diff --git a/.github/workflows/proof.yml b/.github/workflows/proof.yml new file mode 100644 index 00000000..8b119ac2 --- /dev/null +++ b/.github/workflows/proof.yml @@ -0,0 +1,60 @@ +name: Proofing + +on: + push: + branches: [main] + paths: ["crates/proof/**", "Cargo.toml", "Cargo.lock", ".github/workflows/proof.yml"] + pull_request: + paths: ["crates/proof/**", "Cargo.toml", "Cargo.lock", ".github/workflows/proof.yml"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: craft-proof-${{ github.ref }} + cancel-in-progress: true + +jobs: + check: + timeout-minutes: 20 + name: Proofing lint, tests with and without Language data, WASM + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + # The Language data (craft-i18n `language-data/`), embedded by crates/proof/build.rs. + - uses: actions/checkout@v4 + with: + repository: storytold/craft-i18n + ref: 062a6375c322c88b96691a17f7b5fed71c51f7a8 + path: craft-i18n + persist-credentials: false + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + with: + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + key: craft-proof + - run: cargo fmt -p craft-proof --check + - name: Without Language data (a fork, docs.rs, offline) + run: | + cargo clippy -p craft-proof --all-targets --locked -- -D warnings + cargo test -p craft-proof --locked + cargo check -p craft-proof --locked --target wasm32-unknown-unknown + - name: A release without Language data fails + run: | + if CRAFT_I18N_REQUIRED=1 cargo check -p craft-proof --locked 2>build.log; then + echo "::error::CRAFT_I18N_REQUIRED=1 without CRAFT_I18N_DIR built"; exit 1 + fi + grep -q "CRAFT_I18N_REQUIRED=1 but CRAFT_I18N_DIR is unset" build.log + - name: With Language data + env: + CRAFT_I18N_DIR: ${{ github.workspace }}/craft-i18n + CRAFT_I18N_REQUIRED: "1" + run: | + cargo clippy -p craft-proof --all-targets --locked -- -D warnings + cargo test -p craft-proof --locked + cargo check -p craft-proof --locked --target wasm32-unknown-unknown diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 9585c992..30386eac 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -14,6 +14,8 @@ author, source and license. Library code and original assets are under the repos | `crates/cms/tests/data/FOGRA39.txt` (unmodified) | Fogra | [Fogra characterisation data](https://fogra.org/en/downloads/work-tools/characterisation-data), `FOGRA39_to_FOGRA60_v2.zip` | Fogra terms: free use and redistribution, unmodified, with Fogra identified as the source; "FOGRA39" identifies the reference data only and implies no certification, approval or endorsement by Fogra (full text in [NOTICE](NOTICE)) | | `crates/i18n/data/collation.postcard` | Unicode, Inc. (ICU 78.1 collation data, CLDR 48.2.1); generated by `crates/i18n/data/generate-collation.sh` | [ICU](https://github.com/unicode-org/icu/releases/tag/release-78.1rc) | Unicode-3.0 | | `crates/text/assets/hyphen/en-us.dic` | Grady Ward (Moby Hyphenator II), converted by DesignCraft | [Project Gutenberg eBook #3204](https://www.gutenberg.org/ebooks/3204) | Public domain | +| `crates/proof/data/en-words.dic` | Grady Ward (Moby Hyphenator II word list); break points dropped by craft-proof contributors with `crates/proof/examples/en_words.rs` | [Project Gutenberg eBook #3204](https://www.gutenberg.org/ebooks/3204), via craft-i18n `language-data/en-us/en-us.dic` | Public domain | +| `crates/proof/data/en-extra.txt` | WordCraft contributors (original list, no other dictionary copied) | craft-i18n `language-data/en-us/en-extra.txt` | MIT OR Apache-2.0 | | `crates/text/assets/hyphen/en-us.pat` | DesignCraft contributors, trained from the Moby Hyphenator II list | `crates/text/examples/hyphgen.rs` | CC0-1.0 | | Patterns embedded through `hypher` (optional `craft-text` language features) | hyph-utf8 authors, per language | [tex-hyphen](https://github.com/hyphenation/tex-hyphen), as packaged in [hypher](https://github.com/typst/hypher/tree/main/patterns) | Per file header: LPPL, MIT, BSD-3-Clause, Unlicense or a permit-any-use notice (or a choice including one); Slovak (`hyph-sk.tex`, embedded whenever `hypher/slovak` is on: feature `slovak-gpl`, or another dependency through feature unification) is GPL-2.0-or-later; Spanish (`hyph-es.tex` 5.0, Javier Bezos, CervanTeX) is MIT | | Word-break dictionaries and LSTM models embedded through `icu_segmenter_data` (optional `craft-text` features `thai`, `lao`, `khmer`, `burmese`) | Unicode, Inc. and ICU4X contributors | [ICU4X](https://github.com/unicode-org/icu4x) | Unicode-3.0 | diff --git a/Cargo.lock b/Cargo.lock index 06d32761..f19f5306 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1026,6 +1026,16 @@ dependencies = [ "winsafe", ] +[[package]] +name = "craft-proof" +version = "0.1.0" +dependencies = [ + "log", + "miniz_oxide 0.9.1", + "proptest", + "unicode-normalization", +] + [[package]] name = "craft-quit" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index bc32f74d..410091e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ craft-menu = { path = "crates/menu" } craft-pcraft-format = { path = "crates/pcraft-format" } craft-prepress = { path = "crates/prepress" } craft-print = { path = "crates/print" } +craft-proof = { path = "crates/proof" } craft-quit = { path = "crates/quit" } craft-settings = { path = "crates/settings" } craft-text = { path = "crates/text" } diff --git a/NOTICE b/NOTICE index 6fc5f1ea..1c7ac9c9 100644 --- a/NOTICE +++ b/NOTICE @@ -21,6 +21,10 @@ crates/cms/tests/data/FOGRA39.txt: FOGRA39 characterisation data, distributed un imply certification, approval or endorsement by Fogra." crates/text/assets/hyphen/en-us.dic: public domain, derived from the Moby Hyphenator II list by Grady Ward (Project Gutenberg eBook #3204). +crates/proof/data/en-words.dic: public domain, the words of the Moby Hyphenator II list by Grady + Ward (Project Gutenberg eBook #3204), without its break points. +crates/proof/data/en-extra.txt: MIT OR Apache-2.0, WordCraft contributors' own list of modern + English words (a copy of craft-i18n's language-data/en-us/en-extra.txt). crates/text/assets/hyphen/en-us.pat: CC0-1.0, trained from en-us.dic by DesignCraft's trainer. Hyphenation patterns for other languages, embedded through the hypher crate only when the matching craft-text feature is enabled (none are by default). They come from hyph-utf8, diff --git a/README.md b/README.md index 82854e96..740258af 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,9 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = "` embeds `language-data/`. +- Unset (a fork, docs.rs, offline): the crate builds with a warning and no embedded Language + data. English spelling then falls back as below, hyphenation that needs the data is off, and + grammar and the Hunspell languages work as before. +- `CRAFT_I18N_REQUIRED=1` (releases) makes a missing directory or file an error. + +Tests that pin results of the data run only when every file is embedded (`craft_proof_data`); +`src/no_data.rs` runs every public entry point when none is (`craft_proof_no_data`). A partial +data folder runs neither set. + +### English is always checked + +English spelling uses the first of these that is there; `english_source(lang)` says which. + +1. **Embedded:** the Moby word list from `CRAFT_I18N_DIR` (above). Nothing below is used then. +2. **System Hunspell:** the run's own English `.aff`/`.dic` pair (`en_US`, `en_GB` or `en_CA`; + each name in the dictionaries folder and then the system locations before the next name). + Never another English's: an `en_US` dictionary would mark British spellings, so en-GB and + en-CA without their own use the built-in list, which has both spellings. A file that can't be + read is skipped for the next name or folder; one file is read once for every English that uses + it. Compounding (`COMPOUNDMIN`, `COMPOUNDRULE`…, used by English dictionaries only for + ordinals) and `OCONV` are ignored for English rather than refused. None on the web. +3. **Built-in list:** `data/en-words.dic`, the whole Moby word list (about 165,000 words, 344 KB + compressed, words only) built into the crate, with `data/en-extra.txt` (WordCraft's supplement + of modern words, a copy of craft-i18n's) and the same inflection rules as the embedded list. A + word it doesn't know is misspelled. Suggestions are the words one edit away (swap, missing, + added or changed letter), looked up in the list, for words of up to 32 letters: swaps first, + then common words, neighbouring keys, the same first letter. + +A dictionary that is there but can't be read (encoding, size, parse error, no words) is skipped, +and `dictionary_load_error(lang)` gives the reason; it never panics. Text in other languages is +never checked against English: a language without its dictionary has no spelling marks. + +`data/en-words.dic` is public domain: the Moby Hyphenator II list by Grady Ward (Project Gutenberg +eBook #3204), from craft-i18n's `language-data/en-us/en-us.dic` with its break points dropped +and a few entries that are far more often typos left out (`teh`, `abd`, `nad`, `ur`, `u`, `r`; +the list is in the generator). +Regenerate with +`cargo run -p craft-proof --example en_words -- /language-data/en-us/en-us.dic crates/proof/data/en-words.dic`. +`data/en-extra.txt` is WordCraft contributors' own list (MIT OR Apache-2.0); keep it in step with +craft-i18n's copy. + +## Check + +```sh +cargo fmt -p craft-proof --check +cargo clippy -p craft-proof --all-targets -- -D warnings +CRAFT_I18N_DIR=/abs/path/craft-i18n cargo test -p craft-proof +cargo test -p craft-proof # without Language data +cargo check -p craft-proof --target wasm32-unknown-unknown +``` + +CI: `.github/workflows/proof.yml`. + +## Credits + +storytold/craft-libs#30/#31 by brzuchal (Michał Marcin Brzuchalski) started the shared crate. + +## Owners + +WordCraft maintainers: echelon, dylanmtaylor, aiongg. + +## Licence + +MIT OR Apache-2.0. The Language data carries its own licences, recorded in craft-i18n's +`ATTRIBUTION.md`. diff --git a/crates/proof/build.rs b/crates/proof/build.rs new file mode 100644 index 00000000..a0440624 --- /dev/null +++ b/crates/proof/build.rs @@ -0,0 +1,111 @@ +//! Embed the Language data from a craft-i18n checkout (storytold/craft-i18n, `language-data/`). +//! +//! `CRAFT_I18N_DIR`, an absolute path to a craft-i18n checkout: `$OUT_DIR/language_data.rs` holds +//! one constant per file in [`FILES`], each an `include_str!`/`include_bytes!` of +//! `$CRAFT_I18N_DIR/language-data/`. Unset, or not a checkout, or a file missing (a warning): +//! that constant is empty, and the crate has no embedded Language data for it (English spelling +//! falls back to an installed Hunspell dictionary or the crate's own copy of the Moby word list, and +//! hyphenation that needs the data is off). With `CRAFT_I18N_REQUIRED=1` (releases) any of those +//! is an error instead. +//! +//! When every file in [`FILES`] is embedded, the `craft_proof_data` cfg is set (the tests that pin +//! results of the data use it); when none is, `craft_proof_no_data` (the tests of a build without +//! the data). A partial data folder sets neither, so neither set of tests runs against it. +//! +//! `include_str!` takes a string, so a `CRAFT_I18N_DIR` that isn't valid UTF-8 can't be embedded: +//! it is treated like a folder without the data. + +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; + +/// (constant, path under `language-data/`, bytes rather than text). +const FILES: &[(&str, &str, bool)] = &[ + ("EN_US_DIC", "en-us/en-us.dic", true), + ("EN_US_PAT", "en-us/hyphenation.pat", false), + ("EXTRA_EN", "en-us/en-extra.txt", false), + ("EN_VARIANTS", "en-us/en-variants.txt", false), + ("PL_PAT", "pl/hyphenation.pat", false), + ("PT_PAT", "pt/hyphenation.pat", false), + ("IT_PAT", "it/hyphenation.pat", false), + ("DE_PAT", "de/hyphenation.pat", false), + ("EXTRA_RU", "ru/ru-extra.txt", false), +]; + +fn main() { + println!("cargo::rerun-if-changed=build.rs"); + println!("cargo::rerun-if-env-changed=CRAFT_I18N_DIR"); + println!("cargo::rerun-if-env-changed=CRAFT_I18N_REQUIRED"); + println!("cargo::rustc-check-cfg=cfg(craft_proof_data)"); + println!("cargo::rustc-check-cfg=cfg(craft_proof_no_data)"); + let required = std::env::var("CRAFT_I18N_REQUIRED").is_ok_and(|v| v.trim() == "1"); + let level = if required { "error" } else { "warning" }; + let dir = match std::env::var_os("CRAFT_I18N_DIR").filter(|d| !d.is_empty()) { + Some(d) => match language_data(&PathBuf::from(d)) { + Ok(dir) => Some(dir), + Err(e) => { + println!("cargo::{level}=CRAFT_I18N_DIR: {e}; craft-proof embeds no Language data"); + None + } + }, + None => { + if required { + println!("cargo::error=CRAFT_I18N_REQUIRED=1 but CRAFT_I18N_DIR is unset: a release must embed the Language data"); + } else { + println!( + "cargo::warning=CRAFT_I18N_DIR is unset: craft-proof embeds no Language data (English spelling falls back to system Hunspell or the built-in Moby word list; no hyphenation)" + ); + } + None + } + }; + let mut o = String::from("// Generated by craft-proof's build.rs from CRAFT_I18N_DIR/language-data. Do not edit.\n"); + let mut embedded = 0; + for &(name, rel, bytes) in FILES { + let file = dir.as_ref().map(|d| d.join(rel)).filter(|f| f.is_file()); + if let (Some(d), None) = (&dir, &file) { + println!("cargo::{level}=CRAFT_I18N_DIR: {} is missing; craft-proof embeds none", d.join(rel).display()); + } + let (ty, empty, mac) = if bytes { ("&[u8]", "b\"\"", "include_bytes") } else { ("&str", "\"\"", "include_str") }; + // `language_data` only hands back a UTF-8 folder, so this is the path as it is. + let value = match file.as_ref().and_then(|f| f.to_str()) { + Some(f) => { + println!("cargo::rerun-if-changed={f}"); + embedded += 1; + // `{:?}` escapes the path (Windows backslashes) as a Rust string literal. + format!("{mac}!({f:?})") + } + None => empty.to_string(), + }; + let _ = writeln!(o, "/// `language-data/{rel}` (empty when it isn't embedded).\npub(crate) static {name}: {ty} = {value};"); + } + if embedded == FILES.len() { + println!("cargo::rustc-cfg=craft_proof_data"); + } else if embedded == 0 { + println!("cargo::rustc-cfg=craft_proof_no_data"); + } + let out = PathBuf::from(std::env::var_os("OUT_DIR").unwrap_or_default()).join("language_data.rs"); + if let Err(e) = std::fs::write(&out, o) { + println!("cargo::error=writing {}: {e}", out.display()); + } +} + +/// `dir/language-data`, when `dir` is an absolute path to a checkout that has one. +fn language_data(dir: &Path) -> Result { + // Build scripts run in the crate's directory and `include_str!` resolves against the generated + // file, so a relative path would mean two different places. Not canonicalized: on Windows that + // gives `\\?\C:\…`, which `include_str!` can't read. + if !dir.is_absolute() { + return Err(format!("{} is not an absolute path", dir.display())); + } + if dir.to_str().is_none() { + return Err(format!("{} is not valid UTF-8, which include_str! needs", dir.display())); + } + let data = dir.join("language-data"); + if !data.is_dir() { + // Notice when the checkout gains its Language data. + println!("cargo::rerun-if-changed={}", dir.display()); + return Err(format!("{} has no language-data/ (a craft-i18n checkout older than its Language data?)", dir.display())); + } + println!("cargo::rerun-if-changed={}", data.display()); + Ok(data) +} diff --git a/crates/proof/data/en-extra.txt b/crates/proof/data/en-extra.txt new file mode 100644 index 00000000..7d04908e --- /dev/null +++ b/crates/proof/data/en-extra.txt @@ -0,0 +1,241 @@ +# English spelling supplement: common words the public-domain Moby list (1990s) lacks. +# Written by WordCraft contributors (MIT OR Apache-2.0); see ATTRIBUTION.md. +# One lowercase base form per line, sorted. Plurals, -ed, -ing, -er, -ly… come from the +# inflection rules in crates/proof/src/lib.rs, so list "desktop", not "desktops". +airbag +airfare +artcraft +autocomplete +autocorrect +autofill +autoformat +autosave +backend +bitcoin +bitmap +blockchain +blog +blogger +bluetooth +captcha +caregiver +carpool +cellphone +chatbot +chatroom +checkbox +checklist +childcare +clamour +clickable +colourful +colourisation +colourise +colourless +configurable +copyedit +coronavirus +covid +crowdfunding +crowdsource +crypto +cryptocurrency +customizable +customization +cyber +cybersecurity +cyberspace +database +dataset +daycare +desktop +download +downloadable +downsize +dropdown +earbud +ebook +ecommerce +email +emoji +emoticon +endeavour +esports +facebook +favicon +fintech +firewall +flowchart +foodie +freeware +frontend +fundraiser +gameplay +gamification +geeky +geolocation +gigabit +gigabyte +github +glamour +globalization +google +groupthink +hackathon +hardcode +hashtag +healthcare +homepage +homeschool +honour +honourable +honourably +hotspot +hyperlink +hypertext +inbox +incentivize +infographic +instagram +internet +interoperability +intranet +ipad +iphone +javascript +kickstart +kilobyte +laptop +lifestyle +linkedin +linux +livestream +lockdown +login +logout +lowercase +makeover +malware +manoeuvrable +megabyte +megapixel +meme +metadata +microchip +microsoft +middleware +mindset +multicultural +multiplayer +multitask +nerdy +netbook +netflix +newsfeed +offboard +offline +onboard +online +onscreen +outsource +paediatric +paralegal +passcode +petabyte +phishing +photobomb +playdate +playlist +plugin +podcast +popup +prequel +printout +prioritise +prioritization +prioritize +privatise +ransomware +recyclable +rigour +ringtone +roadmap +rollout +runtime +scalability +screencast +screensaver +screenshot +scrollbar +selfie +signup +sitemap +skillset +smartphone +smartwatch +smiley +snowboard +spellcheck +spellchecker +spreadsheet +spyware +stakeholder +standup +startup +subfolder +sunscreen +superfood +superuser +synthesise +tagline +takeout +techie +telecommute +teleconference +telehealth +telemedicine +terabyte +textbox +timeline +timestamp +toolbar +toolkit +touchpad +touchscreen +trackball +trackpad +travelogue +upcycle +upload +uppercase +upsell +upskill +usability +username +videoconference +videogame +vlog +vlogger +voicemail +walkthrough +webcam +webcast +webinar +weblog +webmaster +webpage +website +whatsapp +whiteboard +wifi +wiki +wikipedia +wireframe +wordcraft +workaround +workflow +workforce +workplace +workspace +workstation +youtube diff --git a/crates/proof/data/en-words.dic b/crates/proof/data/en-words.dic new file mode 100644 index 00000000..c151049b Binary files /dev/null and b/crates/proof/data/en-words.dic differ diff --git a/crates/proof/examples/en_words.rs b/crates/proof/examples/en_words.rs new file mode 100644 index 00000000..45a5f9dd --- /dev/null +++ b/crates/proof/examples/en_words.rs @@ -0,0 +1,33 @@ +//! Regenerate `data/en-words.dic`, the English word list built into the crate (see the crate +//! README): every word of the public-domain Moby Hyphenator II list, without its break points. +//! +//! ```sh +//! cargo run -p craft-proof --example en_words -- /language-data/en-us/en-us.dic data/en-words.dic +//! ``` +//! +//! The input is craft-i18n's `en-us.dic`: the Moby Hyphenator II list by Grady Ward (Project +//! Gutenberg eBook #3204, `mhyph.txt`), already stripped of the Project Gutenberg header and +//! licence, with its break points. The output keeps the words only (in [`Dictionary`]'s format, +//! every break mask zero), so it is smaller and carries no hyphenation, and leaves out +//! [`EXCLUDED`]. No other word list or dictionary is read. + +use craft_proof::dict::Dictionary; + +/// Entries of the Moby list that are far more often typing errors than words (`teh` for `the`, +/// `abd` and `nad` for `and`, `ur` for `your`, the letters `u` and `r` for `you` and `are`), so the +/// built-in list marks them. `wont` and `cant` stay: they are words. +const EXCLUDED: &[&str] = &["abd", "nad", "r", "teh", "u", "ur"]; + +fn main() -> Result<(), String> { + let args: Vec = std::env::args().skip(1).collect(); + let [dic, out] = args.as_slice() else { + return Err("usage: en_words ".into()); + }; + let bytes = std::fs::read(dic).map_err(|e| format!("{dic}: {e}"))?; + let moby = Dictionary::from_bytes(&bytes)?; + let words = Dictionary::from_entries(moby.iter().filter(|(w, _)| !EXCLUDED.contains(w)).map(|(w, _)| (w.to_string(), 0)).collect()); + let data = words.to_bytes(); + std::fs::write(out, &data).map_err(|e| format!("{out}: {e}"))?; + eprintln!("{} words, {} bytes", words.len(), data.len()); + Ok(()) +} diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs new file mode 100644 index 00000000..701e168c --- /dev/null +++ b/crates/proof/src/affix.rs @@ -0,0 +1,1847 @@ +//! Hunspell-style affix dictionaries — the subset the SJP.PL Polish dictionary (`pl_PL`) uses. +//! +//! A word is correct when it is a stem, a stem with one suffix, a stem with one prefix, or a stem +//! with a prefix and a suffix that both allow cross products; each rule applies only to stems +//! carrying its flag and ending (suffixes) or starting (prefixes) as its condition says. Highly +//! inflected languages store millions of forms this way in a few hundred thousand stems: the +//! Polish dictionary's ~350k stems and ~7,500 rules accept ~4.5 million forms. +//! +//! The rules and stems come from a Hunspell `.aff`/`.dic` pair ([`AffixDict::from_hunspell`]), +//! read at runtime from the dictionaries installed on the system ([`crate::hunspell`]). They can +//! also be stored in WordCraft's compact format ([`AffixDict::to_bytes`]): the rules as text +//! lines and the stems front-coded, raw-deflated together (the Polish dictionary is under 1 MB). +//! +//! Supported: one-character flags (at most 256 distinct, any Unicode character — `FLAG UTF-8`), +//! flag aliases (`AF`: a number standing for a set of flags), `PFX`/`SFX` with strip, append and +//! condition (`.`, letters, `[abc]`, `[^abc]`), cross products, continuation classes on suffixes +//! (at most two suffixes in a row, as in Hunspell), `FORBIDDENWORD`, `NOSUGGEST`, `KEEPCASE` (the +//! stem is a word only in its own letter case), input conversion (`ICONV`: ligatures such as `fi` +//! read as `fi`) and ignored characters (`IGNORE`), and the suggestion tables `TRY`, `KEY`, `MAP` +//! and `REP`. +//! Morphological data (`AM` aliases, the fields after a stem) is skipped. Rejected: compounding, +//! long or numeric flags, `NEEDAFFIX` and the other directives that change which words are correct +//! (an English dictionary's compounding and `OCONV` are ignored instead: +//! [`AffixDict::from_hunspell_english`]). +//! +//! Lookups never panic: stems sit in one string, found through an open-addressing hash table, and +//! a probe allocates only small scratch strings for the candidate roots it tries. + +use std::collections::HashMap; +use std::hash::Hasher; + +use crate::patterns::{FxHasher, FxMap}; + +const MAGIC: &str = "WCAFF1"; +/// Longest word (in bytes) looked up; longer input is never a dictionary word. +pub const MAX_WORD_BYTES: usize = 120; +/// Empty slot in the stem hash table. +const EMPTY: u32 = u32::MAX; + +/// Directives that change which words are correct in ways this engine doesn't implement. +const UNSUPPORTED: &[&str] = &[ + "CHECKSHARPS", + "CIRCUMFIX", + "COMPLEXPREFIXES", + "COMPOUNDBEGIN", + "COMPOUNDEND", + "COMPOUNDFIRST", + "COMPOUNDFLAG", + "COMPOUNDLAST", + "COMPOUNDMIDDLE", + "COMPOUNDMIN", + "COMPOUNDPERMITFLAG", + "COMPOUNDRULE", + "FULLSTRIP", + "NEEDAFFIX", + "OCONV", + "ONLYINCOMPOUND", + "PSEUDOROOT", +]; + +/// Most distinct flags a dictionary may use: every single-byte flag (the Hungarian dictionary +/// uses 158). +const MAX_FLAGS: usize = 256; + +/// A set of flags, one bit per distinct flag character of the dictionary. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +struct Mask([u64; MAX_FLAGS / 64]); + +impl Mask { + const EMPTY: Mask = Mask([0; MAX_FLAGS / 64]); + + /// The set holding flag number `i` alone (empty when `i` is out of range). + fn bit(i: usize) -> Mask { + let mut m = Mask::EMPTY; + if let Some(w) = m.0.get_mut(i / 64) { + *w = 1 << (i % 64); + } + m + } + + /// The number of the lowest flag in the set, if any. + fn first(self) -> Option { + self.0.iter().enumerate().find(|(_, w)| **w != 0).map(|(i, w)| i * 64 + w.trailing_zeros() as usize) + } + + fn is_empty(self) -> bool { + self.0.iter().all(|&w| w == 0) + } + + /// Do the two sets share a flag? + fn meets(self, other: Mask) -> bool { + !(self & other).is_empty() + } + + /// Does this set hold every flag of `other`? + fn covers(self, other: Mask) -> bool { + (self & other) == other + } +} + +impl std::ops::BitAnd for Mask { + type Output = Mask; + fn bitand(mut self, o: Mask) -> Mask { + for (a, b) in self.0.iter_mut().zip(o.0) { + *a &= b; + } + self + } +} + +impl std::ops::BitOr for Mask { + type Output = Mask; + fn bitor(mut self, o: Mask) -> Mask { + self |= o; + self + } +} + +impl std::ops::BitOrAssign for Mask { + fn bitor_assign(&mut self, o: Mask) { + for (a, b) in self.0.iter_mut().zip(o.0) { + *a |= b; + } + } +} + +/// One character class of a rule's condition. +#[derive(Clone, Debug, PartialEq, Eq)] +enum Cond { + Any, + Char(char), + Set { negated: bool, chars: Box<[char]> }, +} + +impl Cond { + fn matches(&self, c: char) -> bool { + match self { + Cond::Any => true, + Cond::Char(x) => *x == c, + Cond::Set { negated, chars } => chars.contains(&c) != *negated, + } + } +} + +/// Parse a Hunspell condition (`.`, `[^i]e`, `[km]anie`…). +fn parse_cond(text: &str) -> Result, String> { + if text == "." { + return Ok(Box::new([])); + } + let mut out = Vec::new(); + let mut chars = text.chars(); + while let Some(c) = chars.next() { + match c { + '.' => out.push(Cond::Any), + '[' => { + let mut set = Vec::new(); + let mut negated = false; + let mut closed = false; + let mut first = true; + for d in chars.by_ref() { + if d == ']' { + closed = true; + break; + } + if first && d == '^' { + negated = true; + } else { + set.push(d); + } + first = false; + } + if !closed { + return Err(format!("unclosed `[` in condition `{text}`")); + } + out.push(Cond::Set { negated, chars: set.into_boxed_slice() }); + } + ']' => return Err(format!("stray `]` in condition `{text}`")), + c => out.push(Cond::Char(c)), + } + } + Ok(out.into_boxed_slice()) +} + +/// How many suffixes a word may carry in a row through continuation classes (`work` + +/// `able/B` + `s` → `workables` is two). Hunspell strips at most two suffixes, so a third +/// chained suffix is not a word; the bound also keeps hostile input from walking deep recursions. +const MAX_CHAIN: u8 = 2; +/// How many suffixes may sit under the outermost one: the depth `form_has_flag` walks. +const INNER_CHAIN: u8 = MAX_CHAIN - 1; +/// Most forms [`AffixDict::forms`] lists for one stem (real stems have a few hundred at most). +pub const MAX_FORMS: usize = 4096; +/// Most rules [`AffixDict::forms`] tries along continuation chains and prefix–suffix cross +/// products for one stem: real dictionaries need a few thousand, hostile ones are cut off. +const MAX_FORMS_WORK: usize = 1 << 18; + +/// One prefix or suffix rule. +#[derive(Clone, Debug)] +struct Rule { + /// Bit of the rule's flag in the stems' flag masks. + bit: Mask, + flag: char, + cross: bool, + strip: Box, + add: Box, + /// Continuation classes (`SFX A 0 able/B`): flags the derived form carries, licensing + /// further affixes. Chained application is bounded by [`MAX_CHAIN`]. + cont: Mask, + cond: Box<[Cond]>, + cond_text: Box, +} + +impl Rule { + /// Does `root` end (suffix rule) or start (prefix rule) as the condition says? + fn cond_matches(&self, root: &str, suffix: bool) -> bool { + if self.cond.is_empty() { + return true; + } + if suffix { + let mut it = root.chars().rev(); + self.cond.iter().rev().all(|c| it.next().is_some_and(|x| c.matches(x))) + } else { + let mut it = root.chars(); + self.cond.iter().all(|c| it.next().is_some_and(|x| c.matches(x))) + } + } +} + +/// Suffix rules sharing an appended string and a stripped one: one stem lookup serves them all. +#[derive(Clone, Debug, Default)] +struct SfxGroup { + strip: Box, + rules: Vec, +} + +/// A stem list with affix rules (see the module docs). +#[derive(Clone, Debug, Default)] +pub struct AffixDict { + /// All stems, concatenated, sorted by bytes. + words: String, + offs: Vec, + /// Per stem: index into `sets`. + flags: Vec, + /// Distinct flag masks. + sets: Vec, + /// The `FORBIDDENWORD` flag's bit; stems carrying it are not words (0: none declared). + forbidden: Mask, + /// Stems with NOSUGGEST are correct, but none of their forms are suggested. + no_suggest: Mask, + /// Stems with SUBSTANDARD are correct too, and likewise never suggested. + substandard: Mask, + /// Open-addressing hash table of stem indices (`EMPTY` = free). + table: Vec, + /// Flag char of each bit. + flag_chars: Vec, + prefixes: Vec, + suffixes: Vec, + /// Appended string → groups of suffix rules by stripped string. + sfx_index: FxMap, Vec>, + /// Per flag bit: the suffix rules of that flag, in rule order. + sfx_by_flag: Vec>, + /// Longest appended suffix, in chars. + max_sfx_chars: usize, + try_chars: String, + key: String, + map: Vec>, + rep: Vec<(String, String)>, + /// The `KEEPCASE` flag's bit: such stems are words only in their own letter case (0: none). + keepcase: Mask, + /// `ICONV`: input conversions applied before a lookup, `(from, to)`. + iconv: Vec<(String, String)>, + /// `IGNORE`: characters removed from input words (and from the stems and appended texts). + ignore: Box<[char]>, + /// Provenance and licence lines, kept through [`AffixDict::to_bytes`]. + comments: Vec, +} + +fn hash_str(s: &str) -> u64 { + let mut h = FxHasher::default(); + h.write(s.as_bytes()); + h.finish() +} + +/// Letter case of a word. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Casing { + /// No capitals (`kot`). + Lower, + /// First letter capital, no other (`Kot`). + Initial, + /// Every cased letter a capital, at least two (`KOT`). + Upper, + /// Anything else (`McDonald`, `kOT`). + Mixed, +} + +/// The casing of `w`. +pub fn casing(w: &str) -> Casing { + let (mut upper, mut lower, mut first_upper) = (0usize, 0usize, false); + for (i, c) in w.chars().enumerate() { + if c.is_uppercase() { + upper += 1; + if i == 0 { + first_upper = true; + } + } else if c.is_lowercase() { + lower += 1; + } + } + if upper == 0 { + Casing::Lower + } else if upper == 1 && first_upper { + Casing::Initial + } else if lower == 0 { + Casing::Upper + } else { + Casing::Mixed + } +} + +/// `w` with its first letter a capital. +pub fn capitalize(w: &str) -> String { + let mut c = w.chars(); + match c.next() { + Some(f) => f.to_uppercase().chain(c).collect(), + None => String::new(), + } +} + +/// Parsed rules and tables, shared by the Hunspell reader and the compact format. +#[derive(Default)] +struct Rules { + prefixes: Vec, + suffixes: Vec, + try_chars: String, + key: String, + map: Vec>, + rep: Vec<(String, String)>, + /// The `FORBIDDENWORD` flag: stems carrying it are listed but never correct. + forbidden: Option, + no_suggest: Option, + substandard: Option, + /// The `KEEPCASE` flag. + keepcase: Option, + /// The `ONLYINCOMPOUND` flag, read for English only ([`AffixDict::from_hunspell_english`]). + only_in_compound: Option, + iconv: Vec<(String, String)>, + ignore: Vec, + /// `AF` flag aliases: alias `n` (1-based) stands for `aliases[n - 1]`. + aliases: Vec, + /// The `AF` table opened with its count line, so flag fields are alias numbers. A table + /// without one is rejected, as in Hunspell: its lines are ignored. + aliases_on: bool, +} + +impl Rules { + /// Flags as written in a `.dic` entry or a continuation class: the flag characters + /// themselves, or with an `AF` table the alias its leading digits number, as Hunspell reads + /// them (`2x` is alias 2). A field without digits or an undefined alias reads as no flags: + /// the stem is still a word, the rule still applies, without the flags (Hunspell warns). + fn flags<'a>(&'a self, written: &'a str) -> &'a str { + if !self.aliases_on { + return written; + } + let digits = written.bytes().take_while(u8::is_ascii_digit).count(); + written + .get(..digits) + .and_then(|d| d.parse::().ok()) + .and_then(|n| n.checked_sub(1)) + .and_then(|i| self.aliases.get(i)) + .map_or("", String::as_str) + } + + /// `w` without the `IGNORE` characters. + fn strip_ignored(&self, w: &str) -> String { + if self.ignore.is_empty() { w.to_string() } else { w.chars().filter(|c| !self.ignore.contains(c)).collect() } + } +} + +/// The fields of an `.aff`/`.dic` line: Hunspell separates them with spaces and tabs only, so a +/// flag byte read as U+0085 or U+00A0 (see [`crate::hunspell::decode`]) stays a flag. +fn split_fields(line: &str) -> impl Iterator { + line.split([' ', '\t']).filter(|f| !f.is_empty()) +} + +/// The one flag character a directive (`FORBIDDENWORD w`) names. +fn one_flag(arg: &str, dir: &str, n: usize) -> Result { + let mut it = arg.chars().filter(|&c| c != EMOJI_SELECTOR); + match (it.next(), it.next()) { + // Only the field separators of `split_fields` can't be a flag: a flag byte read as + // U+0085 or U+00A0 is one. + (Some(c), None) if !matches!(c, ' ' | '\t') => Ok(c), + _ => Err(format!("line {}: {dir} needs one flag character", n + 1)), + } +} + +/// A rule as parsed from text, before flags become bits: `(flag, cross, strip, add, cont, cond)`. +type RawRule = (char, bool, String, String, String, String); + +/// Most distinct `TRY` characters kept; a hostile `.aff` could otherwise make every suggestion +/// round cost time and memory proportional to the file size. +const MAX_TRY_CHARS: usize = 128; + +/// Keeps the first [`MAX_TRY_CHARS`] distinct characters of a `TRY` value, in order. +fn cap_try(s: &str) -> String { + let mut out = String::new(); + let mut n = 0; + for c in s.chars() { + if n >= MAX_TRY_CHARS { + break; + } + if !out.contains(c) { + out.push(c); + n += 1; + } + } + out +} + +/// `0` means empty in Hunspell's strip and append fields. +fn zero_empty(s: &str) -> String { + if s == "0" { String::new() } else { s.to_string() } +} + +/// U+FE0F VARIATION SELECTOR-16 asks for the emoji form of the character before it. A dictionary +/// that uses an emoji as a flag may write it after the flag (RLA-ES Spanish: `PFX ☎️`, `banco/S☎️`), +/// but it is not a flag of its own: it is dropped wherever flags are read. +const EMOJI_SELECTOR: char = '\u{fe0f}'; + +/// Directives an English dictionary may carry that don't change which English words are correct +/// here, ignored by [`AffixDict::from_hunspell_english`]: compounding (English dictionaries use +/// it only for ordinals such as `1st`, which have digits and are skipped by the spelling check) +/// and output conversion (`OCONV`: how suggestions are written, such as `'` as `’`). +const IGNORED_IN_ENGLISH: &[&str] = &[ + "COMPOUNDBEGIN", + "COMPOUNDEND", + "COMPOUNDFIRST", + "COMPOUNDFLAG", + "COMPOUNDLAST", + "COMPOUNDMIDDLE", + "COMPOUNDMIN", + "COMPOUNDPERMITFLAG", + "COMPOUNDRULE", + "OCONV", +]; + +/// Read Hunspell `.aff` text (already decoded to UTF-8). `english`: ignore the directives in +/// [`IGNORED_IN_ENGLISH`], and keep the `ONLYINCOMPOUND` flag so the stems that are only parts of +/// compounds can be left out. +fn parse_aff(aff: &str, english: bool) -> Result { + let mut r = Rules::default(); + let mut cross: HashMap<(bool, char), bool> = HashMap::new(); + let mut map_header = false; + let mut rep_header = false; + let mut af_header = false; + let mut iconv_header = false; + for (n, line) in aff.lines().enumerate() { + let f: Vec<&str> = split_fields(line).collect(); + let Some(&dir) = f.first() else { continue }; + if dir.starts_with('#') { + continue; + } + let at = |k: usize| f.get(k).copied().unwrap_or(""); + match dir { + "SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS" + | "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {} + // Morphological aliases, and a flag that only matters for morphological generation: + // the words it marks are correct. + "AM" | "LEMMA_PRESENT" => {} + "SUBSTANDARD" => r.substandard = Some(one_flag(at(1), dir, n)?), + // The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a + // trailing comment (`AF 1559 # aliases`). + "AF" => { + if !af_header { + af_header = true; + // A count below 1 rejects the table too: flags stay literal (`AF 0`). + r.aliases_on = at(1).parse::().is_ok_and(|n| n > 0); + } else if r.aliases_on { + r.aliases.push(at(1).to_string()); + } + } + "ICONV" => { + if !iconv_header && at(1).parse::().is_ok() { + iconv_header = true; + } else if f.len() >= 3 { + r.iconv.push((at(1).to_string(), at(2).to_string())); + } + } + "IGNORE" => r.ignore = at(1).chars().collect(), + "KEEPCASE" => r.keepcase = Some(one_flag(at(1), dir, n)?), + // Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work + // with the one-character parsing here; only the multi-character encodings differ. + "FLAG" if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") => { + return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1))); + } + "FLAG" => {} + "FORBIDDENWORD" => r.forbidden = Some(one_flag(at(1), dir, n)?), + "NOSUGGEST" => r.no_suggest = Some(one_flag(at(1), dir, n)?), + "TRY" => r.try_chars = cap_try(at(1)), + "KEY" => r.key = at(1).to_string(), + "MAP" => { + if !map_header && at(1).parse::().is_ok() { + map_header = true; + } else if !at(1).contains('(') { + r.map.push(at(1).chars().collect()); + } + } + "REP" => { + if !rep_header && at(1).parse::().is_ok() { + rep_header = true; + } else if f.len() >= 3 { + r.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))); + } + } + "PFX" | "SFX" => { + let suffix = dir == "SFX"; + let mut flag_chars = at(1).chars().filter(|&c| c != EMOJI_SELECTOR); + let (Some(flag), None) = (flag_chars.next(), flag_chars.next()) else { + return Err(format!("line {}: flag `{}` is not one character", n + 1, at(1))); + }; + // Only a header ignores trailing fields; a rule condition beginning with # is literal. + let is_header = f.len() >= 4 && matches!(at(2), "Y" | "N") && at(3).parse::().is_ok() && !cross.contains_key(&(suffix, flag)); + if is_header { + cross.insert((suffix, flag), at(2) == "Y"); + continue; + } + if f.len() < 4 { + return Err(format!("line {}: short affix rule", n + 1)); + } + // `able/B`: the appended text and its continuation classes. + let (add, cont) = match at(3).split_once('/') { + Some((a, c)) => (a, c), + None => (at(3), ""), + }; + let cond = if f.len() >= 5 { at(4) } else { "." }; + parse_cond(cond).map_err(|e| format!("line {}: {e}", n + 1))?; + let c = cross.get(&(suffix, flag)).copied().unwrap_or(false); + let rule: RawRule = (flag, c, zero_empty(at(2)), zero_empty(add), cont.to_string(), cond.to_string()); + if suffix { + r.suffixes.push(rule); + } else { + r.prefixes.push(rule); + } + } + "ONLYINCOMPOUND" if english => r.only_in_compound = Some(one_flag(at(1), dir, n)?), + d if english && IGNORED_IN_ENGLISH.contains(&d) => {} + d if UNSUPPORTED.contains(&d) => return Err(format!("line {}: {d} is not supported", n + 1)), + _ => {} + } + } + // Aliases may be declared after the rules that use them; resolve once all are read. + // Hunspell removes `IGNORE` characters from the appended text only: a stripped text that + // holds one never matches a stem (stems lose them too), so the rule never applies. + let resolve = |rules: &mut Vec, r: &Rules| { + for rule in rules.iter_mut() { + rule.4 = r.flags(&rule.4).to_string(); + rule.3 = r.strip_ignored(&rule.3); + } + }; + let (mut pfx, mut sfx) = (std::mem::take(&mut r.prefixes), std::mem::take(&mut r.suffixes)); + resolve(&mut pfx, &r); + resolve(&mut sfx, &r); + (r.prefixes, r.suffixes) = (pfx, sfx); + Ok(r) +} + +impl AffixDict { + /// Build from Hunspell `.aff` and `.dic` text (decoded to UTF-8; the `.dic` count line and + /// morphological fields are ignored). `comments` (provenance, licence) are kept with the data. + pub fn from_hunspell(aff: &str, dic: &str, comments: &[&str]) -> Result { + AffixDict::from_hunspell_in(aff, dic, comments, false) + } + + /// [`AffixDict::from_hunspell`] for an English dictionary: compounding and output conversion + /// (`COMPOUNDMIN`, `COMPOUNDRULE`, `OCONV`…), which other languages need but English spelling + /// doesn't, are ignored rather than refused, and the stems marked `ONLYINCOMPOUND` (parts of + /// compounds only) are left out. + pub fn from_hunspell_english(aff: &str, dic: &str, comments: &[&str]) -> Result { + AffixDict::from_hunspell_in(aff, dic, comments, true) + } + + fn from_hunspell_in(aff: &str, dic: &str, comments: &[&str], english: bool) -> Result { + // Some dictionaries open with a UTF-8 BOM; it must not glue itself to the first + // directive or turn the `.dic` count line into a stem. + let aff = aff.strip_prefix('\u{feff}').unwrap_or(aff); + let dic = dic.strip_prefix('\u{feff}').unwrap_or(dic); + let rules = parse_aff(aff, english)?; + let mut stems: Vec<(String, String)> = Vec::new(); + for (n, line) in dic.lines().enumerate() { + // The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it. + // Leading blanks belong to it, as in Hunspell (` lab` is no `lab`). + let tok = line.get(..crate::hunspell::dic_entry_end(line.as_bytes())).unwrap_or_default(); + if tok.is_empty() { + continue; + } + if n == 0 && tok.parse::().is_ok() { + continue; + } + // `\/` is a literal slash in a word; the first plain `/` starts the flags. + let mut word = String::new(); + let mut flags = String::new(); + let mut chars = tok.chars().peekable(); + let mut in_flags = false; + while let Some(c) = chars.next() { + if in_flags { + if c != EMOJI_SELECTOR { + flags.push(c); + } + } else if c == '\\' && chars.peek() == Some(&'/') { + word.push('/'); + chars.next(); + } else if c == '/' { + in_flags = true; + } else { + word.push(c); + } + } + let word = rules.strip_ignored(&word); + if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) { + continue; + } + let flags = rules.flags(&flags).to_string(); + if rules.only_in_compound.is_some_and(|f| flags.contains(f)) { + continue; + } + stems.push((word, flags)); + } + AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect()) + } + + fn build(rules: Rules, mut stems: Vec<(String, String)>, comments: Vec) -> Result { + // Flags: every one a rule or a stem uses gets a bit. + let mut flag_chars: Vec = rules.prefixes.iter().chain(&rules.suffixes).map(|r| r.0).collect(); + flag_chars.extend(stems.iter().flat_map(|s| s.1.chars())); + flag_chars.sort_unstable(); + flag_chars.dedup(); + if flag_chars.len() > MAX_FLAGS { + return Err(format!("{} distinct flags; at most {MAX_FLAGS} are supported", flag_chars.len())); + } + let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(Mask::EMPTY, Mask::bit); + let forbidden = rules.forbidden.map_or(Mask::EMPTY, bit_of); + let keepcase = rules.keepcase.map_or(Mask::EMPTY, bit_of); + // Stems sorted by bytes; homonyms (the same word twice) are merged. + stems.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes())); + let mut d = AffixDict { + words: String::with_capacity(stems.iter().map(|s| s.0.len()).sum()), + offs: Vec::with_capacity(stems.len() + 1), + flag_chars: flag_chars.clone(), + forbidden, + no_suggest: rules.no_suggest.map_or(Mask::EMPTY, bit_of), + substandard: rules.substandard.map_or(Mask::EMPTY, bit_of), + keepcase, + iconv: rules.iconv, + ignore: rules.ignore.into_boxed_slice(), + try_chars: rules.try_chars, + key: rules.key, + map: rules.map, + rep: rules.rep, + comments, + ..AffixDict::default() + }; + d.offs.push(0); + let mut set_ids: HashMap = HashMap::new(); + let mut i = 0; + while i < stems.len() { + let Some((word, _)) = stems.get(i) else { break }; + let mut mask = Mask::EMPTY; + let mut j = i; + while let Some((w, f)) = stems.get(j) + && w == word + { + mask |= f.chars().map(bit_of).fold(Mask::EMPTY, |a, b| a | b); + j += 1; + } + let next_id = u16::try_from(d.sets.len()).map_err(|_| "too many distinct flag sets".to_string())?; + let id = *set_ids.entry(mask).or_insert(next_id); + if usize::from(id) == d.sets.len() { + d.sets.push(mask); + } + d.words.push_str(word); + d.offs.push(u32::try_from(d.words.len()).map_err(|_| "stems too large".to_string())?); + d.flags.push(id); + i = j; + } + let mk = |(flag, cross, strip, add, cont, cond): RawRule| -> Result { + let cont = cont.chars().fold(Mask::EMPTY, |m, c| m | bit_of(c)); + Ok(Rule { bit: bit_of(flag), flag, cross, strip: strip.into(), add: add.into(), cont, cond: parse_cond(&cond)?, cond_text: cond.into() }) + }; + d.prefixes = rules.prefixes.into_iter().map(mk).collect::>()?; + d.suffixes = rules.suffixes.into_iter().map(mk).collect::>()?; + d.index(); + Ok(d) + } + + /// Build the suffix index and the stem hash table. + fn index(&mut self) { + let mut idx: FxMap, Vec> = FxMap::default(); + // Each (appended, stripped) pair's group, found by hash rather than by scanning the + // appended string's groups: a dictionary with many strips per suffix stays linear. + let mut group_of: FxMap<(&str, &str), usize> = FxMap::default(); + let mut by_flag: Vec> = vec![Vec::new(); MAX_FLAGS]; + for (i, r) in self.suffixes.iter().enumerate() { + let groups = idx.entry(r.add.clone()).or_default(); + let Ok(i) = u32::try_from(i) else { break }; + match group_of.get(&(&*r.add, &*r.strip)).and_then(|&g| groups.get_mut(g)) { + Some(g) => g.rules.push(i), + None => { + group_of.insert((&r.add, &r.strip), groups.len()); + groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }); + } + } + if let Some(list) = r.bit.first().and_then(|b| by_flag.get_mut(b)) { + list.push(i); + } + } + self.sfx_by_flag = by_flag; + self.max_sfx_chars = self.suffixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0); + self.sfx_index = idx; + let n = self.len(); + let size = (n.max(1) * 2).next_power_of_two(); + let mut table = vec![EMPTY; size]; + let mask = size - 1; + for i in 0..n { + let Some(w) = self.word(i) else { continue }; + let mut slot = hash_str(w) as usize & mask; + while table.get(slot).is_some_and(|&t| t != EMPTY) { + slot = (slot + 1) & mask; + } + if let (Some(t), Ok(i)) = (table.get_mut(slot), u32::try_from(i)) { + *t = i; + } + } + self.table = table; + } + + /// Number of stems. + pub fn len(&self) -> usize { + self.flags.len() + } + + pub fn is_empty(&self) -> bool { + self.flags.is_empty() + } + + /// Number of prefix and suffix rules. + pub fn rule_count(&self) -> usize { + self.prefixes.len() + self.suffixes.len() + } + + fn word(&self, i: usize) -> Option<&str> { + let a = *self.offs.get(i)? as usize; + let b = *self.offs.get(i + 1)? as usize; + self.words.get(a..b) + } + + /// The flag characters of `mask`, in flag order (how the compact format stores flags). + fn flag_chars_for(&self, mask: Mask) -> String { + self.flag_chars.iter().enumerate().filter(|&(i, _)| mask.meets(Mask::bit(i))).map(|(_, c)| c).collect() + } + + /// The character of `mask` when it holds exactly one flag. + fn flag_char(&self, mask: Mask) -> Option { + let s = self.flag_chars_for(mask); + let mut it = s.chars(); + match (it.next(), it.next()) { + (Some(c), None) => Some(c), + _ => None, + } + } + + /// The stems, in byte order. + pub fn stems(&self) -> impl Iterator + '_ { + (0..self.len()).filter_map(|i| self.word(i)) + } + + /// The stems starting with `prefix`, in byte order (a binary search, then a scan). + pub fn stems_with_prefix<'a>(&'a self, prefix: &str) -> impl Iterator + use<'a> { + let prefix = prefix.to_string(); + let first = { + let (mut lo, mut hi) = (0usize, self.len()); + while lo < hi { + let mid = lo + (hi - lo) / 2; + if self.word(mid).is_some_and(|w| w.as_bytes() < prefix.as_bytes()) { + lo = mid + 1; + } else { + hi = mid; + } + } + lo + }; + (first..self.len()).map_while(move |i| self.word(i).filter(|w| w.starts_with(prefix.as_str()))) + } + + /// Flag mask of a stem, if `w` is one. + fn stem_flags(&self, w: &str) -> Option { + self.stem_flags_except(w, Mask::EMPTY) + } + + /// Flag mask of a stem, if `w` is one and carries none of the flags in `deny`. + fn stem_flags_except(&self, w: &str, deny: Mask) -> Option { + if self.table.is_empty() { + return None; + } + let mask = self.table.len() - 1; + let mut slot = hash_str(w) as usize & mask; + for _ in 0..self.table.len() { + let t = *self.table.get(slot)?; + if t == EMPTY { + return None; + } + if self.word(t as usize) == Some(w) { + let id = *self.flags.get(t as usize)?; + let mask = self.sets.get(usize::from(id)).copied(); + // A stem carrying FORBIDDENWORD is listed but is not a word: no direct hit, + // and (this is the only stem lookup) no affixed form is built on it either. + return mask.filter(|m| !m.meets(self.forbidden | deny)); + } + slot = (slot + 1) & mask; + } + None + } + + /// Does `w` carry any flag of `flags` — as a stem, or as the form of a suffix rule whose + /// continuation classes include one (chained suffixation, depth-bounded)? + /// + /// The rules of a group that could lead to `w` are merged into one mask of their flags and + /// followed once, not once per rule: the answer is the same, but thousands of rules of one + /// flag cost one walk, not one each (#3613). + fn form_has_flag(&self, w: &str, flags: Mask, depth: u8, deny: Mask) -> bool { + if self.stem_flags_except(w, deny).is_some_and(|f| f.meets(flags)) { + return true; + } + if depth == 0 { + return false; + } + let mut root = String::with_capacity(w.len() + 8); + let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0)); + for b in cuts.take(self.max_sfx_chars + 1) { + let (Some(base), Some(add)) = (w.get(..b), w.get(b..)) else { continue }; + let Some(groups) = self.sfx_index.get(add) else { continue }; + for g in groups { + root.clear(); + root.push_str(base); + root.push_str(&g.strip); + let inner = g + .rules + .iter() + .filter_map(|&i| self.suffixes.get(i as usize)) + .filter(|r| r.cont.meets(flags) && r.cond_matches(&root, true)) + .fold(Mask::EMPTY, |m, r| m | r.bit); + if !inner.is_empty() && self.form_has_flag(&root, inner, depth - 1, deny) { + return true; + } + } + } + false + } + + /// Is `w` exactly (case and all) a stem or an affixed form of one? `ICONV` and `IGNORE` + /// apply first. + pub fn contains(&self, w: &str) -> bool { + self.contains_except(&self.normalize(w), Mask::EMPTY) + } + + /// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST and SUBSTANDARD + /// stems excluded). + pub fn contains_suggestible(&self, w: &str) -> bool { + self.contains_except(&self.normalize(w), self.no_suggest | self.substandard) + } + + /// `w` after the `ICONV` conversions (at each position the longest match wins) and without + /// the `IGNORE` characters. + fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> { + if self.iconv.is_empty() && self.ignore.is_empty() { + return std::borrow::Cow::Borrowed(w); + } + // One pass: the input is copied only from the first change on. + let mut out: Option = None; + let mut rest = w; + while let Some(c) = rest.chars().next() { + let done = w.get(..w.len() - rest.len()).unwrap_or_default(); + let hit = self.iconv.iter().filter(|(from, _)| !from.is_empty() && rest.starts_with(from.as_str())).max_by_key(|(from, _)| from.len()); + match hit { + Some((from, to)) => { + out.get_or_insert_with(|| done.to_string()).extend(to.chars().filter(|c| !self.ignore.contains(c))); + rest = rest.get(from.len()..).unwrap_or_default(); + } + None => { + if self.ignore.contains(&c) { + out.get_or_insert_with(|| done.to_string()); + } else if let Some(o) = out.as_mut() { + o.push(c); + } + rest = rest.get(c.len_utf8()..).unwrap_or_default(); + } + } + } + out.map_or(std::borrow::Cow::Borrowed(w), std::borrow::Cow::Owned) + } + + /// [`AffixDict::contains`] on normalized input, where stems carrying a flag in `deny` don't + /// count (case variants of `KEEPCASE` stems). + fn contains_except(&self, w: &str, deny: Mask) -> bool { + if w.is_empty() || w.len() > MAX_WORD_BYTES { + return false; + } + if self.stem_flags_except(w, deny).is_some() || self.suffixed(w, Mask::EMPTY, deny) { + return true; + } + let mut root = String::with_capacity(w.len() + 8); + for r in &self.prefixes { + let Some(rest) = w.strip_prefix(&*r.add) else { continue }; + if rest.is_empty() { + continue; + } + root.clear(); + root.push_str(&r.strip); + root.push_str(rest); + if !r.cond_matches(&root, false) { + continue; + } + if self.stem_flags_except(&root, deny).is_some_and(|f| f.meets(r.bit)) { + return true; + } + if self.form_has_flag(&root, r.bit, INNER_CHAIN, deny) { + return true; + } + if r.cross && self.suffixed(&root, r.bit, deny) { + return true; + } + } + false + } + + /// Is `w` a stem with one suffix? With `need` set (a prefix was taken off), the stem must + /// carry those flags too and the suffix must allow cross products. + fn suffixed(&self, w: &str, need: Mask, deny: Mask) -> bool { + let mut root = String::with_capacity(w.len() + 8); + // Each split point from the end: the appended string is `w[b..]`, at most the longest + // suffix, and something must stay before it. + let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0)); + for b in cuts.take(self.max_sfx_chars + 1) { + let (Some(base), Some(add)) = (w.get(..b), w.get(b..)) else { continue }; + let Some(groups) = self.sfx_index.get(add) else { continue }; + for g in groups { + root.clear(); + root.push_str(base); + root.push_str(&g.strip); + let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)); + // The stem path: `root` is a stem carrying the rule's flag (and the prefix's, + // when a cross product is being checked). + if let Some(f) = self.stem_flags_except(&root, deny) + && f.covers(need) + && rules().any(|r| f.meets(r.bit) && (need.is_empty() || r.cross) && r.cond_matches(&root, true)) + { + return true; + } + // Continuation classes: `root` need not be a stem carrying this rule's flag — + // it may be a form that gained the flag from another rule's continuation. The + // matching rules' flags are looked for together, in one walk. + if need.is_empty() { + let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(Mask::EMPTY, |m, r| m | r.bit); + if !flags.is_empty() && self.form_has_flag(&root, flags, INNER_CHAIN, deny) { + return true; + } + } + } + } + false + } + + /// Is `w` correct, Hunspell style: as written, or — for a capitalized word or one in capitals + /// — in lower case (and capitalized), since a sentence may start with any word and headings + /// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`), and a + /// `KEEPCASE` stem matches only as written. + pub fn check(&self, w: &str) -> bool { + self.check_except(w, Mask::EMPTY) + } + + /// Whether a suggestion is correct in its displayed case and has an eligible stem. + pub fn can_suggest(&self, w: &str) -> bool { + self.check_except(w, self.no_suggest | self.substandard) + } + + /// Whether `w` is a word only through a `KEEPCASE` stem, so it is correct in this letter case + /// alone (`km` from `km/K`, which `Km` and `KM` are not). + pub(crate) fn case_locked(&self, w: &str) -> bool { + self.keepcase != Mask::EMPTY && self.contains(w) && !self.contains_except(&self.normalize(w), self.keepcase) + } + + fn check_except(&self, w: &str, deny: Mask) -> bool { + let w = self.normalize(w); + if self.contains_except(&w, deny) { + return true; + } + let deny = deny | self.keepcase; + match casing(&w) { + Casing::Lower | Casing::Mixed => false, + Casing::Initial => self.contains_except(&w.to_lowercase(), deny), + Casing::Upper => { + let lower = w.to_lowercase(); + self.contains_except(&capitalize(&lower), deny) || self.contains_except(&lower, deny) + } + } + } + + /// Every form of the stem `stem` (itself included), unsorted, possibly with repeats; at most + /// [`MAX_FORMS`], and at most [`MAX_FORMS_WORK`] rules tried along the continuation chains and + /// cross products, so a dictionary with thousands of self-continuing rules can't make one stem + /// cost millions of strings (#3613). + pub fn forms(&self, stem: &str) -> Vec { + let Some(f) = self.stem_flags(stem) else { return Vec::new() }; + let mut out = vec![stem.to_string()]; + let mut cross_sfx = Vec::new(); + // Suffix forms queued with the continuation classes they carry, for chained expansion; + // each (form, classes) pair is expanded once. + let mut chain: Vec<(String, Mask)> = Vec::new(); + let mut queued: std::collections::HashSet<(String, Mask)> = std::collections::HashSet::new(); + let mut work = 0usize; + for r in &self.suffixes { + if !f.meets(r.bit) || !r.cond_matches(stem, true) { + continue; + } + let Some(base) = stem.strip_suffix(&*r.strip) else { continue }; + let form = format!("{base}{}", r.add); + if r.cross { + cross_sfx.push(form.clone()); + } + if !r.cont.is_empty() && queued.insert((form.clone(), r.cont)) { + chain.push((form.clone(), r.cont)); + } + out.push(form); + if out.len() >= MAX_FORMS { + return out; + } + } + // Continuation classes chain suffixes (`workable/B` → `workables`), up to [`MAX_CHAIN`] + // suffixes in all — the same bound the checker walks. + 'chain: for _ in 0..INNER_CHAIN { + let level = std::mem::take(&mut chain); + if level.is_empty() { + break; + } + for (form, flags) in level { + for r in self.suffixes_with(flags).into_iter().filter_map(|i| self.suffixes.get(i as usize)) { + work += 1; + if work > MAX_FORMS_WORK { + break 'chain; + } + if !r.cond_matches(&form, true) { + continue; + } + let Some(base) = form.strip_suffix(&*r.strip) else { continue }; + let next = format!("{base}{}", r.add); + if !r.cont.is_empty() && queued.insert((next.clone(), r.cont)) { + chain.push((next.clone(), r.cont)); + } + out.push(next); + if out.len() >= MAX_FORMS { + return out; + } + } + } + } + for r in &self.prefixes { + if !f.meets(r.bit) || !r.cond_matches(stem, false) { + continue; + } + let Some(rest) = stem.strip_prefix(&*r.strip) else { continue }; + out.push(format!("{}{rest}", r.add)); + if r.cross { + for s in &cross_sfx { + work += 1; + if out.len() >= MAX_FORMS || work > MAX_FORMS_WORK { + return out; + } + if let Some(rest) = s.strip_prefix(&*r.strip) { + out.push(format!("{}{rest}", r.add)); + } + } + } + if out.len() >= MAX_FORMS { + return out; + } + } + out + } + + /// The suffix rules carrying any flag of `mask`, in rule order. + fn suffixes_with(&self, mask: Mask) -> Vec { + let mut rules: Vec = Vec::new(); + let mut lists = 0; + for (bit, list) in self.sfx_by_flag.iter().enumerate() { + if mask.meets(Mask::bit(bit)) && !list.is_empty() { + rules.extend_from_slice(list); + lists += 1; + } + } + if lists > 1 { + rules.sort_unstable(); + } + rules + } + + /// The `TRY` letters, most frequent first (suggestions try these). + pub fn try_chars(&self) -> &str { + &self.try_chars + } + + /// The `KEY` keyboard rows, separated by `|` (adjacent letters are likely mistypes). + pub fn key(&self) -> &str { + &self.key + } + + /// The `MAP` groups: letters often confused with each other (`aą`, `zżź`, `oóu`…). + pub fn map(&self) -> &[Vec] { + &self.map + } + + /// The `REP` table: common misspellings, `(wrong, right)`. + pub fn rep(&self) -> &[(String, String)] { + &self.rep + } + + /// The provenance and licence lines. + pub fn comments(&self) -> &[String] { + &self.comments + } + + /// Serialize to WordCraft's compact format: raw-deflate of `WCAFF1`, the comments, the + /// tables and rules as text lines, a `WORDS` line, then one line per stem, front-coded + /// against the previous one (a byte with the shared prefix length in bytes, the rest of the + /// word, `/` and its flags if any). + pub fn to_bytes(&self) -> Vec { + let mut out = String::new(); + out.push_str(MAGIC); + out.push('\n'); + for c in &self.comments { + out.push_str("# "); + out.push_str(c.trim()); + out.push('\n'); + } + if !self.try_chars.is_empty() { + out.push_str(&format!("TRY {}\n", self.try_chars)); + } + if !self.key.is_empty() { + out.push_str(&format!("KEY {}\n", self.key)); + } + for m in &self.map { + out.push_str(&format!("MAP {}\n", m.iter().collect::())); + } + for (a, b) in &self.rep { + out.push_str(&format!("REP {} {}\n", a.replace(' ', "_"), b.replace(' ', "_"))); + } + for (directive, mask) in + [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest), ("SUBSTANDARD", self.substandard), ("KEEPCASE", self.keepcase)] + { + if !mask.is_empty() + && let Some(c) = self.flag_char(mask) + { + out.push_str(&format!("{directive} {c}\n")); + } + } + for (a, b) in &self.iconv { + out.push_str(&format!("ICONV {a} {b}\n")); + } + if !self.ignore.is_empty() { + out.push_str(&format!("IGNORE {}\n", self.ignore.iter().collect::())); + } + let z = |s: &str| if s.is_empty() { "0".to_string() } else { s.to_string() }; + for (kind, rules) in [("PFX", &self.prefixes), ("SFX", &self.suffixes)] { + for r in rules { + let y = if r.cross { "Y" } else { "N" }; + // Continuation classes ride on the appended field, as in the `.aff` source. + let mut add = z(&r.add); + let cont = self.flag_chars_for(r.cont); + if !cont.is_empty() { + add.push('/'); + add.push_str(&cont); + } + out.push_str(&format!("{kind} {} {y} {} {} {}\n", r.flag, z(&r.strip), add, r.cond_text)); + } + } + out.push_str("WORDS\n"); + let mut raw = out.into_bytes(); + let mut prev: &[u8] = &[]; + for i in 0..self.len() { + let Some(w) = self.word(i) else { continue }; + let b = w.as_bytes(); + let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255); + raw.push(p as u8); + raw.extend_from_slice(b.get(p..).unwrap_or_default()); + let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or_default(); + if !mask.is_empty() { + raw.push(b'/'); + raw.extend_from_slice(self.flag_chars_for(mask).as_bytes()); + } + raw.push(b'\n'); + prev = b; + } + miniz_oxide::deflate::compress_to_vec(&raw, 10) + } + + /// Read data written by [`AffixDict::to_bytes`]. + pub fn from_bytes(data: &[u8]) -> Result { + let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(data, 64 << 20) + .map_err(|e| format!("affix dictionary: inflate failed: {:?}", e.status))?; + let words_at = raw.windows(7).position(|w| w == b"\nWORDS\n").ok_or("affix dictionary: no WORDS section")?; + let head = std::str::from_utf8(raw.get(..words_at).unwrap_or_default()).map_err(|_| "affix dictionary: bad utf-8 in the rules")?; + let mut lines = head.lines(); + if lines.next() != Some(MAGIC) { + return Err("affix dictionary: bad header".into()); + } + let mut rules = Rules::default(); + let mut comments = Vec::new(); + for line in lines { + if let Some(c) = line.strip_prefix("# ") { + comments.push(c.to_string()); + continue; + } + let f: Vec<&str> = split_fields(line).collect(); + let at = |k: usize| f.get(k).copied().unwrap_or(""); + match at(0) { + "TRY" => rules.try_chars = cap_try(at(1)), + "KEY" => rules.key = at(1).to_string(), + directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "SUBSTANDARD" | "KEEPCASE") => { + let c = one_flag(at(1), directive, 0).map_err(|_| format!("affix dictionary: bad {directive} in `{line}`"))?; + match directive { + "FORBIDDENWORD" => rules.forbidden = Some(c), + "NOSUGGEST" => rules.no_suggest = Some(c), + "SUBSTANDARD" => rules.substandard = Some(c), + _ => rules.keepcase = Some(c), + } + } + "ICONV" if f.len() == 3 => rules.iconv.push((at(1).to_string(), at(2).to_string())), + "IGNORE" => rules.ignore = at(1).chars().collect(), + "MAP" => rules.map.push(at(1).chars().collect()), + "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))), + kind @ ("PFX" | "SFX") => { + let mut fc = at(1).chars(); + let (Some(flag), None) = (fc.next(), fc.next()) else { return Err(format!("affix dictionary: bad flag in `{line}`")) }; + if f.len() != 6 { + return Err(format!("affix dictionary: bad rule `{line}`")); + } + // Continuation classes ride on the appended field, as in the `.aff` source. + let (add, cont) = match at(4).split_once('/') { + Some((a, c)) => (zero_empty(a), c.to_string()), + None => (zero_empty(at(4)), String::new()), + }; + let rule: RawRule = (flag, at(2) == "Y", zero_empty(at(3)), add, cont, at(5).to_string()); + if kind == "SFX" { + rules.suffixes.push(rule); + } else { + rules.prefixes.push(rule); + } + } + "" => {} + other => return Err(format!("affix dictionary: unknown line `{other}`")), + } + } + let body = raw.get(words_at + 7..).unwrap_or_default(); + let mut stems: Vec<(String, String)> = Vec::new(); + let mut cur: Vec = Vec::new(); + let mut i = 0; + while let Some(&p) = body.get(i) { + let rest = body.get(i + 1..).unwrap_or_default(); + let len = rest.iter().position(|&b| b == b'\n').ok_or("affix dictionary: truncated")?; + let line = rest.get(..len).unwrap_or_default(); + i += len + 2; + let p = p as usize; + if p > cur.len() { + return Err("affix dictionary: bad prefix".into()); + } + let (tail, flags) = match line.iter().position(|&b| b == b'/') { + Some(k) => (line.get(..k).unwrap_or_default(), line.get(k + 1..).unwrap_or_default()), + None => (line, &[][..]), + }; + cur.truncate(p); + cur.extend_from_slice(tail); + let word = std::str::from_utf8(&cur).map_err(|_| "affix dictionary: bad utf-8 in a stem")?; + let flags = std::str::from_utf8(flags).map_err(|_| "affix dictionary: bad utf-8 in flags")?; + if word.is_empty() || word.len() > MAX_WORD_BYTES { + return Err("affix dictionary: bad stem".into()); + } + stems.push((word.to_string(), flags.to_string())); + } + AffixDict::build(rules, stems, comments) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + const AFF: &str = "SET UTF-8\nTRY aeoiułkr\n\nPFX b Y 1\nPFX b 0 nie .\n\nSFX a Y 3\nSFX a 0 y [^y]\nSFX a ek ka ek\nSFX a 0 em .\n\nSFX c N 1\nSFX c a ą a\n\nMAP 2\nMAP aą\nMAP oóu\nREP 2\nREP ż rz\nREP u ó\n"; + const DIC: &str = "4\nkot/ab\npiesek/a\nkrowa/c\nWarszawa/c\n"; + + fn dict() -> AffixDict { + AffixDict::from_hunspell(AFF, DIC, &["test data"]).unwrap() + } + + #[test] + fn affix_headers_allow_comments_but_rule_conditions_stay_literal() { + let aff = "SET UTF-8\nPFX X Y 1 # prefix comment\nPFX X 0 ne . # rule comment\nSFX y Y 1 # suffix comment\nSFX y 0 y . # rule comment\n"; + let d = AffixDict::from_hunspell(aff, "1\nstrom/Xy\n", &[]).unwrap(); + assert_eq!(d.rule_count(), 2, "headers must not become affix rules"); + for w in ["strom", "stromy", "nestrom", "nestromy"] { + assert!(d.contains(w), "{w}"); + } + for w in ["stromY1", "stromN1", "X", "y"] { + assert!(!d.contains(w), "{w}"); + } + let hash = AffixDict::from_hunspell("SFX H Y 1\nSFX H 0 # .\nTRY a#b\nKEY a#b\n", "1\nstrom/H\n", &[]).unwrap(); + assert!(hash.contains("strom#"), "a hash in an affix field is literal"); + let literal = AffixDict::from_hunspell( + "PFX P Y 1 # header\nPFX P 0 pre #\nSFX S Y 1 # header\nSFX S 0 post #\n", + "3\n#stem/P\nstem#/S\nstem/PS\n", + &[], + ) + .unwrap(); + assert!(literal.contains("pre#stem") && literal.contains("stem#post")); + assert!(!literal.contains("prestem") && !literal.contains("stempost"), "# must remain a literal condition"); + + assert_eq!(hash.try_chars(), "a#b"); + assert_eq!(hash.key(), "a#b"); + assert_eq!(AffixDict::from_bytes(&hash.to_bytes()).unwrap().key(), "a#b"); + } + + #[test] + fn try_alphabet_is_capped_and_deduplicated() { + let aff = format!("TRY {}\n", "abcdefghijklmnopqrstuvwxyz".repeat(10_000)); + let d = AffixDict::from_hunspell(&aff, "1\nkot\n", &[]).unwrap(); + assert_eq!(d.try_chars(), "abcdefghijklmnopqrstuvwxyz"); + let compact = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert_eq!(compact.try_chars(), "abcdefghijklmnopqrstuvwxyz"); + + // More distinct characters than the cap, each repeated: keep the first 128 in order. + let many: String = (0x100u32..0x100 + 300).filter_map(char::from_u32).collect(); + let aff = format!("TRY {many}{many}\n"); + let want: String = many.chars().take(128).collect(); + let d = AffixDict::from_hunspell(&aff, "1\nkot\n", &[]).unwrap(); + assert_eq!(d.try_chars(), want); + let compact = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert_eq!(compact.try_chars(), want); + + // An oversized TRY line fed straight to the compact reader is capped as well. + let raw = format!("{MAGIC}\nTRY {many}{many}\nWORDS\n\0kot\n"); + let bytes = miniz_oxide::deflate::compress_to_vec(raw.as_bytes(), 6); + assert_eq!(AffixDict::from_bytes(&bytes).unwrap().try_chars(), want); + } + + /// Shared hand-written fixture: collectors must reject every path to NOSUGGEST forms. + pub(crate) fn assert_nosuggest_is_honoured(suggest: impl Fn(&AffixDict, &str, usize) -> Vec) { + let aff = "SET UTF-8\nNOSUGGEST N\nFORBIDDENWORD F\nTRY abeklmnostxy\nMAP 1\nMAP ae\nKEY ak\nREP 2\nREP xy toka\nREP zz normal\nPFX X Y 1\nPFX X 0 pre .\nSFX A Y 1\nSFX A 0 ble/B .\nSFX B Y 1\nSFX B 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "4\ntoka/NXA\nnormal/XA\nTkname/N\nbanned/F\n", &[]).unwrap(); + for word in d.forms("toka").into_iter().chain(["TOKA".into(), "Toka".into(), "Tkname".into(), "TKNAME".into()]) { + assert!(d.check(&word), "NOSUGGEST is still correct: {word}"); + assert!(!d.can_suggest(&word), "NOSUGGEST form: {word}"); + } + for input in ["tokax", "tokablesx", "pretokax", "pretokablesx", "toka", "TOKA", "tkname", "teka", "xy", "normaltoka", "toak", "taak"] { + let results = suggest(&d, input, 32); + assert!(results.iter().all(|word| word.split(' ').all(|part| d.can_suggest(part))), "{input}: {results:?}"); + assert!(results.iter().all(|word| !word.to_lowercase().contains("toka") && word.to_lowercase() != "tkname"), "{input}: {results:?}"); + } + assert!(suggest(&d, "norrmal", 8).contains(&"normal".into())); + assert!(suggest(&d, "zz", 8).contains(&"normal".into())); + // Filtering happens before display casing: ordinary mixed-case stems retain the + // existing all-caps suggestion behaviour, with and without a NOSUGGEST declaration. + for aff in ["", "NOSUGGEST N\n"] { + let ordinary = AffixDict::from_hunspell(aff, "2\nmixCase\nhidden/N\n", &[]).unwrap(); + assert!(suggest(&ordinary, "MIXCASX", 8).contains(&"MIXCASE".into())); + } + } + + #[test] + fn nosuggest_is_correct_but_not_suggestible_and_survives_compact_format() { + let d = AffixDict::from_hunspell("NOSUGGEST N\nSFX S Y 1\nSFX S 0 s .\n", "2\nsecret/NS\nnormal/S\n", &[]).unwrap(); + for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] { + for word in ["secret", "secrets", "Secret", "SECRETS"] { + assert!(dict.check(word), "{word}"); + assert!(!dict.can_suggest(word), "{word}"); + } + assert!(dict.contains_suggestible("normal") && dict.can_suggest("NORMALS")); + } + assert!(AffixDict::from_hunspell("NOSUGGEST NN\n", "1\nword\n", &[]).is_err()); + } + + #[test] + fn stems_suffixes_prefixes_and_cross_products() { + let d = dict(); + assert_eq!(d.len(), 4); + for w in ["kot", "koty", "kotem", "piesek", "pieska", "piesekem", "krowa", "krową", "niekot", "niekoty", "niekotem", "Warszawa", "Warszawą"] + { + assert!(d.contains(w), "{w}"); + } + for w in ["kotą", "pieseka", "pieskem", "niepiesek", "niekrową", "krowy", "warszawa", "", "nie", "y", "ka"] { + assert!(!d.contains(w), "{w}"); + } + } + + #[test] + fn case_variants() { + let d = dict(); + assert!(d.check("Kot") && d.check("KOT") && d.check("Koty") && d.check("WARSZAWĄ") && d.check("Warszawą")); + assert!(!d.check("warszawa") && !d.check("kOT") && !d.check("KoT")); + assert_eq!(casing("Kot"), Casing::Initial); + assert_eq!(casing("KOT"), Casing::Upper); + assert_eq!(casing("K"), Casing::Initial); + assert_eq!(casing("kot"), Casing::Lower); + assert_eq!(casing("McDonald"), Casing::Mixed); + assert_eq!(capitalize("żółw"), "Żółw"); + } + + #[test] + fn forms_are_all_accepted() { + let d = dict(); + let mut forms = d.forms("kot"); + forms.sort(); + forms.dedup(); + assert_eq!(forms, ["kot", "kotem", "koty", "niekot", "niekotem", "niekoty"]); + for stem in d.stems().map(str::to_string).collect::>() { + for f in d.forms(&stem) { + assert!(d.contains(&f), "{stem} → {f}"); + } + } + } + + #[test] + fn compact_format_round_trips() { + let d = dict(); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert_eq!(back.len(), d.len()); + assert_eq!(back.rule_count(), d.rule_count()); + assert_eq!(back.comments(), ["test data"]); + assert_eq!(back.rep(), d.rep()); + assert_eq!(back.map(), d.map()); + assert_eq!(back.try_chars(), "aeoiułkr"); + for w in ["kotem", "niekoty", "pieska", "krową", "Warszawą"] { + assert!(back.contains(w), "{w}"); + } + assert!(!back.contains("kotą")); + } + + #[test] + fn flag_utf8_and_default_modes_are_accepted() { + // `FLAG UTF-8` (VERO pt_BR) and `FLAG DEFAULT` are per-character flag modes: the same + // mode the reader already assumes. Only `long` and `num` stay unsupported. + for mode in ["UTF-8", "DEFAULT"] { + let aff = format!("SET UTF-8\nFLAG {mode}\nSFX a Y 1\nSFX a 0 y [^y]\n"); + let d = AffixDict::from_hunspell(&aff, "1\nkot/a\n", &[]).unwrap(); + assert!(d.contains("kot") && d.contains("koty"), "{mode}"); + } + assert!(AffixDict::from_hunspell("FLAG UTF-8\n", "0\n", &[]).is_ok(), "FLAG alone is harmless"); + } + + #[test] + fn emoji_flags_with_a_variation_selector_load() { + // RLA-ES (the Spanish dictionaries in LibreOffice and most Linux distributions) writes its + // telephone flag as ☎ followed by U+FE0F, in the rule and on the stems. + let aff = "SET UTF-8\nFLAG UTF-8\nPFX \u{260e}\u{fe0f} Y 1\nPFX \u{260e}\u{fe0f} 0 tele .\nSFX S Y 1\nSFX S 0 s [aeiou]\n"; + let d = AffixDict::from_hunspell(aff, "2\nbanco/S\u{260e}\u{fe0f}\nbasura/S\n", &[]).unwrap(); + assert!(d.contains("banco") && d.contains("bancos")); + assert!(d.contains("telebanco") && d.contains("telebancos"), "the ☎ prefix applies"); + assert!(d.contains("basuras") && !d.contains("telebasura"), "only to stems that carry it"); + assert!(!d.flag_chars.contains(&EMOJI_SELECTOR), "the selector is not a flag"); + } + + #[test] + fn emoji_forbiddenword_and_nosuggest_flags_load() { + // The same selector after the flag of the directives that name one flag. + let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD \u{260e}\u{fe0f}\nNOSUGGEST \u{1f6ab}\u{fe0f}\n"; + let d = AffixDict::from_hunspell(aff, "3\nbanco\nbanko/\u{260e}\u{fe0f}\nbancu/\u{1f6ab}\u{fe0f}\n", &[]).unwrap(); + assert!(d.contains("banco") && d.contains("bancu")); + assert!(!d.contains("banko"), "a stem with the FORBIDDENWORD flag is not a word"); + } + + #[test] + fn more_than_64_distinct_flags_load() { + // VERO pt_BR uses 106 distinct flags (FLAG UTF-8). Build one with every basic Latin + // letter, the digits and eight accented letters: 70 flags, more than 64. + let mut flags = String::new(); + let mut aff = String::from("SET UTF-8\nFLAG UTF-8\n"); + let chars: Vec = ('a'..='z').chain('A'..='Z').chain('0'..='9').chain(['á', 'é', 'í', 'ó', 'ú', 'ç', 'ã', 'õ']).collect(); + assert_eq!(chars.len(), 70); + for c in chars { + aff.push_str(&format!("SFX {c} N 1\nSFX {c} 0 {c} .\n")); + flags.push(c); + } + let d = AffixDict::from_hunspell(&aff, &format!("1\nverbo/{flags}\n"), &[]).unwrap(); + assert!(d.contains("verbo")); + assert!(d.contains("verboa") && d.contains("verboz") && d.contains("verbo9")); + assert!(d.contains("verboç") && d.contains("verboã"), "multi-byte UTF-8 flags"); + assert!(!d.contains("verboß"), "ß is not a flag and not a stem"); + } + + #[test] + fn more_than_128_distinct_flags_load() { + // The Hungarian dictionary uses 158 single-byte flags: 200 flags here, U+0080 up. + let flags: String = (0x80u32..0x80 + 200).filter_map(char::from_u32).collect(); + let mut aff = String::from("SET UTF-8\n"); + for (i, c) in flags.chars().enumerate() { + aff.push_str(&format!("SFX {c} N 1\nSFX {c} 0 x{i} .\n")); + } + let half: String = flags.chars().step_by(2).collect(); + let d = AffixDict::from_hunspell(&aff, &format!("1\nszó/{half}\n"), &[]).unwrap(); + assert!(d.contains("szóx0") && d.contains("szóx198") && d.contains("szóx130")); + assert!(!d.contains("szóx1") && !d.contains("szóx199") && !d.contains("szóx131")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.contains("szóx198") && !back.contains("szóx199")); + let too_many: String = (0x100u32..0x100 + 257).filter_map(char::from_u32).collect(); + assert!(AffixDict::from_hunspell("SET UTF-8\n", &format!("1\nszó/{too_many}\n"), &[]).is_err()); + } + + #[test] + fn forbidden_words_are_rejected() { + // VERO pt_BR marks spellings it must reject (flag ý under FLAG UTF-8) as stems carrying + // FORBIDDENWORD: listed, but never correct and never inflected. + let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD ý\nSFX a Y 1\nSFX a 0 y [^y]\n"; + let d = AffixDict::from_hunspell(aff, "2\nkot/a\nimundo/ý\n", &[]).unwrap(); + assert!(d.check("kot") && d.check("koty") && d.check("Kot") && d.check("KOT")); + assert!(!d.check("imundo") && !d.check("Imundo") && !d.check("IMUNDO")); + assert!(!d.contains("imundoy"), "a forbidden stem must not inflect"); + assert!(d.forms("imundo").is_empty()); + } + + #[test] + fn continuation_classes_chain_suffixes() { + // `SFX A 0 able/B`: the form derived by A carries flag B, so a suffix of class B may + // follow (workable → workables). The bare stem has A only, so `works` stays wrong. + let aff = "SET UTF-8\nFLAG UTF-8\nSFX A N 1\nSFX A 0 able/B .\nSFX B N 1\nSFX B 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "1\nwork/A\n", &[]).unwrap(); + assert!(d.contains("work") && d.contains("workable") && d.contains("workables")); + assert!(!d.contains("works"), "the stem has A, not B"); + assert!(!d.contains("workabless"), "B declares no continuation"); + assert!(d.check("Workable") && d.check("WORKABLES")); + } + + #[test] + fn continuation_chains_stop_at_two_suffixes() { + // Hunspell strips at most two suffixes: work + able/B + s/C is a word, a third (C) is not. + let aff = "SET UTF-8\nFLAG UTF-8\nSFX A N 1\nSFX A 0 able/B .\nSFX B N 1\nSFX B 0 s/C .\nSFX C N 1\nSFX C 0 x .\n"; + let d = AffixDict::from_hunspell(aff, "1\nwork/A\n", &[]).unwrap(); + assert!(d.contains("workable") && d.contains("workables")); + assert!(!d.contains("workablesx"), "three chained suffixes exceed the limit"); + assert!(!d.forms("work").iter().any(|f| f == "workablesx")); + } + + #[test] + fn forms_include_continuation_chains() { + let aff = "SET UTF-8\nFLAG UTF-8\nSFX A N 1\nSFX A 0 able/B .\nSFX B N 1\nSFX B 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "1\nwork/A\n", &[]).unwrap(); + let mut f = d.forms("work"); + f.sort(); + f.dedup(); + assert_eq!(f, ["work", "workable", "workables"]); + for form in d.forms("work") { + assert!(d.contains(&form), "every listed form is accepted: {form}"); + } + } + + #[test] + fn continuation_classes_merge_the_flags_of_a_group() { + // `s` under B and under C share one group (same append, same strip): a form carrying + // either flag takes it, and the stem carrying neither doesn't. + let aff = "SET UTF-8\nSFX A N 1\nSFX A 0 able/C .\nSFX D N 1\nSFX D 0 er/B .\nSFX B N 1\nSFX B 0 s .\nSFX C N 1\nSFX C 0 s [e]\n"; + let d = AffixDict::from_hunspell(aff, "2\nwork/A\nplay/D\n", &[]).unwrap(); + assert!(d.contains("workables") && d.contains("players")); + assert!(!d.contains("works") && !d.contains("plays") && !d.contains("workabless")); + assert!(d.forms("work").iter().any(|f| f == "workables") && d.forms("play").iter().any(|f| f == "players")); + } + + /// `n` suffix rules of flag A on one stem, `rule(i)` giving each rule's strip, append and condition. + fn many_rules(n: usize, rule: impl Fn(usize) -> String) -> AffixDict { + let mut aff = format!("SET UTF-8\nSFX A Y {n}\n"); + for i in 0..n { + aff.push_str(&format!("SFX A {}\n", rule(i))); + } + AffixDict::from_hunspell(&aff, "1\nkot/A\n", &[]).unwrap() + } + + #[test] + fn self_continuing_rules_check_in_linear_time() { + // Thousands of empty-append rules that continue with their own flag: each check used to + // follow every rule from every rule, millions of steps a word (#3613). The checks take + // about a millisecond; the old checker took 14 s for two of them at 16,000 rules, so at + // 40,000 (quadratic: minutes) the 10 s bound leaves a busy machine a wide margin. + let d = many_rules(40_000, |_| "0 0/A .".into()); + let t = std::time::Instant::now(); + assert!(d.check("kot") && !d.check("pies") && !d.check("kotx")); + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); + } + + #[test] + fn forms_of_self_continuing_rules_are_bounded() { + // Every rule's form continues with every rule: millions of strings for one stem (#3613). + let same = many_rules(3000, |_| "0 0/A .".into()); + let distinct = many_rules(3000, |i| format!("0 x{i}/A .")); + let t = std::time::Instant::now(); + let (a, b) = (same.forms("kot"), distinct.forms("kot")); + // The cap on the count is what tells the old code (9 million forms each) apart; the + // generous time bound (tens of ms here) only backs up the rule-visit cap. + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); + assert!(a.len() <= MAX_FORMS && b.len() <= MAX_FORMS, "{} {}", a.len(), b.len()); + assert!(b.iter().any(|f| f == "kotx0x1"), "chained forms are still listed"); + } + + #[test] + fn many_strips_per_suffix_index_in_linear_time() { + // One append with thousands of strips: the index used to scan the append's groups for + // each rule. Building 200,000 rules takes well under a second; the old index took 3 s at + // 30,000 and over 10 minutes at 300,000. + let n = 200_000; + let t = std::time::Instant::now(); + let d = many_rules(n, |i| format!("k{i} y .")); + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); + assert_eq!(d.sfx_index.get("y").map(Vec::len), Some(n)); + assert!(!d.contains("koty")); + } + + #[test] + fn compact_format_round_trips_continuations_and_forbidden() { + let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD ý\nSFX A N 1\nSFX A 0 able/B .\nSFX B N 1\nSFX B 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "2\nwork/A\nimundo/ý\n", &[]).unwrap(); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.contains("work") && back.contains("workable") && back.contains("workables")); + assert!(!back.contains("works"), "the stem has A, not B, after a round trip"); + assert!(!back.check("imundo") && !back.contains("imundoy"), "forbidden survives a round trip"); + } + + #[test] + fn dic_bom_glued_to_the_count_line_is_skipped() { + // VERO's .dic opens with a UTF-8 BOM glued to its count line; it must not become a stem. + let aff = "SET UTF-8\nSFX a Y 1\nSFX a 0 y [^y]\n"; + let d = AffixDict::from_hunspell(aff, "\u{feff}1\nkot/a\n", &[]).unwrap(); + assert_eq!(d.len(), 1); + assert!(d.contains("kot") && d.contains("koty")); + } + + #[test] + fn flag_aliases_stand_for_flag_sets() { + // `AF`: `/1` in the `.dic` and in a continuation class means the first alias's flags. + // Morphological aliases (`AM`) and the fields after a stem are skipped. + let aff = "SET UTF-8\nAF 2\nAF ab # 1\nAF c # 2\nAM 1\nAM po:noun\nSFX a Y 1\nSFX a 0 ok/2 . 1\nSFX b Y 1\nSFX b 0 at . 1\nSFX c Y 1\nSFX c 0 ban . 1\n"; + let d = AffixDict::from_hunspell(aff, "1\nház/1\t1\n", &[]).unwrap(); + assert!(d.contains("ház") && d.contains("házok") && d.contains("házat") && d.contains("házokban")); + assert!(!d.contains("házban"), "the stem has a and b, not c"); + // An undefined alias reads as no flags, as in Hunspell: the stem loads, without affixes. + let undefined = AffixDict::from_hunspell(aff, "2\nház/3\nkert/1\n", &[]).unwrap(); + assert!(undefined.check("ház") && !undefined.check("házok"), "alias 3 is not defined"); + assert!(undefined.check("kertok"), "the other entries keep their aliases"); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.contains("házokban") && !back.contains("házban"), "aliases are expanded in the compact format"); + } + + #[test] + fn iconv_and_ignore_normalize_input() { + // Ligatures read as their letters; `IGNORE` characters vanish from input and stems. + let aff = "SET UTF-8\nICONV 2\nICONV fi fi\nICONV ffl ffl\nIGNORE ()\nSFX a Y 1\nSFX a 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "2\nfine/a\nwaf(fle)/a\n", &[]).unwrap(); + assert!(d.check("fine") && d.check("fines") && d.check("Fine") && d.contains("fine")); + assert!(d.check("waffle") && d.check("waffles") && d.check("wa(ffle)")); + assert!(!d.check("finer")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.check("fines") && back.check("waffle") && back.check("waf(fle)")); + } + + #[test] + fn alias_numbers_are_read_as_hunspell_reads_them() { + // Checked with the Hunspell 1.7 command line. With an `AF` table a flag field is the alias + // its leading digits number: `fal/2x` is alias 2, `lab/b` has no flags. + let aff = "SET UTF-8\nAF 2\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\nSFX C Y 1\nSFX C 0 x .\nSFX b Y 1\nSFX b 0 q .\n"; + let d = AffixDict::from_hunspell(aff, "3\nlab/b\nfal/2x\nkut/1\n", &[]).unwrap(); + assert!(d.check("lab") && !d.check("labq") && !d.check("labs")); + assert!(d.check("falx") && !d.check("fals") && d.check("kuts")); + // A table without its count line is rejected: no alias applies. + let broken = AffixDict::from_hunspell("SET UTF-8\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\n", "2\nlab/1\nfal/2\n", &[]).unwrap(); + assert!(broken.check("lab") && !broken.check("labs") && !broken.check("fals")); + // `AF 0` is rejected the same way: `lab/B` keeps its literal flag. + let zero = AffixDict::from_hunspell("SET UTF-8\nAF 0\nSFX B Y 1\nSFX B 0 s .\n", "1\nlab/B\n", &[]).unwrap(); + assert!(zero.check("lab") && zero.check("labs")); + // An undefined alias in a continuation class leaves the rule in force, without it. + let cont = AffixDict::from_hunspell("SET UTF-8\nAF 1\nAF a\nSFX a Y 1\nSFX a 0 ok/7 .\n", "1\nhaz/1\n", &[]).unwrap(); + assert!(cont.check("haz") && cont.check("hazok")); + } + + #[test] + fn a_rep_count_line_may_carry_a_comment() { + let d = AffixDict::from_hunspell("SET UTF-8\nREP 1 # confusions\nREP x y\n", "1\nay\n", &[]).unwrap(); + assert_eq!(d.rep(), [("x".to_string(), "y".to_string())]); + } + + #[test] + fn count_lines_may_carry_a_comment() { + // Hunspell reads the first `AF` and `ICONV` line as the count, comment and all: `AF 2 #` + // is no alias, and `ICONV 1 # …` doesn't turn `1` into `#`. + let aff = "SET UTF-8\nAF 2 # aliases\nAF a # 1\nAF b # 2\nICONV 1 # ligatures\nICONV q y\nSFX a Y 1\nSFX a 0 ok .\nSFX b Y 1\nSFX b 0 at .\n"; + let d = AffixDict::from_hunspell(aff, "3\nhaz/1\nkert/2\na1\n", &[]).unwrap(); + assert!(d.check("hazok") && !d.check("hazat") && d.check("kertat") && !d.check("kertok"), "alias 1 is `a`"); + assert!(d.check("a1") && !d.check("a#")); + // The UTF-8 reader keeps such a line as it is, so both readers agree on it. + let (a, dic) = crate::hunspell::decode(aff.as_bytes(), "1\nhaz/1\n".as_bytes()).unwrap(); + assert!(a.contains("AF 2 # aliases") && a.contains("ICONV 1 # ligatures"), "{a}"); + assert!(AffixDict::from_hunspell(&a, &dic, &[]).unwrap().check("hazok")); + } + + #[test] + fn dic_entries_may_hold_spaces() { + // `ad hoc/B` is one entry, as in Hunspell; `ad` alone is no word. + let aff = "SET UTF-8\nSFX B Y 1\nSFX B 0 s .\n"; + // Checked with the Hunspell 1.7 command line: without a tab or a morphological field the + // whole line is the entry (`kot/B 12` has the flag `B`), and leading blanks belong to it + // (` lab` is no `lab`). + let d = AffixDict::from_hunspell(aff, "5\nad hoc/B\tpo:adv\nword/B po:noun\nkot/B 12\n lab/B\nés így/B\n", &[]).unwrap(); + assert!(d.contains("ad hoc") && d.contains("ad hocs") && d.contains("és így") && d.contains("és ígys")); + assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads") && !d.contains("így")); + assert!(d.contains("words") && d.contains("kots") && !d.contains("kot/B")); + assert!(!d.contains("lab") && !d.contains("labs")); + } + + #[test] + fn ignored_characters_leave_the_stripped_text_alone() { + // Hunspell removes `IGNORE` characters from the appended text, not from the stripped + // one: `SFX a bx c` never matches the stem `abx` (stored as `ab`), so `ac` is wrong, + // while `SFX a b cx` appends `c` (checked with the Hunspell 1.7 command line). + let strip = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a bx c .\n", "1\nabx/a\n", &[]).unwrap(); + assert!(strip.check("ab") && strip.check("abx") && !strip.check("ac")); + let add = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a b cx .\n", "1\nab/a\n", &[]).unwrap(); + assert!(add.check("ac") && add.check("acx")); + for d in [&strip, &add] { + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert_eq!(back.check("ac"), d.check("ac")); + } + } + + #[test] + fn normalizing_borrows_when_nothing_changes() { + let d = AffixDict::from_hunspell("SET UTF-8\nICONV 1\nICONV fi fi\nIGNORE ()\n", "1\nfine\n", &[]).unwrap(); + assert!(matches!(d.normalize("fine"), std::borrow::Cow::Borrowed("fine"))); + assert_eq!(d.normalize("(fi)ne"), "fine"); + assert!(matches!(dict().normalize("kot"), std::borrow::Cow::Borrowed("kot"))); + } + + #[test] + fn keepcase_stems_match_only_as_written() { + let aff = "SET UTF-8\nKEEPCASE k\nSFX a Y 1\nSFX a 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "3\npH/k\nkm/ka\nkot/a\n", &[]).unwrap(); + assert!(d.check("pH") && d.check("km") && d.check("kms")); + assert!(!d.check("PH") && !d.check("Km") && !d.check("KM") && !d.check("Kms")); + assert!(d.check("Kot") && d.check("KOTS"), "other stems still follow the sentence and heading case"); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.check("km") && !back.check("Km")); + } + + #[test] + fn substandard_and_lemma_present_words_are_correct() { + // `SUBSTANDARD` and `LEMMA_PRESENT` steer morphological generation, and a SUBSTANDARD + // stem is never suggested; the stems and rules they mark are still correct. + let aff = "SET UTF-8\nSUBSTANDARD x\nLEMMA_PRESENT l\nSFX a Y 2\nSFX a 0 y .\nSFX a 0 em/x .\n"; + let d = AffixDict::from_hunspell(aff, "2\nkot/a\npies/axl\n", &[]).unwrap(); + for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] { + assert!(dict.check("kot") && dict.check("koty") && dict.check("kotem")); + assert!(dict.check("pies") && dict.check("piesy") && dict.check("Piesem")); + assert!(dict.can_suggest("koty") && dict.can_suggest("kotem") && dict.contains_suggestible("kot")); + assert!(!dict.can_suggest("piesy") && !dict.can_suggest("Pies") && !dict.contains_suggestible("pies")); + } + } + + #[test] + fn unsupported_directives_are_refused_and_the_rest_load() { + // A dictionary that needs a directive outside this engine fails to load rather than + // accepting the wrong words. + for dir in UNSUPPORTED { + assert!(AffixDict::from_hunspell(&format!("SET UTF-8\n{dir} X\n"), DIC, &[]).is_err(), "{dir} should be refused"); + } + // The directives this engine reads, with no other change to the dictionary (`AF` would + // turn the flags of `DIC` into alias numbers; it has tests of its own). + for line in ["AM 1\nAM po:noun", "ICONV 1\nICONV fi fi", "IGNORE ()", "KEEPCASE k", "SUBSTANDARD x", "LEMMA_PRESENT l"] { + let d = AffixDict::from_hunspell(&format!("{AFF}{line}\n"), DIC, &[]).unwrap_or_else(|e| panic!("{line}: {e}")); + assert!(d.check("koty") && d.check("niekoty") && !d.check("kotx"), "{line}"); + } + } + + #[test] + fn fields_split_on_spaces_and_tabs_only() { + // A flag byte read as U+00A0 (no-break space) or U+0085 is a flag, not a separator. + let aff = "SET UTF-8\nSFX \u{a0} Y 1\nSFX \u{a0} 0 y .\nSFX \u{85} Y 1\nSFX \u{85} 0 em .\n"; + let d = AffixDict::from_hunspell(aff, "1\nkot/\u{a0}\u{85}\n", &[]).unwrap(); + assert!(d.contains("koty") && d.contains("kotem")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.contains("koty") && back.contains("kotem")); + // The same bytes as one-flag directives, as Hunspell reads them. + let aff = "SET UTF-8\nFORBIDDENWORD \u{a0}\nKEEPCASE \u{85}\nSFX a Y 1\nSFX a 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "3\nkot/a\nrossz/\u{a0}\npH/\u{85}\n", &[]).unwrap(); + for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] { + assert!(dict.check("kots") && !dict.check("rossz")); + assert!(dict.check("pH") && !dict.check("PH")); + } + } + + /// craft-libs#482 review: an English dictionary with compounding (for ordinals) and output + /// conversion, as the usual en_US and en_GB ones have, loads for English; its `ONLYINCOMPOUND` + /// stems are left out. Other languages still refuse those directives. (A synthetic `.aff` + /// written for this test.) + #[test] + fn english_dictionaries_ignore_compounding_and_output_conversion() { + let aff = "SET UTF-8\nTRY esianrtolcdugmphbyfvkwz'\nICONV 1\nICONV ’ '\nOCONV 1\nOCONV ' ’\nNOSUGGEST !\n\ + COMPOUNDMIN 1\nONLYINCOMPOUND c\nCOMPOUNDRULE 2\nCOMPOUNDRULE n*1t\nCOMPOUNDRULE n*mp\nCOMPOUNDFLAG Z\n\ + WORDCHARS 0123456789\nSFX S Y 1\nSFX S 0 s [^sxzhy]\nSFX D Y 1\nSFX D 0 ed [^ey]\n"; + let dic = "5\nhouse/S\nwalk/SD\n0th/pc\n1st/c\nthe\n"; + assert!(AffixDict::from_hunspell(aff, dic, &[]).is_err(), "refused for other languages"); + let d = AffixDict::from_hunspell_english(aff, dic, &[]).unwrap(); + for w in ["house", "houses", "walked", "walks", "the"] { + assert!(d.check(w), "{w}"); + } + assert!(!d.check("0th") && !d.check("1st"), "parts of compounds only"); + assert!(!d.check("housed") && !d.check("hous")); + // The directives that change which words are right are still refused. + for dir in ["NEEDAFFIX", "CIRCUMFIX", "PSEUDOROOT", "FULLSTRIP"] { + assert!(AffixDict::from_hunspell_english(&format!("{dir} X\n"), dic, &[]).is_err(), "{dir}"); + } + } + + #[test] + fn unsupported_and_broken_input_is_an_error() { + assert!(AffixDict::from_hunspell("FLAG long\n", DIC, &[]).is_err()); + assert!(AffixDict::from_hunspell("FLAG num\n", DIC, &[]).is_err()); + assert!(AffixDict::from_hunspell("COMPOUNDFLAG X\n", DIC, &[]).is_err()); + assert!(AffixDict::from_hunspell("SFX a Y 1\nSFX a 0 y [ab\n", DIC, &[]).is_err()); + assert!(AffixDict::from_bytes(b"not deflate").is_err()); + let ok = dict().to_bytes(); + assert!(AffixDict::from_bytes(&ok[..ok.len() / 2]).is_err()); + let garbage = miniz_oxide::deflate::compress_to_vec(b"WCAFF1\nWORDS\n\x09abc\n", 6); + assert!(AffixDict::from_bytes(&garbage).is_err(), "prefix longer than the previous word"); + let empty = AffixDict::default(); + assert!(!empty.contains("kot") && !empty.check("Kot") && empty.forms("kot").is_empty()); + } + + proptest::proptest! { + #[test] + fn lookups_never_panic(s in "\\PC{0,40}", bytes in proptest::collection::vec(proptest::num::u8::ANY, 0..200)) { + let d = dict(); + let _ = d.check(&s); + let _ = d.forms(&s); + let _ = AffixDict::from_bytes(&bytes); + let _ = AffixDict::from_bytes(&miniz_oxide::deflate::compress_to_vec(&bytes, 1)); + } + } +} diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs new file mode 100644 index 00000000..1456f81c --- /dev/null +++ b/crates/proof/src/affix_suggest.rs @@ -0,0 +1,556 @@ +//! Spelling suggestions from an affix dictionary, one engine for every language that reads one +//! (Czech, Croatian, Indonesian, Italian, Polish, Portuguese, Russian, Spanish). Candidates come +//! in rounds of rising cost, each checked against the dictionary: +//! 1. letter case (`warszawa` → `Warszawa`); +//! 2. missing or wrong diacritics and other MAP slips, a few letters changed (`zolw` → `żółw`): +//! adding a missing accent is the commonest slip and ranks first, while taking away one the +//! writer typed ranks after the known confusions; +//! 3. the REP table of common confusions (`żeka` → `rzeka`); +//! 4. a missing space (`napewno` → `na pewno`); +//! 5. keyboard neighbours from the KEY rows, then one typo: a letter left out, added, swapped with +//! the next or mistyped (TRY letters); +//! 6. when that finds fewer than asked for: the stems that start like the word (MAP relatives +//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance in +//! which a MAP slip costs half (two typos at most). +//! +//! The tables come from the dictionary; a language's [`Tuning`] adds the ones its dictionaries may +//! lack and the slips no table can say. The input is bounded ([`MAX_LETTERS`]), and so is every +//! round, so a call costs a few thousand dictionary lookups at most. + +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; + +use crate::affix::{AffixDict, Casing, capitalize, casing}; + +/// Longest word suggestions are looked for, in letters. +pub(crate) const MAX_LETTERS: usize = 40; +const MAX_MAP_VARIANTS: usize = 4096; +/// Stems the scan of round 6 expands. +const SCAN_STEMS: usize = 48; + +const COST_CASE: u32 = 0; +pub(crate) const COST_ADD_DIACRITIC: u32 = 4; +pub(crate) const COST_OTHER_MAP: u32 = 7; +const COST_DROP_DIACRITIC: u32 = 11; +const COST_REP: u32 = 10; +pub(crate) const COST_SPLIT: u32 = 12; +/// A missing space leaving a half shorter than three letters ([`Split::Anywhere`]). +const COST_SPLIT_SHORT: u32 = 17; +/// A missing space between two longer halves with no short word beside it ([`Split::Near`]). +pub(crate) const COST_SPLIT_LONG: u32 = 18; +const COST_KEY: u32 = 13; +const COST_SWAP: u32 = 14; +const COST_DROP_OR_ADD: u32 = 15; +pub(crate) const COST_MISTYPE: u32 = 16; +const COST_SCAN: u32 = 20; + +/// What a language adds to its dictionary's tables. +pub(crate) struct Tuning { + /// Where a missing space may go (round 4). + pub split: Split, + /// TRY letters for a dictionary without a TRY line. + pub alphabet: &'static str, + /// MAP groups the language adds to its dictionary's (each string is one group: `"cčć"`). + pub map: &'static [&'static str], + /// REP confusions the language adds to its dictionary's. + pub rep: &'static [(&'static str, &'static str)], + /// The cost of a REP confusion. + pub rep_cost: u32, + /// Most letters a MAP variant changes. + pub max_map_changes: usize, + /// The cost of changing letter `from` to its MAP relative `to`. + pub map_cost: fn(char, char) -> u32, + /// The cost of typing letter `c` at `i` of the word (`chars`, lower case) instead. + pub mistype_cost: fn(&[char], usize, char) -> u32, +} + +/// Where a missing space may go. +pub(crate) enum Split { + /// Anywhere, dearer when a half is shorter than three letters. + Anywhere, + /// After a short word that often runs into the next one (`na pewno`, `w ogóle`) or before one + /// that runs into the word before (`kocham cię`), dearer between two longer halves, and never + /// leaving a stray short half (`że ka`). + Near { leading: &'static [&'static str], trailing: &'static [&'static str] }, + /// The language's own rule: the cost of the halves `a b`, or `None` to skip them. + Custom(fn(&str, &str) -> Option), +} + +impl Tuning { + /// The dictionary's tables alone (Czech, Indonesian, Spanish). + pub(crate) const DEFAULT: Tuning = + Tuning { split: Split::Anywhere, alphabet: "", map: &[], rep: &[], rep_cost: COST_REP, max_map_changes: 4, map_cost, mistype_cost }; +} + +/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the +/// commonest slip (typing without diacritics); an accent the writer did type is likelier right, +/// so taking one away costs more than a known confusion from the REP table. +pub(crate) fn map_cost(from: char, to: char) -> u32 { + match (from.is_ascii(), to.is_ascii()) { + (true, false) => COST_ADD_DIACRITIC, + (false, true) => COST_DROP_DIACRITIC, + _ => COST_OTHER_MAP, + } +} + +/// Any mistyped letter costs the same. +fn mistype_cost(_: &[char], _: usize, _: char) -> u32 { + COST_MISTYPE +} + +struct Collector<'a> { + d: &'a AffixDict, + /// The word, lower case. + input: String, + /// Candidate → the cheapest cost it was offered at (rounds don't run in rising cost). + tested: HashMap, + /// Suggestion → (cost, order found). + found: HashMap, + seq: usize, +} + +impl Collector<'_> { + /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every + /// word is in the dictionary, as written or capitalized (names). + fn offer(&mut self, cand: &str, cost: u32) { + if cand.is_empty() || cand == self.input { + return; + } + match self.tested.get_mut(cand) { + Some(prev) if *prev <= cost => return, + Some(prev) => *prev = cost, + None => { + self.tested.insert(cand.to_string(), cost); + } + } + let mut words = Vec::new(); + let mut cost = cost; + for part in cand.split(' ') { + if self.d.contains_suggestible(part) { + words.push(part.to_string()); + } else { + let c = capitalize(part); + if !self.d.contains_suggestible(&c) { + return; + } + words.push(c); + // A name ranks after a common word at the same distance (`moze` → `może`, then + // `Mozę`). + cost += 1; + } + } + self.keep(words.join(" "), cost); + } + + /// Keep a suggestion known to be correct, unless the dictionary says never to offer it. + fn keep(&mut self, text: String, cost: u32) { + if !text.split(' ').all(|part| self.d.can_suggest(part)) { + return; + } + self.seq += 1; + let seq = self.seq; + let e = self.found.entry(text).or_insert((cost, seq)); + if cost < e.0 { + *e = (cost, seq); + } + } + + fn count_within(&self, cost: u32) -> usize { + self.found.values().filter(|(c, _)| *c <= cost).count() + } +} + +/// The MAP groups: the dictionary's, then the language's. +fn groups<'a>(d: &'a AffixDict, t: &Tuning) -> Cow<'a, [Vec]> { + if t.map.is_empty() { + return Cow::Borrowed(d.map()); + } + Cow::Owned(d.map().iter().cloned().chain(t.map.iter().map(|g| g.chars().collect())).collect()) +} + +/// The other members of every MAP group of `c` (its likely confusions), each once. +fn alternatives(c: char, map: &[Vec]) -> Vec { + let mut out: Vec = Vec::new(); + for x in map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()) { + if x != c && !out.contains(&x) { + out.push(x); + } + } + out +} + +/// Every way of changing up to `t.max_map_changes` letters of `chars` to a MAP relative (`alts`, +/// letter by letter), with its cost, each written after `buf`. +fn map_variants(chars: &[char], alts: &[Vec], t: &Tuning, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { + if out.len() >= MAX_MAP_VARIANTS { + return; + } + let Some((&c, rest)) = chars.split_first() else { + if changes > 0 { + out.push((buf.iter().collect(), cost)); + } + return; + }; + let (here, later) = alts.split_first().map_or((&[][..], &[][..]), |(h, l)| (h.as_slice(), l)); + buf.push(c); + map_variants(rest, later, t, changes, cost, buf, out); + buf.pop(); + if changes < t.max_map_changes { + for &a in here { + buf.push(a); + map_variants(rest, later, t, changes + 1, cost + (t.map_cost)(c, a), buf, out); + buf.pop(); + } + } +} + +/// A letter's MAP representative (the first of its first group), lower case: `ż` → `z`, `ó` → `o`. +pub(crate) fn fold(c: char, map: &[Vec]) -> char { + let l = c.to_lowercase().next().unwrap_or(c); + map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) +} + +/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other +/// edit 2; `limit + 1` once it is certainly above `limit`. +pub(crate) fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { + let m = b.len(); + let over = limit + 1; + let mut prev2 = vec![0u32; m + 1]; + let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); + let mut cur = vec![0u32; m + 1]; + for (i, &ai) in a.iter().enumerate() { + if let Some(c) = cur.get_mut(0) { + *c = (i as u32 + 1) * 2; + } + let mut row_min = u32::MAX; + for (j, &bj) in b.iter().enumerate() { + let sub = if ai == bj { + 0 + } else if related(ai, bj) { + 1 + } else { + 2 + }; + let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); + let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); + if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { + v = v.min(at(&prev2, j - 1) + 2); + } + if let Some(c) = cur.get_mut(j + 1) { + *c = v; + } + row_min = row_min.min(v); + } + if m > 0 && row_min > limit { + return over; + } + std::mem::swap(&mut prev2, &mut prev); + std::mem::swap(&mut prev, &mut cur); + } + prev.get(m).copied().unwrap_or(over).min(over) +} + +/// The cost of a missing space between `a` and `b`, or `None` when the split isn't offered. +fn split_cost(split: &Split, a: &str, b: &str) -> Option { + let long = |part: &str| part.chars().count() >= 3; + match split { + Split::Anywhere => Some(if long(a) && long(b) { COST_SPLIT } else { COST_SPLIT_SHORT }), + Split::Near { leading, trailing } => { + if (leading.contains(&a) && long(b)) || (long(a) && trailing.contains(&b)) { + Some(COST_SPLIT) + } else if long(a) && long(b) { + Some(COST_SPLIT_LONG) + } else { + None + } + } + Split::Custom(cost) => cost(a, b), + } +} + +/// Spelling suggestions for `word` from `d`, best first, at most `max`, in the word's letter case. +pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> Vec { + let word = word.trim(); + let n = word.chars().count(); + if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { + return Vec::new(); + } + let case = casing(word); + let input = word.to_lowercase(); + let chars: Vec = input.chars().collect(); + let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0 }; + add_case(&mut col, word, &input); + let map = groups(d, t); + add_map_slips(&mut col, &chars, &map, t); + add_confusions(&mut col, &input, t); + add_splits(&mut col, &input, &t.split); + add_key_neighbours(&mut col, &chars); + add_typos(&mut col, &chars, t); + // 6. Further off: scan the stems that start like the word. + if col.count_within(COST_MISTYPE) < max { + scan(&mut col, &chars, &map); + } + let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); + // Among equal costs, the change nearest the word's start is the likelier intent + // (the first changed letter breaks ties). + let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); + found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); + let mut out: Vec = Vec::new(); + for (text, _) in found { + let recased = match case { + Casing::Initial => capitalize(&text), + Casing::Upper if n > 1 => text.to_uppercase(), + _ => text.clone(), + }; + // A KEEPCASE word only exists in its dictionary case: `Km` would be flagged, so offer `km`. + let t = if recased != text && text.split(' ').any(|p| d.case_locked(p)) { text } else { recased }; + if t != word && !out.contains(&t) { + out.push(t); + } + if out.len() >= max { + break; + } + } + out +} + +/// Round 1: letter case. +fn add_case(col: &mut Collector, word: &str, input: &str) { + if col.d.contains(input) && word != input { + col.keep(input.to_string(), COST_CASE); + } else if col.d.contains(&capitalize(input)) && word != capitalize(input) { + col.keep(capitalize(input), COST_CASE); + } +} + +/// Round 2: accents and other MAP slips. +fn add_map_slips(col: &mut Collector, chars: &[char], map: &[Vec], t: &Tuning) { + let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); + let mut variants = Vec::new(); + map_variants(chars, &alts, t, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants); + variants.sort_by_key(|v| v.1); + for (v, cost) in &variants { + col.offer(v, *cost); + } +} + +/// Round 3: common confusions. +fn add_confusions(col: &mut Collector, input: &str, t: &Tuning) { + let rep = col.d.rep().iter().map(|(from, to)| (from.as_str(), to.as_str())).chain(t.rep.iter().copied()); + for (from, to) in rep { + if from.is_empty() { + continue; + } + for (i, _) in input.match_indices(from) { + if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { + col.offer(&format!("{a}{to}{b}"), t.rep_cost); + } + } + } +} + +/// Round 4: a missing space; both halves must be dictionary words. +fn add_splits(col: &mut Collector, input: &str, split: &Split) { + for (i, _) in input.char_indices().skip(1) { + let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; + if let Some(cost) = split_cost(split, a, b) { + col.offer(&format!("{a} {b}"), cost); + } + } +} + +/// Round 5: keyboard neighbours from the dictionary's KEY rows. +fn add_key_neighbours(col: &mut Collector, chars: &[char]) { + for row in col.d.key().split('|').take(32) { + let keys: Vec = row.chars().take(128).collect(); + for pair in keys.windows(2) { + let [a, b] = pair else { continue }; + for (i, &c) in chars.iter().enumerate() { + let replacement = if c == *a { + Some(*b) + } else if c == *b { + Some(*a) + } else { + None + }; + if let Some(replacement) = replacement { + let mut v = chars.to_vec(); + if let Some(x) = v.get_mut(i) { + *x = replacement; + } + col.offer(&v.iter().collect::(), COST_KEY); + } + } + } + } +} + +/// Round 5, second half: one typo, with the TRY letters (the language's when the dictionary has +/// none), bounded and each once, in their order: a TRY line lists the commonest letters first, so +/// equal-cost candidates come out in that order. +fn add_typos(col: &mut Collector, chars: &[char], t: &Tuning) { + let d = col.d; + let n = chars.len(); + let try_chars = if d.try_chars().is_empty() { t.alphabet } else { d.try_chars() }; + let mut letters: Vec = Vec::new(); + for c in try_chars.chars().filter(|c| c.is_lowercase()).take(128) { + if !letters.contains(&c) { + letters.push(c); + } + } + let s = |v: &[char]| v.iter().collect::(); + for i in 0..n.saturating_sub(1) { + if chars.get(i) != chars.get(i + 1) { + let mut v = chars.to_vec(); + v.swap(i, i + 1); + col.offer(&s(&v), COST_SWAP); + } + } + for i in 0..n { + let mut v = chars.to_vec(); + v.remove(i); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + for i in 0..=n { + for &c in &letters { + let mut v = chars.to_vec(); + v.insert(i, c); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + } + for i in 0..n { + for &c in &letters { + if chars.get(i) != Some(&c) { + let mut v = chars.to_vec(); + if let Some(x) = v.get_mut(i) { + *x = c; + } + col.offer(&s(&v), (t.mistype_cost)(chars, i, c)); + } + } + } +} + +/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either +/// case) by shared letter pairs, then keep the forms of the best within two typos. +fn scan(col: &mut Collector, chars: &[char], map: &[Vec]) { + let d = col.d; + let Some(&first) = chars.first() else { return }; + let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); + let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); + let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); + let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); + firsts.extend(upper); + firsts.sort_unstable(); + firsts.dedup(); + let n = folded.len(); + let mut best: Vec<(u32, &str)> = Vec::new(); + for f in firsts { + let prefix = f.to_string(); + for stem in d.stems_with_prefix(&prefix) { + let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); + let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); + if shared < 2 { + continue; + } + // Shared pairs, less the stem's letters the word doesn't explain, less a length gap + // beyond what an ending explains. + let extra = sf.len().saturating_sub(shared); + let gap = sf.len().abs_diff(n).saturating_sub(4); + let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; + if score == 0 { + continue; + } + if best.len() < SCAN_STEMS { + best.push((score, stem)); + } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) + && score > min.0 + { + *min = (score, stem); + } + } + } + best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); + let related = |a: char, b: char| fold(a, map) == fold(b, map); + let limit = 4; + for (_, stem) in best { + for form in d.forms(stem) { + let lf: Vec = form.to_lowercase().chars().collect(); + if lf.len().abs_diff(chars.len()) > 2 { + continue; + } + let dist = weighted_distance(&lf, chars, &related, limit); + if dist <= limit && lf.as_slice() != chars { + col.keep(form, COST_SCAN + dist * 3); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() { + let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap(); + let suggestions = suggest_in(&d, "počítačz", 8, &Tuning::DEFAULT); + assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}"); + assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}"); + } + + /// A candidate offered again at a lower cost (a REP confusion after a MAP diacritic drop) + /// ranks at the lower one. + #[test] + fn a_cheaper_later_offer_wins() { + let d = AffixDict::from_hunspell("", "1\ncafe\n", &[]).unwrap(); + let mut col = Collector { d: &d, input: "cafè".into(), tested: HashMap::new(), found: HashMap::new(), seq: 0 }; + col.offer("cafe", COST_DROP_DIACRITIC); + col.offer("cafe", COST_REP); + col.offer("cafe", COST_SCAN); + assert_eq!(col.found.get("cafe").map(|e| e.0), Some(COST_REP)); + } + + #[test] + fn distance_weighs_map_slips_half() { + let map = vec![vec!['a', 'ą'], vec!['o', 'ó', 'u']]; + let related = |a: char, b: char| fold(a, &map) == fold(b, &map); + let c = |s: &str| s.chars().collect::>(); + assert_eq!(weighted_distance(&c("ktury"), &c("który"), &related, 9), 1); + assert_eq!(weighted_distance(&c("kot"), &c("kto"), &related, 9), 2); + assert_eq!(weighted_distance(&c("kot"), &c("kotek"), &related, 9), 4); + assert_eq!(weighted_distance(&c("kot"), &c("pies"), &related, 2), 3); + assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4); + } + + /// A KEEPCASE word is offered in its dictionary case when the word's own case would be flagged + /// (`Kmm` → `km`, never `Km`), and a capitalised KEEPCASE word gets its dictionary form. + #[test] + fn keepcase_suggestions_are_not_recased_into_errors() { + let d = AffixDict::from_hunspell("KEEPCASE K\n", "2\nkm/K\npies\n", &[]).unwrap(); + assert!(!d.check("Km") && !d.check("KM")); + for (word, want) in [("Kmm", "km"), ("KMM", "km"), ("Km", "km")] { + let got = suggest_in(&d, word, 4, &Tuning::DEFAULT); + assert!(got.iter().all(|s| s.split(' ').all(|p| d.check(p))), "{word}: {got:?}"); + assert_eq!(got.first().map(String::as_str), Some(want), "{word}: {got:?}"); + } + assert_eq!(suggest_in(&d, "Pies", 4, &Tuning::DEFAULT), Vec::::new()); + assert_eq!(suggest_in(&d, "Piesx", 4, &Tuning::DEFAULT).first().map(String::as_str), Some("Pies")); + } + + /// A language's tables come after its dictionary's, a TRY alphabet stands in for a missing TRY + /// line, and a split rule decides which missing spaces are offered. + #[test] + fn tuning_adds_to_the_dictionary_tables() { + let d = AffixDict::from_hunspell("", "3\nčaj\nkot\nna\n", &[]).unwrap(); + let tuned = Tuning { map: &["cč"], alphabet: "t", rep: &[("xy", "kot")], ..Tuning::DEFAULT }; + assert!(!suggest_in(&d, "caj", 4, &Tuning::DEFAULT).contains(&"čaj".into())); + assert!(suggest_in(&d, "caj", 4, &tuned).contains(&"čaj".into()), "the language's MAP group"); + assert!(suggest_in(&d, "xy", 4, &tuned).contains(&"kot".into()), "the language's REP pair"); + assert!(suggest_in(&d, "ko", 4, &tuned).contains(&"kot".into()), "the language's TRY letters"); + let near = Tuning { split: Split::Near { leading: &["na"], trailing: &[] }, ..Tuning::DEFAULT }; + assert!(suggest_in(&d, "nakot", 4, &near).contains(&"na kot".into())); + assert!(!suggest_in(&d, "kotna", 4, &near).contains(&"kot na".into()), "no stray short half"); + assert!(suggest_in(&d, "kotna", 4, &Tuning::DEFAULT).contains(&"kot na".into())); + } +} diff --git a/crates/proof/src/bg.rs b/crates/proof/src/bg.rs new file mode 100644 index 00000000..6b456d8c --- /dev/null +++ b/crates/proof/src/bg.rs @@ -0,0 +1,112 @@ +//! Bulgarian hyphenation by rule ([`ProofLang::Bg`](crate::ProofLang::Bg)). +//! +//! Bulgarian spelling splits words at syllables, and its rules are regular enough to apply +//! directly, so no pattern file or word list is needed (and none is borrowed): +//! - two vowels next to each other may be split (`на-у-ка`); +//! - one consonant between vowels goes with the next syllable (`ма-ма`, `ро-за`); +//! - of two or more consonants between vowels, the first stays with the syllable before and the +//! rest go with the next (`сес-тра`, `въз-дух`, `ком-пю-тър`) — the spelling rules allow any +//! split of a consonant group, and this is the usual one — except that a consonant followed by +//! `р` or `л` starts the next syllable together with it (`до-бре`, `ко-пре`, `та-бли-ца`); +//! - `й` before a consonant stays with the vowel before it (`строй-ка`, `май-ка`); +//! - `ьо` is never split, and the consonant before `ь` goes with it (`шо-фьор`); +//! - `дж` and `дз`, one sound each, are never split (`хо-джа`). +//! +//! A single letter left at the end or start of a line is ruled out by the paragraph's +//! [`Limits`](crate::hyphen::Limits) (`Бъл-га-рия`, not `Бъл-га-ри-я`). + +fn is_vowel(c: char) -> bool { + matches!(c, 'а' | 'е' | 'и' | 'о' | 'у' | 'ъ' | 'ю' | 'я' | 'ѝ') +} + +/// Consonants that may start a syllable before `р` or `л` (`бр`, `пл`, `тр`, `гл` …). +fn is_obstruent(c: char) -> bool { + matches!(c, 'б' | 'в' | 'г' | 'д' | 'з' | 'к' | 'п' | 'с' | 'т' | 'ф' | 'х') +} + +/// A lowercase Bulgarian letter (the words [`points`] splits). +pub(crate) fn is_letter(c: char) -> bool { + matches!(c, 'а'..='я' | 'ѝ') +} + +/// Break points of one lowercase Bulgarian word (char indices; a hyphen goes before that char). +/// Empty for anything that is not entirely Bulgarian letters. +pub fn points(word: &[char]) -> Vec { + if word.len() < 2 || !word.iter().all(|&c| is_letter(c)) { + return Vec::new(); + } + let vowels: Vec = (0..word.len()).filter(|&i| is_vowel(word[i])).collect(); + let mut out = Vec::new(); + for pair in vowels.windows(2) { + let (v1, v2) = (pair[0], pair[1]); + let mut pt = match v2 - v1 - 1 { + 0 | 1 => v1 + 1, + _ => v1 + 2, + }; + // A consonant with `р` or `л` after it starts a syllable together with it. + if v2 - v1 > 2 && matches!(word[v2 - 1], 'р' | 'л') && is_obstruent(word[v2 - 2]) { + pt = v2 - 2; + } + // `дж`, `дз`: one sound, so both go to the next syllable. + if pt > v1 + 1 && word[pt - 1] == 'д' && matches!(word[pt], 'ж' | 'з') { + pt -= 1; + } + // `ь` keeps the consonant before it (`фьо`). + if pt < v2 && word[pt] == 'ь' && pt > v1 + 1 { + pt -= 1; + } + if pt > v1 && pt <= v2 && word[pt - 1] != 'ь' { + out.push(pt); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn split(w: &str) -> String { + let chars: Vec = w.chars().collect(); + let pts = points(&chars); + let mut s = String::new(); + for (i, c) in chars.iter().enumerate() { + if pts.contains(&i) { + s.push('-'); + } + s.push(*c); + } + s + } + + #[test] + fn syllables_follow_the_bulgarian_rules() { + for (word, expected) in [ + ("мама", "ма-ма"), + ("наука", "на-у-ка"), + ("сестра", "сес-тра"), + ("въздух", "въз-дух"), + ("компютър", "ком-пю-тър"), + ("стройка", "строй-ка"), + ("майка", "май-ка"), + ("шофьор", "шо-фьор"), + ("ходжа", "хо-джа"), + ("здравей", "здра-вей"), + ("плевен", "пле-вен"), + ("българия", "бъл-га-ри-я"), + ("сричкопренасяне", "срич-ко-пре-на-ся-не"), + ("добре", "до-бре"), + ("таблица", "та-бли-ца"), + ("отговор", "от-го-вор"), + ] { + assert_eq!(split(word), expected, "{word}"); + } + } + + #[test] + fn non_bulgarian_letters_are_left_alone() { + assert!(points(&"word".chars().collect::>()).is_empty()); + assert!(points(&"съобщение1".chars().collect::>()).is_empty()); + assert!(points(&"объём".chars().collect::>()).is_empty(), "Russian ё is not a Bulgarian letter"); + } +} diff --git a/crates/proof/src/cs/grammar.rs b/crates/proof/src/cs/grammar.rs new file mode 100644 index 00000000..26a14fcd --- /dev/null +++ b/crates/proof/src/cs/grammar.rs @@ -0,0 +1,103 @@ +//! Generic repeated-word and spacing checks with Czech messages. +//! No Czech-specific grammar, comma or sentence-capital rules are inferred. + +use crate::{Issue, IssueKind, ProofLang, words}; + +fn issue(start: usize, end: usize, message: String, suggestions: Vec) -> Issue { + Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::Cs } +} + +pub fn check(text: &str) -> Vec { + let mut issues = Vec::new(); + repeated_words(text, &words(text), &mut issues); + spaces(text, &mut issues); + issues.sort_by_key(|i| (i.start, i.end)); + issues +} + +fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if !gap.is_empty() && gap.chars().all(char::is_whitespace) && w0.chars().all(char::is_alphabetic) && w0.to_lowercase() == w1.to_lowercase() { + v.push(issue(*a0, *b1, format!("Opakované slovo: „{w1}“"), vec![w0.to_string()])); + } + } +} + +fn spaces(text: &str, v: &mut Vec) { + for (i, c) in text.char_indices() { + if c != ' ' { + continue; + } + let rest = text.get(i + 1..).unwrap_or(""); + let mut it = rest.chars(); + let Some(p) = it.next() else { continue }; + let after = it.next(); + // A ratio/score keeps the spaces in digit-space-colon-space-digit. + let ratio = p == ':' + && text.get(..i).and_then(|before| before.chars().next_back()).is_some_and(|c| c.is_ascii_digit()) + && after == Some(' ') + && it.next().is_some_and(|c| c.is_ascii_digit()); + let flag = !ratio + && match p { + ',' => true, + // `;)` and `;-)` are emoticons. + ';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')), + // ` ...` is an ellipsis, ` .5` a number. + '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), + // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. + ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')), + _ => false, + }; + if flag { + v.push(issue(i, i + 1 + p.len_utf8(), "Mezera před interpunkcí".into(), vec![p.to_string()])); + } + } + for (i, _) in text.match_indices(" ") { + if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { + continue; + } + v.push(issue(i, i + 2, "Dvojitá mezera".into(), vec![" ".into()])); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn repeated_words_and_spaces_have_czech_messages() { + let text = "Město město , škola."; + let issues = check(text); + assert_eq!(issues.len(), 3, "{issues:?}"); + assert_eq!(issues[0].message, "Opakované slovo: „město“"); + assert_eq!(issues[0].suggestions, ["Město"]); + assert!(issues.iter().any(|i| i.message == "Mezera před interpunkcí" && &text[i.start..i.end] == " ,")); + assert!(issues.iter().any(|i| i.message == "Dvojitá mezera")); + assert!(issues.iter().all(|i| i.lang == ProofLang::Cs && i.kind == IssueKind::Grammar)); + for text in ["město. škola", "Dr. Novák", "A apple", "Město, město!", "Úsměv :) Číslo .5 a tečky ..."] { + assert!(check(text).is_empty(), "{text}"); + } + } + + #[test] + fn ratios_and_scores_keep_their_spaces() { + for text in ["3 : 1", "1 : 2", "Skóre 12 : 10.", "Poměr 1 : 100."] { + assert!(check(text).is_empty(), "{text}: {:?}", check(text)); + } + for text in ["Slovo : další", "3 : slovo", "slovo : 1", "3 :"] { + assert!(check(text).iter().any(|i| i.message == "Mezera před interpunkcí"), "{text}"); + } + } + + proptest::proptest! { + #[test] + fn never_panics_and_ranges_are_valid(s in "\\PC{0,80}") { + for issue in check(&s) { + proptest::prop_assert!(issue.start <= issue.end && issue.end <= s.len()); + proptest::prop_assert!(s.is_char_boundary(issue.start) && s.is_char_boundary(issue.end)); + } + } + } +} diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs new file mode 100644 index 00000000..9f8bbd0a --- /dev/null +++ b/crates/proof/src/cs/mod.rs @@ -0,0 +1,201 @@ +//! Czech proofing using the installed Hunspell `cs_CZ` dictionary (never bundled). +//! Suggestions use its runtime MAP/REP/TRY/KEY tables. Generic grammar checks have Czech +//! messages; hyphenation keeps the existing fallback, without Czech pattern data. + +pub mod grammar; +pub(crate) mod suggest; +#[cfg(test)] +pub(crate) mod test_dict; + +use std::sync::Arc; + +use crate::affix::AffixDict; +use crate::slot::DictSlot; + +pub use suggest::suggest; + +/// File names of a Czech Hunspell dictionary. +const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"]; + +/// The Czech dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Czech"); + +/// The Czech dictionary: the installed Hunspell `cs_CZ` (read on first use), one given with +/// [`install_dictionary`], or an empty one when none is available (Czech spelling is off +/// then; see [`has_dictionary`]). +pub fn dictionary() -> Arc { + DICT.get() +} + +/// Use this Czech dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to +/// UTF-8), for example one the user picked. On error the current dictionary stays. +pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { + DICT.install(aff, dic) +} + +/// Is a Czech dictionary available (spelling is checked)? +pub fn has_dictionary() -> bool { + DICT.has_dictionary() +} + +/// Is `word` spelled correctly in Czech? Numbers, single letters, short words in capitals +/// (acronyms: `ČR`, `EU`), words with digits and words in other scripts are accepted, and so +/// is every word when no Czech dictionary is installed. +pub fn is_correct(word: &str) -> bool { + is_correct_with(&dictionary(), word) +} + +/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off, +/// short words in capitals are checked against the dictionary too, and with Ignore words that +/// contain numbers off, so are words with digits. +pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool { + check(&dictionary(), word, opts) +} + +/// [`is_correct`] against a given dictionary (an empty one accepts everything). +pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { + check(d, word, &crate::Options::default()) +} + +/// [`is_correct_with_options`] against a given dictionary. +fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { + return true; + }; + if w.contains('-') { + return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + } + known(d, w) +} + +/// Is one word (no hyphens) in the dictionary or the user dictionary? +fn known(d: &AffixDict, w: &str) -> bool { + d.check(w) || crate::in_user_dictionary(&w.to_lowercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dictionary_lookup_accepts_czech_file_names() { + let dir = std::env::temp_dir().join(format!("wordcraft-cs-aliases-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + for name in ["cs_CZ", "cs-CZ", "cs"] { + std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap(); + std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap(); + let (aff, dic) = crate::hunspell::find_in(&dir, NAMES).unwrap(); + let d = crate::hunspell::load(&aff, &dic).unwrap(); + assert!(is_correct_with(&d, "městě")); + std::fs::remove_file(aff).unwrap(); + std::fs::remove_file(dic).unwrap(); + } + std::fs::remove_dir(dir).unwrap(); + } + + #[test] + fn czech_words_affixes_chains_case_and_diacritics() { + test_dict::install(); + let d = dictionary(); + assert!(has_dictionary()); + assert_eq!(d.len(), 20); + assert_eq!(d.rule_count(), 8); + for w in [ + "město", + "města", + "městě", + "neměsta", + "rychle", + "rychlejší", + "učitel", + "učitelé", + "Praha", + "Praze", + "Město", + "POČÍTAČ", + "příliš", + "žluťoučký", + "kůň", + "úpěl", + "ďábelské", + "ódy", + "děkujeme", + "zpráva", + "zprava", + "škola-počítač", + ] { + assert!(is_correct(w), "{w}"); + } + for w in ["městu", "městoě", "rychleejší", "učiteléé", "praha", "mĚsto", "pocitac", "dekujem", "škola-pocitac"] { + assert!(!is_correct(w), "{w}"); + } + for c in "ěščřžýáíéůú".chars() { + assert!(test_dict::DIC.contains(c), "UTF-8 fixture covers {c}"); + } + } + + #[test] + fn forbidden_words_are_rejected_in_every_case() { + test_dict::install(); + for w in ["zqzakázané", "Zqzakázané", "ZQZAKÁZANÉ"] { + assert!(!is_correct(w), "{w}"); + } + } + + #[test] + fn missing_dictionary_and_ignored_inputs_are_quiet() { + let empty = AffixDict::default(); + assert!(is_correct_with(&empty, "pocitac")); + assert!(install_dictionary("FLAG long\n", "1\nstrom\n").is_err()); + let d = AffixDict::from_hunspell(test_dict::AFF, test_dict::DIC, &[]).unwrap(); + for w in ["", "a", "2026", "ČR", "článek2", "日本語"] { + assert!(is_correct_with(&d, w), "{w}"); + } + } + + #[test] + fn user_dictionary_applies() { + test_dict::install(); + assert!(!is_correct("zqčeskévlastníslovo")); + crate::add_word("Zqčeskévlastníslovo"); + assert!(is_correct("zqčeskévlastníslovo")); + } + + #[test] + fn czech_dispatch_messages_options_and_mixed_spans() { + test_dict::install(); + assert!(crate::has_dictionary(Some("cs-CZ"))); + assert!(crate::is_correct_for("městě", Some("cze"))); + let text = "Město pocitac"; + let issues = crate::check_text(text, &[(0..text.len(), crate::ProofLang::Cs)]); + assert_eq!(issues.len(), 1, "{issues:?}"); + assert_eq!(&text[issues[0].start..issues[0].end], "pocitac"); + assert_eq!(issues[0].message, "Možná pravopisná chyba"); + assert_eq!(issues[0].lang, crate::ProofLang::Cs); + assert!(crate::suggest_in("pocitac", 8, crate::ProofLang::Cs).contains(&"počítač".into())); + let text = "Město this is is wrong"; + let issues = crate::check_text(text, &[(0..6, crate::ProofLang::Cs), (7..text.len(), crate::ProofLang::En)]); + assert!(issues.iter().any(|i| i.lang == crate::ProofLang::En && i.message.starts_with("Repeated word"))); + assert!(issues.iter().all(|i| i.start >= 7)); + let opts = crate::Options { ignore_uppercase: false, ..Default::default() }; + assert!(!crate::is_correct_in_with("ZQZAKÁZANÉ", crate::ProofLang::Cs, &opts)); + } + + /// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Czech checks + /// those words against the dictionary instead of accepting them. + #[test] + fn proofing_options_reach_the_czech_check() { + test_dict::install(); + let lang = crate::ProofLang::Cs; + let defaults = crate::Options::default(); + let check_caps = crate::Options { ignore_uppercase: false, ..defaults }; + let check_digits = crate::Options { ignore_numbers: false, ..defaults }; + assert!(crate::is_correct_in_with("ŠKULA", lang, &defaults), "short capitals are acronyms by default"); + assert!(!crate::is_correct_in_with("ŠKULA", lang, &check_caps)); + assert!(crate::is_correct_in_with("ŠKOLA", lang, &check_caps)); + assert!(crate::is_correct_in_with("škula2", lang, &defaults)); + assert!(!crate::is_correct_in_with("škula2", lang, &check_digits)); + assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); + assert!(is_correct("ŠKULA"), "direct checks keep the defaults"); + } +} diff --git a/crates/proof/src/cs/suggest.rs b/crates/proof/src/cs/suggest.rs new file mode 100644 index 00000000..05eba74a --- /dev/null +++ b/crates/proof/src/cs/suggest.rs @@ -0,0 +1,78 @@ +//! Czech suggestions: the shared ranking ([`crate::affix_suggest`]) with the installed +//! dictionary's own MAP/REP/TRY/KEY tables and nothing Czech hard-coded. + +use crate::affix::AffixDict; +use crate::affix_suggest::Tuning; + +/// Spelling suggestions for a Czech word, best first, at most `max`, in the word's letter case. +pub fn suggest(word: &str, max: usize) -> Vec { + suggest_in(&super::dictionary(), word, max) +} + +/// [`suggest`] against a given dictionary. +pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { + crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn suggestions_are_fast_enough() { + crate::cs::test_dict::install(); + let _ = suggest("město", 1); + let t = std::time::Instant::now(); + for word in ["zqčeskévelmidlouhéslovo", "pocitacz", "nejrychllejší"] { + let _ = suggest(word, 6); + } + let ms = t.elapsed().as_secs_f64() * 1000.0; + eprintln!("3 Czech suggestion calls: {ms:.0} ms"); + assert!(ms < 20_000.0, "{ms} ms"); + } + + #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); + } + + #[test] + fn czech_suggestions_use_runtime_tables_and_case() { + crate::cs::test_dict::install(); + for (word, want) in [ + ("mesto", "město"), + ("mňe", "mě"), + ("xůž", "nůž"), + ("škloa", "škola"), + ("pocitac", "počítač"), + ("Mesto", "Město"), + ("praha", "Praha"), + ("toškola", "to škola"), + ] { + assert!(suggest(word, 8).contains(&want.into()), "{word}: {:?}", suggest(word, 8)); + } + // Each table changes the answer without a hard-coded Czech alphabet or confusion list. + for (table, wrong, right) in + [("MAP 1\nMAP cč\n", "caj", "čaj"), ("REP 1\nREP xy město\n", "xy", "město"), ("TRY š\n", "kola", "škola"), ("KEY xn\n", "xůž", "nůž")] + { + let dic = format!("1\n{right}\n"); + let d = AffixDict::from_hunspell(table, &dic, &[]).unwrap(); + assert!(suggest_in(&d, wrong, 4).contains(&right.into()), "{table}"); + let without = AffixDict::from_hunspell("", &dic, &[]).unwrap(); + assert!(!suggest_in(&without, wrong, 4).contains(&right.into()), "must depend on {table}"); + } + } + + #[test] + fn suggestions_stay_bounded_and_skip_forbidden_words() { + crate::cs::test_dict::install(); + for w in ["", "dvě slova", &"ě".repeat(41)] { + assert!(suggest(w, 4).is_empty()); + } + assert!(suggest("mesto", 0).is_empty()); + assert!(suggest_in(&AffixDict::default(), "mesto", 4).is_empty()); + assert!(suggest("město", 4).len() <= 4); + assert!(!suggest("město", 4).contains(&"město".into())); + assert!(!suggest("zqzakázane", 8).contains(&"zqzakázané".into())); + } +} diff --git a/crates/proof/src/cs/test_dict.rs b/crates/proof/src/cs/test_dict.rs new file mode 100644 index 00000000..6cc59759 --- /dev/null +++ b/crates/proof/src/cs/test_dict.rs @@ -0,0 +1,59 @@ +//! Hand-written synthetic Czech test data. No words, rules or suggestion tables were copied +//! from a system dictionary. This is a small fixture, not a redistributable Czech dictionary. + +pub const AFF: &str = "SET UTF-8 +FLAG UTF-8 +FORBIDDENWORD F +TRY aeiouěščřžýáíéůúntklmp +KEY xn|qř +MAP 5 +MAP eěé +MAP cč +MAP rř +MAP uůú +MAP ií +REP 1 +REP mňe mě +SFX A Y 2 # synthetic declension +SFX A o a o +SFX A o ě o +SFX B N 1 +SFX B 0 e/C . +SFX C N 1 +SFX C e ejší e +SFX D N 1 +SFX D 0 itel/E . +SFX E N 1 +SFX E 0 é . +SFX P Y 1 +SFX P ha ze ha +PFX X Y 1 +PFX X 0 ne . +"; + +pub const DIC: &str = "20 +město/AX +rychl/B +uč/D +Praha/P +škola +počítač +příliš +žluťoučký +kůň +úpěl +ďábelské +ódy +zpráva +zprava +děkujeme +mě +nůž +si +to +zqzakázané/F +"; + +pub fn install() { + super::install_dictionary(AFF, DIC).unwrap(); +} diff --git a/crates/proof/src/dict.rs b/crates/proof/src/dict.rs new file mode 100644 index 00000000..3bcedb0b --- /dev/null +++ b/crates/proof/src/dict.rs @@ -0,0 +1,195 @@ +//! Hyphenation exception dictionary: lowercase words with their break points. +//! +//! On-disk format (craft-i18n's `language-data/en-us/en-us.dic`): raw-deflate of +//! `"DCHYPH1\n"` followed by one line per word, sorted by the plain word's bytes. Each line is +//! front-coded against the previous one: a byte with the shared prefix length (in bytes of the +//! hyphenated form), then the rest of the hyphenated form (`-` at every break), then `\n`. + +/// Sorted word list with a break bitmask per word (bit `i` = hyphen before char `i`). +#[derive(Clone, Debug, Default)] +pub struct Dictionary { + words: String, + offs: Vec, + masks: Vec, +} + +const MAGIC: &[u8] = b"DCHYPH1\n"; +/// Largest inflated dictionary, and most bytes of words kept from one (the bundled US English one +/// is about 1 MB). +const MAX_BYTES: usize = 64 << 20; +/// Most words read from one dictionary (the bundled one has about 160k). +const MAX_ENTRIES: usize = 4_000_000; + +impl Dictionary { + /// Build from `(lowercase word, break mask)` pairs (sorted and de-duplicated here; the first + /// entry of a duplicate wins). + pub fn from_entries(mut entries: Vec<(String, u64)>) -> Dictionary { + entries.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes())); + entries.dedup_by(|b, a| a.0 == b.0); + let mut d = Dictionary { words: String::new(), offs: vec![0], masks: Vec::with_capacity(entries.len()) }; + for (w, m) in entries { + d.words.push_str(&w); + d.offs.push(d.words.len() as u32); + d.masks.push(m); + } + d + } + + pub fn len(&self) -> usize { + self.masks.len() + } + + pub fn is_empty(&self) -> bool { + self.masks.is_empty() + } + + fn word(&self, i: usize) -> &str { + &self.words[self.offs[i] as usize..self.offs[i + 1] as usize] + } + + /// Break mask of an exact (already lowercased) word. + pub fn get(&self, word: &str) -> Option { + let (mut lo, mut hi) = (0usize, self.len()); + while lo < hi { + let mid = (lo + hi) / 2; + match self.word(mid).as_bytes().cmp(word.as_bytes()) { + std::cmp::Ordering::Less => lo = mid + 1, + std::cmp::Ordering::Greater => hi = mid, + std::cmp::Ordering::Equal => return Some(self.masks[mid]), + } + } + None + } + + /// Iterate `(word, mask)`. + pub fn iter(&self) -> impl Iterator + '_ { + (0..self.len()).map(|i| (self.word(i), self.masks[i])) + } + + /// Serialize (deflate-compressed). + pub fn to_bytes(&self) -> Vec { + let mut raw = MAGIC.to_vec(); + let mut prev: Vec = Vec::new(); + for (w, m) in self.iter() { + let h = hyphenated(w, m).into_bytes(); + let p = prev.iter().zip(&h).take_while(|(a, b)| a == b).count().min(255); + raw.push(p as u8); + raw.extend_from_slice(&h[p..]); + raw.push(b'\n'); + prev = h; + } + miniz_oxide::deflate::compress_to_vec(&raw, 10) + } + + /// Deserialize data written by [`Dictionary::to_bytes`]. Data that inflates past + /// [`MAX_BYTES`], or holds more than [`MAX_ENTRIES`] words or [`MAX_BYTES`] of them, is refused. + pub fn from_bytes(data: &[u8]) -> Result { + Dictionary::from_bytes_within(data, MAX_BYTES, MAX_ENTRIES) + } + + /// [`Dictionary::from_bytes`] with the limits given. + fn from_bytes_within(data: &[u8], max_bytes: usize, max_entries: usize) -> Result { + let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(data, max_bytes) + .map_err(|e| format!("hyphenation dictionary: inflate failed: {:?}", e.status))?; + let body = raw.strip_prefix(MAGIC).ok_or("hyphenation dictionary: bad header")?; + let mut d = Dictionary { words: String::with_capacity(body.len()), offs: vec![0], masks: Vec::new() }; + let mut cur: Vec = Vec::new(); + let mut i = 0; + while i < body.len() { + let p = body[i] as usize; + let end = body[i + 1..].iter().position(|&b| b == b'\n').map(|e| i + 1 + e).ok_or("hyphenation dictionary: truncated")?; + if p > cur.len() { + return Err("hyphenation dictionary: bad prefix".into()); + } + cur.truncate(p); + cur.extend_from_slice(&body[i + 1..end]); + i = end + 1; + let s = std::str::from_utf8(&cur).map_err(|_| "hyphenation dictionary: bad utf-8")?; + let (w, m) = parse_hyphenated(s); + // Front coding lets a short line repeat a long prefix: bound what is kept, not only + // what is read. + if d.masks.len() >= max_entries || d.words.len() + w.len() > max_bytes { + return Err("hyphenation dictionary: too large".into()); + } + d.words.push_str(&w); + d.offs.push(d.words.len() as u32); + d.masks.push(m); + } + Ok(d) + } +} + +/// `word` with `-` before every char whose bit is set in `mask`. +pub fn hyphenated(word: &str, mask: u64) -> String { + let mut s = String::with_capacity(word.len() + 8); + for (i, c) in word.chars().enumerate() { + if i > 0 && i < 64 && mask & (1 << i) != 0 { + s.push('-'); + } + s.push(c); + } + s +} + +/// Inverse of [`hyphenated`]. +pub fn parse_hyphenated(s: &str) -> (String, u64) { + let mut w = String::with_capacity(s.len()); + let mut m = 0u64; + let mut n = 0; + for c in s.chars() { + if c == '-' { + if n > 0 && n < 64 { + m |= 1 << n; + } + } else { + w.push(c); + n += 1; + } + } + (w, m) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn roundtrip() { + let e: Vec<(String, u64)> = ["ty-pog-ra-phy", "cat", "hap-pen", "o'clock", "naïve-té", "naïve"].iter().map(|s| parse_hyphenated(s)).collect(); + let d = Dictionary::from_entries(e); + let back = Dictionary::from_bytes(&d.to_bytes()).unwrap(); + assert_eq!(back.len(), 6); + assert_eq!(back.get("typography").map(|m| hyphenated("typography", m)), Some("ty-pog-ra-phy".to_string())); + assert_eq!(back.get("cat"), Some(0)); + assert_eq!(back.get("naïveté").map(|m| hyphenated("naïveté", m)), Some("naïve-té".to_string())); + assert!(back.get("dog").is_none()); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn oversized_data_is_refused() { + const LIMIT: usize = 1 << 16; + let within = |raw: &[u8]| Dictionary::from_bytes_within(&miniz_oxide::deflate::compress_to_vec(raw, 6), LIMIT, 1000); + // Data that inflates past the limit is refused before anything is decoded. + let mut raw = MAGIC.to_vec(); + raw.resize(LIMIT + 1, b'a'); + assert!(within(&raw).unwrap_err().contains("inflate failed")); + // Short lines that each repeat a long prefix: small to inflate, but the words they spell + // out would be ~100x larger. Kept storage is bounded too. + let mut raw = MAGIC.to_vec(); + raw.push(0); + raw.extend_from_slice(&[b'a'; 255]); + raw.push(b'\n'); + for _ in 0..(LIMIT / 255 + 2) { + raw.extend_from_slice(&[255, b'\n']); + } + assert!(raw.len() < LIMIT); + assert!(within(&raw).unwrap_err().contains("too large")); + // So is the number of words. + let words: Vec<(String, u64)> = (0..1001).map(|i| (format!("w{i}"), 0)).collect(); + let raw = miniz_oxide::inflate::decompress_to_vec(&Dictionary::from_entries(words).to_bytes()).unwrap(); + assert!(within(&raw).unwrap_err().contains("too large")); + // The real limits leave the bundled dictionary whole. + assert!(Dictionary::en_us().len() > 100_000); + } +} diff --git a/crates/proof/src/en_variants.rs b/crates/proof/src/en_variants.rs new file mode 100644 index 00000000..07c16bb1 --- /dev/null +++ b/crates/proof/src/en_variants.rs @@ -0,0 +1,278 @@ +//! American and British spellings of the same English word (#1281, #1045): which forms each +//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`]) accepts, which it flags, +//! and what it suggests instead (`colour` → `color` in American English). +//! +//! The pairs come from craft-i18n's `language-data/en-us/en-variants.txt`, our own list of base forms with a +//! policy per English. Inflections are generated here by rule for the regular patterns (-or/-our, +//! -er/-re, -ize/-ise, -ense/-ence…), so `colours`, `centred` and `organisation` follow their +//! base forms. A form is right when the shared English word list knows it or knows its other +//! spelling (`realisation` is right in British English because `realization` is a word), so the +//! rules never make a misspelling right. + +use std::collections::HashMap; +use std::sync::OnceLock; + +use crate::ProofLang; + +/// Our list of American/British pairs: `POLICY AMERICAN BRITISH` per line, `#` comments. +use crate::data::EN_VARIANTS as VARIANTS; + +/// What one English does with a pair's two spellings. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Policy { + /// Only the American spelling is right. + American, + /// Only the British spelling is right. + British, + /// Both are right. + Both, +} + +impl Policy { + fn from_char(c: char) -> Option { + match c { + 'u' => Some(Policy::American), + 'b' => Some(Policy::British), + '*' => Some(Policy::Both), + _ => None, + } + } +} + +/// One line of the list: the policies for American, Canadian and British-spelling English, and +/// the two base forms. +struct Pair { + policy: [Policy; 3], + american: &'static str, + british: &'static str, +} + +/// The lines of [`VARIANTS`]; malformed lines are skipped (a test keeps the file clean). +fn pairs() -> &'static [Pair] { + static P: OnceLock> = OnceLock::new(); + P.get_or_init(|| VARIANTS.lines().filter_map(parse_line).collect()) +} + +fn parse_line(line: &'static str) -> Option { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + return None; + } + let mut parts = line.split_whitespace(); + let (policy, american, british) = (parts.next()?, parts.next()?, parts.next()?); + if parts.next().is_some() || american == british { + return None; + } + let mut p = policy.chars().map(Policy::from_char); + let policy = [p.next()??, p.next()??, p.next()??]; + if p.next().is_some() { + return None; + } + let word = |w: &str| !w.is_empty() && w.chars().all(|c| c.is_ascii_lowercase()); + (word(american) && word(british)).then_some(Pair { policy, american, british }) +} + +/// The column of a pair's policy for `lang`; `None` for a language that isn't English. +fn column(lang: ProofLang) -> Option { + match lang { + ProofLang::En => Some(0), + ProofLang::EnCa => Some(1), + ProofLang::EnGb => Some(2), + _ => None, + } +} + +/// `word` with `suffix`: a final `e` drops before a vowel (`centre` + `ing` → `centring`), a +/// final consonant + `y` becomes `i` (`cosy` + `er` → `cosier`), `s` is the plural (`es` after a +/// sibilant, `ies` after a consonant + `y`). +fn add(word: &str, suffix: &str) -> String { + let b = word.as_bytes(); + let last = b.last().copied().unwrap_or(b'_'); + let before = b.len().checked_sub(2).and_then(|i| b.get(i)).copied().unwrap_or(b'_'); + let vowel = |c: u8| b"aeiou".contains(&c); + let head = word.get(..word.len().saturating_sub(1)).unwrap_or(word); + if suffix == "s" { + if last == b'y' && !vowel(before) { + return format!("{head}ies"); + } + if matches!(last, b's' | b'x' | b'z') || word.ends_with("ch") || word.ends_with("sh") { + return format!("{word}es"); + } + return format!("{word}s"); + } + let suffix_vowel = suffix.as_bytes().first().is_some_and(|&c| vowel(c)); + if last == b'e' && suffix_vowel { + return format!("{head}{suffix}"); + } + if last == b'y' && !vowel(before) && !suffix.starts_with('i') { + return format!("{head}i{suffix}"); + } + format!("{word}{suffix}") +} + +/// The suffixes both spellings of a pair take, by the pattern that tells them apart. +fn suffixes(american: &str, british: &str) -> &'static [&'static str] { + // Not -al, -ist or -ite: `humoral` and `humorist` are spelled so in British English too, and + // `behavioural` and `favourite` have lines of their own. + const OUR: &[&str] = &["s", "ed", "ing", "er", "ers", "ful", "fully", "less", "able", "ably"]; + const RE: &[&str] = &["s", "ed", "ing"]; + const IZE: &[&str] = &["s", "ed", "ing", "er", "ers", "ation", "ations", "ational", "able"]; + const E: &[&str] = &["s", "ed", "ing", "er", "ers"]; + const Y: &[&str] = &["s", "er", "est", "ly", "ness"]; + const PLURAL: &[&str] = &["s"]; + let stem = |w: &str, n: usize| w.get(..w.len().saturating_sub(n)).map(str::to_string); + if american.ends_with("or") && british.ends_with("our") && stem(american, 2) == stem(british, 3) { + return OUR; + } + if american.ends_with("er") && british.ends_with("re") && stem(american, 2) == stem(british, 2) { + return RE; + } + // One letter apart (`organize`/`organise`, `defense`/`defence`, `cozy`/`cosy`). + let differ = american.len() == british.len() && american.bytes().zip(british.bytes()).filter(|(a, b)| a != b).count() == 1; + if differ && american.ends_with('e') && british.ends_with('e') { + let ize = american.ends_with("ize") || american.ends_with("yze"); + return if ize { IZE } else { E }; + } + if differ && american.ends_with('y') && british.ends_with('y') { + return Y; + } + PLURAL +} + +/// One English's forms: those it flags (each with the form to suggest), and those it accepts +/// when the shared list knows them or their other spelling (each with that other spelling). +#[derive(Default)] +struct Forms { + foreign: HashMap, + accepted: HashMap, +} + +fn forms(lang: ProofLang) -> Option<&'static Forms> { + static F: [OnceLock; 3] = [OnceLock::new(), OnceLock::new(), OnceLock::new()]; + let col = column(lang)?; + Some(F.get(col)?.get_or_init(|| build(col))) +} + +fn build(col: usize) -> Forms { + let mut f = Forms::default(); + for pair in pairs() { + let Some(&policy) = pair.policy.get(col) else { continue }; + let mut both = vec![(pair.american.to_string(), pair.british.to_string())]; + both.extend(suffixes(pair.american, pair.british).iter().map(|s| (add(pair.american, s), add(pair.british, s)))); + for (am, br) in both { + if am == br { + continue; + } + match policy { + Policy::American => { + f.foreign.insert(br.clone(), am.clone()); + f.accepted.insert(am, br); + } + Policy::British => { + f.foreign.insert(am.clone(), br.clone()); + f.accepted.insert(br, am); + } + Policy::Both => { + f.accepted.insert(am.clone(), br.clone()); + f.accepted.insert(br, am); + } + } + } + } + // A form one pair accepts is never flagged by another, and `analyses`, `paralyses`… are also + // the plurals of `analysis`, `paralysis`… everywhere. + f.foreign.retain(|w, _| !f.accepted.contains_key(w) && !w.ends_with("yses")); + f +} + +/// `lower` without a possessive `'s`. +fn without_possessive(lower: &str) -> &str { + lower.strip_suffix("'s").filter(|s| !s.is_empty()).unwrap_or(lower) +} + +/// The spelling `lang` wants instead of `lower` (a lower-case word, `'` apostrophes), when +/// `lower` is the other dialect's spelling of a word (`colour` in American English: `color`). +/// A possessive keeps its `'s` (`colour's` → `color's`). +pub fn preferred(lower: &str, lang: ProofLang) -> Option { + let f = forms(lang)?; + let base = without_possessive(lower); + let native = f.foreign.get(base)?; + Some(if base.len() < lower.len() { format!("{native}'s") } else { native.clone() }) +} + +/// Is `lower` right in `lang` because the shared word list knows its other spelling (`known` +/// says what the list knows)? `realisation` in British English, `organization` in all three. +pub fn known_variant(lower: &str, lang: ProofLang, known: impl Fn(&str) -> bool) -> bool { + let Some(f) = forms(lang) else { return false }; + let base = without_possessive(lower); + f.accepted.get(base).is_some_and(|other| known(other)) +} + +/// Is `lower` the other dialect's spelling in `lang` (flagged even though it is a word)? +pub fn is_foreign(lower: &str, lang: ProofLang) -> bool { + forms(lang).is_some_and(|f| f.foreign.contains_key(without_possessive(lower))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn every_line_parses() { + let lines = VARIANTS.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')).count(); + assert_eq!(pairs().len(), lines, "a malformed line in en-variants.txt"); + assert!(pairs().len() > 250); + } + + #[test] + fn suffixes_follow_the_pattern() { + assert_eq!(add("centre", "ing"), "centring"); + assert_eq!(add("centre", "ed"), "centred"); + assert_eq!(add("center", "ed"), "centered"); + assert_eq!(add("organise", "ation"), "organisation"); + assert_eq!(add("cosy", "er"), "cosier"); + assert_eq!(add("cosy", "s"), "cosies"); + assert_eq!(add("armoury", "s"), "armouries"); + assert_eq!(add("analyse", "s"), "analyses"); + assert_eq!(add("colour", "ful"), "colourful"); + assert_eq!(add("", "s"), "s"); + assert_eq!(suffixes("color", "colour").len(), 10); + assert_eq!(suffixes("center", "centre"), ["s", "ed", "ing"]); + assert!(suffixes("organize", "organise").contains(&"ation")); + assert_eq!(suffixes("program", "programme"), ["s"], "no -ed: programmed is American too"); + } + + #[test] + fn no_form_is_both_flagged_and_accepted() { + for lang in [ProofLang::En, ProofLang::EnCa, ProofLang::EnGb] { + let Some(f) = forms(lang) else { panic!("{lang:?}") }; + for (w, native) in &f.foreign { + assert!(!f.accepted.contains_key(w), "{lang:?}: {w}"); + assert!(!f.foreign.contains_key(native), "{lang:?}: suggests {native}, itself flagged"); + } + } + // Words both dialects use alike are never flagged. + for w in [ + "programmed", + "tired", + "tires", + "curbs", + "humorous", + "honorary", + "humorist", + "humoral", + "meter", + "license", + "for", + "tour", + "analyses", + "paralyses", + ] { + for lang in [ProofLang::En, ProofLang::EnCa, ProofLang::EnGb] { + assert!(!is_foreign(w, lang), "{lang:?}: {w}"); + } + } + assert!(forms(ProofLang::Pl).is_none() && !is_foreign("color", ProofLang::Other)); + } +} diff --git a/crates/proof/src/english.rs b/crates/proof/src/english.rs new file mode 100644 index 00000000..67265abf --- /dev/null +++ b/crates/proof/src/english.rs @@ -0,0 +1,488 @@ +//! English spelling never goes unchecked. Its words come from the first of these that is there: +//! +//! 1. **Embedded**: the Moby word list embedded from craft-i18n at build time (`CRAFT_I18N_DIR`, +//! see `build.rs`). When it is there, nothing in this module is used. +//! 2. **System Hunspell**: a Hunspell `.aff`/`.dic` pair for the run's own English found by +//! [`crate::hunspell`] (`en_US` for American, `en_GB` for British, `en_CA` for Canadian +//! English; each name in the dictionaries folder and then the system locations before the +//! next name; none on the web). Another English's dictionary is never used: it would mark this +//! English's spellings (`colour` in an `en_US` one), so without its own dictionary an English +//! uses the built-in list, which has both spellings. A file that can't be read is skipped for +//! the next name or folder, and one file is read once for every English that uses it. +//! Directives that only matter for other languages (compounding, output conversion) are +//! ignored ([`crate::hunspell::load_english`]). +//! 3. **Built-in word list**: [`WORDS`], the whole public-domain Moby word list compiled into +//! the crate (without its break points), with WordCraft's own supplement of modern words +//! ([`EXTRA`]) and the usual inflections. A word it doesn't know is misspelled, as with the +//! embedded list. +//! +//! A Hunspell dictionary that is there but can't be read (unsupported encoding, too large, a +//! parse error, no words) is skipped, and its reason is reported by +//! [`crate::dictionary_load_error`]. Non-English text is never checked against any of these. + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, OnceLock, PoisonError, RwLock}; + +use crate::affix::AffixDict; +use crate::dict::Dictionary; +use crate::{Options, ProofLang}; + +/// The English word list built into the crate (`data/en-words.dic`, [`Dictionary`]'s format with +/// no break points): every word of the public-domain Moby Hyphenator II list by Grady Ward +/// (generated by `examples/en_words.rs` from craft-i18n's `en-us.dic`). +const WORDS: &[u8] = include_bytes!("../data/en-words.dic"); + +/// WordCraft's supplement of common modern words the Moby list lacks (`data/en-extra.txt`, a copy +/// of craft-i18n's `language-data/en-us/en-extra.txt`, MIT OR Apache-2.0): one lowercase base +/// form per line, `#` comments. Used when the Language data doesn't embed its own copy. +pub(crate) const EXTRA: &str = include_str!("../data/en-extra.txt"); + +/// Longest word (in characters) suggestions are looked for: longer ones get none, which keeps +/// the edits tried per word bounded. +const MAX_SUGGEST_CHARS: usize = 32; + +/// Most Hunspell `.aff`/`.dic` pairs tried for one English before giving up on Hunspell. +const MAX_CANDIDATES: usize = 8; + +/// Where English spelling comes from now ([`crate::english_source`]). +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EnglishSource { + /// The Moby word list embedded from craft-i18n at build time. + Embedded, + /// An English Hunspell dictionary from the dictionaries folder or the system: its `.dic` file. + Hunspell(PathBuf), + /// The Moby word list built into the crate: a word it doesn't know is misspelled. + CoreList, +} + +/// The words of [`WORDS`] (decoded on first use; empty if they fail to decode, which a test rules +/// out). +fn words() -> &'static Dictionary { + static W: OnceLock = OnceLock::new(); + W.get_or_init(|| { + Dictionary::from_bytes(WORDS).unwrap_or_else(|e| { + log::error!("built-in English word list: {e}"); + Dictionary::default() + }) + }) +} + +/// Is `w` (lower case) a word of the built-in list? +pub(crate) fn word_list_contains(w: &str) -> bool { + words().get(w).is_some() +} + +/// Hunspell file names of `lang`'s own dictionary, the usual one first. Only its own: another +/// English's dictionary would mark its spellings, which the built-in list doesn't. +fn names(lang: ProofLang) -> &'static [&'static str] { + match lang { + ProofLang::EnGb => &["en_GB", "en-GB"], + ProofLang::EnCa => &["en_CA", "en-CA"], + _ => &["en_US", "en-US"], + } +} + +/// The slot of each English in [`State::slots`]. +fn slot(lang: ProofLang) -> usize { + match lang { + ProofLang::EnGb => 1, + ProofLang::EnCa => 2, + _ => 0, + } +} + +/// A loaded Hunspell dictionary and its `.dic` file; `None` when none could be loaded. +type Loaded = Option<(Arc, PathBuf)>; + +/// The English Hunspell dictionaries, under one lock. +struct State { + /// Per English (see [`slot`]): `None` until first use, then what was found. + slots: [Option; 3], + /// Every `.dic` file read, so Englishes that use the same file share one copy. + files: Vec<(PathBuf, Arc)>, + /// Counts reloads, so a load that started before one is not kept. + reloads: u64, +} + +impl State { + const fn new() -> Self { + State { slots: [None, None, None], files: Vec::new(), reloads: 0 } + } +} + +static HUNSPELL: RwLock = RwLock::new(State::new()); + +/// Why the embedded word list failed to decode, if it did. +static EMBEDDED_ERROR: OnceLock = OnceLock::new(); + +/// Remember that the embedded word list is there but failed to decode (English then uses the +/// next source). +pub(crate) fn note_embedded_error(e: String) { + log::error!("embedded English word list: {e}; English spelling falls back to Hunspell or the built-in word list"); + let _ = EMBEDDED_ERROR.set(e); +} + +/// The English Hunspell dictionary for `lang` (read on first use). +fn hunspell(lang: ProofLang) -> Loaded { + let i = slot(lang); + let reloads = { + let s = HUNSPELL.read().unwrap_or_else(PoisonError::into_inner); + if let Some(Some(found)) = s.slots.get(i) { + return found.clone(); + } + s.reloads + }; + let (loaded, error) = load(lang); + keep(&HUNSPELL, i, reloads, loaded, || note_error(lang, error)) +} + +/// Keep in `state` what the load of slot `i`, started after `reloads` reloads, found, and `note` +/// its load error. If the folder was reloaded since, the load may have read the old folder: it +/// serves this one check, its error isn't noted, and the next check loads again. +fn keep(state: &RwLock, i: usize, reloads: u64, loaded: Loaded, note: impl FnOnce()) -> Loaded { + let mut s = state.write().unwrap_or_else(PoisonError::into_inner); + if s.reloads != reloads { + return match s.slots.get(i) { + Some(Some(found)) => found.clone(), + _ => loaded, + }; + } + note(); + let loaded = loaded.map(|(d, dic)| { + // Another English may have read the same file meanwhile: share its copy. + let d = match s.files.iter().find(|(p, _)| *p == dic) { + Some((_, shared)) => shared.clone(), + None => { + s.files.push((dic.clone(), d.clone())); + d + } + }; + (d, dic) + }); + // Another thread may have filled it meanwhile: that one wins. + match s.slots.get_mut(i) { + Some(slot) => slot.get_or_insert(loaded).clone(), + None => loaded, + } +} + +/// The dictionary of the `.dic` file `dic` in `state`, if an English already read it. +fn already_read(state: &RwLock, dic: &Path) -> Option> { + state.read().unwrap_or_else(PoisonError::into_inner).files.iter().find(|(p, _)| p == dic).map(|(_, d)| d.clone()) +} + +/// Forget what `state` holds, so the next check looks again. +fn forget(state: &RwLock) { + let mut s = state.write().unwrap_or_else(PoisonError::into_inner); + s.slots = [None, None, None]; + s.files.clear(); + s.reloads = s.reloads.wrapping_add(1); +} + +/// The key load errors of `lang` are kept under ([`crate::hunspell::load_error`]). +fn key(lang: ProofLang) -> &'static str { + names(lang).first().copied().unwrap_or("en_US") +} + +/// Read the first Hunspell pair of `lang`'s own that loads, trying each name in each folder +/// ([`crate::hunspell::candidates`]), with the first failure before it (the file `lang` would +/// rather use), for [`note_error`]. +fn load(lang: ProofLang) -> (Loaded, Option<(PathBuf, String)>) { + let candidates = crate::hunspell::candidates(names(lang)); + if candidates.is_empty() { + log::info!("English spelling: no Hunspell {} dictionary installed; using the built-in word list", key(lang)); + return (None, None); + } + let mut first_error: Option<(PathBuf, String)> = None; + for (aff, dic) in candidates.into_iter().take(MAX_CANDIDATES) { + let loaded = match already_read(&HUNSPELL, &dic) { + Some(d) => Ok(d), + None => crate::hunspell::load_english(&aff, &dic) + .and_then(|d| if d.is_empty() { Err(format!("{}: no words", dic.display())) } else { Ok(d) }) + .map(Arc::new), + }; + match loaded { + Ok(d) => return (Some((d, dic)), first_error), + Err(e) => { + log::warn!("English spelling: {e}; trying the next dictionary"); + first_error.get_or_insert((dic, e)); + } + } + } + log::warn!("English spelling: no {} Hunspell dictionary could be read; using the built-in word list", key(lang)); + (None, first_error) +} + +/// Remember `lang`'s load error ([`crate::dictionary_load_error`]), or that it has none. +fn note_error(lang: ProofLang, error: Option<(PathBuf, String)>) { + match error { + Some((dic, e)) => { + let _ = crate::hunspell::note_load::<()>(key(lang), &dic, Err(e)); + } + None => crate::hunspell::clear_load_error(key(lang)), + } +} + +/// Forget the loaded dictionaries, so the next check looks for them again (the dictionaries +/// folder changed). +pub(crate) fn reload() { + forget(&HUNSPELL); + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + crate::hunspell::clear_load_error(key(lang)); + } +} + +/// Where the spelling of `lang` (an English) comes from now. Reads the Hunspell dictionary once +/// when there is no embedded data (it is cached afterwards). +pub(crate) fn source(lang: ProofLang) -> EnglishSource { + if crate::english_data() { + EnglishSource::Embedded + } else { + hunspell(lang).map_or(EnglishSource::CoreList, |(_, dic)| EnglishSource::Hunspell(dic)) + } +} + +/// Why a source of `lang`'s spelling failed to load: the embedded word list, or (without it) the +/// Hunspell dictionary found for it. Reads nothing itself. +pub(crate) fn load_error(lang: ProofLang) -> Option { + if let Some(e) = EMBEDDED_ERROR.get() { + return Some(e.clone()); + } + if crate::english_data() { + return None; + } + crate::hunspell::load_error(key(lang)).map(|(_, reason)| reason) +} + +/// Is `word` spelled right in `lang` (an English) when no word list is embedded: against the +/// Hunspell dictionary, or else the built-in word list. +pub(crate) fn is_correct(word: &str, lang: ProofLang, opts: &Options) -> bool { + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’').trim_matches(['\'', '’']); + if crate::accepted_by_options(w, opts) || !w.chars().any(|c| c.is_ascii_alphabetic()) { + return true; // other scripts: not English + } + let w = w.replace('’', "'"); + match hunspell(lang) { + Some((d, _)) => hunspell_correct(&d, &w), + None => list_correct(&w), + } +} + +/// Against an installed English Hunspell dictionary: the word, or each part of a hyphenated one. +fn hunspell_correct(d: &AffixDict, w: &str) -> bool { + let known = |p: &str| p.chars().count() <= 1 || d.check(p) || crate::in_user_dictionary(&p.to_lowercase()); + known(w) || (w.contains('-') && w.split('-').filter(|p| !p.is_empty()).all(known)) +} + +/// Against the built-in word list: its words and their inflections, or each part of a +/// hyphenated word. Anything else is misspelled. (Should the list fail to decode, nothing is +/// marked rather than everything.) +fn list_correct(w: &str) -> bool { + if words().is_empty() { + return true; + } + let lower = w.to_lowercase(); + crate::known_in_word_list(&lower) || (lower.contains('-') && lower.split('-').filter(|p| !p.is_empty()).all(crate::known_in_word_list)) +} + +/// Spelling suggestions in `lang` (an English) when no word list is embedded: from the Hunspell +/// dictionary, or else the built-in word list. +pub(crate) fn suggest(word: &str, max: usize, lang: ProofLang) -> Vec { + match hunspell(lang) { + Some((d, _)) => crate::cs::suggest::suggest_in(&d, word, max), + None => list_suggest(word, max), + } +} + +/// Letters an edit may add or put in a word's place. +const ALPHABET: &str = "abcdefghijklmnopqrstuvwxyz'"; + +/// Are `a` and `b` neighbouring keys on a QWERTY keyboard? (A typing slip: `wprd` for `word`.) +fn neighbour_keys(a: char, b: char) -> bool { + const ROWS: [&str; 3] = ["qwertyuiop", "asdfghjkl", "zxcvbnm"]; + // Each row sits half a key right of the one above: x in half keys. + let at = |c: char| ROWS.iter().enumerate().find_map(|(r, row)| row.find(c).map(|col| (r as i64, 2 * col as i64 + r as i64))); + match (at(a), at(b)) { + (Some((ra, xa)), Some((rb, xb))) => (ra == rb && (xa - xb).abs() == 2) || ((ra - rb).abs() == 1 && (xa - xb).abs() <= 1), + _ => false, + } +} + +/// A suggestion's rank, lowest first: not a swap, not a common word, not a neighbouring key, not +/// the same first letter, the difference in length. +type Rank = (bool, bool, bool, bool, usize); + +/// How an edit changed the word, for ranking. +#[derive(Clone, Copy)] +enum Edit { + /// Two neighbouring letters swapped. + Swap, + /// One letter changed from the first to the second. + Change(char, char), + /// One letter added or missing. + Other, +} + +/// Suggestions from the built-in word list: the words one edit away (two neighbouring letters +/// swapped, one letter missing, added or changed), looked up in the list rather than compared +/// with every word, so the cost is bounded by the word's length (at most +/// [`MAX_SUGGEST_CHARS`]). Ranked: swaps first (`recieve` → `receive`), then the common words +/// (`hte` → `the` before `het`), a letter changed to a neighbouring key's, the same first letter, +/// the nearest in length. One-letter words other than `a` and `I` are never suggested. +fn list_suggest(word: &str, max: usize) -> Vec { + let lower = word.to_lowercase().replace('’', "'"); + let target: Vec = lower.chars().collect(); + let n = target.len(); + if max == 0 || n == 0 || n > MAX_SUGGEST_CHARS { + return Vec::new(); + } + let mut found: Vec<(Rank, String)> = Vec::new(); + let mut tried = std::collections::HashSet::new(); + let mut consider = |c: Vec, edit: Edit| { + let s: String = c.iter().collect(); + if (c.len() < 2 && !matches!(s.as_str(), "a" | "i")) || s == lower || !tried.insert(s.clone()) || !crate::known_in_word_list(&s) { + return; + } + let key = ( + !matches!(edit, Edit::Swap), + !crate::is_common_word(&s), + !matches!(edit, Edit::Change(from, to) if neighbour_keys(from, to)), + c.first() != target.first(), + c.len().abs_diff(n), + ); + found.push((key, s)); + }; + for i in 1..n { + let mut c = target.clone(); + c.swap(i - 1, i); + consider(c, Edit::Swap); + } + for i in 0..n { + let mut c = target.clone(); + c.remove(i); + consider(c, Edit::Other); + } + for i in 0..=n { + for a in ALPHABET.chars() { + if let Some(&from) = target.get(i) { + let mut c = target.clone(); + if let Some(l) = c.get_mut(i) { + *l = a; + } + consider(c, Edit::Change(from, a)); + } + let mut c = target.clone(); + c.insert(i, a); + consider(c, Edit::Other); + } + } + found.sort(); + found.into_iter().take(max).map(|(_, w)| crate::match_case(word, w)).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn built_in_list_is_the_whole_moby_list() { + let d = words(); + assert!(d.len() > 150_000, "{}", d.len()); + assert!(d.iter().all(|(w, m)| m == 0 && !w.is_empty() && w.chars().all(|c| !c.is_uppercase()))); + assert!(WORDS.len() < 400 * 1024, "{} bytes", WORDS.len()); + let extra: Vec<&str> = EXTRA.lines().filter(|l| !l.is_empty() && !l.starts_with('#')).collect(); + assert!(extra.len() > 100 && extra.windows(2).all(|p| p[0] < p[1]), "the supplement is sorted"); + } + + /// The built-in list knows right words, the rare and the modern ones included, and marks + /// every word it doesn't know (craft-libs#482 review: the old core list let any unknown word + /// through unless it was a near miss of a common one, and marked right words that were). + #[test] + fn built_in_list_marks_unknown_words_only() { + for w in [ + "the", + "The", + "houses", + "running", + "looked", + "happily", + "kindness", + "Received", + "well-known", + "photosynthesis", + "fiber", + "theater", + "organize", + "budget", + "phone", + "people's", + "wont", + "cant", + // WordCraft's supplement. + "login", + "blog", + "email", + "emails", + ] { + assert!(list_correct(w), "{w} is not marked"); + } + for w in ["teh", "Teh", "abd", "ur", "recieved", "definately", "seperate", "ocurred", "becuase", "peopel", "beautifull", "zxqv", "Jarndycey"] + { + assert!(!list_correct(w), "{w} is marked"); + } + assert!(!list_correct("well-knwon"), "each part of a compound is checked"); + } + + #[test] + fn built_in_list_suggests_one_edit_away() { + assert_eq!(list_suggest("becuase", 3).first().map(String::as_str), Some("because")); + assert_eq!(list_suggest("Becuase", 3).first().map(String::as_str), Some("Because")); + assert_eq!(list_suggest("hte", 3).first().map(String::as_str), Some("the")); + assert_eq!(list_suggest("recieve", 3).first().map(String::as_str), Some("receive")); + assert_eq!(list_suggest("wprd", 3).first().map(String::as_str), Some("word"), "a neighbouring key"); + assert!(list_suggest("é", 5).iter().all(|s| s.chars().count() > 1 || s == "a" || s == "i"), "{:?}", list_suggest("é", 5)); + assert!(list_suggest("x", 5).iter().all(|s| s.chars().count() > 1 || s == "a" || s == "i"), "{:?}", list_suggest("x", 5)); + assert!(!neighbour_keys('q', 's') && neighbour_keys('q', 'a') && neighbour_keys('o', 'p') && !neighbour_keys('é', 'e')); + assert!(list_suggest("definately", 5).contains(&"definitely".to_string())); + assert!(list_suggest("", 5).is_empty() && list_suggest("becuase", 0).is_empty()); + assert!(list_suggest(&"a".repeat(MAX_SUGGEST_CHARS + 1), 5).is_empty()); + } + + /// Suggestions look words up rather than scanning the list: the longest word is still quick. + #[test] + fn built_in_list_suggestions_are_bounded() { + let _ = words(); + let word: String = "qwertyuiopasdfghjklzxcvbnm".chars().cycle().take(MAX_SUGGEST_CHARS).collect(); + let start = std::time::Instant::now(); + for _ in 0..10 { + let _ = list_suggest(&word, 5); + } + assert!(start.elapsed() < std::time::Duration::from_secs(5), "{:?}", start.elapsed()); + } + + /// craft-libs#482 review: a load that started before a reload (the dictionaries folder + /// changed) serves its own check but isn't kept, so the next check reads the new folder; a + /// load after it is kept, and shared with the Englishes that read the same file. + #[test] + fn a_load_from_before_a_reload_is_not_kept() { + let state = RwLock::new(State::new()); + let noted = std::cell::Cell::new(false); + let dict = || Arc::new(AffixDict::from_hunspell("", "1\nzzyzx\n", &[]).unwrap()); + let file = PathBuf::from("/old/folder/en_US.dic"); + let before = state.read().unwrap().reloads; + forget(&state); // The folder changes while the load runs. + let got = keep(&state, slot(ProofLang::En), before, Some((dict(), file.clone())), || noted.set(true)); + assert_eq!(got.map(|(_, p)| p), Some(file.clone()), "the stale load still answers its own check"); + assert!(state.read().unwrap().slots.iter().all(Option::is_none), "but it isn't kept"); + assert!(!noted.get(), "nor is its load error"); + assert!(already_read(&state, &file).is_none()); + let now = state.read().unwrap().reloads; + let a = keep(&state, slot(ProofLang::En), now, Some((dict(), file.clone())), || noted.set(true)).unwrap(); + assert!(noted.get()); + let b = keep(&state, slot(ProofLang::EnGb), now, Some((dict(), file.clone())), || {}).unwrap(); + assert!(Arc::ptr_eq(&a.0, &b.0), "one copy of a file for every English that reads it"); + assert!(already_read(&state, &file).is_some()); + forget(&state); + assert!(already_read(&state, &file).is_none() && state.read().unwrap().slots.iter().all(Option::is_none)); + } +} diff --git a/crates/proof/src/es/grammar.rs b/crates/proof/src/es/grammar.rs new file mode 100644 index 00000000..42eede50 --- /dev/null +++ b/crates/proof/src/es/grammar.rs @@ -0,0 +1,393 @@ +//! Spanish grammar checks with Spanish messages: the generic repeated-word and spacing checks, +//! and the opening question and exclamation marks Spanish writes before a question or an +//! exclamation (`¿Vienes?`, `¡Qué bien!`): a closing `?` or `!` whose sentence has no opening mark +//! is flagged, with the opening mark offered at the sentence's start, and so is a space after an +//! opening mark (`¿ Vienes?`). Either opening mark may pair with either closing one (`¡Qué +//! dices?`), and `(?)` and `(!)`, which mark doubt or surprise, stand alone. No other +//! Spanish-specific grammar, comma or sentence-capital rules are inferred. + +use unicode_normalization::UnicodeNormalization; + +use crate::{Issue, IssueKind, ProofLang, words}; + +fn issue(start: usize, end: usize, message: String, suggestions: Vec, lang: ProofLang) -> Issue { + Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang } +} + +/// Grammar issues in a paragraph of Spanish text; `lang` is the variety (they explain the same in +/// every one). +pub fn check(text: &str, lang: ProofLang) -> Vec { + let mut issues = Vec::new(); + repeated_words(text, &words(text), lang, &mut issues); + spaces(text, lang, &mut issues); + opening_marks(text, lang, &mut issues); + issues.sort_by_key(|i| (i.start, i.end)); + issues +} + +fn repeated_words(text: &str, ws: &[(usize, usize)], lang: ProofLang, v: &mut Vec) { + // Letters and the combining marks of a decomposed word (`tambie\u{301}n`); a composed and a + // decomposed spelling of one word are the same word. + let letters = |w: &str| w.chars().all(|c| c.is_alphabetic() || unicode_normalization::char::is_combining_mark(c)); + let folded = |w: &str| w.nfc().collect::().to_lowercase(); + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if !gap.is_empty() && gap.chars().all(char::is_whitespace) && letters(w0) && folded(w0) == folded(w1) { + v.push(issue(*a0, *b1, format!("Palabra repetida: «{w1}»"), vec![w0.to_string()], lang)); + } + } +} + +fn spaces(text: &str, lang: ProofLang, v: &mut Vec) { + for (i, c) in text.char_indices() { + // A space after an opening mark: `¿ Vienes?`. + if matches!(c, '¿' | '¡') + && text.get(i + c.len_utf8()..).is_some_and(|r| r.starts_with(' ') && r.trim_start().chars().next().is_some_and(char::is_alphanumeric)) + { + v.push(issue(i, i + c.len_utf8() + 1, "Espacio después de un signo de apertura".into(), vec![c.to_string()], lang)); + continue; + } + if c != ' ' { + continue; + } + let rest = text.get(i + 1..).unwrap_or(""); + let mut it = rest.chars(); + let Some(p) = it.next() else { continue }; + let after = it.next(); + // A ratio/score keeps the spaces in digit-space-colon-space-digit. + let ratio = p == ':' + && text.get(..i).and_then(|before| before.chars().next_back()).is_some_and(|c| c.is_ascii_digit()) + && after == Some(' ') + && it.next().is_some_and(|c| c.is_ascii_digit()); + let flag = !ratio + && match p { + ',' => true, + // `;)` and `;-)` are emoticons. + ';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')), + // ` ...` is an ellipsis, ` .5` a number. + '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), + // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. + ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '»' | '!' | '?')), + _ => false, + }; + if flag { + v.push(issue(i, i + 1 + p.len_utf8(), "Espacio antes de un signo de puntuación".into(), vec![p.to_string()], lang)); + } + } + for (i, _) in text.match_indices(" ") { + if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { + continue; + } + v.push(issue(i, i + 2, "Espacio doble".into(), vec![" ".into()], lang)); + } +} + +/// Does the character `b` at byte `j` end what comes before it, so a question or exclamation after +/// it starts there? Sentence ends, clause breaks, line breaks and the start of a quotation or +/// parenthesis do, but not punctuation inside a word (`3.5`, `10:30`) or inside an address +/// (`example.com`, `https:`), except a mark ending an address (`…/? Vienes mañana?`). +fn ends_clause(text: &str, addresses: &[(usize, usize)], j: usize, b: char) -> bool { + let after = j + b.len_utf8(); + if let Some((_, end)) = address_at(addresses, j) { + return matches!(b, '?' | '!' | '.') && end == after; + } + let inside_word = matches!(b, '.' | ':' | ';') && text.get(after..).and_then(|r| r.chars().next()).is_some_and(char::is_alphanumeric); + !inside_word && matches!(b, '.' | '?' | '!' | '…' | ':' | ';' | '\n' | '\r' | '\t' | '«' | '“' | '"' | '(' | '[' | '—') +} + +/// The one of the `addresses` ([`address_ranges`]) that byte `j` is inside, if any. The ranges come +/// in text order and never overlap, so this is a binary search, O(log m) for m addresses: a paragraph +/// of thousands of addresses costs O(n log m) when every character before a question mark asks +/// (#3562 review), not the O(n m) of a scan per character. +fn address_at(addresses: &[(usize, usize)], j: usize) -> Option<(usize, usize)> { + let after_start = addresses.partition_point(|&(s, _)| s <= j); + let (s, e) = *addresses.get(after_start.checked_sub(1)?)?; + (s <= j && j < e).then_some((s, e)) +} + +/// The byte ranges of the whitespace-delimited tokens of `text` that are web or e-mail addresses +/// (`https://example.com/?`, `www.example.com`, `ana@example.com`), as spelling skips them. +fn address_ranges(text: &str) -> Vec<(usize, usize)> { + let mut out = Vec::new(); + let mut start = None; + // A final space ends the last token. + for (i, c) in text.char_indices().chain(std::iter::once((text.len(), ' '))) { + if !c.is_whitespace() { + start = start.or(Some(i)); + } else if let Some(s) = start.take() + && text.get(s..i).is_some_and(crate::is_address) + { + out.push((s, i)); + } + } + out +} + +/// A closing `?` or `!` with no opening `¿` or `¡` before it. Openers are counted left to right, like +/// brackets, and each closing mark takes one back (either opener pairs with either closer: +/// `¡Qué dices?`), so what sits between them doesn't matter: quotes, parentheses, dashes, an +/// ellipsis or an abbreviation's full stop (`¿Viene el Sr. García?`, `¿Leíste «Rayuela»?`) all belong +/// to the question. Only a closing mark with nothing open is flagged, and then the sentence it +/// closes starts after the last clause break ([`ends_clause`]). Marks in web and e-mail addresses +/// aren't sentence punctuation, and neither is punctuation inside an address or a word +/// (`example.com`, `3.5`, `10:30`) a sentence's start. +fn opening_marks(text: &str, lang: ProofLang, v: &mut Vec) { + let addresses = address_ranges(text); + // Openers not closed yet: a run of them (`¡¡`) counts once, like a run of closers (`!!`). + let mut open = 0usize; + for (i, c) in text.char_indices() { + let before = text.get(..i).unwrap_or(""); + let prev = before.chars().next_back(); + let (opener, message) = match c { + '¿' | '¡' => { + if !matches!(prev, Some('¿' | '¡')) { + open += 1; + } + continue; + } + '?' => ('¿', "Falta el signo de interrogación de apertura (¿)"), + '!' => ('¡', "Falta el signo de exclamación de apertura (¡)"), + _ => continue, + }; + let next = text.get(i + 1..).and_then(|r| r.chars().next()); + // `?!` and `!!!` are handled at their first mark; `(?)` stands alone; `?q=`, `a!b` and a + // mark ending an address (`https://example.com/?`) aren't sentence ends. + if matches!(prev, Some('?' | '!')) + || (prev == Some('(') && next == Some(')')) + || next.is_some_and(char::is_alphanumeric) + || address_at(&addresses, i).is_some() + { + continue; + } + if open > 0 { + open -= 1; + continue; + } + let from = before.char_indices().rev().find(|&(j, b)| matches!(b, '¿' | '¡') || ends_clause(text, &addresses, j, b)); + let start = from.map_or(0, |(j, b)| j + b.len_utf8()); + let Some(sentence) = text.get(start..i) else { continue }; + let lead = sentence.len() - sentence.trim_start().len(); + let start = start + lead; + let Some(sentence) = text.get(start..i + 1) else { continue }; + if !sentence.chars().any(char::is_alphabetic) { + continue; + } + v.push(issue(start, i + 1, message.into(), vec![format!("{opener}{sentence}")], lang)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn messages(text: &str) -> Vec<(String, String)> { + check(text, ProofLang::Es).into_iter().map(|i| (text[i.start..i.end].to_string(), i.message)).collect() + } + + /// The missing-opening-mark issues of `text`: the flagged text and the suggestions offered. + fn missing(text: &str) -> Vec<(String, Vec)> { + check(text, ProofLang::Es) + .into_iter() + .filter(|i| i.message.starts_with("Falta")) + .map(|i| (text[i.start..i.end].to_string(), i.suggestions)) + .collect() + } + + #[test] + fn repeated_words_and_spaces_have_spanish_messages() { + let text = "Es es la casa , dijo."; + let issues = check(text, ProofLang::EsMx); + assert_eq!(issues.len(), 3, "{issues:?}"); + assert_eq!(issues[0].message, "Palabra repetida: «es»"); + assert_eq!(issues[0].suggestions, ["Es"]); + assert!(issues.iter().any(|i| i.message == "Espacio antes de un signo de puntuación" && &text[i.start..i.end] == " ,")); + assert!(issues.iter().any(|i| i.message == "Espacio doble")); + assert!(issues.iter().all(|i| i.lang == ProofLang::EsMx && i.kind == IssueKind::Grammar)); + for text in ["La casa. Es grande", "Sr. García", "No, no lo sé", "Hola :) número .5 y puntos ...", "3 : 1", "teórico-práctico"] { + assert!(check(text, ProofLang::Es).is_empty(), "{text}: {:?}", check(text, ProofLang::Es)); + } + } + + #[test] + fn questions_and_exclamations_need_their_opening_mark() { + assert_eq!(messages("Vienes mañana?"), [("Vienes mañana?".into(), "Falta el signo de interrogación de apertura (¿)".into())]); + assert_eq!(check("Vienes mañana?", ProofLang::Es)[0].suggestions, ["¿Vienes mañana?"]); + assert_eq!(messages("Llegó. Qué bien!"), [("Qué bien!".into(), "Falta el signo de exclamación de apertura (¡)".into())]); + assert_eq!(messages("Le preguntó: vienes?")[0].0, "vienes?"); + assert_eq!(messages("Dijo «vienes?»")[0].0, "vienes?"); + assert_eq!(messages("Hola!!!").len(), 1); + assert_eq!(messages("Qué?!").len(), 1); + for text in [ + "¿Vienes mañana?", + "¡Qué bien!", + "Si vienes, ¿me llamas?", + "¡Qué dices?", + "¿Qué?! ¡No!", + "Nació en 1950 (?) en Montevideo.", + "Lo dijo (!) en público.", + "Ver https://example.com/?q=1 y a!b", + "¿Qué? ¿Cómo?", + "2 + 2 = ?", + "", + "?", + ] { + assert!(messages(text).iter().all(|(_, m)| !m.starts_with("Falta")), "{text}: {:?}", messages(text)); + } + } + + /// Maintainer review on #3562: a URL ending in `/?` isn't a question, and the dots and colons + /// of an address or a word (`example.com`, `https:`, `10:30`) don't start the sentence the + /// missing `¿` is offered for; ordinary questions are still checked. + #[test] + fn addresses_are_not_questions_or_sentence_starts() { + for text in [ + "Ver https://example.com/?", + "Ver https://example.com/? ¿Vienes?", + "Ver http://example.com/buscar/? y www.example.com/?", + "Escribí a ana@example.com!", + "¿Viste https://example.com/?", + ] { + assert!(missing(text).is_empty(), "{text}: {:?}", missing(text)); + } + assert_eq!( + missing("Mirá https://example.com/ y decime si vienes?"), + [("Mirá https://example.com/ y decime si vienes?".to_string(), vec!["¿Mirá https://example.com/ y decime si vienes?".to_string()])] + ); + assert_eq!(missing("Mirá example.com y decime si vienes?")[0].0, "Mirá example.com y decime si vienes?"); + assert_eq!(missing("Son las 10:30, vienes?")[0].0, "Son las 10:30, vienes?"); + assert_eq!(missing("Ver https://example.com/? Vienes mañana?"), [("Vienes mañana?".to_string(), vec!["¿Vienes mañana?".to_string()])]); + assert_eq!(missing("Llegó. Vienes?")[0].0, "Vienes?", "a full stop before a space still ends a sentence"); + } + + /// Maintainer review on #3562: a correctly opened question or exclamation is never flagged, + /// whatever sits between its marks: quotes, an abbreviation's full stop, an ellipsis, a dash + /// aside, parentheses, nested questions, a line break. + #[test] + fn correctly_opened_questions_are_not_flagged() { + for text in [ + "¿Leíste «Rayuela»?", + "¿Viene el Sr. García?", + "¿Y si... no viene?", + "¿Vendrás —preguntó Juan— mañana?", + "¿Qué significa (aquí) la palabra?", + "¿Has leído \"Rayuela\" de Cortázar?", + "¿Leíste “Rayuela”?", + "¿Es 'Rayuela' o 'Los premios'?", + "¿Y tú, qué opinas… vienes?", + "¿Dijo «¿qué?» o no?", + "¿Viene la Dra. López, es decir, la jefa (la de 2.5 años)?", + "¿Dónde: aquí o allí?", + "¿Cuánto cuesta: 3,50 o a las 10:30?", + "¿Vienes a las 10:30; o mejor mañana?", + "¡Qué dices, Dr. Pérez!", + "¡Qué dices, Dr. Pérez? ¡Basta!", + "¡¡Hola!! ¿Qué tal?", + "¡¡¡Basta!!!", + "¿Vienes?!", + "¿Qué?! ¡No!", + "¿Vienes?\n¿O no?", + "¿Vienes\nmañana?", + "Ana dijo: «¿Vienes?» y yo: «Sí».", + "—¿Vienes? —preguntó Juan.", + "¿Viste https://example.com/? Dime.", + "¿Es verdad (!) o no?", + "Nació en 1950 (?) en Montevideo.", + ] { + assert!(missing(text).is_empty(), "{text:?}: {:?}", missing(text)); + } + } + + /// A closing mark with nothing open is still flagged, after a sentence or a clause that closed + /// an earlier question, and with the sentence it closes offered with its opening mark. + #[test] + fn closing_marks_with_nothing_open_are_still_flagged() { + for (text, flagged, offered) in [ + ("Vienes mañana?", "Vienes mañana?", "¿Vienes mañana?"), + ("Llegó. Vienes?", "Vienes?", "¿Vienes?"), + ("Le preguntó: vienes?", "vienes?", "¿vienes?"), + ("Dijo «vienes?»", "vienes?", "¿vienes?"), + ("Qué bien!", "Qué bien!", "¡Qué bien!"), + ("¿Vienes? No sé qué hacer?", "No sé qué hacer?", "¿No sé qué hacer?"), + ("¿Viene el Sr. García? Llegó tarde!", "Llegó tarde!", "¡Llegó tarde!"), + ("¡¡Hola!! Qué bien!", "Qué bien!", "¡Qué bien!"), + ("¿Qué?! No sé qué decir!", "No sé qué decir!", "¡No sé qué decir!"), + ] { + let found = missing(text); + assert_eq!(found.len(), 1, "{text:?}: {found:?}"); + assert_eq!((found[0].0.as_str(), found[0].1.first().map(String::as_str)), (flagged, Some(offered)), "{text:?}"); + } + } + + #[test] + fn address_lookup_finds_the_range_a_byte_is_in() { + let ranges = [(2, 5), (7, 8), (10, 20)]; + let inside = |j| address_at(&ranges, j); + assert_eq!([0, 1].map(inside), [None, None], "before the first"); + assert_eq!([2, 3, 4].map(inside), [Some((2, 5)); 3], "the start is inside, the end is not"); + assert_eq!([5, 6].map(inside), [None, None], "between ranges"); + assert_eq!([7, 10, 19].map(inside), [Some((7, 8)), Some((10, 20)), Some((10, 20))]); + assert_eq!([8, 9, 20, 99].map(inside), [None; 4], "after a range, and after the last"); + assert_eq!(address_at(&[], 3), None); + } + + /// Maintainer review on #3562: a paragraph of thousands of addresses followed by a question was + /// checked in quadratic time (every character before the question scanned every address). + /// About 99 KB (under the 100,000-byte paragraph cap) took hundreds of milliseconds; it is now + /// O(n log m): about 1 to 5 ms in a debug build, against about 300 ms for the scan at + /// `opt-level = 1`. craft-libs builds tests at `opt-level = 0`, where the fixed lookup took up + /// to ~80 ms under a parallel test run and more on a busy machine. The test guards against a + /// return to the quadratic blowup, not machine load, so the bound is 1 s. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn many_addresses_before_a_question_are_checked_quickly() { + for (unit, ending, whole_paragraph) in + [("https://x/ ", "Vienes?", true), ("https://x/? ", "Vienes?", false), ("www.x.com ", "Qué bien!", true)] + { + // As many as fit in about 99 KB: 9,000 of the first, as in the review. + let text = format!("{}{ending}", unit.repeat(99_000 / unit.len())); + assert!(text.len() < 100_000, "{}", text.len()); + let started = std::time::Instant::now(); + let issues = check(&text, ProofLang::Es); + let took = started.elapsed(); + assert!(took < std::time::Duration::from_secs(1), "{unit:?} took {took:?}"); + let missing: Vec<&Issue> = issues.iter().filter(|i| i.message.starts_with("Falta")).collect(); + assert_eq!(missing.len(), 1, "{unit:?}: {missing:?}"); + // The mark's sentence is the whole paragraph when the addresses don't end with a mark + // of their own, and only the last question when they do. + let want_start = if whole_paragraph { 0 } else { text.len() - ending.len() }; + assert_eq!((missing[0].start, missing[0].end), (want_start, text.len()), "{unit:?}"); + } + } + + /// A word repeated once composed and once decomposed is still repeated. + #[test] + fn repeated_words_match_across_normalization_forms() { + let text = "también tambie\u{301}n"; + let issues = check(text, ProofLang::Es); + assert_eq!(issues.len(), 1, "{issues:?}"); + assert_eq!((issues[0].start, issues[0].end), (0, text.len())); + assert!(issues[0].message.starts_with("Palabra repetida")); + } + + #[test] + fn a_space_after_an_opening_mark_is_flagged() { + let text = "¿ Vienes? ¡ Hola!"; + let issues = check(text, ProofLang::EsUy); + let spaced: Vec<&str> = + issues.iter().filter(|i| i.message == "Espacio después de un signo de apertura").map(|i| &text[i.start..i.end]).collect(); + assert_eq!(spaced, ["¿ ", "¡ "], "{issues:?}"); + assert!(issues.iter().all(|i| !i.message.starts_with("Falta")), "{issues:?}"); + assert!(check("¿ ", ProofLang::Es).is_empty()); + } + + proptest::proptest! { + #[test] + fn never_panics_and_ranges_are_valid(s in "\\PC{0,80}|[¿¡?!a-z .,(«»):]{0,40}") { + for issue in check(&s, ProofLang::Es) { + proptest::prop_assert!(issue.start <= issue.end && issue.end <= s.len()); + proptest::prop_assert!(s.is_char_boundary(issue.start) && s.is_char_boundary(issue.end)); + } + } + } +} diff --git a/crates/proof/src/es/mod.rs b/crates/proof/src/es/mod.rs new file mode 100644 index 00000000..5263bd1d --- /dev/null +++ b/crates/proof/src/es/mod.rs @@ -0,0 +1,315 @@ +//! Spanish proofing (#3360, #3553) using the installed Hunspell dictionaries (never bundled; +//! LibreOffice ships them as its `dict-es` extension, Debian and Ubuntu as `hunspell-es`). Each +//! variety reads its own dictionary: Spain ([`ProofLang::Es`]) `es_ES`, Mexico +//! ([`ProofLang::EsMx`]) `es_MX`, Latin America ([`ProofLang::Es419`]) `es_419` and Uruguay +//! ([`ProofLang::EsUy`]) `es_UY`, each falling back to a nearby or generic one. Uruguayan Spanish +//! also accepts the `vos` forms of every verb in the dictionary ([`voseo`]: `tenés`, `vení`, +//! `decime`). Suggestions use the dictionary's own MAP/REP/TRY/KEY tables (the regional +//! dictionaries' REP tables carry their differences, such as `vosotros` forms in Mexico). +//! Grammar is limited to generic repeated-word and spacing checks and the Spanish opening +//! question and exclamation marks, with Spanish messages; hyphenation keeps the English fallback +//! (no Spanish patterns are bundled). + +pub mod grammar; +mod suggest; +#[cfg(test)] +pub(crate) mod test_dict; +pub mod voseo; + +use std::sync::Arc; + +use crate::ProofLang; +use crate::affix::AffixDict; +use crate::slot::DictSlot; + +pub use suggest::suggest; + +/// The dictionary of each variety, found on first use (see [`DictSlot`]). +pub(crate) static SPAIN_DICT: DictSlot = DictSlot::new(SPAIN, "Spanish (Spain)"); +pub(crate) static MEXICO_DICT: DictSlot = DictSlot::new(MEXICO, "Spanish (Mexico)"); +pub(crate) static LATIN_AMERICA_DICT: DictSlot = DictSlot::new(LATIN_AMERICA, "Spanish (Latin America)"); +pub(crate) static URUGUAY_DICT: DictSlot = DictSlot::new(URUGUAY, "Spanish (Uruguay)"); + +/// File names of a Spanish Hunspell dictionary for Spain, most specific first (`es_ANY` is the +/// generic one RLA-ES publishes). +const SPAIN: &[&str] = &["es_ES", "es-ES", "es_ANY", "es"]; +/// For Mexico: its own, then a Latin American or generic one. +const MEXICO: &[&str] = &["es_MX", "es-MX", "es_419", "es-419", "es_ANY", "es"]; +/// For Latin America: its own or a generic one, then the United States' and Mexico's, which +/// LibreOffice ships (it has no `es_419`) and which use Latin American spelling, then +/// Argentina's, so `es-AR` text is checked when that is the only one installed. +const LATIN_AMERICA: &[&str] = &["es_419", "es-419", "es_ANY", "es", "es_US", "es_MX", "es_AR", "es-AR"]; +/// For Uruguay: its own, then Argentina's (the same Rioplatense Spanish), then a Latin American +/// or generic one. +const URUGUAY: &[&str] = &["es_UY", "es-UY", "es_AR", "es-AR", "es_419", "es-419", "es_ANY", "es"]; + +/// `lang`'s variety's dictionary; Spain's for any other language. +fn slot(lang: ProofLang) -> &'static DictSlot { + match lang { + ProofLang::EsMx => &MEXICO_DICT, + ProofLang::Es419 => &LATIN_AMERICA_DICT, + ProofLang::EsUy => &URUGUAY_DICT, + _ => &SPAIN_DICT, + } +} + +/// The Hunspell name each variety's dictionary is known by (`es_ES`, `es_MX`, `es_419`, +/// `es_UY`): the first of its file names, used for the status-bar hint and load errors. +pub fn dictionary_name(lang: ProofLang) -> &'static str { + slot(lang).name() +} + +/// The Spanish dictionary of `lang`'s variety: the installed one (read on first use), one given +/// with [`install_dictionary`], or an empty one when none is available (spelling is off then; +/// see [`has_dictionary`]). +pub fn dictionary(lang: ProofLang) -> Arc { + slot(lang).get() +} + +/// Use this dictionary for `lang`'s variety from now on (Hunspell `.aff` and `.dic` text, +/// decoded to UTF-8), for example one the user picked. On error the current dictionary stays. +pub fn install_dictionary(lang: ProofLang, aff: &str, dic: &str) -> Result<(), String> { + slot(lang).install(aff, dic) +} + +/// Is a dictionary available for `lang`'s variety (spelling is checked)? +pub fn has_dictionary(lang: ProofLang) -> bool { + slot(lang).has_dictionary() +} + +/// Is `word` spelled correctly in `lang`'s Spanish? Numbers, single letters, short words in +/// capitals (acronyms: `ONU`, `CDMX`), words with digits and words in other scripts are accepted, +/// and so is every word when no dictionary is installed for the variety. Hyphenated compounds +/// are checked part by part (`teórico-práctico`, `franco-mexicano`). In Uruguayan Spanish the +/// `vos` forms of the dictionary's verbs are correct too ([`voseo`]). +pub fn is_correct(word: &str, lang: ProofLang) -> bool { + is_correct_with(&dictionary(lang), word, lang) +} + +/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off, +/// short words in capitals are checked against the dictionary too, and with Ignore words that +/// contain numbers off, so are words with digits. +pub fn is_correct_with_options(word: &str, lang: ProofLang, opts: &crate::Options) -> bool { + check(&dictionary(lang), word, lang, opts) +} + +/// [`is_correct`] against a given dictionary (an empty one accepts everything). +pub fn is_correct_with(d: &AffixDict, word: &str, lang: ProofLang) -> bool { + check(d, word, lang, &crate::Options::default()) +} + +/// [`is_correct_with_options`] against a given dictionary. +fn check(d: &AffixDict, word: &str, lang: ProofLang, opts: &crate::Options) -> bool { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { + return true; + }; + if w.contains('-') { + return known(d, w, lang) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p, lang)); + } + known(d, w, lang) +} + +/// Is one word in the dictionary or the user dictionary, or (in Uruguayan Spanish) the `vos` +/// form of one of the dictionary's verbs? +fn known(d: &AffixDict, w: &str, lang: ProofLang) -> bool { + d.check(w) || crate::in_user_dictionary(&w.to_lowercase()) || (lang == ProofLang::EsUy && voseo::is_vos_form(w, |inf| d.check(inf))) +} + +#[cfg(test)] +mod tests { + use super::*; + + const VARIETIES: [ProofLang; 4] = [ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy]; + + #[test] + fn each_variety_looks_for_its_own_dictionary_first() { + let dir = std::env::temp_dir().join(format!("wordcraft-es-aliases-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let (es, mx, la, uy) = (ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy); + // Alone in the folder, each file name is found by the varieties that list it. + for (name, found_by) in [ + ("es_ES", &[es][..]), + ("es-ES", &[es]), + ("es_MX", &[mx, la]), + ("es-MX", &[mx]), + ("es_419", &[mx, la, uy]), + ("es-419", &[mx, la, uy]), + ("es_US", &[la]), + ("es_UY", &[uy]), + ("es-UY", &[uy]), + ("es_AR", &[la, uy]), + ("es-AR", &[la, uy]), + ("es_ANY", &VARIETIES), + ("es", &VARIETIES), + ] { + std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap(); + std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap(); + for lang in VARIETIES { + let found = crate::hunspell::find_in(&dir, slot(lang).names()); + assert_eq!(found.is_some(), found_by.contains(&lang), "{name}: {lang:?}"); + if let Some((aff, dic)) = found { + assert!(is_correct_with(&crate::hunspell::load(&aff, &dic).unwrap(), "canción", lang)); + } + } + std::fs::remove_file(dir.join(format!("{name}.aff"))).unwrap(); + std::fs::remove_file(dir.join(format!("{name}.dic"))).unwrap(); + } + // With them all, each variety reads its own. + for name in ["es_ES", "es_MX", "es_419", "es_UY", "es_AR", "es_ANY"] { + std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap(); + std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap(); + } + for (lang, name) in [(es, "es_ES"), (mx, "es_MX"), (la, "es_419"), (uy, "es_UY")] { + assert_eq!(crate::hunspell::find_in(&dir, slot(lang).names()).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic")))); + assert_eq!(dictionary_name(lang), name); + } + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn spanish_words_affixes_case_and_hyphens() { + test_dict::install(); + let d = dictionary(ProofLang::Es); + assert!(has_dictionary(ProofLang::Es)); + assert!(d.rule_count() > 0); + for w in [ + "canción", + "canciones", + "prueba", + "pruebas", + "corrector", + "correctores", + "también", + "México", + "Canción", + "CANCIÓN", + "teórico-práctico", + "franco-mexicano", + "ONU", + "Año", + "niño", + "pingüino", + ] { + assert!(is_correct(w, ProofLang::Es), "{w}"); + } + for w in ["prueva", "corector", "tambien", "mexico", "cancion", "teórico-prático", "cAnción", "nino"] { + assert!(!is_correct(w, ProofLang::Es), "{w}"); + } + } + + #[test] + fn varieties_keep_their_own_dictionaries() { + test_dict::install(); + // The Americas' fixture lacks `vosotros` forms and has American words; Spain's is the other way round. + assert!(is_correct("habéis", ProofLang::Es) && !is_correct("habéis", ProofLang::EsMx)); + assert!(is_correct("platicar", ProofLang::EsMx) && !is_correct("platicar", ProofLang::Es)); + assert!(is_correct("platicar", ProofLang::Es419) && is_correct("computadora", ProofLang::Es419)); + assert!(!is_correct("habéis", ProofLang::Es419) && !is_correct("habéis", ProofLang::EsUy)); + assert!(is_correct("ordenador", ProofLang::Es) && !is_correct("ordenador", ProofLang::EsMx)); + } + + /// Uruguayan Spanish accepts the `vos` forms of the dictionary's verbs, which the other + /// varieties flag. + #[test] + fn uruguayan_spanish_accepts_vos_forms() { + test_dict::install(); + for w in ["tenés", "vení", "Vení", "decime", "decímelo", "andate", "mirá", "hablás", "querés", "venís", "fijate"] { + assert!(is_correct(w, ProofLang::EsUy), "{w}"); + assert!(!is_correct(w, ProofLang::Es419) && !is_correct(w, ProofLang::Es), "{w}"); + } + for w in ["tenéz", "venì", "decimelo", "decíme", "zqbailá"] { + assert!(!is_correct(w, ProofLang::EsUy), "{w}"); + } + } + + /// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Spanish checks + /// those words against the dictionary instead of accepting them (CodeRabbit review on #3562). + #[test] + fn proofing_options_reach_the_spanish_check() { + test_dict::install(); + let defaults = crate::Options::default(); + let check_caps = crate::Options { ignore_uppercase: false, ..defaults }; + let check_digits = crate::Options { ignore_numbers: false, ..defaults }; + for lang in VARIETIES { + assert!(crate::is_correct_in_with("PRUEVA", lang, &defaults), "{lang:?}: short capitals are acronyms by default"); + assert!(!crate::is_correct_in_with("PRUEVA", lang, &check_caps), "{lang:?}"); + assert!(crate::is_correct_in_with("PRUEBA", lang, &check_caps), "{lang:?}"); + assert!(crate::is_correct_in_with("prueva2", lang, &defaults), "{lang:?}"); + assert!(!crate::is_correct_in_with("prueva2", lang, &check_digits), "{lang:?}"); + assert!(crate::is_correct_in_with("2026", lang, &check_digits), "{lang:?}: numbers alone stay correct"); + } + assert!(is_correct("PRUEVA", ProofLang::Es), "direct checks keep the defaults"); + } + + /// Maintainer review on #3562: a word typed decomposed (NFD: `e` + U+0301) is checked like its + /// composed form (NFC) and keeps its own byte range, accents and all. + #[test] + fn decomposed_accents_are_checked_like_composed_ones() { + test_dict::install(); + for lang in VARIETIES { + for text in ["esto también", "esto tambie\u{301}n", "una cancio\u{301}n y pingu\u{308}ino", "CANCIO\u{301}N", "nin\u{303}o"] { + let issues = crate::check_text(text, &[(0..text.len(), lang)]); + assert!(issues.is_empty(), "{lang:?} {text:?}: {issues:?}"); + } + // A misspelling that ends in a decomposed accent is flagged whole, at its own bytes. + let text = "esto es una prueva tambie\u{301}nn"; + let issues = crate::check_text(text, &[(0..text.len(), lang)]); + let flagged: Vec<(&str, usize, usize)> = issues.iter().map(|i| (&text[i.start..i.end], i.start, i.end)).collect(); + assert_eq!(flagged, [("prueva", 12, 18), ("tambie\u{301}nn", 19, text.len())], "{lang:?}"); + assert!(crate::suggest_in("tambie\u{301}nn", 8, lang).contains(&"también".into()), "{lang:?}"); + } + } + + #[test] + fn forbidden_words_are_rejected_in_every_case() { + test_dict::install(); + for w in ["zqprohibida", "Zqprohibida", "ZQPROHIBIDA"] { + assert!(!is_correct(w, ProofLang::Es), "{w}"); + } + } + + #[test] + fn missing_dictionary_and_ignored_inputs_are_quiet() { + let empty = AffixDict::default(); + assert!(is_correct_with(&empty, "prueva", ProofLang::Es)); + assert!(install_dictionary(ProofLang::Es, "FLAG long\n", "1\ncasa\n").is_err()); + let d = AffixDict::from_hunspell(test_dict::AFF, test_dict::DIC, &[]).unwrap(); + for w in ["", "a", "2026", "EE", "1.º", "日本語", "-", "--"] { + assert!(is_correct_with(&d, w, ProofLang::EsUy), "{w}"); + } + } + + #[test] + fn user_dictionary_applies() { + test_dict::install(); + assert!(!is_correct("zqpalabrapropia", ProofLang::EsMx)); + crate::add_word("Zqpalabrapropia"); + assert!(is_correct("zqpalabrapropia", ProofLang::EsMx)); + } + + #[test] + fn spanish_dispatch_messages_options_and_mixed_spans() { + test_dict::install(); + for tag in ["es-ES", "es-MX", "es-419", "es-AR", "es-UY", "spa"] { + assert!(crate::has_dictionary(Some(tag)), "{tag}"); + } + assert!(crate::is_correct_for("canción", Some("es-MX"))); + assert!(crate::is_correct_for("tenés", Some("es-UY")) && !crate::is_correct_for("tenés", Some("es-MX"))); + // The sentence from #3553. + let text = "esto es una prueva del corector"; + for lang in VARIETIES { + let issues = crate::check_text(text, &[(0..text.len(), lang)]); + let flagged: Vec<&str> = issues.iter().map(|i| &text[i.start..i.end]).collect(); + assert_eq!(flagged, ["prueva", "corector"], "{lang:?}: {issues:?}"); + assert!(issues.iter().all(|i| i.message == "Posible error ortográfico" && i.lang == lang), "{issues:?}"); + assert!(crate::suggest_in("prueva", 8, lang).contains(&"prueba".into()), "{lang:?}"); + assert!(crate::suggest_in("corector", 8, lang).contains(&"corrector".into()), "{lang:?}"); + } + let text = "Canción this is is wrong"; + let issues = crate::check_text(text, &[(0..8, ProofLang::Es419), (9..text.len(), ProofLang::En)]); + assert!(issues.iter().any(|i| i.lang == ProofLang::En && i.message.starts_with("Repeated word"))); + assert!(issues.iter().all(|i| i.start >= 9)); + let opts = crate::Options { ignore_uppercase: false, ..Default::default() }; + assert!(!crate::is_correct_in_with("ZQPROHIBIDA", ProofLang::Es, &opts)); + } +} diff --git a/crates/proof/src/es/suggest.rs b/crates/proof/src/es/suggest.rs new file mode 100644 index 00000000..0568ae87 --- /dev/null +++ b/crates/proof/src/es/suggest.rs @@ -0,0 +1,97 @@ +//! Spanish suggestions. The ranking is the shared dictionary-driven one +//! ([`crate::affix_suggest`]: case, MAP — the Spanish dictionaries map each vowel to +//! its accented forms and `n` to `ñ`, so `tambien` → `también` comes first —, REP, missing spaces, +//! KEY, TRY edits and a stem scan); every table comes from the variety's installed dictionary. +//! In Uruguayan Spanish a `vos` form typed without its accent is offered with it first +//! (`veni` → `vení`, `tenes` → `tenés`; [`super::voseo`]). + +use crate::ProofLang; +use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix_suggest::{MAX_LETTERS, Tuning}; + +/// Spelling suggestions for a word in `lang`'s Spanish, best first, at most `max`, in the word's +/// case. +pub fn suggest(word: &str, max: usize, lang: ProofLang) -> Vec { + suggest_in(&super::dictionary(lang), word, max, lang) +} + +/// [`suggest`] against a given dictionary. +pub fn suggest_in(d: &AffixDict, word: &str, max: usize, lang: ProofLang) -> Vec { + let mut out = Vec::new(); + let word = word.trim(); + if lang == ProofLang::EsUy + && !d.is_empty() + && word.chars().count() <= MAX_LETTERS + && let Some(v) = super::voseo::accented(&word.to_lowercase(), |inf| d.check(inf)) + { + out.push(match casing(word) { + Casing::Upper => v.to_uppercase(), + Casing::Initial => capitalize(&v), + _ => v, + }); + } + for s in crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT) { + if !out.contains(&s) { + out.push(s); + } + } + out.truncate(max); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn spanish_suggestions_use_runtime_tables_and_case() { + crate::es::test_dict::install(); + for lang in [ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy] { + for (word, want) in [ + ("tambien", "también"), + ("cancion", "canción"), + ("Cancion", "Canción"), + ("mexico", "México"), + ("nino", "niño"), + ("pinguino", "pingüino"), + ("prueva", "prueba"), + ("corector", "corrector"), + ("estoes", "esto es"), + ] { + assert!(suggest(word, 8, lang).contains(&want.into()), "{lang:?} {word}: {:?}", suggest(word, 8, lang)); + } + } + // The first suggestion for a missing accent is the accented word. + assert_eq!(suggest("tambien", 4, ProofLang::Es).first().map(String::as_str), Some("también")); + } + + #[test] + fn uruguayan_vos_forms_get_their_accent_back_first() { + crate::es::test_dict::install(); + for (word, want) in [("veni", "vení"), ("tenes", "tenés"), ("Veni", "Vení"), ("TENES", "TENÉS"), ("queres", "querés")] { + assert_eq!(suggest(word, 4, ProofLang::EsUy).first().map(String::as_str), Some(want), "{word}"); + assert!(!suggest(word, 8, ProofLang::Es419).contains(&want.into()), "{word}: no vos forms outside Uruguay"); + } + } + + #[test] + fn suggestions_stay_bounded_and_skip_forbidden_words() { + crate::es::test_dict::install(); + for lang in [ProofLang::Es, ProofLang::EsUy] { + for w in ["", "dos palabras", &"a".repeat(41)] { + assert!(suggest(w, 4, lang).is_empty(), "{w}"); + } + assert!(suggest("prueva", 0, lang).is_empty()); + assert!(suggest("veni", 0, lang).is_empty()); + assert!(suggest_in(&AffixDict::default(), "veni", 4, lang).is_empty()); + assert!(suggest("prueba", 4, lang).len() <= 4); + assert!(!suggest("prueba", 4, lang).contains(&"prueba".into())); + assert!(!suggest("zqprohibda", 8, lang).contains(&"zqprohibida".into())); + } + } + + #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(|d, w, max| suggest_in(d, w, max, ProofLang::Es)); + } +} diff --git a/crates/proof/src/es/test_dict.rs b/crates/proof/src/es/test_dict.rs new file mode 100644 index 00000000..6b3e6b1f --- /dev/null +++ b/crates/proof/src/es/test_dict.rs @@ -0,0 +1,111 @@ +//! Hand-written synthetic Spanish test data. No words, rules or suggestion tables were copied +//! from a system dictionary. This is a small fixture, not a redistributable Spanish dictionary. +//! Spain's word list has `vosotros` forms and Spain's words; the Americas' list (Mexico, Latin +//! America and Uruguay) has theirs instead, so the tests can tell the varieties' dictionaries apart. + +use crate::ProofLang; + +pub const AFF: &str = "SET UTF-8 +FLAG UTF-8 +FORBIDDENWORD F +TRY aeosrnidlctumpbgvyqhfzjñáéíóúüxkw +KEY qwertyuiop|asdfghjklñ|zxcvbnm +MAP 6 +MAP aá +MAP eé +MAP ií +MAP oó +MAP uúü +MAP nñ +REP 2 +REP v b +REP b v +SFX S Y 2 # synthetic plurals +SFX S 0 s [aeiou] +SFX S ón ones ón +SFX E Y 1 +SFX E 0 es [^aeiou] +"; + +/// Spain. +pub const DIC: &str = "32 +hablar +tener +venir +decir +andar +mirar +querer +fijar +poner +canción/S +prueba/S +corrector/E +también +México +esto +es +una +del +de +la +el +y +niño/S +pingüino/S +año/S +teórico +práctico +mexicano +franco +habéis +ordenador/E +zqprohibida/F +"; + +/// Mexico, Latin America and Uruguay. +pub const DIC_AMERICAS: &str = "33 +hablar +tener +venir +decir +andar +mirar +querer +fijar +poner +canción/S +prueba/S +corrector/E +también +México +esto +es +una +del +de +la +el +y +niño/S +pingüino/S +año/S +teórico +práctico +mexicano +franco +platicar +computadora/S +han +zqprohibida/F +"; + +/// Install the fixture for every variety: [`DIC`] for Spain, [`DIC_AMERICAS`] for Mexico, Latin +/// America and Uruguay. +pub fn install() { + for (lang, dic) in [(ProofLang::Es, DIC), (ProofLang::EsMx, DIC_AMERICAS), (ProofLang::Es419, DIC_AMERICAS), (ProofLang::EsUy, DIC_AMERICAS)] { + if let Err(e) = super::install_dictionary(lang, AFF, dic) { + panic!("test dictionary: {e}"); + } + } +} diff --git a/crates/proof/src/es/voseo.rs b/crates/proof/src/es/voseo.rs new file mode 100644 index 00000000..d9753e12 --- /dev/null +++ b/crates/proof/src/es/voseo.rs @@ -0,0 +1,155 @@ +//! The `vos` forms of verbs in Uruguayan Spanish ([`crate::ProofLang::EsUy`]), which dictionaries +//! built around `tú` may lack (#3360). Rioplatense `voseo` builds them regularly from the +//! infinitive, stress on the ending, without the stem changes `tú` has: +//! +//! - the imperative drops the infinitive's `r` and accents the vowel: `hablar` → `hablá`, +//! `tener` → `tené`, `venir` → `vení`, `decir` → `decí` (`ir` uses `andá`, from `andar`); +//! - with one pronoun after it the accent goes (`decime`, `fijate`, `andate`, `poneme`), with two +//! it comes back (`decímelo`, `mandámela`); `-ír` verbs keep theirs (`sonreíte`); +//! - the present indicative ends in `-ás`, `-és`, `-ís`: `hablás`, `tenés`, `venís`. +//! +//! A form is accepted only when the dictionary has its verb's infinitive, so nothing but these +//! endings is hard-coded. Other `vos` forms (`sos`) are ordinary dictionary words. + +/// Pronouns that come first when two follow an imperative (`decímelo`, `dáselo`). +const FIRST_PRONOUNS: [&str; 4] = ["me", "te", "se", "nos"]; +/// Pronouns that come second (or alone). +const SECOND_PRONOUNS: [&str; 6] = ["lo", "la", "le", "los", "las", "les"]; + +/// Shortest infinitive considered (`dar` and `ir` have no `vos` imperative of their own here, and +/// short stems would let nouns such as `bar` pass for verbs). +const MIN_INFINITIVE: usize = 4; + +/// Is `word` (one word, any case) a `vos` form of a verb whose infinitive `is_word` accepts? +pub fn is_vos_form(word: &str, is_word: impl Fn(&str) -> bool) -> bool { + let w = word.to_lowercase(); + if w.chars().count() > crate::affix::MAX_WORD_BYTES { + return false; + } + infinitives(&w).iter().any(|inf| inf.chars().count() >= MIN_INFINITIVE && is_word(inf)) +} + +/// The infinitives `w` (lower case) could be a `vos` form of, by its ending. +fn infinitives(w: &str) -> Vec { + let mut out = Vec::new(); + // Present indicative: `hablás`, `tenés`, `venís`. + if let Some(stem) = w.strip_suffix('s') { + imperative(stem, &mut out); + } + // Imperative: `hablá`, `tené`, `vení`. + imperative(w, &mut out); + // With one pronoun the accent goes: `decime`, `fijate`; `-ír` verbs keep it: `sonreíte`. + for p in FIRST_PRONOUNS.iter().chain(SECOND_PRONOUNS.iter()) { + if let Some(base) = w.strip_suffix(p) + && base.ends_with(['a', 'e', 'i', 'í']) + { + out.push(format!("{base}r")); + } + } + // With two the accent comes back: `decímelo`, `mandámela`. + for second in SECOND_PRONOUNS { + let Some(rest) = w.strip_suffix(second) else { continue }; + for first in FIRST_PRONOUNS { + if let Some(base) = rest.strip_suffix(first) { + imperative(base, &mut out); + } + } + } + out +} + +/// The infinitives of an imperative ending in an accented vowel: `hablá` → `hablar`, and for `í` +/// both `vení` → `venir` and `sonreí` → `sonreír`. +fn imperative(form: &str, out: &mut Vec) { + let mut chars = form.chars(); + let Some(last) = chars.next_back() else { return }; + let stem = chars.as_str(); + match last { + 'á' => out.push(format!("{stem}ar")), + 'é' => out.push(format!("{stem}er")), + 'í' => { + out.push(format!("{stem}ir")); + out.push(format!("{stem}ír")); + } + _ => {} + } +} + +/// The `vos` form `word` (lower case, without its accent) was most likely meant as, when the +/// writer left the accent out: `veni` → `vení`, `tenes` → `tenés`. `None` when it isn't one. +pub fn accented(word: &str, is_word: impl Fn(&str) -> bool) -> Option { + let (body, s) = match word.strip_suffix('s') { + Some(b) => (b, "s"), + None => (word, ""), + }; + let mut chars = body.chars(); + let accent = match chars.next_back()? { + 'a' => 'á', + 'e' => 'é', + 'i' => 'í', + _ => return None, + }; + let form = format!("{}{accent}{s}", chars.as_str()); + is_vos_form(&form, is_word).then_some(form) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn verbs(w: &str) -> bool { + ["hablar", "tener", "venir", "decir", "andar", "mirar", "sonreír", "fijar", "mandar", "poner", "bar", "querer"].contains(&w) + } + + #[test] + fn imperatives_present_and_pronouns_come_from_the_infinitive() { + for w in [ + "hablá", + "tené", + "vení", + "decí", + "andá", + "mirá", + "poné", + "hablás", + "tenés", + "venís", + "querés", + "decime", + "fijate", + "andate", + "poneme", + "sonreíte", + "sonreí", + "decímelo", + "mandámela", + "decinos", + "Vení", + "TENÉS", + ] { + assert!(is_vos_form(w, verbs), "{w}"); + } + // Wrong accents, verbs the dictionary lacks, and short stems (`bame` isn't `bar`). + for w in ["habla", "decíme", "decínos", "decimelo", "tenéz", "cantá", "bá", "bame", "", "s", "á", "més", "lo", "melo"] { + assert!(!is_vos_form(w, verbs), "{w}"); + } + } + + #[test] + fn missing_accents_are_restored() { + assert_eq!(accented("veni", verbs).as_deref(), Some("vení")); + assert_eq!(accented("tenes", verbs).as_deref(), Some("tenés")); + assert_eq!(accented("habla", verbs).as_deref(), Some("hablá")); + for w in ["canta", "casa", "", "s", "pan", "tenés"] { + assert_eq!(accented(w, verbs), None, "{w}"); + } + } + + proptest::proptest! { + #[test] + fn never_panics(s in "\\PC{0,40}") { + let _ = is_vos_form(&s, |_| true); + let _ = accented(&s, |_| true); + } + } +} diff --git a/crates/proof/src/hr/grammar.rs b/crates/proof/src/hr/grammar.rs new file mode 100644 index 00000000..517c5623 --- /dev/null +++ b/crates/proof/src/hr/grammar.rs @@ -0,0 +1,175 @@ +//! Conservative Croatian grammar and punctuation checks. +//! Handles repeated words, whitespace, capitals and duplicated punctuation; +//! intentionally avoids speculative agreement and comma rules. + +use crate::{Issue, IssueKind, ProofLang, words}; + +/// Words Croatian repeats on purpose: interjections, onomatopoeia and emphatic doubling. +const ALLOWED_REPEATS: &[&str] = + &["da", "ne", "no", "ha", "hej", "joj", "aj", "oh", "uh", "hop", "kuc", "tik", "tak", "bum", "malo", "brzo", "polako", "više", "dobro"]; +const ABBREVIATIONS: &[&str] = &[ + "npr", "itd", "itp", "tj", "tzv", "odn", "sl", "str", "br", "god", "st", "dr", "mr", "prof", "doc", "ing", "dipl", "mag", "univ", "g", "god", + "mj", "min", "sek", "čl", "st", "č", "toč", "tab", "sl", "sv", "ul", "trg", "pr", "izd", "ur", "svez", "vol", "vid", "usp", "prim", "engl", + "njem", "hrv", "lat", "tel", "mob", "e", "p", "o", "d", "m", "v", "z", "r", "pon", "uto", "sri", "čet", "pet", "sub", "ned", "sij", "velj", + "ožu", "tra", "svi", "lip", "srp", "kol", "ruj", "lis", "stu", "pro", +]; + +fn issue(start: usize, end: usize, message: String, suggestions: Vec) -> Issue { + Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::Hr } +} + +fn capitalize(w: &str) -> String { + crate::affix::capitalize(w) +} + +/// Is `prev`, before a period, an abbreviation, an initial, a number or a Roman numeral (after +/// which a lower-case word doesn't start a sentence)? +fn abbreviation_or_number(prev: &str) -> bool { + prev.chars().count() <= 1 + || prev.chars().any(|c| c.is_ascii_digit()) + || (prev.chars().all(|c| matches!(c, 'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M'))) + || ABBREVIATIONS.contains(&prev.to_lowercase().as_str()) +} + +const CLOSING: [char; 8] = ['"', '”', '’', '\'', ')', '»', ']', '\u{1}']; +const OPENING: [char; 8] = ['"', '“', '«', '(', '[', '\'', '‚', '—']; + +/// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after +/// an abbreviation), then space, with nothing but quotes and brackets around? +fn ends_sentence(gap: &str, prev: &str) -> bool { + let t = gap.trim_start_matches(CLOSING); + let Some(end) = t.chars().next() else { return false }; + if !matches!(end, '.' | '!' | '?') || t.starts_with("..") { + return false; + } + if end == '.' && abbreviation_or_number(prev) { + return false; + } + let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING); + rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| OPENING.contains(&c)) +} + +/// Grammar issues in a paragraph of Croatian text, in order. +pub fn check(text: &str) -> Vec { + check_with(text, true) +} + +/// [`check`] where `capital_first` says whether the text's first word must start a sentence with a capital +/// (false for a list item, which may be a fragment). +pub fn check_with(text: &str, capital_first: bool) -> Vec { + let ws: Vec<(usize, usize)> = words(text); + let mut v = Vec::new(); + repeated_words(text, &ws, &mut v); + spaces(text, &mut v); + sentence_capitals(text, &ws, capital_first, &mut v); + doubled_punctuation(text, &mut v); + v.sort_by_key(|i| (i.start, i.end)); + v +} + +fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if !gap.is_empty() + && gap.chars().all(char::is_whitespace) + && w0.chars().all(char::is_alphabetic) + && w0.to_lowercase() == w1.to_lowercase() + && !ALLOWED_REPEATS.contains(&w0.to_lowercase().as_str()) + { + v.push(issue(*a0, *b1, format!("Ponovljena riječ: \"{w1}\""), vec![w0.to_string()])); + } + } +} + +fn spaces(text: &str, v: &mut Vec) { + for (i, c) in text.char_indices() { + if c != ' ' { + continue; + } + let rest = text.get(i + 1..).unwrap_or(""); + let mut it = rest.chars(); + let Some(p) = it.next() else { continue }; + let after = it.next(); + let flag = match p { + ',' => true, + // `;)` and `;-)` are emoticons. + ';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')), + // ` ...` is an ellipsis, ` .5` a number. + '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), + // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. + ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')), + _ => false, + }; + if flag { + v.push(issue(i, i + 1 + p.len_utf8(), "Razmak ispred interpunkcijskog znaka".into(), vec![p.to_string()])); + } + } + for (i, _) in text.match_indices(" ") { + if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { + continue; + } + v.push(issue(i, i + 2, "Dvostruki razmak".into(), vec![" ".into()])); + } +} + +fn sentence_capitals(text: &str, ws: &[(usize, usize)], capital_first: bool, v: &mut Vec) { + // List items (`– maçãs,`) may start in lower case. + let list_item = text.trim_end().ends_with([',', ';']); + for (k, &(a, b)) in ws.iter().enumerate() { + let Some(w) = text.get(a..b) else { continue }; + let starts = if k == 0 { + let before = text.get(..a).unwrap_or(""); + capital_first && !list_item && before.chars().all(|c| c.is_whitespace() || OPENING.contains(&c)) + } else { + let Some(&(pa, pb)) = ws.get(k - 1) else { continue }; + ends_sentence(text.get(pb..a).unwrap_or(""), text.get(pa..pb).unwrap_or("")) + }; + if starts && w.chars().next().is_some_and(char::is_lowercase) && w.chars().all(char::is_alphabetic) { + v.push(issue(a, b, "Rečenica treba započeti velikim slovom".into(), vec![capitalize(w)])); + } + } +} + +fn doubled_punctuation(text: &str, v: &mut Vec) { + let bytes = text.as_bytes(); + let mut i = 0; + while let Some(&b) = bytes.get(i) { + if !matches!(b, b',' | b'.' | b';') { + i += 1; + continue; + } + let run = bytes.get(i..).map_or(0, |r| r.iter().take_while(|&&x| x == b).count()); + let flag = match b { + b'.' => run == 2, + _ => run >= 2, + }; + if flag { + let end = i + run; + let suggestions: Vec = match b { + b'.' => vec![".".into(), "…".into()], + _ => vec![char::from(b).to_string()], + }; + v.push(issue(i, end, "Ponovljen interpunkcijski znak".into(), suggestions)); + } + i += run.max(1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn checks_croatian_punctuation() { + let t = "Ovo ovo je primjer , s dva razmaka.."; + let issues = check(t); + assert!(issues.iter().any(|i| i.message.starts_with("Ponovljena riječ"))); + assert!(issues.iter().any(|i| i.message == "Dvostruki razmak")); + assert!(issues.iter().all(|i| i.lang == ProofLang::Hr)); + } + #[test] + fn abbreviations_do_not_trigger_capitals() { + let issues = check("To je npr. sasvim dobar primjer."); + assert!(!issues.iter().any(|i| i.message == "Rečenica treba započeti velikim slovom")); + } +} diff --git a/crates/proof/src/hr/mod.rs b/crates/proof/src/hr/mod.rs new file mode 100644 index 00000000..53edf067 --- /dev/null +++ b/crates/proof/src/hr/mod.rs @@ -0,0 +1,130 @@ +//! Croatian spelling and conservative grammar support. +//! Uses an installed hr_HR Hunspell .aff/.dic pair (not bundled). +//! The engine decodes ISO-8859-2 dictionaries at load time. +//! Suggestions prioritize č/ć, đ, š, ž and common one-character mistakes. +//! Hyphenation remains the host engine's fallback until Croatian patterns are supplied. + +pub mod grammar; +mod suggest; +#[cfg(test)] +pub(crate) mod test_dict; + +use std::sync::Arc; + +use crate::affix::AffixDict; +use crate::slot::DictSlot; + +pub use suggest::suggest; + +/// File names of a Croatian Hunspell dictionary. +const NAMES: &[&str] = &["hr_HR", "hr-HR", "hr"]; + +/// The Croatian dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Croatian"); + +/// The Croatian dictionary: the installed Hunspell `hr_HR` (read on first use), one given with +/// [`install_dictionary`], or an empty one when none is available (Croatian spelling is off +/// then; see [`has_dictionary`]). +pub fn dictionary() -> Arc { + DICT.get() +} + +/// Use this Croatian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to +/// UTF-8), for example one the user picked. On error the current dictionary stays. +pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { + DICT.install(aff, dic) +} + +/// Is a Croatian dictionary available (spelling is checked)? +pub fn has_dictionary() -> bool { + DICT.has_dictionary() +} + +/// Is `word` spelled correctly in Croatian? Numbers, single letters, short words in capitals +/// (acronyms: `HRT`, `RH`), words with digits and words in other scripts are accepted, and so +/// is every word when no Croatian dictionary is installed. +pub fn is_correct(word: &str) -> bool { + is_correct_with(&dictionary(), word) +} + +/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off, +/// short words in capitals are checked against the dictionary too, and with Ignore words that +/// contain numbers off, so are words with digits. +pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool { + check(&dictionary(), word, opts) +} + +/// [`is_correct`] against a given dictionary (an empty one accepts everything). +pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { + check(d, word, &crate::Options::default()) +} + +/// [`is_correct_with_options`] against a given dictionary. +fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { + return true; + }; + if w.contains('-') { + return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + } + known(d, w) +} + +/// Is one word (no hyphens) in the dictionary or the user dictionary? +fn known(d: &AffixDict, w: &str) -> bool { + d.check(w) || crate::in_user_dictionary(&w.to_lowercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_dictionary_loads() { + test_dict::install(); + let d = dictionary(); + assert!(has_dictionary()); + assert!(d.len() > 15); + assert!(d.rule_count() > 0); + assert!(!d.map().is_empty() && !d.rep().is_empty() && !d.try_chars().is_empty()); + } + + #[test] + fn correct_words_and_affixes() { + test_dict::install(); + for w in ["kuća", "kuće", "računalo", "čovjek", "riječ", "dijete", "Hrvatska", "čokolada", "škola", "škole", "HRT", "2026"] { + assert!(is_correct(w), "{w}"); + } + } + + #[test] + fn misspellings_are_flagged() { + test_dict::install(); + for w in ["kuca", "racunalo", "covjek", "rijec", "dijeteh", "skolla"] { + assert!(!is_correct(w), "{w}"); + } + } + + #[test] + fn missing_dictionary_is_silent() { + assert!(is_correct_with(&AffixDict::default(), "nepostojećariječ")); + } + + /// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Croatian checks + /// those words against the dictionary instead of accepting them. + #[test] + fn proofing_options_reach_the_croatian_check() { + test_dict::install(); + let lang = crate::ProofLang::Hr; + let defaults = crate::Options::default(); + let check_caps = crate::Options { ignore_uppercase: false, ..defaults }; + let check_digits = crate::Options { ignore_numbers: false, ..defaults }; + assert!(crate::is_correct_in_with("ŠKOLLA", lang, &defaults), "short capitals are acronyms by default"); + assert!(!crate::is_correct_in_with("ŠKOLLA", lang, &check_caps)); + assert!(crate::is_correct_in_with("ŠKOLA", lang, &check_caps)); + assert!(crate::is_correct_in_with("školla2", lang, &defaults)); + assert!(!crate::is_correct_in_with("školla2", lang, &check_digits)); + assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); + assert!(is_correct("ŠKOLLA"), "direct checks keep the defaults"); + } +} diff --git a/crates/proof/src/hr/suggest.rs b/crates/proof/src/hr/suggest.rs new file mode 100644 index 00000000..d8c19b99 --- /dev/null +++ b/crates/proof/src/hr/suggest.rs @@ -0,0 +1,63 @@ +//! Croatian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with +//! the words Croatian runs into their neighbours (`neznam` → `ne znam`), the letter groups of +//! Croatian diacritics and letters to try, for a dictionary without MAP or TRY lines. + +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; + +/// Short words often run into the next one (`ne znam`, `na primjer`, `da li`). +const LEADING_WORDS: &[&str] = &[ + "a", "ali", "i", "ili", "da", "do", "od", "na", "ne", "ni", "o", "po", "pod", "pri", "sa", "s", "se", "su", "u", "uz", "za", "iz", "je", "će", + "bi", "bih", "što", "koji", "koja", "koje", "kao", "kad", "dok", "bez", "pred", "nad", +]; +const TRAILING_WORDS: &[&str] = + &["je", "su", "se", "ga", "mi", "ti", "mu", "joj", "ih", "im", "li", "će", "ću", "bi", "bih", "i", "a", "u", "na", "za", "od", "do"]; + +/// What Croatian adds to its dictionary's tables. +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "aeiourntsjkvlmdpbgczčćšžđfh", + map: &["cčć", "dđ", "sš", "zž"], + ..Tuning::DEFAULT +}; + +/// Spelling suggestions for a Croatian word, best first, at most `max`, in the word's letter case. +pub fn suggest(word: &str, max: usize) -> Vec { + suggest_in(&super::dictionary(), word, max) +} + +/// [`suggest`] against a given dictionary. +pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { + crate::affix_suggest::suggest_in(d, word, max, &TUNING) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); + } + /// A dictionary without MAP lines still gets the Croatian diacritic groups: `kuca` → `kuća` + /// first, before a typo that costs more (`kuha`). + #[test] + fn diacritics_without_map_lines() { + let d = AffixDict::from_hunspell("", "2\nkuća\nkuha\n", &[]).unwrap(); + assert_eq!(suggest_in(&d, "kuca", 1), ["kuća"]); + } + #[test] + fn croatian_suggestions() { + crate::hr::test_dict::install(); + for (wrong, right) in [("kuca", "kuća"), ("racunalo", "računalo"), ("covjek", "čovjek"), ("rijec", "riječ"), ("skolla", "škola")] { + let result = suggest(wrong, 10); + assert!(result.iter().any(|s| s == right), "{wrong} -> {result:?}, expected {right}"); + } + } + #[test] + fn bounded() { + crate::hr::test_dict::install(); + assert!(suggest("", 5).is_empty()); + assert!(suggest("kuca", 0).is_empty()); + assert!(suggest(&"a".repeat(41), 5).is_empty()); + } +} diff --git a/crates/proof/src/hr/test_dict.rs b/crates/proof/src/hr/test_dict.rs new file mode 100644 index 00000000..038069e8 --- /dev/null +++ b/crates/proof/src/hr/test_dict.rs @@ -0,0 +1,42 @@ +//! Small synthetic UTF-8 Croatian Hunspell dictionary for deterministic tests. +pub const AFF: &str = "SET UTF-8 +TRY abcčćdđefghijklmnoprsštuvzž +MAP 5 +MAP cčć +MAP dđ +MAP sš +MAP zž +MAP iije +REP 3 +REP c č +REP dj đ +REP je ije +SFX A Y 2 +SFX A a e a +SFX A a i a +"; +pub const DIC: &str = "18 +kuća/A +računalo +čovjek +riječ +dijete +Hrvatska +čokolada +škola/A +more +sunce +dobar +hrvatski +jezik +tekst +pogreška +pravopis +provjera +računala +"; +pub fn install() { + if let Err(e) = super::install_dictionary(AFF, DIC) { + panic!("test dictionary: {e}"); + } +} diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs new file mode 100644 index 00000000..98f8c584 --- /dev/null +++ b/crates/proof/src/hunspell.rs @@ -0,0 +1,809 @@ +//! Hunspell dictionaries installed on the system, read at runtime. +//! +//! WordCraft doesn't bundle large spelling dictionaries. For languages with an affix dictionary +//! (Polish, Russian), it reads the standard Hunspell `.aff`/`.dic` pair installed by the operating system, +//! a package manager or LibreOffice, with [`crate::affix::AffixDict::from_hunspell`]. When none is +//! installed the language has no spelling check (no marks, no errors), like Word without the +//! language's proofing tools. +//! +//! Search order: the folder set with [`set_dictionary_dir`], the directory in +//! `WORDCRAFT_DICTIONARIES`, WordCraft's own `dictionaries` folder under the user's application +//! data (on every desktop system), then the usual system locations. On the web there are none. + +use std::path::{Path, PathBuf}; +use std::sync::RwLock; + +use crate::affix::AffixDict; + +/// Largest `.aff` or `.dic` file read (real ones are a few MB). +const MAX_FILE: u64 = 64 << 20; + +/// An extra directory searched first (a setting), set by the front end. +static EXTRA_DIR: RwLock> = RwLock::new(None); + +/// Why an installed dictionary failed to load, by `.dic` name (`pt_BR`): the file and the reason. +/// Without it a file that is there but unreadable looks the same as no file (#2830). +static LOAD_ERRORS: RwLock> = RwLock::new(Vec::new()); + +/// Remember (or, for `Ok`, forget) why the dictionary `key` found at `dic` failed to load, and +/// hand the result back unchanged. +pub fn note_load(key: &str, dic: &Path, result: Result) -> Result { + let mut errors = LOAD_ERRORS.write().unwrap_or_else(std::sync::PoisonError::into_inner); + errors.retain(|(k, _, _)| k != key); + if let Err(e) = &result { + errors.push((key.to_string(), dic.to_path_buf(), e.clone())); + } + result +} + +/// Forget the load error of the dictionary `key` (none found, or it is about to be searched for again). +pub fn clear_load_error(key: &str) { + LOAD_ERRORS.write().unwrap_or_else(std::sync::PoisonError::into_inner).retain(|(k, _, _)| k != key); +} + +/// The `.dic` file and the reason of the last failed load of the dictionary `key`, if any. +pub fn load_error(key: &str) -> Option<(PathBuf, String)> { + LOAD_ERRORS.read().unwrap_or_else(std::sync::PoisonError::into_inner).iter().find(|(k, _, _)| k == key).map(|(_, p, e)| (p.clone(), e.clone())) +} + +/// Search `dir` before the system locations (a "dictionaries folder" setting); `None` clears it. +/// Takes effect for dictionaries not loaded yet. +pub fn set_dictionary_dir(dir: Option) { + *EXTRA_DIR.write().unwrap_or_else(std::sync::PoisonError::into_inner) = dir; + LOAD_ERRORS.write().unwrap_or_else(std::sync::PoisonError::into_inner).clear(); // they belong to the old search path +} + +/// The LibreOffice extension directories a Hunspell dictionary name may live in: LibreOffice's +/// IDs are per dictionary — Polish is `dict-pl`, Brazilian Portuguese `dict-pt-BR` — so both +/// the primary-language folder and the dashed full name are tried. +fn lo_ext_dirs(name: &str) -> Vec { + let primary = name.split(['-', '_']).next().unwrap_or(name); + let dashed = name.replace('_', "-"); + let first = format!("dict-{primary}"); + let full = format!("dict-{dashed}"); + if full == first { vec![first] } else { vec![first, full] } +} + +/// Directories that may hold Hunspell dictionaries, most specific first. `lo_exts`, when a +/// language is being searched for, adds LibreOffice's extension folders for it (`dict-pt-BR`). +pub fn search_dirs(lo_exts: Option<&[String]>) -> Vec { + let mut dirs: Vec = Vec::new(); + if let Some(d) = EXTRA_DIR.read().unwrap_or_else(std::sync::PoisonError::into_inner).clone() { + dirs.push(d); + } + if cfg!(target_arch = "wasm32") { + return dirs; + } + if let Some(d) = std::env::var_os("WORDCRAFT_DICTIONARIES").filter(|v| !v.is_empty()) { + dirs.push(PathBuf::from(d)); + } + // LibreOffice keeps each language under `…/share/extensions/`. + let lo_dirs = + |prefix: &str| -> Vec { lo_exts.map_or_else(Vec::new, |exts| exts.iter().map(|e| PathBuf::from(prefix).join(e)).collect()) }; + let home = std::env::var_os("HOME").filter(|v| !v.is_empty()).map(PathBuf::from); + if cfg!(windows) { + for var in ["LOCALAPPDATA", "APPDATA"] { + if let Some(base) = std::env::var_os(var).filter(|v| !v.is_empty()).map(PathBuf::from) { + dirs.push(base.join("WordCraft").join("dictionaries")); + dirs.push(base.join("hunspell")); + } + } + for var in ["ProgramFiles", "ProgramFiles(x86)"] { + if let Some(base) = std::env::var_os(var).filter(|v| !v.is_empty()).map(PathBuf::from) { + dirs.extend(lo_dirs(&base.join("LibreOffice").join("share").join("extensions").to_string_lossy())); + } + } + // If LibreOffice is installed on another drive (e.g. D:\Program Files\LibreOffice), check drives C..Z + for drive in b'C'..=b'Z' { + let drive_char = drive as char; + for pf in ["Program Files", "Program Files (x86)"] { + let lo_base = PathBuf::from(format!(r"{drive_char}:\{pf}\LibreOffice\share\extensions")); + if lo_base.is_dir() { + dirs.extend(lo_dirs(&lo_base.to_string_lossy())); + } + } + } + } else if cfg!(target_os = "macos") { + if let Some(h) = &home { + // An app started from Finder or the Dock doesn't see `WORDCRAFT_DICTIONARIES`; this + // folder needs no setting (#1063). + dirs.push(h.join("Library/Application Support/WordCraft/dictionaries")); + dirs.push(h.join("Library/Spelling")); + } + dirs.push(PathBuf::from("/Library/Spelling")); + dirs.push(PathBuf::from("/opt/homebrew/share/hunspell")); + dirs.push(PathBuf::from("/usr/local/share/hunspell")); + dirs.extend(lo_dirs("/Applications/LibreOffice.app/Contents/Resources/extensions")); + } else { + // Nor does one started from a desktop launcher (#1063). + if let Some(data) = xdg_data_home(std::env::var_os("XDG_DATA_HOME"), home.as_deref()) { + dirs.push(data.join("wordcraft").join("dictionaries")); + } + if let Some(h) = &home { + dirs.push(h.join(".local/share/hunspell")); + } + for d in ["/usr/share/hunspell", "/usr/share/myspell", "/usr/share/myspell/dicts", "/usr/local/share/hunspell", "/usr/local/share/myspell"] { + dirs.push(PathBuf::from(d)); + } + for d in ["/usr/lib/libreoffice/share/extensions", "/opt/libreoffice/share/extensions"] { + dirs.extend(lo_dirs(d)); + } + } + dirs +} + +/// The user's data folder on Linux and the BSDs: `$XDG_DATA_HOME` when it is an absolute path (the +/// XDG Base Directory spec ignores a relative one), else `~/.local/share`. +fn xdg_data_home(xdg: Option, home: Option<&Path>) -> Option { + xdg.map(PathBuf::from).filter(|p| p.is_absolute()).or_else(|| home.map(|h| h.join(".local/share"))) +} + +/// The first `.aff`/`.dic` pair found for any of `names` (`pl_PL`, `pl`), searching +/// [`search_dirs`] (including LibreOffice's extension folder for the first name). +pub fn find(names: &[&str]) -> Option<(PathBuf, PathBuf)> { + let lo = names.first().copied().map(lo_ext_dirs); + search_dirs(lo.as_deref()).into_iter().find_map(|dir| find_in(&dir, names)) +} + +/// Every `.aff`/`.dic` pair for `names`, name by name, each in every folder +/// ([`search_dirs`], the dictionaries folder first): a file with the first name in a system +/// folder comes before one with the second name in the dictionaries folder. A caller that can't +/// read one tries the next. +pub fn candidates(names: &[&str]) -> Vec<(PathBuf, PathBuf)> { + let lo = names.first().copied().map(lo_ext_dirs); + candidates_in(&search_dirs(lo.as_deref()), names) +} + +/// [`candidates`] in `dirs`. +pub fn candidates_in(dirs: &[PathBuf], names: &[&str]) -> Vec<(PathBuf, PathBuf)> { + let mut out: Vec<(PathBuf, PathBuf)> = Vec::new(); + for n in names { + for dir in dirs { + let aff = dir.join(format!("{n}.aff")); + let dic = dir.join(format!("{n}.dic")); + if aff.is_file() && dic.is_file() && !out.iter().any(|(_, d)| *d == dic) { + out.push((aff, dic)); + } + } + } + out +} + +/// The first `.aff`/`.dic` pair for any of `names` in `dir`. +pub fn find_in(dir: &Path, names: &[&str]) -> Option<(PathBuf, PathBuf)> { + names.iter().find_map(|n| { + let aff = dir.join(format!("{n}.aff")); + let dic = dir.join(format!("{n}.dic")); + (aff.is_file() && dic.is_file()).then_some((aff, dic)) + }) +} + +/// Read a dictionary file: a regular file of at most [`MAX_FILE`] bytes. The read itself is +/// bounded, so a file that grows after its size is checked (or a filesystem that reports a wrong +/// size) can't make it read more (#3661). +pub(crate) fn read_capped(p: &Path) -> Result, String> { + use std::io::Read; + let err = |e: std::io::Error| format!("{}: {e}", p.display()); + let f = std::fs::File::open(p).map_err(err)?; + let meta = f.metadata().map_err(err)?; + if !meta.is_file() { + return Err(format!("{}: not a regular file", p.display())); + } + if meta.len() > MAX_FILE { + return Err(format!("{}: too large ({} bytes)", p.display(), meta.len())); + } + let mut buf = Vec::new(); + f.take(MAX_FILE + 1).read_to_end(&mut buf).map_err(err)?; + if buf.len() as u64 > MAX_FILE { + return Err(format!("{}: too large", p.display())); + } + Ok(buf) +} + +/// Read a Hunspell pair from disk (size-capped), decoded per the `.aff`'s `SET` line. +pub fn load(aff: &Path, dic: &Path) -> Result { + let (aff_text, dic_text) = decode(&read_capped(aff)?, &read_capped(dic)?)?; + let source = format!("Hunspell dictionary {}", dic.display()); + AffixDict::from_hunspell(&aff_text, &dic_text, &[source.as_str()]) +} + +/// [`load`] for an English dictionary: directives that only matter for other languages +/// (compounding, output conversion) are ignored rather than refused +/// ([`AffixDict::from_hunspell_english`]). +pub fn load_english(aff: &Path, dic: &Path) -> Result { + let (aff_text, dic_text) = decode(&read_capped(aff)?, &read_capped(dic)?)?; + let source = format!("Hunspell dictionary {}", dic.display()); + AffixDict::from_hunspell_english(&aff_text, &dic_text, &[source.as_str()]) +} + +/// ISO 8859-1: every byte is the code point of the same value (the original VERO pt_BR uses it). +fn decode_latin1(bytes: &[u8]) -> String { + bytes.iter().map(|&b| char::from(b)).collect() +} + +/// ISO 8859-9 (Latin-5, Turkish): ISO 8859-1 with Ğ İ Ş ğ ı ş in place of Ð Ý Þ ð ý þ. The +/// encoding of older Turkish dictionaries (`SET ISO8859-9`). +fn decode_latin5(bytes: &[u8]) -> String { + bytes + .iter() + .map(|&b| match b { + 0xd0 => 'Ğ', + 0xdd => 'İ', + 0xde => 'Ş', + 0xf0 => 'ğ', + 0xfd => 'ı', + 0xfe => 'ş', + _ => char::from(b), + }) + .collect() +} + +/// ISO 8859-2 code points 0xA0..=0xFF (0x00..=0x9F map to themselves). +const ISO_8859_2_HIGH: &str = "\u{a0}Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\u{ad}ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙"; + +fn decode_latin2(bytes: &[u8]) -> String { + let high: Vec = ISO_8859_2_HIGH.chars().collect(); + bytes.iter().map(|&b| if b < 0xa0 { char::from(b) } else { high.get(usize::from(b - 0xa0)).copied().unwrap_or('\u{fffd}') }).collect() +} + +/// KOI8-R code points 0x80..=0xFF (0x00..=0x7F are ASCII): the encoding of older Russian +/// dictionaries. +const KOI8_R_HIGH: &str = + "─│┌┐└┘├┤┬┴┼▀▄█▌▐░▒▓⌠■∙√≈≤≥\u{a0}⌡°²·÷═║╒ё╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡Ё╢╣╤╥╦╧╨╩╪╫╬©юабцдефгхийклмнопярстужвьызшэщчъЮАБЦДЕФГХИЙКЛМНОПЯРСТУЖВЬЫЗШЭЩЧЪ"; + +/// Windows-1251 code points 0x80..=0xFF (0x98 is unassigned): the other one Russian dictionaries +/// use. +const CP1251_HIGH: &str = "ЂЃ‚ѓ„…†‡€‰Љ‹ЊЌЋЏђ‘’“”•–—\u{fffd}™љ›њќћџ\u{a0}ЎўЈ¤Ґ¦§Ё©Є«¬\u{ad}®Ї°±Ііґµ¶·ё№є»јЅѕїАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя"; + +/// Decode a single-byte encoding whose bytes 0x80..=0xFF are `high` (ASCII below). +fn decode_high(bytes: &[u8], high: &str) -> String { + let high: Vec = high.chars().collect(); + bytes.iter().map(|&b| if b < 0x80 { char::from(b) } else { high.get(usize::from(b - 0x80)).copied().unwrap_or('\u{fffd}') }).collect() +} + +/// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names +/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the +/// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R +/// and Windows-1251 the ones older Russian dictionaries use). +/// +/// Under a UTF-8 `SET` with single-byte flags (no `FLAG` line, or `FLAG DEFAULT`), flags are bytes, +/// as in Hunspell: the Hungarian dictionary writes its affix flags and flag aliases as bytes above +/// 0x7F that aren't UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF, +/// one flag per byte); the words, affixes and conditions around them are UTF-8. With any other +/// `FLAG` (`UTF-8`, `num`, `long`) flags are text and the whole file is UTF-8: `SFX 1234` keeps +/// its number (the Turkish dictionary's `FLAG num`). +pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { + let (set, flag) = directives(aff); + let set = set.unwrap_or_else(|| "UTF-8".into()); + match set.as_str() { + "UTF-8" | "UTF8" => { + if flag.as_deref().is_some_and(|f| f != "DEFAULT") { + let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}")); + return Ok((text(aff, ".aff")?, text(dic, ".dic")?)); + } + Ok((decode_utf8_aff(aff)?, decode_utf8_dic(dic)?)) + } + "ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))), + "ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))), + "ISO8859-9" | "ISO-8859-9" => Ok((decode_latin5(aff), decode_latin5(dic))), + "KOI8-R" | "KOI8R" => Ok((decode_high(aff, KOI8_R_HIGH), decode_high(dic, KOI8_R_HIGH))), + "CP1251" | "WINDOWS-1251" | "MICROSOFT-CP1251" => Ok((decode_high(aff, CP1251_HIGH), decode_high(dic, CP1251_HIGH))), + other => Err(format!("unsupported encoding {other}")), + } +} + +/// The first arguments of the `.aff`'s first `SET` and `FLAG` lines, in capitals, found in one pass +/// over the whole file: a `FLAG UTF-8` after the rules still decides how the `.dic` flags are read, +/// as in Hunspell. Only the first argument counts: `FLAG UTF-8 # Unicode flags` is UTF-8. +fn directives(aff: &[u8]) -> (Option, Option) { + let (mut set, mut flag) = (None, None); + for l in aff.split(|&b| b == b'\n') { + if set.is_some() && flag.is_some() { + break; + } + let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l); + let slot = match l.first() { + Some(b'S') if set.is_none() => (&mut set, &b"SET"[..]), + Some(b'F') if flag.is_none() => (&mut flag, &b"FLAG"[..]), + _ => continue, + }; + let arg = l + .strip_prefix(slot.1) + .filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t'))) + .and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty())); + if let Some(a) = arg { + *slot.0 = Some(String::from_utf8_lossy(a).to_ascii_uppercase()); + } + } + (set, flag) +} + +/// Directives whose one argument is a flag. +const FLAG_DIRECTIVES: &[&str] = &[ + "CIRCUMFIX", + "COMPOUNDBEGIN", + "COMPOUNDEND", + "COMPOUNDFLAG", + "COMPOUNDFORBIDFLAG", + "COMPOUNDLAST", + "COMPOUNDFIRST", + "COMPOUNDMIDDLE", + "COMPOUNDPERMITFLAG", + "COMPOUNDROOT", + "FORBIDDENWORD", + "FORCEUCASE", + "GENERATE", + "HU_KOTOHANGZO", + "KEEPCASE", + "LEMMA_PRESENT", + "NEEDAFFIX", + "NOSUGGEST", + "ONLYINCOMPOUND", + "ONLYROOT", + "PSEUDOROOT", + "SUBSTANDARD", + "WARN", +]; + +/// Directives that only describe the dictionary; a non-UTF-8 byte in them (the Hungarian +/// dictionary's `NAME` is ISO 8859-2) is replaced instead of failing the load. +const METADATA_DIRECTIVES: &[&str] = &["NAME", "HOME", "VERSION", "LANG", "AM"]; + +/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). `what` +/// names the line in an error. +fn text_and_flags(field: &[u8], what: impl Fn() -> String) -> Result { + let mut slash = None; + let mut i = 0; + while let Some(&b) = field.get(i) { + if b == b'\\' && field.get(i + 1) == Some(&b'/') { + i += 2; + continue; + } + if b == b'/' { + slash = Some(i); + break; + } + i += 1; + } + let (text, flags) = match slash { + Some(k) => (field.get(..k).unwrap_or_default(), field.get(k..).unwrap_or_default()), + None => (field, &[][..]), + }; + let mut out = std::str::from_utf8(text).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(); + out.push_str(&decode_latin1(flags)); + Ok(out) +} + +/// Split a line into fields on spaces and tabs, keeping the separators (so a line rebuilt from +/// its parts is the original). +fn fields(line: &[u8]) -> Vec<&[u8]> { + let mut out = Vec::new(); + let mut start = 0; + let mut in_sep = line.first().is_some_and(|b| matches!(b, b' ' | b'\t')); + for (i, b) in line.iter().enumerate() { + let sep = matches!(b, b' ' | b'\t'); + if sep != in_sep { + out.push(line.get(start..i).unwrap_or_default()); + start = i; + in_sep = sep; + } + } + out.push(line.get(start..).unwrap_or_default()); + out +} + +/// A UTF-8 `.aff` with single-byte flags (see [`decode`]). +fn decode_utf8_aff(aff: &[u8]) -> Result { + let mut out = String::with_capacity(aff.len()); + for (n, line) in aff.split(|&b| b == b'\n').enumerate() { + if n > 0 { + out.push('\n'); + } + if let Ok(s) = std::str::from_utf8(line) + && s.is_ascii() + { + out.push_str(s); + continue; + } + let parts = fields(line); + // Field index among the non-separator parts. + let words: Vec = + parts.iter().enumerate().filter(|(_, p)| !p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).map(|(i, _)| i).collect(); + let dir = words + .first() + .and_then(|&i| parts.get(i)) + .map(|p| String::from_utf8_lossy(p).trim_start_matches('\u{feff}').to_string()) + .unwrap_or_default(); + let what = || format!(".aff line {}", n + 1); + if dir.starts_with('#') || METADATA_DIRECTIVES.contains(&dir.as_str()) { + out.push_str(&String::from_utf8_lossy(line)); + continue; + } + // A trailing `# comment` (the Hungarian `AF` lines number themselves) is never a word or + // a flag; from there on the line is kept as text. It can only start after the directive's + // own fields, since a field such as an affix condition may itself begin with `#`. + let field_text = |k: usize| words.get(k).and_then(|&w| parts.get(w)).map(|p| String::from_utf8_lossy(p).into_owned()); + let first_free = comment_field(&dir, field_text); + let comment_at = words.iter().skip(first_free).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + first_free); + // The field index of the next non-separator part. + let mut next_field = 0; + for (i, p) in parts.iter().enumerate() { + let field = (!p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).then(|| { + next_field += 1; + next_field - 1 + }); + let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w)); + let decoded = match (dir.as_str(), field) { + _ if in_comment => String::from_utf8_lossy(p).into_owned(), + ("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => decode_latin1(p), + ("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, what)?, + // A field that names one flag keeps its first byte, as Hunspell's `decode_flag` + // does: `SFX é` is the flag 0xC3, the first of the two flags `kert/é` carries. + ("PFX" | "SFX", Some(1)) => decode_latin1(p.get(..1).unwrap_or_default()), + (d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => decode_latin1(p.get(..1).unwrap_or_default()), + _ => std::str::from_utf8(p).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(), + }; + out.push_str(&decoded); + } + } + Ok(out) +} + +/// The first field of an `.aff` line where a trailing `# comment` may start: after the +/// directive's own fields. A table's count line (`REP 12`, `PFX A Y 3`) has fewer fields than its +/// rows (`REP a b`, `PFX A 0 x .`). `field(k)` is the line's `k`-th field. +fn comment_field(dir: &str, field: impl Fn(usize) -> Option) -> usize { + let number = |k: usize| field(k).is_some_and(|f| f.parse::().is_ok()); + match dir { + "PFX" | "SFX" if field(2).is_some_and(|f| f == "Y" || f == "N") && number(3) => 4, + "PFX" | "SFX" => 5, + "REP" | "ICONV" | "OCONV" | "PHONE" | "MAP" | "BREAK" | "AF" | "AM" | "COMPOUNDRULE" | "CHECKCOMPOUNDPATTERN" + if number(1) && field(2).is_none_or(|f| f.starts_with('#')) => + { + 2 + } + "REP" | "ICONV" | "OCONV" | "PHONE" => 3, + "CHECKCOMPOUNDPATTERN" => 4, + _ => 2, + } +} + +/// Where the `word/flags` entry of a `.dic` line ends, as Hunspell reads it: at a tab or at the +/// first morphological field (a space or tab, two characters and `:`, as in ` po:noun`), whichever +/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the end of the line +/// (`kot/B 12` has the flags `B 12`, as in Hunspell). +pub(crate) fn dic_entry_end(line: &[u8]) -> usize { + let blank = |b: &u8| matches!(b, b' ' | b'\t'); + let morph = line.windows(4).position(|w| matches!(w, [a, b, c, b':'] if blank(a) && !blank(b) && !blank(c))); + let tab = line.iter().position(|&b| b == b'\t'); + let end = match (morph, tab) { + (Some(m), Some(t)) => m.min(t), + (m, t) => match m.or(t) { + Some(e) => e, + None => line.len(), + }, + }; + // Blanks before the morphological field or the line end aren't part of the entry. + line.get(..end).map_or(end, |e| e.iter().rposition(|b| !blank(b)).map_or(0, |k| k + 1)) +} + +/// A UTF-8 `.dic` with single-byte flags: the word is UTF-8, the flags after `/` are bytes. +fn decode_utf8_dic(dic: &[u8]) -> Result { + let mut out = String::with_capacity(dic.len()); + for (n, line) in dic.split(|&b| b == b'\n').enumerate() { + if n > 0 { + out.push('\n'); + } + if let Ok(s) = std::str::from_utf8(line) + && s.split_once('/').is_none_or(|(_, flags)| flags.is_ascii()) + { + out.push_str(s); + continue; + } + let what = || format!(".dic line {}", n + 1); + // The first field is `word/flags` (it may hold spaces, see [`dic_entry_end`]); + // morphological fields after it are UTF-8 text. + let end = dic_entry_end(line); + out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), what)?); + let rest = line.get(end..).unwrap_or_default(); + out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{}: not UTF-8: {e}", what()))?); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// WordCraft's own dictionaries folder under the user's application data is searched on every + /// desktop system, after the setting and before the shared system locations (#1063). + #[test] + fn the_app_folder_comes_before_the_system_locations() { + if cfg!(target_arch = "wasm32") { + return; + } + let user = if cfg!(windows) { "LOCALAPPDATA" } else { "HOME" }; + if std::env::var_os(user).filter(|v| !v.is_empty()).is_none() { + return; // no user folder to derive it from + } + let dirs = search_dirs(None); + let at = |suffix: &str| dirs.iter().position(|d| d.ends_with(suffix)); + let (app, shared) = if cfg!(windows) { + ("WordCraft/dictionaries", "hunspell") + } else if cfg!(target_os = "macos") { + ("Application Support/WordCraft/dictionaries", "/Library/Spelling") + } else { + ("wordcraft/dictionaries", "/usr/share/hunspell") + }; + assert!(matches!((at(app), at(shared)), (Some(app), Some(shared)) if app < shared), "{dirs:?}"); + } + + /// `$XDG_DATA_HOME` counts only as an absolute path; otherwise (unset, empty or relative) the + /// spec's default `~/.local/share` applies, and with no home there is no folder. + #[test] + fn xdg_data_home_follows_the_spec() { + let home = Path::new("/home/a"); + assert_eq!(xdg_data_home(Some("/data".into()), Some(home)), Some(PathBuf::from("/data"))); + assert_eq!(xdg_data_home(Some("/data".into()), None), Some(PathBuf::from("/data"))); + for xdg in [None, Some(""), Some("relative/data")] { + assert_eq!(xdg_data_home(xdg.map(Into::into), Some(home)), Some(home.join(".local/share")), "{xdg:?}"); + assert_eq!(xdg_data_home(xdg.map(Into::into), None), None, "{xdg:?}"); + } + } + + #[test] + fn decodes_latin2_and_utf8() { + let aff = b"SET ISO8859-2\nTRY a\n"; + let (a, d) = decode(aff, b"2\n\xbf\xf3\xb3w\n").unwrap(); + assert!(a.contains("TRY a")); + assert_eq!(d, "2\nżółw\n"); + let (_, d) = decode("SET UTF-8\n".as_bytes(), "żółw".as_bytes()).unwrap(); + assert_eq!(d, "żółw"); + let (_, d) = decode(b"SET ISO8859-9\n", b"\xd0\xdd\xde\xf0\xfd\xfe\xd6\xe7").unwrap(); + assert_eq!(d, "ĞİŞğışÖç"); + assert!(decode(b"SET SHIFT_JIS\n", b"").is_err()); + assert!(decode(b"SET UTF-8\n", b"\xff\xfe").is_err()); + } + + /// #2830: a dictionary that is there but can't be read leaves its reason behind; a good + /// load or no file clears it. + #[test] + fn load_error_is_remembered_until_the_next_good_load() { + let key = "test_unreadable_2830"; + assert_eq!(load_error(key), None); + let dic = Path::new("/dicts/test.dic"); + assert!(note_load(key, dic, decode(b"SET SHIFT_JIS\n", b"")).is_err()); + let (file, reason) = load_error(key).expect("error kept"); + assert_eq!(file, dic); + assert_eq!(reason, "unsupported encoding SHIFT_JIS"); + assert!(note_load(key, dic, decode(b"SET UTF-8\n", b"")).is_ok()); + assert_eq!(load_error(key), None); + let _ = note_load::<()>(key, dic, Err("too large".into())); + clear_load_error(key); + assert_eq!(load_error(key), None); + } + + #[test] + fn decodes_latin1() { + // `orçamento esperávamos` in ISO 8859-1: ç = 0xE7, á = 0xE1. + let (a, d) = decode(b"SET ISO8859-1\nTRY \xe7\n", b"1\nor\xe7amento/A\nesper\xe1vamos\n").unwrap(); + assert!(a.contains("TRY ç"), "{a}"); + assert_eq!(d, "1\norçamento/A\nesperávamos\n"); + assert!(decode(b"SET ISO-8859-1\n", b"\xe7").is_ok()); + } + + #[test] + fn utf8_dictionaries_keep_single_byte_flags() { + // The Hungarian dictionary is `SET UTF-8` but writes flags as single bytes above 0x7F, + // which aren't UTF-8; each byte is one flag. Its `NAME` line is ISO 8859-2. + let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 2\nAF \xd5a # 1\nAF \xe9 # 2\nSFX \xd5 Y 1\nSFX \xd5 0 ok/2 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n"; + let dic = "2\nház/1\t1\nkert/\u{e9}\n".as_bytes(); + let (a, d) = decode(aff, dic).unwrap(); + assert!(a.contains("AF \u{d5}a # 1") && a.contains("AF \u{e9} # 2") && a.contains("SFX \u{d5} 0 ok/2 . 7")); + assert!(d.contains("kert/\u{c3}\u{a9}"), "a UTF-8 é in the flags is two one-byte flags"); + let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap(); + assert!(dict.check("ház") && dict.check("házok") && dict.check("házokban") && dict.check("házat")); + assert!(!dict.check("házban")); + // `FLAG UTF-8` keeps the whole file UTF-8, and words must still be UTF-8. + let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n"); + assert!(decode(b"SET UTF-8\nSFX a Y 1\nSFX a 0 \xe9 .\n", b"").is_err(), "an affix must be UTF-8"); + assert!(decode(b"SET UTF-8\n", b"1\nh\xe1z/a\n").is_err(), "a stem must be UTF-8"); + } + + #[test] + fn single_flag_fields_keep_their_first_byte() { + // Without `FLAG UTF-8`, a UTF-8 `é` (0xC3 0xA9) in a field that names one flag is the + // flag 0xC3; in a `.dic` flag list it is the two flags 0xC3 and 0xA9. + let aff = "SET UTF-8\nKEEPCASE é\nSFX é Y 1\nSFX é 0 ok .\nSYLLABLENUM é\n".as_bytes(); + let (a, d) = decode(aff, "2\nkert/é\nNATO/é\n".as_bytes()).unwrap(); + assert!(a.contains("KEEPCASE \u{c3}\n") && a.contains("SFX \u{c3} Y 1") && a.contains("SFX \u{c3} 0 ok ."), "{a}"); + assert!(a.contains("SYLLABLENUM \u{c3}\u{a9}"), "SYLLABLENUM lists flags: {a}"); + assert!(d.contains("kert/\u{c3}\u{a9}")); + let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap(); + assert!(dict.check("kert") && dict.check("kertok")); + assert!(dict.check("NATO") && !dict.check("Nato"), "KEEPCASE é reaches NATO/é"); + } + + #[test] + fn numeric_and_long_flags_stay_text() { + // `FLAG num` (the Turkish dictionary) and `FLAG long`: no byte-wise flag reading, so a + // number with a non-ASCII rule or comment keeps every digit. + let aff = "SET UTF-8\nFLAG num\nFORBIDDENWORD 6000 # tiltott ő\nSFX 1234 Y 1\nSFX 1234 0 ları .\n"; + let (a, d) = decode(aff.as_bytes(), "1\nkitap/1234\n".as_bytes()).unwrap(); + assert_eq!(a, aff); + assert_eq!(d, "1\nkitap/1234\n"); + let tr = crate::tr::dict::TrDict::from_hunspell(&a, &d).unwrap(); + assert!(tr.contains("kitapları") && tr.contains("kitap")); + let long = "SET UTF-8\nFLAG long\nSFX Aa Y 1\nSFX Aa 0 ős .\n"; + assert_eq!(decode(long.as_bytes(), "1\nkő/Aa\n".as_bytes()).unwrap().0, long); + assert!(decode(b"SET UTF-8\nFLAG num\n", b"1\nh\xe1z/1\n").is_err(), "the .dic must be UTF-8"); + // `FLAG DEFAULT` is the single-byte default. + let (_, d) = decode(b"SET UTF-8\nFLAG default\n", b"1\nkert/\xe9\n").unwrap(); + assert_eq!(d, "1\nkert/\u{e9}\n"); + } + + #[test] + fn flag_utf8_counts_anywhere_in_the_aff() { + // A `FLAG UTF-8` after a long rule table still makes the `.dic` flags Unicode. + let mut aff = b"SET UTF-8\nSFX a Y 400\n".to_vec(); + for _ in 0..400 { + aff.extend_from_slice(b"SFX a 0 aaaaaaaaaaaaaaaaaaaaaaaa .\n"); + } + aff.extend_from_slice(b"FLAG UTF-8\r\n"); + assert!(aff.len() > 8192); + let (_, d) = decode(&aff, "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n"); + } + + #[test] + fn a_comment_after_flag_utf8_keeps_unicode_flags() { + let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8 # Unicode flags\n", "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n"); + let (_, d) = decode(b"SET\tUTF-8 # comment\nFLAG\t\tUTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n", "repeated spaces and tabs separate the argument"); + } + + #[test] + fn decodes_russian_single_byte_encodings() { + // `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0). + let (a, d) = decode(b"SET KOI8-R\nTRY \xcf\xc5\n", b"1\n\xa3\xcc\xcb\xc1/A\n").unwrap(); + assert!(a.contains("TRY ое"), "{a}"); + assert_eq!(d, "1\nёлка/A\n"); + let (_, d) = decode(b"SET microsoft-cp1251\n", b"1\n\xb8\xeb\xea\xe0 \xc4\xee\xec\n").unwrap(); + assert_eq!(d, "1\nёлка Дом\n"); + assert_eq!(decode_high(&[0x98], CP1251_HIGH), "\u{fffd}"); + } + + #[test] + fn comments_start_after_the_directive_fields() { + let f = |line: &'static str| move |k: usize| line.split_whitespace().nth(k).map(str::to_string); + assert_eq!(comment_field("PFX", f("PFX A Y 3 # count")), 4); + assert_eq!(comment_field("SFX", f("SFX A 0 # .")), 5, "a rule's condition may be `#`"); + assert_eq!(comment_field("AF", f("AF 1234 # aliases")), 2); + assert_eq!(comment_field("AF", f("AF Aa # 1")), 2); + assert_eq!(comment_field("REP", f("REP 12")), 2); + assert_eq!(comment_field("REP", f("REP # x")), 3); + assert_eq!(comment_field("CHECKCOMPOUNDPATTERN", f("CHECKCOMPOUNDPATTERN a b c")), 4); + assert_eq!(comment_field("KEEPCASE", f("KEEPCASE k")), 2); + // A byte flag after an append text that begins with `#` stays a flag, not a comment. + let (a, _) = decode(b"SET UTF-8\nSFX \xd5 Y 1 # header\nSFX \xd5 0 #/\xe9 . # rule\n", b"").unwrap(); + assert!(a.contains("SFX \u{d5} 0 #/\u{e9} . # rule"), "{a}"); + } + + #[test] + fn dic_entries_end_where_hunspell_ends_them() { + let end = |l: &str| l.get(..dic_entry_end(l.as_bytes())).unwrap().to_string(); + assert_eq!(end("word/A po:noun\tts:x"), "word/A", "the morphological field comes before the tab"); + assert_eq!(end("ad hoc/B\tpo:adv"), "ad hoc/B", "a tab ends a phrase"); + assert_eq!(end("ad hoc/B po:adv"), "ad hoc/B"); + assert_eq!(end("word/A 123"), "word/A 123", "neither: the whole line, as in Hunspell"); + assert_eq!(end("ad hoc/B "), "ad hoc/B"); + assert_eq!(end("word"), "word"); + assert_eq!(end("a:b/A"), "a:b/A", "a colon inside the word isn't a field"); + // Single-byte flags of a phrase entry are read byte by byte, then the UTF-8 morphology. + let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\tpo:adv\n").unwrap(); + assert_eq!(d, "1\nad hoc/\u{e9}\tpo:adv\n"); + // Without a tab or a morphological field the whole line is the entry: its byte flags + // don't fail the load (the Hungarian dictionary has such lines: `tök jó/396`). + let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\n").unwrap(); + assert_eq!(d, "1\nad hoc/\u{e9}\n"); + } + + #[test] + fn libreoffice_extension_dirs_follow_the_language() { + // LibreOffice's IDs are per dictionary: Polish is `dict-pl`, Brazilian Portuguese + // `dict-pt-BR`, so both the primary-language folder and the dashed full name are tried. + assert_eq!(super::lo_ext_dirs("cs_CZ"), ["dict-cs", "dict-cs-CZ"]); + assert_eq!(super::lo_ext_dirs("id_ID"), ["dict-id", "dict-id-ID"]); + assert_eq!(super::lo_ext_dirs("pl_PL"), ["dict-pl", "dict-pl-PL"]); + assert_eq!(super::lo_ext_dirs("pt_BR"), ["dict-pt", "dict-pt-BR"]); + assert_eq!(super::lo_ext_dirs("de-DE"), ["dict-de", "dict-de-DE"]); + let dirs = search_dirs(Some(&super::lo_ext_dirs("pt_BR"))); + assert!(dirs.iter().any(|d| d.ends_with("dict-pt-BR")), "{dirs:?}"); + assert!(search_dirs(None).iter().all(|d| !d.to_string_lossy().contains("dict-")), "no language: no LibreOffice guess"); + } + + /// craft-libs#482 re-review: a name is looked for in every folder before the next name, so + /// an `en_US` in the dictionaries folder doesn't beat a system `en_GB` for British English. + #[test] + fn candidates_go_name_by_name_across_the_folders() { + let base = std::env::temp_dir().join(format!("wordcraft-candidates-{}", std::process::id())); + let (folder, system) = (base.join("folder"), base.join("system")); + for (dir, name) in [(&folder, "en_US"), (&system, "en_GB"), (&folder, "en-GB")] { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write(dir.join(format!("{name}.aff")), "SET UTF-8\n").unwrap(); + std::fs::write(dir.join(format!("{name}.dic")), "1\nx\n").unwrap(); + } + let dirs = [folder.clone(), system.clone()]; + let dics: Vec = candidates_in(&dirs, &["en_GB", "en-GB"]).into_iter().map(|(_, d)| d).collect(); + assert_eq!(dics, vec![system.join("en_GB.dic"), folder.join("en-GB.dic")]); + let us: Vec = candidates_in(&dirs, &["en_US", "en_GB"]).into_iter().map(|(_, d)| d).collect(); + assert_eq!(us, vec![folder.join("en_US.dic"), system.join("en_GB.dic")]); + assert!(candidates_in(&dirs, &["en_CA"]).is_empty()); + let _ = std::fs::remove_dir_all(&base); + } + + #[test] + fn finds_and_loads_a_pair_in_a_directory() { + let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + assert!(find_in(&dir, &["xx_XX"]).is_none(), "no files: nothing found"); + std::fs::write(dir.join("xx_XX.aff"), "SET UTF-8\nSFX A Y 1\nSFX A 0 y .\n").unwrap(); + assert!(find_in(&dir, &["xx_XX"]).is_none(), "an .aff alone isn't a dictionary"); + std::fs::write(dir.join("xx_XX.dic"), "1\nkot/A\n").unwrap(); + let (aff, dic) = find_in(&dir, &["yy", "xx_XX"]).unwrap(); + let d = load(&aff, &dic).unwrap(); + assert!(d.check("kot") && d.check("koty") && !d.check("kotx")); + assert!(load(&dir.join("missing.aff"), &dic).is_err()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn reads_are_bounded_and_regular_files_only() { + let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-cap-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let aff = dir.join("xx.aff"); + std::fs::write(&aff, "SET UTF-8\n").unwrap(); + // One byte over the cap (sparse: nothing is written) is refused without reading it. + let big = dir.join("big.dic"); + std::fs::File::create(&big).unwrap().set_len(MAX_FILE + 1).unwrap(); + assert!(load(&aff, &big).unwrap_err().contains("too large")); + // A directory, and on Unix an endless device, are not dictionary files. (No clock: a + // reader that read `/dev/zero` would never return, so the error itself is the proof.) + assert!(load(&aff, &dir).unwrap_err().contains("not a regular file")); + #[cfg(unix)] + assert!(load(&aff, Path::new("/dev/zero")).unwrap_err().contains("not a regular file")); + // Turkish reads through the same reader. + let (tr_aff, tr_dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic")); + std::fs::write(&tr_aff, "SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n").unwrap(); + std::fs::write(&tr_dic, "1\nkitap/1\n").unwrap(); + assert!(crate::tr::load(&tr_aff, &tr_dic).unwrap().contains("kitaplar")); + assert!(crate::tr::load(&tr_aff, &big).unwrap_err().contains("too large")); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A Turkish dictionary in ISO 8859-9, the encoding older tr_TR dictionaries use, read from + /// disk through the capped reader and decoded per its `SET` line: its words with Turkish + /// letters (ğ ı ş İ, and ç ö ü from Latin-1) are found, as stems and with a suffix. Before the + /// Turkish dictionary went through this reader it was read as UTF-8, which these bytes are not. + #[test] + fn a_turkish_dictionary_in_iso_8859_9_loads_from_disk() { + let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-latin5-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let (aff, dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic")); + std::fs::write(&aff, b"SET ISO8859-9\nSFX 1 Y 2\nSFX 1 0 lar .\nSFX 1 0 \xfd .\nSFX 2 Y 1\nSFX 2 0 ler .\n").unwrap(); + // ağaç/1 (-lar, -ı), ışık, şişe/2 (-ler), İzmir, öğrenci/2, güneş. + let words: &[u8] = b"6\na\xf0a\xe7/1\n\xfd\xfe\xfdk\n\xfei\xfee/2\n\xddzmir\n\xf6\xf0renci/2\ng\xfcne\xfe\n"; + std::fs::write(&dic, words).unwrap(); + assert!(String::from_utf8(words.to_vec()).is_err(), "the .dic is not UTF-8"); + let d = crate::tr::load(&aff, &dic).unwrap(); + for w in ["ağaç", "ağaçlar", "ağaçı", "ışık", "şişe", "şişeler", "İzmir", "öğrenci", "öğrenciler", "güneş"] { + assert!(d.contains(w), "{w}"); + } + // The six letters ISO 8859-9 puts in place of ISO 8859-1's aren't read as those. + assert!(!d.contains("aðaç") && !d.contains("ýþýk") && !d.contains("Ýzmir")); + assert!(!d.contains("ağaçs") && !d.contains("ışıklar") && !d.contains("şişelar"), "only the rules' suffixes, on the stems that take them"); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs new file mode 100644 index 00000000..e12c5102 --- /dev/null +++ b/crates/proof/src/hyphen.rs @@ -0,0 +1,939 @@ +//! Hyphenation, per proofing language ([`ProofLang`]). +//! +//! **English** (and languages without patterns of their own, as before languages were told +//! apart), lookup order for each word: +//! 1. the **dictionary** ([`Dictionary::en_us`]) — the public-domain Moby Hyphenator II word list +//! by Grady Ward (~160k words), stored compactly in craft-i18n's `language-data/en-us/en-us.dic`; simple +//! inflections (`'s`, `-s`, `-es`, `-ed`, `-d`) inherit the breaks of their stem; +//! 2. **Liang patterns** ([`Patterns::en_us`]) for words not in the dictionary — our own pattern +//! set, trained from the same list by DesignCraft's patgen (generated by DesignCraft's `hyphgen` example); +//! 3. a small syllable **heuristic** for words outside the pattern alphabet. +//! +//! **Polish**: the exception words, then the TeX Polish patterns ([`Patterns::pl`], by Hanna +//! Kołodziejska, Bogusław Jackowski and Marek Ryćko, MIT licence option), then a Polish syllable +//! heuristic for words with letters outside the Polish alphabet (`é`, `ü`…). +//! +//! **Bulgarian**: the Bulgarian syllable rules ([`crate::bg`]); no pattern file or word list. +//! +//! **German**: the TeX German patterns for the reformed spelling ([`Patterns::de`]), also for +//! Swiss text, whose `ss` the patterns split like any double consonant (`Stras-se`). +//! +//! Then the paragraph's limits ([`Limits`]) are applied. + +pub use crate::dict; + +pub use crate::patterns; + +use std::sync::OnceLock; + +pub use crate::lang::ProofLang; +pub use dict::Dictionary; +pub use patterns::Patterns; + +// The embedded Language data (`language-data/` of craft-i18n, see `build.rs`); each is empty when +// it isn't embedded. The TeX files are hyph-utf8's `hyph-pl.tex`, `hyph-pt.tex`, `hyph-it.tex` and +// `hyph-de-1996.tex` (reformed spelling), unmodified, their licence headers in the files. +use crate::data::{DE_PAT, EN_US_DIC, EN_US_PAT, IT_PAT, PL_PAT, PT_PAT}; + +impl Dictionary { + /// The bundled US English dictionary (decoded on first use). + pub fn en_us() -> &'static Dictionary { + static D: OnceLock = OnceLock::new(); + D.get_or_init(|| { + if EN_US_DIC.is_empty() { + return Dictionary::default(); + } + Dictionary::from_bytes(EN_US_DIC).unwrap_or_else(|e| { + crate::english::note_embedded_error(e); + Dictionary::default() + }) + }) + } +} + +impl Patterns { + /// The bundled US English patterns (parsed on first use). + pub fn en_us() -> &'static Patterns { + static P: OnceLock = OnceLock::new(); + P.get_or_init(|| Patterns::parse(EN_US_PAT)) + } + + /// The bundled Polish patterns and exception words (parsed on first use). + pub fn pl() -> &'static Patterns { + static P: OnceLock = OnceLock::new(); + P.get_or_init(|| Patterns::parse(PL_PAT)) + } + + /// The bundled Portuguese patterns and exception words (parsed on first use). + pub fn pt() -> &'static Patterns { + static P: OnceLock = OnceLock::new(); + P.get_or_init(|| Patterns::parse(PT_PAT)) + } + + /// The bundled Italian patterns (parsed on first use; the file has no exception words). + pub fn it() -> &'static Patterns { + static P: OnceLock = OnceLock::new(); + P.get_or_init(|| Patterns::parse(IT_PAT)) + } + + /// The bundled German patterns (parsed on first use; the file has no exception words). + pub fn de() -> &'static Patterns { + static P: OnceLock = OnceLock::new(); + P.get_or_init(|| Patterns::parse(DE_PAT)) + } +} + +/// Hyphenation limits from the paragraph's Hyphenation settings. +#[derive(Clone, Copy, Debug)] +pub struct Limits { + pub min_word: usize, + pub after_first: usize, + pub before_last: usize, + pub capitalized: bool, +} + +impl Default for Limits { + fn default() -> Self { + Limits { min_word: 5, after_first: 2, before_last: 2, capitalized: true } + } +} + +/// Where a word's break points came from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Source { + Dictionary, + Patterns, + Heuristic, + /// A language's syllable rules (Bulgarian, [`crate::bg`]). + Rules, + None, +} + +fn is_apostrophe(c: char) -> bool { + matches!(c, '\'' | '’' | 'ʼ') +} + +fn is_word_char(c: char) -> bool { + c.is_alphabetic() || is_apostrophe(c) +} + +/// Unrestricted break points of one lowercase core (letters and `'` only, no leading/trailing `'`). +fn core_breaks(core: &[char], lang: ProofLang) -> (Vec, Source) { + let n = core.len(); + if n < 2 { + return (vec![], Source::None); + } + if lang == ProofLang::Pl { + return core_breaks_pl(core); + } + // Bulgarian words follow the Bulgarian syllable rules; Latin-script words in a Bulgarian run + // (`WordCraft`, `email`) are hyphenated as English, as in any other language. + if lang == ProofLang::Bg && core.iter().all(|&c| crate::bg::is_letter(c)) { + return (crate::bg::points(core), Source::Rules); + } + if lang == ProofLang::Tr && core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) { + return (crate::tr::hyphen::points(core), Source::Rules); + } + match lang { + ProofLang::Pt => return core_breaks_patterns(Patterns::pt(), core), + ProofLang::It => return core_breaks_patterns(Patterns::it(), core), + ProofLang::De => return core_breaks_patterns(Patterns::de(), core), + _ => {} + } + if lang == ProofLang::Ru && core.iter().all(|&c| crate::ru::is_cyrillic(c)) { + return (crate::ru::hyphen::points(core), Source::Heuristic); + } + let dict = Dictionary::en_us(); + // No English Language data embedded: hyphenation is off. + if dict.is_empty() && Patterns::en_us().is_empty() { + return (vec![], Source::None); + } + let s: String = core.iter().collect(); + let from_mask = |m: u64, len: usize| (1..len.min(64)).filter(|&i| m & (1 << i) != 0).collect::>(); + if let Some(m) = dict.get(&s) { + return (from_mask(m, n), Source::Dictionary); + } + // Inflections inherit the stem's breaks (the suffix adds no break of its own). + for suf in ["'s", "s", "es", "ed", "d"] { + if let Some(stem) = s.strip_suffix(suf) + && stem.chars().count() >= 2 + && let Some(m) = dict.get(stem) + { + return (from_mask(m, stem.chars().count()), Source::Dictionary); + } + } + let pats = Patterns::en_us(); + if !pats.is_empty() + && let Some(p) = pats.points(core) + { + return (p, Source::Patterns); + } + (heuristic_points(core), Source::Heuristic) +} + +/// Polish: exception words, then the patterns, then the Polish syllable heuristic. +fn core_breaks_pl(core: &[char]) -> (Vec, Source) { + let pats = Patterns::pl(); + // No Polish patterns embedded: hyphenation is off. + if pats.is_empty() { + return (vec![], Source::None); + } + let s: String = core.iter().collect(); + if let Some(p) = pats.exception(&s) { + return (p.to_vec(), Source::Dictionary); + } + if !pats.is_empty() + && let Some(p) = pats.points(core) + { + return (p, Source::Patterns); + } + (heuristic_points_pl(core), Source::Heuristic) +} + +/// [`core_breaks_pl`] for a language with bundled patterns (Portuguese, Italian, German): its +/// exception words, the patterns, then the generic heuristic. +fn core_breaks_patterns(pats: &Patterns, core: &[char]) -> (Vec, Source) { + // The language's patterns aren't embedded: hyphenation is off. + if pats.is_empty() { + return (vec![], Source::None); + } + let s: String = core.iter().collect(); + if let Some(p) = pats.exception(&s) { + return (p.to_vec(), Source::Dictionary); + } + if !pats.is_empty() + && let Some(p) = pats.points(core) + { + return (p, Source::Patterns); + } + (heuristic_points(core), Source::Heuristic) +} + +/// Unrestricted break points of an English `word` ([`word_breaks_in`] with [`ProofLang::En`]). +pub fn word_breaks(word: &str) -> (Vec, Source) { + word_breaks_in(word, ProofLang::En) +} + +/// Unrestricted break points of `word` in `lang` (char indices; a hyphen goes before that char), +/// handling case, apostrophes (`don't`, `’s`), surrounding punctuation and compounds (each letter +/// run is hyphenated separately). No break is ever placed next to an apostrophe or non-letter. +pub fn word_breaks_in(word: &str, lang: ProofLang) -> (Vec, Source) { + let chars: Vec = word.chars().collect(); + let mut out = Vec::new(); + let mut src = Source::None; + let mut i = 0; + while i < chars.len() { + if !is_word_char(chars[i]) { + i += 1; + continue; + } + let mut j = i; + while j < chars.len() && is_word_char(chars[j]) { + j += 1; + } + let (mut a, mut b) = (i, j); + while a < b && is_apostrophe(chars[a]) { + a += 1; + } + while b > a && is_apostrophe(chars[b - 1]) { + b -= 1; + } + let core: Vec = chars[a..b].iter().map(|&c| if is_apostrophe(c) { '\'' } else { c.to_lowercase().next().unwrap_or(c) }).collect(); + let (pts, s) = core_breaks(&core, lang); + if src == Source::None { + src = s; + } + out.extend(pts.into_iter().filter(|&p| p > 0 && p < core.len() && core[p - 1] != '\'' && core[p] != '\'').map(|p| a + p)); + i = j; + } + (out, src) +} + +/// Allowed hyphenation points in an English `word` ([`hyphen_points_in`] with [`ProofLang::En`]). +pub fn hyphen_points(word: &str, lim: &Limits) -> Vec { + hyphen_points_in(word, lim, ProofLang::En) +} + +/// Allowed hyphenation points in `word`, a word in `lang`, as char indices (a hyphen goes before +/// that char), with the paragraph limits applied to each letter run. +pub fn hyphen_points_in(word: &str, lim: &Limits, lang: ProofLang) -> Vec { + let chars: Vec = word.chars().collect(); + let (pts, _) = word_breaks_in(word, lang); + if pts.is_empty() { + return pts; + } + let min_before = lim.after_first.max(1); + let min_after = lim.before_last.max(1); + // Counted once, so each point's checks take constant time however long the word: letters and + // uppercase letters before each char, and the letter run (trimmed of apostrophes) of each char. + let (mut letters, mut upper) = (Vec::with_capacity(chars.len() + 1), Vec::with_capacity(chars.len() + 1)); + let (mut l, mut u) = (0usize, 0usize); + for c in &chars { + letters.push(l); + upper.push(u); + if c.is_alphabetic() { + l += 1; + u += usize::from(c.is_uppercase()); + } + } + letters.push(l); + upper.push(u); + let mut run_of: Vec<(usize, usize)> = vec![(0, 0); chars.len()]; + let mut i = 0; + while i < chars.len() { + let mut j = i; + while chars.get(j).is_some_and(|c| is_word_char(*c)) { + j += 1; + } + let (mut a, mut b) = (i, j); + while a < b && chars.get(a).is_some_and(|c| is_apostrophe(*c)) { + a += 1; + } + while b > a && chars.get(b - 1).is_some_and(|c| is_apostrophe(*c)) { + b -= 1; + } + for r in run_of.get_mut(i..j).into_iter().flatten() { + *r = (a, b); + } + i = j.max(i + 1); + } + let count = |v: &[usize], a: usize, b: usize| v.get(b).zip(v.get(a)).map_or(0, |(y, x)| y.saturating_sub(*x)); + pts.into_iter() + .filter(|&p| { + // A point lies between two letters of one run; the char after it gives the run. + let Some(&(a, b)) = run_of.get(p) else { return false }; + let n = count(&letters, a, b); + if n < lim.min_word.max(2) { + return false; + } + if n > 1 && count(&upper, a, b) == n { + return false; + } + if !lim.capitalized && chars.get(a).is_some_and(|c| c.is_uppercase()) { + return false; + } + count(&letters, a, p) >= min_before && count(&letters, p, b) >= min_after + }) + .collect() +} + +/// `word` with `-` at every hyphenation point (tests and diagnostics). +pub fn hyphenate_word(word: &str, lim: &Limits) -> String { + hyphenate_word_in(word, lim, ProofLang::En) +} + +/// `word`, a word in `lang`, with `-` at every hyphenation point (tests and diagnostics). +pub fn hyphenate_word_in(word: &str, lim: &Limits, lang: ProofLang) -> String { + let mut pts = hyphen_points_in(word, lim, lang); + pts.sort_unstable(); + // Walk the points alongside the chars rather than searching them for each char. + let mut next = pts.into_iter().peekable(); + let mut s = String::with_capacity(word.len() + 8); + for (i, ch) in word.chars().enumerate() { + if next.next_if_eq(&i).is_some() { + while next.next_if_eq(&i).is_some() {} + s.push('-'); + } + s.push(ch); + } + s +} + +// ---------- heuristic fallback (words outside the pattern alphabet) ---------- + +/// Letters with a diacritic that are consonants (`ł`, `ż`, `ç`, `ñ`…); any other non-ASCII letter +/// counts as a vowel (`é`, `ø`, `ą`…), and so do letters of other scripts, which keeps the +/// heuristic from breaking them. +const NON_ASCII_CONSONANTS: &str = "łńśćźżçčďđğģķļľĺňñņřŕšşșťţțžßŋðþ"; + +fn is_vowel(c: char) -> bool { + let l = c.to_lowercase().next().unwrap_or(c); + if l.is_ascii() { matches!(l, 'a' | 'e' | 'i' | 'o' | 'u' | 'y') } else { l.is_alphabetic() && !NON_ASCII_CONSONANTS.contains(l) } +} + +fn onset_pair(a: char, b: char) -> bool { + let (a, b) = (a.to_ascii_lowercase(), b.to_ascii_lowercase()); + matches!( + (a, b), + ('c' | 's' | 't' | 'p' | 'w' | 'g', 'h') + | ('b' | 'c' | 'd' | 'f' | 'g' | 'k' | 'p' | 't', 'r') + | ('b' | 'c' | 'f' | 'g' | 'k' | 'p' | 's', 'l') + | ('q', 'u') + | ('s', 't' | 'p' | 'c' | 'k') + ) +} + +/// Syllable heuristic: V-CV ("ty-po"), VC-CV ("hap-pen"), keeping onset clusters ("gra-phy"). +fn heuristic_points(c: &[char]) -> Vec { + let n = c.len(); + if n < 4 || !c.iter().all(|c| c.is_alphabetic()) { + return vec![]; + } + let v: Vec = c.iter().map(|&x| is_vowel(x)).collect(); + // Both parts need a sounded vowel: a trailing silent `e`, `ed` or `es` doesn't count. The part + // after `i` is `c[i..end]`, `end` before that ending; computed once, so each point is O(1). + let end = |i: usize| { + if n - i >= 2 && (c.ends_with(&['e', 'd']) || c.ends_with(&['e', 's'])) { + n - 2 + } else if c.ends_with(&['e']) { + n - 1 + } else { + n + } + }; + // UTF-8 bytes before each char, and the first vowel at or after each char. + let mut bytes = Vec::with_capacity(n + 1); + let mut acc = 0usize; + for ch in c { + bytes.push(acc); + acc += ch.len_utf8(); + } + bytes.push(acc); + let mut next_vowel = vec![n; n + 1]; + for i in (0..n).rev() { + next_vowel[i] = if v[i] { i } else { next_vowel[i + 1] }; + } + let mut out = vec![]; + let mut vowel_before = false; + for i in 1..n { + vowel_before |= v[i - 1]; + let ok = if v[i - 1] && !v[i] { + (i + 1 < n && v[i + 1]) || (i + 2 < n && onset_pair(c[i], c[i + 1]) && v[i + 2]) + } else if !v[i - 1] && !v[i] { + i >= 2 && v[i - 2] && i + 1 < n && v[i + 1] && !onset_pair(c[i - 1], c[i]) && c[i - 1] != c[i] || (c[i - 1] == c[i] && i >= 2 && v[i - 2]) + } else { + false + }; + let e = end(i); + if ok && next_vowel[i] < e && bytes[e] - bytes[i] > 1 && vowel_before { + out.push(i); + } + } + out +} + +/// Polish digraphs, one sound each: never split. +fn is_digraph_pl(a: char, b: char) -> bool { + matches!((a, b), ('c', 'h' | 'z') | ('d', 'z' | 'ź' | 'ż') | ('r' | 's', 'z')) +} + +/// Polish syllable heuristic, for words with letters outside the Polish pattern alphabet (`é`, +/// `ü`…): one break between each pair of syllable nuclei — after a leading sonorant (`kon-ty`, +/// `Pol-ska`) or inside a doubled consonant (`Pöp-pel`), else before the consonant cluster +/// (`sio-stra`). Digraphs (`ch`, `cz`, `dz`, `dź`, `dż`, `rz`, `sz`) stay whole, and an `i` that +/// only softens the consonant before a vowel (`nia`, `sio`) stays with that consonant. +fn heuristic_points_pl(c: &[char]) -> Vec { + let n = c.len(); + if n < 4 || !c.iter().all(|c| c.is_alphabetic()) { + return vec![]; + } + let v: Vec = c.iter().map(|&x| is_vowel(x)).collect(); + let at = |i: usize| v.get(i).copied().unwrap_or(false); + // A syllable nucleus: a vowel, except a softening `i` (after a consonant, before a vowel). + let nucleus: Vec = (0..n).map(|i| at(i) && !(c.get(i) == Some(&'i') && i > 0 && !at(i - 1) && at(i + 1))).collect(); + let nuclei: Vec = (0..n).filter(|&i| nucleus.get(i).copied().unwrap_or(false)).collect(); + let mut out = vec![]; + for pair in nuclei.windows(2) { + let [a, b] = pair else { continue }; + let (start, end) = (a + 1, *b); + if start >= end { + continue; // adjacent vowels (na-u-ka): no break inside a vowel pair here + } + // Consonant units of the cluster: digraphs and a softening `i` join their consonant. + let mut units: Vec = Vec::new(); + let mut i = start; + while i < end { + units.push(i); + i += if i + 1 < end && c.get(i).zip(c.get(i + 1)).is_some_and(|(x, y)| is_digraph_pl(*x, *y)) { 2 } else { 1 }; + if i < end && c.get(i) == Some(&'i') { + i += 1; + } + } + let sonorant = |i: usize| matches!(c.get(i), Some('l' | 'ł' | 'm' | 'n' | 'ń' | 'r' | 'j')); + let first_is_single = units.get(1).is_some_and(|&second| second == start + 1); + // A doubled consonant splits (`lek-ki`, `Pöp-pel`), as does a cluster after a sonorant. + let doubled = c.get(start).is_some_and(|x| c.get(start + 1) == Some(x)); + out.push(if units.len() >= 2 && first_is_single && (sonorant(start) || doubled) { start + 1 } else { start }); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn h(w: &str) -> String { + hyphenate_word(w, &Limits::default()) + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn bundled_data_loads() { + assert!(Dictionary::en_us().len() > 150_000, "{}", Dictionary::en_us().len()); + assert!(Patterns::en_us().len() > 1000, "{}", Patterns::en_us().len()); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn dictionary_words() { + assert_eq!(h("happen"), "hap-pen"); + assert_eq!(h("typography"), "ty-pog-ra-phy"); + assert_eq!(h("composition"), "com-po-si-tion"); + assert_eq!(h("hyphenation"), "hy-phen-a-tion"); + assert_eq!(h("international"), "in-ter-na-tion-al"); + assert_eq!(h("cat"), "cat"); + assert!(!h("changed").contains('-')); + assert_eq!(h("NASA"), "NASA"); + assert_eq!(word_breaks("publication").1, Source::Dictionary); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn case_apostrophes_and_punctuation() { + assert_eq!(h("Typography"), "Ty-pog-ra-phy"); + assert_eq!(h("TYPOGRAPHY"), "TYPOGRAPHY"); + assert_eq!(h("typography,"), "ty-pog-ra-phy,"); + assert_eq!(h("“typography”"), "“ty-pog-ra-phy”"); + assert_eq!(h("typography's"), "ty-pog-ra-phy's"); + assert_eq!(h("typography’s"), "ty-pog-ra-phy’s"); + assert_eq!(h("'typography'"), "'ty-pog-ra-phy'"); + // Compounds: each part separately, never next to the hyphen. + assert_eq!(h("typography-composition"), "ty-pog-ra-phy-com-po-si-tion"); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn inflections_inherit_the_stem() { + assert_eq!(h("ripples"), "rip-ples"); + assert_eq!(h("gradients"), "gra-di-ents"); + assert_eq!(h("restrained"), "re-strained"); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn patterns_cover_unknown_words() { + // Not in the word list: the trained patterns decide. + let (pts, src) = word_breaks("hyperlocalization"); + assert_eq!(src, Source::Patterns); + assert!(pts.len() >= 4, "{}", h("hyperlocalization")); + let (_, src) = word_breaks("zzyzx"); + assert_ne!(src, Source::Dictionary); + } + + #[test] + fn limits_are_respected() { + let lim = Limits { min_word: 5, after_first: 3, before_last: 3, capitalized: false }; + for w in ["typography", "hyphenation", "justification", "Paragraph", "beautiful"] { + for p in hyphen_points(w, &lim) { + assert!(p >= 3 && p <= w.chars().count() - 3, "{w} {p}"); + } + } + assert!(hyphen_points("Paragraph", &lim).is_empty()); + let lim = Limits { min_word: 12, ..Limits::default() }; + assert!(hyphen_points("typography", &lim).is_empty()); + } + + /// The CPU time `f` takes on this thread, in seconds. On Linux it is the scheduler's count of the + /// time the thread actually ran, so the time it waits while other work on a busy machine has the + /// CPU is left out; elsewhere, wall-clock time. + fn cpu_cost(f: &mut impl FnMut()) -> f64 { + let ran = || { + let s = std::fs::read_to_string("/proc/thread-self/schedstat").ok()?; + s.split_whitespace().next()?.parse::().ok() + }; + let (t, before) = (std::time::Instant::now(), ran()); + f(); + match (before, ran()) { + (Some(a), Some(b)) => b.saturating_sub(a) as f64 * 1e-9, + _ => t.elapsed().as_secs_f64(), + } + } + + /// How much more `big` costs than `small`, for `big` an input `scale` times the size of `small`'s: + /// (small, big, big / small), ~`scale` when linear and ~`scale`² when quadratic. Each is the + /// cheapest of several runs after a warm-up, the runs alternating so a busy spell slows both sizes + /// alike; `small` is timed `scale` times over per run, so both timings are long against the clock's + /// resolution. + fn cost_ratio(scale: usize, mut small: impl FnMut(), mut big: impl FnMut()) -> (f64, f64, f64) { + small(); + big(); + let mut small_scaled = || (0..scale).for_each(|_| small()); + let (mut t1, mut tk) = (f64::INFINITY, f64::INFINITY); + for _ in 0..5 { + t1 = t1.min(cpu_cost(&mut small_scaled) / scale as f64); + tk = tk.min(cpu_cost(&mut big)); + } + (t1, tk, tk / t1.max(1e-9)) + } + + /// `hyphen_points`' limit checks as first written (a rescan of the letter run per point): the + /// reference for the linear version. + fn hyphen_points_ref(word: &str, lim: &Limits) -> Vec { + let chars: Vec = word.chars().collect(); + let (pts, _) = word_breaks(word); + let min_before = lim.after_first.max(1); + let min_after = lim.before_last.max(1); + let run_of = |p: usize| { + let mut a = p; + while a > 0 && is_word_char(chars[a - 1]) { + a -= 1; + } + let mut b = p; + while b < chars.len() && is_word_char(chars[b]) { + b += 1; + } + while a < b && is_apostrophe(chars[a]) { + a += 1; + } + while b > a && is_apostrophe(chars[b - 1]) { + b -= 1; + } + (a, b) + }; + pts.into_iter() + .filter(|&p| { + let (a, b) = run_of(p); + let run = &chars[a..b]; + let letters = run.iter().filter(|c| c.is_alphabetic()).count(); + if letters < lim.min_word.max(2) { + return false; + } + if letters > 1 && run.iter().filter(|c| c.is_alphabetic()).all(|c| c.is_uppercase()) { + return false; + } + if !lim.capitalized && run[0].is_uppercase() { + return false; + } + let before = chars[a..p].iter().filter(|c| c.is_alphabetic()).count(); + let after = chars[p..b].iter().filter(|c| c.is_alphabetic()).count(); + before >= min_before && after >= min_after + }) + .collect() + } + + /// `heuristic_points` as first written (the right part rebuilt per point): the reference. + fn heuristic_ref(c: &[char]) -> Vec { + let n = c.len(); + if n < 4 || !c.iter().all(|c| c.is_alphabetic()) { + return vec![]; + } + let v: Vec = c.iter().map(|&x| is_vowel(x)).collect(); + let mut out = vec![]; + for i in 1..n { + let ok = if v[i - 1] && !v[i] { + (i + 1 < n && v[i + 1]) || (i + 2 < n && onset_pair(c[i], c[i + 1]) && v[i + 2]) + } else if !v[i - 1] && !v[i] { + i >= 2 && v[i - 2] && i + 1 < n && v[i + 1] && !onset_pair(c[i - 1], c[i]) && c[i - 1] != c[i] + || (c[i - 1] == c[i] && i >= 2 && v[i - 2]) + } else { + false + }; + let right: String = c[i..].iter().collect(); + let core = right.strip_suffix("ed").or_else(|| right.strip_suffix("es")).or_else(|| right.strip_suffix('e')).unwrap_or(&right); + if ok && core.chars().any(is_vowel) && core.len() > 1 && v[..i].iter().any(|x| *x) { + out.push(i); + } + } + out + } + + #[test] + fn linear_limits_and_heuristic_match_the_reference() { + let long = "hyphenation".repeat(40); + let words = [ + "", + "x", + "''", + "a'b'c", + "cat", + "happen", + "hyphenation", + "Typography's", + "TYPOGRAPHY", + "Typography", + "typography’s", + "'typography'", + "“typography”,", + "typography-composition", + "don't", + "rock'n'roll", + "naïveté", + "crème-brûlée", + "Ünïcödé", + "ÉTUDE", + "strengths", + "ee", + "bed", + "hopped", + "rhythmed", + "aeiouy", + "xxyyzz", + "bookkeeper", + "ʼtwas", + "mid-1990s", + "twenty-one", + "école", + "Ålesund", + "smørrebrød", + "hopes", + "hoped", + "taed", + "teeed", + &long, + ]; + for w in words { + for min_word in [2, 5, 7] { + for (after_first, before_last) in [(1, 1), (2, 2), (3, 2), (2, 4)] { + for capitalized in [true, false] { + let lim = Limits { min_word, after_first, before_last, capitalized }; + assert_eq!(hyphen_points(w, &lim), hyphen_points_ref(w, &lim), "{w:?} {lim:?}"); + } + } + } + let c: Vec = w.to_lowercase().chars().collect(); + assert_eq!(heuristic_points(&c), heuristic_ref(&c), "{w:?}"); + } + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn hyphenating_a_long_word_is_linear() { + // A word of n letters against one of 8n: linear ~8x the cost, the old rescans ~64x. One in + // the pattern alphabet, one outside it (the syllable heuristic). + let pattern = |k: usize| "hyphenation".repeat(k); + let heuristic = |k: usize| "smørrebrød".repeat(k); + assert_eq!(word_breaks(&heuristic(2)).1, Source::Heuristic); + // Each timing repeats the call so the shorter one is milliseconds, well above timer noise. + let ratios: Vec<(String, f64)> = [(pattern as fn(usize) -> String, 20), (heuristic, 40)] + .into_iter() + .map(|(word, repeat)| { + let (small, big) = (word(400), word(8 * 400)); + let time = |w: &str| (0..repeat).for_each(|_| drop(hyphen_points(w, &Limits::default()))); + let (t1, t8, ratio) = cost_ratio(8, || time(&small), || time(&big)); + let what = format!("{}…: t(n) {t1:.4} s, t(8n) {t8:.4} s, ratio {ratio:.1}", word(1).chars().take(10).collect::()); + eprintln!("{what}"); + (what, ratio) + }) + .collect(); + for (what, ratio) in ratios { + assert!(ratio < 24.0, "8x the letters, not linear: {what}"); + } + } + + #[test] + fn russian_hyphenation_and_hyphenated_output_are_linear() { + // A Russian word breaks at every syllable: each break used to be checked against all the + // earlier ones (#3657), and the hyphenated output searched every break for each char. + let word = |k: usize| "мама".repeat(k); + assert_eq!(word_breaks_in(&word(2), ProofLang::Ru).0, [2, 4, 6]); + assert_eq!(hyphenate_word_in(&word(2), &Limits::default(), ProofLang::Ru), "ма-ма-ма-ма"); + // The breaks' rescan is a cheap one per break: it takes longer words to stand out. + for (what, k, repeat, f) in [ + ("points", 1000, 3, (|w: &str| drop(hyphen_points_in(w, &Limits::default(), ProofLang::Ru))) as fn(&str)), + ("hyphenated", 400, 20, |w: &str| drop(hyphenate_word_in(w, &Limits::default(), ProofLang::Ru))), + ] { + let (small, big) = (word(k), word(8 * k)); + let time = |w: &str| (0..repeat).for_each(|_| f(w)); + let (t1, t8, ratio) = cost_ratio(8, || time(&small), || time(&big)); + eprintln!("{what}: t(n) {t1:.4} s, t(8n) {t8:.4} s, ratio {ratio:.1}"); + assert!(ratio < 24.0, "8x the letters, not linear: {what} {ratio:.1}"); + } + } + + #[test] + fn heuristic_fallback() { + let c: Vec = "happen".chars().collect(); + assert_eq!(heuristic_points(&c), vec![3]); + } + + #[test] + fn heuristic_knows_consonants_with_diacritics() { + // `ł`, `ż`, `ç` are consonants (every non-ASCII letter used to count as a vowel). + assert!(!is_vowel('ł') && !is_vowel('Ż') && !is_vowel('ç') && is_vowel('é') && is_vowel('ą') && is_vowel('ø')); + let c: Vec = "façade".chars().collect(); + assert_eq!(heuristic_points(&c), vec![2]); + } + + fn pl(w: &str) -> String { + hyphenate_word_in(w, &Limits::default(), ProofLang::Pl) + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn polish_patterns_load() { + let p = Patterns::pl(); + assert!(p.len() > 4000, "{}", p.len()); + assert_eq!(p.exception_count(), 20); + assert_eq!(p.exception("nigdy"), Some(&[2][..])); + } + + fn pt(w: &str) -> String { + hyphenate_word_in(w, &Limits::default(), ProofLang::Pt) + } + + fn it(w: &str) -> String { + hyphenate_word_in(w, &Limits::default(), ProofLang::It) + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn italian_patterns_load() { + let p = Patterns::it(); + assert!(p.len() > 300, "{}", p.len()); + // The file has no exception words. + assert_eq!(p.exception_count(), 0); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn italian_words_hyphenate_by_italian_rules() { + for (w, want) in [ + ("famiglia", "fa-mi-glia"), + ("finestra", "fi-ne-stra"), + ("bicicletta", "bi-ci-clet-ta"), + ("passeggiata", "pas-seg-gia-ta"), + ("caffè", "caf-fè"), + ("tavolo", "ta-vo-lo"), + ("bambino", "bam-bi-no"), + ("scuola", "scuo-la"), + ("notte", "not-te"), + ("giorno", "gior-no"), + ("acqua", "ac-qua"), + // Under five letters (`Limits::min_word`): no break. + ("casa", "casa"), + ("pane", "pane"), + ("libro", "li-bro"), + ("re", "re"), + ("più", "più"), + ] { + assert_eq!(it(w), want, "{w}"); + } + assert_eq!(word_breaks_in("famiglia", ProofLang::It).1, Source::Patterns); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn portuguese_patterns_load() { + let p = Patterns::pt(); + assert!(p.len() > 300, "{}", p.len()); + // `hard-ware`, `soft-ware` — and the file's last pattern line, a bare `1-`, which the + // reader counts as a (never-matching) exception because it contains a hyphen. + assert_eq!(p.exception_count(), 3); + assert_eq!(p.exception("hardware"), Some(&[4][..])); + assert_eq!(p.exception("software"), Some(&[4][..])); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn portuguese_words_hyphenate_by_portuguese_rules() { + for (w, want) in [ + ("problema", "pro-ble-ma"), + ("coração", "co-ra-ção"), + ("paralelepípedo", "pa-ra-le-le-pí-pe-do"), + ("computador", "com-pu-ta-dor"), + ("escrever", "es-cre-ver"), + ("saudade", "sau-da-de"), + ("extraordinário", "ex-tra-or-di-ná-rio"), + ("automático", "au-to-má-ti-co"), + // One syllable: no break. + ("flor", "flor"), + ("bem", "bem"), + ] { + assert_eq!(pt(w), want, "{w}"); + } + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn polish_words_hyphenate_by_polish_rules() { + for (w, want) in [ + ("konstytucja", "kon-sty-tu-cja"), + ("konstytucyjnego", "kon-sty-tu-cyj-ne-go"), + ("Rzeczpospolita", "Rzecz-po-spo-li-ta"), + ("przedsiębiorstwo", "przed-się-bior-stwo"), + ("pięćdziesięciu", "pięć-dzie-się-ciu"), + ("dziewczyna", "dziew-czy-na"), + ("książka", "książ-ka"), + ("Warszawa", "War-sza-wa"), + ("szczęście", "szczę-ście"), + ("rzeczywistość", "rze-czy-wi-stość"), + ("najpiękniejszy", "naj-pięk-niej-szy"), + ("wyobraźnia", "wy-obraź-nia"), + ("gżegżółka", "gżeg-żół-ka"), + // One syllable, or a single letter left over: no break. + ("chrząszcz", "chrząszcz"), + ("źdźbło", "źdźbło"), + ("obiad", "obiad"), + // The pattern file's exception words. + ("nigdy", "ni-gdy"), + ("przynajmniej", "przy-naj-mniej"), + ("skądinąd", "skąd-inąd"), + // Compounds: each part, never next to the hyphen. + ("biało-czerwony", "bia-ło-czer-wo-ny"), + ] { + assert_eq!(pl(w), want, "{w}"); + } + assert_eq!(word_breaks_in("nigdy", ProofLang::Pl).1, Source::Dictionary); + assert_eq!(word_breaks_in("konstytucja", ProofLang::Pl).1, Source::Patterns); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn polish_heuristic_for_foreign_letters() { + // Letters outside the Polish alphabet: the Polish syllable heuristic, digraphs kept whole. + assert_eq!(word_breaks_in("Pöppelmann", ProofLang::Pl).1, Source::Heuristic); + assert_eq!(pl("Pöppelmann"), "Pöp-pel-mann"); + assert_eq!(pl("Müllerowi"), "Mül-le-ro-wi"); + let c: Vec = "kawiarnié".chars().collect(); + assert_eq!(heuristic_points_pl(&c), vec![2, 6], "ka-wiar-nié: `wi` and `ni` soften, `rn` splits after the sonorant"); + let c: Vec = "szczérze".chars().collect(); + assert_eq!(heuristic_points_pl(&c), vec![5], "szczé-rze: `rz` stays whole"); + } + + #[test] + fn languages_without_patterns_keep_the_english_ones() { + for w in ["hyphenation", "typography", "internationalization"] { + for lang in [ProofLang::Other, ProofLang::Cs, ProofLang::Id, ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy] { + assert_eq!(hyphen_points_in(w, &Limits::default(), lang), hyphen_points(w, &Limits::default()), "{w}"); + } + } + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn german_patterns_load() { + let p = Patterns::de(); + assert_eq!(p.len(), 36_709); + // The file has no exception words, and its `\message{…}` line adds none. + assert_eq!(p.exception_count(), 0); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn german_words_follow_the_german_patterns() { + let de = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::De); + assert_eq!(de("Silbentrennung"), "Sil-ben-tren-nung"); + assert_eq!(de("unterliegende"), "un-ter-lie-gen-de"); + assert_eq!(de("öffentlich-rechtlicher"), "öf-fent-lich-recht-li-cher"); + assert_eq!(de("Begründung"), "Be-grün-dung"); + assert_eq!(de("Zucker"), "Zu-cker"); + // Swiss spelling: `ss` splits like any double consonant. + assert_eq!(de("Strasse"), "Stras-se"); + assert_eq!(word_breaks_in("Unterlagen", ProofLang::De).1, Source::Patterns); + } + + #[test] + fn bulgarian_words_follow_the_bulgarian_rules() { + let bg = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::Bg); + assert_eq!(bg("сричкопренасяне"), "срич-ко-пре-на-ся-не"); + assert_eq!(bg("Таблица"), "Та-бли-ца"); + // A single letter is never left on a line (the paragraph limits): България, not Бъл-га-ри-я. + assert_eq!(bg("България"), "Бъл-га-рия"); + assert_eq!(bg("„шофьор“,"), "„шо-фьор“,"); + assert_eq!(word_breaks_in("таблица", ProofLang::Bg).1, Source::Rules); + // Latin words in a Bulgarian run are hyphenated as English. + for w in ["publication", "WordCraft"] { + assert_eq!(hyphen_points_in(w, &Limits::default(), ProofLang::Bg), hyphen_points(w, &Limits::default()), "{w}"); + } + } +} diff --git a/crates/proof/src/id/grammar.rs b/crates/proof/src/id/grammar.rs new file mode 100644 index 00000000..72ca10c9 --- /dev/null +++ b/crates/proof/src/id/grammar.rs @@ -0,0 +1,105 @@ +//! Generic repeated-word and spacing checks with Indonesian messages. No Indonesian-specific +//! grammar rules are inferred. A word written twice with a space is flagged; since standard +//! Indonesian spelling (PUEBI) writes reduplication with a hyphen (`anak-anak`), the hyphenated +//! form is offered first, then the single word. + +use crate::{Issue, IssueKind, ProofLang, words}; + +fn issue(start: usize, end: usize, message: String, suggestions: Vec) -> Issue { + Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::Id } +} + +pub fn check(text: &str) -> Vec { + let mut issues = Vec::new(); + repeated_words(text, &words(text), &mut issues); + spaces(text, &mut issues); + issues.sort_by_key(|i| (i.start, i.end)); + issues +} + +fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if !gap.is_empty() && gap.chars().all(char::is_whitespace) && w0.chars().all(char::is_alphabetic) && w0.to_lowercase() == w1.to_lowercase() { + v.push(issue(*a0, *b1, format!("Kata berulang: “{w1}”"), vec![format!("{w0}-{w1}"), w0.to_string()])); + } + } +} + +fn spaces(text: &str, v: &mut Vec) { + for (i, c) in text.char_indices() { + if c != ' ' { + continue; + } + let rest = text.get(i + 1..).unwrap_or(""); + let mut it = rest.chars(); + let Some(p) = it.next() else { continue }; + let after = it.next(); + // A ratio/score keeps the spaces in digit-space-colon-space-digit. + let ratio = p == ':' + && text.get(..i).and_then(|before| before.chars().next_back()).is_some_and(|c| c.is_ascii_digit()) + && after == Some(' ') + && it.next().is_some_and(|c| c.is_ascii_digit()); + let flag = !ratio + && match p { + ',' => true, + // `;)` and `;-)` are emoticons. + ';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')), + // ` ...` is an ellipsis, ` .5` a number. + '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), + // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. + ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')), + _ => false, + }; + if flag { + v.push(issue(i, i + 1 + p.len_utf8(), "Spasi sebelum tanda baca".into(), vec![p.to_string()])); + } + } + for (i, _) in text.match_indices(" ") { + if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { + continue; + } + v.push(issue(i, i + 2, "Spasi ganda".into(), vec![" ".into()])); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn repeated_words_and_spaces_have_indonesian_messages() { + let text = "Anak anak , sekolah."; + let issues = check(text); + assert_eq!(issues.len(), 3, "{issues:?}"); + assert_eq!(issues[0].message, "Kata berulang: “anak”"); + assert_eq!(issues[0].suggestions, ["Anak-anak", "Anak"]); + assert!(issues.iter().any(|i| i.message == "Spasi sebelum tanda baca" && &text[i.start..i.end] == " ,")); + assert!(issues.iter().any(|i| i.message == "Spasi ganda")); + assert!(issues.iter().all(|i| i.lang == ProofLang::Id && i.kind == IssueKind::Grammar)); + for text in ["anak-anak bermain", "rumah. Sekolah", "Dr. Budi", "Anak, anak!", "Senyum :) Angka .5 dan titik ...", "kupu-kupu"] { + assert!(check(text).is_empty(), "{text}: {:?}", check(text)); + } + } + + #[test] + fn ratios_and_scores_keep_their_spaces() { + for text in ["3 : 1", "Skor 2 : 1.", "Rasio 1 : 100."] { + assert!(check(text).is_empty(), "{text}: {:?}", check(text)); + } + for text in ["Kata : lain", "3 : kata", "3 :"] { + assert!(check(text).iter().any(|i| i.message == "Spasi sebelum tanda baca"), "{text}"); + } + } + + proptest::proptest! { + #[test] + fn never_panics_and_ranges_are_valid(s in "\\PC{0,80}") { + for issue in check(&s) { + proptest::prop_assert!(issue.start <= issue.end && issue.end <= s.len()); + proptest::prop_assert!(s.is_char_boundary(issue.start) && s.is_char_boundary(issue.end)); + } + } + } +} diff --git a/crates/proof/src/id/mod.rs b/crates/proof/src/id/mod.rs new file mode 100644 index 00000000..04248d63 --- /dev/null +++ b/crates/proof/src/id/mod.rs @@ -0,0 +1,210 @@ +//! Indonesian proofing using the installed Hunspell `id_ID` dictionary (never bundled; Debian and +//! Ubuntu ship it as `hunspell-id`, LibreOffice as its `dict-id` extension). Suggestions use the +//! dictionary's own MAP/REP/TRY/KEY tables. Grammar is limited to generic repeated-word and +//! spacing checks with Indonesian messages; hyphenation keeps the English fallback (no Indonesian +//! patterns are bundled). + +pub mod grammar; +mod suggest; +#[cfg(test)] +pub(crate) mod test_dict; + +use std::sync::Arc; + +use crate::affix::AffixDict; +use crate::slot::DictSlot; + +pub use suggest::suggest; + +/// File names of an Indonesian Hunspell dictionary (`in` is the old ISO 639 code, still used by +/// Java locales). +const NAMES: &[&str] = &["id_ID", "id-ID", "id", "in_ID"]; + +/// The Indonesian dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Indonesian"); + +/// The Indonesian dictionary: the installed Hunspell `id_ID` (read on first use), one given with +/// [`install_dictionary`], or an empty one when none is available (Indonesian spelling is off +/// then; see [`has_dictionary`]). +pub fn dictionary() -> Arc { + DICT.get() +} + +/// Use this Indonesian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to +/// UTF-8), for example one the user picked. On error the current dictionary stays. +pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { + DICT.install(aff, dic) +} + +/// Is an Indonesian dictionary available (spelling is checked)? +pub fn has_dictionary() -> bool { + DICT.has_dictionary() +} + +/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number +/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own. +const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"]; + +/// Is `word` spelled correctly in Indonesian? Numbers, single letters, short words in capitals +/// (acronyms: `DPR`, `KTP`), words with digits and words in other scripts are accepted, and so +/// is every word when no Indonesian dictionary is installed. A hyphenated word is correct when +/// the dictionary has it whole (`kupu-kupu`, `sayur-mayur`) or has each part (`anak-anak`, +/// `rumah-rumah`); a hyphenated clitic (`KTP-nya`, `se-Indonesia`) counts as known. +pub fn is_correct(word: &str) -> bool { + is_correct_with(&dictionary(), word) +} + +/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off, +/// short words in capitals are checked against the dictionary too, and with Ignore words that +/// contain numbers off, so are words with digits. +pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool { + check(&dictionary(), word, opts) +} + +/// [`is_correct`] against a given dictionary (an empty one accepts everything). +pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { + check(d, word, &crate::Options::default()) +} + +/// [`is_correct_with_options`] against a given dictionary. +fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { + return true; + }; + if w.contains('-') { + return known(d, w) + || w.split('-').all(|p| { + p.chars().count() <= 1 + || CLITICS.iter().any(|c| p.eq_ignore_ascii_case(c)) + || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= 6) + || known(d, p) + }); + } + known(d, w) +} + +/// Is one word in the dictionary or the user dictionary? +fn known(d: &AffixDict, w: &str) -> bool { + d.check(w) || crate::in_user_dictionary(&w.to_lowercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dictionary_lookup_accepts_indonesian_file_names() { + let dir = std::env::temp_dir().join(format!("wordcraft-id-aliases-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + for name in ["id_ID", "id-ID", "id", "in_ID"] { + std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap(); + std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap(); + let (aff, dic) = crate::hunspell::find_in(&dir, NAMES).unwrap(); + let d = crate::hunspell::load(&aff, &dic).unwrap(); + assert!(is_correct_with(&d, "rumah")); + std::fs::remove_file(aff).unwrap(); + std::fs::remove_file(dic).unwrap(); + } + std::fs::remove_dir(dir).unwrap(); + } + + #[test] + fn indonesian_words_affixes_case_and_hyphens() { + test_dict::install(); + let d = dictionary(); + assert!(has_dictionary()); + assert_eq!(d.len(), 20); + assert!(d.rule_count() > 0); + for w in [ + "rumah", + "makan", + "makanan", + "dimakan", + "membaca", + "bacaan", + "dibaca", + "Jakarta", + "Rumah", + "SEKOLAH", + "anak-anak", + "rumah-rumah", + "kupu-kupu", + "KTP-nya", + "se-Indonesia", + "Jakarta-lah", + "DPR", + ] { + assert!(is_correct(w), "{w}"); + } + for w in ["rumha", "makanann", "skolah", "jakarta", "kupu", "anak-anka", "mAkan", "bukunya-xyz"] { + assert!(!is_correct(w), "{w}"); + } + } + + #[test] + fn forbidden_words_are_rejected_in_every_case() { + test_dict::install(); + for w in ["zqterlarang", "Zqterlarang", "ZQTERLARANG"] { + assert!(!is_correct(w), "{w}"); + } + } + + #[test] + fn missing_dictionary_and_ignored_inputs_are_quiet() { + let empty = AffixDict::default(); + assert!(is_correct_with(&empty, "rumha")); + assert!(install_dictionary("FLAG long\n", "1\nrumah\n").is_err()); + let d = AffixDict::from_hunspell(test_dict::AFF, test_dict::DIC, &[]).unwrap(); + for w in ["", "a", "2026", "RI", "ke-2", "日本語", "-", "--"] { + assert!(is_correct_with(&d, w), "{w}"); + } + } + + #[test] + fn user_dictionary_applies() { + test_dict::install(); + assert!(!is_correct("zqkatasendiri")); + crate::add_word("Zqkatasendiri"); + assert!(is_correct("zqkatasendiri")); + } + + #[test] + fn indonesian_dispatch_messages_options_and_mixed_spans() { + test_dict::install(); + assert!(crate::has_dictionary(Some("id-ID"))); + assert!(crate::is_correct_for("rumah", Some("ind"))); + let text = "Rumah sekolha"; + let issues = crate::check_text(text, &[(0..text.len(), crate::ProofLang::Id)]); + assert_eq!(issues.len(), 1, "{issues:?}"); + assert_eq!(&text[issues[0].start..issues[0].end], "sekolha"); + assert_eq!(issues[0].message, "Kemungkinan salah eja"); + assert_eq!(issues[0].lang, crate::ProofLang::Id); + assert!(crate::suggest_in("sekolha", 8, crate::ProofLang::Id).contains(&"sekolah".into())); + let text = "Rumah this is is wrong"; + let issues = crate::check_text(text, &[(0..5, crate::ProofLang::Id), (6..text.len(), crate::ProofLang::En)]); + assert!(issues.iter().any(|i| i.lang == crate::ProofLang::En && i.message.starts_with("Repeated word"))); + assert!(issues.iter().all(|i| i.start >= 6)); + let opts = crate::Options { ignore_uppercase: false, ..Default::default() }; + assert!(!crate::is_correct_in_with("ZQTERLARANG", crate::ProofLang::Id, &opts)); + } + + /// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Indonesian checks + /// those words against the dictionary instead of accepting them. + #[test] + fn proofing_options_reach_the_indonesian_check() { + test_dict::install(); + let lang = crate::ProofLang::Id; + let defaults = crate::Options::default(); + let check_caps = crate::Options { ignore_uppercase: false, ..defaults }; + let check_digits = crate::Options { ignore_numbers: false, ..defaults }; + assert!(crate::is_correct_in_with("RUMHA", lang, &defaults), "short capitals are acronyms by default"); + assert!(!crate::is_correct_in_with("RUMHA", lang, &check_caps)); + assert!(crate::is_correct_in_with("RUMAH", lang, &check_caps)); + assert!(crate::is_correct_in_with("RUMHA-nya", lang, &defaults)); + assert!(!crate::is_correct_in_with("RUMHA-nya", lang, &check_caps), "a part in capitals is checked too"); + assert!(crate::is_correct_in_with("rumha2", lang, &defaults)); + assert!(!crate::is_correct_in_with("rumha2", lang, &check_digits)); + assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); + assert!(is_correct("RUMHA"), "direct checks keep the defaults"); + } +} diff --git a/crates/proof/src/id/suggest.rs b/crates/proof/src/id/suggest.rs new file mode 100644 index 00000000..b42efde9 --- /dev/null +++ b/crates/proof/src/id/suggest.rs @@ -0,0 +1,56 @@ +//! Indonesian suggestions: the shared ranking ([`crate::affix_suggest`]: case, MAP, REP, missing +//! spaces, KEY, TRY edits and a stem scan); every table comes from the installed `id_ID` +//! dictionary, so nothing Indonesian is hard-coded here. + +use crate::affix::AffixDict; + +/// Spelling suggestions for an Indonesian word, best first, at most `max`, in the word's case. +pub fn suggest(word: &str, max: usize) -> Vec { + suggest_in(&super::dictionary(), word, max) +} + +/// [`suggest`] against a given dictionary. +pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { + crate::affix_suggest::suggest_in(d, word, max, &crate::affix_suggest::Tuning::DEFAULT) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn indonesian_suggestions_use_runtime_tables_and_case() { + crate::id::test_dict::install(); + for (word, want) in [ + ("rumha", "rumah"), + ("sekolha", "sekolah"), + ("Sekola", "Sekolah"), + ("jakarta", "Jakarta"), + ("makanna", "makanan"), + ("djalan", "jalan"), + ("dsn", "dan"), + ("sayadan", "saya dan"), + ("membcaa", "membaca"), + ] { + assert!(suggest(word, 8).contains(&want.into()), "{word}: {:?}", suggest(word, 8)); + } + } + + #[test] + fn suggestions_stay_bounded_and_skip_forbidden_words() { + crate::id::test_dict::install(); + for w in ["", "dua kata", &"a".repeat(41)] { + assert!(suggest(w, 4).is_empty()); + } + assert!(suggest("rumha", 0).is_empty()); + assert!(suggest_in(&AffixDict::default(), "rumha", 4).is_empty()); + assert!(suggest("rumah", 4).len() <= 4); + assert!(!suggest("rumah", 4).contains(&"rumah".into())); + assert!(!suggest("zqterlarag", 8).contains(&"zqterlarang".into())); + } + + #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); + } +} diff --git a/crates/proof/src/id/test_dict.rs b/crates/proof/src/id/test_dict.rs new file mode 100644 index 00000000..23c8b97e --- /dev/null +++ b/crates/proof/src/id/test_dict.rs @@ -0,0 +1,49 @@ +//! Hand-written synthetic Indonesian test data. No words, rules or suggestion tables were copied +//! from a system dictionary. This is a small fixture, not a redistributable Indonesian dictionary. + +pub const AFF: &str = "SET UTF-8 +FLAG UTF-8 +FORBIDDENWORD F +TRY aniekrumgtslhbdopy +KEY qw|as|zx +MAP 1 +MAP eé +REP 2 +REP oe u +REP dj j +SFX A Y 1 # synthetic -an +SFX A 0 an . +PFX D Y 1 +PFX D 0 di . +PFX M Y 1 +PFX M 0 mem b +"; + +pub const DIC: &str = "20 +rumah +makan/AD +baca/ADM +sekolah +anak +buku +kupu-kupu +Jakarta +Indonesia +saya +kamu +dan +yang +ini +itu +pergi +jalan/A +besar +kecil +zqterlarang/F +"; + +pub fn install() { + if let Err(e) = super::install_dictionary(AFF, DIC) { + panic!("test dictionary: {e}"); + } +} diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs new file mode 100644 index 00000000..46302936 --- /dev/null +++ b/crates/proof/src/it/grammar.rs @@ -0,0 +1,472 @@ +//! Italian grammar and punctuation checks: cheap, rule-based and tuned to stay quiet when +//! unsure (a missed mistake costs less than a wrong mark on correct text). Messages are in +//! Italian. +//! +//! - **Repeated word** (`la la`), except words Italian doubles on purpose (`piano piano`, +//! `sì sì`, interjections). +//! - **Space before punctuation** (`bene ,`) and **double spaces**; emoticons (` :)`) and +//! ellipses are left alone. +//! - **Capital letter at the start of a sentence**, knowing Italian abbreviations (`ecc.`, +//! `dott.`, `prof.`, `sig.`), initials, ordinal numbers (`1. posto`) and ellipses. +//! - **Doubled punctuation** (`,,`, `..`, `;;`). +//! - **`po'`**: `un pò` and `un po` → `un po'` (a truncation of `poco`: apostrophe, no accent). +//! - **`qual è`**: `qual'è`, `qual'era` → `qual è`, `qual era` (a truncation, never elided). +//! - **`È`**: `E'` written for the verb → `È`. +//! - **`un` / `un'`**: the apostrophe only before a feminine word. `un'altro`, `un'anno` → +//! `un altro`, `un anno` (a masculine word in `-o`; `auto` and `eco` are feminine), and +//! `un altra`, `un amica` → `un'altra`, `un'amica` (a word in `-a`, except the masculine and +//! common-gender ones: `-ista`, `-ita`, `-ota`, `-cida`, `-ma`, `-eta`, `atleta`, `acrobata`…). +//! The same for `nessun`, `ciascun`, `qualcun` and `alcun`. +//! +//! No comma rule and no English rule (such as `a`/`an`) applies to Italian text. + +use crate::{Issue, IssueKind, ProofLang, words}; + +/// Words Italian repeats on purpose: emphatic doubling, interjections, onomatopoeia. +const ALLOWED_REPEATS: &[&str] = &[ + "sì", "no", "piano", "bene", "male", "ecco", "oh", "ah", "eh", "mh", "bah", "ehi", "urrà", "evviva", "ciao", "addio", "adagio", "forte", "lento", + "chiaro", "pian", "poco", "quasi", "presto", "subito", "zitto", "via", "così", "tanto", "molto", "pure", "dai", "su", "avanti", "basta", "aiuto", +]; + +/// Indefinite articles (and the like) that elide to `un'` before a feminine word only. +const UN_FORMS: &[&str] = &["un", "nessun", "ciascun", "qualcun", "alcun"]; + +/// Feminine words in `-o` that start with a vowel (`un'auto`, `un'eco`). +const FEMININE_IN_O: &[&str] = &["auto", "autoradio", "eco", "ecografo"]; + +/// Endings of masculine or common-gender words in `-a` (`un artista`, `un idiota`, `un enigma`). +const MASCULINE_A_ENDINGS: &[&str] = &["ista", "ita", "ota", "cida", "ma", "eta", "nauta", "iatra", "arca"]; + +/// Words in `-a` after `un` that aren't feminine nouns: masculine or common-gender ones that fit +/// no ending above, and verbs and pronouns (`un ha`, `un una`). +const NOT_FEMININE_IN_A: &[&str] = &["acrobata", "autodidatta", "alfa", "omega", "ha", "una"]; + +/// Abbreviations written with a final period after which a sentence goes on in lower case +/// (lower case, without the period). +const ABBREVIATIONS: &[&str] = &[ + "ecc", + "es", + "dott", + "prof", + "profssa", + "ing", + "avv", + "sig", + "sigg", + "sigra", + "sigr", + "geom", + "rag", + "dottssa", + "avvoc", + "pag", + "pagg", + "tel", + "art", + "artt", + "fig", + "figg", + "vol", + "voll", + "ed", + "sez", + "cap", + "capp", + "par", + "n", + "num", + "vs", + "cfr", + "cfrt", + "op", + "cit", + "ibid", + "sez", + "all", + "tav", + "pag", + "ditta", + "soc", + "spa", + "srl", + "on", + "sen", + "spett", + "eg", + "ca", + "v", + "vv", + "sec", + "min", + "max", + "lun", + "mar", + "mer", + "gio", + "ven", + "sab", + "dom", + "gen", + "feb", + "mar", + "apr", + "mag", + "giu", + "lug", + "ago", + "set", + "ott", + "nov", + "dic", + "genn", + "febb", + "sett", + "dottor", + "professor", + "cav", + "commend", + "mons", + "fr", + "suor", + "s", + "ss", + "b", + "beato", +]; + +fn issue(start: usize, end: usize, message: String, suggestions: Vec) -> Issue { + Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::It } +} + +/// Is `prev`, before a period, an abbreviation, an initial, a number or a Roman numeral (after +/// which a lower-case word doesn't start a sentence)? +fn abbreviation_or_number(prev: &str) -> bool { + prev.chars().count() <= 1 + || prev.chars().any(|c| c.is_ascii_digit()) + || (prev.chars().all(|c| matches!(c, 'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M'))) + || ABBREVIATIONS.contains(&prev.to_lowercase().as_str()) +} + +/// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after +/// an abbreviation), then space, with nothing but quotes and brackets around? +fn ends_sentence(gap: &str, prev: &str) -> bool { + let t = gap.trim_start_matches(['"', '”', '’', '\'', ')', '»', ']', '\u{1}']); + let Some(end) = t.chars().next() else { return false }; + if !matches!(end, '.' | '!' | '?') || t.starts_with("..") { + return false; + } + if end == '.' && abbreviation_or_number(prev) { + return false; + } + let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(['"', '”', '’', '\'', ')', '»', ']']); + rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—')) +} + +fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if !gap.is_empty() + && gap.chars().all(char::is_whitespace) + && w0.chars().all(char::is_alphabetic) + && w0.to_lowercase() == w1.to_lowercase() + && !ALLOWED_REPEATS.contains(&w0.to_lowercase().as_str()) + { + v.push(issue(*a0, *b1, format!("Parola ripetuta: \"{w1}\""), vec![w0.to_string()])); + } + } +} + +fn spaces(text: &str, v: &mut Vec) { + for (i, c) in text.char_indices() { + if c != ' ' { + continue; + } + let rest = text.get(i + 1..).unwrap_or(""); + let mut it = rest.chars(); + let Some(p) = it.next() else { continue }; + let after = it.next(); + let flag = match p { + ',' => true, + // `;)` and `;-)` are emoticons. + ';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')), + // ` ...` is an ellipsis, ` .5` a number. + '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), + // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. + ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')), + _ => false, + }; + if flag { + v.push(issue(i, i + 1 + p.len_utf8(), "Spazio prima della punteggiatura".into(), vec![p.to_string()])); + } + } + for (i, _) in text.match_indices(" ") { + if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { + continue; + } + v.push(issue(i, i + 2, "Spazio doppio".into(), vec![" ".into()])); + } +} + +fn sentence_capitals(text: &str, ws: &[(usize, usize)], capital_first: bool, v: &mut Vec) { + let mut after_end = capital_first; + for (a, b) in ws { + let Some(w) = text.get(*a..*b) else { continue }; + if after_end && w.chars().next().is_some_and(char::is_lowercase) && w.chars().all(char::is_alphabetic) { + let fixed: String = w.chars().next().map(|f| f.to_uppercase().chain(w.chars().skip(1)).collect()).unwrap_or_default(); + v.push(issue(*a, *b, "Maiuscola a inizio frase".into(), vec![fixed])); + } + after_end = ends_sentence(text.get(*b..).unwrap_or(""), w); + } +} + +fn doubled_punctuation(text: &str, v: &mut Vec) { + for pat in [",,", "..", ";;", "::", "??", "!!"] { + for (i, _) in text.match_indices(pat) { + if pat == ".." && text.get(i + 2..).is_some_and(|r| r.starts_with('.')) { + continue; // ellipsis + } + v.push(issue(i, i + pat.len(), "Punteggiatura doppia".into(), vec![pat[..1].into()])); + } + } +} + +/// Is `c` an apostrophe (straight or typographic)? +fn is_apostrophe(c: char) -> bool { + matches!(c, '\'' | '’') +} + +/// `pò` (never accented) and `po` without its apostrophe → `po'`. `Po` (the river) is left alone. +fn po(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for (a, b) in ws { + let Some(w) = text.get(*a..*b) else { continue }; + let apostrophe = text.get(*b..).and_then(|r| r.chars().next()).is_some_and(is_apostrophe); + let fix = match w { + "pò" => "po'", + "Pò" => "Po'", + "po" if !apostrophe => "po'", + _ => continue, + }; + v.push(issue(*a, *b, "Si scrive «po'», con l'apostrofo e senza accento".into(), vec![fix.into()])); + } +} + +/// `qual'è` → `qual è` (any case): the fix when `word` is `qual` + apostrophe + `è`/`era`. +pub(crate) fn qual_elided(word: &str) -> Option { + let head_end = word.char_indices().take(4).last().map(|(i, c)| i + c.len_utf8())?; + let (head, rest) = (word.get(..head_end)?, word.get(head_end..)?); + let tail = rest.strip_prefix(is_apostrophe)?; + let tail_ok = matches!(tail.to_lowercase().as_str(), "è" | "era"); + (head.eq_ignore_ascii_case("qual") && tail_ok).then(|| format!("{head} {tail}")) +} + +fn qual_e(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for (a, b) in ws { + if let Some(fix) = text.get(*a..*b).and_then(qual_elided) { + v.push(issue(*a, *b, "«Qual è» si scrive senza apostrofo".into(), vec![fix])); + } + } +} + +/// `E'` for the verb → `È`. A quoted letter (`la lettera 'E'`) is left alone. +fn e_apostrophe(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for (a, b) in ws { + if text.get(*a..*b) != Some("E") { + continue; + } + let mut after = text.get(*b..).unwrap_or("").chars(); + let Some(ap) = after.next().filter(|&c| is_apostrophe(c)) else { continue }; + let ends = after.next().is_none_or(char::is_whitespace); + let quoted = text.get(..*a).and_then(|t| t.chars().next_back()).is_some_and(|c| is_apostrophe(c) || c == '‘'); + if ends && !quoted { + v.push(issue(*a, *b + ap.len_utf8(), "«È» si scrive con l'accento, non con l'apostrofo".into(), vec!["È".into()])); + } + } +} + +/// Does `w` (lower case) start with a vowel or `h`, as the words `un'` elides before? +fn starts_with_vowel(w: &str) -> bool { + w.chars().next().is_some_and(|c| "aeiouhàèéìòóù".contains(c)) +} + +/// Is `w` (lower case) a feminine word in `-a` (`altra`, `amica`, `ora`), as far as the ending +/// tells? +fn feminine_in_a(w: &str) -> bool { + w.chars().count() >= 2 && w.ends_with('a') && !MASCULINE_A_ENDINGS.iter().any(|e| w.ends_with(e)) && !NOT_FEMININE_IN_A.contains(&w) +} + +/// `un'` before a masculine word (`un'altro` → `un altro`), and `un` without the apostrophe +/// before a feminine one (`un altra` → `un'altra`). Only lower-case words: a name's gender isn't +/// in its ending. +fn un_apostrophe(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for (a, b) in ws { + let Some(w) = text.get(*a..*b) else { continue }; + let Some((head, tail)) = w.split_once(is_apostrophe) else { continue }; + if UN_FORMS.contains(&head.to_lowercase().as_str()) + && tail.chars().all(|c| c.is_alphabetic() && c.is_lowercase()) + && starts_with_vowel(tail) + && tail.ends_with('o') + && !FEMININE_IN_O.contains(&tail) + { + v.push(issue(*a, *b, format!("Davanti a un nome maschile «{head}» non vuole l'apostrofo"), vec![format!("{head} {tail}")])); + } + } + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if gap == " " + && UN_FORMS.contains(&w0.to_lowercase().as_str()) + && w1.chars().all(|c| c.is_alphabetic() && c.is_lowercase()) + && starts_with_vowel(w1) + && feminine_in_a(w1) + { + v.push(issue(*a0, *b1, format!("Davanti a un nome femminile si scrive «{w0}'», con l'apostrofo"), vec![format!("{w0}'{w1}")])); + } + } +} + +/// Grammar issues in a paragraph of Italian text, in order. +pub fn check(text: &str) -> Vec { + check_with(text, true) +} + +/// [`check`] where `capital_first` says whether the text's first word must start a sentence with a capital +/// (false for a list item, which may be a fragment). +pub fn check_with(text: &str, capital_first: bool) -> Vec { + let ws: Vec<(usize, usize)> = words(text); + let mut v = Vec::new(); + repeated_words(text, &ws, &mut v); + spaces(text, &mut v); + sentence_capitals(text, &ws, capital_first, &mut v); + doubled_punctuation(text, &mut v); + po(text, &ws, &mut v); + qual_e(text, &ws, &mut v); + e_apostrophe(text, &ws, &mut v); + un_apostrophe(text, &ws, &mut v); + v.sort_by_key(|i| (i.start, i.end)); + v +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn italian_grammar_flags() { + test_dict_install(); + let t = "questo questo è un test , fatto male.. Vedi dott. rossi"; + let v = check(t); + let msgs: Vec<&str> = v.iter().map(|i| i.message.as_str()).collect(); + assert!(msgs.iter().any(|m| m.contains("ripetuta")), "{msgs:?}"); + assert!(msgs.iter().any(|m| m.contains("prima della punteggiatura")), "{msgs:?}"); + assert!(msgs.iter().any(|m| m.contains("doppio")), "{msgs:?}"); + assert!(msgs.iter().any(|m| m.contains("doppia")), "{msgs:?}"); + assert!(v.iter().all(|i| i.lang == ProofLang::It), "{v:?}"); + } + + #[test] + fn italian_grammar_stays_quiet() { + test_dict_install(); + assert!( + check("Vado a Roma. Il dott. Rossi è già lì... piano piano!").is_empty(), + "{:?}", + check("Vado a Roma. Il dott. Rossi è già lì... piano piano!") + ); + assert!(check("Sì sì, bene bene :)").is_empty()); + } + + /// The fixes the rules offer for `t`, in order. + fn fixes(t: &str) -> Vec<(String, String)> { + check(t).into_iter().map(|i| (t.get(i.start..i.end).unwrap_or("").to_string(), i.suggestions.join("|"))).collect() + } + + fn pair(a: &str, b: &str) -> (String, String) { + (a.to_string(), b.to_string()) + } + + #[test] + fn po_takes_an_apostrophe() { + assert_eq!(fixes("Dammi un pò di pane"), [pair("pò", "po'")]); + assert_eq!(fixes("Dammi un po di pane"), [pair("po", "po'")]); + assert_eq!(fixes("Pò di più"), [pair("Pò", "Po'")]); + assert!(fixes("Dammi un po' di pane, un po’ d'acqua").is_empty()); + assert!(fixes("Il Po scorre a Torino").is_empty(), "the river"); + } + + #[test] + fn qual_e_has_no_apostrophe() { + assert_eq!(fixes("Dimmi qual'è il problema"), [pair("qual'è", "qual è")]); + assert_eq!(fixes("Qual’era la strada?"), [pair("Qual’era", "Qual era")]); + assert!(fixes("Qual è il problema? Quale era?").is_empty()); + assert!(fixes("Quale'è").is_empty(), "not the truncation"); + assert_eq!(qual_elided("qual'è").as_deref(), Some("qual è")); + assert_eq!(qual_elided("qua"), None); + assert_eq!(qual_elided(""), None); + } + + #[test] + fn e_apostrophe_becomes_e_grave() { + assert_eq!(fixes("E' vero"), [pair("E'", "È")]); + assert_eq!(fixes("Lo so. E’ tardi"), [pair("E’", "È")]); + assert!(fixes("È vero. E poi?").is_empty()); + assert!(fixes("La lettera 'E' è una vocale").is_empty(), "a quoted letter"); + } + + #[test] + fn un_apostrophe_follows_the_gender() { + assert_eq!(fixes("Ho visto un'altro amico"), [pair("un'altro", "un altro")]); + assert_eq!(fixes("È passato un’anno"), [pair("un’anno", "un anno")]); + assert_eq!(fixes("Prendo un altra strada"), [pair("un altra", "un'altra")]); + assert_eq!(fixes("Ho un amica a Roma"), [pair("un amica", "un'amica")]); + assert_eq!(fixes("Non c'è nessun altra via"), [pair("nessun altra", "nessun'altra")]); + for ok in [ + "Un altro amico", + "Un'altra amica", + "Un'auto nuova", + "Un'eco lontana", + "Un artista", + "Un atleta", + "Un idiota", + "Un ipocrita", + "Un enigma", + "Un omicida", + "Un astronauta", + "Un acrobata", + "Un'ora", + "Un'idea", + "Un uomo", + "Un'Alfa", + "Un Andrea", + "Un ha", + ] { + assert!(fixes(ok).is_empty(), "{ok}: {:?}", fixes(ok)); + } + } + + #[test] + fn more_emphatic_doubles_are_allowed() { + for ok in ["Pian piano", "Poco poco", "Zitto zitto", "Via via", "Così così", "Dai dai"] { + assert!(fixes(ok).is_empty(), "{ok}: {:?}", fixes(ok)); + } + } + + proptest::proptest! { + #[test] + fn never_panics(s in "\\PC{0,80}") { + for i in check(&s) { + proptest::prop_assert!(i.start <= i.end && i.end <= s.len()); + } + } + + #[test] + fn never_panics_on_italian_like_text(s in "[a-zàèéìòóùA-ZÈ ,.;:!?'’‘\"«»…\\-]{0,60}") { + for i in check(&s) { + proptest::prop_assert!(i.start <= i.end && i.end <= s.len() && s.is_char_boundary(i.start) && s.is_char_boundary(i.end)); + } + } + } + + fn test_dict_install() { + crate::it::test_dict::install(); + } +} diff --git a/crates/proof/src/it/mod.rs b/crates/proof/src/it/mod.rs new file mode 100644 index 00000000..ee5aaaee --- /dev/null +++ b/crates/proof/src/it/mod.rs @@ -0,0 +1,213 @@ +//! Italian proofing. +//! +//! - **Spelling** against the Hunspell `it_IT` dictionary installed on the system (found by +//! [`crate::hunspell`]: `it_IT.aff`/`it_IT.dic` from the OS, a package manager or +//! LibreOffice; it isn't bundled — the upstream file is GPL-licensed). Without it Italian +//! text gets no spelling marks. Elided forms check the word after the apostrophe +//! (`l'amico`, `un'idea`, `c'è`, `dov'è`, `senz'altro`); truncations keep their apostrophe +//! (`po'`, `mo'`); hyphenated compounds are checked part by part; the user dictionary applies +//! as in English. Forms a grammar rule explains better (`pò`, `qual'è`) are left to it. +//! - **Suggestions** tuned for Italian ([`suggest`]): missing accents first (`caffe` → `caffè`, +//! `perche` → `perché`), then the dictionary's `REP` confusions, a missing space, one-letter +//! typos, and a dictionary scan for anything further off. +//! - **Grammar** ([`grammar`]): cheap, reliable rules with Italian messages, which work with +//! no dictionary installed. + +pub mod grammar; +mod suggest; +#[cfg(test)] +pub(crate) mod test_dict; + +use std::sync::Arc; + +use crate::affix::AffixDict; +use crate::slot::DictSlot; + +pub use suggest::suggest; + +/// File names of an Italian Hunspell dictionary (Italy first, then Switzerland). +const NAMES: &[&str] = &["it_IT", "it_CH", "it-IT", "it"]; + +/// The Italian dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Italian"); + +/// The Italian dictionary: the installed Hunspell `it_IT` (read on first use), one given with +/// [`install_dictionary`], or an empty one when none is available (Italian spelling is off +/// then; see [`has_dictionary`]). +pub fn dictionary() -> Arc { + DICT.get() +} + +/// Use this Italian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to +/// UTF-8), for example one the user picked. On error the current dictionary stays. +pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { + DICT.install(aff, dic) +} + +/// Is an Italian dictionary available (spelling is checked)? +pub fn has_dictionary() -> bool { + DICT.has_dictionary() +} + +/// Proclitics (and verb clippings) that elide before a vowel or `h`: the word after the +/// apostrophe is what gets checked (`l'amico`, `un'idea`, `c'è`, `m'hai`, `dov'è`, `anch'io`). +const PROCLITICS: &[&str] = &[ + "l", "un", "una", "uno", "all", "dall", "dell", "nell", "sull", "coll", "pell", "c", "m", "t", "s", "v", "d", "j", "n", "ch", "gl", "quest", + "quell", "bell", "buon", "sant", "grand", "poc", "nessun", "ciascun", "qualcun", "alcun", "dov", "cos", "com", "quand", "anch", "senz", "tutt", + "nient", "mezz", "sott", "tant", "quant", "dev", +]; + +/// Truncated words written with an apostrophe (`un po'`, `a mo' di`, `be'`): [`crate::words`] +/// trims the apostrophe, so spelling sees the bare word; [`grammar`] checks the apostrophe. +const TRUNCATIONS: &[&str] = &["po", "mo", "be"]; + +/// Misspellings a grammar rule reports with a clearer message and fix (`pò` → `po'`, `qual'è` +/// → `qual è`), so spelling leaves them alone rather than marking the word twice. +fn left_to_grammar(lower: &str) -> bool { + lower == "pò" || grammar::qual_elided(lower).is_some() +} + +/// Is `word` spelled correctly in Italian? Numbers, single letters, short words in capitals +/// (acronyms: `RAI`, `TG`), words with digits and words in other scripts are accepted, and so +/// is every word when no Italian dictionary is installed. +pub fn is_correct(word: &str) -> bool { + is_correct_with(&dictionary(), word) +} + +/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off, +/// short words in capitals are checked against the dictionary too, and with Ignore words that +/// contain numbers off, so are words with digits. +pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool { + check(&dictionary(), word, opts) +} + +/// [`is_correct`] against a given dictionary (an empty one accepts everything). +pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { + check(d, word, &crate::Options::default()) +} + +/// [`is_correct_with_options`] against a given dictionary. +fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { + return true; + }; + let lower = w.to_lowercase(); + if TRUNCATIONS.contains(&lower.as_str()) || left_to_grammar(&lower) { + return true; + } + if w.contains('-') { + return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + } + if let Some((head, tail)) = w.split_once(['\'', '’']) + && PROCLITICS.contains(&head.to_lowercase().as_str()) + && !tail.is_empty() + { + return known(d, tail); + } + known(d, w) +} + +/// Is one word (no hyphens, no apostrophe) in the dictionary or the user dictionary? +fn known(d: &AffixDict, w: &str) -> bool { + d.check(w) || crate::in_user_dictionary(&w.to_lowercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_dictionary_loads() { + test_dict::install(); + let d = dictionary(); + assert!(has_dictionary()); + assert!(d.len() > 50, "{}", d.len()); + assert!(d.rule_count() >= 4, "the test dictionary's affix rules"); + assert!(!d.rep().is_empty() && !d.map().is_empty() && !d.try_chars().is_empty()); + } + + #[test] + fn swiss_dictionary_files_are_found() { + // it_CH is searched too, so Swiss users with only it_CH installed get Italian spelling. + let dir = std::env::temp_dir().join(format!("wc-it-ch-{}", std::process::id())); + let _ = std::fs::create_dir_all(&dir); + std::fs::write(dir.join("it_CH.aff"), test_dict::AFF).expect("aff"); + std::fs::write(dir.join("it_CH.dic"), test_dict::DIC).expect("dic"); + let found = crate::hunspell::find_in(&dir, NAMES); + let ok = found.as_ref().is_some_and(|(a, d)| a.ends_with("it_CH.aff") && d.ends_with("it_CH.dic")); + assert!(ok, "{found:?}"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn without_a_dictionary_nothing_is_flagged() { + let none = AffixDict::default(); + for w in ["caffè", "perché", "xqzv", ""] { + assert!(is_correct_with(&none, w), "{w}"); + } + assert!(install_dictionary("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused"); + } + + #[test] + fn italian_words_are_correct() { + test_dict::install(); + for w in [ + "casa", "case", "libro", "libri", "città", "caffè", "perché", "più", "può", "parlare", "parla", "parlano", "rifare", "bello", "belli", + "nuovo", "bianco", "Roma", "Milano", "l'amico", "un'idea", "L'amico", "RAI", "2026", "e", "non", + ] { + assert!(is_correct(w), "{w}"); + } + } + + #[test] + fn elisions_and_truncations_are_correct() { + test_dict::install(); + for w in ["dov'è", "Com'è", "cos'è", "anch'io", "senz'acqua", "tutt'uno", "quand'è", "nessun'amica", "n'ha", "po'", "po", "mo'"] { + assert!(is_correct(w), "{w}"); + } + } + + #[test] + fn misspellings_are_flagged() { + test_dict::install(); + for w in ["caffé", "perche", "lbiro", "parlre", "amicaa", "citta", "quelo", "dov'lbiro", "xyz'casa"] { + assert!(!is_correct(w), "{w}"); + } + } + + #[test] + fn grammar_owned_forms_are_not_marked_twice() { + test_dict::install(); + // `pò` and `qual'è` are reported by the grammar rules, with the fix; spelling stays quiet. + for w in ["pò", "Pò", "qual'è", "Qual’era"] { + assert!(is_correct(w), "{w}"); + assert!(!grammar::check(w).is_empty(), "{w}: grammar reports it"); + } + } + + #[test] + fn user_dictionary_applies() { + test_dict::install(); + assert!(!is_correct("zxqitalianaparola")); + crate::add_word("Zxqitalianaparola"); + assert!(is_correct("zxqitalianaparola")); + } + + /// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Italian checks + /// those words against the dictionary instead of accepting them. + #[test] + fn proofing_options_reach_the_italian_check() { + test_dict::install(); + let lang = crate::ProofLang::It; + let defaults = crate::Options::default(); + let check_caps = crate::Options { ignore_uppercase: false, ..defaults }; + let check_digits = crate::Options { ignore_numbers: false, ..defaults }; + assert!(crate::is_correct_in_with("LIBRRO", lang, &defaults), "short capitals are acronyms by default"); + assert!(!crate::is_correct_in_with("LIBRRO", lang, &check_caps)); + assert!(crate::is_correct_in_with("LIBRO", lang, &check_caps)); + assert!(crate::is_correct_in_with("librro2", lang, &defaults)); + assert!(!crate::is_correct_in_with("librro2", lang, &check_digits)); + assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); + assert!(is_correct("LIBRRO"), "direct checks keep the defaults"); + } +} diff --git a/crates/proof/src/it/suggest.rs b/crates/proof/src/it/suggest.rs new file mode 100644 index 00000000..55f86909 --- /dev/null +++ b/crates/proof/src/it/suggest.rs @@ -0,0 +1,115 @@ +//! Italian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the +//! words Italian runs into their neighbours (`perme` → `per me`, `amilano` → `a milano`) and +//! letters to try for a dictionary without a TRY line. + +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; + +/// Short words often run into the next one (`per me`, `a Milano`, `di Roma`). +const LEADING_WORDS: &[&str] = &[ + "a", "e", "è", "di", "da", "in", "con", "su", "per", "tra", "fra", "che", "chi", "cui", "non", "si", "ci", "vi", "ne", "più", "meno", "molto", + "tanto", "come", "dove", "quando", "mai", "qui", "qua", "lì", "là", "ogni", "altro", "tutto", "questo", "quello", "bene", "male", "anche", + "ancora", "sempre", "già", "poi", "ed", "od", "ad", "un", "uno", "una", "il", "lo", "la", "i", "gli", "le", "del", "al", "dal", "nel", "sul", + "col", "pel", "allo", "dello", "nello", "sullo", "della", "alla", "dalla", "nella", "sulla", "degli", "agli", "dagli", "negli", "sugli", "delle", + "alle", "dalle", "nelle", "sulle", "dei", "ai", "dai", "nei", "sui", "coi", "dei", +]; +/// Short words often run into the one before (enclitic pronouns: `dimmelo` in writing without +/// the doubling, or a dropped space: `perme` → `per me`). +const TRAILING_WORDS: &[&str] = &[ + "mi", "ti", "si", "ci", "vi", "ne", "lo", "la", "li", "le", "gli", "me", "te", "lui", "lei", "noi", "voi", "loro", "che", "cosa", "come", "dove", + "è", +]; + +/// What Italian adds to its dictionary's tables. +const TUNING: Tuning = + Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, alphabet: "aeiortnslcdupmghfbqvàèéìòóùz", ..Tuning::DEFAULT + }; + +/// Spelling suggestions for an Italian word, best first, at most `max`, in the word's letter case. +pub fn suggest(word: &str, max: usize) -> Vec { + suggest_in(&super::dictionary(), word, max) +} + +/// [`suggest`] against a given dictionary. +pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { + crate::affix_suggest::suggest_in(d, word, max, &TUNING) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::affix_suggest::{fold, weighted_distance}; + + fn has(word: &str, want: &str) { + let s = suggest(word, 8); + assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); + } + + #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); + } + + #[test] + fn italian_misspellings_get_the_right_suggestion() { + crate::it::test_dict::install(); + has("caffe", "caffè"); + has("perche", "perché"); + has("cioe", "cioè"); + has("citta", "città"); + has("lbiro", "libro"); + has("parlre", "parlare"); + has("quelo", "quello"); + has("perme", "per me"); + has("amilano", "a Milano"); + has("roma", "Roma"); + has("Caffe", "Caffè"); + } + + #[test] + fn the_likeliest_fix_comes_first() { + crate::it::test_dict::install(); + assert_eq!(suggest("caffe", 3).first().map(String::as_str), Some("caffè")); + assert_eq!(suggest("perche", 3).first().map(String::as_str), Some("perché")); + } + + #[test] + fn edge_cases_stay_empty_or_bounded() { + crate::it::test_dict::install(); + assert!(suggest("", 5).is_empty()); + assert!(suggest("bene", 0).is_empty()); + assert!(suggest(&"a".repeat(41), 5).is_empty()); + assert!(suggest("due parole", 5).is_empty()); + assert!(suggest_in(&crate::affix::AffixDict::default(), "caffe", 5).is_empty()); + assert!(suggest("caffè", 5).len() <= 5); + assert!(!suggest("caffè", 5).contains(&"caffè".to_string())); + } + + #[test] + fn distance_weighs_accents_half() { + let map = vec![vec!['a', 'à'], vec!['e', 'é', 'è']]; + let related = |a: char, b: char| fold(a, &map) == fold(b, &map); + let c = |s: &str| s.chars().collect::>(); + assert_eq!(weighted_distance(&c("caffe"), &c("caffè"), &related, 9), 1); + assert_eq!(weighted_distance(&c("citta"), &c("città"), &related, 9), 1); + assert_eq!(weighted_distance(&c("bene"), &c("bene"), &related, 9), 0); + assert_eq!(weighted_distance(&c("bene"), &c("male"), &related, 2), 3); + assert_eq!(weighted_distance(&c("bene"), &c("benissimo"), &related, 2), 3); + assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4); + } + + #[test] + fn suggestions_are_fast_enough() { + crate::it::test_dict::install(); + // Warm the dictionary, then time a worst case (a long word nothing near): the scan round. + let _ = suggest("bene", 1); + let t = std::time::Instant::now(); + for w in ["xqzvweitalianaparola", "prsidenti", "parallellepipedo"] { + let _ = suggest(w, 6); + } + let ms = t.elapsed().as_secs_f64() * 1000.0; + eprintln!("3 suggestion calls: {ms:.0} ms"); + assert!(ms < 20_000.0, "{ms} ms"); + } +} diff --git a/crates/proof/src/it/test_dict.rs b/crates/proof/src/it/test_dict.rs new file mode 100644 index 00000000..5d594d2f --- /dev/null +++ b/crates/proof/src/it/test_dict.rs @@ -0,0 +1,139 @@ +//! A tiny synthetic Italian Hunspell dictionary for tests (written for these tests; not taken +//! from any real dictionary). Real dictionaries are installed on the system, never bundled. +//! +//! It exercises what the Italian dictionaries use: the default single-character `FLAG` type, +//! suffixes (verb endings `-are`, noun plurals `-o` → `-i`), a prefix (`ri-`), the suggestion +//! tables (`TRY`, `MAP` for accents, `REP` for common confusions) and elided forms with an +//! apostrophe (`l'amico`). + +pub const AFF: &str = "SET UTF-8 +TRY aeiortnslcdupmghfbqvàèéìòóùz +MAP 5 +MAP aà +MAP eéè +MAP iì +MAP oòó +MAP uù +REP 3 +REP caffé caffè +REP perche perché +REP cioe cioè +SFX A Y 3 +SFX A 0 are . +SFX A 0 a . +SFX A 0 ano . +SFX N Y 2 +SFX N o i o +SFX N a e a +PFX R Y 1 +PFX R 0 ri . +"; + +/// Stems (the count line first, as in Hunspell files). +pub const DIC: &str = "97 +città +caffè +perché +più +può +è +già +così +virtù +tribù +menù +tè +Mosca +Roma +Milano +Torino +casa/N +libro/N +cane +gatto +sole +luna +mare +amico +amica +idea +parl/A +mangiare +dormire +fare/R +scuola +bambino/N +albero +fiore +pane +acqua +terra +cielo +notte +giorno +vecchia +vado +con +ha +ho +bel +cioè +la +un +in +piazza +lavoro +tempo +uomo +donna +bello/N +buono +grande +piccolo +nuovo/N +vecchio +bianco +nero +rosso +verde +giallo +uno +due +tre +tutto +come +quando +dove +senza +sopra +sotto +tra +molto +sempre +mai +qui +lì +sì +no +non +che +chi +cui +mio +tuo +suo +nostro +a +per +me +quello +io +"; + +/// Make [`AFF`]/[`DIC`] the Italian dictionary (every test that needs one calls this; they all +/// install the same one, so tests running in parallel agree). +pub fn install() { + if let Err(e) = super::install_dictionary(AFF, DIC) { + panic!("test dictionary: {e}"); + } +} diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs new file mode 100644 index 00000000..8f681408 --- /dev/null +++ b/crates/proof/src/lang.rs @@ -0,0 +1,388 @@ +//! Proofing languages: which dictionary, grammar rules and hyphenation patterns apply to a run of +//! text. A run's language is its BCP 47 tag (`w:lang` in .docx, `CharProps::lang` in the model), +//! resolved with [`ProofLang::from_tag`]. + +/// A language WordCraft can proof, or [`ProofLang::Other`] for one it can't. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum ProofLang { + /// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`] or + /// [`ProofLang::EnCa`]): the public-domain Moby word list, the English grammar rules and the + /// English hyphenation dictionary and patterns. British spellings (`colour`, `centre`, + /// `organise`) are flagged, with the American form suggested ([`crate::en_variants`]). Also + /// text with no language at all (WordCraft's default editing language is US English, like + /// Word's). + #[default] + En, + /// English with British spelling (`en-GB`, `en-AU`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…): + /// [`ProofLang::En`]'s tools, but American spellings (`color`, `center`, `analyze`) are + /// flagged and `-ize` and `-ise` are both right, as in Oxford spelling. + EnGb, + /// Canadian English (`en-CA`): British `-our`, `-re` and `-ence` (`colour`, `centre`, + /// `defence`) with American `-ize` (`organize`), `tire` and `program`. + EnCa, + /// Polish (`pl`, `pl-PL`): the installed Hunspell dictionary (when there is one), the Polish grammar rules and the Polish + /// hyphenation patterns. + Pl, + /// Bulgarian (`bg`, `bg-BG`): hyphenation by the Bulgarian syllable rules ([`crate::bg`]). + /// There is no Bulgarian dictionary or grammar yet, so, as for [`ProofLang::Other`], no + /// spelling or grammar marks. + Bg, + /// Portuguese (`pt`, `pt-BR`, `pt-PT`): the installed Hunspell dictionary (VERO and + /// friends, when there is one), the Portuguese grammar rules and the Portuguese hyphenation + /// patterns. + Pt, + /// Italian (`it`, `it-IT`, `it-CH`): the installed Hunspell dictionary (`it_IT`, when there + /// is one), the Italian grammar rules and the Italian hyphenation patterns. + It, + /// Croatian (`hr`, `hr-HR`) + Hr, + /// Turkish (`tr`, `tr-TR`, `tr-CY`): the installed Hunspell dictionary (when there is one), + /// Turkish grammar rules, spelling suggestions and Turkish syllable hyphenation rules. + Tr, + /// Russian (`ru`, `ru-RU`): the installed Hunspell dictionary (when there is one), the Russian + /// grammar rules and the Russian word-division rules. + Ru, + /// Czech (`cs`, `cs-CZ`): the installed Hunspell dictionary, dictionary-driven suggestions + /// and generic punctuation checks. Hyphenation keeps the fallback English patterns. + Cs, + /// Indonesian (`id`, `id-ID`; also the old `in` and ISO 639-2 `ind`): the installed Hunspell + /// `id_ID` dictionary, dictionary-driven suggestions and generic checks with Indonesian + /// messages. Hyphenation keeps the fallback English patterns. + Id, + /// Spanish of Spain (`es`, `es-ES`, and every region not listed for [`ProofLang::EsMx`], + /// [`ProofLang::Es419`] or [`ProofLang::EsUy`]; also ISO 639-2 `spa`): the installed + /// Hunspell `es_ES` dictionary (or a generic Spanish one), dictionary-driven suggestions and + /// generic checks with Spanish messages ([`crate::es`]). Hyphenation keeps the fallback + /// English patterns. + Es, + /// Mexican Spanish (`es-MX`): [`ProofLang::Es`]'s tools with the installed `es_MX` + /// dictionary, else a Latin American or generic Spanish one. + EsMx, + /// Latin American Spanish (`es-419`, and the Spanish of the Americas other than Mexico and + /// Uruguay: `es-AR`, `es-CO`, `es-US`…): [`ProofLang::Es`]'s tools with the installed `es_419` + /// dictionary, else a generic or another Latin American Spanish one. + Es419, + /// Uruguayan Spanish (`es-UY`): [`ProofLang::Es`]'s tools with the installed `es_UY` + /// dictionary (else the neighbouring Rioplatense `es_AR`, a Latin American or a generic one), + /// and the `vos` forms of verbs (`tenés`, `vení`, `decime`) accepted ([`crate::es::voseo`]). + EsUy, + /// German (`de`, `de-DE`, `de-AT`, `de-CH`…): hyphenation by the German patterns (reformed + /// spelling). There is no German dictionary or grammar yet, so, as for [`ProofLang::Other`], + /// no spelling or grammar marks. + De, + /// A language without proofing tools here (`fr-FR`, `ja-JP`, `x-none`…). Like Word without + /// the language's proofing tools: no spelling or grammar marks, so its words are never + /// flagged as English misspellings. Hyphenation keeps using the English patterns, as before + /// languages were told apart. + Other, +} + +impl ProofLang { + /// Every language with a spelling dictionary, in display order. + pub const PROOFED: [ProofLang; 15] = [ + ProofLang::En, + ProofLang::EnGb, + ProofLang::EnCa, + ProofLang::Pl, + ProofLang::Pt, + ProofLang::It, + ProofLang::Hr, + ProofLang::Tr, + ProofLang::Ru, + ProofLang::Cs, + ProofLang::Id, + ProofLang::Es, + ProofLang::EsMx, + ProofLang::Es419, + ProofLang::EsUy, + ]; + + /// The proofing language of a BCP 47 tag (`pl-PL`, `en_GB`, `PL`…). The primary subtag + /// counts, and for English and Spanish the region too ([`ProofLang::english`], + /// [`ProofLang::spanish`]). No tag (or an empty one) is English, WordCraft's default editing + /// language. + pub fn from_tag(tag: Option<&str>) -> ProofLang { + let Some(tag) = tag.map(str::trim).filter(|t| !t.is_empty()) else { return ProofLang::En }; + let primary = tag.split(['-', '_']).next().unwrap_or(""); + if primary.eq_ignore_ascii_case("en") || primary.eq_ignore_ascii_case("eng") { + ProofLang::english(tag) + } else if primary.eq_ignore_ascii_case("pl") || primary.eq_ignore_ascii_case("pol") { + ProofLang::Pl + } else if primary.eq_ignore_ascii_case("bg") || primary.eq_ignore_ascii_case("bul") { + ProofLang::Bg + } else if primary.eq_ignore_ascii_case("pt") || primary.eq_ignore_ascii_case("por") { + ProofLang::Pt + } else if primary.eq_ignore_ascii_case("it") || primary.eq_ignore_ascii_case("ita") { + ProofLang::It + } else if primary.eq_ignore_ascii_case("hr") || primary.eq_ignore_ascii_case("hrv") { + ProofLang::Hr + } else if primary.eq_ignore_ascii_case("tr") || primary.eq_ignore_ascii_case("tur") { + ProofLang::Tr + } else if primary.eq_ignore_ascii_case("ru") || primary.eq_ignore_ascii_case("rus") { + ProofLang::Ru + } else if primary.eq_ignore_ascii_case("cs") || primary.eq_ignore_ascii_case("ces") || primary.eq_ignore_ascii_case("cze") { + ProofLang::Cs + } else if primary.eq_ignore_ascii_case("id") || primary.eq_ignore_ascii_case("ind") || primary.eq_ignore_ascii_case("in") { + ProofLang::Id + } else if primary.eq_ignore_ascii_case("es") || primary.eq_ignore_ascii_case("spa") { + ProofLang::spanish(tag) + } else if ["de", "deu", "ger"].iter().any(|c| primary.eq_ignore_ascii_case(c)) { + ProofLang::De + } else { + ProofLang::Other + } + } + + /// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], regions + /// that spell the British way are [`ProofLang::EnGb`], and the United States, any other + /// region and no region are [`ProofLang::En`]. + fn english(tag: &str) -> ProofLang { + match region(tag) { + Some(r) if r.eq_ignore_ascii_case("CA") => ProofLang::EnCa, + Some(r) if BRITISH_SPELLING.iter().any(|b| b.eq_ignore_ascii_case(r)) => ProofLang::EnGb, + _ => ProofLang::En, + } + } + + /// The Spanish of a Spanish tag, by its [`region`]: Mexico is [`ProofLang::EsMx`], Uruguay + /// [`ProofLang::EsUy`], Latin America (`419`) and the other countries of the Americas are + /// [`ProofLang::Es419`], and Spain, any other region and no region are [`ProofLang::Es`]. + fn spanish(tag: &str) -> ProofLang { + match region(tag) { + Some(r) if r.eq_ignore_ascii_case("MX") => ProofLang::EsMx, + Some(r) if r.eq_ignore_ascii_case("UY") => ProofLang::EsUy, + Some(r) if LATIN_AMERICA.iter().any(|l| l.eq_ignore_ascii_case(r)) => ProofLang::Es419, + _ => ProofLang::Es, + } + } + + /// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnCa`])? + pub fn is_english(self) -> bool { + matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) + } + + /// Is this one of the Spanishes ([`ProofLang::Es`], [`ProofLang::EsMx`], [`ProofLang::Es419`], + /// [`ProofLang::EsUy`])? + pub fn is_spanish(self) -> bool { + matches!(self, ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy) + } + + /// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`, `de`, `es`); empty for [`ProofLang::Other`]. + pub fn code(self) -> &'static str { + match self { + ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => "en", + ProofLang::Pl => "pl", + ProofLang::Bg => "bg", + ProofLang::Pt => "pt", + ProofLang::It => "it", + ProofLang::Hr => "hr", + ProofLang::Tr => "tr", + ProofLang::Ru => "ru", + ProofLang::Cs => "cs", + ProofLang::Id => "id", + ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => "es", + ProofLang::De => "de", + ProofLang::Other => "", + } + } + + /// Does WordCraft check spelling and grammar in this language? + pub fn is_proofed(self) -> bool { + Self::PROOFED.contains(&self) + } +} + +/// The region of a tag: the first two-letter or three-digit subtag after the language and +/// script, before any singleton; a POSIX `.UTF-8` or `@…` tail is ignored. +fn region(tag: &str) -> Option<&str> { + let tag = tag.split(['.', '@']).next().unwrap_or(""); + // Subtags after a singleton (`x-…` private use, `u-…` extensions) aren't regions. + tag.split(['-', '_']) + .skip(1) + .take_while(|p| p.len() != 1) + .find(|p| (p.len() == 2 && p.chars().all(|c| c.is_ascii_alphabetic())) || (p.len() == 3 && p.chars().all(|c| c.is_ascii_digit()))) +} + +/// Regions whose Spanish is Latin American ([`ProofLang::Es419`]): Latin America and the +/// Caribbean (`419`) and its parts (South America `005`, Central America `013`, the Caribbean +/// `029`), the Spanish-speaking countries of the Americas other than Mexico and Uruguay, and the +/// United States. +const LATIN_AMERICA: [&str; 22] = + ["419", "005", "013", "029", "AR", "BO", "CL", "CO", "CR", "CU", "DO", "EC", "GT", "HN", "NI", "PA", "PE", "PR", "PY", "SV", "US", "VE"]; + +/// Regions whose English uses British spelling ([`ProofLang::EnGb`]): the United Kingdom (and +/// the non-standard `UK`), Ireland, Australia, New Zealand, South Africa, India and other +/// Commonwealth countries, the Crown dependencies and Europe (`150`). +const BRITISH_SPELLING: [&str; 26] = [ + "GB", "UK", "IE", "AU", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH", + "PK", "150", +]; + +/// A BCP 47 tag in canonical case (`pl-PL`, `en-GB`, `zh-Hant-TW`, `x-none`), or `None` when +/// `tag` isn't shaped like one: 1–35 ASCII letters, digits and `-`/`_` separators, starting with +/// a 2–8 letter language (or `x`/`i` for private and grandfathered tags). `_` becomes `-`. +pub fn normalize_tag(tag: &str) -> Option { + let tag = tag.trim(); + if tag.is_empty() || tag.len() > 35 || !tag.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') { + return None; + } + let parts: Vec<&str> = tag.split(['-', '_']).collect(); + if parts.iter().any(|p| p.is_empty() || p.len() > 8) { + return None; + } + let first = parts.first()?; + let language = (2..=8).contains(&first.len()) && first.chars().all(|c| c.is_ascii_alphabetic()); + if !language && !first.eq_ignore_ascii_case("x") && !first.eq_ignore_ascii_case("i") { + return None; + } + let letters = |p: &str| p.chars().all(|c| c.is_ascii_alphabetic()); + // After a singleton (`x-…` private use, `u-…` extensions) subtags are lower case. + let singleton = parts.iter().position(|p| p.len() == 1).unwrap_or(usize::MAX); + let out: Vec = parts + .iter() + .enumerate() + .map(|(i, p)| { + if i >= singleton { + p.to_ascii_lowercase() + } else if i > 0 && p.len() == 2 && letters(p) { + p.to_ascii_uppercase() // region + } else if i > 0 && p.len() == 4 && letters(p) { + let lower = p.to_ascii_lowercase(); + let mut c = lower.chars(); + c.next().map(|f| f.to_ascii_uppercase().to_string() + c.as_str()).unwrap_or_default() // script + } else { + p.to_ascii_lowercase() + } + }) + .collect(); + Some(out.join("-")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tags_normalize() { + assert_eq!(normalize_tag("PL-pl").as_deref(), Some("pl-PL")); + assert_eq!(normalize_tag(" pl_pl ").as_deref(), Some("pl-PL")); + assert_eq!(normalize_tag("pl").as_deref(), Some("pl")); + assert_eq!(normalize_tag("zh-hant-tw").as_deref(), Some("zh-Hant-TW")); + assert_eq!(normalize_tag("es-419").as_deref(), Some("es-419")); + assert_eq!(normalize_tag("x-none").as_deref(), Some("x-none")); + assert_eq!(normalize_tag("de-DE-u-co-PHONEBK").as_deref(), Some("de-DE-u-co-phonebk")); + for bad in ["", " ", "p", "1pl", "pl--PL", "pl PL", "pl-PL;", "polski język", "-pl", "pl-", "abcdefghi", &"a".repeat(40)] { + assert_eq!(normalize_tag(bad), None, "{bad:?}"); + } + } + + fn assert_tags(tags: &[&str], expected: ProofLang) { + for tag in tags { + assert_eq!(ProofLang::from_tag(Some(tag)), expected, "{tag}"); + } + } + + #[test] + fn tags_resolve_primary_languages() { + assert_tags(&["pl", "pl-PL", "PL-pl", "pl_PL", "pl_PL.UTF-8", " pl-PL ", "pol"], ProofLang::Pl); + assert_tags(&["pt", "pt-BR", "PT_br", "pt-BR.UTF-8", " pt-PT ", "por"], ProofLang::Pt); + assert_tags(&["it", "it-IT", "IT_it", "it-CH", "it-CH.UTF-8", " it-IT ", "ita"], ProofLang::It); + assert_tags(&["cs", "cs-CZ", "CS-cz", "cs_CZ", " ces ", "CES", "cze", "CZE"], ProofLang::Cs); + assert_tags(&["id", "id-ID", "ID-id", "id_ID", "id_ID.UTF-8", " id-ID ", "ind", "in", "in_ID"], ProofLang::Id); + assert_eq!(ProofLang::Cs.code(), "cs"); + assert!(ProofLang::Cs.is_proofed() && ProofLang::PROOFED.contains(&ProofLang::Cs)); + assert_eq!(ProofLang::Id.code(), "id"); + assert!(ProofLang::Id.is_proofed() && ProofLang::PROOFED.contains(&ProofLang::Id)); + } + + #[test] + fn tags_resolve_english_regions() { + assert_tags(&["en", "en-US", "EN_us", "en-Latn-US", "en-PH", "en-001", "eng", "en_US.UTF-8", "en-x-gb"], ProofLang::En); + assert_tags( + &[ + "en-GB", + "EN_gb", + "en_GB.UTF-8", + "en-AU", + "en_AU", + "en-NZ", + "en-IE", + "en-ZA", + "en-IN", + "en-Latn-GB", + "en-UK", + "en-150", + "eng-GB", + "en_GB@euro", + ], + ProofLang::EnGb, + ); + assert_tags(&["en-CA", "en_CA", "EN-ca", "en_CA.UTF-8", "en-Latn-CA"], ProofLang::EnCa); + for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + assert!(language.is_english() && language.is_proofed() && ProofLang::PROOFED.contains(&language), "{language:?}"); + assert_eq!(language.code(), "en"); + } + assert!(!ProofLang::Pl.is_english() && !ProofLang::Other.is_english()); + // The region is the first two-letter or three-digit subtag: a script or variant isn't one. + assert_eq!(ProofLang::from_tag(Some("en-Brai")), ProofLang::En); + assert_eq!(ProofLang::from_tag(Some("en-GB-oxendict")), ProofLang::EnGb); + } + + #[test] + fn tags_resolve_additional_locales_and_defaults() { + assert_tags(&["bg", "bg-BG", "BG_bg", "bg_BG.UTF-8", "bul"], ProofLang::Bg); + assert_tags(&["tr", "tr-TR", "tr-CY", "TR_tr", "tur"], ProofLang::Tr); + assert_tags(&["de", "de-DE", "de-CH", "DE_at", "deu", "ger"], ProofLang::De); + assert_tags(&["ru", "ru-RU", "RU-ru", "ru_RU", "ru_RU.UTF-8", " ru-RU ", "rus"], ProofLang::Ru); + assert!(!ProofLang::De.is_proofed()); + assert!(!ProofLang::Bg.is_proofed()); + assert_eq!(ProofLang::Bg.code(), "bg"); + assert!(ProofLang::Pt.is_proofed() && !ProofLang::Other.is_proofed()); + assert!(ProofLang::Tr.is_proofed()); + assert!(ProofLang::Ru.is_proofed()); + assert_eq!(ProofLang::Ru.code(), "ru"); + assert!(ProofLang::PROOFED.contains(&ProofLang::Ru)); + assert_eq!(ProofLang::Pl.code(), "pl"); + assert_eq!(ProofLang::Pt.code(), "pt"); + assert!(ProofLang::It.is_proofed()); + assert_eq!(ProofLang::It.code(), "it"); + assert!(ProofLang::Hr.is_proofed()); + assert_eq!(ProofLang::Hr.code(), "hr"); + assert_eq!(ProofLang::Tr.code(), "tr"); + assert!(ProofLang::PROOFED.contains(&ProofLang::Pt), "every proofed language is listed"); + assert_eq!(ProofLang::from_tag(None), ProofLang::En); + assert_eq!(ProofLang::from_tag(Some("")), ProofLang::En); + assert_eq!(ProofLang::from_tag(Some(" ")), ProofLang::En); + assert!(ProofLang::Pl.is_proofed() && !ProofLang::Other.is_proofed()); + } + + #[test] + fn tags_reject_unknown_or_partial_language_codes() { + assert_tags(&["fr-FR", "dex", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "esx", "spax", "e", "-"], ProofLang::Other); + } + + /// #3360, #3553: Spanish is proofed, by region: Spain (and no region), Mexico, Uruguay, and + /// Latin America with the other countries of the Americas. + #[test] + fn spanish_tags_resolve_by_region() { + for t in ["es", "es-ES", "ES_es", "es_ES.UTF-8", " es-ES ", "spa", "es-Latn-ES", "es-GQ", "es-PH", "es-x-mx", "es_ES@euro"] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Es, "{t}"); + } + for t in ["es-MX", "es_MX", "ES-mx", "es_MX.UTF-8", "spa-MX", "es-Latn-MX"] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::EsMx, "{t}"); + } + for t in ["es-419", "es_419", "es-AR", "es-CO", "es_CL.UTF-8", "es-US", "ES-ve", "es-PR", "es-Latn-419", "es-005", "spa-PE"] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Es419, "{t}"); + } + for t in ["es-UY", "es_UY", "ES-uy", "es_UY.UTF-8", "spa-UY", "es-Latn-UY"] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::EsUy, "{t}"); + } + for l in [ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy] { + assert!(l.is_spanish() && l.is_proofed() && ProofLang::PROOFED.contains(&l), "{l:?}"); + assert!(!l.is_english()); + assert_eq!(l.code(), "es"); + } + assert!(!ProofLang::En.is_spanish() && !ProofLang::Pt.is_spanish() && !ProofLang::Other.is_spanish()); + assert_eq!(normalize_tag("ES_mx").as_deref(), Some("es-MX")); + } +} diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs new file mode 100644 index 00000000..d1aa3f46 --- /dev/null +++ b/crates/proof/src/lib.rs @@ -0,0 +1,2245 @@ +//! Proofing for the Craft Apps (moved from WordCraft's `wordcraft-proof`), per language. A run's language (its BCP 47 tag) picks the tools through +//! [`ProofLang`]; the `*_in` functions take it, the older English-only ones remain. +//! +//! - **English** ([`ProofLang::En`]; also text with no language): spelling against the +//! public-domain Moby word list (~160k words, the same list that drives hyphenation) plus our +//! supplement of modern words it lacks (craft-i18n's `language-data/en-us/en-extra.txt`: `desktop`, `email`, +//! `download`…), with inflections (`-s`, `-es`, `-ed`, `-ing`, `-ly`, `'s`…), a user dictionary and ignore lists; +//! suggestions by edit distance. Grammar checks that are cheap and reliable: repeated words, +//! `a`/`an`, capital letter after a sentence end, spaces before punctuation, doubled spaces. +//! - **Polish** ([`ProofLang::Pl`], module [`pl`]): spelling against the Hunspell `pl_PL` +//! dictionary installed on the system ([`hunspell`]; not bundled, so without it Polish has no +//! spelling marks), read by the affix engine ([`affix`]), suggestions tuned for Polish +//! (diacritics, `ż`/`rz`, `u`/`ó`, missing spaces), and Polish grammar rules with Polish +//! messages. +//! - **Bulgarian** ([`ProofLang::Bg`], module [`bg`]): hyphenation by the Bulgarian syllable +//! rules; no spelling or grammar marks yet. +//! - **Portuguese** ([`ProofLang::Pt`], module [`pt`]): spelling against the Hunspell `pt_BR` +//! dictionary installed on the system (VERO and friends; not bundled), suggestions tuned for +//! Portuguese (missing accents, `seje`/`seja` confusions, missing spaces), conservative +//! grammar rules with Portuguese messages, and `hyph-utf8`'s Portuguese patterns. +//! - **Italian** ([`ProofLang::It`], module [`it`]): spelling against the Hunspell `it_IT` +//! dictionary installed on the system (not bundled — the upstream file is GPL-licensed), +//! suggestions tuned for Italian (missing accents, `REP` confusions, missing spaces), +//! conservative grammar rules with Italian messages, and `hyph-utf8`'s Italian patterns +//! (Beccari, MIT option). +//! - **Russian** ([`ProofLang::Ru`], module [`ru`]): spelling against the Hunspell `ru_RU` +//! dictionary installed on the system (not bundled either), with `ё` optional and hyphenated +//! words (`кто-то`, `по-русски`, `северо-западный`) understood, suggestions tuned for Russian +//! (`е`/`ё`, `жи`/`ши`, `тся`/`ться`, voiced and voiceless pairs, missing spaces), Russian +//! grammar rules with Russian messages, and Russian word division. +//! - **Czech** ([`ProofLang::Cs`], module [`cs`]): the installed Hunspell `cs_CZ` dictionary +//! (not bundled), dictionary-driven suggestions and generic checks with Czech messages. +//! Hyphenation uses the existing English fallback; no Czech patterns are bundled. +//! - **Indonesian** ([`ProofLang::Id`], module [`id`]): the installed Hunspell `id_ID` +//! dictionary (not bundled), dictionary-driven suggestions and generic checks with Indonesian +//! messages. Hyphenation uses the existing English fallback. +//! - **Spanish** ([`ProofLang::Es`], [`ProofLang::EsMx`], [`ProofLang::Es419`], [`ProofLang::EsUy`], +//! module [`es`]): each variety's installed Hunspell dictionary (`es_ES`, `es_MX`, `es_419`, +//! `es_UY`, with fallbacks; not bundled), dictionary-driven suggestions, Uruguayan `vos` forms, +//! and generic checks plus the opening `¿`/`¡` with Spanish messages. Hyphenation uses the +//! existing English fallback. +//! - **German** ([`ProofLang::De`]): hyphenation by `hyph-utf8`'s German patterns (reformed +//! spelling, MIT); no spelling or grammar marks yet. +//! - **Other languages** ([`ProofLang::Other`]): no spelling or grammar marks, like Word without +//! the language's proofing tools. +//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese, +//! Italian, German), Bulgarian syllable rules, Russian word-division rules, heuristic. +//! +//! [`check_text`] checks a paragraph whose runs are in different languages. +//! +//! **Language data** (the Moby word list, the pattern files, our word supplements) is embedded at +//! build time from craft-i18n's `language-data/` (`CRAFT_I18N_DIR`, see `build.rs`). English +//! spelling never goes unchecked: without the embedded word list it uses an English Hunspell +//! dictionary installed on the system, and without one the whole Moby word list built into the +//! crate (see [`english_source`]). Hyphenation that needs the data is off without it; grammar and the +//! rule-based hyphenation (Bulgarian, Turkish, Russian) are code and still work. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +pub mod affix; +mod affix_suggest; +/// The Language data embedded from craft-i18n by `build.rs` (each constant empty when it isn't). +mod data { + include!(concat!(env!("OUT_DIR"), "/language_data.rs")); +} +pub mod bg; +pub mod cs; +pub mod dict; +pub mod en_variants; +mod english; +pub mod es; +pub mod hr; +pub mod hunspell; +pub mod hyphen; +pub mod id; +pub mod it; +pub mod lang; +pub mod patterns; +pub mod pl; +pub mod pt; +pub mod ru; +mod slot; +mod spelling; +pub mod tr; +pub mod vi; + +use std::collections::HashSet; +use std::ops::Range; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{OnceLock, RwLock}; + +/// What the spelling check skips (File › Options › Proofing). Word's defaults: all on. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Options { + /// Words in capitals (`NASA`, `WORDCRAFT`) are taken as correct. + pub ignore_uppercase: bool, + /// Words containing digits (`B2B`, `mp3`) are taken as correct. + pub ignore_numbers: bool, + /// Web and email addresses and file paths (`https://…`, `a@b.org`, `www.…`) are skipped. + pub ignore_internet: bool, +} + +impl Default for Options { + fn default() -> Self { + Options { ignore_uppercase: true, ignore_numbers: true, ignore_internet: true } + } +} + +pub use english::EnglishSource; +pub use lang::ProofLang; + +/// A problem found in text: byte range, kind, message and suggestions. +#[derive(Clone, Debug, PartialEq)] +pub struct Issue { + pub start: usize, + pub end: usize, + pub kind: IssueKind, + /// In the language of the text it was found in (Polish rules explain in Polish). + pub message: String, + pub suggestions: Vec, + /// The language the text was checked in: spelling suggestions come from its dictionary + /// ([`suggest_in`]). + pub lang: ProofLang, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum IssueKind { + Spelling, + Grammar, +} + +fn user_words() -> &'static RwLock> { + static U: OnceLock>> = OnceLock::new(); + U.get_or_init(|| RwLock::new(HashSet::new())) +} + +/// Bumped whenever the user dictionary changes, so cached proofing results can be dropped. +static DICTIONARY_GENERATION: AtomicU64 = AtomicU64::new(0); + +/// The key a word has in the user dictionary and the Ignore All list: lower case and composed +/// (NFC), as spelling looks words up ([`composed`]). A word added, ignored, removed or loaded +/// decomposed (`zqprue\u{301}ba`) is then the same entry as its composed form (`zqpruéba`), and +/// the other way round (#3562 review). +fn dictionary_key(w: &str) -> String { + composed(&w.to_lowercase()).into_owned() +} + +/// Add a word to the user dictionary ("Add to Dictionary"). It applies to every language. +pub fn add_word(w: &str) { + if user_words().write().unwrap_or_else(std::sync::PoisonError::into_inner).insert(dictionary_key(w)) { + DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed); + } +} + +/// Take a word out of the user dictionary (File › Options › Proofing). Returns whether it was there. +pub fn remove_word(w: &str) -> bool { + let removed = user_words().write().unwrap_or_else(std::sync::PoisonError::into_inner).remove(&dictionary_key(w)); + if removed { + DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed); + } + removed +} + +fn ignored_words() -> &'static RwLock> { + static I: OnceLock>> = OnceLock::new(); + I.get_or_init(|| RwLock::new(HashSet::new())) +} + +/// Ignore a word everywhere until the app closes ("Ignore All"). Unlike [`add_word`] it isn't +/// part of the user dictionary: it is never listed, saved or loaded, as in Word. +pub fn ignore_word(w: &str) { + if ignored_words().write().unwrap_or_else(std::sync::PoisonError::into_inner).insert(dictionary_key(w)) { + DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed); + } +} + +/// A language proofed against an installed Hunspell dictionary: all the functions below need to +/// know about it, so a language joins with one row in [`LANGUAGES`]. +struct Language { + lang: ProofLang, + slot: &'static dyn slot::Slot, + /// The spelling mark's message, in the language. + message: &'static str, + is_correct: fn(&str, ProofLang, &Options) -> bool, + suggest: fn(&str, usize, ProofLang) -> Vec, + /// Grammar issues; the flag says whether the first word must have a capital (not in a list + /// item). + grammar: fn(&str, ProofLang, bool) -> Vec, +} + +/// Every language proofed against a Hunspell dictionary, in the order File › Options › Proofing +/// lists them. English has a built-in word list and is handled apart. +static LANGUAGES: [Language; 12] = [ + Language { + lang: ProofLang::Pl, + slot: &pl::DICT, + message: "Możliwy błąd pisowni", + is_correct: |w, _, o| pl::is_correct_with_options(w, o), + suggest: |w, n, _| pl::suggest(w, n), + grammar: |t, _, c| pl::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Pt, + slot: &pt::DICT, + message: "Possível erro de ortografia", + is_correct: |w, _, o| pt::is_correct_with_options(w, o), + suggest: |w, n, _| pt::suggest(w, n), + grammar: |t, _, c| pt::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::It, + slot: &it::DICT, + message: "Possibile errore di ortografia", + is_correct: |w, _, o| it::is_correct_with_options(w, o), + suggest: |w, n, _| it::suggest(w, n), + grammar: |t, _, c| it::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Hr, + slot: &hr::DICT, + message: "Moguća pravopisna pogreška", + is_correct: |w, _, o| hr::is_correct_with_options(w, o), + suggest: |w, n, _| hr::suggest(w, n), + grammar: |t, _, c| hr::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Tr, + slot: &tr::DICT, + message: "Olası yazım hatası", + is_correct: |w, _, o| tr::is_correct_with_options(w, o), + suggest: |w, n, _| tr::suggest(w, n), + grammar: |t, _, c| tr::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Ru, + slot: &ru::DICT, + message: "Возможна орфографическая ошибка", + is_correct: |w, _, o| ru::is_correct_with_options(w, o), + suggest: |w, n, _| ru::suggest(w, n), + grammar: |t, _, c| ru::grammar::check_with(t, c), + }, + // Czech, Indonesian and Spanish have no sentence-capital rule, so a list item needs no + // exception. + Language { + lang: ProofLang::Cs, + slot: &cs::DICT, + message: "Možná pravopisná chyba", + is_correct: |w, _, o| cs::is_correct_with_options(w, o), + suggest: |w, n, _| cs::suggest(w, n), + grammar: |t, _, _| cs::grammar::check(t), + }, + Language { + lang: ProofLang::Id, + slot: &id::DICT, + message: "Kemungkinan salah eja", + is_correct: |w, _, o| id::is_correct_with_options(w, o), + suggest: |w, n, _| id::suggest(w, n), + grammar: |t, _, _| id::grammar::check(t), + }, + Language { + lang: ProofLang::Es, + slot: &es::SPAIN_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, + Language { + lang: ProofLang::EsMx, + slot: &es::MEXICO_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, + Language { + lang: ProofLang::Es419, + slot: &es::LATIN_AMERICA_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, + Language { + lang: ProofLang::EsUy, + slot: &es::URUGUAY_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, +]; + +/// `lang`'s row in [`LANGUAGES`]; `None` for English and the languages WordCraft doesn't proof. +fn language(lang: ProofLang) -> Option<&'static Language> { + LANGUAGES.iter().find(|l| l.lang == lang) +} + +/// Search `dir` for Hunspell dictionaries before the system locations (File › Options › +/// Proofing › Dictionaries folder, #1063); `None` clears it. The dictionaries load again from +/// the new search path, and text is checked again. +pub fn set_dictionaries_folder(dir: Option) { + hunspell::set_dictionary_dir(dir); + for l in &LANGUAGES { + l.slot.reload(); + } + english::reload(); + DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed); +} + +/// Each language that reads an installed Hunspell dictionary, with the `.dic` file found for it +/// now (`None`: not installed, so the language has no spelling marks). +pub fn installed_dictionaries() -> Vec<(ProofLang, Option)> { + LANGUAGES.iter().map(|l| (l.lang, l.slot.installed_path())).collect() +} + +/// Why the dictionary found for `lang` failed to load (unsupported encoding, too large, a parse +/// error), if it was read and failed. Doesn't read anything itself: a dictionary nobody has +/// checked yet reports `None` (#2830). For an English, the embedded word list or (without it) the +/// English Hunspell dictionary that failed; English then uses the next source ([`english_source`]). +pub fn dictionary_load_error(lang: ProofLang) -> Option { + if matches!(lang, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) { + return english::load_error(lang); + } + hunspell::load_error(language(lang)?.slot.name()).map(|(_, reason)| reason) +} + +/// Where the spelling of an English `lang` comes from now (`None` for other languages): the word +/// list embedded at build time, else an English Hunspell dictionary from the dictionaries folder +/// or the system (the run's English first), else the Moby word list built into the crate. +/// Each English uses only its own Hunspell dictionary (`en_US`, `en_GB`, `en_CA`): without it, +/// the built-in list, which has both spellings. Reads the Hunspell dictionary once when there is no +/// embedded word list (it is cached afterwards). +pub fn english_source(lang: ProofLang) -> Option { + lang.is_english().then(|| english::source(lang)) +} + +/// A number that changes whenever a word is added to or removed from the user dictionary, or the +/// dictionaries folder changes. +pub fn dictionary_generation() -> u64 { + DICTIONARY_GENERATION.load(Ordering::Relaxed) +} + +/// Is `lower` (a lower-case word) in the user dictionary or ignored? It is looked up composed, as +/// the entries are kept ([`dictionary_key`]). +pub(crate) fn in_user_dictionary(lower: &str) -> bool { + let key = composed(lower); + user_words().read().unwrap_or_else(std::sync::PoisonError::into_inner).contains(&*key) + || ignored_words().read().unwrap_or_else(std::sync::PoisonError::into_inner).contains(&*key) +} + +/// Words in the user dictionary. +pub fn user_dictionary() -> Vec { + let mut v: Vec = user_words().read().unwrap_or_else(std::sync::PoisonError::into_inner).iter().cloned().collect(); + v.sort(); + v +} + +/// Longest custom-dictionary word kept, in characters (mirrors `file.options` validation). +pub const MAX_CUSTOM_WORD_CHARS: usize = 64; +/// Most custom-dictionary words read from one file (caps a hostile file). +const MAX_CUSTOM_WORDS: usize = 100_000; +/// Largest custom-dictionary file read, in bytes (caps a hostile file). +const MAX_CUSTOM_FILE_BYTES: u64 = 1_000_000; + +/// `w` as a custom-dictionary word: trimmed and lower-cased, or `None` when it isn't a single +/// word of at most [`MAX_CUSTOM_WORD_CHARS`] characters (empty, spaces, control characters, or +/// bytes that weren't UTF-8). +/// Adding and loading both use it, so a word that can be added is never dropped on the next load. +pub fn custom_word(w: &str) -> Option { + let w = w.trim(); + if w.is_empty() + || w.chars().count() > MAX_CUSTOM_WORD_CHARS + || w.chars().any(|c| c.is_whitespace() || c.is_control() || c == char::REPLACEMENT_CHARACTER) + { + return None; + } + Some(dictionary_key(w)) +} + +/// The valid words of the custom-dictionary file at `path`, reading at most +/// [`MAX_CUSTOM_FILE_BYTES`] of it and keeping at most [`MAX_CUSTOM_WORDS`]. A missing or +/// unreadable file reads as no words. +fn read_dictionary_file(path: &std::path::Path) -> Vec { + use std::io::Read; + let mut bytes = Vec::new(); + let Ok(file) = std::fs::File::open(path) else { return Vec::new() }; + // `take` bounds the read itself, so a huge file is never read whole. + if file.take(MAX_CUSTOM_FILE_BYTES).read_to_end(&mut bytes).is_err() { + return Vec::new(); + } + // Lossy: a cut multi-byte tail becomes U+FFFD and is skipped by validation below, + // instead of discarding the whole file. + String::from_utf8_lossy(&bytes).lines().filter_map(custom_word).take(MAX_CUSTOM_WORDS).collect() +} + +/// Read custom-dictionary words from a file into the user dictionary (one word per line; +/// blank lines are skipped). A missing file reads as empty; a hostile file is capped, +/// never a panic and never an error that stops startup. Returns the words added. +pub fn load_user_dictionary(path: &std::path::Path) -> Vec { + let words = read_dictionary_file(path); + let mut set = user_words().write().unwrap_or_else(std::sync::PoisonError::into_inner); + let mut added = Vec::new(); + for w in words { + if set.len() >= MAX_CUSTOM_WORDS { + break; + } + if set.insert(w.clone()) { + added.push(w); + } + } + if !added.is_empty() { + DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed); + } + added +} + +/// Write the user dictionary to a file (one word per line, sorted), creating its folder. +/// +/// Words another process (a second window, the MCP bridge) saved to the file since it was loaded +/// are merged in first, so neither loses the other's words; `removed` (words just taken out) stay +/// out. The file is replaced atomically: written beside it, then renamed over it. Errors (a +/// read-only folder, a full disk) go back to the caller; the in-memory words are kept. +pub fn save_user_dictionary(path: &std::path::Path, removed: &[String]) -> std::io::Result<()> { + let removed: HashSet = removed.iter().map(|w| dictionary_key(w)).collect(); + let on_disk = read_dictionary_file(path); + { + let mut set = user_words().write().unwrap_or_else(std::sync::PoisonError::into_inner); + let mut changed = false; + for w in on_disk { + if set.len() >= MAX_CUSTOM_WORDS { + break; + } + if !removed.contains(&w) && set.insert(w) { + changed = true; + } + } + for w in &removed { + changed |= set.remove(w); + } + if changed { + DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed); + } + } + let mut out = String::new(); + for w in user_dictionary() { + out.push_str(&w); + out.push('\n'); + } + if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) { + std::fs::create_dir_all(dir)?; + } + let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_else(|| "custom-dictionary.txt".into()); + let tmp = path.with_file_name(format!(".{name}.{}.tmp", std::process::id())); + let written = std::fs::write(&tmp, out).and_then(|()| std::fs::rename(&tmp, path)); + if written.is_err() { + let _ = std::fs::remove_file(&tmp); + } + written +} + +/// Common words the source list lacks (modern vocabulary, contractions). +const COMMON: &str = "a an the and or but if of to in on at by for with from as is am are was were be been being it its we us our you your he him his she her they them their i me my mine this that these those there here what which who whom whose when where why how all any both each few more most other some such no nor not only own same so than too very can will just should now do does did done has have had having may might must shall would could up down out over under again further then once off into onto upon about above below between through during before after until while because per via vs etc ok yes yeah oh ah hi hey bye one two three four five six seven eight nine ten first last next new old get got go goes went gone see saw seen say said make made take took come came give gave know knew think thought look want need use used find found tell told ask asked feel felt try left keep let put mean set run show turn move live play pay hear hold bring begin seem help talk start might"; + +/// Is `w` (lower case) one of [`COMMON`]'s words? +pub(crate) fn is_common_word(w: &str) -> bool { + COMMON.split(' ').any(|c| c == w) +} + +/// Our supplement of common words the Moby list lacks (`desktop`, `email`, `download`…): one +/// lowercase base form per line, `#` comments. Inflections come from [`known_core`]. +use data::EXTRA_EN; + +/// The supplement's words: the Language data's copy, else the crate's own ([`english::EXTRA`]). +fn extra_words() -> &'static HashSet<&'static str> { + static E: OnceLock> = OnceLock::new(); + let extra = if EXTRA_EN.is_empty() { english::EXTRA } else { EXTRA_EN }; + E.get_or_init(|| extra.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')).collect()) +} + +/// A base form in the dictionary or the supplement. +fn base_known(d: &dict::Dictionary, w: &str) -> bool { + d.get(w).is_some() || extra_words().contains(w) +} + +fn known_core(w: &str) -> bool { + let d = dict::Dictionary::en_us(); + known_with(w, &|s| base_known(d, s)) +} + +/// [`known_core`] against the word list built into the crate (no embedded word list). +pub(crate) fn known_in_word_list(w: &str) -> bool { + known_with(w, &|s| english::word_list_contains(s) || extra_words().contains(s)) +} + +/// Is `w` (lower case) a word, by `base_known` (a base form in the word list), the common words, +/// the user dictionary and the inflection and prefix rules? +fn known_with(w: &str, base_known: &dyn Fn(&str) -> bool) -> bool { + if base_known(w) || COMMON.split(' ').any(|c| c == w) { + return true; + } + if in_user_dictionary(w) { + return true; + } + // Inflections. + let stems: [(&str, &[&str]); 13] = [ + ("'s", &[""]), + ("s", &[""]), + ("es", &["", "e"]), + ("ies", &["y"]), + ("ed", &["", "e"]), + ("ied", &["y"]), + ("ing", &["", "e"]), + ("ly", &["", "le"]), + ("ily", &["y"]), + // specifically → specific (#1694): only -ic adjectives, so "desktopally" stays wrong. + ("ically", &["ic"]), + ("er", &["", "e"]), + ("est", &["", "e"]), + ("ness", &[""]), + ]; + for (suf, adds) in stems { + if let Some(stem) = w.strip_suffix(suf) + && stem.len() >= 2 + { + for a in adds { + let s = format!("{stem}{a}"); + if base_known(&s) { + return true; + } + } + // Doubled consonant: running → run, stopped → stop. + let b = stem.as_bytes(); + if (suf == "ing" || suf == "ed" || suf == "er" || suf == "est") + && b.len() >= 3 + && b.last() == b.get(b.len() - 2) + && let Some(s) = stem.get(..stem.len() - 1) + && base_known(s) + { + return true; + } + } + } + // Prefixes. + for pre in ["un", "re", "non", "pre", "over", "under", "co", "mis"] { + if let Some(rest) = w.strip_prefix(pre) + && rest.len() >= 3 + && base_known(rest) + { + return true; + } + } + false +} + +/// Is there a proofing dictionary for this BCP 47 language tag? `None` (and empty tags) mean the +/// document default, which is en-US. Runs in a language without one are skipped (#100) instead +/// of flagging every word against another language's dictionary; see [`ProofLang::from_tag`]. +pub fn has_dictionary(lang: Option<&str>) -> bool { + ProofLang::from_tag(lang).is_proofed() +} + +/// The Hunspell dictionary name (`pl_PL`, `it_IT`) a proofing language needs but doesn't have +/// installed, for a status-bar hint (#1167). `None` when the language's dictionary is available, +/// or when it needs none (English has a built-in word list; unproofed languages aren't offered +/// one). Reads the dictionary once (it is cached afterwards). +pub fn missing_dictionary(lang: Option<&str>) -> Option<&'static str> { + let lang = ProofLang::from_tag(lang); + dictionary_needed(lang, dictionary_loaded(lang)?) +} + +/// The Hunspell name, `.dic` file and reason of a dictionary that is installed for this language +/// but couldn't be read (unsupported `SET` encoding, too large, a parse error). `None` when it +/// loaded, or when no file exists (then [`missing_dictionary`] names it). Reads the dictionary +/// once (it is cached afterwards). +pub fn unreadable_dictionary(lang: Option<&str>) -> Option<(&'static str, std::path::PathBuf, String)> { + let lang = ProofLang::from_tag(lang); + let name = dictionary_needed(lang, dictionary_loaded(lang)?)?; + hunspell::load_error(name).map(|(file, reason)| (name, file, reason)) +} + +/// Did the Hunspell dictionary of `lang` load? `None` for languages that use none (English). +fn dictionary_loaded(lang: ProofLang) -> Option { + language(lang).map(|l| l.slot.has_dictionary()) +} + +/// The Hunspell name `lang` needs when `installed` is false (the first name its module looks for; +/// a Spanish variety's is its own: `es_ES`, `es_MX`, `es_419`, `es_UY`). +fn dictionary_needed(lang: ProofLang, installed: bool) -> Option<&'static str> { + language(lang).filter(|_| !installed).map(|l| l.slot.name()) +} + +/// Is `word` spelled correctly in the run's proofing language? A language without a dictionary +/// can't be checked, so every word counts as correct (no red underlines for it). +pub fn is_correct_for(word: &str, lang: Option<&str>) -> bool { + is_correct_in(word, ProofLang::from_tag(lang)) +} + +/// Is `word` spelled correctly? Numbers, single letters, ALL-CAPS acronyms, URLs and words with +/// digits are accepted. +pub fn is_correct(word: &str) -> bool { + is_correct_with(word, &Options::default()) +} + +/// The words every language's check accepts under `opts`: single letters, words without +/// letters, words with digits (Ignore words that contain numbers), words in capitals +/// (Ignore words in UPPERCASE) and words in the user dictionary. +pub(crate) fn accepted_by_options(word: &str, opts: &Options) -> bool { + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'').trim_matches('\''); + w.chars().count() <= 1 + || !w.chars().any(char::is_alphabetic) + || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) + || (opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase())) + // The whole word: the languages split a hyphenated word into its parts, and an added + // `Unigranrio-RJ` is none of them (#2082). + || in_user_dictionary(&w.to_lowercase()) +} + +/// Is the English word list embedded? Without it English is checked against an installed +/// Hunspell dictionary or the word list built into the crate ([`english`]). +pub(crate) fn english_data() -> bool { + !dict::Dictionary::en_us().is_empty() +} + +/// [`is_correct`] with the File › Options › Proofing choices. +pub fn is_correct_with(word: &str, opts: &Options) -> bool { + if !english_data() { + return english::is_correct(word, ProofLang::En, opts); + } + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\''); + let w = w.trim_matches('\''); + if accepted_by_options(w, opts) { + return true; + } + if !w.is_ascii() && !w.chars().any(|c| c.is_ascii_alphabetic()) { + return true; // other scripts: no dictionary + } + let lower = w.to_lowercase().replace('’', "'"); + if known_core(&lower) { + return true; + } + // Hyphenated compounds: each part. + if lower.contains('-') { + return lower.split('-').all(|p| p.is_empty() || known_core(p)); + } + false +} + +/// Spelling suggestions for a misspelled word (best first, at most `max`). +pub fn suggest(word: &str, max: usize) -> Vec { + if !english_data() { + return english::suggest(word, max, ProofLang::En); + } + suggest_from(word, max, dict::Dictionary::en_us().iter().map(|(w, _)| w)) +} + +/// [`suggest`] from `words` (lower case) and our supplement. +pub(crate) fn suggest_from<'a>(word: &str, max: usize, words: impl Iterator) -> Vec { + let lower = word.to_lowercase(); + let n = lower.chars().count(); + if n == 0 || n > 40 { + return Vec::new(); + } + let mut scored: Vec<(usize, i64, String)> = Vec::new(); + let target: Vec = lower.chars().collect(); + for w in words.chain(extra_words().iter().copied()) { + let wn = w.chars().count(); + if wn + 2 < n || wn > n + 2 { + continue; + } + let dist = damerau(&target, w); + if dist <= 2 { + let same_first = w.chars().next() == target.first().copied(); + scored.push((dist, if same_first { 0 } else { 1 }, w.to_string())); + } + } + scored.sort_by(|a, b| (a.0, a.1, a.2.len().abs_diff(n)).cmp(&(b.0, b.1, b.2.len().abs_diff(n))).then(a.2.cmp(&b.2))); + scored.into_iter().take(max).map(|(_, _, w)| match_case(word, w)).collect() +} + +/// `w` (lower case) in the case of `word`: all capitals, an initial capital, or as it is. +fn match_case(word: &str, w: String) -> String { + let cap = word.chars().next().is_some_and(char::is_uppercase); + let upper = word.chars().all(|c| !c.is_lowercase()); + if upper && word.chars().count() > 1 { + w.to_uppercase() + } else if cap { + let mut c = w.chars(); + c.next().map(|f| f.to_uppercase().chain(c).collect()).unwrap_or(w) + } else { + w + } +} + +/// `lower` in the spelling `lang` (an English) wants, when it or a part of a hyphenated compound +/// is the other dialect's spelling (`colour-blind` → `color-blind` in American English). +fn english_preferred(lower: &str, lang: ProofLang) -> Option { + let mut changed = false; + let parts: Vec = lower + .split('-') + .map(|p| match en_variants::preferred(p, lang) { + Some(n) => { + changed = true; + n + } + None => p.to_string(), + }) + .collect(); + changed.then(|| parts.join("-")) +} + +/// Is `word` spelled right in the English `lang` ([`ProofLang::En`], [`ProofLang::EnGb`], +/// [`ProofLang::EnCa`])? The shared word list's words, less the other dialect's spellings +/// (`colour` in American English, `color` in British), plus this dialect's spellings of the +/// list's words ([`en_variants`]). The user dictionary overrules both. +fn is_correct_english(word: &str, lang: ProofLang, opts: &Options) -> bool { + if !english_data() { + return english::is_correct(word, lang, opts); + } + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'').trim_matches('\''); + if accepted_by_options(w, opts) { + return true; + } + let lower = w.to_lowercase().replace('’', "'"); + if in_user_dictionary(&lower) { + return true; + } + let parts: Vec<&str> = lower.split('-').filter(|p| !p.is_empty()).collect(); + if parts.iter().any(|p| en_variants::is_foreign(p, lang)) { + return false; + } + is_correct_with(word, opts) || (!parts.is_empty() && parts.iter().all(|p| known_core(p) || en_variants::known_variant(p, lang, known_core))) +} + +/// Spelling suggestions in the English `lang`: this dialect's spelling first when `word` is the +/// other dialect's, then the shared suggestions in this dialect's spelling. +fn suggest_english(word: &str, max: usize, lang: ProofLang) -> Vec { + let lower = word.to_lowercase().replace('’', "'"); + let mut out: Vec = Vec::new(); + if let Some(p) = english_preferred(&lower, lang) { + out.push(match_case(word, p)); + } + let shared = if english_data() { suggest(word, max.saturating_add(2)) } else { english::suggest(word, max.saturating_add(2), lang) }; + for s in shared { + let s = english_preferred(&s.to_lowercase(), lang).map_or(s, |p| match_case(word, p)); + if !out.contains(&s) { + out.push(s); + } + } + out.truncate(max); + out +} + +/// Optimal string alignment distance, early-exit at 3. +pub(crate) fn damerau(a: &[char], b: &str) -> usize { + let b: Vec = b.chars().collect(); + let (n, m) = (a.len(), b.len()); + let mut prev2 = vec![0usize; m + 1]; + let mut prev: Vec = (0..=m).collect(); + let mut cur = vec![0usize; m + 1]; + for i in 1..=n { + if let Some(c) = cur.get_mut(0) { + *c = i; + } + let mut row_min = i; + for j in 1..=m { + let (ai, bj) = (a.get(i - 1), b.get(j - 1)); + let cost = usize::from(ai != bj); + let mut v = (prev.get(j).copied().unwrap_or(9) + 1) + .min(cur.get(j - 1).copied().unwrap_or(9) + 1) + .min(prev.get(j - 1).copied().unwrap_or(9) + cost); + if i > 1 && j > 1 && a.get(i - 1) == b.get(j - 2) && a.get(i - 2) == b.get(j - 1) { + v = v.min(prev2.get(j - 2).copied().unwrap_or(9) + 1); + } + if let Some(c) = cur.get_mut(j) { + *c = v; + } + row_min = row_min.min(v); + } + if row_min > 2 { + return 3; + } + std::mem::swap(&mut prev2, &mut prev); + std::mem::swap(&mut prev, &mut cur); + } + prev.get(m).copied().unwrap_or(9) +} + +/// Words in `text`: (byte start, byte end). Combining marks belong to the word they follow, so a +/// word typed decomposed (`tambie\u{301}n`, NFD) is one word, as its precomposed form is. +pub fn words(text: &str) -> Vec<(usize, usize)> { + let mut out = Vec::new(); + let mut start: Option = None; + let is_w = |c: char| c.is_alphanumeric() || c == '\'' || c == '’' || c == '-' || unicode_normalization::char::is_combining_mark(c); + for (i, c) in text.char_indices() { + match (start, is_w(c)) { + (None, true) => start = Some(i), + (Some(s), false) => { + out.push((s, i)); + start = None; + } + _ => {} + } + } + if let Some(s) = start { + out.push((s, text.len())); + } + // Trim surrounding apostrophes/hyphens. + out.into_iter() + .filter_map(|(a, b)| { + let w = text.get(a..b)?; + let lead = w.len() - w.trim_start_matches(['\'', '’', '-']).len(); + let trail = w.len() - w.trim_end_matches(['\'', '’', '-']).len(); + (a + lead < b - trail).then_some((a + lead, b - trail)) + }) + .collect() +} + +/// Spelling issues in a paragraph of English text (no suggestions; ask [`suggest`] on demand). +pub fn check_spelling(text: &str) -> Vec { + check_spelling_with(text, &Options::default()) +} + +/// [`check_spelling`] with the File › Options › Proofing choices. +pub fn check_spelling_with(text: &str, opts: &Options) -> Vec { + spelling_issues(text, |_| ProofLang::En, opts) +} + +/// Spelling issues in a paragraph of text in `lang` (none for a language without a dictionary; +/// no suggestions: ask [`suggest_in`] on demand). +pub fn check_spelling_in(text: &str, lang: ProofLang) -> Vec { + if !lang.is_proofed() { + return Vec::new(); + } + spelling_issues(text, |_| lang, &Options::default()) +} + +/// Spelling issues of every word of `text`, each word spelled in `lang(word_start)`. +/// +/// The web/e-mail classification of the whitespace-delimited token around a word is computed +/// once per token and shared by every word punctuation splits out of it. Classifying per word +/// rescanned the token each time, so a long whitespace-free run of punctuation-separated words +/// cost quadratic scanning (#1335). +fn spelling_issues(text: &str, lang: impl Fn(usize) -> ProofLang, opts: &Options) -> Vec { + let mut token = (0, 0, false); + words(text) + .into_iter() + .filter_map(|(a, b)| { + if a < token.0 || a >= token.1 { + token = enclosing_token(text, a, b); + } + spelling_issue(text, a, b, lang(a), opts, token.2) + }) + .collect() +} + +/// The whitespace-delimited token enclosing the word at `a..b` of `text`: its byte range, and +/// whether it looks like a web or e-mail address. +fn enclosing_token(text: &str, a: usize, b: usize) -> (usize, usize, bool) { + // The token starts after the last white-space character, whatever its width (a no-break + // space is two bytes and an ideographic space is three). + let ts = text.get(..a).and_then(|t| t.char_indices().rev().find(|(_, c)| c.is_whitespace())).map(|(i, c)| i + c.len_utf8()).unwrap_or(0); + let te = text.get(b..).and_then(|t| t.find(char::is_whitespace)).map(|i| b + i).unwrap_or(text.len()); + (ts, te, text.get(ts..te).is_some_and(is_address)) +} + +/// Does a whitespace-delimited token look like a web or e-mail address (`https://…`, `www.…`, +/// `a@b.org`)? Spelling skips its words, and Spanish grammar its punctuation. +pub(crate) fn is_address(token: &str) -> bool { + token.contains("://") || token.contains('@') || token.starts_with("www.") +} + +/// The spelling issue of the word at `a..b` of `text`, if it is misspelled in `lang`. Words in +/// web and e-mail addresses are skipped (unless `opts` says otherwise); `internet` is the +/// classification of the word's enclosing token, from [`enclosing_token`]. +fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Options, internet: bool) -> Option { + let w = text.get(a..b)?; + // A word with a Vietnamese-only letter (a name like Nguyễn in English text) isn't English. + if (opts.ignore_internet && internet) || (lang.is_english() && vi::is_vietnamese_word(w)) || is_correct_in_with(w, lang, opts) { + return None; + } + let message = language(lang).map_or("Possible spelling mistake", |l| l.message); + Some(Issue { start: a, end: b, kind: IssueKind::Spelling, message: message.into(), suggestions: Vec::new(), lang }) +} + +/// Is `word` spelled correctly in `lang`? Always true in a language without a dictionary. +pub fn is_correct_in(word: &str, lang: ProofLang) -> bool { + is_correct_in_with(word, lang, &Options::default()) +} + +/// `word` composed (NFC), as the dictionaries spell their words: `tambie\u{301}n` is looked up as +/// `también`. Words already composed (all ASCII ones among them) are borrowed as they are. +fn composed(word: &str) -> std::borrow::Cow<'_, str> { + use unicode_normalization::{IsNormalized, UnicodeNormalization, is_nfc_quick}; + if word.is_ascii() || is_nfc_quick(word.chars()) == IsNormalized::Yes { + std::borrow::Cow::Borrowed(word) + } else { + std::borrow::Cow::Owned(word.nfc().collect()) + } +} + +/// [`is_correct_in`] with the File › Options › Proofing choices. A word with combining marks is +/// checked as its composed form ([`composed`]). +pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { + let word = &*composed(word); + if lang.is_english() { + return is_correct_english(word, lang, opts); + } + // A language WordCraft doesn't proof (Bulgarian, German, any other) has no spelling marks. + language(lang).is_none_or(|l| accepted_by_options(word, opts) || (l.is_correct)(word, lang, opts)) +} + +/// Spelling suggestions for a word misspelled in `lang` (best first, at most `max`), composed +/// ([`composed`]) like the dictionaries' words. +pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec { + let word = &*composed(word); + if lang.is_english() { + return suggest_english(word, max, lang); + } + language(lang).map_or_else(Vec::new, |l| (l.suggest)(word, max, lang)) +} + +/// Grammar issues in a paragraph of text in `lang`: the English rules, the Polish ones +/// ([`pl::grammar`]), the Portuguese ones ([`pt::grammar`]), the Italian ones ([`it::grammar`]), +/// the Turkish ones ([`tr::grammar`]), Czech and Indonesian generic checks ([`cs::grammar`], [`id::grammar`]), the Spanish ones +/// ([`es::grammar`]), or none for a language WordCraft doesn't proof. +pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec { + grammar_in(text, lang, true) +} + +/// [`check_grammar_in`] where `capital_first` says whether the first word must be capitalized +/// (false for a list item). +fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec { + match lang { + ProofLang::En => grammar_en(text, capital_first), + ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(), + _ => language(lang).map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first)), + } +} + +/// Spelling and grammar issues in a paragraph whose text is in several languages, in order. +/// `spans` gives the language of byte ranges of `text` (in order, not overlapping); text outside +/// them is English. Each word is spelled in the language of its first letter. Each language's +/// grammar rules read the whole paragraph (so sentence boundaries are seen) but report only +/// inside that language's spans: an English quotation in a Polish paragraph gets English +/// spelling and is never told a Polish rule. +pub fn check_text(text: &str, spans: &[(Range, ProofLang)]) -> Vec { + check_text_with(text, spans, &Options::default(), true, false) +} + +/// [`check_text`] with the File › Options › Proofing choices: what spelling skips, and whether +/// grammar is checked at all. +/// +/// `list_item` marks a list paragraph: items are often fragments, so the English rule asking for +/// a capital on the paragraph's first word is off for it (later sentences are still checked). +pub fn check_text_with(text: &str, spans: &[(Range, ProofLang)], opts: &Options, grammar: bool, list_item: bool) -> Vec { + // Sorted, empty ones dropped, neighbours of one language merged. + let mut sorted: Vec<&(Range, ProofLang)> = spans.iter().filter(|(r, _)| r.start < r.end).collect(); + sorted.sort_by_key(|s| s.0.start); + let mut merged: Vec<(Range, ProofLang)> = Vec::new(); + for (r, l) in sorted { + match merged.last_mut() { + Some((m, ml)) if *ml == *l && m.end == r.start => m.end = r.end, + _ => merged.push((r.clone(), *l)), + } + } + // A paragraph typed in Vietnamese under the default English language is Vietnamese, which + // has no proofing tools: no marks (Word detects the language the same way). + let default = if vi::looks_vietnamese(text) { + for (_, l) in merged.iter_mut().filter(|(_, l)| l.is_english()) { + *l = ProofLang::Other; + } + ProofLang::Other + } else { + ProofLang::En + }; + let lang_at = |i: usize| -> ProofLang { + let k = merged.partition_point(|(r, _)| r.end <= i); + merged.get(k).filter(|(r, _)| r.start <= i).map_or(default, |s| s.1) + }; + let mut v: Vec = spelling_issues(text, lang_at, opts); + if !grammar { + v.sort_by_key(|i| (i.start, i.end)); + return v; + } + let mut langs: Vec = merged.iter().map(|s| s.1).collect(); + let mut covered = 0; + for (r, _) in &merged { + if r.start > covered { + break; + } + covered = covered.max(r.end); + } + if covered < text.len() || merged.is_empty() { + langs.push(default); + } + langs.sort(); + langs.dedup(); + for lang in langs { + let found = grammar_in(text, lang, !list_item); + v.extend(found.into_iter().filter(|i| lang_at(i.start) == lang)); + } + v.sort_by_key(|i| (i.start, i.end)); + v +} + +/// Grammar issues in a paragraph of English text. +pub fn check_grammar(text: &str) -> Vec { + grammar_en(text, true) +} + +/// The English rules; `capital_first` is whether the text's first word must start a sentence +/// with a capital (false for a list item, which may be a fragment). +fn grammar_en(text: &str, capital_first: bool) -> Vec { + let mut v = Vec::new(); + let ws = words(text); + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if gap.trim().is_empty() + && w0.eq_ignore_ascii_case(w1) + && w0.chars().all(char::is_alphabetic) + && !matches!(w0.to_lowercase().as_str(), "had" | "that" | "bye" | "so") + { + v.push(Issue { + start: *a0, + end: *b1, + kind: IssueKind::Grammar, + message: format!("Repeated word: \"{w1}\""), + suggestions: vec![w0.to_string()], + lang: ProofLang::En, + }); + } + if gap == " " && (w0 == "a" || w0 == "A") && w1.chars().next().is_some_and(|c| "aeiouAEIOU".contains(c)) && !starts_consonant_sound(w1) { + v.push(Issue { + start: *a0, + end: *b0, + kind: IssueKind::Grammar, + message: "Use \"an\" before a vowel sound".into(), + suggestions: vec![if w0 == "A" { "An" } else { "an" }.into()], + lang: ProofLang::En, + }); + } + if gap == " " + && (w0 == "an" || w0 == "An") + && w1.chars().next().is_some_and(|c| c.is_alphabetic() && !"aeiouAEIOUhH".contains(c)) + && !w1.chars().all(char::is_uppercase) + { + v.push(Issue { + start: *a0, + end: *b0, + kind: IssueKind::Grammar, + message: "Use \"a\" before a consonant sound".into(), + suggestions: vec![if w0 == "An" { "A" } else { "a" }.into()], + lang: ProofLang::En, + }); + } + } + // Space before punctuation; doubled spaces. + for (i, _) in text.match_indices(" ,").chain(text.match_indices(" .")).chain(text.match_indices(" ;")) { + if text.get(i + 2..).is_some_and(|r| r.starts_with('.') || r.starts_with(',')) { + continue; // "..." or numbers like " .5" + } + v.push(Issue { + start: i, + end: i + 2, + kind: IssueKind::Grammar, + message: "Remove the space before the punctuation".into(), + suggestions: vec![text.get(i + 1..i + 2).unwrap_or("").to_string()], + lang: ProofLang::En, + }); + } + for (i, _) in text.match_indices(" ") { + if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { + continue; + } + v.push(Issue { + start: i, + end: i + 2, + kind: IssueKind::Grammar, + message: "Extra space".into(), + suggestions: vec![" ".into()], + lang: ProofLang::En, + }); + } + // Sentence start capital. + let mut after_end = capital_first; + for (a, b) in &ws { + let Some(w) = text.get(*a..*b) else { continue }; + if after_end && w.chars().next().is_some_and(char::is_lowercase) && w.chars().all(char::is_alphabetic) { + let fixed: String = w.chars().next().map(|f| f.to_uppercase().chain(w.chars().skip(1)).collect()).unwrap_or_default(); + v.push(Issue { + start: *a, + end: *b, + kind: IssueKind::Grammar, + message: "Capitalize the first word of a sentence".into(), + suggestions: vec![fixed], + lang: ProofLang::En, + }); + } + let tail = text.get(*b..).unwrap_or(""); + let punctuation = tail.trim_start_matches(['"', '”', ')', '\'']); + // Two or more dots are an ellipsis, not a full stop ending a sentence. + // A Unicode ellipsis (U+2026) does not match sentence-ending punctuation either. + after_end = matches!(punctuation.chars().next(), Some('.' | '!' | '?')) + && !punctuation.starts_with("..") + && punctuation.trim_start_matches(['.', '!', '?', '"', '”', ')']).starts_with(' '); + } + v.sort_by_key(|i| i.start); + v +} + +fn starts_consonant_sound(w: &str) -> bool { + let l = w.to_lowercase(); + ["uni", "use", "usu", "eu", "one", "once", "ubiq", "uti", "ure", "uro"].iter().any(|p| l.starts_with(p)) +} + +/// Held by tests that change the dictionaries folder or rely on what it holds, so a folder change +/// can't land mid-test, and by those that save the whole user dictionary and load it back, which +/// would bring back a word another test just removed. +#[cfg(test)] +pub(crate) static DICTIONARY_TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// The no-Language-data build (`build.rs` sets `craft_proof_no_data` when nothing is embedded). +#[cfg(all(test, craft_proof_no_data))] +mod no_data; + +#[cfg(test)] +mod tests { + #[test] + fn missing_dictionary_is_named_only_for_hunspell_languages_without_one() { + assert_eq!(dictionary_needed(ProofLang::Pl, false), Some("pl_PL")); + assert_eq!(dictionary_needed(ProofLang::Pt, false), Some("pt_BR")); + assert_eq!(dictionary_needed(ProofLang::Pl, true), None); + assert_eq!(dictionary_needed(ProofLang::Pt, true), None); + // Every Hunspell language is named by the first name its module looks for (#3150). + for (l, name) in [ + (ProofLang::It, "it_IT"), + (ProofLang::Hr, "hr_HR"), + (ProofLang::Tr, "tr_TR"), + (ProofLang::Ru, "ru_RU"), + (ProofLang::Cs, "cs_CZ"), + (ProofLang::Id, "id_ID"), + // #3360: each Spanish variety names its own. + (ProofLang::Es, "es_ES"), + (ProofLang::EsMx, "es_MX"), + (ProofLang::Es419, "es_419"), + (ProofLang::EsUy, "es_UY"), + ] { + assert_eq!(dictionary_needed(l, false), Some(name), "{l:?}"); + assert_eq!(dictionary_needed(l, true), None, "{l:?}"); + } + // Every proofed language other than English has a Hunspell dictionary to report. + for l in ProofLang::PROOFED { + let english = matches!(l, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa); + assert_eq!(dictionary_needed(l, false).is_some(), !english, "{l:?}"); + } + for l in [ProofLang::En, ProofLang::Bg, ProofLang::De, ProofLang::Other] { + assert_eq!(dictionary_needed(l, false), None); + } + // Installed dictionaries and the built-in English list: no hint. + pl::test_dict::install(); + pt::test_dict::install(); + es::test_dict::install(); + for tag in [Some("pl-PL"), Some("pt-BR"), Some("es-ES"), Some("es-MX"), Some("es-419"), Some("es-UY"), Some("en-US"), Some("fr-FR"), None] { + assert_eq!(missing_dictionary(tag), None, "{tag:?}"); + } + } + + use super::*; + + /// The non-English test dictionaries every dictionaries-folder test puts in its folder, so + /// tests running meanwhile see the same words whichever of those folders is set. + fn write_test_dictionaries(dir: &std::path::Path) { + std::fs::create_dir_all(dir).unwrap(); + for (name, aff, dic) in [ + ("pl_PL", pl::test_dict::AFF, pl::test_dict::DIC), + ("pt_BR", pt::test_dict::AFF, pt::test_dict::DIC), + ("it_IT", it::test_dict::AFF, it::test_dict::DIC), + ("hr_HR", hr::test_dict::AFF, hr::test_dict::DIC), + ("tr_TR", tr::test_dict::AFF, tr::test_dict::DIC), + ("ru_RU", ru::test_dict::AFF, ru::test_dict::DIC), + ("cs_CZ", cs::test_dict::AFF, cs::test_dict::DIC), + ("id_ID", id::test_dict::AFF, id::test_dict::DIC), + ("es_ES", es::test_dict::AFF, es::test_dict::DIC), + ("es_MX", es::test_dict::AFF, es::test_dict::DIC_AMERICAS), + ("es_419", es::test_dict::AFF, es::test_dict::DIC_AMERICAS), + ("es_UY", es::test_dict::AFF, es::test_dict::DIC_AMERICAS), + ] { + std::fs::write(dir.join(format!("{name}.aff")), aff).unwrap(); + std::fs::write(dir.join(format!("{name}.dic")), dic).unwrap(); + } + } + + /// Set `dir` (with [`write_test_dictionaries`] and `english`, `(name, .aff, .dic)` files) as + /// the dictionaries folder; the English files are removed again by [`drop_english_files`]. + fn folder_with_english(dir: &std::path::Path, english: &[(&str, &[u8], &[u8])]) { + write_test_dictionaries(dir); + for (name, aff, dic) in english { + std::fs::write(dir.join(format!("{name}.aff")), aff).unwrap(); + std::fs::write(dir.join(format!("{name}.dic")), dic).unwrap(); + } + set_dictionaries_folder(Some(dir.to_path_buf())); + } + + /// Remove the English files of `dir` and set it again: English goes back to what it was + /// (the built-in list, or a dictionary installed on the system), for the tests that follow. + fn drop_english_files(dir: &std::path::Path, english: &[(&str, &[u8], &[u8])]) { + for (name, _, _) in english { + let _ = std::fs::remove_file(dir.join(format!("{name}.aff"))); + let _ = std::fs::remove_file(dir.join(format!("{name}.dic"))); + } + set_dictionaries_folder(Some(dir.to_path_buf())); + } + + /// R6.4, English never goes unchecked: with no embedded word list, an English Hunspell + /// dictionary (here in the dictionaries folder) is used before the word list built into the + /// crate; with one, the embedded list stays in use whatever is installed. + #[test] + fn english_hunspell_dictionary_is_used_before_the_built_in_list() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let dir = std::env::temp_dir().join(format!("wc-proof-en-hunspell-{}", std::process::id())); + // A tiny en_US. `hous` and `because` aren't words in it, so a marked `because` is this + // dictionary answering (the built-in list knows it). + let english: &[(&str, &[u8], &[u8])] = + &[("en_US", b"SET UTF-8\nTRY esiarntolcdugmphbyfvkwz\nSFX S Y 1\nSFX S 0 s .\n", b"3\nteh\nthe\nhouse/S\n")]; + folder_with_english(&dir, english); + if english_data() { + assert_eq!(english_source(ProofLang::En), Some(EnglishSource::Embedded)); + // (The Moby list has `teh`.) + assert!(!is_correct_in("becuase", ProofLang::En) && is_correct_in("because", ProofLang::En)); + assert_eq!(dictionary_load_error(ProofLang::En), None); + } else { + let dic = dir.join("en_US.dic"); + assert_eq!(english_source(ProofLang::En), Some(EnglishSource::Hunspell(dic.clone()))); + for w in ["teh", "Teh", "the", "house", "houses", "House"] { + assert!(is_correct_in(w, ProofLang::En), "{w}"); + } + for w in ["hous", "because", "housez"] { + assert!(!is_correct_in(w, ProofLang::En), "{w}: the dictionary's words, not the built-in list's"); + } + assert!(!is_correct("hous") && is_correct("teh"), "the older English-only functions too"); + assert_eq!(check_spelling_in("teh hous", ProofLang::En).len(), 1); + assert!(suggest_in("hous", 3, ProofLang::En).contains(&"house".to_string()), "{:?}", suggest_in("hous", 3, ProofLang::En)); + // No en_GB or en_CA file: never the en_US one, which would mark their spellings + // (craft-libs#482 re-review), but their own installed elsewhere or the built-in list. + for lang in [ProofLang::EnGb, ProofLang::EnCa] { + assert_ne!(english_source(lang), Some(EnglishSource::Hunspell(dic.clone())), "{lang:?}"); + if english_source(lang) == Some(EnglishSource::CoreList) { + for w in ["colour", "organise", "favourite", "centre", "travelled", "because"] { + assert!(is_correct_in(w, lang), "{w} {lang:?}"); + } + assert!(!is_correct_in("becuase", lang), "{lang:?}"); + } + } + } + // Text in a language without proofing is never checked with English. + assert!(check_spelling_in("hous teh zxqv", ProofLang::Other).is_empty()); + assert!(check_text("hous teh", &[(0..8, ProofLang::De)]).is_empty()); + assert_eq!(english_source(ProofLang::Pl), None); + drop_english_files(&dir, english); + if !english_data() { + assert_ne!(english_source(ProofLang::En), Some(EnglishSource::Hunspell(dir.join("en_US.dic")))); + } + } + + /// R6.4: an English Hunspell dictionary that is there but can't be read (an encoding we don't + /// read, bytes that aren't UTF-8, no words) is an error reported for its English, never a + /// panic, and that English falls back to the next dictionary or the built-in word list, which + /// still marks `becuase`. + #[test] + fn unreadable_english_dictionary_is_reported_and_skipped() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let dir = std::env::temp_dir().join(format!("wc-proof-en-unreadable-{}", std::process::id())); + let english: &[(&str, &[u8], &[u8])] = + &[("en_US", b"SET UTF-8\n", b"0\n"), ("en_GB", b"SET SHIFT_JIS\n", b"1\ncolour\n"), ("en_CA", b"SET UTF-8\n", b"1\n\xff\xfe\n")]; + folder_with_english(&dir, english); + for (lang, reason) in [(ProofLang::En, "no words"), (ProofLang::EnGb, "unsupported encoding"), (ProofLang::EnCa, "not UTF-8")] { + if english_data() { + assert_eq!(english_source(lang), Some(EnglishSource::Embedded)); + assert_eq!(dictionary_load_error(lang), None, "{lang:?}: never read"); + } else { + // The built-in list, or a dictionary installed on the system (tried next). + let source = english_source(lang); + assert!(!matches!(&source, Some(EnglishSource::Hunspell(p)) if p.starts_with(&dir)), "{lang:?}: {source:?}"); + let error = dictionary_load_error(lang).unwrap_or_default(); + assert!(error.contains(reason), "{lang:?}: {error}"); + } + assert!(!is_correct_in("becuase", lang) && is_correct_in("because", lang) && is_correct_in("houses", lang), "{lang:?}"); + assert!(suggest_in("becuase", 3, lang).contains(&"because".to_string()), "{lang:?}"); + } + drop_english_files(&dir, english); + assert_eq!(dictionary_load_error(ProofLang::EnGb), None, "a new folder forgets the old errors"); + } + + /// craft-libs#482 review: an English dictionary that can't be read is skipped for the next + /// name or folder of the same English, never for another English's; and a dictionary with + /// compounding and output conversion (as the usual en_US and en_GB ones have) is read. (Only + /// American English gets a dictionary that loads: the British tests don't hold the lock.) + #[test] + fn english_tries_its_next_dictionary_and_reads_real_world_directives() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let dir = std::env::temp_dir().join(format!("wc-proof-en-next-{}", std::process::id())); + let aff: &[u8] = b"SET UTF-8\nICONV 1\nICONV \xe2\x80\x99 '\nOCONV 1\nOCONV ' \xe2\x80\x99\nCOMPOUNDMIN 1\nONLYINCOMPOUND c\n\ + COMPOUNDRULE 1\nCOMPOUNDRULE n*1t\nSFX S Y 1\nSFX S 0 s .\n"; + let english: &[(&str, &[u8], &[u8])] = &[ + ("en_US", b"SET SHIFT_JIS\n", b"1\ncolor\n"), + ("en-US", aff, b"5\nhouse/S\nthe\nqwzzy\nbecause\n1st/c\n"), + ("en_GB", b"SET SHIFT_JIS\n", b"1\ncolour\n"), + ]; + folder_with_english(&dir, english); + if !english_data() { + let us = Some(EnglishSource::Hunspell(dir.join("en-US.dic"))); + // A real en_US installed elsewhere (WORDCRAFT_DICTIONARIES, the system) comes first: + // the same name in every folder before the next name. + if english_source(ProofLang::En) == us { + assert!(is_correct_in("qwzzy", ProofLang::En) && is_correct_in("houses", ProofLang::En)); + assert!(is_correct_in("because", ProofLang::En) && !is_correct_in("becuase", ProofLang::En) && !is_correct_in("hous", ProofLang::En)); + } + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + let source = english_source(lang); + assert!(source == us || !matches!(&source, Some(EnglishSource::Hunspell(p)) if p.starts_with(&dir)), "{lang:?}: {source:?}"); + } + assert_ne!(english_source(ProofLang::EnGb), us, "never another English's dictionary"); + for lang in [ProofLang::En, ProofLang::EnGb] { + let error = dictionary_load_error(lang).unwrap_or_default(); + assert!(error.contains("unsupported encoding"), "{lang:?}: the file it would rather use is reported: {error}"); + } + } + drop_english_files(&dir, english); + } + + /// #1063: the dictionaries folder is searched before the system locations; the dictionaries + /// in it are reported and used, and cached checks are dropped. The folder holds the same test + /// dictionaries the other tests install, so tests running meanwhile see the same words; it + /// stays set for the rest of the run for that reason. + #[test] + fn dictionaries_folder_is_searched_first() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let dir = std::env::temp_dir().join(format!("wc-proof-dictionaries-{}", std::process::id())); + write_test_dictionaries(&dir); + // #3252 review: force Russian and Czech to cache whatever they find before the folder is + // set (empty, with no folder and nothing installed on CI), so the folder change below has + // a stale cache to actually invalidate, not just an untouched `None`. + ru::dictionary(); + cs::dictionary(); + let before = dictionary_generation(); + set_dictionaries_folder(Some(dir.clone())); + assert!(dictionary_generation() > before, "cached checks are dropped"); + // Every language is listed, Russian and Czech too (#3226: they were once missing, so + // neither showed up in File › Options › Proofing). A language reports the folder's file, + // or none while a dictionary given with install_dictionary() is in use (#3252 review); + // the languages' own tests install theirs concurrently, so an exact path would race them. + // The folder is searched first whatever is in use. + let found = installed_dictionaries(); + let names = ["pl_PL", "pt_BR", "it_IT", "hr_HR", "tr_TR", "ru_RU", "cs_CZ", "id_ID", "es_ES", "es_MX", "es_419", "es_UY"]; + assert_eq!( + found.iter().map(|(lang, _)| *lang).collect::>(), + [ + ProofLang::Pl, + ProofLang::Pt, + ProofLang::It, + ProofLang::Hr, + ProofLang::Tr, + ProofLang::Ru, + ProofLang::Cs, + ProofLang::Id, + ProofLang::Es, + ProofLang::EsMx, + ProofLang::Es419, + ProofLang::EsUy + ] + ); + for ((lang, path), name) in found.iter().zip(names) { + let file = dir.join(format!("{name}.dic")); + assert!(path.is_none() || path.as_ref() == Some(&file), "{lang:?}: {path:?}"); + assert_eq!(hunspell::find(&[name]).map(|(_, dic)| dic), Some(file), "{lang:?}"); + } + assert!(is_correct_in("chrząszcz", ProofLang::Pl) && !is_correct_in("chrzonszcz", ProofLang::Pl)); + assert!(is_correct_in("tenés", ProofLang::EsUy) && !is_correct_in("prueva", ProofLang::EsUy)); + assert!(is_correct_in("дом", ProofLang::Ru) && !is_correct_in("дум", ProofLang::Ru)); + } + + /// A dictionary that fails to load says why in File › Options › Proofing in every language + /// that reads one; Russian and Czech were missing from the list that named their dictionaries + /// there. + #[test] + fn every_dictionary_language_reports_its_load_error() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + for (lang, _) in installed_dictionaries() { + let name = dictionary_needed(lang, false).unwrap(); + let reason = format!("zq unreadable {name}"); + let _ = hunspell::note_load::<()>(name, std::path::Path::new("zq.dic"), Err(reason.clone())); + assert_eq!(dictionary_load_error(lang), Some(reason), "{lang:?}"); + hunspell::clear_load_error(name); + } + } + + /// Every proofed language other than English has a row in the language table, and only those. + #[test] + fn the_language_table_covers_the_proofed_languages() { + for l in ProofLang::PROOFED { + assert_eq!(language(l).is_some(), !l.is_english(), "{l:?}"); + } + for (i, row) in LANGUAGES.iter().enumerate() { + assert!(row.lang.is_proofed(), "{:?}", row.lang); + assert!(LANGUAGES.iter().skip(i + 1).all(|other| other.lang != row.lang), "{:?} twice", row.lang); + } + } + + /// #1741: a list item may be a fragment, so its first word needs no capital in any language + /// with the rule; a plain paragraph's does, and later sentences in the item are still checked. + #[test] + fn list_item_first_word_needs_no_capital() { + let grammar = |t: &str, lang: ProofLang, list: bool| -> Vec { + check_text_with(t, &[(0..t.len(), lang)], &Options::default(), true, list) + .into_iter() + .filter(|i| i.kind == IssueKind::Grammar) + .filter_map(|i| t.get(i.start..i.end).map(str::to_string)) + .collect() + }; + for (lang, text) in [ + (ProofLang::En, "item one"), + (ProofLang::Pl, "jabłka i gruszki"), + (ProofLang::Hr, "jabuke i kruške"), + (ProofLang::It, "mele e pere"), + (ProofLang::Tr, "elma ve armut"), + (ProofLang::Pt, "maçãs e peras"), + (ProofLang::Ru, "яблоки и груши"), + ] { + let first = text.split(' ').next().unwrap(); + assert!(grammar(text, lang, true).is_empty(), "{lang:?} list item: {:?}", grammar(text, lang, true)); + assert_eq!(grammar(text, lang, false), [first], "{lang:?} plain paragraph"); + } + assert_eq!(grammar("hello world.", ProofLang::En, false), ["hello"]); + // Later sentences in a list item are still checked. + assert_eq!(grammar("item one. then two", ProofLang::En, true), ["then"]); + assert_eq!(grammar("elma ve armut. sonra kiraz", ProofLang::Tr, true), ["sonra"]); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn dictionary_loads() { + assert!(dict::Dictionary::en_us().len() > 100_000); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn spelling() { + for w in [ + "if", + "we", + "hello", + "world", + "running", + "studios", + "don't", + "Thursday", + "NASA", + "2026", + "co-operate", + "unhappy", + "colour", + "it's", + "stopped", + ] { + assert!(is_correct(w), "{w}"); + } + for w in ["helo", "wrold", "sentense", "recieve"] { + assert!(!is_correct(w), "{w}"); + } + let s = suggest("recieve", 5); + assert!(s.contains(&"receive".to_string()), "{s:?}"); + assert!(suggest("Wrold", 3).contains(&"World".to_string())); + } + + /// #1694: `-ically` adverbs come from their `-ic` adjective (specific → specifically). + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn ically_adverbs_are_known() { + for w in ["specifically", "scientifically", "automatically", "dramatically", "Specifically"] { + assert!(is_correct(w), "{w}"); + } + for w in ["specificaly", "specifcally", "zzzically"] { + assert!(!is_correct(w), "{w}"); + } + } + + /// #1694 review: only `-ic` adjectives take `-ally`; other known words (dictionary or + /// supplement) don't. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn non_ic_words_do_not_take_ally() { + for (base, wrong) in [("desktop", "desktopally"), ("house", "houseally"), ("final", "finalally"), ("green", "greenally")] { + assert!(is_correct(base), "{base}"); + assert!(!is_correct(wrong), "{wrong}"); + } + } + + /// #520: common modern words (and their inflections) are known; misspellings still flagged. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn common_modern_words() { + for w in [ + "desktop", + "Desktops", + "laptop", + "download", + "downloaded", + "uploading", + "email", + "emailing", + "website", + "online", + "offline", + "smartphones", + "login", + "username", + "spreadsheet", + "database", + "screenshots", + "blogging", + "inbox", + "workplace", + "uploader", + ] { + assert!(is_correct(w), "{w}"); + } + for w in ["dekstop", "laptp", "dowload", "spreadshet"] { + assert!(!is_correct(w), "{w}"); + } + assert!(suggest("dekstop", 5).contains(&"desktop".to_string()), "{:?}", suggest("dekstop", 5)); + assert!(check_spelling("My desktop and laptop download email online.").is_empty()); + } + + /// The supplement stays a sorted list of lowercase base forms. + /// #40: British spellings and their inflections are not flagged. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn british_spellings_are_known() { + for w in [ + "judgement", + "defence", + "licence", + "programme", + "catalogue", + "behaviour", + "colour", + "colourful", + "colourless", + "honour", + "honoured", + "honouring", + "honours", + "honourable", + "rigour", + "glamour", + "clamour", + "endeavour", + "paediatric", + "prioritise", + "prioritised", + "prioritising", + "privatise", + "privatised", + "synthesise", + "travelogue", + "manoeuvrable", + ] { + assert!(is_correct(w), "{w}"); + } + let text = "The programme honoured her rigour and prioritised colourful, paediatric care."; + for lang in [ProofLang::EnGb, ProofLang::EnCa] { + let flagged: Vec<&str> = check_spelling_in(text, lang).iter().map(|i| &text[i.start..i.end]).collect(); + let expect: &[&str] = if lang == ProofLang::EnCa { &["prioritised"] } else { &[] }; + assert_eq!(flagged, expect, "{lang:?}"); + } + // American English flags the British spellings (#1281). + let flagged: Vec<&str> = check_spelling(text).iter().map(|i| &text[i.start..i.end]).collect(); + assert_eq!(flagged, ["programme", "honoured", "rigour", "prioritised", "colourful", "paediatric"]); + } + + /// Each English accepts its own spellings and flags the other dialect's, suggesting its own + /// first, as Word does (#1281, #1045). + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn english_dialects_flag_each_others_spellings() { + let (us, gb, ca) = (ProofLang::En, ProofLang::EnGb, ProofLang::EnCa); + // (word, right in en-US, right in en-CA, right in en-GB) + let cases = [ + ("color", true, false, false), + ("colour", false, true, true), + ("colours", false, true, true), + ("coloured", false, true, true), + ("Colourful", false, true, true), + ("colour's", false, true, true), + ("honorable", true, false, false), + ("neighbourhood", false, true, true), + ("center", true, false, false), + ("centre", false, true, true), + ("centred", false, true, true), + ("centring", false, true, true), + ("centering", true, false, false), + ("theater", true, false, false), + ("theatre", true, true, true), + ("organize", true, true, true), + ("organise", false, false, true), + ("organisation", false, false, true), + ("organisations", false, false, true), + ("realised", false, false, true), + ("recognising", false, false, true), + ("analyze", true, true, false), + ("analyse", false, true, true), + ("paralysed", false, true, true), + ("defense", true, false, false), + ("defence", false, true, true), + ("licence", false, true, true), + ("license", true, true, true), + ("practise", false, true, true), + ("traveled", true, false, false), + ("travelled", true, true, true), + ("catalog", true, false, false), + ("catalogue", true, true, true), + ("anaemia", false, true, true), + ("anemia", true, true, false), + ("tire", true, true, true), + ("tyre", false, false, true), + ("programme", false, true, true), + ("program", true, true, true), + ("meter", true, true, true), + ("metre", false, true, true), + ("plough", false, true, true), + ("colour-blind", false, true, true), + // Words spelled alike everywhere, and real misspellings, are unaffected. + ("humorous", true, true, true), + ("honorary", true, true, true), + ("tour", true, true, true), + ("four", true, true, true), + ("programmed", true, true, true), + ("colourr", false, false, false), + ("colorr", false, false, false), + ("organisationz", false, false, false), + ]; + for (w, in_us, in_ca, in_gb) in cases { + assert_eq!(is_correct_in(w, us), in_us, "en-US: {w}"); + assert_eq!(is_correct_in(w, ca), in_ca, "en-CA: {w}"); + assert_eq!(is_correct_in(w, gb), in_gb, "en-GB: {w}"); + assert_eq!(is_correct_for(w, Some("en-AU")), in_gb, "en-AU follows British spelling: {w}"); + } + // The dialect's own spelling comes first, in the word's case. + assert_eq!(suggest_in("colour", 5, us).first().map(String::as_str), Some("color")); + assert_eq!(suggest_in("Colours", 5, us).first().map(String::as_str), Some("Colors")); + assert_eq!(suggest_in("color", 5, gb).first().map(String::as_str), Some("colour")); + assert_eq!(suggest_in("organisation", 5, us).first().map(String::as_str), Some("organization")); + assert_eq!(suggest_in("organisation", 5, ca).first().map(String::as_str), Some("organization")); + assert_eq!(suggest_in("analyze", 5, gb).first().map(String::as_str), Some("analyse")); + assert_eq!(suggest_in("CENTER", 5, gb).first().map(String::as_str), Some("CENTRE")); + assert_eq!(suggest_in("colour's", 5, us).first().map(String::as_str), Some("color's")); + assert_eq!(suggest_in("colour-blind", 5, us).first().map(String::as_str), Some("color-blind")); + // Shared suggestions come in the dialect's spelling, never the other one's. + let gb_sugg = suggest_in("colouur", 8, gb); + assert!(gb_sugg.contains(&"colour".to_string()) && !gb_sugg.contains(&"color".to_string()), "{gb_sugg:?}"); + let us_sugg = suggest_in("colouur", 8, us); + assert!(us_sugg.contains(&"color".to_string()) && !us_sugg.contains(&"colour".to_string()), "{us_sugg:?}"); + // A word in the user dictionary is right in every English. + add_word("zzcolourzz"); + assert!(is_correct_in("zzcolourzz", us)); + // Issues carry the dialect, so suggestions on demand come from the same one. + let issues = check_text("The color is nice.", &[(0..18, gb)]); + assert_eq!(issues.len(), 1, "{issues:?}"); + assert_eq!((issues[0].lang, &"The color is nice."[issues[0].start..issues[0].end]), (gb, "color")); + let grammar = check_text("the colour is nice.", &[(0..19, gb)]); + assert!(grammar.iter().any(|i| i.kind == IssueKind::Grammar && i.lang == gb), "{grammar:?}"); + } + + /// The words #40 (folded into #1281) reported as flagged in British English, with their + /// inflections: right in en-GB, and in en-US flagged or not as `en-variants.txt` decides. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn british_words_reported_in_issue_40_are_right_in_en_gb() { + let (us, gb, ca) = (ProofLang::En, ProofLang::EnGb, ProofLang::EnCa); + // (word, right in en-US, right in en-CA, right in en-GB) + let cases = [ + // Both spellings are right everywhere: `judgment` is also British legal usage, and + // American dictionaries list `judgement` as a variant. + ("judgement", true, true, true), + ("judgements", true, true, true), + ("judgment", true, true, true), + ("judgments", true, true, true), + ("defence", false, true, true), + ("defences", false, true, true), + ("defense", true, false, false), + // `license` is the British verb, so only American English flags `licence`. + ("licence", false, true, true), + ("licences", false, true, true), + ("license", true, true, true), + ("licensed", true, true, true), + ("licenses", true, true, true), + // `program` is British for computing. + ("programme", false, true, true), + ("programmes", false, true, true), + ("program", true, true, true), + ("programs", true, true, true), + // American English accepts both. + ("catalogue", true, true, true), + ("catalogues", true, true, true), + ("catalogued", true, true, true), + ("cataloguing", true, true, true), + ("catalog", true, false, false), + ("catalogs", true, false, false), + ]; + for (w, in_us, in_ca, in_gb) in cases { + assert_eq!(is_correct_in(w, us), in_us, "en-US: {w}"); + assert_eq!(is_correct_in(w, ca), in_ca, "en-CA: {w}"); + assert_eq!(is_correct_in(w, gb), in_gb, "en-GB: {w}"); + } + let text = "The judgement on the defence licence put the programme in the catalogue."; + assert!(check_spelling_in(text, gb).is_empty(), "{:?}", check_spelling_in(text, gb)); + assert_eq!(suggest_in("licences", 5, us).first().map(String::as_str), Some("licenses")); + assert_eq!(suggest_in("programmes", 5, us).first().map(String::as_str), Some("programs")); + } + + /// More British spellings that must stay right in en-GB and en-AU (#1281): -re, single l, ae, + /// with their inflections. + #[test] + fn more_british_spellings_are_right_in_en_gb() { + let gb = ProofLang::EnGb; + assert_eq!(ProofLang::from_tag(Some("en-AU")), gb); + for w in [ + "goitre", + "goitres", + "reconnoitre", + "sceptre", + "sceptres", + "enrol", + "enrols", + "enrolled", + "enrolment", + "fulfil", + "fulfils", + "fulfilled", + "fulfilment", + "instalment", + "instalments", + "skilful", + "skilfully", + "wilful", + "wilfully", + "encyclopaedia", + "encyclopaedias", + ] { + assert!(is_correct_in(w, gb), "en-GB: {w}"); + } + let text = "The wilful knight took the sceptre to fulfil his enrolment, paid the instalment and read the encyclopaedia."; + assert!(check_spelling_in(text, gb).is_empty(), "{:?}", check_spelling_in(text, gb)); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn user_dictionary_accepts_the_other_dialects_spelling() { + add_word("savour"); + assert!(is_correct_in("savour", ProofLang::En), "a word the user added is never flagged"); + assert!(!is_correct_in("savor", ProofLang::EnGb)); + remove_word("savour"); + assert!(!is_correct_in("savour", ProofLang::En)); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn supplement_is_sorted_lowercase() { + let words: Vec<&str> = EXTRA_EN.lines().filter(|l| !l.is_empty() && !l.starts_with('#')).collect(); + assert!(words.len() > 100); + assert!(words.windows(2).all(|p| p[0] < p[1]), "not sorted / duplicate"); + assert!(words.iter().all(|w| w.chars().all(|c| c.is_lowercase() || c == '\'' || c == '-')), "not lowercase"); + } + + /// Without the Language data too: the installed English dictionary or the built-in word list + /// marks both misspellings (R6.4). + #[test] + fn spelling_issues_in_text() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let t = "This sentense has a mispelled word, see https://example.com/xyzq."; + let v = check_spelling(t); + let bad: Vec<&str> = v.iter().map(|i| &t[i.start..i.end]).collect(); + assert_eq!(bad, vec!["sentense", "mispelled"]); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn address_token_covers_every_word_split_out_of_it() { + // #1335: one whitespace-delimited token, however punctuation splits it, has one + // web/e-mail classification, shared by all its words. + let t = "zxq@mail.example.com,blorp;qwirpy wugwug"; + let bad: Vec<&str> = check_spelling(t).iter().map(|i| &t[i.start..i.end]).collect(); + assert_eq!(bad, vec!["wugwug"]); + let opts = Options { ignore_internet: false, ..Options::default() }; + let bad: Vec<&str> = check_spelling_with(t, &opts).iter().map(|i| &t[i.start..i.end]).collect(); + assert_eq!(bad.first(), Some(&"zxq"), "{bad:?}"); + assert!(bad.contains(&"qwirpy"), "{bad:?}"); + assert_eq!(bad.last(), Some(&"wugwug"), "{bad:?}"); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn address_after_a_wide_space_is_an_internet_token() { + // A web or e-mail address after a no-break space, a narrow no-break space, a figure space + // or an ideographic space is skipped, as it is after a plain space. + let t = "see\u{a0}https://xyzq.example/blorp and\u{3000}www.qwirpy.example or\u{a0}zxq@mail.example \ + at\u{202f}https://frobq.example/zibb by\u{2007}snorf@mail.example wugwug"; + let bad: Vec<&str> = check_spelling(t).iter().map(|i| &t[i.start..i.end]).collect(); + assert_eq!(bad, vec!["wugwug"]); + let opts = Options { ignore_internet: false, ..Options::default() }; + let bad: Vec<&str> = check_spelling_with(t, &opts).iter().map(|i| &t[i.start..i.end]).collect(); + assert!(["xyzq", "blorp", "qwirpy", "zxq", "frobq", "zibb", "snorf", "wugwug"].iter().all(|w| bad.contains(w)), "{bad:?}"); + } + + #[test] + fn spelling_scales_over_one_long_token() { + // #1335: thousands of punctuation-separated words inside a single + // whitespace-free token: one classification for the token, one issue per + // misspelled word, through every public spelling entry point. Without the Language data + // too (craft-libs#482 review: the old core list took 14 s here). + let started = std::time::Instant::now(); + let word = |i: usize| { + let mut w = String::from("zxq"); + let mut n = i; + loop { + w.push((b'a' + (n % 26) as u8) as char); + n /= 26; + if n == 0 { + break; + } + } + w + }; + let t = (0..2000).map(word).collect::>().join(","); + assert_eq!(check_spelling(&t).len(), 2000); + assert_eq!(check_spelling_in(&t, ProofLang::En).len(), 2000); + let opts = Options { ignore_internet: false, ..Options::default() }; + assert_eq!(check_spelling_with(&t, &opts).len(), 2000); + assert_eq!(check_text(&t, &[]).into_iter().filter(|i| i.kind == IssueKind::Spelling).count(), 2000); + // Debug builds on a loaded CI machine: a bound against blowups, not a benchmark. + assert!(started.elapsed() < std::time::Duration::from_secs(10), "{:?}", started.elapsed()); + } + + #[test] + fn grammar_issues() { + let t = "this is is a apple , and an car. Done"; + let v = check_grammar(t); + let msgs: Vec<&str> = v.iter().map(|i| i.message.as_str()).collect(); + assert!(msgs.iter().any(|m| m.contains("Repeated")), "{msgs:?}"); + assert!(msgs.iter().any(|m| m.contains("\"an\"")), "{msgs:?}"); + assert!(msgs.iter().any(|m| m.contains("\"a\" before")), "{msgs:?}"); + assert!(msgs.iter().any(|m| m.contains("space before")), "{msgs:?}"); + assert!(msgs.iter().any(|m| m.contains("Capitalize")), "{msgs:?}"); + assert!(msgs.iter().any(|m| m.contains("Extra space")), "{msgs:?}"); + assert!(check_grammar("A university is an hour away.").is_empty(), "{:?}", check_grammar("A university is an hour away.")); + } + + #[test] + fn ellipses_do_not_start_sentences() { + for text in [ + "I saw... a figure right outside.", + "I saw… a figure right outside.", + "I saw.... a figure right outside.", + "I saw... \"a figure\" right outside.", + "I saw… “a figure” right outside.", + ] { + let capitals: Vec<_> = + check_grammar(text).into_iter().filter(|issue| issue.message == "Capitalize the first word of a sentence").collect(); + assert!(capitals.is_empty(), "{text}: {capitals:?}"); + } + + for text in ["I saw. a figure.", "I saw! a figure.", "I saw? a figure.", "I saw.\" a figure."] { + let capitals: Vec<_> = + check_grammar(text).into_iter().filter(|issue| issue.message == "Capitalize the first word of a sentence").collect(); + assert_eq!(capitals.len(), 1, "{text}: {capitals:?}"); + assert_eq!(capitals[0].suggestions, ["A"], "{text}"); + } + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn user_dictionary_works() { + assert!(!is_correct("zxqwordcrafty")); + add_word("zxqwordcrafty"); + assert!(is_correct("zxqwordcrafty")); + } + + /// #40: the user dictionary survives a save and a fresh load. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn user_dictionary_persists_through_a_file() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let dir = std::env::temp_dir().join(format!("wc-dict-{}-{}", std::process::id(), "persist")); + let _ = std::fs::remove_dir_all(&dir); + // The folder is created on the first save. + let path = dir.join("sub/custom-dictionary.txt"); + add_word("zxqpersistone"); + add_word("zxqpersisttwo"); + save_user_dictionary(&path, &[]).expect("save works"); + let saved = std::fs::read_to_string(&path).unwrap(); + assert!(saved.lines().any(|w| w == "zxqpersistone") && saved.lines().any(|w| w == "zxqpersisttwo"), "{saved}"); + // Written atomically: no temporary file is left beside it. + let names: Vec<_> = std::fs::read_dir(dir.join("sub")).unwrap().map(|e| e.unwrap().file_name()).collect(); + assert_eq!(names, ["custom-dictionary.txt"]); + remove_word("zxqpersistone"); + assert!(!is_correct("zxqpersistone")); + load_user_dictionary(&path); + assert!(is_correct("zxqpersistone"), "loading brings the word back"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// #40: a hostile file is capped: invalid lines are skipped, and only the first megabyte is read. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn hostile_dictionary_files_are_capped() { + let dir = std::env::temp_dir().join(format!("wc-dict-{}-{}", std::process::id(), "hostile")); + let _ = std::fs::create_dir_all(&dir); + let path = dir.join("custom-dictionary.txt"); + let mut hostile = vec![0xff, 0xfe, b'\n']; + hostile.extend(b"zxqhostileok\n zxqhostiletrim \nzxq hostile space\nzxq\x07bell\n"); + hostile.extend("zxq".repeat(30).bytes()); // 90 characters: too long + hostile.push(b'\n'); + // Pad past the 1 MB cap with blank lines; the word after it is never read. + hostile.resize(1_000_100, b'\n'); + hostile.extend(b"zxqhostilepastcap\n"); + std::fs::write(&path, &hostile).unwrap(); + let added = load_user_dictionary(&path); + assert!(added.contains(&"zxqhostileok".to_string()) && added.contains(&"zxqhostiletrim".to_string()), "{added:?}"); + assert!(!added.iter().any(|w| w.contains("space") || w.contains("bell") || w.chars().count() > MAX_CUSTOM_WORD_CHARS), "{added:?}"); + assert!(!is_correct("zxqhostilepastcap"), "the file is read only up to the cap"); + assert_eq!(read_dictionary_file(&path).len(), 2); + let _ = std::fs::remove_dir_all(&dir); + } + + /// #40: saving merges in words another process saved, and leaves out the ones just removed. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn saving_merges_with_the_file() { + let dir = std::env::temp_dir().join(format!("wc-dict-{}-{}", std::process::id(), "merge")); + let _ = std::fs::create_dir_all(&dir); + let path = dir.join("custom-dictionary.txt"); + std::fs::write(&path, "zxqmergeother\nzxqmergegone\n").unwrap(); + add_word("zxqmergemine"); + save_user_dictionary(&path, &["zxqmergegone".to_string()]).unwrap(); + let saved = std::fs::read_to_string(&path).unwrap(); + assert!(saved.lines().any(|w| w == "zxqmergeother"), "the other process's word is kept: {saved}"); + assert!(saved.lines().any(|w| w == "zxqmergemine"), "{saved}"); + assert!(!saved.lines().any(|w| w == "zxqmergegone"), "a removed word stays out: {saved}"); + assert!(is_correct("zxqmergeother") && !is_correct("zxqmergegone")); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Ignore All lasts for the session: the word is accepted but isn't in the dictionary. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn ignored_words_are_not_dictionary_words() { + assert!(!is_correct("zxqignoredword")); + ignore_word("ZxqIgnoredWord"); + assert!(is_correct("zxqignoredword")); + assert!(!user_dictionary().contains(&"zxqignoredword".to_string())); + } + + #[test] + fn custom_words_are_single_short_words() { + assert_eq!(custom_word(" Hello "), Some("hello".into())); + for bad in ["", " ", "two words", "tab\tbed", "bell\u{7}", "bad\u{FFFD}", &"x".repeat(MAX_CUSTOM_WORD_CHARS + 1)] { + assert_eq!(custom_word(bad), None, "{bad:?}"); + } + assert!(custom_word(&"é".repeat(MAX_CUSTOM_WORD_CHARS)).is_some(), "the cap counts characters, not bytes"); + } + + /// #40: a missing file loads as empty, never an error. + #[test] + fn missing_dictionary_file_loads_empty() { + let path = std::env::temp_dir().join(format!("wc-dict-{}-missing.txt", std::process::id())); + let _ = std::fs::remove_file(&path); + assert!(load_user_dictionary(&path).is_empty()); + } + + /// #100: runs are proofed when their language has a dictionary (the default language too) + /// and skipped otherwise. + #[test] + fn dictionary_lang_tags() { + for l in [None, Some(""), Some(" "), Some("en"), Some("en-US"), Some("en-GB"), Some("EN-us"), Some("en_US")] { + assert!(has_dictionary(l), "{l:?}"); + } + for l in [Some("lt-LT"), Some("fr-FR"), Some("de"), Some("uk"), Some("-US"), Some("e")] { + assert!(!has_dictionary(l), "{l:?}"); + } + // The gate follows the dictionaries, not English: every proofed language is proofed. + for d in ProofLang::PROOFED.map(ProofLang::code) { + assert!(has_dictionary(Some(d)), "{d}"); + assert!(has_dictionary(Some(&format!("{}-XX", d.to_ascii_uppercase()))), "{d}"); + } + } + + /// Combining marks stay in their word (#3562 review): a decomposed word is one word, with the + /// byte range of what was typed, and it is checked as its composed form in every language. + #[test] + fn decomposed_words_stay_whole() { + let text = "tambie\u{301}n, cafe\u{301} \u{301}x"; + let ws: Vec<&str> = words(text).into_iter().map(|(a, b)| &text[a..b]).collect(); + assert_eq!(ws, ["tambie\u{301}n", "cafe\u{301}", "\u{301}x"]); + assert_eq!(words("tambie\u{301}n"), [(0, 9)]); + assert_eq!(&*composed("tambie\u{301}n"), "también"); + assert!(matches!(composed("también"), std::borrow::Cow::Borrowed(_)) && matches!(composed("plain"), std::borrow::Cow::Borrowed(_))); + pt::test_dict::install(); + assert!(is_correct_in("corac\u{327}a\u{303}o", ProofLang::Pt), "Portuguese gets the same composition"); + } + + /// #3562 review: Add to Dictionary, Ignore All, removal and the dictionary file use the same + /// composed key as spelling, so a word added or ignored decomposed is accepted in both forms, + /// an NFD entry in a saved file loads (and saves) composed, and either form removes it. + #[test] + fn user_dictionary_and_ignore_all_match_both_normalization_forms() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + es::test_dict::install(); + let es = ProofLang::Es; + let (nfd, nfc) = ("zqprue\u{301}ba", "zqpruéba"); + let text = format!("esto {nfd}"); + let issues = check_text(&text, &[(0..text.len(), es)]); + assert_eq!(issues.iter().map(|i| (i.start, i.end)).collect::>(), [(5, text.len())], "flagged at the typed bytes"); + assert!(!is_correct_in(nfc, es)); + add_word(nfd); + assert!(is_correct_in(nfd, es) && is_correct_in(nfc, es)); + assert!(check_text(&text, &[(0..text.len(), es)]).is_empty()); + assert!(user_dictionary().contains(&nfc.to_string()) && !user_dictionary().contains(&nfd.to_string())); + assert!(remove_word(nfc), "the composed form removes the entry added decomposed"); + assert!(!is_correct_in(nfd, es) && !is_correct_in(nfc, es)); + // Ignore All, either way round. + ignore_word("zqigno\u{301}ro"); + assert!(is_correct_in("zqignóro", es) && is_correct_in("zqigno\u{301}ro", es)); + ignore_word("zqcallé"); + assert!(is_correct_in("zqcalle\u{301}", es)); + // A dictionary file with a decomposed entry loads composed and is saved composed. + let path = std::env::temp_dir().join(format!("wc-nfd-dictionary-{}.txt", std::process::id())); + std::fs::write(&path, "zqguarda\u{301}do\nzqotra\n").unwrap(); + let added = load_user_dictionary(&path); + assert!(added.contains(&"zqguardádo".to_string()) && !added.iter().any(|w| w.contains('\u{301}')), "{added:?}"); + assert!(is_correct_in("zqguardádo", es) && is_correct_in("zqguarda\u{301}do", es)); + save_user_dictionary(&path, &["zqotra".to_string()]).unwrap(); + let saved = std::fs::read_to_string(&path).unwrap(); + assert!(saved.lines().any(|l| l == "zqguardádo") && !saved.contains('\u{301}') && !saved.lines().any(|l| l == "zqotra"), "{saved}"); + // Removing the decomposed form takes out the composed entry, and the file follows. + assert!(remove_word("zqguarda\u{301}do")); + save_user_dictionary(&path, &["zqguarda\u{301}do".to_string()]).unwrap(); + assert!(!std::fs::read_to_string(&path).unwrap().lines().any(|l| l == "zqguardádo")); + assert!(!is_correct_in("zqguardádo", es)); + let _ = std::fs::remove_file(&path); + } + + /// #100: runs in a language without a dictionary are never flagged; English still is. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn non_english_words_are_not_flagged() { + for w in ["ąžuolas", "élève", "über", "labas", "bonjour"] { + assert!(is_correct_for(w, Some("lt-LT")), "{w}"); + assert!(is_correct_for(w, Some("fr-FR")), "{w}"); + } + assert!(!is_correct_for("helo", Some("en-US")), "English typos still flagged"); + assert!(!is_correct_for("helo", None), "default language still proofed"); + assert!(is_correct_for("hello", Some("en-US"))); + } + + /// File › Options › Proofing (#485): each "Ignore" choice turned off makes those words count. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn ignore_options_decide_what_is_checked() { + let t = "SENTENSE helo2 www.exampel.org"; + let bad = |o: Options| check_spelling_with(t, &o).iter().map(|i| t[i.start..i.end].to_string()).collect::>(); + assert!(bad(Options::default()).is_empty(), "Word's defaults skip all three"); + let all = Options { ignore_uppercase: false, ignore_numbers: false, ignore_internet: false }; + assert_eq!(bad(all), ["SENTENSE", "helo2", "www", "exampel"]); + assert_eq!(bad(Options { ignore_uppercase: false, ..Options::default() }), ["SENTENSE"]); + assert!(is_correct_with("NASA", &all), "capitals that are a word stay correct"); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn hyphenation() { + let l = hyphen::Limits::default(); + assert!(hyphen::hyphenate_word("hyphenation", &l).matches('-').count() >= 2); + } + + fn flagged(t: &str, v: &[Issue]) -> Vec { + v.iter().filter(|i| i.kind == IssueKind::Spelling).map(|i| t[i.start..i.end].to_string()).collect() + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn each_language_uses_its_own_tools() { + pl::test_dict::install(); + let pl = "Zażółć gęślą jaźń, chrząszcz brzmi w trzcinie."; + assert!(check_spelling_in(pl, ProofLang::Pl).is_empty(), "{:?}", check_spelling_in(pl, ProofLang::Pl)); + assert!(check_spelling_in(pl, ProofLang::En).len() >= 3, "Polish words are English misspellings"); + assert!(check_spelling_in("Nous écrivons en français.", ProofLang::Other).is_empty()); + assert!(check_grammar_in("this is is wrong", ProofLang::Other).is_empty()); + assert!(is_correct_in("anything", ProofLang::Other) && suggest_in("xyzzy", 5, ProofLang::Other).is_empty()); + let v = check_spelling_in("Płynie żeka.", ProofLang::Pl); + assert_eq!(flagged("Płynie żeka.", &v), ["żeka"]); + assert_eq!((v[0].lang, v[0].message.as_str()), (ProofLang::Pl, "Możliwy błąd pisowni")); + assert!(suggest_in("żeka", 5, ProofLang::Pl).contains(&"rzeka".to_string())); + assert!(suggest_in("recieve", 5, ProofLang::En).contains(&"receive".to_string())); + // Polish grammar speaks Polish and has no English a/an rule. + let g = check_grammar_in("Wiem że to jest a apple.", ProofLang::Pl); + assert!(g.iter().any(|i| i.message == "Brak przecinka przed „że”"), "{g:?}"); + assert!(!g.iter().any(|i| i.message.contains("\"an\"")), "{g:?}"); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn italian_uses_its_own_tools() { + it::test_dict::install(); + let it = "La città vecchia ha un bel caffè in piazza."; + assert!(check_spelling_in(it, ProofLang::It).is_empty(), "{:?}", check_spelling_in(it, ProofLang::It)); + assert!(check_spelling_in(it, ProofLang::En).len() >= 3, "Italian words are English misspellings"); + let v = check_spelling_in("Vado a Roma con lbiro.", ProofLang::It); + assert_eq!(flagged("Vado a Roma con lbiro.", &v), ["lbiro"]); + assert_eq!((v[0].lang, v[0].message.as_str()), (ProofLang::It, "Possibile errore di ortografia")); + assert!(suggest_in("caffe", 5, ProofLang::It).contains(&"caffè".to_string())); + // Italian grammar speaks Italian and has no English a/an rule. + let g = check_grammar_in("questo questo è un test", ProofLang::It); + assert!(g.iter().any(|i| i.message.contains("ripetuta")), "{g:?}"); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn mixed_language_paragraphs_check_each_run_in_its_language() { + pl::test_dict::install(); + // A Polish paragraph quoting English: English spelling and rules inside the quote only. + let t = "Napisał „the quick brown fox is is here” i poszedł spać, bo żeka wylała."; + let (a, b) = (t.find("the").unwrap(), t.find('”').unwrap()); + let spans = [(0..a, ProofLang::Pl), (a..b, ProofLang::En), (b..t.len(), ProofLang::Pl)]; + let v = check_text(t, &spans); + assert_eq!(flagged(t, &v), ["żeka"], "{v:?}"); + let rep = v.iter().find(|i| i.kind == IssueKind::Grammar).unwrap(); + assert_eq!((rep.lang, rep.message.as_str()), (ProofLang::En, "Repeated word: \"is\""), "{v:?}"); + // The English rules don't run over the Polish text (no "Capitalize" for the lower-case + // Polish words), and the Polish rules don't run over the English quote. + assert!(v.iter().all(|i| i.lang == ProofLang::Pl || (a..b).contains(&i.start)), "{v:?}"); + // Text outside every span is English; no spans at all is English. + assert_eq!(flagged("helo świat", &check_text("helo świat", &[])), ["helo", "świat"]); + assert_eq!(flagged("helo świat", &check_text("helo świat", &[(5..12, ProofLang::Pl)])), ["helo"]); + // A language without tools: nothing flagged there. + assert!(check_text("Bonjour mes amis", &[(0..16, ProofLang::Other)]).is_empty()); + // Portuguese checks in their span only, with the Portuguese message. + pt::test_dict::install(); + let t = "Ele disse que coraçao e frequencia existem."; + let v = check_text(t, &[(0..t.len(), ProofLang::Pt)]); + assert_eq!(flagged(t, &v), ["coraçao", "frequencia"], "{v:?}"); + assert!(v.iter().all(|i| i.lang == ProofLang::Pt && i.message == "Possível erro de ortografia"), "{v:?}"); + // Italian checks in its span only, with the Italian message. + it::test_dict::install(); + let t = "Vado a Roma con lbiro e caffe."; + let v = check_text(t, &[(0..t.len(), ProofLang::It)]); + assert_eq!(flagged(t, &v), ["lbiro", "caffe"], "{v:?}"); + assert!(v.iter().all(|i| i.lang == ProofLang::It && i.message == "Possibile errore di ortografia"), "{v:?}"); + } + + proptest::proptest! { + #[test] + fn never_panics(s in "\\PC{0,80}") { + let _ = check_spelling(&s); + let _ = check_grammar(&s); + let _ = suggest(&s, 3); + let _ = has_dictionary(Some(&s)); + let _ = is_correct_for(&s, Some(&s)); + let _ = hyphen::hyphen_points(&s, &hyphen::Limits::default()); + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::De, ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy, ProofLang::Other] { + let _ = check_spelling_in(&s, lang); + let _ = check_grammar_in(&s, lang); + let _ = is_correct_in(&s, lang); + let _ = hyphen::hyphen_points_in(&s, &hyphen::Limits::default(), lang); + } + let _ = lang::normalize_tag(&s); + let _ = ProofLang::from_tag(Some(&s)); + } + + #[test] + fn polish_suggestions_never_panic(s in "[a-ząćęłńóśźżA-ZĄĆĘŁŃÓŚŹŻ'’ -]{0,14}") { + let _ = suggest_in(&s, 4, ProofLang::Pl); + } + + #[test] + fn portuguese_suggestions_never_panic(s in "[a-záàâãéêíóôõúüçA-ZÁÀÂÃÉÊÍÓÔÕÚÜÇ'’ -]{0,14}") { + let _ = suggest_in(&s, 4, ProofLang::Pt); + } + + #[test] + fn italian_suggestions_never_panic(s in "[a-zàèéìòóùA-ZÀÈÉÌÒÓÙ'’ -]{0,14}") { + let _ = suggest_in(&s, 4, ProofLang::It); + } + + #[test] + fn turkish_suggestions_never_panic(s in "[a-zçğıöşüA-ZÇĞİÖŞÜ'’ -]{0,14}") { + let _ = suggest_in(&s, 4, ProofLang::Tr); + } + + #[test] + fn czech_suggestions_never_panic(s in "[a-zěščřžýáíéůúĚŠČŘŽÝÁÍÉŮÚ'’ -]{0,14}") { + cs::test_dict::install(); + let _ = suggest_in(&s, 4, ProofLang::Cs); + } + + #[test] + fn indonesian_suggestions_never_panic(s in "[a-zA-Zé'’ -]{0,14}") { + id::test_dict::install(); + let _ = suggest_in(&s, 4, ProofLang::Id); + let _ = is_correct_in(&s, ProofLang::Id); + } + + #[test] + fn spanish_suggestions_never_panic(s in "[a-zA-ZáéíóúüñÁÉÍÓÚÜÑ¿¡'’ -]{0,14}") { + es::test_dict::install(); + for lang in [ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy] { + let _ = suggest_in(&s, 4, lang); + let _ = is_correct_in(&s, lang); + } + } + + #[test] + fn mixed_spans_never_panic(s in "\\PC{0,60}", cuts in proptest::collection::vec((0usize..80, 0usize..80, 0u8..15), 0..6)) { + let langs = [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy, ProofLang::Other]; + let spans: Vec<(Range, ProofLang)> = cuts.iter().map(|&(a, b, l)| (a.min(b)..a.max(b), langs[l as usize % langs.len()])).collect(); + for i in check_text(&s, &spans) { + proptest::prop_assert!(i.start <= i.end && i.end <= s.len()); + } + } + } + + proptest::proptest! { + // Suggestions scan the whole word list: fewer cases keep the test quick. + #![proptest_config(proptest::prelude::ProptestConfig::with_cases(24))] + #[test] + fn english_dialect_suggestions_never_panic(s in "[a-zA-Zéß'’ -]{0,16}|(colou?r|cent(er|re)|organi[sz]ation)['’s-]{0,4}") { + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + let _ = suggest_in(&s, 4, lang); + let _ = is_correct_in(&s, lang); + let _ = en_variants::preferred(&s, lang); + let _ = en_variants::known_variant(&s, lang, |_| true); + } + } + } +} diff --git a/crates/proof/src/no_data.rs b/crates/proof/src/no_data.rs new file mode 100644 index 00000000..0cab5eb9 --- /dev/null +++ b/crates/proof/src/no_data.rs @@ -0,0 +1,163 @@ +//! A build with no Language data embedded (no `CRAFT_I18N_DIR`): every public entry point still +//! works and never panics. English spelling still marks misspellings, against an English +//! Hunspell dictionary installed on the system or else the Moby word list built into the crate; +//! the dialect variants and hyphenation that need the data are off, and grammar (code, not data) +//! still works. + +use crate::hyphen::{self, Limits, Source}; +use crate::{EnglishSource, Options, ProofLang, dict, en_variants, patterns}; + +const LANGS: [ProofLang; 18] = [ + ProofLang::En, + ProofLang::EnGb, + ProofLang::EnCa, + ProofLang::Pl, + ProofLang::Bg, + ProofLang::Pt, + ProofLang::It, + ProofLang::Hr, + ProofLang::Tr, + ProofLang::Ru, + ProofLang::Cs, + ProofLang::Id, + ProofLang::Es, + ProofLang::EsMx, + ProofLang::Es419, + ProofLang::EsUy, + ProofLang::De, + ProofLang::Other, +]; + +/// Words and text that exercise each entry point, hostile ones included. +const SAMPLES: [&str; 10] = [ + "", + "teh colour of hyphenation", + "Ünïcödé — ‘quotes’ don't", + "a a apple .Next sentence", + "https://example.org a@b.org", + "zażółć gęślą jaźń", + "Привет, мир — кто-то", + "¿Qué tal? más", + "\u{0}\u{FFFF}\u{10FFFF}", + "supercalifragilisticexpialidociousxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", +]; + +#[test] +fn nothing_is_embedded() { + assert!(crate::data::EN_US_DIC.is_empty()); + assert!(dict::Dictionary::en_us().is_empty()); + for p in [patterns::Patterns::en_us(), patterns::Patterns::pl(), patterns::Patterns::pt(), patterns::Patterns::it(), patterns::Patterns::de()] { + assert!(p.is_empty()); + assert_eq!(p.exception_count(), 0); + } +} + +/// R6.4, English never goes unchecked: without the embedded word list, English is checked against +/// an installed English Hunspell dictionary or, without one, the Moby word list built into the +/// crate. Either way a misspelling is marked and common words are right. (Before, everything was right.) +#[test] +fn english_spelling_still_marks_misspellings() { + let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + assert!(matches!(crate::english_source(l), Some(EnglishSource::Hunspell(_) | EnglishSource::CoreList)), "{l:?}"); + assert!(!crate::is_correct_in("becuase", l) && !crate::is_correct_in("teh", l), "{l:?}"); + assert!(crate::suggest_in("teh", 5, l).contains(&"the".to_string()), "{l:?}"); + assert!(crate::suggest_in("becuase", 5, l).contains(&"because".to_string()), "{l:?}"); + for w in ["the", "The", "house", "houses", "because", "running", "people", "beautiful", "don't"] { + assert!(crate::is_correct_in(w, l), "{w} {l:?}"); + } + assert_eq!(crate::check_spelling_in("Becuase the house of the people", l).len(), 1, "{l:?}"); + } + assert!( + !crate::is_correct("becuase") && !crate::is_correct_with("becuase", &Options::default()) && !crate::is_correct_for("becuase", Some("en-GB")) + ); + assert!(crate::suggest("becuase", 5).contains(&"because".to_string())); + assert_eq!(crate::check_spelling("Becuase house").len(), 1); + // The dialect variants are Language data: none without it. + for w in ["colour", "color"] { + assert_eq!(en_variants::preferred(w, ProofLang::En), None); + assert!(!en_variants::is_foreign(w, ProofLang::EnGb)); + } + // Text in a language without proofing is never checked with English. + assert!(crate::check_spelling_in("Teh hous zxqv", ProofLang::Other).is_empty()); + assert!(crate::is_correct_for("teh", Some("fr-FR"))); +} + +#[test] +fn grammar_still_works() { + assert!(!crate::check_grammar("This is is wrong.").is_empty()); +} + +#[test] +fn hyphenation_is_off_where_it_needs_data() { + let lim = Limits::default(); + for (w, l) in [ + ("hyphenation", ProofLang::En), + ("hyphenation", ProofLang::Cs), + ("przedsiębiorstwo", ProofLang::Pl), + ("computador", ProofLang::Pt), + ("calcolatore", ProofLang::It), + ("Silbentrennung", ProofLang::De), + ] { + assert_eq!(hyphen::word_breaks_in(w, l), (vec![], Source::None), "{w} {l:?}"); + assert!(hyphen::hyphen_points_in(w, &lim, l).is_empty(), "{w} {l:?}"); + assert_eq!(hyphen::hyphenate_word_in(w, &lim, l), w); + } + assert!(hyphen::hyphen_points("hyphenation", &lim).is_empty()); + assert_eq!(hyphen::hyphenate_word("hyphenation", &lim), "hyphenation"); + assert_eq!(hyphen::word_breaks("hyphenation").1, Source::None); + // Rules are code, not data: Bulgarian still hyphenates. + assert!(!hyphen::word_breaks_in("книжарница", ProofLang::Bg).0.is_empty()); +} + +/// Every public entry point, in every language, on every sample: no panic. +#[test] +fn every_public_function_runs_without_data() { + let lim = Limits { min_word: 0, after_first: 0, before_last: 0, capitalized: true }; + let opts = Options { ignore_uppercase: false, ignore_numbers: false, ignore_internet: false }; + for text in SAMPLES { + let len = text.len(); + let _ = crate::words(text); + let _ = crate::check_spelling(text); + let _ = crate::check_spelling_with(text, &opts); + let _ = crate::check_grammar(text); + let _ = crate::is_correct(text); + let _ = crate::is_correct_with(text, &opts); + let _ = crate::suggest(text, 5); + let _ = crate::custom_word(text); + let _ = crate::has_dictionary(Some(text)); + let _ = crate::missing_dictionary(Some(text)); + let _ = crate::unreadable_dictionary(Some(text)); + let _ = crate::is_correct_for(text, Some(text)); + let _ = hyphen::word_breaks(text); + let _ = hyphen::hyphen_points(text, &lim); + let _ = hyphen::hyphenate_word(text, &lim); + let spans: Vec<(std::ops::Range, ProofLang)> = LANGS.iter().map(|&l| (0..len, l)).collect(); + let _ = crate::check_text(text, &spans); + let _ = crate::check_text_with(text, &spans, &opts, true, true); + // Out-of-range and reversed spans are hostile input too. + #[allow(clippy::reversed_empty_ranges)] + let _ = crate::check_text(text, &[(len + 5..len + 9, ProofLang::En), (3..1, ProofLang::Pl)]); + for l in LANGS { + let _ = crate::check_spelling_in(text, l); + let _ = crate::check_grammar_in(text, l); + let _ = crate::is_correct_in(text, l); + let _ = crate::is_correct_in_with(text, l, &opts); + let _ = crate::suggest_in(text, 5, l); + let _ = crate::dictionary_load_error(l); + let _ = crate::english_source(l); + let _ = hyphen::word_breaks_in(text, l); + let _ = hyphen::hyphen_points_in(text, &lim, l); + let _ = hyphen::hyphenate_word_in(text, &lim, l); + let _ = en_variants::preferred(text, l); + let _ = en_variants::is_foreign(text, l); + let _ = en_variants::known_variant(text, l, |_| true); + } + } + let _ = crate::installed_dictionaries(); + let _ = crate::dictionary_generation(); + let _ = crate::user_dictionary(); + let _ = crate::has_dictionary(None); + let _ = crate::missing_dictionary(None); + let _ = crate::unreadable_dictionary(None); +} diff --git a/crates/proof/src/patterns.rs b/crates/proof/src/patterns.rs new file mode 100644 index 00000000..cb5855d4 --- /dev/null +++ b/crates/proof/src/patterns.rs @@ -0,0 +1,356 @@ +//! Liang hyphenation patterns (TeX notation, e.g. `.ab1c`, `4t1ion`): parsing and application. +//! +//! The bundled English set is *ours*: trained from the public-domain Moby Hyphenator word list by +//! DesignCraft's patgen (see `examples/hyphgen.rs`). The Polish set is the TeX one by Hanna +//! Kołodziejska, Bogusław Jackowski and Marek Ryćko (hyph-utf8 `hyph-pl.tex`, used under its MIT +//! licence option), bundled unmodified: [`Patterns::parse`] reads TeX's `\patterns{…}` and +//! `\hyphenation{…}` blocks directly. Patterns are stored in a hash map keyed by the packed letter +//! string, so applying them to a word costs `O(len × max_pattern_len)` lookups. + +use std::collections::HashMap; +use std::hash::{BuildHasherDefault, Hasher}; + +/// Word-boundary marker in patterns. +pub const DOT: char = '.'; +/// Longest pattern the packed key supports (6 bits a letter in a `u128`; the Polish set's longest +/// is 14 including the dots). +pub const MAX_PATTERN_LEN: usize = 20; + +/// Small, fast multiplicative hasher for packed integer keys and short strings (FxHash-style). +#[derive(Default, Clone, Copy)] +pub struct FxHasher(u64); + +impl Hasher for FxHasher { + fn finish(&self) -> u64 { + self.0 + } + fn write(&mut self, bytes: &[u8]) { + let (chunks, rest) = bytes.as_chunks::<8>(); + for c in chunks { + self.write_u64(u64::from_le_bytes(*c)); + } + for &b in rest { + self.write_u64(b as u64); + } + } + fn write_u8(&mut self, i: u8) { + self.write_u64(i as u64); + } + fn write_usize(&mut self, i: usize) { + self.write_u64(i as u64); + } + fn write_u64(&mut self, i: u64) { + self.0 = (self.0.rotate_left(5) ^ i).wrapping_mul(0x51_7c_c1_b7_27_22_0a_95); + } + fn write_u128(&mut self, i: u128) { + self.write_u64(i as u64); + self.write_u64((i >> 64) as u64); + } +} + +pub type FxMap = HashMap>; + +/// Maps characters to 6-bit codes (1 = word boundary, 2.. = letters); 0 = not in the alphabet. +#[derive(Clone, Debug)] +pub struct Alphabet { + ascii: [u8; 128], + other: Vec<(char, u8)>, + next: u8, +} + +impl Default for Alphabet { + fn default() -> Self { + let mut a = Alphabet { ascii: [0; 128], other: Vec::new(), next: 2 }; + a.ascii[DOT as usize] = 1; + a + } +} + +impl Alphabet { + pub fn code(&self, c: char) -> u8 { + if (c as u32) < 128 { self.ascii[c as usize] } else { self.other.iter().find(|x| x.0 == c).map_or(0, |x| x.1) } + } + /// Code for `c`, adding it if new (None when the 6-bit alphabet is full). + pub fn intern(&mut self, c: char) -> Option { + let k = self.code(c); + if k != 0 { + return Some(k); + } + if self.next >= 63 { + return None; + } + let k = self.next; + self.next += 1; + if (c as u32) < 128 { + self.ascii[c as usize] = k; + } else { + self.other.push((c, k)); + } + Some(k) + } +} + +/// Packed key of a code string (≤ [`MAX_PATTERN_LEN`] codes): 6 bits a code, the length on top. +#[inline] +pub fn pack(codes: &[u8]) -> u128 { + let mut k = 0u128; + for &c in codes { + k = (k << 6) | c as u128; + } + k | ((codes.len() as u128) << 120) +} + +/// A compiled pattern set, with its exception words (TeX's `\hyphenation{…}`). +#[derive(Clone, Debug, Default)] +pub struct Patterns { + alpha: Alphabet, + /// Packed letters → inter-letter values (`len + 1` entries). + map: FxMap>, + max_len: usize, + /// Lowercase word → its break points (char indices), overriding the patterns. + exceptions: HashMap>, +} + +/// Which TeX block [`Patterns::parse`] is reading. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Block { + Patterns, + Exceptions, + /// Another command's argument (`\message{…}`), ignored. + Skip, +} + +impl Patterns { + /// Parse patterns: whitespace-separated, `%` comments to the end of the line (`#` too at the + /// start of a line). A whole TeX pattern file works as is: `\patterns{…}` holds patterns and + /// `\hyphenation{…}` exception words written with their hyphens (`ni-gdy`), and other + /// commands' arguments (`\message{…}`) are skipped; outside any block, a token with a hyphen + /// is an exception and anything else a pattern. + pub fn parse(text: &str) -> Patterns { + let mut p = Patterns::default(); + let mut block = Block::Patterns; + for line in text.lines() { + let line = line.trim(); + if line.starts_with('#') { + continue; + } + let line = line.split('%').next().unwrap_or(""); + for tok in line.split_whitespace() { + let mut tok = tok; + if let Some(cmd) = tok.strip_prefix('\\') { + let (name, rest) = cmd.split_once('{').unwrap_or((cmd, "")); + block = match name { + "hyphenation" => Block::Exceptions, + "patterns" => Block::Patterns, + // shortcut: the argument ends at the first `}`, so nested braces would leak; no bundled file has them. + _ if cmd.contains('{') => Block::Skip, + _ => Block::Patterns, + }; + tok = rest; + } + let closes = tok.ends_with('}'); + let tok = tok.trim_end_matches('}'); + if !tok.is_empty() && block != Block::Skip { + if block == Block::Exceptions || tok.contains('-') { + p.insert_exception(tok); + } else { + p.insert(tok); + } + } + if closes { + block = Block::Patterns; + } + } + } + p + } + + /// Add one pattern like `a1b`; returns false if it can't be represented. + pub fn insert(&mut self, pat: &str) -> bool { + let mut codes = Vec::with_capacity(pat.len()); + let mut vals = vec![0u8]; + for c in pat.chars() { + if let Some(d) = c.to_digit(10) { + *vals.last_mut().unwrap_or(&mut 0) = d as u8; + } else { + let Some(k) = self.alpha.intern(c) else { return false }; + codes.push(k); + vals.push(0); + } + } + if codes.is_empty() || codes.len() > MAX_PATTERN_LEN { + return false; + } + self.max_len = self.max_len.max(codes.len()); + self.map.insert(pack(&codes), vals.into_boxed_slice()); + true + } + + /// Add an exception word written with its hyphens (`ni-gdy`; no hyphen: never broken). + pub fn insert_exception(&mut self, word: &str) { + let mut plain = String::with_capacity(word.len()); + let mut points = Vec::new(); + let mut n = 0usize; + for c in word.chars() { + if c == '-' { + if n > 0 && points.last() != Some(&n) { + points.push(n); + } + } else { + plain.extend(c.to_lowercase()); + n += 1; + } + } + points.retain(|&i| i < n); + if n > 0 { + self.exceptions.insert(plain, points.into_boxed_slice()); + } + } + + /// Break points of an exception word (lowercase), if it is one. + pub fn exception(&self, word: &str) -> Option<&[usize]> { + self.exceptions.get(word).map(|b| &b[..]) + } + + /// Number of exception words. + pub fn exception_count(&self) -> usize { + self.exceptions.len() + } + + /// Make `letters` part of the alphabet even if no pattern mentions them. + pub fn add_letters(&mut self, letters: impl IntoIterator) { + for c in letters { + let _ = self.alpha.intern(c); + } + } + + pub fn len(&self) -> usize { + self.map.len() + } + + pub fn is_empty(&self) -> bool { + self.map.is_empty() + } + + /// Inter-letter values for a lowercase word: `out[i]` is the value *before* char `i` + /// (`out.len() == word.len() + 1`). None if a char is outside the pattern alphabet. + pub fn values(&self, word: &[char]) -> Option> { + let mut codes = Vec::with_capacity(word.len() + 2); + codes.push(1); + for &c in word { + let k = self.alpha.code(c); + if k == 0 { + return None; + } + codes.push(k); + } + codes.push(1); + let mut vals = vec![0u8; codes.len() + 1]; + apply(&self.map, self.max_len, &codes, &mut vals); + // Padded position p sits before padded char p; word char i is padded char i + 1. + vals.get(1..codes.len()).map(<[u8]>::to_vec) + } + + /// Break points (char indices, a hyphen goes before that char) of a lowercase word. + pub fn points(&self, word: &[char]) -> Option> { + let v = self.values(word)?; + Some((1..word.len()).filter(|&i| v.get(i).is_some_and(|x| x % 2 == 1)).collect()) + } + + /// The patterns in TeX notation (sorted by letters), e.g. for writing a pattern file. + pub fn to_tex(&self) -> Vec { + let mut rev: Vec<(u8, char)> = + (0u8..128).filter(|&c| self.alpha.ascii[c as usize] != 0).map(|c| (self.alpha.ascii[c as usize], c as char)).collect(); + rev.extend(self.alpha.other.iter().map(|&(c, k)| (k, c))); + let ch = |k: u8| rev.iter().find(|x| x.0 == k).map_or('?', |x| x.1); + let mut out: Vec<(String, String)> = self + .map + .iter() + .map(|(&key, vals)| { + let n = ((key >> 120) as usize).min(MAX_PATTERN_LEN); + let letters: Vec = (0..n).map(|i| ch(((key >> (6 * (n - 1 - i))) & 63) as u8)).collect(); + let mut s = String::new(); + for (i, l) in letters.iter().enumerate() { + if let Some(&v) = vals.get(i).filter(|v| **v > 0) { + s.push(char::from(b'0' + v.min(9))); + } + s.push(*l); + } + if let Some(&v) = vals.get(n).filter(|v| **v > 0) { + s.push(char::from(b'0' + v.min(9))); + } + (letters.into_iter().collect(), s) + }) + .collect(); + out.sort(); + out.into_iter().map(|x| x.1).collect() + } +} + +/// Apply `map` to padded `codes`, raising `vals` (len = codes.len() + 1; shorter is tolerated). +#[inline] +pub fn apply(map: &FxMap>, max_len: usize, codes: &[u8], vals: &mut [u8]) { + let n = codes.len(); + for s in 0..n { + let mut k = 0u128; + for (len, &c) in codes.iter().skip(s).take(max_len.min(MAX_PATTERN_LEN)).enumerate() { + k = (k << 6) | c as u128; + if let Some(v) = map.get(&(k | (((len + 1) as u128) << 120))) { + for (j, &x) in v.iter().enumerate() { + if let Some(slot) = vals.get_mut(s + j) + && x > *slot + { + *slot = x; + } + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn liang_example() { + // The classic example from Liang's thesis: "hy-phen-ation". + let p = Patterns::parse("hy3ph he2n hena4 hen5at 1na n2at 1tio 2io o2n"); + let w: Vec = "hyphenation".chars().collect(); + assert_eq!(p.points(&w).unwrap(), vec![2, 6]); + let mut tex = p.to_tex(); + tex.sort(); + assert!(tex.contains(&"hen5at".to_string())); + assert!(p.values(&['h', 'é']).is_none()); + } + + #[test] + fn tex_files_parse_with_blocks_comments_and_exceptions() { + let p = Patterns::parse( + "% a comment\n\\patterns{\n1na n2at % trailing comment\n.ab4cdefghijklm5n\n}\n\\hyphenation{\nni-gdy\nna-dal\nzawsze\n}\nhy-phen-ation\n", + ); + assert_eq!(p.len(), 3); + assert_eq!(p.exception("nigdy"), Some(&[2][..])); + assert_eq!(p.exception("nadal"), Some(&[2][..])); + assert_eq!(p.exception("zawsze"), Some(&[][..])); + assert_eq!(p.exception("hyphenation"), Some(&[2, 6][..])); + assert_eq!(p.exception_count(), 4); + // A pattern longer than the old 9-letter limit applies. + let w: Vec = "abcdefghijklmn".chars().collect(); + assert_eq!(p.values(&w).unwrap()[13], 5); + assert!(p.to_tex().contains(&".ab4cdefghijklm5n".to_string())); + } + + #[test] + fn other_commands_arguments_are_skipped() { + let p = Patterns::parse("\\message{Some Patterns `x-y' 2024-02-28}\n\\patterns{%\na1b\n}\n\\endinput\n"); + assert_eq!((p.len(), p.exception_count()), (1, 0)); + } + + #[test] + fn apply_tolerates_short_value_buffers() { + let p = Patterns::parse("a1b"); + let mut vals = [0u8; 1]; + apply(&p.map, p.max_len, &[1, 2, 3, 1], &mut vals); + } +} diff --git a/crates/proof/src/pl/grammar.rs b/crates/proof/src/pl/grammar.rs new file mode 100644 index 00000000..4289f0c7 --- /dev/null +++ b/crates/proof/src/pl/grammar.rs @@ -0,0 +1,548 @@ +//! Polish grammar and punctuation checks: cheap, rule-based and tuned to stay quiet when unsure +//! (a missed comma costs less than a wrong mark on correct text). Messages are in Polish. +//! +//! - **Repeated word** (`się się`), except interjections and words Polish doubles on purpose +//! (`to to`, `tak tak`, `ha ha`). +//! - **Space before punctuation** (`tak ,`) and **double spaces**; emoticons (` :)`) and +//! ellipses are left alone. +//! - **Capital letter at the start of a sentence**, knowing Polish abbreviations (`np.`, `itd.`, +//! `r.`, `m.in.`), initials, ordinal numbers (`5. miejsce`), ellipses and dialogue dashes +//! (`– Tak? – zapytał`), after which lower case is correct. +//! - **Missing comma** before subordinating conjunctions and relative pronouns that always open a +//! clause: `że`, `ponieważ`, `gdyż`, `aby`, `żeby`, `jeśli`, `jeżeli`, `gdyby`, `lecz` and the +//! forms of `który` — unless a conjunction or particle comes first (`mimo że`, `chyba że`, +//! `tak aby`, `i który`) or, for `który`, a preposition (`w którym`: the comma goes before it). +//! - **`we`/`ze`** where the next word's consonant cluster requires them (`we wtorek`, +//! `ze szkoły`, `ze mną`). +//! - **Doubled punctuation** (`,,` — often a typed opening quote `„` — `..`, `;;`). +//! +//! No English rule (such as `a`/`an`) applies to Polish text. + +use crate::{Issue, IssueKind, ProofLang, words}; + +/// Words Polish repeats on purpose: interjections, onomatopoeia and emphatic doubling. +const ALLOWED_REPEATS: &[&str] = &[ + "to", "tak", "nie", "no", "ha", "he", "hi", "hej", "oj", "aj", "ej", "ach", "och", "ech", "puk", "hop", "bum", "cyk", "tik", "pa", "fiu", "la", + "bardzo", "dawno", "daleko", "długo", +]; + +/// Abbreviations written with a final period after which a sentence goes on in lower case +/// (lower case, without the period). +const ABBREVIATIONS: &[&str] = &[ + "np", "itd", "itp", "tzn", "tj", "tzw", "ok", "ul", "al", "pl", "os", "in", "wg", "zob", "por", "str", "ss", "nr", "pkt", "ust", "art", "par", + "rozdz", "tab", "rys", "il", "przyp", "red", "wyd", "oprac", "tłum", "cyt", "jw", "ds", "im", "ks", "św", "bł", "prof", "dr", "doc", "hab", + "inż", "mgr", "lek", "płk", "gen", "mjr", "kpt", "ppłk", "sierż", "szer", "kmdr", "adm", "abp", "bp", "br", "ub", "bm", "pn", "płd", "płn", + "wsch", "zach", "gł", "godz", "min", "sek", "tys", "mln", "mld", "zł", "gr", "kg", "dag", "mies", "tyg", "pon", "wt", "śr", "czw", "pt", "sob", + "niedz", "sty", "lut", "mar", "kwi", "cze", "lip", "sie", "wrz", "paź", "lis", "gru", "ang", "niem", "franc", "łac", "ros", "wł", "hiszp", "pol", + "lit", "dosł", "przen", "pot", "zdrob", "fot", "ryc", "tel", "kom", "ew", "ob", "obyw", "pp", "zm", "ur", "ca", "vs", "etc", "cdn", "dot", "wz", + "zw", "zał", "cz", "rr", "ww", "jęz", "jun", "sen", "mec", "red", "dyr", "kier", "proc", "poj", "lp", "lm", "zaw", "woj", "pow", "gm", "wsp", + "nast", "poprz", "ang", "fr", +]; + +/// Subordinating conjunctions and relative pronouns that open a clause: a comma comes before +/// them. +const CLAUSE_OPENERS: &[&str] = &[ + "że", + "ponieważ", + "gdyż", + "aby", + "żeby", + "jeśli", + "jeżeli", + "gdyby", + "lecz", + "który", + "która", + "które", + "którego", + "której", + "któremu", + "którą", + "którym", + "których", + "którymi", + "którzy", +]; + +/// Words after which a clause opener takes no comma of its own: conjunctions (`i że`), particles +/// (`tylko jeśli`, `zwłaszcza że`) and the first halves of compound conjunctions (`mimo że`, +/// `chyba że`, `tak aby`, `jak gdyby`, `na wypadek gdyby`, `z tym że`). +const NO_COMMA_AFTER: &[&str] = &[ + "i", + "a", + "oraz", + "lub", + "albo", + "ani", + "czy", + "bądź", + "ale", + "lecz", + "bo", + "że", + "aby", + "żeby", + "by", + "niż", + "jak", + "jako", + "niby", + "tylko", + "nawet", + "właśnie", + "zwłaszcza", + "szczególnie", + "głównie", + "częściowo", + "jedynie", + "wyłącznie", + "chyba", + "mianowicie", + "przynajmniej", + "również", + "także", + "też", + "dopiero", + "jeszcze", + "już", + "zaś", + "więc", + "zatem", + "toteż", + "przecież", + "chociaż", + "choć", + "choćby", + "gdy", + "kiedy", + "jeśli", + "jeżeli", + "gdyby", + "skoro", + "zanim", + "dopóki", + "póki", + "aż", + "co", + "tak", + "mimo", + "pomimo", + "wtedy", + "teraz", + "dziś", + "dzisiaj", + "podczas", + "tym", + "bardziej", + "warunkiem", + "wypadek", + "zamiast", + "byle", + "lada", + "wiadomo", + "obojętnie", + "jedno", + "wszystkim", + "może", + "pewnie", + "prawdopodobnie", + "zapewne", + "dlatego", + "tyle", + "dość", + "ponieważ", + "gdyż", + "lecz", + "albowiem", + "oto", + "no", +]; + +/// Prepositions: a relative pronoun after one takes the comma before the preposition instead +/// (`dom, w którym`). +const PREPOSITIONS: &[&str] = &[ + "w", + "we", + "z", + "ze", + "na", + "do", + "od", + "ode", + "o", + "u", + "po", + "za", + "przed", + "przede", + "przez", + "przeze", + "przy", + "nad", + "nade", + "pod", + "pode", + "dla", + "bez", + "beze", + "ku", + "między", + "pomiędzy", + "wśród", + "spośród", + "spod", + "spoza", + "sprzed", + "znad", + "zza", + "poza", + "ponad", + "poprzez", + "dzięki", + "według", + "wobec", + "oprócz", + "prócz", + "obok", + "koło", + "około", + "wokół", + "dookoła", + "naprzeciw", + "naprzeciwko", + "wzdłuż", + "wewnątrz", + "zewnątrz", + "względem", + "przeciw", + "przeciwko", + "wbrew", + "wskutek", + "śród", + "wedle", + "pośród", + "ponad", + "niedaleko", + "blisko", +]; + +fn issue(start: usize, end: usize, message: String, suggestions: Vec) -> Issue { + Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::Pl } +} + +fn capitalize(w: &str) -> String { + crate::affix::capitalize(w) +} + +/// Polish vowels (and the accented vowels of loanwords). +fn is_vowel(c: char) -> bool { + matches!(c.to_lowercase().next().unwrap_or(c), 'a' | 'ą' | 'e' | 'ę' | 'i' | 'o' | 'ó' | 'u' | 'y' | 'é' | 'á' | 'ö' | 'ü' | 'í' | 'ú') +} + +fn is_consonant(c: char) -> bool { + c.is_alphabetic() && !is_vowel(c) +} + +/// Grammar issues in a paragraph of Polish text, in order. +pub fn check(text: &str) -> Vec { + check_with(text, true) +} + +/// [`check`] where `capital_first` says whether the text's first word must start a sentence with a capital +/// (false for a list item, which may be a fragment). +pub fn check_with(text: &str, capital_first: bool) -> Vec { + let ws: Vec<(usize, usize)> = words(text); + let mut v = Vec::new(); + repeated_words(text, &ws, &mut v); + spaces(text, &mut v); + sentence_capitals(text, &ws, capital_first, &mut v); + missing_commas(text, &ws, &mut v); + we_ze(text, &ws, &mut v); + doubled_punctuation(text, &mut v); + v.sort_by_key(|i| (i.start, i.end)); + v +} + +fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if !gap.is_empty() + && gap.chars().all(char::is_whitespace) + && w0.chars().all(char::is_alphabetic) + && w0.to_lowercase() == w1.to_lowercase() + && !ALLOWED_REPEATS.contains(&w0.to_lowercase().as_str()) + { + v.push(issue(*a0, *b1, format!("Powtórzone słowo: „{w1}”"), vec![w0.to_string()])); + } + } +} + +fn spaces(text: &str, v: &mut Vec) { + for (i, c) in text.char_indices() { + if c != ' ' { + continue; + } + let rest = text.get(i + 1..).unwrap_or(""); + let mut it = rest.chars(); + let Some(p) = it.next() else { continue }; + let after = it.next(); + let flag = match p { + ',' => true, + // `;)` and `;-)` are emoticons. + ';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')), + // ` ...` is an ellipsis, ` .5` a number. + '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), + // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. + ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '»' | '’' | '!' | '?')), + _ => false, + }; + if flag { + v.push(issue(i, i + 1 + p.len_utf8(), "Zbędna spacja przed znakiem interpunkcyjnym".into(), vec![p.to_string()])); + } + } + for (i, _) in text.match_indices(" ") { + if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { + continue; + } + v.push(issue(i, i + 2, "Podwójna spacja".into(), vec![" ".into()])); + } +} + +/// Is `prev`, before a period, an abbreviation, an initial, a number or a Roman numeral (after +/// which a lower-case word doesn't start a sentence)? +fn abbreviation_or_number(prev: &str) -> bool { + prev.chars().count() <= 1 + || prev.chars().any(|c| c.is_ascii_digit()) + || (prev.chars().all(|c| matches!(c, 'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M'))) + || ABBREVIATIONS.contains(&prev.to_lowercase().as_str()) +} + +const CLOSING: [char; 8] = ['"', '”', '’', '\'', ')', '»', ']', '\u{1}']; +const OPENING: [char; 8] = ['„', '"', '“', '«', '(', '[', '\'', '‚']; + +/// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after +/// an abbreviation), then space, with nothing but quotes and brackets around? +fn ends_sentence(gap: &str, prev: &str) -> bool { + let t = gap.trim_start_matches(CLOSING); + let Some(end) = t.chars().next() else { return false }; + if !matches!(end, '.' | '!' | '?') || t.starts_with("..") { + return false; + } + if end == '.' && abbreviation_or_number(prev) { + return false; + } + let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING); + rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| OPENING.contains(&c)) +} + +fn sentence_capitals(text: &str, ws: &[(usize, usize)], capital_first: bool, v: &mut Vec) { + // List items (`– jabłka,`) may start in lower case. + let list_item = text.trim_end().ends_with([',', ';']); + for (k, &(a, b)) in ws.iter().enumerate() { + let Some(w) = text.get(a..b) else { continue }; + let starts = if k == 0 { + let before = text.get(..a).unwrap_or(""); + capital_first && !list_item && before.chars().all(|c| c.is_whitespace() || OPENING.contains(&c)) + } else { + let Some(&(pa, pb)) = ws.get(k - 1) else { continue }; + ends_sentence(text.get(pb..a).unwrap_or(""), text.get(pa..pb).unwrap_or("")) + }; + if starts && w.chars().next().is_some_and(char::is_lowercase) && w.chars().all(char::is_alphabetic) { + v.push(issue(a, b, "Wielka litera na początku zdania".into(), vec![capitalize(w)])); + } + } +} + +fn missing_commas(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for k in 1..ws.len() { + let (Some(&(pa, pb)), Some(&(a, b))) = (ws.get(k - 1), ws.get(k)) else { continue }; + let (Some(prev), Some(w), Some(gap)) = (text.get(pa..pb), text.get(a..b), text.get(pb..a)) else { continue }; + if !CLAUSE_OPENERS.contains(&w) || gap.is_empty() || !gap.chars().all(|c| c == ' ' || c == '\u{a0}') { + continue; + } + if !prev.chars().all(char::is_alphabetic) { + continue; + } + let p = prev.to_lowercase(); + let p = p.as_str(); + if NO_COMMA_AFTER.contains(&p) { + continue; + } + if w.starts_with("któr") && (PREPOSITIONS.contains(&p) || p == "to") { + continue; + } + v.push(issue(pb, b, format!("Brak przecinka przed „{w}”"), vec![format!(", {w}")])); + } +} + +fn we_ze(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if gap != " " && gap != "\u{a0}" { + continue; + } + let next = w1.to_lowercase(); + let c: Vec = next.chars().collect(); + let at = |i: usize| c.get(i).copied(); + let (long, wanted) = match w0 { + "w" | "W" => ("we", (at(0) == Some('w') && at(1).is_some_and(is_consonant)) || next == "mnie"), + "z" | "Z" => { + // The first sound: `sz` is one consonant (`z szafy`, but `ze szkoły`). + let after = if at(0) == Some('s') && at(1) == Some('z') { at(2) } else { at(1) }; + let sibilant = matches!(at(0), Some('s' | 'z' | 'ś' | 'ź' | 'ż')); + ("ze", (sibilant && after.is_some_and(is_consonant)) || next == "mną") + } + _ => continue, + }; + if wanted { + let fixed = if w0.starts_with(char::is_uppercase) { capitalize(long) } else { long.to_string() }; + v.push(issue(*a0, *b0, format!("Przed tym słowem piszemy „{long}”"), vec![fixed])); + } + } +} + +fn doubled_punctuation(text: &str, v: &mut Vec) { + let bytes = text.as_bytes(); + let mut i = 0; + while let Some(&b) = bytes.get(i) { + if !matches!(b, b',' | b'.' | b';') { + i += 1; + continue; + } + let run = bytes.get(i..).map_or(0, |r| r.iter().take_while(|&&x| x == b).count()); + let flag = match b { + b'.' => run == 2, + _ => run >= 2, + }; + if flag { + let end = i + run; + let suggestions: Vec = match b { + // `,,` before a word, after a space: a typed opening quote. + b',' if text.get(end..).and_then(|t| t.chars().next()).is_some_and(char::is_alphabetic) + && (i == 0 || text.get(..i).is_some_and(|t| t.ends_with(char::is_whitespace))) => + { + vec!["„".into()] + } + b'.' => vec![".".into(), "…".into()], + _ => vec![char::from(b).to_string()], + }; + v.push(issue(i, end, "Powtórzony znak interpunkcyjny".into(), suggestions)); + } + i += run.max(1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn messages(t: &str) -> Vec { + check(t).into_iter().map(|i| format!("{}@{}", i.message, &t[i.start..i.end])).collect() + } + + fn clean(t: &str) { + let m = messages(t); + assert!(m.is_empty(), "{t:?}: {m:?}"); + } + + #[test] + fn repeated_words() { + let v = check("Ala ma ma kota i się się cieszy."); + assert_eq!(v.len(), 2, "{v:?}"); + assert_eq!(v[0].message, "Powtórzone słowo: „ma”"); + assert_eq!(v[0].suggestions, ["ma"]); + assert!(v.iter().all(|i| i.lang == ProofLang::Pl && i.kind == IssueKind::Grammar)); + clean("Tak tak, to to samo. Ha ha!"); + } + + #[test] + fn spaces_and_punctuation() { + let m = messages("Ala ma kota , a kot ma Alę . Czy na pewno ?"); + assert!(m.contains(&"Zbędna spacja przed znakiem interpunkcyjnym@ ,".to_string()), "{m:?}"); + assert!(m.contains(&"Zbędna spacja przed znakiem interpunkcyjnym@ .".to_string()), "{m:?}"); + assert!(m.contains(&"Zbędna spacja przed znakiem interpunkcyjnym@ ?".to_string()), "{m:?}"); + assert!(m.contains(&"Podwójna spacja@ ".to_string()), "{m:?}"); + clean("Fajnie :) Naprawdę :-) Albo :D Wartość .5 i wielokropek ..."); + let m = messages("Powiedział ,,tak” i poszedł.. Potem;; koniec."); + assert_eq!(m.iter().filter(|x| x.starts_with("Powtórzony znak")).count(), 3, "{m:?}"); + assert_eq!(check("Powiedział ,,tak”.").iter().find(|i| i.message.starts_with("Powtórzony")).unwrap().suggestions, ["„"]); + clean("Czekał… i czekał... aż do rana."); + } + + #[test] + fn sentence_start_capitals() { + let m = messages("to jest zdanie. a to drugie! czy trzecie? tak."); + assert_eq!(m.iter().filter(|x| x.starts_with("Wielka litera")).count(), 4, "{m:?}"); + assert_eq!(check("ala ma kota.")[0].suggestions, ["Ala"]); + // Abbreviations, initials, ordinals, dates, ellipses and dialogue dashes. + clean("Kupiłem owoce, np. jabłka, gruszki itp. oraz warzywa."); + clean("W 1990 r. powstała firma, m.in. dzięki dotacji."); + clean("Spotkałem J. Kowalskiego i prof. Nowaka przy ul. Długiej."); + clean("Zajął 5. miejsce w XX w. sporcie."); + clean("Czekałem... aż w końcu przyszedł."); + clean("– Naprawdę? – zapytał. – Tak – odpowiedziała."); + clean("Powiedział: „Wracam”. Potem wyszedł."); + // A list item may start in lower case. + clean("jabłka,"); + } + + #[test] + fn missing_commas() { + let m = messages("Wiem że przyjdzie. Mam kota który śpi. Zostałem bo musiałem. Idę aby zdążyć. Nie przyszedł ponieważ padało."); + for want in [ + "Brak przecinka przed „że”@ że", + "Brak przecinka przed „który”@ który", + "Brak przecinka przed „aby”@ aby", + "Brak przecinka przed „ponieważ”@ ponieważ", + ] { + assert!(m.contains(&want.to_string()), "{want}: {m:?}"); + } + assert_eq!(check("Wiem że tak.")[0].suggestions, [", że"]); + // Compound conjunctions, particles, conjunctions and prepositions take no comma there. + clean("Przyszedł, mimo że padało, chyba że nie."); + clean("Zrobię to, tylko jeśli zdążę, i to tak aby nikt nie widział."); + clean("To dom, w którym mieszkam, i ogród, przez który idę."); + clean("Wiem, że przyjdzie, i że zostanie."); + clean("Myślę, że gdyby przyszedł, byłoby dobrze."); + clean("Zachowywał się, jak gdyby nigdy nic."); + clean("Który z nich? To który wybierasz?"); + clean("Po raz nie wiadomo który."); + clean("Dlatego że padało, zostaliśmy."); + } + + #[test] + fn we_and_ze() { + let m = messages("Spotkamy się w wtorek w Wrocławiu z szkoły z mną."); + assert_eq!(m.iter().filter(|x| x.starts_with("Przed tym słowem")).count(), 4, "{m:?}"); + let v = check("Z szkoły wracam."); + assert_eq!(v[0].suggestions, ["Ze"]); + clean("Byłem w wodzie, w wieku dziesięciu lat, z szafą, z siostrą i z zamkiem."); + clean("Wracam ze szkoły we wtorek ze mną."); + } + + #[test] + fn correct_polish_text_is_clean() { + clean( + "Zażółć gęślą jaźń. Chrząszcz brzmi w trzcinie, a źdźbło trawy się kołysze. \ + Przyjdę jutro, chyba że będzie padać. Książka, którą czytam, jest ciekawa.", + ); + clean("Pięćdziesięciu uczniów przyszło, ponieważ ogłoszono konkurs."); + } + + proptest::proptest! { + #[test] + fn never_panics(s in "\\PC{0,80}") { + let _ = check(&s); + } + + #[test] + fn never_panics_on_polish_like_text(s in "[a-ząćęłńóśźżA-Z ,.;:!?„”…\\-]{0,60}") { + for i in check(&s) { + proptest::prop_assert!(i.start <= i.end && i.end <= s.len()); + proptest::prop_assert!(s.is_char_boundary(i.start) && s.is_char_boundary(i.end)); + } + } + } +} diff --git a/crates/proof/src/pl/mod.rs b/crates/proof/src/pl/mod.rs new file mode 100644 index 00000000..7cde9d0f --- /dev/null +++ b/crates/proof/src/pl/mod.rs @@ -0,0 +1,193 @@ +//! Polish proofing. +//! +//! - **Spelling** against the SJP.PL Hunspell dictionary installed on the system +//! ([`dictionary`], found by [`crate::hunspell`]: `pl_PL.aff`/`pl_PL.dic` from the OS, a +//! package manager or LibreOffice; it isn't bundled). Without it Polish text gets no spelling +//! marks. Capitalized stems are names (`Warszawa`); a sentence may start with any word and +//! headings may be in capitals. Hyphenated compounds are checked part by part +//! (`biało-czerwony`), foreign names inflected with an apostrophe are accepted +//! (`Kennedy'ego`), and the user dictionary applies as in English. +//! - **Suggestions** tuned for Polish ([`suggest()`]): missing diacritics first (`zolw` → `żółw`), then +//! the common spelling confusions (`ż`/`rz`, `u`/`ó`, `h`/`ch`…), a missing space (`napewno` → +//! `na pewno`), one-letter typos, and a dictionary scan for anything further off. +//! - **Grammar** ([`grammar`]): cheap, reliable rules with Polish messages. + +pub mod grammar; +mod suggest; +#[cfg(test)] +pub(crate) mod test_dict; + +use std::sync::Arc; + +use crate::affix::AffixDict; +use crate::slot::DictSlot; + +pub use suggest::suggest; + +/// File names of a Polish Hunspell dictionary. +const NAMES: &[&str] = &["pl_PL", "pl-PL", "pl"]; + +/// The Polish dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Polish"); + +/// The Polish dictionary: the installed Hunspell `pl_PL` (read on first use), one given with +/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off +/// then; see [`has_dictionary`]). +pub fn dictionary() -> Arc { + DICT.get() +} + +/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to +/// UTF-8), for example one the user picked. On error the current dictionary stays. +pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { + DICT.install(aff, dic) +} + +/// Is a Polish dictionary available (spelling is checked)? +pub fn has_dictionary() -> bool { + DICT.has_dictionary() +} + +/// Endings Polish adds after an apostrophe to foreign names whose last letter isn't pronounced +/// (`Kennedy'ego`, `Wilde'a`, `Shakespeare'owi`). +const APOSTROPHE_ENDINGS: &[&str] = &["a", "ach", "ami", "e", "ego", "em", "emu", "i", "m", "om", "owi", "owie", "u", "y", "ów", "ą", "ę"]; + +/// Is `word` spelled correctly in Polish? Numbers, single letters, short words in capitals +/// (acronyms: `PKO`, `ZUS`), words with digits and words in other scripts are accepted, and so +/// is every word when no Polish dictionary is installed. +pub fn is_correct(word: &str) -> bool { + is_correct_with(&dictionary(), word) +} + +/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off, +/// short words in capitals are checked against the dictionary too, and with Ignore words that +/// contain numbers off, so are words with digits. +pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool { + check(&dictionary(), word, opts) +} + +/// [`is_correct`] against a given dictionary (an empty one accepts everything). +pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { + check(d, word, &crate::Options::default()) +} + +/// [`is_correct_with_options`] against a given dictionary. +fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { + return true; + }; + let w = w.replace('’', "'"); + if w.contains('-') { + return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + } + known(d, &w) +} + +/// Is one word (no hyphens) in the dictionary, the user dictionary, or a foreign name with a +/// Polish ending after an apostrophe? +fn known(d: &AffixDict, w: &str) -> bool { + if d.check(w) || crate::in_user_dictionary(&w.to_lowercase()) { + return true; + } + if let Some((stem, ending)) = w.split_once('\'') { + return stem.chars().next().is_some_and(char::is_uppercase) && stem.chars().count() >= 2 && APOSTROPHE_ENDINGS.contains(&ending); + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_dictionary_loads() { + test_dict::install(); + let d = dictionary(); + assert!(has_dictionary()); + assert!(d.len() > 50, "{}", d.len()); + assert_eq!(d.rule_count(), 5, "the test dictionary's affix rules"); + assert!(!d.rep().is_empty() && !d.map().is_empty() && !d.try_chars().is_empty()); + } + + #[test] + fn without_a_dictionary_nothing_is_flagged_and_nothing_suggested() { + let none = AffixDict::default(); + for w in ["żeka", "kturzy", "xqzv", ""] { + assert!(is_correct_with(&none, w), "{w}"); + } + assert!(suggest::suggest_in(&none, "żeka", 5).is_empty()); + assert!(install_dictionary("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused"); + } + + #[test] + fn polish_words_are_correct() { + test_dict::install(); + for w in [ + "źdźbło", + "gżegżółka", + "chrząszcz", + "pięćdziesięciu", + "rzeka", + "którzy", + "żółw", + "Zażółć", + "gęślą", + "jaźń", + "konstytucja", + "konstytucyjnego", + "niebieskiego", + "nieładnie", + "przyszliśmy", + "zrobilibyśmy", + "Warszawa", + "Warszawie", + "WARSZAWA", + "Kot", + "biało-czerwony", + "polsko-niemieckiej", + "Kennedy'ego", + "Shakespeare’a", + "PKO", + "2026", + "e-mail", + "w", + "się", + ] { + assert!(is_correct(w), "{w}"); + } + } + + #[test] + fn misspellings_are_flagged() { + test_dict::install(); + for w in ["żeka", "kturzy", "napewno", "wogóle", "zolw", "chuśtawka", "bżoza", "warszawa", "niewiem", "pszyjaciel", "Kennedy'xyz"] { + assert!(!is_correct(w), "{w}"); + } + } + + #[test] + fn user_dictionary_applies() { + test_dict::install(); + assert!(!is_correct("zxqpolskisłowo")); + crate::add_word("Zxqpolskisłowo"); + assert!(is_correct("zxqpolskisłowo")); + } + + /// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Polish checks + /// those words against the dictionary instead of accepting them. + #[test] + fn proofing_options_reach_the_polish_check() { + test_dict::install(); + let lang = crate::ProofLang::Pl; + let defaults = crate::Options::default(); + let check_caps = crate::Options { ignore_uppercase: false, ..defaults }; + let check_digits = crate::Options { ignore_numbers: false, ..defaults }; + assert!(crate::is_correct_in_with("ŻEKA", lang, &defaults), "short capitals are acronyms by default"); + assert!(!crate::is_correct_in_with("ŻEKA", lang, &check_caps)); + assert!(crate::is_correct_in_with("RZEKA", lang, &check_caps)); + assert!(crate::is_correct_in_with("żeka2", lang, &defaults)); + assert!(!crate::is_correct_in_with("żeka2", lang, &check_digits)); + assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); + assert!(is_correct("ŻEKA"), "direct checks keep the defaults"); + } +} diff --git a/crates/proof/src/pl/suggest.rs b/crates/proof/src/pl/suggest.rs new file mode 100644 index 00000000..7baeaa54 --- /dev/null +++ b/crates/proof/src/pl/suggest.rs @@ -0,0 +1,132 @@ +//! Polish spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the +//! words Polish runs into their neighbours (`napewno` → `na pewno`, `kochamcię` → `kocham cię`), +//! letters to try for a dictionary without a TRY line, and final devoicing (`chlep` → `chleb`). + +use crate::affix::AffixDict; +use crate::affix_suggest::{COST_ADD_DIACRITIC, COST_MISTYPE, Split, Tuning}; + +/// Short words often run into the next one (`na pewno`, `w ogóle`, `nie wiem`, `przede +/// wszystkim`). +const LEADING_WORDS: &[&str] = &[ + "a", "i", "o", "u", "w", "z", "we", "ze", "na", "po", "do", "od", "za", "co", "to", "nie", "jak", "tak", "przy", "bez", "pod", "nad", "dla", + "że", "by", "czy", "już", "ani", "przed", "przez", "przede", "ku", "aż", "bo", "ten", "ta", "te", "tym", "tej", "też", "jest", "ode", "beze", + "nade", +]; +/// Short words often run into the one before (`kocham cię`, `daj mi`, `bałem się`). +const TRAILING_WORDS: &[&str] = + &["się", "cię", "mi", "mu", "ci", "go", "ją", "je", "nam", "wam", "im", "by", "że", "no", "mnie", "tobie", "jej", "ich"]; + +/// What Polish adds to its dictionary's tables. +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx", + mistype_cost, + ..Tuning::DEFAULT +}; + +/// Final devoicing makes a voiced consonant at the end of a word sound voiceless, so writing it +/// voiceless (`chlep` for `chleb`) is a likelier slip than any other typo. +fn mistype_cost(chars: &[char], i: usize, c: char) -> u32 { + let last = i + 1 == chars.len(); + if last && matches!((chars.get(i), c), (Some('p'), 'b') | (Some('t'), 'd') | (Some('k'), 'g')) { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE } +} + +/// Spelling suggestions for a Polish word, best first, at most `max`, in the word's letter case. +pub fn suggest(word: &str, max: usize) -> Vec { + suggest_in(&super::dictionary(), word, max) +} + +/// [`suggest`] against a given dictionary. +pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { + crate::affix_suggest::suggest_in(d, word, max, &TUNING) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::affix_suggest::{fold, weighted_distance}; + + fn has(word: &str, want: &str) { + let s = suggest(word, 8); + assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); + } + + #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); + } + + #[test] + fn polish_misspellings_get_the_right_suggestion() { + crate::pl::test_dict::install(); + has("żeka", "rzeka"); + has("kturzy", "którzy"); + has("napewno", "na pewno"); + has("wogóle", "w ogóle"); + has("niewiem", "nie wiem"); + has("zolw", "żółw"); + has("gzegzolka", "gżegżółka"); + has("chuśtawka", "huśtawka"); + has("bżoza", "brzoza"); + has("pszyjaciel", "przyjaciel"); + has("warszawa", "Warszawa"); + has("Żeka", "Rzeka"); + has("ZOLW", "ŻÓŁW"); + has("konstytucija", "konstytucja"); + has("wziąść", "wziąć"); + has("dziewczyan", "dziewczyna"); + } + + #[test] + fn voiced_consonant_suggestion_ranks_first() { + crate::pl::test_dict::install(); + let result = suggest("chlep", 6); + assert_eq!(result.first().map(String::as_str), Some("chleb"), "{result:?}"); + } + + #[test] + fn the_likeliest_fix_comes_first() { + crate::pl::test_dict::install(); + assert_eq!(suggest("kturzy", 3).first().map(String::as_str), Some("którzy")); + assert_eq!(suggest("zolw", 3).first().map(String::as_str), Some("żółw")); + assert_eq!(suggest("napewno", 3).first().map(String::as_str), Some("na pewno")); + assert_eq!(suggest("żeka", 3).first().map(String::as_str), Some("rzeka")); + } + + #[test] + fn edge_cases_stay_empty_or_bounded() { + crate::pl::test_dict::install(); + assert!(suggest("", 5).is_empty()); + assert!(suggest("kot", 0).is_empty()); + assert!(suggest(&"a".repeat(41), 5).is_empty()); + assert!(suggest("dwa słowa", 5).is_empty()); + assert!(suggest("żółw", 5).len() <= 5); + assert!(!suggest("żółw", 5).contains(&"żółw".to_string())); + } + + #[test] + fn distance_weighs_diacritics_half() { + let map = vec![vec!['a', 'ą'], vec!['o', 'ó', 'u']]; + let related = |a: char, b: char| fold(a, &map) == fold(b, &map); + let c = |s: &str| s.chars().collect::>(); + assert_eq!(weighted_distance(&c("ktury"), &c("który"), &related, 9), 1); + assert_eq!(weighted_distance(&c("kot"), &c("kto"), &related, 9), 2); + assert_eq!(weighted_distance(&c("kot"), &c("kotek"), &related, 9), 4); + assert_eq!(weighted_distance(&c("kot"), &c("pies"), &related, 2), 3); + assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4); + } + + #[test] + fn suggestions_are_fast_enough() { + crate::pl::test_dict::install(); + // Warm the dictionary, then time a worst case (a long word nothing near): the scan round. + let _ = suggest("kot", 1); + let t = std::time::Instant::now(); + for w in ["xqzvwpolskiegoslowa", "pszyjacielowi", "zażółćgęśląjaźń"] { + let _ = suggest(w, 6); + } + let ms = t.elapsed().as_secs_f64() * 1000.0; + eprintln!("3 suggestion calls: {ms:.0} ms"); + assert!(ms < 20_000.0, "{ms} ms"); + } +} diff --git a/crates/proof/src/pl/test_dict.rs b/crates/proof/src/pl/test_dict.rs new file mode 100644 index 00000000..80c1aa24 --- /dev/null +++ b/crates/proof/src/pl/test_dict.rs @@ -0,0 +1,114 @@ +//! A tiny synthetic Polish Hunspell dictionary for tests (written for these tests; not taken from +//! any real dictionary). Real dictionaries are installed on the system, never bundled. + +/// Affix rules: a few suffixes and one prefix, plus the suggestion tables the Polish +/// suggestions use (`TRY`, `MAP` for diacritics, `REP` for common confusions). +pub const AFF: &str = "SET UTF-8 +TRY aioeznrwcysżtkpdmłjlubąęgchśóźćń +MAP 9 +MAP aą +MAP cć +MAP eę +MAP lł +MAP nń +MAP oó +MAP uó +MAP sś +MAP zżź +REP 6 +REP ż rz +REP rz ż +REP ch h +REP h ch +REP sz rz +REP ść ć +SFX A Y 1 +SFX A a ie a +SFX B Y 3 +SFX B y ego y +SFX B i iego i +SFX B i iej i +PFX N Y 1 +PFX N 0 nie . +"; + +/// Stems (the count line first, as in Hunspell files). +pub const DIC: &str = "69 +źdźbło +gżegżółka +chrząszcz +chleb +chłep +chlap +chlip +chlup +chlej +pięćdziesięciu +rzeka +którzy +żółw +zażółć +gęślą +jaźń +konstytucja +konstytucyjny/B +niebieski/B +niemiecki/B +ładnie/N +przyszliśmy +zrobilibyśmy +Warszawa/A +kot +biało +czerwony +polsko +mail +w +się +huśtawka +brzoza +przyjaciel +wziąć +dziewczyna +na +pewno +ogóle +nie +wiem +płynie +a +i +o +to +że +tak +jest +brzmi +trzcina/A +napisał +poszedł +spać +bo +wylała +świat +dom +ma +ala +kota +dzień +dobry +jutro +bardzo +dobrze +mamy +czas +ten +"; + +/// Make [`AFF`]/[`DIC`] the Polish dictionary (every test that needs one calls this; they all +/// install the same one, so tests running in parallel agree). +pub fn install() { + if let Err(e) = super::install_dictionary(AFF, DIC) { + panic!("test dictionary: {e}"); + } +} diff --git a/crates/proof/src/pt/grammar.rs b/crates/proof/src/pt/grammar.rs new file mode 100644 index 00000000..6b72e322 --- /dev/null +++ b/crates/proof/src/pt/grammar.rs @@ -0,0 +1,251 @@ +//! Portuguese grammar and punctuation checks: cheap, rule-based and tuned to stay quiet when +//! unsure (a missed mistake costs less than a wrong mark on correct text). Messages are in +//! Portuguese. +//! +//! - **Repeated word** (`ele ele`), except interjections and words Portuguese doubles on purpose +//! (`sim sim`, `não não`, `ora ora`). +//! - **Space before punctuation** (`bem ,`) and **double spaces**; emoticons (` :)`) and +//! ellipses are left alone. +//! - **Capital letter at the start of a sentence**, knowing Portuguese abbreviations (`etc.`, +//! `pág.`, `Prof.`, `Dr.`), initials, ordinal numbers (`5. colocado`), ellipses and dialogue +//! dashes (`— Vai — disse`), after which lower case is correct. +//! - **Doubled punctuation** (`,,`, `..`, `;;`). +//! +//! No comma rule (`que` opens clauses only sometimes) and no English rule (such as `a`/`an`) +//! applies to Portuguese text. + +use crate::{Issue, IssueKind, ProofLang, words}; + +/// Words Portuguese repeats on purpose: interjections, onomatopoeia and emphatic doubling. +const ALLOWED_REPEATS: &[&str] = &[ + "sim", "não", "ora", "olha", "já", "bem", "muito", "vai", "lá", "cá", "aí", "ha", "he", "hi", "oh", "ah", "ai", "ui", "uf", "uxa", "oba", "viva", + "psiu", "olá", "tchau", "xau", "adeus", "devagar", +]; + +/// Abbreviations written with a final period after which a sentence goes on in lower case +/// (lower case, without the period). +const ABBREVIATIONS: &[&str] = &[ + "etc", "ex", "pág", "págs", "séc", "sécs", "fl", "av", "vol", "p", "pp", "ed", "tel", "min", "max", "art", "par", "prof", "profa", "dr", "dra", + "sr", "sra", "srta", "eng", "arq", "op", "cit", "cf", "ibid", "id", "vide", "seg", "ter", "qua", "qui", "sex", "sáb", "dom", "jan", "fev", "mar", + "abr", "mai", "jun", "jul", "ago", "set", "out", "nov", "dez", "s.a", "d.c", "a.c", "pg", "ref", "cod", "dept", "univ", "inst", "lat", "ing", + "esp", "fr", "it", "al", "obs", "att", "obs", +]; + +fn issue(start: usize, end: usize, message: String, suggestions: Vec) -> Issue { + Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::Pt } +} + +fn capitalize(w: &str) -> String { + crate::affix::capitalize(w) +} + +/// Is `prev`, before a period, an abbreviation, an initial, a number or a Roman numeral (after +/// which a lower-case word doesn't start a sentence)? +fn abbreviation_or_number(prev: &str) -> bool { + prev.chars().count() <= 1 + || prev.chars().any(|c| c.is_ascii_digit()) + || (prev.chars().all(|c| matches!(c, 'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M'))) + || ABBREVIATIONS.contains(&prev.to_lowercase().as_str()) +} + +const CLOSING: [char; 8] = ['"', '”', '’', '\'', ')', '»', ']', '\u{1}']; +const OPENING: [char; 8] = ['"', '“', '«', '(', '[', '\'', '‚', '—']; + +/// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after +/// an abbreviation), then space, with nothing but quotes and brackets around? +fn ends_sentence(gap: &str, prev: &str) -> bool { + let t = gap.trim_start_matches(CLOSING); + let Some(end) = t.chars().next() else { return false }; + if !matches!(end, '.' | '!' | '?') || t.starts_with("..") { + return false; + } + if end == '.' && abbreviation_or_number(prev) { + return false; + } + let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING); + rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| OPENING.contains(&c)) +} + +/// Grammar issues in a paragraph of Portuguese text, in order. +pub fn check(text: &str) -> Vec { + check_with(text, true) +} + +/// [`check`] where `capital_first` says whether the text's first word must start a sentence with a capital +/// (false for a list item, which may be a fragment). +pub fn check_with(text: &str, capital_first: bool) -> Vec { + let ws: Vec<(usize, usize)> = words(text); + let mut v = Vec::new(); + repeated_words(text, &ws, &mut v); + spaces(text, &mut v); + sentence_capitals(text, &ws, capital_first, &mut v); + doubled_punctuation(text, &mut v); + v.sort_by_key(|i| (i.start, i.end)); + v +} + +fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if !gap.is_empty() + && gap.chars().all(char::is_whitespace) + && w0.chars().all(char::is_alphabetic) + && w0.to_lowercase() == w1.to_lowercase() + && !ALLOWED_REPEATS.contains(&w0.to_lowercase().as_str()) + { + v.push(issue(*a0, *b1, format!("Palavra repetida: \"{w1}\""), vec![w0.to_string()])); + } + } +} + +fn spaces(text: &str, v: &mut Vec) { + for (i, c) in text.char_indices() { + if c != ' ' { + continue; + } + let rest = text.get(i + 1..).unwrap_or(""); + let mut it = rest.chars(); + let Some(p) = it.next() else { continue }; + let after = it.next(); + let flag = match p { + ',' => true, + // `;)` and `;-)` are emoticons. + ';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')), + // ` ...` is an ellipsis, ` .5` a number. + '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), + // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. + ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')), + _ => false, + }; + if flag { + v.push(issue(i, i + 1 + p.len_utf8(), "Espaço antes da pontuação".into(), vec![p.to_string()])); + } + } + for (i, _) in text.match_indices(" ") { + if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { + continue; + } + v.push(issue(i, i + 2, "Espaço duplo".into(), vec![" ".into()])); + } +} + +fn sentence_capitals(text: &str, ws: &[(usize, usize)], capital_first: bool, v: &mut Vec) { + // List items (`– maçãs,`) may start in lower case. + let list_item = text.trim_end().ends_with([',', ';']); + for (k, &(a, b)) in ws.iter().enumerate() { + let Some(w) = text.get(a..b) else { continue }; + let starts = if k == 0 { + let before = text.get(..a).unwrap_or(""); + capital_first && !list_item && before.chars().all(|c| c.is_whitespace() || OPENING.contains(&c)) + } else { + let Some(&(pa, pb)) = ws.get(k - 1) else { continue }; + ends_sentence(text.get(pb..a).unwrap_or(""), text.get(pa..pb).unwrap_or("")) + }; + if starts && w.chars().next().is_some_and(char::is_lowercase) && w.chars().all(char::is_alphabetic) { + v.push(issue(a, b, "Letra maiúscula no início da frase".into(), vec![capitalize(w)])); + } + } +} + +fn doubled_punctuation(text: &str, v: &mut Vec) { + let bytes = text.as_bytes(); + let mut i = 0; + while let Some(&b) = bytes.get(i) { + if !matches!(b, b',' | b'.' | b';') { + i += 1; + continue; + } + let run = bytes.get(i..).map_or(0, |r| r.iter().take_while(|&&x| x == b).count()); + let flag = match b { + b'.' => run == 2, + _ => run >= 2, + }; + if flag { + let end = i + run; + let suggestions: Vec = match b { + b'.' => vec![".".into(), "…".into()], + _ => vec![char::from(b).to_string()], + }; + v.push(issue(i, end, "Sinal repetido".into(), suggestions)); + } + i += run.max(1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn messages(t: &str) -> Vec { + check(t).into_iter().map(|i| format!("{}@{}", i.message, &t[i.start..i.end])).collect() + } + + fn clean(t: &str) { + let m = messages(t); + assert!(m.is_empty(), "{t:?}: {m:?}"); + } + + #[test] + fn repeated_words() { + let v = check("Eu eu vi isso e ele ele saiu."); + assert_eq!(v.len(), 2, "{v:?}"); + assert_eq!(v[0].message, "Palavra repetida: \"eu\""); + assert_eq!(v[0].suggestions, ["Eu"]); + assert!(v.iter().all(|i| i.lang == ProofLang::Pt && i.kind == IssueKind::Grammar)); + clean("Sim, sim! Não, não! Ora ora, que surpresa."); + } + + #[test] + fn spaces_and_punctuation() { + let m = messages("Olá , mundo . Tudo bem ?"); + assert!(m.contains(&"Espaço antes da pontuação@ ,".to_string()), "{m:?}"); + assert!(m.contains(&"Espaço antes da pontuação@ .".to_string()), "{m:?}"); + assert!(m.contains(&"Espaço antes da pontuação@ ?".to_string()), "{m:?}"); + assert!(m.contains(&"Espaço duplo@ ".to_string()), "{m:?}"); + clean("Legal :) Muito bom :-) Ou :D Valor .5 e reticências ..."); + let m = messages("Falou,, e saiu.. depois;; fim."); + assert_eq!(m.iter().filter(|x| x.starts_with("Sinal repetido")).count(), 3, "{m:?}"); + clean("Esperei… e esperei... até ele voltar."); + } + + #[test] + fn sentence_start_capitals() { + let m = messages("isso é um teste. a segunda frase! terceira? ok."); + assert_eq!(m.iter().filter(|x| x.starts_with("Letra maiúscula")).count(), 4, "{m:?}"); + assert_eq!(check("ele tem razão.")[0].suggestions, ["Ele"]); + // Abbreviations, initials, ordinals, ellipses and dialogue dashes. + clean("Comprei frutas, etc. e depois saímos."); + clean("Conheci o Dr. Silva na Av. Paulista."); + clean("Zhou ficou em 5. lugar no séc. XX."); + clean("Esperei... e ele apareceu."); + clean("— Vai — disse ele. — Amanhã eu volto."); + clean("Ele disse: \"Volto amanhã\". E saiu."); + // A list item may start in lower case. + clean("maçãs,"); + } + + #[test] + fn correct_portuguese_text_is_clean() { + clean( + "O rato roeu a roupa do rei de Roma. Quem não tem cão caça com gato, \ + e quem tem paciência colhe o que plantou.", + ); + clean("Brasília, 12 de outubro de 2026 — uma data memorável."); + } + + proptest::proptest! { + #[test] + fn never_panics(s in "\\PC{0,80}") { + let _ = check(&s); + } + + #[test] + fn never_panics_on_portuguese_like_text(s in "[a-záàâãéêíóôõúüçA-Z ,.;:!?\"…\\-—]{0,60}") { + for i in check(&s) { + proptest::prop_assert!(i.start <= i.end && i.end <= s.len()); + proptest::prop_assert!(s.is_char_boundary(i.start) && s.is_char_boundary(i.end)); + } + } + } +} diff --git a/crates/proof/src/pt/mod.rs b/crates/proof/src/pt/mod.rs new file mode 100644 index 00000000..ea91dbdb --- /dev/null +++ b/crates/proof/src/pt/mod.rs @@ -0,0 +1,201 @@ +//! Portuguese proofing. +//! +//! - **Spelling** against the Hunspell `pt_BR` dictionary installed on the system +//! ([`dictionary`], found by [`crate::hunspell`]: `pt_BR.aff`/`pt_BR.dic` — VERO and its +//! derivatives — from the OS, a package manager or LibreOffice; it isn't bundled). Without it +//! Portuguese text gets no spelling marks. Capitalized stems are names (`Brasília`); a sentence +//! may start with any word and headings may be in capitals. Hyphenated compounds are checked +//! part by part (`guarda-roupa`, `ex-marido`), and the user dictionary applies as in English. +//! - **Suggestions** tuned for Portuguese ([`suggest`]): missing accents first (`coraçao` → +//! `coração`, `saude` → `saúde`), then the dictionary's `REP` confusions (`seje` → `seja`), a +//! missing space, one-letter typos, and a dictionary scan for anything further off. +//! - **Grammar** ([`grammar`]): cheap, reliable rules with Portuguese messages. + +pub mod grammar; +mod suggest; +#[cfg(test)] +pub(crate) mod test_dict; + +use std::sync::Arc; + +use crate::affix::AffixDict; +use crate::slot::DictSlot; + +pub use suggest::suggest; + +/// File names of a Portuguese Hunspell dictionary. +const NAMES: &[&str] = &["pt_BR", "pt-BR", "pt"]; + +/// The Portuguese dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Portuguese"); + +/// The Portuguese dictionary: the installed Hunspell `pt_BR` (read on first use), one given with +/// [`install_dictionary`], or an empty one when none is available (Portuguese spelling is off +/// then; see [`has_dictionary`]). +pub fn dictionary() -> Arc { + DICT.get() +} + +/// Use this Portuguese dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to +/// UTF-8), for example one the user picked. On error the current dictionary stays. +pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { + DICT.install(aff, dic) +} + +/// Is a Portuguese dictionary available (spelling is checked)? +pub fn has_dictionary() -> bool { + DICT.has_dictionary() +} + +/// Is `word` spelled correctly in Portuguese? Numbers, single letters, short words in capitals +/// (acronyms: `USP`, `PT`), words with digits and words in other scripts are accepted, and so +/// is every word when no Portuguese dictionary is installed. +pub fn is_correct(word: &str) -> bool { + is_correct_with(&dictionary(), word) +} + +/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off, +/// short words in capitals are checked against the dictionary too, and with Ignore words that +/// contain numbers off, so are words with digits. +pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool { + check(&dictionary(), word, opts) +} + +/// [`is_correct`] against a given dictionary (an empty one accepts everything). +pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { + check(d, word, &crate::Options::default()) +} + +/// [`is_correct_with_options`] against a given dictionary. +fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { + return true; + }; + if w.contains('-') { + return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + } + known(d, w) +} + +/// Is one word (no hyphens) in the dictionary or the user dictionary? +fn known(d: &AffixDict, w: &str) -> bool { + d.check(w) || crate::in_user_dictionary(&w.to_lowercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_dictionary_loads() { + test_dict::install(); + let d = dictionary(); + assert!(has_dictionary()); + assert!(d.len() > 50, "{}", d.len()); + assert!(d.rule_count() >= 5, "the test dictionary's affix rules"); + assert!(!d.rep().is_empty() && !d.map().is_empty() && !d.try_chars().is_empty()); + } + + #[test] + fn without_a_dictionary_nothing_is_flagged() { + let none = AffixDict::default(); + for w in ["coraçao", "saude", "xqzv", ""] { + assert!(is_correct_with(&none, w), "{w}"); + } + assert!(install_dictionary("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused"); + } + + #[test] + fn portuguese_words_are_correct() { + test_dict::install(); + for w in [ + "coração", + "saudade", + "pássaro", + "saúde", + "árvore", + "escrever", + "computador", + "praia", + "conhecer", + "Brasília", + "São", + "autoestima", + "frequência", + "exceção", + "amanhã", + "vocês", + "coragem", + "parabéns", + "cantava", + "cantando", + "provável", + "prováveis", + "descobrir", + "trabalho", + "CORAGEM", + "USP", + "2026", + "e-mail", + "guarda-roupa", + "ex-marido", + "vice-presidente", + "e", + "não", + ] { + assert!(is_correct(w), "{w}"); + } + } + + #[test] + fn misspellings_are_flagged() { + test_dict::install(); + for w in ["coraçao", "saude", "trbalho", "excrever", "prsidente", "frequencia", "amnha", "cantaando"] { + assert!(!is_correct(w), "{w}"); + } + } + + #[test] + fn forbidden_words_are_rejected() { + test_dict::install(); + // `frequencia/F` is listed with FORBIDDENWORD: the old spelling, never correct. + assert!(!is_correct("frequencia") && !is_correct("Frequencia") && !is_correct("FREQUENCIA")); + assert!(is_correct("frequência")); + } + + #[test] + fn user_dictionary_applies() { + test_dict::install(); + assert!(!is_correct("zxqportuguespalavra")); + crate::add_word("Zxqportuguespalavra"); + assert!(is_correct("zxqportuguespalavra")); + } + + #[test] + fn user_dictionary_accepts_a_hyphenated_word_whole() { + test_dict::install(); + let text = "Ele disse que está na Unigranrio-RJ."; + assert_eq!(crate::check_spelling_in(text, crate::ProofLang::Pt).len(), 1, "Unigranrio isn't in the dictionary"); + // Added as the context menu adds it, hyphen and all (#2082). + crate::add_word("Unigranrio-RJ"); + assert!(crate::check_spelling_in(text, crate::ProofLang::Pt).is_empty()); + } + + /// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Portuguese checks + /// those words against the dictionary instead of accepting them. + #[test] + fn proofing_options_reach_the_portuguese_check() { + test_dict::install(); + let lang = crate::ProofLang::Pt; + let defaults = crate::Options::default(); + let check_caps = crate::Options { ignore_uppercase: false, ..defaults }; + let check_digits = crate::Options { ignore_numbers: false, ..defaults }; + assert!(crate::is_correct_in_with("PRAYA", lang, &defaults), "short capitals are acronyms by default"); + assert!(!crate::is_correct_in_with("PRAYA", lang, &check_caps)); + assert!(crate::is_correct_in_with("PRAIA", lang, &check_caps)); + assert!(crate::is_correct_in_with("praya2", lang, &defaults)); + assert!(!crate::is_correct_in_with("praya2", lang, &check_digits)); + assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); + assert!(is_correct("PRAYA"), "direct checks keep the defaults"); + } +} diff --git a/crates/proof/src/pt/suggest.rs b/crates/proof/src/pt/suggest.rs new file mode 100644 index 00000000..f8f33bff --- /dev/null +++ b/crates/proof/src/pt/suggest.rs @@ -0,0 +1,122 @@ +//! Portuguese spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with +//! the words Portuguese runs into their neighbours (`nãoé` → `não é`, `paraela` → `para ela`) and +//! letters to try for a dictionary without a TRY line. + +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; + +/// Short words often run into the next one (`não é`, `da água`, `para isso`). +const LEADING_WORDS: &[&str] = &[ + "a", "o", "e", "é", "de", "do", "da", "dos", "das", "em", "no", "na", "nos", "nas", "por", "para", "com", "sem", "sob", "sobre", "ao", "aos", + "à", "às", "que", "se", "não", "como", "mas", "mais", "já", "até", "entre", "contra", "perante", "desde", "após", "ante", "durante", "mediante", + "segundo", "conforme", "salvo", "fora", "onde", "quando", "bem", "mal", "pois", "logo", "portanto", "porém", "todavia", "contudo", "assim", + "tão", "tanto", "tudo", "nada", "algo", "cada", "outro", "mesmo", "próprio", "ainda", "tampouco", "quase", "talvez", "nunca", "sempre", +]; +/// Short words often run into the one before (`amo-te`, `deu-lhe`; in writing without the +/// hyphen: `amote`, `deulhe`). +const TRAILING_WORDS: &[&str] = &[ + "me", "te", "se", "lhe", "nos", "vos", "lhes", "o", "a", "os", "as", "ele", "ela", "eles", "elas", "isso", "isto", "aquilo", "mim", "ti", "si", + "eu", "tu", "que", "de", "em", "é", +]; + +/// What Portuguese adds to its dictionary's tables. +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw", + ..Tuning::DEFAULT +}; + +/// Spelling suggestions for a Portuguese word, best first, at most `max`, in the word's letter case. +pub fn suggest(word: &str, max: usize) -> Vec { + suggest_in(&super::dictionary(), word, max) +} + +/// [`suggest`] against a given dictionary. +pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { + crate::affix_suggest::suggest_in(d, word, max, &TUNING) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::affix_suggest::{fold, weighted_distance}; + + fn has(word: &str, want: &str) { + let s = suggest(word, 8); + assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); + } + + #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); + } + + #[test] + fn portuguese_misspellings_get_the_right_suggestion() { + crate::pt::test_dict::install(); + has("coraçao", "coração"); + has("saude", "saúde"); + has("seje", "seja"); + has("excessão", "exceção"); + has("previlégio", "privilégio"); + has("prsidente", "presidente"); + has("provavel", "provável"); + has("trbalho", "trabalho"); + has("nãoé", "não é"); + has("paraela", "para ela"); + has("brasília", "Brasília"); + has("Coraçao", "Coração"); + } + + #[test] + fn the_likeliest_fix_comes_first() { + crate::pt::test_dict::install(); + assert_eq!(suggest("coraçao", 3).first().map(String::as_str), Some("coração")); + assert_eq!( + suggest("saude", 3).first().map(String::as_str), + Some("saúde"), + "an accent at the first difference beats one further right (saudê is a word too)" + ); + assert_eq!(suggest("seje", 3).first().map(String::as_str), Some("seja")); + } + + #[test] + fn edge_cases_stay_empty_or_bounded() { + crate::pt::test_dict::install(); + assert!(suggest("", 5).is_empty()); + assert!(suggest("bem", 0).is_empty()); + assert!(suggest(&"a".repeat(41), 5).is_empty()); + assert!(suggest("duas palavras", 5).is_empty()); + assert!(suggest_in(&crate::affix::AffixDict::default(), "coraçao", 5).is_empty()); + assert!(suggest("coração", 5).len() <= 5); + assert!(!suggest("coração", 5).contains(&"coração".to_string())); + } + + #[test] + fn distance_weighs_accents_half() { + let map = vec![vec!['a', 'á', 'à', 'â', 'ã'], vec!['c', 'ç']]; + let related = |a: char, b: char| fold(a, &map) == fold(b, &map); + let c = |s: &str| s.chars().collect::>(); + assert_eq!(weighted_distance(&c("coraçao"), &c("coração"), &related, 9), 1); + assert_eq!(weighted_distance(&c("coracao"), &c("coração"), &related, 9), 2, "ç and ã are two accent slips"); + assert_eq!(weighted_distance(&c("coro"), &c("coro"), &related, 9), 0); + assert_eq!(weighted_distance(&c("bem"), &c("bom"), &related, 9), 2); + assert_eq!(weighted_distance(&c("bem"), &c("bemte"), &related, 9), 4); + assert_eq!(weighted_distance(&c("bem"), &c("ver"), &related, 2), 3); + assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4); + } + + #[test] + fn suggestions_are_fast_enough() { + crate::pt::test_dict::install(); + // Warm the dictionary, then time a worst case (a long word nothing near): the scan round. + let _ = suggest("bem", 1); + let t = std::time::Instant::now(); + for w in ["xqzvweportuguespalavra", "prsidentes", "paralelepípedo"] { + let _ = suggest(w, 6); + } + let ms = t.elapsed().as_secs_f64() * 1000.0; + eprintln!("3 suggestion calls: {ms:.0} ms"); + assert!(ms < 20_000.0, "{ms} ms"); + } +} diff --git a/crates/proof/src/pt/test_dict.rs b/crates/proof/src/pt/test_dict.rs new file mode 100644 index 00000000..e02be3c1 --- /dev/null +++ b/crates/proof/src/pt/test_dict.rs @@ -0,0 +1,121 @@ +//! A tiny synthetic Portuguese Hunspell dictionary for tests (written for these tests; not taken +//! from any real dictionary). Real dictionaries are installed on the system, never bundled. +//! +//! It exercises what VERO-like dictionaries use: `FLAG UTF-8` (multi-byte flag characters — +//! `F` marks a forbidden spelling), continuation classes (`provável` → `prováveis`), the +//! suggestion tables (`TRY`, `MAP` for accents, `REP` for common confusions) and one prefix. + +pub const AFF: &str = "SET UTF-8 +FLAG UTF-8 +FORBIDDENWORD F +TRY aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw +MAP 6 +MAP aáàâã +MAP eéê +MAP ií +MAP oóôõ +MAP uú +MAP cç +REP 4 +REP seje seja +REP excessão exceção +REP previlégio privilégio +REP analize análise +SFX A Y 2 +SFX A a ava a +SFX A a ando a +SFX B N 1 +SFX B 0 ável/C . +SFX C N 1 +SFX C vel veis vel +PFX X Y 1 +PFX X 0 des . +"; + +/// Stems (the count line first, as in Hunspell files). +pub const DIC: &str = "76 +coração +saudade +saudê +pássaro +saúde +árvore +escrever +computador +praia +conhecer +Brasília +São +Paulo +guarda +roupa +mail +marido +ex +vice +presidente +autoestima +frequência +frequencia/F +exceção +privilégio +análise +seja +trabalho +canta/A +prov/B +cobrir/X +avião +pássaros +amanhã +vocês +coragem +parabéns +está +ele +disse +existem +bem +não +sim +que +com +para +dos +das +pela +pelo +muito +mais +como +porque +aqui +também +já +ser +ter +ir +ver +dar +e +o +é +água +ela +da +do +de +em +no +na +um +uma +"; + +/// Make [`AFF`]/[`DIC`] the Portuguese dictionary (every test that needs one calls this; they all +/// install the same one, so tests running in parallel agree). +pub fn install() { + if let Err(e) = super::install_dictionary(AFF, DIC) { + panic!("test dictionary: {e}"); + } +} diff --git a/crates/proof/src/ru/extra.rs b/crates/proof/src/ru/extra.rs new file mode 100644 index 00000000..d9c992ae --- /dev/null +++ b/crates/proof/src/ru/extra.rs @@ -0,0 +1,218 @@ +//! Our supplement of Russian words older Hunspell dictionaries lack (`маркетплейс`, `промокод`, +//! `интеграция`, `бонусный`…), from craft-i18n's `language-data/ru/ru-extra.txt`: each base form with its +//! declension, expanded here into every form. + +use std::collections::HashSet; +use std::sync::OnceLock; + +use crate::data::EXTRA_RU; + +/// Every form of the supplement: common words (lower case) and names (lower case too, but they +/// only match a word written with a capital). +struct Extra { + words: HashSet, + names: HashSet, +} + +fn extra() -> &'static Extra { + static E: OnceLock = OnceLock::new(); + E.get_or_init(|| { + let mut e = Extra { words: HashSet::new(), names: HashSet::new() }; + for line in EXTRA_RU.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')) { + let (base, class) = line.split_once('\t').map_or((line, ""), |(b, c)| (b.trim(), c.trim())); + let name = base.chars().next().is_some_and(char::is_uppercase); + let set = if name { &mut e.names } else { &mut e.words }; + for f in forms(base, class) { + set.insert(f.to_lowercase()); + } + } + e + }) +} + +/// Is `w` a form of a supplement word (a name only when written with a capital)? +pub(super) fn contains(w: &str) -> bool { + let e = extra(); + let lower = w.to_lowercase(); + e.words.contains(&lower) || (w.chars().next().is_some_and(char::is_uppercase) && e.names.contains(&lower)) +} + +fn is_velar(c: char) -> bool { + matches!(c, 'г' | 'к' | 'х') +} + +fn is_hushing(c: char) -> bool { + matches!(c, 'ж' | 'ш' | 'ч' | 'щ') +} + +/// The forms of `base` declined as `class` (see the supplement file's header). +fn forms(base: &str, class: &str) -> Vec { + let with = |stem: &str, ends: &[&str]| ends.iter().map(|e| format!("{stem}{e}")).collect::>(); + let last = |s: &str| s.chars().last().unwrap_or(' '); + if let Some(list) = class.strip_prefix('=') { + let mut v: Vec = list.split_whitespace().map(str::to_string).collect(); + v.push(base.to_string()); + return v; + } + match class { + "m" => { + let c = last(base); + let plural = if is_velar(c) || is_hushing(c) { "и" } else { "ы" }; + let gen_pl = if is_hushing(c) { "ей" } else { "ов" }; + let mut v = with(base, &["", "а", "у", "ом", "е", plural, gen_pl, "ам", "ами", "ах"]); + if is_hushing(c) || c == 'ц' { + v.push(format!("{base}ем")); + } + v + } + "f" => { + let Some(stem) = base.strip_suffix('а') else { return vec![base.to_string()] }; + let c = last(stem); + let plural = if is_velar(c) || is_hushing(c) { "и" } else { "ы" }; + let mut v = with(stem, &["а", plural, "е", "у", "ой", "ою", "", "ам", "ами", "ах"]); + if is_hushing(c) || c == 'ц' { + v.extend(with(stem, &["ей", "ею"])); + } + v + } + "fia" => { + base.strip_suffix("ия").map_or_else(|| vec![base.to_string()], |s| with(s, &["ия", "ии", "ию", "ией", "иею", "ий", "иям", "иями", "иях"])) + } + "fs" => base.strip_suffix('ь').map_or_else(|| vec![base.to_string()], |s| with(s, &["ь", "и", "ью", "ей", "ям", "ями", "ях"])), + "no" => base.strip_suffix('о').map_or_else(|| vec![base.to_string()], |s| with(s, &["о", "а", "у", "ом", "е", "", "ам", "ами", "ах"])), + "ovat" => { + // Verbs in -овать (`колеровать`, `администрировать`): the conjugation, the past, the + // imperative and the participles. + let Some(stem) = base.strip_suffix("овать") else { return vec![base.to_string()] }; + let mut v = with( + stem, + &[ + "овать", + "ую", + "уешь", + "ует", + "уем", + "уете", + "уют", + "овал", + "овала", + "овало", + "овали", + "уя", + "уй", + "уйте", + "уется", + "уются", + "уюсь", + "уешься", + "уемся", + "уетесь", + "овался", + "овалась", + "овалось", + "овались", + "оваться", + "овав", + "овавшись", + ], + ); + for participle in ["уемый", "ованный", "ующий", "овавший", "ующийся", "овавшийся"] { + let adj = format!("{stem}{participle}"); + if let Some(reflexive) = adj.strip_suffix("ся") { + v.extend(forms(reflexive, "adj").into_iter().map(|f| format!("{f}ся"))); + } else { + v.extend(forms(&adj, "adj")); + } + } + v.push(format!("{stem}ован")); + v.extend(with(stem, &["ована", "овано", "ованы"])); + v + } + "nie" => { + base.strip_suffix("ие").map_or_else(|| vec![base.to_string()], |s| with(s, &["ие", "ия", "ию", "ием", "ии", "ий", "иям", "иями", "иях"])) + } + "adj" => { + if let Some(stem) = base.strip_suffix("ый").or_else(|| base.strip_suffix("ой")) { + let mut v = with(stem, &["ого", "ому", "ым", "ом", "ая", "ой", "ую", "ое", "ые", "ых", "ыми"]); + v.push(base.to_string()); + return v; + } + let Some(stem) = base.strip_suffix("ий") else { return vec![base.to_string()] }; + let c = last(stem); + let ends: &[&str] = if is_velar(c) { + &["ий", "ого", "ому", "им", "ом", "ая", "ой", "ую", "ое", "ие", "их", "ими"] + } else if is_hushing(c) { + &["ий", "его", "ему", "им", "ем", "ая", "ей", "ую", "ее", "ие", "их", "ими"] + } else { + &["ий", "его", "ему", "им", "ем", "яя", "ей", "юю", "ее", "ие", "их", "ими"] + }; + with(stem, ends) + } + _ => vec![base.to_string()], + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn supplement_words_decline() { + for w in [ + "маркетплейс", + "маркетплейсов", + "маркетплейсах", + "Маркетплейсами", + "чекбоксы", + "промокода", + "кешбэки", + "инфографики", + "оплат", + "отгрузок", + "интеграций", + "соцсетей", + "согласий", + "бонусных", + "газоиспользующего", + "колеруется", + "заколерованный", + "онлайн", + "чат-ботом", + "Яндекса", + "Битриксе", + ] { + assert!(contains(w), "{w}"); + } + // Names need their capital; nonsense isn't in. + assert!(!contains("яндекс")); + assert!(!contains("маркетплейсыы")); + } + + #[test] + fn declension_classes() { + assert!(forms("бренд", "m").contains(&"брендов".to_string())); + assert!(forms("блог", "m").contains(&"блоги".to_string())); + assert!(forms("мерч", "m").contains(&"мерчей".to_string())); + assert!(forms("книга", "f").contains(&"книги".to_string())); + assert!(forms("синий", "adj").contains(&"синяя".to_string())); + assert!(forms("русский", "adj").contains(&"русская".to_string())); + assert!(forms("свежий", "adj").contains(&"свежая".to_string())); + assert_eq!(forms("медиа", ""), ["медиа"]); + let ovat = forms("модерировать", "ovat"); + for f in [ + "модерирую", + "модерирует", + "модерировали", + "модерируется", + "модерируемого", + "модерированной", + "модерирующая", + "модерирующемуся", + "модерирован", + ] { + assert!(ovat.contains(&f.to_string()), "{f}"); + } + assert!(forms("юрлицо", "no").contains(&"юрлиц".to_string())); + } +} diff --git a/crates/proof/src/ru/grammar.rs b/crates/proof/src/ru/grammar.rs new file mode 100644 index 00000000..70b9a1e6 --- /dev/null +++ b/crates/proof/src/ru/grammar.rs @@ -0,0 +1,712 @@ +//! Russian grammar and punctuation checks: cheap, rule-based and tuned to stay quiet when unsure +//! (a missed comma costs less than a wrong mark on correct text). Messages are in Russian. +//! +//! - **Repeated word** (`в в`), except words Russian doubles on purpose (`да да`, `так так`, +//! `ну ну`, `ха ха`, `очень очень`). +//! - **Space before punctuation** (`да ,`) and **double spaces**; emoticons (` :)`) and ellipses +//! are left alone. +//! - **Capital letter at the start of a sentence**, knowing Russian abbreviations (`т. е.`, `г.`, +//! `руб.`, `ул.`, `и т. д.`), initials, numbers, ellipses and dialogue dashes, after which lower +//! case is correct. +//! - **Missing comma** before conjunctions and relative pronouns that always open a clause: +//! `что`, `чтобы`, `поскольку`, `если`, `хотя` (not `хотя бы`) and the forms of `который`, and +//! before `потому`/`оттого` in `потому что`. Not after a conjunction or particle (`и что`, +//! `только что`, `даже если`, `так что`, `для того чтобы`), not after a preposition (`за что`, +//! `в котором`: the comma goes before it) and not inside `что-то`. +//! - **Doubled punctuation** (`,,` — often a typed opening quote — `..`, `;;`). +//! +//! No English rule (such as `a`/`an`) applies to Russian text. + +use crate::{Issue, IssueKind, ProofLang, words}; + +/// Words Russian repeats on purpose: interjections, onomatopoeia and emphatic doubling. +const ALLOWED_REPEATS: &[&str] = &[ + "да", + "нет", + "так", + "ну", + "ха", + "хи", + "хе", + "ой", + "ай", + "эй", + "ах", + "ох", + "эх", + "ага", + "угу", + "тук", + "кап", + "бах", + "бум", + "тик", + "так", + "ля", + "очень", + "много", + "давно", + "далеко", + "долго", + "еле", + "чуть", + "тихо", + "быстро", + "скорей", + "скорее", + "иди", + "давай", + "пока", + "ладно", + "вот", + "ещё", + "еще", + "всё", + "все", + "кто", + "что", + "то", +]; + +/// Abbreviations written with a final period after which a sentence goes on in lower case +/// (lower case, without the period). +const ABBREVIATIONS: &[&str] = &[ + "т", + "е", + "д", + "п", + "др", + "пр", + "см", + "ср", + "напр", + "т.е", + "т.д", + "т.п", + "т.к", + "т.н", + "т.о", + "г", + "гг", + "в", + "вв", + "н.э", + "до н.э", + "руб", + "коп", + "тыс", + "млн", + "млрд", + "трлн", + "ул", + "пр-т", + "пер", + "наб", + "пл", + "ш", + "д", + "кв", + "корп", + "стр", + "оф", + "эт", + "под", + "обл", + "р-н", + "пос", + "дер", + "им", + "св", + "проф", + "доц", + "акад", + "канд", + "наук", + "зав", + "зам", + "ред", + "изд", + "сост", + "пер", + "рис", + "табл", + "гл", + "разд", + "ст", + "ч", + "п", + "пп", + "абз", + "прим", + "прил", + "илл", + "с", + "сс", + "т", + "тт", + "вып", + "кн", + "экз", + "шт", + "кг", + "мин", + "сек", + "час", + "мес", + "нед", + "янв", + "фев", + "февр", + "мар", + "апр", + "авг", + "сент", + "окт", + "нояб", + "дек", + "пн", + "вт", + "чт", + "пт", + "сб", + "вс", + "англ", + "нем", + "фр", + "франц", + "лат", + "рус", + "греч", + "итал", + "исп", + "букв", + "перен", + "разг", + "устар", + "ок", + "ок", + "прибл", + "мл", + "ст", + "тел", + "моб", + "факс", + "эл", + "почт", + "вкл", + "искл", + "мм", + "см", + "дм", + "км", + "м", + "л", + "мл", + "т", + "ц", + "млн", + "долл", + "евро", + "др", + "проч", + "etc", + "vs", +]; + +/// Conjunctions and relative pronouns that open a clause: a comma comes before them. +const CLAUSE_OPENERS: &[&str] = &[ + "что", + "чтобы", + "поскольку", + "если", + "хотя", + "который", + "которая", + "которое", + "которые", + "которого", + "которой", + "которому", + "которым", + "котором", + "которую", + "которых", + "которыми", +]; + +/// Words after which a clause opener takes no comma of its own: conjunctions (`и что`, `а если`), +/// particles (`только что`, `даже если`, `лишь бы`), interrogatives and the first halves of +/// compound conjunctions (`потому что`, `так что`, `для того чтобы`, `с тем чтобы`, `несмотря на +/// то что`, `тогда как`, `как будто`). +const NO_COMMA_AFTER: &[&str] = &[ + "и", + "а", + "но", + "да", + "или", + "либо", + "ни", + "не", + "же", + "ли", + "бы", + "б", + "то", + "только", + "лишь", + "именно", + "даже", + "особенно", + "потому", + "оттого", + "поэтому", + "так", + "затем", + "тем", + "того", + "ввиду", + "вследствие", + "несмотря", + "благодаря", + "прежде", + "раньше", + "мере", + "пор", + "чем", + "будто", + "словно", + "точно", + "как", + "если", + "хотя", + "хоть", + "чтобы", + "что", + "кто", + "где", + "когда", + "почему", + "зачем", + "куда", + "откуда", + "сколько", + "разве", + "неужели", + "вот", + "вон", + "ну", + "уже", + "это", + "просто", + "чуть", + "мол", + "ведь", + "вроде", + "тоже", + "также", + "ещё", + "еще", + "всё", + "все", + "почти", + "вовсе", + "ой", + "ах", + "эх", + "ох", + "ага", + "нет", + "конечно", + "наверное", + "может", + "пусть", + "лучше", + "условии", + "случае", + "того", + "ради", + "раз", + "покуда", + "пока", + "после", + "перед", + "лишь", + "едва", + "чуть", + "дабы", + "кабы", + "коли", + "ежели", + "абы", + "вдруг", + "иначе", + "причём", + "причем", + "притом", + "зато", + "однако", + "тогда", + "сейчас", + "теперь", + "сначала", + "сперва", + "иногда", + "часто", + "всегда", + "никогда", + "т", + "др", +]; + +/// Prepositions: a relative pronoun or `что` after one takes the comma before the preposition +/// instead (`дом, в котором`, `то, за что`). +const PREPOSITIONS: &[&str] = &[ + "в", + "во", + "на", + "с", + "со", + "к", + "ко", + "о", + "об", + "обо", + "от", + "ото", + "до", + "из", + "изо", + "за", + "по", + "под", + "подо", + "над", + "надо", + "при", + "про", + "без", + "безо", + "для", + "у", + "через", + "чрез", + "между", + "меж", + "среди", + "из-за", + "из-под", + "около", + "возле", + "вокруг", + "мимо", + "против", + "вместо", + "кроме", + "сквозь", + "вдоль", + "внутри", + "вне", + "после", + "перед", + "пред", + "ради", + "благодаря", + "согласно", + "вопреки", + "навстречу", + "насчёт", + "насчет", + "ввиду", + "вследствие", + "посредством", + "путём", + "путем", + "относительно", + "касательно", + "вслед", + "ко", + "подле", + "близ", + "внутрь", + "поверх", +]; + +fn issue(start: usize, end: usize, message: String, suggestions: Vec) -> Issue { + Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::Ru } +} + +fn capitalize(w: &str) -> String { + crate::affix::capitalize(w) +} + +/// Grammar issues in a paragraph of Russian text, in order. +pub fn check(text: &str) -> Vec { + check_with(text, true) +} + +/// [`check`] where `capital_first` says whether the text's first word must start a sentence with a capital +/// (false for a list item, which may be a fragment). +pub fn check_with(text: &str, capital_first: bool) -> Vec { + let ws: Vec<(usize, usize)> = words(text); + let mut v = Vec::new(); + repeated_words(text, &ws, &mut v); + spaces(text, &mut v); + sentence_capitals(text, &ws, capital_first, &mut v); + missing_commas(text, &ws, &mut v); + doubled_punctuation(text, &mut v); + v.sort_by_key(|i| (i.start, i.end)); + v +} + +fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if !gap.is_empty() + && gap.chars().all(char::is_whitespace) + && w0.chars().all(char::is_alphabetic) + && w0.to_lowercase() == w1.to_lowercase() + && !ALLOWED_REPEATS.contains(&w0.to_lowercase().as_str()) + { + v.push(issue(*a0, *b1, format!("Повтор слова: «{w1}»"), vec![w0.to_string()])); + } + } +} + +fn spaces(text: &str, v: &mut Vec) { + for (i, c) in text.char_indices() { + if c != ' ' { + continue; + } + let rest = text.get(i + 1..).unwrap_or(""); + let mut it = rest.chars(); + let Some(p) = it.next() else { continue }; + let after = it.next(); + let flag = match p { + ',' => true, + // `;)` and `;-)` are emoticons. + ';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')), + // ` ...` is an ellipsis, ` .5` a number. + '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), + // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. + ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '»' | '”' | '’' | '!' | '?')), + _ => false, + }; + if flag { + v.push(issue(i, i + 1 + p.len_utf8(), "Лишний пробел перед знаком препинания".into(), vec![p.to_string()])); + } + } + for (i, _) in text.match_indices(" ") { + if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { + continue; + } + v.push(issue(i, i + 2, "Двойной пробел".into(), vec![" ".into()])); + } +} + +/// Is `prev`, before a period, an abbreviation, an initial, a number or a Roman numeral (after +/// which a lower-case word doesn't start a sentence)? +fn abbreviation_or_number(prev: &str) -> bool { + prev.chars().count() <= 1 + || prev.chars().any(|c| c.is_ascii_digit()) + || (prev.chars().all(|c| matches!(c, 'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M'))) + || ABBREVIATIONS.contains(&prev.to_lowercase().as_str()) +} + +const CLOSING: [char; 8] = ['"', '»', '”', '’', '\'', ')', ']', '\u{1}']; +const OPENING: [char; 8] = ['«', '"', '„', '“', '(', '[', '\'', '‚']; + +/// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after +/// an abbreviation), then space, with nothing but quotes and brackets around? +fn ends_sentence(gap: &str, prev: &str) -> bool { + let t = gap.trim_start_matches(CLOSING); + let Some(end) = t.chars().next() else { return false }; + if !matches!(end, '.' | '!' | '?') || t.starts_with("..") { + return false; + } + if end == '.' && abbreviation_or_number(prev) { + return false; + } + let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING); + rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| OPENING.contains(&c)) +} + +fn sentence_capitals(text: &str, ws: &[(usize, usize)], capital_first: bool, v: &mut Vec) { + // List items (`– яблоки,`) may start in lower case. + let list_item = !capital_first || text.trim_end().ends_with([',', ';']); + for (k, &(a, b)) in ws.iter().enumerate() { + let Some(w) = text.get(a..b) else { continue }; + let starts = if k == 0 { + let before = text.get(..a).unwrap_or(""); + !list_item && before.chars().all(|c| c.is_whitespace() || OPENING.contains(&c)) + } else { + let Some(&(pa, pb)) = ws.get(k - 1) else { continue }; + ends_sentence(text.get(pb..a).unwrap_or(""), text.get(pa..pb).unwrap_or("")) + }; + if starts && w.chars().next().is_some_and(char::is_lowercase) && w.chars().all(char::is_alphabetic) { + v.push(issue(a, b, "Предложение начинается со строчной буквы".into(), vec![capitalize(w)])); + } + } +} + +fn missing_commas(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for k in 1..ws.len() { + let (Some(&(pa, pb)), Some(&(a, b))) = (ws.get(k - 1), ws.get(k)) else { continue }; + let (Some(prev), Some(w), Some(gap)) = (text.get(pa..pb), text.get(a..b), text.get(pb..a)) else { continue }; + if gap.is_empty() || !gap.chars().all(|c| c == ' ' || c == '\u{a0}') || !prev.chars().all(char::is_alphabetic) { + continue; + } + let lw = w.to_lowercase(); + let next = ws.get(k + 1).and_then(|&(na, nb)| text.get(na..nb)).map(str::to_lowercase); + let next_gap_plain = ws.get(k + 1).and_then(|&(na, _)| text.get(b..na)).is_some_and(|g| g.chars().all(|c| c == ' ' || c == '\u{a0}')); + // `потому что`, `оттого что`: the comma goes before the first word. + let compound = matches!(lw.as_str(), "потому" | "оттого") && next.as_deref() == Some("что") && next_gap_plain; + if !compound && !CLAUSE_OPENERS.contains(&lw.as_str()) { + continue; + } + // `хотя бы` is a particle (`хотя бы раз`). + if lw == "хотя" && next.as_deref() == Some("бы") { + continue; + } + let p = prev.to_lowercase(); + let p = p.as_str(); + if NO_COMMA_AFTER.contains(&p) || PREPOSITIONS.contains(&p) { + continue; + } + v.push(issue(pb, b, format!("Возможно, пропущена запятая перед «{w}»"), vec![format!(", {w}")])); + } +} + +fn doubled_punctuation(text: &str, v: &mut Vec) { + let bytes = text.as_bytes(); + let mut i = 0; + while let Some(&b) = bytes.get(i) { + if !matches!(b, b',' | b'.' | b';') { + i += 1; + continue; + } + let run = bytes.get(i..).map_or(0, |r| r.iter().take_while(|&&x| x == b).count()); + let flag = match b { + b'.' => run == 2, + _ => run >= 2, + }; + if flag { + let end = i + run; + let suggestions: Vec = match b { + // `,,` before a word, after a space: a typed opening quote. + b',' if text.get(end..).and_then(|t| t.chars().next()).is_some_and(char::is_alphabetic) + && (i == 0 || text.get(..i).is_some_and(|t| t.ends_with(char::is_whitespace))) => + { + vec!["«".into()] + } + b'.' => vec![".".into(), "…".into()], + _ => vec![char::from(b).to_string()], + }; + v.push(issue(i, end, "Повтор знака препинания".into(), suggestions)); + } + i += run.max(1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn messages(t: &str) -> Vec { + check(t).into_iter().map(|i| format!("{}@{}", i.message, &t[i.start..i.end])).collect() + } + + fn clean(t: &str) { + let m = messages(t); + assert!(m.is_empty(), "{t:?}: {m:?}"); + } + + #[test] + fn repeated_words() { + let v = check("Я пошёл в в магазин и и купил хлеб."); + assert_eq!(v.len(), 2, "{v:?}"); + assert_eq!(v[0].message, "Повтор слова: «в»"); + assert_eq!(v[0].suggestions, ["в"]); + assert!(v.iter().all(|i| i.lang == ProofLang::Ru && i.kind == IssueKind::Grammar)); + clean("Да да, так так. Ха ха! Очень очень давно."); + } + + #[test] + fn spaces_and_punctuation() { + let m = messages("Мы пришли , а он ушёл . Правда ?"); + assert!(m.contains(&"Лишний пробел перед знаком препинания@ ,".to_string()), "{m:?}"); + assert!(m.contains(&"Лишний пробел перед знаком препинания@ .".to_string()), "{m:?}"); + assert!(m.contains(&"Лишний пробел перед знаком препинания@ ?".to_string()), "{m:?}"); + assert!(m.contains(&"Двойной пробел@ ".to_string()), "{m:?}"); + clean("Отлично :) Правда :-) Число .5 и многоточие ..."); + let m = messages("Он сказал ,,да» и ушёл.. Потом;; конец."); + assert_eq!(m.iter().filter(|x| x.starts_with("Повтор знака")).count(), 3, "{m:?}"); + assert_eq!(check("Он сказал ,,да».").iter().find(|i| i.message.starts_with("Повтор знака")).unwrap().suggestions, ["«"]); + clean("Ждал… и ждал... до утра."); + } + + #[test] + fn sentence_start_capitals() { + let m = messages("это первое предложение. а это второе! правда? да."); + assert_eq!(m.iter().filter(|x| x.starts_with("Предложение начинается")).count(), 4, "{m:?}"); + assert_eq!(check("мама мыла раму.")[0].suggestions, ["Мама"]); + // Abbreviations, initials, numbers, ellipses and dialogue dashes. + clean("Купили фрукты: яблоки, груши и т. д. и овощи."); + clean("В 1990 г. открылась фирма, т. е. наша компания."); + clean("Цена 500 руб. за штуку, адрес ул. Ленина, д. 5, кв. 3."); + clean("Встретил А. С. Пушкина и проф. Иванова."); + clean("Ждал... и наконец дождался."); + clean("– Правда? – спросил он. – Да, – ответила она."); + clean("Он сказал: «Вернусь». Потом ушёл."); + clean("яблоки,"); + } + + #[test] + fn missing_commas() { + let m = messages( + "Я знаю что он придёт. Это дом который построил Джек. Пришёл чтобы помочь. Ушёл потому что устал. Приходи если сможешь. Пошёл хотя устал.", + ); + for want in [ + "Возможно, пропущена запятая перед «что»@ что", + "Возможно, пропущена запятая перед «который»@ который", + "Возможно, пропущена запятая перед «чтобы»@ чтобы", + "Возможно, пропущена запятая перед «потому»@ потому", + "Возможно, пропущена запятая перед «если»@ если", + "Возможно, пропущена запятая перед «хотя»@ хотя", + ] { + assert!(m.contains(&want.to_string()), "{want}: {m:?}"); + } + assert_eq!(check("Знаю что да.")[0].suggestions, [", что"]); + // Conjunctions, particles, compound conjunctions and prepositions take no comma there. + clean("Я знаю, что он придёт, и что останется."); + clean("Он только что пришёл, так что всё в порядке."); + clean("Это дом, в котором я живу, и сад, через который иду."); + clean("Я сделаю это, даже если не успею, лишь бы ты был рад."); + clean("Пришёл для того чтобы помочь, а не чтобы мешать."); + clean("Ушёл, потому что устал, и не потому что обиделся."); + clean("Расскажи хотя бы что-нибудь, за что тебя хвалят."); + clean("Что ты делаешь? Который час?"); + clean("Кто-то что-то сказал."); + } + + #[test] + fn correct_russian_text_is_clean() { + clean("Съешь же ещё этих мягких французских булок, да выпей чаю. Мы пришли, когда уже стемнело."); + clean("Книга, которую я читаю, очень интересная, поэтому я её советую."); + } + + proptest::proptest! { + #[test] + fn never_panics(s in "\\PC{0,80}") { + let _ = check(&s); + } + + #[test] + fn never_panics_on_russian_like_text(s in "[а-яёА-ЯЁ ,.;:!?«»…\\-]{0,60}") { + for i in check(&s) { + proptest::prop_assert!(i.start <= i.end && i.end <= s.len()); + proptest::prop_assert!(s.is_char_boundary(i.start) && s.is_char_boundary(i.end)); + } + } + } +} diff --git a/crates/proof/src/ru/hyphen.rs b/crates/proof/src/ru/hyphen.rs new file mode 100644 index 00000000..33d19b77 --- /dev/null +++ b/crates/proof/src/ru/hyphen.rs @@ -0,0 +1,134 @@ +//! Russian word division, from the rules of Russian spelling (no pattern file): +//! +//! - every part keeps a vowel, and at least two letters stay on each side; +//! - `ь`, `ъ` and `й` stay with the letter before them (`паль-то`, `вой-на`), and a syllable +//! ends after `ъ` (`объ-ём`, `подъ-езд`); +//! - between two vowels with one consonant between them the consonant starts the next syllable +//! (`ма-ма`, `ро-ди-на`); +//! - a cluster of consonants is divided after its first consonant (`сес-тра`, `кар-ман`, +//! `кас-са`, `под-ход`), which also keeps the common prefixes `под-`, `раз-`, `без-` whole; +//! - two vowels side by side may be divided (`по-эт`, `ма-як`). + +use super::is_cyrillic; + +/// Russian vowels. +fn is_vowel(c: char) -> bool { + matches!(c, 'а' | 'е' | 'ё' | 'и' | 'о' | 'у' | 'ы' | 'э' | 'ю' | 'я') +} + +/// Letters that never begin a line: the soft and hard signs and `й`. +fn sticks_left(c: char) -> bool { + matches!(c, 'ь' | 'ъ' | 'й') +} + +/// Break points of a lowercase Russian word (letters only): each is the index of the letter that +/// starts a new line. Empty for words with letters outside the Russian alphabet. +pub fn points(c: &[char]) -> Vec { + let n = c.len(); + if n < 4 || !c.iter().all(|&x| is_cyrillic(x) && x.is_alphabetic()) { + return vec![]; + } + let vowels: Vec = (0..n).filter(|&i| c.get(i).is_some_and(|&x| is_vowel(x))).collect(); + let mut out = vec![]; + for pair in vowels.windows(2) { + let [a, b] = pair else { continue }; + let (start, end) = (a + 1, *b); + // The letters that must stay on the left: signs and `й` right after the vowel or after a + // consonant of the cluster. + let mut cut = if start == end { + // Two vowels side by side. + end + } else if c.get(end - 1) == Some(&'ъ') { + // The hard sign closes its syllable (`объ-ём`, `подъ-езд`). + end + } else { + // The cluster's consonants, skipping signs that stick to the letter before. + let consonants: Vec = (start..end).filter(|&i| c.get(i).is_some_and(|&x| !sticks_left(x))).collect(); + match consonants.as_slice() { + // Only `й` (`май-онез`): after it. + [] => end, + // One consonant: it starts the next syllable (`ма-ма`, `паль-то`). + [only] => *only, + // Several: after the first one (`сес-тра`, `кар-ман`). + [_, second, ..] => *second, + } + }; + // A sign or `й` never starts a line. + while c.get(cut).is_some_and(|&x| sticks_left(x)) && cut < end { + cut += 1; + } + // Cuts come in increasing order (each lies after the previous pair's vowel), so a repeat + // can only be the last one: no scan of the whole list per syllable (#3657). + if cut >= 2 && n - cut >= 2 && out.last().is_none_or(|&l| l < cut) { + out.push(cut); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn h(w: &str) -> String { + let c: Vec = w.chars().collect(); + let p = points(&c); + c.iter().enumerate().map(|(i, ch)| if p.contains(&i) { format!("-{ch}") } else { ch.to_string() }).collect() + } + + #[test] + fn russian_words_divide_by_the_rules() { + assert_eq!(h("мама"), "ма-ма"); + assert_eq!(h("родина"), "ро-ди-на"); + assert_eq!(h("сестра"), "сес-тра"); + assert_eq!(h("карман"), "кар-ман"); + assert_eq!(h("касса"), "кас-са"); + assert_eq!(h("подход"), "под-ход"); + assert_eq!(h("пальто"), "паль-то"); + assert_eq!(h("объём"), "объ-ём"); + assert_eq!(h("подъезд"), "подъ-езд"); + assert_eq!(h("семья"), "се-мья"); + assert_eq!(h("вьюга"), "вью-га"); + assert_eq!(h("объявление"), "объ-яв-ле-ние"); + assert_eq!(h("война"), "вой-на"); + assert_eq!(h("чайник"), "чай-ник"); + assert_eq!(h("поэзия"), "по-э-зия"); + assert_eq!(h("маяк"), "ма-як"); + assert_eq!(h("жидкие"), "жид-кие"); + assert_eq!(h("обои"), "обои"); + assert_eq!(h("декоративная"), "де-ко-ра-тив-ная"); + assert_eq!(h("штукатурка"), "шту-ка-тур-ка"); + } + + #[test] + fn short_words_and_other_scripts_stay_whole() { + for w in ["дом", "ёж", "оса", "iPhone", "дом2", ""] { + let c: Vec = w.chars().collect(); + assert!(points(&c).is_empty(), "{w}"); + } + } + + #[test] + fn no_line_starts_with_a_sign_or_ends_with_one_letter() { + for w in ["объявление", "подъезд", "семья", "вьюга", "майонез", "яйцо", "ауди", "аорта", "оазис", "ёлочка"] + { + let c: Vec = w.chars().collect(); + for p in points(&c) { + assert!(p >= 2 && c.len() - p >= 2, "{w}: {p}"); + assert!(c.get(p).is_some_and(|&x| !sticks_left(x)), "{w}: {p}"); + assert!(c.get(..p).is_some_and(|l| l.iter().any(|&x| is_vowel(x))), "{w}: {p}"); + assert!(c.get(p..).is_some_and(|r| r.iter().any(|&x| is_vowel(x))), "{w}: {p}"); + } + } + } + + proptest::proptest! { + #[test] + fn never_panics(s in "[а-яё]{0,30}") { + let c: Vec = s.chars().collect(); + for p in points(&c) { + proptest::prop_assert!(p >= 2 && p + 2 <= c.len()); + } + } + } +} diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs new file mode 100644 index 00000000..06d65ef5 --- /dev/null +++ b/crates/proof/src/ru/mod.rs @@ -0,0 +1,486 @@ +//! Russian proofing. +//! +//! - **Spelling** against a Russian Hunspell dictionary installed on the system ([`dictionary`], +//! found by [`crate::hunspell`]: `ru_RU.aff`/`ru_RU.dic` from the OS, a package manager or +//! LibreOffice's `dict-ru`; it isn't bundled). Without it Russian text gets no spelling marks. +//! `ё` is optional, as in ordinary Russian writing: `елка` and `ёлка`, `все` and `всё` are +//! accepted whichever one the dictionary lists. Short words in capitals are abbreviations +//! (`ООО`, `НДС`). Hyphenated words are checked whole first, then by their parts, knowing the +//! particles (`кто-то`, `где-нибудь`, `давай-ка`, `всё-таки`), `кое-`/`кой-`, adverbs with +//! `по-` (`по-русски`, `по-новому`) and compound adjectives whose first part ends in a linking +//! vowel (`северо-западный`, `сине-зелёный`). Compounds written solid are checked the same way +//! (`высокоэластичный`, `газоиспользующий`), and so are adjectives, participles and verbal nouns +//! with `не` (`невскрытый`, `непредоставление`; not verbs: `незнаю` stays wrong). Our supplement +//! adds words older dictionaries lack (`маркетплейс`, `промокод`, `интеграция`: [`extra`]). +//! Words in another script (`iPhone`), with digits or superscripts (`м²`), in camel case +//! (`ВКонтакте`) or abbreviations with an ending (`ГИПа`, `ПДн`) are left alone. +//! - **Suggestions** tuned for Russian ([`suggest()`]): `е`/`ё`, the common confusions (`тся`/ +//! `ться`, `жи`/`ши` written with `ы`, `ча`/`ща` with `я`, voiced and voiceless consonant pairs, +//! doubled consonants), a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`), one-letter +//! typos, and a dictionary scan for anything further off. +//! - **Grammar** ([`grammar`]): cheap, reliable rules with Russian messages. +//! - **Hyphenation** ([`hyphen`]): the rules of Russian word division, written as code. + +mod extra; +pub mod grammar; +pub mod hyphen; +mod suggest; +#[cfg(test)] +pub(crate) mod test_dict; + +use std::sync::Arc; + +use crate::affix::AffixDict; +use crate::slot::DictSlot; + +pub use suggest::suggest; + +/// File names of a Russian Hunspell dictionary. +const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"]; + +/// The Russian dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Russian"); + +/// The Russian dictionary: the installed Hunspell `ru_RU` (read on first use), one given with +/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off +/// then; see [`has_dictionary`]). +pub fn dictionary() -> Arc { + DICT.get() +} + +/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to +/// UTF-8), for example one the user picked. On error the current dictionary stays. +pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { + DICT.install(aff, dic) +} + +/// Is a Russian dictionary available (spelling is checked)? +pub fn has_dictionary() -> bool { + DICT.has_dictionary() +} + +/// Is `c` a letter of the Cyrillic script (the only one the Russian dictionary covers)? +pub(crate) fn is_cyrillic(c: char) -> bool { + matches!(c, '\u{0400}'..='\u{052F}') +} + +/// Particles joined to a word with a hyphen (`кто-то`, `где-нибудь`, `давай-ка`, `всё-таки`, +/// `он-де`). +const PARTICLES: &[&str] = &["то", "либо", "нибудь", "ка", "таки", "де", "с"]; + +/// Prefixes joined to a pronoun with a hyphen (`кое-кто`, `кое-где`, `кой-какой`). +const PRONOUN_PREFIXES: &[&str] = &["кое", "кой"]; + +/// Words of fewer letters than this written in capitals are taken as abbreviations (`ООО`, `НДС`, +/// `МГУ`, `ГОСТ`). +const ABBREVIATION_LETTERS: usize = 6; + +/// Is `word` spelled correctly in Russian? Numbers, single letters, short words in capitals, +/// words with digits and words in other scripts are accepted, and so is every word when no +/// Russian dictionary is installed. +pub fn is_correct(word: &str) -> bool { + is_correct_with(&dictionary(), word) +} + +/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off, +/// short words in capitals are checked against the dictionary too, and with Ignore words that +/// contain numbers off, so are words with digits or superscripts. +pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool { + check(&dictionary(), word, opts) +} + +/// [`is_correct`] against a given dictionary (an empty one accepts everything). +pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { + check(d, word, &crate::Options::default()) +} + +/// [`is_correct_with_options`] against a given dictionary. +fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { + if d.is_empty() { + return true; // no Russian dictionary installed: no spelling marks + } + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '-'); + let w = w.trim_matches('-'); + let n = w.chars().count(); + if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) { + return true; + } + if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS { + return true; + } + // Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word. + let cs: Vec = w.chars().collect(); + if cs.windows(2).any(|p| matches!(p, [a, b] if a.is_lowercase() && b.is_uppercase())) { + return true; + } + // An abbreviation with lower-case letters or an ending (`ГИПа`, `ПДн`, `ИСПДн`). + if n <= 10 && cs.iter().take(2).all(|c| c.is_uppercase()) && cs.iter().any(|c| c.is_lowercase()) { + return true; + } + // Another script, or a mix of scripts (`iPhone`, `SMS-ка`): not this dictionary's to judge. + if w.chars().any(|c| c.is_alphabetic() && !is_cyrillic(c)) { + return true; + } + if known(d, w) { + return true; + } + if w.contains('-') { + return hyphenated(d, w, opts); + } + solid_compound(d, w) +} + +/// Endings of adjectives, participles and nouns that are written together with `не` +/// (`невскрытый`, `неисполнение`, `неготовность`). Verb endings aren't among them: `не` with a +/// verb is written apart (`не знаю`). +const NE_ENDINGS: &[&str] = &[ + "ый", + "ий", + "ой", + "ая", + "яя", + "ое", + "ее", + "ые", + "ие", + "ого", + "его", + "ому", + "ему", + "ым", + "ую", + "юю", + "ых", + "их", + "ыми", + "ими", + "ние", + "ния", + "нию", + "нием", + "нии", + "ость", + "ости", + "остью", + "остей", + "остям", + "остями", + "остях", +]; + +/// Prefixes Russian joins to any word (`ультрастойкий`, `супермаркет`, `экопакет`). +const WORD_PREFIXES: &[&str] = &[ + "ультра", + "супер", + "гипер", + "мега", + "микро", + "макро", + "мульти", + "мини", + "макси", + "анти", + "псевдо", + "квази", + "контр", + "сверх", + "полу", + "само", + "авто", + "видео", + "аудио", + "фото", + "радио", + "теле", + "эко", + "био", + "нано", + "агро", + "медиа", + "интернет", + "веб", + "бизнес", + "онлайн", + "спец", + "гос", + "мед", + "юр", + "орг", + "проф", + "соц", +]; + +/// Prefixes that make a verb perfective (`с-генерировать`, `от-модерировать`, `пере-загрузить`). +const VERB_PREFIXES: &[&str] = &[ + "с", "со", "за", "по", "про", "на", "вы", "от", "ото", "пере", "под", "подо", "при", "раз", "рас", "из", "ис", "у", "о", "об", "обо", "до", + "недо", "пред", "в", "во", "вз", "вс", +]; + +/// Endings of verb forms (infinitive, present, past, reflexive). +const VERB_ENDINGS: &[&str] = &[ + "ть", "ться", "ти", "чь", "ю", "ешь", "ет", "ем", "ете", "ют", "ишь", "ит", "им", "ите", "ят", "ал", "ала", "ало", "али", "ил", "ила", "ило", + "или", "ся", "сь", +]; + +/// A word written solid from known parts: `не` with an adjective, participle or verbal noun +/// (`невскрытой`), or two stems joined by a linking vowel (`высокоэластичная`, `газоиспользующего`). +fn solid_compound(d: &AffixDict, w: &str) -> bool { + let lower = w.to_lowercase(); + let chars: Vec = lower.chars().collect(); + let n = chars.len(); + if !(7..=40).contains(&n) { + return false; + } + if let Some(rest) = lower.strip_prefix("не") + && rest.chars().count() >= 4 + && NE_ENDINGS.iter().any(|e| rest.ends_with(e)) + && known(d, rest) + { + return true; + } + // A productive prefix (`ультрастойкий`, `экопакет`, `микроцемент`) on a known word. + if WORD_PREFIXES.iter().any(|p| lower.strip_prefix(p).is_some_and(|rest| rest.chars().count() >= 4 && known(d, rest))) { + return true; + } + // A verb with a perfective prefix (`сгенерировать`, `отмодерировали`) whose unprefixed form is + // known. + if VERB_PREFIXES.iter().any(|p| { + lower.strip_prefix(p).is_some_and(|rest| rest.chars().count() >= 5 && VERB_ENDINGS.iter().any(|e| rest.ends_with(e)) && known(d, rest)) + }) { + return true; + } + (4..=n.saturating_sub(4)).any(|i| { + let (Some(left), Some(right)) = (chars.get(..i), chars.get(i..)) else { return false }; + if !matches!(left.last(), Some('о' | 'е')) { + return false; + } + let left: String = left.iter().collect(); + let right: String = right.iter().collect(); + known(d, &right) && (known(d, &left) || linking_stem(d, &left)) + }) +} + +/// Is `part`, ending in a linking vowel, a known word's stem (`северо` → `север`, `сине` → +/// `синий`, `газо` → `газ`)? +fn linking_stem(d: &AffixDict, part: &str) -> bool { + let Some(stem) = part.strip_suffix(['о', 'е', 'О', 'Е']) else { return false }; + stem.chars().count() >= 2 && ["", "ый", "ий", "ой", "ь"].iter().any(|end| known(d, &format!("{stem}{end}"))) +} + +/// Is one word in the dictionary or the user dictionary, with `ё` optional? +fn known(d: &AffixDict, w: &str) -> bool { + if d.check(w) || crate::in_user_dictionary(&w.to_lowercase()) || extra::contains(w) { + return true; + } + // `ё` written as `е` is ordinary Russian: try the dictionary's `ё` spelling… + let without = w.replace('ё', "е").replace('Ё', "Е"); + if without != w && (d.check(&without) || crate::in_user_dictionary(&without.to_lowercase())) { + return true; + } + // …and `е` where the dictionary has `ё` (one or two of them: `Черемушки`, `еще`). + let e_positions: Vec = without.char_indices().filter(|(_, c)| matches!(c, 'е' | 'Е')).map(|(i, _)| i).collect(); + if e_positions.is_empty() || e_positions.len() > 4 { + return false; + } + let swap = |s: &str, i: usize| -> Option { + let c = s.get(i..)?.chars().next()?; + let yo = if c == 'Е' { "Ё" } else { "ё" }; + Some(format!("{}{}{}", s.get(..i)?, yo, s.get(i + c.len_utf8()..)?)) + }; + for (k, &i) in e_positions.iter().enumerate() { + let Some(one) = swap(&without, i) else { continue }; + if d.check(&one) { + return true; + } + for &j in e_positions.get(k + 1..).unwrap_or_default() { + // `ё` is two bytes like `е`, so later offsets still hold. + if swap(&one, j).is_some_and(|two| d.check(&two)) { + return true; + } + } + } + false +} + +/// A hyphenated word that isn't in the dictionary whole: its parts, with the particles, +/// prefixes and linking vowels Russian joins with a hyphen. +fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool { + let parts: Vec<&str> = w.split('-').collect(); + if parts.iter().any(|p| p.is_empty()) || parts.len() > 4 { + return false; + } + let lower = w.to_lowercase(); + let lp: Vec<&str> = lower.split('-').collect(); + // `кто-то`, `где-нибудь`, `всё-таки`: a word and its particles. + let first_particle = lp.iter().skip(1).position(|p| PARTICLES.contains(p)).map(|i| i + 1); + if let Some(k) = first_particle + && lp.iter().skip(k).all(|p| PARTICLES.contains(p)) + { + let base = parts.get(..k).map(|b| b.join("-")).unwrap_or_default(); + return known(d, &base) || (base.contains('-') && hyphenated(d, &base, opts)); + } + match lp.as_slice() { + // `кое-кто`, `кой-какой`. + [prefix, rest] if PRONOUN_PREFIXES.contains(prefix) => return parts.get(1).is_some_and(|r| known(d, r)), + // `по-русски` (from `русский`), `по-новому`, `по-моему`. + ["по", rest] => { + let Some(r) = parts.get(1) else { return false }; + return known(d, r) || ((rest.ends_with("ски") || rest.ends_with("цки") || rest.ends_with("ьи")) && known(d, &format!("{r}й"))); + } + _ => {} + } + // Compounds part by part: `северо-западный`, `сине-зелёный`, `интернет-магазин`, `из-за`. + let last = parts.len() - 1; + parts.iter().enumerate().all(|(i, p)| { + // An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word. + if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) { + return true; + } + // A first part with a linking vowel: the stem (`северо` → `север`) or its adjective + // (`сине` → `синий`, `бело` → `белый`). + i != last && linking_stem(d, p) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_dictionary_loads() { + test_dict::install(); + let d = dictionary(); + assert!(has_dictionary()); + assert!(d.len() > 50, "{}", d.len()); + } + + #[test] + fn without_a_dictionary_nothing_is_flagged_and_nothing_suggested() { + let none = AffixDict::default(); + for w in ["превет", "сдесь", "xqzv", ""] { + assert!(is_correct_with(&none, w), "{w}"); + } + assert!(suggest::suggest_in(&none, "превет", 5).is_empty()); + } + + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn russian_words_are_correct() { + test_dict::install(); + for w in [ + "привет", + "Привет", + "ПРИВЕТ", + "дом", + "дома", + "домами", + "красный", + "красного", + "красной", + "читать", + "читаем", + "читаются", + "ёлка", + "елка", + "всё", + "все", + "ещё", + "еще", + "Москва", + "Москве", + "кто-то", + "где-нибудь", + "что-либо", + "давай-ка", + "всё-таки", + "все-таки", + "кое-кто", + "по-русски", + "по-новому", + "северо-западный", + "сине-зелёный", + "сине-зеленый", + "из-за", + "ООО", + "НДС", + "2026", + "5-й", + "iPhone", + "e-mail", + "в", + "м²", + "ВКонтакте", + "МирКрасок", + "ГИПа", + "ПДн", + "ИИ-консультант", + "маркетплейсов", + "промокода", + "невскрытой", + "некрасного", + "непредоставления", + "высокоэластичная", + "краснозелёный", + "Пн-Пт", + "ультракрасный", + "суперновый", + "прочитать", + "администрируется", + "юрлица", + ] { + assert!(is_correct(w), "{w}"); + } + } + + #[test] + fn misspellings_are_flagged() { + test_dict::install(); + for w in [ + "превет", + "сдесь", + "жыть", + "чясто", + "незнаю", + "нечитаю", + "вобщем", + "москва", + "здраствуй", + "кто-нибуть", + "по-руски", + "северо-зопадный", + "ПРИВЕТТТ", + "высокозопадный", + ] { + assert!(!is_correct(w), "{w}"); + } + } + + #[test] + fn user_dictionary_applies() { + test_dict::install(); + assert!(!is_correct("зкщфрусскоеслово")); + crate::add_word("Зкщфрусскоеслово"); + assert!(is_correct("зкщфрусскоеслово")); + } + + /// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Russian checks + /// those words against the dictionary instead of accepting them. + #[test] + fn proofing_options_reach_the_russian_check() { + test_dict::install(); + let lang = crate::ProofLang::Ru; + let defaults = crate::Options::default(); + let check_caps = crate::Options { ignore_uppercase: false, ..defaults }; + let check_digits = crate::Options { ignore_numbers: false, ..defaults }; + assert!(crate::is_correct_in_with("ПРЕВЕТ", lang, &defaults), "short capitals are abbreviations by default"); + assert!(!crate::is_correct_in_with("ПРЕВЕТ", lang, &check_caps)); + assert!(crate::is_correct_in_with("ПРИВЕТ", lang, &check_caps)); + assert!(crate::is_correct_in_with("ИИ-консультант", lang, &defaults)); + assert!(!crate::is_correct_in_with("ИИ-консультант", lang, &check_caps), "a part in capitals is checked too"); + assert!(crate::is_correct_in_with("превет2", lang, &defaults)); + assert!(!crate::is_correct_in_with("превет2", lang, &check_digits)); + assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too"); + assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); + assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults"); + } +} diff --git a/crates/proof/src/ru/suggest.rs b/crates/proof/src/ru/suggest.rs new file mode 100644 index 00000000..135f4152 --- /dev/null +++ b/crates/proof/src/ru/suggest.rs @@ -0,0 +1,258 @@ +//! Russian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with +//! Russian's own letter groups and confusions, since Russian Hunspell dictionaries carry no `MAP` +//! or `REP` tables (`е`/`ё` first, then unstressed vowels, voiced and voiceless pairs, `жы` → +//! `жи`, `тся`/`ться`, dropped or doubled consonants; up to three letters changed), and its rule +//! for a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`). + +use crate::affix::AffixDict; +use crate::affix_suggest::{COST_OTHER_MAP, COST_SPLIT, COST_SPLIT_LONG, Split, Tuning}; + +/// Letters confused with each other: a slip inside a group is cheaper than another typo. +const MAP: &[&str] = &["её", "оа", "еия", "зс", "дт", "бп", "вф", "гк", "жш", "ьъ", "еэ", "ыи"]; +/// Common confusions: a written sequence and what was meant. +const REP: &[(&str, &str)] = &[ + ("жы", "жи"), + ("шы", "ши"), + ("чя", "ча"), + ("щя", "ща"), + ("чю", "чу"), + ("щю", "щу"), + ("тся", "ться"), + ("ться", "тся"), + ("сч", "щ"), + ("щ", "сч"), + ("нн", "н"), + ("н", "нн"), + ("сс", "с"), + ("с", "сс"), + ("лл", "л"), + ("л", "лл"), + ("тт", "т"), + ("т", "тт"), + ("мм", "м"), + ("м", "мм"), + ("рр", "р"), + ("р", "рр"), + ("ье", "ъе"), + ("ъе", "ье"), + ("раст", "рост"), + ("рост", "раст"), + ("лож", "лаг"), + ("лаг", "лож"), + // Unpronounced consonants: `здраствуй`, `чуствовать`, `солнце`, `лесница`, `праздник`. + ("аст", "авст"), + ("уст", "увст"), + ("лнц", "лц"), + ("лц", "лнц"), + ("сн", "стн"), + ("стн", "сн"), + ("зн", "здн"), + ("здн", "зн"), + ("ц", "тс"), + ("тс", "ц"), +]; +/// Short words often run into the next one (`не знаю`, `в общем`, `на верху`, `потому что`). +const LEADING_WORDS: &[&str] = &[ + "не", + "ни", + "в", + "во", + "на", + "с", + "со", + "к", + "ко", + "о", + "об", + "обо", + "от", + "ото", + "до", + "из", + "изо", + "за", + "по", + "под", + "над", + "при", + "про", + "без", + "для", + "у", + "а", + "и", + "но", + "да", + "же", + "как", + "так", + "то", + "что", + "уже", + "еще", + "ещё", + "все", + "всё", + "потому", + "чтобы", + "это", +]; +/// Prepositions among [`LEADING_WORDS`]. +const PREPOSITIONS: &[&str] = &[ + "в", "во", "на", "с", "со", "к", "ко", "о", "об", "обо", "от", "ото", "до", "из", "изо", "за", "по", "под", "над", "при", "про", "без", "для", + "у", +]; +/// Short words often run into the one before (`мог бы`, `он же`, `так ли`). +const TRAILING_WORDS: &[&str] = &["ли", "же", "бы", "ж", "б", "что"]; +/// The Russian alphabet, when the dictionary has no `TRY` line. +const ALPHABET: &str = "оеаинтсрвлкмдпуяыьгзбчйхжшюцщэфъё"; + +/// `е` written for `ё`, or the other way round, is barely a slip. +const COST_YO: u32 = 3; +/// A one- or two-letter preposition run into the next word (`вдоме` → `в доме`). +const COST_SPLIT_PREPOSITION: u32 = 17; + +/// What Russian adds to its dictionary's tables. +const TUNING: Tuning = + Tuning { split: Split::Custom(split), alphabet: ALPHABET, map: MAP, rep: REP, rep_cost: 9, max_map_changes: 3, map_cost, ..Tuning::DEFAULT }; + +/// The cost of changing `from` to `to` within a letter group: `е`/`ё` cheapest. +fn map_cost(from: char, to: char) -> u32 { + if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_OTHER_MAP } +} + +/// A missing space: cheap after a short leading word or before a short trailing one, dearer after +/// a short preposition or between two longer words, never leaving a stray short half. +fn split(a: &str, b: &str) -> Option { + let long = |p: &str| p.chars().count() >= 3; + // A preposition takes a noun, not a verb (`с генерировать` isn't a fix for + // `сгенерировать`); after one or two letters a split is likelier wrong than a typo. + let preposition = PREPOSITIONS.contains(&a); + if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) { + return None; + } + if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) { + Some(COST_SPLIT_PREPOSITION) + } else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { + Some(COST_SPLIT) + } else if long(a) && long(b) { + Some(COST_SPLIT_LONG) + } else { + None + } +} + +/// Spelling suggestions for a Russian word, best first, at most `max`, in the word's letter case. +pub fn suggest(word: &str, max: usize) -> Vec { + suggest_in(&super::dictionary(), word, max) +} + +/// [`suggest`] against a given dictionary. +pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { + crate::affix_suggest::suggest_in(d, word, max, &TUNING) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn has(word: &str, want: &str) { + let s = suggest(word, 8); + assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); + } + + /// Russian's own costs: `е`/`ё` is the cheapest slip, so a `ё` and one more group slip + /// (made-up `нёснаю`) beat a missing space; a REP confusion beats a `ё` and a group slip; + /// a preposition is never split off a verb, and after a one-letter one a typo is likelier. + #[test] + fn russian_costs_rank_its_slips() { + let dict = |words: &[&str]| AffixDict::from_hunspell("", &format!("{}\n{}\n", words.len(), words.join("\n")), &[]).unwrap(); + assert_eq!(suggest_in(&dict(&["не", "знаю", "нёснаю"]), "незнаю", 1), ["нёснаю"]); + assert_eq!(suggest_in(&dict(&["еча", "ёчи"]), "ечя", 1), ["еча"]); + assert!(!suggest_in(&dict(&["с", "генерировать"]), "сгенерировать", 8).contains(&"с генерировать".into())); + let s = suggest_in(&dict(&["в", "доме"]), "вдоме", 4); + assert_eq!(s.first().map(String::as_str), Some("доме"), "{s:?}"); + assert!(s.contains(&"в доме".into()), "{s:?}"); + } + + /// A word the dictionary marks NOSUGGEST (a vulgarism, a rare form) is correct but never + /// offered, in any round or letter case. Russian's MAP and REP tables are its own, so the + /// shared Latin-script check doesn't reach it. + #[test] + fn nosuggest_words_are_never_offered() { + let offered = |aff: &str, input: &str| { + let d = AffixDict::from_hunspell(aff, "2\nпривет/N\nпредмет/N\n", &[]).unwrap(); + suggest_in(&d, input, 16) + }; + assert!(offered("SET UTF-8\n", "превет").contains(&"привет".into()), "offered without the flag declared"); + for input in ["превет", "ПРЕВЕТ", "предмот", "приветт", "привте", "привет"] { + let s = offered("SET UTF-8\nNOSUGGEST N\n", input); + assert!(!s.iter().any(|w| w.to_lowercase().contains("привет") || w.to_lowercase().contains("предмет")), "{input}: {s:?}"); + } + } + + #[test] + fn russian_misspellings_get_the_right_suggestion() { + crate::ru::test_dict::install(); + has("превет", "привет"); + has("сдесь", "здесь"); + has("жыть", "жить"); + has("чясто", "часто"); + has("незнаю", "не знаю"); + has("вобщем", "в общем"); + has("здраствуй", "здравствуй"); + has("москва", "Москва"); + has("Превет", "Привет"); + has("ПРЕВЕТ", "ПРИВЕТ"); + has("читаеться", "читается"); + has("краснного", "красного"); + has("западнный", "западный"); + assert!(!suggest("сгенерировать", 8).contains(&"с генерировать".to_string())); + } + + #[test] + fn the_likeliest_fix_comes_first() { + crate::ru::test_dict::install(); + assert_eq!(suggest("превет", 3).first().map(String::as_str), Some("привет")); + assert_eq!(suggest("незнаю", 3).first().map(String::as_str), Some("не знаю")); + assert_eq!(suggest("жыть", 3).first().map(String::as_str), Some("жить")); + assert_eq!(suggest("зеленый", 3).first().map(String::as_str), Some("зелёный")); + } + + #[test] + fn edge_cases_stay_empty_or_bounded() { + crate::ru::test_dict::install(); + assert!(suggest("", 5).is_empty()); + assert!(suggest("дом", 0).is_empty()); + assert!(suggest(&"а".repeat(41), 5).is_empty()); + assert!(suggest("два слова", 5).is_empty()); + assert!(suggest("привет", 5).len() <= 5); + assert!(!suggest("привет", 5).contains(&"привет".to_string())); + } + + #[test] + fn distance_weighs_letter_groups_half() { + use crate::affix_suggest::{fold, weighted_distance}; + let map: Vec> = MAP.iter().map(|g| g.chars().collect()).collect(); + let related = |a: char, b: char| fold(a, &map) == fold(b, &map); + let c = |s: &str| s.chars().collect::>(); + assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), &related, 9), 1); + assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), &related, 9), 1); + assert_eq!(weighted_distance(&c("дом"), &c("дмо"), &related, 9), 2); + assert_eq!(weighted_distance(&c("дом"), &c("домами"), &related, 9), 6); + assert_eq!(weighted_distance(&c("дом"), &c("книга"), &related, 2), 3); + } + + #[test] + fn suggestions_are_fast_enough() { + crate::ru::test_dict::install(); + let _ = suggest("дом", 1); + let t = std::time::Instant::now(); + for w in ["ыкщзвотрусскогословаааа", "здраствуйте", "краснннного"] { + let _ = suggest(w, 6); + } + let ms = t.elapsed().as_secs_f64() * 1000.0; + eprintln!("3 suggestion calls: {ms:.0} ms"); + assert!(ms < 20_000.0, "{ms} ms"); + } +} diff --git a/crates/proof/src/ru/test_dict.rs b/crates/proof/src/ru/test_dict.rs new file mode 100644 index 00000000..947f3552 --- /dev/null +++ b/crates/proof/src/ru/test_dict.rs @@ -0,0 +1,130 @@ +//! A tiny synthetic Russian Hunspell dictionary for tests (written for these tests; not taken from +//! any real dictionary). Real dictionaries are installed on the system, never bundled. + +/// Affix rules: the endings of a hard masculine noun, a feminine noun in `-а`, hard and soft +/// adjectives and a verb in `-ать`, plus the letters suggestions try. +pub const AFF: &str = "SET UTF-8 +TRY оеаинтсрвлкмдпуяыьгзбчйхжшюцщэфъё +SFX A Y 9 +SFX A 0 а . +SFX A 0 у . +SFX A 0 ом . +SFX A 0 е . +SFX A 0 ы . +SFX A 0 ов . +SFX A 0 ам . +SFX A 0 ами . +SFX A 0 ах . +SFX D Y 5 +SFX D а ы а +SFX D а е а +SFX D а у а +SFX D а ой а +SFX D а ою а +SFX B Y 11 +SFX B ый ого ый +SFX B ый ому ый +SFX B ый ым ый +SFX B ый ом ый +SFX B ый ая ый +SFX B ый ой ый +SFX B ый ую ый +SFX B ый ое ый +SFX B ый ые ый +SFX B ый ых ый +SFX B ый ыми ый +SFX E Y 8 +SFX E ий его ий +SFX E ий ему ий +SFX E ий им ий +SFX E ий ей ий +SFX E ий яя ий +SFX E ий ее ий +SFX E ий ие ий +SFX E ий их ий +SFX C Y 11 +SFX C ть ю ть +SFX C ть ешь ть +SFX C ть ет ть +SFX C ть ем ть +SFX C ть ете ть +SFX C ть ют ть +SFX C ть ются ть +SFX C ть ется ть +SFX C ть л ть +SFX C ть ла ть +SFX C ть ться ть +"; + +/// Stems (the count line first, as in Hunspell files). +pub const DIC: &str = "62 +привет +здесь +здравствуй +жить +часто +знаю +знать/C +не +в +общем +общий/E +дом/A +стол/A +город/A +Москва/D +книга/D +красный/B +новый/B +белый/B +западный/B +синий/E +зелёный/B +русский +английский +читать/C +работать/C +кто +где +что +давай +всё +все +ещё +ёлка/D +север/A +из +за +на +и +а +но +или +он +она +я +ты +мы +вы +это +так +как +когда +который/B +чтобы +потому +если +пришёл +сказал +вскрытый/B +высоко +эластичный/B +консультант/A +"; + +/// Install the test dictionary as the Russian one. +pub fn install() { + if let Err(e) = super::install_dictionary(AFF, DIC) { + panic!("test dictionary: {e}"); + } +} diff --git a/crates/proof/src/slot.rs b/crates/proof/src/slot.rs new file mode 100644 index 00000000..6255ae5a --- /dev/null +++ b/crates/proof/src/slot.rs @@ -0,0 +1,210 @@ +//! The installed Hunspell dictionary of a proofing language, in one shape for every language: +//! found on first use, replaced with [`DictSlot::install`] and forgotten with [`DictSlot::reload`]. + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard}; + +use crate::affix::AffixDict; + +/// A dictionary a language reads: the affix engine's ([`AffixDict`]) or Turkish's own. +pub(crate) trait Dictionary: Default + Send + Sync { + /// The installed `.aff` and `.dic` files, read. + fn load(aff: &Path, dic: &Path) -> Result; + /// Hunspell `.aff` and `.dic` text (decoded to UTF-8), parsed. + fn parse(aff: &str, dic: &str) -> Result; + /// No words: the language has no spelling marks. + fn is_empty(&self) -> bool; +} + +impl Dictionary for AffixDict { + fn load(aff: &Path, dic: &Path) -> Result { + crate::hunspell::load(aff, dic) + } + + fn parse(aff: &str, dic: &str) -> Result { + AffixDict::from_hunspell(aff, dic, &[]) + } + + fn is_empty(&self) -> bool { + AffixDict::is_empty(self) + } +} + +/// What every language's slot answers, whatever its dictionary type (the language table in +/// lib.rs holds them side by side). +pub(crate) trait Slot: Sync { + /// See [`DictSlot::name`]. + fn name(&self) -> &'static str; + /// See [`DictSlot::installed_path`]. + fn installed_path(&self) -> Option; + /// See [`DictSlot::has_dictionary`]. + fn has_dictionary(&self) -> bool; + /// See [`DictSlot::reload`]. + fn reload(&self); +} + +impl Slot for DictSlot { + fn name(&self) -> &'static str { + DictSlot::name(self) + } + + fn installed_path(&self) -> Option { + DictSlot::installed_path(self) + } + + fn has_dictionary(&self) -> bool { + DictSlot::has_dictionary(self) + } + + fn reload(&self) { + DictSlot::reload(self); + } +} + +/// The dictionary in use and whether it came from [`DictSlot::install`] rather than a file on +/// disk (it then has no file to report, even if a matching file also happens to exist): one lock, +/// so a reload and an install can't interleave their two fields. +struct State { + dict: Option>, + in_memory: bool, + /// Counts reloads, so a load that started before one is not kept. + reloads: u64, +} + +/// One language's dictionary. +pub(crate) struct DictSlot { + /// File names the dictionary may have, the usual one first (also the key of its load error). + names: &'static [&'static str], + /// The language's English name, for the log. + english: &'static str, + state: RwLock>, +} + +impl DictSlot { + pub(crate) const fn new(names: &'static [&'static str], english: &'static str) -> Self { + DictSlot { names, english, state: RwLock::new(State { dict: None, in_memory: false, reloads: 0 }) } + } + + fn read(&self) -> RwLockReadGuard<'_, State> { + self.state.read().unwrap_or_else(PoisonError::into_inner) + } + + fn write(&self) -> RwLockWriteGuard<'_, State> { + self.state.write().unwrap_or_else(PoisonError::into_inner) + } + + /// The Hunspell name the dictionary is known by (`pl_PL`): the first of its file names. + pub(crate) fn name(&self) -> &'static str { + self.names.first().copied().unwrap_or_default() + } + + /// The file names the dictionary may have, the usual one first. + #[cfg(test)] + pub(crate) fn names(&self) -> &'static [&'static str] { + self.names + } + + /// The dictionary: the installed one (read on first use), one given with [`Self::install`], or + /// an empty one when none is available (the language's spelling is off then). + pub(crate) fn get(&self) -> Arc { + let reloads = { + let s = self.read(); + if let Some(d) = s.dict.as_ref() { + return d.clone(); + } + s.reloads + }; + self.keep(reloads, Arc::new(self.load_installed())) + } + + /// Keep a dictionary loaded after `reloads` reloads. If the folder was reloaded since, the + /// load may have read the old folder: it serves this one check and the next one loads again. + fn keep(&self, reloads: u64, loaded: Arc) -> Arc { + let mut s = self.write(); + if s.reloads != reloads { + return s.dict.clone().unwrap_or(loaded); + } + // Another thread (or an install) may have filled it meanwhile: that one wins. + s.dict.get_or_insert(loaded).clone() + } + + fn load_installed(&self) -> D { + let name = self.name(); + let Some((aff, dic)) = crate::hunspell::find(self.names) else { + crate::hunspell::clear_load_error(name); + log::info!("{} spelling: no Hunspell {name} dictionary installed", self.english); + return D::default(); + }; + crate::hunspell::note_load(name, &dic, D::load(&aff, &dic)).unwrap_or_else(|e| { + log::warn!("{} spelling: {e}", self.english); + D::default() + }) + } + + /// Use this dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), for + /// example one the user picked. On error the current dictionary stays. + pub(crate) fn install(&self, aff: &str, dic: &str) -> Result<(), String> { + let d = D::parse(aff, dic)?; + let mut slot = self.write(); + slot.dict = Some(Arc::new(d)); + slot.in_memory = true; + Ok(()) + } + + /// Forget the dictionary, so the next check looks for an installed one again (after + /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). + pub(crate) fn reload(&self) { + let mut slot = self.write(); + slot.dict = None; + slot.in_memory = false; + slot.reloads = slot.reloads.wrapping_add(1); + } + + /// The `.dic` file of the installed dictionary, searching the dictionaries folder and the + /// system locations now (File › Options › Proofing shows it); `None` when there is none, or + /// when the dictionary in use was given with [`Self::install`] and so has no file. + pub(crate) fn installed_path(&self) -> Option { + if self.read().in_memory { + return None; + } + crate::hunspell::find(self.names).map(|(_, dic)| dic) + } + + /// Is a dictionary available (spelling is checked)? + pub(crate) fn has_dictionary(&self) -> bool { + !self.get().is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + static SLOT: DictSlot = DictSlot::new(&["zz_SLOT_TEST"], "Test"); + + #[test] + fn an_installed_dictionary_has_no_file_until_reloaded() { + assert!(!SLOT.has_dictionary(), "nothing installed under this name"); + assert!(SLOT.install("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused"); + assert!(!SLOT.has_dictionary(), "a refused install keeps the current one"); + SLOT.install("", "1\nkot\n").unwrap(); + assert!(SLOT.has_dictionary() && SLOT.get().check("kot")); + assert_eq!(SLOT.installed_path(), None); + SLOT.reload(); + assert!(!SLOT.has_dictionary(), "reloaded from disk, where there is none"); + assert_eq!(SLOT.name(), "zz_SLOT_TEST"); + } + + /// A load that started before a dictionaries-folder change is not kept after its reload. + #[test] + fn a_load_from_before_a_reload_is_not_kept() { + let slot: DictSlot = DictSlot::new(&["zz_SLOT_RELOAD_TEST"], "Test"); + let before = slot.read().reloads; + slot.reload(); + let _ = slot.keep(before, Arc::new(AffixDict::default())); + assert!(slot.read().dict.is_none(), "the next check loads from the new folder"); + let now = slot.read().reloads; + let _ = slot.keep(now, Arc::new(AffixDict::default())); + assert!(slot.read().dict.is_some(), "a load since the reload is kept"); + } +} diff --git a/crates/proof/src/spelling.rs b/crates/proof/src/spelling.rs new file mode 100644 index 00000000..ab5e520e --- /dev/null +++ b/crates/proof/src/spelling.rs @@ -0,0 +1,52 @@ +//! The start of every Latin-script dictionary language's spelling check (Czech, Croatian, +//! Indonesian, Italian, Polish, Portuguese, Spanish): the words that need no lookup, under the +//! File › Options › Proofing choices. Each language goes on from there with its own rules. + +use crate::Options; +use crate::affix::AffixDict; + +/// Is `c` a letter of the Latin script (the only one these dictionaries cover)? +fn latin(c: char) -> bool { + c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) +} + +/// `word` trimmed of quotes and apostrophes, when `d` must say whether it is right; `None` when it +/// is right without a lookup: there is no dictionary (no spelling marks), the word is a single +/// letter or has no letters, it has digits or is a short word in capitals (an acronym) and the +/// options skip those, or it is in another script. +pub(crate) fn word_to_check<'a>(d: &AffixDict, word: &'a str, opts: &Options) -> Option<&'a str> { + if d.is_empty() { + return None; + } + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); + let w = w.trim_matches(['\'', '’']); + let n = w.chars().count(); + if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + return None; + } + if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { + return None; + } + w.chars().any(latin).then_some(w) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn words_needing_no_lookup() { + let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap(); + let defaults = Options::default(); + assert_eq!(word_to_check(&AffixDict::default(), "kot", &defaults), None, "no dictionary"); + for w in ["", "a", "2026", "PKO", "B2B", "日本語", "--"] { + assert_eq!(word_to_check(&d, w, &defaults), None, "{w}"); + } + assert_eq!(word_to_check(&d, "„kot”", &defaults), Some("kot")); + assert_eq!(word_to_check(&d, "'kot'", &defaults), Some("kot")); + let strict = Options { ignore_uppercase: false, ignore_numbers: false, ..defaults }; + assert_eq!(word_to_check(&d, "PKO", &strict), Some("PKO")); + assert_eq!(word_to_check(&d, "B2B", &strict), Some("B2B")); + assert_eq!(word_to_check(&d, "PRZEDSIĘBIORSTWO", &defaults), Some("PRZEDSIĘBIORSTWO"), "only short capitals are acronyms"); + } +} diff --git a/crates/proof/src/tr/dict.rs b/crates/proof/src/tr/dict.rs new file mode 100644 index 00000000..8cbacf02 --- /dev/null +++ b/crates/proof/src/tr/dict.rs @@ -0,0 +1,144 @@ +//! Turkish Hunspell dictionary parser with `FLAG num` support. +//! +//! Turkish uses `FLAG num`, storing thousands of numerical suffix flags (0..6465). +//! This engine parses `/usr/share/hunspell/tr_TR.aff` and `.dic` safely without panics. + +use std::collections::HashMap; + +/// A suffix rule for Turkish. +#[derive(Clone, Debug)] +pub struct TrSuffixRule { + pub strip: Box, + pub add: Box, + pub cond: Box, +} + +/// Dedicated Turkish Hunspell dictionary engine. +#[derive(Clone, Debug, Default)] +pub struct TrDict { + /// Stem -> list of numerical flags it carries. + stems: HashMap>, + /// Flag number -> list of suffix rules. + rules: HashMap>, + /// Suffix 'add' string -> list of (flag, rule_index). + suffix_index: HashMap>, + /// Longest suffix 'add' string in characters. + max_suffix_len: usize, +} + +impl crate::slot::Dictionary for TrDict { + fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result { + super::load(aff, dic) + } + + fn parse(aff: &str, dic: &str) -> Result { + TrDict::from_hunspell(aff, dic) + } + + fn is_empty(&self) -> bool { + TrDict::is_empty(self) + } +} + +impl TrDict { + /// Returns true if empty (no dictionary loaded). + pub fn is_empty(&self) -> bool { + self.stems.is_empty() + } + + /// Load from `.aff` and `.dic` contents. + pub fn from_hunspell(aff: &str, dic: &str) -> Result { + let mut rules: HashMap> = HashMap::new(); + let mut max_suffix_len = 0; + + for line in aff.lines() { + let parts: Vec<&str> = line.split_whitespace().collect(); + // SFX [] + if parts.len() >= 4 && parts[0] == "SFX" { + let Ok(flag) = parts[1].parse::() else { continue }; + // Skip header line: `SFX Y/N ` + if parts.len() == 4 && (parts[2] == "Y" || parts[2] == "N") && parts[3].parse::().is_ok() { + continue; + } + let strip = if parts[2] == "0" { "" } else { parts[2] }; + let add = if parts[3] == "0" { "" } else { parts[3] }; + let cond = if parts.len() > 4 { parts[4] } else { "." }; + + max_suffix_len = max_suffix_len.max(add.chars().count()); + + rules.entry(flag).or_default().push(TrSuffixRule { strip: strip.into(), add: add.into(), cond: cond.into() }); + } + } + + let mut suffix_index: HashMap> = HashMap::new(); + for (&flag, r_list) in &rules { + for (idx, r) in r_list.iter().enumerate() { + if let Ok(idx_u16) = u16::try_from(idx) { + suffix_index.entry(r.add.to_string()).or_default().push((flag, idx_u16)); + } + } + } + + let mut stems: HashMap> = HashMap::new(); + for (i, line) in dic.lines().enumerate() { + if i == 0 && line.trim().parse::().is_ok() { + continue; + } + let line = line.trim(); + if line.is_empty() { + continue; + } + if let Some((word, flags_part)) = line.split_once('/') { + let flags: Vec = flags_part.split(',').filter_map(|f| f.parse::().ok()).collect(); + stems.insert(word.to_string(), flags.into_boxed_slice()); + } else { + stems.insert(line.to_string(), Box::new([])); + } + } + + Ok(TrDict { stems, rules, suffix_index, max_suffix_len }) + } + + /// Check if word is correct (stem or affixed). + pub fn contains(&self, word: &str) -> bool { + // Longer input is never a dictionary word, and isn't scanned (#3625). + if self.is_empty() || word.is_empty() || word.len() > crate::affix::MAX_WORD_BYTES { + return false; + } + + // Direct stem hit + if self.stems.contains_key(word) { + return true; + } + + // Try stripping suffixes matching the tail + let word_len = word.chars().count(); + let char_indices: Vec<(usize, char)> = word.char_indices().collect(); + + // Check each possible suffix length up to max_suffix_len + for cut_chars in 0..=self.max_suffix_len.min(word_len) { + let byte_idx = if cut_chars == 0 { word.len() } else { char_indices[word_len - cut_chars].0 }; + + let add = &word[byte_idx..]; + let base = &word[..byte_idx]; + + if let Some(matches) = self.suffix_index.get(add) { + for &(flag, rule_idx) in matches { + let Some(rule_list) = self.rules.get(&flag) else { continue }; + let Some(rule) = rule_list.get(rule_idx as usize) else { continue }; + + // Check condition on root + let mut candidate = String::with_capacity(base.len() + rule.strip.len()); + candidate.push_str(base); + candidate.push_str(&rule.strip); + + if self.stems.get(&candidate).is_some_and(|f_list| f_list.contains(&flag)) { + return true; + } + } + } + } + + false + } +} diff --git a/crates/proof/src/tr/grammar.rs b/crates/proof/src/tr/grammar.rs new file mode 100644 index 00000000..b17e89cb --- /dev/null +++ b/crates/proof/src/tr/grammar.rs @@ -0,0 +1,359 @@ +//! Turkish grammar and punctuation checks: cheap, rule-based and tuned to stay quiet when unsure. +//! Messages are in Turkish. +//! +//! - **Repeated word** (`çok çok`), except idiomatic reduplications and doublings (`ALLOWED_REPEATS`: +//! `yavaş yavaş`, `tıkır tıkır`, `adım adım`). +//! - **Space before punctuation** (`ev ,`) and **double spaces**. +//! - **Capital letter at the start of a sentence**, knowing Turkish abbreviations (`vb.`, `vs.`, +//! `prof.`, `dr.`, `mad.`, `yy.`, `s.`…) after which lowercase or numbers may continue. +//! - **No comma before/after conjunctions**: `ve`, `veya`, `yahut`, `ile`, `de`, `da`, `hem ... hem`, +//! `ne ... ne` conjunctions should not have commas immediately before or after them. +//! - **Doubled punctuation** (`..`, `,,`, `;;`). + +use crate::{Issue, IssueKind, ProofLang, words}; + +/// Turkish reduplications and emphatic word pairings that are intentionally repeated. +const ALLOWED_REPEATS: &[&str] = &[ + "yavaş", + "hızlı", + "koşa", + "güle", + "adım", + "kara", + "tıkır", + "şakır", + "akın", + "ağır", + "açık", + "ayrı", + "bire", + "bir", + "boş", + "bölük", + "buram", + "cana", + "can", + "cılk", + "çifter", + "çit", + "dalga", + "damla", + "deste", + "didik", + "dizi", + "dolu", + "deste", + "döne", + "eğri", + "enine", + "epey", + "fıkır", + "fırıl", + "gece", + "gıcır", + "gide", + "günü", + "güzel", + "halk", + "harıl", + "hık", + "hım", + "horul", + "ılık", + "ışıl", + "için", + "içten", + "iğne", + "iri", + "iyiden", + "iyi", + "kaba", + "kaç", + "kat", + "kaynar", + "kelli", + "kese", + "kıtır", + "kıvrım", + "koca", + "kol", + "kucak", + "kütür", + "mırıl", + "öbek", + "parça", + "patır", + "pıtır", + "sabah", + "salına", + "salkım", + "sap", + "sara", + "sarı", + "serin", + "sıra", + "sırım", + "sıkı", + "sırılsıklam", + "tane", + "teker", + "tek", + "tıpış", + "top", + "tutam", + "uzun", + "üçer", + "üst", + "yalan", + "yana", + "yan", + "yarım", + "yer", + "yudum", + "yığın", + "yürekten", + "yüz", + "zaman", + "zor", +]; + +/// Turkish abbreviations that end with a period and after which lower case may continue. +const ABBREVIATIONS: &[&str] = &[ + "vb", "vs", "vd", "bkz", "bknz", "ör", "örnek", "mad", "md", "s", "sf", "sn", "dk", "sa", "yy", "m", "km", "cm", "mm", "gr", "kg", "mg", "dr", + "prof", "doç", "yrd", "av", "ist", "ank", "izm", "cad", "sok", "mah", "apt", "no", "tel", "faks", "tl", "kr", "krş", "a.ş", "ltd", "şti", "t.c", + "tbmm", "meb", "möt", "mö", "ms", +]; + +/// Conjunctions that must not be preceded by a comma. +const CONJUNCTIONS_NO_PRE_COMMA: &[&str] = &["ve", "veya", "yahut", "velev", "veyahut", "de", "da"]; + +/// Conjunctions that must not be followed by a comma. +const CONJUNCTIONS_NO_POST_COMMA: &[&str] = &["ve", "veya", "yahut", "veyahut", "hem", "ne", "ya", "gerek", "ister", "de", "da"]; + +/// Check grammar and punctuation in a Turkish text block. +pub fn check(text: &str) -> Vec { + check_with(text, true) +} + +/// [`check`] where `capital_first` says whether the text's first word must start a sentence with a capital +/// (false for a list item, which may be a fragment). +pub fn check_with(text: &str, capital_first: bool) -> Vec { + let mut v = Vec::new(); + let ws = words(text); + + // 1. Repeated words (`kelime kelime`) + for pair in ws.windows(2) { + let (w1, w2) = match (text.get(pair[0].0..pair[0].1), text.get(pair[1].0..pair[1].1)) { + (Some(a), Some(b)) => (a, b), + _ => continue, + }; + let gap = match text.get(pair[0].1..pair[1].0) { + Some(g) => g, + None => continue, + }; + let only_spaces = gap.chars().all(|c| c == ' ' || c == '\t'); + let lower1 = w1.to_lowercase(); + let lower2 = w2.to_lowercase(); + if only_spaces && lower1 == lower2 && !ALLOWED_REPEATS.contains(&lower1.as_str()) { + v.push(Issue { + start: pair[1].0, + end: pair[1].1, + kind: IssueKind::Grammar, + message: "Yinelenen sözcük".into(), + suggestions: Vec::new(), + lang: ProofLang::Tr, + }); + } + } + + // 2. Spaces before punctuation and double spaces + let chars: Vec<(usize, char)> = text.char_indices().collect(); + for i in 0..chars.len() { + let (idx, c) = chars[i]; + if matches!(c, ',' | '.' | '?' | '!' | ';' | ':') && i > 0 && chars[i - 1].1 == ' ' { + let prev_sp = chars[i - 1].0; + // Leave emoticons and ellipses alone + let is_dot_in_dots = c == '.' && (i + 1 < chars.len() && chars[i + 1].1 == '.'); + if !is_dot_in_dots { + v.push(Issue { + start: prev_sp, + end: idx + c.len_utf8(), + kind: IssueKind::Grammar, + message: "Noktalama işaretinden önce boşluk bırakılmaz".into(), + suggestions: vec![c.to_string()], + lang: ProofLang::Tr, + }); + } + } + // Double space + if c == ' ' && i + 1 < chars.len() && chars[i + 1].1 == ' ' && (i == 0 || chars[i - 1].1 != ' ') { + let mut end = idx + 1; + while end < text.len() && text.as_bytes().get(end) == Some(&b' ') { + end += 1; + } + v.push(Issue { + start: idx, + end, + kind: IssueKind::Grammar, + message: "Fazladan boşluk".into(), + suggestions: vec![" ".into()], + lang: ProofLang::Tr, + }); + } + } + + // 3. Comma before or after conjunctions (`ali, ve veli` or `ve, veli`) + for (i, &(w_start, w_end)) in ws.iter().enumerate() { + let word = match text.get(w_start..w_end) { + Some(w) => w.to_lowercase(), + None => continue, + }; + // Check comma before conjunction + if CONJUNCTIONS_NO_PRE_COMMA.contains(&word.as_str()) + && i > 0 + && let Some(between) = text.get(ws[i - 1].1..w_start) + && let Some(comma_pos) = between.find(',') + { + let actual_pos = ws[i - 1].1 + comma_pos; + v.push(Issue { + start: actual_pos, + end: actual_pos + 1, + kind: IssueKind::Grammar, + message: format!("'{word}' bağlacından önce virgül konulmaz"), + suggestions: vec![String::new()], + lang: ProofLang::Tr, + }); + } + // Check comma after conjunction + if CONJUNCTIONS_NO_POST_COMMA.contains(&word.as_str()) + && i + 1 < ws.len() + && let Some(between) = text.get(w_end..ws[i + 1].0) + && let Some(comma_pos) = between.find(',') + { + let actual_pos = w_end + comma_pos; + v.push(Issue { + start: actual_pos, + end: actual_pos + 1, + kind: IssueKind::Grammar, + message: format!("'{word}' bağlacından sonra virgül konulmaz"), + suggestions: vec![String::new()], + lang: ProofLang::Tr, + }); + } + } + + // 4. Sentence capitalization knowing Turkish abbreviations + for (i, &(w_start, w_end)) in ws.iter().enumerate() { + if i == 0 { + // First word in paragraph + if capital_first + && let Some(w) = text.get(w_start..w_end) + && w.chars().next().is_some_and(char::is_lowercase) + { + let cap = turkish_capitalize(w); + v.push(Issue { + start: w_start, + end: w_end, + kind: IssueKind::Grammar, + message: "Cümle büyük harfle başlar".into(), + suggestions: vec![cap], + lang: ProofLang::Tr, + }); + } + continue; + } + + // Check if previous character sequence ended a sentence + let prev_end = ws[i - 1].1; + let prev_word = text.get(ws[i - 1].0..prev_end).unwrap_or(""); + if let Some(gap) = text.get(prev_end..w_start) { + let trimmed = gap.trim(); + if (trimmed.ends_with('.') || trimmed.ends_with('?') || trimmed.ends_with('!')) + && !trimmed.contains("...") + && !ABBREVIATIONS.contains(&prev_word.to_lowercase().as_str()) + && let Some(w) = text.get(w_start..w_end) + && w.chars().next().is_some_and(char::is_lowercase) + { + let cap = turkish_capitalize(w); + v.push(Issue { + start: w_start, + end: w_end, + kind: IssueKind::Grammar, + message: "Cümle büyük harfle başlar".into(), + suggestions: vec![cap], + lang: ProofLang::Tr, + }); + } + } + } + + v.sort_by_key(|issue| (issue.start, issue.end)); + v +} + +/// Capitalize according to Turkish rules (`i` -> `İ`, `ı` -> `I`). +pub fn turkish_capitalize(s: &str) -> String { + let mut chars = s.chars(); + let Some(first) = chars.next() else { return String::new() }; + let first_cap = match first { + 'i' => 'İ', + 'ı' => 'I', + other => other.to_uppercase().next().unwrap_or(other), + }; + let mut out = String::with_capacity(s.len() + 1); + out.push(first_cap); + out.push_str(chars.as_str()); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn clean_turkish_text_produces_no_issues() { + let text = "Bu bir test cümlesidir. Kitap ve defter aldım. Çocuk yavaş yavaş yürüdü."; + assert!(check(text).is_empty()); + } + + #[test] + fn repeated_words_flagged_except_allowed() { + let text = "Gitti gitti ama bulamadı."; + let issues = check(text); + assert_eq!(issues.len(), 1); + assert_eq!(issues[0].message, "Yinelenen sözcük"); + + // Allowed repeat + let text2 = "Çocuk yavaş yavaş ilerliyordu."; + assert!(check(text2).is_empty()); + } + + #[test] + fn space_before_punctuation_flagged() { + let text = "Merhaba , dünya!"; + let issues = check(text); + assert_eq!(issues.len(), 1); + assert_eq!(issues[0].message, "Noktalama işaretinden önce boşluk bırakılmaz"); + } + + #[test] + fn comma_around_conjunctions_flagged() { + let text = "Elma, ve armut."; + let issues = check(text); + assert_eq!(issues.len(), 1); + assert!(issues[0].message.contains("virgül konulmaz")); + + let text2 = "Bana kalırsa bu mesele de, haklı olan taraf bizdik."; + let issues2 = check(text2); + assert_eq!(issues2.len(), 1); + assert!(issues2[0].message.contains("'de' bağlacından sonra virgül konulmaz")); + } + + #[test] + fn sentence_capitalization_respects_abbreviations() { + let text = "Dr. ahmet geldi. Fakat geç kaldı."; + let issues = check(text); + // "Dr. ahmet" does not flag "ahmet" because "Dr." is in ABBREVIATIONS. + assert!(issues.is_empty()); + } +} diff --git a/crates/proof/src/tr/hyphen.rs b/crates/proof/src/tr/hyphen.rs new file mode 100644 index 00000000..a8a48108 --- /dev/null +++ b/crates/proof/src/tr/hyphen.rs @@ -0,0 +1,98 @@ +//! Turkish hyphenation by rule ([`ProofLang::Tr`](crate::ProofLang::Tr)). +//! +//! Turkish words are split at syllables according to Turkish Language Association (TDK) rules. +//! Because Turkish syllable boundaries are fully regular, they can be determined directly by rule +//! without needing external pattern files or word lists: +//! - Turkish vowels: `a`, `e`, `ı`, `i`, `o`, `ö`, `u`, `ü` (both lowercase and uppercase). +//! - A single consonant between two vowels joins the following syllable (`ba-lık`, `ö-ğren-ci`). +//! - Of two consonants between two vowels, the first stays with the preceding syllable and the +//! second joins the next (`alt-lık`, `sev-gi`, `tür-kçe`). +//! - Of three consonants between two vowels, the first two stay with the preceding syllable and the +//! third joins the next (`türk-çe`, `sürp-riz`). +//! - In compound words, syllable division follows pronunciation across the boundary (`ba-şöğ-ret-men`, +//! `il-ko-kul`), which matches the general vowel/consonant distribution rules. +//! +//! A single letter left at the end or start of a line is ruled out by the paragraph's +//! [`Limits`](crate::hyphen::Limits). + +/// Is `c` a Turkish vowel? +pub fn is_vowel(c: char) -> bool { + matches!(c, 'a' | 'e' | 'ı' | 'i' | 'o' | 'ö' | 'u' | 'ü' | 'A' | 'E' | 'I' | 'İ' | 'O' | 'Ö' | 'U' | 'Ü') +} + +/// Is `c` a Turkish letter? +pub(crate) fn is_letter(c: char) -> bool { + c.is_alphabetic() +} + +/// Break points of a Turkish word (char indices; a hyphen goes before that char). +/// Empty for words with fewer than two vowels or containing non-alphabetic characters. +pub fn points(word: &[char]) -> Vec { + if word.len() < 2 || !word.iter().all(|&c| is_letter(c)) { + return Vec::new(); + } + let vowels: Vec = (0..word.len()).filter(|&i| is_vowel(word[i])).collect(); + if vowels.len() < 2 { + return Vec::new(); + } + let mut out = Vec::new(); + for pair in vowels.windows(2) { + let (v1, v2) = (pair[0], pair[1]); + if v2 <= v1 { + continue; + } + let num_consonants = v2 - v1 - 1; + let pt = match num_consonants { + 0 => v1 + 1, + 1 => v2 - 1, + 2 => v1 + 2, + _ => v2 - 1, + }; + if pt > v1 && pt <= v2 { + out.push(pt); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn split(w: &str) -> String { + let chars: Vec = w.chars().collect(); + let pts = points(&chars); + let mut out = String::new(); + for (i, c) in chars.iter().enumerate() { + if pts.contains(&i) { + out.push('-'); + } + out.push(*c); + } + out + } + + #[test] + fn syllables_follow_turkish_rules() { + assert_eq!(split("balık"), "ba-lık"); + assert_eq!(split("araba"), "a-ra-ba"); + assert_eq!(split("öğrenci"), "öğ-ren-ci"); + assert_eq!(split("altlık"), "alt-lık"); + assert_eq!(split("sevgi"), "sev-gi"); + assert_eq!(split("türkçe"), "türk-çe"); + assert_eq!(split("saat"), "sa-at"); + assert_eq!(split("şiir"), "şi-ir"); + assert_eq!(split("ilkokul"), "il-ko-kul"); + assert_eq!(split("başöğretmen"), "ba-şöğ-ret-men"); + } + + #[test] + fn short_or_non_words_stay_unsplit() { + assert_eq!(split(""), ""); + assert_eq!(split("a"), "a"); + assert_eq!(split("ve"), "ve"); + assert_eq!(split("Türk"), "Türk"); + assert_eq!(split("123"), "123"); + assert_eq!(split("araba-ev"), "araba-ev"); + } +} diff --git a/crates/proof/src/tr/mod.rs b/crates/proof/src/tr/mod.rs new file mode 100644 index 00000000..967148d8 --- /dev/null +++ b/crates/proof/src/tr/mod.rs @@ -0,0 +1,164 @@ +//! Turkish proofing. +//! +//! - **Spelling** against the Hunspell `tr_TR` dictionary installed on the system +//! ([`dictionary`], found by [`crate::hunspell`]: `tr_TR.aff`/`tr_TR.dic` from the OS, a +//! package manager or LibreOffice; it is not bundled). Without it Turkish text gets no spelling +//! marks. Capitalized stems are proper nouns (`Ankara`); foreign names inflected with an apostrophe +//! are accepted (`WordCraft'ta`), and the user dictionary applies as in English. +//! - **Suggestions** tuned for Turkish ([`suggest`]): ASCII-diacritic restoration (`turkce` -> `Türkçe`), +//! common confusions (`yalnış` -> `yanlış`), detached particles (`geldimi` -> `geldi mi`), and typos. +//! - **Grammar** ([`grammar`]): high-precision rules (reduplications, comma around conjunctions, +//! sentence start with abbreviations). +//! - **Hyphenation** ([`hyphen`]): syllable division by Turkish Language Association (TDK) rules. + +pub mod dict; +pub mod grammar; +pub mod hyphen; +pub mod suggest; +#[cfg(test)] +pub(crate) mod test_dict; + +use std::sync::Arc; + +use crate::slot::DictSlot; + +pub use dict::TrDict; +pub use suggest::suggest; + +/// File names of a Turkish Hunspell dictionary. +const NAMES: &[&str] = &["tr_TR", "tr-TR", "tr"]; + +/// The Turkish dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Turkish"); + +/// The Turkish dictionary: the installed Hunspell `tr_TR` (read on first use), one given with +/// [`install_dictionary`], or an empty one when none is available. +pub fn dictionary() -> Arc { + DICT.get() +} + +/// Read a Turkish Hunspell pair from disk: size-capped and decoded per the `.aff`'s `SET` line, +/// like the other languages' dictionaries (#3661). +pub(crate) fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result { + let (aff_text, dic_text) = crate::hunspell::decode(&crate::hunspell::read_capped(aff)?, &crate::hunspell::read_capped(dic)?)?; + TrDict::from_hunspell(&aff_text, &dic_text) +} + +/// Use this Turkish dictionary from now on (Hunspell `.aff` and `.dic` text). On error the +/// current dictionary stays. +pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { + DICT.install(aff, dic) +} + +/// Is a Turkish dictionary available? +pub fn has_dictionary() -> bool { + DICT.has_dictionary() +} + +/// Is `word` spelled correctly in Turkish? +pub fn is_correct(word: &str) -> bool { + is_correct_with(&dictionary(), word) +} + +/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words that contain +/// numbers off, words with digits are checked against the dictionary too. +pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool { + check(&dictionary(), word, opts) +} + +/// [`is_correct`] against a given dictionary. +pub fn is_correct_with(d: &TrDict, word: &str) -> bool { + check(d, word, &crate::Options::default()) +} + +/// [`is_correct_with_options`] against a given dictionary. +fn check(d: &TrDict, word: &str, opts: &crate::Options) -> bool { + if d.is_empty() { + return true; // No Turkish dictionary installed: no spelling marks + } + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); + let w = w.trim_matches(['\'', '’']); + let n = w.chars().count(); + if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + return true; + } + // Check user dictionary first + if crate::in_user_dictionary(&suggest::turkish_to_lower(w)) || crate::in_user_dictionary(&w.to_lowercase()) { + return true; + } + + // Direct lookup or lowercase lookup + if d.contains(w) { + return true; + } + let lower = suggest::turkish_to_lower(w); + if d.contains(&lower) { + return true; + } + + // Handle apostrophe for proper nouns (e.g., `Ankara'da`, `WordCraft'ta`) + if let Some((stem, _suffix)) = w.split_once(['\'', '’']) + && (d.contains(stem) || d.contains(&suggest::turkish_to_lower(stem))) + { + return true; + } + + // Compound words separated by hyphens (e.g., `afyon-karahisar`) + if w.contains('-') && w.split('-').all(|part| part.is_empty() || check(d, part, opts)) { + return true; + } + + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn without_dictionary_everything_is_correct() { + let empty = TrDict::default(); + assert!(is_correct_with(&empty, "herhangi")); + assert!(is_correct_with(&empty, "xyzabc")); + } + + #[test] + fn apostrophe_and_compounds_are_handled() { + let empty = TrDict::default(); + assert!(is_correct_with(&empty, "Ankara'da")); + assert!(is_correct_with(&empty, "açık-mavi")); + } + + #[test] + fn installed_dictionary_checks_words() { + test_dict::install(); + assert!(is_correct("kitap")); + assert!(is_correct("kitaplar")); + assert!(is_correct("evde")); + assert!(is_correct("olacaklardan")); + assert!(is_correct("sonra")); + assert!(!is_correct("sorna")); + assert!(!is_correct("kitapppp")); + assert!(!is_correct("evdekişşş")); + + // Test user dictionary + assert!(!is_correct("benimbilmedigimkelime")); + crate::add_word("benimbilmedigimkelime"); + assert!(is_correct("benimbilmedigimkelime")); + } + + /// With Ignore words that contain numbers off, Turkish checks words with digits against the + /// dictionary instead of accepting them, hyphen parts included. + #[test] + fn proofing_options_reach_the_turkish_check() { + test_dict::install(); + let lang = crate::ProofLang::Tr; + let defaults = crate::Options::default(); + let check_digits = crate::Options { ignore_numbers: false, ..defaults }; + assert!(crate::is_correct_in_with("kitapp2", lang, &defaults)); + assert!(!crate::is_correct_in_with("kitapp2", lang, &check_digits)); + assert!(!crate::is_correct_in_with("ev-kitapp2", lang, &check_digits)); + assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); + assert!(is_correct("kitapp2"), "direct checks keep the defaults"); + } +} diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs new file mode 100644 index 00000000..08874420 --- /dev/null +++ b/crates/proof/src/tr/suggest.rs @@ -0,0 +1,328 @@ +//! Turkish spelling suggestions, ranked by likelihood. +//! +//! Candidates are generated and ranked by cost: +//! 1. Capitalization / case adjustments (especially dotted/dotless I: `istanbul` -> `İstanbul`). +//! 2. Missing Turkish diacritics / ASCII substitutions (most frequent Turkish typing slip): +//! `c` <-> `ç`, `g` <-> `ğ`, `i` <-> `ı`, `o` <-> `ö`, `s` <-> `ş`, `u` <-> `ü` (`turkce` -> `Türkçe`). +//! 3. Common spelling mistakes and phonetic confusions (`REP` table): +//! `yalnış` -> `yanlış`, `yanlız` -> `yalnız`, `herşey` -> `her şey`, `bir kaç` -> `birkaç`. +//! 4. Missing space for detached particles and conjunctions (`bende` -> `ben de`, `geldinmi` -> `geldin mi`). +//! 5. Single typo (Damerau-Levenshtein distance <= 2). + +use std::collections::HashSet; + +use crate::tr::dict::TrDict; + +/// Common misspellings and their canonical replacements in Turkish. +const COMMON_REPLACEMENTS: &[(&str, &str)] = &[ + ("yalnış", "yanlış"), + ("yanlız", "yalnız"), + ("herşey", "her şey"), + ("herkez", "herkes"), + ("bir kaç", "birkaç"), + ("birşey", "bir şey"), + ("tabiki", "tabii ki"), + ("orjinal", "orijinal"), + ("kiprik", "kirpik"), + ("şöför", "şoför"), + ("sarmısak", "sarımsak"), + ("makina", "makine"), + ("ünvan", "unvan"), + ("rastlantı", "rastlantı"), + ("labaratuvar", "laboratuvar"), + ("laboratuar", "laboratuvar"), + ("kravat", "kravat"), + ("gardolap", "gardırop"), + ("egzoz", "egzoz"), + ("egzozt", "egzoz"), + ("egsoz", "egzoz"), + ("gelcen", "geleceksin"), + ("gelecen", "geleceksin"), + ("gitcen", "gideceksin"), + ("gidecen", "gideceksin"), + ("yapcan", "yapacaksın"), + ("yapacan", "yapacaksın"), + ("edecen", "edeceksin"), + ("deyil", "değil"), +]; + +/// Longest word (in letters) given suggestions. Every candidate is a fresh copy of the word, so +/// the cost grows with the square of its length (#3625); Turkish words run longer than most +/// (`Çekoslovakyalılaştıramadıklarımızdanmışsınız` is 43 letters), so the cap is above the other +/// languages' 40. +const MAX_LETTERS: usize = 64; + +/// Produce spelling suggestions for a misspelled Turkish word. +pub fn suggest(word: &str, max: usize) -> Vec { + let dict = super::dictionary(); + suggest_with(&dict, word, max) +} + +/// Suggest with a given dictionary. +pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { + let n = word.chars().count(); + if n == 0 || n > MAX_LETTERS || max == 0 { + return Vec::new(); + } + + let mut suggestions = Vec::new(); + let mut seen = HashSet::new(); + + let lower = turkish_to_lower(word); + + // 1. Check direct common replacement table + for &(wrong, right) in COMMON_REPLACEMENTS { + if lower == wrong && seen.insert(right.to_string()) { + suggestions.push(right.to_string()); + if suggestions.len() >= max { + return suggestions; + } + } + } + // Only the table takes more than one word (`bir kaç`). + if word.contains(char::is_whitespace) { + return suggestions; + } + + // 2. Diacritic substitutions (`c` -> `ç`, `g` -> `ğ`, `ı` -> `i`, `o` -> `ö`, `s` -> `ş`, `u` -> `ü`) + if !d.is_empty() { + let variants = generate_diacritic_variants(&lower); + for cand in variants { + if cand != lower && d.contains(&cand) && seen.insert(cand.clone()) { + suggestions.push(match_casing(word, &cand)); + if suggestions.len() >= max { + return suggestions; + } + } + } + } + + // 3. Detached suffix check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü` + for suffix in ["de", "da", "te", "ta", "mi", "mı", "mu", "mü"] { + if lower.ends_with(suffix) && lower.len() > suffix.len() + 1 { + let stem = &lower[..lower.len() - suffix.len()]; + if d.is_empty() || d.contains(stem) { + let cand = format!("{stem} {suffix}"); + if seen.insert(cand.clone()) { + suggestions.push(match_casing(word, &cand)); + if suggestions.len() >= max { + return suggestions; + } + } + } + } + } + + // 4. Single typo (transposition, deletion, insertion, substitution) + if !d.is_empty() { + let chars: Vec = lower.chars().collect(); + let n = chars.len(); + + // 4a. Transposition (adjacent swap: `sorna` -> `sonra`) + for i in 0..n.saturating_sub(1) { + if chars.get(i) != chars.get(i + 1) { + let mut swapped = chars.clone(); + swapped.swap(i, i + 1); + let cand: String = swapped.into_iter().collect(); + if d.contains(&cand) && seen.insert(cand.clone()) { + suggestions.push(match_casing(word, &cand)); + if suggestions.len() >= max { + return suggestions; + } + } + } + } + + // Turkish alphabet for insertions and substitutions + const TR_ALPHABET: &[char] = &[ + 'a', 'b', 'c', 'ç', 'd', 'e', 'f', 'g', 'ğ', 'h', 'ı', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'ö', 'p', 'r', 's', 'ş', 't', 'u', 'ü', 'v', + 'y', 'z', + ]; + + // 4b. Deletion (one character left out: `kitapp` -> `kitap`) + for i in 0..n { + let mut deleted = chars.clone(); + deleted.remove(i); + let cand: String = deleted.into_iter().collect(); + if d.contains(&cand) && seen.insert(cand.clone()) { + suggestions.push(match_casing(word, &cand)); + if suggestions.len() >= max { + return suggestions; + } + } + } + + // 4c. Substitution (one wrong letter: `yolculuk` -> ...) + for i in 0..n { + for &c in TR_ALPHABET { + if chars.get(i) != Some(&c) { + let mut substituted = chars.clone(); + if let Some(target) = substituted.get_mut(i) { + *target = c; + } + let cand: String = substituted.into_iter().collect(); + if d.contains(&cand) && seen.insert(cand.clone()) { + suggestions.push(match_casing(word, &cand)); + if suggestions.len() >= max { + return suggestions; + } + } + } + } + } + + // 4d. Insertion (one missing letter: `kitp` -> `kitap`) + for i in 0..=n { + for &c in TR_ALPHABET { + let mut inserted = chars.clone(); + inserted.insert(i, c); + let cand: String = inserted.into_iter().collect(); + if d.contains(&cand) && seen.insert(cand.clone()) { + suggestions.push(match_casing(word, &cand)); + if suggestions.len() >= max { + return suggestions; + } + } + } + } + } + + // 5. Fallback: capitalization + let cap = super::grammar::turkish_capitalize(&lower); + if cap != word && (d.is_empty() || d.contains(&cap)) && seen.insert(cap.clone()) { + suggestions.push(cap); + } + + suggestions +} + +/// Convert ASCII letters to Turkish diacritic equivalents. +fn generate_diacritic_variants(s: &str) -> Vec { + let mut results = vec![String::new()]; + for c in s.chars() { + let (alt1, alt2) = match c { + 'c' => ('c', Some('ç')), + 'ç' => ('ç', Some('c')), + 'g' => ('g', Some('ğ')), + 'ğ' => ('ğ', Some('g')), + 'i' => ('i', Some('ı')), + 'ı' => ('ı', Some('i')), + 'o' => ('o', Some('ö')), + 'ö' => ('ö', Some('o')), + 's' => ('s', Some('ş')), + 'ş' => ('ş', Some('s')), + 'u' => ('u', Some('ü')), + 'ü' => ('ü', Some('u')), + other => (other, None), + }; + + let branch_mult = if alt2.is_some() { 2 } else { 1 }; + if results.len() * branch_mult > 1024 { + // Cap search branch explosion + for res in &mut results { + res.push(c); + } + continue; + } + + let mut next_gen = Vec::with_capacity(results.len() * branch_mult); + for existing in &results { + let mut s1 = existing.clone(); + s1.push(alt1); + next_gen.push(s1); + if let Some(c2) = alt2 { + let mut s2 = existing.clone(); + s2.push(c2); + next_gen.push(s2); + } + } + results = next_gen; + } + results +} + +/// Turkish lowercase with dotted/dotless I support. +pub fn turkish_to_lower(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + 'I' => out.push('ı'), + 'İ' => out.push('i'), + other => { + for lc in other.to_lowercase() { + out.push(lc); + } + } + } + } + out +} + +/// Match original casing on suggestion. +fn match_casing(orig: &str, cand: &str) -> String { + if orig.chars().all(char::is_uppercase) { + cand.to_uppercase() + } else if orig.chars().next().is_some_and(char::is_uppercase) { + super::grammar::turkish_capitalize(cand) + } else { + cand.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn common_replacements_are_suggested() { + let sugs = suggest("yalnış", 3); + assert!(sugs.contains(&"yanlış".to_string())); + + let sugs2 = suggest("herkez", 3); + assert!(sugs2.contains(&"herkes".to_string())); + } + + #[test] + fn detached_suffix_is_suggested() { + let sugs = suggest("geldinmi", 3); + assert!(sugs.contains(&"geldin mi".to_string())); + } + + #[test] + fn overlong_words_get_no_suggestions_promptly() { + let d = TrDict::from_hunspell("SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n", "2\nkitap/1\nsonra\n").unwrap(); + assert!(suggest_with(&d, "sorna", 3).contains(&"sonra".to_string())); + assert!(suggest_with(&d, "kitaplr", 3).contains(&"kitaplar".to_string())); + let t = std::time::Instant::now(); + let long = "a".repeat(100_000); + assert!(suggest_with(&d, &long, 8).is_empty()); + assert!(!d.contains(&long) && !d.contains(&format!("kitap{long}"))); + assert!(!super::super::is_correct_with(&d, &long)); + // Tens of milliseconds with the caps; without them this ran for over 5 minutes. + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); + } + + #[test] + fn two_words_get_only_the_tables_replacement() { + let d = TrDict::from_hunspell("SET UTF-8\n", "2\nbirkaç\nsonra\n").unwrap(); + assert_eq!(suggest_with(&d, "bir kaç", 3), vec!["birkaç".to_string()]); + assert_eq!(suggest_with(&d, "Bir kaç", 3), vec!["birkaç".to_string()]); + assert!(suggest_with(&d, "sorna sorna", 3).is_empty()); + } + + #[test] + fn turkish_to_lower_handles_i() { + assert_eq!(turkish_to_lower("IŞIK"), "ışık"); + assert_eq!(turkish_to_lower("İSTANBUL"), "istanbul"); + } + + #[test] + fn typo_transposition_is_suggested() { + crate::tr::test_dict::install(); + let sugs = suggest("sorna", 3); + assert!(sugs.contains(&"sonra".to_string()), "sorna should suggest sonra, got {sugs:?}"); + + let sugs_gelcen = suggest("gelcen", 3); + assert!(sugs_gelcen.contains(&"geleceksin".to_string()), "gelcen should suggest geleceksin, got {sugs_gelcen:?}"); + } +} diff --git a/crates/proof/src/tr/test_dict.rs b/crates/proof/src/tr/test_dict.rs new file mode 100644 index 00000000..de4a1319 --- /dev/null +++ b/crates/proof/src/tr/test_dict.rs @@ -0,0 +1,22 @@ +//! A tiny synthetic Turkish Hunspell dictionary for tests (written for these tests; not taken from +//! any real dictionary). Real dictionaries are installed on the system, never bundled. + +/// No affix rules: the forms the tests look up are listed whole. +pub const AFF: &str = "SET UTF-8\n"; + +/// The words the Turkish tests expect to be known. +pub const DIC: &str = "7 +kitap +kitaplar +ev +evde +olacaklardan +sonra +geldin +"; + +pub fn install() { + if let Err(e) = super::install_dictionary(AFF, DIC) { + panic!("test dictionary: {e}"); + } +} diff --git a/crates/proof/src/vi.rs b/crates/proof/src/vi.rs new file mode 100644 index 00000000..cef4deb3 --- /dev/null +++ b/crates/proof/src/vi.rs @@ -0,0 +1,96 @@ +//! Telling Vietnamese text apart from English, so a document typed in Vietnamese under the default +//! English proofing language isn't covered in red underlines (as Word's "detect language +//! automatically" does). There are no Vietnamese proofing tools yet, so Vietnamese text is treated +//! like any language without them: no spelling or grammar marks. +//! +//! The test is orthographic. Vietnamese writes most syllables with letters no other Latin-script +//! language shares: the horn letters (`ơ ư`), `ĩ ũ`, and the tone-marked vowels of the Latin +//! Extended Additional block (`ạ ả ấ ầ … ỹ`), precomposed or as combining hook above, dot below +//! and horn. French and Portuguese share `â ê ô à é`, Romanian `ă` and Croatian, Serbian and +//! Bosnian `đ`, so those alone don't count. + +/// Does `c` occur in Vietnamese words but (practically) nowhere else? +fn vietnamese_letter(c: char) -> bool { + matches!(c, 'ơ' | 'Ơ' | 'ư' | 'Ư' | 'ĩ' | 'Ĩ' | 'ũ' | 'Ũ' | '\u{1EA0}'..='\u{1EF9}' | '\u{0309}' | '\u{0323}' | '\u{031B}') +} + +/// Is `word` written with a Vietnamese-only letter (`Việt`, `ơn`)? Such a word is never an +/// English misspelling. +pub fn is_vietnamese_word(word: &str) -> bool { + let combining = |c: char| matches!(c, '\u{0300}'..='\u{036F}'); + // Two marks stacked on one letter (decomposed `ế` = e + circumflex + acute) are Vietnamese too. + word.chars().any(vietnamese_letter) || word.chars().zip(word.chars().skip(1)).any(|(a, b)| combining(a) && combining(b)) +} + +/// Does a paragraph read as Vietnamese? At least two of its words (any, in a heading of up to three +/// words) carry a Vietnamese-only letter, and they are at least a sixth of its words: Vietnamese syllables without +/// such a letter (`anh`, `con`, `ba`) are common, while an English paragraph quoting one +/// Vietnamese name stays English. +pub fn looks_vietnamese(text: &str) -> bool { + let mut total = 0usize; + let mut marked = 0usize; + for w in text.split(|c: char| !(c.is_alphanumeric() || matches!(c, '\u{0300}'..='\u{036F}'))).filter(|w| !w.is_empty()) { + total += 1; + if is_vietnamese_word(w) { + marked += 1; + } + } + marked > 0 && (marked >= 2 || total <= 3) && marked * 6 >= total +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vietnamese_words() { + for w in ["Việt", "ơn", "những", "Ước", "mũi", "Nghĩa", "tie\u{0302}\u{0301}ng", "nho\u{031B}"] { + assert!(is_vietnamese_word(w), "{w}"); + } + // Shared with French, Portuguese, Spanish: not enough on their own. + // Romanian ă and Croatian đ are Vietnamese letters too, but not Vietnamese-only. + for w in ["café", "être", "côte", "água", "señor", "anh", "the", "mănânc", "acasă", "Đurđa", "đak", "đi"] { + assert!(!is_vietnamese_word(w), "{w}"); + } + } + + #[test] + fn paragraphs() { + assert!(looks_vietnamese("Tôi yêu tiếng Việt và những bài thơ của nó.")); + assert!(looks_vietnamese("Hà Nội")); + assert!(looks_vietnamese("Việt")); + // English quoting one Vietnamese name, and French, stay as they are. + assert!(!looks_vietnamese("The meeting with Mr Nguyễn is on Tuesday at the main office.")); + assert!(!looks_vietnamese("Le café de la côte est très bon.")); + assert!(!looks_vietnamese("")); + assert!(!looks_vietnamese("anh em")); + // Romanian and Croatian paragraphs, full of ă and đ, aren't Vietnamese. + assert!(!looks_vietnamese("Mă duc acasă să mănânc cu fratele meu, că e târziu.")); + assert!(!looks_vietnamese("Đurđa i Đorđe su se vratili kući, a Đakovo je daleko.")); + } +} + +#[cfg(test)] +mod checker_tests { + use crate::{Options, ProofLang, check_text_with}; + + #[test] + fn vietnamese_paragraphs_under_english_get_no_marks() { + let text = "Hôm nay tôi đi học, trời rất đẹp và mát mẻ."; + let spans = [(0..text.len(), ProofLang::En)]; + assert!(check_text_with(text, &spans, &Options::default(), true, false).is_empty()); + // No language at all (the document default) behaves the same. + assert!(check_text_with(text, &[], &Options::default(), true, false).is_empty()); + } + + /// Without the Language data too: the installed English dictionary or the built-in word list marks + /// `yesterdy` (R6.4). + #[test] + fn english_keeps_its_marks_and_skips_vietnamese_names() { + let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let text = "We met Nguyễn yesterdy at the office."; + let issues = check_text_with(text, &[(0..text.len(), ProofLang::En)], &Options::default(), false, false); + let flagged: Vec<&str> = issues.iter().filter_map(|i| text.get(i.start..i.end)).collect(); + assert_eq!(flagged, ["yesterdy"]); + } +}