daniel543x/craft-libs
1
0 Fork 0
mirror of https://github.com/storytold/craft-libs.git synced 2026-11-08 19:42:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #482 from benandez/feat/craft-proof

Browse Source 
craft-proof: WordCraft's proofing library, shared (spelling, grammar, hyphenation, Hunspell)
This commit is contained in:
Dylan M. Taylor authored and GitHub committed 2026-11-02 15:48:03 -05:00
2 parents ea1fbeb05b 9e5862a1c1
commit fcf3be172e
68 files changed
+17352
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/proof.yml
+60
View File
@@ -0,0 +1,60 @@
name: Proofing
on:
push:
branches: [main]
paths: ["crates/proof/**", "Cargo.toml", "Cargo.lock", ".github/workflows/proof.yml"]
pull_request:
paths: ["crates/proof/**", "Cargo.toml", "Cargo.lock", ".github/workflows/proof.yml"]
workflow_dispatch:
permissions:
contents: read
concurrency:
group: craft-proof-${{ github.ref }}
cancel-in-progress: true
jobs:
check:
timeout-minutes: 20
name: Proofing lint, tests with and without Language data, WASM
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
# The Language data (craft-i18n `language-data/`), embedded by crates/proof/build.rs.
- uses: actions/checkout@v4
with:
repository: storytold/craft-i18n
ref: 062a6375c322c88b96691a17f7b5fed71c51f7a8
path: craft-i18n
persist-credentials: false
- uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable
with:
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
with:
key: craft-proof
- run: cargo fmt -p craft-proof --check
- name: Without Language data (a fork, docs.rs, offline)
run: |
cargo clippy -p craft-proof --all-targets --locked -- -D warnings
cargo test -p craft-proof --locked
cargo check -p craft-proof --locked --target wasm32-unknown-unknown
- name: A release without Language data fails
run: |
if CRAFT_I18N_REQUIRED=1 cargo check -p craft-proof --locked 2>build.log; then
echo "::error::CRAFT_I18N_REQUIRED=1 without CRAFT_I18N_DIR built"; exit 1
fi
grep -q "CRAFT_I18N_REQUIRED=1 but CRAFT_I18N_DIR is unset" build.log
- name: With Language data
env:
CRAFT_I18N_DIR: ${{ github.workspace }}/craft-i18n
CRAFT_I18N_REQUIRED: "1"
run: |
cargo clippy -p craft-proof --all-targets --locked -- -D warnings
cargo test -p craft-proof --locked
cargo check -p craft-proof --locked --target wasm32-unknown-unknown
ATTRIBUTION.md
+2
View File
@@ -14,6 +14,8 @@ author, source and license. Library code and original assets are under the repos
| `crates/cms/tests/data/FOGRA39.txt` (unmodified) | Fogra | [Fogra characterisation data](https://fogra.org/en/downloads/work-tools/characterisation-data), `FOGRA39_to_FOGRA60_v2.zip` | Fogra terms: free use and redistribution, unmodified, with Fogra identified as the source; "FOGRA39" identifies the reference data only and implies no certification, approval or endorsement by Fogra (full text in [NOTICE](NOTICE)) |
| `crates/i18n/data/collation.postcard` | Unicode, Inc. (ICU 78.1 collation data, CLDR 48.2.1); generated by `crates/i18n/data/generate-collation.sh` | [ICU](https://github.com/unicode-org/icu/releases/tag/release-78.1rc) | Unicode-3.0 |
| `crates/text/assets/hyphen/en-us.dic` | Grady Ward (Moby Hyphenator II), converted by DesignCraft | [Project Gutenberg eBook #3204](https://www.gutenberg.org/ebooks/3204) | Public domain |
| `crates/proof/data/en-words.dic` | Grady Ward (Moby Hyphenator II word list); break points dropped by craft-proof contributors with `crates/proof/examples/en_words.rs` | [Project Gutenberg eBook #3204](https://www.gutenberg.org/ebooks/3204), via craft-i18n `language-data/en-us/en-us.dic` | Public domain |
| `crates/proof/data/en-extra.txt` | WordCraft contributors (original list, no other dictionary copied) | craft-i18n `language-data/en-us/en-extra.txt` | MIT OR Apache-2.0 |
| `crates/text/assets/hyphen/en-us.pat` | DesignCraft contributors, trained from the Moby Hyphenator II list | `crates/text/examples/hyphgen.rs` | CC0-1.0 |
| Patterns embedded through `hypher` (optional `craft-text` language features) | hyph-utf8 authors, per language | [tex-hyphen](https://github.com/hyphenation/tex-hyphen), as packaged in [hypher](https://github.com/typst/hypher/tree/main/patterns) | Per file header: LPPL, MIT, BSD-3-Clause, Unlicense or a permit-any-use notice (or a choice including one); Slovak (`hyph-sk.tex`, embedded whenever `hypher/slovak` is on: feature `slovak-gpl`, or another dependency through feature unification) is GPL-2.0-or-later; Spanish (`hyph-es.tex` 5.0, Javier Bezos, CervanTeX) is MIT |
| Word-break dictionaries and LSTM models embedded through `icu_segmenter_data` (optional `craft-text` features `thai`, `lao`, `khmer`, `burmese`) | Unicode, Inc. and ICU4X contributors | [ICU4X](https://github.com/unicode-org/icu4x) | Unicode-3.0 |
Cargo.lock
Generated
+10
View File
@@ -1026,6 +1026,16 @@ dependencies = [
"winsafe",
]
[[package]]
name = "craft-proof"
version = "0.1.0"
dependencies = [
"log",
"miniz_oxide 0.9.1",
"proptest",
"unicode-normalization",
]
[[package]]
name = "craft-quit"
version = "0.1.0"
Cargo.toml
+1
View File
@@ -42,6 +42,7 @@ craft-menu = { path = "crates/menu" }
craft-pcraft-format = { path = "crates/pcraft-format" }
craft-prepress = { path = "crates/prepress" }
craft-print = { path = "crates/print" }
craft-proof = { path = "crates/proof" }
craft-quit = { path = "crates/quit" }
craft-settings = { path = "crates/settings" }
craft-text = { path = "crates/text" }
NOTICE
+4
View File
@@ -21,6 +21,10 @@ crates/cms/tests/data/FOGRA39.txt: FOGRA39 characterisation data, distributed un
imply certification, approval or endorsement by Fogra."
crates/text/assets/hyphen/en-us.dic: public domain, derived from the Moby Hyphenator II list by
Grady Ward (Project Gutenberg eBook #3204).
crates/proof/data/en-words.dic: public domain, the words of the Moby Hyphenator II list by Grady
Ward (Project Gutenberg eBook #3204), without its break points.
crates/proof/data/en-extra.txt: MIT OR Apache-2.0, WordCraft contributors' own list of modern
English words (a copy of craft-i18n's language-data/en-us/en-extra.txt).
crates/text/assets/hyphen/en-us.pat: CC0-1.0, trained from en-us.dic by DesignCraft's trainer.
Hyphenation patterns for other languages, embedded through the hypher crate only when the
matching craft-text feature is enabled (none are by default). They come from hyph-utf8,
README.md
+3
View File
@@ -87,6 +87,9 @@ craft-<name> = { git = "https://github.com/storytold/craft-libs", rev = "<commit
manifest decode limits and the migration runner. The document model stays in PhotoCraft.
- [`craft-print`](crates/print/README.md): PDF submission to CUPS and Windows, printer discovery,
duplex/copies/colour and driver preferences. PDF generation stays in each app.
- [`craft-proof`](crates/proof/README.md): proofing per language (spelling, suggestions, grammar,
hyphenation, a Hunspell reader, user words). The word lists and patterns are craft-i18n's
`language-data/`, embedded at build time from `CRAFT_I18N_DIR`.
- [`craft-quit`](crates/quit/src/lib.rs): native macOS quit, egui close handling and an optional
app menu. Unsaved-document decisions and save prompts stay in each app.
crates/proof/Cargo.toml
+20
View File
@@ -0,0 +1,20 @@
[package]
name = "craft-proof"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
description = "Proofing per language for the Craft Apps: spelling, suggestions, grammar checks and hyphenation, a Hunspell .aff/.dic reader, and user and ignored words. Language data is embedded from craft-i18n at build time (CRAFT_I18N_DIR)."
build = "build.rs"
[lints]
workspace = true
[dependencies]
log = "0.4"
miniz_oxide = "0.9"
unicode-normalization = "0.1"
[dev-dependencies]
proptest = { workspace = true }
crates/proof/README.md
+117
View File
@@ -0,0 +1,117 @@
# craft-proof
Proofing for the Craft apps, per language: spelling, suggestions, grammar checks and hyphenation,
a Hunspell `.aff`/`.dic` reader, user words, ignored words and a dictionaries folder. It moved here
from WordCraft's `wordcraft-proof` with its history and the same public API. No `unsafe`, no
panics on hostile input, and it builds for `wasm32-unknown-unknown`.
```toml
craft-proof = { git = "https://github.com/storytold/craft-libs", rev = "<commit>" }
```
## Languages
| Language | Spelling | Grammar | Hyphenation |
|---|---|---|---|
| English (en-US, en-GB and the British-spelling Englishes, en-CA) | Moby word list + `en-extra.txt`, dialect variants (`en-variants.txt`); without them the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic |
| Polish, Portuguese, Italian | the installed Hunspell dictionary | yes | TeX patterns |
| Croatian, Turkish, Russian, Czech, Indonesian, Spanish (es, es-MX, es-419, es-UY) | the installed Hunspell dictionary (Russian also `ru-extra.txt`) | yes | Turkish and Russian rules; English fallback otherwise |
| German | — | — | TeX patterns |
| Bulgarian | — | — | syllable rules |
Grammar rules, suggestion tuning and the rule-based hyphenation are code: they work in every
build.
## Use
```rust
use craft_proof::{ProofLang, check_text_with, suggest_in, Options};
use craft_proof::hyphen::{hyphen_points_in, Limits};
// A paragraph whose runs are in different languages (byte ranges of `text`).
let issues = check_text_with(text, &[(0..text.len(), ProofLang::EnGb)], &Options::default(), true, false);
let fixes = suggest_in("colour", 5, ProofLang::En);
let breaks = hyphen_points_in("hyphenation", &Limits::default(), ProofLang::En);
```
Hunspell dictionaries are not bundled; they are read from, in order: the folder given to
`set_dictionaries_folder`, `WORDCRAFT_DICTIONARIES`, WordCraft's own folder under the user's
application data (`%LOCALAPPDATA%\WordCraft\dictionaries` and `%APPDATA%\WordCraft\dictionaries`,
`~/Library/Application Support/WordCraft/dictionaries`, `$XDG_DATA_HOME/wordcraft/dictionaries`),
then the usual Hunspell and LibreOffice locations. None on the web.
State is process-wide: `add_word`, `ignore_word`, `set_dictionaries_folder`,
`load_user_dictionary`/`save_user_dictionary`. The crate reads no document model; callers pass
plain text and language spans.
## Language data
The word lists and patterns live in [craft-i18n](https://github.com/storytold/craft-i18n)
(`language-data/`), not here. `build.rs` embeds them at build time:
- `CRAFT_I18N_DIR=<absolute path to a craft-i18n checkout>` embeds `language-data/`.
- Unset (a fork, docs.rs, offline): the crate builds with a warning and no embedded Language
data. English spelling then falls back as below, hyphenation that needs the data is off, and
grammar and the Hunspell languages work as before.
- `CRAFT_I18N_REQUIRED=1` (releases) makes a missing directory or file an error.
Tests that pin results of the data run only when every file is embedded (`craft_proof_data`);
`src/no_data.rs` runs every public entry point when none is (`craft_proof_no_data`). A partial
data folder runs neither set.
### English is always checked
English spelling uses the first of these that is there; `english_source(lang)` says which.
1. **Embedded:** the Moby word list from `CRAFT_I18N_DIR` (above). Nothing below is used then.
2. **System Hunspell:** the run's own English `.aff`/`.dic` pair (`en_US`, `en_GB` or `en_CA`;
each name in the dictionaries folder and then the system locations before the next name).
Never another English's: an `en_US` dictionary would mark British spellings, so en-GB and
en-CA without their own use the built-in list, which has both spellings. A file that can't be
read is skipped for the next name or folder; one file is read once for every English that uses
it. Compounding (`COMPOUNDMIN`, `COMPOUNDRULE`…, used by English dictionaries only for
ordinals) and `OCONV` are ignored for English rather than refused. None on the web.
3. **Built-in list:** `data/en-words.dic`, the whole Moby word list (about 165,000 words, 344 KB
compressed, words only) built into the crate, with `data/en-extra.txt` (WordCraft's supplement
of modern words, a copy of craft-i18n's) and the same inflection rules as the embedded list. A
word it doesn't know is misspelled. Suggestions are the words one edit away (swap, missing,
added or changed letter), looked up in the list, for words of up to 32 letters: swaps first,
then common words, neighbouring keys, the same first letter.
A dictionary that is there but can't be read (encoding, size, parse error, no words) is skipped,
and `dictionary_load_error(lang)` gives the reason; it never panics. Text in other languages is
never checked against English: a language without its dictionary has no spelling marks.
`data/en-words.dic` is public domain: the Moby Hyphenator II list by Grady Ward (Project Gutenberg
eBook #3204), from craft-i18n's `language-data/en-us/en-us.dic` with its break points dropped
and a few entries that are far more often typos left out (`teh`, `abd`, `nad`, `ur`, `u`, `r`;
the list is in the generator).
Regenerate with
`cargo run -p craft-proof --example en_words -- <craft-i18n>/language-data/en-us/en-us.dic crates/proof/data/en-words.dic`.
`data/en-extra.txt` is WordCraft contributors' own list (MIT OR Apache-2.0); keep it in step with
craft-i18n's copy.
## Check
```sh
cargo fmt -p craft-proof --check
cargo clippy -p craft-proof --all-targets -- -D warnings
CRAFT_I18N_DIR=/abs/path/craft-i18n cargo test -p craft-proof
cargo test -p craft-proof # without Language data
cargo check -p craft-proof --target wasm32-unknown-unknown
```
CI: `.github/workflows/proof.yml`.
## Credits
storytold/craft-libs#30/#31 by brzuchal (Michał Marcin Brzuchalski) started the shared crate.
## Owners
WordCraft maintainers: echelon, dylanmtaylor, aiongg.
## Licence
MIT OR Apache-2.0. The Language data carries its own licences, recorded in craft-i18n's
`ATTRIBUTION.md`.
crates/proof/build.rs
+111
View File
@@ -0,0 +1,111 @@
//! Embed the Language data from a craft-i18n checkout (storytold/craft-i18n, `language-data/`).
//!
//! `CRAFT_I18N_DIR`, an absolute path to a craft-i18n checkout: `$OUT_DIR/language_data.rs` holds
//! one constant per file in [`FILES`], each an `include_str!`/`include_bytes!` of
//! `$CRAFT_I18N_DIR/language-data/<path>`. Unset, or not a checkout, or a file missing (a warning):
//! that constant is empty, and the crate has no embedded Language data for it (English spelling
//! falls back to an installed Hunspell dictionary or the crate's own copy of the Moby word list, and
//! hyphenation that needs the data is off). With `CRAFT_I18N_REQUIRED=1` (releases) any of those
//! is an error instead.
//!
//! When every file in [`FILES`] is embedded, the `craft_proof_data` cfg is set (the tests that pin
//! results of the data use it); when none is, `craft_proof_no_data` (the tests of a build without
//! the data). A partial data folder sets neither, so neither set of tests runs against it.
//!
//! `include_str!` takes a string, so a `CRAFT_I18N_DIR` that isn't valid UTF-8 can't be embedded:
//! it is treated like a folder without the data.
use std::fmt::Write as _;
use std::path::{Path, PathBuf};
/// (constant, path under `language-data/`, bytes rather than text).
const FILES: &[(&str, &str, bool)] = &[
("EN_US_DIC", "en-us/en-us.dic", true),
("EN_US_PAT", "en-us/hyphenation.pat", false),
("EXTRA_EN", "en-us/en-extra.txt", false),
("EN_VARIANTS", "en-us/en-variants.txt", false),
("PL_PAT", "pl/hyphenation.pat", false),
("PT_PAT", "pt/hyphenation.pat", false),
("IT_PAT", "it/hyphenation.pat", false),
("DE_PAT", "de/hyphenation.pat", false),
("EXTRA_RU", "ru/ru-extra.txt", false),
];
fn main() {
println!("cargo::rerun-if-changed=build.rs");
println!("cargo::rerun-if-env-changed=CRAFT_I18N_DIR");
println!("cargo::rerun-if-env-changed=CRAFT_I18N_REQUIRED");
println!("cargo::rustc-check-cfg=cfg(craft_proof_data)");
println!("cargo::rustc-check-cfg=cfg(craft_proof_no_data)");
let required = std::env::var("CRAFT_I18N_REQUIRED").is_ok_and(|v| v.trim() == "1");
let level = if required { "error" } else { "warning" };
let dir = match std::env::var_os("CRAFT_I18N_DIR").filter(|d| !d.is_empty()) {
Some(d) => match language_data(&PathBuf::from(d)) {
Ok(dir) => Some(dir),
Err(e) => {
println!("cargo::{level}=CRAFT_I18N_DIR: {e}; craft-proof embeds no Language data");
None
}
},
None => {
if required {
println!("cargo::error=CRAFT_I18N_REQUIRED=1 but CRAFT_I18N_DIR is unset: a release must embed the Language data");
} else {
println!(
"cargo::warning=CRAFT_I18N_DIR is unset: craft-proof embeds no Language data (English spelling falls back to system Hunspell or the built-in Moby word list; no hyphenation)"
);
}
None
}
};
let mut o = String::from("// Generated by craft-proof's build.rs from CRAFT_I18N_DIR/language-data. Do not edit.\n");
let mut embedded = 0;
for &(name, rel, bytes) in FILES {
let file = dir.as_ref().map(|d| d.join(rel)).filter(|f| f.is_file());
if let (Some(d), None) = (&dir, &file) {
println!("cargo::{level}=CRAFT_I18N_DIR: {} is missing; craft-proof embeds none", d.join(rel).display());
}
let (ty, empty, mac) = if bytes { ("&[u8]", "b\"\"", "include_bytes") } else { ("&str", "\"\"", "include_str") };
// `language_data` only hands back a UTF-8 folder, so this is the path as it is.
let value = match file.as_ref().and_then(|f| f.to_str()) {
Some(f) => {
println!("cargo::rerun-if-changed={f}");
embedded += 1;
// `{:?}` escapes the path (Windows backslashes) as a Rust string literal.
format!("{mac}!({f:?})")
}
None => empty.to_string(),
};
let _ = writeln!(o, "/// `language-data/{rel}` (empty when it isn't embedded).\npub(crate) static {name}: {ty} = {value};");
}
if embedded == FILES.len() {
println!("cargo::rustc-cfg=craft_proof_data");
} else if embedded == 0 {
println!("cargo::rustc-cfg=craft_proof_no_data");
}
let out = PathBuf::from(std::env::var_os("OUT_DIR").unwrap_or_default()).join("language_data.rs");
if let Err(e) = std::fs::write(&out, o) {
println!("cargo::error=writing {}: {e}", out.display());
}
}
/// `dir/language-data`, when `dir` is an absolute path to a checkout that has one.
fn language_data(dir: &Path) -> Result<PathBuf, String> {
// Build scripts run in the crate's directory and `include_str!` resolves against the generated
// file, so a relative path would mean two different places. Not canonicalized: on Windows that
// gives `\\?\C:\…`, which `include_str!` can't read.
if !dir.is_absolute() {
return Err(format!("{} is not an absolute path", dir.display()));
}
if dir.to_str().is_none() {
return Err(format!("{} is not valid UTF-8, which include_str! needs", dir.display()));
}
let data = dir.join("language-data");
if !data.is_dir() {
// Notice when the checkout gains its Language data.
println!("cargo::rerun-if-changed={}", dir.display());
return Err(format!("{} has no language-data/ (a craft-i18n checkout older than its Language data?)", dir.display()));
}
println!("cargo::rerun-if-changed={}", data.display());
Ok(data)
}
crates/proof/data/en-extra.txt
+241
View File
@@ -0,0 +1,241 @@
# English spelling supplement: common words the public-domain Moby list (1990s) lacks.
# Written by WordCraft contributors (MIT OR Apache-2.0); see ATTRIBUTION.md.
# One lowercase base form per line, sorted. Plurals, -ed, -ing, -er, -ly… come from the
# inflection rules in crates/proof/src/lib.rs, so list "desktop", not "desktops".
airbag
airfare
artcraft
autocomplete
autocorrect
autofill
autoformat
autosave
backend
bitcoin
bitmap
blockchain
blog
blogger
bluetooth
captcha
caregiver
carpool
cellphone
chatbot
chatroom
checkbox
checklist
childcare
clamour
clickable
colourful
colourisation
colourise
colourless
configurable
copyedit
coronavirus
covid
crowdfunding
crowdsource
crypto
cryptocurrency
customizable
customization
cyber
cybersecurity
cyberspace
database
dataset
daycare
desktop
download
downloadable
downsize
dropdown
earbud
ebook
ecommerce
email
emoji
emoticon
endeavour
esports
facebook
favicon
fintech
firewall
flowchart
foodie
freeware
frontend
fundraiser
gameplay
gamification
geeky
geolocation
gigabit
gigabyte
github
glamour
globalization
google
groupthink
hackathon
hardcode
hashtag
healthcare
homepage
homeschool
honour
honourable
honourably
hotspot
hyperlink
hypertext
inbox
incentivize
infographic
instagram
internet
interoperability
intranet
ipad
iphone
javascript
kickstart
kilobyte
laptop
lifestyle
linkedin
linux
livestream
lockdown
login
logout
lowercase
makeover
malware
manoeuvrable
megabyte
megapixel
meme
metadata
microchip
microsoft
middleware
mindset
multicultural
multiplayer
multitask
nerdy
netbook
netflix
newsfeed
offboard
offline
onboard
online
onscreen
outsource
paediatric
paralegal
passcode
petabyte
phishing
photobomb
playdate
playlist
plugin
podcast
popup
prequel
printout
prioritise
prioritization
prioritize
privatise
ransomware
recyclable
rigour
ringtone
roadmap
rollout
runtime
scalability
screencast
screensaver
screenshot
scrollbar
selfie
signup
sitemap
skillset
smartphone
smartwatch
smiley
snowboard
spellcheck
spellchecker
spreadsheet
spyware
stakeholder
standup
startup
subfolder
sunscreen
superfood
superuser
synthesise
tagline
takeout
techie
telecommute
teleconference
telehealth
telemedicine
terabyte
textbox
timeline
timestamp
toolbar
toolkit
touchpad
touchscreen
trackball
trackpad
travelogue
upcycle
upload
uppercase
upsell
upskill
usability
username
videoconference
videogame
vlog
vlogger
voicemail
walkthrough
webcam
webcast
webinar
weblog
webmaster
webpage
website
whatsapp
whiteboard
wifi
wiki
wikipedia
wireframe
wordcraft
workaround
workflow
workforce
workplace
workspace
workstation
youtube
crates/proof/data/en-words.dic
BIN
View File
Binary file not shown.
crates/proof/examples/en_words.rs
+33
View File
@@ -0,0 +1,33 @@
//! Regenerate `data/en-words.dic`, the English word list built into the crate (see the crate
//! README): every word of the public-domain Moby Hyphenator II list, without its break points.
//!
//! ```sh
//! cargo run -p craft-proof --example en_words -- <craft-i18n>/language-data/en-us/en-us.dic data/en-words.dic
//! ```
//!
//! The input is craft-i18n's `en-us.dic`: the Moby Hyphenator II list by Grady Ward (Project
//! Gutenberg eBook #3204, `mhyph.txt`), already stripped of the Project Gutenberg header and
//! licence, with its break points. The output keeps the words only (in [`Dictionary`]'s format,
//! every break mask zero), so it is smaller and carries no hyphenation, and leaves out
//! [`EXCLUDED`]. No other word list or dictionary is read.
use craft_proof::dict::Dictionary;
/// Entries of the Moby list that are far more often typing errors than words (`teh` for `the`,
/// `abd` and `nad` for `and`, `ur` for `your`, the letters `u` and `r` for `you` and `are`), so the
/// built-in list marks them. `wont` and `cant` stay: they are words.
const EXCLUDED: &[&str] = &["abd", "nad", "r", "teh", "u", "ur"];
fn main() -> Result<(), String> {
let args: Vec<String> = std::env::args().skip(1).collect();
let [dic, out] = args.as_slice() else {
return Err("usage: en_words <en-us.dic> <out>".into());
};
let bytes = std::fs::read(dic).map_err(|e| format!("{dic}: {e}"))?;
let moby = Dictionary::from_bytes(&bytes)?;
let words = Dictionary::from_entries(moby.iter().filter(|(w, _)| !EXCLUDED.contains(w)).map(|(w, _)| (w.to_string(), 0)).collect());
let data = words.to_bytes();
std::fs::write(out, &data).map_err(|e| format!("{out}: {e}"))?;
eprintln!("{} words, {} bytes", words.len(), data.len());
Ok(())
}
crates/proof/src/affix.rs
+1847
View File
File diff suppressed because it is too large. Load diff
crates/proof/src/affix_suggest.rs
+556
View File
@@ -0,0 +1,556 @@
//! Spelling suggestions from an affix dictionary, one engine for every language that reads one
//! (Czech, Croatian, Indonesian, Italian, Polish, Portuguese, Russian, Spanish). Candidates come
//! in rounds of rising cost, each checked against the dictionary:
//! 1. letter case (`warszawa` → `Warszawa`);
//! 2. missing or wrong diacritics and other MAP slips, a few letters changed (`zolw` → `żółw`):
//! adding a missing accent is the commonest slip and ranks first, while taking away one the
//! writer typed ranks after the known confusions;
//! 3. the REP table of common confusions (`żeka` → `rzeka`);
//! 4. a missing space (`napewno` → `na pewno`);
//! 5. keyboard neighbours from the KEY rows, then one typo: a letter left out, added, swapped with
//! the next or mistyped (TRY letters);
//! 6. when that finds fewer than asked for: the stems that start like the word (MAP relatives
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance in
//! which a MAP slip costs half (two typos at most).
//!
//! The tables come from the dictionary; a language's [`Tuning`] adds the ones its dictionaries may
//! lack and the slips no table can say. The input is bounded ([`MAX_LETTERS`]), and so is every
//! round, so a call costs a few thousand dictionary lookups at most.
use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use crate::affix::{AffixDict, Casing, capitalize, casing};
/// Longest word suggestions are looked for, in letters.
pub(crate) const MAX_LETTERS: usize = 40;
const MAX_MAP_VARIANTS: usize = 4096;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
const COST_CASE: u32 = 0;
pub(crate) const COST_ADD_DIACRITIC: u32 = 4;
pub(crate) const COST_OTHER_MAP: u32 = 7;
const COST_DROP_DIACRITIC: u32 = 11;
const COST_REP: u32 = 10;
pub(crate) const COST_SPLIT: u32 = 12;
/// A missing space leaving a half shorter than three letters ([`Split::Anywhere`]).
const COST_SPLIT_SHORT: u32 = 17;
/// A missing space between two longer halves with no short word beside it ([`Split::Near`]).
pub(crate) const COST_SPLIT_LONG: u32 = 18;
const COST_KEY: u32 = 13;
const COST_SWAP: u32 = 14;
const COST_DROP_OR_ADD: u32 = 15;
pub(crate) const COST_MISTYPE: u32 = 16;
const COST_SCAN: u32 = 20;
/// What a language adds to its dictionary's tables.
pub(crate) struct Tuning {
/// Where a missing space may go (round 4).
pub split: Split,
/// TRY letters for a dictionary without a TRY line.
pub alphabet: &'static str,
/// MAP groups the language adds to its dictionary's (each string is one group: `"cčć"`).
pub map: &'static [&'static str],
/// REP confusions the language adds to its dictionary's.
pub rep: &'static [(&'static str, &'static str)],
/// The cost of a REP confusion.
pub rep_cost: u32,
/// Most letters a MAP variant changes.
pub max_map_changes: usize,
/// The cost of changing letter `from` to its MAP relative `to`.
pub map_cost: fn(char, char) -> u32,
/// The cost of typing letter `c` at `i` of the word (`chars`, lower case) instead.
pub mistype_cost: fn(&[char], usize, char) -> u32,
}
/// Where a missing space may go.
pub(crate) enum Split {
/// Anywhere, dearer when a half is shorter than three letters.
Anywhere,
/// After a short word that often runs into the next one (`na pewno`, `w ogóle`) or before one
/// that runs into the word before (`kocham cię`), dearer between two longer halves, and never
/// leaving a stray short half (`że ka`).
Near { leading: &'static [&'static str], trailing: &'static [&'static str] },
/// The language's own rule: the cost of the halves `a b`, or `None` to skip them.
Custom(fn(&str, &str) -> Option<u32>),
}
impl Tuning {
/// The dictionary's tables alone (Czech, Indonesian, Spanish).
pub(crate) const DEFAULT: Tuning =
Tuning { split: Split::Anywhere, alphabet: "", map: &[], rep: &[], rep_cost: COST_REP, max_map_changes: 4, map_cost, mistype_cost };
}
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
/// commonest slip (typing without diacritics); an accent the writer did type is likelier right,
/// so taking one away costs more than a known confusion from the REP table.
pub(crate) fn map_cost(from: char, to: char) -> u32 {
match (from.is_ascii(), to.is_ascii()) {
(true, false) => COST_ADD_DIACRITIC,
(false, true) => COST_DROP_DIACRITIC,
_ => COST_OTHER_MAP,
}
}
/// Any mistyped letter costs the same.
fn mistype_cost(_: &[char], _: usize, _: char) -> u32 {
COST_MISTYPE
}
struct Collector<'a> {
d: &'a AffixDict,
/// The word, lower case.
input: String,
/// Candidate → the cheapest cost it was offered at (rounds don't run in rising cost).
tested: HashMap<String, u32>,
/// Suggestion → (cost, order found).
found: HashMap<String, (u32, usize)>,
seq: usize,
}
impl Collector<'_> {
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
/// word is in the dictionary, as written or capitalized (names).
fn offer(&mut self, cand: &str, cost: u32) {
if cand.is_empty() || cand == self.input {
return;
}
match self.tested.get_mut(cand) {
Some(prev) if *prev <= cost => return,
Some(prev) => *prev = cost,
None => {
self.tested.insert(cand.to_string(), cost);
}
}
let mut words = Vec::new();
let mut cost = cost;
for part in cand.split(' ') {
if self.d.contains_suggestible(part) {
words.push(part.to_string());
} else {
let c = capitalize(part);
if !self.d.contains_suggestible(&c) {
return;
}
words.push(c);
// A name ranks after a common word at the same distance (`moze` → `może`, then
// `Mozę`).
cost += 1;
}
}
self.keep(words.join(" "), cost);
}
/// Keep a suggestion known to be correct, unless the dictionary says never to offer it.
fn keep(&mut self, text: String, cost: u32) {
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
return;
}
self.seq += 1;
let seq = self.seq;
let e = self.found.entry(text).or_insert((cost, seq));
if cost < e.0 {
*e = (cost, seq);
}
}
fn count_within(&self, cost: u32) -> usize {
self.found.values().filter(|(c, _)| *c <= cost).count()
}
}
/// The MAP groups: the dictionary's, then the language's.
fn groups<'a>(d: &'a AffixDict, t: &Tuning) -> Cow<'a, [Vec<char>]> {
if t.map.is_empty() {
return Cow::Borrowed(d.map());
}
Cow::Owned(d.map().iter().cloned().chain(t.map.iter().map(|g| g.chars().collect())).collect())
}
/// The other members of every MAP group of `c` (its likely confusions), each once.
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
let mut out: Vec<char> = Vec::new();
for x in map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()) {
if x != c && !out.contains(&x) {
out.push(x);
}
}
out
}
/// Every way of changing up to `t.max_map_changes` letters of `chars` to a MAP relative (`alts`,
/// letter by letter), with its cost, each written after `buf`.
fn map_variants(chars: &[char], alts: &[Vec<char>], t: &Tuning, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
if out.len() >= MAX_MAP_VARIANTS {
return;
}
let Some((&c, rest)) = chars.split_first() else {
if changes > 0 {
out.push((buf.iter().collect(), cost));
}
return;
};
let (here, later) = alts.split_first().map_or((&[][..], &[][..]), |(h, l)| (h.as_slice(), l));
buf.push(c);
map_variants(rest, later, t, changes, cost, buf, out);
buf.pop();
if changes < t.max_map_changes {
for &a in here {
buf.push(a);
map_variants(rest, later, t, changes + 1, cost + (t.map_cost)(c, a), buf, out);
buf.pop();
}
}
}
/// A letter's MAP representative (the first of its first group), lower case: `ż` → `z`, `ó` → `o`.
pub(crate) fn fold(c: char, map: &[Vec<char>]) -> char {
let l = c.to_lowercase().next().unwrap_or(c);
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
}
/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other
/// edit 2; `limit + 1` once it is certainly above `limit`.
pub(crate) fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
let m = b.len();
let over = limit + 1;
let mut prev2 = vec![0u32; m + 1];
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
let mut cur = vec![0u32; m + 1];
for (i, &ai) in a.iter().enumerate() {
if let Some(c) = cur.get_mut(0) {
*c = (i as u32 + 1) * 2;
}
let mut row_min = u32::MAX;
for (j, &bj) in b.iter().enumerate() {
let sub = if ai == bj {
0
} else if related(ai, bj) {
1
} else {
2
};
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
v = v.min(at(&prev2, j - 1) + 2);
}
if let Some(c) = cur.get_mut(j + 1) {
*c = v;
}
row_min = row_min.min(v);
}
if m > 0 && row_min > limit {
return over;
}
std::mem::swap(&mut prev2, &mut prev);
std::mem::swap(&mut prev, &mut cur);
}
prev.get(m).copied().unwrap_or(over).min(over)
}
/// The cost of a missing space between `a` and `b`, or `None` when the split isn't offered.
fn split_cost(split: &Split, a: &str, b: &str) -> Option<u32> {
let long = |part: &str| part.chars().count() >= 3;
match split {
Split::Anywhere => Some(if long(a) && long(b) { COST_SPLIT } else { COST_SPLIT_SHORT }),
Split::Near { leading, trailing } => {
if (leading.contains(&a) && long(b)) || (long(a) && trailing.contains(&b)) {
Some(COST_SPLIT)
} else if long(a) && long(b) {
Some(COST_SPLIT_LONG)
} else {
None
}
}
Split::Custom(cost) => cost(a, b),
}
}
/// Spelling suggestions for `word` from `d`, best first, at most `max`, in the word's letter case.
pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> Vec<String> {
let word = word.trim();
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0 };
add_case(&mut col, word, &input);
let map = groups(d, t);
add_map_slips(&mut col, &chars, &map, t);
add_confusions(&mut col, &input, t);
add_splits(&mut col, &input, &t.split);
add_key_neighbours(&mut col, &chars);
add_typos(&mut col, &chars, t);
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars, &map);
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
// Among equal costs, the change nearest the word's start is the likelier intent
// (the first changed letter breaks ties).
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
let mut out: Vec<String> = Vec::new();
for (text, _) in found {
let recased = match case {
Casing::Initial => capitalize(&text),
Casing::Upper if n > 1 => text.to_uppercase(),
_ => text.clone(),
};
// A KEEPCASE word only exists in its dictionary case: `Km` would be flagged, so offer `km`.
let t = if recased != text && text.split(' ').any(|p| d.case_locked(p)) { text } else { recased };
if t != word && !out.contains(&t) {
out.push(t);
}
if out.len() >= max {
break;
}
}
out
}
/// Round 1: letter case.
fn add_case(col: &mut Collector, word: &str, input: &str) {
if col.d.contains(input) && word != input {
col.keep(input.to_string(), COST_CASE);
} else if col.d.contains(&capitalize(input)) && word != capitalize(input) {
col.keep(capitalize(input), COST_CASE);
}
}
/// Round 2: accents and other MAP slips.
fn add_map_slips(col: &mut Collector, chars: &[char], map: &[Vec<char>], t: &Tuning) {
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
let mut variants = Vec::new();
map_variants(chars, &alts, t, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants);
variants.sort_by_key(|v| v.1);
for (v, cost) in &variants {
col.offer(v, *cost);
}
}
/// Round 3: common confusions.
fn add_confusions(col: &mut Collector, input: &str, t: &Tuning) {
let rep = col.d.rep().iter().map(|(from, to)| (from.as_str(), to.as_str())).chain(t.rep.iter().copied());
for (from, to) in rep {
if from.is_empty() {
continue;
}
for (i, _) in input.match_indices(from) {
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
col.offer(&format!("{a}{to}{b}"), t.rep_cost);
}
}
}
}
/// Round 4: a missing space; both halves must be dictionary words.
fn add_splits(col: &mut Collector, input: &str, split: &Split) {
for (i, _) in input.char_indices().skip(1) {
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
if let Some(cost) = split_cost(split, a, b) {
col.offer(&format!("{a} {b}"), cost);
}
}
}
/// Round 5: keyboard neighbours from the dictionary's KEY rows.
fn add_key_neighbours(col: &mut Collector, chars: &[char]) {
for row in col.d.key().split('|').take(32) {
let keys: Vec<char> = row.chars().take(128).collect();
for pair in keys.windows(2) {
let [a, b] = pair else { continue };
for (i, &c) in chars.iter().enumerate() {
let replacement = if c == *a {
Some(*b)
} else if c == *b {
Some(*a)
} else {
None
};
if let Some(replacement) = replacement {
let mut v = chars.to_vec();
if let Some(x) = v.get_mut(i) {
*x = replacement;
}
col.offer(&v.iter().collect::<String>(), COST_KEY);
}
}
}
}
}
/// Round 5, second half: one typo, with the TRY letters (the language's when the dictionary has
/// none), bounded and each once, in their order: a TRY line lists the commonest letters first, so
/// equal-cost candidates come out in that order.
fn add_typos(col: &mut Collector, chars: &[char], t: &Tuning) {
let d = col.d;
let n = chars.len();
let try_chars = if d.try_chars().is_empty() { t.alphabet } else { d.try_chars() };
let mut letters: Vec<char> = Vec::new();
for c in try_chars.chars().filter(|c| c.is_lowercase()).take(128) {
if !letters.contains(&c) {
letters.push(c);
}
}
let s = |v: &[char]| v.iter().collect::<String>();
for i in 0..n.saturating_sub(1) {
if chars.get(i) != chars.get(i + 1) {
let mut v = chars.to_vec();
v.swap(i, i + 1);
col.offer(&s(&v), COST_SWAP);
}
}
for i in 0..n {
let mut v = chars.to_vec();
v.remove(i);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
for i in 0..=n {
for &c in &letters {
let mut v = chars.to_vec();
v.insert(i, c);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
}
for i in 0..n {
for &c in &letters {
if chars.get(i) != Some(&c) {
let mut v = chars.to_vec();
if let Some(x) = v.get_mut(i) {
*x = c;
}
col.offer(&s(&v), (t.mistype_cost)(chars, i, c));
}
}
}
}
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
/// case) by shared letter pairs, then keep the forms of the best within two typos.
fn scan(col: &mut Collector, chars: &[char], map: &[Vec<char>]) {
let d = col.d;
let Some(&first) = chars.first() else { return };
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
firsts.extend(upper);
firsts.sort_unstable();
firsts.dedup();
let n = folded.len();
let mut best: Vec<(u32, &str)> = Vec::new();
for f in firsts {
let prefix = f.to_string();
for stem in d.stems_with_prefix(&prefix) {
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
if shared < 2 {
continue;
}
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
// beyond what an ending explains.
let extra = sf.len().saturating_sub(shared);
let gap = sf.len().abs_diff(n).saturating_sub(4);
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
if score == 0 {
continue;
}
if best.len() < SCAN_STEMS {
best.push((score, stem));
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
&& score > min.0
{
*min = (score, stem);
}
}
}
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let related = |a: char, b: char| fold(a, map) == fold(b, map);
let limit = 4;
for (_, stem) in best {
for form in d.forms(stem) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
}
let dist = weighted_distance(&lf, chars, &related, limit);
if dist <= limit && lf.as_slice() != chars {
col.keep(form, COST_SCAN + dist * 3);
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() {
let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap();
let suggestions = suggest_in(&d, "počítačz", 8, &Tuning::DEFAULT);
assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}");
assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}");
}
/// A candidate offered again at a lower cost (a REP confusion after a MAP diacritic drop)
/// ranks at the lower one.
#[test]
fn a_cheaper_later_offer_wins() {
let d = AffixDict::from_hunspell("", "1\ncafe\n", &[]).unwrap();
let mut col = Collector { d: &d, input: "cafè".into(), tested: HashMap::new(), found: HashMap::new(), seq: 0 };
col.offer("cafe", COST_DROP_DIACRITIC);
col.offer("cafe", COST_REP);
col.offer("cafe", COST_SCAN);
assert_eq!(col.found.get("cafe").map(|e| e.0), Some(COST_REP));
}
#[test]
fn distance_weighs_map_slips_half() {
let map = vec![vec!['a', 'ą'], vec!['o', 'ó', 'u']];
let related = |a: char, b: char| fold(a, &map) == fold(b, &map);
let c = |s: &str| s.chars().collect::<Vec<_>>();
assert_eq!(weighted_distance(&c("ktury"), &c("który"), &related, 9), 1);
assert_eq!(weighted_distance(&c("kot"), &c("kto"), &related, 9), 2);
assert_eq!(weighted_distance(&c("kot"), &c("kotek"), &related, 9), 4);
assert_eq!(weighted_distance(&c("kot"), &c("pies"), &related, 2), 3);
assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4);
}
/// A KEEPCASE word is offered in its dictionary case when the word's own case would be flagged
/// (`Kmm` → `km`, never `Km`), and a capitalised KEEPCASE word gets its dictionary form.
#[test]
fn keepcase_suggestions_are_not_recased_into_errors() {
let d = AffixDict::from_hunspell("KEEPCASE K\n", "2\nkm/K\npies\n", &[]).unwrap();
assert!(!d.check("Km") && !d.check("KM"));
for (word, want) in [("Kmm", "km"), ("KMM", "km"), ("Km", "km")] {
let got = suggest_in(&d, word, 4, &Tuning::DEFAULT);
assert!(got.iter().all(|s| s.split(' ').all(|p| d.check(p))), "{word}: {got:?}");
assert_eq!(got.first().map(String::as_str), Some(want), "{word}: {got:?}");
}
assert_eq!(suggest_in(&d, "Pies", 4, &Tuning::DEFAULT), Vec::<String>::new());
assert_eq!(suggest_in(&d, "Piesx", 4, &Tuning::DEFAULT).first().map(String::as_str), Some("Pies"));
}
/// A language's tables come after its dictionary's, a TRY alphabet stands in for a missing TRY
/// line, and a split rule decides which missing spaces are offered.
#[test]
fn tuning_adds_to_the_dictionary_tables() {
let d = AffixDict::from_hunspell("", "3\nčaj\nkot\nna\n", &[]).unwrap();
let tuned = Tuning { map: &["cč"], alphabet: "t", rep: &[("xy", "kot")], ..Tuning::DEFAULT };
assert!(!suggest_in(&d, "caj", 4, &Tuning::DEFAULT).contains(&"čaj".into()));
assert!(suggest_in(&d, "caj", 4, &tuned).contains(&"čaj".into()), "the language's MAP group");
assert!(suggest_in(&d, "xy", 4, &tuned).contains(&"kot".into()), "the language's REP pair");
assert!(suggest_in(&d, "ko", 4, &tuned).contains(&"kot".into()), "the language's TRY letters");
let near = Tuning { split: Split::Near { leading: &["na"], trailing: &[] }, ..Tuning::DEFAULT };
assert!(suggest_in(&d, "nakot", 4, &near).contains(&"na kot".into()));
assert!(!suggest_in(&d, "kotna", 4, &near).contains(&"kot na".into()), "no stray short half");
assert!(suggest_in(&d, "kotna", 4, &Tuning::DEFAULT).contains(&"kot na".into()));
}
}
crates/proof/src/bg.rs
+112
View File
@@ -0,0 +1,112 @@
//! Bulgarian hyphenation by rule ([`ProofLang::Bg`](crate::ProofLang::Bg)).
//!
//! Bulgarian spelling splits words at syllables, and its rules are regular enough to apply
//! directly, so no pattern file or word list is needed (and none is borrowed):
//! - two vowels next to each other may be split (`на-у-ка`);
//! - one consonant between vowels goes with the next syllable (`ма-ма`, `ро-за`);
//! - of two or more consonants between vowels, the first stays with the syllable before and the
//! rest go with the next (`сес-тра`, `въз-дух`, `ком-пю-тър`) — the spelling rules allow any
//! split of a consonant group, and this is the usual one — except that a consonant followed by
//! `р` or `л` starts the next syllable together with it (`до-бре`, `ко-пре`, `та-бли-ца`);
//! - `й` before a consonant stays with the vowel before it (`строй-ка`, `май-ка`);
//! - `ьо` is never split, and the consonant before `ь` goes with it (`шо-фьор`);
//! - `дж` and `дз`, one sound each, are never split (`хо-джа`).
//!
//! A single letter left at the end or start of a line is ruled out by the paragraph's
//! [`Limits`](crate::hyphen::Limits) (`Бъл-га-рия`, not `Бъл-га-ри-я`).
fn is_vowel(c: char) -> bool {
matches!(c, 'а' | 'е' | 'и' | 'о' | 'у' | 'ъ' | 'ю' | 'я' | 'ѝ')
}
/// Consonants that may start a syllable before `р` or `л` (`бр`, `пл`, `тр`, `гл` …).
fn is_obstruent(c: char) -> bool {
matches!(c, 'б' | 'в' | 'г' | 'д' | 'з' | 'к' | 'п' | 'с' | 'т' | 'ф' | 'х')
}
/// A lowercase Bulgarian letter (the words [`points`] splits).
pub(crate) fn is_letter(c: char) -> bool {
matches!(c, 'а'..='я' | 'ѝ')
}
/// Break points of one lowercase Bulgarian word (char indices; a hyphen goes before that char).
/// Empty for anything that is not entirely Bulgarian letters.
pub fn points(word: &[char]) -> Vec<usize> {
if word.len() < 2 || !word.iter().all(|&c| is_letter(c)) {
return Vec::new();
}
let vowels: Vec<usize> = (0..word.len()).filter(|&i| is_vowel(word[i])).collect();
let mut out = Vec::new();
for pair in vowels.windows(2) {
let (v1, v2) = (pair[0], pair[1]);
let mut pt = match v2 - v1 - 1 {
0 | 1 => v1 + 1,
_ => v1 + 2,
};
// A consonant with `р` or `л` after it starts a syllable together with it.
if v2 - v1 > 2 && matches!(word[v2 - 1], 'р' | 'л') && is_obstruent(word[v2 - 2]) {
pt = v2 - 2;
}
// `дж`, `дз`: one sound, so both go to the next syllable.
if pt > v1 + 1 && word[pt - 1] == 'д' && matches!(word[pt], 'ж' | 'з') {
pt -= 1;
}
// `ь` keeps the consonant before it (`фьо`).
if pt < v2 && word[pt] == 'ь' && pt > v1 + 1 {
pt -= 1;
}
if pt > v1 && pt <= v2 && word[pt - 1] != 'ь' {
out.push(pt);
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
fn split(w: &str) -> String {
let chars: Vec<char> = w.chars().collect();
let pts = points(&chars);
let mut s = String::new();
for (i, c) in chars.iter().enumerate() {
if pts.contains(&i) {
s.push('-');
}
s.push(*c);
}
s
}
#[test]
fn syllables_follow_the_bulgarian_rules() {
for (word, expected) in [
("мама", "ма-ма"),
("наука", "на-у-ка"),
("сестра", "сес-тра"),
("въздух", "въз-дух"),
("компютър", "ком-пю-тър"),
("стройка", "строй-ка"),
("майка", "май-ка"),
("шофьор", "шо-фьор"),
("ходжа", "хо-джа"),
("здравей", "здра-вей"),
("плевен", "пле-вен"),
("българия", "бъл-га-ри-я"),
("сричкопренасяне", "срич-ко-пре-на-ся-не"),
("добре", "до-бре"),
("таблица", "та-бли-ца"),
("отговор", "от-го-вор"),
] {
assert_eq!(split(word), expected, "{word}");
}
}
#[test]
fn non_bulgarian_letters_are_left_alone() {
assert!(points(&"word".chars().collect::<Vec<_>>()).is_empty());
assert!(points(&"съобщение1".chars().collect::<Vec<_>>()).is_empty());
assert!(points(&"объём".chars().collect::<Vec<_>>()).is_empty(), "Russian ё is not a Bulgarian letter");
}
}
crates/proof/src/cs/grammar.rs
+103
View File
@@ -0,0 +1,103 @@
//! Generic repeated-word and spacing checks with Czech messages.
//! No Czech-specific grammar, comma or sentence-capital rules are inferred.
use crate::{Issue, IssueKind, ProofLang, words};
fn issue(start: usize, end: usize, message: String, suggestions: Vec<String>) -> Issue {
Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::Cs }
}
pub fn check(text: &str) -> Vec<Issue> {
let mut issues = Vec::new();
repeated_words(text, &words(text), &mut issues);
spaces(text, &mut issues);
issues.sort_by_key(|i| (i.start, i.end));
issues
}
fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
for pair in ws.windows(2) {
let [(a0, b0), (a1, b1)] = pair else { continue };
let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue };
if !gap.is_empty() && gap.chars().all(char::is_whitespace) && w0.chars().all(char::is_alphabetic) && w0.to_lowercase() == w1.to_lowercase() {
v.push(issue(*a0, *b1, format!("Opakované slovo: „{w1}“"), vec![w0.to_string()]));
}
}
}
fn spaces(text: &str, v: &mut Vec<Issue>) {
for (i, c) in text.char_indices() {
if c != ' ' {
continue;
}
let rest = text.get(i + 1..).unwrap_or("");
let mut it = rest.chars();
let Some(p) = it.next() else { continue };
let after = it.next();
// A ratio/score keeps the spaces in digit-space-colon-space-digit.
let ratio = p == ':'
&& text.get(..i).and_then(|before| before.chars().next_back()).is_some_and(|c| c.is_ascii_digit())
&& after == Some(' ')
&& it.next().is_some_and(|c| c.is_ascii_digit());
let flag = !ratio
&& match p {
',' => true,
// `;)` and `;-)` are emoticons.
';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')),
// ` ...` is an ellipsis, ` .5` a number.
'.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
// ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space.
':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')),
_ => false,
};
if flag {
v.push(issue(i, i + 1 + p.len_utf8(), "Mezera před interpunkcí".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Dvojitá mezera".into(), vec![" ".into()]));
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn repeated_words_and_spaces_have_czech_messages() {
let text = "Město město , škola.";
let issues = check(text);
assert_eq!(issues.len(), 3, "{issues:?}");
assert_eq!(issues[0].message, "Opakované slovo: „město“");
assert_eq!(issues[0].suggestions, ["Město"]);
assert!(issues.iter().any(|i| i.message == "Mezera před interpunkcí" && &text[i.start..i.end] == " ,"));
assert!(issues.iter().any(|i| i.message == "Dvojitá mezera"));
assert!(issues.iter().all(|i| i.lang == ProofLang::Cs && i.kind == IssueKind::Grammar));
for text in ["město. škola", "Dr. Novák", "A apple", "Město, město!", "Úsměv :) Číslo .5 a tečky ..."] {
assert!(check(text).is_empty(), "{text}");
}
}
#[test]
fn ratios_and_scores_keep_their_spaces() {
for text in ["3 : 1", "1 : 2", "Skóre 12 : 10.", "Poměr 1 : 100."] {
assert!(check(text).is_empty(), "{text}: {:?}", check(text));
}
for text in ["Slovo : další", "3 : slovo", "slovo : 1", "3 :"] {
assert!(check(text).iter().any(|i| i.message == "Mezera před interpunkcí"), "{text}");
}
}
proptest::proptest! {
#[test]
fn never_panics_and_ranges_are_valid(s in "\\PC{0,80}") {
for issue in check(&s) {
proptest::prop_assert!(issue.start <= issue.end && issue.end <= s.len());
proptest::prop_assert!(s.is_char_boundary(issue.start) && s.is_char_boundary(issue.end));
}
}
}
}
crates/proof/src/cs/mod.rs
+201
View File
@@ -0,0 +1,201 @@
//! Czech proofing using the installed Hunspell `cs_CZ` dictionary (never bundled).
//! Suggestions use its runtime MAP/REP/TRY/KEY tables. Generic grammar checks have Czech
//! messages; hyphenation keeps the existing fallback, without Czech pattern data.
pub mod grammar;
pub(crate) mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// File names of a Czech Hunspell dictionary.
const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"];
/// The Czech dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Czech");
/// The Czech dictionary: the installed Hunspell `cs_CZ` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Czech spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
DICT.get()
}
/// Use this Czech dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
DICT.install(aff, dic)
}
/// Is a Czech dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
DICT.has_dictionary()
}
/// Is `word` spelled correctly in Czech? Numbers, single letters, short words in capitals
/// (acronyms: `ČR`, `EU`), words with digits and words in other scripts are accepted, and so
/// is every word when no Czech dictionary is installed.
pub fn is_correct(word: &str) -> bool {
is_correct_with(&dictionary(), word)
}
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off,
/// short words in capitals are checked against the dictionary too, and with Ignore words that
/// contain numbers off, so are words with digits.
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
check(&dictionary(), word, opts)
}
/// [`is_correct`] against a given dictionary (an empty one accepts everything).
pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
check(d, word, &crate::Options::default())
}
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
};
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
}
known(d, w)
}
/// Is one word (no hyphens) in the dictionary or the user dictionary?
fn known(d: &AffixDict, w: &str) -> bool {
d.check(w) || crate::in_user_dictionary(&w.to_lowercase())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn dictionary_lookup_accepts_czech_file_names() {
let dir = std::env::temp_dir().join(format!("wordcraft-cs-aliases-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
for name in ["cs_CZ", "cs-CZ", "cs"] {
std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap();
std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap();
let (aff, dic) = crate::hunspell::find_in(&dir, NAMES).unwrap();
let d = crate::hunspell::load(&aff, &dic).unwrap();
assert!(is_correct_with(&d, "městě"));
std::fs::remove_file(aff).unwrap();
std::fs::remove_file(dic).unwrap();
}
std::fs::remove_dir(dir).unwrap();
}
#[test]
fn czech_words_affixes_chains_case_and_diacritics() {
test_dict::install();
let d = dictionary();
assert!(has_dictionary());
assert_eq!(d.len(), 20);
assert_eq!(d.rule_count(), 8);
for w in [
"město",
"města",
"městě",
"neměsta",
"rychle",
"rychlejší",
"učitel",
"učitelé",
"Praha",
"Praze",
"Město",
"POČÍTAČ",
"příliš",
"žluťoučký",
"kůň",
"úpěl",
"ďábelské",
"ódy",
"děkujeme",
"zpráva",
"zprava",
"škola-počítač",
] {
assert!(is_correct(w), "{w}");
}
for w in ["městu", "městoě", "rychleejší", "učiteléé", "praha", "mĚsto", "pocitac", "dekujem", "škola-pocitac"] {
assert!(!is_correct(w), "{w}");
}
for c in "ěščřžýáíéůú".chars() {
assert!(test_dict::DIC.contains(c), "UTF-8 fixture covers {c}");
}
}
#[test]
fn forbidden_words_are_rejected_in_every_case() {
test_dict::install();
for w in ["zqzakázané", "Zqzakázané", "ZQZAKÁZANÉ"] {
assert!(!is_correct(w), "{w}");
}
}
#[test]
fn missing_dictionary_and_ignored_inputs_are_quiet() {
let empty = AffixDict::default();
assert!(is_correct_with(&empty, "pocitac"));
assert!(install_dictionary("FLAG long\n", "1\nstrom\n").is_err());
let d = AffixDict::from_hunspell(test_dict::AFF, test_dict::DIC, &[]).unwrap();
for w in ["", "a", "2026", "ČR", "článek2", "日本語"] {
assert!(is_correct_with(&d, w), "{w}");
}
}
#[test]
fn user_dictionary_applies() {
test_dict::install();
assert!(!is_correct("zqčeskévlastníslovo"));
crate::add_word("Zqčeskévlastníslovo");
assert!(is_correct("zqčeskévlastníslovo"));
}
#[test]
fn czech_dispatch_messages_options_and_mixed_spans() {
test_dict::install();
assert!(crate::has_dictionary(Some("cs-CZ")));
assert!(crate::is_correct_for("městě", Some("cze")));
let text = "Město pocitac";
let issues = crate::check_text(text, &[(0..text.len(), crate::ProofLang::Cs)]);
assert_eq!(issues.len(), 1, "{issues:?}");
assert_eq!(&text[issues[0].start..issues[0].end], "pocitac");
assert_eq!(issues[0].message, "Možná pravopisná chyba");
assert_eq!(issues[0].lang, crate::ProofLang::Cs);
assert!(crate::suggest_in("pocitac", 8, crate::ProofLang::Cs).contains(&"počítač".into()));
let text = "Město this is is wrong";
let issues = crate::check_text(text, &[(0..6, crate::ProofLang::Cs), (7..text.len(), crate::ProofLang::En)]);
assert!(issues.iter().any(|i| i.lang == crate::ProofLang::En && i.message.starts_with("Repeated word")));
assert!(issues.iter().all(|i| i.start >= 7));
let opts = crate::Options { ignore_uppercase: false, ..Default::default() };
assert!(!crate::is_correct_in_with("ZQZAKÁZANÉ", crate::ProofLang::Cs, &opts));
}
/// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Czech checks
/// those words against the dictionary instead of accepting them.
#[test]
fn proofing_options_reach_the_czech_check() {
test_dict::install();
let lang = crate::ProofLang::Cs;
let defaults = crate::Options::default();
let check_caps = crate::Options { ignore_uppercase: false, ..defaults };
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
assert!(crate::is_correct_in_with("ŠKULA", lang, &defaults), "short capitals are acronyms by default");
assert!(!crate::is_correct_in_with("ŠKULA", lang, &check_caps));
assert!(crate::is_correct_in_with("ŠKOLA", lang, &check_caps));
assert!(crate::is_correct_in_with("škula2", lang, &defaults));
assert!(!crate::is_correct_in_with("škula2", lang, &check_digits));
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
assert!(is_correct("ŠKULA"), "direct checks keep the defaults");
}
}
crates/proof/src/cs/suggest.rs
+78
View File
@@ -0,0 +1,78 @@
//! Czech suggestions: the shared ranking ([`crate::affix_suggest`]) with the installed
//! dictionary's own MAP/REP/TRY/KEY tables and nothing Czech hard-coded.
use crate::affix::AffixDict;
use crate::affix_suggest::Tuning;
/// Spelling suggestions for a Czech word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
suggest_in(&super::dictionary(), word, max)
}
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn suggestions_are_fast_enough() {
crate::cs::test_dict::install();
let _ = suggest("město", 1);
let t = std::time::Instant::now();
for word in ["zqčeskévelmidlouhéslovo", "pocitacz", "nejrychllejší"] {
let _ = suggest(word, 6);
}
let ms = t.elapsed().as_secs_f64() * 1000.0;
eprintln!("3 Czech suggestion calls: {ms:.0} ms");
assert!(ms < 20_000.0, "{ms} ms");
}
#[test]
fn nosuggest_stems_and_forms_are_never_offered() {
crate::affix::tests::assert_nosuggest_is_honoured(suggest_in);
}
#[test]
fn czech_suggestions_use_runtime_tables_and_case() {
crate::cs::test_dict::install();
for (word, want) in [
("mesto", "město"),
("mňe", "mě"),
("xůž", "nůž"),
("škloa", "škola"),
("pocitac", "počítač"),
("Mesto", "Město"),
("praha", "Praha"),
("toškola", "to škola"),
] {
assert!(suggest(word, 8).contains(&want.into()), "{word}: {:?}", suggest(word, 8));
}
// Each table changes the answer without a hard-coded Czech alphabet or confusion list.
for (table, wrong, right) in
[("MAP 1\nMAP cč\n", "caj", "čaj"), ("REP 1\nREP xy město\n", "xy", "město"), ("TRY š\n", "kola", "škola"), ("KEY xn\n", "xůž", "nůž")]
{
let dic = format!("1\n{right}\n");
let d = AffixDict::from_hunspell(table, &dic, &[]).unwrap();
assert!(suggest_in(&d, wrong, 4).contains(&right.into()), "{table}");
let without = AffixDict::from_hunspell("", &dic, &[]).unwrap();
assert!(!suggest_in(&without, wrong, 4).contains(&right.into()), "must depend on {table}");
}
}
#[test]
fn suggestions_stay_bounded_and_skip_forbidden_words() {
crate::cs::test_dict::install();
for w in ["", "dvě slova", &"ě".repeat(41)] {
assert!(suggest(w, 4).is_empty());
}
assert!(suggest("mesto", 0).is_empty());
assert!(suggest_in(&AffixDict::default(), "mesto", 4).is_empty());
assert!(suggest("město", 4).len() <= 4);
assert!(!suggest("město", 4).contains(&"město".into()));
assert!(!suggest("zqzakázane", 8).contains(&"zqzakázané".into()));
}
}
crates/proof/src/cs/test_dict.rs
+59
View File
@@ -0,0 +1,59 @@
//! Hand-written synthetic Czech test data. No words, rules or suggestion tables were copied
//! from a system dictionary. This is a small fixture, not a redistributable Czech dictionary.
pub const AFF: &str = "SET UTF-8
FLAG UTF-8
FORBIDDENWORD F
TRY aeiouěščřžýáíéůúntklmp
KEY xn|qř
MAP 5
MAP eěé
MAP cč
MAP rř
MAP uůú
MAP ií
REP 1
REP mňe mě
SFX A Y 2 # synthetic declension
SFX A o a o
SFX A o ě o
SFX B N 1
SFX B 0 e/C .
SFX C N 1
SFX C e ejší e
SFX D N 1
SFX D 0 itel/E .
SFX E N 1
SFX E 0 é .
SFX P Y 1
SFX P ha ze ha
PFX X Y 1
PFX X 0 ne .
";
pub const DIC: &str = "20
město/AX
rychl/B
uč/D
Praha/P
škola
počítač
příliš
žluťoučký
kůň
úpěl
ďábelské
ódy
zpráva
zprava
děkujeme
mě
nůž
si
to
zqzakázané/F
";
pub fn install() {
super::install_dictionary(AFF, DIC).unwrap();
}
crates/proof/src/dict.rs
+195
View File
@@ -0,0 +1,195 @@
//! Hyphenation exception dictionary: lowercase words with their break points.
//!
//! On-disk format (craft-i18n's `language-data/en-us/en-us.dic`): raw-deflate of
//! `"DCHYPH1\n"` followed by one line per word, sorted by the plain word's bytes. Each line is
//! front-coded against the previous one: a byte with the shared prefix length (in bytes of the
//! hyphenated form), then the rest of the hyphenated form (`-` at every break), then `\n`.
/// Sorted word list with a break bitmask per word (bit `i` = hyphen before char `i`).
#[derive(Clone, Debug, Default)]
pub struct Dictionary {
words: String,
offs: Vec<u32>,
masks: Vec<u64>,
}
const MAGIC: &[u8] = b"DCHYPH1\n";
/// Largest inflated dictionary, and most bytes of words kept from one (the bundled US English one
/// is about 1 MB).
const MAX_BYTES: usize = 64 << 20;
/// Most words read from one dictionary (the bundled one has about 160k).
const MAX_ENTRIES: usize = 4_000_000;
impl Dictionary {
/// Build from `(lowercase word, break mask)` pairs (sorted and de-duplicated here; the first
/// entry of a duplicate wins).
pub fn from_entries(mut entries: Vec<(String, u64)>) -> Dictionary {
entries.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes()));
entries.dedup_by(|b, a| a.0 == b.0);
let mut d = Dictionary { words: String::new(), offs: vec![0], masks: Vec::with_capacity(entries.len()) };
for (w, m) in entries {
d.words.push_str(&w);
d.offs.push(d.words.len() as u32);
d.masks.push(m);
}
d
}
pub fn len(&self) -> usize {
self.masks.len()
}
pub fn is_empty(&self) -> bool {
self.masks.is_empty()
}
fn word(&self, i: usize) -> &str {
&self.words[self.offs[i] as usize..self.offs[i + 1] as usize]
}
/// Break mask of an exact (already lowercased) word.
pub fn get(&self, word: &str) -> Option<u64> {
let (mut lo, mut hi) = (0usize, self.len());
while lo < hi {
let mid = (lo + hi) / 2;
match self.word(mid).as_bytes().cmp(word.as_bytes()) {
std::cmp::Ordering::Less => lo = mid + 1,
std::cmp::Ordering::Greater => hi = mid,
std::cmp::Ordering::Equal => return Some(self.masks[mid]),
}
}
None
}
/// Iterate `(word, mask)`.
pub fn iter(&self) -> impl Iterator<Item = (&str, u64)> + '_ {
(0..self.len()).map(|i| (self.word(i), self.masks[i]))
}
/// Serialize (deflate-compressed).
pub fn to_bytes(&self) -> Vec<u8> {
let mut raw = MAGIC.to_vec();
let mut prev: Vec<u8> = Vec::new();
for (w, m) in self.iter() {
let h = hyphenated(w, m).into_bytes();
let p = prev.iter().zip(&h).take_while(|(a, b)| a == b).count().min(255);
raw.push(p as u8);
raw.extend_from_slice(&h[p..]);
raw.push(b'\n');
prev = h;
}
miniz_oxide::deflate::compress_to_vec(&raw, 10)
}
/// Deserialize data written by [`Dictionary::to_bytes`]. Data that inflates past
/// [`MAX_BYTES`], or holds more than [`MAX_ENTRIES`] words or [`MAX_BYTES`] of them, is refused.
pub fn from_bytes(data: &[u8]) -> Result<Dictionary, String> {
Dictionary::from_bytes_within(data, MAX_BYTES, MAX_ENTRIES)
}
/// [`Dictionary::from_bytes`] with the limits given.
fn from_bytes_within(data: &[u8], max_bytes: usize, max_entries: usize) -> Result<Dictionary, String> {
let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(data, max_bytes)
.map_err(|e| format!("hyphenation dictionary: inflate failed: {:?}", e.status))?;
let body = raw.strip_prefix(MAGIC).ok_or("hyphenation dictionary: bad header")?;
let mut d = Dictionary { words: String::with_capacity(body.len()), offs: vec![0], masks: Vec::new() };
let mut cur: Vec<u8> = Vec::new();
let mut i = 0;
while i < body.len() {
let p = body[i] as usize;
let end = body[i + 1..].iter().position(|&b| b == b'\n').map(|e| i + 1 + e).ok_or("hyphenation dictionary: truncated")?;
if p > cur.len() {
return Err("hyphenation dictionary: bad prefix".into());
}
cur.truncate(p);
cur.extend_from_slice(&body[i + 1..end]);
i = end + 1;
let s = std::str::from_utf8(&cur).map_err(|_| "hyphenation dictionary: bad utf-8")?;
let (w, m) = parse_hyphenated(s);
// Front coding lets a short line repeat a long prefix: bound what is kept, not only
// what is read.
if d.masks.len() >= max_entries || d.words.len() + w.len() > max_bytes {
return Err("hyphenation dictionary: too large".into());
}
d.words.push_str(&w);
d.offs.push(d.words.len() as u32);
d.masks.push(m);
}
Ok(d)
}
}
/// `word` with `-` before every char whose bit is set in `mask`.
pub fn hyphenated(word: &str, mask: u64) -> String {
let mut s = String::with_capacity(word.len() + 8);
for (i, c) in word.chars().enumerate() {
if i > 0 && i < 64 && mask & (1 << i) != 0 {
s.push('-');
}
s.push(c);
}
s
}
/// Inverse of [`hyphenated`].
pub fn parse_hyphenated(s: &str) -> (String, u64) {
let mut w = String::with_capacity(s.len());
let mut m = 0u64;
let mut n = 0;
for c in s.chars() {
if c == '-' {
if n > 0 && n < 64 {
m |= 1 << n;
}
} else {
w.push(c);
n += 1;
}
}
(w, m)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn roundtrip() {
let e: Vec<(String, u64)> = ["ty-pog-ra-phy", "cat", "hap-pen", "o'clock", "naïve-té", "naïve"].iter().map(|s| parse_hyphenated(s)).collect();
let d = Dictionary::from_entries(e);
let back = Dictionary::from_bytes(&d.to_bytes()).unwrap();
assert_eq!(back.len(), 6);
assert_eq!(back.get("typography").map(|m| hyphenated("typography", m)), Some("ty-pog-ra-phy".to_string()));
assert_eq!(back.get("cat"), Some(0));
assert_eq!(back.get("naïveté").map(|m| hyphenated("naïveté", m)), Some("naïve-té".to_string()));
assert!(back.get("dog").is_none());
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn oversized_data_is_refused() {
const LIMIT: usize = 1 << 16;
let within = |raw: &[u8]| Dictionary::from_bytes_within(&miniz_oxide::deflate::compress_to_vec(raw, 6), LIMIT, 1000);
// Data that inflates past the limit is refused before anything is decoded.
let mut raw = MAGIC.to_vec();
raw.resize(LIMIT + 1, b'a');
assert!(within(&raw).unwrap_err().contains("inflate failed"));
// Short lines that each repeat a long prefix: small to inflate, but the words they spell
// out would be ~100x larger. Kept storage is bounded too.
let mut raw = MAGIC.to_vec();
raw.push(0);
raw.extend_from_slice(&[b'a'; 255]);
raw.push(b'\n');
for _ in 0..(LIMIT / 255 + 2) {
raw.extend_from_slice(&[255, b'\n']);
}
assert!(raw.len() < LIMIT);
assert!(within(&raw).unwrap_err().contains("too large"));
// So is the number of words.
let words: Vec<(String, u64)> = (0..1001).map(|i| (format!("w{i}"), 0)).collect();
let raw = miniz_oxide::inflate::decompress_to_vec(&Dictionary::from_entries(words).to_bytes()).unwrap();
assert!(within(&raw).unwrap_err().contains("too large"));
// The real limits leave the bundled dictionary whole.
assert!(Dictionary::en_us().len() > 100_000);
}
}
crates/proof/src/en_variants.rs
+278
View File
@@ -0,0 +1,278 @@
//! American and British spellings of the same English word (#1281, #1045): which forms each
//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`]) accepts, which it flags,
//! and what it suggests instead (`colour` → `color` in American English).
//!
//! The pairs come from craft-i18n's `language-data/en-us/en-variants.txt`, our own list of base forms with a
//! policy per English. Inflections are generated here by rule for the regular patterns (-or/-our,
//! -er/-re, -ize/-ise, -ense/-ence…), so `colours`, `centred` and `organisation` follow their
//! base forms. A form is right when the shared English word list knows it or knows its other
//! spelling (`realisation` is right in British English because `realization` is a word), so the
//! rules never make a misspelling right.
use std::collections::HashMap;
use std::sync::OnceLock;
use crate::ProofLang;
/// Our list of American/British pairs: `POLICY AMERICAN BRITISH` per line, `#` comments.
use crate::data::EN_VARIANTS as VARIANTS;
/// What one English does with a pair's two spellings.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Policy {
/// Only the American spelling is right.
American,
/// Only the British spelling is right.
British,
/// Both are right.
Both,
}
impl Policy {
fn from_char(c: char) -> Option<Policy> {
match c {
'u' => Some(Policy::American),
'b' => Some(Policy::British),
'*' => Some(Policy::Both),
_ => None,
}
}
}
/// One line of the list: the policies for American, Canadian and British-spelling English, and
/// the two base forms.
struct Pair {
policy: [Policy; 3],
american: &'static str,
british: &'static str,
}
/// The lines of [`VARIANTS`]; malformed lines are skipped (a test keeps the file clean).
fn pairs() -> &'static [Pair] {
static P: OnceLock<Vec<Pair>> = OnceLock::new();
P.get_or_init(|| VARIANTS.lines().filter_map(parse_line).collect())
}
fn parse_line(line: &'static str) -> Option<Pair> {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
return None;
}
let mut parts = line.split_whitespace();
let (policy, american, british) = (parts.next()?, parts.next()?, parts.next()?);
if parts.next().is_some() || american == british {
return None;
}
let mut p = policy.chars().map(Policy::from_char);
let policy = [p.next()??, p.next()??, p.next()??];
if p.next().is_some() {
return None;
}
let word = |w: &str| !w.is_empty() && w.chars().all(|c| c.is_ascii_lowercase());
(word(american) && word(british)).then_some(Pair { policy, american, british })
}
/// The column of a pair's policy for `lang`; `None` for a language that isn't English.
fn column(lang: ProofLang) -> Option<usize> {
match lang {
ProofLang::En => Some(0),
ProofLang::EnCa => Some(1),
ProofLang::EnGb => Some(2),
_ => None,
}
}
/// `word` with `suffix`: a final `e` drops before a vowel (`centre` + `ing` → `centring`), a
/// final consonant + `y` becomes `i` (`cosy` + `er` → `cosier`), `s` is the plural (`es` after a
/// sibilant, `ies` after a consonant + `y`).
fn add(word: &str, suffix: &str) -> String {
let b = word.as_bytes();
let last = b.last().copied().unwrap_or(b'_');
let before = b.len().checked_sub(2).and_then(|i| b.get(i)).copied().unwrap_or(b'_');
let vowel = |c: u8| b"aeiou".contains(&c);
let head = word.get(..word.len().saturating_sub(1)).unwrap_or(word);
if suffix == "s" {
if last == b'y' && !vowel(before) {
return format!("{head}ies");
}
if matches!(last, b's' | b'x' | b'z') || word.ends_with("ch") || word.ends_with("sh") {
return format!("{word}es");
}
return format!("{word}s");
}
let suffix_vowel = suffix.as_bytes().first().is_some_and(|&c| vowel(c));
if last == b'e' && suffix_vowel {
return format!("{head}{suffix}");
}
if last == b'y' && !vowel(before) && !suffix.starts_with('i') {
return format!("{head}i{suffix}");
}
format!("{word}{suffix}")
}
/// The suffixes both spellings of a pair take, by the pattern that tells them apart.
fn suffixes(american: &str, british: &str) -> &'static [&'static str] {
// Not -al, -ist or -ite: `humoral` and `humorist` are spelled so in British English too, and
// `behavioural` and `favourite` have lines of their own.
const OUR: &[&str] = &["s", "ed", "ing", "er", "ers", "ful", "fully", "less", "able", "ably"];
const RE: &[&str] = &["s", "ed", "ing"];
const IZE: &[&str] = &["s", "ed", "ing", "er", "ers", "ation", "ations", "ational", "able"];
const E: &[&str] = &["s", "ed", "ing", "er", "ers"];
const Y: &[&str] = &["s", "er", "est", "ly", "ness"];
const PLURAL: &[&str] = &["s"];
let stem = |w: &str, n: usize| w.get(..w.len().saturating_sub(n)).map(str::to_string);
if american.ends_with("or") && british.ends_with("our") && stem(american, 2) == stem(british, 3) {
return OUR;
}
if american.ends_with("er") && british.ends_with("re") && stem(american, 2) == stem(british, 2) {
return RE;
}
// One letter apart (`organize`/`organise`, `defense`/`defence`, `cozy`/`cosy`).
let differ = american.len() == british.len() && american.bytes().zip(british.bytes()).filter(|(a, b)| a != b).count() == 1;
if differ && american.ends_with('e') && british.ends_with('e') {
let ize = american.ends_with("ize") || american.ends_with("yze");
return if ize { IZE } else { E };
}
if differ && american.ends_with('y') && british.ends_with('y') {
return Y;
}
PLURAL
}
/// One English's forms: those it flags (each with the form to suggest), and those it accepts
/// when the shared list knows them or their other spelling (each with that other spelling).
#[derive(Default)]
struct Forms {
foreign: HashMap<String, String>,
accepted: HashMap<String, String>,
}
fn forms(lang: ProofLang) -> Option<&'static Forms> {
static F: [OnceLock<Forms>; 3] = [OnceLock::new(), OnceLock::new(), OnceLock::new()];
let col = column(lang)?;
Some(F.get(col)?.get_or_init(|| build(col)))
}
fn build(col: usize) -> Forms {
let mut f = Forms::default();
for pair in pairs() {
let Some(&policy) = pair.policy.get(col) else { continue };
let mut both = vec![(pair.american.to_string(), pair.british.to_string())];
both.extend(suffixes(pair.american, pair.british).iter().map(|s| (add(pair.american, s), add(pair.british, s))));
for (am, br) in both {
if am == br {
continue;
}
match policy {
Policy::American => {
f.foreign.insert(br.clone(), am.clone());
f.accepted.insert(am, br);
}
Policy::British => {
f.foreign.insert(am.clone(), br.clone());
f.accepted.insert(br, am);
}
Policy::Both => {
f.accepted.insert(am.clone(), br.clone());
f.accepted.insert(br, am);
}
}
}
}
// A form one pair accepts is never flagged by another, and `analyses`, `paralyses`… are also
// the plurals of `analysis`, `paralysis`… everywhere.
f.foreign.retain(|w, _| !f.accepted.contains_key(w) && !w.ends_with("yses"));
f
}
/// `lower` without a possessive `'s`.
fn without_possessive(lower: &str) -> &str {
lower.strip_suffix("'s").filter(|s| !s.is_empty()).unwrap_or(lower)
}
/// The spelling `lang` wants instead of `lower` (a lower-case word, `'` apostrophes), when
/// `lower` is the other dialect's spelling of a word (`colour` in American English: `color`).
/// A possessive keeps its `'s` (`colour's` → `color's`).
pub fn preferred(lower: &str, lang: ProofLang) -> Option<String> {
let f = forms(lang)?;
let base = without_possessive(lower);
let native = f.foreign.get(base)?;
Some(if base.len() < lower.len() { format!("{native}'s") } else { native.clone() })
}
/// Is `lower` right in `lang` because the shared word list knows its other spelling (`known`
/// says what the list knows)? `realisation` in British English, `organization` in all three.
pub fn known_variant(lower: &str, lang: ProofLang, known: impl Fn(&str) -> bool) -> bool {
let Some(f) = forms(lang) else { return false };
let base = without_possessive(lower);
f.accepted.get(base).is_some_and(|other| known(other))
}
/// Is `lower` the other dialect's spelling in `lang` (flagged even though it is a word)?
pub fn is_foreign(lower: &str, lang: ProofLang) -> bool {
forms(lang).is_some_and(|f| f.foreign.contains_key(without_possessive(lower)))
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn every_line_parses() {
let lines = VARIANTS.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')).count();
assert_eq!(pairs().len(), lines, "a malformed line in en-variants.txt");
assert!(pairs().len() > 250);
}
#[test]
fn suffixes_follow_the_pattern() {
assert_eq!(add("centre", "ing"), "centring");
assert_eq!(add("centre", "ed"), "centred");
assert_eq!(add("center", "ed"), "centered");
assert_eq!(add("organise", "ation"), "organisation");
assert_eq!(add("cosy", "er"), "cosier");
assert_eq!(add("cosy", "s"), "cosies");
assert_eq!(add("armoury", "s"), "armouries");
assert_eq!(add("analyse", "s"), "analyses");
assert_eq!(add("colour", "ful"), "colourful");
assert_eq!(add("", "s"), "s");
assert_eq!(suffixes("color", "colour").len(), 10);
assert_eq!(suffixes("center", "centre"), ["s", "ed", "ing"]);
assert!(suffixes("organize", "organise").contains(&"ation"));
assert_eq!(suffixes("program", "programme"), ["s"], "no -ed: programmed is American too");
}
#[test]
fn no_form_is_both_flagged_and_accepted() {
for lang in [ProofLang::En, ProofLang::EnCa, ProofLang::EnGb] {
let Some(f) = forms(lang) else { panic!("{lang:?}") };
for (w, native) in &f.foreign {
assert!(!f.accepted.contains_key(w), "{lang:?}: {w}");
assert!(!f.foreign.contains_key(native), "{lang:?}: suggests {native}, itself flagged");
}
}
// Words both dialects use alike are never flagged.
for w in [
"programmed",
"tired",
"tires",
"curbs",
"humorous",
"honorary",
"humorist",
"humoral",
"meter",
"license",
"for",
"tour",
"analyses",
"paralyses",
] {
for lang in [ProofLang::En, ProofLang::EnCa, ProofLang::EnGb] {
assert!(!is_foreign(w, lang), "{lang:?}: {w}");
}
}
assert!(forms(ProofLang::Pl).is_none() && !is_foreign("color", ProofLang::Other));
}
}
crates/proof/src/english.rs
+488
View File
@@ -0,0 +1,488 @@
//! English spelling never goes unchecked. Its words come from the first of these that is there:
//!
//! 1. **Embedded**: the Moby word list embedded from craft-i18n at build time (`CRAFT_I18N_DIR`,
//! see `build.rs`). When it is there, nothing in this module is used.
//! 2. **System Hunspell**: a Hunspell `.aff`/`.dic` pair for the run's own English found by
//! [`crate::hunspell`] (`en_US` for American, `en_GB` for British, `en_CA` for Canadian
//! English; each name in the dictionaries folder and then the system locations before the
//! next name; none on the web). Another English's dictionary is never used: it would mark this
//! English's spellings (`colour` in an `en_US` one), so without its own dictionary an English
//! uses the built-in list, which has both spellings. A file that can't be read is skipped for
//! the next name or folder, and one file is read once for every English that uses it.
//! Directives that only matter for other languages (compounding, output conversion) are
//! ignored ([`crate::hunspell::load_english`]).
//! 3. **Built-in word list**: [`WORDS`], the whole public-domain Moby word list compiled into
//! the crate (without its break points), with WordCraft's own supplement of modern words
//! ([`EXTRA`]) and the usual inflections. A word it doesn't know is misspelled, as with the
//! embedded list.
//!
//! A Hunspell dictionary that is there but can't be read (unsupported encoding, too large, a
//! parse error, no words) is skipped, and its reason is reported by
//! [`crate::dictionary_load_error`]. Non-English text is never checked against any of these.
use std::path::{Path, PathBuf};
use std::sync::{Arc, OnceLock, PoisonError, RwLock};
use crate::affix::AffixDict;
use crate::dict::Dictionary;
use crate::{Options, ProofLang};
/// The English word list built into the crate (`data/en-words.dic`, [`Dictionary`]'s format with
/// no break points): every word of the public-domain Moby Hyphenator II list by Grady Ward
/// (generated by `examples/en_words.rs` from craft-i18n's `en-us.dic`).
const WORDS: &[u8] = include_bytes!("../data/en-words.dic");
/// WordCraft's supplement of common modern words the Moby list lacks (`data/en-extra.txt`, a copy
/// of craft-i18n's `language-data/en-us/en-extra.txt`, MIT OR Apache-2.0): one lowercase base
/// form per line, `#` comments. Used when the Language data doesn't embed its own copy.
pub(crate) const EXTRA: &str = include_str!("../data/en-extra.txt");
/// Longest word (in characters) suggestions are looked for: longer ones get none, which keeps
/// the edits tried per word bounded.
const MAX_SUGGEST_CHARS: usize = 32;
/// Most Hunspell `.aff`/`.dic` pairs tried for one English before giving up on Hunspell.
const MAX_CANDIDATES: usize = 8;
/// Where English spelling comes from now ([`crate::english_source`]).
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum EnglishSource {
/// The Moby word list embedded from craft-i18n at build time.
Embedded,
/// An English Hunspell dictionary from the dictionaries folder or the system: its `.dic` file.
Hunspell(PathBuf),
/// The Moby word list built into the crate: a word it doesn't know is misspelled.
CoreList,
}
/// The words of [`WORDS`] (decoded on first use; empty if they fail to decode, which a test rules
/// out).
fn words() -> &'static Dictionary {
static W: OnceLock<Dictionary> = OnceLock::new();
W.get_or_init(|| {
Dictionary::from_bytes(WORDS).unwrap_or_else(|e| {
log::error!("built-in English word list: {e}");
Dictionary::default()
})
})
}
/// Is `w` (lower case) a word of the built-in list?
pub(crate) fn word_list_contains(w: &str) -> bool {
words().get(w).is_some()
}
/// Hunspell file names of `lang`'s own dictionary, the usual one first. Only its own: another
/// English's dictionary would mark its spellings, which the built-in list doesn't.
fn names(lang: ProofLang) -> &'static [&'static str] {
match lang {
ProofLang::EnGb => &["en_GB", "en-GB"],
ProofLang::EnCa => &["en_CA", "en-CA"],
_ => &["en_US", "en-US"],
}
}
/// The slot of each English in [`State::slots`].
fn slot(lang: ProofLang) -> usize {
match lang {
ProofLang::EnGb => 1,
ProofLang::EnCa => 2,
_ => 0,
}
}
/// A loaded Hunspell dictionary and its `.dic` file; `None` when none could be loaded.
type Loaded = Option<(Arc<AffixDict>, PathBuf)>;
/// The English Hunspell dictionaries, under one lock.
struct State {
/// Per English (see [`slot`]): `None` until first use, then what was found.
slots: [Option<Loaded>; 3],
/// Every `.dic` file read, so Englishes that use the same file share one copy.
files: Vec<(PathBuf, Arc<AffixDict>)>,
/// Counts reloads, so a load that started before one is not kept.
reloads: u64,
}
impl State {
const fn new() -> Self {
State { slots: [None, None, None], files: Vec::new(), reloads: 0 }
}
}
static HUNSPELL: RwLock<State> = RwLock::new(State::new());
/// Why the embedded word list failed to decode, if it did.
static EMBEDDED_ERROR: OnceLock<String> = OnceLock::new();
/// Remember that the embedded word list is there but failed to decode (English then uses the
/// next source).
pub(crate) fn note_embedded_error(e: String) {
log::error!("embedded English word list: {e}; English spelling falls back to Hunspell or the built-in word list");
let _ = EMBEDDED_ERROR.set(e);
}
/// The English Hunspell dictionary for `lang` (read on first use).
fn hunspell(lang: ProofLang) -> Loaded {
let i = slot(lang);
let reloads = {
let s = HUNSPELL.read().unwrap_or_else(PoisonError::into_inner);
if let Some(Some(found)) = s.slots.get(i) {
return found.clone();
}
s.reloads
};
let (loaded, error) = load(lang);
keep(&HUNSPELL, i, reloads, loaded, || note_error(lang, error))
}
/// Keep in `state` what the load of slot `i`, started after `reloads` reloads, found, and `note`
/// its load error. If the folder was reloaded since, the load may have read the old folder: it
/// serves this one check, its error isn't noted, and the next check loads again.
fn keep(state: &RwLock<State>, i: usize, reloads: u64, loaded: Loaded, note: impl FnOnce()) -> Loaded {
let mut s = state.write().unwrap_or_else(PoisonError::into_inner);
if s.reloads != reloads {
return match s.slots.get(i) {
Some(Some(found)) => found.clone(),
_ => loaded,
};
}
note();
let loaded = loaded.map(|(d, dic)| {
// Another English may have read the same file meanwhile: share its copy.
let d = match s.files.iter().find(|(p, _)| *p == dic) {
Some((_, shared)) => shared.clone(),
None => {
s.files.push((dic.clone(), d.clone()));
d
}
};
(d, dic)
});
// Another thread may have filled it meanwhile: that one wins.
match s.slots.get_mut(i) {
Some(slot) => slot.get_or_insert(loaded).clone(),
None => loaded,
}
}
/// The dictionary of the `.dic` file `dic` in `state`, if an English already read it.
fn already_read(state: &RwLock<State>, dic: &Path) -> Option<Arc<AffixDict>> {
state.read().unwrap_or_else(PoisonError::into_inner).files.iter().find(|(p, _)| p == dic).map(|(_, d)| d.clone())
}
/// Forget what `state` holds, so the next check looks again.
fn forget(state: &RwLock<State>) {
let mut s = state.write().unwrap_or_else(PoisonError::into_inner);
s.slots = [None, None, None];
s.files.clear();
s.reloads = s.reloads.wrapping_add(1);
}
/// The key load errors of `lang` are kept under ([`crate::hunspell::load_error`]).
fn key(lang: ProofLang) -> &'static str {
names(lang).first().copied().unwrap_or("en_US")
}
/// Read the first Hunspell pair of `lang`'s own that loads, trying each name in each folder
/// ([`crate::hunspell::candidates`]), with the first failure before it (the file `lang` would
/// rather use), for [`note_error`].
fn load(lang: ProofLang) -> (Loaded, Option<(PathBuf, String)>) {
let candidates = crate::hunspell::candidates(names(lang));
if candidates.is_empty() {
log::info!("English spelling: no Hunspell {} dictionary installed; using the built-in word list", key(lang));
return (None, None);
}
let mut first_error: Option<(PathBuf, String)> = None;
for (aff, dic) in candidates.into_iter().take(MAX_CANDIDATES) {
let loaded = match already_read(&HUNSPELL, &dic) {
Some(d) => Ok(d),
None => crate::hunspell::load_english(&aff, &dic)
.and_then(|d| if d.is_empty() { Err(format!("{}: no words", dic.display())) } else { Ok(d) })
.map(Arc::new),
};
match loaded {
Ok(d) => return (Some((d, dic)), first_error),
Err(e) => {
log::warn!("English spelling: {e}; trying the next dictionary");
first_error.get_or_insert((dic, e));
}
}
}
log::warn!("English spelling: no {} Hunspell dictionary could be read; using the built-in word list", key(lang));
(None, first_error)
}
/// Remember `lang`'s load error ([`crate::dictionary_load_error`]), or that it has none.
fn note_error(lang: ProofLang, error: Option<(PathBuf, String)>) {
match error {
Some((dic, e)) => {
let _ = crate::hunspell::note_load::<()>(key(lang), &dic, Err(e));
}
None => crate::hunspell::clear_load_error(key(lang)),
}
}
/// Forget the loaded dictionaries, so the next check looks for them again (the dictionaries
/// folder changed).
pub(crate) fn reload() {
forget(&HUNSPELL);
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
crate::hunspell::clear_load_error(key(lang));
}
}
/// Where the spelling of `lang` (an English) comes from now. Reads the Hunspell dictionary once
/// when there is no embedded data (it is cached afterwards).
pub(crate) fn source(lang: ProofLang) -> EnglishSource {
if crate::english_data() {
EnglishSource::Embedded
} else {
hunspell(lang).map_or(EnglishSource::CoreList, |(_, dic)| EnglishSource::Hunspell(dic))
}
}
/// Why a source of `lang`'s spelling failed to load: the embedded word list, or (without it) the
/// Hunspell dictionary found for it. Reads nothing itself.
pub(crate) fn load_error(lang: ProofLang) -> Option<String> {
if let Some(e) = EMBEDDED_ERROR.get() {
return Some(e.clone());
}
if crate::english_data() {
return None;
}
crate::hunspell::load_error(key(lang)).map(|(_, reason)| reason)
}
/// Is `word` spelled right in `lang` (an English) when no word list is embedded: against the
/// Hunspell dictionary, or else the built-in word list.
pub(crate) fn is_correct(word: &str, lang: ProofLang, opts: &Options) -> bool {
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’').trim_matches(['\'', '’']);
if crate::accepted_by_options(w, opts) || !w.chars().any(|c| c.is_ascii_alphabetic()) {
return true; // other scripts: not English
}
let w = w.replace('’', "'");
match hunspell(lang) {
Some((d, _)) => hunspell_correct(&d, &w),
None => list_correct(&w),
}
}
/// Against an installed English Hunspell dictionary: the word, or each part of a hyphenated one.
fn hunspell_correct(d: &AffixDict, w: &str) -> bool {
let known = |p: &str| p.chars().count() <= 1 || d.check(p) || crate::in_user_dictionary(&p.to_lowercase());
known(w) || (w.contains('-') && w.split('-').filter(|p| !p.is_empty()).all(known))
}
/// Against the built-in word list: its words and their inflections, or each part of a
/// hyphenated word. Anything else is misspelled. (Should the list fail to decode, nothing is
/// marked rather than everything.)
fn list_correct(w: &str) -> bool {
if words().is_empty() {
return true;
}
let lower = w.to_lowercase();
crate::known_in_word_list(&lower) || (lower.contains('-') && lower.split('-').filter(|p| !p.is_empty()).all(crate::known_in_word_list))
}
/// Spelling suggestions in `lang` (an English) when no word list is embedded: from the Hunspell
/// dictionary, or else the built-in word list.
pub(crate) fn suggest(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
match hunspell(lang) {
Some((d, _)) => crate::cs::suggest::suggest_in(&d, word, max),
None => list_suggest(word, max),
}
}
/// Letters an edit may add or put in a word's place.
const ALPHABET: &str = "abcdefghijklmnopqrstuvwxyz'";
/// Are `a` and `b` neighbouring keys on a QWERTY keyboard? (A typing slip: `wprd` for `word`.)
fn neighbour_keys(a: char, b: char) -> bool {
const ROWS: [&str; 3] = ["qwertyuiop", "asdfghjkl", "zxcvbnm"];
// Each row sits half a key right of the one above: x in half keys.
let at = |c: char| ROWS.iter().enumerate().find_map(|(r, row)| row.find(c).map(|col| (r as i64, 2 * col as i64 + r as i64)));
match (at(a), at(b)) {
(Some((ra, xa)), Some((rb, xb))) => (ra == rb && (xa - xb).abs() == 2) || ((ra - rb).abs() == 1 && (xa - xb).abs() <= 1),
_ => false,
}
}
/// A suggestion's rank, lowest first: not a swap, not a common word, not a neighbouring key, not
/// the same first letter, the difference in length.
type Rank = (bool, bool, bool, bool, usize);
/// How an edit changed the word, for ranking.
#[derive(Clone, Copy)]
enum Edit {
/// Two neighbouring letters swapped.
Swap,
/// One letter changed from the first to the second.
Change(char, char),
/// One letter added or missing.
Other,
}
/// Suggestions from the built-in word list: the words one edit away (two neighbouring letters
/// swapped, one letter missing, added or changed), looked up in the list rather than compared
/// with every word, so the cost is bounded by the word's length (at most
/// [`MAX_SUGGEST_CHARS`]). Ranked: swaps first (`recieve` → `receive`), then the common words
/// (`hte` → `the` before `het`), a letter changed to a neighbouring key's, the same first letter,
/// the nearest in length. One-letter words other than `a` and `I` are never suggested.
fn list_suggest(word: &str, max: usize) -> Vec<String> {
let lower = word.to_lowercase().replace('’', "'");
let target: Vec<char> = lower.chars().collect();
let n = target.len();
if max == 0 || n == 0 || n > MAX_SUGGEST_CHARS {
return Vec::new();
}
let mut found: Vec<(Rank, String)> = Vec::new();
let mut tried = std::collections::HashSet::new();
let mut consider = |c: Vec<char>, edit: Edit| {
let s: String = c.iter().collect();
if (c.len() < 2 && !matches!(s.as_str(), "a" | "i")) || s == lower || !tried.insert(s.clone()) || !crate::known_in_word_list(&s) {
return;
}
let key = (
!matches!(edit, Edit::Swap),
!crate::is_common_word(&s),
!matches!(edit, Edit::Change(from, to) if neighbour_keys(from, to)),
c.first() != target.first(),
c.len().abs_diff(n),
);
found.push((key, s));
};
for i in 1..n {
let mut c = target.clone();
c.swap(i - 1, i);
consider(c, Edit::Swap);
}
for i in 0..n {
let mut c = target.clone();
c.remove(i);
consider(c, Edit::Other);
}
for i in 0..=n {
for a in ALPHABET.chars() {
if let Some(&from) = target.get(i) {
let mut c = target.clone();
if let Some(l) = c.get_mut(i) {
*l = a;
}
consider(c, Edit::Change(from, a));
}
let mut c = target.clone();
c.insert(i, a);
consider(c, Edit::Other);
}
}
found.sort();
found.into_iter().take(max).map(|(_, w)| crate::match_case(word, w)).collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn built_in_list_is_the_whole_moby_list() {
let d = words();
assert!(d.len() > 150_000, "{}", d.len());
assert!(d.iter().all(|(w, m)| m == 0 && !w.is_empty() && w.chars().all(|c| !c.is_uppercase())));
assert!(WORDS.len() < 400 * 1024, "{} bytes", WORDS.len());
let extra: Vec<&str> = EXTRA.lines().filter(|l| !l.is_empty() && !l.starts_with('#')).collect();
assert!(extra.len() > 100 && extra.windows(2).all(|p| p[0] < p[1]), "the supplement is sorted");
}
/// The built-in list knows right words, the rare and the modern ones included, and marks
/// every word it doesn't know (craft-libs#482 review: the old core list let any unknown word
/// through unless it was a near miss of a common one, and marked right words that were).
#[test]
fn built_in_list_marks_unknown_words_only() {
for w in [
"the",
"The",
"houses",
"running",
"looked",
"happily",
"kindness",
"Received",
"well-known",
"photosynthesis",
"fiber",
"theater",
"organize",
"budget",
"phone",
"people's",
"wont",
"cant",
// WordCraft's supplement.
"login",
"blog",
"email",
"emails",
] {
assert!(list_correct(w), "{w} is not marked");
}
for w in ["teh", "Teh", "abd", "ur", "recieved", "definately", "seperate", "ocurred", "becuase", "peopel", "beautifull", "zxqv", "Jarndycey"]
{
assert!(!list_correct(w), "{w} is marked");
}
assert!(!list_correct("well-knwon"), "each part of a compound is checked");
}
#[test]
fn built_in_list_suggests_one_edit_away() {
assert_eq!(list_suggest("becuase", 3).first().map(String::as_str), Some("because"));
assert_eq!(list_suggest("Becuase", 3).first().map(String::as_str), Some("Because"));
assert_eq!(list_suggest("hte", 3).first().map(String::as_str), Some("the"));
assert_eq!(list_suggest("recieve", 3).first().map(String::as_str), Some("receive"));
assert_eq!(list_suggest("wprd", 3).first().map(String::as_str), Some("word"), "a neighbouring key");
assert!(list_suggest("é", 5).iter().all(|s| s.chars().count() > 1 || s == "a" || s == "i"), "{:?}", list_suggest("é", 5));
assert!(list_suggest("x", 5).iter().all(|s| s.chars().count() > 1 || s == "a" || s == "i"), "{:?}", list_suggest("x", 5));
assert!(!neighbour_keys('q', 's') && neighbour_keys('q', 'a') && neighbour_keys('o', 'p') && !neighbour_keys('é', 'e'));
assert!(list_suggest("definately", 5).contains(&"definitely".to_string()));
assert!(list_suggest("", 5).is_empty() && list_suggest("becuase", 0).is_empty());
assert!(list_suggest(&"a".repeat(MAX_SUGGEST_CHARS + 1), 5).is_empty());
}
/// Suggestions look words up rather than scanning the list: the longest word is still quick.
#[test]
fn built_in_list_suggestions_are_bounded() {
let _ = words();
let word: String = "qwertyuiopasdfghjklzxcvbnm".chars().cycle().take(MAX_SUGGEST_CHARS).collect();
let start = std::time::Instant::now();
for _ in 0..10 {
let _ = list_suggest(&word, 5);
}
assert!(start.elapsed() < std::time::Duration::from_secs(5), "{:?}", start.elapsed());
}
/// craft-libs#482 review: a load that started before a reload (the dictionaries folder
/// changed) serves its own check but isn't kept, so the next check reads the new folder; a
/// load after it is kept, and shared with the Englishes that read the same file.
#[test]
fn a_load_from_before_a_reload_is_not_kept() {
let state = RwLock::new(State::new());
let noted = std::cell::Cell::new(false);
let dict = || Arc::new(AffixDict::from_hunspell("", "1\nzzyzx\n", &[]).unwrap());
let file = PathBuf::from("/old/folder/en_US.dic");
let before = state.read().unwrap().reloads;
forget(&state); // The folder changes while the load runs.
let got = keep(&state, slot(ProofLang::En), before, Some((dict(), file.clone())), || noted.set(true));
assert_eq!(got.map(|(_, p)| p), Some(file.clone()), "the stale load still answers its own check");
assert!(state.read().unwrap().slots.iter().all(Option::is_none), "but it isn't kept");
assert!(!noted.get(), "nor is its load error");
assert!(already_read(&state, &file).is_none());
let now = state.read().unwrap().reloads;
let a = keep(&state, slot(ProofLang::En), now, Some((dict(), file.clone())), || noted.set(true)).unwrap();
assert!(noted.get());
let b = keep(&state, slot(ProofLang::EnGb), now, Some((dict(), file.clone())), || {}).unwrap();
assert!(Arc::ptr_eq(&a.0, &b.0), "one copy of a file for every English that reads it");
assert!(already_read(&state, &file).is_some());
forget(&state);
assert!(already_read(&state, &file).is_none() && state.read().unwrap().slots.iter().all(Option::is_none));
}
}
crates/proof/src/es/grammar.rs
+393
View File
@@ -0,0 +1,393 @@
//! Spanish grammar checks with Spanish messages: the generic repeated-word and spacing checks,
//! and the opening question and exclamation marks Spanish writes before a question or an
//! exclamation (`¿Vienes?`, `¡Qué bien!`): a closing `?` or `!` whose sentence has no opening mark
//! is flagged, with the opening mark offered at the sentence's start, and so is a space after an
//! opening mark (`¿ Vienes?`). Either opening mark may pair with either closing one (`¡Qué
//! dices?`), and `(?)` and `(!)`, which mark doubt or surprise, stand alone. No other
//! Spanish-specific grammar, comma or sentence-capital rules are inferred.
use unicode_normalization::UnicodeNormalization;
use crate::{Issue, IssueKind, ProofLang, words};
fn issue(start: usize, end: usize, message: String, suggestions: Vec<String>, lang: ProofLang) -> Issue {
Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang }
}
/// Grammar issues in a paragraph of Spanish text; `lang` is the variety (they explain the same in
/// every one).
pub fn check(text: &str, lang: ProofLang) -> Vec<Issue> {
let mut issues = Vec::new();
repeated_words(text, &words(text), lang, &mut issues);
spaces(text, lang, &mut issues);
opening_marks(text, lang, &mut issues);
issues.sort_by_key(|i| (i.start, i.end));
issues
}
fn repeated_words(text: &str, ws: &[(usize, usize)], lang: ProofLang, v: &mut Vec<Issue>) {
// Letters and the combining marks of a decomposed word (`tambie\u{301}n`); a composed and a
// decomposed spelling of one word are the same word.
let letters = |w: &str| w.chars().all(|c| c.is_alphabetic() || unicode_normalization::char::is_combining_mark(c));
let folded = |w: &str| w.nfc().collect::<String>().to_lowercase();
for pair in ws.windows(2) {
let [(a0, b0), (a1, b1)] = pair else { continue };
let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue };
if !gap.is_empty() && gap.chars().all(char::is_whitespace) && letters(w0) && folded(w0) == folded(w1) {
v.push(issue(*a0, *b1, format!("Palabra repetida: «{w1}»"), vec![w0.to_string()], lang));
}
}
}
fn spaces(text: &str, lang: ProofLang, v: &mut Vec<Issue>) {
for (i, c) in text.char_indices() {
// A space after an opening mark: `¿ Vienes?`.
if matches!(c, '¿' | '¡')
&& text.get(i + c.len_utf8()..).is_some_and(|r| r.starts_with(' ') && r.trim_start().chars().next().is_some_and(char::is_alphanumeric))
{
v.push(issue(i, i + c.len_utf8() + 1, "Espacio después de un signo de apertura".into(), vec![c.to_string()], lang));
continue;
}
if c != ' ' {
continue;
}
let rest = text.get(i + 1..).unwrap_or("");
let mut it = rest.chars();
let Some(p) = it.next() else { continue };
let after = it.next();
// A ratio/score keeps the spaces in digit-space-colon-space-digit.
let ratio = p == ':'
&& text.get(..i).and_then(|before| before.chars().next_back()).is_some_and(|c| c.is_ascii_digit())
&& after == Some(' ')
&& it.next().is_some_and(|c| c.is_ascii_digit());
let flag = !ratio
&& match p {
',' => true,
// `;)` and `;-)` are emoticons.
';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')),
// ` ...` is an ellipsis, ` .5` a number.
'.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
// ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space.
':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '»' | '!' | '?')),
_ => false,
};
if flag {
v.push(issue(i, i + 1 + p.len_utf8(), "Espacio antes de un signo de puntuación".into(), vec![p.to_string()], lang));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Espacio doble".into(), vec![" ".into()], lang));
}
}
/// Does the character `b` at byte `j` end what comes before it, so a question or exclamation after
/// it starts there? Sentence ends, clause breaks, line breaks and the start of a quotation or
/// parenthesis do, but not punctuation inside a word (`3.5`, `10:30`) or inside an address
/// (`example.com`, `https:`), except a mark ending an address (`…/? Vienes mañana?`).
fn ends_clause(text: &str, addresses: &[(usize, usize)], j: usize, b: char) -> bool {
let after = j + b.len_utf8();
if let Some((_, end)) = address_at(addresses, j) {
return matches!(b, '?' | '!' | '.') && end == after;
}
let inside_word = matches!(b, '.' | ':' | ';') && text.get(after..).and_then(|r| r.chars().next()).is_some_and(char::is_alphanumeric);
!inside_word && matches!(b, '.' | '?' | '!' | '…' | ':' | ';' | '\n' | '\r' | '\t' | '«' | '“' | '"' | '(' | '[' | '—')
}
/// The one of the `addresses` ([`address_ranges`]) that byte `j` is inside, if any. The ranges come
/// in text order and never overlap, so this is a binary search, O(log m) for m addresses: a paragraph
/// of thousands of addresses costs O(n log m) when every character before a question mark asks
/// (#3562 review), not the O(n m) of a scan per character.
fn address_at(addresses: &[(usize, usize)], j: usize) -> Option<(usize, usize)> {
let after_start = addresses.partition_point(|&(s, _)| s <= j);
let (s, e) = *addresses.get(after_start.checked_sub(1)?)?;
(s <= j && j < e).then_some((s, e))
}
/// The byte ranges of the whitespace-delimited tokens of `text` that are web or e-mail addresses
/// (`https://example.com/?`, `www.example.com`, `ana@example.com`), as spelling skips them.
fn address_ranges(text: &str) -> Vec<(usize, usize)> {
let mut out = Vec::new();
let mut start = None;
// A final space ends the last token.
for (i, c) in text.char_indices().chain(std::iter::once((text.len(), ' '))) {
if !c.is_whitespace() {
start = start.or(Some(i));
} else if let Some(s) = start.take()
&& text.get(s..i).is_some_and(crate::is_address)
{
out.push((s, i));
}
}
out
}
/// A closing `?` or `!` with no opening `¿` or `¡` before it. Openers are counted left to right, like
/// brackets, and each closing mark takes one back (either opener pairs with either closer:
/// `¡Qué dices?`), so what sits between them doesn't matter: quotes, parentheses, dashes, an
/// ellipsis or an abbreviation's full stop (`¿Viene el Sr. García?`, `¿Leíste «Rayuela»?`) all belong
/// to the question. Only a closing mark with nothing open is flagged, and then the sentence it
/// closes starts after the last clause break ([`ends_clause`]). Marks in web and e-mail addresses
/// aren't sentence punctuation, and neither is punctuation inside an address or a word
/// (`example.com`, `3.5`, `10:30`) a sentence's start.
fn opening_marks(text: &str, lang: ProofLang, v: &mut Vec<Issue>) {
let addresses = address_ranges(text);
// Openers not closed yet: a run of them (`¡¡`) counts once, like a run of closers (`!!`).
let mut open = 0usize;
for (i, c) in text.char_indices() {
let before = text.get(..i).unwrap_or("");
let prev = before.chars().next_back();
let (opener, message) = match c {
'¿' | '¡' => {
if !matches!(prev, Some('¿' | '¡')) {
open += 1;
}
continue;
}
'?' => ('¿', "Falta el signo de interrogación de apertura (¿)"),
'!' => ('¡', "Falta el signo de exclamación de apertura (¡)"),
_ => continue,
};
let next = text.get(i + 1..).and_then(|r| r.chars().next());
// `?!` and `!!!` are handled at their first mark; `(?)` stands alone; `?q=`, `a!b` and a
// mark ending an address (`https://example.com/?`) aren't sentence ends.
if matches!(prev, Some('?' | '!'))
|| (prev == Some('(') && next == Some(')'))
|| next.is_some_and(char::is_alphanumeric)
|| address_at(&addresses, i).is_some()
{
continue;
}
if open > 0 {
open -= 1;
continue;
}
let from = before.char_indices().rev().find(|&(j, b)| matches!(b, '¿' | '¡') || ends_clause(text, &addresses, j, b));
let start = from.map_or(0, |(j, b)| j + b.len_utf8());
let Some(sentence) = text.get(start..i) else { continue };
let lead = sentence.len() - sentence.trim_start().len();
let start = start + lead;
let Some(sentence) = text.get(start..i + 1) else { continue };
if !sentence.chars().any(char::is_alphabetic) {
continue;
}
v.push(issue(start, i + 1, message.into(), vec![format!("{opener}{sentence}")], lang));
}
}
#[cfg(test)]
mod tests {
use super::*;
fn messages(text: &str) -> Vec<(String, String)> {
check(text, ProofLang::Es).into_iter().map(|i| (text[i.start..i.end].to_string(), i.message)).collect()
}
/// The missing-opening-mark issues of `text`: the flagged text and the suggestions offered.
fn missing(text: &str) -> Vec<(String, Vec<String>)> {
check(text, ProofLang::Es)
.into_iter()
.filter(|i| i.message.starts_with("Falta"))
.map(|i| (text[i.start..i.end].to_string(), i.suggestions))
.collect()
}
#[test]
fn repeated_words_and_spaces_have_spanish_messages() {
let text = "Es es la casa , dijo.";
let issues = check(text, ProofLang::EsMx);
assert_eq!(issues.len(), 3, "{issues:?}");
assert_eq!(issues[0].message, "Palabra repetida: «es»");
assert_eq!(issues[0].suggestions, ["Es"]);
assert!(issues.iter().any(|i| i.message == "Espacio antes de un signo de puntuación" && &text[i.start..i.end] == " ,"));
assert!(issues.iter().any(|i| i.message == "Espacio doble"));
assert!(issues.iter().all(|i| i.lang == ProofLang::EsMx && i.kind == IssueKind::Grammar));
for text in ["La casa. Es grande", "Sr. García", "No, no lo sé", "Hola :) número .5 y puntos ...", "3 : 1", "teórico-práctico"] {
assert!(check(text, ProofLang::Es).is_empty(), "{text}: {:?}", check(text, ProofLang::Es));
}
}
#[test]
fn questions_and_exclamations_need_their_opening_mark() {
assert_eq!(messages("Vienes mañana?"), [("Vienes mañana?".into(), "Falta el signo de interrogación de apertura (¿)".into())]);
assert_eq!(check("Vienes mañana?", ProofLang::Es)[0].suggestions, ["¿Vienes mañana?"]);
assert_eq!(messages("Llegó. Qué bien!"), [("Qué bien!".into(), "Falta el signo de exclamación de apertura (¡)".into())]);
assert_eq!(messages("Le preguntó: vienes?")[0].0, "vienes?");
assert_eq!(messages("Dijo «vienes?»")[0].0, "vienes?");
assert_eq!(messages("Hola!!!").len(), 1);
assert_eq!(messages("Qué?!").len(), 1);
for text in [
"¿Vienes mañana?",
"¡Qué bien!",
"Si vienes, ¿me llamas?",
"¡Qué dices?",
"¿Qué?! ¡No!",
"Nació en 1950 (?) en Montevideo.",
"Lo dijo (!) en público.",
"Ver https://example.com/?q=1 y a!b",
"¿Qué? ¿Cómo?",
"2 + 2 = ?",
"",
"?",
] {
assert!(messages(text).iter().all(|(_, m)| !m.starts_with("Falta")), "{text}: {:?}", messages(text));
}
}
/// Maintainer review on #3562: a URL ending in `/?` isn't a question, and the dots and colons
/// of an address or a word (`example.com`, `https:`, `10:30`) don't start the sentence the
/// missing `¿` is offered for; ordinary questions are still checked.
#[test]
fn addresses_are_not_questions_or_sentence_starts() {
for text in [
"Ver https://example.com/?",
"Ver https://example.com/? ¿Vienes?",
"Ver http://example.com/buscar/? y www.example.com/?",
"Escribí a ana@example.com!",
"¿Viste https://example.com/?",
] {
assert!(missing(text).is_empty(), "{text}: {:?}", missing(text));
}
assert_eq!(
missing("Mirá https://example.com/ y decime si vienes?"),
[("Mirá https://example.com/ y decime si vienes?".to_string(), vec!["¿Mirá https://example.com/ y decime si vienes?".to_string()])]
);
assert_eq!(missing("Mirá example.com y decime si vienes?")[0].0, "Mirá example.com y decime si vienes?");
assert_eq!(missing("Son las 10:30, vienes?")[0].0, "Son las 10:30, vienes?");
assert_eq!(missing("Ver https://example.com/? Vienes mañana?"), [("Vienes mañana?".to_string(), vec!["¿Vienes mañana?".to_string()])]);
assert_eq!(missing("Llegó. Vienes?")[0].0, "Vienes?", "a full stop before a space still ends a sentence");
}
/// Maintainer review on #3562: a correctly opened question or exclamation is never flagged,
/// whatever sits between its marks: quotes, an abbreviation's full stop, an ellipsis, a dash
/// aside, parentheses, nested questions, a line break.
#[test]
fn correctly_opened_questions_are_not_flagged() {
for text in [
"¿Leíste «Rayuela»?",
"¿Viene el Sr. García?",
"¿Y si... no viene?",
"¿Vendrás —preguntó Juan— mañana?",
"¿Qué significa (aquí) la palabra?",
"¿Has leído \"Rayuela\" de Cortázar?",
"¿Leíste “Rayuela”?",
"¿Es 'Rayuela' o 'Los premios'?",
"¿Y tú, qué opinas… vienes?",
"¿Dijo «¿qué?» o no?",
"¿Viene la Dra. López, es decir, la jefa (la de 2.5 años)?",
"¿Dónde: aquí o allí?",
"¿Cuánto cuesta: 3,50 o a las 10:30?",
"¿Vienes a las 10:30; o mejor mañana?",
"¡Qué dices, Dr. Pérez!",
"¡Qué dices, Dr. Pérez? ¡Basta!",
"¡¡Hola!! ¿Qué tal?",
"¡¡¡Basta!!!",
"¿Vienes?!",
"¿Qué?! ¡No!",
"¿Vienes?\n¿O no?",
"¿Vienes\nmañana?",
"Ana dijo: «¿Vienes?» y yo: «Sí».",
"—¿Vienes? —preguntó Juan.",
"¿Viste https://example.com/? Dime.",
"¿Es verdad (!) o no?",
"Nació en 1950 (?) en Montevideo.",
] {
assert!(missing(text).is_empty(), "{text:?}: {:?}", missing(text));
}
}
/// A closing mark with nothing open is still flagged, after a sentence or a clause that closed
/// an earlier question, and with the sentence it closes offered with its opening mark.
#[test]
fn closing_marks_with_nothing_open_are_still_flagged() {
for (text, flagged, offered) in [
("Vienes mañana?", "Vienes mañana?", "¿Vienes mañana?"),
("Llegó. Vienes?", "Vienes?", "¿Vienes?"),
("Le preguntó: vienes?", "vienes?", "¿vienes?"),
("Dijo «vienes?»", "vienes?", "¿vienes?"),
("Qué bien!", "Qué bien!", "¡Qué bien!"),
("¿Vienes? No sé qué hacer?", "No sé qué hacer?", "¿No sé qué hacer?"),
("¿Viene el Sr. García? Llegó tarde!", "Llegó tarde!", "¡Llegó tarde!"),
("¡¡Hola!! Qué bien!", "Qué bien!", "¡Qué bien!"),
("¿Qué?! No sé qué decir!", "No sé qué decir!", "¡No sé qué decir!"),
] {
let found = missing(text);
assert_eq!(found.len(), 1, "{text:?}: {found:?}");
assert_eq!((found[0].0.as_str(), found[0].1.first().map(String::as_str)), (flagged, Some(offered)), "{text:?}");
}
}
#[test]
fn address_lookup_finds_the_range_a_byte_is_in() {
let ranges = [(2, 5), (7, 8), (10, 20)];
let inside = |j| address_at(&ranges, j);
assert_eq!([0, 1].map(inside), [None, None], "before the first");
assert_eq!([2, 3, 4].map(inside), [Some((2, 5)); 3], "the start is inside, the end is not");
assert_eq!([5, 6].map(inside), [None, None], "between ranges");
assert_eq!([7, 10, 19].map(inside), [Some((7, 8)), Some((10, 20)), Some((10, 20))]);
assert_eq!([8, 9, 20, 99].map(inside), [None; 4], "after a range, and after the last");
assert_eq!(address_at(&[], 3), None);
}
/// Maintainer review on #3562: a paragraph of thousands of addresses followed by a question was
/// checked in quadratic time (every character before the question scanned every address).
/// About 99 KB (under the 100,000-byte paragraph cap) took hundreds of milliseconds; it is now
/// O(n log m): about 1 to 5 ms in a debug build, against about 300 ms for the scan at
/// `opt-level = 1`. craft-libs builds tests at `opt-level = 0`, where the fixed lookup took up
/// to ~80 ms under a parallel test run and more on a busy machine. The test guards against a
/// return to the quadratic blowup, not machine load, so the bound is 1 s.
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn many_addresses_before_a_question_are_checked_quickly() {
for (unit, ending, whole_paragraph) in
[("https://x/ ", "Vienes?", true), ("https://x/? ", "Vienes?", false), ("www.x.com ", "Qué bien!", true)]
{
// As many as fit in about 99 KB: 9,000 of the first, as in the review.
let text = format!("{}{ending}", unit.repeat(99_000 / unit.len()));
assert!(text.len() < 100_000, "{}", text.len());
let started = std::time::Instant::now();
let issues = check(&text, ProofLang::Es);
let took = started.elapsed();
assert!(took < std::time::Duration::from_secs(1), "{unit:?} took {took:?}");
let missing: Vec<&Issue> = issues.iter().filter(|i| i.message.starts_with("Falta")).collect();
assert_eq!(missing.len(), 1, "{unit:?}: {missing:?}");
// The mark's sentence is the whole paragraph when the addresses don't end with a mark
// of their own, and only the last question when they do.
let want_start = if whole_paragraph { 0 } else { text.len() - ending.len() };
assert_eq!((missing[0].start, missing[0].end), (want_start, text.len()), "{unit:?}");
}
}
/// A word repeated once composed and once decomposed is still repeated.
#[test]
fn repeated_words_match_across_normalization_forms() {
let text = "también tambie\u{301}n";
let issues = check(text, ProofLang::Es);
assert_eq!(issues.len(), 1, "{issues:?}");
assert_eq!((issues[0].start, issues[0].end), (0, text.len()));
assert!(issues[0].message.starts_with("Palabra repetida"));
}
#[test]
fn a_space_after_an_opening_mark_is_flagged() {
let text = "¿ Vienes? ¡ Hola!";
let issues = check(text, ProofLang::EsUy);
let spaced: Vec<&str> =
issues.iter().filter(|i| i.message == "Espacio después de un signo de apertura").map(|i| &text[i.start..i.end]).collect();
assert_eq!(spaced, ["¿ ", "¡ "], "{issues:?}");
assert!(issues.iter().all(|i| !i.message.starts_with("Falta")), "{issues:?}");
assert!(check("¿ ", ProofLang::Es).is_empty());
}
proptest::proptest! {
#[test]
fn never_panics_and_ranges_are_valid(s in "\\PC{0,80}|[¿¡?!a-z .,(«»):]{0,40}") {
for issue in check(&s, ProofLang::Es) {
proptest::prop_assert!(issue.start <= issue.end && issue.end <= s.len());
proptest::prop_assert!(s.is_char_boundary(issue.start) && s.is_char_boundary(issue.end));
}
}
}
}
crates/proof/src/es/mod.rs
+315
View File
@@ -0,0 +1,315 @@
//! Spanish proofing (#3360, #3553) using the installed Hunspell dictionaries (never bundled;
//! LibreOffice ships them as its `dict-es` extension, Debian and Ubuntu as `hunspell-es`). Each
//! variety reads its own dictionary: Spain ([`ProofLang::Es`]) `es_ES`, Mexico
//! ([`ProofLang::EsMx`]) `es_MX`, Latin America ([`ProofLang::Es419`]) `es_419` and Uruguay
//! ([`ProofLang::EsUy`]) `es_UY`, each falling back to a nearby or generic one. Uruguayan Spanish
//! also accepts the `vos` forms of every verb in the dictionary ([`voseo`]: `tenés`, `vení`,
//! `decime`). Suggestions use the dictionary's own MAP/REP/TRY/KEY tables (the regional
//! dictionaries' REP tables carry their differences, such as `vosotros` forms in Mexico).
//! Grammar is limited to generic repeated-word and spacing checks and the Spanish opening
//! question and exclamation marks, with Spanish messages; hyphenation keeps the English fallback
//! (no Spanish patterns are bundled).
pub mod grammar;
mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
pub mod voseo;
use std::sync::Arc;
use crate::ProofLang;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The dictionary of each variety, found on first use (see [`DictSlot`]).
pub(crate) static SPAIN_DICT: DictSlot<AffixDict> = DictSlot::new(SPAIN, "Spanish (Spain)");
pub(crate) static MEXICO_DICT: DictSlot<AffixDict> = DictSlot::new(MEXICO, "Spanish (Mexico)");
pub(crate) static LATIN_AMERICA_DICT: DictSlot<AffixDict> = DictSlot::new(LATIN_AMERICA, "Spanish (Latin America)");
pub(crate) static URUGUAY_DICT: DictSlot<AffixDict> = DictSlot::new(URUGUAY, "Spanish (Uruguay)");
/// File names of a Spanish Hunspell dictionary for Spain, most specific first (`es_ANY` is the
/// generic one RLA-ES publishes).
const SPAIN: &[&str] = &["es_ES", "es-ES", "es_ANY", "es"];
/// For Mexico: its own, then a Latin American or generic one.
const MEXICO: &[&str] = &["es_MX", "es-MX", "es_419", "es-419", "es_ANY", "es"];
/// For Latin America: its own or a generic one, then the United States' and Mexico's, which
/// LibreOffice ships (it has no `es_419`) and which use Latin American spelling, then
/// Argentina's, so `es-AR` text is checked when that is the only one installed.
const LATIN_AMERICA: &[&str] = &["es_419", "es-419", "es_ANY", "es", "es_US", "es_MX", "es_AR", "es-AR"];
/// For Uruguay: its own, then Argentina's (the same Rioplatense Spanish), then a Latin American
/// or generic one.
const URUGUAY: &[&str] = &["es_UY", "es-UY", "es_AR", "es-AR", "es_419", "es-419", "es_ANY", "es"];
/// `lang`'s variety's dictionary; Spain's for any other language.
fn slot(lang: ProofLang) -> &'static DictSlot<AffixDict> {
match lang {
ProofLang::EsMx => &MEXICO_DICT,
ProofLang::Es419 => &LATIN_AMERICA_DICT,
ProofLang::EsUy => &URUGUAY_DICT,
_ => &SPAIN_DICT,
}
}
/// The Hunspell name each variety's dictionary is known by (`es_ES`, `es_MX`, `es_419`,
/// `es_UY`): the first of its file names, used for the status-bar hint and load errors.
pub fn dictionary_name(lang: ProofLang) -> &'static str {
slot(lang).name()
}
/// The Spanish dictionary of `lang`'s variety: the installed one (read on first use), one given
/// with [`install_dictionary`], or an empty one when none is available (spelling is off then;
/// see [`has_dictionary`]).
pub fn dictionary(lang: ProofLang) -> Arc<AffixDict> {
slot(lang).get()
}
/// Use this dictionary for `lang`'s variety from now on (Hunspell `.aff` and `.dic` text,
/// decoded to UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(lang: ProofLang, aff: &str, dic: &str) -> Result<(), String> {
slot(lang).install(aff, dic)
}
/// Is a dictionary available for `lang`'s variety (spelling is checked)?
pub fn has_dictionary(lang: ProofLang) -> bool {
slot(lang).has_dictionary()
}
/// Is `word` spelled correctly in `lang`'s Spanish? Numbers, single letters, short words in
/// capitals (acronyms: `ONU`, `CDMX`), words with digits and words in other scripts are accepted,
/// and so is every word when no dictionary is installed for the variety. Hyphenated compounds
/// are checked part by part (`teórico-práctico`, `franco-mexicano`). In Uruguayan Spanish the
/// `vos` forms of the dictionary's verbs are correct too ([`voseo`]).
pub fn is_correct(word: &str, lang: ProofLang) -> bool {
is_correct_with(&dictionary(lang), word, lang)
}
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off,
/// short words in capitals are checked against the dictionary too, and with Ignore words that
/// contain numbers off, so are words with digits.
pub fn is_correct_with_options(word: &str, lang: ProofLang, opts: &crate::Options) -> bool {
check(&dictionary(lang), word, lang, opts)
}
/// [`is_correct`] against a given dictionary (an empty one accepts everything).
pub fn is_correct_with(d: &AffixDict, word: &str, lang: ProofLang) -> bool {
check(d, word, lang, &crate::Options::default())
}
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, lang: ProofLang, opts: &crate::Options) -> bool {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
};
if w.contains('-') {
return known(d, w, lang) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p, lang));
}
known(d, w, lang)
}
/// Is one word in the dictionary or the user dictionary, or (in Uruguayan Spanish) the `vos`
/// form of one of the dictionary's verbs?
fn known(d: &AffixDict, w: &str, lang: ProofLang) -> bool {
d.check(w) || crate::in_user_dictionary(&w.to_lowercase()) || (lang == ProofLang::EsUy && voseo::is_vos_form(w, |inf| d.check(inf)))
}
#[cfg(test)]
mod tests {
use super::*;
const VARIETIES: [ProofLang; 4] = [ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy];
#[test]
fn each_variety_looks_for_its_own_dictionary_first() {
let dir = std::env::temp_dir().join(format!("wordcraft-es-aliases-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let (es, mx, la, uy) = (ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy);
// Alone in the folder, each file name is found by the varieties that list it.
for (name, found_by) in [
("es_ES", &[es][..]),
("es-ES", &[es]),
("es_MX", &[mx, la]),
("es-MX", &[mx]),
("es_419", &[mx, la, uy]),
("es-419", &[mx, la, uy]),
("es_US", &[la]),
("es_UY", &[uy]),
("es-UY", &[uy]),
("es_AR", &[la, uy]),
("es-AR", &[la, uy]),
("es_ANY", &VARIETIES),
("es", &VARIETIES),
] {
std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap();
std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap();
for lang in VARIETIES {
let found = crate::hunspell::find_in(&dir, slot(lang).names());
assert_eq!(found.is_some(), found_by.contains(&lang), "{name}: {lang:?}");
if let Some((aff, dic)) = found {
assert!(is_correct_with(&crate::hunspell::load(&aff, &dic).unwrap(), "canción", lang));
}
}
std::fs::remove_file(dir.join(format!("{name}.aff"))).unwrap();
std::fs::remove_file(dir.join(format!("{name}.dic"))).unwrap();
}
// With them all, each variety reads its own.
for name in ["es_ES", "es_MX", "es_419", "es_UY", "es_AR", "es_ANY"] {
std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap();
std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap();
}
for (lang, name) in [(es, "es_ES"), (mx, "es_MX"), (la, "es_419"), (uy, "es_UY")] {
assert_eq!(crate::hunspell::find_in(&dir, slot(lang).names()).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic"))));
assert_eq!(dictionary_name(lang), name);
}
std::fs::remove_dir_all(dir).unwrap();
}
#[test]
fn spanish_words_affixes_case_and_hyphens() {
test_dict::install();
let d = dictionary(ProofLang::Es);
assert!(has_dictionary(ProofLang::Es));
assert!(d.rule_count() > 0);
for w in [
"canción",
"canciones",
"prueba",
"pruebas",
"corrector",
"correctores",
"también",
"México",
"Canción",
"CANCIÓN",
"teórico-práctico",
"franco-mexicano",
"ONU",
"Año",
"niño",
"pingüino",
] {
assert!(is_correct(w, ProofLang::Es), "{w}");
}
for w in ["prueva", "corector", "tambien", "mexico", "cancion", "teórico-prático", "cAnción", "nino"] {
assert!(!is_correct(w, ProofLang::Es), "{w}");
}
}
#[test]
fn varieties_keep_their_own_dictionaries() {
test_dict::install();
// The Americas' fixture lacks `vosotros` forms and has American words; Spain's is the other way round.
assert!(is_correct("habéis", ProofLang::Es) && !is_correct("habéis", ProofLang::EsMx));
assert!(is_correct("platicar", ProofLang::EsMx) && !is_correct("platicar", ProofLang::Es));
assert!(is_correct("platicar", ProofLang::Es419) && is_correct("computadora", ProofLang::Es419));
assert!(!is_correct("habéis", ProofLang::Es419) && !is_correct("habéis", ProofLang::EsUy));
assert!(is_correct("ordenador", ProofLang::Es) && !is_correct("ordenador", ProofLang::EsMx));
}
/// Uruguayan Spanish accepts the `vos` forms of the dictionary's verbs, which the other
/// varieties flag.
#[test]
fn uruguayan_spanish_accepts_vos_forms() {
test_dict::install();
for w in ["tenés", "vení", "Vení", "decime", "decímelo", "andate", "mirá", "hablás", "querés", "venís", "fijate"] {
assert!(is_correct(w, ProofLang::EsUy), "{w}");
assert!(!is_correct(w, ProofLang::Es419) && !is_correct(w, ProofLang::Es), "{w}");
}
for w in ["tenéz", "venì", "decimelo", "decíme", "zqbailá"] {
assert!(!is_correct(w, ProofLang::EsUy), "{w}");
}
}
/// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Spanish checks
/// those words against the dictionary instead of accepting them (CodeRabbit review on #3562).
#[test]
fn proofing_options_reach_the_spanish_check() {
test_dict::install();
let defaults = crate::Options::default();
let check_caps = crate::Options { ignore_uppercase: false, ..defaults };
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
for lang in VARIETIES {
assert!(crate::is_correct_in_with("PRUEVA", lang, &defaults), "{lang:?}: short capitals are acronyms by default");
assert!(!crate::is_correct_in_with("PRUEVA", lang, &check_caps), "{lang:?}");
assert!(crate::is_correct_in_with("PRUEBA", lang, &check_caps), "{lang:?}");
assert!(crate::is_correct_in_with("prueva2", lang, &defaults), "{lang:?}");
assert!(!crate::is_correct_in_with("prueva2", lang, &check_digits), "{lang:?}");
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "{lang:?}: numbers alone stay correct");
}
assert!(is_correct("PRUEVA", ProofLang::Es), "direct checks keep the defaults");
}
/// Maintainer review on #3562: a word typed decomposed (NFD: `e` + U+0301) is checked like its
/// composed form (NFC) and keeps its own byte range, accents and all.
#[test]
fn decomposed_accents_are_checked_like_composed_ones() {
test_dict::install();
for lang in VARIETIES {
for text in ["esto también", "esto tambie\u{301}n", "una cancio\u{301}n y pingu\u{308}ino", "CANCIO\u{301}N", "nin\u{303}o"] {
let issues = crate::check_text(text, &[(0..text.len(), lang)]);
assert!(issues.is_empty(), "{lang:?} {text:?}: {issues:?}");
}
// A misspelling that ends in a decomposed accent is flagged whole, at its own bytes.
let text = "esto es una prueva tambie\u{301}nn";
let issues = crate::check_text(text, &[(0..text.len(), lang)]);
let flagged: Vec<(&str, usize, usize)> = issues.iter().map(|i| (&text[i.start..i.end], i.start, i.end)).collect();
assert_eq!(flagged, [("prueva", 12, 18), ("tambie\u{301}nn", 19, text.len())], "{lang:?}");
assert!(crate::suggest_in("tambie\u{301}nn", 8, lang).contains(&"también".into()), "{lang:?}");
}
}
#[test]
fn forbidden_words_are_rejected_in_every_case() {
test_dict::install();
for w in ["zqprohibida", "Zqprohibida", "ZQPROHIBIDA"] {
assert!(!is_correct(w, ProofLang::Es), "{w}");
}
}
#[test]
fn missing_dictionary_and_ignored_inputs_are_quiet() {
let empty = AffixDict::default();
assert!(is_correct_with(&empty, "prueva", ProofLang::Es));
assert!(install_dictionary(ProofLang::Es, "FLAG long\n", "1\ncasa\n").is_err());
let d = AffixDict::from_hunspell(test_dict::AFF, test_dict::DIC, &[]).unwrap();
for w in ["", "a", "2026", "EE", "1.º", "日本語", "-", "--"] {
assert!(is_correct_with(&d, w, ProofLang::EsUy), "{w}");
}
}
#[test]
fn user_dictionary_applies() {
test_dict::install();
assert!(!is_correct("zqpalabrapropia", ProofLang::EsMx));
crate::add_word("Zqpalabrapropia");
assert!(is_correct("zqpalabrapropia", ProofLang::EsMx));
}
#[test]
fn spanish_dispatch_messages_options_and_mixed_spans() {
test_dict::install();
for tag in ["es-ES", "es-MX", "es-419", "es-AR", "es-UY", "spa"] {
assert!(crate::has_dictionary(Some(tag)), "{tag}");
}
assert!(crate::is_correct_for("canción", Some("es-MX")));
assert!(crate::is_correct_for("tenés", Some("es-UY")) && !crate::is_correct_for("tenés", Some("es-MX")));
// The sentence from #3553.
let text = "esto es una prueva del corector";
for lang in VARIETIES {
let issues = crate::check_text(text, &[(0..text.len(), lang)]);
let flagged: Vec<&str> = issues.iter().map(|i| &text[i.start..i.end]).collect();
assert_eq!(flagged, ["prueva", "corector"], "{lang:?}: {issues:?}");
assert!(issues.iter().all(|i| i.message == "Posible error ortográfico" && i.lang == lang), "{issues:?}");
assert!(crate::suggest_in("prueva", 8, lang).contains(&"prueba".into()), "{lang:?}");
assert!(crate::suggest_in("corector", 8, lang).contains(&"corrector".into()), "{lang:?}");
}
let text = "Canción this is is wrong";
let issues = crate::check_text(text, &[(0..8, ProofLang::Es419), (9..text.len(), ProofLang::En)]);
assert!(issues.iter().any(|i| i.lang == ProofLang::En && i.message.starts_with("Repeated word")));
assert!(issues.iter().all(|i| i.start >= 9));
let opts = crate::Options { ignore_uppercase: false, ..Default::default() };
assert!(!crate::is_correct_in_with("ZQPROHIBIDA", ProofLang::Es, &opts));
}
}
crates/proof/src/es/suggest.rs
+97
View File
@@ -0,0 +1,97 @@
//! Spanish suggestions. The ranking is the shared dictionary-driven one
//! ([`crate::affix_suggest`]: case, MAP — the Spanish dictionaries map each vowel to
//! its accented forms and `n` to `ñ`, so `tambien` → `también` comes first —, REP, missing spaces,
//! KEY, TRY edits and a stem scan); every table comes from the variety's installed dictionary.
//! In Uruguayan Spanish a `vos` form typed without its accent is offered with it first
//! (`veni` → `vení`, `tenes` → `tenés`; [`super::voseo`]).
use crate::ProofLang;
use crate::affix::{AffixDict, Casing, capitalize, casing};
use crate::affix_suggest::{MAX_LETTERS, Tuning};
/// Spelling suggestions for a word in `lang`'s Spanish, best first, at most `max`, in the word's
/// case.
pub fn suggest(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
suggest_in(&super::dictionary(lang), word, max, lang)
}
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize, lang: ProofLang) -> Vec<String> {
let mut out = Vec::new();
let word = word.trim();
if lang == ProofLang::EsUy
&& !d.is_empty()
&& word.chars().count() <= MAX_LETTERS
&& let Some(v) = super::voseo::accented(&word.to_lowercase(), |inf| d.check(inf))
{
out.push(match casing(word) {
Casing::Upper => v.to_uppercase(),
Casing::Initial => capitalize(&v),
_ => v,
});
}
for s in crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT) {
if !out.contains(&s) {
out.push(s);
}
}
out.truncate(max);
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn spanish_suggestions_use_runtime_tables_and_case() {
crate::es::test_dict::install();
for lang in [ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy] {
for (word, want) in [
("tambien", "también"),
("cancion", "canción"),
("Cancion", "Canción"),
("mexico", "México"),
("nino", "niño"),
("pinguino", "pingüino"),
("prueva", "prueba"),
("corector", "corrector"),
("estoes", "esto es"),
] {
assert!(suggest(word, 8, lang).contains(&want.into()), "{lang:?} {word}: {:?}", suggest(word, 8, lang));
}
}
// The first suggestion for a missing accent is the accented word.
assert_eq!(suggest("tambien", 4, ProofLang::Es).first().map(String::as_str), Some("también"));
}
#[test]
fn uruguayan_vos_forms_get_their_accent_back_first() {
crate::es::test_dict::install();
for (word, want) in [("veni", "vení"), ("tenes", "tenés"), ("Veni", "Vení"), ("TENES", "TENÉS"), ("queres", "querés")] {
assert_eq!(suggest(word, 4, ProofLang::EsUy).first().map(String::as_str), Some(want), "{word}");
assert!(!suggest(word, 8, ProofLang::Es419).contains(&want.into()), "{word}: no vos forms outside Uruguay");
}
}
#[test]
fn suggestions_stay_bounded_and_skip_forbidden_words() {
crate::es::test_dict::install();
for lang in [ProofLang::Es, ProofLang::EsUy] {
for w in ["", "dos palabras", &"a".repeat(41)] {
assert!(suggest(w, 4, lang).is_empty(), "{w}");
}
assert!(suggest("prueva", 0, lang).is_empty());
assert!(suggest("veni", 0, lang).is_empty());
assert!(suggest_in(&AffixDict::default(), "veni", 4, lang).is_empty());
assert!(suggest("prueba", 4, lang).len() <= 4);
assert!(!suggest("prueba", 4, lang).contains(&"prueba".into()));
assert!(!suggest("zqprohibda", 8, lang).contains(&"zqprohibida".into()));
}
}
#[test]
fn nosuggest_stems_and_forms_are_never_offered() {
crate::affix::tests::assert_nosuggest_is_honoured(|d, w, max| suggest_in(d, w, max, ProofLang::Es));
}
}
crates/proof/src/es/test_dict.rs
+111
View File
@@ -0,0 +1,111 @@
//! Hand-written synthetic Spanish test data. No words, rules or suggestion tables were copied
//! from a system dictionary. This is a small fixture, not a redistributable Spanish dictionary.
//! Spain's word list has `vosotros` forms and Spain's words; the Americas' list (Mexico, Latin
//! America and Uruguay) has theirs instead, so the tests can tell the varieties' dictionaries apart.
use crate::ProofLang;
pub const AFF: &str = "SET UTF-8
FLAG UTF-8
FORBIDDENWORD F
TRY aeosrnidlctumpbgvyqhfzjñáéíóúüxkw
KEY qwertyuiop|asdfghjklñ|zxcvbnm
MAP 6
MAP aá
MAP eé
MAP ií
MAP oó
MAP uúü
MAP nñ
REP 2
REP v b
REP b v
SFX S Y 2 # synthetic plurals
SFX S 0 s [aeiou]
SFX S ón ones ón
SFX E Y 1
SFX E 0 es [^aeiou]
";
/// Spain.
pub const DIC: &str = "32
hablar
tener
venir
decir
andar
mirar
querer
fijar
poner
canción/S
prueba/S
corrector/E
también
México
esto
es
una
del
de
la
el
y
niño/S
pingüino/S
año/S
teórico
práctico
mexicano
franco
habéis
ordenador/E
zqprohibida/F
";
/// Mexico, Latin America and Uruguay.
pub const DIC_AMERICAS: &str = "33
hablar
tener
venir
decir
andar
mirar
querer
fijar
poner
canción/S
prueba/S
corrector/E
también
México
esto
es
una
del
de
la
el
y
niño/S
pingüino/S
año/S
teórico
práctico
mexicano
franco
platicar
computadora/S
han
zqprohibida/F
";
/// Install the fixture for every variety: [`DIC`] for Spain, [`DIC_AMERICAS`] for Mexico, Latin
/// America and Uruguay.
pub fn install() {
for (lang, dic) in [(ProofLang::Es, DIC), (ProofLang::EsMx, DIC_AMERICAS), (ProofLang::Es419, DIC_AMERICAS), (ProofLang::EsUy, DIC_AMERICAS)] {
if let Err(e) = super::install_dictionary(lang, AFF, dic) {
panic!("test dictionary: {e}");
}
}
}
crates/proof/src/es/voseo.rs
+155
View File
@@ -0,0 +1,155 @@
//! The `vos` forms of verbs in Uruguayan Spanish ([`crate::ProofLang::EsUy`]), which dictionaries
//! built around `tú` may lack (#3360). Rioplatense `voseo` builds them regularly from the
//! infinitive, stress on the ending, without the stem changes `tú` has:
//!
//! - the imperative drops the infinitive's `r` and accents the vowel: `hablar` → `hablá`,
//! `tener` → `tené`, `venir` → `vení`, `decir` → `decí` (`ir` uses `andá`, from `andar`);
//! - with one pronoun after it the accent goes (`decime`, `fijate`, `andate`, `poneme`), with two
//! it comes back (`decímelo`, `mandámela`); `-ír` verbs keep theirs (`sonreíte`);
//! - the present indicative ends in `-ás`, `-és`, `-ís`: `hablás`, `tenés`, `venís`.
//!
//! A form is accepted only when the dictionary has its verb's infinitive, so nothing but these
//! endings is hard-coded. Other `vos` forms (`sos`) are ordinary dictionary words.
/// Pronouns that come first when two follow an imperative (`decímelo`, `dáselo`).
const FIRST_PRONOUNS: [&str; 4] = ["me", "te", "se", "nos"];
/// Pronouns that come second (or alone).
const SECOND_PRONOUNS: [&str; 6] = ["lo", "la", "le", "los", "las", "les"];
/// Shortest infinitive considered (`dar` and `ir` have no `vos` imperative of their own here, and
/// short stems would let nouns such as `bar` pass for verbs).
const MIN_INFINITIVE: usize = 4;
/// Is `word` (one word, any case) a `vos` form of a verb whose infinitive `is_word` accepts?
pub fn is_vos_form(word: &str, is_word: impl Fn(&str) -> bool) -> bool {
let w = word.to_lowercase();
if w.chars().count() > crate::affix::MAX_WORD_BYTES {
return false;
}
infinitives(&w).iter().any(|inf| inf.chars().count() >= MIN_INFINITIVE && is_word(inf))
}
/// The infinitives `w` (lower case) could be a `vos` form of, by its ending.
fn infinitives(w: &str) -> Vec<String> {
let mut out = Vec::new();
// Present indicative: `hablás`, `tenés`, `venís`.
if let Some(stem) = w.strip_suffix('s') {
imperative(stem, &mut out);
}
// Imperative: `hablá`, `tené`, `vení`.
imperative(w, &mut out);
// With one pronoun the accent goes: `decime`, `fijate`; `-ír` verbs keep it: `sonreíte`.
for p in FIRST_PRONOUNS.iter().chain(SECOND_PRONOUNS.iter()) {
if let Some(base) = w.strip_suffix(p)
&& base.ends_with(['a', 'e', 'i', 'í'])
{
out.push(format!("{base}r"));
}
}
// With two the accent comes back: `decímelo`, `mandámela`.
for second in SECOND_PRONOUNS {
let Some(rest) = w.strip_suffix(second) else { continue };
for first in FIRST_PRONOUNS {
if let Some(base) = rest.strip_suffix(first) {
imperative(base, &mut out);
}
}
}
out
}
/// The infinitives of an imperative ending in an accented vowel: `hablá` → `hablar`, and for `í`
/// both `vení` → `venir` and `sonreí` → `sonreír`.
fn imperative(form: &str, out: &mut Vec<String>) {
let mut chars = form.chars();
let Some(last) = chars.next_back() else { return };
let stem = chars.as_str();
match last {
'á' => out.push(format!("{stem}ar")),
'é' => out.push(format!("{stem}er")),
'í' => {
out.push(format!("{stem}ir"));
out.push(format!("{stem}ír"));
}
_ => {}
}
}
/// The `vos` form `word` (lower case, without its accent) was most likely meant as, when the
/// writer left the accent out: `veni` → `vení`, `tenes` → `tenés`. `None` when it isn't one.
pub fn accented(word: &str, is_word: impl Fn(&str) -> bool) -> Option<String> {
let (body, s) = match word.strip_suffix('s') {
Some(b) => (b, "s"),
None => (word, ""),
};
let mut chars = body.chars();
let accent = match chars.next_back()? {
'a' => 'á',
'e' => 'é',
'i' => 'í',
_ => return None,
};
let form = format!("{}{accent}{s}", chars.as_str());
is_vos_form(&form, is_word).then_some(form)
}
#[cfg(test)]
mod tests {
use super::*;
fn verbs(w: &str) -> bool {
["hablar", "tener", "venir", "decir", "andar", "mirar", "sonreír", "fijar", "mandar", "poner", "bar", "querer"].contains(&w)
}
#[test]
fn imperatives_present_and_pronouns_come_from_the_infinitive() {
for w in [
"hablá",
"tené",
"vení",
"decí",
"andá",
"mirá",
"poné",
"hablás",
"tenés",
"venís",
"querés",
"decime",
"fijate",
"andate",
"poneme",
"sonreíte",
"sonreí",
"decímelo",
"mandámela",
"decinos",
"Vení",
"TENÉS",
] {
assert!(is_vos_form(w, verbs), "{w}");
}
// Wrong accents, verbs the dictionary lacks, and short stems (`bame` isn't `bar`).
for w in ["habla", "decíme", "decínos", "decimelo", "tenéz", "cantá", "bá", "bame", "", "s", "á", "més", "lo", "melo"] {
assert!(!is_vos_form(w, verbs), "{w}");
}
}
#[test]
fn missing_accents_are_restored() {
assert_eq!(accented("veni", verbs).as_deref(), Some("vení"));
assert_eq!(accented("tenes", verbs).as_deref(), Some("tenés"));
assert_eq!(accented("habla", verbs).as_deref(), Some("hablá"));
for w in ["canta", "casa", "", "s", "pan", "tenés"] {
assert_eq!(accented(w, verbs), None, "{w}");
}
}
proptest::proptest! {
#[test]
fn never_panics(s in "\\PC{0,40}") {
let _ = is_vos_form(&s, |_| true);
let _ = accented(&s, |_| true);
}
}
}
crates/proof/src/hr/grammar.rs
+175
View File
@@ -0,0 +1,175 @@
//! Conservative Croatian grammar and punctuation checks.
//! Handles repeated words, whitespace, capitals and duplicated punctuation;
//! intentionally avoids speculative agreement and comma rules.
use crate::{Issue, IssueKind, ProofLang, words};
/// Words Croatian repeats on purpose: interjections, onomatopoeia and emphatic doubling.
const ALLOWED_REPEATS: &[&str] =
&["da", "ne", "no", "ha", "hej", "joj", "aj", "oh", "uh", "hop", "kuc", "tik", "tak", "bum", "malo", "brzo", "polako", "više", "dobro"];
const ABBREVIATIONS: &[&str] = &[
"npr", "itd", "itp", "tj", "tzv", "odn", "sl", "str", "br", "god", "st", "dr", "mr", "prof", "doc", "ing", "dipl", "mag", "univ", "g", "god",
"mj", "min", "sek", "čl", "st", "č", "toč", "tab", "sl", "sv", "ul", "trg", "pr", "izd", "ur", "svez", "vol", "vid", "usp", "prim", "engl",
"njem", "hrv", "lat", "tel", "mob", "e", "p", "o", "d", "m", "v", "z", "r", "pon", "uto", "sri", "čet", "pet", "sub", "ned", "sij", "velj",
"ožu", "tra", "svi", "lip", "srp", "kol", "ruj", "lis", "stu", "pro",
];
fn issue(start: usize, end: usize, message: String, suggestions: Vec<String>) -> Issue {
Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::Hr }
}
fn capitalize(w: &str) -> String {
crate::affix::capitalize(w)
}
/// Is `prev`, before a period, an abbreviation, an initial, a number or a Roman numeral (after
/// which a lower-case word doesn't start a sentence)?
fn abbreviation_or_number(prev: &str) -> bool {
prev.chars().count() <= 1
|| prev.chars().any(|c| c.is_ascii_digit())
|| (prev.chars().all(|c| matches!(c, 'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M')))
|| ABBREVIATIONS.contains(&prev.to_lowercase().as_str())
}
const CLOSING: [char; 8] = ['"', '”', '’', '\'', ')', '»', ']', '\u{1}'];
const OPENING: [char; 8] = ['"', '“', '«', '(', '[', '\'', '‚', '—'];
/// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after
/// an abbreviation), then space, with nothing but quotes and brackets around?
fn ends_sentence(gap: &str, prev: &str) -> bool {
let t = gap.trim_start_matches(CLOSING);
let Some(end) = t.chars().next() else { return false };
if !matches!(end, '.' | '!' | '?') || t.starts_with("..") {
return false;
}
if end == '.' && abbreviation_or_number(prev) {
return false;
}
let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING);
rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| OPENING.contains(&c))
}
/// Grammar issues in a paragraph of Croatian text, in order.
pub fn check(text: &str) -> Vec<Issue> {
check_with(text, true)
}
/// [`check`] where `capital_first` says whether the text's first word must start a sentence with a capital
/// (false for a list item, which may be a fragment).
pub fn check_with(text: &str, capital_first: bool) -> Vec<Issue> {
let ws: Vec<(usize, usize)> = words(text);
let mut v = Vec::new();
repeated_words(text, &ws, &mut v);
spaces(text, &mut v);
sentence_capitals(text, &ws, capital_first, &mut v);
doubled_punctuation(text, &mut v);
v.sort_by_key(|i| (i.start, i.end));
v
}
fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
for pair in ws.windows(2) {
let [(a0, b0), (a1, b1)] = pair else { continue };
let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue };
if !gap.is_empty()
&& gap.chars().all(char::is_whitespace)
&& w0.chars().all(char::is_alphabetic)
&& w0.to_lowercase() == w1.to_lowercase()
&& !ALLOWED_REPEATS.contains(&w0.to_lowercase().as_str())
{
v.push(issue(*a0, *b1, format!("Ponovljena riječ: \"{w1}\""), vec![w0.to_string()]));
}
}
}
fn spaces(text: &str, v: &mut Vec<Issue>) {
for (i, c) in text.char_indices() {
if c != ' ' {
continue;
}
let rest = text.get(i + 1..).unwrap_or("");
let mut it = rest.chars();
let Some(p) = it.next() else { continue };
let after = it.next();
let flag = match p {
',' => true,
// `;)` and `;-)` are emoticons.
';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')),
// ` ...` is an ellipsis, ` .5` a number.
'.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
// ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space.
':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')),
_ => false,
};
if flag {
v.push(issue(i, i + 1 + p.len_utf8(), "Razmak ispred interpunkcijskog znaka".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Dvostruki razmak".into(), vec![" ".into()]));
}
}
fn sentence_capitals(text: &str, ws: &[(usize, usize)], capital_first: bool, v: &mut Vec<Issue>) {
// List items (`– maçãs,`) may start in lower case.
let list_item = text.trim_end().ends_with([',', ';']);
for (k, &(a, b)) in ws.iter().enumerate() {
let Some(w) = text.get(a..b) else { continue };
let starts = if k == 0 {
let before = text.get(..a).unwrap_or("");
capital_first && !list_item && before.chars().all(|c| c.is_whitespace() || OPENING.contains(&c))
} else {
let Some(&(pa, pb)) = ws.get(k - 1) else { continue };
ends_sentence(text.get(pb..a).unwrap_or(""), text.get(pa..pb).unwrap_or(""))
};
if starts && w.chars().next().is_some_and(char::is_lowercase) && w.chars().all(char::is_alphabetic) {
v.push(issue(a, b, "Rečenica treba započeti velikim slovom".into(), vec![capitalize(w)]));
}
}
}
fn doubled_punctuation(text: &str, v: &mut Vec<Issue>) {
let bytes = text.as_bytes();
let mut i = 0;
while let Some(&b) = bytes.get(i) {
if !matches!(b, b',' | b'.' | b';') {
i += 1;
continue;
}
let run = bytes.get(i..).map_or(0, |r| r.iter().take_while(|&&x| x == b).count());
let flag = match b {
b'.' => run == 2,
_ => run >= 2,
};
if flag {
let end = i + run;
let suggestions: Vec<String> = match b {
b'.' => vec![".".into(), "…".into()],
_ => vec![char::from(b).to_string()],
};
v.push(issue(i, end, "Ponovljen interpunkcijski znak".into(), suggestions));
}
i += run.max(1);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn checks_croatian_punctuation() {
let t = "Ovo ovo je primjer , s dva razmaka..";
let issues = check(t);
assert!(issues.iter().any(|i| i.message.starts_with("Ponovljena riječ")));
assert!(issues.iter().any(|i| i.message == "Dvostruki razmak"));
assert!(issues.iter().all(|i| i.lang == ProofLang::Hr));
}
#[test]
fn abbreviations_do_not_trigger_capitals() {
let issues = check("To je npr. sasvim dobar primjer.");
assert!(!issues.iter().any(|i| i.message == "Rečenica treba započeti velikim slovom"));
}
}
crates/proof/src/hr/mod.rs
+130
View File
@@ -0,0 +1,130 @@
//! Croatian spelling and conservative grammar support.
//! Uses an installed hr_HR Hunspell .aff/.dic pair (not bundled).
//! The engine decodes ISO-8859-2 dictionaries at load time.
//! Suggestions prioritize č/ć, đ, š, ž and common one-character mistakes.
//! Hyphenation remains the host engine's fallback until Croatian patterns are supplied.
pub mod grammar;
mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// File names of a Croatian Hunspell dictionary.
const NAMES: &[&str] = &["hr_HR", "hr-HR", "hr"];
/// The Croatian dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Croatian");
/// The Croatian dictionary: the installed Hunspell `hr_HR` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Croatian spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
DICT.get()
}
/// Use this Croatian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
DICT.install(aff, dic)
}
/// Is a Croatian dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
DICT.has_dictionary()
}
/// Is `word` spelled correctly in Croatian? Numbers, single letters, short words in capitals
/// (acronyms: `HRT`, `RH`), words with digits and words in other scripts are accepted, and so
/// is every word when no Croatian dictionary is installed.
pub fn is_correct(word: &str) -> bool {
is_correct_with(&dictionary(), word)
}
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off,
/// short words in capitals are checked against the dictionary too, and with Ignore words that
/// contain numbers off, so are words with digits.
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
check(&dictionary(), word, opts)
}
/// [`is_correct`] against a given dictionary (an empty one accepts everything).
pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
check(d, word, &crate::Options::default())
}
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
};
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
}
known(d, w)
}
/// Is one word (no hyphens) in the dictionary or the user dictionary?
fn known(d: &AffixDict, w: &str) -> bool {
d.check(w) || crate::in_user_dictionary(&w.to_lowercase())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_dictionary_loads() {
test_dict::install();
let d = dictionary();
assert!(has_dictionary());
assert!(d.len() > 15);
assert!(d.rule_count() > 0);
assert!(!d.map().is_empty() && !d.rep().is_empty() && !d.try_chars().is_empty());
}
#[test]
fn correct_words_and_affixes() {
test_dict::install();
for w in ["kuća", "kuće", "računalo", "čovjek", "riječ", "dijete", "Hrvatska", "čokolada", "škola", "škole", "HRT", "2026"] {
assert!(is_correct(w), "{w}");
}
}
#[test]
fn misspellings_are_flagged() {
test_dict::install();
for w in ["kuca", "racunalo", "covjek", "rijec", "dijeteh", "skolla"] {
assert!(!is_correct(w), "{w}");
}
}
#[test]
fn missing_dictionary_is_silent() {
assert!(is_correct_with(&AffixDict::default(), "nepostojećariječ"));
}
/// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Croatian checks
/// those words against the dictionary instead of accepting them.
#[test]
fn proofing_options_reach_the_croatian_check() {
test_dict::install();
let lang = crate::ProofLang::Hr;
let defaults = crate::Options::default();
let check_caps = crate::Options { ignore_uppercase: false, ..defaults };
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
assert!(crate::is_correct_in_with("ŠKOLLA", lang, &defaults), "short capitals are acronyms by default");
assert!(!crate::is_correct_in_with("ŠKOLLA", lang, &check_caps));
assert!(crate::is_correct_in_with("ŠKOLA", lang, &check_caps));
assert!(crate::is_correct_in_with("školla2", lang, &defaults));
assert!(!crate::is_correct_in_with("školla2", lang, &check_digits));
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
assert!(is_correct("ŠKOLLA"), "direct checks keep the defaults");
}
}
crates/proof/src/hr/suggest.rs
+63
View File
@@ -0,0 +1,63 @@
//! Croatian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with
//! the words Croatian runs into their neighbours (`neznam` → `ne znam`), the letter groups of
//! Croatian diacritics and letters to try, for a dictionary without MAP or TRY lines.
use crate::affix::AffixDict;
use crate::affix_suggest::{Split, Tuning};
/// Short words often run into the next one (`ne znam`, `na primjer`, `da li`).
const LEADING_WORDS: &[&str] = &[
"a", "ali", "i", "ili", "da", "do", "od", "na", "ne", "ni", "o", "po", "pod", "pri", "sa", "s", "se", "su", "u", "uz", "za", "iz", "je", "će",
"bi", "bih", "što", "koji", "koja", "koje", "kao", "kad", "dok", "bez", "pred", "nad",
];
const TRAILING_WORDS: &[&str] =
&["je", "su", "se", "ga", "mi", "ti", "mu", "joj", "ih", "im", "li", "će", "ću", "bi", "bih", "i", "a", "u", "na", "za", "od", "do"];
/// What Croatian adds to its dictionary's tables.
const TUNING: Tuning = Tuning {
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS },
alphabet: "aeiourntsjkvlmdpbgczčćšžđfh",
map: &["cčć", "dđ", "sš", "zž"],
..Tuning::DEFAULT
};
/// Spelling suggestions for a Croatian word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
suggest_in(&super::dictionary(), word, max)
}
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn nosuggest_stems_and_forms_are_never_offered() {
crate::affix::tests::assert_nosuggest_is_honoured(suggest_in);
}
/// A dictionary without MAP lines still gets the Croatian diacritic groups: `kuca` → `kuća`
/// first, before a typo that costs more (`kuha`).
#[test]
fn diacritics_without_map_lines() {
let d = AffixDict::from_hunspell("", "2\nkuća\nkuha\n", &[]).unwrap();
assert_eq!(suggest_in(&d, "kuca", 1), ["kuća"]);
}
#[test]
fn croatian_suggestions() {
crate::hr::test_dict::install();
for (wrong, right) in [("kuca", "kuća"), ("racunalo", "računalo"), ("covjek", "čovjek"), ("rijec", "riječ"), ("skolla", "škola")] {
let result = suggest(wrong, 10);
assert!(result.iter().any(|s| s == right), "{wrong} -> {result:?}, expected {right}");
}
}
#[test]
fn bounded() {
crate::hr::test_dict::install();
assert!(suggest("", 5).is_empty());
assert!(suggest("kuca", 0).is_empty());
assert!(suggest(&"a".repeat(41), 5).is_empty());
}
}
crates/proof/src/hr/test_dict.rs
+42
View File
@@ -0,0 +1,42 @@
//! Small synthetic UTF-8 Croatian Hunspell dictionary for deterministic tests.
pub const AFF: &str = "SET UTF-8
TRY abcčćdđefghijklmnoprsštuvzž
MAP 5
MAP cčć
MAP dđ
MAP sš
MAP zž
MAP iije
REP 3
REP c č
REP dj đ
REP je ije
SFX A Y 2
SFX A a e a
SFX A a i a
";
pub const DIC: &str = "18
kuća/A
računalo
čovjek
riječ
dijete
Hrvatska
čokolada
škola/A
more
sunce
dobar
hrvatski
jezik
tekst
pogreška
pravopis
provjera
računala
";
pub fn install() {
if let Err(e) = super::install_dictionary(AFF, DIC) {
panic!("test dictionary: {e}");
}
}
crates/proof/src/hunspell.rs
+809
View File
@@ -0,0 +1,809 @@
//! Hunspell dictionaries installed on the system, read at runtime.
//!
//! WordCraft doesn't bundle large spelling dictionaries. For languages with an affix dictionary
//! (Polish, Russian), it reads the standard Hunspell `.aff`/`.dic` pair installed by the operating system,
//! a package manager or LibreOffice, with [`crate::affix::AffixDict::from_hunspell`]. When none is
//! installed the language has no spelling check (no marks, no errors), like Word without the
//! language's proofing tools.
//!
//! Search order: the folder set with [`set_dictionary_dir`], the directory in
//! `WORDCRAFT_DICTIONARIES`, WordCraft's own `dictionaries` folder under the user's application
//! data (on every desktop system), then the usual system locations. On the web there are none.
use std::path::{Path, PathBuf};
use std::sync::RwLock;
use crate::affix::AffixDict;
/// Largest `.aff` or `.dic` file read (real ones are a few MB).
const MAX_FILE: u64 = 64 << 20;
/// An extra directory searched first (a setting), set by the front end.
static EXTRA_DIR: RwLock<Option<PathBuf>> = RwLock::new(None);
/// Why an installed dictionary failed to load, by `.dic` name (`pt_BR`): the file and the reason.
/// Without it a file that is there but unreadable looks the same as no file (#2830).
static LOAD_ERRORS: RwLock<Vec<(String, PathBuf, String)>> = RwLock::new(Vec::new());
/// Remember (or, for `Ok`, forget) why the dictionary `key` found at `dic` failed to load, and
/// hand the result back unchanged.
pub fn note_load<T>(key: &str, dic: &Path, result: Result<T, String>) -> Result<T, String> {
let mut errors = LOAD_ERRORS.write().unwrap_or_else(std::sync::PoisonError::into_inner);
errors.retain(|(k, _, _)| k != key);
if let Err(e) = &result {
errors.push((key.to_string(), dic.to_path_buf(), e.clone()));
}
result
}
/// Forget the load error of the dictionary `key` (none found, or it is about to be searched for again).
pub fn clear_load_error(key: &str) {
LOAD_ERRORS.write().unwrap_or_else(std::sync::PoisonError::into_inner).retain(|(k, _, _)| k != key);
}
/// The `.dic` file and the reason of the last failed load of the dictionary `key`, if any.
pub fn load_error(key: &str) -> Option<(PathBuf, String)> {
LOAD_ERRORS.read().unwrap_or_else(std::sync::PoisonError::into_inner).iter().find(|(k, _, _)| k == key).map(|(_, p, e)| (p.clone(), e.clone()))
}
/// Search `dir` before the system locations (a "dictionaries folder" setting); `None` clears it.
/// Takes effect for dictionaries not loaded yet.
pub fn set_dictionary_dir(dir: Option<PathBuf>) {
*EXTRA_DIR.write().unwrap_or_else(std::sync::PoisonError::into_inner) = dir;
LOAD_ERRORS.write().unwrap_or_else(std::sync::PoisonError::into_inner).clear(); // they belong to the old search path
}
/// The LibreOffice extension directories a Hunspell dictionary name may live in: LibreOffice's
/// IDs are per dictionary — Polish is `dict-pl`, Brazilian Portuguese `dict-pt-BR` — so both
/// the primary-language folder and the dashed full name are tried.
fn lo_ext_dirs(name: &str) -> Vec<String> {
let primary = name.split(['-', '_']).next().unwrap_or(name);
let dashed = name.replace('_', "-");
let first = format!("dict-{primary}");
let full = format!("dict-{dashed}");
if full == first { vec![first] } else { vec![first, full] }
}
/// Directories that may hold Hunspell dictionaries, most specific first. `lo_exts`, when a
/// language is being searched for, adds LibreOffice's extension folders for it (`dict-pt-BR`).
pub fn search_dirs(lo_exts: Option<&[String]>) -> Vec<PathBuf> {
let mut dirs: Vec<PathBuf> = Vec::new();
if let Some(d) = EXTRA_DIR.read().unwrap_or_else(std::sync::PoisonError::into_inner).clone() {
dirs.push(d);
}
if cfg!(target_arch = "wasm32") {
return dirs;
}
if let Some(d) = std::env::var_os("WORDCRAFT_DICTIONARIES").filter(|v| !v.is_empty()) {
dirs.push(PathBuf::from(d));
}
// LibreOffice keeps each language under `…/share/extensions/<dict-xx>`.
let lo_dirs =
|prefix: &str| -> Vec<PathBuf> { lo_exts.map_or_else(Vec::new, |exts| exts.iter().map(|e| PathBuf::from(prefix).join(e)).collect()) };
let home = std::env::var_os("HOME").filter(|v| !v.is_empty()).map(PathBuf::from);
if cfg!(windows) {
for var in ["LOCALAPPDATA", "APPDATA"] {
if let Some(base) = std::env::var_os(var).filter(|v| !v.is_empty()).map(PathBuf::from) {
dirs.push(base.join("WordCraft").join("dictionaries"));
dirs.push(base.join("hunspell"));
}
}
for var in ["ProgramFiles", "ProgramFiles(x86)"] {
if let Some(base) = std::env::var_os(var).filter(|v| !v.is_empty()).map(PathBuf::from) {
dirs.extend(lo_dirs(&base.join("LibreOffice").join("share").join("extensions").to_string_lossy()));
}
}
// If LibreOffice is installed on another drive (e.g. D:\Program Files\LibreOffice), check drives C..Z
for drive in b'C'..=b'Z' {
let drive_char = drive as char;
for pf in ["Program Files", "Program Files (x86)"] {
let lo_base = PathBuf::from(format!(r"{drive_char}:\{pf}\LibreOffice\share\extensions"));
if lo_base.is_dir() {
dirs.extend(lo_dirs(&lo_base.to_string_lossy()));
}
}
}
} else if cfg!(target_os = "macos") {
if let Some(h) = &home {
// An app started from Finder or the Dock doesn't see `WORDCRAFT_DICTIONARIES`; this
// folder needs no setting (#1063).
dirs.push(h.join("Library/Application Support/WordCraft/dictionaries"));
dirs.push(h.join("Library/Spelling"));
}
dirs.push(PathBuf::from("/Library/Spelling"));
dirs.push(PathBuf::from("/opt/homebrew/share/hunspell"));
dirs.push(PathBuf::from("/usr/local/share/hunspell"));
dirs.extend(lo_dirs("/Applications/LibreOffice.app/Contents/Resources/extensions"));
} else {
// Nor does one started from a desktop launcher (#1063).
if let Some(data) = xdg_data_home(std::env::var_os("XDG_DATA_HOME"), home.as_deref()) {
dirs.push(data.join("wordcraft").join("dictionaries"));
}
if let Some(h) = &home {
dirs.push(h.join(".local/share/hunspell"));
}
for d in ["/usr/share/hunspell", "/usr/share/myspell", "/usr/share/myspell/dicts", "/usr/local/share/hunspell", "/usr/local/share/myspell"] {
dirs.push(PathBuf::from(d));
}
for d in ["/usr/lib/libreoffice/share/extensions", "/opt/libreoffice/share/extensions"] {
dirs.extend(lo_dirs(d));
}
}
dirs
}
/// The user's data folder on Linux and the BSDs: `$XDG_DATA_HOME` when it is an absolute path (the
/// XDG Base Directory spec ignores a relative one), else `~/.local/share`.
fn xdg_data_home(xdg: Option<std::ffi::OsString>, home: Option<&Path>) -> Option<PathBuf> {
xdg.map(PathBuf::from).filter(|p| p.is_absolute()).or_else(|| home.map(|h| h.join(".local/share")))
}
/// The first `<name>.aff`/`<name>.dic` pair found for any of `names` (`pl_PL`, `pl`), searching
/// [`search_dirs`] (including LibreOffice's extension folder for the first name).
pub fn find(names: &[&str]) -> Option<(PathBuf, PathBuf)> {
let lo = names.first().copied().map(lo_ext_dirs);
search_dirs(lo.as_deref()).into_iter().find_map(|dir| find_in(&dir, names))
}
/// Every `<name>.aff`/`<name>.dic` pair for `names`, name by name, each in every folder
/// ([`search_dirs`], the dictionaries folder first): a file with the first name in a system
/// folder comes before one with the second name in the dictionaries folder. A caller that can't
/// read one tries the next.
pub fn candidates(names: &[&str]) -> Vec<(PathBuf, PathBuf)> {
let lo = names.first().copied().map(lo_ext_dirs);
candidates_in(&search_dirs(lo.as_deref()), names)
}
/// [`candidates`] in `dirs`.
pub fn candidates_in(dirs: &[PathBuf], names: &[&str]) -> Vec<(PathBuf, PathBuf)> {
let mut out: Vec<(PathBuf, PathBuf)> = Vec::new();
for n in names {
for dir in dirs {
let aff = dir.join(format!("{n}.aff"));
let dic = dir.join(format!("{n}.dic"));
if aff.is_file() && dic.is_file() && !out.iter().any(|(_, d)| *d == dic) {
out.push((aff, dic));
}
}
}
out
}
/// The first `<name>.aff`/`<name>.dic` pair for any of `names` in `dir`.
pub fn find_in(dir: &Path, names: &[&str]) -> Option<(PathBuf, PathBuf)> {
names.iter().find_map(|n| {
let aff = dir.join(format!("{n}.aff"));
let dic = dir.join(format!("{n}.dic"));
(aff.is_file() && dic.is_file()).then_some((aff, dic))
})
}
/// Read a dictionary file: a regular file of at most [`MAX_FILE`] bytes. The read itself is
/// bounded, so a file that grows after its size is checked (or a filesystem that reports a wrong
/// size) can't make it read more (#3661).
pub(crate) fn read_capped(p: &Path) -> Result<Vec<u8>, String> {
use std::io::Read;
let err = |e: std::io::Error| format!("{}: {e}", p.display());
let f = std::fs::File::open(p).map_err(err)?;
let meta = f.metadata().map_err(err)?;
if !meta.is_file() {
return Err(format!("{}: not a regular file", p.display()));
}
if meta.len() > MAX_FILE {
return Err(format!("{}: too large ({} bytes)", p.display(), meta.len()));
}
let mut buf = Vec::new();
f.take(MAX_FILE + 1).read_to_end(&mut buf).map_err(err)?;
if buf.len() as u64 > MAX_FILE {
return Err(format!("{}: too large", p.display()));
}
Ok(buf)
}
/// Read a Hunspell pair from disk (size-capped), decoded per the `.aff`'s `SET` line.
pub fn load(aff: &Path, dic: &Path) -> Result<AffixDict, String> {
let (aff_text, dic_text) = decode(&read_capped(aff)?, &read_capped(dic)?)?;
let source = format!("Hunspell dictionary {}", dic.display());
AffixDict::from_hunspell(&aff_text, &dic_text, &[source.as_str()])
}
/// [`load`] for an English dictionary: directives that only matter for other languages
/// (compounding, output conversion) are ignored rather than refused
/// ([`AffixDict::from_hunspell_english`]).
pub fn load_english(aff: &Path, dic: &Path) -> Result<AffixDict, String> {
let (aff_text, dic_text) = decode(&read_capped(aff)?, &read_capped(dic)?)?;
let source = format!("Hunspell dictionary {}", dic.display());
AffixDict::from_hunspell_english(&aff_text, &dic_text, &[source.as_str()])
}
/// ISO 8859-1: every byte is the code point of the same value (the original VERO pt_BR uses it).
fn decode_latin1(bytes: &[u8]) -> String {
bytes.iter().map(|&b| char::from(b)).collect()
}
/// ISO 8859-9 (Latin-5, Turkish): ISO 8859-1 with Ğ İ Ş ğ ı ş in place of Ð Ý Þ ð ý þ. The
/// encoding of older Turkish dictionaries (`SET ISO8859-9`).
fn decode_latin5(bytes: &[u8]) -> String {
bytes
.iter()
.map(|&b| match b {
0xd0 => 'Ğ',
0xdd => 'İ',
0xde => 'Ş',
0xf0 => 'ğ',
0xfd => 'ı',
0xfe => 'ş',
_ => char::from(b),
})
.collect()
}
/// ISO 8859-2 code points 0xA0..=0xFF (0x00..=0x9F map to themselves).
const ISO_8859_2_HIGH: &str = "\u{a0}Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\u{ad}ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙";
fn decode_latin2(bytes: &[u8]) -> String {
let high: Vec<char> = ISO_8859_2_HIGH.chars().collect();
bytes.iter().map(|&b| if b < 0xa0 { char::from(b) } else { high.get(usize::from(b - 0xa0)).copied().unwrap_or('\u{fffd}') }).collect()
}
/// KOI8-R code points 0x80..=0xFF (0x00..=0x7F are ASCII): the encoding of older Russian
/// dictionaries.
const KOI8_R_HIGH: &str =
"─│┌┐└┘├┤┬┴┼▀▄█▌▐░▒▓⌠■∙√≈≤≥\u{a0}⌡°²·÷═║╒ё╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡Ё╢╣╤╥╦╧╨╩╪╫╬©юабцдефгхийклмнопярстужвьызшэщчъЮАБЦДЕФГХИЙКЛМНОПЯРСТУЖВЬЫЗШЭЩЧЪ";
/// Windows-1251 code points 0x80..=0xFF (0x98 is unassigned): the other one Russian dictionaries
/// use.
const CP1251_HIGH: &str = "ЂЃ‚ѓ„…†‡€‰Љ‹ЊЌЋЏђ‘’“”•–—\u{fffd}™љ›њќћџ\u{a0}ЎўЈ¤Ґ¦§Ё©Є«¬\u{ad}®Ї°±Ііґµ¶·ё№є»јЅѕїАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя";
/// Decode a single-byte encoding whose bytes 0x80..=0xFF are `high` (ASCII below).
fn decode_high(bytes: &[u8], high: &str) -> String {
let high: Vec<char> = high.chars().collect();
bytes.iter().map(|&b| if b < 0x80 { char::from(b) } else { high.get(usize::from(b - 0x80)).copied().unwrap_or('\u{fffd}') }).collect()
}
/// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names
/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the
/// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R
/// and Windows-1251 the ones older Russian dictionaries use).
///
/// Under a UTF-8 `SET` with single-byte flags (no `FLAG` line, or `FLAG DEFAULT`), flags are bytes,
/// as in Hunspell: the Hungarian dictionary writes its affix flags and flag aliases as bytes above
/// 0x7F that aren't UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF,
/// one flag per byte); the words, affixes and conditions around them are UTF-8. With any other
/// `FLAG` (`UTF-8`, `num`, `long`) flags are text and the whole file is UTF-8: `SFX 1234` keeps
/// its number (the Turkish dictionary's `FLAG num`).
pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
let (set, flag) = directives(aff);
let set = set.unwrap_or_else(|| "UTF-8".into());
match set.as_str() {
"UTF-8" | "UTF8" => {
if flag.as_deref().is_some_and(|f| f != "DEFAULT") {
let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));
return Ok((text(aff, ".aff")?, text(dic, ".dic")?));
}
Ok((decode_utf8_aff(aff)?, decode_utf8_dic(dic)?))
}
"ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))),
"ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))),
"ISO8859-9" | "ISO-8859-9" => Ok((decode_latin5(aff), decode_latin5(dic))),
"KOI8-R" | "KOI8R" => Ok((decode_high(aff, KOI8_R_HIGH), decode_high(dic, KOI8_R_HIGH))),
"CP1251" | "WINDOWS-1251" | "MICROSOFT-CP1251" => Ok((decode_high(aff, CP1251_HIGH), decode_high(dic, CP1251_HIGH))),
other => Err(format!("unsupported encoding {other}")),
}
}
/// The first arguments of the `.aff`'s first `SET` and `FLAG` lines, in capitals, found in one pass
/// over the whole file: a `FLAG UTF-8` after the rules still decides how the `.dic` flags are read,
/// as in Hunspell. Only the first argument counts: `FLAG UTF-8 # Unicode flags` is UTF-8.
fn directives(aff: &[u8]) -> (Option<String>, Option<String>) {
let (mut set, mut flag) = (None, None);
for l in aff.split(|&b| b == b'\n') {
if set.is_some() && flag.is_some() {
break;
}
let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l);
let slot = match l.first() {
Some(b'S') if set.is_none() => (&mut set, &b"SET"[..]),
Some(b'F') if flag.is_none() => (&mut flag, &b"FLAG"[..]),
_ => continue,
};
let arg = l
.strip_prefix(slot.1)
.filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t')))
.and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty()));
if let Some(a) = arg {
*slot.0 = Some(String::from_utf8_lossy(a).to_ascii_uppercase());
}
}
(set, flag)
}
/// Directives whose one argument is a flag.
const FLAG_DIRECTIVES: &[&str] = &[
"CIRCUMFIX",
"COMPOUNDBEGIN",
"COMPOUNDEND",
"COMPOUNDFLAG",
"COMPOUNDFORBIDFLAG",
"COMPOUNDLAST",
"COMPOUNDFIRST",
"COMPOUNDMIDDLE",
"COMPOUNDPERMITFLAG",
"COMPOUNDROOT",
"FORBIDDENWORD",
"FORCEUCASE",
"GENERATE",
"HU_KOTOHANGZO",
"KEEPCASE",
"LEMMA_PRESENT",
"NEEDAFFIX",
"NOSUGGEST",
"ONLYINCOMPOUND",
"ONLYROOT",
"PSEUDOROOT",
"SUBSTANDARD",
"WARN",
];
/// Directives that only describe the dictionary; a non-UTF-8 byte in them (the Hungarian
/// dictionary's `NAME` is ISO 8859-2) is replaced instead of failing the load.
const METADATA_DIRECTIVES: &[&str] = &["NAME", "HOME", "VERSION", "LANG", "AM"];
/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). `what`
/// names the line in an error.
fn text_and_flags(field: &[u8], what: impl Fn() -> String) -> Result<String, String> {
let mut slash = None;
let mut i = 0;
while let Some(&b) = field.get(i) {
if b == b'\\' && field.get(i + 1) == Some(&b'/') {
i += 2;
continue;
}
if b == b'/' {
slash = Some(i);
break;
}
i += 1;
}
let (text, flags) = match slash {
Some(k) => (field.get(..k).unwrap_or_default(), field.get(k..).unwrap_or_default()),
None => (field, &[][..]),
};
let mut out = std::str::from_utf8(text).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string();
out.push_str(&decode_latin1(flags));
Ok(out)
}
/// Split a line into fields on spaces and tabs, keeping the separators (so a line rebuilt from
/// its parts is the original).
fn fields(line: &[u8]) -> Vec<&[u8]> {
let mut out = Vec::new();
let mut start = 0;
let mut in_sep = line.first().is_some_and(|b| matches!(b, b' ' | b'\t'));
for (i, b) in line.iter().enumerate() {
let sep = matches!(b, b' ' | b'\t');
if sep != in_sep {
out.push(line.get(start..i).unwrap_or_default());
start = i;
in_sep = sep;
}
}
out.push(line.get(start..).unwrap_or_default());
out
}
/// A UTF-8 `.aff` with single-byte flags (see [`decode`]).
fn decode_utf8_aff(aff: &[u8]) -> Result<String, String> {
let mut out = String::with_capacity(aff.len());
for (n, line) in aff.split(|&b| b == b'\n').enumerate() {
if n > 0 {
out.push('\n');
}
if let Ok(s) = std::str::from_utf8(line)
&& s.is_ascii()
{
out.push_str(s);
continue;
}
let parts = fields(line);
// Field index among the non-separator parts.
let words: Vec<usize> =
parts.iter().enumerate().filter(|(_, p)| !p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).map(|(i, _)| i).collect();
let dir = words
.first()
.and_then(|&i| parts.get(i))
.map(|p| String::from_utf8_lossy(p).trim_start_matches('\u{feff}').to_string())
.unwrap_or_default();
let what = || format!(".aff line {}", n + 1);
if dir.starts_with('#') || METADATA_DIRECTIVES.contains(&dir.as_str()) {
out.push_str(&String::from_utf8_lossy(line));
continue;
}
// A trailing `# comment` (the Hungarian `AF` lines number themselves) is never a word or
// a flag; from there on the line is kept as text. It can only start after the directive's
// own fields, since a field such as an affix condition may itself begin with `#`.
let field_text = |k: usize| words.get(k).and_then(|&w| parts.get(w)).map(|p| String::from_utf8_lossy(p).into_owned());
let first_free = comment_field(&dir, field_text);
let comment_at = words.iter().skip(first_free).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + first_free);
// The field index of the next non-separator part.
let mut next_field = 0;
for (i, p) in parts.iter().enumerate() {
let field = (!p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).then(|| {
next_field += 1;
next_field - 1
});
let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w));
let decoded = match (dir.as_str(), field) {
_ if in_comment => String::from_utf8_lossy(p).into_owned(),
("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => decode_latin1(p),
("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, what)?,
// A field that names one flag keeps its first byte, as Hunspell's `decode_flag`
// does: `SFX é` is the flag 0xC3, the first of the two flags `kert/é` carries.
("PFX" | "SFX", Some(1)) => decode_latin1(p.get(..1).unwrap_or_default()),
(d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => decode_latin1(p.get(..1).unwrap_or_default()),
_ => std::str::from_utf8(p).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(),
};
out.push_str(&decoded);
}
}
Ok(out)
}
/// The first field of an `.aff` line where a trailing `# comment` may start: after the
/// directive's own fields. A table's count line (`REP 12`, `PFX A Y 3`) has fewer fields than its
/// rows (`REP a b`, `PFX A 0 x .`). `field(k)` is the line's `k`-th field.
fn comment_field(dir: &str, field: impl Fn(usize) -> Option<String>) -> usize {
let number = |k: usize| field(k).is_some_and(|f| f.parse::<usize>().is_ok());
match dir {
"PFX" | "SFX" if field(2).is_some_and(|f| f == "Y" || f == "N") && number(3) => 4,
"PFX" | "SFX" => 5,
"REP" | "ICONV" | "OCONV" | "PHONE" | "MAP" | "BREAK" | "AF" | "AM" | "COMPOUNDRULE" | "CHECKCOMPOUNDPATTERN"
if number(1) && field(2).is_none_or(|f| f.starts_with('#')) =>
{
2
}
"REP" | "ICONV" | "OCONV" | "PHONE" => 3,
"CHECKCOMPOUNDPATTERN" => 4,
_ => 2,
}
}
/// Where the `word/flags` entry of a `.dic` line ends, as Hunspell reads it: at a tab or at the
/// first morphological field (a space or tab, two characters and `:`, as in ` po:noun`), whichever
/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the end of the line
/// (`kot/B 12` has the flags `B 12`, as in Hunspell).
pub(crate) fn dic_entry_end(line: &[u8]) -> usize {
let blank = |b: &u8| matches!(b, b' ' | b'\t');
let morph = line.windows(4).position(|w| matches!(w, [a, b, c, b':'] if blank(a) && !blank(b) && !blank(c)));
let tab = line.iter().position(|&b| b == b'\t');
let end = match (morph, tab) {
(Some(m), Some(t)) => m.min(t),
(m, t) => match m.or(t) {
Some(e) => e,
None => line.len(),
},
};
// Blanks before the morphological field or the line end aren't part of the entry.
line.get(..end).map_or(end, |e| e.iter().rposition(|b| !blank(b)).map_or(0, |k| k + 1))
}
/// A UTF-8 `.dic` with single-byte flags: the word is UTF-8, the flags after `/` are bytes.
fn decode_utf8_dic(dic: &[u8]) -> Result<String, String> {
let mut out = String::with_capacity(dic.len());
for (n, line) in dic.split(|&b| b == b'\n').enumerate() {
if n > 0 {
out.push('\n');
}
if let Ok(s) = std::str::from_utf8(line)
&& s.split_once('/').is_none_or(|(_, flags)| flags.is_ascii())
{
out.push_str(s);
continue;
}
let what = || format!(".dic line {}", n + 1);
// The first field is `word/flags` (it may hold spaces, see [`dic_entry_end`]);
// morphological fields after it are UTF-8 text.
let end = dic_entry_end(line);
out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), what)?);
let rest = line.get(end..).unwrap_or_default();
out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{}: not UTF-8: {e}", what()))?);
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
/// WordCraft's own dictionaries folder under the user's application data is searched on every
/// desktop system, after the setting and before the shared system locations (#1063).
#[test]
fn the_app_folder_comes_before_the_system_locations() {
if cfg!(target_arch = "wasm32") {
return;
}
let user = if cfg!(windows) { "LOCALAPPDATA" } else { "HOME" };
if std::env::var_os(user).filter(|v| !v.is_empty()).is_none() {
return; // no user folder to derive it from
}
let dirs = search_dirs(None);
let at = |suffix: &str| dirs.iter().position(|d| d.ends_with(suffix));
let (app, shared) = if cfg!(windows) {
("WordCraft/dictionaries", "hunspell")
} else if cfg!(target_os = "macos") {
("Application Support/WordCraft/dictionaries", "/Library/Spelling")
} else {
("wordcraft/dictionaries", "/usr/share/hunspell")
};
assert!(matches!((at(app), at(shared)), (Some(app), Some(shared)) if app < shared), "{dirs:?}");
}
/// `$XDG_DATA_HOME` counts only as an absolute path; otherwise (unset, empty or relative) the
/// spec's default `~/.local/share` applies, and with no home there is no folder.
#[test]
fn xdg_data_home_follows_the_spec() {
let home = Path::new("/home/a");
assert_eq!(xdg_data_home(Some("/data".into()), Some(home)), Some(PathBuf::from("/data")));
assert_eq!(xdg_data_home(Some("/data".into()), None), Some(PathBuf::from("/data")));
for xdg in [None, Some(""), Some("relative/data")] {
assert_eq!(xdg_data_home(xdg.map(Into::into), Some(home)), Some(home.join(".local/share")), "{xdg:?}");
assert_eq!(xdg_data_home(xdg.map(Into::into), None), None, "{xdg:?}");
}
}
#[test]
fn decodes_latin2_and_utf8() {
let aff = b"SET ISO8859-2\nTRY a\n";
let (a, d) = decode(aff, b"2\n\xbf\xf3\xb3w\n").unwrap();
assert!(a.contains("TRY a"));
assert_eq!(d, "2\nżółw\n");
let (_, d) = decode("SET UTF-8\n".as_bytes(), "żółw".as_bytes()).unwrap();
assert_eq!(d, "żółw");
let (_, d) = decode(b"SET ISO8859-9\n", b"\xd0\xdd\xde\xf0\xfd\xfe\xd6\xe7").unwrap();
assert_eq!(d, "ĞİŞğışÖç");
assert!(decode(b"SET SHIFT_JIS\n", b"").is_err());
assert!(decode(b"SET UTF-8\n", b"\xff\xfe").is_err());
}
/// #2830: a dictionary that is there but can't be read leaves its reason behind; a good
/// load or no file clears it.
#[test]
fn load_error_is_remembered_until_the_next_good_load() {
let key = "test_unreadable_2830";
assert_eq!(load_error(key), None);
let dic = Path::new("/dicts/test.dic");
assert!(note_load(key, dic, decode(b"SET SHIFT_JIS\n", b"")).is_err());
let (file, reason) = load_error(key).expect("error kept");
assert_eq!(file, dic);
assert_eq!(reason, "unsupported encoding SHIFT_JIS");
assert!(note_load(key, dic, decode(b"SET UTF-8\n", b"")).is_ok());
assert_eq!(load_error(key), None);
let _ = note_load::<()>(key, dic, Err("too large".into()));
clear_load_error(key);
assert_eq!(load_error(key), None);
}
#[test]
fn decodes_latin1() {
// `orçamento esperávamos` in ISO 8859-1: ç = 0xE7, á = 0xE1.
let (a, d) = decode(b"SET ISO8859-1\nTRY \xe7\n", b"1\nor\xe7amento/A\nesper\xe1vamos\n").unwrap();
assert!(a.contains("TRY ç"), "{a}");
assert_eq!(d, "1\norçamento/A\nesperávamos\n");
assert!(decode(b"SET ISO-8859-1\n", b"\xe7").is_ok());
}
#[test]
fn utf8_dictionaries_keep_single_byte_flags() {
// The Hungarian dictionary is `SET UTF-8` but writes flags as single bytes above 0x7F,
// which aren't UTF-8; each byte is one flag. Its `NAME` line is ISO 8859-2.
let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 2\nAF \xd5a # 1\nAF \xe9 # 2\nSFX \xd5 Y 1\nSFX \xd5 0 ok/2 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n";
let dic = "2\nház/1\t1\nkert/\u{e9}\n".as_bytes();
let (a, d) = decode(aff, dic).unwrap();
assert!(a.contains("AF \u{d5}a # 1") && a.contains("AF \u{e9} # 2") && a.contains("SFX \u{d5} 0 ok/2 . 7"));
assert!(d.contains("kert/\u{c3}\u{a9}"), "a UTF-8 é in the flags is two one-byte flags");
let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();
assert!(dict.check("ház") && dict.check("házok") && dict.check("házokban") && dict.check("házat"));
assert!(!dict.check("házban"));
// `FLAG UTF-8` keeps the whole file UTF-8, and words must still be UTF-8.
let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n");
assert!(decode(b"SET UTF-8\nSFX a Y 1\nSFX a 0 \xe9 .\n", b"").is_err(), "an affix must be UTF-8");
assert!(decode(b"SET UTF-8\n", b"1\nh\xe1z/a\n").is_err(), "a stem must be UTF-8");
}
#[test]
fn single_flag_fields_keep_their_first_byte() {
// Without `FLAG UTF-8`, a UTF-8 `é` (0xC3 0xA9) in a field that names one flag is the
// flag 0xC3; in a `.dic` flag list it is the two flags 0xC3 and 0xA9.
let aff = "SET UTF-8\nKEEPCASE é\nSFX é Y 1\nSFX é 0 ok .\nSYLLABLENUM é\n".as_bytes();
let (a, d) = decode(aff, "2\nkert/é\nNATO/é\n".as_bytes()).unwrap();
assert!(a.contains("KEEPCASE \u{c3}\n") && a.contains("SFX \u{c3} Y 1") && a.contains("SFX \u{c3} 0 ok ."), "{a}");
assert!(a.contains("SYLLABLENUM \u{c3}\u{a9}"), "SYLLABLENUM lists flags: {a}");
assert!(d.contains("kert/\u{c3}\u{a9}"));
let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();
assert!(dict.check("kert") && dict.check("kertok"));
assert!(dict.check("NATO") && !dict.check("Nato"), "KEEPCASE é reaches NATO/é");
}
#[test]
fn numeric_and_long_flags_stay_text() {
// `FLAG num` (the Turkish dictionary) and `FLAG long`: no byte-wise flag reading, so a
// number with a non-ASCII rule or comment keeps every digit.
let aff = "SET UTF-8\nFLAG num\nFORBIDDENWORD 6000 # tiltott ő\nSFX 1234 Y 1\nSFX 1234 0 ları .\n";
let (a, d) = decode(aff.as_bytes(), "1\nkitap/1234\n".as_bytes()).unwrap();
assert_eq!(a, aff);
assert_eq!(d, "1\nkitap/1234\n");
let tr = crate::tr::dict::TrDict::from_hunspell(&a, &d).unwrap();
assert!(tr.contains("kitapları") && tr.contains("kitap"));
let long = "SET UTF-8\nFLAG long\nSFX Aa Y 1\nSFX Aa 0 ős .\n";
assert_eq!(decode(long.as_bytes(), "1\nkő/Aa\n".as_bytes()).unwrap().0, long);
assert!(decode(b"SET UTF-8\nFLAG num\n", b"1\nh\xe1z/1\n").is_err(), "the .dic must be UTF-8");
// `FLAG DEFAULT` is the single-byte default.
let (_, d) = decode(b"SET UTF-8\nFLAG default\n", b"1\nkert/\xe9\n").unwrap();
assert_eq!(d, "1\nkert/\u{e9}\n");
}
#[test]
fn flag_utf8_counts_anywhere_in_the_aff() {
// A `FLAG UTF-8` after a long rule table still makes the `.dic` flags Unicode.
let mut aff = b"SET UTF-8\nSFX a Y 400\n".to_vec();
for _ in 0..400 {
aff.extend_from_slice(b"SFX a 0 aaaaaaaaaaaaaaaaaaaaaaaa .\n");
}
aff.extend_from_slice(b"FLAG UTF-8\r\n");
assert!(aff.len() > 8192);
let (_, d) = decode(&aff, "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n");
}
#[test]
fn a_comment_after_flag_utf8_keeps_unicode_flags() {
let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8 # Unicode flags\n", "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n");
let (_, d) = decode(b"SET\tUTF-8 # comment\nFLAG\t\tUTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n", "repeated spaces and tabs separate the argument");
}
#[test]
fn decodes_russian_single_byte_encodings() {
// `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0).
let (a, d) = decode(b"SET KOI8-R\nTRY \xcf\xc5\n", b"1\n\xa3\xcc\xcb\xc1/A\n").unwrap();
assert!(a.contains("TRY ое"), "{a}");
assert_eq!(d, "1\nёлка/A\n");
let (_, d) = decode(b"SET microsoft-cp1251\n", b"1\n\xb8\xeb\xea\xe0 \xc4\xee\xec\n").unwrap();
assert_eq!(d, "1\nёлка Дом\n");
assert_eq!(decode_high(&[0x98], CP1251_HIGH), "\u{fffd}");
}
#[test]
fn comments_start_after_the_directive_fields() {
let f = |line: &'static str| move |k: usize| line.split_whitespace().nth(k).map(str::to_string);
assert_eq!(comment_field("PFX", f("PFX A Y 3 # count")), 4);
assert_eq!(comment_field("SFX", f("SFX A 0 # .")), 5, "a rule's condition may be `#`");
assert_eq!(comment_field("AF", f("AF 1234 # aliases")), 2);
assert_eq!(comment_field("AF", f("AF Aa # 1")), 2);
assert_eq!(comment_field("REP", f("REP 12")), 2);
assert_eq!(comment_field("REP", f("REP # x")), 3);
assert_eq!(comment_field("CHECKCOMPOUNDPATTERN", f("CHECKCOMPOUNDPATTERN a b c")), 4);
assert_eq!(comment_field("KEEPCASE", f("KEEPCASE k")), 2);
// A byte flag after an append text that begins with `#` stays a flag, not a comment.
let (a, _) = decode(b"SET UTF-8\nSFX \xd5 Y 1 # header\nSFX \xd5 0 #/\xe9 . # rule\n", b"").unwrap();
assert!(a.contains("SFX \u{d5} 0 #/\u{e9} . # rule"), "{a}");
}
#[test]
fn dic_entries_end_where_hunspell_ends_them() {
let end = |l: &str| l.get(..dic_entry_end(l.as_bytes())).unwrap().to_string();
assert_eq!(end("word/A po:noun\tts:x"), "word/A", "the morphological field comes before the tab");
assert_eq!(end("ad hoc/B\tpo:adv"), "ad hoc/B", "a tab ends a phrase");
assert_eq!(end("ad hoc/B po:adv"), "ad hoc/B");
assert_eq!(end("word/A 123"), "word/A 123", "neither: the whole line, as in Hunspell");
assert_eq!(end("ad hoc/B "), "ad hoc/B");
assert_eq!(end("word"), "word");
assert_eq!(end("a:b/A"), "a:b/A", "a colon inside the word isn't a field");
// Single-byte flags of a phrase entry are read byte by byte, then the UTF-8 morphology.
let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\tpo:adv\n").unwrap();
assert_eq!(d, "1\nad hoc/\u{e9}\tpo:adv\n");
// Without a tab or a morphological field the whole line is the entry: its byte flags
// don't fail the load (the Hungarian dictionary has such lines: `tök jó/396`).
let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\n").unwrap();
assert_eq!(d, "1\nad hoc/\u{e9}\n");
}
#[test]
fn libreoffice_extension_dirs_follow_the_language() {
// LibreOffice's IDs are per dictionary: Polish is `dict-pl`, Brazilian Portuguese
// `dict-pt-BR`, so both the primary-language folder and the dashed full name are tried.
assert_eq!(super::lo_ext_dirs("cs_CZ"), ["dict-cs", "dict-cs-CZ"]);
assert_eq!(super::lo_ext_dirs("id_ID"), ["dict-id", "dict-id-ID"]);
assert_eq!(super::lo_ext_dirs("pl_PL"), ["dict-pl", "dict-pl-PL"]);
assert_eq!(super::lo_ext_dirs("pt_BR"), ["dict-pt", "dict-pt-BR"]);
assert_eq!(super::lo_ext_dirs("de-DE"), ["dict-de", "dict-de-DE"]);
let dirs = search_dirs(Some(&super::lo_ext_dirs("pt_BR")));
assert!(dirs.iter().any(|d| d.ends_with("dict-pt-BR")), "{dirs:?}");
assert!(search_dirs(None).iter().all(|d| !d.to_string_lossy().contains("dict-")), "no language: no LibreOffice guess");
}
/// craft-libs#482 re-review: a name is looked for in every folder before the next name, so
/// an `en_US` in the dictionaries folder doesn't beat a system `en_GB` for British English.
#[test]
fn candidates_go_name_by_name_across_the_folders() {
let base = std::env::temp_dir().join(format!("wordcraft-candidates-{}", std::process::id()));
let (folder, system) = (base.join("folder"), base.join("system"));
for (dir, name) in [(&folder, "en_US"), (&system, "en_GB"), (&folder, "en-GB")] {
std::fs::create_dir_all(dir).unwrap();
std::fs::write(dir.join(format!("{name}.aff")), "SET UTF-8\n").unwrap();
std::fs::write(dir.join(format!("{name}.dic")), "1\nx\n").unwrap();
}
let dirs = [folder.clone(), system.clone()];
let dics: Vec<PathBuf> = candidates_in(&dirs, &["en_GB", "en-GB"]).into_iter().map(|(_, d)| d).collect();
assert_eq!(dics, vec![system.join("en_GB.dic"), folder.join("en-GB.dic")]);
let us: Vec<PathBuf> = candidates_in(&dirs, &["en_US", "en_GB"]).into_iter().map(|(_, d)| d).collect();
assert_eq!(us, vec![folder.join("en_US.dic"), system.join("en_GB.dic")]);
assert!(candidates_in(&dirs, &["en_CA"]).is_empty());
let _ = std::fs::remove_dir_all(&base);
}
#[test]
fn finds_and_loads_a_pair_in_a_directory() {
let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
assert!(find_in(&dir, &["xx_XX"]).is_none(), "no files: nothing found");
std::fs::write(dir.join("xx_XX.aff"), "SET UTF-8\nSFX A Y 1\nSFX A 0 y .\n").unwrap();
assert!(find_in(&dir, &["xx_XX"]).is_none(), "an .aff alone isn't a dictionary");
std::fs::write(dir.join("xx_XX.dic"), "1\nkot/A\n").unwrap();
let (aff, dic) = find_in(&dir, &["yy", "xx_XX"]).unwrap();
let d = load(&aff, &dic).unwrap();
assert!(d.check("kot") && d.check("koty") && !d.check("kotx"));
assert!(load(&dir.join("missing.aff"), &dic).is_err());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn reads_are_bounded_and_regular_files_only() {
let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-cap-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let aff = dir.join("xx.aff");
std::fs::write(&aff, "SET UTF-8\n").unwrap();
// One byte over the cap (sparse: nothing is written) is refused without reading it.
let big = dir.join("big.dic");
std::fs::File::create(&big).unwrap().set_len(MAX_FILE + 1).unwrap();
assert!(load(&aff, &big).unwrap_err().contains("too large"));
// A directory, and on Unix an endless device, are not dictionary files. (No clock: a
// reader that read `/dev/zero` would never return, so the error itself is the proof.)
assert!(load(&aff, &dir).unwrap_err().contains("not a regular file"));
#[cfg(unix)]
assert!(load(&aff, Path::new("/dev/zero")).unwrap_err().contains("not a regular file"));
// Turkish reads through the same reader.
let (tr_aff, tr_dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic"));
std::fs::write(&tr_aff, "SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n").unwrap();
std::fs::write(&tr_dic, "1\nkitap/1\n").unwrap();
assert!(crate::tr::load(&tr_aff, &tr_dic).unwrap().contains("kitaplar"));
assert!(crate::tr::load(&tr_aff, &big).unwrap_err().contains("too large"));
let _ = std::fs::remove_dir_all(&dir);
}
/// A Turkish dictionary in ISO 8859-9, the encoding older tr_TR dictionaries use, read from
/// disk through the capped reader and decoded per its `SET` line: its words with Turkish
/// letters (ğ ı ş İ, and ç ö ü from Latin-1) are found, as stems and with a suffix. Before the
/// Turkish dictionary went through this reader it was read as UTF-8, which these bytes are not.
#[test]
fn a_turkish_dictionary_in_iso_8859_9_loads_from_disk() {
let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-latin5-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let (aff, dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic"));
std::fs::write(&aff, b"SET ISO8859-9\nSFX 1 Y 2\nSFX 1 0 lar .\nSFX 1 0 \xfd .\nSFX 2 Y 1\nSFX 2 0 ler .\n").unwrap();
// ağaç/1 (-lar, -ı), ışık, şişe/2 (-ler), İzmir, öğrenci/2, güneş.
let words: &[u8] = b"6\na\xf0a\xe7/1\n\xfd\xfe\xfdk\n\xfei\xfee/2\n\xddzmir\n\xf6\xf0renci/2\ng\xfcne\xfe\n";
std::fs::write(&dic, words).unwrap();
assert!(String::from_utf8(words.to_vec()).is_err(), "the .dic is not UTF-8");
let d = crate::tr::load(&aff, &dic).unwrap();
for w in ["ağaç", "ağaçlar", "ağaçı", "ışık", "şişe", "şişeler", "İzmir", "öğrenci", "öğrenciler", "güneş"] {
assert!(d.contains(w), "{w}");
}
// The six letters ISO 8859-9 puts in place of ISO 8859-1's aren't read as those.
assert!(!d.contains("aðaç") && !d.contains("ýþýk") && !d.contains("Ýzmir"));
assert!(!d.contains("ağaçs") && !d.contains("ışıklar") && !d.contains("şişelar"), "only the rules' suffixes, on the stems that take them");
let _ = std::fs::remove_dir_all(&dir);
}
}
crates/proof/src/hyphen.rs
+939
View File
@@ -0,0 +1,939 @@
//! Hyphenation, per proofing language ([`ProofLang`]).
//!
//! **English** (and languages without patterns of their own, as before languages were told
//! apart), lookup order for each word:
//! 1. the **dictionary** ([`Dictionary::en_us`]) — the public-domain Moby Hyphenator II word list
//! by Grady Ward (~160k words), stored compactly in craft-i18n's `language-data/en-us/en-us.dic`; simple
//! inflections (`'s`, `-s`, `-es`, `-ed`, `-d`) inherit the breaks of their stem;
//! 2. **Liang patterns** ([`Patterns::en_us`]) for words not in the dictionary — our own pattern
//! set, trained from the same list by DesignCraft's patgen (generated by DesignCraft's `hyphgen` example);
//! 3. a small syllable **heuristic** for words outside the pattern alphabet.
//!
//! **Polish**: the exception words, then the TeX Polish patterns ([`Patterns::pl`], by Hanna
//! Kołodziejska, Bogusław Jackowski and Marek Ryćko, MIT licence option), then a Polish syllable
//! heuristic for words with letters outside the Polish alphabet (`é`, `ü`…).
//!
//! **Bulgarian**: the Bulgarian syllable rules ([`crate::bg`]); no pattern file or word list.
//!
//! **German**: the TeX German patterns for the reformed spelling ([`Patterns::de`]), also for
//! Swiss text, whose `ss` the patterns split like any double consonant (`Stras-se`).
//!
//! Then the paragraph's limits ([`Limits`]) are applied.
pub use crate::dict;
pub use crate::patterns;
use std::sync::OnceLock;
pub use crate::lang::ProofLang;
pub use dict::Dictionary;
pub use patterns::Patterns;
// The embedded Language data (`language-data/` of craft-i18n, see `build.rs`); each is empty when
// it isn't embedded. The TeX files are hyph-utf8's `hyph-pl.tex`, `hyph-pt.tex`, `hyph-it.tex` and
// `hyph-de-1996.tex` (reformed spelling), unmodified, their licence headers in the files.
use crate::data::{DE_PAT, EN_US_DIC, EN_US_PAT, IT_PAT, PL_PAT, PT_PAT};
impl Dictionary {
/// The bundled US English dictionary (decoded on first use).
pub fn en_us() -> &'static Dictionary {
static D: OnceLock<Dictionary> = OnceLock::new();
D.get_or_init(|| {
if EN_US_DIC.is_empty() {
return Dictionary::default();
}
Dictionary::from_bytes(EN_US_DIC).unwrap_or_else(|e| {
crate::english::note_embedded_error(e);
Dictionary::default()
})
})
}
}
impl Patterns {
/// The bundled US English patterns (parsed on first use).
pub fn en_us() -> &'static Patterns {
static P: OnceLock<Patterns> = OnceLock::new();
P.get_or_init(|| Patterns::parse(EN_US_PAT))
}
/// The bundled Polish patterns and exception words (parsed on first use).
pub fn pl() -> &'static Patterns {
static P: OnceLock<Patterns> = OnceLock::new();
P.get_or_init(|| Patterns::parse(PL_PAT))
}
/// The bundled Portuguese patterns and exception words (parsed on first use).
pub fn pt() -> &'static Patterns {
static P: OnceLock<Patterns> = OnceLock::new();
P.get_or_init(|| Patterns::parse(PT_PAT))
}
/// The bundled Italian patterns (parsed on first use; the file has no exception words).
pub fn it() -> &'static Patterns {
static P: OnceLock<Patterns> = OnceLock::new();
P.get_or_init(|| Patterns::parse(IT_PAT))
}
/// The bundled German patterns (parsed on first use; the file has no exception words).
pub fn de() -> &'static Patterns {
static P: OnceLock<Patterns> = OnceLock::new();
P.get_or_init(|| Patterns::parse(DE_PAT))
}
}
/// Hyphenation limits from the paragraph's Hyphenation settings.
#[derive(Clone, Copy, Debug)]
pub struct Limits {
pub min_word: usize,
pub after_first: usize,
pub before_last: usize,
pub capitalized: bool,
}
impl Default for Limits {
fn default() -> Self {
Limits { min_word: 5, after_first: 2, before_last: 2, capitalized: true }
}
}
/// Where a word's break points came from.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Source {
Dictionary,
Patterns,
Heuristic,
/// A language's syllable rules (Bulgarian, [`crate::bg`]).
Rules,
None,
}
fn is_apostrophe(c: char) -> bool {
matches!(c, '\'' | '’' | 'ʼ')
}
fn is_word_char(c: char) -> bool {
c.is_alphabetic() || is_apostrophe(c)
}
/// Unrestricted break points of one lowercase core (letters and `'` only, no leading/trailing `'`).
fn core_breaks(core: &[char], lang: ProofLang) -> (Vec<usize>, Source) {
let n = core.len();
if n < 2 {
return (vec![], Source::None);
}
if lang == ProofLang::Pl {
return core_breaks_pl(core);
}
// Bulgarian words follow the Bulgarian syllable rules; Latin-script words in a Bulgarian run
// (`WordCraft`, `email`) are hyphenated as English, as in any other language.
if lang == ProofLang::Bg && core.iter().all(|&c| crate::bg::is_letter(c)) {
return (crate::bg::points(core), Source::Rules);
}
if lang == ProofLang::Tr && core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) {
return (crate::tr::hyphen::points(core), Source::Rules);
}
match lang {
ProofLang::Pt => return core_breaks_patterns(Patterns::pt(), core),
ProofLang::It => return core_breaks_patterns(Patterns::it(), core),
ProofLang::De => return core_breaks_patterns(Patterns::de(), core),
_ => {}
}
if lang == ProofLang::Ru && core.iter().all(|&c| crate::ru::is_cyrillic(c)) {
return (crate::ru::hyphen::points(core), Source::Heuristic);
}
let dict = Dictionary::en_us();
// No English Language data embedded: hyphenation is off.
if dict.is_empty() && Patterns::en_us().is_empty() {
return (vec![], Source::None);
}
let s: String = core.iter().collect();
let from_mask = |m: u64, len: usize| (1..len.min(64)).filter(|&i| m & (1 << i) != 0).collect::<Vec<_>>();
if let Some(m) = dict.get(&s) {
return (from_mask(m, n), Source::Dictionary);
}
// Inflections inherit the stem's breaks (the suffix adds no break of its own).
for suf in ["'s", "s", "es", "ed", "d"] {
if let Some(stem) = s.strip_suffix(suf)
&& stem.chars().count() >= 2
&& let Some(m) = dict.get(stem)
{
return (from_mask(m, stem.chars().count()), Source::Dictionary);
}
}
let pats = Patterns::en_us();
if !pats.is_empty()
&& let Some(p) = pats.points(core)
{
return (p, Source::Patterns);
}
(heuristic_points(core), Source::Heuristic)
}
/// Polish: exception words, then the patterns, then the Polish syllable heuristic.
fn core_breaks_pl(core: &[char]) -> (Vec<usize>, Source) {
let pats = Patterns::pl();
// No Polish patterns embedded: hyphenation is off.
if pats.is_empty() {
return (vec![], Source::None);
}
let s: String = core.iter().collect();
if let Some(p) = pats.exception(&s) {
return (p.to_vec(), Source::Dictionary);
}
if !pats.is_empty()
&& let Some(p) = pats.points(core)
{
return (p, Source::Patterns);
}
(heuristic_points_pl(core), Source::Heuristic)
}
/// [`core_breaks_pl`] for a language with bundled patterns (Portuguese, Italian, German): its
/// exception words, the patterns, then the generic heuristic.
fn core_breaks_patterns(pats: &Patterns, core: &[char]) -> (Vec<usize>, Source) {
// The language's patterns aren't embedded: hyphenation is off.
if pats.is_empty() {
return (vec![], Source::None);
}
let s: String = core.iter().collect();
if let Some(p) = pats.exception(&s) {
return (p.to_vec(), Source::Dictionary);
}
if !pats.is_empty()
&& let Some(p) = pats.points(core)
{
return (p, Source::Patterns);
}
(heuristic_points(core), Source::Heuristic)
}
/// Unrestricted break points of an English `word` ([`word_breaks_in`] with [`ProofLang::En`]).
pub fn word_breaks(word: &str) -> (Vec<usize>, Source) {
word_breaks_in(word, ProofLang::En)
}
/// Unrestricted break points of `word` in `lang` (char indices; a hyphen goes before that char),
/// handling case, apostrophes (`don't`, `’s`), surrounding punctuation and compounds (each letter
/// run is hyphenated separately). No break is ever placed next to an apostrophe or non-letter.
pub fn word_breaks_in(word: &str, lang: ProofLang) -> (Vec<usize>, Source) {
let chars: Vec<char> = word.chars().collect();
let mut out = Vec::new();
let mut src = Source::None;
let mut i = 0;
while i < chars.len() {
if !is_word_char(chars[i]) {
i += 1;
continue;
}
let mut j = i;
while j < chars.len() && is_word_char(chars[j]) {
j += 1;
}
let (mut a, mut b) = (i, j);
while a < b && is_apostrophe(chars[a]) {
a += 1;
}
while b > a && is_apostrophe(chars[b - 1]) {
b -= 1;
}
let core: Vec<char> = chars[a..b].iter().map(|&c| if is_apostrophe(c) { '\'' } else { c.to_lowercase().next().unwrap_or(c) }).collect();
let (pts, s) = core_breaks(&core, lang);
if src == Source::None {
src = s;
}
out.extend(pts.into_iter().filter(|&p| p > 0 && p < core.len() && core[p - 1] != '\'' && core[p] != '\'').map(|p| a + p));
i = j;
}
(out, src)
}
/// Allowed hyphenation points in an English `word` ([`hyphen_points_in`] with [`ProofLang::En`]).
pub fn hyphen_points(word: &str, lim: &Limits) -> Vec<usize> {
hyphen_points_in(word, lim, ProofLang::En)
}
/// Allowed hyphenation points in `word`, a word in `lang`, as char indices (a hyphen goes before
/// that char), with the paragraph limits applied to each letter run.
pub fn hyphen_points_in(word: &str, lim: &Limits, lang: ProofLang) -> Vec<usize> {
let chars: Vec<char> = word.chars().collect();
let (pts, _) = word_breaks_in(word, lang);
if pts.is_empty() {
return pts;
}
let min_before = lim.after_first.max(1);
let min_after = lim.before_last.max(1);
// Counted once, so each point's checks take constant time however long the word: letters and
// uppercase letters before each char, and the letter run (trimmed of apostrophes) of each char.
let (mut letters, mut upper) = (Vec::with_capacity(chars.len() + 1), Vec::with_capacity(chars.len() + 1));
let (mut l, mut u) = (0usize, 0usize);
for c in &chars {
letters.push(l);
upper.push(u);
if c.is_alphabetic() {
l += 1;
u += usize::from(c.is_uppercase());
}
}
letters.push(l);
upper.push(u);
let mut run_of: Vec<(usize, usize)> = vec![(0, 0); chars.len()];
let mut i = 0;
while i < chars.len() {
let mut j = i;
while chars.get(j).is_some_and(|c| is_word_char(*c)) {
j += 1;
}
let (mut a, mut b) = (i, j);
while a < b && chars.get(a).is_some_and(|c| is_apostrophe(*c)) {
a += 1;
}
while b > a && chars.get(b - 1).is_some_and(|c| is_apostrophe(*c)) {
b -= 1;
}
for r in run_of.get_mut(i..j).into_iter().flatten() {
*r = (a, b);
}
i = j.max(i + 1);
}
let count = |v: &[usize], a: usize, b: usize| v.get(b).zip(v.get(a)).map_or(0, |(y, x)| y.saturating_sub(*x));
pts.into_iter()
.filter(|&p| {
// A point lies between two letters of one run; the char after it gives the run.
let Some(&(a, b)) = run_of.get(p) else { return false };
let n = count(&letters, a, b);
if n < lim.min_word.max(2) {
return false;
}
if n > 1 && count(&upper, a, b) == n {
return false;
}
if !lim.capitalized && chars.get(a).is_some_and(|c| c.is_uppercase()) {
return false;
}
count(&letters, a, p) >= min_before && count(&letters, p, b) >= min_after
})
.collect()
}
/// `word` with `-` at every hyphenation point (tests and diagnostics).
pub fn hyphenate_word(word: &str, lim: &Limits) -> String {
hyphenate_word_in(word, lim, ProofLang::En)
}
/// `word`, a word in `lang`, with `-` at every hyphenation point (tests and diagnostics).
pub fn hyphenate_word_in(word: &str, lim: &Limits, lang: ProofLang) -> String {
let mut pts = hyphen_points_in(word, lim, lang);
pts.sort_unstable();
// Walk the points alongside the chars rather than searching them for each char.
let mut next = pts.into_iter().peekable();
let mut s = String::with_capacity(word.len() + 8);
for (i, ch) in word.chars().enumerate() {
if next.next_if_eq(&i).is_some() {
while next.next_if_eq(&i).is_some() {}
s.push('-');
}
s.push(ch);
}
s
}
// ---------- heuristic fallback (words outside the pattern alphabet) ----------
/// Letters with a diacritic that are consonants (`ł`, `ż`, `ç`, `ñ`…); any other non-ASCII letter
/// counts as a vowel (`é`, `ø`, `ą`…), and so do letters of other scripts, which keeps the
/// heuristic from breaking them.
const NON_ASCII_CONSONANTS: &str = "łńśćźżçčďđğģķļľĺňñņřŕšşșťţțžßŋðþ";
fn is_vowel(c: char) -> bool {
let l = c.to_lowercase().next().unwrap_or(c);
if l.is_ascii() { matches!(l, 'a' | 'e' | 'i' | 'o' | 'u' | 'y') } else { l.is_alphabetic() && !NON_ASCII_CONSONANTS.contains(l) }
}
fn onset_pair(a: char, b: char) -> bool {
let (a, b) = (a.to_ascii_lowercase(), b.to_ascii_lowercase());
matches!(
(a, b),
('c' | 's' | 't' | 'p' | 'w' | 'g', 'h')
| ('b' | 'c' | 'd' | 'f' | 'g' | 'k' | 'p' | 't', 'r')
| ('b' | 'c' | 'f' | 'g' | 'k' | 'p' | 's', 'l')
| ('q', 'u')
| ('s', 't' | 'p' | 'c' | 'k')
)
}
/// Syllable heuristic: V-CV ("ty-po"), VC-CV ("hap-pen"), keeping onset clusters ("gra-phy").
fn heuristic_points(c: &[char]) -> Vec<usize> {
let n = c.len();
if n < 4 || !c.iter().all(|c| c.is_alphabetic()) {
return vec![];
}
let v: Vec<bool> = c.iter().map(|&x| is_vowel(x)).collect();
// Both parts need a sounded vowel: a trailing silent `e`, `ed` or `es` doesn't count. The part
// after `i` is `c[i..end]`, `end` before that ending; computed once, so each point is O(1).
let end = |i: usize| {
if n - i >= 2 && (c.ends_with(&['e', 'd']) || c.ends_with(&['e', 's'])) {
n - 2
} else if c.ends_with(&['e']) {
n - 1
} else {
n
}
};
// UTF-8 bytes before each char, and the first vowel at or after each char.
let mut bytes = Vec::with_capacity(n + 1);
let mut acc = 0usize;
for ch in c {
bytes.push(acc);
acc += ch.len_utf8();
}
bytes.push(acc);
let mut next_vowel = vec![n; n + 1];
for i in (0..n).rev() {
next_vowel[i] = if v[i] { i } else { next_vowel[i + 1] };
}
let mut out = vec![];
let mut vowel_before = false;
for i in 1..n {
vowel_before |= v[i - 1];
let ok = if v[i - 1] && !v[i] {
(i + 1 < n && v[i + 1]) || (i + 2 < n && onset_pair(c[i], c[i + 1]) && v[i + 2])
} else if !v[i - 1] && !v[i] {
i >= 2 && v[i - 2] && i + 1 < n && v[i + 1] && !onset_pair(c[i - 1], c[i]) && c[i - 1] != c[i] || (c[i - 1] == c[i] && i >= 2 && v[i - 2])
} else {
false
};
let e = end(i);
if ok && next_vowel[i] < e && bytes[e] - bytes[i] > 1 && vowel_before {
out.push(i);
}
}
out
}
/// Polish digraphs, one sound each: never split.
fn is_digraph_pl(a: char, b: char) -> bool {
matches!((a, b), ('c', 'h' | 'z') | ('d', 'z' | 'ź' | 'ż') | ('r' | 's', 'z'))
}
/// Polish syllable heuristic, for words with letters outside the Polish pattern alphabet (`é`,
/// `ü`…): one break between each pair of syllable nuclei — after a leading sonorant (`kon-ty`,
/// `Pol-ska`) or inside a doubled consonant (`Pöp-pel`), else before the consonant cluster
/// (`sio-stra`). Digraphs (`ch`, `cz`, `dz`, `dź`, `dż`, `rz`, `sz`) stay whole, and an `i` that
/// only softens the consonant before a vowel (`nia`, `sio`) stays with that consonant.
fn heuristic_points_pl(c: &[char]) -> Vec<usize> {
let n = c.len();
if n < 4 || !c.iter().all(|c| c.is_alphabetic()) {
return vec![];
}
let v: Vec<bool> = c.iter().map(|&x| is_vowel(x)).collect();
let at = |i: usize| v.get(i).copied().unwrap_or(false);
// A syllable nucleus: a vowel, except a softening `i` (after a consonant, before a vowel).
let nucleus: Vec<bool> = (0..n).map(|i| at(i) && !(c.get(i) == Some(&'i') && i > 0 && !at(i - 1) && at(i + 1))).collect();
let nuclei: Vec<usize> = (0..n).filter(|&i| nucleus.get(i).copied().unwrap_or(false)).collect();
let mut out = vec![];
for pair in nuclei.windows(2) {
let [a, b] = pair else { continue };
let (start, end) = (a + 1, *b);
if start >= end {
continue; // adjacent vowels (na-u-ka): no break inside a vowel pair here
}
// Consonant units of the cluster: digraphs and a softening `i` join their consonant.
let mut units: Vec<usize> = Vec::new();
let mut i = start;
while i < end {
units.push(i);
i += if i + 1 < end && c.get(i).zip(c.get(i + 1)).is_some_and(|(x, y)| is_digraph_pl(*x, *y)) { 2 } else { 1 };
if i < end && c.get(i) == Some(&'i') {
i += 1;
}
}
let sonorant = |i: usize| matches!(c.get(i), Some('l' | 'ł' | 'm' | 'n' | 'ń' | 'r' | 'j'));
let first_is_single = units.get(1).is_some_and(|&second| second == start + 1);
// A doubled consonant splits (`lek-ki`, `Pöp-pel`), as does a cluster after a sonorant.
let doubled = c.get(start).is_some_and(|x| c.get(start + 1) == Some(x));
out.push(if units.len() >= 2 && first_is_single && (sonorant(start) || doubled) { start + 1 } else { start });
}
out
}
#[cfg(test)]
mod tests {
use super::*;
fn h(w: &str) -> String {
hyphenate_word(w, &Limits::default())
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn bundled_data_loads() {
assert!(Dictionary::en_us().len() > 150_000, "{}", Dictionary::en_us().len());
assert!(Patterns::en_us().len() > 1000, "{}", Patterns::en_us().len());
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn dictionary_words() {
assert_eq!(h("happen"), "hap-pen");
assert_eq!(h("typography"), "ty-pog-ra-phy");
assert_eq!(h("composition"), "com-po-si-tion");
assert_eq!(h("hyphenation"), "hy-phen-a-tion");
assert_eq!(h("international"), "in-ter-na-tion-al");
assert_eq!(h("cat"), "cat");
assert!(!h("changed").contains('-'));
assert_eq!(h("NASA"), "NASA");
assert_eq!(word_breaks("publication").1, Source::Dictionary);
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn case_apostrophes_and_punctuation() {
assert_eq!(h("Typography"), "Ty-pog-ra-phy");
assert_eq!(h("TYPOGRAPHY"), "TYPOGRAPHY");
assert_eq!(h("typography,"), "ty-pog-ra-phy,");
assert_eq!(h("“typography”"), "“ty-pog-ra-phy”");
assert_eq!(h("typography's"), "ty-pog-ra-phy's");
assert_eq!(h("typography’s"), "ty-pog-ra-phy’s");
assert_eq!(h("'typography'"), "'ty-pog-ra-phy'");
// Compounds: each part separately, never next to the hyphen.
assert_eq!(h("typography-composition"), "ty-pog-ra-phy-com-po-si-tion");
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn inflections_inherit_the_stem() {
assert_eq!(h("ripples"), "rip-ples");
assert_eq!(h("gradients"), "gra-di-ents");
assert_eq!(h("restrained"), "re-strained");
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn patterns_cover_unknown_words() {
// Not in the word list: the trained patterns decide.
let (pts, src) = word_breaks("hyperlocalization");
assert_eq!(src, Source::Patterns);
assert!(pts.len() >= 4, "{}", h("hyperlocalization"));
let (_, src) = word_breaks("zzyzx");
assert_ne!(src, Source::Dictionary);
}
#[test]
fn limits_are_respected() {
let lim = Limits { min_word: 5, after_first: 3, before_last: 3, capitalized: false };
for w in ["typography", "hyphenation", "justification", "Paragraph", "beautiful"] {
for p in hyphen_points(w, &lim) {
assert!(p >= 3 && p <= w.chars().count() - 3, "{w} {p}");
}
}
assert!(hyphen_points("Paragraph", &lim).is_empty());
let lim = Limits { min_word: 12, ..Limits::default() };
assert!(hyphen_points("typography", &lim).is_empty());
}
/// The CPU time `f` takes on this thread, in seconds. On Linux it is the scheduler's count of the
/// time the thread actually ran, so the time it waits while other work on a busy machine has the
/// CPU is left out; elsewhere, wall-clock time.
fn cpu_cost(f: &mut impl FnMut()) -> f64 {
let ran = || {
let s = std::fs::read_to_string("/proc/thread-self/schedstat").ok()?;
s.split_whitespace().next()?.parse::<u64>().ok()
};
let (t, before) = (std::time::Instant::now(), ran());
f();
match (before, ran()) {
(Some(a), Some(b)) => b.saturating_sub(a) as f64 * 1e-9,
_ => t.elapsed().as_secs_f64(),
}
}
/// How much more `big` costs than `small`, for `big` an input `scale` times the size of `small`'s:
/// (small, big, big / small), ~`scale` when linear and ~`scale`² when quadratic. Each is the
/// cheapest of several runs after a warm-up, the runs alternating so a busy spell slows both sizes
/// alike; `small` is timed `scale` times over per run, so both timings are long against the clock's
/// resolution.
fn cost_ratio(scale: usize, mut small: impl FnMut(), mut big: impl FnMut()) -> (f64, f64, f64) {
small();
big();
let mut small_scaled = || (0..scale).for_each(|_| small());
let (mut t1, mut tk) = (f64::INFINITY, f64::INFINITY);
for _ in 0..5 {
t1 = t1.min(cpu_cost(&mut small_scaled) / scale as f64);
tk = tk.min(cpu_cost(&mut big));
}
(t1, tk, tk / t1.max(1e-9))
}
/// `hyphen_points`' limit checks as first written (a rescan of the letter run per point): the
/// reference for the linear version.
fn hyphen_points_ref(word: &str, lim: &Limits) -> Vec<usize> {
let chars: Vec<char> = word.chars().collect();
let (pts, _) = word_breaks(word);
let min_before = lim.after_first.max(1);
let min_after = lim.before_last.max(1);
let run_of = |p: usize| {
let mut a = p;
while a > 0 && is_word_char(chars[a - 1]) {
a -= 1;
}
let mut b = p;
while b < chars.len() && is_word_char(chars[b]) {
b += 1;
}
while a < b && is_apostrophe(chars[a]) {
a += 1;
}
while b > a && is_apostrophe(chars[b - 1]) {
b -= 1;
}
(a, b)
};
pts.into_iter()
.filter(|&p| {
let (a, b) = run_of(p);
let run = &chars[a..b];
let letters = run.iter().filter(|c| c.is_alphabetic()).count();
if letters < lim.min_word.max(2) {
return false;
}
if letters > 1 && run.iter().filter(|c| c.is_alphabetic()).all(|c| c.is_uppercase()) {
return false;
}
if !lim.capitalized && run[0].is_uppercase() {
return false;
}
let before = chars[a..p].iter().filter(|c| c.is_alphabetic()).count();
let after = chars[p..b].iter().filter(|c| c.is_alphabetic()).count();
before >= min_before && after >= min_after
})
.collect()
}
/// `heuristic_points` as first written (the right part rebuilt per point): the reference.
fn heuristic_ref(c: &[char]) -> Vec<usize> {
let n = c.len();
if n < 4 || !c.iter().all(|c| c.is_alphabetic()) {
return vec![];
}
let v: Vec<bool> = c.iter().map(|&x| is_vowel(x)).collect();
let mut out = vec![];
for i in 1..n {
let ok = if v[i - 1] && !v[i] {
(i + 1 < n && v[i + 1]) || (i + 2 < n && onset_pair(c[i], c[i + 1]) && v[i + 2])
} else if !v[i - 1] && !v[i] {
i >= 2 && v[i - 2] && i + 1 < n && v[i + 1] && !onset_pair(c[i - 1], c[i]) && c[i - 1] != c[i]
|| (c[i - 1] == c[i] && i >= 2 && v[i - 2])
} else {
false
};
let right: String = c[i..].iter().collect();
let core = right.strip_suffix("ed").or_else(|| right.strip_suffix("es")).or_else(|| right.strip_suffix('e')).unwrap_or(&right);
if ok && core.chars().any(is_vowel) && core.len() > 1 && v[..i].iter().any(|x| *x) {
out.push(i);
}
}
out
}
#[test]
fn linear_limits_and_heuristic_match_the_reference() {
let long = "hyphenation".repeat(40);
let words = [
"",
"x",
"''",
"a'b'c",
"cat",
"happen",
"hyphenation",
"Typography's",
"TYPOGRAPHY",
"Typography",
"typography’s",
"'typography'",
"“typography”,",
"typography-composition",
"don't",
"rock'n'roll",
"naïveté",
"crème-brûlée",
"Ünïcödé",
"ÉTUDE",
"strengths",
"ee",
"bed",
"hopped",
"rhythmed",
"aeiouy",
"xxyyzz",
"bookkeeper",
"ʼtwas",
"mid-1990s",
"twenty-one",
"école",
"Ålesund",
"smørrebrød",
"hopes",
"hoped",
"taed",
"teeed",
&long,
];
for w in words {
for min_word in [2, 5, 7] {
for (after_first, before_last) in [(1, 1), (2, 2), (3, 2), (2, 4)] {
for capitalized in [true, false] {
let lim = Limits { min_word, after_first, before_last, capitalized };
assert_eq!(hyphen_points(w, &lim), hyphen_points_ref(w, &lim), "{w:?} {lim:?}");
}
}
}
let c: Vec<char> = w.to_lowercase().chars().collect();
assert_eq!(heuristic_points(&c), heuristic_ref(&c), "{w:?}");
}
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn hyphenating_a_long_word_is_linear() {
// A word of n letters against one of 8n: linear ~8x the cost, the old rescans ~64x. One in
// the pattern alphabet, one outside it (the syllable heuristic).
let pattern = |k: usize| "hyphenation".repeat(k);
let heuristic = |k: usize| "smørrebrød".repeat(k);
assert_eq!(word_breaks(&heuristic(2)).1, Source::Heuristic);
// Each timing repeats the call so the shorter one is milliseconds, well above timer noise.
let ratios: Vec<(String, f64)> = [(pattern as fn(usize) -> String, 20), (heuristic, 40)]
.into_iter()
.map(|(word, repeat)| {
let (small, big) = (word(400), word(8 * 400));
let time = |w: &str| (0..repeat).for_each(|_| drop(hyphen_points(w, &Limits::default())));
let (t1, t8, ratio) = cost_ratio(8, || time(&small), || time(&big));
let what = format!("{}…: t(n) {t1:.4} s, t(8n) {t8:.4} s, ratio {ratio:.1}", word(1).chars().take(10).collect::<String>());
eprintln!("{what}");
(what, ratio)
})
.collect();
for (what, ratio) in ratios {
assert!(ratio < 24.0, "8x the letters, not linear: {what}");
}
}
#[test]
fn russian_hyphenation_and_hyphenated_output_are_linear() {
// A Russian word breaks at every syllable: each break used to be checked against all the
// earlier ones (#3657), and the hyphenated output searched every break for each char.
let word = |k: usize| "мама".repeat(k);
assert_eq!(word_breaks_in(&word(2), ProofLang::Ru).0, [2, 4, 6]);
assert_eq!(hyphenate_word_in(&word(2), &Limits::default(), ProofLang::Ru), "ма-ма-ма-ма");
// The breaks' rescan is a cheap one per break: it takes longer words to stand out.
for (what, k, repeat, f) in [
("points", 1000, 3, (|w: &str| drop(hyphen_points_in(w, &Limits::default(), ProofLang::Ru))) as fn(&str)),
("hyphenated", 400, 20, |w: &str| drop(hyphenate_word_in(w, &Limits::default(), ProofLang::Ru))),
] {
let (small, big) = (word(k), word(8 * k));
let time = |w: &str| (0..repeat).for_each(|_| f(w));
let (t1, t8, ratio) = cost_ratio(8, || time(&small), || time(&big));
eprintln!("{what}: t(n) {t1:.4} s, t(8n) {t8:.4} s, ratio {ratio:.1}");
assert!(ratio < 24.0, "8x the letters, not linear: {what} {ratio:.1}");
}
}
#[test]
fn heuristic_fallback() {
let c: Vec<char> = "happen".chars().collect();
assert_eq!(heuristic_points(&c), vec![3]);
}
#[test]
fn heuristic_knows_consonants_with_diacritics() {
// `ł`, `ż`, `ç` are consonants (every non-ASCII letter used to count as a vowel).
assert!(!is_vowel('ł') && !is_vowel('Ż') && !is_vowel('ç') && is_vowel('é') && is_vowel('ą') && is_vowel('ø'));
let c: Vec<char> = "façade".chars().collect();
assert_eq!(heuristic_points(&c), vec![2]);
}
fn pl(w: &str) -> String {
hyphenate_word_in(w, &Limits::default(), ProofLang::Pl)
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn polish_patterns_load() {
let p = Patterns::pl();
assert!(p.len() > 4000, "{}", p.len());
assert_eq!(p.exception_count(), 20);
assert_eq!(p.exception("nigdy"), Some(&[2][..]));
}
fn pt(w: &str) -> String {
hyphenate_word_in(w, &Limits::default(), ProofLang::Pt)
}
fn it(w: &str) -> String {
hyphenate_word_in(w, &Limits::default(), ProofLang::It)
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn italian_patterns_load() {
let p = Patterns::it();
assert!(p.len() > 300, "{}", p.len());
// The file has no exception words.
assert_eq!(p.exception_count(), 0);
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn italian_words_hyphenate_by_italian_rules() {
for (w, want) in [
("famiglia", "fa-mi-glia"),
("finestra", "fi-ne-stra"),
("bicicletta", "bi-ci-clet-ta"),
("passeggiata", "pas-seg-gia-ta"),
("caffè", "caf-fè"),
("tavolo", "ta-vo-lo"),
("bambino", "bam-bi-no"),
("scuola", "scuo-la"),
("notte", "not-te"),
("giorno", "gior-no"),
("acqua", "ac-qua"),
// Under five letters (`Limits::min_word`): no break.
("casa", "casa"),
("pane", "pane"),
("libro", "li-bro"),
("re", "re"),
("più", "più"),
] {
assert_eq!(it(w), want, "{w}");
}
assert_eq!(word_breaks_in("famiglia", ProofLang::It).1, Source::Patterns);
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn portuguese_patterns_load() {
let p = Patterns::pt();
assert!(p.len() > 300, "{}", p.len());
// `hard-ware`, `soft-ware` — and the file's last pattern line, a bare `1-`, which the
// reader counts as a (never-matching) exception because it contains a hyphen.
assert_eq!(p.exception_count(), 3);
assert_eq!(p.exception("hardware"), Some(&[4][..]));
assert_eq!(p.exception("software"), Some(&[4][..]));
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn portuguese_words_hyphenate_by_portuguese_rules() {
for (w, want) in [
("problema", "pro-ble-ma"),
("coração", "co-ra-ção"),
("paralelepípedo", "pa-ra-le-le-pí-pe-do"),
("computador", "com-pu-ta-dor"),
("escrever", "es-cre-ver"),
("saudade", "sau-da-de"),
("extraordinário", "ex-tra-or-di-ná-rio"),
("automático", "au-to-má-ti-co"),
// One syllable: no break.
("flor", "flor"),
("bem", "bem"),
] {
assert_eq!(pt(w), want, "{w}");
}
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn polish_words_hyphenate_by_polish_rules() {
for (w, want) in [
("konstytucja", "kon-sty-tu-cja"),
("konstytucyjnego", "kon-sty-tu-cyj-ne-go"),
("Rzeczpospolita", "Rzecz-po-spo-li-ta"),
("przedsiębiorstwo", "przed-się-bior-stwo"),
("pięćdziesięciu", "pięć-dzie-się-ciu"),
("dziewczyna", "dziew-czy-na"),
("książka", "książ-ka"),
("Warszawa", "War-sza-wa"),
("szczęście", "szczę-ście"),
("rzeczywistość", "rze-czy-wi-stość"),
("najpiękniejszy", "naj-pięk-niej-szy"),
("wyobraźnia", "wy-obraź-nia"),
("gżegżółka", "gżeg-żół-ka"),
// One syllable, or a single letter left over: no break.
("chrząszcz", "chrząszcz"),
("źdźbło", "źdźbło"),
("obiad", "obiad"),
// The pattern file's exception words.
("nigdy", "ni-gdy"),
("przynajmniej", "przy-naj-mniej"),
("skądinąd", "skąd-inąd"),
// Compounds: each part, never next to the hyphen.
("biało-czerwony", "bia-ło-czer-wo-ny"),
] {
assert_eq!(pl(w), want, "{w}");
}
assert_eq!(word_breaks_in("nigdy", ProofLang::Pl).1, Source::Dictionary);
assert_eq!(word_breaks_in("konstytucja", ProofLang::Pl).1, Source::Patterns);
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn polish_heuristic_for_foreign_letters() {
// Letters outside the Polish alphabet: the Polish syllable heuristic, digraphs kept whole.
assert_eq!(word_breaks_in("Pöppelmann", ProofLang::Pl).1, Source::Heuristic);
assert_eq!(pl("Pöppelmann"), "Pöp-pel-mann");
assert_eq!(pl("Müllerowi"), "Mül-le-ro-wi");
let c: Vec<char> = "kawiarnié".chars().collect();
assert_eq!(heuristic_points_pl(&c), vec![2, 6], "ka-wiar-nié: `wi` and `ni` soften, `rn` splits after the sonorant");
let c: Vec<char> = "szczérze".chars().collect();
assert_eq!(heuristic_points_pl(&c), vec![5], "szczé-rze: `rz` stays whole");
}
#[test]
fn languages_without_patterns_keep_the_english_ones() {
for w in ["hyphenation", "typography", "internationalization"] {
for lang in [ProofLang::Other, ProofLang::Cs, ProofLang::Id, ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy] {
assert_eq!(hyphen_points_in(w, &Limits::default(), lang), hyphen_points(w, &Limits::default()), "{w}");
}
}
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn german_patterns_load() {
let p = Patterns::de();
assert_eq!(p.len(), 36_709);
// The file has no exception words, and its `\message{…}` line adds none.
assert_eq!(p.exception_count(), 0);
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn german_words_follow_the_german_patterns() {
let de = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::De);
assert_eq!(de("Silbentrennung"), "Sil-ben-tren-nung");
assert_eq!(de("unterliegende"), "un-ter-lie-gen-de");
assert_eq!(de("öffentlich-rechtlicher"), "öf-fent-lich-recht-li-cher");
assert_eq!(de("Begründung"), "Be-grün-dung");
assert_eq!(de("Zucker"), "Zu-cker");
// Swiss spelling: `ss` splits like any double consonant.
assert_eq!(de("Strasse"), "Stras-se");
assert_eq!(word_breaks_in("Unterlagen", ProofLang::De).1, Source::Patterns);
}
#[test]
fn bulgarian_words_follow_the_bulgarian_rules() {
let bg = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::Bg);
assert_eq!(bg("сричкопренасяне"), "срич-ко-пре-на-ся-не");
assert_eq!(bg("Таблица"), "Та-бли-ца");
// A single letter is never left on a line (the paragraph limits): България, not Бъл-га-ри-я.
assert_eq!(bg("България"), "Бъл-га-рия");
assert_eq!(bg("„шофьор“,"), "„шо-фьор“,");
assert_eq!(word_breaks_in("таблица", ProofLang::Bg).1, Source::Rules);
// Latin words in a Bulgarian run are hyphenated as English.
for w in ["publication", "WordCraft"] {
assert_eq!(hyphen_points_in(w, &Limits::default(), ProofLang::Bg), hyphen_points(w, &Limits::default()), "{w}");
}
}
}
crates/proof/src/id/grammar.rs
+105
View File
@@ -0,0 +1,105 @@
//! Generic repeated-word and spacing checks with Indonesian messages. No Indonesian-specific
//! grammar rules are inferred. A word written twice with a space is flagged; since standard
//! Indonesian spelling (PUEBI) writes reduplication with a hyphen (`anak-anak`), the hyphenated
//! form is offered first, then the single word.
use crate::{Issue, IssueKind, ProofLang, words};
fn issue(start: usize, end: usize, message: String, suggestions: Vec<String>) -> Issue {
Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::Id }
}
pub fn check(text: &str) -> Vec<Issue> {
let mut issues = Vec::new();
repeated_words(text, &words(text), &mut issues);
spaces(text, &mut issues);
issues.sort_by_key(|i| (i.start, i.end));
issues
}
fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
for pair in ws.windows(2) {
let [(a0, b0), (a1, b1)] = pair else { continue };
let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue };
if !gap.is_empty() && gap.chars().all(char::is_whitespace) && w0.chars().all(char::is_alphabetic) && w0.to_lowercase() == w1.to_lowercase() {
v.push(issue(*a0, *b1, format!("Kata berulang: “{w1}”"), vec![format!("{w0}-{w1}"), w0.to_string()]));
}
}
}
fn spaces(text: &str, v: &mut Vec<Issue>) {
for (i, c) in text.char_indices() {
if c != ' ' {
continue;
}
let rest = text.get(i + 1..).unwrap_or("");
let mut it = rest.chars();
let Some(p) = it.next() else { continue };
let after = it.next();
// A ratio/score keeps the spaces in digit-space-colon-space-digit.
let ratio = p == ':'
&& text.get(..i).and_then(|before| before.chars().next_back()).is_some_and(|c| c.is_ascii_digit())
&& after == Some(' ')
&& it.next().is_some_and(|c| c.is_ascii_digit());
let flag = !ratio
&& match p {
',' => true,
// `;)` and `;-)` are emoticons.
';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')),
// ` ...` is an ellipsis, ` .5` a number.
'.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
// ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space.
':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')),
_ => false,
};
if flag {
v.push(issue(i, i + 1 + p.len_utf8(), "Spasi sebelum tanda baca".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Spasi ganda".into(), vec![" ".into()]));
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn repeated_words_and_spaces_have_indonesian_messages() {
let text = "Anak anak , sekolah.";
let issues = check(text);
assert_eq!(issues.len(), 3, "{issues:?}");
assert_eq!(issues[0].message, "Kata berulang: “anak”");
assert_eq!(issues[0].suggestions, ["Anak-anak", "Anak"]);
assert!(issues.iter().any(|i| i.message == "Spasi sebelum tanda baca" && &text[i.start..i.end] == " ,"));
assert!(issues.iter().any(|i| i.message == "Spasi ganda"));
assert!(issues.iter().all(|i| i.lang == ProofLang::Id && i.kind == IssueKind::Grammar));
for text in ["anak-anak bermain", "rumah. Sekolah", "Dr. Budi", "Anak, anak!", "Senyum :) Angka .5 dan titik ...", "kupu-kupu"] {
assert!(check(text).is_empty(), "{text}: {:?}", check(text));
}
}
#[test]
fn ratios_and_scores_keep_their_spaces() {
for text in ["3 : 1", "Skor 2 : 1.", "Rasio 1 : 100."] {
assert!(check(text).is_empty(), "{text}: {:?}", check(text));
}
for text in ["Kata : lain", "3 : kata", "3 :"] {
assert!(check(text).iter().any(|i| i.message == "Spasi sebelum tanda baca"), "{text}");
}
}
proptest::proptest! {
#[test]
fn never_panics_and_ranges_are_valid(s in "\\PC{0,80}") {
for issue in check(&s) {
proptest::prop_assert!(issue.start <= issue.end && issue.end <= s.len());
proptest::prop_assert!(s.is_char_boundary(issue.start) && s.is_char_boundary(issue.end));
}
}
}
}
crates/proof/src/id/mod.rs
+210
View File
@@ -0,0 +1,210 @@
//! Indonesian proofing using the installed Hunspell `id_ID` dictionary (never bundled; Debian and
//! Ubuntu ship it as `hunspell-id`, LibreOffice as its `dict-id` extension). Suggestions use the
//! dictionary's own MAP/REP/TRY/KEY tables. Grammar is limited to generic repeated-word and
//! spacing checks with Indonesian messages; hyphenation keeps the English fallback (no Indonesian
//! patterns are bundled).
pub mod grammar;
mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// File names of an Indonesian Hunspell dictionary (`in` is the old ISO 639 code, still used by
/// Java locales).
const NAMES: &[&str] = &["id_ID", "id-ID", "id", "in_ID"];
/// The Indonesian dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Indonesian");
/// The Indonesian dictionary: the installed Hunspell `id_ID` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Indonesian spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
DICT.get()
}
/// Use this Indonesian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
DICT.install(aff, dic)
}
/// Is an Indonesian dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
DICT.has_dictionary()
}
/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number
/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own.
const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"];
/// Is `word` spelled correctly in Indonesian? Numbers, single letters, short words in capitals
/// (acronyms: `DPR`, `KTP`), words with digits and words in other scripts are accepted, and so
/// is every word when no Indonesian dictionary is installed. A hyphenated word is correct when
/// the dictionary has it whole (`kupu-kupu`, `sayur-mayur`) or has each part (`anak-anak`,
/// `rumah-rumah`); a hyphenated clitic (`KTP-nya`, `se-Indonesia`) counts as known.
pub fn is_correct(word: &str) -> bool {
is_correct_with(&dictionary(), word)
}
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off,
/// short words in capitals are checked against the dictionary too, and with Ignore words that
/// contain numbers off, so are words with digits.
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
check(&dictionary(), word, opts)
}
/// [`is_correct`] against a given dictionary (an empty one accepts everything).
pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
check(d, word, &crate::Options::default())
}
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
};
if w.contains('-') {
return known(d, w)
|| w.split('-').all(|p| {
p.chars().count() <= 1
|| CLITICS.iter().any(|c| p.eq_ignore_ascii_case(c))
|| (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= 6)
|| known(d, p)
});
}
known(d, w)
}
/// Is one word in the dictionary or the user dictionary?
fn known(d: &AffixDict, w: &str) -> bool {
d.check(w) || crate::in_user_dictionary(&w.to_lowercase())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn dictionary_lookup_accepts_indonesian_file_names() {
let dir = std::env::temp_dir().join(format!("wordcraft-id-aliases-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
for name in ["id_ID", "id-ID", "id", "in_ID"] {
std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap();
std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap();
let (aff, dic) = crate::hunspell::find_in(&dir, NAMES).unwrap();
let d = crate::hunspell::load(&aff, &dic).unwrap();
assert!(is_correct_with(&d, "rumah"));
std::fs::remove_file(aff).unwrap();
std::fs::remove_file(dic).unwrap();
}
std::fs::remove_dir(dir).unwrap();
}
#[test]
fn indonesian_words_affixes_case_and_hyphens() {
test_dict::install();
let d = dictionary();
assert!(has_dictionary());
assert_eq!(d.len(), 20);
assert!(d.rule_count() > 0);
for w in [
"rumah",
"makan",
"makanan",
"dimakan",
"membaca",
"bacaan",
"dibaca",
"Jakarta",
"Rumah",
"SEKOLAH",
"anak-anak",
"rumah-rumah",
"kupu-kupu",
"KTP-nya",
"se-Indonesia",
"Jakarta-lah",
"DPR",
] {
assert!(is_correct(w), "{w}");
}
for w in ["rumha", "makanann", "skolah", "jakarta", "kupu", "anak-anka", "mAkan", "bukunya-xyz"] {
assert!(!is_correct(w), "{w}");
}
}
#[test]
fn forbidden_words_are_rejected_in_every_case() {
test_dict::install();
for w in ["zqterlarang", "Zqterlarang", "ZQTERLARANG"] {
assert!(!is_correct(w), "{w}");
}
}
#[test]
fn missing_dictionary_and_ignored_inputs_are_quiet() {
let empty = AffixDict::default();
assert!(is_correct_with(&empty, "rumha"));
assert!(install_dictionary("FLAG long\n", "1\nrumah\n").is_err());
let d = AffixDict::from_hunspell(test_dict::AFF, test_dict::DIC, &[]).unwrap();
for w in ["", "a", "2026", "RI", "ke-2", "日本語", "-", "--"] {
assert!(is_correct_with(&d, w), "{w}");
}
}
#[test]
fn user_dictionary_applies() {
test_dict::install();
assert!(!is_correct("zqkatasendiri"));
crate::add_word("Zqkatasendiri");
assert!(is_correct("zqkatasendiri"));
}
#[test]
fn indonesian_dispatch_messages_options_and_mixed_spans() {
test_dict::install();
assert!(crate::has_dictionary(Some("id-ID")));
assert!(crate::is_correct_for("rumah", Some("ind")));
let text = "Rumah sekolha";
let issues = crate::check_text(text, &[(0..text.len(), crate::ProofLang::Id)]);
assert_eq!(issues.len(), 1, "{issues:?}");
assert_eq!(&text[issues[0].start..issues[0].end], "sekolha");
assert_eq!(issues[0].message, "Kemungkinan salah eja");
assert_eq!(issues[0].lang, crate::ProofLang::Id);
assert!(crate::suggest_in("sekolha", 8, crate::ProofLang::Id).contains(&"sekolah".into()));
let text = "Rumah this is is wrong";
let issues = crate::check_text(text, &[(0..5, crate::ProofLang::Id), (6..text.len(), crate::ProofLang::En)]);
assert!(issues.iter().any(|i| i.lang == crate::ProofLang::En && i.message.starts_with("Repeated word")));
assert!(issues.iter().all(|i| i.start >= 6));
let opts = crate::Options { ignore_uppercase: false, ..Default::default() };
assert!(!crate::is_correct_in_with("ZQTERLARANG", crate::ProofLang::Id, &opts));
}
/// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Indonesian checks
/// those words against the dictionary instead of accepting them.
#[test]
fn proofing_options_reach_the_indonesian_check() {
test_dict::install();
let lang = crate::ProofLang::Id;
let defaults = crate::Options::default();
let check_caps = crate::Options { ignore_uppercase: false, ..defaults };
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
assert!(crate::is_correct_in_with("RUMHA", lang, &defaults), "short capitals are acronyms by default");
assert!(!crate::is_correct_in_with("RUMHA", lang, &check_caps));
assert!(crate::is_correct_in_with("RUMAH", lang, &check_caps));
assert!(crate::is_correct_in_with("RUMHA-nya", lang, &defaults));
assert!(!crate::is_correct_in_with("RUMHA-nya", lang, &check_caps), "a part in capitals is checked too");
assert!(crate::is_correct_in_with("rumha2", lang, &defaults));
assert!(!crate::is_correct_in_with("rumha2", lang, &check_digits));
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
assert!(is_correct("RUMHA"), "direct checks keep the defaults");
}
}
crates/proof/src/id/suggest.rs
+56
View File
@@ -0,0 +1,56 @@
//! Indonesian suggestions: the shared ranking ([`crate::affix_suggest`]: case, MAP, REP, missing
//! spaces, KEY, TRY edits and a stem scan); every table comes from the installed `id_ID`
//! dictionary, so nothing Indonesian is hard-coded here.
use crate::affix::AffixDict;
/// Spelling suggestions for an Indonesian word, best first, at most `max`, in the word's case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
suggest_in(&super::dictionary(), word, max)
}
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
crate::affix_suggest::suggest_in(d, word, max, &crate::affix_suggest::Tuning::DEFAULT)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn indonesian_suggestions_use_runtime_tables_and_case() {
crate::id::test_dict::install();
for (word, want) in [
("rumha", "rumah"),
("sekolha", "sekolah"),
("Sekola", "Sekolah"),
("jakarta", "Jakarta"),
("makanna", "makanan"),
("djalan", "jalan"),
("dsn", "dan"),
("sayadan", "saya dan"),
("membcaa", "membaca"),
] {
assert!(suggest(word, 8).contains(&want.into()), "{word}: {:?}", suggest(word, 8));
}
}
#[test]
fn suggestions_stay_bounded_and_skip_forbidden_words() {
crate::id::test_dict::install();
for w in ["", "dua kata", &"a".repeat(41)] {
assert!(suggest(w, 4).is_empty());
}
assert!(suggest("rumha", 0).is_empty());
assert!(suggest_in(&AffixDict::default(), "rumha", 4).is_empty());
assert!(suggest("rumah", 4).len() <= 4);
assert!(!suggest("rumah", 4).contains(&"rumah".into()));
assert!(!suggest("zqterlarag", 8).contains(&"zqterlarang".into()));
}
#[test]
fn nosuggest_stems_and_forms_are_never_offered() {
crate::affix::tests::assert_nosuggest_is_honoured(suggest_in);
}
}
crates/proof/src/id/test_dict.rs
+49
View File
@@ -0,0 +1,49 @@
//! Hand-written synthetic Indonesian test data. No words, rules or suggestion tables were copied
//! from a system dictionary. This is a small fixture, not a redistributable Indonesian dictionary.
pub const AFF: &str = "SET UTF-8
FLAG UTF-8
FORBIDDENWORD F
TRY aniekrumgtslhbdopy
KEY qw|as|zx
MAP 1
MAP eé
REP 2
REP oe u
REP dj j
SFX A Y 1 # synthetic -an
SFX A 0 an .
PFX D Y 1
PFX D 0 di .
PFX M Y 1
PFX M 0 mem b
";
pub const DIC: &str = "20
rumah
makan/AD
baca/ADM
sekolah
anak
buku
kupu-kupu
Jakarta
Indonesia
saya
kamu
dan
yang
ini
itu
pergi
jalan/A
besar
kecil
zqterlarang/F
";
pub fn install() {
if let Err(e) = super::install_dictionary(AFF, DIC) {
panic!("test dictionary: {e}");
}
}
crates/proof/src/it/grammar.rs
+472
View File
@@ -0,0 +1,472 @@
//! Italian grammar and punctuation checks: cheap, rule-based and tuned to stay quiet when
//! unsure (a missed mistake costs less than a wrong mark on correct text). Messages are in
//! Italian.
//!
//! - **Repeated word** (`la la`), except words Italian doubles on purpose (`piano piano`,
//! `sì sì`, interjections).
//! - **Space before punctuation** (`bene ,`) and **double spaces**; emoticons (` :)`) and
//! ellipses are left alone.
//! - **Capital letter at the start of a sentence**, knowing Italian abbreviations (`ecc.`,
//! `dott.`, `prof.`, `sig.`), initials, ordinal numbers (`1. posto`) and ellipses.
//! - **Doubled punctuation** (`,,`, `..`, `;;`).
//! - **`po'`**: `un pò` and `un po` → `un po'` (a truncation of `poco`: apostrophe, no accent).
//! - **`qual è`**: `qual'è`, `qual'era` → `qual è`, `qual era` (a truncation, never elided).
//! - **`È`**: `E'` written for the verb → `È`.
//! - **`un` / `un'`**: the apostrophe only before a feminine word. `un'altro`, `un'anno` →
//! `un altro`, `un anno` (a masculine word in `-o`; `auto` and `eco` are feminine), and
//! `un altra`, `un amica` → `un'altra`, `un'amica` (a word in `-a`, except the masculine and
//! common-gender ones: `-ista`, `-ita`, `-ota`, `-cida`, `-ma`, `-eta`, `atleta`, `acrobata`…).
//! The same for `nessun`, `ciascun`, `qualcun` and `alcun`.
//!
//! No comma rule and no English rule (such as `a`/`an`) applies to Italian text.
use crate::{Issue, IssueKind, ProofLang, words};
/// Words Italian repeats on purpose: emphatic doubling, interjections, onomatopoeia.
const ALLOWED_REPEATS: &[&str] = &[
"sì", "no", "piano", "bene", "male", "ecco", "oh", "ah", "eh", "mh", "bah", "ehi", "urrà", "evviva", "ciao", "addio", "adagio", "forte", "lento",
"chiaro", "pian", "poco", "quasi", "presto", "subito", "zitto", "via", "così", "tanto", "molto", "pure", "dai", "su", "avanti", "basta", "aiuto",
];
/// Indefinite articles (and the like) that elide to `un'` before a feminine word only.
const UN_FORMS: &[&str] = &["un", "nessun", "ciascun", "qualcun", "alcun"];
/// Feminine words in `-o` that start with a vowel (`un'auto`, `un'eco`).
const FEMININE_IN_O: &[&str] = &["auto", "autoradio", "eco", "ecografo"];
/// Endings of masculine or common-gender words in `-a` (`un artista`, `un idiota`, `un enigma`).
const MASCULINE_A_ENDINGS: &[&str] = &["ista", "ita", "ota", "cida", "ma", "eta", "nauta", "iatra", "arca"];
/// Words in `-a` after `un` that aren't feminine nouns: masculine or common-gender ones that fit
/// no ending above, and verbs and pronouns (`un ha`, `un una`).
const NOT_FEMININE_IN_A: &[&str] = &["acrobata", "autodidatta", "alfa", "omega", "ha", "una"];
/// Abbreviations written with a final period after which a sentence goes on in lower case
/// (lower case, without the period).
const ABBREVIATIONS: &[&str] = &[
"ecc",
"es",
"dott",
"prof",
"profssa",
"ing",
"avv",
"sig",
"sigg",
"sigra",
"sigr",
"geom",
"rag",
"dottssa",
"avvoc",
"pag",
"pagg",
"tel",
"art",
"artt",
"fig",
"figg",
"vol",
"voll",
"ed",
"sez",
"cap",
"capp",
"par",
"n",
"num",
"vs",
"cfr",
"cfrt",
"op",
"cit",
"ibid",
"sez",
"all",
"tav",
"pag",
"ditta",
"soc",
"spa",
"srl",
"on",
"sen",
"spett",
"eg",
"ca",
"v",
"vv",
"sec",
"min",
"max",
"lun",
"mar",
"mer",
"gio",
"ven",
"sab",
"dom",
"gen",
"feb",
"mar",
"apr",
"mag",
"giu",
"lug",
"ago",
"set",
"ott",
"nov",
"dic",
"genn",
"febb",
"sett",
"dottor",
"professor",
"cav",
"commend",
"mons",
"fr",
"suor",
"s",
"ss",
"b",
"beato",
];
fn issue(start: usize, end: usize, message: String, suggestions: Vec<String>) -> Issue {
Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::It }
}
/// Is `prev`, before a period, an abbreviation, an initial, a number or a Roman numeral (after
/// which a lower-case word doesn't start a sentence)?
fn abbreviation_or_number(prev: &str) -> bool {
prev.chars().count() <= 1
|| prev.chars().any(|c| c.is_ascii_digit())
|| (prev.chars().all(|c| matches!(c, 'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M')))
|| ABBREVIATIONS.contains(&prev.to_lowercase().as_str())
}
/// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after
/// an abbreviation), then space, with nothing but quotes and brackets around?
fn ends_sentence(gap: &str, prev: &str) -> bool {
let t = gap.trim_start_matches(['"', '”', '’', '\'', ')', '»', ']', '\u{1}']);
let Some(end) = t.chars().next() else { return false };
if !matches!(end, '.' | '!' | '?') || t.starts_with("..") {
return false;
}
if end == '.' && abbreviation_or_number(prev) {
return false;
}
let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(['"', '”', '’', '\'', ')', '»', ']']);
rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—'))
}
fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
for pair in ws.windows(2) {
let [(a0, b0), (a1, b1)] = pair else { continue };
let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue };
if !gap.is_empty()
&& gap.chars().all(char::is_whitespace)
&& w0.chars().all(char::is_alphabetic)
&& w0.to_lowercase() == w1.to_lowercase()
&& !ALLOWED_REPEATS.contains(&w0.to_lowercase().as_str())
{
v.push(issue(*a0, *b1, format!("Parola ripetuta: \"{w1}\""), vec![w0.to_string()]));
}
}
}
fn spaces(text: &str, v: &mut Vec<Issue>) {
for (i, c) in text.char_indices() {
if c != ' ' {
continue;
}
let rest = text.get(i + 1..).unwrap_or("");
let mut it = rest.chars();
let Some(p) = it.next() else { continue };
let after = it.next();
let flag = match p {
',' => true,
// `;)` and `;-)` are emoticons.
';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')),
// ` ...` is an ellipsis, ` .5` a number.
'.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
// ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space.
':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')),
_ => false,
};
if flag {
v.push(issue(i, i + 1 + p.len_utf8(), "Spazio prima della punteggiatura".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Spazio doppio".into(), vec![" ".into()]));
}
}
fn sentence_capitals(text: &str, ws: &[(usize, usize)], capital_first: bool, v: &mut Vec<Issue>) {
let mut after_end = capital_first;
for (a, b) in ws {
let Some(w) = text.get(*a..*b) else { continue };
if after_end && w.chars().next().is_some_and(char::is_lowercase) && w.chars().all(char::is_alphabetic) {
let fixed: String = w.chars().next().map(|f| f.to_uppercase().chain(w.chars().skip(1)).collect()).unwrap_or_default();
v.push(issue(*a, *b, "Maiuscola a inizio frase".into(), vec![fixed]));
}
after_end = ends_sentence(text.get(*b..).unwrap_or(""), w);
}
}
fn doubled_punctuation(text: &str, v: &mut Vec<Issue>) {
for pat in [",,", "..", ";;", "::", "??", "!!"] {
for (i, _) in text.match_indices(pat) {
if pat == ".." && text.get(i + 2..).is_some_and(|r| r.starts_with('.')) {
continue; // ellipsis
}
v.push(issue(i, i + pat.len(), "Punteggiatura doppia".into(), vec![pat[..1].into()]));
}
}
}
/// Is `c` an apostrophe (straight or typographic)?
fn is_apostrophe(c: char) -> bool {
matches!(c, '\'' | '’')
}
/// `pò` (never accented) and `po` without its apostrophe → `po'`. `Po` (the river) is left alone.
fn po(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
for (a, b) in ws {
let Some(w) = text.get(*a..*b) else { continue };
let apostrophe = text.get(*b..).and_then(|r| r.chars().next()).is_some_and(is_apostrophe);
let fix = match w {
"pò" => "po'",
"Pò" => "Po'",
"po" if !apostrophe => "po'",
_ => continue,
};
v.push(issue(*a, *b, "Si scrive «po'», con l'apostrofo e senza accento".into(), vec![fix.into()]));
}
}
/// `qual'è` → `qual è` (any case): the fix when `word` is `qual` + apostrophe + `è`/`era`.
pub(crate) fn qual_elided(word: &str) -> Option<String> {
let head_end = word.char_indices().take(4).last().map(|(i, c)| i + c.len_utf8())?;
let (head, rest) = (word.get(..head_end)?, word.get(head_end..)?);
let tail = rest.strip_prefix(is_apostrophe)?;
let tail_ok = matches!(tail.to_lowercase().as_str(), "è" | "era");
(head.eq_ignore_ascii_case("qual") && tail_ok).then(|| format!("{head} {tail}"))
}
fn qual_e(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
for (a, b) in ws {
if let Some(fix) = text.get(*a..*b).and_then(qual_elided) {
v.push(issue(*a, *b, "«Qual è» si scrive senza apostrofo".into(), vec![fix]));
}
}
}
/// `E'` for the verb → `È`. A quoted letter (`la lettera 'E'`) is left alone.
fn e_apostrophe(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
for (a, b) in ws {
if text.get(*a..*b) != Some("E") {
continue;
}
let mut after = text.get(*b..).unwrap_or("").chars();
let Some(ap) = after.next().filter(|&c| is_apostrophe(c)) else { continue };
let ends = after.next().is_none_or(char::is_whitespace);
let quoted = text.get(..*a).and_then(|t| t.chars().next_back()).is_some_and(|c| is_apostrophe(c) || c == '‘');
if ends && !quoted {
v.push(issue(*a, *b + ap.len_utf8(), "«È» si scrive con l'accento, non con l'apostrofo".into(), vec!["È".into()]));
}
}
}
/// Does `w` (lower case) start with a vowel or `h`, as the words `un'` elides before?
fn starts_with_vowel(w: &str) -> bool {
w.chars().next().is_some_and(|c| "aeiouhàèéìòóù".contains(c))
}
/// Is `w` (lower case) a feminine word in `-a` (`altra`, `amica`, `ora`), as far as the ending
/// tells?
fn feminine_in_a(w: &str) -> bool {
w.chars().count() >= 2 && w.ends_with('a') && !MASCULINE_A_ENDINGS.iter().any(|e| w.ends_with(e)) && !NOT_FEMININE_IN_A.contains(&w)
}
/// `un'` before a masculine word (`un'altro` → `un altro`), and `un` without the apostrophe
/// before a feminine one (`un altra` → `un'altra`). Only lower-case words: a name's gender isn't
/// in its ending.
fn un_apostrophe(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
for (a, b) in ws {
let Some(w) = text.get(*a..*b) else { continue };
let Some((head, tail)) = w.split_once(is_apostrophe) else { continue };
if UN_FORMS.contains(&head.to_lowercase().as_str())
&& tail.chars().all(|c| c.is_alphabetic() && c.is_lowercase())
&& starts_with_vowel(tail)
&& tail.ends_with('o')
&& !FEMININE_IN_O.contains(&tail)
{
v.push(issue(*a, *b, format!("Davanti a un nome maschile «{head}» non vuole l'apostrofo"), vec![format!("{head} {tail}")]));
}
}
for pair in ws.windows(2) {
let [(a0, b0), (a1, b1)] = pair else { continue };
let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue };
if gap == " "
&& UN_FORMS.contains(&w0.to_lowercase().as_str())
&& w1.chars().all(|c| c.is_alphabetic() && c.is_lowercase())
&& starts_with_vowel(w1)
&& feminine_in_a(w1)
{
v.push(issue(*a0, *b1, format!("Davanti a un nome femminile si scrive «{w0}'», con l'apostrofo"), vec![format!("{w0}'{w1}")]));
}
}
}
/// Grammar issues in a paragraph of Italian text, in order.
pub fn check(text: &str) -> Vec<Issue> {
check_with(text, true)
}
/// [`check`] where `capital_first` says whether the text's first word must start a sentence with a capital
/// (false for a list item, which may be a fragment).
pub fn check_with(text: &str, capital_first: bool) -> Vec<Issue> {
let ws: Vec<(usize, usize)> = words(text);
let mut v = Vec::new();
repeated_words(text, &ws, &mut v);
spaces(text, &mut v);
sentence_capitals(text, &ws, capital_first, &mut v);
doubled_punctuation(text, &mut v);
po(text, &ws, &mut v);
qual_e(text, &ws, &mut v);
e_apostrophe(text, &ws, &mut v);
un_apostrophe(text, &ws, &mut v);
v.sort_by_key(|i| (i.start, i.end));
v
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn italian_grammar_flags() {
test_dict_install();
let t = "questo questo è un test , fatto male.. Vedi dott. rossi";
let v = check(t);
let msgs: Vec<&str> = v.iter().map(|i| i.message.as_str()).collect();
assert!(msgs.iter().any(|m| m.contains("ripetuta")), "{msgs:?}");
assert!(msgs.iter().any(|m| m.contains("prima della punteggiatura")), "{msgs:?}");
assert!(msgs.iter().any(|m| m.contains("doppio")), "{msgs:?}");
assert!(msgs.iter().any(|m| m.contains("doppia")), "{msgs:?}");
assert!(v.iter().all(|i| i.lang == ProofLang::It), "{v:?}");
}
#[test]
fn italian_grammar_stays_quiet() {
test_dict_install();
assert!(
check("Vado a Roma. Il dott. Rossi è già lì... piano piano!").is_empty(),
"{:?}",
check("Vado a Roma. Il dott. Rossi è già lì... piano piano!")
);
assert!(check("Sì sì, bene bene :)").is_empty());
}
/// The fixes the rules offer for `t`, in order.
fn fixes(t: &str) -> Vec<(String, String)> {
check(t).into_iter().map(|i| (t.get(i.start..i.end).unwrap_or("").to_string(), i.suggestions.join("|"))).collect()
}
fn pair(a: &str, b: &str) -> (String, String) {
(a.to_string(), b.to_string())
}
#[test]
fn po_takes_an_apostrophe() {
assert_eq!(fixes("Dammi un pò di pane"), [pair("pò", "po'")]);
assert_eq!(fixes("Dammi un po di pane"), [pair("po", "po'")]);
assert_eq!(fixes("Pò di più"), [pair("Pò", "Po'")]);
assert!(fixes("Dammi un po' di pane, un po’ d'acqua").is_empty());
assert!(fixes("Il Po scorre a Torino").is_empty(), "the river");
}
#[test]
fn qual_e_has_no_apostrophe() {
assert_eq!(fixes("Dimmi qual'è il problema"), [pair("qual'è", "qual è")]);
assert_eq!(fixes("Qual’era la strada?"), [pair("Qual’era", "Qual era")]);
assert!(fixes("Qual è il problema? Quale era?").is_empty());
assert!(fixes("Quale'è").is_empty(), "not the truncation");
assert_eq!(qual_elided("qual'è").as_deref(), Some("qual è"));
assert_eq!(qual_elided("qua"), None);
assert_eq!(qual_elided(""), None);
}
#[test]
fn e_apostrophe_becomes_e_grave() {
assert_eq!(fixes("E' vero"), [pair("E'", "È")]);
assert_eq!(fixes("Lo so. E’ tardi"), [pair("E’", "È")]);
assert!(fixes("È vero. E poi?").is_empty());
assert!(fixes("La lettera 'E' è una vocale").is_empty(), "a quoted letter");
}
#[test]
fn un_apostrophe_follows_the_gender() {
assert_eq!(fixes("Ho visto un'altro amico"), [pair("un'altro", "un altro")]);
assert_eq!(fixes("È passato un’anno"), [pair("un’anno", "un anno")]);
assert_eq!(fixes("Prendo un altra strada"), [pair("un altra", "un'altra")]);
assert_eq!(fixes("Ho un amica a Roma"), [pair("un amica", "un'amica")]);
assert_eq!(fixes("Non c'è nessun altra via"), [pair("nessun altra", "nessun'altra")]);
for ok in [
"Un altro amico",
"Un'altra amica",
"Un'auto nuova",
"Un'eco lontana",
"Un artista",
"Un atleta",
"Un idiota",
"Un ipocrita",
"Un enigma",
"Un omicida",
"Un astronauta",
"Un acrobata",
"Un'ora",
"Un'idea",
"Un uomo",
"Un'Alfa",
"Un Andrea",
"Un ha",
] {
assert!(fixes(ok).is_empty(), "{ok}: {:?}", fixes(ok));
}
}
#[test]
fn more_emphatic_doubles_are_allowed() {
for ok in ["Pian piano", "Poco poco", "Zitto zitto", "Via via", "Così così", "Dai dai"] {
assert!(fixes(ok).is_empty(), "{ok}: {:?}", fixes(ok));
}
}
proptest::proptest! {
#[test]
fn never_panics(s in "\\PC{0,80}") {
for i in check(&s) {
proptest::prop_assert!(i.start <= i.end && i.end <= s.len());
}
}
#[test]
fn never_panics_on_italian_like_text(s in "[a-zàèéìòóùA-ZÈ ,.;:!?'’‘\"«»…\\-]{0,60}") {
for i in check(&s) {
proptest::prop_assert!(i.start <= i.end && i.end <= s.len() && s.is_char_boundary(i.start) && s.is_char_boundary(i.end));
}
}
}
fn test_dict_install() {
crate::it::test_dict::install();
}
}
crates/proof/src/it/mod.rs
+213
View File
@@ -0,0 +1,213 @@
//! Italian proofing.
//!
//! - **Spelling** against the Hunspell `it_IT` dictionary installed on the system (found by
//! [`crate::hunspell`]: `it_IT.aff`/`it_IT.dic` from the OS, a package manager or
//! LibreOffice; it isn't bundled — the upstream file is GPL-licensed). Without it Italian
//! text gets no spelling marks. Elided forms check the word after the apostrophe
//! (`l'amico`, `un'idea`, `c'è`, `dov'è`, `senz'altro`); truncations keep their apostrophe
//! (`po'`, `mo'`); hyphenated compounds are checked part by part; the user dictionary applies
//! as in English. Forms a grammar rule explains better (`pò`, `qual'è`) are left to it.
//! - **Suggestions** tuned for Italian ([`suggest`]): missing accents first (`caffe` → `caffè`,
//! `perche` → `perché`), then the dictionary's `REP` confusions, a missing space, one-letter
//! typos, and a dictionary scan for anything further off.
//! - **Grammar** ([`grammar`]): cheap, reliable rules with Italian messages, which work with
//! no dictionary installed.
pub mod grammar;
mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// File names of an Italian Hunspell dictionary (Italy first, then Switzerland).
const NAMES: &[&str] = &["it_IT", "it_CH", "it-IT", "it"];
/// The Italian dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Italian");
/// The Italian dictionary: the installed Hunspell `it_IT` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Italian spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
DICT.get()
}
/// Use this Italian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
DICT.install(aff, dic)
}
/// Is an Italian dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
DICT.has_dictionary()
}
/// Proclitics (and verb clippings) that elide before a vowel or `h`: the word after the
/// apostrophe is what gets checked (`l'amico`, `un'idea`, `c'è`, `m'hai`, `dov'è`, `anch'io`).
const PROCLITICS: &[&str] = &[
"l", "un", "una", "uno", "all", "dall", "dell", "nell", "sull", "coll", "pell", "c", "m", "t", "s", "v", "d", "j", "n", "ch", "gl", "quest",
"quell", "bell", "buon", "sant", "grand", "poc", "nessun", "ciascun", "qualcun", "alcun", "dov", "cos", "com", "quand", "anch", "senz", "tutt",
"nient", "mezz", "sott", "tant", "quant", "dev",
];
/// Truncated words written with an apostrophe (`un po'`, `a mo' di`, `be'`): [`crate::words`]
/// trims the apostrophe, so spelling sees the bare word; [`grammar`] checks the apostrophe.
const TRUNCATIONS: &[&str] = &["po", "mo", "be"];
/// Misspellings a grammar rule reports with a clearer message and fix (`pò` → `po'`, `qual'è`
/// → `qual è`), so spelling leaves them alone rather than marking the word twice.
fn left_to_grammar(lower: &str) -> bool {
lower == "pò" || grammar::qual_elided(lower).is_some()
}
/// Is `word` spelled correctly in Italian? Numbers, single letters, short words in capitals
/// (acronyms: `RAI`, `TG`), words with digits and words in other scripts are accepted, and so
/// is every word when no Italian dictionary is installed.
pub fn is_correct(word: &str) -> bool {
is_correct_with(&dictionary(), word)
}
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off,
/// short words in capitals are checked against the dictionary too, and with Ignore words that
/// contain numbers off, so are words with digits.
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
check(&dictionary(), word, opts)
}
/// [`is_correct`] against a given dictionary (an empty one accepts everything).
pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
check(d, word, &crate::Options::default())
}
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
};
let lower = w.to_lowercase();
if TRUNCATIONS.contains(&lower.as_str()) || left_to_grammar(&lower) {
return true;
}
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
}
if let Some((head, tail)) = w.split_once(['\'', '’'])
&& PROCLITICS.contains(&head.to_lowercase().as_str())
&& !tail.is_empty()
{
return known(d, tail);
}
known(d, w)
}
/// Is one word (no hyphens, no apostrophe) in the dictionary or the user dictionary?
fn known(d: &AffixDict, w: &str) -> bool {
d.check(w) || crate::in_user_dictionary(&w.to_lowercase())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_dictionary_loads() {
test_dict::install();
let d = dictionary();
assert!(has_dictionary());
assert!(d.len() > 50, "{}", d.len());
assert!(d.rule_count() >= 4, "the test dictionary's affix rules");
assert!(!d.rep().is_empty() && !d.map().is_empty() && !d.try_chars().is_empty());
}
#[test]
fn swiss_dictionary_files_are_found() {
// it_CH is searched too, so Swiss users with only it_CH installed get Italian spelling.
let dir = std::env::temp_dir().join(format!("wc-it-ch-{}", std::process::id()));
let _ = std::fs::create_dir_all(&dir);
std::fs::write(dir.join("it_CH.aff"), test_dict::AFF).expect("aff");
std::fs::write(dir.join("it_CH.dic"), test_dict::DIC).expect("dic");
let found = crate::hunspell::find_in(&dir, NAMES);
let ok = found.as_ref().is_some_and(|(a, d)| a.ends_with("it_CH.aff") && d.ends_with("it_CH.dic"));
assert!(ok, "{found:?}");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn without_a_dictionary_nothing_is_flagged() {
let none = AffixDict::default();
for w in ["caffè", "perché", "xqzv", ""] {
assert!(is_correct_with(&none, w), "{w}");
}
assert!(install_dictionary("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused");
}
#[test]
fn italian_words_are_correct() {
test_dict::install();
for w in [
"casa", "case", "libro", "libri", "città", "caffè", "perché", "più", "può", "parlare", "parla", "parlano", "rifare", "bello", "belli",
"nuovo", "bianco", "Roma", "Milano", "l'amico", "un'idea", "L'amico", "RAI", "2026", "e", "non",
] {
assert!(is_correct(w), "{w}");
}
}
#[test]
fn elisions_and_truncations_are_correct() {
test_dict::install();
for w in ["dov'è", "Com'è", "cos'è", "anch'io", "senz'acqua", "tutt'uno", "quand'è", "nessun'amica", "n'ha", "po'", "po", "mo'"] {
assert!(is_correct(w), "{w}");
}
}
#[test]
fn misspellings_are_flagged() {
test_dict::install();
for w in ["caffé", "perche", "lbiro", "parlre", "amicaa", "citta", "quelo", "dov'lbiro", "xyz'casa"] {
assert!(!is_correct(w), "{w}");
}
}
#[test]
fn grammar_owned_forms_are_not_marked_twice() {
test_dict::install();
// `pò` and `qual'è` are reported by the grammar rules, with the fix; spelling stays quiet.
for w in ["pò", "Pò", "qual'è", "Qual’era"] {
assert!(is_correct(w), "{w}");
assert!(!grammar::check(w).is_empty(), "{w}: grammar reports it");
}
}
#[test]
fn user_dictionary_applies() {
test_dict::install();
assert!(!is_correct("zxqitalianaparola"));
crate::add_word("Zxqitalianaparola");
assert!(is_correct("zxqitalianaparola"));
}
/// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Italian checks
/// those words against the dictionary instead of accepting them.
#[test]
fn proofing_options_reach_the_italian_check() {
test_dict::install();
let lang = crate::ProofLang::It;
let defaults = crate::Options::default();
let check_caps = crate::Options { ignore_uppercase: false, ..defaults };
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
assert!(crate::is_correct_in_with("LIBRRO", lang, &defaults), "short capitals are acronyms by default");
assert!(!crate::is_correct_in_with("LIBRRO", lang, &check_caps));
assert!(crate::is_correct_in_with("LIBRO", lang, &check_caps));
assert!(crate::is_correct_in_with("librro2", lang, &defaults));
assert!(!crate::is_correct_in_with("librro2", lang, &check_digits));
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
assert!(is_correct("LIBRRO"), "direct checks keep the defaults");
}
}
crates/proof/src/it/suggest.rs
+115
View File
@@ -0,0 +1,115 @@
//! Italian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the
//! words Italian runs into their neighbours (`perme` → `per me`, `amilano` → `a milano`) and
//! letters to try for a dictionary without a TRY line.
use crate::affix::AffixDict;
use crate::affix_suggest::{Split, Tuning};
/// Short words often run into the next one (`per me`, `a Milano`, `di Roma`).
const LEADING_WORDS: &[&str] = &[
"a", "e", "è", "di", "da", "in", "con", "su", "per", "tra", "fra", "che", "chi", "cui", "non", "si", "ci", "vi", "ne", "più", "meno", "molto",
"tanto", "come", "dove", "quando", "mai", "qui", "qua", "lì", "là", "ogni", "altro", "tutto", "questo", "quello", "bene", "male", "anche",
"ancora", "sempre", "già", "poi", "ed", "od", "ad", "un", "uno", "una", "il", "lo", "la", "i", "gli", "le", "del", "al", "dal", "nel", "sul",
"col", "pel", "allo", "dello", "nello", "sullo", "della", "alla", "dalla", "nella", "sulla", "degli", "agli", "dagli", "negli", "sugli", "delle",
"alle", "dalle", "nelle", "sulle", "dei", "ai", "dai", "nei", "sui", "coi", "dei",
];
/// Short words often run into the one before (enclitic pronouns: `dimmelo` in writing without
/// the doubling, or a dropped space: `perme` → `per me`).
const TRAILING_WORDS: &[&str] = &[
"mi", "ti", "si", "ci", "vi", "ne", "lo", "la", "li", "le", "gli", "me", "te", "lui", "lei", "noi", "voi", "loro", "che", "cosa", "come", "dove",
"è",
];
/// What Italian adds to its dictionary's tables.
const TUNING: Tuning =
Tuning {
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, alphabet: "aeiortnslcdupmghfbqvàèéìòóùz", ..Tuning::DEFAULT
};
/// Spelling suggestions for an Italian word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
suggest_in(&super::dictionary(), word, max)
}
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::affix_suggest::{fold, weighted_distance};
fn has(word: &str, want: &str) {
let s = suggest(word, 8);
assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}");
}
#[test]
fn nosuggest_stems_and_forms_are_never_offered() {
crate::affix::tests::assert_nosuggest_is_honoured(suggest_in);
}
#[test]
fn italian_misspellings_get_the_right_suggestion() {
crate::it::test_dict::install();
has("caffe", "caffè");
has("perche", "perché");
has("cioe", "cioè");
has("citta", "città");
has("lbiro", "libro");
has("parlre", "parlare");
has("quelo", "quello");
has("perme", "per me");
has("amilano", "a Milano");
has("roma", "Roma");
has("Caffe", "Caffè");
}
#[test]
fn the_likeliest_fix_comes_first() {
crate::it::test_dict::install();
assert_eq!(suggest("caffe", 3).first().map(String::as_str), Some("caffè"));
assert_eq!(suggest("perche", 3).first().map(String::as_str), Some("perché"));
}
#[test]
fn edge_cases_stay_empty_or_bounded() {
crate::it::test_dict::install();
assert!(suggest("", 5).is_empty());
assert!(suggest("bene", 0).is_empty());
assert!(suggest(&"a".repeat(41), 5).is_empty());
assert!(suggest("due parole", 5).is_empty());
assert!(suggest_in(&crate::affix::AffixDict::default(), "caffe", 5).is_empty());
assert!(suggest("caffè", 5).len() <= 5);
assert!(!suggest("caffè", 5).contains(&"caffè".to_string()));
}
#[test]
fn distance_weighs_accents_half() {
let map = vec![vec!['a', 'à'], vec!['e', 'é', 'è']];
let related = |a: char, b: char| fold(a, &map) == fold(b, &map);
let c = |s: &str| s.chars().collect::<Vec<_>>();
assert_eq!(weighted_distance(&c("caffe"), &c("caffè"), &related, 9), 1);
assert_eq!(weighted_distance(&c("citta"), &c("città"), &related, 9), 1);
assert_eq!(weighted_distance(&c("bene"), &c("bene"), &related, 9), 0);
assert_eq!(weighted_distance(&c("bene"), &c("male"), &related, 2), 3);
assert_eq!(weighted_distance(&c("bene"), &c("benissimo"), &related, 2), 3);
assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4);
}
#[test]
fn suggestions_are_fast_enough() {
crate::it::test_dict::install();
// Warm the dictionary, then time a worst case (a long word nothing near): the scan round.
let _ = suggest("bene", 1);
let t = std::time::Instant::now();
for w in ["xqzvweitalianaparola", "prsidenti", "parallellepipedo"] {
let _ = suggest(w, 6);
}
let ms = t.elapsed().as_secs_f64() * 1000.0;
eprintln!("3 suggestion calls: {ms:.0} ms");
assert!(ms < 20_000.0, "{ms} ms");
}
}
crates/proof/src/it/test_dict.rs
+139
View File
@@ -0,0 +1,139 @@
//! A tiny synthetic Italian Hunspell dictionary for tests (written for these tests; not taken
//! from any real dictionary). Real dictionaries are installed on the system, never bundled.
//!
//! It exercises what the Italian dictionaries use: the default single-character `FLAG` type,
//! suffixes (verb endings `-are`, noun plurals `-o` → `-i`), a prefix (`ri-`), the suggestion
//! tables (`TRY`, `MAP` for accents, `REP` for common confusions) and elided forms with an
//! apostrophe (`l'amico`).
pub const AFF: &str = "SET UTF-8
TRY aeiortnslcdupmghfbqvàèéìòóùz
MAP 5
MAP aà
MAP eéè
MAP iì
MAP oòó
MAP uù
REP 3
REP caffé caffè
REP perche perché
REP cioe cioè
SFX A Y 3
SFX A 0 are .
SFX A 0 a .
SFX A 0 ano .
SFX N Y 2
SFX N o i o
SFX N a e a
PFX R Y 1
PFX R 0 ri .
";
/// Stems (the count line first, as in Hunspell files).
pub const DIC: &str = "97
città
caffè
perché
più
può
è
già
così
virtù
tribù
menù
tè
Mosca
Roma
Milano
Torino
casa/N
libro/N
cane
gatto
sole
luna
mare
amico
amica
idea
parl/A
mangiare
dormire
fare/R
scuola
bambino/N
albero
fiore
pane
acqua
terra
cielo
notte
giorno
vecchia
vado
con
ha
ho
bel
cioè
la
un
in
piazza
lavoro
tempo
uomo
donna
bello/N
buono
grande
piccolo
nuovo/N
vecchio
bianco
nero
rosso
verde
giallo
uno
due
tre
tutto
come
quando
dove
senza
sopra
sotto
tra
molto
sempre
mai
qui
lì
sì
no
non
che
chi
cui
mio
tuo
suo
nostro
a
per
me
quello
io
";
/// Make [`AFF`]/[`DIC`] the Italian dictionary (every test that needs one calls this; they all
/// install the same one, so tests running in parallel agree).
pub fn install() {
if let Err(e) = super::install_dictionary(AFF, DIC) {
panic!("test dictionary: {e}");
}
}
crates/proof/src/lang.rs
+388
View File
@@ -0,0 +1,388 @@
//! Proofing languages: which dictionary, grammar rules and hyphenation patterns apply to a run of
//! text. A run's language is its BCP 47 tag (`w:lang` in .docx, `CharProps::lang` in the model),
//! resolved with [`ProofLang::from_tag`].
/// A language WordCraft can proof, or [`ProofLang::Other`] for one it can't.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum ProofLang {
/// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`] or
/// [`ProofLang::EnCa`]): the public-domain Moby word list, the English grammar rules and the
/// English hyphenation dictionary and patterns. British spellings (`colour`, `centre`,
/// `organise`) are flagged, with the American form suggested ([`crate::en_variants`]). Also
/// text with no language at all (WordCraft's default editing language is US English, like
/// Word's).
#[default]
En,
/// English with British spelling (`en-GB`, `en-AU`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…):
/// [`ProofLang::En`]'s tools, but American spellings (`color`, `center`, `analyze`) are
/// flagged and `-ize` and `-ise` are both right, as in Oxford spelling.
EnGb,
/// Canadian English (`en-CA`): British `-our`, `-re` and `-ence` (`colour`, `centre`,
/// `defence`) with American `-ize` (`organize`), `tire` and `program`.
EnCa,
/// Polish (`pl`, `pl-PL`): the installed Hunspell dictionary (when there is one), the Polish grammar rules and the Polish
/// hyphenation patterns.
Pl,
/// Bulgarian (`bg`, `bg-BG`): hyphenation by the Bulgarian syllable rules ([`crate::bg`]).
/// There is no Bulgarian dictionary or grammar yet, so, as for [`ProofLang::Other`], no
/// spelling or grammar marks.
Bg,
/// Portuguese (`pt`, `pt-BR`, `pt-PT`): the installed Hunspell dictionary (VERO and
/// friends, when there is one), the Portuguese grammar rules and the Portuguese hyphenation
/// patterns.
Pt,
/// Italian (`it`, `it-IT`, `it-CH`): the installed Hunspell dictionary (`it_IT`, when there
/// is one), the Italian grammar rules and the Italian hyphenation patterns.
It,
/// Croatian (`hr`, `hr-HR`)
Hr,
/// Turkish (`tr`, `tr-TR`, `tr-CY`): the installed Hunspell dictionary (when there is one),
/// Turkish grammar rules, spelling suggestions and Turkish syllable hyphenation rules.
Tr,
/// Russian (`ru`, `ru-RU`): the installed Hunspell dictionary (when there is one), the Russian
/// grammar rules and the Russian word-division rules.
Ru,
/// Czech (`cs`, `cs-CZ`): the installed Hunspell dictionary, dictionary-driven suggestions
/// and generic punctuation checks. Hyphenation keeps the fallback English patterns.
Cs,
/// Indonesian (`id`, `id-ID`; also the old `in` and ISO 639-2 `ind`): the installed Hunspell
/// `id_ID` dictionary, dictionary-driven suggestions and generic checks with Indonesian
/// messages. Hyphenation keeps the fallback English patterns.
Id,
/// Spanish of Spain (`es`, `es-ES`, and every region not listed for [`ProofLang::EsMx`],
/// [`ProofLang::Es419`] or [`ProofLang::EsUy`]; also ISO 639-2 `spa`): the installed
/// Hunspell `es_ES` dictionary (or a generic Spanish one), dictionary-driven suggestions and
/// generic checks with Spanish messages ([`crate::es`]). Hyphenation keeps the fallback
/// English patterns.
Es,
/// Mexican Spanish (`es-MX`): [`ProofLang::Es`]'s tools with the installed `es_MX`
/// dictionary, else a Latin American or generic Spanish one.
EsMx,
/// Latin American Spanish (`es-419`, and the Spanish of the Americas other than Mexico and
/// Uruguay: `es-AR`, `es-CO`, `es-US`…): [`ProofLang::Es`]'s tools with the installed `es_419`
/// dictionary, else a generic or another Latin American Spanish one.
Es419,
/// Uruguayan Spanish (`es-UY`): [`ProofLang::Es`]'s tools with the installed `es_UY`
/// dictionary (else the neighbouring Rioplatense `es_AR`, a Latin American or a generic one),
/// and the `vos` forms of verbs (`tenés`, `vení`, `decime`) accepted ([`crate::es::voseo`]).
EsUy,
/// German (`de`, `de-DE`, `de-AT`, `de-CH`…): hyphenation by the German patterns (reformed
/// spelling). There is no German dictionary or grammar yet, so, as for [`ProofLang::Other`],
/// no spelling or grammar marks.
De,
/// A language without proofing tools here (`fr-FR`, `ja-JP`, `x-none`…). Like Word without
/// the language's proofing tools: no spelling or grammar marks, so its words are never
/// flagged as English misspellings. Hyphenation keeps using the English patterns, as before
/// languages were told apart.
Other,
}
impl ProofLang {
/// Every language with a spelling dictionary, in display order.
pub const PROOFED: [ProofLang; 15] = [
ProofLang::En,
ProofLang::EnGb,
ProofLang::EnCa,
ProofLang::Pl,
ProofLang::Pt,
ProofLang::It,
ProofLang::Hr,
ProofLang::Tr,
ProofLang::Ru,
ProofLang::Cs,
ProofLang::Id,
ProofLang::Es,
ProofLang::EsMx,
ProofLang::Es419,
ProofLang::EsUy,
];
/// The proofing language of a BCP 47 tag (`pl-PL`, `en_GB`, `PL`…). The primary subtag
/// counts, and for English and Spanish the region too ([`ProofLang::english`],
/// [`ProofLang::spanish`]). No tag (or an empty one) is English, WordCraft's default editing
/// language.
pub fn from_tag(tag: Option<&str>) -> ProofLang {
let Some(tag) = tag.map(str::trim).filter(|t| !t.is_empty()) else { return ProofLang::En };
let primary = tag.split(['-', '_']).next().unwrap_or("");
if primary.eq_ignore_ascii_case("en") || primary.eq_ignore_ascii_case("eng") {
ProofLang::english(tag)
} else if primary.eq_ignore_ascii_case("pl") || primary.eq_ignore_ascii_case("pol") {
ProofLang::Pl
} else if primary.eq_ignore_ascii_case("bg") || primary.eq_ignore_ascii_case("bul") {
ProofLang::Bg
} else if primary.eq_ignore_ascii_case("pt") || primary.eq_ignore_ascii_case("por") {
ProofLang::Pt
} else if primary.eq_ignore_ascii_case("it") || primary.eq_ignore_ascii_case("ita") {
ProofLang::It
} else if primary.eq_ignore_ascii_case("hr") || primary.eq_ignore_ascii_case("hrv") {
ProofLang::Hr
} else if primary.eq_ignore_ascii_case("tr") || primary.eq_ignore_ascii_case("tur") {
ProofLang::Tr
} else if primary.eq_ignore_ascii_case("ru") || primary.eq_ignore_ascii_case("rus") {
ProofLang::Ru
} else if primary.eq_ignore_ascii_case("cs") || primary.eq_ignore_ascii_case("ces") || primary.eq_ignore_ascii_case("cze") {
ProofLang::Cs
} else if primary.eq_ignore_ascii_case("id") || primary.eq_ignore_ascii_case("ind") || primary.eq_ignore_ascii_case("in") {
ProofLang::Id
} else if primary.eq_ignore_ascii_case("es") || primary.eq_ignore_ascii_case("spa") {
ProofLang::spanish(tag)
} else if ["de", "deu", "ger"].iter().any(|c| primary.eq_ignore_ascii_case(c)) {
ProofLang::De
} else {
ProofLang::Other
}
}
/// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], regions
/// that spell the British way are [`ProofLang::EnGb`], and the United States, any other
/// region and no region are [`ProofLang::En`].
fn english(tag: &str) -> ProofLang {
match region(tag) {
Some(r) if r.eq_ignore_ascii_case("CA") => ProofLang::EnCa,
Some(r) if BRITISH_SPELLING.iter().any(|b| b.eq_ignore_ascii_case(r)) => ProofLang::EnGb,
_ => ProofLang::En,
}
}
/// The Spanish of a Spanish tag, by its [`region`]: Mexico is [`ProofLang::EsMx`], Uruguay
/// [`ProofLang::EsUy`], Latin America (`419`) and the other countries of the Americas are
/// [`ProofLang::Es419`], and Spain, any other region and no region are [`ProofLang::Es`].
fn spanish(tag: &str) -> ProofLang {
match region(tag) {
Some(r) if r.eq_ignore_ascii_case("MX") => ProofLang::EsMx,
Some(r) if r.eq_ignore_ascii_case("UY") => ProofLang::EsUy,
Some(r) if LATIN_AMERICA.iter().any(|l| l.eq_ignore_ascii_case(r)) => ProofLang::Es419,
_ => ProofLang::Es,
}
}
/// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnCa`])?
pub fn is_english(self) -> bool {
matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa)
}
/// Is this one of the Spanishes ([`ProofLang::Es`], [`ProofLang::EsMx`], [`ProofLang::Es419`],
/// [`ProofLang::EsUy`])?
pub fn is_spanish(self) -> bool {
matches!(self, ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy)
}
/// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`, `de`, `es`); empty for [`ProofLang::Other`].
pub fn code(self) -> &'static str {
match self {
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => "en",
ProofLang::Pl => "pl",
ProofLang::Bg => "bg",
ProofLang::Pt => "pt",
ProofLang::It => "it",
ProofLang::Hr => "hr",
ProofLang::Tr => "tr",
ProofLang::Ru => "ru",
ProofLang::Cs => "cs",
ProofLang::Id => "id",
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => "es",
ProofLang::De => "de",
ProofLang::Other => "",
}
}
/// Does WordCraft check spelling and grammar in this language?
pub fn is_proofed(self) -> bool {
Self::PROOFED.contains(&self)
}
}
/// The region of a tag: the first two-letter or three-digit subtag after the language and
/// script, before any singleton; a POSIX `.UTF-8` or `@…` tail is ignored.
fn region(tag: &str) -> Option<&str> {
let tag = tag.split(['.', '@']).next().unwrap_or("");
// Subtags after a singleton (`x-…` private use, `u-…` extensions) aren't regions.
tag.split(['-', '_'])
.skip(1)
.take_while(|p| p.len() != 1)
.find(|p| (p.len() == 2 && p.chars().all(|c| c.is_ascii_alphabetic())) || (p.len() == 3 && p.chars().all(|c| c.is_ascii_digit())))
}
/// Regions whose Spanish is Latin American ([`ProofLang::Es419`]): Latin America and the
/// Caribbean (`419`) and its parts (South America `005`, Central America `013`, the Caribbean
/// `029`), the Spanish-speaking countries of the Americas other than Mexico and Uruguay, and the
/// United States.
const LATIN_AMERICA: [&str; 22] =
["419", "005", "013", "029", "AR", "BO", "CL", "CO", "CR", "CU", "DO", "EC", "GT", "HN", "NI", "PA", "PE", "PR", "PY", "SV", "US", "VE"];
/// Regions whose English uses British spelling ([`ProofLang::EnGb`]): the United Kingdom (and
/// the non-standard `UK`), Ireland, Australia, New Zealand, South Africa, India and other
/// Commonwealth countries, the Crown dependencies and Europe (`150`).
const BRITISH_SPELLING: [&str; 26] = [
"GB", "UK", "IE", "AU", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH",
"PK", "150",
];
/// A BCP 47 tag in canonical case (`pl-PL`, `en-GB`, `zh-Hant-TW`, `x-none`), or `None` when
/// `tag` isn't shaped like one: 1–35 ASCII letters, digits and `-`/`_` separators, starting with
/// a 2–8 letter language (or `x`/`i` for private and grandfathered tags). `_` becomes `-`.
pub fn normalize_tag(tag: &str) -> Option<String> {
let tag = tag.trim();
if tag.is_empty() || tag.len() > 35 || !tag.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') {
return None;
}
let parts: Vec<&str> = tag.split(['-', '_']).collect();
if parts.iter().any(|p| p.is_empty() || p.len() > 8) {
return None;
}
let first = parts.first()?;
let language = (2..=8).contains(&first.len()) && first.chars().all(|c| c.is_ascii_alphabetic());
if !language && !first.eq_ignore_ascii_case("x") && !first.eq_ignore_ascii_case("i") {
return None;
}
let letters = |p: &str| p.chars().all(|c| c.is_ascii_alphabetic());
// After a singleton (`x-…` private use, `u-…` extensions) subtags are lower case.
let singleton = parts.iter().position(|p| p.len() == 1).unwrap_or(usize::MAX);
let out: Vec<String> = parts
.iter()
.enumerate()
.map(|(i, p)| {
if i >= singleton {
p.to_ascii_lowercase()
} else if i > 0 && p.len() == 2 && letters(p) {
p.to_ascii_uppercase() // region
} else if i > 0 && p.len() == 4 && letters(p) {
let lower = p.to_ascii_lowercase();
let mut c = lower.chars();
c.next().map(|f| f.to_ascii_uppercase().to_string() + c.as_str()).unwrap_or_default() // script
} else {
p.to_ascii_lowercase()
}
})
.collect();
Some(out.join("-"))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tags_normalize() {
assert_eq!(normalize_tag("PL-pl").as_deref(), Some("pl-PL"));
assert_eq!(normalize_tag(" pl_pl ").as_deref(), Some("pl-PL"));
assert_eq!(normalize_tag("pl").as_deref(), Some("pl"));
assert_eq!(normalize_tag("zh-hant-tw").as_deref(), Some("zh-Hant-TW"));
assert_eq!(normalize_tag("es-419").as_deref(), Some("es-419"));
assert_eq!(normalize_tag("x-none").as_deref(), Some("x-none"));
assert_eq!(normalize_tag("de-DE-u-co-PHONEBK").as_deref(), Some("de-DE-u-co-phonebk"));
for bad in ["", " ", "p", "1pl", "pl--PL", "pl PL", "pl-PL;", "polski język", "-pl", "pl-", "abcdefghi", &"a".repeat(40)] {
assert_eq!(normalize_tag(bad), None, "{bad:?}");
}
}
fn assert_tags(tags: &[&str], expected: ProofLang) {
for tag in tags {
assert_eq!(ProofLang::from_tag(Some(tag)), expected, "{tag}");
}
}
#[test]
fn tags_resolve_primary_languages() {
assert_tags(&["pl", "pl-PL", "PL-pl", "pl_PL", "pl_PL.UTF-8", " pl-PL ", "pol"], ProofLang::Pl);
assert_tags(&["pt", "pt-BR", "PT_br", "pt-BR.UTF-8", " pt-PT ", "por"], ProofLang::Pt);
assert_tags(&["it", "it-IT", "IT_it", "it-CH", "it-CH.UTF-8", " it-IT ", "ita"], ProofLang::It);
assert_tags(&["cs", "cs-CZ", "CS-cz", "cs_CZ", " ces ", "CES", "cze", "CZE"], ProofLang::Cs);
assert_tags(&["id", "id-ID", "ID-id", "id_ID", "id_ID.UTF-8", " id-ID ", "ind", "in", "in_ID"], ProofLang::Id);
assert_eq!(ProofLang::Cs.code(), "cs");
assert!(ProofLang::Cs.is_proofed() && ProofLang::PROOFED.contains(&ProofLang::Cs));
assert_eq!(ProofLang::Id.code(), "id");
assert!(ProofLang::Id.is_proofed() && ProofLang::PROOFED.contains(&ProofLang::Id));
}
#[test]
fn tags_resolve_english_regions() {
assert_tags(&["en", "en-US", "EN_us", "en-Latn-US", "en-PH", "en-001", "eng", "en_US.UTF-8", "en-x-gb"], ProofLang::En);
assert_tags(
&[
"en-GB",
"EN_gb",
"en_GB.UTF-8",
"en-AU",
"en_AU",
"en-NZ",
"en-IE",
"en-ZA",
"en-IN",
"en-Latn-GB",
"en-UK",
"en-150",
"eng-GB",
"en_GB@euro",
],
ProofLang::EnGb,
);
assert_tags(&["en-CA", "en_CA", "EN-ca", "en_CA.UTF-8", "en-Latn-CA"], ProofLang::EnCa);
for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
assert!(language.is_english() && language.is_proofed() && ProofLang::PROOFED.contains(&language), "{language:?}");
assert_eq!(language.code(), "en");
}
assert!(!ProofLang::Pl.is_english() && !ProofLang::Other.is_english());
// The region is the first two-letter or three-digit subtag: a script or variant isn't one.
assert_eq!(ProofLang::from_tag(Some("en-Brai")), ProofLang::En);
assert_eq!(ProofLang::from_tag(Some("en-GB-oxendict")), ProofLang::EnGb);
}
#[test]
fn tags_resolve_additional_locales_and_defaults() {
assert_tags(&["bg", "bg-BG", "BG_bg", "bg_BG.UTF-8", "bul"], ProofLang::Bg);
assert_tags(&["tr", "tr-TR", "tr-CY", "TR_tr", "tur"], ProofLang::Tr);
assert_tags(&["de", "de-DE", "de-CH", "DE_at", "deu", "ger"], ProofLang::De);
assert_tags(&["ru", "ru-RU", "RU-ru", "ru_RU", "ru_RU.UTF-8", " ru-RU ", "rus"], ProofLang::Ru);
assert!(!ProofLang::De.is_proofed());
assert!(!ProofLang::Bg.is_proofed());
assert_eq!(ProofLang::Bg.code(), "bg");
assert!(ProofLang::Pt.is_proofed() && !ProofLang::Other.is_proofed());
assert!(ProofLang::Tr.is_proofed());
assert!(ProofLang::Ru.is_proofed());
assert_eq!(ProofLang::Ru.code(), "ru");
assert!(ProofLang::PROOFED.contains(&ProofLang::Ru));
assert_eq!(ProofLang::Pl.code(), "pl");
assert_eq!(ProofLang::Pt.code(), "pt");
assert!(ProofLang::It.is_proofed());
assert_eq!(ProofLang::It.code(), "it");
assert!(ProofLang::Hr.is_proofed());
assert_eq!(ProofLang::Hr.code(), "hr");
assert_eq!(ProofLang::Tr.code(), "tr");
assert!(ProofLang::PROOFED.contains(&ProofLang::Pt), "every proofed language is listed");
assert_eq!(ProofLang::from_tag(None), ProofLang::En);
assert_eq!(ProofLang::from_tag(Some("")), ProofLang::En);
assert_eq!(ProofLang::from_tag(Some(" ")), ProofLang::En);
assert!(ProofLang::Pl.is_proofed() && !ProofLang::Other.is_proofed());
}
#[test]
fn tags_reject_unknown_or_partial_language_codes() {
assert_tags(&["fr-FR", "dex", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "esx", "spax", "e", "-"], ProofLang::Other);
}
/// #3360, #3553: Spanish is proofed, by region: Spain (and no region), Mexico, Uruguay, and
/// Latin America with the other countries of the Americas.
#[test]
fn spanish_tags_resolve_by_region() {
for t in ["es", "es-ES", "ES_es", "es_ES.UTF-8", " es-ES ", "spa", "es-Latn-ES", "es-GQ", "es-PH", "es-x-mx", "es_ES@euro"] {
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Es, "{t}");
}
for t in ["es-MX", "es_MX", "ES-mx", "es_MX.UTF-8", "spa-MX", "es-Latn-MX"] {
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::EsMx, "{t}");
}
for t in ["es-419", "es_419", "es-AR", "es-CO", "es_CL.UTF-8", "es-US", "ES-ve", "es-PR", "es-Latn-419", "es-005", "spa-PE"] {
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Es419, "{t}");
}
for t in ["es-UY", "es_UY", "ES-uy", "es_UY.UTF-8", "spa-UY", "es-Latn-UY"] {
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::EsUy, "{t}");
}
for l in [ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy] {
assert!(l.is_spanish() && l.is_proofed() && ProofLang::PROOFED.contains(&l), "{l:?}");
assert!(!l.is_english());
assert_eq!(l.code(), "es");
}
assert!(!ProofLang::En.is_spanish() && !ProofLang::Pt.is_spanish() && !ProofLang::Other.is_spanish());
assert_eq!(normalize_tag("ES_mx").as_deref(), Some("es-MX"));
}
}
crates/proof/src/lib.rs
+2245
View File
File diff suppressed because it is too large. Load diff
crates/proof/src/no_data.rs
+163
View File
@@ -0,0 +1,163 @@
//! A build with no Language data embedded (no `CRAFT_I18N_DIR`): every public entry point still
//! works and never panics. English spelling still marks misspellings, against an English
//! Hunspell dictionary installed on the system or else the Moby word list built into the crate;
//! the dialect variants and hyphenation that need the data are off, and grammar (code, not data)
//! still works.
use crate::hyphen::{self, Limits, Source};
use crate::{EnglishSource, Options, ProofLang, dict, en_variants, patterns};
const LANGS: [ProofLang; 18] = [
ProofLang::En,
ProofLang::EnGb,
ProofLang::EnCa,
ProofLang::Pl,
ProofLang::Bg,
ProofLang::Pt,
ProofLang::It,
ProofLang::Hr,
ProofLang::Tr,
ProofLang::Ru,
ProofLang::Cs,
ProofLang::Id,
ProofLang::Es,
ProofLang::EsMx,
ProofLang::Es419,
ProofLang::EsUy,
ProofLang::De,
ProofLang::Other,
];
/// Words and text that exercise each entry point, hostile ones included.
const SAMPLES: [&str; 10] = [
"",
"teh colour of hyphenation",
"Ünïcödé — ‘quotes’ don't",
"a a apple .Next sentence",
"https://example.org a@b.org",
"zażółć gęślą jaźń",
"Привет, мир — кто-то",
"¿Qué tal? más",
"\u{0}\u{FFFF}\u{10FFFF}",
"supercalifragilisticexpialidociousxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
];
#[test]
fn nothing_is_embedded() {
assert!(crate::data::EN_US_DIC.is_empty());
assert!(dict::Dictionary::en_us().is_empty());
for p in [patterns::Patterns::en_us(), patterns::Patterns::pl(), patterns::Patterns::pt(), patterns::Patterns::it(), patterns::Patterns::de()] {
assert!(p.is_empty());
assert_eq!(p.exception_count(), 0);
}
}
/// R6.4, English never goes unchecked: without the embedded word list, English is checked against
/// an installed English Hunspell dictionary or, without one, the Moby word list built into the
/// crate. Either way a misspelling is marked and common words are right. (Before, everything was right.)
#[test]
fn english_spelling_still_marks_misspellings() {
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
assert!(matches!(crate::english_source(l), Some(EnglishSource::Hunspell(_) | EnglishSource::CoreList)), "{l:?}");
assert!(!crate::is_correct_in("becuase", l) && !crate::is_correct_in("teh", l), "{l:?}");
assert!(crate::suggest_in("teh", 5, l).contains(&"the".to_string()), "{l:?}");
assert!(crate::suggest_in("becuase", 5, l).contains(&"because".to_string()), "{l:?}");
for w in ["the", "The", "house", "houses", "because", "running", "people", "beautiful", "don't"] {
assert!(crate::is_correct_in(w, l), "{w} {l:?}");
}
assert_eq!(crate::check_spelling_in("Becuase the house of the people", l).len(), 1, "{l:?}");
}
assert!(
!crate::is_correct("becuase") && !crate::is_correct_with("becuase", &Options::default()) && !crate::is_correct_for("becuase", Some("en-GB"))
);
assert!(crate::suggest("becuase", 5).contains(&"because".to_string()));
assert_eq!(crate::check_spelling("Becuase house").len(), 1);
// The dialect variants are Language data: none without it.
for w in ["colour", "color"] {
assert_eq!(en_variants::preferred(w, ProofLang::En), None);
assert!(!en_variants::is_foreign(w, ProofLang::EnGb));
}
// Text in a language without proofing is never checked with English.
assert!(crate::check_spelling_in("Teh hous zxqv", ProofLang::Other).is_empty());
assert!(crate::is_correct_for("teh", Some("fr-FR")));
}
#[test]
fn grammar_still_works() {
assert!(!crate::check_grammar("This is is wrong.").is_empty());
}
#[test]
fn hyphenation_is_off_where_it_needs_data() {
let lim = Limits::default();
for (w, l) in [
("hyphenation", ProofLang::En),
("hyphenation", ProofLang::Cs),
("przedsiębiorstwo", ProofLang::Pl),
("computador", ProofLang::Pt),
("calcolatore", ProofLang::It),
("Silbentrennung", ProofLang::De),
] {
assert_eq!(hyphen::word_breaks_in(w, l), (vec![], Source::None), "{w} {l:?}");
assert!(hyphen::hyphen_points_in(w, &lim, l).is_empty(), "{w} {l:?}");
assert_eq!(hyphen::hyphenate_word_in(w, &lim, l), w);
}
assert!(hyphen::hyphen_points("hyphenation", &lim).is_empty());
assert_eq!(hyphen::hyphenate_word("hyphenation", &lim), "hyphenation");
assert_eq!(hyphen::word_breaks("hyphenation").1, Source::None);
// Rules are code, not data: Bulgarian still hyphenates.
assert!(!hyphen::word_breaks_in("книжарница", ProofLang::Bg).0.is_empty());
}
/// Every public entry point, in every language, on every sample: no panic.
#[test]
fn every_public_function_runs_without_data() {
let lim = Limits { min_word: 0, after_first: 0, before_last: 0, capitalized: true };
let opts = Options { ignore_uppercase: false, ignore_numbers: false, ignore_internet: false };
for text in SAMPLES {
let len = text.len();
let _ = crate::words(text);
let _ = crate::check_spelling(text);
let _ = crate::check_spelling_with(text, &opts);
let _ = crate::check_grammar(text);
let _ = crate::is_correct(text);
let _ = crate::is_correct_with(text, &opts);
let _ = crate::suggest(text, 5);
let _ = crate::custom_word(text);
let _ = crate::has_dictionary(Some(text));
let _ = crate::missing_dictionary(Some(text));
let _ = crate::unreadable_dictionary(Some(text));
let _ = crate::is_correct_for(text, Some(text));
let _ = hyphen::word_breaks(text);
let _ = hyphen::hyphen_points(text, &lim);
let _ = hyphen::hyphenate_word(text, &lim);
let spans: Vec<(std::ops::Range<usize>, ProofLang)> = LANGS.iter().map(|&l| (0..len, l)).collect();
let _ = crate::check_text(text, &spans);
let _ = crate::check_text_with(text, &spans, &opts, true, true);
// Out-of-range and reversed spans are hostile input too.
#[allow(clippy::reversed_empty_ranges)]
let _ = crate::check_text(text, &[(len + 5..len + 9, ProofLang::En), (3..1, ProofLang::Pl)]);
for l in LANGS {
let _ = crate::check_spelling_in(text, l);
let _ = crate::check_grammar_in(text, l);
let _ = crate::is_correct_in(text, l);
let _ = crate::is_correct_in_with(text, l, &opts);
let _ = crate::suggest_in(text, 5, l);
let _ = crate::dictionary_load_error(l);
let _ = crate::english_source(l);
let _ = hyphen::word_breaks_in(text, l);
let _ = hyphen::hyphen_points_in(text, &lim, l);
let _ = hyphen::hyphenate_word_in(text, &lim, l);
let _ = en_variants::preferred(text, l);
let _ = en_variants::is_foreign(text, l);
let _ = en_variants::known_variant(text, l, |_| true);
}
}
let _ = crate::installed_dictionaries();
let _ = crate::dictionary_generation();
let _ = crate::user_dictionary();
let _ = crate::has_dictionary(None);
let _ = crate::missing_dictionary(None);
let _ = crate::unreadable_dictionary(None);
}
crates/proof/src/patterns.rs
+356
View File
@@ -0,0 +1,356 @@
//! Liang hyphenation patterns (TeX notation, e.g. `.ab1c`, `4t1ion`): parsing and application.
//!
//! The bundled English set is *ours*: trained from the public-domain Moby Hyphenator word list by
//! DesignCraft's patgen (see `examples/hyphgen.rs`). The Polish set is the TeX one by Hanna
//! Kołodziejska, Bogusław Jackowski and Marek Ryćko (hyph-utf8 `hyph-pl.tex`, used under its MIT
//! licence option), bundled unmodified: [`Patterns::parse`] reads TeX's `\patterns{…}` and
//! `\hyphenation{…}` blocks directly. Patterns are stored in a hash map keyed by the packed letter
//! string, so applying them to a word costs `O(len × max_pattern_len)` lookups.
use std::collections::HashMap;
use std::hash::{BuildHasherDefault, Hasher};
/// Word-boundary marker in patterns.
pub const DOT: char = '.';
/// Longest pattern the packed key supports (6 bits a letter in a `u128`; the Polish set's longest
/// is 14 including the dots).
pub const MAX_PATTERN_LEN: usize = 20;
/// Small, fast multiplicative hasher for packed integer keys and short strings (FxHash-style).
#[derive(Default, Clone, Copy)]
pub struct FxHasher(u64);
impl Hasher for FxHasher {
fn finish(&self) -> u64 {
self.0
}
fn write(&mut self, bytes: &[u8]) {
let (chunks, rest) = bytes.as_chunks::<8>();
for c in chunks {
self.write_u64(u64::from_le_bytes(*c));
}
for &b in rest {
self.write_u64(b as u64);
}
}
fn write_u8(&mut self, i: u8) {
self.write_u64(i as u64);
}
fn write_usize(&mut self, i: usize) {
self.write_u64(i as u64);
}
fn write_u64(&mut self, i: u64) {
self.0 = (self.0.rotate_left(5) ^ i).wrapping_mul(0x51_7c_c1_b7_27_22_0a_95);
}
fn write_u128(&mut self, i: u128) {
self.write_u64(i as u64);
self.write_u64((i >> 64) as u64);
}
}
pub type FxMap<K, V> = HashMap<K, V, BuildHasherDefault<FxHasher>>;
/// Maps characters to 6-bit codes (1 = word boundary, 2.. = letters); 0 = not in the alphabet.
#[derive(Clone, Debug)]
pub struct Alphabet {
ascii: [u8; 128],
other: Vec<(char, u8)>,
next: u8,
}
impl Default for Alphabet {
fn default() -> Self {
let mut a = Alphabet { ascii: [0; 128], other: Vec::new(), next: 2 };
a.ascii[DOT as usize] = 1;
a
}
}
impl Alphabet {
pub fn code(&self, c: char) -> u8 {
if (c as u32) < 128 { self.ascii[c as usize] } else { self.other.iter().find(|x| x.0 == c).map_or(0, |x| x.1) }
}
/// Code for `c`, adding it if new (None when the 6-bit alphabet is full).
pub fn intern(&mut self, c: char) -> Option<u8> {
let k = self.code(c);
if k != 0 {
return Some(k);
}
if self.next >= 63 {
return None;
}
let k = self.next;
self.next += 1;
if (c as u32) < 128 {
self.ascii[c as usize] = k;
} else {
self.other.push((c, k));
}
Some(k)
}
}
/// Packed key of a code string (≤ [`MAX_PATTERN_LEN`] codes): 6 bits a code, the length on top.
#[inline]
pub fn pack(codes: &[u8]) -> u128 {
let mut k = 0u128;
for &c in codes {
k = (k << 6) | c as u128;
}
k | ((codes.len() as u128) << 120)
}
/// A compiled pattern set, with its exception words (TeX's `\hyphenation{…}`).
#[derive(Clone, Debug, Default)]
pub struct Patterns {
alpha: Alphabet,
/// Packed letters → inter-letter values (`len + 1` entries).
map: FxMap<u128, Box<[u8]>>,
max_len: usize,
/// Lowercase word → its break points (char indices), overriding the patterns.
exceptions: HashMap<String, Box<[usize]>>,
}
/// Which TeX block [`Patterns::parse`] is reading.
#[derive(Clone, Copy, PartialEq, Eq)]
enum Block {
Patterns,
Exceptions,
/// Another command's argument (`\message{…}`), ignored.
Skip,
}
impl Patterns {
/// Parse patterns: whitespace-separated, `%` comments to the end of the line (`#` too at the
/// start of a line). A whole TeX pattern file works as is: `\patterns{…}` holds patterns and
/// `\hyphenation{…}` exception words written with their hyphens (`ni-gdy`), and other
/// commands' arguments (`\message{…}`) are skipped; outside any block, a token with a hyphen
/// is an exception and anything else a pattern.
pub fn parse(text: &str) -> Patterns {
let mut p = Patterns::default();
let mut block = Block::Patterns;
for line in text.lines() {
let line = line.trim();
if line.starts_with('#') {
continue;
}
let line = line.split('%').next().unwrap_or("");
for tok in line.split_whitespace() {
let mut tok = tok;
if let Some(cmd) = tok.strip_prefix('\\') {
let (name, rest) = cmd.split_once('{').unwrap_or((cmd, ""));
block = match name {
"hyphenation" => Block::Exceptions,
"patterns" => Block::Patterns,
// shortcut: the argument ends at the first `}`, so nested braces would leak; no bundled file has them.
_ if cmd.contains('{') => Block::Skip,
_ => Block::Patterns,
};
tok = rest;
}
let closes = tok.ends_with('}');
let tok = tok.trim_end_matches('}');
if !tok.is_empty() && block != Block::Skip {
if block == Block::Exceptions || tok.contains('-') {
p.insert_exception(tok);
} else {
p.insert(tok);
}
}
if closes {
block = Block::Patterns;
}
}
}
p
}
/// Add one pattern like `a1b`; returns false if it can't be represented.
pub fn insert(&mut self, pat: &str) -> bool {
let mut codes = Vec::with_capacity(pat.len());
let mut vals = vec![0u8];
for c in pat.chars() {
if let Some(d) = c.to_digit(10) {
*vals.last_mut().unwrap_or(&mut 0) = d as u8;
} else {
let Some(k) = self.alpha.intern(c) else { return false };
codes.push(k);
vals.push(0);
}
}
if codes.is_empty() || codes.len() > MAX_PATTERN_LEN {
return false;
}
self.max_len = self.max_len.max(codes.len());
self.map.insert(pack(&codes), vals.into_boxed_slice());
true
}
/// Add an exception word written with its hyphens (`ni-gdy`; no hyphen: never broken).
pub fn insert_exception(&mut self, word: &str) {
let mut plain = String::with_capacity(word.len());
let mut points = Vec::new();
let mut n = 0usize;
for c in word.chars() {
if c == '-' {
if n > 0 && points.last() != Some(&n) {
points.push(n);
}
} else {
plain.extend(c.to_lowercase());
n += 1;
}
}
points.retain(|&i| i < n);
if n > 0 {
self.exceptions.insert(plain, points.into_boxed_slice());
}
}
/// Break points of an exception word (lowercase), if it is one.
pub fn exception(&self, word: &str) -> Option<&[usize]> {
self.exceptions.get(word).map(|b| &b[..])
}
/// Number of exception words.
pub fn exception_count(&self) -> usize {
self.exceptions.len()
}
/// Make `letters` part of the alphabet even if no pattern mentions them.
pub fn add_letters(&mut self, letters: impl IntoIterator<Item = char>) {
for c in letters {
let _ = self.alpha.intern(c);
}
}
pub fn len(&self) -> usize {
self.map.len()
}
pub fn is_empty(&self) -> bool {
self.map.is_empty()
}
/// Inter-letter values for a lowercase word: `out[i]` is the value *before* char `i`
/// (`out.len() == word.len() + 1`). None if a char is outside the pattern alphabet.
pub fn values(&self, word: &[char]) -> Option<Vec<u8>> {
let mut codes = Vec::with_capacity(word.len() + 2);
codes.push(1);
for &c in word {
let k = self.alpha.code(c);
if k == 0 {
return None;
}
codes.push(k);
}
codes.push(1);
let mut vals = vec![0u8; codes.len() + 1];
apply(&self.map, self.max_len, &codes, &mut vals);
// Padded position p sits before padded char p; word char i is padded char i + 1.
vals.get(1..codes.len()).map(<[u8]>::to_vec)
}
/// Break points (char indices, a hyphen goes before that char) of a lowercase word.
pub fn points(&self, word: &[char]) -> Option<Vec<usize>> {
let v = self.values(word)?;
Some((1..word.len()).filter(|&i| v.get(i).is_some_and(|x| x % 2 == 1)).collect())
}
/// The patterns in TeX notation (sorted by letters), e.g. for writing a pattern file.
pub fn to_tex(&self) -> Vec<String> {
let mut rev: Vec<(u8, char)> =
(0u8..128).filter(|&c| self.alpha.ascii[c as usize] != 0).map(|c| (self.alpha.ascii[c as usize], c as char)).collect();
rev.extend(self.alpha.other.iter().map(|&(c, k)| (k, c)));
let ch = |k: u8| rev.iter().find(|x| x.0 == k).map_or('?', |x| x.1);
let mut out: Vec<(String, String)> = self
.map
.iter()
.map(|(&key, vals)| {
let n = ((key >> 120) as usize).min(MAX_PATTERN_LEN);
let letters: Vec<char> = (0..n).map(|i| ch(((key >> (6 * (n - 1 - i))) & 63) as u8)).collect();
let mut s = String::new();
for (i, l) in letters.iter().enumerate() {
if let Some(&v) = vals.get(i).filter(|v| **v > 0) {
s.push(char::from(b'0' + v.min(9)));
}
s.push(*l);
}
if let Some(&v) = vals.get(n).filter(|v| **v > 0) {
s.push(char::from(b'0' + v.min(9)));
}
(letters.into_iter().collect(), s)
})
.collect();
out.sort();
out.into_iter().map(|x| x.1).collect()
}
}
/// Apply `map` to padded `codes`, raising `vals` (len = codes.len() + 1; shorter is tolerated).
#[inline]
pub fn apply(map: &FxMap<u128, Box<[u8]>>, max_len: usize, codes: &[u8], vals: &mut [u8]) {
let n = codes.len();
for s in 0..n {
let mut k = 0u128;
for (len, &c) in codes.iter().skip(s).take(max_len.min(MAX_PATTERN_LEN)).enumerate() {
k = (k << 6) | c as u128;
if let Some(v) = map.get(&(k | (((len + 1) as u128) << 120))) {
for (j, &x) in v.iter().enumerate() {
if let Some(slot) = vals.get_mut(s + j)
&& x > *slot
{
*slot = x;
}
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn liang_example() {
// The classic example from Liang's thesis: "hy-phen-ation".
let p = Patterns::parse("hy3ph he2n hena4 hen5at 1na n2at 1tio 2io o2n");
let w: Vec<char> = "hyphenation".chars().collect();
assert_eq!(p.points(&w).unwrap(), vec![2, 6]);
let mut tex = p.to_tex();
tex.sort();
assert!(tex.contains(&"hen5at".to_string()));
assert!(p.values(&['h', 'é']).is_none());
}
#[test]
fn tex_files_parse_with_blocks_comments_and_exceptions() {
let p = Patterns::parse(
"% a comment\n\\patterns{\n1na n2at % trailing comment\n.ab4cdefghijklm5n\n}\n\\hyphenation{\nni-gdy\nna-dal\nzawsze\n}\nhy-phen-ation\n",
);
assert_eq!(p.len(), 3);
assert_eq!(p.exception("nigdy"), Some(&[2][..]));
assert_eq!(p.exception("nadal"), Some(&[2][..]));
assert_eq!(p.exception("zawsze"), Some(&[][..]));
assert_eq!(p.exception("hyphenation"), Some(&[2, 6][..]));
assert_eq!(p.exception_count(), 4);
// A pattern longer than the old 9-letter limit applies.
let w: Vec<char> = "abcdefghijklmn".chars().collect();
assert_eq!(p.values(&w).unwrap()[13], 5);
assert!(p.to_tex().contains(&".ab4cdefghijklm5n".to_string()));
}
#[test]
fn other_commands_arguments_are_skipped() {
let p = Patterns::parse("\\message{Some Patterns `x-y' 2024-02-28}\n\\patterns{%\na1b\n}\n\\endinput\n");
assert_eq!((p.len(), p.exception_count()), (1, 0));
}
#[test]
fn apply_tolerates_short_value_buffers() {
let p = Patterns::parse("a1b");
let mut vals = [0u8; 1];
apply(&p.map, p.max_len, &[1, 2, 3, 1], &mut vals);
}
}
crates/proof/src/pl/grammar.rs
+548
View File
@@ -0,0 +1,548 @@
//! Polish grammar and punctuation checks: cheap, rule-based and tuned to stay quiet when unsure
//! (a missed comma costs less than a wrong mark on correct text). Messages are in Polish.
//!
//! - **Repeated word** (`się się`), except interjections and words Polish doubles on purpose
//! (`to to`, `tak tak`, `ha ha`).
//! - **Space before punctuation** (`tak ,`) and **double spaces**; emoticons (` :)`) and
//! ellipses are left alone.
//! - **Capital letter at the start of a sentence**, knowing Polish abbreviations (`np.`, `itd.`,
//! `r.`, `m.in.`), initials, ordinal numbers (`5. miejsce`), ellipses and dialogue dashes
//! (`– Tak? – zapytał`), after which lower case is correct.
//! - **Missing comma** before subordinating conjunctions and relative pronouns that always open a
//! clause: `że`, `ponieważ`, `gdyż`, `aby`, `żeby`, `jeśli`, `jeżeli`, `gdyby`, `lecz` and the
//! forms of `który` — unless a conjunction or particle comes first (`mimo że`, `chyba że`,
//! `tak aby`, `i który`) or, for `który`, a preposition (`w którym`: the comma goes before it).
//! - **`we`/`ze`** where the next word's consonant cluster requires them (`we wtorek`,
//! `ze szkoły`, `ze mną`).
//! - **Doubled punctuation** (`,,` — often a typed opening quote `„` — `..`, `;;`).
//!
//! No English rule (such as `a`/`an`) applies to Polish text.
use crate::{Issue, IssueKind, ProofLang, words};
/// Words Polish repeats on purpose: interjections, onomatopoeia and emphatic doubling.
const ALLOWED_REPEATS: &[&str] = &[
"to", "tak", "nie", "no", "ha", "he", "hi", "hej", "oj", "aj", "ej", "ach", "och", "ech", "puk", "hop", "bum", "cyk", "tik", "pa", "fiu", "la",
"bardzo", "dawno", "daleko", "długo",
];
/// Abbreviations written with a final period after which a sentence goes on in lower case
/// (lower case, without the period).
const ABBREVIATIONS: &[&str] = &[
"np", "itd", "itp", "tzn", "tj", "tzw", "ok", "ul", "al", "pl", "os", "in", "wg", "zob", "por", "str", "ss", "nr", "pkt", "ust", "art", "par",
"rozdz", "tab", "rys", "il", "przyp", "red", "wyd", "oprac", "tłum", "cyt", "jw", "ds", "im", "ks", "św", "bł", "prof", "dr", "doc", "hab",
"inż", "mgr", "lek", "płk", "gen", "mjr", "kpt", "ppłk", "sierż", "szer", "kmdr", "adm", "abp", "bp", "br", "ub", "bm", "pn", "płd", "płn",
"wsch", "zach", "gł", "godz", "min", "sek", "tys", "mln", "mld", "zł", "gr", "kg", "dag", "mies", "tyg", "pon", "wt", "śr", "czw", "pt", "sob",
"niedz", "sty", "lut", "mar", "kwi", "cze", "lip", "sie", "wrz", "paź", "lis", "gru", "ang", "niem", "franc", "łac", "ros", "wł", "hiszp", "pol",
"lit", "dosł", "przen", "pot", "zdrob", "fot", "ryc", "tel", "kom", "ew", "ob", "obyw", "pp", "zm", "ur", "ca", "vs", "etc", "cdn", "dot", "wz",
"zw", "zał", "cz", "rr", "ww", "jęz", "jun", "sen", "mec", "red", "dyr", "kier", "proc", "poj", "lp", "lm", "zaw", "woj", "pow", "gm", "wsp",
"nast", "poprz", "ang", "fr",
];
/// Subordinating conjunctions and relative pronouns that open a clause: a comma comes before
/// them.
const CLAUSE_OPENERS: &[&str] = &[
"że",
"ponieważ",
"gdyż",
"aby",
"żeby",
"jeśli",
"jeżeli",
"gdyby",
"lecz",
"który",
"która",
"które",
"którego",
"której",
"któremu",
"którą",
"którym",
"których",
"którymi",
"którzy",
];
/// Words after which a clause opener takes no comma of its own: conjunctions (`i że`), particles
/// (`tylko jeśli`, `zwłaszcza że`) and the first halves of compound conjunctions (`mimo że`,
/// `chyba że`, `tak aby`, `jak gdyby`, `na wypadek gdyby`, `z tym że`).
const NO_COMMA_AFTER: &[&str] = &[
"i",
"a",
"oraz",
"lub",
"albo",
"ani",
"czy",
"bądź",
"ale",
"lecz",
"bo",
"że",
"aby",
"żeby",
"by",
"niż",
"jak",
"jako",
"niby",
"tylko",
"nawet",
"właśnie",
"zwłaszcza",
"szczególnie",
"głównie",
"częściowo",
"jedynie",
"wyłącznie",
"chyba",
"mianowicie",
"przynajmniej",
"również",
"także",
"też",
"dopiero",
"jeszcze",
"już",
"zaś",
"więc",
"zatem",
"toteż",
"przecież",
"chociaż",
"choć",
"choćby",
"gdy",
"kiedy",
"jeśli",
"jeżeli",
"gdyby",
"skoro",
"zanim",
"dopóki",
"póki",
"aż",
"co",
"tak",
"mimo",
"pomimo",
"wtedy",
"teraz",
"dziś",
"dzisiaj",
"podczas",
"tym",
"bardziej",
"warunkiem",
"wypadek",
"zamiast",
"byle",
"lada",
"wiadomo",
"obojętnie",
"jedno",
"wszystkim",
"może",
"pewnie",
"prawdopodobnie",
"zapewne",
"dlatego",
"tyle",
"dość",
"ponieważ",
"gdyż",
"lecz",
"albowiem",
"oto",
"no",
];
/// Prepositions: a relative pronoun after one takes the comma before the preposition instead
/// (`dom, w którym`).
const PREPOSITIONS: &[&str] = &[
"w",
"we",
"z",
"ze",
"na",
"do",
"od",
"ode",
"o",
"u",
"po",
"za",
"przed",
"przede",
"przez",
"przeze",
"przy",
"nad",
"nade",
"pod",
"pode",
"dla",
"bez",
"beze",
"ku",
"między",
"pomiędzy",
"wśród",
"spośród",
"spod",
"spoza",
"sprzed",
"znad",
"zza",
"poza",
"ponad",
"poprzez",
"dzięki",
"według",
"wobec",
"oprócz",
"prócz",
"obok",
"koło",
"około",
"wokół",
"dookoła",
"naprzeciw",
"naprzeciwko",
"wzdłuż",
"wewnątrz",
"zewnątrz",
"względem",
"przeciw",
"przeciwko",
"wbrew",
"wskutek",
"śród",
"wedle",
"pośród",
"ponad",
"niedaleko",
"blisko",
];
fn issue(start: usize, end: usize, message: String, suggestions: Vec<String>) -> Issue {
Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::Pl }
}
fn capitalize(w: &str) -> String {
crate::affix::capitalize(w)
}
/// Polish vowels (and the accented vowels of loanwords).
fn is_vowel(c: char) -> bool {
matches!(c.to_lowercase().next().unwrap_or(c), 'a' | 'ą' | 'e' | 'ę' | 'i' | 'o' | 'ó' | 'u' | 'y' | 'é' | 'á' | 'ö' | 'ü' | 'í' | 'ú')
}
fn is_consonant(c: char) -> bool {
c.is_alphabetic() && !is_vowel(c)
}
/// Grammar issues in a paragraph of Polish text, in order.
pub fn check(text: &str) -> Vec<Issue> {
check_with(text, true)
}
/// [`check`] where `capital_first` says whether the text's first word must start a sentence with a capital
/// (false for a list item, which may be a fragment).
pub fn check_with(text: &str, capital_first: bool) -> Vec<Issue> {
let ws: Vec<(usize, usize)> = words(text);
let mut v = Vec::new();
repeated_words(text, &ws, &mut v);
spaces(text, &mut v);
sentence_capitals(text, &ws, capital_first, &mut v);
missing_commas(text, &ws, &mut v);
we_ze(text, &ws, &mut v);
doubled_punctuation(text, &mut v);
v.sort_by_key(|i| (i.start, i.end));
v
}
fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
for pair in ws.windows(2) {
let [(a0, b0), (a1, b1)] = pair else { continue };
let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue };
if !gap.is_empty()
&& gap.chars().all(char::is_whitespace)
&& w0.chars().all(char::is_alphabetic)
&& w0.to_lowercase() == w1.to_lowercase()
&& !ALLOWED_REPEATS.contains(&w0.to_lowercase().as_str())
{
v.push(issue(*a0, *b1, format!("Powtórzone słowo: „{w1}”"), vec![w0.to_string()]));
}
}
}
fn spaces(text: &str, v: &mut Vec<Issue>) {
for (i, c) in text.char_indices() {
if c != ' ' {
continue;
}
let rest = text.get(i + 1..).unwrap_or("");
let mut it = rest.chars();
let Some(p) = it.next() else { continue };
let after = it.next();
let flag = match p {
',' => true,
// `;)` and `;-)` are emoticons.
';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')),
// ` ...` is an ellipsis, ` .5` a number.
'.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
// ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space.
':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '»' | '’' | '!' | '?')),
_ => false,
};
if flag {
v.push(issue(i, i + 1 + p.len_utf8(), "Zbędna spacja przed znakiem interpunkcyjnym".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Podwójna spacja".into(), vec![" ".into()]));
}
}
/// Is `prev`, before a period, an abbreviation, an initial, a number or a Roman numeral (after
/// which a lower-case word doesn't start a sentence)?
fn abbreviation_or_number(prev: &str) -> bool {
prev.chars().count() <= 1
|| prev.chars().any(|c| c.is_ascii_digit())
|| (prev.chars().all(|c| matches!(c, 'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M')))
|| ABBREVIATIONS.contains(&prev.to_lowercase().as_str())
}
const CLOSING: [char; 8] = ['"', '”', '’', '\'', ')', '»', ']', '\u{1}'];
const OPENING: [char; 8] = ['„', '"', '“', '«', '(', '[', '\'', '‚'];
/// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after
/// an abbreviation), then space, with nothing but quotes and brackets around?
fn ends_sentence(gap: &str, prev: &str) -> bool {
let t = gap.trim_start_matches(CLOSING);
let Some(end) = t.chars().next() else { return false };
if !matches!(end, '.' | '!' | '?') || t.starts_with("..") {
return false;
}
if end == '.' && abbreviation_or_number(prev) {
return false;
}
let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING);
rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| OPENING.contains(&c))
}
fn sentence_capitals(text: &str, ws: &[(usize, usize)], capital_first: bool, v: &mut Vec<Issue>) {
// List items (`– jabłka,`) may start in lower case.
let list_item = text.trim_end().ends_with([',', ';']);
for (k, &(a, b)) in ws.iter().enumerate() {
let Some(w) = text.get(a..b) else { continue };
let starts = if k == 0 {
let before = text.get(..a).unwrap_or("");
capital_first && !list_item && before.chars().all(|c| c.is_whitespace() || OPENING.contains(&c))
} else {
let Some(&(pa, pb)) = ws.get(k - 1) else { continue };
ends_sentence(text.get(pb..a).unwrap_or(""), text.get(pa..pb).unwrap_or(""))
};
if starts && w.chars().next().is_some_and(char::is_lowercase) && w.chars().all(char::is_alphabetic) {
v.push(issue(a, b, "Wielka litera na początku zdania".into(), vec![capitalize(w)]));
}
}
}
fn missing_commas(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
for k in 1..ws.len() {
let (Some(&(pa, pb)), Some(&(a, b))) = (ws.get(k - 1), ws.get(k)) else { continue };
let (Some(prev), Some(w), Some(gap)) = (text.get(pa..pb), text.get(a..b), text.get(pb..a)) else { continue };
if !CLAUSE_OPENERS.contains(&w) || gap.is_empty() || !gap.chars().all(|c| c == ' ' || c == '\u{a0}') {
continue;
}
if !prev.chars().all(char::is_alphabetic) {
continue;
}
let p = prev.to_lowercase();
let p = p.as_str();
if NO_COMMA_AFTER.contains(&p) {
continue;
}
if w.starts_with("któr") && (PREPOSITIONS.contains(&p) || p == "to") {
continue;
}
v.push(issue(pb, b, format!("Brak przecinka przed „{w}”"), vec![format!(", {w}")]));
}
}
fn we_ze(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
for pair in ws.windows(2) {
let [(a0, b0), (a1, b1)] = pair else { continue };
let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue };
if gap != " " && gap != "\u{a0}" {
continue;
}
let next = w1.to_lowercase();
let c: Vec<char> = next.chars().collect();
let at = |i: usize| c.get(i).copied();
let (long, wanted) = match w0 {
"w" | "W" => ("we", (at(0) == Some('w') && at(1).is_some_and(is_consonant)) || next == "mnie"),
"z" | "Z" => {
// The first sound: `sz` is one consonant (`z szafy`, but `ze szkoły`).
let after = if at(0) == Some('s') && at(1) == Some('z') { at(2) } else { at(1) };
let sibilant = matches!(at(0), Some('s' | 'z' | 'ś' | 'ź' | 'ż'));
("ze", (sibilant && after.is_some_and(is_consonant)) || next == "mną")
}
_ => continue,
};
if wanted {
let fixed = if w0.starts_with(char::is_uppercase) { capitalize(long) } else { long.to_string() };
v.push(issue(*a0, *b0, format!("Przed tym słowem piszemy „{long}”"), vec![fixed]));
}
}
}
fn doubled_punctuation(text: &str, v: &mut Vec<Issue>) {
let bytes = text.as_bytes();
let mut i = 0;
while let Some(&b) = bytes.get(i) {
if !matches!(b, b',' | b'.' | b';') {
i += 1;
continue;
}
let run = bytes.get(i..).map_or(0, |r| r.iter().take_while(|&&x| x == b).count());
let flag = match b {
b'.' => run == 2,
_ => run >= 2,
};
if flag {
let end = i + run;
let suggestions: Vec<String> = match b {
// `,,` before a word, after a space: a typed opening quote.
b',' if text.get(end..).and_then(|t| t.chars().next()).is_some_and(char::is_alphabetic)
&& (i == 0 || text.get(..i).is_some_and(|t| t.ends_with(char::is_whitespace))) =>
{
vec!["„".into()]
}
b'.' => vec![".".into(), "…".into()],
_ => vec![char::from(b).to_string()],
};
v.push(issue(i, end, "Powtórzony znak interpunkcyjny".into(), suggestions));
}
i += run.max(1);
}
}
#[cfg(test)]
mod tests {
use super::*;
fn messages(t: &str) -> Vec<String> {
check(t).into_iter().map(|i| format!("{}@{}", i.message, &t[i.start..i.end])).collect()
}
fn clean(t: &str) {
let m = messages(t);
assert!(m.is_empty(), "{t:?}: {m:?}");
}
#[test]
fn repeated_words() {
let v = check("Ala ma ma kota i się się cieszy.");
assert_eq!(v.len(), 2, "{v:?}");
assert_eq!(v[0].message, "Powtórzone słowo: „ma”");
assert_eq!(v[0].suggestions, ["ma"]);
assert!(v.iter().all(|i| i.lang == ProofLang::Pl && i.kind == IssueKind::Grammar));
clean("Tak tak, to to samo. Ha ha!");
}
#[test]
fn spaces_and_punctuation() {
let m = messages("Ala ma kota , a kot ma Alę . Czy na pewno ?");
assert!(m.contains(&"Zbędna spacja przed znakiem interpunkcyjnym@ ,".to_string()), "{m:?}");
assert!(m.contains(&"Zbędna spacja przed znakiem interpunkcyjnym@ .".to_string()), "{m:?}");
assert!(m.contains(&"Zbędna spacja przed znakiem interpunkcyjnym@ ?".to_string()), "{m:?}");
assert!(m.contains(&"Podwójna spacja@ ".to_string()), "{m:?}");
clean("Fajnie :) Naprawdę :-) Albo :D Wartość .5 i wielokropek ...");
let m = messages("Powiedział ,,tak” i poszedł.. Potem;; koniec.");
assert_eq!(m.iter().filter(|x| x.starts_with("Powtórzony znak")).count(), 3, "{m:?}");
assert_eq!(check("Powiedział ,,tak”.").iter().find(|i| i.message.starts_with("Powtórzony")).unwrap().suggestions, ["„"]);
clean("Czekał… i czekał... aż do rana.");
}
#[test]
fn sentence_start_capitals() {
let m = messages("to jest zdanie. a to drugie! czy trzecie? tak.");
assert_eq!(m.iter().filter(|x| x.starts_with("Wielka litera")).count(), 4, "{m:?}");
assert_eq!(check("ala ma kota.")[0].suggestions, ["Ala"]);
// Abbreviations, initials, ordinals, dates, ellipses and dialogue dashes.
clean("Kupiłem owoce, np. jabłka, gruszki itp. oraz warzywa.");
clean("W 1990 r. powstała firma, m.in. dzięki dotacji.");
clean("Spotkałem J. Kowalskiego i prof. Nowaka przy ul. Długiej.");
clean("Zajął 5. miejsce w XX w. sporcie.");
clean("Czekałem... aż w końcu przyszedł.");
clean("– Naprawdę? – zapytał. – Tak – odpowiedziała.");
clean("Powiedział: „Wracam”. Potem wyszedł.");
// A list item may start in lower case.
clean("jabłka,");
}
#[test]
fn missing_commas() {
let m = messages("Wiem że przyjdzie. Mam kota który śpi. Zostałem bo musiałem. Idę aby zdążyć. Nie przyszedł ponieważ padało.");
for want in [
"Brak przecinka przed „że”@ że",
"Brak przecinka przed „który”@ który",
"Brak przecinka przed „aby”@ aby",
"Brak przecinka przed „ponieważ”@ ponieważ",
] {
assert!(m.contains(&want.to_string()), "{want}: {m:?}");
}
assert_eq!(check("Wiem że tak.")[0].suggestions, [", że"]);
// Compound conjunctions, particles, conjunctions and prepositions take no comma there.
clean("Przyszedł, mimo że padało, chyba że nie.");
clean("Zrobię to, tylko jeśli zdążę, i to tak aby nikt nie widział.");
clean("To dom, w którym mieszkam, i ogród, przez który idę.");
clean("Wiem, że przyjdzie, i że zostanie.");
clean("Myślę, że gdyby przyszedł, byłoby dobrze.");
clean("Zachowywał się, jak gdyby nigdy nic.");
clean("Który z nich? To który wybierasz?");
clean("Po raz nie wiadomo który.");
clean("Dlatego że padało, zostaliśmy.");
}
#[test]
fn we_and_ze() {
let m = messages("Spotkamy się w wtorek w Wrocławiu z szkoły z mną.");
assert_eq!(m.iter().filter(|x| x.starts_with("Przed tym słowem")).count(), 4, "{m:?}");
let v = check("Z szkoły wracam.");
assert_eq!(v[0].suggestions, ["Ze"]);
clean("Byłem w wodzie, w wieku dziesięciu lat, z szafą, z siostrą i z zamkiem.");
clean("Wracam ze szkoły we wtorek ze mną.");
}
#[test]
fn correct_polish_text_is_clean() {
clean(
"Zażółć gęślą jaźń. Chrząszcz brzmi w trzcinie, a źdźbło trawy się kołysze. \
Przyjdę jutro, chyba że będzie padać. Książka, którą czytam, jest ciekawa.",
);
clean("Pięćdziesięciu uczniów przyszło, ponieważ ogłoszono konkurs.");
}
proptest::proptest! {
#[test]
fn never_panics(s in "\\PC{0,80}") {
let _ = check(&s);
}
#[test]
fn never_panics_on_polish_like_text(s in "[a-ząćęłńóśźżA-Z ,.;:!?„”…\\-]{0,60}") {
for i in check(&s) {
proptest::prop_assert!(i.start <= i.end && i.end <= s.len());
proptest::prop_assert!(s.is_char_boundary(i.start) && s.is_char_boundary(i.end));
}
}
}
}
crates/proof/src/pl/mod.rs
+193
View File
@@ -0,0 +1,193 @@
//! Polish proofing.
//!
//! - **Spelling** against the SJP.PL Hunspell dictionary installed on the system
//! ([`dictionary`], found by [`crate::hunspell`]: `pl_PL.aff`/`pl_PL.dic` from the OS, a
//! package manager or LibreOffice; it isn't bundled). Without it Polish text gets no spelling
//! marks. Capitalized stems are names (`Warszawa`); a sentence may start with any word and
//! headings may be in capitals. Hyphenated compounds are checked part by part
//! (`biało-czerwony`), foreign names inflected with an apostrophe are accepted
//! (`Kennedy'ego`), and the user dictionary applies as in English.
//! - **Suggestions** tuned for Polish ([`suggest()`]): missing diacritics first (`zolw` → `żółw`), then
//! the common spelling confusions (`ż`/`rz`, `u`/`ó`, `h`/`ch`…), a missing space (`napewno` →
//! `na pewno`), one-letter typos, and a dictionary scan for anything further off.
//! - **Grammar** ([`grammar`]): cheap, reliable rules with Polish messages.
pub mod grammar;
mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// File names of a Polish Hunspell dictionary.
const NAMES: &[&str] = &["pl_PL", "pl-PL", "pl"];
/// The Polish dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Polish");
/// The Polish dictionary: the installed Hunspell `pl_PL` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
DICT.get()
}
/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
DICT.install(aff, dic)
}
/// Is a Polish dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
DICT.has_dictionary()
}
/// Endings Polish adds after an apostrophe to foreign names whose last letter isn't pronounced
/// (`Kennedy'ego`, `Wilde'a`, `Shakespeare'owi`).
const APOSTROPHE_ENDINGS: &[&str] = &["a", "ach", "ami", "e", "ego", "em", "emu", "i", "m", "om", "owi", "owie", "u", "y", "ów", "ą", "ę"];
/// Is `word` spelled correctly in Polish? Numbers, single letters, short words in capitals
/// (acronyms: `PKO`, `ZUS`), words with digits and words in other scripts are accepted, and so
/// is every word when no Polish dictionary is installed.
pub fn is_correct(word: &str) -> bool {
is_correct_with(&dictionary(), word)
}
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off,
/// short words in capitals are checked against the dictionary too, and with Ignore words that
/// contain numbers off, so are words with digits.
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
check(&dictionary(), word, opts)
}
/// [`is_correct`] against a given dictionary (an empty one accepts everything).
pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
check(d, word, &crate::Options::default())
}
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
};
let w = w.replace('’', "'");
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
}
known(d, &w)
}
/// Is one word (no hyphens) in the dictionary, the user dictionary, or a foreign name with a
/// Polish ending after an apostrophe?
fn known(d: &AffixDict, w: &str) -> bool {
if d.check(w) || crate::in_user_dictionary(&w.to_lowercase()) {
return true;
}
if let Some((stem, ending)) = w.split_once('\'') {
return stem.chars().next().is_some_and(char::is_uppercase) && stem.chars().count() >= 2 && APOSTROPHE_ENDINGS.contains(&ending);
}
false
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_dictionary_loads() {
test_dict::install();
let d = dictionary();
assert!(has_dictionary());
assert!(d.len() > 50, "{}", d.len());
assert_eq!(d.rule_count(), 5, "the test dictionary's affix rules");
assert!(!d.rep().is_empty() && !d.map().is_empty() && !d.try_chars().is_empty());
}
#[test]
fn without_a_dictionary_nothing_is_flagged_and_nothing_suggested() {
let none = AffixDict::default();
for w in ["żeka", "kturzy", "xqzv", ""] {
assert!(is_correct_with(&none, w), "{w}");
}
assert!(suggest::suggest_in(&none, "żeka", 5).is_empty());
assert!(install_dictionary("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused");
}
#[test]
fn polish_words_are_correct() {
test_dict::install();
for w in [
"źdźbło",
"gżegżółka",
"chrząszcz",
"pięćdziesięciu",
"rzeka",
"którzy",
"żółw",
"Zażółć",
"gęślą",
"jaźń",
"konstytucja",
"konstytucyjnego",
"niebieskiego",
"nieładnie",
"przyszliśmy",
"zrobilibyśmy",
"Warszawa",
"Warszawie",
"WARSZAWA",
"Kot",
"biało-czerwony",
"polsko-niemieckiej",
"Kennedy'ego",
"Shakespeare’a",
"PKO",
"2026",
"e-mail",
"w",
"się",
] {
assert!(is_correct(w), "{w}");
}
}
#[test]
fn misspellings_are_flagged() {
test_dict::install();
for w in ["żeka", "kturzy", "napewno", "wogóle", "zolw", "chuśtawka", "bżoza", "warszawa", "niewiem", "pszyjaciel", "Kennedy'xyz"] {
assert!(!is_correct(w), "{w}");
}
}
#[test]
fn user_dictionary_applies() {
test_dict::install();
assert!(!is_correct("zxqpolskisłowo"));
crate::add_word("Zxqpolskisłowo");
assert!(is_correct("zxqpolskisłowo"));
}
/// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Polish checks
/// those words against the dictionary instead of accepting them.
#[test]
fn proofing_options_reach_the_polish_check() {
test_dict::install();
let lang = crate::ProofLang::Pl;
let defaults = crate::Options::default();
let check_caps = crate::Options { ignore_uppercase: false, ..defaults };
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
assert!(crate::is_correct_in_with("ŻEKA", lang, &defaults), "short capitals are acronyms by default");
assert!(!crate::is_correct_in_with("ŻEKA", lang, &check_caps));
assert!(crate::is_correct_in_with("RZEKA", lang, &check_caps));
assert!(crate::is_correct_in_with("żeka2", lang, &defaults));
assert!(!crate::is_correct_in_with("żeka2", lang, &check_digits));
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
assert!(is_correct("ŻEKA"), "direct checks keep the defaults");
}
}
crates/proof/src/pl/suggest.rs
+132
View File
@@ -0,0 +1,132 @@
//! Polish spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the
//! words Polish runs into their neighbours (`napewno` → `na pewno`, `kochamcię` → `kocham cię`),
//! letters to try for a dictionary without a TRY line, and final devoicing (`chlep` → `chleb`).
use crate::affix::AffixDict;
use crate::affix_suggest::{COST_ADD_DIACRITIC, COST_MISTYPE, Split, Tuning};
/// Short words often run into the next one (`na pewno`, `w ogóle`, `nie wiem`, `przede
/// wszystkim`).
const LEADING_WORDS: &[&str] = &[
"a", "i", "o", "u", "w", "z", "we", "ze", "na", "po", "do", "od", "za", "co", "to", "nie", "jak", "tak", "przy", "bez", "pod", "nad", "dla",
"że", "by", "czy", "już", "ani", "przed", "przez", "przede", "ku", "aż", "bo", "ten", "ta", "te", "tym", "tej", "też", "jest", "ode", "beze",
"nade",
];
/// Short words often run into the one before (`kocham cię`, `daj mi`, `bałem się`).
const TRAILING_WORDS: &[&str] =
&["się", "cię", "mi", "mu", "ci", "go", "ją", "je", "nam", "wam", "im", "by", "że", "no", "mnie", "tobie", "jej", "ich"];
/// What Polish adds to its dictionary's tables.
const TUNING: Tuning = Tuning {
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS },
alphabet: "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx",
mistype_cost,
..Tuning::DEFAULT
};
/// Final devoicing makes a voiced consonant at the end of a word sound voiceless, so writing it
/// voiceless (`chlep` for `chleb`) is a likelier slip than any other typo.
fn mistype_cost(chars: &[char], i: usize, c: char) -> u32 {
let last = i + 1 == chars.len();
if last && matches!((chars.get(i), c), (Some('p'), 'b') | (Some('t'), 'd') | (Some('k'), 'g')) { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE }
}
/// Spelling suggestions for a Polish word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
suggest_in(&super::dictionary(), word, max)
}
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::affix_suggest::{fold, weighted_distance};
fn has(word: &str, want: &str) {
let s = suggest(word, 8);
assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}");
}
#[test]
fn nosuggest_stems_and_forms_are_never_offered() {
crate::affix::tests::assert_nosuggest_is_honoured(suggest_in);
}
#[test]
fn polish_misspellings_get_the_right_suggestion() {
crate::pl::test_dict::install();
has("żeka", "rzeka");
has("kturzy", "którzy");
has("napewno", "na pewno");
has("wogóle", "w ogóle");
has("niewiem", "nie wiem");
has("zolw", "żółw");
has("gzegzolka", "gżegżółka");
has("chuśtawka", "huśtawka");
has("bżoza", "brzoza");
has("pszyjaciel", "przyjaciel");
has("warszawa", "Warszawa");
has("Żeka", "Rzeka");
has("ZOLW", "ŻÓŁW");
has("konstytucija", "konstytucja");
has("wziąść", "wziąć");
has("dziewczyan", "dziewczyna");
}
#[test]
fn voiced_consonant_suggestion_ranks_first() {
crate::pl::test_dict::install();
let result = suggest("chlep", 6);
assert_eq!(result.first().map(String::as_str), Some("chleb"), "{result:?}");
}
#[test]
fn the_likeliest_fix_comes_first() {
crate::pl::test_dict::install();
assert_eq!(suggest("kturzy", 3).first().map(String::as_str), Some("którzy"));
assert_eq!(suggest("zolw", 3).first().map(String::as_str), Some("żółw"));
assert_eq!(suggest("napewno", 3).first().map(String::as_str), Some("na pewno"));
assert_eq!(suggest("żeka", 3).first().map(String::as_str), Some("rzeka"));
}
#[test]
fn edge_cases_stay_empty_or_bounded() {
crate::pl::test_dict::install();
assert!(suggest("", 5).is_empty());
assert!(suggest("kot", 0).is_empty());
assert!(suggest(&"a".repeat(41), 5).is_empty());
assert!(suggest("dwa słowa", 5).is_empty());
assert!(suggest("żółw", 5).len() <= 5);
assert!(!suggest("żółw", 5).contains(&"żółw".to_string()));
}
#[test]
fn distance_weighs_diacritics_half() {
let map = vec![vec!['a', 'ą'], vec!['o', 'ó', 'u']];
let related = |a: char, b: char| fold(a, &map) == fold(b, &map);
let c = |s: &str| s.chars().collect::<Vec<_>>();
assert_eq!(weighted_distance(&c("ktury"), &c("który"), &related, 9), 1);
assert_eq!(weighted_distance(&c("kot"), &c("kto"), &related, 9), 2);
assert_eq!(weighted_distance(&c("kot"), &c("kotek"), &related, 9), 4);
assert_eq!(weighted_distance(&c("kot"), &c("pies"), &related, 2), 3);
assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4);
}
#[test]
fn suggestions_are_fast_enough() {
crate::pl::test_dict::install();
// Warm the dictionary, then time a worst case (a long word nothing near): the scan round.
let _ = suggest("kot", 1);
let t = std::time::Instant::now();
for w in ["xqzvwpolskiegoslowa", "pszyjacielowi", "zażółćgęśląjaźń"] {
let _ = suggest(w, 6);
}
let ms = t.elapsed().as_secs_f64() * 1000.0;
eprintln!("3 suggestion calls: {ms:.0} ms");
assert!(ms < 20_000.0, "{ms} ms");
}
}
crates/proof/src/pl/test_dict.rs
+114
View File
@@ -0,0 +1,114 @@
//! A tiny synthetic Polish Hunspell dictionary for tests (written for these tests; not taken from
//! any real dictionary). Real dictionaries are installed on the system, never bundled.
/// Affix rules: a few suffixes and one prefix, plus the suggestion tables the Polish
/// suggestions use (`TRY`, `MAP` for diacritics, `REP` for common confusions).
pub const AFF: &str = "SET UTF-8
TRY aioeznrwcysżtkpdmłjlubąęgchśóźćń
MAP 9
MAP aą
MAP cć
MAP eę
MAP lł
MAP nń
MAP oó
MAP uó
MAP sś
MAP zżź
REP 6
REP ż rz
REP rz ż
REP ch h
REP h ch
REP sz rz
REP ść ć
SFX A Y 1
SFX A a ie a
SFX B Y 3
SFX B y ego y
SFX B i iego i
SFX B i iej i
PFX N Y 1
PFX N 0 nie .
";
/// Stems (the count line first, as in Hunspell files).
pub const DIC: &str = "69
źdźbło
gżegżółka
chrząszcz
chleb
chłep
chlap
chlip
chlup
chlej
pięćdziesięciu
rzeka
którzy
żółw
zażółć
gęślą
jaźń
konstytucja
konstytucyjny/B
niebieski/B
niemiecki/B
ładnie/N
przyszliśmy
zrobilibyśmy
Warszawa/A
kot
biało
czerwony
polsko
mail
w
się
huśtawka
brzoza
przyjaciel
wziąć
dziewczyna
na
pewno
ogóle
nie
wiem
płynie
a
i
o
to
że
tak
jest
brzmi
trzcina/A
napisał
poszedł
spać
bo
wylała
świat
dom
ma
ala
kota
dzień
dobry
jutro
bardzo
dobrze
mamy
czas
ten
";
/// Make [`AFF`]/[`DIC`] the Polish dictionary (every test that needs one calls this; they all
/// install the same one, so tests running in parallel agree).
pub fn install() {
if let Err(e) = super::install_dictionary(AFF, DIC) {
panic!("test dictionary: {e}");
}
}
crates/proof/src/pt/grammar.rs
+251
View File
@@ -0,0 +1,251 @@
//! Portuguese grammar and punctuation checks: cheap, rule-based and tuned to stay quiet when
//! unsure (a missed mistake costs less than a wrong mark on correct text). Messages are in
//! Portuguese.
//!
//! - **Repeated word** (`ele ele`), except interjections and words Portuguese doubles on purpose
//! (`sim sim`, `não não`, `ora ora`).
//! - **Space before punctuation** (`bem ,`) and **double spaces**; emoticons (` :)`) and
//! ellipses are left alone.
//! - **Capital letter at the start of a sentence**, knowing Portuguese abbreviations (`etc.`,
//! `pág.`, `Prof.`, `Dr.`), initials, ordinal numbers (`5. colocado`), ellipses and dialogue
//! dashes (`— Vai — disse`), after which lower case is correct.
//! - **Doubled punctuation** (`,,`, `..`, `;;`).
//!
//! No comma rule (`que` opens clauses only sometimes) and no English rule (such as `a`/`an`)
//! applies to Portuguese text.
use crate::{Issue, IssueKind, ProofLang, words};
/// Words Portuguese repeats on purpose: interjections, onomatopoeia and emphatic doubling.
const ALLOWED_REPEATS: &[&str] = &[
"sim", "não", "ora", "olha", "já", "bem", "muito", "vai", "lá", "cá", "aí", "ha", "he", "hi", "oh", "ah", "ai", "ui", "uf", "uxa", "oba", "viva",
"psiu", "olá", "tchau", "xau", "adeus", "devagar",
];
/// Abbreviations written with a final period after which a sentence goes on in lower case
/// (lower case, without the period).
const ABBREVIATIONS: &[&str] = &[
"etc", "ex", "pág", "págs", "séc", "sécs", "fl", "av", "vol", "p", "pp", "ed", "tel", "min", "max", "art", "par", "prof", "profa", "dr", "dra",
"sr", "sra", "srta", "eng", "arq", "op", "cit", "cf", "ibid", "id", "vide", "seg", "ter", "qua", "qui", "sex", "sáb", "dom", "jan", "fev", "mar",
"abr", "mai", "jun", "jul", "ago", "set", "out", "nov", "dez", "s.a", "d.c", "a.c", "pg", "ref", "cod", "dept", "univ", "inst", "lat", "ing",
"esp", "fr", "it", "al", "obs", "att", "obs",
];
fn issue(start: usize, end: usize, message: String, suggestions: Vec<String>) -> Issue {
Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::Pt }
}
fn capitalize(w: &str) -> String {
crate::affix::capitalize(w)
}
/// Is `prev`, before a period, an abbreviation, an initial, a number or a Roman numeral (after
/// which a lower-case word doesn't start a sentence)?
fn abbreviation_or_number(prev: &str) -> bool {
prev.chars().count() <= 1
|| prev.chars().any(|c| c.is_ascii_digit())
|| (prev.chars().all(|c| matches!(c, 'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M')))
|| ABBREVIATIONS.contains(&prev.to_lowercase().as_str())
}
const CLOSING: [char; 8] = ['"', '”', '’', '\'', ')', '»', ']', '\u{1}'];
const OPENING: [char; 8] = ['"', '“', '«', '(', '[', '\'', '‚', '—'];
/// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after
/// an abbreviation), then space, with nothing but quotes and brackets around?
fn ends_sentence(gap: &str, prev: &str) -> bool {
let t = gap.trim_start_matches(CLOSING);
let Some(end) = t.chars().next() else { return false };
if !matches!(end, '.' | '!' | '?') || t.starts_with("..") {
return false;
}
if end == '.' && abbreviation_or_number(prev) {
return false;
}
let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING);
rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| OPENING.contains(&c))
}
/// Grammar issues in a paragraph of Portuguese text, in order.
pub fn check(text: &str) -> Vec<Issue> {
check_with(text, true)
}
/// [`check`] where `capital_first` says whether the text's first word must start a sentence with a capital
/// (false for a list item, which may be a fragment).
pub fn check_with(text: &str, capital_first: bool) -> Vec<Issue> {
let ws: Vec<(usize, usize)> = words(text);
let mut v = Vec::new();
repeated_words(text, &ws, &mut v);
spaces(text, &mut v);
sentence_capitals(text, &ws, capital_first, &mut v);
doubled_punctuation(text, &mut v);
v.sort_by_key(|i| (i.start, i.end));
v
}
fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
for pair in ws.windows(2) {
let [(a0, b0), (a1, b1)] = pair else { continue };
let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue };
if !gap.is_empty()
&& gap.chars().all(char::is_whitespace)
&& w0.chars().all(char::is_alphabetic)
&& w0.to_lowercase() == w1.to_lowercase()
&& !ALLOWED_REPEATS.contains(&w0.to_lowercase().as_str())
{
v.push(issue(*a0, *b1, format!("Palavra repetida: \"{w1}\""), vec![w0.to_string()]));
}
}
}
fn spaces(text: &str, v: &mut Vec<Issue>) {
for (i, c) in text.char_indices() {
if c != ' ' {
continue;
}
let rest = text.get(i + 1..).unwrap_or("");
let mut it = rest.chars();
let Some(p) = it.next() else { continue };
let after = it.next();
let flag = match p {
',' => true,
// `;)` and `;-)` are emoticons.
';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')),
// ` ...` is an ellipsis, ` .5` a number.
'.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
// ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space.
':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')),
_ => false,
};
if flag {
v.push(issue(i, i + 1 + p.len_utf8(), "Espaço antes da pontuação".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Espaço duplo".into(), vec![" ".into()]));
}
}
fn sentence_capitals(text: &str, ws: &[(usize, usize)], capital_first: bool, v: &mut Vec<Issue>) {
// List items (`– maçãs,`) may start in lower case.
let list_item = text.trim_end().ends_with([',', ';']);
for (k, &(a, b)) in ws.iter().enumerate() {
let Some(w) = text.get(a..b) else { continue };
let starts = if k == 0 {
let before = text.get(..a).unwrap_or("");
capital_first && !list_item && before.chars().all(|c| c.is_whitespace() || OPENING.contains(&c))
} else {
let Some(&(pa, pb)) = ws.get(k - 1) else { continue };
ends_sentence(text.get(pb..a).unwrap_or(""), text.get(pa..pb).unwrap_or(""))
};
if starts && w.chars().next().is_some_and(char::is_lowercase) && w.chars().all(char::is_alphabetic) {
v.push(issue(a, b, "Letra maiúscula no início da frase".into(), vec![capitalize(w)]));
}
}
}
fn doubled_punctuation(text: &str, v: &mut Vec<Issue>) {
let bytes = text.as_bytes();
let mut i = 0;
while let Some(&b) = bytes.get(i) {
if !matches!(b, b',' | b'.' | b';') {
i += 1;
continue;
}
let run = bytes.get(i..).map_or(0, |r| r.iter().take_while(|&&x| x == b).count());
let flag = match b {
b'.' => run == 2,
_ => run >= 2,
};
if flag {
let end = i + run;
let suggestions: Vec<String> = match b {
b'.' => vec![".".into(), "…".into()],
_ => vec![char::from(b).to_string()],
};
v.push(issue(i, end, "Sinal repetido".into(), suggestions));
}
i += run.max(1);
}
}
#[cfg(test)]
mod tests {
use super::*;
fn messages(t: &str) -> Vec<String> {
check(t).into_iter().map(|i| format!("{}@{}", i.message, &t[i.start..i.end])).collect()
}
fn clean(t: &str) {
let m = messages(t);
assert!(m.is_empty(), "{t:?}: {m:?}");
}
#[test]
fn repeated_words() {
let v = check("Eu eu vi isso e ele ele saiu.");
assert_eq!(v.len(), 2, "{v:?}");
assert_eq!(v[0].message, "Palavra repetida: \"eu\"");
assert_eq!(v[0].suggestions, ["Eu"]);
assert!(v.iter().all(|i| i.lang == ProofLang::Pt && i.kind == IssueKind::Grammar));
clean("Sim, sim! Não, não! Ora ora, que surpresa.");
}
#[test]
fn spaces_and_punctuation() {
let m = messages("Olá , mundo . Tudo bem ?");
assert!(m.contains(&"Espaço antes da pontuação@ ,".to_string()), "{m:?}");
assert!(m.contains(&"Espaço antes da pontuação@ .".to_string()), "{m:?}");
assert!(m.contains(&"Espaço antes da pontuação@ ?".to_string()), "{m:?}");
assert!(m.contains(&"Espaço duplo@ ".to_string()), "{m:?}");
clean("Legal :) Muito bom :-) Ou :D Valor .5 e reticências ...");
let m = messages("Falou,, e saiu.. depois;; fim.");
assert_eq!(m.iter().filter(|x| x.starts_with("Sinal repetido")).count(), 3, "{m:?}");
clean("Esperei… e esperei... até ele voltar.");
}
#[test]
fn sentence_start_capitals() {
let m = messages("isso é um teste. a segunda frase! terceira? ok.");
assert_eq!(m.iter().filter(|x| x.starts_with("Letra maiúscula")).count(), 4, "{m:?}");
assert_eq!(check("ele tem razão.")[0].suggestions, ["Ele"]);
// Abbreviations, initials, ordinals, ellipses and dialogue dashes.
clean("Comprei frutas, etc. e depois saímos.");
clean("Conheci o Dr. Silva na Av. Paulista.");
clean("Zhou ficou em 5. lugar no séc. XX.");
clean("Esperei... e ele apareceu.");
clean("— Vai — disse ele. — Amanhã eu volto.");
clean("Ele disse: \"Volto amanhã\". E saiu.");
// A list item may start in lower case.
clean("maçãs,");
}
#[test]
fn correct_portuguese_text_is_clean() {
clean(
"O rato roeu a roupa do rei de Roma. Quem não tem cão caça com gato, \
e quem tem paciência colhe o que plantou.",
);
clean("Brasília, 12 de outubro de 2026 — uma data memorável.");
}
proptest::proptest! {
#[test]
fn never_panics(s in "\\PC{0,80}") {
let _ = check(&s);
}
#[test]
fn never_panics_on_portuguese_like_text(s in "[a-záàâãéêíóôõúüçA-Z ,.;:!?\"…\\-—]{0,60}") {
for i in check(&s) {
proptest::prop_assert!(i.start <= i.end && i.end <= s.len());
proptest::prop_assert!(s.is_char_boundary(i.start) && s.is_char_boundary(i.end));
}
}
}
}
crates/proof/src/pt/mod.rs
+201
View File
@@ -0,0 +1,201 @@
//! Portuguese proofing.
//!
//! - **Spelling** against the Hunspell `pt_BR` dictionary installed on the system
//! ([`dictionary`], found by [`crate::hunspell`]: `pt_BR.aff`/`pt_BR.dic` — VERO and its
//! derivatives — from the OS, a package manager or LibreOffice; it isn't bundled). Without it
//! Portuguese text gets no spelling marks. Capitalized stems are names (`Brasília`); a sentence
//! may start with any word and headings may be in capitals. Hyphenated compounds are checked
//! part by part (`guarda-roupa`, `ex-marido`), and the user dictionary applies as in English.
//! - **Suggestions** tuned for Portuguese ([`suggest`]): missing accents first (`coraçao` →
//! `coração`, `saude` → `saúde`), then the dictionary's `REP` confusions (`seje` → `seja`), a
//! missing space, one-letter typos, and a dictionary scan for anything further off.
//! - **Grammar** ([`grammar`]): cheap, reliable rules with Portuguese messages.
pub mod grammar;
mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// File names of a Portuguese Hunspell dictionary.
const NAMES: &[&str] = &["pt_BR", "pt-BR", "pt"];
/// The Portuguese dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Portuguese");
/// The Portuguese dictionary: the installed Hunspell `pt_BR` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Portuguese spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
DICT.get()
}
/// Use this Portuguese dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
DICT.install(aff, dic)
}
/// Is a Portuguese dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
DICT.has_dictionary()
}
/// Is `word` spelled correctly in Portuguese? Numbers, single letters, short words in capitals
/// (acronyms: `USP`, `PT`), words with digits and words in other scripts are accepted, and so
/// is every word when no Portuguese dictionary is installed.
pub fn is_correct(word: &str) -> bool {
is_correct_with(&dictionary(), word)
}
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off,
/// short words in capitals are checked against the dictionary too, and with Ignore words that
/// contain numbers off, so are words with digits.
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
check(&dictionary(), word, opts)
}
/// [`is_correct`] against a given dictionary (an empty one accepts everything).
pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
check(d, word, &crate::Options::default())
}
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
};
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
}
known(d, w)
}
/// Is one word (no hyphens) in the dictionary or the user dictionary?
fn known(d: &AffixDict, w: &str) -> bool {
d.check(w) || crate::in_user_dictionary(&w.to_lowercase())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_dictionary_loads() {
test_dict::install();
let d = dictionary();
assert!(has_dictionary());
assert!(d.len() > 50, "{}", d.len());
assert!(d.rule_count() >= 5, "the test dictionary's affix rules");
assert!(!d.rep().is_empty() && !d.map().is_empty() && !d.try_chars().is_empty());
}
#[test]
fn without_a_dictionary_nothing_is_flagged() {
let none = AffixDict::default();
for w in ["coraçao", "saude", "xqzv", ""] {
assert!(is_correct_with(&none, w), "{w}");
}
assert!(install_dictionary("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused");
}
#[test]
fn portuguese_words_are_correct() {
test_dict::install();
for w in [
"coração",
"saudade",
"pássaro",
"saúde",
"árvore",
"escrever",
"computador",
"praia",
"conhecer",
"Brasília",
"São",
"autoestima",
"frequência",
"exceção",
"amanhã",
"vocês",
"coragem",
"parabéns",
"cantava",
"cantando",
"provável",
"prováveis",
"descobrir",
"trabalho",
"CORAGEM",
"USP",
"2026",
"e-mail",
"guarda-roupa",
"ex-marido",
"vice-presidente",
"e",
"não",
] {
assert!(is_correct(w), "{w}");
}
}
#[test]
fn misspellings_are_flagged() {
test_dict::install();
for w in ["coraçao", "saude", "trbalho", "excrever", "prsidente", "frequencia", "amnha", "cantaando"] {
assert!(!is_correct(w), "{w}");
}
}
#[test]
fn forbidden_words_are_rejected() {
test_dict::install();
// `frequencia/F` is listed with FORBIDDENWORD: the old spelling, never correct.
assert!(!is_correct("frequencia") && !is_correct("Frequencia") && !is_correct("FREQUENCIA"));
assert!(is_correct("frequência"));
}
#[test]
fn user_dictionary_applies() {
test_dict::install();
assert!(!is_correct("zxqportuguespalavra"));
crate::add_word("Zxqportuguespalavra");
assert!(is_correct("zxqportuguespalavra"));
}
#[test]
fn user_dictionary_accepts_a_hyphenated_word_whole() {
test_dict::install();
let text = "Ele disse que está na Unigranrio-RJ.";
assert_eq!(crate::check_spelling_in(text, crate::ProofLang::Pt).len(), 1, "Unigranrio isn't in the dictionary");
// Added as the context menu adds it, hyphen and all (#2082).
crate::add_word("Unigranrio-RJ");
assert!(crate::check_spelling_in(text, crate::ProofLang::Pt).is_empty());
}
/// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Portuguese checks
/// those words against the dictionary instead of accepting them.
#[test]
fn proofing_options_reach_the_portuguese_check() {
test_dict::install();
let lang = crate::ProofLang::Pt;
let defaults = crate::Options::default();
let check_caps = crate::Options { ignore_uppercase: false, ..defaults };
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
assert!(crate::is_correct_in_with("PRAYA", lang, &defaults), "short capitals are acronyms by default");
assert!(!crate::is_correct_in_with("PRAYA", lang, &check_caps));
assert!(crate::is_correct_in_with("PRAIA", lang, &check_caps));
assert!(crate::is_correct_in_with("praya2", lang, &defaults));
assert!(!crate::is_correct_in_with("praya2", lang, &check_digits));
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
assert!(is_correct("PRAYA"), "direct checks keep the defaults");
}
}
crates/proof/src/pt/suggest.rs
+122
View File
@@ -0,0 +1,122 @@
//! Portuguese spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with
//! the words Portuguese runs into their neighbours (`nãoé` → `não é`, `paraela` → `para ela`) and
//! letters to try for a dictionary without a TRY line.
use crate::affix::AffixDict;
use crate::affix_suggest::{Split, Tuning};
/// Short words often run into the next one (`não é`, `da água`, `para isso`).
const LEADING_WORDS: &[&str] = &[
"a", "o", "e", "é", "de", "do", "da", "dos", "das", "em", "no", "na", "nos", "nas", "por", "para", "com", "sem", "sob", "sobre", "ao", "aos",
"à", "às", "que", "se", "não", "como", "mas", "mais", "já", "até", "entre", "contra", "perante", "desde", "após", "ante", "durante", "mediante",
"segundo", "conforme", "salvo", "fora", "onde", "quando", "bem", "mal", "pois", "logo", "portanto", "porém", "todavia", "contudo", "assim",
"tão", "tanto", "tudo", "nada", "algo", "cada", "outro", "mesmo", "próprio", "ainda", "tampouco", "quase", "talvez", "nunca", "sempre",
];
/// Short words often run into the one before (`amo-te`, `deu-lhe`; in writing without the
/// hyphen: `amote`, `deulhe`).
const TRAILING_WORDS: &[&str] = &[
"me", "te", "se", "lhe", "nos", "vos", "lhes", "o", "a", "os", "as", "ele", "ela", "eles", "elas", "isso", "isto", "aquilo", "mim", "ti", "si",
"eu", "tu", "que", "de", "em", "é",
];
/// What Portuguese adds to its dictionary's tables.
const TUNING: Tuning = Tuning {
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS },
alphabet: "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw",
..Tuning::DEFAULT
};
/// Spelling suggestions for a Portuguese word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
suggest_in(&super::dictionary(), word, max)
}
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::affix_suggest::{fold, weighted_distance};
fn has(word: &str, want: &str) {
let s = suggest(word, 8);
assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}");
}
#[test]
fn nosuggest_stems_and_forms_are_never_offered() {
crate::affix::tests::assert_nosuggest_is_honoured(suggest_in);
}
#[test]
fn portuguese_misspellings_get_the_right_suggestion() {
crate::pt::test_dict::install();
has("coraçao", "coração");
has("saude", "saúde");
has("seje", "seja");
has("excessão", "exceção");
has("previlégio", "privilégio");
has("prsidente", "presidente");
has("provavel", "provável");
has("trbalho", "trabalho");
has("nãoé", "não é");
has("paraela", "para ela");
has("brasília", "Brasília");
has("Coraçao", "Coração");
}
#[test]
fn the_likeliest_fix_comes_first() {
crate::pt::test_dict::install();
assert_eq!(suggest("coraçao", 3).first().map(String::as_str), Some("coração"));
assert_eq!(
suggest("saude", 3).first().map(String::as_str),
Some("saúde"),
"an accent at the first difference beats one further right (saudê is a word too)"
);
assert_eq!(suggest("seje", 3).first().map(String::as_str), Some("seja"));
}
#[test]
fn edge_cases_stay_empty_or_bounded() {
crate::pt::test_dict::install();
assert!(suggest("", 5).is_empty());
assert!(suggest("bem", 0).is_empty());
assert!(suggest(&"a".repeat(41), 5).is_empty());
assert!(suggest("duas palavras", 5).is_empty());
assert!(suggest_in(&crate::affix::AffixDict::default(), "coraçao", 5).is_empty());
assert!(suggest("coração", 5).len() <= 5);
assert!(!suggest("coração", 5).contains(&"coração".to_string()));
}
#[test]
fn distance_weighs_accents_half() {
let map = vec![vec!['a', 'á', 'à', 'â', 'ã'], vec!['c', 'ç']];
let related = |a: char, b: char| fold(a, &map) == fold(b, &map);
let c = |s: &str| s.chars().collect::<Vec<_>>();
assert_eq!(weighted_distance(&c("coraçao"), &c("coração"), &related, 9), 1);
assert_eq!(weighted_distance(&c("coracao"), &c("coração"), &related, 9), 2, "ç and ã are two accent slips");
assert_eq!(weighted_distance(&c("coro"), &c("coro"), &related, 9), 0);
assert_eq!(weighted_distance(&c("bem"), &c("bom"), &related, 9), 2);
assert_eq!(weighted_distance(&c("bem"), &c("bemte"), &related, 9), 4);
assert_eq!(weighted_distance(&c("bem"), &c("ver"), &related, 2), 3);
assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4);
}
#[test]
fn suggestions_are_fast_enough() {
crate::pt::test_dict::install();
// Warm the dictionary, then time a worst case (a long word nothing near): the scan round.
let _ = suggest("bem", 1);
let t = std::time::Instant::now();
for w in ["xqzvweportuguespalavra", "prsidentes", "paralelepípedo"] {
let _ = suggest(w, 6);
}
let ms = t.elapsed().as_secs_f64() * 1000.0;
eprintln!("3 suggestion calls: {ms:.0} ms");
assert!(ms < 20_000.0, "{ms} ms");
}
}
crates/proof/src/pt/test_dict.rs
+121
View File
@@ -0,0 +1,121 @@
//! A tiny synthetic Portuguese Hunspell dictionary for tests (written for these tests; not taken
//! from any real dictionary). Real dictionaries are installed on the system, never bundled.
//!
//! It exercises what VERO-like dictionaries use: `FLAG UTF-8` (multi-byte flag characters —
//! `F` marks a forbidden spelling), continuation classes (`provável` → `prováveis`), the
//! suggestion tables (`TRY`, `MAP` for accents, `REP` for common confusions) and one prefix.
pub const AFF: &str = "SET UTF-8
FLAG UTF-8
FORBIDDENWORD F
TRY aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw
MAP 6
MAP aáàâã
MAP eéê
MAP ií
MAP oóôõ
MAP uú
MAP cç
REP 4
REP seje seja
REP excessão exceção
REP previlégio privilégio
REP analize análise
SFX A Y 2
SFX A a ava a
SFX A a ando a
SFX B N 1
SFX B 0 ável/C .
SFX C N 1
SFX C vel veis vel
PFX X Y 1
PFX X 0 des .
";
/// Stems (the count line first, as in Hunspell files).
pub const DIC: &str = "76
coração
saudade
saudê
pássaro
saúde
árvore
escrever
computador
praia
conhecer
Brasília
São
Paulo
guarda
roupa
mail
marido
ex
vice
presidente
autoestima
frequência
frequencia/F
exceção
privilégio
análise
seja
trabalho
canta/A
prov/B
cobrir/X
avião
pássaros
amanhã
vocês
coragem
parabéns
está
ele
disse
existem
bem
não
sim
que
com
para
dos
das
pela
pelo
muito
mais
como
porque
aqui
também
já
ser
ter
ir
ver
dar
e
o
é
água
ela
da
do
de
em
no
na
um
uma
";
/// Make [`AFF`]/[`DIC`] the Portuguese dictionary (every test that needs one calls this; they all
/// install the same one, so tests running in parallel agree).
pub fn install() {
if let Err(e) = super::install_dictionary(AFF, DIC) {
panic!("test dictionary: {e}");
}
}
crates/proof/src/ru/extra.rs
+218
View File
@@ -0,0 +1,218 @@
//! Our supplement of Russian words older Hunspell dictionaries lack (`маркетплейс`, `промокод`,
//! `интеграция`, `бонусный`…), from craft-i18n's `language-data/ru/ru-extra.txt`: each base form with its
//! declension, expanded here into every form.
use std::collections::HashSet;
use std::sync::OnceLock;
use crate::data::EXTRA_RU;
/// Every form of the supplement: common words (lower case) and names (lower case too, but they
/// only match a word written with a capital).
struct Extra {
words: HashSet<String>,
names: HashSet<String>,
}
fn extra() -> &'static Extra {
static E: OnceLock<Extra> = OnceLock::new();
E.get_or_init(|| {
let mut e = Extra { words: HashSet::new(), names: HashSet::new() };
for line in EXTRA_RU.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')) {
let (base, class) = line.split_once('\t').map_or((line, ""), |(b, c)| (b.trim(), c.trim()));
let name = base.chars().next().is_some_and(char::is_uppercase);
let set = if name { &mut e.names } else { &mut e.words };
for f in forms(base, class) {
set.insert(f.to_lowercase());
}
}
e
})
}
/// Is `w` a form of a supplement word (a name only when written with a capital)?
pub(super) fn contains(w: &str) -> bool {
let e = extra();
let lower = w.to_lowercase();
e.words.contains(&lower) || (w.chars().next().is_some_and(char::is_uppercase) && e.names.contains(&lower))
}
fn is_velar(c: char) -> bool {
matches!(c, 'г' | 'к' | 'х')
}
fn is_hushing(c: char) -> bool {
matches!(c, 'ж' | 'ш' | 'ч' | 'щ')
}
/// The forms of `base` declined as `class` (see the supplement file's header).
fn forms(base: &str, class: &str) -> Vec<String> {
let with = |stem: &str, ends: &[&str]| ends.iter().map(|e| format!("{stem}{e}")).collect::<Vec<_>>();
let last = |s: &str| s.chars().last().unwrap_or(' ');
if let Some(list) = class.strip_prefix('=') {
let mut v: Vec<String> = list.split_whitespace().map(str::to_string).collect();
v.push(base.to_string());
return v;
}
match class {
"m" => {
let c = last(base);
let plural = if is_velar(c) || is_hushing(c) { "и" } else { "ы" };
let gen_pl = if is_hushing(c) { "ей" } else { "ов" };
let mut v = with(base, &["", "а", "у", "ом", "е", plural, gen_pl, "ам", "ами", "ах"]);
if is_hushing(c) || c == 'ц' {
v.push(format!("{base}ем"));
}
v
}
"f" => {
let Some(stem) = base.strip_suffix('а') else { return vec![base.to_string()] };
let c = last(stem);
let plural = if is_velar(c) || is_hushing(c) { "и" } else { "ы" };
let mut v = with(stem, &["а", plural, "е", "у", "ой", "ою", "", "ам", "ами", "ах"]);
if is_hushing(c) || c == 'ц' {
v.extend(with(stem, &["ей", "ею"]));
}
v
}
"fia" => {
base.strip_suffix("ия").map_or_else(|| vec![base.to_string()], |s| with(s, &["ия", "ии", "ию", "ией", "иею", "ий", "иям", "иями", "иях"]))
}
"fs" => base.strip_suffix('ь').map_or_else(|| vec![base.to_string()], |s| with(s, &["ь", "и", "ью", "ей", "ям", "ями", "ях"])),
"no" => base.strip_suffix('о').map_or_else(|| vec![base.to_string()], |s| with(s, &["о", "а", "у", "ом", "е", "", "ам", "ами", "ах"])),
"ovat" => {
// Verbs in -овать (`колеровать`, `администрировать`): the conjugation, the past, the
// imperative and the participles.
let Some(stem) = base.strip_suffix("овать") else { return vec![base.to_string()] };
let mut v = with(
stem,
&[
"овать",
"ую",
"уешь",
"ует",
"уем",
"уете",
"уют",
"овал",
"овала",
"овало",
"овали",
"уя",
"уй",
"уйте",
"уется",
"уются",
"уюсь",
"уешься",
"уемся",
"уетесь",
"овался",
"овалась",
"овалось",
"овались",
"оваться",
"овав",
"овавшись",
],
);
for participle in ["уемый", "ованный", "ующий", "овавший", "ующийся", "овавшийся"] {
let adj = format!("{stem}{participle}");
if let Some(reflexive) = adj.strip_suffix("ся") {
v.extend(forms(reflexive, "adj").into_iter().map(|f| format!("{f}ся")));
} else {
v.extend(forms(&adj, "adj"));
}
}
v.push(format!("{stem}ован"));
v.extend(with(stem, &["ована", "овано", "ованы"]));
v
}
"nie" => {
base.strip_suffix("ие").map_or_else(|| vec![base.to_string()], |s| with(s, &["ие", "ия", "ию", "ием", "ии", "ий", "иям", "иями", "иях"]))
}
"adj" => {
if let Some(stem) = base.strip_suffix("ый").or_else(|| base.strip_suffix("ой")) {
let mut v = with(stem, &["ого", "ому", "ым", "ом", "ая", "ой", "ую", "ое", "ые", "ых", "ыми"]);
v.push(base.to_string());
return v;
}
let Some(stem) = base.strip_suffix("ий") else { return vec![base.to_string()] };
let c = last(stem);
let ends: &[&str] = if is_velar(c) {
&["ий", "ого", "ому", "им", "ом", "ая", "ой", "ую", "ое", "ие", "их", "ими"]
} else if is_hushing(c) {
&["ий", "его", "ему", "им", "ем", "ая", "ей", "ую", "ее", "ие", "их", "ими"]
} else {
&["ий", "его", "ему", "им", "ем", "яя", "ей", "юю", "ее", "ие", "их", "ими"]
};
with(stem, ends)
}
_ => vec![base.to_string()],
}
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn supplement_words_decline() {
for w in [
"маркетплейс",
"маркетплейсов",
"маркетплейсах",
"Маркетплейсами",
"чекбоксы",
"промокода",
"кешбэки",
"инфографики",
"оплат",
"отгрузок",
"интеграций",
"соцсетей",
"согласий",
"бонусных",
"газоиспользующего",
"колеруется",
"заколерованный",
"онлайн",
"чат-ботом",
"Яндекса",
"Битриксе",
] {
assert!(contains(w), "{w}");
}
// Names need their capital; nonsense isn't in.
assert!(!contains("яндекс"));
assert!(!contains("маркетплейсыы"));
}
#[test]
fn declension_classes() {
assert!(forms("бренд", "m").contains(&"брендов".to_string()));
assert!(forms("блог", "m").contains(&"блоги".to_string()));
assert!(forms("мерч", "m").contains(&"мерчей".to_string()));
assert!(forms("книга", "f").contains(&"книги".to_string()));
assert!(forms("синий", "adj").contains(&"синяя".to_string()));
assert!(forms("русский", "adj").contains(&"русская".to_string()));
assert!(forms("свежий", "adj").contains(&"свежая".to_string()));
assert_eq!(forms("медиа", ""), ["медиа"]);
let ovat = forms("модерировать", "ovat");
for f in [
"модерирую",
"модерирует",
"модерировали",
"модерируется",
"модерируемого",
"модерированной",
"модерирующая",
"модерирующемуся",
"модерирован",
] {
assert!(ovat.contains(&f.to_string()), "{f}");
}
assert!(forms("юрлицо", "no").contains(&"юрлиц".to_string()));
}
}
crates/proof/src/ru/grammar.rs
+712
View File
@@ -0,0 +1,712 @@
//! Russian grammar and punctuation checks: cheap, rule-based and tuned to stay quiet when unsure
//! (a missed comma costs less than a wrong mark on correct text). Messages are in Russian.
//!
//! - **Repeated word** (`в в`), except words Russian doubles on purpose (`да да`, `так так`,
//! `ну ну`, `ха ха`, `очень очень`).
//! - **Space before punctuation** (`да ,`) and **double spaces**; emoticons (` :)`) and ellipses
//! are left alone.
//! - **Capital letter at the start of a sentence**, knowing Russian abbreviations (`т. е.`, `г.`,
//! `руб.`, `ул.`, `и т. д.`), initials, numbers, ellipses and dialogue dashes, after which lower
//! case is correct.
//! - **Missing comma** before conjunctions and relative pronouns that always open a clause:
//! `что`, `чтобы`, `поскольку`, `если`, `хотя` (not `хотя бы`) and the forms of `который`, and
//! before `потому`/`оттого` in `потому что`. Not after a conjunction or particle (`и что`,
//! `только что`, `даже если`, `так что`, `для того чтобы`), not after a preposition (`за что`,
//! `в котором`: the comma goes before it) and not inside `что-то`.
//! - **Doubled punctuation** (`,,` — often a typed opening quote — `..`, `;;`).
//!
//! No English rule (such as `a`/`an`) applies to Russian text.
use crate::{Issue, IssueKind, ProofLang, words};
/// Words Russian repeats on purpose: interjections, onomatopoeia and emphatic doubling.
const ALLOWED_REPEATS: &[&str] = &[
"да",
"нет",
"так",
"ну",
"ха",
"хи",
"хе",
"ой",
"ай",
"эй",
"ах",
"ох",
"эх",
"ага",
"угу",
"тук",
"кап",
"бах",
"бум",
"тик",
"так",
"ля",
"очень",
"много",
"давно",
"далеко",
"долго",
"еле",
"чуть",
"тихо",
"быстро",
"скорей",
"скорее",
"иди",
"давай",
"пока",
"ладно",
"вот",
"ещё",
"еще",
"всё",
"все",
"кто",
"что",
"то",
];
/// Abbreviations written with a final period after which a sentence goes on in lower case
/// (lower case, without the period).
const ABBREVIATIONS: &[&str] = &[
"т",
"е",
"д",
"п",
"др",
"пр",
"см",
"ср",
"напр",
"т.е",
"т.д",
"т.п",
"т.к",
"т.н",
"т.о",
"г",
"гг",
"в",
"вв",
"н.э",
"до н.э",
"руб",
"коп",
"тыс",
"млн",
"млрд",
"трлн",
"ул",
"пр-т",
"пер",
"наб",
"пл",
"ш",
"д",
"кв",
"корп",
"стр",
"оф",
"эт",
"под",
"обл",
"р-н",
"пос",
"дер",
"им",
"св",
"проф",
"доц",
"акад",
"канд",
"наук",
"зав",
"зам",
"ред",
"изд",
"сост",
"пер",
"рис",
"табл",
"гл",
"разд",
"ст",
"ч",
"п",
"пп",
"абз",
"прим",
"прил",
"илл",
"с",
"сс",
"т",
"тт",
"вып",
"кн",
"экз",
"шт",
"кг",
"мин",
"сек",
"час",
"мес",
"нед",
"янв",
"фев",
"февр",
"мар",
"апр",
"авг",
"сент",
"окт",
"нояб",
"дек",
"пн",
"вт",
"чт",
"пт",
"сб",
"вс",
"англ",
"нем",
"фр",
"франц",
"лат",
"рус",
"греч",
"итал",
"исп",
"букв",
"перен",
"разг",
"устар",
"ок",
"ок",
"прибл",
"мл",
"ст",
"тел",
"моб",
"факс",
"эл",
"почт",
"вкл",
"искл",
"мм",
"см",
"дм",
"км",
"м",
"л",
"мл",
"т",
"ц",
"млн",
"долл",
"евро",
"др",
"проч",
"etc",
"vs",
];
/// Conjunctions and relative pronouns that open a clause: a comma comes before them.
const CLAUSE_OPENERS: &[&str] = &[
"что",
"чтобы",
"поскольку",
"если",
"хотя",
"который",
"которая",
"которое",
"которые",
"которого",
"которой",
"которому",
"которым",
"котором",
"которую",
"которых",
"которыми",
];
/// Words after which a clause opener takes no comma of its own: conjunctions (`и что`, `а если`),
/// particles (`только что`, `даже если`, `лишь бы`), interrogatives and the first halves of
/// compound conjunctions (`потому что`, `так что`, `для того чтобы`, `с тем чтобы`, `несмотря на
/// то что`, `тогда как`, `как будто`).
const NO_COMMA_AFTER: &[&str] = &[
"и",
"а",
"но",
"да",
"или",
"либо",
"ни",
"не",
"же",
"ли",
"бы",
"б",
"то",
"только",
"лишь",
"именно",
"даже",
"особенно",
"потому",
"оттого",
"поэтому",
"так",
"затем",
"тем",
"того",
"ввиду",
"вследствие",
"несмотря",
"благодаря",
"прежде",
"раньше",
"мере",
"пор",
"чем",
"будто",
"словно",
"точно",
"как",
"если",
"хотя",
"хоть",
"чтобы",
"что",
"кто",
"где",
"когда",
"почему",
"зачем",
"куда",
"откуда",
"сколько",
"разве",
"неужели",
"вот",
"вон",
"ну",
"уже",
"это",
"просто",
"чуть",
"мол",
"ведь",
"вроде",
"тоже",
"также",
"ещё",
"еще",
"всё",
"все",
"почти",
"вовсе",
"ой",
"ах",
"эх",
"ох",
"ага",
"нет",
"конечно",
"наверное",
"может",
"пусть",
"лучше",
"условии",
"случае",
"того",
"ради",
"раз",
"покуда",
"пока",
"после",
"перед",
"лишь",
"едва",
"чуть",
"дабы",
"кабы",
"коли",
"ежели",
"абы",
"вдруг",
"иначе",
"причём",
"причем",
"притом",
"зато",
"однако",
"тогда",
"сейчас",
"теперь",
"сначала",
"сперва",
"иногда",
"часто",
"всегда",
"никогда",
"т",
"др",
];
/// Prepositions: a relative pronoun or `что` after one takes the comma before the preposition
/// instead (`дом, в котором`, `то, за что`).
const PREPOSITIONS: &[&str] = &[
"в",
"во",
"на",
"с",
"со",
"к",
"ко",
"о",
"об",
"обо",
"от",
"ото",
"до",
"из",
"изо",
"за",
"по",
"под",
"подо",
"над",
"надо",
"при",
"про",
"без",
"безо",
"для",
"у",
"через",
"чрез",
"между",
"меж",
"среди",
"из-за",
"из-под",
"около",
"возле",
"вокруг",
"мимо",
"против",
"вместо",
"кроме",
"сквозь",
"вдоль",
"внутри",
"вне",
"после",
"перед",
"пред",
"ради",
"благодаря",
"согласно",
"вопреки",
"навстречу",
"насчёт",
"насчет",
"ввиду",
"вследствие",
"посредством",
"путём",
"путем",
"относительно",
"касательно",
"вслед",
"ко",
"подле",
"близ",
"внутрь",
"поверх",
];
fn issue(start: usize, end: usize, message: String, suggestions: Vec<String>) -> Issue {
Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::Ru }
}
fn capitalize(w: &str) -> String {
crate::affix::capitalize(w)
}
/// Grammar issues in a paragraph of Russian text, in order.
pub fn check(text: &str) -> Vec<Issue> {
check_with(text, true)
}
/// [`check`] where `capital_first` says whether the text's first word must start a sentence with a capital
/// (false for a list item, which may be a fragment).
pub fn check_with(text: &str, capital_first: bool) -> Vec<Issue> {
let ws: Vec<(usize, usize)> = words(text);
let mut v = Vec::new();
repeated_words(text, &ws, &mut v);
spaces(text, &mut v);
sentence_capitals(text, &ws, capital_first, &mut v);
missing_commas(text, &ws, &mut v);
doubled_punctuation(text, &mut v);
v.sort_by_key(|i| (i.start, i.end));
v
}
fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
for pair in ws.windows(2) {
let [(a0, b0), (a1, b1)] = pair else { continue };
let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue };
if !gap.is_empty()
&& gap.chars().all(char::is_whitespace)
&& w0.chars().all(char::is_alphabetic)
&& w0.to_lowercase() == w1.to_lowercase()
&& !ALLOWED_REPEATS.contains(&w0.to_lowercase().as_str())
{
v.push(issue(*a0, *b1, format!("Повтор слова: «{w1}»"), vec![w0.to_string()]));
}
}
}
fn spaces(text: &str, v: &mut Vec<Issue>) {
for (i, c) in text.char_indices() {
if c != ' ' {
continue;
}
let rest = text.get(i + 1..).unwrap_or("");
let mut it = rest.chars();
let Some(p) = it.next() else { continue };
let after = it.next();
let flag = match p {
',' => true,
// `;)` and `;-)` are emoticons.
';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')),
// ` ...` is an ellipsis, ` .5` a number.
'.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
// ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space.
':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '»' | '”' | '’' | '!' | '?')),
_ => false,
};
if flag {
v.push(issue(i, i + 1 + p.len_utf8(), "Лишний пробел перед знаком препинания".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Двойной пробел".into(), vec![" ".into()]));
}
}
/// Is `prev`, before a period, an abbreviation, an initial, a number or a Roman numeral (after
/// which a lower-case word doesn't start a sentence)?
fn abbreviation_or_number(prev: &str) -> bool {
prev.chars().count() <= 1
|| prev.chars().any(|c| c.is_ascii_digit())
|| (prev.chars().all(|c| matches!(c, 'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M')))
|| ABBREVIATIONS.contains(&prev.to_lowercase().as_str())
}
const CLOSING: [char; 8] = ['"', '»', '”', '’', '\'', ')', ']', '\u{1}'];
const OPENING: [char; 8] = ['«', '"', '„', '“', '(', '[', '\'', '‚'];
/// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after
/// an abbreviation), then space, with nothing but quotes and brackets around?
fn ends_sentence(gap: &str, prev: &str) -> bool {
let t = gap.trim_start_matches(CLOSING);
let Some(end) = t.chars().next() else { return false };
if !matches!(end, '.' | '!' | '?') || t.starts_with("..") {
return false;
}
if end == '.' && abbreviation_or_number(prev) {
return false;
}
let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING);
rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| OPENING.contains(&c))
}
fn sentence_capitals(text: &str, ws: &[(usize, usize)], capital_first: bool, v: &mut Vec<Issue>) {
// List items (`– яблоки,`) may start in lower case.
let list_item = !capital_first || text.trim_end().ends_with([',', ';']);
for (k, &(a, b)) in ws.iter().enumerate() {
let Some(w) = text.get(a..b) else { continue };
let starts = if k == 0 {
let before = text.get(..a).unwrap_or("");
!list_item && before.chars().all(|c| c.is_whitespace() || OPENING.contains(&c))
} else {
let Some(&(pa, pb)) = ws.get(k - 1) else { continue };
ends_sentence(text.get(pb..a).unwrap_or(""), text.get(pa..pb).unwrap_or(""))
};
if starts && w.chars().next().is_some_and(char::is_lowercase) && w.chars().all(char::is_alphabetic) {
v.push(issue(a, b, "Предложение начинается со строчной буквы".into(), vec![capitalize(w)]));
}
}
}
fn missing_commas(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
for k in 1..ws.len() {
let (Some(&(pa, pb)), Some(&(a, b))) = (ws.get(k - 1), ws.get(k)) else { continue };
let (Some(prev), Some(w), Some(gap)) = (text.get(pa..pb), text.get(a..b), text.get(pb..a)) else { continue };
if gap.is_empty() || !gap.chars().all(|c| c == ' ' || c == '\u{a0}') || !prev.chars().all(char::is_alphabetic) {
continue;
}
let lw = w.to_lowercase();
let next = ws.get(k + 1).and_then(|&(na, nb)| text.get(na..nb)).map(str::to_lowercase);
let next_gap_plain = ws.get(k + 1).and_then(|&(na, _)| text.get(b..na)).is_some_and(|g| g.chars().all(|c| c == ' ' || c == '\u{a0}'));
// `потому что`, `оттого что`: the comma goes before the first word.
let compound = matches!(lw.as_str(), "потому" | "оттого") && next.as_deref() == Some("что") && next_gap_plain;
if !compound && !CLAUSE_OPENERS.contains(&lw.as_str()) {
continue;
}
// `хотя бы` is a particle (`хотя бы раз`).
if lw == "хотя" && next.as_deref() == Some("бы") {
continue;
}
let p = prev.to_lowercase();
let p = p.as_str();
if NO_COMMA_AFTER.contains(&p) || PREPOSITIONS.contains(&p) {
continue;
}
v.push(issue(pb, b, format!("Возможно, пропущена запятая перед «{w}»"), vec![format!(", {w}")]));
}
}
fn doubled_punctuation(text: &str, v: &mut Vec<Issue>) {
let bytes = text.as_bytes();
let mut i = 0;
while let Some(&b) = bytes.get(i) {
if !matches!(b, b',' | b'.' | b';') {
i += 1;
continue;
}
let run = bytes.get(i..).map_or(0, |r| r.iter().take_while(|&&x| x == b).count());
let flag = match b {
b'.' => run == 2,
_ => run >= 2,
};
if flag {
let end = i + run;
let suggestions: Vec<String> = match b {
// `,,` before a word, after a space: a typed opening quote.
b',' if text.get(end..).and_then(|t| t.chars().next()).is_some_and(char::is_alphabetic)
&& (i == 0 || text.get(..i).is_some_and(|t| t.ends_with(char::is_whitespace))) =>
{
vec!["«".into()]
}
b'.' => vec![".".into(), "…".into()],
_ => vec![char::from(b).to_string()],
};
v.push(issue(i, end, "Повтор знака препинания".into(), suggestions));
}
i += run.max(1);
}
}
#[cfg(test)]
mod tests {
use super::*;
fn messages(t: &str) -> Vec<String> {
check(t).into_iter().map(|i| format!("{}@{}", i.message, &t[i.start..i.end])).collect()
}
fn clean(t: &str) {
let m = messages(t);
assert!(m.is_empty(), "{t:?}: {m:?}");
}
#[test]
fn repeated_words() {
let v = check("Я пошёл в в магазин и и купил хлеб.");
assert_eq!(v.len(), 2, "{v:?}");
assert_eq!(v[0].message, "Повтор слова: «в»");
assert_eq!(v[0].suggestions, ["в"]);
assert!(v.iter().all(|i| i.lang == ProofLang::Ru && i.kind == IssueKind::Grammar));
clean("Да да, так так. Ха ха! Очень очень давно.");
}
#[test]
fn spaces_and_punctuation() {
let m = messages("Мы пришли , а он ушёл . Правда ?");
assert!(m.contains(&"Лишний пробел перед знаком препинания@ ,".to_string()), "{m:?}");
assert!(m.contains(&"Лишний пробел перед знаком препинания@ .".to_string()), "{m:?}");
assert!(m.contains(&"Лишний пробел перед знаком препинания@ ?".to_string()), "{m:?}");
assert!(m.contains(&"Двойной пробел@ ".to_string()), "{m:?}");
clean("Отлично :) Правда :-) Число .5 и многоточие ...");
let m = messages("Он сказал ,,да» и ушёл.. Потом;; конец.");
assert_eq!(m.iter().filter(|x| x.starts_with("Повтор знака")).count(), 3, "{m:?}");
assert_eq!(check("Он сказал ,,да».").iter().find(|i| i.message.starts_with("Повтор знака")).unwrap().suggestions, ["«"]);
clean("Ждал… и ждал... до утра.");
}
#[test]
fn sentence_start_capitals() {
let m = messages("это первое предложение. а это второе! правда? да.");
assert_eq!(m.iter().filter(|x| x.starts_with("Предложение начинается")).count(), 4, "{m:?}");
assert_eq!(check("мама мыла раму.")[0].suggestions, ["Мама"]);
// Abbreviations, initials, numbers, ellipses and dialogue dashes.
clean("Купили фрукты: яблоки, груши и т. д. и овощи.");
clean("В 1990 г. открылась фирма, т. е. наша компания.");
clean("Цена 500 руб. за штуку, адрес ул. Ленина, д. 5, кв. 3.");
clean("Встретил А. С. Пушкина и проф. Иванова.");
clean("Ждал... и наконец дождался.");
clean("– Правда? – спросил он. – Да, – ответила она.");
clean("Он сказал: «Вернусь». Потом ушёл.");
clean("яблоки,");
}
#[test]
fn missing_commas() {
let m = messages(
"Я знаю что он придёт. Это дом который построил Джек. Пришёл чтобы помочь. Ушёл потому что устал. Приходи если сможешь. Пошёл хотя устал.",
);
for want in [
"Возможно, пропущена запятая перед «что»@ что",
"Возможно, пропущена запятая перед «который»@ который",
"Возможно, пропущена запятая перед «чтобы»@ чтобы",
"Возможно, пропущена запятая перед «потому»@ потому",
"Возможно, пропущена запятая перед «если»@ если",
"Возможно, пропущена запятая перед «хотя»@ хотя",
] {
assert!(m.contains(&want.to_string()), "{want}: {m:?}");
}
assert_eq!(check("Знаю что да.")[0].suggestions, [", что"]);
// Conjunctions, particles, compound conjunctions and prepositions take no comma there.
clean("Я знаю, что он придёт, и что останется.");
clean("Он только что пришёл, так что всё в порядке.");
clean("Это дом, в котором я живу, и сад, через который иду.");
clean("Я сделаю это, даже если не успею, лишь бы ты был рад.");
clean("Пришёл для того чтобы помочь, а не чтобы мешать.");
clean("Ушёл, потому что устал, и не потому что обиделся.");
clean("Расскажи хотя бы что-нибудь, за что тебя хвалят.");
clean("Что ты делаешь? Который час?");
clean("Кто-то что-то сказал.");
}
#[test]
fn correct_russian_text_is_clean() {
clean("Съешь же ещё этих мягких французских булок, да выпей чаю. Мы пришли, когда уже стемнело.");
clean("Книга, которую я читаю, очень интересная, поэтому я её советую.");
}
proptest::proptest! {
#[test]
fn never_panics(s in "\\PC{0,80}") {
let _ = check(&s);
}
#[test]
fn never_panics_on_russian_like_text(s in "[а-яёА-ЯЁ ,.;:!?«»…\\-]{0,60}") {
for i in check(&s) {
proptest::prop_assert!(i.start <= i.end && i.end <= s.len());
proptest::prop_assert!(s.is_char_boundary(i.start) && s.is_char_boundary(i.end));
}
}
}
}
crates/proof/src/ru/hyphen.rs
+134
View File
@@ -0,0 +1,134 @@
//! Russian word division, from the rules of Russian spelling (no pattern file):
//!
//! - every part keeps a vowel, and at least two letters stay on each side;
//! - `ь`, `ъ` and `й` stay with the letter before them (`паль-то`, `вой-на`), and a syllable
//! ends after `ъ` (`объ-ём`, `подъ-езд`);
//! - between two vowels with one consonant between them the consonant starts the next syllable
//! (`ма-ма`, `ро-ди-на`);
//! - a cluster of consonants is divided after its first consonant (`сес-тра`, `кар-ман`,
//! `кас-са`, `под-ход`), which also keeps the common prefixes `под-`, `раз-`, `без-` whole;
//! - two vowels side by side may be divided (`по-эт`, `ма-як`).
use super::is_cyrillic;
/// Russian vowels.
fn is_vowel(c: char) -> bool {
matches!(c, 'а' | 'е' | 'ё' | 'и' | 'о' | 'у' | 'ы' | 'э' | 'ю' | 'я')
}
/// Letters that never begin a line: the soft and hard signs and `й`.
fn sticks_left(c: char) -> bool {
matches!(c, 'ь' | 'ъ' | 'й')
}
/// Break points of a lowercase Russian word (letters only): each is the index of the letter that
/// starts a new line. Empty for words with letters outside the Russian alphabet.
pub fn points(c: &[char]) -> Vec<usize> {
let n = c.len();
if n < 4 || !c.iter().all(|&x| is_cyrillic(x) && x.is_alphabetic()) {
return vec![];
}
let vowels: Vec<usize> = (0..n).filter(|&i| c.get(i).is_some_and(|&x| is_vowel(x))).collect();
let mut out = vec![];
for pair in vowels.windows(2) {
let [a, b] = pair else { continue };
let (start, end) = (a + 1, *b);
// The letters that must stay on the left: signs and `й` right after the vowel or after a
// consonant of the cluster.
let mut cut = if start == end {
// Two vowels side by side.
end
} else if c.get(end - 1) == Some(&'ъ') {
// The hard sign closes its syllable (`объ-ём`, `подъ-езд`).
end
} else {
// The cluster's consonants, skipping signs that stick to the letter before.
let consonants: Vec<usize> = (start..end).filter(|&i| c.get(i).is_some_and(|&x| !sticks_left(x))).collect();
match consonants.as_slice() {
// Only `й` (`май-онез`): after it.
[] => end,
// One consonant: it starts the next syllable (`ма-ма`, `паль-то`).
[only] => *only,
// Several: after the first one (`сес-тра`, `кар-ман`).
[_, second, ..] => *second,
}
};
// A sign or `й` never starts a line.
while c.get(cut).is_some_and(|&x| sticks_left(x)) && cut < end {
cut += 1;
}
// Cuts come in increasing order (each lies after the previous pair's vowel), so a repeat
// can only be the last one: no scan of the whole list per syllable (#3657).
if cut >= 2 && n - cut >= 2 && out.last().is_none_or(|&l| l < cut) {
out.push(cut);
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
fn h(w: &str) -> String {
let c: Vec<char> = w.chars().collect();
let p = points(&c);
c.iter().enumerate().map(|(i, ch)| if p.contains(&i) { format!("-{ch}") } else { ch.to_string() }).collect()
}
#[test]
fn russian_words_divide_by_the_rules() {
assert_eq!(h("мама"), "ма-ма");
assert_eq!(h("родина"), "ро-ди-на");
assert_eq!(h("сестра"), "сес-тра");
assert_eq!(h("карман"), "кар-ман");
assert_eq!(h("касса"), "кас-са");
assert_eq!(h("подход"), "под-ход");
assert_eq!(h("пальто"), "паль-то");
assert_eq!(h("объём"), "объ-ём");
assert_eq!(h("подъезд"), "подъ-езд");
assert_eq!(h("семья"), "се-мья");
assert_eq!(h("вьюга"), "вью-га");
assert_eq!(h("объявление"), "объ-яв-ле-ние");
assert_eq!(h("война"), "вой-на");
assert_eq!(h("чайник"), "чай-ник");
assert_eq!(h("поэзия"), "по-э-зия");
assert_eq!(h("маяк"), "ма-як");
assert_eq!(h("жидкие"), "жид-кие");
assert_eq!(h("обои"), "обои");
assert_eq!(h("декоративная"), "де-ко-ра-тив-ная");
assert_eq!(h("штукатурка"), "шту-ка-тур-ка");
}
#[test]
fn short_words_and_other_scripts_stay_whole() {
for w in ["дом", "ёж", "оса", "iPhone", "дом2", ""] {
let c: Vec<char> = w.chars().collect();
assert!(points(&c).is_empty(), "{w}");
}
}
#[test]
fn no_line_starts_with_a_sign_or_ends_with_one_letter() {
for w in ["объявление", "подъезд", "семья", "вьюга", "майонез", "яйцо", "ауди", "аорта", "оазис", "ёлочка"]
{
let c: Vec<char> = w.chars().collect();
for p in points(&c) {
assert!(p >= 2 && c.len() - p >= 2, "{w}: {p}");
assert!(c.get(p).is_some_and(|&x| !sticks_left(x)), "{w}: {p}");
assert!(c.get(..p).is_some_and(|l| l.iter().any(|&x| is_vowel(x))), "{w}: {p}");
assert!(c.get(p..).is_some_and(|r| r.iter().any(|&x| is_vowel(x))), "{w}: {p}");
}
}
}
proptest::proptest! {
#[test]
fn never_panics(s in "[а-яё]{0,30}") {
let c: Vec<char> = s.chars().collect();
for p in points(&c) {
proptest::prop_assert!(p >= 2 && p + 2 <= c.len());
}
}
}
}
crates/proof/src/ru/mod.rs
+486
View File
@@ -0,0 +1,486 @@
//! Russian proofing.
//!
//! - **Spelling** against a Russian Hunspell dictionary installed on the system ([`dictionary`],
//! found by [`crate::hunspell`]: `ru_RU.aff`/`ru_RU.dic` from the OS, a package manager or
//! LibreOffice's `dict-ru`; it isn't bundled). Without it Russian text gets no spelling marks.
//! `ё` is optional, as in ordinary Russian writing: `елка` and `ёлка`, `все` and `всё` are
//! accepted whichever one the dictionary lists. Short words in capitals are abbreviations
//! (`ООО`, `НДС`). Hyphenated words are checked whole first, then by their parts, knowing the
//! particles (`кто-то`, `где-нибудь`, `давай-ка`, `всё-таки`), `кое-`/`кой-`, adverbs with
//! `по-` (`по-русски`, `по-новому`) and compound adjectives whose first part ends in a linking
//! vowel (`северо-западный`, `сине-зелёный`). Compounds written solid are checked the same way
//! (`высокоэластичный`, `газоиспользующий`), and so are adjectives, participles and verbal nouns
//! with `не` (`невскрытый`, `непредоставление`; not verbs: `незнаю` stays wrong). Our supplement
//! adds words older dictionaries lack (`маркетплейс`, `промокод`, `интеграция`: [`extra`]).
//! Words in another script (`iPhone`), with digits or superscripts (`м²`), in camel case
//! (`ВКонтакте`) or abbreviations with an ending (`ГИПа`, `ПДн`) are left alone.
//! - **Suggestions** tuned for Russian ([`suggest()`]): `е`/`ё`, the common confusions (`тся`/
//! `ться`, `жи`/`ши` written with `ы`, `ча`/`ща` with `я`, voiced and voiceless consonant pairs,
//! doubled consonants), a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`), one-letter
//! typos, and a dictionary scan for anything further off.
//! - **Grammar** ([`grammar`]): cheap, reliable rules with Russian messages.
//! - **Hyphenation** ([`hyphen`]): the rules of Russian word division, written as code.
mod extra;
pub mod grammar;
pub mod hyphen;
mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// File names of a Russian Hunspell dictionary.
const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"];
/// The Russian dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Russian");
/// The Russian dictionary: the installed Hunspell `ru_RU` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
DICT.get()
}
/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
DICT.install(aff, dic)
}
/// Is a Russian dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
DICT.has_dictionary()
}
/// Is `c` a letter of the Cyrillic script (the only one the Russian dictionary covers)?
pub(crate) fn is_cyrillic(c: char) -> bool {
matches!(c, '\u{0400}'..='\u{052F}')
}
/// Particles joined to a word with a hyphen (`кто-то`, `где-нибудь`, `давай-ка`, `всё-таки`,
/// `он-де`).
const PARTICLES: &[&str] = &["то", "либо", "нибудь", "ка", "таки", "де", "с"];
/// Prefixes joined to a pronoun with a hyphen (`кое-кто`, `кое-где`, `кой-какой`).
const PRONOUN_PREFIXES: &[&str] = &["кое", "кой"];
/// Words of fewer letters than this written in capitals are taken as abbreviations (`ООО`, `НДС`,
/// `МГУ`, `ГОСТ`).
const ABBREVIATION_LETTERS: usize = 6;
/// Is `word` spelled correctly in Russian? Numbers, single letters, short words in capitals,
/// words with digits and words in other scripts are accepted, and so is every word when no
/// Russian dictionary is installed.
pub fn is_correct(word: &str) -> bool {
is_correct_with(&dictionary(), word)
}
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off,
/// short words in capitals are checked against the dictionary too, and with Ignore words that
/// contain numbers off, so are words with digits or superscripts.
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
check(&dictionary(), word, opts)
}
/// [`is_correct`] against a given dictionary (an empty one accepts everything).
pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
check(d, word, &crate::Options::default())
}
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Russian dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '-');
let w = w.trim_matches('-');
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS {
return true;
}
// Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word.
let cs: Vec<char> = w.chars().collect();
if cs.windows(2).any(|p| matches!(p, [a, b] if a.is_lowercase() && b.is_uppercase())) {
return true;
}
// An abbreviation with lower-case letters or an ending (`ГИПа`, `ПДн`, `ИСПДн`).
if n <= 10 && cs.iter().take(2).all(|c| c.is_uppercase()) && cs.iter().any(|c| c.is_lowercase()) {
return true;
}
// Another script, or a mix of scripts (`iPhone`, `SMS-ка`): not this dictionary's to judge.
if w.chars().any(|c| c.is_alphabetic() && !is_cyrillic(c)) {
return true;
}
if known(d, w) {
return true;
}
if w.contains('-') {
return hyphenated(d, w, opts);
}
solid_compound(d, w)
}
/// Endings of adjectives, participles and nouns that are written together with `не`
/// (`невскрытый`, `неисполнение`, `неготовность`). Verb endings aren't among them: `не` with a
/// verb is written apart (`не знаю`).
const NE_ENDINGS: &[&str] = &[
"ый",
"ий",
"ой",
"ая",
"яя",
"ое",
"ее",
"ые",
"ие",
"ого",
"его",
"ому",
"ему",
"ым",
"ую",
"юю",
"ых",
"их",
"ыми",
"ими",
"ние",
"ния",
"нию",
"нием",
"нии",
"ость",
"ости",
"остью",
"остей",
"остям",
"остями",
"остях",
];
/// Prefixes Russian joins to any word (`ультрастойкий`, `супермаркет`, `экопакет`).
const WORD_PREFIXES: &[&str] = &[
"ультра",
"супер",
"гипер",
"мега",
"микро",
"макро",
"мульти",
"мини",
"макси",
"анти",
"псевдо",
"квази",
"контр",
"сверх",
"полу",
"само",
"авто",
"видео",
"аудио",
"фото",
"радио",
"теле",
"эко",
"био",
"нано",
"агро",
"медиа",
"интернет",
"веб",
"бизнес",
"онлайн",
"спец",
"гос",
"мед",
"юр",
"орг",
"проф",
"соц",
];
/// Prefixes that make a verb perfective (`с-генерировать`, `от-модерировать`, `пере-загрузить`).
const VERB_PREFIXES: &[&str] = &[
"с", "со", "за", "по", "про", "на", "вы", "от", "ото", "пере", "под", "подо", "при", "раз", "рас", "из", "ис", "у", "о", "об", "обо", "до",
"недо", "пред", "в", "во", "вз", "вс",
];
/// Endings of verb forms (infinitive, present, past, reflexive).
const VERB_ENDINGS: &[&str] = &[
"ть", "ться", "ти", "чь", "ю", "ешь", "ет", "ем", "ете", "ют", "ишь", "ит", "им", "ите", "ят", "ал", "ала", "ало", "али", "ил", "ила", "ило",
"или", "ся", "сь",
];
/// A word written solid from known parts: `не` with an adjective, participle or verbal noun
/// (`невскрытой`), or two stems joined by a linking vowel (`высокоэластичная`, `газоиспользующего`).
fn solid_compound(d: &AffixDict, w: &str) -> bool {
let lower = w.to_lowercase();
let chars: Vec<char> = lower.chars().collect();
let n = chars.len();
if !(7..=40).contains(&n) {
return false;
}
if let Some(rest) = lower.strip_prefix("не")
&& rest.chars().count() >= 4
&& NE_ENDINGS.iter().any(|e| rest.ends_with(e))
&& known(d, rest)
{
return true;
}
// A productive prefix (`ультрастойкий`, `экопакет`, `микроцемент`) on a known word.
if WORD_PREFIXES.iter().any(|p| lower.strip_prefix(p).is_some_and(|rest| rest.chars().count() >= 4 && known(d, rest))) {
return true;
}
// A verb with a perfective prefix (`сгенерировать`, `отмодерировали`) whose unprefixed form is
// known.
if VERB_PREFIXES.iter().any(|p| {
lower.strip_prefix(p).is_some_and(|rest| rest.chars().count() >= 5 && VERB_ENDINGS.iter().any(|e| rest.ends_with(e)) && known(d, rest))
}) {
return true;
}
(4..=n.saturating_sub(4)).any(|i| {
let (Some(left), Some(right)) = (chars.get(..i), chars.get(i..)) else { return false };
if !matches!(left.last(), Some('о' | 'е')) {
return false;
}
let left: String = left.iter().collect();
let right: String = right.iter().collect();
known(d, &right) && (known(d, &left) || linking_stem(d, &left))
})
}
/// Is `part`, ending in a linking vowel, a known word's stem (`северо` → `север`, `сине` →
/// `синий`, `газо` → `газ`)?
fn linking_stem(d: &AffixDict, part: &str) -> bool {
let Some(stem) = part.strip_suffix(['о', 'е', 'О', 'Е']) else { return false };
stem.chars().count() >= 2 && ["", "ый", "ий", "ой", "ь"].iter().any(|end| known(d, &format!("{stem}{end}")))
}
/// Is one word in the dictionary or the user dictionary, with `ё` optional?
fn known(d: &AffixDict, w: &str) -> bool {
if d.check(w) || crate::in_user_dictionary(&w.to_lowercase()) || extra::contains(w) {
return true;
}
// `ё` written as `е` is ordinary Russian: try the dictionary's `ё` spelling…
let without = w.replace('ё', "е").replace('Ё', "Е");
if without != w && (d.check(&without) || crate::in_user_dictionary(&without.to_lowercase())) {
return true;
}
// …and `е` where the dictionary has `ё` (one or two of them: `Черемушки`, `еще`).
let e_positions: Vec<usize> = without.char_indices().filter(|(_, c)| matches!(c, 'е' | 'Е')).map(|(i, _)| i).collect();
if e_positions.is_empty() || e_positions.len() > 4 {
return false;
}
let swap = |s: &str, i: usize| -> Option<String> {
let c = s.get(i..)?.chars().next()?;
let yo = if c == 'Е' { "Ё" } else { "ё" };
Some(format!("{}{}{}", s.get(..i)?, yo, s.get(i + c.len_utf8()..)?))
};
for (k, &i) in e_positions.iter().enumerate() {
let Some(one) = swap(&without, i) else { continue };
if d.check(&one) {
return true;
}
for &j in e_positions.get(k + 1..).unwrap_or_default() {
// `ё` is two bytes like `е`, so later offsets still hold.
if swap(&one, j).is_some_and(|two| d.check(&two)) {
return true;
}
}
}
false
}
/// A hyphenated word that isn't in the dictionary whole: its parts, with the particles,
/// prefixes and linking vowels Russian joins with a hyphen.
fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool {
let parts: Vec<&str> = w.split('-').collect();
if parts.iter().any(|p| p.is_empty()) || parts.len() > 4 {
return false;
}
let lower = w.to_lowercase();
let lp: Vec<&str> = lower.split('-').collect();
// `кто-то`, `где-нибудь`, `всё-таки`: a word and its particles.
let first_particle = lp.iter().skip(1).position(|p| PARTICLES.contains(p)).map(|i| i + 1);
if let Some(k) = first_particle
&& lp.iter().skip(k).all(|p| PARTICLES.contains(p))
{
let base = parts.get(..k).map(|b| b.join("-")).unwrap_or_default();
return known(d, &base) || (base.contains('-') && hyphenated(d, &base, opts));
}
match lp.as_slice() {
// `кое-кто`, `кой-какой`.
[prefix, rest] if PRONOUN_PREFIXES.contains(prefix) => return parts.get(1).is_some_and(|r| known(d, r)),
// `по-русски` (from `русский`), `по-новому`, `по-моему`.
["по", rest] => {
let Some(r) = parts.get(1) else { return false };
return known(d, r) || ((rest.ends_with("ски") || rest.ends_with("цки") || rest.ends_with("ьи")) && known(d, &format!("{r}й")));
}
_ => {}
}
// Compounds part by part: `северо-западный`, `сине-зелёный`, `интернет-магазин`, `из-за`.
let last = parts.len() - 1;
parts.iter().enumerate().all(|(i, p)| {
// An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word.
if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) {
return true;
}
// A first part with a linking vowel: the stem (`северо` → `север`) or its adjective
// (`сине` → `синий`, `бело` → `белый`).
i != last && linking_stem(d, p)
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_dictionary_loads() {
test_dict::install();
let d = dictionary();
assert!(has_dictionary());
assert!(d.len() > 50, "{}", d.len());
}
#[test]
fn without_a_dictionary_nothing_is_flagged_and_nothing_suggested() {
let none = AffixDict::default();
for w in ["превет", "сдесь", "xqzv", ""] {
assert!(is_correct_with(&none, w), "{w}");
}
assert!(suggest::suggest_in(&none, "превет", 5).is_empty());
}
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn russian_words_are_correct() {
test_dict::install();
for w in [
"привет",
"Привет",
"ПРИВЕТ",
"дом",
"дома",
"домами",
"красный",
"красного",
"красной",
"читать",
"читаем",
"читаются",
"ёлка",
"елка",
"всё",
"все",
"ещё",
"еще",
"Москва",
"Москве",
"кто-то",
"где-нибудь",
"что-либо",
"давай-ка",
"всё-таки",
"все-таки",
"кое-кто",
"по-русски",
"по-новому",
"северо-западный",
"сине-зелёный",
"сине-зеленый",
"из-за",
"ООО",
"НДС",
"2026",
"5-й",
"iPhone",
"e-mail",
"в",
"м²",
"ВКонтакте",
"МирКрасок",
"ГИПа",
"ПДн",
"ИИ-консультант",
"маркетплейсов",
"промокода",
"невскрытой",
"некрасного",
"непредоставления",
"высокоэластичная",
"краснозелёный",
"Пн-Пт",
"ультракрасный",
"суперновый",
"прочитать",
"администрируется",
"юрлица",
] {
assert!(is_correct(w), "{w}");
}
}
#[test]
fn misspellings_are_flagged() {
test_dict::install();
for w in [
"превет",
"сдесь",
"жыть",
"чясто",
"незнаю",
"нечитаю",
"вобщем",
"москва",
"здраствуй",
"кто-нибуть",
"по-руски",
"северо-зопадный",
"ПРИВЕТТТ",
"высокозопадный",
] {
assert!(!is_correct(w), "{w}");
}
}
#[test]
fn user_dictionary_applies() {
test_dict::install();
assert!(!is_correct("зкщфрусскоеслово"));
crate::add_word("Зкщфрусскоеслово");
assert!(is_correct("зкщфрусскоеслово"));
}
/// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Russian checks
/// those words against the dictionary instead of accepting them.
#[test]
fn proofing_options_reach_the_russian_check() {
test_dict::install();
let lang = crate::ProofLang::Ru;
let defaults = crate::Options::default();
let check_caps = crate::Options { ignore_uppercase: false, ..defaults };
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
assert!(crate::is_correct_in_with("ПРЕВЕТ", lang, &defaults), "short capitals are abbreviations by default");
assert!(!crate::is_correct_in_with("ПРЕВЕТ", lang, &check_caps));
assert!(crate::is_correct_in_with("ПРИВЕТ", lang, &check_caps));
assert!(crate::is_correct_in_with("ИИ-консультант", lang, &defaults));
assert!(!crate::is_correct_in_with("ИИ-консультант", lang, &check_caps), "a part in capitals is checked too");
assert!(crate::is_correct_in_with("превет2", lang, &defaults));
assert!(!crate::is_correct_in_with("превет2", lang, &check_digits));
assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too");
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults");
}
}
crates/proof/src/ru/suggest.rs
+258
View File
@@ -0,0 +1,258 @@
//! Russian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with
//! Russian's own letter groups and confusions, since Russian Hunspell dictionaries carry no `MAP`
//! or `REP` tables (`е`/`ё` first, then unstressed vowels, voiced and voiceless pairs, `жы` →
//! `жи`, `тся`/`ться`, dropped or doubled consonants; up to three letters changed), and its rule
//! for a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`).
use crate::affix::AffixDict;
use crate::affix_suggest::{COST_OTHER_MAP, COST_SPLIT, COST_SPLIT_LONG, Split, Tuning};
/// Letters confused with each other: a slip inside a group is cheaper than another typo.
const MAP: &[&str] = &["её", "оа", "еия", "зс", "дт", "бп", "вф", "гк", "жш", "ьъ", "еэ", "ыи"];
/// Common confusions: a written sequence and what was meant.
const REP: &[(&str, &str)] = &[
("жы", "жи"),
("шы", "ши"),
("чя", "ча"),
("щя", "ща"),
("чю", "чу"),
("щю", "щу"),
("тся", "ться"),
("ться", "тся"),
("сч", "щ"),
("щ", "сч"),
("нн", "н"),
("н", "нн"),
("сс", "с"),
("с", "сс"),
("лл", "л"),
("л", "лл"),
("тт", "т"),
("т", "тт"),
("мм", "м"),
("м", "мм"),
("рр", "р"),
("р", "рр"),
("ье", "ъе"),
("ъе", "ье"),
("раст", "рост"),
("рост", "раст"),
("лож", "лаг"),
("лаг", "лож"),
// Unpronounced consonants: `здраствуй`, `чуствовать`, `солнце`, `лесница`, `праздник`.
("аст", "авст"),
("уст", "увст"),
("лнц", "лц"),
("лц", "лнц"),
("сн", "стн"),
("стн", "сн"),
("зн", "здн"),
("здн", "зн"),
("ц", "тс"),
("тс", "ц"),
];
/// Short words often run into the next one (`не знаю`, `в общем`, `на верху`, `потому что`).
const LEADING_WORDS: &[&str] = &[
"не",
"ни",
"в",
"во",
"на",
"с",
"со",
"к",
"ко",
"о",
"об",
"обо",
"от",
"ото",
"до",
"из",
"изо",
"за",
"по",
"под",
"над",
"при",
"про",
"без",
"для",
"у",
"а",
"и",
"но",
"да",
"же",
"как",
"так",
"то",
"что",
"уже",
"еще",
"ещё",
"все",
"всё",
"потому",
"чтобы",
"это",
];
/// Prepositions among [`LEADING_WORDS`].
const PREPOSITIONS: &[&str] = &[
"в", "во", "на", "с", "со", "к", "ко", "о", "об", "обо", "от", "ото", "до", "из", "изо", "за", "по", "под", "над", "при", "про", "без", "для",
"у",
];
/// Short words often run into the one before (`мог бы`, `он же`, `так ли`).
const TRAILING_WORDS: &[&str] = &["ли", "же", "бы", "ж", "б", "что"];
/// The Russian alphabet, when the dictionary has no `TRY` line.
const ALPHABET: &str = "оеаинтсрвлкмдпуяыьгзбчйхжшюцщэфъё";
/// `е` written for `ё`, or the other way round, is barely a slip.
const COST_YO: u32 = 3;
/// A one- or two-letter preposition run into the next word (`вдоме` → `в доме`).
const COST_SPLIT_PREPOSITION: u32 = 17;
/// What Russian adds to its dictionary's tables.
const TUNING: Tuning =
Tuning { split: Split::Custom(split), alphabet: ALPHABET, map: MAP, rep: REP, rep_cost: 9, max_map_changes: 3, map_cost, ..Tuning::DEFAULT };
/// The cost of changing `from` to `to` within a letter group: `е`/`ё` cheapest.
fn map_cost(from: char, to: char) -> u32 {
if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_OTHER_MAP }
}
/// A missing space: cheap after a short leading word or before a short trailing one, dearer after
/// a short preposition or between two longer words, never leaving a stray short half.
fn split(a: &str, b: &str) -> Option<u32> {
let long = |p: &str| p.chars().count() >= 3;
// A preposition takes a noun, not a verb (`с генерировать` isn't a fix for
// `сгенерировать`); after one or two letters a split is likelier wrong than a typo.
let preposition = PREPOSITIONS.contains(&a);
if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) {
return None;
}
if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) {
Some(COST_SPLIT_PREPOSITION)
} else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
Some(COST_SPLIT)
} else if long(a) && long(b) {
Some(COST_SPLIT_LONG)
} else {
None
}
}
/// Spelling suggestions for a Russian word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
suggest_in(&super::dictionary(), word, max)
}
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
}
#[cfg(test)]
mod tests {
use super::*;
fn has(word: &str, want: &str) {
let s = suggest(word, 8);
assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}");
}
/// Russian's own costs: `е`/`ё` is the cheapest slip, so a `ё` and one more group slip
/// (made-up `нёснаю`) beat a missing space; a REP confusion beats a `ё` and a group slip;
/// a preposition is never split off a verb, and after a one-letter one a typo is likelier.
#[test]
fn russian_costs_rank_its_slips() {
let dict = |words: &[&str]| AffixDict::from_hunspell("", &format!("{}\n{}\n", words.len(), words.join("\n")), &[]).unwrap();
assert_eq!(suggest_in(&dict(&["не", "знаю", "нёснаю"]), "незнаю", 1), ["нёснаю"]);
assert_eq!(suggest_in(&dict(&["еча", "ёчи"]), "ечя", 1), ["еча"]);
assert!(!suggest_in(&dict(&["с", "генерировать"]), "сгенерировать", 8).contains(&"с генерировать".into()));
let s = suggest_in(&dict(&["в", "доме"]), "вдоме", 4);
assert_eq!(s.first().map(String::as_str), Some("доме"), "{s:?}");
assert!(s.contains(&"в доме".into()), "{s:?}");
}
/// A word the dictionary marks NOSUGGEST (a vulgarism, a rare form) is correct but never
/// offered, in any round or letter case. Russian's MAP and REP tables are its own, so the
/// shared Latin-script check doesn't reach it.
#[test]
fn nosuggest_words_are_never_offered() {
let offered = |aff: &str, input: &str| {
let d = AffixDict::from_hunspell(aff, "2\nпривет/N\nпредмет/N\n", &[]).unwrap();
suggest_in(&d, input, 16)
};
assert!(offered("SET UTF-8\n", "превет").contains(&"привет".into()), "offered without the flag declared");
for input in ["превет", "ПРЕВЕТ", "предмот", "приветт", "привте", "привет"] {
let s = offered("SET UTF-8\nNOSUGGEST N\n", input);
assert!(!s.iter().any(|w| w.to_lowercase().contains("привет") || w.to_lowercase().contains("предмет")), "{input}: {s:?}");
}
}
#[test]
fn russian_misspellings_get_the_right_suggestion() {
crate::ru::test_dict::install();
has("превет", "привет");
has("сдесь", "здесь");
has("жыть", "жить");
has("чясто", "часто");
has("незнаю", "не знаю");
has("вобщем", "в общем");
has("здраствуй", "здравствуй");
has("москва", "Москва");
has("Превет", "Привет");
has("ПРЕВЕТ", "ПРИВЕТ");
has("читаеться", "читается");
has("краснного", "красного");
has("западнный", "западный");
assert!(!suggest("сгенерировать", 8).contains(&"с генерировать".to_string()));
}
#[test]
fn the_likeliest_fix_comes_first() {
crate::ru::test_dict::install();
assert_eq!(suggest("превет", 3).first().map(String::as_str), Some("привет"));
assert_eq!(suggest("незнаю", 3).first().map(String::as_str), Some("не знаю"));
assert_eq!(suggest("жыть", 3).first().map(String::as_str), Some("жить"));
assert_eq!(suggest("зеленый", 3).first().map(String::as_str), Some("зелёный"));
}
#[test]
fn edge_cases_stay_empty_or_bounded() {
crate::ru::test_dict::install();
assert!(suggest("", 5).is_empty());
assert!(suggest("дом", 0).is_empty());
assert!(suggest(&"а".repeat(41), 5).is_empty());
assert!(suggest("два слова", 5).is_empty());
assert!(suggest("привет", 5).len() <= 5);
assert!(!suggest("привет", 5).contains(&"привет".to_string()));
}
#[test]
fn distance_weighs_letter_groups_half() {
use crate::affix_suggest::{fold, weighted_distance};
let map: Vec<Vec<char>> = MAP.iter().map(|g| g.chars().collect()).collect();
let related = |a: char, b: char| fold(a, &map) == fold(b, &map);
let c = |s: &str| s.chars().collect::<Vec<_>>();
assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), &related, 9), 1);
assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), &related, 9), 1);
assert_eq!(weighted_distance(&c("дом"), &c("дмо"), &related, 9), 2);
assert_eq!(weighted_distance(&c("дом"), &c("домами"), &related, 9), 6);
assert_eq!(weighted_distance(&c("дом"), &c("книга"), &related, 2), 3);
}
#[test]
fn suggestions_are_fast_enough() {
crate::ru::test_dict::install();
let _ = suggest("дом", 1);
let t = std::time::Instant::now();
for w in ["ыкщзвотрусскогословаааа", "здраствуйте", "краснннного"] {
let _ = suggest(w, 6);
}
let ms = t.elapsed().as_secs_f64() * 1000.0;
eprintln!("3 suggestion calls: {ms:.0} ms");
assert!(ms < 20_000.0, "{ms} ms");
}
}
crates/proof/src/ru/test_dict.rs
+130
View File
@@ -0,0 +1,130 @@
//! A tiny synthetic Russian Hunspell dictionary for tests (written for these tests; not taken from
//! any real dictionary). Real dictionaries are installed on the system, never bundled.
/// Affix rules: the endings of a hard masculine noun, a feminine noun in `-а`, hard and soft
/// adjectives and a verb in `-ать`, plus the letters suggestions try.
pub const AFF: &str = "SET UTF-8
TRY оеаинтсрвлкмдпуяыьгзбчйхжшюцщэфъё
SFX A Y 9
SFX A 0 а .
SFX A 0 у .
SFX A 0 ом .
SFX A 0 е .
SFX A 0 ы .
SFX A 0 ов .
SFX A 0 ам .
SFX A 0 ами .
SFX A 0 ах .
SFX D Y 5
SFX D а ы а
SFX D а е а
SFX D а у а
SFX D а ой а
SFX D а ою а
SFX B Y 11
SFX B ый ого ый
SFX B ый ому ый
SFX B ый ым ый
SFX B ый ом ый
SFX B ый ая ый
SFX B ый ой ый
SFX B ый ую ый
SFX B ый ое ый
SFX B ый ые ый
SFX B ый ых ый
SFX B ый ыми ый
SFX E Y 8
SFX E ий его ий
SFX E ий ему ий
SFX E ий им ий
SFX E ий ей ий
SFX E ий яя ий
SFX E ий ее ий
SFX E ий ие ий
SFX E ий их ий
SFX C Y 11
SFX C ть ю ть
SFX C ть ешь ть
SFX C ть ет ть
SFX C ть ем ть
SFX C ть ете ть
SFX C ть ют ть
SFX C ть ются ть
SFX C ть ется ть
SFX C ть л ть
SFX C ть ла ть
SFX C ть ться ть
";
/// Stems (the count line first, as in Hunspell files).
pub const DIC: &str = "62
привет
здесь
здравствуй
жить
часто
знаю
знать/C
не
в
общем
общий/E
дом/A
стол/A
город/A
Москва/D
книга/D
красный/B
новый/B
белый/B
западный/B
синий/E
зелёный/B
русский
английский
читать/C
работать/C
кто
где
что
давай
всё
все
ещё
ёлка/D
север/A
из
за
на
и
а
но
или
он
она
я
ты
мы
вы
это
так
как
когда
который/B
чтобы
потому
если
пришёл
сказал
вскрытый/B
высоко
эластичный/B
консультант/A
";
/// Install the test dictionary as the Russian one.
pub fn install() {
if let Err(e) = super::install_dictionary(AFF, DIC) {
panic!("test dictionary: {e}");
}
}
crates/proof/src/slot.rs
+210
View File
@@ -0,0 +1,210 @@
//! The installed Hunspell dictionary of a proofing language, in one shape for every language:
//! found on first use, replaced with [`DictSlot::install`] and forgotten with [`DictSlot::reload`].
use std::path::{Path, PathBuf};
use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
use crate::affix::AffixDict;
/// A dictionary a language reads: the affix engine's ([`AffixDict`]) or Turkish's own.
pub(crate) trait Dictionary: Default + Send + Sync {
/// The installed `.aff` and `.dic` files, read.
fn load(aff: &Path, dic: &Path) -> Result<Self, String>;
/// Hunspell `.aff` and `.dic` text (decoded to UTF-8), parsed.
fn parse(aff: &str, dic: &str) -> Result<Self, String>;
/// No words: the language has no spelling marks.
fn is_empty(&self) -> bool;
}
impl Dictionary for AffixDict {
fn load(aff: &Path, dic: &Path) -> Result<Self, String> {
crate::hunspell::load(aff, dic)
}
fn parse(aff: &str, dic: &str) -> Result<Self, String> {
AffixDict::from_hunspell(aff, dic, &[])
}
fn is_empty(&self) -> bool {
AffixDict::is_empty(self)
}
}
/// What every language's slot answers, whatever its dictionary type (the language table in
/// lib.rs holds them side by side).
pub(crate) trait Slot: Sync {
/// See [`DictSlot::name`].
fn name(&self) -> &'static str;
/// See [`DictSlot::installed_path`].
fn installed_path(&self) -> Option<PathBuf>;
/// See [`DictSlot::has_dictionary`].
fn has_dictionary(&self) -> bool;
/// See [`DictSlot::reload`].
fn reload(&self);
}
impl<D: Dictionary> Slot for DictSlot<D> {
fn name(&self) -> &'static str {
DictSlot::name(self)
}
fn installed_path(&self) -> Option<PathBuf> {
DictSlot::installed_path(self)
}
fn has_dictionary(&self) -> bool {
DictSlot::has_dictionary(self)
}
fn reload(&self) {
DictSlot::reload(self);
}
}
/// The dictionary in use and whether it came from [`DictSlot::install`] rather than a file on
/// disk (it then has no file to report, even if a matching file also happens to exist): one lock,
/// so a reload and an install can't interleave their two fields.
struct State<D> {
dict: Option<Arc<D>>,
in_memory: bool,
/// Counts reloads, so a load that started before one is not kept.
reloads: u64,
}
/// One language's dictionary.
pub(crate) struct DictSlot<D> {
/// File names the dictionary may have, the usual one first (also the key of its load error).
names: &'static [&'static str],
/// The language's English name, for the log.
english: &'static str,
state: RwLock<State<D>>,
}
impl<D: Dictionary> DictSlot<D> {
pub(crate) const fn new(names: &'static [&'static str], english: &'static str) -> Self {
DictSlot { names, english, state: RwLock::new(State { dict: None, in_memory: false, reloads: 0 }) }
}
fn read(&self) -> RwLockReadGuard<'_, State<D>> {
self.state.read().unwrap_or_else(PoisonError::into_inner)
}
fn write(&self) -> RwLockWriteGuard<'_, State<D>> {
self.state.write().unwrap_or_else(PoisonError::into_inner)
}
/// The Hunspell name the dictionary is known by (`pl_PL`): the first of its file names.
pub(crate) fn name(&self) -> &'static str {
self.names.first().copied().unwrap_or_default()
}
/// The file names the dictionary may have, the usual one first.
#[cfg(test)]
pub(crate) fn names(&self) -> &'static [&'static str] {
self.names
}
/// The dictionary: the installed one (read on first use), one given with [`Self::install`], or
/// an empty one when none is available (the language's spelling is off then).
pub(crate) fn get(&self) -> Arc<D> {
let reloads = {
let s = self.read();
if let Some(d) = s.dict.as_ref() {
return d.clone();
}
s.reloads
};
self.keep(reloads, Arc::new(self.load_installed()))
}
/// Keep a dictionary loaded after `reloads` reloads. If the folder was reloaded since, the
/// load may have read the old folder: it serves this one check and the next one loads again.
fn keep(&self, reloads: u64, loaded: Arc<D>) -> Arc<D> {
let mut s = self.write();
if s.reloads != reloads {
return s.dict.clone().unwrap_or(loaded);
}
// Another thread (or an install) may have filled it meanwhile: that one wins.
s.dict.get_or_insert(loaded).clone()
}
fn load_installed(&self) -> D {
let name = self.name();
let Some((aff, dic)) = crate::hunspell::find(self.names) else {
crate::hunspell::clear_load_error(name);
log::info!("{} spelling: no Hunspell {name} dictionary installed", self.english);
return D::default();
};
crate::hunspell::note_load(name, &dic, D::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("{} spelling: {e}", self.english);
D::default()
})
}
/// Use this dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), for
/// example one the user picked. On error the current dictionary stays.
pub(crate) fn install(&self, aff: &str, dic: &str) -> Result<(), String> {
let d = D::parse(aff, dic)?;
let mut slot = self.write();
slot.dict = Some(Arc::new(d));
slot.in_memory = true;
Ok(())
}
/// Forget the dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub(crate) fn reload(&self) {
let mut slot = self.write();
slot.dict = None;
slot.in_memory = false;
slot.reloads = slot.reloads.wrapping_add(1);
}
/// The `.dic` file of the installed dictionary, searching the dictionaries folder and the
/// system locations now (File › Options › Proofing shows it); `None` when there is none, or
/// when the dictionary in use was given with [`Self::install`] and so has no file.
pub(crate) fn installed_path(&self) -> Option<PathBuf> {
if self.read().in_memory {
return None;
}
crate::hunspell::find(self.names).map(|(_, dic)| dic)
}
/// Is a dictionary available (spelling is checked)?
pub(crate) fn has_dictionary(&self) -> bool {
!self.get().is_empty()
}
}
#[cfg(test)]
mod tests {
use super::*;
static SLOT: DictSlot<AffixDict> = DictSlot::new(&["zz_SLOT_TEST"], "Test");
#[test]
fn an_installed_dictionary_has_no_file_until_reloaded() {
assert!(!SLOT.has_dictionary(), "nothing installed under this name");
assert!(SLOT.install("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused");
assert!(!SLOT.has_dictionary(), "a refused install keeps the current one");
SLOT.install("", "1\nkot\n").unwrap();
assert!(SLOT.has_dictionary() && SLOT.get().check("kot"));
assert_eq!(SLOT.installed_path(), None);
SLOT.reload();
assert!(!SLOT.has_dictionary(), "reloaded from disk, where there is none");
assert_eq!(SLOT.name(), "zz_SLOT_TEST");
}
/// A load that started before a dictionaries-folder change is not kept after its reload.
#[test]
fn a_load_from_before_a_reload_is_not_kept() {
let slot: DictSlot<AffixDict> = DictSlot::new(&["zz_SLOT_RELOAD_TEST"], "Test");
let before = slot.read().reloads;
slot.reload();
let _ = slot.keep(before, Arc::new(AffixDict::default()));
assert!(slot.read().dict.is_none(), "the next check loads from the new folder");
let now = slot.read().reloads;
let _ = slot.keep(now, Arc::new(AffixDict::default()));
assert!(slot.read().dict.is_some(), "a load since the reload is kept");
}
}
crates/proof/src/spelling.rs
+52
View File
@@ -0,0 +1,52 @@
//! The start of every Latin-script dictionary language's spelling check (Czech, Croatian,
//! Indonesian, Italian, Polish, Portuguese, Spanish): the words that need no lookup, under the
//! File › Options › Proofing choices. Each language goes on from there with its own rules.
use crate::Options;
use crate::affix::AffixDict;
/// Is `c` a letter of the Latin script (the only one these dictionaries cover)?
fn latin(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
}
/// `word` trimmed of quotes and apostrophes, when `d` must say whether it is right; `None` when it
/// is right without a lookup: there is no dictionary (no spelling marks), the word is a single
/// letter or has no letters, it has digits or is a short word in capitals (an acronym) and the
/// options skip those, or it is in another script.
pub(crate) fn word_to_check<'a>(d: &AffixDict, word: &'a str, opts: &Options) -> Option<&'a str> {
if d.is_empty() {
return None;
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
return None;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return None;
}
w.chars().any(latin).then_some(w)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn words_needing_no_lookup() {
let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap();
let defaults = Options::default();
assert_eq!(word_to_check(&AffixDict::default(), "kot", &defaults), None, "no dictionary");
for w in ["", "a", "2026", "PKO", "B2B", "日本語", "--"] {
assert_eq!(word_to_check(&d, w, &defaults), None, "{w}");
}
assert_eq!(word_to_check(&d, "„kot”", &defaults), Some("kot"));
assert_eq!(word_to_check(&d, "'kot'", &defaults), Some("kot"));
let strict = Options { ignore_uppercase: false, ignore_numbers: false, ..defaults };
assert_eq!(word_to_check(&d, "PKO", &strict), Some("PKO"));
assert_eq!(word_to_check(&d, "B2B", &strict), Some("B2B"));
assert_eq!(word_to_check(&d, "PRZEDSIĘBIORSTWO", &defaults), Some("PRZEDSIĘBIORSTWO"), "only short capitals are acronyms");
}
}
crates/proof/src/tr/dict.rs
+144
View File
@@ -0,0 +1,144 @@
//! Turkish Hunspell dictionary parser with `FLAG num` support.
//!
//! Turkish uses `FLAG num`, storing thousands of numerical suffix flags (0..6465).
//! This engine parses `/usr/share/hunspell/tr_TR.aff` and `.dic` safely without panics.
use std::collections::HashMap;
/// A suffix rule for Turkish.
#[derive(Clone, Debug)]
pub struct TrSuffixRule {
pub strip: Box<str>,
pub add: Box<str>,
pub cond: Box<str>,
}
/// Dedicated Turkish Hunspell dictionary engine.
#[derive(Clone, Debug, Default)]
pub struct TrDict {
/// Stem -> list of numerical flags it carries.
stems: HashMap<String, Box<[u16]>>,
/// Flag number -> list of suffix rules.
rules: HashMap<u16, Vec<TrSuffixRule>>,
/// Suffix 'add' string -> list of (flag, rule_index).
suffix_index: HashMap<String, Vec<(u16, u16)>>,
/// Longest suffix 'add' string in characters.
max_suffix_len: usize,
}
impl crate::slot::Dictionary for TrDict {
fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result<Self, String> {
super::load(aff, dic)
}
fn parse(aff: &str, dic: &str) -> Result<Self, String> {
TrDict::from_hunspell(aff, dic)
}
fn is_empty(&self) -> bool {
TrDict::is_empty(self)
}
}
impl TrDict {
/// Returns true if empty (no dictionary loaded).
pub fn is_empty(&self) -> bool {
self.stems.is_empty()
}
/// Load from `.aff` and `.dic` contents.
pub fn from_hunspell(aff: &str, dic: &str) -> Result<Self, String> {
let mut rules: HashMap<u16, Vec<TrSuffixRule>> = HashMap::new();
let mut max_suffix_len = 0;
for line in aff.lines() {
let parts: Vec<&str> = line.split_whitespace().collect();
// SFX <flag_num> <strip> <add> [<cond>]
if parts.len() >= 4 && parts[0] == "SFX" {
let Ok(flag) = parts[1].parse::<u16>() else { continue };
// Skip header line: `SFX <flag> Y/N <count>`
if parts.len() == 4 && (parts[2] == "Y" || parts[2] == "N") && parts[3].parse::<usize>().is_ok() {
continue;
}
let strip = if parts[2] == "0" { "" } else { parts[2] };
let add = if parts[3] == "0" { "" } else { parts[3] };
let cond = if parts.len() > 4 { parts[4] } else { "." };
max_suffix_len = max_suffix_len.max(add.chars().count());
rules.entry(flag).or_default().push(TrSuffixRule { strip: strip.into(), add: add.into(), cond: cond.into() });
}
}
let mut suffix_index: HashMap<String, Vec<(u16, u16)>> = HashMap::new();
for (&flag, r_list) in &rules {
for (idx, r) in r_list.iter().enumerate() {
if let Ok(idx_u16) = u16::try_from(idx) {
suffix_index.entry(r.add.to_string()).or_default().push((flag, idx_u16));
}
}
}
let mut stems: HashMap<String, Box<[u16]>> = HashMap::new();
for (i, line) in dic.lines().enumerate() {
if i == 0 && line.trim().parse::<usize>().is_ok() {
continue;
}
let line = line.trim();
if line.is_empty() {
continue;
}
if let Some((word, flags_part)) = line.split_once('/') {
let flags: Vec<u16> = flags_part.split(',').filter_map(|f| f.parse::<u16>().ok()).collect();
stems.insert(word.to_string(), flags.into_boxed_slice());
} else {
stems.insert(line.to_string(), Box::new([]));
}
}
Ok(TrDict { stems, rules, suffix_index, max_suffix_len })
}
/// Check if word is correct (stem or affixed).
pub fn contains(&self, word: &str) -> bool {
// Longer input is never a dictionary word, and isn't scanned (#3625).
if self.is_empty() || word.is_empty() || word.len() > crate::affix::MAX_WORD_BYTES {
return false;
}
// Direct stem hit
if self.stems.contains_key(word) {
return true;
}
// Try stripping suffixes matching the tail
let word_len = word.chars().count();
let char_indices: Vec<(usize, char)> = word.char_indices().collect();
// Check each possible suffix length up to max_suffix_len
for cut_chars in 0..=self.max_suffix_len.min(word_len) {
let byte_idx = if cut_chars == 0 { word.len() } else { char_indices[word_len - cut_chars].0 };
let add = &word[byte_idx..];
let base = &word[..byte_idx];
if let Some(matches) = self.suffix_index.get(add) {
for &(flag, rule_idx) in matches {
let Some(rule_list) = self.rules.get(&flag) else { continue };
let Some(rule) = rule_list.get(rule_idx as usize) else { continue };
// Check condition on root
let mut candidate = String::with_capacity(base.len() + rule.strip.len());
candidate.push_str(base);
candidate.push_str(&rule.strip);
if self.stems.get(&candidate).is_some_and(|f_list| f_list.contains(&flag)) {
return true;
}
}
}
}
false
}
}
crates/proof/src/tr/grammar.rs
+359
View File
@@ -0,0 +1,359 @@
//! Turkish grammar and punctuation checks: cheap, rule-based and tuned to stay quiet when unsure.
//! Messages are in Turkish.
//!
//! - **Repeated word** (`çok çok`), except idiomatic reduplications and doublings (`ALLOWED_REPEATS`:
//! `yavaş yavaş`, `tıkır tıkır`, `adım adım`).
//! - **Space before punctuation** (`ev ,`) and **double spaces**.
//! - **Capital letter at the start of a sentence**, knowing Turkish abbreviations (`vb.`, `vs.`,
//! `prof.`, `dr.`, `mad.`, `yy.`, `s.`…) after which lowercase or numbers may continue.
//! - **No comma before/after conjunctions**: `ve`, `veya`, `yahut`, `ile`, `de`, `da`, `hem ... hem`,
//! `ne ... ne` conjunctions should not have commas immediately before or after them.
//! - **Doubled punctuation** (`..`, `,,`, `;;`).
use crate::{Issue, IssueKind, ProofLang, words};
/// Turkish reduplications and emphatic word pairings that are intentionally repeated.
const ALLOWED_REPEATS: &[&str] = &[
"yavaş",
"hızlı",
"koşa",
"güle",
"adım",
"kara",
"tıkır",
"şakır",
"akın",
"ağır",
"açık",
"ayrı",
"bire",
"bir",
"boş",
"bölük",
"buram",
"cana",
"can",
"cılk",
"çifter",
"çit",
"dalga",
"damla",
"deste",
"didik",
"dizi",
"dolu",
"deste",
"döne",
"eğri",
"enine",
"epey",
"fıkır",
"fırıl",
"gece",
"gıcır",
"gide",
"günü",
"güzel",
"halk",
"harıl",
"hık",
"hım",
"horul",
"ılık",
"ışıl",
"için",
"içten",
"iğne",
"iri",
"iyiden",
"iyi",
"kaba",
"kaç",
"kat",
"kaynar",
"kelli",
"kese",
"kıtır",
"kıvrım",
"koca",
"kol",
"kucak",
"kütür",
"mırıl",
"öbek",
"parça",
"patır",
"pıtır",
"sabah",
"salına",
"salkım",
"sap",
"sara",
"sarı",
"serin",
"sıra",
"sırım",
"sıkı",
"sırılsıklam",
"tane",
"teker",
"tek",
"tıpış",
"top",
"tutam",
"uzun",
"üçer",
"üst",
"yalan",
"yana",
"yan",
"yarım",
"yer",
"yudum",
"yığın",
"yürekten",
"yüz",
"zaman",
"zor",
];
/// Turkish abbreviations that end with a period and after which lower case may continue.
const ABBREVIATIONS: &[&str] = &[
"vb", "vs", "vd", "bkz", "bknz", "ör", "örnek", "mad", "md", "s", "sf", "sn", "dk", "sa", "yy", "m", "km", "cm", "mm", "gr", "kg", "mg", "dr",
"prof", "doç", "yrd", "av", "ist", "ank", "izm", "cad", "sok", "mah", "apt", "no", "tel", "faks", "tl", "kr", "krş", "a.ş", "ltd", "şti", "t.c",
"tbmm", "meb", "möt", "mö", "ms",
];
/// Conjunctions that must not be preceded by a comma.
const CONJUNCTIONS_NO_PRE_COMMA: &[&str] = &["ve", "veya", "yahut", "velev", "veyahut", "de", "da"];
/// Conjunctions that must not be followed by a comma.
const CONJUNCTIONS_NO_POST_COMMA: &[&str] = &["ve", "veya", "yahut", "veyahut", "hem", "ne", "ya", "gerek", "ister", "de", "da"];
/// Check grammar and punctuation in a Turkish text block.
pub fn check(text: &str) -> Vec<Issue> {
check_with(text, true)
}
/// [`check`] where `capital_first` says whether the text's first word must start a sentence with a capital
/// (false for a list item, which may be a fragment).
pub fn check_with(text: &str, capital_first: bool) -> Vec<Issue> {
let mut v = Vec::new();
let ws = words(text);
// 1. Repeated words (`kelime kelime`)
for pair in ws.windows(2) {
let (w1, w2) = match (text.get(pair[0].0..pair[0].1), text.get(pair[1].0..pair[1].1)) {
(Some(a), Some(b)) => (a, b),
_ => continue,
};
let gap = match text.get(pair[0].1..pair[1].0) {
Some(g) => g,
None => continue,
};
let only_spaces = gap.chars().all(|c| c == ' ' || c == '\t');
let lower1 = w1.to_lowercase();
let lower2 = w2.to_lowercase();
if only_spaces && lower1 == lower2 && !ALLOWED_REPEATS.contains(&lower1.as_str()) {
v.push(Issue {
start: pair[1].0,
end: pair[1].1,
kind: IssueKind::Grammar,
message: "Yinelenen sözcük".into(),
suggestions: Vec::new(),
lang: ProofLang::Tr,
});
}
}
// 2. Spaces before punctuation and double spaces
let chars: Vec<(usize, char)> = text.char_indices().collect();
for i in 0..chars.len() {
let (idx, c) = chars[i];
if matches!(c, ',' | '.' | '?' | '!' | ';' | ':') && i > 0 && chars[i - 1].1 == ' ' {
let prev_sp = chars[i - 1].0;
// Leave emoticons and ellipses alone
let is_dot_in_dots = c == '.' && (i + 1 < chars.len() && chars[i + 1].1 == '.');
if !is_dot_in_dots {
v.push(Issue {
start: prev_sp,
end: idx + c.len_utf8(),
kind: IssueKind::Grammar,
message: "Noktalama işaretinden önce boşluk bırakılmaz".into(),
suggestions: vec![c.to_string()],
lang: ProofLang::Tr,
});
}
}
// Double space
if c == ' ' && i + 1 < chars.len() && chars[i + 1].1 == ' ' && (i == 0 || chars[i - 1].1 != ' ') {
let mut end = idx + 1;
while end < text.len() && text.as_bytes().get(end) == Some(&b' ') {
end += 1;
}
v.push(Issue {
start: idx,
end,
kind: IssueKind::Grammar,
message: "Fazladan boşluk".into(),
suggestions: vec![" ".into()],
lang: ProofLang::Tr,
});
}
}
// 3. Comma before or after conjunctions (`ali, ve veli` or `ve, veli`)
for (i, &(w_start, w_end)) in ws.iter().enumerate() {
let word = match text.get(w_start..w_end) {
Some(w) => w.to_lowercase(),
None => continue,
};
// Check comma before conjunction
if CONJUNCTIONS_NO_PRE_COMMA.contains(&word.as_str())
&& i > 0
&& let Some(between) = text.get(ws[i - 1].1..w_start)
&& let Some(comma_pos) = between.find(',')
{
let actual_pos = ws[i - 1].1 + comma_pos;
v.push(Issue {
start: actual_pos,
end: actual_pos + 1,
kind: IssueKind::Grammar,
message: format!("'{word}' bağlacından önce virgül konulmaz"),
suggestions: vec![String::new()],
lang: ProofLang::Tr,
});
}
// Check comma after conjunction
if CONJUNCTIONS_NO_POST_COMMA.contains(&word.as_str())
&& i + 1 < ws.len()
&& let Some(between) = text.get(w_end..ws[i + 1].0)
&& let Some(comma_pos) = between.find(',')
{
let actual_pos = w_end + comma_pos;
v.push(Issue {
start: actual_pos,
end: actual_pos + 1,
kind: IssueKind::Grammar,
message: format!("'{word}' bağlacından sonra virgül konulmaz"),
suggestions: vec![String::new()],
lang: ProofLang::Tr,
});
}
}
// 4. Sentence capitalization knowing Turkish abbreviations
for (i, &(w_start, w_end)) in ws.iter().enumerate() {
if i == 0 {
// First word in paragraph
if capital_first
&& let Some(w) = text.get(w_start..w_end)
&& w.chars().next().is_some_and(char::is_lowercase)
{
let cap = turkish_capitalize(w);
v.push(Issue {
start: w_start,
end: w_end,
kind: IssueKind::Grammar,
message: "Cümle büyük harfle başlar".into(),
suggestions: vec![cap],
lang: ProofLang::Tr,
});
}
continue;
}
// Check if previous character sequence ended a sentence
let prev_end = ws[i - 1].1;
let prev_word = text.get(ws[i - 1].0..prev_end).unwrap_or("");
if let Some(gap) = text.get(prev_end..w_start) {
let trimmed = gap.trim();
if (trimmed.ends_with('.') || trimmed.ends_with('?') || trimmed.ends_with('!'))
&& !trimmed.contains("...")
&& !ABBREVIATIONS.contains(&prev_word.to_lowercase().as_str())
&& let Some(w) = text.get(w_start..w_end)
&& w.chars().next().is_some_and(char::is_lowercase)
{
let cap = turkish_capitalize(w);
v.push(Issue {
start: w_start,
end: w_end,
kind: IssueKind::Grammar,
message: "Cümle büyük harfle başlar".into(),
suggestions: vec![cap],
lang: ProofLang::Tr,
});
}
}
}
v.sort_by_key(|issue| (issue.start, issue.end));
v
}
/// Capitalize according to Turkish rules (`i` -> `İ`, `ı` -> `I`).
pub fn turkish_capitalize(s: &str) -> String {
let mut chars = s.chars();
let Some(first) = chars.next() else { return String::new() };
let first_cap = match first {
'i' => 'İ',
'ı' => 'I',
other => other.to_uppercase().next().unwrap_or(other),
};
let mut out = String::with_capacity(s.len() + 1);
out.push(first_cap);
out.push_str(chars.as_str());
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn clean_turkish_text_produces_no_issues() {
let text = "Bu bir test cümlesidir. Kitap ve defter aldım. Çocuk yavaş yavaş yürüdü.";
assert!(check(text).is_empty());
}
#[test]
fn repeated_words_flagged_except_allowed() {
let text = "Gitti gitti ama bulamadı.";
let issues = check(text);
assert_eq!(issues.len(), 1);
assert_eq!(issues[0].message, "Yinelenen sözcük");
// Allowed repeat
let text2 = "Çocuk yavaş yavaş ilerliyordu.";
assert!(check(text2).is_empty());
}
#[test]
fn space_before_punctuation_flagged() {
let text = "Merhaba , dünya!";
let issues = check(text);
assert_eq!(issues.len(), 1);
assert_eq!(issues[0].message, "Noktalama işaretinden önce boşluk bırakılmaz");
}
#[test]
fn comma_around_conjunctions_flagged() {
let text = "Elma, ve armut.";
let issues = check(text);
assert_eq!(issues.len(), 1);
assert!(issues[0].message.contains("virgül konulmaz"));
let text2 = "Bana kalırsa bu mesele de, haklı olan taraf bizdik.";
let issues2 = check(text2);
assert_eq!(issues2.len(), 1);
assert!(issues2[0].message.contains("'de' bağlacından sonra virgül konulmaz"));
}
#[test]
fn sentence_capitalization_respects_abbreviations() {
let text = "Dr. ahmet geldi. Fakat geç kaldı.";
let issues = check(text);
// "Dr. ahmet" does not flag "ahmet" because "Dr." is in ABBREVIATIONS.
assert!(issues.is_empty());
}
}
crates/proof/src/tr/hyphen.rs
+98
View File
@@ -0,0 +1,98 @@
//! Turkish hyphenation by rule ([`ProofLang::Tr`](crate::ProofLang::Tr)).
//!
//! Turkish words are split at syllables according to Turkish Language Association (TDK) rules.
//! Because Turkish syllable boundaries are fully regular, they can be determined directly by rule
//! without needing external pattern files or word lists:
//! - Turkish vowels: `a`, `e`, `ı`, `i`, `o`, `ö`, `u`, `ü` (both lowercase and uppercase).
//! - A single consonant between two vowels joins the following syllable (`ba-lık`, `ö-ğren-ci`).
//! - Of two consonants between two vowels, the first stays with the preceding syllable and the
//! second joins the next (`alt-lık`, `sev-gi`, `tür-kçe`).
//! - Of three consonants between two vowels, the first two stay with the preceding syllable and the
//! third joins the next (`türk-çe`, `sürp-riz`).
//! - In compound words, syllable division follows pronunciation across the boundary (`ba-şöğ-ret-men`,
//! `il-ko-kul`), which matches the general vowel/consonant distribution rules.
//!
//! A single letter left at the end or start of a line is ruled out by the paragraph's
//! [`Limits`](crate::hyphen::Limits).
/// Is `c` a Turkish vowel?
pub fn is_vowel(c: char) -> bool {
matches!(c, 'a' | 'e' | 'ı' | 'i' | 'o' | 'ö' | 'u' | 'ü' | 'A' | 'E' | 'I' | 'İ' | 'O' | 'Ö' | 'U' | 'Ü')
}
/// Is `c` a Turkish letter?
pub(crate) fn is_letter(c: char) -> bool {
c.is_alphabetic()
}
/// Break points of a Turkish word (char indices; a hyphen goes before that char).
/// Empty for words with fewer than two vowels or containing non-alphabetic characters.
pub fn points(word: &[char]) -> Vec<usize> {
if word.len() < 2 || !word.iter().all(|&c| is_letter(c)) {
return Vec::new();
}
let vowels: Vec<usize> = (0..word.len()).filter(|&i| is_vowel(word[i])).collect();
if vowels.len() < 2 {
return Vec::new();
}
let mut out = Vec::new();
for pair in vowels.windows(2) {
let (v1, v2) = (pair[0], pair[1]);
if v2 <= v1 {
continue;
}
let num_consonants = v2 - v1 - 1;
let pt = match num_consonants {
0 => v1 + 1,
1 => v2 - 1,
2 => v1 + 2,
_ => v2 - 1,
};
if pt > v1 && pt <= v2 {
out.push(pt);
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
fn split(w: &str) -> String {
let chars: Vec<char> = w.chars().collect();
let pts = points(&chars);
let mut out = String::new();
for (i, c) in chars.iter().enumerate() {
if pts.contains(&i) {
out.push('-');
}
out.push(*c);
}
out
}
#[test]
fn syllables_follow_turkish_rules() {
assert_eq!(split("balık"), "ba-lık");
assert_eq!(split("araba"), "a-ra-ba");
assert_eq!(split("öğrenci"), "öğ-ren-ci");
assert_eq!(split("altlık"), "alt-lık");
assert_eq!(split("sevgi"), "sev-gi");
assert_eq!(split("türkçe"), "türk-çe");
assert_eq!(split("saat"), "sa-at");
assert_eq!(split("şiir"), "şi-ir");
assert_eq!(split("ilkokul"), "il-ko-kul");
assert_eq!(split("başöğretmen"), "ba-şöğ-ret-men");
}
#[test]
fn short_or_non_words_stay_unsplit() {
assert_eq!(split(""), "");
assert_eq!(split("a"), "a");
assert_eq!(split("ve"), "ve");
assert_eq!(split("Türk"), "Türk");
assert_eq!(split("123"), "123");
assert_eq!(split("araba-ev"), "araba-ev");
}
}
crates/proof/src/tr/mod.rs
+164
View File
@@ -0,0 +1,164 @@
//! Turkish proofing.
//!
//! - **Spelling** against the Hunspell `tr_TR` dictionary installed on the system
//! ([`dictionary`], found by [`crate::hunspell`]: `tr_TR.aff`/`tr_TR.dic` from the OS, a
//! package manager or LibreOffice; it is not bundled). Without it Turkish text gets no spelling
//! marks. Capitalized stems are proper nouns (`Ankara`); foreign names inflected with an apostrophe
//! are accepted (`WordCraft'ta`), and the user dictionary applies as in English.
//! - **Suggestions** tuned for Turkish ([`suggest`]): ASCII-diacritic restoration (`turkce` -> `Türkçe`),
//! common confusions (`yalnış` -> `yanlış`), detached particles (`geldimi` -> `geldi mi`), and typos.
//! - **Grammar** ([`grammar`]): high-precision rules (reduplications, comma around conjunctions,
//! sentence start with abbreviations).
//! - **Hyphenation** ([`hyphen`]): syllable division by Turkish Language Association (TDK) rules.
pub mod dict;
pub mod grammar;
pub mod hyphen;
pub mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::Arc;
use crate::slot::DictSlot;
pub use dict::TrDict;
pub use suggest::suggest;
/// File names of a Turkish Hunspell dictionary.
const NAMES: &[&str] = &["tr_TR", "tr-TR", "tr"];
/// The Turkish dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<TrDict> = DictSlot::new(NAMES, "Turkish");
/// The Turkish dictionary: the installed Hunspell `tr_TR` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available.
pub fn dictionary() -> Arc<TrDict> {
DICT.get()
}
/// Read a Turkish Hunspell pair from disk: size-capped and decoded per the `.aff`'s `SET` line,
/// like the other languages' dictionaries (#3661).
pub(crate) fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result<TrDict, String> {
let (aff_text, dic_text) = crate::hunspell::decode(&crate::hunspell::read_capped(aff)?, &crate::hunspell::read_capped(dic)?)?;
TrDict::from_hunspell(&aff_text, &dic_text)
}
/// Use this Turkish dictionary from now on (Hunspell `.aff` and `.dic` text). On error the
/// current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
DICT.install(aff, dic)
}
/// Is a Turkish dictionary available?
pub fn has_dictionary() -> bool {
DICT.has_dictionary()
}
/// Is `word` spelled correctly in Turkish?
pub fn is_correct(word: &str) -> bool {
is_correct_with(&dictionary(), word)
}
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words that contain
/// numbers off, words with digits are checked against the dictionary too.
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
check(&dictionary(), word, opts)
}
/// [`is_correct`] against a given dictionary.
pub fn is_correct_with(d: &TrDict, word: &str) -> bool {
check(d, word, &crate::Options::default())
}
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &TrDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // No Turkish dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
return true;
}
// Check user dictionary first
if crate::in_user_dictionary(&suggest::turkish_to_lower(w)) || crate::in_user_dictionary(&w.to_lowercase()) {
return true;
}
// Direct lookup or lowercase lookup
if d.contains(w) {
return true;
}
let lower = suggest::turkish_to_lower(w);
if d.contains(&lower) {
return true;
}
// Handle apostrophe for proper nouns (e.g., `Ankara'da`, `WordCraft'ta`)
if let Some((stem, _suffix)) = w.split_once(['\'', '’'])
&& (d.contains(stem) || d.contains(&suggest::turkish_to_lower(stem)))
{
return true;
}
// Compound words separated by hyphens (e.g., `afyon-karahisar`)
if w.contains('-') && w.split('-').all(|part| part.is_empty() || check(d, part, opts)) {
return true;
}
false
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn without_dictionary_everything_is_correct() {
let empty = TrDict::default();
assert!(is_correct_with(&empty, "herhangi"));
assert!(is_correct_with(&empty, "xyzabc"));
}
#[test]
fn apostrophe_and_compounds_are_handled() {
let empty = TrDict::default();
assert!(is_correct_with(&empty, "Ankara'da"));
assert!(is_correct_with(&empty, "açık-mavi"));
}
#[test]
fn installed_dictionary_checks_words() {
test_dict::install();
assert!(is_correct("kitap"));
assert!(is_correct("kitaplar"));
assert!(is_correct("evde"));
assert!(is_correct("olacaklardan"));
assert!(is_correct("sonra"));
assert!(!is_correct("sorna"));
assert!(!is_correct("kitapppp"));
assert!(!is_correct("evdekişşş"));
// Test user dictionary
assert!(!is_correct("benimbilmedigimkelime"));
crate::add_word("benimbilmedigimkelime");
assert!(is_correct("benimbilmedigimkelime"));
}
/// With Ignore words that contain numbers off, Turkish checks words with digits against the
/// dictionary instead of accepting them, hyphen parts included.
#[test]
fn proofing_options_reach_the_turkish_check() {
test_dict::install();
let lang = crate::ProofLang::Tr;
let defaults = crate::Options::default();
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
assert!(crate::is_correct_in_with("kitapp2", lang, &defaults));
assert!(!crate::is_correct_in_with("kitapp2", lang, &check_digits));
assert!(!crate::is_correct_in_with("ev-kitapp2", lang, &check_digits));
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
assert!(is_correct("kitapp2"), "direct checks keep the defaults");
}
}
crates/proof/src/tr/suggest.rs
+328
View File
@@ -0,0 +1,328 @@
//! Turkish spelling suggestions, ranked by likelihood.
//!
//! Candidates are generated and ranked by cost:
//! 1. Capitalization / case adjustments (especially dotted/dotless I: `istanbul` -> `İstanbul`).
//! 2. Missing Turkish diacritics / ASCII substitutions (most frequent Turkish typing slip):
//! `c` <-> `ç`, `g` <-> `ğ`, `i` <-> `ı`, `o` <-> `ö`, `s` <-> `ş`, `u` <-> `ü` (`turkce` -> `Türkçe`).
//! 3. Common spelling mistakes and phonetic confusions (`REP` table):
//! `yalnış` -> `yanlış`, `yanlız` -> `yalnız`, `herşey` -> `her şey`, `bir kaç` -> `birkaç`.
//! 4. Missing space for detached particles and conjunctions (`bende` -> `ben de`, `geldinmi` -> `geldin mi`).
//! 5. Single typo (Damerau-Levenshtein distance <= 2).
use std::collections::HashSet;
use crate::tr::dict::TrDict;
/// Common misspellings and their canonical replacements in Turkish.
const COMMON_REPLACEMENTS: &[(&str, &str)] = &[
("yalnış", "yanlış"),
("yanlız", "yalnız"),
("herşey", "her şey"),
("herkez", "herkes"),
("bir kaç", "birkaç"),
("birşey", "bir şey"),
("tabiki", "tabii ki"),
("orjinal", "orijinal"),
("kiprik", "kirpik"),
("şöför", "şoför"),
("sarmısak", "sarımsak"),
("makina", "makine"),
("ünvan", "unvan"),
("rastlantı", "rastlantı"),
("labaratuvar", "laboratuvar"),
("laboratuar", "laboratuvar"),
("kravat", "kravat"),
("gardolap", "gardırop"),
("egzoz", "egzoz"),
("egzozt", "egzoz"),
("egsoz", "egzoz"),
("gelcen", "geleceksin"),
("gelecen", "geleceksin"),
("gitcen", "gideceksin"),
("gidecen", "gideceksin"),
("yapcan", "yapacaksın"),
("yapacan", "yapacaksın"),
("edecen", "edeceksin"),
("deyil", "değil"),
];
/// Longest word (in letters) given suggestions. Every candidate is a fresh copy of the word, so
/// the cost grows with the square of its length (#3625); Turkish words run longer than most
/// (`Çekoslovakyalılaştıramadıklarımızdanmışsınız` is 43 letters), so the cap is above the other
/// languages' 40.
const MAX_LETTERS: usize = 64;
/// Produce spelling suggestions for a misspelled Turkish word.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
let dict = super::dictionary();
suggest_with(&dict, word, max)
}
/// Suggest with a given dictionary.
pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec<String> {
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 {
return Vec::new();
}
let mut suggestions = Vec::new();
let mut seen = HashSet::new();
let lower = turkish_to_lower(word);
// 1. Check direct common replacement table
for &(wrong, right) in COMMON_REPLACEMENTS {
if lower == wrong && seen.insert(right.to_string()) {
suggestions.push(right.to_string());
if suggestions.len() >= max {
return suggestions;
}
}
}
// Only the table takes more than one word (`bir kaç`).
if word.contains(char::is_whitespace) {
return suggestions;
}
// 2. Diacritic substitutions (`c` -> `ç`, `g` -> `ğ`, `ı` -> `i`, `o` -> `ö`, `s` -> `ş`, `u` -> `ü`)
if !d.is_empty() {
let variants = generate_diacritic_variants(&lower);
for cand in variants {
if cand != lower && d.contains(&cand) && seen.insert(cand.clone()) {
suggestions.push(match_casing(word, &cand));
if suggestions.len() >= max {
return suggestions;
}
}
}
}
// 3. Detached suffix check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü`
for suffix in ["de", "da", "te", "ta", "mi", "mı", "mu", "mü"] {
if lower.ends_with(suffix) && lower.len() > suffix.len() + 1 {
let stem = &lower[..lower.len() - suffix.len()];
if d.is_empty() || d.contains(stem) {
let cand = format!("{stem} {suffix}");
if seen.insert(cand.clone()) {
suggestions.push(match_casing(word, &cand));
if suggestions.len() >= max {
return suggestions;
}
}
}
}
}
// 4. Single typo (transposition, deletion, insertion, substitution)
if !d.is_empty() {
let chars: Vec<char> = lower.chars().collect();
let n = chars.len();
// 4a. Transposition (adjacent swap: `sorna` -> `sonra`)
for i in 0..n.saturating_sub(1) {
if chars.get(i) != chars.get(i + 1) {
let mut swapped = chars.clone();
swapped.swap(i, i + 1);
let cand: String = swapped.into_iter().collect();
if d.contains(&cand) && seen.insert(cand.clone()) {
suggestions.push(match_casing(word, &cand));
if suggestions.len() >= max {
return suggestions;
}
}
}
}
// Turkish alphabet for insertions and substitutions
const TR_ALPHABET: &[char] = &[
'a', 'b', 'c', 'ç', 'd', 'e', 'f', 'g', 'ğ', 'h', 'ı', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'ö', 'p', 'r', 's', 'ş', 't', 'u', 'ü', 'v',
'y', 'z',
];
// 4b. Deletion (one character left out: `kitapp` -> `kitap`)
for i in 0..n {
let mut deleted = chars.clone();
deleted.remove(i);
let cand: String = deleted.into_iter().collect();
if d.contains(&cand) && seen.insert(cand.clone()) {
suggestions.push(match_casing(word, &cand));
if suggestions.len() >= max {
return suggestions;
}
}
}
// 4c. Substitution (one wrong letter: `yolculuk` -> ...)
for i in 0..n {
for &c in TR_ALPHABET {
if chars.get(i) != Some(&c) {
let mut substituted = chars.clone();
if let Some(target) = substituted.get_mut(i) {
*target = c;
}
let cand: String = substituted.into_iter().collect();
if d.contains(&cand) && seen.insert(cand.clone()) {
suggestions.push(match_casing(word, &cand));
if suggestions.len() >= max {
return suggestions;
}
}
}
}
}
// 4d. Insertion (one missing letter: `kitp` -> `kitap`)
for i in 0..=n {
for &c in TR_ALPHABET {
let mut inserted = chars.clone();
inserted.insert(i, c);
let cand: String = inserted.into_iter().collect();
if d.contains(&cand) && seen.insert(cand.clone()) {
suggestions.push(match_casing(word, &cand));
if suggestions.len() >= max {
return suggestions;
}
}
}
}
}
// 5. Fallback: capitalization
let cap = super::grammar::turkish_capitalize(&lower);
if cap != word && (d.is_empty() || d.contains(&cap)) && seen.insert(cap.clone()) {
suggestions.push(cap);
}
suggestions
}
/// Convert ASCII letters to Turkish diacritic equivalents.
fn generate_diacritic_variants(s: &str) -> Vec<String> {
let mut results = vec![String::new()];
for c in s.chars() {
let (alt1, alt2) = match c {
'c' => ('c', Some('ç')),
'ç' => ('ç', Some('c')),
'g' => ('g', Some('ğ')),
'ğ' => ('ğ', Some('g')),
'i' => ('i', Some('ı')),
'ı' => ('ı', Some('i')),
'o' => ('o', Some('ö')),
'ö' => ('ö', Some('o')),
's' => ('s', Some('ş')),
'ş' => ('ş', Some('s')),
'u' => ('u', Some('ü')),
'ü' => ('ü', Some('u')),
other => (other, None),
};
let branch_mult = if alt2.is_some() { 2 } else { 1 };
if results.len() * branch_mult > 1024 {
// Cap search branch explosion
for res in &mut results {
res.push(c);
}
continue;
}
let mut next_gen = Vec::with_capacity(results.len() * branch_mult);
for existing in &results {
let mut s1 = existing.clone();
s1.push(alt1);
next_gen.push(s1);
if let Some(c2) = alt2 {
let mut s2 = existing.clone();
s2.push(c2);
next_gen.push(s2);
}
}
results = next_gen;
}
results
}
/// Turkish lowercase with dotted/dotless I support.
pub fn turkish_to_lower(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for c in s.chars() {
match c {
'I' => out.push('ı'),
'İ' => out.push('i'),
other => {
for lc in other.to_lowercase() {
out.push(lc);
}
}
}
}
out
}
/// Match original casing on suggestion.
fn match_casing(orig: &str, cand: &str) -> String {
if orig.chars().all(char::is_uppercase) {
cand.to_uppercase()
} else if orig.chars().next().is_some_and(char::is_uppercase) {
super::grammar::turkish_capitalize(cand)
} else {
cand.to_string()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn common_replacements_are_suggested() {
let sugs = suggest("yalnış", 3);
assert!(sugs.contains(&"yanlış".to_string()));
let sugs2 = suggest("herkez", 3);
assert!(sugs2.contains(&"herkes".to_string()));
}
#[test]
fn detached_suffix_is_suggested() {
let sugs = suggest("geldinmi", 3);
assert!(sugs.contains(&"geldin mi".to_string()));
}
#[test]
fn overlong_words_get_no_suggestions_promptly() {
let d = TrDict::from_hunspell("SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n", "2\nkitap/1\nsonra\n").unwrap();
assert!(suggest_with(&d, "sorna", 3).contains(&"sonra".to_string()));
assert!(suggest_with(&d, "kitaplr", 3).contains(&"kitaplar".to_string()));
let t = std::time::Instant::now();
let long = "a".repeat(100_000);
assert!(suggest_with(&d, &long, 8).is_empty());
assert!(!d.contains(&long) && !d.contains(&format!("kitap{long}")));
assert!(!super::super::is_correct_with(&d, &long));
// Tens of milliseconds with the caps; without them this ran for over 5 minutes.
assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed());
}
#[test]
fn two_words_get_only_the_tables_replacement() {
let d = TrDict::from_hunspell("SET UTF-8\n", "2\nbirkaç\nsonra\n").unwrap();
assert_eq!(suggest_with(&d, "bir kaç", 3), vec!["birkaç".to_string()]);
assert_eq!(suggest_with(&d, "Bir kaç", 3), vec!["birkaç".to_string()]);
assert!(suggest_with(&d, "sorna sorna", 3).is_empty());
}
#[test]
fn turkish_to_lower_handles_i() {
assert_eq!(turkish_to_lower("IŞIK"), "ışık");
assert_eq!(turkish_to_lower("İSTANBUL"), "istanbul");
}
#[test]
fn typo_transposition_is_suggested() {
crate::tr::test_dict::install();
let sugs = suggest("sorna", 3);
assert!(sugs.contains(&"sonra".to_string()), "sorna should suggest sonra, got {sugs:?}");
let sugs_gelcen = suggest("gelcen", 3);
assert!(sugs_gelcen.contains(&"geleceksin".to_string()), "gelcen should suggest geleceksin, got {sugs_gelcen:?}");
}
}
crates/proof/src/tr/test_dict.rs
+22
View File
@@ -0,0 +1,22 @@
//! A tiny synthetic Turkish Hunspell dictionary for tests (written for these tests; not taken from
//! any real dictionary). Real dictionaries are installed on the system, never bundled.
/// No affix rules: the forms the tests look up are listed whole.
pub const AFF: &str = "SET UTF-8\n";
/// The words the Turkish tests expect to be known.
pub const DIC: &str = "7
kitap
kitaplar
ev
evde
olacaklardan
sonra
geldin
";
pub fn install() {
if let Err(e) = super::install_dictionary(AFF, DIC) {
panic!("test dictionary: {e}");
}
}
crates/proof/src/vi.rs
+96
View File
@@ -0,0 +1,96 @@
//! Telling Vietnamese text apart from English, so a document typed in Vietnamese under the default
//! English proofing language isn't covered in red underlines (as Word's "detect language
//! automatically" does). There are no Vietnamese proofing tools yet, so Vietnamese text is treated
//! like any language without them: no spelling or grammar marks.
//!
//! The test is orthographic. Vietnamese writes most syllables with letters no other Latin-script
//! language shares: the horn letters (`ơ ư`), `ĩ ũ`, and the tone-marked vowels of the Latin
//! Extended Additional block (`ạ ả ấ ầ … ỹ`), precomposed or as combining hook above, dot below
//! and horn. French and Portuguese share `â ê ô à é`, Romanian `ă` and Croatian, Serbian and
//! Bosnian `đ`, so those alone don't count.
/// Does `c` occur in Vietnamese words but (practically) nowhere else?
fn vietnamese_letter(c: char) -> bool {
matches!(c, 'ơ' | 'Ơ' | 'ư' | 'Ư' | 'ĩ' | 'Ĩ' | 'ũ' | 'Ũ' | '\u{1EA0}'..='\u{1EF9}' | '\u{0309}' | '\u{0323}' | '\u{031B}')
}
/// Is `word` written with a Vietnamese-only letter (`Việt`, `ơn`)? Such a word is never an
/// English misspelling.
pub fn is_vietnamese_word(word: &str) -> bool {
let combining = |c: char| matches!(c, '\u{0300}'..='\u{036F}');
// Two marks stacked on one letter (decomposed `ế` = e + circumflex + acute) are Vietnamese too.
word.chars().any(vietnamese_letter) || word.chars().zip(word.chars().skip(1)).any(|(a, b)| combining(a) && combining(b))
}
/// Does a paragraph read as Vietnamese? At least two of its words (any, in a heading of up to three
/// words) carry a Vietnamese-only letter, and they are at least a sixth of its words: Vietnamese syllables without
/// such a letter (`anh`, `con`, `ba`) are common, while an English paragraph quoting one
/// Vietnamese name stays English.
pub fn looks_vietnamese(text: &str) -> bool {
let mut total = 0usize;
let mut marked = 0usize;
for w in text.split(|c: char| !(c.is_alphanumeric() || matches!(c, '\u{0300}'..='\u{036F}'))).filter(|w| !w.is_empty()) {
total += 1;
if is_vietnamese_word(w) {
marked += 1;
}
}
marked > 0 && (marked >= 2 || total <= 3) && marked * 6 >= total
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn vietnamese_words() {
for w in ["Việt", "ơn", "những", "Ước", "mũi", "Nghĩa", "tie\u{0302}\u{0301}ng", "nho\u{031B}"] {
assert!(is_vietnamese_word(w), "{w}");
}
// Shared with French, Portuguese, Spanish: not enough on their own.
// Romanian ă and Croatian đ are Vietnamese letters too, but not Vietnamese-only.
for w in ["café", "être", "côte", "água", "señor", "anh", "the", "mănânc", "acasă", "Đurđa", "đak", "đi"] {
assert!(!is_vietnamese_word(w), "{w}");
}
}
#[test]
fn paragraphs() {
assert!(looks_vietnamese("Tôi yêu tiếng Việt và những bài thơ của nó."));
assert!(looks_vietnamese("Hà Nội"));
assert!(looks_vietnamese("Việt"));
// English quoting one Vietnamese name, and French, stay as they are.
assert!(!looks_vietnamese("The meeting with Mr Nguyễn is on Tuesday at the main office."));
assert!(!looks_vietnamese("Le café de la côte est très bon."));
assert!(!looks_vietnamese(""));
assert!(!looks_vietnamese("anh em"));
// Romanian and Croatian paragraphs, full of ă and đ, aren't Vietnamese.
assert!(!looks_vietnamese("Mă duc acasă să mănânc cu fratele meu, că e târziu."));
assert!(!looks_vietnamese("Đurđa i Đorđe su se vratili kući, a Đakovo je daleko."));
}
}
#[cfg(test)]
mod checker_tests {
use crate::{Options, ProofLang, check_text_with};
#[test]
fn vietnamese_paragraphs_under_english_get_no_marks() {
let text = "Hôm nay tôi đi học, trời rất đẹp và mát mẻ.";
let spans = [(0..text.len(), ProofLang::En)];
assert!(check_text_with(text, &spans, &Options::default(), true, false).is_empty());
// No language at all (the document default) behaves the same.
assert!(check_text_with(text, &[], &Options::default(), true, false).is_empty());
}
/// Without the Language data too: the installed English dictionary or the built-in word list marks
/// `yesterdy` (R6.4).
#[test]
fn english_keeps_its_marks_and_skips_vietnamese_names() {
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
let text = "We met Nguyễn yesterdy at the office.";
let issues = check_text_with(text, &[(0..text.len(), ProofLang::En)], &Options::default(), false, false);
let flagged: Vec<&str> = issues.iter().filter_map(|i| text.get(i.start..i.end)).collect();
assert_eq!(flagged, ["yesterdy"]);
}
}