mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-08 19:42:12 +00:00
Add craft-proof: Hunspell affix dictionaries, dictionary discovery and language tags
The language-independent spelling mechanism from WordCraft's wordcraft-proof (storytold/craft-libs#30): the affix engine (AffixDict, with its compact serialized form), finding and loading an installed Hunspell .aff/.dic pair, and BCP 47 tag normalization. What was WordCraft policy is now a parameter: hunspell::Search carries the app's dictionaries-folder setting, its environment variable and its Windows folder name, instead of a global and hard-coded WordCraft names. Which languages an app proofs, suggestion ranking, grammar rules and user dictionaries stay in the apps. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Michał Marcin BrzuchalskiandClaude Opus 5.5 committed
1 parent
3c9702725d
commit
20dfbeb48a
9 files changed
+1649
-6
No files matched your search
Generated
+174
-6
@@ -144,7 +144,16 @@ version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09ec2f926cc3060f09db9ebc5b52823d85268d24bb917e472c0c4bea35780a7d"
|
||||
dependencies = [
|
||||
"bit-vec",
|
||||
"bit-vec 0.9.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bit-set"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c"
|
||||
dependencies = [
|
||||
"bit-vec 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -153,6 +162,16 @@ version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51"
|
||||
|
||||
[[package]]
|
||||
name = "bit-vec"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269"
|
||||
dependencies = [
|
||||
"borsh",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
@@ -183,6 +202,30 @@ dependencies = [
|
||||
"objc2 0.6.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "borsh"
|
||||
version = "1.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a"
|
||||
dependencies = [
|
||||
"borsh-derive",
|
||||
"bytes",
|
||||
"cfg_aliases",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "borsh-derive"
|
||||
version = "1.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"proc-macro-crate",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.3"
|
||||
@@ -283,6 +326,17 @@ version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
|
||||
|
||||
[[package]]
|
||||
name = "chacha20"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clipboard-win"
|
||||
version = "5.4.1"
|
||||
@@ -371,6 +425,21 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core_detect"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-print"
|
||||
version = "0.1.0"
|
||||
@@ -379,6 +448,14 @@ dependencies = [
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-proof"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"miniz_oxide 0.8.9",
|
||||
"proptest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-ui"
|
||||
version = "0.1.0"
|
||||
@@ -752,6 +829,12 @@ version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4"
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
||||
|
||||
[[package]]
|
||||
name = "foldhash"
|
||||
version = "0.2.0"
|
||||
@@ -837,6 +920,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1259,7 +1343,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a616d2fb8c89516ac2723a581f69d6c18576046bed761bd6b305e5618e6ae130"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bit-set",
|
||||
"bit-set 0.10.0",
|
||||
"bitflags 2.13.2",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
@@ -1878,12 +1962,38 @@ version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5"
|
||||
|
||||
[[package]]
|
||||
name = "proptest"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0"
|
||||
dependencies = [
|
||||
"bit-set 0.11.1",
|
||||
"bit-vec 0.10.1",
|
||||
"bitflags 2.13.2",
|
||||
"chacha20",
|
||||
"core_detect",
|
||||
"num-traits",
|
||||
"rand",
|
||||
"rand_xorshift",
|
||||
"regex-syntax",
|
||||
"rusty-fork",
|
||||
"tempfile",
|
||||
"unarray",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pxfm"
|
||||
version = "0.1.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"
|
||||
|
||||
[[package]]
|
||||
name = "quick-error"
|
||||
version = "1.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
|
||||
|
||||
[[package]]
|
||||
name = "quick-error"
|
||||
version = "2.0.1"
|
||||
@@ -1914,6 +2024,31 @@ version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.10.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
|
||||
|
||||
[[package]]
|
||||
name = "rand_xorshift"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf"
|
||||
dependencies = [
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "range-alloc"
|
||||
version = "0.1.5"
|
||||
@@ -1976,6 +2111,12 @@ dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
||||
|
||||
[[package]]
|
||||
name = "renderdoc-sys"
|
||||
version = "1.1.0"
|
||||
@@ -2064,6 +2205,18 @@ version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
||||
|
||||
[[package]]
|
||||
name = "rusty-fork"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2"
|
||||
dependencies = [
|
||||
"fnv",
|
||||
"quick-error 1.2.3",
|
||||
"tempfile",
|
||||
"wait-timeout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "same-file"
|
||||
version = "1.0.6"
|
||||
@@ -2397,7 +2550,7 @@ dependencies = [
|
||||
"fax",
|
||||
"flate2",
|
||||
"half",
|
||||
"quick-error",
|
||||
"quick-error 2.0.1",
|
||||
"weezl",
|
||||
"zune-jpeg",
|
||||
]
|
||||
@@ -2506,6 +2659,12 @@ dependencies = [
|
||||
"rustc-hash 2.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unarray"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-general-category"
|
||||
version = "1.1.0"
|
||||
@@ -2591,6 +2750,15 @@ version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "wait-timeout"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.5.0"
|
||||
@@ -2842,8 +3010,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14c018fce9b6270aa203c2fdd56f3cce996713534bd757e4ea58c8560b121f14"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bit-set",
|
||||
"bit-vec",
|
||||
"bit-set 0.10.0",
|
||||
"bit-vec 0.9.1",
|
||||
"bitflags 2.13.2",
|
||||
"bytemuck",
|
||||
"cfg_aliases",
|
||||
@@ -2915,7 +3083,7 @@ dependencies = [
|
||||
"android_system_properties",
|
||||
"arrayvec",
|
||||
"ash",
|
||||
"bit-set",
|
||||
"bit-set 0.10.0",
|
||||
"bitflags 2.13.2",
|
||||
"block2 0.6.2",
|
||||
"bytemuck",
|
||||
|
||||
@@ -12,6 +12,7 @@ repository = "https://github.com/storytold/craft-libs"
|
||||
[workspace.dependencies]
|
||||
# internal: add each crate here as `craft-<name> = { path = "crates/<name>" }`
|
||||
craft-print = { path = "crates/print" }
|
||||
craft-proof = { path = "crates/proof" }
|
||||
craft-ui = { path = "crates/ui" }
|
||||
egui = { version = "0.36", default-features = false }
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
[package]
|
||||
name = "craft-proof"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
description = "Spelling for the Craft apps: Hunspell affix dictionaries, installed-dictionary discovery and language tags"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
miniz_oxide = "0.8"
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = "1"
|
||||
@@ -0,0 +1,31 @@
|
||||
# craft-proof
|
||||
|
||||
Spelling mechanism shared by the Craft apps: read a Hunspell `.aff`/`.dic` pair installed on the
|
||||
system and check words against it. Seeded from WordCraft's `wordcraft-proof`; GridCraft,
|
||||
DeckCraft and PdfCraft are the next consumers (storytold/craft-libs#30). No dependency on any
|
||||
app or UI framework. Requires Rust 1.95.
|
||||
|
||||
```rust,no_run
|
||||
let search = craft_proof::hunspell::Search::for_app("WordCraft");
|
||||
if let Some((aff, dic)) = search.find(&["pl_PL", "pl"]) {
|
||||
let dict = craft_proof::hunspell::load(&aff, &dic)?;
|
||||
assert!(dict.check("kotami"));
|
||||
}
|
||||
# Ok::<(), String>(())
|
||||
```
|
||||
|
||||
- **Dictionaries are not bundled.** Apps read what the OS, a package manager or LibreOffice
|
||||
installed; without one a language simply has no spelling check.
|
||||
- **Search order** (`Search`): the app's dictionaries folder setting, `<APP>_DICTIONARIES`, the
|
||||
app's folder under the Windows application-data folders, then the usual Hunspell and
|
||||
LibreOffice locations for the platform. Nothing on wasm.
|
||||
- **Affix engine** (`AffixDict`): the Hunspell subset real dictionaries such as SJP.PL (Polish)
|
||||
and VERO (Brazilian Portuguese) use: one-character or `FLAG UTF-8` flags, prefixes and
|
||||
suffixes with conditions and cross products, continuation classes, `FORBIDDENWORD`, and the
|
||||
`TRY`/`MAP`/`REP` suggestion tables. Compounding and long or numeric flags are rejected with
|
||||
an error rather than misread.
|
||||
- **Compact form**: `AffixDict::to_bytes`/`from_bytes` store a loaded dictionary deflated
|
||||
(Polish is under 1 MB), so an app can cache it instead of parsing the `.aff`/`.dic` each start.
|
||||
|
||||
What stays in each app: which languages it proofs, suggestion ranking and grammar rules per
|
||||
language, the user dictionary, and how marks are shown.
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,40 @@
|
||||
//! A small, fast multiplicative hasher (FxHash-style) for the affix engine's stem table and rule
|
||||
//! maps: keys are short strings and packed integers, and HashDoS resistance is not needed for
|
||||
//! dictionary data the app chose to load.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::hash::{BuildHasherDefault, Hasher};
|
||||
|
||||
/// Small, fast multiplicative hasher for packed integer keys and short strings (FxHash-style).
|
||||
#[derive(Default, Clone, Copy)]
|
||||
pub struct FxHasher(u64);
|
||||
|
||||
impl Hasher for FxHasher {
|
||||
fn finish(&self) -> u64 {
|
||||
self.0
|
||||
}
|
||||
fn write(&mut self, bytes: &[u8]) {
|
||||
let (chunks, rest) = bytes.as_chunks::<8>();
|
||||
for c in chunks {
|
||||
self.write_u64(u64::from_le_bytes(*c));
|
||||
}
|
||||
for &b in rest {
|
||||
self.write_u64(b as u64);
|
||||
}
|
||||
}
|
||||
fn write_u8(&mut self, i: u8) {
|
||||
self.write_u64(i as u64);
|
||||
}
|
||||
fn write_usize(&mut self, i: usize) {
|
||||
self.write_u64(i as u64);
|
||||
}
|
||||
fn write_u64(&mut self, i: u64) {
|
||||
self.0 = (self.0.rotate_left(5) ^ i).wrapping_mul(0x51_7c_c1_b7_27_22_0a_95);
|
||||
}
|
||||
fn write_u128(&mut self, i: u128) {
|
||||
self.write_u64(i as u64);
|
||||
self.write_u64((i >> 64) as u64);
|
||||
}
|
||||
}
|
||||
|
||||
pub type FxMap<K, V> = HashMap<K, V, BuildHasherDefault<FxHasher>>;
|
||||
@@ -0,0 +1,208 @@
|
||||
//! Hunspell dictionaries installed on the system, read at runtime.
|
||||
//!
|
||||
//! The Craft apps don't bundle large spelling dictionaries. For a language with an affix
|
||||
//! dictionary, an app reads the standard Hunspell `.aff`/`.dic` pair installed by the operating
|
||||
//! system, a package manager or LibreOffice, with [`load`]. When none is installed the language
|
||||
//! has no spelling check (no marks, no errors), like Word without the language's proofing tools.
|
||||
//!
|
||||
//! Where to look is per app ([`Search`]): its own dictionaries folder (a setting), an environment
|
||||
//! variable and a folder under the user's application data, then the usual system locations. On
|
||||
//! the web there are none.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
|
||||
/// Largest `.aff` or `.dic` file read (real ones are a few MB).
|
||||
const MAX_FILE: u64 = 64 << 20;
|
||||
|
||||
/// Where one app looks for Hunspell dictionaries, most specific first.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Search {
|
||||
/// A folder searched before everything else, such as a "dictionaries folder" setting.
|
||||
pub extra_dir: Option<PathBuf>,
|
||||
/// An environment variable naming a folder searched next (`WORDCRAFT_DICTIONARIES`).
|
||||
pub env_var: Option<String>,
|
||||
/// The app's folder name under the Windows application-data folders (`WordCraft` searches
|
||||
/// `%LOCALAPPDATA%\WordCraft\dictionaries`).
|
||||
pub app_dir: Option<String>,
|
||||
}
|
||||
|
||||
impl Search {
|
||||
/// The search for an app called `app` (`WordCraft`): its environment variable is the
|
||||
/// upper-cased name with `_DICTIONARIES` (`WORDCRAFT_DICTIONARIES`) and its Windows folder the
|
||||
/// name itself.
|
||||
pub fn for_app(app: &str) -> Search {
|
||||
Search { extra_dir: None, env_var: Some(format!("{}_DICTIONARIES", app.to_ascii_uppercase())), app_dir: Some(app.to_string()) }
|
||||
}
|
||||
|
||||
/// Directories that may hold Hunspell dictionaries, most specific first. `lo_exts`, when a
|
||||
/// language is being searched for, adds LibreOffice's extension folders for it (`dict-pt-BR`).
|
||||
pub fn dirs(&self, lo_exts: Option<&[String]>) -> Vec<PathBuf> {
|
||||
let mut dirs: Vec<PathBuf> = self.extra_dir.iter().cloned().collect();
|
||||
if cfg!(target_arch = "wasm32") {
|
||||
return dirs;
|
||||
}
|
||||
if let Some(d) = self.env_var.as_deref().and_then(std::env::var_os).filter(|v| !v.is_empty()) {
|
||||
dirs.push(PathBuf::from(d));
|
||||
}
|
||||
// LibreOffice keeps each language under `…/share/extensions/<dict-xx>`.
|
||||
let lo_dirs = |base: &Path| -> Vec<PathBuf> { lo_exts.map_or_else(Vec::new, |exts| exts.iter().map(|e| base.join(e)).collect()) };
|
||||
let home = std::env::var_os("HOME").filter(|v| !v.is_empty()).map(PathBuf::from);
|
||||
if cfg!(windows) {
|
||||
for var in ["LOCALAPPDATA", "APPDATA"] {
|
||||
if let Some(base) = std::env::var_os(var).filter(|v| !v.is_empty()).map(PathBuf::from) {
|
||||
if let Some(app) = &self.app_dir {
|
||||
dirs.push(base.join(app).join("dictionaries"));
|
||||
}
|
||||
dirs.push(base.join("hunspell"));
|
||||
}
|
||||
}
|
||||
for var in ["ProgramFiles", "ProgramFiles(x86)"] {
|
||||
if let Some(base) = std::env::var_os(var).filter(|v| !v.is_empty()).map(PathBuf::from) {
|
||||
dirs.extend(lo_dirs(&base.join("LibreOffice").join("share").join("extensions")));
|
||||
}
|
||||
}
|
||||
} else if cfg!(target_os = "macos") {
|
||||
if let Some(h) = &home {
|
||||
dirs.push(h.join("Library/Spelling"));
|
||||
}
|
||||
dirs.push(PathBuf::from("/Library/Spelling"));
|
||||
dirs.push(PathBuf::from("/opt/homebrew/share/hunspell"));
|
||||
dirs.push(PathBuf::from("/usr/local/share/hunspell"));
|
||||
dirs.extend(lo_dirs(Path::new("/Applications/LibreOffice.app/Contents/Resources/extensions")));
|
||||
} else {
|
||||
if let Some(h) = &home {
|
||||
dirs.push(h.join(".local/share/hunspell"));
|
||||
}
|
||||
for d in
|
||||
["/usr/share/hunspell", "/usr/share/myspell", "/usr/share/myspell/dicts", "/usr/local/share/hunspell", "/usr/local/share/myspell"]
|
||||
{
|
||||
dirs.push(PathBuf::from(d));
|
||||
}
|
||||
for d in ["/usr/lib/libreoffice/share/extensions", "/opt/libreoffice/share/extensions"] {
|
||||
dirs.extend(lo_dirs(Path::new(d)));
|
||||
}
|
||||
}
|
||||
dirs
|
||||
}
|
||||
|
||||
/// The first `<name>.aff`/`<name>.dic` pair found for any of `names` (`pl_PL`, `pl`),
|
||||
/// searching [`Search::dirs`] (including LibreOffice's extension folders for the first name).
|
||||
pub fn find(&self, names: &[&str]) -> Option<(PathBuf, PathBuf)> {
|
||||
let lo = names.first().copied().map(lo_ext_dirs);
|
||||
self.dirs(lo.as_deref()).into_iter().find_map(|dir| find_in(&dir, names))
|
||||
}
|
||||
}
|
||||
|
||||
/// The LibreOffice extension directories a Hunspell dictionary name may live in: LibreOffice's
|
||||
/// IDs are per dictionary — Polish is `dict-pl`, Brazilian Portuguese `dict-pt-BR` — so both
|
||||
/// the primary-language folder and the dashed full name are tried.
|
||||
fn lo_ext_dirs(name: &str) -> Vec<String> {
|
||||
let primary = name.split(['-', '_']).next().unwrap_or(name);
|
||||
let dashed = name.replace('_', "-");
|
||||
let first = format!("dict-{primary}");
|
||||
let full = format!("dict-{dashed}");
|
||||
if full == first { vec![first] } else { vec![first, full] }
|
||||
}
|
||||
|
||||
/// The first `<name>.aff`/`<name>.dic` pair for any of `names` in `dir`.
|
||||
pub fn find_in(dir: &Path, names: &[&str]) -> Option<(PathBuf, PathBuf)> {
|
||||
names.iter().find_map(|n| {
|
||||
let aff = dir.join(format!("{n}.aff"));
|
||||
let dic = dir.join(format!("{n}.dic"));
|
||||
(aff.is_file() && dic.is_file()).then_some((aff, dic))
|
||||
})
|
||||
}
|
||||
|
||||
/// Read a Hunspell pair from disk (size-capped), decoded per the `.aff`'s `SET` line.
|
||||
pub fn load(aff: &Path, dic: &Path) -> Result<AffixDict, String> {
|
||||
let read = |p: &Path| -> Result<Vec<u8>, String> {
|
||||
let len = std::fs::metadata(p).map_err(|e| format!("{}: {e}", p.display()))?.len();
|
||||
if len > MAX_FILE {
|
||||
return Err(format!("{}: too large ({len} bytes)", p.display()));
|
||||
}
|
||||
std::fs::read(p).map_err(|e| format!("{}: {e}", p.display()))
|
||||
};
|
||||
let (aff_text, dic_text) = decode(&read(aff)?, &read(dic)?)?;
|
||||
let source = format!("Hunspell dictionary {}", dic.display());
|
||||
AffixDict::from_hunspell(&aff_text, &dic_text, &[source.as_str()])
|
||||
}
|
||||
|
||||
/// ISO 8859-2 code points 0xA0..=0xFF (0x00..=0x9F map to themselves).
|
||||
const ISO_8859_2_HIGH: &str = "\u{a0}Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\u{ad}ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙";
|
||||
|
||||
fn decode_latin2(bytes: &[u8]) -> String {
|
||||
let high: Vec<char> = ISO_8859_2_HIGH.chars().collect();
|
||||
bytes.iter().map(|&b| if b < 0xa0 { char::from(b) } else { high.get(usize::from(b - 0xa0)).copied().unwrap_or('\u{fffd}') }).collect()
|
||||
}
|
||||
|
||||
/// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names
|
||||
/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use).
|
||||
pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
|
||||
let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string();
|
||||
let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase();
|
||||
match set.as_str() {
|
||||
"UTF-8" | "UTF8" => {
|
||||
let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));
|
||||
Ok((text(aff, ".aff")?, text(dic, ".dic")?))
|
||||
}
|
||||
"ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))),
|
||||
other => Err(format!("unsupported encoding {other}")),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn decodes_latin2_and_utf8() {
|
||||
let aff = b"SET ISO8859-2\nTRY a\n";
|
||||
let (a, d) = decode(aff, b"2\n\xbf\xf3\xb3w\n").unwrap();
|
||||
assert!(a.contains("TRY a"));
|
||||
assert_eq!(d, "2\nżółw\n");
|
||||
let (_, d) = decode("SET UTF-8\n".as_bytes(), "żółw".as_bytes()).unwrap();
|
||||
assert_eq!(d, "żółw");
|
||||
assert!(decode(b"SET KOI8-R\n", b"").is_err());
|
||||
assert!(decode(b"SET UTF-8\n", b"\xff\xfe").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn libreoffice_extension_dirs_follow_the_language() {
|
||||
assert_eq!(lo_ext_dirs("pl_PL"), ["dict-pl", "dict-pl-PL"]);
|
||||
assert_eq!(lo_ext_dirs("pt_BR"), ["dict-pt", "dict-pt-BR"]);
|
||||
assert_eq!(lo_ext_dirs("de-DE"), ["dict-de", "dict-de-DE"]);
|
||||
let search = Search::default();
|
||||
let dirs = search.dirs(Some(&lo_ext_dirs("pt_BR")));
|
||||
assert!(dirs.iter().any(|d| d.ends_with("dict-pt-BR")), "{dirs:?}");
|
||||
assert!(search.dirs(None).iter().all(|d| !d.to_string_lossy().contains("dict-")), "no language: no LibreOffice guess");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_app_names_its_own_folder_and_variable() {
|
||||
let s = Search::for_app("GridCraft");
|
||||
assert_eq!(s.env_var.as_deref(), Some("GRIDCRAFT_DICTIONARIES"));
|
||||
assert_eq!(s.app_dir.as_deref(), Some("GridCraft"));
|
||||
let extra = std::env::temp_dir().join("craft-proof-extra");
|
||||
let s = Search { extra_dir: Some(extra.clone()), ..Search::for_app("PdfCraft") };
|
||||
assert_eq!(s.dirs(None).first(), Some(&extra), "the setting comes first");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finds_and_loads_a_pair_in_a_directory() {
|
||||
let dir = std::env::temp_dir().join(format!("craft-proof-hunspell-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
assert!(find_in(&dir, &["xx_XX"]).is_none(), "no files: nothing found");
|
||||
std::fs::write(dir.join("xx_XX.aff"), "SET UTF-8\nSFX A Y 1\nSFX A 0 y .\n").unwrap();
|
||||
assert!(find_in(&dir, &["xx_XX"]).is_none(), "an .aff alone isn't a dictionary");
|
||||
std::fs::write(dir.join("xx_XX.dic"), "1\nkot/A\n").unwrap();
|
||||
let (aff, dic) = find_in(&dir, &["yy", "xx_XX"]).unwrap();
|
||||
let d = load(&aff, &dic).unwrap();
|
||||
assert!(d.check("kot") && d.check("koty") && !d.check("kotx"));
|
||||
assert!(load(&dir.join("missing.aff"), &dic).is_err());
|
||||
let found = Search { extra_dir: Some(dir.clone()), ..Search::default() }.find(&["xx_XX"]);
|
||||
assert_eq!(found, Some((aff, dic)), "the extra folder is searched");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
//! Spelling mechanism shared by the Craft apps: Hunspell affix dictionaries and where to find
|
||||
//! them, with no opinion on which languages an app proofs or how it shows the result.
|
||||
//!
|
||||
//! - [`affix`]: an affix dictionary ([`affix::AffixDict`]) built from a Hunspell `.aff`/`.dic`
|
||||
//! pair, word checks, generated forms, the suggestion tables (`TRY`, `MAP`, `REP`) and a
|
||||
//! compact serialized form ([`affix::AffixDict::to_bytes`]) for fast loading.
|
||||
//! - [`hunspell`]: finding an installed pair per app ([`hunspell::Search`]), decoding it from
|
||||
//! UTF-8 or ISO 8859-2 and loading it.
|
||||
//! - [`tag`]: canonical BCP 47 tags and primary subtags, for mapping a text's language to a
|
||||
//! dictionary name.
|
||||
//!
|
||||
//! Per-app policy stays in the apps: the list of proofed languages, suggestion ranking and
|
||||
//! grammar rules per language, user dictionaries and how marks are drawn. The code comes from
|
||||
//! WordCraft's `wordcraft-proof` (storytold/craft-libs#30).
|
||||
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
pub mod affix;
|
||||
mod fx;
|
||||
pub mod hunspell;
|
||||
pub mod tag;
|
||||
|
||||
pub use affix::AffixDict;
|
||||
@@ -0,0 +1,80 @@
|
||||
//! Language tags as apps receive them (`w:lang`, `a:rPr/@lang`, `LANG`, `navigator.language`):
|
||||
//! canonical BCP 47 form and the primary language subtag. Which languages an app proofs, and
|
||||
//! with which rules, stays in the app.
|
||||
|
||||
/// A BCP 47 tag in canonical case (`pl-PL`, `en-GB`, `zh-Hant-TW`, `x-none`), or `None` when
|
||||
/// `tag` isn't shaped like one: 1–35 ASCII letters, digits and `-`/`_` separators, starting with
|
||||
/// a 2–8 letter language (or `x`/`i` for private and grandfathered tags). `_` becomes `-`.
|
||||
pub fn normalize_tag(tag: &str) -> Option<String> {
|
||||
let tag = tag.trim();
|
||||
if tag.is_empty() || tag.len() > 35 || !tag.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') {
|
||||
return None;
|
||||
}
|
||||
let parts: Vec<&str> = tag.split(['-', '_']).collect();
|
||||
if parts.iter().any(|p| p.is_empty() || p.len() > 8) {
|
||||
return None;
|
||||
}
|
||||
let first = parts.first()?;
|
||||
let language = (2..=8).contains(&first.len()) && first.chars().all(|c| c.is_ascii_alphabetic());
|
||||
if !language && !first.eq_ignore_ascii_case("x") && !first.eq_ignore_ascii_case("i") {
|
||||
return None;
|
||||
}
|
||||
let letters = |p: &str| p.chars().all(|c| c.is_ascii_alphabetic());
|
||||
// After a singleton (`x-…` private use, `u-…` extensions) subtags are lower case.
|
||||
let singleton = parts.iter().position(|p| p.len() == 1).unwrap_or(usize::MAX);
|
||||
let out: Vec<String> = parts
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, p)| {
|
||||
if i >= singleton {
|
||||
p.to_ascii_lowercase()
|
||||
} else if i > 0 && p.len() == 2 && letters(p) {
|
||||
p.to_ascii_uppercase() // region
|
||||
} else if i > 0 && p.len() == 4 && letters(p) {
|
||||
let lower = p.to_ascii_lowercase();
|
||||
let mut c = lower.chars();
|
||||
c.next().map(|f| f.to_ascii_uppercase().to_string() + c.as_str()).unwrap_or_default() // script
|
||||
} else {
|
||||
p.to_ascii_lowercase()
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
Some(out.join("-"))
|
||||
}
|
||||
|
||||
/// The primary language subtag of a tag or POSIX locale, lower case (`pl` for `pl-PL`,
|
||||
/// `pl_PL.UTF-8` and `PL`); `None` when there is none.
|
||||
pub fn primary_subtag(tag: &str) -> Option<String> {
|
||||
let primary = tag.trim().split(['-', '_', '.', '@']).next().unwrap_or("");
|
||||
(!primary.is_empty() && primary.chars().all(|c| c.is_ascii_alphabetic())).then(|| primary.to_ascii_lowercase())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn tags_normalize() {
|
||||
assert_eq!(normalize_tag("PL-pl").as_deref(), Some("pl-PL"));
|
||||
assert_eq!(normalize_tag(" pl_pl ").as_deref(), Some("pl-PL"));
|
||||
assert_eq!(normalize_tag("pl").as_deref(), Some("pl"));
|
||||
assert_eq!(normalize_tag("zh-hant-tw").as_deref(), Some("zh-Hant-TW"));
|
||||
assert_eq!(normalize_tag("es-419").as_deref(), Some("es-419"));
|
||||
assert_eq!(normalize_tag("x-none").as_deref(), Some("x-none"));
|
||||
assert_eq!(normalize_tag("de-DE-u-co-PHONEBK").as_deref(), Some("de-DE-u-co-phonebk"));
|
||||
for bad in ["", " ", "p", "1pl", "pl--PL", "pl PL", "pl-PL;", "polski język", "-pl", "pl-", "abcdefghi", &"a".repeat(40)] {
|
||||
assert_eq!(normalize_tag(bad), None, "{bad:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn primary_subtags() {
|
||||
for t in ["pl", "pl-PL", "PL_pl", "pl_PL.UTF-8", " pl-PL ", "pl_PL@euro"] {
|
||||
assert_eq!(primary_subtag(t).as_deref(), Some("pl"), "{t}");
|
||||
}
|
||||
assert_eq!(primary_subtag("zh-Hant-TW").as_deref(), Some("zh"));
|
||||
for t in ["", " ", "-pl", "1pl"] {
|
||||
assert_eq!(primary_subtag(t), None, "{t:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user