diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..1eb59a05 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,4 @@ +# craft-proof moved here from WordCraft's crates/proof; the WordCraft maintainers own it. +# Confirm these handles before merge (WordCraft has no CODEOWNERS file to copy from). +/crates/proof/ @dylanmtaylor @aiongg @skylartaylor +/.github/workflows/proof.yml @dylanmtaylor @aiongg @skylartaylor diff --git a/.github/workflows/proof.yml b/.github/workflows/proof.yml new file mode 100644 index 00000000..db171ee1 --- /dev/null +++ b/.github/workflows/proof.yml @@ -0,0 +1,60 @@ +name: Proofing + +on: + push: + branches: [main] + paths: ["crates/proof/**", "Cargo.toml", "Cargo.lock", ".github/workflows/proof.yml"] + pull_request: + paths: ["crates/proof/**", "Cargo.toml", "Cargo.lock", ".github/workflows/proof.yml"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: craft-proof-${{ github.ref }} + cancel-in-progress: true + +jobs: + check: + timeout-minutes: 20 + name: Proofing lint, tests with and without Language data, WASM + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + # The Language data (craft-i18n `language-data/`), embedded by crates/proof/build.rs. + # TODO(CP1.2): pin `ref` to the craft-i18n commit that adds language-data/ once it merges. + - uses: actions/checkout@v4 + with: + repository: storytold/craft-i18n + path: craft-i18n + persist-credentials: false + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + with: + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + key: craft-proof + - run: cargo fmt -p craft-proof --check + - name: Without Language data (a fork, docs.rs, offline) + run: | + cargo clippy -p craft-proof --all-targets --locked -- -D warnings + cargo test -p craft-proof --locked + cargo check -p craft-proof --locked --target wasm32-unknown-unknown + - name: A release without Language data fails + run: | + if CRAFT_I18N_REQUIRED=1 cargo check -p craft-proof --locked 2>build.log; then + echo "::error::CRAFT_I18N_REQUIRED=1 without CRAFT_I18N_DIR built"; exit 1 + fi + grep -q "CRAFT_I18N_REQUIRED=1 but CRAFT_I18N_DIR is unset" build.log + - name: With Language data + env: + CRAFT_I18N_DIR: ${{ github.workspace }}/craft-i18n + CRAFT_I18N_REQUIRED: "1" + run: | + cargo clippy -p craft-proof --all-targets --locked -- -D warnings + cargo test -p craft-proof --locked + cargo check -p craft-proof --locked --target wasm32-unknown-unknown diff --git a/Cargo.lock b/Cargo.lock index 4fcd223b..368a83a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1002,6 +1002,16 @@ dependencies = [ "winsafe", ] +[[package]] +name = "craft-proof" +version = "0.1.0" +dependencies = [ + "log", + "miniz_oxide 0.9.1", + "proptest", + "unicode-normalization", +] + [[package]] name = "craft-quit" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a98e231b..3946fd50 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ craft-media = { path = "crates/media" } craft-menu = { path = "crates/menu" } craft-prepress = { path = "crates/prepress" } craft-print = { path = "crates/print" } +craft-proof = { path = "crates/proof" } craft-quit = { path = "crates/quit" } craft-settings = { path = "crates/settings" } craft-text = { path = "crates/text" } diff --git a/README.md b/README.md index 30d08db6..58b561d6 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,9 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = "` embeds `language-data/`. +- Unset (a fork, docs.rs, offline): the crate builds with a warning and no embedded Language + data. English spelling then marks nothing and suggests nothing, and hyphenation that needs the + data is off; grammar and the Hunspell languages work as before. +- `CRAFT_I18N_REQUIRED=1` (releases) makes a missing directory or file an error. + +Tests that pin results of the data are ignored without it; `src/no_data.rs` runs every public +entry point with no data embedded. + +## Check + +```sh +cargo fmt -p craft-proof --check +cargo clippy -p craft-proof --all-targets -- -D warnings +CRAFT_I18N_DIR=/abs/path/craft-i18n cargo test -p craft-proof +cargo test -p craft-proof # without Language data +cargo check -p craft-proof --target wasm32-unknown-unknown +``` + +CI: `.github/workflows/proof.yml`. + +## Licence + +MIT OR Apache-2.0. The Language data carries its own licences, recorded in craft-i18n's +`ATTRIBUTION.md`. diff --git a/crates/proof/build.rs b/crates/proof/build.rs new file mode 100644 index 00000000..2620e54f --- /dev/null +++ b/crates/proof/build.rs @@ -0,0 +1,97 @@ +//! Embed the Language data from a craft-i18n checkout (storytold/craft-i18n, `language-data/`). +//! +//! `CRAFT_I18N_DIR`, an absolute path to a craft-i18n checkout: `$OUT_DIR/language_data.rs` holds +//! one constant per file in [`FILES`], each an `include_str!`/`include_bytes!` of +//! `$CRAFT_I18N_DIR/language-data/`. Unset, or not a checkout, or a file missing (a warning): +//! that constant is empty, and the crate has no embedded Language data for it (English spelling +//! marks nothing, hyphenation that needs the data is off). With `CRAFT_I18N_REQUIRED=1` (releases) +//! any of those is an error instead. +//! +//! When the English word list is embedded, the `craft_proof_data` cfg is set (the tests that pin +//! results of the data use it). + +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; + +/// (constant, path under `language-data/`, bytes rather than text). +const FILES: &[(&str, &str, bool)] = &[ + ("EN_US_DIC", "en-us/en-us.dic", true), + ("EN_US_PAT", "en-us/en-us.pat", false), + ("EXTRA_EN", "en-us/en-extra.txt", false), + ("EN_VARIANTS", "en-us/en-variants.txt", false), + ("PL_PAT", "pl/pl.pat", false), + ("PT_PAT", "pt/pt.pat", false), + ("IT_PAT", "it/it.pat", false), + ("DE_PAT", "de/de.pat", false), + ("EXTRA_RU", "ru/ru-extra.txt", false), +]; + +fn main() { + println!("cargo::rerun-if-changed=build.rs"); + println!("cargo::rerun-if-env-changed=CRAFT_I18N_DIR"); + println!("cargo::rerun-if-env-changed=CRAFT_I18N_REQUIRED"); + println!("cargo::rustc-check-cfg=cfg(craft_proof_data)"); + let required = std::env::var("CRAFT_I18N_REQUIRED").is_ok_and(|v| v.trim() == "1"); + let level = if required { "error" } else { "warning" }; + let dir = match std::env::var_os("CRAFT_I18N_DIR").filter(|d| !d.is_empty()) { + Some(d) => match language_data(&PathBuf::from(d)) { + Ok(dir) => Some(dir), + Err(e) => { + println!("cargo::{level}=CRAFT_I18N_DIR: {e}; craft-proof embeds no Language data"); + None + } + }, + None => { + if required { + println!("cargo::error=CRAFT_I18N_REQUIRED=1 but CRAFT_I18N_DIR is unset: a release must embed the Language data"); + } else { + println!("cargo::warning=CRAFT_I18N_DIR is unset: craft-proof embeds no Language data (no English spelling marks, no hyphenation)"); + } + None + } + }; + let mut o = String::from("// Generated by craft-proof's build.rs from CRAFT_I18N_DIR/language-data. Do not edit.\n"); + let mut english = false; + for &(name, rel, bytes) in FILES { + let file = dir.as_ref().map(|d| d.join(rel)).filter(|f| f.is_file()); + if let (Some(d), None) = (&dir, &file) { + println!("cargo::{level}=CRAFT_I18N_DIR: {} is missing; craft-proof embeds none", d.join(rel).display()); + } + let (ty, empty, mac) = if bytes { ("&[u8]", "b\"\"", "include_bytes") } else { ("&str", "\"\"", "include_str") }; + let value = match &file { + Some(f) => { + println!("cargo::rerun-if-changed={}", f.display()); + if name == "EN_US_DIC" { + english = true; + } + // `{:?}` escapes the path (Windows backslashes) as a Rust string literal. + format!("{mac}!({:?})", f.display().to_string()) + } + None => empty.to_string(), + }; + let _ = writeln!(o, "/// `language-data/{rel}` (empty when it isn't embedded).\npub(crate) static {name}: {ty} = {value};"); + } + if english { + println!("cargo::rustc-cfg=craft_proof_data"); + } + let out = PathBuf::from(std::env::var_os("OUT_DIR").unwrap_or_default()).join("language_data.rs"); + if let Err(e) = std::fs::write(&out, o) { + println!("cargo::error=writing {}: {e}", out.display()); + } +} + +/// `dir/language-data`, when `dir` is an absolute path to a checkout that has one. +fn language_data(dir: &Path) -> Result { + // Build scripts run in the crate's directory and `include_str!` resolves against the generated + // file, so a relative path would mean two different places. Not canonicalized: on Windows that + // gives `\\?\C:\…`, which `include_str!` can't read. + if !dir.is_absolute() { + return Err(format!("{} is not an absolute path", dir.display())); + } + let data = dir.join("language-data"); + if !data.is_dir() { + return Err(format!("{} has no language-data/ (a craft-i18n checkout older than its Language data?)", dir.display())); + } + println!("cargo::rerun-if-changed={}", data.display()); + Ok(data) +} diff --git a/crates/proof/src/dict.rs b/crates/proof/src/dict.rs index 81eb0de1..3bcedb0b 100644 --- a/crates/proof/src/dict.rs +++ b/crates/proof/src/dict.rs @@ -1,6 +1,6 @@ //! Hyphenation exception dictionary: lowercase words with their break points. //! -//! On-disk format (`assets/hyphenation/en-us.dic`): raw-deflate of +//! On-disk format (craft-i18n's `language-data/en-us/en-us.dic`): raw-deflate of //! `"DCHYPH1\n"` followed by one line per word, sorted by the plain word's bytes. Each line is //! front-coded against the previous one: a byte with the shared prefix length (in bytes of the //! hyphenated form), then the rest of the hyphenated form (`-` at every break), then `\n`. @@ -165,6 +165,7 @@ mod tests { assert!(back.get("dog").is_none()); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn oversized_data_is_refused() { const LIMIT: usize = 1 << 16; diff --git a/crates/proof/src/en_variants.rs b/crates/proof/src/en_variants.rs index 56b9f26f..07c16bb1 100644 --- a/crates/proof/src/en_variants.rs +++ b/crates/proof/src/en_variants.rs @@ -2,7 +2,7 @@ //! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`]) accepts, which it flags, //! and what it suggests instead (`colour` → `color` in American English). //! -//! The pairs come from `assets/spelling/en-variants.txt`, our own list of base forms with a +//! The pairs come from craft-i18n's `language-data/en-us/en-variants.txt`, our own list of base forms with a //! policy per English. Inflections are generated here by rule for the regular patterns (-or/-our, //! -er/-re, -ize/-ise, -ense/-ence…), so `colours`, `centred` and `organisation` follow their //! base forms. A form is right when the shared English word list knows it or knows its other @@ -15,7 +15,7 @@ use std::sync::OnceLock; use crate::ProofLang; /// Our list of American/British pairs: `POLICY AMERICAN BRITISH` per line, `#` comments. -const VARIANTS: &str = include_str!("../../../assets/spelling/en-variants.txt"); +use crate::data::EN_VARIANTS as VARIANTS; /// What one English does with a pair's two spellings. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -217,6 +217,7 @@ pub fn is_foreign(lower: &str, lang: ProofLang) -> bool { mod tests { use super::*; + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn every_line_parses() { let lines = VARIANTS.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')).count(); diff --git a/crates/proof/src/es/grammar.rs b/crates/proof/src/es/grammar.rs index 614a1f07..15806edb 100644 --- a/crates/proof/src/es/grammar.rs +++ b/crates/proof/src/es/grammar.rs @@ -336,7 +336,9 @@ mod tests { /// About 99 KB (under the 100,000-byte paragraph cap) took hundreds of milliseconds; it is now /// O(n log m): about 1 to 5 ms in a debug build, against about 300 ms for the scan, so the /// bound below leaves room on a loaded machine and still catches a return to the quadratic - /// lookup. + /// lookup. craft-libs builds tests at `opt-level = 0` (WordCraft used 1), where the linear + /// lookup took up to ~80 ms under a parallel test run, so the bound is 150 ms. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn many_addresses_before_a_question_are_checked_quickly() { for (unit, ending, whole_paragraph) in @@ -348,7 +350,7 @@ mod tests { let started = std::time::Instant::now(); let issues = check(&text, ProofLang::Es); let took = started.elapsed(); - assert!(took < std::time::Duration::from_millis(75), "{unit:?} took {took:?}"); + assert!(took < std::time::Duration::from_millis(150), "{unit:?} took {took:?}"); let missing: Vec<&Issue> = issues.iter().filter(|i| i.message.starts_with("Falta")).collect(); assert_eq!(missing.len(), 1, "{unit:?}: {missing:?}"); // The mark's sentence is the whole paragraph when the addresses don't end with a mark diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs index b203ff1a..60f4269c 100644 --- a/crates/proof/src/hyphen.rs +++ b/crates/proof/src/hyphen.rs @@ -3,7 +3,7 @@ //! **English** (and languages without patterns of their own, as before languages were told //! apart), lookup order for each word: //! 1. the **dictionary** ([`Dictionary::en_us`]) — the public-domain Moby Hyphenator II word list -//! by Grady Ward (~160k words), stored compactly in `assets/hyphenation/en-us.dic`; simple +//! by Grady Ward (~160k words), stored compactly in craft-i18n's `language-data/en-us/en-us.dic`; simple //! inflections (`'s`, `-s`, `-es`, `-ed`, `-d`) inherit the breaks of their stem; //! 2. **Liang patterns** ([`Patterns::en_us`]) for words not in the dictionary — our own pattern //! set, trained from the same list by DesignCraft's patgen (generated by DesignCraft's `hyphgen` example); @@ -30,18 +30,10 @@ pub use crate::lang::ProofLang; pub use dict::Dictionary; pub use patterns::Patterns; -static EN_US_DIC: &[u8] = include_bytes!("../../../assets/hyphenation/en-us.dic"); -static EN_US_PAT: &str = include_str!("../../../assets/hyphenation/en-us.pat"); -/// hyph-utf8's `hyph-pl.tex`, unmodified (its licence header is in the file). -static PL_PAT: &str = include_str!("../../../assets/hyphenation/pl.pat"); -/// hyph-utf8's `hyph-pt.tex`, unmodified (its BSD 3-clause header is in the file). -static PT_PAT: &str = include_str!("../../../assets/hyphenation/pt.pat"); -/// hyph-utf8's `hyph-it.tex`, unmodified (offered under MIT or LPPL; WordCraft uses MIT, whose -/// notice is in the file's header). -static IT_PAT: &str = include_str!("../../../assets/hyphenation/it.pat"); -/// hyph-utf8's `hyph-de-1996.tex` (reformed spelling), unmodified (MIT; the notice is in the -/// file's header). -static DE_PAT: &str = include_str!("../../../assets/hyphenation/de.pat"); +// The embedded Language data (`language-data/` of craft-i18n, see `build.rs`); each is empty when +// it isn't embedded. The TeX files are hyph-utf8's `hyph-pl.tex`, `hyph-pt.tex`, `hyph-it.tex` and +// `hyph-de-1996.tex` (reformed spelling), unmodified, their licence headers in the files. +use crate::data::{DE_PAT, EN_US_DIC, EN_US_PAT, IT_PAT, PL_PAT, PT_PAT}; impl Dictionary { /// The bundled US English dictionary (decoded on first use). @@ -152,6 +144,10 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec, Source) { return (crate::ru::hyphen::points(core), Source::Heuristic); } let dict = Dictionary::en_us(); + // No English Language data embedded: hyphenation is off. + if dict.is_empty() && Patterns::en_us().is_empty() { + return (vec![], Source::None); + } let s: String = core.iter().collect(); let from_mask = |m: u64, len: usize| (1..len.min(64)).filter(|&i| m & (1 << i) != 0).collect::>(); if let Some(m) = dict.get(&s) { @@ -178,6 +174,10 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec, Source) { /// Polish: exception words, then the patterns, then the Polish syllable heuristic. fn core_breaks_pl(core: &[char]) -> (Vec, Source) { let pats = Patterns::pl(); + // No Polish patterns embedded: hyphenation is off. + if pats.is_empty() { + return (vec![], Source::None); + } let s: String = core.iter().collect(); if let Some(p) = pats.exception(&s) { return (p.to_vec(), Source::Dictionary); @@ -193,6 +193,10 @@ fn core_breaks_pl(core: &[char]) -> (Vec, Source) { /// [`core_breaks_pl`] for a language with bundled patterns (Portuguese, Italian, German): its /// exception words, the patterns, then the generic heuristic. fn core_breaks_patterns(pats: &Patterns, core: &[char]) -> (Vec, Source) { + // The language's patterns aren't embedded: hyphenation is off. + if pats.is_empty() { + return (vec![], Source::None); + } let s: String = core.iter().collect(); if let Some(p) = pats.exception(&s) { return (p.to_vec(), Source::Dictionary); @@ -462,12 +466,14 @@ mod tests { hyphenate_word(w, &Limits::default()) } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn bundled_data_loads() { assert!(Dictionary::en_us().len() > 150_000, "{}", Dictionary::en_us().len()); assert!(Patterns::en_us().len() > 1000, "{}", Patterns::en_us().len()); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn dictionary_words() { assert_eq!(h("happen"), "hap-pen"); @@ -481,6 +487,7 @@ mod tests { assert_eq!(word_breaks("publication").1, Source::Dictionary); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn case_apostrophes_and_punctuation() { assert_eq!(h("Typography"), "Ty-pog-ra-phy"); @@ -494,6 +501,7 @@ mod tests { assert_eq!(h("typography-composition"), "ty-pog-ra-phy-com-po-si-tion"); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn inflections_inherit_the_stem() { assert_eq!(h("ripples"), "rip-ples"); @@ -501,6 +509,7 @@ mod tests { assert_eq!(h("restrained"), "re-strained"); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn patterns_cover_unknown_words() { // Not in the word list: the trained patterns decide. @@ -686,6 +695,7 @@ mod tests { } } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn hyphenating_a_long_word_is_linear() { // A word of n letters against one of 8n: linear ~8x the cost, the old rescans ~64x. One in @@ -748,6 +758,7 @@ mod tests { hyphenate_word_in(w, &Limits::default(), ProofLang::Pl) } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn polish_patterns_load() { let p = Patterns::pl(); @@ -764,6 +775,7 @@ mod tests { hyphenate_word_in(w, &Limits::default(), ProofLang::It) } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn italian_patterns_load() { let p = Patterns::it(); @@ -772,6 +784,7 @@ mod tests { assert_eq!(p.exception_count(), 0); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn italian_words_hyphenate_by_italian_rules() { for (w, want) in [ @@ -798,6 +811,7 @@ mod tests { assert_eq!(word_breaks_in("famiglia", ProofLang::It).1, Source::Patterns); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn portuguese_patterns_load() { let p = Patterns::pt(); @@ -809,6 +823,7 @@ mod tests { assert_eq!(p.exception("software"), Some(&[4][..])); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn portuguese_words_hyphenate_by_portuguese_rules() { for (w, want) in [ @@ -828,6 +843,7 @@ mod tests { } } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn polish_words_hyphenate_by_polish_rules() { for (w, want) in [ @@ -861,6 +877,7 @@ mod tests { assert_eq!(word_breaks_in("konstytucja", ProofLang::Pl).1, Source::Patterns); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn polish_heuristic_for_foreign_letters() { // Letters outside the Polish alphabet: the Polish syllable heuristic, digraphs kept whole. @@ -882,6 +899,7 @@ mod tests { } } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn german_patterns_load() { let p = Patterns::de(); @@ -890,6 +908,7 @@ mod tests { assert_eq!(p.exception_count(), 0); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn german_words_follow_the_german_patterns() { let de = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::De); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index bf2cde13..a917f217 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -1,9 +1,9 @@ -//! WordCraft proofing, per language. A run's language (its BCP 47 tag) picks the tools through +//! Proofing for the Craft Apps (moved from WordCraft's `wordcraft-proof`), per language. A run's language (its BCP 47 tag) picks the tools through //! [`ProofLang`]; the `*_in` functions take it, the older English-only ones remain. //! //! - **English** ([`ProofLang::En`]; also text with no language): spelling against the //! public-domain Moby word list (~160k words, the same list that drives hyphenation) plus our -//! supplement of modern words it lacks (`assets/spelling/en-extra.txt`: `desktop`, `email`, +//! supplement of modern words it lacks (craft-i18n's `language-data/en-us/en-extra.txt`: `desktop`, `email`, //! `download`…), with inflections (`-s`, `-es`, `-ed`, `-ing`, `-ly`, `'s`…), a user dictionary and ignore lists; //! suggestions by edit distance. Grammar checks that are cheap and reliable: repeated words, //! `a`/`an`, capital letter after a sentence end, spaces before punctuation, doubled spaces. @@ -47,9 +47,19 @@ //! Italian, German), Bulgarian syllable rules, Russian word-division rules, heuristic. //! //! [`check_text`] checks a paragraph whose runs are in different languages. +//! +//! **Language data** (the Moby word list, the pattern files, our word supplements) is embedded at +//! build time from craft-i18n's `language-data/` (`CRAFT_I18N_DIR`, see `build.rs`). A build +//! without it has none: English spelling marks nothing and has no suggestions, and hyphenation +//! that needs the data is off; grammar and the rule-based hyphenation (Bulgarian, Turkish, +//! Russian) are code and still work. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] pub mod affix; +/// The Language data embedded from craft-i18n by `build.rs` (each constant empty when it isn't). +mod data { + include!(concat!(env!("OUT_DIR"), "/language_data.rs")); +} pub mod bg; pub mod cs; pub mod dict; @@ -339,7 +349,7 @@ const COMMON: &str = "a an the and or but if of to in on at by for with from as /// Our supplement of common words the Moby list lacks (`desktop`, `email`, `download`…): one /// lowercase base form per line, `#` comments. Inflections come from [`known_core`]. -const EXTRA_EN: &str = include_str!("../../../assets/spelling/en-extra.txt"); +use data::EXTRA_EN; fn extra_words() -> &'static HashSet<&'static str> { static E: OnceLock> = OnceLock::new(); @@ -496,8 +506,17 @@ fn accepted_by_options(word: &str, opts: &Options) -> bool { || in_user_dictionary(&w.to_lowercase()) } +/// Is the English word list embedded? Without it English can't be checked, so every English +/// word counts as correct (no red underlines), as in a language without a dictionary. +fn english_data() -> bool { + !dict::Dictionary::en_us().is_empty() +} + /// [`is_correct`] with the File › Options › Proofing choices. pub fn is_correct_with(word: &str, opts: &Options) -> bool { + if !english_data() { + return true; + } let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\''); let w = w.trim_matches('\''); if accepted_by_options(w, opts) { @@ -578,6 +597,9 @@ fn english_preferred(lower: &str, lang: ProofLang) -> Option { /// (`colour` in American English, `color` in British), plus this dialect's spellings of the /// list's words ([`en_variants`]). The user dictionary overrules both. fn is_correct_english(word: &str, lang: ProofLang, opts: &Options) -> bool { + if !english_data() { + return true; + } let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'').trim_matches('\''); if accepted_by_options(w, opts) { return true; @@ -1016,6 +1038,11 @@ fn starts_consonant_sound(w: &str) -> bool { ["uni", "use", "usu", "eu", "one", "once", "ubiq", "uti", "ure", "uro"].iter().any(|p| l.starts_with(p)) } +/// The no-Language-data build (`build.rs` sets `craft_proof_data` when the English word list is +/// embedded). +#[cfg(all(test, not(craft_proof_data)))] +mod no_data; + #[cfg(test)] mod tests { #[test] @@ -1161,11 +1188,13 @@ mod tests { assert_eq!(grammar("elma ve armut. sonra kiraz", ProofLang::Tr, true), ["sonra"]); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn dictionary_loads() { assert!(dict::Dictionary::en_us().len() > 100_000); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn spelling() { for w in [ @@ -1196,6 +1225,7 @@ mod tests { } /// #1694: `-ically` adverbs come from their `-ic` adjective (specific → specifically). + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn ically_adverbs_are_known() { for w in ["specifically", "scientifically", "automatically", "dramatically", "Specifically"] { @@ -1208,6 +1238,7 @@ mod tests { /// #1694 review: only `-ic` adjectives take `-ally`; other known words (dictionary or /// supplement) don't. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn non_ic_words_do_not_take_ally() { for (base, wrong) in [("desktop", "desktopally"), ("house", "houseally"), ("final", "finalally"), ("green", "greenally")] { @@ -1217,6 +1248,7 @@ mod tests { } /// #520: common modern words (and their inflections) are known; misspellings still flagged. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn common_modern_words() { for w in [ @@ -1253,6 +1285,7 @@ mod tests { /// The supplement stays a sorted list of lowercase base forms. /// #40: British spellings and their inflections are not flagged. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn british_spellings_are_known() { for w in [ @@ -1299,6 +1332,7 @@ mod tests { /// Each English accepts its own spellings and flags the other dialect's, suggesting its own /// first, as Word does (#1281, #1045). + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn english_dialects_flag_each_others_spellings() { let (us, gb, ca) = (ProofLang::En, ProofLang::EnGb, ProofLang::EnCa); @@ -1391,6 +1425,7 @@ mod tests { /// The words #40 (folded into #1281) reported as flagged in British English, with their /// inflections: right in en-GB, and in en-US flagged or not as `en-variants.txt` decides. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn british_words_reported_in_issue_40_are_right_in_en_gb() { let (us, gb, ca) = (ProofLang::En, ProofLang::EnGb, ProofLang::EnCa); @@ -1470,6 +1505,7 @@ mod tests { assert!(check_spelling_in(text, gb).is_empty(), "{:?}", check_spelling_in(text, gb)); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn user_dictionary_accepts_the_other_dialects_spelling() { add_word("savour"); @@ -1479,6 +1515,7 @@ mod tests { assert!(!is_correct_in("savour", ProofLang::En)); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn supplement_is_sorted_lowercase() { let words: Vec<&str> = EXTRA_EN.lines().filter(|l| !l.is_empty() && !l.starts_with('#')).collect(); @@ -1487,6 +1524,7 @@ mod tests { assert!(words.iter().all(|w| w.chars().all(|c| c.is_lowercase() || c == '\'' || c == '-')), "not lowercase"); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn spelling_issues_in_text() { let t = "This sentense has a mispelled word, see https://example.com/xyzq."; @@ -1495,6 +1533,7 @@ mod tests { assert_eq!(bad, vec!["sentense", "mispelled"]); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn address_token_covers_every_word_split_out_of_it() { // #1335: one whitespace-delimited token, however punctuation splits it, has one @@ -1509,6 +1548,7 @@ mod tests { assert_eq!(bad.last(), Some(&"wugwug"), "{bad:?}"); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn address_after_a_wide_space_is_an_internet_token() { // A web or e-mail address after a no-break space or an ideographic space is skipped, @@ -1521,6 +1561,7 @@ mod tests { assert!(["xyzq", "blorp", "qwirpy", "zxq", "wugwug"].iter().all(|w| bad.contains(w)), "{bad:?}"); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn spelling_scales_over_one_long_token() { // #1335: thousands of punctuation-separated words inside a single @@ -1582,6 +1623,7 @@ mod tests { } } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn user_dictionary_works() { assert!(!is_correct("zxqwordcrafty")); @@ -1590,6 +1632,7 @@ mod tests { } /// #40: the user dictionary survives a save and a fresh load. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn user_dictionary_persists_through_a_file() { let dir = std::env::temp_dir().join(format!("wc-dict-{}-{}", std::process::id(), "persist")); @@ -1612,6 +1655,7 @@ mod tests { } /// #40: a hostile file is capped: invalid lines are skipped, and only the first megabyte is read. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn hostile_dictionary_files_are_capped() { let dir = std::env::temp_dir().join(format!("wc-dict-{}-{}", std::process::id(), "hostile")); @@ -1634,6 +1678,7 @@ mod tests { } /// #40: saving merges in words another process saved, and leaves out the ones just removed. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn saving_merges_with_the_file() { let dir = std::env::temp_dir().join(format!("wc-dict-{}-{}", std::process::id(), "merge")); @@ -1651,6 +1696,7 @@ mod tests { } /// Ignore All lasts for the session: the word is accepted but isn't in the dictionary. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn ignored_words_are_not_dictionary_words() { assert!(!is_correct("zxqignoredword")); @@ -1749,6 +1795,7 @@ mod tests { } /// #100: runs in a language without a dictionary are never flagged; English still is. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn non_english_words_are_not_flagged() { for w in ["ąžuolas", "élève", "über", "labas", "bonjour"] { @@ -1761,6 +1808,7 @@ mod tests { } /// File › Options › Proofing (#485): each "Ignore" choice turned off makes those words count. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn ignore_options_decide_what_is_checked() { let t = "SENTENSE helo2 www.exampel.org"; @@ -1772,6 +1820,7 @@ mod tests { assert!(is_correct_with("NASA", &all), "capitals that are a word stay correct"); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn hyphenation() { let l = hyphen::Limits::default(); @@ -1782,6 +1831,7 @@ mod tests { v.iter().filter(|i| i.kind == IssueKind::Spelling).map(|i| t[i.start..i.end].to_string()).collect() } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn each_language_uses_its_own_tools() { pl::test_dict::install(); @@ -1802,6 +1852,7 @@ mod tests { assert!(!g.iter().any(|i| i.message.contains("\"an\"")), "{g:?}"); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn italian_uses_its_own_tools() { it::test_dict::install(); @@ -1817,6 +1868,7 @@ mod tests { assert!(g.iter().any(|i| i.message.contains("ripetuta")), "{g:?}"); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn mixed_language_paragraphs_check_each_run_in_its_language() { pl::test_dict::install(); diff --git a/crates/proof/src/no_data.rs b/crates/proof/src/no_data.rs new file mode 100644 index 00000000..c2cdc53e --- /dev/null +++ b/crates/proof/src/no_data.rs @@ -0,0 +1,147 @@ +//! A build with no Language data embedded (no `CRAFT_I18N_DIR`): every public entry point still +//! works and never panics. English spelling marks nothing and suggests nothing, hyphenation that +//! needs the data is off, and grammar (code, not data) still works. + +use crate::hyphen::{self, Limits, Source}; +use crate::{Options, ProofLang, dict, en_variants, patterns}; + +const LANGS: [ProofLang; 18] = [ + ProofLang::En, + ProofLang::EnGb, + ProofLang::EnCa, + ProofLang::Pl, + ProofLang::Bg, + ProofLang::Pt, + ProofLang::It, + ProofLang::Hr, + ProofLang::Tr, + ProofLang::Ru, + ProofLang::Cs, + ProofLang::Id, + ProofLang::Es, + ProofLang::EsMx, + ProofLang::Es419, + ProofLang::EsUy, + ProofLang::De, + ProofLang::Other, +]; + +/// Words and text that exercise each entry point, hostile ones included. +const SAMPLES: [&str; 10] = [ + "", + "teh colour of hyphenation", + "Ünïcödé — ‘quotes’ don't", + "a a apple .Next sentence", + "https://example.org a@b.org", + "zażółć gęślą jaźń", + "Привет, мир — кто-то", + "¿Qué tal? más", + "\u{0}\u{FFFF}\u{10FFFF}", + "supercalifragilisticexpialidociousxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", +]; + +#[test] +fn nothing_is_embedded() { + assert!(crate::data::EN_US_DIC.is_empty()); + assert!(dict::Dictionary::en_us().is_empty()); + for p in [patterns::Patterns::en_us(), patterns::Patterns::pl(), patterns::Patterns::pt(), patterns::Patterns::it(), patterns::Patterns::de()] { + assert!(p.is_empty()); + assert_eq!(p.exception_count(), 0); + } +} + +#[test] +fn english_spelling_marks_nothing_and_suggests_nothing() { + for w in ["teh", "colour", "color", "zxqv", "desktop"] { + assert!(crate::is_correct(w), "{w}"); + assert!(crate::is_correct_with(w, &Options::default()), "{w}"); + assert!(crate::is_correct_for(w, Some("en-GB")), "{w}"); + for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + assert!(crate::is_correct_in(w, l), "{w} {l:?}"); + assert!(crate::suggest_in(w, 5, l).is_empty(), "{w} {l:?}"); + } + assert!(crate::suggest(w, 5).is_empty(), "{w}"); + assert_eq!(en_variants::preferred(w, ProofLang::En), None); + assert!(!en_variants::is_foreign(w, ProofLang::EnGb)); + } + assert!(crate::check_spelling("Teh colour of zxqv").is_empty()); + assert!(crate::check_spelling_in("Teh colour", ProofLang::EnGb).is_empty()); +} + +#[test] +fn grammar_still_works() { + assert!(!crate::check_grammar("This is is wrong.").is_empty()); +} + +#[test] +fn hyphenation_is_off_where_it_needs_data() { + let lim = Limits::default(); + for (w, l) in [ + ("hyphenation", ProofLang::En), + ("hyphenation", ProofLang::Cs), + ("przedsiębiorstwo", ProofLang::Pl), + ("computador", ProofLang::Pt), + ("calcolatore", ProofLang::It), + ("Silbentrennung", ProofLang::De), + ] { + assert_eq!(hyphen::word_breaks_in(w, l), (vec![], Source::None), "{w} {l:?}"); + assert!(hyphen::hyphen_points_in(w, &lim, l).is_empty(), "{w} {l:?}"); + assert_eq!(hyphen::hyphenate_word_in(w, &lim, l), w); + } + assert!(hyphen::hyphen_points("hyphenation", &lim).is_empty()); + assert_eq!(hyphen::hyphenate_word("hyphenation", &lim), "hyphenation"); + assert_eq!(hyphen::word_breaks("hyphenation").1, Source::None); + // Rules are code, not data: Bulgarian still hyphenates. + assert!(!hyphen::word_breaks_in("книжарница", ProofLang::Bg).0.is_empty()); +} + +/// Every public entry point, in every language, on every sample: no panic. +#[test] +fn every_public_function_runs_without_data() { + let lim = Limits { min_word: 0, after_first: 0, before_last: 0, capitalized: true }; + let opts = Options { ignore_uppercase: false, ignore_numbers: false, ignore_internet: false }; + for text in SAMPLES { + let len = text.len(); + let _ = crate::words(text); + let _ = crate::check_spelling(text); + let _ = crate::check_spelling_with(text, &opts); + let _ = crate::check_grammar(text); + let _ = crate::is_correct(text); + let _ = crate::is_correct_with(text, &opts); + let _ = crate::suggest(text, 5); + let _ = crate::custom_word(text); + let _ = crate::has_dictionary(Some(text)); + let _ = crate::missing_dictionary(Some(text)); + let _ = crate::unreadable_dictionary(Some(text)); + let _ = crate::is_correct_for(text, Some(text)); + let _ = hyphen::word_breaks(text); + let _ = hyphen::hyphen_points(text, &lim); + let _ = hyphen::hyphenate_word(text, &lim); + let spans: Vec<(std::ops::Range, ProofLang)> = LANGS.iter().map(|&l| (0..len, l)).collect(); + let _ = crate::check_text(text, &spans); + let _ = crate::check_text_with(text, &spans, &opts, true, true); + // Out-of-range and reversed spans are hostile input too. + #[allow(clippy::reversed_empty_ranges)] + let _ = crate::check_text(text, &[(len + 5..len + 9, ProofLang::En), (3..1, ProofLang::Pl)]); + for l in LANGS { + let _ = crate::check_spelling_in(text, l); + let _ = crate::check_grammar_in(text, l); + let _ = crate::is_correct_in(text, l); + let _ = crate::is_correct_in_with(text, l, &opts); + let _ = crate::suggest_in(text, 5, l); + let _ = crate::dictionary_load_error(l); + let _ = hyphen::word_breaks_in(text, l); + let _ = hyphen::hyphen_points_in(text, &lim, l); + let _ = hyphen::hyphenate_word_in(text, &lim, l); + let _ = en_variants::preferred(text, l); + let _ = en_variants::is_foreign(text, l); + let _ = en_variants::known_variant(text, l, |_| true); + } + } + let _ = crate::installed_dictionaries(); + let _ = crate::dictionary_generation(); + let _ = crate::user_dictionary(); + let _ = crate::has_dictionary(None); + let _ = crate::missing_dictionary(None); + let _ = crate::unreadable_dictionary(None); +} diff --git a/crates/proof/src/ru/extra.rs b/crates/proof/src/ru/extra.rs index 95397e61..d9c992ae 100644 --- a/crates/proof/src/ru/extra.rs +++ b/crates/proof/src/ru/extra.rs @@ -1,11 +1,11 @@ //! Our supplement of Russian words older Hunspell dictionaries lack (`маркетплейс`, `промокод`, -//! `интеграция`, `бонусный`…), from `assets/spelling/ru-extra.txt`: each base form with its +//! `интеграция`, `бонусный`…), from craft-i18n's `language-data/ru/ru-extra.txt`: each base form with its //! declension, expanded here into every form. use std::collections::HashSet; use std::sync::OnceLock; -const EXTRA_RU: &str = include_str!("../../../../assets/spelling/ru-extra.txt"); +use crate::data::EXTRA_RU; /// Every form of the supplement: common words (lower case) and names (lower case too, but they /// only match a word written with a capital). @@ -156,6 +156,7 @@ fn forms(base: &str, class: &str) -> Vec { mod tests { use super::*; + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn supplement_words_decline() { for w in [ diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index c92e2712..504c3853 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -399,6 +399,7 @@ mod tests { assert!(suggest::suggest_in(&none, "превет", 5).is_empty()); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn russian_words_are_correct() { test_dict::install(); diff --git a/crates/proof/src/vi.rs b/crates/proof/src/vi.rs index 1abf7d4a..0276fd00 100644 --- a/crates/proof/src/vi.rs +++ b/crates/proof/src/vi.rs @@ -83,6 +83,7 @@ mod checker_tests { assert!(check_text_with(text, &[], &Options::default(), true, false).is_empty()); } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn english_keeps_its_marks_and_skips_vietnamese_names() { let text = "We met Nguyễn yesterdy at the office.";