mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-08 19:42:12 +00:00
storytold/wordcraft#498: Indonesian (id-ID) spelling proofing (storytold/wordcraft#2576)
Indonesian spelling reads the installed Hunspell id_ID dictionary at runtime (Debian/Ubuntu hunspell-id, LibreOffice dict-id, the Dictionaries folder or WORDCRAFT_DICTIONARIES), like Czech, Polish and Portuguese: nothing is bundled, and without a dictionary Indonesian text gets no spelling marks. - ProofLang::Id for id, id-ID, ind and the legacy in tag - suggestions reuse the dictionary-driven Czech ranking (MAP/REP/TRY/KEY of the loaded dictionary), nothing Indonesian hard-coded - hyphenated reduplication accepted whole or by parts (kupu-kupu, anak-anak); hyphenated clitics after capitals/abbreviations (KTP-nya, se-Indonesia) - generic grammar only: repeated word (offers anak-anak first), space before punctuation, double space, with Indonesian messages - Review > Language, status bar, Set Proofing Language, File > Options > Proofing list Indonesian; name translated in every catalog - tests use a hand-written synthetic dictionary; engine end-to-end test Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Brandon Thomas <echelon@gmail.com>
This commit is contained in:
1 parent
3cd526ca7e
commit
4da49a6fe8
10 files changed
+483
-13
No files matched your search
@@ -3,7 +3,7 @@
|
||||
//! messages; hyphenation keeps the existing fallback, without Czech pattern data.
|
||||
|
||||
pub mod grammar;
|
||||
mod suggest;
|
||||
pub(crate) mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
//! Czech suggestions: bounded MAP variants, REP replacements, KEY neighbours, TRY edits
|
||||
//! and a stem scan. All linguistic tables are read from the installed dictionary.
|
||||
//! and a stem scan. All linguistic tables are read from the installed dictionary, so Indonesian
|
||||
//! ([`crate::id`]) uses the same ranking with its own dictionary.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
|
||||
@@ -201,6 +201,7 @@ mod tests {
|
||||
// LibreOffice's IDs are per dictionary: Polish is `dict-pl`, Brazilian Portuguese
|
||||
// `dict-pt-BR`, so both the primary-language folder and the dashed full name are tried.
|
||||
assert_eq!(super::lo_ext_dirs("cs_CZ"), ["dict-cs", "dict-cs-CZ"]);
|
||||
assert_eq!(super::lo_ext_dirs("id_ID"), ["dict-id", "dict-id-ID"]);
|
||||
assert_eq!(super::lo_ext_dirs("pl_PL"), ["dict-pl", "dict-pl-PL"]);
|
||||
assert_eq!(super::lo_ext_dirs("pt_BR"), ["dict-pt", "dict-pt-BR"]);
|
||||
assert_eq!(super::lo_ext_dirs("de-DE"), ["dict-de", "dict-de-DE"]);
|
||||
|
||||
@@ -856,7 +856,7 @@ mod tests {
|
||||
#[test]
|
||||
fn languages_without_patterns_keep_the_english_ones() {
|
||||
for w in ["hyphenation", "typography", "internationalization"] {
|
||||
for lang in [ProofLang::Other, ProofLang::Cs] {
|
||||
for lang in [ProofLang::Other, ProofLang::Cs, ProofLang::Id] {
|
||||
assert_eq!(hyphen_points_in(w, &Limits::default(), lang), hyphen_points(w, &Limits::default()), "{w}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
//! Generic repeated-word and spacing checks with Indonesian messages. No Indonesian-specific
|
||||
//! grammar rules are inferred. A word written twice with a space is flagged; since standard
|
||||
//! Indonesian spelling (PUEBI) writes reduplication with a hyphen (`anak-anak`), the hyphenated
|
||||
//! form is offered first, then the single word.
|
||||
|
||||
use crate::{Issue, IssueKind, ProofLang, words};
|
||||
|
||||
fn issue(start: usize, end: usize, message: String, suggestions: Vec<String>) -> Issue {
|
||||
Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::Id }
|
||||
}
|
||||
|
||||
pub fn check(text: &str) -> Vec<Issue> {
|
||||
let mut issues = Vec::new();
|
||||
repeated_words(text, &words(text), &mut issues);
|
||||
spaces(text, &mut issues);
|
||||
issues.sort_by_key(|i| (i.start, i.end));
|
||||
issues
|
||||
}
|
||||
|
||||
fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
|
||||
for pair in ws.windows(2) {
|
||||
let [(a0, b0), (a1, b1)] = pair else { continue };
|
||||
let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue };
|
||||
if !gap.is_empty() && gap.chars().all(char::is_whitespace) && w0.chars().all(char::is_alphabetic) && w0.to_lowercase() == w1.to_lowercase() {
|
||||
v.push(issue(*a0, *b1, format!("Kata berulang: “{w1}”"), vec![format!("{w0}-{w1}"), w0.to_string()]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
for (i, c) in text.char_indices() {
|
||||
if c != ' ' {
|
||||
continue;
|
||||
}
|
||||
let rest = text.get(i + 1..).unwrap_or("");
|
||||
let mut it = rest.chars();
|
||||
let Some(p) = it.next() else { continue };
|
||||
let after = it.next();
|
||||
// A ratio/score keeps the spaces in digit-space-colon-space-digit.
|
||||
let ratio = p == ':'
|
||||
&& text.get(..i).and_then(|before| before.chars().next_back()).is_some_and(|c| c.is_ascii_digit())
|
||||
&& after == Some(' ')
|
||||
&& it.next().is_some_and(|c| c.is_ascii_digit());
|
||||
let flag = !ratio
|
||||
&& match p {
|
||||
',' => true,
|
||||
// `;)` and `;-)` are emoticons.
|
||||
';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')),
|
||||
// ` ...` is an ellipsis, ` .5` a number.
|
||||
'.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
|
||||
// ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space.
|
||||
':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')),
|
||||
_ => false,
|
||||
};
|
||||
if flag {
|
||||
v.push(issue(i, i + 1 + p.len_utf8(), "Spasi sebelum tanda baca".into(), vec![p.to_string()]));
|
||||
}
|
||||
}
|
||||
for (i, _) in text.match_indices(" ") {
|
||||
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
|
||||
continue;
|
||||
}
|
||||
v.push(issue(i, i + 2, "Spasi ganda".into(), vec![" ".into()]));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn repeated_words_and_spaces_have_indonesian_messages() {
|
||||
let text = "Anak anak , sekolah.";
|
||||
let issues = check(text);
|
||||
assert_eq!(issues.len(), 3, "{issues:?}");
|
||||
assert_eq!(issues[0].message, "Kata berulang: “anak”");
|
||||
assert_eq!(issues[0].suggestions, ["Anak-anak", "Anak"]);
|
||||
assert!(issues.iter().any(|i| i.message == "Spasi sebelum tanda baca" && &text[i.start..i.end] == " ,"));
|
||||
assert!(issues.iter().any(|i| i.message == "Spasi ganda"));
|
||||
assert!(issues.iter().all(|i| i.lang == ProofLang::Id && i.kind == IssueKind::Grammar));
|
||||
for text in ["anak-anak bermain", "rumah. Sekolah", "Dr. Budi", "Anak, anak!", "Senyum :) Angka .5 dan titik ...", "kupu-kupu"] {
|
||||
assert!(check(text).is_empty(), "{text}: {:?}", check(text));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ratios_and_scores_keep_their_spaces() {
|
||||
for text in ["3 : 1", "Skor 2 : 1.", "Rasio 1 : 100."] {
|
||||
assert!(check(text).is_empty(), "{text}: {:?}", check(text));
|
||||
}
|
||||
for text in ["Kata : lain", "3 : kata", "3 :"] {
|
||||
assert!(check(text).iter().any(|i| i.message == "Spasi sebelum tanda baca"), "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
proptest::proptest! {
|
||||
#[test]
|
||||
fn never_panics_and_ranges_are_valid(s in "\\PC{0,80}") {
|
||||
for issue in check(&s) {
|
||||
proptest::prop_assert!(issue.start <= issue.end && issue.end <= s.len());
|
||||
proptest::prop_assert!(s.is_char_boundary(issue.start) && s.is_char_boundary(issue.end));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
//! Indonesian proofing using the installed Hunspell `id_ID` dictionary (never bundled; Debian and
|
||||
//! Ubuntu ship it as `hunspell-id`, LibreOffice as its `dict-id` extension). Suggestions use the
|
||||
//! dictionary's own MAP/REP/TRY/KEY tables. Grammar is limited to generic repeated-word and
|
||||
//! spacing checks with Indonesian messages; hyphenation keeps the English fallback (no Indonesian
|
||||
//! patterns are bundled).
|
||||
|
||||
pub mod grammar;
|
||||
mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Indonesian dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
|
||||
|
||||
/// File names of an Indonesian Hunspell dictionary (`in` is the old ISO 639 code, still used by
|
||||
/// Java locales).
|
||||
const NAMES: &[&str] = &["id_ID", "id-ID", "id", "in_ID"];
|
||||
|
||||
/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number
|
||||
/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own.
|
||||
const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"];
|
||||
|
||||
/// The Indonesian dictionary: the installed Hunspell `id_ID` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Indonesian spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner());
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
log::info!("Indonesian spelling: no Hunspell id_ID dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::load(&aff, &dic).unwrap_or_else(|e| {
|
||||
log::warn!("Indonesian spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
}
|
||||
|
||||
/// Use this Indonesian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
*DICT.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(|e| e.into_inner()) = None;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the Hunspell `id_ID` dictionary found in the dictionaries folder or the
|
||||
/// system locations now (File › Options › Proofing shows it); `None` when there is none. A
|
||||
/// dictionary given with [`install_dictionary`] has no file and isn't reported.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
}
|
||||
|
||||
/// Is an Indonesian dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Indonesian dictionary covers)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
}
|
||||
|
||||
/// Is `word` spelled correctly in Indonesian? Numbers, single letters, short words in capitals
|
||||
/// (acronyms: `DPR`, `KTP`), words with digits and words in other scripts are accepted, and so
|
||||
/// is every word when no Indonesian dictionary is installed. A hyphenated word is correct when
|
||||
/// the dictionary has it whole (`kupu-kupu`, `sayur-mayur`) or has each part (`anak-anak`,
|
||||
/// `rumah-rumah`); a hyphenated clitic (`KTP-nya`, `se-Indonesia`) counts as known.
|
||||
pub fn is_correct(word: &str) -> bool {
|
||||
is_correct_with(&dictionary(), word)
|
||||
}
|
||||
|
||||
/// [`is_correct`] against a given dictionary (an empty one accepts everything).
|
||||
pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Indonesian dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || w.chars().any(|c| c.is_ascii_digit()) || !w.chars().any(char::is_alphabetic) {
|
||||
return true;
|
||||
}
|
||||
if w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
if w.contains('-') {
|
||||
return known(d, w)
|
||||
|| w.split('-').all(|p| {
|
||||
p.chars().count() <= 1
|
||||
|| CLITICS.iter().any(|c| p.eq_ignore_ascii_case(c))
|
||||
|| (p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= 6)
|
||||
|| known(d, p)
|
||||
});
|
||||
}
|
||||
known(d, w)
|
||||
}
|
||||
|
||||
/// Is one word in the dictionary or the user dictionary?
|
||||
fn known(d: &AffixDict, w: &str) -> bool {
|
||||
d.check(w) || crate::in_user_dictionary(&w.to_lowercase())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn dictionary_lookup_accepts_indonesian_file_names() {
|
||||
let dir = std::env::temp_dir().join(format!("wordcraft-id-aliases-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
for name in ["id_ID", "id-ID", "id", "in_ID"] {
|
||||
std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap();
|
||||
std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap();
|
||||
let (aff, dic) = crate::hunspell::find_in(&dir, NAMES).unwrap();
|
||||
let d = crate::hunspell::load(&aff, &dic).unwrap();
|
||||
assert!(is_correct_with(&d, "rumah"));
|
||||
std::fs::remove_file(aff).unwrap();
|
||||
std::fs::remove_file(dic).unwrap();
|
||||
}
|
||||
std::fs::remove_dir(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn indonesian_words_affixes_case_and_hyphens() {
|
||||
test_dict::install();
|
||||
let d = dictionary();
|
||||
assert!(has_dictionary());
|
||||
assert_eq!(d.len(), 20);
|
||||
assert!(d.rule_count() > 0);
|
||||
for w in [
|
||||
"rumah",
|
||||
"makan",
|
||||
"makanan",
|
||||
"dimakan",
|
||||
"membaca",
|
||||
"bacaan",
|
||||
"dibaca",
|
||||
"Jakarta",
|
||||
"Rumah",
|
||||
"SEKOLAH",
|
||||
"anak-anak",
|
||||
"rumah-rumah",
|
||||
"kupu-kupu",
|
||||
"KTP-nya",
|
||||
"se-Indonesia",
|
||||
"Jakarta-lah",
|
||||
"DPR",
|
||||
] {
|
||||
assert!(is_correct(w), "{w}");
|
||||
}
|
||||
for w in ["rumha", "makanann", "skolah", "jakarta", "kupu", "anak-anka", "mAkan", "bukunya-xyz"] {
|
||||
assert!(!is_correct(w), "{w}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn forbidden_words_are_rejected_in_every_case() {
|
||||
test_dict::install();
|
||||
for w in ["zqterlarang", "Zqterlarang", "ZQTERLARANG"] {
|
||||
assert!(!is_correct(w), "{w}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_dictionary_and_ignored_inputs_are_quiet() {
|
||||
let empty = AffixDict::default();
|
||||
assert!(is_correct_with(&empty, "rumha"));
|
||||
assert!(install_dictionary("FLAG long\n", "1\nrumah\n").is_err());
|
||||
let d = AffixDict::from_hunspell(test_dict::AFF, test_dict::DIC, &[]).unwrap();
|
||||
for w in ["", "a", "2026", "RI", "ke-2", "日本語", "-", "--"] {
|
||||
assert!(is_correct_with(&d, w), "{w}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_dictionary_applies() {
|
||||
test_dict::install();
|
||||
assert!(!is_correct("zqkatasendiri"));
|
||||
crate::add_word("Zqkatasendiri");
|
||||
assert!(is_correct("zqkatasendiri"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn indonesian_dispatch_messages_options_and_mixed_spans() {
|
||||
test_dict::install();
|
||||
assert!(crate::has_dictionary(Some("id-ID")));
|
||||
assert!(crate::is_correct_for("rumah", Some("ind")));
|
||||
let text = "Rumah sekolha";
|
||||
let issues = crate::check_text(text, &[(0..text.len(), crate::ProofLang::Id)]);
|
||||
assert_eq!(issues.len(), 1, "{issues:?}");
|
||||
assert_eq!(&text[issues[0].start..issues[0].end], "sekolha");
|
||||
assert_eq!(issues[0].message, "Kemungkinan salah eja");
|
||||
assert_eq!(issues[0].lang, crate::ProofLang::Id);
|
||||
assert!(crate::suggest_in("sekolha", 8, crate::ProofLang::Id).contains(&"sekolah".into()));
|
||||
let text = "Rumah this is is wrong";
|
||||
let issues = crate::check_text(text, &[(0..5, crate::ProofLang::Id), (6..text.len(), crate::ProofLang::En)]);
|
||||
assert!(issues.iter().any(|i| i.lang == crate::ProofLang::En && i.message.starts_with("Repeated word")));
|
||||
assert!(issues.iter().all(|i| i.start >= 6));
|
||||
let opts = crate::Options { ignore_uppercase: false, ..Default::default() };
|
||||
assert!(!crate::is_correct_in_with("ZQTERLARANG", crate::ProofLang::Id, &opts));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
//! Indonesian suggestions. The ranking is the dictionary-driven one Czech uses
|
||||
//! ([`crate::cs::suggest::suggest_in`]: case, MAP, REP, missing spaces, KEY, TRY edits and a stem
|
||||
//! scan); every table comes from the installed `id_ID` dictionary, so nothing Indonesian is
|
||||
//! hard-coded here.
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
|
||||
/// Spelling suggestions for an Indonesian word, best first, at most `max`, in the word's case.
|
||||
pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
suggest_in(&super::dictionary(), word, max)
|
||||
}
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
crate::cs::suggest::suggest_in(d, word, max)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn indonesian_suggestions_use_runtime_tables_and_case() {
|
||||
crate::id::test_dict::install();
|
||||
for (word, want) in [
|
||||
("rumha", "rumah"),
|
||||
("sekolha", "sekolah"),
|
||||
("Sekola", "Sekolah"),
|
||||
("jakarta", "Jakarta"),
|
||||
("makanna", "makanan"),
|
||||
("djalan", "jalan"),
|
||||
("dsn", "dan"),
|
||||
("sayadan", "saya dan"),
|
||||
("membcaa", "membaca"),
|
||||
] {
|
||||
assert!(suggest(word, 8).contains(&want.into()), "{word}: {:?}", suggest(word, 8));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suggestions_stay_bounded_and_skip_forbidden_words() {
|
||||
crate::id::test_dict::install();
|
||||
for w in ["", "dua kata", &"a".repeat(41)] {
|
||||
assert!(suggest(w, 4).is_empty());
|
||||
}
|
||||
assert!(suggest("rumha", 0).is_empty());
|
||||
assert!(suggest_in(&AffixDict::default(), "rumha", 4).is_empty());
|
||||
assert!(suggest("rumah", 4).len() <= 4);
|
||||
assert!(!suggest("rumah", 4).contains(&"rumah".into()));
|
||||
assert!(!suggest("zqterlarag", 8).contains(&"zqterlarang".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nosuggest_stems_and_forms_are_never_offered() {
|
||||
crate::affix::tests::assert_nosuggest_is_honoured(suggest_in);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
//! Hand-written synthetic Indonesian test data. No words, rules or suggestion tables were copied
|
||||
//! from a system dictionary. This is a small fixture, not a redistributable Indonesian dictionary.
|
||||
|
||||
pub const AFF: &str = "SET UTF-8
|
||||
FLAG UTF-8
|
||||
FORBIDDENWORD F
|
||||
TRY aniekrumgtslhbdopy
|
||||
KEY qw|as|zx
|
||||
MAP 1
|
||||
MAP eé
|
||||
REP 2
|
||||
REP oe u
|
||||
REP dj j
|
||||
SFX A Y 1 # synthetic -an
|
||||
SFX A 0 an .
|
||||
PFX D Y 1
|
||||
PFX D 0 di .
|
||||
PFX M Y 1
|
||||
PFX M 0 mem b
|
||||
";
|
||||
|
||||
pub const DIC: &str = "20
|
||||
rumah
|
||||
makan/AD
|
||||
baca/ADM
|
||||
sekolah
|
||||
anak
|
||||
buku
|
||||
kupu-kupu
|
||||
Jakarta
|
||||
Indonesia
|
||||
saya
|
||||
kamu
|
||||
dan
|
||||
yang
|
||||
ini
|
||||
itu
|
||||
pergi
|
||||
jalan/A
|
||||
besar
|
||||
kecil
|
||||
zqterlarang/F
|
||||
";
|
||||
|
||||
pub fn install() {
|
||||
if let Err(e) = super::install_dictionary(AFF, DIC) {
|
||||
panic!("test dictionary: {e}");
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,10 @@ pub enum ProofLang {
|
||||
/// Czech (`cs`, `cs-CZ`): the installed Hunspell dictionary, dictionary-driven suggestions
|
||||
/// and generic punctuation checks. Hyphenation keeps the fallback English patterns.
|
||||
Cs,
|
||||
/// Indonesian (`id`, `id-ID`; also the old `in` and ISO 639-2 `ind`): the installed Hunspell
|
||||
/// `id_ID` dictionary, dictionary-driven suggestions and generic checks with Indonesian
|
||||
/// messages. Hyphenation keeps the fallback English patterns.
|
||||
Id,
|
||||
/// A language without proofing tools here (`fr-FR`, `ja-JP`, `x-none`…). Like Word without
|
||||
/// the language's proofing tools: no spelling or grammar marks, so its words are never
|
||||
/// flagged as English misspellings. Hyphenation keeps using the English patterns, as before
|
||||
@@ -44,8 +48,8 @@ pub enum ProofLang {
|
||||
|
||||
impl ProofLang {
|
||||
/// Every language with a spelling dictionary, in display order.
|
||||
pub const PROOFED: [ProofLang; 8] =
|
||||
[ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs];
|
||||
pub const PROOFED: [ProofLang; 9] =
|
||||
[ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id];
|
||||
|
||||
/// The proofing language of a BCP 47 tag (`pl-PL`, `en_GB`, `PL`…). Only the primary subtag
|
||||
/// counts. No tag (or an empty one) is English, WordCraft's default editing language.
|
||||
@@ -70,12 +74,14 @@ impl ProofLang {
|
||||
ProofLang::Ru
|
||||
} else if primary.eq_ignore_ascii_case("cs") || primary.eq_ignore_ascii_case("ces") || primary.eq_ignore_ascii_case("cze") {
|
||||
ProofLang::Cs
|
||||
} else if primary.eq_ignore_ascii_case("id") || primary.eq_ignore_ascii_case("ind") || primary.eq_ignore_ascii_case("in") {
|
||||
ProofLang::Id
|
||||
} else {
|
||||
ProofLang::Other
|
||||
}
|
||||
}
|
||||
|
||||
/// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`); empty for [`ProofLang::Other`].
|
||||
/// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`); empty for [`ProofLang::Other`].
|
||||
pub fn code(self) -> &'static str {
|
||||
match self {
|
||||
ProofLang::En => "en",
|
||||
@@ -87,6 +93,7 @@ impl ProofLang {
|
||||
ProofLang::Tr => "tr",
|
||||
ProofLang::Ru => "ru",
|
||||
ProofLang::Cs => "cs",
|
||||
ProofLang::Id => "id",
|
||||
ProofLang::Other => "",
|
||||
}
|
||||
}
|
||||
@@ -171,6 +178,11 @@ mod tests {
|
||||
}
|
||||
assert_eq!(ProofLang::Cs.code(), "cs");
|
||||
assert!(ProofLang::Cs.is_proofed() && ProofLang::PROOFED.contains(&ProofLang::Cs));
|
||||
for t in ["id", "id-ID", "ID-id", "id_ID", "id_ID.UTF-8", " id-ID ", "ind", "in", "in_ID"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Id, "{t}");
|
||||
}
|
||||
assert_eq!(ProofLang::Id.code(), "id");
|
||||
assert!(ProofLang::Id.is_proofed() && ProofLang::PROOFED.contains(&ProofLang::Id));
|
||||
for t in ["en", "en-US", "en-GB", "EN_au", "en-Latn-US"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::En, "{t}");
|
||||
}
|
||||
@@ -180,7 +192,7 @@ mod tests {
|
||||
for t in ["tr", "tr-TR", "tr-CY", "TR_tr", "tur"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Tr, "{t}");
|
||||
}
|
||||
for t in ["fr-FR", "de", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "e", "-"] {
|
||||
for t in ["fr-FR", "de", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "e", "-"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Other, "{t}");
|
||||
}
|
||||
assert_eq!(ProofLang::from_tag(None), ProofLang::En);
|
||||
|
||||
+25
-6
@@ -31,6 +31,9 @@
|
||||
//! - **Czech** ([`ProofLang::Cs`], module [`cs`]): the installed Hunspell `cs_CZ` dictionary
|
||||
//! (not bundled), dictionary-driven suggestions and generic checks with Czech messages.
|
||||
//! Hyphenation uses the existing English fallback; no Czech patterns are bundled.
|
||||
//! - **Indonesian** ([`ProofLang::Id`], module [`id`]): the installed Hunspell `id_ID`
|
||||
//! dictionary (not bundled), dictionary-driven suggestions and generic checks with Indonesian
|
||||
//! messages. Hyphenation uses the existing English fallback.
|
||||
//! - **Other languages** ([`ProofLang::Other`]): no spelling or grammar marks, like Word without
|
||||
//! the language's proofing tools.
|
||||
//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese),
|
||||
@@ -46,6 +49,7 @@ pub mod dict;
|
||||
pub mod hr;
|
||||
pub mod hunspell;
|
||||
pub mod hyphen;
|
||||
pub mod id;
|
||||
pub mod it;
|
||||
pub mod lang;
|
||||
pub mod patterns;
|
||||
@@ -146,6 +150,7 @@ pub fn set_dictionaries_folder(dir: Option<std::path::PathBuf>) {
|
||||
it::reload_dictionary();
|
||||
hr::reload_dictionary();
|
||||
tr::reload_dictionary();
|
||||
id::reload_dictionary();
|
||||
DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
@@ -158,6 +163,7 @@ pub fn installed_dictionaries() -> Vec<(ProofLang, Option<std::path::PathBuf>)>
|
||||
(ProofLang::It, it::installed_path()),
|
||||
(ProofLang::Hr, hr::installed_path()),
|
||||
(ProofLang::Tr, tr::installed_path()),
|
||||
(ProofLang::Id, id::installed_path()),
|
||||
]
|
||||
}
|
||||
|
||||
@@ -598,6 +604,7 @@ fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Option
|
||||
ProofLang::Tr => "Olası yazım hatası",
|
||||
ProofLang::Ru => "Возможна орфографическая ошибка",
|
||||
ProofLang::Cs => "Možná pravopisná chyba",
|
||||
ProofLang::Id => "Kemungkinan salah eja",
|
||||
_ => "Possible spelling mistake",
|
||||
};
|
||||
Some(Issue { start: a, end: b, kind: IssueKind::Spelling, message: message.into(), suggestions: Vec::new(), lang })
|
||||
@@ -619,6 +626,7 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool {
|
||||
ProofLang::Tr => accepted_by_options(word, opts) || tr::is_correct(word),
|
||||
ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct(word),
|
||||
ProofLang::Cs => accepted_by_options(word, opts) || cs::is_correct(word),
|
||||
ProofLang::Id => accepted_by_options(word, opts) || id::is_correct(word),
|
||||
ProofLang::Bg | ProofLang::Other => true,
|
||||
}
|
||||
}
|
||||
@@ -634,13 +642,14 @@ pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
|
||||
ProofLang::Tr => tr::suggest(word, max),
|
||||
ProofLang::Ru => ru::suggest(word, max),
|
||||
ProofLang::Cs => cs::suggest(word, max),
|
||||
ProofLang::Id => id::suggest(word, max),
|
||||
ProofLang::Bg | ProofLang::Other => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Grammar issues in a paragraph of text in `lang`: the English rules, the Polish ones
|
||||
/// ([`pl::grammar`]), the Portuguese ones ([`pt::grammar`]), the Italian ones ([`it::grammar`]),
|
||||
/// the Turkish ones ([`tr::grammar`]), Czech generic checks ([`cs::grammar`]), or none for a language WordCraft doesn't proof.
|
||||
/// the Turkish ones ([`tr::grammar`]), Czech and Indonesian generic checks ([`cs::grammar`], [`id::grammar`]), or none for a language WordCraft doesn't proof.
|
||||
pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec<Issue> {
|
||||
match lang {
|
||||
ProofLang::En => check_grammar(text),
|
||||
@@ -651,6 +660,7 @@ pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec<Issue> {
|
||||
ProofLang::Tr => tr::grammar::check(text),
|
||||
ProofLang::Ru => ru::grammar::check(text),
|
||||
ProofLang::Cs => cs::grammar::check(text),
|
||||
ProofLang::Id => id::grammar::check(text),
|
||||
ProofLang::Bg | ProofLang::Other => Vec::new(),
|
||||
}
|
||||
}
|
||||
@@ -856,6 +866,7 @@ mod tests {
|
||||
("pt_BR", pt::test_dict::AFF, pt::test_dict::DIC),
|
||||
("it_IT", it::test_dict::AFF, it::test_dict::DIC),
|
||||
("hr_HR", hr::test_dict::AFF, hr::test_dict::DIC),
|
||||
("id_ID", id::test_dict::AFF, id::test_dict::DIC),
|
||||
] {
|
||||
std::fs::write(dir.join(format!("{name}.aff")), aff).unwrap();
|
||||
std::fs::write(dir.join(format!("{name}.dic")), dic).unwrap();
|
||||
@@ -863,9 +874,10 @@ mod tests {
|
||||
let before = dictionary_generation();
|
||||
set_dictionaries_folder(Some(dir.clone()));
|
||||
assert!(dictionary_generation() > before, "cached checks are dropped");
|
||||
// Turkish has no test dictionary to put in the folder; it's listed after the others.
|
||||
// Turkish has no test dictionary to put in the folder; Indonesian is listed after it.
|
||||
let found = installed_dictionaries();
|
||||
assert_eq!(found.last().map(|(lang, _)| *lang), Some(ProofLang::Tr));
|
||||
assert_eq!(found.get(4).map(|(lang, _)| *lang), Some(ProofLang::Tr));
|
||||
assert_eq!(found.last(), Some(&(ProofLang::Id, Some(dir.join("id_ID.dic")))));
|
||||
assert_eq!(
|
||||
found[..4],
|
||||
[
|
||||
@@ -1298,7 +1310,7 @@ mod tests {
|
||||
let _ = has_dictionary(Some(&s));
|
||||
let _ = is_correct_for(&s, Some(&s));
|
||||
let _ = hyphen::hyphen_points(&s, &hyphen::Limits::default());
|
||||
for lang in [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Other] {
|
||||
for lang in [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Other] {
|
||||
let _ = check_spelling_in(&s, lang);
|
||||
let _ = check_grammar_in(&s, lang);
|
||||
let _ = is_correct_in(&s, lang);
|
||||
@@ -1335,8 +1347,15 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_spans_never_panic(s in "\\PC{0,60}", cuts in proptest::collection::vec((0usize..80, 0usize..80, 0u8..8), 0..6)) {
|
||||
let langs = [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Other];
|
||||
fn indonesian_suggestions_never_panic(s in "[a-zA-Zé'’ -]{0,14}") {
|
||||
id::test_dict::install();
|
||||
let _ = suggest_in(&s, 4, ProofLang::Id);
|
||||
let _ = is_correct_in(&s, ProofLang::Id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_spans_never_panic(s in "\\PC{0,60}", cuts in proptest::collection::vec((0usize..80, 0usize..80, 0u8..9), 0..6)) {
|
||||
let langs = [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Other];
|
||||
let spans: Vec<(Range<usize>, ProofLang)> = cuts.iter().map(|&(a, b, l)| (a.min(b)..a.max(b), langs[l as usize % langs.len()])).collect();
|
||||
for i in check_text(&s, &spans) {
|
||||
proptest::prop_assert!(i.start <= i.end && i.end <= s.len());
|
||||
|
||||
Reference in new issue
Block a user