mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-15 23:12:15 +00:00
proof: read FLAG num (and long) affix dictionaries (storytold/wordcraft#4967)
The affix engine read one-character flags only, at most 128 of them, as bits of a u128. It now reads `FLAG num` (comma-separated numbers 1-65535, leading digits as in Hunspell) and `FLAG long` (two characters), with up to 65,536 distinct flags (storytold/wordcraft#4966): - One reader (`FlagMode`) parses and writes every flag field: .dic flags, PFX/SFX flags and continuation classes, FORBIDDENWORD and NOSUGGEST. - Each flag gets a dense u16 id; a stem's flags and a rule's continuation classes are sorted id sets interned in one flat table (set 0 is empty). - Hostile input: at most 1,024 flags per entry (Belarusian uses 264), a bad .aff flag is an error, a .dic entry whose flags don't read is skipped, and a FLAG line after flags were read is refused. - Lookups skip the continuation walk for groups and flags no continuation class can reach, so checks are as fast or faster with the Polish, Portuguese, Czech, Italian and Russian dictionaries (same answers). - The compact format writes a `FLAG num`/`FLAG long` line only for those modes, so one-character data is byte-for-byte unchanged and old data still loads. LibreOffice's Serbian (Cyrillic and Latin), Belarusian, Sinhala, Kannada and Nepali dictionaries now load; Danish, Galician, Icelandic and Korean still stop at COMPOUNDMIN, NEEDAFFIX, FULLSTRIP and ICONV. Ported from storytold/wordcraft#5142 (f5778bb76) onto craft-proof. craft-proof already had storytold/wordcraft#735 (AF aliases, SUBSTANDARD, KEEPCASE, ICONV, IGNORE, 256 one-character flags as a bitmask), which this commit predates; conflicts resolved by carrying those through the flag ids: AF aliases resolve to flag text read in the FLAG mode, SUBSTANDARD and KEEPCASE are flag ids like FORBIDDENWORD and NOSUGGEST, and lookups exclude up to four flags (Excluded). The FLAG long test lines in the language modules and slot.rs now use an unknown mode. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2
This commit is contained in:
1 parent
6e45fda67c
commit
e4182149a5
8 files changed
+737
-306
No files matched your search
+730
-299
File diff suppressed because it is too large.
Load diff
@@ -146,7 +146,7 @@ mod tests {
|
||||
fn missing_dictionary_and_ignored_inputs_are_quiet() {
|
||||
let empty = AffixDict::default();
|
||||
assert!(is_correct_with(&empty, "pocitac"));
|
||||
assert!(install_dictionary("FLAG long\n", "1\nstrom\n").is_err());
|
||||
assert!(install_dictionary("FLAG bogus\n", "1\nstrom\n").is_err());
|
||||
let d = AffixDict::from_hunspell(test_dict::AFF, test_dict::DIC, &[]).unwrap();
|
||||
for w in ["", "a", "2026", "ČR", "článek2", "日本語"] {
|
||||
assert!(is_correct_with(&d, w), "{w}");
|
||||
|
||||
@@ -272,7 +272,7 @@ mod tests {
|
||||
fn missing_dictionary_and_ignored_inputs_are_quiet() {
|
||||
let empty = AffixDict::default();
|
||||
assert!(is_correct_with(&empty, "prueva", ProofLang::Es));
|
||||
assert!(install_dictionary(ProofLang::Es, "FLAG long\n", "1\ncasa\n").is_err());
|
||||
assert!(install_dictionary(ProofLang::Es, "FLAG bogus\n", "1\ncasa\n").is_err());
|
||||
let d = AffixDict::from_hunspell(test_dict::AFF, test_dict::DIC, &[]).unwrap();
|
||||
for w in ["", "a", "2026", "EE", "1.º", "日本語", "-", "--"] {
|
||||
assert!(is_correct_with(&d, w, ProofLang::EsUy), "{w}");
|
||||
|
||||
@@ -153,7 +153,7 @@ mod tests {
|
||||
fn missing_dictionary_and_ignored_inputs_are_quiet() {
|
||||
let empty = AffixDict::default();
|
||||
assert!(is_correct_with(&empty, "rumha"));
|
||||
assert!(install_dictionary("FLAG long\n", "1\nrumah\n").is_err());
|
||||
assert!(install_dictionary("FLAG bogus\n", "1\nrumah\n").is_err());
|
||||
let d = AffixDict::from_hunspell(test_dict::AFF, test_dict::DIC, &[]).unwrap();
|
||||
for w in ["", "a", "2026", "RI", "ke-2", "日本語", "-", "--"] {
|
||||
assert!(is_correct_with(&d, w), "{w}");
|
||||
|
||||
@@ -145,7 +145,7 @@ mod tests {
|
||||
for w in ["caffè", "perché", "xqzv", ""] {
|
||||
assert!(is_correct_with(&none, w), "{w}");
|
||||
}
|
||||
assert!(install_dictionary("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused");
|
||||
assert!(install_dictionary("FLAG bogus\n", "1\nkot\n").is_err(), "unsupported files are refused");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -116,7 +116,7 @@ mod tests {
|
||||
assert!(is_correct_with(&none, w), "{w}");
|
||||
}
|
||||
assert!(suggest::suggest_in(&none, "żeka", 5).is_empty());
|
||||
assert!(install_dictionary("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused");
|
||||
assert!(install_dictionary("FLAG bogus\n", "1\nkot\n").is_err(), "unsupported files are refused");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -102,7 +102,7 @@ mod tests {
|
||||
for w in ["coraçao", "saude", "xqzv", ""] {
|
||||
assert!(is_correct_with(&none, w), "{w}");
|
||||
}
|
||||
assert!(install_dictionary("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused");
|
||||
assert!(install_dictionary("FLAG bogus\n", "1\nkot\n").is_err(), "unsupported files are refused");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -185,7 +185,7 @@ mod tests {
|
||||
#[test]
|
||||
fn an_installed_dictionary_has_no_file_until_reloaded() {
|
||||
assert!(!SLOT.has_dictionary(), "nothing installed under this name");
|
||||
assert!(SLOT.install("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused");
|
||||
assert!(SLOT.install("FLAG bogus\n", "1\nkot\n").is_err(), "unsupported files are refused");
|
||||
assert!(!SLOT.has_dictionary(), "a refused install keeps the current one");
|
||||
SLOT.install("", "1\nkot\n").unwrap();
|
||||
assert!(SLOT.has_dictionary() && SLOT.get().check("kot"));
|
||||
|
||||
Reference in new issue
Block a user