From 2dc51d7866a67af0dced1fe286532f16484ce466 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sun, 1 Nov 2026 22:57:01 +0000 Subject: [PATCH] =?UTF-8?q?CP6.13:=20craft-libs#495=20review=20=E2=80=94?= =?UTF-8?q?=20a=20.dic=20entry=20may=20carry=20up=20to=208,192=20flags=20(?= =?UTF-8?q?was=201,024),=20so=20LibreOffice's=20Turkish=20stems=20with=20t?= =?UTF-8?q?housands=20(abart,=20sava=C5=9F,=20inan:=20up=20to=203,926)=20a?= =?UTF-8?q?re=20words=20again?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The total flag-set bound (MAX_SET_DATA, 8.4M) stays the hostile-input guard: LibreOffice's tr_TR loads with 5.2M of it in 0.6 s, and abart, abartmıştım, savaşıyor, inandım, aranıyor, açılamadı and bilinmiyor are right, as hunspell 1.7 answers. The merged-homonym cap is now 4 × the entry cap (32,768), under the 65,536 distinct flags. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/affix.rs | 13 ++++++++----- crates/proof/src/tr/dict.rs | 11 +++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index c9b8d339..63d6f289 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -55,15 +55,17 @@ const MAX_INPUT_BYTES: usize = 4 * MAX_WORD_BYTES; const MAX_CONV_PAIRS: usize = 1 << 16; /// Longest text on either side of an `ICONV` or `OCONV` pair, in bytes. const MAX_CONV_BYTES: usize = 64; -/// Most flags one `.dic` entry or one continuation class field may carry. Real dictionaries use a -/// few hundred at most (Belarusian: 264); a longer field is hostile and is refused (`.aff`) or -/// skipped (`.dic`, though its `FORBIDDENWORD` and `NOSUGGEST` flags still apply to the word). -pub const MAX_FLAGS_PER_ENTRY: usize = 1024; +/// Most flags one `.dic` entry or one continuation class field may carry. LibreOffice's Turkish +/// dictionary has 920 entries over 1,024 flags (`abart`, `savaş`: up to 3,926), so the cap sits +/// well above that; a longer field is hostile and is refused (`.aff`) or skipped (`.dic`, though +/// its `FORBIDDENWORD` and `NOSUGGEST` flags still apply to the word). What all the flags of a +/// dictionary may cost together is bounded by `MAX_SET_DATA`. +pub const MAX_FLAGS_PER_ENTRY: usize = 8192; /// Most flags one stem may carry once its homonyms (the same word on several `.dic` lines) are /// merged, and so most flags a stem line of the compact format may carry. Larger than /// [`MAX_FLAGS_PER_ENTRY`] because merged sets aren't `.dic` fields; a homonym that would take the /// set past it adds only its `FORBIDDENWORD` and `NOSUGGEST` flags, as an exclusion is never lost. -pub const MAX_FLAGS_PER_STEM: usize = 8 * MAX_FLAGS_PER_ENTRY; +pub const MAX_FLAGS_PER_STEM: usize = 4 * MAX_FLAGS_PER_ENTRY; /// Most `AF` aliases, and most bytes of alias text in all: each alias is read once and shared by /// the stems using it, so these bound the flags an `AF` table can make a dictionary hold /// (the largest Language pack's table: 1,596 aliases, 19 KB). @@ -72,6 +74,7 @@ const MAX_ALIAS_BYTES: usize = 4 << 20; /// Most flags all distinct flag sets (stems' and continuation classes') may hold together: a lookup /// tests each distinct continuation set it meets at most once, so this bounds its cost. const MAX_SET_DATA: usize = 1 << 23; + /// Most distinct flags a dictionary may use: each gets a `u16` id. const MAX_DISTINCT_FLAGS: usize = 1 << 16; diff --git a/crates/proof/src/tr/dict.rs b/crates/proof/src/tr/dict.rs index 78209653..117d5378 100644 --- a/crates/proof/src/tr/dict.rs +++ b/crates/proof/src/tr/dict.rs @@ -62,4 +62,15 @@ mod tests { let d = TrDict::from_hunspell(aff, "1\nkitap/1,2\tpo:noun\n").unwrap(); assert!(d.contains("kitaplar") && d.contains("kitapde"), "both flags before the field"); } + + /// #495 review: LibreOffice's `tr_TR.dic` has 920 entries with more than 1,024 flags + /// (`abart`, `savaş`, `inan`: up to 3,926); such a stem and its forms are words. + #[test] + fn a_stem_with_thousands_of_flags_is_a_word() { + let aff = "SET UTF-8\nFLAG num\nSFX 1 Y 1\nSFX 1 0 de .\nSFX 3999 Y 1\nSFX 3999 0 lar .\n"; + let flags = (1..=4000).map(|i| i.to_string()).collect::>().join(","); + let d = TrDict::from_hunspell(aff, &format!("2\nabart/{flags}\nkitap\n")).unwrap(); + assert!(d.contains("abart") && d.contains("abartde") && d.contains("abartlar") && d.contains("kitap")); + assert!(!d.contains("kitaplar")); + } }