diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index c9b8d339..63d6f289 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -55,15 +55,17 @@ const MAX_INPUT_BYTES: usize = 4 * MAX_WORD_BYTES; const MAX_CONV_PAIRS: usize = 1 << 16; /// Longest text on either side of an `ICONV` or `OCONV` pair, in bytes. const MAX_CONV_BYTES: usize = 64; -/// Most flags one `.dic` entry or one continuation class field may carry. Real dictionaries use a -/// few hundred at most (Belarusian: 264); a longer field is hostile and is refused (`.aff`) or -/// skipped (`.dic`, though its `FORBIDDENWORD` and `NOSUGGEST` flags still apply to the word). -pub const MAX_FLAGS_PER_ENTRY: usize = 1024; +/// Most flags one `.dic` entry or one continuation class field may carry. LibreOffice's Turkish +/// dictionary has 920 entries over 1,024 flags (`abart`, `savaş`: up to 3,926), so the cap sits +/// well above that; a longer field is hostile and is refused (`.aff`) or skipped (`.dic`, though +/// its `FORBIDDENWORD` and `NOSUGGEST` flags still apply to the word). What all the flags of a +/// dictionary may cost together is bounded by `MAX_SET_DATA`. +pub const MAX_FLAGS_PER_ENTRY: usize = 8192; /// Most flags one stem may carry once its homonyms (the same word on several `.dic` lines) are /// merged, and so most flags a stem line of the compact format may carry. Larger than /// [`MAX_FLAGS_PER_ENTRY`] because merged sets aren't `.dic` fields; a homonym that would take the /// set past it adds only its `FORBIDDENWORD` and `NOSUGGEST` flags, as an exclusion is never lost. -pub const MAX_FLAGS_PER_STEM: usize = 8 * MAX_FLAGS_PER_ENTRY; +pub const MAX_FLAGS_PER_STEM: usize = 4 * MAX_FLAGS_PER_ENTRY; /// Most `AF` aliases, and most bytes of alias text in all: each alias is read once and shared by /// the stems using it, so these bound the flags an `AF` table can make a dictionary hold /// (the largest Language pack's table: 1,596 aliases, 19 KB). @@ -72,6 +74,7 @@ const MAX_ALIAS_BYTES: usize = 4 << 20; /// Most flags all distinct flag sets (stems' and continuation classes') may hold together: a lookup /// tests each distinct continuation set it meets at most once, so this bounds its cost. const MAX_SET_DATA: usize = 1 << 23; + /// Most distinct flags a dictionary may use: each gets a `u16` id. const MAX_DISTINCT_FLAGS: usize = 1 << 16; diff --git a/crates/proof/src/tr/dict.rs b/crates/proof/src/tr/dict.rs index 78209653..117d5378 100644 --- a/crates/proof/src/tr/dict.rs +++ b/crates/proof/src/tr/dict.rs @@ -62,4 +62,15 @@ mod tests { let d = TrDict::from_hunspell(aff, "1\nkitap/1,2\tpo:noun\n").unwrap(); assert!(d.contains("kitaplar") && d.contains("kitapde"), "both flags before the field"); } + + /// #495 review: LibreOffice's `tr_TR.dic` has 920 entries with more than 1,024 flags + /// (`abart`, `savaş`, `inan`: up to 3,926); such a stem and its forms are words. + #[test] + fn a_stem_with_thousands_of_flags_is_a_word() { + let aff = "SET UTF-8\nFLAG num\nSFX 1 Y 1\nSFX 1 0 de .\nSFX 3999 Y 1\nSFX 3999 0 lar .\n"; + let flags = (1..=4000).map(|i| i.to_string()).collect::>().join(","); + let d = TrDict::from_hunspell(aff, &format!("2\nabart/{flags}\nkitap\n")).unwrap(); + assert!(d.contains("abart") && d.contains("abartde") && d.contains("abartlar") && d.contains("kitap")); + assert!(!d.contains("kitaplar")); + } }