CP4.2: review — pack hyphenation patterns only take words in the language's script; others fall through as before

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2
This commit is contained in:
Martin BallandClaude Opus 5.5 committed 2026-11-01 22:34:32 +00:00
1 parent 930365bb6d
commit 867f44a3d8
3 files changed
+44 -4

No files matched your search

+3 -2
View File
@@ -126,8 +126,9 @@ locale (`nl`, `pt-br`, `sr-latn`):
**Lookup order.** Spelling: the dictionaries folder, then an installed pack, then the system's
Hunspell dictionaries, then (English only) the built-in core list. A slot's Hunspell names map to
pack locales (`pt_BR` → `pt-br`, then `pt`, then any other `pt-*` pack). Hyphenation, every
language but English: an installed pack's patterns, then the embedded ones (a Language build),
then the language's rules or the English fallback as before.
language but English: an installed pack's patterns for a word in the language's script (Russian,
Bulgarian, Turkish: their script; others: letters the patterns know), then the embedded ones (a
Language build), then the language's rules or the English fallback as before.
**Hostile packs.** Every file is read only if it is a regular file (never through a symbolic
link), size-capped (`pack.toml` 1 MB, licences 4 MB, dictionaries and patterns 64 MB), and a pack
+19 -2
View File
@@ -128,8 +128,12 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec<usize>, Source) {
if n < 2 {
return (vec![], Source::None);
}
// An installed Language pack's patterns come before the embedded ones and the rules.
if let Some(p) = crate::packs::hyphenation(lang) {
// An installed Language pack's patterns come before the embedded ones and the rules, for a
// word in the language's script; any other word goes on as before (a Latin word in a Russian
// run is hyphenated as English).
if let Some(p) = crate::packs::hyphenation(lang)
&& pack_patterns_fit(lang, &p, core)
{
return if lang == ProofLang::Pl { core_breaks_pl_with(&p, core) } else { core_breaks_patterns(&p, core) };
}
if lang == ProofLang::Pl {
@@ -180,6 +184,19 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec<usize>, Source) {
(heuristic_points(core), Source::Heuristic)
}
/// Does a pack's `pats` hyphenate `core` in `lang`? Russian, Bulgarian and Turkish: a word in
/// their script (the guard their rules use). Polish, Portuguese, Italian and German: every word,
/// as their embedded patterns do. Any other language: a word whose letters the patterns know.
fn pack_patterns_fit(lang: ProofLang, pats: &Patterns, core: &[char]) -> bool {
match lang {
ProofLang::Ru => core.iter().all(|&c| crate::ru::is_cyrillic(c)),
ProofLang::Bg => core.iter().all(|&c| crate::bg::is_letter(c)),
ProofLang::Tr => core.iter().all(|&c| crate::tr::hyphen::is_letter(c)),
ProofLang::Pl | ProofLang::Pt | ProofLang::It | ProofLang::De => true,
_ => pats.values(core).is_some(),
}
}
/// Polish: exception words, then the patterns, then the Polish syllable heuristic.
fn core_breaks_pl(core: &[char]) -> (Vec<usize>, Source) {
core_breaks_pl_with(Patterns::pl(), core)
+22
View File
@@ -1563,6 +1563,28 @@ mod tests {
let _ = std::fs::remove_dir_all(&root);
}
/// Review of #485: a pack's patterns hyphenate only words in the language's script: a Latin
/// word in a Russian or Dutch run is hyphenated as it was without the pack.
#[test]
fn pack_patterns_only_hyphenate_their_script() {
let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
set_dictionaries_folder(Some(shared_test_folder()));
let root = std::env::temp_dir().join(format!("wc-proof-script-hyphen-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
let lim = hyphen::Limits::default();
let (ru, el) = (ProofLang::Ru, ProofLang::from_tag(Some("el")));
let before = [hyphen::hyphenate_word_in("hyphenation", &lim, ru), hyphen::hyphenate_word_in("hyphenation", &lim, el)];
write_pack(&root, "ru", &[("hyphenation.pat", "1на 1ка\n".as_bytes())]);
write_pack(&root, "el", &[("hyphenation.pat", "1να 1κα\n".as_bytes())]);
packs::set_packs_dir(Some(root.clone()));
assert_eq!([hyphen::hyphenate_word_in("hyphenation", &lim, ru), hyphen::hyphenate_word_in("hyphenation", &lim, el)], before);
assert_eq!(hyphen::hyphenate_word_in("канака", &lim, ru), "ка-на-ка", "a Russian word uses the pack");
assert_eq!(hyphen::hyphenate_word_in("κανακα", &lim, el), "κα-να-κα");
packs::set_packs_dir(None);
set_dictionaries_folder(Some(shared_test_folder()));
let _ = std::fs::remove_dir_all(&root);
}
/// Pack languages for the cache tests: `qa`, `qb`… each knowing `word<locale>`.
fn many_packs(root: &std::path::Path, n: usize) -> Vec<ProofLang> {
(0..n)