mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-20 17:25:24 +00:00
CP4.2: review — pack hyphenation patterns only take words in the language's script; others fall through as before
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2
This commit is contained in:
1 parent
930365bb6d
commit
867f44a3d8
3 files changed
+44
-4
No files matched your search
@@ -126,8 +126,9 @@ locale (`nl`, `pt-br`, `sr-latn`):
|
||||
**Lookup order.** Spelling: the dictionaries folder, then an installed pack, then the system's
|
||||
Hunspell dictionaries, then (English only) the built-in core list. A slot's Hunspell names map to
|
||||
pack locales (`pt_BR` → `pt-br`, then `pt`, then any other `pt-*` pack). Hyphenation, every
|
||||
language but English: an installed pack's patterns, then the embedded ones (a Language build),
|
||||
then the language's rules or the English fallback as before.
|
||||
language but English: an installed pack's patterns for a word in the language's script (Russian,
|
||||
Bulgarian, Turkish: their script; others: letters the patterns know), then the embedded ones (a
|
||||
Language build), then the language's rules or the English fallback as before.
|
||||
|
||||
**Hostile packs.** Every file is read only if it is a regular file (never through a symbolic
|
||||
link), size-capped (`pack.toml` 1 MB, licences 4 MB, dictionaries and patterns 64 MB), and a pack
|
||||
|
||||
@@ -128,8 +128,12 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec<usize>, Source) {
|
||||
if n < 2 {
|
||||
return (vec![], Source::None);
|
||||
}
|
||||
// An installed Language pack's patterns come before the embedded ones and the rules.
|
||||
if let Some(p) = crate::packs::hyphenation(lang) {
|
||||
// An installed Language pack's patterns come before the embedded ones and the rules, for a
|
||||
// word in the language's script; any other word goes on as before (a Latin word in a Russian
|
||||
// run is hyphenated as English).
|
||||
if let Some(p) = crate::packs::hyphenation(lang)
|
||||
&& pack_patterns_fit(lang, &p, core)
|
||||
{
|
||||
return if lang == ProofLang::Pl { core_breaks_pl_with(&p, core) } else { core_breaks_patterns(&p, core) };
|
||||
}
|
||||
if lang == ProofLang::Pl {
|
||||
@@ -180,6 +184,19 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec<usize>, Source) {
|
||||
(heuristic_points(core), Source::Heuristic)
|
||||
}
|
||||
|
||||
/// Does a pack's `pats` hyphenate `core` in `lang`? Russian, Bulgarian and Turkish: a word in
|
||||
/// their script (the guard their rules use). Polish, Portuguese, Italian and German: every word,
|
||||
/// as their embedded patterns do. Any other language: a word whose letters the patterns know.
|
||||
fn pack_patterns_fit(lang: ProofLang, pats: &Patterns, core: &[char]) -> bool {
|
||||
match lang {
|
||||
ProofLang::Ru => core.iter().all(|&c| crate::ru::is_cyrillic(c)),
|
||||
ProofLang::Bg => core.iter().all(|&c| crate::bg::is_letter(c)),
|
||||
ProofLang::Tr => core.iter().all(|&c| crate::tr::hyphen::is_letter(c)),
|
||||
ProofLang::Pl | ProofLang::Pt | ProofLang::It | ProofLang::De => true,
|
||||
_ => pats.values(core).is_some(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Polish: exception words, then the patterns, then the Polish syllable heuristic.
|
||||
fn core_breaks_pl(core: &[char]) -> (Vec<usize>, Source) {
|
||||
core_breaks_pl_with(Patterns::pl(), core)
|
||||
|
||||
@@ -1563,6 +1563,28 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// Review of #485: a pack's patterns hyphenate only words in the language's script: a Latin
|
||||
/// word in a Russian or Dutch run is hyphenated as it was without the pack.
|
||||
#[test]
|
||||
fn pack_patterns_only_hyphenate_their_script() {
|
||||
let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
set_dictionaries_folder(Some(shared_test_folder()));
|
||||
let root = std::env::temp_dir().join(format!("wc-proof-script-hyphen-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
let lim = hyphen::Limits::default();
|
||||
let (ru, el) = (ProofLang::Ru, ProofLang::from_tag(Some("el")));
|
||||
let before = [hyphen::hyphenate_word_in("hyphenation", &lim, ru), hyphen::hyphenate_word_in("hyphenation", &lim, el)];
|
||||
write_pack(&root, "ru", &[("hyphenation.pat", "1на 1ка\n".as_bytes())]);
|
||||
write_pack(&root, "el", &[("hyphenation.pat", "1να 1κα\n".as_bytes())]);
|
||||
packs::set_packs_dir(Some(root.clone()));
|
||||
assert_eq!([hyphen::hyphenate_word_in("hyphenation", &lim, ru), hyphen::hyphenate_word_in("hyphenation", &lim, el)], before);
|
||||
assert_eq!(hyphen::hyphenate_word_in("канака", &lim, ru), "ка-на-ка", "a Russian word uses the pack");
|
||||
assert_eq!(hyphen::hyphenate_word_in("κανακα", &lim, el), "κα-να-κα");
|
||||
packs::set_packs_dir(None);
|
||||
set_dictionaries_folder(Some(shared_test_folder()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// Pack languages for the cache tests: `qa`, `qb`… each knowing `word<locale>`.
|
||||
fn many_packs(root: &std::path::Path, n: usize) -> Vec<ProofLang> {
|
||||
(0..n)
|
||||
|
||||
Reference in new issue
Block a user