Merge PR storytold/wordcraft#1761: Hyphenation: German text follows the German patterns

crates/proof/src/lang.rs: main's English-dialect helpers kept; code() doc lists `de`.
crates/proof/src/lib.rs: hostile-input test loops over main's languages plus De.
crates/ui-egui/src/options.rs: language_name maps EnGb/EnCa and De to no dictionary name.
ROADMAP.md: main's status lines; both progress-log and revision rows kept.
docs/gaps.md: main's status line; gap 8 row is main's with German hyphenation added; revision rows unioned.
docs/layout-parity.md: main's status line, Tabs and Drop caps rows; the PR's Hyphenation row; revision rows unioned.
docs/localization-parity.md: main's status line; revision rows unioned.
This commit is contained in:
aiongg committed 2026-10-22 01:20:31 +00:00
commit 0712924629
4 files changed
+82 -37

No files matched your search

+41 -24
View File
@@ -15,6 +15,9 @@
//!
//! **Bulgarian**: the Bulgarian syllable rules ([`crate::bg`]); no pattern file or word list.
//!
//! **German**: the TeX German patterns for the reformed spelling ([`Patterns::de`]), also for
//! Swiss text, whose `ss` the patterns split like any double consonant (`Stras-se`).
//!
//! Then the paragraph's limits ([`Limits`]) are applied.
pub use crate::dict;
@@ -36,6 +39,9 @@ static PT_PAT: &str = include_str!("../../../assets/hyphenation/pt.pat");
/// hyph-utf8's `hyph-it.tex`, unmodified (offered under MIT or LPPL; WordCraft uses MIT, whose
/// notice is in the file's header).
static IT_PAT: &str = include_str!("../../../assets/hyphenation/it.pat");
/// hyph-utf8's `hyph-de-1996.tex` (reformed spelling), unmodified (MIT; the notice is in the
/// file's header).
static DE_PAT: &str = include_str!("../../../assets/hyphenation/de.pat");
impl Dictionary {
/// The bundled US English dictionary (decoded on first use).
@@ -77,6 +83,12 @@ impl Patterns {
static P: OnceLock<Patterns> = OnceLock::new();
P.get_or_init(|| Patterns::parse(IT_PAT))
}
/// The bundled German patterns (parsed on first use; the file has no exception words).
pub fn de() -> &'static Patterns {
static P: OnceLock<Patterns> = OnceLock::new();
P.get_or_init(|| Patterns::parse(DE_PAT))
}
}
/// Hyphenation limits from the paragraph's Hyphenation settings.
@@ -130,11 +142,11 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec<usize>, Source) {
if lang == ProofLang::Tr && core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) {
return (crate::tr::hyphen::points(core), Source::Rules);
}
if lang == ProofLang::Pt {
return core_breaks_pt(core);
}
if lang == ProofLang::It {
return core_breaks_it(core);
match lang {
ProofLang::Pt => return core_breaks_patterns(Patterns::pt(), core),
ProofLang::It => return core_breaks_patterns(Patterns::it(), core),
ProofLang::De => return core_breaks_patterns(Patterns::de(), core),
_ => {}
}
if lang == ProofLang::Ru && core.iter().all(|&c| crate::ru::is_cyrillic(c)) {
return (crate::ru::hyphen::points(core), Source::Heuristic);
@@ -178,25 +190,9 @@ fn core_breaks_pl(core: &[char]) -> (Vec<usize>, Source) {
(heuristic_points_pl(core), Source::Heuristic)
}
/// [`core_breaks_pl`] for Portuguese: the bundled patterns, then the generic heuristic.
fn core_breaks_pt(core: &[char]) -> (Vec<usize>, Source) {
let pats = Patterns::pt();
let s: String = core.iter().collect();
if let Some(p) = pats.exception(&s) {
return (p.to_vec(), Source::Dictionary);
}
if !pats.is_empty()
&& let Some(p) = pats.points(core)
{
return (p, Source::Patterns);
}
(heuristic_points(core), Source::Heuristic)
}
/// [`core_breaks_pt`] for Italian: the bundled patterns (the file has no exception words),
/// then the generic heuristic.
fn core_breaks_it(core: &[char]) -> (Vec<usize>, Source) {
let pats = Patterns::it();
/// [`core_breaks_pl`] for a language with bundled patterns (Portuguese, Italian, German): its
/// exception words, the patterns, then the generic heuristic.
fn core_breaks_patterns(pats: &Patterns, core: &[char]) -> (Vec<usize>, Source) {
let s: String = core.iter().collect();
if let Some(p) = pats.exception(&s) {
return (p.to_vec(), Source::Dictionary);
@@ -862,6 +858,27 @@ mod tests {
}
}
#[test]
fn german_patterns_load() {
let p = Patterns::de();
assert_eq!(p.len(), 36_709);
// The file has no exception words, and its `\message{…}` line adds none.
assert_eq!(p.exception_count(), 0);
}
#[test]
fn german_words_follow_the_german_patterns() {
let de = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::De);
assert_eq!(de("Silbentrennung"), "Sil-ben-tren-nung");
assert_eq!(de("unterliegende"), "un-ter-lie-gen-de");
assert_eq!(de("öffentlich-rechtlicher"), "öf-fent-lich-recht-li-cher");
assert_eq!(de("Begründung"), "Be-grün-dung");
assert_eq!(de("Zucker"), "Zu-cker");
// Swiss spelling: `ss` splits like any double consonant.
assert_eq!(de("Strasse"), "Stras-se");
assert_eq!(word_breaks_in("Unterlagen", ProofLang::De).1, Source::Patterns);
}
#[test]
fn bulgarian_words_follow_the_bulgarian_rules() {
let bg = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::Bg);
+13 -2
View File
@@ -49,6 +49,10 @@ pub enum ProofLang {
/// `id_ID` dictionary, dictionary-driven suggestions and generic checks with Indonesian
/// messages. Hyphenation keeps the fallback English patterns.
Id,
/// German (`de`, `de-DE`, `de-AT`, `de-CH`…): hyphenation by the German patterns (reformed
/// spelling). There is no German dictionary or grammar yet, so, as for [`ProofLang::Other`],
/// no spelling or grammar marks.
De,
/// A language without proofing tools here (`fr-FR`, `ja-JP`, `x-none`…). Like Word without
/// the language's proofing tools: no spelling or grammar marks, so its words are never
/// flagged as English misspellings. Hyphenation keeps using the English patterns, as before
@@ -98,6 +102,8 @@ impl ProofLang {
ProofLang::Cs
} else if primary.eq_ignore_ascii_case("id") || primary.eq_ignore_ascii_case("ind") || primary.eq_ignore_ascii_case("in") {
ProofLang::Id
} else if ["de", "deu", "ger"].iter().any(|c| primary.eq_ignore_ascii_case(c)) {
ProofLang::De
} else {
ProofLang::Other
}
@@ -127,7 +133,7 @@ impl ProofLang {
matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa)
}
/// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`); empty for [`ProofLang::Other`].
/// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`, `de`); empty for [`ProofLang::Other`].
pub fn code(self) -> &'static str {
match self {
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => "en",
@@ -140,6 +146,7 @@ impl ProofLang {
ProofLang::Ru => "ru",
ProofLang::Cs => "cs",
ProofLang::Id => "id",
ProofLang::De => "de",
ProofLang::Other => "",
}
}
@@ -275,7 +282,11 @@ mod tests {
for t in ["tr", "tr-TR", "tr-CY", "TR_tr", "tur"] {
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Tr, "{t}");
}
for t in ["fr-FR", "de", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "e", "-"] {
for t in ["de", "de-DE", "de-CH", "DE_at", "deu", "ger"] {
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::De, "{t}");
}
assert!(!ProofLang::De.is_proofed());
for t in ["fr-FR", "dex", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "e", "-"] {
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Other, "{t}");
}
assert_eq!(ProofLang::from_tag(None), ProofLang::En);
+9 -7
View File
@@ -34,10 +34,12 @@
//! - **Indonesian** ([`ProofLang::Id`], module [`id`]): the installed Hunspell `id_ID`
//! dictionary (not bundled), dictionary-driven suggestions and generic checks with Indonesian
//! messages. Hyphenation uses the existing English fallback.
//! - **German** ([`ProofLang::De`]): hyphenation by `hyph-utf8`'s German patterns (reformed
//! spelling, MIT); no spelling or grammar marks yet.
//! - **Other languages** ([`ProofLang::Other`]): no spelling or grammar marks, like Word without
//! the language's proofing tools.
//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese),
//! Bulgarian syllable rules, Russian word-division rules, heuristic.
//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese,
//! Italian, German), Bulgarian syllable rules, Russian word-division rules, heuristic.
//!
//! [`check_text`] checks a paragraph whose runs are in different languages.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
@@ -724,7 +726,7 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool {
ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct(word),
ProofLang::Cs => accepted_by_options(word, opts) || cs::is_correct(word),
ProofLang::Id => accepted_by_options(word, opts) || id::is_correct(word),
ProofLang::Bg | ProofLang::Other => true,
ProofLang::Bg | ProofLang::De | ProofLang::Other => true,
}
}
@@ -740,7 +742,7 @@ pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
ProofLang::Ru => ru::suggest(word, max),
ProofLang::Cs => cs::suggest(word, max),
ProofLang::Id => id::suggest(word, max),
ProofLang::Bg | ProofLang::Other => Vec::new(),
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
}
}
@@ -766,7 +768,7 @@ fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec<Issue> {
// Czech has no sentence-capital rule, so a list item needs no exception.
ProofLang::Cs => cs::grammar::check(text),
ProofLang::Id => id::grammar::check(text),
ProofLang::Bg | ProofLang::Other => Vec::new(),
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
}
}
@@ -955,7 +957,7 @@ mod tests {
assert_eq!(dictionary_needed(ProofLang::Pt, false), Some("pt_BR"));
assert_eq!(dictionary_needed(ProofLang::Pl, true), None);
assert_eq!(dictionary_needed(ProofLang::Pt, true), None);
for l in [ProofLang::En, ProofLang::Bg, ProofLang::Other] {
for l in [ProofLang::En, ProofLang::Bg, ProofLang::De, ProofLang::Other] {
assert_eq!(dictionary_needed(l, false), None);
}
// Installed dictionaries and the built-in English list: no hint.
@@ -1682,7 +1684,7 @@ mod tests {
let _ = has_dictionary(Some(&s));
let _ = is_correct_for(&s, Some(&s));
let _ = hyphen::hyphen_points(&s, &hyphen::Limits::default());
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Other] {
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::De, ProofLang::Other] {
let _ = check_spelling_in(&s, lang);
let _ = check_grammar_in(&s, lang);
let _ = is_correct_in(&s, lang);
+19 -4
View File
@@ -116,13 +116,16 @@ pub struct Patterns {
enum Block {
Patterns,
Exceptions,
/// Another command's argument (`\message{…}`), ignored.
Skip,
}
impl Patterns {
/// Parse patterns: whitespace-separated, `%` comments to the end of the line (`#` too at the
/// start of a line). A whole TeX pattern file works as is: `\patterns{…}` holds patterns and
/// `\hyphenation{…}` exception words written with their hyphens (`ni-gdy`); outside any
/// block, a token with a hyphen is an exception and anything else a pattern.
/// `\hyphenation{…}` exception words written with their hyphens (`ni-gdy`), and other
/// commands' arguments (`\message{…}`) are skipped; outside any block, a token with a hyphen
/// is an exception and anything else a pattern.
pub fn parse(text: &str) -> Patterns {
let mut p = Patterns::default();
let mut block = Block::Patterns;
@@ -136,12 +139,18 @@ impl Patterns {
let mut tok = tok;
if let Some(cmd) = tok.strip_prefix('\\') {
let (name, rest) = cmd.split_once('{').unwrap_or((cmd, ""));
block = if name == "hyphenation" { Block::Exceptions } else { Block::Patterns };
block = match name {
"hyphenation" => Block::Exceptions,
"patterns" => Block::Patterns,
// shortcut: the argument ends at the first `}`, so nested braces would leak; no bundled file has them.
_ if cmd.contains('{') => Block::Skip,
_ => Block::Patterns,
};
tok = rest;
}
let closes = tok.ends_with('}');
let tok = tok.trim_end_matches('}');
if !tok.is_empty() {
if !tok.is_empty() && block != Block::Skip {
if block == Block::Exceptions || tok.contains('-') {
p.insert_exception(tok);
} else {
@@ -337,6 +346,12 @@ mod tests {
assert!(p.to_tex().contains(&".ab4cdefghijklm5n".to_string()));
}
#[test]
fn other_commands_arguments_are_skipped() {
let p = Patterns::parse("\\message{Some Patterns `x-y' 2024-02-28}\n\\patterns{%\na1b\n}\n\\endinput\n");
assert_eq!((p.len(), p.exception_count()), (1, 0));
}
#[test]
fn apply_tolerates_short_value_buffers() {
let p = Patterns::parse("a1b");