mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-17 07:44:40 +00:00
Merge PR storytold/wordcraft#1761: Hyphenation: German text follows the German patterns
crates/proof/src/lang.rs: main's English-dialect helpers kept; code() doc lists `de`. crates/proof/src/lib.rs: hostile-input test loops over main's languages plus De. crates/ui-egui/src/options.rs: language_name maps EnGb/EnCa and De to no dictionary name. ROADMAP.md: main's status lines; both progress-log and revision rows kept. docs/gaps.md: main's status line; gap 8 row is main's with German hyphenation added; revision rows unioned. docs/layout-parity.md: main's status line, Tabs and Drop caps rows; the PR's Hyphenation row; revision rows unioned. docs/localization-parity.md: main's status line; revision rows unioned.
This commit is contained in:
commit
0712924629
4 files changed
+82
-37
No files matched your search
+41
-24
@@ -15,6 +15,9 @@
|
||||
//!
|
||||
//! **Bulgarian**: the Bulgarian syllable rules ([`crate::bg`]); no pattern file or word list.
|
||||
//!
|
||||
//! **German**: the TeX German patterns for the reformed spelling ([`Patterns::de`]), also for
|
||||
//! Swiss text, whose `ss` the patterns split like any double consonant (`Stras-se`).
|
||||
//!
|
||||
//! Then the paragraph's limits ([`Limits`]) are applied.
|
||||
|
||||
pub use crate::dict;
|
||||
@@ -36,6 +39,9 @@ static PT_PAT: &str = include_str!("../../../assets/hyphenation/pt.pat");
|
||||
/// hyph-utf8's `hyph-it.tex`, unmodified (offered under MIT or LPPL; WordCraft uses MIT, whose
|
||||
/// notice is in the file's header).
|
||||
static IT_PAT: &str = include_str!("../../../assets/hyphenation/it.pat");
|
||||
/// hyph-utf8's `hyph-de-1996.tex` (reformed spelling), unmodified (MIT; the notice is in the
|
||||
/// file's header).
|
||||
static DE_PAT: &str = include_str!("../../../assets/hyphenation/de.pat");
|
||||
|
||||
impl Dictionary {
|
||||
/// The bundled US English dictionary (decoded on first use).
|
||||
@@ -77,6 +83,12 @@ impl Patterns {
|
||||
static P: OnceLock<Patterns> = OnceLock::new();
|
||||
P.get_or_init(|| Patterns::parse(IT_PAT))
|
||||
}
|
||||
|
||||
/// The bundled German patterns (parsed on first use; the file has no exception words).
|
||||
pub fn de() -> &'static Patterns {
|
||||
static P: OnceLock<Patterns> = OnceLock::new();
|
||||
P.get_or_init(|| Patterns::parse(DE_PAT))
|
||||
}
|
||||
}
|
||||
|
||||
/// Hyphenation limits from the paragraph's Hyphenation settings.
|
||||
@@ -130,11 +142,11 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec<usize>, Source) {
|
||||
if lang == ProofLang::Tr && core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) {
|
||||
return (crate::tr::hyphen::points(core), Source::Rules);
|
||||
}
|
||||
if lang == ProofLang::Pt {
|
||||
return core_breaks_pt(core);
|
||||
}
|
||||
if lang == ProofLang::It {
|
||||
return core_breaks_it(core);
|
||||
match lang {
|
||||
ProofLang::Pt => return core_breaks_patterns(Patterns::pt(), core),
|
||||
ProofLang::It => return core_breaks_patterns(Patterns::it(), core),
|
||||
ProofLang::De => return core_breaks_patterns(Patterns::de(), core),
|
||||
_ => {}
|
||||
}
|
||||
if lang == ProofLang::Ru && core.iter().all(|&c| crate::ru::is_cyrillic(c)) {
|
||||
return (crate::ru::hyphen::points(core), Source::Heuristic);
|
||||
@@ -178,25 +190,9 @@ fn core_breaks_pl(core: &[char]) -> (Vec<usize>, Source) {
|
||||
(heuristic_points_pl(core), Source::Heuristic)
|
||||
}
|
||||
|
||||
/// [`core_breaks_pl`] for Portuguese: the bundled patterns, then the generic heuristic.
|
||||
fn core_breaks_pt(core: &[char]) -> (Vec<usize>, Source) {
|
||||
let pats = Patterns::pt();
|
||||
let s: String = core.iter().collect();
|
||||
if let Some(p) = pats.exception(&s) {
|
||||
return (p.to_vec(), Source::Dictionary);
|
||||
}
|
||||
if !pats.is_empty()
|
||||
&& let Some(p) = pats.points(core)
|
||||
{
|
||||
return (p, Source::Patterns);
|
||||
}
|
||||
(heuristic_points(core), Source::Heuristic)
|
||||
}
|
||||
|
||||
/// [`core_breaks_pt`] for Italian: the bundled patterns (the file has no exception words),
|
||||
/// then the generic heuristic.
|
||||
fn core_breaks_it(core: &[char]) -> (Vec<usize>, Source) {
|
||||
let pats = Patterns::it();
|
||||
/// [`core_breaks_pl`] for a language with bundled patterns (Portuguese, Italian, German): its
|
||||
/// exception words, the patterns, then the generic heuristic.
|
||||
fn core_breaks_patterns(pats: &Patterns, core: &[char]) -> (Vec<usize>, Source) {
|
||||
let s: String = core.iter().collect();
|
||||
if let Some(p) = pats.exception(&s) {
|
||||
return (p.to_vec(), Source::Dictionary);
|
||||
@@ -862,6 +858,27 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn german_patterns_load() {
|
||||
let p = Patterns::de();
|
||||
assert_eq!(p.len(), 36_709);
|
||||
// The file has no exception words, and its `\message{…}` line adds none.
|
||||
assert_eq!(p.exception_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn german_words_follow_the_german_patterns() {
|
||||
let de = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::De);
|
||||
assert_eq!(de("Silbentrennung"), "Sil-ben-tren-nung");
|
||||
assert_eq!(de("unterliegende"), "un-ter-lie-gen-de");
|
||||
assert_eq!(de("öffentlich-rechtlicher"), "öf-fent-lich-recht-li-cher");
|
||||
assert_eq!(de("Begründung"), "Be-grün-dung");
|
||||
assert_eq!(de("Zucker"), "Zu-cker");
|
||||
// Swiss spelling: `ss` splits like any double consonant.
|
||||
assert_eq!(de("Strasse"), "Stras-se");
|
||||
assert_eq!(word_breaks_in("Unterlagen", ProofLang::De).1, Source::Patterns);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bulgarian_words_follow_the_bulgarian_rules() {
|
||||
let bg = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::Bg);
|
||||
|
||||
@@ -49,6 +49,10 @@ pub enum ProofLang {
|
||||
/// `id_ID` dictionary, dictionary-driven suggestions and generic checks with Indonesian
|
||||
/// messages. Hyphenation keeps the fallback English patterns.
|
||||
Id,
|
||||
/// German (`de`, `de-DE`, `de-AT`, `de-CH`…): hyphenation by the German patterns (reformed
|
||||
/// spelling). There is no German dictionary or grammar yet, so, as for [`ProofLang::Other`],
|
||||
/// no spelling or grammar marks.
|
||||
De,
|
||||
/// A language without proofing tools here (`fr-FR`, `ja-JP`, `x-none`…). Like Word without
|
||||
/// the language's proofing tools: no spelling or grammar marks, so its words are never
|
||||
/// flagged as English misspellings. Hyphenation keeps using the English patterns, as before
|
||||
@@ -98,6 +102,8 @@ impl ProofLang {
|
||||
ProofLang::Cs
|
||||
} else if primary.eq_ignore_ascii_case("id") || primary.eq_ignore_ascii_case("ind") || primary.eq_ignore_ascii_case("in") {
|
||||
ProofLang::Id
|
||||
} else if ["de", "deu", "ger"].iter().any(|c| primary.eq_ignore_ascii_case(c)) {
|
||||
ProofLang::De
|
||||
} else {
|
||||
ProofLang::Other
|
||||
}
|
||||
@@ -127,7 +133,7 @@ impl ProofLang {
|
||||
matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa)
|
||||
}
|
||||
|
||||
/// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`); empty for [`ProofLang::Other`].
|
||||
/// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`, `de`); empty for [`ProofLang::Other`].
|
||||
pub fn code(self) -> &'static str {
|
||||
match self {
|
||||
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => "en",
|
||||
@@ -140,6 +146,7 @@ impl ProofLang {
|
||||
ProofLang::Ru => "ru",
|
||||
ProofLang::Cs => "cs",
|
||||
ProofLang::Id => "id",
|
||||
ProofLang::De => "de",
|
||||
ProofLang::Other => "",
|
||||
}
|
||||
}
|
||||
@@ -275,7 +282,11 @@ mod tests {
|
||||
for t in ["tr", "tr-TR", "tr-CY", "TR_tr", "tur"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Tr, "{t}");
|
||||
}
|
||||
for t in ["fr-FR", "de", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "e", "-"] {
|
||||
for t in ["de", "de-DE", "de-CH", "DE_at", "deu", "ger"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::De, "{t}");
|
||||
}
|
||||
assert!(!ProofLang::De.is_proofed());
|
||||
for t in ["fr-FR", "dex", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "e", "-"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Other, "{t}");
|
||||
}
|
||||
assert_eq!(ProofLang::from_tag(None), ProofLang::En);
|
||||
|
||||
@@ -34,10 +34,12 @@
|
||||
//! - **Indonesian** ([`ProofLang::Id`], module [`id`]): the installed Hunspell `id_ID`
|
||||
//! dictionary (not bundled), dictionary-driven suggestions and generic checks with Indonesian
|
||||
//! messages. Hyphenation uses the existing English fallback.
|
||||
//! - **German** ([`ProofLang::De`]): hyphenation by `hyph-utf8`'s German patterns (reformed
|
||||
//! spelling, MIT); no spelling or grammar marks yet.
|
||||
//! - **Other languages** ([`ProofLang::Other`]): no spelling or grammar marks, like Word without
|
||||
//! the language's proofing tools.
|
||||
//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese),
|
||||
//! Bulgarian syllable rules, Russian word-division rules, heuristic.
|
||||
//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese,
|
||||
//! Italian, German), Bulgarian syllable rules, Russian word-division rules, heuristic.
|
||||
//!
|
||||
//! [`check_text`] checks a paragraph whose runs are in different languages.
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
@@ -724,7 +726,7 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool {
|
||||
ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct(word),
|
||||
ProofLang::Cs => accepted_by_options(word, opts) || cs::is_correct(word),
|
||||
ProofLang::Id => accepted_by_options(word, opts) || id::is_correct(word),
|
||||
ProofLang::Bg | ProofLang::Other => true,
|
||||
ProofLang::Bg | ProofLang::De | ProofLang::Other => true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -740,7 +742,7 @@ pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
|
||||
ProofLang::Ru => ru::suggest(word, max),
|
||||
ProofLang::Cs => cs::suggest(word, max),
|
||||
ProofLang::Id => id::suggest(word, max),
|
||||
ProofLang::Bg | ProofLang::Other => Vec::new(),
|
||||
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -766,7 +768,7 @@ fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec<Issue> {
|
||||
// Czech has no sentence-capital rule, so a list item needs no exception.
|
||||
ProofLang::Cs => cs::grammar::check(text),
|
||||
ProofLang::Id => id::grammar::check(text),
|
||||
ProofLang::Bg | ProofLang::Other => Vec::new(),
|
||||
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -955,7 +957,7 @@ mod tests {
|
||||
assert_eq!(dictionary_needed(ProofLang::Pt, false), Some("pt_BR"));
|
||||
assert_eq!(dictionary_needed(ProofLang::Pl, true), None);
|
||||
assert_eq!(dictionary_needed(ProofLang::Pt, true), None);
|
||||
for l in [ProofLang::En, ProofLang::Bg, ProofLang::Other] {
|
||||
for l in [ProofLang::En, ProofLang::Bg, ProofLang::De, ProofLang::Other] {
|
||||
assert_eq!(dictionary_needed(l, false), None);
|
||||
}
|
||||
// Installed dictionaries and the built-in English list: no hint.
|
||||
@@ -1682,7 +1684,7 @@ mod tests {
|
||||
let _ = has_dictionary(Some(&s));
|
||||
let _ = is_correct_for(&s, Some(&s));
|
||||
let _ = hyphen::hyphen_points(&s, &hyphen::Limits::default());
|
||||
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Other] {
|
||||
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::De, ProofLang::Other] {
|
||||
let _ = check_spelling_in(&s, lang);
|
||||
let _ = check_grammar_in(&s, lang);
|
||||
let _ = is_correct_in(&s, lang);
|
||||
|
||||
@@ -116,13 +116,16 @@ pub struct Patterns {
|
||||
enum Block {
|
||||
Patterns,
|
||||
Exceptions,
|
||||
/// Another command's argument (`\message{…}`), ignored.
|
||||
Skip,
|
||||
}
|
||||
|
||||
impl Patterns {
|
||||
/// Parse patterns: whitespace-separated, `%` comments to the end of the line (`#` too at the
|
||||
/// start of a line). A whole TeX pattern file works as is: `\patterns{…}` holds patterns and
|
||||
/// `\hyphenation{…}` exception words written with their hyphens (`ni-gdy`); outside any
|
||||
/// block, a token with a hyphen is an exception and anything else a pattern.
|
||||
/// `\hyphenation{…}` exception words written with their hyphens (`ni-gdy`), and other
|
||||
/// commands' arguments (`\message{…}`) are skipped; outside any block, a token with a hyphen
|
||||
/// is an exception and anything else a pattern.
|
||||
pub fn parse(text: &str) -> Patterns {
|
||||
let mut p = Patterns::default();
|
||||
let mut block = Block::Patterns;
|
||||
@@ -136,12 +139,18 @@ impl Patterns {
|
||||
let mut tok = tok;
|
||||
if let Some(cmd) = tok.strip_prefix('\\') {
|
||||
let (name, rest) = cmd.split_once('{').unwrap_or((cmd, ""));
|
||||
block = if name == "hyphenation" { Block::Exceptions } else { Block::Patterns };
|
||||
block = match name {
|
||||
"hyphenation" => Block::Exceptions,
|
||||
"patterns" => Block::Patterns,
|
||||
// shortcut: the argument ends at the first `}`, so nested braces would leak; no bundled file has them.
|
||||
_ if cmd.contains('{') => Block::Skip,
|
||||
_ => Block::Patterns,
|
||||
};
|
||||
tok = rest;
|
||||
}
|
||||
let closes = tok.ends_with('}');
|
||||
let tok = tok.trim_end_matches('}');
|
||||
if !tok.is_empty() {
|
||||
if !tok.is_empty() && block != Block::Skip {
|
||||
if block == Block::Exceptions || tok.contains('-') {
|
||||
p.insert_exception(tok);
|
||||
} else {
|
||||
@@ -337,6 +346,12 @@ mod tests {
|
||||
assert!(p.to_tex().contains(&".ab4cdefghijklm5n".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_commands_arguments_are_skipped() {
|
||||
let p = Patterns::parse("\\message{Some Patterns `x-y' 2024-02-28}\n\\patterns{%\na1b\n}\n\\endinput\n");
|
||||
assert_eq!((p.len(), p.exception_count()), (1, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_tolerates_short_value_buffers() {
|
||||
let p = Patterns::parse("a1b");
|
||||
|
||||
Reference in new issue
Block a user