Files
designcraft/crates/compose/src/hyphen/mod.rs
T
Brandon ThomasandClaude Opus 5.5 83915e2fc2 M2.5: dictionary hyphenation, letter/glyph justification, keeps
- hyphen: bundled en-US dictionary from the public-domain Moby Hyphenator II
  list (Grady Ward, Gutenberg #3204), front-coded + deflated (433 KB), with
  case/apostrophe/punctuation/compound handling and inflection inheritance;
  our own Liang patterns (19.5k, 129 KB) trained by a patgen-style trainer
  (hyphen::patgen, examples/hyphgen.rs) for unknown words; heuristic last.
  Discretionary hyphens replace automatic points.
- breaker: prioritised word/letter/glyph-scaling glue in Knuth-Plass and
  layout; uncapped badness; hyphen limit as a hard constraint (hyphen-count
  states); hyphenation zone for ragged text (both composers); ragged
  hyphenation measured correctly; optical margin alignment; balance ragged
  lines; Knuth's node pruning.
- compose: Keep Options (keep with next, keep all lines, keep first/last)
  via paragraph snapshots; desired spacing applied to all alignments;
  justified last lines shrink instead of overflowing; word shaping and
  hyphenation caches (300k-char story: ~40 ms warm, ~60 ms cold).
- tests: golden corpus, perf (ignored), keeps, balance, optical, zone,
  limit, letter/glyph scaling, discretionary hyphens.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-01 12:04:43 -04:00

328 lines
11 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! English hyphenation.
//!
//! Lookup order for each word:
//! 1. the **dictionary** ([`Dictionary::en_us`]) — the public-domain Moby Hyphenator II word list
//! by Grady Ward (~160k words), stored compactly in `assets/hyphenation/en-us.dic`; simple
//! inflections (`'s`, `-s`, `-es`, `-ed`, `-d`) inherit the breaks of their stem;
//! 2. **Liang patterns** ([`Patterns::en_us`]) for words not in the dictionary — our own pattern
//! set, trained from the same list by [`patgen`] (`examples/hyphgen.rs` regenerates both files);
//! 3. a small syllable **heuristic** for words outside the pattern alphabet.
//!
//! Then the paragraph's limits ([`Limits`]) are applied.
pub mod dict;
pub mod patgen;
pub mod patterns;
use std::sync::OnceLock;
pub use dict::Dictionary;
pub use patterns::Patterns;
static EN_US_DIC: &[u8] = include_bytes!("../../../../assets/hyphenation/en-us.dic");
static EN_US_PAT: &str = include_str!("../../../../assets/hyphenation/en-us.pat");
impl Dictionary {
/// The bundled US English dictionary (decoded on first use).
pub fn en_us() -> &'static Dictionary {
static D: OnceLock<Dictionary> = OnceLock::new();
D.get_or_init(|| {
if EN_US_DIC.is_empty() {
return Dictionary::default();
}
Dictionary::from_bytes(EN_US_DIC).unwrap_or_else(|e| {
log::error!("{e}");
Dictionary::default()
})
})
}
}
impl Patterns {
/// The bundled US English patterns (parsed on first use).
pub fn en_us() -> &'static Patterns {
static P: OnceLock<Patterns> = OnceLock::new();
P.get_or_init(|| Patterns::parse(EN_US_PAT))
}
}
/// Hyphenation limits from the paragraph's Hyphenation settings.
#[derive(Clone, Copy, Debug)]
pub struct Limits {
pub min_word: usize,
pub after_first: usize,
pub before_last: usize,
pub capitalized: bool,
}
impl Default for Limits {
fn default() -> Self {
Limits { min_word: 5, after_first: 2, before_last: 2, capitalized: true }
}
}
/// Where a word's break points came from.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Source {
Dictionary,
Patterns,
Heuristic,
None,
}
fn is_apostrophe(c: char) -> bool {
matches!(c, '\'' | '’' | 'ʼ')
}
fn is_word_char(c: char) -> bool {
c.is_alphabetic() || is_apostrophe(c)
}
/// Unrestricted break points of one lowercase core (letters and `'` only, no leading/trailing `'`).
fn core_breaks(core: &[char]) -> (Vec<usize>, Source) {
let n = core.len();
if n < 2 {
return (vec![], Source::None);
}
let dict = Dictionary::en_us();
let s: String = core.iter().collect();
let from_mask = |m: u64, len: usize| (1..len.min(64)).filter(|&i| m & (1 << i) != 0).collect::<Vec<_>>();
if let Some(m) = dict.get(&s) {
return (from_mask(m, n), Source::Dictionary);
}
// Inflections inherit the stem's breaks (the suffix adds no break of its own).
for suf in ["'s", "s", "es", "ed", "d"] {
if let Some(stem) = s.strip_suffix(suf)
&& stem.chars().count() >= 2
&& let Some(m) = dict.get(stem)
{
return (from_mask(m, stem.chars().count()), Source::Dictionary);
}
}
let pats = Patterns::en_us();
if !pats.is_empty()
&& let Some(p) = pats.points(core)
{
return (p, Source::Patterns);
}
(heuristic_points(core), Source::Heuristic)
}
/// Unrestricted break points of `word` (char indices; a hyphen goes before that char), handling
/// case, apostrophes (`don't`, `’s`), surrounding punctuation and compounds (each letter run is
/// hyphenated separately). No break is ever placed next to an apostrophe or non-letter.
pub fn word_breaks(word: &str) -> (Vec<usize>, Source) {
let chars: Vec<char> = word.chars().collect();
let mut out = Vec::new();
let mut src = Source::None;
let mut i = 0;
while i < chars.len() {
if !is_word_char(chars[i]) {
i += 1;
continue;
}
let mut j = i;
while j < chars.len() && is_word_char(chars[j]) {
j += 1;
}
let (mut a, mut b) = (i, j);
while a < b && is_apostrophe(chars[a]) {
a += 1;
}
while b > a && is_apostrophe(chars[b - 1]) {
b -= 1;
}
let core: Vec<char> = chars[a..b].iter().map(|&c| if is_apostrophe(c) { '\'' } else { c.to_lowercase().next().unwrap_or(c) }).collect();
let (pts, s) = core_breaks(&core);
if src == Source::None {
src = s;
}
out.extend(pts.into_iter().filter(|&p| p > 0 && p < core.len() && core[p - 1] != '\'' && core[p] != '\'').map(|p| a + p));
i = j;
}
(out, src)
}
/// Allowed hyphenation points in `word` as char indices (a hyphen goes before that char), with
/// the paragraph limits applied to each letter run.
pub fn hyphen_points(word: &str, lim: &Limits) -> Vec<usize> {
let chars: Vec<char> = word.chars().collect();
let (pts, _) = word_breaks(word);
if pts.is_empty() {
return pts;
}
let min_before = lim.after_first.max(1);
let min_after = lim.before_last.max(1);
// Run bounds of each point's letter run (trimmed of apostrophes).
let run_of = |p: usize| {
let mut a = p;
while a > 0 && is_word_char(chars[a - 1]) {
a -= 1;
}
let mut b = p;
while b < chars.len() && is_word_char(chars[b]) {
b += 1;
}
while a < b && is_apostrophe(chars[a]) {
a += 1;
}
while b > a && is_apostrophe(chars[b - 1]) {
b -= 1;
}
(a, b)
};
pts.into_iter()
.filter(|&p| {
let (a, b) = run_of(p);
let run = &chars[a..b];
let letters = run.iter().filter(|c| c.is_alphabetic()).count();
if letters < lim.min_word.max(2) {
return false;
}
if letters > 1 && run.iter().filter(|c| c.is_alphabetic()).all(|c| c.is_uppercase()) {
return false;
}
if !lim.capitalized && run[0].is_uppercase() {
return false;
}
let before = chars[a..p].iter().filter(|c| c.is_alphabetic()).count();
let after = chars[p..b].iter().filter(|c| c.is_alphabetic()).count();
before >= min_before && after >= min_after
})
.collect()
}
/// `word` with `-` at every hyphenation point (tests and diagnostics).
pub fn hyphenate_word(word: &str, lim: &Limits) -> String {
let pts = hyphen_points(word, lim);
let mut s = String::with_capacity(word.len() + pts.len());
for (i, ch) in word.chars().enumerate() {
if pts.contains(&i) {
s.push('-');
}
s.push(ch);
}
s
}
// ---------- heuristic fallback (words outside the pattern alphabet) ----------
fn is_vowel(c: char) -> bool {
matches!(c.to_ascii_lowercase(), 'a' | 'e' | 'i' | 'o' | 'u' | 'y') || (c.is_alphabetic() && !c.is_ascii())
}
fn onset_pair(a: char, b: char) -> bool {
let (a, b) = (a.to_ascii_lowercase(), b.to_ascii_lowercase());
matches!(
(a, b),
('c' | 's' | 't' | 'p' | 'w' | 'g', 'h')
| ('b' | 'c' | 'd' | 'f' | 'g' | 'k' | 'p' | 't', 'r')
| ('b' | 'c' | 'f' | 'g' | 'k' | 'p' | 's', 'l')
| ('q', 'u')
| ('s', 't' | 'p' | 'c' | 'k')
)
}
/// Syllable heuristic: V-CV ("ty-po"), VC-CV ("hap-pen"), keeping onset clusters ("gra-phy").
fn heuristic_points(c: &[char]) -> Vec<usize> {
let n = c.len();
if n < 4 || !c.iter().all(|c| c.is_alphabetic()) {
return vec![];
}
let v: Vec<bool> = c.iter().map(|&x| is_vowel(x)).collect();
let mut out = vec![];
for i in 1..n {
let ok = if v[i - 1] && !v[i] {
(i + 1 < n && v[i + 1]) || (i + 2 < n && onset_pair(c[i], c[i + 1]) && v[i + 2])
} else if !v[i - 1] && !v[i] {
i >= 2 && v[i - 2] && i + 1 < n && v[i + 1] && !onset_pair(c[i - 1], c[i]) && c[i - 1] != c[i] || (c[i - 1] == c[i] && i >= 2 && v[i - 2])
} else {
false
};
// Both parts need a sounded vowel: a trailing silent `e`, `ed` or `es` doesn't count.
let right: String = c[i..].iter().collect();
let core = right.strip_suffix("ed").or_else(|| right.strip_suffix("es")).or_else(|| right.strip_suffix('e')).unwrap_or(&right);
if ok && core.chars().any(is_vowel) && core.len() > 1 && v[..i].iter().any(|x| *x) {
out.push(i);
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
fn h(w: &str) -> String {
hyphenate_word(w, &Limits::default())
}
#[test]
fn bundled_data_loads() {
assert!(Dictionary::en_us().len() > 150_000, "{}", Dictionary::en_us().len());
assert!(Patterns::en_us().len() > 1000, "{}", Patterns::en_us().len());
}
#[test]
fn dictionary_words() {
assert_eq!(h("happen"), "hap-pen");
assert_eq!(h("typography"), "ty-pog-ra-phy");
assert_eq!(h("composition"), "com-po-si-tion");
assert_eq!(h("hyphenation"), "hy-phen-a-tion");
assert_eq!(h("international"), "in-ter-na-tion-al");
assert_eq!(h("cat"), "cat");
assert!(!h("changed").contains('-'));
assert_eq!(h("NASA"), "NASA");
assert_eq!(word_breaks("publication").1, Source::Dictionary);
}
#[test]
fn case_apostrophes_and_punctuation() {
assert_eq!(h("Typography"), "Ty-pog-ra-phy");
assert_eq!(h("TYPOGRAPHY"), "TYPOGRAPHY");
assert_eq!(h("typography,"), "ty-pog-ra-phy,");
assert_eq!(h("“typography”"), "“ty-pog-ra-phy”");
assert_eq!(h("typography's"), "ty-pog-ra-phy's");
assert_eq!(h("typography’s"), "ty-pog-ra-phy’s");
assert_eq!(h("'typography'"), "'ty-pog-ra-phy'");
// Compounds: each part separately, never next to the hyphen.
assert_eq!(h("typography-composition"), "ty-pog-ra-phy-com-po-si-tion");
}
#[test]
fn inflections_inherit_the_stem() {
assert_eq!(h("ripples"), "rip-ples");
assert_eq!(h("gradients"), "gra-di-ents");
assert_eq!(h("restrained"), "re-strained");
}
#[test]
fn patterns_cover_unknown_words() {
// Not in the word list: the trained patterns decide.
let (pts, src) = word_breaks("hyperlocalization");
assert_eq!(src, Source::Patterns);
assert!(pts.len() >= 4, "{}", h("hyperlocalization"));
let (_, src) = word_breaks("zzyzx");
assert_ne!(src, Source::Dictionary);
}
#[test]
fn limits_are_respected() {
let lim = Limits { min_word: 5, after_first: 3, before_last: 3, capitalized: false };
for w in ["typography", "hyphenation", "justification", "Paragraph", "beautiful"] {
for p in hyphen_points(w, &lim) {
assert!(p >= 3 && p <= w.chars().count() - 3, "{w} {p}");
}
}
assert!(hyphen_points("Paragraph", &lim).is_empty());
let lim = Limits { min_word: 12, ..Limits::default() };
assert!(hyphen_points("typography", &lim).is_empty());
}
#[test]
fn heuristic_fallback() {
let c: Vec<char> = "happen".chars().collect();
assert_eq!(heuristic_points(&c), vec![3]);
}
}