daniel543x/craft-libs
1
0 Fork 0
mirror of https://github.com/storytold/craft-libs.git synced 2026-10-31 07:39:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #337 from aiongg/lang-1-text

Browse Source 
craft-text: hyphenation, line-break segmentation, bidi and line breaking
This commit is contained in:
aiongg authored and GitHub committed 2026-10-25 23:08:26 -04:00
2 parents 1e6f127844 f4c6718d68
commit ee4103fe78
26 files changed
+6563
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

ATTRIBUTION.md
+4
View File
@@ -10,6 +10,10 @@ author, source and license. Library code and original assets are under the repos
| `crates/cms/profiles/craft-coated-fogra39.icc` | craft-libs contributors; generated by `crates/cms/tests/fogra39.rs` from the FOGRA39 data below | this repository | CC0-1.0 |
| `crates/cms/tests/data/FOGRA39.txt` (unmodified) | Fogra | [Fogra characterisation data](https://fogra.org/en/downloads/work-tools/characterisation-data), `FOGRA39_to_FOGRA60_v2.zip` | Fogra terms: free use and redistribution, unmodified, with Fogra identified as the source; "FOGRA39" identifies the reference data only and implies no certification, approval or endorsement by Fogra (full text in [NOTICE](NOTICE)) |
| `crates/i18n/data/collation.postcard` | Unicode, Inc. (ICU 78.1 collation data, CLDR 48.2.1); generated by `crates/i18n/data/generate-collation.sh` | [ICU](https://github.com/unicode-org/icu/releases/tag/release-78.1rc) | Unicode-3.0 |
| `crates/text/assets/hyphen/en-us.dic` | Grady Ward (Moby Hyphenator II), converted by DesignCraft | [Project Gutenberg eBook #3204](https://www.gutenberg.org/ebooks/3204) | Public domain |
| `crates/text/assets/hyphen/en-us.pat` | DesignCraft contributors, trained from the Moby Hyphenator II list | `crates/text/examples/hyphgen.rs` | CC0-1.0 |
| Patterns embedded through `hypher` (optional `craft-text` language features) | hyph-utf8 authors, per language | [tex-hyphen](https://github.com/hyphenation/tex-hyphen), as packaged in [hypher](https://github.com/typst/hypher/tree/main/patterns) | Per file header: LPPL, MIT, BSD-3-Clause, Unlicense or a permit-any-use notice (or a choice including one); Slovak (`hyph-sk.tex`, embedded whenever `hypher/slovak` is on: feature `slovak-gpl`, or another dependency through feature unification) is GPL-2.0-or-later; Spanish (`hyph-es.tex` 5.0, Javier Bezos, CervanTeX) is MIT |
| Word-break dictionaries and LSTM models embedded through `icu_segmenter_data` (optional `craft-text` features `thai`, `lao`, `khmer`, `burmese`) | Unicode, Inc. and ICU4X contributors | [ICU4X](https://github.com/unicode-org/icu4x) | Unicode-3.0 |
Original assets are covered by the repository licence.
Cargo.lock
Generated
+49
View File
@@ -940,6 +940,20 @@ dependencies = [
"toml_edit",
]
[[package]]
name = "craft-text"
version = "0.1.0"
dependencies = [
"hypher",
"icu_segmenter",
"miniz_oxide 0.8.9",
"unicode-bidi",
"unicode-bidi-mirroring",
"unicode-general-category",
"unicode-linebreak",
"unicode-segmentation",
]
[[package]]
name = "craft-ui"
version = "0.1.0"
@@ -2247,6 +2261,12 @@ dependencies = [
"windows-registry",
]
[[package]]
name = "hypher"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45f2b4f6b5da34d400d4c112eadcc14512cc5d04307a8eee182586d9bebd7ce6"
[[package]]
name = "iana-time-zone"
version = "0.1.65"
@@ -2469,6 +2489,29 @@ dependencies = [
"zerovec",
]
[[package]]
name = "icu_segmenter"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82d07aafccd67af15d02512a6adf5896fbc5ed00f2e99b471d2efa14016db3db"
dependencies = [
"core_maths",
"icu_collections",
"icu_locale_fallback",
"icu_provider",
"icu_segmenter_data",
"potential_utf",
"smallvec",
"utf8_iter",
"zerovec",
]
[[package]]
name = "icu_segmenter_data"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae293c039020f9ec10710af98d29ce6aa2051486638b49c9a6409f3b4a9e98ad"
[[package]]
name = "idna"
version = "1.1.0"
@@ -5459,6 +5502,12 @@ version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
[[package]]
name = "unicode-linebreak"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"
[[package]]
name = "unicode-normalization"
version = "0.1.25"
Cargo.toml
+1
View File
@@ -36,6 +36,7 @@ craft-prepress = { path = "crates/prepress" }
craft-print = { path = "crates/print" }
craft-quit = { path = "crates/quit" }
craft-settings = { path = "crates/settings" }
craft-text = { path = "crates/text" }
craft-ui = { path = "crates/ui" }
craft-update = { path = "crates/update" }
craft-window = { path = "crates/window" }
NOTICE
+41
View File
@@ -19,6 +19,40 @@ crates/cms/tests/data/FOGRA39.txt: FOGRA39 characterisation data, distributed un
the data are distributed unmodified and Fogra is identified as the source. The designation
FOGRAxx may be used solely to identify the respective reference data. Such use does not
imply certification, approval or endorsement by Fogra."
crates/text/assets/hyphen/en-us.dic: public domain, derived from the Moby Hyphenator II list by
Grady Ward (Project Gutenberg eBook #3204).
crates/text/assets/hyphen/en-us.pat: CC0-1.0, trained from en-us.dic by DesignCraft's trainer.
Hyphenation patterns for other languages, embedded through the hypher crate only when the
matching craft-text feature is enabled (none are by default). They come from hyph-utf8,
https://github.com/hyphenation/tex-hyphen, as packaged in https://github.com/typst/hypher
(patterns/). Each file is under the license stated in its header: LPPL, MIT, BSD-3-Clause, the
Unlicense, a notice permitting any use, or a choice of licenses that includes one of these. An
application that enables a language ships that file's copyright and license notice. The
exception is Slovak (hyph-sk.tex, Copyright (C) 1992 Jana Chlebíková), under the GNU General
Public License, version 2 or later (SPDX: GPL-2.0-or-later). craft-text asks for it through the
feature "slovak-gpl", which "all-languages" does not include; an application that enables it
ships GPL-licensed data. Cargo unifies features, so another dependency that enables hypher's
"slovak" feature (hypher's default features do) embeds the same data; a build that must exclude
it checks its dependency graph. The Spanish patterns (feature "spanish") are hyph-es.tex 5.0.
Copyright (C) 1993, 1997 Javier Bezos, 2001-2019 Javier Bezos, CervanTeX. MIT License:
Permission is hereby granted, free of charge, to any person obtaining a copy of this software
and associated documentation files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions: The above copyright notice
and this permission notice shall be included in all copies or substantial portions of the
Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE
AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- Word-break dictionaries and LSTM models for Thai, Lao, Khmer and Burmese, embedded through the
icu_segmenter_data crate only when craft-text's "thai", "lao", "khmer" or "burmese" feature is
enabled (none are by default). From ICU4X, https://github.com/unicode-org/icu4x.
Copyright (c) 2020-2024 Unicode, Inc. Unicode License v3 (SPDX: Unicode-3.0); portions adapted
from ICU4C, (c) 1995-2016 International Business Machines Corporation and others. An
application that enables one of these features ships the Unicode License v3 copyright and
permission notice, https://www.unicode.org/license.txt.
The optional PDF writer bundles four CC0 ICC profiles by Clinton Ingram. See ATTRIBUTION.md for per-file sources and hashes.
@@ -53,6 +87,13 @@ The transactional panel model and drop-zone semantics in crates/ui/src/docking d
FilmCraft/EffectCraft split, tab and accordion layouts and VectorCraft floating groups listed
above. The generic transaction validation, optional renderer and workspace-location helpers
are implemented here without dependencies on application engines or native window helpers.
The hyphenation code, data and generator in crates/text (src/hyphen/, assets/hyphen/ and
examples/hyphgen.rs) are adapted from:
- DesignCraft, crates/compose/src/hyphen/, crates/compose/examples/hyphgen.rs and
assets/hyphenation/, revision e161878e4c7744fa06f9a2eede8103cb4c296355 (above).
- WordCraft, crates/proof/src/hyphen.rs, revision e4b7d155f15a7784197ba34eeea0b54490d70eb2.
https://github.com/storytold/wordcraft
Copyright (c) 2026 ArtCraft Team and the WordCraft contributors.
All sources listed above are licensed MIT OR Apache-2.0. No icon artwork is copied here.
The PDF spooler in crates/print is extracted from:
README.md
+3
View File
@@ -65,6 +65,9 @@ craft-<name> = { git = "https://github.com/storytold/craft-libs", rev = "<commit
app menu. Unsaved-document decisions and save prompts stay in each app.
- [`craft-settings`](crates/settings/README.md): `config.toml` settings storage for every app: platform folders, portable mode and overrides, atomic saves while running, backup, damaged-file recovery and schema migrations. Policy in [docs/settings-storage.md](docs/settings-storage.md).
- [`craft-text`](crates/text/README.md): text processing below the composer. Its `hyphen` module finds
hyphenation points, with a bundled US English dictionary and patterns and optional hypher patterns
for 47 more languages (one cargo feature each).
- [`craft-ui`](crates/ui/README.md): SVG catalogs, icon buttons, tabs, pane controls, transactional docking, print settings and theme helpers for egui frontends;
assets, themes, localization and command handling stay in each app. Requires Rust 1.95 or later.
- [Shared UI component audit](docs/ui-components-audit.md): source inventory, extraction priorities
crates/text/Cargo.toml
+156
View File
@@ -0,0 +1,156 @@
[package]
name = "craft-text"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
description = "Text processing below the Craft composer: hyphenation with a bundled US English dictionary and patterns, other languages through hypher"
[lints]
workspace = true
[dependencies]
hypher = { version = "0.1.8", default-features = false, features = ["alloc"], optional = true }
miniz_oxide = { version = "0.8", optional = true }
unicode-general-category = { version = "1.1", optional = true }
unicode-linebreak = { version = "0.1.5", optional = true }
unicode-segmentation = { version = "1.13", optional = true }
icu_segmenter = { version = "2.3", default-features = false, features = ["compiled_data"], optional = true }
unicode-bidi = { version = "0.3.18", optional = true }
unicode-bidi-mirroring = { version = "0.4.0", optional = true }
[dev-dependencies]
unicode-segmentation = "1.13"
[[example]]
name = "hyphgen"
required-features = ["generator"]
[features]
default = ["hyphen", "bidi", "segment", "linebreak"]
# Hyphenation points; English (bundled dictionary and patterns) needs no further feature.
hyphen = ["dep:hypher", "dep:miniz_oxide", "dep:unicode-general-category"]
# Bidirectional text: embedding levels, line reordering and mirroring (UAX #9).
bidi = ["dep:unicode-bidi", "dep:unicode-bidi-mirroring"]
# Line breaking on a box/glue/penalty model (Knuth–Plass, greedy, balanced); no dependencies.
linebreak = []
# The dictionary writer (`hyphen::dict`) and pattern trainer (`hyphen::patgen`) used by
# examples/hyphgen.rs.
generator = ["hyphen"]
# Line-break opportunities (UAX #14 with kinsoku) and grapheme-cluster emergency breaks; Thai, Lao,
# Khmer and Burmese words need their language feature.
segment = ["dep:unicode-linebreak", "dep:unicode-segmentation"]
# With `thai`, `lao`, `khmer` or `burmese`: find their words with ICU4X's LSTM models instead of its
# dictionaries. Much smaller (for web builds), slower to run, and its words differ slightly.
lstm = ["segment", "icu_segmenter?/lstm"]
# Each feature below adds the data for one more language: for `hyphen`, hypher's patterns (see
# NOTICE for their licenses; Slovak is `slovak-gpl`, outside `all-languages`); for `segment`, ICU4X's word data for Thai, Lao, Khmer and Burmese
# (one shared data set, Unicode License v3).
all-languages = [
"afrikaans",
"albanian",
"assamese",
"belarusian",
"bengali",
"bulgarian",
"burmese",
"catalan",
"croatian",
"czech",
"danish",
"dutch",
"estonian",
"finnish",
"french",
"galician",
"georgian",
"german",
"greek",
"gujarati",
"hindi",
"hungarian",
"icelandic",
"italian",
"kannada",
"khmer",
"kurmanji",
"lao",
"latin",
"lithuanian",
"malayalam",
"marathi",
"mongolian",
"norwegian",
"oriya",
"panjabi",
"polish",
"portuguese",
"russian",
"sanskrit",
"serbian",
"slovenian",
"spanish",
"swedish",
"tamil",
"telugu",
"thai",
"turkish",
"turkmen",
"ukrainian",
]
afrikaans = ["hyphen", "hypher/afrikaans"]
albanian = ["hyphen", "hypher/albanian"]
assamese = ["hyphen", "hypher/assamese"]
belarusian = ["hyphen", "hypher/belarusian"]
bengali = ["hyphen", "hypher/bengali"]
bulgarian = ["hyphen", "hypher/bulgarian"]
catalan = ["hyphen", "hypher/catalan"]
croatian = ["hyphen", "hypher/croatian"]
czech = ["hyphen", "hypher/czech"]
danish = ["hyphen", "hypher/danish"]
dutch = ["hyphen", "hypher/dutch"]
estonian = ["hyphen", "hypher/estonian"]
finnish = ["hyphen", "hypher/finnish"]
french = ["hyphen", "hypher/french"]
galician = ["hyphen", "hypher/galician"]
georgian = ["hyphen", "hypher/georgian"]
german = ["hyphen", "hypher/german"]
greek = ["hyphen", "hypher/greek"]
gujarati = ["hyphen", "hypher/gujarati"]
hindi = ["hyphen", "hypher/hindi"]
hungarian = ["hyphen", "hypher/hungarian"]
icelandic = ["hyphen", "hypher/icelandic"]
italian = ["hyphen", "hypher/italian"]
kannada = ["hyphen", "hypher/kannada"]
kurmanji = ["hyphen", "hypher/kurmanji"]
latin = ["hyphen", "hypher/latin"]
lithuanian = ["hyphen", "hypher/lithuanian"]
malayalam = ["hyphen", "hypher/malayalam"]
marathi = ["hyphen", "hypher/marathi"]
mongolian = ["hyphen", "hypher/mongolian"]
norwegian = ["hyphen", "hypher/norwegian"]
oriya = ["hyphen", "hypher/oriya"]
panjabi = ["hyphen", "hypher/panjabi"]
polish = ["hyphen", "hypher/polish"]
portuguese = ["hyphen", "hypher/portuguese"]
russian = ["hyphen", "hypher/russian"]
sanskrit = ["hyphen", "hypher/sanskrit"]
serbian = ["hyphen", "hypher/serbian"]
# Slovak patterns are under the GPL, version 2 or later (hyph-sk.tex), unlike every other language
# here, so `all-languages` leaves them out. A build that enables this ships GPL-licensed data. Any
# other dependency that enables `hypher/slovak` (hypher's defaults do) embeds them too, through
# feature unification.
slovak-gpl = ["hyphen", "hypher/slovak"]
slovenian = ["hyphen", "hypher/slovenian"]
spanish = ["hyphen", "hypher/spanish"]
swedish = ["hyphen", "hypher/swedish"]
tamil = ["hyphen", "hypher/tamil"]
telugu = ["hyphen", "hypher/telugu"]
turkish = ["hyphen", "hypher/turkish"]
turkmen = ["hyphen", "hypher/turkmen"]
ukrainian = ["hyphen", "hypher/ukrainian"]
burmese = ["segment", "dep:icu_segmenter"]
khmer = ["segment", "dep:icu_segmenter"]
lao = ["segment", "dep:icu_segmenter"]
thai = ["segment", "dep:icu_segmenter"]
crates/text/README.md
+313
View File
@@ -0,0 +1,313 @@
# Craft Text
`craft-text` holds the text processing that sits below each app's composer: functions that take
plain text and a language and return what layout needs before it shapes and places glyphs.
| Module | Feature | Provides |
| --- | --- | --- |
| `hyphen` | `hyphen` (default) | Hyphenation points |
| `segment` | `segment` (default) | Line-break opportunities with kinsoku; Thai, Lao, Khmer and Burmese words; grapheme-cluster emergency breaks |
| `bidi` | `bidi` (default) | Bidirectional text: embedding levels, line reordering, mirroring |
| `linebreak` | `linebreak` (default) | Line breaking on a box/glue/penalty model |
Language features (`spanish`, `german`, …, `all-languages`) add the data for one language to every
module that uses it, and turn those modules on. Build with `default-features = false` to leave
out all modules and data.
## Hyphenation
`craft_text::hyphen` finds hyphenation points. It returns byte offsets into the word: a hyphen
may go before the char that starts at each offset. Line breaking, discretionary hyphens and
exception lists stay in each application.
### English
English is always available with `hyphen` and needs no language feature. Each word is looked up
in this order:
1. the bundled dictionary (`Dictionary::en_us`): the Moby Hyphenator II word list, about 160,000
words. Simple inflections (`'s`, `-s`, `-es`, `-ed`, `-d`) inherit the breaks of their stem;
2. the bundled Liang patterns (`assets/hyphen/en-us.pat`), trained from the same list by the crate's
patgen-style trainer;
3. a small syllable heuristic for words outside the pattern alphabet.
Case, apostrophes (`don't`, `’s`), surrounding punctuation and compounds are handled; no break is
placed next to an apostrophe or a non-letter. `Dictionary::en_us()` also serves as an English word
list, for example for spelling checks.
### Other languages
Other languages use the patterns compiled into [hypher](https://crates.io/crates/hypher) 0.1.8.
Each has a cargo feature named as in hypher (`spanish`, `german`, `norwegian`, …); `all-languages`
enables 46 of them. Patterns for languages whose feature is off are not compiled in.
Slovak's patterns are under the GPL, version 2 or later; the patterns of every other language are
under a permissive license. craft-text asks for Slovak through the feature `slovak-gpl`, which
`all-languages` leaves out; a build that enables it ships GPL-licensed data. Cargo unifies features,
so another dependency that enables hypher's `slovak` feature (hypher's default features do) embeds
the same data. A build that must exclude it checks its dependency graph
(`cargo tree -e features -i hypher`).
```rust
use craft_text::hyphen::{Lang, Limits, hyphen_points_in};
fn points(word: &str, language: &str) -> Vec<usize> {
match Lang::for_language(language) {
Some(lang) => hyphen_points_in(word, &Limits::default(), lang),
None => Vec::new(), // unknown language or feature off: don't hyphenate
}
}
```
`Lang::for_language` accepts English language names as InDesign and IDML write them (`Spanish`,
`German: 2006 Reform`, `$ID/Norwegian: Bokmal`) and locale codes (`es`, `es_ES`, `pt-BR`,
`de_DE_2006`). It returns `None` for unknown languages and for languages not compiled in.
`Lang` is `Copy + Eq + Hash`, so it can key a cache of computed points. Its `Patterns` variant
holds an opaque `PatternLang`; build one with `Lang::for_language`.
### Limits
`Limits` holds the paragraph's settings: minimum word length, letters before the first and after
the last hyphen, and whether capitalized words are hyphenated. Words in all capitals are never
hyphenated. Limit checks are linear in word length.
### Regenerating the English data
`examples/hyphgen.rs` rebuilds `assets/hyphen/en-us.dic` and `assets/hyphen/en-us.pat` from the
Moby list. It needs the `generator` feature, which exposes the dictionary writer (`hyphen::dict`)
and the pattern trainer (`hyphen::patgen`); applications don't need it.
```sh
curl -LO https://www.gutenberg.org/files/3204/files/mhyph.txt
cargo run --release -p craft-text --features generator --example hyphgen -- mhyph.txt crates/text/assets/hyphen --validate
```
## Segmentation
`craft_text::segment` finds where a line may break. It takes plain text and returns byte offsets;
measuring, choosing breaks and hanging punctuation stay in each application's composer.
Consolidated from vectorcraft, wordcraft, deckcraft and designcraft.
```rust
use craft_text::segment::{Kinsoku, Options, line_breaks};
let options = Options { kinsoku: Kinsoku::Soft, ..Options::default() };
for b in line_breaks("日本語のテキスト。", &options) {
// A line may end before byte b.offset; it must if b.mandatory.
}
```
`line_breaks` starts from UAX #14 (the `unicode-linebreak` crate) and adjusts it:
- `Options::kinsoku` applies the Japanese and Chinese line-start and line-end rules (JLREQ,
CLREQ) at breaks next to an East Asian character. Text with no East Asian characters gets
UAX #14's breaks unchanged.
- `Hard` (default): UAX #14's breaks, minus those kinsoku forbids.
- `Soft`: as `Hard`, and 々, ー and small kana may start a line.
- `None`: UAX #14's breaks plus a break between any two East Asian characters.
- `Options::korean_words` sets Korean word by word: no break between two Hangul letters or Han
ideographs. Off, Korean breaks between syllables as UAX #14 does.
- Mandatory breaks (newlines and the end of the text) are always kept.
`Options::for_language(language)` gives the options for a paragraph's BCP 47 language tag, or for
an unknown language (`None`): `korean_words` for Korean, the defaults otherwise.
When no opportunity leaves a line short enough, `emergency_breaks(text, range)` gives the places
to break the piece `text[range]` anyway: byte offsets into `text` between extended grapheme
clusters (UAX #29, through `unicode-segmentation`). An emoji ZWJ sequence, a flag or a letter with
its marks is never split.
`kinsoku_allows(before, after, mode)` is the same rule for composers that build their own
opportunity lists.
### Each app's kinsoku as a `Kinsoku` value
| App | App setting | `Kinsoku` | Rules the app keeps |
| --- | --- | --- | --- |
| vectorcraft | Kinsoku set None, Hard, Soft | `None`, `Hard`, `Soft` | none |
| wordcraft | `kinsoku` on, off | `Hard`, `None` | `word_wrap` off: Latin words break at any character |
| designcraft | no set (the breaker's own rules); Hard Kinsoku | `Hard` | custom document tables, inseparable pairs, bunri-kinshi, rensuuji, roman word break |
| designcraft | Soft, Simplified Chinese, Traditional Chinese, Korean Kinsoku | `Soft` | as above |
| designcraft | Kinsoku set None | `None` | as above |
Push-in and push-out priority and hanging punctuation decide how a line fits, not where it may
break, and stay in the composer. deckcraft has no kinsoku of its own (see below).
### Kinsoku sets for adopters
The sets are vectorcraft's (`no_line_start`, `soft_line_start`, `no_line_end`), plus these from
the other apps:
- Can't start a line: 〞 (closing double prime quote), ゛ ゜ ﾞ ﾟ (spacing sound marks), ％ ￠
(postfixed abbreviations, JLREQ cl-13). From wordcraft.
- Can't end a line: ＄ ￡ ￥ (prefixed abbreviations, JLREQ cl-12). From wordcraft.
What changes for each app:
- vectorcraft: the additions above. Breaks come from UAX #14 rather than "after any CJK
character", so `Hard` and `Soft` also keep every break UAX #14 forbids. Kinsoku no longer
applies between two non-East-Asian characters (a space before an en dash in Latin text may
break).
- wordcraft: its kinsoku on is `Hard`, which also keeps the prolonged sound mark, small kana,
hyphens (‐ – 〜 ゠), ‼ ⁇ ⁈ ⁉ and the Latin closing marks next to East Asian text off the line
start, and « ‘ “ ( [ { off the line end. Its kinsoku off is `None`. Breaking Latin words
anywhere (`word_wrap` off) and the no-break-after-slash rule stay in wordcraft.
- designcraft: its breaker's set is `Hard` without the additions above and without ’ ” 〛 〟 ｠
〜 ～ ゠ ‐ – ‼ ⁇ ⁈ ⁉ », ゕ ゖ, ㇰ–ㇿ and the half-width forms (can't start), « ‘ “ 〚 〝 ｟ ｢
(can't end). Its Soft set also lets ゝ ゞ ヽ ヾ 〻 start a line; here they stay forbidden, as in
InDesign's Soft set. Document kinsoku tables (`Kinsoku` in `doc/cjk.rs`) stay in designcraft.
`korean_char_breaks: false` is `korean_words: true`.
- deckcraft: `Hard` is its UAX #14 breaks minus those kinsoku forbids. `None` adds breaks between
East Asian characters, so no mode returns plain UAX #14 for East Asian text.
### Thai, Lao, Khmer and Burmese
UAX #14 gives these scripts class SA and leaves breaking inside them to a dictionary. Without the
script's language feature, `line_breaks` treats a run of them as one word: it never breaks inside
the run, only at its edges where UAX #14 allows a break (after a space, for example). A long Thai
sentence with no spaces is one unbreakable piece, so the composer needs an emergency break for it.
The features `thai`, `lao`, `khmer` and `burmese` split runs of that
script into words with ICU4X's segmenter (`icu_segmenter`). Any one of them links the data for all
four; the default build has no ICU data. The data is under the Unicode License v3 (see NOTICE).
The segmenter has two models, chosen by the `lstm` feature:
| Model | Feature | Added to a wasm32 release build | Gzipped |
| --- | --- | --- | --- |
| Dictionaries (desktop default) | none | 1.86 MB | 0.89 MB |
| LSTM models (for web) | `lstm` | 0.42 MB | 0.33 MB |
Measured on a minimal `line_breaks` example, against the same example with `segment` alone. The
LSTM models are slower to run and split a few words differently, mostly compounds broken between
syllables (Lao ລາດ|ຊະການ, Burmese မြန်|မာ), which are still valid line breaks.
## Bidi
`craft_text::bidi` runs the Unicode Bidirectional Algorithm (UAX #9, through
[unicode-bidi](https://crates.io/crates/unicode-bidi)) on document text. It works on plain text and
needs no fonts; shaping, glyph mirroring and caret placement stay in each application.
The pipeline is the one UAX #9 prescribes:
1. `Paragraph::new(text, Direction::Rtl)` resolves embedding levels once for the logical
paragraph. `Direction::Auto` takes the direction from the first strong character.
`Paragraph::with_overrides` also forces byte ranges to one direction (as LRO/RLO would),
keeping offsets into the original text. A paragraph separator (`\n`, U+2029) in the text starts
a new paragraph with its own level: `paragraphs()` lists each one's byte range and level, and
`level()` is the first one's.
2. The application shapes and breaks lines in logical order.
3. Each line is reordered on its own: `Paragraph::visual_runs(line)` returns its runs of one level
(byte range and level) left to right as displayed, with rule L1 applied (trailing whitespace,
tabs and separators at the level of the paragraph they are in). Zero-width joiners and tag
characters keep the level of the character before them, so no run splits a grapheme cluster.
A run with an odd level displays its characters from end to start. Lines should not cross a
paragraph separator.
Lines made of other units (clusters, glyphs, cells) use the free functions on level slices:
`line_levels(chars, levels, para_level)` for rule L1 and `visual_order(levels)` for rule L2.
`mirror(c)` returns the mirrored character for rule L4 (`(` for `)`), or `None` when the font has
to supply a mirrored glyph.
Offsets are bytes into the paragraph's `&str`: `Paragraph::levels()` holds one level per byte,
as unicode-bidi resolves them, and line ranges and runs are byte ranges. This matches the shaper
cluster offsets and line ranges the apps already keep, so no char-index table is needed;
`char_levels()` gives one level per character for code that works on `char` slices. Levels are
plain `u8` (even left to right, odd right to left), so unicode-bidi's types don't leak into
callers.
Text with nothing right to left in a left-to-right or `Auto` paragraph skips the algorithm.
The module started from DeckCraft's `text::bidi` (levels, rule L1 and L2 over chars). The
per-byte levels, the run-level reordering test cases and the hostile-input checks come from
WordCraft's layout; overrides and mirroring follow DesignCraft's composer.
## Line breaking
`craft_text::linebreak` breaks paragraphs into lines on Knuth and Plass's box/glue/penalty model.
It knows nothing about fonts: the composer turns shaped glyphs into `Item`s, calls a breaker and
places the glyphs of each returned `Line`. Widths are in the caller's units, and `width(k)` gives
the measure of line `k`, so first-line indents and shaped frames work. The core comes from
vectorcraft's every-line composer; tiered justification, emergency breaks, the hyphenation zone,
greedy and balanced lines come from designcraft's breaker.
- **Items.** Boxes, glue and penalties. Glue has a natural width and stretch and shrink in three
tiers (word space, letter space, glyph scale) that are used in that order, as in InDesign's
justification settings. Boxes may carry letter and glyph tiers too. Penalties have a cost and a
flag (hyphens); ±10 000 forbids or forces the break. `Item::fill()` before the final
`Item::forced()` leaves the last line unjustified. Optional fields, set on top of a constructor
(`Item { hang, ..Item::glue(w, st, sh) }`): `hang` (optical margin), `edge` (an `Edge` of
stretch and shrink gained at a line end or start, such as mojikumi), `hyphenation`,
`emergency`.
- **Breakers.** `knuth_plass` runs one total-fit pass. `break_paragraph` runs passes at rising
tolerances, then falls back to `greedy`; the returned `Breaks` says in `method` which one
produced the lines. `greedy_with` resolves position-dependent widths (tabs). `balanced` evens
the lines (meant for ragged text) and sets `Breaks::balanced` when it did. `overlong` finds
words wider than the measure, where the composer adds `Item::emergency()` breaks.
- **Params.** Demerits, fitness classes, tolerance, hyphen limit, tier costs, `uniform_from`,
and `Rag` for ragged text: rag stretch, short-last-line rating and hyphenation zone.
`uniform_from` is the first line index from which `width(k)` stays the same. The default 0 is
right for a constant measure; a caller with a first-line indent, drop cap or shaped frame sets
it to the first line of the constant tail, or its breaks may not be the best ones. Negative or
NaN demerits count as 0, and a NaN measure as 0.
- **Lines.** Each `Line` gives its item range, natural width, adjustment ratio, the fraction of
each tier used and the width left over, so the composer can distribute the space.
```rust
use craft_text::linebreak::{Item, Params, Rag, break_paragraph};
let space = Item::glue(0.5, 0.0, 0.0);
let items = [Item::boxed(3.0), space, Item::boxed(3.0), space, Item::boxed(5.0), Item::fill(), Item::forced()];
let ragged = Params { rag: Some(Rag::default()), ..Params::default() };
let lines = break_paragraph(&items, &|_line| 8.0, &ragged, &[1.0, f64::INFINITY]).lines;
```
### Adopting
- **vectorcraft** `composer::compose`: a box per glyph (`hang` = `start_credit(i)` for `i > 0`),
glue per space (justified: stretch 0.5 and shrink 0.2 of its width; ragged: none), a penalty per
`Breakpoint` (width = its hyphen, flagged when it has one, cost 50 justified or 25 ragged), and
`fill` + `forced` at the end unless Justify All. `Params`: `first_fitness: None`,
`max_badness: 10_000`, `fitness_demerits` 10 000 or 1 000, `fitness` bounds
`[-0.5, 0.5, 1.0]` or `[0.15, 0.4, 0.75]`, `uniform_from` as `layout.rs` computes it (the
first index from which `widths` stay at the last width); ragged adds
`Rag { stretch: 0, width_fraction: 1/6, short_last: 1/3, short_last_unit: 1/3 }`. Its two
`compose` calls become two `knuth_plass` calls (tolerance 1, then 4 or ∞); `None` still falls
back to the single-line composer.
- **designcraft** `breaker::items` builds `linebreak::Item`s: `hang`, `edge` (mojikumi
end/start elasticity), `overflow` (`space_overflow`), `hyphenation` (`auto`), `emergency`.
Ragged text uses plain spaces and `Rag { stretch: ragged_stretch, width_fraction: 0,
hyphenation_zone: hyph_zone }` instead of glue/penalty/glue triples. `Spacing` maps to
`Params { hyphen_limit, overrun: FIT_EPS, uniform_from, .. }` with `uniform_from` the number
of drop-cap lines, else 1 with a first-line indent, else 0. Kinsoku priority 1 or 2 maps
to `stretch_weight` or `shrink_weight` 4; `TIER_COST` is `tier_cost`. `knuth_plass` becomes
`overlong` + emergency items + `break_paragraph(.., &[2.5, 12.0, f64::INFINITY])`; `greedy`
becomes `greedy_with` (tabs through `advance`; `Item::no_break()` where an emergency split is
not allowed); `balanced` becomes `balanced`. Mapping lines back to glyphs
(`breaks_from_positions`) stays in the app.
- **effectcraft** `every_line`: a box per segment between line-break opportunities, glue for its
trailing whitespace, `Rag { width_fraction: 1.0, .. }`, `uniform_from: 1` (the first line's
measure differs when it is indented) and `break_paragraph(.., &[f64::INFINITY])`.
Differences from the seeds:
- vectorcraft: a Justify All last line is accepted at any ratio, like every line that ends at a
forced break.
- designcraft: fill glue stretches infinitely instead of by 10 000 units. In ragged text a line
ending at a forced break keeps its neighbour's fitness class. Greedy counts every flagged break
toward the hyphen limit, and splits overlong words at the composer's emergency breaks instead
of scanning for kinsoku. Past 4096 active breaks a pass fails and `break_paragraph` falls back
to greedy, reported as `Method::Greedy`. Lines before `uniform_from` (first-line indent, drop
cap) are kept apart by line number, where designcraft treats every line alike.
- effectcraft: lines are rated by Knuth's cubed ratio, not squared slack.
## Licenses
The word list is public domain; the trained patterns are CC0-1.0. hypher's patterns come from
hyph-utf8 under their own permissive licenses (LPPL, MPL, MIT, BSD-3); see the repository's
[NOTICE](../../NOTICE) and [ATTRIBUTION.md](../../ATTRIBUTION.md).
## Checks
Some tests depend on language features, so run the suite with them on:
```sh
cargo test -p craft-text --all-features
```
crates/text/assets/hyphen/en-us.dic
BIN
View File
Binary file not shown.
crates/text/assets/hyphen/en-us.pat
+1633
View File
File diff suppressed because it is too large. Load diff
crates/text/examples/hyphgen.rs
+174
View File
@@ -0,0 +1,174 @@
//! Regenerates the bundled English hyphenation data from the Moby Hyphenator II word list.
//!
//! ```text
//! curl -LO https://www.gutenberg.org/files/3204/files/mhyph.txt # Project Gutenberg #3204, public domain
//! cargo run --release -p craft-text --features generator --example hyphgen -- mhyph.txt crates/text/assets/hyphen [--validate]
//! ```
//!
//! Writes `en-us.dic` (the word list, front-coded and deflated; see `craft_text::hyphen::dict`)
//! and `en-us.pat` (Liang patterns trained from the list by `craft_text::hyphen::patgen`).
//! `--validate` first trains on 90% of the words and reports accuracy on the held-out 10%.
use std::collections::HashMap;
use std::path::PathBuf;
use craft_text::hyphen::dict::{Dictionary, hyphenated};
use craft_text::hyphen::patgen::{self, DEFAULT_LEVELS};
/// Mac OS Roman, 0x80..=0xFF (the list's encoding; 0xA5 `•` marks breaks).
const MAC_ROMAN_HIGH: &str =
"ÄÅÇÉÑÖÜáàâäãåçéèêëíìîïñóòôöõúùûü†°¢£§•¶ß®©™´¨≠ÆØ∞±≤≥¥µ∂∑∏π∫ªºΩæø¿¡¬√ƒ≈∆«»…\u{A0}ÀÃÕŒœ–—“”‘’÷◊ÿŸ⁄€‹›fifl‡·‚„‰ÂÊÁËÈÍÎÏÌÓÔ\u{F8FF}ÒÚÛÙıˆ˜¯˘˙˚¸˝˛ˇ";
fn mac_roman(bytes: &[u8]) -> String {
let high: Vec<char> = MAC_ROMAN_HIGH.chars().collect();
assert_eq!(high.len(), 128);
bytes.iter().map(|&b| if b < 0x80 { b as char } else { high[b as usize - 0x80] }).collect()
}
const BREAK: char = '•';
fn is_word_char(c: char) -> bool {
c.is_alphabetic() || c == '\'' || c == BREAK
}
/// `(lowercase word, mask)` of one token like `Aa•ron's`, or None if it isn't a plain word.
fn token(t: &str) -> Option<(String, u64)> {
let t = t.trim_matches('\'');
if t.is_empty() || !t.chars().all(is_word_char) || t.starts_with(BREAK) || t.ends_with(BREAK) {
return None;
}
let mut w = String::new();
let mut m = 0u64;
let mut n = 0usize;
for c in t.chars() {
if c == BREAK {
if n < 64 {
m |= 1 << n;
}
} else {
w.extend(c.to_lowercase());
n += 1;
}
}
((1..64).contains(&n) && w.chars().count() == n).then_some((w, m))
}
fn main() {
let args: Vec<String> = std::env::args().skip(1).collect();
let validate = args.iter().any(|a| a == "--validate");
let pos: Vec<&String> = args.iter().filter(|a| !a.starts_with("--")).collect();
let Some(src) = pos.first() else {
eprintln!("usage: hyphgen <mhyph.txt> [out-dir] [--validate]");
std::process::exit(2);
};
let out_dir = PathBuf::from(pos.get(1).map_or("crates/text/assets/hyphen", |s| s.as_str()));
let text = mac_roman(&std::fs::read(src).expect("read word list"));
// key → (lowercase-origin masks, other masks); compound parts only fill gaps.
let mut primary: HashMap<String, (Vec<u64>, Vec<u64>)> = HashMap::new();
let mut parts: HashMap<String, u64> = HashMap::new();
let mut entries = 0usize;
for line in text.lines() {
let line = line.trim_end_matches('\r').trim();
if line.is_empty() {
continue;
}
entries += 1;
if let Some((w, m)) = token(line) {
let lower = !line.chars().any(char::is_uppercase);
let e = primary.entry(w).or_default();
if lower { e.0.push(m) } else { e.1.push(m) }
} else {
for t in line.split([' ', '-', '/']) {
if let Some((w, m)) = token(t) {
parts.entry(w).or_insert(m);
}
}
}
}
let mut list: Vec<(String, u64)> = primary
.into_iter()
.map(|(w, (lower, other))| {
// Prefer lowercase entries; where spellings disagree (re•cord / rec•ord) keep only the
// breaks they share.
let v = if lower.is_empty() { other } else { lower };
(w, v.iter().fold(u64::MAX, |a, b| a & b))
})
.collect();
let have: std::collections::HashSet<String> = list.iter().map(|x| x.0.clone()).collect();
let extra: Vec<(String, u64)> = parts.into_iter().filter(|(w, _)| !have.contains(w)).collect();
println!("{entries} entries → {} words (+{} from compounds)", list.len(), extra.len());
list.extend(extra);
let dict = Dictionary::from_entries(list);
let bytes = dict.to_bytes();
std::fs::create_dir_all(&out_dir).expect("mkdir");
std::fs::write(out_dir.join("en-us.dic"), &bytes).expect("write en-us.dic");
let back = Dictionary::from_bytes(&bytes).expect("roundtrip");
assert_eq!(back.len(), dict.len());
println!("en-us.dic: {} words, {} bytes", dict.len(), bytes.len());
// Training data: plain lowercase ASCII words (+ apostrophes).
let train: Vec<(Vec<char>, Vec<usize>)> = dict
.iter()
.filter(|(w, _)| w.chars().all(|c| c.is_ascii_lowercase() || c == '\''))
.map(|(w, m)| (w.chars().collect(), (1..w.len().min(64)).filter(|&i| m & (1 << i) != 0).collect()))
.collect();
println!("training on {} words", train.len());
let mut log = |s: String| println!(" {s}");
// Optional override for tuning: HYPHGEN_LEVELS="1,4,1,2,20;2,5,2,1,8;…" (min,max,good,bad,threshold).
let custom: Option<Vec<patgen::Level>> = std::env::var("HYPHGEN_LEVELS").ok().map(|s| {
s.split(';')
.map(|l| {
let v: Vec<u32> = l.split(',').map(|x| x.trim().parse().expect("number")).collect();
patgen::Level { min_len: v[0] as usize, max_len: v[1] as usize, good_weight: v[2], bad_weight: v[3], threshold: v[4] }
})
.collect()
});
let levels = custom.as_deref().unwrap_or(DEFAULT_LEVELS);
if validate {
let (fit, hold): (Vec<_>, Vec<_>) = train.iter().cloned().enumerate().partition(|(i, _)| i % 10 != 0);
let fit: Vec<_> = fit.into_iter().map(|x| x.1).collect();
let hold: Vec<_> = hold.into_iter().map(|x| x.1).collect();
let p = patgen::train(&fit, levels, &mut log);
let s = patgen::score(&p, &hold);
println!(
"held-out: {} words, {} breaks: found {} ({:.1}%), missed {}, wrong {} ({:.2}% of reported)",
s.words,
s.breaks,
s.found,
100.0 * s.found as f64 / s.breaks.max(1) as f64,
s.missed(),
s.wrong,
100.0 * s.wrong as f64 / (s.found + s.wrong).max(1) as f64
);
}
if std::env::var_os("HYPHGEN_VALIDATE_ONLY").is_some() {
return;
}
let pats = patgen::train(&train, levels, &mut log);
let s = patgen::score(&pats, &train);
println!(
"training set: found {}/{} ({:.1}%), wrong {} — {} patterns",
s.found,
s.breaks,
100.0 * s.found as f64 / s.breaks.max(1) as f64,
s.wrong,
pats.len()
);
let mut out = String::new();
out.push_str("% Craft US English hyphenation patterns (Liang/TeX notation).\n");
out.push_str("% Generated by crates/text/examples/hyphgen.rs: trained with craft-text's patgen-style trainer\n");
out.push_str("% from the Moby Hyphenator II word list by Grady Ward (public domain, Project Gutenberg #3204).\n");
out.push_str("% Released into the public domain (CC0-1.0) by the DesignCraft and craft-libs contributors.\n");
for line in pats.to_tex().chunks(12) {
out.push_str(&line.join(" "));
out.push('\n');
}
std::fs::write(out_dir.join("en-us.pat"), &out).expect("write en-us.pat");
println!("en-us.pat: {} patterns, {} bytes", pats.len(), out.len());
for w in ["hyphenation", "typography", "composition", "photographer", "unquestionably", "localization", "blockchain", "smartphone"] {
let c: Vec<char> = w.chars().collect();
let m = pats.points(&c).unwrap_or_default().iter().fold(0u64, |m, &i| m | 1 << i);
println!(" patterns: {}", hyphenated(w, m));
}
}
crates/text/src/bidi/mod.rs
+367
View File
@@ -0,0 +1,367 @@
//! Bidirectional text (UAX #9) for document layout, on plain text and independent of fonts.
//!
//! The pipeline is the one the algorithm prescribes: [`Paragraph::new`] resolves embedding levels
//! once for the whole logical paragraph (or for each paragraph, in text with paragraph
//! separators); shaping and line breaking happen in logical order; then each line is reordered on
//! its own. [`Paragraph::line_levels`] applies rule L1 and
//! [`Paragraph::visual_runs`] rule L2 to a byte range of the paragraph. For lines built from other
//! units (clusters, glyphs, cells), [`line_levels`] and [`visual_order`] apply the same rules to
//! slices of levels. [`mirror`] gives the character for rule L4; drawing it, or a font's mirrored
//! glyph, is left to the shaper.
//!
//! Levels are `u8` embedding levels: even runs left to right, odd runs right to left.
use std::ops::Range;
use unicode_bidi::BidiClass::{self, AL, AN, B, BN, FSI, LRE, LRI, LRO, PDF, PDI, R, RLE, RLI, RLO, S, WS};
use unicode_bidi::{BidiInfo, Level, bidi_class};
/// A paragraph's base direction (rules P2–P3, or the higher-level protocol HL1 when set).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum Direction {
/// Left to right: paragraph level 0.
Ltr,
/// Right to left: paragraph level 1.
Rtl,
/// From the first strong character, left to right when there is none.
Auto,
}
/// A directional override (LRO or RLO … PDF) over a byte range of a paragraph: every character in
/// it is treated as strongly left to right or right to left.
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct Override {
pub range: Range<usize>,
pub rtl: bool,
}
/// A maximal run of one embedding level in a line, as [`Paragraph::visual_runs`] returns it.
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub struct Run {
/// Byte range in the paragraph text, in logical order.
pub range: Range<usize>,
pub level: u8,
}
impl Run {
/// Is the run displayed right to left (its characters from the end of `range` to its start)?
pub fn is_rtl(&self) -> bool {
self.level % 2 == 1
}
}
/// Resolved embedding levels (rules up to I2) for a paragraph's text, one per byte.
///
/// A paragraph separator (`\n`, U+2029 and the other class B characters) inside the text ends a
/// paragraph and belongs to it; the next one starts after it with its own paragraph level, which
/// `Direction::Auto` resolves from that paragraph's own first strong character. Rule L1 uses the
/// level of the paragraph each byte is in.
#[derive(Clone, Debug)]
pub struct Paragraph<'t> {
text: &'t str,
levels: Vec<u8>,
/// Each paragraph's byte range and level, in order and covering the text; never empty.
paragraphs: Vec<(Range<usize>, u8)>,
}
impl<'t> Paragraph<'t> {
/// Resolve the levels of `text`. Text with nothing right to left in a left-to-right or `Auto`
/// paragraph skips the algorithm.
pub fn new(text: &'t str, base: Direction) -> Self {
if base != Direction::Rtl && !needs_bidi(text) {
let paragraphs = split_paragraphs(text).into_iter().map(|r| (r, 0)).collect();
return Self { text, levels: vec![0; text.len()], paragraphs };
}
let (levels, paragraphs) = resolve(text, base);
Self { text, levels, paragraphs }
}
/// Like [`new`](Self::new), with directional overrides. Overrides that are empty, out of
/// range, off char boundaries or overlapping an earlier one are ignored. An override that
/// spans a paragraph separator continues in the next paragraph; the separator keeps its
/// paragraph's level.
pub fn with_overrides(text: &'t str, base: Direction, overrides: &[Override]) -> Self {
let mut sorted: Vec<&Override> = overrides.iter().filter(|o| o.range.start < o.range.end && text.get(o.range.clone()).is_some()).collect();
sorted.sort_by_key(|o| o.range.start);
// The text with LRO/RLO … PDF around each override, and where each source char went.
let mut wrapped = String::with_capacity(text.len() + sorted.len() * 6);
let mut at: Vec<usize> = Vec::with_capacity(text.len());
let mut copy = |wrapped: &mut String, piece: Range<usize>| {
if let Some(s) = text.get(piece) {
for (_, c) in s.char_indices() {
at.extend(std::iter::repeat_n(wrapped.len(), c.len_utf8()));
wrapped.push(c);
}
}
};
let mut pos = 0;
for o in sorted {
if o.range.start < pos {
continue;
}
copy(&mut wrapped, pos..o.range.start);
// A paragraph separator ends every embedding (rule X8), so the override is closed
// before each separator and opened again after it.
let mut start = o.range.start;
while start < o.range.end {
let rest = text.get(start..o.range.end).unwrap_or_default();
let (end, next) = rest
.char_indices()
.find(|&(_, c)| bidi_class(c) == B)
.map_or((o.range.end, o.range.end), |(i, c)| (start + i, start + i + c.len_utf8()));
if start < end {
wrapped.push(if o.rtl { '\u{202E}' } else { '\u{202D}' });
copy(&mut wrapped, start..end);
wrapped.push('\u{202C}');
}
copy(&mut wrapped, end..next);
start = next;
}
pos = o.range.end;
}
if pos == 0 {
return Self::new(text, base);
}
copy(&mut wrapped, pos..text.len());
let (wrapped_levels, wrapped_paragraphs) = resolve(&wrapped, base);
// The inserted controls are not separators, so the paragraphs are the same ones; their
// ranges map back through `at`, which is ascending.
let back = |w: usize| at.partition_point(|&a| a < w);
let paragraphs: Vec<(Range<usize>, u8)> = wrapped_paragraphs.iter().map(|(r, l)| (back(r.start)..back(r.end), *l)).collect();
let levels = at.iter().enumerate().map(|(b, &w)| wrapped_levels.get(w).copied().unwrap_or_else(|| level_in(&paragraphs, b))).collect();
Self { text, levels, paragraphs }
}
pub fn text(&self) -> &'t str {
self.text
}
/// The first paragraph's embedding level: 0 or 1. Text with paragraph separators has one
/// level per paragraph: see [`paragraphs`](Self::paragraphs).
pub fn level(&self) -> u8 {
self.paragraphs.first().map_or(0, |p| p.1)
}
/// Is the first paragraph right to left?
pub fn is_rtl(&self) -> bool {
self.level() % 2 == 1
}
/// Each paragraph's byte range (its separator included) and embedding level, in order. There
/// is always at least one; empty text has one empty paragraph.
pub fn paragraphs(&self) -> &[(Range<usize>, u8)] {
&self.paragraphs
}
/// The embedding level of the paragraph containing byte `byte` (the last one's past the end).
pub fn paragraph_level_at(&self, byte: usize) -> u8 {
level_in(&self.paragraphs, byte)
}
/// Does any line need reordering (is any level above 0)?
pub fn needs_reordering(&self) -> bool {
self.levels.iter().any(|l| *l > 0)
}
/// The level of each byte of the text.
pub fn levels(&self) -> &[u8] {
&self.levels
}
pub fn into_levels(self) -> Vec<u8> {
self.levels
}
/// The level of the character at byte `byte` (the last paragraph's level past the end).
pub fn level_at(&self, byte: usize) -> u8 {
self.levels.get(byte).copied().unwrap_or_else(|| self.paragraph_level_at(byte))
}
/// The level of each character, in order.
pub fn char_levels(&self) -> Vec<u8> {
self.text.char_indices().map(|(b, _)| self.level_at(b)).collect()
}
/// Rule L1 for the line `line` (a byte range of the text): the level of each of its bytes,
/// with segment and paragraph separators, whitespace before them and whitespace at the end of
/// the line at their paragraph's level; zero-width joiners and tag characters keep the level of
/// the char before them. Empty when `line` is not a char-boundary range of the text. A line
/// should lie within one paragraph; one that crosses a separator takes each byte's own
/// paragraph level here and is reordered as a whole by [`visual_runs`](Self::visual_runs).
pub fn line_levels(&self, line: Range<usize>) -> Vec<u8> {
let Some(s) = self.text.get(line.clone()) else { return Vec::new() };
let mut out: Vec<u8> = line.clone().map(|b| self.level_at(b)).collect();
let mut trailing = true;
for (b, c) in s.char_indices().rev() {
if l1_reset(bidi_class(c), &mut trailing)
&& let Some(bytes) = out.get_mut(b..b + c.len_utf8())
{
bytes.fill(self.paragraph_level_at(line.start + b));
}
}
let mut prev = None;
for (b, c) in s.char_indices() {
if joins_previous(c)
&& let Some(l) = prev
&& let Some(bytes) = out.get_mut(b..b + c.len_utf8())
{
bytes.fill(l);
}
prev = out.get(b).copied();
}
out
}
/// Rules L1 and L2 for the line `line`: its runs of one level, left to right as displayed.
/// Empty when `line` is not a char-boundary range of the text.
pub fn visual_runs(&self, line: Range<usize>) -> Vec<Run> {
let levels = self.line_levels(line.clone());
let Some(s) = self.text.get(line.clone()) else { return Vec::new() };
let mut runs: Vec<Run> = Vec::new();
for (b, c) in s.char_indices() {
let level = levels.get(b).copied().unwrap_or_else(|| self.paragraph_level_at(line.start + b));
let end = line.start + b + c.len_utf8();
match runs.last_mut() {
Some(run) if run.level == level => run.range.end = end,
_ => runs.push(Run { range: line.start + b..end, level }),
}
}
let run_levels: Vec<u8> = runs.iter().map(|r| r.level).collect();
visual_order(&run_levels).into_iter().filter_map(|i| runs.get(i).cloned()).collect()
}
}
/// Rule L1 over a line's characters `chars` and their resolved `levels`: segment and paragraph
/// separators, any whitespace before them and whitespace at the end of the line take `para_level`;
/// zero-width joiners and tag characters keep the level of the char before them. Missing levels are
/// taken as `para_level`.
pub fn line_levels(chars: &[char], levels: &[u8], para_level: u8) -> Vec<u8> {
let mut out = levels.to_vec();
out.resize(chars.len(), para_level);
let mut trailing = true;
for (i, c) in chars.iter().enumerate().rev() {
if l1_reset(bidi_class(*c), &mut trailing)
&& let Some(l) = out.get_mut(i)
{
*l = para_level;
}
}
for i in 1..out.len() {
if chars.get(i).is_some_and(|&c| joins_previous(c))
&& let Some(&l) = out.get(i - 1)
&& let Some(cur) = out.get_mut(i)
{
*cur = l;
}
}
out
}
/// Zero-width joiners and tag characters extend the grapheme cluster of the char before them
/// (UAX #29), so they take its level. Rule L1 counts them as whitespace and would otherwise give a
/// trailing one the paragraph level, splitting its cluster across two runs.
fn joins_previous(c: char) -> bool {
matches!(c, '\u{200C}' | '\u{200D}' | '\u{E0020}'..='\u{E007F}')
}
/// Rule L2: the display order, left to right, of items with these levels (indices into `levels`).
/// From the highest level down to the lowest odd one, every maximal run at that level or higher is
/// reversed.
pub fn visual_order(levels: &[u8]) -> Vec<usize> {
let mut order: Vec<usize> = (0..levels.len()).collect();
let Some(min_odd) = levels.iter().copied().filter(|l| l % 2 == 1).min() else { return order };
let max = levels.iter().copied().max().unwrap_or(0);
let level_at = |order: &[usize], i: usize| order.get(i).and_then(|&o| levels.get(o)).copied().unwrap_or(0);
for lvl in (min_odd..=max).rev() {
let mut i = 0;
while i < order.len() {
if level_at(&order, i) < lvl {
i += 1;
continue;
}
let mut j = i;
while j < order.len() && level_at(&order, j) >= lvl {
j += 1;
}
if let Some(run) = order.get_mut(i..j) {
run.reverse();
}
i = j;
}
}
order
}
/// The mirrored counterpart of `c` (Bidi_Mirroring_Glyph), for a character at an odd level (rule
/// L4): `(` for `)`, `»` for `«`. `None` when it has none; some Bidi_Mirrored characters (`∑`)
/// need a mirrored glyph from the font instead.
pub fn mirror(c: char) -> Option<char> {
unicode_bidi_mirroring::get_mirrored(c)
}
/// Does anything in the text need the full algorithm (strong right-to-left characters, Arabic
/// numbers or explicit embeddings)?
fn needs_bidi(text: &str) -> bool {
text.chars().any(|c| matches!(bidi_class(c), R | AL | AN | LRE | RLE | LRO | RLO | PDF | LRI | RLI | FSI | PDI))
}
/// Levels per byte of `text`, and each paragraph's byte range and level.
fn resolve(text: &str, base: Direction) -> (Vec<u8>, Vec<(Range<usize>, u8)>) {
let default = match base {
Direction::Ltr => Some(Level::ltr()),
Direction::Rtl => Some(Level::rtl()),
Direction::Auto => None,
};
let info = BidiInfo::new(text, default);
let mut paragraphs: Vec<(Range<usize>, u8)> = info.paragraphs.iter().map(|p| (p.range.clone(), p.level.number())).collect();
if paragraphs.is_empty() {
paragraphs.push((0..text.len(), u8::from(base == Direction::Rtl)));
}
let levels = (0..text.len()).map(|b| info.levels.get(b).map_or_else(|| level_in(&paragraphs, b), |l| l.number())).collect();
(levels, paragraphs)
}
/// The byte ranges of the paragraphs in `text` as rule P1 (and unicode-bidi) splits them: each
/// ends after a paragraph separator, and there is always at least one.
fn split_paragraphs(text: &str) -> Vec<Range<usize>> {
let mut out = Vec::new();
let mut start = 0;
for (b, c) in text.char_indices() {
if bidi_class(c) == B {
out.push(start..b + c.len_utf8());
start = b + c.len_utf8();
}
}
if start < text.len() || out.is_empty() {
out.push(start..text.len());
}
out
}
/// The level of the paragraph containing byte `byte`; the last paragraph's past the end.
fn level_in(paragraphs: &[(Range<usize>, u8)], byte: usize) -> u8 {
let i = paragraphs.partition_point(|(r, _)| r.end <= byte);
paragraphs.get(i).or(paragraphs.last()).map_or(0, |p| p.1)
}
/// Rule L1 walking a line backwards: whether the character of class `class` takes the paragraph
/// level. `trailing` is true while everything after it up to the line end or a separator is
/// whitespace.
fn l1_reset(class: BidiClass, trailing: &mut bool) -> bool {
match class {
S | B => {
*trailing = true;
true
}
WS | FSI | LRI | RLI | PDI | BN | LRE | RLE | LRO | RLO | PDF => *trailing,
_ => {
*trailing = false;
false
}
}
}
#[cfg(test)]
mod tests;
crates/text/src/bidi/tests.rs
+254
View File
@@ -0,0 +1,254 @@
use super::*;
fn rev(s: &str) -> String {
s.chars().rev().collect()
}
/// Byte offset of char `n` of `s`.
fn at(s: &str, n: usize) -> usize {
s.char_indices().nth(n).map_or(s.len(), |(i, _)| i)
}
/// The line as displayed, left to right, built from its visual runs.
fn shown(p: &Paragraph<'_>, line: Range<usize>) -> String {
p.visual_runs(line).iter().map(|r| if r.is_rtl() { rev(&p.text()[r.range.clone()]) } else { p.text()[r.range.clone()].to_string() }).collect()
}
fn shown_all(text: &str, base: Direction) -> String {
shown(&Paragraph::new(text, base), 0..text.len())
}
#[test]
fn mixed_arabic_latin_numbers_and_brackets() {
let s = "سنة 2024 (Retrait) فقط";
let p = Paragraph::new(s, Direction::Rtl);
let l = p.char_levels();
let chars: Vec<char> = s.chars().collect();
assert_eq!(l.len(), chars.len());
// Arabic letters at the paragraph level 1; European digits and Latin at 2.
assert_eq!(l[0], 1);
assert_eq!(&l[4..8], &[2, 2, 2, 2]);
assert_eq!(&l[10..17], &[2; 7]);
assert_eq!(l[19], 1);
// Brackets around Latin inside a right-to-left paragraph take the embedding direction (N0).
assert_eq!(l[9] % 2, 1, "( is right to left");
assert_eq!(l[17] % 2, 1, ") is right to left");
// The Arabic words reversed, Latin and digits kept, the brackets swapped (the shaper mirrors
// them back, rule L4). Char-level reordering and byte runs agree.
let order = visual_order(&line_levels(&chars, &l, p.level()));
let vis: String = order.iter().map(|&i| chars[i]).collect();
assert_eq!(vis, "طقف )Retrait( 2024 ةنس");
assert_eq!(shown(&p, 0..s.len()), vis);
}
#[test]
fn levels_for_persian_with_latin_and_numbers() {
let p = Paragraph::new("plain English", Direction::Ltr);
assert!(!p.needs_reordering(), "pure left-to-right paragraphs skip the algorithm");
assert_eq!(p.levels(), &[0; 13]);
let s = "سلام Word ۱۴۰۳";
let p = Paragraph::new(s, Direction::Rtl);
assert_eq!(p.levels().len(), s.len());
assert!(p.needs_reordering());
assert_eq!(p.level_at(0), 1, "Persian letters are right to left");
assert_eq!(p.level_at(at(s, 5)), 2, "Latin inside a right-to-left paragraph is embedded at level 2");
assert_eq!(p.level_at(at(s, 10)), 2, "Persian digits are numbers: left to right inside right-to-left text");
assert_eq!(p.level_at(s.len()), 1, "past the end: the paragraph level");
// In a left-to-right paragraph Persian is embedded at level 1.
let s = "Hello سلام";
let p = Paragraph::new(s, Direction::Ltr);
assert_eq!((p.level_at(0), p.level_at(at(s, 6))), (0, 1));
// A right-to-left paragraph of Latin text keeps it left to right inside it.
assert_eq!(Paragraph::new("ab", Direction::Rtl).char_levels(), vec![2, 2]);
}
#[test]
fn auto_direction_follows_the_first_strong_character() {
assert!(Paragraph::new("سلام Word", Direction::Auto).is_rtl());
assert!(!Paragraph::new("Word سلام", Direction::Auto).is_rtl());
assert!(Paragraph::new("123 سلام", Direction::Auto).is_rtl(), "digits are weak");
assert_eq!(Paragraph::new("123", Direction::Auto).level(), 0);
assert_eq!(Paragraph::new("", Direction::Auto).level(), 0);
assert_eq!(Paragraph::new("", Direction::Rtl).level(), 1);
assert_eq!(Paragraph::new("Word", Direction::Rtl).level(), 1, "an explicit direction wins");
}
#[test]
fn whole_lines_read_in_display_order() {
let s = "سلام دنیا";
assert_eq!(shown_all(s, Direction::Rtl), rev(s));
assert_eq!(shown_all("Hello سلام world", Direction::Ltr), "Hello مالس world");
// "Price 250 toman" in Persian, with a Latin product name, Persian punctuation and digits:
// numbers and Latin keep their order, the words around them read right to left, and the final
// full stop is at the paragraph's end (the far left).
let s = "قیمت 250 تومان برای Word، امسال ۱۴۰۳.";
let want = format!(".۱۴۰۳ {} ،Word {} {} 250 {}", rev("امسال"), rev("برای"), rev("تومان"), rev("قیمت"));
assert_eq!(shown_all(s, Direction::Rtl), want);
}
#[test]
fn each_line_is_reordered_on_its_own() {
// A right-to-left paragraph of Latin words broken after "ab ": the trailing space takes the
// paragraph level (rule L1), so it sits at the line's end edge, on the left.
let p = Paragraph::new("ab cd", Direction::Rtl);
assert_eq!(p.visual_runs(0..3), vec![Run { range: 2..3, level: 1 }, Run { range: 0..2, level: 2 }]);
assert_eq!(p.visual_runs(3..5), vec![Run { range: 3..5, level: 2 }]);
// Line ranges that aren't char-boundary ranges of the text give nothing.
let p = Paragraph::new("سلام", Direction::Rtl);
assert!(p.visual_runs(1..2).is_empty());
assert!(p.line_levels(0..99).is_empty());
#[allow(clippy::reversed_empty_ranges)]
let backwards = 4..2;
assert!(p.visual_runs(backwards).is_empty());
assert!(p.visual_runs(2..2).is_empty());
}
#[test]
fn l1_resets_trailing_whitespace_and_tabs() {
let chars: Vec<char> = "ab\tcd ".chars().collect();
assert_eq!(line_levels(&chars, &[2; 7], 1), vec![2, 2, 1, 2, 2, 1, 1]);
assert_eq!(Paragraph::new("ab\tcd ", Direction::Rtl).line_levels(0..7), vec![2, 2, 1, 2, 2, 1, 1]);
// Short level slices never panic.
assert_eq!(line_levels(&['a', 'b'], &[], 1), vec![1, 1]);
}
#[test]
fn visual_order_follows_rule_l2() {
// The example from UAX #9 / unicode-bidi.
assert_eq!(visual_order(&[0, 0, 0, 1, 1, 1, 2, 2]), vec![0, 1, 2, 6, 7, 5, 4, 3]);
assert!(visual_order(&[]).is_empty());
assert_eq!(visual_order(&[0, 0]), vec![0, 1]);
assert_eq!(visual_order(&[1, 1, 1]), vec![2, 1, 0]);
// Numbers (level 2) inside right-to-left text keep their order.
assert_eq!(visual_order(&[1, 2, 2, 1]), vec![3, 1, 2, 0]);
assert_eq!(visual_order(&[125, 125]), vec![1, 0]);
assert_eq!(visual_order(&[255, 0, 255]), vec![0, 1, 2]);
}
#[test]
fn overrides_force_a_direction() {
let p = Paragraph::with_overrides("abc def", Direction::Ltr, &[Override { range: 0..3, rtl: true }]);
assert_eq!(p.levels().len(), 7);
assert_eq!(shown(&p, 0..7), "cba def");
let s = "سلام دنیا";
let p = Paragraph::with_overrides(s, Direction::Rtl, &[Override { range: 0..at(s, 4), rtl: false }]);
assert_eq!(shown(&p, 0..s.len()), format!("{} سلام", rev("دنیا")));
// Overrides off char boundaries, empty, out of range or overlapping an earlier one are ignored.
let bad = [Override { range: 1..2, rtl: false }, Override { range: 3..3, rtl: false }, Override { range: 0..99, rtl: false }];
assert_eq!(Paragraph::with_overrides(s, Direction::Rtl, &bad).levels(), Paragraph::new(s, Direction::Rtl).levels());
let overlapping = [Override { range: 2..5, rtl: true }, Override { range: 0..3, rtl: true }];
assert_eq!(shown(&Paragraph::with_overrides("abcdef", Direction::Ltr, &overlapping), 0..6), "cbadef");
}
#[test]
fn each_paragraph_has_its_own_level() {
// The second paragraph resolves right to left on its own, so rule L1 puts its trailing space at
// level 1: at the line's end edge, on the left.
let s = "a\nאב ";
let p = Paragraph::new(s, Direction::Auto);
assert_eq!(p.paragraphs(), &[(0..2, 0), (2..7, 1)]);
assert_eq!((p.level(), p.is_rtl()), (0, false), "level() is the first paragraph's");
assert_eq!((p.paragraph_level_at(1), p.paragraph_level_at(2), p.level_at(s.len())), (0, 1, 1));
assert_eq!(p.line_levels(2..7), vec![1; 5]);
assert_eq!(p.visual_runs(2..7), vec![Run { range: 2..7, level: 1 }]);
assert_eq!(p.line_levels(0..2), vec![0, 0]);
// Left-to-right text skips the algorithm but still splits paragraphs.
assert_eq!(Paragraph::new("ab\u{2029}c", Direction::Ltr).paragraphs(), &[(0..5, 0), (5..6, 0)]);
assert_eq!(Paragraph::new("", Direction::Auto).paragraphs(), &[(0..0, 0)]);
}
#[test]
fn overrides_continue_across_paragraph_separators() {
let s = "ab\ncd";
let p = Paragraph::with_overrides(s, Direction::Ltr, &[Override { range: 0..5, rtl: true }]);
assert_eq!(p.paragraphs(), &[(0..3, 0), (3..5, 0)]);
assert_eq!(p.line_levels(0..3), vec![1, 1, 0], "the separator keeps the paragraph level");
assert_eq!(shown(&p, 0..3), "ba\n");
assert_eq!(shown(&p, 3..5), "dc");
// Each paragraph still resolves its own direction.
let s = "a\nאב ";
let p = Paragraph::with_overrides(s, Direction::Auto, &[Override { range: 0..1, rtl: false }]);
assert_eq!(p.paragraphs(), &[(0..2, 0), (2..7, 1)]);
assert_eq!(p.visual_runs(2..7), vec![Run { range: 2..7, level: 1 }]);
}
#[test]
fn mirrored_characters() {
assert_eq!(mirror('('), Some(')'));
assert_eq!(mirror(']'), Some('['));
assert_eq!(mirror('«'), Some('»'));
assert_eq!(mirror('a'), None);
assert_eq!(mirror('∑'), None, "mirrored, but only by glyph");
}
#[test]
fn hostile_text_never_panics() {
// Explicit embeddings, isolates, unmatched pops, marks with no base, deep nesting.
let rlo = "\u{202E}".repeat(200);
let junk = [
"\u{202B}\u{202B}\u{202C}\u{202C}\u{202C}abc\u{2067}سلام\u{2069}\u{2069}",
"\u{064E}\u{064E}\u{200C}\u{200D}",
"\u{200F}\u{200E}\t\u{200F}\n\u{200E}",
"(((((((((سلام]]]]]]]",
"a\u{2029}سلام\u{2029}b",
&rlo,
];
for s in junk {
for base in [Direction::Ltr, Direction::Rtl, Direction::Auto] {
let overrides = [Override { range: 0..s.len() / 2, rtl: true }, Override { range: s.len() / 2..s.len(), rtl: false }];
for p in [Paragraph::new(s, base), Paragraph::with_overrides(s, base, &overrides)] {
assert_eq!(p.levels().len(), s.len());
let paras = p.paragraphs();
assert_eq!((paras.first().map(|p| p.0.start), paras.last().map(|p| p.0.end)), (Some(0), Some(s.len())), "{s:?}");
assert!(paras.windows(2).all(|w| w[0].0.end == w[1].0.start), "{s:?}");
let bounds: Vec<usize> = (0..=s.len()).filter(|&b| s.is_char_boundary(b)).collect();
for &b in &bounds {
for line in [0..b, b..s.len()] {
let runs = p.visual_runs(line.clone());
assert_eq!(runs.iter().map(|r| r.range.len()).sum::<usize>(), line.len(), "{s:?} {line:?}");
assert!(runs.iter().all(|r| !r.range.is_empty() && line.start <= r.range.start && r.range.end <= line.end));
}
}
}
}
}
}
#[test]
fn runs_never_split_a_grapheme_cluster() {
use unicode_segmentation::UnicodeSegmentation;
let cases = [
// Arabic with harakat, among Latin and digits.
("كَتَبَ abc مُحَمَّدٌ 123 بِسْمِ", Direction::Rtl),
("see كَتَبَ, مُحَمَّدٌ here", Direction::Ltr),
// Hebrew with niqqud (and a dagesh-shin-dot stack).
("שָׁלוֹם hello עוֹלָם", Direction::Rtl),
("hello שָׁלוֹם world", Direction::Ltr),
// A Devanagari conjunct (virama, and an explicit half form with ZWJ) in right-to-left text.
("سلام क्षत्रिय क्\u{200D}ष سلام", Direction::Rtl),
("سلام क्\u{200D} ", Direction::Rtl),
// A combining mark after a Latin letter in a right-to-left paragraph, and after a Hebrew
// letter in a left-to-right one, each at a run edge.
("שלום e\u{0301}te\u{0301} עולם", Direction::Rtl),
("שלום cafe\u{0301}", Direction::Rtl),
("café ש\u{05B8}\u{05BC} next", Direction::Ltr),
("abc ש\u{05B8}", Direction::Ltr),
("a\u{0301}\u{0302}ש", Direction::Auto),
];
for (s, base) in cases {
let p = Paragraph::new(s, base);
let bounds: Vec<usize> = s.grapheme_indices(true).map(|(i, _)| i).chain([s.len()]).collect();
for &a in &bounds {
for &b in bounds.iter().filter(|&&b| b > a) {
for r in p.visual_runs(a..b) {
assert!(bounds.contains(&r.range.start) && bounds.contains(&r.range.end), "{s:?} line {a}..{b}: run {r:?}");
}
}
}
}
// The char-level L1 keeps a trailing joiner with its conjunct too.
let s = "سلام क्\u{200D} ";
let chars: Vec<char> = s.chars().collect();
let l = line_levels(&chars, &Paragraph::new(s, Direction::Rtl).char_levels(), 1);
assert_eq!((l[6], l[7], l[8]), (2, 2, 1));
}
crates/text/src/hyphen/dict.rs
+159
View File
@@ -0,0 +1,159 @@
//! Hyphenation exception dictionary: lowercase words with their break points.
//!
//! On-disk format (`assets/en-us.dic`): raw-deflate of
//! `"DCHYPH1\n"` followed by one line per word, sorted by the plain word's bytes. Each line is
//! front-coded against the previous one: a byte with the shared prefix length (in bytes of the
//! hyphenated form), then the rest of the hyphenated form (`-` at every break), then `\n`.
/// Sorted word list with a break bitmask per word (bit `i` = hyphen before char `i`).
#[derive(Clone, Debug, Default)]
pub struct Dictionary {
words: String,
offs: Vec<u32>,
masks: Vec<u64>,
}
const MAGIC: &[u8] = b"DCHYPH1\n";
impl Dictionary {
pub fn len(&self) -> usize {
self.masks.len()
}
pub fn is_empty(&self) -> bool {
self.masks.is_empty()
}
fn word(&self, i: usize) -> &str {
&self.words[self.offs[i] as usize..self.offs[i + 1] as usize]
}
/// Break mask of an exact (already lowercased) word.
pub fn get(&self, word: &str) -> Option<u64> {
let (mut lo, mut hi) = (0usize, self.len());
while lo < hi {
let mid = (lo + hi) / 2;
match self.word(mid).as_bytes().cmp(word.as_bytes()) {
std::cmp::Ordering::Less => lo = mid + 1,
std::cmp::Ordering::Greater => hi = mid,
std::cmp::Ordering::Equal => return Some(self.masks[mid]),
}
}
None
}
/// Iterate `(word, mask)`.
pub fn iter(&self) -> impl Iterator<Item = (&str, u64)> + '_ {
(0..self.len()).map(|i| (self.word(i), self.masks[i]))
}
/// Deserialize data written by `Dictionary::to_bytes`.
pub(crate) fn decode(data: &[u8]) -> Result<Dictionary, String> {
let raw = miniz_oxide::inflate::decompress_to_vec(data).map_err(|e| format!("hyphenation dictionary: inflate failed: {e:?}"))?;
let body = raw.strip_prefix(MAGIC).ok_or("hyphenation dictionary: bad header")?;
let mut d = Dictionary { words: String::with_capacity(body.len()), offs: vec![0], masks: Vec::new() };
let mut cur: Vec<u8> = Vec::new();
let mut i = 0;
while i < body.len() {
let p = body[i] as usize;
let end = body[i + 1..].iter().position(|&b| b == b'\n').map(|e| i + 1 + e).ok_or("hyphenation dictionary: truncated")?;
if p > cur.len() {
return Err("hyphenation dictionary: bad prefix".into());
}
cur.truncate(p);
cur.extend_from_slice(&body[i + 1..end]);
i = end + 1;
let s = std::str::from_utf8(&cur).map_err(|_| "hyphenation dictionary: bad utf-8")?;
let (w, m) = parse_hyphenated(s);
d.words.push_str(&w);
d.offs.push(d.words.len() as u32);
d.masks.push(m);
}
Ok(d)
}
}
#[cfg(any(test, feature = "generator"))]
impl Dictionary {
/// Build from `(lowercase word, break mask)` pairs (sorted and de-duplicated here; the first
/// entry of a duplicate wins).
pub fn from_entries(mut entries: Vec<(String, u64)>) -> Dictionary {
entries.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes()));
entries.dedup_by(|b, a| a.0 == b.0);
let mut d = Dictionary { words: String::new(), offs: vec![0], masks: Vec::with_capacity(entries.len()) };
for (w, m) in entries {
d.words.push_str(&w);
d.offs.push(d.words.len() as u32);
d.masks.push(m);
}
d
}
/// Serialize (deflate-compressed).
pub fn to_bytes(&self) -> Vec<u8> {
let mut raw = MAGIC.to_vec();
let mut prev: Vec<u8> = Vec::new();
for (w, m) in self.iter() {
let h = hyphenated(w, m).into_bytes();
let p = prev.iter().zip(&h).take_while(|(a, b)| a == b).count().min(255);
raw.push(p as u8);
raw.extend_from_slice(&h[p..]);
raw.push(b'\n');
prev = h;
}
miniz_oxide::deflate::compress_to_vec(&raw, 10)
}
/// Deserialize data written by [`Dictionary::to_bytes`].
pub fn from_bytes(data: &[u8]) -> Result<Dictionary, String> {
Dictionary::decode(data)
}
}
/// `word` with `-` before every char whose bit is set in `mask`.
#[cfg(any(test, feature = "generator"))]
pub fn hyphenated(word: &str, mask: u64) -> String {
let mut s = String::with_capacity(word.len() + 8);
for (i, c) in word.chars().enumerate() {
if i > 0 && i < 64 && mask & (1 << i) != 0 {
s.push('-');
}
s.push(c);
}
s
}
/// A word written with `-` at its breaks, as `(word, break mask)`.
pub fn parse_hyphenated(s: &str) -> (String, u64) {
let mut w = String::with_capacity(s.len());
let mut m = 0u64;
let mut n = 0;
for c in s.chars() {
if c == '-' {
if n > 0 && n < 64 {
m |= 1 << n;
}
} else {
w.push(c);
n += 1;
}
}
(w, m)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn roundtrip() {
let e: Vec<(String, u64)> = ["ty-pog-ra-phy", "cat", "hap-pen", "o'clock", "naïve-té", "naïve"].iter().map(|s| parse_hyphenated(s)).collect();
let d = Dictionary::from_entries(e);
let back = Dictionary::from_bytes(&d.to_bytes()).unwrap();
assert_eq!(back.len(), 6);
assert_eq!(back.get("typography").map(|m| hyphenated("typography", m)), Some("ty-pog-ra-phy".to_string()));
assert_eq!(back.get("cat"), Some(0));
assert_eq!(back.get("naïveté").map(|m| hyphenated("naïveté", m)), Some("naïve-té".to_string()));
assert!(back.get("dog").is_none());
}
}
crates/text/src/hyphen/mod.rs
+509
View File
@@ -0,0 +1,509 @@
//! Hyphenation points for text layout. English uses the bundled dictionary and patterns below.
//! Other languages use the Liang patterns compiled into [`hypher`], one cargo feature per
//! language (`spanish`, `german`, …, or `all-languages`); their licenses are listed in NOTICE.
//!
//! English lookup order for each word:
//! 1. the **dictionary** ([`Dictionary::en_us`]) — the public-domain Moby Hyphenator II word list
//! by Grady Ward (~160k words), stored compactly in `assets/hyphen/en-us.dic`; simple
//! inflections (`'s`, `-s`, `-es`, `-ed`, `-d`) inherit the breaks of their stem;
//! 2. **Liang patterns** (`assets/hyphen/en-us.pat`) for words not in the dictionary — our own
//! pattern set, trained from the same list by the patgen-style trainer behind the `generator`
//! feature (`examples/hyphgen.rs` regenerates both files);
//! 3. a small syllable **heuristic** for words outside the pattern alphabet.
//!
//! Then the paragraph's limits ([`Limits`]) are applied.
#[cfg(feature = "generator")]
pub mod dict;
#[cfg(not(feature = "generator"))]
mod dict;
#[cfg(feature = "generator")]
pub mod patgen;
mod patterns;
use std::sync::OnceLock;
pub use dict::Dictionary;
#[cfg(feature = "generator")]
pub use patterns::Patterns;
#[cfg(not(feature = "generator"))]
use patterns::Patterns;
static EN_US_DIC: &[u8] = include_bytes!("../../assets/hyphen/en-us.dic");
static EN_US_PAT: &str = include_str!("../../assets/hyphen/en-us.pat");
impl Dictionary {
/// The bundled US English dictionary (decoded on first use; empty if the data is corrupt).
pub fn en_us() -> &'static Dictionary {
static D: OnceLock<Dictionary> = OnceLock::new();
D.get_or_init(|| Dictionary::decode(EN_US_DIC).unwrap_or_default())
}
}
impl Patterns {
/// The bundled US English patterns (parsed on first use).
pub fn en_us() -> &'static Patterns {
static P: OnceLock<Patterns> = OnceLock::new();
P.get_or_init(|| Patterns::parse(EN_US_PAT))
}
}
/// Hyphenation limits from the paragraph's Hyphenation settings.
#[derive(Clone, Copy, Debug)]
pub struct Limits {
pub min_word: usize,
pub after_first: usize,
pub before_last: usize,
pub capitalized: bool,
}
impl Default for Limits {
fn default() -> Self {
Limits { min_word: 5, after_first: 2, before_last: 2, capitalized: true }
}
}
/// The hyphenator for a run of text.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Lang {
/// The bundled dictionary, patterns and heuristic.
English,
/// Patterns compiled into [`hypher`] (one cargo feature per language).
Patterns(PatternLang),
}
/// A language whose patterns are compiled into [`hypher`]; obtained from [`Lang::for_language`].
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct PatternLang(hypher::Lang);
/// English names of every language hypher can hyphenate, with their ISO 639-1 codes. Languages
/// whose cargo feature is off are not compiled into hypher, so `from_iso` returns None for them.
const LANGUAGES: &[(&str, [u8; 2])] = &[
("afrikaans", *b"af"),
("albanian", *b"sq"),
("assamese", *b"as"),
("belarusian", *b"be"),
("bengali", *b"bn"),
("bangla", *b"bn"),
("bulgarian", *b"bg"),
("catalan", *b"ca"),
("croatian", *b"hr"),
("czech", *b"cs"),
("danish", *b"da"),
("dutch", *b"nl"),
("estonian", *b"et"),
("finnish", *b"fi"),
("french", *b"fr"),
("galician", *b"gl"),
("georgian", *b"ka"),
("german", *b"de"),
("greek", *b"el"),
("gujarati", *b"gu"),
("hindi", *b"hi"),
("hungarian", *b"hu"),
("icelandic", *b"is"),
("italian", *b"it"),
("kannada", *b"kn"),
("kurmanji", *b"ku"),
("latin", *b"la"),
("lithuanian", *b"lt"),
("malayalam", *b"ml"),
("marathi", *b"mr"),
("mongolian", *b"mn"),
("norwegian", *b"no"),
("bokmål", *b"nb"),
("bokmal", *b"nb"),
("nynorsk", *b"nn"),
("oriya", *b"or"),
("odia", *b"or"),
("panjabi", *b"pa"),
("punjabi", *b"pa"),
("polish", *b"pl"),
("portuguese", *b"pt"),
("russian", *b"ru"),
("sanskrit", *b"sa"),
("serbian", *b"sr"),
("slovak", *b"sk"),
("slovenian", *b"sl"),
("slovene", *b"sl"),
("spanish", *b"es"),
("castilian", *b"es"),
("swedish", *b"sv"),
("tamil", *b"ta"),
("telugu", *b"te"),
("turkish", *b"tr"),
("turkmen", *b"tk"),
("ukrainian", *b"uk"),
];
impl Lang {
/// The hyphenator for a language name or locale code: an English name as InDesign and IDML
/// write it ("Spanish", "German: 2006 Reform", "$ID/Norwegian: Bokmal"), or a locale code
/// (`es`, `es_ES`, `pt-BR`, `de_DE_2006`). Case and punctuation don't matter. None when the
/// language is unknown or its cargo feature is off.
pub fn for_language(language: &str) -> Option<Lang> {
if language.starts_with("English") {
return Some(Lang::English);
}
let language = language.trim();
let language = language.strip_prefix("$ID/").unwrap_or(language);
let l = language.to_lowercase();
let words: Vec<&str> = l.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()).collect();
let first = *words.first()?;
let code = match locale_language(&words) {
Some((code, script)) => {
let code = <[u8; 2]>::try_from(code.as_bytes()).ok()?;
// Patterns for one script find nothing in another: `sr-Latn` has none.
if script.is_some_and(|script| script != pattern_script(code)) {
return None;
}
if code == *b"en" {
return Some(Lang::English);
}
code
}
None if first == "english" => return Some(Lang::English),
None => LANGUAGES.iter().find(|(name, _)| *name == first)?.1,
};
hypher::Lang::from_iso(code).map(|l| Lang::Patterns(PatternLang(l)))
}
}
/// The script of hypher's patterns for the language `code`, lowercase, as its documentation
/// lists them.
fn pattern_script(code: [u8; 2]) -> &'static str {
match &code {
b"as" | b"bn" => "beng",
b"be" | b"bg" | b"mn" | b"ru" | b"sr" | b"uk" => "cyrl",
b"hi" | b"mr" | b"sa" => "deva",
b"ka" => "geor",
b"el" => "grek",
b"gu" => "gujr",
b"pa" => "guru",
b"kn" => "knda",
b"ml" => "mlym",
b"or" => "orya",
b"ta" => "taml",
b"te" => "telu",
_ => "latn",
}
}
/// The language and script of a locale code split into lowercase `words`: a language (2 or 3
/// letters), then optionally a script (4 letters), a region (2 letters or 3 digits) and a reform
/// year (4 digits), in that order.
fn locale_language<'a>(words: &[&'a str]) -> Option<(&'a str, Option<&'a str>)> {
fn letters(w: &str, n: std::ops::RangeInclusive<usize>) -> bool {
n.contains(&w.len()) && w.bytes().all(|b| b.is_ascii_lowercase())
}
fn digits(w: &str, n: usize) -> bool {
w.len() == n && w.bytes().all(|b| b.is_ascii_digit())
}
let (&lang, mut rest) = words.split_first()?;
if !letters(lang, 2..=3) {
return None;
}
let script = rest.first().copied().filter(|w| letters(w, 4..=4));
if script.is_some() {
rest = &rest[1..];
}
let region = |w: &str| letters(w, 2..=2) || digits(w, 3);
let year = |w: &str| digits(w, 4);
for subtag in [region as fn(&str) -> bool, year] {
if let Some((&w, tail)) = rest.split_first()
&& subtag(w)
{
rest = tail;
}
}
rest.is_empty().then_some((lang, script))
}
/// Where a word's break points came from.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Source {
Dictionary,
Patterns,
Heuristic,
None,
}
fn is_apostrophe(c: char) -> bool {
matches!(c, '\'' | '’' | 'ʼ')
}
/// Letters, apostrophes, and the combining marks and joiners inside a word: a virama or a
/// decomposed accent doesn't split it.
fn is_word_char(c: char) -> bool {
c.is_alphabetic() || is_apostrophe(c) || is_mark(c) || is_joiner(c)
}
/// A combining mark (general category M): it belongs to the letter before it.
fn is_mark(c: char) -> bool {
use unicode_general_category::{GeneralCategory, get_general_category};
matches!(get_general_category(c), GeneralCategory::NonspacingMark | GeneralCategory::SpacingMark | GeneralCategory::EnclosingMark)
}
/// Zero-width joiner and non-joiner, which shape the letters on both sides.
fn is_joiner(c: char) -> bool {
matches!(c, '\u{200C}' | '\u{200D}')
}
/// Unrestricted break points of one lowercase core (letters and `'` only, no leading/trailing `'`).
fn core_breaks(core: &[char], lang: Lang) -> (Vec<usize>, Source) {
let n = core.len();
if n < 2 {
return (vec![], Source::None);
}
if let Lang::Patterns(PatternLang(l)) = lang {
// The whole core, apostrophes included: French, Italian and Catalan patterns encode
// elisions (`l'or-ga-ni-sa-tion`, `del-l'am-mi-ni-stra-zio-ne`).
let s: String = core.iter().collect();
let mut out = Vec::new();
let mut at = 0;
let syl: Vec<&str> = hypher::hyphenate_bounded(&s, l, 1, 1).collect();
if let Some((_, init)) = syl.split_last() {
for part in init {
at += part.chars().count();
out.push(at);
}
}
return (out, Source::Patterns);
}
let dict = Dictionary::en_us();
let s: String = core.iter().collect();
let from_mask = |m: u64, len: usize| (1..len.min(64)).filter(|&i| m & (1 << i) != 0).collect::<Vec<_>>();
if let Some(m) = dict.get(&s) {
return (from_mask(m, n), Source::Dictionary);
}
// Inflections inherit the stem's breaks (the suffix adds no break of its own).
for suf in ["'s", "s", "es", "ed", "d"] {
if let Some(stem) = s.strip_suffix(suf)
&& stem.chars().count() >= 2
&& let Some(m) = dict.get(stem)
{
return (from_mask(m, stem.chars().count()), Source::Dictionary);
}
}
let pats = Patterns::en_us();
if !pats.is_empty()
&& let Some(p) = pats.points(core)
{
return (p, Source::Patterns);
}
(heuristic_points(core), Source::Heuristic)
}
/// Unrestricted break points of `word` as byte offsets (a hyphen goes before the char that starts
/// there), handling case, apostrophes (`don't`, `’s`), surrounding punctuation and compounds (each
/// letter run is hyphenated separately). No break is ever placed next to an apostrophe or
/// non-letter, before a combining mark or next to a joiner; marks and joiners don't end a run.
pub fn word_breaks(word: &str) -> (Vec<usize>, Source) {
word_breaks_in(word, Lang::English)
}
/// [`word_breaks`] for a language.
pub fn word_breaks_in(word: &str, lang: Lang) -> (Vec<usize>, Source) {
let (pts, src) = char_breaks(word, lang);
(byte_offsets(word, pts), src)
}
/// Char indices into `word` as byte offsets.
fn byte_offsets(word: &str, pts: Vec<usize>) -> Vec<usize> {
let starts: Vec<usize> = word.char_indices().map(|(b, _)| b).collect();
pts.into_iter().filter_map(|p| starts.get(p).copied()).collect()
}
/// [`word_breaks_in`] as char indices.
fn char_breaks(word: &str, lang: Lang) -> (Vec<usize>, Source) {
let chars: Vec<char> = word.chars().collect();
let mut out = Vec::new();
let mut src = Source::None;
let mut i = 0;
while i < chars.len() {
if !is_word_char(chars[i]) {
i += 1;
continue;
}
let mut j = i;
while j < chars.len() && is_word_char(chars[j]) {
j += 1;
}
let (mut a, mut b) = (i, j);
while a < b && is_apostrophe(chars[a]) {
a += 1;
}
while b > a && is_apostrophe(chars[b - 1]) {
b -= 1;
}
let core: Vec<char> = chars[a..b].iter().map(|&c| if is_apostrophe(c) { '\'' } else { c.to_lowercase().next().unwrap_or(c) }).collect();
let (pts, s) = core_breaks(&core, lang);
if src == Source::None {
src = s;
}
// Never before a mark or next to a joiner: that would split a shaping cluster.
let allowed = |p: usize| match (core.get(p.wrapping_sub(1)), core.get(p)) {
(Some(&x), Some(&y)) => x != '\'' && y != '\'' && !is_mark(y) && !is_joiner(x) && !is_joiner(y),
_ => false,
};
out.extend(pts.into_iter().filter(|&p| p > 0 && allowed(p)).map(|p| a + p));
i = j;
}
(out, src)
}
/// Allowed hyphenation points in `word` as byte offsets (a hyphen goes before the char that
/// starts there), with the paragraph limits applied to each letter run.
pub fn hyphen_points(word: &str, lim: &Limits) -> Vec<usize> {
hyphen_points_in(word, lim, Lang::English)
}
/// [`hyphen_points`] for a language.
pub fn hyphen_points_in(word: &str, lim: &Limits, lang: Lang) -> Vec<usize> {
byte_offsets(word, char_points(word, lim, lang))
}
/// [`hyphen_points_in`] as char indices.
fn char_points(word: &str, lim: &Limits, lang: Lang) -> Vec<usize> {
let chars: Vec<char> = word.chars().collect();
let (pts, _) = char_breaks(word, lang);
if pts.is_empty() {
return pts;
}
let min_before = lim.after_first.max(1);
let min_after = lim.before_last.max(1);
// Counted once, so each point's checks take constant time however long the word: letters and
// uppercase letters before each char, and the letter run (trimmed of apostrophes) of each char.
let (mut letters, mut upper) = (Vec::with_capacity(chars.len() + 1), Vec::with_capacity(chars.len() + 1));
let (mut l, mut u) = (0usize, 0usize);
for c in &chars {
letters.push(l);
upper.push(u);
if c.is_alphabetic() {
l += 1;
u += usize::from(c.is_uppercase());
}
}
letters.push(l);
upper.push(u);
let mut run_of: Vec<(usize, usize)> = vec![(0, 0); chars.len()];
let mut i = 0;
while i < chars.len() {
let mut j = i;
while chars.get(j).is_some_and(|c| is_word_char(*c)) {
j += 1;
}
let (mut a, mut b) = (i, j);
while a < b && chars.get(a).is_some_and(|c| is_apostrophe(*c)) {
a += 1;
}
while b > a && chars.get(b - 1).is_some_and(|c| is_apostrophe(*c)) {
b -= 1;
}
for r in run_of.get_mut(i..j).into_iter().flatten() {
*r = (a, b);
}
i = j.max(i + 1);
}
let count = |v: &[usize], a: usize, b: usize| v.get(b).zip(v.get(a)).map_or(0, |(y, x)| y.saturating_sub(*x));
pts.into_iter()
.filter(|&p| {
// A point lies between two letters of one run; the char after it gives the run.
let Some(&(a, b)) = run_of.get(p) else { return false };
let n = count(&letters, a, b);
if n < lim.min_word.max(2) {
return false;
}
if n > 1 && count(&upper, a, b) == n {
return false;
}
if !lim.capitalized && chars.get(a).is_some_and(|c| c.is_uppercase()) {
return false;
}
count(&letters, a, p) >= min_before && count(&letters, p, b) >= min_after
})
.collect()
}
/// `word` with `-` at every hyphenation point (tests and diagnostics).
pub fn hyphenate_word(word: &str, lim: &Limits) -> String {
let pts = hyphen_points(word, lim);
let mut s = String::with_capacity(word.len() + pts.len());
for (i, ch) in word.char_indices() {
if pts.contains(&i) {
s.push('-');
}
s.push(ch);
}
s
}
// ---------- heuristic fallback (words outside the pattern alphabet) ----------
fn is_vowel(c: char) -> bool {
matches!(c.to_ascii_lowercase(), 'a' | 'e' | 'i' | 'o' | 'u' | 'y') || (c.is_alphabetic() && !c.is_ascii())
}
fn onset_pair(a: char, b: char) -> bool {
let (a, b) = (a.to_ascii_lowercase(), b.to_ascii_lowercase());
matches!(
(a, b),
('c' | 's' | 't' | 'p' | 'w' | 'g', 'h')
| ('b' | 'c' | 'd' | 'f' | 'g' | 'k' | 'p' | 't', 'r')
| ('b' | 'c' | 'f' | 'g' | 'k' | 'p' | 's', 'l')
| ('q', 'u')
| ('s', 't' | 'p' | 'c' | 'k')
)
}
/// Syllable heuristic: V-CV ("ty-po"), VC-CV ("hap-pen"), keeping onset clusters ("gra-phy").
fn heuristic_points(c: &[char]) -> Vec<usize> {
let n = c.len();
if n < 4 || !c.iter().all(|c| c.is_alphabetic()) {
return vec![];
}
let v: Vec<bool> = c.iter().map(|&x| is_vowel(x)).collect();
// Both parts need a sounded vowel: a trailing silent `e`, `ed` or `es` doesn't count. The part
// after `i` is `c[i..end]`, `end` before that ending; computed once, so each point is O(1).
let end = |i: usize| {
if n - i >= 2 && (c.ends_with(&['e', 'd']) || c.ends_with(&['e', 's'])) {
n - 2
} else if c.ends_with(&['e']) {
n - 1
} else {
n
}
};
// UTF-8 bytes before each char, and the first vowel at or after each char.
let mut bytes = Vec::with_capacity(n + 1);
let mut acc = 0usize;
for ch in c {
bytes.push(acc);
acc += ch.len_utf8();
}
bytes.push(acc);
let mut next_vowel = vec![n; n + 1];
for i in (0..n).rev() {
next_vowel[i] = if v[i] { i } else { next_vowel[i + 1] };
}
let mut out = vec![];
let mut vowel_before = false;
for i in 1..n {
vowel_before |= v[i - 1];
let ok = if v[i - 1] && !v[i] {
(i + 1 < n && v[i + 1]) || (i + 2 < n && onset_pair(c[i], c[i + 1]) && v[i + 2])
} else if !v[i - 1] && !v[i] {
i >= 2 && v[i - 2] && i + 1 < n && v[i + 1] && !onset_pair(c[i - 1], c[i]) && c[i - 1] != c[i] || (c[i - 1] == c[i] && i >= 2 && v[i - 2])
} else {
false
};
let e = end(i);
if ok && next_vowel[i] < e && bytes[e] - bytes[i] > 1 && vowel_before {
out.push(i);
}
}
out
}
#[cfg(test)]
mod tests;
crates/text/src/hyphen/patgen.rs
+251
View File
@@ -0,0 +1,251 @@
//! A small patgen-style trainer (ours): learns Liang hyphenation patterns from a hyphenated word
//! list, level by level. Odd levels add hyphenating patterns, even levels add inhibiting ones.
//!
//! At level `L`, for every pattern length, each inter-letter position whose current value is
//! below `L` is classified as *good* (applying value `L` would fix it) or *bad* (it would break a
//! position that is currently right). A candidate (letters + value position) is selected when
//! `good·good_weight − bad·bad_weight ≥ threshold`; selected patterns are applied to every word
//! before the next length is considered. Used offline by `examples/hyphgen.rs`.
use super::patterns::{Alphabet, FxMap, MAX_PATTERN_LEN, Patterns, apply, pack};
/// Parameters of one level.
#[derive(Clone, Copy, Debug)]
pub struct Level {
pub min_len: usize,
pub max_len: usize,
pub good_weight: u32,
pub bad_weight: u32,
pub threshold: u32,
}
/// Defaults tuned on the Moby list (see `examples/hyphgen.rs`).
pub const DEFAULT_LEVELS: &[Level] = &[
Level { min_len: 1, max_len: 4, good_weight: 1, bad_weight: 2, threshold: 20 },
Level { min_len: 2, max_len: 5, good_weight: 2, bad_weight: 1, threshold: 8 },
Level { min_len: 3, max_len: 6, good_weight: 1, bad_weight: 4, threshold: 7 },
Level { min_len: 3, max_len: 7, good_weight: 3, bad_weight: 2, threshold: 2 },
Level { min_len: 4, max_len: 8, good_weight: 1, bad_weight: 20, threshold: 6 },
Level { min_len: 4, max_len: 8, good_weight: 3, bad_weight: 1, threshold: 2 },
];
struct Word {
/// Padded codes: boundary, letters…, boundary.
codes: Vec<u8>,
/// Correct hyphen before padded char `p`.
hyph: Vec<bool>,
vals: Vec<u8>,
}
/// Accuracy of a pattern set over a word list.
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub struct Score {
pub words: usize,
pub breaks: usize,
pub found: usize,
pub wrong: usize,
}
impl Score {
pub fn missed(&self) -> usize {
self.breaks.saturating_sub(self.found)
}
}
/// Train patterns from `(lowercase word, break char indices)` pairs. `log` receives progress lines.
pub fn train(words: &[(Vec<char>, Vec<usize>)], levels: &[Level], log: &mut dyn FnMut(String)) -> Patterns {
let mut alpha = Alphabet::default();
let mut data: Vec<Word> = Vec::with_capacity(words.len());
'w: for (w, br) in words {
let mut codes = Vec::with_capacity(w.len() + 2);
codes.push(1);
for &c in w {
let Some(k) = alpha.intern(c) else { continue 'w };
codes.push(k);
}
codes.push(1);
let mut hyph = vec![false; codes.len() + 1];
for &b in br {
if b > 0 && b < w.len() {
hyph[b + 1] = true;
}
}
let vals = vec![0; codes.len() + 1];
data.push(Word { codes, hyph, vals });
}
let mut pats: FxMap<u64, Box<[u8]>> = FxMap::default();
let mut max_len = 0usize;
for (li, lv) in levels.iter().enumerate() {
let level = (li + 1) as u8;
let hyphenating = level % 2 == 1;
for len in lv.min_len..=lv.max_len.min(MAX_PATTERN_LEN) {
// (packed letters | dot) → (good, bad)
let mut counts: FxMap<u64, (u32, u32)> = FxMap::default();
for w in &data {
let n = w.codes.len();
// Positions strictly between the word's letters.
for p in 2..n - 1 {
let v = w.vals[p];
if v >= level {
continue;
}
let (good, bad) = if hyphenating {
(w.hyph[p] && v % 2 == 0, !w.hyph[p] && v % 2 == 0)
} else {
(!w.hyph[p] && v % 2 == 1, w.hyph[p] && v % 2 == 1)
};
if !good && !bad {
continue;
}
for dot in 0..=len {
if p < dot || p - dot + len > n {
continue;
}
let s = p - dot;
let key = pack(&w.codes[s..s + len]) | ((dot as u64) << 54);
let e = counts.entry(key).or_default();
if good {
e.0 += 1;
} else {
e.1 += 1;
}
}
}
}
// Select.
let mut new: FxMap<u64, Box<[u8]>> = FxMap::default();
let mut selected = 0usize;
for (key, (g, b)) in counts {
// Widened first: u32 × u32 fills a u64, and the difference needs a sign.
let score = i128::from(g) * i128::from(lv.good_weight) - i128::from(b) * i128::from(lv.bad_weight);
if g > 0 && score >= i128::from(lv.threshold) {
let dot = ((key >> 54) & 15) as usize;
let letters = key & !(15u64 << 54);
let e = new.entry(letters).or_insert_with(|| vec![0u8; len + 1].into_boxed_slice());
e[dot] = level;
selected += 1;
}
}
if selected == 0 {
continue;
}
for w in &mut data {
apply(&new, len, &w.codes, &mut w.vals);
}
for (k, v) in new {
let e = pats.entry(k).or_insert_with(|| vec![0u8; len + 1].into_boxed_slice());
for (a, b) in e.iter_mut().zip(v.iter()) {
*a = (*a).max(*b);
}
}
max_len = max_len.max(len);
let sc = score_vals(&data);
log(format!("level {level} len {len}: +{selected} patterns ({} total); found {}/{} wrong {}", pats.len(), sc.found, sc.breaks, sc.wrong));
}
}
// Re-express through the public pattern type (TeX strings) so the alphabet matches at runtime.
let tmp = Patterns::from_parts(alpha, pats, max_len);
let mut out = Patterns::default();
out.add_letters(words.iter().flat_map(|w| w.0.iter().copied()));
for t in tmp.to_tex() {
out.insert(&t);
}
out
}
fn score_vals(data: &[Word]) -> Score {
let mut s = Score { words: data.len(), ..Default::default() };
for w in data {
for p in 2..w.codes.len() - 1 {
let on = w.vals[p] % 2 == 1;
if w.hyph[p] {
s.breaks += 1;
if on {
s.found += 1;
}
} else if on {
s.wrong += 1;
}
}
}
s
}
/// Score `pats` against a word list.
pub fn score(pats: &Patterns, words: &[(Vec<char>, Vec<usize>)]) -> Score {
let mut s = Score::default();
for (w, br) in words {
let Some(pts) = pats.points(w) else { continue };
s.words += 1;
s.breaks += br.len();
for p in &pts {
if br.contains(p) {
s.found += 1;
} else {
s.wrong += 1;
}
}
}
s
}
#[cfg(test)]
mod tests {
use super::*;
fn w(s: &str) -> (Vec<char>, Vec<usize>) {
let mut chars = Vec::new();
let mut br = Vec::new();
for c in s.chars() {
if c == '-' {
br.push(chars.len());
} else {
chars.push(c);
}
}
(chars, br)
}
/// Weights near `u32::MAX` select the pattern instead of overflowing.
#[test]
fn large_weights_do_not_overflow() {
let words = vec![(vec!['a', 'b'], vec![1]); 2];
let levels = [Level { min_len: 1, max_len: 1, good_weight: 1 << 31, bad_weight: 1, threshold: 1 }];
let patterns = train(&words, &levels, &mut |_| {});
assert_eq!(patterns.points(&['a', 'b']), Some(vec![1]));
}
#[test]
fn learns_a_tiny_list() {
let list: Vec<_> = [
"hap-pen",
"hap-pens",
"hap-pened",
"pen-cil",
"pen-cils",
"rab-bit",
"rab-bits",
"kit-ten",
"kit-tens",
"but-ter",
"bet-ter",
"let-ter",
"lit-ter",
"sit-ter",
]
.iter()
.map(|s| w(s))
.collect();
let levels = [
Level { min_len: 1, max_len: 4, good_weight: 1, bad_weight: 1, threshold: 2 },
Level { min_len: 1, max_len: 5, good_weight: 1, bad_weight: 1, threshold: 1 },
];
let p = train(&list, &levels, &mut |_| {});
let sc = score(&p, &list);
assert_eq!(sc.breaks, list.len());
assert!(sc.found >= sc.breaks - 1, "{sc:?}");
assert!(sc.wrong <= 1, "{sc:?}");
// Generalises to an unseen word with the same shape.
assert_eq!(p.points(&"bitter".chars().collect::<Vec<_>>()).unwrap(), vec![3]);
}
}
crates/text/src/hyphen/patterns.rs
+241
View File
@@ -0,0 +1,241 @@
//! Liang hyphenation patterns (TeX notation, e.g. `.ab1c`, `4t1ion`): parsing and application.
//!
//! The bundled English set is *ours*: trained from the public-domain Moby Hyphenator word list by
//! the crate's patgen-style trainer (see `examples/hyphgen.rs`). Patterns are stored in a hash map keyed by the
//! packed letter string, so applying them to a word costs `O(len × max_pattern_len)` lookups.
use std::collections::HashMap;
use std::hash::{BuildHasherDefault, Hasher};
/// Word-boundary marker in patterns.
pub(crate) const DOT: char = '.';
/// Longest pattern the packed key supports.
pub(crate) const MAX_PATTERN_LEN: usize = 9;
/// Small, fast multiplicative hasher for packed `u64` keys (FxHash-style).
#[derive(Default, Clone, Copy)]
pub(crate) struct FxHasher(u64);
impl Hasher for FxHasher {
fn finish(&self) -> u64 {
self.0
}
fn write(&mut self, bytes: &[u8]) {
for &b in bytes {
self.write_u64(b as u64);
}
}
fn write_u64(&mut self, i: u64) {
self.0 = (self.0.rotate_left(5) ^ i).wrapping_mul(0x51_7c_c1_b7_27_22_0a_95);
}
}
pub(crate) type FxMap<K, V> = HashMap<K, V, BuildHasherDefault<FxHasher>>;
/// Maps characters to 6-bit codes (1 = word boundary, 2.. = letters); 0 = not in the alphabet.
#[derive(Clone, Debug)]
pub(crate) struct Alphabet {
ascii: [u8; 128],
other: Vec<(char, u8)>,
next: u8,
}
impl Default for Alphabet {
fn default() -> Self {
let mut a = Alphabet { ascii: [0; 128], other: Vec::new(), next: 2 };
a.ascii[DOT as usize] = 1;
a
}
}
impl Alphabet {
pub(crate) fn code(&self, c: char) -> u8 {
if (c as u32) < 128 { self.ascii[c as usize] } else { self.other.iter().find(|x| x.0 == c).map_or(0, |x| x.1) }
}
/// Code for `c`, adding it if new (None when the 6-bit alphabet is full).
pub(crate) fn intern(&mut self, c: char) -> Option<u8> {
let k = self.code(c);
if k != 0 {
return Some(k);
}
if self.next >= 63 {
return None;
}
let k = self.next;
self.next += 1;
if (c as u32) < 128 {
self.ascii[c as usize] = k;
} else {
self.other.push((c, k));
}
Some(k)
}
}
/// Packed key of a code string (≤ [`MAX_PATTERN_LEN`] codes).
#[inline]
pub(crate) fn pack(codes: &[u8]) -> u64 {
let mut k = 0u64;
for &c in codes {
k = (k << 6) | c as u64;
}
k | ((codes.len() as u64) << 58)
}
/// A compiled pattern set.
#[derive(Clone, Debug, Default)]
pub struct Patterns {
alpha: Alphabet,
/// Packed letters → inter-letter values (`len + 1` entries).
map: FxMap<u64, Box<[u8]>>,
max_len: usize,
}
impl Patterns {
/// Parse TeX-style patterns: whitespace-separated, `%`/`#` start a comment line.
pub fn parse(text: &str) -> Patterns {
let mut p = Patterns::default();
for line in text.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('%') || line.starts_with('#') {
continue;
}
for tok in line.split_whitespace() {
p.insert(tok);
}
}
p
}
/// Add one pattern like `a1b`; returns false if it can't be represented.
pub(crate) fn insert(&mut self, pat: &str) -> bool {
let mut codes = Vec::with_capacity(pat.len());
let mut vals = vec![0u8];
for c in pat.chars() {
if let Some(d) = c.to_digit(10) {
*vals.last_mut().unwrap_or(&mut 0) = d as u8;
} else {
let Some(k) = self.alpha.intern(c) else { return false };
codes.push(k);
vals.push(0);
}
}
if codes.is_empty() || codes.len() > MAX_PATTERN_LEN {
return false;
}
self.max_len = self.max_len.max(codes.len());
self.map.insert(pack(&codes), vals.into_boxed_slice());
true
}
#[cfg(feature = "generator")]
pub(crate) fn from_parts(alpha: Alphabet, map: FxMap<u64, Box<[u8]>>, max_len: usize) -> Patterns {
Patterns { alpha, map, max_len }
}
/// Make `letters` part of the alphabet even if no pattern mentions them.
#[cfg(feature = "generator")]
pub(crate) fn add_letters(&mut self, letters: impl IntoIterator<Item = char>) {
for c in letters {
let _ = self.alpha.intern(c);
}
}
#[cfg(any(test, feature = "generator"))]
pub fn len(&self) -> usize {
self.map.len()
}
pub fn is_empty(&self) -> bool {
self.map.is_empty()
}
/// Inter-letter values for a lowercase word: `out[i]` is the value *before* char `i`
/// (`out.len() == word.len() + 1`). None if a char is outside the pattern alphabet.
pub(crate) fn values(&self, word: &[char]) -> Option<Vec<u8>> {
let mut codes = Vec::with_capacity(word.len() + 2);
codes.push(1);
for &c in word {
let k = self.alpha.code(c);
if k == 0 {
return None;
}
codes.push(k);
}
codes.push(1);
let mut vals = vec![0u8; codes.len() + 1];
apply(&self.map, self.max_len, &codes, &mut vals);
// Padded position p sits before padded char p; word char i is padded char i + 1.
Some(vals[1..codes.len()].to_vec())
}
/// Break points (char indices, a hyphen goes before that char) of a lowercase word.
pub fn points(&self, word: &[char]) -> Option<Vec<usize>> {
let v = self.values(word)?;
Some((1..word.len()).filter(|&i| v[i] % 2 == 1).collect())
}
/// The patterns in TeX notation (sorted by letters), e.g. for writing a pattern file.
#[cfg(any(test, feature = "generator"))]
pub fn to_tex(&self) -> Vec<String> {
let mut rev: Vec<(u8, char)> =
(0u8..128).filter(|&c| self.alpha.ascii[c as usize] != 0).map(|c| (self.alpha.ascii[c as usize], c as char)).collect();
rev.extend(self.alpha.other.iter().map(|&(c, k)| (k, c)));
let ch = |k: u8| rev.iter().find(|x| x.0 == k).map_or('?', |x| x.1);
let mut out: Vec<(String, String)> = self
.map
.iter()
.map(|(&key, vals)| {
let n = (key >> 58) as usize;
let letters: Vec<char> = (0..n).map(|i| ch(((key >> (6 * (n - 1 - i))) & 63) as u8)).collect();
let mut s = String::new();
for (i, l) in letters.iter().enumerate() {
if vals[i] > 0 {
s.push(char::from(b'0' + vals[i]));
}
s.push(*l);
}
if vals[n] > 0 {
s.push(char::from(b'0' + vals[n]));
}
(letters.into_iter().collect(), s)
})
.collect();
out.sort();
out.into_iter().map(|x| x.1).collect()
}
}
/// Apply `map` to padded `codes`, raising `vals` (len = codes.len() + 1).
#[inline]
pub(crate) fn apply(map: &FxMap<u64, Box<[u8]>>, max_len: usize, codes: &[u8], vals: &mut [u8]) {
let n = codes.len();
for s in 0..n {
let mut k = 0u64;
for len in 1..=max_len.min(n - s) {
k = (k << 6) | codes[s + len - 1] as u64;
if let Some(v) = map.get(&(k | ((len as u64) << 58))) {
for (slot, &x) in vals.iter_mut().skip(s).zip(v.iter()) {
*slot = (*slot).max(x);
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn liang_example() {
// The classic example from Liang's thesis: "hy-phen-ation".
let p = Patterns::parse("hy3ph he2n hena4 hen5at 1na n2at 1tio 2io o2n");
let w: Vec<char> = "hyphenation".chars().collect();
assert_eq!(p.points(&w).unwrap(), vec![2, 6]);
let mut tex = p.to_tex();
tex.sort();
assert!(tex.contains(&"hen5at".to_string()));
assert!(p.values(&['h', 'é']).is_none());
}
}
crates/text/src/hyphen/tests.rs
+364
View File
@@ -0,0 +1,364 @@
use super::*;
fn h(w: &str) -> String {
hyphenate_word(w, &Limits::default())
}
#[test]
fn bundled_data_loads() {
assert!(Dictionary::en_us().len() > 150_000, "{}", Dictionary::en_us().len());
assert!(Patterns::en_us().len() > 1000, "{}", Patterns::en_us().len());
}
#[test]
fn dictionary_words() {
assert_eq!(h("happen"), "hap-pen");
assert_eq!(h("typography"), "ty-pog-ra-phy");
assert_eq!(h("composition"), "com-po-si-tion");
assert_eq!(h("hyphenation"), "hy-phen-a-tion");
assert_eq!(h("international"), "in-ter-na-tion-al");
assert_eq!(h("cat"), "cat");
assert!(!h("changed").contains('-'));
assert_eq!(h("NASA"), "NASA");
assert_eq!(word_breaks("publication").1, Source::Dictionary);
}
#[test]
fn case_apostrophes_and_punctuation() {
assert_eq!(h("Typography"), "Ty-pog-ra-phy");
assert_eq!(h("TYPOGRAPHY"), "TYPOGRAPHY");
assert_eq!(h("typography,"), "ty-pog-ra-phy,");
assert_eq!(h("“typography”"), "“ty-pog-ra-phy”");
assert_eq!(h("typography's"), "ty-pog-ra-phy's");
assert_eq!(h("typography’s"), "ty-pog-ra-phy’s");
assert_eq!(h("'typography'"), "'ty-pog-ra-phy'");
// Compounds: each part separately, never next to the hyphen.
assert_eq!(h("typography-composition"), "ty-pog-ra-phy-com-po-si-tion");
// Points are byte offsets: the opening quote takes three bytes.
assert_eq!(hyphen_points("“typography”", &Limits::default()), vec![5, 8, 10]);
}
#[test]
fn inflections_inherit_the_stem() {
assert_eq!(h("ripples"), "rip-ples");
assert_eq!(h("gradients"), "gra-di-ents");
assert_eq!(h("restrained"), "re-strained");
}
#[test]
fn patterns_cover_unknown_words() {
// Not in the word list: the trained patterns decide.
let (pts, src) = word_breaks("hyperlocalization");
assert_eq!(src, Source::Patterns);
assert!(pts.len() >= 4, "{}", h("hyperlocalization"));
let (_, src) = word_breaks("zzyzx");
assert_ne!(src, Source::Dictionary);
}
#[test]
fn limits_are_respected() {
let lim = Limits { min_word: 5, after_first: 3, before_last: 3, capitalized: false };
for w in ["typography", "hyphenation", "justification", "Paragraph", "beautiful"] {
for p in hyphen_points(w, &lim) {
assert!(p >= 3 && p <= w.chars().count() - 3, "{w} {p}");
}
}
assert!(hyphen_points("Paragraph", &lim).is_empty());
let lim = Limits { min_word: 12, ..Limits::default() };
assert!(hyphen_points("typography", &lim).is_empty());
}
#[test]
fn language_names_and_codes() {
for name in ["English", "English: USA", "english", "$ID/English: UK", "en", "en_US", "en-GB", "EN", "en-Latn-US"] {
assert_eq!(Lang::for_language(name), Some(Lang::English), "{name}");
}
assert_eq!(Lang::for_language("en-Cyrl"), None);
// Each language resolves by name and by code exactly when its feature is on.
for &(name, code) in LANGUAGES {
let want = hypher::Lang::from_iso(code).map(|l| Lang::Patterns(PatternLang(l)));
let code = std::str::from_utf8(&code).unwrap();
let mut title = name.chars();
let title: String = title.next().map(|c| c.to_uppercase().chain(title).collect()).unwrap_or_default();
let upper = code.to_uppercase();
let script = pattern_script(code.as_bytes().try_into().unwrap());
for spelling in [
name.to_string(),
format!("{title}: Reform"),
format!("$ID/{title}"),
code.to_string(),
format!("{code}_{upper}"),
format!("{code}-419"),
format!("{code}_{upper}_2006"),
format!("{code}-{script}"),
format!("{code}-{script}-{upper}"),
format!("{code}-1996"),
format!("{code}-{script}-1996"),
] {
assert_eq!(Lang::for_language(&spelling), want, "{spelling}");
}
let other = if script == "latn" { "Cyrl" } else { "Latn" };
assert_eq!(Lang::for_language(&format!("{code}-{other}-{upper}")), None, "{code}: patterns of another script");
assert_eq!(Lang::for_language(&format!("{code}_{upper}_x")), None, "{code}: not a locale code");
}
for unknown in ["", "[No Language]", "Klingon", "zh_CN", "Chinese: Simplified", "tlh"] {
assert_eq!(Lang::for_language(unknown), None, "{unknown}");
}
}
#[cfg(feature = "all-languages")]
#[test]
fn every_language_is_available() {
for &(name, code) in LANGUAGES {
if code == *b"sk" && !cfg!(feature = "slovak-gpl") {
continue;
}
assert!(hypher::Lang::from_iso(code).is_some(), "{name}");
assert!(Lang::for_language(name).is_some(), "{name}");
}
}
/// A virama doesn't split a word: the limits count the whole word, and no point falls before a
/// mark or next to a joiner.
#[cfg(feature = "hindi")]
#[test]
fn indic_word_limits_span_viramas() {
let hi = Lang::for_language("hi").unwrap();
// न म स ् क ा र: unrestricted points before म, स and र; only स has two letters on each side.
assert_eq!(word_breaks_in("नमस्कार", hi).0, vec![3, 6, 18]);
assert_eq!(hyphen_points_in("नमस्कार", &Limits::default(), hi), vec![6]);
for w in ["नमस्कार", "हिन्दुस्तानी", "प्रौद्योगिकी", "विश्वविद्यालय", "क्ष\u{200D}त्रिय"]
{
for p in word_breaks_in(w, hi).0 {
let (before, after) = w.split_at(p);
let (Some(x), Some(y)) = (before.chars().next_back(), after.chars().next()) else { panic!("{w} at {p}") };
assert!(!is_mark(y) && !is_joiner(x) && !is_joiner(y), "{w} at {p}");
}
}
}
/// Slovak's patterns are GPL-2.0-or-later: only their own feature turns them on, never
/// `all-languages`.
#[test]
fn all_languages_leaves_out_the_gpl_slovak_patterns() {
let manifest = include_str!("../../Cargo.toml");
let all = manifest.split("all-languages = [").nth(1).and_then(|s| s.split(']').next()).unwrap_or_default();
assert!(all.contains("\"spanish\""), "all-languages list not found");
assert!(!all.contains("slovak"), "{all}");
assert!(manifest.contains("\nslovak-gpl = [\"hyphen\", \"hypher/slovak\"]"));
}
#[cfg(feature = "spanish")]
#[test]
fn spanish_uses_the_bezos_patterns() {
let es = Lang::for_language("Spanish").unwrap();
assert_eq!(es, Lang::for_language("es_ES").unwrap());
let h = |w: &str| {
let pts = hyphen_points_in(w, &Limits::default(), es);
assert!(pts.iter().all(|&p| w.is_char_boundary(p)), "{w} {pts:?}");
w.char_indices().map(|(i, c)| if pts.contains(&i) { format!("-{c}") } else { c.to_string() }).collect::<String>()
};
// Offsets are bytes: `ú` and `ó` take two each.
assert_eq!(hyphen_points_in("música", &Limits::default(), es), vec![3, 5]);
assert_eq!(hyphen_points_in("ejército", &Limits::default(), es), vec![5, 7]);
assert_eq!(h("transición"), "tran-si-ción");
assert_eq!(h("pingüino"), "pin-güino");
assert_eq!(h("Construcción"), "Cons-truc-ción");
assert_eq!(h("desesperación"), "de-ses-pe-ra-ción");
assert_eq!(h("teatro"), "tea-tro", "a vowel run stays whole");
assert_eq!(h("calle"), "ca-lle", "ll stays whole");
#[cfg(not(feature = "french"))]
assert!(Lang::for_language("French").is_none(), "only languages built in");
}
/// An elided article or preposition keeps the breaks of the word after it, and no break falls
/// next to the apostrophe.
#[cfg(feature = "french")]
#[test]
fn french_elisions_keep_their_breaks() {
let fr = Lang::for_language("French").unwrap();
let h = |w: &str| {
let pts = hyphen_points_in(w, &Limits::default(), fr);
w.char_indices().map(|(i, c)| if pts.contains(&i) { format!("-{c}") } else { c.to_string() }).collect::<String>()
};
assert_eq!(h("organisation"), "or-ga-ni-sa-tion");
assert_eq!(h("l'organisation"), "l'or-ga-ni-sa-tion");
assert_eq!(h("L’organisation"), "L’or-ga-ni-sa-tion");
assert_eq!(h("d'information"), "d'in-for-ma-tion");
assert_eq!(h("aujourd'hui"), "au-jour-d'hui");
}
/// The fastest of three timings of `f`, after a warm-up.
fn fastest(f: impl Fn()) -> f64 {
f();
(0..3)
.map(|_| {
let t = std::time::Instant::now();
f();
t.elapsed().as_secs_f64()
})
.fold(f64::INFINITY, f64::min)
}
/// `hyphen_points`' limit checks as first written (a rescan of the letter run per point): the
/// reference for the linear version.
fn hyphen_points_ref(word: &str, lim: &Limits) -> Vec<usize> {
let chars: Vec<char> = word.chars().collect();
let (pts, _) = char_breaks(word, Lang::English);
let min_before = lim.after_first.max(1);
let min_after = lim.before_last.max(1);
let run_of = |p: usize| {
let mut a = p;
while a > 0 && is_word_char(chars[a - 1]) {
a -= 1;
}
let mut b = p;
while b < chars.len() && is_word_char(chars[b]) {
b += 1;
}
while a < b && is_apostrophe(chars[a]) {
a += 1;
}
while b > a && is_apostrophe(chars[b - 1]) {
b -= 1;
}
(a, b)
};
pts.into_iter()
.filter(|&p| {
let (a, b) = run_of(p);
let run = &chars[a..b];
let letters = run.iter().filter(|c| c.is_alphabetic()).count();
if letters < lim.min_word.max(2) {
return false;
}
if letters > 1 && run.iter().filter(|c| c.is_alphabetic()).all(|c| c.is_uppercase()) {
return false;
}
if !lim.capitalized && run[0].is_uppercase() {
return false;
}
let before = chars[a..p].iter().filter(|c| c.is_alphabetic()).count();
let after = chars[p..b].iter().filter(|c| c.is_alphabetic()).count();
before >= min_before && after >= min_after
})
.collect()
}
/// `heuristic_points` as first written (the right part rebuilt per point): the reference.
fn heuristic_ref(c: &[char]) -> Vec<usize> {
let n = c.len();
if n < 4 || !c.iter().all(|c| c.is_alphabetic()) {
return vec![];
}
let v: Vec<bool> = c.iter().map(|&x| is_vowel(x)).collect();
let mut out = vec![];
for i in 1..n {
let ok = if v[i - 1] && !v[i] {
(i + 1 < n && v[i + 1]) || (i + 2 < n && onset_pair(c[i], c[i + 1]) && v[i + 2])
} else if !v[i - 1] && !v[i] {
i >= 2 && v[i - 2] && i + 1 < n && v[i + 1] && !onset_pair(c[i - 1], c[i]) && c[i - 1] != c[i] || (c[i - 1] == c[i] && i >= 2 && v[i - 2])
} else {
false
};
let right: String = c[i..].iter().collect();
let core = right.strip_suffix("ed").or_else(|| right.strip_suffix("es")).or_else(|| right.strip_suffix('e')).unwrap_or(&right);
if ok && core.chars().any(is_vowel) && core.len() > 1 && v[..i].iter().any(|x| *x) {
out.push(i);
}
}
out
}
#[test]
fn linear_limits_and_heuristic_match_the_reference() {
let long = "hyphenation".repeat(40);
let words = [
"",
"x",
"''",
"a'b'c",
"cat",
"happen",
"hyphenation",
"Typography's",
"TYPOGRAPHY",
"Typography",
"typography’s",
"'typography'",
"“typography”,",
"typography-composition",
"don't",
"rock'n'roll",
"naïveté",
"crème-brûlée",
"Ünïcödé",
"ÉTUDE",
"strengths",
"ee",
"bed",
"hopped",
"rhythmed",
"aeiouy",
"xxyyzz",
"bookkeeper",
"ʼtwas",
"mid-1990s",
"twenty-one",
"école",
"Ålesund",
"smørrebrød",
"cafe\u{301}-cre\u{300}me",
"e\u{301}tude",
"नमस्कार",
"hopes",
"hoped",
"taed",
"teeed",
&long,
];
for w in words {
for min_word in [2, 5, 7] {
for (after_first, before_last) in [(1, 1), (2, 2), (3, 2), (2, 4)] {
for capitalized in [true, false] {
let lim = Limits { min_word, after_first, before_last, capitalized };
assert_eq!(hyphen_points(w, &lim), byte_offsets(w, hyphen_points_ref(w, &lim)), "{w:?} {lim:?}");
}
}
}
let c: Vec<char> = w.to_lowercase().chars().collect();
assert_eq!(heuristic_points(&c), heuristic_ref(&c), "{w:?}");
}
}
#[test]
#[ignore = "wall-clock timing; run with --ignored"]
fn hyphenating_a_long_word_is_linear() {
// A word of n letters against one of 4n: linear ~4x the time, the old rescans ~16x. One in
// the pattern alphabet, one outside it (the syllable heuristic).
let pattern = |k: usize| "hyphenation".repeat(k);
let heuristic = |k: usize| "smørrebrød".repeat(k);
assert_eq!(word_breaks(&heuristic(2)).1, Source::Heuristic);
// Each timing repeats the call so the shorter one is milliseconds, well above timer noise.
let ratios: Vec<(String, f64)> = [(pattern as fn(usize) -> String, 20), (heuristic, 40)]
.into_iter()
.map(|(word, repeat)| {
let (small, big) = (word(400), word(1600));
let time = |w: &str| fastest(|| (0..repeat).for_each(|_| drop(hyphen_points(w, &Limits::default()))));
let (t1, t4) = (time(&small), time(&big));
let ratio = t4 / t1.max(1e-9);
let what = format!("{}…: t(n) {t1:.4} s, t(4n) {t4:.4} s, ratio {ratio:.1}", word(1).chars().take(10).collect::<String>());
eprintln!("{what}");
(what, ratio)
})
.collect();
for (what, ratio) in ratios {
assert!(ratio < 8.0, "4x the letters, not linear: {what}");
}
}
#[test]
fn heuristic_fallback() {
let c: Vec<char> = "happen".chars().collect();
assert_eq!(heuristic_points(&c), vec![3]);
}
crates/text/src/lib.rs
+25
View File
@@ -0,0 +1,25 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
//! Shared text processing for the Craft apps. It sits below the composer: layout code calls it on
//! plain text and a language before it shapes and places glyphs.
//!
//! Modules, each behind the cargo feature of the same name:
//! - [`hyphen`] (default): hyphenation points, with bundled US English data and optional
//! hypher patterns for 47 more languages (Slovak, under the GPL, only with `slovak-gpl`).
//! - [`segment`] (default): line-break opportunities (UAX #14 with kinsoku), and word breaks for
//! Thai, Lao, Khmer and Burmese with their language features.
//! - [`bidi`] (default): bidirectional text (UAX #9): embedding levels, line reordering and
//! mirroring.
//! - [`linebreak`] (default): line breaking on a font-independent box/glue/penalty model.
//!
//! Language features (`spanish`, `german`, …, `all-languages`) add data for that language and
//! turn on the modules that use it.
#[cfg(feature = "bidi")]
pub mod bidi;
#[cfg(feature = "hyphen")]
pub mod hyphen;
#[cfg(feature = "linebreak")]
pub mod linebreak;
#[cfg(feature = "segment")]
pub mod segment;
crates/text/src/linebreak/greedy.rs
+108
View File
@@ -0,0 +1,108 @@
//! Greedy first fit: each line takes as much as fits.
use super::{INFINITE_PENALTY, Item, Kind, Line, Params, Sums, clamp_measure};
/// Break `items` first fit: each line ends at the last break where it still fits its measure
/// `width(k)` with its shrink. A line with no such break ends at the last [`Item::emergency`]
/// break that fits, else before the first box that overflows (never after an
/// [`Item::no_break`]).
pub fn greedy(items: &[Item], width: &dyn Fn(usize) -> f64, p: &Params) -> Vec<Line> {
greedy_with(items, width, p, &|_, _, k| items.get(k).map_or(0.0, |i| i.width))
}
/// [`greedy`] with box widths that depend on position: `advance(line, x, k)` is the width of box
/// `k` when it starts `x` from the start of line `line` (a tab reaching its stop). It returns
/// `items[k].width` for other boxes.
pub fn greedy_with(items: &[Item], width: &dyn Fn(usize) -> f64, p: &Params, advance: &dyn Fn(usize, f64, usize) -> f64) -> Vec<Line> {
let p = &p.sanitized();
let width = |k: usize| clamp_measure(width(k));
let s = Sums::new(items);
let m = items.len();
// Japanese "push out first": break where the line fits without shrinking when there is one.
let avoid_shrink = p.shrink_weight > p.stretch_weight;
let zone = p.rag.map_or(0.0, |r| r.hyphenation_zone);
let zoned = |b: usize| zone > 0.0 && items.get(b).is_some_and(|i| i.is_flagged() && i.hyphenation);
let no_break_before = |k: usize| k > 0 && items.get(k - 1).is_some_and(|i| i.kind == Kind::Penalty && i.cost >= INFINITE_PENALTY);
let mut out = Vec::new();
let mut start = 0;
let mut line = 0;
let mut hyphens = 0;
loop {
let measure = width(line);
// extra[k - start]: width the boxes before item `k` gained from `advance`.
let mut extra = Vec::new();
let mut ext = 0.0;
let mut last_ok = None;
let mut last_natural = None;
let mut last_emergency = None;
// (break, line width there)
let mut last_space: Option<(usize, f64)> = None;
let mut k = start;
let (end, next) = loop {
extra.push(ext);
let Some(it) = items.get(k) else { break (m, m) };
if s.is_break(k) {
if it.is_forced() {
break (k, s.next_start(k));
}
let blocked = it.is_flagged() && p.hyphen_limit > 0 && hyphens >= p.hyphen_limit;
let f = s.fit(start, k, Some(it), false, measure, p, ext);
if !blocked && f.feasible {
if it.emergency {
last_emergency = Some(k);
} else {
last_ok = Some(k);
if f.width <= measure + p.overrun {
last_natural = Some(k);
}
}
}
if it.kind == Kind::Glue {
last_space = Some((k, f.width));
}
} else if it.kind == Kind::Box {
let x = s.fit(start, k, None, false, measure, p, ext).width;
ext += advance(line, x, k) - it.width;
if k > start {
// Would a line ending right after this box overflow? (The following break's
// hang and edge elasticity apply; a hyphen is checked at its own break.)
let after = items.get(k + 1).filter(|i| i.kind != Kind::Box).map(|i| Item { width: 0.0, emergency: false, ..*i });
let f = s.fit(start, k + 1, after.as_ref(), false, measure, p, ext);
if avoid_shrink
&& f.width > measure + p.overrun
&& let Some(b) = last_natural
{
break (b, s.next_start(b));
}
if !f.feasible {
match (last_ok, last_emergency) {
(Some(b), _) => {
// Hyphenation zone: break before the word if that leaves little space.
if zoned(b)
&& let Some((sp, xs)) = last_space
&& measure - xs < zone
{
break (sp, s.next_start(sp));
}
break (b, s.next_start(b));
}
(None, Some(b)) => break (b, s.next_start(b)),
(None, None) if !no_break_before(k) => break (k, k),
(None, None) => {}
}
}
}
}
k += 1;
};
hyphens = if items.get(end).is_some_and(Item::is_flagged) { hyphens + 1 } else { 0 };
let ext_end = extra.get(end.saturating_sub(start)).copied().unwrap_or(ext);
out.push(s.line(start, end, next, measure, p, ext_end));
if next >= m {
break;
}
start = next;
line += 1;
}
out
}
crates/text/src/linebreak/knuth_plass.rs
+223
View File
@@ -0,0 +1,223 @@
//! Knuth–Plass total fit: the breaks that minimise the paragraph's summed demerits.
use super::{Item, Kind, Line, Params, Sums, clamp_measure, fitness_class};
/// Give up (return `None`) beyond this many active breaks.
const MAX_ACTIVE: usize = 4096;
/// Ratios beyond this are rated as this (an unstretchable line is infinitely loose).
const MAX_RATIO: f64 = 10_000.0;
/// Demerits of an overfull line accepted because nothing else reaches a break.
const OVERFULL_DEMERITS: f64 = 1e8;
#[derive(Clone, Copy, Debug)]
struct Node {
/// The break this node ends a line at.
pos: usize,
/// First item of the next line.
start: usize,
/// Number of the next line.
line: usize,
fitness: Option<u8>,
demerits: f64,
/// Consecutive lines ending at flagged breaks, up to here.
hyphens: usize,
flagged: bool,
prev: Option<usize>,
}
/// The best way found to reach the current break for one (line class, fitness, hyphen state).
/// Line classes are line numbers below [`Params::uniform_from`], then one class for the rest.
struct Candidate {
line: usize,
fitness: u8,
hyphens: usize,
demerits: f64,
from: usize,
}
/// One total-fit pass over `items`. `width(k)` is the measure of line `k`; set
/// [`Params::uniform_from`] when it varies. Returns `None` when no set of breaks keeps every line
/// within [`Params::tolerance`] (unless [`Params::overfull`]), or past the active-break limit.
pub fn knuth_plass(items: &[Item], width: &dyn Fn(usize) -> f64, p: &Params) -> Option<Vec<Line>> {
let p = &p.sanitized();
let width = |k: usize| clamp_measure(width(k));
let s = Sums::new(items);
let m = items.len();
let limit = p.hyphen_limit;
// Hyphen states 0..=limit; unlimited still tells a flagged line (1) from others (0).
let max_hyphens = if limit == 0 { 1 } else { limit };
let zone = p.rag.map_or(0.0, |r| r.hyphenation_zone);
let mut nodes = vec![Node { pos: 0, start: 0, line: 0, fitness: p.first_fitness, demerits: 0.0, hyphens: 0, flagged: false, prev: None }];
let mut active: Vec<usize> = vec![0];
let mut keep: Vec<usize> = Vec::new();
let mut best: Vec<Candidate> = Vec::new();
// The last break at a space (glue), for the hyphenation zone.
let mut last_space: Option<usize> = None;
for b in 0..=m {
if !s.is_break(b) {
continue;
}
let item = items.get(b);
let forced = s.forced(b);
let flagged = item.is_some_and(Item::is_flagged);
let cost = item.filter(|i| i.kind == Kind::Penalty).map_or(0.0, |i| i.cost);
best.clear();
keep.clear();
let mut last_removed = None;
for &a in &active {
let Some(n) = nodes.get(a).copied() else { continue };
// The node's next line starts after `b` (the penalties between were discarded): no
// line ends here, but one may end at a later break.
if b < n.start {
keep.push(a);
continue;
}
let measure = width(n.line);
let f = s.measure(n.start, b, measure, p);
// An overfull node is dropped even at a penalty with width (a hyphen), as in TeX,
// although a later break without that width might still fit.
if f.feasible && !forced {
keep.push(a);
} else {
last_removed = Some(a);
}
if !f.feasible || (f.ratio > p.tolerance && !forced) {
continue;
}
if flagged {
if limit > 0 && n.hyphens >= limit {
continue;
}
// Hyphenation zone: break at the space before the word instead when that leaves
// less than the zone at the line end.
if zone > 0.0
&& item.is_some_and(|i| i.hyphenation)
&& let Some(sp) = last_space.filter(|&sp| sp >= n.start)
&& measure - s.measure(n.start, sp, measure, p).width < zone
{
continue;
}
}
let weight = if f.ratio > 0.0 {
p.stretch_weight
} else if f.ratio < 0.0 {
p.shrink_weight
} else {
1.0
};
let badness = (weight * 100.0 * f.ratio.abs().min(MAX_RATIO).powi(3)).min(p.max_badness);
let lp = p.line_penalty + badness;
let mut d = if forced {
lp * lp
} else if cost >= 0.0 {
lp * lp + cost * cost
} else {
lp * lp - cost * cost
};
if flagged && n.flagged {
d += p.flagged_demerits;
}
// A ragged line ending at a forced break says nothing about the rag: it keeps its
// neighbour's class.
let class = fitness_class(f.ratio, &p.fitness);
let fitness = if forced && p.rag.is_some() { n.fitness.unwrap_or(class) } else { class };
if n.fitness.is_some_and(|nf| fitness.abs_diff(nf) > 1) {
d += p.fitness_demerits;
}
let hyphens = if flagged { (n.hyphens + 1).min(max_hyphens) } else { 0 };
let line = (n.line + 1).min(p.uniform_from);
let demerits = n.demerits + d;
match best.iter_mut().find(|c| c.line == line && c.fitness == fitness && c.hyphens == hyphens) {
Some(c) if c.demerits <= demerits => {}
Some(c) => {
c.demerits = demerits;
c.from = a;
}
None => best.push(Candidate { line, fitness, hyphens, demerits, from: a }),
}
}
if p.overfull
&& keep.is_empty()
&& best.is_empty()
&& let Some(a) = last_removed
&& let Some(n) = nodes.get(a)
{
// An overfull line ending here would leave the next line empty up to a forced break:
// end it at that forced break instead. Nor may it pass the hyphen limit: it ends at a
// later break.
let empty_next = !forced && s.items.get(s.next_start(b)).is_some_and(Item::is_forced);
let too_many_hyphens = flagged && limit > 0 && n.hyphens >= limit;
if empty_next || too_many_hyphens {
keep.push(a);
} else {
best.push(Candidate {
line: (n.line + 1).min(p.uniform_from),
fitness: 0,
hyphens: 0,
demerits: n.demerits + OVERFULL_DEMERITS,
from: a,
});
}
}
std::mem::swap(&mut active, &mut keep);
if item.is_some_and(|i| i.kind == Kind::Glue) {
last_space = Some(b);
} else if forced {
last_space = None;
}
if best.is_empty() {
if active.is_empty() {
return None;
}
continue;
}
let start = s.next_start(b);
// Knuth's pruning: within a line class and hyphen state, a node more than the fitness
// demerits behind the best can never win.
for c in &best {
let min = best.iter().filter(|o| o.line == c.line && o.hyphens == c.hyphens).map(|o| o.demerits).fold(f64::INFINITY, f64::min);
if c.demerits > min + p.fitness_demerits {
continue;
}
let Some(prev) = nodes.get(c.from).copied() else { continue };
nodes.push(Node {
pos: b,
start,
line: prev.line + 1,
fitness: Some(c.fitness),
demerits: c.demerits,
hyphens: if flagged { prev.hyphens + 1 } else { 0 },
flagged,
prev: Some(c.from),
});
active.push(nodes.len() - 1);
}
if active.len() > MAX_ACTIVE {
return None;
}
}
let end = if s.ends_forced() { m - 1 } else { m };
let last = active
.iter()
.copied()
.filter(|&a| nodes.get(a).is_some_and(|n| n.pos == end && n.prev.is_some()))
.min_by(|&a, &b| nodes[a].demerits.total_cmp(&nodes[b].demerits))?;
let mut chain = Vec::new();
let mut cur = Some(last);
while let Some(i) = cur {
let n = nodes.get(i)?;
if n.prev.is_some() {
chain.push(n.pos);
}
cur = n.prev;
}
chain.reverse();
let mut start = 0;
let mut lines = Vec::with_capacity(chain.len());
for (k, &b) in chain.iter().enumerate() {
let next = s.next_start(b);
lines.push(s.line(start, b, next, width(k), p, 0.0));
start = next;
}
Some(lines)
}
crates/text/src/linebreak/mod.rs
+637
View File
@@ -0,0 +1,637 @@
//! Line breaking on Knuth and Plass's box/glue/penalty model, independent of fonts.
//!
//! A composer turns a paragraph's shaped glyphs into [`Item`]s (boxes for glyphs or words, glue
//! for spaces, penalties for other break opportunities), calls a breaker, and places the glyphs of
//! each returned [`Line`]. Widths are in the caller's units.
//!
//! - [`knuth_plass`]: one total-fit pass at [`Params::tolerance`].
//! - [`break_paragraph`]: Knuth–Plass passes at rising tolerances, the last accepting overfull
//! lines, then [`greedy`] if no pass succeeds. [`Breaks::method`] says which produced the lines.
//! - [`greedy`] / [`greedy_with`]: first fit, each line takes as much as fits.
//! - [`balanced`]: the line count of [`break_paragraph`] with lines as even as possible.
//! - [`overlong`]: runs of boxes wider than the measure, where a composer may add
//! [`Item::emergency`] breaks.
//!
//! Elasticity comes in [`TIERS`] tiers, used in order: tier 0 (word spaces), then tier 1 (letter
//! spacing), then tier 2 (glyph scaling). Each tier is charged [`Params::tier_cost`] per unit of
//! its ratio, so a breaker prefers word-space changes, then letter spacing, then glyph scaling.
//!
//! Breakers take the measure of line `k` from `width(k)`. They clamp each measure to
//! `0.0..=f64::MAX` (NaN counts as 0) and treat lines from [`Params::uniform_from`] on as having
//! the same measure.
mod greedy;
mod knuth_plass;
#[cfg(test)]
mod tests;
use std::ops::Range;
pub use greedy::{greedy, greedy_with};
pub use knuth_plass::knuth_plass;
/// Number of elasticity tiers: word space, letter space, glyph scale.
pub const TIERS: usize = 3;
/// Stretch or shrink per tier.
pub type Tiers = [f64; TIERS];
/// A penalty at least this high forbids a break; one at most its negative forces a break.
pub const INFINITE_PENALTY: f64 = 10_000.0;
/// Cost of an [`Item::emergency`] break: high, so a paragraph takes no more than it must, but legal.
pub const EMERGENCY_PENALTY: f64 = 5_000.0;
/// What an [`Item`] is.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum Kind {
/// Fixed material (a glyph, a word). Never a break.
Box,
/// Elastic space. A break when it follows a box; discarded at the start of a line.
Glue,
/// A break opportunity with a cost; adds its width when a line ends there.
Penalty,
}
/// Tier-0 stretch and shrink that a line gains at one of its ends (see [`Item::edge`]).
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub struct Edge {
pub stretch: f64,
pub shrink: f64,
}
/// One item of a paragraph.
///
/// Build items from a constructor ([`Item::boxed`], [`Item::glue`], [`Item::penalty`], …) and
/// override the optional fields with struct update syntax, such as
/// `Item { hang: 2.0, ..Item::glue(3.0, 1.5, 0.6) }`. Fields added later get neutral values in the
/// constructors, so code written this way keeps compiling.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Item {
pub kind: Kind,
/// Natural width (boxes, glue), or the width a line broken here gains (penalties: a hyphen).
pub width: f64,
/// Stretch per tier. Boxes may carry letter-spacing and glyph-scaling stretch.
pub stretch: Tiers,
/// Shrink per tier.
pub shrink: Tiers,
/// Glue: infinite stretch. A line holding fill glue gives it all its leftover space.
pub fill: bool,
/// Glue: the unit in which stretch past every tier is rated. Lines whose glue has none rate it
/// against their total stretch.
pub overflow: f64,
/// Penalty cost: [`INFINITE_PENALTY`] forbids the break, its negative forces it.
pub cost: f64,
/// Penalty: a flagged (hyphen) break. Consecutive flagged lines cost
/// [`Params::flagged_demerits`] and count toward [`Params::hyphen_limit`].
pub flagged: bool,
/// Breaks: width a line ending here gives back (punctuation hanging into the margin).
/// Boxes: width a line starting with this box gives back.
pub hang: f64,
/// Tier-0 stretch and shrink that a line ending at this break (glue, penalties) or starting
/// with this box gains; negative values remove elasticity. The line's tier-0 totals are
/// clamped at zero.
pub edge: Edge,
/// Penalty: an automatic or discretionary hyphenation point, subject to
/// [`Rag::hyphenation_zone`].
pub hyphenation: bool,
/// Penalty: a break inside a word wider than the measure. A justified line ending here that
/// must stretch is rated by its leftover space as a fraction of the measure, so the fullest
/// piece wins.
pub emergency: bool,
}
const BLANK: Item = Item {
kind: Kind::Box,
width: 0.0,
stretch: [0.0; TIERS],
shrink: [0.0; TIERS],
fill: false,
overflow: 0.0,
cost: 0.0,
flagged: false,
hang: 0.0,
edge: Edge { stretch: 0.0, shrink: 0.0 },
hyphenation: false,
emergency: false,
};
impl Item {
/// A box of `width`.
pub const fn boxed(width: f64) -> Item {
Item { width, ..BLANK }
}
/// Glue of natural `width` with tier-0 `stretch` and `shrink`.
pub const fn glue(width: f64, stretch: f64, shrink: f64) -> Item {
Item { kind: Kind::Glue, width, stretch: [stretch, 0.0, 0.0], shrink: [shrink, 0.0, 0.0], ..BLANK }
}
/// Glue of zero width and infinite stretch: ends a paragraph (before [`Item::forced`]) whose
/// last line is not justified.
pub const fn fill() -> Item {
Item { kind: Kind::Glue, fill: true, ..BLANK }
}
/// A break opportunity costing `cost`, adding `width` when taken.
pub const fn penalty(width: f64, cost: f64, flagged: bool) -> Item {
Item { kind: Kind::Penalty, width, cost, flagged, ..BLANK }
}
/// A forced break.
pub const fn forced() -> Item {
Item::penalty(0.0, -INFINITE_PENALTY, false)
}
/// A break that is never taken (keeps [`greedy`] from splitting two boxes).
pub const fn no_break() -> Item {
Item::penalty(0.0, INFINITE_PENALTY, false)
}
/// An emergency break inside an overlong word (see [`overlong`]).
pub const fn emergency() -> Item {
Item { emergency: true, ..Item::penalty(0.0, EMERGENCY_PENALTY, false) }
}
fn is_forced(&self) -> bool {
self.kind == Kind::Penalty && self.cost <= -INFINITE_PENALTY
}
fn is_flagged(&self) -> bool {
self.kind == Kind::Penalty && self.flagged
}
}
/// Ragged (unjustified) setting. Spaces keep their width; a line is rated by its leftover space
/// against a rag stretch, as if every line ended with glue of that stretch.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Rag {
/// Rag stretch of every line, in width units.
pub stretch: f64,
/// Rag stretch of every line, as a fraction of its measure (added to `stretch`).
pub width_fraction: f64,
/// A line ending at a forced break that is shorter than this fraction of its measure is rated
/// by the shortfall instead of its leftover space (0 = off). Such lines otherwise cost nothing.
pub short_last: f64,
/// The unit of that shortfall, as a fraction of the measure.
pub short_last_unit: f64,
/// Hyphenation zone: a [`Item::hyphenation`] break is not taken when breaking at the space
/// before its word would leave less than this much space at the end of the line (0 = off).
pub hyphenation_zone: f64,
}
impl Default for Rag {
fn default() -> Self {
Rag { stretch: 0.0, width_fraction: 1.0 / 6.0, short_last: 0.0, short_last_unit: 1.0 / 3.0, hyphenation_zone: 0.0 }
}
}
/// How lines are rated and chosen.
///
/// Breakers clamp `flagged_demerits` and `fitness_demerits` to at least 0, with NaN counting as 0
/// (negative demerits would let pruning drop the best candidate). Other fields are used as given.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Params {
/// Largest adjustment ratio of a line not ending at a forced break ([`knuth_plass`]).
pub tolerance: f64,
/// When no line reaches a break, accept the overfull line from the last node that could
/// ([`knuth_plass`]), so the pass always succeeds.
pub overfull: bool,
/// Added to every line's badness before squaring.
pub line_penalty: f64,
/// Added when two consecutive lines end at flagged breaks.
pub flagged_demerits: f64,
/// Added when neighbouring lines' fitness classes differ by more than one.
pub fitness_demerits: f64,
/// Upper ratio bounds of fitness classes 0 (tight, `r <` bound), 1 and 2 (`r <=` bound);
/// class 3 (very loose) is above.
pub fitness: [f64; 3],
/// Fitness class assumed before the first line; `None`: the first line pays no fitness demerits.
pub first_fitness: Option<u8>,
/// Most consecutive lines ending at flagged breaks (0 = unlimited). A hard constraint: a
/// Knuth–Plass pass that cannot meet it fails, and [`greedy`] breaks before the word instead.
pub hyphen_limit: usize,
/// Ratio charged per unit of each tier.
pub tier_cost: Tiers,
/// Badness cap (badness is `100·|r|³`, times the weight below).
pub max_badness: f64,
/// Badness multiplier of stretched lines.
pub stretch_weight: f64,
/// Badness multiplier of shrunk lines. [`greedy`] avoids shrinking when this exceeds
/// `stretch_weight`.
pub shrink_weight: f64,
/// A line longer than its measure by at most this fits exactly (rounded widths).
pub overrun: f64,
/// First line index from which `width(k)` no longer changes. [`knuth_plass`] keeps the ways
/// to reach a break apart by line number only for earlier lines; from here on they compete
/// directly, which keeps the number of active breaks small.
///
/// The default 0 suits a constant measure. A caller whose measures vary (a first-line indent,
/// a drop cap, a shaped frame) must set it to the first index of the constant tail, such as 1
/// for a first-line indent; with a smaller value the lines still get their own measures, but
/// the breaks may not be the best ones. Large values make long paragraphs slow and can exceed
/// the active-break limit, so that [`break_paragraph`] falls back to [`greedy`].
pub uniform_from: usize,
/// Ragged setting; `None` justifies.
pub rag: Option<Rag>,
}
impl Default for Params {
fn default() -> Self {
Params {
tolerance: 2.5,
overfull: false,
line_penalty: 10.0,
flagged_demerits: 3000.0,
fitness_demerits: 3000.0,
fitness: [-0.5, 0.5, 1.0],
first_fitness: Some(1),
hyphen_limit: 0,
tier_cost: [1.0, 0.75, 1.0],
max_badness: f64::INFINITY,
stretch_weight: 1.0,
shrink_weight: 1.0,
overrun: 1e-6,
uniform_from: 0,
rag: None,
}
}
}
impl Params {
/// These parameters with the clamps documented on [`Params`] applied.
fn sanitized(&self) -> Params {
let non_negative = |x: f64| if x.is_nan() { 0.0 } else { x.max(0.0) };
Params { flagged_demerits: non_negative(self.flagged_demerits), fitness_demerits: non_negative(self.fitness_demerits), ..*self }
}
}
/// A measure from a caller's `width`, clamped to `0.0..=f64::MAX` (NaN counts as 0).
fn clamp_measure(m: f64) -> f64 {
if m.is_nan() { 0.0 } else { m.clamp(0.0, f64::MAX) }
}
/// One line chosen by a breaker.
#[derive(Clone, Copy, Debug, PartialEq)]
#[non_exhaustive]
pub struct Line {
/// First item on the line.
pub start: usize,
/// The break: items `start..end` are on the line. `items.len()` for a paragraph that does not
/// end with a forced break; a box's index where [`greedy`] split a word with no break in it.
pub end: usize,
/// First item of the next line (glue and penalties after the break are discarded).
pub next: usize,
/// Natural width: boxes and glue on the line, plus the break's penalty width, minus hangs.
pub width: f64,
/// The line's measure.
pub measure: f64,
/// Adjustment ratio as rated: positive stretches, negative shrinks. Each tier adds
/// [`Params::tier_cost`] per unit used, so a line that fits may shrink below −1; an overfull
/// line has a negative [`excess`](Self::excess) instead.
pub ratio: f64,
/// Fraction of each tier's stretch (positive) or shrink (negative) the line uses.
pub tiers: Tiers,
/// Width the tiers leave: taken by fill glue or the rag, stretch past every tier when
/// justified (spread over the glue's `overflow`, else its stretch), or the overfull amount
/// when negative.
pub excess: f64,
/// Ended at a flagged penalty.
pub flagged: bool,
/// Ended at a forced break or the paragraph end.
pub forced: bool,
}
/// Runs of boxes, as item ranges, whose width fully shrunk exceeds `measure`: words no line can
/// hold. A composer adds [`Item::emergency`] breaks inside them where its script allows.
pub fn overlong(items: &[Item], measure: f64) -> Vec<Range<usize>> {
let mut out = Vec::new();
// (first box, last box, width)
let mut run: Option<(usize, usize, f64)> = None;
let mut close = |run: &mut Option<(usize, usize, f64)>| {
if let Some((first, last, w)) = run.take()
&& w > measure
{
out.push(first..last + 1);
}
};
for (k, it) in items.iter().enumerate() {
match it.kind {
Kind::Box => {
let w = it.width - it.shrink.iter().sum::<f64>();
run = Some(run.map_or((k, k, w), |(first, _, acc)| (first, k, acc + w)));
}
Kind::Penalty if it.cost >= INFINITE_PENALTY => {}
Kind::Glue | Kind::Penalty => close(&mut run),
}
}
close(&mut run);
out
}
/// Which breaker produced a paragraph's lines.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum Method {
/// The [`knuth_plass`] pass at `tolerances[pass]`.
KnuthPlass { pass: usize },
/// [`greedy`], because every Knuth–Plass pass failed or there were none.
Greedy,
}
/// Lines from [`break_paragraph`] or [`balanced`].
#[derive(Clone, Debug, PartialEq)]
#[non_exhaustive]
pub struct Breaks {
pub lines: Vec<Line>,
/// The breaker that produced `lines`.
pub method: Method,
/// [`balanced`] evened the lines. Always false from [`break_paragraph`].
pub balanced: bool,
}
/// [`knuth_plass`] at each of `tolerances` in turn, the last pass also accepting overfull lines;
/// [`greedy`] if every pass fails or `tolerances` is empty. The last pass fails only past the
/// active-break limit (see [`Params::uniform_from`]).
pub fn break_paragraph(items: &[Item], width: &dyn Fn(usize) -> f64, params: &Params, tolerances: &[f64]) -> Breaks {
for (pass, &tolerance) in tolerances.iter().enumerate() {
let overfull = params.overfull || pass + 1 == tolerances.len();
if let Some(lines) = knuth_plass(items, width, &Params { tolerance, overfull, ..*params }) {
return Breaks { lines, method: Method::KnuthPlass { pass }, balanced: false };
}
}
Breaks { lines: greedy(items, width, params), method: Method::Greedy, balanced: false }
}
/// Paragraphs with more items than this are not balanced (each balancing step breaks the whole
/// paragraph again).
pub const MAX_BALANCED_ITEMS: usize = 6000;
/// Balance lines: keep [`break_paragraph`]'s line count but make the lines as even as possible,
/// by breaking against the narrowest scaled measure that still gives that many lines. The lines
/// are measured against the real `width`. Meant for ragged text; justified text is balanced the
/// same way, so its lines stretch more.
///
/// Returns [`break_paragraph`]'s lines with [`Breaks::balanced`] false when the paragraph has
/// fewer than two lines, a forced break before its last line, more than [`MAX_BALANCED_ITEMS`]
/// items, or when no narrower measure keeps the line count.
pub fn balanced(items: &[Item], width: &dyn Fn(usize) -> f64, params: &Params, tolerances: &[f64]) -> Breaks {
let base = break_paragraph(items, width, params, tolerances);
let n = base.lines.len();
if n < 2 || items.len() > MAX_BALANCED_ITEMS || base.lines[..n - 1].iter().any(|l| l.forced) {
return base;
}
let attempt = |f: f64| -> Option<Breaks> {
let w = |j: usize| clamp_measure(width(j)) * f;
let b = break_paragraph(items, &w, params, tolerances);
(b.lines.len() == n && b.lines.iter().enumerate().all(|(j, l)| l.width <= w(j) + params.overrun)).then_some(b)
};
let (mut lo, mut hi) = (0.3, 1.0);
let mut best = None;
for _ in 0..14 {
let mid = (lo + hi) / 2.0;
match attempt(mid) {
Some(lines) => {
best = Some(lines);
hi = mid;
}
None => lo = mid,
}
}
let Some(best) = best else { return base };
let sums = Sums::new(items);
let p = params.sanitized();
let lines = best.lines.iter().enumerate().map(|(j, l)| sums.line(l.start, l.end, l.next, clamp_measure(width(j)), &p, 0.0)).collect();
Breaks { lines, method: best.method, balanced: true }
}
/// How a line fits its measure.
#[derive(Clone, Copy, Debug)]
struct Fit {
width: f64,
ratio: f64,
tiers: Tiers,
excess: f64,
/// Fits with the shrink available.
feasible: bool,
}
/// Prefix sums of a paragraph's items, for measuring any line in constant time.
struct Sums<'a> {
items: &'a [Item],
/// Before item `k`: width of boxes and glue, their stretch and shrink, glue overflow and the
/// number of fill glues.
w: Vec<f64>,
y: Vec<Tiers>,
z: Vec<Tiers>,
over: Vec<f64>,
fill: Vec<u32>,
}
impl<'a> Sums<'a> {
fn new(items: &'a [Item]) -> Self {
let m = items.len();
let (mut w, mut y, mut z, mut over, mut fill) =
(Vec::with_capacity(m + 1), Vec::with_capacity(m + 1), Vec::with_capacity(m + 1), Vec::with_capacity(m + 1), Vec::with_capacity(m + 1));
let (mut sw, mut sy, mut sz, mut so, mut sf) = (0.0, [0.0; TIERS], [0.0; TIERS], 0.0, 0);
for it in items {
w.push(sw);
y.push(sy);
z.push(sz);
over.push(so);
fill.push(sf);
if it.kind != Kind::Penalty {
sw += it.width;
for t in 0..TIERS {
sy[t] += it.stretch[t];
sz[t] += it.shrink[t];
}
}
if it.kind == Kind::Glue {
so += it.overflow;
sf += u32::from(it.fill);
}
}
w.push(sw);
y.push(sy);
z.push(sz);
over.push(so);
fill.push(sf);
Sums { items, w, y, z, over, fill }
}
fn len(&self) -> usize {
self.items.len()
}
/// The paragraph ends with a forced penalty (else its end is an implicit forced break).
fn ends_forced(&self) -> bool {
self.items.last().is_some_and(Item::is_forced)
}
/// Is `b` a legal break? `len()` is one unless the items end with a forced penalty.
fn is_break(&self, b: usize) -> bool {
match self.items.get(b) {
None => b == self.len() && !self.ends_forced(),
Some(it) => match it.kind {
Kind::Box => false,
Kind::Penalty => it.cost < INFINITE_PENALTY,
Kind::Glue => b > 0 && self.items.get(b - 1).is_some_and(|p| p.kind == Kind::Box),
},
}
}
fn forced(&self, b: usize) -> bool {
self.items.get(b).map_or(b >= self.len(), Item::is_forced)
}
/// First item of the line after a break at `b`: glue and penalties are discarded up to the next
/// box or forced break.
fn next_start(&self, b: usize) -> usize {
let mut i = b + 1;
while let Some(it) = self.items.get(i) {
if it.kind == Kind::Box || it.is_forced() {
break;
}
i += 1;
}
i.min(self.len())
}
fn at(&self, k: usize) -> (f64, Tiers, Tiers, f64, u32) {
let k = k.min(self.len());
(
self.w.get(k).copied().unwrap_or(0.0),
self.y.get(k).copied().unwrap_or_default(),
self.z.get(k).copied().unwrap_or_default(),
self.over.get(k).copied().unwrap_or(0.0),
self.fill.get(k).copied().unwrap_or(0),
)
}
/// How items `start..end` fit `measure`, ending at `brk` (a glue or penalty whose hang, edge
/// elasticity and penalty width apply; `None` at the paragraph end or a split between boxes).
/// `extra` is added to the natural width (boxes whose width depends on position).
#[allow(clippy::too_many_arguments)]
fn fit(&self, start: usize, end: usize, brk: Option<&Item>, forced: bool, measure: f64, p: &Params, extra: f64) -> Fit {
let (w0, y0, z0, o0, f0) = self.at(start);
let (w1, y1, z1, o1, f1) = self.at(end);
let (start_hang, start_edge) =
self.items.get(start).filter(|i| i.kind == Kind::Box && end > start).map_or((0.0, Edge::default()), |b| (b.hang, b.edge));
let (pw, end_hang, end_edge, emergency) = brk
.filter(|i| i.kind != Kind::Box)
.map_or((0.0, 0.0, Edge::default(), false), |i| (if i.kind == Kind::Penalty { i.width } else { 0.0 }, i.hang, i.edge, i.emergency));
let width = w1 - w0 + pw - start_hang - end_hang + extra;
let mut y = [0.0; TIERS];
let mut z = [0.0; TIERS];
for t in 0..TIERS {
y[t] = y1[t] - y0[t];
z[t] = z1[t] - z0[t];
}
y[0] = (y[0] + start_edge.stretch + end_edge.stretch).max(0.0);
z[0] = (z[0] + start_edge.shrink + end_edge.shrink).max(0.0);
let mut d = measure - width;
if d < 0.0 && d > -p.overrun {
d = 0.0;
}
let free = |ratio: f64| Fit { width, ratio, tiers: [0.0; TIERS], excess: d, feasible: true };
if d < 0.0 {
let (ratio, tiers, excess, feasible) = tiered(d, &z, 0.0, &p.tier_cost);
return Fit { width, ratio, tiers, excess, feasible };
}
if let Some(rag) = p.rag {
if forced && rag.short_last > 0.0 {
let short = measure * rag.short_last - width;
let unit = measure * rag.short_last_unit;
return free(if short <= 0.0 {
0.0
} else if unit > 0.0 {
short / unit
} else {
f64::INFINITY
});
}
if f1 > f0 {
return free(0.0);
}
let s = rag.stretch + rag.width_fraction * measure;
return free(if d <= 0.0 {
0.0
} else if s > 0.0 {
d / s
} else {
f64::INFINITY
});
}
if f1 > f0 {
return free(0.0);
}
let (mut ratio, tiers, excess, feasible) = tiered(d, &y, o1 - o0, &p.tier_cost);
if emergency && ratio > 0.0 && measure > 0.0 {
ratio = d / measure;
}
Fit { width, ratio, tiers, excess, feasible }
}
/// Measure the break at `b` for a line starting at `start` (`b == len()`: the paragraph end).
fn measure(&self, start: usize, b: usize, measure: f64, p: &Params) -> Fit {
self.fit(start, b, self.items.get(b), self.forced(b), measure, p, 0.0)
}
fn line(&self, start: usize, end: usize, next: usize, measure: f64, p: &Params, extra: f64) -> Line {
let brk = self.items.get(end).filter(|i| i.kind != Kind::Box);
let forced = brk.map_or(end >= self.len(), Item::is_forced);
let f = self.fit(start, end, brk, forced, measure, p, extra);
Line {
start,
end,
next,
width: f.width,
measure,
ratio: f.ratio,
tiers: f.tiers,
excess: f.excess,
flagged: brk.is_some_and(Item::is_flagged),
forced,
}
}
}
/// Adjustment ratio for `d` units of stretch (`d > 0`) or shrink (`d < 0`) from tier capacities
/// `cap`, used in tier order and charged `cost` per unit of each tier's ratio. Returns the ratio,
/// the fraction of each tier used (negative when shrinking), the width left over and whether the
/// line is feasible: shrinking past every tier is not. Stretch past every tier is rated in units
/// of `over`, or of the total stretch when `over` is zero.
fn tiered(d: f64, cap: &Tiers, over: f64, cost: &Tiers) -> (f64, Tiers, f64, bool) {
const EPS: f64 = 1e-9;
let mut used = [0.0; TIERS];
if d.abs() < EPS {
return (0.0, used, d, true);
}
let sign = d.signum();
let mut rem = d.abs();
let mut r = 0.0;
for t in 0..TIERS {
if cap[t] > EPS && rem > EPS {
let take = rem.min(cap[t]);
r += cost[t] * take / cap[t];
used[t] = sign * take / cap[t];
rem -= take;
}
}
let excess = sign * rem;
if d < 0.0 {
return (-r, used, excess, rem <= 1e-6);
}
if rem > 1e-6 {
let unit = if over > EPS { over } else { cap.iter().sum::<f64>() };
r = if unit > EPS { r + rem / unit } else { f64::INFINITY };
}
(r, used, excess, true)
}
/// Fitness class 0 (tight) … 3 (very loose) of ratio `r`.
fn fitness_class(r: f64, bounds: &[f64; 3]) -> u8 {
if r < bounds[0] {
0
} else if r <= bounds[1] {
1
} else if r <= bounds[2] {
2
} else {
3
}
}
crates/text/src/linebreak/tests.rs
+476
View File
@@ -0,0 +1,476 @@
//! Breaker tests on a monospaced model: letters are 6 units wide, spaces 3, hyphens 4. Parameter
//! sets mirror vectorcraft's and designcraft's composers.
use super::*;
const CHAR: f64 = 6.0;
const SPACE: f64 = 3.0;
const HYPHEN: f64 = 4.0;
const COPY: &str = "Typography is the craft of arranging type to make written language legible, readable and appealing when displayed. \
The arrangement of type involves selecting typefaces, point sizes, line lengths, line spacing and letter spacing, and adjusting the space \
between pairs of letters.";
/// A paragraph's items and the text each item stands for.
struct Para {
items: Vec<Item>,
text: Vec<String>,
}
impl Para {
/// A box per word and glue per space (elastic unless ragged); with `hyphen_cost`, a
/// hyphenation point every two letters inside words of six letters or more. Ends with fill and
/// a forced break.
fn new(text: &str, p: &Params, hyphen_cost: Option<f64>) -> Para {
let mut para = Para { items: Vec::new(), text: Vec::new() };
for (i, word) in text.split(' ').enumerate() {
if i > 0 {
let glue = if p.rag.is_some() { Item::glue(SPACE, 0.0, 0.0) } else { Item::glue(SPACE, SPACE / 2.0, SPACE / 5.0) };
para.push(glue, " ");
}
let chars: Vec<char> = word.chars().collect();
let points: Vec<usize> = match hyphen_cost {
Some(_) if chars.len() >= 6 => (2..chars.len() - 2).step_by(2).collect(),
_ => Vec::new(),
};
let mut from = 0;
for &at in points.iter().chain([chars.len()].iter()) {
let piece: String = chars[from..at].iter().collect();
para.push(Item::boxed(width_of(&piece)), &piece);
if at < chars.len() {
para.push(Item { hyphenation: true, ..Item::penalty(HYPHEN, hyphen_cost.unwrap_or(0.0), true) }, "");
}
from = at;
}
}
para.push(Item::fill(), "");
para.push(Item::forced(), "");
para
}
fn push(&mut self, item: Item, text: &str) {
self.items.push(item);
self.text.push(text.to_string());
}
fn lines(&self, lines: &[Line]) -> Vec<String> {
lines
.iter()
.map(|l| {
let s: String = self.text[l.start..l.end.min(self.text.len())].concat();
let s = s.trim_end().to_string();
if l.flagged { s + "-" } else { s }
})
.collect()
}
}
fn width_of(s: &str) -> f64 {
s.chars().map(|c| if c == ' ' { SPACE } else { CHAR }).sum()
}
fn vectorcraft(ragged: bool) -> Params {
Params {
fitness_demerits: if ragged { 1000.0 } else { 10_000.0 },
fitness: if ragged { [0.15, 0.4, 0.75] } else { [-0.5, 0.5, 1.0] },
first_fitness: None,
max_badness: 10_000.0,
rag: ragged.then_some(Rag {
stretch: 0.0,
width_fraction: 1.0 / 6.0,
short_last: 1.0 / 3.0,
short_last_unit: 1.0 / 3.0,
hyphenation_zone: 0.0,
}),
..Params::default()
}
}
/// vectorcraft's every-line composer: a strict pass, then a loose one, then single-line.
fn every_line(items: &[Item], width: f64, ragged: bool) -> Vec<Line> {
let p = vectorcraft(ragged);
let w = |_| width;
let loose = if ragged { f64::INFINITY } else { 4.0 };
knuth_plass(items, &w, &Params { tolerance: 1.0, ..p })
.or_else(|| knuth_plass(items, &w, &Params { tolerance: loose, ..p }))
.unwrap_or_else(|| greedy(items, &w, &p))
}
fn designcraft(ragged: bool) -> Params {
Params {
hyphen_limit: 3,
overrun: 0.01,
uniform_from: 0,
rag: ragged.then_some(Rag { stretch: 24.0, width_fraction: 0.0, ..Rag::default() }),
..Params::default()
}
}
const PASSES: [f64; 3] = [2.5, 12.0, f64::INFINITY];
/// Lines are contiguous and cover every item.
fn assert_covers(items: &[Item], lines: &[Line]) {
assert_eq!(lines.first().map(|l| l.start), Some(0));
for pair in lines.windows(2) {
assert_eq!(pair[0].next, pair[1].start, "{lines:?}");
}
assert_eq!(lines.last().map(|l| l.next), Some(items.len()));
}
#[test]
fn tiers_are_used_in_order() {
let cost = Params::default().tier_cost;
// Word stretch alone.
let (r, used, _, ok) = tiered(5.0, &[10.0, 10.0, 10.0], 10.0, &cost);
assert!(ok && (r - 0.5).abs() < 1e-9 && used == [0.5, 0.0, 0.0]);
// Word saturated, half the letter tier.
let (r, used, _, _) = tiered(15.0, &[10.0, 10.0, 10.0], 10.0, &cost);
assert!((r - (1.0 + 0.5 * cost[1])).abs() < 1e-9 && used == [1.0, 0.5, 0.0]);
// Beyond every tier: unbounded word spacing.
let (r, _, excess, _) = tiered(40.0, &[10.0, 10.0, 10.0], 10.0, &cost);
assert!(r > 2.0 && (excess - 10.0).abs() < 1e-9);
// Shrink beyond the tiers is infeasible.
assert!(!tiered(-25.0, &[10.0, 10.0, 0.0], 0.0, &cost).3);
assert!(tiered(-15.0, &[10.0, 10.0, 0.0], 0.0, &cost).3);
}
#[test]
fn lines_fit_and_cover_the_paragraph() {
let text = [COPY; 2].join(" ");
for ragged in [false, true] {
for hyphen in [None, Some(50.0)] {
let p = vectorcraft(ragged);
let para = Para::new(&text, &p, hyphen);
for (name, lines) in [("every-line", every_line(&para.items, 180.0, ragged)), ("greedy", greedy(&para.items, &|_| 180.0, &p))] {
assert!(lines.len() > 10, "{name}");
assert_covers(&para.items, &lines);
for l in &lines {
assert!(l.excess >= -p.overrun, "{name} ragged={ragged} {l:?}");
if ragged {
assert!(l.width <= 180.0 + 1e-6, "{name} {l:?}");
}
}
assert!(lines.last().is_some_and(|l| l.forced));
}
}
}
}
/// Every-line breaking spreads the spacing more evenly than greedy breaking.
#[test]
fn every_line_is_more_even_than_greedy() {
let text = [COPY; 2].join(" ");
let p = vectorcraft(false);
let para = Para::new(&text, &p, None);
for width in [150.0, 190.0, 240.0] {
let worst = |lines: &[Line]| lines[..lines.len() - 1].iter().map(|l| l.ratio).fold(0.0, f64::max);
let (kp, gr) = (worst(&every_line(&para.items, width, false)), worst(&greedy(&para.items, &|_| width, &p)));
assert!(kp <= gr + 1e-9, "{width}: every-line worst ratio {kp} vs greedy {gr}");
}
}
/// Ragged: every-line breaks before a word that would fit, so the last line is not a lone short
/// word; greedy keeps filling the first line.
#[test]
fn every_line_balances_ragged_lines() {
const S: &str = "Destroy all creatures with toughness X or less.";
let width = width_of("Destroy all creatures with toughness X or") + 2.0;
let p = vectorcraft(true);
let para = Para::new(S, &p, None);
assert_eq!(para.lines(&greedy(&para.items, &|_| width, &p)), ["Destroy all creatures with toughness X or", "less."]);
assert_eq!(para.lines(&every_line(&para.items, width, true)), ["Destroy all creatures with toughness X", "or less."]);
}
/// Every-line evens the rag of a long ragged paragraph: the leftover space is no worse than greedy
/// breaking, with at most one line more. Hyphenated, it trades some rag for fewer hyphens.
#[test]
fn every_line_evens_the_rag() {
let text = [COPY; 2].join(" ");
let p = vectorcraft(true);
for (width, hyphen) in [(150.0, None), (190.0, None), (240.0, Some(25.0)), (310.0, None)] {
let para = Para::new(&text, &p, hyphen);
let rag = |lines: &[Line]| {
for l in lines {
assert!(l.width <= width + 1e-6, "{width}: {l:?}");
}
let slack = lines[..lines.len() - 1].iter().map(|l| (width - l.width).powi(2)).sum::<f64>();
(slack, lines.len(), lines.iter().filter(|l| l.flagged).count())
};
let ((kp, kp_n, kp_hy), (gr, gr_n, gr_hy)) = (rag(&every_line(&para.items, width, true)), rag(&greedy(&para.items, &|_| width, &p)));
if hyphen.is_some() {
assert!(kp_hy < gr_hy, "{width}: {kp_hy} hyphens vs greedy {gr_hy}");
} else {
assert!(kp <= gr + 1e-6, "{width}: rag {kp} vs greedy {gr}");
}
assert!(kp_n <= gr_n + 1, "{width}: {kp_n} lines vs {gr_n}");
}
}
/// Greedy: each line takes as many words as fit.
#[test]
fn greedy_fills_each_line() {
let width = 190.0;
let p = vectorcraft(true);
let para = Para::new(COPY, &p, None);
let lines = para.lines(&greedy(&para.items, &|_| width, &p));
for pair in lines.windows(2) {
let next_word = pair[1].split(' ').next().unwrap_or_default();
assert!(width_of(&format!("{} {next_word}", pair[0])) > width, "{pair:?}");
}
}
#[test]
fn hyphen_limit_is_a_hard_constraint() {
let text = [COPY; 4].join(" ");
for limit in [1, 2, 9] {
let p = Params { hyphen_limit: limit, ..designcraft(false) };
let para = Para::new(&text, &p, Some(0.0));
for lines in [break_paragraph(&para.items, &|_| 130.0, &p, &PASSES).lines, greedy(&para.items, &|_| 130.0, &p)] {
let mut run = 0;
for l in &lines {
run = if l.flagged { run + 1 } else { 0 };
assert!(run <= limit, "{run} consecutive hyphens with limit {limit}");
}
assert!(lines.iter().any(|l| l.flagged));
}
}
}
#[test]
fn hyphenation_zone_limits_ragged_hyphens() {
let text = [COPY; 4].join(" ");
let measure = 110.0;
let count = |zone: f64, every_line: bool| {
let base = designcraft(true);
let p = Params { rag: base.rag.map(|r| Rag { hyphenation_zone: zone, ..r }), ..base };
let para = Para::new(&text, &p, Some(0.0));
let lines = if every_line { break_paragraph(&para.items, &|_| measure, &p, &PASSES).lines } else { greedy(&para.items, &|_| measure, &p) };
// A hyphenated line leaves at least the zone after the last space before its last word.
for l in lines.iter().filter(|l| l.flagged) {
if let Some(space) = (l.start..l.end).rfind(|&k| para.items[k].kind == Kind::Glue) {
let before: f64 = para.items[l.start..space].iter().filter(|i| i.kind != Kind::Penalty).map(|i| i.width).sum();
assert!(measure - before >= zone, "every_line={every_line} zone {zone}: {:?}", para.lines(&[*l]));
}
}
lines.iter().filter(|l| l.flagged).count()
};
for every_line in [true, false] {
let (free, zoned, huge) = (count(0.0, every_line), count(36.0, every_line), count(1000.0, every_line));
// A huge zone leaves only words that start a line hyphenable.
assert!(free > 0 && zoned > 0 && huge * 4 <= free, "every_line={every_line}: {free} {zoned} {huge}");
}
}
#[test]
fn balanced_lines_even_a_headline() {
let p = Params { rag: Some(Rag { stretch: 24.0, width_fraction: 0.0, ..Rag::default() }), ..designcraft(true) };
let para = Para::new("Balanced headlines read better than a stub", &p, None);
let w = |_| 150.0;
let plain = break_paragraph(&para.items, &w, &p, &PASSES).lines;
let bal = balanced(&para.items, &w, &p, &PASSES);
assert!(bal.balanced);
let bal = bal.lines;
assert!(plain.len() >= 2 && plain.len() == bal.len(), "{:?} vs {:?}", para.lines(&plain), para.lines(&bal));
let spread = |lines: &[Line]| lines.iter().map(|l| l.width).fold(f64::MIN, f64::max) - lines.iter().map(|l| l.width).fold(f64::MAX, f64::min);
assert!(spread(&bal) < spread(&plain), "{:?} vs {:?}", para.lines(&bal), para.lines(&plain));
assert!(bal.iter().all(|l| l.measure == 150.0));
}
/// A word wider than the measure breaks at its emergency points, or, for greedy without them,
/// before the box that overflows.
#[test]
fn an_overlong_word_breaks_at_the_measure() {
let build = |emergency: bool, ragged: bool| {
let p = designcraft(ragged);
let glue = if ragged { Item::glue(SPACE, 0.0, 0.0) } else { Item::glue(SPACE, SPACE / 2.0, SPACE / 5.0) };
let mut items = vec![Item::boxed(4.0 * CHAR), glue];
for i in 0..60 {
if i > 0 && emergency {
items.push(Item::emergency());
}
items.push(Item::boxed(CHAR));
}
items.extend([glue, Item::boxed(3.0 * CHAR), glue, Item::boxed(2.0 * CHAR), Item::fill(), Item::forced()]);
(items, p)
};
let (plain, _) = build(false, false);
assert_eq!(overlong(&plain, 72.0), vec![2..62]);
for ragged in [false, true] {
let (items, p) = build(true, ragged);
let (no_points, _) = build(false, ragged);
for (name, lines) in [
("every-line", break_paragraph(&items, &|_| 72.0, &p, &PASSES).lines),
("greedy", greedy(&items, &|_| 72.0, &p)),
("greedy, no emergency points", greedy(&no_points, &|_| 72.0, &p)),
] {
assert!(lines.len() >= 6, "{name} ragged={ragged}: {lines:?}");
for l in &lines {
assert!(l.width <= 72.0 + 0.01, "{name} ragged={ragged}: {l:?}");
}
}
}
}
#[test]
fn forced_breaks_end_lines_and_the_paragraph_end_is_one() {
let p = designcraft(false);
// No forced penalty at the end: the paragraph end is the last break.
let items = [Item::boxed(30.0), Item::glue(SPACE, 1.5, 0.6), Item::boxed(30.0), Item::fill(), Item::forced(), Item::boxed(30.0)];
for lines in [break_paragraph(&items, &|_| 100.0, &p, &PASSES).lines, greedy(&items, &|_| 100.0, &p)] {
assert_eq!(lines.iter().map(|l| (l.start, l.end, l.forced)).collect::<Vec<_>>(), [(0, 4, true), (5, 6, true)]);
// The last line is justified: it has no fill and nothing to stretch.
assert!(lines[1].ratio.is_infinite());
}
// Empty paragraph: one empty line.
for lines in [break_paragraph(&[], &|_| 100.0, &p, &PASSES).lines, greedy(&[], &|_| 100.0, &p)] {
assert_eq!(lines.iter().map(|l| (l.start, l.end, l.next)).collect::<Vec<_>>(), [(0, 0, 0)]);
}
}
/// The measure is per line: a first-line indent narrows line 0 only.
#[test]
fn each_line_gets_its_own_measure() {
let p = Params { uniform_from: 1, ..vectorcraft(true) };
let para = Para::new(COPY, &p, None);
let w = |k: usize| if k == 0 { 120.0 } else { 200.0 };
for lines in [break_paragraph(&para.items, &w, &p, &[1.0, f64::INFINITY]).lines, greedy(&para.items, &w, &p)] {
for (k, l) in lines.iter().enumerate() {
assert_eq!(l.measure, w(k));
assert!(l.width <= w(k) + 1e-6, "{k}: {l:?}");
}
assert!(lines[1].width > 120.0);
}
}
/// Punctuation hanging into the margin gives its width back: the line fits with the word after it.
#[test]
fn a_hanging_break_gives_its_width_back() {
let p = designcraft(true);
let items = |hang: f64| {
[
Item::boxed(60.0),
Item::glue(5.0, 0.0, 0.0),
Item::boxed(40.0),
Item { hang, ..Item::glue(5.0, 0.0, 0.0) },
Item::boxed(30.0),
Item::fill(),
Item::forced(),
]
};
for (hang, end) in [(0.0, 1), (5.0, 3)] {
let items = items(hang);
for lines in [break_paragraph(&items, &|_| 100.0, &p, &PASSES).lines, greedy(&items, &|_| 100.0, &p)] {
assert_eq!(lines[0].end, end, "hang {hang}: {lines:?}");
}
}
}
/// Justification uses word spacing first, then letter spacing.
#[test]
fn justification_uses_tiers_in_order() {
let p = designcraft(false);
let letters = Item { stretch: [0.0, 10.0, 0.0], ..Item::boxed(40.0) };
let items = [letters, Item::glue(10.0, 6.0, 0.0), letters, Item::forced()];
let lines = break_paragraph(&items, &|_| 100.0, &p, &PASSES).lines;
let l = lines[0];
assert_eq!(l.tiers, [1.0, 0.2, 0.0]);
assert!((l.ratio - (1.0 + 0.2 * p.tier_cost[1])).abs() < 1e-9, "{l:?}");
}
/// Letter spacing and glyph scaling absorb what word spacing alone cannot in a narrow column.
#[test]
fn letter_and_glyph_tiers_ease_a_narrow_column() {
let worst = |box_stretch: f64| {
let p = designcraft(false);
let mut para = Para::new(COPY, &p, Some(275.0));
for it in para.items.iter_mut().filter(|i| i.kind == Kind::Box) {
it.stretch = [0.0, it.width * box_stretch, it.width * box_stretch];
it.shrink = [0.0, it.width * box_stretch / 2.0, it.width * box_stretch / 2.0];
}
let lines = break_paragraph(&para.items, &|_| 100.0, &p, &PASSES).lines;
lines[..lines.len() - 1].iter().map(|l| l.ratio).fold(0.0, f64::max)
};
let (words, all) = (worst(0.0), worst(0.03));
assert!(all < words, "{all} vs {words}");
}
/// A long ragged paragraph with a constant measure breaks by Knuth–Plass under the default
/// parameters and effectcraft's infinitely tolerant rag, without reaching the active-break limit.
#[test]
fn a_long_ragged_paragraph_stays_on_knuth_plass() {
let mut seed = 1u32;
let mut items = Vec::new();
for i in 0..5000 {
if i > 0 {
items.push(Item::glue(1.0, 0.0, 0.0));
}
seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345);
items.push(Item::boxed(f64::from(3 + (seed >> 16) % 5)));
}
items.extend([Item::fill(), Item::forced()]);
let default = Params { rag: Some(Rag::default()), ..Params::default() };
let effectcraft = Params { rag: Some(Rag { width_fraction: 1.0, ..Rag::default() }), ..Params::default() };
for (p, tolerances) in [(default, &[2.5][..]), (effectcraft, &[f64::INFINITY][..])] {
let b = break_paragraph(&items, &|_| 60.0, &p, tolerances);
assert_eq!(b.method, Method::KnuthPlass { pass: 0 }, "{p:?}");
assert_covers(&items, &b.lines);
}
}
/// A NaN measure counts as 0 and negative fitness demerits as 0.
#[test]
fn invalid_params_are_clamped() {
let p = designcraft(false);
let para = Para::new(COPY, &p, None);
let lines = break_paragraph(&para.items, &|_| f64::NAN, &p, &PASSES).lines;
assert_eq!(lines, break_paragraph(&para.items, &|_| 0.0, &p, &PASSES).lines);
assert!(lines.iter().all(|l| l.measure == 0.0 && !l.excess.is_nan()));
let negative = Params { fitness_demerits: -5000.0, ..p };
let zero = Params { fitness_demerits: 0.0, ..p };
assert_eq!(break_paragraph(&para.items, &|_| 150.0, &negative, &PASSES), break_paragraph(&para.items, &|_| 150.0, &zero, &PASSES));
}
/// Two consecutive breaks: the line after the first starts past the second, which must not end a
/// line `4..3` from there.
#[test]
fn lines_never_end_before_they_start() {
let items = [
Item::boxed(10.0),
Item::glue(1.0, 0.0, 0.0),
Item::penalty(0.0, -1000.0, false),
Item::penalty(0.0, -1000.0, false),
Item::boxed(10.0),
Item::fill(),
Item::forced(),
];
let p = Params { rag: Some(Rag { width_fraction: 1.0, ..Rag::default() }), ..Params::default() };
let lines = break_paragraph(&items, &|_| 11.0, &p, &[2.5, f64::INFINITY]).lines;
assert!(lines.iter().all(|l| l.start <= l.end && l.end <= l.next), "{lines:?}");
assert_covers(&items, &lines);
}
fn spans(lines: &[Line]) -> Vec<(usize, usize)> {
lines.iter().map(|l| (l.start, l.end)).collect()
}
/// A final word wider than the measure is one overfull line: the fill before the forced break
/// doesn't end it and leave an empty line. Consecutive forced breaks still make an empty line.
#[test]
fn an_overlong_last_word_adds_no_empty_line() {
let fill = [Item::boxed(20.0), Item::fill(), Item::forced()];
assert_eq!(spans(&break_paragraph(&fill, &|_| 11.0, &Params::default(), &[2.5, f64::INFINITY]).lines), [(0, 2)]);
let forced = [Item::boxed(20.0), Item::forced(), Item::forced()];
assert_eq!(spans(&break_paragraph(&forced, &|_| 11.0, &Params::default(), &[2.5, f64::INFINITY]).lines), [(0, 1), (2, 2)]);
}
/// Overfull lines keep the hyphen limit too: with a limit of 1, no two consecutive lines end at a
/// hyphen even when every line is overfull.
#[test]
fn overfull_lines_keep_the_hyphen_limit() {
let items =
[Item::boxed(10.0), Item::penalty(1.0, 0.0, true), Item::boxed(10.0), Item::penalty(1.0, 0.0, true), Item::boxed(10.0), Item::forced()];
let p = Params { hyphen_limit: 1, ..Params::default() };
let lines = break_paragraph(&items, &|_| 5.0, &p, &[2.5, f64::INFINITY]).lines;
assert!(lines.windows(2).all(|w| !(w[0].flagged && w[1].flagged)), "{lines:?}");
assert_eq!(spans(&lines), [(0, 1), (2, 5)]);
}
crates/text/src/segment/complex.rs
+50
View File
@@ -0,0 +1,50 @@
//! Word breaks inside Thai, Lao, Khmer and Burmese runs. UAX #14 gives these scripts class SA
//! ("complex context") and leaves breaking inside them to a dictionary; ICU4X's segmenter supplies
//! it, with its dictionaries or, with the `lstm` feature, its LSTM models. Only runs of the scripts
//! whose language feature is on are segmented.
use icu_segmenter::options::LineBreakOptions;
use icu_segmenter::{LineSegmenter, LineSegmenterBorrowed};
use unicode_linebreak::{BreakClass, break_property};
/// Builds ICU4X's line segmenter with one model for the complex scripts. The models are static
/// data; building the segmenter only looks them up.
pub(super) type Model = fn(LineBreakOptions<'static>) -> LineSegmenterBorrowed<'static>;
/// The model `line_breaks` uses: the LSTM models with the `lstm` feature, else the dictionaries.
#[cfg(feature = "lstm")]
pub(super) const MODEL: Model = LineSegmenter::new_lstm;
#[cfg(not(feature = "lstm"))]
pub(super) const MODEL: Model = LineSegmenter::new_dictionary;
/// Byte offsets inside `text` where a run of an enabled script may break between words, in order.
/// Run starts and ends are left to UAX #14.
pub(super) fn word_breaks(text: &str, model: Model) -> Vec<usize> {
let mut out = Vec::new();
let mut segmenter = None;
let mut run_start = None;
let ends = text.char_indices().map(|(i, c)| (i, segmented(c))).chain(std::iter::once((text.len(), false)));
for (i, inside) in ends {
match (run_start, inside) {
(None, true) => run_start = Some(i),
(Some(start), false) => {
run_start = None;
let Some(run) = text.get(start..i) else { continue };
let seg = *segmenter.get_or_insert_with(|| model(LineBreakOptions::default()));
out.extend(seg.segment_str(run).filter(|&o| o > 0 && o < run.len()).map(|o| start + o));
}
_ => {}
}
}
out
}
/// A character of an enabled script that UAX #14 leaves to the word segmenter.
fn segmented(c: char) -> bool {
let u = c as u32;
let enabled = (cfg!(feature = "thai") && matches!(u, 0x0E00..=0x0E7F))
|| (cfg!(feature = "lao") && matches!(u, 0x0E80..=0x0EFF))
|| (cfg!(feature = "khmer") && matches!(u, 0x1780..=0x17FF | 0x19E0..=0x19FF))
|| (cfg!(feature = "burmese") && matches!(u, 0x1000..=0x109F | 0xA9E0..=0xA9FF | 0xAA60..=0xAA7F));
enabled && break_property(u) == BreakClass::ComplexContext
}
crates/text/src/segment/kinsoku.rs
+119
View File
@@ -0,0 +1,119 @@
//! Kinsoku character sets (JLREQ 3.1.7–3.1.8, CLREQ 3.1.7). The sets are VectorCraft's, with the
//! additions listed in the crate README.
use super::Kinsoku;
/// Whether kinsoku lets a line break between `before` and `after` (`None`: the end of the
/// paragraph). `Hard` keeps every character of the set off the line start and end; `Soft` lets
/// 々, ー and small kana start a line; `None` allows every break. Breaks with no East Asian
/// character on either side are always allowed, so Latin text keeps its own rules.
pub fn kinsoku_allows(before: char, after: Option<char>, mode: Kinsoku) -> bool {
if mode == Kinsoku::None || (!is_east_asian(before) && !after.is_some_and(is_east_asian)) {
return true;
}
let start_ok = |c: char| !no_line_start(c) || (mode == Kinsoku::Soft && soft_line_start(c));
!no_line_end(before) && after.is_none_or(start_ok)
}
/// East Asian text for kinsoku: Han, kana, bopomofo, Hangul syllables and compatibility jamo, CJK
/// punctuation and symbols, full-width and half-width forms. Not the ideographic space, conjoining
/// jamo or the (combining and spacing) kana sound marks, which never take a break before them.
pub(crate) fn is_east_asian(c: char) -> bool {
matches!(c as u32,
0x2E80..=0x2FFF // radicals, ideographic description
| 0x3001..=0x3096 // CJK symbols and punctuation, hiragana
| 0x309D..=0x30FF // hiragana iteration marks, katakana
| 0x3100..=0x312F // bopomofo
| 0x3131..=0x318F // Hangul compatibility jamo
| 0x3190..=0x33FF // kanbun, strokes, small katakana, enclosed and compatibility forms
| 0x3400..=0x4DBF // CJK extension A
| 0x4E00..=0x9FFF // CJK unified ideographs
| 0xAC00..=0xD7A3 // Hangul syllables
| 0xF900..=0xFAFF // compatibility ideographs
| 0xFE30..=0xFE4F // CJK compatibility forms
| 0xFF01..=0xFF9D // full-width forms, half-width katakana
| 0xFFA0..=0xFFEF // half-width Hangul, full-width signs
| 0x20000..=0x3FFFF) // supplementary ideographs
}
/// A character that can't start a line: closing brackets and quotes, hyphens, dividing
/// punctuation, middle dots, colons, full stops and commas, iteration marks, the prolonged sound
/// mark, small kana, kana sound marks and postfixed abbreviations (JLREQ cl-02 to cl-07, cl-09 to
/// cl-11, cl-13).
pub(crate) fn no_line_start(c: char) -> bool {
matches!(c,
')' | ']' | '}' | ',' | '.' | ':' | ';' | '!' | '?' | '»' | '’' | '”' | '‐' | '–' | '‼' | '⁇' | '⁈' | '⁉'
| '、' | '。' | '〉' | '》' | '」' | '』' | '】' | '〕' | '〗' | '〙' | '〛' | '〞' | '〟' | '〜' | '゠' | '・' | '｠'
| 'ー' | 'ゝ' | 'ゞ' | 'ヽ' | 'ヾ' | '々' | '〻' | '゛' | '゜'
| 'ぁ' | 'ぃ' | 'ぅ' | 'ぇ' | 'ぉ' | 'っ' | 'ゃ' | 'ゅ' | 'ょ' | 'ゎ' | 'ゕ' | 'ゖ'
| 'ァ' | 'ィ' | 'ゥ' | 'ェ' | 'ォ' | 'ッ' | 'ャ' | 'ュ' | 'ョ' | 'ヮ' | 'ヵ' | 'ヶ' | 'ㇰ'..='ㇿ'
| '！' | '％' | '）' | '，' | '．' | '：' | '；' | '？' | '］' | '｝' | '～' | '｡' | '｣' | '､' | '･' | 'ｰ' | 'ｧ'..='ｯ'
| 'ﾞ' | 'ﾟ' | '￠')
}
/// A character of [`no_line_start`] that may start a line under [`Kinsoku::Soft`]: 々, the
/// prolonged sound mark ー (JLREQ cl-10) and small kana (cl-11), as JLREQ's level 3 rules allow
/// (Appendix C.3, <https://www.w3.org/TR/jlreq/#addendum_a3>). Half-width forms aren't in those
/// classes.
pub(crate) fn soft_line_start(c: char) -> bool {
matches!(
c,
'々' | 'ー'
| 'ぁ'
| 'ぃ'
| 'ぅ'
| 'ぇ'
| 'ぉ'
| 'っ'
| 'ゃ'
| 'ゅ'
| 'ょ'
| 'ゎ'
| 'ゕ'
| 'ゖ'
| 'ァ'
| 'ィ'
| 'ゥ'
| 'ェ'
| 'ォ'
| 'ッ'
| 'ャ'
| 'ュ'
| 'ョ'
| 'ヮ'
| 'ヵ'
| 'ヶ'
| 'ㇰ'..='ㇿ'
)
}
/// A character that can't end a line: opening brackets and quotes, and prefixed abbreviations
/// (JLREQ cl-01, cl-12).
pub(crate) fn no_line_end(c: char) -> bool {
matches!(
c,
'(' | '['
| '{'
| '«'
| '‘'
| '“'
| '〈'
| '《'
| '「'
| '『'
| '【'
| '〔'
| '〖'
| '〘'
| '〚'
| '〝'
| '（'
| '［'
| '｛'
| '｟'
| '｢'
| '＄'
| '￡'
| '￥'
)
}
crates/text/src/segment/mod.rs
+164
View File
@@ -0,0 +1,164 @@
//! Line-break opportunities for text layout: where a composer may end a line, as byte offsets into
//! the paragraph. Font-independent; measuring and choosing among the breaks stays in the composer.
//!
//! [`line_breaks`] starts from the Unicode line-breaking algorithm (UAX #14, through
//! [`unicode_linebreak`]) and adjusts it:
//! - **Kinsoku** ([`Kinsoku`]): the Japanese and Chinese line-start and line-end rules (JLREQ,
//! CLREQ). Closing brackets, commas, full stops, small kana and the like don't start a line;
//! opening brackets don't end one. The rules apply only next to East Asian characters, so text
//! without them gets UAX #14's breaks unchanged.
//! - **Korean words** ([`Options::korean_words`]): Korean set word by word, breaking at spaces.
//! - **Thai, Lao, Khmer and Burmese**: UAX #14 leaves these scripts (class SA) to a dictionary and
//! never breaks inside a run of them. With the language feature (`thai`, `lao`, `khmer`,
//! `burmese`) the run is split into words by ICU4X's segmenter: its dictionaries by default, its
//! much smaller LSTM models with the `lstm` feature.
//!
//! [`Options::for_language`] picks the options for a paragraph's language. When no opportunity
//! leaves a line short enough, [`emergency_breaks`] gives the places to break a word anyway:
//! between grapheme clusters, never inside one.
//!
//! Composers that build their own opportunity lists can filter them with [`kinsoku_allows`].
#[cfg(any(feature = "thai", feature = "lao", feature = "khmer", feature = "burmese"))]
mod complex;
mod kinsoku;
#[cfg(test)]
mod tests;
use std::ops::Range;
use unicode_linebreak::{BreakClass, BreakOpportunity, break_property};
pub use kinsoku::kinsoku_allows;
use kinsoku::{is_east_asian, soft_line_start};
use unicode_segmentation::UnicodeSegmentation;
/// Which kinsoku (line-start and line-end) rules apply.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum Kinsoku {
/// No kinsoku: a line may also break between any two East Asian characters, punctuation
/// included.
None,
/// Closing brackets, commas, full stops, iteration marks, the prolonged sound mark, small kana
/// and the like don't start a line; opening brackets don't end one.
#[default]
Hard,
/// As `Hard`, except that 々, the prolonged sound mark ー and small kana may start a line
/// (JLREQ's relaxed rules, Appendix C.3).
Soft,
}
/// How [`line_breaks`] adjusts UAX #14.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
pub struct Options {
pub kinsoku: Kinsoku,
/// Set Korean word by word (KLREQ): no break between two Hangul letters or Han ideographs
/// (hanja), only at spaces and punctuation. For Korean-language text only, since Han
/// ideographs count as Korean letters. Off, Korean breaks between syllables as UAX #14 does.
pub korean_words: bool,
}
impl Options {
/// The options for text in `language`, a BCP 47 tag such as `ja` or `ko-KR` (`None`: unknown).
/// Korean is set word by word; every other language, and an unknown one, gets the defaults.
/// Thai, Lao, Khmer and Burmese words are found by script, whatever the language.
pub fn for_language(language: Option<&str>) -> Self {
let primary = language.and_then(|tag| tag.split(['-', '_']).next()).unwrap_or_default();
Self { korean_words: primary.eq_ignore_ascii_case("ko"), ..Self::default() }
}
}
/// A place where a line may end: the next line starts at byte `offset`.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub struct Break {
pub offset: usize,
/// The line must end here (after a newline or another hard break, and at the end of the text).
pub mandatory: bool,
}
/// Line-break opportunities in `text`, in order. The last one is the end of the text (mandatory);
/// empty text has none.
pub fn line_breaks(text: &str, options: &Options) -> Vec<Break> {
#[cfg(any(feature = "thai", feature = "lao", feature = "khmer", feature = "burmese"))]
let words = complex::word_breaks(text, complex::MODEL);
#[cfg(not(any(feature = "thai", feature = "lao", feature = "khmer", feature = "burmese")))]
let words = Vec::new();
breaks_with_words(text, options, words)
}
/// [`line_breaks`], given the word breaks inside Thai, Lao, Khmer and Burmese runs (sorted).
fn breaks_with_words(text: &str, options: &Options, words: Vec<usize>) -> Vec<Break> {
if text.is_empty() {
return Vec::new();
}
let mut uax = unicode_linebreak::linebreaks(text).peekable();
let mut words = words.into_iter().peekable();
let mut out = Vec::new();
// The last character before the boundary that isn't a combining mark: kinsoku looks at the
// character a mark belongs to.
let mut base: Option<char> = None;
let mut after_mark = false;
for (i, c) in text.char_indices() {
if i > 0 {
// Both lists are sorted by offset: take the entry for `i` from each, if it has one.
let uax_here = uax.next_if(|&(o, _)| o == i).map(|(_, op)| op);
let word = words.next_if(|&o| o == i).is_some();
let allowed = uax_here.is_some() || word;
let brk = if uax_here == Some(BreakOpportunity::Mandatory) {
Some(true)
} else {
adjust(allowed, base, after_mark, c, options).then_some(false)
};
if let Some(mandatory) = brk {
out.push(Break { offset: i, mandatory });
}
}
after_mark = is_mark(c);
if !after_mark {
base = Some(c);
}
}
out.push(Break { offset: text.len(), mandatory: true });
out
}
/// Emergency breaks for `text[range]`, a piece between two [`line_breaks`] that is too long for a
/// line: byte offsets into `text`, strictly inside `range` and in order, between extended grapheme
/// clusters (UAX #29). An emoji ZWJ sequence, a flag or a letter with its marks stays whole. Empty
/// if `range` isn't on char boundaries of `text`.
pub fn emergency_breaks(text: &str, range: Range<usize>) -> Vec<usize> {
let Some(piece) = text.get(range.clone()) else { return Vec::new() };
piece.grapheme_indices(true).skip(1).map(|(o, _)| range.start + o).collect()
}
/// Whether a line may break between `before` and `after`, given whether UAX #14 (or the word
/// segmenter) allows it. `after_mark`: the character right before the boundary is a combining
/// mark, joiner or variation selector that belongs to `before`.
fn adjust(allowed: bool, before: Option<char>, after_mark: bool, after: char, options: &Options) -> bool {
let Some(before) = before else { return allowed };
// The East Asian breaks UAX #14 lacks go only between two base characters: never inside a
// cluster, so not after a joiner or before a mark.
let east_asian_pair = !after_mark && is_east_asian(before) && is_east_asian(after) && !is_mark(after);
let allowed = match options.kinsoku {
Kinsoku::None => allowed || east_asian_pair,
// UAX #14 keeps small kana, ー and 々 (classes CJ and NS) off the line start.
Kinsoku::Soft => (allowed || (east_asian_pair && soft_line_start(after))) && kinsoku_allows(before, Some(after), Kinsoku::Soft),
Kinsoku::Hard => allowed && kinsoku_allows(before, Some(after), Kinsoku::Hard),
};
allowed && !(options.korean_words && is_korean_letter(before) && is_korean_letter(after))
}
/// A combining mark, variation selector or joiner (UAX #14 classes CM and ZWJ): it belongs to the
/// character before it.
fn is_mark(c: char) -> bool {
matches!(break_property(c as u32), BreakClass::CombiningMark | BreakClass::ZeroWidthJoiner)
}
/// Hangul in every form, and Han ideographs (read as hanja).
fn is_korean_letter(c: char) -> bool {
matches!(c as u32,
0x1100..=0x11FF | 0x3131..=0x318F | 0xA960..=0xA97F | 0xAC00..=0xD7FF | 0xFFA0..=0xFFDC
| 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF | 0x20000..=0x3FFFF)
}
crates/text/src/segment/tests.rs
+242
View File
@@ -0,0 +1,242 @@
use super::kinsoku::{no_line_end, no_line_start};
use super::*;
fn opts(kinsoku: Kinsoku) -> Options {
Options { kinsoku, ..Options::default() }
}
/// Byte offsets of the breaks before the end of the text.
fn offsets(text: &str, options: &Options) -> Vec<usize> {
line_breaks(text, options).iter().map(|b| b.offset).filter(|&o| o < text.len()).collect()
}
/// The text from each break before the end to the next break.
fn pieces(text: &str, options: &Options) -> Vec<String> {
let mut from = 0;
let mut out = Vec::new();
for b in line_breaks(text, options) {
out.push(text.get(from..b.offset).unwrap_or_default().to_string());
from = b.offset;
}
out
}
/// The character each line after the first could start with.
fn starts(text: &str, options: &Options) -> Vec<char> {
offsets(text, options).iter().filter_map(|&o| text.get(o..).and_then(|t| t.chars().next())).collect()
}
#[test]
fn kinsoku_keeps_closing_marks_off_line_starts_and_opening_ones_off_line_ends() {
for (a, b) in
[('字', '。'), ('弧', '」'), ('」', '、'), ('ャ', 'ー'), ('カ', 'ッ'), ('「', 'か'), ('（', '雅'), ('字', '〞'), ('１', '％'), ('＄', '１')]
{
assert!(!kinsoku_allows(a, Some(b), Kinsoku::Hard), "{a}{b}");
}
for (a, b) in [('。', '雅'), ('」', 'は'), ('字', '「'), ('の', '演')] {
assert!(kinsoku_allows(a, Some(b), Kinsoku::Hard), "{a}{b}");
}
assert!(kinsoku_allows('。', None, Kinsoku::Hard));
// No East Asian character on either side: Latin rules apply.
assert!(kinsoku_allows(' ', Some('–'), Kinsoku::Hard));
assert!(!kinsoku_allows('字', Some(')'), Kinsoku::Hard));
}
/// Soft kinsoku lets 々, ー and small kana (JLREQ cl-09's 々, cl-10, cl-11) start a line and keeps
/// the rest of the set (half-width forms aren't in those classes); None allows any break.
#[test]
fn soft_kinsoku_lets_small_kana_and_the_prolonged_sound_mark_start_a_line() {
for (a, b) in [('ャ', 'ー'), ('カ', 'ッ'), ('時', '々'), ('ト', 'ㇰ'), ('か', 'ゃ')] {
assert!(!kinsoku_allows(a, Some(b), Kinsoku::Hard), "{a}{b}");
assert!(kinsoku_allows(a, Some(b), Kinsoku::Soft), "{a}{b}");
}
for (a, b) in [('字', '。'), ('弧', '」'), ('「', 'か'), ('ｶ', 'ｯ'), ('ｶ', 'ｰ'), ('字', 'ゝ')] {
assert!(!kinsoku_allows(a, Some(b), Kinsoku::Soft), "{a}{b}");
assert!(kinsoku_allows(a, Some(b), Kinsoku::None), "{a}{b}");
}
}
#[test]
fn no_break_puts_a_closing_mark_at_a_line_start_or_an_opening_one_at_a_line_end() {
let text = "あいうえお。かきくけ「こさしすせそ」";
let breaks = offsets(text, &Options::default());
assert!(breaks.len() > 10, "{breaks:?}");
for o in breaks {
let (head, tail) = text.split_at(o);
let (Some(last), Some(first)) = (head.chars().next_back(), tail.chars().next()) else { continue };
assert!(!no_line_start(first) && !no_line_end(last), "break at {o}: {last}|{first}");
}
}
/// あカッ breaks before ッ with Soft and None (a small kana may start a line) and not with Hard; a
/// full stop never starts a line, Soft or not.
#[test]
fn kinsoku_mode_decides_whether_a_small_kana_may_start_a_line() {
assert_eq!(starts("あカッ", &opts(Kinsoku::Hard)), ['カ']);
assert_eq!(starts("あカッ", &opts(Kinsoku::Soft)), ['カ', 'ッ']);
assert_eq!(starts("あカッ", &opts(Kinsoku::None)), ['カ', 'ッ']);
assert_eq!(starts("あカ。", &opts(Kinsoku::Soft)), ['カ']);
assert_eq!(starts("あカ。", &opts(Kinsoku::None)), ['カ', '。']);
}
#[test]
fn kinsoku_adjusts_uax14_next_to_east_asian_text_only() {
let ja = "日本語、「漢字」。テスト";
let on = starts(ja, &opts(Kinsoku::Hard));
for c in ['、', '」', '。', '漢'] {
assert!(!on.contains(&c), "kinsoku: no line starts with {c}: {on:?}");
}
assert!(on.contains(&'「') && on.contains(&'テ'), "{on:?}");
let off = starts(ja, &opts(Kinsoku::None));
for c in ['、', '」', '。', '漢'] {
assert!(off.contains(&c), "no kinsoku: a line may start with {c}: {off:?}");
}
let en = "Hello wonderful (world), 50%.";
for k in [Kinsoku::None, Kinsoku::Soft] {
assert_eq!(starts(en, &opts(k)), starts(en, &opts(Kinsoku::Hard)));
}
assert_eq!(starts(en, &Options::default()), ['w', '(', '5']);
}
#[test]
fn breaks_are_byte_offsets_and_newlines_are_mandatory() {
let b = line_breaks("a b\nc", &Options::default());
let got: Vec<(usize, bool)> = b.iter().map(|b| (b.offset, b.mandatory)).collect();
assert_eq!(got, [(2, false), (4, true), (5, true)]);
assert!(line_breaks("", &Options::default()).is_empty());
// A variation selector belongs to its ideograph: no break before it, kinsoku reads the base.
assert_eq!(pieces("字\u{FE00}字\u{FE00}。", &Options::default()), ["字\u{FE00}", "字\u{FE00}。"]);
}
/// The East Asian breaks that None and Soft add to UAX #14 never split a cluster: not after a
/// joiner, not before a combining mark or variation selector.
#[test]
fn no_kinsoku_mode_breaks_inside_a_cluster() {
for k in [Kinsoku::None, Kinsoku::Soft, Kinsoku::Hard] {
let o = opts(k);
assert_eq!(pieces("日本\u{200D}語", &o), ["日", "本\u{200D}語"], "{k:?}");
assert_eq!(pieces("あカ\u{200D}ッ", &o), ["あ", "カ\u{200D}ッ"], "{k:?}");
assert_eq!(pieces("葛\u{E0100}城", &o), ["葛\u{E0100}", "城"], "{k:?}");
assert_eq!(pieces("字\u{3099}\u{FE00}字", &o), ["字\u{3099}\u{FE00}", "字"], "{k:?}");
}
}
#[test]
fn korean_words_break_at_spaces_only() {
let text = "한국어 문장(文章)입니다";
assert_eq!(pieces(text, &Options::default()), ["한", "국", "어 ", "문", "장", "(文", "章)", "입", "니", "다"]);
let words = Options { korean_words: true, ..Options::default() };
assert_eq!(pieces(text, &words), ["한국어 ", "문장", "(文章)", "입니다"]);
}
#[cfg(not(feature = "thai"))]
#[test]
fn thai_without_its_feature_breaks_as_uax14() {
assert_eq!(pieces("ภาษาไทยภาษาไทย ไทย", &Options::default()), ["ภาษาไทยภาษาไทย ", "ไทย"]);
}
/// Each model available in this build, for the complex-script tests.
#[cfg(any(feature = "thai", feature = "lao", feature = "khmer", feature = "burmese"))]
fn models() -> Vec<(&'static str, complex::Model)> {
use icu_segmenter::LineSegmenter;
#[allow(unused_mut)]
let mut models: Vec<(&str, complex::Model)> = vec![("dictionary", LineSegmenter::new_dictionary)];
#[cfg(feature = "lstm")]
models.push(("lstm", LineSegmenter::new_lstm));
models
}
/// Breaks every model puts in `words.concat()` fall between two of `words`, and there are at least
/// `min` of them. `words` splits compounds where a model may break them.
#[cfg(any(feature = "thai", feature = "lao", feature = "khmer", feature = "burmese"))]
fn assert_word_breaks(words: &[&str], min: usize) {
let text = words.concat();
let bounds: Vec<usize> = words
.iter()
.scan(0, |end, w| {
*end += w.len();
Some(*end)
})
.collect();
for (name, model) in models() {
let breaks = breaks_with_words(&text, &Options::default(), complex::word_breaks(&text, model));
let inside: Vec<usize> = breaks.iter().map(|b| b.offset).filter(|&o| o < text.len()).collect();
let shown: Vec<&str> = std::iter::once(0)
.chain(inside.iter().copied())
.zip(inside.iter().copied().chain([text.len()]))
.filter_map(|(a, b)| text.get(a..b))
.collect();
assert!(inside.iter().all(|o| bounds.contains(o)), "{name}: {shown:?}");
assert!(inside.len() >= min, "{name}: {shown:?}");
}
}
#[cfg(feature = "thai")]
#[test]
fn thai_breaks_between_words() {
assert_word_breaks(&["Thai: ", "ภาษา", "ไทย", "เป็น", "ภาษา", "ราชการ", "ของ", "ประเทศ", "ไทย."], 6);
}
#[cfg(feature = "lao")]
#[test]
fn lao_breaks_between_words() {
assert_word_breaks(&["ພາສາ", "ລາວ", "ເປັນ", "ພາສາ", "ລາດ", "ຊະ", "ການ", "ຂອງ", "ປະເທດ", "ລາວ"], 5);
}
#[cfg(feature = "khmer")]
#[test]
fn khmer_breaks_between_words() {
assert_word_breaks(&["ភាសា", "ខ្មែរ", "ជា", "ភាសា", "ផ្លូវ", "ការ", "នៃ", "ប្រទេស", "កម្ពុជា"], 5);
}
/// Burmese may break between any two syllables, and the LSTM model splits some words there
/// (မြန်|မာ): the expected breaks are the syllable boundaries.
#[cfg(feature = "burmese")]
#[test]
fn burmese_breaks_between_syllables() {
let syllables = ["မြန်", "မာ", "ဘာ", "သာ", "စ", "ကား", "သည်", "မြန်", "မာ", "နိုင်", "ငံ", "၏", "ရုံး", "သုံး", "ဘာ", "သာ", "စ", "ကား", "ဖြစ်", "သည်"];
assert_word_breaks(&syllables, 5);
}
/// Chinese full-width punctuation: no line starts with a closing mark, none ends with an opening
/// one.
#[test]
fn chinese_kinsoku_keeps_punctuation_with_its_text() {
let text = "他说：“《红楼梦》是一部小说，（很长）。”";
let o = Options::for_language(Some("zh-Hans"));
let breaks = offsets(text, &o);
assert!(breaks.len() > 8, "{:?}", pieces(text, &o));
for b in breaks {
let (head, tail) = text.split_at(b);
let (Some(last), Some(first)) = (head.chars().next_back(), tail.chars().next()) else { continue };
assert!(!"：”》，）。".contains(first), "line starts with {first}: {:?}", pieces(text, &o));
assert!(!"“《（".contains(last), "line ends with {last}: {:?}", pieces(text, &o));
}
}
#[test]
fn language_picks_korean_words() {
for tag in ["ko", "ko-KR", "KO_kr"] {
assert!(Options::for_language(Some(tag)).korean_words, "{tag}");
}
for tag in [Some("ja"), Some("kok"), Some("en-KO"), None] {
assert_eq!(Options::for_language(tag), Options::default(), "{tag:?}");
}
}
/// Emergency breaks fall between grapheme clusters: never inside an emoji ZWJ sequence, a flag, a
/// skin-tone modifier or a letter with its combining marks.
#[test]
fn emergency_breaks_keep_grapheme_clusters_whole() {
let family = "👨\u{200D}👩\u{200D}👧\u{200D}👦";
let text = format!("ab{family}e\u{301}🇯🇵👍🏽");
let breaks = emergency_breaks(&text, 0..text.len());
let a = family.len() + 2;
let b = a + "e\u{301}".len();
let c = b + "🇯🇵".len();
assert_eq!(breaks, [1, 2, a, b, c]);
// Offsets are into `text`; a range off a char boundary has none.
assert_eq!(emergency_breaks(&text, 1..b), [2, a]);
assert!(emergency_breaks(&text, 3..b).is_empty());
assert!(emergency_breaks("", 0..0).is_empty());
}