M3: merge main's soft hyphen, zero width and CJK line-join search with the ligature and line-end hyphen readings (#4469)

Main's #6595 made a soft hyphen invisible and a line ending in one run on, so
the line-end hyphen readings now cover HYPHEN-MINUS and HYPHEN only, pass over
invisible characters, and keep main's zero width space word boundaries in each
reading. The reference oracle combines both.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Pq7HyCHD1R3rywQBPCHXZA
This commit is contained in:
mtlcl410andClaude Opus 5.5 committed 2026-11-09 10:29:15 +08:00
1 parent 2904866c3b
commit f86f709c3e
2 files changed
+11 -3

No files matched your search

+8 -1
View File
@@ -199,7 +199,14 @@ fn characters(page: &PageText, texts: &[String], divided: &[bool], reading: Hyph
/// Non-overlapping hits of `needle` (with its KMP `prefix` table) in `chars`, as glyph ranges.
/// `zero_width_space` says which characters come right after a skipped zero width space.
fn search(chars: &[(char, usize)], zero_width_space: &[bool], needle: &[char], prefix: &[usize], whole_words: bool, limit: usize) -> Vec<Range<usize>> {
fn search(
chars: &[(char, usize)],
zero_width_space: &[bool],
needle: &[char],
prefix: &[usize],
whole_words: bool,
limit: usize,
) -> Vec<Range<usize>> {
if needle.len() > chars.len() {
return Vec::new();
}
+3 -2
View File
@@ -202,8 +202,9 @@ fn whole_word_rejection_keeps_overlapping_candidates() {
#[test]
fn search_matches_reference_for_unicode_gaps_limits_and_word_boundaries() {
let alphabet =
["a", "b", " ", "-", "İ", "ı", "ς", "Σ", "\t", "\n", "ffi", "", "界", "e\u{301}", "العربية", "\u{FB01}", "b\u{AD}", "1", "\u{AD}", "a\u{200B}", "\u{2010}"];
let alphabet = ["a", "b", " ", "-", "İ", "ı", "ς", "Σ", "\t", "\n", "ffi", "", "界", "e\u{301}", "العربية", "\u{AD}", "a\u{200B}"];
// Ligatures, digits and HYPHEN exercise the readings of a line-end hyphen.
let alphabet: Vec<&str> = alphabet.into_iter().chain(["\u{FB01}", "1", "\u{2010}"]).collect();
let mut seed = 1234567u64;
let mut next = || {
seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);