M3: Find matches words divided by soft hyphens, zero width characters or a Chinese or Japanese line end (view.find) (#6595)

crates/render/src/text.rs

@@ -131,7 +131,9 @@ impl PageText {
     }
 
     /// Case-insensitive search. Returns glyph ranges of every match (words may span line breaks,
-    /// which count as a single space).
+    /// which count as a single space, or as nothing where the line runs on between two Chinese or
+    /// Japanese characters or ends in a soft hyphen: there a query with a space across the break
+    /// doesn't match, and one without does).
     pub fn find(&self, needle: &str) -> Vec<std::ops::Range<usize>> {
         self.find_opts(needle, false, false)
     }

crates/render/src/text/search.rs

@@ -1,4 +1,8 @@
 //! Literal search in normalized page text, retaining the original glyph owners.
+//!
+//! Words and lines are separated by one space, except where a line of Chinese or Japanese runs on
+//! to the next: a break between two such characters is no word break, so it reads as nothing and
+//! "検知機構" is found when the line ends after "検知".
 use super::{PageText, fold_case};
 use std::ops::Range;
 
@@ -7,21 +11,51 @@ pub(super) fn find(page: &PageText, query: &str, sensitive: bool, whole_words: b
         return Vec::new();
     }
     let fold = |s: &str| if sensitive { s.to_owned() } else { fold_case(s) };
-    let needle: Vec<char> = fold(query).split_whitespace().collect::<Vec<_>>().join(" ").chars().collect();
+    let query: String = fold(query).chars().filter(|&c| !is_invisible(c)).collect();
+    let needle: Vec<char> = query.split_whitespace().collect::<Vec<_>>().join(" ").chars().collect();
     if needle.is_empty() {
         return Vec::new();
     }
     let mut chars: Vec<(char, usize)> = Vec::new();
+    // Whether a skipped ZERO WIDTH SPACE came right before each character: it draws nothing, but
+    // it is a word boundary (UAX #29 breaks on both sides of it), which Thai and Khmer write between words
+    // with no visible gap, so whole-word search still sees two words there.
+    let mut zero_width_space: Vec<bool> = Vec::new();
+    let mut pending_zero_width_space = false;
+    // The last character pushed other than a variation selector, kept as characters are pushed so
+    // a long run of selectors costs no rescan.
+    let mut last_base: Option<char> = None;
     for (i, glyph) in page.glyphs.iter().enumerate() {
+        let text = fold(&glyph.text);
         let line_break = i.checked_sub(1).and_then(|previous| page.line_of.get(previous)).zip(page.line_of.get(i)).is_some_and(|(a, b)| a != b);
-        let gap = page.space_before.get(i).copied().unwrap_or(false) || line_break;
+        // A soft hyphen marks where a word may be divided at a line end (ISO 32000-2 §14.8.2.3), so
+        // a line that ends in one runs on to the next with no word break between them.
+        // Other invisible characters after the soft hyphen don't hide it.
+        let soft_break = line_break && last_kept(page, i) == Some('\u{AD}');
+        // So does a line of Chinese or Japanese that runs on to the next. The characters on both
+        // sides of the break are the visible ones: invisible characters are skipped below, and a
+        // variation selector after the last one only picks its glyph.
+        let joined = line_break && last_base.is_some_and(unspaced) && first_visible(page, i).is_some_and(unspaced) && wraps(page, i);
+        let gap = page.space_before.get(i).copied().unwrap_or(false) || (line_break && !(joined || soft_break));
         if i > 0 && gap && chars.last().is_some_and(|c| c.0 != ' ') {
             chars.push((' ', i));
+            last_base = Some(' ');
+            zero_width_space.push(false);
+            pending_zero_width_space = false;
         }
-        for c in fold(&glyph.text).chars() {
+        for c in text.chars() {
+            if is_invisible(c) {
+                pending_zero_width_space |= c == '\u{200B}';
+                continue;
+            }
             let c = if c.is_whitespace() { ' ' } else { c };
             if c != ' ' || chars.last().is_none_or(|last| last.0 != ' ') {
                 chars.push((c, i));
+                if !is_variation_selector(c) {
+                    last_base = Some(c);
+                }
+                zero_width_space.push(pending_zero_width_space && c != ' ');
+                pending_zero_width_space = false;
             }
         }
     }
@@ -48,7 +82,9 @@ pub(super) fn find(page: &PageText, query: &str, sensitive: bool, whole_words: b
         }
         let end = i.saturating_add(1);
         let start = end.saturating_sub(needle.len());
-        let bounded = !whole_words || (!word(start.checked_sub(1)) && !word(Some(end)));
+        // A skipped zero width space before the first character or after the last is a boundary.
+        let after_break = |i: usize| zero_width_space.get(i).copied().unwrap_or(false);
+        let bounded = !whole_words || ((after_break(start) || !word(start.checked_sub(1))) && (after_break(end) || !word(Some(end))));
         if bounded {
             if let Some((_, first)) = chars.get(start)
                 && let Some(last) = owner.checked_add(1)
@@ -68,6 +104,100 @@ pub(super) fn find(page: &PageText, query: &str, sensitive: bool, whole_words: b
     hits
 }
 
+/// A character that draws nothing inside a word and only tells layout where a line may break or
+/// letters may join: SOFT HYPHEN, ZERO WIDTH SPACE, ZERO WIDTH NON-JOINER and JOINER, WORD JOINER
+/// and ZERO WIDTH NO-BREAK SPACE. All six are Unicode default-ignorable format characters, so
+/// find skips them in the page text and in the query alike: "Mitarbeiterinnen" matches a word
+/// stored with soft hyphens. A visible word gap is its own space in the page text, so skipping a
+/// zero width space next to one still keeps the two words apart.
+fn is_invisible(c: char) -> bool {
+    matches!(c, '\u{00AD}' | '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{2060}' | '\u{FEFF}')
+}
+
+/// A variation selector (VS1-VS16 and VS17-VS256): it picks a variant glyph of the character
+/// before it and is no character of its own.
+fn is_variation_selector(c: char) -> bool {
+    matches!(c as u32, 0xFE00..=0xFE0F | 0xE0100..=0xE01EF)
+}
+
+/// The last character before glyph `i` on the line that glyph `i - 1` ends, other than the
+/// invisible characters find skips; a soft hyphen counts, since it marks a divided word. Looks
+/// back no further than that line's start, so each line is read once.
+fn last_kept(page: &PageText, i: usize) -> Option<char> {
+    let end = i.checked_sub(1)?;
+    let line = page.line_of.get(end)?;
+    (0..=end)
+        .rev()
+        .take_while(|&k| page.line_of.get(k) == Some(line))
+        .filter_map(|k| page.glyphs.get(k))
+        .flat_map(|g| g.text.chars().rev())
+        .find(|&c| c == '\u{AD}' || !is_invisible(c))
+}
+
+/// Whether `c` is set solid, with no spaces between words, in text that a line break may divide
+/// anywhere: East Asian Width Wide, Fullwidth or Halfwidth (Unicode Standard Annex #11) Han,
+/// kana, bopomofo, CJK symbols and punctuation and fullwidth forms other than Latin letters and
+/// digits. Hangul is left out: Korean separates its words with spaces, so a line end there is a
+/// word break. A line break between two such characters is the CSS segment-break rule's case for removing the break, not turning
+/// it into a space (CSS Text Module, segment break transformation).
+fn unspaced(c: char) -> bool {
+    matches!(c as u32,
+        0x2E80..=0x2FFF // CJK radicals, Kangxi radicals, ideographic description characters
+        | 0x3001..=0x303E // CJK symbols and punctuation (U+3000 is a space; U+303F is narrow)
+        | 0x3041..=0x30FF // Hiragana, Katakana
+        | 0x3105..=0x312F // Bopomofo
+        | 0x3190..=0x31FF // Kanbun, Bopomofo extended, CJK strokes, Katakana phonetic extensions
+        | 0x3200..=0x4DBF // Enclosed CJK letters, CJK compatibility, Han extension A
+        | 0x4E00..=0x9FFF // Han
+        | 0xF900..=0xFAFF // CJK compatibility ideographs
+        | 0xFE10..=0xFE19 | 0xFE30..=0xFE6F // vertical, compatibility and small forms
+        // Fullwidth forms, less the fullwidth digits and Latin letters (U+FF10-U+FF19, U+FF21-U+FF3A,
+        // U+FF41-U+FF5A): those are Latin text set wide, whose words still end at a line end.
+        | 0xFF01..=0xFF0F | 0xFF1A..=0xFF20 | 0xFF3B..=0xFF40 | 0xFF5B..=0xFF60 | 0xFFE0..=0xFFE6
+        | 0xFF61..=0xFF9F // halfwidth CJK punctuation and Katakana (not halfwidth Hangul)
+        | 0x16FE0..=0x16FFF | 0x1AFF0..=0x1B2FF // ideographic symbols, kana supplements and extensions
+        | 0x20000..=0x3FFFD // Han extensions B and later, compatibility supplement
+    ) && !matches!(c as u32, 0x3200..=0x321E | 0x3260..=0x327E) // parenthesised and circled Hangul
+}
+
+/// Whether glyph `i` starts the line that the text of glyph `i - 1` runs on to: both run the same
+/// way, it sits past that glyph across the lines, by more than half the glyph's line height and
+/// at most three, and starts no further along the line than half a line height past it, as a
+/// wrapped line does (a line of one character starts level with the one before). Text set side by
+/// side on one line (table cells, labels), a cell set lower and further along the row, a column
+/// that starts back at the top, a block further down the page and a flow in another direction
+/// keep their word break.
+fn wraps(page: &PageText, i: usize) -> bool {
+    // Measured on the glyphs that own the visible characters either side of the break: an
+    // invisible glyph or a variation selector of its own (drawn with no height) says nothing about
+    // the lines. Each search stays within one line.
+    let Some(end) = i.checked_sub(1) else { return false };
+    let visible = |k: &usize| page.glyphs.get(*k).is_some_and(|g| g.text.chars().any(|c| !is_invisible(c) && !is_variation_selector(c)));
+    let same_line = |k: &usize, of: usize| page.line_of.get(*k).is_some() && page.line_of.get(*k) == page.line_of.get(of);
+    let before = (0..=end).rev().take_while(|k| same_line(k, end)).find(visible).unwrap_or(end);
+    let after = (i..page.glyphs.len()).take_while(|k| same_line(k, i)).find(visible).unwrap_or(i);
+    let Some((before, after)) = page.glyphs.get(before).zip(page.glyphs.get(after)) else { return false };
+    // Unit directions: the same flow has a dot product of 1.
+    let same_flow = before.direction[0] * after.direction[0] + before.direction[1] * after.direction[1] > 0.99;
+    let centre = |q: &[[f32; 2]; 4]| [q.iter().map(|p| p[0]).sum::<f32>() / 4.0, q.iter().map(|p| p[1]).sum::<f32>() / 4.0];
+    let (a, b) = (centre(&before.quad), centre(&after.quad));
+    // Towards the next line: the writing direction turned a quarter clockwise in y-down view space.
+    let [dx, dy] = before.direction;
+    let across = (b[0] - a[0]) * -dy + (b[1] - a[1]) * dx;
+    let along = (b[0] - a[0]) * dx + (b[1] - a[1]) * dy;
+    let [near, far] = [before.quad[0], before.quad[3]];
+    let height = (near[0] - far[0]).hypot(near[1] - far[1]);
+    // NaN compares false: no join.
+    same_flow && across > height * 0.5 && across <= height * 3.0 && along < height * 0.5
+}
+
+/// The first visible character on the line that glyph `i` starts, skipping the invisible characters
+/// that find drops, so a line that opens with a zero width character still joins on its first letter.
+fn first_visible(page: &PageText, i: usize) -> Option<char> {
+    let line = page.line_of.get(i)?;
+    page.glyphs.iter().zip(&page.line_of).skip(i).take_while(|(_, l)| *l == line).flat_map(|(g, _)| g.text.chars()).find(|&c| !is_invisible(c))
+}
+
 /// Extend a prefix, falling back through strictly shorter proper prefixes on a mismatch.
 fn advance(needle: &[char], prefix: &[usize], mut matched: usize, c: char) -> usize {
     while matched > 0 && needle.get(matched) != Some(&c) {