Merge #4470: M3: Find matches ligatures and words split by a line-end hyphen (#4469)

This commit is contained in:
bflatastic committed 2026-11-09 18:19:30 +07:00
commit 93e45776f2
6 files changed
+424 -79

No files matched your search

+84
View File
@@ -0,0 +1,84 @@
//! `text_find` on a ligature glyph named in the encoding and on a word divided by a hyphen at a
//! line end.
use std::path::PathBuf;
use pdfcraft_automation::{Automation, Content};
use serde_json::{Value, json};
/// A fresh directory for one test.
fn tmp(name: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("pdfcraft-find-text-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
fn ok(a: &mut Automation, tool: &str, args: Value) -> Value {
match a.call(tool, &args) {
Ok(mut c) => match c.remove(0) {
Content::Json(v) => v,
other => panic!("{tool}: expected JSON, got {other:?}"),
},
Err(e) => panic!("{tool} {args}: {e}"),
}
}
/// One page: `of<fi>ce` in a Helvetica whose code 128 is named `/fi`, and `exam-` at the end of
/// one line with `ple` starting the next.
fn find_pdf() -> Vec<u8> {
let body = "BT /F2 14 Tf 90 700 Td (of\\200ce) Tj ET\nBT /F1 14 Tf 90 650 Td (exam-) Tj 0 -16 Td (ple) Tj ET\n";
let objs = [
"<< /Type /Catalog /Pages 2 0 R >>".to_string(),
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(),
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R /F2 6 0 R >> >> >>".to_string(),
format!("<< /Length {} >>\nstream\n{body}\nendstream", body.len()),
"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".to_string(),
"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding << /Type /Encoding /BaseEncoding /WinAnsiEncoding /Differences [128 /fi] >> >>"
.to_string(),
];
let mut out = b"%PDF-1.7\n".to_vec();
let mut offsets = Vec::new();
for (i, o) in objs.iter().enumerate() {
offsets.push(out.len());
out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes());
}
let xref = out.len();
out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes());
for o in offsets {
out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes());
}
out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes());
out
}
/// "office" finds the ligature, and "example" both halves of the divided word, highlighted line by
/// line; the hyphenated form is still found as written.
#[test]
fn text_find_matches_ligatures_and_words_divided_at_a_line_end() {
let dir = tmp("forms");
std::fs::write(dir.join("find.pdf"), find_pdf()).unwrap();
let mut a = Automation::new().with_root(&dir).unwrap();
let doc = ok(&mut a, "doc_open", json!({ "path": "find.pdf" }))["doc"].as_u64().unwrap();
let find = |a: &mut Automation, q: &str| ok(a, "text_find", json!({ "doc": doc, "query": q }));
let office = find(&mut a, "office");
assert_eq!(office["count"], 1, "{office}");
let rects = office["matches"][0]["rects"].as_array().unwrap();
assert_eq!(rects.len(), 1);
assert!((rects[0][0].as_f64().unwrap() - 90.0).abs() < 0.5, "from the left edge of the o: {rects:?}");
let example = find(&mut a, "example");
assert_eq!(example["count"], 1, "{example}");
assert_eq!(example["matches"][0]["text"], "exam-\nple");
let rects = example["matches"][0]["rects"].as_array().unwrap();
assert_eq!(rects.len(), 2, "one highlight on each line: {rects:?}");
let top = |k: usize| rects[k][1].as_f64().unwrap();
assert!(top(1) > top(0) + 10.0, "the second on the next line down: {rects:?}");
for q in ["exam-ple", "Exam- ple", "exam-"] {
assert_eq!(find(&mut a, q)["count"], 1, "{q}");
}
assert_eq!(find(&mut a, "examples")["count"], 0);
let _ = std::fs::remove_dir_all(&dir);
}
+1
View File
@@ -9,6 +9,7 @@ mod batch;
mod batch_failures;
mod batch_secrets;
mod button_caption;
mod find_text;
mod form_bulk;
mod hostile_args;
mod list_selection;
+2 -1
View File
@@ -137,7 +137,8 @@ impl PageText {
/// Case-insensitive search. Returns glyph ranges of every match (words may span line breaks,
/// which count as a single space, or as nothing where the line runs on between two Chinese or
/// Japanese characters or ends in a soft hyphen: there a query with a space across the break
/// doesn't match, and one without does).
/// doesn't match, and one without does; a ligature matches its letters, and a word divided by a
/// hyphen at a line end matches whole as well as with the hyphen).
pub fn find(&self, needle: &str) -> Vec<std::ops::Range<usize>> {
self.find_opts(needle, false, false)
}
+174 -25
View File
@@ -1,33 +1,157 @@
//! Literal search in normalized page text, retaining the original glyph owners.
//!
//! Words and lines are separated by one space, except where a line of Chinese or Japanese runs on
//! to the next: a break between two such characters is no word break, so it reads as nothing and
//! "検知機構" is found when the line ends after "検知".
//! The page text is read as characters, each owned by the glyph it came from. Words and lines are
//! separated by one space, except where a line of Chinese or Japanese runs on to the next: a break
//! between two such characters is no word break, so it reads as nothing and "検知機構" is found
//! when the line ends after "検知". A Latin compatibility ligature (U+FB00–U+FB06) reads as its
//! letters in the page and in the query, so "office" finds "o\u{FB03}ce". A word divided by a
//! hyphen at the end of a line is also found whole: where a line ends in a hyphen between two
//! letters, the text is searched a second time without the hyphen and the line break ("example"
//! finds "exam-" + "ple") and a third time with the hyphen but without the break ("exam-ple"). A
//! hit in any reading counts; where hits from different readings overlap, the one that starts
//! first is kept (the longer on a tie). A soft hyphen needs no second reading: it draws nothing,
//! and a line that ends in one always runs on.
use super::{PageText, fold_case};
use std::ops::Range;
/// How a line ending in a hyphen between two letters is read.
#[derive(Clone, Copy, PartialEq, Eq)]
enum Hyphen {
/// As written: the hyphen, then a line break (a space).
Written,
/// One word: neither the hyphen nor the break.
Joined,
/// A hyphenated compound: the hyphen, but no break.
Kept,
}
/// The letters of a Latin compatibility ligature (U+FB00–U+FB06), their compatibility decomposition.
fn ligature(c: char) -> Option<&'static str> {
match c {
'\u{FB00}' => Some("ff"),
'\u{FB01}' => Some("fi"),
'\u{FB02}' => Some("fl"),
'\u{FB03}' => Some("ffi"),
'\u{FB04}' => Some("ffl"),
'\u{FB05}' | '\u{FB06}' => Some("st"),
_ => None,
}
}
/// `s` with each Latin compatibility ligature written as its letters. Nothing else is normalised.
fn letters(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for c in s.chars() {
match ligature(c) {
Some(l) => out.push_str(l),
None => out.push(c),
}
}
out
}
/// A visible hyphen that can divide a word at a line end: HYPHEN-MINUS and HYPHEN. A SOFT HYPHEN
/// (ISO 32000-2 §14.8.2.3) is invisible instead: find skips it, and a line that ends in one runs on.
fn is_hyphen(c: char) -> bool {
matches!(c, '-' | '\u{2010}')
}
/// Whether a line break separates glyph `i` from the glyph before it.
fn breaks_before(page: &PageText, i: usize) -> bool {
i.checked_sub(1).and_then(|previous| page.line_of.get(previous)).zip(page.line_of.get(i)).is_some_and(|(a, b)| a != b)
}
pub(super) fn find(page: &PageText, query: &str, sensitive: bool, whole_words: bool, limit: usize) -> Vec<Range<usize>> {
if limit == 0 {
return Vec::new();
}
let fold = |s: &str| if sensitive { s.to_owned() } else { fold_case(s) };
let fold = |s: &str| if sensitive { letters(s) } else { fold_case(&letters(s)) };
let query: String = fold(query).chars().filter(|&c| !is_invisible(c)).collect();
let needle: Vec<char> = query.split_whitespace().collect::<Vec<_>>().join(" ").chars().collect();
if needle.is_empty() {
return Vec::new();
}
let texts: Vec<String> = page.glyphs.iter().map(|g| fold(&g.text)).collect();
let divided: Vec<bool> = (0..texts.len()).map(|i| divides(page, &texts, i)).collect();
let readings: &[Hyphen] = if divided.contains(&true) { &[Hyphen::Written, Hyphen::Joined, Hyphen::Kept] } else { &[Hyphen::Written] };
// KMP reuses a matched prefix instead of comparing a long query again at every position.
// The table is bounded by the page's already materialized character count.
let mut prefix = vec![0; needle.len()];
let mut matched = 0;
for (i, &c) in needle.iter().enumerate().skip(1) {
matched = advance(&needle, &prefix, matched, c);
if let Some(slot) = prefix.get_mut(i) {
*slot = matched;
}
}
let mut hits: Vec<(Range<usize>, usize)> = Vec::new();
for (r, &reading) in readings.iter().enumerate() {
let (chars, zero_width_space) = characters(page, &texts, &divided, reading);
let found = search(&chars, &zero_width_space, &needle, &prefix, whole_words, limit);
hits.extend(found.into_iter().map(|hit| (hit, r)));
}
if readings.len() == 1 {
return hits.into_iter().map(|(hit, _)| hit).collect();
}
// Each reading's hits are in order and apart in its characters, though two can share a glyph
// ("f" twice in an "ffi" glyph); they are all kept. Across readings, the earliest start wins
// (the longer on a tie): a hit is left out where it overlaps one kept from another reading.
// The sort is stable, so the earlier reading wins an exact tie.
hits.sort_by(|a, b| a.0.start.cmp(&b.0.start).then(b.0.end.cmp(&a.0.end)));
// The furthest end of the hits kept from each reading. Hits come in order of start, so one
// overlaps an earlier kept hit exactly when it starts before that hit's end.
let mut ends = [0usize; 3];
let mut kept: Vec<Range<usize>> = Vec::new();
for (hit, r) in hits {
if ends.iter().enumerate().all(|(other, &end)| other == r || hit.start >= end) {
if let Some(end) = ends.get_mut(r) {
*end = (*end).max(hit.end);
}
kept.push(hit);
if kept.len() == limit {
break;
}
}
}
kept
}
/// Whether glyph `i` ends its line with a hyphen that follows a letter of its word, and the next
/// line starts with a letter. `texts` are the glyphs' folded texts; the invisible characters find
/// skips are passed over on both sides.
fn divides(page: &PageText, texts: &[String], i: usize) -> bool {
let Some(text) = texts.get(i) else { return false };
let mut back = text.chars().rev().filter(|&c| !is_invisible(c));
if !back.next().is_some_and(is_hyphen) {
return false;
}
let before = match back.next() {
Some(c) => Some(c),
// A hyphen glyph of its own: the letter is the end of the glyph before, on its line with no word gap.
None if !page.space_before.get(i).copied().unwrap_or(false) && !breaks_before(page, i) => {
i.checked_sub(1).and_then(|p| texts.get(p)).and_then(|t| t.chars().rev().find(|&c| !is_invisible(c)))
}
None => None,
};
let Some(next) = i.checked_add(1) else { return false };
before.is_some_and(char::is_alphabetic)
&& breaks_before(page, next)
&& texts.get(next).and_then(|t| t.chars().find(|&c| !is_invisible(c))).is_some_and(char::is_alphabetic)
}
/// The page's characters in one `reading`, each with the glyph it came from, and whether a
/// skipped ZERO WIDTH SPACE came right before each one: it draws nothing, but it is a word
/// boundary (UAX #29 breaks on both sides of it), which Thai and Khmer write between words with
/// no visible gap, so whole-word search still sees two words there.
fn characters(page: &PageText, texts: &[String], divided: &[bool], reading: Hyphen) -> (Vec<(char, usize)>, Vec<bool>) {
let mut chars: Vec<(char, usize)> = Vec::new();
// Whether a skipped ZERO WIDTH SPACE came right before each character: it draws nothing, but
// it is a word boundary (UAX #29 breaks on both sides of it), which Thai and Khmer write between words
// with no visible gap, so whole-word search still sees two words there.
let mut zero_width_space: Vec<bool> = Vec::new();
let mut pending_zero_width_space = false;
// The last character pushed other than a variation selector, kept as characters are pushed so
// a long run of selectors costs no rescan.
let mut last_base: Option<char> = None;
for (i, glyph) in page.glyphs.iter().enumerate() {
let text = fold(&glyph.text);
let line_break = i.checked_sub(1).and_then(|previous| page.line_of.get(previous)).zip(page.line_of.get(i)).is_some_and(|(a, b)| a != b);
for (i, text) in texts.iter().enumerate() {
let line_break = breaks_before(page, i);
// A soft hyphen marks where a word may be divided at a line end (ISO 32000-2 §14.8.2.3), so
// a line that ends in one runs on to the next with no word break between them.
// Other invisible characters after the soft hyphen don't hide it.
@@ -36,18 +160,29 @@ pub(super) fn find(page: &PageText, query: &str, sensitive: bool, whole_words: b
// sides of the break are the visible ones: invisible characters are skipped below, and a
// variation selector after the last one only picks its glyph.
let joined = line_break && last_base.is_some_and(unspaced) && first_visible(page, i).is_some_and(unspaced) && wraps(page, i);
let gap = page.space_before.get(i).copied().unwrap_or(false) || (line_break && !(joined || soft_break));
// And, in the readings that join it, a line that ends in a visible hyphen between letters.
let after_divide = reading != Hyphen::Written && i.checked_sub(1).and_then(|p| divided.get(p)).copied().unwrap_or(false);
let gap = page.space_before.get(i).copied().unwrap_or(false) || (line_break && !(joined || soft_break || after_divide));
if i > 0 && gap && chars.last().is_some_and(|c| c.0 != ' ') {
chars.push((' ', i));
last_base = Some(' ');
zero_width_space.push(false);
pending_zero_width_space = false;
}
for c in text.chars() {
// The divided glyph's hyphen is its last visible character, left out of the joined reading.
let dropped = if reading == Hyphen::Joined && divided.get(i).copied().unwrap_or(false) {
text.char_indices().rev().find(|&(_, c)| !is_invisible(c)).map(|(at, _)| at)
} else {
None
};
for (at, c) in text.char_indices() {
if is_invisible(c) {
pending_zero_width_space |= c == '\u{200B}';
continue;
}
if Some(at) == dropped {
continue;
}
let c = if c.is_whitespace() { ' ' } else { c };
if c != ' ' || chars.last().is_none_or(|last| last.0 != ' ') {
chars.push((c, i));
@@ -59,35 +194,38 @@ pub(super) fn find(page: &PageText, query: &str, sensitive: bool, whole_words: b
}
}
}
(chars, zero_width_space)
}
/// Non-overlapping hits of `needle` (with its KMP `prefix` table) in `chars`, as glyph ranges.
/// `zero_width_space` says which characters come right after a skipped zero width space.
fn search(
chars: &[(char, usize)],
zero_width_space: &[bool],
needle: &[char],
prefix: &[usize],
whole_words: bool,
limit: usize,
) -> Vec<Range<usize>> {
if needle.len() > chars.len() {
return Vec::new();
}
// KMP reuses a matched prefix instead of comparing a long query again at every position.
// The table is bounded by the page's already materialized character count.
let mut prefix = vec![0; needle.len()];
let mut matched = 0;
for (i, &c) in needle.iter().enumerate().skip(1) {
matched = advance(&needle, &prefix, matched, c);
if let Some(slot) = prefix.get_mut(i) {
*slot = matched;
}
}
matched = 0;
// A neighbour bounds a word only in a script that puts spaces between words. Han, kana, Thai
// and the like run their words together, so a character of theirs next to the match is no
// reason to reject it: Acrobat finds 台北 inside 台北市 with Whole words only on. Digits still
// bound, in any script: ๑ is not a whole word inside the number ๑๒.
let word = |i: Option<usize>| i.and_then(|i| chars.get(i)).is_some_and(|c| c.0.is_alphanumeric() && (c.0.is_numeric() || !unspaced_script(c.0)));
// A skipped zero width space before the first character or after the last is a boundary.
let after_break = |i: usize| zero_width_space.get(i).copied().unwrap_or(false);
let mut hits = Vec::new();
for (i, &(c, owner)) in chars.iter().enumerate() {
matched = advance(&needle, &prefix, matched, c);
matched = advance(needle, prefix, matched, c);
if matched != needle.len() {
continue;
}
let end = i.saturating_add(1);
let start = end.saturating_sub(needle.len());
// A skipped zero width space before the first character or after the last is a boundary.
let after_break = |i: usize| zero_width_space.get(i).copied().unwrap_or(false);
let bounded = !whole_words || ((after_break(start) || !word(start.checked_sub(1))) && (after_break(end) || !word(Some(end))));
if bounded {
if let Some((_, first)) = chars.get(start)
@@ -240,3 +378,14 @@ fn unspaced_script(c: char) -> bool {
| 0x20000..=0x3FFFF
)
}
#[cfg(test)]
mod tests {
use super::letters;
#[test]
fn ligatures_read_as_their_letters() {
assert_eq!(letters("o\u{FB03}ce \u{FB00}\u{FB01}\u{FB02}\u{FB04}\u{FB05}\u{FB06}"), "office fffiflfflstst");
assert_eq!(letters("\u{FB13}é\u{FF21}\u{2460}"), "\u{FB13}é\u{FF21}\u{2460}", "nothing else is normalised");
}
}
+161 -51
View File
@@ -88,68 +88,172 @@ const UNSPACED_BLOCK_SAMPLES: &[(char, bool, &str)] = &[
('\u{323AF}', true, "CJK Unified Ideographs Extension H (31350..323AF), last"),
];
// The original straightforward search is an independent oracle for valid page metadata.
// The original straightforward search is an independent oracle for valid page metadata, extended
// with ligature letters and the three readings of a line-end hyphen between letters: as written,
// joined (no hyphen, no break) and kept (hyphen, no break).
fn reference(page: &PageText, query: &str, sensitive: bool, whole: bool, limit: usize) -> Vec<Range<usize>> {
let fold = |s: &str| if sensitive { s.to_owned() } else { fold_case(s) };
let letters = |s: &str| {
s.replace('\u{FB00}', "ff")
.replace('\u{FB01}', "fi")
.replace('\u{FB02}', "fl")
.replace('\u{FB03}', "ffi")
.replace('\u{FB04}', "ffl")
.replace(['\u{FB05}', '\u{FB06}'], "st")
};
let fold = |s: &str| if sensitive { letters(s) } else { fold_case(&letters(s)) };
// Soft hyphens and zero width characters draw nothing and are skipped on both sides.
let visible =
|s: String| s.chars().filter(|c| !matches!(c, '\u{AD}' | '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{2060}' | '\u{FEFF}')).collect::<String>();
let invisible = |c: char| matches!(c, '\u{AD}' | '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{2060}' | '\u{FEFF}');
let visible = |s: String| s.chars().filter(|&c| !invisible(c)).collect::<String>();
let needle: Vec<char> = visible(fold(query)).split_whitespace().collect::<Vec<_>>().join(" ").chars().collect();
if needle.is_empty() || limit == 0 {
return Vec::new();
}
// Each character, its glyph, and whether a skipped zero width space came right before it.
let mut chars: Vec<(char, usize, bool)> = Vec::new();
let mut after_zwsp = false;
for (i, glyph) in page.glyphs.iter().enumerate() {
// A line that ends in a soft hyphen, past any empty or other invisible glyphs and
// characters after it on that line, runs on to the next.
let ends_in_soft_hyphen = |end: usize| {
(0..=end)
.rev()
.take_while(|&k| page.line_of[k] == page.line_of[end])
.flat_map(|k| page.glyphs[k].text.chars().rev())
.find(|&c| c == '\u{AD}' || !visible(c.to_string()).is_empty())
== Some('\u{AD}')
};
let line_break = i > 0 && page.line_of[i] != page.line_of[i - 1] && !ends_in_soft_hyphen(i - 1);
if i > 0 && (page.space_before[i] || line_break) && chars.last().is_some_and(|&(c, ..)| c != ' ') {
chars.push((' ', i, false));
after_zwsp = false;
}
for c in fold(&glyph.text).chars() {
after_zwsp |= c == '\u{200B}';
if visible(c.to_string()).is_empty() {
continue;
}
let c = if c.is_whitespace() { ' ' } else { c };
if c != ' ' || chars.last().is_none_or(|&(last, ..)| last != ' ') {
chars.push((c, i, after_zwsp && c != ' '));
let texts: Vec<String> = page.glyphs.iter().map(|g| fold(&g.text)).collect();
let n = texts.len();
let new_line = |i: usize| i > 0 && page.line_of[i] != page.line_of[i - 1];
// A line that ends in a soft hyphen, past any empty or other invisible glyphs and characters
// after it on that line, runs on to the next.
let ends_in_soft_hyphen = |end: usize| {
(0..=end)
.rev()
.take_while(|&k| page.line_of[k] == page.line_of[end])
.flat_map(|k| page.glyphs[k].text.chars().rev())
.find(|&c| c == '\u{AD}' || !invisible(c))
== Some('\u{AD}')
};
// A visible hyphen between letters at a line end.
let hyphen = |c: char| matches!(c, '-' | '\u{2010}');
let divided: Vec<bool> = (0..n)
.map(|i| {
let chars: Vec<char> = visible(texts[i].clone()).chars().collect();
let Some(&last) = chars.last() else { return false };
let before = if chars.len() >= 2 {
Some(chars[chars.len() - 2])
} else if i > 0 && !page.space_before[i] && !new_line(i) {
visible(texts[i - 1].clone()).chars().last()
} else {
None
};
hyphen(last)
&& before.is_some_and(char::is_alphabetic)
&& i + 1 < n
&& new_line(i + 1)
&& visible(texts[i + 1].clone()).chars().next().is_some_and(char::is_alphabetic)
})
.collect();
let mut all = Vec::new();
let readings = if divided.contains(&true) { 3 } else { 1 };
for reading in 0..readings {
// Each character, its glyph, and whether a skipped zero width space came right before it.
let mut chars: Vec<(char, usize, bool)> = Vec::new();
let mut after_zwsp = false;
for i in 0..n {
let joined = reading > 0 && i > 0 && divided[i - 1];
let line_break = new_line(i) && !ends_in_soft_hyphen(i - 1) && !joined;
if i > 0 && (page.space_before[i] || line_break) && chars.last().is_some_and(|&(c, ..)| c != ' ') {
chars.push((' ', i, false));
after_zwsp = false;
}
}
}
// A neighbour bounds a word only in a script written with spaces between words, or when it
// is a digit.
let word = |i: Option<usize>| i.and_then(|i| chars.get(i)).is_some_and(|&(c, ..)| c.is_alphanumeric() && (c.is_numeric() || !unspaced(c)));
// A word ends at a skipped zero width space as it does at a space.
let edge = |i: usize| chars.get(i).is_some_and(|&(.., zwsp)| zwsp);
let mut hits = Vec::new();
let mut start = 0;
while start + needle.len() <= chars.len() {
let end = start + needle.len();
let bounded = (edge(start) || !word(start.checked_sub(1))) && (edge(end) || !word(Some(end)));
if (!whole || bounded) && chars[start..end].iter().map(|&(c, ..)| c).eq(needle.iter().copied()) {
hits.push(chars[start].1..chars[end - 1].1 + 1);
if hits.len() == limit {
break;
let mut glyph: Vec<char> = texts[i].chars().collect();
if reading == 1 && divided[i] {
// The hyphen: the last visible character.
if let Some(at) = glyph.iter().rposition(|&c| !invisible(c)) {
glyph.remove(at);
}
}
start = end;
} else {
start += 1;
for c in glyph {
after_zwsp |= c == '\u{200B}';
if invisible(c) {
continue;
}
let c = if c.is_whitespace() { ' ' } else { c };
if c != ' ' || chars.last().is_none_or(|&(last, ..)| last != ' ') {
chars.push((c, i, after_zwsp && c != ' '));
after_zwsp = false;
}
}
}
// A neighbour bounds a word only in a script written with spaces between words, or when it
// is a digit.
let word = |i: Option<usize>| i.and_then(|i| chars.get(i)).is_some_and(|&(c, ..)| c.is_alphanumeric() && (c.is_numeric() || !unspaced(c)));
// A word ends at a skipped zero width space as it does at a space.
let edge = |i: usize| chars.get(i).is_some_and(|&(.., zwsp)| zwsp);
let mut hits = Vec::new();
let mut start = 0;
while start + needle.len() <= chars.len() {
let end = start + needle.len();
let bounded = (edge(start) || !word(start.checked_sub(1))) && (edge(end) || !word(Some(end)));
if (!whole || bounded) && chars[start..end].iter().map(|&(c, ..)| c).eq(needle.iter().copied()) {
hits.push(chars[start].1..chars[end - 1].1 + 1);
if hits.len() == limit {
break;
}
start = end;
} else {
start += 1;
}
}
all.push(hits);
}
if readings == 1 {
return all.remove(0);
}
// A hit is left out where it overlaps one kept from another reading; one reading's hits are
// all kept, though two can share a glyph.
let mut hits: Vec<(Range<usize>, usize)> = all.into_iter().enumerate().flat_map(|(r, hits)| hits.into_iter().map(move |h| (h, r))).collect();
hits.sort_by_key(|(h, _)| (h.start, std::cmp::Reverse(h.end)));
let mut out: Vec<(Range<usize>, usize)> = Vec::new();
for (h, r) in hits {
let overlaps = out.iter().any(|(k, kr)| *kr != r && h.start < k.end && k.start < h.end);
if !overlaps && out.len() < limit {
out.push((h, r));
}
}
hits
out.into_iter().map(|(h, _)| h).collect()
}
/// One glyph per entry of `text`, each on the line given in `lines`.
fn lined(text: &[&str], lines: &[u32]) -> PageText {
let mut p = page(text);
p.line_of = lines.to_vec();
p
}
#[test]
fn a_ligature_matches_its_letters() {
let text = page(&["o", "\u{FB03}", "c", "e", " ", "o", "f", "\u{FB01}", "c", "e"]);
assert_eq!(text.find("office"), vec![0..4, 5..10]);
assert_eq!(text.find("OFFICE"), vec![0..4, 5..10]);
assert_eq!(text.find_opts("office", true, true), vec![0..4, 5..10], "case-sensitive and whole words too");
assert_eq!(text.find("o\u{FB03}ce"), vec![0..4, 5..10], "a ligature in the query reads as its letters too");
assert_eq!(text.find("fi"), vec![1..2, 7..8], "a match inside a ligature takes its glyph");
assert!(text.find("o\u{FB13}").is_empty(), "no other compatibility folding");
}
#[test]
fn a_word_divided_by_a_line_end_hyphen_matches_whole_and_as_written() {
// "exam-" ends line 0, "ple" starts line 1; the hyphen is its own glyph.
let text = lined(&["e", "x", "a", "m", "-", "p", "l", "e"], &[0, 0, 0, 0, 0, 1, 1, 1]);
assert_eq!(text.find("example"), vec![0..8], "both parts, and the hyphen between them");
assert_eq!(text.find_opts("example", false, true), vec![0..8], "a whole word");
assert_eq!(text.find("exam-ple"), vec![0..8], "a query with the hyphen");
assert_eq!(text.find("exam- ple"), vec![0..8], "and with the line break as a space");
assert_eq!(text.find("exam"), vec![0..4]);
// The hyphen in the glyph of the last letter, a soft hyphen and U+2010.
for hyphen in ["m-", "m\u{AD}", "m\u{2010}"] {
let text = lined(&["e", "x", "a", hyphen, "p", "l", "e"], &[0, 0, 0, 0, 1, 1, 1]);
assert_eq!(text.find("example"), vec![0..7], "{hyphen:?}");
}
// Not divided: a hyphen inside a line, a dash after a word gap, a hyphen between digits.
assert!(page(&["e", "x", "a", "m", "-", "p", "l", "e"]).find("example").is_empty(), "inside a line");
let mut spaced = lined(&["e", "x", "a", "m", "-", "p", "l", "e"], &[0, 0, 0, 0, 0, 1, 1, 1]);
spaced.space_before[4] = true;
assert!(spaced.find("example").is_empty(), "a dash after a word gap");
assert!(spaced.find("exam ple").is_empty(), "a dash after a word gap is never dropped");
assert!(lined(&["1", "0", "-", "2", "0"], &[0, 0, 0, 1, 1]).find("1020").is_empty(), "between digits");
assert!(lined(&["a", "-", "2", "0"], &[0, 0, 1, 1]).find("a20").is_empty(), "before a number");
assert!(lined(&["1", "0", "-", "a", "b"], &[0, 0, 0, 1, 1]).find("10ab").is_empty(), "after a number");
assert!(lined(&["a", "-"], &[0, 0]).find("a-").len() == 1, "at the end of the page");
}
#[test]
@@ -171,6 +275,10 @@ fn whole_word_rejection_keeps_overlapping_candidates() {
// Accepted matches still do not overlap, even within one multi-character glyph.
assert_eq!(page(&["aaaaa"]).find("aaa"), vec![0..1]);
assert_eq!(page(&["aaaaa"]).find("aa"), vec![0..1, 0..1]);
// So does a page with a word divided at a line end, which is searched in three readings.
let divided = lined(&["\u{FB03}", " ", "exam", "-", "ple"], &[0, 0, 0, 0, 1]);
assert_eq!(divided.find("f"), vec![0..1, 0..1]);
assert_eq!(divided.find("example"), vec![2..5]);
}
/// A letter from a block in `UNSPACED_BLOCK_SAMPLES` next to a whole-word match leaves it standing
@@ -191,6 +299,8 @@ fn whole_words_follow_the_unicode_blocks_of_unspaced_scripts() {
#[test]
fn search_matches_reference_for_unicode_gaps_limits_and_word_boundaries() {
let alphabet = ["a", "b", " ", "-", "İ", "ı", "ς", "Σ", "\t", "\n", "ffi", "", "界", "ก", "한", "e\u{301}", "العربية", "\u{AD}", "a\u{200B}"];
// Ligatures, digits and HYPHEN exercise the readings of a line-end hyphen.
let alphabet: Vec<&str> = alphabet.into_iter().chain(["\u{FB01}", "1", "\u{2010}"]).collect();
let mut seed = 1234567u64;
let mut next = || {
seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);
+2 -2
View File
@@ -1418,8 +1418,8 @@ milestone = "M3"
status = "shipped"
commands = ["edit.find"]
tools = ["text_find"]
evidence = ["crates/ui-egui/tests/it/ui.rs::find_counts_matches_across_pages", "crates/ui-egui/tests/it/ui.rs::find_reports_no_matches", "crates/automation/tests/it/automation.rs::open_inspect_render_and_find", "crates/render/src/text/tests.rs::identity_v_columns_are_searchable_in_reading_order", "crates/render/src/text/tests.rs::toc_leaders_do_not_hide_word_gaps", "crates/render/src/text/tests.rs::short_tracked_ellipses_remain_searchable", "crates/render/tests/it/search.rs::find_skips_soft_hyphens_and_zero_width_characters", "crates/render/tests/it/search.rs::find_joins_a_word_divided_at_a_line_end_soft_hyphen", "crates/render/tests/it/search.rs::a_zero_width_space_is_a_word_boundary_for_whole_words", "crates/render/tests/it/search.rs::a_chinese_or_japanese_word_divided_by_a_line_end_is_found_on_both_lines", "crates/render/tests/it/search.rs::a_line_break_stays_a_space_beside_latin_or_hangul_and_between_side_by_side_text", "crates/render/tests/it/search.rs::a_word_divided_between_vertical_columns_is_found_in_both", "crates/render/tests/it/search.rs::each_unspaced_range_joins_and_spaced_scripts_do_not", "crates/render/tests/it/search.rs::a_soft_hyphen_and_a_chinese_or_japanese_line_end_combine", "crates/render/src/text/tests.rs::thai_marks_drawn_one_glyph_at_a_time_keep_their_place_and_words"]
notes = "Horizontal and WMode 1 vertical text are grouped independently, including a horizontal heading above right-to-left vertical columns. Dotted TOC leaders preserve word spaces for phrase search (#3086), while short tracked ellipses remain intact and searchable; sparse lines and condensed OCR gaps below the existing height-based threshold remain limitations. Combining marks (Mn/Me) of left-to-right scripts drawn right after their letter stay with it in content order, and Thai or Indic word gaps without a space glyph are found as spaces (#6517). Complex interleaved flows still use a geometric reading-order heuristic. Soft hyphens, zero width spaces and joiners, word joiners and U+FEFF inside a word are skipped, in the page and in the query; a skipped zero width space still ends a word for Whole words only. A line break reads as no word break when the line ends in a soft hyphen, or when it runs on between two Chinese or Japanese characters (East Asian Wide, Fullwidth or Halfwidth, not Hangul and not fullwidth Latin letters or digits), so a divided word is found, in horizontal lines and in vertical columns that continue to the left; the break stays a space beside Latin or Korean text and between text set side by side on one line or in staggered cells."
evidence = ["crates/ui-egui/tests/it/ui.rs::find_counts_matches_across_pages", "crates/ui-egui/tests/it/ui.rs::find_reports_no_matches", "crates/automation/tests/it/automation.rs::open_inspect_render_and_find", "crates/render/src/text/tests.rs::identity_v_columns_are_searchable_in_reading_order", "crates/render/src/text/tests.rs::toc_leaders_do_not_hide_word_gaps", "crates/render/src/text/tests.rs::short_tracked_ellipses_remain_searchable", "crates/render/tests/it/search.rs::find_skips_soft_hyphens_and_zero_width_characters", "crates/render/tests/it/search.rs::find_joins_a_word_divided_at_a_line_end_soft_hyphen", "crates/render/tests/it/search.rs::a_zero_width_space_is_a_word_boundary_for_whole_words", "crates/render/tests/it/search.rs::a_chinese_or_japanese_word_divided_by_a_line_end_is_found_on_both_lines", "crates/render/tests/it/search.rs::a_line_break_stays_a_space_beside_latin_or_hangul_and_between_side_by_side_text", "crates/render/tests/it/search.rs::a_word_divided_between_vertical_columns_is_found_in_both", "crates/render/tests/it/search.rs::each_unspaced_range_joins_and_spaced_scripts_do_not", "crates/render/tests/it/search.rs::a_soft_hyphen_and_a_chinese_or_japanese_line_end_combine", "crates/render/src/text/tests.rs::thai_marks_drawn_one_glyph_at_a_time_keep_their_place_and_words", "crates/render/tests/it/search.rs::a_ligature_matches_its_letters", "crates/render/tests/it/search.rs::a_word_divided_by_a_line_end_hyphen_matches_whole_and_as_written", "crates/automation/tests/it/find_text.rs::text_find_matches_ligatures_and_words_divided_at_a_line_end"]
notes = "Horizontal and WMode 1 vertical text are grouped independently, including a horizontal heading above right-to-left vertical columns. Dotted TOC leaders preserve word spaces for phrase search (#3086), while short tracked ellipses remain intact and searchable; sparse lines and condensed OCR gaps below the existing height-based threshold remain limitations. Combining marks (Mn/Me) of left-to-right scripts drawn right after their letter stay with it in content order, and Thai or Indic word gaps without a space glyph are found as spaces (#6517). Complex interleaved flows still use a geometric reading-order heuristic. Soft hyphens, zero width spaces and joiners, word joiners and U+FEFF inside a word are skipped, in the page and in the query; a skipped zero width space still ends a word for Whole words only. A line break reads as no word break when the line ends in a soft hyphen, or when it runs on between two Chinese or Japanese characters (East Asian Wide, Fullwidth or Halfwidth, not Hangul and not fullwidth Latin letters or digits), so a divided word is found, in horizontal lines and in vertical columns that continue to the left; the break stays a space beside Latin or Korean text and between text set side by side on one line or in staggered cells. A Latin ligature (U+FB00–FB06) matches its letters, and a word divided by a hyphen (U+002D or U+2010) between letters at a line end matches whole, highlighted on both lines, as well as with the hyphen. Missing: accent-insensitive matching (an unaccented query finding accented text; an open pull request adds it)."
[[feature]]
id = "view.find-next-previous"