mirror of
https://github.com/storytold/pdfcraft.git
synced 2026-11-16 15:53:54 +00:00
This commit is contained in:
commit
93e45776f2
6 files changed
+424
-79
No files matched your search
@@ -0,0 +1,84 @@
|
||||
//! `text_find` on a ligature glyph named in the encoding and on a word divided by a hyphen at a
|
||||
//! line end.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use pdfcraft_automation::{Automation, Content};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
/// A fresh directory for one test.
|
||||
fn tmp(name: &str) -> PathBuf {
|
||||
let d = std::env::temp_dir().join(format!("pdfcraft-find-text-{}-{name}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
std::fs::create_dir_all(&d).unwrap();
|
||||
d
|
||||
}
|
||||
|
||||
fn ok(a: &mut Automation, tool: &str, args: Value) -> Value {
|
||||
match a.call(tool, &args) {
|
||||
Ok(mut c) => match c.remove(0) {
|
||||
Content::Json(v) => v,
|
||||
other => panic!("{tool}: expected JSON, got {other:?}"),
|
||||
},
|
||||
Err(e) => panic!("{tool} {args}: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// One page: `of<fi>ce` in a Helvetica whose code 128 is named `/fi`, and `exam-` at the end of
|
||||
/// one line with `ple` starting the next.
|
||||
fn find_pdf() -> Vec<u8> {
|
||||
let body = "BT /F2 14 Tf 90 700 Td (of\\200ce) Tj ET\nBT /F1 14 Tf 90 650 Td (exam-) Tj 0 -16 Td (ple) Tj ET\n";
|
||||
let objs = [
|
||||
"<< /Type /Catalog /Pages 2 0 R >>".to_string(),
|
||||
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(),
|
||||
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R /F2 6 0 R >> >> >>".to_string(),
|
||||
format!("<< /Length {} >>\nstream\n{body}\nendstream", body.len()),
|
||||
"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".to_string(),
|
||||
"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding << /Type /Encoding /BaseEncoding /WinAnsiEncoding /Differences [128 /fi] >> >>"
|
||||
.to_string(),
|
||||
];
|
||||
let mut out = b"%PDF-1.7\n".to_vec();
|
||||
let mut offsets = Vec::new();
|
||||
for (i, o) in objs.iter().enumerate() {
|
||||
offsets.push(out.len());
|
||||
out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes());
|
||||
}
|
||||
let xref = out.len();
|
||||
out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes());
|
||||
for o in offsets {
|
||||
out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes());
|
||||
}
|
||||
out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes());
|
||||
out
|
||||
}
|
||||
|
||||
/// "office" finds the ligature, and "example" both halves of the divided word, highlighted line by
|
||||
/// line; the hyphenated form is still found as written.
|
||||
#[test]
|
||||
fn text_find_matches_ligatures_and_words_divided_at_a_line_end() {
|
||||
let dir = tmp("forms");
|
||||
std::fs::write(dir.join("find.pdf"), find_pdf()).unwrap();
|
||||
let mut a = Automation::new().with_root(&dir).unwrap();
|
||||
let doc = ok(&mut a, "doc_open", json!({ "path": "find.pdf" }))["doc"].as_u64().unwrap();
|
||||
let find = |a: &mut Automation, q: &str| ok(a, "text_find", json!({ "doc": doc, "query": q }));
|
||||
|
||||
let office = find(&mut a, "office");
|
||||
assert_eq!(office["count"], 1, "{office}");
|
||||
let rects = office["matches"][0]["rects"].as_array().unwrap();
|
||||
assert_eq!(rects.len(), 1);
|
||||
assert!((rects[0][0].as_f64().unwrap() - 90.0).abs() < 0.5, "from the left edge of the o: {rects:?}");
|
||||
|
||||
let example = find(&mut a, "example");
|
||||
assert_eq!(example["count"], 1, "{example}");
|
||||
assert_eq!(example["matches"][0]["text"], "exam-\nple");
|
||||
let rects = example["matches"][0]["rects"].as_array().unwrap();
|
||||
assert_eq!(rects.len(), 2, "one highlight on each line: {rects:?}");
|
||||
let top = |k: usize| rects[k][1].as_f64().unwrap();
|
||||
assert!(top(1) > top(0) + 10.0, "the second on the next line down: {rects:?}");
|
||||
|
||||
for q in ["exam-ple", "Exam- ple", "exam-"] {
|
||||
assert_eq!(find(&mut a, q)["count"], 1, "{q}");
|
||||
}
|
||||
assert_eq!(find(&mut a, "examples")["count"], 0);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
@@ -9,6 +9,7 @@ mod batch;
|
||||
mod batch_failures;
|
||||
mod batch_secrets;
|
||||
mod button_caption;
|
||||
mod find_text;
|
||||
mod form_bulk;
|
||||
mod hostile_args;
|
||||
mod list_selection;
|
||||
|
||||
@@ -137,7 +137,8 @@ impl PageText {
|
||||
/// Case-insensitive search. Returns glyph ranges of every match (words may span line breaks,
|
||||
/// which count as a single space, or as nothing where the line runs on between two Chinese or
|
||||
/// Japanese characters or ends in a soft hyphen: there a query with a space across the break
|
||||
/// doesn't match, and one without does).
|
||||
/// doesn't match, and one without does; a ligature matches its letters, and a word divided by a
|
||||
/// hyphen at a line end matches whole as well as with the hyphen).
|
||||
pub fn find(&self, needle: &str) -> Vec<std::ops::Range<usize>> {
|
||||
self.find_opts(needle, false, false)
|
||||
}
|
||||
|
||||
@@ -1,33 +1,157 @@
|
||||
//! Literal search in normalized page text, retaining the original glyph owners.
|
||||
//!
|
||||
//! Words and lines are separated by one space, except where a line of Chinese or Japanese runs on
|
||||
//! to the next: a break between two such characters is no word break, so it reads as nothing and
|
||||
//! "検知機構" is found when the line ends after "検知".
|
||||
//! The page text is read as characters, each owned by the glyph it came from. Words and lines are
|
||||
//! separated by one space, except where a line of Chinese or Japanese runs on to the next: a break
|
||||
//! between two such characters is no word break, so it reads as nothing and "検知機構" is found
|
||||
//! when the line ends after "検知". A Latin compatibility ligature (U+FB00–U+FB06) reads as its
|
||||
//! letters in the page and in the query, so "office" finds "o\u{FB03}ce". A word divided by a
|
||||
//! hyphen at the end of a line is also found whole: where a line ends in a hyphen between two
|
||||
//! letters, the text is searched a second time without the hyphen and the line break ("example"
|
||||
//! finds "exam-" + "ple") and a third time with the hyphen but without the break ("exam-ple"). A
|
||||
//! hit in any reading counts; where hits from different readings overlap, the one that starts
|
||||
//! first is kept (the longer on a tie). A soft hyphen needs no second reading: it draws nothing,
|
||||
//! and a line that ends in one always runs on.
|
||||
use super::{PageText, fold_case};
|
||||
use std::ops::Range;
|
||||
|
||||
/// How a line ending in a hyphen between two letters is read.
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum Hyphen {
|
||||
/// As written: the hyphen, then a line break (a space).
|
||||
Written,
|
||||
/// One word: neither the hyphen nor the break.
|
||||
Joined,
|
||||
/// A hyphenated compound: the hyphen, but no break.
|
||||
Kept,
|
||||
}
|
||||
|
||||
/// The letters of a Latin compatibility ligature (U+FB00–U+FB06), their compatibility decomposition.
|
||||
fn ligature(c: char) -> Option<&'static str> {
|
||||
match c {
|
||||
'\u{FB00}' => Some("ff"),
|
||||
'\u{FB01}' => Some("fi"),
|
||||
'\u{FB02}' => Some("fl"),
|
||||
'\u{FB03}' => Some("ffi"),
|
||||
'\u{FB04}' => Some("ffl"),
|
||||
'\u{FB05}' | '\u{FB06}' => Some("st"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// `s` with each Latin compatibility ligature written as its letters. Nothing else is normalised.
|
||||
fn letters(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for c in s.chars() {
|
||||
match ligature(c) {
|
||||
Some(l) => out.push_str(l),
|
||||
None => out.push(c),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A visible hyphen that can divide a word at a line end: HYPHEN-MINUS and HYPHEN. A SOFT HYPHEN
|
||||
/// (ISO 32000-2 §14.8.2.3) is invisible instead: find skips it, and a line that ends in one runs on.
|
||||
fn is_hyphen(c: char) -> bool {
|
||||
matches!(c, '-' | '\u{2010}')
|
||||
}
|
||||
|
||||
/// Whether a line break separates glyph `i` from the glyph before it.
|
||||
fn breaks_before(page: &PageText, i: usize) -> bool {
|
||||
i.checked_sub(1).and_then(|previous| page.line_of.get(previous)).zip(page.line_of.get(i)).is_some_and(|(a, b)| a != b)
|
||||
}
|
||||
|
||||
pub(super) fn find(page: &PageText, query: &str, sensitive: bool, whole_words: bool, limit: usize) -> Vec<Range<usize>> {
|
||||
if limit == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
let fold = |s: &str| if sensitive { s.to_owned() } else { fold_case(s) };
|
||||
let fold = |s: &str| if sensitive { letters(s) } else { fold_case(&letters(s)) };
|
||||
let query: String = fold(query).chars().filter(|&c| !is_invisible(c)).collect();
|
||||
let needle: Vec<char> = query.split_whitespace().collect::<Vec<_>>().join(" ").chars().collect();
|
||||
if needle.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let texts: Vec<String> = page.glyphs.iter().map(|g| fold(&g.text)).collect();
|
||||
let divided: Vec<bool> = (0..texts.len()).map(|i| divides(page, &texts, i)).collect();
|
||||
let readings: &[Hyphen] = if divided.contains(&true) { &[Hyphen::Written, Hyphen::Joined, Hyphen::Kept] } else { &[Hyphen::Written] };
|
||||
// KMP reuses a matched prefix instead of comparing a long query again at every position.
|
||||
// The table is bounded by the page's already materialized character count.
|
||||
let mut prefix = vec![0; needle.len()];
|
||||
let mut matched = 0;
|
||||
for (i, &c) in needle.iter().enumerate().skip(1) {
|
||||
matched = advance(&needle, &prefix, matched, c);
|
||||
if let Some(slot) = prefix.get_mut(i) {
|
||||
*slot = matched;
|
||||
}
|
||||
}
|
||||
let mut hits: Vec<(Range<usize>, usize)> = Vec::new();
|
||||
for (r, &reading) in readings.iter().enumerate() {
|
||||
let (chars, zero_width_space) = characters(page, &texts, ÷d, reading);
|
||||
let found = search(&chars, &zero_width_space, &needle, &prefix, whole_words, limit);
|
||||
hits.extend(found.into_iter().map(|hit| (hit, r)));
|
||||
}
|
||||
if readings.len() == 1 {
|
||||
return hits.into_iter().map(|(hit, _)| hit).collect();
|
||||
}
|
||||
// Each reading's hits are in order and apart in its characters, though two can share a glyph
|
||||
// ("f" twice in an "ffi" glyph); they are all kept. Across readings, the earliest start wins
|
||||
// (the longer on a tie): a hit is left out where it overlaps one kept from another reading.
|
||||
// The sort is stable, so the earlier reading wins an exact tie.
|
||||
hits.sort_by(|a, b| a.0.start.cmp(&b.0.start).then(b.0.end.cmp(&a.0.end)));
|
||||
// The furthest end of the hits kept from each reading. Hits come in order of start, so one
|
||||
// overlaps an earlier kept hit exactly when it starts before that hit's end.
|
||||
let mut ends = [0usize; 3];
|
||||
let mut kept: Vec<Range<usize>> = Vec::new();
|
||||
for (hit, r) in hits {
|
||||
if ends.iter().enumerate().all(|(other, &end)| other == r || hit.start >= end) {
|
||||
if let Some(end) = ends.get_mut(r) {
|
||||
*end = (*end).max(hit.end);
|
||||
}
|
||||
kept.push(hit);
|
||||
if kept.len() == limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
kept
|
||||
}
|
||||
|
||||
/// Whether glyph `i` ends its line with a hyphen that follows a letter of its word, and the next
|
||||
/// line starts with a letter. `texts` are the glyphs' folded texts; the invisible characters find
|
||||
/// skips are passed over on both sides.
|
||||
fn divides(page: &PageText, texts: &[String], i: usize) -> bool {
|
||||
let Some(text) = texts.get(i) else { return false };
|
||||
let mut back = text.chars().rev().filter(|&c| !is_invisible(c));
|
||||
if !back.next().is_some_and(is_hyphen) {
|
||||
return false;
|
||||
}
|
||||
let before = match back.next() {
|
||||
Some(c) => Some(c),
|
||||
// A hyphen glyph of its own: the letter is the end of the glyph before, on its line with no word gap.
|
||||
None if !page.space_before.get(i).copied().unwrap_or(false) && !breaks_before(page, i) => {
|
||||
i.checked_sub(1).and_then(|p| texts.get(p)).and_then(|t| t.chars().rev().find(|&c| !is_invisible(c)))
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let Some(next) = i.checked_add(1) else { return false };
|
||||
before.is_some_and(char::is_alphabetic)
|
||||
&& breaks_before(page, next)
|
||||
&& texts.get(next).and_then(|t| t.chars().find(|&c| !is_invisible(c))).is_some_and(char::is_alphabetic)
|
||||
}
|
||||
|
||||
/// The page's characters in one `reading`, each with the glyph it came from, and whether a
|
||||
/// skipped ZERO WIDTH SPACE came right before each one: it draws nothing, but it is a word
|
||||
/// boundary (UAX #29 breaks on both sides of it), which Thai and Khmer write between words with
|
||||
/// no visible gap, so whole-word search still sees two words there.
|
||||
fn characters(page: &PageText, texts: &[String], divided: &[bool], reading: Hyphen) -> (Vec<(char, usize)>, Vec<bool>) {
|
||||
let mut chars: Vec<(char, usize)> = Vec::new();
|
||||
// Whether a skipped ZERO WIDTH SPACE came right before each character: it draws nothing, but
|
||||
// it is a word boundary (UAX #29 breaks on both sides of it), which Thai and Khmer write between words
|
||||
// with no visible gap, so whole-word search still sees two words there.
|
||||
let mut zero_width_space: Vec<bool> = Vec::new();
|
||||
let mut pending_zero_width_space = false;
|
||||
// The last character pushed other than a variation selector, kept as characters are pushed so
|
||||
// a long run of selectors costs no rescan.
|
||||
let mut last_base: Option<char> = None;
|
||||
for (i, glyph) in page.glyphs.iter().enumerate() {
|
||||
let text = fold(&glyph.text);
|
||||
let line_break = i.checked_sub(1).and_then(|previous| page.line_of.get(previous)).zip(page.line_of.get(i)).is_some_and(|(a, b)| a != b);
|
||||
for (i, text) in texts.iter().enumerate() {
|
||||
let line_break = breaks_before(page, i);
|
||||
// A soft hyphen marks where a word may be divided at a line end (ISO 32000-2 §14.8.2.3), so
|
||||
// a line that ends in one runs on to the next with no word break between them.
|
||||
// Other invisible characters after the soft hyphen don't hide it.
|
||||
@@ -36,18 +160,29 @@ pub(super) fn find(page: &PageText, query: &str, sensitive: bool, whole_words: b
|
||||
// sides of the break are the visible ones: invisible characters are skipped below, and a
|
||||
// variation selector after the last one only picks its glyph.
|
||||
let joined = line_break && last_base.is_some_and(unspaced) && first_visible(page, i).is_some_and(unspaced) && wraps(page, i);
|
||||
let gap = page.space_before.get(i).copied().unwrap_or(false) || (line_break && !(joined || soft_break));
|
||||
// And, in the readings that join it, a line that ends in a visible hyphen between letters.
|
||||
let after_divide = reading != Hyphen::Written && i.checked_sub(1).and_then(|p| divided.get(p)).copied().unwrap_or(false);
|
||||
let gap = page.space_before.get(i).copied().unwrap_or(false) || (line_break && !(joined || soft_break || after_divide));
|
||||
if i > 0 && gap && chars.last().is_some_and(|c| c.0 != ' ') {
|
||||
chars.push((' ', i));
|
||||
last_base = Some(' ');
|
||||
zero_width_space.push(false);
|
||||
pending_zero_width_space = false;
|
||||
}
|
||||
for c in text.chars() {
|
||||
// The divided glyph's hyphen is its last visible character, left out of the joined reading.
|
||||
let dropped = if reading == Hyphen::Joined && divided.get(i).copied().unwrap_or(false) {
|
||||
text.char_indices().rev().find(|&(_, c)| !is_invisible(c)).map(|(at, _)| at)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
for (at, c) in text.char_indices() {
|
||||
if is_invisible(c) {
|
||||
pending_zero_width_space |= c == '\u{200B}';
|
||||
continue;
|
||||
}
|
||||
if Some(at) == dropped {
|
||||
continue;
|
||||
}
|
||||
let c = if c.is_whitespace() { ' ' } else { c };
|
||||
if c != ' ' || chars.last().is_none_or(|last| last.0 != ' ') {
|
||||
chars.push((c, i));
|
||||
@@ -59,35 +194,38 @@ pub(super) fn find(page: &PageText, query: &str, sensitive: bool, whole_words: b
|
||||
}
|
||||
}
|
||||
}
|
||||
(chars, zero_width_space)
|
||||
}
|
||||
|
||||
/// Non-overlapping hits of `needle` (with its KMP `prefix` table) in `chars`, as glyph ranges.
|
||||
/// `zero_width_space` says which characters come right after a skipped zero width space.
|
||||
fn search(
|
||||
chars: &[(char, usize)],
|
||||
zero_width_space: &[bool],
|
||||
needle: &[char],
|
||||
prefix: &[usize],
|
||||
whole_words: bool,
|
||||
limit: usize,
|
||||
) -> Vec<Range<usize>> {
|
||||
if needle.len() > chars.len() {
|
||||
return Vec::new();
|
||||
}
|
||||
// KMP reuses a matched prefix instead of comparing a long query again at every position.
|
||||
// The table is bounded by the page's already materialized character count.
|
||||
let mut prefix = vec![0; needle.len()];
|
||||
let mut matched = 0;
|
||||
for (i, &c) in needle.iter().enumerate().skip(1) {
|
||||
matched = advance(&needle, &prefix, matched, c);
|
||||
if let Some(slot) = prefix.get_mut(i) {
|
||||
*slot = matched;
|
||||
}
|
||||
}
|
||||
matched = 0;
|
||||
// A neighbour bounds a word only in a script that puts spaces between words. Han, kana, Thai
|
||||
// and the like run their words together, so a character of theirs next to the match is no
|
||||
// reason to reject it: Acrobat finds 台北 inside 台北市 with Whole words only on. Digits still
|
||||
// bound, in any script: ๑ is not a whole word inside the number ๑๒.
|
||||
let word = |i: Option<usize>| i.and_then(|i| chars.get(i)).is_some_and(|c| c.0.is_alphanumeric() && (c.0.is_numeric() || !unspaced_script(c.0)));
|
||||
// A skipped zero width space before the first character or after the last is a boundary.
|
||||
let after_break = |i: usize| zero_width_space.get(i).copied().unwrap_or(false);
|
||||
let mut hits = Vec::new();
|
||||
for (i, &(c, owner)) in chars.iter().enumerate() {
|
||||
matched = advance(&needle, &prefix, matched, c);
|
||||
matched = advance(needle, prefix, matched, c);
|
||||
if matched != needle.len() {
|
||||
continue;
|
||||
}
|
||||
let end = i.saturating_add(1);
|
||||
let start = end.saturating_sub(needle.len());
|
||||
// A skipped zero width space before the first character or after the last is a boundary.
|
||||
let after_break = |i: usize| zero_width_space.get(i).copied().unwrap_or(false);
|
||||
let bounded = !whole_words || ((after_break(start) || !word(start.checked_sub(1))) && (after_break(end) || !word(Some(end))));
|
||||
if bounded {
|
||||
if let Some((_, first)) = chars.get(start)
|
||||
@@ -240,3 +378,14 @@ fn unspaced_script(c: char) -> bool {
|
||||
| 0x20000..=0x3FFFF
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::letters;
|
||||
|
||||
#[test]
|
||||
fn ligatures_read_as_their_letters() {
|
||||
assert_eq!(letters("o\u{FB03}ce \u{FB00}\u{FB01}\u{FB02}\u{FB04}\u{FB05}\u{FB06}"), "office fffiflfflstst");
|
||||
assert_eq!(letters("\u{FB13}é\u{FF21}\u{2460}"), "\u{FB13}é\u{FF21}\u{2460}", "nothing else is normalised");
|
||||
}
|
||||
}
|
||||
@@ -88,68 +88,172 @@ const UNSPACED_BLOCK_SAMPLES: &[(char, bool, &str)] = &[
|
||||
('\u{323AF}', true, "CJK Unified Ideographs Extension H (31350..323AF), last"),
|
||||
];
|
||||
|
||||
// The original straightforward search is an independent oracle for valid page metadata.
|
||||
// The original straightforward search is an independent oracle for valid page metadata, extended
|
||||
// with ligature letters and the three readings of a line-end hyphen between letters: as written,
|
||||
// joined (no hyphen, no break) and kept (hyphen, no break).
|
||||
fn reference(page: &PageText, query: &str, sensitive: bool, whole: bool, limit: usize) -> Vec<Range<usize>> {
|
||||
let fold = |s: &str| if sensitive { s.to_owned() } else { fold_case(s) };
|
||||
let letters = |s: &str| {
|
||||
s.replace('\u{FB00}', "ff")
|
||||
.replace('\u{FB01}', "fi")
|
||||
.replace('\u{FB02}', "fl")
|
||||
.replace('\u{FB03}', "ffi")
|
||||
.replace('\u{FB04}', "ffl")
|
||||
.replace(['\u{FB05}', '\u{FB06}'], "st")
|
||||
};
|
||||
let fold = |s: &str| if sensitive { letters(s) } else { fold_case(&letters(s)) };
|
||||
// Soft hyphens and zero width characters draw nothing and are skipped on both sides.
|
||||
let visible =
|
||||
|s: String| s.chars().filter(|c| !matches!(c, '\u{AD}' | '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{2060}' | '\u{FEFF}')).collect::<String>();
|
||||
let invisible = |c: char| matches!(c, '\u{AD}' | '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{2060}' | '\u{FEFF}');
|
||||
let visible = |s: String| s.chars().filter(|&c| !invisible(c)).collect::<String>();
|
||||
let needle: Vec<char> = visible(fold(query)).split_whitespace().collect::<Vec<_>>().join(" ").chars().collect();
|
||||
if needle.is_empty() || limit == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
// Each character, its glyph, and whether a skipped zero width space came right before it.
|
||||
let mut chars: Vec<(char, usize, bool)> = Vec::new();
|
||||
let mut after_zwsp = false;
|
||||
for (i, glyph) in page.glyphs.iter().enumerate() {
|
||||
// A line that ends in a soft hyphen, past any empty or other invisible glyphs and
|
||||
// characters after it on that line, runs on to the next.
|
||||
let ends_in_soft_hyphen = |end: usize| {
|
||||
(0..=end)
|
||||
.rev()
|
||||
.take_while(|&k| page.line_of[k] == page.line_of[end])
|
||||
.flat_map(|k| page.glyphs[k].text.chars().rev())
|
||||
.find(|&c| c == '\u{AD}' || !visible(c.to_string()).is_empty())
|
||||
== Some('\u{AD}')
|
||||
};
|
||||
let line_break = i > 0 && page.line_of[i] != page.line_of[i - 1] && !ends_in_soft_hyphen(i - 1);
|
||||
if i > 0 && (page.space_before[i] || line_break) && chars.last().is_some_and(|&(c, ..)| c != ' ') {
|
||||
chars.push((' ', i, false));
|
||||
after_zwsp = false;
|
||||
}
|
||||
for c in fold(&glyph.text).chars() {
|
||||
after_zwsp |= c == '\u{200B}';
|
||||
if visible(c.to_string()).is_empty() {
|
||||
continue;
|
||||
}
|
||||
let c = if c.is_whitespace() { ' ' } else { c };
|
||||
if c != ' ' || chars.last().is_none_or(|&(last, ..)| last != ' ') {
|
||||
chars.push((c, i, after_zwsp && c != ' '));
|
||||
let texts: Vec<String> = page.glyphs.iter().map(|g| fold(&g.text)).collect();
|
||||
let n = texts.len();
|
||||
let new_line = |i: usize| i > 0 && page.line_of[i] != page.line_of[i - 1];
|
||||
// A line that ends in a soft hyphen, past any empty or other invisible glyphs and characters
|
||||
// after it on that line, runs on to the next.
|
||||
let ends_in_soft_hyphen = |end: usize| {
|
||||
(0..=end)
|
||||
.rev()
|
||||
.take_while(|&k| page.line_of[k] == page.line_of[end])
|
||||
.flat_map(|k| page.glyphs[k].text.chars().rev())
|
||||
.find(|&c| c == '\u{AD}' || !invisible(c))
|
||||
== Some('\u{AD}')
|
||||
};
|
||||
// A visible hyphen between letters at a line end.
|
||||
let hyphen = |c: char| matches!(c, '-' | '\u{2010}');
|
||||
let divided: Vec<bool> = (0..n)
|
||||
.map(|i| {
|
||||
let chars: Vec<char> = visible(texts[i].clone()).chars().collect();
|
||||
let Some(&last) = chars.last() else { return false };
|
||||
let before = if chars.len() >= 2 {
|
||||
Some(chars[chars.len() - 2])
|
||||
} else if i > 0 && !page.space_before[i] && !new_line(i) {
|
||||
visible(texts[i - 1].clone()).chars().last()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
hyphen(last)
|
||||
&& before.is_some_and(char::is_alphabetic)
|
||||
&& i + 1 < n
|
||||
&& new_line(i + 1)
|
||||
&& visible(texts[i + 1].clone()).chars().next().is_some_and(char::is_alphabetic)
|
||||
})
|
||||
.collect();
|
||||
let mut all = Vec::new();
|
||||
let readings = if divided.contains(&true) { 3 } else { 1 };
|
||||
for reading in 0..readings {
|
||||
// Each character, its glyph, and whether a skipped zero width space came right before it.
|
||||
let mut chars: Vec<(char, usize, bool)> = Vec::new();
|
||||
let mut after_zwsp = false;
|
||||
for i in 0..n {
|
||||
let joined = reading > 0 && i > 0 && divided[i - 1];
|
||||
let line_break = new_line(i) && !ends_in_soft_hyphen(i - 1) && !joined;
|
||||
if i > 0 && (page.space_before[i] || line_break) && chars.last().is_some_and(|&(c, ..)| c != ' ') {
|
||||
chars.push((' ', i, false));
|
||||
after_zwsp = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
// A neighbour bounds a word only in a script written with spaces between words, or when it
|
||||
// is a digit.
|
||||
let word = |i: Option<usize>| i.and_then(|i| chars.get(i)).is_some_and(|&(c, ..)| c.is_alphanumeric() && (c.is_numeric() || !unspaced(c)));
|
||||
// A word ends at a skipped zero width space as it does at a space.
|
||||
let edge = |i: usize| chars.get(i).is_some_and(|&(.., zwsp)| zwsp);
|
||||
let mut hits = Vec::new();
|
||||
let mut start = 0;
|
||||
while start + needle.len() <= chars.len() {
|
||||
let end = start + needle.len();
|
||||
let bounded = (edge(start) || !word(start.checked_sub(1))) && (edge(end) || !word(Some(end)));
|
||||
if (!whole || bounded) && chars[start..end].iter().map(|&(c, ..)| c).eq(needle.iter().copied()) {
|
||||
hits.push(chars[start].1..chars[end - 1].1 + 1);
|
||||
if hits.len() == limit {
|
||||
break;
|
||||
let mut glyph: Vec<char> = texts[i].chars().collect();
|
||||
if reading == 1 && divided[i] {
|
||||
// The hyphen: the last visible character.
|
||||
if let Some(at) = glyph.iter().rposition(|&c| !invisible(c)) {
|
||||
glyph.remove(at);
|
||||
}
|
||||
}
|
||||
start = end;
|
||||
} else {
|
||||
start += 1;
|
||||
for c in glyph {
|
||||
after_zwsp |= c == '\u{200B}';
|
||||
if invisible(c) {
|
||||
continue;
|
||||
}
|
||||
let c = if c.is_whitespace() { ' ' } else { c };
|
||||
if c != ' ' || chars.last().is_none_or(|&(last, ..)| last != ' ') {
|
||||
chars.push((c, i, after_zwsp && c != ' '));
|
||||
after_zwsp = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
// A neighbour bounds a word only in a script written with spaces between words, or when it
|
||||
// is a digit.
|
||||
let word = |i: Option<usize>| i.and_then(|i| chars.get(i)).is_some_and(|&(c, ..)| c.is_alphanumeric() && (c.is_numeric() || !unspaced(c)));
|
||||
// A word ends at a skipped zero width space as it does at a space.
|
||||
let edge = |i: usize| chars.get(i).is_some_and(|&(.., zwsp)| zwsp);
|
||||
let mut hits = Vec::new();
|
||||
let mut start = 0;
|
||||
while start + needle.len() <= chars.len() {
|
||||
let end = start + needle.len();
|
||||
let bounded = (edge(start) || !word(start.checked_sub(1))) && (edge(end) || !word(Some(end)));
|
||||
if (!whole || bounded) && chars[start..end].iter().map(|&(c, ..)| c).eq(needle.iter().copied()) {
|
||||
hits.push(chars[start].1..chars[end - 1].1 + 1);
|
||||
if hits.len() == limit {
|
||||
break;
|
||||
}
|
||||
start = end;
|
||||
} else {
|
||||
start += 1;
|
||||
}
|
||||
}
|
||||
all.push(hits);
|
||||
}
|
||||
if readings == 1 {
|
||||
return all.remove(0);
|
||||
}
|
||||
// A hit is left out where it overlaps one kept from another reading; one reading's hits are
|
||||
// all kept, though two can share a glyph.
|
||||
let mut hits: Vec<(Range<usize>, usize)> = all.into_iter().enumerate().flat_map(|(r, hits)| hits.into_iter().map(move |h| (h, r))).collect();
|
||||
hits.sort_by_key(|(h, _)| (h.start, std::cmp::Reverse(h.end)));
|
||||
let mut out: Vec<(Range<usize>, usize)> = Vec::new();
|
||||
for (h, r) in hits {
|
||||
let overlaps = out.iter().any(|(k, kr)| *kr != r && h.start < k.end && k.start < h.end);
|
||||
if !overlaps && out.len() < limit {
|
||||
out.push((h, r));
|
||||
}
|
||||
}
|
||||
hits
|
||||
out.into_iter().map(|(h, _)| h).collect()
|
||||
}
|
||||
|
||||
/// One glyph per entry of `text`, each on the line given in `lines`.
|
||||
fn lined(text: &[&str], lines: &[u32]) -> PageText {
|
||||
let mut p = page(text);
|
||||
p.line_of = lines.to_vec();
|
||||
p
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_ligature_matches_its_letters() {
|
||||
let text = page(&["o", "\u{FB03}", "c", "e", " ", "o", "f", "\u{FB01}", "c", "e"]);
|
||||
assert_eq!(text.find("office"), vec![0..4, 5..10]);
|
||||
assert_eq!(text.find("OFFICE"), vec![0..4, 5..10]);
|
||||
assert_eq!(text.find_opts("office", true, true), vec![0..4, 5..10], "case-sensitive and whole words too");
|
||||
assert_eq!(text.find("o\u{FB03}ce"), vec![0..4, 5..10], "a ligature in the query reads as its letters too");
|
||||
assert_eq!(text.find("fi"), vec![1..2, 7..8], "a match inside a ligature takes its glyph");
|
||||
assert!(text.find("o\u{FB13}").is_empty(), "no other compatibility folding");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_word_divided_by_a_line_end_hyphen_matches_whole_and_as_written() {
|
||||
// "exam-" ends line 0, "ple" starts line 1; the hyphen is its own glyph.
|
||||
let text = lined(&["e", "x", "a", "m", "-", "p", "l", "e"], &[0, 0, 0, 0, 0, 1, 1, 1]);
|
||||
assert_eq!(text.find("example"), vec![0..8], "both parts, and the hyphen between them");
|
||||
assert_eq!(text.find_opts("example", false, true), vec![0..8], "a whole word");
|
||||
assert_eq!(text.find("exam-ple"), vec![0..8], "a query with the hyphen");
|
||||
assert_eq!(text.find("exam- ple"), vec![0..8], "and with the line break as a space");
|
||||
assert_eq!(text.find("exam"), vec![0..4]);
|
||||
// The hyphen in the glyph of the last letter, a soft hyphen and U+2010.
|
||||
for hyphen in ["m-", "m\u{AD}", "m\u{2010}"] {
|
||||
let text = lined(&["e", "x", "a", hyphen, "p", "l", "e"], &[0, 0, 0, 0, 1, 1, 1]);
|
||||
assert_eq!(text.find("example"), vec![0..7], "{hyphen:?}");
|
||||
}
|
||||
// Not divided: a hyphen inside a line, a dash after a word gap, a hyphen between digits.
|
||||
assert!(page(&["e", "x", "a", "m", "-", "p", "l", "e"]).find("example").is_empty(), "inside a line");
|
||||
let mut spaced = lined(&["e", "x", "a", "m", "-", "p", "l", "e"], &[0, 0, 0, 0, 0, 1, 1, 1]);
|
||||
spaced.space_before[4] = true;
|
||||
assert!(spaced.find("example").is_empty(), "a dash after a word gap");
|
||||
assert!(spaced.find("exam ple").is_empty(), "a dash after a word gap is never dropped");
|
||||
assert!(lined(&["1", "0", "-", "2", "0"], &[0, 0, 0, 1, 1]).find("1020").is_empty(), "between digits");
|
||||
assert!(lined(&["a", "-", "2", "0"], &[0, 0, 1, 1]).find("a20").is_empty(), "before a number");
|
||||
assert!(lined(&["1", "0", "-", "a", "b"], &[0, 0, 0, 1, 1]).find("10ab").is_empty(), "after a number");
|
||||
assert!(lined(&["a", "-"], &[0, 0]).find("a-").len() == 1, "at the end of the page");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -171,6 +275,10 @@ fn whole_word_rejection_keeps_overlapping_candidates() {
|
||||
// Accepted matches still do not overlap, even within one multi-character glyph.
|
||||
assert_eq!(page(&["aaaaa"]).find("aaa"), vec![0..1]);
|
||||
assert_eq!(page(&["aaaaa"]).find("aa"), vec![0..1, 0..1]);
|
||||
// So does a page with a word divided at a line end, which is searched in three readings.
|
||||
let divided = lined(&["\u{FB03}", " ", "exam", "-", "ple"], &[0, 0, 0, 0, 1]);
|
||||
assert_eq!(divided.find("f"), vec![0..1, 0..1]);
|
||||
assert_eq!(divided.find("example"), vec![2..5]);
|
||||
}
|
||||
|
||||
/// A letter from a block in `UNSPACED_BLOCK_SAMPLES` next to a whole-word match leaves it standing
|
||||
@@ -191,6 +299,8 @@ fn whole_words_follow_the_unicode_blocks_of_unspaced_scripts() {
|
||||
#[test]
|
||||
fn search_matches_reference_for_unicode_gaps_limits_and_word_boundaries() {
|
||||
let alphabet = ["a", "b", " ", "-", "İ", "ı", "ς", "Σ", "\t", "\n", "ffi", "", "界", "ก", "한", "e\u{301}", "العربية", "\u{AD}", "a\u{200B}"];
|
||||
// Ligatures, digits and HYPHEN exercise the readings of a line-end hyphen.
|
||||
let alphabet: Vec<&str> = alphabet.into_iter().chain(["\u{FB01}", "1", "\u{2010}"]).collect();
|
||||
let mut seed = 1234567u64;
|
||||
let mut next = || {
|
||||
seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);
|
||||
|
||||
@@ -1418,8 +1418,8 @@ milestone = "M3"
|
||||
status = "shipped"
|
||||
commands = ["edit.find"]
|
||||
tools = ["text_find"]
|
||||
evidence = ["crates/ui-egui/tests/it/ui.rs::find_counts_matches_across_pages", "crates/ui-egui/tests/it/ui.rs::find_reports_no_matches", "crates/automation/tests/it/automation.rs::open_inspect_render_and_find", "crates/render/src/text/tests.rs::identity_v_columns_are_searchable_in_reading_order", "crates/render/src/text/tests.rs::toc_leaders_do_not_hide_word_gaps", "crates/render/src/text/tests.rs::short_tracked_ellipses_remain_searchable", "crates/render/tests/it/search.rs::find_skips_soft_hyphens_and_zero_width_characters", "crates/render/tests/it/search.rs::find_joins_a_word_divided_at_a_line_end_soft_hyphen", "crates/render/tests/it/search.rs::a_zero_width_space_is_a_word_boundary_for_whole_words", "crates/render/tests/it/search.rs::a_chinese_or_japanese_word_divided_by_a_line_end_is_found_on_both_lines", "crates/render/tests/it/search.rs::a_line_break_stays_a_space_beside_latin_or_hangul_and_between_side_by_side_text", "crates/render/tests/it/search.rs::a_word_divided_between_vertical_columns_is_found_in_both", "crates/render/tests/it/search.rs::each_unspaced_range_joins_and_spaced_scripts_do_not", "crates/render/tests/it/search.rs::a_soft_hyphen_and_a_chinese_or_japanese_line_end_combine", "crates/render/src/text/tests.rs::thai_marks_drawn_one_glyph_at_a_time_keep_their_place_and_words"]
|
||||
notes = "Horizontal and WMode 1 vertical text are grouped independently, including a horizontal heading above right-to-left vertical columns. Dotted TOC leaders preserve word spaces for phrase search (#3086), while short tracked ellipses remain intact and searchable; sparse lines and condensed OCR gaps below the existing height-based threshold remain limitations. Combining marks (Mn/Me) of left-to-right scripts drawn right after their letter stay with it in content order, and Thai or Indic word gaps without a space glyph are found as spaces (#6517). Complex interleaved flows still use a geometric reading-order heuristic. Soft hyphens, zero width spaces and joiners, word joiners and U+FEFF inside a word are skipped, in the page and in the query; a skipped zero width space still ends a word for Whole words only. A line break reads as no word break when the line ends in a soft hyphen, or when it runs on between two Chinese or Japanese characters (East Asian Wide, Fullwidth or Halfwidth, not Hangul and not fullwidth Latin letters or digits), so a divided word is found, in horizontal lines and in vertical columns that continue to the left; the break stays a space beside Latin or Korean text and between text set side by side on one line or in staggered cells."
|
||||
evidence = ["crates/ui-egui/tests/it/ui.rs::find_counts_matches_across_pages", "crates/ui-egui/tests/it/ui.rs::find_reports_no_matches", "crates/automation/tests/it/automation.rs::open_inspect_render_and_find", "crates/render/src/text/tests.rs::identity_v_columns_are_searchable_in_reading_order", "crates/render/src/text/tests.rs::toc_leaders_do_not_hide_word_gaps", "crates/render/src/text/tests.rs::short_tracked_ellipses_remain_searchable", "crates/render/tests/it/search.rs::find_skips_soft_hyphens_and_zero_width_characters", "crates/render/tests/it/search.rs::find_joins_a_word_divided_at_a_line_end_soft_hyphen", "crates/render/tests/it/search.rs::a_zero_width_space_is_a_word_boundary_for_whole_words", "crates/render/tests/it/search.rs::a_chinese_or_japanese_word_divided_by_a_line_end_is_found_on_both_lines", "crates/render/tests/it/search.rs::a_line_break_stays_a_space_beside_latin_or_hangul_and_between_side_by_side_text", "crates/render/tests/it/search.rs::a_word_divided_between_vertical_columns_is_found_in_both", "crates/render/tests/it/search.rs::each_unspaced_range_joins_and_spaced_scripts_do_not", "crates/render/tests/it/search.rs::a_soft_hyphen_and_a_chinese_or_japanese_line_end_combine", "crates/render/src/text/tests.rs::thai_marks_drawn_one_glyph_at_a_time_keep_their_place_and_words", "crates/render/tests/it/search.rs::a_ligature_matches_its_letters", "crates/render/tests/it/search.rs::a_word_divided_by_a_line_end_hyphen_matches_whole_and_as_written", "crates/automation/tests/it/find_text.rs::text_find_matches_ligatures_and_words_divided_at_a_line_end"]
|
||||
notes = "Horizontal and WMode 1 vertical text are grouped independently, including a horizontal heading above right-to-left vertical columns. Dotted TOC leaders preserve word spaces for phrase search (#3086), while short tracked ellipses remain intact and searchable; sparse lines and condensed OCR gaps below the existing height-based threshold remain limitations. Combining marks (Mn/Me) of left-to-right scripts drawn right after their letter stay with it in content order, and Thai or Indic word gaps without a space glyph are found as spaces (#6517). Complex interleaved flows still use a geometric reading-order heuristic. Soft hyphens, zero width spaces and joiners, word joiners and U+FEFF inside a word are skipped, in the page and in the query; a skipped zero width space still ends a word for Whole words only. A line break reads as no word break when the line ends in a soft hyphen, or when it runs on between two Chinese or Japanese characters (East Asian Wide, Fullwidth or Halfwidth, not Hangul and not fullwidth Latin letters or digits), so a divided word is found, in horizontal lines and in vertical columns that continue to the left; the break stays a space beside Latin or Korean text and between text set side by side on one line or in staggered cells. A Latin ligature (U+FB00–FB06) matches its letters, and a word divided by a hyphen (U+002D or U+2010) between letters at a line end matches whole, highlighted on both lines, as well as with the hyphen. Missing: accent-insensitive matching (an unaccented query finding accented text; an open pull request adds it)."
|
||||
|
||||
[[feature]]
|
||||
id = "view.find-next-previous"
|
||||
|
||||
Reference in new issue
Block a user