From cc1d76f3fd9fab7d8ce9d9a140b74d49a4076431 Mon Sep 17 00:00:00 2001 From: mtlcl410 <2368091+mtlcl410@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:09:20 +0800 Subject: [PATCH] M3: whole-word Find matches inside Chinese, Japanese and Thai text (view.find-options) (#2075) --- crates/render/src/text/search.rs | 37 +++++++++++- crates/render/src/text/tests.rs | 42 ++++++++++++++ crates/render/tests/it/search.rs | 99 +++++++++++++++++++++++++++++++- parity/acrobat-features.toml | 4 +- 4 files changed, 177 insertions(+), 5 deletions(-) diff --git a/crates/render/src/text/search.rs b/crates/render/src/text/search.rs index 71670cc271..bf64334d6e 100644 --- a/crates/render/src/text/search.rs +++ b/crates/render/src/text/search.rs @@ -73,7 +73,11 @@ pub(super) fn find(page: &PageText, query: &str, sensitive: bool, whole_words: b } } matched = 0; - let word = |i: Option| i.and_then(|i| chars.get(i)).is_some_and(|c| c.0.is_alphanumeric()); + // A neighbour bounds a word only in a script that puts spaces between words. Han, kana, Thai + // and the like run their words together, so a character of theirs next to the match is no + // reason to reject it: Acrobat finds 台北 inside 台北市 with Whole words only on. Digits still + // bound, in any script: ๑ is not a whole word inside the number ๑๒. + let word = |i: Option| i.and_then(|i| chars.get(i)).is_some_and(|c| c.0.is_alphanumeric() && (c.0.is_numeric() || !unspaced_script(c.0))); let mut hits = Vec::new(); for (i, &(c, owner)) in chars.iter().enumerate() { matched = advance(&needle, &prefix, matched, c); @@ -205,3 +209,34 @@ fn advance(needle: &[char], prefix: &[usize], mut matched: usize, c: char) -> us } if needle.get(matched) == Some(&c) { matched.saturating_add(1) } else { 0 } } + +/// Whether `c` is a letter of a script written without spaces between words (CJK ideographs, +/// kana, bopomofo, Thai, Lao, Myanmar, Khmer, Tibetan), so that a whole-word search cannot take a +/// neighbouring letter as a word boundary. +fn unspaced_script(c: char) -> bool { + matches!( + u32::from(c), + 0x0E00..=0x0EFF // Thai, Lao + | 0x0F00..=0x0FFF // Tibetan + | 0x1000..=0x109F // Myanmar + | 0x1780..=0x17FF // Khmer + | 0x2E80..=0x2FDF // CJK and Kangxi radicals + | 0x3005..=0x3007 // 々 〆 〇 + | 0x3031..=0x3035 // vertical kana repeat marks + | 0x303B..=0x303C // 〻 〼 + | 0x3040..=0x30FF // hiragana, katakana + | 0x3100..=0x312F // bopomofo + | 0x31A0..=0x31BF // bopomofo extended + | 0x31F0..=0x31FF // katakana phonetic extensions + | 0x3400..=0x4DBF // CJK extension A + | 0x4E00..=0x9FFF // CJK unified ideographs + | 0xA9E0..=0xA9FF // Myanmar extended-B + | 0xAA60..=0xAA7F // Myanmar extended-A + | 0xF900..=0xFAFF // CJK compatibility ideographs + | 0xFF66..=0xFF9F // halfwidth katakana + | 0x1AFF0..=0x1B16F // kana extended-B, kana supplement, kana extended-A, small kana + // The supplementary and tertiary ideographic planes, all CJK ideographs (extensions B + // to J, compatibility supplement). + | 0x20000..=0x3FFFF + ) +} diff --git a/crates/render/src/text/tests.rs b/crates/render/src/text/tests.rs index a3c67696fe..c7601998b8 100644 --- a/crates/render/src/text/tests.rs +++ b/crates/render/src/text/tests.rs @@ -1052,6 +1052,48 @@ fn upright_mapping_round_trips() { assert_eq!(to_upright([290.0, 0.0, 300.0, 10.0], 1, 300.0, 200.0), [0.0, 0.0, 10.0, 10.0]); } +#[test] +fn whole_words_ignore_neighbours_in_scripts_written_without_spaces() { + // "台北市的PDF檔案" in one run: Chinese writes no spaces between words, so "台北" is a + // whole word inside "台北市" and "PDF" inside "的PDF檔", as Acrobat finds them with Whole + // words only on. A Latin neighbour still bounds: "cat" is not a whole word in "category". + let mut v = Vec::new(); + word(&mut v, "台北市的PDF檔案", 10.0, 10.0, 6.0); + word(&mut v, "category", 74.0, 10.0, 6.0); + let t = layout(v); + assert_eq!(t.plain_text(), "台北市的PDF檔案 category"); + assert_eq!(t.find_opts("台北", false, true), vec![0..2], "a word inside a longer run of ideographs"); + assert_eq!(t.find_opts("北市", false, true), vec![1..3]); + assert_eq!(t.find_opts("PDF", false, true), vec![4..7], "Latin between ideographs"); + assert_eq!(t.find_opts("pdf檔", false, true), vec![4..8]); + assert!(t.find_opts("cat", false, true).is_empty(), "Latin neighbours still bound a word"); + assert_eq!(t.find_opts("category", false, true), vec![9..17]); + // Thai and Japanese kana are written without spaces too. + let mut v = Vec::new(); + word(&mut v, "ภาษาไทย", 10.0, 30.0, 6.0); + word(&mut v, "ひらがなとカタカナ", 62.0, 30.0, 6.0); + let t = layout(v); + assert_eq!(t.find_opts("ไทย", false, true), vec![4..7]); + assert_eq!(t.find_opts("カタカナ", false, true), vec![12..16]); +} + +/// #2075 review: digits still bound a whole word in those scripts (๑ is not a word inside +/// the number ๑๒), and letters from the later Myanmar and Han blocks don't. +#[test] +fn whole_words_in_unspaced_scripts_keep_digits_and_later_blocks() { + let one = |text: &str| { + let mut v = Vec::new(); + word(&mut v, text, 10.0, 10.0, 6.0); + layout(v) + }; + assert!(one("๑๒").find_opts("๑", false, true).is_empty(), "a Thai digit beside it bounds"); + assert!(one("台北3").find_opts("台北", false, true).is_empty(), "so does an ASCII digit"); + // Myanmar Extended-A (U+AA60), Myanmar Extended-B (U+A9E0), CJK Extension H (U+31350). + for text in ["\u{AA60}PDF", "\u{A9E0}PDF", "\u{31350}PDF"] { + assert_eq!(one(text).find_opts("PDF", false, true), vec![1..4], "{text}"); + } +} + #[test] fn turkish_dotted_and_dotless_i_match_in_any_case() { // "İSTANBUL IRMAK KIŞ" on one line: each word typed in lower case must find it, and the diff --git a/crates/render/tests/it/search.rs b/crates/render/tests/it/search.rs index 5b2a1e4c46..0c20e2c7b5 100644 --- a/crates/render/tests/it/search.rs +++ b/crates/render/tests/it/search.rs @@ -9,6 +9,84 @@ fn page(parts: &[&str]) -> PageText { } } +/// The letters of the property test's alphabet that come from scripts written without spaces +/// between words: 界 (CJK Unified Ideographs, U+4E00..U+9FFF) and ก (Thai, U+0E00..U+0E7F). +/// 한 (Hangul Syllables) is written with spaces and bounds a word. The reference knows only +/// these three letters, so it does not share the block table in `text/search.rs`; that table is +/// checked against `UNSPACED_BLOCK_SAMPLES` instead. +fn unspaced(c: char) -> bool { + matches!(c, '界' | 'ก') +} + +/// Letters at the edges of each block of a script written without spaces (`true`), and letters +/// just outside them (`false`), named by their block as Unicode's Blocks.txt gives it. Each one is +/// alphabetic and not a digit, so whether it bounds a whole word depends on the table alone. +const UNSPACED_BLOCK_SAMPLES: &[(char, bool, &str)] = &[ + ('\u{0DC6}', false, "Sinhala (0D80..0DFF), before Thai"), + ('\u{0E01}', true, "Thai (0E00..0E7F), first letter"), + ('\u{0E46}', true, "Thai, MAIYAMOK"), + ('\u{0E81}', true, "Lao (0E80..0EFF), first letter"), + ('\u{0EDF}', true, "Lao, last letter"), + ('\u{0F00}', true, "Tibetan (0F00..0FFF), first letter"), + ('\u{0F8C}', true, "Tibetan, last letter"), + ('\u{1000}', true, "Myanmar (1000..109F), first letter"), + ('\u{108E}', true, "Myanmar, last letter"), + ('\u{10A0}', false, "Georgian (10A0..10FF), after Myanmar"), + ('\u{1770}', false, "Tagbanwa (1760..177F), before Khmer"), + ('\u{1780}', true, "Khmer (1780..17FF), first letter"), + ('\u{17DC}', true, "Khmer, last letter"), + ('\u{1820}', false, "Mongolian (1800..18AF), after Khmer"), + ('\u{3005}', true, "CJK Symbols and Punctuation (3000..303F), ideographic iteration mark"), + ('\u{3006}', true, "CJK Symbols and Punctuation, ideographic closing mark"), + ('\u{3031}', true, "CJK Symbols and Punctuation, vertical kana repeat mark"), + ('\u{3035}', true, "CJK Symbols and Punctuation, vertical kana repeat mark lower half"), + ('\u{303B}', true, "CJK Symbols and Punctuation, vertical ideographic iteration mark"), + ('\u{303C}', true, "CJK Symbols and Punctuation, masu mark"), + ('\u{3041}', true, "Hiragana (3040..309F), first letter"), + ('\u{309F}', true, "Hiragana, last letter"), + ('\u{30A1}', true, "Katakana (30A0..30FF), first letter"), + ('\u{30FF}', true, "Katakana, last letter"), + ('\u{3105}', true, "Bopomofo (3100..312F), first letter"), + ('\u{312F}', true, "Bopomofo, last letter"), + ('\u{3131}', false, "Hangul Compatibility Jamo (3130..318F), after Bopomofo"), + ('\u{318E}', false, "Hangul Compatibility Jamo, last letter, before Bopomofo Extended"), + ('\u{31A0}', true, "Bopomofo Extended (31A0..31BF), first letter"), + ('\u{31BF}', true, "Bopomofo Extended, last letter"), + ('\u{31F0}', true, "Katakana Phonetic Extensions (31F0..31FF), first letter"), + ('\u{31FF}', true, "Katakana Phonetic Extensions, last letter"), + ('\u{3400}', true, "CJK Unified Ideographs Extension A (3400..4DBF), first"), + ('\u{4DBF}', true, "CJK Unified Ideographs Extension A, last"), + ('\u{4E00}', true, "CJK Unified Ideographs (4E00..9FFF), first"), + ('\u{9FFF}', true, "CJK Unified Ideographs, last"), + ('\u{A000}', false, "Yi Syllables (A000..A48F), after CJK Unified Ideographs"), + ('\u{A9CF}', false, "Javanese (A980..A9DF), before Myanmar Extended-B"), + ('\u{A9E0}', true, "Myanmar Extended-B (A9E0..A9FF), first letter"), + ('\u{A9FE}', true, "Myanmar Extended-B, last letter"), + ('\u{AA00}', false, "Cham (AA00..AA5F), after Myanmar Extended-B"), + ('\u{AA4B}', false, "Cham, before Myanmar Extended-A"), + ('\u{AA60}', true, "Myanmar Extended-A (AA60..AA7F), first letter"), + ('\u{AA7F}', true, "Myanmar Extended-A, last letter"), + ('\u{AA80}', false, "Tai Viet (AA80..AADF), after Myanmar Extended-A"), + ('\u{D7FB}', false, "Hangul Jamo Extended-B (D7B0..D7FF), before CJK Compatibility Ideographs"), + ('\u{F900}', true, "CJK Compatibility Ideographs (F900..FAFF), first"), + ('\u{FAD9}', true, "CJK Compatibility Ideographs, last"), + ('\u{FB00}', false, "Alphabetic Presentation Forms (FB00..FB4F), after CJK Compatibility Ideographs"), + ('\u{FF5A}', false, "Halfwidth and Fullwidth Forms (FF00..FFEF), fullwidth z, before halfwidth katakana"), + ('\u{FF66}', true, "Halfwidth and Fullwidth Forms, first halfwidth katakana"), + ('\u{FF9F}', true, "Halfwidth and Fullwidth Forms, last halfwidth katakana"), + ('\u{FFA1}', false, "Halfwidth and Fullwidth Forms, halfwidth Hangul, after halfwidth katakana"), + ('\u{1AFF0}', true, "Kana Extended-B (1AFF0..1AFFF), first letter"), + ('\u{1B000}', true, "Kana Supplement (1B000..1B0FF), first letter"), + ('\u{1B122}', true, "Kana Extended-A (1B100..1B12F), last letter"), + ('\u{1B132}', true, "Small Kana Extension (1B130..1B16F), first letter"), + ('\u{1B167}', true, "Small Kana Extension, last letter"), + ('\u{1B170}', false, "Nushu (1B170..1B2FF), after Small Kana Extension"), + ('\u{20000}', true, "CJK Unified Ideographs Extension B (20000..2A6DF), first"), + ('\u{2A6DF}', true, "CJK Unified Ideographs Extension B, last"), + ('\u{2F800}', true, "CJK Compatibility Ideographs Supplement (2F800..2FA1F), first"), + ('\u{323AF}', true, "CJK Unified Ideographs Extension H (31350..323AF), last"), +]; + // The original straightforward search is an independent oracle for valid page metadata. fn reference(page: &PageText, query: &str, sensitive: bool, whole: bool, limit: usize) -> Vec> { let fold = |s: &str| if sensitive { s.to_owned() } else { fold_case(s) }; @@ -50,7 +128,9 @@ fn reference(page: &PageText, query: &str, sensitive: bool, whole: bool, limit: } } } - let word = |i: Option| i.and_then(|i| chars.get(i)).is_some_and(|&(c, ..)| c.is_alphanumeric()); + // A neighbour bounds a word only in a script written with spaces between words, or when it + // is a digit. + let word = |i: Option| i.and_then(|i| chars.get(i)).is_some_and(|&(c, ..)| c.is_alphanumeric() && (c.is_numeric() || !unspaced(c))); // A word ends at a skipped zero width space as it does at a space. let edge = |i: usize| chars.get(i).is_some_and(|&(.., zwsp)| zwsp); let mut hits = Vec::new(); @@ -92,9 +172,24 @@ fn whole_word_rejection_keeps_overlapping_candidates() { assert_eq!(page(&["aaaaa"]).find("aa"), vec![0..1, 0..1]); } +/// A letter from a block in `UNSPACED_BLOCK_SAMPLES` next to a whole-word match leaves it standing +/// when the block's script is written without spaces, and bounds it otherwise, on either side. +#[test] +fn whole_words_follow_the_unicode_blocks_of_unspaced_scripts() { + for &(c, unspaced, block) in UNSPACED_BLOCK_SAMPLES { + let (neighbour, cp) = (c.to_string(), u32::from(c)); + assert!(c.is_alphanumeric() && !c.is_numeric(), "U+{cp:04X} ({block}) is a letter, so the sample tests the table"); + // Case-sensitive, so that case folding does not move a neighbour into another block + // (Georgian U+10A0 folds to U+2D00). + let (before, after) = if unspaced { (Some(1..2), Some(0..1)) } else { (None, None) }; + assert_eq!(page(&[&neighbour, "PDF"]).find_opts("PDF", true, true), Vec::from_iter(before), "U+{cp:04X} ({block}) before the match"); + assert_eq!(page(&["PDF", &neighbour]).find_opts("PDF", true, true), Vec::from_iter(after), "U+{cp:04X} ({block}) after the match"); + } +} + #[test] fn search_matches_reference_for_unicode_gaps_limits_and_word_boundaries() { - let alphabet = ["a", "b", " ", "-", "İ", "ı", "ς", "Σ", "\t", "\n", "ffi", "", "界", "e\u{301}", "العربية", "\u{AD}", "a\u{200B}"]; + let alphabet = ["a", "b", " ", "-", "İ", "ı", "ς", "Σ", "\t", "\n", "ffi", "", "界", "ก", "한", "e\u{301}", "العربية", "\u{AD}", "a\u{200B}"]; let mut seed = 1234567u64; let mut next = || { seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 0c614c67fb..4ec064518b 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -1430,8 +1430,8 @@ milestone = "M3" status = "shipped" commands = ["edit.find"] tools = [] -evidence = ["crates/ui-egui/tests/it/viewer.rs::view_history_select_all_and_find_options"] -notes = "whole words only, case-sensitive" +evidence = ["crates/ui-egui/tests/it/viewer.rs::view_history_select_all_and_find_options", "crates/render/src/text/tests.rs::whole_words_ignore_neighbours_in_scripts_written_without_spaces", "crates/render/tests/it/search.rs::whole_words_follow_the_unicode_blocks_of_unspaced_scripts"] +notes = "whole words only, case-sensitive. Whole words: a neighbouring letter bounds the match only in a script that writes spaces between words; next to Han, kana, bopomofo, Thai, Lao, Myanmar, Khmer or Tibetan letters the match stands, so 台北 is found in 台北市 as in Acrobat" [[feature]] id = "view.find-in-comments-bookmarks"