mirror of
https://github.com/storytold/pdfcraft.git
synced 2026-11-10 13:01:29 +00:00
M3: a taken-back space moves only inside right-to-left text, past marks drawn before its letter (view.copy-text)
A neutral taken back over a space keeps the space before it only when the nearest strong glyphs on both sides are right-to-left (or the line edge), so a parenthesis opening Latin text is placed by x as on a line without Hebrew. Combining marks drawn between the space and its letter no longer stop the move, and the space, the marks and the letter keep drawing order. The pass now runs after zero-width marks are merged, since that merge relies on x order. Adds tests for the pdf.js #11656 content stream and for two taken-back spaces on one line. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Pq7HyCHD1R3rywQBPCHXZA
This commit is contained in:
mtlcl410andClaude Opus 5.5 committed
1 parent
3cbf001256
commit
fc8a4d6974
2 files changed
+249
-12
No files matched your search
@@ -305,7 +305,6 @@ pub(super) fn place_glyphs(glyphs: &[&TextGlyph], height: &dyn Fn(usize) -> f32,
|
||||
} else {
|
||||
collide(glyphs, height, s.h, sc);
|
||||
}
|
||||
keep_taken_back_blanks(glyphs, &mut sc.placed);
|
||||
// Each mark goes before the first placed glyph whose x is greater than its own, or its letter's when it is a
|
||||
// combining mark riding on one. Marks are visited in that order, and that position never moves back, so one pass
|
||||
// over `placed` puts all of them in.
|
||||
@@ -323,8 +322,15 @@ pub(super) fn place_glyphs(glyphs: &[&TextGlyph], height: &dyn Fn(usize) -> f32,
|
||||
sc.merged.extend_from_slice(&sc.placed[next..]);
|
||||
std::mem::swap(&mut sc.placed, &mut sc.merged);
|
||||
}
|
||||
// After the marks: their merge relies on `placed` being in x order, which this breaks on purpose.
|
||||
keep_taken_back_blanks(glyphs, &mut sc.placed);
|
||||
}
|
||||
|
||||
/// How many glyphs [`keep_taken_back_blanks`] looks at past a blank: marks between it and its letter, in drawing order
|
||||
/// and in x order, and glyphs without a strong direction on each side of a taken-back neutral. Each scan stops there,
|
||||
/// so the pass stays linear in the line's glyphs.
|
||||
const TAKEN_BACK_SCAN: usize = 8;
|
||||
|
||||
/// Puts a blank glyph (a space) back before the glyph drawn right after it on a line with right-to-left text, when that
|
||||
/// glyph's adjustment took the blank's advance back: it starts at or left of the blank's own start and covers at least
|
||||
/// half of it. Right-to-left text drawn in reverse order starts a show string with the space that ends its word
|
||||
@@ -333,26 +339,93 @@ pub(super) fn place_glyphs(glyphs: &[&TextGlyph], height: &dyn Fn(usize) -> f32,
|
||||
/// letter it belongs on, and sorting by x split "המדינה" as "המדינ ה". The space is a word break in the character
|
||||
/// stream (§14.8.2.6.2), and drawing order is the only evidence of its place.
|
||||
///
|
||||
/// Only a right-to-left letter or a neutral (punctuation, a symbol) after the blank counts: left-to-right text has no
|
||||
/// reverse-order show strings, and in scanned pages' invisible text layers its spaces overlap letters at random. A
|
||||
/// glyph that only touches the blank's start was drawn beside it, as text set letter by letter leftwards is, and x
|
||||
/// order already reads it right. A blank that a glyph placed before them also reaches over sits inside a cluster (a
|
||||
/// letter and the marks drawn on it), where its place is no clearer, so it stays where x order put it.
|
||||
/// Combining marks (bidi class NSM) drawn between the blank and its letter are passed over, and the blank, those marks
|
||||
/// and the letter keep drawing order, which puts each mark after its letter once the line is reversed, as Unicode
|
||||
/// stores it. Only a right-to-left letter or a neutral (punctuation, a symbol) after the blank counts: left-to-right
|
||||
/// text has no reverse-order show strings, and in scanned pages' invisible text layers its spaces overlap letters at
|
||||
/// random. A neutral counts only inside right-to-left text: the nearest glyph with a strong direction before it, and
|
||||
/// the nearest after the blank, is right-to-left or the line's edge. A parenthesis that opens Latin text then stays
|
||||
/// where x order puts it, as on a line without right-to-left text. A glyph that only touches the blank's start was
|
||||
/// drawn beside it, as text set letter by letter leftwards is, and x order already reads it right. A blank that a
|
||||
/// glyph placed before them also reaches over sits inside a cluster (a letter and the marks drawn on it), where its
|
||||
/// place is no clearer, so it stays where x order put it.
|
||||
fn keep_taken_back_blanks(glyphs: &[&TextGlyph], placed: &mut [usize]) {
|
||||
let glyph = |placed: &[usize], p: usize| placed.get(p).and_then(|&i| glyphs.get(i)).copied();
|
||||
let blank = |g: &TextGlyph| !g.text.is_empty() && g.text.chars().all(char::is_whitespace);
|
||||
let left_to_right = |g: &TextGlyph| g.text.chars().any(|c| c.is_alphanumeric() && !is_rtl(c));
|
||||
let mark = |g: &TextGlyph| !g.text.is_empty() && g.text.chars().all(|c| unicode_bidi::bidi_class(c) == unicode_bidi::BidiClass::NSM);
|
||||
// Some(true) for right-to-left text, Some(false) for left-to-right text (Latin letters, European digits), None for
|
||||
// a glyph without a strong direction.
|
||||
let strong = |g: &TextGlyph| {
|
||||
if g.text.chars().any(|c| c.is_alphanumeric() && !is_rtl(c)) {
|
||||
Some(false)
|
||||
} else if g.text.chars().any(|c| c.is_alphanumeric()) {
|
||||
Some(true)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
if !placed.iter().any(|&i| glyphs.get(i).is_some_and(|g| g.text.chars().any(is_rtl))) {
|
||||
return;
|
||||
}
|
||||
// Whether the nearest glyph with a strong direction from position `from` on, stepping by `step`, is right-to-left,
|
||||
// or there is none before the line's edge.
|
||||
let rtl_side = |placed: &[usize], from: Option<usize>, step: fn(usize) -> Option<usize>| {
|
||||
let mut p = from;
|
||||
for _ in 0..TAKEN_BACK_SCAN {
|
||||
let Some(g) = p.and_then(|p| glyph(placed, p)) else { return true };
|
||||
if let Some(rtl) = strong(g) {
|
||||
return rtl;
|
||||
}
|
||||
p = p.and_then(step);
|
||||
}
|
||||
false
|
||||
};
|
||||
for p in 1..placed.len() {
|
||||
let (Some(&ink), Some(&space)) = (placed.get(p - 1), placed.get(p)) else { continue };
|
||||
let (Some(s), Some(g)) = (glyphs.get(space), glyphs.get(ink)) else { continue };
|
||||
let Some(&space) = placed.get(p) else { continue };
|
||||
let Some(s) = glyphs.get(space).copied() else { continue };
|
||||
if !blank(s) {
|
||||
continue;
|
||||
}
|
||||
// The glyph drawn after the blank, past any marks drawn between them.
|
||||
let mut next = space.checked_add(1);
|
||||
for _ in 0..TAKEN_BACK_SCAN {
|
||||
match next.and_then(|i| glyphs.get(i)) {
|
||||
Some(g) if mark(g) => next = next.and_then(|i| i.checked_add(1)),
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
let Some(ink) = next.filter(|&i| glyphs.get(i).is_some_and(|g| !mark(g))) else { continue };
|
||||
// The marks drawn between them, wherever x order put them next to the blank, go with them.
|
||||
let between = |i: usize| i > space && i < ink;
|
||||
let mut q = p.checked_sub(1);
|
||||
for _ in 0..TAKEN_BACK_SCAN {
|
||||
match q.and_then(|q| placed.get(q)) {
|
||||
Some(&i) if between(i) => q = q.and_then(|q| q.checked_sub(1)),
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
let Some(q) = q.filter(|&q| placed.get(q) == Some(&ink)) else { continue };
|
||||
let mut r = p;
|
||||
for _ in 0..TAKEN_BACK_SCAN {
|
||||
match r.checked_add(1).and_then(|r| placed.get(r)) {
|
||||
Some(&i) if between(i) => r += 1,
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
let Some(g) = glyphs.get(ink).copied() else { continue };
|
||||
let width = s.rect[2] - s.rect[0];
|
||||
let covered = g.rect[2].min(s.rect[2]) - s.rect[0];
|
||||
let taken_back = g.rect[0] <= s.rect[0] && width > 0.0 && covered >= 0.5 * width;
|
||||
let alone = p.checked_sub(2).and_then(|q| placed.get(q)).and_then(|&q| glyphs.get(q)).is_none_or(|q| q.rect[2] <= s.rect[0]);
|
||||
if space.checked_add(1) == Some(ink) && blank(s) && !blank(g) && !left_to_right(g) && taken_back && alone {
|
||||
placed.swap(p - 1, p);
|
||||
let alone = q.checked_sub(1).and_then(|b| glyph(placed, b)).is_none_or(|before| before.rect[2] <= s.rect[0]);
|
||||
let direction = match strong(g) {
|
||||
Some(rtl) => rtl,
|
||||
None => rtl_side(placed, q.checked_sub(1), |p| p.checked_sub(1)) && rtl_side(placed, r.checked_add(1), |p| p.checked_add(1)),
|
||||
};
|
||||
if !blank(g) && direction && taken_back && alone {
|
||||
// The blank, its marks and its letter in drawing order.
|
||||
if let Some(group) = placed.get_mut(q..=r) {
|
||||
group.sort_unstable();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1248,6 +1248,170 @@ fn a_space_taken_back_under_the_next_letter_stays_before_it() {
|
||||
assert_eq!(layout(v).plain_text(), "a bc");
|
||||
}
|
||||
|
||||
/// A neutral taken back over a space keeps the space before it only inside right-to-left text. On a right-to-left line
|
||||
/// drawn in logical order, a parenthesis that opens Latin text and is kerned back into the space before it stays
|
||||
/// where its position puts it, as it would on a line without the Hebrew word.
|
||||
#[test]
|
||||
fn a_taken_back_neutral_beside_latin_text_keeps_its_place() {
|
||||
// Drawn right to left: "שלום", the space, "(" pulled back into the space, then "AB" to its left.
|
||||
let mut v = Vec::new();
|
||||
let mut x = 60.0;
|
||||
for c in ["ש", "ל", "ו", "ם"] {
|
||||
v.push(em_box(c, x - 6.5, 0.0, x, 11.0));
|
||||
x -= 6.5;
|
||||
}
|
||||
v.push(em_box(" ", x - 3.5, 0.0, x, 11.0));
|
||||
x -= 3.5;
|
||||
v.push(em_box("(", x - 2.0, 0.0, x + 2.0, 11.0));
|
||||
x -= 2.0;
|
||||
v.push(em_box("A", x - 13.0, 0.0, x - 6.5, 11.0));
|
||||
v.push(em_box("B", x - 6.5, 0.0, x, 11.0));
|
||||
assert_eq!(layout(v).plain_text(), "שלום (AB");
|
||||
}
|
||||
|
||||
/// A mark drawn between a space and the letter taken back over it does not keep them apart: the space still ends the
|
||||
/// word before, and the mark follows its letter, whether its box starts right or left of the space's start.
|
||||
#[test]
|
||||
fn a_mark_between_a_space_and_its_taken_back_letter_keeps_the_space_before_both() {
|
||||
let line = |mark_x0: f32| {
|
||||
let mut v = vec![em_box(" ", 10.0, 0.0, 13.5, 11.0), em_box("\u{5B8}", mark_x0, 0.0, 17.5, 11.0), em_box("ה", 9.9, 0.0, 17.5, 11.0)];
|
||||
let mut x = 17.5;
|
||||
for c in ["נ", "י", "ד", "מ", "ה"] {
|
||||
v.push(em_box(c, x, 0.0, x + 6.5, 11.0));
|
||||
x += 6.5;
|
||||
}
|
||||
x += 3.5;
|
||||
for c in ["ר", "ק", "ב", "מ"] {
|
||||
v.push(em_box(c, x, 0.0, x + 6.5, 11.0));
|
||||
x += 6.5;
|
||||
}
|
||||
layout(v).plain_text()
|
||||
};
|
||||
assert_eq!(line(12.0).trim_end(), "מבקר המדינה\u{5B8}");
|
||||
assert_eq!(line(9.95).trim_end(), "מבקר המדינה\u{5B8}");
|
||||
}
|
||||
|
||||
/// Two taken-back spaces on one line each stay before their letter, and a second space drawn before a taken-back one
|
||||
/// stays with it.
|
||||
#[test]
|
||||
fn two_taken_back_spaces_on_a_line_each_stay_before_their_letter() {
|
||||
// Drawn left to right: " הנידמה" and " רקבמ", each space's advance taken back by its next letter, then " לע".
|
||||
let word = |v: &mut Vec<TextGlyph>, x: &mut f32, first: &str, rest: &[&str]| {
|
||||
v.push(em_box(" ", *x, 0.0, *x + 3.5, 11.0));
|
||||
v.push(em_box(first, *x - 0.1, 0.0, *x + 6.4, 11.0));
|
||||
*x += 6.4;
|
||||
for c in rest {
|
||||
v.push(em_box(c, *x, 0.0, *x + 6.5, 11.0));
|
||||
*x += 6.5;
|
||||
}
|
||||
};
|
||||
let mut v = Vec::new();
|
||||
let mut x = 10.0;
|
||||
word(&mut v, &mut x, "ה", &["נ", "י", "ד", "מ", "ה"]);
|
||||
word(&mut v, &mut x, "ר", &["ק", "ב", "מ"]);
|
||||
// The last word's space keeps its advance.
|
||||
v.push(em_box(" ", x, 0.0, x + 3.5, 11.0));
|
||||
x += 3.5;
|
||||
for c in ["ל", "ע"] {
|
||||
v.push(em_box(c, x, 0.0, x + 6.5, 11.0));
|
||||
x += 6.5;
|
||||
}
|
||||
assert_eq!(layout(v).plain_text().trim(), "על מבקר המדינה");
|
||||
// Two spaces in a row, the second taken back: both stay before the letter.
|
||||
let mut v = vec![em_box(" ", 6.5, 0.0, 10.0, 11.0), em_box(" ", 10.0, 0.0, 13.5, 11.0), em_box("ה", 9.9, 0.0, 17.5, 11.0)];
|
||||
let mut x = 17.5;
|
||||
for c in ["נ", "י", "ד", "מ", "ה"] {
|
||||
v.push(em_box(c, x, 0.0, x + 6.5, 11.0));
|
||||
x += 6.5;
|
||||
}
|
||||
assert_eq!(layout(v).plain_text().trim(), "המדינה");
|
||||
}
|
||||
|
||||
/// pdf.js issue 11656 (https://github.com/mozilla/pdf.js/issues/11656): its content stream rebuilt with the same
|
||||
/// codes, widths, adjustments and `/ToUnicode` on an unembedded font. Each right-to-left show string starts with a
|
||||
/// space taken back under the next letter or punctuation; copied text keeps whole words and punctuation after its
|
||||
/// word, and the parentheses of "15(ב)(4)" read as before.
|
||||
#[test]
|
||||
fn issue_11656_layout_reads_whole_words() {
|
||||
use lopdf::{Document, Object, Stream, dictionary};
|
||||
let shows: [(f32, f32, &str); 10] = [
|
||||
(
|
||||
386.83,
|
||||
708.96,
|
||||
"[<0003>324<0022>-3<02AD>11<02AC02A5>4<02AB0003>10<02AD02A502AC>4<02B90003>8<02A402AE000302AD>9<02AC02A5>4<02B2>-3<000302AD02A502AC02B9>] TJ",
|
||||
),
|
||||
(380.71, 708.96, "<0003> Tj"),
|
||||
(
|
||||
329.69,
|
||||
686.62,
|
||||
"[<0003>324<001102A902BA>3<02A902AE02A0000302A302AC>4<02A9000302B8>4<02A102AB>11<000302A902B0>10<02A00003>-2<000F02AD02A902A8>4<02A502A7>13<02A4000302AF02AE000302A302B8>6<02A502A9>5<000302A902B002A0>] TJ",
|
||||
),
|
||||
(323.57, 686.62, "<0003> Tj"),
|
||||
(446.35, 664.3, "[<0011>10<02AD02A902A3>6<02AC02A9>4<02A4000302AC02AB000302A502AE02AB>] TJ"),
|
||||
(
|
||||
378.19,
|
||||
641.98,
|
||||
"[<000302B3>-2<02A902B202B1>11<000302A902B402AC0003>10<02A402AC02A0000302BA02A5>3<02AC02A5>4<02B2>7<02B4000302A1>-3<02A502A9>5<02A702AC>] TJ",
|
||||
),
|
||||
(366.19, 641.98, "-0.0278 Tc [<00140018>] TJ 0 Tc"),
|
||||
(346.51, 641.98, "[<000C000B>5<02A1000C>] TJ"),
|
||||
(340.49, 641.98, "[<0017>] TJ"),
|
||||
(236.33, 641.98, "[<0003>324<02A402B002A902A3>4<02AE02A4000302B802B7>16<02A102AE0003>-2<02B7>3<02A502A7>13<02AC0003000B>] TJ"),
|
||||
];
|
||||
let ops: String = shows.iter().map(|(x, y, show)| format!("BT /F1 11.04 Tf 1 0 0 1 {x} {y} Tm {show} ET\n")).collect();
|
||||
let cmap = "/CIDInit /ProcSet findresource begin 12 dict begin begincmap /CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def /CMapName /Adobe-Identity-UCS def /CMapType 2 def 1 begincodespacerange <0000> <FFFF> endcodespacerange 1 beginbfchar <0003> <0020> endbfchar 1 beginbfrange <000B> <000C> [<0029> <0028>] endbfrange 3 beginbfchar <000F> <002C> <0011> <002E> <0014> <0031> endbfchar 1 beginbfrange <0017> <0018> <0034> endbfrange 1 beginbfchar <0022> <003F> endbfchar 5 beginbfrange <02A0> <02A1> <05D0> <02A3> <02A5> <05D3> <02A7> <02A9> <05D7> <02AB> <02B4> <05DB> <02B7> <02BA> <05E7> endbfrange endcmap CMapName currentdict /CMap defineresource pop end end";
|
||||
let ranges: [(i64, &[i64]); 13] = [
|
||||
(0, &[1000]),
|
||||
(3, &[313]),
|
||||
(11, &[383, 383]),
|
||||
(15, &[303]),
|
||||
(17, &[303]),
|
||||
(20, &[546]),
|
||||
(23, &[546, 546]),
|
||||
(34, &[474]),
|
||||
(672, &[673, 640]),
|
||||
(675, &[601, 685, 274]),
|
||||
(679, &[685, 699, 274]),
|
||||
(683, &[621, 621, 685, 685, 362, 401, 730, 648, 662, 652]),
|
||||
(695, &[677, 569, 815, 709]),
|
||||
];
|
||||
let w: Vec<Object> = ranges
|
||||
.iter()
|
||||
.flat_map(|(first, widths)| [Object::Integer(*first), Object::Array(widths.iter().map(|&n| Object::Integer(n)).collect())])
|
||||
.collect();
|
||||
let mut doc = Document::with_version("1.7");
|
||||
let unicode = doc.add_object(Stream::new(dictionary! {}, cmap.as_bytes().to_vec()));
|
||||
let cid = doc.add_object(dictionary! {
|
||||
"Type" => "Font", "Subtype" => "CIDFontType2", "BaseFont" => "TestHebrew",
|
||||
"CIDSystemInfo" => dictionary! { "Registry" => Object::string_literal("Adobe"), "Ordering" => Object::string_literal("Identity"), "Supplement" => 0 },
|
||||
"DW" => 1000, "W" => w, "CIDToGIDMap" => "Identity"
|
||||
});
|
||||
let font = doc.add_object(dictionary! { "Type" => "Font", "Subtype" => "Type0", "BaseFont" => "TestHebrew", "Encoding" => "Identity-H", "DescendantFonts" => vec![Object::Reference(cid)], "ToUnicode" => unicode });
|
||||
let content = doc.add_object(Stream::new(dictionary! {}, ops.into_bytes()));
|
||||
let pages = doc.new_object_id();
|
||||
let page = doc.add_object(dictionary! { "Type" => "Page", "Parent" => pages, "MediaBox" => vec![Object::Integer(0), Object::Integer(0), Object::Integer(612), Object::Integer(792)], "Resources" => dictionary! {"Font" => dictionary! {"F1" => font}}, "Contents" => content });
|
||||
doc.objects.insert(pages, dictionary! {"Type" => "Pages", "Kids" => vec![Object::Reference(page)], "Count" => 1}.into());
|
||||
let catalog = doc.add_object(dictionary! {"Type" => "Catalog", "Pages" => pages});
|
||||
doc.trailer.set("Root", catalog);
|
||||
let mut bytes = Vec::new();
|
||||
doc.save_to(&mut bytes).unwrap();
|
||||
let text = extract_page(&Pdf::new(std::sync::Arc::new(bytes)).unwrap(), 0, &InterpreterSettings::default()).unwrap();
|
||||
let plain = text.plain_text();
|
||||
let lines: Vec<&str> = plain.lines().map(str::trim).collect();
|
||||
assert_eq!(
|
||||
lines,
|
||||
[
|
||||
"שלום עולם מה שלום כולם?",
|
||||
"אני יורד מן החוטים, אני כבר ילד אמיתי.",
|
||||
"כמו כל הילדים.",
|
||||
"לחיוב פעולות אלה לפי סעיף 15(ב)(4) לחוק מבקר המדינה"
|
||||
]
|
||||
);
|
||||
for word in ["המדינה", "כולם?", "אמיתי.", "15(ב)(4)", "(ב)"] {
|
||||
assert_eq!(text.find(word).len(), 1, "{word:?} in {lines:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// #2667: a number or punctuation on a right-to-left line must not split it into separately reversed words.
|
||||
#[test]
|
||||
fn rtl_line_with_number_and_punctuation_reads_in_logical_order() {
|
||||
|
||||
Reference in new issue
Block a user