Merge remote-tracking branch 'upstream/main' into feat/pdf-portfolio

# Conflicts:
#	crates/automation/src/tools.rs
This commit is contained in:
Eric Kaden committed 2026-11-07 10:03:23 -05:00
commit 60cb52d635
673 files changed
+111153 -99814

No files matched your search

+2 -2
View File
@@ -13,6 +13,8 @@ pdfcraft-cos = { workspace = true }
pdfcraft-fonts = { workspace = true }
hayro = { workspace = true }
hayro-syntax = { workspace = true }
# Picks the face of an installed font collection (the version pdfcraft-fonts already uses).
skrifa = "0.44"
lopdf = { workspace = true }
kurbo = { workspace = true }
thiserror = { workspace = true }
@@ -27,7 +29,5 @@ bytemuck = { version = "1" }
getrandom = { version = "0.4", features = ["wasm_js"] }
[dev-dependencies]
# Checks the Japanese substitute's character coverage; the version pdfcraft-fonts already uses.
skrifa = "0.44"
# Builds the decompression-bomb streams in the raster tests.
flate2 = { workspace = true }
+15 -3
View File
@@ -311,11 +311,23 @@ impl<'a> Inspector<'a> {
}
pub(super) fn text(&self, d: &Dictionary, key: &[u8]) -> Option<String> {
let s = trimmed_text(self.resolve(d.get(key).ok()?))?;
(!s.is_empty()).then_some(s)
// Typewriter outer whitespace belongs to the text (keep in step with annot's summaries).
let s = if key == b"Contents"
&& self.name(d, b"Subtype").as_deref() == Some("FreeText")
&& self.name(d, b"IT").as_deref() == Some("FreeTextTypeWriter")
{
match self.resolve(d.get(key).ok()?) {
Object::String(bytes, _) => text_string(bytes).trim_matches('\0').to_owned(),
Object::Name(bytes) => String::from_utf8_lossy(bytes).trim_matches('\0').to_owned(),
_ => return None,
}
} else {
trimmed_text(self.resolve(d.get(key).ok()?))?
};
(!s.trim().is_empty()).then_some(s)
}
/// [`Self::text`], converted and spent from `budget` as [`spend`] does, so text that trims
/// Whitespace-trimmed metadata, spent from `budget` as [`spend`] does, so text that trims
/// short or to nothing still spends its raw length.
pub(super) fn budgeted_text(&self, d: &Dictionary, key: &[u8], budget: &mut usize) -> Option<String> {
let o = self.resolve(d.get(key).ok()?);
+42
View File
@@ -1870,3 +1870,45 @@ trailer << /Root 1 0 R >>
assert!((p.width - 3874.96).abs() < 0.1, "width expected ~3874.96, got {}", p.width);
assert!((p.height - 5493.54).abs() < 0.1, "height expected ~5493.54, got {}", p.height);
}
/// Inspection keeps every object it reads, converted, for as long as it runs, and nothing it
/// reports comes from `/PieceInfo`. InDesign writes kilobytes of that private data into every
/// page, and a 1,412-page book held over a gigabyte of it (#5873). It is left out of what
/// inspection keeps, and what inspection reports is the same as for the page without it.
#[test]
fn inspection_keeps_no_private_page_data() {
use lopdf::dictionary;
let make = |private: bool| -> Vec<u8> {
let mut doc = Document::with_version("1.7");
let pages_id = doc.new_object_id();
let mut page = dictionary! {
"Type" => "Page",
"Parent" => pages_id,
"MediaBox" => vec![0.into(), 0.into(), 200.into(), 300.into()],
"Rotate" => 90,
};
if private {
let map: Dictionary = (0..50).map(|i| (i.to_string(), Object::Array(vec![Object::Integer(i), Object::Real(116.22)]))).collect();
page.set("PieceInfo", dictionary! { "InDesign" => dictionary! { "PageItemUIDToLocationDataMap" => map } });
}
let page_id = doc.add_object(page);
doc.objects.insert(pages_id, dictionary! { "Type" => "Pages", "Kids" => vec![page_id.into()], "Count" => 1 }.into());
let catalog = doc.add_object(dictionary! { "Type" => "Catalog", "Pages" => pages_id });
doc.trailer.set("Root", catalog);
let mut out = Vec::new();
doc.save_to(&mut out).unwrap();
out
};
let (with, without) = (make(true), make(false));
let lazy = LazyStructure::new(Arc::new(with.clone()), None).unwrap();
let page = lazy.get_object((2, 0)).unwrap().as_dict().unwrap();
assert!(!page.has(b"PieceInfo"), "the private data is not kept");
assert!(page.has(b"MediaBox") && page.has(b"Rotate") && page.has(b"Parent"), "the rest of the page is");
let report = |bytes: Vec<u8>| {
let mut info = inspect(Arc::new(bytes), None).unwrap();
info.file_size = 0;
format!("{info:?}")
};
assert_eq!(report(with), report(without), "what inspection reports does not change");
}
+1
View File
@@ -13,6 +13,7 @@ mod pixels;
mod raster;
mod reading;
mod structure;
mod system_font;
pub mod text;
pub mod transition;
+56 -3
View File
@@ -78,16 +78,43 @@ pub struct RenderConfig {
/// The page clears to transparent instead of white (or the reading background): for a page
/// holding only what is laid over another raster, such as a comment while it is dragged.
pub transparent: bool,
/// Draw a CID font of an Adobe CJK collection that the PDF does not embed with one face already
/// installed on this machine (see `system_font`). Off by default and on only for on-screen
/// viewer and preview rasters: exports, printing and OCR render without it, so nothing drawn with an
/// installed font is saved or printed (AGENTS.md §1.2, §1.4).
pub system_fonts: bool,
/// Steps of JBIG2 decoding a render may take across the page's images, instead of the usual
/// `hayro_syntax::MAX_PAGE_JBIG2_WORK`. Those past it aren't drawn, like an image that can't
/// be decoded.
pub jbig2_work: Option<u64>,
}
impl RenderConfig {
/// This configuration for a raster that is only shown on screen (the viewer, Combine
/// thumbnails, figure previews): installed CJK faces may draw fonts the PDF doesn't embed.
/// Never use it for anything that can be saved, printed, recognised or returned to an agent
/// (AGENTS.md §1.2, §1.4).
#[must_use]
pub fn on_screen(self) -> Self {
Self { system_fonts: true, ..self }
}
/// Build interpreter settings with layer and comment preferences and a font resolver that
/// tries bundled Japanese faces, then opt-in installed CJK faces, then standard substitutes.
/// Passwords belong to parser construction; request-specific preview overrides are applied
/// by `render_page`, which also preserves the caller's warning and cancellation hooks.
fn settings(&self) -> InterpreterSettings {
let standard = InterpreterSettings::default().font_resolver;
let defaults = InterpreterSettings::default();
let standard = defaults.font_resolver.clone();
let system_fonts = self.system_fonts;
InterpreterSettings {
ocg_overrides: self.layers.clone(),
hide_comments: self.hide_comments,
font_resolver: Arc::new(move |query| japanese_fallback(query).or_else(|| standard(query))),
..InterpreterSettings::default()
font_resolver: Arc::new(move |query| {
japanese_fallback(query).or_else(|| system_fonts.then(|| system_fallback(query)).flatten()).or_else(|| standard(query))
}),
jbig2_work: self.jbig2_work.unwrap_or(defaults.jbig2_work),
..defaults
}
}
}
@@ -119,6 +146,14 @@ fn japanese_fallback(query: &FontQuery) -> Option<(FontData, u32)> {
Some((Arc::new(face.bytes), 0))
}
/// An installed face for a CID font of an Adobe CJK collection that the PDF doesn't embed and
/// craft-fonts has no face for (Traditional and Simplified Chinese, Korean, and Japanese in a
/// build without craft-fonts). `None` for every other font.
fn system_fallback(query: &FontQuery) -> Option<(FontData, u32)> {
let FontQuery::Fallback(f) = query else { return None };
crate::system_font::fallback(f)
}
#[derive(Debug, PartialEq)]
enum JapaneseFace {
Mincho,
@@ -182,6 +217,7 @@ pub struct RenderRequest {
/// The tag bit of a print-preview raster. Such requests always render the document's true
/// colours, ignoring `RenderConfig::reading_colors`: a preview's job is fidelity to the
/// printout, and the print itself — imposed from the document — is never recoloured.
/// Installed-font fallback is also disabled, matching the output's font policy.
pub const PRINT_TAG: u64 = 1 << 60;
/// The reading colours a request renders with: print previews ([`PRINT_TAG`]) ignore the
@@ -319,6 +355,23 @@ fn render_page<'a>(
config: &RenderConfig,
req: RenderRequest,
) -> Result<Output, (String, bool)> {
// Print previews share the viewer pool, but must use the output font policy. Keep the
// caller's cancellation/warning hooks, layers and annotation settings. A fresh cache is
// essential: hayro keys cached fonts by PDF dictionary, not by the font resolver, so the
// viewer cache could supply an installed face even after changing the resolver (and a
// cached missing font from print preview could blank the viewer on switching back).
let print_settings;
let print_cache;
let (settings, cache) = if config.system_fonts && req.tag & PRINT_TAG != 0 {
print_settings = InterpreterSettings {
font_resolver: RenderConfig { system_fonts: false, ..config.clone() }.settings().font_resolver,
..settings.clone()
};
print_cache = RenderCache::new();
(&print_settings, &print_cache)
} else {
(settings, cache)
};
let reading = request_reading(config, &req);
if req.kind == RequestKind::Text {
return match catch_unwind(AssertUnwindSafe(|| crate::text::extract_page(pdf, req.page, settings))) {
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,60 @@
//! A same-machine pixel oracle, independent of GBK decoding and installed-face selection.
//! All font reads and reference rasters stay in memory; nothing is embedded or published.
use super::*;
use hayro::hayro_interpret::font::FontData;
use skrifa::MetadataProvider as _;
use skrifa::string::StringId;
/// Compare the real GBK PDF path with explicit Unicode and a known installed face. Unlike an
/// ink count or extracted text, this fails if drawing chooses another glyph or font family.
pub(super) fn assert_expected_glyphs(pdf: &Arc<Vec<u8>>, base_font: &str) {
let (expected, file) = match base_font {
"SimSun" | "ABCDEF+FZSSK--GBK1-0" => ("SimSun", "simsun.ttc"),
"SimHei" | "ABCDEF+FZHTK--GBK1-0" => ("SimHei", "simhei.ttf"),
"KaiTi" | "ABCDEF+FZKTK--GBK1-0" => ("KaiTi", "simkai.ttf"),
"FangSong" | "ABCDEF+FZFSK--GBK1-0" => ("FangSong", "simfang.ttf"),
other => panic!("no independent GBK oracle for {other}"),
};
// Do not use the production candidate list, resolver or face-selection helpers here.
let directory = std::env::var_os("WINDIR").or_else(|| std::env::var_os("SystemRoot")).unwrap_or_else(|| r"C:\Windows".into());
let path = std::path::PathBuf::from(directory).join("Fonts").join(file);
if !path.is_file() {
eprintln!("{base_font}: exact {expected} face unavailable; pixel oracle skipped");
return;
}
let bytes = Arc::new(std::fs::read(path).expect("read installed reference font"));
let index = (0..32)
.find(|&index| {
skrifa::FontRef::from_index(&bytes, index)
.is_ok_and(|font| font.localized_strings(StringId::POSTSCRIPT_NAME).any(|name| name.chars().collect::<String>() == expected))
})
.expect("reference file contains the expected PostScript face");
let face = skrifa::FontRef::from_index(&bytes, index).unwrap();
for character in ['\u{4e2d}', '\u{6587}'] {
assert!(face.charmap().map(character).is_some_and(|id| id.to_u32() != 0), "{expected} must contain {character}");
}
// Explicit ToUnicode maps arbitrary reference codes to U+4E2D and U+6587, bypassing GBK
// and Adobe-GB1 CID lookup. The fixed resolver below bypasses family classification too.
let cmap = "begincmap /CMapType 2 def 1 begincodespacerange <0000> <FFFF> endcodespacerange \
2 beginbfchar <0001> <4E2D> <0002> <6587> endbfchar endcmap";
let source = std::str::from_utf8(pdf).unwrap();
assert!(source.contains("/Encoding /GBK-EUC-H") && source.contains("<D6D0CEC4>"));
let reference = source
.replace("/Encoding /GBK-EUC-H", "/Encoding /Identity-H /ToUnicode 8 0 R")
.replace("<D6D0CEC4>", "<00010002>")
.replace("trailer <<", &format!("8 0 obj << /Length {} >> stream\n{cmap}\nendstream endobj\ntrailer <<", cmap.len()));
let parsed = Pdf::new(Arc::new(reference.into_bytes())).unwrap();
let data: FontData = bytes;
let settings = InterpreterSettings { font_resolver: Arc::new(move |_| Some((data.clone(), index))), ..Default::default() };
let reference = hayro::render(&parsed.pages()[0], &RenderCache::new(), &settings, &RenderSettings { bg_color: WHITE, ..Default::default() });
let mut renderer = PageRenderer::new(pdf.clone(), RenderConfig::default().on_screen());
let actual = renderer.render(RenderRequest { page: 0, kind: RequestKind::Pixels, scale: 1.0, ..Default::default() });
assert!(actual.error.is_none(), "{base_font}: {:?}", actual.error);
let expected_pixels = reference.data_as_u8_slice();
assert_eq!(actual.rgba.len(), expected_pixels.len());
assert!(expected_pixels.as_chunks::<4>().0.iter().filter(|pixel| pixel[0] < 128).count() > 300, "reference must draw two visible glyphs");
assert!(actual.rgba.as_ref() == expected_pixels, "{base_font}: GBK raster must draw the exact {expected} glyphs for U+4E2D U+6587");
eprintln!("{base_font}: exact {expected} Unicode-reference pixels matched");
}
@@ -0,0 +1,69 @@
//! #5430: Flate images with PNG predictors (ISO 32000-2 §7.4.4.4) are undone a byte row at a
//! time (vendored hayro-syntax patch 35). The bit-level decoder they went through before dropped
//! the last byte of each filtered row that ends part-way through a pixel, so every later row
//! started a byte early and the picture came out sheared.
use super::rgb_at;
use crate::{PageRenderer, RenderConfig, RenderRequest, RequestKind};
use flate2::{Compression, write::ZlibEncoder};
use std::io::Write;
use std::sync::Arc;
/// The page is 150 × 200 pt, drawn at 72 dpi; the image fills it.
const WIDTH: usize = 150;
const HEIGHT: usize = 200;
/// The colour of each of the image's four rows, top first, as 4-bit samples.
const ROWS: [[u8; 3]; 4] = [[15, 0, 0], [0, 15, 0], [0, 0, 15], [15, 15, 0]];
#[test]
fn a_four_bit_rgb_image_of_odd_width_with_up_predicted_rows_is_not_sheared() {
// Three 4-bit RGB pixels are 4.5 bytes, so each row is five bytes and ends half-way through a
// two-byte pixel.
let rows: Vec<Vec<u8>> = ROWS
.iter()
.map(|rgb| {
let nibbles: Vec<u8> = (0..3).flat_map(|_| rgb.iter().copied()).chain([0]).collect();
nibbles.chunks(2).map(|pair| (pair[0] << 4) | pair[1]).collect()
})
.collect();
let mut predicted = Vec::new();
let mut above = vec![0u8; 5];
for row in &rows {
predicted.push(2);
predicted.extend(row.iter().zip(&above).map(|(b, a)| b.wrapping_sub(*a)));
above.clone_from(row);
}
let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
encoder.write_all(&predicted).unwrap();
let data = encoder.finish().unwrap();
let content = "q 150 0 0 200 0 0 cm /Im0 Do Q";
let mut pdf = format!(
"%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n\
2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\
3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 150 200] /Resources << /XObject << /Im0 5 0 R >> >> /Contents 4 0 R >> endobj\n\
4 0 obj << /Length {} >> stream\n{content}\nendstream endobj\n\
5 0 obj << /Type /XObject /Subtype /Image /Width 3 /Height 4 /ColorSpace /DeviceRGB /BitsPerComponent 4 \
/Filter /FlateDecode /DecodeParms << /Predictor 15 /Colors 3 /BitsPerComponent 4 /Columns 3 >> /Length {} >> stream\n",
content.len(),
data.len()
)
.into_bytes();
pdf.extend_from_slice(&data);
pdf.extend_from_slice(b"\nendstream endobj\ntrailer << /Root 1 0 R >>\n%%EOF");
let mut renderer = PageRenderer::new(Arc::new(pdf), RenderConfig::default());
let page = renderer.render(RenderRequest { page: 0, kind: RequestKind::Pixels, tile: None, scale: 1.0, tag: 0, preview: false });
assert!(page.error.is_none(), "{:?}", page.error);
assert_eq!((page.width, page.height), (WIDTH as u32, HEIGHT as u32));
// Each image pixel is 50 pt square; look at the middle of each.
for (row, rgb) in ROWS.iter().enumerate() {
let expected = rgb.map(|sample| sample * 17);
for column in 0..3 {
let (x, y) = (column * 50 + 25, row * 50 + 25);
assert_eq!(rgb_at(&page.rgba, WIDTH, x, y), expected, "image row {row}, column {column}");
}
}
}
+12 -1
View File
@@ -187,9 +187,20 @@ fn convert_dict(dict: &pdfcraft_cos::Dict, depth: usize) -> Result<Dictionary> {
if depth > 100 {
return Err(Error::RecursionLimit);
}
dict.iter().map(|(key, value)| Ok((key.clone(), convert(value, depth + 1)?))).collect::<Result<Vec<_>>>().map(|items| items.into_iter().collect())
dict.iter()
.filter(|(key, _)| key.as_slice() != PRIVATE_DATA)
.map(|(key, value)| Ok((key.clone(), convert(value, depth + 1)?)))
.collect::<Result<Vec<_>>>()
.map(|items| items.into_iter().collect())
}
/// `/PieceInfo`: an application's private data about a page, form or document (ISO 32000-1
/// §14.5), left out of what inspection keeps. Nothing here reads it, and every object read is
/// kept converted for the inspection's lifetime: InDesign writes kilobytes of it into every page,
/// and a 1,412-page book held over a gigabyte of it (#5873). The file and the editing document are
/// untouched; this is the read-only view inspection walks.
const PRIVATE_DATA: &[u8] = b"PieceInfo";
fn convert(object: &CosObject, depth: usize) -> Result<Object> {
if depth > 100 {
return Err(Error::RecursionLimit);
+519
View File
@@ -0,0 +1,519 @@
//! Installed CJK faces for CID fonts a PDF doesn't embed.
//!
//! Acrobat and other tools often write Chinese, Japanese or Korean text with one of the Adobe
//! collections (Adobe-Japan1, GB1, CNS1, Korea1) and a font name such as `KozGoPr6N-Medium` or
//! `MSung-Light` without embedding the font. hayro maps those CIDs to Unicode itself, but it needs
//! a face to draw them with; without one the text is blank. The craft-fonts Japanese faces come
//! first (see `japanese_fallback` in `raster.rs`); this module is the last resort: one face
//! already installed on this machine, used for the viewer only (`RenderConfig::system_fonts`, off
//! for exports, printing and OCR). It is read at runtime and never embedded or shipped
//! (AGENTS.md §1.4), Noto CJK and Source Han are left out on purpose (§1.1), and
//! `PDFCRAFT_SYSTEM_FONTS=0` turns it off, as for the interface font.
use std::collections::HashMap;
use std::io::Read as _;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, OnceLock, PoisonError};
use hayro::hayro_interpret::font::{FallbackFontQuery, FontData};
use hayro::hayro_interpret::hayro_cmap::CidFamily;
use skrifa::MetadataProvider as _;
use skrifa::string::StringId;
/// Larger files are not read: a font path is still untrusted input. CJK system faces are big
/// collections (PingFang is tens of megabytes).
const MAX_BYTES: u64 = 96 << 20;
/// Faces tried in a collection (`.ttc`).
const MAX_FACES: u32 = 32;
/// Downloaded font assets looked at per kind on macOS.
const MAX_ASSETS: usize = 512;
/// The writing system a character collection is for.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
enum Script {
Japanese,
SimplifiedChinese,
TraditionalChinese,
Korean,
}
impl Script {
/// Map the four supported Adobe CJK collections to their writing systems. Identity and
/// custom collections provide no regional hint and cannot select an installed face here.
fn of(family: &CidFamily) -> Option<Self> {
match family {
CidFamily::AdobeJapan1 => Some(Self::Japanese),
CidFamily::AdobeGB1 => Some(Self::SimplifiedChinese),
CidFamily::AdobeCNS1 => Some(Self::TraditionalChinese),
CidFamily::AdobeKorea1 => Some(Self::Korean),
CidFamily::AdobeIdentity | CidFamily::Custom { .. } => None,
}
}
/// A character the face must have to be worth using. Each is common and not shared by the
/// other scripts' faces: 物 (Japanese; also Chinese, so it only rules out Latin-only faces),
/// 欢 (Simplified only), 說 (Traditional only), 한 (Hangul).
fn probe(self) -> char {
match self {
Self::Japanese => '\u{7269}',
Self::SimplifiedChinese => '\u{6B22}',
Self::TraditionalChinese => '\u{8AAA}',
Self::Korean => '\u{D55C}',
}
}
}
/// One installed file and the words (in a face's family name) that pick the face for the right
/// region out of a collection that holds several (PingFang SC, TC and HK share one file).
struct Candidate {
path: PathBuf,
region: &'static [&'static str],
}
/// Keep named Windows body, heading, calligraphic and FangSong faces distinct. A cache keyed only
/// by script and weight would make the first Chinese font on a page stand in for all of them.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
enum ChineseFamily {
Default,
Song,
Hei,
Kai,
FangSong,
}
impl ChineseFamily {
/// The family a font name asks for: the Windows and Founder (方正) names that Chinese PDFs,
/// CNKI papers in particular, use without embedding the font. FangSong is tested first, as
/// its names contain "song" too. Founder PostScript names are matched by prefix (`FZSSK--GBK1-0`
/// is 方正书宋, `FZXBSK--GBK1-0` 方正小标宋, `FZHTK--GBK1-0` 方正黑体, `FZKTK--GBK1-0` 方正楷体,
/// `FZFSK--GBK1-0` 方正仿宋).
fn of(name: &str) -> Self {
// PDF subset names prepend six uppercase ASCII letters and '+'. Match the actual
// PostScript name so Founder prefixes work for subsets too.
let name = match name.split_once('+') {
Some((tag, family)) if tag.len() == 6 && tag.bytes().all(|b| b.is_ascii_uppercase()) => family,
_ => name,
};
let name = name.to_ascii_lowercase();
let has = |hints: &[&str]| hints.iter().any(|hint| name.contains(hint));
let starts = |prefixes: &[&str]| prefixes.iter().any(|p| name.starts_with(p));
if has(&["fangsong", "simfang", "仿宋"]) || starts(&["fzfs"]) {
Self::FangSong
} else if has(&["simsun", "stsong", "songti", "song", "宋"]) || starts(&["fzss", "fzbs", "fzxbs", "fzcs"]) {
Self::Song
} else if has(&["simhei", "stheiti", "stxihei", "heiti", "黑"]) || starts(&["fzht", "fzhei", "fzlth", "fzdh"]) {
Self::Hei
} else if has(&["kaiti", "stkaiti", "kai", "楷"]) || starts(&["fzkt", "fzkai"]) {
Self::Kai
} else {
Self::Default
}
}
}
/// The installed face to draw a CID font of `query`'s collection with, as `(data, face index)`.
/// `None` when it is turned off, the collection isn't one of the four, or nothing fits.
pub(crate) fn fallback(query: &FallbackFontQuery) -> Option<(FontData, u32)> {
if cfg!(target_arch = "wasm32") || std::env::var_os("PDFCRAFT_SYSTEM_FONTS").is_some_and(|v| v == "0") {
return None;
}
let script = Script::of(&query.character_collection.as_ref()?.family)?;
let bold = query.is_bold || query.font_weight >= 600;
let family = ChineseFamily::of(query.post_script_name.as_deref().unwrap_or_default());
/// A face file and the index of the chosen face in it, or `None` when nothing fits.
type Face = Option<(Arc<Vec<u8>>, u32)>;
type Faces = HashMap<(Script, bool, ChineseFamily), Face>;
static CACHE: Mutex<Option<Faces>> = Mutex::new(None);
let mut cache = CACHE.lock().unwrap_or_else(PoisonError::into_inner);
let found = cache.get_or_insert_with(HashMap::new).entry((script, bold, family)).or_insert_with(|| {
candidates(script, family, bold).iter().find_map(|c| {
read(c, script.probe(), bold).inspect(|(_, index)| {
log::info!("CJK font fallback for {script:?}/{family:?}: {} (face {index})", c.path.display());
})
})
});
found.as_ref().map(|(bytes, index)| (Arc::clone(bytes) as FontData, *index))
}
/// Read a candidate through the bounded file cache and select a face that maps `probe`,
/// preferring the requested region and weight. Unreadable files and unsuitable faces return
/// `None`, allowing the caller to try the next candidate without changing the PDF.
fn read(candidate: &Candidate, probe: char, bold: bool) -> Option<(Arc<Vec<u8>>, u32)> {
let bytes = file_bytes(&candidate.path)?;
let index = pick_face(&bytes, probe, candidate.region, bold)?;
Some((bytes, index))
}
/// The bytes of an installed font file, read once for the whole process and shared: a collection
/// such as `PingFang.ttc` is tens of megabytes and is the candidate of several scripts and both
/// weights, so the per-script cache keeps only a handle and a face index. A file that cannot be
/// used is remembered too, so it is not tried again for every script or family.
fn file_bytes(path: &Path) -> Option<Arc<Vec<u8>>> {
/// The bytes of each file tried, or `None` for one that cannot be used.
type Files = HashMap<PathBuf, Option<Arc<Vec<u8>>>>;
static FILES: Mutex<Option<Files>> = Mutex::new(None);
let mut files = FILES.lock().unwrap_or_else(PoisonError::into_inner);
files.get_or_insert_with(HashMap::new).entry(path.to_path_buf()).or_insert_with(|| read_file(path, MAX_BYTES).map(Arc::new)).clone()
}
/// A regular file of at most `max` bytes, or `None`: a font path is untrusted input.
fn read_file(path: &Path, max: u64) -> Option<Vec<u8>> {
// Reject special files before opening them (in particular a FIFO can block on open).
let meta = std::fs::metadata(path).ok()?;
if !meta.is_file() || meta.len() > max {
return None;
}
let file = std::fs::File::open(path).ok()?;
let meta = file.metadata().ok()?;
if !meta.is_file() || meta.len() > max {
return None;
}
// Bound the actual read too: the file can grow after the metadata check.
let mut bytes = Vec::new();
file.take(max.saturating_add(1)).read_to_end(&mut bytes).ok()?;
(bytes.len() as u64 <= max).then_some(bytes)
}
/// The best face of the file that parses and maps `probe`: the region's face when the file has
/// one, then the regular (or, for bold text, the semibold or bold) weight, then the first.
fn pick_face(bytes: &[u8], probe: char, region: &[&str], bold: bool) -> Option<u32> {
let mut best: Option<(u32, u32)> = None;
for index in 0..MAX_FACES {
let Ok(font) = skrifa::FontRef::from_index(bytes, index) else { continue };
if font.charmap().map(probe).is_none() {
continue;
}
let names = |id: StringId| font.localized_strings(id).map(|s| s.chars().collect::<String>()).collect::<Vec<_>>();
let family = names(StringId::FAMILY_NAME).join(" ");
let style = names(StringId::SUBFAMILY_NAME).join(" ").to_ascii_lowercase();
let mut score = 1;
if region.iter().any(|r| family.split_whitespace().any(|w| w == *r)) {
score += 4;
}
let weighty = style.contains("bold") || style.contains("semibold") || style.contains("heavy");
let plain = style.is_empty() || style.contains("regular") || style.contains("normal") || style.contains("medium") && !bold;
if (bold && weighty) || (!bold && plain && !weighty) {
score += 2;
}
if best.is_none_or(|(_, s)| score > s) {
best = Some((index, score));
}
}
best.map(|(index, _)| index)
}
/// List installed-font paths in preference order, with regional face-name hints for collections.
/// Windows Simplified Chinese first tries the named document family; its generic list also
/// prefers a bold face for bold requests. macOS includes bounded downloaded-font locations.
/// Only fixed platform filenames are used: a PDF font name never becomes a filesystem path.
fn candidates(script: Script, family: ChineseFamily, bold: bool) -> Vec<Candidate> {
let region: &'static [&'static str] = match script {
Script::Japanese => &["JP", "Japanese"],
Script::SimplifiedChinese => &["SC", "GB"],
Script::TraditionalChinese => &["TC", "HK", "CNS"],
Script::Korean => &["KR", "Korean"],
};
let files: Vec<PathBuf> = if cfg!(windows) {
let dir = windows_fonts();
let names: &[&str] = match script {
Script::Japanese => &["YuGothR.ttc", "meiryo.ttc", "msgothic.ttc"],
Script::SimplifiedChinese => match family {
ChineseFamily::Song => &["simsun.ttc", "msyh.ttc", "msjh.ttc"],
ChineseFamily::Hei => &["simhei.ttf", "msyh.ttc", "simsun.ttc", "msjh.ttc"],
ChineseFamily::Kai => &["simkai.ttf", "simsun.ttc", "msyh.ttc", "msjh.ttc"],
ChineseFamily::FangSong => &["simfang.ttf", "simsun.ttc", "msyh.ttc", "msjh.ttc"],
ChineseFamily::Default if bold => &["msyhbd.ttc", "msyh.ttc", "simsun.ttc", "msjh.ttc"],
ChineseFamily::Default => &["msyh.ttc", "simsun.ttc", "msjh.ttc"],
},
Script::TraditionalChinese => &["msjh.ttc", "mingliu.ttc", "msyh.ttc"],
Script::Korean => &["malgun.ttf", "gulim.ttc"],
};
names.iter().map(|f| dir.join(f)).collect()
} else if cfg!(target_os = "macos") {
let names: &[&str] = match script {
Script::Japanese => &["ヒラギノ角ゴシック W3.ttc", "Hiragino Sans GB.ttc", "PingFang.ttc"],
Script::SimplifiedChinese => &["PingFang.ttc", "Hiragino Sans GB.ttc", "STHeiti Light.ttc"],
Script::TraditionalChinese => &["PingFang.ttc", "STHeiti Light.ttc", "Hiragino Sans GB.ttc"],
Script::Korean => &["AppleSDGothicNeo.ttc", "AppleGothic.ttf", "PingFang.ttc"],
};
// The system folder first, then the downloaded font assets: on recent macOS a face such as
// PingFang lives in `/System/Library/AssetsV2/com_apple_MobileAsset_Font*/<id>.asset/AssetData`.
let mut dirs = vec![PathBuf::from("/System/Library/Fonts")];
dirs.extend(macos_font_asset_dirs().iter().cloned());
names.iter().flat_map(|f| dirs.iter().map(move |d| d.join(f))).collect()
} else {
let names: &[&str] = match script {
Script::Korean => {
&["truetype/nanum/NanumGothic.ttf", "nanum/NanumGothic.ttf", "truetype/baekmuk/gulim.ttf", "truetype/unfonts-core/UnDotum.ttf"]
}
Script::Japanese => &[
"truetype/fonts-japanese-gothic.ttf",
"ipa-gothic/ipag.ttf",
"truetype/takao-gothic/TakaoGothic.ttf",
"truetype/vlgothic/VL-Gothic-Regular.ttf",
],
Script::SimplifiedChinese | Script::TraditionalChinese => &[
"truetype/wqy/wqy-microhei.ttc",
"wenquanyi/wqy-microhei/wqy-microhei.ttc",
"wqy-microhei/wqy-microhei.ttc",
"truetype/wqy/wqy-zenhei.ttc",
"wenquanyi/wqy-zenhei/wqy-zenhei.ttc",
"wqy-zenhei/wqy-zenhei.ttc",
"truetype/droid/DroidSansFallbackFull.ttf",
"google-droid-sans-fonts/DroidSansFallbackFull.ttf",
"truetype/arphic/uming.ttc",
"arphic-uming/uming.ttc",
],
};
["/usr/share/fonts", "/usr/local/share/fonts"].iter().flat_map(|dir| names.iter().map(move |f| Path::new(dir).join(f))).collect()
};
files.into_iter().map(|path| Candidate { path, region }).collect()
}
/// The `AssetData` folders of the downloaded font assets of macOS, found once. The ids in their
/// names change with each asset version, so they are listed rather than named; at most
/// [`MAX_ASSETS`] are looked at.
fn macos_font_asset_dirs() -> &'static [PathBuf] {
static DIRS: OnceLock<Vec<PathBuf>> = OnceLock::new();
DIRS.get_or_init(|| {
let root = Path::new("/System/Library/AssetsV2");
let Ok(kinds) = std::fs::read_dir(root) else { return Vec::new() };
let mut dirs = Vec::new();
for kind in kinds.flatten().filter(|k| k.file_name().to_string_lossy().starts_with("com_apple_MobileAsset_Font")) {
let Ok(assets) = std::fs::read_dir(kind.path()) else { continue };
for asset in assets.flatten().take(MAX_ASSETS) {
let data = asset.path().join("AssetData");
if data.is_dir() {
dirs.push(data);
}
}
}
dirs
})
}
/// Locate the Windows Fonts directory using `WINDIR`, then `SystemRoot`, then `C:\Windows`.
/// This only constructs the path; candidate reads still enforce regular-file and size limits.
fn windows_fonts() -> PathBuf {
let dir = std::env::var_os("WINDIR").or_else(|| std::env::var_os("SystemRoot")).map_or_else(|| PathBuf::from(r"C:\Windows"), PathBuf::from);
dir.join("Fonts")
}
#[cfg(test)]
mod tests {
use super::*;
const SCRIPTS: [Script; 4] = [Script::Japanese, Script::SimplifiedChinese, Script::TraditionalChinese, Script::Korean];
/// Standard, localized and subset-prefixed names retain their Chinese family hint;
/// unrelated Latin names and generic YaHei requests use the default candidate list.
#[test]
fn chinese_family_hints_include_subset_and_localized_names() {
for name in ["SimSun", "ABCDEF+SimSun", "STSong-Light", "宋体"] {
assert_eq!(ChineseFamily::of(name), ChineseFamily::Song, "{name}");
}
for name in ["SimHei", "STHeiti", "黑体"] {
assert_eq!(ChineseFamily::of(name), ChineseFamily::Hei, "{name}");
}
for name in ["KaiTi", "STKaiti", "楷体"] {
assert_eq!(ChineseFamily::of(name), ChineseFamily::Kai, "{name}");
}
assert_eq!(ChineseFamily::of("Helvetica"), ChineseFamily::Default);
assert_eq!(ChineseFamily::of("MicrosoftYaHei"), ChineseFamily::Default);
}
/// FangSong names contain "song" and must not become Song; Founder (方正) names in CNKI papers
/// pick the matching installed family instead of YaHei for everything (#3587).
#[test]
fn fangsong_and_founder_names_pick_their_family() {
for name in ["FangSong", "FangSong_GB2312", "STFangsong", "SimFang", "仿宋", "FZFSK--GBK1-0", "ABCDEF+FZFSK--GBK1-0"] {
assert_eq!(ChineseFamily::of(name), ChineseFamily::FangSong, "{name}");
}
for name in ["FZSSK--GBK1-0", "FZBSK--GBK1-0", "FZXBSK--GBK1-0", "FZShuSong-Z01", "STZhongsong", "NSimSun", "ABCDEF+FZSSK--GBK1-0"] {
assert_eq!(ChineseFamily::of(name), ChineseFamily::Song, "{name}");
}
for name in ["FZHTK--GBK1-0", "FZLTHJW--GB1-0", "STXihei", "ABCDEF+FZHTK--GBK1-0"] {
assert_eq!(ChineseFamily::of(name), ChineseFamily::Hei, "{name}");
}
for name in ["FZKTK--GBK1-0", "KaiTi_GB2312", "ABCDEF+FZKTK--GBK1-0"] {
assert_eq!(ChineseFamily::of(name), ChineseFamily::Kai, "{name}");
}
}
/// Every family has a Windows list that ends in faces any Chinese Windows has, so a missing
/// named face still draws the text.
#[cfg(windows)]
#[test]
fn every_windows_chinese_family_falls_back_to_common_faces() {
for family in [ChineseFamily::Default, ChineseFamily::Song, ChineseFamily::Hei, ChineseFamily::Kai, ChineseFamily::FangSong] {
let list = candidates(Script::SimplifiedChinese, family, false);
assert!(list.iter().any(|c| c.path.ends_with("msyh.ttc")), "{family:?}");
}
let fangsong = candidates(Script::SimplifiedChinese, ChineseFamily::FangSong, false);
assert!(fangsong.first().is_some_and(|c| c.path.ends_with("simfang.ttf")));
}
/// Invalid font bytes, missing paths and directories produce no candidate face rather
/// than preventing the resolver from trying a later installed font.
#[test]
fn broken_and_missing_files_are_skipped() {
assert_eq!(pick_face(b"", '\u{7269}', &[], false), None);
assert_eq!(pick_face(b"not a font at all", '\u{7269}', &[], false), None);
assert_eq!(pick_face(&[0u8; 4096], '\u{7269}', &[], false), None);
let missing = Candidate { path: PathBuf::from("definitely/not/here.ttc"), region: &[] };
assert!(read(&missing, '\u{7269}', false).is_none());
// A directory is not a font.
let dir = Candidate { path: std::env::temp_dir(), region: &[] };
assert!(read(&dir, '\u{7269}', false).is_none());
}
/// The files are bounded by `max`: a larger one is not read.
#[test]
fn a_file_over_the_cap_is_not_read() {
let inter = Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/../../assets/fonts/Inter-Regular.ttf"));
assert!(read_file(inter, MAX_BYTES).is_some());
assert!(read_file(inter, 1024).is_none());
}
/// Two faces drawn from one installed file (two scripts, or regular and bold) hold the same
/// bytes, not a copy each; a file that can't be used is remembered as unusable.
#[test]
fn one_installed_file_is_read_once_and_shared() {
let inter = Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/../../assets/fonts/Inter-Regular.ttf"));
let first = file_bytes(inter).expect("readable");
let second = file_bytes(inter).expect("readable");
assert!(Arc::ptr_eq(&first, &second), "one allocation for the file");
let missing = Path::new("definitely/not/here.ttc");
assert!(file_bytes(missing).is_none() && file_bytes(missing).is_none());
}
/// A Latin-only face fails every CJK probe even though its Latin glyph remains readable,
/// distinguishing unsupported script coverage from an unparseable font file.
#[test]
fn a_latin_face_is_rejected_for_every_script() {
let inter = include_bytes!("../../../assets/fonts/Inter-Regular.ttf");
for script in SCRIPTS {
assert_eq!(pick_face(inter, script.probe(), &[], false), None, "{script:?}");
}
assert_eq!(pick_face(inter, 'A', &[], false), Some(0));
}
/// Platform candidates are absolute TrueType paths and exclude the font families barred
/// by the asset policy, independently of whether those files are installed locally.
#[test]
fn candidates_are_absolute_font_files_without_adobe_designs() {
for script in SCRIPTS {
let list = candidates(script, ChineseFamily::Default, false);
assert!(!list.is_empty(), "{script:?}");
for c in list {
assert!(c.path.is_absolute(), "{}", c.path.display());
assert!(c.path.extension().is_some_and(|e| e == "ttf" || e == "ttc"), "{}", c.path.display());
let name = c.path.to_string_lossy().to_lowercase();
assert!(
!["notosanscjk", "notoserifcjk", "sourcehan", "source-han", "adobe", "kozuka", "minion"].iter().any(|b| name.contains(b)),
"{name}"
);
}
}
}
/// The four regional Adobe collections select their corresponding scripts; Identity
/// carries no regional information and must not acquire an installed fallback by guess.
#[test]
fn only_the_four_adobe_collections_have_a_script() {
assert_eq!(Script::of(&CidFamily::AdobeCNS1), Some(Script::TraditionalChinese));
assert_eq!(Script::of(&CidFamily::AdobeGB1), Some(Script::SimplifiedChinese));
assert_eq!(Script::of(&CidFamily::AdobeKorea1), Some(Script::Korean));
assert_eq!(Script::of(&CidFamily::AdobeJapan1), Some(Script::Japanese));
assert_eq!(Script::of(&CidFamily::AdobeIdentity), None);
}
/// A real Windows GBK book names these faces. Replacing both with YaHei hides the
/// distinction between serif body text and sans headings even when the exact face exists.
#[cfg(windows)]
#[test]
fn windows_preserves_installed_chinese_document_families() {
use hayro::hayro_interpret::hayro_cmap::CharacterCollection;
for (name, expected, file) in [
("SimSun", "SimSun", "simsun.ttc"),
("SimHei", "SimHei", "simhei.ttf"),
("KaiTi", "KaiTi", "simkai.ttf"),
("FangSong", "FangSong", "simfang.ttf"),
("ABCDEF+FZSSK--GBK1-0", "SimSun", "simsun.ttc"),
("ABCDEF+FZHTK--GBK1-0", "SimHei", "simhei.ttf"),
("ABCDEF+FZKTK--GBK1-0", "KaiTi", "simkai.ttf"),
("ABCDEF+FZFSK--GBK1-0", "FangSong", "simfang.ttf"),
] {
if !windows_fonts().join(file).is_file() {
continue;
}
let query = FallbackFontQuery {
post_script_name: Some(name.into()),
character_collection: Some(CharacterCollection { family: CidFamily::AdobeGB1, supplement: 4 }),
..FallbackFontQuery::default()
};
let (bytes, index) = fallback(&query).expect("installed Chinese font");
let font = skrifa::FontRef::from_index(bytes.as_ref().as_ref(), index).unwrap();
let names: Vec<String> = font.localized_strings(StringId::POSTSCRIPT_NAME).map(|n| n.chars().collect()).collect();
assert!(names.iter().any(|n| n.eq_ignore_ascii_case(expected)), "{name} resolved to {names:?}, expected {expected}");
}
}
/// On a Mac the face for Traditional Chinese is the TC one of PingFang, not the first of the
/// collection (which is another region's), and it maps a character only Traditional has.
#[cfg(target_os = "macos")]
#[test]
fn macos_picks_the_regional_pingfang_face() {
// Found where this macOS keeps it: the system folder, or the downloaded font assets.
let path = candidates(Script::TraditionalChinese, ChineseFamily::Default, false)
.into_iter()
.map(|c| c.path)
.find(|p| p.file_name().is_some_and(|n| n == "PingFang.ttc") && p.is_file());
let Some(bytes) = path.and_then(|p| std::fs::read(p).ok()) else {
eprintln!("PingFang is not installed here: skipped");
return;
};
for (script, region) in [(Script::TraditionalChinese, "TC"), (Script::SimplifiedChinese, "SC")] {
let index = pick_face(&bytes, script.probe(), &[region], false).expect("PingFang covers it");
let font = skrifa::FontRef::from_index(&bytes, index).unwrap();
let family: String = font.localized_strings(StringId::FAMILY_NAME).next().unwrap().chars().collect();
assert!(family.contains(region), "{script:?} got {family:?}");
}
}
/// A face that this macOS keeps only in its downloaded font assets (PingFang, on recent
/// versions) is among the candidates. The assets are searched here on their own, not by the
/// code under test; where there are none, nothing is checked.
#[cfg(target_os = "macos")]
#[test]
fn macos_candidates_include_a_pingfang_that_lives_in_the_font_assets() {
/// Independently locate an installed asset by filename within the test's depth limit,
/// providing an oracle for the production candidate list without using its discovery.
fn find(dir: &Path, name: &str, depth: u32) -> Option<PathBuf> {
for entry in std::fs::read_dir(dir).ok()?.flatten() {
let path = entry.path();
if path.file_name().is_some_and(|n| n == name) && path.is_file() {
return Some(path);
}
if depth > 0
&& path.is_dir()
&& !path.is_symlink()
&& let Some(found) = find(&path, name, depth - 1)
{
return Some(found);
}
}
None
}
let Some(asset) = find(Path::new("/System/Library/AssetsV2"), "PingFang.ttc", 3) else {
eprintln!("no PingFang in the font assets here: skipped");
return;
};
for script in [Script::SimplifiedChinese, Script::TraditionalChinese] {
assert!(
candidates(script, ChineseFamily::Default, false).iter().any(|c| c.path == asset),
"{script:?}: {} is not a candidate",
asset.display()
);
}
}
}
+1 -1
View File
@@ -385,7 +385,7 @@ pub use layout::layout;
use layout::{from_upright, to_upright};
use layout::{is_rtl, union};
#[cfg(test)]
use order::{Run, Scratch, collide, restarts_run};
use order::{Run, Scratch, Seg, collide, restarts_run};
#[cfg(test)]
mod tests;
+37 -8
View File
@@ -1,5 +1,5 @@
use super::device::{DIRECTION_TOLERANCE, Drawn, WORD_GAP};
use super::order::{NONE, REDRAW_CANDIDATES, Run, Scratch, Seg, fingerprint, place_glyphs, redraws, restarts_run, reverse_rtl};
use super::order::{NONE, REDRAW_CANDIDATES, Run, Scratch, Seg, fingerprint, is_script_on, place_glyphs, redraws, restarts_run, reverse_rtl};
use super::{HashMap, PageText, TextGlyph};
/// Glyphs in reading order, as indices into the page's glyphs, with their lines and word gaps.
@@ -177,14 +177,19 @@ fn arrange(glyphs: &[TextGlyph]) -> (Vec<usize>, Vec<u32>, Vec<bool>) {
let mut furthest_progress = 0.0;
for i in 0..n {
let g = glyphs[i].rect;
let hi = height(i);
let cont = segs.last().is_some_and(|s| {
let Some(p) = s.first.end.checked_sub(1) else { return false };
let ph = height(p).min(height(i));
let ph = height(p).min(hi);
let sh = s.h.min(hi);
let gap = g[0] - glyphs[p].rect[2];
// A sub- or superscript is a smaller glyph on a shifted baseline: its em box centre sits off the line by
// more than half its own height, but by less than half the height of the full-size glyph beside it.
let centre_limit = if gap.abs() < ph { height(p).max(height(i)) * 0.5 } else { ph * 0.5 };
(cy(i) - cy(p)).abs() < centre_limit && gap < ph * 3.0 && g[0] > glyphs[p].rect[0] - ph * 2.0
let centre_limit = if gap.abs() < ph { height(p).max(hi) * 0.5 } else { ph * 0.5 };
let same_line = ((cy(i) - cy(p)).abs() < centre_limit && gap < ph * 3.0)
|| ((cy(i) - s.cy()).abs() < sh * 0.3 && (s.h / hi - 1.0).abs() < 0.35 && gap < sh * 3.0);
let on_seg = gap < s.h * 1.2 && (is_script_on(&s.bbox, s.h, &g, hi) || is_script_on(&g, hi, &s.bbox, s.h));
(same_line || on_seg) && g[0] > glyphs[p].rect[0] - sh * 2.0
});
let reset = cont && segs.last().is_some_and(|s| restarts_run(&glyphs, s.first.start, i, furthest_progress));
if cont
@@ -192,11 +197,10 @@ fn arrange(glyphs: &[TextGlyph]) -> (Vec<usize>, Vec<u32>, Vec<bool>) {
&& let Some(s) = segs.last_mut()
{
s.first.end = i + 1;
union(&mut s.bbox, &g);
s.h = s.h.max(height(i));
s.extend_box(&g, hi);
furthest_progress = furthest_progress.max((g[0] - run_start_x).abs()).max((g[2] - run_start_x).abs());
} else {
segs.push(Seg::new(Run { start: i, end: i + 1 }, g, height(i)));
segs.push(Seg::new(Run { start: i, end: i + 1 }, g, hi));
run_start_x = g[0];
furthest_progress = 0.0;
}
@@ -245,7 +249,8 @@ fn arrange(glyphs: &[TextGlyph]) -> (Vec<usize>, Vec<u32>, Vec<bool>) {
kept_runs
};
segs.truncate(retained);
// Merge same-baseline segments that nearly touch.
// Merge same-baseline segments that nearly touch, then attach any remaining superscript or
// subscript segments to the closest host line segment they sit on.
let mut merged = true;
while merged {
merged = false;
@@ -266,6 +271,30 @@ fn arrange(glyphs: &[TextGlyph]) -> (Vec<usize>, Vec<u32>, Vec<bool>) {
}
}
}
if !merged {
let mut b = 0;
while b < segs.len() {
let sb = &segs[b];
let best_host = (0..segs.len())
.filter(|&a| {
if a == b {
return false;
}
let sa = &segs[a];
let gap = (sb.bbox[0] - sa.bbox[2]).max(sa.bbox[0] - sb.bbox[2]);
sa.bbox[2] - sa.bbox[0] > sb.bbox[2] - sb.bbox[0] && gap < sa.h * 1.2 && is_script_on(&sa.bbox, sa.h, &sb.bbox, sb.h)
})
.min_by(|&a1, &a2| (segs[a1].cy() - sb.cy()).abs().total_cmp(&(segs[a2].cy() - sb.cy()).abs()));
if let Some(a) = best_host {
let sb = segs.remove(b);
let a = if b < a { a - 1 } else { a };
segs[a].merge(sb);
merged = true;
} else {
b += 1;
}
}
}
}
// 2. Blocks.
+48 -4
View File
@@ -1,5 +1,5 @@
use super::TextGlyph;
use super::layout::{is_rtl, union};
use super::layout::is_rtl;
/// Glyphs `start..end` of the glyph list [`arrange`] lays out, drawn one after another.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -37,6 +37,32 @@ struct SegExtra {
more: Vec<Run>,
}
/// Whether `(script_bbox, script_h)` is a superscript or subscript on `(host_bbox, host_h)`.
pub(super) fn is_script_on(host_bbox: &[f32; 4], host_h: f32, script_bbox: &[f32; 4], script_h: f32) -> bool {
let host_cy = (host_bbox[1] + host_bbox[3]) / 2.0;
let script_cy = (script_bbox[1] + script_bbox[3]) / 2.0;
let script_w = script_bbox[2] - script_bbox[0];
let host_w = host_bbox[2] - host_bbox[0];
let smaller = script_h >= host_h * 0.45 && script_w < host_h * 3.0 && (script_h < host_h * 0.9 || (script_h <= host_h && host_w > script_w));
if !smaller {
return false;
}
if script_cy < host_cy {
// Superscript: raised above the host baseline, with its bottom inside the host line.
host_cy - script_cy >= host_h * 0.12
&& host_cy - script_cy < host_h * 0.7
&& host_bbox[3] - script_bbox[3] >= host_h * 0.15
&& script_bbox[3] > host_bbox[1] + host_h * 0.25
} else {
// Subscript: lowered below the host baseline, with its centre still inside the host line.
script_cy - host_cy >= host_h * 0.12
&& script_cy - host_cy < host_h * 0.5
&& script_bbox[3] > host_bbox[3]
&& host_bbox[3] > script_bbox[1] + script_h * 0.5
&& script_bbox[1] >= host_bbox[1] - script_h * 0.2
}
}
impl Seg {
pub(super) fn new(first: Run, bbox: [f32; 4], h: f32) -> Self {
Seg { first, extra: None, bbox, h }
@@ -50,9 +76,25 @@ impl Seg {
std::iter::once(self.first).chain(self.extra.iter().flat_map(|e| e.more.iter().copied()))
}
/// Extend `self`'s box and height with `(bbox, h)`. A smaller script (superscript or
/// subscript) widens the line horizontally without pulling its vertical baseline band up or
/// down; when a line begins with a script and continues with taller body text, the body text's
/// vertical band becomes the line's.
pub(super) fn extend_box(&mut self, bbox: &[f32; 4], h: f32) {
if h > self.h * 1.15 || is_script_on(bbox, h, &self.bbox, self.h) {
self.bbox[1] = bbox[1];
self.bbox[3] = bbox[3];
} else if !(self.h > h * 1.15 || is_script_on(&self.bbox, self.h, bbox, h)) {
self.bbox[1] = self.bbox[1].min(bbox[1]);
self.bbox[3] = self.bbox[3].max(bbox[3]);
}
self.bbox[0] = self.bbox[0].min(bbox[0]);
self.bbox[2] = self.bbox[2].max(bbox[2]);
self.h = self.h.max(h);
}
pub(super) fn merge(&mut self, other: Seg) {
union(&mut self.bbox, &other.bbox);
self.h = self.h.max(other.h);
self.extend_box(&other.bbox, other.h);
let extra = self.extra.get_or_insert_with(Default::default);
extra.more.push(other.first);
if let Some(o) = other.extra {
@@ -274,8 +316,10 @@ pub(super) fn collide(glyphs: &[&TextGlyph], height: &dyn Fn(usize) -> f32, h: f
}
budget -= 1;
let (ra, rb) = (union_find(&mut sc.parent, ri), union_find(&mut sc.parent, rj));
let hj = height(j);
let script = is_script_on(&glyphs[i].rect, hi, &glyphs[j].rect, hj) || is_script_on(&glyphs[j].rect, hj, &glyphs[i].rect, hi);
let overlap = x1.min(glyphs[j].rect[2]) - x0(j);
if ra != rb && overlap > RUN_COLLISION * hi.min(height(j)) {
if ra != rb && !script && overlap > RUN_COLLISION * hi.min(hj) {
sc.parent[ra] = rb;
collided = true;
}
+131
View File
@@ -1760,6 +1760,137 @@ fn truetype_without_an_encoding_reads_its_codes_as_win_ansi() {
assert_eq!(ocr_layer_text(32, true), expected, "nonsymbolic, /ToUnicode for ASCII only");
}
/// Issue #2929: a footnote reference (superscript) on a line stays in that line instead of
/// splitting out as a raised segment that steals the block's tail and pushes the host line
/// after the paragraph.
#[test]
fn paragraph_selection_includes_lines_with_footnote_references() {
for inline in [true, false] {
let ops = if inline {
"BT /F1 12 Tf 1 0 0 1 40 300 Tm (First line of the paragraph) Tj \
1 0 0 1 40 285 Tm (Second line with a note) Tj \
/F1 8 Tf 5.5 Ts (1) Tj 0 Ts \
/F1 12 Tf ( and more text) Tj \
1 0 0 1 40 270 Tm (Third line of the paragraph) Tj \
/F1 7 Tf 1 0 0 1 40 104 Tm (1) Tj \
/F1 10 Tf 1 0 0 1 47 100 Tm (Footnote at the bottom) Tj ET"
} else {
// Superscripts drawn in a separate pass after the body text.
"BT /F1 12 Tf 1 0 0 1 40 300 Tm (First line of the paragraph) Tj \
1 0 0 1 40 285 Tm (Second line with a note) Tj \
1 0 0 1 173 285 Tm (and more text) Tj \
1 0 0 1 40 270 Tm (Third line of the paragraph) Tj \
/F1 10 Tf 1 0 0 1 47 100 Tm (Footnote at the bottom) Tj \
/F1 8 Tf 1 0 0 1 165 290.5 Tm (1) Tj \
/F1 7 Tf 1 0 0 1 40 104 Tm (1) Tj ET"
};
let text = page_glyphs(ops);
let start = text.nearest(40.0, 95.0).unwrap();
let end = text.nearest(185.0, 128.0).unwrap();
assert_eq!(
text.text_of(start..end + 1),
"First line of the paragraph\nSecond line with a note1 and more text\nThird line of the paragraph",
"inline={inline}"
);
assert_eq!(text.line_rects(start..end + 1).len(), 3, "three selected lines, inline={inline}");
assert_eq!(
text.plain_text(),
"First line of the paragraph\nSecond line with a note1 and more text\nThird line of the paragraph\n1 Footnote at the bottom",
"inline={inline}"
);
}
// Superscript on the first line, multi-character bracketed reference `[12]` on the second
// line, and subscript on the third line (both inline and out-of-order).
for inline in [true, false] {
let ops = if inline {
"BT /F1 12 Tf 1 0 0 1 40 300 Tm (Opening claim) Tj \
/F1 8 Tf 5.5 Ts (1) Tj 0 Ts \
/F1 12 Tf ( in the first line) Tj \
1 0 0 1 40 285 Tm (Middle citation) Tj \
/F1 8 Tf 5.5 Ts ([12]) Tj 0 Ts \
/F1 12 Tf ( continues here) Tj \
1 0 0 1 40 270 Tm (Formula H) Tj \
/F1 8 Tf -3 Ts (2) Tj 0 Ts \
/F1 12 Tf (O in the third line) Tj ET"
} else {
"BT /F1 12 Tf 1 0 0 1 40 300 Tm (Opening claim) Tj \
1 0 0 1 125 300 Tm (in the first line) Tj \
1 0 0 1 40 285 Tm (Middle citation) Tj \
1 0 0 1 135 285 Tm (continues here) Tj \
1 0 0 1 40 270 Tm (Formula H) Tj \
1 0 0 1 101 270 Tm (O in the third line) Tj \
/F1 8 Tf 1 0 0 1 117 305.5 Tm (1) Tj \
1 0 0 1 117 290.5 Tm ([12]) Tj \
1 0 0 1 96.5 267 Tm (2) Tj ET"
};
let text = page_glyphs(ops);
assert_eq!(
text.plain_text(),
"Opening claim1 in the first line\nMiddle citation[12] continues here\nFormula H2O in the third line",
"inline={inline}"
);
assert_eq!(text.line_rects(0..text.glyphs.len()).len(), 3, "inline={inline}");
}
// Justified wide gap (18pt = 1.5 * h) after an inline superscript stays on the same line,
// and an out-of-order superscript kerned slightly over punctuation does not break the host
// word via overprint collision.
let justified = page_glyphs(
"BT /F1 12 Tf 1 0 0 1 40 300 Tm (Justified clause.) Tj \
/F1 8 Tf 5.5 Ts (1) Tj 0 Ts \
/F1 12 Tf 1 0 0 1 151 300 Tm (Next sentence on the same line) Tj \
1 0 0 1 40 285 Tm (Kerned clause.) Tj \
1 0 0 1 130 285 Tm (continues cleanly) Tj \
/F1 8 Tf 1 0 0 1 118 290.5 Tm (2) Tj ET",
);
assert_eq!(justified.plain_text(), "Justified clause.1 Next sentence on the same line\nKerned clause.2 continues cleanly");
assert_eq!(justified.line_rects(0..justified.glyphs.len()).len(), 2);
// A line that begins with a raised superscript marker and continues with taller body text
// adopts the body text's vertical band (Seg::extend_box first branch) so a later
// out-of-order superscript on that line still measures against the body baseline.
let leading_script = page_glyphs(
"BT /F1 8 Tf 1 0 0 1 40 300 Tm 6 Ts (1) Tj 0 Ts \
/F1 12 Tf ( See the appendix) Tj \
1 0 0 1 146 300 Tm (for details) Tj \
/F1 8 Tf 1 0 0 1 139 303.6 Tm (2) Tj ET",
);
assert_eq!(leading_script.plain_text(), "1 See the appendix2 for details");
let marker = leading_script.glyphs.first().unwrap().rect;
let body = leading_script.glyphs.iter().find(|g| g.text == "S").unwrap().rect;
let last = leading_script.glyphs.last().unwrap().rect;
let center = |r: [f32; 4]| ((r[0] + r[2]) / 2.0, (r[1] + r[3]) / 2.0);
let (start_x, start_y) = center(marker);
let (end_x, end_y) = center(last);
let start = leading_script.nearest(start_x, start_y).unwrap();
let end = leading_script.nearest(end_x, end_y).unwrap();
assert_eq!(leading_script.text_of(start..end + 1), "1 See the appendix2 for details");
let rects = leading_script.line_rects(start..end + 1);
assert_eq!(rects.len(), 1);
assert!(marker[1] < body[1] && marker[3] < body[3], "marker must be raised above the body");
let close = |actual: f32, expected: f32| assert!((actual - expected).abs() < 0.001, "{actual} != {expected}");
// Highlight rectangles cover the actual glyphs, including the raised marker. They do
// not expose the segment's baseline band used for line grouping.
for (actual, expected) in rects[0].into_iter().zip([marker[0], marker[1], last[2], body[3]]) {
close(actual, expected);
}
close(rects[0][3] - rects[0][1], 14.8);
close(body[3] - 0.2 * 12.0, 100.0); // 300 pt PDF baseline on a 400 pt page.
// Pin that internal band too: replacing the first extend_box condition with `false`
// must fail here even when plain text and highlight rectangles remain unchanged.
let mut segment = Seg::new(Run { start: 0, end: 1 }, marker, marker[3] - marker[1]);
segment.extend_box(&body, body[3] - body[1]);
for (actual, expected) in segment.bbox.into_iter().zip([marker[0], body[1], body[2], body[3]]) {
close(actual, expected);
}
close(segment.h, 12.0);
close(segment.bbox[3] - segment.bbox[1], 12.0);
close(segment.bbox[3] - 0.2 * segment.h, 100.0);
close(segment.cy(), (body[1] + body[3]) / 2.0);
}
#[test]
fn text_outside_the_crop_box_is_clipped() {
use lopdf::{Document, Object, Stream, dictionary};
+1
View File
@@ -8,6 +8,7 @@ mod luminosity_group_space;
mod named_form_properties;
mod nonseparable_gray;
mod open_type_cff_simple_font;
mod rotated_image_snap;
mod search;
mod separation_all;
mod stray_q_in_a_form;
@@ -0,0 +1,163 @@
//! Contributor-original synthetic pages (#5530): a shrunk image on a page turned by `/Rotate` is
//! drawn exactly as on the upright page, turned. The crop box and scale put every page edge on a
//! whole device pixel, so a turn is a pure permutation of pixels and the turned render, turned
//! back, must match the upright one. The image itself sits at fractional coordinates: an upright
//! image there is snapped to whole pixels and shrunk once, and before the fix a turned one was
//! not, so it was shrunk to a rounded-up size and resampled a second time at a fractional scale
//! and offset (most of its pixels off, by up to 217 levels).
//!
//! Every kind of image that reaches the snap is drawn: an 8-bit and a 1-bit grey image, a stencil
//! mask (a CCITT scan is often one), and images with a soft mask of their own size and of another
//! size. The last needs no `/Interpolate`: a shrunk soft mask of another size is filtered, and so
//! snapped, whatever the image asks (hayro patch 15). The other kinds without `/Interpolate` are
//! not snapped upright or turned, and were already the same both ways; they stay in the test as a
//! guard.
use pdfcraft_render::{PageRenderer, RenderConfig, RenderRequest, RenderedPage};
use std::sync::Arc;
/// A PDF of the given objects (object 1 the catalog), with a cross-reference table.
fn pdf(objects: &[Vec<u8>]) -> Vec<u8> {
let mut bytes = b"%PDF-1.7\n".to_vec();
let mut offsets = Vec::new();
for (i, object) in objects.iter().enumerate() {
offsets.push(bytes.len());
bytes.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes());
bytes.extend_from_slice(object);
bytes.extend_from_slice(b"\nendobj\n");
}
let xref = bytes.len();
bytes.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).as_bytes());
for offset in offsets {
bytes.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
}
bytes.extend_from_slice(format!("trailer << /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objects.len() + 1).as_bytes());
bytes
}
fn stream(dict: &str, data: &[u8]) -> Vec<u8> {
let mut bytes = format!("<< {dict} /Length {} >>\nstream\n", data.len()).into_bytes();
bytes.extend_from_slice(data);
bytes.extend_from_slice(b"\nendstream");
bytes
}
/// Image size in samples.
const W: usize = 400;
const H: usize = 500;
/// A busy grey pattern with edges both ways, as a scan has.
fn grey_samples() -> Vec<u8> {
(0..H).flat_map(|y| (0..W).map(move |x| ((x * 7 + y * 13) % 256) as u8 ^ if (x / 3 + y / 5) % 2 == 0 { 0 } else { 0x80 })).collect()
}
/// One bit a sample, rows padded to whole bytes: a text-like pattern of thin strokes.
fn bit_samples() -> Vec<u8> {
let row = W.div_ceil(8);
let mut out = vec![0u8; row * H];
for y in 0..H {
for x in 0..W {
if (x / 2 + y / 3) % 3 == 0 || (x * 5 + y * 3) % 11 < 3 {
out[y * row + x / 8] |= 0x80 >> (x % 8);
}
}
}
out
}
fn alpha_samples(w: usize, h: usize) -> Vec<u8> {
(0..h).flat_map(|y| (0..w).map(move |x| ((x * 3 + y * 5) % 256) as u8)).collect()
}
/// The kinds of image drawn.
const KINDS: [&str; 5] = ["8-bit grey", "1-bit grey", "stencil mask", "soft mask", "smaller soft mask"];
/// The image of `kind` as objects 5 (and 6, its soft mask).
fn image(kind: &str, interpolate: bool) -> Vec<Vec<u8>> {
let image = |depth: u8, extra: &str, data: &[u8]| {
let dict = format!(
"/Type /XObject /Subtype /Image /Width {W} /Height {H} /ColorSpace /DeviceGray /BitsPerComponent {depth} /Interpolate {interpolate}{extra}"
);
stream(&dict, data)
};
let mask = |w: usize, h: usize| {
let dict =
format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /ColorSpace /DeviceGray /BitsPerComponent 8 /Interpolate {interpolate}");
stream(&dict, &alpha_samples(w, h))
};
match kind {
"8-bit grey" => vec![image(8, "", &grey_samples())],
"1-bit grey" => vec![image(1, "", &bit_samples())],
"stencil mask" => vec![stream(
&format!("/Type /XObject /Subtype /Image /Width {W} /Height {H} /ImageMask true /BitsPerComponent 1 /Interpolate {interpolate}"),
&bit_samples(),
)],
"soft mask" => vec![image(8, " /SMask 6 0 R", &grey_samples()), mask(W, H)],
_ => vec![image(8, " /SMask 6 0 R", &grey_samples()), mask(300, 350)],
}
}
/// A page whose crop box sits inside a larger media box, as in the report, with the image drawn
/// at fractional coordinates inside it, in a colour (a stencil mask paints it).
fn page(rotate: u32, kind: &str, interpolate: bool) -> Vec<u8> {
let mut objects = vec![
b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(),
b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(),
format!(
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 794 1224] /CropBox [114 510 680 1210] /Rotate {rotate} /Contents 4 0 R /Resources << /XObject << /Im 5 0 R >> >> >>"
)
.into_bytes(),
stream("", b"0.1 0.2 0.6 rg q 500.3 0 0 600.7 150.15 560.45 cm /Im Do Q"),
];
objects.extend(image(kind, interpolate));
pdf(&objects)
}
/// At half a pixel per point every crop-box edge, upright or turned, is a whole device pixel,
/// and the 400 × 500 image is shrunk to about 250 × 300 pixels.
fn render(rotate: u32, kind: &str, interpolate: bool) -> RenderedPage {
let mut renderer = PageRenderer::new(Arc::new(page(rotate, kind, interpolate)), RenderConfig::default());
let out = renderer.render(RenderRequest { page: 0, scale: 0.5, ..Default::default() });
assert!(out.error.is_none(), "{kind}: {:?}", out.error);
out
}
fn rgb(out: &RenderedPage, x: u32, y: u32) -> [u8; 3] {
let i = ((y * out.width + x) * 4) as usize;
[out.rgba[i], out.rgba[i + 1], out.rgba[i + 2]]
}
/// The largest channel difference between the turned render and the upright one turned the same
/// way.
fn turned_difference(rotate: u32, kind: &str, interpolate: bool) -> u8 {
let upright = render(0, kind, interpolate);
let turned = render(rotate, kind, interpolate);
let (w, h) = (upright.width, upright.height);
// `/Rotate` turns the page clockwise.
let expected = |x: u32, y: u32| match rotate {
90 => rgb(&upright, y, h - 1 - x),
180 => rgb(&upright, w - 1 - x, h - 1 - y),
_ => rgb(&upright, w - 1 - y, x),
};
let (tw, th) = if rotate == 180 { (w, h) } else { (h, w) };
assert_eq!((turned.width, turned.height), (tw, th), "{kind}, /Rotate {rotate}");
let mut worst = 0;
for y in 0..th {
for x in 0..tw {
let (got, want) = (rgb(&turned, x, y), expected(x, y));
worst = (0..3).map(|c| got[c].abs_diff(want[c])).fold(worst, u8::max);
}
}
worst
}
#[test]
fn a_shrunk_image_on_a_turned_page_is_the_upright_render_turned() {
for kind in KINDS {
for interpolate in [true, false] {
for rotate in [90, 180, 270] {
let worst = turned_difference(rotate, kind, interpolate);
assert!(worst <= 1, "{kind}, /Rotate {rotate}, /Interpolate {interpolate}: off by up to {worst} levels");
}
}
}
}