From c3fe680a0f799cf571193960ada552b9b230e46b Mon Sep 17 00:00:00 2001 From: Bryan Mark Date: Wed, 18 Nov 2026 18:28:07 +0900 Subject: [PATCH] Fonts: fallback no longer keeps unneeded font families in memory Font fallback records the file stamp with the first coverage read, so a rescan does not keep loading families that are no longer needed, and a replaced file still invalidates the cache. Originally authored by @arcanii in #2345. --- crates/fonts/src/fontdb.rs | 4 +- crates/fonts/src/fontdb/fallback.rs | 102 +++++++------ crates/fonts/src/fontdb/fallback_data.rs | 177 ++++++++++++++++------ crates/fonts/src/fontdb/names.rs | 113 +++++++++----- crates/fonts/src/fontdb/tests/sysfonts.rs | 165 ++++++++++++++++++-- 5 files changed, 415 insertions(+), 146 deletions(-) diff --git a/crates/fonts/src/fontdb.rs b/crates/fonts/src/fontdb.rs index 6ae53ac460..d338b0b803 100644 --- a/crates/fonts/src/fontdb.rs +++ b/crates/fonts/src/fontdb.rs @@ -26,11 +26,11 @@ mod names; use face::{BUNDLED, FontBytes}; pub use face::{FALLBACK_FAMILY, FaceRef, FontFace, bundled}; +#[cfg(test)] +use fallback_data::{Coverage, RUN_FALLBACK_CACHE_CHARS, SYS_FALLBACK_MAX_FILE, file_coverage, ranges_hold}; #[cfg(not(target_arch = "wasm32"))] use fallback_data::{CoverageIndex, SysFallback}; use fallback_data::{FallbackCache, RunFallbackCache}; -#[cfg(test)] -use fallback_data::{RUN_FALLBACK_CACHE_CHARS, coverage_ranges, ranges_hold}; #[cfg(not(target_arch = "wasm32"))] use names::CatalogEntry; pub(crate) use names::face_names; diff --git a/crates/fonts/src/fontdb/fallback.rs b/crates/fonts/src/fontdb/fallback.rs index 8c18abb3cb..ffe5fa7b4e 100644 --- a/crates/fonts/src/fontdb/fallback.rs +++ b/crates/fonts/src/fontdb/fallback.rs @@ -1,6 +1,6 @@ use super::face::{FALLBACK_FAMILY, FontFace}; #[cfg(not(target_arch = "wasm32"))] -use super::fallback_data::{CoverageIndex, IndexedFile, SYS_MISSES_MAX, SYSTEM_FALLBACKS, coverage_ranges, ranges_hold, read_fallback_file}; +use super::fallback_data::{CoverageIndex, IndexedFile, SYS_MISSES_MAX, SYSTEM_FALLBACKS, file_stamp, ranges_hold, read_fallback_file}; use super::fallback_data::{FALLBACK_CACHE_MAX, RTL_FALLBACKS, RUN_FALLBACK_CACHE_CHARS, RUN_FALLBACK_CACHE_MAX, arabic_block, fallback_order}; use super::{Arc, FontDb}; #[cfg(not(target_arch = "wasm32"))] @@ -235,7 +235,7 @@ impl FontDb { } /// Load a system font covering `c` (preferred fallback families first, then any cataloged - /// file under 40 MB, found through the coverage index). Returns true if one was loaded. + /// file under 40 MB, both found through the coverage index). Returns true if one was loaded. /// Misses are remembered. #[cfg(not(target_arch = "wasm32"))] fn system_fallback(&self, c: char) -> bool { @@ -257,6 +257,11 @@ impl FontDb { if self.is_loaded(fam) || !cataloged.iter().any(|f| f.eq_ignore_ascii_case(fam)) { continue; } + // A family none of whose files maps `c` is not loaded for it: it would stay loaded + // and help nothing (an emoji loaded every CJK family a Mac has first, 500 MB). + if !self.family_maps(fam, c) { + continue; + } if !self.sys.lock().unwrap_or_else(std::sync::PoisonError::into_inner).tried.insert(fam) { continue; } @@ -276,8 +281,47 @@ impl FontDb { false } - /// Load the first cataloged font file (in path order) that maps `c`, reading each file not - /// yet indexed as the search reaches it. Returns whether a loaded face now covers `c`. + /// Does a file of the installed `family` map `c`? Answered by the coverage index, which reads + /// a file's character maps the first time it is asked about it. + #[cfg(not(target_arch = "wasm32"))] + pub(super) fn family_maps(&self, family: &str, c: char) -> bool { + let paths = self.family_paths(family); + let mut index = self.coverage_index(); + paths.iter().any(|p| index.file_holds(p, u32::from(c))) + } + + /// The coverage index, its file list brought up to date with the catalog. + #[cfg(not(target_arch = "wasm32"))] + fn coverage_index(&self) -> std::sync::MutexGuard<'_, CoverageIndex> { + let mut index = self.coverage.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let generation = self.catalog_gen.load(Ordering::Acquire); + if index.generation != Some(generation) { + // A new scan (or the first search): keep what is known of files still installed, + // unless a file was replaced since its character maps were read. + let mut known: HashMap = + std::mem::take(&mut index.files).into_iter().map(|f| (f.path.clone(), f)).collect(); + let mut paths: Vec = + self.catalog.read().unwrap_or_else(std::sync::PoisonError::into_inner).iter().map(|e| e.path.clone()).collect(); + paths.sort(); + paths.dedup(); + let files = paths.into_iter().map(|path| match known.remove(&path) { + Some(mut file) => { + if file.ranges.is_some() && file.stamp != file_stamp(&file.path) { + file.ranges = None; + } + file + } + None => IndexedFile { used: index.loaded.contains(&path), path, ranges: None, stamp: None }, + }); + index.files = files.collect(); + index.generation = Some(generation); + } + index + } + + /// Load the first cataloged font file (in path order) that maps `c`, reading the character + /// maps of each file not yet indexed as the search reaches it. Returns whether a loaded face + /// now covers `c`. #[cfg(not(target_arch = "wasm32"))] fn indexed_fallback(&self, c: char) -> bool { self.load_indexed(&[c], false) @@ -294,49 +338,22 @@ impl FontDb { self.load_indexed(chars, true); } - /// Load the cataloged font files (in path order) that map every char of `chars`, reading each - /// file not yet indexed as the search reaches it, until a loaded face covers them all; with - /// `whole_family`, each such file's family is loaded with it (whatever of it isn't loaded - /// yet). A collection's faces only together may map them: the search then goes on. Returns - /// whether a loaded face now covers them all. + /// Load the cataloged font files (in path order) that map every char of `chars`, reading the + /// character maps of each file not yet indexed as the search reaches it, until a loaded face + /// covers them all; with `whole_family`, each such file's family is loaded with it (whatever + /// of it isn't loaded yet). A collection's faces only together may map them: the search then + /// goes on. Returns whether a loaded face now covers them all. #[cfg(not(target_arch = "wasm32"))] fn load_indexed(&self, chars: &[char], whole_family: bool) -> bool { let covered = |db: &FontDb| db.read_faces().iter().any(|f| chars.iter().all(|c| f.covers(*c))); let cps: Vec = chars.iter().map(|c| u32::from(*c)).collect(); let maps_all = |ranges: &[(u32, u32)]| cps.iter().all(|cp| ranges_hold(ranges, *cp)); - let mut index = self.coverage.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - let generation = self.catalog_gen.load(Ordering::Acquire); - if index.generation != Some(generation) { - // A new scan (or the first search): keep what is known of files still installed. - let mut known: HashMap = - std::mem::take(&mut index.files).into_iter().map(|f| (f.path.clone(), f)).collect(); - let mut paths: Vec = - self.catalog.read().unwrap_or_else(std::sync::PoisonError::into_inner).iter().map(|e| e.path.clone()).collect(); - paths.sort(); - paths.dedup(); - let files = paths - .into_iter() - .map(|path| known.remove(&path).unwrap_or_else(|| IndexedFile { used: index.loaded.contains(&path), path, ranges: None })); - index.files = files.collect(); - index.generation = Some(generation); - } + let mut index = self.coverage_index(); let CoverageIndex { files, reads, .. } = &mut *index; for i in 0..files.len() { let Some(file) = files.get_mut(i).filter(|f| !f.used) else { continue }; - let mut data = None; - match &file.ranges { - Some(r) if !maps_all(r) => continue, - Some(_) => {} - None => { - *reads += 1; - data = read_fallback_file(&file.path); - let ranges = data.as_deref().map(coverage_ranges).unwrap_or_default(); - let hit = maps_all(&ranges); - file.ranges = Some(ranges); - if !hit { - continue; - } - } + if !maps_all(file.ranges(reads)) { + continue; } // The file maps them: load it (once; its faces are loaded fonts from now on). file.used = true; @@ -351,17 +368,12 @@ impl FontDb { load.push(f.path.clone()); } let fonts: Vec<(Vec, Option)> = load.iter().filter_map(|p| read_fallback_file(p)).map(|b| (b, None)).collect(); - *reads += fonts.len(); if self.add_fonts(fonts) > 0 && covered(self) { return true; } continue; } - if data.is_none() { - *reads += 1; - data = read_fallback_file(&file.path); - } - if let Some(data) = data + if let Some(data) = read_fallback_file(&file.path) && self.add_font(data) > 0 && covered(self) { diff --git a/crates/fonts/src/fontdb/fallback_data.rs b/crates/fonts/src/fontdb/fallback_data.rs index 91ffb08b64..a831c75e8a 100644 --- a/crates/fonts/src/fontdb/fallback_data.rs +++ b/crates/fonts/src/fontdb/fallback_data.rs @@ -1,7 +1,9 @@ -use super::face::{FALLBACK_FAMILY, FontFace}; -use super::{Arc, HashMap}; #[cfg(not(target_arch = "wasm32"))] -use super::{FileRef, MetadataProvider}; +use super::MetadataProvider; +use super::face::{FALLBACK_FAMILY, FontFace}; +#[cfg(not(target_arch = "wasm32"))] +use super::names::{TableReader, mini_font}; +use super::{Arc, HashMap}; #[cfg(not(target_arch = "wasm32"))] #[derive(Default)] @@ -17,14 +19,16 @@ pub(super) struct SysFallback { #[cfg(not(target_arch = "wasm32"))] pub(super) const SYS_MISSES_MAX: usize = 65_536; -/// Installed font files larger than this are not searched for fallback characters. +/// Installed font files larger than this are not loaded for a fallback character the coverage +/// index finds in them (a preferred fallback family loads whatever its size). #[cfg(not(target_arch = "wasm32"))] -const SYS_FALLBACK_MAX_FILE: u64 = 40 << 20; +pub(super) const SYS_FALLBACK_MAX_FILE: u64 = 40 << 20; /// Which characters each cataloged font file maps (#605). Before it, every distinct character no /// loaded font covered read and parsed every installed font file again: 500 distinct Tangut -/// characters froze the app for most of a minute. Now each file is read once, when a search -/// first gets to it, and later searches answer from its ranges, reading only a file that matches. +/// characters froze the app for most of a minute. Now each file's character maps are read once, +/// when a search first gets to it, and later searches answer from its ranges, loading only a file +/// that matches. The system fallback asks it too before loading a preferred family. #[cfg(not(target_arch = "wasm32"))] #[derive(Default)] pub(super) struct CoverageIndex { @@ -32,23 +36,58 @@ pub(super) struct CoverageIndex { pub(super) generation: Option, /// The cataloged font files, sorted. pub(super) files: Vec, - /// Font files read by fallback searches (each at most once to index it, once more to load it). + /// Font files whose character maps fallback searches read (each once, and again only when + /// a rescan finds it replaced). pub(super) reads: usize, /// Files loaded by family name ([`FontDb::load_cataloged`]), whether or not they are indexed /// yet: a search never loads them a second time. pub(super) loaded: std::collections::HashSet, } +#[cfg(not(target_arch = "wasm32"))] +impl CoverageIndex { + /// Does the cataloged file at `path` map `cp`? Its character maps are read the first time. + pub(super) fn file_holds(&mut self, path: &std::path::Path, cp: u32) -> bool { + let Ok(i) = self.files.binary_search_by(|f| f.path.as_path().cmp(path)) else { return false }; + let Some(file) = self.files.get_mut(i) else { return false }; + ranges_hold(file.ranges(&mut self.reads), cp) + } +} + #[cfg(not(target_arch = "wasm32"))] pub(super) struct IndexedFile { pub(super) path: std::path::PathBuf, /// The code points the file's faces map, as sorted, disjoint inclusive ranges (`None` until - /// the file is read; empty when it can't be read or is too large). + /// its character maps are read; empty when they can't be). pub(super) ranges: Option>, + /// The file's size and modification time when its character maps were read: a rescan + /// drops the ranges of a file replaced since. + pub(super) stamp: Option<(u64, std::time::SystemTime)>, /// Loaded for a character it covers: its faces are searched as loaded fonts from then on. pub(super) used: bool, } +#[cfg(not(target_arch = "wasm32"))] +impl IndexedFile { + /// The code points the file maps. Its character maps are read the first time (counted in + /// `reads`), and its stamp kept with them, so a rescan keeps them unless the file is replaced. + pub(super) fn ranges(&mut self, reads: &mut usize) -> &[(u32, u32)] { + let (path, stamp) = (&self.path, &mut self.stamp); + self.ranges.get_or_insert_with(|| { + *reads += 1; + *stamp = file_stamp(path); + file_coverage(path) + }) + } +} + +/// The size and modification time of the file at `path` (`None` when it can't be read). +#[cfg(not(target_arch = "wasm32"))] +pub(super) fn file_stamp(path: &std::path::Path) -> Option<(u64, std::time::SystemTime)> { + let meta = std::fs::metadata(path).ok()?; + Some((meta.len(), meta.modified().ok()?)) +} + /// Does `ranges` (sorted, disjoint, inclusive) hold `cp`? #[cfg(not(target_arch = "wasm32"))] pub(super) fn ranges_hold(ranges: &[(u32, u32)], cp: u32) -> bool { @@ -56,56 +95,94 @@ pub(super) fn ranges_hold(ranges: &[(u32, u32)], cp: u32) -> bool { ranges.get(i).is_some_and(|r| r.0 <= cp) } -/// The code points mapped by the faces of `data` (a font file or collection), as sorted, -/// disjoint, inclusive ranges. Bounded however damaged the file: at most 256 faces, a budget of -/// mappings read across them, and a cap on the ranges kept. +/// The code points a font file's faces map, gathered face by face into sorted, disjoint, +/// inclusive ranges. Bounded however damaged the file: a budget of mappings read across its +/// faces, and a cap on the ranges kept. #[cfg(not(target_arch = "wasm32"))] -pub(super) fn coverage_ranges(data: &[u8]) -> Box<[(u32, u32)]> { - const MAX_FACES: u32 = 256; - const MAX_MAPPINGS: usize = 4 * 0x11_0000; - const MAX_RANGES: usize = 1 << 17; - fn compact(r: &mut Vec<(u32, u32)>) { - r.sort_unstable(); - let mut out: Vec<(u32, u32)> = Vec::with_capacity(r.len()); - for &(s, e) in r.iter() { +#[derive(Default)] +pub(super) struct Coverage { + ranges: Vec<(u32, u32)>, + mappings: usize, +} + +#[cfg(not(target_arch = "wasm32"))] +impl Coverage { + pub(super) const MAX_MAPPINGS: usize = 4 * 0x11_0000; + pub(super) const MAX_RANGES: usize = 1 << 17; + + /// Add the code points `f` maps. False once a bound is reached: later faces are left out. + pub(super) fn add(&mut self, f: &skrifa::FontRef<'_>) -> bool { + f.charmap().mappings().all(|(cp, _)| self.push(cp)) + } + + /// Add one mapped code point. False once a bound is reached. + pub(super) fn push(&mut self, cp: u32) -> bool { + if self.mappings >= Self::MAX_MAPPINGS { + return false; + } + self.mappings += 1; + match self.ranges.last_mut() { + Some(last) if last.1.checked_add(1) == Some(cp) => last.1 = cp, + Some(last) if last.0 <= cp && cp <= last.1 => {} + _ => { + if self.ranges.len() >= Self::MAX_RANGES { + self.compact(); + // A compaction must free a quarter of the cap, or a crafted file could make + // every later mapping sort all the ranges again. + if self.ranges.len() > Self::MAX_RANGES / 4 * 3 { + return false; + } + } + self.ranges.push((cp, cp)); + } + } + true + } + + fn compact(&mut self) { + self.ranges.sort_unstable(); + let mut out: Vec<(u32, u32)> = Vec::with_capacity(self.ranges.len()); + for &(s, e) in self.ranges.iter() { match out.last_mut() { Some(last) if s <= last.1.saturating_add(1) => last.1 = last.1.max(e), _ => out.push((s, e)), } } - *r = out; + self.ranges = out; } - let count = match FileRef::new(data) { - Ok(FileRef::Font(_)) => 1, - Ok(FileRef::Collection(c)) => c.len().min(MAX_FACES), - Err(_) => 0, - }; - let mut ranges: Vec<(u32, u32)> = Vec::new(); - let mut budget = MAX_MAPPINGS; - 'faces: for i in 0..count { - let Ok(f) = skrifa::FontRef::from_index(data, i) else { continue }; - for (cp, _) in f.charmap().mappings() { - budget = match budget.checked_sub(1) { - Some(b) => b, - None => break 'faces, - }; - match ranges.last_mut() { - Some(last) if last.1.checked_add(1) == Some(cp) => last.1 = cp, - Some(last) if last.0 <= cp && cp <= last.1 => {} - _ => { - if ranges.len() >= MAX_RANGES { - compact(&mut ranges); - if ranges.len() >= MAX_RANGES { - break 'faces; - } - } - ranges.push((cp, cp)); - } - } + + pub(super) fn finish(mut self) -> Box<[(u32, u32)]> { + self.compact(); + self.ranges.into_boxed_slice() + } +} + +/// The code points the faces of the font file at `path` map, as sorted, disjoint, inclusive +/// ranges, read from their character maps (`cmap`) alone: Apple Color Emoji's 180 MB cost a few +/// kilobytes. Empty when the file can't be read. +#[cfg(not(target_arch = "wasm32"))] +pub(super) fn file_coverage(path: &std::path::Path) -> Box<[(u32, u32)]> { + /// The longest character map read (a format 12 map of every code point is 13 MB). + const MAX_CMAP: u32 = 16 << 20; + let mut coverage = Coverage::default(); + let Some(mut file) = TableReader::open(path) else { return coverage.finish() }; + // The faces of a collection often share one map: each is read once. + let mut read: Vec<(u32, u32)> = Vec::new(); + for face in 0..file.faces.len() { + let Some(at) = file.locate(face, b"cmap") else { continue }; + if read.contains(&at) { + continue; + } + read.push(at); + let Some(cmap) = file.read(at, MAX_CMAP) else { continue }; + // A font holding just the map, to read it as the font itself would. + let font = mini_font(&[(*b"cmap", cmap)]); + let Ok(f) = skrifa::FontRef::new(&font) else { continue }; + if !coverage.add(&f) { + break; } } - compact(&mut ranges); - ranges.into_boxed_slice() + coverage.finish() } /// An installed font file's bytes, `None` when it can't be read or is larger than diff --git a/crates/fonts/src/fontdb/names.rs b/crates/fonts/src/fontdb/names.rs index efa66719ee..3d097d87b9 100644 --- a/crates/fonts/src/fontdb/names.rs +++ b/crates/fonts/src/fontdb/names.rs @@ -145,43 +145,15 @@ const COLOR_BITMAP_TABLES: [&[u8; 4]; 2] = [b"sbix", b"CBDT"]; #[cfg(not(target_arch = "wasm32"))] pub(super) fn file_face_names(path: &std::path::Path) -> Vec { use crate::ui_fallback::OUTLINE_TABLES; - use std::io::{Read, Seek, SeekFrom}; - /// Caps on what a (possibly damaged) file can make the scan read. - const MAX_FACES: u32 = 256; + /// The longest `name`, `OS/2` or `fvar` table read. const MAX_TABLE: u32 = 1 << 20; - let Ok(mut file) = std::fs::File::open(path) else { return vec![] }; - let mut read_at = |offset: u64, len: usize| -> Option> { - let mut buf = vec![0; len]; - file.seek(SeekFrom::Start(offset)).ok()?; - file.read_exact(&mut buf).ok()?; - Some(buf) - }; - let be32 = |b: &[u8], at: usize| b.get(at..at + 4).and_then(|s| s.try_into().ok()).map(u32::from_be_bytes); - let Some(head) = read_at(0, 12) else { return vec![] }; - // A collection lists where each face's table directory starts. - let starts: Vec = if head.starts_with(b"ttcf") { - let n = be32(&head, 8).unwrap_or(0).min(MAX_FACES) as usize; - read_at(12, n * 4).map(|b| b.as_chunks::<4>().0.iter().map(|c| u32::from_be_bytes(*c)).collect()).unwrap_or_default() - } else { - vec![0] - }; - starts - .into_iter() - .filter_map(|start| { - let dir = read_at(start.into(), 12)?; - let tables = u16::from_be_bytes(dir.get(4..6)?.try_into().ok()?) as usize; - let records = read_at(u64::from(start) + 12, tables * 16)?; - if !records.as_chunks::<16>().0.iter().any(|r| OUTLINE_TABLES.iter().chain(&COLOR_BITMAP_TABLES).any(|t| r.starts_with(*t))) { + let Some(mut file) = TableReader::open(path) else { return vec![] }; + (0..file.faces.len()) + .filter_map(|face| { + if !OUTLINE_TABLES.iter().chain(&COLOR_BITMAP_TABLES).any(|t| file.locate(face, t).is_some()) { return None; } - let mut table = |tag: &[u8; 4]| -> Option> { - let rec = records.as_chunks::<16>().0.iter().find(|r| r.starts_with(tag))?; - let (offset, len) = (be32(rec, 8)?, be32(rec, 12)?); - if len > MAX_TABLE { - return None; - } - read_at(offset.into(), len as usize) - }; + let mut table = |tag: &[u8; 4]| file.locate(face, tag).and_then(|at| file.read(at, MAX_TABLE)); let name = table(b"name")?; // Variable fonts: the named instances; OS/2: the code pages (East Asian fonts). let mut tables: Vec<([u8; 4], Vec)> = table(b"OS/2").map(|t| (*b"OS/2", t)).into_iter().collect(); @@ -195,6 +167,79 @@ pub(super) fn file_face_names(path: &std::path::Path) -> Vec { .collect() } +/// A font file (or collection) read table by table instead of whole: the font folder scan and +/// the fallback coverage index open hundreds of files, many of them megabytes long (CJK +/// collections run to 20 MB and more, Apple Color Emoji to 180 MB). +#[cfg(not(target_arch = "wasm32"))] +pub(super) struct TableReader { + file: std::fs::File, + /// Each face's table records (tag, checksum, offset, length), for the faces whose table + /// directory could be read. + pub(super) faces: Vec>, +} + +#[cfg(not(target_arch = "wasm32"))] +impl TableReader { + /// Most faces read from a (possibly damaged) collection. + const MAX_FACES: u32 = 256; + + /// Open the font file at `path` and read its table directories; `None` when it can't be read. + pub(super) fn open(path: &std::path::Path) -> Option { + let file = std::fs::File::open(path).ok()?; + let mut reader = TableReader { file, faces: Vec::new() }; + let head = reader.read_at(0, 12)?; + // A collection lists where each face's table directory starts. + let starts: Vec = if head.starts_with(b"ttcf") { + let n = be32(&head, 8).unwrap_or(0).min(Self::MAX_FACES) as usize; + reader.read_at(12, n * 4).map(|b| b.as_chunks::<4>().0.iter().map(|c| u32::from_be_bytes(*c)).collect()).unwrap_or_default() + } else { + vec![0] + }; + for start in starts { + if let Some(records) = reader.directory(start) { + reader.faces.push(records); + } + } + Some(reader) + } + + /// The table records of the face whose table directory starts at `start`. + fn directory(&mut self, start: u32) -> Option> { + let dir = self.read_at(start.into(), 12)?; + let tables = u16::from_be_bytes(dir.get(4..6)?.try_into().ok()?) as usize; + let records = self.read_at(u64::from(start) + 12, tables * 16)?; + Some(records.as_chunks::<16>().0.to_vec()) + } + + /// Where face `face`'s table `tag` is: (offset, length). + pub(super) fn locate(&self, face: usize, tag: &[u8; 4]) -> Option<(u32, u32)> { + let rec = self.faces.get(face)?.iter().find(|r| r.starts_with(tag))?; + Some((be32(rec, 8)?, be32(rec, 12)?)) + } + + /// The table at `(offset, length)`; `None` when it is longer than `max_len` or can't be read. + pub(super) fn read(&mut self, (offset, len): (u32, u32), max_len: u32) -> Option> { + if len > max_len { + return None; + } + self.read_at(offset.into(), len as usize) + } + + fn read_at(&mut self, offset: u64, len: usize) -> Option> { + use std::io::{Read, Seek, SeekFrom}; + let mut buf = vec![0; len]; + self.file.seek(SeekFrom::Start(offset)).ok()?; + self.file.read_exact(&mut buf).ok()?; + Some(buf) + } +} + +/// The big-endian `u32` at `at` in `b`. +#[cfg(not(target_arch = "wasm32"))] +fn be32(b: &[u8], at: usize) -> Option { + b.get(at..at.checked_add(4)?).and_then(|s| s.try_into().ok()).map(u32::from_be_bytes) +} + /// A font file holding just `tables` (sorted by tag), each padded to four bytes. #[cfg(not(target_arch = "wasm32"))] pub(super) fn mini_font(tables: &[([u8; 4], Vec)]) -> Vec { diff --git a/crates/fonts/src/fontdb/tests/sysfonts.rs b/crates/fonts/src/fontdb/tests/sysfonts.rs index 7502c360ee..35a1909f95 100644 --- a/crates/fonts/src/fontdb/tests/sysfonts.rs +++ b/crates/fonts/src/fontdb/tests/sysfonts.rs @@ -305,13 +305,11 @@ fn fallback_cache_is_dropped_when_a_face_loads_by_its_legacy_family_name() { assert!(!cache.map.contains_key(&(missing, latin.id())), "answers from before the load are dropped"); } -#[test] -fn fallback_reads_each_installed_font_file_once_however_many_characters_miss() { - // #605: every distinct character no font covers read every installed font file again. - let dir = font_dir("coverage"); - // The installed Regular also maps U+30000 (no bundled font does): its last cmap group, the - // single character U+1F916, moved there (still the highest, so the groups stay sorted). - let mut regular = renamed("SourceSans3-Regular.ttf"); +/// A bundled Source Sans 3 Regular renamed `family` (13 characters) that also maps U+30000 (no +/// bundled font does): its last cmap group, the single character U+1F916, moved there (still the +/// highest, so the groups stay sorted). +fn mapping_u30000(family: &str) -> Vec { + let mut regular = renamed_to("SourceSans3-Regular.ttf", family); let (from, to) = ([0x0001_F916_u32.to_be_bytes(); 2].concat(), [0x0003_0000_u32.to_be_bytes(); 2].concat()); let mut patched = 0; while let Some(at) = regular.windows(from.len()).position(|w| w == from) { @@ -319,7 +317,14 @@ fn fallback_reads_each_installed_font_file_once_however_many_characters_miss() { patched += 1; } assert!(patched > 0); - std::fs::write(dir.join("Sysfont-Regular.ttf"), regular).unwrap(); + regular +} + +#[test] +fn fallback_reads_each_installed_font_file_once_however_many_characters_miss() { + // #605: every distinct character no font covers read every installed font file again. + let dir = font_dir("coverage"); + std::fs::write(dir.join("Sysfont-Regular.ttf"), mapping_u30000(FAMILY)).unwrap(); let db = FontDb::with_font_dirs(vec![dir]); let latin = db.face(FALLBACK_FAMILY, "Regular"); let found = db.fallback_for('\u{30000}', latin.id()).expect("the installed font maps it"); @@ -333,7 +338,7 @@ fn fallback_reads_each_installed_font_file_once_however_many_characters_miss() { } let index = db.coverage.lock().unwrap(); assert_eq!(index.files.len(), 2, "Regular and Bold are cataloged"); - assert_eq!(index.reads, 2, "each file read once"); + assert_eq!(index.reads, 2, "each file's character map read once"); } /// A bundled Source Sans 3 `file` renamed `family` whose last cmap group (the single character @@ -421,6 +426,45 @@ fn a_family_partly_loaded_by_a_character_fallback_is_loaded_whole_for_a_run() { assert_eq!(db.read_faces().iter().filter(|f| f.family == "Sysfont Famly").count(), 3); } +#[test] +fn a_preferred_fallback_family_loads_only_when_it_maps_the_character() { + // A character no loaded font covers loaded every installed family in the fallback list up + // to one that covers it, and kept them all: an emoji loaded every CJK family a Mac has, + // about 500 MB. Malgun Gothic comes before Apple Symbols in that list. + let dir = font_dir("preferred"); + std::fs::write(dir.join("Malgun.ttf"), renamed_to("SourceSans3-Regular.ttf", "Malgun Gothic")).unwrap(); + std::fs::write(dir.join("Symbols.ttf"), mapping_u30000("Apple Symbols")).unwrap(); + let db = FontDb::with_font_dirs(vec![dir.clone()]); + let latin = db.face(FALLBACK_FAMILY, "Regular"); + let found = db.fallback_for('\u{30000}', latin.id()).expect("Apple Symbols maps it"); + assert_eq!(found.family, "Apple Symbols"); + assert!(!db.is_loaded("Malgun Gothic"), "a family that lacks the character isn't loaded for it"); + assert_eq!(db.coverage.lock().unwrap().reads, 2, "only the two families' character maps were read"); + // The fallback order is unchanged: when both map it, the first in the list is chosen. + std::fs::write(dir.join("Malgun.ttf"), mapping_u30000("Malgun Gothic")).unwrap(); + let db = FontDb::with_font_dirs(vec![dir]); + let found = db.fallback_for('\u{30000}', latin.id()).expect("both map it"); + assert_eq!(found.family, "Malgun Gothic"); + assert!(!db.is_loaded("Apple Symbols")); +} + +#[test] +fn a_preferred_fallback_family_in_a_very_large_file_is_still_found() { + // The coverage index reads character maps, not whole files, so a preferred family is found + // whatever its size (PingFang is 75 MB, Apple Color Emoji 180 MB): here a font padded past + // the limit on files the index loads by itself. + let dir = font_dir("large"); + let mut data = mapping_u30000("Apple Symbols"); + data.resize(data.len() + SYS_FALLBACK_MAX_FILE as usize, 0); + std::fs::write(dir.join("Symbols.ttf"), &data).unwrap(); + drop(data); + let db = FontDb::with_font_dirs(vec![dir.clone()]); + let latin = db.face(FALLBACK_FAMILY, "Regular"); + let found = db.fallback_for('\u{30000}', latin.id()).map(|f| f.family.clone()); + std::fs::remove_file(dir.join("Symbols.ttf")).unwrap(); + assert_eq!(found.as_deref(), Some("Apple Symbols")); +} + #[test] fn screen_hinting_runs_a_tricky_fonts_instructions_with_the_monochrome_target() { // FreeType's "tricky" fonts (old CJK fonts listed by name, such as DynaLab's "cpop") assemble @@ -455,20 +499,111 @@ fn screen_hinting_runs_a_tricky_fonts_instructions_with_the_monochrome_target() } #[test] -fn coverage_ranges_hold_exactly_the_mapped_characters() { +fn coverage_holds_exactly_the_mapped_characters() { + let dir = font_dir("coverage-ranges"); let data = renamed("SourceSans3-Regular.ttf"); - let ranges = coverage_ranges(&data); + let ranges = file_coverage(&dir.join("Sysfont-Regular.ttf")); assert!(ranges.windows(2).all(|w| w[0].1 + 1 < w[1].0), "sorted, disjoint and merged"); let font = skrifa::FontRef::new(&data).unwrap(); let mapped: Vec = font.charmap().mappings().map(|(cp, _)| cp).collect(); assert!(ranges.len() < mapped.len() / 4, "ranges, not a set of characters"); assert!(mapped.iter().all(|cp| ranges_hold(&ranges, *cp))); - for cp in [0x0, 0x9, 0x0E01, 0x17000, 0x10_FFFF] { + for cp in [0x0, 0x9, 0x0E01, 0x17000, 0x3_0000, 0x10_FFFF] { assert_eq!(ranges_hold(&ranges, cp), font.charmap().map(cp).is_some(), "U+{cp:04X}"); } - // Damaged data indexes as nothing. - assert!(coverage_ranges(b"ttcf\0\x01\0\0\xff\xff\xff\xff").is_empty()); - assert!(coverage_ranges(&data[..100]).is_empty()); + // A collection: what its faces map together. + std::fs::write(dir.join("both.ttc"), collection(&[data.clone(), mapping_u30000(FAMILY)])).unwrap(); + let both = file_coverage(&dir.join("both.ttc")); + assert!(ranges_hold(&both, 0x3_0000) && mapped.iter().all(|cp| ranges_hold(&both, *cp))); + // Damaged, cut short, not a font or missing: nothing. + std::fs::write(dir.join("short.ttf"), &data[..100]).unwrap(); + for name in ["damaged.ttf", "short.ttf", "readme.txt", "missing.ttf"] { + assert!(file_coverage(&dir.join(name)).is_empty(), "{name}"); + } +} + +#[test] +fn coverage_stops_when_a_compaction_frees_too_little() { + // A crafted file whose faces interleave their code points made every mapping past the cap + // sort all the ranges again, freeing one: a compaction must free a quarter of the cap. + let cap = u32::try_from(Coverage::MAX_RANGES).unwrap(); + let mut c = Coverage::default(); + // One face maps every third code point, one range short of the cap; the next face maps the + // code points after them: the first fills the cap, the second compacts it and frees one. + assert!((0..cap - 1).all(|i| c.push(3 * i))); + assert!(c.push(1)); + assert!(!c.push(4), "a compaction that frees one range ends the file's coverage"); + // A compaction that frees enough goes on. + let mut c = Coverage::default(); + assert!((0..cap / 2).all(|i| c.push(2 * i)) && (0..cap / 2).all(|i| c.push(2 * i + 1))); + assert!(c.push(cap + 1), "the compaction merges every range"); + assert_eq!(&*c.finish(), &[(0, cap - 1), (cap + 1, cap + 1)]); +} + +#[test] +fn a_rescan_reads_a_replaced_font_files_character_maps_again() { + // A font updated in place since its character maps were read: what the update maps counts. + let dir = font_dir("replaced"); + let db = FontDb::with_font_dirs(vec![dir.clone()]); + assert!(!db.family_maps(FAMILY, '\u{30000}')); + // The update is a few bytes longer, so it differs however coarse the file times are. + let mut data = mapping_u30000(FAMILY); + data.extend_from_slice(&[0; 4]); + std::fs::write(dir.join("Sysfont-Regular.ttf"), data).unwrap(); + assert!(!db.family_maps(FAMILY, '\u{30000}'), "the index answers until the folders are scanned again"); + db.load_system_fonts(); + assert!(db.family_maps(FAMILY, '\u{30000}')); + assert_eq!(db.coverage.lock().unwrap().reads, 3, "the unchanged Bold file isn't read again"); +} + +#[test] +fn a_rescan_keeps_the_character_maps_a_fallback_search_read() { + // Files indexed by a fallback search, not only by a family lookup, keep their character maps + // through a rescan: #605's hundreds of files aren't read again after each font folder change. + let dir = font_dir("rescan-fallback"); + std::fs::write(dir.join("Sysfont-Regular.ttf"), mapping_u30000(FAMILY)).unwrap(); + let db = FontDb::with_font_dirs(vec![dir]); + let latin = db.face(FALLBACK_FAMILY, "Regular"); + assert!(db.fallback_for('\u{30000}', latin.id()).is_some(), "the installed font maps it"); + assert_eq!(db.coverage.lock().unwrap().reads, 2, "the Bold file and then the Regular"); + db.load_system_fonts(); + // Nothing maps it: the search reaches the Bold file, which isn't loaded, and asks its ranges. + assert!(db.fallback_for('\u{F0001}', latin.id()).is_none()); + assert_eq!(db.coverage.lock().unwrap().reads, 2, "the rescan kept both files' character maps"); +} + +#[test] +fn coverage_of_installed_fonts_is_what_their_loaded_faces_map() { + // Read from the character maps alone, as the whole file maps. Uses installed fonts when + // there are some, collections among them. + let paths: Vec<&Path> = [ + "/System/Library/Fonts/Helvetica.ttc", + "/System/Library/Fonts/Hiragino Sans GB.ttc", + "/System/Library/Fonts/Supplemental/Arial Unicode.ttf", + "C:\\Windows\\Fonts\\arial.ttf", + "C:\\Windows\\Fonts\\msyh.ttc", + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + "/usr/share/fonts/noto/NotoSans-Regular.ttf", + ] + .iter() + .map(Path::new) + .filter(|p| p.exists()) + .collect(); + if paths.is_empty() { + eprintln!("no known font installed: nothing to check"); + } + for path in paths { + let data = std::fs::read(path).unwrap(); + let faces = match FileRef::new(&data).unwrap() { + FileRef::Font(_) => 1, + FileRef::Collection(c) => c.len(), + }; + let mut whole = Coverage::default(); + for i in 0..faces { + assert!(whole.add(&skrifa::FontRef::from_index(&data, i).unwrap())); + } + assert_eq!(file_coverage(path), whole.finish(), "{}", path.display()); + } } /// A bundled font renamed by a `name` table holding just its family names: (Windows language id,