From 630e42f42c49eb8514fd14a780061afe2e8ed7cb Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sun, 1 Nov 2026 17:58:01 +0000 Subject: [PATCH] CP4.1: font store item kind language-pack (pinned index, verified .tar.zst install, remove, list) Additive: new src/language_packs.rs (+ tests); store.rs only widens a few helpers to pub(crate). Developer override CRAFT_LANGUAGE_PACK_INDEX for local end-to-end testing (no index pin; archives still verified). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- Cargo.lock | 9 + crates/font-store/Cargo.toml | 4 + crates/font-store/README.md | 29 + crates/font-store/src/http.rs | 6 + crates/font-store/src/language_packs.rs | 513 ++++++++++++++++++ crates/font-store/src/language_packs/tests.rs | 320 +++++++++++ crates/font-store/src/lib.rs | 12 +- crates/font-store/src/store.rs | 24 +- 8 files changed, 904 insertions(+), 13 deletions(-) create mode 100644 crates/font-store/src/language_packs.rs create mode 100644 crates/font-store/src/language_packs/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 4fcd223b..34c09d42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -837,10 +837,13 @@ dependencies = [ "craft-atomic", "craft-font-catalog", "flate2", + "log", + "ruzstd", "serde", "serde_json", "sha2", "skrifa 0.47.0", + "tar", "thiserror 2.0.21", "ureq", "write-fonts 0.54.1", @@ -4646,6 +4649,12 @@ dependencies = [ "unicode-script", ] +[[package]] +name = "ruzstd" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e4ce4a039f42bf25920b95667e4d789e98749ab11bbb8d2c2936f788421bb5" + [[package]] name = "ryu" version = "1.0.23" diff --git a/crates/font-store/Cargo.toml b/crates/font-store/Cargo.toml index 5e3ed270..02c35a55 100644 --- a/crates/font-store/Cargo.toml +++ b/crates/font-store/Cargo.toml @@ -25,6 +25,10 @@ thiserror = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] craft-atomic = { workspace = true } craft-font-catalog = { workspace = true } +log = "0.4" +# Language packs (`language_packs`): `.tar.zst` archives, unpacked in pure Rust. +ruzstd = { version = "0.9", default-features = false, features = ["std"] } +tar = { version = "0.4", default-features = false } # Blocking HTTPS with rustls and webpki roots: no async runtime, no OpenSSL. ureq = { version = "3", default-features = false, features = ["rustls", "gzip"] } diff --git a/crates/font-store/README.md b/crates/font-store/README.md index 12216302..26df5c44 100644 --- a/crates/font-store/README.md +++ b/crates/font-store/README.md @@ -94,6 +94,8 @@ settings.json the network setting and the sources the user added manifest.json what is installed indexes/ the last indexes fetched: sources.json, .json.gz, user- / the font files and the licence +language-packs/ installed Language packs, one folder per locale +language-packs.json which Language packs are installed .staging/ downloads being installed .lock held while the store changes ``` @@ -113,6 +115,33 @@ A Google Fonts family's folder is google/fonts' own folder name (`lato/`); anoth The catalog scans the store folder. With `with_catalog(catalog)`, every install and removal ends with `catalog.rescan()`, which reads only new and changed files, so the app sees the fonts at once. Other running apps notice through `Catalog::rescan_if_changed` (when their window is activated) or `Catalog::watch`. +## Language packs + +The store's second item kind, `language-pack`: a language's spelling dictionary, hyphenation patterns and licence texts, published by [storytold/craft-i18n](https://github.com/storytold/craft-i18n) as `.tar.zst` release assets listed in an `index.json`, and read at runtime by `craft-proof` (`craft_proof::packs`). It reuses the store's folder, consent, HTTP client, staging and sha256 checks; there is no second downloader. + +```rust +const PACKS: PackIndexPin = PackIndexPin { url: "https://github.com/storytold/craft-i18n/releases/download/language-packs-1/index.json", sha256: "" }; +let index = store.fetch_language_pack_index(&PACKS, &Cancel::new())?; // or cached_language_pack_index(&PACKS) +let nl = index.pack("nl").unwrap(); +store.install_language_pack(&index, nl, &Cancel::new(), &mut |p| ())?; // network on, like fonts +craft_proof::packs::set_packs_dir(Some(store.language_packs_dir())); // then packs::reload() after each change +store.remove_language_pack("nl")?; +let installed = store.installed_language_packs(); +``` + +| Call | What it does | +|---|---| +| `fetch_language_pack_index(pin, cancel)` | Downloads the index, checks it against the pin's sha256, saves it as `indexes/language-packs.json`. Needs the network on. | +| `cached_language_pack_index(pin)` | The saved index, when it is the pinned one. | +| `install_language_pack(index, pack, cancel, progress)` | Downloads the archive (at most 128 MiB), checks size and sha256, unpacks it into `.staging/` and moves it into `language-packs//` with one rename (an older pack is replaced the same way). Records it in `language-packs.json`. | +| `remove_language_pack(locale)` | Moves the folder out at once, updates the record, deletes the files. No network needed. | +| `installed_language_packs()` | The recorded packs whose folder is there. | +| `language_packs_dir()` | `/language-packs`, for `craft_proof::packs::set_packs_dir`. | + +**Unpacking is hostile-input safe:** pure-Rust zstd (`ruzstd`, MIT) and `tar` (MIT OR Apache-2.0); regular files and folders only (no links or devices), plain relative paths at most two levels deep (`LICENCES/`), no `..`, roots, drives or backslashes, each file at most 64 MiB, 256 MiB and 256 entries per pack, `pack.toml` required. Index entries need a lowercase BCP 47 locale (listed once), a sha256 and an `https://` URL. + +**Developer override (testing only):** `CRAFT_LANGUAGE_PACK_INDEX=` replaces the pinned index URL with an `http://` URL on a loopback host (`127.0.0.1`, `localhost`, `[::1]`), an `https://` URL or a local file path, and **skips the index's sha256 pin**. Archives are still checked against the sha256 and size in that index, and may then also be loopback `http://` URLs or local paths. Off unless the variable is set; a warning is logged whenever it is used. Never set it in a release. + ## Web The index, the lookups, `verify_font` and `IndexPin` build for `wasm32-unknown-unknown`. The store folder and the HTTP client are native only; the web keeps fonts elsewhere. diff --git a/crates/font-store/src/http.rs b/crates/font-store/src/http.rs index 94a6bdd8..3c6d007b 100644 --- a/crates/font-store/src/http.rs +++ b/crates/font-store/src/http.rs @@ -111,6 +111,12 @@ impl Client { } } + /// A client that also allows plain `http://`, used only for the developer override of the + /// Language pack index (`CRAFT_LANGUAGE_PACK_INDEX`), which accepts only loopback hosts. + pub(crate) fn allowing_http(t: Timeouts) -> Self { + Self::build(t, false) + } + /// Plain `http://` to a local test server, with `https://` prefixes mapped onto it. #[cfg(test)] pub(crate) fn for_tests(t: Timeouts, rewrite: Vec<(String, String)>) -> Self { diff --git a/crates/font-store/src/language_packs.rs b/crates/font-store/src/language_packs.rs new file mode 100644 index 00000000..19f66b74 --- /dev/null +++ b/crates/font-store/src/language_packs.rs @@ -0,0 +1,513 @@ +//! The store's second item kind, `language-pack`: a language's spelling dictionary, hyphenation +//! patterns and licence texts, published by storytold/craft-i18n as `.tar.zst` release +//! assets listed in an `index.json`, and read at runtime by `craft-proof` (`craft_proof::packs`). +//! +//! It reuses the store's folder, consent ([`FontStore::enable_network`]), HTTP client and sha256 +//! checks: the app pins the index by URL and sha256 ([`PackIndexPin`]), each archive is checked +//! against the size and sha256 the index gives, then unpacked into `.staging/` and moved into +//! `language-packs//` with one rename. Packs have no update checks of their own: a new +//! pin is a new index. +//! +//! ```text +//! /language-packs// pack.toml, spelling.aff, spelling.dic, hyphenation.pat, LICENCES/… +//! /language-packs.json which packs are installed, from which index entry +//! /indexes/language-packs.json the last pack index fetched +//! ``` +//! +//! **Developer override (testing only):** `CRAFT_LANGUAGE_PACK_INDEX=` replaces the +//! pinned index URL with an `http://` URL on a loopback host (`127.0.0.1`, `localhost`, `[::1]`), +//! an `https://` URL or a local file path, and skips the index's sha256 pin. Archives are still +//! checked against the sha256 and size in that index, and may then also be loopback `http://` +//! URLs or local paths. It is off unless the variable is set, and logged as a warning when used. + +use std::io::Read; +use std::path::{Component, Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::http::{Cancel, Client, Timeouts}; +use crate::store::{Error, FontStore, INDEX_CACHE, Progress, Stage, io, read_limited}; +use crate::verify::{VerifyError, sha256_hex}; + +/// The store's folder of installed Language packs. +pub const LANGUAGE_PACKS: &str = "language-packs"; +/// The developer override of the pinned pack index (see the module docs). Testing only. +pub const LANGUAGE_PACK_INDEX_ENV: &str = "CRAFT_LANGUAGE_PACK_INDEX"; +/// Largest pack index read. +pub const MAX_PACK_INDEX_BYTES: u64 = 4 << 20; +/// Largest pack archive downloaded. +pub const MAX_PACK_ARCHIVE_BYTES: u64 = 128 << 20; +/// Largest file unpacked from a pack (craft-proof's dictionary cap). +pub const MAX_PACK_FILE_BYTES: u64 = 64 << 20; +/// Largest total unpacked size of a pack. +pub const MAX_PACK_UNPACKED_BYTES: u64 = 256 << 20; +/// Most files and folders in a pack archive. +pub const MAX_PACK_ENTRIES: usize = 256; +/// Most packs in an index. +const MAX_PACKS: usize = 1024; +const INSTALLED: &str = "language-packs.json"; +const INDEX_CACHE_NAME: &str = "language-packs.json"; + +/// The pack index an app trusts: `index.json` of a craft-i18n `language-packs-` release, by +/// URL and sha256 (the app's Pin). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PackIndexPin { + /// `https://github.com/storytold/craft-i18n/releases/download/language-packs-/index.json`. + pub url: &'static str, + /// The sha256 of that file, lowercase hex. + pub sha256: &'static str, +} + +/// `index.json` of a pack release. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PackIndex { + /// The format; 1. + pub format: u32, + /// The release (`language-packs-1`). + pub release: String, + /// The packs, by locale. + pub packs: Vec, +} + +/// One pack of a [`PackIndex`]. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PackEntry { + /// Lowercase BCP 47 tag (`nl`, `pt-br`); the pack's folder name. + pub locale: String, + /// The language's own name (`Nederlands`). + pub name: String, + /// The craft-i18n commit it was built from. + #[serde(default)] + pub version: String, + /// The archive (`.tar.zst`). + pub url: String, + /// The archive's sha256, lowercase hex. + pub sha256: String, + /// The archive's size in bytes. + pub size: u64, + /// SPDX licence expression. + pub licence: String, + /// The licence class: `Embeddable` or `Download-only`. + #[serde(default)] + pub class: String, + /// Whether it has a spelling dictionary. + #[serde(default)] + pub spelling: bool, + /// Whether it has hyphenation patterns. + #[serde(default)] + pub hyphenation: bool, +} + +/// An installed pack, as recorded when it was installed. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct InstalledLanguagePack { + /// Lowercase BCP 47 tag; the folder under [`LANGUAGE_PACKS`]. + pub locale: String, + /// The language's own name. + pub name: String, + /// The release it came from. + pub release: String, + /// The craft-i18n commit it was built from. + pub version: String, + /// SPDX licence expression. + pub licence: String, + /// The archive's sha256. + pub sha256: String, + /// The archive's size in bytes. + pub size: u64, +} + +#[derive(Default, Serialize, Deserialize)] +struct Installed { + version: u32, + packs: Vec, +} + +impl PackIndex { + /// Parses and checks an index: format 1, every locale a lowercase tag and listed once, every + /// archive at most [`MAX_PACK_ARCHIVE_BYTES`] with a sha256. Archive URLs must be `https://` + /// (with `local`, the developer override, also loopback `http://` or a path). + pub fn from_json(bytes: &[u8]) -> Result { + Self::parse(bytes, false) + } + + fn parse(bytes: &[u8], local: bool) -> Result { + let bad = |why: String| Error::Io { what: "reading the Language pack index".into(), source: std::io::Error::other(why) }; + let index: PackIndex = serde_json::from_slice(bytes).map_err(|e| bad(e.to_string()))?; + if index.format != 1 { + return Err(bad(format!("format {} is not supported", index.format))); + } + if index.packs.len() > MAX_PACKS { + return Err(bad("too many packs".into())); + } + for (i, p) in index.packs.iter().enumerate() { + if !is_locale(&p.locale) || index.packs.iter().take(i).any(|q| q.locale == p.locale) { + return Err(bad(format!("pack {:?}: not a lowercase language tag, or listed twice", p.locale.chars().take(40).collect::()))); + } + if p.size > MAX_PACK_ARCHIVE_BYTES || !is_sha256(&p.sha256) { + return Err(bad(format!("pack {}: bad size or sha256", p.locale))); + } + if !(p.url.starts_with("https://") || local && local_source(&p.url).is_some()) { + return Err(bad(format!("pack {}: {} is not an https URL", p.locale, p.url.chars().take(200).collect::()))); + } + } + Ok(index) + } + + /// The pack for `locale`. + pub fn pack(&self, locale: &str) -> Option<&PackEntry> { + self.packs.iter().find(|p| p.locale == locale) + } +} + +/// Where the pack index comes from: the pin, or the developer override. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum IndexSource { + /// The pinned URL, checked against the pinned sha256. + Pinned(PackIndexPin), + /// `CRAFT_LANGUAGE_PACK_INDEX`: no index pin; loopback `http://` and paths allowed. + Override(String), +} + +impl IndexSource { + /// The pin, unless [`LANGUAGE_PACK_INDEX_ENV`] is set. + fn from_env(pin: &PackIndexPin) -> Self { + match std::env::var(LANGUAGE_PACK_INDEX_ENV).ok().map(|v| v.trim().to_string()).filter(|v| !v.is_empty()) { + Some(v) => { + log::warn!( + "{LANGUAGE_PACK_INDEX_ENV}={v}: the Language pack index is NOT the pinned one and its sha256 is not checked (testing only)" + ); + Self::Override(v) + } + None => Self::Pinned(*pin), + } + } + + fn is_override(&self) -> bool { + matches!(self, Self::Override(_)) + } +} + +/// A source the developer override may read: `Ok(url)` for an `https://` URL or an `http://` +/// one on a loopback host, `Err(path)` for a local file path; `None` for anything else. +fn local_source(s: &str) -> Option> { + if s.starts_with("https://") { + return Some(Ok(s)); + } + if let Some(rest) = s.strip_prefix("http://") { + let host = rest.split(['/', '?', '#']).next().unwrap_or_default(); + let host = host.rsplit_once('@').map_or(host, |(_, h)| h); + let name = if host.starts_with('[') { + host.split(']').next().map(|h| format!("{h}]")).unwrap_or_default() + } else { + host.split(':').next().unwrap_or_default().to_string() + }; + return ["127.0.0.1", "localhost", "[::1]"].contains(&name.as_str()).then_some(Ok(s)); + } + if s.contains("://") { + return None; + } + let path = s.strip_prefix("file://").unwrap_or(s); + Some(Err(PathBuf::from(path))) +} + +/// A lowercase BCP 47 tag as packs are named (craft-proof's rule): ASCII lowercase letters and +/// digits in parts of 1 to 8 joined by `-`, at most 35 characters. +fn is_locale(s: &str) -> bool { + !s.is_empty() + && s.len() <= 35 + && s.split('-').all(|p| !p.is_empty() && p.len() <= 8 && p.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit())) +} + +fn is_sha256(s: &str) -> bool { + s.len() == 64 && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +impl FontStore { + /// The folder of installed Language packs (`/language-packs`): give it to + /// `craft_proof::packs::set_packs_dir`. + pub fn language_packs_dir(&self) -> PathBuf { + self.dir().join(LANGUAGE_PACKS) + } + + /// The pack index saved by the last [`FontStore::fetch_language_pack_index`], when it is + /// `pin`'s (or the developer override is set). + pub fn cached_language_pack_index(&self, pin: &PackIndexPin) -> Option { + self.cached_pack_index_from(&IndexSource::from_env(pin)) + } + + pub(crate) fn cached_pack_index_from(&self, source: &IndexSource) -> Option { + let bytes = read_limited(&self.dir().join(INDEX_CACHE).join(INDEX_CACHE_NAME), MAX_PACK_INDEX_BYTES)?; + match source { + IndexSource::Pinned(pin) if sha256_hex(&bytes) != pin.sha256 => None, + _ => PackIndex::parse(&bytes, source.is_override()).ok(), + } + } + + /// Downloads the pack index `pin` names, checks it against the pin and saves it for + /// [`FontStore::cached_language_pack_index`]. Needs the network on. + pub fn fetch_language_pack_index(&self, pin: &PackIndexPin, cancel: &Cancel) -> Result { + self.fetch_pack_index_from(&IndexSource::from_env(pin), cancel) + } + + pub(crate) fn fetch_pack_index_from(&self, source: &IndexSource, cancel: &Cancel) -> Result { + self.require_network()?; + let bytes = match source { + IndexSource::Pinned(pin) => { + let check = |b: &[u8]| if sha256_hex(b) == pin.sha256 { Ok(()) } else { Err(VerifyError::Sha256) }; + self.fetch(&[pin.url.to_string()], MAX_PACK_INDEX_BYTES, "the Language pack index", cancel, &mut |_| {}, &check)? + } + IndexSource::Override(at) => self.get_local(at, MAX_PACK_INDEX_BYTES, "the Language pack index", cancel, &mut |_| {}, &|_| Ok(()))?, + }; + let index = PackIndex::parse(&bytes, source.is_override())?; + std::fs::create_dir_all(self.dir().join(INDEX_CACHE)).map_err(io("creating the index folder"))?; + self.write_file(&format!("{INDEX_CACHE}/{INDEX_CACHE_NAME}"), &bytes)?; + Ok(index) + } + + /// A download under the developer override: an https or loopback http URL, or a file. + fn get_local( + &self, + at: &str, + max: u64, + what: &str, + cancel: &Cancel, + on_bytes: &mut dyn FnMut(u64), + check: &dyn Fn(&[u8]) -> Result<(), VerifyError>, + ) -> Result, Error> { + let bytes = match local_source(at) { + Some(Ok(url)) if url.starts_with("https://") => return self.fetch(&[url.to_string()], max, what, cancel, on_bytes, check), + Some(Ok(url)) => Client::allowing_http(Timeouts::default()) + .get(url, max, cancel, on_bytes) + .map_err(|e| Error::Download { what: what.to_string(), reasons: vec![e.to_string()] })?, + Some(Err(path)) => { + let mut buf = Vec::new(); + std::fs::File::open(&path) + .and_then(|f| f.take(max.saturating_add(1)).read_to_end(&mut buf)) + .map_err(io(format!("reading {}", path.display())))?; + if buf.len() as u64 > max { + return Err(Error::Download { what: what.to_string(), reasons: vec![format!("larger than {max} bytes")] }); + } + on_bytes(buf.len() as u64); + buf + } + None => return Err(Error::BadUrl(at.chars().take(200).collect())), + }; + check(&bytes).map_err(|error| Error::Verify { file: what.to_string(), error })?; + Ok(bytes) + } + + /// Downloads `pack` from `index`, checks its size and sha256, unpacks it and moves it into + /// `language-packs//` in one rename, replacing an older one. Needs the network on. + /// Then call `craft_proof::packs::reload()`. + pub fn install_language_pack( + &self, + index: &PackIndex, + pack: &PackEntry, + cancel: &Cancel, + progress: &mut dyn FnMut(Progress), + ) -> Result { + let local = std::env::var(LANGUAGE_PACK_INDEX_ENV).is_ok_and(|v| !v.trim().is_empty()); + self.install_pack_from(local, index, pack, cancel, progress) + } + + pub(crate) fn install_pack_from( + &self, + local: bool, + index: &PackIndex, + pack: &PackEntry, + cancel: &Cancel, + progress: &mut dyn FnMut(Progress), + ) -> Result { + self.require_network()?; + // The entry is checked again here: callers may build one by hand. + if !is_locale(&pack.locale) || !is_sha256(&pack.sha256) || pack.size > MAX_PACK_ARCHIVE_BYTES { + return Err(Error::BadUrl(pack.locale.chars().take(40).collect())); + } + let what = format!("{}.tar.zst", pack.locale); + let (size, sha) = (pack.size, pack.sha256.clone()); + let check = move |b: &[u8]| { + if b.len() as u64 != size { + Err(VerifyError::Size { expected: size, actual: b.len() as u64 }) + } else if sha256_hex(b) != sha { + Err(VerifyError::Sha256) + } else { + Ok(()) + } + }; + let mut on_bytes = |done: u64| progress(Progress { file: 0, files: 1, done, total: size }); + let bytes = if local { + self.get_local(&pack.url, size, &what, cancel, &mut on_bytes, &check)? + } else if pack.url.starts_with("https://") { + self.fetch(std::slice::from_ref(&pack.url), size, &what, cancel, &mut on_bytes, &check)? + } else { + return Err(Error::BadUrl(pack.url.chars().take(200).collect())); + }; + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + self.sweep_staging(); + let stage = Stage::new(self.dir())?; + let unpacked = stage.dir.join(&pack.locale); + unpack(&bytes, &unpacked).map_err(|e| Error::Io { what: format!("unpacking {what}"), source: std::io::Error::other(e) })?; + let record = InstalledLanguagePack { + locale: pack.locale.clone(), + name: pack.name.chars().take(200).collect(), + release: index.release.chars().take(200).collect(), + version: pack.version.chars().take(200).collect(), + licence: pack.licence.chars().take(200).collect(), + sha256: pack.sha256.clone(), + size: pack.size, + }; + let _lock = self.lock()?; + let packs = self.language_packs_dir(); + std::fs::create_dir_all(&packs).map_err(io("creating the Language packs folder"))?; + let dest = packs.join(&pack.locale); + let old = stage.dir.join(format!("{}.old", pack.locale)); + let had = std::fs::symlink_metadata(&dest).is_ok(); + if had { + std::fs::rename(&dest, &old).map_err(io(format!("replacing the {} pack", pack.locale)))?; + } + if let Err(e) = std::fs::rename(&unpacked, &dest) { + if had { + let _ = std::fs::rename(&old, &dest); + } + return Err(io(format!("installing the {} pack", pack.locale))(e)); + } + let mut installed = self.read_installed(); + installed.packs.retain(|p| p.locale != record.locale); + installed.packs.push(record.clone()); + installed.packs.sort_by(|a, b| a.locale.cmp(&b.locale)); + if let Err(e) = self.write_installed(&installed) { + // Put the old pack back, so folder and record agree. + let _ = std::fs::remove_dir_all(&dest); + if had { + let _ = std::fs::rename(&old, &dest); + } + return Err(e); + } + Ok(record) + } + + /// The installed Language packs whose folder is there, by locale. + pub fn installed_language_packs(&self) -> Vec { + let dir = self.language_packs_dir(); + self.read_installed().packs.into_iter().filter(|p| is_locale(&p.locale) && dir.join(&p.locale).is_dir()).collect() + } + + /// Removes the pack for `locale`: its folder disappears at once (one rename) and is deleted + /// after. No network needed. Then call `craft_proof::packs::reload()`. + pub fn remove_language_pack(&self, locale: &str) -> Result<(), Error> { + if !is_locale(locale) { + return Err(Error::NotInstalled(locale.chars().take(40).collect())); + } + let stage = Stage::new(self.dir())?; + let _lock = self.lock()?; + let dest = self.language_packs_dir().join(locale); + let mut installed = self.read_installed(); + let known = installed.packs.iter().any(|p| p.locale == locale); + if !known && std::fs::symlink_metadata(&dest).is_err() { + return Err(Error::NotInstalled(locale.to_string())); + } + if std::fs::symlink_metadata(&dest).is_ok() { + std::fs::rename(&dest, stage.dir.join(locale)).map_err(io(format!("removing the {locale} pack")))?; + } + installed.packs.retain(|p| p.locale != locale); + self.write_installed(&installed) + // `stage` deletes the old folder when dropped. + } + + fn read_installed(&self) -> Installed { + read_limited(&self.dir().join(INSTALLED), MAX_PACK_INDEX_BYTES).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default() + } + + fn write_installed(&self, installed: &Installed) -> Result<(), Error> { + let bytes = serde_json::to_vec_pretty(&Installed { version: 1, packs: installed.packs.clone() }) + .map_err(|e| io("saving the Language pack list")(std::io::Error::other(e)))?; + self.write_file(INSTALLED, &bytes) + } +} + +/// Unpacks a `.tar.zst` pack into `dest` (created): regular files and folders only, at most two +/// levels deep (`LICENCES/`), every path plain and relative, each file at most +/// [`MAX_PACK_FILE_BYTES`], all of them [`MAX_PACK_UNPACKED_BYTES`], at most +/// [`MAX_PACK_ENTRIES`]; `pack.toml` required. +pub(crate) fn unpack(archive: &[u8], dest: &Path) -> Result<(), String> { + let zstd = ruzstd::decoding::StreamingDecoder::new(archive).map_err(|e| format!("not zstd: {e}"))?; + // The decompressed stream is capped too: tar padding and headers beyond the files' bytes. + let limited = zstd.take(MAX_PACK_UNPACKED_BYTES.saturating_add((MAX_PACK_ENTRIES as u64 + 2) * 1024)); + let mut tar = tar::Archive::new(limited); + std::fs::create_dir_all(dest).map_err(|e| e.to_string())?; + let (mut total, mut count) = (0u64, 0usize); + let mut written: Vec = Vec::new(); + for entry in tar.entries().map_err(|e| format!("not a tar archive: {e}"))? { + let mut entry = entry.map_err(|e| format!("damaged archive: {e}"))?; + count += 1; + if count > MAX_PACK_ENTRIES { + return Err("too many files".into()); + } + let raw = entry.path().map_err(|e| format!("bad file name: {e}"))?.into_owned(); + let rel = safe_entry_path(&raw).ok_or_else(|| format!("{} is outside the pack", raw.display()))?; + let kind = entry.header().entry_type(); + if kind.is_dir() { + if rel.components().count() > 1 { + return Err(format!("{}: folders nest at most one level", rel.display())); + } + if !rel.as_os_str().is_empty() { + std::fs::create_dir_all(dest.join(&rel)).map_err(|e| e.to_string())?; + } + continue; + } + if !kind.is_file() { + return Err(format!("{}: only files and folders are allowed", rel.display())); + } + if rel.components().count() > 2 || written.contains(&rel) { + return Err(format!("{}: too deep, or listed twice", rel.display())); + } + let size = entry.header().size().map_err(|e| e.to_string())?; + if size > MAX_PACK_FILE_BYTES { + return Err(format!("{}: larger than {MAX_PACK_FILE_BYTES} bytes", rel.display())); + } + total = total.saturating_add(size); + if total > MAX_PACK_UNPACKED_BYTES { + return Err(format!("larger than {MAX_PACK_UNPACKED_BYTES} bytes unpacked")); + } + let path = dest.join(&rel); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| e.to_string())?; + } + let mut out = std::fs::File::options().write(true).create_new(true).open(&path).map_err(|e| format!("{}: {e}", rel.display()))?; + let copied = std::io::copy(&mut (&mut entry).take(MAX_PACK_FILE_BYTES), &mut out).map_err(|e| format!("{}: {e}", rel.display()))?; + if copied != size { + return Err(format!("{}: truncated", rel.display())); + } + written.push(rel); + } + if !written.iter().any(|p| p.as_os_str() == "pack.toml") { + return Err("no pack.toml".into()); + } + Ok(()) +} + +/// `p` as a plain relative path (`./` prefixes dropped), or `None` for a root, a drive, `..` or a +/// name that isn't UTF-8 or holds a backslash, colon or control character. +fn safe_entry_path(p: &Path) -> Option { + let mut out = PathBuf::new(); + for c in p.components() { + match c { + Component::CurDir => {} + Component::Normal(n) => { + let n = n.to_str()?; + if n.is_empty() || n.contains(['\\', ':']) || n.chars().any(char::is_control) { + return None; + } + out.push(n); + } + Component::ParentDir | Component::RootDir | Component::Prefix(_) => return None, + } + } + Some(out) +} + +#[cfg(test)] +mod tests; diff --git a/crates/font-store/src/language_packs/tests.rs b/crates/font-store/src/language_packs/tests.rs new file mode 100644 index 00000000..f9ef14e8 --- /dev/null +++ b/crates/font-store/src/language_packs/tests.rs @@ -0,0 +1,320 @@ +use std::collections::HashMap; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use super::*; +use crate::store::Caller; + +const RELEASE: &str = "https://github.com/storytold/craft-i18n/releases/download/language-packs-1/"; + +/// A local HTTP server answering 200 with the body set for a path, else 404. +struct Server { + base: String, + routes: Arc>>>, +} + +impl Server { + fn start() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}/", listener.local_addr().unwrap()); + let routes: Arc>>> = Arc::default(); + let r = routes.clone(); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let r = r.clone(); + std::thread::spawn(move || answer(stream, &r)); + } + }); + Self { base, routes } + } + + fn serve(&self, path: &str, body: &[u8]) { + self.routes.lock().unwrap().insert(format!("/{path}"), body.to_vec()); + } + + /// A client that maps the release URL onto this server. + fn client(&self) -> Client { + let t = Timeouts { connect: Duration::from_secs(2), response: Duration::from_secs(5), overall: Duration::from_secs(10) }; + Client::for_tests(t, vec![(RELEASE.into(), format!("{}release/", self.base))]) + } +} + +fn answer(mut s: TcpStream, routes: &Mutex>>) { + let mut head = Vec::new(); + let mut b = [0u8; 1]; + while !head.ends_with(b"\r\n\r\n") { + match s.read(&mut b) { + Ok(1) => head.push(b[0]), + _ => return, + } + } + let path = String::from_utf8_lossy(&head).split_whitespace().nth(1).unwrap_or("/").to_string(); + let body = routes.lock().unwrap().get(&path).cloned(); + match body { + Some(body) => { + let _ = write!(s, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + let _ = s.write_all(&body); + } + None => { + let _ = write!(s, "HTTP/1.1 404 Nope\r\nContent-Length: 4\r\nConnection: close\r\n\r\nnope"); + } + } +} + +fn temp(name: &str) -> PathBuf { + static N: AtomicUsize = AtomicUsize::new(0); + let d = std::env::temp_dir().join(format!("craft-language-packs-{name}-{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed))); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +/// A tar of `files` (name, kind, bytes), with raw header names so hostile ones can be written. +fn tar_of(files: &[(&str, tar::EntryType, &[u8])]) -> Vec { + let mut b = tar::Builder::new(Vec::new()); + for (name, kind, bytes) in files { + let mut h = tar::Header::new_old(); + let field = &mut h.as_old_mut().name; + field[..name.len()].copy_from_slice(name.as_bytes()); + h.set_entry_type(*kind); + h.set_size(bytes.len() as u64); + h.set_mode(0o644); + if *kind == tar::EntryType::Symlink || *kind == tar::EntryType::Link { + h.set_link_name("/etc/passwd").unwrap(); + } + h.set_cksum(); + b.append(&h, *bytes).unwrap(); + } + b.into_inner().unwrap() +} + +fn zst(bytes: &[u8]) -> Vec { + ruzstd::encoding::compress_to_vec(bytes, ruzstd::encoding::CompressionLevel::Fastest) +} + +const FILE: tar::EntryType = tar::EntryType::Regular; + +/// A good pack archive for `locale`, with `marker` in its word list. +fn pack_archive(locale: &str, marker: &str) -> Vec { + let toml = format!("locale = \"{locale}\"\nname = \"Test\"\nlicence = \"MIT\"\nclass = \"Embeddable\"\n"); + let dic = format!("2\nhuis\n{marker}\n"); + zst(&tar_of(&[ + ("pack.toml", FILE, toml.as_bytes()), + ("spelling.aff", FILE, b"SET UTF-8\n"), + ("spelling.dic", FILE, dic.as_bytes()), + ("hyphenation.pat", FILE, b"1na n2at\n"), + ("LICENCES/", tar::EntryType::Directory, b""), + ("LICENCES/COPYING", FILE, b"Test licence."), + ])) +} + +fn entry(locale: &str, url: &str, archive: &[u8]) -> serde_json::Value { + serde_json::json!({"locale": locale, "name": "Nederlands", "version": "abc", "url": url, "sha256": sha256_hex(archive), + "size": archive.len(), "licence": "MIT", "class": "Embeddable", "spelling": true, "hyphenation": true}) +} + +fn index_json(packs: &[serde_json::Value]) -> Vec { + serde_json::to_vec(&serde_json::json!({"format": 1, "release": "language-packs-1", "packs": packs})).unwrap() +} + +/// A store whose network is on, a server with the index and the `nl` pack, and the pin. +fn setup(name: &str) -> (FontStore, Server, PackIndexPin, Vec) { + let server = Server::start(); + let archive = pack_archive("nl", "zzmarker"); + let index = index_json(&[entry("nl", &format!("{RELEASE}nl.tar.zst"), &archive)]); + server.serve("release/nl.tar.zst", &archive); + server.serve("release/index.json", &index); + let sha: &'static str = Box::leak(sha256_hex(&index).into_boxed_str()); + let pin = PackIndexPin { url: "https://github.com/storytold/craft-i18n/releases/download/language-packs-1/index.json", sha256: sha }; + let store = FontStore::new(temp(name), Caller::User).with_client(server.client()); + store.enable_network().unwrap(); + (store, server, pin, archive) +} + +#[test] +fn install_list_replace_and_remove_a_pack() { + let (store, server, pin, _) = setup("install"); + let source = IndexSource::Pinned(pin); + let index = store.fetch_pack_index_from(&source, &Cancel::new()).unwrap(); + assert_eq!(store.cached_pack_index_from(&source), Some(index.clone())); + let nl = index.pack("nl").unwrap().clone(); + let mut seen = Vec::new(); + let got = store.install_pack_from(false, &index, &nl, &Cancel::new(), &mut |p| seen.push(p)).unwrap(); + assert_eq!((got.locale.as_str(), got.release.as_str(), got.licence.as_str()), ("nl", "language-packs-1", "MIT")); + assert!(seen.last().is_some_and(|p| p.done == nl.size && p.total == nl.size)); + let dir = store.language_packs_dir().join("nl"); + assert_eq!(store.language_packs_dir(), store.dir().join("language-packs")); + for f in ["pack.toml", "spelling.aff", "spelling.dic", "hyphenation.pat", "LICENCES/COPYING"] { + assert!(dir.join(f).is_file(), "{f}"); + } + assert!(std::fs::read_to_string(dir.join("spelling.dic")).unwrap().contains("zzmarker")); + assert_eq!(store.installed_language_packs(), vec![got]); + // A newer archive replaces the old one in one move. + let newer = pack_archive("nl", "zznewer"); + server.serve("release/nl.tar.zst", &newer); + let newer_entry = PackEntry { sha256: sha256_hex(&newer), size: newer.len() as u64, ..nl.clone() }; + store.install_pack_from(false, &index, &newer_entry, &Cancel::new(), &mut |_| {}).unwrap(); + assert!(std::fs::read_to_string(dir.join("spelling.dic")).unwrap().contains("zznewer")); + assert_eq!(store.installed_language_packs().len(), 1); + assert!(std::fs::read_dir(store.dir().join(".staging")).into_iter().flatten().flatten().next().is_none(), "nothing left staged"); + store.remove_language_pack("nl").unwrap(); + assert!(!dir.exists() && store.installed_language_packs().is_empty()); + assert!(matches!(store.remove_language_pack("nl"), Err(Error::NotInstalled(_)))); + assert!(matches!(store.remove_language_pack("../x"), Err(Error::NotInstalled(_)))); +} + +#[test] +fn network_off_pins_and_checksums_are_enforced() { + let (store, server, pin, archive) = setup("pins"); + let source = IndexSource::Pinned(pin); + store.disable_network().unwrap(); + assert!(matches!(store.fetch_pack_index_from(&source, &Cancel::new()), Err(Error::NetworkOff))); + store.enable_network().unwrap(); + // Another index than the pinned one. + let wrong = IndexSource::Pinned(PackIndexPin { sha256: "0000000000000000000000000000000000000000000000000000000000000000", ..pin }); + assert!(matches!(store.fetch_pack_index_from(&wrong, &Cancel::new()), Err(Error::Verify { error: VerifyError::Sha256, .. }))); + let index = store.fetch_pack_index_from(&source, &Cancel::new()).unwrap(); + assert_eq!(store.cached_pack_index_from(&wrong), None, "the cache is the pinned index only"); + let nl = index.pack("nl").unwrap().clone(); + // An archive that isn't the one the index names. + server.serve("release/nl.tar.zst", &pack_archive("nl", "tampered")); + assert!(store.install_pack_from(false, &index, &nl, &Cancel::new(), &mut |_| {}).is_err()); + server.serve("release/nl.tar.zst", &archive); + let bad_sha = PackEntry { sha256: sha256_hex(b"other"), ..nl.clone() }; + assert!(matches!( + store.install_pack_from(false, &index, &bad_sha, &Cancel::new(), &mut |_| {}), + Err(Error::Verify { error: VerifyError::Sha256, .. }) + )); + assert!(store.installed_language_packs().is_empty() && !store.language_packs_dir().join("nl").exists()); + // Hand-made entries are checked again. + for e in [PackEntry { locale: "../nl".into(), ..nl.clone() }, PackEntry { url: "http://example.org/nl.tar.zst".into(), ..nl.clone() }] { + assert!(store.install_pack_from(false, &index, &e, &Cancel::new(), &mut |_| {}).is_err()); + } + store.disable_network().unwrap(); + assert!(matches!(store.install_pack_from(false, &index, &nl, &Cancel::new(), &mut |_| {}), Err(Error::NetworkOff))); +} + +#[test] +fn indexes_are_checked() { + let a = pack_archive("nl", "x"); + let ok = index_json(&[entry("nl", &format!("{RELEASE}nl.tar.zst"), &a)]); + assert_eq!(PackIndex::from_json(&ok).unwrap().packs.len(), 1); + let bad = [ + serde_json::to_vec(&serde_json::json!({"format": 2, "release": "r", "packs": []})).unwrap(), + index_json(&[entry("NL", &format!("{RELEASE}nl.tar.zst"), &a)]), + index_json(&[entry("../x", &format!("{RELEASE}nl.tar.zst"), &a)]), + index_json(&[entry("nl", &format!("{RELEASE}nl.tar.zst"), &a), entry("nl", &format!("{RELEASE}nl.tar.zst"), &a)]), + index_json(&[entry("nl", "http://127.0.0.1/nl.tar.zst", &a)]), + index_json(&[entry("nl", "/tmp/nl.tar.zst", &a)]), + b"not json".to_vec(), + ]; + for b in bad { + assert!(PackIndex::from_json(&b).is_err(), "{}", String::from_utf8_lossy(&b)); + } + // The developer override also takes loopback http and paths, never other http hosts. + assert!(PackIndex::parse(&index_json(&[entry("nl", "http://127.0.0.1:9/nl.tar.zst", &a)]), true).is_ok()); + assert!(PackIndex::parse(&index_json(&[entry("nl", "/tmp/nl.tar.zst", &a)]), true).is_ok()); + assert!(PackIndex::parse(&index_json(&[entry("nl", "http://example.org/nl.tar.zst", &a)]), true).is_err()); + for (s, ok) in [ + ("http://127.0.0.1:8000/i.json", true), + ("http://localhost/i.json", true), + ("http://[::1]:8/i.json", true), + ("http://127.0.0.1.evil.org/i.json", false), + ("http://evil.org@127.0.0.1/i.json", true), + ("http://127.0.0.1@evil.org/i.json", false), + ("ftp://127.0.0.1/i.json", false), + ] { + assert_eq!(local_source(s).is_some(), ok, "{s}"); + } +} + +/// The developer override (`CRAFT_LANGUAGE_PACK_INDEX`): an index from a local path or a +/// loopback http server, with no index pin, while archives are still checked against its sha256. +#[test] +fn developer_override_reads_a_local_index_without_the_pin() { + let (store, server, _, archive) = setup("override"); + // A local file index pointing at a local archive. + let dir = temp("override-files"); + std::fs::write(dir.join("nl.tar.zst"), &archive).unwrap(); + let index = index_json(&[entry("nl", &dir.join("nl.tar.zst").display().to_string(), &archive)]); + std::fs::write(dir.join("index.json"), &index).unwrap(); + let source = IndexSource::Override(dir.join("index.json").display().to_string()); + let got = store.fetch_pack_index_from(&source, &Cancel::new()).unwrap(); + assert_eq!(store.cached_pack_index_from(&source), Some(got.clone())); + store.install_pack_from(true, &got, got.pack("nl").unwrap(), &Cancel::new(), &mut |_| {}).unwrap(); + assert!(store.language_packs_dir().join("nl/spelling.dic").is_file()); + // A loopback http index and archive. + let http_index = index_json(&[entry("nl", &format!("{}local/nl.tar.zst", server.base), &archive)]); + server.serve("local/index.json", &http_index); + server.serve("local/nl.tar.zst", &archive); + let source = IndexSource::Override(format!("{}local/index.json", server.base)); + let got = store.fetch_pack_index_from(&source, &Cancel::new()).unwrap(); + store.install_pack_from(true, &got, got.pack("nl").unwrap(), &Cancel::new(), &mut |_| {}).unwrap(); + // The archive is still verified. + server.serve("local/nl.tar.zst", &pack_archive("nl", "tampered")); + assert!(store.install_pack_from(true, &got, got.pack("nl").unwrap(), &Cancel::new(), &mut |_| {}).is_err()); + // Only loopback http. + let source = IndexSource::Override("http://example.org/index.json".into()); + assert!(matches!(store.fetch_pack_index_from(&source, &Cancel::new()), Err(Error::BadUrl(_)))); + // Without the variable the pin is used. + let pin = PackIndexPin { url: "https://example.org/index.json", sha256: "00" }; + if std::env::var_os(LANGUAGE_PACK_INDEX_ENV).is_none() { + assert_eq!(IndexSource::from_env(&pin), IndexSource::Pinned(pin)); + } +} + +#[test] +fn hostile_archives_are_refused() { + let toml: &[u8] = b"locale = \"nl\"\nlicence = \"MIT\"\n"; + let big = vec![0u8; 10]; + let cases: Vec<(&str, Vec)> = vec![ + ("traversal", zst(&tar_of(&[("pack.toml", FILE, toml), ("../evil", FILE, b"x")]))), + ("deep traversal", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/../../evil", FILE, b"x")]))), + ("absolute", zst(&tar_of(&[("pack.toml", FILE, toml), ("/tmp/evil", FILE, b"x")]))), + ("symlink", zst(&tar_of(&[("pack.toml", FILE, toml), ("spelling.dic", tar::EntryType::Symlink, b"")]))), + ("hard link", zst(&tar_of(&[("pack.toml", FILE, toml), ("spelling.dic", tar::EntryType::Link, b"")]))), + ("too deep", zst(&tar_of(&[("pack.toml", FILE, toml), ("a/b/c", FILE, b"x")]))), + ("duplicate", zst(&tar_of(&[("pack.toml", FILE, toml), ("pack.toml", FILE, toml)]))), + ("backslash", zst(&tar_of(&[("pack.toml", FILE, toml), ("..\\evil", FILE, b"x")]))), + ("no pack.toml", zst(&tar_of(&[("spelling.dic", FILE, b"1\nx\n")]))), + ("not zstd", tar_of(&[("pack.toml", FILE, toml)])), + ("not tar", zst(b"hello, this is not a tar archive at all")), + ("empty", Vec::new()), + ("truncated", zst(&tar_of(&[("pack.toml", FILE, toml), ("spelling.dic", FILE, &big)]))[..20].to_vec()), + ]; + for (what, archive) in cases { + let root = temp("hostile"); + let dest = root.join("nl"); + assert!(unpack(&archive, &dest).is_err(), "{what}"); + assert!(!root.join("evil").exists() && !root.parent().unwrap().join("evil").exists(), "{what}"); + } + // Too many entries. + let names: Vec = (0..=MAX_PACK_ENTRIES).map(|i| format!("f{i}")).collect(); + let mut files: Vec<(&str, tar::EntryType, &[u8])> = vec![("pack.toml", FILE, toml)]; + files.extend(names.iter().map(|n| (n.as_str(), FILE, &b""[..]))); + assert!(unpack(&zst(&tar_of(&files)), &temp("many").join("nl")).is_err()); + // A header claiming a file larger than the cap is refused before anything is read. + let mut t = tar::Header::new_gnu(); + t.set_path("pack.toml").unwrap(); + t.set_size(toml.len() as u64); + t.set_cksum(); + let mut b = tar::Builder::new(Vec::new()); + b.append(&t, toml).unwrap(); + let mut tarball = b.into_inner().unwrap(); + tarball.truncate(tarball.len() - 1024); // drop the end marker + let mut h = tar::Header::new_gnu(); + h.set_path("spelling.dic").unwrap(); + h.set_size(MAX_PACK_FILE_BYTES + 1); + h.set_cksum(); + tarball.extend_from_slice(h.as_bytes()); + tarball.extend_from_slice(&[0u8; 1024]); + let err = unpack(&zst(&tarball), &temp("huge").join("nl")).unwrap_err(); + assert!(err.contains("larger than"), "{err}"); + // A good one unpacks. + let dest = temp("good").join("nl"); + unpack(&pack_archive("nl", "x"), &dest).unwrap(); + assert!(dest.join("LICENCES/COPYING").is_file()); +} diff --git a/crates/font-store/src/lib.rs b/crates/font-store/src/lib.rs index bd79a3ab..3997a5d3 100644 --- a/crates/font-store/src/lib.rs +++ b/crates/font-store/src/lib.rs @@ -18,8 +18,9 @@ //! # } //! ``` //! -//! The index and its lookups, [`verify_font`] and [`IndexPin`] build on every target. The store -//! and the HTTP client are native only. +//! The index and its lookups, [`verify_font`] and [`IndexPin`] build on every target. The store, +//! the HTTP client and Language packs (the store's second item kind, `language-pack`: see +//! [`FontStore::install_language_pack`]) are native only. mod index; mod pin; @@ -30,6 +31,8 @@ mod verify; #[cfg(not(target_arch = "wasm32"))] mod http; #[cfg(not(target_arch = "wasm32"))] +mod language_packs; +#[cfg(not(target_arch = "wasm32"))] mod store; pub use index::{ @@ -43,6 +46,11 @@ pub use verify::{VerifyError, is_font, sha256_hex, verify_font}; #[cfg(not(target_arch = "wasm32"))] pub use http::{Cancel, Client, FetchError, Timeouts, USER_AGENT}; #[cfg(not(target_arch = "wasm32"))] +pub use language_packs::{ + InstalledLanguagePack, LANGUAGE_PACK_INDEX_ENV, LANGUAGE_PACKS, MAX_PACK_ARCHIVE_BYTES, MAX_PACK_ENTRIES, MAX_PACK_FILE_BYTES, + MAX_PACK_INDEX_BYTES, MAX_PACK_UNPACKED_BYTES, PackEntry, PackIndex, PackIndexPin, +}; +#[cfg(not(target_arch = "wasm32"))] pub use store::{Caller, Error, FontStore, InstalledFamily, InstalledFile, Network, Problem, Progress, UserSource}; #[cfg(test)] diff --git a/crates/font-store/src/store.rs b/crates/font-store/src/store.rs index 880d9857..e2cd1161 100644 --- a/crates/font-store/src/store.rs +++ b/crates/font-store/src/store.rs @@ -7,6 +7,8 @@ //! manifest.json what is installed //! indexes/ the last indexes fetched: sources.json, one file per source //! / google/fonts' folder name, or /: the fonts and the licence +//! language-packs/ installed Language packs, one folder per locale (`crate::language_packs`) +//! language-packs.json which Language packs are installed, from which index entry //! .staging/ downloads being installed; nothing a font scan picks up //! .lock held while the store changes //! ``` @@ -30,7 +32,7 @@ use crate::verify::{VerifyError, is_font, sha256_hex, verify_font}; const SETTINGS: &str = "settings.json"; const MANIFEST: &str = "manifest.json"; -const INDEX_CACHE: &str = "indexes"; +pub(crate) const INDEX_CACHE: &str = "indexes"; const SOURCES_CACHE: &str = "sources.json"; const STAGING: &str = ".staging"; const LOCK: &str = ".lock"; @@ -111,7 +113,7 @@ pub enum Error { }, } -fn io(what: impl Into) -> impl FnOnce(std::io::Error) -> Error { +pub(crate) fn io(what: impl Into) -> impl FnOnce(std::io::Error) -> Error { let what = what.into(); move |source| Error::Io { what, source } } @@ -336,7 +338,7 @@ impl FontStore { self.dir.join(INDEX_CACHE).join(format!("user-{key}")) } - fn require_network(&self) -> Result<(), Error> { + pub(crate) fn require_network(&self) -> Result<(), Error> { if self.network() == Network::On { Ok(()) } else { Err(Error::NetworkOff) } } @@ -545,7 +547,7 @@ impl FontStore { } /// Downloads from each URL in turn until one gives bytes that pass `check`. - fn fetch( + pub(crate) fn fetch( &self, urls: &[String], max: u64, @@ -712,7 +714,7 @@ impl FontStore { } } - fn lock(&self) -> Result { + pub(crate) fn lock(&self) -> Result { std::fs::create_dir_all(&self.dir).map_err(io("creating the font store folder"))?; let file = File::options().create(true).truncate(false).write(true).open(self.dir.join(LOCK)).map_err(io("locking the font store"))?; file.lock().map_err(io("locking the font store"))?; @@ -740,12 +742,12 @@ impl FontStore { self.write_file(MANIFEST, &bytes) } - fn write_file(&self, name: &str, bytes: &[u8]) -> Result<(), Error> { + pub(crate) fn write_file(&self, name: &str, bytes: &[u8]) -> Result<(), Error> { std::fs::create_dir_all(&self.dir).map_err(io("creating the font store folder"))?; craft_atomic::atomic_write(&self.dir.join(name), bytes).map_err(io(format!("saving {name}"))) } - fn sweep_staging(&self) { + pub(crate) fn sweep_staging(&self) { for e in std::fs::read_dir(self.dir.join(STAGING)).into_iter().flatten().flatten() { let old = e.metadata().and_then(|m| m.modified()).is_ok_and(|t| SystemTime::now().duration_since(t).is_ok_and(|d| d > STALE)); if old { @@ -818,7 +820,7 @@ fn user_index(bytes: &[u8], user: &UserSource) -> Result { Ok(index) } -fn read_limited(path: &Path, max: u64) -> Option> { +pub(crate) fn read_limited(path: &Path, max: u64) -> Option> { if std::fs::metadata(path).ok()?.len() > max { return None; } @@ -830,12 +832,12 @@ fn host(url: &str) -> &str { } /// A folder of downloads under `.staging`, deleted when dropped. -struct Stage { - dir: PathBuf, +pub(crate) struct Stage { + pub(crate) dir: PathBuf, } impl Stage { - fn new(store: &Path) -> Result { + pub(crate) fn new(store: &Path) -> Result { static N: AtomicU64 = AtomicU64::new(0); let dir = store.join(STAGING).join(format!("{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed))); std::fs::create_dir_all(&dir).map_err(io("creating a download folder"))?;