Merge remote-tracking branch 'upstream/main' into review-2070

# Conflicts:
#	Cargo.toml
This commit is contained in:
Dylan M. Taylor committed 2026-11-17 14:29:26 -05:00
commit deda581e53
35 files changed
+4313 -25

No files matched your search

+9
View File
@@ -119,6 +119,7 @@ The XML reader in `crates/plist/src/xml.rs` and its base64 compatibility helper
`crates/palette` extracts the GPL text parser from AniCraft `crates/doc/src/swatch.rs` at `d875a7feb250e6ab06aa28a37fb78a0699e88330` (MIT OR Apache-2.0), introduced by Tanish Manku in `a64130a9da9bdde545463b852d95d6a6774b1dfd` (AniCraft #270). The import preserves Tanish as primary author. Neutral RGB/name adaptation separates application display-name sanitization and the `Untitled` policy; validation tests and integration are subsequent Dylan M. Taylor changes.
The legacy ZIP Zstandard method-20 mapping and bounded-read regression in `crates/zip/src/lenient` are ported from RRFZ's Craft Anything commit `29badae7c9280fdf4c24795927bf3a91f51694c6` (MIT OR Apache-2.0). RRFZ remains the primary Git author. Firmware scanner detection remains app-owned and is not part of this port.
## TAR/CPIO compatibility updates
The old GNU sparse-extension traversal, GNU CRC-symlink compatibility and PAX-size precedence fixes and their regression tests in `crates/tar` are ported from RRFZ's Craft Anything PRs #1973 (`8ea2ec66a1bc80f0420f3695baac877a0cb9b9a5`), #1984 (`b226a475f7dce9710e3af7a069386fe7a9720df7`) and #2007 (`d29465c20b0d570ea4973be512a0cb4e212bb60f`). Source is MIT OR Apache-2.0; RRFZ remains the primary Git author of each port. The PAX update adapts to the shared parser's existing `PaxOverrides` structure without changing the source policy.
@@ -130,14 +131,22 @@ The USDZ alignment requirement and compatibility fixture for `Writer::add_stored
The raw gradient wire ordering in `craft-lottie::Gradient::flat_values` follows
AniCraft's Lottie exporter by Tanish Manku (`63b39355`, `b9258b7b`), MIT OR Apache-2.0.
Sanitization, animation fitting and numeric output policy remain caller-owned.
## Screen Video key frames
`crates/video/src/screen_video.rs` extracts AniCraft `crates/xfl/src/video.rs` codec-3 decoding and its synthetic partial-block fixture, introduced by Tanish Manku in `1f3e603b7ff53a3da364e41ec4da8d43dc83aab0` with pixel-loop cleanup `2b7571363b4b647266df758f95b6e397021cfd4d` (MIT OR Apache-2.0). Import commits preserve Tanish as primary author. Caller-owned bounds and neutral RGBA output separate codec work from the consuming app's FLV first-frame/report policy. Boundary tests and integration are subsequent Dylan M. Taylor changes.
## Source Map v3 parser and lookup
`crates/sourcemap` extracts the bounded parser, source inventory and generated-position lookup from Craft Anything `crates/formats/src/sourcemap.rs` at `e5c9c94106747ed44b9a3fb0decce867c2285f93` (MIT OR Apache-2.0). Core-history ports preserve Brandon Thomas, BFlat, Afloat, RRFZ, Nicolas Bohn and Alex Joaquim Pereira as primary authors, with exact source revisions in each commit. Six regression files preserve their source authors; nine core unit tests retain the original tests. App annotation scanning, data URLs, base64 and file I/O are intentionally not imported. Workspace wiring is a subsequent change; existing limits and errors are unchanged.
## JSONC span-preserving editing
`crates/jsonc` is extracted from Craft Anything `crates/formats/src/jsonc.rs` at `e5c9c94106747ed44b9a3fb0decce867c2285f93` (MIT OR Apache-2.0). The original implementation and hardening by BFlat, newline preservation by NetterSilver, and subsequent RRFZ/Afloat correctness fixes retain separate primary-author port commits with immutable source revisions. Shared API wiring and optional input-byte admission are subsequent changes. Compatibility parsing and all existing formatting/editing tests are retained; application file I/O and policy are not imported.
## WebAssembly inspection
`crates/wasm` extracts Craft Anything `crates/formats/src/wasm.rs` at `5979f9b13f16dcf1df3c7ae7f8e7735f452866f2`, with its checked byte-reader subset and original synthetic WASM/testkit fixtures (MIT OR Apache-2.0). Eight core-history ports preserve Brandon Thomas, Afloat and RRFZ as primary authors; component, repeated-custom-section and referenced-string regression imports retain source authors and commit references. Fixtures come from `crates/testkit/src/formats.rs` and deterministic mutation helpers from `crates/testkit/src/lib.rs`; no third-party binaries. Shared error naming and workspace wiring are subsequent changes. Existing inspection limits and report behavior are unchanged; this crate validates/inspects/renders, never executes modules.
## Packet capture and protocol inspection
`crates/pcap` extracts Craft Anything `crates/formats/src/pcap.rs`, its checked byte reader primitives and synthetic packet fixtures/regressions at `5979f9b13f16dcf1df3c7ae7f8e7735f452866f2` (MIT OR Apache-2.0). Brandon Thomas introduced the implementation, byte reader and fixtures in `f64ae3bf3c8863a982af1304e06de20653803968`. Subsequent fixes by phatloi0010, GGWA, BFlat, Afloat, NetterSilver, Max Mittelstädt and RRFZ retain separate primary-author source-history commits with immutable source revisions. Shared error/API wiring and chrono-based timestamp formatting with exhaustive day-boundary parity tests are subsequent Dylan M. Taylor changes. No packet reassembly, decryption, live capture or app analysis policy is imported.
Generated
+42 -1
View File
@@ -3113,6 +3113,15 @@ dependencies = [
name = "craft-sniff"
version = "0.1.0"
[[package]]
name = "craft-sourcemap"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-ssip"
version = "0.1.0"
@@ -3368,6 +3377,16 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-wasm"
version = "0.1.0"
dependencies = [
"serde",
"thiserror 2.0.21",
"wasmparser 0.261.0",
"wasmprinter",
]
[[package]]
name = "craft-web-api"
version = "0.1.0"
@@ -12051,7 +12070,7 @@ dependencies = [
"wasmi_collections",
"wasmi_core",
"wasmi_ir",
"wasmparser",
"wasmparser 0.228.0",
]
[[package]]
@@ -12091,6 +12110,28 @@ dependencies = [
"indexmap",
]
[[package]]
name = "wasmparser"
version = "0.261.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f20f20e44f7e8aeb6744823ea9d869ede51e51be4fdaedede2852282e54d2d8"
dependencies = [
"bitflags 2.13.2",
"indexmap",
"semver",
]
[[package]]
name = "wasmprinter"
version = "0.261.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86a21c9e3325655ba02de3e645bc8357a8b3651a2e3ada25b6dbc37560deb9b7"
dependencies = [
"anyhow",
"termcolor",
"wasmparser 0.261.0",
]
[[package]]
name = "wayland-backend"
version = "0.3.17"
+2
View File
@@ -12,6 +12,7 @@ repository = "https://github.com/storytold/craft-libs"
[workspace.dependencies]
craft-evm = { path = "crates/evm" }
craft-sourcemap = { path = "crates/sourcemap" }
craft-pcap = { path = "crates/pcap" }
craft-har = { path = "crates/har" }
craft-jsonc = { path = "crates/jsonc" }
@@ -29,6 +30,7 @@ acadrust = "=0.6.6"
log = "0.4"
# internal: add each crate here as `craft-<name> = { path = "crates/<name>" }`
craft-plist = { path = "crates/plist" }
craft-wasm = { path = "crates/wasm" }
craft-platform = { path = "crates/platform" }
craft-aep = { path = "crates/aep" }
craft-affinity = { path = "crates/affinity" }
+6
View File
@@ -81,6 +81,12 @@ Buffers are checked once per call, before any pixel is written.
the exact and float paths compare each pixel with white; all other colours are left as the
profiles give them. With a profile whose white falls between its nodes (`coated-cmyk`: 0.4 %
C), the last step to white is that much larger than its neighbours.
* Black preservation (opt-in, `BlackPreserving`): for CMYK → CMYK, a colour with exactly zero
C, M and Y comes out as exactly `(0, 0, 0, K')`, where `K'` is the destination black whose
K-only L* equals the source's (both K ramps sampled at 1024 points when the transform is
built, with its intent and BPC). The test is exact zero, so an input with any chromatic ink
takes the ordinary path. Absolute colorimetric, non-CMYK profiles and K ramps that reverse at
one of the 1024 samples are refused (a reversal between samples is not detected).
## Built-in profiles (all CC0-1.0, generated by this crate)
Binary file not shown.
+205
View File
@@ -0,0 +1,205 @@
//! K-only black preservation for CMYK → CMYK transforms.
//!
//! A CMYK → CMYK transform goes through the three-component PCS, so black alone (C = M = Y = 0)
//! comes out as a four-ink mix. ISO 32000-2 §8.6.5.7 notes that this loses fidelity in the black
//! component. ICC defines no black preservation; this is a named policy the caller opts into.
//!
//! [`BlackPreserving`] converts like its [`Transform`], except that a colour with exactly zero
//! cyan, magenta and yellow becomes exactly `(0, 0, 0, K')`. `K'` is the destination black whose
//! K-only L* equals the source's, both measured to Lab with the transform's own intent and BPC
//! setting: both ramps are sampled once, at construction, and the destination ramp is inverted.
//! A source black darker than the destination's maximum black maps to K = 1. The test is exact zero, so the transform is discontinuous at C = M = Y = 0: an input
//! with any chromatic ink takes the ordinary path.
use crate::transform::{Transform, TransformOptions};
use crate::{Builtin, CmsError, ColorSpace, Intent, Profile, invalid_arg};
/// Samples along each K-only ramp. Linear interpolation between samples of a smooth ramp is far
/// below the 16-bit quantum.
const K_SAMPLES: usize = 1024;
/// Slack for the monotonicity checks (L* units, K units).
const MONOTONIC_EPS: f64 = 1e-6;
/// A CMYK → CMYK transform that keeps black-only colours black-only.
#[derive(Debug)]
pub struct BlackPreserving {
transform: Transform,
/// Destination K for source K at `i / (K_SAMPLES - 1)`; `K_SAMPLES` values in `[0, 1]`.
curve: Vec<f32>,
/// Source K from which the source black is at least as dark as the destination's darkest:
/// mapped to exactly 1. Infinite when the destination's black is the darker.
saturate: f32,
/// The mapping at the 256 8-bit source codes, already quantised.
curve8: [u8; 256],
}
impl BlackPreserving {
/// Transform from `src` to `dst` with black preservation.
///
/// Errors with [`CmsError::InvalidArgument`] when either profile is not CMYK or the intent is
/// absolute colorimetric (which reproduces the source's paper and black, so there is no
/// black to map), and with [`CmsError::Unsupported`] when a profile's K-only ramp is not
/// monotonic at its 1024 samples (black that gets lighter as K rises has no equal-lightness
/// inverse; a reversal between samples is not detected). Other errors
/// are those of [`Transform::with_options`].
pub fn new(src: &Profile, dst: &Profile, opts: TransformOptions) -> Result<BlackPreserving, CmsError> {
if src.color_space != ColorSpace::Cmyk || dst.color_space != ColorSpace::Cmyk {
return Err(invalid_arg("black preservation needs CMYK source and destination profiles"));
}
if opts.intent == Intent::AbsoluteColorimetric {
return Err(invalid_arg("black preservation does not apply to absolute colorimetric"));
}
let transform = Transform::with_options(src, dst, opts)?;
let src_l = k_ramp_lightness(src, opts)?;
let dst_l = k_ramp_lightness(dst, opts)?;
let (curve, saturate) = k_curve(&src_l, &dst_l)?;
let mut curve8 = [0u8; 256];
for (code, slot) in curve8.iter_mut().enumerate() {
*slot = quantise(map(&curve, saturate, code as f32 / 255.0), 255.0) as u8;
}
Ok(BlackPreserving { transform, curve, saturate, curve8 })
}
/// The ordinary transform, which every colour with chromatic ink goes through. Converting with
/// it directly does not preserve black.
pub fn transform(&self) -> &Transform {
&self.transform
}
/// Destination K for a black-only source colour of `k` (clamped to `[0, 1]`).
pub fn map_k(&self, k: f32) -> f32 {
map(&self.curve, self.saturate, k)
}
/// Exact evaluation of one colour; slices as in [`Transform::eval`].
pub fn eval(&self, input: &[f32], out: &mut [f32]) -> Result<(), CmsError> {
self.transform.eval(input, out)?;
self.patch_one(input, out);
Ok(())
}
/// Lookup-table evaluation of one colour; slices as in [`Transform::eval`].
pub fn eval_fast(&self, input: &[f32], out: &mut [f32]) -> Result<(), CmsError> {
self.transform.eval_fast(input, out)?;
self.patch_one(input, out);
Ok(())
}
/// Converts float pixels; layout and errors as in [`Transform::convert_f32`].
pub fn convert_f32(&self, src: &[f32], src_stride: usize, dst: &mut [f32], dst_stride: usize, copy_extra: bool) -> Result<(), CmsError> {
self.transform.convert_f32(src, src_stride, dst, dst_stride, copy_extra)?;
for (s, d) in src.chunks_exact(src_stride).zip(dst.chunks_exact_mut(dst_stride)) {
self.patch_one(s, d);
}
Ok(())
}
/// Converts 8-bit pixels; layout and errors as in [`Transform::convert_u8`].
pub fn convert_u8(&self, src: &[u8], src_stride: usize, dst: &mut [u8], dst_stride: usize, copy_extra: bool) -> Result<(), CmsError> {
self.transform.convert_u8(src, src_stride, dst, dst_stride, copy_extra)?;
for (s, d) in src.chunks_exact(src_stride).zip(dst.chunks_exact_mut(dst_stride)) {
if let ([0, 0, 0, k, ..], [c, m, y, kd, ..]) = (s, d) {
(*c, *m, *y) = (0, 0, 0);
*kd = self.curve8[usize::from(*k)];
}
}
Ok(())
}
/// Converts 16-bit pixels; layout and errors as in [`Transform::convert_u16`].
pub fn convert_u16(&self, src: &[u16], src_stride: usize, dst: &mut [u16], dst_stride: usize, copy_extra: bool) -> Result<(), CmsError> {
self.transform.convert_u16(src, src_stride, dst, dst_stride, copy_extra)?;
for (s, d) in src.chunks_exact(src_stride).zip(dst.chunks_exact_mut(dst_stride)) {
if let ([0, 0, 0, k, ..], [c, m, y, kd, ..]) = (s, d) {
(*c, *m, *y) = (0, 0, 0);
*kd = quantise(self.map_k(f32::from(*k) / 65535.0), 65535.0) as u16;
}
}
Ok(())
}
fn patch_one(&self, input: &[f32], out: &mut [f32]) {
if let ([0.0, 0.0, 0.0, k, ..], [c, m, y, kd, ..]) = (input, out) {
(*c, *m, *y) = (0.0, 0.0, 0.0);
*kd = self.map_k(*k);
}
}
}
/// The sampled mapping from the source to the destination K-only ramp, and the source K at
/// which the destination's darkest black is reached (the `saturate` field).
fn k_curve(src_l: &[f64], dst_l: &[f64]) -> Result<(Vec<f32>, f32), CmsError> {
let curve: Vec<f32> = src_l.iter().map(|&l| invert(dst_l, l) as f32).collect();
if !curve.windows(2).all(|w| f64::from(w[0]) - f64::from(w[1]) <= MONOTONIC_EPS) {
return Err(CmsError::Unsupported("black preservation: the K mapping is not monotonic".into()));
}
let saturate = match (src_l.last(), dst_l.last()) {
(Some(&s), Some(&d)) if s < d => invert(src_l, d) as f32,
_ => f32::INFINITY,
};
Ok((curve, saturate))
}
/// L* of `p`'s K-only ramp at `K_SAMPLES` points, light to dark; refused when it is not
/// non-increasing.
fn k_ramp_lightness(p: &Profile, opts: TransformOptions) -> Result<Vec<f64>, CmsError> {
let to_lab = Transform::with_options(p, Builtin::LabD50.profile(), TransformOptions { precise_float: true, ..opts })?;
let mut ramp = Vec::with_capacity(K_SAMPLES);
let mut lab = [0.0f32; 3];
for i in 0..K_SAMPLES {
let k = i as f32 / (K_SAMPLES - 1) as f32;
to_lab.eval(&[0.0, 0.0, 0.0, k], &mut lab)?;
ramp.push(f64::from(lab[0]) * 100.0);
}
if !ramp.iter().all(|l| l.is_finite()) || !ramp.windows(2).all(|w| w[1] - w[0] <= MONOTONIC_EPS) {
return Err(CmsError::Unsupported("black preservation: a K-only ramp is not monotonic".into()));
}
Ok(ramp)
}
/// The K in `[0, 1]` at which the non-increasing `ramp` reaches `target`, clamped to its ends.
fn invert(ramp: &[f64], target: f64) -> f64 {
let (Some(&first), Some(&last)) = (ramp.first(), ramp.last()) else { return 0.0 };
if ramp.len() < 2 || target >= first {
return 0.0;
}
if target <= last {
return 1.0;
}
// ramp[lo] > target >= ramp[hi]
let (mut lo, mut hi) = (0usize, ramp.len() - 1);
while hi - lo > 1 {
let mid = lo + (hi - lo) / 2;
if ramp.get(mid).is_some_and(|&v| v > target) {
lo = mid;
} else {
hi = mid;
}
}
let (a, b) = (ramp.get(lo).copied().unwrap_or(first), ramp.get(hi).copied().unwrap_or(last));
let t = if a - b > MONOTONIC_EPS { (a - target) / (a - b) } else { 0.0 };
(lo as f64 + t) / (ramp.len() - 1) as f64
}
/// Linear interpolation in `curve` (samples over `[0, 1]`) at `k`, clamped; 1 from `saturate`.
fn map(curve: &[f32], saturate: f32, k: f32) -> f32 {
let Some(&last) = curve.last() else { return 0.0 };
let k = if k.is_nan() { 0.0 } else { k.clamp(0.0, 1.0) };
if k >= saturate {
return 1.0;
}
let x = k * (curve.len() - 1) as f32;
let i = x.floor() as usize;
match (curve.get(i), curve.get(i + 1)) {
(Some(&a), Some(&b)) => a + (b - a) * (x - i as f32),
_ => last,
}
}
fn quantise(v: f32, max: f32) -> f32 {
(v.clamp(0.0, 1.0) * max + 0.5).floor()
}
#[cfg(test)]
#[path = "black_tests.rs"]
mod tests;
+149
View File
@@ -0,0 +1,149 @@
use super::*;
fn coated() -> &'static Profile {
Builtin::CoatedCmyk.profile()
}
fn fogra() -> &'static Profile {
Builtin::CoatedFogra39.profile()
}
fn opts() -> TransformOptions {
TransformOptions::default()
}
/// L* of a CMYK colour under `p`, measured as the transform with `o` measures it.
fn lightness(p: &Profile, o: TransformOptions, cmyk: [f32; 4]) -> f32 {
let t = Transform::with_options(p, Builtin::LabD50.profile(), o).unwrap();
let mut lab = [0.0f32; 3];
t.eval(&cmyk, &mut lab).unwrap();
lab[0] * 100.0
}
#[test]
fn the_plain_transform_turns_black_into_four_inks() {
// What the policy is for: without it, black alone gains chromatic ink across presses.
let t = Transform::with_options(coated(), fogra(), opts()).unwrap();
let mut out = [0.0f32; 4];
t.eval(&[0.0, 0.0, 0.0, 0.5], &mut out).unwrap();
assert!(out[..3].iter().any(|v| *v > 0.01), "{out:?}");
}
#[test]
fn black_only_stays_black_only_on_every_path() {
for (src, dst) in [(coated(), fogra()), (fogra(), coated()), (fogra(), fogra())] {
let bp = BlackPreserving::new(src, dst, opts()).unwrap();
for k in [0.0f32, 0.1, 0.5, 0.9, 1.0] {
for fast in [false, true] {
let mut out = [1.0f32; 4];
if fast { bp.eval_fast(&[0.0, 0.0, 0.0, k], &mut out) } else { bp.eval(&[0.0, 0.0, 0.0, k], &mut out) }.unwrap();
assert_eq!(out[..3], [0.0; 3], "k {k} fast {fast}");
assert_eq!(out[3], bp.map_k(k));
}
}
let src8: Vec<u8> = (0..=255u8).flat_map(|k| [0, 0, 0, k, 7]).collect();
let mut dst8 = vec![0u8; src8.len()];
bp.convert_u8(&src8, 5, &mut dst8, 5, true).unwrap();
for (k, px) in dst8.as_chunks::<5>().0.iter().enumerate() {
assert_eq!(px[..3], [0; 3], "u8 k {k}");
assert!((f32::from(px[3]) - bp.map_k(k as f32 / 255.0) * 255.0).abs() <= 0.5, "u8 k {k}: {}", px[3]);
assert_eq!(px[4], 7, "extra channel copied");
}
let src16: Vec<u16> = (0..=64u16).flat_map(|k| [0, 0, 0, k * 1023]).collect();
let mut dst16 = vec![0u16; src16.len()];
bp.convert_u16(&src16, 4, &mut dst16, 4, false).unwrap();
for (px, k) in dst16.as_chunks::<4>().0.iter().zip(src16.as_chunks::<4>().0) {
assert_eq!(px[..3], [0; 3], "u16 {k:?}");
assert!((f32::from(px[3]) - bp.map_k(f32::from(k[3]) / 65535.0) * 65535.0).abs() <= 0.5, "u16 {k:?}: {}", px[3]);
}
let srcf: Vec<f32> = (0..=10).flat_map(|k| [0.0, 0.0, 0.0, k as f32 / 10.0]).collect();
let mut dstf = vec![0.0f32; srcf.len()];
bp.convert_f32(&srcf, 4, &mut dstf, 4, false).unwrap();
for (px, k) in dstf.as_chunks::<4>().0.iter().zip(srcf.as_chunks::<4>().0) {
assert_eq!(*px, [0.0, 0.0, 0.0, bp.map_k(k[3])], "f32 {k:?}");
}
}
}
#[test]
fn the_destination_black_matches_the_source_lightness() {
// Measured worst error over 201 K steps, both directions, both intents, BPC on and off:
// 0.007 L*.
for intent in [Intent::RelativeColorimetric, Intent::Perceptual] {
for bpc in [true, false] {
let o = TransformOptions { intent, bpc, ..opts() };
for (src, dst) in [(coated(), fogra()), (fogra(), coated())] {
let bp = BlackPreserving::new(src, dst, o).unwrap();
let dst_black = lightness(dst, o, [0.0, 0.0, 0.0, 1.0]);
for k in [0.1f32, 0.3, 0.5, 0.7, 0.9, 1.0] {
let want = lightness(src, o, [0.0, 0.0, 0.0, k]);
if want <= dst_black {
assert_eq!(bp.map_k(k), 1.0, "darker than the destination's black clamps to K = 1");
continue;
}
let got = lightness(dst, o, [0.0, 0.0, 0.0, bp.map_k(k)]);
assert!((got - want).abs() < 0.02, "{intent:?} bpc {bpc} k {k}: L* {got} for {want}");
}
assert_eq!(bp.map_k(0.0), 0.0, "paper stays paper");
assert!(bp.map_k(1.0) > 0.95, "full black stays near full black: {}", bp.map_k(1.0));
assert!(bp.map_k(0.25) < bp.map_k(0.75));
}
}
}
let same = BlackPreserving::new(fogra(), fogra(), opts()).unwrap();
for k in [0.0f32, 0.2, 0.6, 1.0] {
assert!((same.map_k(k) - k).abs() < 1e-3, "{k} -> {}", same.map_k(k));
}
}
#[test]
fn colours_with_any_chromatic_ink_take_the_ordinary_path() {
let bp = BlackPreserving::new(coated(), fogra(), opts()).unwrap();
for input in [[1e-6f32, 0.0, 0.0, 0.5], [0.0, 0.2, 0.0, 0.5], [0.3, 0.3, 0.3, 0.0], [0.0, 0.0, 1.0, 1.0]] {
let (mut a, mut b) = ([0.0f32; 4], [0.0f32; 4]);
bp.eval(&input, &mut a).unwrap();
bp.transform().eval(&input, &mut b).unwrap();
assert_eq!(a, b, "{input:?}");
}
let src = [1u8, 0, 0, 128, 0, 1, 0, 128];
let (mut a, mut b) = ([0u8; 8], [0u8; 8]);
bp.convert_u8(&src, 4, &mut a, 4, false).unwrap();
bp.transform().convert_u8(&src, 4, &mut b, 4, false).unwrap();
assert_eq!(a, b);
}
#[test]
fn refusals_and_hostile_values() {
let srgb = Builtin::Srgb.profile();
assert!(matches!(BlackPreserving::new(srgb, fogra(), opts()), Err(CmsError::InvalidArgument(_))));
assert!(matches!(BlackPreserving::new(fogra(), srgb, opts()), Err(CmsError::InvalidArgument(_))));
let abs = TransformOptions { intent: Intent::AbsoluteColorimetric, ..opts() };
assert!(matches!(BlackPreserving::new(fogra(), coated(), abs), Err(CmsError::InvalidArgument(_))));
let bp = BlackPreserving::new(coated(), fogra(), opts()).unwrap();
for k in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY, -1.0, 2.0] {
assert!((0.0..=1.0).contains(&bp.map_k(k)), "{k}");
}
let mut out = [0.0f32; 4];
assert!(bp.eval(&[0.0; 3], &mut out).is_err(), "short input");
assert!(bp.convert_u8(&[0, 0, 0, 9, 0], 4, &mut [0u8; 4], 4, false).is_err(), "partial pixel");
assert_eq!(invert(&[], 50.0), 0.0);
assert_eq!(invert(&[100.0], 50.0), 0.0);
assert_eq!(invert(&[100.0, 0.0], 150.0), 0.0);
assert_eq!(invert(&[100.0, 0.0], -5.0), 1.0);
assert!((invert(&[100.0, 50.0, 0.0], 25.0) - 0.75).abs() < 1e-12);
assert_eq!(map(&[], f32::INFINITY, 0.5), 0.0);
}
#[test]
fn black_beyond_the_destination_maximum_maps_to_full_black_between_samples() {
// Source L* = 100 (1 - K), destination L* = 100 - 50 K: from K = 0.5 the source is darker
// than any destination black, and 0.5 falls between the 1024 samples.
let ramp = |f: fn(f64) -> f64| (0..K_SAMPLES).map(|i| f(i as f64 / (K_SAMPLES - 1) as f64)).collect::<Vec<_>>();
let (src, dst) = (ramp(|k| 100.0 * (1.0 - k)), ramp(|k| 100.0 - 50.0 * k));
let (curve, saturate) = k_curve(&src, &dst).unwrap();
assert!((saturate - 0.5).abs() < 1e-6, "{saturate}");
for k in [0.5001f32, 0.6, 1.0] {
assert_eq!(map(&curve, saturate, k), 1.0, "{k}");
}
assert!((map(&curve, saturate, 0.25) - 0.5).abs() < 1e-4);
let (_, never) = k_curve(&dst, &src).unwrap();
assert_eq!(never, f32::INFINITY, "a lighter source never saturates");
}
+2
View File
@@ -19,6 +19,7 @@
//! CIE 15, and Adobe's published black point compensation paper.
#![forbid(unsafe_code)]
pub mod black;
pub mod builtin;
mod builtin_v2;
pub mod clut;
@@ -39,6 +40,7 @@ pub mod transform;
mod write;
pub mod write_v2;
pub use black::BlackPreserving;
pub use builtin::Builtin;
pub use clut::Clut;
pub use curve::Curve;
+322 -15
View File
@@ -19,6 +19,9 @@
//! L* is scaled onto the device black, chroma is softly compressed against a gamut boundary
//! descriptor sampled from the forward model — then solved as above.
//!
//! That is [`GamutMapping::Legacy`]. [`GamutMapping::Oklab`] maps out-of-gamut colours in Oklab
//! instead and gives saturation its own table; the Coated FOGRA39 built-in uses it.
//!
//! The model is plausible for coated offset, not a measured characterisation: use a real
//! profile from your print provider for production print.
//!
@@ -49,6 +52,11 @@ pub struct CmykParams {
pub k_max: f64,
/// Exponent of the black generation curve above `gcr_start`.
pub gcr_gamma: f64,
/// Black generation as a curve of `[gray component, black]` points with increasing gray
/// components in `[0, 1]`, interpolated linearly and held flat beyond the ends. Empty by
/// default: black is then `k_max` times the ramp of `gcr_start` and `gcr_gamma` (the curve
/// of the shipped Photocraft profile).
pub gcr_curve: Vec<[f64; 2]>,
/// Tone value increase at 50 % for C, M, Y, K.
pub tvi: [f64; 4],
/// Grid points of the AToB (4D) and BToA (3D) tables.
@@ -60,6 +68,32 @@ pub struct CmykParams {
/// encoding and separates to a trace of ink. Off by default, which keeps the bytes of the
/// shipped Photocraft profile.
pub white_on_node: bool,
/// How the BToA tables bring colours into the gamut. [`GamutMapping::Legacy`] by default,
/// which keeps the bytes of the shipped Photocraft profile.
pub mapping: GamutMapping,
}
/// How the BToA tables map colours the printer cannot reach.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum GamutMapping {
/// The shipped Photocraft profile: out-of-gamut colours land on the closest reachable
/// colour (ΔE76); perceptual scales L* onto the device black and compresses Lab chroma;
/// saturation shares the perceptual table.
Legacy,
/// Gamut mapping in Oklab, whose hue lines stay perceptually constant (a CIELAB hue line
/// from a saturated blue toward grey drifts toward purple; an Oklab one does not):
///
/// * **Colorimetric**: in-gamut colours separate exactly. An out-of-gamut colour takes the
/// closest colour the printer reaches in Oklab, lightness and hue weighted above chroma,
/// and more so the further outside it is; black is generated from the clipped colour, so
/// inks the clip drives to 100 % stay there.
/// * **Perceptual** and **saturation**: the ICC v4 perceptual reference black is mapped
/// onto the device black linearly in XYZ (as black point compensation does) for neutrals,
/// fading out with Oklab chroma, so greys separate as with colorimetric + BPC and colourful
/// colours keep their lightness. Perceptual then compresses Oklab chroma above `knee` ×
/// the gamut boundary into the boundary; saturation scales Oklab chroma by `saturation`
/// and clips as colorimetric does.
Oklab { knee: f64, saturation: f64 },
}
impl Default for CmykParams {
@@ -71,10 +105,12 @@ impl Default for CmykParams {
gcr_start: 0.2,
k_max: 0.95,
gcr_gamma: 1.1,
gcr_curve: Vec::new(),
tvi: [0.14, 0.14, 0.14, 0.17],
grid_a2b: 11,
grid_b2a: 21,
white_on_node: false,
mapping: GamutMapping::Legacy,
}
}
}
@@ -183,6 +219,11 @@ impl Forward for TableForward {
/// Bounded least-squares solve for C, M, Y given K, minimising ΔE76 to `target`, subject to
/// `C + M + Y ≤ limit` (projected Levenberg–Marquardt with a numerical Jacobian).
fn solve_cmy<F: Forward + ?Sized>(model: &F, target: [f64; 3], k: f64, limit: f64, seeds: &[[f64; 3]]) -> ([f64; 3], f64) {
solve_cmy_by(model, k, limit, seeds, |l| [l[0] - target[0], l[1] - target[1], l[2] - target[2]])
}
/// [`solve_cmy`] minimising the norm of `residual(lab)` instead of ΔE76.
fn solve_cmy_by<F: Forward + ?Sized>(model: &F, k: f64, limit: f64, seeds: &[[f64; 3]], residual: impl Fn([f64; 3]) -> [f64; 3]) -> ([f64; 3], f64) {
let project = |mut x: [f64; 3]| {
for _ in 0..4 {
for v in &mut x {
@@ -202,10 +243,7 @@ fn solve_cmy<F: Forward + ?Sized>(model: &F, target: [f64; 3], k: f64, limit: f6
}
x
};
let err = |x: [f64; 3]| {
let l = model.lab([x[0], x[1], x[2], k]);
[l[0] - target[0], l[1] - target[1], l[2] - target[2]]
};
let err = |x: [f64; 3]| residual(model.lab([x[0], x[1], x[2], k]));
let norm2 = |e: [f64; 3]| e[0] * e[0] + e[1] * e[1] + e[2] * e[2];
let mut best = ([0.0; 3], f64::INFINITY);
for seed in seeds {
@@ -275,13 +313,32 @@ fn solve_cmy<F: Forward + ?Sized>(model: &F, target: [f64; 3], k: f64, limit: f6
const SEEDS: [[f64; 3]; 5] = [[0.3, 0.3, 0.3], [0.05, 0.05, 0.05], [0.8, 0.8, 0.8], [0.9, 0.2, 0.2], [0.2, 0.9, 0.9]];
/// Black for a gray component `g` (the smallest of C, M, Y without black).
fn gcr(p: &CmykParams, g: f64) -> f64 {
let Some(first) = p.gcr_curve.first() else {
// The Photocraft profile's curve.
return if g <= p.gcr_start { 0.0 } else { p.k_max * ((g - p.gcr_start) / (1.0 - p.gcr_start)).powf(p.gcr_gamma) };
};
let mut k = first[1];
for w in p.gcr_curve.windows(2) {
let ([g0, k0], [g1, k1]) = (w[0], w[1]);
if g >= g1 {
k = k1;
} else if g > g0 {
k = k0 + (k1 - k0) * (g - g0) / (g1 - g0);
break;
}
}
k.clamp(0.0, 1.0)
}
/// Colorimetric separation of a relative Lab target: GCR black generation then a CMY solve.
pub fn separate<F: Forward + ?Sized>(model: &F, p: &CmykParams, target: [f64; 3], warm: Option<[f64; 4]>) -> ([f64; 4], f64) {
let mut seeds: Vec<[f64; 3]> = warm.map(|w| vec![[w[0], w[1], w[2]]]).unwrap_or_default();
seeds.extend_from_slice(&SEEDS);
let (cmy0, _) = solve_cmy(model, target, 0.0, 3.0, &seeds);
let g = cmy0[0].min(cmy0[1]).min(cmy0[2]);
let k = if g <= p.gcr_start { 0.0 } else { p.k_max * ((g - p.gcr_start) / (1.0 - p.gcr_start)).powf(p.gcr_gamma) };
let k = gcr(p, g);
let mut seeds2 = vec![[(cmy0[0] - k).max(0.0), (cmy0[1] - k).max(0.0), (cmy0[2] - k).max(0.0)], cmy0];
seeds2.extend_from_slice(&seeds);
let (cmy, de) = solve_cmy(model, target, k, (p.tac - k).max(0.0), &seeds2);
@@ -383,6 +440,187 @@ impl Gbd {
}
}
/// ΔE76 under which a separation counts as reaching its target.
const REACH: f64 = 1.5;
/// Oklab chroma beyond any printable colour: the top of the boundary search.
const OKLAB_CHROMA_MAX: f64 = 0.5;
/// Oklab chroma above which the perceptual and saturation tables no longer lift black: they lift
/// neutrals fully, and less the more colourful a colour is.
const LIFT_CHROMA: f64 = 0.1;
/// Weight of lightness and hue against chroma when a colour just outside the gamut is clipped.
const CLIP_WEIGHT: f64 = 1.5;
/// Growth of that weight with the squared share of chroma the clip loses.
const CLIP_WEIGHT_GROWTH: f64 = 200.0;
// Oklab (Björn Ottosson, 2020): XYZ (D65) → cone response → cube root → Lab-like axes.
const OKLAB_M1: math::Mat3 =
[[0.8189330101, 0.3618667424, -0.1288597137], [0.0329845436, 0.9293118715, 0.0361456387], [0.0482003018, 0.2643662691, 0.6338517070]];
const OKLAB_M2: math::Mat3 =
[[0.2104542553, 0.7936177850, -0.0040720468], [1.9779984951, -2.4285922050, 0.4505937099], [0.0259040371, 0.7827717662, -0.8086757660]];
const OKLAB_M1_INV: math::Mat3 =
[[1.2270138511, -0.5577999807, 0.2812561490], [-0.0405801784, 1.1122568696, -0.0716766787], [-0.0763812845, -0.4214819784, 1.5861632204]];
const OKLAB_M2_INV: math::Mat3 = [[1.0, 0.3963377774, 0.2158037573], [1.0, -0.1055613458, -0.0638541728], [1.0, -0.0894841775, -1.2914855480]];
/// The [`GamutMapping::Oklab`] separations of one forward model.
struct OklabMapper<'a, F: Forward + ?Sized> {
model: &'a F,
p: &'a CmykParams,
knee: f64,
saturation: f64,
/// XYZ scale that maps the v4 perceptual reference black onto the device black (the white
/// stays fixed): `v' = a·v + w·(1 − a)`.
black_scale: f64,
to_d65: math::Mat3,
to_d50: math::Mat3,
}
impl<'a, F: Forward + ?Sized> OklabMapper<'a, F> {
fn new(model: &'a F, p: &'a CmykParams, knee: f64, saturation: f64) -> Self {
let d65 = math::xy_to_xyz(math::D65_XY);
// The darkest neutral the printer reaches.
let (black, _) = separate(model, p, [0.0, 0.0, 0.0], None);
let device = math::lab_to_xyz([model.lab(black)[0], 0.0, 0.0], math::D50)[1];
let reference = crate::transform::PERCEPTUAL_BLACK[1];
OklabMapper {
model,
p,
knee,
saturation,
black_scale: (1.0 - device) / (1.0 - reference),
to_d65: math::bradford(math::D50, d65),
to_d50: math::bradford(d65, math::D50),
}
}
fn oklab(&self, lab: [f64; 3]) -> [f64; 3] {
let xyz = math::apply(&self.to_d65, math::lab_to_xyz(lab, math::D50));
math::apply(&OKLAB_M2, math::apply(&OKLAB_M1, xyz).map(f64::cbrt))
}
fn lab(&self, ok: [f64; 3]) -> [f64; 3] {
let lms = math::apply(&OKLAB_M2_INV, ok).map(|v| v * v * v);
math::xyz_to_lab(math::apply(&self.to_d50, math::apply(&OKLAB_M1_INV, lms)), math::D50)
}
/// The most chroma (scale of `ok`'s chroma in `[0, top]`) the printer reaches at `ok`'s
/// lightness and hue, and its separation; `None` when even the neutral is out of reach.
fn boundary(&self, ok: [f64; 3], top: f64, warm: Option<[f64; 4]>) -> Option<(f64, [f64; 4])> {
let (grey, d) = separate(self.model, self.p, self.lab([ok[0], 0.0, 0.0]), warm);
if d > REACH {
return None;
}
let (mut lo, mut hi, mut best) = (0.0, top, grey);
for _ in 0..14 {
let mid = 0.5 * (lo + hi);
let (c, d) = separate(self.model, self.p, self.lab([ok[0], ok[1] * mid, ok[2] * mid]), Some(best));
if d <= REACH {
lo = mid;
best = c;
} else {
hi = mid;
}
}
Some((lo, best))
}
/// Colorimetric: exact in gamut; outside, the closest colour the printer reaches in Oklab,
/// lightness and hue weighted above chroma, and more so the further outside the colour is.
fn clip(&self, lab: [f64; 3], warm: Option<[f64; 4]>) -> [f64; 4] {
let (cmyk, de) = separate(self.model, self.p, lab, warm);
if de <= REACH {
return cmyk;
}
let ok = self.oklab(lab);
let c0 = ok[1].hypot(ok[2]);
// Unit vectors along the colour's chroma and hue directions.
let (u, v) = if c0 > 1e-9 { ([ok[1] / c0, ok[2] / c0], [-ok[2] / c0, ok[1] / c0]) } else { ([1.0, 0.0], [0.0, 1.0]) };
// Lightness and hue errors weighted by `w`, chroma error by 1; scaled by 100 so the
// solver's step tolerances suit it as they suit ΔE76.
let residual = |w: f64| {
move |l: [f64; 3]| {
let o = self.oklab(l);
let (da, db) = (o[1] - ok[1], o[2] - ok[2]);
[100.0 * w * (o[0] - ok[0]), 100.0 * (da * u[0] + db * u[1]), 100.0 * w * (da * v[0] + db * v[1])]
}
};
let solve = |w: f64, k: f64, from: [f64; 3]| {
let mut seeds = vec![from];
seeds.extend_from_slice(&SEEDS);
solve_cmy_by(self.model, k, (self.p.tac - k).max(0.0), &seeds, residual(w)).0
};
// A first clip without black measures how far outside the colour is. A colour just
// outside may give up a little lightness and hue to keep its chroma (a red reaches the
// full magenta and yellow edge); one far outside keeps its lightness and hue and loses
// chroma (a saturated blue does not slide along the gamut surface).
let first = solve(CLIP_WEIGHT, 0.0, [cmyk[0], cmyk[1], cmyk[2]]);
let o = self.oklab(self.model.lab([first[0], first[1], first[2], 0.0]));
let loss = if c0 > 1e-9 { (1.0 - o[1].hypot(o[2]) / c0).clamp(0.0, 1.0) } else { 0.0 };
let w = CLIP_WEIGHT * (1.0 + CLIP_WEIGHT_GROWTH * loss * loss);
let free = solve(w, 0.0, first);
let norm = |c: [f64; 4]| residual(w)(self.model.lab(c)).iter().map(|v| v * v).sum::<f64>();
// Candidates, the closest kept: the separation itself (a colour it nearly reaches stays
// continuous with its in-gamut neighbours), the clip with the separation's black, and
// the clip with black generated from the black-free clip's gray component (inks the
// clip drives to 100 % stay there and black replaces the rest).
let mut best = (norm(cmyk), cmyk);
for k in [cmyk[3], gcr(self.p, free[0].min(free[1]).min(free[2]))] {
for from in [[cmyk[0], cmyk[1], cmyk[2]], free] {
let cmy = solve(w, k, from);
let c = [cmy[0], cmy[1], cmy[2], k];
let n = norm(c);
if n < best.0 {
best = (n, c);
}
}
}
// Near paper white the gamut narrows to a point: the separation's closest colour keeps
// light tints' chroma where holding lightness would drop them toward bare paper, and
// the table nodes there carry the in-gamut tints between them. The inks blend from the
// clip at L* 85 to the separation at L* 95.
let clip_share = ((95.0 - lab[0]) / 10.0).clamp(0.0, 1.0);
std::array::from_fn(|i| clip_share * best.1[i] + (1.0 - clip_share) * cmyk[i])
}
/// Maps the v4 perceptual reference black onto the device black, linearly in XYZ, for
/// neutrals; colourful colours keep their lightness.
fn lift_black(&self, lab: [f64; 3]) -> [f64; 3] {
let ok = self.oklab(lab);
let neutral = (1.0 - ok[1].hypot(ok[2]) / LIFT_CHROMA).clamp(0.0, 1.0);
let a = 1.0 - neutral * (1.0 - self.black_scale);
let xyz = math::lab_to_xyz(lab, math::D50);
math::xyz_to_lab(std::array::from_fn(|i| a * xyz[i] + math::D50[i] * (1.0 - a)), math::D50)
}
fn perceptual(&self, lab: [f64; 3], warm: Option<[f64; 4]>) -> [f64; 4] {
let lab = self.lift_black(lab);
let ok = self.oklab(lab);
let c = ok[1].hypot(ok[2]);
if c < 1e-9 {
return self.clip(lab, warm);
}
let Some((s, edge)) = self.boundary(ok, OKLAB_CHROMA_MAX / c, warm) else {
return self.clip(lab, warm);
};
// Chroma above the knee is compressed smoothly into the boundary chroma `cb`.
let cb = s * c;
let knee = self.knee * cb;
if c <= knee {
return self.clip(lab, warm);
}
let c2 = knee + (cb - knee) * ((c - knee) / (cb - knee).max(1e-9)).tanh();
let f = c2 / c;
if f >= s {
return edge;
}
self.clip(self.lab([ok[0], ok[1] * f, ok[2] * f]), Some(edge))
}
fn saturation(&self, lab: [f64; 3], warm: Option<[f64; 4]>) -> [f64; 4] {
let ok = self.oklab(self.lift_black(lab));
self.clip(self.lab([ok[0], ok[1] * self.saturation, ok[2] * self.saturation]), warm)
}
}
fn lab_node_legacy(i: usize, n: usize) -> f64 {
i as f64 / (n - 1) as f64
}
@@ -419,6 +657,18 @@ fn white_node_curve(l_axis: bool, x: f64) -> f64 {
}
fn b2a_table<F: Forward + ?Sized>(model: &F, p: &CmykParams, gbd: Option<&Gbd>) -> Result<Clut, CmsError> {
b2a_table_with(p, |lab, warm| {
let target = match gbd {
Some(g) => g.compress(lab),
None => lab,
};
separate(model, p, target, warm).0
})
}
/// Samples a BToA table: `node` separates each relative Lab node, warm-started from the
/// previous node along b.
fn b2a_table_with(p: &CmykParams, node: impl Fn([f64; 3], Option<[f64; 4]>) -> [f64; 4] + Sync) -> Result<Clut, CmsError> {
let n = p.grid_b2a.max(2);
let nodes = n * n * n;
let step = (n - 1) as f64;
@@ -434,11 +684,7 @@ fn b2a_table<F: Forward + ?Sized>(model: &F, p: &CmykParams, gbd: Option<&Gbd>)
} else {
decode_v2([lab_node_legacy(li, n), lab_node_legacy(ai, n), lab_node_legacy(bi, n)])
};
let target = match gbd {
Some(g) => g.compress(lab),
None => lab,
};
let (cmyk, _) = if p.white_on_node && lab == [100.0, 0.0, 0.0] { ([0.0; 4], 0.0) } else { separate(model, p, target, warm) };
let cmyk = if p.white_on_node && lab == [100.0, 0.0, 0.0] { [0.0; 4] } else { node(lab, warm) };
warm = Some(cmyk);
out.extend(cmyk.map(|v| v as f32));
}
@@ -460,6 +706,15 @@ fn check_grids(p: &CmykParams) -> Result<(), CmsError> {
if p.white_on_node && p.grid_b2a.is_multiple_of(2) {
return Err(invalid_arg(format!("white on a node needs an odd BToA grid, not {}", p.grid_b2a)));
}
let valid = |v: &[f64; 2]| v.iter().all(|x| (0.0..=1.0).contains(x));
if !p.gcr_curve.iter().all(valid) || p.gcr_curve.windows(2).any(|w| w[1][0] <= w[0][0]) {
return Err(invalid_arg(format!("black generation curve {:?} needs points in [0, 1] with increasing gray components", p.gcr_curve)));
}
if let GamutMapping::Oklab { knee, saturation } = p.mapping
&& !((0.0..1.0).contains(&knee) && saturation.is_finite() && saturation > 0.0)
{
return Err(invalid_arg(format!("Oklab gamut mapping needs a knee in [0, 1) and a positive saturation, not {knee} and {saturation}")));
}
Ok(())
}
@@ -518,11 +773,23 @@ pub fn cmyk_profile_from_model<F: Forward + ?Sized>(model: &F, paper: [f64; 3],
}
fn lut_profile<F: Forward + ?Sized>(model: &F, a2b: Clut, paper: [f64; 3], p: &CmykParams) -> Result<Profile, CmsError> {
let gbd = Gbd::new(model, p);
let colorimetric = b2a_table(model, p, None)?;
let perceptual = b2a_table(model, p, Some(&gbd))?;
let (perceptual, colorimetric, saturation) = match p.mapping {
GamutMapping::Legacy => {
let gbd = Gbd::new(model, p);
let colorimetric = b2a_table(model, p, None)?;
let perceptual = b2a_table(model, p, Some(&gbd))?;
(perceptual.clone(), colorimetric, perceptual)
}
GamutMapping::Oklab { knee, saturation } => {
let m = OklabMapper::new(model, p, knee, saturation);
let colorimetric = b2a_table_with(p, |lab, warm| m.clip(lab, warm))?;
let perceptual = b2a_table_with(p, |lab, warm| m.perceptual(lab, warm))?;
let saturation = b2a_table_with(p, |lab, warm| m.saturation(lab, warm))?;
(perceptual, colorimetric, saturation)
}
};
let a2b = lut16(a2b, false);
let (b0, b1) = (lut16(perceptual, p.white_on_node), lut16(colorimetric, p.white_on_node));
let (b0, b1, b2) = (lut16(perceptual, p.white_on_node), lut16(colorimetric, p.white_on_node), lut16(saturation, p.white_on_node));
Ok(Profile {
version: (4, 0x30),
class: ProfileClass::Output,
@@ -538,7 +805,7 @@ fn lut_profile<F: Forward + ?Sized>(model: &F, a2b: Clut, paper: [f64; 3], p: &C
trc: None,
gray_trc: None,
a2b: [Some(a2b.clone()), Some(a2b.clone()), Some(a2b)],
b2a: [Some(b0.clone()), Some(b1), Some(b0)],
b2a: [Some(b0), Some(b1), Some(b2)],
bytes: None,
hash: Default::default(),
}
@@ -576,6 +843,46 @@ mod tests {
}
}
#[test]
fn oklab_mapping_refuses_bad_parameters() {
for (knee, saturation) in [(1.0, 1.0), (-0.1, 1.0), (f64::NAN, 1.0), (0.5, 0.0), (0.5, f64::INFINITY), (0.5, f64::NAN)] {
let p = CmykParams { grid_a2b: 2, grid_b2a: 2, mapping: GamutMapping::Oklab { knee, saturation }, ..CmykParams::default() };
assert!(cmyk_profile(&p).is_err(), "knee {knee}, saturation {saturation}");
}
}
#[test]
fn black_generation_curve() {
let p = CmykParams { gcr_curve: vec![[0.0, 0.0], [0.5, 0.2], [1.0, 0.9]], ..CmykParams::default() };
for (g, k) in [(-1.0, 0.0), (0.0, 0.0), (0.25, 0.1), (0.5, 0.2), (0.75, 0.55), (1.0, 0.9), (2.0, 0.9)] {
assert!((gcr(&p, g) - k).abs() < 1e-12, "g {g}: {} != {k}", gcr(&p, g));
}
for curve in [vec![[0.5, 0.2], [0.5, 0.3]], vec![[0.5, 0.2], [0.2, 0.3]], vec![[0.0, 1.5]], vec![[f64::NAN, 0.0]]] {
let p = CmykParams { grid_a2b: 2, grid_b2a: 2, gcr_curve: curve.clone(), ..CmykParams::default() };
assert!(cmyk_profile(&p).is_err(), "{curve:?}");
}
}
#[test]
fn oklab_round_trips() {
let m = CmykModel::coated(CmykParams::default().tvi);
let p = CmykParams::default();
let mapper = OklabMapper::new(&m, &p, 0.65, 1.05);
for lab in [[50.0, 0.0, 0.0], [44.0, 63.7, 47.0], [31.9, 32.9, -76.8], [90.0, -5.0, 80.0]] {
let back = mapper.lab(mapper.oklab(lab));
// The published matrices carry 10 digits, so the inverses are exact to about 1e-5 ΔE.
assert!(math::delta_e76(lab, back) < 1e-3, "{lab:?} -> {back:?}");
}
}
#[test]
fn oklab_profile_has_three_tables() {
let p = CmykParams { grid_a2b: 5, grid_b2a: 5, mapping: GamutMapping::Oklab { knee: 0.65, saturation: 1.05 }, ..CmykParams::default() };
let profile = cmyk_profile(&p).unwrap();
let [Some(b0), Some(b1), Some(b2)] = &profile.b2a else { panic!("missing BToA tables") };
assert!(b0 != b1 && b0 != b2, "perceptual shares a table");
}
#[test]
fn separation_hits_in_gamut_targets() {
let m = CmykModel::coated(CmykParams::default().tvi);
+1 -1
View File
@@ -46,7 +46,7 @@ const OUT_TABLE: usize = GpuMatrixShaper::OUTPUT_ENTRIES;
const _: () = assert!(OUT_TABLE.is_power_of_two());
/// ICC v4 perceptual reference medium black (ICC.1:2010 §6.3.4.4), XYZ.
const PERCEPTUAL_BLACK: [f64; 3] = [0.00336, 0.0034731, 0.00287];
pub(crate) const PERCEPTUAL_BLACK: [f64; 3] = [0.00336, 0.0034731, 0.00287];
/// Accelerated representation of a transform.
#[derive(Debug)]
+60 -1
View File
@@ -12,7 +12,7 @@
//! ([`synth::cmyk_profile_from_model`]).
use craft_cms::math::{self, delta_e2000};
use craft_cms::synth::{self, CmykParams, Forward};
use craft_cms::synth::{self, CmykParams, Forward, GamutMapping};
use craft_cms::{Builtin, Intent, Profile, Transform};
const DATA: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/data/FOGRA39.txt");
@@ -346,8 +346,10 @@ fn regen_coated_fogra39() {
description: Builtin::CoatedFogra39.description().into(),
copyright: craft_cms::builtin::COATED_FOGRA39_COPYRIGHT.into(),
tac: 3.3,
gcr_curve: vec![[0.0, 0.0], [0.2, 0.05], [0.5, 0.205], [0.67, 0.5], [1.0, 0.95]],
grid_b2a: 33,
white_on_node: true,
mapping: GamutMapping::Oklab { knee: 0.65, saturation: 1.05 },
..CmykParams::default()
};
let profile = synth::cmyk_profile_from_model(&f, paper(&p), &params).unwrap();
@@ -564,3 +566,60 @@ fn stand_in_separations_respect_the_ink_limit() {
}
}
}
/// sRGB patches and their separations by the reference application (Convert to Profile to
/// Coated FOGRA39, dither on), CMYK % per intent: relative + BPC, relative, perceptual,
/// saturation, absolute. Measured for storytold/photocraft#14267.
const REFERENCE_SEPARATIONS: [([u8; 3], [[f32; 4]; 5]); 3] = [
(
[200, 30, 30],
[[14.5, 97.7, 95.3, 4.3], [13.3, 100.0, 100.0, 5.5], [18.0, 96.9, 92.9, 8.6], [14.9, 100.0, 100.0, 5.9], [8.6, 100.0, 100.0, 2.4]],
),
([30, 60, 200], [[91.8, 75.3, 0.0, 0.0], [94.5, 76.5, 0.0, 0.0], [94.5, 77.7, 0.0, 0.0], [93.7, 76.9, 0.0, 0.0], [92.6, 74.9, 0.0, 0.0]]),
(
[128, 128, 128],
[[49.0, 38.4, 38.4, 20.0], [49.8, 39.2, 38.8, 21.2], [49.0, 38.4, 38.4, 20.0], [49.0, 38.4, 38.4, 20.0], [46.7, 36.9, 38.4, 18.4]],
),
];
const SEPARATION_INTENTS: [(Intent, bool); 5] = [
(Intent::RelativeColorimetric, true),
(Intent::RelativeColorimetric, false),
(Intent::Perceptual, true),
(Intent::Saturation, true),
(Intent::AbsoluteColorimetric, false),
];
fn separation(intent: Intent, bpc: bool, rgb: [u8; 3]) -> [f32; 4] {
let t = Transform::new(Builtin::Srgb.profile(), fogra39(), intent, bpc).unwrap();
let mut o = [0.0f32; 4];
t.eval(&rgb.map(|v| v as f32 / 255.0), &mut o).unwrap();
o.map(|v| v * 100.0)
}
/// sRGB → Coated FOGRA39 matches the reference application's inks on a red, a saturated blue
/// and a mid grey for every intent. Most land within 2 % per ink; red at the magenta–yellow
/// edge lands between table nodes that are and are not clipped, which costs up to 2.2 % with
/// the 33³ BToA tables.
#[test]
fn separates_like_the_reference_application() {
let mut worst = 0.0f32;
for (rgb, want) in REFERENCE_SEPARATIONS {
for ((intent, bpc), want) in SEPARATION_INTENTS.into_iter().zip(want) {
let got = separation(intent, bpc, rgb);
let diff = (0..4).map(|i| (got[i] - want[i]).abs()).fold(0.0f32, f32::max);
eprintln!("{rgb:?} {intent:?} bpc {bpc}: {got:.1?} reference {want:?} (max {diff:.1})");
worst = worst.max(diff);
}
}
assert!(worst <= 2.5, "worst ink difference {worst} %");
}
/// Saturation has its own table: a red separates with more magenta and yellow than
/// perceptual, whose chroma compression takes some back.
#[test]
fn saturation_and_perceptual_separate_differently() {
let red = [200, 30, 30];
let (perceptual, saturation) = (separation(Intent::Perceptual, true, red), separation(Intent::Saturation, true, red));
assert!(saturation[1] > perceptual[1] + 1.0 && saturation[2] > perceptual[2] + 1.0, "perceptual {perceptual:?}, saturation {saturation:?}");
}
+241
View File
@@ -86,6 +86,83 @@ fn held_coefficients(marker: u8, seg: &[u8], scan_components: Option<u8>) -> u64
factors.iter().map(|&(fh, fv)| mcus * fh * fv * 64 * 2).sum()
}
/// Bytes of DCT coefficients to budget for before decoding hostile `b`: the larger of
/// [`crate::JpegInfo::held_coefficients`]'s rule (first frame; SOF2, or SOF0/SOF1 whose first
/// scan lacks components; 8-bit only; sampling factors as written) and a coarser one that
/// keeps a progressive JPEG's coefficients for the whole picture:
/// - every SOF2, SOF6, SOF10 and SOF14 header before the first scan counts, at any precision, and
/// the largest wins; each charges 2 bytes a coefficient: per component, `2 * 64 * h * v` per
/// MCU, with `h` and `v` clamped to 1..=4 and the MCUs covering the frame at the largest `h`, `v`;
/// - a header that can't be followed (a segment that is cut, a length under 2, a frame header
/// without its components, no `0xFF` where a marker belongs) charges the costliest frame,
/// four components sampled 4 × 4, at the first frame's size (as `scan_metadata` reads it), and 0 without one.
///
/// Never less than either rule. Frames `craft-jpeg` decodes (arithmetic, lossless, 12-bit)
/// enforce `max_alloc` on their own buffers, and SOF6/SOF14 (hierarchical) are not decoded at all,
/// so charging them is only a conservative budget. Arithmetic saturates; the scan is linear.
pub(crate) fn worst_case_coefficients(b: &[u8]) -> u64 {
let m = scan_metadata(b);
let mut at = 2usize;
let mut most = 0u64;
let first: Option<(u64, u64)> = m.frame.map(|(w, h, _)| (u64::from(w), u64::from(h)));
let mut broken = false;
while !broken {
if b.get(at) != Some(&0xFF) {
broken = true;
break;
}
while b.get(at) == Some(&0xFF) {
at += 1;
}
let Some(&code) = b.get(at) else {
broken = true;
break;
};
at += 1;
match code {
0xD8 | 0x01 | 0xD0..=0xD7 => continue,
0xD9 | 0xDA => break,
_ => {}
}
let len = match (b.get(at), b.get(at + 1)) {
(Some(&hi), Some(&lo)) => usize::from(u16::from_be_bytes([hi, lo])),
_ => 0,
};
let Some(seg) = (len >= 2).then(|| b.get(at + 2..at + len)).flatten() else {
broken = true;
break;
};
if matches!(code, 0xC2 | 0xC6 | 0xCA | 0xCE) {
match progressive_frame_coefficients(seg) {
Some(held) => most = most.max(held),
None => {
broken = true;
break;
}
}
}
at += len;
}
if broken && let Some((w, h)) = first {
let mcus = w.div_ceil(32).saturating_mul(h.div_ceil(32));
most = most.max(mcus.saturating_mul(2 * 64 * 16 * 4));
}
most.max(m.held_coefficients)
}
/// [`worst_case_coefficients`]'s charge for one frame header `sof` (after its length).
fn progressive_frame_coefficients(sof: &[u8]) -> Option<u64> {
let height = u64::from(u16::from_be_bytes([*sof.get(1)?, *sof.get(2)?]));
let width = u64::from(u16::from_be_bytes([*sof.get(3)?, *sof.get(4)?]));
let n = usize::from(*sof.get(5)?);
let comps = sof.get(6..6usize.checked_add(n.checked_mul(3)?)?)?;
let factors: Vec<(u64, u64)> =
comps.as_chunks::<3>().0.iter().map(|&[_, f, _]| (u64::from(f >> 4).clamp(1, 4), u64::from(f & 15).clamp(1, 4))).collect();
let (hmax, vmax) = factors.iter().fold((1, 1), |(h, v), &(a, b)| (h.max(a), v.max(b)));
let mcus = width.div_ceil(8 * hmax).saturating_mul(height.div_ceil(8 * vmax));
Some(factors.iter().fold(0u64, |sum, &(h, v)| sum.saturating_add(mcus.saturating_mul(2 * 64 * h * v))))
}
/// Walk the marker segments up to SOS and collect metadata.
pub(crate) fn scan_metadata(b: &[u8]) -> JpegMeta {
let mut m = JpegMeta::default();
@@ -825,6 +902,170 @@ mod tests {
assert_eq!(held(&frame(0xC2, 12, &[0x11, 0x11, 0x11], None)), Some(0));
}
/// The consumer's own estimate (WordCraft's `jpeg_coefficients`), ported as the oracle that
/// `worst_case_coefficients` must never undercut.
fn oracle(bytes: &[u8]) -> u64 {
fn worst(bytes: &[u8]) -> u64 {
let Some((w, h)) = info(bytes).map(|i| (i.width, i.height)) else { return 0 };
let mcus = u64::from(w).div_ceil(32) * u64::from(h).div_ceil(32);
mcus.saturating_mul(2 * 64 * 16 * 4)
}
fn frame(sof: &[u8]) -> Option<u64> {
let height = u64::from(u16::from_be_bytes([*sof.get(1)?, *sof.get(2)?]));
let width = u64::from(u16::from_be_bytes([*sof.get(3)?, *sof.get(4)?]));
let n = usize::from(*sof.get(5)?);
let s: Vec<(u64, u64)> = sof
.get(6..6 + 3 * n)?
.as_chunks::<3>()
.0
.iter()
.map(|&[_, s, _]| (u64::from(s >> 4).clamp(1, 4), u64::from(s & 15).clamp(1, 4)))
.collect();
let (hm, vm) = s.iter().fold((1, 1), |(h, v), &(a, b)| (h.max(a), v.max(b)));
let mcus = width.div_ceil(8 * hm).checked_mul(height.div_ceil(8 * vm))?;
s.iter().try_fold(0u64, |sum, &(h, v)| sum.checked_add(mcus.checked_mul(2 * 64 * h * v)?))
}
let mut at = 2;
let mut most = 0u64;
loop {
if bytes.get(at) != Some(&0xFF) {
return worst(bytes).max(most);
}
while bytes.get(at) == Some(&0xFF) {
at += 1;
}
let Some(&code) = bytes.get(at) else { return worst(bytes).max(most) };
at += 1;
match code {
0xD8 | 0x01 | 0xD0..=0xD7 => continue,
0xD9 | 0xDA => return most,
_ => {}
}
let len = match (bytes.get(at), bytes.get(at + 1)) {
(Some(&hi), Some(&lo)) => usize::from(u16::from_be_bytes([hi, lo])),
_ => 0,
};
let Some(seg) = bytes.get(at + 2..at + len).filter(|_| len >= 2) else { return worst(bytes).max(most) };
if matches!(code, 0xC2 | 0xC6 | 0xCA | 0xCE) {
let Some(h) = frame(seg) else { return worst(bytes).max(most) };
most = most.max(h);
}
at += len;
}
}
/// A JPEG header: each frame is (marker, precision, width, height, sampling bytes), then an
/// optional scan of `scan` components, then EOI.
fn header(frames: &[(u8, u8, u16, u16, &[u8])], scan: Option<u8>) -> Vec<u8> {
let mut b = vec![0xFF, 0xD8];
for &(marker, precision, w, h, f) in frames {
b.extend_from_slice(&[0xFF, marker]);
b.extend_from_slice(&(8 + 3 * f.len() as u16).to_be_bytes());
b.push(precision);
b.extend_from_slice(&h.to_be_bytes());
b.extend_from_slice(&w.to_be_bytes());
b.push(f.len() as u8);
for (i, &s) in f.iter().enumerate() {
b.extend_from_slice(&[i as u8 + 1, s, 0]);
}
}
if let Some(n) = scan {
b.extend_from_slice(&[0xFF, 0xDA, 0, 6 + 2 * n, n]);
(1..=n).for_each(|c| b.extend_from_slice(&[c, 0]));
b.extend_from_slice(&[0, 63, 0]);
}
b.extend_from_slice(&[0xFF, 0xD9]);
b
}
/// Never less than the oracle or `held_coefficients`; equal to the larger on valid headers.
#[test]
fn worst_case_coefficients_covers_both_rule_sets() {
const C3: &[u8] = &[0x22, 0x11, 0x11];
const G: &[u8] = &[0x11];
let cases: Vec<Vec<u8>> = vec![
header(&[(0xC0, 8, 640, 480, C3)], Some(3)),
header(&[(0xC0, 8, 640, 480, G)], Some(1)),
header(&[(0xC0, 8, 640, 480, C3)], Some(1)),
header(&[(0xC1, 8, 640, 480, C3)], None),
header(&[(0xC2, 8, 640, 480, C3)], Some(1)),
header(&[(0xC2, 12, 640, 480, C3)], Some(1)),
header(&[(0xC6, 8, 640, 480, C3)], None),
header(&[(0xCA, 8, 640, 480, C3)], None),
header(&[(0xCE, 8, 640, 480, C3)], None),
header(&[(0xC0, 8, 64, 64, G), (0xC2, 8, 6000, 4000, C3), (0xC2, 8, 100, 100, C3)], Some(1)),
header(&[(0xC2, 8, 640, 480, &[0x00, 0x00, 0x00])], None),
header(&[(0xC2, 8, 640, 480, &[0xFF, 0xFF, 0xFF, 0xFF])], None),
header(&[(0xC2, 8, 65535, 65535, &[0x44; 4])], None),
header(&[(0xC0, 8, 65535, 65535, &[0xFF; 4])], Some(1)),
header(&[], None),
vec![0xFF, 0xD8],
vec![],
b"not a jpeg at all".to_vec(),
];
for b in &cases {
let held = scan_metadata(b).held_coefficients;
let got = worst_case_coefficients(b);
assert!(got >= oracle(b) && got >= held, "{b:02x?}");
assert!(got >= info(b).map_or(0, |i| i.held_coefficients));
}
let at = |i: usize| worst_case_coefficients(&cases[i]);
// Equal to the oracle where the shared rule charges nothing extra.
for i in [0, 1, 6, 7, 8, 9, 10, 12, 14] {
assert_eq!(at(i), oracle(&cases[i]), "case {i}");
}
// Where it does, the shared figure wins (the oracle misses a short first scan).
assert_eq!(oracle(&cases[2]), 0);
assert_eq!(at(2), scan_metadata(&cases[2]).held_coefficients);
// Unclamped factors (15 × 15) charge the shared rule more than the clamped oracle.
assert_eq!(at(11), scan_metadata(&cases[11]).held_coefficients);
assert!(at(11) > oracle(&cases[11]));
assert!(at(2) > 0 && at(3) == 0 && at(0) == 0);
// 12-bit and arithmetic progressive frames are charged though craft-jpeg limits itself.
assert_eq!(at(5), at(4));
// Sampling factors clamp: 0 acts as 1, 15 as 4.
assert_eq!(at(10), 80 * 60 * 3 * 128);
assert_eq!(at(12), 2048 * 2048 * 4 * 2 * 64 * 16);
// The largest of several frames wins.
assert_eq!(at(9), oracle(&header(&[(0xC2, 8, 6000, 4000, C3)], None)));
}
/// A cut header charges the costliest frame at its size; every prefix and random damage stays
/// bounded, never panics and never undercuts the oracle.
#[test]
fn worst_case_coefficients_hostile_input() {
let full = header(&[(0xC2, 8, 6000, 4000, &[0x22, 0x11, 0x11])], Some(1));
let cut = &full[..full.len() - 8];
// Cut inside the component list: a frame the scan can't read, at its own size.
let sof_cut = &full[..14];
assert_eq!(worst_case_coefficients(sof_cut), oracle(sof_cut));
assert_eq!(worst_case_coefficients(cut), oracle(cut));
let mut bases = vec![full.clone(), header(&[(0xC0, 8, 6000, 4000, &[0x11; 3])], Some(1))];
bases.push(header(&[(0xC2, 8, 65535, 65535, &[0xFF; 4])], None));
let mut seed = 0x9E37_79B9_7F4A_7C15u64;
let mut next = move || {
seed = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407);
(seed >> 33) as usize
};
for base in &bases {
for n in 0..=base.len() {
let b = &base[..n];
let got = worst_case_coefficients(b);
assert!(got >= oracle(b) && got < 1 << 50);
}
for _ in 0..2000 {
let mut b = base.clone();
for _ in 0..=next() % 4 {
let i = next() % b.len();
b[i] = next() as u8;
}
b.truncate(next() % (b.len() + 1));
let got = worst_case_coefficients(&b);
assert!(got >= oracle(&b) && got >= scan_metadata(&b).held_coefficients && got < 1 << 50);
}
}
}
/// An APP1 Extended XMP fragment (GUID, total size, offset, data).
fn extended_xmp(offset: u32) -> Vec<u8> {
let mut seg = EXTENDED_XMP_HEADER.to_vec();
+9
View File
@@ -307,6 +307,15 @@ pub fn jpeg_info(bytes: &[u8]) -> Option<JpegInfo> {
jpeg::info(bytes)
}
/// Bytes of DCT coefficients to budget for a JPEG before decoding it: never less than
/// [`JpegInfo::held_coefficients`] nor than the whole-picture estimate of every progressive
/// frame header (SOF2, SOF6, SOF10, SOF14 at any precision, sampling clamped to 1..=4), and the
/// costliest frame at the first size read when the header can't be followed. 0 without a
/// readable frame. See the function body's rules for the precise charges.
pub fn jpeg_worst_case_coefficients(bytes: &[u8]) -> u64 {
jpeg::worst_case_coefficients(bytes)
}
/// PCX's dimensions are available from its fixed-size header without decoding the RLE rows.
pub fn pcx_dimensions(bytes: &[u8]) -> Option<(u32, u32)> {
codecs::pcx::dimensions(bytes)
+223
View File
@@ -136,6 +136,93 @@ pub fn color_chunks(b: &[u8]) -> Option<ColorChunks> {
Some(result)
}
/// Most chunks [`pixel_model`] walks before the image data; each chunk is at least 12 bytes.
const MAX_PROBE_CHUNKS: usize = 1 << 16;
/// Pixel model of a PNG, from `IHDR` and the chunk headers before the first `IDAT`.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct PixelModel {
pub width: u32,
pub height: u32,
/// `IHDR` bit depth (1, 2, 4, 8 or 16).
pub bit_depth: u8,
/// `IHDR` colour type (0 grey, 2 RGB, 3 palette, 4 grey+alpha, 6 RGBA).
pub color_type: u8,
/// A `tRNS` chunk precedes the image data; it adds alpha to grey, RGB and palette images.
pub has_trns: bool,
}
impl PixelModel {
/// Channels of the decoded buffer (palette expands to RGB; `tRNS` adds an alpha channel).
pub fn channels(&self) -> u8 {
let base = match self.color_type {
0 => 1,
2 | 3 => 3,
4 => 2,
_ => 4,
};
base + u8::from(self.has_trns && matches!(self.color_type, 0 | 2 | 3))
}
/// Bytes per pixel of the buffer a decoder that expands sub-byte samples to 8 bits gives:
/// `channels * (2 if bit_depth == 16 else 1)`.
pub fn bytes_per_pixel(&self) -> u8 {
self.channels() * if self.bit_depth == 16 { 2 } else { 1 }
}
/// Whether the image has transparency (alpha colour type or `tRNS`).
pub fn has_alpha(&self) -> bool {
matches!(self.color_type, 4 | 6) || self.has_trns
}
}
/// Dimensions, bit depth, colour type and `tRNS` presence without inflating anything (an `iCCP`
/// profile is skipped, not decompressed). Walks at most 65,536 chunk headers and stops at `IDAT`;
/// `IEND` or the end of the input before `IDAT` is an error. Does not validate CRCs.
/// A caller gets its `(width, height, bytes_per_pixel, alpha)` as
/// `(m.width, m.height, u64::from(m.bytes_per_pixel()), m.has_alpha())`.
pub fn pixel_model(b: &[u8]) -> Result<PixelModel, CodecError> {
if !b.starts_with(b"\x89PNG\r\n\x1a\n") {
return Err(err("bad signature"));
}
let be32 = |p: usize| b.get(p..p.checked_add(4)?).and_then(|v| <[u8; 4]>::try_from(v).ok()).map(u32::from_be_bytes);
if be32(8) != Some(13) || b.get(12..16) != Some(b"IHDR") {
return Err(err("missing IHDR"));
}
let ihdr = b.get(16..29).ok_or_else(|| err("truncated IHDR"))?;
let (width, height) = (be32(16).unwrap_or(0), be32(20).unwrap_or(0));
let (bit_depth, color_type) = (ihdr[8], ihdr[9]);
if width == 0 || height == 0 || width > i32::MAX as u32 || height > i32::MAX as u32 {
return Err(err("dimensions out of range"));
}
let depth_ok = match color_type {
0 => matches!(bit_depth, 1 | 2 | 4 | 8 | 16),
3 => matches!(bit_depth, 1 | 2 | 4 | 8),
2 | 4 | 6 => matches!(bit_depth, 8 | 16),
_ => false,
};
if !depth_ok {
return Err(err("invalid IHDR"));
}
let mut has_trns = false;
let mut p = 8usize;
for _ in 0..MAX_PROBE_CHUNKS {
let len = be32(p).ok_or_else(|| err("truncated"))?;
let kind = p.checked_add(4).and_then(|k| b.get(k..k.checked_add(4)?)).ok_or_else(|| err("truncated"))?;
match kind {
b"IDAT" => return Ok(PixelModel { width, height, bit_depth, color_type, has_trns }),
b"IEND" => return Err(err("no image data")),
b"tRNS" => has_trns = true,
_ => {}
}
if len > i32::MAX as u32 {
return Err(err("chunk length out of range"));
}
p = p.checked_add(12).and_then(|q| q.checked_add(len as usize)).ok_or_else(|| err("truncated"))?;
}
Err(CodecError::LimitExceeded("PNG chunk count".into()))
}
#[cfg(test)]
mod tests {
use super::*;
@@ -187,4 +274,140 @@ mod tests {
assert!(!color.srgb);
assert!(color_chunks(b"not png").is_none());
}
/// WordCraft's `png_chunks` (crates/geom/src/picture.rs) as the differential oracle.
fn oracle(bytes: &[u8]) -> Option<(u32, u32, u64, bool)> {
let be = |at: usize| bytes.get(at..at.checked_add(4)?).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(u32::from_be_bytes);
if bytes.get(12..16) != Some(b"IHDR".as_slice()) {
return None;
}
let (width, height) = (be(16)?, be(20)?);
let (depth, color) = (*bytes.get(24)?, *bytes.get(25)?);
let mut trns = false;
let mut at = 8usize;
loop {
let len = usize::try_from(be(at)?).ok()?;
match bytes.get(at.checked_add(4)?..at.checked_add(8)?)? {
b"IDAT" => break,
b"IEND" => return None,
b"tRNS" => trns = true,
_ => {}
}
at = at.checked_add(12)?.checked_add(len)?;
}
let (channels, alpha) = match color {
0 => (1 + u64::from(trns), trns),
2 | 3 => (3 + u64::from(trns), trns),
4 => (2, true),
6 => (4, true),
_ => return None,
};
Some((width, height, channels * if depth == 16 { 2 } else { 1 }, alpha))
}
fn tuple(b: &[u8]) -> Option<(u32, u32, u64, bool)> {
pixel_model(b).ok().map(|m| (m.width, m.height, u64::from(m.bytes_per_pixel()), m.has_alpha()))
}
fn png_with(depth: u8, color: u8, before_idat: &[(&[u8; 4], Vec<u8>)]) -> Vec<u8> {
let mut bytes = b"\x89PNG\r\n\x1a\n".to_vec();
let mut ihdr = Vec::new();
ihdr.extend_from_slice(&5u32.to_be_bytes());
ihdr.extend_from_slice(&7u32.to_be_bytes());
ihdr.extend_from_slice(&[depth, color, 0, 0, 0]);
chunk(&mut bytes, b"IHDR", &ihdr);
for (k, d) in before_idat {
chunk(&mut bytes, k, d);
}
chunk(&mut bytes, b"IDAT", &[1, 2, 3]);
chunk(&mut bytes, b"IEND", &[]);
bytes
}
#[test]
fn pixel_model_matches_oracle_for_every_color_type_depth_and_trns() {
let valid: [(u8, &[u8]); 5] = [(0, &[1, 2, 4, 8, 16]), (2, &[8, 16]), (3, &[1, 2, 4, 8]), (4, &[8, 16]), (6, &[8, 16])];
for (color, depths) in valid {
for &depth in depths {
for trns in [false, true] {
let extra: Vec<(&[u8; 4], Vec<u8>)> = if trns { vec![(b"tRNS", vec![0, 0])] } else { vec![] };
let bytes = png_with(depth, color, &extra);
assert!(tuple(&bytes).is_some(), "color {color} depth {depth}");
assert_eq!(tuple(&bytes), oracle(&bytes), "color {color} depth {depth} trns {trns}");
}
}
}
}
#[test]
fn pixel_model_known_values() {
let m = pixel_model(&png_with(16, 2, &[(b"tRNS", vec![0; 6])])).unwrap();
assert_eq!((m.width, m.height, m.channels(), m.bytes_per_pixel(), m.has_alpha()), (5, 7, 4, 8, true));
let m = pixel_model(&png_with(4, 3, &[])).unwrap();
assert_eq!((m.bytes_per_pixel(), m.has_alpha()), (3, false));
}
#[test]
fn pixel_model_skips_a_huge_iccp_without_inflating() {
let junk = vec![0xAB; 20 << 20];
let bytes = png_with(8, 2, &[(b"iCCP", junk), (b"tRNS", vec![0; 6])]);
assert_eq!(tuple(&bytes), oracle(&bytes));
assert_eq!(tuple(&bytes), Some((5, 7, 4, true)));
}
#[test]
fn pixel_model_malformed_inputs_agree_with_oracle_and_never_panic() {
let full = png_with(8, 6, &[(b"tRNS", vec![0]), (b"gAMA", vec![0; 4])]);
for n in 0..=full.len() {
let cut = &full[..n];
let got = tuple(cut);
// an Err where the oracle answers can only be a stricter header check
if got.is_some() {
assert_eq!(got, oracle(cut), "prefix {n}");
}
}
assert!(pixel_model(&full[..33]).is_err());
// missing IHDR
let mut bad = full.clone();
bad[12..16].copy_from_slice(b"IHDX");
assert!(pixel_model(&bad).is_err() && oracle(&bad).is_none());
// chunk length overflow: a length past the input, then one past i32::MAX
for len in [u32::MAX, i32::MAX as u32, 0x7fff_fff0] {
let mut o = full.clone();
o[33..37].copy_from_slice(&len.to_be_bytes());
assert!(pixel_model(&o).is_err(), "len {len:#x}");
assert_eq!(tuple(&o), oracle(&o));
}
// IEND before IDAT, unknown colour type, zero size, bad signature
let mut iend = png_with(8, 2, &[]);
let at = iend.windows(4).position(|w| w == b"IDAT").unwrap();
iend[at..at + 4].copy_from_slice(b"IEND");
assert!(pixel_model(&iend).is_err() && oracle(&iend).is_none());
assert!(pixel_model(&png_with(8, 5, &[])).is_err());
let mut zero = png_with(8, 2, &[]);
zero[16..20].copy_from_slice(&0u32.to_be_bytes());
assert!(pixel_model(&zero).is_err());
assert!(pixel_model(b"not a png").is_err());
// deterministic pseudo-random corruption
let mut s = 0x9e37_79b9u32;
for _ in 0..2000 {
let mut f = full.clone();
for _ in 0..3 {
s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
let i = (s >> 8) as usize % f.len();
f[i] = (s >> 24) as u8;
}
let _ = pixel_model(&f);
}
}
#[test]
fn pixel_model_many_tiny_chunks_and_the_chunk_cap() {
let many: Vec<(&[u8; 4], Vec<u8>)> = (0..10_000).map(|_| (b"tEXt", Vec::new())).chain([(b"tRNS", vec![0])]).collect();
let bytes = png_with(8, 0, &many);
assert_eq!(tuple(&bytes), oracle(&bytes));
assert_eq!(tuple(&bytes), Some((5, 7, 2, true)));
let over: Vec<(&[u8; 4], Vec<u8>)> = (0..MAX_PROBE_CHUNKS + 1).map(|_| (b"tEXt", Vec::new())).collect();
assert!(matches!(pixel_model(&png_with(8, 0, &over)), Err(CodecError::LimitExceeded(_))));
}
}
+54
View File
@@ -32,6 +32,25 @@ pub fn header(bytes: &[u8]) -> Result<(u32, u32, u16), CodecError> {
Ok((w, h, orientation))
}
/// Whether a WebP carries transparency, from the first chunk only (nothing is decoded):
/// `VP8X` flag bit 0x10, `VP8L` `alpha_is_used`, plain `VP8` never.
pub fn has_alpha(bytes: &[u8]) -> Result<bool, CodecError> {
let err = |s: &str| CodecError::malformed(Format::WebP, s);
if bytes.get(..4) != Some(b"RIFF") || bytes.get(8..12) != Some(b"WEBP") {
return Err(err("bad signature"));
}
match bytes.get(12..16) {
Some(b"VP8X") => Ok(bytes.get(20).ok_or_else(|| err("truncated VP8X"))? & 0x10 != 0),
// 0x2f signature byte, then 14+14 size bits, then the alpha_is_used bit (bit 28)
Some(b"VP8L") => match bytes.get(20..25) {
Some([0x2f, a, b, c, d]) => Ok(u32::from_le_bytes([*a, *b, *c, *d]) >> 28 & 1 != 0),
_ => Err(err("bad VP8L header")),
},
Some(b"VP8 ") => Ok(false),
_ => Err(err("no image data")),
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -71,4 +90,39 @@ mod tests {
bytes.push(0);
assert!(header(&bytes).is_err());
}
#[test]
fn has_alpha_reads_vp8x_vp8l_and_plain_vp8() {
for (layout, want) in [(crate::ChannelLayout::Rgb, false), (crate::ChannelLayout::Rgba, true)] {
let n = layout.channels();
let image = crate::Image::from_u8(2, 3, layout, vec![42; 6 * n]).unwrap();
let bytes = crate::encode(&image, Format::WebP, &crate::EncodeOptions::default()).unwrap();
assert_eq!(has_alpha(&bytes).unwrap(), want);
}
let riff = |chunk: &[u8; 4], body: &[u8]| {
let mut v = b"RIFF\0\0\0\0WEBP".to_vec();
v.extend_from_slice(chunk);
v.extend_from_slice(&(body.len() as u32).to_le_bytes());
v.extend_from_slice(body);
v
};
let mut x = [0u8; 10];
assert!(!has_alpha(&riff(b"VP8X", &x)).unwrap());
x[0] = 0x10;
assert!(has_alpha(&riff(b"VP8X", &x)).unwrap());
assert!(!has_alpha(&riff(b"VP8 ", &[0; 10])).unwrap());
assert!(has_alpha(&riff(b"VP8L", &[0x2f, 0, 0, 0, 0x10])).unwrap());
assert!(!has_alpha(&riff(b"VP8L", &[0x2f, 0, 0, 0, 0])).unwrap());
}
#[test]
fn has_alpha_rejects_malformed_prefixes() {
let full = [b"RIFF\0\0\0\0WEBPVP8X\x0a\0\0\0".as_slice(), &[0x10; 10]].concat();
for n in 0..20 {
assert!(has_alpha(&full[..n]).is_err(), "prefix {n}");
}
assert!(has_alpha(&full[..21]).is_ok());
assert!(has_alpha(b"RIFF\0\0\0\0WEBPVP8L\x05\0\0\0\x2e").is_err());
assert!(has_alpha(b"junk").is_err());
}
}
+6 -4
View File
@@ -191,10 +191,12 @@ mod tests {
let p = Plates { inks, unseparated: Vec::new(), width: 2, height: 1, dpi: 72.0, conversion: Conversion::DeviceFormulas, planes };
let output = Output::new(fogra39(), fogra39(), Intent::RelativeColorimetric, false).unwrap();
let cmyk = p.composite_to(&[], &output);
// Magenta's round trip through the profile adds a little yellow.
assert!(near(&cmyk[0..4], &[255, 255, 8, 0], 8), "spot over cyan: {:?}", &cmyk[0..4]);
assert!(near(&cmyk[4..8], &[0, 255, 8, 0], 8), "spot alone: {:?}", &cmyk[4..8]);
assert!(near(&p.composite_to(&[true, true, true, true, false], &output)[0..4], &[255, 0, 0, 0], 6), "spot hidden");
// The press profile's round trip: its BToA tables generate their own black, so a
// full-tone ink comes back a little short with a trace of black, as a printer would
// separate it. The tolerance is the profile's own round trip, not an exchange total.
assert!(near(&cmyk[0..4], &[250, 247, 17, 2], 8), "spot over cyan: {:?}", &cmyk[0..4]);
assert!(near(&cmyk[4..8], &[8, 247, 11, 1], 8), "spot alone: {:?}", &cmyk[4..8]);
assert!(near(&p.composite_to(&[true, true, true, true, false], &output)[0..4], &[250, 7, 6, 1], 8), "spot hidden");
}
#[test]
+15
View File
@@ -0,0 +1,15 @@
[package]
name = "craft-sourcemap"
description = "Bounded Source Map v3 parser, source inventory and generated-position lookup"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
+816
View File
@@ -0,0 +1,816 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
//! Source maps (revision 3), including indexed maps with sections.
//!
//! Lookups follow the "original positions for a generated position" rule: on the generated line,
//! take the greatest segment column not after the requested column and return every segment that
//! starts there. Lines are 0-based here (callers add one for display); columns are 0-based UTF-16
//! code units as the format defines.
use serde::de::{self, DeserializeSeed, MapAccess, SeqAccess, Visitor};
use std::cell::Cell;
use std::fmt;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Limits {
/// Biggest map JSON accepted.
pub max_bytes: usize,
/// Most decoded segments across the whole map.
pub max_segments: usize,
/// Most generated lines across the whole map, including empty lines and inline sections.
pub max_lines: usize,
/// Deepest section nesting (the format allows one level; we tolerate a few).
pub max_depth: usize,
/// Most `sources`, `sourcesContent`, `names` and ignore-list entries and index-map sections
/// across the whole map, counted while the JSON is read, so a map of tiny entries is refused
/// before it is kept. An entry a repeated key replaces gives its slot back.
pub max_entries: usize,
}
impl Default for Limits {
fn default() -> Self {
Limits { max_bytes: 64 * 1024 * 1024, max_segments: 4_000_000, max_lines: 1_000_000, max_depth: 8, max_entries: 4_000_000 }
}
}
/// Limits on retained source names after applying each map's root.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct InventoryLimits {
pub max_sources: usize,
/// Total UTF-8 bytes in the expanded source paths, across all sections.
pub max_bytes: usize,
}
impl Default for InventoryLimits {
fn default() -> Self {
Self { max_sources: 1_000_000, max_bytes: 64 * 1024 * 1024 }
}
}
/// An ordered prefix of the map's source names, with their complete identities.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct SourceInventory {
pub sources: Vec<String>,
/// More names existed than the limits or available allocations could retain.
pub truncated: bool,
/// Index-map sections that pointed at an external map nobody read, so the sources they
/// would add are unknown (not absent).
pub external_sections: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum SourceMapError {
#[error("source map is {0} bytes; the limit is {1}")]
TooLarge(usize, usize),
#[error("source map is not valid JSON: {0}")]
BadJson(String),
#[error("not a version 3 source map")]
BadVersion,
#[error("source map mappings are malformed at segment {0}")]
BadMappings(usize),
#[error("source map index has invalid sections")]
BadSections,
#[error("source map has more than {0} segments")]
TooManySegments(usize),
#[error("source map has more than {0} generated lines")]
TooManyLines(usize),
#[error("could not reserve memory for generated source-map lines")]
LineAllocationFailed,
#[error("source map sections nest deeper than {0}")]
TooDeep(usize),
#[error("source map has more than {0} source, content, name, ignore-list and section entries")]
TooManyEntries(usize),
}
/// One decoded mapping segment.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Segment {
pub generated_column: u32,
/// (source index, original line, original column, name index)
pub original: Option<(u32, u32, u32, Option<u32>)>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct SourceMap {
pub file: Option<String>,
pub source_root: Option<String>,
pub sources: Vec<Option<String>>,
pub sources_content: Vec<Option<String>>,
pub names: Vec<String>,
pub ignore_list: Vec<u32>,
/// Segments per generated line, sorted by column.
pub lines: Vec<Vec<Segment>>,
pub segments: usize,
/// Indexed maps: (line offset, column offset, map).
pub sections: Vec<(u32, u32, SourceMap)>,
/// Index-map sections that point at an external map (`url`), which isn't fetched here: the
/// sources that map would add are unknown, not absent.
pub external_sections: usize,
}
/// A position in an original source.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct OriginalPosition {
pub source_index: u32,
pub source: Option<String>,
/// 0-based.
pub line: u32,
/// 0-based.
pub column: u32,
pub name: Option<String>,
/// 0-based generated column of the matched segment.
pub generated_column: u32,
}
impl SourceMap {
/// The source path with `sourceRoot` prefixed before URL interpretation.
pub fn source_path(&self, index: u32) -> Option<String> {
let s = self.sources.get(index as usize)?.as_ref()?;
// Generators commonly emit an empty root to mean no root. Keep those source identities
// unchanged even though the literal ECMA-426 prefix algorithm would prepend a slash.
Some(match self.source_root.as_deref().filter(|root| !root.is_empty()) {
Some(root) => {
let join = if root.ends_with('/') { "" } else { "/" };
format!("{root}{join}{s}")
}
None => s.clone(),
})
}
pub fn source_content(&self, index: u32) -> Option<&str> {
self.sources_content.get(index as usize)?.as_deref()
}
/// Every original position for a generated (line, column), both 0-based.
pub fn original_positions(&self, line: u32, column: u32) -> Vec<OriginalPosition> {
if !self.sections.is_empty() {
// Last section that starts at or before the position.
let end = self.sections.partition_point(|(l, c, _)| (*l, *c) <= (line, column));
let Some((l, c, map)) = end.checked_sub(1).and_then(|i| self.sections.get(i)) else { return Vec::new() };
let rel_line = line.saturating_sub(*l);
let rel_col = if rel_line == 0 { column.saturating_sub(*c) } else { column };
return map
.original_positions(rel_line, rel_col)
.into_iter()
.filter_map(|mut position| {
if rel_line == 0 {
position.generated_column = position.generated_column.checked_add(*c)?;
}
Some(position)
})
.collect();
}
let Some(segs) = self.lines.get(line as usize) else { return Vec::new() };
// Greatest generated column <= column.
let idx = segs.partition_point(|s| s.generated_column <= column);
let Some(best) = idx.checked_sub(1).and_then(|i| segs.get(i)) else { return Vec::new() };
let col = best.generated_column;
let first = segs.partition_point(|s| s.generated_column < col);
segs.get(first..idx)
.unwrap_or_default()
.iter()
.filter_map(|s| {
let (src, l, c, n) = s.original?;
Some(OriginalPosition {
source_index: src,
source: self.source_path(src),
line: l,
column: c,
name: n.and_then(|n| self.names.get(n as usize).cloned()),
generated_column: col,
})
})
.collect()
}
/// Sources flattened across sections, within the default inventory limits.
/// Use `source_inventory` to choose limits and observe truncation.
pub fn all_sources(&self) -> Vec<String> {
self.source_inventory(InventoryLimits::default()).sources
}
/// Retain a bounded prefix, checking expanded lengths before constructing paths.
pub fn source_inventory(&self, limits: InventoryLimits) -> SourceInventory {
let mut sources = Vec::new();
let mut bytes_left = limits.max_bytes;
let truncated = !self.collect_sources(limits.max_sources, &mut bytes_left, &mut sources);
SourceInventory { sources, truncated, external_sections: self.external_sections_total() }
}
/// This map's external sections plus those of every map it holds.
fn external_sections_total(&self) -> usize {
self.sections.iter().fold(self.external_sections, |n, (_, _, map)| n.saturating_add(map.external_sections_total()))
}
/// Append this map and its sections using one shared entry and byte budget.
fn collect_sources(&self, max_sources: usize, bytes_left: &mut usize, out: &mut Vec<String>) -> bool {
let root = self.source_root.as_deref().unwrap_or_default();
let join = if root.is_empty() || root.ends_with('/') { "" } else { "/" };
for source in self.sources.iter().flatten() {
let Some(bytes) = root.len().checked_add(join.len()).and_then(|n| n.checked_add(source.len())) else { return false };
if out.len() >= max_sources || bytes > *bytes_left {
return false;
}
let mut path = String::new();
if out.try_reserve(1).is_err() || path.try_reserve_exact(bytes).is_err() {
return false;
}
path.push_str(root);
path.push_str(join);
path.push_str(source);
out.push(path);
*bytes_left = bytes_left.saturating_sub(bytes);
}
for (_, _, map) in &self.sections {
if !map.collect_sources(max_sources, bytes_left, out) {
return false;
}
}
true
}
}
/// Parse a source map.
pub fn parse(json: &[u8], limits: Limits) -> Result<SourceMap, SourceMapError> {
read(json, limits, true)
}
/// Read a map's sources, their content, names and ignore list without decoding its mappings or
/// checking its section offsets, so a map [`parse`] refuses for those
/// ([`SourceMapError::BadMappings`] or [`SourceMapError::BadSections`]) can still list its sources.
/// The map has no lines or segments. A map whose mappings, as far as they decode, name a source
/// past the end of its `sources` list is still refused, since the list is known to be short, and so are a section
/// that has neither a map nor a URL, or both, and a `sections` value that isn't an array.
pub fn parse_sources(json: &[u8], limits: Limits) -> Result<SourceMap, SourceMapError> {
read(json, limits, false)
}
fn read(json: &[u8], limits: Limits, mappings: bool) -> Result<SourceMap, SourceMapError> {
if json.len() > limits.max_bytes {
return Err(SourceMapError::TooLarge(json.len(), limits.max_bytes));
}
// Some servers prefix maps with `)]}'` to defeat XSSI; the format allows skipping that line.
let body = if json.starts_with(b")]}") { json.iter().position(|b| *b == b'\n').and_then(|p| json.get(p + 1..)).unwrap_or(&[]) } else { json };
let entries = Entries { left: Cell::new(limits.max_entries), exhausted: Cell::new(false) };
let mut de = serde_json::Deserializer::from_slice(body);
let raw = RawMapSeed(&entries).deserialize(&mut de).and_then(|raw| de.end().map(|()| raw));
let raw = match raw {
Ok(raw) => raw,
Err(_) if entries.exhausted.get() => return Err(SourceMapError::TooManyEntries(limits.max_entries)),
Err(e) => return Err(SourceMapError::BadJson(e.to_string())),
};
let mut budget = limits.max_segments;
let mut line_budget = limits.max_lines;
from_raw(raw, 0, limits, &mut budget, &mut line_budget, mappings)
}
/// `mappings`: decode the mappings and check section offsets, as [`parse`] does.
fn from_raw(
raw: Option<RawMap>,
depth: usize,
limits: Limits,
budget: &mut usize,
line_budget: &mut usize,
mappings: bool,
) -> Result<SourceMap, SourceMapError> {
if depth > limits.max_depth {
return Err(SourceMapError::TooDeep(limits.max_depth));
}
let Some(raw) = raw.filter(|raw| raw.version == Some(3)) else {
return Err(SourceMapError::BadVersion);
};
if let Some(sections) = raw.sections {
// The index has its own file name. Only its section maps describe original sources.
let mut map = SourceMap { file: raw.file, ..Default::default() };
let sections = sections.ok_or(SourceMapError::BadSections)?;
for section in sections {
let offset = |n: Option<u64>| n.and_then(|n| u32::try_from(n).ok()).ok_or(SourceMapError::BadSections);
let (line, col) = section.offset.unwrap_or_default();
let (line, col) = if mappings { (offset(line)?, offset(col)?) } else { (offset(line).unwrap_or(0), offset(col).unwrap_or(0)) };
if mappings && map.sections.last().is_some_and(|(l, c, _)| (*l, *c) > (line, col)) {
return Err(SourceMapError::BadSections);
}
let (sub, external) = match (section.map, section.url) {
(Some(m), None) => (from_raw(m.map(|m| *m), depth + 1, limits, budget, line_budget, mappings)?, false),
// External maps are never fetched here. Retain their start so a lookup
// cannot leak through to the previous inline map, and count it so the
// sources it would add are reported unknown rather than absent.
(None, Some(true)) => (SourceMap::default(), true),
_ => return Err(SourceMapError::BadSections),
};
if external {
map.external_sections = map.external_sections.saturating_add(1);
}
map.segments = map.segments.saturating_add(sub.segments);
map.sections.push((line, col, sub));
}
return Ok(map);
}
let mut map = SourceMap {
file: raw.file,
source_root: raw.source_root,
sources: raw.sources,
sources_content: raw.sources_content,
names: raw.names.into_iter().map(Option::unwrap_or_default).collect(),
ignore_list: raw
.ignore_list
.or(raw.google_ignore_list)
.unwrap_or_default()
.into_iter()
.filter_map(|n| n.and_then(|n| u32::try_from(n).ok()))
.collect(),
..Default::default()
};
let m = raw.mappings.as_deref().unwrap_or_default();
if mappings {
decode_mappings(m, &mut map, budget, line_budget, limits.max_lines)?;
} else if let Some(count) = past_sources(m, map.sources.len()) {
return Err(SourceMapError::BadMappings(count));
}
Ok(map)
}
/// The entry budget shared by every list in a map and its sections while the JSON is read.
struct Entries {
left: Cell<usize>,
exhausted: Cell<bool>,
}
impl Entries {
fn take<E: de::Error>(&self) -> Result<(), E> {
let Some(left) = self.left.get().checked_sub(1) else {
self.exhausted.set(true);
return Err(E::custom("too many source-map entries"));
};
self.left.set(left);
Ok(())
}
/// Return the slots of entries that won't be kept after all. Callers drop those entries
/// first, so a replaced list and its replacement are never held together.
fn give_back(&self, n: usize) {
self.left.set(self.left.get().saturating_add(n));
}
}
/// The fields a map is built from, read without keeping anything else. Any JSON value is
/// accepted where the format expects another kind; it reads as absent, as before.
#[derive(Default)]
struct RawMap {
version: Option<u64>,
file: Option<String>,
source_root: Option<String>,
sources: Vec<Option<String>>,
sources_content: Vec<Option<String>>,
names: Vec<Option<String>>,
/// Present even when it isn't an array, so `x_google_ignoreList` stays unused then.
ignore_list: Option<Vec<Option<u64>>>,
google_ignore_list: Option<Vec<Option<u64>>>,
mappings: Option<String>,
/// `Some(None)`: present but not an array.
sections: Option<Option<Vec<RawSection>>>,
}
impl RawMap {
/// The entry slots this map and its sections hold.
fn held(&self) -> usize {
let lists = |l: &Option<Vec<Option<u64>>>| l.as_ref().map_or(0, Vec::len);
self.sources.len()
+ self.sources_content.len()
+ self.names.len()
+ lists(&self.ignore_list)
+ lists(&self.google_ignore_list)
+ held_by(&self.sections)
}
}
fn held_by(sections: &Option<Option<Vec<RawSection>>>) -> usize {
sections.iter().flatten().flatten().map(|s| 1 + s.map.iter().flatten().map(|m| m.held()).sum::<usize>()).sum()
}
#[derive(Default)]
struct RawSection {
offset: Option<(Option<u64>, Option<u64>)>,
/// `Some(None)`: present but not an object. Boxed, so a section that isn't a map stays small.
map: Option<Option<Box<RawMap>>>,
/// Whether the URL is a string.
url: Option<bool>,
}
/// Visitor methods that read any other kind of value, without keeping it, as the default.
macro_rules! skip {
($de:lifetime; bool) => { fn visit_bool<E: de::Error>(self, _: bool) -> Result<Self::Value, E> { Ok(Default::default()) } };
($de:lifetime; i64) => { fn visit_i64<E: de::Error>(self, _: i64) -> Result<Self::Value, E> { Ok(Default::default()) } };
($de:lifetime; u64) => { fn visit_u64<E: de::Error>(self, _: u64) -> Result<Self::Value, E> { Ok(Default::default()) } };
($de:lifetime; f64) => { fn visit_f64<E: de::Error>(self, _: f64) -> Result<Self::Value, E> { Ok(Default::default()) } };
($de:lifetime; str) => { fn visit_str<E: de::Error>(self, _: &str) -> Result<Self::Value, E> { Ok(Default::default()) } };
($de:lifetime; unit) => { fn visit_unit<E: de::Error>(self) -> Result<Self::Value, E> { Ok(Default::default()) } };
($de:lifetime; seq) => {
fn visit_seq<A: SeqAccess<$de>>(self, mut seq: A) -> Result<Self::Value, A::Error> {
while seq.next_element_seed(Skip)?.is_some() {}
Ok(Default::default())
}
};
($de:lifetime; map) => {
fn visit_map<A: MapAccess<$de>>(self, mut map: A) -> Result<Self::Value, A::Error> {
while map.next_key_seed(Skip)?.is_some() {
map.next_value_seed(Skip)?;
}
Ok(Default::default())
}
};
($de:lifetime; $($kind:ident),+) => { $(skip!($de; $kind);)+ };
}
/// Read a value the map doesn't keep, with the checks a full read makes (string escapes and
/// UTF-8, number range, nesting depth), so skipping it can't let a broken map through.
#[derive(Clone, Copy)]
struct Skip;
impl<'de> DeserializeSeed<'de> for Skip {
type Value = ();
fn deserialize<D: de::Deserializer<'de>>(self, d: D) -> Result<(), D::Error> {
d.deserialize_any(self)
}
}
impl<'de> Visitor<'de> for Skip {
type Value = ();
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
skip!('de; bool, i64, u64, f64, str, unit, seq, map);
}
/// A string, or `None` for any other value.
#[derive(Clone, Copy)]
struct MaybeStr;
impl<'de> Visitor<'de> for MaybeStr {
type Value = Option<String>;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
fn visit_str<E: de::Error>(self, s: &str) -> Result<Self::Value, E> {
Ok(Some(s.to_owned()))
}
fn visit_string<E: de::Error>(self, s: String) -> Result<Self::Value, E> {
Ok(Some(s))
}
skip!('de; bool, i64, u64, f64, unit, seq, map);
}
/// A non-negative integer, or `None` for any other value.
#[derive(Clone, Copy)]
struct MaybeU64;
impl<'de> Visitor<'de> for MaybeU64 {
type Value = Option<u64>;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
fn visit_u64<E: de::Error>(self, n: u64) -> Result<Self::Value, E> {
Ok(Some(n))
}
skip!('de; bool, i64, f64, str, unit, seq, map);
}
/// One value read through `V`, after taking its slot from the entry budget.
struct Entry<'a, V>(&'a Entries, V);
impl<'de, V: Visitor<'de>> DeserializeSeed<'de> for Entry<'_, V> {
type Value = V::Value;
fn deserialize<D: de::Deserializer<'de>>(self, d: D) -> Result<Self::Value, D::Error> {
self.0.take()?;
d.deserialize_any(self.1)
}
}
/// A list whose entries count against the budget; any other value reads as an empty list.
struct EntryList<'a, V>(&'a Entries, V);
impl<'de, V: Visitor<'de> + Copy> Visitor<'de> for EntryList<'_, V> {
type Value = Vec<V::Value>;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Self::Value, A::Error> {
let mut out = Vec::new();
while let Some(value) = seq.next_element_seed(Entry(self.0, self.1))? {
out.push(value);
}
Ok(out)
}
skip!('de; bool, i64, u64, f64, str, unit, map);
}
impl<'de, V: Visitor<'de> + Copy> DeserializeSeed<'de> for EntryList<'_, V> {
type Value = Vec<V::Value>;
fn deserialize<D: de::Deserializer<'de>>(self, d: D) -> Result<Self::Value, D::Error> {
d.deserialize_any(self)
}
}
/// A map object, or `None` for any other value.
#[derive(Clone, Copy)]
struct RawMapSeed<'a>(&'a Entries);
impl<'de> Visitor<'de> for RawMapSeed<'_> {
type Value = Option<RawMap>;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
fn visit_map<A: MapAccess<'de>>(self, mut fields: A) -> Result<Self::Value, A::Error> {
let mut raw = RawMap::default();
// A repeated key replaces the earlier value, as it did when the whole JSON was read.
while let Some(key) = fields.next_key::<Field>()? {
match key {
Field::Version => raw.version = fields.next_value_seed(Any(MaybeU64))?,
Field::File => raw.file = fields.next_value_seed(Any(MaybeStr))?,
Field::SourceRoot => raw.source_root = fields.next_value_seed(Any(MaybeStr))?,
Field::Sources => {
self.0.give_back(std::mem::take(&mut raw.sources).len());
raw.sources = fields.next_value_seed(EntryList(self.0, MaybeStr))?;
}
Field::SourcesContent => {
self.0.give_back(std::mem::take(&mut raw.sources_content).len());
raw.sources_content = fields.next_value_seed(EntryList(self.0, MaybeStr))?;
}
Field::Names => {
self.0.give_back(std::mem::take(&mut raw.names).len());
raw.names = fields.next_value_seed(EntryList(self.0, MaybeStr))?;
}
// `ignoreList` wins over `x_google_ignoreList` wherever either appears.
Field::IgnoreList => {
let unused = raw.ignore_list.take().into_iter().chain(raw.google_ignore_list.take());
self.0.give_back(unused.map(|l| l.len()).sum());
raw.ignore_list = Some(fields.next_value_seed(EntryList(self.0, MaybeU64))?);
}
Field::GoogleIgnoreList if raw.ignore_list.is_some() => fields.next_value_seed(Skip)?,
Field::GoogleIgnoreList => {
self.0.give_back(raw.google_ignore_list.take().map_or(0, |l| l.len()));
raw.google_ignore_list = Some(fields.next_value_seed(EntryList(self.0, MaybeU64))?);
}
Field::Mappings => raw.mappings = fields.next_value_seed(Any(MaybeStr))?,
Field::Sections => {
self.0.give_back(held_by(&raw.sections.take()));
raw.sections = Some(fields.next_value_seed(Any(Sections(self.0)))?);
}
Field::Other => fields.next_value_seed(Skip)?,
}
}
Ok(Some(raw))
}
skip!('de; bool, i64, u64, f64, str, unit, seq);
}
impl<'de> DeserializeSeed<'de> for RawMapSeed<'_> {
type Value = Option<RawMap>;
fn deserialize<D: de::Deserializer<'de>>(self, d: D) -> Result<Self::Value, D::Error> {
d.deserialize_any(self)
}
}
/// Read any JSON value through `V`.
struct Any<V>(V);
impl<'de, V: Visitor<'de>> DeserializeSeed<'de> for Any<V> {
type Value = V::Value;
fn deserialize<D: de::Deserializer<'de>>(self, d: D) -> Result<Self::Value, D::Error> {
d.deserialize_any(self.0)
}
}
#[derive(serde::Deserialize)]
#[serde(field_identifier, rename_all = "camelCase")]
enum Field {
Version,
File,
SourceRoot,
Sources,
SourcesContent,
Names,
IgnoreList,
#[serde(rename = "x_google_ignoreList")]
GoogleIgnoreList,
Mappings,
Sections,
#[serde(other)]
Other,
}
/// The sections array, or `None` for any other value.
struct Sections<'a>(&'a Entries);
impl<'de> Visitor<'de> for Sections<'_> {
type Value = Option<Vec<RawSection>>;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Self::Value, A::Error> {
let mut out = Vec::new();
while let Some(section) = seq.next_element_seed(Entry(self.0, Section(self.0)))? {
out.push(section);
}
Ok(Some(out))
}
skip!('de; bool, i64, u64, f64, str, unit, map);
}
/// One section; any value that isn't an object reads as a section without an offset.
struct Section<'a>(&'a Entries);
impl<'de> Visitor<'de> for Section<'_> {
type Value = RawSection;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
fn visit_map<A: MapAccess<'de>>(self, mut fields: A) -> Result<Self::Value, A::Error> {
let mut section = RawSection::default();
while let Some(key) = fields.next_key::<SectionField>()? {
match key {
SectionField::Offset => section.offset = Some(fields.next_value_seed(Any(Offset))?),
SectionField::Map => {
self.0.give_back(section.map.take().flatten().map_or(0, |m| m.held()));
section.map = Some(fields.next_value_seed(RawMapSeed(self.0))?.map(Box::new));
}
SectionField::Url => section.url = Some(fields.next_value_seed(Any(MaybeStr))?.is_some()),
SectionField::Other => fields.next_value_seed(Skip)?,
}
}
Ok(section)
}
skip!('de; bool, i64, u64, f64, str, unit, seq);
}
#[derive(serde::Deserialize)]
#[serde(field_identifier, rename_all = "lowercase")]
enum SectionField {
Offset,
Map,
Url,
#[serde(other)]
Other,
}
/// A section offset's line and column; any value that isn't an object has neither.
struct Offset;
impl<'de> Visitor<'de> for Offset {
type Value = (Option<u64>, Option<u64>);
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
fn visit_map<A: MapAccess<'de>>(self, mut fields: A) -> Result<Self::Value, A::Error> {
let (mut line, mut column) = (None, None);
while let Some(key) = fields.next_key::<OffsetField>()? {
match key {
OffsetField::Line => line = fields.next_value_seed(Any(MaybeU64))?,
OffsetField::Column => column = fields.next_value_seed(Any(MaybeU64))?,
OffsetField::Other => fields.next_value_seed(Skip)?,
}
}
Ok((line, column))
}
skip!('de; bool, i64, u64, f64, str, unit, seq);
}
#[derive(serde::Deserialize)]
#[serde(field_identifier, rename_all = "lowercase")]
enum OffsetField {
Line,
Column,
#[serde(other)]
Other,
}
fn b64(c: u8) -> Option<i64> {
Some(match c {
b'A'..=b'Z' => i64::from(c - b'A'),
b'a'..=b'z' => i64::from(c - b'a') + 26,
b'0'..=b'9' => i64::from(c - b'0') + 52,
b'+' => 62,
b'/' => 63,
_ => return None,
})
}
/// Decode one VLQ field from `bytes` at `*pos`.
fn vlq(bytes: &[u8], pos: &mut usize) -> Option<i64> {
let mut result = 0u32;
let mut shift = 0u32;
loop {
let d = b64(*bytes.get(*pos)?)? as u32;
*pos += 1;
let bits = d & 31;
if bits != 0 {
if bits > u32::MAX.checked_shr(shift)? {
return None;
}
result |= bits << shift;
}
if d & 32 == 0 {
break;
}
// Redundant zero groups do not change the value.
shift = shift.saturating_add(5);
}
let magnitude = i64::from(result >> 1);
Some(match (result & 1, magnitude) {
(1, 0) => i64::from(i32::MIN),
(1, n) => -n,
_ => magnitude,
})
}
/// Decode mapping deltas, rejecting fields that cannot name a declared source or symbol.
fn decode_mappings(m: &str, map: &mut SourceMap, budget: &mut usize, line_budget: &mut usize, line_limit: usize) -> Result<(), SourceMapError> {
let bytes = m.as_bytes();
let (mut src, mut ol, mut oc, mut name) = (0i64, 0i64, 0i64, 0i64);
let mut pos = 0usize;
let mut line: Vec<Segment> = Vec::new();
let mut col = 0i64;
let mut count = 0usize;
let to_u32 = |v: i64, count: usize| u32::try_from(v).map_err(|_| SourceMapError::BadMappings(count));
while pos <= bytes.len() {
match bytes.get(pos) {
None | Some(b';') => {
if *line_budget == 0 {
return Err(SourceMapError::TooManyLines(line_limit));
}
*line_budget -= 1;
line.sort_by_key(|s| s.generated_column);
map.lines.try_reserve(1).map_err(|_| SourceMapError::LineAllocationFailed)?;
map.lines.push(std::mem::take(&mut line));
col = 0;
pos += 1;
continue;
}
Some(b',') => {
pos += 1;
continue;
}
_ => {}
}
if *budget == 0 {
return Err(SourceMapError::TooManySegments(count));
}
*budget -= 1;
count += 1;
col += vlq(bytes, &mut pos).ok_or(SourceMapError::BadMappings(count))?;
let mut seg = Segment { generated_column: to_u32(col, count)?, original: None };
if !matches!(bytes.get(pos), None | Some(b',') | Some(b';')) {
src += vlq(bytes, &mut pos).ok_or(SourceMapError::BadMappings(count))?;
ol += vlq(bytes, &mut pos).ok_or(SourceMapError::BadMappings(count))?;
oc += vlq(bytes, &mut pos).ok_or(SourceMapError::BadMappings(count))?;
let mut n = None;
if !matches!(bytes.get(pos), None | Some(b',') | Some(b';')) {
name += vlq(bytes, &mut pos).ok_or(SourceMapError::BadMappings(count))?;
n = Some(to_u32(name, count)?);
}
let source = to_u32(src, count)?;
if source as usize >= map.sources.len() || n.is_some_and(|index| index as usize >= map.names.len()) {
return Err(SourceMapError::BadMappings(count));
}
seg.original = Some((source, to_u32(ol, count)?, to_u32(oc, count)?, n));
}
if !matches!(bytes.get(pos), None | Some(b',') | Some(b';')) {
return Err(SourceMapError::BadMappings(count));
}
line.push(seg);
}
map.segments = count;
Ok(())
}
/// The segment, counted as [`decode_mappings`] does, that names a source past the end of a list
/// of `sources`. Mappings that can't be decoded end the search: past them nothing is known.
fn past_sources(m: &str, sources: usize) -> Option<usize> {
let bytes = m.as_bytes();
let (mut pos, mut src, mut count) = (0usize, 0i64, 0usize);
let at_end = |pos: usize| matches!(bytes.get(pos), None | Some(b',') | Some(b';'));
while pos < bytes.len() {
if matches!(bytes[pos], b',' | b';') {
pos += 1;
continue;
}
count += 1;
vlq(bytes, &mut pos)?;
if at_end(pos) {
continue;
}
src += vlq(bytes, &mut pos)?;
// A negative index is wrong too, but the deltas after it still decode.
if usize::try_from(src).is_ok_and(|src| src >= sources) {
return Some(count);
}
while !at_end(pos) {
vlq(bytes, &mut pos)?;
}
}
None
}
#[cfg(test)]
mod tests;
+122
View File
@@ -0,0 +1,122 @@
use super::*;
#[test]
fn mapping_indices_must_refer_to_declared_entries() {
for mappings in ["ACAA", "AAAAC"] {
let json = serde_json::json!({"version":3,"sources":["source.js"],"names":["name"],"mappings":mappings});
assert!(matches!(parse(json.to_string().as_bytes(), Limits::default()), Err(SourceMapError::BadMappings(1))));
}
let valid = br#"{"version":3,"sources":[null],"names":["name"],"mappings":"AAAAA"}"#;
assert!(parse(valid, Limits::default()).is_ok());
}
#[test]
fn lookup_keeps_coincident_mappings_and_stops_at_an_unmapped_segment() {
let map = parse(br#"{"version":3,"sources":["a","b"],"mappings":"EAAA,ACAA,E"}"#, Limits::default()).unwrap();
assert!(map.original_positions(0, 1).is_empty());
let matches = map.original_positions(0, 3);
assert_eq!(matches.iter().map(|p| p.source_index).collect::<Vec<_>>(), [0, 1]);
assert!(map.original_positions(0, 4).is_empty());
}
#[test]
fn relative_sources_keep_the_url_root_delimiter() {
for (root, expected) in [
("webpack:///", "webpack:///src/main.js"),
("file:///", "file:///src/main.js"),
("https://host/scripts/", "https://host/scripts/src/main.js"),
("src", "src/src/main.js"),
] {
let json = serde_json::json!({"version":3,"sourceRoot":root,"sources":["src/main.js"],"mappings":"AAAA"});
let map = parse(json.to_string().as_bytes(), Limits::default()).unwrap();
assert_eq!(map.all_sources(), [expected]);
assert_eq!(map.original_positions(0, 0)[0].source.as_deref(), Some(expected));
}
}
#[test]
fn decodes_and_looks_up() {
// Generated "a;b" from two lines of src.js.
let m = br#"{"version":3,"sources":["src.js"],"names":["foo"],"mappings":"AAAAA,EAAE;AACA","sourcesContent":["x\ny"]}"#;
let map = parse(m, Limits::default()).unwrap();
assert_eq!(map.lines.len(), 2);
let p = map.original_positions(0, 1);
assert_eq!(p[0].source.as_deref(), Some("src.js"));
assert_eq!(p[0].name.as_deref(), Some("foo"));
assert_eq!((p[0].line, p[0].column), (0, 0));
let q = map.original_positions(0, 2);
assert_eq!(q[0].column, 2);
let r = map.original_positions(1, 0);
assert_eq!(r[0].line, 1);
assert!(map.original_positions(5, 0).is_empty());
assert_eq!(map.source_content(0), Some("x\ny"));
}
#[test]
fn indexed_maps() {
let m = br#"{"version":3,"sections":[{"offset":{"line":0,"column":0},"map":{"version":3,"sources":["a.js"],"names":[],"mappings":"AAAA"}},{"offset":{"line":1,"column":0},"map":{"version":3,"sources":["b.js"],"names":[],"mappings":"AAAA"}}]}"#;
let map = parse(m, Limits::default()).unwrap();
assert_eq!(map.original_positions(1, 3)[0].source.as_deref(), Some("b.js"));
assert_eq!(map.all_sources(), vec!["a.js", "b.js"]);
}
#[test]
fn validates_index_map_shape_and_offsets() {
let inline = serde_json::json!({"version":3,"mappings":"A"});
for sections in [
serde_json::json!({}),
serde_json::json!([{"map":inline}]),
serde_json::json!([{"offset":{"line":-1,"column":0},"map":inline}]),
serde_json::json!([{"offset":{"line":0,"column":4294967296_u64},"map":inline}]),
serde_json::json!([{"offset":{"line":0,"column":0}}]),
serde_json::json!([{"offset":{"line":0,"column":0},"map":inline,"url":"later.map"}]),
serde_json::json!([{"offset":{"line":0,"column":2},"map":inline},{"offset":{"line":0,"column":1},"map":inline}]),
] {
let json = serde_json::json!({"version":3,"sections":sections}).to_string();
assert_eq!(parse(json.as_bytes(), Limits::default()), Err(SourceMapError::BadSections));
}
}
#[test]
fn an_external_section_ends_the_previous_inline_mapping() {
let inline = serde_json::json!({"version":3,"sources":["first.js"],"mappings":"AAAA"});
let sections = serde_json::json!([
{"offset":{"line":0,"column":0},"map":inline},
{"offset":{"line":0,"column":4},"url":"external.map"},
{"offset":{"line":1,"column":0},"map":inline}
]);
let json = serde_json::json!({"version":3,"sections":sections}).to_string();
let map = parse(json.as_bytes(), Limits::default()).unwrap();
assert_eq!(map.original_positions(0, 3)[0].source.as_deref(), Some("first.js"));
assert!(map.original_positions(0, 4).is_empty());
assert_eq!(map.original_positions(1, 0)[0].generated_column, 0);
}
#[test]
fn vlq_signed_extremes_and_overflow() {
for (field, value) in [
("B", Some(i64::from(i32::MIN))),
("+/////D", Some(i64::from(i32::MAX))),
("//////D", Some(-i64::from(i32::MAX))),
("ggggggE", None),
("ggggggggggA", Some(0)),
] {
assert_eq!(vlq(field.as_bytes(), &mut 0), value, "{field}");
}
let map = parse(br#"{"version":3,"mappings":"+/////D,C,B"}"#, Limits::default()).unwrap();
assert_eq!(map.lines[0].iter().map(|s| s.generated_column).collect::<Vec<_>>(), [0, 2147483647, 2147483648]);
assert!(parse(br#"{"version":3,"mappings":"B"}"#, Limits::default()).is_err());
}
#[test]
fn hostile_mappings_do_not_panic() {
for m in ["", ";;;", "A", "gggggggggggggggggggggggggg", "AAAA,", "////", "AAAAAAAAAAAAAAA", "D", "AD,AA", "+", "!!"] {
let j = format!(r#"{{"version":3,"sources":[],"names":[],"mappings":"{m}"}}"#);
if let Ok(map) = parse(j.as_bytes(), Limits::default()) {
let _ = map.original_positions(0, 0);
let _ = map.original_positions(u32::MAX, u32::MAX);
}
}
let tight = Limits { max_segments: 1, ..Limits::default() };
assert!(parse(br#"{"version":3,"sources":[],"names":[],"mappings":"A,A"}"#, tight).is_err());
assert!(parse(b"[]", Limits::default()).is_err());
}
+145
View File
@@ -0,0 +1,145 @@
use craft_sourcemap::{Limits, SourceMap, SourceMapError, parse};
use serde_json::json;
fn with(max_entries: usize) -> Limits {
Limits { max_entries, ..Limits::default() }
}
fn decode(value: serde_json::Value, max_entries: usize) -> Result<SourceMap, SourceMapError> {
parse(&serde_json::to_vec(&value).unwrap(), with(max_entries))
}
#[test]
fn entries_up_to_the_limit_are_kept() {
let value = json!({"version":3,"sources":["a.js","b.js"],"sourcesContent":["a();",null],"names":["x"],"mappings":"AAAAA,CCAA"});
let map = decode(value.clone(), 5).unwrap();
assert_eq!(map.sources, vec![Some("a.js".to_string()), Some("b.js".to_string())]);
assert_eq!(map.sources_content, vec![Some("a();".to_string()), None]);
assert_eq!(map.names, vec!["x".to_string()]);
assert_eq!(decode(value, 4), Err(SourceMapError::TooManyEntries(4)));
}
#[test]
fn every_kind_of_entry_counts_once() {
for value in [
json!({"version":3,"sources":["a","b","c"],"mappings":""}),
json!({"version":3,"sourcesContent":["a","b","c"],"mappings":""}),
json!({"version":3,"names":["a","b","c"],"mappings":""}),
json!({"version":3,"ignoreList":[0,1,2],"mappings":""}),
json!({"version":3,"x_google_ignoreList":[0,1,2],"mappings":""}),
// Entries that are not strings still take a slot each.
json!({"version":3,"sources":[null, 1, {"a":[1,2,3]}],"mappings":""}),
] {
assert_eq!(decode(value.clone(), 2), Err(SourceMapError::TooManyEntries(2)), "{value}");
assert!(decode(value.clone(), 3).is_ok(), "{value}");
}
}
#[test]
fn entries_a_repeated_key_replaces_give_their_slots_back() {
let json = br#"{"version":3,"mappings":"","sources":["a","b"],"sources":["c"],"names":["x"],"names":[]}"#;
let map = parse(json, with(2)).unwrap();
assert_eq!(map.sources, vec![Some("c".to_string())]);
assert!(map.names.is_empty());
// `ignoreList` is kept over `x_google_ignoreList`, before or after it.
let first = br#"{"version":3,"mappings":"","x_google_ignoreList":[0,0],"ignoreList":[0]}"#;
assert_eq!(parse(first, with(2)).unwrap().ignore_list, vec![0]);
let after = br#"{"version":3,"mappings":"","ignoreList":[0],"x_google_ignoreList":[0,0,0]}"#;
assert_eq!(parse(after, with(1)).unwrap().ignore_list, vec![0]);
// A replaced sections array gives back its sections and what their maps held: without that,
// the second array would need slots four and five.
let sections = br#"{"version":3,"sections":[{"offset":{"line":0,"column":0},"map":{"version":3,"sources":["a"],"mappings":""}},{"offset":{"line":1,"column":0},"url":"b.map"}],"sections":[{"offset":{"line":0,"column":0},"url":"c.map"}]}"#;
assert_eq!(parse(sections, with(3)).unwrap().sections.len(), 1);
// So does a section's replaced map: one section, then one source instead of two.
let map = br#"{"version":3,"sections":[{"offset":{"line":0,"column":0},"map":{"version":3,"sources":["a","b"],"mappings":""},"map":{"version":3,"sources":["c"],"mappings":""}}]}"#;
assert_eq!(parse(map, with(3)).unwrap().sections[0].2.sources, vec![Some("c".to_string())]);
}
#[test]
fn sections_and_their_maps_share_the_budget() {
let section = |line: u32, name: &str| json!({"offset":{"line":line,"column":0},"map":{"version":3,"sources":[name],"mappings":"AAAA"}});
let value = json!({"version":3,"sections":[section(0, "a.js"), section(1, "b.js")]});
// Two sections and one source in each.
assert_eq!(decode(value.clone(), 4).unwrap().sections.len(), 2);
assert_eq!(decode(value, 3), Err(SourceMapError::TooManyEntries(3)));
// Sections without a map count too, so a long list of them is refused early.
let external = json!({"offset":{"line":0,"column":0},"url":"a.map"});
let value = json!({"version":3,"sections":[external.clone(), external.clone(), external]});
assert_eq!(decode(value.clone(), 3).unwrap().sections.len(), 3);
assert_eq!(decode(value, 2), Err(SourceMapError::TooManyEntries(2)));
let not_sections = br#"{"version":3,"sections":[0,0,0]}"#;
assert_eq!(parse(not_sections, with(2)), Err(SourceMapError::TooManyEntries(2)));
assert_eq!(parse(not_sections, with(3)), Err(SourceMapError::BadSections));
}
#[test]
fn reading_stops_at_the_limit() {
// Everything after the entry that goes over the limit is never read: the broken tail would
// otherwise be reported as bad JSON.
let json = br#"{"version":3,"sources":["a","b","c","d" this is not JSON"#;
assert_eq!(parse(json, with(2)), Err(SourceMapError::TooManyEntries(2)));
assert!(matches!(parse(json, Limits::default()), Err(SourceMapError::BadJson(_))));
}
#[test]
fn unknown_fields_are_skipped_without_counting() {
let value = json!({"version":3,"x_vendor":[1,2,3,4,5,6,7,8],"sources":["a.js"],"mappings":"AAAA"});
assert_eq!(decode(value, 1).unwrap().sources.len(), 1);
}
#[test]
fn values_the_map_does_not_keep_are_still_checked() {
let deep = format!("{{\"version\":3,\"mappings\":\"\",\"x\":{}{}}}", "[".repeat(200), "]".repeat(200));
for json in [
br#"{"version":3,"mappings":"","x":"\ud800"}"#.as_slice(),
br#"{"version":3,"mappings":"","sources":{"a":"\udc00"}}"#,
br#"{"version":3,"mappings":"","x":1e400}"#,
br#"{"version":3,"mappings":"","sources":[{"a":1e400}]}"#,
b"{\"version\":3,\"mappings\":\"\",\"x\":\"\xff\"}",
br#"{"version":3,"mappings":"","x":[1,]}"#,
deep.as_bytes(),
] {
let got = parse(json, Limits::default());
let full = serde_json::from_slice::<serde_json::Value>(json).unwrap_err().to_string();
assert_eq!(got, Err(SourceMapError::BadJson(full)), "{}", String::from_utf8_lossy(json));
}
}
#[test]
fn a_hostile_map_of_empty_names_is_refused_by_default() {
// About 21 million empty strings fit in the 64 MiB a map may have.
let limit = Limits::default().max_entries;
let names = vec!["\"\""; limit + 1].join(",");
let json = format!("{{\"version\":3,\"mappings\":\"\",\"names\":[{names}]}}").into_bytes();
assert_eq!(parse(&json, Limits::default()), Err(SourceMapError::TooManyEntries(limit)));
// The same map is well formed: with room for every name it reads.
assert_eq!(parse(&json, with(limit + 1)).unwrap().names.len(), limit + 1);
}
#[test]
fn sources_are_read_from_a_map_with_broken_mappings_or_offsets() {
use craft_sourcemap::parse_sources;
let bad = br#"{"version":3,"sourceRoot":"src","sources":["a.js"],"sourcesContent":["a();"],"names":["x"],"mappings":"!!"}"#;
assert_eq!(parse(bad, Limits::default()), Err(SourceMapError::BadMappings(1)));
let map = parse_sources(bad, Limits::default()).unwrap();
assert_eq!((map.source_path(0).as_deref(), map.sources_content.len(), map.names.len(), map.lines.len()), (Some("src/a.js"), 1, 1, 0));
let unordered = br#"{"version":3,"sections":[{"offset":{"line":2,"column":0},"map":{"version":3,"sources":["a.js"],"mappings":"AAAA"}},{"offset":{"line":-1,"column":0},"map":{"version":3,"sources":["b.js"],"mappings":""}},{"offset":{"line":0,"column":0},"map":{"version":3,"sources":["c.js"],"mappings":"!"}}]}"#;
assert_eq!(parse(unordered, Limits::default()), Err(SourceMapError::BadSections));
let map = parse_sources(unordered, Limits::default()).unwrap();
assert_eq!(map.source_inventory(Default::default()).sources, vec!["a.js".to_string(), "b.js".to_string(), "c.js".to_string()]);
// Past mappings that can't be decoded, the indices aren't known.
assert_eq!(parse_sources(br#"{"version":3,"sources":["a"],"mappings":"AAAA,!,ACAA"}"#, Limits::default()).unwrap().sources.len(), 1);
// What a section holds must still be known, and the version and limits still apply.
for (json, limit, err) in [
(br#"{"version":3,"sections":[{"offset":{"line":0,"column":0}}]}"#.as_slice(), 9, SourceMapError::BadSections),
(br#"{"version":3,"sections":{}}"#, 9, SourceMapError::BadSections),
(br#"{"version":3,"sections":[{"offset":{"line":0,"column":0},"map":{"version":2,"sources":["a"]}}]}"#, 9, SourceMapError::BadVersion),
(br#"{"version":3,"sources":["a","b"],"mappings":"!"}"#, 1, SourceMapError::TooManyEntries(1)),
// Mappings that name a source past the end of the list show it is short.
(br#"{"version":3,"mappings":"AAAA"}"#, 9, SourceMapError::BadMappings(1)),
(br#"{"version":3,"sources":["a"],"mappings":"AAAA;ACAA,!"}"#, 9, SourceMapError::BadMappings(2)),
(br#"{"version":3,"sources":["a"],"mappings":"AAAA,ADAA,AEAA"}"#, 9, SourceMapError::BadMappings(3)),
] {
assert_eq!(parse_sources(json, with(limit)), Err(err), "{}", String::from_utf8_lossy(json));
}
}
@@ -0,0 +1,84 @@
//! An index-map section that points at an external map (`url`) adds no sources here, but its
//! sources are unknown rather than absent: the inventory says how many such sections it saw.
use craft_sourcemap::{InventoryLimits, Limits, SourceMap, parse, parse_sources};
use serde_json::{Value, json};
fn decode(value: Value) -> SourceMap {
parse(&serde_json::to_vec(&value).unwrap(), Limits::default()).unwrap()
}
/// An external section adds no sources but is counted, so the inventory can say part of the map
/// was never read.
#[test]
fn an_external_section_is_counted_and_adds_no_sources() {
let map = decode(json!({
"version": 3,
"sections": [
{"offset": {"line": 0, "column": 0}, "map": {"version": 3, "sources": ["a.js"], "mappings": ""}},
{"offset": {"line": 10, "column": 0}, "url": "vendor.js.map"}
]
}));
assert_eq!(map.all_sources(), ["a.js"]);
assert_eq!(map.external_sections, 1);
let inventory = map.source_inventory(InventoryLimits::default());
assert_eq!(inventory.sources, ["a.js"]);
assert!(!inventory.truncated);
assert_eq!(inventory.external_sections, 1);
}
/// The count reaches an external section inside a nested index map.
#[test]
fn external_sections_are_counted_through_nested_index_maps() {
let map = decode(json!({
"version": 3,
"sections": [
{"offset": {"line": 0, "column": 0}, "map": {"version": 3, "sections": [
{"offset": {"line": 0, "column": 0}, "map": {"version": 3, "sources": ["a.js"], "mappings": ""}},
{"offset": {"line": 4, "column": 0}, "url": "inner.js.map"}
]}},
{"offset": {"line": 20, "column": 0}, "url": "outer.js.map"}
]
}));
assert_eq!(map.all_sources(), ["a.js"]);
assert_eq!(map.source_inventory(InventoryLimits::default()).external_sections, 2);
}
/// A map with only inline sections reports nothing unread.
#[test]
fn an_inline_only_map_counts_no_external_sections() {
let map = decode(json!({
"version": 3,
"sections": [{"offset": {"line": 0, "column": 0}, "map": {"version": 3, "sources": ["a.js"], "mappings": ""}}]
}));
assert_eq!(map.source_inventory(InventoryLimits::default()).external_sections, 0);
}
/// A map read for its sources alone (its mappings are refused) still says an external section
/// left part of the list unread.
#[test]
fn a_sources_only_read_still_counts_external_sections() {
let bytes = serde_json::to_vec(&json!({
"version": 3,
"sections": [
{"offset": {"line": 0, "column": 0}, "map": {"version": 3, "sources": ["a.js"]}},
{"offset": {"line": 8, "column": 0}, "url": "vendor.js.map"}
]
}))
.unwrap();
let map = parse_sources(&bytes, Limits::default()).unwrap();
assert_eq!(map.source_inventory(InventoryLimits::default()).external_sections, 1);
}
/// The count spends no inventory budget: a map with many external sections still retains its
/// inline sources in full.
#[test]
fn many_external_sections_do_not_spend_the_source_budget() {
let mut sections: Vec<Value> = (0..1_000).map(|i| json!({"offset": {"line": i, "column": 0}, "url": format!("{i}.map")})).collect();
sections.push(json!({"offset": {"line": 1_000, "column": 0}, "map": {"version": 3, "sources": ["a.js", "b.js"], "mappings": ""}}));
let map = decode(json!({"version": 3, "sections": sections}));
let inventory = map.source_inventory(InventoryLimits { max_sources: 2, max_bytes: 16 });
assert_eq!(inventory.sources, ["a.js", "b.js"]);
assert!(!inventory.truncated);
assert_eq!(inventory.external_sections, 1_000);
}
+104
View File
@@ -0,0 +1,104 @@
use craft_sourcemap::{InventoryLimits, Limits, SourceMap, parse};
use serde_json::{Value, json};
fn decode(value: Value) -> SourceMap {
parse(&serde_json::to_vec(&value).unwrap(), Limits::default()).unwrap()
}
#[test]
fn all_sources_bounds_expanded_path_bytes_before_collecting() {
let root = "r".repeat(2 * 1024 * 1024);
let bytes = serde_json::to_vec(&json!({
"version": 3, "sourceRoot": root, "sources": vec!["/x.js"; 40], "mappings": ""
}))
.unwrap();
let map = parse(&bytes, Limits::default()).unwrap();
let sources = map.all_sources();
assert!(sources.iter().map(String::len).sum::<usize>() <= 64 * 1024 * 1024);
assert_eq!(sources.len(), 31);
assert!(sources.iter().all(|source| source.ends_with("//x.js")));
}
#[test]
fn repeated_large_roots_stop_before_relative_and_absolute_looking_paths_expand() {
for source in ["x.js", "/x.js", "https://example.test/x.js"] {
let bytes = serde_json::to_vec(&json!({
"version": 3, "sourceRoot": "r".repeat(1024 * 1024),
"sources": vec![source; 10_000], "mappings": ""
}))
.unwrap();
assert!(bytes.len() < 2 * 1024 * 1024);
let map = parse(&bytes, Limits::default()).unwrap();
let inventory = map.source_inventory(InventoryLimits { max_sources: 5000, max_bytes: 2 * 1024 * 1024 });
assert_eq!(inventory.sources.len(), 1);
assert_eq!(inventory.sources[0].len(), 1024 * 1024 + 1 + source.len());
assert!(inventory.sources[0].ends_with(source));
assert!(inventory.truncated);
}
}
#[test]
fn byte_limits_preserve_complete_utf8_identities_at_the_exact_boundary() {
let map = decode(json!({"version": 3, "sourceRoot": "é/", "sources": ["東京.js", "東京.js", null], "mappings": ""}));
let expected = "é/東京.js";
let limits = InventoryLimits { max_sources: 2, max_bytes: 2 * expected.len() };
let exact = map.source_inventory(limits);
assert_eq!(exact.sources, [expected, expected]);
assert!(!exact.truncated);
let short = map.source_inventory(InventoryLimits { max_bytes: limits.max_bytes - 1, ..limits });
assert_eq!(short.sources, [expected]);
assert!(short.truncated);
let too_small = map.source_inventory(InventoryLimits { max_bytes: expected.len() - 1, ..limits });
assert!(too_small.sources.is_empty());
assert!(too_small.truncated);
}
#[test]
fn entry_limits_count_empty_paths_but_not_null_sources() {
let map = decode(json!({"version": 3, "sourceRoot": "", "sources": [null, "", "", null], "mappings": ""}));
let exact = map.source_inventory(InventoryLimits { max_sources: 2, max_bytes: 0 });
assert_eq!(exact.sources, ["", ""]);
assert!(!exact.truncated);
let one = map.source_inventory(InventoryLimits { max_sources: 1, max_bytes: 0 });
assert_eq!(one.sources, [""]);
assert!(one.truncated);
let zero = map.source_inventory(InventoryLimits { max_sources: 0, max_bytes: 0 });
assert!(zero.sources.is_empty());
assert!(zero.truncated);
let absent = decode(json!({"version": 3, "sources": [null], "mappings": ""}));
assert!(!absent.source_inventory(InventoryLimits { max_sources: 0, max_bytes: 0 }).truncated);
}
#[test]
fn indexed_sections_share_entry_and_byte_limits_in_source_order() {
let map = decode(json!({"version": 3, "sourceRoot": "outer", "sections": [
{"offset": {"line": 0, "column": 0}, "map": {
"version": 3, "sourceRoot": "src", "sources": [null, "/first.js"], "mappings": ""
}},
{"offset": {"line": 1, "column": 0}, "url": "external.map"},
{"offset": {"line": 2, "column": 0}, "map": {"version": 3, "sections": [
{"offset": {"line": 0, "column": 0}, "map": {
"version": 3, "sourceRoot": "file:///", "sources": ["second.js", "third.js"], "mappings": ""
}}
]}}
]}));
let expected = ["src//first.js", "file:///second.js", "file:///third.js"];
let complete = map.source_inventory(InventoryLimits { max_sources: 3, max_bytes: expected.iter().map(|s| s.len()).sum() });
assert_eq!(complete.sources, expected);
assert!(!complete.truncated);
let bytes = map.source_inventory(InventoryLimits { max_sources: 3, max_bytes: expected[0].len() + expected[1].len() });
assert_eq!(bytes.sources, expected[..2]);
assert!(bytes.truncated);
let entries = map.source_inventory(InventoryLimits { max_sources: 1, max_bytes: 100 });
assert_eq!(entries.sources, expected[..1]);
assert!(entries.truncated);
}
#[test]
fn an_oversized_path_stops_the_prefix_without_shortening_or_skipping_identities() {
let map = decode(json!({"version": 3, "sources": ["a.js", "b".repeat(40), "c.js"], "mappings": ""}));
let inventory = map.source_inventory(InventoryLimits { max_sources: 3, max_bytes: 8 });
assert_eq!(inventory.sources, ["a.js"]);
assert!(inventory.truncated);
assert_eq!(map.source_path(1).unwrap(), "b".repeat(40));
}
+72
View File
@@ -0,0 +1,72 @@
use craft_sourcemap as sourcemap;
use craft_sourcemap::{Limits, SourceMapError};
use serde_json::{Value, json};
fn decode(value: Value, limits: Limits) -> Result<sourcemap::SourceMap, SourceMapError> {
sourcemap::parse(&serde_json::to_vec(&value).unwrap(), limits)
}
fn map(mappings: &str) -> Value {
json!({"version":3,"sources":["input.js"],"names":[],"mappings":mappings})
}
#[test]
fn empty_lines_spend_the_default_budget_even_with_no_segments() {
let limits = Limits { max_segments: 0, ..Limits::default() };
assert_eq!(decode(map(&";".repeat(1_000_000)), limits), Err(SourceMapError::TooManyLines(1_000_000)));
}
#[test]
fn exact_and_zero_line_limits_include_the_final_empty_line() {
let limits = Limits { max_lines: 3, max_segments: 0, ..Limits::default() };
let parsed = decode(map(";;"), limits).unwrap();
assert_eq!(parsed.lines.len(), 3);
assert_eq!(parsed.segments, 0);
assert_eq!(decode(map(";;;"), limits), Err(SourceMapError::TooManyLines(3)));
assert_eq!(decode(map(""), Limits { max_lines: 0, ..limits }), Err(SourceMapError::TooManyLines(0)));
assert_eq!(decode(map(""), Limits { max_lines: 1, ..limits }).unwrap().lines.len(), 1);
}
#[test]
fn nested_sections_share_one_generated_line_budget() {
let value = json!({"version":3,"sections":[
{"offset":{"line":0,"column":0},"map":map(";")},
{"offset":{"line":2,"column":0},"map":{"version":3,"sections":[
{"offset":{"line":0,"column":0},"map":map(";")}
]}}
]});
let limits = Limits { max_lines: 3, max_segments: 0, ..Limits::default() };
assert_eq!(decode(value.clone(), limits), Err(SourceMapError::TooManyLines(3)));
let parsed = decode(value, Limits { max_lines: 4, ..limits }).unwrap();
assert_eq!(parsed.sections[0].2.lines.len(), 2);
assert_eq!(parsed.sections[1].2.sections[0].2.lines.len(), 2);
}
#[test]
fn external_sections_do_not_consume_undecoded_lines() {
let value = json!({"version":3,"sections":[
{"offset":{"line":0,"column":0},"url":"other.map"},
{"offset":{"line":2,"column":0},"map":map("")}
]});
let parsed = decode(value, Limits { max_lines: 1, max_segments: 0, ..Limits::default() }).unwrap();
assert!(parsed.sections[0].2.lines.is_empty());
assert_eq!(parsed.sections[1].2.lines.len(), 1);
}
#[test]
fn positions_after_empty_lines_keep_their_coordinates() {
let parsed = decode(map("AAAA;;AACA"), Limits { max_lines: 3, max_segments: 2, ..Limits::default() }).unwrap();
assert_eq!(parsed.lines.len(), 3);
assert!(parsed.original_positions(1, 0).is_empty());
let position = &parsed.original_positions(2, 0)[0];
assert_eq!(position.source.as_deref(), Some("input.js"));
assert_eq!((position.line, position.column), (1, 0));
}
#[test]
fn line_and_segment_limits_are_independent() {
let limits = Limits { max_lines: 10, max_segments: 1, ..Limits::default() };
assert!(matches!(decode(map("A,A"), limits), Err(SourceMapError::TooManySegments(_))));
assert_eq!(decode(map(";"), Limits { max_lines: 1, max_segments: 0, ..limits }), Err(SourceMapError::TooManyLines(1)));
assert_eq!(decode(map(";"), Limits { max_lines: 2, max_segments: 0, ..limits }).unwrap().segments, 0);
}
+80
View File
@@ -0,0 +1,80 @@
use craft_sourcemap::{Limits, SourceMap, parse};
use serde_json::{Value, json};
fn decode(value: Value) -> SourceMap {
parse(&serde_json::to_vec(&value).unwrap(), Limits::default()).unwrap()
}
#[test]
fn root_prefix_precedes_absolute_looking_sources() {
for (root, source, expected) in [
("webpack:///", "/src/main.js", "webpack:////src/main.js"),
("https://example.test/build", "/src/main.js", "https://example.test/build//src/main.js"),
("src", "https://example.test/main.js", "src/https://example.test/main.js"),
("src/", "//example.test/main.js", "src///example.test/main.js"),
("sources", "node:internal/main", "sources/node:internal/main"),
] {
let map = decode(json!({"version": 3, "sourceRoot": root, "sources": [source], "mappings": "AAAA"}));
assert_eq!(map.original_positions(0, 0)[0].source.as_deref(), Some(expected), "root {root:?}, source {source:?}");
assert_eq!(map.all_sources(), [expected]);
}
}
#[test]
fn empty_root_keeps_the_same_identity_as_an_absent_or_null_root() {
for source in ["src/main.js", "/src/main.js", "https://example.test/main.js"] {
let mut value = json!({"version": 3, "sources": [source], "mappings": "AAAA"});
assert_eq!(decode(value.clone()).all_sources(), [source]);
value["sourceRoot"] = Value::Null;
assert_eq!(decode(value.clone()).all_sources(), [source]);
value["sourceRoot"] = json!("");
let map = decode(value);
assert_eq!(map.all_sources(), [source]);
assert_eq!(map.original_positions(0, 0)[0].source.as_deref(), Some(source));
}
}
#[test]
fn each_source_keeps_its_index_and_embedded_content() {
let map = decode(json!({
"version": 3, "sourceRoot": "https://example.test/originals/",
"sources": ["/first.js", null, "https://other.test/third.js"],
"sourcesContent": ["first();", null, "third();"], "mappings": "AAAA,CEAA"
}));
let first = map.original_positions(0, 0);
let third = map.original_positions(0, 1);
assert_eq!(first[0].source_index, 0);
assert_eq!(third[0].source_index, 2);
assert_eq!(first[0].source.as_deref(), Some("https://example.test/originals//first.js"));
assert_eq!(third[0].source.as_deref(), Some("https://example.test/originals/https://other.test/third.js"));
assert_eq!(map.source_content(0), Some("first();"));
assert_eq!(map.source_content(2), Some("third();"));
assert_eq!(map.source_path(1), None);
assert_eq!(map.source_path(3), None);
}
#[test]
fn indexed_maps_use_each_embedded_maps_own_root() {
let map = decode(json!({
"version": 3, "sourceRoot": "outer", "sections": [
{"offset": {"line": 0, "column": 3}, "map": {
"version": 3, "sourceRoot": "file:///", "sources": ["/first.js"], "mappings": "AAAA"
}},
{"offset": {"line": 1, "column": 0}, "map": {
"version": 3, "sources": ["second.js"], "mappings": "AAAA"
}}
]
}));
assert_eq!(map.original_positions(0, 3)[0].source.as_deref(), Some("file:////first.js"));
assert_eq!(map.original_positions(1, 0)[0].source.as_deref(), Some("second.js"));
assert_eq!(map.all_sources(), ["file:////first.js", "second.js"]);
}
#[test]
fn relative_sources_still_preserve_url_root_delimiters() {
for (root, expected) in [("webpack:///", "webpack:///main.js"), ("file:///", "file:///main.js"), ("src", "src/main.js"), ("src/", "src/main.js")]
{
let map = decode(json!({"version": 3, "sourceRoot": root, "sources": ["main.js"], "mappings": "AAAA"}));
assert_eq!(map.all_sources(), [expected]);
}
}
+24
View File
@@ -0,0 +1,24 @@
use craft_sourcemap::{Limits, parse};
use serde_json::json;
#[test]
fn indexed_lookup_reports_columns_in_the_generated_file() {
let json = json!({"version":3,"sections":[{"offset":{"line":3,"column":11},"map":{"version":3,"sources":["one.js"],"mappings":"IAAA;IACA"}}]});
let map = parse(json.to_string().as_bytes(), Limits::default()).unwrap();
assert!(map.original_positions(3, 14).is_empty());
let first = map.original_positions(3, 17);
assert_eq!(first[0].generated_column, 15);
assert_eq!((first[0].line, first[0].column), (0, 0));
assert_eq!(map.original_positions(4, 4)[0].generated_column, 4);
}
#[test]
fn nested_section_columns_are_added_only_on_each_first_line() {
for (inner_line, query_line, query_column) in [(0, 3, 18), (1, 4, 7)] {
let json = json!({"version":3,"sections":[{"offset":{"line":3,"column":11},"map":{"version":3,"sections":[{"offset":{"line":inner_line,"column":3},"map":{"version":3,"sources":["nested.js"],"mappings":"IAAA"}}]}}]});
let map = parse(json.to_string().as_bytes(), Limits::default()).unwrap();
let positions = map.original_positions(query_line, query_column);
assert_eq!(positions[0].generated_column, query_column);
assert_eq!(positions[0].source.as_deref(), Some("nested.js"));
}
}
+15
View File
@@ -0,0 +1,15 @@
[package]
name = "craft-wasm"
description = "Bounded WebAssembly module/component inspection, validation reports and WAT rendering; never executes code"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
serde = { workspace = true }
thiserror = { workspace = true }
wasmparser = { version = "0.261", default-features = false, features = ["std", "validate", "features", "component-model", "simd"] }
wasmprinter = { version = "0.261", default-features = false, features = ["component-model"] }
+32
View File
@@ -0,0 +1,32 @@
// Checked reader subset retained from Craft Anything; MIT OR Apache-2.0.
pub(crate) struct Reader<'a> {
data: &'a [u8],
}
impl<'a> Reader<'a> {
pub fn new(data: &'a [u8]) -> Self {
Self { data }
}
pub fn u8(&self, off: usize) -> Option<u8> {
self.data.get(off).copied()
}
pub fn slice(&self, off: usize, len: usize) -> Option<&'a [u8]> {
self.data.get(off..off.checked_add(len)?)
}
/// Unsigned LEB128 at `off`: (value, bytes consumed). At most 10 bytes.
pub fn uleb128(&self, off: usize) -> Option<(u64, usize)> {
let mut result = 0u64;
for i in 0..10usize {
let b = self.u8(off.checked_add(i)?)?;
let payload = u64::from(b & 0x7f);
let shift = 7 * i as u32;
if shift > 0 && payload.checked_shl(shift)? >> shift != payload {
return None;
}
result |= payload << shift;
if b & 0x80 == 0 {
return Some((result, i + 1));
}
}
None
}
}
+973
View File
@@ -0,0 +1,973 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
//! WebAssembly modules: validation, sections, imports, exports, functions with their signatures
//! and names, memories, tables, globals, tags, custom sections (names, producers, target features,
//! source map URL) and a text (WAT) rendering. Built on the pure-Rust `wasmparser` and
//! `wasmprinter` crates; never executes anything.
use crate::bytes::Reader;
use WasmError as FormatError;
use serde::Serialize;
use std::collections::{BTreeMap, btree_map::Entry};
use wasmparser::{CompositeInnerType, Encoding, ExternalKind, FuncType, KnownCustom, Name, Parser, Payload, TypeRef, Validator, WasmFeatures};
/// Largest module we read (256 MiB).
pub const MAX_INPUT: usize = 256 << 20;
/// Most entries kept per list.
pub const MAX_ITEMS: usize = 100_000;
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct WasmSection {
pub id: u8,
pub name: String,
pub offset: u64,
pub size: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct WasmImport {
pub module: String,
pub name: String,
/// func, table, memory, global or tag.
pub kind: &'static str,
pub type_index: Option<u32>,
/// Function signature or a short description of the imported item.
pub signature: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct WasmExport {
pub name: String,
pub kind: &'static str,
pub index: u32,
/// Name of the exported function from the name section, when known.
pub target_name: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct WasmFunction {
/// Index in the function index space (imports first).
pub index: u32,
pub type_index: u32,
pub signature: String,
pub body_offset: u64,
pub body_size: u64,
pub locals: u64,
pub name: Option<String>,
pub exported_as: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct WasmMemory {
pub index: u32,
pub initial: u64,
pub maximum: Option<u64>,
pub shared: bool,
pub memory64: bool,
pub imported: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct WasmTable {
pub index: u32,
pub element_type: String,
pub initial: u64,
pub maximum: Option<u64>,
pub imported: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct WasmGlobal {
pub index: u32,
pub value_type: String,
pub mutable: bool,
pub imported: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct WasmTag {
/// Index in the tag index space (imports first).
pub index: u32,
pub type_index: u32,
pub signature: String,
pub imported: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct WasmCustom {
pub name: String,
pub offset: u64,
pub size: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct ProducerValue {
pub name: String,
pub version: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct ProducerField {
pub field: String,
pub values: Vec<ProducerValue>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct WasmValidation {
pub valid: bool,
pub error: Option<String>,
pub offset: Option<u64>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct WasmReport {
/// `module` or `component`.
pub encoding: &'static str,
pub version: u16,
pub size: usize,
pub validation: WasmValidation,
/// Set when the structure could not be read to the end.
pub parse_error: Option<String>,
pub sections: Vec<WasmSection>,
pub types: usize,
pub imports: Vec<WasmImport>,
pub imported_functions: u32,
pub exports: Vec<WasmExport>,
/// Functions defined in the module (imports are listed under `imports`).
pub functions: Vec<WasmFunction>,
pub memories: Vec<WasmMemory>,
pub tables: Vec<WasmTable>,
pub globals: Vec<WasmGlobal>,
pub tags: Vec<WasmTag>,
pub start_function: Option<u32>,
pub data_segments: u32,
pub element_segments: u32,
pub data_count: Option<u32>,
pub custom_sections: Vec<WasmCustom>,
pub module_name: Option<String>,
pub producers: Vec<ProducerField>,
pub target_features: Vec<String>,
pub source_mapping_url: Option<String>,
pub external_debug_info: Option<String>,
pub truncated: bool,
pub warnings: Vec<String>,
}
/// True when the bytes start with the WebAssembly magic.
pub fn is_wasm(data: &[u8]) -> bool {
data.starts_with(b"\0asm")
}
fn sig(f: &FuncType) -> String {
let p: Vec<String> = f.params().iter().map(|v| v.to_string()).collect();
let r: Vec<String> = f.results().iter().map(|v| v.to_string()).collect();
format!("({}) -> ({})", p.join(", "), r.join(", "))
}
struct ReferencedStrings {
remaining: usize,
exhausted: bool,
}
impl ReferencedStrings {
fn claim(&mut self, bytes: usize) -> bool {
if self.exhausted || bytes > self.remaining {
self.exhausted = true;
return false;
}
self.remaining -= bytes;
true
}
fn copy(&mut self, value: &str) -> Option<String> {
self.claim(value.len()).then(|| value.to_owned())
}
}
fn type_sig(types: &[Option<FuncType>], t: u32, cache: &mut BTreeMap<u32, String>, strings: &mut ReferencedStrings) -> Option<String> {
if strings.exhausted {
return None;
}
let signature = match cache.entry(t) {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => {
let value = match types.get(t as usize) {
Some(Some(f)) => sig(f),
Some(None) => format!("type {t} (not a function type)"),
None => format!("type {t} (missing)"),
};
if !strings.claim(value.len()) {
return None;
}
entry.insert(value)
}
};
strings.copy(signature)
}
fn section_name(id: u8) -> &'static str {
match id {
0 => "custom",
1 => "type",
2 => "import",
3 => "function",
4 => "table",
5 => "memory",
6 => "global",
7 => "export",
8 => "start",
9 => "element",
10 => "code",
11 => "data",
12 => "datacount",
13 => "tag",
_ => "unknown",
}
}
/// Read the outer component layout without interpreting the contents of embedded binaries.
fn component_sections(reader: &mut wasmparser::BinaryReader<'_>, rep: &mut WasmReport) -> Result<(), wasmparser::BinaryReaderError> {
while !reader.eof() {
if rep.sections.len() == MAX_ITEMS {
rep.truncated = true;
break;
}
let id = reader.read_u8()?;
let body = reader.read_reader()?;
let range = body.range();
let name = match id {
0 => format!("custom:{}", wasmparser::CustomSectionReader::new(body)?.name()),
1 => "core module".into(),
2 => "core instance".into(),
3 => "core type".into(),
4 => "component".into(),
5 => "instance".into(),
6 => "alias".into(),
7 => "type".into(),
8 => "canonical".into(),
9 => "start".into(),
10 => "import".into(),
11 => "export".into(),
_ => "unknown".into(),
};
rep.sections.push(WasmSection { id, name, offset: range.start, size: range.end.saturating_sub(range.start) });
}
Ok(())
}
fn kind_name(k: ExternalKind) -> &'static str {
match k {
ExternalKind::Func | ExternalKind::FuncExact => "func",
ExternalKind::Table => "table",
ExternalKind::Memory => "memory",
ExternalKind::Global => "global",
ExternalKind::Tag => "tag",
}
}
/// Validate with every feature enabled; the result never fails the caller.
pub fn validate(data: &[u8]) -> WasmValidation {
match Validator::new_with_features(WasmFeatures::all()).validate_all(data) {
Ok(_) => WasmValidation { valid: true, error: None, offset: None },
Err(e) => WasmValidation { valid: false, error: Some(e.message().to_string()), offset: Some(e.offset()) },
}
}
/// A length-prefixed UTF-8 string at the start of a custom section payload.
fn leading_string(data: &[u8]) -> Option<String> {
// inspect bounds the enclosing module by MAX_INPUT, including any inline source map.
wasmparser::BinaryReader::new(data, 0).read_unlimited_string().ok().map(str::to_owned)
}
fn target_features(data: &[u8]) -> Vec<String> {
let r = Reader::new(data);
let mut out = Vec::new();
let Some((n, mut p)) = r.uleb128(0) else { return out };
for _ in 0..n.min(1024) {
let Some(prefix) = r.u8(p) else { break };
let Some((len, used)) = r.uleb128(p + 1) else { break };
let Some(s) = usize::try_from(len).ok().and_then(|l| r.slice(p + 1 + used, l)) else { break };
out.push(format!("{}{}", char::from(prefix), String::from_utf8_lossy(s)));
p = p + 1 + used + s.len();
}
out
}
fn cap(len: usize, truncated: &mut bool) -> bool {
if len >= MAX_ITEMS {
*truncated = true;
false
} else {
true
}
}
fn read_types(
r: wasmparser::TypeSectionReader<'_>,
types: &mut Vec<Option<FuncType>>,
rep: &mut WasmReport,
) -> Result<(), wasmparser::BinaryReaderError> {
for rg in r {
for st in rg?.into_types() {
if types.len() >= MAX_ITEMS {
rep.truncated = true;
break;
}
types.push(match st.composite_type.inner {
CompositeInnerType::Func(f) => Some(f),
_ => None,
});
}
}
rep.types = types.len();
Ok(())
}
fn read_custom(c: wasmparser::CustomSectionReader<'_>, rep: &mut WasmReport, fn_names: &mut BTreeMap<u32, String>, producer_values: &mut usize) {
if cap(rep.custom_sections.len(), &mut rep.truncated) {
let range = c.range();
rep.custom_sections.push(WasmCustom { name: c.name().to_string(), offset: range.start, size: range.end.saturating_sub(range.start) });
}
match c.name() {
"sourceMappingURL" => rep.source_mapping_url = leading_string(c.data()),
"external_debug_info" => rep.external_debug_info = leading_string(c.data()),
"target_features" => rep.target_features = target_features(c.data()),
_ => {}
}
match c.as_known() {
KnownCustom::Name(names) => {
// A damaged name section is reported but does not stop the rest.
for (i, sub) in names.into_iter().take(65).enumerate() {
if i == 64 {
rep.truncated = true;
break;
}
let Ok(sub) = sub else {
rep.warnings.push("the name section is damaged".into());
break;
};
match sub {
Name::Module { name, .. } => rep.module_name = Some(name.to_string()),
Name::Function(map) => {
for (i, n) in map.into_iter().take(MAX_ITEMS + 1).enumerate() {
if i == MAX_ITEMS {
rep.truncated = true;
break;
}
match n {
Ok(n) => {
if let Some(name) = fn_names.get_mut(&n.index) {
// Repeated sections may replace an already retained name at the limit.
*name = n.name.to_string();
} else if cap(fn_names.len(), &mut rep.truncated) {
fn_names.insert(n.index, n.name.to_string());
}
}
Err(_) => {
rep.warnings.push("a function name entry is damaged".into());
break;
}
}
}
}
_ => {}
}
}
}
KnownCustom::Producers(p) => {
for (i, f) in p.into_iter().take(65).enumerate() {
if i == 64 || !cap(rep.producers.len(), &mut rep.truncated) {
rep.truncated = true;
break;
}
let Ok(f) = f else {
rep.warnings.push("the producers section is damaged".into());
break;
};
let mut values = Vec::new();
for (i, value) in f.values.into_iter().take(65).enumerate() {
if i == 64 || !cap(*producer_values, &mut rep.truncated) {
rep.truncated = true;
break;
}
if let Ok(value) = value {
values.push(ProducerValue { name: value.name.to_string(), version: value.version.to_string() });
*producer_values += 1;
}
}
rep.producers.push(ProducerField { field: f.name.to_string(), values });
}
}
_ => {}
}
}
fn read_code(
body: wasmparser::FunctionBody<'_>,
types: &[Option<FuncType>],
func_types: &[u32],
code_index: &mut u32,
rep: &mut WasmReport,
type_signatures: &mut BTreeMap<u32, String>,
strings: &mut ReferencedStrings,
) -> Result<(), wasmparser::BinaryReaderError> {
let ti = func_types.get(*code_index as usize).copied();
let mut locals = 0u64;
let lr = body.get_locals_reader()?;
for l in lr {
let (n, _) = l?;
locals = locals.saturating_add(u64::from(n));
}
let range = body.range();
if cap(rep.functions.len(), &mut rep.truncated) {
let signature = match ti {
Some(t) => type_sig(types, t, type_signatures, strings),
None => strings.copy("no matching function section entry"),
};
if let Some(signature) = signature {
rep.functions.push(WasmFunction {
index: rep.imported_functions.saturating_add(*code_index),
type_index: ti.unwrap_or(u32::MAX),
signature,
body_offset: range.start,
body_size: range.end.saturating_sub(range.start),
locals,
name: None,
exported_as: Vec::new(),
});
}
}
*code_index = code_index.saturating_add(1);
Ok(())
}
fn read_functions(
r: wasmparser::FunctionSectionReader<'_>,
func_types: &mut Vec<u32>,
declared_functions: &mut u32,
rep: &mut WasmReport,
) -> Result<(), wasmparser::BinaryReaderError> {
*declared_functions = r.count();
for t in r {
if func_types.len() >= MAX_ITEMS {
rep.truncated = true;
break;
}
func_types.push(t?);
}
Ok(())
}
/// Read a module. Fails only when the bytes are not WebAssembly at all or too large; structural
/// damage is reported in `parse_error` and `validation`.
pub fn inspect(data: &[u8]) -> Result<WasmReport, FormatError> {
if data.len() > MAX_INPUT {
return Err(FormatError::TooLarge(format!("{} bytes; the limit is {MAX_INPUT}", data.len())));
}
if !is_wasm(data) {
return Err(FormatError::Malformed("no WebAssembly magic (\\0asm)".into()));
}
let mut rep = WasmReport {
encoding: "module",
version: 0,
size: data.len(),
validation: validate(data),
parse_error: None,
sections: Vec::new(),
types: 0,
imports: Vec::new(),
imported_functions: 0,
exports: Vec::new(),
functions: Vec::new(),
memories: Vec::new(),
tables: Vec::new(),
globals: Vec::new(),
tags: Vec::new(),
start_function: None,
data_segments: 0,
element_segments: 0,
data_count: None,
custom_sections: Vec::new(),
module_name: None,
producers: Vec::new(),
target_features: Vec::new(),
source_mapping_url: None,
external_debug_info: None,
truncated: false,
warnings: Vec::new(),
};
let mut types: Vec<Option<FuncType>> = Vec::new();
let mut func_types: Vec<u32> = Vec::new();
let mut fn_names: BTreeMap<u32, String> = BTreeMap::new();
let mut producer_values = 0usize;
let mut imported_tables = 0u32;
let mut imported_memories = 0u32;
let mut imported_globals = 0u32;
let mut imported_tags = 0u32;
let mut declared_functions = 0;
let mut code_index = 0u32;
// Type and name references can expand a small module into much larger display strings.
// Charge both the signature cache and every retained copy before allocating that copy.
let reference_budget = (64 << 20).max(data.len().saturating_mul(3));
let mut strings = ReferencedStrings { remaining: reference_budget, exhausted: false };
let mut type_signatures = BTreeMap::new();
let result: Result<(), wasmparser::BinaryReaderError> = (|| {
for payload in Parser::new(0).parse_all(data) {
let payload = payload?;
if let Some((id, range)) = payload.as_section()
&& cap(rep.sections.len(), &mut rep.truncated)
{
let name = match &payload {
Payload::CustomSection(c) => format!("custom:{}", c.name()),
_ => section_name(id).to_string(),
};
rep.sections.push(WasmSection { id, name, offset: range.start, size: range.end.saturating_sub(range.start) });
}
match payload {
Payload::Version { num, encoding, range } => {
rep.version = num;
if encoding == Encoding::Component {
rep.encoding = "component";
rep.warnings.push("this is a component, not a core module; only validation and the top-level layout are reported".into());
if let Some(body) = usize::try_from(range.end).ok().and_then(|end| data.get(end..)) {
component_sections(&mut wasmparser::BinaryReader::new(body, range.end), &mut rep)?;
}
break;
}
}
Payload::TypeSection(r) => {
read_types(r, &mut types, &mut rep)?;
}
Payload::ImportSection(r) => {
for imp in r.into_imports() {
let imp = imp?;
let (kind, type_index, signature) = match imp.ty {
TypeRef::Func(t) | TypeRef::FuncExact(t) => {
rep.imported_functions = rep.imported_functions.saturating_add(1);
let signature =
if rep.imports.len() < MAX_ITEMS { type_sig(&types, t, &mut type_signatures, &mut strings) } else { None };
("func", Some(t), signature)
}
TypeRef::Table(t) => {
if cap(rep.tables.len(), &mut rep.truncated) {
rep.tables.push(WasmTable {
index: imported_tables,
element_type: format!("{:?}", t.element_type),
initial: t.initial,
maximum: t.maximum,
imported: true,
});
}
imported_tables = imported_tables.saturating_add(1);
("table", None, Some(format!("table {} min {}", format!("{:?}", t.element_type).to_lowercase(), t.initial)))
}
TypeRef::Memory(m) => {
if cap(rep.memories.len(), &mut rep.truncated) {
rep.memories.push(WasmMemory {
index: imported_memories,
initial: m.initial,
maximum: m.maximum,
shared: m.shared,
memory64: m.memory64,
imported: true,
});
}
imported_memories = imported_memories.saturating_add(1);
("memory", None, Some(format!("memory min {} pages", m.initial)))
}
TypeRef::Global(g) => {
if cap(rep.globals.len(), &mut rep.truncated) {
rep.globals.push(WasmGlobal {
index: imported_globals,
value_type: g.content_type.to_string(),
mutable: g.mutable,
imported: true,
});
}
imported_globals = imported_globals.saturating_add(1);
("global", None, Some(format!("{}{}", if g.mutable { "mut " } else { "" }, g.content_type)))
}
TypeRef::Tag(t) => {
let keep_tag = cap(rep.tags.len(), &mut rep.truncated);
let signature = if rep.imports.len() < MAX_ITEMS {
type_sig(&types, t.func_type_idx, &mut type_signatures, &mut strings)
} else {
None
};
if keep_tag {
let tag_signature = match &signature {
Some(value) => strings.copy(value),
None => type_sig(&types, t.func_type_idx, &mut type_signatures, &mut strings),
};
if let Some(signature) = tag_signature {
rep.tags.push(WasmTag { index: imported_tags, type_index: t.func_type_idx, signature, imported: true });
}
}
// The import list and tag list are capped independently.
imported_tags = imported_tags.saturating_add(1);
("tag", Some(t.func_type_idx), signature)
}
};
if cap(rep.imports.len(), &mut rep.truncated)
&& let Some(signature) = signature
{
rep.imports.push(WasmImport { module: imp.module.to_string(), name: imp.name.to_string(), kind, type_index, signature });
}
}
}
Payload::FunctionSection(r) => {
read_functions(r, &mut func_types, &mut declared_functions, &mut rep)?;
}
Payload::TableSection(r) => {
for (i, t) in r.into_iter().enumerate() {
let t = t?;
if !cap(rep.tables.len(), &mut rep.truncated) {
break;
}
rep.tables.push(WasmTable {
index: imported_tables.saturating_add(u32::try_from(i).unwrap_or(u32::MAX)),
element_type: format!("{:?}", t.ty.element_type),
initial: t.ty.initial,
maximum: t.ty.maximum,
imported: false,
});
}
}
Payload::MemorySection(r) => {
for (i, m) in r.into_iter().enumerate() {
let m = m?;
if !cap(rep.memories.len(), &mut rep.truncated) {
break;
}
rep.memories.push(WasmMemory {
index: imported_memories.saturating_add(u32::try_from(i).unwrap_or(u32::MAX)),
initial: m.initial,
maximum: m.maximum,
shared: m.shared,
memory64: m.memory64,
imported: false,
});
}
}
Payload::TagSection(r) => {
for (i, t) in r.into_iter().enumerate() {
let t = t?;
if !cap(rep.tags.len(), &mut rep.truncated) {
break;
}
if let Some(signature) = type_sig(&types, t.func_type_idx, &mut type_signatures, &mut strings) {
rep.tags.push(WasmTag {
index: imported_tags.saturating_add(u32::try_from(i).unwrap_or(u32::MAX)),
type_index: t.func_type_idx,
signature,
imported: false,
});
}
}
}
Payload::GlobalSection(r) => {
for (i, g) in r.into_iter().enumerate() {
let g = g?;
if !cap(rep.globals.len(), &mut rep.truncated) {
break;
}
rep.globals.push(WasmGlobal {
index: imported_globals.saturating_add(u32::try_from(i).unwrap_or(u32::MAX)),
value_type: g.ty.content_type.to_string(),
mutable: g.ty.mutable,
imported: false,
});
}
}
Payload::ExportSection(r) => {
for e in r {
let e = e?;
if !cap(rep.exports.len(), &mut rep.truncated) {
break;
}
rep.exports.push(WasmExport { name: e.name.to_string(), kind: kind_name(e.kind), index: e.index, target_name: None });
}
}
Payload::StartSection { func, .. } => rep.start_function = Some(func),
Payload::ElementSection(r) => rep.element_segments = r.count(),
Payload::DataCountSection { count, .. } => rep.data_count = Some(count),
Payload::DataSection(r) => rep.data_segments = r.count(),
Payload::CodeSectionEntry(body) => {
read_code(body, &types, &func_types, &mut code_index, &mut rep, &mut type_signatures, &mut strings)?;
}
Payload::CustomSection(c) => {
read_custom(c, &mut rep, &mut fn_names, &mut producer_values);
}
Payload::End(_) => {}
_ => {}
}
}
Ok(())
})();
if let Err(e) = result {
rep.parse_error = Some(format!("{} (at byte {})", e.message(), e.offset()));
}
// Attach names and exports to functions.
let mut exported: BTreeMap<u32, Vec<String>> = BTreeMap::new();
for e in rep.exports.iter_mut() {
if e.kind == "func" {
e.target_name = fn_names.get(&e.index).and_then(|name| strings.copy(name));
exported.entry(e.index).or_default().push(e.name.clone());
}
}
for f in rep.functions.iter_mut() {
f.name = fn_names.get(&f.index).and_then(|name| strings.copy(name));
f.exported_as = exported.remove(&f.index).unwrap_or_default();
}
if declared_functions != code_index && rep.parse_error.is_none() {
rep.warnings.push(format!("{declared_functions} function declarations but {code_index} bodies"));
}
if strings.exhausted {
rep.truncated = true;
rep.warnings
.push(format!("referenced type signatures and function names exceed the {reference_budget}-byte budget; some metadata is omitted"));
}
Ok(rep)
}
struct CappedText {
out: String,
cap: usize,
truncated: bool,
}
impl wasmprinter::Print for CappedText {
fn write_str(&mut self, s: &str) -> std::io::Result<()> {
if self.out.len().saturating_add(s.len()) > self.cap {
let room = self.cap.saturating_sub(self.out.len());
let mut end = room.min(s.len());
while end > 0 && !s.is_char_boundary(end) {
end -= 1;
}
self.out.push_str(s.get(..end).unwrap_or(""));
self.truncated = true;
return Err(std::io::Error::other("output cap reached"));
}
self.out.push_str(s);
Ok(())
}
}
/// The text format of a module, at most `cap` bytes: (text, truncated). Only call this on
/// modules that passed [`validate`]; damaged modules are not rendered.
pub fn to_wat(data: &[u8], cap: usize) -> Result<(String, bool), String> {
let mut p = CappedText { out: String::new(), cap, truncated: false };
let mut cfg = wasmprinter::Config::new();
cfg.name_unnamed(false);
match cfg.print(data, &mut p) {
Ok(()) => Ok((p.out, false)),
Err(_) if p.truncated => Ok((p.out, true)),
Err(e) => Err(e.to_string()),
}
}
/// What a JavaScript glue file says about the module.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct GlueScan {
/// String literals ending in `.wasm`.
pub wasm_references: Vec<String>,
/// Imports (`module.name`) whose module and field names both appear in the glue.
pub matched_imports: Vec<String>,
pub mentions_instantiate: bool,
}
/// Scan glue JavaScript for `.wasm` literals and the import names the module expects.
pub fn scan_glue(text: &str, imports: &[WasmImport]) -> GlueScan {
let mut refs = Vec::new();
let b = text.as_bytes();
let mut i = 0usize;
while let Some(p) = text.get(i..).and_then(|t| t.find(".wasm")) {
let at = i + p;
let end = at + 5;
if let Some(&q) = b.get(end)
&& matches!(q, b'"' | b'\'' | b'`')
{
let start = b.get(..at).and_then(|h| h.iter().rposition(|&c| c == q)).map(|s| s + 1);
if let Some(s) = start
&& at - s <= 256
&& let Some(lit) = text.get(s..end)
&& !lit.contains(['\n', ' '])
&& !refs.iter().any(|r: &String| r == lit)
&& refs.len() < 256
{
refs.push(lit.to_string());
}
}
i = end;
}
let matched = imports
.iter()
.filter(|im| text.contains(im.module.as_str()) && text.contains(im.name.as_str()))
.take(MAX_ITEMS)
.map(|im| format!("{}.{}", im.module, im.name))
.collect();
GlueScan { wasm_references: refs, matched_imports: matched, mentions_instantiate: text.contains("WebAssembly.instantiate") }
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_support::formats as fx;
/// The output inventory limit applies to retained functions, not the local groups within one retained function.
#[test]
fn local_counts_include_groups_after_ten_thousand() {
// One valid () -> () function containing 10,001 runs of one i32 local.
let mut bytes = b"\0asm\x01\0\0\0\x01\x04\x01\x60\0\0\x03\x02\x01\0".to_vec();
// section size = 1 + 3 + 20005, body size = 2 + 20002 + 1.
bytes.extend([10, 0xa9, 0x9c, 1, 1, 0xa5, 0x9c, 1, 0x91, 0x4e]);
for _ in 0..10_001 {
bytes.extend([1, 0x7f]);
}
bytes.push(0x0b);
let report = inspect(&bytes).unwrap();
assert!(report.validation.valid, "{:?}", report.validation);
assert_eq!(report.functions[0].locals, 10_001);
}
#[test]
fn debug_links_need_valid_wasm_names() {
assert_eq!(leading_string(b"\x04test").as_deref(), Some("test"));
assert!(leading_string(&[1, 0xff]).is_none());
assert!(leading_string(b"\x81\x80\x80\x80\x80\0x").is_none());
assert!(leading_string(b"\x04abc").is_none());
}
#[test]
fn inspect_preserves_long_source_mapping_urls() {
let url = format!("data:application/json;base64,{}", "A".repeat(100_000));
let report = inspect(&fx::wasm_module_with_source_mapping_url(&url)).unwrap();
assert_eq!(report.source_mapping_url.as_deref(), Some(url.as_str()));
}
#[test]
fn large_function_tables_do_not_report_missing_declarations() {
fn leb(mut value: usize) -> Vec<u8> {
let mut out = Vec::new();
while value >= 128 {
out.push((value as u8 & 0x7f) | 0x80);
value >>= 7;
}
out.push(value as u8);
out
}
let count = MAX_ITEMS + 1;
let mut bytes = b"\0asm\x01\0\0\0\x01\x04\x01\x60\0\0".to_vec();
let mut declarations = leb(count);
declarations.resize(declarations.len() + count, 0);
let mut bodies = leb(count);
for _ in 0..count {
bodies.extend([2, 0, 0x0b]);
}
for (id, section) in [(3, declarations), (10, bodies)] {
bytes.push(id);
bytes.extend(leb(section.len()));
bytes.extend(section);
}
let report = inspect(&bytes).unwrap();
assert!(report.validation.valid && report.truncated);
assert_eq!(report.functions.len(), MAX_ITEMS);
assert!(!report.warnings.iter().any(|s| s.contains("function declarations")), "{:?}", report.warnings);
}
#[test]
fn reads_the_sample_module() {
let m = fx::wasm_module();
let r = inspect(&m).unwrap();
assert!(r.validation.valid, "{:?}", r.validation);
assert!(r.parse_error.is_none());
assert_eq!(r.types, 3);
assert_eq!(r.imports.len(), 1);
assert_eq!(r.imports[0].signature, "(i32) -> ()");
assert_eq!(r.imported_functions, 1);
assert_eq!(r.functions.len(), 2);
assert_eq!(r.functions[0].name.as_deref(), Some("add"));
assert_eq!(r.functions[0].signature, "(i32, i32) -> (i32)");
assert_eq!(r.functions[0].exported_as, vec!["add"]);
assert_eq!(r.exports.len(), 3);
assert_eq!(r.memories.len(), 1);
assert_eq!(r.globals.len(), 1);
assert!(r.globals[0].mutable);
assert_eq!(r.data_segments, 1);
assert_eq!(r.module_name.as_deref(), Some("demo"));
assert_eq!(r.producers.len(), 2);
assert_eq!(r.producers[1].values[0].name, "rustc");
assert_eq!(r.source_mapping_url.as_deref(), Some("demo.wasm.map"));
assert_eq!(r.target_features, vec!["+mutable-globals"]);
let (wat, trunc) = to_wat(&m, 1 << 20).unwrap();
assert!(!trunc);
assert!(wat.contains("(module $demo"), "{wat}");
assert!(wat.contains("i32.add"));
let (short, trunc) = to_wat(&m, 40).unwrap();
assert!(trunc && short.len() <= 40);
let g = scan_glue(fx::WASM_GLUE, &r.imports);
assert_eq!(g.wasm_references, vec!["./demo.wasm"]);
assert_eq!(g.matched_imports, vec!["env.log"]);
assert!(g.mentions_instantiate);
}
#[test]
fn damaged_modules_are_reported_not_fatal() {
let m = fx::wasm_module();
let mut bad = m.clone();
let len = bad.len();
bad.truncate(len - 10);
let r = inspect(&bad).unwrap();
assert!(!r.validation.valid);
assert!(r.parse_error.is_some());
assert!(inspect(b"not wasm").is_err());
}
#[test]
fn hostile_inputs_do_not_panic() {
let m = fx::wasm_module();
for x in crate::test_support::mutations(&m) {
if let Ok(r) = inspect(&x)
&& r.validation.valid
{
let _ = to_wat(&x, 4096);
}
}
for n in 0..m.len() {
let _ = inspect(&m[..n]);
}
for seed in 0..100 {
let mut n = crate::test_support::noise(seed, 256);
n[..8].copy_from_slice(b"\0asm\x01\0\0\0");
let _ = inspect(&n);
let _ = scan_glue(&String::from_utf8_lossy(&n), &[]);
}
}
}
mod bytes;
#[cfg(test)]
mod test_support;
/// Admission errors; structural damage stays in the inspection report.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum WasmError {
#[error("the file is malformed ({0})")]
Malformed(String),
#[error("too large: {0}")]
TooLarge(String),
}
#[cfg(test)]
#[path = "regressions/component_layout.rs"]
mod component_layout;
#[cfg(test)]
#[path = "regressions/custom_inventory_budget.rs"]
mod custom_inventory_budget;
#[cfg(test)]
#[path = "regressions/reference_bytes.rs"]
mod reference_bytes;
@@ -0,0 +1,42 @@
use crate::{MAX_ITEMS, inspect};
const COMPONENT: &[u8] = b"\0asm\x0d\0\x01\0";
/// Only outer sections appear, including sections after embedded binaries.
#[test]
fn lists_component_sections_without_descending_into_embedded_modules() {
let mut bytes = COMPONENT.to_vec();
bytes.extend(b"\x00\x04\x03one");
bytes.extend(b"\x01\x0b\0asm\x01\0\0\0\x01\x01\0");
bytes.extend(b"\x04\x0e\0asm\x0d\0\x01\0\x00\x04\x03two");
bytes.extend(b"\x00\x06\x05three");
let report = inspect(&bytes).unwrap();
assert!(report.validation.valid, "{:?}", report.validation);
let layout: Vec<_> = report.sections.iter().map(|s| (s.id, s.name.as_str(), s.offset, s.size)).collect();
assert_eq!(layout, [(0, "custom:one", 10, 4), (1, "core module", 16, 11), (4, "component", 29, 14), (0, "custom:three", 45, 6)]);
assert!(report.functions.is_empty());
assert!(report.parse_error.is_none());
}
/// A short section reports damage without losing the preceding layout.
#[test]
fn truncated_component_keeps_its_complete_prefix() {
let mut bytes = COMPONENT.to_vec();
bytes.extend(b"\x00\x04\x03one\x01\x08\0asm");
let report = inspect(&bytes).unwrap();
assert_eq!(report.sections.len(), 1);
assert!(report.parse_error.is_some());
}
/// Component layouts use the same cap as core module layouts.
#[test]
fn component_section_limit_is_disclosed() {
let mut bytes = COMPONENT.to_vec();
for _ in 0..=MAX_ITEMS {
bytes.extend(b"\x00\x02\x01x");
}
let report = inspect(&bytes).unwrap();
assert!(report.validation.valid);
assert_eq!(report.sections.len(), MAX_ITEMS);
assert!(report.truncated);
}
@@ -0,0 +1,115 @@
use crate::test_support::formats::wasm_module;
use crate::{MAX_ITEMS, inspect};
use crate::test_support::{leb, string};
fn custom(module: &mut Vec<u8>, name: &str, data: &[u8]) {
let mut body = Vec::new();
string(&mut body, name);
body.extend(data);
module.push(0);
module.extend(leb(body.len()));
module.extend(body);
}
fn names(module: &mut Vec<u8>, entries: &[(usize, &str)]) {
let mut map = leb(entries.len());
for (index, name) in entries {
map.extend(leb(*index));
string(&mut map, name);
}
let mut body = vec![1];
body.extend(leb(map.len()));
body.extend(map);
custom(module, "name", &body);
}
fn producers(fields: usize, values: usize) -> Vec<u8> {
let mut body = leb(fields);
for _ in 0..fields {
string(&mut body, "language");
body.extend(leb(values));
for _ in 0..values {
string(&mut body, "fixture");
string(&mut body, "1");
}
}
body
}
#[test]
fn repeated_name_sections_share_a_limit_but_can_update_retained_functions() {
let mut bytes = b"\0asm\x01\0\0\0\x01\x04\x01\x60\0\0\x03\x02\x01\0\x0a\x04\x01\x02\0\x0b".to_vec();
let entries: Vec<_> = (0..MAX_ITEMS).map(|index| (index, "kept")).collect();
names(&mut bytes, &entries);
let complete = inspect(&bytes).unwrap();
assert!(complete.validation.valid);
assert!(!complete.truncated);
assert_eq!(complete.functions[0].name.as_deref(), Some("kept"));
names(&mut bytes, &[(0, "updated"), (MAX_ITEMS, "omitted")]);
let report = inspect(&bytes).unwrap();
assert!(report.validation.valid);
assert!(report.parse_error.is_none());
assert!(report.truncated);
assert_eq!(report.functions[0].name.as_deref(), Some("updated"));
}
#[test]
fn producer_values_are_bounded_across_repeated_sections() {
let mut bytes = b"\0asm\x01\0\0\0".to_vec();
let section = producers(1, 64);
for _ in 0..MAX_ITEMS / 64 + 1 {
custom(&mut bytes, "producers", &section);
}
let report = inspect(&bytes).unwrap();
assert!(report.validation.valid);
assert!(report.parse_error.is_none());
assert!(report.truncated);
assert_eq!(report.producers.iter().map(|field| field.values.len()).sum::<usize>(), MAX_ITEMS);
assert_eq!(report.producers[0].values[0].name, "fixture");
}
#[test]
fn empty_producer_fields_still_spend_the_global_field_limit() {
let mut bytes = b"\0asm\x01\0\0\0".to_vec();
let section = producers(64, 0);
for _ in 0..MAX_ITEMS / 64 + 1 {
custom(&mut bytes, "producers", &section);
}
let report = inspect(&bytes).unwrap();
assert!(report.validation.valid);
assert_eq!(report.producers.len(), MAX_ITEMS);
assert!(report.truncated);
}
#[test]
fn existing_per_section_limits_are_reported_as_truncation() {
for (fields, values) in [(65, 0), (1, 65)] {
let mut bytes = b"\0asm\x01\0\0\0".to_vec();
custom(&mut bytes, "producers", &producers(fields, values));
let report = inspect(&bytes).unwrap();
assert!(report.truncated, "{fields} fields with {values} values");
}
let mut bytes = b"\0asm\x01\0\0\0".to_vec();
let mut subsections = vec![0, 2, 1, b'm'];
// Name subsection IDs must increase, including unknown extension IDs.
for id in 20..84 {
subsections.extend([id, 0]);
}
custom(&mut bytes, "name", &subsections);
let report = inspect(&bytes).unwrap();
assert!(report.truncated);
assert_eq!(report.module_name.as_deref(), Some("m"));
}
#[test]
fn ordinary_names_and_producers_remain_complete() {
let report = inspect(&wasm_module()).unwrap();
assert!(report.validation.valid);
assert!(!report.truncated);
assert_eq!(report.functions[0].name.as_deref(), Some("add"));
assert_eq!(report.module_name.as_deref(), Some("demo"));
assert_eq!(report.producers.len(), 2);
assert_eq!(report.producers[1].values[0].name, "rustc");
assert_eq!(report.exports.iter().find(|export| export.name == "add").unwrap().target_name.as_deref(), Some("add"));
}
@@ -0,0 +1,140 @@
use crate::{WasmReport, inspect};
use crate::test_support::{leb, string};
fn section(module: &mut Vec<u8>, id: u8, body: &[u8]) {
module.push(id);
module.extend(leb(body.len()));
module.extend(body);
}
fn shared_type(functions: usize, imports: usize, imported_tags: usize, defined_tags: usize, params: usize) -> Vec<u8> {
let mut module = b"\0asm\x01\0\0\0".to_vec();
let mut ty = vec![1, 0x60];
ty.extend(leb(params));
ty.extend(std::iter::repeat_n(0x7f, params));
ty.push(0);
section(&mut module, 1, &ty);
if imports + imported_tags > 0 {
let mut body = leb(imports + imported_tags);
for i in 0..imports + imported_tags {
string(&mut body, "env");
string(&mut body, "item");
if i < imports {
body.extend([0, 0]);
} else {
body.extend([4, 0, 0]);
}
}
section(&mut module, 2, &body);
}
if functions > 0 {
let mut body = leb(functions);
body.extend(std::iter::repeat_n(0, functions));
section(&mut module, 3, &body);
}
if defined_tags > 0 {
let mut body = leb(defined_tags);
for _ in 0..defined_tags {
body.extend([0, 0]);
}
section(&mut module, 13, &body);
}
if functions > 0 {
let mut body = leb(functions);
for _ in 0..functions {
body.extend([2, 0, 0x0b]);
}
section(&mut module, 10, &body);
}
module
}
fn derived_bytes(report: &WasmReport) -> usize {
report.imports.iter().map(|item| item.signature.len()).sum::<usize>()
+ report.tags.iter().map(|item| item.signature.len()).sum::<usize>()
+ report.functions.iter().map(|item| item.signature.len() + item.name.as_ref().map_or(0, String::len)).sum::<usize>()
+ report.exports.iter().map(|item| item.target_name.as_ref().map_or(0, String::len)).sum::<usize>()
}
fn bounded(module: &[u8]) -> WasmReport {
assert!(module.len() < 1 << 20);
let report = inspect(module).unwrap();
assert!(report.parse_error.is_none());
assert!(derived_bytes(&report) <= 64 << 20, "{} input bytes retain {} derived string bytes", module.len(), derived_bytes(&report));
assert!(report.truncated);
assert!(report.warnings.iter().any(|warning| warning.contains("byte budget")), "{:?}", report.warnings);
report
}
#[test]
fn repeated_function_signatures_have_a_byte_budget() {
let report = bounded(&shared_type(16_384, 0, 0, 0, 1000));
assert!(report.validation.valid, "{:?}", report.validation);
assert!(!report.functions.is_empty());
assert!(report.functions.len() < 16_384);
assert!(
report.functions.iter().enumerate().all(|(index, function)| function.index as usize == index && function.signature.starts_with("(i32, i32"))
);
assert!(!report.warnings.iter().any(|warning| warning.contains("function declarations")));
}
#[test]
fn repeated_import_signatures_stay_bounded_even_when_validation_fails() {
let report = bounded(&shared_type(0, 16_384, 0, 0, 1000));
assert!(!report.validation.valid);
assert!(report.validation.error.as_deref().unwrap().contains("effective type size"));
assert_eq!(report.imported_functions, 16_384);
assert!(!report.imports.is_empty());
assert!(report.imports.len() < 16_384);
}
#[test]
fn imported_and_defined_tags_share_the_same_budget_as_functions() {
for (functions, imports, imported_tags, defined_tags) in [(0, 0, 8192, 0), (0, 0, 0, 16_384), (4096, 4096, 4096, 4096)] {
let report = bounded(&shared_type(functions, imports, imported_tags, defined_tags, 1000));
assert_eq!(report.imported_functions as usize, imports);
assert!(report.tags.len() < imported_tags + defined_tags);
assert!(report.tags.iter().all(|tag| tag.signature.starts_with("(i32, i32")));
assert!(report.tags.iter().filter(|tag| !tag.imported).all(|tag| tag.index as usize >= imported_tags));
}
}
#[test]
fn exported_function_names_cannot_multiply_a_small_name_section() {
let mut module = shared_type(0, 1, 0, 0, 0);
let name = "λ".repeat(32_768);
let mut exports = leb(1280);
for index in 0..1280 {
string(&mut exports, &format!("export_{index}"));
exports.extend([0, 0]);
}
section(&mut module, 7, &exports);
let mut map = vec![1, 0];
string(&mut map, &name);
let mut names = vec![4, b'n', b'a', b'm', b'e', 1];
names.extend(leb(map.len()));
names.extend(map);
section(&mut module, 0, &names);
let report = bounded(&module);
assert!(report.validation.valid, "{:?}", report.validation);
assert_eq!(report.exports.len(), 1280);
assert!(report.exports.iter().any(|export| export.target_name.is_none()));
assert!(report.exports.iter().filter_map(|export| export.target_name.as_deref()).all(|value| value == name));
}
#[test]
fn ordinary_shared_signatures_and_names_remain_complete() {
let report = inspect(&shared_type(2, 2, 2, 2, 3)).unwrap();
assert!(report.validation.valid, "{:?}", report.validation);
assert!(!report.truncated, "{:?}", report.warnings);
assert_eq!(report.imported_functions, 2);
assert_eq!(report.functions.len(), 2);
assert_eq!(report.tags.len(), 4);
assert!(report.imports.iter().all(|item| item.signature == "(i32, i32, i32) -> ()"));
assert!(report.tags.iter().all(|tag| tag.signature == "(i32, i32, i32) -> ()"));
assert_eq!(report.functions.iter().map(|function| function.index).collect::<Vec<_>>(), [2, 3]);
assert_eq!(report.tags.iter().map(|tag| tag.index).collect::<Vec<_>>(), [0, 1, 2, 3]);
assert!(report.tags[..2].iter().all(|tag| tag.imported));
assert!(report.tags[2..].iter().all(|tag| !tag.imported));
}
+163
View File
@@ -0,0 +1,163 @@
// Synthetic fixtures and mutations from Craft Anything testkit, same source revision.
/// Deterministic pseudo-random bytes (xorshift) for fuzz-like loops.
pub fn noise(seed: u64, len: usize) -> Vec<u8> {
let mut x = seed.wrapping_mul(0x9e37_79b9_7f4a_7c15) | 1;
(0..len)
.map(|_| {
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
x as u8
})
.collect()
}
/// Every way to damage a sample used by robustness tests: truncations at many lengths and
/// single-byte corruptions spread across the file.
pub fn mutations(sample: &[u8]) -> Vec<Vec<u8>> {
let mut out = Vec::new();
let n = sample.len();
let step = (n / 64).max(1);
let mut len = 0;
while len < n {
out.push(sample[..len].to_vec());
len += step;
}
for (i, seed) in (0..n).step_by(step).zip(1u64..) {
let mut m = sample.to_vec();
let noise = noise(seed, 8);
for (j, v) in noise.iter().enumerate() {
if let Some(b) = m.get_mut(i + j) {
*b = *v;
}
}
out.push(m);
let mut m = sample.to_vec();
if let Some(b) = m.get_mut(i) {
*b = 0xff;
}
out.push(m);
}
out
}
pub mod formats {
fn leb(mut v: u32, out: &mut Vec<u8>) {
loop {
let b = (v & 0x7f) as u8;
v >>= 7;
if v == 0 {
out.push(b);
return;
}
out.push(b | 0x80);
}
}
fn name(s: &str, out: &mut Vec<u8>) {
leb(s.len() as u32, out);
out.extend_from_slice(s.as_bytes());
}
fn section(id: u8, body: &[u8], out: &mut Vec<u8>) {
out.push(id);
leb(body.len() as u32, out);
out.extend_from_slice(body);
}
fn custom(sname: &str, body: &[u8], out: &mut Vec<u8>) {
let mut c = Vec::new();
name(sname, &mut c);
c.extend_from_slice(body);
section(0, &c, out);
}
/// A small valid module: imports `env.log(i32)`, defines `add(i32,i32)->i32` and `_start`,
/// one memory, one mutable global, one data segment, exports `add`, `memory` and `_start`, and
/// carries `name`, `producers` and `sourceMappingURL` custom sections.
pub fn wasm_module() -> Vec<u8> {
wasm_module_with_source_mapping_url("demo.wasm.map")
}
/// The sample module with a caller-provided source map URL.
pub fn wasm_module_with_source_mapping_url(url: &str) -> Vec<u8> {
let mut m = b"\0asm\x01\0\0\0".to_vec();
section(1, &[3, 0x60, 2, 0x7f, 0x7f, 1, 0x7f, 0x60, 0, 0, 0x60, 1, 0x7f, 0], &mut m);
let mut imp = vec![1];
name("env", &mut imp);
name("log", &mut imp);
imp.extend_from_slice(&[0, 2]);
section(2, &imp, &mut m);
section(3, &[2, 0, 1], &mut m);
section(5, &[1, 0, 1], &mut m);
section(6, &[1, 0x7f, 1, 0x41, 42, 0x0b], &mut m);
let mut exp = vec![3];
name("add", &mut exp);
exp.extend_from_slice(&[0, 1]);
name("memory", &mut exp);
exp.extend_from_slice(&[2, 0]);
name("_start", &mut exp);
exp.extend_from_slice(&[0, 2]);
section(7, &exp, &mut m);
let add = [0u8, 0x20, 0, 0x20, 1, 0x6a, 0x0b];
let start = [0u8, 0x41, 7, 0x10, 0, 0x0b];
let mut code = vec![2, add.len() as u8];
code.extend_from_slice(&add);
code.push(start.len() as u8);
code.extend_from_slice(&start);
section(10, &code, &mut m);
section(11, &[1, 0, 0x41, 0, 0x0b, 2, b'h', b'i'], &mut m);
// name section
let mut n = Vec::new();
let mut module_name = Vec::new();
name("demo", &mut module_name);
n.push(0);
leb(module_name.len() as u32, &mut n);
n.extend_from_slice(&module_name);
let mut fnames = vec![3];
for (i, f) in ["log", "add", "_start"].iter().enumerate() {
leb(i as u32, &mut fnames);
name(f, &mut fnames);
}
n.push(1);
leb(fnames.len() as u32, &mut n);
n.extend_from_slice(&fnames);
custom("name", &n, &mut m);
let mut p = vec![2];
name("language", &mut p);
p.push(1);
name("Rust", &mut p);
name("", &mut p);
name("processed-by", &mut p);
p.push(1);
name("rustc", &mut p);
name("1.80.0", &mut p);
custom("producers", &p, &mut m);
let mut s = Vec::new();
name(url, &mut s);
custom("sourceMappingURL", &s, &mut m);
let mut tf = vec![1, b'+'];
name("mutable-globals", &mut tf);
custom("target_features", &tf, &mut m);
m
}
/// JavaScript glue that loads the module.
pub const WASM_GLUE: &str =
"const imports = { env: { log: (x) => console.log(x) } };\nfetch(\"./demo.wasm\").then(r => WebAssembly.instantiateStreaming(r, imports));\n";
}
pub fn leb(mut value: usize) -> Vec<u8> {
let mut out = Vec::new();
while value >= 128 {
out.push((value as u8 & 0x7f) | 0x80);
value >>= 7;
}
out.push(value as u8);
out
}
pub fn string(out: &mut Vec<u8>, value: &str) {
out.extend(leb(value.len()));
out.extend(value.as_bytes());
}
+1 -1
View File
@@ -35,7 +35,7 @@ lenient = ["dep:miniz_oxide"]
lenient-bzip2 = ["lenient", "dep:bzip2-rs"]
# Methods 14 (ZIP LZMA) and 95 (XZ).
lenient-lzma = ["lenient", "dep:lzma-rs"]
# Method 93.
# Zstandard methods 20 (legacy) and 93.
lenient-zstd = ["lenient", "dep:ruzstd"]
serde = ["dep:serde"]
+2 -2
View File
@@ -91,7 +91,7 @@ impl ZipEntry {
9 => "deflate64",
12 => "bzip2",
14 => "lzma",
93 => "zstd",
20 | 93 => "zstd",
95 => "xz",
99 => "aes",
_ => "other",
@@ -463,7 +463,7 @@ pub fn read_entry_unverified(data: &[u8], e: &ZipEntry, max_bytes: u64) -> Resul
})?,
12 | 95 => decode_bzip2_or_xz(raw, e, max_bytes)?,
14 => decode_lzma(raw, e, max_bytes)?,
93 => decode_zstd(raw, e, max_bytes)?,
20 | 93 => decode_zstd(raw, e, max_bytes)?,
m => return Err(ZipError::Unsupported(format!("compression method {m} ({}) for {}", e.method_name(), e.name))),
};
if out.len() as u64 != e.uncompressed_size {
+37
View File
@@ -391,3 +391,40 @@ fn a_method_left_out_of_the_build_is_unsupported() {
data[8..10].copy_from_slice(&93u16.to_le_bytes());
assert!(matches!(read_entry_unverified(&data, &entry, 100), Err(ZipError::Unsupported(m)) if m.contains("not built in")));
}
// RRFZ, Craft Anything 29badae7: deprecated ZIP method 20 is Zstandard.
#[cfg(feature = "lenient-zstd")]
mod legacy_zstandard {
use crate::lenient as zip;
use crate::lenient::testkit::zip_stored;
const BODY: &[u8] = b"zstandard payload";
/// Build a ZIP with one single-segment Zstandard frame containing a raw final block.
fn sample(method: u16) -> Vec<u8> {
let size = BODY.len() as u8;
let mut frame = vec![0x28, 0xb5, 0x2f, 0xfd, 0x20, size, (size << 3) | 1, 0, 0];
frame.extend_from_slice(BODY);
let mut data = zip_stored(&[("a.txt", &frame)]);
let cd = 30 + "a.txt".len() + frame.len();
data[8..10].copy_from_slice(&method.to_le_bytes());
data[14..18].copy_from_slice(&zip::crc32(BODY).to_le_bytes());
data[22..26].copy_from_slice(&u32::from(size).to_le_bytes());
data[cd + 10..cd + 12].copy_from_slice(&method.to_le_bytes());
data[cd + 16..cd + 20].copy_from_slice(&zip::crc32(BODY).to_le_bytes());
data[cd + 24..cd + 28].copy_from_slice(&u32::from(size).to_le_bytes());
data
}
/// The deprecated and current method IDs both decode through the bounded ZIP reader.
#[test]
fn reads_zstandard_members_under_both_method_ids() {
for method in [93, 20] {
let data = sample(method);
let entry = zip::parse(&data, 1).unwrap().entries.remove(0);
assert_eq!(entry.method_name(), "zstd", "method {method}");
assert_eq!(zip::read_entry(&data, &entry, 100).unwrap(), BODY);
assert!(matches!(zip::read_entry(&data, &entry, 2), Err(zip::ZipError::TooLarge(2))));
}
}
}