daniel543x/craft-libs
1
0 Fork 0
mirror of https://github.com/storytold/craft-libs.git synced 2026-11-11 04:52:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

craft-doc: add cooperative cancellation to document reads

Browse Source
This commit is contained in:
dotCooCoo committed 2026-11-03 05:56:54 -08:00
1 parent 55332dd301
commit 5f3aed6309
18 files changed
+987 -178
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/doc/README.md
+20 -1
View File
@@ -44,10 +44,29 @@ let outline = craft_doc::outline(&bytes, &limits)?;
- `renditions(bytes, &Limits) -> Result<Vec<Rendition>, Error>`: each `Rendition` has a `kind`
(`Pdf`, `Thumbnail`, `Composite`), a `mime` and the `bytes`.
- `outline(bytes, &Limits) -> Result<Outline, Error>`: `canvas`, `pages` and `layers`.
- `Error` is `Unrecognized`, `Malformed`, `TooNew`, `Unsupported` or `LimitExceeded`. A file
- `Error` is `Unrecognized`, `Malformed`, `TooNew`, `Unsupported`, `LimitExceeded` or `Cancelled`. A file
whose version is newer than the crate reads gives `TooNew` from `renditions` and `outline`. A
ZIP archive whose `manifest.json` has no `format_version` gives `Unrecognized`.
### Cooperative cancellation
`info_with_cancel`, `renditions_with_cancel` and `outline_with_cancel` take the same bytes and
limits plus `&dyn Fn() -> bool`. The callback runs on the calling thread. Once it returns true,
that call stops with `Error::Cancelled`, even if a later callback result would be false.
```rust
use std::sync::atomic::{AtomicBool, Ordering};
let stopped = AtomicBool::new(false);
let renditions = craft_doc::renditions_with_cancel(&bytes, &limits, &|| stopped.load(Ordering::Relaxed))?;
```
Checks occur during format detection, JSON scans and parsing, gzip input and output, rendition
copies, ZIP directory and entry reads, CRC traversal, and outline conversion, including shared
header, manifest and base64 helpers.
The existing entry points keep their behavior and limits. PDF parsing/rendering and optional
PhotoCraft layer pixel decoding are separate operations.
## What each format gives
| | `.vectorcraft` | `.pcraft` | `.ecproj` |
crates/doc/src/cancel.rs
+116
View File
@@ -0,0 +1,116 @@
//! Per-read cancellation. Callbacks run on the reader's thread and a request stays latched.
use std::cell::Cell;
use std::io::{self, Read};
use crate::Error;
pub(crate) const CHUNK: usize = 64 * 1024;
pub(crate) fn check(control: Option<&Control<'_>>) -> Result<(), Error> {
control.map_or(Ok(()), Control::check)
}
/// A byte scanner can skip over escaped characters, so test a threshold rather than a modulus.
pub(crate) struct Scan<'a> {
control: Option<&'a Control<'a>>,
next: usize,
}
impl<'a> Scan<'a> {
pub(crate) fn new(control: Option<&'a Control<'a>>) -> Self {
Self { control, next: 0 }
}
pub(crate) fn check(&mut self, at: usize) -> Result<(), Error> {
if self.control.is_some() && at >= self.next {
check(self.control)?;
self.next = at.saturating_add(CHUNK);
}
Ok(())
}
}
pub(crate) struct Control<'a> {
cancelled: &'a dyn Fn() -> bool,
stopped: Cell<bool>,
}
impl<'a> Control<'a> {
fn new(cancelled: &'a dyn Fn() -> bool) -> Self {
Self { cancelled, stopped: Cell::new(false) }
}
pub(crate) fn check(&self) -> Result<(), Error> {
if self.stopped.get() || (self.cancelled)() {
self.stopped.set(true);
Err(Error::Cancelled)
} else {
Ok(())
}
}
}
pub(crate) fn run<T>(cancelled: &dyn Fn() -> bool, read: impl FnOnce(&Control<'_>) -> Result<T, Error>) -> Result<T, Error> {
let control = Control::new(cancelled);
control.check()?;
let result = read(&control);
control.check()?;
result
}
/// Poll before each bounded portion of input. In particular, a one-byte JSON reader does not
/// call the callback for every byte, and a decoder cannot consume an unbounded read request.
pub(crate) struct Reader<'a, R> {
inner: R,
control: Option<&'a Control<'a>>,
left: usize,
}
impl<'a, R> Reader<'a, R> {
pub(crate) fn new(inner: R, control: Option<&'a Control<'a>>) -> Self {
Self { inner, control, left: 0 }
}
}
impl<R: Read> Read for Reader<'_, R> {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
let Some(control) = self.control else { return self.inner.read(buf) };
if buf.is_empty() {
return Ok(0);
}
if self.left == 0 {
control.check().map_err(|_| io::Error::other("document read canceled"))?;
self.left = CHUNK;
}
let len = buf.len().min(self.left);
let n = self.inner.read(&mut buf[..len])?;
self.left -= n;
Ok(n)
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used)]
use super::*;
#[test]
fn a_reader_stops_after_a_partial_read_and_latches_the_request() {
let polls = Cell::new(0);
let cancelled = || {
polls.set(polls.get() + 1);
polls.get() == 2
};
let control = Control::new(&cancelled);
let mut input = io::Cursor::new(vec![1; CHUNK * 3]);
let mut reader = Reader::new(&mut input, Some(&control));
let mut bytes = vec![0; CHUNK * 3];
assert_eq!(reader.read(&mut bytes).unwrap(), CHUNK);
assert!(reader.read(&mut bytes).is_err());
assert!(reader.read(&mut bytes).is_err(), "a one-shot true remains canceled");
assert_eq!(input.position(), CHUNK as u64, "the remaining input was not consumed");
assert_eq!(polls.get(), 2);
}
}
crates/doc/src/ecproj.rs
+75 -20
View File
@@ -23,17 +23,27 @@ const TICKS_PER_SECOND: f64 = 254_016_000_000.0;
/// Calls `stop` with each member name of the JSON object `text` starts with, in order, as far as
/// `text` goes, until it returns `true`. Members of nested objects are skipped. A name is given as
/// written, escapes included.
fn top_level_keys<'a>(text: &'a [u8], mut stop: impl FnMut(&'a [u8]) -> bool) {
let Some(body) = text.trim_ascii_start().strip_prefix(b"{") else { return };
fn top_level_keys<'a>(text: &'a [u8], control: Option<&crate::cancel::Control<'_>>, mut stop: impl FnMut(&'a [u8]) -> bool) -> Result<(), Error> {
let mut scan = crate::cancel::Scan::new(control);
let mut start = 0;
while text.get(start).is_some_and(u8::is_ascii_whitespace) {
scan.check(start)?;
start += 1;
}
let Some(body) = text.get(start..).and_then(|s| s.strip_prefix(b"{")) else { return Ok(()) };
// Start a new offset domain after leading whitespace.
let mut scan = crate::cancel::Scan::new(control);
let (mut depth, mut want_key, mut i) = (0u32, true, 0);
while let Some(&b) = body.get(i) {
scan.check(i)?;
match b {
b'"' => {
let start = i + 1;
let mut end = start;
loop {
scan.check(end)?;
match body.get(end) {
None => return,
None => return Ok(()),
Some(b'\\') => end += 2,
Some(b'"') => break,
Some(_) => end += 1,
@@ -44,26 +54,31 @@ fn top_level_keys<'a>(text: &'a [u8], mut stop: impl FnMut(&'a [u8]) -> bool) {
&& let Some(key) = body.get(start..end)
&& stop(key)
{
return;
return Ok(());
}
want_key = false;
i = end;
}
b'{' | b'[' => depth += 1,
b'}' | b']' if depth == 0 => return,
b'}' | b']' if depth == 0 => return Ok(()),
b'}' | b']' => depth -= 1,
b',' if depth == 0 => want_key = true,
_ => {}
}
i += 1;
}
Ok(())
}
/// A project starts with `savedBy` followed by `schema`, or, saved before `savedBy` existed, with
/// `schema` followed by `items` or `savedBy`, all top-level members.
pub(crate) fn sniff(head: &[u8]) -> bool {
sniff_checked(head, None).unwrap_or(false)
}
pub(crate) fn sniff_checked(head: &[u8], control: Option<&crate::cancel::Control<'_>>) -> Result<bool, Error> {
let (mut first, mut found) = (None::<&[u8]>, false);
top_level_keys(head, |key| match first {
top_level_keys(head, control, |key| match first {
None => {
first = Some(key);
!matches!(key, b"savedBy" | b"schema")
@@ -76,8 +91,8 @@ pub(crate) fn sniff(head: &[u8]) -> bool {
found = key == b"items" || key == b"savedBy";
found
}
});
found
})?;
Ok(found)
}
fn json<'a>(bytes: &'a [u8], budget: &Budget) -> Result<&'a [u8], Error> {
@@ -88,9 +103,14 @@ fn json<'a>(bytes: &'a [u8], budget: &Budget) -> Result<&'a [u8], Error> {
/// The project's [`Head`] read with [`Head::read`]; [`ReadError::TooNew`] when its schema is
/// newer than this crate reads.
fn read_head(json: &[u8], budget: &Budget) -> Result<Result<Head, ReadError>, Error> {
check_depth(json, budget.limits.max_depth)?;
check_depth(json, budget.limits.max_depth, budget.control)?;
let text = std::str::from_utf8(json).map_err(|e| malformed(format!("the project is not UTF-8: {e}")))?;
Ok(Head::read(text))
let head = match budget.control {
Some(control) => Head::read_from(crate::cancel::Reader::new(json, Some(control))),
None => Head::read(text),
};
budget.check_cancelled()?;
Ok(head)
}
/// The project's [`Head`], refused when its schema is newer than this crate reads.
@@ -185,12 +205,23 @@ struct SwitchesM {
pub(crate) fn outline(bytes: &[u8], budget: &mut Budget) -> Result<Outline, Error> {
let json = json(bytes, budget)?;
head(json, budget)?;
let p: Project = parse(json, budget.limits)?;
let mut comps: Vec<ItemM> = p.items.into_values().filter(|i| i.kind.kind == "Comp").collect();
let p: Project = parse(json, budget)?;
project_outline(p, budget)
}
fn project_outline(p: Project, budget: &Budget) -> Result<Outline, Error> {
let mut comps = Vec::new();
for i in p.items.into_values() {
budget.check_cancelled()?;
if i.kind.kind == "Comp" {
comps.push(i);
}
}
comps.sort_by_key(|i| i.id);
let pages = comps
.into_iter()
.map(|i| {
budget.check_cancelled()?;
let k = i.kind;
let (Some(width), Some(height)) = (k.width, k.height) else {
return Err(malformed(format!("composition `{}` has no size", i.name)));
@@ -205,17 +236,41 @@ pub(crate) fn outline(bytes: &[u8], budget: &mut Budget) -> Result<Outline, Erro
.unwrap_or_default()
.into_iter()
.rev()
.map(|l| Layer {
name: l.name,
kind: l.source.map(|s| s.kind).unwrap_or_default(),
visible: l.switches.is_none_or(|s| s.video),
opacity: None,
blend: l.blend_mode,
children: vec![],
.map(|l| {
budget.check_cancelled()?;
Ok(Layer {
name: l.name,
kind: l.source.map(|s| s.kind).unwrap_or_default(),
visible: l.switches.is_none_or(|s| s.video),
opacity: None,
blend: l.blend_mode,
children: vec![],
})
})
.collect();
.collect::<Result<_, Error>>()?;
Ok(Page { name: i.name, x: 0.0, y: 0.0, width: f64::from(width), height: f64::from(height), timing, layers })
})
.collect::<Result<_, Error>>()?;
Ok(Outline { canvas: None, pages, layers: vec![] })
}
#[cfg(test)]
mod cancellation_tests {
#![allow(clippy::unwrap_used)]
use super::*;
use std::cell::Cell;
#[test]
fn conversion_stops_inside_composition_layers() {
let project: Project =
serde_json::from_str(r#"{"items":{"1":{"id":1,"kind":{"type":"Comp","width":1,"height":1,"layers":[{},{},{},{},{},{}]}}}}"#).unwrap();
let polls = Cell::new(0);
let cancel = || {
polls.set(polls.get() + 1);
polls.get() == 6
};
let result = crate::cancel::run(&cancel, |control| project_outline(project, &Budget::controlled(&crate::Limits::default(), control)));
assert_eq!(result, Err(Error::Cancelled));
assert_eq!(polls.get(), 6);
}
}
crates/doc/src/lib.rs
+152 -50
View File
@@ -26,6 +26,7 @@
//! assert!(info.supported);
//! ```
mod cancel;
mod ecproj;
mod pcraft;
#[cfg(feature = "pcraft-pixels")]
@@ -98,6 +99,9 @@ pub struct Info {
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[non_exhaustive]
pub enum Error {
/// A cancellation-aware read observed the caller's cancellation request.
#[error("document read canceled")]
Cancelled,
/// [`sniff`] does not recognize the bytes as a Craft document, or a ZIP archive's
/// `manifest.json` is a JSON object without `format_version`, so the archive is not a
/// PhotoCraft bundle.
@@ -273,15 +277,45 @@ pub fn sniff(head: &[u8]) -> Option<Kind> {
}
}
fn read_kind(bytes: &[u8], budget: &Budget) -> Result<Kind, Error> {
if pcraft::sniff_checked(bytes, budget.control)? {
Ok(Kind::PhotoCraft)
} else if vectorcraft::sniff_checked(bytes, budget.control)? {
Ok(Kind::VectorCraft)
} else if ecproj::sniff_checked(bytes, budget.control)? {
Ok(Kind::EffectCraft)
} else {
Err(Error::Unrecognized)
}
}
/// The document's kind, version and writer. A version newer than this crate reads is not an
/// error here: [`Info::supported`] is `false`.
pub fn info(bytes: &[u8], limits: &Limits) -> Result<Info, Error> {
let mut budget = Budget::new(limits);
let kind = sniff(bytes).ok_or(Error::Unrecognized)?;
read_info(bytes, &mut budget)
}
/// [`info`] with cooperative cancellation. The callback runs synchronously on this thread;
/// returning true stops the read at a checkpoint with [`Error::Cancelled`]. The limits and
/// returned data are the same as [`info`].
///
/// ```
/// use std::sync::atomic::{AtomicBool, Ordering};
/// let stopped = AtomicBool::new(true);
/// let result = craft_doc::info_with_cancel(b"", &craft_doc::Limits::default(), &|| stopped.load(Ordering::Relaxed));
/// assert_eq!(result, Err(craft_doc::Error::Cancelled));
/// ```
pub fn info_with_cancel(bytes: &[u8], limits: &Limits, cancelled: &dyn Fn() -> bool) -> Result<Info, Error> {
cancel::run(cancelled, |control| read_info(bytes, &mut Budget::controlled(limits, control)))
}
fn read_info(bytes: &[u8], budget: &mut Budget) -> Result<Info, Error> {
let kind = read_kind(bytes, budget)?;
let (version, generator) = match kind {
Kind::PhotoCraft => pcraft::info(bytes, &mut budget)?,
Kind::VectorCraft => vectorcraft::info(bytes, &mut budget)?,
Kind::EffectCraft => ecproj::info(bytes, &mut budget)?,
Kind::PhotoCraft => pcraft::info(bytes, budget)?,
Kind::VectorCraft => vectorcraft::info(bytes, budget)?,
Kind::EffectCraft => ecproj::info(bytes, budget)?,
};
Ok(Info { kind, version, generator, supported: version <= kind.supported_version() })
}
@@ -292,20 +326,39 @@ pub fn info(bytes: &[u8], limits: &Limits) -> Result<Info, Error> {
/// manifest names; an `.ecproj` project has none.
pub fn renditions(bytes: &[u8], limits: &Limits) -> Result<Vec<Rendition>, Error> {
let mut budget = Budget::new(limits);
match sniff(bytes).ok_or(Error::Unrecognized)? {
Kind::PhotoCraft => pcraft::renditions(bytes, &mut budget),
Kind::VectorCraft => vectorcraft::renditions(bytes, &mut budget),
Kind::EffectCraft => ecproj::renditions(bytes, &mut budget),
read_renditions(bytes, &mut budget)
}
/// [`renditions`] with cooperative cancellation, as [`info_with_cancel`]. This interrupts
/// extraction; it does not parse or render the PDF or decode the image a rendition contains.
pub fn renditions_with_cancel(bytes: &[u8], limits: &Limits, cancelled: &dyn Fn() -> bool) -> Result<Vec<Rendition>, Error> {
cancel::run(cancelled, |control| read_renditions(bytes, &mut Budget::controlled(limits, control)))
}
fn read_renditions(bytes: &[u8], budget: &mut Budget) -> Result<Vec<Rendition>, Error> {
match read_kind(bytes, budget)? {
Kind::PhotoCraft => pcraft::renditions(bytes, budget),
Kind::VectorCraft => vectorcraft::renditions(bytes, budget),
Kind::EffectCraft => ecproj::renditions(bytes, budget),
}
}
/// The document's canvas, pages and layers (see [`Outline`]).
pub fn outline(bytes: &[u8], limits: &Limits) -> Result<Outline, Error> {
let mut budget = Budget::new(limits);
match sniff(bytes).ok_or(Error::Unrecognized)? {
Kind::PhotoCraft => pcraft::outline(bytes, &mut budget),
Kind::VectorCraft => vectorcraft::outline(bytes, &mut budget),
Kind::EffectCraft => ecproj::outline(bytes, &mut budget),
read_outline(bytes, &mut budget)
}
/// [`outline`] with cooperative cancellation, as [`info_with_cancel`].
pub fn outline_with_cancel(bytes: &[u8], limits: &Limits, cancelled: &dyn Fn() -> bool) -> Result<Outline, Error> {
cancel::run(cancelled, |control| read_outline(bytes, &mut Budget::controlled(limits, control)))
}
fn read_outline(bytes: &[u8], budget: &mut Budget) -> Result<Outline, Error> {
match read_kind(bytes, budget)? {
Kind::PhotoCraft => pcraft::outline(bytes, budget),
Kind::VectorCraft => vectorcraft::outline(bytes, budget),
Kind::EffectCraft => ecproj::outline(bytes, budget),
}
}
@@ -316,12 +369,38 @@ fn malformed(m: impl std::fmt::Display) -> Error {
/// The bytes one call may still decompress, read out of ZIP entries or return as renditions.
pub(crate) struct Budget<'a> {
pub(crate) limits: &'a Limits,
pub(crate) control: Option<&'a cancel::Control<'a>>,
left: u64,
}
impl<'a> Budget<'a> {
fn new(limits: &'a Limits) -> Self {
Budget { limits, left: limits.max_total }
Budget { limits, control: None, left: limits.max_total }
}
fn controlled(limits: &'a Limits, control: &'a cancel::Control<'a>) -> Self {
Budget { limits, control: Some(control), left: limits.max_total }
}
pub(crate) fn check_cancelled(&self) -> Result<(), Error> {
if let Some(control) = self.control {
control.check()?;
}
Ok(())
}
/// Copy bytes whose size the caller has already checked against the budget.
pub(crate) fn copy(&self, bytes: &[u8]) -> Result<Vec<u8>, Error> {
if self.control.is_none() {
return Ok(bytes.to_vec());
}
self.check_cancelled()?;
let mut out = Vec::with_capacity(bytes.len());
for chunk in bytes.chunks(cancel::CHUNK) {
self.check_cancelled()?;
out.extend_from_slice(chunk);
}
Ok(out)
}
/// Bytes still allowed.
@@ -331,6 +410,7 @@ impl<'a> Budget<'a> {
/// Count `n` bytes of `what` against the total.
pub(crate) fn take(&mut self, n: u64, what: &str) -> Result<(), Error> {
self.check_cancelled()?;
self.left = self
.left
.checked_sub(n)
@@ -340,6 +420,7 @@ impl<'a> Budget<'a> {
/// Check one rendition or ZIP entry of `n` bytes against `max_entry`.
pub(crate) fn check_entry(&self, n: u64, what: &str) -> Result<(), Error> {
self.check_cancelled()?;
if n > self.limits.max_entry {
return Err(Error::LimitExceeded(format!("{what} is {n} bytes, more than max_entry ({})", self.limits.max_entry)));
}
@@ -354,6 +435,7 @@ impl<'a> Budget<'a> {
/// Check `n` bytes of JSON against `max_json`.
pub(crate) fn check_json(&self, n: u64, what: &str) -> Result<(), Error> {
self.check_cancelled()?;
if n > self.limits.max_json {
return Err(Error::LimitExceeded(format!("{what} is more than max_json ({} bytes)", self.limits.max_json)));
}
@@ -363,29 +445,34 @@ impl<'a> Budget<'a> {
/// Is the JSON text nested at most `max` levels deep (arrays and objects), and at most
/// [`MAX_DEPTH`]?
pub(crate) fn check_depth(json: &[u8], max: u32) -> Result<(), Error> {
pub(crate) fn check_depth(json: &[u8], max: u32, control: Option<&cancel::Control<'_>>) -> Result<(), Error> {
let max = max.min(MAX_DEPTH);
let (mut depth, mut in_str, mut escaped) = (0u32, false, false);
for &b in json {
if in_str {
for chunk in json.chunks(cancel::CHUNK) {
if let Some(control) = control {
control.check()?;
}
for &b in chunk {
if in_str {
match b {
_ if escaped => escaped = false,
b'\\' => escaped = true,
b'"' => in_str = false,
_ => {}
}
continue;
}
match b {
_ if escaped => escaped = false,
b'\\' => escaped = true,
b'"' => in_str = false,
b'"' => in_str = true,
b'[' | b'{' => {
depth += 1;
if depth > max {
return Err(Error::LimitExceeded(format!("JSON nested deeper than max_depth ({max} levels)")));
}
}
b']' | b'}' => depth = depth.saturating_sub(1),
_ => {}
}
continue;
}
match b {
b'"' => in_str = true,
b'[' | b'{' => {
depth += 1;
if depth > max {
return Err(Error::LimitExceeded(format!("JSON nested deeper than max_depth ({max} levels)")));
}
}
b']' | b'}' => depth = depth.saturating_sub(1),
_ => {}
}
}
Ok(())
@@ -491,18 +578,23 @@ pub(crate) fn counted_map<'de, D: Deserializer<'de>, T: Deserialize<'de>>(d: D)
}
/// Parse one JSON value of type `T` from `json`, after checking its nesting against the limits.
pub(crate) fn parse<T: DeserializeOwned>(json: &[u8], limits: &Limits) -> Result<T, Error> {
check_depth(json, limits.max_depth)?;
counting(limits.max_items, || parse_checked(json).map_err(malformed))
pub(crate) fn parse<T: DeserializeOwned>(json: &[u8], budget: &Budget) -> Result<T, Error> {
check_depth(json, budget.limits.max_depth, budget.control)?;
counting(budget.limits.max_items, || parse_checked(json, budget).map_err(malformed))
}
/// Parse JSON whose nesting [`check_depth`] has accepted.
fn parse_checked<T: DeserializeOwned>(json: &[u8]) -> Result<T, serde_json::Error> {
let mut de = serde_json::Deserializer::from_slice(json);
de.disable_recursion_limit();
let v = T::deserialize(&mut de)?;
de.end()?;
Ok(v)
fn parse_checked<T: DeserializeOwned>(json: &[u8], budget: &Budget) -> Result<T, serde_json::Error> {
fn read<'de, T: DeserializeOwned, R: serde_json::de::Read<'de>>(mut de: serde_json::Deserializer<R>) -> Result<T, serde_json::Error> {
de.disable_recursion_limit();
let v = T::deserialize(&mut de)?;
de.end()?;
Ok(v)
}
match budget.control {
Some(control) => read(serde_json::Deserializer::from_reader(cancel::Reader::new(json, Some(control)))),
None => read(serde_json::Deserializer::from_slice(json)),
}
}
/// The top-level member that holds a document's version number: VectorCraft's `version`,
@@ -545,13 +637,23 @@ impl<'de> Visitor<'de> for VersionIn {
/// Parse `T` from `json`; when that fails, report [`Error::TooNew`] instead if the file's version
/// (its top-level member that [`version_member`] names) is newer than `kind` supports, since a
/// newer format may change what `T` expects.
pub(crate) fn parse_versioned<T: DeserializeOwned>(json: &[u8], limits: &Limits, kind: Kind) -> Result<T, Error> {
check_depth(json, limits.max_depth)?;
counting(limits.max_items, || {
parse_checked(json).map_err(|e| {
let mut de = serde_json::Deserializer::from_slice(json);
de.disable_recursion_limit();
let version = (&mut de).deserialize_map(VersionIn(version_member(kind))).and_then(|v| de.end().map(|()| v));
pub(crate) fn parse_versioned<T: DeserializeOwned>(json: &[u8], budget: &Budget, kind: Kind) -> Result<T, Error> {
fn version<'de, R: serde_json::de::Read<'de>>(mut de: serde_json::Deserializer<R>, kind: Kind) -> Result<Option<u32>, serde_json::Error> {
de.disable_recursion_limit();
let version = (&mut de).deserialize_map(VersionIn(version_member(kind)))?;
de.end()?;
Ok(version)
}
check_depth(json, budget.limits.max_depth, budget.control)?;
counting(budget.limits.max_items, || {
parse_checked(json, budget).map_err(|e| {
if let Err(cancelled) = budget.check_cancelled() {
return cancelled;
}
let version = match budget.control {
Some(control) => version(serde_json::Deserializer::from_reader(cancel::Reader::new(json, Some(control))), kind),
None => version(serde_json::Deserializer::from_slice(json), kind),
};
match version {
Ok(Some(version)) if version > kind.supported_version() => too_new(kind, version),
_ => malformed(e),
@@ -684,9 +786,9 @@ mod tests {
#[test]
fn depth_counts_brackets_outside_strings() {
assert!(check_depth(br#"[[["[[[[[[[["]]]"#, 3).is_ok());
assert!(check_depth(br#"[[[["\"[["]]]]"#, 3).is_err());
assert!(check_depth(br#"{"a":{"b":[1]}}"#, 3).is_ok());
assert!(check_depth(br#"[[["[[[[[[[["]]]"#, 3, None).is_ok());
assert!(check_depth(br#"[[[["\"[["]]]]"#, 3, None).is_err());
assert!(check_depth(br#"{"a":{"b":[1]}}"#, 3, None).is_ok());
}
#[test]
crates/doc/src/pcraft.rs
+64 -23
View File
@@ -18,17 +18,24 @@ use crate::{Budget, Canvas, Error, Kind, Layer, Outline, Rendition, RenditionKin
/// its central directory decide, as [`Zip::find`] reads them; otherwise the names in the local
/// headers that `head` holds.
pub(crate) fn sniff(head: &[u8]) -> bool {
head.starts_with(b"PK\x03\x04")
&& match Zip::open(head) {
Ok(zip) => zip.find(MANIFEST).is_some(),
Err(_) => local_names_include(head, MANIFEST),
}
sniff_checked(head, None).unwrap_or(false)
}
pub(crate) fn sniff_checked(head: &[u8], control: Option<&crate::cancel::Control<'_>>) -> Result<bool, Error> {
if !head.starts_with(b"PK\x03\x04") {
return Ok(false);
}
match Zip::open(head, control) {
Ok(zip) => Ok(zip.find(MANIFEST, control)?.is_some()),
Err(Error::Cancelled) => Err(Error::Cancelled),
Err(_) => local_names_include(head, MANIFEST, control),
}
}
/// The archive and its `manifest.json`.
fn manifest<'a>(bytes: &'a [u8], budget: &mut Budget) -> Result<(Zip<'a>, Vec<u8>), Error> {
let zip = Zip::open(bytes)?;
let entry = zip.find(MANIFEST).ok_or_else(|| malformed("the bundle has no manifest.json"))?;
let zip = Zip::open(bytes, budget.control)?;
let entry = zip.find(MANIFEST, budget.control)?.ok_or_else(|| malformed("the bundle has no manifest.json"))?;
let max = budget.limits.max_json.min(budget.limits.max_entry);
let json = zip.read(entry, max, budget, MANIFEST)?;
Ok((zip, json))
@@ -76,7 +83,7 @@ fn present<'de, D: Deserializer<'de>, T: Deserialize<'de>>(d: D) -> Result<Optio
/// The manifest's version and writer. A manifest without `format_version` is not PhotoCraft's:
/// [`Error::Unrecognized`]. A version of 0 is refused.
fn stamp(json: &[u8], budget: &Budget) -> Result<(u32, Option<String>), Error> {
let Object(stamp) = parse::<Object<Stamp>>(json, budget.limits)?;
let Object(stamp) = parse::<Object<Stamp>>(json, budget)?;
let version = stamp.format_version.ok_or(Error::Unrecognized)?;
if version == 0 {
return Err(malformed("format_version 0 is invalid"));
@@ -107,7 +114,12 @@ fn read<D: DeserializeOwned>(json: &[u8], budget: &Budget) -> Result<Envelope<D>
let mut limits = format::Limits::default();
limits.max_manifest_bytes = json.len();
limits.max_depth = (budget.limits.max_depth as usize).min(manifest_depth(MAX_GROUP_DEPTH));
format::check_manifest(json, &limits).map_err(|e| match e {
let checked = match budget.control {
Some(control) => format::check_manifest_with_cancel(json, &limits, &|| control.check().is_err()),
None => format::check_manifest(json, &limits),
};
checked.map_err(|e| match e {
format::Error::Cancelled => Error::Cancelled,
format::Error::LimitExceeded(m) => Error::LimitExceeded(m),
e => malformed(e),
})?;
@@ -115,7 +127,8 @@ fn read<D: DeserializeOwned>(json: &[u8], budget: &Budget) -> Result<Envelope<D>
if version > FORMAT_VERSION {
return Err(too_new(Kind::PhotoCraft, version));
}
let Object(m) = parse::<Object<Envelope<D>>>(json, budget.limits)?;
budget.check_cancelled()?;
let Object(m) = parse::<Object<Envelope<D>>>(json, budget)?;
for (member, entry, expected) in [("thumbnail", &m.thumbnail, THUMB), ("composite", &m.composite, COMPOSITE)] {
if let Some(entry) = entry
&& entry != expected
@@ -137,7 +150,8 @@ pub(crate) fn renditions(bytes: &[u8], budget: &mut Budget) -> Result<Vec<Rendit
let mut out = vec![];
for (kind, path) in [(RenditionKind::Thumbnail, m.thumbnail), (RenditionKind::Composite, m.composite)] {
let Some(path) = path else { continue };
let entry = zip.find(&path).ok_or_else(|| malformed(format!("the manifest names `{path}`, which the bundle does not have")))?;
let entry =
zip.find(&path, budget.control)?.ok_or_else(|| malformed(format!("the manifest names `{path}`, which the bundle does not have")))?;
let max = budget.limits.max_entry;
let bytes = zip.read(entry, max, budget, &path)?;
out.push(Rendition { kind, mime: "image/png".into(), bytes });
@@ -178,29 +192,56 @@ struct ContentM {
children: Vec<LayerM>,
}
impl From<LayerM> for Layer {
fn from(l: LayerM) -> Self {
Layer {
name: l.name,
kind: l.content.kind,
visible: l.visible,
opacity: Some(l.opacity),
blend: Some(l.blend),
children: l.content.children.into_iter().map(Layer::from).collect(),
}
}
fn layer(l: LayerM, budget: &Budget) -> Result<Layer, Error> {
budget.check_cancelled()?;
Ok(Layer {
name: l.name,
kind: l.content.kind,
visible: l.visible,
opacity: Some(l.opacity),
blend: Some(l.blend),
children: l.content.children.into_iter().map(|l| layer(l, budget)).collect::<Result<_, _>>()?,
})
}
pub(crate) fn outline(bytes: &[u8], budget: &mut Budget) -> Result<Outline, Error> {
let (_, json) = manifest(bytes, budget)?;
let d = read::<DocM>(&json, budget)?.document;
document_outline(d, budget)
}
fn document_outline(d: DocM, budget: &Budget) -> Result<Outline, Error> {
Ok(Outline {
canvas: Some(Canvas { width: d.size.width, height: d.size.height, resolution_dpi: d.resolution_dpi, mode: d.mode, depth: d.depth }),
pages: vec![],
layers: d.layers.into_iter().map(Layer::from).collect(),
layers: d.layers.into_iter().map(|l| layer(l, budget)).collect::<Result<_, _>>()?,
})
}
#[cfg(test)]
mod cancellation_tests {
#![allow(clippy::unwrap_used)]
use super::*;
use std::cell::Cell;
#[test]
fn conversion_stops_inside_group_children() {
let child = r#"{"name":"child","visible":true,"blend":"Normal","opacity":1,"content":{"kind":"Raster"}}"#;
let parent = format!(
r#"{{"name":"group","visible":true,"blend":"Normal","opacity":1,"content":{{"kind":"Group","children":[{child},{child},{child},{child},{child}]}}}}"#
);
let node: LayerM = serde_json::from_str(&parent).unwrap();
let polls = Cell::new(0);
let cancel = || {
polls.set(polls.get() + 1);
polls.get() == 4
};
let result = crate::cancel::run(&cancel, |control| layer(node, &Budget::controlled(&crate::Limits::default(), control)));
assert_eq!(result, Err(Error::Cancelled));
assert_eq!(polls.get(), 4);
}
}
/// The archive and the manifest's document read as `D`, checked as [`outline`] reads it.
#[cfg(feature = "pcraft-pixels")]
pub(crate) fn document<'a, D: DeserializeOwned>(bytes: &'a [u8], budget: &mut Budget) -> Result<(Zip<'a>, D), Error> {
crates/doc/src/vectorcraft.rs
+125 -47
View File
@@ -25,6 +25,7 @@ use flate2::read::GzDecoder;
use serde::Deserialize;
use serde::de::{self, Deserializer, IgnoredAny, MapAccess, SeqAccess, Visitor};
use crate::cancel::Reader;
use crate::{
Budget, Error, Kind, Layer, Lenient, Outline, Page, Rendition, RenditionKind, Scalar, Shaped, counted, malformed, parse, parse_versioned, too_new,
};
@@ -36,12 +37,37 @@ pub(crate) fn sniff(head: &[u8]) -> bool {
craft_vectorcraft_format::sniff(head)
}
pub(crate) fn sniff_checked(head: &[u8], control: Option<&crate::cancel::Control<'_>>) -> Result<bool, Error> {
if control.is_none() || !is_compressed(head) {
return Ok(sniff(head));
}
// Match the format crate's 256-byte probe, including truncated gzip input. Bound input
// reads too: optional gzip header fields can exceed the decoded probe's length.
let mut prefix = Vec::new();
let _ = GzDecoder::new(Reader::new(head, control)).take(256).read_to_end(&mut prefix);
crate::cancel::check(control)?;
Ok(!is_compressed(&prefix) && sniff(&prefix))
}
/// Where the JSON ends and the data after the marker starts, when `text` holds a NUL. JSON holds
/// no raw NUL and the marker's second byte is one, so the JSON ends right before the first NUL.
/// The marker must follow the JSON's closing `}`, `]` or `"`, as
/// [`craft_vectorcraft_format::marker_at`] requires; any other NUL makes the file malformed.
fn marker(text: &[u8]) -> Result<(usize, usize), Error> {
marker_at(text).map(|at| (at, at + BLOB_MAGIC.len())).ok_or_else(|| malformed("the file holds a NUL byte outside the marker after its JSON"))
fn marker(text: &[u8], nul: usize) -> Result<(usize, usize), Error> {
// Start at the byte before the marker so the shared check inspects a bounded suffix,
// rather than scanning the already searched JSON again.
let found = nul.checked_sub(2).and_then(|start| marker_at(text.get(start..)?).map(|at| start + at));
found.map(|at| (at, at + BLOB_MAGIC.len())).ok_or_else(|| malformed("the file holds a NUL byte outside the marker after its JSON"))
}
fn first_nul(bytes: &[u8], budget: &Budget) -> Result<Option<usize>, Error> {
for (i, chunk) in bytes.chunks(crate::cancel::CHUNK).enumerate() {
budget.check_cancelled()?;
if chunk.contains(&0) {
return Ok(chunk.iter().position(|&b| b == 0).map(|at| i * crate::cancel::CHUNK + at));
}
}
Ok(None)
}
fn damaged(e: std::io::Error) -> Error {
@@ -49,21 +75,21 @@ fn damaged(e: std::io::Error) -> Error {
}
/// A file's JSON, and the data after it.
struct Unpacked<'a> {
struct Unpacked<'a, 'c> {
json: Cow<'a, [u8]>,
tail: Tail<'a>,
tail: Tail<'a, 'c>,
}
/// The data after the marker.
enum Tail<'a> {
enum Tail<'a, 'c> {
/// In an uncompressed file.
Slice(&'a [u8]),
/// Still in the gzip stream: `read` holds what was decompressed past the marker already, not
/// yet counted against the budget.
Stream { dec: Box<GzDecoder<&'a [u8]>>, read: Vec<u8> },
Stream { dec: Box<GzDecoder<Reader<'c, &'a [u8]>>>, read: Vec<u8> },
}
impl Tail<'_> {
impl Tail<'_, '_> {
/// The bytes each `[offset, length]` in `ranges` names, or `None` for a range the file ends
/// before. A compressed file is decompressed only as far as the last range ends, and that
/// offset is checked against the budget before anything past the marker is decompressed.
@@ -71,7 +97,7 @@ impl Tail<'_> {
fn ranges(self, ranges: &[[u64; 2]], budget: &mut Budget) -> Result<Vec<Option<Vec<u8>>>, Error> {
const WHAT: &str = "the decompressed document";
let (mut dec, read) = match self {
Tail::Slice(s) => return Ok(ranges.iter().map(|&r| entry(s, r).map(<[u8]>::to_vec)).collect()),
Tail::Slice(s) => return ranges.iter().map(|&r| entry(s, r).map(|bytes| budget.copy(bytes)).transpose()).collect(),
Tail::Stream { dec, read } => (dec, read),
};
let end = ranges.iter().map(|&[offset, len]| offset.saturating_add(len)).max().unwrap_or(0);
@@ -96,7 +122,7 @@ impl Tail<'_> {
let stated = ranges.iter().fold(0u64, |n, &[_, len]| n.saturating_add(len));
let spanned = merged.iter().fold(0u64, |n, &[start, end]| n.saturating_add(end - start));
budget.take(stated.saturating_sub(spanned), WHAT)?;
let mut src = read.as_slice().chain(&mut *dec);
let mut src = Reader::new(read.as_slice().chain(&mut *dec), budget.control);
let (mut at, mut kept) = (0u64, Vec::<(u64, Vec<u8>)>::new());
for [start, end] in merged {
let skipped = std::io::copy(&mut (&mut src).take(start.saturating_sub(at)), &mut std::io::sink()).map_err(damaged)?;
@@ -114,22 +140,24 @@ impl Tail<'_> {
break;
}
}
Ok(ranges
ranges
.iter()
.map(|&[offset, len]| kept.iter().find_map(|(start, bytes)| entry(bytes, [offset.checked_sub(*start)?, len])).map(<[u8]>::to_vec))
.collect())
.map(|&[offset, len]| {
kept.iter().find_map(|(start, bytes)| entry(bytes, [offset.checked_sub(*start)?, len])).map(|bytes| budget.copy(bytes)).transpose()
})
.collect()
}
}
/// A file's JSON and where the data after it is. A compressed file is decompressed only as far
/// as the end of the marker, every byte used counted against the budget, and its JSON is
/// checked against `max_json` as it grows.
fn unpack<'a>(bytes: &'a [u8], budget: &mut Budget) -> Result<Unpacked<'a>, Error> {
fn unpack<'a, 'c>(bytes: &'a [u8], budget: &mut Budget<'c>) -> Result<Unpacked<'a, 'c>, Error> {
const JSON: &str = "the document's JSON";
let too_long = |budget: &Budget| Error::LimitExceeded(format!("{JSON} is more than max_json ({} bytes)", budget.limits.max_json));
if !is_compressed(bytes) {
let (json, tail) = if bytes.contains(&0) {
let (end, tail) = marker(bytes)?;
let (json, tail) = if let Some(nul) = first_nul(bytes, budget)? {
let (end, tail) = marker(bytes, nul)?;
(bytes.get(..end).unwrap_or_default(), bytes.get(tail..).unwrap_or_default())
} else {
(bytes, &[][..])
@@ -137,18 +165,18 @@ fn unpack<'a>(bytes: &'a [u8], budget: &mut Budget) -> Result<Unpacked<'a>, Erro
budget.check_json(json.len() as u64, JSON)?;
return Ok(Unpacked { json: Cow::Borrowed(json), tail: Tail::Slice(tail) });
}
let mut dec = Box::new(GzDecoder::new(bytes));
let mut dec = Box::new(GzDecoder::new(Reader::new(bytes, budget.control)));
let mut buf = Vec::new();
let mut split: Option<(usize, usize)> = None;
let mut nul: Option<usize> = None;
loop {
let start = buf.len();
let want = CHUNK.min(budget.left().saturating_add(1));
let n = (&mut *dec).take(want).read_to_end(&mut buf).map_err(damaged)?;
let n = Reader::new(&mut *dec, budget.control).take(want).read_to_end(&mut buf).map_err(damaged)?;
// The JSON ends before the first NUL: until there is one, all of it is JSON. The marker
// may end in a later chunk than its first NUL.
if nul.is_none() {
nul = buf.get(start..).and_then(|b| b.iter().position(|&c| c == 0)).map(|p| p + start);
nul = first_nul(buf.get(start..).unwrap_or_default(), budget)?.map(|p| p + start);
}
if nul.unwrap_or(buf.len()).saturating_sub(1) as u64 > budget.limits.max_json {
return Err(too_long(budget));
@@ -157,7 +185,7 @@ fn unpack<'a>(bytes: &'a [u8], budget: &mut Budget) -> Result<Unpacked<'a>, Erro
if let Some(nul) = nul
&& buf.len() >= nul + BLOB_MAGIC.len() - 1
{
let s = marker(&buf)?;
let s = marker(&buf, nul)?;
split = Some(s);
// The data after the marker in this chunk is counted when it is read.
used = s.1.saturating_sub(start).min(n);
@@ -170,7 +198,7 @@ fn unpack<'a>(bytes: &'a [u8], budget: &mut Budget) -> Result<Unpacked<'a>, Erro
let (json_len, tail_at) = match (split, nul) {
(Some(s), _) => s,
// The stream ended inside the marker.
(None, Some(_)) => marker(&buf)?,
(None, Some(nul)) => marker(&buf, nul)?,
(None, None) => (buf.len(), buf.len()),
};
budget.check_json(json_len as u64, JSON)?;
@@ -408,7 +436,7 @@ impl<'de, const IMAGES: bool> Deserialize<'de> for Stamp<IMAGES> {
/// The file's [`Stamp`], refused when the format is not VectorCraft's, and when the version is
/// newer than this crate reads unless `newer` is true.
fn stamp<const IMAGES: bool>(json: &[u8], budget: &Budget, newer: bool) -> Result<Stamp<IMAGES>, Error> {
let s: Stamp<IMAGES> = if newer { parse(json, budget.limits)? } else { parse_versioned(json, budget.limits, Kind::VectorCraft)? };
let s: Stamp<IMAGES> = if newer { parse(json, budget)? } else { parse_versioned(json, budget, Kind::VectorCraft)? };
match craft_vectorcraft_format::check(&s.format, s.version) {
Ok(()) => Ok(s),
Err(craft_vectorcraft_format::Error::Format(format)) => Err(malformed(format!("format is `{format}`"))),
@@ -424,6 +452,15 @@ pub(crate) fn info(bytes: &[u8], budget: &mut Budget) -> Result<(u32, Option<Str
Ok((s.version, s.generator))
}
fn inline_bytes(image: &Image, budget: &Budget, what: &str) -> Result<Vec<u8>, Error> {
let bytes = if budget.control.is_some() {
image.bytes_at_most_with_check(&[], u64::MAX, || budget.check_cancelled())?
} else {
image.bytes_at_most(&[], u64::MAX)
};
bytes.ok_or_else(|| malformed(format!("the {what} is not valid base64")))
}
pub(crate) fn renditions(bytes: &[u8], budget: &mut Budget) -> Result<Vec<Rendition>, Error> {
let Unpacked { json, tail } = unpack(bytes, budget)?;
let head = stamp::<true>(&json, budget, false)?;
@@ -437,10 +474,11 @@ pub(crate) fn renditions(bytes: &[u8], budget: &mut Budget) -> Result<Vec<Rendit
// member is counted here.
let compressed = matches!(tail, Tail::Stream { .. });
for (_, m, what) in &members {
let len = if budget.control.is_some() { m.len_with_check(|| budget.check_cancelled())? } else { m.len() };
if compressed && m.blob.is_some() {
budget.check_entry(m.len(), what)?;
budget.check_entry(len, what)?;
} else {
budget.take_entry(m.len(), what)?;
budget.take_entry(len, what)?;
}
}
let ranges: Vec<[u64; 2]> = members.iter().filter_map(|(_, m, _)| m.blob).collect();
@@ -448,10 +486,12 @@ pub(crate) fn renditions(bytes: &[u8], budget: &mut Budget) -> Result<Vec<Rendit
members
.into_iter()
.map(|(kind, m, what)| {
budget.check_cancelled()?;
let bytes = match m.blob {
Some(_) => blobs.next().flatten().ok_or_else(|| malformed(format!("the file ends before its {what}")))?,
None => m.bytes_at_most(&[], u64::MAX).ok_or_else(|| malformed(format!("the {what} is not valid base64")))?,
None => inline_bytes(&m, budget, what)?,
};
budget.check_cancelled()?;
Ok(Rendition { kind, mime: m.mime, bytes })
})
.collect()
@@ -517,34 +557,42 @@ struct KindM {
pub(crate) fn outline(bytes: &[u8], budget: &mut Budget) -> Result<Outline, Error> {
let json = unpack(bytes, budget)?.json;
stamp::<false>(&json, budget, false)?;
let f: WithDocument = parse(&json, budget.limits)?;
let pages = f
.document
let f: WithDocument = parse(&json, budget)?;
document_outline(f.document, budget)
}
fn document_outline(document: DocM, budget: &Budget) -> Result<Outline, Error> {
let pages = document
.artboards
.into_iter()
.map(|a| Page {
name: a.name,
x: a.rect.x0.min(a.rect.x1),
y: a.rect.y0.min(a.rect.y1),
width: (a.rect.x1 - a.rect.x0).abs(),
height: (a.rect.y1 - a.rect.y0).abs(),
timing: None,
layers: vec![],
.map(|a| {
budget.check_cancelled()?;
Ok(Page {
name: a.name,
x: a.rect.x0.min(a.rect.x1),
y: a.rect.y0.min(a.rect.y1),
width: (a.rect.x1 - a.rect.x0).abs(),
height: (a.rect.y1 - a.rect.y0).abs(),
timing: None,
layers: vec![],
})
})
.collect();
let layers = f
.document
.collect::<Result<_, Error>>()?;
let layers = document
.layers
.into_iter()
.map(|n| Layer {
name: n.name.unwrap_or_default(),
kind: n.kind.kind,
visible: n.visible,
opacity: Some(n.opacity),
blend: Some(n.blend.unwrap_or_else(|| "Normal".into())),
children: vec![],
.map(|n| {
budget.check_cancelled()?;
Ok(Layer {
name: n.name.unwrap_or_default(),
kind: n.kind.kind,
visible: n.visible,
opacity: Some(n.opacity),
blend: Some(n.blend.unwrap_or_else(|| "Normal".into())),
children: vec![],
})
})
.collect();
.collect::<Result<_, Error>>()?;
Ok(Outline { canvas: None, pages, layers })
}
@@ -554,6 +602,36 @@ mod tests {
use crate::tests::{unread_array, unread_object};
use serde::de::value::{Error as ValueError, MapDeserializer, SeqDeserializer, StrDeserializer};
#[test]
#[allow(clippy::unwrap_used)]
fn conversion_stops_inside_document_layers() {
use std::cell::Cell;
let node = r#"{"kind":{"type":"Path"}}"#;
let json = format!(r#"{{"layers":[{node},{node},{node},{node},{node}]}}"#);
let document: DocM = serde_json::from_str(&json).unwrap();
let polls = Cell::new(0);
let cancel = || {
polls.set(polls.get() + 1);
polls.get() == 4
};
let result = crate::cancel::run(&cancel, |control| document_outline(document, &Budget::controlled(&crate::Limits::default(), control)));
assert_eq!(result, Err(Error::Cancelled));
assert_eq!(polls.get(), 4);
}
#[test]
fn inline_cancellation_interrupts_extraction() {
let image = Image::inline("application/pdf", &vec![1; 2 << 20]);
let polls = std::cell::Cell::new(0);
let cancelled = || {
polls.set(polls.get() + 1);
polls.get() >= 10
};
let result = crate::cancel::run(&cancelled, |control| inline_bytes(&image, &Budget::controlled(&crate::Limits::default(), control), "PDF"));
assert!(matches!(result, Err(Error::Cancelled)));
assert_eq!(polls.get(), 10);
}
#[test]
fn an_image_of_the_wrong_shape_is_skipped_without_reading_it_whole() {
assert!(Lenient::<Image>::deserialize(unread_object(1000)).unwrap().0.is_none());
@@ -612,7 +690,7 @@ mod tests {
for shape in shapes {
let json = format!(r#"{{"format":"vectorcraft","version":4,"generator":{shape},"preview":{shape},"pdf":{shape},"document":0}}"#);
let file = craft_vectorcraft_format::File::<IgnoredAny>::read(json.as_bytes()).unwrap();
let stamp: Stamp<true> = crate::parse(json.as_bytes(), &crate::Limits::default()).unwrap();
let stamp: Stamp<true> = crate::parse(json.as_bytes(), &crate::Budget::new(&crate::Limits::default())).unwrap();
assert_eq!((&stamp.generator, &stamp.preview, &stamp.pdf), (&file.generator, &file.preview, &file.pdf), "{shape}");
}
}
crates/doc/src/zip.rs
+29 -12
View File
@@ -55,7 +55,8 @@ pub(crate) struct Zip<'a> {
}
impl<'a> Zip<'a> {
pub(crate) fn open(bytes: &'a [u8]) -> Result<Self, Error> {
pub(crate) fn open(bytes: &'a [u8], control: Option<&crate::cancel::Control<'_>>) -> Result<Self, Error> {
crate::cancel::check(control)?;
let last = bytes.len().checked_sub(EOCD_LEN).ok_or_else(|| malformed("not a ZIP archive (too short)"))?;
// The comment may itself hold the signature: accept only a record whose comment ends the
// file and whose central directory ends where the record starts.
@@ -87,6 +88,7 @@ impl<'a> Zip<'a> {
let mut entries = Vec::with_capacity(count.min(dir_size / CENTRAL_LEN));
let mut p = dir;
for _ in 0..count {
crate::cancel::check(control)?;
if u32_at(bytes, p) != Some(CENTRAL_SIG) || p + CENTRAL_LEN > eocd {
return Err(malformed("bad ZIP central directory entry"));
}
@@ -123,10 +125,16 @@ impl<'a> Zip<'a> {
/// The first entry named `name`. Each call walks the central directory; for many lookups,
/// use [`Zip::index`].
pub(crate) fn find(&self, name: &str) -> Option<&Entry> {
pub(crate) fn find(&self, name: &str, control: Option<&crate::cancel::Control<'_>>) -> Result<Option<&Entry>, Error> {
#[cfg(test)]
SCANS.with(|n| n.set(n.get() + 1));
self.entries.iter().find(|e| self.bytes.get(e.name.clone()) == Some(name.as_bytes()))
for e in &self.entries {
crate::cancel::check(control)?;
if self.bytes.get(e.name.clone()) == Some(name.as_bytes()) {
return Ok(Some(e));
}
}
Ok(None)
}
/// Every entry by name, the first of each name as [`Zip::find`] gives it, built in one walk of
@@ -163,18 +171,26 @@ impl<'a> Zip<'a> {
if e.compressed != size {
return Err(malformed(format!("{what} has the wrong size")));
}
raw.to_vec()
budget.copy(raw)?
} else {
// DEFLATE expands at most 1032 times.
let mut out = Vec::with_capacity(size.min(raw.len().saturating_mul(1032)));
flate2::read::DeflateDecoder::new(raw).take(e.size + 1).read_to_end(&mut out).map_err(|err| malformed(format!("{what}: {err}")))?;
let input = crate::cancel::Reader::new(raw, budget.control);
let decoder = flate2::read::DeflateDecoder::new(input);
crate::cancel::Reader::new(decoder, budget.control)
.take(e.size + 1)
.read_to_end(&mut out)
.map_err(|err| malformed(format!("{what}: {err}")))?;
if out.len() != size {
return Err(malformed(format!("{what} has the wrong size")));
}
out
};
let mut crc = flate2::Crc::new();
crc.update(&data);
for chunk in data.chunks(crate::cancel::CHUNK) {
budget.check_cancelled()?;
crc.update(chunk);
}
if crc.sum() != e.crc {
return Err(malformed(format!("{what} failed its CRC check")));
}
@@ -185,26 +201,27 @@ impl<'a> Zip<'a> {
/// Does a local header in `head` name `name`? Walks the local headers from the start, for a
/// head too short to reach the central directory; stops at an entry whose size is only in a
/// data descriptor.
pub(crate) fn local_names_include(head: &[u8], name: &str) -> bool {
pub(crate) fn local_names_include(head: &[u8], name: &str, control: Option<&crate::cancel::Control<'_>>) -> Result<bool, Error> {
let mut p = 0usize;
while u32_at(head, p) == Some(LOCAL_SIG) {
crate::cancel::check(control)?;
let (Some(flags), Some(compressed), Some(name_len), Some(extra_len)) =
(u16_at(head, p + 6), u32_at(head, p + 18), u16_at(head, p + 26), u16_at(head, p + 28))
else {
return false;
return Ok(false);
};
let start = p + LOCAL_LEN;
let end = start + usize::from(name_len);
if head.get(start..end) == Some(name.as_bytes()) {
return true;
return Ok(true);
}
if flags & 8 != 0 {
return false;
return Ok(false);
}
p = match end.checked_add(usize::from(extra_len)).and_then(|q| q.checked_add(compressed as usize)) {
Some(q) => q,
None => return false,
None => return Ok(false),
};
}
false
Ok(false)
}
crates/doc/tests/cancellation.rs
+135
View File
@@ -0,0 +1,135 @@
//! Cancellation is observed inside reads, with no timing or worker-thread scheduling assumptions.
mod common;
use std::cell::Cell;
use common::*;
use craft_doc::{Error, Limits, info, info_with_cancel, outline, outline_with_cancel, renditions, renditions_with_cancel};
type Read = fn(&[u8], &Limits, &dyn Fn() -> bool) -> Result<(), Error>;
const READS: [Read; 3] = [
|bytes, limits, cancel| info_with_cancel(bytes, limits, cancel).map(|_| ()),
|bytes, limits, cancel| renditions_with_cancel(bytes, limits, cancel).map(|_| ()),
|bytes, limits, cancel| outline_with_cancel(bytes, limits, cancel).map(|_| ()),
];
#[test]
fn cancellation_interrupts_detection_before_an_unrecognized_result() {
let json = format!(r#"{{"savedBy":"{}","unrelated":0}}"#, "\\n".repeat(1 << 20));
// Optional filename and comment fields are consumed before the decoded prefix.
let mut header = vec![0x1f, 0x8b, 8, 24, 0, 0, 0, 0, 0, 3];
header.extend(std::iter::repeat_n(b'x', 60_000));
header.push(0);
header.extend(std::iter::repeat_n(b'x', 60_000));
header.push(0);
header.extend(&gzip(b"unrelated")[10..]);
let entries: Vec<_> = (0..200).map(|_| store("other", b"")).collect();
let archive = zip(&entries);
// The truncated bundle forces detection to walk local headers instead of the directory.
let local = &archive[..archive.len() / 4];
for (case, bytes) in [json.as_bytes(), header.as_slice(), archive.as_slice(), local].into_iter().enumerate() {
for read in READS {
let threshold = if case == 1 { 3 } else { 10 };
let polls = Cell::new(0);
let cancel = || {
polls.set(polls.get() + 1);
polls.get() == threshold
};
assert_eq!(read(bytes, &Limits::default(), &cancel), Err(Error::Cancelled), "detection case {case}, polls {}", polls.get());
assert_eq!(polls.get(), threshold);
}
}
}
#[test]
fn reads_honor_an_initial_request_even_for_unrecognized_input() {
for read in READS {
assert_eq!(read(b"not a document", &Limits::default(), &|| true), Err(Error::Cancelled));
}
}
#[test]
fn uncanceled_reads_preserve_results_and_errors() {
let vc = vectorcraft_json(4, "");
for file in [vc.as_bytes().to_vec(), gzip(vc.as_bytes()), pcraft(&png(1, 1, 0), &png(2, 2, 0)), ecproj(2).into_bytes(), vec![1]] {
let limits = Limits::default();
assert_eq!(info_with_cancel(&file, &limits, &|| false), info(&file, &limits));
assert_eq!(renditions_with_cancel(&file, &limits, &|| false), renditions(&file, &limits));
assert_eq!(outline_with_cancel(&file, &limits, &|| false), outline(&file, &limits));
}
}
#[test]
fn cancellation_after_work_starts_interrupts_plain_and_gzip_json() {
let json = vectorcraft_json(4, &format!(r#""padding":"{}""#, "x".repeat(2 << 20)));
for file in [json.as_bytes().to_vec(), gzip(json.as_bytes())] {
for read in READS {
let polls = Cell::new(0);
let cancel = || {
polls.set(polls.get() + 1);
polls.get() >= 20
};
assert_eq!(read(&file, &Limits::default(), &cancel), Err(Error::Cancelled));
assert_eq!(polls.get(), 20, "the request is latched instead of being polled again");
}
}
}
#[test]
fn cancellation_interrupts_zip_manifest_reads() {
let json = pcraft_manifest(1, "[]", &format!(r#","padding":"{}""#, "x".repeat(2 << 20)));
for entry in [store("manifest.json", json.as_bytes()), deflated("manifest.json", json.as_bytes())] {
let file = zip(&[entry]);
for read in READS {
let polls = Cell::new(0);
let cancel = || {
polls.set(polls.get() + 1);
polls.get() >= 20
};
assert_eq!(read(&file, &Limits::default(), &cancel), Err(Error::Cancelled));
}
}
}
#[test]
fn cancellation_interrupts_the_effectcraft_header_parse() {
let original = ecproj(2);
let prefix = original.trim_end().strip_suffix('}').unwrap();
let json = format!(r#"{prefix},"padding":"{}"}}"#, "x".repeat(2 << 20));
// Allow the initial callback, input-size check and complete depth scan. Cancellation then
// arrives after the header's own reader has consumed some input.
let threshold = json.len().div_ceil(64 * 1024) + 6;
let polls = Cell::new(0);
let cancel = || {
polls.set(polls.get() + 1);
polls.get() >= threshold
};
assert_eq!(info_with_cancel(json.as_bytes(), &Limits::default(), &cancel), Err(Error::Cancelled));
}
#[test]
fn controlled_reads_keep_limit_and_malformed_input_errors() {
let json = vectorcraft_json(4, r#""preview":{"mime":"image/png","data":"AQID"}"#);
let files = [
json.as_bytes().to_vec(),
gzip(json.as_bytes()),
json.as_bytes()[..json.len() - 1].to_vec(),
vectorcraft_json(99, "").into_bytes(),
pcraft(&png(1, 1, 0), &png(2, 2, 0)),
];
for file in files {
let mut bounds = vec![Limits::default(); 6];
bounds[1].max_json = 16;
bounds[2].max_entry = 0;
bounds[3].max_total = 8;
bounds[4].max_depth = 1;
bounds[5].max_items = 0;
for limits in bounds {
assert_eq!(info_with_cancel(&file, &limits, &|| false), info(&file, &limits));
assert_eq!(renditions_with_cancel(&file, &limits, &|| false), renditions(&file, &limits));
assert_eq!(outline_with_cancel(&file, &limits, &|| false), outline(&file, &limits));
}
}
}
crates/ecproj-format/README.md
+2
View File
@@ -38,6 +38,8 @@ let head = Head::read(&text)?;
flattened into it. `to_json()` gives the file's text indented with two spaces, and
`to_json_compact()` gives it on one line. Neither checks the values: `serde_json` writes NaN and
infinity as `null`, so the caller checks its values before saving.
- `Head::read_from(reader)` applies the same header parsing to a `std::io::Read`, allowing a
caller to bound or interrupt input. File/network readers should be buffered.
- `Head::read(text) -> Result<Head, ReadError>` reads `savedBy` and `schema` from one JSON
object, and skips other members. `savedBy` is `None` when it is absent, is not a string, or
appears more than once. Text that is not a JSON object, has no `schema` that is a `u32`, or has
crates/ecproj-format/src/lib.rs
+10 -1
View File
@@ -139,7 +139,16 @@ impl Head {
/// members are skipped, and two `schema` members are refused. A schema newer than
/// [`SCHEMA_VERSION`] is refused with [`ReadError::TooNew`], which holds the file's `savedBy`.
pub fn read(text: &str) -> Result<Head, ReadError> {
let mut deserializer = serde_json::Deserializer::from_str(text);
Self::read_deserializer(serde_json::Deserializer::from_str(text))
}
/// [`Head::read`] from a reader, which may enforce input limits or cooperative
/// interruption. Buffer file or network readers to avoid one-byte I/O requests.
pub fn read_from(reader: impl std::io::Read) -> Result<Head, ReadError> {
Self::read_deserializer(serde_json::Deserializer::from_reader(reader))
}
fn read_deserializer<'de, R: serde_json::de::Read<'de>>(mut deserializer: serde_json::Deserializer<R>) -> Result<Head, ReadError> {
let head = deserializer.deserialize_map(HeadVisitor)?;
deserializer.end()?;
if let Err(too_new) = check_schema(head.schema) {
crates/ecproj-format/tests/envelope.rs
+50
View File
@@ -120,6 +120,56 @@ fn a_project_saved_before_saved_by_has_a_head_without_it() {
assert_eq!(Head::read(text).unwrap(), Head { saved_by: None, schema: 1 });
}
#[test]
fn reader_heads_preserve_string_reader_behavior() {
for text in [
r#"{"schema":2,"savedBy":"writer","items":{}}"#,
r#"{"schema":99,"savedBy":"new writer"}"#,
r#"{"schema":2,"savedBy":"first","savedBy":"second"}"#,
r#"{"schema":2,"schema":2}"#,
r#"{"schema":"bad"}"#,
"[]",
"",
] {
match (Head::read_from(text.as_bytes()), Head::read(text)) {
(Ok(from_reader), Ok(from_str)) => assert_eq!(from_reader, from_str),
(Err(ReadError::Json(from_reader)), Err(ReadError::Json(from_str))) => {
assert_eq!(from_reader.classify(), from_str.classify());
// serde's reader may consume lookahead before reporting the same error.
assert_eq!(from_reader.to_string().split(" at line ").next(), from_str.to_string().split(" at line ").next());
}
(Err(ReadError::TooNew { too_new: a, saved_by: a_writer, .. }), Err(ReadError::TooNew { too_new: b, saved_by: b_writer, .. })) => {
assert_eq!(a.schema, b.schema);
assert_eq!(a_writer, b_writer);
}
(from_reader, from_str) => panic!("different header results: {from_reader:?}, {from_str:?}"),
}
}
}
#[test]
fn reader_heads_propagate_an_interruption_before_consuming_all_input() {
struct Interrupted {
input: std::io::Cursor<Vec<u8>>,
left: usize,
}
impl std::io::Read for Interrupted {
fn read(&mut self, bytes: &mut [u8]) -> std::io::Result<usize> {
if self.left == 0 {
return Err(std::io::Error::other("stopped"));
}
let len = bytes.len().min(self.left);
let n = std::io::Read::read(&mut self.input, &mut bytes[..len])?;
self.left -= n;
Ok(n)
}
}
let json = format!(r#"{{"schema":2,"padding":"{}"}}"#, "x".repeat(128 * 1024));
let mut reader = Interrupted { input: std::io::Cursor::new(json.into_bytes()), left: 32 * 1024 };
assert!(matches!(Head::read_from(&mut reader), Err(ReadError::Json(e)) if e.is_io()));
assert_eq!(reader.input.position(), 32 * 1024);
}
#[test]
fn a_head_needs_a_schema() {
assert!(Head::read(r#"{"savedBy": "0.12.0", "items": {}}"#).is_err());
crates/pcraft-format/README.md
+3 -1
View File
@@ -57,11 +57,13 @@ let manifest: Manifest<MyDocument> = craft_pcraft_format::read_manifest(&manifes
without recursion. PhotoCraft's own check of a manifest it saves tests the nesting and the
item count only. `parse_manifest(json, &Limits)` runs that check and then parses into a
`serde_json::Value` with serde_json's 128-level recursion limit turned off.
- `check_manifest_with_cancel(json, &Limits, &cancelled)` applies those same checks while
polling a callback between bounded portions of JSON. A true result stops with `Error::Cancelled`.
- `migrate(&mut value, steps, target)` upgrades the JSON in place, where `steps[i]` upgrades
version `i + 1` to `i + 2`, and returns the version the file had. `Step<E>` is
`fn(&mut Value) -> Result<(), E>`; with `E: From<Error>`, the steps return the caller's own
error type.
- `Error` is `Corrupt`, `Unsupported`, `TooNew`, `LimitExceeded` or `Json`. A manifest whose
- `Error` is `Corrupt`, `Unsupported`, `TooNew`, `LimitExceeded`, `Json` or `Cancelled`. A manifest whose
`format_version` is newer than `target` gives `TooNew`, and one with no step to the next
version gives `Unsupported`. A missing or invalid `format_version` gives `Corrupt`, and any
other missing or mistyped member gives `Json`. `TooNew` holds `found`, `supported` and the manifest's
crates/pcraft-format/src/lib.rs
+6 -1
View File
@@ -37,7 +37,9 @@ mod read;
use serde::{Deserialize, Serialize};
pub use limits::{Limits, MAX_DEPTH, MAX_GROUP_DEPTH, MAX_MANIFEST_ITEMS, check_manifest, manifest_depth, parse_manifest};
pub use limits::{
Limits, MAX_DEPTH, MAX_GROUP_DEPTH, MAX_MANIFEST_ITEMS, check_manifest, check_manifest_with_cancel, manifest_depth, parse_manifest,
};
pub use migrate::{Step, migrate};
pub use read::{read_manifest, read_manifest_with};
@@ -168,6 +170,9 @@ impl TileRef {
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum Error {
/// A cancellation-aware validation observed the caller's request.
#[error("manifest check canceled")]
Cancelled,
/// The manifest breaks the format.
#[error("corrupt .pcraft bundle: {0}")]
Corrupt(String),
crates/pcraft-format/src/limits.rs
+39 -21
View File
@@ -65,37 +65,55 @@ impl Default for Limits {
/// This check also refuses JSON larger than [`Limits::max_manifest_bytes`], so a writer that runs
/// it before saving refuses a manifest of more than that size.
pub fn check_manifest(json: &[u8], limits: &Limits) -> Result<(), Error> {
checked_manifest(json, limits, None)
}
/// [`check_manifest`] with cooperative cancellation while scanning the JSON. The callback
/// runs on this thread; returning true stops validation with [`Error::Cancelled`].
pub fn check_manifest_with_cancel(json: &[u8], limits: &Limits, cancelled: &dyn Fn() -> bool) -> Result<(), Error> {
checked_manifest(json, limits, Some(cancelled))
}
fn checked_manifest(json: &[u8], limits: &Limits, cancelled: Option<&dyn Fn() -> bool>) -> Result<(), Error> {
let check = || match cancelled {
Some(cancelled) if cancelled() => Err(Error::Cancelled),
_ => Ok(()),
};
check()?;
if json.len() > limits.max_manifest_bytes {
return Err(Error::LimitExceeded(format!("`{}` exceeds {} bytes", crate::MANIFEST, limits.max_manifest_bytes)));
}
let max_depth = limits.max_depth.min(MAX_DEPTH);
let (mut depth, mut items, mut in_str, mut escaped) = (0usize, 0usize, false, false);
for &b in json {
if in_str {
for chunk in json.chunks(64 * 1024) {
check()?;
for &b in chunk {
if in_str {
match b {
_ if escaped => escaped = false,
b'\\' => escaped = true,
b'"' => in_str = false,
_ => {}
}
continue;
}
match b {
_ if escaped => escaped = false,
b'\\' => escaped = true,
b'"' => in_str = false,
b'"' => in_str = true,
b'[' | b'{' => {
depth += 1;
if depth > max_depth {
return Err(Error::LimitExceeded(format!("manifest JSON nested deeper than {max_depth} levels")));
}
}
b']' | b'}' => depth = depth.saturating_sub(1),
_ => {}
}
continue;
}
match b {
b'"' => in_str = true,
b'[' | b'{' => {
depth += 1;
if depth > max_depth {
return Err(Error::LimitExceeded(format!("manifest JSON nested deeper than {max_depth} levels")));
if matches!(b, b'[' | b'{' | b',') {
items += 1;
if items > limits.max_items {
return Err(Error::LimitExceeded(format!("manifest JSON holds more than {} values", limits.max_items)));
}
}
b']' | b'}' => depth = depth.saturating_sub(1),
_ => {}
}
if matches!(b, b'[' | b'{' | b',') {
items += 1;
if items > limits.max_items {
return Err(Error::LimitExceeded(format!("manifest JSON holds more than {} values", limits.max_items)));
}
}
}
Ok(())
crates/pcraft-format/tests/limits.rs
+21 -1
View File
@@ -1,4 +1,24 @@
use craft_pcraft_format::{Error, Limits, MAX_DEPTH, MAX_GROUP_DEPTH, MAX_MANIFEST_ITEMS, check_manifest, manifest_depth, parse_manifest};
use craft_pcraft_format::{
Error, Limits, MAX_DEPTH, MAX_GROUP_DEPTH, MAX_MANIFEST_ITEMS, check_manifest, check_manifest_with_cancel, manifest_depth, parse_manifest,
};
#[test]
fn cancellation_stops_manifest_validation_after_it_starts() {
let polls = std::cell::Cell::new(0);
let cancelled = || {
polls.set(polls.get() + 1);
polls.get() >= 3
};
let json = format!(r#"{{"padding":"{}"}}"#, "x".repeat(256 * 1024));
assert!(matches!(check_manifest_with_cancel(json.as_bytes(), &Limits::default(), &cancelled), Err(Error::Cancelled)));
assert_eq!(polls.get(), 3);
for limits in [Limits::default(), small(1, 1)] {
assert_eq!(
check_manifest_with_cancel(json.as_bytes(), &limits, &|| false).map_err(|e| e.to_string()),
check_manifest(json.as_bytes(), &limits).map_err(|e| e.to_string())
);
}
}
fn nested(depth: usize) -> Vec<u8> {
let mut json = "[".repeat(depth).into_bytes();
crates/vectorcraft-format/README.md
+4
View File
@@ -94,6 +94,10 @@ let pdf = file.pdf.map(|p| entries.bytes(&p)).transpose()?.flatten();
for version 4 and entries of at least `INLINE_MAX` bytes, inline otherwise. `Tail::write_to`
writes the marker and the entries.
- `base64::encode`, `decode` and `decoded_len` handle the `data` members.
- `Image::len_with_check` and `bytes_at_most_with_check` accept a fallible checkpoint called
during bounded scans, base64 decoding and blob copying. The checkpoint error is returned
unchanged; `Ok(None)` keeps `bytes_at_most`'s meaning. `base64::decoded_len_with_check` and
`decode_with_check` expose the same checkpoints for standalone base64 data.
Size limits on the JSON and on decompression stay with each reader.
crates/vectorcraft-format/src/base64.rs
+104
View File
@@ -41,6 +41,69 @@ pub fn decoded_len(s: &str) -> u64 {
(n / 4 * 3).saturating_sub(pad)
}
pub(crate) const CHECK_CHUNK: usize = 64 * 1024;
/// [`decoded_len`] with a fallible checkpoint before each bounded portion of input.
pub fn decoded_len_with_check<E>(s: &str, mut check: impl FnMut() -> Result<(), E>) -> Result<u64, E> {
let (n, pad) = measured_with_check(s, &mut check)?;
Ok((n / 4 * 3).saturating_sub(pad))
}
fn measured_with_check<E>(s: &str, mut check: impl FnMut() -> Result<(), E>) -> Result<(u64, u64), E> {
check()?;
let (mut n, mut pad) = (0u64, 0u64);
for chunk in s.as_bytes().chunks(CHECK_CHUNK) {
check()?;
for &b in chunk {
if !b.is_ascii_whitespace() {
n += 1;
pad = if b == b'=' { (pad + 1).min(2) } else { 0 };
}
}
}
Ok((n, pad))
}
/// [`decode`] with fallible checkpoints during length calculation and decoding. Whitespace,
/// padding and non-canonical groups follow [`decode`]; `Ok(None)` means invalid base64.
pub fn decode_with_check<E>(s: &str, mut check: impl FnMut() -> Result<(), E>) -> Result<Option<Vec<u8>>, E> {
let (n, pad) = measured_with_check(s, &mut check)?;
if !n.is_multiple_of(4) {
return Ok(None);
}
let Ok(len) = usize::try_from((n / 4 * 3).saturating_sub(pad)) else { return Ok(None) };
decode_chunks(s, len, &mut check)
}
fn decode_chunks<E>(s: &str, capacity: usize, check: &mut impl FnMut() -> Result<(), E>) -> Result<Option<Vec<u8>>, E> {
let mut out = Vec::with_capacity(capacity);
let mut clean = Vec::with_capacity(CHECK_CHUNK);
let mut ended = false;
for chunk in s.as_bytes().chunks(CHECK_CHUNK) {
check()?;
for &b in chunk {
if b.is_ascii_whitespace() {
continue;
}
if ended {
return Ok(None);
}
clean.push(b);
if clean.len() == CHECK_CHUNK {
let Ok(text) = std::str::from_utf8(&clean) else { return Ok(None) };
let Some(decoded) = decode(text) else { return Ok(None) };
out.extend_from_slice(&decoded);
ended = clean.contains(&b'=');
clean.clear();
}
}
}
let Ok(text) = std::str::from_utf8(&clean) else { return Ok(None) };
let Some(decoded) = decode(text) else { return Ok(None) };
out.extend_from_slice(&decoded);
Ok(Some(out))
}
/// The bytes `s` encodes; `None` when it is not valid padded base64. Padding is allowed only at
/// the end. The unused bits before the padding are not checked, so a non-canonical group decodes
/// like its canonical form (`Zh==` gives the bytes of `Zg==`, and `Zm9=` those of `Zm8=`), as
@@ -99,6 +162,8 @@ mod tests {
assert_eq!(encode(dec.as_bytes()), enc);
assert_eq!(decode(enc).as_deref(), Some(dec.as_bytes()), "{enc}");
assert_eq!(decoded_len(enc), dec.len() as u64, "{enc}");
assert_eq!(decode_with_check(enc, || Ok::<_, ()>(())).unwrap(), decode(enc));
assert_eq!(decoded_len_with_check(enc, || Ok::<_, ()>(())).unwrap(), decoded_len(enc));
}
assert_eq!(decode("Zm9v\nYmFy").as_deref(), Some(&b"foobar"[..]));
}
@@ -125,6 +190,45 @@ mod tests {
fn misplaced_padding_and_foreign_characters_are_refused() {
for s in ["====", "A===", "AA=A", "=AAA", "QQ==QUJD", "QUI=QUJD", "QUJ", "QU*D", "QUJDé"] {
assert_eq!(decode(s), None, "{s}");
assert_eq!(decode_with_check(s, || Ok::<_, ()>(())).unwrap(), None, "{s}");
}
}
#[test]
fn checked_blocks_preserve_padding_and_whitespace() {
for pad in ["QUJD", "Zh==", "Zm9=", "A===", "QUJ"] {
let data = format!("{}\n{pad}\t", "QUJD".repeat(CHECK_CHUNK / 4 - 1));
for data in [data.clone(), format!("{data}QUJD")] {
assert_eq!(decode_with_check(&data, || Ok::<_, ()>(())).unwrap(), decode(&data));
assert_eq!(decoded_len_with_check(&data, || Ok::<_, ()>(())).unwrap(), decoded_len(&data));
}
}
}
#[test]
fn decoding_can_stop_after_its_first_block() {
let data = "QUJD".repeat(CHECK_CHUNK);
let mut polls = 0;
let mut check = || {
polls += 1;
if polls == 2 { Err("stopped") } else { Ok(()) }
};
assert_eq!(decode_chunks(&data, 0, &mut check), Err("stopped"));
assert_eq!(polls, 2);
}
#[test]
fn checked_image_reads_keep_bounds_and_blob_ranges() {
let mut image = crate::Image::inline("application/pdf", b"abcdef");
for max in [0, 5, 6, u64::MAX] {
assert_eq!(image.bytes_at_most_with_check(b"", max, || Ok::<_, ()>(())).unwrap(), image.bytes_at_most(b"", max));
}
image.blob = Some([2, 3]);
for tail in [&b"123456"[..], b"12"] {
for max in [0, 2, 3, u64::MAX] {
assert_eq!(image.bytes_at_most_with_check(tail, max, || Ok::<_, ()>(())).unwrap(), image.bytes_at_most(tail, max));
}
}
assert_eq!(image.bytes_at_most_with_check(b"123456", 3, || Err(17)), Err(17));
}
}
crates/vectorcraft-format/src/lib.rs
+32
View File
@@ -406,6 +406,18 @@ impl Image {
self.len() == 0
}
/// [`Image::len`] with fallible checkpoints while scanning inline data. The checkpoint's
/// error is returned unchanged.
pub fn len_with_check<E>(&self, mut check: impl FnMut() -> Result<(), E>) -> Result<u64, E> {
match self.blob {
Some([_, len]) => {
check()?;
Ok(len)
}
None => base64::decoded_len_with_check(&self.data, check),
}
}
/// The entry's bytes: `data` decoded, or the range `blob` names in `tail` (the data after the
/// JSON, from [`split_tail`]). `None` when the entry states more than [`MAX_ENTRY`] bytes,
/// when `data` is not valid base64, or when `tail` ends before the range does. Each call
@@ -426,6 +438,26 @@ impl Image {
None => base64::decode(&self.data),
}
}
/// [`Image::bytes_at_most`] with fallible checkpoints during size checks, base64 decoding
/// and copying. `Ok(None)` has the same meaning as [`Image::bytes_at_most`].
pub fn bytes_at_most_with_check<E>(&self, tail: &[u8], max: u64, mut check: impl FnMut() -> Result<(), E>) -> Result<Option<Vec<u8>>, E> {
if self.len_with_check(&mut check)? > max {
return Ok(None);
}
match self.blob {
Some(at) => {
let Some(bytes) = entry(tail, at) else { return Ok(None) };
let mut out = Vec::with_capacity(bytes.len());
for chunk in bytes.chunks(base64::CHECK_CHUNK) {
check()?;
out.extend_from_slice(chunk);
}
Ok(Some(out))
}
None => base64::decode_with_check(&self.data, check),
}
}
}
/// The bytes `[offset, length]` names in `tail`; `None` when `tail` ends before them.