Merge remote-tracking branch 'origin/codex/shared-dng-opcode-envelope' into w1647

This commit is contained in:
FredApps committed 2026-11-16 08:32:30 +01:00
commit b611b03a29
140 files changed
+14812 -988

No files matched your search

+1 -1
View File
@@ -23,5 +23,5 @@ jobs:
# The codec and format code behind features that are off by default. Without these the
# coverage build never compiles it, and new code there reads as untested.
cargo-features: >-
craft-video/dnx,craft-video/vp8,craft-video/prores,craft-video/h264,craft-video/hevc,craft-video/vp9,craft-video/av1,craft-video/threads,craft-audio/decode,craft-audio/encode,craft-audio/peaks,craft-audio/rayon,craft-audio/serde,craft-media/matroska,craft-media/ogg,craft-media/bitstream,craft-media/serde,craft-mesh/stl,craft-mesh/obj,craft-mesh/ply,craft-mesh/gltf,craft-metafile/write
craft-video/dnx,craft-video/vp8,craft-video/prores,craft-video/h264,craft-video/hevc,craft-video/vp9,craft-video/av1,craft-video/threads,craft-audio/decode,craft-audio/encode,craft-audio/peaks,craft-audio/rayon,craft-audio/serde,craft-media/matroska,craft-media/ogg,craft-media/bitstream,craft-media/serde,craft-mesh/stl,craft-mesh/obj,craft-mesh/ply,craft-mesh/gltf,craft-metafile/write,craft-image-codecs/sgi,craft-image-codecs/sunraster,craft-jsonrpc/async-lines
secrets: inherit
+1
View File
@@ -101,4 +101,5 @@ and the exact transfer revision are recorded in each vendored crate's SOURCE.jso
`crates/plist` is extracted from `storytold/craft-anything` (`crates/formats/src/plist.rs` and checked byte reads), at source main `88d1d9cac852aec1c7fd97886db451e005e49d2a`. MIT OR Apache-2.0; source license texts are retained in the crate. Primary import Git authors preserve Brandon Thomas, BFlat, RRFZ and Afloat. Regression imports preserve BFlat. XML, JSON projection, calendar formatting and app policy were not imported into the public API.
`crates/tga/src/metadata_tests.rs` and metadata-boundary helpers in `crates/tga/src/lib.rs` are adapted from nuclearspike's FilmCraft #1932 at `e7842f76f887005c460c64a8ab51d59d086e6f27` (MIT OR Apache-2.0). Synthetic fixtures are original contributor test data; primary Git author retained.
The XML reader in `crates/plist/src/xml.rs` and its base64 compatibility helper are ported from Craft Anything `8f3b775157d331dd31cfa4da911e26d8856b8c77` (same MIT OR Apache-2.0 source licenses). Brandon Thomas is the primary import author; retained hardening credits include Afloat, BFlat, Rahul, RRFZ and NetterSilver. The finite-real fix preserves gmrmad1 as author from Craft Anything #1905 (`d85e104f`); quoted-attribute/line-ending regressions preserve RRFZ. Bounds/adaptation and bounded entity scanning are subsequent Dylan M. Taylor changes. App-specific JSON projections and keyed-archive interpretation remain outside this crate.
Generated
+52 -5
View File
@@ -1785,9 +1785,12 @@ name = "craft-font-catalog"
version = "0.1.0"
dependencies = [
"fontique",
"harfrust 0.13.3",
"kurbo 0.13.1",
"skrifa 0.47.0",
"thiserror 2.0.21",
"unicode-bidi",
"unicode-segmentation",
"windows-registry",
"write-fonts 0.54.1",
]
@@ -1846,6 +1849,16 @@ dependencies = [
"craft-fonts-build",
]
[[package]]
name = "craft-format"
version = "0.1.0"
dependencies = [
"craft-zip",
"proptest",
"serde_json",
"sha2 0.10.9",
]
[[package]]
name = "craft-gainmap"
version = "0.1.0"
@@ -1868,6 +1881,15 @@ dependencies = [
"image",
]
[[package]]
name = "craft-gridcraft-format"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-gzip"
version = "0.1.0"
@@ -1887,6 +1909,7 @@ name = "craft-heif"
version = "0.1.0"
dependencies = [
"heic-rs",
"proptest",
"rayon",
]
@@ -1903,6 +1926,10 @@ dependencies = [
"proptest",
]
[[package]]
name = "craft-http-url"
version = "0.1.0"
[[package]]
name = "craft-i18n"
version = "0.1.0"
@@ -2009,6 +2036,7 @@ name = "craft-jsonrpc"
version = "0.1.0"
dependencies = [
"serde_json",
"tokio",
]
[[package]]
@@ -2151,6 +2179,7 @@ version = "0.1.0"
dependencies = [
"craft-opc",
"craft-xml",
"craft-zip",
"miniz_oxide 0.9.1",
"serde_json",
]
@@ -2580,7 +2609,6 @@ dependencies = [
"craft-crw",
"craft-metadata",
"craft-tiff",
"image",
"jpeg-encoder 0.7.1",
"jxl-oxide",
"miniz_oxide 0.9.1",
@@ -2645,6 +2673,10 @@ dependencies = [
name = "craft-sheet"
version = "0.1.0"
[[package]]
name = "craft-shx"
version = "0.1.0"
[[package]]
name = "craft-silhouette-studio"
version = "0.1.0"
@@ -2772,6 +2804,7 @@ version = "0.1.0"
dependencies = [
"hypher",
"icu_segmenter",
"kurbo 0.13.1",
"miniz_oxide 0.8.9",
"unicode-bidi",
"unicode-bidi-mirroring",
@@ -6128,6 +6161,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519"
dependencies = [
"cfg-if",
"rayon",
]
[[package]]
@@ -7965,6 +7999,7 @@ dependencies = [
"loop9",
"quick-error 2.0.1",
"rav1e",
"rayon",
"rgb",
]
@@ -8516,9 +8551,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f"
[[package]]
name = "rustls-webpki"
version = "0.103.15"
version = "0.103.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281"
dependencies = [
"aws-lc-rs",
"ring",
@@ -9908,9 +9943,21 @@ dependencies = [
"mio",
"pin-project-lite",
"socket2",
"tokio-macros",
"windows-sys 0.61.2",
]
[[package]]
name = "tokio-macros"
version = "2.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.6",
]
[[package]]
name = "tokio-rustls"
version = "0.26.6"
@@ -11929,9 +11976,9 @@ dependencies = [
[[package]]
name = "zip"
version = "9.1.0"
version = "9.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2886b00003a72c4ccee3f42cfc34908d33b2508a74d827dc2a4244706475c704"
checksum = "1156b39d142a34b999c2310a50455e0cb3760bf7efad0122f796d813bee26ccb"
dependencies = [
"crc32fast",
"flate2",
+3
View File
@@ -11,6 +11,8 @@ rust-version = "1.95"
repository = "https://github.com/storytold/craft-libs"
[workspace.dependencies]
craft-shx = { path = "crates/shx" }
craft-http-url = { path = "crates/http-url" }
craft-usd = { path = "crates/usd" }
craft-smime = { path = "crates/smime" }
craft-ber = { path = "crates/ber" }
@@ -40,6 +42,7 @@ bytemuck = "1"
kurbo = "0.13"
thiserror = "2"
craft-gzip = { path = "crates/gzip" }
craft-gridcraft-format = { path = "crates/gridcraft-format" }
craft-aaf = { path = "crates/aaf" }
craft-atomic = { path = "crates/atomic" }
craft-captions = { path = "crates/captions" }
+3
View File
@@ -176,6 +176,9 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
every frame's position, delay, disposal and transparent colour, the loop count and the first
frame's colour table, without decompressing pixels. Tells an animation from a still and whether
a file can hold the picture it declares before a decoder runs.
- [`craft-gridcraft-format`](crates/gridcraft-format/README.md): GridCraft's native JSON workbook
(`.json`, `.scjson`): the extensions, the reader, the pretty writer, and newer entries kept
unread and written back into a list. The workbook model and what follows a read stay in GridCraft.
- [`craft-hdr`](crates/hdr/src/lib.rs): Radiance HDR (RGBE) picture reader, from FilmCraft: flat,
old and new run-length scan lines and all eight axis orders, to linear float RGB. Scan lines are
validated before the picture is allocated.
+45 -2
View File
@@ -57,13 +57,56 @@ pub struct WriteOptions<'a> {
/// reported.
pub fn write(doc: &Document, opts: &WriteOptions<'_>, report: &mut Report) -> Result<Vec<u8>> {
let version = if opts.small_sectors { craft_cfb::Version::V3 } else { craft_cfb::Version::V4 };
write::write(doc, version, &opts.identity, report)
write::write(doc, version, &opts.identity, &ReferenceOptions::default(), report)
}
/// Optional interpretation choices for applications retaining older interchange behavior.
#[derive(Clone, Copy, Debug)]
pub struct ReadOptions {
/// Interpret a source clip's start using its target slot's edit rate.
pub source_slot_rate: bool,
/// Use the referenced mob name when the clip has no explicit name.
pub infer_clip_name: bool,
/// Decode percent escapes in relative network locators.
pub decode_relative_locator: bool,
/// Report selector and nested-scope flattening.
pub report_flattening: bool,
/// Accept truncated PCM WAVE essence summaries.
pub permissive_pcm: bool,
}
impl Default for ReadOptions {
fn default() -> Self {
Self { source_slot_rate: true, infer_clip_name: true, decode_relative_locator: true, report_flattening: true, permissive_pcm: false }
}
}
/// Read using explicitly selected interpretation choices.
pub fn read_with_options(bytes: &[u8], options: &ReadOptions, report: &mut Report) -> Result<Document> {
read::read(bytes, options, report)
}
/// Optional compatibility choices for writing source references and locators.
#[derive(Clone, Copy, Debug)]
pub struct ReferenceOptions {
/// Write source offsets in target-slot units.
pub source_slot_rate: bool,
/// Percent-encode relative locators.
pub encode_relative_locator: bool,
}
impl Default for ReferenceOptions {
fn default() -> Self {
Self { source_slot_rate: true, encode_relative_locator: true }
}
}
/// Write with explicitly selected reference and locator conventions.
pub fn write_with_references(doc: &Document, opts: &WriteOptions<'_>, references: &ReferenceOptions, report: &mut Report) -> Result<Vec<u8>> {
let version = if opts.small_sectors { craft_cfb::Version::V3 } else { craft_cfb::Version::V4 };
write::write(doc, version, &opts.identity, references, report)
}
/// Read an AAF file: its top-level compositions, the compositions nested in them and their
/// sources. What cannot be represented is skipped and named in `report`.
pub fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
read::read(bytes, report)
read::read(bytes, &ReadOptions::default(), report)
}
/// Whether `bytes` look like an AAF file (a compound file whose root has the AAF root class or a
+25 -7
View File
@@ -67,6 +67,7 @@ fn indirect_f64(d: &[u8]) -> Option<f64> {
struct R<'a> {
report: &'a mut Report,
options: super::ReadOptions,
mobs: HashMap<MobId, &'a Obj>,
essence: HashMap<MobId, &'a [u8]>,
doc: Document,
@@ -90,7 +91,7 @@ struct NestClip<'a> {
/// Compositions nested deeper than this are read as gaps.
const MAX_NESTING: usize = 16;
pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
pub(crate) fn read(bytes: &[u8], options: &super::ReadOptions, report: &mut Report) -> Result<Document> {
let root = store::read(bytes).map_err(|e| Error::Parse { format: "AAF", message: e })?;
let header = root.strong(pid::ROOT_HEADER).ok_or_else(|| Error::Parse { format: "AAF", message: "no header object".into() })?;
let content = header.strong(pid::CONTENT).ok_or_else(|| Error::Parse { format: "AAF", message: "no content storage".into() })?;
@@ -106,7 +107,8 @@ pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
essence.insert(id, d);
}
}
let mut r = R { report, mobs, essence, doc: Document::default(), sources: HashMap::new(), nests: HashMap::new(), open: Vec::new() };
let mut r =
R { report, options: *options, mobs, essence, doc: Document::default(), sources: HashMap::new(), nests: HashMap::new(), open: Vec::new() };
// Top-level compositions: those no other composition uses, among the ones tagged top-level
// when any are. (Premiere Pro tags a nested sequence's composition top-level too; it is still
// only a nested sequence.) Compositions that only use each other in a circle all count. The
@@ -331,7 +333,9 @@ impl R<'_> {
items
}
x if x == cls::SELECTOR => {
self.report.info("a selector was flattened to its selected input");
if self.options.report_flattening {
self.report.info("a selector was flattened to its selected input");
}
match c.strong(pid::SELECTED) {
Some(s) => {
let mut items = self.segment(s, rate, kind, depth);
@@ -342,7 +346,9 @@ impl R<'_> {
}
}
x if x == cls::NESTED_SCOPE => {
self.report.info("a nested scope was flattened to its final input");
if self.options.report_flattening {
self.report.info("a nested scope was flattened to its final input");
}
match c.objs(pid::NESTED_SLOTS).last() {
Some(s) => {
let mut items = self.segment(s, rate, kind, depth);
@@ -388,7 +394,12 @@ impl R<'_> {
let len = units_to_ticks(len_of(c), rate.0, rate.1);
let tagged_name = tagged_values(c, pid::COMPONENT_USER_COMMENTS).into_iter().find(|(k, _)| k == "Clip Name").map(|(_, v)| v);
let (Some(id), Some(slot)) = (c.mob_id(pid::SOURCE_ID), c.u32(pid::SOURCE_MOB_SLOT_ID)) else { return CItem::Filler(len) };
let name = tagged_name.or_else(|| self.mobs.get(&id).and_then(|mob| mob.string(pid::MOB_NAME))).unwrap_or_default();
let name = if self.options.infer_clip_name {
tagged_name.or_else(|| self.mobs.get(&id).and_then(|mob| mob.string(pid::MOB_NAME)))
} else {
tagged_name
}
.unwrap_or_default();
if id == [0; 32] {
return CItem::Filler(len);
}
@@ -401,6 +412,7 @@ impl R<'_> {
.and_then(|mob| mob.objs(pid::SLOTS).iter().find(|s| s.u32(pid::SLOT_ID) == Some(slot)))
.and_then(|s| rate_of(s, pid::EDIT_RATE))
.unwrap_or(rate);
let source_rate = if self.options.source_slot_rate { source_rate } else { rate };
let start = units_to_ticks(c.i64(pid::START_TIME).unwrap_or(0), source_rate.0, source_rate.1);
match self.resolve(id, slot, start, kind, master_key, 0) {
Some((source, t)) => CItem::Clip(CClip { len, source, start: t, gain: None, name }),
@@ -506,6 +518,7 @@ impl R<'_> {
.and_then(|m| m.objs(pid::SLOTS).iter().find(|s| s.u32(pid::SLOT_ID) == Some(next_slot)))
.and_then(|s| rate_of(s, pid::EDIT_RATE))
.unwrap_or(rate);
let source_rate = if self.options.source_slot_rate { source_rate } else { rate };
let inner = units_to_ticks(clip.i64(pid::START_TIME).unwrap_or(0), source_rate.0, source_rate.1);
if next == [0; 32] {
return None;
@@ -545,7 +558,11 @@ impl R<'_> {
DataKind::Sound => CKind::Sound,
_ => kind,
};
let path = d.objs(pid::LOCATOR).iter().find_map(|l| l.string(pid::URL_STRING)).map(|u| locator_path(&u));
let path = d
.objs(pid::LOCATOR)
.iter()
.find_map(|l| l.string(pid::URL_STRING))
.map(|u| if self.options.decode_relative_locator { locator_path(&u) } else { craft_timeline::path::file_url_to_path(&u) });
let file_rate = rate_of(d, pid::SAMPLE_RATE).unwrap_or(rate);
let length = d
.i64(pid::FILE_LENGTH)
@@ -604,7 +621,8 @@ impl R<'_> {
if let Some(data) = self.essence.get(&id) {
s.embedded = Some(data.to_vec());
// embedded essence may be a whole WAVE / AIFF file
if let Some((pcm, ch, sr, bits)) = crate::pcm::parse_wav(data) {
let pcm = if self.options.permissive_pcm { craft_audio::essence_pcm::parse_wav(data) } else { crate::pcm::parse_wav(data) };
if let Some((pcm, ch, sr, bits)) = pcm {
s.embedded = Some(pcm);
s.file_channels = ch as u32;
s.channels = if s.channel.is_some() { 1 } else { ch as u32 };
+29
View File
@@ -557,3 +557,32 @@ fn relative_media_locator_round_trips_escaped_characters() {
let got = read_clean(&bytes);
assert!(got.sources.iter().any(|s| s.path.as_deref() == Some("Audio Files/100%20 #1.wav")));
}
#[test]
fn compatibility_options_keep_relative_paths_and_parent_rate() {
let mut doc = crate::testdoc::sound_only(None);
doc.compositions[0].sample_rate = 44_100;
doc.sources[0].path = Some("Audio Files/100%20 #1.wav".into());
let mut report = Report::default();
let references = ReferenceOptions { source_slot_rate: false, encode_relative_locator: false };
let bytes = write_with_references(&doc, &WriteOptions { identity: IDENTITY, small_sectors: false }, &references, &mut report).unwrap();
let root = store::read(&bytes).unwrap();
let mut all = Vec::new();
walk(&root, &mut all);
assert!(
all.iter().filter(|o| o.class == cls::NETWORK_LOCATOR).any(|o| o.string(pid::URL_STRING).as_deref() == Some("Audio Files/100%20 #1.wav"))
);
let starts: Vec<i64> = all.iter().filter(|o| o.class == cls::SOURCE_CLIP).filter_map(|o| o.i64(pid::START_TIME)).collect();
assert!(starts.contains(&17_640), "parent rate start: {starts:?}");
let opts = ReadOptions {
source_slot_rate: false,
infer_clip_name: false,
decode_relative_locator: false,
report_flattening: false,
permissive_pcm: true,
};
let back = read_with_options(&bytes, &opts, &mut report).unwrap();
assert!(back.sources.iter().any(|s| s.path.as_deref() == Some("Audio Files/100%20 #1.wav")));
let clip = back.compositions[0].tracks[0].items.iter().find_map(|i| if let CItem::Clip(c) = i { Some(c) } else { None }).unwrap();
assert_eq!(clip.start, FrameRate::FPS_25.tick_of(10));
}
+14 -4
View File
@@ -75,6 +75,7 @@ fn definition(class: Auid, id: Auid, name: &str) -> Obj {
struct W<'a> {
report: &'a mut Report,
inexact: bool,
references: super::ReferenceOptions,
}
struct IdGen {
@@ -350,9 +351,15 @@ fn add_tape_mob(first: &Source, tape_id: [u8; 32], timecode: Option<(i64, FrameR
}
/// Serialise a composition document.
pub(crate) fn write(doc: &Document, version: craft_cfb::Version, identity: &Identity<'_>, report: &mut Report) -> Result<Vec<u8>> {
pub(crate) fn write(
doc: &Document,
version: craft_cfb::Version,
identity: &Identity<'_>,
references: &super::ReferenceOptions,
report: &mut Report,
) -> Result<Vec<u8>> {
let comp = doc.compositions.first().ok_or(Error::Empty)?;
let mut w = W { report, inexact: false };
let mut w = W { report, inexact: false, references: *references };
let mut ids = IdGen::for_document(doc, comp);
let MediaOutput { mut mobs, essence, master_ref } = write_media(doc, &mut w, &mut ids);
let mut used_ops: Vec<Auid> = Vec::new();
@@ -360,7 +367,8 @@ pub(crate) fn write(doc: &Document, version: craft_cfb::Version, identity: &Iden
// ---- nested compositions: a clip refers to one as it does to a master mob, through the
// composition's first track of the clip's kind (a broken-out channel: that channel's track)
let mut refs = Refs { mobs: master_ref, rates: doc.sources.iter().enumerate().map(|(i, s)| (i, source_rate(s))).collect() };
let rates = if references.source_slot_rate { doc.sources.iter().enumerate().map(|(i, s)| (i, source_rate(s))).collect() } else { HashMap::new() };
let mut refs = Refs { mobs: master_ref, rates };
let nested_ids: Vec<[u8; 32]> = doc.nested.iter().map(|_| ids.next()).collect();
for (si, s) in doc.sources.iter().enumerate() {
let Some((comp, id)) = s.nested.and_then(|n| Some((doc.nested.get(n)?, nested_ids.get(n)?))) else { continue };
@@ -795,8 +803,10 @@ fn descriptor(w: &mut W, s: &Source) -> Obj {
if let Some(p) = &s.path {
let url = if craft_timeline::path::is_absolute(p) {
craft_timeline::path::path_to_file_url(p, false)
} else {
} else if w.references.encode_relative_locator {
craft_timeline::path::encode_uri_path(p)
} else {
p.clone()
};
d.set(pid::LOCATOR, Value::StrongVec(vec![Obj::new(cls::NETWORK_LOCATOR).data_prop(pid::URL_STRING, utf16z(&url))]));
}
+4
View File
@@ -29,7 +29,11 @@ mod roundtrip_tests;
#[cfg(test)]
mod testdoc;
/// Diagnostic types used by readers and writers.
pub use craft_timeline::report;
pub use craft_timeline::report::Report;
/// Exact time types used by the composition model.
pub use craft_timeline::time;
/// Error from a reader or writer. Unsupported *features* never error; they go in the [`Report`].
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
+340 -10
View File
@@ -12,6 +12,10 @@ type Uid = [u8; 12];
const MAX_REFERENCE_VISITS: usize = 65_536;
/// A source-clip search stopped at its work limit.
#[derive(Clone, Copy)]
struct SearchLimit;
struct R<'a, 'b> {
c: &'b Container<'a>,
report: &'b mut Report,
@@ -21,15 +25,49 @@ struct R<'a, 'b> {
media: HashMap<Uid, Vec<u8>>,
doc: Document,
sources: HashMap<(Uid, u32, String), usize>,
/// Component visits left for [`R::segment`] over the whole document.
budget: usize,
/// Outcome of the source-clip search under each segment: deterministic, so it is made once.
clip_search: HashMap<u32, std::result::Result<Option<u32>, SearchLimit>>,
/// Visits and queued references left for source-clip searches over the whole document:
/// distinct roots over one shared graph would otherwise each search it in full.
search_budget: usize,
}
/// Components one document may expand into timeline items; a small file can reference the same
/// sequence from many places and would otherwise multiply without bound.
const MAX_COMPONENT_VISITS: usize = 1_000_000;
/// Reported when the shared component budget runs out.
const BUDGET_WARNING: &str = "OMF composition is too large to import completely (the remaining components are dropped)";
/// Most visited components, and most queued references, one source-clip search may spend.
const MAX_CLIP_SEARCH_VISITS: usize = 100_000;
const MAX_CLIP_SEARCH_PENDING: usize = 1_000_000;
/// Work all source-clip searches in one document may spend together.
const MAX_DOCUMENT_SEARCH_WORK: usize = 2_000_000;
pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
read_with_budget(bytes, report, MAX_COMPONENT_VISITS)
}
fn read_with_budget(bytes: &[u8], report: &mut Report, budget: usize) -> Result<Document> {
let perr = |m: String| Error::Parse { format: "OMF", message: m };
let c = Container::open(bytes).map_err(perr)?;
let class_of = |o: u32| c.get(o, "OMFI:OOBJ:ObjClass").and_then(|v| v.get(..4).map(|b| [b[0], b[1], b[2], b[3]]));
let head = c.objects.keys().copied().find(|&o| class_of(o) == Some(*b"HEAD")).ok_or_else(|| perr("no OMF header object".into()))?;
let be = c.get(head, "OMFI:HEAD:ByteOrder").is_none_or(|v| v.first() != Some(&0x49));
let mut r = R { c: &c, report, be, mobs: HashMap::new(), media: HashMap::new(), doc: Document::default(), sources: HashMap::new() };
let mut r = R {
c: &c,
report,
be,
mobs: HashMap::new(),
media: HashMap::new(),
doc: Document::default(),
sources: HashMap::new(),
budget,
clip_search: HashMap::new(),
search_budget: MAX_DOCUMENT_SEARCH_WORK,
};
let mob_list = r.refs(head, "OMFI:HEAD:Mobs");
let mob_list = if mob_list.is_empty() {
c.objects.keys().copied().filter(|&o| matches!(class_of(o).as_ref(), Some(b"CMOB" | b"MMOB" | b"SMOB"))).collect()
@@ -66,8 +104,14 @@ pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
cmobs.iter().copied().filter(|m| r.uid(*m, "OMFI:MOBJ:MobID").is_none_or(|id| !referenced.contains(&id))).collect()
};
for m in top {
// every composition spends from the budget too: a list of up to 65,535 repeated entries must not yield that many
if r.budget == 0 && !r.doc.compositions.is_empty() {
r.report.warn(BUDGET_WARNING);
break;
}
let comp = r.composition(m);
r.doc.compositions.push(comp);
r.budget = r.budget.saturating_sub(1);
}
if r.doc.compositions.is_empty() {
return Err(Error::Empty);
@@ -166,6 +210,13 @@ impl R<'_, '_> {
let slots = self.refs(m, "OMFI:MOBJ:Slots");
let mut na = 0;
for s in slots {
if self.budget == 0 {
self.report.warn(BUDGET_WARNING);
break;
}
// Every slot spends a unit, skipped ones too: a list of up to 65,535 repeated slots
// must not cost that much work in each of 65,535 repeated compositions.
self.budget -= 1;
let Some(seg) = self.r#ref(s, "OMFI:MSLT:Segment") else { continue };
let rate = self.rational(s, "OMFI:MSLT:EditRate").filter(|r| r.0 > 0 && r.1 > 0).unwrap_or((25, 1));
let kind = self.data_kind(seg);
@@ -213,13 +264,32 @@ impl R<'_, '_> {
fn segment(&mut self, seg: u32, rate: (i64, i64), depth: usize) -> Vec<CItem> {
let len = units_to_ticks(self.len(seg), rate.0, rate.1);
// Depth alone does not bound a graph that references the same component many times at
// every level, so each visit also spends from a per-document budget.
// A visit stopped by depth spends from it too.
if self.budget == 0 {
self.report.warn(BUDGET_WARNING);
return vec![CItem::Filler(len)];
}
self.budget -= 1;
if depth > 32 {
return vec![CItem::Filler(len)];
}
match &self.class(seg) {
b"SEQU" => {
let mut out = Vec::new();
let mut expanded = 0i64;
for c in self.refs(seg, "OMFI:SEQU:Components") {
if self.budget == 0 {
// the rest of the sequence keeps its duration as one gap, not a gap per reference
self.report.warn(BUDGET_WARNING);
let rest = self.len(seg).saturating_sub(expanded);
if rest > 0 {
out.push(CItem::Filler(units_to_ticks(rest, rate.0, rate.1)));
}
break;
}
expanded = expanded.saturating_add(self.len(c));
out.extend(self.segment(c, rate, depth + 1));
}
out
@@ -306,12 +376,66 @@ impl R<'_, '_> {
.find(|&s| self.r#ref(s, "OMFI:MSLT:TrackDesc").and_then(|t| self.u32(t, "OMFI:TRKD:TrackID")) == Some(track))
}
fn first_clip(&self, seg: u32) -> Option<u32> {
match &self.class(seg) {
b"SCLP" => Some(seg),
b"SEQU" => self.refs(seg, "OMFI:SEQU:Components").into_iter().find_map(|c| self.first_clip(c)),
_ => None,
/// First source clip under `seg`, depth-first; a sequence referencing itself (directly or
/// through a cycle) ends the search rather than the stack. `Err` when the search spent its
/// budget of visited components or queued references before finishing (the answer is unknown).
fn first_clip(&mut self, seg: u32) -> std::result::Result<Option<u32>, SearchLimit> {
if let Some(found) = self.clip_search.get(&seg) {
return *found;
}
let mut spent = 0;
let found = if self.search_budget == 0 {
Err(SearchLimit)
} else {
let visits = MAX_CLIP_SEARCH_VISITS.min(self.search_budget);
let pending = MAX_CLIP_SEARCH_PENDING.min(self.search_budget);
self.first_clip_within(seg, visits, pending, &mut spent)
};
self.search_budget = self.search_budget.saturating_sub(spent);
self.clip_search.insert(seg, found);
found
}
/// `spent` receives the visits plus queued references the search used.
fn first_clip_within(&self, seg: u32, max_visits: usize, max_pending: usize, spent: &mut usize) -> std::result::Result<Option<u32>, SearchLimit> {
let mut visited = HashSet::new();
let mut todo = vec![seg];
let mut queued = 1usize;
let found = Self::search_from(self, &mut visited, &mut todo, &mut queued, max_visits, max_pending);
*spent = visited.len().saturating_add(queued);
found
}
fn search_from(
&self,
visited: &mut HashSet<u32>,
todo: &mut Vec<u32>,
queued: &mut usize,
max_visits: usize,
max_pending: usize,
) -> std::result::Result<Option<u32>, SearchLimit> {
while let Some(seg) = todo.pop() {
if !visited.insert(seg) {
continue;
}
if visited.len() > max_visits {
return Err(SearchLimit);
}
match &self.class(seg) {
b"SCLP" => return Ok(Some(seg)),
b"SEQU" => {
let comps = self.refs(seg, "OMFI:SEQU:Components");
// sequences may share one component list, so queued references grow with expansions, not visits
*queued = queued.saturating_add(comps.len());
if *queued > max_pending {
return Err(SearchLimit);
}
todo.extend(comps.into_iter().rev());
}
_ => {}
}
}
Ok(None)
}
fn resolve(&mut self, id: Uid, track: u32, start: Tick, master: Option<String>, depth: usize) -> Option<(usize, Tick)> {
@@ -328,7 +452,13 @@ impl R<'_, '_> {
match &self.class(mob) {
b"MMOB" => {
let key = master.unwrap_or_else(|| id.iter().map(|b| format!("{b:02x}")).collect());
let clip = self.first_clip(seg)?;
let clip = match self.first_clip(seg) {
Ok(c) => c?,
Err(SearchLimit) => {
self.report.warn("a master mob is too large to search for its source clip (imported as a gap)");
return None;
}
};
let inner = units_to_ticks(self.i32(clip, "OMFI:SCLP:StartTime").unwrap_or(0), rate.0, rate.1);
let next = self.uid(clip, "OMFI:SCLP:SourceID")?;
let next_track = self.u32(clip, "OMFI:SCLP:SourceTrackID")?;
@@ -353,7 +483,9 @@ impl R<'_, '_> {
let file_rate = sr.filter(|r| r.0 > 0 && r.1 > 0).unwrap_or(rate);
let length = units_to_ticks(self.i32(desc, "OMFI:MDFL:Length").unwrap_or(0), file_rate.0, file_rate.1);
let summary = self.c.get(desc, "OMFI:WAVD:Summary").or_else(|| self.c.get(desc, "OMFI:AIFD:Summary")).unwrap_or_default();
let fmt = crate::pcm::parse_wav(&summary).or_else(|| crate::pcm::parse_aiff(&summary)).map(|(_, ch, sr, bits)| (ch as u32, sr, bits));
let fmt = craft_audio::essence_pcm::parse_wav(&summary)
.or_else(|| craft_audio::essence_pcm::parse_aiff(&summary))
.map(|(_, ch, sr, bits)| (ch as u32, sr, bits));
let mut path = None;
for l in self.refs(desc, "OMFI:MDES:Locator") {
let p = match &self.class(l) {
@@ -373,7 +505,8 @@ impl R<'_, '_> {
let (file_channels, sample_rate, bits) = fmt.unwrap_or((1, file_rate.0 as u32, 16));
let channel = (file_channels > 1 && sound_slots > 1).then(|| track.saturating_sub(1).min(file_channels - 1));
// timecode of the physical source
let tc = self.first_clip(seg).and_then(|clip| {
let first = self.first_clip(seg).ok().flatten();
let tc = first.and_then(|clip| {
let tid = self.uid(clip, "OMFI:SCLP:SourceID")?;
let tape = *self.mobs.get(&tid)?;
let (start_tc, fps, trate) = self.refs(tape, "OMFI:MOBJ:Slots").into_iter().find_map(|s| {
@@ -415,7 +548,7 @@ impl R<'_, '_> {
nested: None,
};
if let Some(d) = self.media.get(&id) {
if let Some((pcm, ch, sr, b)) = crate::pcm::parse_wav(d).or_else(|| crate::pcm::parse_aiff(d)) {
if let Some((pcm, ch, sr, b)) = craft_audio::essence_pcm::parse_wav(d).or_else(|| craft_audio::essence_pcm::parse_aiff(d)) {
s.embedded = Some(pcm);
s.file_channels = ch as u32;
s.channels = if s.channel.is_some() { 1 } else { ch as u32 };
@@ -437,3 +570,200 @@ impl R<'_, '_> {
}
}
}
#[cfg(test)]
mod tests {
use super::super::bento::Writer;
use super::*;
fn obj(w: &mut Writer, class: &[u8; 4]) -> u32 {
let id = w.new_object();
w.set(id, "OMFI:OOBJ:ObjClass", "omfi:ClassID", class);
id
}
fn list(w: &mut Writer, o: u32, p: &str, targets: &[u32]) {
let mut v = (targets.len() as u16).to_be_bytes().to_vec();
for t in targets {
v.extend_from_slice(&t.to_be_bytes());
}
w.set(o, p, "omfi:ObjRefArray", &v);
}
fn search(max_visits: usize, max_pending: usize, build: impl FnOnce(&mut Writer) -> u32) -> std::result::Result<Option<u32>, SearchLimit> {
let mut w = Writer::new();
let root = build(&mut w);
let bytes = w.finish();
let c = Container::open(&bytes).unwrap();
let mut report = Report::default();
let r = R {
c: &c,
report: &mut report,
be: true,
mobs: HashMap::new(),
media: HashMap::new(),
doc: Document::default(),
sources: HashMap::new(),
budget: 0,
clip_search: HashMap::new(),
search_budget: MAX_DOCUMENT_SEARCH_WORK,
};
r.first_clip_within(root, max_visits, max_pending, &mut 0)
}
#[test]
fn clip_search_reports_its_limits() {
// found within the limits
assert!(matches!(
search(10, 10, |w| {
let (s, c) = (obj(w, b"SEQU"), obj(w, b"SCLP"));
list(w, s, "OMFI:SEQU:Components", &[c]);
s
}),
Ok(Some(_))
));
// too many distinct components before any clip
assert!(
search(5, 1000, |w| {
let (s, f) = (obj(w, b"SEQU"), (0..20).map(|_| obj(w, b"FILL")).collect::<Vec<_>>());
list(w, s, "OMFI:SEQU:Components", &f);
s
})
.is_err()
);
// few visits, but sequences that each queue a long list of references
assert!(
search(1000, 500, |w| {
let f = obj(w, b"FILL");
let inner: Vec<u32> = (0..20).map(|_| obj(w, b"SEQU")).collect();
for &i in &inner {
list(w, i, "OMFI:SEQU:Components", &[f; 100]);
}
let s = obj(w, b"SEQU");
list(w, s, "OMFI:SEQU:Components", &inner);
s
})
.is_err()
);
// a search that ends without a clip is not a limit
assert!(matches!(search(10, 10, |w| obj(w, b"FILL")), Ok(None)));
}
#[test]
fn skipped_slots_spend_the_budget() {
let mut w = Writer::new();
let head = obj(&mut w, b"HEAD");
let cm = obj(&mut w, b"CMOB");
let slot = obj(&mut w, b"MSLT");
list(&mut w, cm, "OMFI:MOBJ:Slots", &[slot; 200]);
list(&mut w, head, "OMFI:HEAD:Mobs", &[cm]);
list(&mut w, head, "OMFI:HEAD:PrimaryMobs", &[cm]);
let bytes = w.finish();
let mut report = Report::default();
read_with_budget(&bytes, &mut report, 50).unwrap();
assert!(report.mentions(BUDGET_WARNING), "slots with no segment cost nothing before");
}
#[test]
fn a_source_clip_search_is_made_once_per_segment() {
let mut w = Writer::new();
let (s, c) = (obj(&mut w, b"SEQU"), obj(&mut w, b"SCLP"));
list(&mut w, s, "OMFI:SEQU:Components", &[c]);
let bytes = w.finish();
let c = Container::open(&bytes).unwrap();
let mut report = Report::default();
let mut r = R {
c: &c,
report: &mut report,
be: true,
mobs: HashMap::new(),
media: HashMap::new(),
doc: Document::default(),
sources: HashMap::new(),
budget: 0,
clip_search: HashMap::new(),
search_budget: MAX_DOCUMENT_SEARCH_WORK,
};
let first = r.first_clip(s).ok();
assert!(matches!(first, Some(Some(_))));
assert_eq!(r.clip_search.len(), 1);
assert_eq!(r.first_clip(s).ok(), first);
assert_eq!(r.clip_search.len(), 1);
}
#[test]
fn repeated_slots_and_compositions_spend_the_budget() {
let mut w = Writer::new();
let head = obj(&mut w, b"HEAD");
let cm = obj(&mut w, b"CMOB");
let dd = obj(&mut w, b"DDEF");
w.set(dd, "OMFI:DDEF:DataKindID", "omfi:UniqueName", b"omfi:data:Sound\0");
let fill = obj(&mut w, b"FILL");
w.set(fill, "OMFI:CPNT:DataKind", "omfi:ObjRef", &dd.to_be_bytes());
let slot = obj(&mut w, b"MSLT");
w.set(slot, "OMFI:MSLT:Segment", "omfi:ObjRef", &fill.to_be_bytes());
list(&mut w, cm, "OMFI:MOBJ:Slots", &[slot; 200]);
list(&mut w, head, "OMFI:HEAD:Mobs", &[cm]);
list(&mut w, head, "OMFI:HEAD:PrimaryMobs", &[cm; 200]);
let bytes = w.finish();
let mut report = Report::default();
let doc = read_with_budget(&bytes, &mut report, 50).unwrap();
let tracks: usize = doc.compositions.iter().map(|c| c.tracks.len()).sum();
assert!(tracks <= 50, "{tracks} tracks from a budget of 50");
assert!(doc.compositions.len() <= 2, "{} compositions", doc.compositions.len());
assert!(report.has_warnings());
}
fn with_reader<T>(bytes: &[u8], budget: usize, search_budget: usize, f: impl FnOnce(&mut R) -> T) -> (T, Report) {
let c = Container::open(bytes).unwrap();
let mut report = Report::default();
let mut r = R {
c: &c,
report: &mut report,
be: true,
mobs: HashMap::new(),
media: HashMap::new(),
doc: Document::default(),
sources: HashMap::new(),
budget,
clip_search: HashMap::new(),
search_budget,
};
let out = f(&mut r);
(out, report)
}
#[test]
fn depth_limited_components_spend_the_budget() {
// a chain 33 sequences deep whose last one repeats a depth-limited sequence many times
let mut w = Writer::new();
let deep = obj(&mut w, b"SEQU");
let chain: Vec<u32> = (0..33).map(|_| obj(&mut w, b"SEQU")).collect();
for pair in chain.windows(2) {
list(&mut w, pair[0], "OMFI:SEQU:Components", &[pair[1]]);
}
list(&mut w, chain[32], "OMFI:SEQU:Components", &[deep; 1000]);
let bytes = w.finish();
let (items, report) = with_reader(&bytes, 100, MAX_DOCUMENT_SEARCH_WORK, |r| r.segment(chain[0], (48_000, 1), 0));
assert!(items.len() <= 100, "{} items from a budget of 100", items.len());
assert!(report.mentions(BUDGET_WARNING));
}
#[test]
fn source_clip_searches_share_a_document_budget() {
// distinct roots over one shared clip-free graph
let mut w = Writer::new();
let fills: Vec<u32> = (0..50).map(|_| obj(&mut w, b"FILL")).collect();
let shared = obj(&mut w, b"SEQU");
list(&mut w, shared, "OMFI:SEQU:Components", &fills);
let roots: Vec<u32> = (0..30).map(|_| obj(&mut w, b"SEQU")).collect();
for &root in &roots {
list(&mut w, root, "OMFI:SEQU:Components", &[shared]);
}
let bytes = w.finish();
let (results, _) = with_reader(&bytes, 0, 500, |r| roots.iter().map(|&root| r.first_clip(root)).collect::<Vec<_>>());
assert!(matches!(results[0], Ok(None)), "the first search fits");
assert!(results.iter().any(|r| r.is_err()), "later searches run out of the shared budget");
let (results, _) = with_reader(&bytes, 0, MAX_DOCUMENT_SEARCH_WORK, |r| roots.iter().map(|&root| r.first_clip(root)).collect::<Vec<_>>());
assert!(results.iter().all(|r| matches!(r, Ok(None))));
}
}
-5
View File
@@ -4,8 +4,3 @@
pub(crate) fn parse_wav(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
craft_audio::wav_pcm(b).map(|p| (p.data, p.channels, p.sample_rate, p.bits))
}
/// The same for an AIFF / AIFF-C (`NONE`, `twos`, `sowt`) file.
pub(crate) fn parse_aiff(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
craft_audio::aiff_pcm(b).map(|p| (p.data, p.channels, p.sample_rate, p.bits))
}
+189
View File
@@ -0,0 +1,189 @@
//! Compatibility WAVE and AIFF wrappers for embedded PCM essence in interchange files.
//! PCM inside the model is interleaved little-endian signed integers (8-bit unsigned for WAVE).
pub use crate::{pcm_aiff_header as aiff_header, pcm_wav_file as wav_file, pcm_wav_header as wav_header};
/// PCM, channels, sample rate and bits of a WAVE file (integer PCM only).
pub fn parse_wav(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
if b.len() < 12 || &b[..4] != b"RIFF" || &b[8..12] != b"WAVE" {
return None;
}
let mut at = 12;
let mut fmt = None;
while at + 8 <= b.len() {
let id = &b[at..at + 4];
let len = u32::from_le_bytes(b[at + 4..at + 8].try_into().ok()?) as usize;
let body = &b[at + 8..(at + 8).saturating_add(len).min(b.len())];
if id == b"fmt " && body.len() >= 16 {
let tag = u16::from_le_bytes([body[0], body[1]]);
let ch = u16::from_le_bytes([body[2], body[3]]);
let sr = u32::from_le_bytes(body[4..8].try_into().ok()?);
let bits = u16::from_le_bytes([body[14], body[15]]);
if tag != 1 && tag != 0xFFFE {
return None;
}
if tag == 0xFFFE {
// The first two bytes of the sub-format GUID carry the real format code.
let sub = body.get(24..26)?;
if u16::from_le_bytes([sub[0], sub[1]]) != 1 {
return None;
}
}
fmt = Some((ch, sr, bits));
} else if id == b"data" {
let (ch, sr, bits) = fmt?;
return Some((body.to_vec(), ch, sr, bits));
}
at = at.checked_add(8usize.checked_add(len)?.checked_add(len & 1)?)?;
}
None
}
/// A complete AIFF file (big-endian PCM) from little-endian PCM.
pub fn aiff_file(pcm: &[u8], channels: u16, sample_rate: u32, bits: u16) -> Vec<u8> {
let bps = bits.div_ceil(8) as usize;
let frames = pcm.len() / (bps * channels.max(1) as usize).max(1);
let mut v = aiff_header(frames as u32, channels, sample_rate, bits);
for s in pcm.chunks_exact(bps.max(1)) {
if bps == 1 {
v.push(s[0] ^ 0x80); // WAVE 8-bit is unsigned, AIFF signed
} else {
v.extend(s.iter().rev());
}
}
if (v.len() - 8) & 1 == 1 {
v.push(0);
}
v
}
/// PCM (converted to little-endian), channels, sample rate and bits of an AIFF / AIFF-C (`NONE`
/// / `twos` / `sowt`) file.
pub fn parse_aiff(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
if b.len() < 12 || &b[..4] != b"FORM" || !(&b[8..12] == b"AIFF" || &b[8..12] == b"AIFC") {
return None;
}
let aifc = &b[8..12] == b"AIFC";
let mut at = 12;
let mut comm = None;
while at + 8 <= b.len() {
let id = &b[at..at + 4];
let len = u32::from_be_bytes(b[at + 4..at + 8].try_into().ok()?) as usize;
let body = &b[at + 8..(at + 8).saturating_add(len).min(b.len())];
if id == b"COMM" && body.len() >= 18 {
let ch = u16::from_be_bytes([body[0], body[1]]);
let bits = u16::from_be_bytes([body[6], body[7]]);
let sr = from_extended(body[8..18].try_into().ok()?).round() as u32;
// AIFF has no compression code (trailing COMM bytes mean nothing); AIFF-C must name one
// of the integer PCM codes, or the samples are not integer PCM at all
let little = if aifc {
match body.get(18..22)? {
b"NONE" | b"twos" => false,
b"sowt" => true,
_ => return None,
}
} else {
false
};
comm = Some((ch, sr, bits, little));
} else if id == b"SSND" && body.len() >= 8 {
let (ch, sr, bits, little) = comm?;
let off = u32::from_be_bytes(body[..4].try_into().ok()?) as usize;
let data = body.get(8usize.checked_add(off)?..)?;
let bps = bits.div_ceil(8) as usize;
let mut pcm = Vec::with_capacity(data.len());
for s in data.chunks_exact(bps.max(1)) {
if bps == 1 {
pcm.push(s[0] ^ 0x80);
} else if little {
pcm.extend_from_slice(s);
} else {
pcm.extend(s.iter().rev());
}
}
return Some((pcm, ch, sr, bits));
}
at = at.checked_add(8usize.checked_add(len)?.checked_add(len & 1)?)?;
}
None
}
fn from_extended(b: [u8; 10]) -> f64 {
let e = (u16::from_be_bytes([b[0], b[1]]) & 0x7FFF) as i32;
let m = u64::from_be_bytes(b[2..10].try_into().unwrap_or([0; 8]));
if e == 0 && m == 0 {
return 0.0;
}
m as f64 / (1u64 << 63) as f64 * 2f64.powi(e - 16383)
}
#[cfg(test)]
mod tests {
/// A mutated EDL whose clip ran for days produced an OMF descriptor for more than 4 GiB of
/// PCM: the RIFF size field overflowed ("attempt to add with overflow").
#[test]
fn huge_wav_header_saturates() {
let h = super::wav_header(u32::MAX, 2, u32::MAX, 24);
assert_eq!(&h[4..8], &u32::MAX.to_le_bytes());
}
use super::*;
#[test]
fn accepts_truncated_essence_summaries() {
let mut wav = wav_file(&[1, 2, 3, 4], 1, 48_000, 16);
wav.truncate(46);
assert_eq!(parse_wav(&wav), Some((vec![1, 2], 1, 48_000, 16)));
let mut aiff = aiff_file(&[1, 2, 3, 4], 1, 48_000, 16);
aiff.truncate(56);
assert_eq!(parse_aiff(&aiff), Some((vec![1, 2], 1, 48_000, 16)));
}
#[test]
fn wav_and_aiff_round_trip() {
let pcm: Vec<u8> = (0..48u8).collect(); // 8 stereo 24-bit frames
let w = wav_file(&pcm, 2, 48_000, 24);
assert_eq!(parse_wav(&w), Some((pcm.clone(), 2, 48_000, 24)));
let a = aiff_file(&pcm, 2, 44_100, 24);
assert_eq!(parse_aiff(&a), Some((pcm.clone(), 2, 44_100, 24)));
let p16: Vec<u8> = (0..32u8).collect();
assert_eq!(parse_aiff(&aiff_file(&p16, 1, 96_000, 16)), Some((p16, 1, 96_000, 16)));
assert!(parse_wav(b"RIFF").is_none());
assert!(parse_aiff(&a[..30]).is_none());
for cut in 0..w.len() {
let _ = parse_wav(&w[..cut]);
let _ = parse_aiff(&a[..cut.min(a.len())]);
}
}
#[test]
fn aifc_needs_an_integer_pcm_compression_code() {
let file = |kind: &[u8; 4], code: &[u8]| {
let mut comm = vec![0, 1, 0, 0, 0, 1, 0, 16];
comm.extend_from_slice(&[0x40, 0x0E, 0xBB, 0x80, 0, 0, 0, 0, 0, 0]);
comm.extend_from_slice(code);
let mut body = kind.to_vec();
body.extend_from_slice(b"COMM");
body.extend_from_slice(&(comm.len() as u32).to_be_bytes());
body.extend_from_slice(&comm);
if comm.len() & 1 == 1 {
body.push(0);
}
body.extend_from_slice(b"SSND");
body.extend_from_slice(&10u32.to_be_bytes());
body.extend_from_slice(&[0; 8]);
body.extend_from_slice(&[0x12, 0x34]);
let mut v = b"FORM".to_vec();
v.extend_from_slice(&(body.len() as u32).to_be_bytes());
v.extend_from_slice(&body);
v
};
assert_eq!(parse_aiff(&file(b"AIFC", b"NONE")).map(|p| p.0), Some(vec![0x34, 0x12]));
assert_eq!(parse_aiff(&file(b"AIFC", b"twos")).map(|p| p.0), Some(vec![0x34, 0x12]));
assert_eq!(parse_aiff(&file(b"AIFC", b"sowt")).map(|p| p.0), Some(vec![0x12, 0x34]));
assert!(parse_aiff(&file(b"AIFC", b"fl32")).is_none(), "float samples are not integer PCM");
assert!(parse_aiff(&file(b"AIFC", b"")).is_none(), "AIFF-C without a compression code");
// plain AIFF ignores trailing COMM bytes
assert_eq!(parse_aiff(&file(b"AIFF", b"sowt")).map(|p| p.0), Some(vec![0x34, 0x12]));
}
}
+8
View File
@@ -76,6 +76,11 @@ pub mod wav_markers;
))]
mod write;
/// PCM WAVE and AIFF wrappers for interchange essence.
#[cfg(feature = "pcm-encode")]
pub mod essence_pcm;
/// Streaming WAVE and AIFF headers and sample packing.
pub mod pcm_export;
/// Interleaved PCM packet decoding.
pub mod pcm_interleaved;
/// Range-backed PCM WAVE access and a 16-bit PCM writer.
@@ -767,3 +772,6 @@ mod tests {
assert!(matches!(e, AudioError::Encode(_)), "{e}");
}
}
/// DVD and Blu-ray LPCM packet decoding.
pub mod lpcm;
+118
View File
@@ -0,0 +1,118 @@
//! DVD-Video and Blu-ray LPCM packet grammar and sample decoding.
/// Blu-ray / AVCHD LPCM packet header (4 bytes before the samples).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct BlurayLpcm {
/// Sample rate in hertz.
pub sample_rate: u32,
/// Number of decoded channels.
pub channels: usize,
/// Significant bits per sample.
pub bits: u32,
}
/// Channels of each channel_assignment code (mono, stereo, 3/0, 2/1, 3/1, 2/2, 3/2, 3/2+LFE,
/// 3/4, 3/4+LFE).
const BD_LPCM_CHANNELS: [usize; 12] = [0, 1, 0, 2, 3, 3, 4, 4, 5, 6, 7, 8];
/// Parse and validate the four-byte Blu-ray LPCM header.
pub fn bluray_lpcm_header(p: &[u8]) -> Option<BlurayLpcm> {
if p.len() < 4 {
return None;
}
let channels = *BD_LPCM_CHANNELS.get((p[2] >> 4) as usize).filter(|&&c| c > 0)?;
let sample_rate = match p[2] & 15 {
1 => 48_000,
4 => 96_000,
5 => 192_000,
_ => return None,
};
let bits = match p[3] >> 6 {
1 => 16,
2 => 20,
3 => 24,
_ => return None,
};
Some(BlurayLpcm { sample_rate, channels, bits })
}
/// Decode one Blu-ray LPCM packet (big-endian; channels padded to an even count; 20-bit
/// samples are stored in 24 bits).
pub fn decode_bluray_lpcm(p: &[u8]) -> Option<Vec<Vec<f32>>> {
let h = bluray_lpcm_header(p)?;
let coded = (h.channels + 1) & !1;
let bps = if h.bits == 16 { 2 } else { 3 };
let data = &p[4..];
let n = data.len() / (coded * bps);
let mut out = vec![Vec::with_capacity(n); h.channels];
for k in 0..n {
for (c, o) in out.iter_mut().enumerate() {
let at = (k * coded + c) * bps;
let v = if bps == 2 {
i16::from_be_bytes([data[at], data[at + 1]]) as f32 / 32768.0
} else {
(i32::from_be_bytes([data[at], data[at + 1], data[at + 2], 0]) >> 8) as f32 / 8_388_608.0
};
o.push(v);
}
}
Some(out)
}
/// Bytes per two sample frames of DVD LPCM. The caller must bound `channels` to the DVD channel range.
pub fn dvd_lpcm_group_bytes(channels: usize, bits: u32) -> usize {
match bits {
16 => 4 * channels,
20 => 5 * channels,
_ => 6 * channels,
}
}
/// Decode DVD-Video LPCM (big-endian). 16-bit samples are interleaved; 20- and 24-bit samples
/// come in groups of two frames: the upper 16 bits of all the group's samples, then their low
/// 4 / 8 bits. The caller must bound `channels` to the DVD channel range.
pub fn decode_dvd_lpcm(data: &[u8], channels: usize, bits: u32) -> Vec<Vec<f32>> {
let ch = channels.max(1);
let group = dvd_lpcm_group_bytes(ch, bits);
let groups = data.len() / group;
let mut out = vec![Vec::with_capacity(groups * 2); ch];
for g in 0..groups {
let b = &data[g * group..(g + 1) * group];
for f in 0..2 {
for (c, o) in out.iter_mut().enumerate() {
let k = f * ch + c;
let hi = i16::from_be_bytes([b[2 * k], b[2 * k + 1]]) as i32;
let v = match bits {
16 => hi as f32 / 32768.0,
20 => {
let nib = b[4 * ch + k / 2];
let lo = if k.is_multiple_of(2) { nib >> 4 } else { nib & 15 } as i32;
((hi << 4) | lo) as f32 / 524_288.0
}
_ => ((hi << 8) | b[4 * ch + k] as i32) as f32 / 8_388_608.0,
};
o.push(v);
}
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn bluray_header_and_even_channel_padding() {
let packet = [0, 0, 0x31, 0x40, 0x40, 0, 0xc0, 0];
assert_eq!(bluray_lpcm_header(&packet), Some(BlurayLpcm { sample_rate: 48_000, channels: 2, bits: 16 }));
assert_eq!(decode_bluray_lpcm(&packet), Some(vec![vec![0.5], vec![-0.5]]));
assert_eq!(decode_bluray_lpcm(&packet[..3]), None);
}
#[test]
fn dvd_lpcm_16_20_and_24_bit_groups() {
assert_eq!(decode_dvd_lpcm(&[0x40, 0, 0xc0, 0], 1, 16), vec![vec![0.5, -0.5]]);
assert_eq!(decode_dvd_lpcm(&[0x40, 0, 0xc0, 0, 0], 1, 20), vec![vec![0.5, -0.5]]);
assert_eq!(decode_dvd_lpcm(&[0x40, 0, 0xc0, 0, 0, 0], 1, 24), vec![vec![0.5, -0.5]]);
assert_eq!(decode_dvd_lpcm(&[1, 2, 3], 1, 16), vec![Vec::<f32>::new()]);
}
}
+287
View File
@@ -0,0 +1,287 @@
//! Audio-only files: WAV (RIFF, little-endian PCM; RF64 past 4 GiB) and AIFF (big-endian PCM),
//! 16, 24 or 32-bit integer or 32-bit float ([`SampleFormat`]), for streaming export.
//! Float WAV is `WAVE_FORMAT_IEEE_FLOAT` with a `fact` chunk; float AIFF is AIFF-C (`fl32`).
//!
//! The header is written first from the known length (`header`), then the samples in chunks
//! (`encode`) and the pad byte (`pad`), so an export streams to disk instead of building the
//! file in memory.
/// PCM representation in an exported file.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SampleFormat {
/// 16-bit signed integer.
Int16,
/// 24-bit signed integer.
Int24,
/// 32-bit signed integer.
Int32,
/// 32-bit IEEE float.
Float32,
}
impl SampleFormat {
/// Bits per sample.
pub fn bits(self) -> u16 {
match self {
Self::Int16 => 16,
Self::Int24 => 24,
Self::Int32 | Self::Float32 => 32,
}
}
/// Bytes per sample.
pub fn bytes(self) -> usize {
usize::from(self.bits() / 8)
}
/// Whether this is a floating point format.
pub fn is_float(self) -> bool {
self == Self::Float32
}
/// Clip and quantize one sample with symmetric integer full scale; NaN becomes silence.
pub fn quantise(self, s: f32) -> i32 {
let s = s.clamp(-1.0, 1.0);
match self {
Self::Int16 => (s * 32_767.0).round() as i32,
Self::Int24 => (s * 8_388_607.0).round() as i32,
Self::Int32 => (f64::from(s) * 2_147_483_647.0).round() as i32,
Self::Float32 => 0,
}
}
/// Sanitize a floating sample: NaN becomes silence, infinity becomes the largest finite value.
pub fn float(s: f32) -> f32 {
if s.is_nan() { 0.0 } else { s.clamp(f32::MIN, f32::MAX) }
}
}
/// The container of an audio-only export.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum PcmContainer {
/// RIFF/WAVE or RF64.
Wav,
/// AIFF or AIFF-C.
Aiff,
}
/// Bytes of sample data for `frames` frames, or None when that does not fit 64 bits.
fn data_len(frames: u64, channels: u16, format: SampleFormat) -> Option<u64> {
frames.checked_mul(u64::from(channels))?.checked_mul(format.bytes() as u64)
}
/// The pad byte after an odd-sized data chunk (RIFF and AIFF chunks are word-aligned).
pub fn pad(frames: u64, channels: u16, format: SampleFormat) -> &'static [u8] {
if data_len(frames, channels, format).is_some_and(|n| n & 1 == 1) { &[0] } else { &[] }
}
/// `KSDATAFORMAT_SUBTYPE_PCM` / `_IEEE_FLOAT` after their first two bytes (the format tag).
const SUBTYPE_TAIL: [u8; 14] = [0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71];
/// AIFF-C format version 1 (`FVER`).
const AIFC_VERSION_1: u32 = 0xA280_5140;
/// The AIFF-C compression name of `fl32`, as a Pascal string of even length.
const FL32_NAME: &[u8] = b"\x1532-bit floating point";
/// Everything before the samples of a file of `frames` frames. A WAV whose RIFF size would not
/// fit 32 bits (4 GiB: 24-bit 5.1 at 96 kHz after about 41 minutes) is written as RF64 (EBU Tech
/// 3306), with the 64-bit sizes in a `ds64` chunk. AIFF has no 64-bit form, so a longer AIFF is
/// an error naming WAV instead.
pub fn header(container: PcmContainer, channels: u16, sample_rate: u32, format: SampleFormat, frames: u64) -> Result<Vec<u8>, String> {
let too_long = || "the audio is too long to write".to_string();
if channels == 0 {
return Err("audio needs at least one channel".into());
}
if sample_rate == 0 {
return Err("audio needs a sample rate".into());
}
let bits = format.bits();
let bps = bits / 8;
let float = format.is_float();
let data = data_len(frames, channels, format).ok_or_else(too_long)?;
let block_align = channels.checked_mul(bps).ok_or("too many audio channels")?;
match container {
PcmContainer::Wav => {
let byte_rate =
u32::try_from(u64::from(sample_rate) * u64::from(block_align)).map_err(|_| "the audio sample rate is too high".to_string())?;
let ext = channels > 2;
// plain PCM has no cbSize; IEEE float has an empty one
let fmt_len: u32 = if ext {
40
} else if float {
18
} else {
16
};
// every format but integer PCM has a `fact` chunk (the frame count; RF64: in `ds64`)
let fact_len: u64 = if float { 12 } else { 0 };
let padded = data.checked_add(data & 1).ok_or_else(too_long)?;
// "WAVE" + fmt chunk + fact chunk + data chunk header + samples
let riff = (4 + 8 + u64::from(fmt_len) + fact_len + 8).checked_add(padded).ok_or_else(too_long)?;
let rf64 = riff > u64::from(u32::MAX);
let mut v = Vec::with_capacity(100);
if rf64 {
const DS64: u64 = 8 + 28;
v.extend_from_slice(b"RF64");
v.extend_from_slice(&u32::MAX.to_le_bytes());
v.extend_from_slice(b"WAVEds64");
v.extend_from_slice(&28u32.to_le_bytes());
v.extend_from_slice(&riff.checked_add(DS64).ok_or_else(too_long)?.to_le_bytes());
v.extend_from_slice(&data.to_le_bytes());
v.extend_from_slice(&frames.to_le_bytes());
v.extend_from_slice(&0u32.to_le_bytes()); // no table entries
} else {
v.extend_from_slice(b"RIFF");
v.extend_from_slice(&(riff as u32).to_le_bytes());
v.extend_from_slice(b"WAVE");
}
let tag: u16 = if float { 3 } else { 1 };
v.extend_from_slice(b"fmt ");
v.extend_from_slice(&fmt_len.to_le_bytes());
v.extend_from_slice(&(if ext { 0xFFFE } else { tag }).to_le_bytes());
v.extend_from_slice(&channels.to_le_bytes());
v.extend_from_slice(&sample_rate.to_le_bytes());
v.extend_from_slice(&byte_rate.to_le_bytes());
v.extend_from_slice(&block_align.to_le_bytes());
v.extend_from_slice(&bits.to_le_bytes());
if ext {
// cbSize, valid bits, channel mask, KSDATAFORMAT_SUBTYPE_PCM / _IEEE_FLOAT
v.extend_from_slice(&22u16.to_le_bytes());
v.extend_from_slice(&bits.to_le_bytes());
let mask: u32 = if channels == 6 { 0x3F } else { 0 };
v.extend_from_slice(&mask.to_le_bytes());
v.extend_from_slice(&tag.to_le_bytes());
v.extend_from_slice(&SUBTYPE_TAIL);
} else if float {
v.extend_from_slice(&0u16.to_le_bytes());
}
if float {
v.extend_from_slice(b"fact");
v.extend_from_slice(&4u32.to_le_bytes());
v.extend_from_slice(&(if rf64 { u32::MAX } else { u32::try_from(frames).unwrap_or(u32::MAX) }).to_le_bytes());
}
v.extend_from_slice(b"data");
v.extend_from_slice(&(if rf64 { u32::MAX } else { data as u32 }).to_le_bytes());
Ok(v)
}
PcmContainer::Aiff => {
let aiff_too_long = || "AIFF holds at most 4 GB of audio: export WAV, which has no such limit".to_string();
let frames32 = u32::try_from(frames).map_err(|_| aiff_too_long())?;
// SSND: offset + block size + samples
let ssnd = u32::try_from(data.checked_add(8).ok_or_else(aiff_too_long)?).map_err(|_| aiff_too_long())?;
// floats need AIFF-C: a version chunk, and the compression type and name in COMM
let comm: u32 = if float { 18 + 4 + FL32_NAME.len() as u32 } else { 18 };
let fver: u32 = if float { 8 + 4 } else { 0 };
let form = (4u32 + fver + (8 + comm))
.checked_add(8)
.and_then(|n| n.checked_add(ssnd))
.and_then(|n| n.checked_add(ssnd & 1))
.ok_or_else(aiff_too_long)?;
let mut v = Vec::with_capacity(90);
v.extend_from_slice(b"FORM");
v.extend_from_slice(&form.to_be_bytes());
if float {
v.extend_from_slice(b"AIFCFVER");
v.extend_from_slice(&4u32.to_be_bytes());
v.extend_from_slice(&AIFC_VERSION_1.to_be_bytes());
} else {
v.extend_from_slice(b"AIFF");
}
v.extend_from_slice(b"COMM");
v.extend_from_slice(&comm.to_be_bytes());
v.extend_from_slice(&channels.to_be_bytes());
v.extend_from_slice(&frames32.to_be_bytes());
v.extend_from_slice(&bits.to_be_bytes());
v.extend_from_slice(&extended(sample_rate));
if float {
v.extend_from_slice(b"fl32");
v.extend_from_slice(FL32_NAME);
}
v.extend_from_slice(b"SSND");
v.extend_from_slice(&ssnd.to_be_bytes());
v.extend_from_slice(&0u32.to_be_bytes()); // offset
v.extend_from_slice(&0u32.to_be_bytes()); // block size
Ok(v)
}
}
}
/// Append interleaved samples in `format` (little-endian for WAV, big-endian for AIFF).
pub fn encode(container: PcmContainer, interleaved: &[f32], format: SampleFormat, out: &mut Vec<u8>) {
let bps = format.bytes();
out.reserve(interleaved.len().saturating_mul(bps));
for &s in interleaved {
let q = if format.is_float() { SampleFormat::float(s).to_bits() as i32 } else { format.quantise(s) };
match container {
PcmContainer::Wav => out.extend_from_slice(q.to_le_bytes().get(..bps).unwrap_or_default()),
PcmContainer::Aiff => out.extend_from_slice(q.to_be_bytes().get(4 - bps..).unwrap_or_default()),
}
}
}
/// A whole file in memory (short audio; exports stream with [`header`] and [`encode`]).
fn write_whole(container: PcmContainer, interleaved: &[f32], channels: u16, sample_rate: u32, format: SampleFormat) -> Result<Vec<u8>, String> {
let per_frame = usize::from(channels.max(1));
if !interleaved.len().is_multiple_of(per_frame) {
return Err("the samples do not make whole frames".into());
}
let frames = (interleaved.len() / per_frame) as u64;
let mut v = header(container, channels, sample_rate, format, frames)?;
encode(container, interleaved, format, &mut v);
v.extend_from_slice(pad(frames, channels, format));
Ok(v)
}
/// A RIFF/WAVE file of interleaved samples. More than two channels are written as
/// `WAVE_FORMAT_EXTENSIBLE` with the speaker mask of the layout (6 channels: L, R, C, LFE, Ls, Rs
/// = `0x3F`).
pub fn write_wav(interleaved: &[f32], channels: u16, sample_rate: u32, format: SampleFormat) -> Result<Vec<u8>, String> {
write_whole(PcmContainer::Wav, interleaved, channels, sample_rate, format)
}
/// IEEE 754 80-bit extended encoding of a sample rate (AIFF `COMM`).
pub fn extended(rate: u32) -> [u8; 10] {
let mut out = [0u8; 10];
if rate == 0 {
return out;
}
let shift = rate.leading_zeros();
let mant = (rate as u64) << (32 + shift);
let exp = 16383 + 31 - shift as u16;
out[..2].copy_from_slice(&exp.to_be_bytes());
out[2..].copy_from_slice(&mant.to_be_bytes());
out
}
/// An AIFF file (`FORM`/`AIFF`, `COMM` + `SSND`; AIFF-C for floats) of interleaved samples.
pub fn write_aiff(interleaved: &[f32], channels: u16, sample_rate: u32, format: SampleFormat) -> Result<Vec<u8>, String> {
write_whole(PcmContainer::Aiff, interleaved, channels, sample_rate, format)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn streaming_formats_preserve_samples_and_bounds() {
let samples = [1.0, -1.0, f32::NAN];
let wav = write_wav(&samples, 1, 48_000, SampleFormat::Int24).unwrap();
assert_eq!(&wav[..4], b"RIFF");
assert_eq!(&wav[44..53], &[255, 255, 127, 1, 0, 128, 0, 0, 0]);
let h = header(PcmContainer::Wav, 6, 96_000, SampleFormat::Int24, 96_000 * 3600).unwrap();
assert_eq!(&h[..4], b"RF64");
assert!(header(PcmContainer::Aiff, 2, 48_000, SampleFormat::Int24, u64::MAX).is_err());
let aiff = write_aiff(&[0.25, -3.0], 1, 44_100, SampleFormat::Float32).unwrap();
assert_eq!(&aiff[8..12], b"AIFC");
assert_eq!(&aiff[aiff.len() - 8..], [0.25f32.to_be_bytes(), (-3.0f32).to_be_bytes()].concat());
}
#[test]
fn zero_channels_rates_and_partial_frames_are_refused() {
for container in [PcmContainer::Wav, PcmContainer::Aiff] {
assert!(header(container, 0, 48_000, SampleFormat::Int16, 1).is_err());
assert!(header(container, 1, 0, SampleFormat::Int16, 1).is_err());
assert!(header(container, 2, 48_000, SampleFormat::Int16, 0).is_ok(), "zero frames stay valid");
}
assert!(write_wav(&[0.5], 0, 48_000, SampleFormat::Int16).is_err());
assert!(write_wav(&[0.1, 0.2, 0.3], 2, 48_000, SampleFormat::Int16).is_err());
assert!(write_aiff(&[0.1, 0.2, 0.3], 2, 48_000, SampleFormat::Int16).is_err());
assert!(write_aiff(&[0.1, 0.2], 2, 48_000, SampleFormat::Int16).is_ok());
}
}
+3
View File
@@ -29,8 +29,10 @@ pub mod lut3d;
pub mod lutfile;
pub mod math;
pub mod named;
mod normalize;
pub mod pipeline;
pub mod profile;
pub mod sony_lut;
pub mod spi1d;
pub mod synth;
pub mod transform;
@@ -44,6 +46,7 @@ pub use gamut::GamutCheck;
pub use gpu_matrix_shaper::GpuMatrixShaper;
pub use lut3d::Lut3d;
pub use named::{NamedColor, NamedColors};
pub use normalize::normalize_for_embedding;
pub use profile::{ColorSpace, Pcs, Profile, ProfileClass};
pub use transform::{SampleKind, Transform, TransformOptions, black_point, cached};
pub use write::profile_id;
+142
View File
@@ -0,0 +1,142 @@
//! Bounded encoded ICC layout normalization, originating in LightCraft.
use std::collections::BTreeMap;
/// Zero the creation date, align tag payloads to four bytes, preserve shared payloads,
/// update the profile length and clear its stale profile ID. No colour tags are interpreted.
///
/// This is a layout operation, not profile validation. Malformed tables or layouts exceeding
/// 4096 tags or 64 MiB output are returned unchanged except for the creation date. The owned
/// input is reused on failure, so rejected input does not require an additional full copy.
/// Consumers needing rejection of invalid profiles must separately use `Profile::parse`.
pub fn normalize_for_embedding(mut bytes: Vec<u8>) -> Vec<u8> {
if let Some(date) = bytes.get_mut(24..36) {
date.fill(0);
}
align(&bytes).unwrap_or(bytes)
}
const MAX_BYTES: usize = 64 * 1024 * 1024;
const MAX_TAGS: usize = 4096;
fn be(bytes: &[u8], at: usize) -> Option<usize> {
let s = bytes.get(at..at.checked_add(4)?)?;
usize::try_from(u32::from_be_bytes([s[0], s[1], s[2], s[3]])).ok()
}
fn align(bytes: &[u8]) -> Option<Vec<u8>> {
if bytes.len() > MAX_BYTES {
return None;
}
let count = be(bytes, 128)?;
if count > MAX_TAGS {
return None;
}
let end = 132usize.checked_add(count.checked_mul(12)?)?;
bytes.get(..end)?;
let mut placements = BTreeMap::new();
let mut offsets = Vec::new();
offsets.try_reserve_exact(count).ok()?;
let mut size = end;
for index in 0..count {
let entry = 132 + index * 12;
let off = be(bytes, entry + 4)?;
let len = be(bytes, entry + 8)?;
if off < end {
return None;
}
bytes.get(off..off.checked_add(len)?)?;
let new_off = if let Some(&placed) = placements.get(&(off, len)) {
placed
} else {
let aligned = size.checked_add(3)? & !3;
size = aligned.checked_add(len)?;
if size > MAX_BYTES {
return None;
}
placements.insert((off, len), aligned);
aligned
};
offsets.push(new_off);
}
size = size.checked_add(3)? & !3;
if size > MAX_BYTES {
return None;
}
let mut out = Vec::new();
out.try_reserve_exact(size).ok()?;
out.resize(size, 0);
out[..end].copy_from_slice(&bytes[..end]);
for ((off, len), target) in placements {
out[target..target + len].copy_from_slice(&bytes[off..off + len]);
}
for (index, off) in offsets.into_iter().enumerate() {
let field = 132 + index * 12 + 4;
out[field..field + 4].copy_from_slice(&u32::try_from(off).ok()?.to_be_bytes());
}
out[..4].copy_from_slice(&u32::try_from(size).ok()?.to_be_bytes());
out[84..100].fill(0);
Some(out)
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
#[path = "normalize_legacy_tests.rs"]
mod legacy;
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use super::*;
#[test]
fn matches_frozen_writer_and_preserves_aliases() {
for count in 0..20usize {
let end = 132 + count * 12;
let mut bytes = vec![17; end + 31];
bytes[128..132].copy_from_slice(&(count as u32).to_be_bytes());
for index in 0..count {
let field = 132 + index * 12 + 4;
let off = end + index % 3;
bytes[field..field + 4].copy_from_slice(&(off as u32).to_be_bytes());
bytes[field + 4..field + 8].copy_from_slice(&7u32.to_be_bytes());
}
let expected = legacy::align_tags(&legacy::fixed_date(bytes.clone()));
let result = normalize_for_embedding(bytes);
assert_eq!(result, expected);
assert_eq!(normalize_for_embedding(result.clone()), result);
}
}
#[test]
fn real_profile_preserves_color_tags_and_legacy_bytes() {
let encoded = crate::Builtin::Srgb.profile().clone().with_encoded_bytes();
let input = encoded.to_bytes().to_vec();
let normalized = normalize_for_embedding(input.clone());
assert_eq!(normalized, legacy::align_tags(&legacy::fixed_date(input)));
assert_eq!(&normalized[24..36], &[0; 12]);
assert_eq!(&normalized[84..100], &[0; 16]);
assert!(crate::Profile::parse(&normalized).is_ok());
for index in 0..be(&normalized, 128).unwrap() {
assert_eq!(be(&normalized, 136 + index * 12).unwrap() % 4, 0);
}
}
#[test]
fn hostile_tables_return_owned_input_without_expansion() {
for len in 0..132 {
let bytes = vec![1; len];
assert_eq!(normalize_for_embedding(bytes.clone()), legacy::fixed_date(bytes));
}
for count in [4097u32, u32::MAX] {
let mut bytes = vec![1; 144];
bytes[128..132].copy_from_slice(&count.to_be_bytes());
assert_eq!(normalize_for_embedding(bytes.clone()), legacy::fixed_date(bytes));
}
let mut bytes = vec![1; 144];
bytes[128..132].copy_from_slice(&1u32.to_be_bytes());
bytes[136..140].copy_from_slice(&u32::MAX.to_be_bytes());
bytes[140..144].copy_from_slice(&u32::MAX.to_be_bytes());
assert_eq!(normalize_for_embedding(bytes.clone()), legacy::fixed_date(bytes));
}
}
+43
View File
@@ -0,0 +1,43 @@
//! Encoded ICC normalization extracted from LightCraft.
pub(super) fn align_tags(b: &[u8]) -> Vec<u8> {
let be = |i: usize| b.get(i..i + 4).map(|s| u32::from_be_bytes([s[0], s[1], s[2], s[3]]) as usize);
let Some(n) = be(128) else { return b.to_vec() };
let table_end = 132 + 12 * n;
let tags: Option<Vec<(usize, usize)>> = (0..n).map(|t| Some((be(132 + 12 * t + 4)?, be(132 + 12 * t + 8)?))).collect();
let Some(tags) = tags.filter(|t| table_end <= b.len() && t.iter().all(|&(o, l)| o >= table_end && o + l <= b.len())) else {
return b.to_vec();
};
let mut out = b[..table_end].to_vec();
let mut placed: Vec<((usize, usize), usize)> = Vec::new();
for (t, &(off, len)) in tags.iter().enumerate() {
let new_off = match placed.iter().find(|(k, _)| *k == (off, len)) {
// tags sharing data keep sharing it
Some(&(_, o)) => o,
None => {
while !out.len().is_multiple_of(4) {
out.push(0);
}
let o = out.len();
out.extend_from_slice(&b[off..off + len]);
placed.push(((off, len), o));
o
}
};
out[132 + 12 * t + 4..132 + 12 * t + 8].copy_from_slice(&(new_off as u32).to_be_bytes());
}
while !out.len().is_multiple_of(4) {
out.push(0);
}
let size = out.len() as u32;
out[0..4].copy_from_slice(&size.to_be_bytes());
// the profile ID (MD5) would no longer match: clear it (= "not computed")
out[84..100].fill(0);
out
}
pub(super) fn fixed_date(mut b: Vec<u8>) -> Vec<u8> {
if let Some(date) = b.get_mut(24..36) {
date.fill(0);
}
b
}
+82
View File
@@ -0,0 +1,82 @@
//! Sony Imageworks SPI3D and SPIMTX text readers, extracted from EffectCraft.
//!
//! These preserve the source reader's format tolerance. SPI3D sample values are not
//! clamped or sanitized; rendering policy belongs to the caller. Duplicate coordinates
//! retain their last value, and the total row count must match the lattice volume.
/// Maximum lattice side accepted by the source reader.
pub const MAX_SIDE: usize = 129;
/// Row-major homogeneous transform with a separate normalized code-value offset.
#[derive(Clone, Debug, PartialEq)]
pub struct Matrix {
pub matrix: [f64; 16],
pub offset: [f64; 4],
}
/// A cubic RGB table, red coordinate varying fastest.
#[derive(Clone, Debug, PartialEq)]
pub struct Lattice {
pub size: usize,
pub values: Vec<[f32; 3]>,
}
/// Read exactly twelve finite SPIMTX numbers, dividing the offset column by 65535.
/// Parsing uses a fixed array, not storage proportional to an oversized input.
pub fn parse_spimtx(text: &str) -> Option<Matrix> {
let mut words = text.split_whitespace();
let mut v = [0.0f64; 12];
for value in &mut v {
*value = words.next()?.parse().ok()?;
if !value.is_finite() {
return None;
}
}
if words.next().is_some() {
return None;
}
let mut matrix = [0.0; 16];
let mut offset = [0.0; 4];
for (r, row) in v.as_chunks::<4>().0.iter().enumerate() {
matrix[r * 4..r * 4 + 3].copy_from_slice(&row[..3]);
offset[r] = row[3] / 65535.0;
}
matrix[15] = 1.0;
Some(Matrix { matrix, offset })
}
/// Read a SPI3D table with a bounded 2..=129 cubic lattice and exactly six fields per row.
/// A `SPILUT` prefix is accepted and the second header line is ignored, as in EffectCraft.
pub fn parse_spi3d(text: &str) -> Option<Lattice> {
let mut lines = text.lines().map(str::trim).filter(|l| !l.is_empty());
if !lines.next()?.starts_with("SPILUT") {
return None;
}
lines.next()?;
let mut sizes = lines.next()?.split_whitespace();
let n: usize = sizes.next()?.parse().ok()?;
let g: usize = sizes.next()?.parse().ok()?;
let b: usize = sizes.next()?.parse().ok()?;
if sizes.next().is_some() || g != n || b != n || !(2..=MAX_SIDE).contains(&n) {
return None;
}
let count = n * n * n;
let mut values = vec![[0.0f32; 3]; count];
let mut seen = 0usize;
for line in lines {
if seen == count {
return None;
}
let mut fields = line.split_whitespace();
let r: usize = fields.next()?.parse().ok()?;
let g: usize = fields.next()?.parse().ok()?;
let b: usize = fields.next()?.parse().ok()?;
let color = [fields.next()?.parse().ok()?, fields.next()?.parse().ok()?, fields.next()?.parse().ok()?];
if fields.next().is_some() || r >= n || g >= n || b >= n {
return None;
}
*values.get_mut(r + g * n + b * n * n)? = color;
seen += 1;
}
(seen == count).then_some(Lattice { size: n, values })
}
+46
View File
@@ -0,0 +1,46 @@
use craft_cms::sony_lut::{MAX_SIDE, parse_spi3d, parse_spimtx};
#[test]
fn spimtx_is_finite_exact_and_scales_offsets() {
let m = parse_spimtx("2 0 0 6553.5\n0 3 0 -65535\n0 0 4 0").unwrap();
assert_eq!(m.matrix, [2.0, 0.0, 0.0, 0.0, 0.0, 3.0, 0.0, 0.0, 0.0, 0.0, 4.0, 0.0, 0.0, 0.0, 0.0, 1.0]);
assert_eq!(m.offset, [0.1, -1.0, 0.0, 0.0]);
for bad in ["", "1 0 0 0", "NaN 0 0 0 0 1 0 0 0 0 1 0", "inf 0 0 0 0 1 0 0 0 0 1 0", "1 0 0 0 0 1 0 0 0 0 1 0 9"] {
assert!(parse_spimtx(bad).is_none());
}
}
fn table() -> String {
let mut text = String::from("SPILUT 1.0\n3 3\n2 2 2\n");
// Reverse row order to prove coordinates, rather than row order, determine storage.
for i in (0..8).rev() {
let (r, g, b) = (i & 1, (i >> 1) & 1, i >> 2);
text.push_str(&format!("{r} {g} {b} {r} {g} {b}\n"));
}
text
}
#[test]
fn spi3d_red_fast_coordinates_and_limits() {
let text = table();
let lut = parse_spi3d(&text).unwrap();
assert_eq!(lut.size, 2);
for (i, row) in lut.values.iter().enumerate() {
assert_eq!(*row, [(i & 1) as f32, ((i >> 1) & 1) as f32, (i >> 2) as f32]);
}
assert!(parse_spi3d(&format!("{text}0 0 0 0 0 0\n")).is_none());
assert!(parse_spi3d("SPILUT 1.0\n3 3\n2 2 2\n0 0 0 0 0 0\n").is_none());
assert!(parse_spi3d(&format!("SPILUT 1.0\n3 3\n{0} {0} {0}\n", MAX_SIDE + 1)).is_none());
assert!(parse_spi3d(&text.replace("2 2 2", "2 2 3")).is_none());
assert!(parse_spi3d(&text.replace("0 0 0 0 0 0", "2 0 0 0 0 0")).is_none());
assert!(parse_spi3d(&text.replace("0 0 0 0 0 0", "0 0 0 0 0 0 extra")).is_none());
}
#[test]
fn inherited_tolerance_is_explicit() {
let text = "SPILUT alternative\nignored\n2 2 2\n".to_owned() + &"0 0 0 -1 2 NaN\n".repeat(8);
let lut = parse_spi3d(&text).unwrap();
assert_eq!(&lut.values[0][..2], &[-1.0, 2.0]);
assert!(lut.values[0][2].is_nan());
assert_eq!(lut.values[1], [0.0; 3]);
}
+3
View File
@@ -34,6 +34,9 @@
//! Input is treated as hostile: every header read is bounds-checked, the picture size is capped
//! by [`Limits`] and the image data must be present in full before anything is allocated.
/// RGBA8-to-10-bit DPX encoding.
pub mod rgba8_dpx10;
#[cfg(test)]
mod tests;
+122
View File
@@ -0,0 +1,122 @@
//! Uncompressed RGB(A) output from top-down RGBA8 pixels.
/// Why the supplied frame cannot be encoded.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Rgba8EncodeError {
/// Empty, overflowing, or mismatched dimensions.
FrameSize,
/// A dimension cannot fit the output format.
TooLarge,
/// The output buffer could not be allocated.
OutOfMemory,
}
/// Reserves the DPX output up front; a failed reservation becomes [`Rgba8EncodeError::OutOfMemory`].
fn output_buffer(size: usize) -> std::result::Result<Vec<u8>, Rgba8EncodeError> {
let mut buf = Vec::new();
match buf.try_reserve_exact(size) {
Ok(()) => Ok(buf),
Err(_) => Err(Rgba8EncodeError::OutOfMemory),
}
}
fn checked_len(rgba: &[u8], w: u32, h: u32) -> std::result::Result<usize, Rgba8EncodeError> {
let n = (w as usize).checked_mul(h as usize).and_then(|n| n.checked_mul(4)).ok_or(Rgba8EncodeError::FrameSize)?;
if w == 0 || h == 0 || rgba.len() != n {
return Err(Rgba8EncodeError::FrameSize);
}
Ok(n)
}
/// Header size of the DPX files we write (file + image + orientation + film + TV headers).
const DPX_HEADER: usize = 2048;
/// A DPX file: 10-bit RGB (8-bit values widened by bit replication, so 0 → 0 and 255 → 1023).
pub fn encode_rgba8_dpx10(rgba: &[u8], w: u32, h: u32, creator: &str) -> std::result::Result<Vec<u8>, Rgba8EncodeError> {
let n = checked_len(rgba, w, h)?;
// one 32-bit word per pixel; `n` is already `w * h * 4`
let size = n.checked_add(DPX_HEADER).ok_or(Rgba8EncodeError::FrameSize)?;
let file_size = u32::try_from(size).map_err(|_| Rgba8EncodeError::TooLarge)?;
// SMPTE ST 268: undefined numeric fields are all ones, undefined text fields are NUL
let mut hd = vec![0xFF_u8; DPX_HEADER];
for (a, b) in [
(36, 660), // file name, creation time, creator, project, copyright
(664, 768), // reserved
(1356, 1408), // reserved
(1432, 1620), // source file name, time, input device, serial number
(1644, 1664), // reserved
(1664, 1712), // film manufacturer, type, offset, prefix, count, format
(1732, 1920), // frame id, slate, reserved
(1972, 2048), // reserved
] {
hd[a..b].fill(0);
}
for e in 0..8 {
let d = 780 + e * 72 + 40;
hd[d..d + 32].fill(0); // element descriptions
}
let put32 = |hd: &mut Vec<u8>, at: usize, v: u32| hd[at..at + 4].copy_from_slice(&v.to_be_bytes());
let put16 = |hd: &mut Vec<u8>, at: usize, v: u16| hd[at..at + 2].copy_from_slice(&v.to_be_bytes());
// file information header
hd[0..4].copy_from_slice(b"SDPX");
put32(&mut hd, 4, DPX_HEADER as u32); // offset to the image data
hd[8..16].copy_from_slice(b"V2.0\0\0\0\0");
put32(&mut hd, 16, file_size);
put32(&mut hd, 20, 1); // ditto key: new frame
put32(&mut hd, 24, 1664); // generic header length
put32(&mut hd, 28, 384); // industry header length
put32(&mut hd, 32, 0); // user data length
let creator = creator.as_bytes();
let n = creator.len().min(100);
hd[160..160 + n].copy_from_slice(&creator[..n]);
// image information header
put16(&mut hd, 768, 0); // orientation: left to right, top to bottom
put16(&mut hd, 770, 1); // one image element
put32(&mut hd, 772, w);
put32(&mut hd, 776, h);
let el = 780;
put32(&mut hd, el, 0); // unsigned
put32(&mut hd, el + 4, 0); // reference low data code
put32(&mut hd, el + 12, 1023); // reference high data code
hd[el + 20] = 50; // descriptor: RGB
hd[el + 21] = 6; // transfer: ITU-R BT.709
hd[el + 22] = 6; // colorimetric: ITU-R BT.709
hd[el + 23] = 10; // bit depth
put16(&mut hd, el + 24, 1); // packing: filled to 32-bit words, method A
put16(&mut hd, el + 26, 0); // no encoding (uncompressed)
put32(&mut hd, el + 28, DPX_HEADER as u32); // offset to the element's data
put32(&mut hd, el + 32, 0); // end-of-line padding
put32(&mut hd, el + 36, 0); // end-of-image padding
// image orientation header: no offset, square pixels
put32(&mut hd, 1408, 0);
put32(&mut hd, 1412, 0);
put32(&mut hd, 1424, w);
put32(&mut hd, 1428, h);
put32(&mut hd, 1628, 1);
put32(&mut hd, 1632, 1);
// television header: progressive
hd[1928] = 0;
// The caller already holds the frame; reserve the output fallibly so a large export reports
// an error instead of aborting the process.
let mut out = output_buffer(size)?;
out.extend_from_slice(&hd);
let ten = |v: u8| (u32::from(v) << 2) | (u32::from(v) >> 6);
for &[r, g, b, _] in rgba.as_chunks::<4>().0 {
out.extend_from_slice(&((ten(r) << 22) | (ten(g) << 12) | (ten(b) << 2)).to_be_bytes());
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn packed_rgb_and_creator() {
let bytes = encode_rgba8_dpx10(&[255, 0, 128, 255], 1, 1, "FilmCraft").unwrap();
assert_eq!(&bytes[..4], b"SDPX");
assert_eq!(&bytes[160..169], b"FilmCraft");
assert_eq!(&bytes[2048..], &((1023u32 << 22) | (514u32 << 2)).to_be_bytes());
assert_eq!(encode_rgba8_dpx10(&[], 0, 1, ""), Err(Rgba8EncodeError::FrameSize));
}
#[test]
fn output_reservation_failure_maps_to_out_of_memory() {
assert!(matches!(output_buffer(usize::MAX), Err(Rgba8EncodeError::OutOfMemory)));
assert_eq!(output_buffer(DPX_HEADER).map(|b| b.capacity() >= DPX_HEADER), Ok(true));
}
}
+3
View File
@@ -15,9 +15,12 @@ default = ["system"]
system = ["dep:fontique"]
[dependencies]
harfrust = "0.13"
kurbo = { workspace = true }
skrifa = { version = "0.47", default-features = false, features = ["std"] }
thiserror = { workspace = true }
unicode-bidi = "0.3.18"
unicode-segmentation = "1.13"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
fontique = { version = "0.11.1", default-features = false, features = ["system", "fontconfig-dlopen"], optional = true }
+89
View File
@@ -0,0 +1,89 @@
//! Shared ownership of a font file and a face within it.
use std::sync::Arc;
/// Font file bytes and the index of a face in that file.
///
/// Clones share owned bytes, so collection faces and variable instances do not copy the file.
/// Static bytes let applications use embedded fonts without allocating a copy.
#[derive(Clone, Debug)]
pub struct FaceData {
bytes: Bytes,
index: u32,
}
#[derive(Clone, Debug)]
enum Bytes {
Static(&'static [u8]),
Owned(Arc<Vec<u8>>),
Shared(Arc<[u8]>),
}
impl FaceData {
/// A face in statically embedded font bytes.
pub fn from_static(bytes: &'static [u8], index: u32) -> Self {
Self { bytes: Bytes::Static(bytes), index }
}
/// A face in a shared vector, without copying its allocation.
pub fn from_owned(bytes: Arc<Vec<u8>>, index: u32) -> Self {
Self { bytes: Bytes::Owned(bytes), index }
}
/// A face in bytes shared by a catalog.
pub fn from_shared(bytes: Arc<[u8]>, index: u32) -> Self {
Self { bytes: Bytes::Shared(bytes), index }
}
/// The entire font file.
pub fn bytes(&self) -> &[u8] {
match &self.bytes {
Bytes::Static(bytes) => bytes,
Bytes::Owned(bytes) => bytes.as_slice(),
Bytes::Shared(bytes) => bytes,
}
}
/// The index of this face in the file.
pub fn index(&self) -> u32 {
self.index
}
/// Parse this face for font metadata and outlines.
pub fn skrifa(&self) -> Option<skrifa::FontRef<'_>> {
skrifa::FontRef::from_index(self.bytes(), self.index).ok()
}
/// Parse this face for shaping.
pub fn harfrust(&self) -> Option<harfrust::FontRef<'_>> {
harfrust::FontRef::from_index(self.bytes(), self.index).ok()
}
}
impl From<crate::FontData> for FaceData {
fn from(data: crate::FontData) -> Self {
Self::from_shared(data.bytes, data.index)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sources_keep_bytes_and_face_index_without_copying() {
static DATA: &[u8] = b"not a font";
let owned = Arc::new(DATA.to_vec());
let shared: Arc<[u8]> = Arc::from(DATA);
for data in [FaceData::from_static(DATA, 2), FaceData::from_owned(owned.clone(), 2), FaceData::from_shared(shared.clone(), 2)] {
let clone = data.clone();
assert_eq!(clone.bytes(), DATA);
assert_eq!(clone.bytes().as_ptr(), data.bytes().as_ptr());
assert_eq!(clone.index(), 2);
assert!(clone.skrifa().is_none());
assert!(clone.harfrust().is_none());
}
assert_eq!(Arc::strong_count(&owned), 1);
assert_eq!(Arc::strong_count(&shared), 1);
}
}
+62
View File
@@ -0,0 +1,62 @@
//! Face choice for a grapheme cluster.
/// Characters requiring their own glyph, after numeral or other caller-supplied mapping.
/// Controls, joiners and variation selectors affect shaping but do not need a separate glyph.
pub fn drawable_characters<'a>(cluster: &'a str, map: impl Fn(char) -> char + Copy + 'a) -> impl Iterator<Item = char> + 'a {
cluster
.chars()
.map(map)
.filter(|&c| !(c.is_control() || matches!(c, '\u{200C}' | '\u{200D}' | '\u{2028}' | '\u{FE00}'..='\u{FE0F}' | '\u{E0100}'..='\u{E01EF}')))
}
/// Select one face for a cluster: prefer the primary if it covers every drawable character;
/// otherwise try each character's fallback until a face covers the whole cluster. If none does,
/// use the first character's fallback when the primary lacks it, or keep the primary.
/// The caller controls the font catalog, ordering and style matching.
pub fn cluster_face<F: Clone>(
cluster: &str,
primary: &F,
map: impl Fn(char) -> char + Copy,
primary_covers: impl Fn(char) -> bool,
mut fallback: impl FnMut(char) -> Option<F>,
covers: impl Fn(&F, char) -> bool,
) -> F {
let characters = || drawable_characters(cluster, map);
if characters().all(&primary_covers) {
return primary.clone();
}
for c in characters() {
if let Some(face) = fallback(c)
&& characters().all(|ch| covers(&face, ch))
{
return face;
}
}
if let Some(c) = characters().next()
&& !primary_covers(c)
&& let Some(face) = fallback(c)
{
return face;
}
primary.clone()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn chooses_one_face_for_a_base_and_mark() {
let fallback = |c| match c {
'a' => Some(1),
'\u{301}' => Some(2),
_ => None,
};
let covers = |face: &u8, c| *face == 2 || (*face == 1 && c == 'a');
assert_eq!(cluster_face("a\u{301}\u{200d}", &0, |c| c, |_| false, fallback, covers), 2);
assert_eq!(drawable_characters("a\u{200d}\u{fe0f}", |c| c).collect::<String>(), "a");
}
#[test]
fn uses_first_character_fallback_when_no_face_covers_cluster() {
assert_eq!(cluster_face("ab", &0, |c| c, |_| false, |c| if c == 'a' { Some(1) } else { Some(2) }, |face, c| (*face, c) == (1, 'a')), 1);
}
}
+16
View File
@@ -23,21 +23,37 @@
mod catalog;
mod embedding;
mod env;
mod face_data;
mod fallback;
mod fontconfig;
mod ligature;
mod line_break;
mod outline;
mod plan_cache;
mod registry;
mod scan;
mod segments;
/// Bounded SFNT face metadata discovery over caller-provided random-access readers.
pub mod sfnt;
mod shape;
mod suitcase;
pub mod system;
mod visual_order;
pub use catalog::{Catalog, FontData, FontFileError, Limits, Lister, LoadedFace, Match, Watch};
pub use embedding::{SubsetEmbedding, subset_embedding};
pub use env::{FontEnv, Os, font_dirs, font_store_dir, office_font_dirs};
pub use face_data::FaceData;
pub use fallback::{cluster_face, drawable_characters};
pub use fontconfig::fontconfig_dirs;
pub use ligature::ligature_caret_stops;
pub use line_break::settle_line_break;
pub use outline::{FontOutline, GlyphPen, OutlineError};
pub use plan_cache::ShapePlanCache;
pub use scan::{FaceInfo, LocalizedName, Source, is_font_file, style_italic, style_weight, style_width};
pub use segments::SegmentBoundary;
pub use shape::{RunGlyph, RunInput, ShapeRunError, shape_run};
pub use visual_order::{VisualGlyph, visual_order};
#[cfg(test)]
mod tests;
+33
View File
@@ -0,0 +1,33 @@
//! Map ligature caret fractions to grapheme boundaries.
use unicode_segmentation::UnicodeSegmentation;
/// Return logical source-byte caret stops inside a shaped cluster. `font_carets` receives the
/// number of stops and may return their normalized positions along the glyph's logical width.
/// Missing or incorrectly sized font data falls back to equal divisions of the width.
pub fn ligature_caret_stops(text: &str, byte: usize, len: usize, font_carets: impl FnOnce(usize) -> Option<Vec<f64>>) -> Vec<(usize, f64)> {
let Some(s) = byte.checked_add(len).and_then(|end| text.get(byte..end)).filter(|_| len > 1) else { return Vec::new() };
let starts: Vec<usize> = s.grapheme_indices(true).skip(1).map(|(i, _)| byte + i).collect();
if starts.is_empty() {
return Vec::new();
}
let parts = starts.len() + 1;
let mut at =
font_carets(starts.len()).filter(|c| c.len() == starts.len()).unwrap_or_else(|| (1..parts).map(|j| j as f64 / parts as f64).collect());
at.sort_by(f64::total_cmp);
starts.into_iter().zip(at).collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn maps_graphemes_and_sorts_font_positions() {
assert_eq!(ligature_caret_stops("ábc", 0, 5, |_| Some(vec![0.8, 0.3])), vec![(3, 0.3), (4, 0.8)]);
}
#[test]
fn falls_back_and_rejects_invalid_ranges() {
assert_eq!(ligature_caret_stops("abc", 0, 3, |_| Some(vec![0.5])), vec![(1, 1.0 / 3.0), (2, 2.0 / 3.0)]);
assert!(ligature_caret_stops("abc", usize::MAX, 2, |_| None).is_empty());
assert!(ligature_caret_stops("é", 1, 1, |_| None).is_empty());
}
}
+49
View File
@@ -0,0 +1,49 @@
//! Choose a line break after measuring candidate lines as drawn.
/// Choose the latest candidate that fits when a tab changes the line's drawn width.
///
/// `breaks` must be sorted by glyph index and the boolean marks hyphenated breaks.
/// `checks` limits the candidates measured individually from the end; older non-hyphenated
/// candidates are bisected, assuming their drawn widths are nondecreasing. If none fits,
/// the earliest eligible break is returned. Without a tab or a measurement callback the
/// original `chosen` break is returned.
pub fn settle_line_break(
tabbed: bool,
checks: usize,
start: usize,
chosen: (usize, bool),
breaks: &[(usize, bool)],
fits: Option<&dyn Fn(usize, bool) -> bool>,
) -> (usize, bool) {
let Some(fits) = fits.filter(|_| tabbed) else { return chosen };
if fits(chosen.0, chosen.1) {
return chosen;
}
let earlier = breaks.get(breaks.partition_point(|b| b.0 <= start)..breaks.partition_point(|b| b.0 < chosen.0)).unwrap_or_default();
if let Some(&b) = earlier.iter().rev().take(checks).find(|b| fits(b.0, b.1)) {
return b;
}
let rest = earlier.get(..earlier.len().saturating_sub(checks)).unwrap_or_default();
let plain: Vec<&(usize, bool)> = rest.iter().filter(|b| !b.1).collect();
let fitting = plain.partition_point(|b| fits(b.0, b.1));
fitting.checked_sub(1).and_then(|k| plain.get(k).copied()).or(earlier.first()).copied().unwrap_or(chosen)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn measures_latest_candidates_and_preserves_hyphenation() {
let breaks = [(1, false), (2, false), (3, true), (4, false)];
let fits = |i, hyphen| i <= 2 || (i == 3 && hyphen);
assert_eq!(settle_line_break(true, 2, 0, (4, false), &breaks, Some(&fits)), (3, true));
assert_eq!(settle_line_break(false, 2, 0, (4, false), &breaks, Some(&fits)), (4, false));
}
#[test]
fn bisects_older_plain_breaks_and_falls_back_to_first_eligible() {
let breaks = [(1, false), (2, true), (3, false), (4, false), (5, false)];
assert_eq!(settle_line_break(true, 1, 0, (5, false), &breaks, Some(&|i, _| i <= 3)), (3, false));
assert_eq!(settle_line_break(true, 1, 1, (5, false), &breaks, Some(&|_, _| false)), (2, true));
assert_eq!(settle_line_break(true, 1, 5, (5, false), &breaks, Some(&|_, _| false)), (5, false));
}
}
+52
View File
@@ -0,0 +1,52 @@
//! Per-face bounded shaping plan reuse.
use std::sync::{Arc, Mutex};
/// A cache for one font face and variation instance. Do not share it across faces or instances.
/// Plans are created outside the lock, then checked again to avoid retaining duplicates.
#[derive(Default)]
pub struct ShapePlanCache {
plans: Mutex<Vec<Arc<harfrust::ShapePlan>>>,
}
impl ShapePlanCache {
/// Maximum number of plans retained; the oldest is evicted when this is reached.
pub const CAPACITY: usize = 32;
/// Returns a matching plan or builds and retains one for this face.
pub fn get_or_create(
&self,
shaper: &harfrust::Shaper<'_>,
instance: Option<&harfrust::ShaperInstance>,
direction: harfrust::Direction,
script: Option<harfrust::Script>,
language: Option<&harfrust::Language>,
features: &[harfrust::Feature],
) -> Arc<harfrust::ShapePlan> {
let key = harfrust::ShapePlanKey::new(script, direction).language(language).instance(instance).features(features);
let kept = |plans: &[Arc<harfrust::ShapePlan>]| plans.iter().rev().find(|p| key.matches(p)).cloned();
if let Some(plan) = kept(&self.plans.lock().unwrap_or_else(|e| e.into_inner())) {
return plan;
}
let plan = Arc::new(harfrust::ShapePlan::new(shaper, direction, script, language, features));
let mut plans = self.plans.lock().unwrap_or_else(|e| e.into_inner());
if let Some(made) = kept(&plans) {
return made;
}
if plans.len() >= Self::CAPACITY {
plans.remove(0);
}
plans.push(plan.clone());
plan
}
/// Number of retained plans.
pub fn len(&self) -> usize {
self.plans.lock().unwrap_or_else(|e| e.into_inner()).len()
}
/// Whether this cache contains no plans.
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
+55
View File
@@ -0,0 +1,55 @@
//! Boundaries between runs that need separate shaping.
/// Tracks the properties of the current shaping segment while walking grapheme clusters.
/// The caller resolves font faces, bidi levels and script classification; marks stay with the
/// preceding segment even when their face, level or case differs.
pub struct SegmentBoundary<F, L, S> {
face: F,
level: L,
small_caps: bool,
strong_script: Option<S>,
}
impl<F: Copy + Eq, L: Copy + Eq, S: Copy + Eq> SegmentBoundary<F, L, S> {
/// Begin at the first segment's face and bidi level.
pub fn new(face: F, level: L) -> Self {
Self { face, level, small_caps: false, strong_script: None }
}
/// Observe a cluster and return whether it begins a new shaping segment. Pass `None` for
/// common or inherited script and `true` for marks, joiners and variation selectors.
pub fn advance(&mut self, face: F, level: L, small_caps: bool, strong_script: Option<S>, mark: bool) -> bool {
let script_change = matches!((self.strong_script, strong_script), (Some(a), Some(b)) if a != b);
let split = (self.face != face || self.level != level || self.small_caps != small_caps || script_change) && !mark;
if split {
self.face = face;
self.level = level;
self.small_caps = small_caps;
}
if let Some(script) = strong_script {
self.strong_script = Some(script);
}
split
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn preserves_marks_and_splits_on_face_direction_case_and_script() {
let mut state = SegmentBoundary::<u8, u8, u8>::new(1, 0);
assert!(!state.advance(1, 0, false, Some(1), false));
assert!(!state.advance(2, 1, true, None, true));
assert!(state.advance(2, 1, true, Some(2), false));
assert!(!state.advance(2, 1, true, None, false));
assert!(state.advance(2, 1, true, Some(1), false));
}
#[test]
fn strong_script_on_a_mark_updates_the_following_boundary() {
let mut state = SegmentBoundary::<u8, u8, u8>::new(1, 0);
assert!(!state.advance(1, 0, false, Some(1), false));
assert!(!state.advance(1, 0, false, Some(2), true));
assert!(!state.advance(1, 0, false, Some(2), false));
}
}
+181
View File
@@ -0,0 +1,181 @@
//! Shape a run with its source clusters and surrounding text context.
use std::sync::Arc;
use harfrust::{BufferClusterLevel, Direction, Feature, Language, Script, ShapeOptions, ShapePlan, Shaper, UnicodeBuffer};
/// Parameters for a single directional run. Text is in logical order and `source_start` is its
/// byte offset in the complete source. Context influences joining but emits no glyphs.
pub struct RunInput<'a> {
/// Text in this run.
pub text: &'a str,
/// Its byte offset in the complete source.
pub source_start: usize,
/// Text before the run, within the same shaping context.
pub pre_context: &'a str,
/// Text after the run, within the same shaping context.
pub post_context: &'a str,
/// Direction of shaping.
pub direction: Direction,
/// Language used for language-specific forms.
pub language: Option<&'a Language>,
/// OpenType features with ranges in source byte coordinates.
pub features: &'a [Feature],
/// Use character-level instead of monotone grapheme clusters.
pub character_clusters: bool,
/// Uppercase each mapped character without losing its source cluster.
pub uppercase: bool,
}
/// A glyph in design units, with its source-byte cluster and directional advances and offsets.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct RunGlyph {
pub glyph_id: u32,
pub cluster: u32,
pub x_advance: i32,
pub y_advance: i32,
pub x_offset: i32,
pub y_offset: i32,
}
/// A run's source offsets cannot fit HarfRust's 32-bit clusters.
#[derive(Debug, thiserror::Error)]
#[error("shaping cluster exceeds the 32-bit source offset range")]
pub struct ShapeRunError;
/// Shape a run using a caller's face-specific plan cache. `map_character` applies digit or other
/// character mapping before case conversion; expansions retain the original byte cluster.
/// The plan callback receives the guessed script, direction, language and feature set so it can
/// reuse plans for the particular shaper and its variable-font instance.
pub fn shape_run(
shaper: &Shaper<'_>,
input: &RunInput<'_>,
mut map_character: impl FnMut(char) -> char,
plan: impl FnOnce(&Shaper<'_>, Direction, Option<Script>, Option<&Language>, &[Feature]) -> Arc<ShapePlan>,
) -> Result<Vec<RunGlyph>, ShapeRunError> {
let mut buffer = UnicodeBuffer::new();
for (offset, character) in input.text.char_indices() {
let cluster = input.source_start.checked_add(offset).and_then(|n| u32::try_from(n).ok()).ok_or(ShapeRunError)?;
let character = map_character(character);
if input.uppercase {
for upper in character.to_uppercase() {
buffer.add(upper, cluster);
}
} else {
buffer.add(character, cluster);
}
}
buffer.set_direction(input.direction);
buffer.guess_segment_properties();
if input.character_clusters {
buffer.set_cluster_level(BufferClusterLevel::Characters);
}
if let Some(language) = input.language {
buffer.set_language(language.clone());
}
buffer.set_pre_context(input.pre_context);
buffer.set_post_context(input.post_context);
let script = Some(buffer.script()).filter(|s| *s != harfrust::script::UNKNOWN);
let plan = plan(shaper, buffer.direction(), script, input.language, input.features);
let glyphs = shaper.shape(buffer, ShapeOptions::new().features(input.features).plan(Some(&plan)));
Ok(glyphs
.glyph_infos()
.iter()
.zip(glyphs.glyph_positions())
.map(|(info, pos)| RunGlyph {
glyph_id: info.glyph_id,
cluster: info.cluster,
x_advance: pos.x_advance,
y_advance: pos.y_advance,
x_offset: pos.x_offset,
y_offset: pos.y_offset,
})
.collect())
}
#[cfg(test)]
mod tests {
use super::*;
use harfrust::{FontRef, ShaperData};
#[test]
fn synthetic_face_runs_without_installed_fonts() {
// A tiny face with a single .notdef glyph. The shaper still processes source
// clusters, direction, mapping, features and context without external font files.
let mut builder = write_fonts::FontBuilder::new();
let mut head = vec![0; 54];
head[18..20].copy_from_slice(&1000u16.to_be_bytes());
let mut hhea = vec![0; 36];
hhea[34..36].copy_from_slice(&1u16.to_be_bytes());
for (tag, bytes) in [(b"head", head), (b"hhea", hhea), (b"hmtx", vec![0, 100, 0, 0]), (b"maxp", vec![0, 0, 0x50, 0, 0, 1])] {
builder.add_raw(write_fonts::types::Tag::new(tag), bytes);
}
check_run(&builder.build());
}
#[test]
fn run_matches_direct_shaping_with_source_clusters_and_context() {
let Ok(dir) = std::env::var("CRAFT_FONTS_DIR") else { return };
let bytes = std::fs::read(std::path::Path::new(&dir).join("fonts/noto-sans-arabic/NotoSansArabic.ttf")).unwrap();
check_run(&bytes);
}
fn check_run(bytes: &[u8]) {
let font = FontRef::from_index(bytes, 0).unwrap();
let data = ShaperData::new(&font);
let shaper = data.shaper(&font).build();
let overflow = RunInput {
text: "ab",
source_start: usize::MAX,
pre_context: "",
post_context: "",
direction: Direction::LeftToRight,
language: None,
features: &[],
character_clusters: false,
uppercase: false,
};
assert!(shape_run(&shaper, &overflow, |c| c, |s, d, script, lang, f| Arc::new(ShapePlan::new(s, d, script, lang, f))).is_err());
let language: Language = "ar".parse().unwrap();
let features = [Feature::new(harfrust::Tag::new(b"kern"), 0, 19..25)];
for direction in [Direction::LeftToRight, Direction::RightToLeft, Direction::TopToBottom] {
let input = RunInput {
text: "بaß1",
source_start: 19,
pre_context: "س",
post_context: "م",
direction,
language: Some(&language),
features: &features,
character_clusters: true,
uppercase: true,
};
let map = |c| if c == '1' { '٢' } else { c };
let actual =
shape_run(&shaper, &input, map, |s, d, script, lang, features| Arc::new(ShapePlan::new(s, d, script, lang, features))).unwrap();
let mut buffer = UnicodeBuffer::new();
for (i, c) in input.text.char_indices() {
for upper in map(c).to_uppercase() {
buffer.add(upper, 19 + i as u32);
}
}
buffer.set_direction(direction);
buffer.guess_segment_properties();
buffer.set_cluster_level(BufferClusterLevel::Characters);
buffer.set_language(language.clone());
buffer.set_pre_context(input.pre_context);
buffer.set_post_context(input.post_context);
let script = Some(buffer.script()).filter(|s| *s != harfrust::script::UNKNOWN);
let plan = ShapePlan::new(&shaper, direction, script, Some(&language), &features);
let expected = shaper.shape(buffer, ShapeOptions::new().features(&features).plan(Some(&plan)));
let tuples: Vec<_> = expected
.glyph_infos()
.iter()
.zip(expected.glyph_positions())
.map(|(i, p)| (i.glyph_id, i.cluster, p.x_advance, p.y_advance, p.x_offset, p.y_offset))
.collect();
assert_eq!(actual.iter().map(|g| (g.glyph_id, g.cluster, g.x_advance, g.y_advance, g.x_offset, g.y_offset)).collect::<Vec<_>>(), tuples);
assert!(!actual.is_empty());
}
}
}
+19
View File
@@ -569,6 +569,25 @@ fn a_read_is_capped_where_the_file_has_grown() {
let _ = std::fs::remove_file(&path);
}
#[test]
fn shape_plan_cache_reuses_and_evicts_plans() {
let data = font(&spec("Test", "Regular", 400));
let face = harfrust::FontRef::from_index(&data, 0).unwrap();
let shaper_data = harfrust::ShaperData::new(&face);
let shaper = shaper_data.shaper(&face).build();
let cache = ShapePlanCache::default();
let make = |language: &harfrust::Language| cache.get_or_create(&shaper, None, harfrust::Direction::LeftToRight, None, Some(language), &[]);
let languages: Vec<harfrust::Language> = ('a'..='z').flat_map(|a| ['a', 'b'].map(move |b| format!("{a}{b}").parse().unwrap())).collect();
let first = make(&languages[0]);
assert!(Arc::ptr_eq(&first, &make(&languages[0])));
for language in languages.iter().skip(1) {
make(language);
}
assert_eq!(cache.len(), ShapePlanCache::CAPACITY);
assert!(!Arc::ptr_eq(&first, &make(&languages[0])));
assert_eq!(cache.len(), ShapePlanCache::CAPACITY);
}
#[test]
fn bounded_sfnt_reader_preserves_face_names_and_collection_indices() {
let mut bold = spec("Example", "Bold Italic", 700);
+69
View File
@@ -0,0 +1,69 @@
//! Visual ordering of shaped clusters after line bidi levels are resolved.
use std::ops::Range;
use unicode_bidi::{BidiInfo, Level};
/// A glyph's source byte, source length and original resolved level.
/// Zero-length glyphs represent list markers.
#[derive(Clone, Copy)]
pub struct VisualGlyph {
/// Byte offset in the original text.
pub byte: usize,
/// Source length in bytes, or zero for a list marker.
pub len: usize,
/// Original level, also used when the line levels lack this byte.
pub level: Level,
}
/// Order glyph indices by cluster using caller-resolved line levels (UAX #9 L1–L2).
/// `levels` is indexed relative to `paragraph_start`. Adjacent marker glyphs at the
/// same byte form their own cluster and reverse internally at a right-to-left level.
pub fn visual_order(glyphs: impl IntoIterator<Item = VisualGlyph>, paragraph_start: usize, levels: &[Level]) -> Vec<usize> {
let mut clusters: Vec<(Range<usize>, VisualGlyph)> = Vec::new();
for (i, glyph) in glyphs.into_iter().enumerate() {
if let Some((range, _)) = clusters.last_mut().filter(|(_, prev)| prev.byte == glyph.byte && (prev.len == 0) == (glyph.len == 0)) {
range.end = i + 1;
} else {
clusters.push((i..i + 1, glyph));
}
}
let cluster_levels: Vec<_> = clusters
.iter()
.map(|(_, g)| if g.len == 0 { g.level } else { levels.get(g.byte.saturating_sub(paragraph_start)).copied().unwrap_or(g.level) })
.collect();
BidiInfo::reorder_visual(&cluster_levels)
.into_iter()
.filter_map(|i| clusters.get(i))
.flat_map(|(range, glyph)| {
let mut indices: Vec<_> = range.clone().collect();
if glyph.len == 0 && glyph.level.is_rtl() {
indices.reverse();
}
indices
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn clusters_and_markers_keep_their_distinct_order() {
let l = Level::ltr();
let r = Level::rtl();
let glyphs = [
VisualGlyph { byte: 5, len: 0, level: r },
VisualGlyph { byte: 5, len: 0, level: r },
VisualGlyph { byte: 5, len: 2, level: r },
VisualGlyph { byte: 5, len: 2, level: r },
VisualGlyph { byte: 7, len: 1, level: l },
];
assert_eq!(visual_order(glyphs, 5, &[r, r, l]), vec![2, 3, 1, 0, 4]);
assert!(visual_order([], 0, &[]).is_empty());
}
#[test]
fn missing_levels_fall_back_to_glyph_level() {
let r = Level::rtl();
let glyphs = [VisualGlyph { byte: 2, len: 1, level: r }, VisualGlyph { byte: 3, len: 1, level: r }];
assert_eq!(visual_order(glyphs, 0, &[]), vec![1, 0]);
}
}
+20
View File
@@ -0,0 +1,20 @@
[package]
name = "craft-format"
description = "Versioned JSON file formats: the format tag, the registry of versions (hand-written steps, then one folder of changes per version, then the pending changes), the upgrade of an older file and the telling of a newer one"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
serde_json = { workspace = true }
# The container: content-addressed blobs (SHA-256) in a ZIP package.
sha2 = { version = "0.10", default-features = false, features = ["std"] }
craft-zip = { workspace = true }
[dev-dependencies]
proptest = { workspace = true }
+237
View File
@@ -0,0 +1,237 @@
//! For a build script: the generated lists of a family's changes. Each change is one `.rs` file in
//! the family's folder, in `v<N>/` once a release fixed its number, else in `pending/`; the lists
//! are generated so that a change adds a file and never edits a line another change also edits.
//!
//! The generated code, for the app to `include!`, is per family a `FOLDED` list (each folded
//! format and its parts) and a `PENDING` list, both prefixed by the family's `list_prefix`, then a
//! module per change file. It names the app's `Part` type, which the including module has in scope
//! (`type Part = craft_format::Part<AppError>;`), and a change file's step as `step` (a
//! `pub fn step(v: &mut Value) -> Result<(), AppError>`, when the file has one).
use std::fmt::Write as _;
use std::path::{Path, PathBuf};
/// A family's change files: their folder (relative to the crate's manifest), the last format
/// before them, and the prefix of the family's lists.
pub struct Family {
pub folder: &'static str,
pub base: u32,
pub list_prefix: &'static str,
}
/// What [`lists`] made: the code to write, and the folders the build script reruns on.
pub struct Lists {
pub code: String,
pub watch: Vec<PathBuf>,
}
/// The `FOLDED` and `PENDING` lists of every family, and a module for each change file. Refuses
/// versions that are not contiguous from a family's base, a change file not named with lowercase
/// letters, digits and dashes, and one that doesn't start with a `//!` note.
pub fn lists(manifest: &Path, families: &[Family]) -> Result<Lists, String> {
let mut code = String::new();
let mut mods = String::new();
let mut watch = Vec::new();
for family in families {
let root = manifest.join(family.folder);
family_lists(family, &root, &mut code, &mut mods)?;
watch.push(root);
}
code.push_str(&mods);
Ok(Lists { code, watch })
}
/// A family's `FOLDED` and `PENDING` lists; its folded formats follow on from its base.
fn family_lists(family: &Family, root: &Path, out: &mut String, mods: &mut String) -> Result<(), String> {
let Family { folder, base, list_prefix } = family;
let mut folded: Vec<(u32, PathBuf)> = Vec::new();
for entry in std::fs::read_dir(root).map_err(|e| format!("{}: {e}", root.display()))? {
// an entry that can't be read could be a version folder: fail rather than drop it
let path = entry.map_err(|e| format!("{}: {e}", root.display()))?.path();
let version = path.file_name().and_then(|n| n.to_str()).and_then(|n| n.strip_prefix('v')).and_then(|n| n.parse().ok());
if let Some(version) = version
&& path.is_dir()
{
folded.push((version, path));
}
}
folded.sort();
for (i, (version, _)) in folded.iter().enumerate() {
let expected = base + 1 + i as u32;
if *version != expected {
return Err(format!("{folder}/: expected v{expected}/ next, found v{version}/ (versions are contiguous)"));
}
}
// Module names start with the family's (`drawing_pending_…`), so two families' changes of one
// name never clash.
let module = list_prefix.to_ascii_lowercase();
let _ = writeln!(
out,
"/// The changes of each format after the base, oldest first (`{folder}/v<N>/`).\nconst {list_prefix}FOLDED: &[(u32, &[Part])] = &["
);
for (version, dir) in &folded {
let _ = writeln!(out, " ({version}, &[");
parts(dir, &format!("{module}v{version}"), false, out, mods)?;
out.push_str(" ]),\n");
}
let _ = writeln!(out, "];\n\n/// The changes no release has a number for yet (`{folder}/pending/`).\nconst {list_prefix}PENDING: &[Part] = &[");
parts(&root.join("pending"), &format!("{module}pending"), true, out, mods)?;
out.push_str("];\n\n");
Ok(())
}
/// Whether a change file declares its `step` function (`fn step(`, `pub fn step (` …) on a line of
/// code: a mention in a comment or a note is not a declaration.
fn declares_step(text: &str) -> bool {
let code = without_comments(text);
let ident = |c: char| c.is_alphanumeric() || c == '_';
// `fn`, whitespace, `step`, then `(` (or generics), across lines
code.match_indices("fn").any(|(at, _)| {
let before_ok = code[..at].chars().next_back().is_none_or(|c| !ident(c));
let rest = &code[at + 2..];
before_ok
&& rest.starts_with(char::is_whitespace)
&& rest
.trim_start()
.strip_prefix("step")
.is_some_and(|rest| matches!(rest.trim_start().chars().next(), Some('(' | '<')) && !rest.starts_with(ident))
})
}
/// `text` as code only: its `//` and (nested) `/* */` comments become a space, and the contents
/// of its string literals (plain, byte and raw) are dropped, line breaks kept throughout. A comment
/// marker inside a string is not a comment, and `fn/* note */step` still reads as `fn step`.
fn without_comments(text: &str) -> String {
let chars: Vec<char> = text.chars().collect();
let at = |i: usize| chars.get(i).copied();
let mut out = String::with_capacity(text.len());
let (mut i, mut depth) = (0usize, 0usize);
while let Some(c) = at(i) {
let next = at(i + 1);
if depth > 0 {
match (c, next) {
('/', Some('*')) => (depth, i) = (depth + 1, i + 2),
('*', Some('/')) => (depth, i) = (depth - 1, i + 2),
_ => {
if c == '\n' {
out.push('\n');
}
i += 1;
}
}
if depth == 0 {
out.push(' ');
}
continue;
}
let ident_before = i > 0 && at(i - 1).is_some_and(|p| p.is_alphanumeric() || p == '_');
match (c, next) {
('/', Some('/')) => {
while at(i).is_some_and(|c| c != '\n') {
i += 1;
}
out.push(' ');
}
('/', Some('*')) => (depth, i) = (1, i + 2),
('"', _) => {
out.push('"');
i += 1;
while let Some(c) = at(i) {
i += 1;
match c {
'\\' => i += 1,
'"' => break,
'\n' => out.push('\n'),
_ => {}
}
}
out.push('"');
}
// a raw string, `r"…"` or `r#"…"#`, after a byte prefix or on its own
('r', Some('"' | '#'))
if !ident_before || (i > 0 && at(i - 1) == Some('b') && !(i > 1 && at(i - 2).is_some_and(|p| p.is_alphanumeric() || p == '_'))) =>
{
let mut j = i + 1;
let mut hashes = 0;
while at(j) == Some('#') {
hashes += 1;
j += 1;
}
if at(j) != Some('"') {
out.push(c);
i += 1;
continue;
}
out.push('"');
i = j + 1;
while let Some(c) = at(i) {
if c == '"' && (1..=hashes).all(|k| at(i + k) == Some('#')) {
i += 1 + hashes;
break;
}
if c == '\n' {
out.push('\n');
}
i += 1;
}
out.push('"');
}
// a character literal (`'"'`, `'\''`), not a lifetime
('\'', Some('\\')) => {
i += 2;
while at(i).is_some_and(|c| c != '\'') {
i += 1;
}
i += 1;
out.push_str("' '");
}
('\'', Some(_)) if at(i + 2) == Some('\'') => {
i += 3;
out.push_str("' '");
}
_ => {
out.push(c);
i += 1;
}
}
}
out
}
/// The `Part`s of the change files in `dir`, in file-name order, and a module for each. A missing
/// folder is an empty list only where it may be absent (`optional`: `pending/`); any other error
/// reading it fails the build rather than leave a change out of the lists.
fn parts(dir: &Path, prefix: &str, optional: bool, out: &mut String, mods: &mut String) -> Result<(), String> {
let entries = match std::fs::read_dir(dir) {
Ok(entries) => entries,
Err(e) if optional && e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(e) => return Err(format!("{}: {e}", dir.display())),
};
let mut files: Vec<PathBuf> = Vec::new();
for entry in entries {
let path = entry.map_err(|e| format!("{}: {e}", dir.display()))?.path();
if path.extension().is_some_and(|x| x == "rs") {
files.push(path);
}
}
files.sort();
for path in files {
let name = path.file_stem().and_then(|s| s.to_str()).unwrap_or_default().to_string();
if name.is_empty() || !name.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') {
return Err(format!("{}: name a format change file with lowercase letters, digits and dashes", path.display()));
}
let text = std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?;
if !text.trim_start().starts_with("//!") {
return Err(format!("{}: start a format change with a `//!` note", path.display()));
}
let module = format!("{prefix}_{}", name.replace('-', "_"));
let step = if declares_step(&text) { format!("Some({module}::step)") } else { "None".into() };
let _ = writeln!(out, " Part {{ name: {name:?}, step: {step} }},");
let _ = writeln!(mods, "#[path = {:?}]\nmod {module};", path.display().to_string());
}
Ok(())
}
#[cfg(test)]
#[path = "build_tests.rs"]
mod tests;
+115
View File
@@ -0,0 +1,115 @@
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
/// A fresh, empty folder for one test.
fn scratch(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("craft-format-build-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
dir
}
fn write(path: &Path, text: &str) {
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, text).unwrap();
}
/// The error of a run that must fail.
fn refused(dir: &Path) -> String {
match lists(dir, &[DRAWING]) {
Ok(_) => panic!("expected an error"),
Err(e) => e,
}
}
const DRAWING: Family = Family { folder: "drawing", base: 3, list_prefix: "DRAWING_" };
#[test]
fn lists_folded_and_pending_changes_with_their_modules() {
let dir = scratch("ok");
write(&dir.join("drawing/v4/add-layers.rs"), "//! Layers.\npub fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n");
write(&dir.join("drawing/v4/notes.txt"), "not a change");
write(&dir.join("drawing/v5/rename.rs"), "\n//! A rename with no step.\n");
write(&dir.join("drawing/pending/next-1.rs"), "//! Pending.\nfn step(v: &mut Value) {}\n");
std::fs::create_dir_all(dir.join("drawing/v6x")).unwrap();
write(&dir.join("drawing/v7"), "a file, not a version folder");
let lists = lists(&dir, &[DRAWING]).unwrap();
assert_eq!(lists.watch, vec![dir.join("drawing")]);
let code = &lists.code;
assert!(code.contains("const DRAWING_FOLDED: &[(u32, &[Part])] = &["), "{code}");
assert!(code.contains(" (4, &[\n Part { name: \"add-layers\", step: Some(drawing_v4_add_layers::step) },\n ]),"), "{code}");
assert!(code.contains("Part { name: \"rename\", step: None }"), "{code}");
assert!(
code.contains("const DRAWING_PENDING: &[Part] = &[\n Part { name: \"next-1\", step: Some(drawing_pending_next_1::step) },"),
"{code}"
);
assert!(code.contains("mod drawing_v5_rename;"), "{code}");
assert!(!code.contains("notes"), "{code}");
assert!(code.find("const DRAWING_PENDING").unwrap() < code.find("#[path").unwrap(), "modules follow the lists");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn an_empty_family_has_empty_lists() {
let dir = scratch("empty");
std::fs::create_dir_all(dir.join("drawing")).unwrap();
let code = lists(&dir, &[DRAWING]).unwrap().code;
assert!(code.contains("const DRAWING_FOLDED: &[(u32, &[Part])] = &[\n];"), "{code}");
assert!(code.contains("const DRAWING_PENDING: &[Part] = &[\n];"), "{code}");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn refuses_gaps_bad_names_missing_notes_and_missing_folders() {
let dir = scratch("bad");
assert!(refused(&dir).contains("drawing"));
write(&dir.join("drawing/v5/a.rs"), "//! a");
assert!(refused(&dir).contains("expected v4/ next, found v5/"));
std::fs::rename(dir.join("drawing/v5"), dir.join("drawing/v4")).unwrap();
write(&dir.join("drawing/v4/Bad_Name.rs"), "//! b");
assert!(refused(&dir).contains("lowercase letters"));
std::fs::remove_file(dir.join("drawing/v4/Bad_Name.rs")).unwrap();
write(&dir.join("drawing/pending/no-note.rs"), "pub fn step() {}");
assert!(refused(&dir).contains("`//!` note"));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_step_is_a_declaration_not_a_mention() {
assert!(!declares_step("//! No fn step() is needed.\n"));
assert!(!declares_step("//! a note\n// fn step(v) was removed\nconst X: u8 = 1;\n"));
assert!(declares_step("//! a note\npub fn step (v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
// a declaration inside a block comment, nested or not, is not one
assert!(!declares_step("//! Documentation only.\n/*\nfn step() {}\n*/\n"));
assert!(!declares_step("/* outer /* inner */\nfn step() {}\n*/\nconst X: u8 = 1;\n"));
assert!(declares_step("/* a note */\nfn step(v: &mut Value) {}\n"));
// comment markers inside strings and character literals are not comments
assert!(declares_step("//! A change with a step.\nconst MARKER: &str = \"/*\";\npub fn step(_v: &mut Value) -> Result { Ok(()) }\n"));
assert!(declares_step("const A: &str = r#\"/* \"quoted\" \"#;\nconst B: &[u8] = br\"/*\";\nfn step(v: &mut Value) {}\n"));
assert!(declares_step("const Q: char = '\"';\nconst E: char = '\\'';\nfn step(v: &mut Value) { let _: &'static str = \"\"; }\n"));
assert!(declares_step("const S: &str = \"a \\\" /* b\";\nfn step(v: &mut Value) {}\n"));
assert!(!declares_step("const S: &str = \"\nfn step(v) \";\n"), "inside a string");
// a comment between the words is whitespace
assert!(declares_step("pub fn/* migration */step(v: &mut Value) {}\n"));
// across lines
assert!(declares_step("//! A change with a step.\npub fn step\n (_v: &mut Value) -> Result {\n Ok(())\n}\n"));
assert!(declares_step("pub fn\nstep(v: &mut Value) {}\n"));
assert!(!declares_step("fn stepper(v: &mut Value) {}\nfn prefn step() {}\n"), "other names");
assert!(declares_step("const X: u8 = 1; // fn step() is below\nfn step(v: &mut Value) {}\n"));
assert!(declares_step("//! a note\npub(crate) fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
assert!(declares_step("//! a note\n fn step(v: &mut Value) {}\n"));
assert!(!declares_step("//! a note\nfn stepper(v: &mut Value) {}\nfn fnstep() {}\n"));
}
#[test]
fn a_documentation_only_change_has_no_step_and_a_missing_version_folder_fails() {
let dir = scratch("doc-only");
write(&dir.join("drawing/v4/doc.rs"), "//! No fn step() is needed.\n");
let code = lists(&dir, &[DRAWING]).unwrap().code;
assert!(code.contains("Part { name: \"doc\", step: None }"), "{code}");
// a version folder that can't be listed is an error, not an empty format
let missing = parts(&dir.join("drawing/v9"), "x", false, &mut String::new(), &mut String::new());
assert!(missing.is_err());
assert!(parts(&dir.join("drawing/pending"), "x", true, &mut String::new(), &mut String::new()).is_ok());
let _ = std::fs::remove_dir_all(&dir);
}
+440
View File
@@ -0,0 +1,440 @@
//! The file container: a ZIP package of named parts (the file's JSON, binary payloads, optional
//! caches), never extracted to filesystem paths. Payloads are content-addressed blobs,
//! `blobs/<sha256>.<extension>`, stored once however often they are referred to. Every size is
//! bounded by the app's [`Limits`]: the file, each part, all parts expanded, the payloads resolved,
//! and the number of entries; a part is read no further than its declared size (a decompression
//! bomb stops there). What a part holds, and how a payload is encoded, is the app's.
use std::collections::{BTreeMap, BTreeSet};
use std::io::{Read, Seek, SeekFrom};
use std::sync::Arc;
use serde_json::Value;
use sha2::{Digest, Sha256};
use crate::FormatError;
/// How hard the payloads are compressed: Fast favours interactive file operations; Compact trades
/// CPU time for space.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum Compression {
#[default]
Fast,
Compact,
}
/// An app's budgets for its containers. The functions are read each time they are needed, so a
/// limit the app lets the user raise takes effect at once.
#[derive(Clone, Copy)]
pub struct Limits {
/// The most entries a container holds.
pub max_entries: usize,
/// The most bytes all parts may expand to, and all payloads may resolve to.
pub max_expanded: fn() -> usize,
/// The most bytes a part of this name may expand to.
pub part_limit: fn(&str) -> usize,
/// The Deflate level of a compressed part of this name and size.
pub level: fn(&str, usize) -> i64,
/// Whether a container of this many bytes may be read, or why not.
pub check_size: fn(u64) -> Result<(), String>,
/// Whether the work is cancelled (a save the person stopped): checked before each part, and
/// a long part is written a piece at a time and stops before the next piece when this says
/// so ([`FormatError::Cancelled`]). `None`: it cannot be cancelled here.
pub cancelled: Option<fn() -> bool>,
}
/// The most bytes of a part written between two cancellation checks.
const PIECE: usize = 1 << 20;
fn invalid(message: impl std::fmt::Display) -> FormatError {
FormatError::Invalid(message.to_string())
}
/// Whether the bytes start as a container does, whatever the file is called.
pub fn is_container(bytes: &[u8]) -> bool {
bytes.starts_with(b"PK\x03\x04")
}
/// The SHA-256 of the bytes.
pub fn sha256(bytes: &[u8]) -> [u8; 32] {
Sha256::digest(bytes).into()
}
fn hex(digest: &[u8; 32]) -> String {
use std::fmt::Write as _;
let mut out = String::with_capacity(64);
for byte in digest {
// Writing to a `String` cannot fail.
let _ = write!(&mut out, "{byte:02x}");
}
out
}
/// The SHA-256 of the bytes, in lowercase hexadecimal: a blob's name, and how a cache is keyed to
/// the file it was written for.
pub fn digest(bytes: &[u8]) -> String {
hex(&sha256(bytes))
}
/// One part a [`Writer`] wrote, with the SHA-256 of its uncompressed content. A caller that signs
/// a container uses these to bind the parts without reading the file back.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PartDigest {
pub name: String,
pub len: u64,
pub digest: [u8; 32],
}
/// A container being written.
pub struct Writer {
zip: craft_zip::Writer,
names: BTreeSet<String>,
expanded: usize,
resolved: usize,
limits: Limits,
/// Which written parts to record for a signature, and the records so far.
tracked: Option<TrackedParts>,
}
/// A tracking closure and the part digests collected so far.
type TrackedParts = (fn(&str) -> bool, Vec<PartDigest>);
impl Writer {
pub fn new(limits: Limits) -> Writer {
Writer { zip: craft_zip::Writer::new(), names: BTreeSet::new(), expanded: 0, resolved: 0, limits, tracked: None }
}
/// Record the parts written from now on whose name `include` accepts, with their SHA-256, for
/// [`Writer::take_parts`]. Used by a caller that signs the container.
pub fn track_parts(&mut self, include: fn(&str) -> bool) {
self.tracked = Some((include, Vec::new()));
}
/// The tracked parts written so far, oldest first, and stop tracking.
pub fn take_parts(&mut self) -> Option<Vec<PartDigest>> {
self.tracked.take().map(|(_, parts)| parts)
}
/// The container's bytes.
pub fn finish(self) -> Result<Vec<u8>, FormatError> {
self.zip.finish().map_err(invalid)
}
/// Add a part, compressed or stored. A name already written is written once.
pub fn part(&mut self, name: &str, bytes: &[u8], compressed: bool) -> Result<(), FormatError> {
self.part_with_digest(name, bytes, compressed, None)
}
/// [`Writer::part`] with a digest the caller already worked out (a blob's name is its digest),
/// so a signed save does not hash the same bytes twice.
fn part_with_digest(&mut self, name: &str, bytes: &[u8], compressed: bool, digest: Option<[u8; 32]>) -> Result<(), FormatError> {
if bytes.len() > (self.limits.part_limit)(name) {
return Err(invalid("part too large"));
}
if self.names.contains(name) {
return Ok(());
}
// Check the counters this part would leave, and commit them only once it is written: a
// refused part is not remembered as written.
let expanded = self.expanded.checked_add(bytes.len()).ok_or_else(|| invalid("expanded size overflow"))?;
if expanded > (self.limits.max_expanded)() || self.names.len().saturating_add(1) > self.limits.max_entries {
return Err(invalid("package too large"));
}
let cancelled = self.limits.cancelled;
if cancelled.is_some_and(|c| c()) {
return Err(FormatError::Cancelled);
}
// the configured level, clamped to the Deflate levels craft-zip writes (1 to 9)
let compression = if compressed {
craft_zip::Compression::Level((self.limits.level)(name, bytes.len()).clamp(1, 9) as u8)
} else {
craft_zip::Compression::Stored
};
match cancelled {
Some(cancelled) if bytes.len() > PIECE => {
let mut stopped = false;
let written = self.zip.add_with(name, compression, |out| {
for piece in bytes.chunks(PIECE) {
if cancelled() {
stopped = true;
return Err(std::io::Error::other("cancelled"));
}
out.write_all(piece)?;
}
Ok(())
});
if stopped {
return Err(FormatError::Cancelled);
}
written.map_err(invalid)?;
}
_ => self.zip.add(name, bytes, compression).map_err(invalid)?,
}
self.names.insert(name.to_owned());
self.expanded = expanded;
if let Some((include, parts)) = &mut self.tracked
&& include(name)
{
let digest = digest.unwrap_or_else(|| sha256(bytes));
parts.push(PartDigest { name: name.to_owned(), len: bytes.len() as u64, digest });
}
Ok(())
}
/// Would `entries` more optional parts of `extra` bytes, stored uncompressed, still keep the
/// file within `budget` and its expanded size within the reader's budget? The parts already
/// written count at their expanded size (an upper bound on what they take in the file).
pub fn room(&self, entries: usize, extra: usize, budget: usize) -> bool {
self.remaining(entries, budget).is_some_and(|left| extra <= left)
}
/// The bytes `entries` more optional parts may hold together ([`Writer::room`]).
pub fn space(&self, entries: usize, budget: usize) -> usize {
self.remaining(entries, budget).unwrap_or(0)
}
/// The bytes left for `entries` more parts, or `None` when the entries don't fit or what is
/// written already is past `budget` or the reader's budget.
fn remaining(&self, entries: usize, budget: usize) -> Option<usize> {
let count = self.names.len().checked_add(entries).filter(|&n| n <= self.limits.max_entries)?;
let overhead = count.saturating_mul(256).saturating_add(65_536);
let in_file = budget.checked_sub(overhead.saturating_add(self.expanded))?;
let expanded = (self.limits.max_expanded)().checked_sub(self.expanded)?;
Some(in_file.min(expanded))
}
/// Add a payload as a blob, `blobs/<sha256>.<extension>`, and say its name, for the JSON that
/// refers to it.
pub fn blob(&mut self, bytes: &[u8], extension: &str, compressed: bool) -> Result<Value, FormatError> {
let resolved = self
.resolved
.checked_add(bytes.len())
.filter(|&n| n <= (self.limits.max_expanded)())
.ok_or_else(|| invalid("resolved payloads too large"))?;
let digest = sha256(bytes);
let name = format!("blobs/{}.{extension}", hex(&digest));
self.part_with_digest(&name, bytes, compressed, Some(digest))?;
self.resolved = resolved;
Ok(Value::String(name))
}
}
/// A container being read. Opening reads all bytes into memory and checks the directory
/// (the entry count, unique and plain names, every part's
/// declared size), not the parts.
pub struct Reader {
archive: craft_zip::Archive<'static>,
blobs: BTreeMap<String, Vec<u8>>,
/// The bytes of payloads resolved so far, repeats counted: bounded by the expanded budget.
pub hydrated: usize,
document: Option<Arc<[u8]>>,
limits: Limits,
/// Every byte part reads have expanded to, whatever the budget was at the time.
read_total: u64,
}
impl Reader {
pub fn new(mut source: impl Read + Seek, limits: Limits) -> Result<Self, FormatError> {
let length = source.seek(SeekFrom::End(0)).map_err(invalid)?;
(limits.check_size)(length).map_err(invalid)?;
// Read all bytes into memory for the archive, no more than the length just approved: a file
// that grew since is refused rather than read past its budget.
source.seek(SeekFrom::Start(0)).map_err(invalid)?;
let capacity = usize::try_from(length).map_err(|_| invalid("file too large"))?;
let mut bytes = Vec::new();
bytes.try_reserve_exact(capacity).map_err(|_| invalid("file too large"))?;
source.take(length.saturating_add(1)).read_to_end(&mut bytes).map_err(invalid)?;
if bytes.len() as u64 != length {
return Err(invalid("the file changed while it was read"));
}
let mut archive = craft_zip::Archive::open_owned(
bytes,
craft_zip::Limits { max_entries: limits.max_entries as u64, max_total_bytes: (limits.max_expanded)() as u64 },
)
.map_err(invalid)?;
// the directory lists distinct names, so a repeated name is caught here
if archive.has_duplicate_names() {
return Err(invalid("duplicate or invalid archive entry"));
}
// Validate entry names match format rules: no empty, no backslashes,
// no path components that are "." or "..", and no duplicates; and every part's declared
// size within its configured limit.
let mut name_set = BTreeSet::new();
for entry_idx in 0..archive.len() {
// an entry whose name can't be read would be left out of the package unnoticed
let Ok(Some(name)) = archive.entry_name(entry_idx) else {
return Err(invalid("duplicate or invalid archive entry"));
};
{
if name.is_empty()
|| name.contains('\\')
|| name.split('/').any(|c| c.is_empty() || c == "." || c == "..")
|| !name_set.insert(name.clone())
{
return Err(invalid("duplicate or invalid archive entry"));
}
if archive.declared_size(&name).is_some_and(|size| size > (limits.part_limit)(&name) as u64) {
return Err(invalid("part too large"));
}
}
}
Ok(Self { archive, blobs: BTreeMap::new(), hydrated: 0, document: None, limits, read_total: 0 })
}
/// The file's own part, `document.json`, read once and kept: provisional caches and the
/// authoritative reading are bound to the same bytes, even if another process overwrites the
/// open file in place. Blobs remain hash-checked.
pub fn document(&mut self, limit: usize) -> Result<Arc<[u8]>, FormatError> {
if self.document.is_none() {
let bytes = self.limited_part("document.json", limit)?;
self.document = Some(bytes.into());
}
// a cached document still keeps the current part limit
let limit = limit.min((self.limits.part_limit)("document.json"));
self.document.as_ref().filter(|d| d.len() <= limit).cloned().ok_or_else(|| invalid("metadata budget"))
}
/// Whether the container has a part of this name.
pub fn has_part(&self, name: &str) -> bool {
self.archive.contains(name)
}
/// The names of every part, in the container's order.
pub fn part_names(&self) -> Vec<String> {
self.archive.names().map(|n| n.into_owned()).collect()
}
/// The part of this name, at most `limit` bytes and never more than its configured limit.
pub fn limited_part(&mut self, name: &str, limit: usize) -> Result<Vec<u8>, FormatError> {
let limit = limit.min((self.limits.part_limit)(name));
let expected = self.archive.declared_size(name).ok_or_else(|| invalid("part not found"))?;
if expected > limit as u64 {
return Err(invalid("part too large"));
}
// the expanded budget is read each time: a raised (or lowered) one applies at once, the
// bytes already read still counting against it
let budget = self.archive.budget_mut();
*budget = craft_zip::Budget::new(budget.what(), (self.limits.max_expanded)() as u64);
if budget.charge(self.read_total).is_err() {
return Err(invalid("package too large"));
}
// Read within budget. craft-zip's read() handles decompression bombs via the limit.
let bytes = self.archive.read(name, limit as u64).map_err(invalid)?.ok_or_else(|| invalid("part not found"))?;
if bytes.len() as u64 != expected {
return Err(invalid("part length mismatch"));
}
self.read_total = self.read_total.saturating_add(expected);
Ok(bytes)
}
/// The blob `value`'s `field` names, of this extension, within its part limit.
pub fn blob(&mut self, value: &Value, field: &str, extension: &str) -> Result<&[u8], FormatError> {
let limit = (self.limits.part_limit)(&format!("blob.{extension}"));
self.limited_blob(value, field, extension, limit)
}
/// [`Reader::blob`] within `limit` bytes. Its hash is checked when it is first read, and every
/// reference counts against the resolved budget.
pub fn limited_blob(&mut self, value: &Value, field: &str, extension: &str, limit: usize) -> Result<&[u8], FormatError> {
let name = value.get(field).and_then(Value::as_str).ok_or_else(|| invalid(format!("missing `{field}` reference")))?;
// a cached blob still keeps the part's current limit
let limit = limit.min((self.limits.part_limit)(name));
let hash = name
.strip_prefix("blobs/")
.and_then(|n| n.strip_suffix(&format!(".{extension}")))
.filter(|h| h.len() == 64 && h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))
.ok_or_else(|| invalid("invalid blob reference"))?;
if !self.blobs.contains_key(name) {
let bytes = self.limited_part(name, limit)?;
if digest(&bytes) != hash {
return Err(invalid(format!("blob hash mismatch: {name}")));
}
self.blobs.insert(name.to_owned(), bytes);
}
let bytes = self.blobs.get(name).ok_or_else(|| invalid("missing blob"))?;
if bytes.len() > limit {
return Err(invalid("blob exceeds its resource budget"));
}
// Count repeated references too: a tiny archive must not expand into unbounded owned
// strings or arrays when the same blob is referenced repeatedly.
self.hydrated = self
.hydrated
.checked_add(bytes.len())
.filter(|&n| n <= (self.limits.max_expanded)())
.ok_or_else(|| invalid("resolved payloads too large"))?;
Ok(bytes)
}
}
/// A small PNG picture of the file, kept in the container as a disposable cache keyed to the
/// exact file it shows: `cache/thumbnail.json` (its key) and `cache/thumbnail.png`. Reading it
/// never resolves a payload; a picture that isn't one, or is stale, is a miss.
#[derive(Clone, Copy, Debug)]
pub struct Thumbnail {
/// The key's format tag.
pub format: &'static str,
/// The build that wrote it: only the same build reads it back.
pub app: &'static str,
/// Its width and height in pixels.
pub size: [usize; 2],
/// The most bytes it may have.
pub max_bytes: usize,
}
impl Thumbnail {
/// Whether the bytes start as a PNG of this size within the byte budget (not a decode: a
/// consumer decodes with its own dimension and allocation limits).
pub fn valid(&self, bytes: &[u8]) -> bool {
bytes.len() <= self.max_bytes
&& bytes.get(..8) == Some(b"\x89PNG\r\n\x1a\n")
&& bytes.get(8..16) == Some(b"\0\0\0\rIHDR")
&& bytes.get(16..20) == Some((self.size[0] as u32).to_be_bytes().as_slice())
&& bytes.get(20..24) == Some((self.size[1] as u32).to_be_bytes().as_slice())
}
/// Add the picture, keyed to `document` (the file's own part), when it is one and there is
/// room for it within `budget`; otherwise leave it out, never failing the save.
pub fn write(&self, writer: &mut Writer, png: &[u8], document: &[u8], budget: usize) -> Result<(), FormatError> {
if !self.valid(png) {
return Ok(());
}
let key =
serde_json::to_vec(&serde_json::json!({"format": self.format, "app": self.app, "document": digest(document), "payload": digest(png)}))
.map_err(invalid)?;
let fits = |name: &str, len: usize| len <= (writer.limits.part_limit)(name);
if !fits("cache/thumbnail.json", key.len())
|| !fits("cache/thumbnail.png", png.len())
|| !writer.room(2, key.len().saturating_add(png.len()), budget)
{
return Ok(());
}
writer.part("cache/thumbnail.json", &key, false)?;
writer.part("cache/thumbnail.png", png, false)
}
/// The picture, when this build wrote it for exactly the file's own part (`document.json`,
/// read within `document_limit`).
pub fn read(&self, reader: &mut Reader, document_limit: usize) -> Result<Vec<u8>, FormatError> {
let key: Value = serde_json::from_slice(&reader.limited_part("cache/thumbnail.json", 4096)?).map_err(invalid)?;
let document = reader.document(document_limit)?;
if key.get("format").and_then(Value::as_str) != Some(self.format)
|| key.get("app").and_then(Value::as_str) != Some(self.app)
|| key.get("document").and_then(Value::as_str) != Some(digest(&document).as_str())
{
return Err(invalid("thumbnail key"));
}
let png = reader.limited_part("cache/thumbnail.png", self.max_bytes)?;
if !self.valid(&png) || key.get("payload").and_then(Value::as_str) != Some(digest(&png).as_str()) {
return Err(invalid("thumbnail payload"));
}
Ok(png)
}
}
#[cfg(test)]
#[path = "container_tests.rs"]
mod tests;
+260
View File
@@ -0,0 +1,260 @@
use super::*;
use std::io::Cursor;
const LIMITS: Limits = Limits {
max_entries: 8,
max_expanded: || 1 << 20,
part_limit: |name| if name.ends_with(".big") { 1 << 20 } else { 1024 },
level: |_, _| 1,
check_size: |length| if length <= 1 << 20 { Ok(()) } else { Err("too large".into()) },
cancelled: None,
};
/// Parts and blobs written are read back as they were; a blob is stored once, named by its hash,
/// and checked against it.
#[test]
fn parts_and_blobs_read_back() {
let mut writer = Writer::new(LIMITS);
writer.part("document.json", br#"{"a":1}"#, true).unwrap();
let first = writer.blob(b"payload", "bin", false).unwrap();
let again = writer.blob(b"payload", "bin", true).unwrap();
assert_eq!(first, again, "the same payload is one blob");
assert_eq!(first, Value::String(format!("blobs/{}.bin", digest(b"payload"))));
let bytes = writer.finish().unwrap();
assert!(is_container(&bytes));
let mut reader = Reader::new(Cursor::new(bytes), LIMITS).unwrap();
assert_eq!(reader.part_names().len(), 2);
assert!(reader.has_part("document.json") && !reader.has_part("other.json"));
assert_eq!(&*reader.document(64).unwrap(), br#"{"a":1}"#);
let reference = serde_json::json!({"blob": first});
assert_eq!(reader.blob(&reference, "blob", "bin").unwrap(), b"payload");
assert!(reader.blob(&reference, "blob", "png").is_err(), "another extension");
assert_eq!(reader.hydrated, 7);
reader.hydrated = (LIMITS.max_expanded)();
assert!(reader.blob(&reference, "blob", "bin").unwrap_err().to_string().contains("resolved payloads too large"));
}
/// A part past its limit, or more entries than the limit, are refused as they are written.
#[test]
fn budgets_are_kept_when_writing() {
let mut writer = Writer::new(LIMITS);
assert_eq!(writer.part("small.json", &[0; 1025], false), Err(FormatError::Invalid("part too large".into())));
assert!(writer.part("large.big", &[0; 1025], true).is_ok());
for i in 0..7 {
writer.part(&format!("{i}.json"), b"x", false).unwrap();
}
assert_eq!(writer.part("one-more.json", b"x", false), Err(FormatError::Invalid("package too large".into())));
assert!(!writer.room(1, 0, 1 << 20), "no room for an entry past the limit");
}
/// A thumbnail is kept only for the file it shows.
#[test]
fn a_thumbnail_is_keyed_to_its_file() {
let picture = Thumbnail { format: "app-thumbnail/1", app: "1.0", size: [2, 1], max_bytes: 1024 };
let mut png = b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR".to_vec();
png.extend_from_slice(&2u32.to_be_bytes());
png.extend_from_slice(&1u32.to_be_bytes());
assert!(picture.valid(&png));
let mut writer = Writer::new(LIMITS);
writer.part("document.json", b"{}", false).unwrap();
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
assert_eq!(picture.read(&mut reader, 64).unwrap(), png);
let mut writer = Writer::new(LIMITS);
writer.part("document.json", b"{\"changed\":1}", false).unwrap();
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
assert!(picture.read(&mut reader, 64).is_err(), "stale");
}
/// A refused part is not remembered as written: once the budget is raised it is written.
#[test]
fn a_refused_part_can_be_written_later() {
static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(4);
let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::SeqCst), ..LIMITS };
let mut writer = Writer::new(limits);
assert_eq!(writer.part("a.json", b"12345678", false), Err(FormatError::Invalid("package too large".into())));
assert!(writer.blob(b"12345678", "bin", false).is_err());
BUDGET.store(1 << 20, std::sync::atomic::Ordering::SeqCst);
writer.part("a.json", b"12345678", false).unwrap();
writer.blob(b"12345678", "bin", false).unwrap();
let reader = Reader::new(Cursor::new(writer.finish().unwrap()), limits).unwrap();
assert_eq!(reader.part_names().len(), 2, "both parts were written on the retry");
}
/// A cancelled save writes no more parts, whether the part is small or written in pieces.
#[test]
fn cancellation_stops_the_save() {
let cancelled = Limits { cancelled: Some(|| true), ..LIMITS };
let mut writer = Writer::new(cancelled);
assert_eq!(writer.part("a.json", b"a", false), Err(FormatError::Cancelled));
// cancelled after the first piece of a long part: checks before the part, then per piece
static CHECKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
let midway = Limits {
cancelled: Some(|| CHECKS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) >= 2),
max_expanded: || 1 << 22,
part_limit: |_| 1 << 22,
..LIMITS
};
let mut writer = Writer::new(midway);
assert_eq!(writer.part("big.big", &vec![0; PIECE + 1], true), Err(FormatError::Cancelled));
assert_eq!(CHECKS.load(std::sync::atomic::Ordering::SeqCst), 3);
let going = Limits { cancelled: Some(|| false), max_expanded: || 1 << 22, part_limit: |_| 1 << 22, ..LIMITS };
let mut writer = Writer::new(going);
writer.part("big.big", &vec![7; PIECE + 1], true).unwrap();
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), going).unwrap();
assert_eq!(reader.limited_part("big.big", 1 << 22).unwrap().len(), PIECE + 1);
}
/// Room counts what is already written, and the reader keeps every part's configured limit.
#[test]
fn room_counts_written_parts_and_reads_keep_part_limits() {
let mut writer = Writer::new(LIMITS);
writer.part("large.big", &[0; 500 * 1024], false).unwrap();
assert!(!writer.room(1, 200 * 1024, 600 * 1024));
assert!(writer.room(1, 1024, 1 << 20));
// what is written already is past the budget: not even an empty part fits
assert!(!writer.room(1, 0, 400 * 1024));
assert_eq!(writer.space(1, 400 * 1024), 0);
let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS };
let mut writer = Writer::new(wide);
writer.part("document.json", &[b' '; 2048], false).unwrap();
let bytes = writer.finish().unwrap();
assert!(Reader::new(Cursor::new(bytes.clone()), LIMITS).is_err(), "declared past its configured limit");
let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap();
assert!(reader.limited_part("document.json", 4096).is_ok());
}
/// A thumbnail past a cache part's own limit is left out, never failing the save.
#[test]
fn a_thumbnail_past_its_part_limit_is_left_out() {
let picture = Thumbnail { format: "app-thumbnail/1", app: "1.0", size: [2, 1], max_bytes: 2048 };
let mut png = b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR".to_vec();
png.extend_from_slice(&2u32.to_be_bytes());
png.extend_from_slice(&1u32.to_be_bytes());
png.resize(1500, 0);
let mut writer = Writer::new(LIMITS);
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
let reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
assert!(!reader.has_part("cache/thumbnail.json") && !reader.has_part("cache/thumbnail.png"));
}
/// A source that grows after its length was approved is refused.
#[test]
fn a_source_that_grows_while_read_is_refused() {
struct Growing(Cursor<Vec<u8>>, bool);
impl Read for Growing {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
if !self.1 {
self.1 = true;
let end = self.0.get_ref().len();
self.0.get_mut().extend_from_slice(&[0; 16]);
self.0.set_position(self.0.position().min(end as u64));
}
self.0.read(buf)
}
}
impl Seek for Growing {
fn seek(&mut self, to: SeekFrom) -> std::io::Result<u64> {
self.0.seek(to)
}
}
let mut writer = Writer::new(LIMITS);
writer.part("document.json", b"{}", false).unwrap();
let bytes = writer.finish().unwrap();
assert!(Reader::new(Growing(Cursor::new(bytes), false), LIMITS).err().is_some_and(|e| e.to_string().contains("changed")));
}
/// Compressed parts use the configured Deflate level.
#[test]
fn compressed_parts_use_the_configured_level() {
// words in a varied order, so the levels compress them differently
let words = ["alpha ", "beta ", "gamma ", "delta ", "epsilon ", "zeta ", "eta ", "theta "];
let mut seed = 7u32;
let text: Vec<u8> = (0..150)
.flat_map(|_| {
seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345);
words[(seed >> 16) as usize % words.len()].bytes()
})
.take(1000)
.collect();
let size = |level: fn(&str, usize) -> i64| {
let mut writer = Writer::new(Limits { level, ..LIMITS });
writer.part("document.json", &text, true).unwrap();
writer.finish().unwrap().len()
};
assert_ne!(size(|_, _| 1), size(|_, _| 9), "the level is used");
assert_eq!(size(|_, _| 42), size(|_, _| 9), "past 9 is clamped");
assert_eq!(size(|_, _| -3), size(|_, _| 1), "below 1 is clamped");
}
/// An archive naming one part twice is refused.
#[test]
fn a_repeated_part_name_is_refused() {
let mut writer = Writer::new(LIMITS);
writer.part("document.json", b"{}", false).unwrap();
writer.part("documenX.json", b"[]", false).unwrap();
let mut bytes = writer.finish().unwrap();
let (from, to) = (b"documenX.json", b"document.json");
let mut at = 0;
while let Some(i) = bytes[at..].windows(from.len()).position(|w| w == from) {
bytes[at + i..at + i + to.len()].copy_from_slice(to);
at += i + to.len();
}
assert!(Reader::new(Cursor::new(bytes), LIMITS).is_err());
}
/// Cached documents and blobs keep the current part limits.
#[test]
fn cached_parts_keep_the_current_part_limits() {
let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS };
let mut writer = Writer::new(wide);
writer.part("document.json", &[b' '; 512], false).unwrap();
let blob = writer.blob(&[7; 512], "bin", false).unwrap();
let bytes = writer.finish().unwrap();
let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap();
let reference = serde_json::json!({"blob": blob});
assert!(reader.document(4096).is_ok());
assert!(reader.blob(&reference, "blob", "bin").is_ok());
reader.limits = Limits { part_limit: |_| 128, ..wide };
assert!(reader.document(4096).is_err());
assert!(reader.limited_blob(&reference, "blob", "bin", 4096).is_err());
}
/// A raised expanded budget applies to the open reader, and what was read still counts.
#[test]
fn a_raised_expanded_budget_applies_at_once() {
static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(600);
let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::Relaxed), part_limit: |_| 1 << 20, ..LIMITS };
let wide = Limits { max_expanded: || 1 << 20, ..limits };
let mut writer = Writer::new(wide);
writer.part("document.json", &[b' '; 256], false).unwrap();
writer.part("other.big", &[1; 512], false).unwrap();
let bytes = writer.finish().unwrap();
BUDGET.store(1 << 20, std::sync::atomic::Ordering::Relaxed);
let mut reader = Reader::new(Cursor::new(bytes), limits).unwrap();
BUDGET.store(600, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("document.json", 4096).is_ok());
assert!(reader.limited_part("other.big", 4096).is_err(), "256 read + 512 is past 600");
BUDGET.store(1024, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("other.big", 4096).is_ok(), "768 fits the raised 1024");
// a lowered budget does not forget what was read: 768 + 256 is past 1000 after a dip to 100
BUDGET.store(100, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("document.json", 4096).is_err());
BUDGET.store(1000, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("document.json", 4096).is_err(), "1024 is past 1000");
BUDGET.store(1024, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("document.json", 4096).is_ok());
}
/// An entry whose local header can't be read is refused at open, not skipped.
#[test]
fn an_unreadable_entry_is_refused() {
let mut writer = Writer::new(LIMITS);
writer.part("document.json", b"{}", false).unwrap();
writer.part("other.json", b"[]", false).unwrap();
let mut bytes = writer.finish().unwrap();
let second = bytes.windows(4).enumerate().filter(|(_, w)| *w == b"PK\x03\x04").map(|(i, _)| i).nth(1).unwrap();
bytes[second + 2] = 0;
assert!(Reader::new(Cursor::new(bytes), LIMITS).is_err());
}
+257
View File
@@ -0,0 +1,257 @@
//! Versioned file formats: what an app needs to read every file it ever wrote, and to open a file
//! a newer build of it wrote.
//!
//! A file starts with `"format": "<prefix>N"`. A [`Family`] is one kind of file (an app's designs,
//! its drawings): its tag, its own numbers, and how a file of an older number is brought up to
//! this build's, step by step (N → N+1 → … → current). The app's code only knows the current
//! format; all translation lives in the steps:
//! - the steps written out by hand for the first formats, one per format ([`Formats::hand`]);
//! - then one folder per format, `v<N>/`, one file per change in it ([`Part`]), each with a step
//! when files from before the change are rewritten as they open ([`Formats::folded`]);
//! - then the changes no release has a number for yet, in `pending/` ([`Formats::pending`]):
//! while there are any, the build writes the next number, with their names in the file's
//! `format_pending`, so a later build runs just the steps a file is missing. A release moves
//! them into the next `v<N>/`. [`build`] lists the folders for a build script.
//!
//! A file a newer build wrote says so by a number past this build's or a change name this build
//! doesn't have ([`Family::newer`]); the app then reads what it can and keeps the rest
//! ([`newer`]). A file is saved as JSON, or in a [`container`] with its binary payloads.
//!
//! Steps are JSON-level (the file's shape) and return the app's own error type; every error this
//! crate makes is a [`FormatError`], which the app's error type takes `From`.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
use serde_json::Value;
pub mod build;
pub mod container;
pub mod newer;
/// Why a file is refused: its text says what is wrong, as the user reads it.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum FormatError {
/// The file isn't one of the family's, or it is damaged.
Invalid(String),
/// The work was cancelled (a save the person stopped): nothing more was written.
Cancelled,
}
impl std::fmt::Display for FormatError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
FormatError::Invalid(why) => f.write_str(why),
FormatError::Cancelled => f.write_str("cancelled"),
}
}
}
impl std::error::Error for FormatError {}
/// One JSON-level upgrade step: rewrites a file in place, or says why it can't.
pub type Step<E> = fn(&mut Value) -> Result<(), E>;
/// One change to a family's file format: a file in its `v<N>/` or `pending/` folder.
pub struct Part<E> {
/// The file's name, which a file written while the change was pending lists.
pub name: &'static str,
/// How a file from before the change is rewritten as it opens, if it is.
pub step: Option<Step<E>>,
}
/// Another spelling of a change name that a file of one format may carry (a build that named the
/// change otherwise): read as the change it stands for, never written.
#[derive(Clone, Copy, Debug)]
pub struct Alias {
/// The format whose files may say it.
pub format: u32,
/// The spelling in the file.
pub name: &'static str,
/// The change it stands for.
pub known_as: &'static str,
}
/// A family's upgrade path: the steps written out by hand, the folded formats after them (each a
/// list of changes) and the pending changes, which make the next format.
pub struct Formats<'a, E> {
/// Entry `i` turns format `i + 1` into `i + 2`.
pub hand: &'a [Step<E>],
/// Each format after the hand-written ones, oldest first, and its changes.
pub folded: &'a [(u32, &'a [Part<E>])],
/// The changes no release has a number for yet.
pub pending: &'a [Part<E>],
}
/// A kind of file whose format is versioned: its own format tag, its own numbers and its own
/// changes. Reading a file's format, telling a file a newer build saved and the upgrade are the
/// same for every family.
pub struct Family<'a, E> {
/// The app, as messages name it.
pub app: &'static str,
/// What a file of the family is called in messages.
pub noun: &'static str,
/// The tag before the number in a file's `format`.
pub prefix: &'static str,
/// The tag files saved under an older name carry: read, never written.
pub legacy_prefix: Option<&'static str>,
/// The format of a file that names none, if such files exist.
pub untagged: Option<u32>,
/// Other spellings of change names that files may carry.
pub aliases: &'a [Alias],
/// The upgrade path.
pub formats: Formats<'a, E>,
}
/// How a file shows a newer build saved it ([`Family::newer`]).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Newer {
/// The number the file says.
pub format: u32,
/// The number this build reads up to.
pub reads_up_to: u32,
/// The changes the file names that this build doesn't have.
pub unknown_changes: Vec<String>,
}
impl<E> Formats<'_, E> {
/// The format this build writes: the last folded one, or the next while changes are pending.
pub const fn current(&self) -> u32 {
self.hand.len() as u32 + 1 + self.folded.len() as u32 + if self.pending.is_empty() { 0 } else { 1 }
}
/// The changes format `version` is made of, if it is made of changes.
fn parts(&self, version: u32) -> Option<&[Part<E>]> {
let after = version.checked_sub(self.hand.len() as u32 + 2)? as usize;
match self.folded.get(after) {
Some((_, parts)) => Some(parts),
None if after == self.folded.len() && !self.pending.is_empty() => Some(self.pending),
None => None,
}
}
/// Whether a change of this name is anywhere in this build's history.
fn knows_part_name(&self, name: &str) -> bool {
self.folded.iter().flat_map(|(_, parts)| *parts).chain(self.pending).any(|part| part.name == name)
}
}
impl<E> Family<'_, E> {
/// The format this build writes: the last folded one, or the next while changes are pending.
pub const fn current(&self) -> u32 {
self.formats.current()
}
/// What a file of the family is called in messages.
pub const fn noun(&self) -> &'static str {
self.noun
}
/// The names of the pending changes this build writes into a file (`format_pending`).
pub fn pending_names(&self) -> Vec<String> {
self.formats.pending.iter().map(|p| p.name.to_string()).collect()
}
/// The format tag of format `n`.
pub fn format_string(&self, n: u32) -> String {
format!("{}{n}", self.prefix)
}
/// The version in one of this family's format tags; `None` for any other tag, another
/// family's included.
pub fn version_of(&self, s: &str) -> Option<u32> {
// each tag form parses on its own: a legacy prefix may start with the primary one
// (`app/` and `app/design/`)
let parse = |prefix: &str| s.strip_prefix(prefix).and_then(|n| n.parse().ok());
parse(self.prefix).or_else(|| self.legacy_prefix.and_then(parse))
}
/// The change a name a file of format `version` lists stands for: its alias's, or itself.
/// An alias counts only while this build has the change it stands for, so recognising it
/// never grants support to another reader.
fn canonical_applied_name<'name>(&self, version: u32, name: &'name str) -> &'name str {
match self.aliases.iter().find(|a| a.format == version && a.name == name) {
Some(alias) if self.formats.knows_part_name(alias.known_as) => alias.known_as,
_ => name,
}
}
/// What says a file of format `version` with the changes `applied` came from a newer build:
/// a number past this build's, or a change this build doesn't have anywhere in its history
/// (a branch's pending change may have landed at a later number). A list beside a format that
/// isn't made of changes means nothing (a tool or a test set an older number on a newer file).
/// `None` for a file this build fully knows.
pub fn newer(&self, version: u32, applied: Option<&[String]>) -> Option<Newer> {
let current = self.current();
let listed = applied.filter(|_| version > current || self.formats.parts(version).is_some()).unwrap_or_default();
let unknown_changes: Vec<String> =
listed.iter().filter(|name| !self.formats.knows_part_name(self.canonical_applied_name(version, name))).cloned().collect();
(version > current || !unknown_changes.is_empty()).then_some(Newer { format: version, reads_up_to: current, unknown_changes })
}
}
impl<E: From<FormatError>> Family<'_, E> {
/// Read a file's format and take its `format_pending` out of its JSON, before anything else
/// in it is read: a file that isn't this family's is refused here. One a newer build saved is
/// not: [`Family::newer`] says so.
pub fn take_version(&self, v: &mut Value) -> Result<(u32, Option<Vec<String>>), E> {
let (app, noun) = (self.app, self.noun);
let not_ours = |what: String| E::from(FormatError::Invalid(format!("not a {app} {noun} ({what})")));
let Some(obj) = v.as_object() else { return Err(not_ours("expected a JSON object".into())) };
let version = match obj.get("format") {
None => self.untagged.ok_or_else(|| not_ours("no format".into()))?,
Some(Value::String(f)) => self.version_of(f).ok_or_else(|| not_ours(format!("format `{f}`")))?,
Some(_) => return Err(not_ours("format is not a string".into())),
};
let applied = take_names(v, "format_pending", noun).map_err(E::from)?;
if version == 0 {
return Err(not_ours("format 0".into()));
}
Ok((version, applied))
}
/// Bring a file of format `version` up to this build's; `applied` is its `format_pending`
/// (both from [`Family::take_version`]): the changes of its format it already has, when it
/// was written while they were pending. The others' steps run first, and already-applied
/// changes never run again at their later number. A file past this build's number has no
/// steps to run.
pub fn upgrade(&self, v: &mut Value, version: u32, applied: Option<&[String]>) -> Result<(), E> {
let formats = &self.formats;
if version == 0 {
return Err(E::from(FormatError::Invalid(format!("not a {} {} (format 0)", self.app, self.noun))));
}
// A list beside a format that isn't made of changes is ignored, like any field an upgrade
// drops.
let applied = applied.filter(|_| formats.parts(version).is_some());
let already_applied =
|part: &Part<E>| applied.is_some_and(|names| names.iter().any(|name| self.canonical_applied_name(version, name) == part.name));
if applied.is_some() {
for part in formats.parts(version).unwrap_or_default().iter().filter(|part| !already_applied(part)) {
part.step.map_or(Ok(()), |step| step(v))?;
}
}
for from in version..self.current() {
match formats.hand.get(from as usize - 1) {
Some(step) => step(v)?,
None => {
for part in formats.parts(from + 1).unwrap_or_default().iter().filter(|part| !already_applied(part)) {
part.step.map_or(Ok(()), |step| step(v))?;
}
}
}
}
Ok(())
}
}
/// Take a short list of names out of a file's JSON: its `format_pending` (the pending changes it
/// has).
fn take_names(v: &mut Value, key: &str, noun: &str) -> Result<Option<Vec<String>>, FormatError> {
let Some(list) = v.as_object_mut().and_then(|o| o.remove(key)) else { return Ok(None) };
let invalid = || FormatError::Invalid(format!("damaged {noun}: `{key}` is not a short list of names"));
let list = list.as_array().filter(|l| l.len() <= 1000).ok_or_else(invalid)?;
list.iter().map(|n| n.as_str().filter(|n| n.len() <= 128).map(str::to_string).ok_or_else(invalid)).collect::<Result<_, _>>().map(Some)
}
#[cfg(test)]
#[path = "lib_tests.rs"]
mod tests;
+142
View File
@@ -0,0 +1,142 @@
use super::*;
use serde_json::json;
/// The app's error, as an app has one.
#[derive(Debug, PartialEq)]
struct AppError(String);
impl From<FormatError> for AppError {
fn from(e: FormatError) -> Self {
AppError(e.to_string())
}
}
/// Each step appends its name to the file's `log`, so the order steps ran in can be read back.
fn log(v: &mut Value, step: &str) {
if let Some(list) = v.get_mut("log").and_then(Value::as_array_mut) {
list.push(json!(step));
}
}
const HAND: [Step<AppError>; 2] = [
|v| {
log(v, "1to2");
Ok(())
},
|v| {
log(v, "2to3");
Ok(())
},
];
const V4: [Part<AppError>; 2] = [
Part {
name: "a",
step: Some(|v| {
log(v, "a");
Ok(())
}),
},
Part { name: "b", step: None },
];
const V5: [Part<AppError>; 1] = [Part {
name: "c",
step: Some(|v| {
log(v, "c");
Ok(())
}),
}];
const FOLDED: [(u32, &[Part<AppError>]); 2] = [(4, &V4), (5, &V5)];
const PENDING: [Part<AppError>; 2] = [
Part {
name: "d",
step: Some(|v| {
log(v, "d");
Ok(())
}),
},
Part {
name: "e",
step: Some(|v| {
log(v, "e");
Ok(())
}),
},
];
const ALIASES: [Alias; 1] = [Alias { format: 5, name: "old-c", known_as: "c" }];
/// Format 1 untagged; hand steps to 3; folded 4 and 5; pending changes make 6.
const FAMILY: Family<'static, AppError> = Family {
app: "App",
noun: "design",
prefix: "app/",
legacy_prefix: Some("old/"),
untagged: Some(1),
aliases: &ALIASES,
formats: Formats { hand: &HAND, folded: &FOLDED, pending: &PENDING },
};
fn steps_run(version: u32, applied: Option<&[String]>) -> Vec<String> {
let mut v = json!({"log": []});
FAMILY.upgrade(&mut v, version, applied).unwrap();
v["log"].as_array().unwrap().iter().map(|s| s.as_str().unwrap().to_string()).collect()
}
#[test]
fn tags_and_numbers() {
assert_eq!(FAMILY.current(), 6);
assert_eq!(FAMILY.format_string(6), "app/6");
assert_eq!(FAMILY.version_of("app/4"), Some(4));
assert_eq!(FAMILY.version_of("old/3"), Some(3), "the legacy tag is read");
assert_eq!(FAMILY.version_of("other/3"), None);
assert_eq!(FAMILY.pending_names(), vec!["d".to_string(), "e".to_string()]);
}
#[test]
fn a_file_is_taken_apart_or_refused_with_the_apps_words() {
let mut v = json!({"format": "app/6", "format_pending": ["d"], "x": 1});
assert_eq!(FAMILY.take_version(&mut v), Ok((6, Some(vec!["d".to_string()]))));
assert!(v.get("format_pending").is_none(), "taken out");
assert_eq!(FAMILY.take_version(&mut json!({})), Ok((1, None)), "untagged");
let refused = |mut v: Value| FAMILY.take_version(&mut v).unwrap_err().0;
assert_eq!(refused(json!([])), "not a App design (expected a JSON object)");
assert_eq!(refused(json!({"format": "other/2"})), "not a App design (format `other/2`)");
assert_eq!(refused(json!({"format": 2})), "not a App design (format is not a string)");
assert_eq!(refused(json!({"format": "app/0"})), "not a App design (format 0)");
assert_eq!(refused(json!({"format": "app/2", "format_pending": [1]})), "damaged design: `format_pending` is not a short list of names");
}
#[test]
fn every_step_runs_once_in_order() {
assert_eq!(steps_run(1, None), ["1to2", "2to3", "a", "c", "d", "e"]);
assert_eq!(steps_run(4, None), ["c", "d", "e"]);
assert_eq!(steps_run(6, None), Vec::<String>::new());
// A file written while `d` was pending has it: only `e` runs.
assert_eq!(steps_run(6, Some(&["d".to_string()])), ["e"]);
// A file of format 5 written while `c` was still pending, without it: `c` runs first.
assert_eq!(steps_run(5, Some(&[])), ["c", "d", "e"]);
// Under its other spelling `c` counts as had.
assert_eq!(steps_run(5, Some(&["old-c".to_string()])), ["d", "e"]);
// A list beside a hand-written format means nothing.
assert_eq!(steps_run(2, Some(&["d".to_string()])), ["2to3", "a", "c", "d", "e"]);
let mut v = json!({});
assert_eq!(FAMILY.upgrade(&mut v, 0, None), Err(AppError("not a App design (format 0)".into())));
}
#[test]
fn a_newer_file_is_told_by_its_number_or_an_unknown_change() {
assert_eq!(FAMILY.newer(6, None), None);
assert_eq!(FAMILY.newer(6, Some(&["d".to_string()])), None);
assert_eq!(FAMILY.newer(7, None), Some(Newer { format: 7, reads_up_to: 6, unknown_changes: Vec::new() }));
assert_eq!(FAMILY.newer(6, Some(&["z".to_string()])), Some(Newer { format: 6, reads_up_to: 6, unknown_changes: vec!["z".to_string()] }));
assert_eq!(FAMILY.newer(5, Some(&["old-c".to_string()])), None, "an alias of a known change");
assert_eq!(FAMILY.newer(4, Some(&["old-c".to_string()])), Some(Newer { format: 4, reads_up_to: 6, unknown_changes: vec!["old-c".to_string()] }));
assert_eq!(FAMILY.newer(2, Some(&["z".to_string()])), None, "a list beside a hand-written format means nothing");
}
#[test]
fn a_legacy_prefix_that_extends_the_primary_one_is_read() {
let nested = Family { prefix: "app/", legacy_prefix: Some("app/design/"), ..FAMILY };
assert_eq!(nested.version_of("app/design/3"), Some(3));
assert_eq!(nested.version_of("app/4"), Some(4));
assert_eq!(nested.version_of("app/design/x"), None);
}
+470
View File
@@ -0,0 +1,470 @@
//! A file a newer build saved: it opens, with a warning, and saving it loses nothing.
//!
//! A newer format number, or a change the file names that this build doesn't have, says a newer
//! build saved the file ([`crate::Family::newer`]). It may use nothing this build lacks, so the app
//! reads it as usual. When part of it can't be read, that part stays out of the way and the rest
//! opens: the app reads the elements of its lists it can ([`NewerFile::keep_element`]), then any
//! other top-level section it can't read is left out whole ([`leave_out_unreadable`],
//! [`read_around`]), and a field it doesn't know, which typed reading would drop, is noticed by
//! comparing the file with what was read ([`NewerFile::compare`]).
//!
//! All of these are named in [`NewerFile::not_understood`] and go back into the file when it is
//! saved ([`NewerFile::restore_into`]), as they came. What can't be put back because the file was
//! changed around it is what `restore_into` returns.
use std::collections::{BTreeMap, HashMap};
use std::sync::Arc;
use serde_json::{Map, Value, json};
use crate::Newer;
/// The most sections left out one at a time to find the one a file can't be read with.
const MAX_SECTION_TRIES: usize = 64;
/// What the file says and holds that this build doesn't read.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct NewerFile {
/// The app, as the notice names it.
pub app: &'static str,
/// The number the file says.
pub format: u32,
/// The number this build reads up to.
pub reads_up_to: u32,
/// The changes the file names that this build doesn't have.
pub unknown_changes: Vec<String>,
/// What this build couldn't read, by name.
pub not_understood: Vec<String>,
/// Fields known records lost in typed reading.
kept: Option<Kept>,
/// Top-level sections left out whole.
sections: Vec<Section>,
/// Elements left out of a top-level list: its key and the element.
elements: Vec<(String, Value)>,
/// The parts of a container this build has no use for: name and bytes. A blob only one of the
/// kept pieces refers to is one of them.
pub parts: Vec<(String, Arc<[u8]>)>,
/// The file's `container.json`, when this build doesn't know what it says.
pub container: Option<Value>,
}
#[derive(Clone, Debug, PartialEq)]
struct Section {
key: String,
value: Value,
/// This build's record of the section, without the file's: while it is unchanged, the file's
/// goes back.
empty: Option<Value>,
}
/// The fields of a record that typed reading dropped.
#[derive(Clone, Debug, PartialEq)]
enum Kept {
Field(Value),
Fields(BTreeMap<String, Kept>),
Items(Vec<(Anchor, Kept)>),
}
/// Which element of a list some kept fields belong to.
#[derive(Clone, Debug, PartialEq)]
enum Anchor {
Id(u64),
Name(String),
/// Its place, good only while the list keeps its length.
Index {
at: usize,
of: usize,
},
}
impl NewerFile {
/// A file of `app` that `newer` says a newer build saved, nothing left out of it yet.
pub fn new(app: &'static str, newer: &Newer) -> NewerFile {
NewerFile {
app,
format: newer.format,
reads_up_to: newer.reads_up_to,
unknown_changes: newer.unknown_changes.clone(),
..NewerFile::default()
}
}
/// What to tell the user, whatever else is said.
pub fn notice(&self) -> String {
let (app, reads, format) = (self.app, self.reads_up_to, self.format);
match self.unknown_changes.first() {
Some(change) if format <= reads => {
format!("Saved by a newer {app} (format {format} with `{change}`, which this version doesn't know). If something's missing, update.")
}
_ => format!("Saved by a newer {app} (format {format}, this version reads up to {reads}). If something's missing, update."),
}
}
/// Every line to report: the notice, then what couldn't be read, by name.
pub fn warnings(&self) -> Vec<String> {
let unread = self.not_understood.iter().map(|what| format!("Not read: {what}. Saving keeps it."));
std::iter::once(self.notice()).chain(unread).collect()
}
/// What a reply says about it.
pub fn report(&self) -> Value {
json!({
"format": self.format,
"reads_up_to": self.reads_up_to,
"unknown_changes": self.unknown_changes,
"not_understood": self.not_understood,
"notice": self.notice(),
})
}
/// Whether the next save needs a container to keep what the file holds: plain JSON has no room
/// for parts beside the file's own, nor for the container's own `container.json`.
pub fn needs_container(&self) -> bool {
!self.parts.is_empty() || self.container.is_some()
}
/// Whether anything kept back names this text (a blob's name in a container).
pub fn mentions(&self, text: &str) -> bool {
fn kept(k: &Kept, text: &str) -> bool {
match k {
Kept::Field(v) => value_mentions(v, text),
Kept::Fields(m) => m.values().any(|x| kept(x, text)),
Kept::Items(l) => l.iter().any(|(_, x)| kept(x, text)),
}
}
self.kept.as_ref().is_some_and(|k| kept(k, text))
|| self.sections.iter().any(|s| value_mentions(&s.value, text))
|| self.elements.iter().any(|(_, v)| value_mentions(v, text))
}
/// Say that a section the reader couldn't finish reading is left out. `empty` is what the
/// app writes for the section while it holds nothing, if anything.
pub fn keep_section(&mut self, key: &str, value: Value, empty: Option<Value>, why: &str) {
self.not_understood.push(format!("section `{key}` ({why})"));
self.sections.push(Section { key: key.into(), value, empty });
}
/// Say that an element of the top-level list `key` is left out, as `note` names it; it goes
/// back at the end of the list when the file is saved.
pub fn keep_element(&mut self, key: &str, element: Value, note: String) {
self.not_understood.push(note);
self.elements.push((key.into(), element));
}
/// Compare the file as it reads (`file`, after the same upgrade as what was read) with what was
/// read of it (`known`, its typed record written out): what `known` lacks is kept and named.
/// An object whose `type` is `placeholder_tag` (an element the app kept whole) is not
/// compared; nor are these top-level fields: those in `container_only` (what a container adds
/// beside the file's own), `format` and `format_pending`, and the sections kept whole (they go
/// back whole).
pub fn compare(&mut self, file: &Value, known: &Value, placeholder_tag: &str, container_only: &[&str]) {
let mut paths = Vec::new();
let mut top_only: Vec<&str> = container_only.to_vec();
top_only.extend(["format", "format_pending"]);
top_only.extend(self.sections.iter().map(|s| s.key.as_str()));
// a list typed writing leaves out holds only elements kept on their own: not kept twice
top_only.extend(self.elements.iter().map(|(key, _)| key.as_str()).filter(|key| known.get(*key).is_none()));
let kept = vanished(file, known, "", &mut paths, placeholder_tag, &top_only);
self.kept = kept;
self.not_understood.extend(paths);
for section in &mut self.sections {
section.empty = known.get(&section.key).cloned();
}
}
/// Write back what was kept of the file into `value`, the file as this build writes it.
/// Returns what couldn't go back, because the file was changed around it.
pub fn restore_into(&self, value: &mut Value) -> Vec<String> {
let mut lost = Vec::new();
if let Some(kept) = &self.kept {
apply(kept, value, "", &mut lost);
}
if let Some(top) = value.as_object_mut() {
for section in &self.sections {
if top.get(&section.key) == section.empty.as_ref() {
top.insert(section.key.clone(), section.value.clone());
} else {
lost.push(format!("section `{}`, replaced by what was edited here", section.key));
}
}
// each list's records indexed once, by the identity records are matched by
let mut indexes: HashMap<&str, Positions> = HashMap::new();
for (key, element) in &self.elements {
let name = element.get("name");
let anchor = anchor_of(element);
match top.entry(key.clone()).or_insert_with(|| json!([])) {
Value::Array(list) => {
let index = indexes.entry(key.as_str()).or_insert_with(|| Positions::of(list));
if anchor.as_ref().is_some_and(|a| index.find(list.len(), a).is_some()) {
lost.push(format!(
"`{key}` element {}, replaced by what was made here",
name.and_then(Value::as_str).unwrap_or("unnamed")
));
} else {
index.add(element, list.len());
list.push(element.clone());
}
}
_ => lost.push(format!("`{key}` element {}, replaced by what was made here", name.and_then(Value::as_str).unwrap_or("unnamed"))),
}
}
}
lost
}
}
/// Whether a string anywhere in `v` is `text`; iterative, so no nesting depth overflows the stack.
fn value_mentions(v: &Value, text: &str) -> bool {
let mut todo = vec![v];
while let Some(v) = todo.pop() {
match v {
Value::String(s) if s == text => return true,
Value::Array(a) => todo.extend(a),
Value::Object(o) => todo.extend(o.values()),
_ => {}
}
}
false
}
/// The first line of a reader's error, without the list of every kind it knows, and without the
/// `"<noun> file: "` its reader starts it with.
pub fn short<'a>(why: &'a str, noun: &str) -> &'a str {
let why = why.split(", expected").next().unwrap_or(why);
why.strip_prefix(&format!("{noun} file: ")).unwrap_or(why)
}
/// Leave a section out and keep it. A section the file needs is left empty.
fn move_out(top: &mut Map<String, Value>, key: &str, why: &str, noun: &str, file: &mut NewerFile) {
let Some(value) = top.remove(key) else { return };
file.not_understood.push(format!("section `{key}` ({})", short(why, noun)));
file.sections.push(Section { key: key.into(), value, empty: None });
}
/// Every top-level section of `top` that doesn't read by itself (`piece`, the app's reading of a
/// file holding just that section) is left out and kept in `file`. `noun` is what a file of the
/// app's is called, which its reader's errors start with.
pub fn leave_out_unreadable<E: std::fmt::Display>(
top: &mut Map<String, Value>,
file: &mut NewerFile,
noun: &str,
piece: impl Fn(&str, &Value) -> Result<(), E>,
) {
let keys: Vec<String> = top.keys().cloned().collect();
for key in &keys {
let value = top.get(key).cloned().unwrap_or_default();
if let Err(why) = piece(key, &value) {
move_out(top, key, &why.to_string(), noun, file);
}
}
}
/// Read `top` with `attempt`, leaving out the sections it can't be read with: if it doesn't read,
/// each section but the `core` ones (which the file can't be read without) is tried left out alone,
/// and as a last resort every one of them is. What is left out is kept in `file`. The error is the
/// first attempt's, when even the core alone doesn't read.
pub fn read_around<D, E: std::fmt::Display>(
top: &mut Map<String, Value>,
file: &mut NewerFile,
core: &[&str],
noun: &str,
attempt: impl Fn(&Map<String, Value>) -> Result<D, E>,
) -> Result<D, String> {
let first = match attempt(top) {
Ok(doc) => return Ok(doc),
Err(why) => why.to_string(),
};
// Sections that are fine by themselves but not together with the rest.
let keys: Vec<String> = top.keys().filter(|k| !core.contains(&k.as_str())).cloned().collect();
// Each try reads the whole file: a file of many sections goes straight to the last resort.
for key in keys.iter().filter(|_| keys.len() <= MAX_SECTION_TRIES) {
let mut without = top.clone();
without.remove(key);
if let Ok(doc) = attempt(&without) {
move_out(top, key, &first, noun, file);
return Ok(doc);
}
}
let mut bare = top.clone();
bare.retain(|key, _| core.contains(&key.as_str()));
match attempt(&bare) {
Ok(doc) => {
for key in &keys {
move_out(top, key, &first, noun, file);
}
Ok(doc)
}
Err(_) => Err(first),
}
}
/// A value a writer leaves out without losing anything: not worth telling anyone about.
pub fn looks_default(v: &Value) -> bool {
match v {
Value::Null => true,
Value::Bool(b) => !b,
Value::Number(n) => n.as_f64() == Some(0.0),
Value::String(s) => s.is_empty(),
Value::Array(a) => a.is_empty(),
Value::Object(o) => o.is_empty(),
}
}
/// What `raw` has that `known`, what was read of it, doesn't. `paths` gets those fields' names.
/// `top_only` are top-level keys left out of the comparison.
fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec<String>, placeholder_tag: &str, top_only: &[&str]) -> Option<Kept> {
match (raw, known) {
(Value::Object(raw), Value::Object(known)) => {
if known.get("type").and_then(Value::as_str) == Some(placeholder_tag) {
return None;
}
let mut kept = BTreeMap::new();
for (key, value) in raw {
if path.is_empty() && top_only.contains(&key.as_str()) {
continue;
}
let inner = if path.is_empty() { key.clone() } else { format!("{path}.{key}") };
match known.get(key) {
None => {
if !looks_default(value) {
paths.push(format!("field `{inner}`"));
}
kept.insert(key.clone(), Kept::Field(value.clone()));
}
Some(known) => {
if let Some(k) = vanished(value, known, &inner, paths, placeholder_tag, top_only) {
kept.insert(key.clone(), k);
}
}
}
}
(!kept.is_empty()).then_some(Kept::Fields(kept))
}
(Value::Array(raw), Value::Array(known)) => {
let mut items = Vec::new();
let positions = Positions::of(known);
for (at, element) in raw.iter().enumerate() {
// an identity typed reading kept: an id it dropped falls back to the name
let anchor = surviving_anchor(element, &positions).unwrap_or(Anchor::Index { at, of: raw.len() });
let Some(partner) = positions.find(known.len(), &anchor).and_then(|i| known.get(i)) else { continue };
let label = match (element.get("name").and_then(Value::as_str), &anchor) {
(Some(name), _) => format!("{path}[{name}]"),
(None, Anchor::Id(id)) => format!("{path}[{id}]"),
_ => format!("{path}[{at}]"),
};
if let Some(k) = vanished(element, partner, &label, paths, placeholder_tag, top_only) {
items.push((anchor, k));
}
}
(!items.is_empty()).then_some(Kept::Items(items))
}
_ => None,
}
}
fn anchor_of(element: &Value) -> Option<Anchor> {
if let Some(id) = element.get("id").and_then(Value::as_u64) {
return Some(Anchor::Id(id));
}
element.get("name").and_then(Value::as_str).map(|name| Anchor::Name(name.to_string()))
}
/// The identity of `element` that `known`'s records still carry: its id, or else its name. When
/// typed reading kept neither, there is none, and the record is matched by its place.
fn surviving_anchor(element: &Value, known: &Positions) -> Option<Anchor> {
let id = element.get("id").and_then(Value::as_u64).map(Anchor::Id);
let name = element.get("name").and_then(Value::as_str).map(|n| Anchor::Name(n.to_string()));
[id, name].into_iter().flatten().find(|anchor| known.find(usize::MAX, anchor).is_some())
}
/// Where each id and name first appears in a list, so matching records is not a scan per record.
struct Positions {
ids: HashMap<u64, usize>,
names: HashMap<String, usize>,
}
impl Positions {
fn of(list: &[Value]) -> Positions {
let (mut ids, mut names) = (HashMap::new(), HashMap::new());
for (i, x) in list.iter().enumerate() {
if let Some(id) = x.get("id").and_then(Value::as_u64) {
ids.entry(id).or_insert(i);
}
if let Some(name) = x.get("name").and_then(Value::as_str) {
names.entry(name.to_string()).or_insert(i);
}
}
Positions { ids, names }
}
/// Count `x`, now at `i`, in.
fn add(&mut self, x: &Value, i: usize) {
if let Some(id) = x.get("id").and_then(Value::as_u64) {
self.ids.entry(id).or_insert(i);
}
if let Some(name) = x.get("name").and_then(Value::as_str) {
self.names.entry(name.to_string()).or_insert(i);
}
}
/// The position of the record `anchor` names in a list of `len` records (an index anchor
/// only while the list keeps its length).
fn find(&self, len: usize, anchor: &Anchor) -> Option<usize> {
match anchor {
Anchor::Id(id) => self.ids.get(id).copied(),
Anchor::Name(name) => self.names.get(name).copied(),
Anchor::Index { at, of } => (len == *of && at < of).then_some(*at),
}
}
}
/// Put the kept fields back. An element that is gone was deleted here, its fields with it; one
/// whose place can't be told any more, and a field whose record is gone, go into `lost`.
fn apply(kept: &Kept, value: &mut Value, path: &str, lost: &mut Vec<String>) {
match kept {
Kept::Field(_) => {}
Kept::Fields(fields) => {
let Some(object) = value.as_object_mut() else {
lost.push(format!("field `{path}`"));
return;
};
for (key, sub) in fields {
let inner = if path.is_empty() { key.clone() } else { format!("{path}.{key}") };
match (sub, object.get_mut(key)) {
(Kept::Field(v), None) => {
object.insert(key.clone(), v.clone());
}
// the edited value stays; a retained one it replaced is reported
(Kept::Field(v), Some(existing)) => {
if existing != v {
lost.push(format!("field `{inner}`"));
}
}
(sub, Some(inner_value)) => apply(sub, inner_value, &inner, lost),
// The list it was in is empty and not written: its elements were deleted.
(Kept::Items(_), None) => {}
(_, None) => lost.push(format!("field `{inner}`")),
}
}
}
Kept::Items(items) => {
let Some(list) = value.as_array_mut() else {
lost.push(format!("field `{path}`"));
return;
};
let positions = Positions::of(list);
let len = list.len();
for (anchor, sub) in items {
match (positions.find(len, anchor).and_then(|i| list.get_mut(i)), anchor) {
(Some(element), _) => apply(sub, element, path, lost),
(None, Anchor::Index { at, .. }) => lost.push(format!("element {at} of `{path}`")),
(None, _) => {}
}
}
}
}
}
#[cfg(test)]
#[path = "newer_tests.rs"]
mod tests;
+218
View File
@@ -0,0 +1,218 @@
use super::*;
fn newer() -> Newer {
Newer { format: 9, reads_up_to: 8, unknown_changes: Vec::new() }
}
/// What typed reading dropped is named and goes back on saving, where it was.
#[test]
fn dropped_fields_are_named_and_put_back() {
let file = json!({"name": "A", "format": "app/9", "items": [{"id": 1, "size": 2, "colour": "red"}], "extra": {"x": 1}});
let known = json!({"name": "A", "items": [{"id": 1, "size": 2}]});
let mut said = NewerFile::new("App", &newer());
said.compare(&file, &known, "placeholder", &[]);
assert_eq!(said.not_understood, ["field `extra`", "field `items[1].colour`"]);
let mut saved = json!({"name": "A", "items": [{"id": 1, "size": 3}]});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"name": "A", "items": [{"id": 1, "size": 3, "colour": "red"}], "extra": {"x": 1}}));
assert_eq!(said.notice(), "Saved by a newer App (format 9, this version reads up to 8). If something's missing, update.");
}
/// A section that can't be read is left out, kept, and goes back while it is unchanged here.
#[test]
fn unreadable_sections_are_left_out_and_kept() {
let mut top = match json!({"name": "A", "good": 1, "bad": 2}) {
Value::Object(m) => m,
_ => unreachable!(),
};
let mut said = NewerFile::new("App", &newer());
leave_out_unreadable(&mut top, &mut said, "design", |key: &str, _: &Value| if key == "bad" { Err("design file: no") } else { Ok(()) });
assert!(!top.contains_key("bad"));
assert_eq!(said.not_understood, ["section `bad` (no)"]);
let read =
read_around(
&mut top,
&mut said,
&["name"],
"design",
|top: &Map<String, Value>| {
if top.contains_key("good") { Err("design file: clash") } else { Ok(top.len()) }
},
);
assert_eq!(read, Ok(1));
assert_eq!(said.not_understood, ["section `bad` (no)", "section `good` (clash)"]);
let mut saved = json!({"name": "A"});
said.compare(&saved.clone(), &saved.clone(), "placeholder", &[]);
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"name": "A", "good": 1, "bad": 2}));
}
/// Notices, warnings and reports name what was left out, and kept values can be searched.
#[test]
fn warnings_report_and_mentions_cover_everything_kept() {
let mut said = NewerFile::new("App", &Newer { format: 8, reads_up_to: 8, unknown_changes: vec!["curves".into()] });
assert_eq!(said.notice(), "Saved by a newer App (format 8 with `curves`, which this version doesn't know). If something's missing, update.");
assert!(!said.needs_container());
said.keep_section("plugins", json!({"tool": "ruler"}), Some(json!({})), "unknown");
said.keep_element("shapes", json!({"name": "star", "points": ["tip"]}), "shape `star`".into());
let file = json!({"shapes": [{"name": "a", "tags": ["deep"], "more": {"x": "nested"}}], "n": 1});
said.compare(&file, &json!({"shapes": [{"name": "a"}], "n": 1}), "placeholder", &[]);
assert_eq!(said.warnings()[0], said.notice());
assert!(said.warnings().contains(&"Not read: section `plugins` (unknown). Saving keeps it.".to_string()));
let report = said.report();
assert_eq!(report["format"], 8);
assert_eq!(report["unknown_changes"], json!(["curves"]));
assert_eq!(report["notice"], json!(said.notice()));
for text in ["ruler", "tip", "deep", "nested"] {
assert!(said.mentions(text), "{text}");
}
assert!(!said.mentions("absent"));
said.parts.push(("blob".into(), Arc::from(&b"x"[..])));
assert!(said.needs_container());
}
/// A kept section or element that was replaced here is reported lost, not written over.
#[test]
fn replaced_sections_and_elements_are_reported_lost() {
let mut said = NewerFile::new("App", &newer());
said.keep_section("plugins", json!({"tool": "ruler"}), None, "unknown");
said.keep_element("shapes", json!({"name": "star"}), "shape `star`".into());
said.keep_element("shapes", json!({"points": 3}), "an unnamed shape".into());
let mut saved = json!({"plugins": {"edited": true}, "shapes": [{"name": "star"}]});
let lost = said.restore_into(&mut saved);
assert_eq!(lost, ["section `plugins`, replaced by what was edited here", "`shapes` element star, replaced by what was made here"]);
assert_eq!(saved["shapes"], json!([{"name": "star"}, {"points": 3}]));
let mut fresh = json!({});
said.sections.clear();
assert!(said.restore_into(&mut fresh).is_empty());
assert_eq!(fresh["shapes"], json!([{"name": "star"}, {"points": 3}]));
}
/// Retained container metadata alone needs a container to survive the save.
#[test]
fn container_metadata_alone_needs_a_container() {
let mut said = NewerFile::new("App", &newer());
assert!(!said.needs_container());
said.container = Some(json!({"layout": 2}));
assert!(said.needs_container());
}
/// An edit that replaces a retained field keeps the edit and reports the replaced value.
#[test]
fn a_replaced_retained_field_is_reported() {
let mut said = NewerFile::new("App", &newer());
said.compare(&json!({"extra": 1}), &json!({}), "placeholder", &[]);
let mut edited = json!({"extra": 2});
assert_eq!(said.restore_into(&mut edited), ["field `extra`"]);
assert_eq!(edited, json!({"extra": 2}));
let mut same = json!({"extra": 1});
assert!(said.restore_into(&mut same).is_empty());
}
/// Records are matched by id and by name across long lists, and an index anchor only while the
/// list keeps its length.
#[test]
fn records_match_by_id_name_or_unchanged_index() {
let records: Vec<Value> = (0..2000).map(|i| json!({"id": i, "extra": i})).collect();
let known: Vec<Value> = (0..2000).map(|i| json!({"id": i})).collect();
let mut said = NewerFile::new("App", &newer());
said.compare(&json!({"items": records}), &json!({"items": known}), "placeholder", &[]);
let mut saved = json!({"items": (0..2000).rev().map(|i| json!({"id": i})).collect::<Vec<_>>()});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved["items"][0], json!({"id": 1999, "extra": 1999}));
let mut by_name = NewerFile::new("App", &newer());
by_name.compare(&json!({"l": [{"name": "a", "x": 1}, [1, 2]]}), &json!({"l": [{"name": "a"}, [1]]}), "placeholder", &[]);
let mut renamed = json!({"l": [{"name": "a"}]});
by_name.restore_into(&mut renamed);
assert_eq!(renamed["l"][0], json!({"name": "a", "x": 1}));
}
/// A section kept whole goes back whole, even when this build's record of it has fields.
#[test]
fn a_section_kept_whole_is_not_compared_field_by_field() {
let mut said = NewerFile::new("App", &newer());
said.keep_section("plugins", json!({"mode": "future", "payload": 1}), None, "unreadable");
let file = json!({"name": "A", "plugins": {"mode": "future", "payload": 1}});
said.compare(&file, &json!({"name": "A", "plugins": {"mode": "none"}}), "placeholder", &[]);
let mut saved = json!({"name": "A", "plugins": {"mode": "none"}});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, file);
}
/// A kept element goes back only while no record has its id or name.
#[test]
fn a_kept_element_keeps_record_identity() {
let mut said = NewerFile::new("App", &newer());
said.keep_element("items", json!({"id": 7, "kind": "future"}), "element 7".into());
said.keep_element("items", json!({"kind": "anonymous"}), "unnamed".into());
let mut clash = json!({"items": [{"id": 7, "kind": "made here"}]});
assert_eq!(said.restore_into(&mut clash).len(), 1);
assert_eq!(clash, json!({"items": [{"id": 7, "kind": "made here"}, {"kind": "anonymous"}]}));
let mut free = json!({"items": []});
assert!(said.restore_into(&mut free).is_empty());
assert_eq!(free, json!({"items": [{"id": 7, "kind": "future"}, {"kind": "anonymous"}]}));
}
/// Container-only fields are left out at the top level only.
#[test]
fn container_only_fields_are_top_level() {
let mut said = NewerFile::new("App", &newer());
let file = json!({"thumbnail": "t", "layer": {"thumbnail": "nested"}});
said.compare(&file, &json!({"layer": {}}), "placeholder", &["thumbnail"]);
assert_eq!(said.not_understood, ["field `layer.thumbnail`"]);
let mut saved = json!({"layer": {}});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"layer": {"thumbnail": "nested"}}));
}
/// An element kept on its own is not kept again with its list when typed writing leaves the list out.
#[test]
fn a_kept_element_is_not_restored_twice() {
let mut said = NewerFile::new("App", &newer());
said.keep_element("items", json!({"kind": "future"}), "unnamed".into());
said.compare(&json!({"items": [{"kind": "future"}]}), &json!({}), "placeholder", &[]);
let mut saved = json!({});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"items": [{"kind": "future"}]}));
}
/// A record whose id typed reading drops is matched by its name, and its fields go back.
#[test]
fn a_dropped_id_falls_back_to_the_name() {
let mut said = NewerFile::new("App", &newer());
said.compare(&json!({"items": [{"id": 7, "name": "a", "extra": 1}]}), &json!({"items": [{"name": "a"}]}), "placeholder", &[]);
let mut saved = json!({"items": [{"name": "a"}]});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"items": [{"name": "a", "id": 7, "extra": 1}]}));
}
/// Kept elements sharing a list are checked against one index, the ones put back included.
#[test]
fn kept_elements_check_each_other() {
let mut said = NewerFile::new("App", &newer());
said.keep_element("items", json!({"id": 1, "v": "first"}), "1".into());
said.keep_element("items", json!({"id": 1, "v": "again"}), "1 again".into());
let mut saved = json!({"items": []});
assert_eq!(said.restore_into(&mut saved).len(), 1);
assert_eq!(saved, json!({"items": [{"id": 1, "v": "first"}]}));
// deep nesting is searched without recursion
let mut deep = json!("needle");
for _ in 0..20_000 {
deep = Value::Array(vec![deep]);
}
said.keep_section("deep", deep, None, "test");
assert!(said.mentions("needle"));
// serde_json drops deep values recursively; leak it rather than overflow the test thread
std::mem::forget(said);
}
/// A record whose whole identity typed reading drops is matched by its place while the list keeps
/// its length, so an unchanged save keeps its fields.
#[test]
fn a_record_without_a_surviving_identity_is_matched_by_place() {
let mut said = NewerFile::new("App", &newer());
said.compare(&json!({"items": [{"id": 7, "extra": 1}]}), &json!({"items": [{}]}), "placeholder", &[]);
let mut saved = json!({"items": [{}]});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"items": [{"id": 7, "extra": 1}]}));
}
+16
View File
@@ -0,0 +1,16 @@
[package]
name = "craft-gridcraft-format"
description = "GridCraft's native JSON workbook file: its extensions, reader and pretty writer"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
+17
View File
@@ -0,0 +1,17 @@
# craft-gridcraft-format
GridCraft's native workbook file is one pretty-printed UTF-8 JSON object, with a two-space indent
and no newline at the end. It is saved as `.json`, or as `.scjson` by older builds.
The workbook is a type parameter. GridCraft owns the model, and after a read it rebuilds the
model's indexes, drops cells outside the grid and reads the macros. This crate owns the bytes:
- `EXTENSIONS` and `is_extension` name the format.
- `read(bytes)` reads a workbook. A failure reads as "not a GridCraft JSON workbook: …".
- `write(workbook)` writes one with `serde_json`'s pretty printer.
- `write_with_kept(workbook, field, kept)` appends entries that a newer version wrote, and this
build kept without reading, to the top-level list `field`, as the file held them. With nothing
kept, the output is exactly `write`'s. Otherwise the workbook goes through a `serde_json::Value`
first, so its object keys come out in map order, as GridCraft has always written them.
There is no `unsafe` and no panics, and it builds for `wasm32-unknown-unknown`.
+59
View File
@@ -0,0 +1,59 @@
//! GridCraft's native workbook file: the workbook as one pretty-printed UTF-8 JSON object,
//! saved as `.json` (or `.scjson`, its older name).
//!
//! GridCraft owns the workbook model and what happens after a read: rebuilding its indexes,
//! dropping cells outside the grid, and reading its macros. This crate owns the bytes: which
//! extensions name the format, how the JSON is read and written, and how entries kept from a
//! newer version are written back into a list as the file held them.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
use serde::Serialize;
use serde::de::DeserializeOwned;
use serde_json::Value;
/// The extensions of the format, lower case. `scjson` is the name older builds saved.
pub const EXTENSIONS: [&str; 2] = ["json", "scjson"];
/// Whether `extension` (without the dot, any case) names the format.
pub fn is_extension(extension: &str) -> bool {
EXTENSIONS.iter().any(|e| e.eq_ignore_ascii_case(extension))
}
#[derive(Debug, thiserror::Error)]
pub enum Error {
/// The bytes are not a workbook: not JSON, or JSON of another shape.
#[error("not a GridCraft JSON workbook: {0}")]
Read(serde_json::Error),
/// The workbook could not be written as JSON.
#[error("{0}")]
Write(serde_json::Error),
}
/// Read a workbook. Unknown fields are left to the model's own serde rules.
pub fn read<T: DeserializeOwned>(bytes: &[u8]) -> Result<T, Error> {
serde_json::from_slice(bytes).map_err(Error::Read)
}
/// Write a workbook as pretty-printed JSON (two-space indent, no newline at the end).
pub fn write<T: Serialize>(workbook: &T) -> Result<Vec<u8>, Error> {
serde_json::to_vec_pretty(workbook).map_err(Error::Write)
}
/// [`write`], with `kept` appended to the top-level list `field` (made if absent): entries a
/// newer version wrote that this build kept without reading. With nothing kept this is exactly
/// [`write`]. Otherwise the workbook goes through a [`Value`] first, so its object keys are
/// written in `serde_json`'s map order.
pub fn write_with_kept<T: Serialize>(workbook: &T, field: &str, kept: &[Value]) -> Result<Vec<u8>, Error> {
if kept.is_empty() {
return write(workbook);
}
let mut v = serde_json::to_value(workbook).map_err(Error::Write)?;
if let Some(list) = v.as_object_mut().map(|o| o.entry(field).or_insert_with(|| Value::Array(vec![]))).and_then(Value::as_array_mut) {
list.extend(kept.iter().cloned());
}
serde_json::to_vec_pretty(&v).map_err(Error::Write)
}
#[cfg(test)]
mod tests;
+87
View File
@@ -0,0 +1,87 @@
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use serde::Deserialize;
use serde_json::json;
use super::*;
/// A stand-in for an app's workbook: fields in declaration order, one of them a list.
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Book {
sheets: Vec<String>,
active: u32,
#[serde(default)]
macros: Vec<Value>,
}
fn book() -> Book {
Book { sheets: vec!["Sheet1".into(), "Été ✓".into()], active: 1, macros: vec![json!({"v": 1, "name": "Fill"})] }
}
#[test]
fn extensions_are_json_and_scjson_in_any_case() {
for yes in ["json", "scjson", "JSON", "ScJson"] {
assert!(is_extension(yes), "{yes}");
}
for no in ["", "xlsx", "jsonl", ".json", "scjs"] {
assert!(!is_extension(no), "{no}");
}
}
#[test]
fn a_workbook_is_written_pretty_in_field_order() {
let bytes = write(&book()).unwrap();
let text = String::from_utf8(bytes).unwrap();
assert_eq!(
text,
"{\n \"sheets\": [\n \"Sheet1\",\n \"Été ✓\"\n ],\n \"active\": 1,\n \"macros\": [\n {\n \"name\": \"Fill\",\n \"v\": 1\n }\n ]\n}"
);
assert_eq!(read::<Book>(text.as_bytes()).unwrap(), book());
}
#[test]
fn kept_entries_are_appended_as_the_file_held_them() {
let kept = [json!({"v": 9, "name": "Future", "extra": [1, 2]})];
let bytes = write_with_kept(&book(), "macros", &kept).unwrap();
let back: Book = read(&bytes).unwrap();
assert_eq!(back.macros, [json!({"v": 1, "name": "Fill"}), kept[0].clone()]);
// Through a JSON value, keys are in map order: `active` before `macros` before `sheets`.
let text = String::from_utf8(bytes).unwrap();
assert!(text.find("\"active\"") < text.find("\"macros\"") && text.find("\"macros\"") < text.find("\"sheets\""), "{text}");
// With nothing kept, the plain writer's bytes.
assert_eq!(write_with_kept(&book(), "macros", &[]).unwrap(), write(&book()).unwrap());
}
#[test]
fn kept_entries_make_the_list_when_it_is_absent() {
#[derive(Serialize)]
struct Bare {
sheets: Vec<String>,
}
let bytes = write_with_kept(&Bare { sheets: vec![] }, "macros", &[json!(1)]).unwrap();
assert_eq!(serde_json::from_slice::<Value>(&bytes).unwrap(), json!({"sheets": [], "macros": [1]}));
// A field that is not a list is left as it is.
let bytes = write_with_kept(&json!({"macros": "text"}), "macros", &[json!(1)]).unwrap();
assert_eq!(serde_json::from_slice::<Value>(&bytes).unwrap(), json!({"macros": "text"}));
}
#[test]
fn what_is_not_a_workbook_says_so() {
for bad in [&b""[..], b"{", b"[]", b"{\"sheets\": 7, \"active\": 0}", b"\xff\xfe"] {
let err = read::<Book>(bad).unwrap_err();
assert!(matches!(err, Error::Read(_)));
assert!(err.to_string().starts_with("not a GridCraft JSON workbook: "), "{err}");
}
assert_eq!(
read::<Book>(b"{\"sheets\": 7}").unwrap_err().to_string(),
"not a GridCraft JSON workbook: invalid type: integer `7`, expected a sequence at line 1 column 12"
);
}
#[test]
fn a_value_json_cannot_hold_is_a_write_error() {
use std::collections::BTreeMap;
let bad: BTreeMap<Vec<u8>, u8> = [(vec![1], 1)].into();
assert!(matches!(write(&bad), Err(Error::Write(_))));
assert!(matches!(write_with_kept(&bad, "macros", &[json!(1)]), Err(Error::Write(_))));
}
+3
View File
@@ -19,6 +19,9 @@ heic-rs = { version = "=0.1.4", default-features = false, features = ["std"], op
heic-rs = { version = "=0.1.4", default-features = false, features = ["std", "parallel"], optional = true }
rayon = { workspace = true, optional = true }
[dev-dependencies]
proptest = { workspace = true }
[features]
default = ["decode"]
# The decoder (probe, decode, decode_thumbnail, read_metadata): heic-rs for the container and HEVC.
+17 -4
View File
@@ -36,11 +36,24 @@ Errors are `Error::Unsupported` (image sequences, overlays, 4:2:2/4:4:4, …), `
`Error::Malformed`. It never panics: every call into heic-rs runs under `catch_unwind`, and a panic
inside it (PhotoCraft's fuzzing found two in 0.1.1) becomes `Error::Malformed`.
Features: `decode` (default) is the decoder above. `metadata` adds `craft_heif::items::read`: the
first `Exif` item and XMP packet from the box structure alone (`iinf`, `iloc`, `idat`; at most 4096
boxes, 1024 items, 1024 extents per item, 32 MiB per item), without heic-rs. A build with
Features: `decode` (default) is the decoder above. `metadata` adds `craft_heif::items`, which reads
the `Exif` and XMP items from the box structure alone (`pitm`, `iinf`, `iref`, `iloc`, `idat`),
without heic-rs:
- `items::read` returns the photo's own EXIF and XMP: the items `cdsc` ties to the primary image or
to an image it is made from (`dimg`: grid tiles, `iden`, `iovl`, `tmap`), else those tied to no
image. A thumbnail's or depth map's metadata is not the photo's.
- `items::scan` returns every EXIF and XMP item, whichever image it describes, and says when it
can't be sure it saw them all: `Scan::complete` is false for an incomplete or damaged item
catalog, and an item whose bytes can't be read comes back as `Unreadable`. An app that must
prove a file has no location (before an export that removes it) refuses when unsure.
- `items::exif_tiff` finds the TIFF stream in an `Exif` item, with or without a correct offset
field or an `Exif\0\0` prefix.
The reader walks at most 65,536 boxes and walks `iloc` once. It gathers at most 16 MiB per item,
and a scan gathers at most 64 MiB in all. A build with
`default-features = false, features = ["metadata"]` has no heic-rs dependency, and its
`read_metadata` is that reader. With both features, a test checks that the two readers agree.
`read_metadata` is `items::read`. With both features, a test checks that the two readers agree.
Feature `testdata` exposes `craft_heif::testdata`: lossless synthetic HEIF files (PCM-coded
32 × 32 HEVC pictures, grids, every property above) for the tests of the crates above.
+666 -414
View File
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+11
View File
@@ -0,0 +1,11 @@
[package]
name = "craft-http-url"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license = "Apache-2.0"
repository.workspace = true
description = "Bounded HTTP URL parsing and redirect resolution without network access"
[lints]
workspace = true
+25
View File
@@ -0,0 +1,25 @@
# craft-http-url
HTTP/HTTPS URL parsing, dot-segment removal, relative redirects, HTTPS downgrade rejection,
and query-redacted display. No dependencies or network operations. Parsing rejects inputs
larger than 16 KiB and redirects are bounded before concatenation.
This is not an SSRF filter, DNS resolver or HTTP client. Local/private hosts are allowed.
Applications keep destination permission, authentication, TLS, request/response limits and
redirect-count policy. Public fields preserve the original API; manually constructed values
must not be treated as validated URLs.
## Attribution
Extracted from LightCraft `crates/fetch/src/http.rs` at
`2dcb258ba40738230a07484a95eb27dc84ed66fe` (Apache-2.0).
Original parser: Daniel Voshart, `7fa7506db46d69d609e85e39b338f33786ed0293`.
HTTPS downgrade protection: MarekSpodBiedry, `4a91a1c218bf3d0ac49cc73ebb60621a21ae55e2`.
URL normalization, scheme, fragment and authority fixes: NetterSilver, including
`edc369565eef92161b56b8f1e39ef53629c05489`,
`78f0e6e650444886069f4d4ae4e6593f221b98d4`,
`cee93496587d31c514aaeced3e6b8027cfdb41a0`,
`dee5f7910ad8c24cf28bab528f8d007550af2b23`,
`c3a5d9b7f1f837dc14c01942e3d0293ecbf36345` and
`13849307a9343504339d68b086f1a32cdcbbb457`.
Source-author commits are retained separately in this extraction.
+277
View File
@@ -0,0 +1,277 @@
//! Bounded HTTP/HTTPS URL parsing and redirect resolution, extracted from LightCraft.
//!
//! This is syntax handling, not a network permission or SSRF policy. It permits local/private
//! hosts and performs no DNS, socket or TLS operations. Callers own destination allowlists,
//! credential forwarding, redirect counts, connection deadlines and response limits.
//! Only parsed values have validated fields; constructing public fields bypasses parsing.
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
/// A URL or redirect cannot be represented by this bounded HTTP URL syntax.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UrlError(pub String);
impl std::fmt::Display for UrlError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
impl std::error::Error for UrlError {}
/// A parsed `http://` or `https://` URL.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Url {
pub tls: bool,
pub host: String,
pub port: u16,
/// Path and query, starting with `/`.
pub path: String,
}
/// Remove `.` and `..` segments from the path part of `path` (RFC 3986 section 5.2.4); the query is kept as is.
fn remove_dot_segments(path: &str) -> String {
let (p, query) = match path.find('?') {
Some(i) => (path.get(..i).unwrap_or_default(), path.get(i..).unwrap_or_default()),
None => (path, ""),
};
let mut out: Vec<&str> = Vec::new();
let mut segments = p.split('/').skip(1).peekable();
let mut trailing_slash = false;
while let Some(seg) = segments.next() {
let last = segments.peek().is_none();
match seg {
"." => trailing_slash = last,
".." => {
out.pop();
trailing_slash = last;
}
_ => {
out.push(seg);
trailing_slash = false;
}
}
}
let mut result = format!("/{}", out.join("/"));
if trailing_slash && !result.ends_with('/') {
result.push('/');
}
result.push_str(query);
result
}
/// `s` without the leading `prefix`, matched ASCII case-insensitively (RFC 3986 §3.1).
fn strip_scheme<'a>(s: &'a str, prefix: &str) -> Option<&'a str> {
let head = s.get(..prefix.len())?;
head.eq_ignore_ascii_case(prefix).then(|| s.get(prefix.len()..).unwrap_or_default())
}
impl Url {
pub fn parse(s: &str) -> Result<Url, UrlError> {
let bad = |why: &str| UrlError(why.to_string());
if s.len() > 16 * 1024 {
return Err(bad("URL exceeds 16 KiB"));
}
let s = s.trim();
if s.chars().any(|c| c.is_control() || c == ' ') {
return Err(bad("contains spaces or control characters"));
}
let (tls, rest) = if let Some(r) = strip_scheme(s, "https://") {
(true, r)
} else if let Some(r) = strip_scheme(s, "http://") {
(false, r)
} else {
return Err(bad("only http:// and https:// are supported"));
};
let rest = rest.split('#').next().unwrap_or_default();
let (authority, path) = match rest.find(['/', '?']) {
Some(i) => (rest.get(..i).unwrap_or_default(), rest.get(i..).unwrap_or_default()),
None => (rest, ""),
};
let path = if path.starts_with('/') { path.to_string() } else { format!("/{path}") };
let path = remove_dot_segments(&path);
if authority.contains('@') {
return Err(bad("user names and passwords in URLs are not supported"));
}
let (host, port) = if let Some(v6) = authority.strip_prefix('[') {
let end = v6.find(']').ok_or_else(|| bad("unterminated IPv6 address"))?;
let host = v6.get(..end).unwrap_or_default();
let after = v6.get(end + 1..).unwrap_or_default();
if !after.is_empty() && !after.starts_with(':') {
return Err(bad("unexpected text after IPv6 address"));
}
(host.to_string(), after.strip_prefix(':'))
} else {
match authority.rsplit_once(':') {
Some((h, p)) => (h.to_string(), Some(p)),
None => (authority.to_string(), None),
}
};
let port = match port {
Some(p) if p.bytes().all(|b| b.is_ascii_digit()) => p.parse::<u16>().ok().filter(|p| *p != 0).ok_or_else(|| bad("bad port"))?,
Some(_) => return Err(bad("bad port")),
None if tls => 443,
None => 80,
};
if host.is_empty() || !host.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '.' | ':' | '_')) {
return Err(bad("bad host name"));
}
Ok(Url { tls, host: host.to_ascii_lowercase(), port, path })
}
/// `Location` of a redirect, resolved against this URL.
pub fn join(&self, location: &str) -> Result<Url, UrlError> {
// Reject before concatenating a hostile Location with the base URL.
if location.len() > 16 * 1024 || self.host.len().saturating_add(self.path.len()) > 16 * 1024 {
return Err(UrlError("URL exceeds 16 KiB".into()));
}
let l = location.trim();
if strip_scheme(l, "http://").is_some() || strip_scheme(l, "https://").is_some() {
let url = Url::parse(l)?;
if self.tls && !url.tls {
return Err(UrlError("redirect from https to http refused".into()));
}
return Ok(url);
}
// another scheme (`ftp:`, `file:`, `data:`…)
if let Some((scheme, _)) = l.split_once(':')
&& !scheme.is_empty()
&& scheme.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
{
return Err(UrlError(format!("redirect to an unsupported location `{scheme}:`")));
}
let scheme = if self.tls { "https" } else { "http" };
let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() };
let base = format!("{scheme}://{host}:{}", self.port);
if let Some(rest) = l.strip_prefix("//") {
return Url::parse(&format!("{scheme}://{rest}"));
}
if l.starts_with('/') {
return Url::parse(&format!("{base}{l}"));
}
// empty or fragment-only: the same document
if l.is_empty() || l.starts_with('#') {
return Ok(self.clone());
}
// relative to the current directory
let dir = self.path.split('?').next().unwrap_or("/");
// a query-only reference keeps the whole path, file name included (RFC 3986 §5.2.2)
if l.starts_with('?') {
return Url::parse(&format!("{base}{dir}{l}"));
}
let dir = dir.rsplit_once('/').map_or("", |(d, _)| d);
Url::parse(&format!("{base}{dir}/{l}"))
}
/// Authority formatted for an HTTP Host field.
pub fn host_header(&self) -> String {
let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() };
let default = if self.tls { 443 } else { 80 };
if self.port == default { host } else { format!("{host}:{}", self.port) }
}
/// The URL without its query (for messages: a query may hold a signed token).
pub fn display(&self) -> String {
let scheme = if self.tls { "https" } else { "http" };
format!("{scheme}://{}{}", self.host_header(), self.path.split('?').next().unwrap_or("/"))
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::{Url, UrlError};
#[test]
fn fragment_only_or_empty_location_keeps_file_name_and_query() {
let base = Url::parse("https://a.example/x/y/file?q").unwrap();
for l in ["#top", "", " "] {
let u = base.join(l).unwrap();
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/x/y/file?q"), "{l:?}");
}
}
#[test]
fn rejects_oversized_and_control_character_urls() {
let base = Url::parse("https://example.com/path").unwrap();
assert!(base.join(&format!("#{}", "x".repeat(16 * 1024))).is_err());
assert!(base.join(&"x".repeat(16 * 1024 + 1)).is_err());
assert!(Url::parse(&format!("https://example.com/{}", "x".repeat(16 * 1024))).is_err());
assert!(Url::parse("https://example.com/x?secret=abc\r\nInjected: 1").is_err());
}
#[test]
fn join_query_only_keeps_file_name() {
let u = Url::parse("https://a.example/x/y/file?q=1").unwrap();
assert_eq!(u.join("?page=2").unwrap().path, "/x/y/file?page=2");
}
#[test]
fn join_removes_dot_segments() {
let base = Url::parse("https://a.example/x/y/file").unwrap();
for (loc, want) in
[("../z", "/x/z"), ("./z", "/x/y/z"), ("../../../z", "/z"), ("/a/./b/../c", "/a/c"), ("../z?q=1", "/x/z?q=1"), ("..", "/x/")]
{
assert_eq!(base.join(loc).unwrap().path, want, "{loc}");
}
}
#[test]
fn url_scheme_is_matched_case_insensitively() {
let u = Url::parse("HTTPS://A.example/z").unwrap();
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/z"));
assert!(Url::parse("Http://a.example/").is_ok());
let j = Url::parse("https://a.example/start").unwrap().join("HTTPS://cdn.example/file").unwrap();
assert_eq!((j.tls, j.host.as_str(), j.path.as_str()), (true, "cdn.example", "/file"));
}
#[test]
fn parses_urls() {
let u = Url::parse("https://cdn.example.com/models/sam3/model.safetensors?sig=1#x").unwrap();
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "cdn.example.com", 443, "/models/sam3/model.safetensors?sig=1"));
assert_eq!(u.display(), "https://cdn.example.com/models/sam3/model.safetensors");
let u = Url::parse("http://127.0.0.1:8080").unwrap();
assert_eq!((u.tls, u.port, u.path.as_str()), (false, 8080, "/"));
let u = Url::parse("http://[::1]:9/a").unwrap();
assert_eq!((u.host.as_str(), u.port), ("::1", 9));
for bad in ["ftp://x/y", "https://", "https://a b/", "https://u:p@h/", "http://h:0/", "http://h:99999/", "https://h/\r\nX: y", "file:///etc"]
{
assert!(Url::parse(bad).is_err(), "{bad}");
}
}
#[test]
fn rejects_signed_ports() {
for bad in ["http://example.com:+80/x", "http://example.com:-80/x", "http://[::1]:+80/x"] {
assert!(Url::parse(bad).is_err(), "{bad}");
}
assert!(Url::parse("http://a.test/").unwrap().join("http://example.com:+80/x").is_err());
assert_eq!(Url::parse("http://example.com:80/x").unwrap().port, 80);
}
#[test]
fn rejects_text_after_a_bracketed_ipv6_host() {
for bad in ["http://[::1]junk/x", "https://[::1]junk/x", "http://[::1]8080/x", "http://[::1]8080"] {
assert!(matches!(Url::parse(bad), Err(UrlError(_))), "{bad}");
}
let u = Url::parse("http://[::1]:8080/x").unwrap();
assert_eq!((u.host.as_str(), u.port), ("::1", 8080));
let u = Url::parse("http://[::1]/x").unwrap();
assert_eq!((u.host.as_str(), u.port), ("::1", 80));
}
#[test]
fn resolves_redirects() {
let u = Url::parse("https://a.example/x/y/file?q").unwrap();
assert_eq!(u.join("https://b.example/z").unwrap().host, "b.example");
assert_eq!(u.join("/root").unwrap().path, "/root");
assert_eq!(u.join("other").unwrap().path, "/x/y/other");
assert_eq!(u.join("//c.example/p").unwrap().host, "c.example");
assert!(u.join("ftp://nope").is_err());
// no downgrade from https to http, however the redirect is written
assert!(u.join("http://a.example/z").is_err());
let plain = Url::parse("http://127.0.0.1:8080/a").unwrap();
assert!(!plain.join("http://127.0.0.1:9/b").unwrap().tls);
assert!(plain.join("https://b.example/z").unwrap().tls);
}
}
+4
View File
@@ -13,6 +13,10 @@ workspace = true
[features]
corpus = []
default = []
# Opt in to threaded AVIF encoding without enabling AVIF (or assembly) by itself.
threads = ["ravif?/threading", "rav1e?/threading"]
sgi = []
sunraster = []
avif = ["image/avif", "dep:ravif", "dep:rav1e", "dep:avif-serialize"]
heif = ["dep:craft-heif"]
# AVIF decode: craft-video's clean-room pure-Rust AV1 decoder. Native targets only (on wasm32 a
+2
View File
@@ -57,6 +57,8 @@ craft-image-codecs = { git = "https://github.com/storytold/craft-libs", rev = "<
| Cineon | yes | yes | U16 (10-bit code values read and written exactly through U16) | Gray, RGB | no | no | no | no | no | no | no | built in |
| DICOM | yes | **no** | U8, U16 | Gray, RGB | no | no | no | no | no | no | n/a | built in |
| Amiga IFF (ILBM/PBM) | yes | yes (24-bit ILBM) | U8 | RGB, RGBA | yes (1-bit mask plane on write) | no | no | no | no | no | no | built in |
| Sun Raster (`.ras`) | with feature `sunraster` (incl. RLE and colour maps) | with feature `sunraster` (uncompressed) | U8 | Gray, RGB, RGBA | yes | no | no | no | no | no | no | `sunraster` (off by default) |
| SGI (`.sgi`, `.rgb`, `.bw`) | with feature `sgi` | with feature `sgi` (RLE) | U8, U16 | Gray, Gray+A, RGB, RGBA | yes | no | no | no | no | no | no | `sgi` (off by default) |
| JPEG-LS (`.jls`) | yes (any NEAR; one scan per component) | yes (lossless) | U8, U16 | Gray, RGB | no | no | no | no | no | no | no | built in |
| AVIF | with feature `avif-decode` (native only) | only with feature `avif` | U8, U16 (10/12-bit decodes to U16; PQ/HLG to linear F32) | RGB, RGBA | yes (auxiliary alpha) | read (`colr` prof/rICC) | read | read | no | no | yes | `craft-video` AV1 decoder; `image` + `ravif` to write |
| HEIF/HEIC | with feature `heif` | **no** | U8, U16 (10/12-bit decodes to U16) | RGB, RGBA | yes (auxiliary alpha) | yes (`colr` prof) | yes | yes | no | no | n/a | `heic-rs` |
+4
View File
@@ -20,6 +20,10 @@ pub(crate) mod jxl;
pub(crate) mod pcx;
pub(crate) mod png;
pub(crate) mod pnm;
#[cfg(feature = "sgi")]
pub(crate) mod sgi;
#[cfg(feature = "sunraster")]
pub(crate) mod sunraster;
pub(crate) mod tga;
pub(crate) mod tiff;
pub(crate) mod tiff_exif;
+104 -21
View File
@@ -198,6 +198,15 @@ fn data_end(b: &[u8]) -> (DataEnd, u64) {
data_end_mode(b, false)
}
/// Whether JPEG `b` holds too little scan data for a `w` × `h` frame to be worth decoding: no scan
/// data at all, or under one byte for every [`MAX_PIXELS_PER_BYTE`] pixels (counted on the scan
/// data the marker walk crosses, so APP padding doesn't count). `zune-jpeg` allocates the whole
/// output before it reads a scan, so a few bytes could otherwise ask for a frame of any size.
pub(crate) fn scan_too_short(b: &[u8], w: u32, h: u32) -> bool {
let (end, scan_bytes) = data_end(b);
end == DataEnd::Empty || u64::from(w) * u64::from(h) / MAX_PIXELS_PER_BYTE > scan_bytes
}
/// Whether the marker walk actually reaches the first EOI, with malformed segment lengths refused.
pub(crate) fn reaches_eoi(b: &[u8]) -> bool {
data_end_mode(b, true).0 == DataEnd::Complete
@@ -322,6 +331,40 @@ fn without_extended_xmp(b: &[u8]) -> Option<Vec<u8>> {
Some(out)
}
/// Whether the first frame header of `b` samples its first component below the largest
/// horizontal or vertical sampling factor of the frame. zune-jpeg 0.5 sizes its upsampler's
/// scratch row from the first component and asserts on some such frames (jpeg-encoder writes
/// CMYK that way: C, M and Y at 1×1, K larger). `false` when no frame header can be read.
fn first_component_subsampled(b: &[u8]) -> bool {
if b.get(..2) != Some(&[0xFF, 0xD8]) {
return false;
}
let mut i = 2;
while let (Some(&0xFF), Some(&m)) = (b.get(i), b.get(i + 1)) {
match m {
0xFF => i += 1,
0x01 | 0xD0..=0xD8 => i += 2,
0xD9 | 0xDA => return false,
_ => {
let Some(len) = b.get(i + 2..i + 4).map(|l| usize::from(u16::from_be_bytes([l[0], l[1]]))) else { return false };
if matches!(m, 0xC0..=0xCF) && !matches!(m, 0xC4 | 0xC8 | 0xCC) {
let Some(seg) = b.get(i + 4..i + 2 + len.max(2)) else { return false };
let n = usize::from(seg.get(5).copied().unwrap_or(0));
let factors: Vec<(u8, u8)> = (0..n).filter_map(|c| seg.get(6 + c * 3 + 1)).map(|f| (f >> 4, f & 0x0F)).collect();
let (Some(&(h0, v0)), Some(hmax), Some(vmax)) =
(factors.first(), factors.iter().map(|f| f.0).max(), factors.iter().map(|f| f.1).max())
else {
return false;
};
return h0 < hmax || v0 < vmax;
}
i += 2 + len;
}
}
}
false
}
/// Whether `craft-jpeg` (rather than `zune-jpeg`) decodes the file: arithmetic coding
/// (SOF9–SOF11), lossless (SOF3), or a DCT frame whose precision isn't 8 bits (#4934, #5003).
/// This includes 12-bit progressive Huffman (SOF2), which `craft-jpeg` supports.
@@ -389,6 +432,39 @@ fn decode_craft_jpeg(bytes: &[u8], limits: &Limits, adobe_transform: Option<u8>,
/// truncated or not, are far above this.
const MAX_PIXELS_PER_BYTE: u64 = 2048;
/// Decode with craft-jpeg: the files [`uses_craft_jpeg`] routes away from `zune-jpeg`, and the
/// ones `zune-jpeg` panics on.
fn decode_whole(bytes: &[u8], limits: &Limits, meta: JpegMeta, end: DataEnd, scan_bytes: u64) -> Result<Image, CodecError> {
// craft-jpeg sizes its buffers from the frame header before it reads a scan.
if scan_bytes == 0 {
return Err(err("the file has no scan data"));
}
if let Some((w, h, _)) = meta.frame
&& end == DataEnd::Truncated
// Only truncated scans: arithmetic coding (SOF9–SOF11) can spend well under a bit on
// a flat block, so a complete scan may be shorter than this bound.
&& u64::from(w) * u64::from(h) / MAX_PIXELS_PER_BYTE > scan_bytes
{
return Err(err("the truncated scan is too short for the frame it declares"));
}
let (mut img, damaged) = decode_craft_jpeg(bytes, limits, meta.adobe_transform, meta.adobe_marker)?;
img.icc = meta.icc;
img.meta = Metadata { exif: meta.exif, xmp: meta.xmp, dpi: meta.dpi, photoshop_resources: meta.photoshop, ..Default::default() };
if end == DataEnd::Truncated || damaged {
img.warnings.push(DecodeWarning::Truncated { format: F });
}
Ok(img)
}
/// The stored size and EXIF block [`decode`] would find, from the marker segments before the
/// first scan: the first frame header's size, checked against `limits`, and the APP1 EXIF block.
pub(crate) fn probe(bytes: &[u8], limits: &Limits) -> Result<(u32, u32, Option<Vec<u8>>), CodecError> {
let meta = scan_metadata(bytes);
let (w, h, _) = meta.frame.ok_or_else(|| err("no frame header before the image data"))?;
limits.check_size(w, h)?;
Ok((w, h, meta.exif))
}
pub(crate) fn decode(bytes: &[u8], limits: &Limits) -> Result<Image, CodecError> {
let (end, scan_bytes) = data_end(bytes);
if end == DataEnd::Empty {
@@ -398,26 +474,9 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits) -> Result<Image, CodecError>
if let Some((w, h, nc)) = meta.frame {
limits.check_bytes(w, h, u64::from(nc.max(1)))?;
}
if uses_craft_jpeg(bytes) {
// craft-jpeg sizes its buffers from the frame header before it reads a scan.
if scan_bytes == 0 {
return Err(err("the file has no scan data"));
}
if let Some((w, h, _)) = meta.frame
&& end == DataEnd::Truncated
// Only truncated scans: arithmetic coding (SOF9–SOF11) can spend well under a bit on
// a flat block, so a complete scan may be shorter than this bound.
&& u64::from(w) * u64::from(h) / MAX_PIXELS_PER_BYTE > scan_bytes
{
return Err(err("the truncated scan is too short for the frame it declares"));
}
let (mut img, damaged) = decode_craft_jpeg(bytes, limits, meta.adobe_transform, meta.adobe_marker)?;
img.icc = meta.icc;
img.meta = Metadata { exif: meta.exif, xmp: meta.xmp, dpi: meta.dpi, photoshop_resources: meta.photoshop, ..Default::default() };
if end == DataEnd::Truncated || damaged {
img.warnings.push(DecodeWarning::Truncated { format: F });
}
return Ok(img);
// Where a panic aborts (wasm32), the frames zune-jpeg may panic on can't be caught below.
if uses_craft_jpeg(bytes) || (cfg!(panic = "abort") && first_component_subsampled(bytes)) {
return decode_whole(bytes, limits, meta, end, scan_bytes);
}
let options = DecoderOptions::default().set_strict_mode(false).set_max_width(65535).set_max_height(65535);
let stripped = without_extended_xmp(bytes);
@@ -443,7 +502,12 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits) -> Result<Image, CodecError>
return Err(err(format!("{what} is too short for the frame it declares")));
}
dec.set_options(dec.options().jpeg_set_out_colorspace(out_cs));
let mut px = dec.decode().map_err(err)?;
// zune-jpeg 0.5 asserts on some subsampled four-component frames whose first component isn't
// the largest (its upsampler's scratch row is sized from that component): craft-jpeg reads them.
let Ok(decoded) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| dec.decode())) else {
return decode_whole(bytes, limits, meta, end, scan_bytes);
};
let mut px = decoded.map_err(err)?;
let expected = w as usize * h as usize * layout.channels();
if px.len() < expected {
return Err(err("short pixel buffer"));
@@ -1009,6 +1073,25 @@ mod tests {
(v * 65535.0 / 4095.0).round() as u16
}
#[test]
fn first_component_subsampling() {
let sof = |comps: &[(u8, u8)]| {
let mut s = vec![0xFF, 0xD8, 0xFF, 0xE0, 0, 4, 0, 0, 0xFF, 0xC2, 0, (8 + 3 * comps.len()) as u8, 8, 0, 8, 0, 8, comps.len() as u8];
for (i, &(h, v)) in comps.iter().enumerate() {
s.extend([i as u8 + 1, (h << 4) | v, 0]);
}
s
};
assert!(first_component_subsampled(&sof(&[(1, 1), (1, 1), (1, 1), (2, 2)])));
assert!(first_component_subsampled(&sof(&[(1, 2), (1, 2), (2, 1)])));
assert!(!first_component_subsampled(&sof(&[(2, 2), (1, 1), (1, 1)])));
assert!(!first_component_subsampled(&sof(&[(1, 1); 4])));
assert!(!first_component_subsampled(&sof(&[])));
assert!(!first_component_subsampled(&[0xFF, 0xD8, 0xFF, 0xDA, 0, 2]));
assert!(!first_component_subsampled(&[0xFF, 0xD8, 0xFF, 0xC0, 0, 20, 8]));
assert!(!first_component_subsampled(b"not a jpeg"));
}
#[test]
fn routing() {
assert!(!uses_craft_jpeg(&[0xFF, 0xD8, 0xFF, 0xC0, 0, 11, 8, 0, 8, 0, 8, 1, 1, 0x11, 0]));
+20
View File
@@ -27,6 +27,26 @@ pub(crate) fn dpi_of_ppm(ppm: u32) -> f32 {
}
/// Decodes a whole PNG, by driving a [`PngDecodeRows`] to its end.
/// The stored size and EXIF block [`decode`] would find, from the chunks before the image data:
/// the same header checks (limits, a file too small for its image) and the same frame (an fcTL
/// before the IDAT overrides the header's size), without inflating any pixels or text.
pub(crate) fn probe(bytes: &[u8], limits: &Limits) -> Result<(u32, u32, Option<Vec<u8>>), CodecError> {
let mut decoder = png::Decoder::new_with_limits(Cursor::new(bytes), png::Limits { bytes: limits.alloc_usize() });
decoder.set_ignore_text_chunk(true);
{
let info = decoder.read_header_info().map_err(map_png_err)?;
let (w, h) = info.size();
limits.check_size(w, h)?;
let stored = u64::from(w).saturating_mul(u64::from(h)).saturating_mul(info.bits_per_pixel() as u64).div_ceil(8);
Limits::check_plausible(stored, bytes.len(), 1100)?;
}
let reader = decoder.read_info().map_err(map_png_err)?;
let info = reader.info();
let (w, h) = info.frame_control.map_or_else(|| info.size(), |f| (f.width, f.height));
limits.check_size(w, h)?;
Ok((w, h, info.exif_metadata.as_ref().map(|c| c.to_vec())))
}
pub(crate) fn decode(bytes: &[u8], limits: &Limits) -> Result<Image, CodecError> {
PngDecodeRows::new(bytes, limits)?.step(&mut || true)?.ok_or_else(|| err("the decoder stopped before the image ended"))
}
+395
View File
@@ -0,0 +1,395 @@
//! SGI image files (`.sgi`, `.rgb`, `.rgba`, `.bw`): 8- and 16-bit gray, gray + alpha, RGB and
//! RGBA, verbatim or run-length encoded.
//!
//! Read and write follow the published SGI image file format description: a 512-byte big-endian
//! header, then one plane per channel with the rows stored bottom to top. Palette-mapped files
//! (colormap ID other than 0) are rejected. The writer uses the run-length encoding.
use crate::error::CodecError;
use crate::fidelity::Plan;
use crate::format::Format;
use crate::image::{ChannelLayout, Image, SampleType};
use crate::options::Limits;
const F: Format = Format::Sgi;
const HEADER: usize = 512;
const MAGIC: u16 = 474;
fn bad(reason: impl std::fmt::Display) -> CodecError {
CodecError::malformed(F, reason)
}
fn be16(b: &[u8], i: usize) -> u16 {
u16::from_be_bytes([b[i], b[i + 1]])
}
fn be32(b: &[u8], i: usize) -> u32 {
u32::from_be_bytes([b[i], b[i + 1], b[i + 2], b[i + 3]])
}
/// The fields of the header that decoding needs, already checked for plausibility.
struct Header {
rle: bool,
wide: bool,
width: u32,
height: u32,
planes: usize,
}
/// Parse the 512-byte header. `None` when the bytes are not a plausible SGI header.
fn header(b: &[u8]) -> Option<Header> {
if b.len() < HEADER || be16(b, 0) != MAGIC {
return None;
}
let (storage, bpc, dimension) = (b[2], b[3], be16(b, 4));
let (width, height, planes) = (u32::from(be16(b, 6)), u32::from(be16(b, 8)), usize::from(be16(b, 10)));
if storage > 1 || !(1..=2).contains(&bpc) || !(1..=3).contains(&dimension) {
return None;
}
// A one-dimensional image has one row, a two-dimensional one a single plane.
let (height, planes) = match dimension {
1 => (1, 1),
2 => (height, 1),
_ => (height, planes),
};
if width == 0 || height == 0 || !(1..=4).contains(&planes) {
return None;
}
Some(Header { rle: storage == 1, wide: bpc == 2, width, height, planes })
}
/// Whether `b` starts with a plausible SGI header (the magic number plus consistent fields).
pub(crate) fn looks_like_sgi(b: &[u8]) -> bool {
header(b).is_some()
}
fn layout_of(planes: usize) -> ChannelLayout {
match planes {
1 => ChannelLayout::Gray,
2 => ChannelLayout::GrayA,
3 => ChannelLayout::Rgb,
_ => ChannelLayout::Rgba,
}
}
/// Expand one run-length encoded row into exactly `out.len()` samples.
fn read_rle_row(src: &[u8], wide: bool, out: &mut [u16]) -> Result<(), CodecError> {
let step = if wide { 2 } else { 1 };
let sample = |pos: usize| -> Result<u16, CodecError> {
let s = src.get(pos..pos + step).ok_or_else(|| bad("truncated RLE row"))?;
Ok(if wide { u16::from_be_bytes([s[0], s[1]]) } else { u16::from(s[0]) })
};
let (mut pos, mut n) = (0usize, 0usize);
while n < out.len() {
let code = sample(pos)?;
pos += step;
let count = usize::from(code & 0x7f);
if count == 0 {
return Err(bad("RLE row ends before the image width"));
}
if count > out.len() - n {
return Err(bad("RLE packet runs past the end of the row"));
}
if code & 0x80 != 0 {
for slot in &mut out[n..n + count] {
*slot = sample(pos)?;
pos += step;
}
} else {
let value = sample(pos)?;
pos += step;
out[n..n + count].fill(value);
}
n += count;
}
Ok(())
}
pub(crate) fn decode(b: &[u8], limits: &Limits) -> Result<Image, CodecError> {
let h = header(b).ok_or_else(|| bad("invalid SGI header"))?;
if be32(b, 104) != 0 {
return Err(CodecError::unsupported(F, "palette-mapped SGI files are not supported"));
}
let layout = layout_of(h.planes);
let sample = if h.wide { SampleType::U16 } else { SampleType::U8 };
limits.check(h.width, h.height, layout, sample)?;
let (w, ht, planes) = (h.width as usize, h.height as usize, h.planes);
let rows = ht.checked_mul(planes).ok_or_else(|| bad("image size overflow"))?;
let bps = if h.wide { 2 } else { 1 };
let total = w.checked_mul(rows).ok_or_else(|| bad("image size overflow"))?;
// Reject inputs that cannot hold the declared pixels before allocating anything for them.
let table = if h.rle { rows.checked_mul(8).ok_or_else(|| bad("image size overflow"))? } else { 0 };
let min_len = if h.rle {
// Both offset tables must fit; rows may share compressed data, so the data itself is
// checked row by row against the offsets below.
HEADER + table
} else {
HEADER + total.checked_mul(bps).ok_or_else(|| bad("image size overflow"))?
};
if b.len() < min_len {
return Err(bad("the file is too short for the declared image"));
}
let mut out = Vec::new();
out.try_reserve_exact(total).map_err(|_| bad("pixel buffer is too large"))?;
out.resize(total, 0u16);
let mut row = vec![0u16; w];
for z in 0..planes {
for y in 0..ht {
let index = y + z * ht;
if h.rle {
let start = be32(b, HEADER + index * 4) as usize;
let len = be32(b, HEADER + rows * 4 + index * 4) as usize;
let end = start
.checked_add(len)
.filter(|&e| e <= b.len() && start >= HEADER + table)
.ok_or_else(|| bad("RLE row lies outside the file"))?;
read_rle_row(&b[start..end], h.wide, &mut row)?;
} else {
let start = HEADER + index * w * bps;
let s = &b[start..start + w * bps];
for (x, v) in row.iter_mut().enumerate() {
*v = if h.wide { u16::from_be_bytes([s[2 * x], s[2 * x + 1]]) } else { u16::from(s[x]) };
}
}
// Rows are stored bottom to top; planes are separate.
let top_y = ht - 1 - y;
for (x, &v) in row.iter().enumerate() {
out[(top_y * w + x) * planes + z] = v;
}
}
}
if h.wide {
Image::from_u16(h.width, h.height, layout, &out)
} else {
Image::from_u8(h.width, h.height, layout, out.iter().map(|&v| v as u8).collect())
}
}
/// Append one run-length encoded row (one packet list and a terminator).
fn write_rle_row(row: &[u16], wide: bool, out: &mut Vec<u8>) {
let put = |out: &mut Vec<u8>, v: u16| {
if wide {
out.extend_from_slice(&v.to_be_bytes());
} else {
out.push(v as u8);
}
};
let mut i = 0;
while i < row.len() {
let mut run = 1;
while i + run < row.len() && run < 127 && row[i + run] == row[i] {
run += 1;
}
if run >= 2 {
put(out, run as u16);
put(out, row[i]);
i += run;
} else {
// A literal packet extends until the next run of two or more.
let mut end = i + 1;
while end < row.len() && end - i < 127 && !(end + 1 < row.len() && row[end] == row[end + 1]) {
end += 1;
}
put(out, 0x80 | (end - i) as u16);
for &v in &row[i..end] {
put(out, v);
}
i = end;
}
}
put(out, 0);
}
/// Writes a run-length encoded SGI file: 8 or 16 bits per channel, gray, gray + alpha, RGB or RGBA.
pub(crate) fn encode(src: &Image, plan: Plan) -> Result<Vec<u8>, CodecError> {
let enc = |msg: String| CodecError::encode(F, msg);
let img = src.converted(plan.layout, plan.sample);
let wide = match img.sample_type() {
SampleType::U8 => false,
SampleType::U16 => true,
s => return Err(enc(format!("SGI stores 8 or 16 bits per channel, not {s:?}"))),
};
let planes = img.layout().channels();
if !matches!(img.layout(), ChannelLayout::Gray | ChannelLayout::GrayA | ChannelLayout::Rgb | ChannelLayout::Rgba) {
return Err(enc(format!("no SGI layout for {:?}", img.layout())));
}
let (w, ht) = img.dimensions();
let (Ok(w16), Ok(h16)) = (u16::try_from(w), u16::try_from(ht)) else {
return Err(enc(format!("dimensions exceed 65535x65535: {w}x{ht}")));
};
let (w, ht) = (w as usize, ht as usize);
let bps = if wide { 2 } else { 1 };
let data = img.data();
if data.len() != w * ht * planes * bps {
return Err(enc("the image has the wrong number of samples".into()));
}
let sample_at = |i: usize| -> u16 { if wide { u16::from_ne_bytes([data[2 * i], data[2 * i + 1]]) } else { u16::from(data[i]) } };
let rows = ht * planes;
let table = HEADER;
let mut starts = Vec::with_capacity(rows);
let mut lens = Vec::with_capacity(rows);
let mut body = Vec::new();
let mut row = vec![0u16; w];
let base = u32::try_from(HEADER + rows * 8).map_err(|_| enc("image too large".into()))?;
for z in 0..planes {
for y in 0..ht {
let src_y = ht - 1 - y; // stored bottom to top
for (x, v) in row.iter_mut().enumerate() {
*v = sample_at((src_y * w + x) * planes + z);
}
let before = body.len();
write_rle_row(&row, wide, &mut body);
let start = u32::try_from(base as usize + before).map_err(|_| enc("image too large".into()))?;
starts.push(start);
lens.push((body.len() - before) as u32);
}
}
let mut out = Vec::with_capacity(table + rows * 8 + body.len());
out.extend_from_slice(&MAGIC.to_be_bytes());
out.push(1); // RLE
out.push(bps as u8);
out.extend_from_slice(&(if planes > 1 { 3u16 } else { 2u16 }).to_be_bytes());
out.extend_from_slice(&w16.to_be_bytes());
out.extend_from_slice(&h16.to_be_bytes());
out.extend_from_slice(&(planes as u16).to_be_bytes());
out.extend_from_slice(&0u32.to_be_bytes()); // pixmin
out.extend_from_slice(&(if wide { 65535u32 } else { 255 }).to_be_bytes()); // pixmax
out.resize(HEADER, 0); // dummy, image name, colormap 0, padding
for s in &starts {
out.extend_from_slice(&s.to_be_bytes());
}
for l in &lens {
out.extend_from_slice(&l.to_be_bytes());
}
out.extend_from_slice(&body);
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
fn gradient(w: u32, h: u32, layout: ChannelLayout) -> Image {
let ch = layout.channels();
let data = (0..w as usize * h as usize * ch).map(|i| ((i * 7 + i / 5) % 251) as u8).collect();
Image::from_u8(w, h, layout, data).unwrap()
}
#[test]
fn rle_round_trips_every_layout_in_8_bits() {
for layout in [ChannelLayout::Gray, ChannelLayout::GrayA, ChannelLayout::Rgb, ChannelLayout::Rgba] {
let img = gradient(37, 11, layout);
let plan = Plan { layout, sample: SampleType::U8 };
let bytes = encode(&img, plan).unwrap();
assert!(looks_like_sgi(&bytes));
let back = decode(&bytes, &Limits::default()).unwrap();
assert_eq!(back.layout(), layout);
assert_eq!(back.data(), img.data());
}
}
#[test]
fn rle_round_trips_16_bits_with_long_runs() {
let (w, h) = (300u32, 4u32);
let samples: Vec<u16> = (0..w * h * 3).map(|i| if (i / 3) % 200 < 150 { 4242 } else { (i * 31) as u16 }).collect();
let img = Image::from_u16(w, h, ChannelLayout::Rgb, &samples).unwrap();
let bytes = encode(&img, Plan { layout: ChannelLayout::Rgb, sample: SampleType::U16 }).unwrap();
let back = decode(&bytes, &Limits::default()).unwrap();
assert_eq!(back.sample_type(), SampleType::U16);
assert_eq!(back.data(), img.data());
}
/// A hand-built verbatim file: two rows, one plane, stored bottom to top.
#[test]
fn verbatim_files_read_bottom_to_top() {
let mut f = vec![0u8; HEADER];
f[0..2].copy_from_slice(&MAGIC.to_be_bytes());
f[2] = 0; // verbatim
f[3] = 1;
f[4..6].copy_from_slice(&2u16.to_be_bytes());
f[6..8].copy_from_slice(&3u16.to_be_bytes()); // x
f[8..10].copy_from_slice(&2u16.to_be_bytes()); // y
f[10..12].copy_from_slice(&1u16.to_be_bytes()); // z
f.extend_from_slice(&[10, 11, 12, 20, 21, 22]); // bottom row first
let img = decode(&f, &Limits::default()).unwrap();
assert_eq!(img.layout(), ChannelLayout::Gray);
assert_eq!(img.data(), &[20, 21, 22, 10, 11, 12]);
}
#[test]
fn hostile_input_is_an_error_not_a_panic() {
let img = gradient(16, 8, ChannelLayout::Rgb);
let good = encode(&img, Plan { layout: ChannelLayout::Rgb, sample: SampleType::U8 }).unwrap();
// Every truncation fails cleanly.
for len in 0..good.len() {
let _ = decode(&good[..len], &Limits::default());
}
// A row table pointing outside the file, and a packet that overruns its row.
let mut bad_table = good.clone();
bad_table[HEADER..HEADER + 4].copy_from_slice(&u32::MAX.to_be_bytes());
assert!(decode(&bad_table, &Limits::default()).is_err());
let mut overrun = good.clone();
let start = be32(&overrun, HEADER) as usize;
overrun[start] = 127; // a run longer than the 16-pixel row
assert!(decode(&overrun, &Limits::default()).is_err());
// A huge declared size is stopped by the limits or the length check, before allocating.
let mut huge = good;
huge[6..8].copy_from_slice(&u16::MAX.to_be_bytes());
huge[8..10].copy_from_slice(&u16::MAX.to_be_bytes());
assert!(decode(&huge, &Limits::default()).is_err());
}
/// A 1-plane, 8-bit RLE file whose `rows` all start where `starts[i]` says, each `len` bytes long.
fn rle_file(w: u16, starts: &[u32], len: u32, payload: &[u8]) -> Vec<u8> {
let mut f = vec![0u8; HEADER];
f[0..2].copy_from_slice(&MAGIC.to_be_bytes());
f[2] = 1;
f[3] = 1;
f[4..6].copy_from_slice(&2u16.to_be_bytes());
f[6..8].copy_from_slice(&w.to_be_bytes());
f[8..10].copy_from_slice(&(starts.len() as u16).to_be_bytes());
f[10..12].copy_from_slice(&1u16.to_be_bytes());
for s in starts {
f.extend_from_slice(&s.to_be_bytes());
}
for _ in starts {
f.extend_from_slice(&len.to_be_bytes());
}
f.extend_from_slice(payload);
f
}
/// The format lets table entries share one compressed row: eight rows, one payload.
#[test]
fn rows_may_share_compressed_data() {
let table = HEADER + 8 * 8;
let f = rle_file(1, &[table as u32; 8], 3, &[0x81, 255, 0]);
assert_eq!(f.len(), table + 3);
let img = decode(&f, &Limits::default()).unwrap();
assert_eq!((img.dimensions(), img.data()), ((1, 8), &[255u8; 8][..]));
}
/// Compressed rows start after both tables: an offset into a table is refused.
#[test]
fn rows_may_not_start_inside_the_offset_tables() {
// 3x1: the tables end at 520; 515 reads the table's own bytes as a packet list.
let f = rle_file(3, &[515], 3, &[0, 0, 0, 0, 0]);
assert!(decode(&f, &Limits::default()).is_err());
}
/// FFmpeg's decoder, run as an oracle, also reads a row that ends with its last packet and no
/// zero terminator; so does this one.
#[test]
fn a_row_without_its_terminator_still_reads() {
let f = rle_file(1, &[520], 2, &[0x81, 42]);
assert_eq!(decode(&f, &Limits::default()).unwrap().data(), &[42]);
}
#[test]
fn palette_mapped_files_are_unsupported() {
let img = gradient(4, 4, ChannelLayout::Gray);
let mut f = encode(&img, Plan { layout: ChannelLayout::Gray, sample: SampleType::U8 }).unwrap();
f[104..108].copy_from_slice(&1u32.to_be_bytes());
assert!(matches!(decode(&f, &Limits::default()), Err(CodecError::Unsupported { .. })));
}
}
+281
View File
@@ -0,0 +1,281 @@
//! Sun Raster files (`.ras`, `.sun`): 1-bit, 8-bit (gray or colour-mapped), 24-bit RGB and 32-bit
//! RGBA, uncompressed or byte-encoded (RLE) on read; uncompressed on write.
//!
//! Follows the published rasterfile format: a 32-byte big-endian header, an optional colour map,
//! then rows padded to an even number of bytes. In the "standard" type (1) 24- and 32-bit pixels
//! are stored blue-first; in the "RGB" type (3) red-first. Other types (TIFF, IFF, experimental)
//! are reported as unsupported.
use crate::error::CodecError;
use crate::fidelity::Plan;
use crate::format::Format;
use crate::image::{ChannelLayout, Image, SampleType};
use crate::options::Limits;
const F: Format = Format::SunRaster;
const HEADER: usize = 32;
const MAGIC: u32 = 0x59a6_6a95;
fn bad(reason: impl std::fmt::Display) -> CodecError {
CodecError::malformed(F, reason)
}
fn be32(b: &[u8], i: usize) -> u32 {
u32::from_be_bytes([b[i], b[i + 1], b[i + 2], b[i + 3]])
}
/// Whether `b` starts with a Sun rasterfile header.
pub(crate) fn looks_like_sunraster(b: &[u8]) -> bool {
b.len() >= HEADER && be32(b, 0) == MAGIC && matches!(be32(b, 12), 1 | 8 | 24 | 32)
}
/// Expand byte-encoded data (`0x80` escapes a run) until `want` bytes are produced.
fn unrle(src: &[u8], want: usize) -> Result<Vec<u8>, CodecError> {
let mut out = Vec::new();
out.try_reserve_exact(want).map_err(|_| bad("pixel buffer is too large"))?;
let mut i = 0;
while out.len() < want {
let b = *src.get(i).ok_or_else(|| bad("truncated RLE data"))?;
i += 1;
if b != 0x80 {
out.push(b);
continue;
}
let n = *src.get(i).ok_or_else(|| bad("truncated RLE data"))?;
i += 1;
if n == 0 {
out.push(0x80);
continue;
}
let v = *src.get(i).ok_or_else(|| bad("truncated RLE data"))?;
i += 1;
let count = usize::from(n) + 1;
if count > want - out.len() {
return Err(bad("RLE run passes the end of the image"));
}
out.resize(out.len() + count, v);
}
Ok(out)
}
pub(crate) fn decode(b: &[u8], limits: &Limits) -> Result<Image, CodecError> {
if !looks_like_sunraster(b) {
return Err(bad("invalid Sun raster header"));
}
let (w, h, depth, kind, map_type, map_len) = (be32(b, 4), be32(b, 8), be32(b, 12) as usize, be32(b, 20), be32(b, 24), be32(b, 28) as usize);
if !matches!(kind, 0..=3) {
return Err(CodecError::unsupported(F, format!("Sun raster type {kind} is not supported")));
}
if map_type > 2 {
return Err(CodecError::unsupported(F, format!("Sun raster map type {map_type} is not supported")));
}
let layout = match depth {
1 | 8 if map_type != 1 => ChannelLayout::Gray,
1 | 8 | 24 => ChannelLayout::Rgb,
_ => ChannelLayout::Rgba,
};
limits.check(w, h, layout, SampleType::U8)?;
let (wu, hu) = (w as usize, h as usize);
let stride = wu.checked_mul(depth).and_then(|n| n.checked_add(15)).ok_or_else(|| bad("image size overflow"))? / 16 * 2;
let want = stride.checked_mul(hu).ok_or_else(|| bad("image size overflow"))?;
let data_at = HEADER.checked_add(map_len).ok_or_else(|| bad("colour map size overflow"))?;
let map = b.get(HEADER..data_at).ok_or_else(|| bad("truncated colour map"))?;
let rest = &b[data_at..];
let raw;
let rows: &[u8] = if kind == 2 {
raw = unrle(rest, want)?;
&raw
} else {
rest.get(..want).ok_or_else(|| bad("the file is too short for the declared image"))?
};
let channels = layout.channels();
let mut out = Vec::new();
out.try_reserve_exact(wu * hu * channels).map_err(|_| bad("pixel buffer is too large"))?;
let entries = map_len / 3;
for y in 0..hu {
let row = &rows[y * stride..(y + 1) * stride];
for x in 0..wu {
match depth {
1 => {
let bit = (row[x / 8] >> (7 - x % 8)) & 1;
if map_type == 1 {
// The bit is an index into the (two-entry or longer) colour map.
let i = usize::from(bit);
if i >= entries {
return Err(bad("pixel index is outside the colour map"));
}
out.extend_from_slice(&[map[i], map[entries + i], map[2 * entries + i]]);
} else {
// A set bit is black.
out.push(if bit == 1 { 0 } else { 255 });
}
}
8 if map_type == 1 => {
let i = usize::from(row[x]);
if i >= entries {
return Err(bad("pixel index is outside the colour map"));
}
out.extend_from_slice(&[map[i], map[entries + i], map[2 * entries + i]]);
}
8 => out.push(row[x]),
24 => {
let p = &row[3 * x..3 * x + 3];
out.extend_from_slice(&if kind == 3 { [p[0], p[1], p[2]] } else { [p[2], p[1], p[0]] });
}
_ => {
let p = &row[4 * x..4 * x + 4];
// In the standard types the first byte (X in XBGR) is unused, not alpha.
out.extend_from_slice(&if kind == 3 { [p[1], p[2], p[3], p[0]] } else { [p[3], p[2], p[1], 255] });
}
}
}
}
Image::from_u8(w, h, layout, out)
}
/// A buffer of `cap` bytes, or an error instead of an abort when the allocation cannot be made.
fn reserve(cap: usize) -> std::result::Result<Vec<u8>, std::collections::TryReserveError> {
let mut v = Vec::new();
v.try_reserve_exact(cap)?;
Ok(v)
}
/// Writes an uncompressed rasterfile: 8-bit gray, 24-bit RGB or 32-bit RGBA, in the RGB-ordered type.
pub(crate) fn encode(src: &Image, plan: Plan) -> Result<Vec<u8>, CodecError> {
let enc = |msg: String| CodecError::encode(F, msg);
let img = src.converted(plan.layout, plan.sample);
if img.sample_type() != SampleType::U8 {
return Err(enc(format!("Sun raster stores 8 bits per channel, not {:?}", img.sample_type())));
}
let depth: usize = match img.layout() {
ChannelLayout::Gray => 8,
ChannelLayout::Rgb => 24,
ChannelLayout::Rgba => 32,
l => return Err(enc(format!("no Sun raster layout for {l:?}"))),
};
let (w, h) = img.dimensions();
let (wu, hu) = (w as usize, h as usize);
let bytes = depth / 8;
let stride = (wu * bytes).next_multiple_of(2);
let len = stride.checked_mul(hu).and_then(|n| u32::try_from(n).ok()).ok_or_else(|| enc("image too large".into()))?;
let data = img.data();
if data.len() != wu * hu * bytes {
return Err(enc("the image has the wrong number of samples".into()));
}
let mut out = reserve(HEADER.checked_add(len as usize).ok_or_else(|| enc("image too large".into()))?)
.map_err(|_| enc("the image is too large to encode".into()))?;
for v in [MAGIC, w, h, depth as u32, len, 3, 0, 0] {
out.extend_from_slice(&v.to_be_bytes());
}
for y in 0..hu {
let row = &data[y * wu * bytes..(y + 1) * wu * bytes];
if depth == 32 {
// Stored alpha first.
for p in row.as_chunks::<4>().0 {
out.extend_from_slice(&[p[3], p[0], p[1], p[2]]);
}
} else {
out.extend_from_slice(row);
}
out.resize(HEADER + (y + 1) * stride, 0);
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
fn pattern(w: u32, h: u32, layout: ChannelLayout) -> Image {
let n = w as usize * h as usize * layout.channels();
Image::from_u8(w, h, layout, (0..n).map(|i| ((i * 13 + i / 3) % 253) as u8).collect()).unwrap()
}
#[test]
fn round_trips_gray_rgb_and_rgba_with_odd_widths() {
for layout in [ChannelLayout::Gray, ChannelLayout::Rgb, ChannelLayout::Rgba] {
let img = pattern(13, 7, layout);
let bytes = encode(&img, Plan { layout, sample: SampleType::U8 }).unwrap();
assert!(looks_like_sunraster(&bytes));
let back = decode(&bytes, &Limits::default()).unwrap();
assert_eq!((back.layout(), back.data()), (layout, img.data()));
}
}
fn file(depth: u32, kind: u32, map: &[u8], map_type: u32, w: u32, h: u32, data: &[u8]) -> Vec<u8> {
let mut f = Vec::new();
for v in [MAGIC, w, h, depth, data.len() as u32, kind, map_type, map.len() as u32] {
f.extend_from_slice(&v.to_be_bytes());
}
f.extend_from_slice(map);
f.extend_from_slice(data);
f
}
#[test]
fn standard_type_is_blue_first_and_rle_expands() {
let f = file(24, 1, &[], 0, 1, 1, &[1, 2, 3, 0]);
assert_eq!(decode(&f, &Limits::default()).unwrap().data(), &[3, 2, 1]);
// 0x80 0x00 is a literal 0x80; 0x80 n v is n + 1 copies of v. The 2x2 image has stride 2.
let f = file(8, 2, &[], 0, 2, 2, &[0x80, 0x00, 0x80, 0x00, 0x80, 0x01, 7]);
assert_eq!(decode(&f, &Limits::default()).unwrap().data(), &[0x80, 0x80, 7, 7]);
// A run that passes the end of the image is an error.
assert!(decode(&file(8, 2, &[], 0, 2, 2, &[0x80, 0x09, 7]), &Limits::default()).is_err());
}
#[test]
fn colour_mapped_and_one_bit_files() {
// Two entries: red plane, green plane, blue plane.
let map = [10, 20, 30, 40, 50, 60];
let f = file(8, 1, &map, 1, 2, 1, &[0, 1]);
let img = decode(&f, &Limits::default()).unwrap();
assert_eq!((img.layout(), img.data()), (ChannelLayout::Rgb, &[10, 30, 50, 20, 40, 60][..]));
let f = file(8, 1, &map, 1, 1, 1, &[5, 0]);
assert!(decode(&f, &Limits::default()).is_err());
let f = file(1, 1, &[], 0, 3, 1, &[0b1010_0000, 0]);
assert_eq!(decode(&f, &Limits::default()).unwrap().data(), &[0, 255, 0]);
}
#[test]
fn one_bit_files_with_a_colour_map_read_as_colour() {
// Two entries: red plane, green plane, blue plane; bit 0 is entry 0.
let map = [10, 20, 30, 40, 50, 60];
let f = file(1, 1, &map, 1, 3, 1, &[0b0100_0000, 0]);
let img = decode(&f, &Limits::default()).unwrap();
assert_eq!((img.layout(), img.data()), (ChannelLayout::Rgb, &[10, 30, 50, 20, 40, 60, 10, 30, 50][..]));
// A one-entry map has no entry for a set bit.
assert!(decode(&file(1, 1, &[1, 2, 3], 1, 1, 1, &[0b1000_0000, 0]), &Limits::default()).is_err());
}
#[test]
fn the_unused_first_byte_of_standard_32_bit_pixels_is_not_alpha() {
let img = decode(&file(32, 1, &[], 0, 1, 1, &[0, 30, 20, 10]), &Limits::default()).unwrap();
assert_eq!((img.layout(), img.data()), (ChannelLayout::Rgba, &[10, 20, 30, 255][..]));
// In the RGB-ordered type the first byte is the alpha the writer stores there.
let img = decode(&file(32, 3, &[], 0, 1, 1, &[7, 10, 20, 30]), &Limits::default()).unwrap();
assert_eq!(img.data(), &[10, 20, 30, 7]);
}
#[test]
fn an_output_buffer_too_large_to_allocate_is_an_error() {
assert!(reserve(usize::MAX).is_err());
assert!(reserve(16).is_ok());
}
#[test]
fn hostile_input_is_an_error_not_a_panic() {
let img = pattern(9, 5, ChannelLayout::Rgb);
let good = encode(&img, Plan { layout: ChannelLayout::Rgb, sample: SampleType::U8 }).unwrap();
for len in 0..good.len() {
let _ = decode(&good[..len], &Limits::default());
}
let mut huge = good.clone();
huge[4..8].copy_from_slice(&u32::MAX.to_be_bytes());
huge[8..12].copy_from_slice(&u32::MAX.to_be_bytes());
assert!(decode(&huge, &Limits::default()).is_err());
let mut bad_map = good;
bad_map[28..32].copy_from_slice(&u32::MAX.to_be_bytes());
assert!(decode(&bad_map, &Limits::default()).is_err());
assert!(matches!(decode(&file(24, 4, &[], 0, 1, 1, &[0; 4]), &Limits::default()), Err(CodecError::Unsupported { .. })));
}
}
+93 -2
View File
@@ -71,6 +71,18 @@ const TAG_DNG_VERSION: u16 = 50706;
/// first IFD is often a reduced preview, so this runs before decoding.
/// Reads only bounds-checked header bytes; anything malformed is "not raw".
fn camera_raw(b: &[u8]) -> bool {
camera_raw_mode(b, false)
}
/// The older LightCraft RAW/TIFF classification policy, sharing this bounded directory walk.
pub(crate) fn legacy_camera_raw(b: &[u8]) -> bool {
camera_raw_mode(b, true)
}
fn camera_raw_mode(b: &[u8], legacy: bool) -> bool {
if legacy && b.get(8..10) == Some(b"CR") {
return true;
}
let le = match b.get(0..4) {
Some(b"II*\0") => true,
Some(b"MM\0*") => false,
@@ -92,8 +104,48 @@ fn camera_raw(b: &[u8]) -> bool {
let value = |e: usize| match u16_at(e.saturating_add(2))? {
3 => u16_at(e.saturating_add(8)).map(u32::from),
4 | 13 => u32_at(e.saturating_add(8)),
1 | 7 if legacy => b.get(e.saturating_add(8)).copied().map(u32::from),
_ => None,
};
// The compatibility policy recognizes Sony lossless YCbCr only when IFD0 names Sony.
let is_sony = if legacy {
u32_at(4)
.and_then(|v| usize::try_from(v).ok())
.and_then(|ifd| {
let n = usize::from(u16_at(ifd)?);
if n > 4096 {
return None;
}
for i in 0..n {
let e = ifd.checked_add(2)?.checked_add(i.checked_mul(12)?)?;
let tag = u16_at(e)?;
let ty = u16_at(e + 2)?;
let count = u32_at(e + 4)? as usize;
if tag == 0x010f {
let matches = (|| {
let size = match ty {
1 | 2 | 6 | 7 => 1usize,
3 | 8 => 2,
4 | 9 | 11 | 13 => 4,
5 | 10 | 12 => 8,
_ => return None,
};
let len = size.checked_mul(count)?;
let start = if len <= 4 { e + 8 } else { u32_at(e + 8)? as usize };
Some(b.get(start..start.checked_add(len)?)?.get(..4)?.eq_ignore_ascii_case(b"SONY"))
})()
.unwrap_or(false);
if matches {
return Some(true);
}
}
}
Some(false)
})
.unwrap_or(false)
} else {
false
};
let mut queue: Vec<u32> = u32_at(4).into_iter().collect();
let mut visited = 0;
while let Some(ifd) = queue.pop() {
@@ -104,6 +156,44 @@ fn camera_raw(b: &[u8]) -> bool {
}
let ifd = ifd as usize;
let Some(n) = u16_at(ifd) else { continue };
if legacy
&& (n > 4096
|| (n > 0
&& ifd
.checked_add(2)
.and_then(|p| usize::from(n - 1).checked_mul(12).and_then(|v| p.checked_add(v)))
.and_then(|p| p.checked_add(8))
.is_none_or(|end| end > b.len())))
{
continue;
}
if is_sony {
let mut cfa = false;
let (mut compression, mut photo, mut samples) = (None, None, None);
let (mut saw_compression, mut saw_photo, mut saw_samples) = (false, false, false);
for i in 0..usize::from(n) {
let e = ifd.saturating_add(2 + 12 * i);
match u16_at(e) {
Some(0x828e) => cfa = true,
Some(0x0103) if !saw_compression => {
saw_compression = true;
compression = value(e);
}
Some(0x0106) if !saw_photo => {
saw_photo = true;
photo = value(e);
}
Some(0x0115) if !saw_samples => {
saw_samples = true;
samples = value(e);
}
_ => {}
}
}
if cfa && compression == Some(7) && photo == Some(6) && samples == Some(3) {
return true;
}
}
for i in 0..usize::from(n) {
let e = ifd.saturating_add(2 + 12 * i);
let (Some(tag), Some(count)) = (u16_at(e), u32_at(e.saturating_add(4))) else {
@@ -111,6 +201,7 @@ fn camera_raw(b: &[u8]) -> bool {
};
match tag {
TAG_DNG_VERSION => return true,
0x0103 if legacy && matches!(value(e), Some(32767 | 32769 | 32770 | 34316 | 34713 | 65000 | 65535)) => return true,
TAG_PHOTOMETRIC if matches!(value(e), Some(PHOTOMETRIC_CFA | PHOTOMETRIC_LINEAR_RAW)) => {
return true;
}
@@ -125,7 +216,7 @@ fn camera_raw(b: &[u8]) -> bool {
_ => {}
}
}
if let Some(next) = u32_at(ifd.saturating_add(2 + 12 * usize::from(n))).filter(|&o| o != 0) {
if !legacy && let Some(next) = u32_at(ifd.saturating_add(2 + 12 * usize::from(n))).filter(|&o| o != 0) {
queue.push(next);
}
}
@@ -171,7 +262,7 @@ mod tags;
mod ycbcr;
pub use decode::TiffBands;
pub(crate) use decode::{decode, decode_page, has_unretained_samples, orientation};
pub(crate) use decode::{decode, decode_page, has_unretained_samples, orientation, probe};
pub(crate) use encode::encode;
#[cfg(test)]
use encode::{Extras, try_with_capacity};
@@ -13,6 +13,17 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits) -> Result<Image, CodecError>
decode_page(bytes, None, limits)
}
/// The stored size and Orientation (1–8) of the page [`decode`] opens, from its directory alone:
/// the same page choice and refusals (camera raw, no directory), checked against `limits`,
/// without reading any strips or tiles.
pub(crate) fn probe(bytes: &[u8], limits: &Limits) -> Result<(u32, u32, u16), CodecError> {
let (file, dir, _) = open_page(bytes, None)?;
let side = |tag: u16| file.tag_uint(&dir, tag).and_then(|v| u32::try_from(v).ok()).unwrap_or(0);
let (w, h) = (side(ifd::TAG_WIDTH), side(ifd::TAG_HEIGHT));
limits.check_size(w, h)?;
Ok((w, h, ifd::orientation_of(&file, &dir)))
}
/// Decodes page `page` of [`ifd::tiff_info`]'s list (`None`: the default page). The image
/// carries the page's metadata, a [`DecodeWarning::MorePages`] when the file holds more
/// full-resolution pages, and a [`DecodeWarning::Truncated`] when image data is cut off.
+32 -2
View File
@@ -45,6 +45,10 @@ pub enum Format {
Iff,
/// JPEG 2000 (JP2 container or raw J2K codestream): read-only; see [`ASYMMETRIC_EXCEPTIONS`].
Jp2,
/// Sun Raster (`.ras`): 1-, 8-, 24- and 32-bit, uncompressed or RLE.
SunRaster,
/// SGI image files (`.sgi`, `.rgb`, `.bw`): 8- and 16-bit gray, gray + alpha, RGB and RGBA.
Sgi,
/// JPEG-LS (`.jls`): lossless and near-lossless gray and RGB.
JpegLs,
}
@@ -118,7 +122,7 @@ const TIFF_LAYOUTS: &[ChannelLayout] = &[L::Gray, L::GrayA, L::Rgb, L::Rgba, L::
impl Format {
/// Every format known to the crate (enabled or not).
pub const ALL: [Format; 25] = [
pub const ALL: [Format; 27] = [
Format::Png,
Format::Jpeg,
Format::Jp2,
@@ -143,6 +147,8 @@ impl Format {
Format::Cineon,
Format::Dicom,
Format::Iff,
Format::SunRaster,
Format::Sgi,
Format::JpegLs,
];
@@ -182,6 +188,8 @@ impl Format {
Format::Dicom => "DICOM",
Format::Iff => "Amiga IFF",
Format::Jp2 => "JPEG 2000",
Format::SunRaster => "Sun Raster",
Format::Sgi => "SGI",
Format::JpegLs => "JPEG-LS",
}
}
@@ -214,6 +222,8 @@ impl Format {
Format::Dicom => &["dcm", "dicom", "dc3"],
Format::Iff => &["iff", "lbm", "ilbm"],
Format::Jp2 => &["jp2", "j2k", "j2c", "jpf", "jpx"],
Format::SunRaster => &["ras", "sun"],
Format::Sgi => &["sgi", "rgb", "rgba", "bw"],
Format::JpegLs => &["jls"],
}
}
@@ -244,6 +254,8 @@ impl Format {
Format::Dicom => "application/dicom",
Format::Iff => "image/x-iff",
Format::Jp2 => "image/jp2",
Format::SunRaster => "image/x-sun-raster",
Format::Sgi => "image/x-sgi",
Format::JpegLs => "image/jls",
}
}
@@ -251,7 +263,7 @@ impl Format {
/// Maximum encodable dimensions, if the format has a hard limit.
pub fn max_dimensions(self) -> Option<(u32, u32)> {
match self {
Format::Jpeg | Format::Gif | Format::Iff | Format::JpegLs => Some((65535, 65535)),
Format::Jpeg | Format::Gif | Format::Iff | Format::Sgi | Format::JpegLs => Some((65535, 65535)),
Format::WebP => Some((16384, 16384)),
Format::Ico => Some((256, 256)),
_ => None,
@@ -321,6 +333,16 @@ pub fn caps(format: Format) -> FormatCaps {
Format::Cineon => FormatCaps { depths: &[S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base },
Format::Dicom => FormatCaps { read: true, write: false, depths: &[S::U8, S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base },
Format::Iff => FormatCaps { depths: &[S::U8], layouts: &[L::Rgb, L::Rgba], alpha: true, ..base },
Format::SunRaster => FormatCaps {
read: cfg!(feature = "sunraster"),
write: cfg!(feature = "sunraster"),
layouts: &[L::Gray, L::Rgb, L::Rgba],
alpha: true,
..base
},
Format::Sgi => {
FormatCaps { read: cfg!(feature = "sgi"), write: cfg!(feature = "sgi"), depths: &[S::U8, S::U16], layouts: RGB_GRAY, alpha: true, ..base }
}
Format::JpegLs => FormatCaps { depths: &[S::U8, S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base },
Format::Jp2 => FormatCaps { read: true, write: false, layouts: &[L::Gray, L::GrayA, L::Rgb, L::Rgba, L::Cmyk, L::CmykA], icc: true, ..base },
}
@@ -403,6 +425,14 @@ pub fn detect(bytes: &[u8]) -> Option<Format> {
if b.starts_with(b"/* XPM */") || b.starts_with(b"! XPM2\n") || b.starts_with(b"! XPM2\r\n") {
return Some(Format::Xpm);
}
#[cfg(feature = "sunraster")]
if crate::codecs::sunraster::looks_like_sunraster(b) {
return Some(Format::SunRaster);
}
#[cfg(feature = "sgi")]
if crate::codecs::sgi::looks_like_sgi(b) {
return Some(Format::Sgi);
}
if crate::codecs::pcx::looks_like_pcx(b) {
return Some(Format::Pcx);
}
+98
View File
@@ -0,0 +1,98 @@
//! A JPEG's headers and four-component samples read the way `zune-jpeg` and libjpeg-style readers
//! give them, for apps that keep a CMYK JPEG's inks: the component count and ICC profile from the
//! headers alone, and the ink amounts of a CMYK or YCCK JPEG.
//!
//! [`crate::decode`] reads the same files differently, on purpose: it keeps the samples of a
//! four-component JPEG without Adobe's marker as they are, rounds its YCCK conversion, and hands
//! arithmetic-coded, lossless and 12-bit files to its whole-coefficient decoder. The readers here
//! follow the older convention instead, so a document that kept a CMYK JPEG's inks that way reads
//! the same: every four-component file is taken to hold Adobe's inverted samples, YCCK is turned
//! into CMYK as libjpeg does (truncating), and only the 8-bit Huffman-coded frames `zune-jpeg`
//! decodes are read.
use zune_core::bytestream::ZCursor;
use zune_core::colorspace::ColorSpace;
use zune_core::options::DecoderOptions;
use zune_jpeg::JpegDecoder;
use crate::Format;
use crate::error::CodecError;
use crate::image::{ChannelLayout, Image, SampleType};
use crate::options::Limits;
/// A decoder of `bytes` whose headers have been read, with the default options (16384 pixels a
/// side at most); `None` when they aren't a JPEG or its headers can't be read.
fn headers(bytes: &[u8]) -> Option<JpegDecoder<ZCursor<&[u8]>>> {
if !bytes.starts_with(&[0xFF, 0xD8]) {
return None;
}
let mut d = JpegDecoder::new_with_options(ZCursor::new(bytes), DecoderOptions::default());
d.decode_headers().ok()?;
Some(d)
}
/// The number of colour components the frame header of JPEG `bytes` declares, whatever it is;
/// `None` when they aren't a JPEG whose headers can be read.
pub fn jpeg_components(bytes: &[u8]) -> Option<usize> {
Some(usize::from(headers(bytes)?.info()?.components))
}
/// The width and height the frame header of JPEG `bytes` declares, for any colour type; `None`
/// when they aren't a JPEG whose headers can be read (which includes sides past 16384 pixels).
pub fn jpeg_dimensions(bytes: &[u8]) -> Option<(u32, u32)> {
let (w, h) = headers(bytes)?.dimensions()?;
Some((u32::try_from(w).ok()?, u32::try_from(h).ok()?))
}
/// The ICC profile of JPEG `bytes`, put together from all its numbered `ICC_PROFILE` APP2
/// segments; `None` without one, when it is empty, and when the headers can't be read.
pub fn jpeg_icc_profile(bytes: &[u8]) -> Option<Vec<u8>> {
headers(bytes)?.icc_profile().filter(|icc| !icc.is_empty())
}
/// The ink amounts of a four-component (CMYK or YCCK) JPEG: C, M, Y and K bytes a pixel, 0 for
/// no ink, as a [`ChannelLayout::Cmyk`] image. The samples are taken to be Adobe's inverted ones
/// whether or not the file has Adobe's marker; YCCK is first turned into CMYK as libjpeg does
/// (the complement of the YCbCr colour's RGB, each value truncated).
///
/// Errors: bytes that aren't a JPEG with readable headers ([`CodecError::InvalidImage`]); one of
/// other than four components, or a frame `zune-jpeg` doesn't decode, such as an arithmetic-coded
/// or 12-bit one ([`CodecError::Unsupported`]); a size past `limits`; scan data too short for the
/// frame (under a byte for every 2048 pixels, refused before anything is allocated); and data that
/// doesn't decode,
/// including frames `zune-jpeg` panics on, which are caught.
pub fn decode_jpeg_inks(bytes: &[u8], limits: &Limits) -> Result<Image, CodecError> {
let bad = |m: &str| CodecError::InvalidImage(format!("JPEG inks: {m}"));
let mut d = headers(bytes).ok_or_else(|| bad("not a JPEG with readable headers"))?;
let space = d.input_colorspace().ok_or_else(|| bad("no colour space"))?;
let components = d.info().map_or(0, |i| i.components);
if components != 4 || !matches!(space, ColorSpace::CMYK | ColorSpace::YCCK) {
return Err(CodecError::unsupported(Format::Jpeg, format!("inks of a {components}-component {space:?} image")));
}
let (w, h) = d.dimensions().ok_or_else(|| bad("no size"))?;
let (w, h) = (u32::try_from(w).map_err(|_| bad("width"))?, u32::try_from(h).map_err(|_| bad("height"))?);
limits.check(w, h, ChannelLayout::Cmyk, SampleType::U8)?;
if crate::codecs::jpeg::scan_too_short(bytes, w, h) {
return Err(bad("the scan data is too short for the frame it declares"));
}
d.set_options(DecoderOptions::default().jpeg_set_out_colorspace(space));
// zune-jpeg 0.5 asserts on some subsampled four-component frames whose first component isn't
// the largest (a scratch row too short for its upsampler): that is an error here, not a panic.
let decoded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| d.decode()));
let mut data = decoded
.map_err(|_| CodecError::unsupported(Format::Jpeg, "inks: zune-jpeg failed on this file"))?
.map_err(|e| CodecError::unsupported(Format::Jpeg, format!("inks: {e}")))?;
for p in data.as_chunks_mut::<4>().0 {
if space == ColorSpace::YCCK {
let [y, cb, cr] = [p[0], p[1], p[2]].map(f32::from);
// libjpeg's YCCK → CMYK: the complement of the YCbCr colour's RGB.
p[0] = (434.456 - y - 1.402 * cr).clamp(0.0, 255.0) as u8;
p[1] = (119.541 - y + 0.344 * cb + 0.714 * cr).clamp(0.0, 255.0) as u8;
p[2] = (481.816 - y - 1.772 * cb).clamp(0.0, 255.0) as u8;
}
for v in p.iter_mut() {
*v = 255 - *v;
}
}
Image::from_u8(w, h, ChannelLayout::Cmyk, data)
}
+64 -5
View File
@@ -26,6 +26,7 @@ mod fidelity;
mod format;
mod image;
mod inspection;
pub mod jpeg_inks;
pub mod jpeg_sef;
mod lab;
mod options;
@@ -35,6 +36,7 @@ pub mod resolution;
pub mod source_settings;
pub mod tiff_probe;
pub mod web;
pub mod webp_probe;
pub use crate::allocation::{probe_allocation_size, probe_stored_size};
pub use crate::cicp::Cicp;
pub use crate::cicp::ContentLightLevel;
@@ -67,6 +69,10 @@ pub use crate::resolution::{exif_resolution, export_exif, export_xmp, photoshop_
pub use crate::source_settings::{WebpEncoding, jpeg_quality, webp_encoding};
pub use half::f16;
#[cfg(feature = "sgi")]
use crate::codecs::sgi;
#[cfg(feature = "sunraster")]
use crate::codecs::sunraster;
use crate::codecs::{dds, dicom, dpx, exr, fits, heif, iff, jp2, jpeg, jpegls, jxl, pcx, png, pnm, tiff, via_image, webp, xpm};
/// Detect the format and decode with default [`Limits`].
@@ -106,6 +112,26 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res
Format::Cineon => dpx::decode_cineon(bytes, l),
Format::Dicom => dicom::decode(bytes, l),
Format::Iff => iff::decode(bytes, l),
Format::SunRaster => {
#[cfg(feature = "sunraster")]
{
sunraster::decode(bytes, l)
}
#[cfg(not(feature = "sunraster"))]
{
Err(CodecError::unsupported(format, "Sun Raster support requires the sunraster feature"))
}
}
Format::Sgi => {
#[cfg(feature = "sgi")]
{
sgi::decode(bytes, l)
}
#[cfg(not(feature = "sgi"))]
{
Err(CodecError::unsupported(format, "SGI support requires the sgi feature"))
}
}
Format::JpegLs => jpegls::decode(bytes, l),
Format::OpenExr => exr::decode(bytes, l),
Format::Heif => heif::decode(bytes, l, opts.keep_orientation),
@@ -213,11 +239,12 @@ pub fn decode_png_rows<E: From<CodecError>>(
}
/// The upright pixel size [`decode`] would return, read from the file's header without decoding
/// pixels, for WebP, OpenEXR (flat), AVIF (with `avif-decode`) and the GIF/BMP/TGA/ICO/QOI/HDR family. It runs the same
/// header reads and size checks as the decoder (default [`Limits`]) and swaps the size for an
/// EXIF orientation of 5–8 (AVIF: its container's `clap` and `irot`), so wherever [`decode`]
/// succeeds the two agree. Other formats (and deep EXR) return [`CodecError::Unsupported`]: decode
/// those to learn their size.
/// pixels, for PNG, JPEG, TIFF (the page [`decode`] opens), WebP, OpenEXR (flat), AVIF (with
/// `avif-decode`) and the GIF/BMP/TGA/ICO/QOI/HDR family. It runs the same header reads and size
/// checks as the decoder (default [`Limits`]) and swaps the size for an orientation of 5–8 (EXIF,
/// the TIFF page's own tag, or AVIF's container `clap` and `irot`), so wherever [`decode`]
/// succeeds the two agree. Other formats (and deep EXR) return [`CodecError::Unsupported`]:
/// decode those to learn their size.
pub fn probe_size(bytes: &[u8]) -> Result<(u32, u32), CodecError> {
let format = detect(bytes).ok_or(CodecError::UnknownFormat)?;
if !caps(format).read {
@@ -225,6 +252,12 @@ pub fn probe_size(bytes: &[u8]) -> Result<(u32, u32), CodecError> {
}
let l = &Limits::default();
let (w, h, exif) = match format {
Format::Png => png::probe(bytes, l)?,
Format::Jpeg => jpeg::probe(bytes, l)?,
Format::Tiff => {
let (w, h, o) = tiff::probe(bytes, l)?;
return Ok(if (5..=8).contains(&o) { (h, w) } else { (w, h) });
}
Format::WebP => webp::probe(bytes, l)?,
Format::OpenExr => exr::probe(bytes, l).map(|(w, h)| (w, h, None))?,
#[cfg(all(feature = "avif-decode", not(target_arch = "wasm32")))]
@@ -238,6 +271,12 @@ pub fn probe_size(bytes: &[u8]) -> Result<(u32, u32), CodecError> {
Ok(if (5..=8).contains(&o) { (h, w) } else { (w, h) })
}
/// Classify classic TIFF camera RAWs using LightCraft's historical bounded heuristic.
/// This compatibility policy differs from the decoder's own RAW rejection policy.
pub fn legacy_tiff_raw_hint(bytes: &[u8]) -> bool {
tiff::legacy_camera_raw(bytes)
}
/// A JPEG's frame as its header declares it, read without decoding pixels: the stored size, the
/// layout [`decode`] returns, whether [`decode`] hands the file to its whole-coefficient
/// decoder (arithmetic-coded, lossless or 12-bit frames), whose working set is several times the
@@ -481,6 +520,26 @@ pub fn encode(image: &Image, format: Format, opts: &EncodeOptions) -> Result<Vec
Format::Cineon => dpx::encode_cineon(image, plan),
Format::Dicom => Err(CodecError::unsupported(format, "DICOM is import-only; save as TIFF or PNG instead")),
Format::Iff => iff::encode(image, plan),
Format::SunRaster => {
#[cfg(feature = "sunraster")]
{
sunraster::encode(image, plan)
}
#[cfg(not(feature = "sunraster"))]
{
Err(CodecError::unsupported(format, "Sun Raster support requires the sunraster feature"))
}
}
Format::Sgi => {
#[cfg(feature = "sgi")]
{
sgi::encode(image, plan)
}
#[cfg(not(feature = "sgi"))]
{
Err(CodecError::unsupported(format, "SGI support requires the sgi feature"))
}
}
Format::JpegLs => jpegls::encode(image, plan),
Format::OpenExr => exr::encode(image, plan, opts),
Format::Heif => Err(CodecError::unsupported(format, "encoding is not available in this build")),
+1 -1
View File
@@ -88,7 +88,7 @@ pub fn header_with_max_pixels(b: &[u8], max_pixels: u64) -> Result<(u32, u32, u1
if !after_idat {
return Err(err("truncated image data"));
}
Ok((w, h, exif.map_or(1, crate::exif_orientation)))
Ok((w, h, exif.map_or(1, craft_metadata::compat_orientation::orientation)))
}
/// Container color declarations preceding image data. Raw values leave color policy to callers.
+59
View File
@@ -0,0 +1,59 @@
//! Header-only WebP inspection with image-chunk completeness checks.
use crate::{CodecError, Format};
/// Stored width, height and EXIF orientation without decoding pixels. The image payload must
/// fit in the input, but its compressed contents are not validated. Uses the default decode limits.
pub fn header(bytes: &[u8]) -> Result<(u32, u32, u16), CodecError> {
let err = |s: &str| CodecError::malformed(Format::WebP, s);
let mut p = 12usize;
let mut has_image = false;
let mut exif = None;
while let Some(h) = bytes.get(p..p.saturating_add(8)) {
let size = u32::from_le_bytes([h[4], h[5], h[6], h[7]]) as usize;
let end = p.saturating_add(8).saturating_add(size);
if matches!(&h[..4], b"VP8 " | b"VP8L" | b"ANMF") {
if end > bytes.len() {
return Err(err("truncated image data"));
}
has_image = true;
} else if &h[..4] == b"EXIF" && end <= bytes.len() && exif.is_none() {
exif = bytes.get(p + 8..end);
}
p = end.saturating_add(size & 1);
}
if !has_image {
return Err(err("no image data"));
}
let (w, h) = crate::probe_size(bytes)?;
let orientation = exif.map_or(1, craft_metadata::compat_orientation::orientation);
let (w, h) = if (5..=8).contains(&orientation) { (h, w) } else { (w, h) };
Ok((w, h, orientation))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn preserves_stored_dimensions_with_exif_orientation() {
let mut image = crate::Image::from_u8(2, 3, crate::ChannelLayout::Rgb, vec![42; 18]).unwrap();
image.meta.exif = Some(b"II\x2a\x00\x08\x00\x00\x00\x01\x00\x12\x01\x03\x00\x01\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00".to_vec());
let mut bytes = crate::encode(&image, Format::WebP, &crate::EncodeOptions::default()).unwrap();
// The writer correctly makes exported pixels upright; simulate a camera file carrying orientation 6.
let exif = bytes.windows(4).position(|v| v == b"EXIF").unwrap();
let tiff = exif + 8;
bytes[tiff + 18] = 6;
assert_eq!(header(&bytes).unwrap(), (2, 3, 6));
}
#[test]
fn rejects_missing_and_truncated_image_chunks() {
let mut bytes = b"RIFF\x00\x00\x00\x00WEBP".to_vec();
assert!(header(&bytes).is_err());
bytes.extend_from_slice(b"VP8 ");
bytes.extend_from_slice(&10u32.to_le_bytes());
bytes.push(0);
assert!(header(&bytes).is_err());
}
}
@@ -62,6 +62,10 @@ fn default_build_formats_all_symmetric() {
Format::Dpx,
Format::Cineon,
Format::Iff,
#[cfg(feature = "sunraster")]
Format::SunRaster,
#[cfg(feature = "sgi")]
Format::Sgi,
Format::JpegLs,
];
for f in expected {
@@ -224,6 +228,23 @@ fn detect_magic_iff() {
assert_eq!(detect(b"FORM\0\0\0\x14AIFF"), None);
assert!(decode_as(Format::Iff, b"FORM\0\0\0\x14ILBM").is_err());
}
#[cfg(feature = "sgi")]
#[test]
fn detect_magic_sgi() {
use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode};
let img = Image::from_u8(5, 3, ChannelLayout::Rgba, (0..60).collect()).unwrap();
let bytes = encode(&img, Format::Sgi, &EncodeOptions::default()).unwrap();
assert_eq!(detect(&bytes), Some(Format::Sgi));
assert_eq!(from_extension("scan.RGB"), Some(Format::Sgi));
assert_eq!(from_extension("scan.bw"), Some(Format::Sgi));
assert_eq!(decode(&bytes).unwrap().data(), img.data());
assert!(!ASYMMETRIC_EXCEPTIONS.iter().any(|(f, _)| *f == Format::Sgi));
assert_eq!(Format::Sgi.max_dimensions(), Some((65535, 65535)));
// The magic number alone, without a full header, is not SGI.
assert_eq!(detect(&[0x01, 0xda, 0, 1, 0, 2]), None);
assert!(decode_as(Format::Sgi, &[0x01, 0xda]).is_err());
}
#[test]
fn detect_magic_jpeg_ls() {
use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode};
@@ -239,6 +260,20 @@ fn detect_magic_jpeg_ls() {
assert!(decode_as(Format::JpegLs, &[0xFF, 0xD8, 0xFF, 0xF7]).is_err());
}
#[cfg(feature = "sunraster")]
#[test]
fn detect_magic_sun_raster() {
use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode};
let img = Image::from_u8(5, 3, ChannelLayout::Rgb, (0..45).collect()).unwrap();
let bytes = encode(&img, Format::SunRaster, &EncodeOptions::default()).unwrap();
assert_eq!(detect(&bytes), Some(Format::SunRaster));
assert_eq!(from_extension("scan.RAS"), Some(Format::SunRaster));
assert_eq!(decode(&bytes).unwrap().data(), img.data());
assert!(!ASYMMETRIC_EXCEPTIONS.iter().any(|(f, _)| *f == Format::SunRaster));
assert_eq!(detect(&[0x59, 0xa6, 0x6a, 0x95]), None);
assert!(decode_as(Format::SunRaster, &[0x59, 0xa6, 0x6a, 0x95]).is_err());
}
#[test]
fn detect_magic_avif() {
assert_eq!(detect(b"\0\0\0\x1cftypavif\0\0\0\0avifmif1miaf"), Some(Format::Avif));
@@ -328,3 +363,28 @@ fn extensions_unique_across_formats() {
all.dedup();
assert_eq!(all.len(), n);
}
#[cfg(not(feature = "sgi"))]
#[test]
fn sgi_requires_explicit_feature() {
use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode};
assert!(!caps(Format::Sgi).read && !caps(Format::Sgi).write);
assert!(decode_as(Format::Sgi, &[]).is_err());
let image = Image::from_u8(1, 1, ChannelLayout::Gray, vec![0]).unwrap();
assert!(encode(&image, Format::Sgi, &EncodeOptions::default()).is_err());
let mut header = vec![0; 512];
header[..12].copy_from_slice(&[1, 218, 0, 1, 0, 2, 0, 1, 0, 1, 0, 1]);
assert_eq!(detect(&header), None);
}
#[cfg(not(feature = "sunraster"))]
#[test]
fn sun_raster_disabled_by_default() {
use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode};
let bytes = [0x59, 0xa6, 0x6a, 0x95, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0];
assert_ne!(detect(&bytes), Some(Format::SunRaster));
assert!(!caps(Format::SunRaster).read && !caps(Format::SunRaster).write);
assert!(decode_as(Format::SunRaster, &bytes).is_err());
let image = Image::from_u8(1, 1, ChannelLayout::Gray, vec![42]).unwrap();
assert!(encode(&image, Format::SunRaster, &EncodeOptions::default()).is_err());
}
+215
View File
@@ -0,0 +1,215 @@
//! `jpeg_inks`: four-component JPEGs read as Adobe-inverted inks, as `zune-jpeg`-based readers
//! kept them, beside `decode`'s own reading.
use craft_image_codecs::jpeg_inks::{decode_jpeg_inks, jpeg_components, jpeg_dimensions, jpeg_icc_profile};
use craft_image_codecs::*;
/// A `w` × `h` four-component JPEG of `samples` at quality 100, written by `jpeg-encoder` as
/// `ty`, with ICC profile `icc` when given.
fn four(w: u16, h: u16, samples: &[u8], ty: jpeg_encoder::ColorType, icc: Option<&[u8]>) -> Vec<u8> {
let mut out = Vec::new();
let mut enc = jpeg_encoder::Encoder::new(&mut out, 100);
enc.set_sampling_factor(jpeg_encoder::SamplingFactor::R_4_4_4);
if let Some(icc) = icc {
enc.add_icc_profile(icc).unwrap();
}
enc.encode(samples, w, h, ty).unwrap();
out
}
/// Flat 8×8 blocks of ink, so the coding keeps each block's value.
fn blocks(w: usize, h: usize) -> Vec<u8> {
(0..w * h)
.flat_map(|i| {
let b = ((i % w) / 8 + (i / w) / 8 * 7) as u8;
[b.wrapping_mul(37), b.wrapping_mul(91), 255 - b.wrapping_mul(13), b.wrapping_mul(53)]
})
.collect()
}
/// The segments of `bytes` up to the scan without Adobe's APP14 marker.
fn without_adobe(bytes: &[u8]) -> Vec<u8> {
let mut out = bytes[..2].to_vec();
let mut at = 2;
loop {
let marker = bytes[at + 1];
if marker == 0xDA {
out.extend_from_slice(&bytes[at..]);
return out;
}
let len = usize::from(u16::from_be_bytes([bytes[at + 2], bytes[at + 3]]));
if !(marker == 0xEE && &bytes[at + 4..at + 9] == b"Adobe") {
out.extend_from_slice(&bytes[at..at + 2 + len]);
}
at += 2 + len;
}
}
/// Adobe CMYK reads to the inks it was written with, as `decode` reads it; without the marker
/// the samples are still taken as inverted (where `decode` keeps them as they are), and YCCK
/// turns into the same inks closely, truncating where `decode` rounds.
#[test]
fn four_component_jpegs_read_as_inverted_inks() {
let (w, h) = (32u16, 16u16);
let inks = blocks(usize::from(w), usize::from(h));
let cmyk = four(w, h, &inks, jpeg_encoder::ColorType::Cmyk, None);
let read = decode_jpeg_inks(&cmyk, &Limits::default()).unwrap();
assert_eq!((read.dimensions(), read.layout()), ((32, 16), ChannelLayout::Cmyk));
assert!(read.data().iter().zip(&inks).all(|(a, b)| a.abs_diff(*b) <= 2), "Adobe CMYK keeps its inks");
assert_eq!(read.data(), decode(&cmyk).unwrap().data(), "as decode reads Adobe CMYK");
// No marker: inverted all the same, where decode keeps the stored samples.
let bare = without_adobe(&cmyk);
assert_ne!(bare, cmyk);
let unmarked = decode_jpeg_inks(&bare, &Limits::default()).unwrap();
assert_eq!(unmarked.data(), read.data());
let kept = decode(&bare).unwrap();
assert!(kept.data().iter().zip(read.data()).all(|(k, r)| *k == 255 - *r), "decode keeps them as stored");
// YCCK: libjpeg's conversion, truncated.
let ycck = four(w, h, &inks, jpeg_encoder::ColorType::CmykAsYcck, None);
let read = decode_jpeg_inks(&ycck, &Limits::default()).unwrap();
assert!(read.data().iter().zip(&inks).all(|(a, b)| a.abs_diff(*b) <= 3), "YCCK keeps its inks closely");
let rounded = decode(&ycck).unwrap();
let differ = read.data().iter().zip(rounded.data()).filter(|(a, b)| a != b).count();
assert!(read.data().iter().zip(rounded.data()).all(|(a, b)| a.abs_diff(*b) <= 1), "within a step of decode's rounding");
assert!(differ > 0, "truncation differs from rounding somewhere");
}
/// The headers alone give the component count of any JPEG and its ICC profile, put together from
/// its numbered segments.
#[test]
fn headers_give_components_and_icc() {
let icc: Vec<u8> = (0..200_000u32).map(|i| (i * 7 % 251) as u8).collect();
let cmyk = four(8, 8, &[9; 256], jpeg_encoder::ColorType::Cmyk, Some(&icc));
assert_eq!((jpeg_components(&cmyk), jpeg_dimensions(&cmyk)), (Some(4), Some((8, 8))));
assert_eq!(jpeg_icc_profile(&cmyk).as_deref(), Some(icc.as_slice()), "three APP2 segments");
let gray = encode(&Image::from_u8(8, 8, ChannelLayout::Gray, vec![50; 64]).unwrap(), Format::Jpeg, &EncodeOptions::default()).unwrap();
assert_eq!((jpeg_components(&gray), jpeg_icc_profile(&gray)), (Some(1), None));
for not in [&b""[..], b"\xFF\xD8", b"\x89PNG\r\n\x1a\n"] {
assert_eq!((jpeg_components(not), jpeg_icc_profile(not), jpeg_dimensions(not)), (None, None, None));
}
}
/// A frame header past 16384 pixels a side, `zune-jpeg`'s default, isn't read.
#[test]
fn sides_past_zunes_default_are_not_read() {
let mut gray = encode(&Image::from_u8(8, 8, ChannelLayout::Gray, vec![50; 64]).unwrap(), Format::Jpeg, &EncodeOptions::default()).unwrap();
let sof = gray.windows(2).position(|w| w == [0xFF, 0xC0]).unwrap();
gray[sof + 5..sof + 9].copy_from_slice(&[0x40, 0x00, 0x40, 0x00]);
assert_eq!(jpeg_dimensions(&gray), Some((16384, 16384)));
gray[sof + 7..sof + 9].copy_from_slice(&[0x40, 0x01]);
assert_eq!((jpeg_dimensions(&gray), jpeg_components(&gray)), (None, None));
}
/// What isn't a four-component image `zune-jpeg` decodes is refused: other component counts, an
/// arithmetic-coded or 12-bit frame (`decode` hands those to craft-jpeg), a size past the limits,
/// and data that isn't a JPEG.
#[test]
fn other_jpegs_are_refused() {
let gray = encode(&Image::from_u8(8, 8, ChannelLayout::Gray, vec![50; 64]).unwrap(), Format::Jpeg, &EncodeOptions::default()).unwrap();
assert!(matches!(decode_jpeg_inks(&gray, &Limits::default()), Err(CodecError::Unsupported { .. })));
let cmyk = four(16, 8, &[40; 16 * 8 * 4], jpeg_encoder::ColorType::Cmyk, None);
let sof = cmyk.windows(2).position(|w| w == [0xFF, 0xC0]).unwrap();
for (marker, precision) in [(0xC9u8, 8u8), (0xC1, 12)] {
let mut other = cmyk.clone();
other[sof + 1] = marker;
other[sof + 4] = precision;
assert!(decode_jpeg_inks(&other, &Limits::default()).is_err(), "SOF {marker:#x}, {precision} bits");
}
// A tiny file declaring a large frame is refused before zune-jpeg allocates its output.
let mut huge = cmyk.clone();
huge[sof + 5..sof + 9].copy_from_slice(&[0x20, 0x00, 0x20, 0x00]);
assert!(matches!(decode_jpeg_inks(&huge, &Limits::default()), Err(CodecError::InvalidImage(m)) if m.contains("too short")));
let tight = Limits { max_pixels: 100, ..Limits::default() };
assert!(matches!(decode_jpeg_inks(&cmyk, &tight), Err(CodecError::LimitExceeded(_))));
assert!(matches!(decode_jpeg_inks(b"not a jpeg", &Limits::default()), Err(CodecError::InvalidImage(_))));
assert!(decode_jpeg_inks(&cmyk[..12], &Limits::default()).is_err(), "headers cut short");
}
/// The ink reader four-component JPEGs were read with before (vectorcraft's `jpeg_inks`, on
/// `zune-jpeg` as this crate uses it), kept as the reference the readers here match.
fn reference_inks(bytes: &[u8]) -> Option<(usize, usize, Vec<u8>)> {
use zune_core::{bytestream::ZCursor, colorspace::ColorSpace, options::DecoderOptions};
let mut d = zune_jpeg::JpegDecoder::new_with_options(ZCursor::new(bytes), DecoderOptions::default());
d.decode_headers().ok()?;
let space = d.input_colorspace()?;
if d.info()?.components != 4 || !matches!(space, ColorSpace::CMYK | ColorSpace::YCCK) {
return None;
}
let (w, h) = d.dimensions()?;
d.set_options(DecoderOptions::default().jpeg_set_out_colorspace(space));
let mut data = d.decode().ok()?;
for p in data.as_chunks_mut::<4>().0 {
if space == ColorSpace::YCCK {
let [y, cb, cr] = [p[0], p[1], p[2]].map(f32::from);
p[0] = (434.456 - y - 1.402 * cr).clamp(0.0, 255.0) as u8;
p[1] = (119.541 - y + 0.344 * cb + 0.714 * cr).clamp(0.0, 255.0) as u8;
p[2] = (481.816 - y - 1.772 * cb).clamp(0.0, 255.0) as u8;
}
for v in p.iter_mut() {
*v = 255 - *v;
}
}
Some((w, h, data))
}
/// Over CMYK and YCCK JPEGs of many sizes, samplings, qualities and pictures, with and without
/// Adobe's marker, baseline and progressive, the inks and component counts read the same as the
/// reference reader.
#[test]
fn matches_the_reference_reader() {
use jpeg_encoder::{ColorType, Encoder, SamplingFactor};
let (mut seen, mut refused) = (0, 0);
for (i, (w, h)) in [(1u16, 1u16), (7, 3), (8, 8), (17, 9), (33, 31), (64, 5), (100, 77)].into_iter().enumerate() {
let (wu, n) = (usize::from(w), usize::from(w) * usize::from(h));
let noise: Vec<u8> = (0..n * 4).map(|k| ((k as u64).wrapping_mul(2_654_435_761).wrapping_add(i as u64 * 97) >> 7) as u8).collect();
let ramp: Vec<u8> = (0..n * 4).map(|k| (k / 4 % wu * 255 / wu.max(2)) as u8).collect();
for samples in [&noise, &ramp] {
for ty in [ColorType::Cmyk, ColorType::CmykAsYcck] {
for (q, sampling, progressive) in
[(100, SamplingFactor::R_4_4_4, false), (75, SamplingFactor::R_4_2_0, false), (40, SamplingFactor::R_4_2_2, true)]
{
let mut out = Vec::new();
let mut enc = Encoder::new(&mut out, q);
enc.set_sampling_factor(sampling);
enc.set_progressive(progressive);
enc.encode(samples, w, h, ty).unwrap();
for file in [without_adobe(&out), out] {
let read = decode_jpeg_inks(&file, &Limits::default());
assert_eq!(jpeg_components(&file), Some(4));
let (dw, dh) = jpeg_dimensions(&file).unwrap();
assert_eq!((usize::from(w), usize::from(h)), (dw as usize, dh as usize));
let Some((rw, rh, inks)) = reference_inks(&file) else {
eprintln!("refused: {w}x{h} {ty:?} q{q} progressive {progressive}");
assert!(read.is_err(), "refused by both: {w}x{h} {ty:?} q{q}");
refused += 1;
continue;
};
let read = read.unwrap();
assert_eq!(read.dimensions(), (rw as u32, rh as u32));
assert_eq!(read.data(), inks.as_slice(), "{w}x{h} {ty:?} q{q}");
seen += 1;
}
}
}
}
}
eprintln!("read {seen}, refused {refused}");
assert_eq!(seen + refused, 7 * 2 * 2 * 3 * 2);
assert!(seen > refused);
}
/// zune-jpeg 0.5 panics on some subsampled CMYK frames whose first component isn't the largest
/// (here `jpeg-encoder`'s 2×2 K with C, M and Y at 1×1, progressive): an error, not a panic.
#[test]
fn frames_zune_panics_on_are_errors() {
for (w, h) in [(1u16, 1u16), (3, 3)] {
let mut out = Vec::new();
let mut enc = jpeg_encoder::Encoder::new(&mut out, 70);
enc.set_sampling_factor(jpeg_encoder::SamplingFactor::F_2_2);
enc.set_progressive(true);
enc.encode(&vec![9; usize::from(w * h) * 4], w, h, jpeg_encoder::ColorType::Cmyk).unwrap();
assert!(std::panic::catch_unwind(|| reference_inks(&out)).is_err(), "the reference reader panics on {w}x{h}");
assert!(matches!(decode_jpeg_inks(&out, &Limits::default()), Err(CodecError::Unsupported { .. })), "{w}x{h}");
assert_eq!((jpeg_components(&out), jpeg_dimensions(&out)), (Some(4), Some((u32::from(w), u32::from(h)))));
}
}
+29
View File
@@ -671,3 +671,32 @@ fn highly_compressible_jpeg_and_gif_still_decode() {
assert_eq!(decode(&b).unwrap().width(), 3000, "{f:?}");
}
}
/// `probe_size` refuses the headers `decode` refuses, from the header alone: a size past the
/// limits, and one at the limit that a tiny file cannot hold.
#[test]
fn probe_size_refuses_png_bomb_headers() {
for side in [100_000u32, 32768] {
let mut ihdr = Vec::new();
ihdr.extend_from_slice(&side.to_be_bytes());
ihdr.extend_from_slice(&side.to_be_bytes());
ihdr.extend_from_slice(&[16, 6, 0, 0, 0]);
let mut b = b"\x89PNG\r\n\x1a\n".to_vec();
b.extend(png_chunk(b"IHDR", &ihdr));
b.extend(png_chunk(b"IDAT", &[0x78, 0x9C, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01]));
b.extend(png_chunk(b"IEND", &[]));
assert!(matches!(probe_size(&b), Err(CodecError::LimitExceeded(_))), "{side}");
}
}
/// A JPEG header declaring a size past the default limits is refused by `probe_size` as by
/// `decode`, without reading the scan.
#[test]
fn probe_size_refuses_an_oversized_jpeg_header() {
let mut jpeg = encoded(Format::Jpeg);
assert_eq!(probe_size(&jpeg).unwrap(), (64, 48));
let sof = jpeg.windows(2).position(|w| matches!(w, [0xFF, 0xC0..=0xC2])).unwrap();
jpeg[sof + 5..sof + 9].copy_from_slice(&[0xFF, 0xFF, 0xFF, 0xFF]);
assert!(decode(&jpeg).is_err());
assert!(matches!(probe_size(&jpeg), Err(CodecError::LimitExceeded(_))), "{:?}", probe_size(&jpeg));
}
+1
View File
@@ -14,6 +14,7 @@ mod exr_stream;
mod fidelity;
mod gif_frames;
mod heif;
mod jpeg_inks;
mod jpegls_oracle;
mod limits;
mod malformed;
+18 -2
View File
@@ -202,6 +202,7 @@ fn png_opens_upright_for_every_orientation() {
let file = png_with_exif(&png, &exif(u32::from(o), o % 2 == 0, false));
let img = decode(&file).unwrap();
assert_eq!(img.dimensions(), (w as u32, h as u32), "o={o}");
assert_eq!(probe_size(&file).unwrap(), img.dimensions(), "the header probe turns upright too, o={o}");
assert_eq!(img.data(), &up[..], "o={o}");
assert_eq!(exif_orientation(img.meta.exif.as_deref().unwrap()), 1, "o={o}: metadata says upright");
// Decoded in steps and finished by `finish_png_decode`, it is the same image.
@@ -309,8 +310,10 @@ fn tiff_opens_upright_for_every_orientation() {
let up = exact_scene(w, h);
for o in 1..=8u16 {
let (s, sw, sh) = stored(&up, w, h, o);
let img = decode(&tiff_with_orientation(&s, sw as u32, sh as u32, o)).unwrap();
let file = tiff_with_orientation(&s, sw as u32, sh as u32, o);
let img = decode(&file).unwrap();
assert_eq!(img.dimensions(), (w as u32, h as u32), "o={o}");
assert_eq!(probe_size(&file).unwrap(), img.dimensions(), "the header probe turns upright too, o={o}");
assert_eq!(img.data(), &up[..], "o={o}");
}
}
@@ -358,6 +361,7 @@ fn jpeg_opens_upright_for_every_orientation() {
let file = jpeg_with_exif(&jpeg, &exif(u32::from(o), o > 4, o == 7));
let img = decode(&file).unwrap();
assert_blocks(&img, &up, w, h, &format!("o={o}"));
assert_eq!(probe_size(&file).unwrap(), img.dimensions(), "the header probe turns upright too, o={o}");
assert_eq!(exif_orientation(img.meta.exif.as_deref().unwrap()), 1);
}
}
@@ -659,7 +663,19 @@ fn long_orientation_reads_and_rewrites_in_both_byte_orders() {
#[test]
fn probe_size_matches_decode() {
let img = synth(7, 5, ChannelLayout::Rgb, SampleType::U8, 3, 0.0);
let probed = [Format::WebP, Format::Gif, Format::Bmp, Format::Tga, Format::Qoi, Format::Hdr, Format::Ico, Format::OpenExr];
let probed = [
Format::Png,
Format::Jpeg,
Format::Tiff,
Format::WebP,
Format::Gif,
Format::Bmp,
Format::Tga,
Format::Qoi,
Format::Hdr,
Format::Ico,
Format::OpenExr,
];
assert!(probed.iter().all(|f| rw_formats().contains(f)));
for format in rw_formats() {
let bytes = encode(&img, format, &EncodeOptions::default()).unwrap();
+35
View File
@@ -759,3 +759,38 @@ fn bounded_png_stream_preserves_late_exif_across_metadata_yields() {
assert_eq!(extras.orientation, 6);
assert_eq!(extras.meta.text.len(), 150);
}
/// zune-jpeg 0.5 asserts on some subsampled CMYK frames whose first component isn't the largest
/// (jpeg-encoder writes C, M and Y at 1×1 and K larger); craft-jpeg reads those instead, and they
/// decode to the inks they were written with. The others of these files read as before.
#[test]
fn cmyk_frames_zune_jpeg_panics_on_decode() {
use jpeg_encoder::{ColorType, Encoder, SamplingFactor as S};
let mut seen = 0;
for (s, progressive) in
[(S::F_2_2, true), (S::R_4_2_0, true), (S::F_4_1, false), (S::F_4_2, false), (S::F_2_4, false), (S::R_4_1_1, false), (S::R_4_1_0, false)]
{
for (w, h) in [(1u16, 1u16), (9, 1), (1, 9), (17, 13), (40, 40)] {
let inks: Vec<u8> = (0..usize::from(w) * usize::from(h)).flat_map(|_| [30u8, 90, 150, 210]).collect();
let mut out = Vec::new();
let mut e = Encoder::new(&mut out, 100);
e.set_sampling_factor(s);
e.set_progressive(progressive);
e.encode(&inks, w, h, ColorType::Cmyk).unwrap();
let zune = std::panic::catch_unwind(|| {
let mut d = zune_jpeg::JpegDecoder::new(zune_core::bytestream::ZCursor::new(&out[..]));
d.decode().is_ok()
});
if zune.is_ok() {
// Read or refused by zune-jpeg as before; no panic either way.
let _ = decode(&out);
continue;
}
seen += 1;
let img = decode(&out).unwrap_or_else(|e| panic!("{s:?} {w}x{h}: {e}"));
assert_eq!((img.dimensions(), img.layout()), ((u32::from(w), u32::from(h)), ChannelLayout::Cmyk));
assert!(img.data().iter().zip(&inks).all(|(a, b)| a.abs_diff(*b) <= 3), "{s:?} progressive {progressive} {w}x{h}");
}
}
assert!(seen >= 10, "zune-jpeg panicked on {seen} of them");
}
@@ -539,6 +539,7 @@ fn every_ifd_and_subifd_is_listed_and_decodable() {
// Like Photoshop: the first full-resolution page opens; the thumbnail is skipped.
let img = decode_ok(&b.bytes);
assert_eq!((img.dimensions(), img.data()[0]), ((9, 7), 2));
assert_eq!(probe_size(&b.bytes).unwrap(), (9, 7), "the header probe sizes the page decode opens");
assert_eq!(img.warnings, [DecodeWarning::MorePages { total: Some(2) }]);
// Every directory decodes on request.
for (i, (w, level)) in [(4, 1u8), (9, 2), (3, 5), (2, 6), (9, 3), (5, 4)].into_iter().enumerate() {
@@ -554,6 +555,7 @@ fn a_file_of_only_thumbnails_still_opens() {
let b = build(true, false, &[gray_page(4, 3, 7, Some(1))], &[0]);
assert_eq!(tiff_info(&b.bytes).unwrap().default_page(), Some(0));
assert_eq!(decode_ok(&b.bytes).data()[0], 7);
assert_eq!(probe_size(&b.bytes).unwrap(), (4, 3));
}
#[test]
@@ -698,6 +700,9 @@ fn hostile_sizes_and_counts_are_refused_without_allocating() {
patch_value(&mut b, little, big, 0, 256, 0, wv);
patch_value(&mut b, little, big, 0, 257, 0, hv);
assert!(decode(&b.bytes).is_err(), "{wv}x{hv}");
if wv > 300_000 || hv > 300_000 {
assert!(matches!(probe_size(&b.bytes), Err(CodecError::LimitExceeded(_))), "{wv}x{hv}: probe");
}
assert!(decode_with(&b.bytes, &DecodeOptions { limits: Limits::none(), ..Default::default() }).is_err(), "{wv}x{hv} without limits");
}
// A strip-offset count far past the end of the file.
@@ -805,6 +810,7 @@ fn poke(bytes: &[u8]) {
let _ = decode_with(bytes, &DecodeOptions { limits: Limits { max_alloc: 16 << 20, ..Limits::default() }, ..Default::default() });
let _ = exif_orientation(bytes);
let _ = tiff_orientation(bytes);
let _ = probe_size(bytes);
if let Ok(info) = tiff_info(bytes) {
for i in 0..info.pages.len().min(8) {
let _ = decode_tiff_page(bytes, i, &DecodeOptions::default());
@@ -0,0 +1,65 @@
use craft_image_codecs::legacy_tiff_raw_hint;
fn tiff(tag: u16, ty: u16, value: u32) -> Vec<u8> {
let mut b = b"II\x2a\0\x08\0\0\0\x01\0".to_vec();
b.extend_from_slice(&tag.to_le_bytes());
b.extend_from_slice(&ty.to_le_bytes());
b.extend_from_slice(&1u32.to_le_bytes());
b.extend_from_slice(&value.to_le_bytes());
b.extend_from_slice(&0u32.to_le_bytes());
b
}
#[test]
fn recognizes_legacy_raw_compression_and_broad_cr_marker() {
assert!(legacy_tiff_raw_hint(&tiff(0x0103, 3, 34713)));
assert!(!legacy_tiff_raw_hint(&tiff(0x0103, 3, 1)));
assert!(legacy_tiff_raw_hint(b"II*\0\x08\0\0\0CR"));
let mut cut = tiff(0xc612, 1, 1);
cut.truncate(17);
assert!(!legacy_tiff_raw_hint(&cut));
}
/// A little-endian TIFF with one IFD of (tag, type, count, value) entries and `tail` after it.
fn ifd(entries: &[(u16, u16, u32, [u8; 4])], tail: &[u8]) -> Vec<u8> {
let mut b = b"II\x2a\0\x08\0\0\0".to_vec();
b.extend_from_slice(&(entries.len() as u16).to_le_bytes());
for (tag, ty, count, value) in entries {
b.extend_from_slice(&tag.to_le_bytes());
b.extend_from_slice(&ty.to_le_bytes());
b.extend_from_slice(&count.to_le_bytes());
b.extend_from_slice(value);
}
b.extend_from_slice(&0u32.to_le_bytes());
b.extend_from_slice(tail);
b
}
#[test]
fn sony_lossless_ycbcr_needs_a_sony_make_and_every_layout_tag() {
let short = |v: u16| [v.to_le_bytes()[0], v.to_le_bytes()[1], 0, 0];
let layout = [(0x828e, 1, 4, [0, 1, 1, 2]), (0x0103, 3, 1, short(7)), (0x0106, 3, 1, short(6)), (0x0115, 3, 1, short(3))];
// Make stored after the IFD (ASCII, 5 bytes): 8 + 2 + 5 * 12 + 4 = 74.
let with_make = |make: (u16, u16, u32, [u8; 4]), tail: &[u8]| ifd(&[&[make], &layout[..]].concat(), tail);
assert!(legacy_tiff_raw_hint(&with_make((0x010f, 2, 5, 74u32.to_le_bytes()), b"SONY\0")));
assert!(legacy_tiff_raw_hint(&with_make((0x010f, 7, 4, *b"sony"), b"")));
assert!(!legacy_tiff_raw_hint(&with_make((0x010f, 2, 6, 74u32.to_le_bytes()), b"Nikon\0")));
assert!(!legacy_tiff_raw_hint(&with_make((0x010f, 99, 4, *b"SONY"), b"")));
assert!(!legacy_tiff_raw_hint(&with_make((0x010f, 2, 5, 9999u32.to_le_bytes()), b"")));
// A Sony file without the full lossless layout is not a RAW by this rule.
let partial = ifd(&[(0x010f, 7, 4, *b"SONY"), layout[0], layout[1], layout[2]], b"");
assert!(!legacy_tiff_raw_hint(&partial));
// Repeated tags: the first value wins.
let repeated = ifd(&[&[(0x010f, 7, 4, *b"SONY")], &layout[..], &[(0x0103, 3, 1, short(1))]].concat(), b"");
assert!(legacy_tiff_raw_hint(&repeated));
}
#[test]
fn oversized_or_cut_directories_are_skipped() {
let mut huge = b"II\x2a\0\x08\0\0\0".to_vec();
huge.extend_from_slice(&5000u16.to_le_bytes());
assert!(!legacy_tiff_raw_hint(&huge));
let mut cut = ifd(&[(0x010f, 7, 4, *b"SONY"), (0x0103, 3, 1, [7, 0, 0, 0])], b"");
cut.truncate(20);
assert!(!legacy_tiff_raw_hint(&cut));
}
@@ -0,0 +1,7 @@
# Seeds for failure cases proptest has generated in the past. It is
# automatically read and these particular cases re-run before any
# novel cases are generated.
#
# It is recommended to check this file in to source control so that
# everyone who runs the test benefits from these saved cases.
cc f92b7316155ea46006de7467faa6182a90782b7befd08ed047ebeb70be15c3f7 # shrinks to which = 0, pos = 0, val = 0, cut = 0
+4
View File
@@ -10,5 +10,9 @@ repository.workspace = true
[lints]
workspace = true
[features]
async-lines = ["dep:tokio"]
[dependencies]
serde_json = { workspace = true }
tokio = { version = "1", optional = true, features = ["rt", "sync", "io-util", "macros"] }
+223
View File
@@ -0,0 +1,223 @@
//! JSON-lines adapter that forwards valid JSON to a service and emits JSON-RPC parse errors.
use serde_json::{Value, json};
use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWrite, AsyncWriteExt, BufReader, DuplexStream};
use tokio::task::JoinHandle;
/// The service-facing streams and the tasks forwarding the client streams.
pub struct Adapter {
/// Input for the JSON-RPC service.
pub service_in: DuplexStream,
/// Output from the JSON-RPC service.
pub service_out: DuplexStream,
/// Forwarding tasks, to finish after the service exits.
pub tasks: Tasks,
}
/// Background forwarding tasks.
pub struct Tasks {
reader: JoinHandle<()>,
writer: JoinHandle<()>,
}
impl Tasks {
/// Stop the reader and allow queued output to finish, as after a service exits.
/// This waits for a blocked output sink; callers needing cancellation can call `abort` instead.
pub async fn finish(self) {
self.reader.abort();
let _ = self.writer.await;
}
/// Cancel both forwarding tasks, including a writer blocked on output.
pub fn abort(self) {
self.reader.abort();
self.writer.abort();
}
}
async fn bounded_line(reader: &mut BufReader<impl AsyncRead + Unpin>, bytes: &mut Vec<u8>, limit: usize) -> std::io::Result<Option<String>> {
loop {
let chunk = reader.fill_buf().await?;
if chunk.is_empty() {
if bytes.is_empty() {
return Ok(None);
}
if bytes.len() > limit {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "JSON-RPC line exceeds limit"));
}
break;
}
let newline = chunk.iter().position(|b| *b == b'\n');
let count = newline.unwrap_or(chunk.len());
let size = bytes.len().saturating_add(count);
let last = if count == 0 { bytes.last().copied() } else { Some(chunk[count - 1]) };
if size > limit && !(size == limit.saturating_add(1) && last == Some(b'\r')) {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "JSON-RPC line exceeds limit"));
}
bytes.try_reserve(count).map_err(|_| std::io::Error::new(std::io::ErrorKind::OutOfMemory, "JSON-RPC line allocation failed"))?;
bytes.extend_from_slice(&chunk[..count]);
reader.consume(count + usize::from(newline.is_some()));
if newline.is_some() {
if bytes.last() == Some(&b'\r') {
bytes.pop();
}
break;
}
}
String::from_utf8(std::mem::take(bytes)).map(Some).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))
}
/// Spawn a line adapter. Invalid JSON gets a JSON-RPC parse error; valid values pass through.
/// The bounded error queue applies backpressure if the client stops reading. Input and output lines
/// above their respective limits close their forwarding direction, including unterminated lines.
/// Buffers grow incrementally; forwarding and JSON parsing can temporarily hold additional copies.
pub fn spawn(
input: impl AsyncRead + Unpin + Send + 'static,
output: impl AsyncWrite + Unpin + Send + 'static,
input_limit: usize,
output_limit: usize,
) -> Adapter {
const PIPE_BYTES: usize = 1 << 20;
const MAX_QUEUED_PARSE_ERRORS: usize = 64;
let (mut to_service, service_in) = tokio::io::duplex(PIPE_BYTES);
let (service_out, from_service) = tokio::io::duplex(PIPE_BYTES);
let (err_tx, mut err_rx) = tokio::sync::mpsc::channel::<String>(MAX_QUEUED_PARSE_ERRORS);
let reader = tokio::spawn(async move {
let mut lines = BufReader::new(input);
let mut partial = Vec::new();
loop {
let line = tokio::select! {
l = bounded_line(&mut lines, &mut partial, input_limit) => match l { Ok(Some(l)) => l, _ => break },
() = err_tx.closed() => break,
};
if line.trim().is_empty() {
continue;
}
match serde_json::from_str::<Value>(&line) {
Ok(_) => {
if to_service.write_all(format!("{line}\n").as_bytes()).await.is_err() {
break;
}
}
Err(e) => {
let reply = json!({"jsonrpc": "2.0", "id": null, "error": {"code": -32700, "message": format!("parse error: {e}")}});
if err_tx.send(reply.to_string()).await.is_err() {
break;
}
}
}
}
let _ = to_service.shutdown().await;
});
let writer = tokio::spawn(async move {
let mut output = output;
let mut replies = BufReader::new(from_service);
let mut partial = Vec::new();
loop {
let line = tokio::select! {
l = bounded_line(&mut replies, &mut partial, output_limit) => match l { Ok(Some(l)) => l, _ => break },
Some(e) = err_rx.recv() => e,
};
if output.write_all(format!("{line}\n").as_bytes()).await.is_err() || output.flush().await.is_err() {
break;
}
}
});
Adapter { service_in, service_out, tasks: Tasks { reader, writer } }
}
#[cfg(test)]
mod tests {
use super::*;
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
#[tokio::test(flavor = "current_thread")]
async fn forwards_valid_lines_and_returns_parse_errors() {
let (mut client_in, input) = tokio::io::duplex(1024);
let (output, client_out) = tokio::io::duplex(1024);
let Adapter { mut service_in, mut service_out, tasks } = spawn(input, output, 1024, 1024);
client_in.write_all(b"\n{bad}\n{\"id\":1}\n").await.unwrap();
client_in.shutdown().await.unwrap();
let mut forwarded = String::new();
service_in.read_to_string(&mut forwarded).await.unwrap();
assert_eq!(forwarded, "{\"id\":1}\n");
let mut lines = BufReader::new(client_out).lines();
let error: Value = serde_json::from_str(&lines.next_line().await.unwrap().unwrap()).unwrap();
assert_eq!(error["error"]["code"], -32700);
assert!(error["id"].is_null());
service_out.write_all(b"{\"id\":1,\"result\":true}\n").await.unwrap();
let result: Value = serde_json::from_str(&lines.next_line().await.unwrap().unwrap()).unwrap();
assert_eq!(result["result"], true);
assert_eq!(result["id"], 1);
service_out.shutdown().await.unwrap();
tasks.finish().await;
}
#[tokio::test(flavor = "current_thread")]
async fn cancelled_line_read_retains_consumed_prefix() {
use std::future::{Future, poll_fn};
use std::task::Poll;
let (mut writer, stream) = tokio::io::duplex(32);
let mut reader = BufReader::new(stream);
let mut partial = Vec::new();
writer.write_all(b"prefix").await.unwrap();
{
let mut read = Box::pin(bounded_line(&mut reader, &mut partial, 32));
poll_fn(|cx| {
assert!(matches!(read.as_mut().poll(cx), Poll::Pending));
Poll::Ready(())
})
.await;
}
assert_eq!(partial, b"prefix");
writer.write_all(b" suffix\n").await.unwrap();
assert_eq!(bounded_line(&mut reader, &mut partial, 32).await.unwrap().as_deref(), Some("prefix suffix"));
}
#[tokio::test(flavor = "current_thread")]
async fn parse_error_does_not_discard_partial_service_reply() {
let (mut client, input) = tokio::io::duplex(128);
let (output, client_out) = tokio::io::duplex(128);
let Adapter { service_in: _, mut service_out, tasks } = spawn(input, output, 128, 128);
service_out.write_all(b"{\"id\":1,").await.unwrap();
tokio::task::yield_now().await;
client.write_all(b"{bad}\n").await.unwrap();
let mut lines = BufReader::new(client_out).lines();
let error: Value = serde_json::from_str(&lines.next_line().await.unwrap().unwrap()).unwrap();
assert_eq!(error["error"]["code"], -32700);
service_out.write_all(b"\"result\":true}\n").await.unwrap();
let reply: Value = serde_json::from_str(&lines.next_line().await.unwrap().unwrap()).unwrap();
assert_eq!(reply, json!({"id": 1, "result": true}));
tasks.abort();
}
#[tokio::test(flavor = "current_thread")]
async fn crlf_at_exact_limit_does_not_count_the_terminator() {
let mut reader = BufReader::new(&b"1234\r\n"[..]);
assert_eq!(bounded_line(&mut reader, &mut Vec::new(), 4).await.unwrap().as_deref(), Some("1234"));
let mut reader = BufReader::new(&b"1234\r"[..]);
assert!(bounded_line(&mut reader, &mut Vec::new(), 4).await.is_err());
}
#[tokio::test(flavor = "current_thread")]
async fn oversized_unterminated_lines_stop_both_directions() {
let (mut client, input) = tokio::io::duplex(128);
let (output, _client_out) = tokio::io::duplex(128);
let Adapter { mut service_in, service_out, tasks } = spawn(input, output, 8, 8);
client.write_all(b"123456789").await.unwrap();
let mut forwarded = Vec::new();
service_in.read_to_end(&mut forwarded).await.unwrap();
assert!(forwarded.is_empty());
drop(service_out);
tasks.finish().await;
let (_client, input) = tokio::io::duplex(128);
let (output, mut client_out) = tokio::io::duplex(128);
let Adapter { service_in: _, mut service_out, tasks } = spawn(input, output, 8, 8);
service_out.write_all(b"123456789").await.unwrap();
let mut received = Vec::new();
client_out.read_to_end(&mut received).await.unwrap();
assert!(received.is_empty());
tasks.abort();
}
#[tokio::test(flavor = "current_thread")]
async fn blocked_output_tasks_can_be_cancelled() {
let (_client, input) = tokio::io::duplex(16);
let (output, _blocked) = tokio::io::duplex(1);
let Adapter { service_in: _, mut service_out, tasks } = spawn(input, output, 64, 64);
service_out.write_all(b"123456789\n").await.unwrap();
tokio::task::yield_now().await;
tasks.abort();
}
}
+2
View File
@@ -8,6 +8,8 @@
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#[cfg(feature = "async-lines")]
pub mod async_lines;
pub mod control;
use serde_json::{Value, json};
+114
View File
@@ -0,0 +1,114 @@
//! Tolerant fields from AVC and HEVC configuration records.
/// NAL length prefix width from AVC configuration, defaulting to four on a short record.
pub fn avcc_length_size(avcc: &[u8]) -> usize {
avcc.get(4).map_or(4, |b| (b & 3) as usize + 1)
}
/// NAL length prefix width and highest temporal ID from HEVC configuration.
pub fn hvcc_length_size_and_tid(hvcc: &[u8]) -> (usize, Option<u8>) {
let length_size = hvcc.get(21).map_or(4, |b| (b & 3) as usize + 1);
let highest_tid = hvcc.get(21).map(|b| (b >> 3) & 7).filter(|&n| n > 0).map(|n| n - 1);
(length_size, highest_tid)
}
/// objectTypeIndication of the DecoderConfigDescriptor inside an `esds` box in `raw`.
pub fn esds_object_type(raw: &[u8]) -> Option<u8> {
let at = raw.windows(4).position(|w| w == b"esds")? + 8;
let mut p = at;
let size = |p: &mut usize| -> Option<usize> {
let mut n = 0usize;
for _ in 0..4 {
let b = *raw.get(*p)?;
*p += 1;
n = (n << 7) | (b & 0x7F) as usize;
if b & 0x80 == 0 {
break;
}
}
Some(n)
};
if *raw.get(p)? != 0x03 {
return None;
}
p += 1;
size(&mut p)?;
let flags = *raw.get(p + 2)?;
p += 3;
if flags & 0x80 != 0 {
p += 2;
}
if flags & 0x40 != 0 {
p += 1 + *raw.get(p)? as usize;
}
if flags & 0x20 != 0 {
p += 2;
}
if *raw.get(p)? != 0x04 {
return None;
}
p += 1;
size(&mut p)?;
raw.get(p).copied()
}
/// E-AC-3 `ec-3` entry: the program-layout fields are the 24 bits starting at byte 2 of the
/// descriptor, then `bsmod(5) | acmod(3) | lfeon(1)`. Matches FFmpeg's `dec3` demuxer.
pub fn dec3_layout(payload: &[u8]) -> Option<(u32, u32)> {
if payload.len() < 5 {
return None;
}
let bits = u32::from(payload[2]) << 16 | u32::from(payload[3]) << 8 | u32::from(payload[4]);
let acmod = ((bits >> 9) & 0x7) as u8;
let lfeon = bits & 0x100 != 0;
let channels = dac3_channels(acmod, lfeon)?;
Some((0, channels))
}
/// AC-3 `ac-3` entry: `dac3` begins with the syncframe bitstream information: sample rate code,
/// bit stream id, bit stream mode, channel mode and LFE flag. The descriptor has no sample rate,
/// so the caller keeps the sample-entry rate.
pub fn dac3_layout(payload: &[u8]) -> Option<(u32, u32)> {
if payload.len() < 3 {
return None;
}
// The descriptor begins with the bitstream information: fscod(2), bsid(5), bsmod(3),
// acmod(3) and lfeon(1), continuing in the second payload byte.
let bits = u32::from_be_bytes([0, 0, payload[1], payload[2]]);
let acmod = ((bits >> 11) & 0x7) as u8;
let lfeon = bits & 0x400 != 0;
let channels = dac3_channels(acmod, lfeon)?;
Some((0, channels))
}
/// Full-bandwidth channels per AC-3 channel mode (Table 5.8) plus the optional LFE channel.
fn dac3_channels(acmod: u8, lfeon: bool) -> Option<u32> {
let n = [2u32, 1, 2, 3, 3, 4, 4, 5].get(usize::from(acmod & 7))?.checked_add(u32::from(lfeon))?;
(n > 0).then_some(n)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn audio_layout_and_elementary_descriptor_are_bounded() {
assert_eq!(dac3_layout(&[0, 0, 0]), Some((0, 2)));
assert_eq!(dec3_layout(&[0, 0, 0, 0, 0]), Some((0, 2)));
assert_eq!(dac3_layout(&[0, 0]), None);
assert_eq!(dec3_layout(&[0; 4]), None);
let esds = [b'e', b's', b'd', b's', 0, 0, 0, 0, 0x03, 0x04, 0, 0, 0, 0x04, 0x01, 0x60];
assert_eq!(esds_object_type(&esds), Some(0x60));
for end in 0..esds.len() {
assert_eq!(esds_object_type(&esds[..end]), None);
}
}
#[test]
fn short_records_and_temporal_layers() {
assert_eq!(avcc_length_size(&[]), 4);
assert_eq!(avcc_length_size(&[0, 0, 0, 0, 1]), 2);
assert_eq!(hvcc_length_size_and_tid(&[]), (4, None));
let mut h = [0; 22];
h[21] = (3 << 3) | 2;
assert_eq!(hvcc_length_size_and_tid(&h), (3, Some(2)));
}
}
+2
View File
@@ -14,6 +14,7 @@ pub mod boxes;
mod bytes;
pub mod chapters;
mod codec;
mod config_fields;
mod demux;
mod error;
mod mux;
@@ -25,6 +26,7 @@ pub use codec::{
AacConfig, ApvConfig, AudioParams, Av1Config, AvcConfig, BitRate, CleanAperture, CodecConfig, ColorInfo, FieldInfo, FlacConfig, HevcConfig,
HevcNalArray, MasteringDisplay, OpusConfig, PcmConfig, SampleEntry, TimecodeConfig, VideoParams, VpcConfig, parse_asc,
};
pub use config_fields::{avcc_length_size, dac3_layout, dec3_layout, esds_object_type, hvcc_length_size_and_tid};
pub use demux::{Edit, Metadata, Mp4File, Sample, SampleGroup, Track, TrackKind, open};
pub use error::{Error, Result};
pub use mux::{Brand, FragmentedWriter, Mp4Writer, TrackConfig, WriteSample, WriterOptions};
+2
View File
@@ -22,6 +22,7 @@ mod mux;
#[cfg(test)]
mod synthetic_tests;
mod track;
mod vp9_features;
pub use crate::source::{ByteSource, ReadSeekSource};
pub use codec::Codec;
@@ -32,6 +33,7 @@ pub use meta::{
};
pub use mux::{LacingMode, MkvWriter, MuxOptions, TrackSpec};
pub use track::{AudioInfo, Color, ContentEncoding, MasteringMetadata, Projection, Sample, Track, TrackKind, VideoInfo};
pub use vp9_features::{Vp9Features, vp9_features};
/// Element IDs used by the demuxer (marker bits included), for callers inspecting raw elements.
pub mod element_ids {
+42
View File
@@ -0,0 +1,42 @@
//! VP9 CodecPrivate feature records in Matroska/WebM.
/// The fields used to configure VP9 decoding and sample descriptions.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Vp9Features {
pub profile: u8,
pub level: u8,
pub bit_depth: u8,
pub chroma_subsampling: u8,
}
/// Read ID/length/value triples, retaining defaults when a trailing record is truncated.
pub fn vp9_features(private: &[u8]) -> Vp9Features {
let mut result = Vp9Features { profile: 0, level: 0, bit_depth: 8, chroma_subsampling: 0 };
let mut at = 0;
while at + 2 <= private.len() {
let (id, len) = (private[at], private[at + 1] as usize);
let Some(value) = private.get(at + 2..at + 2 + len) else { break };
let value = value.first().copied().unwrap_or(0);
match id {
1 => result.profile = value,
2 => result.level = value,
3 => result.bit_depth = value,
4 => result.chroma_subsampling = value,
_ => {}
}
at += 2 + len;
}
result
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn features_keep_earlier_values_when_a_later_record_is_truncated() {
let f = vp9_features(&[1, 1, 2, 3, 1, 10, 4, 2, 9]);
assert_eq!(f, Vp9Features { profile: 2, level: 0, bit_depth: 10, chroma_subsampling: 0 });
assert_eq!(vp9_features(&[3, 0]).bit_depth, 0);
assert_eq!(vp9_features(&[]).bit_depth, 8);
}
}
+2 -1
View File
@@ -16,13 +16,14 @@ stl = []
obj = []
ply = []
gltf = ["dep:serde_json"]
fbx = ["dep:miniz_oxide"]
fbx = ["dep:miniz_oxide", "dep:craft-zip"]
threemf = ["dep:craft-opc", "dep:craft-xml"]
[dependencies]
serde_json = { workspace = true, optional = true }
# zlib-compressed arrays in binary FBX.
miniz_oxide = { version = "0.9", optional = true }
craft-zip = { workspace = true, optional = true }
# 3MF is an OPC package of XML parts.
craft-opc = { workspace = true, optional = true }
craft-xml = { workspace = true, optional = true }
+521 -156
View File
@@ -39,21 +39,23 @@
//! name is listed in [`Scene::external`]. The names come from the file: a resolver must not trust
//! them as paths.
//!
//! Not read: FBX 5 and older, animation stacks, skin deformers, blend shapes, cameras and
//! lights, NURBS and patches.
//! Not read: FBX 5 and older, animation stacks, skin deformers, blend shapes, lights, NURBS and patches.
//!
//! [`Limits::max_data_bytes`] bounds what one file may keep in memory: its parsed values (an
//! array element counts 8 bytes whatever its type in the file), the meshes built from them and
//! the image bytes.
use std::cell::Cell;
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use crate::{
AlphaMode, Corner, IDENTITY, Image, Limits, Mat4, Material, Mesh, NONE, Node, Part, Resolver, Result, Scene, TextureRef, base64_decode,
count_warning, malformed, mat_mul, srgb_to_linear, too_large,
AlphaMode, Camera, Corner, IDENTITY, Image, Limits, Mat4, Material, Mesh, NONE, Node, Part, Projection, Resolver, Result, Scene, TextureRef,
base64_decode, count_warning, malformed, mat_mul, srgb_to_linear, too_large,
};
/// Bounded ZIP packages containing an FBX and its texture files.
pub mod package;
const KIND: &str = "FBX";
/// The binary file's magic (`Kaydara FBX Binary`, two spaces, NUL, 0x1A, 0x00).
@@ -89,7 +91,8 @@ impl Budget {
}
/// Bytes the values under `records` keep. Used for ASCII files, which hold array elements as one
/// [`Prop`] each and are parsed before they can be counted (the text itself bounds them).
/// [`Prop`] each.
#[cfg(test)]
pub(crate) fn value_bytes(records: &[Record]) -> Option<usize> {
records.iter().try_fold(0usize, |acc, n| {
let own = n.props.iter().try_fold(0usize, |a, p| {
@@ -109,7 +112,13 @@ pub(crate) fn value_bytes(records: &[Record]) -> Option<usize> {
/// Bytes a mesh keeps.
pub(crate) fn mesh_bytes(m: &Mesh) -> Option<usize> {
let uvs = m.uv_sets.iter().try_fold(0usize, |a, s| a.checked_add(s.len().checked_mul(8)?))?;
let pools = m.positions.len().checked_mul(24)?.checked_add(m.normals.len().checked_mul(24)?)?.checked_add(uvs)?;
let pools = m
.positions
.len()
.checked_mul(24)?
.checked_add(m.normals.len().checked_mul(24)?)?
.checked_add(uvs)?
.checked_add(m.colors.len().checked_mul(16)?)?;
m.parts.iter().try_fold(pools, |a, p| a.checked_add(p.corners.len().checked_mul(12)?)?.checked_add(p.face_sizes.len().checked_mul(4)?))
}
@@ -229,13 +238,29 @@ pub fn is_binary(bytes: &[u8]) -> bool {
/// Read an FBX file (binary or ASCII) into its records.
pub fn read_document(bytes: &[u8], limits: &Limits) -> Result<Document> {
read_document_impl(bytes, limits, false)
}
/// Read records strictly, checking ASCII delimiters and the binary closing record.
pub fn read_document_strict(bytes: &[u8], limits: &Limits) -> Result<Document> {
read_document_impl(bytes, limits, true)
}
fn read_document_impl(bytes: &[u8], limits: &Limits, strict: bool) -> Result<Document> {
let budget = Budget::new(limits.max_data_bytes);
let (version, records) = if is_binary(bytes) {
read_binary(bytes, &budget)?
read_binary(bytes, &budget, strict)?
} else {
let (version, records) = read_ascii(&String::from_utf8_lossy(bytes))?;
budget.take(value_bytes(&records), "its values")?;
(version, records)
let text = match std::str::from_utf8(bytes) {
Ok(text) => std::borrow::Cow::Borrowed(text),
Err(_) => {
if bytes.len() > limits.max_data_bytes {
return Err(too_large(KIND, "invalid UTF-8 text exceeds the memory limit"));
}
String::from_utf8_lossy(bytes)
}
};
read_ascii(&text, &budget, strict)?
};
Ok(Document { version, records, left: budget.left.get() })
}
@@ -254,6 +279,7 @@ struct Bin<'a> {
/// 64-bit record offsets (version 7500 and later).
wide: bool,
budget: &'a Budget,
records_budget: &'a Budget,
}
impl Bin<'_> {
@@ -313,6 +339,8 @@ impl Bin<'_> {
if end <= start || end > limit.min(self.d.len()) {
return Err(malformed(KIND, format!("bad record end {end} at byte {start}")));
}
// Bound record and property-vector allocations before copying names or reserving storage.
self.records_budget.take(std::mem::size_of::<Record>().checked_add(nlen.saturating_mul(3)), "its records")?;
let name = String::from_utf8_lossy(self.take(pos, nlen)?).into_owned();
let pstart = *pos;
let pend =
@@ -321,7 +349,9 @@ impl Bin<'_> {
if nprops > plen as u64 {
return Err(malformed(KIND, format!("`{name}`: {nprops} properties in {plen} bytes")));
}
let mut props = Vec::with_capacity(nprops as usize);
let count = usize::try_from(nprops).map_err(|_| too_large(KIND, "too many properties"))?;
self.records_budget.take(count.checked_mul(std::mem::size_of::<Prop>()), "its properties")?;
let mut props = Vec::with_capacity(count);
for _ in 0..nprops {
props.push(self.prop(pos, pend)?);
}
@@ -332,7 +362,12 @@ impl Bin<'_> {
}
fn prop(&self, pos: &mut usize, pend: usize) -> Result<Prop> {
let sub = Bin { d: self.d.get(..pend).ok_or_else(|| malformed(KIND, "truncated property"))?, wide: self.wide, budget: self.budget };
let sub = Bin {
d: self.d.get(..pend).ok_or_else(|| malformed(KIND, "truncated property"))?,
wide: self.wide,
budget: self.budget,
records_budget: self.records_budget,
};
let t = sub.arr::<1>(pos)?[0];
Ok(match t {
b'Y' => Prop::Int(i64::from(i16::from_le_bytes(sub.arr(pos)?))),
@@ -343,10 +378,12 @@ impl Bin<'_> {
b'D' => Prop::Float(f64::from_le_bytes(sub.arr(pos)?)),
b'S' => {
let n = sub.u32(pos)? as usize;
sub.budget.take(n.checked_mul(3), "a string")?;
Prop::Str(object_name(sub.take(pos, n)?))
}
b'R' => {
let n = sub.u32(pos)? as usize;
sub.budget.take(Some(n), "binary data")?;
Prop::Bytes(sub.take(pos, n)?.to_vec())
}
b'f' | b'd' | b'l' | b'i' | b'b' => {
@@ -398,143 +435,178 @@ fn object_name(b: &[u8]) -> String {
}
}
fn read_binary(bytes: &[u8], budget: &Budget) -> Result<(u32, Vec<Record>)> {
fn read_binary(bytes: &[u8], budget: &Budget, strict: bool) -> Result<(u32, Vec<Record>)> {
let mut pos = BINARY_MAGIC.len();
let version = Bin { d: bytes, wide: false, budget }.u32(&mut pos)?;
let bin = Bin { d: bytes, wide: version >= 7500, budget };
let records_budget = Budget::new(budget.limit);
let version = Bin { d: bytes, wide: false, budget, records_budget: &records_budget }.u32(&mut pos)?;
let bin = Bin { d: bytes, wide: version >= 7500, budget, records_budget: &records_budget };
let mut records = vec![];
let mut closed = false;
while pos < bytes.len() {
match bin.record(&mut pos, bytes.len(), 0) {
Ok(Some(n)) => records.push(n),
Ok(None) => break,
Ok(None) => {
closed = true;
break;
}
// A damaged tail (footer, padding) after complete sections is ignored.
Err(crate::Error::Malformed(_)) if records.iter().any(|n| n.name == "Objects") => break,
Err(crate::Error::Malformed(_)) if !strict && records.iter().any(|n| n.name == "Objects") => break,
Err(e) => return Err(e),
}
}
if strict && !closed {
return Err(malformed(KIND, "missing binary closing record"));
}
Ok((version, records))
}
// ---------------------------------------------------------------------------------------------
// ASCII
#[derive(Clone, Debug, PartialEq)]
enum Tok {
Key(String),
Val(Prop),
/// `*N` array length (the values follow in an `a:` child).
#[derive(Clone, Copy, Debug, PartialEq)]
enum Tok<'a> {
Key(&'a str),
Value(&'a str, bool),
Count,
Comma,
Open,
Close,
}
fn tokenize(text: &str) -> Vec<Tok> {
let mut out = vec![];
let mut it = text.char_indices().peekable();
while let Some(&(i, c)) = it.peek() {
match c {
c if c.is_whitespace() => {
it.next();
/// Scan on demand so punctuation and ignored input cannot build an unbounded token vector.
struct Ascii<'a> {
text: &'a str,
pos: usize,
lookahead: Option<Tok<'a>>,
strict: bool,
unterminated: bool,
}
impl<'a> Ascii<'a> {
fn next_token(&mut self) -> Option<Tok<'a>> {
loop {
let rest = self.text.get(self.pos..)?;
let c = rest.chars().next()?;
if c.is_whitespace() {
self.pos += c.len_utf8();
continue;
}
';' => {
for (_, c) in it.by_ref() {
if c == '\n' {
break;
}
if c == ';' {
self.pos += rest.find('\n').map_or(rest.len(), |i| i + 1);
continue;
}
match c {
',' | '{' | '}' => {
self.pos += 1;
return Some(match c {
',' => Tok::Comma,
'{' => Tok::Open,
_ => Tok::Close,
});
}
}
',' => {
it.next();
out.push(Tok::Comma);
}
'{' => {
it.next();
out.push(Tok::Open);
}
'}' => {
it.next();
out.push(Tok::Close);
}
'"' => {
it.next();
let start = i + 1;
let mut end = text.len();
for (j, c) in it.by_ref() {
if c == '"' {
end = j;
break;
}
'"' => {
let tail = rest.get(1..).unwrap_or("");
let end = tail.find('"').unwrap_or(tail.len());
self.unterminated |= end == tail.len();
self.pos += 1 + end + usize::from(end < tail.len());
return Some(Tok::Value(tail.get(..end).unwrap_or(""), true));
}
out.push(Tok::Val(Prop::Str(text.get(start..end).unwrap_or("").replace("&quot;", "\""))));
}
_ => {
let start = i;
let mut end = text.len();
while let Some(&(j, c)) = it.peek() {
if c.is_whitespace() || matches!(c, ',' | '{' | '}' | '"' | ';' | ':') {
end = j;
break;
_ => {
let end = rest.find(|c: char| c.is_whitespace() || matches!(c, ',' | '{' | '}' | '"' | ';' | ':')).unwrap_or(rest.len());
let word = rest.get(..end).unwrap_or("");
self.pos += end;
if rest.as_bytes().get(end) == Some(&b':') {
self.pos += 1;
return Some(Tok::Key(word));
}
if word.starts_with('*') {
return Some(Tok::Count);
}
if !word.is_empty() {
return Some(Tok::Value(word, false));
}
it.next();
}
let word = text.get(start..end).unwrap_or("");
if it.peek().is_some_and(|(_, c)| *c == ':') {
it.next();
out.push(Tok::Key(word.to_string()));
} else if word.starts_with('*') {
out.push(Tok::Count);
} else if let Ok(i) = word.parse::<i64>() {
out.push(Tok::Val(Prop::Int(i)));
} else if let Ok(f) = word.parse::<f64>() {
out.push(Tok::Val(Prop::Float(f)));
} else if !word.is_empty() {
out.push(Tok::Val(Prop::Str(word.to_string())));
} else {
// A stray `:` with no key.
it.next();
self.pos += 1;
}
}
}
}
out
fn peek(&mut self) -> Option<Tok<'a>> {
if self.lookahead.is_none() {
self.lookahead = self.next_token();
}
self.lookahead
}
fn take(&mut self) -> Option<Tok<'a>> {
self.lookahead.take().or_else(|| self.next_token())
}
}
fn ascii_list(toks: &[Tok], i: &mut usize, depth: usize) -> Result<Vec<Record>> {
fn ascii_list(scan: &mut Ascii<'_>, budget: &Budget, records_budget: &Budget, depth: usize) -> Result<Vec<Record>> {
if depth > MAX_DEPTH {
return Err(too_large(KIND, "records nested too deeply"));
}
let mut out = vec![];
while let Some(t) = toks.get(*i) {
*i += 1;
while let Some(t) = scan.take() {
match t {
Tok::Close if depth > 0 => return Ok(out),
Tok::Close if scan.strict => return Err(malformed(KIND, "unbalanced closing brace")),
Tok::Key(name) => {
let mut record = Record { name: name.clone(), ..Record::default() };
records_budget.take(std::mem::size_of::<Record>().checked_add(name.len()), "its records")?;
let mut record = Record { name: name.to_string(), ..Record::default() };
let mut needs_comma = false;
loop {
match toks.get(*i) {
Some(Tok::Val(v)) => record.props.push(v.clone()),
Some(Tok::Count | Tok::Comma) => {}
match scan.peek() {
Some(Tok::Value(..) | Tok::Count) if scan.strict && needs_comma => break,
Some(Tok::Value(value, quoted)) => {
let number = if quoted {
None
} else {
value.parse::<i64>().ok().map(Prop::Int).or_else(|| value.parse::<f64>().ok().map(Prop::Float))
};
let size = std::mem::size_of::<Prop>().checked_add(if number.is_some() { 0 } else { value.len() });
// Charge before copying a string or growing the property vector.
budget.take(size, "its values")?;
record
.props
.push(number.unwrap_or_else(|| Prop::Str(if quoted { value.replace("&quot;", "\"") } else { value.to_string() })));
}
Some(Tok::Count) => {}
Some(Tok::Comma) => {
needs_comma = false;
scan.take();
continue;
}
_ => break,
}
*i += 1;
needs_comma = true;
scan.take();
}
if toks.get(*i) == Some(&Tok::Open) {
*i += 1;
record.children = ascii_list(toks, i, depth + 1)?;
if scan.peek() == Some(Tok::Open) {
scan.take();
record.children = ascii_list(scan, budget, records_budget, depth + 1)?;
}
out.push(record);
}
_ if scan.strict => return Err(malformed(KIND, "expected a property name")),
_ => {}
}
}
if scan.strict && depth > 0 {
return Err(malformed(KIND, "missing closing brace"));
}
Ok(out)
}
fn read_ascii(text: &str) -> Result<(u32, Vec<Record>)> {
let toks = tokenize(text);
let mut i = 0;
let records = ascii_list(&toks, &mut i, 0)?;
fn read_ascii(text: &str, budget: &Budget, strict: bool) -> Result<(u32, Vec<Record>)> {
let mut scan = Ascii { text, pos: 0, lookahead: None, strict, unterminated: false };
let records_budget = Budget::new(budget.limit);
let records = ascii_list(&mut scan, budget, &records_budget, 0)?;
if strict && scan.unterminated {
return Err(malformed(KIND, "unterminated string"));
}
let version = records
.iter()
.find(|n| n.name == "FBXHeaderExtension")
@@ -814,53 +886,244 @@ impl Document {
}
}
// One mesh per Model with Mesh geometry (material slots are per Model). Models instancing
// the same geometry with the same materials share one mesh rather than each building a
// copy.
let mut mesh_of: HashMap<(Key, Vec<usize>), usize> = HashMap::new();
for (id, o) in &models {
// FBX cameras look along +X; a child turns the neutral camera's -Z onto +X.
// Keep this separate from the Model so its children retain their original transforms.
for (id, model) in &models {
let class_index = if matches!(model.props.first(), Some(Prop::Int(_))) { 2 } else { 1 };
if model.str_at(class_index) != Some("Camera") {
continue;
}
let name = short_name(model.first_str().unwrap_or(""));
if name.starts_with("Producer ") {
continue;
}
let Some(ni) = node_of.get(id).copied() else { continue };
let Some((gid, geom)) = b.geometry_of(*id) else { continue };
let mut mats: Vec<usize> = vec![];
let lens = b.children_of(*id, "NodeAttribute").last().and_then(|(key, _)| b.by_key.get(key)).copied().unwrap_or(model);
let props = Props::of(lens);
let positive = |key| props.f(key).filter(|v| *v > 0.0);
let projection = if props.f("CameraProjectionType") == Some(1.0) {
let mag = positive("OrthoZoom").unwrap_or(1.0);
Projection::Orthographic { xmag: mag, ymag: mag, znear: 0.01, zfar: 1000.0 }
} else {
let yfov = match positive("FocalLength") {
Some(focal) => 2.0 * (positive("FilmHeight").unwrap_or(0.612) * 25.4 / (2.0 * focal)).atan(),
None => positive("FieldOfViewY").or_else(|| positive("FieldOfView")).unwrap_or(40.0).to_radians(),
};
Projection::Perspective { yfov, aspect: None, znear: 0.01, zfar: None }
};
if b.scene.cameras.len() >= limits.max_extras {
return Err(too_large(KIND, "too many cameras"));
}
b.node_room()?;
let ci = b.scene.cameras.len();
b.scene.cameras.push(Camera { name: name.to_string(), projection });
let child = b.scene.nodes.len();
b.scene.nodes.push(Node { name: format!("{name} camera"), transform: rotate(1, -90.0), camera: Some(ci), ..Node::default() });
if let Some(node) = b.scene.nodes.get_mut(ni) {
node.children.push(child);
}
}
// One mesh per geometry and material-slot combination, shared by instances.
let mut mesh_of: HashMap<(Key, Vec<usize>), usize> = HashMap::new();
let mut placed = HashSet::new();
for (id, model) in &models {
let Some(ni) = node_of.get(id).copied() else { continue };
let geometries = b.geometries_of(*id);
if geometries.is_empty() {
continue;
}
let mut mats = Vec::new();
for (mid, _) in b.children_of(*id, "Material") {
mats.extend(b.material(mid)?);
}
let name = b.scene.nodes.get(ni).map(|n| n.name.clone()).unwrap_or_default();
let mesh_index = match mesh_of.get(&(gid, mats.clone())) {
Some(i) => *i,
None => {
let Some(mesh) = b.mesh(&name, geom, &mats)? else { continue };
b.budget.take(mesh_bytes(&mesh), &format!("mesh `{name}`"))?;
if b.scene.meshes.len() >= limits.max_meshes {
return Err(too_large(KIND, format!("more than {} meshes", limits.max_meshes)));
}
b.scene.meshes.push(mesh);
mesh_of.insert((gid, mats), b.scene.meshes.len() - 1);
b.scene.meshes.len() - 1
for (gid, geom) in geometries {
if !placed.insert((gid, Some(*id))) {
continue;
}
};
match geometric(&Props::of(o)) {
Some(transform) => {
let mesh_index = match mesh_of.get(&(gid, mats.clone())) {
Some(i) => *i,
None => {
let Some(mesh) = b.mesh(&name, geom, &mats)? else { continue };
b.budget.take(mesh_bytes(&mesh), &format!("mesh `{name}`"))?;
if b.scene.meshes.len() >= limits.max_meshes {
return Err(too_large(KIND, format!("more than {} meshes", limits.max_meshes)));
}
b.scene.meshes.push(mesh);
mesh_of.insert((gid, mats.clone()), b.scene.meshes.len() - 1);
b.scene.meshes.len() - 1
}
};
let transform = geometric(&Props::of(model));
let occupied = b.scene.nodes.get(ni).is_some_and(|n| n.mesh.is_some());
if transform.is_some() || occupied {
b.node_room()?;
let gi = b.scene.nodes.len();
b.scene.nodes.push(Node { name: format!("{name} geometry"), transform, mesh: Some(mesh_index), ..Node::default() });
if let Some(n) = b.scene.nodes.get_mut(ni) {
n.children.push(gi);
}
}
None => {
if let Some(n) = b.scene.nodes.get_mut(ni) {
n.mesh = Some(mesh_index);
let child = b.scene.nodes.len();
b.scene.nodes.push(Node {
name: format!("{name} geometry"),
transform: transform.unwrap_or(IDENTITY),
mesh: Some(mesh_index),
..Node::default()
});
if let Some(node) = b.scene.nodes.get_mut(ni) {
node.children.push(child);
}
} else if let Some(node) = b.scene.nodes.get_mut(ni) {
node.mesh = Some(mesh_index);
}
}
}
// Geometry with no Model connection still contributes a mesh at the origin.
let placed_ids: HashSet<Key<'_>> = placed.iter().map(|(gid, _)| *gid).collect();
for geom in objects.children.iter().filter(|o| o.name == "Geometry" && o.child("Vertices").is_some()) {
let Some(gid) = key(geom.props.first()) else { continue };
if placed_ids.contains(&gid) {
continue;
}
let Some(mesh) = b.mesh("", geom, &[])? else { continue };
b.budget.take(mesh_bytes(&mesh), "unplaced mesh")?;
if b.scene.meshes.len() >= limits.max_meshes {
return Err(too_large(KIND, format!("more than {} meshes", limits.max_meshes)));
}
b.node_room()?;
let mesh_index = b.scene.meshes.len();
b.scene.meshes.push(mesh);
b.scene.roots.push(b.scene.nodes.len());
b.scene.nodes.push(Node { mesh: Some(mesh_index), ..Node::default() });
}
let mut scene = b.scene;
count_warning(&mut scene.warnings, b.skipped, "polygons with fewer than three corners or a vertex that is not there were left out");
Ok(scene)
}
}
/// Diffuse material values as stored in FBX, before color-space conversion.
#[derive(Clone, Debug, PartialEq)]
pub struct DiffuseMaterial {
pub name: String,
pub factor: [f32; 4],
/// Encoded embedded or resolved image bytes, when available.
pub image: Option<Vec<u8>>,
}
impl Document {
/// Read all FBX diffuse materials, including unconnected materials. The resolver receives
/// the relative path followed by the absolute path's leaf, in preference order.
pub fn diffuse_materials(&self, limits: &Limits, resolve: &mut dyn FnMut(&[&str]) -> Option<Vec<u8>>) -> Result<Vec<DiffuseMaterial>> {
let objects = self.top("Objects").ok_or_else(|| malformed(KIND, "no Objects section"))?;
let mut materials = HashMap::new();
let mut textures = HashMap::new();
let mut videos = HashMap::new();
for record in &objects.children {
let Some(id) = key(record.props.first()) else { continue };
match record.name.as_str() {
"Material" => {
materials.insert(id, record);
}
"Texture" => {
textures.insert(id, record);
}
"Video" => {
videos.insert(id, record);
}
_ => {}
}
}
if materials.len() > limits.max_materials {
return Err(too_large(KIND, "too many materials"));
}
let mut video_of = HashMap::new();
let mut diffuse = HashMap::new();
for link in self.top("Connections").into_iter().flat_map(|c| c.children.iter()).filter(|c| c.name == "C" || c.name == "Connect") {
let (Some(child), Some(parent)) = (key(link.props.get(1)), key(link.props.get(2))) else { continue };
if videos.contains_key(&child) && textures.contains_key(&parent) {
video_of.insert(parent, child);
} else if textures.contains_key(&child) && materials.contains_key(&parent) {
match (link.str_at(0), link.str_at(3)) {
(Some("OP"), Some("DiffuseColor" | "Maya|baseColor")) => {
diffuse.insert(parent, child);
}
(Some("OO"), _) => {
diffuse.entry(parent).or_insert(child);
}
_ => {}
}
}
}
let budget = Budget { left: Cell::new(self.left.min(limits.max_data_bytes)), limit: limits.max_data_bytes };
let mut out = Vec::new();
for (id, material) in materials {
let tex = diffuse.get(&id).and_then(|t| textures.get(t)).copied();
let video = diffuse.get(&id).and_then(|t| video_of.get(t)).and_then(|v| videos.get(v)).copied();
let mut image = None;
if let Some(content) = video.or(tex).and_then(|n| n.child("Content")) {
match content.props.first() {
Some(Prop::Bytes(bytes)) if !bytes.is_empty() => {
budget.take(Some(bytes.len()), "an embedded image")?;
image = Some(bytes.clone());
}
Some(Prop::Str(_)) => {
let size = content.props.iter().filter_map(Prop::as_str).try_fold(0usize, |n, s| n.checked_add(s.len()));
// Reserve a conservative bound before concatenating and decoding.
budget.take(size, "an embedded image")?;
let text: String = content.props.iter().filter_map(Prop::as_str).collect();
image = base64_decode(text.trim()).filter(|b| !b.is_empty());
}
_ => {}
}
}
if image.is_none() {
for record in [tex, video].into_iter().flatten() {
let relative = record.child("RelativeFilename").and_then(|n| n.str_at(0)).unwrap_or("");
let absolute = ["FileName", "Filename"]
.into_iter()
.find_map(|n| record.child(n).and_then(|v| v.str_at(0)).filter(|s| !s.is_empty()))
.unwrap_or("");
let leaf = absolute.rsplit(['/', '\\']).next().unwrap_or("");
let mut paths = Vec::new();
for path in [relative, leaf] {
if !path.is_empty() && !paths.contains(&path) {
paths.push(path);
}
}
if !paths.is_empty()
&& let Some(bytes) = resolve(&paths)
{
budget.take(Some(bytes.len()), "a resolved image")?;
image = Some(bytes);
break;
}
}
}
let props = Props::of(material);
let values = props.values("DiffuseColor");
let factor = match values.as_slice() {
[r, g, b, ..] => {
let f = props.f("DiffuseFactor").unwrap_or(1.0);
[(*r * f) as f32, (*g * f) as f32, (*b * f) as f32, 1.0]
}
_ => [1.0; 4],
};
if factor.iter().any(|v| !v.is_finite()) || (image.is_none() && values.len() < 3) {
continue;
}
let raw_name = short_name(material.first_str().unwrap_or(""));
let name = if raw_name.is_empty() {
match id {
Key::Id(i) => format!("Material_{i}"),
Key::Name(s) => format!("Material_{s}"),
}
} else {
raw_name.to_string()
};
out.push(DiffuseMaterial { name, factor, image });
}
out.sort_by(|a, b| a.name.cmp(&b.name));
Ok(out)
}
}
/// What connections name an object by: its id, or its `Class::Name`.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
enum Key<'a> {
@@ -899,7 +1162,7 @@ struct Layer {
impl Layer {
/// The layer and its flat array of `width` numbers per entry.
fn of(geom: &Record, element: &str, data: &str, index: &[&str], width: usize) -> Option<(Layer, Vec<f64>)> {
let e = geom.child(element)?;
let e = geom.children_named(element).min_by_key(|e| e.props.first().and_then(Prop::as_i64) != Some(0))?;
let mapping = match e.child("MappingInformationType").and_then(|m| m.str_at(0)).unwrap_or("ByPolygonVertex") {
"ByPolygonVertex" => Mapping::PolygonVertex,
"ByVertex" | "ByVertice" | "ByControlPoint" => Mapping::Vertex,
@@ -908,7 +1171,7 @@ impl Layer {
_ => return None,
};
let reference = e.child("ReferenceInformationType").and_then(|m| m.str_at(0)).unwrap_or("Direct");
let index = (reference != "Direct").then(|| index.iter().find_map(|n| e.child(n)).map(Record::ints)).flatten();
let index = if matches!(reference, "IndexToDirect" | "Index") { Some(index.iter().find_map(|n| e.child(n))?.ints()) } else { None };
let data = e.child(data)?.floats();
Some((Layer { mapping, index, len: data.len() / width }, data))
}
@@ -1011,14 +1274,18 @@ impl<'a> Build<'a, '_> {
Ok(())
}
/// The mesh geometry of Model `id`, and what it is shared by.
fn geometry_of(&self, id: Key<'a>) -> Option<(Key<'a>, &'a Record)> {
// 6.x: the Model holds its own mesh.
/// The geometries connected to a Model, including its own FBX 6 mesh.
fn geometries_of(&self, id: Key<'a>) -> Vec<(Key<'a>, &'a Record)> {
let mut geometries = Vec::new();
if let Some(model) = self.by_key.get(&id).copied().filter(|o| o.child("Vertices").is_some()) {
return Some((id, model));
geometries.push((id, model));
}
let (gid, _) = self.children_of(id, "Geometry").into_iter().next()?;
Some((gid, self.by_key.get(&gid).copied()?))
for (gid, _) in self.children_of(id, "Geometry") {
if let Some(geometry) = self.by_key.get(&gid).copied().filter(|o| o.child("Vertices").is_some()) {
geometries.push((gid, geometry));
}
}
geometries
}
/// The image of Texture `tid`.
@@ -1086,7 +1353,18 @@ impl<'a> Build<'a, '_> {
return Ok(Some(*m));
}
let Some(record) = self.by_key.get(&mid).copied() else { return Ok(None) };
let mut m = material(short_name(record.first_str().unwrap_or("")), record);
let name = short_name(record.first_str().unwrap_or(""));
let fallback;
let name = if name.is_empty() {
fallback = match mid {
Key::Id(i) => format!("Material_{i}"),
Key::Name(s) => format!("Material_{s}"),
};
&fallback
} else {
name
};
let mut m = material(name, record);
for (tid, prop) in self.children_of(mid, "Texture") {
let slot = match prop {
Some("DiffuseColor") => &mut m.base_color_texture,
@@ -1128,13 +1406,14 @@ impl<'a> Build<'a, '_> {
return Err(malformed(KIND, "mesh array contains a non-numeric, non-finite or fractional index value"));
}
for (element, names) in [
("LayerElementNormal", &["Normals", "NormalsIndex"][..]),
("LayerElementNormal", &["Normals", "NormalsIndex", "NormalIndex"][..]),
("LayerElementUV", &["UV", "UVIndex"][..]),
("LayerElementMaterial", &["Materials"][..]),
("LayerElementColor", &["Colors", "ColorIndex"][..]),
("LayerElementColor", &["Colors", "ColorIndex", "ColorsIndex"][..]),
] {
if let Some(layer) = geom.children_named(element).min_by_key(|e| e.props.first().and_then(Prop::as_i64) != Some(0))
&& names.iter().filter_map(|n| layer.child(n)).any(|r| !valid(r, false))
// index arrays and material ids are integers; the data arrays are floats
&& names.iter().filter_map(|n| layer.child(n).map(|r| (r, n.ends_with("Index") || *n == "Materials"))).any(|(r, integers)| !valid(r, integers))
{
return Err(malformed(KIND, "mesh array contains a non-numeric or non-finite value"));
}
@@ -1143,13 +1422,30 @@ impl<'a> Build<'a, '_> {
let pool = |what: &str, n: usize| {
if n > limits.max_vertices { Err(too_large(KIND, format!("more than {} {what}", limits.max_vertices))) } else { Ok(()) }
};
let positions: Vec<[f64; 3]> = vertices.floats().as_chunks::<3>().0.iter().map(|c| c.map(finite)).collect();
let mut positions: Vec<[f64; 3]> = vertices.floats().as_chunks::<3>().0.iter().map(|c| c.map(finite)).collect();
pool("vertices", positions.len())?;
let original_positions = positions.len();
let normals = Layer::of(geom, "LayerElementNormal", "Normals", &["NormalsIndex", "NormalIndex"], 3);
let uvs = Layer::of(geom, "LayerElementUV", "UV", &["UVIndex"], 2);
let slots = geom.child("LayerElementMaterial").and_then(|e| {
let colors = Layer::of(geom, "LayerElementColor", "Colors", &["ColorIndex", "ColorsIndex"], 4);
if colors.is_some() && positions.len().checked_mul(16).is_none_or(|n| n > self.budget.left.get()) {
return Err(too_large(KIND, "vertex colors exceed the memory limit"));
}
let mut vertex_colors = colors.as_ref().map(|_| vec![[1.0; 4]; positions.len()]).unwrap_or_default();
let mut color_indices = HashMap::<(u32, u32), u32>::new();
let mut first_color = HashMap::<u32, u32>::new();
let mut colors_complete = colors.is_some();
let slots = geom.children_named("LayerElementMaterial").min_by_key(|e| e.props.first().and_then(Prop::as_i64) != Some(0)).and_then(|e| {
let mapping = e.child("MappingInformationType").and_then(|m| m.str_at(0)).unwrap_or("AllSame");
Some((mapping == "ByPolygon", e.child("Materials")?.ints()))
// a polygon goes to one part, so per-corner and per-vertex IDs use its first corner
let by = match mapping {
"ByPolygon" => Mapping::Polygon,
"ByPolygonVertex" => Mapping::PolygonVertex,
"ByVertex" | "ByVertice" | "ByControlPoint" => Mapping::Vertex,
"AllSame" => Mapping::AllSame,
_ => return None,
};
Some((by, e.child("Materials")?.ints()))
});
let (normal_layer, uv_layer) = (normals.as_ref().map(|l| &l.0), uvs.as_ref().map(|l| &l.0));
@@ -1159,15 +1455,58 @@ impl<'a> Build<'a, '_> {
let (mut broken, mut poly) = (false, 0usize);
let polygon_indices = pvi.ints();
let index_count = polygon_indices.len();
for (pv, raw) in polygon_indices.into_iter().enumerate() {
let mut poly_start = 0usize;
for (pv, raw) in polygon_indices.iter().copied().enumerate() {
// The last corner of a polygon is stored as `-index - 1`.
let (v, last) = if raw < 0 { (!raw, true) } else { (raw, self.strict_geometry && pv + 1 == index_count) };
match u32::try_from(v).ok().filter(|v| (*v as usize) < positions.len()) {
Some(p) => face.push(Corner {
position: p,
normal: Layer::at(normal_layer, pv, p as usize, poly),
uv: Layer::at(uv_layer, pv, p as usize, poly),
}),
match u32::try_from(v).ok().filter(|v| (*v as usize) < original_positions) {
Some(p) => {
let normal = Layer::at(normal_layer, pv, p as usize, poly);
let uv = Layer::at(uv_layer, pv, p as usize, poly);
let mut position = p;
if let Some((layer, values)) = &colors {
let color = Layer::at(Some(layer), pv, p as usize, poly);
let rgba = usize::try_from(color)
.ok()
.and_then(|i| i.checked_mul(4))
.and_then(|i| i.checked_add(4).and_then(|end| values.get(i..end)));
if let Some(rgba) = rgba {
if let Some(existing) = color_indices.get(&(p, color)) {
position = *existing;
} else {
// Color seams need two maps as well as the output pools; do not
// grow them past the remaining file budget on hostile corner lists.
let staging = color_indices
.len()
.checked_add(1)
.and_then(|n| n.checked_mul(64))
.and_then(|n| positions.len().checked_mul(40).and_then(|pool| n.checked_add(pool)));
if staging.is_none_or(|n| n > self.budget.left.get()) {
return Err(too_large(KIND, "color seams exceed the memory limit"));
}
if first_color.insert(p, color).is_some() {
pool("vertices", positions.len().saturating_add(1))?;
if positions.len().saturating_add(1).checked_mul(40).is_none_or(|n| n > self.budget.left.get()) {
return Err(too_large(KIND, "colored vertices exceed the memory limit"));
}
position = u32::try_from(positions.len()).map_err(|_| too_large(KIND, "too many vertices"))?;
let Some(original) = positions.get(p as usize).copied() else { continue };
positions.push(original);
vertex_colors.push([1.0; 4]);
}
if let Some(dst) = vertex_colors.get_mut(position as usize)
&& let [r, g, b, a] = rgba
{
*dst = [finite(*r) as f32, finite(*g) as f32, finite(*b) as f32, finite(*a) as f32];
}
color_indices.insert((p, color), position);
}
} else {
colors_complete = false;
}
}
face.push(Corner { position, normal, uv });
}
None if self.strict_geometry => return Err(malformed(KIND, format!("polygon vertex index {raw} out of range"))),
None => broken = true,
}
@@ -1178,6 +1517,7 @@ impl<'a> Build<'a, '_> {
continue;
}
let (corners, p_index, was_broken) = (std::mem::take(&mut face), poly, std::mem::take(&mut broken));
let first_pv = std::mem::replace(&mut poly_start, pv + 1);
poly += 1;
if was_broken || corners.len() < 3 {
self.skipped += 1;
@@ -1187,11 +1527,22 @@ impl<'a> Build<'a, '_> {
if self.triangles > limits.max_triangles {
return Err(too_large(KIND, format!("more than {} triangles", limits.max_triangles)));
}
let slot = slots.as_ref().and_then(|(by_poly, ids)| {
let k = if *by_poly { *ids.get(p_index)? } else { *ids.first()? };
materials.get(usize::try_from(k).ok()?).copied()
});
let slot = slot.or_else(|| materials.first().copied());
let slot = match &slots {
Some((by, ids)) => {
let k = match by {
Mapping::Polygon => ids.get(p_index),
Mapping::PolygonVertex => ids.get(first_pv),
Mapping::Vertex => polygon_indices
.get(first_pv)
.map(|&r| if r < 0 { !r } else { r })
.and_then(|v| usize::try_from(v).ok())
.and_then(|v| ids.get(v)),
Mapping::AllSame => ids.first(),
};
k.and_then(|k| usize::try_from(*k).ok()).and_then(|k| materials.get(k)).copied()
}
None => materials.first().copied(),
};
let at = match part_of.get(&slot) {
Some(at) => *at,
None => {
@@ -1221,6 +1572,20 @@ impl<'a> Build<'a, '_> {
let uvs: Option<Vec<[f32; 2]>> =
uvs.map(|(_, d)| d.as_chunks::<2>().0.iter().map(|c| [finite(c[0]) as f32, 1.0 - finite(c[1]) as f32]).collect());
pool("UVs", uvs.as_ref().map_or(0, Vec::len))?;
Ok(Some(Mesh { name: name.to_string(), positions, normals, uv_sets: uvs.into_iter().collect(), parts, ..Mesh::default() }))
if !colors_complete {
vertex_colors.clear();
if colors.is_some() {
self.scene.warnings.push("FBX: incomplete vertex color layer was omitted".into());
}
}
Ok(Some(Mesh {
name: name.to_string(),
positions,
normals,
uv_sets: uvs.into_iter().collect(),
colors: vertex_colors,
parts,
..Mesh::default()
}))
}
}
+301
View File
@@ -0,0 +1,301 @@
//! FBX and referenced texture files in a ZIP container.
use super::{Document, KIND};
use crate::{Error, Limits, Result, malformed, too_large};
const ZIP_MAGIC: &[u8] = b"PK\x03\x04";
/// Whether bytes begin with a ZIP local-file header.
pub fn is_package(bytes: &[u8]) -> bool {
bytes.starts_with(ZIP_MAGIC)
}
/// Limits for an FBX package and each referenced texture.
#[derive(Clone, Copy, Debug)]
pub struct PackageLimits {
pub max_entries: u64,
pub max_model_bytes: u64,
pub max_texture_bytes: u64,
pub max_total_bytes: u64,
}
/// An open FBX package. Reads are charged to its cumulative decompression budget.
pub struct Package<'a> {
zip: craft_zip::Archive<'a>,
limits: PackageLimits,
name: String,
/// Textures read so far by name: a texture several materials share is read, and charged, once.
textures: std::collections::HashMap<String, Vec<u8>>,
}
impl<'a> Package<'a> {
pub fn open(bytes: &'a [u8], limits: PackageLimits) -> Result<Self> {
let zip = craft_zip::Archive::open(bytes, craft_zip::Limits { max_entries: limits.max_entries, max_total_bytes: limits.max_total_bytes })
.map_err(|e| malformed(KIND, format!("package: {e}")))?;
let name = zip
.names()
.find(|n| n.rsplit('.').next().is_some_and(|ext| ext.eq_ignore_ascii_case("fbx")))
.map(|n| n.into_owned())
.ok_or_else(|| malformed(KIND, "the package contains no FBX file"))?;
Ok(Self { zip, limits, name, textures: std::collections::HashMap::new() })
}
pub fn model(&mut self) -> Result<Vec<u8>> {
let bytes = self
.zip
.read(&self.name, self.limits.max_model_bytes)
.map_err(|e| malformed(KIND, format!("package: {e}")))?
.ok_or_else(|| malformed(KIND, "package FBX is missing"))?;
if is_package(&bytes) {
return Err(malformed(KIND, "the package holds another package"));
}
Ok(bytes)
}
/// Resolve candidate names within the package; outside paths are never read.
pub fn texture(&mut self, paths: &[&str]) -> Option<Vec<u8>> {
for name in paths.iter().filter_map(|p| below(p)) {
if let Some(data) = self.textures.get(&name) {
return Some(data.clone());
}
if let Some(data) = self.zip.read(&name, self.limits.max_texture_bytes).ok().flatten() {
self.textures.insert(name, data.clone());
return Some(data);
}
}
None
}
}
/// Resolve a safe relative path inside a package.
fn below(path: &str) -> Option<String> {
let path = path.replace('\\', "/");
if path.starts_with('/') || path.contains(':') {
return None;
}
let mut parts = Vec::new();
for part in path.split('/') {
match part {
"" | "." => {}
".." => {
parts.pop()?;
}
part => parts.push(part),
}
}
(!parts.is_empty()).then(|| parts.join("/"))
}
/// Package the FBX and its externally resolved diffuse textures, if any. Returns unresolved paths.
pub fn write(
doc: &Document,
file_name: &str,
bytes: &[u8],
limits: &Limits,
package_limits: PackageLimits,
found: &mut dyn FnMut(&str) -> Option<Vec<u8>>,
) -> Result<(Option<Vec<u8>>, Vec<String>)> {
if is_package(bytes) {
return Ok((None, Vec::new()));
}
let mut files: Vec<(String, Vec<u8>)> = Vec::new();
let mut by_name = std::collections::HashMap::<String, usize>::new();
let mut too_many = false;
let mut missing = Vec::new();
let mut missing_seen = std::collections::HashSet::<String>::new();
let mut total = bytes.len() as u64;
if total > package_limits.max_total_bytes || total > package_limits.max_model_bytes {
return Err(too_large(KIND, "the FBX and its textures exceed the package limit"));
}
let mut exceeded = false;
doc.diffuse_materials(limits, &mut |paths| {
// candidates in order of preference: an earlier one that exists wins over a later cached one
for path in paths {
let Some(name) = below(path) else { continue };
if let Some(data) = by_name.get(&name).and_then(|&i| files.get(i)).map(|(_, d)| d.clone()) {
return Some(data);
}
// once a resolved texture has not fit, the package fails: stop calling the resolver
if too_many || exceeded {
return None;
}
if let Some(data) = found(path).filter(|d| d.len() as u64 <= package_limits.max_texture_bytes) {
// the model is an entry too; only a texture that resolves needs a new entry
if files.len() as u64 + 1 >= package_limits.max_entries {
too_many = true;
return None;
}
total = total.saturating_add(data.len() as u64);
if total > package_limits.max_total_bytes {
exceeded = true;
return None;
}
by_name.insert(name.clone(), files.len());
files.push((name, data.clone()));
return Some(data);
}
}
if let Some(first) = paths.first().filter(|p| missing_seen.insert((**p).to_string())) {
missing.push((*first).to_string());
}
None
})?;
if exceeded {
return Err(too_large(KIND, "the FBX and its textures exceed the package limit"));
}
if too_many {
return Err(too_large(KIND, "too many package entries"));
}
if files.is_empty() {
return Ok((None, missing));
}
let leaf = file_name.rsplit(['/', '\\']).next().filter(|n| !n.is_empty()).unwrap_or("mesh.fbx");
// the package is found by its `.fbx` entry, so the model always carries that extension
let leaf = if leaf.rsplit_once('.').is_some_and(|(_, ext)| ext.eq_ignore_ascii_case("fbx")) { leaf.to_string() } else { format!("{leaf}.fbx") };
let leaf = leaf.as_str();
let mut writer = craft_zip::Writer::new();
for (name, data) in std::iter::once((leaf, bytes)).chain(files.iter().map(|(n, b)| (n.as_str(), b.as_slice()))) {
writer.add(name, data, craft_zip::Compression::Stored).map_err(|e| Error::Malformed(format!("FBX package: {e}")))?;
}
Ok((Some(writer.finish().map_err(|e| Error::Malformed(format!("FBX package: {e}")))?), missing))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::fbx::read_document;
const MODEL: &[u8] = b"FBXHeaderExtension: { FBXVersion: 7400 } Objects: { Material: 1, \"Material::M\", \"\" { } Texture: 2, \"Texture::T\", \"\" { RelativeFilename: \"tex/a.png\" } } Connections: { C: \"OP\",2,1,\"DiffuseColor\" }";
fn bounds() -> PackageLimits {
PackageLimits { max_entries: 4, max_model_bytes: 4096, max_texture_bytes: 32, max_total_bytes: 8192 }
}
#[test]
fn package_roundtrip_and_limits() {
let limits = Limits::default();
let doc = read_document(MODEL, &limits).unwrap();
let (bytes, missing) = write(&doc, "asset.fbx", MODEL, &limits, bounds(), &mut |p| (p == "tex/a.png").then(|| b"image".to_vec())).unwrap();
assert!(missing.is_empty());
let bytes = bytes.unwrap();
let mut package = Package::open(&bytes, bounds()).unwrap();
assert_eq!(package.model().unwrap(), MODEL);
assert_eq!(package.texture(&["../tex/a.png", "tex/a.png"]), Some(b"image".to_vec()));
assert!(package.texture(&["../tex/a.png"]).is_none());
let small = PackageLimits { max_total_bytes: MODEL.len() as u64 + 4, ..bounds() };
assert!(matches!(write(&doc, "a.fbx", MODEL, &limits, small, &mut |_| Some(b"image".to_vec())), Err(Error::TooLarge(_))));
assert!(Package::open(&bytes, PackageLimits { max_entries: 1, ..bounds() }).is_err());
}
#[test]
fn an_earlier_candidate_wins_over_a_cached_later_one() {
let model = b"FBXHeaderExtension: { FBXVersion: 7400 } Objects: { Material: 1, \"Material::A\", \"\" { } Texture: 2, \"Texture::A\", \"\" { RelativeFilename: \"base.png\" } Material: 3, \"Material::B\", \"\" { } Texture: 4, \"Texture::B\", \"\" { RelativeFilename: \"maps/base.png\" FileName: \"C:/art/base.png\" } } Connections: { C: \"OP\",2,1,\"DiffuseColor\" C: \"OP\",4,3,\"DiffuseColor\" }";
let limits = Limits::default();
let doc = read_document(model, &limits).unwrap();
let mut found = |p: &str| match p {
"base.png" => Some(b"flat".to_vec()),
"maps/base.png" => Some(b"maps".to_vec()),
_ => None,
};
let (bytes, missing) = write(&doc, "asset.fbx", model, &limits, bounds(), &mut found).unwrap();
assert!(missing.is_empty());
let bytes = bytes.unwrap();
let mut package = Package::open(&bytes, bounds()).unwrap();
assert_eq!(package.texture(&["maps/base.png"]), Some(b"maps".to_vec()));
assert_eq!(package.texture(&["base.png"]), Some(b"flat".to_vec()));
}
#[test]
fn the_model_entry_always_has_an_fbx_extension() {
let limits = Limits::default();
let doc = read_document(MODEL, &limits).unwrap();
let (bytes, _) = write(&doc, "dir/asset", MODEL, &limits, bounds(), &mut |_| Some(b"image".to_vec())).unwrap();
let bytes = bytes.unwrap();
let mut package = Package::open(&bytes, bounds()).unwrap();
assert_eq!(package.model().unwrap(), MODEL);
}
#[test]
fn texture_resolution_stops_at_the_entry_limit() {
let mut model = String::from("FBXHeaderExtension: { FBXVersion: 7400 } Objects: {");
let mut links = String::new();
for i in 0..20 {
model.push_str(&format!(
" Material: {m}, \"Material::M{i}\", \"\" {{ }} Texture: {t}, \"Texture::T{i}\", \"\" {{ RelativeFilename: \"t{i}.png\" }}",
m = 2 * i + 1,
t = 2 * i + 2
));
links.push_str(&format!(" C: \"OP\",{t},{m},\"DiffuseColor\"", m = 2 * i + 1, t = 2 * i + 2));
}
model.push_str(&format!(" }} Connections: {{{links} }}"));
let limits = Limits::default();
let doc = read_document(model.as_bytes(), &limits).unwrap();
let mut calls = 0;
let result = write(&doc, "a.fbx", model.as_bytes(), &limits, bounds(), &mut |_| {
calls += 1;
Some(b"image".to_vec())
});
assert!(matches!(result, Err(Error::TooLarge(_))));
assert!(calls <= bounds().max_entries as usize, "{calls} resolver calls past the entry limit");
}
#[test]
fn resolution_stops_once_the_byte_limit_is_passed() {
let mut model = String::from("FBXHeaderExtension: { FBXVersion: 7400 } Objects: {");
let mut links = String::new();
for i in 0..10 {
model.push_str(&format!(
" Material: {m}, \"Material::M{i}\", \"\" {{ }} Texture: {t}, \"Texture::T{i}\", \"\" {{ RelativeFilename: \"big.png\" }}",
m = 2 * i + 1,
t = 2 * i + 2
));
links.push_str(&format!(" C: \"OP\",{t},{m},\"DiffuseColor\"", m = 2 * i + 1, t = 2 * i + 2));
}
model.push_str(&format!(" }} Connections: {{{links} }}"));
let limits = Limits::default();
let doc = read_document(model.as_bytes(), &limits).unwrap();
let tight = PackageLimits { max_total_bytes: model.len() as u64 + 4, max_model_bytes: 1 << 20, ..bounds() };
let mut calls = 0;
let result = write(&doc, "a.fbx", model.as_bytes(), &limits, tight, &mut |_| {
calls += 1;
Some(b"image".to_vec())
});
assert!(matches!(result, Err(Error::TooLarge(_))));
assert_eq!(calls, 1, "one texture past the limit ends resolution");
// a model past its own limit is refused before any texture is resolved
let small = PackageLimits { max_model_bytes: 8, ..bounds() };
let mut calls = 0;
assert!(matches!(
write(&doc, "a.fbx", model.as_bytes(), &limits, small, &mut |_| {
calls += 1;
None
}),
Err(Error::TooLarge(_))
));
assert_eq!(calls, 0);
}
#[test]
fn the_entry_limit_counts_only_textures_that_resolve() {
let limits = Limits::default();
let doc = read_document(MODEL, &limits).unwrap();
// the model alone fills a one-entry package, so a resolved texture cannot fit
let one = PackageLimits { max_entries: 1, ..bounds() };
assert!(matches!(write(&doc, "a.fbx", MODEL, &limits, one, &mut |_| Some(b"image".to_vec())), Err(Error::TooLarge(_))));
// an unresolved texture adds no entry, so it is reported missing rather than overflowing
let (bytes, missing) = write(&doc, "a.fbx", MODEL, &limits, one, &mut |_| None).unwrap();
assert!(bytes.is_none());
assert_eq!(missing, vec!["tex/a.png".to_string()]);
}
#[test]
fn a_shared_texture_is_read_and_charged_once() {
let limits = Limits::default();
let doc = read_document(MODEL, &limits).unwrap();
let (bytes, _) = write(&doc, "asset.fbx", MODEL, &limits, bounds(), &mut |_| Some(b"image".to_vec())).unwrap();
let bytes = bytes.unwrap();
// room for the model and one read of the texture
let tight = PackageLimits { max_total_bytes: MODEL.len() as u64 + 5, ..bounds() };
let mut package = Package::open(&bytes, tight).unwrap();
package.model().unwrap();
assert_eq!(package.texture(&["tex/a.png"]), Some(b"image".to_vec()));
assert_eq!(package.texture(&["tex/a.png"]), Some(b"image".to_vec()), "served from the cache");
}
}
+215
View File
@@ -628,6 +628,16 @@ fn fbx6_model_holds_its_mesh() {
}
}
#[test]
fn layer_zero_takes_precedence_and_an_indexed_layer_needs_indices() {
let layer_one = "LayerElementUV: 1 { MappingInformationType: \"ByPolygonVertex\" ReferenceInformationType: \"Direct\" UV: 0.25,0.25, 0.25,0.25, 0.25,0.25, 0.25,0.25, 0.25,0.25, 0.25,0.25, 0.25,0.25 }\n";
let text = ASCII.replace("LayerElementUV: 0 {", &format!("{layer_one} LayerElementUV: 0 {{"));
let scene = parse(text.as_bytes()).unwrap();
assert_eq!(scene.meshes[0].uv_sets[0][0], [0.0, 1.0]);
let missing = ASCII.replace("UVIndex: *7", "UnusedIndex: *7");
assert!(parse(missing.as_bytes()).unwrap().meshes[0].uv_sets.is_empty());
}
#[test]
fn optional_inactive_pre_rotation_keeps_legacy_placements() {
let text = "FBXHeaderExtension: { FBXVersion: 7400 } Objects: { Model: 1, \"Model::A\", \"Null\" { Properties70: { P: \"PreRotation\", \"\", \"\", \"\", 0,0,90 } } }";
@@ -672,3 +682,208 @@ fn object_only_hierarchy_ignores_property_connections() {
assert_eq!(scene.roots.len(), 2);
assert_eq!(doc.scene(&Limits::default(), &mut no_external).unwrap().roots.len(), 1);
}
#[test]
fn binary_scalar_properties_are_bounded_before_reserving() {
let mut bytes = BINARY_MAGIC.to_vec();
bytes.extend(7400u32.to_le_bytes());
let count = 1000u32;
let end = bytes.len() + 13 + 1 + count as usize * 2;
bytes.extend((end as u32).to_le_bytes());
bytes.extend(count.to_le_bytes());
bytes.extend((count * 2).to_le_bytes());
bytes.push(1);
bytes.push(b'A');
for _ in 0..count {
bytes.extend([b'C', 1]);
}
let limits = Limits { max_data_bytes: 1024, ..Limits::default() };
assert!(matches!(read_document(&bytes, &limits), Err(Error::TooLarge(_))));
assert!(read_document(&bytes, &Limits::default()).is_ok());
}
#[test]
fn fbx_invalid_material_slot_is_unassigned_and_unnamed_material_has_id() {
let text = ASCII.replace("a: 0,1\n", "a: 0,99\n").replace("\"Material::Red\"", "\"\"");
let scene = parse(text.as_bytes()).unwrap();
assert!(scene.materials.iter().any(|m| m.name == "Material_400"));
let parts = &scene.meshes[0].parts;
assert!(parts.iter().any(|p| p.material.is_none()));
assert!(parts.iter().any(|p| p.material.is_some()));
}
#[test]
fn cameras_keep_lens_and_model_hierarchy() {
let text = r#"FBXHeaderExtension: { FBXVersion: 7400 }
Objects: {
Model: 1, "Model::Rig", "Null" { Properties70: { P: "Lcl Translation", "", "", "", 1,2,3 } }
Model: 2, "Model::View", "Camera" { }
NodeAttribute: 3, "NodeAttribute::Lens", "Camera" { Properties70: {
P: "FocalLength", "", "", "", 50
P: "FilmHeight", "", "", "", 1
} }
Model: 4, "Model::Ortho", "Camera" { Properties70: {
P: "CameraProjectionType", "", "", "", 1
P: "OrthoZoom", "", "", "", 3
} }
Model: 5, "Model::Producer Perspective", "Camera" { }
}
Connections: {
C: "OO",2,1
C: "OO",3,2
}
"#;
let scene = parse(text.as_bytes()).unwrap();
assert_eq!(scene.cameras.len(), 2);
let crate::Projection::Perspective { yfov, .. } = scene.cameras[0].projection else { panic!("perspective") };
assert!((yfov - 2.0 * (25.4_f64 / 100.0).atan()).abs() < 1e-12);
assert!(matches!(scene.cameras[1].projection, crate::Projection::Orthographic { ymag: 3.0, .. }));
let camera_node = scene.nodes.iter().find(|n| n.camera == Some(0)).unwrap();
let forward = crate::transform_point(&camera_node.transform, [0.0, 0.0, -1.0]);
assert!((forward[0] - 1.0).abs() < 1e-12);
let rig = scene.nodes.iter().find(|n| n.name == "Rig").unwrap();
assert!(rig.children.iter().any(|i| scene.nodes[*i].name == "View"));
let limits = Limits { max_extras: 1, ..Limits::default() };
assert!(matches!(read(text.as_bytes(), &limits, &mut no_external), Err(Error::TooLarge(_))));
}
#[test]
fn all_connected_and_unplaced_geometries_are_instances() {
let geometry = |id| format!("Geometry: {id}, \"Geometry::Part{id}\", \"Mesh\" {{ Vertices: 0,0,0,1,0,0,0,1,0 PolygonVertexIndex: 0,1,-3 }}\n");
let text = format!(
"FBXHeaderExtension: {{ FBXVersion: 7400 }}\nObjects: {{\n{}{}{} Model: 10, \"Model::M\", \"Mesh\" {{}} }}\nConnections: {{ C: \"OO\",1,10 C: \"OO\",2,10 C: \"OO\",2,10 }}",
geometry(1),
geometry(2),
geometry(3)
);
let scene = parse(text.as_bytes()).unwrap();
assert_eq!(scene.meshes.len(), 3);
assert_eq!(scene.instances(&Limits::default()).unwrap().len(), 3);
assert_eq!(scene.roots.len(), 2);
}
#[test]
fn fbx_corner_colors_keep_seams_and_incomplete_layers_are_omitted() {
let layer = r#"LayerElementColor: 0 {
MappingInformationType: "ByPolygonVertex"
ReferenceInformationType: "Direct"
Colors: 1,0,0,1, 0,1,0,1, 0,0,1,1, 1,1,0,1, 0,1,0,1, 0,0,1,1
}"#;
let text = ASCII.replace("LayerElementNormal: 0 {", &format!("{layer}\n LayerElementNormal: 0 {{"));
// Add a color for every corner of the original quad and triangle.
let text = text.replace("0,0,1,1\n}", "0,0,1,1, 1,0,1,1\n}");
let scene = parse(text.as_bytes()).unwrap();
let mesh = &scene.meshes[0];
let colors: Vec<_> = mesh.parts.iter().flat_map(|part| part.corners.iter()).map(|corner| mesh.colors[corner.position as usize]).collect();
assert_eq!(colors.len(), 7);
assert!(colors.contains(&[1.0, 0.0, 0.0, 1.0]));
assert!(colors.contains(&[0.0, 1.0, 0.0, 1.0]));
assert!(colors.contains(&[1.0, 0.0, 1.0, 1.0]));
let incomplete = text.replace(", 1,0,1,1\n}", "\n}");
assert!(parse(incomplete.as_bytes()).unwrap().meshes[0].colors.is_empty());
}
#[test]
fn ascii_budget_applies_while_reading_values_and_records() {
let limits = Limits { max_data_bytes: 512, ..Limits::default() };
let many_values = format!("Objects: {{\n A: {}\n}}", "1,".repeat(1000));
assert!(matches!(read_document(many_values.as_bytes(), &limits), Err(Error::TooLarge(_))));
let many_records = format!("Objects: {{\n{}\n}}", "A: {}\n".repeat(1000));
assert!(matches!(read_document(many_records.as_bytes(), &limits), Err(Error::TooLarge(_))));
let giant_string = format!("Objects: {{ A: \"{}\" }}", "x".repeat(1024));
assert!(matches!(read_document(giant_string.as_bytes(), &limits), Err(Error::TooLarge(_))));
// Empty punctuation and comments do not consume memory in proportion to their length.
let ignored = format!(";{}\nObjects: {{}}", ",".repeat(10_000));
assert!(read_document(ignored.as_bytes(), &limits).is_ok());
}
#[test]
fn strict_ascii_reports_unbalanced_and_unterminated_records() {
let strict = |s: &str| crate::fbx::read_document_strict(s.as_bytes(), &Limits::default());
assert!(strict(ASCII).is_ok());
for s in ["Objects: { A: 1", "Objects: { } }", "Objects: { A: \"unterminated", "Objects: { A: 1 2 }"] {
assert!(strict(s).is_err(), "{s}");
}
assert!(read_document(b"Objects: { A: 1", &Limits::default()).is_ok());
}
#[test]
fn strict_binary_requires_a_closing_record() {
let doc = document(ASCII.as_bytes());
let bytes = write_binary(&doc.records, 7400, false);
assert!(crate::fbx::read_document_strict(&bytes, &Limits::default()).is_ok());
let truncated = &bytes[..bytes.len() - 29];
assert!(read_document(truncated, &Limits::default()).is_ok());
assert!(crate::fbx::read_document_strict(truncated, &Limits::default()).is_err());
}
#[test]
fn diffuse_materials_preserve_factors_and_texture_candidates() {
let text = with(
"Texture: 600, \"Texture::Base\", \"\" { RelativeFilename: \"maps/base.png\" FileName: \"C:/a/base.png\" }\n",
"C: \"OP\",600,400,\"Maya|baseColor\"\n",
);
let doc = document(text.as_bytes());
let mut asked = Vec::new();
let materials = doc
.diffuse_materials(&Limits::default(), &mut |paths| {
asked.push(paths.iter().map(|s| s.to_string()).collect::<Vec<_>>());
Some(b"image".to_vec())
})
.unwrap();
assert_eq!(asked, vec![vec!["maps/base.png", "base.png"]]);
let red = materials.iter().find(|m| m.name == "Red").unwrap();
assert_eq!(red.factor, [1.0, 0.0, 0.0, 1.0]);
assert_eq!(red.image.as_deref(), Some(&b"image"[..]));
let small = Limits { max_data_bytes: 4, ..Limits::default() };
assert!(matches!(doc.diffuse_materials(&small, &mut |_| Some(b"image".to_vec())), Err(Error::TooLarge(_))));
let embedded_text = with(
&format!("Texture: 600, \"Texture::Base\", \"\" {{}}\nVideo: 700, \"Video::Base\", \"Clip\" {{ Content: \"{}\" }}\n", b64(b"embedded")),
"C: \"OP\",600,400,\"DiffuseColor\"\nC: \"OO\",700,600\n",
);
let embedded = document(embedded_text.as_bytes()).diffuse_materials(&Limits::default(), &mut |_| None).unwrap();
assert_eq!(embedded.iter().find(|m| m.name == "Red").unwrap().image.as_deref(), Some(&b"embedded"[..]));
}
#[test]
fn per_corner_and_per_vertex_material_ids_use_each_polygons_first_corner() {
let by_corner = ASCII.replace("MappingInformationType: \"ByPolygon\"\n\t\t\tReferenceInformationType: \"IndexToDirect\"\n\t\t\tMaterials: *2 {\n\t\t\t\ta: 0,1", "MappingInformationType: \"ByPolygonVertex\"\n\t\t\tReferenceInformationType: \"IndexToDirect\"\n\t\t\tMaterials: *7 {\n\t\t\t\ta: 0,0,0,0,1,1,1");
assert_ne!(by_corner, ASCII);
let parts = &parse(by_corner.as_bytes()).unwrap().meshes[0].parts;
assert_eq!(parts.len(), 2, "the triangle takes material 1 from its first corner");
// The triangle starts at control point 4, whose material is 1.
let by_vertex = ASCII.replace("a: 0,1,2,-4,0,1,-5", "a: 0,1,2,-4,4,0,-2").replace(
"MappingInformationType: \"ByPolygon\"\n\t\t\tReferenceInformationType: \"IndexToDirect\"\n\t\t\tMaterials: *2 {\n\t\t\t\ta: 0,1",
"MappingInformationType: \"ByControlPoint\"\n\t\t\tReferenceInformationType: \"IndexToDirect\"\n\t\t\tMaterials: *5 {\n\t\t\t\ta: 0,0,0,0,1",
);
let scene = parse(by_vertex.as_bytes()).unwrap();
assert_eq!(scene.meshes[0].parts.len(), 2);
assert!(scene.meshes[0].parts.iter().all(|p| p.material.is_some()));
}
#[test]
fn strict_colors_validate_and_read_the_same_layer() {
let good = "LayerElementColor: 0 { MappingInformationType: \"AllSame\" ReferenceInformationType: \"Direct\" Colors: 0,1,0,1 }\n";
let bad = "LayerElementColor: 1 { MappingInformationType: \"AllSame\" ReferenceInformationType: \"Direct\" Colors: \"bad\",1,0,0,1 }\n";
let text = ASCII.replace("LayerElementNormal: 0 {", &format!("{bad} {good} LayerElementNormal: 0 {{"));
let options = crate::fbx::SceneOptions { strict_geometry: true, ..Default::default() };
let scene = document(text.as_bytes()).scene_with_options(&Limits::default(), &mut no_external, &options).unwrap();
assert!(scene.meshes[0].colors.iter().all(|c| *c == [0.0, 1.0, 0.0, 1.0]));
assert!(!scene.meshes[0].colors.is_empty());
let only_bad = ASCII.replace("LayerElementNormal: 0 {", &format!("{} LayerElementNormal: 0 {{", bad.replace(": 1 {", ": 0 {")));
assert!(document(only_bad.as_bytes()).scene_with_options(&Limits::default(), &mut no_external, &options).is_err());
}
#[test]
fn strict_geometry_validates_the_colors_index_alias() {
let layer = |index: &str| {
format!(
"LayerElementColor: 0 {{ MappingInformationType: \"AllSame\" ReferenceInformationType: \"IndexToDirect\" Colors: 0,1,0,1 ColorsIndex: {index} }}\n"
)
};
let with = |index: &str| ASCII.replace("LayerElementNormal: 0 {", &format!("{} LayerElementNormal: 0 {{", layer(index)));
let options = crate::fbx::SceneOptions { strict_geometry: true, ..Default::default() };
assert!(document(with("0").as_bytes()).scene_with_options(&Limits::default(), &mut no_external, &options).is_ok());
assert!(document(with("\"bad\",0").as_bytes()).scene_with_options(&Limits::default(), &mut no_external, &options).is_err());
assert!(document(with("0.5").as_bytes()).scene_with_options(&Limits::default(), &mut no_external, &options).is_err(), "a fractional index");
}
+237
View File
@@ -0,0 +1,237 @@
//! Compatibility EXIF summary and helpers for legacy clients that accepted loosely formed classic TIFF.
//! The ordinary strict EXIF orientation API remains unchanged.
/// Read a legacy orientation, with an optional `Exif\0\0` prefix.
pub fn orientation(bytes: &[u8]) -> u16 {
summarize(bytes.strip_prefix(b"Exif\0\0").unwrap_or(bytes)).orientation.unwrap_or(1)
}
/// Byte-order aware reader over a TIFF-structured blob.
#[derive(Clone, Copy)]
struct Tiff<'a> {
pub data: &'a [u8],
le: bool,
}
#[derive(Clone, Copy, Debug)]
struct Entry {
pub tag: u16,
pub typ: u16,
pub count: u32,
/// Offset of the 4-byte value/offset field within `data`.
pub value_pos: usize,
}
impl<'a> Tiff<'a> {
pub fn new(data: &'a [u8]) -> Option<Tiff<'a>> {
let le = match data.get(0..2)? {
b"II" => true,
b"MM" => false,
_ => return None,
};
Some(Tiff { data, le })
}
pub fn u16_at(&self, pos: usize) -> Option<u16> {
let b = self.data.get(pos..pos.checked_add(2)?)?;
Some(if self.le { u16::from_le_bytes([b[0], b[1]]) } else { u16::from_be_bytes([b[0], b[1]]) })
}
pub fn u32_at(&self, pos: usize) -> Option<u32> {
let b = self.data.get(pos..pos.checked_add(4)?)?;
let a = [b[0], b[1], b[2], b[3]];
Some(if self.le { u32::from_le_bytes(a) } else { u32::from_be_bytes(a) })
}
pub fn first_ifd(&self) -> Option<usize> {
let magic = self.u16_at(2)?;
if magic != 42 {
return None;
}
Some(self.u32_at(4)? as usize)
}
/// Entries of the IFD at `pos` and the offset of the next IFD (0 = none).
pub fn ifd(&self, pos: usize) -> Option<(Vec<Entry>, usize)> {
let n = self.u16_at(pos)? as usize;
if n > 4096 {
return None;
}
let mut v = Vec::with_capacity(n);
for i in 0..n {
let e = pos.checked_add(2)?.checked_add(i.checked_mul(12)?)?;
v.push(Entry { tag: self.u16_at(e)?, typ: self.u16_at(e + 2)?, count: self.u32_at(e + 4)?, value_pos: e + 8 });
}
let next =
pos.checked_add(2).and_then(|p| n.checked_mul(12).and_then(|v| p.checked_add(v))).and_then(|p| self.u32_at(p)).unwrap_or(0) as usize;
Some((v, next))
}
/// First value of a SHORT/LONG entry.
pub fn uint(&self, e: &Entry) -> Option<u32> {
match e.typ {
3 => self.u16_at(e.value_pos).map(u32::from),
4 | 13 => self.u32_at(e.value_pos),
1 | 7 => self.data.get(e.value_pos).map(|b| *b as u32),
_ => None,
}
}
/// Raw bytes of an entry's value (for ASCII/UNDEFINED/BYTE types).
pub fn bytes(&self, e: &Entry) -> Option<&'a [u8]> {
let size = match e.typ {
1 | 2 | 6 | 7 => 1usize,
3 | 8 => 2,
4 | 9 | 11 | 13 => 4,
5 | 10 | 12 => 8,
_ => return None,
};
let len = size.checked_mul(e.count as usize)?;
let start = if len <= 4 { e.value_pos } else { self.u32_at(e.value_pos)? as usize };
self.data.get(start..start.checked_add(len)?)
}
}
/// What the decoder cares about from an EXIF (TIFF-structured) blob.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct ExifSummary {
/// Valid orientation (1–8), if present.
pub orientation: Option<u16>,
/// (offset, length) of the IFD1 JPEG thumbnail within the blob.
pub thumbnail: Option<(usize, usize)>,
/// EXIF ColorSpace = 0xFFFF with InteropIndex "R03" (Adobe RGB (1998) "option" files).
pub adobe_rgb_hint: bool,
}
pub fn summarize(blob: &[u8]) -> ExifSummary {
let mut s = ExifSummary::default();
let Some(t) = Tiff::new(blob) else { return s };
let Some(ifd0) = t.first_ifd() else { return s };
let Some((entries, next)) = t.ifd(ifd0) else { return s };
let mut exif_ifd = None;
for e in &entries {
match e.tag {
0x0112 => s.orientation = t.uint(e).map(|v| v as u16).filter(|v| (1..=8).contains(v)),
0x8769 => exif_ifd = t.uint(e).map(|v| v as usize),
_ => {}
}
}
if next != 0
&& next != ifd0
&& let Some((e1, _)) = t.ifd(next)
{
let off = e1.iter().find(|e| e.tag == 0x0201).and_then(|e| t.uint(e));
let len = e1.iter().find(|e| e.tag == 0x0202).and_then(|e| t.uint(e));
if let (Some(o), Some(l)) = (off, len) {
let (o, l) = (o as usize, l as usize);
if l > 4 && o.checked_add(l).is_some_and(|end| end <= blob.len()) && blob[o..o + 2] == [0xFF, 0xD8] {
s.thumbnail = Some((o, l));
}
}
}
if let Some(p) = exif_ifd
&& let Some((ee, _)) = t.ifd(p)
{
let cs = ee.iter().find(|e| e.tag == 0xA001).and_then(|e| t.uint(e));
let interop = ee.iter().find(|e| e.tag == 0xA005).and_then(|e| t.uint(e));
if cs == Some(0xFFFF)
&& let Some((ie, _)) = interop.and_then(|p| t.ifd(p as usize))
{
s.adobe_rgb_hint = ie.iter().any(|e| e.tag == 1 && t.bytes(e).is_some_and(|b| b.starts_with(b"R03")));
}
}
s
}
/// Build a tiny EXIF blob (little-endian TIFF) with just an orientation tag. Used by tests and by
/// callers that need to stamp orientation on export when no source EXIF exists.
pub fn minimal_exif(orientation: u16) -> Vec<u8> {
let mut v = Vec::with_capacity(26);
v.extend_from_slice(b"II");
v.extend_from_slice(&42u16.to_le_bytes());
v.extend_from_slice(&8u32.to_le_bytes());
v.extend_from_slice(&1u16.to_le_bytes());
v.extend_from_slice(&0x0112u16.to_le_bytes());
v.extend_from_slice(&3u16.to_le_bytes());
v.extend_from_slice(&1u32.to_le_bytes());
v.extend_from_slice(&orientation.to_le_bytes());
v.extend_from_slice(&[0, 0]);
v.extend_from_slice(&0u32.to_le_bytes());
v
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn accepts_legacy_count_zero_but_bounds_the_table() {
let mut bytes = minimal_exif(6);
bytes[14..18].fill(0);
assert_eq!(orientation(&bytes), 6);
bytes.truncate(19);
assert_eq!(orientation(&bytes), 1);
}
#[test]
fn orientation_roundtrip() {
for o in 1..=8 {
assert_eq!(summarize(&minimal_exif(o)).orientation, Some(o));
}
}
#[test]
fn garbage_ok() {
assert_eq!(summarize(b"II*\0\xff\xff\xff\xff"), ExifSummary::default());
assert_eq!(summarize(&[]), ExifSummary::default());
}
/// Big-endian IFD: `entries` are (tag, type, count, value-or-offset), value left-justified.
fn be_ifd(out: &mut Vec<u8>, entries: &[(u16, u16, u32, [u8; 4])], next: u32) {
out.extend_from_slice(&(entries.len() as u16).to_be_bytes());
for (tag, typ, count, value) in entries {
out.extend_from_slice(&tag.to_be_bytes());
out.extend_from_slice(&typ.to_be_bytes());
out.extend_from_slice(&count.to_be_bytes());
out.extend_from_slice(value);
}
out.extend_from_slice(&next.to_be_bytes());
}
#[test]
fn big_endian_thumbnail_and_adobe_rgb_interop_hint() {
let (ifd1, exif, interop, rational, thumb) = (38u32, 68u32, 98u32, 164u32, 172u32);
let mut b = b"MM\0\x2a\0\0\0\x08".to_vec();
be_ifd(&mut b, &[(0x0112, 3, 1, [0, 6, 0, 0]), (0x8769, 4, 1, exif.to_be_bytes())], ifd1);
be_ifd(&mut b, &[(0x0201, 4, 1, thumb.to_be_bytes()), (0x0202, 1, 1, [6, 0, 0, 0])], 0);
be_ifd(&mut b, &[(0xA001, 3, 1, [0xFF, 0xFF, 0, 0]), (0xA005, 13, 1, interop.to_be_bytes())], 0);
// Every tag-1 candidate is read; only the ASCII "R03" one matches.
be_ifd(
&mut b,
&[(1, 99, 1, [0; 4]), (1, 5, 1, rational.to_be_bytes()), (1, 4, 1, *b"R98\0"), (1, 3, 1, *b"R9\0\0"), (1, 2, 4, *b"R03\0")],
0,
);
assert_eq!(b.len(), rational as usize);
b.extend_from_slice(&[0, 0, 0, 1, 0, 0, 0, 2]);
b.extend_from_slice(&[0xFF, 0xD8, 0xFF, 0xD9, 0, 0]);
let s = summarize(&b);
assert_eq!(s, ExifSummary { orientation: Some(6), thumbnail: Some((thumb as usize, 6)), adobe_rgb_hint: true });
// A thumbnail that is not a JPEG, or runs past the blob, is ignored.
let mut not_jpeg = b.clone();
not_jpeg[thumb as usize] = 0;
assert_eq!(summarize(&not_jpeg).thumbnail, None);
assert_eq!(summarize(&b[..b.len() - 1]).thumbnail, None);
// Without ColorSpace 0xFFFF the interop index is not consulted.
let mut srgb = b.clone();
srgb[(exif + 10) as usize] = 0;
assert!(!summarize(&srgb).adobe_rgb_hint);
}
#[test]
fn rejects_bad_headers_types_and_oversized_tables() {
assert_eq!(summarize(b"XX\0\x2a\0\0\0\x08"), ExifSummary::default());
assert_eq!(summarize(b"MM\0\x2b\0\0\0\x08"), ExifSummary::default());
let mut huge = b"MM\0\x2a\0\0\0\x08".to_vec();
huge.extend_from_slice(&5000u16.to_be_bytes());
assert_eq!(summarize(&huge), ExifSummary::default());
// Orientation stored with an unsupported type (SRATIONAL) is ignored.
let mut bytes = minimal_exif(6);
bytes[12..14].copy_from_slice(&10u16.to_le_bytes());
assert_eq!(orientation(&bytes), 1);
assert_eq!(orientation(&[b"Exif\0\0".as_slice(), &minimal_exif(3)].concat()), 3);
}
}
+11 -1
View File
@@ -51,8 +51,18 @@ fn pos(v: Option<f64>) -> Option<f64> {
/// the Exif's tag 700 states; a viewer displays a photo by the Exif's alone, so that one can't tell how a photo
/// is stored.
pub(crate) fn ifd0_orientation(bytes: &[u8]) -> Option<Orientation> {
ifd0_orientation_value(bytes).filter(|v| (1..=8).contains(v)).map(Orientation::from_exif)
}
fn ifd0_orientation_value(bytes: &[u8]) -> Option<u16> {
let tiff = Tiff::parse_with(strip_exif_header(bytes), &ParseOptions::default()).ok()?;
tiff.ifds.first().and_then(|i| i.u16(t::ORIENTATION)).filter(|v| (1..=8).contains(v)).map(Orientation::from_exif)
tiff.ifds.first().and_then(|i| i.u16(t::ORIENTATION))
}
/// Read the first IFD's orientation, treating out-of-range values as normal.
/// Missing or malformed values return `None`.
pub fn ifd0_orientation_compat(bytes: &[u8]) -> Option<Orientation> {
ifd0_orientation_value(bytes).map(Orientation::from_exif)
}
/// The ISO an Exif IFD states beyond the 16-bit ISOSpeedRatings: RecommendedExposureIndex, else
+2 -1
View File
@@ -20,6 +20,7 @@
//! [`Embedded`] and call [`from_embedded`].
pub mod camera_fields;
pub mod compat_orientation;
mod container;
pub mod cr3;
mod datetime;
@@ -39,7 +40,7 @@ mod xmp_merge;
pub use container::{Embedded, JpegInspection, embedded, inspect_jpeg, jpeg_segments, jpeg_segments_with_all_xmp, png_chunks, webp_chunks};
pub use datetime::{DatePrecision, DateTime};
pub use exif::{from_tiff, read_exif, strip_exif_header, try_read_exif, try_read_exif_with, write_exif};
pub use exif::{from_tiff, ifd0_orientation_compat, read_exif, strip_exif_header, try_read_exif, try_read_exif_with, write_exif};
pub use focus::FocusPoint;
pub use fuji::{FujiRecipe, FujiRow, might_hold_fuji_recipe, read_recipe_file, recipe_from_bytes, recipe_from_exif};
pub use geom::{Orientation, Point, Rect};
@@ -0,0 +1,8 @@
#[test]
fn missing_and_invalid_values_remain_distinct() {
let mut data = b"II*\0\x08\0\0\0\x01\0\x12\x01\x03\0\x01\0\0\0\x06\0\0\0\0\0\0\0".to_vec();
assert_eq!(craft_metadata::ifd0_orientation_compat(&data).map(|o| o.to_exif()), Some(6));
data[18] = 0;
assert_eq!(craft_metadata::ifd0_orientation_compat(&data).map(|o| o.to_exif()), Some(1));
assert_eq!(craft_metadata::ifd0_orientation_compat(b"invalid"), None);
}
+2 -2
View File
@@ -53,14 +53,14 @@ decoder is where the size, pixel and work limits have to be enforced.
| Group | Records |
|---|---|
| Headers | Placeable header, memory and disk metafile headers. |
| Mapping | `SETMAPMODE`, `SETWINDOWORG`, `SETWINDOWEXT`, `SETVIEWPORTORG`, `SETVIEWPORTEXT`, all map modes (text, metric, isotropic, anisotropic). A placeable file is played into its box the way Office plays it. |
| Mapping | `SETMAPMODE`, `SETWINDOWORG`, `SETWINDOWEXT`, `SETVIEWPORTORG`, `SETVIEWPORTEXT`, `OFFSETWINDOWORG`, `OFFSETVIEWPORTORG`, `SCALEWINDOWEXT`, `SCALEVIEWPORTEXT`, all map modes (text, metric, isotropic, anisotropic). A placeable file is played into its box the way Office plays it. |
| State | `SAVEDC`, `RESTOREDC`, `SETPOLYFILLMODE`, `SETROP2`, `SETBKMODE`, `SETBKCOLOR`, `SETTEXTCOLOR`, `SETTEXTALIGN`. |
| Objects | `CREATEPENINDIRECT` (style, cap and join bits), `CREATEBRUSHINDIRECT` (solid, null, hatched), `DIBCREATEPATTERNBRUSH`, `CREATEFONTINDIRECT`, `SELECTOBJECT`, `DELETEOBJECT`, stock objects. `CREATEPALETTE`, `CREATEPATTERNBRUSH` and `CREATEREGION` take their handle slot. |
| Drawing | `MOVETO`, `LINETO`, `POLYLINE`, `POLYGON`, `POLYPOLYGON`, `RECTANGLE`, `ROUNDRECT`, `ELLIPSE`, `ARC`, `PIE`, `CHORD`, `PATBLT`. |
| Bitmaps | `STRETCHDIB`, `DIBSTRETCHBLT`, `DIBBITBLT` (with or without a source). |
| Text | `EXTTEXTOUT` (rectangle, opaque, clipped, dx), `TEXTOUT`, decoded in the selected font's character set: code pages 1250 to 1258 and 874, Shift_JIS, UHC, GBK and Big5 (a double-byte character takes the advances of both its bytes), and the symbol character set (also the Symbol font under `DEFAULT_CHARSET`), whose codes stay in the private-use area at U+F000. Face names are read in the font's code page. Johab pairs are blanks. |
| Clipping | `INTERSECTCLIPRECT`, `EXCLUDECLIPRECT`. |
| Not read | `OFFSETWINDOWORG`, `OFFSETVIEWPORTORG`, `SCALEWINDOWEXT`, `SCALEVIEWPORTEXT`, flood fills, `SETPIXEL`, region records (`FILLREGION`, `FRAMEREGION`, `PAINTREGION`, `INVERTREGION`, `SELECTCLIPREGION`, `OFFSETCLIPRGN`), the device-dependent `BITBLT`, `STRETCHBLT` and `SETDIBTODEV`. They are counted in `skipped_records`. |
| Not read | Flood fills, `SETPIXEL`, region records (`FILLREGION`, `FRAMEREGION`, `PAINTREGION`, `INVERTREGION`, `SELECTCLIPREGION`, `OFFSETCLIPRGN`), the device-dependent `BITBLT`, `STRETCHBLT` and `SETDIBTODEV`. They are counted in `skipped_records`. |
### EMF (MS-EMF)
+8 -2
View File
@@ -822,8 +822,14 @@ impl Canvas {
if self.arc_clockwise { (e, s) } else { (s, e) }
}
/// SCALEVIEWPORTEXTEX and SCALEWINDOWEXTEX: the extent times `num / den` on each axis. A zero
/// denominator leaves the extent alone.
/// OFFSETWINDOWORG and OFFSETVIEWPORTORG (WMF): the window or viewport origin moved by `d`.
pub fn offset_org(&mut self, window: bool, d: P) {
let o = if window { &mut self.map.wo } else { &mut self.map.vo };
*o = (o.0 + d.0, o.1 + d.1);
}
/// SCALEVIEWPORTEXTEX and SCALEWINDOWEXTEX (and WMF's SCALEVIEWPORTEXT and SCALEWINDOWEXT): the
/// extent times `num / den` on each axis. A zero denominator leaves the extent alone.
pub fn scale_ext(&mut self, window: bool, x: (f64, f64), y: (f64, f64)) {
if x.1 == 0.0 || y.1 == 0.0 {
return;
+104
View File
@@ -2654,3 +2654,107 @@ fn canvas_budget_counts_bytes_held_elsewhere() {
assert!(cv.full());
assert!(cv.items.is_empty());
}
/// The box through a path's segment ends (an ellipse's four cubics end on its box).
fn end_extent(item: &Item) -> (f32, f32, f32, f32) {
let (segs, ..) = path_segs(item);
let (xs, ys): (Vec<f32>, Vec<f32>) = segs
.iter()
.filter_map(|s| match s {
Seg::Move(x, y) | Seg::Line(x, y) | Seg::Cubic(.., x, y) => Some((*x, *y)),
Seg::Close => None,
})
.unzip();
let lo = |v: &[f32]| v.iter().copied().fold(f32::MAX, f32::min);
let hi = |v: &[f32]| v.iter().copied().fold(f32::MIN, f32::max);
(lo(&xs), lo(&ys), hi(&xs), hi(&ys))
}
/// Two sub-pictures drawn into halves of the frame, as charts pasted from older Office versions
/// are stored (deckcraft#2069): each saves the DC, halves the viewport with SCALEVIEWPORTEXT, sets
/// its own 400 x 400 window (the second shifted 400 to the left, so it lands in the right half),
/// draws a circle and restores the DC. The left circle stays in the left half and the right one
/// lands in the right half of the 800 x 400 frame.
#[test]
fn wmf_scale_viewport_ext_draws_sub_pictures_into_parts_of_the_frame() {
let sub = |org_x: i32| {
vec![
wrec(0x001E, &[]),
wrec(0x0103, &h16(8)),
// yDenom, yNum, xDenom, xNum: x * 1/2, y * 1/1.
wrec(0x0412, &cat(vec![h16(1), h16(1), h16(2), h16(1)])),
wrec(0x020B, &cat(vec![h16(0), h16(org_x)])),
wrec(0x020C, &cat(vec![h16(400), h16(400)])),
wrec(0x0418, &cat(vec![h16(380), h16(380), h16(20), h16(20)])),
wrec(0x0127, &h16(-1)),
]
};
let mut recs = vec![wrec(0x020B, &cat(vec![h16(0), h16(0)])), wrec(0x020C, &cat(vec![h16(400), h16(800)]))];
recs.extend(sub(0));
recs.extend(sub(-400));
let p = parse(&wmf(&recs, None)).unwrap();
assert_eq!((p.width, p.height), (800.0, 400.0));
assert_eq!((p.items.len(), p.skipped_records), (2, 0), "both circles, and every record played");
for (item, want) in p.items.iter().zip([(20.0, 20.0, 380.0, 380.0), (420.0, 20.0, 780.0, 380.0)]) {
let got = end_extent(item);
assert!(close(got.0, want.0) && close(got.1, want.1) && close(got.2, want.2) && close(got.3, want.3), "{got:?}, want {want:?}");
}
}
/// SCALEWINDOWEXT scales the window extent, and OFFSETWINDOWORG and OFFSETVIEWPORTORG move the
/// window and viewport origins by their offsets, on top of what the SET records gave them. The
/// 100 x 100 window of a plain WMF is the frame, and the records after it place a rectangle in it.
#[test]
fn wmf_scale_window_ext_and_offset_origins_change_the_mapping() {
let rect = || wrec(0x041B, &cat(vec![h16(20), h16(20), h16(10), h16(10)]));
let base = || vec![wrec(0x0103, &h16(8)), wrec(0x020B, &cat(vec![h16(0), h16(0)])), wrec(0x020C, &cat(vec![h16(100), h16(100)]))];
let extent = |extra: Vec<Vec<u8>>| {
let mut recs = base();
recs.extend(extra);
recs.push(rect());
rect_extent(&parse(&wmf(&recs, None)).unwrap().items[0])
};
// The window doubled (x * 2/1, y * 4/2): the rectangle shows at half size.
let got = extent(vec![wrec(0x0410, &cat(vec![h16(2), h16(4), h16(1), h16(2)]))]);
assert!(close(got.0, 5.0) && close(got.1, 5.0) && close(got.2, 10.0) && close(got.3, 10.0), "scaled window: {got:?}");
// The window origin moved by (5, 10): the rectangle moves the other way.
let got = extent(vec![wrec(0x020F, &cat(vec![h16(10), h16(5)]))]);
assert!(close(got.0, 5.0) && close(got.1, 0.0) && close(got.2, 15.0) && close(got.3, 10.0), "window origin: {got:?}");
// The viewport origin moved by (30, -5): the rectangle moves with it.
let got = extent(vec![wrec(0x0211, &cat(vec![h16(-5), h16(30)]))]);
assert!(close(got.0, 40.0) && close(got.1, 5.0) && close(got.2, 50.0) && close(got.3, 15.0), "viewport origin: {got:?}");
// A zero denominator leaves the extent alone.
let got = extent(vec![wrec(0x0412, &cat(vec![h16(0), h16(1), h16(0), h16(1)]))]);
assert!(close(got.0, 10.0) && close(got.1, 10.0) && close(got.2, 20.0) && close(got.3, 20.0), "zero denominator: {got:?}");
}
#[test]
fn wmf_viewport_scaling_does_not_resize_the_player_frame() {
for numerator in [1, 4] {
let recs = [
wrec(0x0103, &h16(8)),
wrec(0x020C, &cat(vec![h16(100), h16(100)])),
wrec(0x041B, &cat(vec![h16(100), h16(100), h16(0), h16(0)])),
// Change the mapping after drawing, without a matching RESTOREDC.
wrec(0x0412, &cat(vec![h16(1), h16(1), h16(2), h16(numerator)])),
];
let p = parse(&wmf(&recs, None)).unwrap();
assert_eq!((p.width, p.height), (100.0, 100.0));
assert_eq!(p.skipped_records, 0);
assert_eq!(rect_extent(&p.items[0]), (0.0, 0.0, 100.0, 100.0));
}
}
#[test]
fn wmf_viewport_scaling_before_first_window_is_retained() {
let recs = [
wrec(0x0103, &h16(8)),
wrec(0x0412, &cat(vec![h16(1), h16(1), h16(2), h16(1)])),
wrec(0x020C, &cat(vec![h16(100), h16(100)])),
wrec(0x041B, &cat(vec![h16(100), h16(100), h16(0), h16(0)])),
];
let p = parse(&wmf(&recs, None)).unwrap();
assert_eq!((p.width, p.height), (100.0, 100.0));
assert_eq!(p.skipped_records, 0);
assert_eq!(rect_extent(&p.items[0]), (0.0, 0.0, 50.0, 100.0));
}
+20 -7
View File
@@ -50,7 +50,8 @@ pub(crate) fn parse(input: Bytes) -> Result<(Canvas, Option<Frame>), Error> {
cv.set_clip_surface([0.0, 0.0, size.0, size.1]);
cv.pattern_px = PT_PER_PX;
}
let mut player_viewport = bounds.is_some();
// Keep the player target independent of subsequent mapping changes.
let mut player_target = bounds.map(|(_, size)| size);
let mut parsed = 0usize;
while !cv.full() {
let (Some(words), Some(func)) = (input.u32(pos), pos.checked_add(4).and_then(|o| input.u16(o))) else { break };
@@ -60,10 +61,14 @@ pub(crate) fn parse(input: Bytes) -> Result<(Canvas, Option<Frame>), Error> {
break;
}
record(&mut cv, func, Bytes(rec));
if func == 0x020C && cv.window_set && !player_viewport && !cv.viewport_set {
if func == 0x020C && cv.window_set && player_target.is_none() && !cv.viewport_set {
let we = cv.map.window_ext();
cv.play_into(None, (we.0.abs(), we.1.abs()));
player_viewport = true;
let target = (we.0.abs(), we.1.abs());
// Before an explicit viewport, its default (1, 1) retains any
// SCALEVIEWPORTEXT factors already applied by the file.
let scale = cv.map.viewport_ext();
cv.play_into(None, (target.0 * scale.0, target.1 * scale.1));
player_target = Some(target);
}
parsed += 1;
let Some(next) = pos.checked_add(len) else { break };
@@ -82,9 +87,8 @@ pub(crate) fn parse(input: Bytes) -> Result<(Canvas, Option<Frame>), Error> {
let rect = [a.0.min(b.0), a.1.min(b.1), (b.0 - a.0).abs(), (b.1 - a.1).abs()];
(rect[2] > 0.0 && rect[3] > 0.0).then_some(Frame { rect, size: Some(size) })
}
None if player_viewport && !cv.viewport_set && cv.map.uses_extents() => {
let ve = cv.map.viewport_ext();
Some(Frame { rect: [0.0, 0.0, ve.0, ve.1], size: None })
None if !cv.viewport_set && cv.map.uses_extents() && player_target.is_some() => {
player_target.map(|(w, h)| Frame { rect: [0.0, 0.0, w, h], size: None })
}
None if cv.window_set => Some(Frame { rect: cv.map.window_rect(), size: None }),
None => None,
@@ -105,6 +109,15 @@ fn record(cv: &mut Canvas, func: u16, r: Bytes) {
0x020C => xy16(r, 6).map(|p| cv.set_window_ext(p)),
0x020D => xy16(r, 6).map(|p| cv.set_viewport_org(p)),
0x020E => xy16(r, 6).map(|p| cv.set_viewport_ext(p)),
// OFFSETWINDOWORG and OFFSETVIEWPORTORG: yOffset, xOffset, added to the origin.
0x020F | 0x0211 => xy16(r, 6).map(|d| cv.offset_org(func == 0x020F, d)),
// SCALEWINDOWEXT and SCALEVIEWPORTEXT: yDenom, yNum, xDenom, xNum ([MS-WMF] 2.3.5). A
// sub-picture drawn into part of the frame halves the viewport this way (deckcraft#2069).
0x0410 | 0x0412 => (|| {
let [y_den, y_num, x_den, x_num] = [r.i16(6)?, r.i16(8)?, r.i16(10)?, r.i16(12)?].map(f64::from);
cv.scale_ext(func == 0x0410, (x_num, x_den), (y_num, y_den));
Some(())
})(),
0x001E => {
cv.save_dc();
Some(())
+4 -1
View File
@@ -15,7 +15,10 @@ mod smf;
pub mod sync;
pub use sequence::{CtrlEvent, CtrlKind, Note, Sequence};
pub use smf::{LenientRead, MAX_EVENTS, MAX_TRACKS, MAX_WARNINGS, Smf, SmfTrack, read_smf, read_smf_lenient, write_smf};
pub use smf::{
LenientRead, MAX_EVENTS, MAX_TRACKS, MAX_WARNINGS, Smf, SmfTrack, TimedEvent, encode_meta_event, read_smf, read_smf_lenient, write_smf,
write_timed_smf,
};
/// Ticks per quarter note used throughout the model.
pub const PPQ: u32 = 960;
+98 -7
View File
@@ -598,10 +598,68 @@ fn push_vlq(out: &mut Vec<u8>, v: u64) -> Result<(), MidiError> {
}
/// An event ready to be serialised: `(tick, order, bytes)`. `order` breaks ties at one tick.
struct WEvent {
tick: u64,
order: u8,
bytes: Vec<u8>,
/// An already encoded SMF event at an absolute source tick.
///
/// `bytes` must contain one complete channel, meta, or SysEx event, including its
/// status and any length, but no delta time. The caller owns payload validity.
/// Do not provide running-status fragments or End-of-Track (added by the writer).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TimedEvent {
pub tick: u64,
pub order: u8,
pub bytes: Vec<u8>,
}
type WEvent = TimedEvent;
/// Encode an SMF meta event, without its delta time.
pub fn encode_meta_event(kind: u8, data: &[u8]) -> Result<Vec<u8>, MidiError> {
meta(kind, data)
}
/// Serialize source-timed events without normalizing ticks or trimming notes.
///
/// Tracks and same-tick events retain their input order (with `order` as the tie
/// breaker). Format must be 0 (exactly one track) or 1. Division is PPQ 1..=32767.
/// Channel running-status compression is disabled. `clamp_deltas` is an explicit
/// compatibility option: gaps above the SMF 28-bit maximum are saturated rather
/// than rejected. It can shorten represented time and should normally be false.
///
/// `max_bytes` bounds the conservative serialized size (four bytes per delta),
/// not caller-owned events or sorting memory. Encoded payloads are trusted; this
/// is a framing primitive, not validation of arbitrary untrusted event bytes.
pub fn write_timed_smf(format: u16, division: u16, tracks: Vec<Vec<TimedEvent>>, clamp_deltas: bool, max_bytes: usize) -> Result<Vec<u8>, MidiError> {
if format > 1 || (format == 0 && tracks.len() != 1) || division == 0 || division > 0x7fff {
return Err(MidiError::Write("invalid format, track count, or PPQ division".into()));
}
let count = u16::try_from(tracks.len()).map_err(|_| MidiError::Write("more than 65535 tracks".into()))?;
let mut bound = 14usize;
for track in &tracks {
bound = bound.checked_add(12).ok_or_else(|| MidiError::Write("output size overflow".into()))?;
let mut body = 4usize;
for event in track {
let size = event.bytes.len().checked_add(4).ok_or_else(|| MidiError::Write("event size overflow".into()))?;
body = body.checked_add(size).ok_or_else(|| MidiError::Write("track size overflow".into()))?;
bound = bound.checked_add(size).ok_or_else(|| MidiError::Write("output size overflow".into()))?;
}
if body > u32::MAX as usize {
return Err(MidiError::Write("track larger than 4 GiB".into()));
}
}
if bound > max_bytes {
return Err(MidiError::Write("serialized output exceeds byte budget".into()));
}
let mut out = Vec::new();
out.try_reserve(bound).map_err(|_| MidiError::Write("output allocation failed".into()))?;
out.extend_from_slice(b"MThd");
out.extend_from_slice(&6u32.to_be_bytes());
out.extend_from_slice(&format.to_be_bytes());
out.extend_from_slice(&count.to_be_bytes());
out.extend_from_slice(&division.to_be_bytes());
for track in tracks {
write_track_policy(&mut out, track, clamp_deltas, false)?;
}
Ok(out)
}
fn clamp_tick(t: i64) -> u64 {
@@ -615,17 +673,22 @@ fn meta(kind: u8, data: &[u8]) -> Result<Vec<u8>, MidiError> {
Ok(b)
}
fn write_track(out: &mut Vec<u8>, mut events: Vec<WEvent>) -> Result<(), MidiError> {
fn write_track(out: &mut Vec<u8>, events: Vec<WEvent>) -> Result<(), MidiError> {
write_track_policy(out, events, false, true)
}
fn write_track_policy(out: &mut Vec<u8>, mut events: Vec<WEvent>, clamp_deltas: bool, running_status: bool) -> Result<(), MidiError> {
events.sort_by_key(|e| (e.tick, e.order));
let mut body = Vec::new();
let mut last_tick = 0u64;
let mut running: Option<u8> = None;
for e in &events {
push_vlq(&mut body, e.tick - last_tick)?;
let delta = e.tick - last_tick;
push_vlq(&mut body, if clamp_deltas { delta.min(MAX_VLQ) } else { delta })?;
last_tick = e.tick;
let status = e.bytes.first().copied().unwrap_or(0);
if (0x80..=0xEF).contains(&status) {
if running == Some(status) {
if running_status && running == Some(status) {
body.extend_from_slice(e.bytes.get(1..).unwrap_or(&[]));
} else {
body.extend_from_slice(&e.bytes);
@@ -847,4 +910,32 @@ mod tests {
// of two notes starting together only the longer survives.
assert_eq!(got, vec![(0, 100), (100, 200), (1000, 100), (2000, 900)]);
}
#[test]
fn timed_writer_preserves_source_ticks_payloads_and_stable_order() {
let events = vec![
TimedEvent { tick: 96, order: 3, bytes: vec![0x90, 60, 100] },
TimedEvent { tick: 96, order: 1, bytes: vec![0x80, 60, 0] },
TimedEvent { tick: 96, order: 3, bytes: vec![0x90, 60, 110] },
];
let out = write_timed_smf(0, 96, vec![events], false, 53).unwrap();
assert_eq!(&out[..14], b"MThd\0\0\0\x06\0\0\0\x01\0\x60");
assert_eq!(&out[22..], &[96, 0x80, 60, 0, 0, 0x90, 60, 100, 0, 0x90, 60, 110, 0, 0xff, 0x2f, 0]);
}
#[test]
fn timed_writer_checks_budget_format_division_and_delta_policy() {
let event = TimedEvent { tick: MAX_VLQ + 1, order: 0, bytes: vec![0xc0, 1] };
assert!(write_timed_smf(0, 480, vec![vec![event.clone()]], false, 32).is_err());
let out = write_timed_smf(0, 480, vec![vec![event.clone()]], true, 32).unwrap();
assert_eq!(&out[22..26], &[0xff, 0xff, 0xff, 0x7f]);
assert!(write_timed_smf(0, 480, vec![vec![event]], true, 31).is_err());
assert!(write_timed_smf(0, 96, vec![], false, 32).is_err());
assert!(write_timed_smf(2, 96, vec![], false, 32).is_err());
for division in [0, 0x8000, u16::MAX] {
assert!(write_timed_smf(1, division, vec![], false, 32).is_err());
}
assert_eq!(write_timed_smf(1, 96, vec![], false, 14).unwrap().len(), 14);
assert!(write_timed_smf(1, 96, vec![], false, 13).is_err());
assert_eq!(encode_meta_event(4, b"Piano").unwrap(), b"\xff\x04\x05Piano");
}
}
+16
View File
@@ -369,6 +369,22 @@ fn layer_selection_ids(data: &[u8]) -> Result<Vec<u32>> {
(0..count).map(|_| r.u32()).collect()
}
/// Read the unique EXIF and XMP image resources, rejecting duplicates and invalid XMP UTF-8.
/// EXIF bytes retain any optional container prefix; trailing XMP NULs are removed.
pub fn unique_exif_xmp(resources: &[ImageResource]) -> Option<(Option<&[u8]>, Option<&str>)> {
let (mut exif, mut xmp) = (None, None);
for resource in resources {
match resource.id {
ids::EXIF if exif.is_some() => return None,
ids::EXIF => exif = Some(resource.data.as_slice()),
ids::XMP if xmp.is_some() => return None,
ids::XMP => xmp = Some(std::str::from_utf8(&resource.data).ok()?.trim_end_matches('\0')),
_ => {}
}
}
Some((exif, xmp))
}
impl ImageResource {
/// A new `8BIM` resource with an empty name.
pub fn new(id: u16, data: Vec<u8>) -> Self {
+14
View File
@@ -0,0 +1,14 @@
use craft_psd::ImageResource;
use craft_psd::resources::{ids, unique_exif_xmp};
#[test]
fn metadata_resources_reject_duplicates_and_invalid_xmp() {
let exif = ImageResource::new(ids::EXIF, b"Exif\0\0II*\0".to_vec());
let xmp = ImageResource::new(ids::XMP, b"<x/>\0".to_vec());
let resources = [exif.clone(), xmp.clone()];
assert_eq!(unique_exif_xmp(&resources), Some((Some(b"Exif\0\0II*\0".as_slice()), Some("<x/>"))));
assert!(unique_exif_xmp(&[exif.clone(), exif]).is_none());
assert!(unique_exif_xmp(&[xmp.clone(), xmp]).is_none());
assert!(unique_exif_xmp(&[ImageResource::new(ids::XMP, vec![255])]).is_none());
assert_eq!(unique_exif_xmp(&[]), Some((None, None)));
}
Loaded 100 of 140 files, more files were not shown because too many files have changed in this diff. Show more