Merge remote-tracking branch 'origin/main' into HEAD

This commit is contained in:
Dylan M. Taylor committed 2026-11-15 10:12:11 +00:00
commit 2369c13f28
92 files changed
+11204 -276

No files matched your search

+1 -1
View File
@@ -158,7 +158,7 @@ dependencies:
repo_id: 192625525
'dtolnay/rust-toolchain@stable':
ref: 'stable'
commit: 'sha1-b725eda525362e7cf34af9a7243cac1e3f01a7ec'
commit: 'sha1-39008827e6b4319a349e9a42dfc90579dd25f1ca'
owner_id: 1940490
repo_id: 260749683
'step-security/dtolnay-rust-toolchain@v1.0.0':
+9
View File
@@ -10,6 +10,14 @@ on:
permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (push to main, a release tag, a manual run) gets a group of its own: it is neither
# cancelled nor replaced while queued, so a main run always finishes and saves the dependency
# cache, and a release run is never cancelled.
group: workspace-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
native:
name: Workspace
@@ -39,6 +47,7 @@ jobs:
printf 'CRAFT_PSD_CORPUS=%s\n' "$GITHUB_WORKSPACE/$corpus_fetcher/corpus" >> "$GITHUB_ENV"
printf 'CRAFT_PDF_TEST_FONT_DIR=%s\n' "$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV"
command: |
python3 scripts/test-workspace-concurrency.py
cargo fmt --check
cargo clippy --workspace --all-targets --locked -- -D warnings
cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
Generated
+17 -1
View File
@@ -1352,6 +1352,7 @@ version = "0.1.0"
dependencies = [
"craft-media",
"craft-media-testkit",
"craft-xml",
"log",
"proptest",
"rayon",
@@ -1362,6 +1363,10 @@ dependencies = [
"vorbis_rs",
]
[[package]]
name = "craft-ber"
version = "0.1.0"
[[package]]
name = "craft-captions"
version = "0.1.0"
@@ -1875,6 +1880,7 @@ dependencies = [
"ravif",
"rayon",
"serde_json",
"sha2 0.10.9",
"thiserror 2.0.21",
"tiff",
"zune-core",
@@ -2380,6 +2386,16 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "craft-proto"
version = "0.1.0"
dependencies = [
"base64 0.23.1",
"log",
"proptest",
"zeroize",
]
[[package]]
name = "craft-ps"
version = "0.1.0"
@@ -11150,7 +11166,7 @@ dependencies = [
[[package]]
name = "winprint"
version = "0.2.1"
source = "git+https://github.com/storytold/winprint.rs?rev=72951d7455ebcab569fbd9386c513d3d9f53fb02#72951d7455ebcab569fbd9386c513d3d9f53fb02"
source = "git+https://github.com/storytold/winprint.rs?rev=8ae097ef941b432b0e7671b1ec3a506800e65e57#8ae097ef941b432b0e7671b1ec3a506800e65e57"
dependencies = [
"flate2",
"fmt-derive",
+2
View File
@@ -11,6 +11,8 @@ rust-version = "1.95"
repository = "https://github.com/storytold/craft-libs"
[workspace.dependencies]
craft-ber = { path = "crates/ber" }
craft-proto = { path = "crates/proto" }
acadrust = "=0.6.6"
log = "0.4"
# internal: add each crate here as `craft-<name> = { path = "crates/<name>" }`
+2 -1
View File
@@ -19,7 +19,7 @@ workspace = true
# µ-law and free-format MP3 through symphonia.
decode = ["decode-native", "symphonia"]
# `decode` without symphonia: only the formats with a native decoder (others are Unsupported).
decode-native = ["aac-decode", "alac-decode", "opus-decode", "ac3-decode", "mpa-decode", "mp3-decode", "craft-media/isobmff", "craft-media/matroska", "craft-media/ogg", "dep:log"]
decode-native = ["aac-decode", "alac-decode", "opus-decode", "ac3-decode", "mpa-decode", "mp3-decode", "craft-media/isobmff", "craft-media/matroska", "craft-media/ogg", "dep:log", "dep:craft-xml"]
# The symphonia fallback for the formats with no native decoder yet.
symphonia = ["decode-native", "dep:symphonia"]
# Every encoder: the lossless file writers and the AAC, ALAC, AC-3/E-AC-3, MP3 and Opus encoders.
@@ -60,6 +60,7 @@ serde = ["dep:serde", "craft-media/serde"]
[dependencies]
craft-media = { workspace = true }
craft-xml = { workspace = true, optional = true }
thiserror = { workspace = true }
serde = { workspace = true, optional = true }
log = { version = "0.4", optional = true }
+5
View File
@@ -761,3 +761,8 @@ The codec modules' public items keep their names, so apps can re-export them
CAF.
MP4 and Matroska files decode to presentation length. A packet that fails is concealed, not
dropped. A failed source read is `AudioError::Io`.
WAV timestamps use `bext` when present, including a zero timestamp. Otherwise a bounded iXML
`BWFXML/BEXT` time reference supplies the timestamp (two unsigned 32-bit words; an omitted high
word is zero). Only the first iXML chunk is considered. Invalid or oversized optional XML is
ignored without rejecting the audio. This works for byte-backed and streaming sources.
+62 -2
View File
@@ -115,6 +115,50 @@ fn parse_bext(c: &[u8]) -> Option<BwfInfo> {
})
}
/// Cap optional metadata independently of audio length, including on range-backed sources.
const IXML_MAX: usize = 1 << 20;
/// iXML's BEXT mirror is a fallback only: a real `bext` always wins, including timestamp zero.
/// Read the two unsigned 32-bit words, not arbitrary text that merely contains their tag names.
fn ixml_time_reference(bytes: &[u8]) -> Option<u64> {
let options = craft_xml::Options {
doctype: craft_xml::Doctype::Reject,
..craft_xml::Options::default().strict().with_limits(craft_xml::Limits {
max_input: IXML_MAX,
max_depth: 32,
max_nodes: 8192,
max_attrs: 64,
max_attrs_total: 8192,
..craft_xml::Limits::default()
})
};
let root = options.parse(bytes).ok()?;
if root.name != "BWFXML" {
return None;
}
let mut bexts = root.children("BEXT");
let bext = bexts.next()?;
if bexts.next().is_some() {
return None;
}
let word = |name: &str| -> Option<Option<u32>> {
let mut fields = bext.children(name);
let Some(field) = fields.next() else { return Some(None) };
if fields.next().is_some() || field.children.iter().any(|n| matches!(n, craft_xml::Node::El(_))) {
return None;
}
let text = field.text();
let text = text.trim();
if text.is_empty() || !text.bytes().all(|b| b.is_ascii_digit()) {
return None;
}
Some(Some(text.parse().ok()?))
};
let low = word("BWF_TIME_REFERENCE_LOW")??;
let high = word("BWF_TIME_REFERENCE_HIGH")?.unwrap_or(0);
Some((u64::from(high) << 32) | u64::from(low))
}
/// Refuse the WAVE file at `at`, which symphonia reads (A-law, ADPCM, ...), if an INFO entry
/// claims more bytes than the file holds: symphonia allocates each entry's length. Chunks are
/// walked where symphonia reads them: after the bytes a chunk's parser used, not its length.
@@ -239,7 +283,7 @@ fn pcm_layout(fmt: &Fmt) -> Result<Option<(PcmKind, SampleFormat)>> {
}
/// Parse a WAV file's chunks: its format, `bext` metadata and where its samples are. Only chunk
/// headers and the small `fmt `, `bext` and `ds64` bodies are read.
/// headers, small `fmt `, `bext` and `ds64` bodies, and at most one 1 MiB iXML document are read.
pub(crate) fn parse(src: &Src<'_>) -> Result<WavParse> {
if !is_wav(&src.bytes_upto(0, 12)) {
return Err(AudioError::Malformed("not a RIFF/RF64 WAVE file".into()));
@@ -249,6 +293,8 @@ pub(crate) fn parse(src: &Src<'_>) -> Result<WavParse> {
let mut ds64_data: Option<u64> = None;
let mut fmt: Option<Fmt> = None;
let mut bwf: Option<BwfInfo> = None;
let mut ixml_time = None;
let mut ixml_seen = false;
let mut data: Option<(u64, u64)> = None;
let mut pos: u64 = 12;
@@ -288,7 +334,18 @@ pub(crate) fn parse(src: &Src<'_>) -> Result<WavParse> {
let small = |n: u64| src.bytes(body, usize::try_from(declared.min(n)).unwrap_or(0)).unwrap_or_default();
match &id {
b"fmt " => fmt = Some(parse_fmt(&small(256))?),
b"bext" => bwf = parse_bext(&small(4096)),
b"bext" => {
if let Some(info) = parse_bext(&small(4096)) {
bwf = Some(info);
}
}
b"iXML" if !ixml_seen => {
// Inspect at most one bounded document, not 100,000 individually bounded trees.
ixml_seen = true;
if declared <= IXML_MAX as u64 {
ixml_time = ixml_time_reference(&small(IXML_MAX as u64));
}
}
b"ds64" => ds64_data = le_u64(&small(64), 8),
_ => {}
}
@@ -297,6 +354,9 @@ pub(crate) fn parse(src: &Src<'_>) -> Result<WavParse> {
pos = next;
}
if bwf.is_none() {
bwf = ixml_time.map(|time_reference| BwfInfo { time_reference, ..BwfInfo::default() });
}
let fmt = fmt.ok_or_else(|| AudioError::Malformed("WAV has no fmt chunk".into()))?;
if fmt.channels == 0 {
return Err(AudioError::Malformed("WAV declares zero channels".into()));
+1
View File
@@ -10,3 +10,4 @@ mod raw_pcm;
mod roundtrip;
mod stream;
mod wav_empty_data;
mod wav_ixml;
+87
View File
@@ -0,0 +1,87 @@
//! iXML timestamp parity with FilmCraft's streaming WAV reader, including hostile metadata.
use std::io::Cursor;
use craft_audio::{DecodeOptions, Decoder};
use craft_media::ReadSeekSource;
fn chunk(id: &[u8; 4], body: &[u8]) -> Vec<u8> {
let mut out = id.to_vec();
out.extend_from_slice(&(body.len() as u32).to_le_bytes());
out.extend_from_slice(body);
if !body.len().is_multiple_of(2) {
out.push(0);
}
out
}
fn wav(metadata: &[Vec<u8>]) -> Vec<u8> {
let mut out = b"RIFF\0\0\0\0WAVE".to_vec();
out.extend(chunk(b"fmt ", &[1, 0, 1, 0, 0x80, 0xbb, 0, 0, 0, 0x77, 1, 0, 2, 0, 16, 0]));
for m in metadata {
out.extend(m);
}
out.extend(chunk(b"data", &[0, 0, 1, 0]));
let len = (out.len() - 8) as u32;
out[4..8].copy_from_slice(&len.to_le_bytes());
out
}
fn xml(low: &str, high: Option<&str>) -> Vec<u8> {
let high = high.map(|v| format!("<BWF_TIME_REFERENCE_HIGH>{v}</BWF_TIME_REFERENCE_HIGH>")).unwrap_or_default();
chunk(b"iXML", format!("<BWFXML><BEXT><BWF_TIME_REFERENCE_LOW>{low}</BWF_TIME_REFERENCE_LOW>{high}</BEXT></BWFXML>").as_bytes())
}
fn check(file: Vec<u8>, expected: Option<u64>) {
let options = DecodeOptions::default();
let mut bytes = Decoder::from_bytes(&file, Some("wav"), &options).unwrap();
assert_eq!(bytes.info().bwf.as_ref().map(|b| b.time_reference), expected);
let source = ReadSeekSource::new(Cursor::new(file.clone())).unwrap();
let mut stream = Decoder::open(source, Some("wav"), &options).unwrap();
assert_eq!(stream.info().bwf.as_ref().map(|b| b.time_reference), expected);
assert_eq!(bytes.info().frames, 2);
assert_eq!(stream.next_block().unwrap().unwrap().frames(), 2);
assert_eq!(bytes.next_block().unwrap().unwrap().frames(), 2);
}
#[test]
fn ixml_fallback_preserves_large_timestamps_and_optional_high_word() {
check(wav(&[xml(" 5 ", Some("1"))]), Some((1 << 32) + 5));
check(wav(&[xml("0", None)]), Some(0));
check(wav(&[xml("4294967295", Some("4294967295"))]), Some(u64::MAX));
}
#[test]
fn bext_wins_in_either_chunk_order_including_timestamp_zero() {
let bext = chunk(b"bext", &[0; 602]);
let ixml = xml("5", Some("1"));
check(wav(&[bext.clone(), ixml.clone()]), Some(0));
check(wav(&[ixml.clone(), bext.clone()]), Some(0));
check(wav(&[bext, chunk(b"bext", &[0; 12]), ixml.clone()]), Some(0));
check(wav(&[chunk(b"bext", &[0; 12]), ixml]), Some((1 << 32) + 5));
}
#[test]
fn invalid_metadata_never_fabricates_a_timestamp_or_blocks_audio() {
for low in ["4294967296", "-1", "+1", "", "1.5", "<x>5</x>"] {
check(wav(&[xml(low, Some("0"))]), None);
}
for high in ["4294967296", "-1", "", "oops"] {
check(wav(&[xml("5", Some(high))]), None);
}
for text in [
"<BWFXML><BEXT><BWF_TIME_REFERENCE_LOW>5</BWF_TIME_REFERENCE_LOW>",
"<BWFXML><!-- <BWF_TIME_REFERENCE_LOW>5</BWF_TIME_REFERENCE_LOW> --></BWFXML>",
"<BWFXML><BEXT><BWF_TIME_REFERENCE_LOW>5</BWF_TIME_REFERENCE_LOW><BWF_TIME_REFERENCE_LOW>6</BWF_TIME_REFERENCE_LOW></BEXT></BWFXML>",
"<!DOCTYPE BWFXML><BWFXML><BEXT><BWF_TIME_REFERENCE_LOW>5</BWF_TIME_REFERENCE_LOW></BEXT></BWFXML>",
] {
check(wav(&[chunk(b"iXML", text.as_bytes())]), None);
}
check(wav(&[chunk(b"iXML", &vec![b' '; (1 << 20) + 1])]), None);
let deep = format!("<BWFXML>{}{}</BWFXML>", "<a>".repeat(40), "</a>".repeat(40));
check(wav(&[chunk(b"iXML", deep.as_bytes())]), None);
}
#[test]
fn only_first_ixml_document_is_considered() {
check(wav(&[xml("5", None), xml("6", None)]), Some(5));
}
+11
View File
@@ -0,0 +1,11 @@
[package]
name = "craft-ber"
description = "ASN.1 BER and DER (X.690): reading TLVs, BER to DER, and what an opaque signed CMS holds"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
+432
View File
@@ -0,0 +1,432 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
//! Reading the TLVs of ASN.1 BER and DER (X.690), and turning BER into DER where they differ in
//! what CMS writers use: indefinite lengths (which streaming signers write) and OCTET STRINGs in
//! pieces (constructed). What is DER already is kept byte for byte, an element with a definite
//! length inside an indefinite one too, so that the bytes a signature covers (signed attributes,
//! a certificate's body) are those the signer wrote. For S/MIME: `craft-smime` checks
//! signatures with it, and a message's reader takes out what an opaque signed one holds
//! ([`signed_content`]).
//!
//! The input is hostile: lengths are checked against what is there, nesting is bounded
//! ([`MAX_DEPTH`]), and the work is linear in the input (times the bounded depth).
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
/// CMS's id-signedData and id-data (RFC 5652 §5.1, §4), as OBJECT IDENTIFIER values.
const SIGNED_DATA: &[u8] = &[0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x07, 0x02];
const DATA: &[u8] = &[0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x07, 0x01];
/// What the CMS `cms` holds when it is SignedData with its content inside, of type id-data: the
/// MIME entity an opaque signed message (`application/pkcs7-mime`, RFC 8551 §3.5) signed, for
/// reading as the message. Nothing is checked here (`craft-smime` checks it); `None` for any
/// other CMS (an Encrypted message's enveloped data) or bytes that aren't one.
pub fn signed_content(cms: &[u8]) -> Option<Vec<u8>> {
let der = to_der(cms)?;
let (info, rest) = read(&der)?;
if info.tag != 0x30 || !rest.is_empty() {
return None;
}
let info = children(info.value)?;
let [oid, explicit] = info.as_slice() else { return None };
if oid.tag != 0x06 || oid.value != SIGNED_DATA || explicit.tag != 0xa0 {
return None;
}
let (signed, rest) = read(explicit.value)?;
if signed.tag != 0x30 || !rest.is_empty() {
return None;
}
let signed = children(signed.value)?;
let [version, digests, encap, tail @ ..] = signed.as_slice() else { return None };
if version.tag != 0x02 || !matches!(version.value, [1 | 3 | 4 | 5]) || digests.tag != 0x31 || encap.tag != 0x30 {
return None;
}
if children(digests.value)?.iter().any(|algorithm| algorithm.tag != 0x30) {
return None;
}
// Certificates and revocation lists are optional, but ordered and present at most once.
let tail = if tail.first().is_some_and(|field| field.tag == 0xa0) { tail.get(1..)? } else { tail };
let tail = if tail.first().is_some_and(|field| field.tag == 0xa1) { tail.get(1..)? } else { tail };
let [signers] = tail else { return None };
if signers.tag != 0x31 || children(signers.value)?.iter().any(|signer| signer.tag != 0x30) {
return None;
}
let encap = children(encap.value)?;
let [kind, content] = encap.as_slice() else { return None };
if kind.tag != 0x06 || kind.value != DATA || content.tag != 0xa0 {
return None;
}
let (octets, rest) = read(content.value)?;
(octets.tag == 0x04 && rest.is_empty()).then(|| octets.value.to_vec())
}
/// How deep elements may nest.
pub const MAX_DEPTH: usize = 48;
/// Maximum elements materialized or indexed in one parsing operation. This bounds the
/// allocation amplification from hostile, shallow input containing many empty TLVs.
pub const MAX_ELEMENTS: usize = 65_536;
/// The universal tag of an OCTET STRING, and of one in pieces.
const OCTET_STRING: u8 = 0x04;
const OCTET_STRING_PIECES: u8 = 0x24;
/// One element: its tag bytes, and its value (the content octets).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Tlv<'a> {
/// The tag's first byte (class, constructed bit, number below 31).
pub tag: u8,
/// The whole element, tag and length included.
pub whole: &'a [u8],
/// Its content octets.
pub value: &'a [u8],
}
/// The tag bytes at the start of `bytes`: one, or a high tag number in up to four more.
fn tag_len(bytes: &[u8]) -> Option<usize> {
let first = *bytes.first()?;
if first & 0x1f != 0x1f {
return Some(1);
}
// High tag number: base-128 bytes, the last without its top bit.
for (i, b) in bytes.iter().enumerate().skip(1).take(4) {
if b & 0x80 == 0 {
return i.checked_add(1);
}
}
None
}
/// The length at the start of `bytes`: `(length, bytes it took)`, `None` for an indefinite one
/// (`Some((None, 1))`) is told apart by the caller.
fn length(bytes: &[u8]) -> Option<(Option<usize>, usize)> {
let first = *bytes.first()?;
if first < 0x80 {
return Some((Some(usize::from(first)), 1));
}
if first == 0x80 {
return Some((None, 1));
}
let n = usize::from(first & 0x7f);
if n > 4 {
return None;
}
let digits = bytes.get(1..n.checked_add(1)?)?;
let len = digits.iter().fold(0usize, |acc, d| (acc << 8) | usize::from(*d));
Some((Some(len), n.checked_add(1)?))
}
/// The element at the start of `bytes` with a definite length, and what follows it. `None` for
/// an indefinite length, or one longer than what is there.
pub fn read(bytes: &[u8]) -> Option<(Tlv<'_>, &[u8])> {
let t = tag_len(bytes)?;
let tag = *bytes.first()?;
let (len, l) = length(bytes.get(t..)?)?;
let len = len?;
let start = t.checked_add(l)?;
let end = start.checked_add(len)?;
let value = bytes.get(start..end)?;
let whole = bytes.get(..end)?;
Some((Tlv { tag, whole, value }, bytes.get(end..)?))
}
/// The elements of `value`, each with a definite length; `None` when one can't be read.
pub fn children(mut value: &[u8]) -> Option<Vec<Tlv<'_>>> {
let mut out = Vec::new();
while !value.is_empty() {
if out.len() >= MAX_ELEMENTS {
return None;
}
let (tlv, rest) = read(value)?;
out.push(tlv);
value = rest;
}
Some(out)
}
/// `bytes` as DER as far as CMS needs it: indefinite lengths made definite and OCTET STRINGs in
/// pieces joined into one, everything else as it was. One element is read, and must be all of
/// `bytes` (bar trailing zero bytes, which some writers pad with). `None` when it isn't BER.
pub fn to_der(bytes: &[u8]) -> Option<Vec<u8>> {
if let Some((_, rest)) = read(bytes)
&& rest.iter().all(|b| *b == 0)
&& der_shaped(bytes.get(..bytes.len().saturating_sub(rest.len()))?, 0)
{
return bytes.get(..bytes.len().saturating_sub(rest.len())).map(<[u8]>::to_vec);
}
let mut out = Vec::with_capacity(bytes.len());
let used = element(bytes, 0, &mut out)?;
let rest = bytes.get(used..)?;
rest.iter().all(|b| *b == 0).then_some(out)
}
/// Whether the elements of `bytes` all have definite lengths and no OCTET STRING is in pieces,
/// as deep as they go: nothing to change.
fn der_shaped(mut bytes: &[u8], depth: usize) -> bool {
if depth > MAX_DEPTH {
return false;
}
while !bytes.is_empty() {
let Some((tlv, rest)) = read(bytes) else { return false };
if tlv.tag == OCTET_STRING_PIECES || (tlv.tag & 0x20 != 0 && !der_shaped(tlv.value, depth.saturating_add(1))) {
return false;
}
bytes = rest;
}
true
}
/// What [`measure`] found of one element: how many bytes of the input it was, and how long its
/// content is as DER.
#[derive(Clone, Copy)]
struct Size {
used: usize,
content: usize,
/// It is DER-shaped already (a definite length, and nothing in it to change): copied as it
/// is, its lengths as the writer wrote them, since a signature may cover them.
verbatim: bool,
}
/// The sizes of the constructed elements of the input, by where each starts in it: worked out
/// once, so that writing them is one pass with no copy of a level for each level above it.
type Sizes = std::collections::HashMap<usize, Size>;
/// The element at `at` in `input`: how many bytes it was, and its length as DER, header and all.
fn measure(input: &[u8], at: usize, depth: usize, sizes: &mut Sizes) -> Option<(usize, usize)> {
if depth > MAX_DEPTH || sizes.len() >= MAX_ELEMENTS {
return None;
}
let bytes = input.get(at..)?;
if let Some((tlv, _)) = read(bytes)
&& der_shaped(tlv.whole, depth)
{
sizes.insert(at, Size { used: tlv.whole.len(), content: tlv.value.len(), verbatim: true });
return Some((tlv.whole.len(), tlv.whole.len()));
}
let t = tag_len(bytes)?;
let tag = *bytes.first()?;
let (len, l) = length(bytes.get(t..)?)?;
let start = t.checked_add(l)?;
if tag & 0x20 == 0 {
let used = start.checked_add(len?)?;
bytes.get(..used)?;
return Some((used, used_der(t, used.checked_sub(start)?)?));
}
// The children, one after another, to the end of a definite length or to the end mark.
let mut content = 0usize;
let mut pos = start;
let end = match len {
Some(len) => Some(start.checked_add(len)?),
None => None,
};
loop {
match end {
Some(end) if pos >= end => break,
None if bytes.get(pos..)?.starts_with(&[0, 0]) => {
pos = pos.checked_add(2)?;
break;
}
_ => {}
}
// (The pieces of an OCTET STRING are OCTET STRINGs, in pieces themselves or not.)
if tag == OCTET_STRING_PIECES && !matches!(bytes.get(pos), Some(&OCTET_STRING | &OCTET_STRING_PIECES)) {
return None;
}
let (n, der) = measure(input, at.checked_add(pos)?, depth.checked_add(1)?, sizes)?;
if n == 0 {
return None;
}
// An OCTET STRING in pieces is one of their contents together.
content = content.checked_add(if tag == OCTET_STRING_PIECES { der_content(input, at.checked_add(pos)?, sizes)? } else { der })?;
pos = pos.checked_add(n)?;
}
if let Some(end) = end
&& pos != end
{
return None;
}
if sizes.len() >= MAX_ELEMENTS {
return None;
}
sizes.insert(at, Size { used: pos, content, verbatim: false });
let header_tag = if tag == OCTET_STRING_PIECES { 1 } else { t };
Some((pos, used_der(header_tag, content)?))
}
/// The length of a DER element with a tag of `tag` bytes and `content` bytes of content.
fn used_der(tag: usize, content: usize) -> Option<usize> {
let mut l = Vec::new();
put_length(&mut l, content);
tag.checked_add(l.len())?.checked_add(content)
}
/// The DER content length of the (measured) element at `at`: a primitive's own.
fn der_content(input: &[u8], at: usize, sizes: &Sizes) -> Option<usize> {
if let Some(s) = sizes.get(&at) {
return Some(s.content);
}
let bytes = input.get(at..)?;
let t = tag_len(bytes)?;
let (len, _) = length(bytes.get(t..)?)?;
len
}
/// Write the (measured) element at `at` in `input` to `out` as DER; how many bytes it was.
fn write(input: &[u8], at: usize, sizes: &Sizes, pieces: bool, out: &mut Vec<u8>) -> Option<usize> {
if let Some(s) = sizes.get(&at).filter(|s| s.verbatim && !pieces) {
out.extend_from_slice(input.get(at..at.checked_add(s.used)?)?);
return Some(s.used);
}
let bytes = input.get(at..)?;
let t = tag_len(bytes)?;
let tag_bytes = bytes.get(..t)?;
let tag = *tag_bytes.first()?;
let (len, l) = length(bytes.get(t..)?)?;
let start = t.checked_add(l)?;
if tag & 0x20 == 0 {
let used = start.checked_add(len?)?;
let value = bytes.get(start..used)?;
// (A piece of an OCTET STRING in pieces gives its bytes alone.)
if !pieces {
out.extend_from_slice(tag_bytes);
put_length(out, value.len());
}
out.extend_from_slice(value);
return Some(used);
}
let size = *sizes.get(&at)?;
if tag == OCTET_STRING_PIECES {
if !pieces {
out.push(OCTET_STRING);
put_length(out, size.content);
}
} else {
out.extend_from_slice(tag_bytes);
put_length(out, size.content);
}
let inner_pieces = tag == OCTET_STRING_PIECES;
let mut pos = start;
while pos < size.used {
if bytes.get(pos..)?.starts_with(&[0, 0]) && len.is_none() {
break;
}
let n = write(input, at.checked_add(pos)?, sizes, inner_pieces, out)?;
pos = pos.checked_add(n)?;
}
Some(size.used)
}
/// Write the element at the start of `bytes` to `out` as DER; how many bytes of `bytes` it was.
fn element(bytes: &[u8], depth: usize, out: &mut Vec<u8>) -> Option<usize> {
let mut sizes = Sizes::new();
let (used, der) = measure(bytes, 0, depth, &mut sizes)?;
out.reserve(der);
write(bytes, 0, &sizes, false, out)?;
Some(used)
}
/// A definite length in DER's shortest form.
fn put_length(out: &mut Vec<u8>, len: usize) {
if len < 0x80 {
out.push(len as u8);
return;
}
let bytes = len.to_be_bytes();
let skip = bytes.iter().take_while(|b| **b == 0).count();
let digits = bytes.get(skip..).unwrap_or_default();
out.push(0x80 | digits.len() as u8);
out.extend_from_slice(digits);
}
#[cfg(test)]
mod tests {
use super::*;
fn tlv(tag: u8, value: &[u8]) -> Vec<u8> {
let mut out = vec![tag];
put_length(&mut out, value.len());
out.extend_from_slice(value);
out
}
fn cms(outer_tag: u8, signed_tag: u8, oid_tag: u8, long_oid: bool, trailing: bool) -> Vec<u8> {
let oid = |value: &[u8]| {
if long_oid {
let mut out = vec![oid_tag, 0x81, 9];
out.extend_from_slice(value);
out
} else {
tlv(oid_tag, value)
}
};
let mut encap = oid(DATA);
encap.extend(tlv(0xa0, &tlv(0x04, b"hello")));
let mut signed = tlv(0x02, &[1]);
signed.extend(tlv(0x31, &[]));
signed.extend(tlv(0x30, &encap));
signed.extend(tlv(0x31, &[]));
let mut signed = tlv(signed_tag, &signed);
if trailing {
signed.extend(tlv(0x05, &[]));
}
let mut info = oid(SIGNED_DATA);
info.extend(tlv(0xa0, &signed));
tlv(outer_tag, &info)
}
#[test]
fn extracts_content_with_der_and_ber_oid_lengths() {
for long_oid in [false, true] {
assert_eq!(signed_content(&cms(0x30, 0x30, 0x06, long_oid, false)), Some(b"hello".to_vec()));
}
}
#[test]
fn rejects_wrong_cms_container_and_oid_tags() {
for (outer, signed, oid) in [(0x31, 0x30, 0x06), (0x30, 0x31, 0x06), (0x30, 0x30, 0x04)] {
assert_eq!(signed_content(&cms(outer, signed, oid, false, false)), None);
}
}
#[test]
fn rejects_missing_mandatory_signed_data_fields() {
let mut encap = tlv(0x06, DATA);
encap.extend(tlv(0xa0, &tlv(0x04, b"hello")));
for prefix in [vec![0x05, 0, 0x05, 0], vec![0x02, 1, 1, 0x31, 0]] {
let mut malformed = prefix;
malformed.extend(tlv(0x30, &encap));
let mut info = tlv(0x06, SIGNED_DATA);
info.extend(tlv(0xa0, &tlv(0x30, &malformed)));
assert_eq!(signed_content(&tlv(0x30, &info)), None);
}
}
#[test]
fn rejects_extra_signed_data_inside_explicit_container() {
assert_eq!(signed_content(&cms(0x30, 0x30, 0x06, false, true)), None);
}
#[test]
fn rejects_wide_inputs_before_unbounded_metadata_allocation() {
let children_bytes = [0x05, 0].repeat(MAX_ELEMENTS + 1);
assert!(children(&children_bytes).is_none());
let mut indefinite = vec![0x30, 0x80];
indefinite.extend_from_slice(&children_bytes);
indefinite.extend_from_slice(&[0, 0]);
assert!(to_der(&indefinite).is_none());
}
#[test]
fn joins_indefinite_octet_string_pieces() {
assert_eq!(to_der(&[0x24, 0x80, 0x04, 1, b'a', 0x04, 1, b'b', 0, 0]), Some(vec![0x04, 2, b'a', b'b']));
}
#[test]
fn rejects_truncated_and_over_nested_input() {
assert_eq!(to_der(&[0x04, 2, b'a']), None);
let mut nested = tlv(0x04, &[]);
for _ in 0..=MAX_DEPTH {
nested = tlv(0x30, &nested);
}
assert_eq!(to_der(&nested), None);
}
}
+1
View File
@@ -45,6 +45,7 @@ exr = { version = "1.73", default-features = false, features = ["rayon"] }
[dev-dependencies]
proptest = { workspace = true }
image = { version = "0.25", default-features = false, features = ["png", "jpeg", "tiff"] }
sha2 = { version = "0.10", default-features = false, features = ["std"] }
[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
jxl-oxide = { version = "=0.12.6", default-features = false, features = ["rayon"] }
+16 -1
View File
@@ -302,9 +302,11 @@ embedded table (and a PNG's transparent entry) so `photocraft-io` can open them
`DecodeOptions { limits: Limits { max_width, max_height, max_pixels, max_alloc }, .. }` guards
decoding. Header dimensions are checked before the pixel buffer is allocated, and the budget is
also passed to the underlying decoders. A violation returns `CodecError::LimitExceeded`. The
defaults are 262144 px per side, 2^30 pixels and 8 GiB (2 GiB on 32-bit targets such as wasm).
defaults are 262144 px per side, 2^30 pixels and 8 GiB (1 GiB on 32-bit targets such as wasm).
PNG also bounds the aggregate decoded text and XMP (UTF-8 keywords and values) by `max_alloc`,
including chunks after IDAT. This budget is separate from pixels, not a total process-memory cap.
`Image::to_rgba_f32(&limits)` checks its output buffer against the same `Limits` and returns
`CodecError::LimitExceeded` before allocating when the RGBA f32 copy would exceed them.
PNG `cICP` and `cLLI` values are retained as image color metadata. `encode_png_pq` writes linear-light
Rec. 2020 RGB as 16-bit BT.2100 PQ, with measured content-light levels and the required chunks.
@@ -324,3 +326,16 @@ Rec. 2020 RGB as 16-bit BT.2100 PQ, with measured content-light levels and the r
If `corpus/pngsuite/*.png` exists at the repo root, every file in it is compared against the
`image` crate's decoder. Without it, that test is skipped.
With the `corpus` feature, the deep EXR test reads `11.deep.exr`, `42.deep.exr`, `64.deep.exr`,
`multivariate.deep.exr` and `objectid.deep.exr` from `corpus/exr/` at the repo root. These are the
deep test images in `src/test/bin/test_images` of the
[OpenEXR repository at commit `69b2604fc76e370615438bdc8d2cd95b9349c12e`](https://github.com/AcademySoftwareFoundation/openexr/tree/69b2604fc76e370615438bdc8d2cd95b9349c12e/src/test/bin/test_images).
The test checks each file against the SHA-256 pinned in `REAL_FILE_SHA256` in
`tests/it/deep_exr.rs` before decoding it, and fails when a file is missing, is not a regular
file, is larger than 4 MiB or does not match. It reads no other file in the folder. To fill the
folder, copy the five files from that commit into `corpus/exr/`, or run
`cargo xtask corpus --exr` in a checkout of [PhotoCraft](https://github.com/storytold/PhotoCraft)
at commit `74330157cb601573ae4523a95bb9132822fc3918` and copy its `corpus/exr/` here, as CI does.
The images are not committed here; OpenEXR distributes them under its
[license](https://github.com/AcademySoftwareFoundation/openexr/blob/69b2604fc76e370615438bdc8d2cd95b9349c12e/LICENSE.md).
+101 -26
View File
@@ -8,6 +8,7 @@ use std::fmt;
use crate::error::CodecError;
use crate::format::Format;
use crate::options::Limits;
use half::f16;
/// Channel layout of an [`Image`]. Alpha, when present, is always the last
@@ -716,9 +717,31 @@ impl Image {
self.convert(ChannelLayout::Rgba, SampleType::U16).to_u16_samples().unwrap_or_default()
}
/// Interleaved RGBA as f32 (integer inputs normalized to `[0,1]`).
pub fn to_rgba_f32(&self) -> Vec<f32> {
convert_layout(&self.to_normalized(), self.layout, ChannelLayout::Rgba)
/// Interleaved RGBA as f32 (integer inputs normalized to `[0,1]`), with the same values as
/// [`Image::convert`] to RGBA F32. The output buffer is checked against `limits` (dimensions,
/// pixel count and `max_alloc`) and reserved before any pixel is converted, and the pixels are
/// written straight into it. A size over the limits, or a reservation the allocator refuses,
/// returns [`CodecError::LimitExceeded`]. An image with no pixels gives an empty vector.
pub fn to_rgba_f32(&self, limits: &Limits) -> Result<Vec<f32>, CodecError> {
if self.width == 0 || self.height == 0 {
return Ok(Vec::new());
}
limits.check(self.width, self.height, ChannelLayout::Rgba, SampleType::F32)?;
let len = self
.pixel_count()
.checked_mul(ChannelLayout::Rgba.channels())
.ok_or_else(|| CodecError::LimitExceeded("RGBA f32 size overflows usize".into()))?;
let mut out = Vec::new();
out.try_reserve_exact(len).map_err(|e| CodecError::LimitExceeded(format!("cannot reserve {len} f32 samples for RGBA: {e}")))?;
let fc = self.layout.channels();
let mut px = [0.0f32; 5];
for p in 0..self.pixel_count() {
for (c, v) in px[..fc].iter_mut().enumerate() {
*v = read_normalized(&self.data, self.sample, p * fc + c);
}
convert_pixel(&px[..fc], self.layout, ChannelLayout::Rgba, &mut out);
}
Ok(out)
}
}
@@ -890,33 +913,39 @@ pub(crate) fn convert_layout(src: &[f32], from: ChannelLayout, to: ChannelLayout
let n = src.len() / fc;
let mut out = Vec::with_capacity(n * tc);
for px in src.chunks_exact(fc) {
let alpha = if from.has_alpha() { px[fc - 1] } else { 1.0 };
let color = &px[..from.color_channels()];
if from.same_model(to) {
out.extend_from_slice(color);
} else {
let rgb = match from {
ChannelLayout::Gray | ChannelLayout::GrayA => [color[0]; 3],
ChannelLayout::Rgb | ChannelLayout::Rgba => [color[0], color[1], color[2]],
ChannelLayout::Cmyk | ChannelLayout::CmykA => cmyk_to_rgb(color[0], color[1], color[2], color[3]),
ChannelLayout::Lab | ChannelLayout::LabA => crate::lab::lab_to_srgb([color[0], color[1], color[2]]),
};
match to {
ChannelLayout::Gray | ChannelLayout::GrayA => {
out.push(rgb[0] * LUMA[0] + rgb[1] * LUMA[1] + rgb[2] * LUMA[2]);
}
ChannelLayout::Rgb | ChannelLayout::Rgba => out.extend_from_slice(&rgb),
ChannelLayout::Cmyk | ChannelLayout::CmykA => out.extend_from_slice(&rgb_to_cmyk(rgb[0], rgb[1], rgb[2])),
ChannelLayout::Lab | ChannelLayout::LabA => out.extend_from_slice(&crate::lab::srgb_to_lab(rgb)),
}
}
if to.has_alpha() {
out.push(alpha);
}
convert_pixel(px, from, to, &mut out);
}
out
}
/// One pixel of [`convert_layout`]: the normalized samples `px` (`from` layout) appended to `out`
/// in the `to` layout.
fn convert_pixel(px: &[f32], from: ChannelLayout, to: ChannelLayout, out: &mut Vec<f32>) {
let alpha = if from.has_alpha() { px[from.channels() - 1] } else { 1.0 };
let color = &px[..from.color_channels()];
if from.same_model(to) {
out.extend_from_slice(color);
} else {
let rgb = match from {
ChannelLayout::Gray | ChannelLayout::GrayA => [color[0]; 3],
ChannelLayout::Rgb | ChannelLayout::Rgba => [color[0], color[1], color[2]],
ChannelLayout::Cmyk | ChannelLayout::CmykA => cmyk_to_rgb(color[0], color[1], color[2], color[3]),
ChannelLayout::Lab | ChannelLayout::LabA => crate::lab::lab_to_srgb([color[0], color[1], color[2]]),
};
match to {
ChannelLayout::Gray | ChannelLayout::GrayA => {
out.push(rgb[0] * LUMA[0] + rgb[1] * LUMA[1] + rgb[2] * LUMA[2]);
}
ChannelLayout::Rgb | ChannelLayout::Rgba => out.extend_from_slice(&rgb),
ChannelLayout::Cmyk | ChannelLayout::CmykA => out.extend_from_slice(&rgb_to_cmyk(rgb[0], rgb[1], rgb[2])),
ChannelLayout::Lab | ChannelLayout::LabA => out.extend_from_slice(&crate::lab::srgb_to_lab(rgb)),
}
}
if to.has_alpha() {
out.push(alpha);
}
}
#[cfg(test)]
mod tests {
#[test]
@@ -1042,6 +1071,52 @@ mod tests {
assert!(Image::from_raw(2, 2, ChannelLayout::Rgb, SampleType::U8, vec![0; 11]).is_err());
}
#[test]
fn rgba_f32_limit_rejects_before_allocation() {
// 3x2 pixels: the RGBA f32 output is 3 * 2 * 4 * 4 = 96 bytes. Most sources here are
// smaller than that, so the source fitting the budget does not let the output through.
for sample in SampleType::ALL {
for layout in ChannelLayout::ALL {
let img = Image::new(3, 2, layout, sample).unwrap();
let tight = Limits::new(3, 2, 6, 95);
assert!(matches!(img.to_rgba_f32(&tight), Err(CodecError::LimitExceeded(_))), "{layout:?} {sample:?}: max_alloc");
let exact = Limits::new(3, 2, 6, 96);
assert_eq!(img.to_rgba_f32(&exact).map(|v| v.len()).ok(), Some(24), "{layout:?} {sample:?}");
for limits in [Limits::new(2, 2, 6, 96), Limits::new(3, 1, 6, 96), Limits::new(3, 2, 5, 96)] {
assert!(matches!(img.to_rgba_f32(&limits), Err(CodecError::LimitExceeded(_))), "{layout:?} {sample:?}: {limits:?}");
}
}
}
}
#[test]
fn rgba_f32_matches_existing_conversions() {
// Edge, mid and (for float samples) out-of-range values, so the CMYK, Lab and alpha
// formulas and the f16 and f32 reads are all compared bit for bit.
let base = [0.0, 1.0, 0.5, 0.25, 0.75, 1.0 / 255.0, 0.3, 0.9, 1.5, -0.25];
for sample in SampleType::ALL {
for layout in ChannelLayout::ALL {
let n = 7 * 3 * layout.channels();
let vals: Vec<f32> = (0..n).map(|i| base[(i * 7 + i / 3) % base.len()]).collect();
let img = Image::from_normalized(7, 3, layout, sample, &vals).unwrap();
let got = img.to_rgba_f32(&Limits::default()).unwrap();
let want = convert_layout(&img.to_normalized(), layout, ChannelLayout::Rgba);
let via_convert = img.convert(ChannelLayout::Rgba, SampleType::F32).to_f32_samples().unwrap();
let bits = |v: &[f32]| v.iter().map(|x| x.to_bits()).collect::<Vec<_>>();
assert_eq!(bits(&got), bits(&want), "{layout:?} {sample:?}");
assert_eq!(bits(&got), bits(&via_convert), "{layout:?} {sample:?}: convert");
}
}
}
#[test]
fn rgba_f32_of_an_empty_image_is_empty() {
for (w, h) in [(0, 0), (0, 3), (4, 0)] {
let img = Image::new(w, h, ChannelLayout::Rgb, SampleType::U8).unwrap();
assert_eq!(img.to_rgba_f32(&Limits::new(1, 1, 1, 0)).ok(), Some(Vec::new()), "{w}x{h}");
}
}
#[test]
fn rgba8_of_gray() {
let img = Image::from_u8(1, 1, ChannelLayout::GrayA, vec![10, 20]).unwrap();
+119 -8
View File
@@ -2,11 +2,14 @@
//! specification (NONE, RLE and ZIPS × scanline and tile), the oracle tests check the
//! compositing math against hand-computed values, and malformed inputs must error, never
//! panic. With the `corpus` feature, the OpenEXR project's deep test images (`corpus/exr/`,
//! pinned) are checked against the ID manifests their sidecars list.
//! each checked against its pinned SHA-256 before decoding) are checked against the ID
//! manifests their sidecars list.
use craft_image_codecs::CodecError;
use craft_image_codecs::f16;
use craft_image_codecs::{Limits, decode, decode_deep_exr};
use sha2::{Digest, Sha256};
use std::io::Read as _;
use std::io::Write as _;
// ---------------------------------------------------------------------------------------------
@@ -797,10 +800,55 @@ fn inconsistent_count_table_errors() {
}
// ---------------------------------------------------------------------------------------------
// Real reference files: the deep test images of the OpenEXR repository, pinned in
// `xtask/src/corpus_pins.rs` and fetched into `corpus/exr/` by `cargo xtask corpus --exr`
// (cargo feature `corpus`; `cargo xtask test-corpus`). Their `.txt` sidecars upstream list the
// ID manifest of each file, an independent oracle for the u32 ID channels.
// Real reference files: the deep test images of the OpenEXR repository at commit
// `69b2604fc76e370615438bdc8d2cd95b9349c12e`, read from `corpus/exr/` at the repo root with the
// cargo feature `corpus`. `REAL_FILE_SHA256` pins each file's SHA-256, and the test refuses a
// file that is missing, is not a regular file, is larger than `MAX_REAL_FILE_BYTES` or does not
// match its digest. The crate README says how to fill the folder. Their `.txt` sidecars upstream
// list the ID manifest of each file, an independent oracle for the u32 ID channels.
/// The pinned files as name and lowercase hex SHA-256, the same five names as `REAL_FILES`.
const REAL_FILE_SHA256: &[(&str, &str)] = &[
("11.deep.exr", "800ec94ffb2085da9a730dcda2f9611f5fb62f922a3f2aef77287e36106b9667"),
("42.deep.exr", "439907f3448480894264f0c8847f6c572f645441db34a5ad9f36f3f020d101b5"),
("64.deep.exr", "4d9bfa646323b80630933e114a3112095488e8c008cdf908e210979638e65fe4"),
("multivariate.deep.exr", "2335ed6052258fe693cb795f2bea57f22ba0564281691c82f2987d31d96e9e74"),
("objectid.deep.exr", "af8940912bfa856b2650d2005462fd348656385161ccf0745b91b143e934c5f6"),
];
/// The largest file read from `corpus/exr/`. The largest pinned file is 611,840 bytes.
#[cfg(feature = "corpus")]
const MAX_REAL_FILE_BYTES: u64 = 4 * 1024 * 1024;
/// The lowercase hex SHA-256 of `bytes`.
fn sha256_hex(bytes: &[u8]) -> String {
Sha256::digest(bytes).iter().map(|b| format!("{b:02x}")).collect()
}
/// Reads a regular file of at most `limit` bytes. A symbolic link, a folder or a larger file is an
/// error, and a file that grows during the read is cut off at `limit + 1` bytes and refused.
fn read_bounded(path: &std::path::Path, limit: u64) -> Result<Vec<u8>, String> {
let meta = std::fs::symlink_metadata(path).map_err(|e| format!("{}: {e}", path.display()))?;
if !meta.is_file() || meta.len() > limit {
return Err(format!("{}: expected a regular file of at most {limit} bytes", path.display()));
}
let mut bytes = Vec::new();
std::fs::File::open(path).and_then(|f| f.take(limit + 1).read_to_end(&mut bytes)).map_err(|e| format!("{}: {e}", path.display()))?;
if bytes.len() as u64 > limit {
return Err(format!("{}: exceeds {limit} bytes", path.display()));
}
Ok(bytes)
}
/// Reads `dir/name` within `limit` bytes and checks its SHA-256 against `pinned`.
fn read_pinned(dir: &std::path::Path, name: &str, pinned: &str, limit: u64) -> Result<Vec<u8>, String> {
let bytes = read_bounded(&dir.join(name), limit)?;
let got = sha256_hex(&bytes);
if got != pinned {
return Err(format!("{name}: SHA-256 {got} does not match the pinned {pinned}"));
}
Ok(bytes)
}
/// One pinned file: name, total samples, most samples in one pixel, and per ID channel the
/// manifest IDs its sidecar lists.
@@ -846,9 +894,12 @@ const REAL_FILES: &[RealFile] = &[
#[test]
fn real_deep_files_decode() {
let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus/exr");
for &(name, total, max_per_pixel, ids) in REAL_FILES {
let path = dir.join(name);
let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{}: {e} (run `cargo xtask corpus --exr`)", path.display()));
let decoded: Vec<&str> = REAL_FILES.iter().map(|f| f.0).collect();
let pinned: Vec<&str> = REAL_FILE_SHA256.iter().map(|p| p.0).collect();
assert_eq!(decoded, pinned, "REAL_FILES and REAL_FILE_SHA256 must list the same files");
for (&(name, total, max_per_pixel, ids), &(_, sha256)) in REAL_FILES.iter().zip(REAL_FILE_SHA256) {
let bytes = read_pinned(&dir, name, sha256, MAX_REAL_FILE_BYTES)
.unwrap_or_else(|e| panic!("{}: {e}; place the pinned OpenEXR deep test images in this folder (see the crate README)", dir.display()));
let deep = decode_deep_exr(&bytes, &Limits::none()).unwrap_or_else(|e| panic!("{name}: {e}"));
assert_eq!((deep.width, deep.height), (256, 256), "{name}");
assert_eq!(deep.total_samples(), total, "{name}");
@@ -874,6 +925,66 @@ fn real_deep_files_decode() {
}
}
#[test]
fn real_file_pins_are_well_formed() {
let mut names = std::collections::BTreeSet::new();
for &(name, sha256) in REAL_FILE_SHA256 {
assert!(name.ends_with(".deep.exr") && !name.contains(['/', '\\']), "{name}");
assert!(names.insert(name), "{name} is pinned twice");
assert_eq!(sha256.len(), 64, "{name}");
assert!(sha256.bytes().all(|c| matches!(c, b'0'..=b'9' | b'a'..=b'f')), "{name}: {sha256}");
}
assert_eq!(names.len(), 5);
}
/// A folder under the system temp directory, removed when dropped.
struct ScratchDir(std::path::PathBuf);
impl ScratchDir {
fn new(tag: &str) -> Self {
let dir = std::env::temp_dir().join(format!("craft-image-codecs-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).expect("create scratch dir");
Self(dir)
}
}
impl Drop for ScratchDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
#[test]
fn pinned_read_refuses_a_changed_byte() {
let dir = ScratchDir::new("deep-exr-pin");
// SHA-256 of "abc" (FIPS 180-2, appendix B.1).
let abc = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad";
std::fs::write(dir.0.join("a.deep.exr"), b"abc").expect("write");
assert_eq!(read_pinned(&dir.0, "a.deep.exr", abc, 16).as_deref(), Ok(&b"abc"[..]));
std::fs::write(dir.0.join("a.deep.exr"), b"abd").expect("write");
let err = read_pinned(&dir.0, "a.deep.exr", abc, 16).expect_err("a changed byte must be refused");
assert!(err.starts_with("a.deep.exr: ") && err.contains(abc) && err.contains(&sha256_hex(b"abd")), "{err}");
}
#[test]
fn pinned_read_refuses_missing_oversize_and_folders() {
let dir = ScratchDir::new("deep-exr-bound");
let pin = sha256_hex(b"12345678");
std::fs::write(dir.0.join("eight.deep.exr"), b"12345678").expect("write");
assert!(read_pinned(&dir.0, "eight.deep.exr", &pin, 8).is_ok());
let err = read_pinned(&dir.0, "eight.deep.exr", &pin, 7).expect_err("a file over the limit must be refused");
assert!(err.contains("at most 7 bytes"), "{err}");
let err = read_pinned(&dir.0, "missing.deep.exr", &pin, 8).expect_err("a missing file must be refused");
assert!(err.contains("missing.deep.exr"), "{err}");
std::fs::create_dir(dir.0.join("folder.deep.exr")).expect("mkdir");
let err = read_pinned(&dir.0, "folder.deep.exr", &pin, 8).expect_err("a folder must be refused");
assert!(err.contains("regular file"), "{err}");
}
/// `gen_deep` for a cropped render: the data window sits at `(dwx, dwy)` inside a larger
/// display window of `sw × sh` (the stage the renderer composed onto). Scanline chunk Y
/// coordinates are absolute display rows, so they carry the data-window offset.
+162 -11
View File
@@ -351,28 +351,70 @@ fn trak(bytes: &[u8], t: &BoxRef, budget: &mut usize) -> Option<Cr3Track> {
}
}
let stbl = stbl?;
let (mut kind, mut size, mut offset) = (None, None, None);
let (mut kind, mut size, mut offset, mut mapping, mut sample_count) = (None, None, None, None, 0);
for b in boxes(bytes, stbl.body, stbl.end, budget) {
let body = bytes.get(b.body..b.end)?;
match &b.kind {
// full box: version/flags, entry count, entries
b"stsd" if be32(body, 4)? > 0 => kind = boxes(bytes, b.body + 8, b.end, budget).first().map(|e| sample_entry(bytes, e, budget)),
b"stsc" => {
// Only the first sample is exposed. It must map chunk 1 to
// the first sample description; don't trust offsets without
// that mapping, or a damaged/raw track could become a JPEG.
// A malformed mapping invalidates sample data, not the track's
// identity: dropping the largest raw track could select a reduced
// raw successfully and disguise corruption in the main image.
mapping = (|| {
let count = be32(body, 4)? as usize;
if be32(body, 0)? != 0 || count == 0 || count > body.len().saturating_sub(8) / 12 {
return None;
}
Some((be32(body, 8)?, be32(body, 12)?, be32(body, 16)?))
})();
}
// version/flags, sample_size (0 = per-sample table), count, sizes
b"stsz" => {
size = match be32(body, 4)? {
0 if be32(body, 8)? > 0 => be32(body, 12),
0 => None,
n if be32(body, 8)? > 0 => Some(n),
// A truncated size table invalidates data without hiding a known track.
sample_count = be32(body, 8).unwrap_or(0);
size = match be32(body, 4) {
// A first entry cannot justify an incomplete per-sample
// table, nor an impossible constant-size sample count.
Some(0) if sample_count > 0 && sample_count as usize <= body.len().saturating_sub(12) / 4 => be32(body, 12),
Some(n)
if n > 0
&& sample_count > 0
&& u64::from(n).checked_mul(u64::from(sample_count)).is_some_and(|total| total <= bytes.len() as u64) =>
{
Some(n)
}
_ => None,
}
}
b"co64" if be32(body, 4)? > 0 => offset = be64(body, 8).and_then(|v| usize::try_from(v).ok()),
b"stco" if be32(body, 4)? > 0 => offset = be32(body, 8).map(|v| v as usize),
// Missing or truncated offset-table counts deny sample access, but the known
// track must remain available to the largest-raw-track selector.
b"co64" => {
offset = match be32(body, 4) {
Some(count) if count > 0 && count as usize <= body.len().saturating_sub(8) / 8 => {
be64(body, 8).and_then(|v| usize::try_from(v).ok())
}
_ => None,
};
}
b"stco" => {
offset = match be32(body, 4) {
Some(count) if count > 0 && count as usize <= body.len().saturating_sub(8) / 4 => be32(body, 8).map(|v| v as usize),
_ => None,
};
}
_ => {}
}
}
let data = match (offset, size) {
(Some(o), Some(s)) if o.checked_add(s as usize).is_some_and(|e| e <= bytes.len()) => Some((o, s as usize)),
let data = match (offset, size, mapping) {
(Some(o), Some(s), Some((1, samples, 1)))
if samples > 0 && samples <= sample_count && o.checked_add(s as usize).is_some_and(|e| e <= bytes.len()) =>
{
Some((o, s as usize))
}
_ => None,
};
Some(Cr3Track { kind: kind?, data })
@@ -495,6 +537,13 @@ mod tests {
bx(b"CRAW", &b)
}
fn trak(entry: Vec<u8>, offset: u64, size: u32) -> Vec<u8> {
let stsc = [1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>();
trak_with_mapping(entry, offset, size, &stsc)
}
fn trak_with_mapping(entry: Vec<u8>, offset: u64, size: u32, mapping: &[u8]) -> Vec<u8> {
trak_with_table(entry, offset, size, b"stsc", mapping)
}
fn trak_with_table(entry: Vec<u8>, offset: u64, size: u32, table: &[u8; 4], payload: &[u8]) -> Vec<u8> {
let mut stsd = 1u32.to_be_bytes().to_vec();
stsd.extend(entry);
let mut stsz = 0u32.to_be_bytes().to_vec();
@@ -502,7 +551,15 @@ mod tests {
stsz.extend(size.to_be_bytes());
let mut co64 = 1u32.to_be_bytes().to_vec();
co64.extend(offset.to_be_bytes());
let stbl = [full(b"stsd", &stsd), full(b"stsz", &stsz), full(b"co64", &co64)].concat();
let stsc = [1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>();
let offsets = if table == b"stco" { full(b"stco", payload) } else { full(b"co64", if table == b"co64" { payload } else { &co64 }) };
let stbl = [
full(b"stsd", &stsd),
full(b"stsc", if table == b"stsc" { payload } else { &stsc }),
full(b"stsz", if table == b"stsz" { payload } else { &stsz }),
offsets,
]
.concat();
bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &stbl))))
}
fn tiff_with(tag: u16, v: craft_tiff::Value) -> Vec<u8> {
@@ -543,6 +600,97 @@ mod tests {
assert_eq!(c.tracks[1].data, Some((4112, 32)));
}
#[test]
fn first_sample_requires_a_complete_valid_sample_to_chunk_mapping() {
let file = |track: Vec<u8>| {
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
file.extend(bx(b"moov", &track));
file.resize(4112, 0);
file
};
let valid = trak(craw(16, 16, &bx(b"JPEG", &[0; 4])), 4096, 16);
assert_eq!(parse_cr3(&file(valid.clone())).unwrap().tracks[0].data, Some((4096, 16)));
let at = valid.windows(4).position(|bytes| bytes == b"stsc").unwrap();
// Missing table, impossible first chunk, zero samples, a different
// description and a count that extends beyond this box all deny data.
for (field, value) in [(0, u32::from_be_bytes(*b"xxxx")), (8, 0), (8, 2), (12, 0), (12, 2), (16, 0), (16, 2), (4, u32::MAX)] {
let mut damaged = valid.clone();
let offset = if field == 0 { at } else { at + 4 + field };
damaged[offset..offset + 4].copy_from_slice(&value.to_be_bytes());
let bytes = file(damaged);
let parsed = parse_cr3(&bytes).unwrap();
assert_eq!(parsed.tracks.len(), 1, "invalid mapping must not erase an identifiable track");
assert_eq!(parsed.tracks[0].kind, Cr3TrackKind::Jpeg);
assert!(parsed.tracks[0].data.is_none());
}
for entry in [craw(16, 16, &[]), craw(16, 16, &bx(b"CMP1", &[0xff, 0x10, 0, 0x30]))] {
let bytes = file(trak(entry, 4096, 16));
let parsed = parse_cr3(&bytes).unwrap();
assert!(!matches!(parsed.tracks[0].kind, Cr3TrackKind::Jpeg));
}
}
#[test]
fn truncated_sample_mapping_preserves_larger_raw_track_identity_and_valid_reduced_track() {
let reduced = trak(craw(4, 4, &bx(b"CMP1", &compression_payload(0))), 4096, 16);
for table in [b"stsc", b"stsz", b"co64", b"stco"] {
for payload in [Vec::new(), vec![0; 4], 1u32.to_be_bytes().to_vec()] {
// full() supplies version/flags; an empty payload leaves only
// those four bytes, with the table's required count missing.
let main = trak_with_table(craw(8, 4, &bx(b"CMP1", &compression_payload(0))), 4112, 32, table, &payload);
for main_first in [false, true] {
let tracks = if main_first { [main.clone(), reduced.clone()].concat() } else { [reduced.clone(), main.clone()].concat() };
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
file.extend(bx(b"moov", &tracks));
file.resize(4144, 0);
let parsed = parse_cr3(&file).unwrap();
assert_eq!(parsed.tracks.len(), 2, "truncated {table:?} cannot hide the larger track");
let (main, reduced) = if main_first { (&parsed.tracks[0], &parsed.tracks[1]) } else { (&parsed.tracks[1], &parsed.tracks[0]) };
assert!(matches!(reduced.kind, Cr3TrackKind::Raw { width: 4, height: 4, .. }));
assert_eq!(reduced.data, Some((4096, 16)));
assert!(matches!(main.kind, Cr3TrackKind::Raw { width: 8, height: 4, cmp1: Some(_), .. }));
assert!(main.data.is_none());
assert!(main.compression(&file).is_some(), "only sample location is invalid");
}
}
let mut offsets = 1u32.to_be_bytes().to_vec();
offsets.extend(4112u64.to_be_bytes());
let valid = match table {
b"stsc" => [1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>(),
b"stsz" => [0u32, 1, 32].into_iter().flat_map(u32::to_be_bytes).collect(),
b"stco" => [1u32, 4112].into_iter().flat_map(u32::to_be_bytes).collect(),
_ => offsets,
};
let main = trak_with_table(craw(8, 4, &bx(b"CMP1", &compression_payload(0))), 4112, 32, table, &valid);
for main_first in [false, true] {
let tracks = if main_first { [main.clone(), reduced.clone()].concat() } else { [reduced.clone(), main.clone()].concat() };
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
file.extend(bx(b"moov", &tracks));
file.resize(4144, 0);
let parsed = parse_cr3(&file).unwrap();
assert_eq!(parsed.tracks.len(), 2);
let main = &parsed.tracks[if main_first { 0 } else { 1 }];
assert_eq!(main.data, Some((4112, 32)), "complete valid {table:?} stays usable");
}
}
}
#[test]
fn first_sample_size_requires_a_complete_bounded_sample_table() {
let valid = trak(craw(16, 16, &bx(b"JPEG", &[0; 4])), 4096, 16);
let at = valid.windows(4).position(|bytes| bytes == b"stsz").unwrap();
for (constant, count, accepted) in [(0u32, 1u32, true), (16, 1, true), (0, 2, false), (0, u32::MAX, false), (16, u32::MAX, false)] {
let mut track = valid.clone();
track[at + 8..at + 12].copy_from_slice(&constant.to_be_bytes());
track[at + 12..at + 16].copy_from_slice(&count.to_be_bytes());
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
file.extend(bx(b"moov", &track));
file.resize(4112, 0);
let parsed = parse_cr3(&file).unwrap();
assert_eq!(parsed.tracks[0].data, accepted.then_some((4096, 16)), "size {constant}, count {count}");
}
}
#[test]
fn hevc_preview_is_distinct_from_an_equal_size_raw_track() {
let preview = trak(craw(6000, 4000, &bx(b"HEVC", &[0; 4])), 4096, 16);
@@ -714,7 +862,10 @@ mod tests {
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
file.extend(bx(b"moov", &bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &stbl))))));
file.resize(512, 0);
assert!(parse_cr3(&file).unwrap().tracks.is_empty());
let parsed = parse_cr3(&file).unwrap();
assert_eq!(parsed.tracks.len(), 1, "a truncated table must not hide the identifiable raw track");
assert!(matches!(parsed.tracks[0].kind, Cr3TrackKind::Raw { width: 16, height: 16, .. }));
assert!(parsed.tracks[0].data.is_none(), "the empty stsz must not borrow size bytes from the following co64 box");
}
#[test]
+15 -1
View File
@@ -1,6 +1,6 @@
# craft-midi
MIDI file formats. Bytes in, a neutral model out, and back. No file access: callers pass bytes,
MIDI file formats and synchronization wire messages. Bytes in, a neutral model out, and back. No file access: callers pass bytes,
so the crate builds for wasm32.
| Format | Read | Write |
@@ -79,3 +79,17 @@ of two, more than 65535 tracks or a gap beyond the 28-bit delta limit is an erro
Written from the public Standard MIDI Files 1.0 specification (the MIDI Association). No other
product's code, assets or strings were used. The code comes from SoundCraft's `soundcraft-midi`
crate; the tests build every file they read in code, so the crate ships no binary test data.
## Synchronization wire messages
`sync` provides MIDI Clock/Start/Continue/Stop and Song Position Pointer parsing,
clamped song-position encoding, and full-frame/quarter-frame MIDI Time Code serialization.
It is allocation-free and contains no port access, clocks, tempo estimation or transport scheduling.
`SyncMessage::parse` accepts a status-first prefix and ignores trailing bytes, matching LoopCraft;
it is not a byte-stream parser. Time code supports only 24/25/30 non-drop fps, not 29.97 drop-frame.
These wire helpers are extracted from LoopCraft's `crates/midi/src/sync.rs` at
[`18c68c6e`](https://github.com/storytold/loopcraft/blob/18c68c6e18b13d90db3001dd9f95462cdc4b05a8/crates/midi/src/sync.rs).
The source history credits bflatastic. Transport state and scheduling remain app-owned.
Tests cover known wire vectors, truncated/invalid input, all song-position input values and
legacy conversion behavior. No application consumer cutover is claimed by this library addition.
+1
View File
@@ -12,6 +12,7 @@
mod sequence;
mod smf;
pub mod sync;
pub use sequence::{CtrlEvent, CtrlKind, Note, Sequence};
pub use smf::{LenientRead, MAX_EVENTS, MAX_TRACKS, MAX_WARNINGS, Smf, SmfTrack, read_smf, read_smf_lenient, write_smf};
+135
View File
@@ -0,0 +1,135 @@
//! MIDI synchronization wire messages, separate from transport scheduling.
//!
//! Extracted from LoopCraft `crates/midi/src/sync.rs` at
//! `18c68c6e18b13d90db3001dd9f95462cdc4b05a8` (source history credits bflatastic).
//! This module preserves its prefix-parser, clamped song-position and non-drop time-code
//! behavior. It does not parse a byte stream or schedule clocks; hosts retain those policies.
//!
//! Only 24, 25 and 30 non-drop frames per second are represented, not 29.97 drop-frame.
/// MIDI clock: 24 to the quarter note.
pub const CLOCKS_PER_QUARTER: u32 = 24;
pub const CLOCK: u8 = 0xF8;
pub const START: u8 = 0xFA;
pub const CONTINUE: u8 = 0xFB;
pub const STOP: u8 = 0xFC;
pub const SONG_POSITION: u8 = 0xF2;
pub const QUARTER_FRAME: u8 = 0xF1;
/// The furthest Song Position Pointer reaches, in sixteenth notes (fourteen bits).
pub const MAX_POSITION: u16 = 0x3FFF;
/// What a device that leads the time sends.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SyncMessage {
Clock,
Start,
Continue,
Stop,
/// Song Position Pointer: sixteenth notes from the start of the song.
Position(u16),
}
impl SyncMessage {
/// Parse a status-first message prefix; trailing bytes are ignored for compatibility.
/// Returns `None` for an unknown status, truncated song position, or non-seven-bit data.
/// This is not a stream parser and does not remove interleaved real-time messages.
pub fn parse(message: &[u8]) -> Option<SyncMessage> {
Some(match *message {
[CLOCK, ..] => SyncMessage::Clock,
[START, ..] => SyncMessage::Start,
[CONTINUE, ..] => SyncMessage::Continue,
[STOP, ..] => SyncMessage::Stop,
[SONG_POSITION, low, high, ..] if low < 0x80 && high < 0x80 => SyncMessage::Position(u16::from(high) << 7 | u16::from(low)),
_ => return None,
})
}
}
/// Song Position Pointer at `sixteenths` from the start, the low seven bits first.
pub fn song_position(sixteenths: u16) -> [u8; 3] {
let at = sixteenths.min(MAX_POSITION);
[SONG_POSITION, (at & 0x7F) as u8, (at >> 7) as u8]
}
/// The frame rates time code is sent at (non-drop).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum Rate {
Fps24,
#[default]
Fps25,
Fps30,
}
impl Rate {
/// 24 and 30 frames a second are themselves; anything else is 25.
pub fn of(fps: u8) -> Rate {
match fps {
24 => Rate::Fps24,
30 => Rate::Fps30,
_ => Rate::Fps25,
}
}
pub fn fps(self) -> u8 {
match self {
Rate::Fps24 => 24,
Rate::Fps25 => 25,
Rate::Fps30 => 30,
}
}
/// As the hours byte carries it: 0 for 24, 1 for 25, 3 for 30 non-drop.
fn code(self) -> u8 {
match self {
Rate::Fps24 => 0,
Rate::Fps25 => 1,
Rate::Fps30 => 3,
}
}
}
/// Hours, minutes, seconds and frames at `secs`; the hours wrap at a day.
fn timecode(secs: f64, rate: Rate) -> [u8; 4] {
let fps = u64::from(rate.fps());
// `as` saturates: before the start, and NaN, is the start. The nudge keeps a frame's own
// start in it when the sum lands a hair short.
let frames = (secs * fps as f64 + 1e-6) as u64;
let s = frames / fps;
[(s / 3600 % 24) as u8, (s / 60 % 60) as u8, (s % 60) as u8, (frames % fps) as u8]
}
/// The full-frame message (universal real-time SysEx, all devices) locating to `secs`.
///
/// Compatibility conversion: negative values and NaN locate to zero; positive overflow
/// saturates the frame count at `u64::MAX`. Hours wrap at 24. No drop-frame conversion.
pub fn full_frame(secs: f64, rate: Rate) -> [u8; 10] {
let [h, m, s, f] = timecode(secs, rate);
[0xF0, 0x7F, 0x7F, 0x01, 0x01, rate.code() << 5 | h, m, s, f, 0xF7]
}
/// Quarter frame `piece` (0..8) of `time` (hours, minutes, seconds, frames): eight of them, a
/// quarter of a frame apart, carry the whole time, the frames' low nibble first.
fn quarter_frame(time: [u8; 4], rate: Rate, piece: u8) -> [u8; 2] {
let [h, m, s, f] = time;
let piece = piece & 7;
let nibble = match piece {
0 => f,
1 => f >> 4,
2 => s,
3 => s >> 4,
4 => m,
5 => m >> 4,
6 => h,
_ => h >> 4 | rate.code() << 1,
};
[QUARTER_FRAME, piece << 4 | nibble & 0x0F]
}
/// All eight quarter-frame wire messages for a single time-code snapshot, in wire order.
///
/// This is serialization only: the host must send each piece at the appropriate time and
/// account for transport direction/latency. Seconds use the same conversion as [`full_frame`].
pub fn quarter_frames(secs: f64, rate: Rate) -> [[u8; 2]; 8] {
let time = timecode(secs, rate);
[0, 1, 2, 3, 4, 5, 6, 7].map(|piece| quarter_frame(time, rate, piece))
}
+1
View File
@@ -3,3 +3,4 @@
mod lenient;
mod sequence;
mod smf;
mod sync;
+54
View File
@@ -0,0 +1,54 @@
use craft_midi::sync::{CLOCK, CONTINUE, MAX_POSITION, Rate, SONG_POSITION, START, STOP, SyncMessage, full_frame, quarter_frames, song_position};
#[test]
fn transport_prefix_compatibility() {
for (status, expected) in [(CLOCK, SyncMessage::Clock), (START, SyncMessage::Start), (CONTINUE, SyncMessage::Continue), (STOP, SyncMessage::Stop)]
{
assert_eq!(SyncMessage::parse(&[status]), Some(expected));
assert_eq!(SyncMessage::parse(&[status, 0, 0xff]), Some(expected));
}
for invalid in [&[][..], &[0x90, 60, 127], &[SONG_POSITION], &[SONG_POSITION, 0], &[SONG_POSITION, 0x80, 0], &[SONG_POSITION, 0, 0x80]] {
assert_eq!(SyncMessage::parse(invalid), None);
}
}
#[test]
fn song_position_preserves_clamping_and_seven_bit_order() {
for value in 0..=u16::MAX {
let bytes = song_position(value);
assert!(bytes[1] < 128 && bytes[2] < 128);
assert_eq!(SyncMessage::parse(&bytes), Some(SyncMessage::Position(value.min(MAX_POSITION))));
}
assert_eq!(song_position(128), [0xf2, 0, 1]);
assert_eq!(SyncMessage::parse(&[0xf2, 1, 2, 0xff]), Some(SyncMessage::Position(257)));
}
#[test]
fn full_frame_known_vectors_and_legacy_conversion() {
for (rate, bits) in [(Rate::Fps24, 0), (Rate::Fps25, 0x20), (Rate::Fps30, 0x60)] {
assert_eq!(full_frame(3723.0, rate), [0xf0, 0x7f, 0x7f, 1, 1, bits | 1, 2, 3, 0, 0xf7]);
assert_eq!(full_frame(86400.0, rate), full_frame(0.0, rate));
assert_eq!(full_frame(-1.0, rate), full_frame(0.0, rate));
assert_eq!(full_frame(f64::NAN, rate), full_frame(0.0, rate));
assert_eq!(full_frame(1.0 / f64::from(rate.fps()), rate)[8], 1);
}
assert_eq!(Rate::of(24), Rate::Fps24);
assert_eq!(Rate::of(30), Rate::Fps30);
assert_eq!(Rate::of(29), Rate::Fps25);
}
#[test]
fn quarter_frames_known_vector() {
assert_eq!(
quarter_frames(3723.04, Rate::Fps25),
[[0xf1, 1], [0xf1, 0x10], [0xf1, 0x23], [0xf1, 0x30], [0xf1, 0x42], [0xf1, 0x50], [0xf1, 0x61], [0xf1, 0x72]]
);
for rate in [Rate::Fps24, Rate::Fps25, Rate::Fps30] {
for secs in [0.0, 59.99, 3599.99, 86399.99, f64::INFINITY, f64::NEG_INFINITY, f64::NAN] {
for bytes in quarter_frames(secs, rate) {
assert_eq!(bytes[0], 0xf1);
assert!(bytes[1] < 128);
}
}
}
}
+41 -21
View File
@@ -370,7 +370,8 @@ impl DirectoryBundleWriter {
/// Saves an entry set incrementally. Existing objects are reused only after a hash check;
/// unchanged file signatures may skip checks until they enter the rolling verification
/// window. The manifest is atomically published last, then stale previews and objects are
/// collected while the bundle lock is held.
/// collected while the bundle lock is held. Collection is best-effort after publication:
/// inaccessible or unsafe leftovers cannot turn a committed save into a reported failure.
pub fn save_directory(&mut self, root: &Path, bundle: ZipBundleWriter) -> Result<DirectoryStats> {
if !bundle.entries.contains_key(craft_pcraft_format::MANIFEST) {
return Err(Error::InvalidBundle("manifest not set".into()));
@@ -401,6 +402,10 @@ impl DirectoryBundleWriter {
Err(std::fs::TryLockError::Error(_)) => break None,
}
};
// A known unsafe manifest cannot be published. Refuse it before writing objects or
// previews, and still recheck immediately before publication below.
let manifest_path = root.join(craft_pcraft_format::MANIFEST);
refuse_symlink(&manifest_path)?;
self.generation = self.generation.saturating_add(1);
let generation = self.generation;
let recheck = self.rolling_recheck(&bundle.entries);
@@ -449,43 +454,33 @@ impl DirectoryBundleWriter {
}
let manifest = bundle.entries.get(craft_pcraft_format::MANIFEST).ok_or_else(|| Error::InvalidBundle("manifest not set".into()))?;
let manifest_path = root.join(craft_pcraft_format::MANIFEST);
refuse_symlink(&manifest_path)?;
craft_atomic::atomic_write(&manifest_path, manifest).map_err(io_error)?;
stats.written = stats.written.saturating_add(1);
let mut removed_paths = Vec::new();
// The durable manifest is the commit boundary. Cleanup owns only canonical regular
// object files and previews; leave unsafe, inaccessible or locked leftovers for later.
for kind in [craft_pcraft_format::TILES_DIR, craft_pcraft_format::BLOBS_DIR] {
let dir = root.join(kind);
if !dir.exists() {
if !cleanup_directory(&dir) {
continue;
}
refuse_symlink(&dir)?;
for entry in fs::read_dir(&dir).map_err(io_error)? {
let entry = entry.map_err(io_error)?;
let Ok(entries) = fs::read_dir(&dir) else { continue };
for entry in entries {
let Ok(entry) = entry else { continue };
let name = entry.file_name();
let path = format!("{kind}/{}", name.to_string_lossy());
if is_object_path(&path) && !bundle.entries.contains_key(&path) {
let metadata = fs::symlink_metadata(entry.path()).map_err(io_error)?;
if metadata.file_type().is_symlink() || !metadata.is_file() {
return Err(Error::InvalidBundle(format!("refusing to collect non-file object {path}")));
}
fs::remove_file(entry.path()).map_err(io_error)?;
removed_paths.push(path);
if is_object_path(&path) && !bundle.entries.contains_key(&path) && remove_cleanup_file(&entry.path()) {
stats.removed = stats.removed.saturating_add(1);
}
}
}
for path in [craft_pcraft_format::THUMB, craft_pcraft_format::COMPOSITE, craft_pcraft_format::FULL_COMPOSITE] {
if !bundle.entries.contains_key(path) {
let target = root.join(path);
refuse_symlink(&target)?;
if fs::symlink_metadata(&target).is_ok() {
fs::remove_file(&target).map_err(io_error)?;
}
let _ = remove_cleanup_file(&root.join(path));
}
}
self.verified.retain(|path, _| bundle.entries.contains_key(path) && !removed_paths.contains(path));
self.verified.retain(|path, _| bundle.entries.contains_key(path));
Ok(stats)
}
@@ -510,7 +505,32 @@ impl DirectoryBundleWriter {
}
fn is_object_path(path: &str) -> bool {
path.starts_with("tiles/") || path.starts_with("blobs/")
valid_object_path(path)
}
/// Do not traverse symlinked object/preview directories, including Windows junctions.
fn cleanup_directory(path: &Path) -> bool {
fs::symlink_metadata(path).is_ok_and(|metadata| metadata.is_dir() && !cleanup_link(&metadata))
}
fn cleanup_link(metadata: &fs::Metadata) -> bool {
#[cfg(windows)]
{
use std::os::windows::fs::MetadataExt;
metadata.file_attributes() & 0x400 != 0 // FILE_ATTRIBUTE_REPARSE_POINT includes junctions.
}
#[cfg(not(windows))]
{
metadata.file_type().is_symlink()
}
}
/// Cleanup never follows or removes a directory/link and never reports a post-commit failure.
fn remove_cleanup_file(path: &Path) -> bool {
if path.parent().is_none_or(|parent| !cleanup_directory(parent)) {
return false;
}
fs::symlink_metadata(path).is_ok_and(|metadata| metadata.is_file() && !cleanup_link(&metadata)) && fs::remove_file(path).is_ok()
}
fn file_signature(path: &Path) -> Result<Option<FileSignature>> {
@@ -0,0 +1,344 @@
//! Publication succeeds independently of best-effort cleanup of the previous entry set.
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use craft_pcraft_bundle::{DirectoryBundle, DirectoryBundleWriter, Limits, ObjectKind, ZipBundleWriter};
use craft_pcraft_format::{COMPOSITE, FULL_COMPOSITE, MANIFEST, THUMB};
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
const OLD: &[u8] = br#"{"revision":1}"#;
const NEW: &[u8] = br#"{"revision":2}"#;
struct Temporary(PathBuf);
impl Temporary {
fn new() -> Self {
static NEXT: AtomicU64 = AtomicU64::new(0);
let path = std::env::temp_dir().join(format!("craft-bundle-cleanup-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed)));
fs::create_dir(&path).unwrap();
Self(path)
}
fn bundle(&self) -> PathBuf {
self.0.join("document.pcraft")
}
}
impl Drop for Temporary {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
fn entries(manifest: &[u8]) -> ZipBundleWriter {
let mut bundle = ZipBundleWriter::default();
bundle.set_manifest(manifest.to_vec()).unwrap();
bundle
}
fn assert_manifest(root: &Path, expected: &[u8]) {
assert_eq!(fs::read(root.join(MANIFEST)).unwrap(), expected);
assert_eq!(DirectoryBundle::open(root, Limits::default()).unwrap().manifest().unwrap(), expected);
}
#[test]
fn non_file_stale_object_does_not_fail_the_committed_save() {
let temp = Temporary::new();
let root = temp.bundle();
let mut writer = DirectoryBundleWriter::default();
writer.save_directory(&root, entries(OLD)).unwrap();
let stuck = root.join("tiles").join(format!("{}.zst", "0".repeat(64)));
fs::create_dir_all(stuck.join("inner")).unwrap();
fs::write(stuck.join("inner/keep"), b"not an object file").unwrap();
let stats = writer.save_directory(&root, entries(NEW)).expect("cleanup cannot fail after manifest publication");
assert_manifest(&root, NEW);
assert_eq!(stats.removed, 0);
assert_eq!(fs::read(stuck.join("inner/keep")).unwrap(), b"not an object file");
}
#[test]
fn cleanup_removes_only_canonical_stale_regular_objects_and_reuses_current_objects() {
let temp = Temporary::new();
let root = temp.bundle();
let mut writer = DirectoryBundleWriter::default();
let mut first = entries(OLD);
let retained = first.add_object(ObjectKind::Tile, b"still referenced").unwrap();
let stale = first.add_object(ObjectKind::Blob, b"old blob").unwrap();
for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] {
first.set_preview(preview, b"old preview".to_vec()).unwrap();
}
writer.save_directory(&root, first).unwrap();
let retained_bytes = fs::read(root.join(&retained)).unwrap();
let unrelated = [
"tiles/personal-notes.zst".to_string(),
"blobs/012345.zst".to_string(),
format!("blobs/{}.zst", "A".repeat(64)),
format!("tiles/{}.zst.backup", "b".repeat(64)),
];
for path in &unrelated {
fs::write(root.join(path), b"unrelated").unwrap();
}
let nested = root.join("blobs/not-an-object-folder");
fs::create_dir(&nested).unwrap();
fs::write(nested.join(format!("{}.zst", "c".repeat(64))), b"unrelated nested file").unwrap();
let mut second = entries(NEW);
assert_eq!(second.add_object(ObjectKind::Tile, b"still referenced").unwrap(), retained);
let stats = writer.save_directory(&root, second).unwrap();
assert_manifest(&root, NEW);
assert_eq!(stats.removed, 1);
assert_eq!(stats.reused, 1);
assert_eq!(stats.reused_objects.as_slice(), std::slice::from_ref(&retained));
assert!(stats.written_objects.is_empty());
assert!(!root.join(stale).exists());
assert_eq!(fs::read(root.join(retained)).unwrap(), retained_bytes);
for path in unrelated {
assert_eq!(fs::read(root.join(path)).unwrap(), b"unrelated");
}
assert!(nested.is_dir());
for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] {
assert!(!root.join(preview).exists());
}
}
#[test]
fn stale_preview_directories_are_preserved_after_commit() {
let temp = Temporary::new();
let root = temp.bundle();
let mut writer = DirectoryBundleWriter::default();
writer.save_directory(&root, entries(OLD)).unwrap();
for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] {
fs::create_dir_all(root.join(preview)).unwrap();
fs::write(root.join(preview).join("keep"), b"directory contents").unwrap();
}
let stats = writer.save_directory(&root, entries(NEW)).unwrap();
assert_manifest(&root, NEW);
assert_eq!(stats.removed, 0);
for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] {
assert_eq!(fs::read(root.join(preview).join("keep")).unwrap(), b"directory contents");
}
}
#[test]
fn required_non_file_objects_and_previews_fail_before_manifest_publication() {
let temp = Temporary::new();
let root = temp.bundle();
let mut writer = DirectoryBundleWriter::default();
writer.save_directory(&root, entries(OLD)).unwrap();
let mut changed = entries(NEW);
let object = changed.add_object(ObjectKind::Tile, b"required object").unwrap();
fs::create_dir_all(root.join(&object)).unwrap();
assert!(writer.save_directory(&root, changed).is_err());
assert_manifest(&root, OLD);
for preview in [THUMB, COMPOSITE] {
fs::create_dir_all(root.join(preview)).unwrap();
let mut changed = entries(NEW);
changed.set_preview(preview, b"required preview".to_vec()).unwrap();
assert!(writer.save_directory(&root, changed).is_err());
assert_manifest(&root, OLD);
}
}
#[cfg(windows)]
#[test]
fn locked_stale_object_and_preview_do_not_fail_commit_and_are_collected_on_retry() {
use std::fs::OpenOptions;
use std::os::windows::fs::OpenOptionsExt;
let temp = Temporary::new();
let root = temp.bundle();
let mut writer = DirectoryBundleWriter::default();
let mut first = entries(OLD);
let stale = first.add_object(ObjectKind::Tile, b"locked stale tile").unwrap();
first.set_preview(FULL_COMPOSITE, b"locked preview".to_vec()).unwrap();
writer.save_directory(&root, first).unwrap();
let tile_lock = OpenOptions::new().read(true).share_mode(0).open(root.join(&stale)).unwrap();
let preview_lock = OpenOptions::new().read(true).share_mode(0).open(root.join(FULL_COMPOSITE)).unwrap();
let stats = writer.save_directory(&root, entries(NEW)).unwrap();
assert_manifest(&root, NEW);
assert_eq!(stats.removed, 0);
assert!(root.join(&stale).exists());
assert!(root.join(FULL_COMPOSITE).exists());
drop(tile_lock);
drop(preview_lock);
let retry = writer.save_directory(&root, entries(NEW)).unwrap();
assert_manifest(&root, NEW);
assert_eq!(retry.removed, 1);
assert!(!root.join(stale).exists());
assert!(!root.join(FULL_COMPOSITE).exists());
}
#[cfg(unix)]
#[test]
fn cleanup_preserves_symlinks_and_does_not_traverse_linked_directories() {
use std::os::unix::fs::symlink;
let temp = Temporary::new();
let root = temp.bundle();
let outside = temp.0.join("outside");
fs::create_dir(&outside).unwrap();
let object_name = format!("{}.zst", "d".repeat(64));
fs::write(outside.join(&object_name), b"outside object").unwrap();
fs::write(outside.join("preview.png"), b"outside preview").unwrap();
let mut writer = DirectoryBundleWriter::default();
writer.save_directory(&root, entries(OLD)).unwrap();
symlink(&outside, root.join("blobs")).unwrap();
symlink(&outside, root.join("composite")).unwrap();
symlink(outside.join("preview.png"), root.join(THUMB)).unwrap();
fs::create_dir(root.join("tiles")).unwrap();
let linked_object = root.join("tiles").join(&object_name);
symlink(outside.join(&object_name), &linked_object).unwrap();
let stats = writer.save_directory(&root, entries(NEW)).unwrap();
assert_manifest(&root, NEW);
assert_eq!(stats.removed, 0);
assert_eq!(fs::read(outside.join(object_name)).unwrap(), b"outside object");
assert_eq!(fs::read(outside.join("preview.png")).unwrap(), b"outside preview");
for link in [root.join("blobs"), root.join("composite"), root.join(THUMB), linked_object] {
assert!(fs::symlink_metadata(link).unwrap().file_type().is_symlink());
}
}
#[cfg(unix)]
#[test]
fn required_symlink_objects_and_previews_still_fail_before_commit() {
use std::os::unix::fs::symlink;
let temp = Temporary::new();
let root = temp.bundle();
let outside = temp.0.join("outside-file");
fs::write(&outside, b"unchanged outside bytes").unwrap();
let mut writer = DirectoryBundleWriter::default();
writer.save_directory(&root, entries(OLD)).unwrap();
let mut changed = entries(NEW);
let object = changed.add_object(ObjectKind::Tile, b"required object").unwrap();
fs::create_dir(root.join("tiles")).unwrap();
symlink(&outside, root.join(object)).unwrap();
assert!(writer.save_directory(&root, changed).is_err());
assert_manifest(&root, OLD);
symlink(&outside, root.join(THUMB)).unwrap();
let mut changed = entries(NEW);
changed.set_preview(THUMB, b"new preview".to_vec()).unwrap();
assert!(writer.save_directory(&root, changed).is_err());
assert_manifest(&root, OLD);
assert_eq!(fs::read(outside).unwrap(), b"unchanged outside bytes");
}
#[cfg(windows)]
#[test]
fn cleanup_does_not_traverse_windows_directory_junctions() {
use std::os::windows::process::CommandExt;
struct Junction(PathBuf);
impl Drop for Junction {
fn drop(&mut self) {
let _ = fs::remove_dir(&self.0);
}
}
fn junction(link: PathBuf, target: &Path) -> Junction {
// Fixture-owned absolute paths only; junctions require no symlink privilege.
assert!(link.is_absolute() && target.is_absolute());
let output = std::process::Command::new("cmd")
.args(["/C", "mklink", "/J"])
.arg(&link)
.arg(target)
.creation_flags(0x0800_0000) // CREATE_NO_WINDOW
.output()
.unwrap();
assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr));
Junction(link)
}
let temp = Temporary::new();
let root = temp.bundle();
let outside = temp.0.join("outside");
fs::create_dir(&outside).unwrap();
let object_name = format!("{}.zst", "e".repeat(64));
fs::write(outside.join(&object_name), b"outside object").unwrap();
fs::write(outside.join("preview.png"), b"outside preview").unwrap();
fs::write(outside.join("full.png"), b"outside full preview").unwrap();
let mut writer = DirectoryBundleWriter::default();
writer.save_directory(&root, entries(OLD)).unwrap();
let _objects = junction(root.join("blobs"), &outside);
let _previews = junction(root.join("composite"), &outside);
let stats = writer.save_directory(&root, entries(NEW)).unwrap();
assert_manifest(&root, NEW);
assert_eq!(stats.removed, 0);
assert_eq!(fs::read(outside.join(object_name)).unwrap(), b"outside object");
assert_eq!(fs::read(outside.join("preview.png")).unwrap(), b"outside preview");
assert_eq!(fs::read(outside.join("full.png")).unwrap(), b"outside full preview");
assert!(root.join("blobs").exists());
assert!(root.join("composite").exists());
}
#[cfg(any(unix, windows))]
#[test]
fn manifest_symlink_is_refused_before_writing_any_objects_or_previews() {
let temp = Temporary::new();
let root = temp.bundle();
let mut writer = DirectoryBundleWriter::default();
writer.save_directory(&root, entries(OLD)).unwrap();
assert_manifest(&root, OLD); // Ordinary publication remains supported.
let external = temp.0.join("external-manifest.json");
fs::write(&external, b"unchanged external manifest").unwrap();
fs::remove_file(root.join(MANIFEST)).unwrap();
#[cfg(unix)]
std::os::unix::fs::symlink(&external, root.join(MANIFEST)).unwrap();
#[cfg(windows)]
if let Err(error) = std::os::windows::fs::symlink_file(&external, root.join(MANIFEST)) {
if error.raw_os_error() == Some(1314) || error.kind() == std::io::ErrorKind::Unsupported {
eprintln!("SKIP manifest symlink fixture: {error}");
return;
}
panic!("create manifest symlink fixture: {error}");
}
let mut changed = entries(NEW);
changed.add_object(ObjectKind::Tile, b"must not write tile").unwrap();
changed.add_object(ObjectKind::Blob, b"must not write blob").unwrap();
changed.set_preview(THUMB, b"must not write thumbnail".to_vec()).unwrap();
changed.set_preview(FULL_COMPOSITE, b"must not write composite".to_vec()).unwrap();
assert!(writer.save_directory(&root, changed).is_err());
assert_eq!(fs::read(&external).unwrap(), b"unchanged external manifest");
assert!(fs::symlink_metadata(root.join(MANIFEST)).unwrap().file_type().is_symlink());
for unwritten in ["tiles", "blobs", THUMB, "composite"] {
assert!(!root.join(unwritten).exists(), "manifest refusal must precede {unwritten} publication");
}
}
#[cfg(unix)]
#[test]
fn chosen_root_symlink_cleanup_works_with_search_only_root_and_composite_permissions() {
use std::os::unix::fs::{PermissionsExt, symlink};
struct Restore(PathBuf, fs::Permissions);
impl Restore {
fn restrict(path: &Path) -> Self {
let old = fs::metadata(path).unwrap().permissions();
fs::set_permissions(path, fs::Permissions::from_mode(0o300)).unwrap();
Self(path.to_path_buf(), old)
}
}
impl Drop for Restore {
fn drop(&mut self) {
let _ = fs::set_permissions(&self.0, self.1.clone());
}
}
let temp = Temporary::new();
let root = temp.bundle();
let mut writer = DirectoryBundleWriter::default();
let mut first = entries(OLD);
let stale = first.add_object(ObjectKind::Blob, b"stale blob").unwrap();
first.set_preview(THUMB, b"stale thumbnail".to_vec()).unwrap();
first.set_preview(COMPOSITE, b"stale preview".to_vec()).unwrap();
writer.save_directory(&root, first).unwrap();
let chosen = temp.0.join("chosen.pcraft");
symlink(&root, &chosen).unwrap();
let composite_mode = Restore::restrict(&root.join("composite"));
let root_mode = Restore::restrict(&root);
let result = writer.save_directory(&chosen, entries(NEW));
drop(root_mode);
drop(composite_mode);
let stats = result.expect("chosen root resolves before cleanup; traversal does not require directory read permission");
assert_manifest(&root, NEW);
assert_eq!(stats.removed, 1);
assert!(!root.join(stale).exists());
assert!(!root.join(THUMB).exists());
assert!(!root.join(COMPOSITE).exists());
assert!(fs::symlink_metadata(chosen).unwrap().file_type().is_symlink());
}
+3 -2
View File
@@ -35,8 +35,9 @@ workspace = true
# ticket (winprint.rs#12, storytold/pdfcraft#3823), and reads whitespace-padded integers in
# PrintCapabilities (winprint.rs#3, storytold/pdfcraft#2430). Pinned to an immutable revision;
# review before bumping. The settings chosen in a driver's own dialog become a ticket through
# `PrintTicket::from_dev_mode`.
winprint = { git = "https://github.com/storytold/winprint.rs", rev = "72951d7455ebcab569fbd9386c513d3d9f53fb02", default-features = false }
# `PrintTicket::from_dev_mode` (winprint.rs#24, pinned at its merge commit on winprint.rs's
# `main`).
winprint = { git = "https://github.com/storytold/winprint.rs", rev = "8ae097ef941b432b0e7671b1ec3a506800e65e57", default-features = false }
# GetDefaultPrinter (MIT).
winsafe = { version = "0.0.29", default-features = false, features = ["winspool"] }
# The job file's private folder.
+20
View File
@@ -0,0 +1,20 @@
[package]
name = "craft-proto"
description = "Mail protocols without sockets: IMAP4rev1 and SMTP clients over any Read + Write stream"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
base64 = "0.23"
zeroize = "1"
# Internationalization: error messages and user-facing text.
log = { workspace = true }
[dev-dependencies]
proptest = { workspace = true }
+246
View File
@@ -0,0 +1,246 @@
// SPDX-FileCopyrightText: Craft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
//! Error message handling for protocol errors.
//!
//! Provides the `said!` macro and related types for creating user-facing error messages.
pub use crate::said;
use std::fmt;
/// A value that can be converted to an argument for error messages.
pub trait ToArg {
fn to_arg(&self) -> Arg;
}
impl<T: fmt::Display> ToArg for T {
fn to_arg(&self) -> Arg {
Arg::Plain(format!("{}", self))
}
}
/// An argument for error messages.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Arg {
Plain(String),
}
impl fmt::Display for Arg {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Arg::Plain(s) => f.write_str(s),
}
}
}
impl Arg {
/// Create an argument from a string.
pub fn said(t: Text) -> Self {
Arg::Plain(format!("{}", t))
}
}
/// A text with values: `said!("no folder named {name}", name)`, or
/// `said!("{name} is larger than {mb} MB", name, mb = MAX >> 20)`.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Text {
template: String,
}
impl Text {
/// Preserve text supplied by a server or host without interpreting it.
pub fn verbatim(text: String) -> Self {
Self { template: text }
}
/// Create a new text with the given template and arguments.
pub fn new(template: &str, args: Vec<(&str, Arg)>) -> Self {
// Substitute only placeholders in the template, never text inserted by a server.
let mut filled = String::new();
let mut rest = template;
while let Some(open) = rest.find('{') {
filled.push_str(rest.get(..open).unwrap_or_default());
let after = rest.get(open.saturating_add(1)..).unwrap_or_default();
let Some(close) = after.find('}') else {
filled.push_str(rest.get(open..).unwrap_or_default());
rest = "";
break;
};
let key = after.get(..close).unwrap_or_default();
if let Some((_, arg)) = args.iter().find(|(name, _)| *name == key) {
filled.push_str(&arg.to_string());
} else {
filled.push('{');
filled.push_str(key);
filled.push('}');
}
rest = after.get(close.saturating_add(1)..).unwrap_or_default();
}
filled.push_str(rest);
Text { template: filled }
}
}
impl fmt::Display for Text {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.template)
}
}
impl From<&str> for Text {
fn from(s: &str) -> Self {
Text { template: s.to_string() }
}
}
impl From<String> for Text {
fn from(s: String) -> Self {
Self::verbatim(s)
}
}
/// The `said!` macro for creating error messages.
#[macro_export]
macro_rules! said {
(@arg $k:ident) => { $crate::i18n::ToArg::to_arg(&$k) };
(@arg $k:ident = $v:expr) => { $crate::i18n::ToArg::to_arg(&$v) };
($t:literal $(, $k:ident $(= $v:expr)?)* $(,)?) => {
$crate::i18n::Text::new($t, vec![$((stringify!($k), $crate::said!(@arg $k $(= $v)?))),*])
};
}
/// Trace only recognized protocol structure, never arbitrary reply text or command arguments.
/// Uppercase text can be private too; capitalization must not determine what is safe to log.
pub fn for_trace(line: &str) -> String {
let line = line.trim_end();
let mut words = line.split_ascii_whitespace();
let first = words.next().unwrap_or_default();
let known = |word: &str| {
matches!(
word,
"OK" | "NO"
| "BAD"
| "BYE"
| "PREAUTH"
| "CAPABILITY"
| "LOGIN"
| "LOGOUT"
| "AUTHENTICATE"
| "SELECT"
| "EXAMINE"
| "CREATE"
| "DELETE"
| "RENAME"
| "SUBSCRIBE"
| "UNSUBSCRIBE"
| "LIST"
| "LSUB"
| "STATUS"
| "APPEND"
| "CHECK"
| "CLOSE"
| "EXPUNGE"
| "SEARCH"
| "FETCH"
| "STORE"
| "COPY"
| "MOVE"
| "UID"
| "IDLE"
| "DONE"
| "NOOP"
| "EHLO"
| "HELO"
| "STARTTLS"
| "AUTH"
| "MAIL"
| "RCPT"
| "DATA"
| "RSET"
| "QUIT"
| "EXISTS"
| "RECENT"
| "FLAGS"
)
};
let smtp_code = first.get(..3).is_some_and(|code| code.bytes().all(|b| b.is_ascii_digit()))
&& first.as_bytes().first().is_some_and(|b| matches!(b, b'1'..=b'5'));
let tag = first.strip_prefix('A').is_some_and(|digits| !digits.is_empty() && digits.len() <= 16 && digits.bytes().all(|b| b.is_ascii_digit()));
let end = if smtp_code && (first.len() == 3 || first.get(3..4) == Some("-")) {
if first.len() > 3 { 4 } else { 3 }
} else if known(first) {
first.len()
} else if first == "*" || tag {
let second = words.next().unwrap_or_default();
let last = if first == "*" && !second.is_empty() && second.bytes().all(|b| b.is_ascii_digit()) {
words.next().filter(|word| known(word))
} else {
known(second).then_some(second)
};
last.and_then(|word| line.find(word)?.checked_add(word.len())).unwrap_or(first.len())
} else if first == "+" {
1
} else {
0
};
let kept = line.get(..end).unwrap_or_default();
let left = line.get(end..).unwrap_or_default().trim_start().len();
match (kept.is_empty(), left) {
(_, 0) => kept.to_owned(),
(true, _) => format!("({left} bytes)"),
(false, _) => format!("{kept} … ({left} bytes)"),
}
}
/// Server text made fit for an error message: one line, no control characters, not too long.
pub fn tidy(text: &str) -> String {
const MAX_CHARS: usize = 300;
let mut out = String::new();
let mut space = false;
for (n, c) in text.trim().chars().enumerate() {
if n >= MAX_CHARS {
out.push('…');
break;
}
if c.is_control() || c.is_whitespace() {
space = !out.is_empty();
} else {
if space {
out.push(' ');
space = false;
}
out.push(c);
}
}
out
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[test]
fn inserted_arguments_are_not_interpreted() {
let msg = Text::new("{a} {b}", vec![("a", Arg::Plain("{b}".into())), ("b", Arg::Plain("secret".into()))]);
assert_eq!(msg.to_string(), "{b} secret");
}
#[test]
fn simple_message() {
let msg = said!("the server closed the connection");
assert_eq!(msg.to_string(), "the server closed the connection");
}
#[test]
fn message_with_args() {
let msg = said!("the server sent {n} bytes", n = 123);
assert_eq!(msg.to_string(), "the server sent 123 bytes");
}
#[test]
fn tidy_removes_control_chars() {
assert_eq!(tidy("hello\x01world"), "hello world");
assert_eq!(tidy("hello\nworld"), "hello world");
}
}
+230
View File
@@ -0,0 +1,230 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
//! Gmail's labels of a message (`X-GM-LABELS`, Google's "Gmail IMAP extensions", #1153).
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use proptest::prelude::*;
use super::*;
use crate::script::Script;
fn labels(server: &str) -> Vec<(u32, Option<Vec<String>>)> {
let mut s = Script::new(server);
let got = Imap::resume(&mut s).uid_fetch_flags_gmail("1:*").unwrap_or_default();
got.into_iter().map(|f| (f.uid, f.gmail_labels)).collect()
}
fn owned(labels: &[&str]) -> Option<Vec<String>> {
Some(labels.iter().map(|l| l.to_string()).collect())
}
#[test]
fn labels_come_with_the_flags() {
let mut server = String::new();
// Atoms, quoted strings, Gmail's own with their backslash, an empty list.
server += "* 1 FETCH (UID 1 FLAGS (\\Seen) X-GM-MSGID 11 X-GM-LABELS (\\Inbox \\Sent Important \"Muy Importante\"))\r\n";
server += "* 2 FETCH (X-GM-LABELS (foo) UID 2 FLAGS ())\r\n";
server += "* 3 FETCH (UID 3 FLAGS () X-GM-LABELS ())\r\n";
// A literal, and modified UTF-7 as mailbox names are written.
server += "* 4 FETCH (UID 4 FLAGS () X-GM-LABELS ({12}\r\nWork/Clients \"Caf&AOk-\" &AOk-t&AOk-))\r\n";
// UTF-8 as it stands, and an ampersand that isn't the start of a shifted run.
server += "* 5 FETCH (UID 5 FLAGS () X-GM-LABELS (\"Été\" \"Tom & Jerry\" \\Drafts))\r\n";
// A message the server gave none for.
server += "* 6 FETCH (UID 6 FLAGS ())\r\n";
server += "A0001 OK done\r\n";
let mut s = Script::new(server);
let got = Imap::resume(&mut s).uid_fetch_flags_gmail("1:*").unwrap();
assert_eq!(s.sent(), "A0001 UID FETCH 1:* (UID FLAGS X-GM-MSGID X-GM-LABELS)\r\n");
let got: Vec<(u32, Option<Vec<String>>)> = got.into_iter().map(|f| (f.uid, f.gmail_labels)).collect();
assert_eq!(
got,
[
(1, owned(&["\\Inbox", "\\Sent", "Important", "Muy Importante"])),
(2, owned(&["foo"])),
(3, owned(&[])),
(4, owned(&["Work/Clients", "Café", "été"])),
(5, owned(&["Été", "Tom & Jerry", "\\Drafts"])),
(6, None),
]
);
}
#[test]
fn the_later_word_counts_and_other_servers_are_not_asked() {
let got = labels(
"* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (a))\r\n* 1 FETCH (UID 1 X-GM-LABELS (b c))\r\n* 1 FETCH (UID 1 FLAGS ())\r\nA0001 OK done\r\n",
);
assert_eq!(got, [(1, owned(&["b", "c"]))]);
// Without asking for them, labels the server gives anyway are not taken.
let mut s = Script::new("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (a))\r\nA0001 OK done\r\n");
let plain = Imap::resume(&mut s).uid_fetch_flags("1:*").unwrap();
assert_eq!(plain.first().and_then(|f| f.gmail_labels.clone()), None);
assert_eq!(s.sent(), "A0001 UID FETCH 1:* (UID FLAGS)\r\n");
}
#[test]
fn hostile_label_lists_are_kept_small() {
// A huge list: only so many are kept.
let many: String = (0..1000).map(|n| format!(" L{n}")).collect();
let got = labels(&format!("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS ({many}))\r\nA0001 OK done\r\n"));
assert_eq!(got.first().and_then(|(_, l)| l.as_ref()).map(Vec::len), Some(MAX_GMAIL_LABELS));
// A long label is cut, at a character boundary.
let long = "é".repeat(400);
let got = labels(&format!("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (\"{long}\"))\r\nA0001 OK done\r\n"));
let label = got.first().and_then(|(_, l)| l.as_ref()).and_then(|l| l.first()).cloned().unwrap_or_default();
assert!(label.len() <= MAX_GMAIL_LABEL_LEN && !label.is_empty() && label.chars().all(|c| c == 'é'));
// A NUL in a literal is carried as text for the model to clean; a list inside the list is
// skipped; a list that isn't closed is read as far as it goes.
let got = labels("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS ({3}\r\na\0b (nested list) ok))\r\nA0001 OK done\r\n");
assert_eq!(got, [(1, owned(&["a\0b", "ok"]))]);
let got = labels("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (a (b\r\nA0001 OK done\r\n");
assert!(got.len() <= 1);
// Something other than a list is no labels.
let got = labels("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS NIL)\r\nA0001 OK done\r\n");
assert_eq!(got, [(1, None)]);
}
#[test]
fn a_fetch_keeps_no_more_label_bytes_than_its_budget() {
// Labels as literals, which the cap on a command's text doesn't count: each message with as
// many as a response may carry, of 256 bytes each. Past 32 MiB the rest come without.
let label = "x".repeat(MAX_GMAIL_LABEL_LEN);
let list: String = (0..MAX_LITERALS).map(|_| format!(" {{{}}}\r\n{label}", label.len())).collect();
let count = MAX_GMAIL_LABEL_BYTES / (MAX_GMAIL_LABEL_LEN * MAX_LITERALS) + 4;
let mut server = String::new();
for uid in 1..=count {
server += &format!("* {uid} FETCH (UID {uid} FLAGS () X-GM-LABELS ({list}))\r\n");
}
server += "A0001 OK done\r\n";
let mut s = Script::new(server);
let mut imap = Imap::resume(&mut s);
imap.selected = Selected { exists: u32::try_from(count).unwrap_or(u32::MAX), uid_validity: 1, uid_next: 0 };
let got = imap.uid_fetch_flags_gmail("1:*").unwrap();
assert_eq!(got.len(), count);
let kept: usize = got.iter().filter_map(|f| f.gmail_labels.as_ref()).flatten().map(String::len).sum();
assert!(kept <= MAX_GMAIL_LABEL_BYTES);
assert!(got.last().is_some_and(|f| f.gmail_labels.is_none()));
}
proptest! {
#[test]
fn arbitrary_label_lists_never_panic(inside in proptest::collection::vec(any::<u8>(), 0..400)) {
let mut server = b"* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (".to_vec();
server.extend(inside.iter().filter(|b| **b != b'\r' && **b != b'\n'));
server.extend_from_slice(b"))\r\nA0001 OK done\r\n");
let mut s = Script::new(server);
if let Ok(got) = Imap::resume(&mut s).uid_fetch_flags_gmail("1:*") {
for labels in got.iter().filter_map(|f| f.gmail_labels.as_ref()) {
prop_assert!(labels.len() <= MAX_GMAIL_LABELS);
}
}
}
#[test]
fn arbitrary_server_bytes_never_panic_in_a_gmail_fetch(server in proptest::collection::vec(any::<u8>(), 0..600)) {
let mut s = Script::new(server);
let _ = Imap::resume(&mut s).uid_fetch_flags_gmail("1:*");
}
}
#[test]
fn a_folder_other_than_the_inbox_asks_for_labels_alone() {
let mut s = Script::new("* 1 FETCH (UID 1 FLAGS () X-GM-MSGID 5 X-GM-LABELS (a))\r\nA0001 OK done\r\n");
let got = Imap::resume(&mut s).uid_fetch_flags_labels("1:*").unwrap();
assert_eq!(s.sent(), "A0001 UID FETCH 1:* (UID FLAGS X-GM-LABELS)\r\n");
// An id it wasn't asked for is not taken.
assert_eq!(got.into_iter().map(|f| (f.uid, f.gmail_id, f.gmail_labels)).collect::<Vec<_>>(), [(1, None, owned(&["a"]))]);
}
#[test]
fn nil_in_a_list_is_no_label() {
assert_eq!(labels("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (NIL))\r\nA0001 OK done\r\n"), [(1, owned(&[]))]);
// A label called so comes quoted, and is one.
assert_eq!(labels("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (nil \"NIL\"))\r\nA0001 OK done\r\n"), [(1, owned(&["NIL"]))]);
}
#[test]
fn tiny_labels_count_against_the_budget_too() {
// Empty labels, as many as a message keeps: each costs its overhead, so the budget still ends.
let list: String = (0..MAX_GMAIL_LABELS).map(|_| " \"\"").collect();
let per_message = MAX_GMAIL_LABELS * LABEL_OVERHEAD;
let count = MAX_GMAIL_LABEL_BYTES / per_message + 4;
let mut server = String::new();
for uid in 1..=count {
server += &format!("* {uid} FETCH (UID {uid} FLAGS () X-GM-LABELS ({list}))\r\n");
}
server += "A0001 OK done\r\n";
let mut s = Script::new(server);
let mut imap = Imap::resume(&mut s);
imap.selected = Selected { exists: u32::try_from(count).unwrap_or(u32::MAX), uid_validity: 1, uid_next: 0 };
let got = imap.uid_fetch_flags_labels("1:*").unwrap();
assert_eq!(got.len(), count);
let kept = got.iter().filter(|f| f.gmail_labels.is_some()).count();
assert!(kept * per_message <= MAX_GMAIL_LABEL_BYTES && kept < count);
}
#[test]
fn a_big_folders_listing_is_not_cut_short_for_its_size() {
// 300,000 messages with labels: more text than one command may otherwise bring (32 MiB).
let count: u32 = 300_000;
let mut server = String::with_capacity(count as usize * 200);
for uid in 1..=count {
server += &format!(
"* {uid} FETCH (FLAGS (\\Seen) UID {uid} X-GM-LABELS (\\Inbox \\Important \"Work/Clients and partners\" \"Receipts 2026\" \"Travel/Japan\"))\r\n"
);
}
assert!(server.len() > MAX_COMMAND_TEXT);
server += "A0001 OK done\r\n";
let mut s = Script::new(server);
let mut imap = Imap::resume(&mut s);
imap.selected = Selected { exists: count, uid_validity: 1, uid_next: count + 1 };
assert_eq!(imap.uid_fetch_flags_labels("1:*").map(|got| got.len()), Ok(count as usize));
// The room is the listing's alone: the next command has the usual cap again.
assert_eq!(imap.text_cap, MAX_COMMAND_TEXT);
}
#[test]
fn a_mailbox_claimed_huge_cant_fill_memory_with_flags() {
// Any server (no Gmail here): ten million messages claimed, then message after message with
// every flag it may have. The text it may send is capped, and so are the flags kept: the
// listing fails, and never hands back messages that seem to have lost their flags.
let flags: Vec<String> = (0..MAX_FLAGS).map(|n| format!("f{n}")).collect();
let flags = flags.join(" ");
let mut server = String::new();
let mut uid = 0u32;
while server.len() < MAX_FETCH_TEXT / 4 {
uid += 1;
server += &format!("* {uid} FETCH (UID {uid} FLAGS ({flags}))\r\n");
}
server += "A0001 OK done\r\n";
let mut s = Script::new(server);
let mut imap = Imap::resume(&mut s);
imap.selected = Selected { exists: 10_000_000, uid_validity: 1, uid_next: 0 };
let got = imap.uid_fetch_flags("1:*");
assert!(matches!(&got, Err(ProtoError::Protocol(m)) if format!("{m:?}").contains("more flags")), "{:?}", got.as_ref().map(Vec::len));
assert_eq!(imap.text_cap, MAX_COMMAND_TEXT);
// And what it may send at all stays within the ceiling, however many messages are claimed.
let mut s = Script::new(format!("* OK {}\r\n", "x".repeat(60_000)).repeat(MAX_FETCH_TEXT / 60_000 + 8));
let mut imap = Imap::resume(&mut s);
imap.selected = Selected { exists: u32::MAX, uid_validity: 1, uid_next: 0 };
assert!(matches!(imap.uid_fetch_flags("1:*"), Err(ProtoError::Protocol(_))));
}
#[test]
fn an_honest_big_mailboxs_flags_fit_the_budget() {
// A million messages with three flags each.
const { assert!(1_000_000 * (ENTRY_OVERHEAD + 3 * (FLAG_OVERHEAD + 8)) <= MAX_FLAG_BYTES) };
}
#[test]
fn a_small_folders_listing_still_has_its_cap() {
// One message, and more than 32 MiB of other lines before it: refused as before.
let junk = format!("* OK {}\r\n", "x".repeat(60_000));
let mut server = junk.repeat(MAX_COMMAND_TEXT / 60_000 + 8);
server += "* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (a))\r\nA0001 OK done\r\n";
let mut s = Script::new(server);
let mut imap = Imap::resume(&mut s);
imap.selected = Selected { exists: 1, uid_validity: 1, uid_next: 2 };
assert!(matches!(imap.uid_fetch_flags_labels("1:*"), Err(ProtoError::Protocol(_))));
assert_eq!(imap.text_cap, MAX_COMMAND_TEXT);
}
+79
View File
@@ -0,0 +1,79 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
//! Gmail's IMAP extensions (#911): `X-GM-MSGID` and `X-GM-RAW`.
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::script::Script;
#[test]
fn gmail_ids_come_with_the_flags() {
let mut server = String::new();
server += "* 1 FETCH (UID 4 FLAGS (\\Seen) X-GM-MSGID 1278455344230334865)\r\n";
server += "* 2 FETCH (X-GM-MSGID 18446744073709551615 UID 7 FLAGS ())\r\n";
// An id that is no number, or 0, is none; a later word about a message without one keeps it.
server += "* 3 FETCH (UID 9 FLAGS () X-GM-MSGID 12ab)\r\n";
server += "* 4 FETCH (UID 10 FLAGS () X-GM-MSGID 0)\r\n";
server += "* 2 FETCH (UID 7 FLAGS (\\Flagged))\r\n";
server += "A0001 OK done\r\n";
let mut s = Script::new(server);
let got = Imap::resume(&mut s).uid_fetch_flags_gmail("1:*").unwrap();
let ids: Vec<(u32, Option<u64>)> = got.iter().map(|f| (f.uid, f.gmail_id)).collect();
assert_eq!(ids, [(4, Some(1_278_455_344_230_334_865)), (7, Some(u64::MAX)), (9, None), (10, None)]);
assert_eq!(got.get(1).map(|f| f.flags.clone()), Some(vec!["\\Flagged".to_string()]));
assert_eq!(s.sent(), "A0001 UID FETCH 1:* (UID FLAGS X-GM-MSGID X-GM-LABELS)\r\n");
// Without asking for them, an id the server gives anyway is not taken.
let mut s = Script::new("* 1 FETCH (UID 4 FLAGS () X-GM-MSGID 5)\r\nA0001 OK done\r\n");
assert_eq!(Imap::resume(&mut s).uid_fetch_flags("1:*").unwrap().first().and_then(|f| f.gmail_id), None);
}
#[test]
fn gmail_search_is_quoted_and_kept_to_its_set() {
let mut s = Script::new("* SEARCH 9 4 0 x 4 12 600\r\nA0001 OK done\r\nA0002 OK done\r\n");
let mut imap = Imap::resume(&mut s);
// In UID order, once each, and only what the set names.
assert_eq!(imap.uid_search_gmail("4:12", "category:social"), Ok(vec![4, 9, 12]));
// Quotes are quoted; CR, LF and NUL never go out.
assert_eq!(imap.uid_search_gmail("1:*", "category:\"x\"\r\nA9 DELETE INBOX\0"), Ok(vec![]));
assert_eq!(
s.sent(),
"A0001 UID SEARCH UID 4:12 UNDELETED X-GM-RAW \"category:social\"\r\n\
A0002 UID SEARCH UID 1:* UNDELETED X-GM-RAW \"category:\\\"x\\\"A9 DELETE INBOX\"\r\n"
);
// A refusal is said.
let mut s = Script::new("A0001 BAD no such search\r\n");
assert!(matches!(Imap::resume(&mut s).uid_search_gmail("1:*", "category:forums"), Err(ProtoError::Rejected(_))));
// What can't be sent is refused unsent.
for (set, query) in [("1:*", ""), ("1:*", "\r\n"), ("1:*", "kategorie:äöü"), ("1 2", "category:social"), ("", "category:social")] {
let mut s = Script::new("A0001 OK done\r\n");
assert!(matches!(Imap::resume(&mut s).uid_search_gmail(set, query), Err(ProtoError::Protocol(_))), "{set:?} {query:?}");
assert_eq!(s.sent(), "", "{set:?} {query:?}");
}
}
#[test]
fn gmail_search_takes_no_more_than_its_cap() {
let many: String = (1..=6000).map(|n| format!(" {n}")).collect();
let mut s = Script::new(format!("* SEARCH{many}\r\nA0001 OK done\r\n"));
assert_eq!(Imap::resume(&mut s).uid_search_gmail("1:*", "category:promotions").map(|uids| uids.len()), Ok(MAX_GMAIL_FOUND));
// As many as the cap, at the longest a UID is written, fit in one line.
let longest: String = (0..MAX_GMAIL_FOUND).map(|_| " 4294967295").collect();
assert!(format!("* SEARCH{longest}").len() < crate::MAX_LINE);
}
#[test]
fn a_gmail_answer_too_long_is_refused_and_the_session_goes_on() {
let long: String = (0..20_000).map(|n| format!(" {}", 1_000_000_000u32 + n)).collect();
assert!(long.len() > crate::MAX_LINE);
let server = format!("* SEARCH{long}\r\n* SEARCH 3\r\nA0001 OK done\r\n* SEARCH 5\r\nA0002 OK done\r\n");
let mut s = Script::new(server);
let mut imap = Imap::resume(&mut s);
assert!(matches!(imap.uid_search_gmail("1:*", "category:social"), Err(ProtoError::Rejected(_))));
// The next command is answered as if nothing had happened.
assert_eq!(imap.uid_search_gmail("1:*", "category:social"), Ok(vec![5]));
// Elsewhere a line that long still ends the session, and a tagged one is never read past.
let mut s = Script::new(format!("* SEARCH{long}\r\nA0001 OK done\r\n"));
assert!(matches!(Imap::resume(&mut s).uid_search_header("Message-ID", "x"), Err(ProtoError::Protocol(_))));
let mut s = Script::new(format!("A0001 OK{long}\r\n"));
assert!(matches!(Imap::resume(&mut s).uid_search_gmail("1:*", "category:social"), Err(ProtoError::Protocol(_))));
}
+185
View File
@@ -0,0 +1,185 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
//! IDLE (RFC 2177): the server says when a mailbox changes, instead of being asked. The session
//! is in one of two states, and this module only moves it between them: a mailbox is selected
//! (EXAMINE), [`Imap::idle_begin`] puts the server to waiting, [`Imap::idle_poll`] hears what it
//! says (a line a second at most, so that the caller can look at the clock and at whether it was
//! told to stop), and [`Imap::idle_end`] says DONE and reads on to the command's end.
//!
//! The server's word is bounded as everywhere: a line over [`crate::MAX_LINE`] and a line that
//! announces a literal end the session, and the caller counts what it hears.
use std::io::{Read, Write};
use super::Imap;
use super::parse;
use crate::i18n::{Arg, said};
use crate::{ProtoError, for_trace};
/// Most lines [`Imap::idle_end`] reads on the way to the command's end.
const MAX_ENDING: usize = 256;
/// Most quiet reads [`Imap::idle_end`] waits through for one line.
const MAX_QUIET: usize = 30;
/// What the server said while it was waiting.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum IdleEvent {
/// `* n EXISTS`: the mailbox now has `n` messages.
Exists(u32),
/// `* n EXPUNGE`: the message with that sequence number is gone.
Expunge(u32),
/// `* n RECENT`: `n` messages are new since the mailbox was opened.
Recent(u32),
/// Anything else the server said unasked (a flag change, a status line): not news of mail.
Other,
/// The server ended the command itself (`tag OK`): IDLE has to be begun again.
Ended,
}
/// From the text after `* `: `3 EXISTS`.
fn event_from(rest: &[u8], secrets: &crate::Secrets) -> Result<IdleEvent, ProtoError> {
let text = String::from_utf8_lossy(rest);
let mut words = text.split_whitespace();
let (first, second) = (words.next().unwrap_or(""), words.next().unwrap_or(""));
let count = first.parse::<u32>().ok();
Ok(match (count, second.to_ascii_uppercase().as_str()) {
(Some(n), "EXISTS") => IdleEvent::Exists(n),
(Some(n), "EXPUNGE") => IdleEvent::Expunge(n),
(Some(n), "RECENT") => IdleEvent::Recent(n),
_ if first.eq_ignore_ascii_case("BYE") => {
return Err(ProtoError::Io(said!("the server ended the session: {reason}", reason = crate::tidy(&secrets.hide(&text)))));
}
_ => IdleEvent::Other,
})
}
impl<S: Read + Write> Imap<S> {
/// Send IDLE and wait for the server's go-ahead. The session has to have a mailbox selected;
/// a server that refuses (NO, BAD) is an error. While idle, the only commands that may follow
/// are [`Imap::idle_poll`] and [`Imap::idle_end`].
pub fn idle_begin(&mut self) -> Result<(), ProtoError> {
if self.idle_tag.is_some() {
return Err(ProtoError::Protocol(said!("the server is already idling")));
}
self.idle_pending.clear();
let tag = self.tag();
self.text_seen = 0;
log::trace!("C: {tag} IDLE");
self.wire.write(format!("{tag} IDLE\r\n").as_bytes())?;
self.wire.flush()?;
self.idle_tag = Some(tag.clone());
// Untagged lines may come before the `+`; the tagged answer is a refusal.
for _ in 0..MAX_ENDING {
match self.read_response()? {
super::Response::GoAhead => return Ok(()),
super::Response::Status { tag: Some(answered), line } => {
self.idle_tag = None;
if answered != tag {
return Err(ProtoError::Protocol(said!("the server answered a command that wasn't sent")));
}
return Err(ProtoError::Rejected(self.refusal(&line)));
}
super::Response::Status { tag: None, line } => {
if line.status == parse::Status::Bye {
self.idle_tag = None;
return Err(ProtoError::Io(said!("the server ended the session: {reason}", reason = Arg::said(self.refusal(&line)))));
}
self.note_status(&line);
}
super::Response::Data(values) => {
self.note_data(&values);
let event = match (values.first().and_then(parse::Value::number), values.get(1).and_then(parse::Value::atom)) {
(Some(n), Some(word)) if word.eq_ignore_ascii_case("EXISTS") => IdleEvent::Exists(n),
(Some(n), Some(word)) if word.eq_ignore_ascii_case("EXPUNGE") => IdleEvent::Expunge(n),
(Some(n), Some(word)) if word.eq_ignore_ascii_case("RECENT") => IdleEvent::Recent(n),
_ => IdleEvent::Other,
};
self.idle_pending.push_back(event);
}
}
}
self.idle_tag = None;
Err(ProtoError::Protocol(said!("the server sent too many responses to one command")))
}
/// The next thing the server says while idle, or `None` after one read if the line is
/// incomplete or no bytes arrived (see [`crate::MAX_LINE`] for the bound on a line).
pub fn idle_poll(&mut self) -> Result<Option<IdleEvent>, ProtoError> {
let Some(tag) = self.idle_tag.clone() else {
return Err(ProtoError::Protocol(said!("the server isn't idling")));
};
if let Some(event) = self.idle_pending.pop_front() {
return Ok(Some(event));
}
let Some(line) = self.wire.poll_line()? else { return Ok(None) };
if log::log_enabled!(log::Level::Trace) {
log::trace!("S: {}", for_trace(&self.secrets.hide(&String::from_utf8_lossy(&line))));
}
if let Some(rest) = line.strip_prefix(b"* ") {
// A literal in what is said while idle is nothing a server has a reason to send.
if parse::literal_marker(rest).is_some() {
return Err(ProtoError::Protocol(said!("the server sent data it has no reason to send while idle")));
}
return event_from(rest, &self.secrets).map(Some);
}
let text = String::from_utf8_lossy(&line);
if text.strip_prefix(tag.as_str()).is_some_and(|rest| rest.starts_with(' ')) {
// The server ended IDLE by itself (a timeout of its own, say).
self.idle_tag = None;
let status = text.split_ascii_whitespace().nth(1).and_then(parse::Status::parse);
return match status {
Some(parse::Status::Ok) => Ok(Some(IdleEvent::Ended)),
Some(parse::Status::No | parse::Status::Bad) => Err(ProtoError::Rejected(said!("the server didn't end IDLE properly"))),
_ => Err(ProtoError::Protocol(said!("the server returned an invalid IDLE completion"))),
};
}
Err(ProtoError::Protocol(said!("the server said something that is no news while idle")))
}
/// Say DONE and read what the server still has to say, up to the end of the command. What was
/// said on the way is given back. After this the session is as it was before
/// [`Imap::idle_begin`].
pub fn idle_end(&mut self) -> Result<Vec<IdleEvent>, ProtoError> {
let Some(tag) = self.idle_tag.take() else {
return Err(ProtoError::Protocol(said!("the server isn't idling")));
};
log::trace!("C: DONE");
self.wire.write(b"DONE\r\n")?;
self.wire.flush()?;
let mut events: Vec<_> = self.idle_pending.drain(..).collect();
for _ in 0..MAX_ENDING {
// The server may take a moment to answer: a few polls of the stream's own wait.
let mut polled = None;
for _ in 0..MAX_QUIET {
polled = self.wire.poll_line()?;
if polled.is_some() {
break;
}
}
let Some(line) = polled else { return Err(ProtoError::Io(said!("the server took too long to answer"))) };
if let Some(rest) = line.strip_prefix(b"* ") {
if parse::literal_marker(rest).is_some() {
return Err(ProtoError::Protocol(said!("the server sent data it has no reason to send while idle")));
}
events.push(event_from(rest, &self.secrets)?);
continue;
}
let text = String::from_utf8_lossy(&line);
let mut words = text.splitn(3, ' ');
let (answered, status) = (words.next().unwrap_or(""), words.next().unwrap_or(""));
if answered != tag {
return Err(ProtoError::Protocol(said!("the server answered a command that wasn't sent")));
}
return match parse::Status::parse(status) {
Some(parse::Status::Ok) => Ok(events),
_ => Err(ProtoError::Rejected(said!("the server didn't end IDLE properly"))),
};
}
Err(ProtoError::Protocol(said!("the server sent too many responses to one command")))
}
/// Whether the server said it can do IDLE (upper-cased capability list as last said).
pub fn offers_idle(&self) -> bool {
self.offers("IDLE")
}
}
+139
View File
@@ -0,0 +1,139 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use proptest::prelude::*;
use super::*;
use crate::script::Script;
// Polling now yields after each underlying read, including partial lines.
fn next_event(imap: &mut Imap<&mut Script>) -> Result<Option<IdleEvent>, ProtoError> {
for _ in 0..crate::MAX_LINE + 2 {
match imap.idle_poll() {
Ok(None) => continue,
result => return result,
}
}
panic!("script failed to produce an event within its bounded input");
}
#[test]
fn idle_goes_ahead_hears_news_and_ends() {
// The go-ahead is the first line; the rest is heard by polling.
let mut s2 = Script::new("+ idling\r\n* 4 EXISTS\r\n* 1 RECENT\r\n* 2 FETCH (FLAGS (\\Seen))\r\n* 3 EXPUNGE\r\n");
let mut imap2 = Imap::resume(&mut s2);
imap2.idle_begin().unwrap();
assert_eq!(next_event(&mut imap2).unwrap(), Some(IdleEvent::Exists(4)));
assert_eq!(next_event(&mut imap2).unwrap(), Some(IdleEvent::Recent(1)));
assert_eq!(next_event(&mut imap2).unwrap(), Some(IdleEvent::Other));
assert_eq!(next_event(&mut imap2).unwrap(), Some(IdleEvent::Expunge(3)));
assert_eq!(s2.sent(), "A0001 IDLE\r\n");
let mut s3 = Script::new("+ idling\r\n* 5 EXISTS\r\nA0001 OK IDLE terminated\r\n");
let mut imap3 = Imap::resume(&mut s3);
imap3.idle_begin().unwrap();
assert_eq!(imap3.idle_end().unwrap(), vec![IdleEvent::Exists(5)]);
assert_eq!(s3.sent(), "A0001 IDLE\r\nDONE\r\n");
}
#[test]
fn a_refusal_or_an_early_end_or_a_goodbye_is_said() {
let mut s = Script::new("A0001 NO [CANNOT] not now\r\n");
assert!(matches!(Imap::resume(&mut s).idle_begin(), Err(ProtoError::Rejected(_))));
let mut s = Script::new("+ idling\r\nA0001 OK the server gave up waiting\r\n");
let mut imap = Imap::resume(&mut s);
imap.idle_begin().unwrap();
assert_eq!(next_event(&mut imap).unwrap(), Some(IdleEvent::Ended));
// Not idling any more: nothing to poll or to end.
assert!(matches!(next_event(&mut imap), Err(ProtoError::Protocol(_))));
assert!(matches!(imap.idle_end(), Err(ProtoError::Protocol(_))));
let mut s = Script::new("+ idling\r\n* BYE going away\r\n");
let mut imap = Imap::resume(&mut s);
imap.idle_begin().unwrap();
assert!(matches!(next_event(&mut imap), Err(ProtoError::Io(_))));
}
#[test]
fn what_a_server_has_no_reason_to_send_while_idle_ends_the_session() {
for hostile in ["* 1 FETCH (BODY[] {5}\r\n", "A0099 OK that was never asked\r\n", "garbage\r\n", "* 1 EXISTS\r\n".repeat(0).as_str()] {
let mut s = Script::new(format!("+ idling\r\n{hostile}"));
let mut imap = Imap::resume(&mut s);
imap.idle_begin().unwrap();
let got = next_event(&mut imap);
assert!(got.is_err() || hostile.is_empty(), "{hostile:?}: {got:?}");
}
// A line longer than any a server is allowed.
let mut long = b"+ idling\r\n* ".to_vec();
long.extend(std::iter::repeat_n(b'x', crate::MAX_LINE + 10));
long.extend_from_slice(b"\r\n");
let mut s = Script::whole(long);
let mut imap = Imap::resume(&mut s);
imap.idle_begin().unwrap();
assert!(matches!(next_event(&mut imap), Err(ProtoError::Protocol(_))));
}
proptest! {
#[test]
fn arbitrary_server_bytes_never_panic_while_idle(server in proptest::collection::vec(any::<u8>(), 0..800), greeted in any::<bool>()) {
let mut bytes = if greeted { b"+ idling\r\n".to_vec() } else { Vec::new() };
bytes.extend_from_slice(&server);
let mut s = Script::new(&bytes);
let mut imap = Imap::resume(&mut s);
if imap.idle_begin().is_ok() {
// However much is said, polling ends (the script runs out) and nothing is kept without bound.
for _ in 0..2_000 {
match next_event(&mut imap) {
Ok(Some(_)) => {}
Ok(None) | Err(_) => break,
}
}
let _ = imap.idle_end();
}
}
#[test]
fn idle_shaped_noise_never_panics(words in proptest::collection::vec(proptest::sample::select(&["* ", "1 ", "EXISTS", "EXPUNGE", "RECENT", " ", "\r\n", "A0001 OK ", "BYE ", "{9}", "FETCH (", ")"][..]), 0..60)) {
let mut s = Script::new(format!("+ go\r\n{}", words.concat()));
let mut imap = Imap::resume(&mut s);
if imap.idle_begin().is_ok() {
for _ in 0..200 {
if !matches!(next_event(&mut imap), Ok(Some(_))) {
break;
}
}
let _ = imap.idle_end();
}
}
}
#[test]
fn notifications_before_continuation_are_delivered() {
let mut s = Script::new("* 4 EXISTS\r\n* 2 EXPUNGE\r\n+ idling\r\nA0001 OK done\r\n");
let mut imap = Imap::resume(&mut s);
imap.idle_begin().unwrap();
assert_eq!(imap.idle_poll().unwrap(), Some(IdleEvent::Exists(4)));
assert_eq!(imap.idle_end().unwrap(), vec![IdleEvent::Expunge(2)]);
}
#[test]
fn idle_bye_errors_hide_registered_secrets() {
for ending in [false, true] {
let mut s = Script::new("+ idling\r\n* BYE test-token\r\n");
let mut imap = Imap::resume(&mut s);
imap.secrets.add("test-token");
imap.idle_begin().unwrap();
let error = if ending { imap.idle_end().unwrap_err() } else { next_event(&mut imap).unwrap_err() };
assert!(!error.to_string().contains("test-token"));
}
}
#[test]
fn idle_completion_requires_a_valid_success_status() {
for (status, rejected) in [("NO", true), ("BAD", true), ("nonsense", false)] {
let mut s = Script::new(format!("+ idling\r\nA0001 {status} failed\r\n"));
let mut imap = Imap::resume(&mut s);
imap.idle_begin().unwrap();
let error = next_event(&mut imap).unwrap_err();
assert!(if rejected { matches!(error, ProtoError::Rejected(_)) } else { matches!(error, ProtoError::Protocol(_)) });
}
}
+215
View File
@@ -0,0 +1,215 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
//! What one command may make the client keep or read, however much the server sends.
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use proptest::prelude::*;
use super::*;
use crate::script::Script;
/// A session whose open mailbox has `exists` messages and the next UID `uid_next`.
fn opened(s: &mut Script, exists: u32, uid_next: u32) -> Imap<&mut Script> {
let mut imap = Imap::resume(s);
imap.selected = Selected { exists, uid_validity: 1, uid_next };
imap
}
/// `lines` lines made by `line(n)`.
fn flood(lines: usize, line: impl Fn(usize) -> String) -> String {
(0..lines).map(line).collect()
}
fn padding() -> String {
"x".repeat(60 * 1024)
}
#[test]
fn a_flood_of_long_lines_ends_the_command() {
let pad = padding();
// 600 lines of 60 KB are more than the 32 MB one command may bring.
let server = flood(600, |n| format!("* {n} FETCH (UID {} FLAGS ({pad}))\r\n", n + 1)) + "A0001 OK done\r\n";
let mut s = Script::new(server);
let err = opened(&mut s, 1000, 2000).uid_fetch_flags("1:*").unwrap_err();
assert!(matches!(err, ProtoError::Protocol(ref m) if m.to_string().contains("too much")), "{err:?}");
// The same in a list.
let server = flood(600, |n| format!("* LIST () \"/\" \"{n}{pad}\"\r\n")) + "A0001 OK done\r\n";
let mut s = Script::new(server);
assert!(matches!(opened(&mut s, 0, 0).list(), Err(ProtoError::Protocol(_))));
}
#[test]
fn a_flood_of_responses_ends_the_command() {
let server = "* OK still here\r\n".repeat(MAX_RESPONSES + 1) + "A0001 OK done\r\n";
let mut s = Script::new(server);
let err = opened(&mut s, 0, 0).uid_fetch_sizes("1:*").unwrap_err();
assert!(matches!(err, ProtoError::Protocol(ref m) if m.to_string().contains("too many")), "{err:?}");
}
#[test]
fn a_list_keeps_a_capped_number_of_mailboxes() {
let server = flood(MAX_MAILBOXES + 500, |n| format!("* LIST () \"/\" f{n}\r\n")) + "A0001 OK done\r\n";
let mut s = Script::new(server);
let got = opened(&mut s, 0, 0).list().unwrap();
assert_eq!(got.len(), MAX_MAILBOXES);
assert_eq!(got.first().map(|m| m.raw.as_str()), Some("f0"));
// #991: and how many were dropped is said; none, for a list kept whole.
let server = flood(MAX_MAILBOXES + 500, |n| format!("* LIST () \"/\" f{n}\r\n")) + "A0001 OK done\r\n";
let mut s = Script::new(server);
assert_eq!(opened(&mut s, 0, 0).list_counted().unwrap().1, 500);
let mut s = Script::new("* LIST () \"/\" a\r\n* LIST () \"/\" a\r\nA0001 OK done\r\n");
assert_eq!(
opened(&mut s, 0, 0).list_counted().unwrap(),
(vec![Mailbox { name: "a".into(), raw: "a".into(), delimiter: Some('/'), flags: Vec::new() }], 0)
);
}
/// #991: a list keeps a bounded number of bytes too, however many mailboxes that is.
#[test]
fn a_list_keeps_a_capped_number_of_bytes() {
let flags: Vec<String> = (0..MAX_FLAGS).map(|n| format!("\\F{n}{}", "x".repeat(MAX_FLAG_LEN - 8))).collect();
let flags = flags.join(" ");
let server = flood(1_500, |n| format!("* LIST ({flags}) \"/\" f{n}\r\n")) + "A0001 OK done\r\n";
let mut s = Script::new(server);
let (got, dropped) = opened(&mut s, 0, 0).list_counted().unwrap();
assert_eq!(got.len() + dropped, 1_500);
let kept: usize = got.iter().map(|m| m.raw.len() * 2 + m.name.len() + m.flags.iter().map(String::len).sum::<usize>()).sum();
assert!(!got.is_empty() && got.len() < 1_500, "{}", got.len());
// The last one kept may take it over by one mailbox's worth, never more.
assert!(kept < MAX_LIST_BYTES + MAX_LINE, "{kept}");
}
#[test]
fn names_and_flags_are_kept_short() {
let long_flag = "é".repeat(400);
let long_name = "n".repeat(MAX_MAILBOX_NAME + 1);
let server = format!("* LIST ({long_flag} \\Sent) \"/\" Fine\r\n* LIST () \"/\" {long_name}\r\nA0001 OK done\r\n");
let mut s = Script::new(server);
let got = opened(&mut s, 0, 0).list().unwrap();
// The mailbox with the oversized name is left out, not cut into another one.
assert_eq!(got.len(), 1);
assert!(got[0].flags[0].len() <= MAX_FLAG_LEN && got[0].flags[0].chars().all(|c| c == 'é'));
assert_eq!(got[0].flags[1], "\\Sent");
let server = format!("* 1 FETCH (UID 1 FLAGS ({long_flag}))\r\nA0001 OK done\r\n");
let mut s = Script::new(server);
let got = opened(&mut s, 1, 2).uid_fetch_flags("1:*").unwrap();
assert!(got[0].flags[0].len() <= MAX_FLAG_LEN);
}
#[test]
fn only_uids_that_were_asked_for_and_exist_are_kept() {
let server = "* 1 FETCH (UID 3 FLAGS (\\Seen))\r\n* 2 FETCH (UID 7 FLAGS ())\r\n* 3 FETCH (UID 9 FLAGS ())\r\n* 4 FETCH (UID 4000000000 FLAGS ())\r\n\
* 5 FETCH (UID 10 FLAGS ())\r\nA0001 OK done\r\n";
// Not in the set.
let mut s = Script::new(server);
let got = opened(&mut s, 5, 11).uid_fetch_flags("3,8:9").unwrap();
assert_eq!(got.iter().map(|f| f.uid).collect::<Vec<_>>(), [3, 9]);
// UIDNEXT is only a snapshot: later deliveries must not be discarded.
let sizes = server.replace("FLAGS (\\Seen)", "RFC822.SIZE 5").replace("FLAGS ()", "RFC822.SIZE 6");
let mut s = Script::new(sizes);
let got = opened(&mut s, 5, 10).uid_fetch_sizes("1:*").unwrap();
assert_eq!(got, [(3, 5), (7, 6), (9, 6), (10, 6), (4000000000, 6)]);
}
#[test]
fn more_distinct_uids_than_the_mailbox_can_hold_fail_the_command() {
// 3 messages: room for 2 * 3 + FETCH_SLACK distinct UIDs, and not one more.
let room = 2 * 3 + FETCH_SLACK;
let many = |n: usize| flood(n, |i| format!("* {i} FETCH (UID {} FLAGS ())\r\n", i + 1)) + "A0001 OK done\r\n";
let mut s = Script::new(many(room));
assert_eq!(opened(&mut s, 3, u32::MAX).uid_fetch_flags("1:*").unwrap().len(), room);
let mut s = Script::new(many(room + 1));
assert!(matches!(opened(&mut s, 3, u32::MAX).uid_fetch_flags("1:*"), Err(ProtoError::Protocol(_))));
let sizes = flood(room + 1, |i| format!("* {i} FETCH (UID {} RFC822.SIZE 1)\r\n", i + 1)) + "A0001 OK done\r\n";
let mut s = Script::new(sizes);
assert!(matches!(opened(&mut s, 3, u32::MAX).uid_fetch_sizes("1:*"), Err(ProtoError::Protocol(_))));
// The same UID again and again is one.
let same = flood(room * 3, |_| "* 1 FETCH (UID 1 FLAGS ())\r\n".to_string()) + "A0001 OK done\r\n";
let mut s = Script::new(same);
assert_eq!(opened(&mut s, 3, 100).uid_fetch_flags("1:*").unwrap().len(), 1);
}
#[test]
fn set_ranges_read_what_is_written() {
assert_eq!(set_ranges("1:*", 9), Some(vec![(1, 9)]));
assert_eq!(set_ranges("5,9,20:12", 9), Some(vec![(5, 5), (9, 9), (12, 20)]));
assert_eq!(set_ranges("1:x", 9), None);
assert_eq!(set_ranges("", 9), None);
}
proptest! {
/// Whatever the server sends, the command ends, and what it keeps is bounded by the caps.
#[test]
fn arbitrary_responses_are_kept_within_bounds(lines in proptest::collection::vec(("[ -~]{0,200}", any::<u32>(), 0usize..4), 0..400)) {
let server: String = lines
.iter()
.map(|(text, uid, kind)| match kind {
0 => format!("* LIST (\\{text}) \"/\" {text}\r\n"),
1 => format!("* 1 FETCH (UID {uid} FLAGS ({text}))\r\n"),
2 => format!("* 1 FETCH (UID {uid} RFC822.SIZE {uid})\r\n"),
_ => format!("* {text}\r\n"),
})
.collect::<String>()
+ "A0001 OK done\r\n";
let mut s = Script::new(server.clone());
if let Ok(list) = opened(&mut s, 10, 50).list() {
prop_assert!(list.len() <= MAX_MAILBOXES);
prop_assert!(list.iter().all(|m| m.raw.len() <= MAX_MAILBOX_NAME && m.flags.len() <= MAX_FLAGS && m.flags.iter().all(|f| f.len() <= MAX_FLAG_LEN)));
}
let mut s = Script::new(server.clone());
// A fixed range retains its UID bound even if the server sends newer deliveries.
if let Ok(got) = opened(&mut s, 10, 50).uid_fetch_flags("1:49") {
prop_assert!(got.len() <= 2 * 10 + FETCH_SLACK);
prop_assert!(got.iter().all(|f| f.uid >= 1 && f.uid < 50 && f.flags.len() <= MAX_FLAGS && f.flags.iter().all(|x| x.len() <= MAX_FLAG_LEN)));
}
let mut s = Script::new(server.clone());
// Wildcards admit deliveries newer than SELECT's UIDNEXT, but retain all storage caps.
if let Ok(got) = opened(&mut s, 10, 50).uid_fetch_flags("1:*") {
prop_assert!(got.len() <= 2 * 10 + FETCH_SLACK);
prop_assert!(got.iter().all(|f| f.uid >= 1 && f.flags.len() <= MAX_FLAGS && f.flags.iter().all(|x| x.len() <= MAX_FLAG_LEN)));
}
let mut s = Script::new(server);
if let Ok(got) = opened(&mut s, 10, 50).uid_fetch_sizes("5:20") {
prop_assert!(got.iter().all(|(uid, _)| (5..=20).contains(uid)));
}
}
}
#[test]
fn a_folder_with_more_messages_than_other_commands_may_answer_still_reads() {
let exists = 300_000u32;
let lines = |n: u32| flood(n as usize, |i| format!("* {i} FETCH (UID {} FLAGS ())\r\n", i + 1)) + "A0001 OK done\r\n";
let mut s = Script::new(lines(exists));
assert_eq!(opened(&mut s, exists, exists + 1).uid_fetch_flags("1:*").unwrap().len(), exists as usize);
let sizes = flood(exists as usize, |i| format!("* {i} FETCH (UID {} RFC822.SIZE 1)\r\n", i + 1)) + "A0001 OK done\r\n";
let mut s = Script::new(sizes);
assert_eq!(opened(&mut s, exists, exists + 1).uid_fetch_sizes("1:*").unwrap().len(), exists as usize);
// Responses that are not messages still end at the cap for that folder, and the next command is back to the usual one.
let noise = "* OK still here\r\n".repeat(2 * exists as usize + FETCH_SLACK + 1) + "A0001 OK done\r\n";
let mut s = Script::new(noise);
let mut imap = opened(&mut s, exists, exists + 1);
assert!(matches!(imap.uid_fetch_flags("1:*"), Err(ProtoError::Protocol(_))));
assert_eq!(imap.response_cap, MAX_RESPONSES);
}
#[test]
fn short_flags_release_large_backing_allocations() {
let flags = short_flags(vec![Value::Atom("A".repeat(1024 * 1024))]);
assert_eq!(flags[0].len(), MAX_FLAG_LEN);
assert!(flags[0].capacity() <= MAX_FLAG_LEN);
}
#[test]
fn wildcard_fetch_accepts_deliveries_after_select() {
let server = "* 1 FETCH (UID 10 FLAGS ())\r\nA0001 OK done\r\n";
let mut s = Script::new(server);
assert_eq!(opened(&mut s, 1, 10).uid_fetch_flags("*").unwrap()[0].uid, 10);
}
#[test]
fn brackets_in_flag_keywords_do_not_consume_body_sections() {
for flag in ["x[", "BODY["] {
let mut s = Script::new(format!("* 1 FETCH (UID 3 FLAGS ({flag}) BODY[] {{2}}\r\nok)\r\nA0001 OK done\r\n"));
assert_eq!(opened(&mut s, 1, 4).uid_fetch_message(3).unwrap().unwrap().body, Some(b"ok".to_vec()));
}
}
File diff suppressed because it is too large. Load diff
+346
View File
@@ -0,0 +1,346 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
//! Reading IMAP responses: the pieces a response arrives in, the values in it, and status lines.
use crate::ProtoError;
use crate::i18n::said;
/// Deepest that parenthesised lists may nest.
const MAX_DEPTH: usize = 32;
/// A response as it came off the wire: lines of text with the literals between them.
#[derive(Debug)]
pub(crate) enum Piece {
Text(Vec<u8>),
Literal(Vec<u8>),
}
/// One value of a response.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum Value {
/// A bare word: a number, a flag, `NIL`, `BODY[]`.
Atom(String),
/// A quoted string or a literal.
Str(Vec<u8>),
List(Vec<Value>),
}
impl Value {
pub(crate) fn atom(&self) -> Option<&str> {
match self {
Value::Atom(a) => Some(a),
_ => None,
}
}
pub(crate) fn number(&self) -> Option<u32> {
self.atom()?.parse().ok()
}
/// The text of a string or a bare word (what the grammar calls an `astring`).
pub(crate) fn into_text(self) -> Option<String> {
match self {
Value::Atom(a) => Some(a),
Value::Str(bytes) => Some(String::from_utf8_lossy(&bytes).into_owned()),
Value::List(_) => None,
}
}
}
/// Where a line ends in a literal marker (`{123}`, also `{123+}`) outside any quoted string: the
/// marker's position and the size it names.
pub(crate) fn literal_marker(line: &[u8]) -> Option<(usize, u64)> {
let inner_end = line.len().checked_sub(1).filter(|last| line.get(*last) == Some(&b'}'))?;
let open = line.iter().rposition(|b| *b == b'{')?;
let inner = line.get(open + 1..inner_end)?;
let digits = inner.strip_suffix(b"+").unwrap_or(inner);
if digits.is_empty() || !digits.iter().all(u8::is_ascii_digit) {
return None;
}
// A brace inside a quoted string is just text.
let (mut quoted, mut escaped) = (false, false);
for b in line.get(..open)? {
if escaped {
escaped = false;
} else if quoted && *b == b'\\' {
escaped = true;
} else if *b == b'"' {
quoted = !quoted;
}
}
if quoted {
return None;
}
// A size beyond u64 is as large as can be, and refused like any other that is too large.
let size = digits.iter().try_fold(0u64, |n, d| n.checked_mul(10)?.checked_add(u64::from(d.wrapping_sub(b'0')))).unwrap_or(u64::MAX);
Some((open, size))
}
/// The values of a response. Unbalanced parentheses are forgiven; lists nested too deeply are not.
pub(crate) fn values(pieces: Vec<Piece>) -> Result<Vec<Value>, ProtoError> {
// The list being filled is on top; the response itself is at the bottom.
let mut stack: Vec<Vec<Value>> = vec![Vec::new()];
let mut pieces = pieces.into_iter().peekable();
while let Some(piece) = pieces.next() {
match piece {
Piece::Literal(bytes) => push(&mut stack, Value::Str(bytes)),
Piece::Text(text) => {
// The marker of the literal that follows is not a value.
let end = match (pieces.peek(), literal_marker(&text)) {
(Some(Piece::Literal(_)), Some((at, _))) => at,
_ => text.len(),
};
scan(text.get(..end).unwrap_or(&[]), &mut stack)?;
}
}
}
while stack.len() > 1 {
close(&mut stack);
}
Ok(stack.pop().unwrap_or_default())
}
fn push(stack: &mut [Vec<Value>], value: Value) {
if let Some(top) = stack.last_mut() {
top.push(value);
}
}
fn close(stack: &mut Vec<Vec<Value>>) {
if stack.len() > 1
&& let Some(list) = stack.pop()
{
push(stack, Value::List(list));
}
}
fn scan(text: &[u8], stack: &mut Vec<Vec<Value>>) -> Result<(), ProtoError> {
// A `[` only opens a section (`BODY[HEADER.FIELDS (TO)]`) when a `]` follows somewhere.
let last_close = text.iter().rposition(|b| *b == b']');
let mut i = 0;
// Every arm moves `i` forward.
while let Some(&b) = text.get(i) {
match b {
b' ' | b'\t' => i += 1,
b'(' => {
if stack.len() > MAX_DEPTH {
return Err(ProtoError::Protocol(said!("the server's response is nested too deeply")));
}
stack.push(Vec::new());
i += 1;
}
b')' => {
close(stack);
i += 1;
}
b'"' => {
let (string, next) = quoted(text, i + 1);
push(stack, Value::Str(string));
i = next;
}
_ => {
let fetch_attribute = stack.len() == 2
&& stack.first().and_then(|root| root.get(1)).and_then(Value::atom).is_some_and(|word| word.eq_ignore_ascii_case("FETCH"));
let end = atom_end(text, i, if fetch_attribute { last_close } else { None });
push(stack, Value::Atom(String::from_utf8_lossy(text.get(i..end).unwrap_or(&[])).into_owned()));
i = end;
}
}
}
Ok(())
}
/// The contents of the quoted string that starts at `start` (after its opening quote), and where
/// the text goes on. A string the line ends in the middle of is taken as far as it goes.
fn quoted(text: &[u8], start: usize) -> (Vec<u8>, usize) {
let mut out = Vec::new();
let mut i = start;
while let Some(&b) = text.get(i) {
match b {
b'"' => return (out, i + 1),
b'\\' => {
out.extend(text.get(i + 1));
i += 2;
}
_ => {
out.push(b);
i += 1;
}
}
}
(out, i)
}
/// Where the bare word starting at `start` ends: at a space or parenthesis outside `[…]`.
fn atom_end(text: &[u8], start: usize, last_close: Option<usize>) -> usize {
let mut i = start;
while let Some(&b) = text.get(i) {
match b {
b' ' | b'\t' | b'(' | b')' if i > start => break,
b'[' if last_close.is_some_and(|close| close > i)
&& text.get(start..i).is_some_and(|prefix| prefix.eq_ignore_ascii_case(b"BODY") || prefix.eq_ignore_ascii_case(b"BODY.PEEK")) =>
{
let close = text.get(i..).and_then(|rest| rest.iter().position(|b| *b == b']')).unwrap_or(0);
i += close + 1;
}
_ => i += 1,
}
}
i
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Status {
Ok,
No,
Bad,
Bye,
Preauth,
}
impl Status {
pub(crate) fn parse(word: &str) -> Option<Status> {
match word.to_ascii_uppercase().as_str() {
"OK" => Some(Status::Ok),
"NO" => Some(Status::No),
"BAD" => Some(Status::Bad),
"BYE" => Some(Status::Bye),
"PREAUTH" => Some(Status::Preauth),
_ => None,
}
}
}
/// `OK [UIDVALIDITY 3857529045] UIDs valid`, tagged or not.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct StatusLine {
pub(crate) status: Status,
/// The name of the response code, upper-cased (`UIDVALIDITY`); empty when there is none.
pub(crate) code: String,
/// What follows the name inside the brackets.
pub(crate) arg: String,
/// Everything after the status word, code included.
pub(crate) text: String,
}
impl StatusLine {
pub(crate) fn new(status: Status, rest: &str) -> StatusLine {
let text = rest.trim();
let inner = text.strip_prefix('[').and_then(|r| r.split_once(']')).map_or("", |(inner, _)| inner.trim());
let (code, arg) = inner.split_once(' ').unwrap_or((inner, ""));
StatusLine { status, code: code.to_ascii_uppercase(), arg: arg.trim().to_string(), text: text.to_string() }
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
fn text(s: &str) -> Piece {
Piece::Text(s.as_bytes().to_vec())
}
fn atom(s: &str) -> Value {
Value::Atom(s.into())
}
fn string(s: &str) -> Value {
Value::Str(s.as_bytes().to_vec())
}
fn parse(s: &str) -> Vec<Value> {
values(vec![text(s)]).unwrap()
}
#[test]
fn atoms_strings_and_lists() {
assert_eq!(
parse(r#"LIST (\HasNoChildren \Sent) "/" "Sent \"old\" \\ mail""#),
vec![atom("LIST"), Value::List(vec![atom("\\HasNoChildren"), atom("\\Sent")]), string("/"), string("Sent \"old\" \\ mail")]
);
assert_eq!(
parse("12 FETCH (UID 7 FLAGS ())"),
vec![atom("12"), atom("FETCH"), Value::List(vec![atom("UID"), atom("7"), atom("FLAGS"), Value::List(vec![])])]
);
assert_eq!(
parse("a ((b) (c (d))) NIL"),
vec![atom("a"), Value::List(vec![Value::List(vec![atom("b")]), Value::List(vec![atom("c"), Value::List(vec![atom("d")])])]), atom("NIL")]
);
assert_eq!(parse(""), vec![]);
assert_eq!(parse(" "), vec![]);
}
#[test]
fn sections_are_part_of_the_word() {
assert_eq!(
parse("12 FETCH (BODY[HEADER.FIELDS (FROM TO)] x BODY[]<0> y)"),
vec![atom("12"), atom("FETCH"), Value::List(vec![atom("BODY[HEADER.FIELDS (FROM TO)]"), atom("x"), atom("BODY[]<0>"), atom("y")])]
);
assert_eq!(parse(r#"LIST () "/" [Gmail]/Trash"#), vec![atom("LIST"), Value::List(vec![]), string("/"), atom("[Gmail]/Trash")]);
// A bracket that never closes is an ordinary character.
assert_eq!(parse("(a[b c)"), vec![Value::List(vec![atom("a[b"), atom("c")])]);
assert_eq!(parse("[[[[ x"), vec![atom("[[[["), atom("x")]);
}
#[test]
fn broken_text_is_read_as_far_as_it_goes() {
assert_eq!(parse("(a (b"), vec![Value::List(vec![atom("a"), Value::List(vec![atom("b")])])]);
assert_eq!(parse("a) b))"), vec![atom("a"), atom("b")]);
assert_eq!(parse(r#""never closed"#), vec![string("never closed")]);
assert_eq!(parse(r#""ends in a backslash\"#), vec![string("ends in a backslash")]);
assert_eq!(parse("a\"b c"), vec![atom("a\"b"), atom("c")]);
let deep = "(".repeat(MAX_DEPTH + 5);
assert!(matches!(values(vec![text(&deep)]), Err(ProtoError::Protocol(_))));
assert!(values(vec![text(&"(".repeat(MAX_DEPTH))]).is_ok());
}
#[test]
fn literals_take_the_place_of_their_marker() {
let pieces =
vec![text("1 FETCH (BODY[] {5}"), Piece::Literal(b"a (\"b".to_vec()), text(" UID 3 X {0}"), Piece::Literal(Vec::new()), text(")")];
assert_eq!(
values(pieces).unwrap(),
vec![atom("1"), atom("FETCH"), Value::List(vec![atom("BODY[]"), string("a (\"b"), atom("UID"), atom("3"), atom("X"), string("")])]
);
// Without a literal after it, a marker is only a word.
assert_eq!(parse("a {5}"), vec![atom("a"), atom("{5}")]);
}
#[test]
fn markers() {
assert_eq!(literal_marker(b"* 1 FETCH (BODY[] {123}"), Some((18, 123)));
assert_eq!(literal_marker(b"{0}"), Some((0, 0)));
assert_eq!(literal_marker(b"x {7+}"), Some((2, 7)));
assert_eq!(literal_marker(b"\"quoted\" {7}"), Some((9, 7)));
assert_eq!(literal_marker(b"x {99999999999999999999999}"), Some((2, u64::MAX)));
for not in ["", "}", "{}", "{+}", "{5} ", "{-5}", "{5 }", "\"in a string {5}", "\"a \\\" {5}", "{5}x", "{ 5}", "5}"] {
assert_eq!(literal_marker(not.as_bytes()), None, "{not:?}");
}
}
#[test]
fn status_lines() {
let s = StatusLine::new(Status::Ok, " [UIDVALIDITY 3857529045] UIDs valid ");
assert_eq!((s.code.as_str(), s.arg.as_str(), s.text.as_str()), ("UIDVALIDITY", "3857529045", "[UIDVALIDITY 3857529045] UIDs valid"));
let s = StatusLine::new(Status::No, "[alert] disk full");
assert_eq!((s.code.as_str(), s.arg.as_str()), ("ALERT", ""));
let s = StatusLine::new(Status::Ok, "done [not a code]");
assert_eq!((s.code.as_str(), s.text.as_str()), ("", "done [not a code]"));
assert_eq!(StatusLine::new(Status::Bad, "[unclosed").code, "");
assert_eq!(StatusLine::new(Status::Bad, "").text, "");
assert_eq!(Status::parse("preauth"), Some(Status::Preauth));
assert_eq!(Status::parse("FETCH"), None);
}
#[test]
fn values_as_numbers_and_text() {
assert_eq!(atom("42").number(), Some(42));
assert_eq!(atom("4294967296").number(), None);
assert_eq!(atom("-1").number(), None);
assert_eq!(string("42").number(), None);
assert_eq!(string("Entw&APw-rfe").into_text().as_deref(), Some("Entw&APw-rfe"));
assert_eq!(Value::Str(vec![0xFF, b'a']).into_text().as_deref(), Some("\u{FFFD}a"));
assert_eq!(Value::List(vec![]).into_text(), None);
}
}
+72
View File
@@ -0,0 +1,72 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
//! Password authentication from RFC 4616 and RFC 4959, over an already secured stream.
use super::{Imap, ProtoError, Read, Status, Write, ignore, said};
use base64::{Engine as _, engine::general_purpose::STANDARD};
impl<S: Read + Write> Imap<S> {
/// Prefer the server's advertised PLAIN mechanism; otherwise use LOGIN. The host must
/// secure the stream first. Unknown capabilities are queried here; known ones are reused.
/// A refusal never triggers a second attempt with another mechanism.
pub fn authenticate_password(&mut self, user: &str, password: &str) -> Result<(), ProtoError> {
if password.is_empty() {
return Err(ProtoError::Auth(said!("enter a password to sign in")));
}
if user.is_empty() {
return Err(ProtoError::Auth(said!("enter a user name to sign in")));
}
if self.capabilities.is_none() {
self.capabilities()?;
}
if !self.offers("AUTH=PLAIN") {
return self.login(user, password);
}
if user.contains('\0') || password.contains('\0') {
return Err(ProtoError::Auth(said!("PLAIN needs a user name and password without NUL characters")));
}
if user.len().saturating_add(password.len()) > 32 * 1024 {
return Err(ProtoError::Auth(said!("the user name or password is too long")));
}
// Empty authorization identity: the server derives it from the authentication identity.
let response = STANDARD.encode(format!("\0{user}\0{password}").as_bytes());
self.secrets.add(password);
self.secrets.add(&response);
let inline = self.offers("SASL-IR");
let tag = self.tag();
let command = if inline { format!("{tag} AUTHENTICATE PLAIN {response}\r\n") } else { format!("{tag} AUTHENTICATE PLAIN\r\n") };
log::trace!("C: {tag} AUTHENTICATE PLAIN{}", if inline { " ***" } else { "" });
self.wire.write(command.as_bytes())?;
self.wire.flush()?;
let mut sent = inline;
let mut cancelled = false;
let verdict = loop {
if let Some(verdict) = self.read_until(&tag, true, &mut ignore)? {
break verdict;
}
if cancelled {
return Err(ProtoError::Protocol(said!("the server continued a cancelled password authentication")));
}
let answer = if sent {
cancelled = true;
"*"
} else {
sent = true;
response.as_str()
};
log::trace!("C: ***");
self.wire.write(format!("{answer}\r\n").as_bytes())?;
self.wire.flush()?;
};
match verdict.status {
_ if cancelled => Err(ProtoError::Auth(said!("the server requested more than one PLAIN response"))),
Status::Ok if !sent => Err(ProtoError::Protocol(said!("the server accepted a password that was not sent"))),
Status::Ok => {
self.authenticated = true;
Ok(())
}
Status::No => Err(ProtoError::Auth(self.refusal(&verdict))),
_ => Err(ProtoError::Rejected(self.refusal(&verdict))),
}
}
}
+112
View File
@@ -0,0 +1,112 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::script::Script;
use base64::{Engine as _, engine::general_purpose::STANDARD};
fn greeting(caps: &str, replies: &str) -> String {
format!("* OK [CAPABILITY IMAP4rev1 {caps}] ready\r\n{replies}")
}
#[test]
fn plain_uses_an_initial_response_only_when_advertised() {
let payload = STANDARD.encode(b"\0user@example.org\0invented");
for inline in [false, true] {
let caps = if inline { "AUTH=PLAIN SASL-IR LOGINDISABLED" } else { "AUTH=PLAIN LOGINDISABLED" };
let replies = if inline { "A0001 OK welcome\r\n" } else { "+ continue\r\nA0001 OK welcome\r\n" };
let mut stream = Script::new(greeting(caps, replies));
let mut imap = Imap::connect(&mut stream).unwrap();
imap.authenticate_password("user@example.org", "invented").unwrap();
assert!(imap.is_authenticated());
let expected = if inline { format!("A0001 AUTHENTICATE PLAIN {payload}\r\n") } else { format!("A0001 AUTHENTICATE PLAIN\r\n{payload}\r\n") };
assert_eq!(stream.sent(), expected);
}
}
#[test]
fn login_remains_available_when_plain_is_not_advertised() {
let mut stream = Script::new(greeting("SASL-IR", "A0001 OK welcome\r\n"));
Imap::connect(&mut stream).unwrap().authenticate_password("user@example.org", "invented").unwrap();
assert_eq!(stream.sent(), "A0001 LOGIN \"user@example.org\" \"invented\"\r\n");
}
#[test]
fn a_plain_refusal_is_redacted_and_never_retried_with_login() {
let payload = STANDARD.encode(b"\0user@example.org\0invented");
let mut stream = Script::new(greeting("AUTH=PLAIN SASL-IR", &format!("A0001 NO [AUTHENTICATIONFAILED] invented {payload}\r\n")));
let mut imap = Imap::connect(&mut stream).unwrap();
let error = imap.authenticate_password("user@example.org", "invented").unwrap_err();
assert!(matches!(error, ProtoError::Auth(_)));
assert!(!format!("{error:?} {imap:?}").contains("invented"));
assert!(!format!("{error:?} {imap:?}").contains(&payload));
assert!(!imap.is_authenticated());
assert!(!stream.sent().contains(" LOGIN "));
}
#[test]
fn a_refusal_before_the_challenge_sends_no_password() {
let mut stream = Script::new(greeting("AUTH=PLAIN", "A0001 NO unavailable\r\n"));
assert!(Imap::connect(&mut stream).unwrap().authenticate_password("user@example.org", "invented").is_err());
assert_eq!(stream.sent(), "A0001 AUTHENTICATE PLAIN\r\n");
}
#[test]
fn repeated_challenges_cancel_and_never_authenticate() {
for replies in ["+ first\r\n+ second\r\nA0001 OK ignored\r\n", "+ first\r\n+ second\r\n+ third\r\n"] {
let mut stream = Script::new(greeting("AUTH=PLAIN", replies));
let mut imap = Imap::connect(&mut stream).unwrap();
assert!(imap.authenticate_password("user@example.org", "invented").is_err());
assert!(!imap.is_authenticated());
assert!(stream.sent().ends_with("\r\n*\r\n"));
}
}
#[test]
fn premature_success_and_invalid_credentials_are_refused() {
let mut stream = Script::new(greeting("AUTH=PLAIN", "A0001 OK too early\r\n"));
assert!(Imap::connect(&mut stream).unwrap().authenticate_password("user@example.org", "invented").is_err());
for (user, password) in [("", "invented"), ("user@example.org", ""), ("bad\0user", "invented"), ("user@example.org", "bad\0password")] {
let mut stream = Script::new(greeting("AUTH=PLAIN", ""));
assert!(Imap::connect(&mut stream).unwrap().authenticate_password(user, password).is_err());
assert!(stream.sent().is_empty());
}
let mut stream = Script::new(greeting("AUTH=PLAIN", ""));
assert!(Imap::connect(&mut stream).unwrap().authenticate_password("user@example.org", &"x".repeat(32769)).is_err());
assert!(stream.sent().is_empty());
}
#[test]
fn utf8_and_line_breaks_are_encoded_not_protocol_lines() {
let mut stream = Script::new(greeting("auth=plain sasl-ir", "A0001 OK welcome\r\n"));
Imap::connect(&mut stream).unwrap().authenticate_password("user@example.org", "päss\r\nword").unwrap();
assert_eq!(stream.sent().lines().count(), 1);
assert!(stream.sent().contains(&STANDARD.encode("\0user@example.org\0päss\r\nword".as_bytes())));
}
#[test]
fn unknown_capabilities_are_queried_once_by_password_authentication() {
let mut stream = Script::new("* OK ready\r\n* CAPABILITY IMAP4rev1 AUTH=PLAIN SASL-IR\r\nA0001 OK done\r\nA0002 OK welcome\r\n");
Imap::connect(&mut stream).unwrap().authenticate_password("user@example.org", "invented").unwrap();
assert!(stream.sent().starts_with("A0001 CAPABILITY\r\nA0002 AUTHENTICATE PLAIN "));
assert_eq!(stream.sent().matches(" CAPABILITY").count(), 1);
}
#[test]
fn a_missing_password_has_its_own_message_without_network_commands() {
let mut stream = Script::new("* OK ready\r\n");
let error = Imap::connect(&mut stream).unwrap().authenticate_password("user@example.org", "").unwrap_err();
assert!(error.to_string().contains("enter a password"));
assert!(!error.to_string().contains("NUL"));
assert!(stream.sent().is_empty());
}
#[test]
fn an_empty_user_name_is_explained_before_any_commands() {
for text in ["* OK ready\r\n".to_string(), greeting("AUTH=PLAIN", ""), greeting("", "")] {
let mut stream = Script::new(text);
let error = Imap::connect(&mut stream).unwrap().authenticate_password("", "invented").unwrap_err();
assert!(error.to_string().contains("enter a user name"));
assert!(!error.to_string().contains("NUL"));
assert!(stream.sent().is_empty());
}
}
+164
View File
@@ -0,0 +1,164 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
//! STATUS (RFC 3501 §6.3.10): what a mailbox holds, asked without opening it, for a program to
//! tell whether it has changed since it was last read (#276). Many mailboxes are asked in one
//! go: the commands are written together and the answers read together (pipelining, RFC 3501
//! §5.5), so a hundred mailboxes cost about one round trip, not a hundred.
use std::io::{Read, Write};
use super::parse::{Status, Value};
use super::{Command, Imap, MAX_COMMAND_TEXT, MAX_QUOTED, Part, Response};
use crate::i18n::said;
use crate::{ProtoError, for_trace};
/// Most mailboxes asked in one go: their commands are a few kilobytes at most, so the server
/// can't be left waiting to be read while we write.
pub const MAX_STATUS_BATCH: usize = 50;
/// What STATUS says of a mailbox: all four counts, or nothing at all (a server that leaves one
/// out gets its mailbox read, never passed over on a guess).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct FolderStatus {
/// How many messages there are.
pub messages: u32,
/// The UID the next message to arrive gets.
pub uid_next: u32,
pub uid_validity: u32,
/// How many are not marked `\Seen`.
pub unseen: u32,
}
/// From `STATUS name (MESSAGES 3 UIDNEXT 9 …)`; `None` when it isn't one, or a count is missing.
fn status_from(values: Vec<Value>) -> Option<FolderStatus> {
let mut values = values.into_iter();
if !values.next()?.atom()?.eq_ignore_ascii_case("STATUS") {
return None;
}
values.next()?;
let Value::List(pairs) = values.next()? else { return None };
let (mut messages, mut uid_next, mut uid_validity, mut unseen) = (None, None, None, None);
let mut pairs = pairs.into_iter();
while let (Some(name), Some(value)) = (pairs.next(), pairs.next()) {
let Some(number) = value.number() else { continue };
match name.atom().map(str::to_ascii_uppercase).as_deref() {
Some("MESSAGES") => messages = Some(number),
Some("UIDNEXT") => uid_next = Some(number),
Some("UIDVALIDITY") => uid_validity = Some(number),
Some("UNSEEN") => unseen = Some(number),
_ => {}
}
}
Some(FolderStatus { messages: messages?, uid_next: uid_next?, uid_validity: uid_validity?, unseen: unseen? })
}
fn command(raw_name: &str) -> Command<'_> {
Command::new("STATUS").string(raw_name.as_bytes()).arg("(MESSAGES UIDNEXT UIDVALIDITY UNSEEN)")
}
impl<S: Read + Write> Imap<S> {
/// STATUS of one mailbox; `None` when the server refuses or leaves a count out.
pub fn status(&mut self, raw_name: &str) -> Result<Option<FolderStatus>, ProtoError> {
let mut found = None;
let line = self.exchange(command(raw_name), &mut |values| {
if found.is_none() {
found = status_from(values);
}
})?;
Ok(found.filter(|_| line.status == Status::Ok))
}
/// STATUS of each of `raw_names`, in the same order, `None` for one the server refuses or
/// leaves a count out of. At most [`MAX_STATUS_BATCH`] are asked in one go; a name that
/// can't go in a quoted string is asked alone.
///
/// An error is the connection's: the answers read before it are lost with it.
pub fn status_many(&mut self, raw_names: &[&str]) -> Result<Vec<Option<FolderStatus>>, ProtoError> {
let mut out = Vec::with_capacity(raw_names.len());
for batch in raw_names.chunks(MAX_STATUS_BATCH) {
let plain = batch.iter().all(|name| name.len() <= MAX_QUOTED && name.bytes().all(|b| matches!(b, b' '..=b'~')));
if plain && batch.len() > 1 {
out.extend(self.status_batch(batch)?);
} else {
for name in batch {
out.push(self.status(name)?);
}
}
}
Ok(out)
}
/// One batch, written together: the answers come in the order of the commands.
fn status_batch(&mut self, batch: &[&str]) -> Result<Vec<Option<FolderStatus>>, ProtoError> {
let mut tags = Vec::with_capacity(batch.len());
let mut out = Vec::with_capacity(batch.len().saturating_mul(64));
for name in batch {
let command = command(name);
if command.unsendable {
return Err(ProtoError::Protocol(said!("the text has a character that can't be sent")));
}
let tag = self.tag();
if log::log_enabled!(log::Level::Trace) {
log::trace!("C: {tag} {}", for_trace(&self.secrets.hide(&command.for_log())));
}
out.extend_from_slice(tag.as_bytes());
out.push(b' ');
for part in &command.parts {
if let Part::Text(text) = part {
out.extend_from_slice(text);
}
}
out.extend_from_slice(b"\r\n");
tags.push(tag);
}
self.wire.write(&out)?;
self.wire.flush()?;
self.text_seen = 0;
let mut found: Vec<Option<FolderStatus>> = vec![None; batch.len()];
let mut answered = 0usize;
// Each command brings one STATUS and its tagged answer; the rest is slack for what a
// server says unasked.
let cap = batch.len().saturating_mul(2).saturating_add(64);
for _ in 0..cap {
if answered >= tags.len() {
break;
}
match self.read_response()? {
Response::GoAhead => return Err(ProtoError::Protocol(said!("the server asked for more than the command had"))),
Response::Status { tag: Some(tag), line } => {
if tags.get(answered) != Some(&tag) || !matches!(line.status, Status::Ok | Status::No | Status::Bad) {
return Err(ProtoError::Protocol(said!("the server answered a command that wasn't sent")));
}
self.note_status(&line);
if line.status != Status::Ok
&& let Some(slot) = found.get_mut(answered)
{
*slot = None;
}
answered += 1;
}
Response::Status { tag: None, line } => {
if line.status == Status::Bye && !self.leaving {
return Err(ProtoError::Io(said!("the server ended the session")));
}
self.note_status(&line);
}
Response::Data(values) => {
if self.text_seen > MAX_COMMAND_TEXT {
return Err(ProtoError::Protocol(said!("the server sent too much in answer to one command")));
}
self.note_data(&values);
if let Some(slot) = found.get_mut(answered)
&& slot.is_none()
{
*slot = status_from(values);
}
}
}
}
if answered < tags.len() {
return Err(ProtoError::Protocol(said!("the server sent too many responses to one command")));
}
Ok(found)
}
}
+78
View File
@@ -0,0 +1,78 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use proptest::prelude::*;
use super::*;
use crate::script::Script;
fn status(messages: u32, uid_next: u32, uid_validity: u32, unseen: u32) -> FolderStatus {
FolderStatus { messages, uid_next, uid_validity, unseen }
}
#[test]
fn one_mailbox_is_asked_its_four_counts() {
let mut s = Script::new("* STATUS \"Work\" (MESSAGES 3 UIDNEXT 9 UIDVALIDITY 77 UNSEEN 1)\r\nA0001 OK done\r\n");
let got = Imap::resume(&mut s).status("Work").unwrap();
assert_eq!(got, Some(status(3, 9, 77, 1)));
assert_eq!(s.sent(), "A0001 STATUS \"Work\" (MESSAGES UIDNEXT UIDVALIDITY UNSEEN)\r\n");
}
#[test]
fn a_count_left_out_or_a_refusal_is_no_status() {
let mut s = Script::new("* STATUS Work (MESSAGES 3 UIDNEXT 9 UIDVALIDITY 77)\r\nA0001 OK done\r\nA0002 NO [NONEXISTENT] no such mailbox\r\n");
let mut imap = Imap::resume(&mut s);
assert_eq!(imap.status("Work").unwrap(), None);
assert_eq!(imap.status("Gone").unwrap(), None);
}
#[test]
fn many_mailboxes_are_asked_in_one_write() {
let mut s = Script::new(
"* STATUS A (MESSAGES 1 UIDNEXT 2 UIDVALIDITY 5 UNSEEN 0)\r\nA0001 OK done\r\n\
A0002 NO [NONEXISTENT] no\r\n\
* STATUS \"C d\" (UNSEEN 4 UIDVALIDITY 7 UIDNEXT 8 MESSAGES 9 HIGHESTMODSEQ 12)\r\nA0003 OK done\r\n",
);
let got = Imap::resume(&mut s).status_many(&["A", "B", "C d"]).unwrap();
assert_eq!(got, vec![Some(status(1, 2, 5, 0)), None, Some(status(9, 8, 7, 4))]);
let sent = s.sent();
assert_eq!(sent.matches("STATUS").count(), 3);
assert!(sent.starts_with("A0001 STATUS \"A\" ") && sent.contains("A0003 STATUS \"C d\" "), "{sent}");
}
#[test]
fn answers_out_of_turn_or_too_many_end_the_session_not_the_program() {
let mut s = Script::new("A0002 OK done\r\n");
assert!(matches!(Imap::resume(&mut s).status_many(&["A", "B"]), Err(ProtoError::Protocol(_))));
let flood = "* 1 EXISTS\r\n".repeat(500);
let mut s = Script::new(flood);
assert!(matches!(Imap::resume(&mut s).status_many(&["A", "B"]), Err(ProtoError::Protocol(_))));
let mut s = Script::new("* BYE going\r\n");
assert!(matches!(Imap::resume(&mut s).status_many(&["A", "B"]), Err(ProtoError::Io(_))));
}
#[test]
fn a_name_that_needs_a_literal_is_asked_alone() {
let mut s = Script::new("+ go\r\n* STATUS x (MESSAGES 1 UIDNEXT 2 UIDVALIDITY 3 UNSEEN 0)\r\nA0001 OK done\r\n");
let got = Imap::resume(&mut s).status_many(&["é", "A"]).unwrap_or_default();
assert!(got.len() <= 2);
}
proptest! {
#[test]
fn arbitrary_server_bytes_never_panic_in_status(server in proptest::collection::vec(any::<u8>(), 0..600), names in 0usize..120) {
let list: Vec<String> = (0..names).map(|n| format!("F{n}")).collect();
let refs: Vec<&str> = list.iter().map(String::as_str).collect();
let mut s = Script::new(&server);
let got = Imap::resume(&mut s).status_many(&refs);
if let Ok(got) = got {
prop_assert_eq!(got.len(), names);
}
}
#[test]
fn any_status_line_is_read_or_not(line in "[ -~]{0,80}") {
let mut s = Script::new(format!("* STATUS {line}\r\nA0001 OK done\r\n"));
let _ = Imap::resume(&mut s).status("x");
}
}
File diff suppressed because it is too large. Load diff
+175
View File
@@ -0,0 +1,175 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
//! Modified UTF-7, the encoding of mailbox names (RFC 3501 §5.1.3).
//!
//! Printable ASCII stands for itself, except `&`, which is written `&-`. Everything else is
//! UTF-16 in base64 (with `,` in place of `/`, and no padding) between `&` and `-`.
const DIGITS: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,";
fn digit(six_bits: u32) -> char {
DIGITS.get((six_bits & 0x3F) as usize).map_or('A', |b| char::from(*b))
}
/// A mailbox name as people read it. Text that isn't valid modified UTF-7 is kept as it is.
pub fn decode_utf7(name: &str) -> String {
let mut out = String::with_capacity(name.len());
let mut rest = name;
// Every round takes at least the `&` off `rest`.
while let Some(at) = rest.find('&') {
out.push_str(rest.get(..at).unwrap_or(""));
let after = rest.get(at + 1..).unwrap_or("");
rest = match after.find('-') {
Some(0) => {
out.push('&');
after.get(1..).unwrap_or("")
}
Some(end) => match shifted(after.get(..end).unwrap_or("")) {
Some(text) => {
out.push_str(&text);
after.get(end + 1..).unwrap_or("")
}
// Not base64 after all: an ampersand like any other character.
None => {
out.push('&');
after
}
},
None => {
out.push('&');
after
}
};
}
out.push_str(rest);
out
}
/// The text of one base64 run, or `None` when it isn't one.
fn shifted(encoded: &str) -> Option<String> {
let mut units = Vec::with_capacity(encoded.len() * 3 / 8 + 1);
let (mut bits, mut count) = (0u32, 0u32);
for b in encoded.bytes() {
let value = DIGITS.iter().position(|d| *d == b)? as u32;
bits = (bits << 6) | value;
count += 6;
if count >= 16 {
count -= 16;
units.push((bits >> count) as u16);
bits &= (1 << count) - 1;
}
}
// What is left over is padding: less than one digit's worth.
if count >= 6 || units.is_empty() {
return None;
}
String::from_utf16(&units).ok()
}
/// A mailbox name as a server wants it.
pub fn encode_utf7(name: &str) -> String {
let mut out = String::with_capacity(name.len());
let mut pending: Vec<u16> = Vec::new();
for c in name.chars() {
if matches!(c, ' '..='~') {
flush(&mut pending, &mut out);
out.push(c);
if c == '&' {
out.push('-');
}
} else {
let mut units = [0u16; 2];
pending.extend_from_slice(c.encode_utf16(&mut units));
}
}
flush(&mut pending, &mut out);
out
}
/// Write the characters waiting to be encoded as one base64 run.
fn flush(pending: &mut Vec<u16>, out: &mut String) {
if pending.is_empty() {
return;
}
out.push('&');
let (mut bits, mut count) = (0u32, 0u32);
for unit in pending.drain(..) {
bits = (bits << 16) | u32::from(unit);
count += 16;
while count >= 6 {
count -= 6;
out.push(digit(bits >> count));
}
bits &= (1 << count) - 1;
}
if count > 0 {
out.push(digit(bits << (6 - count)));
}
out.push('-');
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use proptest::prelude::*;
use super::*;
#[test]
fn the_examples_of_the_rfc() {
// RFC 3501 §5.1.3.
assert_eq!(decode_utf7("~peter/mail/&U,BTFw-/&ZeVnLIqe-"), "~peter/mail/台北/日本語");
assert_eq!(encode_utf7("~peter/mail/台北/日本語"), "~peter/mail/&U,BTFw-/&ZeVnLIqe-");
assert_eq!(decode_utf7("&Jjo-!"), "☺!");
assert_eq!(encode_utf7("☺!"), "&Jjo-!");
}
#[test]
fn ordinary_names() {
assert_eq!(encode_utf7("INBOX"), "INBOX");
assert_eq!(encode_utf7("Tom & Jerry"), "Tom &- Jerry");
assert_eq!(decode_utf7("Tom &- Jerry"), "Tom & Jerry");
assert_eq!(encode_utf7("Entwürfe"), "Entw&APw-rfe");
assert_eq!(decode_utf7("Entw&APw-rfe"), "Entwürfe");
assert_eq!(encode_utf7("Gelöschte Elemente"), "Gel&APY-schte Elemente");
// Outside the basic plane: a surrogate pair.
assert_eq!(decode_utf7(&encode_utf7("📬 Post")), "📬 Post");
assert_eq!(encode_utf7("📬"), "&2D3c7A-");
// Control characters are encoded too.
assert_eq!(encode_utf7("a\tb"), "a&AAk-b");
assert_eq!(encode_utf7(""), "");
}
#[test]
fn junk_is_kept_as_it_is() {
for junk in ["&", "a&b", "&&&", "&!!!-", "&A-", "&AAAAA-x", "R&D", "&2D0-", "&-&", "é&é-"] {
let decoded = decode_utf7(junk);
assert!(!decoded.is_empty(), "{junk}");
}
assert_eq!(decode_utf7("R&D"), "R&D");
assert_eq!(decode_utf7("&!!!-"), "&!!!-");
// A lone surrogate is not text.
assert_eq!(decode_utf7("&2D0-"), "&2D0-");
assert_eq!(decode_utf7("AT&T-Mobile"), "AT&T-Mobile");
}
proptest! {
#[test]
fn any_name_round_trips(name in "\\PC{0,40}") {
let encoded = encode_utf7(&name);
prop_assert!(encoded.bytes().all(|b| (0x20..=0x7E).contains(&b)), "{encoded}");
prop_assert_eq!(decode_utf7(&encoded), name);
}
#[test]
fn any_text_round_trips_with_control_characters(name in proptest::collection::vec(any::<char>(), 0..40)) {
let name: String = name.into_iter().collect();
prop_assert_eq!(decode_utf7(&encode_utf7(&name)), name);
}
#[test]
fn decoding_never_panics(name in "[&A-Za-z0-9+,\\-é ]{0,60}") {
let _ = decode_utf7(&name);
}
}
}
+229
View File
@@ -0,0 +1,229 @@
// SPDX-FileCopyrightText: Craft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
//! Mail protocols without sockets.
//!
//! [`smtp::Smtp`] and [`imap::Imap`] speak SMTP (RFC 5321, 3207, 4954) and IMAP4rev1 (RFC 3501,
//! 4315, 4959, 6154, 6851) over any `Read + Write` stream: a TLS socket on the desktop, a scripted
//! transcript in the tests. Everything a server sends is treated as hostile: lines, literals and
//! response counts are capped, and nothing here panics on junk.
//!
//! Passwords and access tokens never reach a log or an error text, even when a server echoes
//! them back.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
pub mod i18n;
pub mod imap;
mod line;
pub mod smtp;
#[cfg(test)]
mod script;
use base64::{Engine as _, engine::general_purpose::STANDARD};
use i18n::{Arg, Text};
pub(crate) use i18n::{for_trace, tidy};
/// Longest line a server may send.
pub const MAX_LINE: usize = 64 * 1024;
/// Largest counted block (an IMAP literal, so: the largest message) a server may send.
pub const MAX_LITERAL: usize = 64 * 1024 * 1024;
/// Longest user name and access token, together, that one sign-in may carry.
const MAX_XOAUTH2: usize = 32 * 1024;
/// What went wrong. Each holds the reason as a [`Text`]: Craft's own words are a template
/// displayed in its language ([`said!`]), and what a server or the system said is kept as
/// it said it ([`Text::verbatim`]). The English ([`Display`](std::fmt::Display)) is what the CLI, MCP
/// and the log read, and it is the same either way.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ProtoError {
Io(Text),
Protocol(Text),
/// The server refused the command (IMAP NO/BAD, SMTP 4xx/5xx); the text is the server's.
Rejected(Text),
Auth(Text),
}
impl ProtoError {
/// The underlying reason, without its error category prefix.
pub fn reason(&self) -> &Text {
match self {
Self::Io(why) | Self::Protocol(why) | Self::Rejected(why) | Self::Auth(why) => why,
}
}
/// The error as the user is told it: what kind of trouble, and why.
pub fn said(&self) -> Text {
match self {
ProtoError::Io(why) => said!("connection problem: {reason}", reason = Arg::said(why.clone())),
ProtoError::Protocol(why) => said!("unexpected server response: {reason}", reason = Arg::said(why.clone())),
ProtoError::Rejected(why) => said!("the server said: {reason}", reason = Arg::said(why.clone())),
ProtoError::Auth(why) => said!("sign-in failed: {reason}", reason = Arg::said(why.clone())),
}
}
}
impl std::fmt::Display for ProtoError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.said(), f)
}
}
impl std::error::Error for ProtoError {}
/// Secrets: passwords and tokens, never shown in logs or errors.
#[derive(Default)]
pub(crate) struct Secrets(Vec<zeroize::Zeroizing<String>>);
impl Secrets {
pub(crate) fn add(&mut self, secret: &str) {
if !secret.is_empty() && !self.0.iter().any(|s| s.as_str() == secret) {
self.0.push(zeroize::Zeroizing::new(secret.to_owned()));
// Longer overlapping secrets must be hidden before their prefixes.
self.0.sort_by_key(|s| std::cmp::Reverse(s.len()));
}
}
pub(crate) fn hide(&self, text: &str) -> String {
self.0.iter().fold(text.to_owned(), |text, secret| text.replace(secret.as_str(), "***"))
}
}
/// Build a bounded, base64-encoded SASL XOAUTH2 initial response.
pub(crate) fn xoauth2(user: &str, token: &str) -> Result<String, ProtoError> {
if user.is_empty() || token.is_empty() || user.chars().chain(token.chars()).any(char::is_control) {
return Err(ProtoError::Auth(said!("the user name or token has a character that can't be sent")));
}
if user.len().saturating_add(token.len()) > MAX_XOAUTH2 {
return Err(ProtoError::Auth(said!("the user name or token is too long")));
}
let plain = zeroize::Zeroizing::new(format!("user={user}\x01auth=Bearer {token}\x01\x01"));
Ok(STANDARD.encode(plain.as_bytes()))
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[test]
fn secrets_redact_overlapping_values() {
let mut secrets = Secrets::default();
secrets.add("token");
secrets.add("token-long");
assert_eq!(secrets.hide("token-long token"), "*** ***");
}
#[test]
fn xoauth2_rejects_injection_and_oversized_values() {
assert!(xoauth2("user\x01auth=Bearer injected", "token").is_err());
assert!(xoauth2("user", &"x".repeat(MAX_XOAUTH2)).is_err());
}
#[test]
fn error_display() {
let err = ProtoError::Io(said!("connection problem"));
assert_eq!(err.to_string(), "connection problem: connection problem");
assert_eq!(err.reason().to_string(), "connection problem");
}
#[test]
fn error_said() {
let err = ProtoError::Auth(said!("invalid password"));
assert_eq!(err.said().to_string(), "sign-in failed: invalid password");
}
#[test]
fn xoauth2_format() {
let result = xoauth2("user@example.com", "token123").unwrap();
let result = String::from_utf8(STANDARD.decode(result).unwrap()).unwrap();
assert!(result.contains("user=user@example.com"));
assert!(result.contains("auth=Bearer token123"));
}
}
#[cfg(test)]
mod source_regression_tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[test]
fn the_wire_trace_keeps_keywords_and_no_address_name_or_text() {
// (Line, what the trace shows.)
for (line, _old_shown) in [
("A0001 OK ada@example.org authenticated (Success)\r\n", "A0001 OK … (39 bytes)"),
("A0002 LOGIN ***", "A0002 LOGIN ***"),
("MAIL FROM:<ada@example.org> SIZE=120", "MAIL FROM: … (26 bytes)"),
("RCPT TO:<bob@example.org>", "RCPT TO: … (17 bytes)"),
("550 5.1.1 <bob@example.org>: Recipient address rejected", "550 5.1.1 … (45 bytes)"),
("250-mx.example.org at your service", "250- … (30 bytes)"),
("250-AUTH LOGIN PLAIN XOAUTH2", "250-AUTH LOGIN PLAIN XOAUTH2"),
("A0003 SELECT \"Letters from Grace\"", "A0003 SELECT … (20 bytes)"),
("* LIST (\\HasNoChildren) \"/\" Grace", "* LIST … (26 bytes)"),
("* 12 FETCH (UID 40 FLAGS (\\Seen) BODY[HEADER.FIELDS (SUBJECT)] \"Subject: Quarterly figures\")", "* 12 FETCH … (81 bytes)"),
("* 3 EXISTS", "* 3 EXISTS"),
("UID FETCH 1:40,42 (UID FLAGS)", "UID FETCH 1:40,42 … (11 bytes)"),
("hello there", "(11 bytes)"),
("", ""),
] {
let traced = for_trace(line);
assert!(traced.len() <= line.len().saturating_add(32), "{line:?}");
for private in ["ada", "bob", "Grace", "Quarterly", "mx.example"] {
assert!(!traced.contains(private), "{line:?}: {traced}");
}
}
assert!(for_trace(&format!("A0001 OK {}", "X ".repeat(500))).len() < 160);
assert_eq!(for_trace("ünïcödé ä"), "(14 bytes)");
}
#[test]
fn uppercase_private_text_is_not_protocol_structure() {
assert_eq!(for_trace("550 CUSTOMER ALICE PAYMENT OVERDUE"), "550 … (30 bytes)");
assert_eq!(for_trace("A0003 SELECT PRIVATE_MAILBOX"), "A0003 SELECT … (15 bytes)");
assert_eq!(for_trace("250-PRIVATE_CUSTOMER_DOMAIN"), "250- … (23 bytes)");
assert_eq!(for_trace("A0001 OK PRIVATE CUSTOMER"), "A0001 OK … (16 bytes)");
}
#[test]
fn secrets_are_hidden_longest_first() {
let mut s = Secrets::default();
s.add("hunter2");
s.add("hunter2-and-more");
s.add("");
assert_eq!(s.hide("no hunter2-and-more, no hunter2, nothing"), "no ***, no ***, nothing");
assert_eq!(Secrets::default().hide("plain"), "plain");
}
#[test]
fn the_xoauth2_response_is_user_and_bearer_token_between_control_a() {
let decoded = |payload: String| STANDARD.decode(payload).unwrap();
assert_eq!(decoded(xoauth2("ada@example.org", "tok.en-123").unwrap()), b"user=ada@example.org\x01auth=Bearer tok.en-123\x01\x01");
// The example of the mechanism's description.
assert_eq!(
xoauth2("someuser@example.com", "ya29.vF9dft4qmTc2Nvb3RlckBhdHRhdmlzdGEuY29tCg").unwrap(),
"dXNlcj1zb21ldXNlckBleGFtcGxlLmNvbQFhdXRoPUJlYXJlciB5YTI5LnZGOWRmdDRxbVRjMk52YjNSbGNrQmhkSFJoZG1semRHRXVZMjl0Q2cBAQ=="
);
assert_eq!(decoded(xoauth2("ädä", "t").unwrap()), "user=ädä\x01auth=Bearer t\x01\x01".as_bytes());
// Nothing that could end the name, the token or the line early.
for (user, token) in [("ada\x01auth=Bearer other", "t"), ("ada", "t\x01"), ("ada\r\n", "t"), ("ada", "t\nQUIT"), ("a\0", "t"), ("ada", "\0")]
{
assert!(matches!(xoauth2(user, token), Err(ProtoError::Auth(_))), "{user:?} {token:?}");
}
assert!(matches!(xoauth2("ada", ""), Err(ProtoError::Auth(_))));
assert!(xoauth2("ada", &"t".repeat(MAX_XOAUTH2 - 3)).is_ok());
let err = xoauth2("ada", &"t".repeat(MAX_XOAUTH2)).unwrap_err();
assert!(matches!(&err, ProtoError::Auth(why) if why.to_string().len() < 100), "{err:?}");
}
#[test]
fn server_text_is_tidied() {
assert_eq!(tidy(" 5.7.8\tBad\r\n credentials \u{7}"), "5.7.8 Bad credentials");
let long = tidy(&"é".repeat(1000));
assert_eq!(long.chars().count(), 301);
assert!(long.ends_with('…'));
assert_eq!(tidy(""), "");
}
}
+318
View File
@@ -0,0 +1,318 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
//! The bounded reader both protocols share: lines and counted blocks from a stream that may be hostile.
use std::io::{ErrorKind, Read, Write};
use crate::said;
use crate::{MAX_LINE, MAX_LITERAL, ProtoError};
/// How much is asked of the stream in one read.
const CHUNK: usize = 16 * 1024;
/// Most bytes of one over-long untagged line read past ([`Wire::read_line_or_skip`]).
const MAX_SKIPPED: usize = 4 * 1024 * 1024;
/// Reads interrupted in a row before giving up.
const MAX_INTERRUPTS: u32 = 1_000;
/// A stream with the bytes read ahead of what was asked for.
pub(crate) struct Wire<S> {
stream: S,
/// Bytes read from the stream and not handed out yet (from `start`).
buf: Vec<u8>,
start: usize,
// Bytes already searched for a newline in the current pending polling line.
poll_scanned: usize,
}
fn closed() -> ProtoError {
ProtoError::Io(said!("the server closed the connection"))
}
fn io_error(e: &std::io::Error) -> ProtoError {
match e.kind() {
ErrorKind::TimedOut | ErrorKind::WouldBlock => ProtoError::Io(said!("the server took too long to answer")),
ErrorKind::UnexpectedEof | ErrorKind::ConnectionAborted | ErrorKind::ConnectionReset | ErrorKind::BrokenPipe => closed(),
// The system's own words, as it said them.
_ => ProtoError::Io(crate::tidy(&e.to_string()).into()),
}
}
impl<S> Wire<S> {
pub(crate) fn new(stream: S) -> Self {
Wire { stream, buf: Vec::new(), start: 0, poll_scanned: 0 }
}
/// Whether the server sent more than has been read so far.
pub(crate) fn has_pending(&self) -> bool {
self.start < self.buf.len()
}
pub(crate) fn into_inner(self) -> S {
self.stream
}
/// The stream itself, for its owner to set up (what a socket waits for, say). Reading from
/// it or writing to it here would take the session apart.
pub(crate) fn get_mut(&mut self) -> &mut S {
&mut self.stream
}
}
impl<S: Read + Write> Wire<S> {
/// Read some more into the buffer; the end of the stream is an error.
fn fill(&mut self) -> Result<(), ProtoError> {
self.poll_scanned = 0;
if self.start > 0 {
self.buf.drain(..self.start.min(self.buf.len()));
self.start = 0;
}
let mut tmp = vec![0u8; CHUNK];
let n = read_some(&mut self.stream, &mut tmp)?;
self.buf.extend_from_slice(tmp.get(..n).unwrap_or(&[]));
Ok(())
}
/// The next line without its line ending (CRLF, or a bare LF).
pub(crate) fn read_line(&mut self) -> Result<Vec<u8>, ProtoError> {
self.next_line(false)?.ok_or_else(too_long)
}
/// The next line, or `None` for an untagged one (`* …`) longer than [`MAX_LINE`], which is
/// read past to its end (at most [`MAX_SKIPPED`] bytes of it) so that the session goes on.
/// Any other line that long is an error, as for [`Wire::read_line`].
pub(crate) fn read_line_or_skip(&mut self) -> Result<Option<Vec<u8>>, ProtoError> {
self.next_line(true)
}
fn next_line(&mut self, skip: bool) -> Result<Option<Vec<u8>>, ProtoError> {
self.poll_scanned = 0;
// How much of the pending bytes has been searched for a line feed already.
let mut scanned = 0usize;
// Bytes of an over-long line dropped so far, when one is being read past.
let mut skipped: Option<usize> = None;
loop {
let pending = self.buf.get(self.start..).unwrap_or(&[]);
if let Some(i) = pending.get(scanned..).and_then(|rest| rest.iter().position(|b| *b == b'\n')) {
let end = scanned.saturating_add(i);
let line = pending.get(..end).unwrap_or(&[]);
let line = line.strip_suffix(b"\r").unwrap_or(line);
let line = if skipped.is_some() {
None
} else if line.len() > MAX_LINE {
if !(skip && line.starts_with(b"* ")) {
return Err(too_long());
}
None
} else {
Some(line.to_vec())
};
self.start = self.start.saturating_add(end).saturating_add(1);
return Ok(line);
}
scanned = pending.len();
// The limit is on the line itself; a CR may still come on top.
if scanned > MAX_LINE + 1 {
let dropped = match skipped {
Some(n) => n,
None if skip && pending.starts_with(b"* ") => 0,
None => return Err(too_long()),
};
let dropped = dropped.saturating_add(scanned);
if dropped > MAX_SKIPPED {
return Err(too_long());
}
skipped = Some(dropped);
self.start = self.buf.len();
scanned = 0;
}
// Each round reads at least one byte, so this ends at the line feed, the cap or the end of the stream.
self.fill()?;
}
}
/// The next line after at most one underlying read; `None` for a partial line or when
/// the stream times out (IDLE waits like this, a second at a time, so that its owner can
/// look at the clock and at whether it was told to stop). Bytes of a line that came so far are
/// kept for the next call. A line over [`MAX_LINE`] is an error, as for [`Wire::read_line`].
pub(crate) fn poll_line(&mut self) -> Result<Option<Vec<u8>>, ProtoError> {
let mut read_once = false;
loop {
let pending = self.buf.get(self.start..).unwrap_or(&[]);
let scanned = self.poll_scanned.min(pending.len());
if let Some(offset) = pending.get(scanned..).unwrap_or(&[]).iter().position(|b| *b == b'\n') {
let end = scanned.saturating_add(offset);
let line = pending.get(..end).unwrap_or(&[]);
let line = line.strip_suffix(b"\r").unwrap_or(line);
if line.len() > MAX_LINE {
return Err(too_long());
}
let line = line.to_vec();
self.start = self.start.saturating_add(end).saturating_add(1);
self.poll_scanned = 0;
return Ok(Some(line));
}
if pending.len() > MAX_LINE + 1 {
return Err(too_long());
}
self.poll_scanned = pending.len();
// One underlying read per poll lets the caller check cancellation even when
// a hostile peer trickles bytes without ever triggering the read timeout.
if read_once {
return Ok(None);
}
if self.start > 0 {
self.buf.drain(..self.start.min(self.buf.len()));
self.start = 0;
}
let mut tmp = vec![0u8; CHUNK];
read_once = true;
match self.stream.read(&mut tmp) {
Ok(0) => return Err(closed()),
Ok(n) => self.buf.extend_from_slice(tmp.get(..n.min(tmp.len())).unwrap_or(&[])),
Err(e) if matches!(e.kind(), ErrorKind::TimedOut | ErrorKind::WouldBlock | ErrorKind::Interrupted) => return Ok(None),
Err(e) => return Err(io_error(&e)),
}
}
}
/// Exactly `n` bytes (at most [`MAX_LITERAL`]).
pub(crate) fn read_block(&mut self, n: usize) -> Result<Vec<u8>, ProtoError> {
self.poll_scanned = 0;
if n > MAX_LITERAL {
return Err(ProtoError::Protocol(said!("the server sent {n} bytes in one piece, more than MailCraft accepts", n)));
}
// The size is the server's word: memory is taken as the bytes arrive.
let mut out = Vec::with_capacity(n.min(CHUNK));
let pending = self.buf.get(self.start..).unwrap_or(&[]);
let take = pending.len().min(n);
out.extend_from_slice(pending.get(..take).unwrap_or(&[]));
self.start = self.start.saturating_add(take);
let mut tmp = vec![0u8; CHUNK];
while out.len() < n {
let want = (n - out.len()).min(tmp.len());
let got = read_some(&mut self.stream, tmp.get_mut(..want).unwrap_or(&mut []))?;
out.extend_from_slice(tmp.get(..got.min(want)).unwrap_or(&[]));
}
Ok(out)
}
pub(crate) fn write(&mut self, mut bytes: &[u8]) -> Result<(), ProtoError> {
// Not `write_all`: that retries an interrupted write for ever.
let mut interrupts = 0;
while !bytes.is_empty() {
match self.stream.write(bytes) {
Ok(0) => return Err(closed()),
Ok(n) => bytes = bytes.get(n..).unwrap_or(&[]),
Err(e) if e.kind() == ErrorKind::Interrupted && interrupts < MAX_INTERRUPTS => interrupts += 1,
Err(e) => return Err(io_error(&e)),
}
}
Ok(())
}
pub(crate) fn flush(&mut self) -> Result<(), ProtoError> {
self.stream.flush().map_err(|e| io_error(&e))
}
}
fn too_long() -> ProtoError {
ProtoError::Protocol(said!("the server sent a line longer than {kb} KB", kb = MAX_LINE / 1024))
}
/// One read of at least one byte; the end of the stream is an error.
fn read_some<S: Read>(stream: &mut S, buf: &mut [u8]) -> Result<usize, ProtoError> {
let mut interrupts = 0;
loop {
match stream.read(buf) {
Ok(0) => return Err(closed()),
// A stream that claims more than the buffer holds is lying.
Ok(n) => return Ok(n.min(buf.len())),
Err(e) if e.kind() == ErrorKind::Interrupted && interrupts < MAX_INTERRUPTS => interrupts += 1,
Err(e) => return Err(io_error(&e)),
}
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::script::{Broken, Script};
#[test]
fn polling_reads_once_and_retains_linear_scan_progress() {
struct ByteAtATime {
bytes: std::io::Cursor<Vec<u8>>,
reads: usize,
}
impl Read for ByteAtATime {
fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
self.reads += 1;
self.bytes.read(&mut out[..1])
}
}
impl Write for ByteAtATime {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
let mut wire = Wire::new(ByteAtATime { bytes: std::io::Cursor::new(b"abc\n".to_vec()), reads: 0 });
for scanned in 1..=3 {
assert_eq!(wire.poll_line().unwrap(), None);
assert_eq!(wire.poll_scanned, scanned);
assert_eq!(wire.get_mut().reads, scanned);
}
assert_eq!(wire.poll_line().unwrap(), Some(b"abc".to_vec()));
assert_eq!(wire.get_mut().reads, 4);
assert_eq!(wire.poll_scanned, 0);
}
#[test]
fn lines_and_blocks_arrive_in_pieces() {
let mut s = Script::new("first\r\nsecond\nab\r\ncd\r\nlast\r\n");
let mut w = Wire::new(&mut s);
assert_eq!(w.read_line().unwrap(), b"first");
assert_eq!(w.read_line().unwrap(), b"second");
assert_eq!(w.read_block(6).unwrap(), b"ab\r\ncd");
assert_eq!(w.read_block(0).unwrap(), b"");
assert_eq!(w.read_line().unwrap(), b"");
assert_eq!(w.read_line().unwrap(), b"last");
assert!(!w.has_pending());
}
#[test]
fn the_end_of_the_stream_is_an_error() {
let mut s = Script::new("no line ending");
let mut w = Wire::new(&mut s);
assert_eq!(w.read_line(), Err(ProtoError::Io(said!("the server closed the connection"))));
let mut s = Script::new("short");
assert!(matches!(Wire::new(&mut s).read_block(10), Err(ProtoError::Io(_))));
}
#[test]
fn long_lines_and_large_blocks_are_refused() {
let mut s = Script::whole(vec![b'x'; MAX_LINE + CHUNK * 2]);
assert!(matches!(Wire::new(&mut s).read_line(), Err(ProtoError::Protocol(_))));
// A line of exactly the limit is fine.
let mut ok = vec![b'x'; MAX_LINE];
ok.extend_from_slice(b"\r\n");
let mut s = Script::whole(ok);
assert_eq!(Wire::new(&mut s).read_line().unwrap().len(), MAX_LINE);
// The claim alone is refused: nothing is read or allocated.
let mut s = Script::new("");
assert!(matches!(Wire::new(&mut s).read_block(MAX_LITERAL + 1), Err(ProtoError::Protocol(_))));
}
#[test]
fn stream_errors_become_connection_problems() {
let mut w = Wire::new(Broken(ErrorKind::TimedOut));
assert_eq!(w.read_line(), Err(ProtoError::Io(said!("the server took too long to answer"))));
assert!(matches!(w.write(b"x"), Err(ProtoError::Io(_))));
// A stream that is interrupted for ever hangs neither the reader nor the writer.
assert!(matches!(Wire::new(Broken(ErrorKind::Interrupted)).read_line(), Err(ProtoError::Io(_))));
assert!(matches!(Wire::new(Broken(ErrorKind::Interrupted)).write(b"x"), Err(ProtoError::Io(_))));
}
}
+84
View File
@@ -0,0 +1,84 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
//! Test streams: a scripted server, and a stream that only fails.
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use std::io::{Error, ErrorKind, Read, Write};
/// A canned server: reads hand out what the server "says" (a few bytes at a time, so every
/// boundary gets split somewhere), writes are kept for the test to look at.
#[derive(Debug)]
pub(crate) struct Script {
input: Vec<u8>,
pos: usize,
step: usize,
trickle: bool,
pub(crate) written: Vec<u8>,
}
impl Script {
/// Hands the bytes out 1–7 at a time.
pub(crate) fn new(server: impl AsRef<[u8]>) -> Script {
Script { input: server.as_ref().to_vec(), pos: 0, step: 0, trickle: true, written: Vec::new() }
}
/// Hands out as much as each read asks for.
pub(crate) fn whole(server: impl AsRef<[u8]>) -> Script {
Script { trickle: false, ..Script::new(server) }
}
/// What the client wrote, as text.
pub(crate) fn sent(&self) -> String {
String::from_utf8_lossy(&self.written).into_owned()
}
/// Whether the client read everything the script had.
pub(crate) fn finished(&self) -> bool {
self.pos >= self.input.len()
}
}
impl Read for Script {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let left = &self.input[self.pos.min(self.input.len())..];
let mut n = left.len().min(buf.len());
if self.trickle {
n = n.min([3, 1, 7, 2, 5, 4, 6][self.step % 7]);
self.step += 1;
}
buf[..n].copy_from_slice(&left[..n]);
self.pos += n;
Ok(n)
}
}
impl Write for Script {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.written.extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
/// A stream whose every read and write fails.
#[derive(Debug)]
pub(crate) struct Broken(pub(crate) ErrorKind);
impl Read for Broken {
fn read(&mut self, _: &mut [u8]) -> std::io::Result<usize> {
Err(Error::from(self.0))
}
}
impl Write for Broken {
fn write(&mut self, _: &[u8]) -> std::io::Result<usize> {
Err(Error::from(self.0))
}
fn flush(&mut self) -> std::io::Result<()> {
Err(Error::from(self.0))
}
}
+419
View File
@@ -0,0 +1,419 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
//! An SMTP client (RFC 5321) with STARTTLS (RFC 3207) and AUTH PLAIN / LOGIN / XOAUTH2 (RFC 4954).
use std::io::{Read, Write};
use base64::{Engine as _, engine::general_purpose::STANDARD};
use crate::i18n::{Arg, Text};
use crate::line::Wire;
use crate::said;
use crate::{ProtoError, Secrets, for_trace, tidy, xoauth2};
/// Most lines one reply may have.
const MAX_REPLY_LINES: usize = 256;
/// Longest command line without its line ending (RFC 5321 §4.5.3.1.4).
const MAX_COMMAND: usize = 510;
/// Longest address (RFC 5321 §4.5.3.1.3 allows 256; some servers take more).
const MAX_ADDRESS: usize = 320;
/// Most recipients one message may have.
const MAX_RECIPIENTS: usize = 10_000;
/// An SMTP session on a stream.
pub struct Smtp<S> {
wire: Wire<S>,
ext: Extensions,
secrets: Secrets,
}
/// What the server offered in its answer to EHLO.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Extensions {
pub starttls: bool,
pub auth_plain: bool,
pub auth_login: bool,
/// The server takes an OAuth access token instead of a password (SASL XOAUTH2).
pub auth_xoauth2: bool,
/// The largest message the server takes; `Some(0)` when it names no limit.
pub size: Option<u64>,
pub eight_bit_mime: bool,
}
/// Neither the stream nor anything secret is shown.
impl<S> std::fmt::Debug for Smtp<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Smtp").field("extensions", &self.ext).finish_non_exhaustive()
}
}
/// One reply: `250-first`, `250 last`.
struct Reply {
code: u16,
lines: Vec<String>,
}
impl Reply {
fn class(&self) -> u16 {
self.code / 100
}
/// `550 5.1.1 No such user`, as the server wrote it.
fn text(&self) -> String {
format!("{} {}", self.code, self.lines.join(" "))
}
}
impl<S: Read + Write> Smtp<S> {
/// Read the 220 greeting.
pub fn connect(stream: S) -> Result<Self, ProtoError> {
let mut smtp = Self::resume(stream);
let greeting = smtp.reply()?;
if greeting.code == 220 { Ok(smtp) } else { Err(smtp.refused(&greeting)) }
}
/// Continue on a stream that already greeted (after the caller wrapped it in TLS).
pub fn resume(stream: S) -> Self {
Smtp { wire: Wire::new(stream), ext: Extensions::default(), secrets: Secrets::default() }
}
/// What the last [`hello`](Self::hello) learned.
pub fn extensions(&self) -> &Extensions {
&self.ext
}
/// The stream the session runs on, for its owner to set up between commands (how long a
/// socket waits, say). Reading from it or writing to it would take the session apart.
pub fn get_mut(&mut self) -> &mut S {
self.wire.get_mut()
}
/// EHLO (falling back to HELO); remembers and returns what the server offers.
pub fn hello(&mut self, client_name: &str) -> Result<Extensions, ProtoError> {
// A name that could break the command line is not sent.
let name = if is_word(client_name) { client_name } else { "localhost" };
let reply = self.command(&format!("EHLO {name}"))?;
if reply.code == 250 {
self.ext = extensions(&reply.lines);
return Ok(self.ext.clone());
}
if reply.class() != 5 {
return Err(self.refused(&reply));
}
// A server from before 1995 (or one hiding its extensions).
let reply = self.command(&format!("HELO {name}"))?;
if reply.code != 250 {
return Err(self.refused(&reply));
}
self.ext = Extensions::default();
Ok(self.ext.clone())
}
/// Send STARTTLS and hand the stream back for the caller to wrap; call `resume` + `hello` afterwards.
pub fn starttls(mut self) -> Result<S, ProtoError> {
let reply = self.command("STARTTLS")?;
if reply.code != 220 {
return Err(self.refused(&reply));
}
// Anything after the go-ahead was slipped in before encryption started (RFC 3207 §6).
if self.wire.has_pending() {
return Err(ProtoError::Protocol(said!("the server sent data before the secure connection started")));
}
Ok(self.wire.into_inner())
}
/// AUTH PLAIN when offered, else AUTH LOGIN. 535 and friends → `ProtoError::Auth`.
pub fn auth(&mut self, user: &str, password: &str) -> Result<(), ProtoError> {
if user.contains('\0') || password.contains('\0') {
return Err(ProtoError::Auth(said!("the user name or password has a character that can't be sent")));
}
self.secrets.add(password);
if self.ext.auth_plain {
let payload = STANDARD.encode(format!("\0{user}\0{password}"));
self.secrets.add(&payload);
let line = format!("AUTH PLAIN {payload}");
let reply = if line.len() <= MAX_COMMAND {
self.secret_command(&line, "AUTH PLAIN ***")?
} else {
// Too long for one line: wait for the server to ask (RFC 4954 §4).
let ask = self.command("AUTH PLAIN")?;
if ask.code != 334 {
return Err(self.sign_in_failed(&ask));
}
self.secret_command(&payload, "***")?
};
return self.signed_in(&reply);
}
if self.ext.auth_login {
let ask = self.command("AUTH LOGIN")?;
if ask.code != 334 {
return Err(self.sign_in_failed(&ask));
}
let ask = self.secret_command(&STANDARD.encode(user), "***")?;
if ask.code != 334 {
return Err(self.sign_in_failed(&ask));
}
let encoded = STANDARD.encode(password);
self.secrets.add(&encoded);
let reply = self.secret_command(&encoded, "***")?;
return self.signed_in(&reply);
}
Err(ProtoError::Auth(said!("the server offers no way to sign in with a password")))
}
/// AUTH XOAUTH2: an OAuth access token in place of a password. A server that doesn't offer
/// it isn't shown the token. 535 and friends → `ProtoError::Auth`.
pub fn auth_xoauth2(&mut self, user: &str, token: &str) -> Result<(), ProtoError> {
let payload = xoauth2(user, token)?;
if !self.ext.auth_xoauth2 {
return Err(ProtoError::Auth(said!("the server offers no way to sign in with an access token")));
}
self.secrets.add(token);
self.secrets.add(&payload);
let line = format!("AUTH XOAUTH2 {payload}");
let reply = if line.len() <= MAX_COMMAND {
self.secret_command(&line, "AUTH XOAUTH2 ***")?
} else {
// Too long for one line, as tokens mostly are: wait for the server to ask (RFC 4954 §4).
let ask = self.command("AUTH XOAUTH2")?;
if ask.code != 334 {
return Err(self.sign_in_failed(&ask));
}
self.secret_command(&payload, "***")?
};
match reply.code {
235 => Ok(()),
334 => {
// A refusal comes in two parts: the reasons (JSON in base64), and, once an empty
// line has acknowledged them, the verdict.
let verdict = self.command("")?;
match verdict.code {
235 => Ok(()),
334 => {
let _ = self.command("*");
Err(ProtoError::Auth(said!("the server asked for more than a user name and access token")))
}
_ => Err(self.sign_in_failed(&verdict)),
}
}
_ => Err(self.sign_in_failed(&reply)),
}
}
/// MAIL FROM, RCPT TO (every recipient must be accepted), DATA with CRLF normalisation and dot-stuffing.
pub fn send(&mut self, from: &str, recipients: &[&str], message: &[u8]) -> Result<(), ProtoError> {
// Nothing is sent unless every address is safe to put on a command line.
if !from.is_empty() && !is_address(from) {
return Err(bad_address(from));
}
if recipients.is_empty() {
return Err(ProtoError::Protocol(said!("the message has no recipients")));
}
if recipients.len() > MAX_RECIPIENTS {
return Err(ProtoError::Protocol(said!("the message has too many recipients")));
}
if let Some(bad) = recipients.iter().find(|r| !is_address(r)) {
return Err(bad_address(bad));
}
let (data, size) = data_block(message);
let mut mail = format!("MAIL FROM:<{from}>");
if self.ext.size.is_some() {
mail.push_str(&format!(" SIZE={size}"));
}
if self.ext.eight_bit_mime && !message.is_ascii() {
mail.push_str(" BODY=8BITMIME");
}
let reply = self.command(&mail)?;
if reply.class() != 2 {
return Err(self.refused(&reply));
}
for rcpt in recipients {
let reply = self.command(&format!("RCPT TO:<{rcpt}>"))?;
if reply.class() != 2 {
let refused = match self.refused(&reply) {
// (The address, then what the server said of it: no words of MailCraft's to say.)
ProtoError::Rejected(text) => {
ProtoError::Rejected(Text::new("{rcpt}: {text}", vec![("rcpt", Arg::Plain(rcpt.to_string())), ("text", Arg::said(text))]))
}
other => other,
};
return Err(self.give_up(refused));
}
}
let reply = self.command("DATA")?;
if reply.code != 354 {
let refused = self.refused(&reply);
return Err(self.give_up(refused));
}
log::trace!("C: ({size} bytes)");
self.wire.write(&data)?;
self.wire.flush()?;
let reply = self.reply()?;
if reply.class() != 2 {
return Err(self.refused(&reply));
}
Ok(())
}
/// QUIT. A server that hangs up without answering has quit all the same.
pub fn quit(mut self) -> Result<(), ProtoError> {
match self.command("QUIT") {
Ok(reply) if reply.class() == 2 => Ok(()),
Ok(reply) => Err(self.refused(&reply)),
// A server that hangs up instead of saying goodbye has quit all the same.
Err(ProtoError::Io(_)) => Ok(()),
Err(e) => Err(e),
}
}
/// Forget the message being composed, so the session can go on; hands the error back.
fn give_up(&mut self, error: ProtoError) -> ProtoError {
let _ = self.command("RSET");
error
}
fn command(&mut self, line: &str) -> Result<Reply, ProtoError> {
self.secret_command(line, line)
}
/// Send `line`; the log gets `shown` instead.
fn secret_command(&mut self, line: &str, shown: &str) -> Result<Reply, ProtoError> {
log::trace!("C: {}", for_trace(&self.secrets.hide(shown)));
let mut out = Vec::with_capacity(line.len() + 2);
out.extend_from_slice(line.as_bytes());
out.extend_from_slice(b"\r\n");
self.wire.write(&out)?;
self.wire.flush()?;
self.reply()
}
fn reply(&mut self) -> Result<Reply, ProtoError> {
let mut reply = Reply { code: 0, lines: Vec::new() };
for n in 0..MAX_REPLY_LINES {
let raw = self.wire.read_line()?;
let line = String::from_utf8_lossy(&raw);
log::trace!("S: {}", for_trace(&self.secrets.hide(&line)));
// Three digits, then `-` (more lines follow), a space or nothing (the last line).
let code = line.get(..3).filter(|d| d.bytes().all(|b| b.is_ascii_digit())).and_then(|d| d.parse::<u16>().ok());
let rest = line.get(3..).unwrap_or("");
let more = match (code, rest.chars().next()) {
(Some(code), Some('-')) if n == 0 || code == reply.code => true,
(Some(code), Some(' ') | None) if n == 0 || code == reply.code => false,
_ => return Err(ProtoError::Protocol(said!("\"{line}\" is not an SMTP reply", line = tidy(&self.secrets.hide(&line))))),
};
reply.code = code.unwrap_or(0);
reply.lines.push(rest.get(1..).unwrap_or("").to_string());
if !more {
return Ok(reply);
}
}
Err(ProtoError::Protocol(said!("the server's reply has too many lines")))
}
/// The error for a reply that isn't the one hoped for.
fn refused(&self, reply: &Reply) -> ProtoError {
// Hidden before it is shortened: a secret cut in half would no longer be recognised.
let text = tidy(&self.secrets.hide(&reply.text()));
if matches!(reply.class(), 4 | 5) { ProtoError::Rejected(text.into()) } else { ProtoError::Protocol(text.into()) }
}
fn sign_in_failed(&mut self, reply: &Reply) -> ProtoError {
match self.refused(reply) {
ProtoError::Rejected(text) => ProtoError::Auth(text),
other => other,
}
}
fn signed_in(&mut self, reply: &Reply) -> Result<(), ProtoError> {
match reply.code {
235 => Ok(()),
334 => {
// Another question we have no answer to: cancel the exchange (RFC 4954 §4).
let _ = self.command("*");
Err(ProtoError::Auth(said!("the server asked for more than a user name and password")))
}
_ => Err(self.sign_in_failed(reply)),
}
}
}
/// The extensions named in an EHLO reply (its first line is the server's name).
fn extensions(lines: &[String]) -> Extensions {
let mut ext = Extensions::default();
for line in lines.iter().skip(1) {
let upper = line.to_ascii_uppercase();
// `AUTH=PLAIN LOGIN` is how some old servers spell `AUTH PLAIN LOGIN`.
let mut words = upper.split([' ', '\t', '=']).filter(|w| !w.is_empty());
match words.next() {
Some("STARTTLS") => ext.starttls = true,
Some("8BITMIME") => ext.eight_bit_mime = true,
Some("SIZE") => ext.size = Some(words.next().and_then(|n| n.parse().ok()).unwrap_or(0)),
Some("AUTH") => {
for mechanism in words {
match mechanism {
"PLAIN" => ext.auth_plain = true,
"LOGIN" => ext.auth_login = true,
"XOAUTH2" => ext.auth_xoauth2 = true,
_ => {}
}
}
}
_ => {}
}
}
ext
}
/// Visible ASCII without spaces: safe as one word of a command line.
fn is_word(s: &str) -> bool {
!s.is_empty() && s.len() <= 255 && s.bytes().all(|b| b.is_ascii_graphic())
}
/// An address that can go between `<` and `>` as it is.
fn is_address(s: &str) -> bool {
is_word(s) && s.len() <= MAX_ADDRESS && !s.contains(['<', '>'])
}
fn bad_address(address: &str) -> ProtoError {
ProtoError::Protocol(said!("\"{address}\" can't be used as an address", address = tidy(address)))
}
/// The message as DATA wants it: every line ending CRLF, a leading dot doubled, the closing
/// `.` line added. Also returns the size without the added dots and closing line (RFC 1870).
fn data_block(message: &[u8]) -> (Vec<u8>, usize) {
let mut out = Vec::with_capacity(message.len().saturating_add(message.len() / 32).saturating_add(8));
let mut dots = 0usize;
let mut line_start = true;
let mut bytes = message.iter().copied().peekable();
while let Some(b) = bytes.next() {
match b {
b'\r' => {
bytes.next_if_eq(&b'\n');
out.extend_from_slice(b"\r\n");
line_start = true;
}
b'\n' => {
out.extend_from_slice(b"\r\n");
line_start = true;
}
b'.' if line_start => {
out.extend_from_slice(b"..");
dots += 1;
line_start = false;
}
_ => {
out.push(b);
line_start = false;
}
}
}
if !line_start {
out.extend_from_slice(b"\r\n");
}
let size = out.len().saturating_sub(dots);
out.extend_from_slice(b".\r\n");
(out, size)
}
#[cfg(test)]
mod tests;
+455
View File
@@ -0,0 +1,455 @@
// SPDX-FileCopyrightText: MailCraft contributors
// SPDX-License-Identifier: MIT OR Apache-2.0
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use std::io::ErrorKind;
use proptest::prelude::*;
use super::*;
use crate::script::{Broken, Script};
const EHLO: &str =
"250-mail.example.org greets you\r\n250-SIZE 35882577\r\n250-8BITMIME\r\n250-STARTTLS\r\n250-AUTH LOGIN PLAIN XOAUTH2\r\n250 HELP\r\n";
fn rejected(r: Result<(), ProtoError>) -> String {
match r {
Err(ProtoError::Rejected(text)) => text.to_string(),
other => panic!("expected a rejection, got {other:?}"),
}
}
#[test]
fn a_whole_session() {
let mut s = Script::new(format!(
"220-mail.example.org ESMTP\r\n220 ready\r\n{EHLO}235 2.7.0 Accepted\r\n250 OK\r\n250 OK\r\n251 will forward\r\n354 go ahead\r\n250 queued as 1\r\n221 bye\r\n"
));
let mut smtp = Smtp::connect(&mut s).unwrap();
let ext = smtp.hello("[127.0.0.1]").unwrap();
assert_eq!(
ext,
Extensions { starttls: true, auth_plain: true, auth_login: true, auth_xoauth2: true, size: Some(35_882_577), eight_bit_mime: true }
);
assert_eq!(smtp.extensions(), &ext);
smtp.auth("ada@example.org", "hunter2").unwrap();
smtp.send("ada@example.org", &["bob@example.org", "carol@example.org"], b"Subject: Hi\r\n\r\nHello.\r\n").unwrap();
smtp.quit().unwrap();
assert!(s.finished());
assert_eq!(
s.sent(),
"EHLO [127.0.0.1]\r\n\
AUTH PLAIN AGFkYUBleGFtcGxlLm9yZwBodW50ZXIy\r\n\
MAIL FROM:<ada@example.org> SIZE=23\r\n\
RCPT TO:<bob@example.org>\r\n\
RCPT TO:<carol@example.org>\r\n\
DATA\r\n\
Subject: Hi\r\n\r\nHello.\r\n.\r\n\
QUIT\r\n"
);
}
#[test]
fn the_plain_payload_is_nul_user_nul_password() {
let mut s = Script::new("250-x\r\n250 AUTH PLAIN\r\n235 ok\r\n");
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
smtp.auth("grace", "pässword \"1\"").unwrap();
let sent = s.sent();
let payload = sent.lines().nth(1).unwrap().strip_prefix("AUTH PLAIN ").unwrap();
assert_eq!(STANDARD.decode(payload).unwrap(), "\0grace\0pässword \"1\"".as_bytes());
}
#[test]
fn a_long_plain_payload_waits_for_the_server_to_ask() {
let mut s = Script::new("250-x\r\n250 AUTH PLAIN\r\n334 \r\n235 ok\r\n");
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
let password = "p".repeat(600);
smtp.auth("grace", &password).unwrap();
let sent = s.sent();
let lines: Vec<&str> = sent.lines().collect();
assert_eq!(lines[1], "AUTH PLAIN");
assert_eq!(STANDARD.decode(lines[2]).unwrap(), format!("\0grace\0{password}").as_bytes());
}
#[test]
fn login_is_used_when_plain_is_not_offered() {
let mut s = Script::new("250-x\r\n250 AUTH=LOGIN CRAM-MD5\r\n334 VXNlcm5hbWU6\r\n334 UGFzc3dvcmQ6\r\n235 ok\r\n");
let mut smtp = Smtp::resume(&mut s);
let ext = smtp.hello("localhost").unwrap();
assert!(ext.auth_login && !ext.auth_plain);
smtp.auth("ada", "hunter2").unwrap();
assert_eq!(s.sent(), "EHLO localhost\r\nAUTH LOGIN\r\nYWRh\r\naHVudGVyMg==\r\n");
}
#[test]
fn a_wrong_password_is_an_auth_error_that_never_shows_it() {
// A careless server repeats what it was sent.
let echo = "535-5.7.8 no such password: hunter2\r\n535 5.7.8 (AGFkYQBodW50ZXIy)\r\n";
let mut s = Script::new(format!("250-x\r\n250 AUTH PLAIN\r\n{echo}"));
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
let err = smtp.auth("ada", "hunter2").unwrap_err();
assert_eq!(err, ProtoError::Auth("535 5.7.8 no such password: *** 5.7.8 (***)".into()));
assert!(!format!("{err} {err:?} {smtp:?}").contains("hunter2"));
// The same over LOGIN, where the password travels alone.
let mut s = Script::new("250-x\r\n250 AUTH LOGIN\r\n334 VXNlcm5hbWU6\r\n334 UGFzc3dvcmQ6\r\n535 bad aHVudGVyMg== / hunter2\r\n");
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
assert_eq!(smtp.auth("ada", "hunter2"), Err(ProtoError::Auth("535 bad *** / ***".into())));
}
#[test]
fn sign_in_needs_a_method_and_sendable_text() {
let mut s = Script::new("250-x\r\n250 AUTH XOAUTH2\r\n");
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
assert!(matches!(smtp.auth("ada", "hunter2"), Err(ProtoError::Auth(_))));
assert!(matches!(smtp.auth("ada", "hun\0ter2"), Err(ProtoError::Auth(_))));
// Neither attempt put anything on the wire.
assert_eq!(s.sent(), "EHLO localhost\r\n");
}
/// What an XOAUTH2 response says under its base64.
fn bearer(payload: &str) -> String {
String::from_utf8(STANDARD.decode(payload).unwrap()).unwrap()
}
#[test]
fn a_token_signs_in_with_xoauth2() {
let mut s = Script::new("250-x\r\n250 AUTH LOGIN XOAUTH2\r\n235 2.7.0 Authentication successful\r\n");
let mut smtp = Smtp::resume(&mut s);
assert!(smtp.hello("localhost").unwrap().auth_xoauth2);
smtp.auth_xoauth2("ada@example.org", "tok.en-123").unwrap();
assert!(s.finished());
let sent = s.sent();
let lines: Vec<&str> = sent.lines().collect();
assert_eq!(lines.len(), 2);
assert_eq!(bearer(lines[1].strip_prefix("AUTH XOAUTH2 ").unwrap()), "user=ada@example.org\x01auth=Bearer tok.en-123\x01\x01");
}
#[test]
fn a_long_token_waits_for_the_server_to_ask() {
// Real tokens run to a couple of thousand characters.
let token = "t".repeat(2_000);
let mut s = Script::new("250-x\r\n250 AUTH XOAUTH2\r\n334 \r\n235 ok\r\n");
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
smtp.auth_xoauth2("ada@example.org", &token).unwrap();
let sent = s.sent();
let lines: Vec<&str> = sent.lines().collect();
assert_eq!(lines[1], "AUTH XOAUTH2");
assert_eq!(bearer(lines[2]), format!("user=ada@example.org\x01auth=Bearer {token}\x01\x01"));
// The longest that still fits goes in one line.
let token = "t".repeat(349);
let mut s = Script::new("250-x\r\n250 AUTH XOAUTH2\r\n235 ok\r\n");
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
smtp.auth_xoauth2("ada", &token).unwrap();
let sent = s.sent();
assert_eq!((sent.lines().count(), sent.lines().nth(1).unwrap().len()), (2, MAX_COMMAND - 1));
// A server that won't ask is not shown the token.
let token = "t".repeat(2_000);
let mut s = Script::new("250-x\r\n250 AUTH XOAUTH2\r\n504 5.5.4 not that way\r\n");
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
assert_eq!(smtp.auth_xoauth2("ada", &token), Err(ProtoError::Auth("504 5.5.4 not that way".into())));
assert_eq!(s.sent(), "EHLO localhost\r\nAUTH XOAUTH2\r\n");
}
#[test]
fn a_refused_token_is_an_auth_error_that_never_shows_it() {
let payload = xoauth2("ada", "tok.en-123").unwrap();
let reasons = STANDARD.encode(r#"{"status":"401","schemes":"bearer","scope":"https://mail.example.org/"}"#);
// The reasons come first and the verdict after an empty line; a careless server repeats what it was sent.
let mut s = Script::new(format!("250-x\r\n250 AUTH XOAUTH2\r\n334 {reasons}\r\n535-5.7.3 no good: tok.en-123\r\n535 5.7.3 ({payload})\r\n"));
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
let err = smtp.auth_xoauth2("ada", "tok.en-123").unwrap_err();
assert_eq!(err, ProtoError::Auth("535 5.7.3 no good: *** 5.7.3 (***)".into()));
assert!(!format!("{err} {err:?} {smtp:?}").contains("tok.en-123"));
assert_eq!(s.sent(), format!("EHLO localhost\r\nAUTH XOAUTH2 {payload}\r\n\r\n"));
assert!(s.finished());
// The same when the token went on a line of its own.
let token = "t".repeat(2_000);
let mut s = Script::new(format!("250-x\r\n250 AUTH XOAUTH2\r\n334 \r\n334 {reasons}\r\n535 5.7.3 Authentication unsuccessful\r\n"));
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
assert_eq!(smtp.auth_xoauth2("ada", &token), Err(ProtoError::Auth("535 5.7.3 Authentication unsuccessful".into())));
assert_eq!(s.sent(), format!("EHLO localhost\r\nAUTH XOAUTH2\r\n{}\r\n\r\n", xoauth2("ada", &token).unwrap()));
assert!(s.finished());
// Refused in one word, without reasons.
let mut s = Script::new("250-x\r\n250 AUTH XOAUTH2\r\n535 5.7.3 Authentication unsuccessful\r\n");
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
assert_eq!(smtp.auth_xoauth2("ada", "tok.en-123"), Err(ProtoError::Auth("535 5.7.3 Authentication unsuccessful".into())));
assert_eq!(s.sent(), format!("EHLO localhost\r\nAUTH XOAUTH2 {payload}\r\n"));
// A question after the reasons has no answer: the exchange is cancelled.
let mut s = Script::new(format!("250-x\r\n250 AUTH XOAUTH2\r\n334 {reasons}\r\n334 more?\r\n501 cancelled\r\n"));
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
assert!(matches!(smtp.auth_xoauth2("ada", "tok.en-123"), Err(ProtoError::Auth(_))));
assert!(s.sent().ends_with("\r\n\r\n*\r\n"));
}
#[test]
fn a_token_echoed_in_a_long_reply_is_hidden_whole() {
// The reply is shortened for the error text: not before the token is out of it.
let token = format!("head.{}.tail", "x".repeat(2_000));
let mut s = Script::new(format!("250-x\r\n250 AUTH XOAUTH2\r\n334 \r\n535 5.7.3 {} bad token {token}\r\n", "no ".repeat(80)));
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
let err = smtp.auth_xoauth2("ada", &token).unwrap_err();
let shown = format!("{err} {err:?}");
assert!(matches!(err, ProtoError::Auth(_)));
assert!(!shown.contains("head.") && !shown.contains("xxxx"), "{shown}");
}
#[test]
fn a_token_needs_a_server_that_takes_one_and_sendable_text() {
let mut s = Script::new("250-x\r\n250 AUTH PLAIN LOGIN\r\n");
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
assert_eq!(smtp.auth_xoauth2("ada", "tok.en-123"), Err(ProtoError::Auth(said!("the server offers no way to sign in with an access token"))));
assert_eq!(s.sent(), "EHLO localhost\r\n");
let mut s = Script::new("250-x\r\n250 AUTH XOAUTH2\r\n235 ok\r\n");
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
for (user, token) in
[("ada", ""), ("ada", "tok\r\nQUIT"), ("ada", "tok\x01"), ("ada\x01auth=Bearer x", "tok"), ("a\0da", "tok"), ("ada\n", "tok")]
{
assert!(matches!(smtp.auth_xoauth2(user, token), Err(ProtoError::Auth(_))), "{user:?} {token:?}");
}
assert_eq!(s.sent(), "EHLO localhost\r\n");
}
#[test]
fn an_unexpected_second_question_cancels_the_sign_in() {
let mut s = Script::new("250-x\r\n250 AUTH PLAIN\r\n334 more?\r\n501 cancelled\r\n");
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
assert!(matches!(smtp.auth("ada", "hunter2"), Err(ProtoError::Auth(_))));
assert!(s.sent().ends_with("\r\n*\r\n"));
}
#[test]
fn helo_is_the_fallback() {
let mut s = Script::new("220 old.example.org\r\n502 what?\r\n250 old.example.org\r\n");
let mut smtp = Smtp::connect(&mut s).unwrap();
assert_eq!(smtp.hello("localhost").unwrap(), Extensions::default());
assert_eq!(s.sent(), "EHLO localhost\r\nHELO localhost\r\n");
// A server on its way out is not asked twice.
let mut s = Script::new("421 shutting down\r\n");
assert_eq!(Smtp::resume(&mut s).hello("localhost"), Err(ProtoError::Rejected("421 shutting down".into())));
assert_eq!(s.sent(), "EHLO localhost\r\n");
}
#[test]
fn a_client_name_that_could_break_the_line_is_replaced() {
for bad in ["", "two words", "evil\r\nMAIL FROM:<x>", "nön-ascii"] {
let mut s = Script::new("250 ok\r\n");
Smtp::resume(&mut s).hello(bad).unwrap();
assert_eq!(s.sent(), "EHLO localhost\r\n", "{bad:?}");
}
}
#[test]
fn extensions_are_read_whatever_their_case() {
let mut s = Script::new("250-x\r\n250-starttls\r\n250-Size\r\n250-auth plain\r\n250 8bitmime\r\n");
let ext = Smtp::resume(&mut s).hello("localhost").unwrap();
assert_eq!(ext, Extensions { starttls: true, auth_plain: true, auth_login: false, auth_xoauth2: false, size: Some(0), eight_bit_mime: true });
let mut s = Script::new("250-x\r\n250 auth=login xoauth2 OAUTHBEARER\r\n");
let ext = Smtp::resume(&mut s).hello("localhost").unwrap();
assert_eq!(ext, Extensions { auth_login: true, auth_xoauth2: true, ..Default::default() });
}
#[test]
fn starttls_hands_the_stream_back() {
let mut s = Script::new("220 hi\r\n250-x\r\n250 STARTTLS\r\n220 go ahead\r\n");
let mut smtp = Smtp::connect(&mut s).unwrap();
assert!(smtp.hello("localhost").unwrap().starttls);
let stream = smtp.starttls().unwrap();
assert_eq!(stream.sent(), "EHLO localhost\r\nSTARTTLS\r\n");
let mut s = Script::new("454 TLS not available\r\n");
assert_eq!(Smtp::resume(&mut s).starttls().unwrap_err(), ProtoError::Rejected("454 TLS not available".into()));
}
#[test]
fn text_slipped_in_before_tls_is_refused() {
// Both lines arrive in one read, as an attacker in the middle would send them.
let mut s = Script::whole("220 go ahead\r\n250 injected\r\n");
assert!(matches!(Smtp::resume(&mut s).starttls(), Err(ProtoError::Protocol(_))));
}
#[test]
fn data_is_normalised_and_dot_stuffed() {
let message = b"Subject: dots\n\n.leading\r\n..two\rbare CR\n.\nno dot. here\n\nlast line without an ending";
let mut s = Script::new("250 ok\r\n250 ok\r\n354 go\r\n250 ok\r\n");
let mut smtp = Smtp::resume(&mut s);
smtp.send("ada@example.org", &["bob@example.org"], message).unwrap();
let sent = s.sent();
let data = sent.split_once("DATA\r\n").unwrap().1;
assert_eq!(data, "Subject: dots\r\n\r\n..leading\r\n...two\r\nbare CR\r\n..\r\nno dot. here\r\n\r\nlast line without an ending\r\n.\r\n");
// No SIZE or BODY parameter for a server that didn't offer them.
assert!(sent.starts_with("MAIL FROM:<ada@example.org>\r\n"));
}
#[test]
fn data_blocks() {
assert_eq!(data_block(b""), (b".\r\n".to_vec(), 0));
assert_eq!(data_block(b"a"), (b"a\r\n.\r\n".to_vec(), 3));
assert_eq!(data_block(b"."), (b"..\r\n.\r\n".to_vec(), 3));
assert_eq!(data_block(b"\r\r\n\n"), (b"\r\n\r\n\r\n.\r\n".to_vec(), 6));
assert_eq!(data_block(b"a\r"), (b"a\r\n.\r\n".to_vec(), 3));
}
#[test]
fn eight_bit_mail_says_so_when_the_server_takes_it() {
let mut s = Script::new("250-x\r\n250 8BITMIME\r\n250 ok\r\n250 ok\r\n354 go\r\n250 ok\r\n250 ok\r\n250 ok\r\n354 go\r\n250 ok\r\n");
let mut smtp = Smtp::resume(&mut s);
smtp.hello("localhost").unwrap();
smtp.send("ada@example.org", &["bob@example.org"], "Subject: é\r\n\r\n".as_bytes()).unwrap();
smtp.send("", &["bob@example.org"], b"Subject: e\r\n\r\n").unwrap();
let sent = s.sent();
assert!(sent.contains("MAIL FROM:<ada@example.org> BODY=8BITMIME\r\n"));
// An empty sender is the bounce address.
assert!(sent.contains("MAIL FROM:<>\r\n"));
}
#[test]
fn a_refused_recipient_stops_the_message_and_resets() {
let mut s = Script::new("250 ok\r\n250 ok\r\n550 5.1.1 no such user\r\n250 reset\r\n");
let mut smtp = Smtp::resume(&mut s);
let text = rejected(smtp.send("ada@example.org", &["bob@example.org", "nobody@example.org", "carol@example.org"], b"x"));
assert_eq!(text, "nobody@example.org: 550 5.1.1 no such user");
assert!(s.sent().ends_with("RCPT TO:<nobody@example.org>\r\nRSET\r\n"));
}
#[test]
fn refusals_at_each_step() {
// MAIL FROM.
let mut s = Script::new("552 5.3.4 too big\r\n");
assert_eq!(rejected(Smtp::resume(&mut s).send("a@example.org", &["b@example.org"], b"x")), "552 5.3.4 too big");
// DATA.
let mut s = Script::new("250 ok\r\n250 ok\r\n554 no\r\n250 reset\r\n");
assert_eq!(rejected(Smtp::resume(&mut s).send("a@example.org", &["b@example.org"], b"x")), "554 no");
assert!(s.sent().ends_with("DATA\r\nRSET\r\n"));
// After the message.
let mut s = Script::new("250 ok\r\n250 ok\r\n354 go\r\n451 4.3.0 try later\r\n");
assert_eq!(rejected(Smtp::resume(&mut s).send("a@example.org", &["b@example.org"], b"x")), "451 4.3.0 try later");
// The greeting.
let mut s = Script::new("554 no service for you\r\n");
assert_eq!(Smtp::connect(&mut s).unwrap_err(), ProtoError::Rejected("554 no service for you".into()));
}
#[test]
fn unsafe_addresses_are_never_sent() {
let cases: [(&str, &[&str]); 6] = [
("ada@example.org", &[]),
("ada@example.org", &["bob@example.org>\r\nRCPT TO:<eve@example.org"]),
("ada@example.org", &["bob smith@example.org"]),
("ada@example.org", &[""]),
("ada@example.org\r\nDATA", &["bob@example.org"]),
("ada@example.org", &["bøb@example.org"]),
];
for (from, to) in cases {
let mut s = Script::new("250 ok\r\n250 ok\r\n354 go\r\n250 ok\r\n");
assert!(matches!(Smtp::resume(&mut s).send(from, to, b"x"), Err(ProtoError::Protocol(_))), "{from:?} {to:?}");
assert_eq!(s.sent(), "", "{from:?} {to:?}");
}
}
#[test]
fn junk_replies_are_protocol_errors() {
for junk in ["hello there\r\n", "25 ok\r\n", "250ok\r\n", "250-first\r\n251 second\r\n", "\r\n", "2éé ok\r\n"] {
let mut s = Script::new(junk);
assert!(matches!(Smtp::resume(&mut s).hello("localhost"), Err(ProtoError::Protocol(_))), "{junk:?}");
}
// A reply that never ends.
let mut s = Script::whole("250-more\r\n".repeat(MAX_REPLY_LINES + 1));
assert_eq!(Smtp::resume(&mut s).hello("localhost"), Err(ProtoError::Protocol(said!("the server's reply has too many lines"))));
// A greeting that is a success code but not 220.
let mut s = Script::new("250 hello\r\n");
assert!(matches!(Smtp::connect(&mut s), Err(ProtoError::Protocol(_))));
}
#[test]
fn a_dead_connection_is_an_io_error() {
let mut s = Script::new("250-half a rep");
assert_eq!(Smtp::resume(&mut s).hello("localhost"), Err(ProtoError::Io(said!("the server closed the connection"))));
assert!(matches!(Smtp::connect(Broken(ErrorKind::TimedOut)), Err(ProtoError::Io(_))));
assert!(matches!(Smtp::resume(Broken(ErrorKind::ConnectionReset)).send("a@example.org", &["b@example.org"], b"x"), Err(ProtoError::Io(_))));
}
#[test]
fn quitting() {
let mut s = Script::new("221 bye\r\n");
assert_eq!(Smtp::resume(&mut s).quit(), Ok(()));
// A server that just hangs up has quit too.
let mut s = Script::new("");
assert_eq!(Smtp::resume(&mut s).quit(), Ok(()));
assert_eq!(s.sent(), "QUIT\r\n");
let mut s = Script::new("500 what\r\n");
assert!(matches!(Smtp::resume(&mut s).quit(), Err(ProtoError::Rejected(_))));
}
/// Everything a client does, against whatever the "server" says.
fn drive(server: &[u8]) {
let mut s = Script::new(server);
if let Ok(mut smtp) = Smtp::connect(&mut s) {
let _ = smtp.hello("localhost");
let _ = smtp.auth("ada@example.org", "hunter2");
let _ = smtp.send("ada@example.org", &["bob@example.org"], b"Subject: x\r\n\r\n.\r\n");
let _ = smtp.quit();
}
let mut s = Script::new(server);
let mut smtp = Smtp::resume(&mut s);
smtp.ext = Extensions { auth_login: true, ..Default::default() };
let _ = smtp.auth("ada", "hunter2");
let _ = smtp.starttls();
// With a token, on one line and on its own.
for token in ["tok.en-123".to_string(), "t".repeat(700)] {
let mut s = Script::new(server);
let mut smtp = Smtp::resume(&mut s);
smtp.ext = Extensions { auth_xoauth2: true, ..Default::default() };
let _ = smtp.auth_xoauth2("ada@example.org", &token);
let _ = smtp.send("ada@example.org", &["bob@example.org"], b"x");
}
}
proptest! {
#[test]
fn arbitrary_server_bytes_never_panic_or_hang(server in proptest::collection::vec(any::<u8>(), 0..600)) {
drive(&server);
}
#[test]
fn reply_shaped_noise_never_panics(lines in proptest::collection::vec(("[0-9]{3}", "[- ]?", "[ -~]{0,30}"), 0..40)) {
let server: String = lines.iter().map(|(code, sep, text)| format!("{code}{sep}{text}\r\n")).collect();
drive(server.as_bytes());
}
#[test]
fn data_blocks_never_end_early(message in proptest::collection::vec(prop_oneof![Just(b'.'), Just(b'\r'), Just(b'\n'), any::<u8>()], 0..300)) {
let (data, size) = data_block(&message);
prop_assert!(data.ends_with(b".\r\n"));
let body = &data[..data.len() - 3];
prop_assert!(body.is_empty() || body.ends_with(b"\r\n"));
// No bare CR or LF, and no line that is just a dot before the end.
let text = String::from_utf8_lossy(body);
prop_assert_eq!(body.iter().filter(|b| **b == b'\r').count(), body.iter().filter(|b| **b == b'\n').count());
prop_assert!(!text.contains("\r\n.\r\n") && !text.starts_with(".\r\n"));
prop_assert_eq!(body.windows(2).filter(|w| w[0] == b'\r' && w[1] == b'\n').count(), body.iter().filter(|b| **b == b'\n').count());
// Taking the stuffing out gives the size that was announced.
let unstuffed: usize = body.split_inclusive(|b| *b == b'\n').map(|l| l.len() - usize::from(l.starts_with(b"."))).sum();
prop_assert_eq!(unstuffed, size);
}
}
+74 -5
View File
@@ -629,6 +629,19 @@ fn raw_ifd(tiff: &Tiff) -> Option<&Ifd> {
.max_by_key(|i| i.u64(t::IMAGE_WIDTH).unwrap_or(0).saturating_mul(i.u64(t::IMAGE_LENGTH).unwrap_or(0)))
}
fn valid_arw2_curve(curve: &[u64]) -> bool {
curve.len() == 4 && curve.iter().all(|&v| v <= 16383) && curve.windows(2).all(|v| (v[0] >> 2) < (v[1] >> 2))
}
fn declared_arw2_curve(raw: &Ifd, compression: u16, one_byte_per_sample: bool) -> Result<Option<Vec<u64>>> {
let curve = raw.u64s(TONE_CURVE);
if compression == 32767 && one_byte_per_sample && curve.as_ref().is_some_and(|curve| !valid_arw2_curve(curve)) {
// Reordering or clamping malformed breakpoints fabricates a different transfer curve.
return Err(RawError::Unsupported("Sony ARW2 with an invalid tone curve".into()));
}
Ok(curve)
}
pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawImage> {
let tiff = Tiff::parse(bytes)?;
let ifd0 = tiff.ifds.first().ok_or_else(|| RawError::Corrupt("TIFF without IFDs".into()))?;
@@ -645,6 +658,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
// archive measure exactly width x height. A200/A230/A350 files carry a strip 4-23 % larger than that (and A290/A390 one
// smaller), which is a different packing, so they are refused rather than read with the wrong layout.
let one_byte_per_sample = chunks.len() == 1 && strip_len == samples;
let declared_curve = declared_arw2_curve(raw, info.compression, one_byte_per_sample)?;
let packed12 = info.compression == 32767 && is_packed12(&info, chunks.len(), strip_len);
let (data, out_bits) = match info.compression {
7 if linear_rgb => (RawData::U16(read_ycbcr_tiles(bytes, &info, raw, mode, limits)?), 14),
@@ -720,13 +734,16 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
// agree too. Genuinely packed 12-bit strips retain their 12-bit scale.
let scale_bits = if info.compression == 1 && out_bits < 14 && strip_len >= (w * h * 2) as u64 { 14 } else { out_bits };
let cfa = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
let (cfa, assumed_cfa) = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
(Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => {
Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }
(Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }, false)
}
_ => Cfa::bayer_static("RGGB"),
_ => (Cfa::bayer_static("RGGB"), true),
};
let default_black = if scale_bits >= 14 { 512.0 } else { 128.0 };
let curve_black = (info.compression == 32767 && one_byte_per_sample)
.then(|| declared_curve.as_ref().and_then(|curve| code_curve(curve).get(256).copied()).map(f32::from))
.flatten();
let default_black = curve_black.unwrap_or(if scale_bits >= 14 { 512.0 } else { 128.0 });
// the decrypted SR2SubIFD, read once and only when a plain black level or white balance is missing
let sr2_cell = std::cell::OnceCell::new();
let sr2 = || sr2_cell.get_or_init(|| crate::sr2::SubIfd::read(bytes, ifd0, tiff.order)).as_ref();
@@ -741,7 +758,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
_ if linear_rgb => (BlackLevel::uniform(default_black + YCBCR_OFFSET), crate::LevelSource::Fallback),
_ => match (scale_bits >= 14).then(|| sr2().and_then(sr2_black)).flatten() {
Some(b) => (BlackLevel::uniform(b), crate::LevelSource::Metadata),
None => (BlackLevel::uniform(default_black), crate::LevelSource::Fallback),
None => (BlackLevel::uniform(default_black), curve_black.map_or(crate::LevelSource::Fallback, |_| crate::LevelSource::Metadata)),
},
};
// ARW6 samples are in 2 x 14-bit units, so the file's 14-bit levels double; without the tag the white is the
@@ -842,6 +859,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: LensMetadata { sony: sony_lens },
level_sources: crate::LevelSources { black: black_source, white: white_source },
notices: crate::RawNotices { assumed_cfa: assumed_cfa && !linear_rgb, ..Default::default() },
};
img.validate_for(mode)?;
Ok(img)
@@ -873,6 +891,8 @@ mod tests {
assert_eq!(header.info(), full.info());
assert_eq!((full.width, full.height, full.cpp), (4, 4, 3));
assert!(full.cfa.is_none());
assert!(!header.notices.assumed_cfa);
assert!(!full.notices.assumed_cfa);
assert_eq!(full.data.len(), 48);
assert_eq!(full.wb_multipliers, Some([1.0; 3]));
// the YCbCr data sit 512 above the recorded sensor black level; white moves with it
@@ -911,6 +931,10 @@ mod tests {
/// A 32767-compressed ARW whose single strip is `strip` bytes for a 32 x 4 image.
fn arw_with_strip(strip: usize) -> Vec<u8> {
arw_with_strip_and_cfa(strip, None)
}
fn arw_with_strip_and_cfa(strip: usize, cfa: Option<(Vec<u16>, Vec<u8>)>) -> Vec<u8> {
use craft_tiff::{IfdBuilder, ImageData, TiffWriter, Value};
let mut raw = IfdBuilder::new();
raw.set(t::MAKE, Value::Ascii("SONY".into()));
@@ -921,11 +945,37 @@ mod tests {
raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1]));
raw.set(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA]));
raw.set(t::COMPRESSION, Value::Short(vec![32767]));
if let Some((dimensions, pattern)) = cfa {
raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(dimensions));
raw.set(t::CFA_PATTERN_EP, Value::Byte(pattern));
}
raw.set(TONE_CURVE, Value::Short(vec![8000, 10400, 12900, 14100]));
raw.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![vec![0u8; strip]] });
TiffWriter::default().write(&[raw]).unwrap()
}
#[test]
fn cfa_fallback_notices_follow_missing_malformed_and_valid_source_tags() {
for (tags, expected, assumed) in [
(None, "RGGB", true),
(Some((vec![1, 4], vec![0, 1, 1, 2])), "RGGB", true),
(Some((vec![2, 2], vec![0, 1, 3, 2])), "RGGB", true),
(Some((vec![2, 2], vec![0, 1, 2])), "RGGB", true),
(Some((vec![2, 2], vec![1, 0, 2, 1])), "GRBG", false),
] {
let file = arw_with_strip_and_cfa(32 * 4, tags);
let header = decode(&file, Mode::Header, &Limits::default()).unwrap();
let full = decode(&file, Mode::Full, &Limits::default()).unwrap();
assert_eq!(header.info(), full.info());
for image in [&header, &full] {
assert_eq!(image.cfa.as_ref().unwrap().name(), expected);
assert_eq!(image.notices.assumed_cfa, assumed);
}
assert!(header.data.is_empty());
assert_eq!(full.data.len(), 32 * 4);
}
}
/// Strip byte counts that add up past `u64::MAX` (LONG8 `[u64::MAX, 1]`) are an error or a picture, not an overflow.
#[test]
fn strip_byte_counts_past_u64_max_do_not_overflow() {
@@ -1110,6 +1160,25 @@ mod tests {
let _ = code_curve(&[60000, 1, 0, 0]);
}
#[test]
fn arw2_rejects_non_increasing_declared_curve_in_header_and_full_decode() {
let valid = arw_with_strip(32 * 4);
assert!(crate::decode(&valid).is_ok());
let mut damaged = valid.clone();
let tiff = Tiff::parse(&damaged).unwrap();
let curve = raw_ifd(&tiff).unwrap().get(TONE_CURVE).unwrap();
let at = curve.offset as usize;
for (dst, value) in damaged[at..at + 8].as_chunks_mut::<2>().0.iter_mut().zip([9000u16, 8000, 12000, 13000]) {
dst.copy_from_slice(&value.to_le_bytes());
}
for mode in [Mode::Header, Mode::Full] {
assert!(matches!(decode(&damaged, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("curve")));
let image = decode(&valid, mode, &Limits::default()).unwrap();
assert_eq!(image.black.mean(), 512.0);
assert_eq!(image.level_sources.black, crate::LevelSource::Metadata, "the declared curve maps the black code");
}
}
#[test]
fn decipher_inverts_the_cube_substitution() {
let mut seen = [false; 256];
+128 -2
View File
@@ -10,8 +10,11 @@
//! - Colour filter layout: IFD#3 tag `0xc5e0` (`CR2CFAPattern` in the ExifTool EXIF tag-name docs): 1 = RGGB,
//! 2 = BGGR, 3 = GBRG, 4 = GRBG, anchored at the top-left of the full decoded sensor (masked borders included).
//! It differs by model (e.g. 3 on the 50D/60D/7D/550D/5D Mark II, 1 on the 40D/5D Mark III/6D/5DS R), so it is
//! read per file (no model table); files without the tag fall back to RGGB (storytold/lightcraft#85). The parity of the
//! read per file first. Without a known tag, observed model IDs provide the fallback; scene-derived
//! measurements are not authoritative enough to establish sensor phase. The parity of the
//! `SensorInfo` borders does *not* predict it (RGGB files come with both even and odd top borders).
//! Header-only probes do not decode pixels for this fallback: without the tag their reported CFA
//! is assumed and can refine on full decode, as indicated by `RawNotices::assumed_cfa`.
//! - `SensorInfo` (maker note `0x00e0`): sensor width/height and the left/top/right/bottom borders of the image
//! area. Light already reaches some columns left of that border (EOS 6D: masked columns 0–70, border 84; 10–36
//! such columns on every corpus body), so the masked columns that give the black level are measured from the
@@ -38,6 +41,7 @@ const SRAW_TYPE: u16 = 0xc6c5;
/// `SRAW_TYPE` value of the YCbCr frames of sRAW / mRAW (IFD2's preview image carries 3).
const SRAW_YCC: u32 = 4;
const CR2_CFA_PATTERN: u16 = 0xc5e0;
const CANON_MODEL_ID: u16 = 0x0010;
const SENSOR_INFO: u16 = 0x00e0;
const COLOR_BALANCE: u16 = 0x4001;
/// ShotInfo, whose value 7 is the white-balance setting.
@@ -61,6 +65,13 @@ fn cfa_from_tag(v: Option<u64>) -> Option<Cfa> {
Some(Cfa::bayer_static(name))
}
/// Fallback for files without CR2CFAPattern. Image content cannot establish sensor phase safely.
fn fallback_cfa(_data: &[u16], _width: usize, _area: Rect, _black: f32, _white: f32, model: Option<u64>) -> (Cfa, bool) {
// Observed CC0 raw.pixls.us captures: 5D II, 7D, 550D, 600D and 1200D.
let green_first = [0x8000_0218, 0x8000_0250, 0x8000_0270, 0x8000_0286, 0x8000_0327].into_iter().any(|id| model == Some(id));
(Cfa::bayer_static(if green_first { "GBRG" } else { "RGGB" }), true)
}
/// The 16-bit words of a maker-note array. Canon writes `ColorData` as SHORT in most models, but as UNDEFINED
/// bytes (count = byte length, e.g. 5120) in the PowerShot / EOS M models with ColorData versions -3 and -4;
/// those bytes are the same 16-bit words in the maker note's byte order, so they are paired up, not read one by
@@ -198,6 +209,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
let chunk = craft_tiff::image::Chunk { index: 0, x: 0, y: 0, width: 0, height: 0, plane: 0, offset: off, len };
let src = chunk_bytes(bytes, &chunk).ok_or_else(|| RawError::Corrupt("raw strip outside file".into()))?;
let (fw, fh, nc, prec) = ljpeg::frame_info(src)?;
let tagged_cfa = cfa_from_tag(raw.u64(CR2_CFA_PATTERN));
let frame = match mode {
Mode::Full => {
let total = (fw as u64)
@@ -270,12 +282,19 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
let color_data = mn.as_ref().and_then(|m| m.ifd.value(COLOR_BALANCE).map(|v| words(v, m.order)));
let kelvin_presets = color_data.as_deref().and_then(kelvin_presets_from_color_data);
let wb = color_data.as_deref().and_then(wb_from_color_balance).or_else(|| wb_from_wb_info(mn.as_ref()?));
let cfa = cfa_from_tag(raw.u64(CR2_CFA_PATTERN)).unwrap_or_else(|| Cfa::bayer_static("RGGB"));
let (white, white_source) = white_from_data_sourced(&active_rows(&data, width, active), prec as u32);
let (black, black_source) = match masked_columns(&data, width, active, white) {
cols if cols.len() >= 2 => black_from_columns_sourced(&data, width, cols, active.y..active.y + active.height, active),
_ => (BlackLevel::uniform(0.0), crate::LevelSource::Fallback),
};
let model = mn.as_ref().and_then(|m| m.ifd.u64(CANON_MODEL_ID));
let (cfa, assumed_cfa) = match tagged_cfa {
Some(cfa) => (cfa, false),
None => {
let (cfa, guessed) = fallback_cfa(&data, width, active, black.mean(), white, model);
(cfa, guessed || mode == Mode::Header)
}
};
let img = RawImage {
format: RawFormat::Cr2,
width,
@@ -298,6 +317,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: crate::LevelSources { black: black_source, white: white_source },
notices: crate::RawNotices { assumed_cfa, ..Default::default() },
};
img.validate_for(mode)?;
Ok(img)
@@ -397,6 +417,112 @@ pub(crate) mod tests {
assert_eq!(name(None), None);
}
#[test]
fn absent_cfa_tag_uses_safe_phase_without_shifting_for_crop_parity() {
let (w, h) = (192usize, 96usize);
for red_row in [0usize, 1] {
let samples: Vec<u16> = (0..w * h)
.map(|i| {
let (x, y) = (i % w, i / w);
let texture = ((x / 2 + y / 2) % 17) as u16;
if i % 97 == 0 {
return 13000;
}
100 + if (y & 1) == red_row {
if x & 1 == 0 { 500 + texture } else { 1000 + texture }
} else if x & 1 == 0 {
1000 + texture
} else {
667 + texture
}
})
.collect();
for (left, top) in [(16, 4), (17, 4), (16, 5), (17, 5)] {
let area = Rect::new(left, top, w - left, h - top);
let (cfa, assumed) = fallback_cfa(&samples, w, area, 100.0, 4000.0, None);
assert_eq!(cfa.name(), "RGGB");
assert!(assumed);
}
let mut bytes = synthetic_cr2_of(&samples, w, h, None, None);
let marker = [0xe0, 0xc5, 4, 0, 1, 0, 0, 0];
let at = bytes.windows(marker.len()).position(|b| b == marker).unwrap();
// The explicit tag remains authoritative even when the observed sensor phase disagrees.
bytes[at + 8..at + 12].copy_from_slice(&4u32.to_le_bytes());
let explicit = crate::decode(&bytes).unwrap();
assert_eq!(explicit.cfa.as_ref().unwrap().name(), "GRBG");
assert_eq!(explicit.info(), crate::probe_info(&bytes).unwrap());
assert!(!explicit.notices.assumed_cfa);
bytes[at..at + 2].copy_from_slice(&0xc5dfu16.to_le_bytes());
let full = crate::decode(&bytes).unwrap();
assert_eq!(full.cfa.as_ref().unwrap().name(), "RGGB");
assert!(full.notices.assumed_cfa);
let header = decode(&bytes, Mode::Header, &Limits::default()).unwrap();
assert!(header.notices.assumed_cfa);
assert!(matches!(header.data, RawData::U16(ref v) if v.is_empty()));
assert_eq!(full.data, RawData::U16(samples));
}
}
#[test]
fn inconclusive_cfa_measurement_keeps_observed_model_fallback_and_safe_geometry() {
let data = vec![512; 64 * 32];
let area = Rect::new(0, 0, 64, 32);
assert_eq!(fallback_cfa(&data, 64, area, 512.0, 16000.0, Some(0x8000_0250)).0.name(), "GBRG");
assert!(fallback_cfa(&data, 64, area, 512.0, 16000.0, Some(0x8000_0250)).1);
assert_eq!(fallback_cfa(&data, 64, area, 512.0, 16000.0, None).0.name(), "RGGB");
assert!(fallback_cfa(&data, 64, area, 512.0, 16000.0, None).1);
let gradient: Vec<u16> = (0..192 * 96)
.map(|i| {
let (x, y) = (i % 192, i / 192);
if y % 2 == 0 {
if x % 2 == 0 { 500 } else { 1000 + 8 * x as u16 }
} else if x % 2 == 0 {
1000 + 8 * x as u16
} else {
500
}
})
.collect();
let (cfa, assumed) = fallback_cfa(&gradient, 192, Rect::new(0, 0, 192, 96), 0.0, 4000.0, None);
assert_eq!(cfa.name(), "RGGB");
assert!(assumed);
let mut bytes = synthetic_cr2_of(&gradient, 192, 96, None, None);
let marker = [0xe0, 0xc5, 4, 0, 1, 0, 0, 0];
let at = bytes.windows(marker.len()).position(|b| b == marker).unwrap();
// Exercise the decoder's missing-tag fallback, not the fixture's default CFA tag.
bytes[at..at + 2].copy_from_slice(&0xc5dfu16.to_le_bytes());
let decoded = crate::decode(&bytes).unwrap();
assert_eq!(decoded.cfa.as_ref().unwrap().name(), "RGGB");
assert!(decoded.notices.assumed_cfa);
}
#[test]
fn model_cfa_fallback_is_disclosed_in_header_and_full_decodes() {
// A single inline LONG maker-note entry needs no external offsets.
let mut note = Vec::new();
note.extend_from_slice(&1u16.to_le_bytes());
note.extend_from_slice(&CANON_MODEL_ID.to_le_bytes());
note.extend_from_slice(&4u16.to_le_bytes());
note.extend_from_slice(&1u32.to_le_bytes());
note.extend_from_slice(&0x8000_0250u32.to_le_bytes());
note.extend_from_slice(&0u32.to_le_bytes());
let samples = vec![512u16; 64 * 32];
let mut bytes = synthetic_cr2_of(&samples, 64, 32, None, Some(note));
let marker = [0xe0, 0xc5, 4, 0, 1, 0, 0, 0];
let at = bytes.windows(marker.len()).position(|b| b == marker).unwrap();
// Remove only the CFA tag; the model ID and encoded samples stay intact.
bytes[at..at + 2].copy_from_slice(&0xc5dfu16.to_le_bytes());
let header = decode(&bytes, Mode::Header, &Limits::default()).unwrap();
let full = decode(&bytes, Mode::Full, &Limits::default()).unwrap();
for image in [&header, &full] {
assert_eq!(image.cfa.as_ref().unwrap().name(), "GBRG");
assert!(image.notices.assumed_cfa);
}
assert_eq!(header.info(), full.info());
assert_eq!(full.data, RawData::U16(samples));
assert!(header.data.is_empty());
}
#[test]
fn wb_probe() {
let mut v = vec![0u64; 120];
+1
View File
@@ -393,6 +393,7 @@ pub(super) fn decode(bytes: &[u8], tiff: &Tiff, raw: &Ifd, mode: Mode, limits: &
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
lens_metadata: Default::default(),
notices: Default::default(),
level_sources: Default::default(),
};
img.validate_for(mode)?;
+119 -1
View File
@@ -274,6 +274,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
camera: ifd0.map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
lens_metadata: Default::default(),
notices: Default::default(),
level_sources: Default::default(),
};
// Canon's stated white level can sit below the sensor's true saturation (EOS RP: 14558
@@ -404,6 +405,15 @@ mod tests {
}
fn track(entry: &[u8], offset: u64, size: u32) -> Vec<u8> {
let mapping = [0u32, 1, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>();
track_with_mapping(entry, offset, size, &mapping)
}
fn track_with_mapping(entry: &[u8], offset: u64, size: u32, mapping: &[u8]) -> Vec<u8> {
track_with_table(entry, offset, size, b"stsc", mapping)
}
fn track_with_table(entry: &[u8], offset: u64, size: u32, table: &[u8; 4], payload: &[u8]) -> Vec<u8> {
let mut stsd = vec![0u8; 4];
stsd.extend(1u32.to_be_bytes());
stsd.extend(entry);
@@ -413,7 +423,15 @@ mod tests {
let mut co64 = vec![0u8; 4];
co64.extend(1u32.to_be_bytes());
co64.extend(offset.to_be_bytes());
let tables = [box_bytes(b"stsd", &stsd), box_bytes(b"stsz", &stsz), box_bytes(b"co64", &co64)].concat();
let mapping = [0u32, 1, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>();
let offsets = if table == b"stco" { box_bytes(b"stco", payload) } else { box_bytes(b"co64", if table == b"co64" { payload } else { &co64 }) };
let tables = [
box_bytes(b"stsd", &stsd),
box_bytes(b"stsc", if table == b"stsc" { payload } else { &mapping }),
box_bytes(b"stsz", if table == b"stsz" { payload } else { &stsz }),
offsets,
]
.concat();
box_bytes(b"trak", &box_bytes(b"mdia", &box_bytes(b"minf", &box_bytes(b"stbl", &tables))))
}
@@ -636,6 +654,105 @@ mod tests {
assert!(matches!(decode(&missing, Mode::Header, &Limits::default()), Err(RawError::Corrupt(_))));
}
/// Independently framed constant-value CRX: each 4x4 tile has four 2x2
/// planes whose one-byte 0x1e entropy streams reconstruct sensor value8193.
fn flat_raw_track(tiles: u32) -> (Vec<u8>, Vec<u8>) {
let mut sample = Vec::new();
let marker = |out: &mut Vec<u8>, code: u16, size: u32, flags: u32| {
out.extend(code.to_be_bytes());
out.extend(8u16.to_be_bytes());
out.extend(size.to_be_bytes());
out.extend(flags.to_be_bytes());
};
for tile in 0..tiles {
marker(&mut sample, 0xff01, 4, tile << 16);
for plane in 0..4 {
marker(&mut sample, 0xff02, 1, (plane << 28) | 0x0800_0000);
marker(&mut sample, 0xff03, 1, 0x0020_0000);
}
}
let header_size = sample.len() as u32;
sample.extend(std::iter::repeat_n(0x1e, tiles as usize * 4));
let mut coding = vec![0u8; 52];
coding[..2].copy_from_slice(&0xff00u16.to_be_bytes());
coding[2..4].copy_from_slice(&48u16.to_be_bytes());
coding[4..6].copy_from_slice(&0x100u16.to_be_bytes());
for (offset, value) in [(8, tiles * 4), (12, 4), (16, 4), (20, 4), (28, header_size)] {
coding[offset..offset + 4].copy_from_slice(&value.to_be_bytes());
}
coding[24] = 14;
coding[25] = 0x40;
let mut entry = vec![0u8; 82];
entry[24..26].copy_from_slice(&((tiles * 4) as u16).to_be_bytes());
entry[26..28].copy_from_slice(&4u16.to_be_bytes());
entry.extend(box_bytes(b"CMP1", &coding));
(box_bytes(b"CRAW", &entry), sample)
}
#[test]
fn truncated_main_sample_mapping_cannot_silently_select_a_valid_reduced_raw() {
let mapping = [0u32, 1, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>();
let (reduced_entry, reduced_sample) = flat_raw_track(1);
let (main_entry, main_sample) = flat_raw_track(2);
let main_offset = 4096 + reduced_sample.len();
let file = |main_table: Option<(&[u8; 4], &[u8])>, main_first: bool| {
let reduced = track_with_mapping(&reduced_entry, 4096, reduced_sample.len() as u32, &mapping);
let tracks = if let Some((table, payload)) = main_table {
let main = track_with_table(&main_entry, main_offset as u64, main_sample.len() as u32, table, payload);
if main_first { [main, reduced].concat() } else { [reduced, main].concat() }
} else {
reduced
};
let mut bytes = box_bytes(b"ftyp", b"crx \0\0\0\x01crx isom");
bytes.extend(box_bytes(b"moov", &tracks));
assert!(bytes.len() < 4096);
bytes.resize(4096, 0);
bytes.extend_from_slice(&reduced_sample);
bytes.extend_from_slice(&main_sample);
bytes
};
let mut co64 = vec![0u8; 4];
co64.extend(1u32.to_be_bytes());
co64.extend((main_offset as u64).to_be_bytes());
let valid_tables = [
(b"stsc", mapping.clone()),
(b"stsz", [0u32, main_sample.len() as u32, 1].into_iter().flat_map(u32::to_be_bytes).collect()),
(b"co64", co64),
(b"stco", [0u32, 1, main_offset as u32].into_iter().flat_map(u32::to_be_bytes).collect()),
];
for mode in [Mode::Header, Mode::Full] {
// The smaller source is genuinely decodable, so dropping the larger
// track would return success rather than merely a different failure.
let reduced = decode(&file(None, false), mode, &Limits::default()).unwrap();
assert_eq!((reduced.width, reduced.height), (4, 4));
if mode == Mode::Full {
assert_eq!(reduced.data, RawData::U16(vec![8193; 16]));
}
for main_first in [false, true] {
for (table, payload) in &valid_tables {
let valid = decode(&file(Some((table, payload)), main_first), mode, &Limits::default()).unwrap();
assert_eq!((valid.width, valid.height), (8, 4));
if mode == Mode::Full {
assert_eq!(valid.data, RawData::U16(vec![8193; 32]));
}
for truncated in [vec![], vec![0; 4], vec![0, 0, 0, 0, 0, 0, 0, 1]] {
let bytes = file(Some((table, &truncated)), main_first);
let container = parse_cr3(&bytes).unwrap();
assert_eq!(container.tracks.len(), 2, "mapping damage cannot erase the main track");
let selected = main_track(&container.tracks).unwrap();
assert!(matches!(selected.kind, Cr3TrackKind::Raw { width: 8, height: 4, .. }));
assert!(selected.data.is_none());
assert!(selected.compression(&bytes).is_some());
assert!(
matches!(decode(&bytes, mode, &Limits::default()), Err(RawError::Corrupt(message)) if message.contains("sample")),
"{mode:?}/{table:?}: main mapping damage cannot decode the reduced source"
);
}
}
}
}
}
#[test]
fn color_data_uses_versioned_as_shot_offsets() {
for (version, wb_at, black_at, white_at) in
@@ -712,6 +829,7 @@ mod tests {
camera: Camera::default(),
cinema: Default::default(),
lens_metadata: Default::default(),
notices: Default::default(),
level_sources: Default::default(),
}
}
+1
View File
@@ -65,6 +65,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, caller: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: crate::RawNotices::default(),
};
img.validate_for(mode)?;
Ok(img)
+82 -35
View File
@@ -5,8 +5,8 @@ use crate::gaintable::GainTableMap;
use crate::profile::{HsvTable, ProfileLook, ToneCurve};
use crate::tiffraw::{Packing, read_image_in};
use crate::{
BlackLevel, Camera, Cfa, CinemaTags, ColorData, Limits, Matrix3, Mode, Orientation, RawData, RawError, RawFormat, RawImage, Rect, Result,
Timecode, opcodes,
BlackLevel, Camera, Cfa, CinemaTags, ColorData, LevelSource, LevelSources, Limits, Matrix3, Mode, Orientation, RawData, RawError, RawFormat,
RawImage, RawNotices, Rect, Result, Timecode, opcodes,
};
use craft_tiff::tags::{self as t, photometric};
use craft_tiff::{ByteOrder, Ifd, Tiff, Value};
@@ -178,7 +178,15 @@ pub(crate) fn is_plain_cfa_tiff(tiff: &Tiff, bytes: &[u8]) -> bool {
if !raw.contains(t::AS_SHOT_NEUTRAL) && !tiff.ifds.first().is_some_and(|i| i.contains(t::AS_SHOT_NEUTRAL)) {
return false;
}
let Ok(info) = raw.image() else { return false };
let Ok(info) = raw.image() else {
// Recognition must not turn a damaged, otherwise supported plain CFA
// image into an unsupported container. Let decode_as report its layout
// error; these tags alone never decompress or allocate pixel samples.
return raw.u16(t::COMPRESSION).unwrap_or(1) == t::compression::NONE
&& raw.u16(t::SAMPLES_PER_PIXEL).unwrap_or(1) == 1
&& (1..=16).contains(&raw.u16(t::BITS_PER_SAMPLE).unwrap_or(1))
&& raw.u16(t::SAMPLE_FORMAT).unwrap_or(1) != 3;
};
if info.samples_per_pixel != 1 || !(1..=16).contains(&info.bits()) || info.sample_format == 3 {
return false;
}
@@ -274,8 +282,13 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
let bits = info.bits() as u32;
// DNG 1.4 stores lossy JPEG at 8 bits per sample: the tiles are 8-bit baseline JPEGs whatever the tag says,
// and reading them as a deeper sample would put the picture at the bottom of its range
if info.compression == t::compression::LOSSY_JPEG && bits != 8 {
return Err(RawError::Unsupported(format!("lossy JPEG DNG with {bits}-bit samples (DNG allows only 8)")));
if info.compression == t::compression::LOSSY_JPEG {
if bits != 8 {
return Err(RawError::Unsupported(format!("lossy JPEG DNG with {bits}-bit samples (DNG allows only 8)")));
}
if !matches!(cpp, 1 | 3) {
return Err(RawError::Unsupported(format!("lossy JPEG DNG with {cpp} samples per pixel")));
}
}
let mut data = read_image_in(mode, bytes, &info, tiff.order, Packing::Msb, limits)?;
@@ -312,6 +325,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
// (a DOUBLE past `f32::MAX` casts to infinity, which would black the picture out: ignored like a zero one)
let mut white: Vec<f32> =
levels(t::WHITE_LEVEL).f64s(t::WHITE_LEVEL).unwrap_or_default().into_iter().map(|v| v as f32).filter(|v| v.is_finite() && *v > 0.0).collect();
let white_source = if !white.is_empty() || table_max.is_some() { LevelSource::Metadata } else { LevelSource::Fallback };
if white.is_empty() {
white = vec![default_white];
}
@@ -333,7 +347,13 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
};
// (a FLOAT / DOUBLE tag can hold NaN or infinity: such a level reads as 0, in its place)
let level = |v: f64| Some(v as f32).filter(|v| v.is_finite()).unwrap_or(0.0);
let mut values: Vec<f32> = levels(t::BLACK_LEVEL).f64s(t::BLACK_LEVEL).unwrap_or_default().into_iter().map(level).collect();
let declared_black = levels(t::BLACK_LEVEL).f64s(t::BLACK_LEVEL).unwrap_or_default();
let black_source = if !declared_black.is_empty() && declared_black.iter().all(|value| (*value as f32).is_finite()) {
LevelSource::Metadata
} else {
LevelSource::Fallback
};
let mut values: Vec<f32> = declared_black.into_iter().map(level).collect();
// A TIFF that is not a DNG may state four black levels without `BlackLevelRepeatDim`: one per site of its 2 × 2
// CFA, row-major (PhotoCraft's reading of Hasselblad files). In a DNG the count has to match the repeat.
let (br, bc) = if format != RawFormat::Dng && cpp == 1 && values.len() == 4 && !levels(t::BLACK_LEVEL).contains(t::BLACK_LEVEL_REPEAT_DIM) {
@@ -364,7 +384,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
}
_ => Rect::new(0, 0, full_w, full_h),
};
let crop = default_crop(raw, active_area);
let (crop, default_crop_adjusted) = default_crop(raw, active_area, full_w, full_h);
let (active_area, crop) = if (full_w, full_h) == (w, h) {
(active_area, crop)
} else {
@@ -378,6 +398,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
(area, scale(crop).clipped(area.width, area.height))
};
let mut assumed_cfa = false;
let cfa = if info.photometric == photometric::CFA {
if raw.u16(t::CFA_LAYOUT).unwrap_or(1) != 1 {
return Err(RawError::Unsupported("non-rectangular CFA layout".into()));
@@ -391,7 +412,10 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
let assumed = [0u8, 1, 1, 2];
let pat = match raw.bytes(t::CFA_PATTERN_EP) {
Some(p) => p,
None if format != RawFormat::Dng && (rows, cols) == (2, 2) => &assumed,
None if format != RawFormat::Dng && (rows, cols) == (2, 2) => {
assumed_cfa = true;
&assumed
}
None => return Err(RawError::Corrupt("missing CFAPattern".into())),
};
if pat.len() != rows * cols {
@@ -422,6 +446,9 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
let opcodes =
crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) };
let color = color_data(ifd0, raw, tiff.order);
let gain_table_omitted = [raw, ifd0].iter().any(|ifd| ifd.contains(t::PROFILE_GAIN_TABLE_MAP) || ifd.contains(t::PROFILE_GAIN_TABLE_MAP_2))
&& color.profile.gain_table_map.is_none();
let img = RawImage {
format,
width: w,
@@ -435,7 +462,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
active_area,
crop,
orientation: Orientation::from_exif(ifd0.u16(t::ORIENTATION).unwrap_or(1)),
color: color_data(ifd0, raw, tiff.order),
color,
wb_multipliers: None,
linearized,
pixel_aspect: default_scale_aspect(raw.f64s(t::DEFAULT_SCALE).as_deref()),
@@ -443,7 +470,8 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
camera: Camera::from_ifd0(ifd0),
cinema: cinema_tags(ifd0, raw),
lens_metadata: Default::default(),
level_sources: Default::default(),
level_sources: LevelSources { black: black_source, white: white_source },
notices: RawNotices { default_crop_adjusted, gain_table_omitted, assumed_cfa },
};
img.validate_for(mode)?;
Ok(img)
@@ -472,33 +500,45 @@ fn cinema_tags(ifd0: &Ifd, raw: &Ifd) -> CinemaTags {
}
}
/// The default crop, relative to the active area. Each tag has its own default (DNG 1.7.1, `DefaultCropOrigin`
/// and `DefaultCropSize`): the origin (0, 0), the size the whole area, so a file with only a size crops from the
/// area's corner. A tag that is present must hold two finite, non-negative values, and the crop must start inside
/// the area and be at least one pixel each way; otherwise the crop is the whole active area.
fn default_crop(raw: &Ifd, active_area: Rect) -> Rect {
/// The default crop is relative to ActiveArea. DNG 1.7.1 defines independent
/// defaults: origin (0, 0), size ImageWidth/ImageLength. An unusable optional crop
/// preserves the entire active image. Valid partly overlapping rectangles retain the
/// existing clipped-crop policy, with an explicit notice rather than silent adjustment.
fn default_crop(raw: &Ifd, active_area: Rect, full_width: usize, full_height: usize) -> (Rect, bool) {
let whole = Rect::new(0, 0, active_area.width, active_area.height);
let pair = |tag: u16, default: [f64; 2]| -> Option<[f64; 2]> {
if !raw.contains(tag) {
return Some(default);
}
match raw.f64s(tag).as_deref() {
Some([a, b]) if a.is_finite() && b.is_finite() && *a >= 0.0 && *b >= 0.0 => Some([*a, *b]),
_ => None,
}
};
let (Some([x, y]), Some([cw, ch])) =
(pair(t::DEFAULT_CROP_ORIGIN, [0.0, 0.0]), pair(t::DEFAULT_CROP_SIZE, [active_area.width as f64, active_area.height as f64]))
else {
return whole;
};
if cw < 1.0 || ch < 1.0 {
return whole;
if !raw.contains(t::DEFAULT_CROP_ORIGIN) && !raw.contains(t::DEFAULT_CROP_SIZE) {
return (whole, false);
}
// (`as usize` saturates, and `clipped` holds the origin to the area)
let crop =
Rect::new(x.round() as usize, y.round() as usize, cw.round() as usize, ch.round() as usize).clipped(active_area.width, active_area.height);
if crop.width == 0 || crop.height == 0 { whole } else { crop }
let pair = |tag: u16, default: [f64; 2]| -> Option<[usize; 2]> {
let values = if raw.contains(tag) {
match raw.f64s(tag).as_deref() {
Some([a, b]) => [*a, *b],
_ => return None,
}
} else {
default
};
// Float-to-integer casts saturate. Refuse nonfinite/unrepresentable numbers before
// casting so a hostile origin cannot become a seemingly valid clipped rectangle.
if values.iter().any(|value| !value.is_finite() || value.round() >= usize::MAX as f64) {
return None;
}
if values.iter().any(|value| *value < 0.0) {
return None;
}
Some(values.map(|value| value.round() as usize))
};
let (Some([x, y]), Some([width, height])) =
(pair(t::DEFAULT_CROP_ORIGIN, [0.0, 0.0]), pair(t::DEFAULT_CROP_SIZE, [full_width as f64, full_height as f64]))
else {
return (whole, true);
};
let requested = Rect::new(x, y, width, height);
let crop = requested.clipped(active_area.width, active_area.height);
if crop.width == 0 || crop.height == 0 {
return (whole, true);
}
(crop, crop != requested)
}
/// Pixel height over width from `DefaultScale` (horizontal, vertical): only a clear 1.5 to 4 ratio counts (the rows stretch; a wider-than-tall pixel is left alone).
@@ -509,6 +549,9 @@ fn default_scale_aspect(scale: Option<&[f64]>) -> f32 {
}
}
#[cfg(test)]
mod compat_review_tests;
#[cfg(test)]
mod tests {
#[test]
@@ -621,3 +664,7 @@ mod tests {
assert!(deinterleave(&mut hostile, 8, 6, 1, 1 << 20, 1).is_err());
}
}
#[cfg(all(test, feature = "hasselblad"))]
#[path = "dng/probe_review_tests.rs"]
mod probe_review_tests;
+120
View File
@@ -0,0 +1,120 @@
use super::*;
use craft_tiff::{Entry, IfdBuilder, ImageData, TiffWriter};
fn ifd(tags: Vec<(u16, Value)>) -> Ifd {
let mut entries: Vec<_> = tags.into_iter().map(|(tag, value)| Entry { tag, value, offset: 0 }).collect();
entries.sort_by_key(|entry| entry.tag);
Ifd { entries, ..Default::default() }
}
#[test]
fn optional_crop_adjustments_are_explicit_and_valid_clipping_is_preserved() {
let active = Rect::new(3, 2, 11, 8);
let whole = Rect::new(0, 0, 11, 8);
let clipped = ifd(vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![10, 0])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]);
assert_eq!(default_crop(&clipped, active, 16, 12), (Rect::new(10, 0, 1, 4), true));
assert_eq!(default_crop(&ifd(vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![1, 1]))]), active, 16, 12), (Rect::new(1, 1, 10, 7), true));
for fields in [
vec![(t::DEFAULT_CROP_SIZE, Value::Long(vec![4]))],
vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![0, 0])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![0, 4]))],
] {
assert_eq!(default_crop(&ifd(fields), active, 16, 12), (whole, true));
}
assert_eq!(default_crop(&Ifd::default(), active, 16, 12), (whole, false));
assert_eq!(default_crop(&ifd(vec![(t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]), active, 16, 12), (Rect::new(0, 0, 5, 4), false));
assert_eq!(
default_crop(&ifd(vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![2, 1])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]), active, 16, 12),
(Rect::new(2, 1, 5, 4), false)
);
}
#[test]
fn hostile_crop_coordinates_fall_back_explicitly_before_saturating_casts() {
for bad in [1e300, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
for tag in [t::DEFAULT_CROP_ORIGIN, t::DEFAULT_CROP_SIZE] {
let raw = ifd(vec![(tag, Value::Double(vec![bad, 1.]))]);
assert_eq!(default_crop(&raw, Rect::new(0, 0, 16, 12), 16, 12), (Rect::new(0, 0, 16, 12), true));
}
}
}
fn linear(extra: &[(u16, Value)]) -> Vec<u8> {
let mut raw = IfdBuilder::new();
for (tag, value) in [
(t::IMAGE_WIDTH, Value::Long(vec![8])),
(t::IMAGE_LENGTH, Value::Long(vec![6])),
(t::BITS_PER_SAMPLE, Value::Short(vec![16])),
(t::SAMPLES_PER_PIXEL, Value::Short(vec![1])),
(t::PHOTOMETRIC, Value::Short(vec![photometric::LINEAR_RAW])),
(t::COMPRESSION, Value::Short(vec![1])),
] {
raw.set(tag, value);
}
for (tag, value) in extra {
raw.set(*tag, value.clone());
}
raw.set_image(ImageData::Strips { rows_per_strip: 6, strips: vec![(0u16..48).flat_map(u16::to_le_bytes).collect()] });
TiffWriter::new(ByteOrder::Little, false).write(&[raw]).unwrap()
}
#[test]
fn dng_and_generic_cfa_tiff_report_actual_level_sources_instead_of_unknown() {
for format in [RawFormat::Dng, RawFormat::CfaTiff] {
let missing = decode_as(&linear(&[]), Mode::Full, format, &Limits::default()).unwrap();
assert_eq!(missing.level_sources, LevelSources { black: LevelSource::Fallback, white: LevelSource::Fallback });
let declared = decode_as(
&linear(&[(t::BLACK_LEVEL, Value::Short(vec![17])), (t::WHITE_LEVEL, Value::Short(vec![1000]))]),
Mode::Full,
format,
&Limits::default(),
)
.unwrap();
assert_eq!(declared.level_sources, LevelSources { black: LevelSource::Metadata, white: LevelSource::Metadata });
assert_eq!(declared.black.values, vec![17.]);
assert_eq!(declared.white, vec![1000.]);
let invalid = decode_as(
&linear(&[(t::BLACK_LEVEL, Value::Double(vec![f64::NAN])), (t::WHITE_LEVEL, Value::Double(vec![f64::INFINITY]))]),
Mode::Full,
format,
&Limits::default(),
)
.unwrap();
assert_eq!(invalid.level_sources, LevelSources { black: LevelSource::Fallback, white: LevelSource::Fallback });
}
let table = decode_as(&linear(&[(t::LINEARIZATION_TABLE, Value::Short(vec![0, 1000]))]), Mode::Full, RawFormat::Dng, &Limits::default()).unwrap();
assert_eq!(table.white, vec![1000.]);
assert_eq!(table.level_sources.white, LevelSource::Metadata);
}
#[test]
fn decoded_notices_record_actual_crop_gain_table_and_cfa_decisions() {
let plain = decode_as(&linear(&[]), Mode::Full, RawFormat::Dng, &Limits::default()).unwrap();
assert_eq!(plain.notices, RawNotices::default());
let adjusted = decode_as(
&linear(&[(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![7, 0])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]),
Mode::Full,
RawFormat::Dng,
&Limits::default(),
)
.unwrap();
assert_eq!(adjusted.crop, Rect::new(7, 0, 1, 4));
assert!(adjusted.notices.default_crop_adjusted);
let invalid_gain =
decode_as(&linear(&[(t::PROFILE_GAIN_TABLE_MAP, Value::Undefined(vec![0; 16]))]), Mode::Full, RawFormat::Dng, &Limits::default()).unwrap();
assert!(invalid_gain.notices.gain_table_omitted);
assert!(invalid_gain.color.profile.gain_table_map.is_none());
let assumed =
decode_as(&linear(&[(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA]))]), Mode::Full, RawFormat::CfaTiff, &Limits::default()).unwrap();
assert!(assumed.notices.assumed_cfa);
assert_eq!(assumed.cfa.as_ref().unwrap().pattern, vec![0, 1, 1, 2]);
let tagged = decode_as(
&linear(&[(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])), (t::CFA_PATTERN_EP, Value::Byte(vec![0, 1, 1, 2]))]),
Mode::Full,
RawFormat::Dng,
&Limits::default(),
)
.unwrap();
assert!(!tagged.notices.assumed_cfa);
let limits = Limits { max_samples: 47, ..Default::default() };
assert!(matches!(decode_as(&linear(&[]), Mode::Full, RawFormat::Dng, &limits), Err(RawError::Limit(_))));
}
+64
View File
@@ -0,0 +1,64 @@
use super::*;
use craft_tiff::{IfdBuilder, ImageData, TiffWriter};
#[allow(clippy::unwrap_used)] // Test fixture construction must fail the test if its independent writer fails.
fn plain() -> Vec<u8> {
let mut raw = IfdBuilder::new();
for (tag, value) in [
(t::IMAGE_WIDTH, Value::Long(vec![2])),
(t::IMAGE_LENGTH, Value::Long(vec![2])),
(t::BITS_PER_SAMPLE, Value::Short(vec![16])),
(t::SAMPLES_PER_PIXEL, Value::Short(vec![1])),
(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])),
(t::COMPRESSION, Value::Short(vec![1])),
(t::AS_SHOT_NEUTRAL, Value::Rational(vec![(1, 2), (1, 1), (2, 3)])),
] {
raw.set(tag, value);
}
raw.set_image(ImageData::Strips { rows_per_strip: 2, strips: vec![[17u16, 31, 53, 79].into_iter().flat_map(u16::to_le_bytes).collect()] });
TiffWriter::new(ByteOrder::Little, false).write(&[raw]).unwrap()
}
#[allow(clippy::unwrap_used)] // Only known tags in this module's synthetic TIFF are inspected.
fn entry(bytes: &[u8], tag: u16) -> usize {
let at = u32::from_le_bytes(bytes[4..8].try_into().unwrap()) as usize;
let count = u16::from_le_bytes(bytes[at..at + 2].try_into().unwrap()) as usize;
(0..count).map(|i| at + 2 + i * 12).find(|&i| u16::from_le_bytes(bytes[i..i + 2].try_into().unwrap()) == tag).unwrap()
}
#[test]
fn plain_cfa_probe_keeps_malformed_geometry_and_tables_in_the_supported_decoder() {
let valid = plain();
assert_eq!(crate::probe(&valid), Some(RawFormat::CfaTiff));
let decoded = crate::decode(&valid).unwrap();
assert_eq!(decoded.data, RawData::U16(vec![17, 31, 53, 79]));
for tag in [t::IMAGE_WIDTH, t::IMAGE_LENGTH] {
let mut damaged = valid.clone();
let at = entry(&damaged, tag);
damaged[at + 8..at + 12].copy_from_slice(&0u32.to_le_bytes());
assert_eq!(crate::probe(&damaged), Some(RawFormat::CfaTiff));
assert!(matches!(crate::probe_info(&damaged), Err(RawError::Tiff(_))), "zero geometry remains malformed before pixel decode");
assert!(matches!(crate::decode(&damaged), Err(RawError::Tiff(_))));
}
let mut missing = valid.clone();
let at = entry(&missing, t::STRIP_OFFSETS);
missing[at..at + 2].copy_from_slice(&0xf123u16.to_le_bytes());
assert_eq!(crate::probe(&missing), Some(RawFormat::CfaTiff));
assert!(matches!(crate::probe_info(&missing), Err(RawError::Tiff(_))));
assert!(matches!(crate::decode(&missing), Err(RawError::Tiff(_))));
}
#[test]
fn plain_cfa_probe_does_not_claim_unknown_tiff_codings_or_missing_color_contracts() {
for tag in [t::AS_SHOT_NEUTRAL, t::COMPRESSION] {
let mut unknown = plain();
let at = entry(&unknown, tag);
if tag == t::COMPRESSION {
unknown[at + 8..at + 10].copy_from_slice(&65000u16.to_le_bytes());
} else {
unknown[at..at + 2].copy_from_slice(&0xf123u16.to_le_bytes());
}
assert_eq!(crate::probe(&unknown), Some(RawFormat::OtherTiff));
assert!(matches!(crate::decode(&unknown), Err(RawError::Unsupported(_))));
}
}
+1
View File
@@ -462,6 +462,7 @@ mod tests {
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: crate::RawNotices::default(),
};
let opts = |tile: u32| DngWriteOptions { compression: DngCompression::Deflate { tile, half: false }, ..DngWriteOptions::default() };
let hostile = write_dng(&raw, &opts(u32::MAX)).expect("the clamped tile writes");
+1
View File
@@ -128,6 +128,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: crate::LevelSources { black: black_source, white: white_source },
notices: crate::RawNotices::default(),
pixel_aspect: 1.0,
};
img.validate_for(mode)?;
+1
View File
@@ -121,6 +121,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: crate::RawNotices::default(),
};
img.validate_for(mode)?;
Ok(img)
+35
View File
@@ -997,6 +997,20 @@ pub struct ColorData {
pub kelvin_presets: Option<kelvin::KelvinPresets>,
}
/// Bounded diagnostics for decisions actually made by a raw decoder. An absent optional tag
/// does not itself set a notice; hosts can translate these flags into their own warning text
/// without inferring a fallback from missing metadata or retaining source-controlled strings.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct RawNotices {
/// Explicit default-crop metadata needed correction or a fallback to the active area.
pub default_crop_adjusted: bool,
/// A source profile gain table could not be used and was omitted from the decoded image.
pub gain_table_omitted: bool,
/// The CFA layout was selected by a decoder fallback instead of usable source metadata.
pub assumed_cfa: bool,
}
/// A decoded raw image.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct RawImage {
@@ -1041,6 +1055,9 @@ pub struct RawImage {
/// measured levels are placeholders.
#[serde(default)]
pub level_sources: LevelSources,
/// Fixed-size decoder decisions, separate from ordinary missing optional metadata.
#[serde(default)]
pub notices: RawNotices,
}
impl RawImage {
@@ -1173,3 +1190,21 @@ mod checked_arithmetic_tests {
assert_eq!(Orientation::from_exif(9), Orientation::Normal);
}
}
#[cfg(test)]
mod raw_notices_tests {
use super::*;
#[test]
fn missing_notices_default_to_no_claims_and_explicit_flags_are_independent() {
type Error = serde::de::value::Error;
let empty = serde::de::value::MapDeserializer::<_, Error>::new(std::iter::empty::<(&str, bool)>());
assert_eq!(RawNotices::deserialize(empty).unwrap(), RawNotices::default());
let one = serde::de::value::MapDeserializer::<_, Error>::new([("assumed_cfa", true)].into_iter());
let notices = RawNotices::deserialize(one).unwrap();
let copied = notices;
assert_eq!(notices, copied);
assert_eq!(notices, RawNotices { assumed_cfa: true, default_crop_adjusted: false, gain_table_omitted: false });
assert!(!std::mem::needs_drop::<RawNotices>(), "decoder notices own no allocated strings");
}
}
+1
View File
@@ -128,6 +128,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: crate::LevelSources { black: crate::LevelSource::Fallback, white: white_source },
notices: crate::RawNotices::default(),
pixel_aspect: 1.0,
};
img.validate_for(mode)?;
+1
View File
@@ -170,6 +170,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: crate::LevelSources { black: crate::LevelSource::Fallback, white: white_source },
notices: crate::RawNotices::default(),
pixel_aspect: 1.0,
};
img.validate_for(mode)?;
+144 -23
View File
@@ -13,7 +13,7 @@
//! refused.
use crate::nefc;
use crate::shared::white_from_data;
use crate::shared::white_from_data_sourced;
use crate::tiffraw::{Packing, check_image, read_image};
use crate::{BlackLevel, Camera, Cfa, ColorData, Limits, Mode, OpcodeLists, Orientation, RawData, RawError, RawFormat, RawImage, Rect, Result};
use craft_tiff::image::ImageInfo;
@@ -266,35 +266,44 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
// D7500 and Z 50 store 600 / 1008 / 400 / 400 / 1008 while their darkest samples are 150 / ~252 / 99 / 99 / ~250
// (our measurements on CC0 raw.pixls.us samples; the same bodies' 14-bit files match the tag as is).
let black_scale = if bits == 12 { 0.25 } else { 1.0 };
let black = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_LEVEL)).as_deref() {
Some([a, b, c, d]) if [a, b, c, d].iter().all(|v| **v < 16384.0) => BlackLevel {
repeat_rows: 2,
repeat_cols: 2,
values: [a, b, c, d].iter().map(|v| (**v * black_scale) as f32).collect(),
..Default::default()
let (black, black_source) = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_LEVEL)).as_deref() {
Some([a, b, c, d]) if [a, b, c, d].iter().all(|v| **v < 16384.0) => (
BlackLevel {
repeat_rows: 2,
repeat_cols: 2,
values: [a, b, c, d].iter().map(|v| (**v * black_scale) as f32).collect(),
..Default::default()
},
crate::LevelSource::Metadata,
),
_ => match nrw_black(mn.as_ref(), bits) {
Some(value) => (BlackLevel::uniform(value), crate::LevelSource::Metadata),
None => {
let (value, source) = be16_black(&info, bytes.len() as u64, mn.as_ref(), samples);
(BlackLevel::uniform(value), source)
}
},
_ => BlackLevel::uniform(nrw_black(mn.as_ref(), bits).unwrap_or_else(|| be16_black(&info, bytes.len() as u64, mn.as_ref(), samples))),
};
let wb = mn
.as_ref()
.and_then(|m| m.ifd.f64s(WB_RB_LEVELS))
.filter(|v| v.len() >= 2 && v[0] > 0.1 && v[1] > 0.1 && v[0] < 10.0 && v[1] < 10.0)
.map(|v| [v[0] as f32, 1.0, v[1] as f32]);
let cfa = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
let (cfa, assumed_cfa) = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
(Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => {
Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }
(Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }, false)
}
_ => Cfa::bayer_static("RGGB"),
_ => (Cfa::bayer_static("RGGB"), true),
};
let model = ifd0.string(t::MODEL).unwrap_or_default();
let body = measured_body(&model, w, h, bits);
let active_w = trailing_masked_columns(samples, w, bits);
// the white level is measured from the whole image (or known for a measured body); the probe (no samples in its
// result) takes the nominal one
let white = match (body.and_then(|b| b.white), mode) {
(Some(white), _) => white,
(None, Mode::Full) => white_from_data(samples, bits),
(None, Mode::Header) => ((1u32 << bits.clamp(1, 16)) - 1) as f32,
let (white, white_source) = match (body.and_then(|b| b.white), mode) {
(Some(white), _) => (white, crate::LevelSource::Fallback),
(None, Mode::Full) => white_from_data_sourced(samples, bits),
(None, Mode::Header) => (((1u32 << bits.clamp(1, 16)) - 1) as f32, crate::LevelSource::Fallback),
};
let crop = match body {
Some(&MeasuredBody { crop: (x, y, cw, ch), .. }) if x + cw <= active_w && y + ch <= h => Rect::new(x, y, cw, ch),
@@ -329,7 +338,8 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
level_sources: crate::LevelSources { black: black_source, white: white_source },
notices: crate::RawNotices { assumed_cfa, ..Default::default() },
};
img.validate_for(mode)?;
Ok(img)
@@ -378,8 +388,8 @@ fn uncompressed(
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum PackedLayout {
/// LSB-first bit stream of `bits` per sample (12: 3 bytes per 2 samples, 14: 7 bytes per 4), rows byte aligned.
/// 12-bit rows are `3w/2` bytes (older bodies) or padded to 16 bytes (Z 6, D6); 14-bit rows are `7w/4` bytes
/// padded to 16 bytes.
/// Rows use `ceil(bits * width / 8)` bytes, optionally padded to 16 bytes. Without an explicit packed maker
/// mode, recognition requires complete sample groups and 14-bit rows padded to 16 bytes.
Lsb { stride: usize },
/// D100: 16-byte chunks of fifteen data bytes holding ten MSB-first 12-bit samples, plus one zero pad byte. The
/// rows carry more samples than the IFD width (3040 against 3034); the extra columns are not image.
@@ -399,7 +409,11 @@ fn packed_layout(w: usize, h: usize, bits: u32, strip: u64, stated: bool) -> Opt
}
let stride = strip / h;
let tight = w.checked_mul(bits as usize)?.div_ceil(8);
let tight = if bits == 12 && w.is_multiple_of(2) || bits == 14 && w.is_multiple_of(4) { tight } else { return None };
// Unmarked data must still match a measured complete-group layout: do not guess a packing for compressed
// data. An explicit packed maker mode permits a final partial group, byte-aligned independently on each row.
if !stated && !(bits == 12 && w.is_multiple_of(2) || bits == 14 && w.is_multiple_of(4)) {
return None;
}
let padded = tight.checked_next_multiple_of(16)?;
let lsb_ok = match bits {
12 => stride == tight || stride == padded,
@@ -436,14 +450,14 @@ fn be16_layout(w: usize, h: usize, bits: u32, strip: u64) -> Option<PackedLayout
/// 223), and the other 12-bit Coolpix NRWs state 200 in their NRW block; so 200, never above the darkest sample.
const BE16_BLACK: f32 = 200.0;
fn be16_black(info: &ImageInfo, file_len: u64, mn: Option<&makernote::MakerNote>, samples: &[u16]) -> f32 {
fn be16_black(info: &ImageInfo, file_len: u64, mn: Option<&makernote::MakerNote>, samples: &[u16]) -> (f32, crate::LevelSource) {
let has_table = mn.is_some_and(|m| m.ifd.bytes(LINEARIZATION_TABLE).is_some());
let single = matches!(info.chunks(file_len).as_slice(), [c] if be16_layout(info.width as usize, info.height as usize, info.bits() as u32, c.len).is_some());
if info.compression != t::compression::NIKON || has_table || info.samples_per_pixel != 1 || !single {
return 0.0;
return (0.0, crate::LevelSource::Fallback);
}
let darkest = samples.iter().copied().min().unwrap_or(0);
BE16_BLACK.min(f32::from(darkest))
(BE16_BLACK.min(f32::from(darkest)), crate::LevelSource::Inferred)
}
/// Unpack a [`PackedLayout`] strip into `w * h` samples. Short input leaves the missing samples zero.
@@ -524,7 +538,7 @@ fn compressed(bytes: &[u8], info: &ImageInfo, mn: Option<&makernote::MakerNote>,
}
/// A "compressed" strip of packed samples: the strip size must match a measured layout exactly. `stated` is set
/// when the maker note names the packed mode, which also allows 14-bit rows without padding.
/// when the maker note names the packed mode, which also allows partial groups and 14-bit rows without padding.
fn packed(bytes: &[u8], info: &ImageInfo, rows: usize, stated: bool, limits: &Limits) -> Result<RawData> {
let (w, h) = (info.width as usize, info.height as usize);
let bits = info.bits() as u32;
@@ -663,6 +677,7 @@ fn small_raw(bytes: &[u8], tiff: &Tiff, info: &ImageInfo, strip: craft_tiff::ima
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: Default::default(),
};
img.validate_for(mode)?;
Ok(img)
@@ -725,6 +740,21 @@ mod tests {
TiffWriter::new(order, false).write(&[ifd0]).unwrap()
}
#[test]
fn level_provenance_distinguishes_missing_black_from_recorded_black() {
let strip: Vec<u8> = [1000u16; 16].into_iter().flat_map(u16::to_be_bytes).collect();
let fallback = nef(1, 12, vec![strip.clone()], 4, 4, 4);
let mut note = IfdBuilder::new();
note.set(BLACK_LEVEL, Value::Short(vec![400; 4]));
let recorded = nef_with_note(1, 12, vec![strip], 4, 4, 4, Some(note));
for mode in [Mode::Header, Mode::Full] {
let missing = decode(&fallback, mode, &Limits::default()).unwrap();
assert_eq!((missing.black.mean(), missing.level_sources.black), (0.0, crate::LevelSource::Fallback));
let stated = decode(&recorded, mode, &Limits::default()).unwrap();
assert_eq!((stated.black.mean(), stated.level_sources.black), (100.0, crate::LevelSource::Metadata));
}
}
/// A small uncompressed NEF with a real NikonNEFInfo TIFF block, without binary fixtures.
fn nikon_distortion_file(model: &str, mode: u8) -> Vec<u8> {
use crate::nef_lens::tests::{distortion_block, nef_info_blob, offset_block};
@@ -1527,4 +1557,95 @@ mod tests {
}
}
}
mod partial_row_tests {
use super::*;
// Independent wire oracle: place each sample bit directly, leaving nonzero tail bits and padding.
fn image(data: &[u16], w: usize, bits: u32, padded: bool) -> Vec<u8> {
let tight = (w * bits as usize).div_ceil(8);
let stride = if padded { tight.next_multiple_of(16) } else { tight };
let mut bytes = vec![0xA5; stride * (data.len() / w)];
for (i, &sample) in data.iter().enumerate() {
for bit in 0..bits as usize {
let at = (i % w) * bits as usize + bit;
let byte = &mut bytes[(i / w) * stride + at / 8];
*byte = (*byte & !(1 << (at % 8))) | (((sample >> bit) as u8 & 1) << (at % 8));
}
}
bytes
}
fn note(order: ByteOrder, modern: bool, mode: u16) -> IfdBuilder {
let mut n = IfdBuilder::new();
if modern {
let mut record = b"01000400\0\0".to_vec();
record.extend(if order == ByteOrder::Big { mode.to_be_bytes() } else { mode.to_le_bytes() });
n.set(NEF_COMPRESSION_RECORD, Value::Undefined(record));
} else {
n.set(NEF_COMPRESSION, Value::Short(vec![mode]));
}
n
}
#[test]
fn declared_packed_partial_rows_preserve_every_sample() {
let h = 5;
for (bits, mode) in [(12u32, 6), (14, 10)] {
for w in [1, 3, 5, 22, 23] {
let data = samples(w * h, bits);
for padded in [false, true] {
let wire = image(&data, w, bits, padded);
if w == 22 && bits == 14 {
assert_eq!(wire.len() / h, if padded { 48 } else { 39 });
}
for order in [ByteOrder::Big, ByteOrder::Little] {
for modern in [false, true] {
let file = nef_in(
order,
34713,
bits as u16,
vec![wire.clone()],
w as u32,
h as u32,
h as u32,
Some(note(order, modern, mode)),
);
let raw = crate::decode(&file).unwrap();
assert_eq!((raw.width, raw.height, raw.cpp), (w, h, 1));
assert!(!raw.notices.assumed_cfa, "valid CFA tag is retained");
let RawData::U16(got) = raw.data else { panic!("integer samples") };
assert_eq!(got, data, "{bits}-bit width {w}, padded {padded}, {order:?}, modern {modern}");
}
}
}
}
}
}
#[test]
fn partial_rows_keep_mode_depth_and_strip_guards() {
let (w, h) = (22, 5);
let data = samples(w * h, 14);
for padded in [false, true] {
let wire = image(&data, w, 14, padded);
let file = |bytes, mode: Option<u16>| {
nef_with_note(34713, 14, vec![bytes], w as u32, h as u32, h as u32, mode.map(|m| note(ByteOrder::Big, false, m)))
};
for mode in [None, Some(0), Some(6), Some(13), Some(14)] {
assert!(matches!(crate::decode(&file(wire.clone(), mode)), Err(RawError::Unsupported(_))), "mode {mode:?}");
}
let good = file(wire.clone(), Some(10));
assert!(crate::decode(&good).is_ok());
let short = truncated_tail_strip(good, wire.len() - 1);
assert!(matches!(crate::decode(&short), Err(RawError::Corrupt(_))));
for len in [wire.len() - 1, wire.len() + 1] {
assert!(matches!(crate::decode(&file(vec![0; len], Some(10))), Err(RawError::Unsupported(_))));
}
}
for (w, h, bits) in [(0, 5, 14), (22, 0, 14), (22, 5, 13), (usize::MAX, 5, 14)] {
assert!(packed_layout(w, h, bits, u64::MAX, true).is_none());
}
}
}
}
+65
View File
@@ -207,6 +207,19 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
let info = ifd0.image()?;
let (w, h) = (info.width as usize, info.height as usize);
let n = w.checked_mul(h).filter(|n| *n > 0 && *n <= limits.samples()).ok_or(RawError::Limit("image too large"))?;
// A partial TIFF directory is damage, not evidence of an unsupported compressed layout.
// Do not infer packing from an absent strip table or from clipped sensor bytes.
if info.compression == 1
&& (info.offsets.is_empty()
|| info.offsets.len() != info.byte_counts.len()
|| info
.offsets
.iter()
.zip(&info.byte_counts)
.any(|(&at, &len)| len == 0 || at.checked_add(len).is_none_or(|end| end > bytes.len() as u64)))
{
return Err(RawError::Corrupt("ORF strip table is missing or outside file".into()));
}
let chunks = info.chunks(bytes.len() as u64);
let total = chunks.iter().fold(0u64, |a, c| a.saturating_add(c.len));
let stated = cfa_from_exif(&tiff);
@@ -302,6 +315,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
}
_ => Rect::new(0, 0, w, h),
};
let assumed_cfa = stated.is_none();
let cfa = stated.unwrap_or_else(|| cfa_from_data(samples, w, active));
let (black, black_source) = match ip.as_ref().and_then(|i| i.f64s(BLACK)).as_deref() {
Some(v @ [_, _, _, _]) => {
@@ -348,6 +362,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
lens_metadata: Default::default(),
notices: crate::RawNotices { assumed_cfa, ..Default::default() },
level_sources: crate::LevelSources { black: black_source, white: white_source },
};
img.validate_for(mode)?;
@@ -416,6 +431,34 @@ mod tests {
}
}
#[test]
fn truncated_uncompressed_directory_and_strip_are_corrupt_not_compressed() {
let mut bytes = orf_with(4, 4, 16, vec![0; 32], Some(&[2, 0, 2, 0, 0, 1, 1, 2]));
// TiffWriter puts image chunks before its directory/metadata, so EOF is not a
// required sensor byte. Relocate the declared strip to EOF without changing it.
let directory = u32::from_le_bytes(bytes[4..8].try_into().unwrap()) as usize;
let count = u16::from_le_bytes(bytes[directory..directory + 2].try_into().unwrap()) as usize;
let entry = (0..count)
.map(|i| directory + 2 + i * 12)
.find(|&at| u16::from_le_bytes(bytes[at..at + 2].try_into().unwrap()) == t::STRIP_OFFSETS)
.unwrap();
let old_offset = u32::from_le_bytes(bytes[entry + 8..entry + 12].try_into().unwrap()) as usize;
let pixels = bytes[old_offset..old_offset + 32].to_vec();
let offset = bytes.len();
bytes.extend(pixels);
bytes[entry + 8..entry + 12].copy_from_slice(&(offset as u32).to_le_bytes());
assert!(crate::decode(&bytes).is_ok());
for cut in 0..bytes.len() {
for result in [crate::decode(&bytes[..cut]).map(|_| ()), crate::probe_info(&bytes[..cut]).map(|_| ())] {
assert!(!matches!(result, Err(RawError::Unsupported(ref why)) if why.contains("compressed")), "cut {cut}: {result:?}");
}
}
let result = crate::decode(&bytes[..bytes.len() - 1]);
assert!(matches!(result, Err(RawError::Corrupt(_))), "required final strip byte missing: {result:?}");
let result = crate::probe_info(&bytes[..bytes.len() - 1]);
assert!(matches!(result, Err(RawError::Corrupt(_))), "required final strip byte missing during probe: {result:?}");
}
/// Strip byte counts that add up past `u64::MAX` (LONG8 `[u64::MAX, 1]`) are an error or a picture, not an overflow.
#[test]
fn strip_byte_counts_past_u64_max_do_not_overflow() {
@@ -627,6 +670,28 @@ mod tests {
}
}
#[test]
fn cfa_notice_distinguishes_missing_and_unusable_exif_from_stated_patterns() {
let (w, h) = (160usize, 144usize);
let strip = mosaic(w, h, |cx, cy| [600, 1000, 1000, 300].map(|value| value + shade(cx, cy)));
let expected: Vec<u16> = strip.as_chunks::<2>().0.iter().map(|word| u16::from_le_bytes(*word)).collect();
for (tag, assumed, pattern) in [
(None, true, "RGGB"),
(Some(&[2, 0, 2, 0, 0, 1][..]), true, "RGGB"),
(Some(&[2, 0, 2, 0, 0, 1, 1, 3][..]), true, "RGGB"),
(Some(BGGR), false, "BGGR"),
(Some(GRBG), false, "GRBG"),
] {
let bytes = orf_with(w as u32, h as u32, 16, strip.clone(), tag);
for mode in [Mode::Header, Mode::Full] {
let image = decode(&bytes, mode, &Limits::default()).unwrap();
assert_eq!(image.cfa, Some(Cfa::bayer_static(pattern)));
assert_eq!(image.data, RawData::U16(expected.clone()), "word16 samples remain unchanged");
assert_eq!(image.notices, crate::RawNotices { assumed_cfa: assumed, ..Default::default() });
}
}
}
#[test]
fn packed_12_bit_probe_agrees_with_decode() {
let (w, h) = (16usize, 4usize);
+47 -1
View File
@@ -194,6 +194,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
Some(r) => r,
None => dark_trimmed(samples, w, hgt, black_point.as_deref().map(|v| v.iter().sum::<f64>() / 4.0), white),
};
let mut assumed_cfa = false;
let cfa = match (ifd0.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), ifd0.bytes(t::CFA_PATTERN_EP)) {
(Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => {
Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }
@@ -202,7 +203,10 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
// of the image area: the *ist D (origin 19, 11) and the 645D (80, 59) only fit unshifted. The 12-bit 10 MP
// bodies and everything from the K-3 on say RGGB; the K-7 / K-5 / 645D generation says BGGR. Files without the
// tag keep BGGR.
_ => cfa_from_exif(&tiff).unwrap_or_else(|| Cfa::bayer_static("BGGR")),
_ => cfa_from_exif(&tiff).unwrap_or_else(|| {
assumed_cfa = true;
Cfa::bayer_static("BGGR")
}),
};
let black = match black_point.as_deref() {
Some(v @ [_, _, _, _]) => {
@@ -240,6 +244,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: crate::RawNotices { assumed_cfa, ..Default::default() },
};
img.validate_for(mode)?;
Ok(img)
@@ -487,6 +492,47 @@ mod tests {
assert_eq!(img.cfa, Some(Cfa::bayer_static("BGGR")));
}
#[test]
fn fixed_cfa_fallback_is_noticed_but_either_usable_metadata_form_is_not_assumed() {
use craft_tiff::{IfdBuilder, ImageData, TiffWriter, Value};
let expected: Vec<u16> = (0..16).map(|i| 100 + i * 7).collect();
let valid_exif = &[2, 0, 2, 0, 1, 0, 2, 1][..];
let invalid_exif = &[2, 0, 2, 0, 1, 0][..];
for (primary, exif, assumed, pattern) in [
(None, None, true, "BGGR"),
(Some([0, 1, 1, 3]), None, true, "BGGR"),
(None, Some(invalid_exif), true, "BGGR"),
(Some([0, 1, 1, 3]), Some(invalid_exif), true, "BGGR"),
(Some([0, 1, 1, 3]), Some(valid_exif), false, "GRBG"),
(None, Some(valid_exif), false, "GRBG"),
(Some([0, 1, 1, 2]), None, false, "RGGB"),
(Some([0, 1, 1, 2]), Some(valid_exif), false, "RGGB"),
] {
let mut raw = IfdBuilder::new();
raw.set(t::IMAGE_WIDTH, Value::Long(vec![4]));
raw.set(t::IMAGE_LENGTH, Value::Long(vec![4]));
raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![16]));
raw.set(t::COMPRESSION, Value::Short(vec![1]));
raw.set(t::PHOTOMETRIC, Value::Short(vec![32803]));
raw.set(t::MAKE, Value::Ascii("PENTAX Corporation".into()));
if let Some(pattern) = primary {
raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2, 2]));
raw.set(t::CFA_PATTERN_EP, Value::Byte(pattern.to_vec()));
}
if let Some(pattern) = exif {
raw.set_child(t::EXIF_IFD, IfdBuilder::new().with(crate::EXIF_CFA_PATTERN, Value::Undefined(pattern.to_vec())));
}
raw.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![expected.iter().flat_map(|value| value.to_be_bytes()).collect()] });
let bytes = TiffWriter::new(ByteOrder::Big, false).write(&[raw]).unwrap();
for mode in [Mode::Header, Mode::Full] {
let image = decode(&bytes, mode, &Limits::default()).unwrap();
assert_eq!(image.cfa, Some(Cfa::bayer_static(pattern)));
assert_eq!(image.data, RawData::U16(expected.clone()));
assert_eq!(image.notices, crate::RawNotices { assumed_cfa: assumed, ..Default::default() });
}
}
}
/// A `w x h` big-endian 14-bit PEF of plain 16-bit words without area tags, whose maker note has only a
/// BlackPoint of `black`, with `px(x, y)` as the samples.
fn pef_with_black_point(w: usize, h: usize, black: u16, px: impl Fn(usize, usize) -> u16) -> Vec<u8> {
+9 -3
View File
@@ -643,9 +643,15 @@ fn cr3_preview(bytes: &[u8]) -> Option<&[u8]> {
/// A CR3's full-size JPEG track, when it holds a DCT JPEG.
#[cfg(feature = "cr3")]
fn cr3_full_jpeg(bytes: &[u8]) -> Option<&[u8]> {
let full = craft_metadata::cr3::parse_cr3(bytes)
.and_then(|c| c.tracks.iter().find(|t| t.kind == craft_metadata::cr3::Cr3TrackKind::Jpeg).and_then(|t| t.data));
full.and_then(|(at, len)| bytes.get(at..at.checked_add(len)?)).filter(|j| is_dct_jpeg(j))
let cr3 = craft_metadata::cr3::parse_cr3(bytes)?;
// An unusable preview track must not hide a later usable one. Share the marker budget
// across candidates so overlapping hostile track ranges cannot cause quadratic walks.
let mut work = CR3_MAX_WALK;
cr3.tracks.iter().filter(|t| t.kind == craft_metadata::cr3::Cr3TrackKind::Jpeg).find_map(|t| {
let (at, len) = t.data?;
let jpeg = bytes.get(at..at.checked_add(len)?)?;
first_frame_within(jpeg, &mut work).filter(|(m, _)| matches!(m, 0xc0..=0xc2)).map(|_| jpeg)
})
}
/// Marker-walk budget of the CR3 fallback: candidates can overlap (each claiming the rest of the file) and
+94 -4
View File
@@ -582,7 +582,19 @@ fn cr3_prefers_the_full_size_jpeg_track() {
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat());
let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat());
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat()))));
let trak = bx(
b"trak",
&bx(
b"mdia",
&bx(
b"minf",
&bx(
b"stbl",
&[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>()), stsz, co64].concat(),
),
),
),
);
file.extend(bx(b"moov", &trak));
file.resize(at as usize, 0);
file.extend_from_slice(&big);
@@ -592,6 +604,48 @@ fn cr3_prefers_the_full_size_jpeg_track() {
assert_eq!(embedded_preview(&file).unwrap(), small);
}
#[cfg(feature = "cr3")]
#[test]
fn cr3_skips_bad_preview_tracks_but_never_uses_sensor_tracks_as_jpeg() {
let bx = |kind: &[u8; 4], body: &[u8]| -> Vec<u8> { [&((body.len() + 8) as u32).to_be_bytes()[..], kind, body].concat() };
let full = |kind: &[u8; 4], body: &[u8]| bx(kind, &[&[0u8; 4][..], body].concat());
let track = |jpeg: bool, at: u64, len: u32| {
let mut entry = vec![0u8; 82];
if jpeg {
entry.extend(bx(b"JPEG", &[0; 4]));
}
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &entry)].concat());
let stsz = full(b"stsz", &[len.to_be_bytes(), 1u32.to_be_bytes()].concat());
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
bx(
b"trak",
&bx(
b"mdia",
&bx(
b"minf",
&bx(
b"stbl",
&[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>()), stsz, co64].concat(),
),
),
),
)
};
let good = fake_jpeg(100);
for first in [track(true, u64::MAX, 32), track(true, 4096, 32)] {
let mut bytes = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
bytes.extend(bx(b"moov", &[first, track(true, 4128, good.len() as u32)].concat()));
bytes.resize(4128, 0);
bytes.extend(&good);
assert_eq!(embedded_preview(&bytes).as_deref(), Some(good.as_slice()));
}
let mut bytes = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
bytes.extend(bx(b"moov", &track(false, 4096, good.len() as u32)));
bytes.resize(4096, 0);
bytes.extend(&good);
assert_eq!(embedded_preview(&bytes), None, "a sensor sample is not a rendered preview even if its bytes look like JPEG");
}
#[cfg(feature = "cr3")]
/// A CR3 whose `PRVW` box holds `prvw` and whose JPEG track holds `big`.
fn cr3_with(prvw_jpeg: &[u8], big: &[u8]) -> Vec<u8> {
@@ -608,7 +662,19 @@ fn cr3_with(prvw_jpeg: &[u8], big: &[u8]) -> Vec<u8> {
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat());
let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat());
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat()))));
let trak = bx(
b"trak",
&bx(
b"mdia",
&bx(
b"minf",
&bx(
b"stbl",
&[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>()), stsz, co64].concat(),
),
),
),
);
file.extend(bx(b"moov", &trak));
file.resize(at as usize, 0);
file.extend_from_slice(big);
@@ -656,7 +722,19 @@ fn the_smallest_preview_covering_the_size_is_chosen() {
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat());
let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat());
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat()))));
let trak = bx(
b"trak",
&bx(
b"mdia",
&bx(
b"minf",
&bx(
b"stbl",
&[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>()), stsz, co64].concat(),
),
),
),
);
file.extend(bx(b"moov", &trak));
file.resize(at as usize, 0);
file.extend_from_slice(&big);
@@ -769,7 +847,19 @@ fn a_cr3_with_two_prvw_boxes_offers_both() {
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat());
let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat());
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat()))));
let trak = bx(
b"trak",
&bx(
b"mdia",
&bx(
b"minf",
&bx(
b"stbl",
&[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>()), stsz, co64].concat(),
),
),
),
);
file.extend(bx(b"moov", &trak));
// (the track in the media data, as a camera writes it, after a `free` box up to it)
let pad = at as usize - 16 - file.len();
+242 -17
View File
@@ -18,7 +18,10 @@
//! - CFA: the X-Trans layout record (36 bytes, 0 = R, 1 = G, 2 = B) is stored in reverse order and, reversed,
//! is anchored at raw pixel (0, 0) — verified on X-Trans I and III samples by minimising the difference
//! between horizontally / vertically adjacent "green" samples over all 72 orientations/phases, and by colour
//! renders. Bayer bodies (X-A series) use RGGB at (0, 0), verified by colour renders.
//! renders. Bayer FujiLayout (`0x0130`) gives the reversed 2×2 pattern in its low two bits
//! (0 = R, 1/3 = G, 2 = B); absent layout records retain the assumed RGGB fallback.
//! - RawExposureBias (`0x9650`) is a signed big-endian 16-bit fraction; its negative is the
//! baseline exposure used to restore the camera's dynamic-range exposure reduction.
//! - Striped predictive compression is decoded by `rafc`; see `docs/raf-compression.md`.
use crate::shared::white_from_data_sourced;
@@ -87,7 +90,7 @@ fn pair(h: &Header, tag: u16) -> Option<(usize, usize)> {
pub(crate) fn raw_ifd(raw: &[u8]) -> Result<Ifd> {
let tiff = Tiff::parse(raw).map_err(|_| RawError::Unsupported("RAF without a raw IFD (older FinePix layout)".into()))?;
let off = tiff.ifds.first().and_then(|i| i.u64(0xf000)).ok_or_else(|| RawError::Unsupported("RAF without a raw IFD".into()))?;
let (ifd, _) = craft_tiff::parse_ifd_at(raw, off, ByteOrder::Little, 0, false, &Default::default())?;
let (ifd, _) = craft_tiff::parse_ifd_at(raw, off, tiff.order, 0, false, &Default::default())?;
Ok(ifd)
}
@@ -122,14 +125,42 @@ fn camera_name(b: &[u8]) -> String {
String::from_utf8_lossy(name.get(..end).unwrap_or_default()).trim().to_string()
}
/// The CFA: X-Trans from the (reversed) layout record, else RGGB.
fn cfa(h: &Header) -> Cfa {
match record(h, XTRANS_LAYOUT) {
Some(l) if l.len() == 36 && l.iter().all(|&c| c <= 2) => {
Cfa { width: 6, height: 6, pattern: l.iter().rev().copied().collect(), origin_x: 0, origin_y: 0 }
/// Layout records are reversed and anchored at sensor (0, 0). FujiLayout's low
/// two bits encode Bayer colour; both 1 and 3 denote green.
fn cfa(h: &Header) -> Result<Cfa> {
if let Some(l) = record(h, XTRANS_LAYOUT) {
if l.len() != 36 || l.iter().any(|&c| c > 2) {
return Err(RawError::Unsupported("Fujifilm RAF with an unknown X-Trans layout".into()));
}
_ => Cfa::bayer_static("RGGB"),
return Ok(Cfa { width: 6, height: 6, pattern: l.iter().rev().copied().collect(), origin_x: 0, origin_y: 0 });
}
let Some(l) = record(h, 0x0130) else { return Ok(Cfa::bayer_static("RGGB")) };
let cfa = Cfa { width: 2, height: 2, pattern: l.iter().rev().map(|&c| if c & 3 == 3 { 1 } else { c & 3 }).collect(), origin_x: 0, origin_y: 0 };
// Colour counts alone also accept adjacent green sites; the two greens
// must occupy opposite corners of the 2x2 Bayer repeat.
if l.len() != 4 || !matches!(cfa.pattern.as_slice(), [0, 1, 1, 2] | [2, 1, 1, 0] | [1, 0, 2, 1] | [1, 2, 0, 1]) {
return Err(RawError::Unsupported("Fujifilm RAF with an unknown Bayer layout".into()));
}
Ok(cfa)
}
/// Fuji RawExposureBias is a signed big-endian 16-bit fraction, undone like
/// DNG BaselineExposure. Invalid or implausible metadata does not alter exposure.
fn baseline_exposure(h: &Header) -> f64 {
record(h, 0x9650)
.and_then(|d| {
let num = i16::from_be_bytes(d.get(..2)?.try_into().ok()?);
let den = i16::from_be_bytes(d.get(2..4)?.try_into().ok()?);
(den != 0).then(|| -f64::from(num) / f64::from(den))
})
.filter(|v| v.is_finite() && v.abs() <= 8.0)
.unwrap_or(0.0)
}
/// The codec fixes the green-site geometry; exchanging the two non-green
/// planes only exchanges their colour labels, so RGGB and BGGR are both valid.
fn compressed_layout_supported(cfa: &Cfa) -> bool {
*cfa == Cfa::xtrans() || *cfa == Cfa::bayer_static("RGGB") || *cfa == Cfa::bayer_static("BGGR")
}
/// Unpack one row of `bits`-bit samples in Fujifilm's packing. `words32`: the raw IFD's `0xf006` flag is set, the
@@ -137,7 +168,7 @@ fn cfa(h: &Header) -> Cfa {
/// LSB-first bit stream, or (other depths) the same 32-bit words.
pub(crate) fn unpack_row(src: &[u8], bits: u32, words32: bool, out: &mut [u16]) {
match bits {
16 => out.iter_mut().zip(src.as_chunks::<2>().0).for_each(|(o, c)| *o = u16::from_le_bytes([c[0], c[1]])),
16 if !words32 => out.iter_mut().zip(src.as_chunks::<2>().0).for_each(|(o, c)| *o = u16::from_le_bytes([c[0], c[1]])),
12 if !words32 => unpack_lsb(src, 12, out),
_ => unpack_words32_msb(src, ByteOrder::Little, bits, out),
}
@@ -171,7 +202,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
let src = raw
.get(off..off.checked_add(len).ok_or_else(|| RawError::Corrupt("RAF strip size overflow".into()))?)
.ok_or_else(|| RawError::Corrupt("RAF strip outside file".into()))?;
let cfa = cfa(&h);
let cfa = cfa(&h)?;
let compression = be32(bytes, 108).unwrap_or(0);
let compressed = match compression {
0 => false,
@@ -180,8 +211,13 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
};
let mut data = Vec::new();
if compressed {
if cfa.width == 6 && cfa != Cfa::xtrans() {
return Err(RawError::Unsupported("Fujifilm compressed RAF with an unknown X-Trans layout".into()));
if !compressed_layout_supported(&cfa) {
return Err(RawError::Unsupported("Fujifilm compressed RAF with an unsupported CFA layout".into()));
}
if !src.starts_with(b"IS") {
// A recognized RAF compression mode can wrap an unknown packing. Do not
// feed it to the IS codec; damage inside an identified IS stream stays Corrupt.
return Err(RawError::Unsupported("Fujifilm compressed RAF without a supported IS signature".into()));
}
let flag = if compression == 2 { 1 } else { 0 };
if src.get(2) != Some(&flag) {
@@ -190,9 +226,18 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
data = crate::rafc::decode(src, w, hgt, bits, cfa.width == 6, mode, limits)?;
} else {
let words32 = ifd.u64(PACKED_WORDS).is_some_and(|v| v != 0);
// Samples may use 16-bit words (e.g. 14-bit X-T20); otherwise they are packed at `bits`.
let store = if len as u64 >= (n as u64) * 2 { 16 } else { bits };
let packed_row = w.checked_mul(store as usize).map(|n| n.div_ceil(8)).ok_or(RawError::Limit("RAF row too large"))?;
// An explicit packed-word flag wins over the size heuristic: padding in short
// packed rows can occupy as many bytes as unpacked 16-bit samples.
// Unflagged samples may use 16-bit words (e.g. 14-bit X-T20).
let store = if !words32 && len as u64 >= (n as u64) * 2 { 16 } else { bits };
let row_bits = w.checked_mul(store as usize).ok_or(RawError::Limit("RAF row too large"))?;
// Word-packed rows store the high bits in the last little-endian word's final bytes.
// Reading only ceil(bits / 8) bytes would lose those samples, not just padding.
let packed_row = if words32 || (store != 16 && store != 12) {
row_bits.div_ceil(32).checked_mul(4).ok_or(RawError::Limit("RAF row too large"))?
} else {
row_bits.div_ceil(8)
};
let stride = match get(0xf005)?.unwrap_or(0) {
0 => packed_row,
words => words.checked_mul(4).ok_or(RawError::Limit("RAF stride too large"))?.max(packed_row),
@@ -206,7 +251,11 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
data.par_chunks_mut(w).enumerate().for_each(|(y, row)| {
// `required <= src.len()` proves these row offsets fit, including the final row.
let s = src.get(y * stride..y * stride + packed_row).unwrap_or(&[]);
unpack_row(s, store, words32, row);
if !words32 && store == 16 && raw.starts_with(b"MM") {
crate::unpack::read_u16s(s, ByteOrder::Big, row);
} else {
unpack_row(s, store, words32, row);
}
});
}
}
@@ -246,7 +295,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
active_area: active,
crop: Rect::new(0, 0, active.width, active.height),
orientation,
color: ColorData { kelvin_presets, ..Default::default() },
color: ColorData { kelvin_presets, baseline_exposure: baseline_exposure(&h), ..Default::default() },
wb_multipliers: wb,
linearized: false,
pixel_aspect: 1.0,
@@ -256,6 +305,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
.unwrap_or_else(|| Camera { model: Some(camera_name(bytes)).filter(|n| !n.is_empty()), ..Default::default() }),
cinema: Default::default(),
lens_metadata: Default::default(),
notices: crate::RawNotices { assumed_cfa: record(&h, XTRANS_LAYOUT).is_none() && record(&h, 0x0130).is_none(), ..Default::default() },
level_sources: crate::LevelSources { black: black_source, white: white_source },
};
img.validate_for(mode)?;
@@ -480,6 +530,181 @@ pub(super) mod tests {
assert!(matches!(crate::decode(&bytes), Err(RawError::Unsupported(_))));
}
#[test]
fn word_packed_rows_keep_the_final_partial_word_and_reject_truncation() {
for (w, bits, flag) in [(30u32, 14u32, 0u32), (22, 12, 1), (6, 12, 1), (5, 16, 1)] {
let h = 6u32;
let px: Vec<u16> = (0..w * h).map(|i| ((i * 977 + 101) % (1 << bits)) as u16).collect();
// Independently place each sample bit into whole little-endian 32-bit row words.
let strip: Vec<u8> = px
.chunks(w as usize)
.flat_map(|row| {
let mut words = vec![0u32; (row.len() * bits as usize).div_ceil(32)];
for (i, &sample) in row.iter().enumerate() {
for bit in 0..bits as usize {
let at = i * bits as usize + bit;
words[at / 32] |= u32::from((sample >> (bits as usize - 1 - bit)) & 1) << (31 - at % 32);
}
}
words.into_iter().flat_map(u32::to_le_bytes).collect::<Vec<_>>()
})
.collect();
let bytes = raf_flagged(w, h, bits, strip, None, &[], flag);
assert_eq!(crate::decode(&bytes).unwrap().data, RawData::U16(px));
assert!(crate::decode(&bytes[..bytes.len() - 1]).is_err());
}
}
#[test]
fn big_endian_words_and_bayer_layout_preserve_exact_sensor_values() {
use craft_tiff::{IfdBuilder, TiffWriter, Value};
let (w, h) = (6u32, 4u32);
let px: Vec<u16> = (0..w * h).map(|i| (i * 977 + 123) as u16).collect();
for order in [ByteOrder::Little, ByteOrder::Big] {
let build = |at| {
let mut sensor = IfdBuilder::new();
for (tag, value) in [(0xf001, w), (0xf002, h), (0xf003, 16), (0xf007, at), (0xf008, w * h * 2)] {
sensor.set(tag, Value::Long(vec![value]));
}
let mut root = IfdBuilder::new();
root.set_child(0xf000, sensor);
TiffWriter::new(order, false).write(&[root]).unwrap()
};
let mut raw = build(build(0).len() as u32);
raw.extend(px.iter().flat_map(|v| if order == ByteOrder::Big { v.to_be_bytes() } else { v.to_le_bytes() }));
let mut bytes = raf(w, h, 16, vec![0; (w * h * 2) as usize], None, &[]);
let raw_at = be32(&bytes, 100).unwrap();
bytes.truncate(raw_at);
bytes[104..108].copy_from_slice(&(raw.len() as u32).to_be_bytes());
bytes.extend(raw);
// A four-site Bayer record is reversed in storage, just like the X-Trans record.
let dir_at = be32(&bytes, 92).unwrap();
let mut records = bytes[dir_at..raw_at].to_vec();
let n = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1;
records[..4].copy_from_slice(&n.to_be_bytes());
records.extend_from_slice(&[0x01, 0x30, 0, 4, 0x08, 0x09, 0x0b, 0x0a]);
bytes.splice(dir_at..raw_at, records.clone());
bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes());
bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes());
let image = crate::decode(&bytes).unwrap();
assert_eq!(image.data, RawData::U16(px.clone()));
assert_eq!(image.cfa.unwrap().name(), "BGGR");
}
}
#[test]
fn raw_exposure_bias_is_preserved_as_baseline_exposure_and_invalid_values_are_ignored() {
for (num, den, expected) in [(-72i16, 100i16, 0.72f64), (1, 0, 0.0), (32767, 1, 0.0)] {
let mut bytes = raf(6, 4, 16, pack(&[1000; 24], 16), None, &[]);
let (dir_at, raw_at) = (be32(&bytes, 92).unwrap(), be32(&bytes, 100).unwrap());
let mut records = bytes[dir_at..raw_at].to_vec();
let n = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1;
records[..4].copy_from_slice(&n.to_be_bytes());
records.extend_from_slice(&[0x96, 0x50, 0, 4]);
records.extend(num.to_be_bytes());
records.extend(den.to_be_bytes());
bytes.splice(dir_at..raw_at, records.clone());
bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes());
bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes());
for mode in [Mode::Header, Mode::Full] {
assert!((decode(&bytes, mode, &Limits::default()).unwrap().color.baseline_exposure - expected).abs() < 1e-9);
}
}
}
#[test]
fn invalid_declared_cfa_layouts_are_not_silently_replaced() {
for (tag, data) in [(0x0130, vec![8, 8, 10, 10]), (0x0130, vec![1, 2, 1, 0]), (0x0131, vec![3; 36])] {
let header = Header { jpeg: None, records: vec![(tag, data.as_slice())], raw: None };
assert!(matches!(cfa(&header), Err(RawError::Unsupported(_))));
}
}
fn with_bayer_record(mut bytes: Vec<u8>, pattern: [u8; 4]) -> Vec<u8> {
let (dir_at, raw_at) = (be32(&bytes, 92).unwrap(), be32(&bytes, 100).unwrap());
let mut records = bytes[dir_at..raw_at].to_vec();
let count = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1;
records[..4].copy_from_slice(&count.to_be_bytes());
records.extend_from_slice(&[0x01, 0x30, 0, 4]);
records.extend_from_slice(&pattern);
bytes.splice(dir_at..raw_at, records.clone());
bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes());
bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes());
bytes
}
#[test]
fn non_bayer_geometry_is_rejected_in_header_and_full_modes() {
// Reversal gives RGBG: correct colour counts, but both greens in one column.
let bytes = with_bayer_record(raf(6, 4, 16, pack(&[1000; 24], 16), None, &[]), [1, 2, 1, 0]);
for mode in [Mode::Header, Mode::Full] {
assert!(matches!(decode(&bytes, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("Bayer layout")));
}
}
#[test]
fn all_four_bayer_geometries_preserve_the_stored_samples() {
let pixels: Vec<u16> = (0..24).map(|i| 123 + i * 977).collect();
for (stored, expected) in [([2, 1, 1, 0], "RGGB"), ([0, 1, 1, 2], "BGGR"), ([1, 2, 0, 1], "GRBG"), ([1, 0, 2, 1], "GBRG")] {
let bytes = with_bayer_record(raf(6, 4, 16, pack(&pixels, 16), None, &[]), stored);
for mode in [Mode::Header, Mode::Full] {
let image = decode(&bytes, mode, &Limits::default()).unwrap();
assert_eq!((image.width, image.height), (6, 4));
assert_eq!(image.cfa.as_ref().unwrap().name(), expected);
assert_eq!(image.data, RawData::U16(if mode == Mode::Full { pixels.clone() } else { Vec::new() }));
}
}
}
#[test]
fn compressed_bayer_requires_the_layout_assumed_by_its_predictor() {
let strip = b"IS\x01\x00\x0e\0\x12\x06\0\x03\x0c\x03\0\x02\0\x03".to_vec();
let mut bytes = raf(780, 18, 14, strip, None, &[]);
let (dir_at, raw_at) = (be32(&bytes, 92).unwrap(), be32(&bytes, 100).unwrap());
let mut records = bytes[dir_at..raw_at].to_vec();
let n = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1;
records[..4].copy_from_slice(&n.to_be_bytes());
// Reversed GRBG: a valid Bayer pattern, but not the RGGB order of this codec.
records.extend_from_slice(&[0x01, 0x30, 0, 4, 0x0b, 0x0a, 0x08, 0x09]);
bytes.splice(dir_at..raw_at, records.clone());
bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes());
bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes());
for mode in [Mode::Header, Mode::Full] {
assert!(matches!(decode(&bytes, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("layout")));
}
}
#[test]
fn compressed_layout_accepts_both_red_blue_orders_without_accepting_wrong_green_phase() {
for name in ["RGGB", "BGGR"] {
assert!(compressed_layout_supported(&Cfa::bayer_static(name)), "{name}");
}
for name in ["GRBG", "GBRG"] {
assert!(!compressed_layout_supported(&Cfa::bayer_static(name)), "{name}");
}
let mut other = Cfa::xtrans();
assert!(compressed_layout_supported(&other));
other.pattern.swap(0, 2);
assert!(!compressed_layout_supported(&other));
}
#[test]
fn unknown_compressed_signature_is_unsupported_but_identified_stream_damage_is_corrupt() {
let header = b"IS\x01\x00\x0e\0\x12\x06\0\x03\x0c\x03\0\x02\0\x03".to_vec();
let known = raf(780, 18, 14, header, None, &[]);
let raw_at = be32(&known, 100).unwrap();
let strip_at = raw_at + raw_ifd(&known[raw_at..]).unwrap().u64(0xf007).unwrap() as usize;
let mut unknown = known.clone();
unknown[strip_at..strip_at + 2].copy_from_slice(b"XX");
for mode in [Mode::Header, Mode::Full] {
assert!(matches!(decode(&unknown, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("signature")));
assert!(
matches!(decode(&known, mode, &Limits::default()), Err(RawError::Corrupt(_))),
"identified but truncated IS stream must remain corrupt"
);
}
}
/// Raw IFD flag `0xf006` selects the 32-bit-word packing at 12 bits; without it 12 bits are an LSB-first stream.
#[test]
fn flag_selects_word_packed_12_bit() {
+57 -5
View File
@@ -615,11 +615,18 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
}
_ => Rect::new(0, 0, active.width, active.height),
};
if !data.is_empty() {
// Packed formats 2/5/7 are lossless sample containers: zero is an ordinary
// code, not evidence of a mapped-out photosite. The documented zero repair
// applies to the predictive format 4 and older word containers only.
if !data.is_empty() && matches!(encoding, Encoding::Words | Encoding::Blocks { kind: Block::Format4, .. }) {
fill_defects(&mut data, w, active);
}
let cfa = cfa(ifd0.u16(CFA_PATTERN).unwrap_or(1));
let cfa_code = ifd0.u16(CFA_PATTERN);
// RGGB is still usable when metadata is absent or unknown, but callers
// must be able to distinguish that fallback from a recorded sensor phase.
let assumed_cfa = !matches!(cfa_code, Some(1..=4));
let cfa = cfa(cfa_code.unwrap_or(1));
let read = BLACK_RGB.map(|tag| ifd0.f64(tag).filter(|v| v.is_finite() && *v >= 0.0));
let tags = read.map(|v| v.unwrap_or(0.0) as f32);
let offset = match encoding {
@@ -671,6 +678,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: crate::LevelSources { black: black_source, white: white_source },
notices: crate::RawNotices { assumed_cfa, ..Default::default() },
};
img.validate_for(mode)?;
Ok(img)
@@ -1069,18 +1077,62 @@ mod tests {
}
}
#[test]
fn cfa_fallback_provenance_preserves_pixels_in_header_and_full_modes() {
let pixels: Vec<u16> = (0..72).map(|i| 17 + i * 113).collect();
let payload = pack(&pixels, 14);
for (code, expected, assumed) in [
(None, "RGGB", true),
(Some(0), "RGGB", true),
(Some(5), "RGGB", true),
(Some(u16::MAX), "RGGB", true),
(Some(1), "RGGB", false),
(Some(2), "GRBG", false),
(Some(3), "GBRG", false),
(Some(4), "BGGR", false),
] {
let mut tags = vec![short(BITS, 14), short(RAW_FORMAT, 7)];
if let Some(code) = code {
tags.push(short(CFA_PATTERN, code));
}
let bytes = file(18, 4, &tags, &payload, &|off| vec![(RAW_OFFSET, Value::Long(vec![off]))]);
for mode in [Mode::Header, Mode::Full] {
let image = decode(&bytes, mode, &Limits::default()).unwrap();
assert_eq!((image.width, image.height), (18, 4));
assert_eq!(image.cfa.as_ref().unwrap().name(), expected, "code {code:?}");
assert_eq!(image.notices, crate::RawNotices { assumed_cfa: assumed, ..Default::default() }, "code {code:?}, {mode:?}");
assert_eq!(samples(&image), if mode == Mode::Full { pixels.as_slice() } else { &[] });
}
}
}
#[test]
fn lossless_packed_formats_preserve_zero_codes_inside_the_active_area() {
let (w, h) = (90usize, 12usize);
for (format, bits, rotated) in [(5u16, 12u32, true), (5, 14, true), (7, 14, false), (2, 12, false)] {
let mut px = vec![1000u16; w * h];
px[5 * w + 20] = 0;
let payload = pack(&px, bits);
let bytes = rw2(w as u16, h as u16, bits as u16, format, 100, &if rotated { rotate(payload) } else { payload });
assert_eq!(samples(&crate::decode(&bytes).unwrap()), px, "format {format}/{bits}");
}
}
#[test]
fn levels_crop_defects_and_cfa() {
let (w, h) = (90usize, 60usize);
let mut px: Vec<u16> = (0..w * h).map(|i| 300 + (i % 7) as u16).collect();
// a saturation plateau sets the white level; zeros inside the active area are defects
// A saturation plateau sets the white level. In the older word container,
// isolated zeros inside the active area are documented mapped-out defects.
px[10 * w..20 * w].fill(3900);
px[30 * w + 40] = 0;
px[1] = 0; // in the border: left alone
let mut tags = vec![short(CFA_PATTERN, 2), short(BITS, 12), short(RAW_FORMAT, 5)];
let mut tags = vec![short(CFA_PATTERN, 2), short(BITS, 12), short(COMPRESSION, 34828)];
tags.extend([0x002f, 0x0030, 0x0031, 0x0032].iter().zip([12u16, 10, 52, 80]).map(|(&tag, v)| short(tag, v)));
tags.extend(LINEARITY_LIMIT.map(|tag| short(tag, 2000))); // recorded limits are not used
let bytes = file(w as u16, h as u16, &tags, &rotate(pack(&px, 12)), &|off| vec![(RAW_OFFSET, Value::Long(vec![off]))]);
// Compression 34828 stores left-justified samples in little-endian u16 words.
let words: Vec<u8> = px.iter().flat_map(|&v| (v << 4).to_le_bytes()).collect();
let bytes = file(w as u16, h as u16, &tags, &words, &|off| vec![(t::STRIP_OFFSETS, Value::Long(vec![off]))]);
let r = crate::decode(&bytes).unwrap();
let d = samples(&r);
assert_eq!(d[30 * w + 40], ((px[30 * w + 38] as u32 + px[30 * w + 42] as u32 + px[28 * w + 40] as u32 + px[32 * w + 40] as u32) / 4) as u16);
+1
View File
@@ -192,6 +192,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: crate::LevelSources { black: black_source, white: white_source },
notices: crate::RawNotices::default(),
};
img.validate_for(mode)?;
Ok(img)
+1
View File
@@ -49,6 +49,7 @@ fn an_active_area_that_wraps_past_the_image_is_corrupt() {
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: crate::RawNotices::default(),
pixel_aspect: 1.0,
};
assert!(raw.validate().is_err(), "`x + width` wrapped: the area is outside the image");
+72 -13
View File
@@ -115,6 +115,7 @@ pub(crate) fn read_image_in(
crate::Mode::Full => read_image(data, info, order, packing, limits),
crate::Mode::Header => {
check_image(data, info, limits)?;
check_uncompressed_chunks(data, info, &info.chunks(data.len() as u64), packing)?;
// a lossless-JPEG layout the decoder rejects (subsampled components, e.g. Sony's
// lossless M/S sizes) must fail here too, as the full decode will; likewise a JPEG XL
// chunk whose headers don't match the IFD
@@ -151,6 +152,7 @@ pub(crate) fn read_image(data: &[u8], info: &ImageInfo, order: ByteOrder, packin
let bits = info.bits() as u32;
let float = info.sample_format == 3;
let chunks = info.chunks(data.len() as u64);
check_uncompressed_chunks(data, info, &chunks, packing)?;
let planar = info.planar == 2 && cpp > 1;
let ccpp = if planar { 1 } else { cpp };
// The chunks are decoded and copied into the output in groups whose combined nominal size stays within
@@ -222,6 +224,26 @@ pub(crate) fn read_image(data: &[u8], info: &ImageInfo, order: ByteOrder, packin
Ok(if float { RawData::F32(out_f32) } else { RawData::U16(out_u16) })
}
/// Framing precedes allocation even in header-only probes: missing source rows may not
/// be reported as a decodable image and later silently filled with zero samples.
fn check_uncompressed_chunks(data: &[u8], info: &ImageInfo, chunks: &[Chunk], packing: Packing) -> Result<()> {
if info.compression != comp::NONE {
return Ok(());
}
let cpp = if info.planar == 2 { 1 } else { info.samples_per_pixel as usize };
for chunk in chunks {
let src = chunk_bytes(data, chunk).ok_or_else(|| RawError::Corrupt("uncompressed chunk is outside the file".into()))?;
let (cw, _, vw, vh) = chunk_dims(info, chunk);
let row = packed_row_bytes(cw as u64 * cpp as u64, info.bits() as u32, packing, info.sample_format == 3);
let last = packed_row_bytes(vw as u64 * cpp as u64, info.bits() as u32, packing, info.sample_format == 3);
let need = row.saturating_mul(vh.saturating_sub(1) as u64).saturating_add(last);
if (src.len() as u64) < need {
return Err(RawError::Corrupt(format!("an uncompressed chunk has {} bytes, expected at least {need}", src.len())));
}
}
Ok(())
}
/// Bytes that `n` samples take in a row of an uncompressed or inflated chunk, laid out as
/// [`unpack_chunk`] reads them: whole bytes or 16-bit words, or 32-bit words for `Words32Msb`.
fn packed_row_bytes(n: u64, bits: u32, packing: Packing, float: bool) -> u64 {
@@ -285,10 +307,12 @@ fn decode_chunk(
}
match info.compression {
comp::NONE => {
// uncompressed: missing rows are filled with zeros, but a chunk holding a small fraction of what its
// samples need is no picture: refused before the nominal chunk is allocated (storytold/lightcraft#1512)
let need = (n as u64).saturating_mul(u64::from(bits)) / 8;
if (src.len() as u64).saturating_mul(64) < need {
// Require every visible sample. Nominal edge-tile padding may be absent, but
// missing image rows must not turn a truncated file into a zero-filled success.
let row_bytes = packed_row_bytes(cw as u64 * cpp as u64, bits, packing, float);
let last_row = packed_row_bytes(vw as u64 * cpp as u64, bits, packing, float);
let need = row_bytes.saturating_mul(vh.saturating_sub(1) as u64).saturating_add(last_row);
if (src.len() as u64) < need {
return Err(RawError::Corrupt(format!("an uncompressed chunk of {n} samples has only {} bytes", src.len())));
}
unpack_chunk(src, order, packing, cw, ch, cpp, bits, float, 1)
@@ -314,13 +338,7 @@ fn decode_chunk(
let mcu = |v: usize| v.div_ceil(32).saturating_mul(32);
let tile_budget =
mcu(cw).saturating_mul(mcu(ch)).saturating_mul(if cpp == 1 { 1 } else { 3 }).clamp(1, limits.max_lossy_tile_samples.max(1));
let (px, jw, jh, jc) = lossy_jpeg(src, cpp, tile_budget)?;
// An edge tile's JPEG may be smaller than the nominal tile, but it has to cover the part of the tile
// inside the image: one that doesn't would leave a black region in a picture reported as decoded
// (PhotoCraft's and FilmCraft's readers refuse a JPEG smaller than its tile; this keeps the edge case).
if jw < vw || jh < vh {
return Err(RawError::Corrupt(format!("a lossy JPEG of {jw} × {jh} does not cover the {vw} × {vh} of its tile inside the image")));
}
let (px, jw, jh, jc) = lossy_jpeg_with_tile(src, cpp, tile_budget, Some((cw, ch, vw, vh)))?;
// the buffer holds the rows the decoded JPEG covers (the caller zero-fills the tile's
// remainder and reads only rows it fills): a tile-many-larger decoded JPEG does not
// get the nominal tile's full buffer (storytold/lightcraft#1517)
@@ -408,9 +426,21 @@ const MAX_PADDING_FACTOR: usize = 32;
#[cfg(feature = "dng")]
const MAX_WRITER_TILE_SAMPLES: usize = 2048 * 2048;
#[cfg(feature = "dng")]
#[cfg(all(test, feature = "dng"))]
/// Decode one baseline JPEG chunk to 8-bit samples: (samples, width, height, channels).
pub(crate) fn lossy_jpeg(src: &[u8], cpp: usize, max_samples: usize) -> Result<(Vec<u8>, usize, usize, usize)> {
lossy_jpeg_with_tile(src, cpp, max_samples, None)
}
/// Validate the actual frame before decode allocates its output. Edge tiles may store
/// a smaller frame covering all visible samples or bounded whole-MCU padding.
#[cfg(feature = "dng")]
fn lossy_jpeg_with_tile(
src: &[u8],
cpp: usize,
max_samples: usize,
tile: Option<(usize, usize, usize, usize)>,
) -> Result<(Vec<u8>, usize, usize, usize)> {
use zune_core::bytestream::ZCursor;
use zune_core::colorspace::ColorSpace;
use zune_core::options::DecoderOptions;
@@ -422,7 +452,27 @@ pub(crate) fn lossy_jpeg(src: &[u8], cpp: usize, max_samples: usize) -> Result<(
d.decode_headers().map_err(|e| RawError::Corrupt(format!("lossy JPEG tile: {e}")))?;
let info = d.info().ok_or_else(|| RawError::Corrupt("lossy JPEG tile without a header".into()))?;
let (w, h) = (info.width as usize, info.height as usize);
let c = if cpp == 1 { 1 } else { 3 };
if !matches!(cpp, 1 | 3) {
return Err(RawError::Unsupported(format!("lossy JPEG with {cpp} samples per pixel")));
}
if usize::from(info.components) != cpp {
return Err(RawError::Corrupt(format!("lossy JPEG has {} components, expected {cpp}", info.components)));
}
if let Some((cw, ch, vw, vh)) = tile {
if w < vw || h < vh {
return Err(RawError::Corrupt(format!("a lossy JPEG of {w} x {h} does not cover the {vw} x {vh} of its tile inside the image")));
}
// Writers can store a whole MCU around an edge tile. Keep the documented
// 32-pixel padding allowance without letting one tiny tile decode an arbitrary frame.
let mcu_extent = |value: usize| value.checked_add(31).map(|rounded| rounded / 32).and_then(|blocks| blocks.checked_mul(32));
let (Some(max_w), Some(max_h)) = (mcu_extent(cw), mcu_extent(ch)) else {
return Err(RawError::Corrupt("lossy JPEG tile dimensions overflow MCU padding".into()));
};
if w > max_w || h > max_h {
return Err(RawError::Corrupt(format!("a lossy JPEG of {w} x {h} exceeds its {max_w} x {max_h} MCU-padded tile dimensions")));
}
}
let c = cpp;
let out = w as u64 * h as u64 * c as u64;
if w == 0 || h == 0 || out > max_samples as u64 {
return Err(RawError::Corrupt(format!("a lossy JPEG of {w} × {h} × {c} would exceed the {max_samples} output samples allowed")));
@@ -519,6 +569,12 @@ fn unpack_chunk(
}
} else {
undo_diff_u16(row, cpp * factor);
// Horizontal differences wrap in the declared sample precision, not in
// the u16 container used by the unpacker (TIFF 6.0 Predictor 2).
if bits < 16 {
let mask = ((1u32 << bits) - 1) as u16;
row.iter_mut().for_each(|sample| *sample &= mask);
}
}
}
}
@@ -982,3 +1038,6 @@ mod budget_tests {
assert_eq!(decode_groups(&big.chunks(64), 1, 1 << 20), vec![0..1]);
}
}
#[cfg(all(test, feature = "dng"))]
mod compat_review_tests;
@@ -0,0 +1,115 @@
use super::*;
use craft_tiff::image::Layout;
fn image(width: u32, height: u32, bits: u16, cpp: u16, compression: u16, len: usize) -> ImageInfo {
ImageInfo {
width,
height,
bits_per_sample: vec![bits; cpp as usize],
samples_per_pixel: cpp,
compression,
photometric: 34892,
planar: 1,
predictor: 1,
sample_format: 1,
new_subfile_type: 0,
layout: Layout::Strips { rows_per_strip: height },
offsets: vec![0],
byte_counts: vec![len as u64],
}
}
#[test]
fn truncated_uncompressed_rows_are_rejected_in_full_and_header_modes() {
for (bits, packing, float, bytes) in [
(8, Packing::Msb, false, 8),
(12, Packing::Msb, false, 12),
(12, Packing::Word16, false, 16),
(16, Packing::Msb, false, 16),
(32, Packing::Msb, true, 32),
] {
let data = vec![0u8; bytes];
let mut info = image(4, 2, bits, 1, comp::NONE, bytes);
info.sample_format = if float { 3 } else { 1 };
assert!(read_image_in(crate::Mode::Full, &data, &info, ByteOrder::Little, packing, &Limits::default()).is_ok());
info.byte_counts[0] -= 1;
for mode in [crate::Mode::Full, crate::Mode::Header] {
assert!(
matches!(read_image_in(mode, &data[..bytes - 1], &info, ByteOrder::Little, packing, &Limits::default()), Err(RawError::Corrupt(message)) if message.contains("uncompressed chunk")),
"{bits}/{float}/{mode:?}"
);
}
}
}
#[test]
fn edge_padding_may_be_absent_but_visible_uncompressed_samples_must_all_exist() {
let data = vec![31u8; 16 * 7 + 8];
let mut info = image(8, 8, 8, 1, comp::NONE, data.len());
info.layout = Layout::Tiles { tile_width: 16, tile_height: 16 };
let RawData::U16(values) = read_image(&data, &info, ByteOrder::Little, Packing::Msb, &Limits::default()).unwrap() else { panic!("integer") };
assert_eq!(values, vec![31; 64]);
info.byte_counts[0] -= 1;
assert!(read_image(&data[..data.len() - 1], &info, ByteOrder::Little, Packing::Msb, &Limits::default()).is_err());
}
#[test]
fn packed_predictor_two_wraps_at_declared_twelve_bits() {
// Samples [250, 10, 4090, 20]; differences modulo4096 are 0FA,F10,FF0,01A.
let packed = [0x0f, 0xaf, 0x10, 0xff, 0x00, 0x1a];
let ChunkPx::U16(values) = unpack_chunk(&packed, ByteOrder::Little, Packing::Msb, 4, 1, 1, 12, false, 2).unwrap() else { panic!("integer") };
assert_eq!(values, vec![250, 10, 4090, 20]);
}
#[allow(clippy::unwrap_used)] // Test-only fixture encoder; failed construction must fail the test.
fn jpeg(cpp: usize, width: u16, height: u16) -> Vec<u8> {
let mut data = Vec::new();
let color = if cpp == 1 { jpeg_encoder::ColorType::Luma } else { jpeg_encoder::ColorType::Rgb };
jpeg_encoder::Encoder::new(&mut data, 100).encode(&vec![90; width as usize * height as usize * cpp], width, height, color).unwrap();
data
}
#[test]
fn baseline_jpeg_components_are_checked_before_forced_colorspace_conversion() {
for (actual, expected) in [(3, 1), (1, 3)] {
let data = jpeg(actual, 8, 8);
let error = lossy_jpeg(&data, expected, 1024).unwrap_err();
assert!(matches!(error, RawError::Corrupt(message) if message.contains(&format!("{actual} components"))));
}
assert!(matches!(lossy_jpeg(&jpeg(3, 8, 8), 4, 1024), Err(RawError::Unsupported(message)) if message.contains("4 samples")));
for cpp in [1, 3] {
assert_eq!(lossy_jpeg(&jpeg(cpp, 8, 8), cpp, 1024).unwrap().3, cpp);
}
}
#[test]
fn jpeg_tile_geometry_and_limits_are_checked_at_the_header_with_valid_edge_controls() {
let data = jpeg(3, 8, 8);
assert!(lossy_jpeg_with_tile(&data, 3, 1024, Some((16, 16, 8, 8))).is_ok());
assert!(
matches!(lossy_jpeg_with_tile(&data, 3, 1024, Some((16, 16, 9, 8))), Err(RawError::Corrupt(message)) if message.contains("does not cover"))
);
assert!(matches!(lossy_jpeg_with_tile(&data, 3, 32, Some((16, 16, 8, 8))), Err(RawError::Corrupt(message)) if message.contains("would exceed")));
// Mutate only SOF dimensions, retaining a valid header but no entropy for the claimed frame.
// The geometry error proves refusal precedes decode's short/corrupt entropy failure.
for (width, height) in [(33u16, 8u16), (8, 33), (256, 256)] {
let mut oversized = data.clone();
let sof = oversized.windows(2).position(|bytes| bytes == [0xff, 0xc0]).unwrap();
oversized[sof + 5..sof + 7].copy_from_slice(&height.to_be_bytes());
oversized[sof + 7..sof + 9].copy_from_slice(&width.to_be_bytes());
assert!(
matches!(lossy_jpeg_with_tile(&oversized, 3, 1 << 20, Some((16, 16, 8, 8))), Err(RawError::Corrupt(message)) if message.contains("exceeds its"))
);
}
// A genuine full MCU remains valid even when only a 4x4 part of the nominal16 tile is visible.
let padded = jpeg(3, 32, 32);
assert!(lossy_jpeg_with_tile(&padded, 3, 32 * 32 * 3, Some((16, 16, 4, 4))).is_ok());
assert!(
matches!(lossy_jpeg_with_tile(&data, 3, 1 << 20, Some((usize::MAX, 16, 8, 8))), Err(RawError::Corrupt(message)) if message.contains("overflow MCU padding"))
);
let mut info = image(8, 8, 8, 3, comp::LOSSY_JPEG, data.len());
info.layout = Layout::Tiles { tile_width: 16, tile_height: 16 };
let RawData::U16(values) = read_image(&data, &info, ByteOrder::Little, Packing::Msb, &Limits::default()).unwrap() else { panic!("integer") };
assert_eq!(values.len(), 8 * 8 * 3);
assert!(values.iter().all(|sample| (i32::from(*sample) - 90).abs() <= 3));
}
+17 -3
View File
@@ -64,6 +64,7 @@ fn synthetic(w: usize, h: usize, cfa: Option<Cfa>, cpp: usize) -> RawImage {
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: craft_raw::RawNotices::default(),
}
}
@@ -948,18 +949,31 @@ fn malformed_semantic_masks_are_skipped() {
let mut zero = mask_ifd(2, 2, "zero size", vec![0; 4]);
zero.set(t::IMAGE_WIDTH, Value::Long(vec![0]));
ifd0.add_sub_ifd(zero);
// A 4x4 8-bit mask needs 16 visible samples; three bytes must not be
// accepted as a synthetic mask with thirteen zero-filled samples.
let mut short = mask_ifd(4, 4, "short strip", vec![200; 3]);
short.set(t::MASK_SUB_AREA, Value::Long(vec![u32::MAX, u32::MAX, u32::MAX, u32::MAX]));
ifd0.add_sub_ifd(short);
let mut valid = mask_ifd(4, 2, "valid crop", vec![10, 20, 30, 40, 50, 60, 70, 255]);
valid.set(t::MASK_SUB_AREA, Value::Long(vec![1, 2, 8, 4]));
ifd0.add_sub_ifd(valid);
let mut outside = mask_ifd(2, 2, "sub area too small", vec![255; 4]);
outside.set(t::MASK_SUB_AREA, Value::Long(vec![3, 0, 2, 4]));
ifd0.add_sub_ifd(outside);
});
assert_eq!(decode(&bytes).unwrap().cpp, 3);
let raw = decode(&bytes).unwrap();
let baseline = decode(&profile_dng(ByteOrder::Little, |_| {})).unwrap();
assert_eq!((raw.width, raw.height, raw.cpp), (8, 6, 3));
assert_eq!(raw.data, baseline.data, "optional damaged masks must not change base image samples");
let masks = craft_raw::semantic_masks(&bytes);
let names: Vec<&str> = masks.iter().map(|m| m.name.as_str()).collect();
assert_eq!(names, ["short strip", "sub area too small"]);
assert!(masks.iter().all(|m| m.sub_area.is_none()), "sub areas the mask doesn't fit are ignored");
assert_eq!(names, ["valid crop", "sub area too small"], "the truncated mask is omitted, and later usable masks survive");
assert_eq!(masks[0].data, [10u16, 20, 30, 40, 50, 60, 70, 255].map(|sample| sample * 257));
assert_eq!(masks[0].sub_area, Some(craft_raw::semantic::MaskSubArea { top: 1, left: 2, full_width: 8, full_height: 4 }));
assert!((masks[0].value_at(2.5, 1.5) - 10. / 255.).abs() < 1e-4);
assert_eq!(masks[0].value_at(1.5, 1.5), 0.);
assert!(masks[1].sub_area.is_none(), "the complete mask survives while a sub area it doesn't fit is ignored");
assert_eq!(masks[1].data, vec![65535; 4]);
for m in &masks {
assert_eq!(m.data.len(), m.width * m.height);
}
+1
View File
@@ -28,6 +28,7 @@ fn sample(comp: DngCompression) -> Vec<u8> {
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: craft_raw::RawNotices::default(),
};
craft_raw::write_dng(&raw, &DngWriteOptions { compression: comp, ..Default::default() }).unwrap()
}
+31 -1
View File
@@ -302,6 +302,7 @@ struct Used {
raster_limit: AtomicUsize,
invalid_svg: AtomicUsize,
svg_limit: AtomicUsize,
svg_resource_limit: AtomicUsize,
}
/// What resolving one drawing's embedded images used or refused.
@@ -323,6 +324,10 @@ pub struct ImageUsage {
pub invalid_svg_images: usize,
/// Embedded SVG data refused by byte or nesting limits.
pub svg_limit_images: usize,
/// Subset of `svg_limit_images` refused by unpacked byte or embedded-document-chain limits.
/// XML/tree-depth omissions and invalid or corrupt data do not set this counter. Consumers
/// can reject resource overruns while still allowing an optional, over-deep child to be omitted.
pub svg_resource_limit_images: usize,
}
/// A data-URL resolver and the accounting for the one drawing it serves.
@@ -350,6 +355,7 @@ impl ImageResolver {
raster_limit: AtomicUsize::new(0),
invalid_svg: AtomicUsize::new(0),
svg_limit: AtomicUsize::new(0),
svg_resource_limit: AtomicUsize::new(0),
}),
}
}
@@ -376,6 +382,7 @@ impl ImageResolver {
raster_limit_images: self.used.raster_limit.load(Ordering::Relaxed),
invalid_svg_images: self.used.invalid_svg.load(Ordering::Relaxed),
svg_limit_images: self.used.svg_limit.load(Ordering::Relaxed),
svg_resource_limit_images: self.used.svg_resource_limit.load(Ordering::Relaxed),
}
}
}
@@ -438,6 +445,7 @@ fn raster_image(used: &Used, limits: &Limits, data: Arc<Vec<u8>>, format: craft_
fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<'_>) -> Option<usvg::ImageKind> {
if used.depth.load(Ordering::Relaxed) >= limits.embedded_depth {
used.svg_limit.fetch_add(1, Ordering::Relaxed);
used.svg_resource_limit.fetch_add(1, Ordering::Relaxed);
return None;
}
let left = limits.embedded_bytes.saturating_sub(used.bytes.load(Ordering::Relaxed));
@@ -445,15 +453,37 @@ fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<'
Ok(text) => text,
// What a refused document unpacked before it was refused was work too: it uses up what
// is left, so the next image can't repeat it.
Err(_) => {
Err(error) => {
used.bytes.fetch_add(left, Ordering::Relaxed);
used.svg_limit.fetch_add(1, Ordering::Relaxed);
if matches!(error, svgz::Error::TooLarge(_)) {
used.svg_resource_limit.fetch_add(1, Ordering::Relaxed);
}
return None;
}
};
// The raw/inflated bytes can fit while UTF-16/Latin-1 decoding expands
// beyond the aggregate allowance. A refusal consumes the remaining work.
if text.len() > left {
used.bytes.fetch_add(left, Ordering::Relaxed);
used.svg_limit.fetch_add(1, Ordering::Relaxed);
used.svg_resource_limit.fetch_add(1, Ordering::Relaxed);
return None;
}
used.bytes.fetch_add(text.len(), Ordering::Relaxed);
let xml = match checked(&text, limits) {
Ok(xml) => xml,
// Decoding UTF-16 or Latin-1 can exceed the text cap after the
// raw/inflated bytes fitted. This is a resource refusal, not invalid XML.
Err(Error::Text(svgz::Error::TooLarge(_))) => {
used.svg_limit.fetch_add(1, Ordering::Relaxed);
used.svg_resource_limit.fetch_add(1, Ordering::Relaxed);
return None;
}
Err(Error::TooDeep(_)) => {
used.svg_limit.fetch_add(1, Ordering::Relaxed);
return None;
}
Err(_) => {
used.invalid_svg.fetch_add(1, Ordering::Relaxed);
return None;
+189
View File
@@ -245,6 +245,195 @@ fn embedded_svgs_are_read_within_their_budgets() {
assert_eq!(found(&svg(r#"<image width="5" height="5" xlink:href="/etc/passwd.svg"/>"#), shallow()), [false; 0]);
}
mod resource_limit_usage {
use super::*;
fn data_image(bytes: &[u8]) -> String {
let encoded: String = bytes.iter().map(|byte| format!("%{byte:02X}")).collect();
format!(r#"<image width="5" height="5" xlink:href="data:image/svg+xml,{encoded}"/>"#)
}
fn packed(text: &str, gzip: bool) -> Vec<u8> {
if gzip { svgz::compress(text.as_bytes()) } else { text.as_bytes().to_vec() }
}
/// The outer document has its own generous text allowance, so tiny resolver limits
/// test the public embedded admission contract rather than rejecting the outer XML.
fn resolved(text: &str, limits: Limits, outer_depth: usize) -> (Vec<bool>, ImageUsage) {
let resolver = ImageResolver::new(limits);
resolver.set_document_depth(outer_depth);
let options = usvg::Options { image_href_resolver: resolver.resolver(), ..Default::default() };
let tree = parse(text.as_bytes(), &shallow(), &options).unwrap();
let mut found = Vec::new();
images(tree.root(), &mut found);
(found, resolver.usage())
}
fn expanding_encoded_children() -> Vec<(&'static str, Vec<u8>)> {
let latin =
format!("<?xml version='1.0' encoding='ISO-8859-1'?>{}", svg(&format!("<title>{}</title><rect width='1' height='1'/>", "é".repeat(100))));
let latin = latin.chars().map(|c| u8::try_from(u32::from(c)).unwrap()).collect();
let wide = svg(&format!("<title>{}</title><rect width='1' height='1'/>", "ࠀ".repeat(400)));
let mut little = vec![0xff, 0xfe];
little.extend(wide.encode_utf16().flat_map(u16::to_le_bytes));
let mut big = vec![0xfe, 0xff];
big.extend(wide.encode_utf16().flat_map(u16::to_be_bytes));
vec![("Latin-1", latin), ("UTF-16LE", little), ("UTF-16BE", big)]
}
#[test]
fn decoded_text_expansion_is_a_resource_refusal_for_plain_and_gzip_children() {
for (encoding, raw) in expanding_encoded_children() {
let decoded = svgz::decode(&raw).unwrap();
let cap = raw.len();
assert!(decoded.len() > cap, "{encoding} fixture must expand after inflation");
for gzip in [false, true] {
let data = if gzip { svgz::compress(&raw) } else { raw.clone() };
assert_eq!(svgz::inflate(&data, cap).unwrap().len(), cap, "raw bytes fit exactly");
assert_eq!(svgz::text(&data, cap).unwrap().len(), decoded.len(), "decoding itself succeeds");
let limits = Limits { bytes: cap, embedded_bytes: decoded.len(), ..shallow() };
assert!(matches!(checked(&decoded, &limits), Err(Error::Text(svgz::Error::TooLarge(n))) if n == cap));
let image = data_image(&data);
let (found, usage) = resolved(&svg(&image), limits, 1);
assert!(found.is_empty(), "{encoding}, gzip {gzip}");
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (1, 1, 0));
assert_eq!(usage.embedded_svg_bytes, decoded.len(), "spent decoded work remains charged");
let (found, usage) = resolved(&svg(&format!("{image}{image}")), limits, 1);
assert!(found.is_empty());
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (2, 2, 0));
assert_eq!(usage.embedded_svg_bytes, decoded.len(), "a refused sibling cannot repeat exhausted work");
}
}
}
#[test]
fn decoded_text_expansion_respects_aggregate_bytes_even_when_each_child_fits() {
for (encoding, raw) in expanding_encoded_children() {
let decoded = svgz::decode(&raw).unwrap();
let limits = Limits { bytes: decoded.len(), embedded_bytes: raw.len(), ..shallow() };
assert!(decoded.len() > limits.embedded_bytes);
checked(&decoded, &limits).unwrap(); // The independent per-image cap allows this complete XML.
for gzip in [false, true] {
let data = if gzip { svgz::compress(&raw) } else { raw.clone() };
assert_eq!(svgz::inflate(&data, limits.embedded_bytes).unwrap().len(), raw.len());
let image = data_image(&data);
let (found, usage) = resolved(&svg(&image), limits, 1);
assert!(found.is_empty(), "{encoding}, gzip {gzip}");
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (1, 1, 0));
assert_eq!(usage.embedded_svg_bytes, limits.embedded_bytes, "refusal consumes exactly the remaining allowance");
let (found, usage) = resolved(&svg(&format!("{image}{image}")), limits, 1);
assert!(found.is_empty());
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (2, 2, 0));
assert_eq!(usage.embedded_svg_bytes, limits.embedded_bytes, "a later child cannot repeat exhausted work");
let (found, usage) = resolved(&svg(&image), Limits { embedded_bytes: decoded.len(), ..limits }, 1);
assert_eq!(found, [true]);
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (0, 0, 0));
assert_eq!(usage.embedded_svg_bytes, decoded.len());
}
}
}
#[test]
fn decoded_text_expansion_at_the_cap_is_valid_and_malformed_xml_stays_invalid() {
for (encoding, raw) in expanding_encoded_children() {
let decoded = svgz::decode(&raw).unwrap();
for gzip in [false, true] {
let data = if gzip { svgz::compress(&raw) } else { raw.clone() };
let limits = Limits { bytes: decoded.len(), embedded_bytes: decoded.len(), ..shallow() };
let (found, usage) = resolved(&svg(&data_image(&data)), limits, 1);
assert_eq!(found, [true], "{encoding}, gzip {gzip}");
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (0, 0, 0));
assert_eq!(usage.embedded_svg_bytes, decoded.len());
// Preserve the declared/BOM encoding when making the XML malformed;
// re-encoding Latin-1 text as UTF-8 would test a second expansion instead.
let closing_bytes = if encoding == "Latin-1" { 6 } else { 12 };
let malformed = &raw[..raw.len() - closing_bytes];
let data = if gzip { svgz::compress(malformed) } else { malformed.to_vec() };
let (found, usage) = resolved(&svg(&data_image(&data)), limits, 1);
assert!(found.is_empty());
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (0, 0, 1));
}
}
}
#[test]
fn embedded_chain_caps_are_explicit_resource_refusals_for_plain_and_gzip_children() {
for gzip in [false, true] {
let inner = svg("<rect width='1' height='1'/>");
let middle = svg(&data_image(&packed(&inner, gzip)));
let outer = svg(&data_image(&packed(&middle, gzip)));
let (found, usage) = resolved(&outer, Limits { embedded_depth: 1, ..shallow() }, 1);
assert_eq!(found, [true]);
assert_eq!(usage.embedded_svg_depth, 1);
assert_eq!(usage.svg_limit_images, 1, "strict consumers still see the existing limit flag");
assert_eq!(usage.svg_resource_limit_images, 1, "a chain cap must not become a successful partial bounded preview");
let (found, usage) = resolved(&outer, Limits { embedded_depth: 2, ..shallow() }, 1);
assert_eq!(found, [true, true]);
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0));
}
}
#[test]
fn embedded_per_image_byte_caps_are_resource_refusals_for_plain_and_gzip_children() {
let inner = svg("<rect width='1' height='1'/>");
for gzip in [false, true] {
let outer = svg(&data_image(&packed(&inner, gzip)));
let (_, usage) = resolved(&outer, Limits { bytes: inner.len() - 1, embedded_bytes: inner.len() * 3, ..shallow() }, 1);
assert_eq!(usage.embedded_svg_count, 0);
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (1, 1));
let (found, usage) = resolved(&outer, Limits { bytes: inner.len(), ..shallow() }, 1);
assert_eq!(found, [true]);
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0));
}
}
#[test]
fn aggregate_embedded_bytes_are_resource_refusals_even_when_each_child_fits() {
let inner = svg("<rect width='1' height='1'/>");
for gzip in [false, true] {
let image = data_image(&packed(&inner, gzip));
let outer = svg(&format!("{image}{image}"));
let cap = inner.len() * 2 - 1;
let (found, usage) = resolved(&outer, Limits { embedded_bytes: cap, ..shallow() }, 1);
assert_eq!(found, [true]);
assert_eq!(usage.embedded_svg_bytes, cap, "a refused child still spends the remaining work allowance");
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (1, 1));
let (found, usage) = resolved(&outer, Limits { embedded_bytes: inner.len() * 2, ..shallow() }, 1);
assert_eq!(found, [true, true]);
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0));
}
}
#[test]
fn combined_xml_depth_omissions_remain_distinct_from_resource_refusals() {
let inner = svg("<rect width='1' height='1'/>");
for gzip in [false, true] {
let image = data_image(&packed(&inner, gzip));
let outer = svg(&format!("{}{}{}", "<g>".repeat(4), image, "</g>".repeat(4)));
let (found, usage) = resolved(&outer, Limits { tree_depth: 6, ..shallow() }, 5);
assert!(found.is_empty());
assert_eq!(usage.svg_limit_images, 1, "strict consumers must continue to reject this omission");
assert_eq!(usage.svg_resource_limit_images, 0, "non-strict consumers may omit this child without disguising a byte/chain overrun");
let (found, usage) = resolved(&svg(&image), Limits { tree_depth: 6, ..shallow() }, 1);
assert_eq!(found, [true]);
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0));
}
}
#[test]
fn invalid_and_corrupt_children_do_not_claim_a_resource_limit() {
let mut corrupt = svgz::compress(svg("<rect width='1' height='1'/>").as_bytes());
let crc = corrupt.len() - 8;
corrupt[crc] ^= 1;
for data in [b"not XML".to_vec(), corrupt] {
let (found, usage) = resolved(&svg(&data_image(&data)), shallow(), 1);
assert!(found.is_empty());
assert!(usage.invalid_svg_images > 0 || usage.svg_limit_images > 0, "strict consumers still observe unreadable content");
assert_eq!(usage.svg_resource_limit_images, 0);
}
}
}
mod draw_checks {
use super::*;
+124 -11
View File
@@ -125,6 +125,16 @@ struct Asset {
has_audio: bool,
}
// Bound semantic resource chains separately from XML syntax: flat media resources can
// reference one another without adding any XML nesting.
const MAX_MEDIA_DEPTH: usize = 64;
const MAX_XML_DEPTH: usize = 128;
// Retrying truncated resources is useful at shallower depths, but must remain bounded.
const MAX_REIMPORTS: usize = 4096;
// A hostile document may reference the same large cached graph repeatedly. Bound total
// reachability work, not just one walk, and refuse reuse when safety cannot be established.
const MAX_CACHE_EDGES: usize = 1_000_000;
struct Imp<'a, 'c, 'r> {
b: Builder<'c>,
opts: &'a ImportOptions<'a>,
@@ -135,6 +145,13 @@ struct Imp<'a, 'c, 'r> {
media_nodes: HashMap<String, Node<'a>>,
media_items: HashMap<String, ItemId>,
media_active: HashSet<String>,
depth_hits: usize,
// Depths at which a cached resource was truncated. A retry may improve neither limit
// at the expense of the other, and reuses the same item so earlier references stay valid.
media_truncated_at: HashMap<String, (usize, usize)>,
reimports_left: usize,
cache_edges_left: usize,
depth: usize,
effects: HashMap<String, String>,
}
@@ -186,6 +203,11 @@ pub fn import(text: &str, opts: &ImportOptions, report: &mut Report) -> Result<I
media_nodes: HashMap::new(),
media_items: HashMap::new(),
media_active: HashSet::new(),
depth_hits: 0,
media_truncated_at: HashMap::new(),
reimports_left: MAX_REIMPORTS,
cache_edges_left: MAX_CACHE_EDGES,
depth: 0,
effects: HashMap::new(),
};
if let Some(res) = child(root, "resources") {
@@ -229,7 +251,7 @@ pub fn import(text: &str, opts: &ImportOptions, report: &mut Report) -> Result<I
"project" => {
if let Some(s) = child(c, "sequence") {
let name = c.attribute("name").unwrap_or("Project");
let id = imp.sequence(s, name, Some(bin));
let id = imp.sequence(s, name, Some(bin), None);
imp.b.top.push(id);
}
}
@@ -244,7 +266,7 @@ pub fn import(text: &str, opts: &ImportOptions, report: &mut Report) -> Result<I
}
for p in children(root, "project") {
if let Some(s) = child(p, "sequence") {
let id = imp.sequence(s, p.attribute("name").unwrap_or("Project"), None);
let id = imp.sequence(s, p.attribute("name").unwrap_or("Project"), None, None);
imp.b.top.push(id);
}
}
@@ -308,34 +330,100 @@ impl<'a> Imp<'a, '_, '_> {
Some(a)
}
/// A rebuilt item retains its ID. A cached ancestor that already points to it must
/// not be inserted underneath it, even when the resource names are not directly active.
fn cache_reaches_active(&mut self, root: ItemId) -> bool {
let active: HashSet<ItemId> = self.media_active.iter().filter_map(|id| self.media_items.get(id).copied()).collect();
if active.is_empty() {
return false;
}
let mut seen = HashSet::from([root]);
let mut pending = vec![root];
while let Some(id) = pending.pop() {
if active.contains(&id) {
return true;
}
let Some(seq) = self.b.p.sequence(id) else { continue };
for item in seq.all_tracks().flat_map(|track| &track.items) {
if self.cache_edges_left == 0 {
return true;
}
self.cache_edges_left -= 1;
if active.contains(&item.item) {
return true;
}
if seen.insert(item.item) {
pending.push(item.item);
}
}
}
false
}
/// Nested sequence from a `<media>` resource.
fn media(&mut self, id: &str) -> Option<ItemId> {
if let Some(i) = self.media_items.get(id) {
return Some(*i);
}
let n = *self.media_nodes.get(id)?;
let s = child(n, "sequence")?;
if !self.media_active.insert(id.into()) {
// Check before the cache: a truncated resource may already have an item while it
// is being rebuilt, but a back-edge must never turn that item into a self-cycle.
if self.media_active.contains(id) {
self.report.warn(format!("recursive FCPXML media resource {id:?} ignored"));
return None;
}
let depth = self.media_active.len();
let mut again = None;
if let Some(i) = self.media_items.get(id).copied() {
// Imported with a nested one cut off: import it again (into the same item) where that
// one now fits.
let shallower = self
.media_truncated_at
.get(id)
.is_some_and(|&(media, xml)| depth <= media && self.depth <= xml && (depth < media || self.depth < xml))
&& !self.media_active.contains(id);
if !shallower || self.reimports_left == 0 {
if self.cache_reaches_active(i) {
self.depth_hits += 1;
self.report.warn(format!(
"FCPXML cached media {id:?} could form a recursive graph or exceeds the cache traversal budget; imported as a gap"
));
return None;
}
return Some(i);
}
self.reimports_left -= 1;
again = Some(i);
}
let n = *self.media_nodes.get(id)?;
let s = child(n, "sequence")?;
// The active set is the current nesting chain: a long acyclic chain of compound clips
// would otherwise recurse once per resource.
if depth >= MAX_MEDIA_DEPTH {
self.depth_hits += 1;
self.report.warn(format!("FCPXML media resource {id:?} is nested too deeply; imported as a gap"));
return None;
}
self.media_active.insert(id.into());
// Mark the resource before descending: a compound clip can reference itself through a
// ref-clip before a completed sequence is available to cache.
let name = n.attribute("name").unwrap_or("Compound Clip").to_string();
let item = self.sequence(s, &name, None);
let hits_before = self.depth_hits;
let item = self.sequence(s, &name, None, again);
self.media_active.remove(id);
self.media_items.insert(id.into(), item);
if self.depth_hits > hits_before {
self.media_truncated_at.insert(id.into(), (depth, self.depth));
} else {
self.media_truncated_at.remove(id);
}
Some(item)
}
fn sequence(&mut self, s: Node<'a>, name: &str, bin: Option<BinId>) -> ItemId {
fn sequence(&mut self, s: Node<'a>, name: &str, bin: Option<BinId>, reuse: Option<ItemId>) -> ItemId {
let (rate, w, h) = s.attribute("format").and_then(|f| self.formats.get(f)).copied().unwrap_or((FrameRate::default(), 1920, 1080));
let df = s.attribute("tcFormat") == Some("DF");
let mut settings = settings_for(rate, w, h, df);
if let Some(sr) = s.attribute("audioRate").and_then(parse_audio_rate) {
settings.sample_rate = sr;
}
let id = self.b.reserve_sequence(name, settings.clone(), bin);
let id = reuse.unwrap_or_else(|| self.b.reserve_sequence(name, settings.clone(), bin));
let mut seq = empty_sequence(settings);
let tc_start = attr_time(s, "tcStart").unwrap_or(Tick::ZERO);
seq.start_timecode = rate.frame_at(tc_start);
@@ -353,7 +441,19 @@ impl<'a> Imp<'a, '_, '_> {
/// Lay out a storyline. `origin` maps the storyline's offsets to timeline time
/// (`timeline = origin + offset`). `lane` is the storyline's lane (0 = primary).
/// Refuses nesting past [`MAX_XML_DEPTH`].
fn storyline(&mut self, seq: &mut Sequence, ctx: &Ctx, spine: Node<'a>, origin: Tick, lane: i32, primary: bool) {
if self.depth >= MAX_XML_DEPTH {
self.depth_hits += 1;
self.report.warn(format!("XML storylines are nested too deeply (more than {MAX_XML_DEPTH}); deeper clips were skipped"));
return;
}
self.depth += 1;
self.storyline_inner(seq, ctx, spine, origin, lane, primary);
self.depth -= 1;
}
fn storyline_inner(&mut self, seq: &mut Sequence, ctx: &Ctx, spine: Node<'a>, origin: Tick, lane: i32, primary: bool) {
// (timeline start, timeline end, video clip on this storyline's track)
let mut laid: Vec<(Tick, Tick, Option<ClipId>)> = Vec::new();
let mut transitions: Vec<(usize, Node)> = Vec::new();
@@ -421,7 +521,20 @@ impl<'a> Imp<'a, '_, '_> {
}
/// Lay out one storyline element at timeline time `t0`. Returns (timeline end, video clip).
/// Refuses nesting past [`MAX_XML_DEPTH`].
fn element(&mut self, seq: &mut Sequence, ctx: &Ctx, el: Node<'a>, t0: Tick, lane: i32, primary: bool) -> (Tick, Option<ClipId>) {
if self.depth >= MAX_XML_DEPTH {
self.depth_hits += 1;
self.report.warn(format!("XML elements are nested too deeply (more than {MAX_XML_DEPTH}); deeper clips were skipped"));
return (t0, None);
}
self.depth += 1;
let out = self.element_inner(seq, ctx, el, t0, lane, primary);
self.depth -= 1;
out
}
fn element_inner(&mut self, seq: &mut Sequence, ctx: &Ctx, el: Node<'a>, t0: Tick, lane: i32, primary: bool) -> (Tick, Option<ClipId>) {
let tag = el.name();
let dur = attr_time(el, "duration").unwrap_or(Tick::ZERO);
let local_start = attr_time(el, "start").unwrap_or(Tick::ZERO);
+111
View File
@@ -503,3 +503,114 @@ fn relative_media_urls_are_percent_decoded() {
let boats = &p.sequence(only_seq(&imp)).unwrap().video_tracks[0].items[1];
assert_eq!(media_key(p, boats.item), "/proj/media/boats b.mov");
}
// Parity regressions from FilmCraft e6df318 (including #4799), adapted to the neutral model.
#[test]
fn media_cut_off_by_the_depth_limit_is_complete_when_reached_higher_up() {
// m0 … m64 nest one inside the next. Project A reaches m63 at the depth limit, so m64 is left
// out of it; project B refers to m63 directly, where m64 fits, and must not get the cut-off copy.
let mut media = String::new();
for i in 0..64 {
media.push_str(&format!(
r#"<media id="m{i}" name="M{i}"><sequence format="r1"><spine><ref-clip ref="m{}" offset="0s" start="0s" duration="1s"/></spine></sequence></media>"#,
i + 1
));
}
media.push_str(r#"<media id="m64" name="M64"><sequence format="r1"><spine/></sequence></media>"#);
let xml = format!(
r#"<fcpxml version="1.10"><resources><format id="r1" frameDuration="1/24s" width="1920" height="1080"/>{media}</resources>
<library><event>
<project name="A"><sequence format="r1"><spine><ref-clip ref="m0" offset="0s" start="0s" duration="1s"/></spine></sequence></project>
<project name="B"><sequence format="r1"><spine><ref-clip ref="m63" offset="0s" start="0s" duration="1s"/></spine></sequence></project>
</event></library></fcpxml>"#
);
let (imp, rep) = import(xml.as_bytes(), Format::Fcpxml, None).unwrap();
assert!(rep.mentions("nested too deeply"), "{rep:?}");
let m63 = imp.project.items.values().find(|i| i.name == "M63").and_then(|i| imp.project.sequence(i.id)).unwrap();
assert_eq!(m63.video_tracks[0].items.len(), 1, "M63 keeps its nested M64");
}
#[test]
fn media_cut_off_by_xml_layout_is_complete_when_revisited_at_the_same_media_depth() {
// Both references have media-chain depth zero, but the first is reached through connected
// clips. Its resource must not remain cached as complete after the XML-layout guard fires.
for depth in [125, 126] {
let nested = format!(
"{}<ref-clip lane=\"1\" ref=\"m0\" offset=\"0s\" duration=\"1s\"/>{}",
"<gap lane=\"1\" offset=\"0s\" duration=\"1s\">".repeat(depth),
"</gap>".repeat(depth)
);
let xml = format!(
r#"<fcpxml version="1.10"><resources>
<format id="r1" frameDuration="1/24s" width="1920" height="1080"/>
<media id="m0" name="M0"><sequence format="r1"><spine>
<ref-clip ref="m1" offset="0s" duration="1s"/>
</spine></sequence></media>
<media id="m1" name="M1"><sequence format="r1"><spine/></sequence></media>
</resources><library><event>
<project name="A"><sequence format="r1"><spine>{nested}</spine></sequence></project>
<project name="B"><sequence format="r1"><spine>
<ref-clip ref="m0" offset="0s" duration="1s"/>
</spine></sequence></project>
</event></library></fcpxml>"#
);
let (imp, rep) = import(xml.as_bytes(), Format::Fcpxml, None).unwrap();
assert!(rep.mentions("XML") && rep.mentions("nested"), "{rep:?}");
let m0 = imp.project.items.values().find(|i| i.name == "M0").and_then(|i| imp.project.sequence(i.id)).unwrap();
assert_eq!(m0.video_tracks[0].items.len(), 1, "M0 keeps its nested M1 after the shallower revisit");
}
}
#[test]
fn flat_resource_chain_is_bounded_even_when_xml_is_shallow() {
let mut media = String::new();
for i in 0..512 {
media.push_str(&format!(
r#"<media id="m{i}" name="M{i}"><sequence><spine><ref-clip ref="m{}" duration="1s"/></spine></sequence></media>"#,
i + 1
));
}
let xml = format!(
r#"<fcpxml version="1.10"><resources>{media}</resources><project name="P"><sequence><spine><ref-clip ref="m0" duration="1s"/></spine></sequence></project></fcpxml>"#
);
let (imp, rep) = import(xml.as_bytes(), Format::Fcpxml, None).unwrap();
assert!(rep.mentions("nested too deeply"), "{rep:?}");
assert!(imp.project.items.len() <= 65, "the flat resource graph is not expanded beyond the semantic limit");
}
#[test]
fn shallower_retry_does_not_link_a_cached_ancestor_back_to_the_rebuilt_item() {
let mut media = String::new();
for i in 0..64 {
media.push_str(&format!(
r#"<media id="m{i}" name="M{i}"><sequence><spine><ref-clip ref="m{}" duration="1s"/></spine></sequence></media>"#,
(i + 1) % 64
));
}
let xml = format!(
r#"<fcpxml version="1.10"><resources>{media}</resources>
<project name="A"><sequence><spine><ref-clip ref="m0" duration="1s"/></spine></sequence></project>
<project name="B"><sequence><spine><ref-clip ref="m62" duration="1s"/></spine></sequence></project></fcpxml>"#
);
let (imp, rep) = import(xml.as_bytes(), Format::Fcpxml, None).unwrap();
assert!(rep.mentions("cached media"), "{rep:?}");
// Check the final graph, not just the resource-name recursion stack during import.
fn visit(p: &Project, id: ItemId, active: &mut std::collections::HashSet<ItemId>, done: &mut std::collections::HashSet<ItemId>) {
if done.contains(&id) {
return;
}
assert!(active.insert(id), "cached media introduced a cycle at {id:?}");
if let Some(seq) = p.sequence(id) {
for item in seq.all_tracks().flat_map(|track| &track.items) {
visit(p, item.item, active, done);
}
}
active.remove(&id);
done.insert(id);
}
let mut active = std::collections::HashSet::new();
let mut done = std::collections::HashSet::new();
for id in imp.project.items.keys() {
visit(&imp.project, *id, &mut active, &mut done);
}
}
+1
View File
@@ -170,6 +170,7 @@ fn scene(ui: &mut egui::Ui, ours: bool) {
}
fn render(ours: bool, hover: Option<&str>, open: Option<&str>) -> image::RgbaImage {
let _gpu = crate::gpu_test_lock();
// Two of the combo boxes have no label: the scene compares looks, not names.
let mut h = Harness::builder().with_size(egui::vec2(480.0, 700.0)).with_accessibility_check(false).build_ui(move |ui| scene(ui, ours));
h.run();
+11
View File
@@ -27,3 +27,14 @@ pub mod text;
pub mod theme;
pub use combo::ComboBox;
/// Lets one unit test at a time hold a wgpu renderer. Each renderer creates its own Vulkan instance
/// and device. When one test uses its device while a test on another thread creates or destroys
/// one, the Vulkan loader now and then crashes the test binary with SIGSEGV. Take the lock before
/// building the harness (`let _gpu = crate::gpu_test_lock();`), so it is released after the harness
/// is dropped. A test that failed while holding it does not block the others.
#[cfg(test)]
pub(crate) fn gpu_test_lock() -> std::sync::MutexGuard<'static, ()> {
static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
+1
View File
@@ -123,6 +123,7 @@ mod tests {
#[test]
fn selected_tab_erases_a_baseline_centered_on_its_bottom_edge() {
let _gpu = crate::gpu_test_lock();
let mut h = egui_kittest::Harness::builder().with_size(Vec2::new(120.0, 80.0)).wgpu().build_ui(|ui| {
ui.painter().rect_filled(ui.max_rect(), 0.0, Color32::WHITE);
let rect = Rect::from_min_size(egui::pos2(10.0, 10.0), Vec2::new(100.0, 26.0));
+16
View File
@@ -9,6 +9,7 @@
mod fixtures;
use std::path::PathBuf;
use std::sync::{Mutex, MutexGuard, PoisonError};
use egui::{Color32, Context, Event, FullOutput, Modifiers, PointerButton, Pos2, RawInput, Rect, Response, TextureId, Ui};
use egui_kittest::{TestRenderer, wgpu::WgpuTestRenderer};
@@ -28,6 +29,17 @@ struct Capture {
meshes: Option<Vec<Mesh>>,
}
/// Lets one test at a time hold a wgpu renderer. Each renderer creates its own Vulkan instance and
/// device. When one test uses its device while a test on another thread creates or destroys one,
/// the Vulkan loader now and then crashes the test binary with SIGSEGV.
static GPU_LOCK: Mutex<()> = Mutex::new(());
/// Take this before creating a renderer (`let _gpu = gpu_lock();`), so it is released after the
/// renderer is dropped. A test that failed while holding it does not block the others.
fn gpu_lock() -> MutexGuard<'static, ()> {
GPU_LOCK.lock().unwrap_or_else(PoisonError::into_inner)
}
fn context(theme: Theme, scale: f32) -> Context {
let ctx = Context::default();
ctx.set_theme(theme.egui());
@@ -184,6 +196,7 @@ impl Artifacts {
#[test]
fn rendered_controls_match_independent_reference_pixels_and_meshes() {
let _gpu = gpu_lock();
let mut renderer = WgpuTestRenderer::new();
let mut artifacts = Artifacts::new("controls");
let mut count = 0;
@@ -213,6 +226,7 @@ fn rendered_controls_match_independent_reference_pixels_and_meshes() {
#[test]
fn visual_comparisons_detect_color_spacing_and_stroke_mutations() {
let _gpu = gpu_lock();
let mut renderer = WgpuTestRenderer::new();
let mut artifacts = Artifacts::new("negative-controls");
let mut count = 0;
@@ -241,6 +255,7 @@ fn toolbar_fitting_matches_literal_group_and_overflow_positions() {
use craft_ui::layout::fit_toolbar;
use egui::{Align, Layout, Stroke, StrokeKind, UiBuilder, pos2, vec2};
let _gpu = gpu_lock();
let mut renderer = WgpuTestRenderer::new();
let mut artifacts = Artifacts::new("toolbar");
let mut count = 0;
@@ -295,6 +310,7 @@ fn widget_palette_renders_all_five_states_with_existing_borders_and_expansion()
use craft_ui::theme::WidgetPalette;
use egui::{CornerRadius, Stroke, StrokeKind, pos2, vec2};
let _gpu = gpu_lock();
let mut renderer = WgpuTestRenderer::new();
let mut artifacts = Artifacts::new("widget-palette");
for theme in [Theme::Light, Theme::Dark] {
+59 -43
View File
@@ -56,48 +56,8 @@ pub fn animation(width: u32, height: u32, frames: &[AnimFrame], opts: &AnimOptio
if frames.is_empty() {
return Err(Error::Encode("an animation needs a frame"));
}
let mut anmf = Vec::new();
let mut any_alpha = opts.alpha;
for f in frames {
if f.width == 0 || f.height == 0 || f.x % 2 != 0 || f.y % 2 != 0 {
return Err(Error::Encode("a frame is empty or not at an even position"));
}
if f.x.checked_add(f.width).is_none_or(|r| r > width) || f.y.checked_add(f.height).is_none_or(|b| b > height) {
return Err(Error::Encode("a frame reaches past the canvas"));
}
if f.duration_ms >= 1 << 24 {
return Err(Error::Encode("a frame duration is 2^24 ms or more"));
}
let mut body = Vec::new();
u24(&mut body, f.x / 2);
u24(&mut body, f.y / 2);
u24(&mut body, f.width - 1);
u24(&mut body, f.height - 1);
u24(&mut body, f.duration_ms);
// Bit 1: do not blend; bit 0: dispose to the background.
body.push(u8::from(!f.blend) << 1 | u8::from(f.dispose_to_background));
match f.payload {
Payload::Lossy { vp8, alpha } => {
if f.width > MAX_VP8_DIMENSION || f.height > MAX_VP8_DIMENSION {
return Err(Error::Encode("a lossy frame is at most 16383 pixels a side"));
}
if let Some(a) = alpha {
if a.len() as u64 != u64::from(f.width) * u64::from(f.height) {
return Err(Error::Encode("an alpha plane does not match its frame"));
}
any_alpha = true;
let mut alph = Vec::with_capacity(a.len() + 1);
// Header byte: no pre-processing, no filtering, no compression.
alph.push(0);
alph.extend_from_slice(a);
chunk(&mut body, b"ALPH", &alph)?;
}
chunk(&mut body, b"VP8 ", vp8)?;
}
Payload::Lossless { vp8l } => chunk(&mut body, b"VP8L", vp8l)?,
}
chunk(&mut anmf, b"ANMF", &body)?;
}
// Determine the header flag before writing, without retaining another full animation.
let any_alpha = opts.alpha || frames.iter().any(|f| matches!(f.payload, Payload::Lossy { alpha: Some(_), .. }));
let mut body = b"WEBP".to_vec();
let flags = [(opts.icc.is_some(), 0x20u8), (any_alpha, 0x10), (opts.exif.is_some(), 0x08), (opts.xmp.is_some(), 0x04), (true, 0x02)]
.iter()
@@ -113,7 +73,46 @@ pub fn animation(width: u32, height: u32, frames: &[AnimFrame], opts: &AnimOptio
let mut anim = opts.background_bgra.to_vec();
anim.extend_from_slice(&opts.loop_count.to_le_bytes());
chunk(&mut body, b"ANIM", &anim)?;
body.extend_from_slice(&anmf);
for f in frames {
if f.width == 0 || f.height == 0 || f.x % 2 != 0 || f.y % 2 != 0 {
return Err(Error::Encode("a frame is empty or not at an even position"));
}
if f.x.checked_add(f.width).is_none_or(|r| r > width) || f.y.checked_add(f.height).is_none_or(|b| b > height) {
return Err(Error::Encode("a frame reaches past the canvas"));
}
if f.duration_ms >= 1 << 24 {
return Err(Error::Encode("a frame duration is 2^24 ms or more"));
}
let mut frame_body = Vec::new();
u24(&mut frame_body, f.x / 2);
u24(&mut frame_body, f.y / 2);
u24(&mut frame_body, f.width - 1);
u24(&mut frame_body, f.height - 1);
u24(&mut frame_body, f.duration_ms);
// Bit 1: do not blend; bit 0: dispose to the background.
frame_body.push(u8::from(!f.blend) << 1 | u8::from(f.dispose_to_background));
match f.payload {
Payload::Lossy { vp8, alpha } => {
if f.width > MAX_VP8_DIMENSION || f.height > MAX_VP8_DIMENSION {
return Err(Error::Encode("a lossy frame is at most 16383 pixels a side"));
}
if let Some(a) = alpha {
if a.len() as u64 != u64::from(f.width) * u64::from(f.height) {
return Err(Error::Encode("an alpha plane does not match its frame"));
}
let mut alph = Vec::with_capacity(a.len() + 1);
// Header byte: no pre-processing, no filtering, no compression.
alph.push(0);
alph.extend_from_slice(a);
chunk(&mut frame_body, b"ALPH", &alph)?;
}
chunk(&mut frame_body, b"VP8 ", vp8)?;
}
Payload::Lossless { vp8l } => chunk(&mut frame_body, b"VP8L", vp8l)?,
}
chunk(&mut body, b"ANMF", &frame_body)?;
}
if let Some(exif) = opts.exif {
chunk(&mut body, b"EXIF", exif)?;
}
@@ -231,6 +230,23 @@ mod tests {
assert_eq!(&file[anim + 8..anim + 14], &[1, 2, 3, 4, 0, 0]);
}
#[test]
fn lossy_container_keeps_alpha_payload_padding_and_xmp() {
// Container-only fixture: the writer must not reinterpret an encoder's VP8 bytes.
let vp8 = [11, 22, 33];
let alpha = [128];
let f = frame(Payload::Lossy { vp8: &vp8, alpha: Some(&alpha) }, 0, 0, 1, 1, 17);
let file = animation(1, 1, &[f], &AnimOptions { xmp: Some(b"xmp"), ..Default::default() }).unwrap();
let info = scan(&file, 10).unwrap();
assert!(info.alpha && info.complete);
assert_eq!(info.frames[0].duration_ms, 17);
let at = file.windows(4).position(|c| c == b"ANMF").unwrap();
let payload = at + 8 + 16;
assert_eq!(&file[payload..payload + 10], &[b'A', b'L', b'P', b'H', 2, 0, 0, 0, 0, 128]);
assert_eq!(&file[payload + 10..payload + 22], &[b'V', b'P', b'8', b' ', 3, 0, 0, 0, 11, 22, 33, 0]);
assert_eq!(&file[file.len() - 12..], &[b'X', b'M', b'P', b' ', 3, 0, 0, 0, b'x', b'm', b'p', 0]);
}
#[test]
fn what_does_not_fit_is_refused() {
let px = vp8l(2, 2, [0; 4]);
+1
View File
@@ -17,6 +17,7 @@
//! Input is treated as hostile: every read is bounds-checked, chunk sizes are checked against the
//! file, and the number of frames recorded is capped by the caller.
pub mod anim;
#[cfg(feature = "frames")]
pub mod frames;
#[cfg(test)]
+6
View File
@@ -0,0 +1,6 @@
//! Guard the public module wiring: the implementation must be usable by consumers.
#[test]
fn public_animation_api_refuses_empty_movies() {
let result = craft_webp::anim::animation(1, 1, &[], &craft_webp::anim::AnimOptions::default());
assert!(result.is_err());
}
+348 -28
View File
@@ -3,6 +3,7 @@
//! This layer recognizes the document, stage, scene, layer and keyframe fields defined by XFL.
//! It does not construct a consumer's animation model. More detailed payloads remain available
//! through [`Document::root`] for the application adapter.
use crate::shape::XflElement;
use craft_xml::{Doctype, El, Error as XmlError, Limits, Local, Options, Pat, UnknownEntity};
use std::borrow::Cow;
@@ -104,36 +105,102 @@ pub struct FrameModel {
}
/// A timeline (`DOMTimeline`) listed under `DOMDocument/timelines`.
#[derive(Clone, Copy, Debug)]
pub struct Timeline<'a> {
element: &'a El,
#[derive(Debug)]
pub struct Timeline<'a, E: XflElement = El> {
element: &'a E,
}
impl<E: XflElement> Copy for Timeline<'_, E> {}
impl<E: XflElement> Clone for Timeline<'_, E> {
fn clone(&self) -> Self {
*self
}
}
/// A native `DOMLayer` element.
#[derive(Clone, Copy, Debug)]
pub struct Layer<'a> {
element: &'a El,
#[derive(Debug)]
pub struct Layer<'a, E: XflElement = El> {
element: &'a E,
}
impl<E: XflElement> Copy for Layer<'_, E> {}
impl<E: XflElement> Clone for Layer<'_, E> {
fn clone(&self) -> Self {
*self
}
}
/// A native `DOMFrame` element.
#[derive(Clone, Copy, Debug)]
pub struct Frame<'a> {
element: &'a El,
#[derive(Debug)]
pub struct Frame<'a, E: XflElement = El> {
element: &'a E,
}
fn attr_last<'a>(element: &'a El, name: &str) -> Option<&'a str> {
element.attrs.iter().rev().find(|(key, _)| key == name).map(|(_, value)| value.as_str())
/// An `Include` entry in the document's `<symbols>` library list.
#[derive(Debug)]
pub struct SymbolInclude<'a, E: XflElement = El> {
element: &'a E,
}
fn number_f32(element: &El, name: &str) -> Option<f32> {
/// A folder entry in the document's `<folders>` library list.
#[derive(Debug)]
pub struct LibraryFolder<'a, E: XflElement = El> {
element: &'a E,
}
/// Native metadata stored on a library `DOMSymbolItem`.
#[derive(Debug)]
pub struct SymbolItem<'a, E: XflElement = El> {
element: &'a E,
}
impl<E: XflElement> Copy for Frame<'_, E> {}
impl<E: XflElement> Clone for Frame<'_, E> {
fn clone(&self) -> Self {
*self
}
}
impl<E: XflElement> Copy for SymbolInclude<'_, E> {}
impl<E: XflElement> Clone for SymbolInclude<'_, E> {
fn clone(&self) -> Self {
*self
}
}
impl<E: XflElement> Copy for LibraryFolder<'_, E> {}
impl<E: XflElement> Clone for LibraryFolder<'_, E> {
fn clone(&self) -> Self {
*self
}
}
impl<E: XflElement> Copy for SymbolItem<'_, E> {}
impl<E: XflElement> Clone for SymbolItem<'_, E> {
fn clone(&self) -> Self {
*self
}
}
fn attr_last<'a>(element: &'a impl XflElement, name: &str) -> Option<&'a str> {
element.attr(name)
}
fn number_f32(element: &impl XflElement, name: &str) -> Option<f32> {
attr_last(element, name).and_then(|value| value.trim().parse::<f32>().ok()).filter(|value| value.is_finite())
}
fn number_u32(element: &El, name: &str) -> Option<u32> {
fn number_u32(element: &impl XflElement, name: &str) -> Option<u32> {
attr_last(element, name).and_then(|value| value.trim().parse::<u32>().ok())
}
fn boolean(element: &El, name: &str) -> Option<bool> {
fn boolean(element: &impl XflElement, name: &str) -> Option<bool> {
attr_last(element, name).map(|value| value.trim() == "true")
}
@@ -187,7 +254,7 @@ impl Document {
let profile = self
.root
.elements()
.find(|element| Local("flash_profile").matches(&element.name) && boolean(element, "current") == Some(true))
.find(|element| Local("flash_profile").matches(&element.name) && boolean(*element, "current") == Some(true))
.or_else(|| self.root.elements().find(|element| Local("flash_profile").matches(&element.name)))
.unwrap_or(&self.root);
let properties = profile.child(Local("PublishFlashProperties")).unwrap_or(profile);
@@ -237,23 +304,28 @@ impl Document {
}
}
impl<'a> Timeline<'a> {
impl<'a, E: XflElement> Timeline<'a, E> {
/// Creates a borrowed XFL timeline view over a consumer's compatible XML element.
pub fn new(element: &'a E) -> Self {
Self { element }
}
pub fn name(self) -> Option<&'a str> {
attr_last(self.element, "name")
}
/// Raw timeline XML for application-specific payload conversion.
pub fn element(self) -> &'a El {
pub fn element(self) -> &'a E {
self.element
}
/// Layers in native order. Non-`DOMLayer` extension nodes are skipped.
pub fn layers(self) -> impl Iterator<Item = Layer<'a>> {
pub fn layers(self) -> impl Iterator<Item = Layer<'a, E>> {
self.element
.child(Local("layers"))
.child("layers")
.into_iter()
.flat_map(El::elements)
.filter(|element| Local("DOMLayer").matches(&element.name))
.flat_map(|element| element.elements())
.filter(|element| element.local_name() == "DOMLayer")
.map(|element| Layer { element })
}
@@ -262,7 +334,12 @@ impl<'a> Timeline<'a> {
}
}
impl<'a> Layer<'a> {
impl<'a, E: XflElement> Layer<'a, E> {
/// Creates a borrowed XFL layer view over a consumer's compatible XML element.
pub fn new(element: &'a E) -> Self {
Self { element }
}
pub fn name(self) -> Option<&'a str> {
attr_last(self.element, "name")
}
@@ -296,12 +373,16 @@ impl<'a> Layer<'a> {
}
/// Frames in native order. Non-`DOMFrame` extension nodes are skipped.
pub fn frames(self) -> impl Iterator<Item = Frame<'a>> {
pub fn element(self) -> &'a E {
self.element
.child(Local("frames"))
}
pub fn frames(self) -> impl Iterator<Item = Frame<'a, E>> {
self.element
.child("frames")
.into_iter()
.flat_map(El::elements)
.filter(|element| Local("DOMFrame").matches(&element.name))
.flat_map(|element| element.elements())
.filter(|element| element.local_name() == "DOMFrame")
.map(|element| Frame { element })
}
@@ -325,7 +406,12 @@ impl<'a> Layer<'a> {
}
}
impl<'a> Frame<'a> {
impl<'a, E: XflElement> Frame<'a, E> {
/// Creates a borrowed XFL frame view over a consumer's compatible XML element.
pub fn new(element: &'a E) -> Self {
Self { element }
}
pub fn index(self) -> Option<u32> {
number_u32(self.element, "index")
}
@@ -347,7 +433,7 @@ impl<'a> Frame<'a> {
}
/// Raw frame XML for application-specific shape, action, sound and tween conversion.
pub fn element(self) -> &'a El {
pub fn element(self) -> &'a E {
self.element
}
@@ -367,6 +453,112 @@ impl<'a> Frame<'a> {
}
}
impl<'a, E: XflElement> SymbolInclude<'a, E> {
/// Creates a borrowed view over a native `Include` element.
pub fn new(element: &'a E) -> Self {
Self { element }
}
/// The package-relative library XML path from `href`, kept in native string form.
pub fn href(self) -> Option<&'a str> {
attr_last(self.element, "href")
}
/// The source element for application-specific path resolution.
pub fn element(self) -> &'a E {
self.element
}
}
impl<'a, E: XflElement> LibraryFolder<'a, E> {
/// Creates a borrowed view over a native `DOMFolderItem` element.
pub fn new(element: &'a E) -> Self {
Self { element }
}
/// The folder path from `name`, kept in native string form.
pub fn name(self) -> Option<&'a str> {
attr_last(self.element, "name")
}
/// The source element for application-specific path validation.
pub fn element(self) -> &'a E {
self.element
}
}
impl<'a, E: XflElement> SymbolItem<'a, E> {
/// Creates a borrowed view over a native `DOMSymbolItem` element.
pub fn new(element: &'a E) -> Self {
Self { element }
}
pub fn name(self) -> Option<&'a str> {
attr_last(self.element, "name")
}
pub fn symbol_type(self) -> Option<&'a str> {
attr_last(self.element, "symbolType")
}
pub fn linkage_export_for_action_script(self) -> Option<&'a str> {
attr_last(self.element, "linkageExportForAS")
}
pub fn linkage_class_name(self) -> Option<&'a str> {
attr_last(self.element, "linkageClassName")
}
pub fn linkage_identifier(self) -> Option<&'a str> {
attr_last(self.element, "linkageIdentifier")
}
pub fn linkage_base_class(self) -> Option<&'a str> {
attr_last(self.element, "linkageBaseClass")
}
/// Whether the symbol declares any native 9-slice scale-grid attribute, even if malformed.
pub fn has_scale_grid(self) -> bool {
["scaleGridLeft", "scaleGridRight", "scaleGridTop", "scaleGridBottom"].iter().any(|name| attr_last(self.element, name).is_some())
}
pub fn scale_grid_left(self) -> Option<f32> {
number_f32(self.element, "scaleGridLeft")
}
pub fn scale_grid_right(self) -> Option<f32> {
number_f32(self.element, "scaleGridRight")
}
pub fn scale_grid_top(self) -> Option<f32> {
number_f32(self.element, "scaleGridTop")
}
pub fn scale_grid_bottom(self) -> Option<f32> {
number_f32(self.element, "scaleGridBottom")
}
/// The symbol's optional timeline wrapper, for the consumer to map into its own model.
pub fn timeline(self) -> Option<&'a E> {
self.element.child("timeline")
}
/// The source element for application-specific symbol validation and mapping.
pub fn element(self) -> &'a E {
self.element
}
}
/// Symbol library entries in source order, skipping non-`Include` extension nodes.
pub fn symbol_includes<'a, E: XflElement>(root: &'a E) -> impl Iterator<Item = SymbolInclude<'a, E>> + 'a {
root.child("symbols").into_iter().flat_map(XflElement::elements).filter(|element| element.local_name() == "Include").map(SymbolInclude::new)
}
/// Library folders in source order, skipping non-`DOMFolderItem` extension nodes.
pub fn library_folders<'a, E: XflElement>(root: &'a E) -> impl Iterator<Item = LibraryFolder<'a, E>> + 'a {
root.child("folders").into_iter().flat_map(XflElement::elements).filter(|element| element.local_name() == "DOMFolderItem").map(LibraryFolder::new)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -408,6 +600,91 @@ mod tests {
assert_eq!(frame.element().child("elements").unwrap().child("Shape").unwrap().name, "Shape");
}
#[test]
fn borrowed_timeline_views_accept_compatible_element_adapters() {
#[derive(Debug)]
struct AppElement {
name: String,
attributes: Vec<(String, String)>,
children: Vec<AppElement>,
}
impl XflElement for AppElement {
type Elements<'a> = std::slice::Iter<'a, Self>;
fn local_name(&self) -> &str {
self.name.rsplit(':').next().unwrap_or(&self.name)
}
fn attr(&self, name: &str) -> Option<&str> {
self.attributes.iter().rev().find(|(key, _)| key == name).map(|(_, value)| value.as_str())
}
fn elements(&self) -> Self::Elements<'_> {
self.children.iter()
}
fn text_content(&self) -> Cow<'_, str> {
Cow::Borrowed("")
}
}
fn element(name: &str, attributes: &[(&str, &str)], children: Vec<AppElement>) -> AppElement {
AppElement {
name: name.to_string(),
attributes: attributes.iter().map(|(key, value)| ((*key).to_string(), (*value).to_string())).collect(),
children,
}
}
let root = element(
"x:DOMDocument",
&[],
vec![element(
"x:timelines",
&[],
vec![element(
"x:DOMTimeline",
&[("name", "Old"), ("name", "Scene")],
vec![element(
"x:layers",
&[],
vec![element(
"x:DOMLayer",
&[("name", "Old"), ("name", "Layer"), ("color", "#abcdef"), ("visible", "false")],
vec![element(
"x:frames",
&[],
vec![element(
"x:DOMFrame",
&[
("index", "999"),
("index", "3"),
("duration", "2"),
("name", "Intro"),
("labelType", "name"),
("tweenType", "motion"),
],
vec![],
)],
)],
)],
)],
)],
)],
);
let timeline_element = root.child("timelines").unwrap().child("DOMTimeline").unwrap();
let timeline = Timeline::new(timeline_element);
assert_eq!(timeline.name(), Some("Scene"));
let layer = timeline.layers().next().unwrap();
assert_eq!(layer.name(), Some("Layer"));
assert_eq!(layer.color(), Some("#abcdef"));
assert_eq!(layer.visible(), Some(false));
let frame = layer.frames().next().unwrap();
assert_eq!((frame.index(), frame.duration()), (Some(3), Some(2)));
assert_eq!((frame.name(), frame.label_type(), frame.tween_type()), (Some("Intro"), Some("name"), Some("motion")));
}
#[test]
fn uses_last_duplicate_attribute_like_the_legacy_xfl_adapter() {
let document = Document::parse(br##"<DOMDocument width="1" width="2"><timelines/></DOMDocument>"##).unwrap();
@@ -521,4 +798,47 @@ mod tests {
assert_eq!(settings.action_script_version.as_deref(), Some("3"));
assert_eq!(settings.document_class.as_deref(), Some("RootClass"));
}
#[test]
fn reads_symbol_library_entries_and_native_symbol_metadata() {
let document = Document::parse(
br#"<DOMDocument><symbols><Extension/><Include href="LIBRARY/Shape.xml"/><Include href="LIBRARY/Button.xml"/></symbols><folders><Extension/><DOMFolderItem name="Characters/Hero"/></folders></DOMDocument>"#,
)
.unwrap();
let includes: Vec<_> = symbol_includes(document.root()).map(SymbolInclude::href).collect();
assert_eq!(includes, [Some("LIBRARY/Shape.xml"), Some("LIBRARY/Button.xml")]);
let include = symbol_includes(document.root()).next().unwrap();
assert_eq!(include.element().local_name(), "Include");
let folders: Vec<_> = library_folders(document.root()).map(LibraryFolder::name).collect();
assert_eq!(folders, [Some("Characters/Hero")]);
let folder = library_folders(document.root()).next().unwrap();
assert_eq!(folder.element().local_name(), "DOMFolderItem");
let symbol = Document::parse(
br#"<x:DOMSymbolItem xmlns:x="urn:adobe:xfl" name="Characters/Hero" name="Characters/Final" symbolType="graphic" linkageExportForAS="true" linkageClassName="example.Hero" linkageIdentifier="legacy.Hero" linkageBaseClass="flash.display.MovieClip" scaleGridLeft="10" scaleGridRight="90" scaleGridTop="5" scaleGridBottom="75"><x:timeline><x:DOMTimeline/></x:timeline></x:DOMSymbolItem>"#,
)
.unwrap();
let item = SymbolItem::new(symbol.root());
assert_eq!(item.element().local_name(), "DOMSymbolItem");
assert_eq!(item.name(), Some("Characters/Final"), "the last duplicate attribute wins");
assert_eq!(item.symbol_type(), Some("graphic"));
assert_eq!(item.linkage_export_for_action_script(), Some("true"));
assert_eq!(item.linkage_class_name(), Some("example.Hero"));
assert_eq!(item.linkage_identifier(), Some("legacy.Hero"));
assert_eq!(item.linkage_base_class(), Some("flash.display.MovieClip"));
assert_eq!(
(item.scale_grid_left(), item.scale_grid_right(), item.scale_grid_top(), item.scale_grid_bottom()),
(Some(10.0), Some(90.0), Some(5.0), Some(75.0))
);
assert!(item.has_scale_grid());
assert!(item.timeline().is_some());
let malformed = Document::parse(br#"<DOMSymbolItem scaleGridLeft="not-a-number"/>"#).unwrap();
let malformed = SymbolItem::new(malformed.root());
assert!(malformed.has_scale_grid(), "attribute presence remains observable when its value is malformed");
assert_eq!(malformed.scale_grid_left(), None);
let absent = Document::parse(b"<DOMSymbolItem/>").unwrap();
assert!(!SymbolItem::new(absent.root()).has_scale_grid());
}
}
+239
View File
@@ -0,0 +1,239 @@
//! Borrowed views of native XFL filter records.
//!
//! This module parses the wire-level filter names, fields, and gradient-ramp entries. Consumers
//! remain responsible for mapping those values into their own rendering/filter models.
use crate::shape::{self, Color, XflElement, boolean, integer, number};
/// Native filter variants understood by XFL.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum FilterKind {
Blur,
DropShadow,
Glow,
AdjustColor,
Bevel,
GradientGlow,
GradientBevel,
}
/// Bevel/glow placement recorded by the native `type` attribute.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum FilterSide {
Inner,
Outer,
Full,
}
/// The native `<filters>` list on an element, preserving source order and unknown records.
#[derive(Clone, Copy, Debug)]
pub struct FilterList<'a, E: XflElement> {
wrapper: Option<&'a E>,
}
impl<'a, E: XflElement> FilterList<'a, E> {
pub fn new(element: &'a E) -> Self {
Self { wrapper: element.child("filters") }
}
pub fn iter(&self) -> impl Iterator<Item = Filter<'a, E>> + 'a {
let wrapper = self.wrapper;
wrapper.into_iter().flat_map(XflElement::elements).map(|element| Filter { element })
}
}
/// One native filter record. Unknown records are retained so the consumer can report/skip them.
#[derive(Clone, Copy, Debug)]
pub struct Filter<'a, E: XflElement> {
element: &'a E,
}
impl<'a, E: XflElement> Filter<'a, E> {
pub fn kind(&self) -> Option<FilterKind> {
match self.element.local_name() {
"BlurFilter" => Some(FilterKind::Blur),
"DropShadowFilter" => Some(FilterKind::DropShadow),
"GlowFilter" => Some(FilterKind::Glow),
"AdjustColorFilter" => Some(FilterKind::AdjustColor),
"BevelFilter" => Some(FilterKind::Bevel),
"GradientGlowFilter" => Some(FilterKind::GradientGlow),
"GradientBevelFilter" => Some(FilterKind::GradientBevel),
_ => None,
}
}
/// XFL treats filters as enabled unless explicitly set to false.
pub fn enabled(&self) -> bool {
boolean(self.element, "enabled") != Some(false)
}
/// Quality is a native integer in the range 1 through 3, defaulting to 1.
pub fn quality(&self) -> u8 {
integer(self.element, "quality").unwrap_or(1).clamp(1, 3) as u8
}
pub fn blur_x(&self) -> Option<f32> {
number(self.element, "blurX")
}
pub fn blur_y(&self) -> Option<f32> {
number(self.element, "blurY")
}
pub fn strength(&self) -> Option<f32> {
number(self.element, "strength")
}
pub fn angle(&self) -> Option<f32> {
number(self.element, "angle")
}
pub fn distance(&self) -> Option<f32> {
number(self.element, "distance")
}
pub fn brightness(&self) -> Option<f32> {
number(self.element, "brightness")
}
pub fn contrast(&self) -> Option<f32> {
number(self.element, "contrast")
}
pub fn saturation(&self) -> Option<f32> {
number(self.element, "saturation")
}
pub fn hue(&self) -> Option<f32> {
number(self.element, "hue")
}
pub fn inner(&self) -> Option<bool> {
boolean(self.element, "inner")
}
pub fn knockout(&self) -> Option<bool> {
boolean(self.element, "knockout")
}
pub fn hide_object(&self) -> Option<bool> {
boolean(self.element, "hideObject")
}
pub fn side(&self) -> Option<FilterSide> {
match self.element.attr("type").map(str::trim) {
Some("inner") => Some(FilterSide::Inner),
Some("outer") => Some(FilterSide::Outer),
Some("full") => Some(FilterSide::Full),
_ => None,
}
}
pub fn color(&self) -> Color {
shape::color(self.element, "color", "alpha")
}
pub fn shadow_color(&self) -> Color {
shape::color(self.element, "shadowColor", "shadowAlpha")
}
/// Explicit highlight color, absent when the native attribute is missing so consumers can
/// apply the format-specific white highlight default.
pub fn highlight_color(&self) -> Option<Color> {
self.element.attr("highlightColor").map(|_| shape::color(self.element, "highlightColor", "highlightAlpha"))
}
pub fn highlight_alpha(&self) -> Option<f32> {
number(self.element, "highlightAlpha")
}
/// Native gradient entries, either direct children or one wrapper level below the filter.
pub fn ramp_entries(&self) -> impl Iterator<Item = RampEntry<'a, E>> + 'a {
self.element
.elements()
.flat_map(|child| std::iter::once(child).chain(child.elements()))
.filter(|entry| entry.local_name() == "GradientEntry")
.map(|element| RampEntry { element })
}
}
/// One XFL gradient-ramp entry.
#[derive(Clone, Copy, Debug)]
pub struct RampEntry<'a, E: XflElement> {
element: &'a E,
}
impl<'a, E: XflElement> RampEntry<'a, E> {
pub fn ratio(&self) -> Option<f32> {
number(self.element, "ratio")
}
pub fn color(&self) -> Color {
shape::color(self.element, "color", "alpha")
}
}
#[cfg(test)]
mod tests {
use super::*;
fn parse(source: &[u8]) -> craft_xml::El {
craft_xml::parse(source).expect("well-formed test XML")
}
#[test]
fn recognizes_filter_kinds_and_preserves_unknown_entries_and_order() {
let root = parse(br##"<DOMSymbolInstance><filters><BlurFilter/><x:GlowFilter xmlns:x="urn:xfl" enabled="false"/><FutureFilter/><GradientBevelFilter/></filters></DOMSymbolInstance>"##);
let filters: Vec<_> = FilterList::new(&root).iter().collect();
assert_eq!(filters.len(), 4);
assert_eq!(filters[0].kind(), Some(FilterKind::Blur));
assert_eq!(filters[1].kind(), Some(FilterKind::Glow));
assert!(!filters[1].enabled());
assert_eq!(filters[2].kind(), None);
assert_eq!(filters[3].kind(), Some(FilterKind::GradientBevel));
}
#[test]
fn reads_native_filter_values_colors_and_gradient_entries_in_order() {
let root = parse(
br##"<DOMSymbolInstance><filters><DropShadowFilter blurX="7.5" blurY="8" strength="1.25" quality="9" angle="45" distance="4" color="#123456" alpha="0.5" inner="true" knockout="false" hideObject="true"/><BevelFilter type="full" shadowColor="#010203" shadowAlpha="0.25" highlightColor="#aabbcc" highlightAlpha="0.75"/><GradientGlowFilter><GradientEntry ratio="255" color="#ff0000"/><Ramp><GradientEntry ratio="64" color="#00ff00"/></Ramp></GradientGlowFilter></filters></DOMSymbolInstance>"##,
);
let filters: Vec<_> = FilterList::new(&root).iter().collect();
let shadow = &filters[0];
assert_eq!(shadow.quality(), 3);
assert_eq!(
(shadow.blur_x(), shadow.blur_y(), shadow.strength(), shadow.angle(), shadow.distance()),
(Some(7.5), Some(8.0), Some(1.25), Some(45.0), Some(4.0))
);
assert_eq!(shadow.color(), Color { rgb: 0x123456, alpha: 0.5 });
assert_eq!(shadow.inner(), Some(true));
assert_eq!(shadow.knockout(), Some(false));
assert_eq!(shadow.hide_object(), Some(true));
let bevel = &filters[1];
assert_eq!(bevel.side(), Some(FilterSide::Full));
assert_eq!(bevel.shadow_color(), Color { rgb: 0x010203, alpha: 0.25 });
assert_eq!(bevel.highlight_color(), Some(Color { rgb: 0xaabbcc, alpha: 0.75 }));
let ramp: Vec<_> = filters[2].ramp_entries().collect();
assert_eq!(ramp.iter().map(RampEntry::ratio).collect::<Vec<_>>(), [Some(255.0), Some(64.0)]);
assert_eq!(ramp[0].color(), Color { rgb: 0xff0000, alpha: 1.0 });
}
#[test]
fn absent_lists_are_empty_and_values_use_shared_wire_validation() {
let root = parse(b"<DOMSymbolInstance/>");
assert_eq!(FilterList::new(&root).iter().count(), 0);
let root = parse(br##"<BlurFilter quality="bad" blurX="NaN" enabled="unexpected"/>"##);
let filter = FilterList::new(&root).iter().next();
assert_eq!(filter.map(|filter| filter.kind()), None);
let parent =
parse(br##"<DOMSymbolInstance><filters><BlurFilter quality="bad" blurX="NaN" enabled="unexpected"/></filters></DOMSymbolInstance>"##);
let filter = FilterList::new(&parent).iter().next().expect("filter entry");
assert_eq!(filter.kind(), Some(FilterKind::Blur));
assert_eq!(filter.quality(), 1);
assert_eq!(filter.blur_x(), None);
assert!(!filter.enabled());
}
}
+10
View File
@@ -27,10 +27,20 @@ use std::collections::HashMap;
pub mod document;
/// XFL's edge-attribute grammar, independent of any consuming application's geometry types.
pub mod edges;
/// Borrowed views of native XFL filter records.
pub mod filters;
/// Borrowed views of native XFL media library records.
pub mod media;
/// Borrowed views of native XFL shape and style elements.
pub mod shape;
/// Borrowed views of native XFL shape-hint records.
pub mod shape_hints;
/// Borrowed views of native XFL frame-sound and envelope records.
pub mod sound;
/// Borrowed views of native XFL text-field and text-run schema.
pub mod text;
/// Borrowed views of native XFL tween and custom-ease records.
pub mod tween;
/// Largest single entry inflated (bytes).
pub const MAX_ENTRY: usize = 256 * 1024 * 1024;
+90 -5
View File
@@ -5,6 +5,7 @@
//! compatible borrowed element adapters implemented by consumers.
use crate::edges::{self, Segment};
use std::borrow::Cow;
/// An element source used by the borrowed XFL schema views.
///
@@ -27,6 +28,9 @@ pub trait XflElement {
/// Direct child elements in document order.
fn elements(&self) -> Self::Elements<'_>;
/// This element's direct character data, with XML references resolved.
fn text_content(&self) -> Cow<'_, str>;
/// The first direct child whose local name matches.
fn child(&self, name: &str) -> Option<&Self> {
self.elements().find(|element| element.local_name() == name)
@@ -68,6 +72,10 @@ impl XflElement for craft_xml::El {
fn elements(&self) -> Self::Elements<'_> {
XmlElements { nodes: self.children.iter() }
}
fn text_content(&self) -> Cow<'_, str> {
self.text()
}
}
/// XFL matrix coefficients, in the native units stored in XML.
@@ -88,28 +96,78 @@ pub struct Color {
pub alpha: f32,
}
/// A native XFL instance color record (`<color><Color .../></color>`).
///
/// The view parses the authored tint, brightness, channel multipliers and offsets but does not
/// choose a consumer's color-effect style or clamp values into an application's model. Offsets
/// remain in XFL's native 0–255 scale. Invalid or non-finite channel multipliers use 1, and
/// invalid or non-finite offsets use 0, matching the historical AniCraft importer.
#[derive(Clone, Copy, Debug)]
pub struct NativeColorEffect<'a, E: XflElement> {
element: &'a E,
}
impl Matrix {
/// The matrix used when an XFL element omits its transform.
pub const IDENTITY: Matrix = Matrix { a: 1.0, b: 0.0, c: 0.0, d: 1.0, tx: 0.0, ty: 0.0 };
}
fn number(element: &impl XflElement, name: &str) -> Option<f32> {
impl<'a, E: XflElement> NativeColorEffect<'a, E> {
/// Views a native XFL `Color` element without converting it to an application's model.
pub fn new(element: &'a E) -> Self {
Self { element }
}
/// The native tint amount, when authored.
pub fn tint_multiplier(&self) -> Option<f32> {
number(self.element, "tintMultiplier")
}
/// The native tint color. Missing or malformed color values use black as other XFL colors do.
pub fn tint_color(&self) -> Color {
color(self.element, "tintColor", "__none")
}
/// The native brightness value, when authored.
pub fn brightness(&self) -> Option<f32> {
number(self.element, "brightness")
}
/// Red, green, blue, and alpha multipliers, with identity defaults for absent or invalid data.
pub fn multipliers(&self) -> [f32; 4] {
["redMultiplier", "greenMultiplier", "blueMultiplier", "alphaMultiplier"].map(|name| number(self.element, name).unwrap_or(1.0))
}
/// Red, green, blue, and alpha offsets in XFL's native 0–255 scale.
pub fn offsets(&self) -> [f32; 4] {
["redOffset", "greenOffset", "blueOffset", "alphaOffset"].map(|name| number(self.element, name).unwrap_or(0.0))
}
}
pub(crate) fn number(element: &impl XflElement, name: &str) -> Option<f32> {
element.attr(name)?.trim().parse::<f32>().ok().filter(|value| value.is_finite())
}
fn integer(element: &impl XflElement, name: &str) -> Option<u32> {
pub(crate) fn integer(element: &impl XflElement, name: &str) -> Option<u32> {
element.attr(name)?.trim().parse::<u32>().ok()
}
fn boolean(element: &impl XflElement, name: &str) -> Option<bool> {
pub(crate) fn boolean(element: &impl XflElement, name: &str) -> Option<bool> {
element.attr(name).map(|value| value.trim() == "true")
}
/// Parses XFL's hexadecimal color and alpha attributes. Missing or malformed colors use black;
/// missing, malformed, or non-finite alpha uses full opacity, matching the legacy importer.
pub fn color(element: &impl XflElement, color_attribute: &str, alpha_attribute: &str) -> Color {
let rgb = element.attr(color_attribute).and_then(|value| u32::from_str_radix(value.trim().trim_start_matches('#'), 16).ok()).unwrap_or(0);
let alpha = number(element, alpha_attribute).unwrap_or(1.0).clamp(0.0, 1.0);
color_from_attributes(element.attr(color_attribute), element.attr(alpha_attribute))
}
/// Parses native color and alpha strings after a consumer has read them from its own element
/// adapter. This keeps wire-value interpretation shared even when an application already has a
/// borrowed schema view for the containing element.
pub fn color_from_attributes(color: Option<&str>, alpha: Option<&str>) -> Color {
let rgb = color.and_then(|value| u32::from_str_radix(value.trim().trim_start_matches('#'), 16).ok()).unwrap_or(0);
let alpha = alpha.and_then(|value| value.trim().parse::<f32>().ok()).filter(|value| value.is_finite()).unwrap_or(1.0).clamp(0.0, 1.0);
Color { rgb, alpha }
}
@@ -534,6 +592,14 @@ mod tests {
assert_eq!(fill.color_value(), Color { rgb: 0, alpha: 1.0 });
}
#[test]
fn color_attribute_values_keep_native_defaults_and_clamping() {
assert_eq!(color_from_attributes(Some(" #123456 "), Some("0.4")), Color { rgb: 0x123456, alpha: 0.4 });
assert_eq!(color_from_attributes(Some("invalid"), Some("NaN")), Color { rgb: 0, alpha: 1.0 });
assert_eq!(color_from_attributes(Some("#abcdef"), Some("2")), Color { rgb: 0xabcdef, alpha: 1.0 });
assert_eq!(color_from_attributes(None, Some("-1")), Color { rgb: 0, alpha: 0.0 });
}
#[test]
fn style_table_views_keep_the_legacy_first_child_and_index_behavior() {
let root = parse(b"<DOMShape><fills><LegacyFill index='5'><SolidColor color='#123456'/></LegacyFill></fills><strokes><LegacyStroke index='6'><SolidStroke weight='2'/></LegacyStroke></strokes></DOMShape>");
@@ -565,6 +631,9 @@ mod tests {
fn elements(&self) -> Self::Elements<'_> {
self.children.iter()
}
fn text_content(&self) -> Cow<'_, str> {
Cow::Borrowed("")
}
}
let app = AppElement { name: "x:DOMShape".into(), attrs: vec![("isDrawingObject".into(), "true".into())], children: Vec::new() };
assert_eq!(Shape::new(&app).unwrap().is_drawing_object(), Some(true));
@@ -587,4 +656,20 @@ mod tests {
assert_eq!(Primitive::new(&oval).unwrap().kind(), PrimitiveKind::Oval);
assert!(Primitive::new(&parse(b"<DOMShape/>")).is_none());
}
#[test]
fn native_color_effect_keeps_xfl_values_and_safe_channel_defaults() {
let element = parse(
br##"<x:Color xmlns:x="urn:xfl" tintMultiplier=".25" tintColor="#123456"
brightness="-.4" alphaMultiplier=".5" redMultiplier="Infinity" greenMultiplier="-2"
blueMultiplier=".5" redOffset="255" greenOffset="-255" blueOffset="NaN" alphaOffset=".25"/>"##,
);
let effect = NativeColorEffect::new(&element);
assert_eq!(effect.tint_multiplier(), Some(0.25));
assert_eq!(effect.tint_color(), Color { rgb: 0x123456, alpha: 1.0 });
assert_eq!(effect.brightness(), Some(-0.4));
assert_eq!(effect.multipliers(), [1.0, -2.0, 0.5, 0.5]);
assert_eq!(effect.offsets(), [255.0, -255.0, 0.0, 0.25]);
}
}
+119
View File
@@ -0,0 +1,119 @@
//! Borrowed views of native XFL shape-hint records.
//!
//! These parse the native point encodings and hint ordering. Snapping points onto consumer
//! geometry and pairing start/end keyframes remain application responsibilities.
use crate::shape::{XflElement, integer, number};
/// A native XFL point in document coordinates.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Point {
pub x: f32,
pub y: f32,
}
/// The optional `<shapeHints>` wrapper on a keyframe.
#[derive(Clone, Copy, Debug)]
pub struct ShapeHints<'a, E: XflElement> {
wrapper: Option<&'a E>,
}
impl<'a, E: XflElement> ShapeHints<'a, E> {
pub fn new(keyframe: &'a E) -> Self {
Self { wrapper: keyframe.child("shapeHints") }
}
pub fn is_empty(&self) -> bool {
self.wrapper.is_none_or(|wrapper| wrapper.elements().next().is_none())
}
/// Records in source order; unrecognized child names are retained because the legacy reader
/// interpreted their point and ordering attributes too.
pub fn iter(&self) -> impl Iterator<Item = ShapeHint<'a, E>> + 'a {
let wrapper = self.wrapper;
wrapper.into_iter().flat_map(XflElement::elements).map(|element| ShapeHint { element })
}
}
/// One native shape-hint record.
#[derive(Clone, Copy, Debug)]
pub struct ShapeHint<'a, E: XflElement> {
element: &'a E,
}
impl<'a, E: XflElement> ShapeHint<'a, E> {
/// Native numeric order, or its single-letter `name` fallback (`a` is zero).
pub fn index(&self) -> Option<usize> {
integer(self.element, "index").map(|value| value as usize).or_else(|| self.element.attr("name").and_then(letter_index))
}
/// The keyframe's point, in either joined or separate native coordinate attributes.
pub fn start_point(&self) -> Option<Point> {
read_point(self.element, "startPoint", "startX", "startY").or_else(|| read_point(self.element, "point", "x", "y"))
}
/// The next keyframe's point, when the record contains one.
pub fn end_point(&self) -> Option<Point> {
read_point(self.element, "endPoint", "endX", "endY")
}
}
fn letter_index(name: &str) -> Option<usize> {
let [byte] = name.as_bytes() else { return None };
match byte.to_ascii_lowercase() {
b'a'..=b'z' => Some((byte.to_ascii_lowercase() - b'a') as usize),
_ => None,
}
}
fn read_point(element: &impl XflElement, joined: &str, x: &str, y: &str) -> Option<Point> {
let point = match element.attr(joined) {
Some(value) => {
let mut values = value.split([',', ' ']).filter(|part| !part.trim().is_empty()).map(|part| part.trim().parse::<f32>().ok());
Point { x: values.next()??, y: values.next()?? }
}
None => Point { x: number(element, x)?, y: number(element, y)? },
};
(point.x.is_finite() && point.y.is_finite()).then_some(point)
}
#[cfg(test)]
mod tests {
use super::*;
fn parse(source: &[u8]) -> craft_xml::El {
craft_xml::parse(source).expect("well-formed test XML")
}
#[test]
fn reads_joined_and_split_native_points_in_source_order() {
let keyframe = parse(br#"<DOMFrame><shapeHints><Hint name="B" startPoint="12.5, -2" endX="30" endY="40"/><Hint index="0" startX="1" startY="2" endPoint="3 4"/></shapeHints></DOMFrame>"#);
let hints = ShapeHints::new(&keyframe);
assert!(!hints.is_empty());
let records: Vec<_> = hints.iter().collect();
assert_eq!(records[0].index(), Some(1));
assert_eq!(records[0].start_point(), Some(Point { x: 12.5, y: -2.0 }));
assert_eq!(records[0].end_point(), Some(Point { x: 30.0, y: 40.0 }));
assert_eq!(records[1].index(), Some(0));
assert_eq!(records[1].start_point(), Some(Point { x: 1.0, y: 2.0 }));
assert_eq!(records[1].end_point(), Some(Point { x: 3.0, y: 4.0 }));
}
#[test]
fn malformed_joined_values_fall_back_to_the_other_native_point_field() {
let keyframe = parse(br#"<DOMFrame><shapeHints><Any name="z" startPoint="bad" x="7" y="8" endPoint="nan, 2"/></shapeHints></DOMFrame>"#);
let hint = ShapeHints::new(&keyframe).iter().next().expect("hint record");
assert_eq!(hint.index(), Some(25));
assert_eq!(hint.start_point(), Some(Point { x: 7.0, y: 8.0 }));
assert_eq!(hint.end_point(), None);
}
#[test]
fn missing_lists_and_non_letter_names_remain_optional() {
let keyframe = parse(b"<DOMFrame/>");
assert!(ShapeHints::new(&keyframe).is_empty());
let keyframe = parse(br#"<DOMFrame><shapeHints><Hint name="AB" startX="1" startY="2"/></shapeHints></DOMFrame>"#);
let hint = ShapeHints::new(&keyframe).iter().next().expect("hint record");
assert_eq!(hint.index(), None);
}
}
+136
View File
@@ -0,0 +1,136 @@
//! Borrowed views of native XFL frame-sound and envelope records.
//!
//! These parse the frame fields, effect-name spelling, and envelope wire values. Resolving sound
//! library items and mapping effects/envelopes into a consumer's audio model remain application
//! responsibilities.
use crate::shape::{XflElement, integer, number};
/// Native frame-sound synchronization mode. Unknown values use Animate's event behavior.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum SyncMode {
Start,
Stop,
Stream,
#[default]
Event,
}
/// A native sound instruction stored on a `DOMFrame`.
#[derive(Clone, Copy, Debug)]
pub struct FrameSound<'a, E: XflElement> {
frame: &'a E,
}
impl<'a, E: XflElement> FrameSound<'a, E> {
pub fn new(frame: &'a E) -> Self {
Self { frame }
}
pub fn name(&self) -> Option<&'a str> {
self.frame.attr("soundName")
}
pub fn sync_mode(&self) -> SyncMode {
match self.frame.attr("soundSync").map(str::trim) {
Some("start") => SyncMode::Start,
Some("stop") => SyncMode::Stop,
Some("stream") => SyncMode::Stream,
_ => SyncMode::Event,
}
}
pub fn looping(&self) -> bool {
self.frame.attr("soundLoopMode").map(str::trim) == Some("loop")
}
pub fn repeat(&self) -> Option<u32> {
integer(self.frame, "soundLoop")
}
pub fn has_in_point(&self) -> bool {
self.frame.attr("inPoint44").is_some()
}
pub fn has_out_point(&self) -> bool {
self.frame.attr("outPoint44").is_some()
}
/// The effect menu label normalized to lowercase ASCII letters and digits, matching the
/// legacy reader's recognition of labels such as `Fade Left to Right`.
pub fn effect_key(&self) -> String {
self.frame
.attr("soundEffect")
.unwrap_or("")
.chars()
.filter(char::is_ascii_alphanumeric)
.map(|character| character.to_ascii_lowercase())
.collect()
}
pub fn envelope_points(&self) -> impl Iterator<Item = EnvelopePoint<'a, E>> + 'a {
self.frame
.child("SoundEnvelope")
.into_iter()
.flat_map(|envelope| envelope.elements_named("SoundEnvelopePoint"))
.map(|element| EnvelopePoint { element })
}
}
/// One native `SoundEnvelopePoint` record. Levels are raw SWF/XFL integer units.
#[derive(Clone, Copy, Debug)]
pub struct EnvelopePoint<'a, E: XflElement> {
element: &'a E,
}
impl<E: XflElement> EnvelopePoint<'_, E> {
pub fn mark44(&self) -> Option<f32> {
number(self.element, "mark44")
}
pub fn level0(&self) -> Option<f32> {
number(self.element, "level0")
}
pub fn level1(&self) -> Option<f32> {
number(self.element, "level1")
}
}
#[cfg(test)]
mod tests {
use super::*;
fn parse(source: &[u8]) -> craft_xml::El {
craft_xml::parse(source).expect("well-formed test XML")
}
#[test]
fn reads_frame_sound_modes_repeat_trim_and_effect_name() {
let frame = parse(br#"<x:DOMFrame xmlns:x="urn:xfl" soundName="Music" soundSync=" stream " soundLoopMode="loop" soundLoop="12" inPoint44="2" outPoint44="100" soundEffect="Fade Left to Right!"/>"#);
let sound = FrameSound::new(&frame);
assert_eq!(sound.name(), Some("Music"));
assert_eq!(sound.sync_mode(), SyncMode::Stream);
assert!(sound.looping());
assert_eq!(sound.repeat(), Some(12));
assert!(sound.has_in_point());
assert!(sound.has_out_point());
assert_eq!(sound.effect_key(), "fadelefttoright");
}
#[test]
fn defaults_and_envelope_records_keep_native_optional_values() {
let frame = parse(br#"<DOMFrame soundSync="unknown" soundLoopMode="repeat" soundEffect="CuStOm"><SoundEnvelope><SoundEnvelopePoint mark44="44100" level0="32768"/><Other/><SoundEnvelopePoint mark44="bad" level0="0" level1="nan"/></SoundEnvelope></DOMFrame>"#);
let sound = FrameSound::new(&frame);
assert_eq!(sound.sync_mode(), SyncMode::Event);
assert!(!sound.looping());
assert_eq!(sound.repeat(), None);
assert!(!sound.has_in_point());
assert!(!sound.has_out_point());
assert_eq!(sound.effect_key(), "custom");
let points: Vec<_> = sound.envelope_points().collect();
assert_eq!(points.len(), 2);
assert_eq!((points[0].mark44(), points[0].level0(), points[0].level1()), (Some(44100.0), Some(32768.0), None));
assert_eq!((points[1].mark44(), points[1].level0(), points[1].level1()), (None, Some(0.0), None));
}
}
+273
View File
@@ -0,0 +1,273 @@
//! Borrowed views of native XFL text-field, run and style records.
//!
//! The views expose format attributes and direct text only. Text layout, font selection,
//! unsupported-style reporting and conversion into an application's text model stay with the
//! consumer.
use crate::shape::{self, Color, XflElement};
use std::borrow::Cow;
/// A native static, dynamic or input text field.
#[derive(Clone, Copy, Debug)]
pub struct TextField<'a, E: XflElement> {
element: &'a E,
kind: TextFieldKind,
}
/// Kind of native XFL text field.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum TextFieldKind {
Static,
Dynamic,
Input,
}
impl<'a, E: XflElement> TextField<'a, E> {
/// Views a recognized XFL text field.
pub fn new(element: &'a E) -> Option<Self> {
let kind = match element.local_name() {
"DOMStaticText" => TextFieldKind::Static,
"DOMDynamicText" => TextFieldKind::Dynamic,
"DOMInputText" => TextFieldKind::Input,
_ => return None,
};
Some(Self { element, kind })
}
pub fn kind(&self) -> TextFieldKind {
self.kind
}
/// Native field orientation (`horizontal`, `vertical`, etc.).
pub fn orientation(&self) -> Option<&'a str> {
self.element.attr("orientation")
}
/// Native `autoExpand` flag, if present and parseable.
pub fn auto_expand(&self) -> Option<bool> {
boolean(self.element, "autoExpand")
}
pub fn width(&self) -> Option<f32> {
number(self.element, "width")
}
pub fn height(&self) -> Option<f32> {
number(self.element, "height")
}
pub fn left(&self) -> Option<f32> {
number(self.element, "left")
}
pub fn top(&self) -> Option<f32> {
number(self.element, "top")
}
pub fn line_type(&self) -> Option<&'a str> {
self.element.attr("lineType")
}
pub fn border(&self) -> Option<bool> {
boolean(self.element, "border")
}
pub fn is_selectable(&self) -> Option<bool> {
boolean(self.element, "isSelectable")
}
pub fn render_as_html(&self) -> Option<bool> {
boolean(self.element, "renderAsHTML")
}
pub fn max_characters(&self) -> Option<f32> {
number(self.element, "maxCharacters")
}
pub fn variable_name(&self) -> Option<&'a str> {
self.element.attr("variableName")
}
/// Whether the native field has any filter records, which consumers may need to report as
/// unsupported while keeping the field.
pub fn has_filters(&self) -> bool {
self.element.child("filters").is_some_and(|wrapper| wrapper.elements().next().is_some())
}
/// Text runs in document order. Non-`DOMTextRun` children are ignored.
pub fn runs(&self) -> impl Iterator<Item = TextRun<'a, E>> + 'a {
self.element.child("textRuns").into_iter().flat_map(|wrapper| wrapper.elements_named("DOMTextRun")).map(|element| TextRun { element })
}
/// Number of element records inside the native `<textRuns>` wrapper, including unknown
/// records. This lets consumers retain their prior malformed-record and resource-limit
/// policy while parsing recognized runs through [`Self::runs`].
pub fn run_entries_count(&self) -> usize {
self.element.child("textRuns").map_or(0, |wrapper| wrapper.elements().count())
}
}
/// One native text run and its optional style record.
#[derive(Clone, Copy, Debug)]
pub struct TextRun<'a, E: XflElement> {
element: &'a E,
}
impl<'a, E: XflElement> TextRun<'a, E> {
/// Direct character data of this run's `<characters>` element.
pub fn characters(&self) -> Cow<'a, str> {
self.element.child("characters").map_or(Cow::Borrowed(""), XflElement::text_content)
}
/// The native text attributes nested under `<textAttrs><DOMTextAttrs .../>`.
pub fn style(&self) -> Option<TextStyle<'a, E>> {
self.element.child("textAttrs").and_then(|wrapper| wrapper.child("DOMTextAttrs")).map(|element| TextStyle { element })
}
}
/// Native `DOMTextAttrs` fields. Values are parsed from the wire schema but not laid out or
/// normalized for any application.
#[derive(Clone, Copy, Debug)]
pub struct TextStyle<'a, E: XflElement> {
element: &'a E,
}
impl<'a, E: XflElement> TextStyle<'a, E> {
pub fn face(&self) -> Option<&'a str> {
self.element.attr("face")
}
pub fn size(&self) -> Option<f32> {
number(self.element, "size")
}
pub fn bold(&self) -> Option<bool> {
boolean(self.element, "bold")
}
pub fn italic(&self) -> Option<bool> {
boolean(self.element, "italic")
}
pub fn fill_color(&self) -> Option<&'a str> {
self.element.attr("fillColor")
}
/// Parsed native fill color and alpha, with XFL's black/full-opacity defaults.
pub fn color(&self) -> Color {
shape::color(self.element, "fillColor", "alpha")
}
pub fn alpha(&self) -> Option<f32> {
number(self.element, "alpha")
}
pub fn alignment(&self) -> Option<&'a str> {
self.element.attr("alignment")
}
pub fn indent(&self) -> Option<f32> {
number(self.element, "indent")
}
pub fn character_position(&self) -> Option<&'a str> {
self.element.attr("characterPosition")
}
pub fn letter_spacing(&self) -> Option<f32> {
number(self.element, "letterSpacing")
}
pub fn line_spacing(&self) -> Option<f32> {
number(self.element, "lineSpacing")
}
pub fn left_margin(&self) -> Option<f32> {
number(self.element, "leftMargin")
}
pub fn right_margin(&self) -> Option<f32> {
number(self.element, "rightMargin")
}
}
fn number(element: &impl XflElement, name: &str) -> Option<f32> {
element.attr(name)?.trim().parse::<f32>().ok().filter(|value| value.is_finite())
}
fn boolean(element: &impl XflElement, name: &str) -> Option<bool> {
element.attr(name).map(|value| value.trim() == "true")
}
#[cfg(test)]
mod tests {
use super::*;
fn parse(source: &[u8]) -> craft_xml::El {
craft_xml::parse(source).expect("well-formed test XML")
}
#[test]
fn reads_prefixed_text_fields_runs_and_native_style_values() {
let root = parse(
br##"<x:DOMDynamicText xmlns:x="urn:xfl" orientation="horizontal" autoExpand="true" width="320" height="40" left="12" top="3" lineType="multiline" border="true" isSelectable="false" renderAsHTML="false" maxCharacters="45" variableName="message"><x:filters><x:BlurFilter/></x:filters><x:textRuns><x:DOMTextRun><x:characters>one &amp; two</x:characters><x:textAttrs><x:DOMTextAttrs face="Arial" size="24" bold="true" italic="false" fillColor="#123456" alpha="0.5" alignment="center" indent="3" characterPosition="superscript" letterSpacing="1.25" lineSpacing="2" leftMargin="4" rightMargin="5"/></x:textAttrs></x:DOMTextRun><x:Other/><x:DOMTextRun><x:characters><![CDATA[third]]></x:characters></x:DOMTextRun></x:textRuns></x:DOMDynamicText>"##,
);
let field = TextField::new(&root).expect("dynamic text field");
assert_eq!(field.kind(), TextFieldKind::Dynamic);
assert_eq!(field.orientation(), Some("horizontal"));
assert_eq!(field.auto_expand(), Some(true));
assert_eq!((field.width(), field.height(), field.left(), field.top()), (Some(320.0), Some(40.0), Some(12.0), Some(3.0)));
assert_eq!(field.line_type(), Some("multiline"));
assert_eq!(field.border(), Some(true));
assert_eq!(field.is_selectable(), Some(false));
assert_eq!(field.render_as_html(), Some(false));
assert_eq!(field.max_characters(), Some(45.0));
assert_eq!(field.variable_name(), Some("message"));
assert!(field.has_filters());
let mut runs = field.runs();
let first = runs.next().expect("first run");
assert_eq!(first.characters(), "one & two");
let style = first.style().expect("text style");
assert_eq!(style.face(), Some("Arial"));
assert_eq!(style.size(), Some(24.0));
assert_eq!(style.bold(), Some(true));
assert_eq!(style.italic(), Some(false));
assert_eq!(style.fill_color(), Some("#123456"));
assert_eq!(style.alpha(), Some(0.5));
assert_eq!(style.alignment(), Some("center"));
assert_eq!(style.indent(), Some(3.0));
assert_eq!(style.character_position(), Some("superscript"));
assert_eq!(style.letter_spacing(), Some(1.25));
assert_eq!(style.line_spacing(), Some(2.0));
assert_eq!(style.left_margin(), Some(4.0));
assert_eq!(style.right_margin(), Some(5.0));
let second = runs.next().expect("second run");
assert_eq!(second.characters(), "third");
assert!(second.style().is_none());
assert!(runs.next().is_none());
}
#[test]
fn recognizes_all_native_field_kinds_and_ignores_non_text_elements() {
for (name, expected) in
[("DOMStaticText", TextFieldKind::Static), ("DOMDynamicText", TextFieldKind::Dynamic), ("DOMInputText", TextFieldKind::Input)]
{
let xml = format!("<{name}/>");
let root = craft_xml::parse(xml.as_bytes()).expect("test XML");
assert_eq!(TextField::new(&root).map(|field| field.kind()), Some(expected));
}
let root = parse(b"<DOMShape/>");
assert!(TextField::new(&root).is_none());
}
#[test]
fn malformed_and_non_finite_numbers_remain_absent() {
let root = parse(b"<DOMStaticText><textRuns><DOMTextRun><textAttrs><DOMTextAttrs size='NaN' letterSpacing='bad'/></textAttrs></DOMTextRun></textRuns></DOMStaticText>");
let field = TextField::new(&root).expect("text field");
let style = field.runs().next().and_then(|run| run.style()).expect("style");
assert_eq!(style.size(), None);
assert_eq!(style.letter_spacing(), None);
}
}
+117
View File
@@ -0,0 +1,117 @@
//! Borrowed views of native XFL frame-tween and custom-ease records.
//!
//! This parses the native attributes and point lists. Interpreting easing curves and applying
//! them to an application's timeline model remain consumer responsibilities.
use crate::shape::{XflElement, boolean, number};
/// Native motion/shape tween settings on a `DOMFrame`.
#[derive(Clone, Copy, Debug)]
pub struct FrameTween<'a, E: XflElement> {
frame: &'a E,
}
impl<'a, E: XflElement> FrameTween<'a, E> {
pub fn new(frame: &'a E) -> Self {
Self { frame }
}
/// The classic slider value, when present and finite.
pub fn acceleration(&self) -> Option<f32> {
number(self.frame, "acceleration")
}
pub fn has_custom_ease(&self) -> Option<bool> {
boolean(self.frame, "hasCustomEase")
}
/// Animate's default is a single curve unless this is explicitly false.
pub fn use_single_ease_curve(&self) -> Option<bool> {
boolean(self.frame, "useSingleEaseCurve")
}
pub fn orient_to_path(&self) -> Option<bool> {
boolean(self.frame, "motionTweenOrientToPath")
}
/// Raw native rotation setting; interpretation stays in the consuming application.
pub fn rotate(&self) -> Option<&'a str> {
self.frame.attr("motionTweenRotate")
}
/// Custom ease curves in source order; unknown tween entries are skipped.
pub fn custom_eases(&self) -> impl Iterator<Item = CustomEase<'a, E>> + 'a {
self.frame.child("tweens").into_iter().flat_map(|wrapper| wrapper.elements_named("CustomEase")).map(|element| CustomEase { element })
}
}
/// A native custom-ease curve.
#[derive(Clone, Copy, Debug)]
pub struct CustomEase<'a, E: XflElement> {
element: &'a E,
}
impl<'a, E: XflElement> CustomEase<'a, E> {
/// The property-group name, defaulting to `all` in XFL.
pub fn target(&self) -> &'a str {
self.element.attr("target").map(str::trim).unwrap_or("all")
}
pub fn points(&self) -> impl Iterator<Item = EasePoint<'a, E>> + 'a {
self.element.elements_named("Point").map(|element| EasePoint { element })
}
}
/// One native custom-ease point.
#[derive(Clone, Copy, Debug)]
pub struct EasePoint<'a, E: XflElement> {
element: &'a E,
}
impl<E: XflElement> EasePoint<'_, E> {
pub fn x(&self) -> Option<f32> {
number(self.element, "x")
}
pub fn y(&self) -> Option<f32> {
number(self.element, "y")
}
}
#[cfg(test)]
mod tests {
use super::*;
fn parse(source: &[u8]) -> craft_xml::El {
craft_xml::parse(source).expect("well-formed test XML")
}
#[test]
fn reads_frame_tween_settings_and_custom_eases_in_order() {
let frame = parse(
br#"<x:DOMFrame xmlns:x="urn:xfl" acceleration="-45" hasCustomEase="true" useSingleEaseCurve="false" motionTweenOrientToPath="true" motionTweenRotate="clockwise"><x:tweens><x:Other/><x:CustomEase target="position"><x:Point x="0" y="0"/><x:Point x="0.5" y="0.25"/></x:CustomEase><x:CustomEase><x:Point x="1" y="1"/></x:CustomEase></x:tweens></x:DOMFrame>"#,
);
let tween = FrameTween::new(&frame);
assert_eq!(tween.acceleration(), Some(-45.0));
assert_eq!(tween.has_custom_ease(), Some(true));
assert_eq!(tween.use_single_ease_curve(), Some(false));
assert_eq!(tween.orient_to_path(), Some(true));
assert_eq!(tween.rotate(), Some("clockwise"));
let curves: Vec<_> = tween.custom_eases().collect();
assert_eq!(curves.len(), 2);
assert_eq!(curves[0].target(), "position");
assert_eq!(curves[0].points().map(|point| (point.x(), point.y())).collect::<Vec<_>>(), [(Some(0.0), Some(0.0)), (Some(0.5), Some(0.25))]);
assert_eq!(curves[1].target(), "all");
assert_eq!(curves[1].points().next().map(|point| (point.x(), point.y())), Some((Some(1.0), Some(1.0))));
}
#[test]
fn absent_and_malformed_values_remain_optional() {
let frame = parse(br#"<DOMFrame acceleration="NaN" hasCustomEase="bad" useSingleEaseCurve="false"/>"#);
let tween = FrameTween::new(&frame);
assert_eq!(tween.acceleration(), None);
assert_eq!(tween.has_custom_ease(), Some(false));
assert_eq!(tween.use_single_ease_curve(), Some(false));
assert!(tween.custom_eases().next().is_none());
}
}
+162
View File
@@ -0,0 +1,162 @@
#!/usr/bin/env python3
"""Check the Workspace workflow's pull-request-only cancellation policy.
The policy is the top-level `concurrency` block of .github/workflows/workspace.yml. This script
reads those fields, requires the documented expression shapes, and checks which group each event
would be given. The rest of the workflow is left to diff review, not frozen here.
Use --workflow PATH to read a baseline or a deliberately wrong variant without touching this
checkout.
"""
import argparse
import re
import sys
import unittest
from pathlib import Path
try: # cross-check only; the reader below is what a plain `python3` run needs
import yaml
except ImportError:
yaml = None
DEFAULT_WORKFLOW = Path(__file__).resolve().parent.parent / ".github" / "workflows" / "workspace.yml"
PULL_REQUEST, PUSH, DISPATCH = "pull_request", "push", "workflow_dispatch"
# The two shapes this policy may use. GitHub reads `a && b || c` as "b, or c when a is false"; these
# patterns are the whole of that expression language this script understands, so a reworded policy
# has to be reworded here too, on purpose. github.ref, never github.head_ref or github.sha.
GROUP_SHAPE = re.compile(
r"(?P<prefix>[A-Za-z0-9._-]+)-\$\{\{\s*github\.event_name\s*==\s*'(?P<event>[a-z_]+)'\s*"
r"&&\s*github\.ref\s*\|\|\s*github\.run_id\s*\}\}"
)
CANCEL_SHAPE = re.compile(r"\$\{\{\s*github\.event_name\s*==\s*'(?P<event>[a-z_]+)'\s*\}\}")
def read_concurrency(text):
"""The top-level `concurrency` mapping. A deliberately limited reader: two-space fields."""
lines = text.splitlines()
starts = [index for index, line in enumerate(lines) if line.startswith("concurrency:")]
if len(starts) != 1:
raise AssertionError(f"expected one top-level concurrency block, found {len(starts)}")
header = lines[starts[0]].partition("#")[0].strip()
if header != "concurrency:":
# `concurrency: |` and `concurrency: some-string` are scalars, not a mapping. Reading the
# lines under one as fields would let a workflow that is not this policy pass every check.
raise AssertionError(f"concurrency must be a mapping, not a scalar: {lines[starts[0]]!r}")
fields = {}
for line in lines[starts[0] + 1:]:
stripped = line.strip()
if stripped and not line.startswith(" "):
break # the next top-level key
if not stripped or stripped.startswith("#"):
continue
if not line.startswith(" ") or line.startswith(" "):
raise AssertionError(f"unexpected indentation under concurrency: {line!r}")
key, separator, value = stripped.partition(":")
if not separator:
raise AssertionError(f"not a field of concurrency: {line!r}")
fields[key.strip()] = value.strip()
return fields
def pull_request(number, run_id):
# github.ref for a pull request is refs/pull/<n>/merge: the PR number, never the branch name.
return (PULL_REQUEST, f"refs/pull/{number}/merge", run_id)
def other_run(event_name, ref, run_id):
return (event_name, ref, run_id)
MAIN, TAG = ("refs/heads/main", "refs/tags/v1.2.3")
SAME_GROUP = [
("a newer commit on the same pull request", pull_request(1316, 43599251877), pull_request(1316, 43600899878)),
("a rerun of one main run", other_run(PUSH, MAIN, 7001), other_run(PUSH, MAIN, 7001)),
]
DIFFERENT_GROUPS = [
("two pull requests", pull_request(1316, 1), pull_request(1317, 1)),
("the same branch name in two forks", pull_request(1316, 2), pull_request(1400, 2)),
("two pushes to main", other_run(PUSH, MAIN, 3), other_run(PUSH, MAIN, 4)),
("two release tag runs", other_run(PUSH, TAG, 5), other_run(PUSH, TAG, 6)),
("two manual runs", other_run(DISPATCH, MAIN, 7), other_run(DISPATCH, MAIN, 8)),
("a main push and a pull request", other_run(PUSH, MAIN, 9), pull_request(1316, 9)),
("a release tag and a pull request", other_run(PUSH, TAG, 10), pull_request(1316, 10)),
]
CANCELLATION = [
("a push supersedes the previous run of its pull request", pull_request(1316, 11), True),
("a push to main is never cancelled", other_run(PUSH, MAIN, 12), False),
("a release tag run is never cancelled", other_run(PUSH, TAG, 13), False),
("a manual run is never cancelled", other_run(DISPATCH, MAIN, 14), False),
]
class ConcurrencyTests(unittest.TestCase):
workflow = DEFAULT_WORKFLOW
@classmethod
def setUpClass(cls):
cls.text = cls.workflow.read_text()
cls.fields = read_concurrency(cls.text)
cls.group_shape = GROUP_SHAPE.fullmatch(cls.fields.get("group", ""))
cls.cancel_shape = CANCEL_SHAPE.fullmatch(cls.fields.get("cancel-in-progress", ""))
if not cls.group_shape or not cls.cancel_shape:
raise AssertionError(f"the concurrency fields are not the documented shapes: {cls.fields}")
def cancels(self, event_name):
return event_name == self.cancel_shape["event"]
def group(self, event_name, ref, run_id):
return self.group_shape["prefix"] + "-" + (ref if event_name == self.group_shape["event"] else str(run_id))
def test_policy_is_the_two_documented_fields(self):
self.assertEqual(sorted(self.fields), ["cancel-in-progress", "group"])
self.assertEqual(self.group_shape["prefix"], "workspace")
self.assertEqual(self.group_shape["event"], PULL_REQUEST)
self.assertEqual(self.cancel_shape["event"], PULL_REQUEST)
def test_a_scalar_concurrency_value_is_not_read_as_a_mapping(self):
cases = {
"a block scalar": "concurrency: |\n group: workspace-x\n cancel-in-progress: true\n",
"a plain scalar": "concurrency: workspace-x\njobs: {}\n",
}
for description, text in cases.items():
with self.subTest(description):
with self.assertRaises(AssertionError):
read_concurrency(text)
def test_one_group_per_pull_request_and_run(self):
for description, first, second in SAME_GROUP:
with self.subTest(description):
self.assertEqual(self.group(*first), self.group(*second), description)
for description, first, second in DIFFERENT_GROUPS:
with self.subTest(description):
self.assertNotEqual(self.group(*first), self.group(*second), description)
def test_only_a_pull_request_cancels(self):
for description, context, expected in CANCELLATION:
with self.subTest(description):
self.assertEqual(self.cancels(context[0]), expected, description)
def test_pyyaml_reads_the_same_fields(self):
if yaml is None:
self.skipTest("PyYAML is not installed; the limited reader above ran alone")
document = yaml.load(self.text, Loader=yaml.BaseLoader) # BaseLoader keeps `on:` a string
self.assertEqual(document["concurrency"], self.fields)
def test_no_per_job_concurrency_is_added(self):
indented = [line for line in self.text.splitlines()
if line.strip().startswith("concurrency:") and not line.startswith("concurrency:")]
self.assertEqual(indented, [], "the policy belongs to the workflow, not to one job")
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--workflow", type=Path, default=DEFAULT_WORKFLOW, help="workflow to read (default: %(default)s)")
arguments, remaining = parser.parse_known_args()
ConcurrencyTests.workflow = arguments.workflow
unittest.main(argv=[sys.argv[0], *remaining])
if __name__ == "__main__":
main()
+5 -1
View File
@@ -7,7 +7,11 @@ sonar.organization=storytold
# *_tests.rs), which includes the browser fault-injection fixtures in crates/platform/tests.
sonar.sources=.
sonar.tests=.
sonar.test.inclusions=**/tests/**,**/tests.rs,**/*_tests.rs
# scripts/test-workspace-concurrency.py is a unit-test module for the Workspace workflow's
# cancellation policy: its unittest cases read a workflow and assert on it, so it is test
# code, analysed as a test and still linted, indexed and reported, not counted as production
# code that needs coverage. Its name is listed exactly, so no other script is classified.
sonar.test.inclusions=**/tests/**,**/tests.rs,**/*_tests.rs,scripts/test-workspace-concurrency.py
# Codec constant tables are transcribed from the format specifications; their repeated rows are
# data, not copy-pasted logic.