mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-16 23:42:27 +00:00
Merge remote-tracking branch 'origin/main' into HEAD
This commit is contained in:
commit
2369c13f28
92 files changed
+11204
-276
No files matched your search
@@ -158,7 +158,7 @@ dependencies:
|
||||
repo_id: 192625525
|
||||
'dtolnay/rust-toolchain@stable':
|
||||
ref: 'stable'
|
||||
commit: 'sha1-b725eda525362e7cf34af9a7243cac1e3f01a7ec'
|
||||
commit: 'sha1-39008827e6b4319a349e9a42dfc90579dd25f1ca'
|
||||
owner_id: 1940490
|
||||
repo_id: 260749683
|
||||
'step-security/dtolnay-rust-toolchain@v1.0.0':
|
||||
|
||||
@@ -10,6 +10,14 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
|
||||
# run (push to main, a release tag, a manual run) gets a group of its own: it is neither
|
||||
# cancelled nor replaced while queued, so a main run always finishes and saves the dependency
|
||||
# cache, and a release run is never cancelled.
|
||||
group: workspace-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
native:
|
||||
name: Workspace
|
||||
@@ -39,6 +47,7 @@ jobs:
|
||||
printf 'CRAFT_PSD_CORPUS=%s\n' "$GITHUB_WORKSPACE/$corpus_fetcher/corpus" >> "$GITHUB_ENV"
|
||||
printf 'CRAFT_PDF_TEST_FONT_DIR=%s\n' "$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV"
|
||||
command: |
|
||||
python3 scripts/test-workspace-concurrency.py
|
||||
cargo fmt --check
|
||||
cargo clippy --workspace --all-targets --locked -- -D warnings
|
||||
cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
|
||||
|
||||
Generated
+17
-1
@@ -1352,6 +1352,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"craft-media",
|
||||
"craft-media-testkit",
|
||||
"craft-xml",
|
||||
"log",
|
||||
"proptest",
|
||||
"rayon",
|
||||
@@ -1362,6 +1363,10 @@ dependencies = [
|
||||
"vorbis_rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-ber"
|
||||
version = "0.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "craft-captions"
|
||||
version = "0.1.0"
|
||||
@@ -1875,6 +1880,7 @@ dependencies = [
|
||||
"ravif",
|
||||
"rayon",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.21",
|
||||
"tiff",
|
||||
"zune-core",
|
||||
@@ -2380,6 +2386,16 @@ dependencies = [
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-proto"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"log",
|
||||
"proptest",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-ps"
|
||||
version = "0.1.0"
|
||||
@@ -11150,7 +11166,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "winprint"
|
||||
version = "0.2.1"
|
||||
source = "git+https://github.com/storytold/winprint.rs?rev=72951d7455ebcab569fbd9386c513d3d9f53fb02#72951d7455ebcab569fbd9386c513d3d9f53fb02"
|
||||
source = "git+https://github.com/storytold/winprint.rs?rev=8ae097ef941b432b0e7671b1ec3a506800e65e57#8ae097ef941b432b0e7671b1ec3a506800e65e57"
|
||||
dependencies = [
|
||||
"flate2",
|
||||
"fmt-derive",
|
||||
|
||||
@@ -11,6 +11,8 @@ rust-version = "1.95"
|
||||
repository = "https://github.com/storytold/craft-libs"
|
||||
|
||||
[workspace.dependencies]
|
||||
craft-ber = { path = "crates/ber" }
|
||||
craft-proto = { path = "crates/proto" }
|
||||
acadrust = "=0.6.6"
|
||||
log = "0.4"
|
||||
# internal: add each crate here as `craft-<name> = { path = "crates/<name>" }`
|
||||
|
||||
@@ -19,7 +19,7 @@ workspace = true
|
||||
# µ-law and free-format MP3 through symphonia.
|
||||
decode = ["decode-native", "symphonia"]
|
||||
# `decode` without symphonia: only the formats with a native decoder (others are Unsupported).
|
||||
decode-native = ["aac-decode", "alac-decode", "opus-decode", "ac3-decode", "mpa-decode", "mp3-decode", "craft-media/isobmff", "craft-media/matroska", "craft-media/ogg", "dep:log"]
|
||||
decode-native = ["aac-decode", "alac-decode", "opus-decode", "ac3-decode", "mpa-decode", "mp3-decode", "craft-media/isobmff", "craft-media/matroska", "craft-media/ogg", "dep:log", "dep:craft-xml"]
|
||||
# The symphonia fallback for the formats with no native decoder yet.
|
||||
symphonia = ["decode-native", "dep:symphonia"]
|
||||
# Every encoder: the lossless file writers and the AAC, ALAC, AC-3/E-AC-3, MP3 and Opus encoders.
|
||||
@@ -60,6 +60,7 @@ serde = ["dep:serde", "craft-media/serde"]
|
||||
|
||||
[dependencies]
|
||||
craft-media = { workspace = true }
|
||||
craft-xml = { workspace = true, optional = true }
|
||||
thiserror = { workspace = true }
|
||||
serde = { workspace = true, optional = true }
|
||||
log = { version = "0.4", optional = true }
|
||||
|
||||
@@ -761,3 +761,8 @@ The codec modules' public items keep their names, so apps can re-export them
|
||||
CAF.
|
||||
MP4 and Matroska files decode to presentation length. A packet that fails is concealed, not
|
||||
dropped. A failed source read is `AudioError::Io`.
|
||||
|
||||
WAV timestamps use `bext` when present, including a zero timestamp. Otherwise a bounded iXML
|
||||
`BWFXML/BEXT` time reference supplies the timestamp (two unsigned 32-bit words; an omitted high
|
||||
word is zero). Only the first iXML chunk is considered. Invalid or oversized optional XML is
|
||||
ignored without rejecting the audio. This works for byte-backed and streaming sources.
|
||||
@@ -115,6 +115,50 @@ fn parse_bext(c: &[u8]) -> Option<BwfInfo> {
|
||||
})
|
||||
}
|
||||
|
||||
/// Cap optional metadata independently of audio length, including on range-backed sources.
|
||||
const IXML_MAX: usize = 1 << 20;
|
||||
|
||||
/// iXML's BEXT mirror is a fallback only: a real `bext` always wins, including timestamp zero.
|
||||
/// Read the two unsigned 32-bit words, not arbitrary text that merely contains their tag names.
|
||||
fn ixml_time_reference(bytes: &[u8]) -> Option<u64> {
|
||||
let options = craft_xml::Options {
|
||||
doctype: craft_xml::Doctype::Reject,
|
||||
..craft_xml::Options::default().strict().with_limits(craft_xml::Limits {
|
||||
max_input: IXML_MAX,
|
||||
max_depth: 32,
|
||||
max_nodes: 8192,
|
||||
max_attrs: 64,
|
||||
max_attrs_total: 8192,
|
||||
..craft_xml::Limits::default()
|
||||
})
|
||||
};
|
||||
let root = options.parse(bytes).ok()?;
|
||||
if root.name != "BWFXML" {
|
||||
return None;
|
||||
}
|
||||
let mut bexts = root.children("BEXT");
|
||||
let bext = bexts.next()?;
|
||||
if bexts.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
let word = |name: &str| -> Option<Option<u32>> {
|
||||
let mut fields = bext.children(name);
|
||||
let Some(field) = fields.next() else { return Some(None) };
|
||||
if fields.next().is_some() || field.children.iter().any(|n| matches!(n, craft_xml::Node::El(_))) {
|
||||
return None;
|
||||
}
|
||||
let text = field.text();
|
||||
let text = text.trim();
|
||||
if text.is_empty() || !text.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return None;
|
||||
}
|
||||
Some(Some(text.parse().ok()?))
|
||||
};
|
||||
let low = word("BWF_TIME_REFERENCE_LOW")??;
|
||||
let high = word("BWF_TIME_REFERENCE_HIGH")?.unwrap_or(0);
|
||||
Some((u64::from(high) << 32) | u64::from(low))
|
||||
}
|
||||
|
||||
/// Refuse the WAVE file at `at`, which symphonia reads (A-law, ADPCM, ...), if an INFO entry
|
||||
/// claims more bytes than the file holds: symphonia allocates each entry's length. Chunks are
|
||||
/// walked where symphonia reads them: after the bytes a chunk's parser used, not its length.
|
||||
@@ -239,7 +283,7 @@ fn pcm_layout(fmt: &Fmt) -> Result<Option<(PcmKind, SampleFormat)>> {
|
||||
}
|
||||
|
||||
/// Parse a WAV file's chunks: its format, `bext` metadata and where its samples are. Only chunk
|
||||
/// headers and the small `fmt `, `bext` and `ds64` bodies are read.
|
||||
/// headers, small `fmt `, `bext` and `ds64` bodies, and at most one 1 MiB iXML document are read.
|
||||
pub(crate) fn parse(src: &Src<'_>) -> Result<WavParse> {
|
||||
if !is_wav(&src.bytes_upto(0, 12)) {
|
||||
return Err(AudioError::Malformed("not a RIFF/RF64 WAVE file".into()));
|
||||
@@ -249,6 +293,8 @@ pub(crate) fn parse(src: &Src<'_>) -> Result<WavParse> {
|
||||
let mut ds64_data: Option<u64> = None;
|
||||
let mut fmt: Option<Fmt> = None;
|
||||
let mut bwf: Option<BwfInfo> = None;
|
||||
let mut ixml_time = None;
|
||||
let mut ixml_seen = false;
|
||||
let mut data: Option<(u64, u64)> = None;
|
||||
|
||||
let mut pos: u64 = 12;
|
||||
@@ -288,7 +334,18 @@ pub(crate) fn parse(src: &Src<'_>) -> Result<WavParse> {
|
||||
let small = |n: u64| src.bytes(body, usize::try_from(declared.min(n)).unwrap_or(0)).unwrap_or_default();
|
||||
match &id {
|
||||
b"fmt " => fmt = Some(parse_fmt(&small(256))?),
|
||||
b"bext" => bwf = parse_bext(&small(4096)),
|
||||
b"bext" => {
|
||||
if let Some(info) = parse_bext(&small(4096)) {
|
||||
bwf = Some(info);
|
||||
}
|
||||
}
|
||||
b"iXML" if !ixml_seen => {
|
||||
// Inspect at most one bounded document, not 100,000 individually bounded trees.
|
||||
ixml_seen = true;
|
||||
if declared <= IXML_MAX as u64 {
|
||||
ixml_time = ixml_time_reference(&small(IXML_MAX as u64));
|
||||
}
|
||||
}
|
||||
b"ds64" => ds64_data = le_u64(&small(64), 8),
|
||||
_ => {}
|
||||
}
|
||||
@@ -297,6 +354,9 @@ pub(crate) fn parse(src: &Src<'_>) -> Result<WavParse> {
|
||||
pos = next;
|
||||
}
|
||||
|
||||
if bwf.is_none() {
|
||||
bwf = ixml_time.map(|time_reference| BwfInfo { time_reference, ..BwfInfo::default() });
|
||||
}
|
||||
let fmt = fmt.ok_or_else(|| AudioError::Malformed("WAV has no fmt chunk".into()))?;
|
||||
if fmt.channels == 0 {
|
||||
return Err(AudioError::Malformed("WAV declares zero channels".into()));
|
||||
|
||||
@@ -10,3 +10,4 @@ mod raw_pcm;
|
||||
mod roundtrip;
|
||||
mod stream;
|
||||
mod wav_empty_data;
|
||||
mod wav_ixml;
|
||||
@@ -0,0 +1,87 @@
|
||||
//! iXML timestamp parity with FilmCraft's streaming WAV reader, including hostile metadata.
|
||||
use std::io::Cursor;
|
||||
|
||||
use craft_audio::{DecodeOptions, Decoder};
|
||||
use craft_media::ReadSeekSource;
|
||||
|
||||
fn chunk(id: &[u8; 4], body: &[u8]) -> Vec<u8> {
|
||||
let mut out = id.to_vec();
|
||||
out.extend_from_slice(&(body.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(body);
|
||||
if !body.len().is_multiple_of(2) {
|
||||
out.push(0);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn wav(metadata: &[Vec<u8>]) -> Vec<u8> {
|
||||
let mut out = b"RIFF\0\0\0\0WAVE".to_vec();
|
||||
out.extend(chunk(b"fmt ", &[1, 0, 1, 0, 0x80, 0xbb, 0, 0, 0, 0x77, 1, 0, 2, 0, 16, 0]));
|
||||
for m in metadata {
|
||||
out.extend(m);
|
||||
}
|
||||
out.extend(chunk(b"data", &[0, 0, 1, 0]));
|
||||
let len = (out.len() - 8) as u32;
|
||||
out[4..8].copy_from_slice(&len.to_le_bytes());
|
||||
out
|
||||
}
|
||||
|
||||
fn xml(low: &str, high: Option<&str>) -> Vec<u8> {
|
||||
let high = high.map(|v| format!("<BWF_TIME_REFERENCE_HIGH>{v}</BWF_TIME_REFERENCE_HIGH>")).unwrap_or_default();
|
||||
chunk(b"iXML", format!("<BWFXML><BEXT><BWF_TIME_REFERENCE_LOW>{low}</BWF_TIME_REFERENCE_LOW>{high}</BEXT></BWFXML>").as_bytes())
|
||||
}
|
||||
|
||||
fn check(file: Vec<u8>, expected: Option<u64>) {
|
||||
let options = DecodeOptions::default();
|
||||
let mut bytes = Decoder::from_bytes(&file, Some("wav"), &options).unwrap();
|
||||
assert_eq!(bytes.info().bwf.as_ref().map(|b| b.time_reference), expected);
|
||||
let source = ReadSeekSource::new(Cursor::new(file.clone())).unwrap();
|
||||
let mut stream = Decoder::open(source, Some("wav"), &options).unwrap();
|
||||
assert_eq!(stream.info().bwf.as_ref().map(|b| b.time_reference), expected);
|
||||
assert_eq!(bytes.info().frames, 2);
|
||||
assert_eq!(stream.next_block().unwrap().unwrap().frames(), 2);
|
||||
assert_eq!(bytes.next_block().unwrap().unwrap().frames(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ixml_fallback_preserves_large_timestamps_and_optional_high_word() {
|
||||
check(wav(&[xml(" 5 ", Some("1"))]), Some((1 << 32) + 5));
|
||||
check(wav(&[xml("0", None)]), Some(0));
|
||||
check(wav(&[xml("4294967295", Some("4294967295"))]), Some(u64::MAX));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bext_wins_in_either_chunk_order_including_timestamp_zero() {
|
||||
let bext = chunk(b"bext", &[0; 602]);
|
||||
let ixml = xml("5", Some("1"));
|
||||
check(wav(&[bext.clone(), ixml.clone()]), Some(0));
|
||||
check(wav(&[ixml.clone(), bext.clone()]), Some(0));
|
||||
check(wav(&[bext, chunk(b"bext", &[0; 12]), ixml.clone()]), Some(0));
|
||||
check(wav(&[chunk(b"bext", &[0; 12]), ixml]), Some((1 << 32) + 5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_metadata_never_fabricates_a_timestamp_or_blocks_audio() {
|
||||
for low in ["4294967296", "-1", "+1", "", "1.5", "<x>5</x>"] {
|
||||
check(wav(&[xml(low, Some("0"))]), None);
|
||||
}
|
||||
for high in ["4294967296", "-1", "", "oops"] {
|
||||
check(wav(&[xml("5", Some(high))]), None);
|
||||
}
|
||||
for text in [
|
||||
"<BWFXML><BEXT><BWF_TIME_REFERENCE_LOW>5</BWF_TIME_REFERENCE_LOW>",
|
||||
"<BWFXML><!-- <BWF_TIME_REFERENCE_LOW>5</BWF_TIME_REFERENCE_LOW> --></BWFXML>",
|
||||
"<BWFXML><BEXT><BWF_TIME_REFERENCE_LOW>5</BWF_TIME_REFERENCE_LOW><BWF_TIME_REFERENCE_LOW>6</BWF_TIME_REFERENCE_LOW></BEXT></BWFXML>",
|
||||
"<!DOCTYPE BWFXML><BWFXML><BEXT><BWF_TIME_REFERENCE_LOW>5</BWF_TIME_REFERENCE_LOW></BEXT></BWFXML>",
|
||||
] {
|
||||
check(wav(&[chunk(b"iXML", text.as_bytes())]), None);
|
||||
}
|
||||
check(wav(&[chunk(b"iXML", &vec![b' '; (1 << 20) + 1])]), None);
|
||||
let deep = format!("<BWFXML>{}{}</BWFXML>", "<a>".repeat(40), "</a>".repeat(40));
|
||||
check(wav(&[chunk(b"iXML", deep.as_bytes())]), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_first_ixml_document_is_considered() {
|
||||
check(wav(&[xml("5", None), xml("6", None)]), Some(5));
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
[package]
|
||||
name = "craft-ber"
|
||||
description = "ASN.1 BER and DER (X.690): reading TLVs, BER to DER, and what an opaque signed CMS holds"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,432 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
//! Reading the TLVs of ASN.1 BER and DER (X.690), and turning BER into DER where they differ in
|
||||
//! what CMS writers use: indefinite lengths (which streaming signers write) and OCTET STRINGs in
|
||||
//! pieces (constructed). What is DER already is kept byte for byte, an element with a definite
|
||||
//! length inside an indefinite one too, so that the bytes a signature covers (signed attributes,
|
||||
//! a certificate's body) are those the signer wrote. For S/MIME: `craft-smime` checks
|
||||
//! signatures with it, and a message's reader takes out what an opaque signed one holds
|
||||
//! ([`signed_content`]).
|
||||
//!
|
||||
//! The input is hostile: lengths are checked against what is there, nesting is bounded
|
||||
//! ([`MAX_DEPTH`]), and the work is linear in the input (times the bounded depth).
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
/// CMS's id-signedData and id-data (RFC 5652 §5.1, §4), as OBJECT IDENTIFIER values.
|
||||
const SIGNED_DATA: &[u8] = &[0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x07, 0x02];
|
||||
const DATA: &[u8] = &[0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x07, 0x01];
|
||||
|
||||
/// What the CMS `cms` holds when it is SignedData with its content inside, of type id-data: the
|
||||
/// MIME entity an opaque signed message (`application/pkcs7-mime`, RFC 8551 §3.5) signed, for
|
||||
/// reading as the message. Nothing is checked here (`craft-smime` checks it); `None` for any
|
||||
/// other CMS (an Encrypted message's enveloped data) or bytes that aren't one.
|
||||
pub fn signed_content(cms: &[u8]) -> Option<Vec<u8>> {
|
||||
let der = to_der(cms)?;
|
||||
let (info, rest) = read(&der)?;
|
||||
if info.tag != 0x30 || !rest.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let info = children(info.value)?;
|
||||
let [oid, explicit] = info.as_slice() else { return None };
|
||||
if oid.tag != 0x06 || oid.value != SIGNED_DATA || explicit.tag != 0xa0 {
|
||||
return None;
|
||||
}
|
||||
let (signed, rest) = read(explicit.value)?;
|
||||
if signed.tag != 0x30 || !rest.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let signed = children(signed.value)?;
|
||||
let [version, digests, encap, tail @ ..] = signed.as_slice() else { return None };
|
||||
if version.tag != 0x02 || !matches!(version.value, [1 | 3 | 4 | 5]) || digests.tag != 0x31 || encap.tag != 0x30 {
|
||||
return None;
|
||||
}
|
||||
if children(digests.value)?.iter().any(|algorithm| algorithm.tag != 0x30) {
|
||||
return None;
|
||||
}
|
||||
// Certificates and revocation lists are optional, but ordered and present at most once.
|
||||
let tail = if tail.first().is_some_and(|field| field.tag == 0xa0) { tail.get(1..)? } else { tail };
|
||||
let tail = if tail.first().is_some_and(|field| field.tag == 0xa1) { tail.get(1..)? } else { tail };
|
||||
let [signers] = tail else { return None };
|
||||
if signers.tag != 0x31 || children(signers.value)?.iter().any(|signer| signer.tag != 0x30) {
|
||||
return None;
|
||||
}
|
||||
let encap = children(encap.value)?;
|
||||
let [kind, content] = encap.as_slice() else { return None };
|
||||
if kind.tag != 0x06 || kind.value != DATA || content.tag != 0xa0 {
|
||||
return None;
|
||||
}
|
||||
let (octets, rest) = read(content.value)?;
|
||||
(octets.tag == 0x04 && rest.is_empty()).then(|| octets.value.to_vec())
|
||||
}
|
||||
|
||||
/// How deep elements may nest.
|
||||
pub const MAX_DEPTH: usize = 48;
|
||||
|
||||
/// Maximum elements materialized or indexed in one parsing operation. This bounds the
|
||||
/// allocation amplification from hostile, shallow input containing many empty TLVs.
|
||||
pub const MAX_ELEMENTS: usize = 65_536;
|
||||
|
||||
/// The universal tag of an OCTET STRING, and of one in pieces.
|
||||
const OCTET_STRING: u8 = 0x04;
|
||||
const OCTET_STRING_PIECES: u8 = 0x24;
|
||||
|
||||
/// One element: its tag bytes, and its value (the content octets).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Tlv<'a> {
|
||||
/// The tag's first byte (class, constructed bit, number below 31).
|
||||
pub tag: u8,
|
||||
/// The whole element, tag and length included.
|
||||
pub whole: &'a [u8],
|
||||
/// Its content octets.
|
||||
pub value: &'a [u8],
|
||||
}
|
||||
|
||||
/// The tag bytes at the start of `bytes`: one, or a high tag number in up to four more.
|
||||
fn tag_len(bytes: &[u8]) -> Option<usize> {
|
||||
let first = *bytes.first()?;
|
||||
if first & 0x1f != 0x1f {
|
||||
return Some(1);
|
||||
}
|
||||
// High tag number: base-128 bytes, the last without its top bit.
|
||||
for (i, b) in bytes.iter().enumerate().skip(1).take(4) {
|
||||
if b & 0x80 == 0 {
|
||||
return i.checked_add(1);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// The length at the start of `bytes`: `(length, bytes it took)`, `None` for an indefinite one
|
||||
/// (`Some((None, 1))`) is told apart by the caller.
|
||||
fn length(bytes: &[u8]) -> Option<(Option<usize>, usize)> {
|
||||
let first = *bytes.first()?;
|
||||
if first < 0x80 {
|
||||
return Some((Some(usize::from(first)), 1));
|
||||
}
|
||||
if first == 0x80 {
|
||||
return Some((None, 1));
|
||||
}
|
||||
let n = usize::from(first & 0x7f);
|
||||
if n > 4 {
|
||||
return None;
|
||||
}
|
||||
let digits = bytes.get(1..n.checked_add(1)?)?;
|
||||
let len = digits.iter().fold(0usize, |acc, d| (acc << 8) | usize::from(*d));
|
||||
Some((Some(len), n.checked_add(1)?))
|
||||
}
|
||||
|
||||
/// The element at the start of `bytes` with a definite length, and what follows it. `None` for
|
||||
/// an indefinite length, or one longer than what is there.
|
||||
pub fn read(bytes: &[u8]) -> Option<(Tlv<'_>, &[u8])> {
|
||||
let t = tag_len(bytes)?;
|
||||
let tag = *bytes.first()?;
|
||||
let (len, l) = length(bytes.get(t..)?)?;
|
||||
let len = len?;
|
||||
let start = t.checked_add(l)?;
|
||||
let end = start.checked_add(len)?;
|
||||
let value = bytes.get(start..end)?;
|
||||
let whole = bytes.get(..end)?;
|
||||
Some((Tlv { tag, whole, value }, bytes.get(end..)?))
|
||||
}
|
||||
|
||||
/// The elements of `value`, each with a definite length; `None` when one can't be read.
|
||||
pub fn children(mut value: &[u8]) -> Option<Vec<Tlv<'_>>> {
|
||||
let mut out = Vec::new();
|
||||
while !value.is_empty() {
|
||||
if out.len() >= MAX_ELEMENTS {
|
||||
return None;
|
||||
}
|
||||
let (tlv, rest) = read(value)?;
|
||||
out.push(tlv);
|
||||
value = rest;
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// `bytes` as DER as far as CMS needs it: indefinite lengths made definite and OCTET STRINGs in
|
||||
/// pieces joined into one, everything else as it was. One element is read, and must be all of
|
||||
/// `bytes` (bar trailing zero bytes, which some writers pad with). `None` when it isn't BER.
|
||||
pub fn to_der(bytes: &[u8]) -> Option<Vec<u8>> {
|
||||
if let Some((_, rest)) = read(bytes)
|
||||
&& rest.iter().all(|b| *b == 0)
|
||||
&& der_shaped(bytes.get(..bytes.len().saturating_sub(rest.len()))?, 0)
|
||||
{
|
||||
return bytes.get(..bytes.len().saturating_sub(rest.len())).map(<[u8]>::to_vec);
|
||||
}
|
||||
let mut out = Vec::with_capacity(bytes.len());
|
||||
let used = element(bytes, 0, &mut out)?;
|
||||
let rest = bytes.get(used..)?;
|
||||
rest.iter().all(|b| *b == 0).then_some(out)
|
||||
}
|
||||
|
||||
/// Whether the elements of `bytes` all have definite lengths and no OCTET STRING is in pieces,
|
||||
/// as deep as they go: nothing to change.
|
||||
fn der_shaped(mut bytes: &[u8], depth: usize) -> bool {
|
||||
if depth > MAX_DEPTH {
|
||||
return false;
|
||||
}
|
||||
while !bytes.is_empty() {
|
||||
let Some((tlv, rest)) = read(bytes) else { return false };
|
||||
if tlv.tag == OCTET_STRING_PIECES || (tlv.tag & 0x20 != 0 && !der_shaped(tlv.value, depth.saturating_add(1))) {
|
||||
return false;
|
||||
}
|
||||
bytes = rest;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// What [`measure`] found of one element: how many bytes of the input it was, and how long its
|
||||
/// content is as DER.
|
||||
#[derive(Clone, Copy)]
|
||||
struct Size {
|
||||
used: usize,
|
||||
content: usize,
|
||||
/// It is DER-shaped already (a definite length, and nothing in it to change): copied as it
|
||||
/// is, its lengths as the writer wrote them, since a signature may cover them.
|
||||
verbatim: bool,
|
||||
}
|
||||
|
||||
/// The sizes of the constructed elements of the input, by where each starts in it: worked out
|
||||
/// once, so that writing them is one pass with no copy of a level for each level above it.
|
||||
type Sizes = std::collections::HashMap<usize, Size>;
|
||||
|
||||
/// The element at `at` in `input`: how many bytes it was, and its length as DER, header and all.
|
||||
fn measure(input: &[u8], at: usize, depth: usize, sizes: &mut Sizes) -> Option<(usize, usize)> {
|
||||
if depth > MAX_DEPTH || sizes.len() >= MAX_ELEMENTS {
|
||||
return None;
|
||||
}
|
||||
let bytes = input.get(at..)?;
|
||||
if let Some((tlv, _)) = read(bytes)
|
||||
&& der_shaped(tlv.whole, depth)
|
||||
{
|
||||
sizes.insert(at, Size { used: tlv.whole.len(), content: tlv.value.len(), verbatim: true });
|
||||
return Some((tlv.whole.len(), tlv.whole.len()));
|
||||
}
|
||||
let t = tag_len(bytes)?;
|
||||
let tag = *bytes.first()?;
|
||||
let (len, l) = length(bytes.get(t..)?)?;
|
||||
let start = t.checked_add(l)?;
|
||||
if tag & 0x20 == 0 {
|
||||
let used = start.checked_add(len?)?;
|
||||
bytes.get(..used)?;
|
||||
return Some((used, used_der(t, used.checked_sub(start)?)?));
|
||||
}
|
||||
// The children, one after another, to the end of a definite length or to the end mark.
|
||||
let mut content = 0usize;
|
||||
let mut pos = start;
|
||||
let end = match len {
|
||||
Some(len) => Some(start.checked_add(len)?),
|
||||
None => None,
|
||||
};
|
||||
loop {
|
||||
match end {
|
||||
Some(end) if pos >= end => break,
|
||||
None if bytes.get(pos..)?.starts_with(&[0, 0]) => {
|
||||
pos = pos.checked_add(2)?;
|
||||
break;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
// (The pieces of an OCTET STRING are OCTET STRINGs, in pieces themselves or not.)
|
||||
if tag == OCTET_STRING_PIECES && !matches!(bytes.get(pos), Some(&OCTET_STRING | &OCTET_STRING_PIECES)) {
|
||||
return None;
|
||||
}
|
||||
let (n, der) = measure(input, at.checked_add(pos)?, depth.checked_add(1)?, sizes)?;
|
||||
if n == 0 {
|
||||
return None;
|
||||
}
|
||||
// An OCTET STRING in pieces is one of their contents together.
|
||||
content = content.checked_add(if tag == OCTET_STRING_PIECES { der_content(input, at.checked_add(pos)?, sizes)? } else { der })?;
|
||||
pos = pos.checked_add(n)?;
|
||||
}
|
||||
if let Some(end) = end
|
||||
&& pos != end
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if sizes.len() >= MAX_ELEMENTS {
|
||||
return None;
|
||||
}
|
||||
sizes.insert(at, Size { used: pos, content, verbatim: false });
|
||||
let header_tag = if tag == OCTET_STRING_PIECES { 1 } else { t };
|
||||
Some((pos, used_der(header_tag, content)?))
|
||||
}
|
||||
|
||||
/// The length of a DER element with a tag of `tag` bytes and `content` bytes of content.
|
||||
fn used_der(tag: usize, content: usize) -> Option<usize> {
|
||||
let mut l = Vec::new();
|
||||
put_length(&mut l, content);
|
||||
tag.checked_add(l.len())?.checked_add(content)
|
||||
}
|
||||
|
||||
/// The DER content length of the (measured) element at `at`: a primitive's own.
|
||||
fn der_content(input: &[u8], at: usize, sizes: &Sizes) -> Option<usize> {
|
||||
if let Some(s) = sizes.get(&at) {
|
||||
return Some(s.content);
|
||||
}
|
||||
let bytes = input.get(at..)?;
|
||||
let t = tag_len(bytes)?;
|
||||
let (len, _) = length(bytes.get(t..)?)?;
|
||||
len
|
||||
}
|
||||
|
||||
/// Write the (measured) element at `at` in `input` to `out` as DER; how many bytes it was.
|
||||
fn write(input: &[u8], at: usize, sizes: &Sizes, pieces: bool, out: &mut Vec<u8>) -> Option<usize> {
|
||||
if let Some(s) = sizes.get(&at).filter(|s| s.verbatim && !pieces) {
|
||||
out.extend_from_slice(input.get(at..at.checked_add(s.used)?)?);
|
||||
return Some(s.used);
|
||||
}
|
||||
let bytes = input.get(at..)?;
|
||||
let t = tag_len(bytes)?;
|
||||
let tag_bytes = bytes.get(..t)?;
|
||||
let tag = *tag_bytes.first()?;
|
||||
let (len, l) = length(bytes.get(t..)?)?;
|
||||
let start = t.checked_add(l)?;
|
||||
if tag & 0x20 == 0 {
|
||||
let used = start.checked_add(len?)?;
|
||||
let value = bytes.get(start..used)?;
|
||||
// (A piece of an OCTET STRING in pieces gives its bytes alone.)
|
||||
if !pieces {
|
||||
out.extend_from_slice(tag_bytes);
|
||||
put_length(out, value.len());
|
||||
}
|
||||
out.extend_from_slice(value);
|
||||
return Some(used);
|
||||
}
|
||||
let size = *sizes.get(&at)?;
|
||||
if tag == OCTET_STRING_PIECES {
|
||||
if !pieces {
|
||||
out.push(OCTET_STRING);
|
||||
put_length(out, size.content);
|
||||
}
|
||||
} else {
|
||||
out.extend_from_slice(tag_bytes);
|
||||
put_length(out, size.content);
|
||||
}
|
||||
let inner_pieces = tag == OCTET_STRING_PIECES;
|
||||
let mut pos = start;
|
||||
while pos < size.used {
|
||||
if bytes.get(pos..)?.starts_with(&[0, 0]) && len.is_none() {
|
||||
break;
|
||||
}
|
||||
let n = write(input, at.checked_add(pos)?, sizes, inner_pieces, out)?;
|
||||
pos = pos.checked_add(n)?;
|
||||
}
|
||||
Some(size.used)
|
||||
}
|
||||
|
||||
/// Write the element at the start of `bytes` to `out` as DER; how many bytes of `bytes` it was.
|
||||
fn element(bytes: &[u8], depth: usize, out: &mut Vec<u8>) -> Option<usize> {
|
||||
let mut sizes = Sizes::new();
|
||||
let (used, der) = measure(bytes, 0, depth, &mut sizes)?;
|
||||
out.reserve(der);
|
||||
write(bytes, 0, &sizes, false, out)?;
|
||||
Some(used)
|
||||
}
|
||||
|
||||
/// A definite length in DER's shortest form.
|
||||
fn put_length(out: &mut Vec<u8>, len: usize) {
|
||||
if len < 0x80 {
|
||||
out.push(len as u8);
|
||||
return;
|
||||
}
|
||||
let bytes = len.to_be_bytes();
|
||||
let skip = bytes.iter().take_while(|b| **b == 0).count();
|
||||
let digits = bytes.get(skip..).unwrap_or_default();
|
||||
out.push(0x80 | digits.len() as u8);
|
||||
out.extend_from_slice(digits);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn tlv(tag: u8, value: &[u8]) -> Vec<u8> {
|
||||
let mut out = vec![tag];
|
||||
put_length(&mut out, value.len());
|
||||
out.extend_from_slice(value);
|
||||
out
|
||||
}
|
||||
|
||||
fn cms(outer_tag: u8, signed_tag: u8, oid_tag: u8, long_oid: bool, trailing: bool) -> Vec<u8> {
|
||||
let oid = |value: &[u8]| {
|
||||
if long_oid {
|
||||
let mut out = vec![oid_tag, 0x81, 9];
|
||||
out.extend_from_slice(value);
|
||||
out
|
||||
} else {
|
||||
tlv(oid_tag, value)
|
||||
}
|
||||
};
|
||||
let mut encap = oid(DATA);
|
||||
encap.extend(tlv(0xa0, &tlv(0x04, b"hello")));
|
||||
let mut signed = tlv(0x02, &[1]);
|
||||
signed.extend(tlv(0x31, &[]));
|
||||
signed.extend(tlv(0x30, &encap));
|
||||
signed.extend(tlv(0x31, &[]));
|
||||
let mut signed = tlv(signed_tag, &signed);
|
||||
if trailing {
|
||||
signed.extend(tlv(0x05, &[]));
|
||||
}
|
||||
let mut info = oid(SIGNED_DATA);
|
||||
info.extend(tlv(0xa0, &signed));
|
||||
tlv(outer_tag, &info)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_content_with_der_and_ber_oid_lengths() {
|
||||
for long_oid in [false, true] {
|
||||
assert_eq!(signed_content(&cms(0x30, 0x30, 0x06, long_oid, false)), Some(b"hello".to_vec()));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_wrong_cms_container_and_oid_tags() {
|
||||
for (outer, signed, oid) in [(0x31, 0x30, 0x06), (0x30, 0x31, 0x06), (0x30, 0x30, 0x04)] {
|
||||
assert_eq!(signed_content(&cms(outer, signed, oid, false, false)), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_missing_mandatory_signed_data_fields() {
|
||||
let mut encap = tlv(0x06, DATA);
|
||||
encap.extend(tlv(0xa0, &tlv(0x04, b"hello")));
|
||||
for prefix in [vec![0x05, 0, 0x05, 0], vec![0x02, 1, 1, 0x31, 0]] {
|
||||
let mut malformed = prefix;
|
||||
malformed.extend(tlv(0x30, &encap));
|
||||
let mut info = tlv(0x06, SIGNED_DATA);
|
||||
info.extend(tlv(0xa0, &tlv(0x30, &malformed)));
|
||||
assert_eq!(signed_content(&tlv(0x30, &info)), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_extra_signed_data_inside_explicit_container() {
|
||||
assert_eq!(signed_content(&cms(0x30, 0x30, 0x06, false, true)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_wide_inputs_before_unbounded_metadata_allocation() {
|
||||
let children_bytes = [0x05, 0].repeat(MAX_ELEMENTS + 1);
|
||||
assert!(children(&children_bytes).is_none());
|
||||
let mut indefinite = vec![0x30, 0x80];
|
||||
indefinite.extend_from_slice(&children_bytes);
|
||||
indefinite.extend_from_slice(&[0, 0]);
|
||||
assert!(to_der(&indefinite).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn joins_indefinite_octet_string_pieces() {
|
||||
assert_eq!(to_der(&[0x24, 0x80, 0x04, 1, b'a', 0x04, 1, b'b', 0, 0]), Some(vec![0x04, 2, b'a', b'b']));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_truncated_and_over_nested_input() {
|
||||
assert_eq!(to_der(&[0x04, 2, b'a']), None);
|
||||
let mut nested = tlv(0x04, &[]);
|
||||
for _ in 0..=MAX_DEPTH {
|
||||
nested = tlv(0x30, &nested);
|
||||
}
|
||||
assert_eq!(to_der(&nested), None);
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,7 @@ exr = { version = "1.73", default-features = false, features = ["rayon"] }
|
||||
[dev-dependencies]
|
||||
proptest = { workspace = true }
|
||||
image = { version = "0.25", default-features = false, features = ["png", "jpeg", "tiff"] }
|
||||
sha2 = { version = "0.10", default-features = false, features = ["std"] }
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
|
||||
jxl-oxide = { version = "=0.12.6", default-features = false, features = ["rayon"] }
|
||||
@@ -302,9 +302,11 @@ embedded table (and a PNG's transparent entry) so `photocraft-io` can open them
|
||||
`DecodeOptions { limits: Limits { max_width, max_height, max_pixels, max_alloc }, .. }` guards
|
||||
decoding. Header dimensions are checked before the pixel buffer is allocated, and the budget is
|
||||
also passed to the underlying decoders. A violation returns `CodecError::LimitExceeded`. The
|
||||
defaults are 262144 px per side, 2^30 pixels and 8 GiB (2 GiB on 32-bit targets such as wasm).
|
||||
defaults are 262144 px per side, 2^30 pixels and 8 GiB (1 GiB on 32-bit targets such as wasm).
|
||||
PNG also bounds the aggregate decoded text and XMP (UTF-8 keywords and values) by `max_alloc`,
|
||||
including chunks after IDAT. This budget is separate from pixels, not a total process-memory cap.
|
||||
`Image::to_rgba_f32(&limits)` checks its output buffer against the same `Limits` and returns
|
||||
`CodecError::LimitExceeded` before allocating when the RGBA f32 copy would exceed them.
|
||||
PNG `cICP` and `cLLI` values are retained as image color metadata. `encode_png_pq` writes linear-light
|
||||
Rec. 2020 RGB as 16-bit BT.2100 PQ, with measured content-light levels and the required chunks.
|
||||
|
||||
@@ -324,3 +326,16 @@ Rec. 2020 RGB as 16-bit BT.2100 PQ, with measured content-light levels and the r
|
||||
|
||||
If `corpus/pngsuite/*.png` exists at the repo root, every file in it is compared against the
|
||||
`image` crate's decoder. Without it, that test is skipped.
|
||||
|
||||
With the `corpus` feature, the deep EXR test reads `11.deep.exr`, `42.deep.exr`, `64.deep.exr`,
|
||||
`multivariate.deep.exr` and `objectid.deep.exr` from `corpus/exr/` at the repo root. These are the
|
||||
deep test images in `src/test/bin/test_images` of the
|
||||
[OpenEXR repository at commit `69b2604fc76e370615438bdc8d2cd95b9349c12e`](https://github.com/AcademySoftwareFoundation/openexr/tree/69b2604fc76e370615438bdc8d2cd95b9349c12e/src/test/bin/test_images).
|
||||
The test checks each file against the SHA-256 pinned in `REAL_FILE_SHA256` in
|
||||
`tests/it/deep_exr.rs` before decoding it, and fails when a file is missing, is not a regular
|
||||
file, is larger than 4 MiB or does not match. It reads no other file in the folder. To fill the
|
||||
folder, copy the five files from that commit into `corpus/exr/`, or run
|
||||
`cargo xtask corpus --exr` in a checkout of [PhotoCraft](https://github.com/storytold/PhotoCraft)
|
||||
at commit `74330157cb601573ae4523a95bb9132822fc3918` and copy its `corpus/exr/` here, as CI does.
|
||||
The images are not committed here; OpenEXR distributes them under its
|
||||
[license](https://github.com/AcademySoftwareFoundation/openexr/blob/69b2604fc76e370615438bdc8d2cd95b9349c12e/LICENSE.md).
|
||||
@@ -8,6 +8,7 @@ use std::fmt;
|
||||
|
||||
use crate::error::CodecError;
|
||||
use crate::format::Format;
|
||||
use crate::options::Limits;
|
||||
use half::f16;
|
||||
|
||||
/// Channel layout of an [`Image`]. Alpha, when present, is always the last
|
||||
@@ -716,9 +717,31 @@ impl Image {
|
||||
self.convert(ChannelLayout::Rgba, SampleType::U16).to_u16_samples().unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Interleaved RGBA as f32 (integer inputs normalized to `[0,1]`).
|
||||
pub fn to_rgba_f32(&self) -> Vec<f32> {
|
||||
convert_layout(&self.to_normalized(), self.layout, ChannelLayout::Rgba)
|
||||
/// Interleaved RGBA as f32 (integer inputs normalized to `[0,1]`), with the same values as
|
||||
/// [`Image::convert`] to RGBA F32. The output buffer is checked against `limits` (dimensions,
|
||||
/// pixel count and `max_alloc`) and reserved before any pixel is converted, and the pixels are
|
||||
/// written straight into it. A size over the limits, or a reservation the allocator refuses,
|
||||
/// returns [`CodecError::LimitExceeded`]. An image with no pixels gives an empty vector.
|
||||
pub fn to_rgba_f32(&self, limits: &Limits) -> Result<Vec<f32>, CodecError> {
|
||||
if self.width == 0 || self.height == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
limits.check(self.width, self.height, ChannelLayout::Rgba, SampleType::F32)?;
|
||||
let len = self
|
||||
.pixel_count()
|
||||
.checked_mul(ChannelLayout::Rgba.channels())
|
||||
.ok_or_else(|| CodecError::LimitExceeded("RGBA f32 size overflows usize".into()))?;
|
||||
let mut out = Vec::new();
|
||||
out.try_reserve_exact(len).map_err(|e| CodecError::LimitExceeded(format!("cannot reserve {len} f32 samples for RGBA: {e}")))?;
|
||||
let fc = self.layout.channels();
|
||||
let mut px = [0.0f32; 5];
|
||||
for p in 0..self.pixel_count() {
|
||||
for (c, v) in px[..fc].iter_mut().enumerate() {
|
||||
*v = read_normalized(&self.data, self.sample, p * fc + c);
|
||||
}
|
||||
convert_pixel(&px[..fc], self.layout, ChannelLayout::Rgba, &mut out);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -890,33 +913,39 @@ pub(crate) fn convert_layout(src: &[f32], from: ChannelLayout, to: ChannelLayout
|
||||
let n = src.len() / fc;
|
||||
let mut out = Vec::with_capacity(n * tc);
|
||||
for px in src.chunks_exact(fc) {
|
||||
let alpha = if from.has_alpha() { px[fc - 1] } else { 1.0 };
|
||||
let color = &px[..from.color_channels()];
|
||||
if from.same_model(to) {
|
||||
out.extend_from_slice(color);
|
||||
} else {
|
||||
let rgb = match from {
|
||||
ChannelLayout::Gray | ChannelLayout::GrayA => [color[0]; 3],
|
||||
ChannelLayout::Rgb | ChannelLayout::Rgba => [color[0], color[1], color[2]],
|
||||
ChannelLayout::Cmyk | ChannelLayout::CmykA => cmyk_to_rgb(color[0], color[1], color[2], color[3]),
|
||||
ChannelLayout::Lab | ChannelLayout::LabA => crate::lab::lab_to_srgb([color[0], color[1], color[2]]),
|
||||
};
|
||||
match to {
|
||||
ChannelLayout::Gray | ChannelLayout::GrayA => {
|
||||
out.push(rgb[0] * LUMA[0] + rgb[1] * LUMA[1] + rgb[2] * LUMA[2]);
|
||||
}
|
||||
ChannelLayout::Rgb | ChannelLayout::Rgba => out.extend_from_slice(&rgb),
|
||||
ChannelLayout::Cmyk | ChannelLayout::CmykA => out.extend_from_slice(&rgb_to_cmyk(rgb[0], rgb[1], rgb[2])),
|
||||
ChannelLayout::Lab | ChannelLayout::LabA => out.extend_from_slice(&crate::lab::srgb_to_lab(rgb)),
|
||||
}
|
||||
}
|
||||
if to.has_alpha() {
|
||||
out.push(alpha);
|
||||
}
|
||||
convert_pixel(px, from, to, &mut out);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// One pixel of [`convert_layout`]: the normalized samples `px` (`from` layout) appended to `out`
|
||||
/// in the `to` layout.
|
||||
fn convert_pixel(px: &[f32], from: ChannelLayout, to: ChannelLayout, out: &mut Vec<f32>) {
|
||||
let alpha = if from.has_alpha() { px[from.channels() - 1] } else { 1.0 };
|
||||
let color = &px[..from.color_channels()];
|
||||
if from.same_model(to) {
|
||||
out.extend_from_slice(color);
|
||||
} else {
|
||||
let rgb = match from {
|
||||
ChannelLayout::Gray | ChannelLayout::GrayA => [color[0]; 3],
|
||||
ChannelLayout::Rgb | ChannelLayout::Rgba => [color[0], color[1], color[2]],
|
||||
ChannelLayout::Cmyk | ChannelLayout::CmykA => cmyk_to_rgb(color[0], color[1], color[2], color[3]),
|
||||
ChannelLayout::Lab | ChannelLayout::LabA => crate::lab::lab_to_srgb([color[0], color[1], color[2]]),
|
||||
};
|
||||
match to {
|
||||
ChannelLayout::Gray | ChannelLayout::GrayA => {
|
||||
out.push(rgb[0] * LUMA[0] + rgb[1] * LUMA[1] + rgb[2] * LUMA[2]);
|
||||
}
|
||||
ChannelLayout::Rgb | ChannelLayout::Rgba => out.extend_from_slice(&rgb),
|
||||
ChannelLayout::Cmyk | ChannelLayout::CmykA => out.extend_from_slice(&rgb_to_cmyk(rgb[0], rgb[1], rgb[2])),
|
||||
ChannelLayout::Lab | ChannelLayout::LabA => out.extend_from_slice(&crate::lab::srgb_to_lab(rgb)),
|
||||
}
|
||||
}
|
||||
if to.has_alpha() {
|
||||
out.push(alpha);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
@@ -1042,6 +1071,52 @@ mod tests {
|
||||
assert!(Image::from_raw(2, 2, ChannelLayout::Rgb, SampleType::U8, vec![0; 11]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rgba_f32_limit_rejects_before_allocation() {
|
||||
// 3x2 pixels: the RGBA f32 output is 3 * 2 * 4 * 4 = 96 bytes. Most sources here are
|
||||
// smaller than that, so the source fitting the budget does not let the output through.
|
||||
for sample in SampleType::ALL {
|
||||
for layout in ChannelLayout::ALL {
|
||||
let img = Image::new(3, 2, layout, sample).unwrap();
|
||||
let tight = Limits::new(3, 2, 6, 95);
|
||||
assert!(matches!(img.to_rgba_f32(&tight), Err(CodecError::LimitExceeded(_))), "{layout:?} {sample:?}: max_alloc");
|
||||
let exact = Limits::new(3, 2, 6, 96);
|
||||
assert_eq!(img.to_rgba_f32(&exact).map(|v| v.len()).ok(), Some(24), "{layout:?} {sample:?}");
|
||||
for limits in [Limits::new(2, 2, 6, 96), Limits::new(3, 1, 6, 96), Limits::new(3, 2, 5, 96)] {
|
||||
assert!(matches!(img.to_rgba_f32(&limits), Err(CodecError::LimitExceeded(_))), "{layout:?} {sample:?}: {limits:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rgba_f32_matches_existing_conversions() {
|
||||
// Edge, mid and (for float samples) out-of-range values, so the CMYK, Lab and alpha
|
||||
// formulas and the f16 and f32 reads are all compared bit for bit.
|
||||
let base = [0.0, 1.0, 0.5, 0.25, 0.75, 1.0 / 255.0, 0.3, 0.9, 1.5, -0.25];
|
||||
for sample in SampleType::ALL {
|
||||
for layout in ChannelLayout::ALL {
|
||||
let n = 7 * 3 * layout.channels();
|
||||
let vals: Vec<f32> = (0..n).map(|i| base[(i * 7 + i / 3) % base.len()]).collect();
|
||||
let img = Image::from_normalized(7, 3, layout, sample, &vals).unwrap();
|
||||
let got = img.to_rgba_f32(&Limits::default()).unwrap();
|
||||
let want = convert_layout(&img.to_normalized(), layout, ChannelLayout::Rgba);
|
||||
let via_convert = img.convert(ChannelLayout::Rgba, SampleType::F32).to_f32_samples().unwrap();
|
||||
let bits = |v: &[f32]| v.iter().map(|x| x.to_bits()).collect::<Vec<_>>();
|
||||
assert_eq!(bits(&got), bits(&want), "{layout:?} {sample:?}");
|
||||
assert_eq!(bits(&got), bits(&via_convert), "{layout:?} {sample:?}: convert");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rgba_f32_of_an_empty_image_is_empty() {
|
||||
for (w, h) in [(0, 0), (0, 3), (4, 0)] {
|
||||
let img = Image::new(w, h, ChannelLayout::Rgb, SampleType::U8).unwrap();
|
||||
assert_eq!(img.to_rgba_f32(&Limits::new(1, 1, 1, 0)).ok(), Some(Vec::new()), "{w}x{h}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rgba8_of_gray() {
|
||||
let img = Image::from_u8(1, 1, ChannelLayout::GrayA, vec![10, 20]).unwrap();
|
||||
|
||||
@@ -2,11 +2,14 @@
|
||||
//! specification (NONE, RLE and ZIPS × scanline and tile), the oracle tests check the
|
||||
//! compositing math against hand-computed values, and malformed inputs must error, never
|
||||
//! panic. With the `corpus` feature, the OpenEXR project's deep test images (`corpus/exr/`,
|
||||
//! pinned) are checked against the ID manifests their sidecars list.
|
||||
//! each checked against its pinned SHA-256 before decoding) are checked against the ID
|
||||
//! manifests their sidecars list.
|
||||
|
||||
use craft_image_codecs::CodecError;
|
||||
use craft_image_codecs::f16;
|
||||
use craft_image_codecs::{Limits, decode, decode_deep_exr};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::io::Read as _;
|
||||
use std::io::Write as _;
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
@@ -797,10 +800,55 @@ fn inconsistent_count_table_errors() {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Real reference files: the deep test images of the OpenEXR repository, pinned in
|
||||
// `xtask/src/corpus_pins.rs` and fetched into `corpus/exr/` by `cargo xtask corpus --exr`
|
||||
// (cargo feature `corpus`; `cargo xtask test-corpus`). Their `.txt` sidecars upstream list the
|
||||
// ID manifest of each file, an independent oracle for the u32 ID channels.
|
||||
// Real reference files: the deep test images of the OpenEXR repository at commit
|
||||
// `69b2604fc76e370615438bdc8d2cd95b9349c12e`, read from `corpus/exr/` at the repo root with the
|
||||
// cargo feature `corpus`. `REAL_FILE_SHA256` pins each file's SHA-256, and the test refuses a
|
||||
// file that is missing, is not a regular file, is larger than `MAX_REAL_FILE_BYTES` or does not
|
||||
// match its digest. The crate README says how to fill the folder. Their `.txt` sidecars upstream
|
||||
// list the ID manifest of each file, an independent oracle for the u32 ID channels.
|
||||
|
||||
/// The pinned files as name and lowercase hex SHA-256, the same five names as `REAL_FILES`.
|
||||
const REAL_FILE_SHA256: &[(&str, &str)] = &[
|
||||
("11.deep.exr", "800ec94ffb2085da9a730dcda2f9611f5fb62f922a3f2aef77287e36106b9667"),
|
||||
("42.deep.exr", "439907f3448480894264f0c8847f6c572f645441db34a5ad9f36f3f020d101b5"),
|
||||
("64.deep.exr", "4d9bfa646323b80630933e114a3112095488e8c008cdf908e210979638e65fe4"),
|
||||
("multivariate.deep.exr", "2335ed6052258fe693cb795f2bea57f22ba0564281691c82f2987d31d96e9e74"),
|
||||
("objectid.deep.exr", "af8940912bfa856b2650d2005462fd348656385161ccf0745b91b143e934c5f6"),
|
||||
];
|
||||
|
||||
/// The largest file read from `corpus/exr/`. The largest pinned file is 611,840 bytes.
|
||||
#[cfg(feature = "corpus")]
|
||||
const MAX_REAL_FILE_BYTES: u64 = 4 * 1024 * 1024;
|
||||
|
||||
/// The lowercase hex SHA-256 of `bytes`.
|
||||
fn sha256_hex(bytes: &[u8]) -> String {
|
||||
Sha256::digest(bytes).iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
/// Reads a regular file of at most `limit` bytes. A symbolic link, a folder or a larger file is an
|
||||
/// error, and a file that grows during the read is cut off at `limit + 1` bytes and refused.
|
||||
fn read_bounded(path: &std::path::Path, limit: u64) -> Result<Vec<u8>, String> {
|
||||
let meta = std::fs::symlink_metadata(path).map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
if !meta.is_file() || meta.len() > limit {
|
||||
return Err(format!("{}: expected a regular file of at most {limit} bytes", path.display()));
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
std::fs::File::open(path).and_then(|f| f.take(limit + 1).read_to_end(&mut bytes)).map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
if bytes.len() as u64 > limit {
|
||||
return Err(format!("{}: exceeds {limit} bytes", path.display()));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
/// Reads `dir/name` within `limit` bytes and checks its SHA-256 against `pinned`.
|
||||
fn read_pinned(dir: &std::path::Path, name: &str, pinned: &str, limit: u64) -> Result<Vec<u8>, String> {
|
||||
let bytes = read_bounded(&dir.join(name), limit)?;
|
||||
let got = sha256_hex(&bytes);
|
||||
if got != pinned {
|
||||
return Err(format!("{name}: SHA-256 {got} does not match the pinned {pinned}"));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
/// One pinned file: name, total samples, most samples in one pixel, and per ID channel the
|
||||
/// manifest IDs its sidecar lists.
|
||||
@@ -846,9 +894,12 @@ const REAL_FILES: &[RealFile] = &[
|
||||
#[test]
|
||||
fn real_deep_files_decode() {
|
||||
let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus/exr");
|
||||
for &(name, total, max_per_pixel, ids) in REAL_FILES {
|
||||
let path = dir.join(name);
|
||||
let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{}: {e} (run `cargo xtask corpus --exr`)", path.display()));
|
||||
let decoded: Vec<&str> = REAL_FILES.iter().map(|f| f.0).collect();
|
||||
let pinned: Vec<&str> = REAL_FILE_SHA256.iter().map(|p| p.0).collect();
|
||||
assert_eq!(decoded, pinned, "REAL_FILES and REAL_FILE_SHA256 must list the same files");
|
||||
for (&(name, total, max_per_pixel, ids), &(_, sha256)) in REAL_FILES.iter().zip(REAL_FILE_SHA256) {
|
||||
let bytes = read_pinned(&dir, name, sha256, MAX_REAL_FILE_BYTES)
|
||||
.unwrap_or_else(|e| panic!("{}: {e}; place the pinned OpenEXR deep test images in this folder (see the crate README)", dir.display()));
|
||||
let deep = decode_deep_exr(&bytes, &Limits::none()).unwrap_or_else(|e| panic!("{name}: {e}"));
|
||||
assert_eq!((deep.width, deep.height), (256, 256), "{name}");
|
||||
assert_eq!(deep.total_samples(), total, "{name}");
|
||||
@@ -874,6 +925,66 @@ fn real_deep_files_decode() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn real_file_pins_are_well_formed() {
|
||||
let mut names = std::collections::BTreeSet::new();
|
||||
for &(name, sha256) in REAL_FILE_SHA256 {
|
||||
assert!(name.ends_with(".deep.exr") && !name.contains(['/', '\\']), "{name}");
|
||||
assert!(names.insert(name), "{name} is pinned twice");
|
||||
assert_eq!(sha256.len(), 64, "{name}");
|
||||
assert!(sha256.bytes().all(|c| matches!(c, b'0'..=b'9' | b'a'..=b'f')), "{name}: {sha256}");
|
||||
}
|
||||
assert_eq!(names.len(), 5);
|
||||
}
|
||||
|
||||
/// A folder under the system temp directory, removed when dropped.
|
||||
struct ScratchDir(std::path::PathBuf);
|
||||
|
||||
impl ScratchDir {
|
||||
fn new(tag: &str) -> Self {
|
||||
let dir = std::env::temp_dir().join(format!("craft-image-codecs-{tag}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).expect("create scratch dir");
|
||||
Self(dir)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScratchDir {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pinned_read_refuses_a_changed_byte() {
|
||||
let dir = ScratchDir::new("deep-exr-pin");
|
||||
// SHA-256 of "abc" (FIPS 180-2, appendix B.1).
|
||||
let abc = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad";
|
||||
std::fs::write(dir.0.join("a.deep.exr"), b"abc").expect("write");
|
||||
assert_eq!(read_pinned(&dir.0, "a.deep.exr", abc, 16).as_deref(), Ok(&b"abc"[..]));
|
||||
|
||||
std::fs::write(dir.0.join("a.deep.exr"), b"abd").expect("write");
|
||||
let err = read_pinned(&dir.0, "a.deep.exr", abc, 16).expect_err("a changed byte must be refused");
|
||||
assert!(err.starts_with("a.deep.exr: ") && err.contains(abc) && err.contains(&sha256_hex(b"abd")), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pinned_read_refuses_missing_oversize_and_folders() {
|
||||
let dir = ScratchDir::new("deep-exr-bound");
|
||||
let pin = sha256_hex(b"12345678");
|
||||
std::fs::write(dir.0.join("eight.deep.exr"), b"12345678").expect("write");
|
||||
assert!(read_pinned(&dir.0, "eight.deep.exr", &pin, 8).is_ok());
|
||||
let err = read_pinned(&dir.0, "eight.deep.exr", &pin, 7).expect_err("a file over the limit must be refused");
|
||||
assert!(err.contains("at most 7 bytes"), "{err}");
|
||||
|
||||
let err = read_pinned(&dir.0, "missing.deep.exr", &pin, 8).expect_err("a missing file must be refused");
|
||||
assert!(err.contains("missing.deep.exr"), "{err}");
|
||||
|
||||
std::fs::create_dir(dir.0.join("folder.deep.exr")).expect("mkdir");
|
||||
let err = read_pinned(&dir.0, "folder.deep.exr", &pin, 8).expect_err("a folder must be refused");
|
||||
assert!(err.contains("regular file"), "{err}");
|
||||
}
|
||||
|
||||
/// `gen_deep` for a cropped render: the data window sits at `(dwx, dwy)` inside a larger
|
||||
/// display window of `sw × sh` (the stage the renderer composed onto). Scanline chunk Y
|
||||
/// coordinates are absolute display rows, so they carry the data-window offset.
|
||||
|
||||
+162
-11
@@ -351,28 +351,70 @@ fn trak(bytes: &[u8], t: &BoxRef, budget: &mut usize) -> Option<Cr3Track> {
|
||||
}
|
||||
}
|
||||
let stbl = stbl?;
|
||||
let (mut kind, mut size, mut offset) = (None, None, None);
|
||||
let (mut kind, mut size, mut offset, mut mapping, mut sample_count) = (None, None, None, None, 0);
|
||||
for b in boxes(bytes, stbl.body, stbl.end, budget) {
|
||||
let body = bytes.get(b.body..b.end)?;
|
||||
match &b.kind {
|
||||
// full box: version/flags, entry count, entries
|
||||
b"stsd" if be32(body, 4)? > 0 => kind = boxes(bytes, b.body + 8, b.end, budget).first().map(|e| sample_entry(bytes, e, budget)),
|
||||
b"stsc" => {
|
||||
// Only the first sample is exposed. It must map chunk 1 to
|
||||
// the first sample description; don't trust offsets without
|
||||
// that mapping, or a damaged/raw track could become a JPEG.
|
||||
// A malformed mapping invalidates sample data, not the track's
|
||||
// identity: dropping the largest raw track could select a reduced
|
||||
// raw successfully and disguise corruption in the main image.
|
||||
mapping = (|| {
|
||||
let count = be32(body, 4)? as usize;
|
||||
if be32(body, 0)? != 0 || count == 0 || count > body.len().saturating_sub(8) / 12 {
|
||||
return None;
|
||||
}
|
||||
Some((be32(body, 8)?, be32(body, 12)?, be32(body, 16)?))
|
||||
})();
|
||||
}
|
||||
// version/flags, sample_size (0 = per-sample table), count, sizes
|
||||
b"stsz" => {
|
||||
size = match be32(body, 4)? {
|
||||
0 if be32(body, 8)? > 0 => be32(body, 12),
|
||||
0 => None,
|
||||
n if be32(body, 8)? > 0 => Some(n),
|
||||
// A truncated size table invalidates data without hiding a known track.
|
||||
sample_count = be32(body, 8).unwrap_or(0);
|
||||
size = match be32(body, 4) {
|
||||
// A first entry cannot justify an incomplete per-sample
|
||||
// table, nor an impossible constant-size sample count.
|
||||
Some(0) if sample_count > 0 && sample_count as usize <= body.len().saturating_sub(12) / 4 => be32(body, 12),
|
||||
Some(n)
|
||||
if n > 0
|
||||
&& sample_count > 0
|
||||
&& u64::from(n).checked_mul(u64::from(sample_count)).is_some_and(|total| total <= bytes.len() as u64) =>
|
||||
{
|
||||
Some(n)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
b"co64" if be32(body, 4)? > 0 => offset = be64(body, 8).and_then(|v| usize::try_from(v).ok()),
|
||||
b"stco" if be32(body, 4)? > 0 => offset = be32(body, 8).map(|v| v as usize),
|
||||
// Missing or truncated offset-table counts deny sample access, but the known
|
||||
// track must remain available to the largest-raw-track selector.
|
||||
b"co64" => {
|
||||
offset = match be32(body, 4) {
|
||||
Some(count) if count > 0 && count as usize <= body.len().saturating_sub(8) / 8 => {
|
||||
be64(body, 8).and_then(|v| usize::try_from(v).ok())
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
b"stco" => {
|
||||
offset = match be32(body, 4) {
|
||||
Some(count) if count > 0 && count as usize <= body.len().saturating_sub(8) / 4 => be32(body, 8).map(|v| v as usize),
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let data = match (offset, size) {
|
||||
(Some(o), Some(s)) if o.checked_add(s as usize).is_some_and(|e| e <= bytes.len()) => Some((o, s as usize)),
|
||||
let data = match (offset, size, mapping) {
|
||||
(Some(o), Some(s), Some((1, samples, 1)))
|
||||
if samples > 0 && samples <= sample_count && o.checked_add(s as usize).is_some_and(|e| e <= bytes.len()) =>
|
||||
{
|
||||
Some((o, s as usize))
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
Some(Cr3Track { kind: kind?, data })
|
||||
@@ -495,6 +537,13 @@ mod tests {
|
||||
bx(b"CRAW", &b)
|
||||
}
|
||||
fn trak(entry: Vec<u8>, offset: u64, size: u32) -> Vec<u8> {
|
||||
let stsc = [1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>();
|
||||
trak_with_mapping(entry, offset, size, &stsc)
|
||||
}
|
||||
fn trak_with_mapping(entry: Vec<u8>, offset: u64, size: u32, mapping: &[u8]) -> Vec<u8> {
|
||||
trak_with_table(entry, offset, size, b"stsc", mapping)
|
||||
}
|
||||
fn trak_with_table(entry: Vec<u8>, offset: u64, size: u32, table: &[u8; 4], payload: &[u8]) -> Vec<u8> {
|
||||
let mut stsd = 1u32.to_be_bytes().to_vec();
|
||||
stsd.extend(entry);
|
||||
let mut stsz = 0u32.to_be_bytes().to_vec();
|
||||
@@ -502,7 +551,15 @@ mod tests {
|
||||
stsz.extend(size.to_be_bytes());
|
||||
let mut co64 = 1u32.to_be_bytes().to_vec();
|
||||
co64.extend(offset.to_be_bytes());
|
||||
let stbl = [full(b"stsd", &stsd), full(b"stsz", &stsz), full(b"co64", &co64)].concat();
|
||||
let stsc = [1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>();
|
||||
let offsets = if table == b"stco" { full(b"stco", payload) } else { full(b"co64", if table == b"co64" { payload } else { &co64 }) };
|
||||
let stbl = [
|
||||
full(b"stsd", &stsd),
|
||||
full(b"stsc", if table == b"stsc" { payload } else { &stsc }),
|
||||
full(b"stsz", if table == b"stsz" { payload } else { &stsz }),
|
||||
offsets,
|
||||
]
|
||||
.concat();
|
||||
bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &stbl))))
|
||||
}
|
||||
fn tiff_with(tag: u16, v: craft_tiff::Value) -> Vec<u8> {
|
||||
@@ -543,6 +600,97 @@ mod tests {
|
||||
assert_eq!(c.tracks[1].data, Some((4112, 32)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_sample_requires_a_complete_valid_sample_to_chunk_mapping() {
|
||||
let file = |track: Vec<u8>| {
|
||||
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
|
||||
file.extend(bx(b"moov", &track));
|
||||
file.resize(4112, 0);
|
||||
file
|
||||
};
|
||||
let valid = trak(craw(16, 16, &bx(b"JPEG", &[0; 4])), 4096, 16);
|
||||
assert_eq!(parse_cr3(&file(valid.clone())).unwrap().tracks[0].data, Some((4096, 16)));
|
||||
let at = valid.windows(4).position(|bytes| bytes == b"stsc").unwrap();
|
||||
// Missing table, impossible first chunk, zero samples, a different
|
||||
// description and a count that extends beyond this box all deny data.
|
||||
for (field, value) in [(0, u32::from_be_bytes(*b"xxxx")), (8, 0), (8, 2), (12, 0), (12, 2), (16, 0), (16, 2), (4, u32::MAX)] {
|
||||
let mut damaged = valid.clone();
|
||||
let offset = if field == 0 { at } else { at + 4 + field };
|
||||
damaged[offset..offset + 4].copy_from_slice(&value.to_be_bytes());
|
||||
let bytes = file(damaged);
|
||||
let parsed = parse_cr3(&bytes).unwrap();
|
||||
assert_eq!(parsed.tracks.len(), 1, "invalid mapping must not erase an identifiable track");
|
||||
assert_eq!(parsed.tracks[0].kind, Cr3TrackKind::Jpeg);
|
||||
assert!(parsed.tracks[0].data.is_none());
|
||||
}
|
||||
for entry in [craw(16, 16, &[]), craw(16, 16, &bx(b"CMP1", &[0xff, 0x10, 0, 0x30]))] {
|
||||
let bytes = file(trak(entry, 4096, 16));
|
||||
let parsed = parse_cr3(&bytes).unwrap();
|
||||
assert!(!matches!(parsed.tracks[0].kind, Cr3TrackKind::Jpeg));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_sample_mapping_preserves_larger_raw_track_identity_and_valid_reduced_track() {
|
||||
let reduced = trak(craw(4, 4, &bx(b"CMP1", &compression_payload(0))), 4096, 16);
|
||||
for table in [b"stsc", b"stsz", b"co64", b"stco"] {
|
||||
for payload in [Vec::new(), vec![0; 4], 1u32.to_be_bytes().to_vec()] {
|
||||
// full() supplies version/flags; an empty payload leaves only
|
||||
// those four bytes, with the table's required count missing.
|
||||
let main = trak_with_table(craw(8, 4, &bx(b"CMP1", &compression_payload(0))), 4112, 32, table, &payload);
|
||||
for main_first in [false, true] {
|
||||
let tracks = if main_first { [main.clone(), reduced.clone()].concat() } else { [reduced.clone(), main.clone()].concat() };
|
||||
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
|
||||
file.extend(bx(b"moov", &tracks));
|
||||
file.resize(4144, 0);
|
||||
let parsed = parse_cr3(&file).unwrap();
|
||||
assert_eq!(parsed.tracks.len(), 2, "truncated {table:?} cannot hide the larger track");
|
||||
let (main, reduced) = if main_first { (&parsed.tracks[0], &parsed.tracks[1]) } else { (&parsed.tracks[1], &parsed.tracks[0]) };
|
||||
assert!(matches!(reduced.kind, Cr3TrackKind::Raw { width: 4, height: 4, .. }));
|
||||
assert_eq!(reduced.data, Some((4096, 16)));
|
||||
assert!(matches!(main.kind, Cr3TrackKind::Raw { width: 8, height: 4, cmp1: Some(_), .. }));
|
||||
assert!(main.data.is_none());
|
||||
assert!(main.compression(&file).is_some(), "only sample location is invalid");
|
||||
}
|
||||
}
|
||||
let mut offsets = 1u32.to_be_bytes().to_vec();
|
||||
offsets.extend(4112u64.to_be_bytes());
|
||||
let valid = match table {
|
||||
b"stsc" => [1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>(),
|
||||
b"stsz" => [0u32, 1, 32].into_iter().flat_map(u32::to_be_bytes).collect(),
|
||||
b"stco" => [1u32, 4112].into_iter().flat_map(u32::to_be_bytes).collect(),
|
||||
_ => offsets,
|
||||
};
|
||||
let main = trak_with_table(craw(8, 4, &bx(b"CMP1", &compression_payload(0))), 4112, 32, table, &valid);
|
||||
for main_first in [false, true] {
|
||||
let tracks = if main_first { [main.clone(), reduced.clone()].concat() } else { [reduced.clone(), main.clone()].concat() };
|
||||
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
|
||||
file.extend(bx(b"moov", &tracks));
|
||||
file.resize(4144, 0);
|
||||
let parsed = parse_cr3(&file).unwrap();
|
||||
assert_eq!(parsed.tracks.len(), 2);
|
||||
let main = &parsed.tracks[if main_first { 0 } else { 1 }];
|
||||
assert_eq!(main.data, Some((4112, 32)), "complete valid {table:?} stays usable");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_sample_size_requires_a_complete_bounded_sample_table() {
|
||||
let valid = trak(craw(16, 16, &bx(b"JPEG", &[0; 4])), 4096, 16);
|
||||
let at = valid.windows(4).position(|bytes| bytes == b"stsz").unwrap();
|
||||
for (constant, count, accepted) in [(0u32, 1u32, true), (16, 1, true), (0, 2, false), (0, u32::MAX, false), (16, u32::MAX, false)] {
|
||||
let mut track = valid.clone();
|
||||
track[at + 8..at + 12].copy_from_slice(&constant.to_be_bytes());
|
||||
track[at + 12..at + 16].copy_from_slice(&count.to_be_bytes());
|
||||
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
|
||||
file.extend(bx(b"moov", &track));
|
||||
file.resize(4112, 0);
|
||||
let parsed = parse_cr3(&file).unwrap();
|
||||
assert_eq!(parsed.tracks[0].data, accepted.then_some((4096, 16)), "size {constant}, count {count}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hevc_preview_is_distinct_from_an_equal_size_raw_track() {
|
||||
let preview = trak(craw(6000, 4000, &bx(b"HEVC", &[0; 4])), 4096, 16);
|
||||
@@ -714,7 +862,10 @@ mod tests {
|
||||
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
|
||||
file.extend(bx(b"moov", &bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &stbl))))));
|
||||
file.resize(512, 0);
|
||||
assert!(parse_cr3(&file).unwrap().tracks.is_empty());
|
||||
let parsed = parse_cr3(&file).unwrap();
|
||||
assert_eq!(parsed.tracks.len(), 1, "a truncated table must not hide the identifiable raw track");
|
||||
assert!(matches!(parsed.tracks[0].kind, Cr3TrackKind::Raw { width: 16, height: 16, .. }));
|
||||
assert!(parsed.tracks[0].data.is_none(), "the empty stsz must not borrow size bytes from the following co64 box");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+15
-1
@@ -1,6 +1,6 @@
|
||||
# craft-midi
|
||||
|
||||
MIDI file formats. Bytes in, a neutral model out, and back. No file access: callers pass bytes,
|
||||
MIDI file formats and synchronization wire messages. Bytes in, a neutral model out, and back. No file access: callers pass bytes,
|
||||
so the crate builds for wasm32.
|
||||
|
||||
| Format | Read | Write |
|
||||
@@ -79,3 +79,17 @@ of two, more than 65535 tracks or a gap beyond the 28-bit delta limit is an erro
|
||||
Written from the public Standard MIDI Files 1.0 specification (the MIDI Association). No other
|
||||
product's code, assets or strings were used. The code comes from SoundCraft's `soundcraft-midi`
|
||||
crate; the tests build every file they read in code, so the crate ships no binary test data.
|
||||
|
||||
## Synchronization wire messages
|
||||
|
||||
`sync` provides MIDI Clock/Start/Continue/Stop and Song Position Pointer parsing,
|
||||
clamped song-position encoding, and full-frame/quarter-frame MIDI Time Code serialization.
|
||||
It is allocation-free and contains no port access, clocks, tempo estimation or transport scheduling.
|
||||
`SyncMessage::parse` accepts a status-first prefix and ignores trailing bytes, matching LoopCraft;
|
||||
it is not a byte-stream parser. Time code supports only 24/25/30 non-drop fps, not 29.97 drop-frame.
|
||||
|
||||
These wire helpers are extracted from LoopCraft's `crates/midi/src/sync.rs` at
|
||||
[`18c68c6e`](https://github.com/storytold/loopcraft/blob/18c68c6e18b13d90db3001dd9f95462cdc4b05a8/crates/midi/src/sync.rs).
|
||||
The source history credits bflatastic. Transport state and scheduling remain app-owned.
|
||||
Tests cover known wire vectors, truncated/invalid input, all song-position input values and
|
||||
legacy conversion behavior. No application consumer cutover is claimed by this library addition.
|
||||
@@ -12,6 +12,7 @@
|
||||
|
||||
mod sequence;
|
||||
mod smf;
|
||||
pub mod sync;
|
||||
|
||||
pub use sequence::{CtrlEvent, CtrlKind, Note, Sequence};
|
||||
pub use smf::{LenientRead, MAX_EVENTS, MAX_TRACKS, MAX_WARNINGS, Smf, SmfTrack, read_smf, read_smf_lenient, write_smf};
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
//! MIDI synchronization wire messages, separate from transport scheduling.
|
||||
//!
|
||||
//! Extracted from LoopCraft `crates/midi/src/sync.rs` at
|
||||
//! `18c68c6e18b13d90db3001dd9f95462cdc4b05a8` (source history credits bflatastic).
|
||||
//! This module preserves its prefix-parser, clamped song-position and non-drop time-code
|
||||
//! behavior. It does not parse a byte stream or schedule clocks; hosts retain those policies.
|
||||
//!
|
||||
//! Only 24, 25 and 30 non-drop frames per second are represented, not 29.97 drop-frame.
|
||||
|
||||
/// MIDI clock: 24 to the quarter note.
|
||||
pub const CLOCKS_PER_QUARTER: u32 = 24;
|
||||
pub const CLOCK: u8 = 0xF8;
|
||||
pub const START: u8 = 0xFA;
|
||||
pub const CONTINUE: u8 = 0xFB;
|
||||
pub const STOP: u8 = 0xFC;
|
||||
pub const SONG_POSITION: u8 = 0xF2;
|
||||
pub const QUARTER_FRAME: u8 = 0xF1;
|
||||
/// The furthest Song Position Pointer reaches, in sixteenth notes (fourteen bits).
|
||||
pub const MAX_POSITION: u16 = 0x3FFF;
|
||||
|
||||
/// What a device that leads the time sends.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum SyncMessage {
|
||||
Clock,
|
||||
Start,
|
||||
Continue,
|
||||
Stop,
|
||||
/// Song Position Pointer: sixteenth notes from the start of the song.
|
||||
Position(u16),
|
||||
}
|
||||
|
||||
impl SyncMessage {
|
||||
/// Parse a status-first message prefix; trailing bytes are ignored for compatibility.
|
||||
/// Returns `None` for an unknown status, truncated song position, or non-seven-bit data.
|
||||
/// This is not a stream parser and does not remove interleaved real-time messages.
|
||||
pub fn parse(message: &[u8]) -> Option<SyncMessage> {
|
||||
Some(match *message {
|
||||
[CLOCK, ..] => SyncMessage::Clock,
|
||||
[START, ..] => SyncMessage::Start,
|
||||
[CONTINUE, ..] => SyncMessage::Continue,
|
||||
[STOP, ..] => SyncMessage::Stop,
|
||||
[SONG_POSITION, low, high, ..] if low < 0x80 && high < 0x80 => SyncMessage::Position(u16::from(high) << 7 | u16::from(low)),
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Song Position Pointer at `sixteenths` from the start, the low seven bits first.
|
||||
pub fn song_position(sixteenths: u16) -> [u8; 3] {
|
||||
let at = sixteenths.min(MAX_POSITION);
|
||||
[SONG_POSITION, (at & 0x7F) as u8, (at >> 7) as u8]
|
||||
}
|
||||
|
||||
/// The frame rates time code is sent at (non-drop).
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum Rate {
|
||||
Fps24,
|
||||
#[default]
|
||||
Fps25,
|
||||
Fps30,
|
||||
}
|
||||
|
||||
impl Rate {
|
||||
/// 24 and 30 frames a second are themselves; anything else is 25.
|
||||
pub fn of(fps: u8) -> Rate {
|
||||
match fps {
|
||||
24 => Rate::Fps24,
|
||||
30 => Rate::Fps30,
|
||||
_ => Rate::Fps25,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn fps(self) -> u8 {
|
||||
match self {
|
||||
Rate::Fps24 => 24,
|
||||
Rate::Fps25 => 25,
|
||||
Rate::Fps30 => 30,
|
||||
}
|
||||
}
|
||||
|
||||
/// As the hours byte carries it: 0 for 24, 1 for 25, 3 for 30 non-drop.
|
||||
fn code(self) -> u8 {
|
||||
match self {
|
||||
Rate::Fps24 => 0,
|
||||
Rate::Fps25 => 1,
|
||||
Rate::Fps30 => 3,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Hours, minutes, seconds and frames at `secs`; the hours wrap at a day.
|
||||
fn timecode(secs: f64, rate: Rate) -> [u8; 4] {
|
||||
let fps = u64::from(rate.fps());
|
||||
// `as` saturates: before the start, and NaN, is the start. The nudge keeps a frame's own
|
||||
// start in it when the sum lands a hair short.
|
||||
let frames = (secs * fps as f64 + 1e-6) as u64;
|
||||
let s = frames / fps;
|
||||
[(s / 3600 % 24) as u8, (s / 60 % 60) as u8, (s % 60) as u8, (frames % fps) as u8]
|
||||
}
|
||||
|
||||
/// The full-frame message (universal real-time SysEx, all devices) locating to `secs`.
|
||||
///
|
||||
/// Compatibility conversion: negative values and NaN locate to zero; positive overflow
|
||||
/// saturates the frame count at `u64::MAX`. Hours wrap at 24. No drop-frame conversion.
|
||||
pub fn full_frame(secs: f64, rate: Rate) -> [u8; 10] {
|
||||
let [h, m, s, f] = timecode(secs, rate);
|
||||
[0xF0, 0x7F, 0x7F, 0x01, 0x01, rate.code() << 5 | h, m, s, f, 0xF7]
|
||||
}
|
||||
|
||||
/// Quarter frame `piece` (0..8) of `time` (hours, minutes, seconds, frames): eight of them, a
|
||||
/// quarter of a frame apart, carry the whole time, the frames' low nibble first.
|
||||
fn quarter_frame(time: [u8; 4], rate: Rate, piece: u8) -> [u8; 2] {
|
||||
let [h, m, s, f] = time;
|
||||
let piece = piece & 7;
|
||||
let nibble = match piece {
|
||||
0 => f,
|
||||
1 => f >> 4,
|
||||
2 => s,
|
||||
3 => s >> 4,
|
||||
4 => m,
|
||||
5 => m >> 4,
|
||||
6 => h,
|
||||
_ => h >> 4 | rate.code() << 1,
|
||||
};
|
||||
[QUARTER_FRAME, piece << 4 | nibble & 0x0F]
|
||||
}
|
||||
|
||||
/// All eight quarter-frame wire messages for a single time-code snapshot, in wire order.
|
||||
///
|
||||
/// This is serialization only: the host must send each piece at the appropriate time and
|
||||
/// account for transport direction/latency. Seconds use the same conversion as [`full_frame`].
|
||||
pub fn quarter_frames(secs: f64, rate: Rate) -> [[u8; 2]; 8] {
|
||||
let time = timecode(secs, rate);
|
||||
[0, 1, 2, 3, 4, 5, 6, 7].map(|piece| quarter_frame(time, rate, piece))
|
||||
}
|
||||
@@ -3,3 +3,4 @@
|
||||
mod lenient;
|
||||
mod sequence;
|
||||
mod smf;
|
||||
mod sync;
|
||||
@@ -0,0 +1,54 @@
|
||||
use craft_midi::sync::{CLOCK, CONTINUE, MAX_POSITION, Rate, SONG_POSITION, START, STOP, SyncMessage, full_frame, quarter_frames, song_position};
|
||||
|
||||
#[test]
|
||||
fn transport_prefix_compatibility() {
|
||||
for (status, expected) in [(CLOCK, SyncMessage::Clock), (START, SyncMessage::Start), (CONTINUE, SyncMessage::Continue), (STOP, SyncMessage::Stop)]
|
||||
{
|
||||
assert_eq!(SyncMessage::parse(&[status]), Some(expected));
|
||||
assert_eq!(SyncMessage::parse(&[status, 0, 0xff]), Some(expected));
|
||||
}
|
||||
for invalid in [&[][..], &[0x90, 60, 127], &[SONG_POSITION], &[SONG_POSITION, 0], &[SONG_POSITION, 0x80, 0], &[SONG_POSITION, 0, 0x80]] {
|
||||
assert_eq!(SyncMessage::parse(invalid), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn song_position_preserves_clamping_and_seven_bit_order() {
|
||||
for value in 0..=u16::MAX {
|
||||
let bytes = song_position(value);
|
||||
assert!(bytes[1] < 128 && bytes[2] < 128);
|
||||
assert_eq!(SyncMessage::parse(&bytes), Some(SyncMessage::Position(value.min(MAX_POSITION))));
|
||||
}
|
||||
assert_eq!(song_position(128), [0xf2, 0, 1]);
|
||||
assert_eq!(SyncMessage::parse(&[0xf2, 1, 2, 0xff]), Some(SyncMessage::Position(257)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_frame_known_vectors_and_legacy_conversion() {
|
||||
for (rate, bits) in [(Rate::Fps24, 0), (Rate::Fps25, 0x20), (Rate::Fps30, 0x60)] {
|
||||
assert_eq!(full_frame(3723.0, rate), [0xf0, 0x7f, 0x7f, 1, 1, bits | 1, 2, 3, 0, 0xf7]);
|
||||
assert_eq!(full_frame(86400.0, rate), full_frame(0.0, rate));
|
||||
assert_eq!(full_frame(-1.0, rate), full_frame(0.0, rate));
|
||||
assert_eq!(full_frame(f64::NAN, rate), full_frame(0.0, rate));
|
||||
assert_eq!(full_frame(1.0 / f64::from(rate.fps()), rate)[8], 1);
|
||||
}
|
||||
assert_eq!(Rate::of(24), Rate::Fps24);
|
||||
assert_eq!(Rate::of(30), Rate::Fps30);
|
||||
assert_eq!(Rate::of(29), Rate::Fps25);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quarter_frames_known_vector() {
|
||||
assert_eq!(
|
||||
quarter_frames(3723.04, Rate::Fps25),
|
||||
[[0xf1, 1], [0xf1, 0x10], [0xf1, 0x23], [0xf1, 0x30], [0xf1, 0x42], [0xf1, 0x50], [0xf1, 0x61], [0xf1, 0x72]]
|
||||
);
|
||||
for rate in [Rate::Fps24, Rate::Fps25, Rate::Fps30] {
|
||||
for secs in [0.0, 59.99, 3599.99, 86399.99, f64::INFINITY, f64::NEG_INFINITY, f64::NAN] {
|
||||
for bytes in quarter_frames(secs, rate) {
|
||||
assert_eq!(bytes[0], 0xf1);
|
||||
assert!(bytes[1] < 128);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -370,7 +370,8 @@ impl DirectoryBundleWriter {
|
||||
/// Saves an entry set incrementally. Existing objects are reused only after a hash check;
|
||||
/// unchanged file signatures may skip checks until they enter the rolling verification
|
||||
/// window. The manifest is atomically published last, then stale previews and objects are
|
||||
/// collected while the bundle lock is held.
|
||||
/// collected while the bundle lock is held. Collection is best-effort after publication:
|
||||
/// inaccessible or unsafe leftovers cannot turn a committed save into a reported failure.
|
||||
pub fn save_directory(&mut self, root: &Path, bundle: ZipBundleWriter) -> Result<DirectoryStats> {
|
||||
if !bundle.entries.contains_key(craft_pcraft_format::MANIFEST) {
|
||||
return Err(Error::InvalidBundle("manifest not set".into()));
|
||||
@@ -401,6 +402,10 @@ impl DirectoryBundleWriter {
|
||||
Err(std::fs::TryLockError::Error(_)) => break None,
|
||||
}
|
||||
};
|
||||
// A known unsafe manifest cannot be published. Refuse it before writing objects or
|
||||
// previews, and still recheck immediately before publication below.
|
||||
let manifest_path = root.join(craft_pcraft_format::MANIFEST);
|
||||
refuse_symlink(&manifest_path)?;
|
||||
self.generation = self.generation.saturating_add(1);
|
||||
let generation = self.generation;
|
||||
let recheck = self.rolling_recheck(&bundle.entries);
|
||||
@@ -449,43 +454,33 @@ impl DirectoryBundleWriter {
|
||||
}
|
||||
|
||||
let manifest = bundle.entries.get(craft_pcraft_format::MANIFEST).ok_or_else(|| Error::InvalidBundle("manifest not set".into()))?;
|
||||
let manifest_path = root.join(craft_pcraft_format::MANIFEST);
|
||||
refuse_symlink(&manifest_path)?;
|
||||
craft_atomic::atomic_write(&manifest_path, manifest).map_err(io_error)?;
|
||||
stats.written = stats.written.saturating_add(1);
|
||||
|
||||
let mut removed_paths = Vec::new();
|
||||
// The durable manifest is the commit boundary. Cleanup owns only canonical regular
|
||||
// object files and previews; leave unsafe, inaccessible or locked leftovers for later.
|
||||
for kind in [craft_pcraft_format::TILES_DIR, craft_pcraft_format::BLOBS_DIR] {
|
||||
let dir = root.join(kind);
|
||||
if !dir.exists() {
|
||||
if !cleanup_directory(&dir) {
|
||||
continue;
|
||||
}
|
||||
refuse_symlink(&dir)?;
|
||||
for entry in fs::read_dir(&dir).map_err(io_error)? {
|
||||
let entry = entry.map_err(io_error)?;
|
||||
let Ok(entries) = fs::read_dir(&dir) else { continue };
|
||||
for entry in entries {
|
||||
let Ok(entry) = entry else { continue };
|
||||
let name = entry.file_name();
|
||||
let path = format!("{kind}/{}", name.to_string_lossy());
|
||||
if is_object_path(&path) && !bundle.entries.contains_key(&path) {
|
||||
let metadata = fs::symlink_metadata(entry.path()).map_err(io_error)?;
|
||||
if metadata.file_type().is_symlink() || !metadata.is_file() {
|
||||
return Err(Error::InvalidBundle(format!("refusing to collect non-file object {path}")));
|
||||
}
|
||||
fs::remove_file(entry.path()).map_err(io_error)?;
|
||||
removed_paths.push(path);
|
||||
if is_object_path(&path) && !bundle.entries.contains_key(&path) && remove_cleanup_file(&entry.path()) {
|
||||
stats.removed = stats.removed.saturating_add(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
for path in [craft_pcraft_format::THUMB, craft_pcraft_format::COMPOSITE, craft_pcraft_format::FULL_COMPOSITE] {
|
||||
if !bundle.entries.contains_key(path) {
|
||||
let target = root.join(path);
|
||||
refuse_symlink(&target)?;
|
||||
if fs::symlink_metadata(&target).is_ok() {
|
||||
fs::remove_file(&target).map_err(io_error)?;
|
||||
}
|
||||
let _ = remove_cleanup_file(&root.join(path));
|
||||
}
|
||||
}
|
||||
self.verified.retain(|path, _| bundle.entries.contains_key(path) && !removed_paths.contains(path));
|
||||
self.verified.retain(|path, _| bundle.entries.contains_key(path));
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
@@ -510,7 +505,32 @@ impl DirectoryBundleWriter {
|
||||
}
|
||||
|
||||
fn is_object_path(path: &str) -> bool {
|
||||
path.starts_with("tiles/") || path.starts_with("blobs/")
|
||||
valid_object_path(path)
|
||||
}
|
||||
|
||||
/// Do not traverse symlinked object/preview directories, including Windows junctions.
|
||||
fn cleanup_directory(path: &Path) -> bool {
|
||||
fs::symlink_metadata(path).is_ok_and(|metadata| metadata.is_dir() && !cleanup_link(&metadata))
|
||||
}
|
||||
|
||||
fn cleanup_link(metadata: &fs::Metadata) -> bool {
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::fs::MetadataExt;
|
||||
metadata.file_attributes() & 0x400 != 0 // FILE_ATTRIBUTE_REPARSE_POINT includes junctions.
|
||||
}
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
metadata.file_type().is_symlink()
|
||||
}
|
||||
}
|
||||
|
||||
/// Cleanup never follows or removes a directory/link and never reports a post-commit failure.
|
||||
fn remove_cleanup_file(path: &Path) -> bool {
|
||||
if path.parent().is_none_or(|parent| !cleanup_directory(parent)) {
|
||||
return false;
|
||||
}
|
||||
fs::symlink_metadata(path).is_ok_and(|metadata| metadata.is_file() && !cleanup_link(&metadata)) && fs::remove_file(path).is_ok()
|
||||
}
|
||||
|
||||
fn file_signature(path: &Path) -> Result<Option<FileSignature>> {
|
||||
|
||||
@@ -0,0 +1,344 @@
|
||||
//! Publication succeeds independently of best-effort cleanup of the previous entry set.
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
|
||||
use craft_pcraft_bundle::{DirectoryBundle, DirectoryBundleWriter, Limits, ObjectKind, ZipBundleWriter};
|
||||
use craft_pcraft_format::{COMPOSITE, FULL_COMPOSITE, MANIFEST, THUMB};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
const OLD: &[u8] = br#"{"revision":1}"#;
|
||||
const NEW: &[u8] = br#"{"revision":2}"#;
|
||||
|
||||
struct Temporary(PathBuf);
|
||||
|
||||
impl Temporary {
|
||||
fn new() -> Self {
|
||||
static NEXT: AtomicU64 = AtomicU64::new(0);
|
||||
let path = std::env::temp_dir().join(format!("craft-bundle-cleanup-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed)));
|
||||
fs::create_dir(&path).unwrap();
|
||||
Self(path)
|
||||
}
|
||||
|
||||
fn bundle(&self) -> PathBuf {
|
||||
self.0.join("document.pcraft")
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Temporary {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
fn entries(manifest: &[u8]) -> ZipBundleWriter {
|
||||
let mut bundle = ZipBundleWriter::default();
|
||||
bundle.set_manifest(manifest.to_vec()).unwrap();
|
||||
bundle
|
||||
}
|
||||
|
||||
fn assert_manifest(root: &Path, expected: &[u8]) {
|
||||
assert_eq!(fs::read(root.join(MANIFEST)).unwrap(), expected);
|
||||
assert_eq!(DirectoryBundle::open(root, Limits::default()).unwrap().manifest().unwrap(), expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_file_stale_object_does_not_fail_the_committed_save() {
|
||||
let temp = Temporary::new();
|
||||
let root = temp.bundle();
|
||||
let mut writer = DirectoryBundleWriter::default();
|
||||
writer.save_directory(&root, entries(OLD)).unwrap();
|
||||
let stuck = root.join("tiles").join(format!("{}.zst", "0".repeat(64)));
|
||||
fs::create_dir_all(stuck.join("inner")).unwrap();
|
||||
fs::write(stuck.join("inner/keep"), b"not an object file").unwrap();
|
||||
|
||||
let stats = writer.save_directory(&root, entries(NEW)).expect("cleanup cannot fail after manifest publication");
|
||||
assert_manifest(&root, NEW);
|
||||
assert_eq!(stats.removed, 0);
|
||||
assert_eq!(fs::read(stuck.join("inner/keep")).unwrap(), b"not an object file");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cleanup_removes_only_canonical_stale_regular_objects_and_reuses_current_objects() {
|
||||
let temp = Temporary::new();
|
||||
let root = temp.bundle();
|
||||
let mut writer = DirectoryBundleWriter::default();
|
||||
let mut first = entries(OLD);
|
||||
let retained = first.add_object(ObjectKind::Tile, b"still referenced").unwrap();
|
||||
let stale = first.add_object(ObjectKind::Blob, b"old blob").unwrap();
|
||||
for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] {
|
||||
first.set_preview(preview, b"old preview".to_vec()).unwrap();
|
||||
}
|
||||
writer.save_directory(&root, first).unwrap();
|
||||
let retained_bytes = fs::read(root.join(&retained)).unwrap();
|
||||
let unrelated = [
|
||||
"tiles/personal-notes.zst".to_string(),
|
||||
"blobs/012345.zst".to_string(),
|
||||
format!("blobs/{}.zst", "A".repeat(64)),
|
||||
format!("tiles/{}.zst.backup", "b".repeat(64)),
|
||||
];
|
||||
for path in &unrelated {
|
||||
fs::write(root.join(path), b"unrelated").unwrap();
|
||||
}
|
||||
let nested = root.join("blobs/not-an-object-folder");
|
||||
fs::create_dir(&nested).unwrap();
|
||||
fs::write(nested.join(format!("{}.zst", "c".repeat(64))), b"unrelated nested file").unwrap();
|
||||
let mut second = entries(NEW);
|
||||
assert_eq!(second.add_object(ObjectKind::Tile, b"still referenced").unwrap(), retained);
|
||||
let stats = writer.save_directory(&root, second).unwrap();
|
||||
|
||||
assert_manifest(&root, NEW);
|
||||
assert_eq!(stats.removed, 1);
|
||||
assert_eq!(stats.reused, 1);
|
||||
assert_eq!(stats.reused_objects.as_slice(), std::slice::from_ref(&retained));
|
||||
assert!(stats.written_objects.is_empty());
|
||||
assert!(!root.join(stale).exists());
|
||||
assert_eq!(fs::read(root.join(retained)).unwrap(), retained_bytes);
|
||||
for path in unrelated {
|
||||
assert_eq!(fs::read(root.join(path)).unwrap(), b"unrelated");
|
||||
}
|
||||
assert!(nested.is_dir());
|
||||
for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] {
|
||||
assert!(!root.join(preview).exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_preview_directories_are_preserved_after_commit() {
|
||||
let temp = Temporary::new();
|
||||
let root = temp.bundle();
|
||||
let mut writer = DirectoryBundleWriter::default();
|
||||
writer.save_directory(&root, entries(OLD)).unwrap();
|
||||
for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] {
|
||||
fs::create_dir_all(root.join(preview)).unwrap();
|
||||
fs::write(root.join(preview).join("keep"), b"directory contents").unwrap();
|
||||
}
|
||||
let stats = writer.save_directory(&root, entries(NEW)).unwrap();
|
||||
assert_manifest(&root, NEW);
|
||||
assert_eq!(stats.removed, 0);
|
||||
for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] {
|
||||
assert_eq!(fs::read(root.join(preview).join("keep")).unwrap(), b"directory contents");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn required_non_file_objects_and_previews_fail_before_manifest_publication() {
|
||||
let temp = Temporary::new();
|
||||
let root = temp.bundle();
|
||||
let mut writer = DirectoryBundleWriter::default();
|
||||
writer.save_directory(&root, entries(OLD)).unwrap();
|
||||
let mut changed = entries(NEW);
|
||||
let object = changed.add_object(ObjectKind::Tile, b"required object").unwrap();
|
||||
fs::create_dir_all(root.join(&object)).unwrap();
|
||||
assert!(writer.save_directory(&root, changed).is_err());
|
||||
assert_manifest(&root, OLD);
|
||||
|
||||
for preview in [THUMB, COMPOSITE] {
|
||||
fs::create_dir_all(root.join(preview)).unwrap();
|
||||
let mut changed = entries(NEW);
|
||||
changed.set_preview(preview, b"required preview".to_vec()).unwrap();
|
||||
assert!(writer.save_directory(&root, changed).is_err());
|
||||
assert_manifest(&root, OLD);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn locked_stale_object_and_preview_do_not_fail_commit_and_are_collected_on_retry() {
|
||||
use std::fs::OpenOptions;
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
let temp = Temporary::new();
|
||||
let root = temp.bundle();
|
||||
let mut writer = DirectoryBundleWriter::default();
|
||||
let mut first = entries(OLD);
|
||||
let stale = first.add_object(ObjectKind::Tile, b"locked stale tile").unwrap();
|
||||
first.set_preview(FULL_COMPOSITE, b"locked preview".to_vec()).unwrap();
|
||||
writer.save_directory(&root, first).unwrap();
|
||||
let tile_lock = OpenOptions::new().read(true).share_mode(0).open(root.join(&stale)).unwrap();
|
||||
let preview_lock = OpenOptions::new().read(true).share_mode(0).open(root.join(FULL_COMPOSITE)).unwrap();
|
||||
let stats = writer.save_directory(&root, entries(NEW)).unwrap();
|
||||
assert_manifest(&root, NEW);
|
||||
assert_eq!(stats.removed, 0);
|
||||
assert!(root.join(&stale).exists());
|
||||
assert!(root.join(FULL_COMPOSITE).exists());
|
||||
drop(tile_lock);
|
||||
drop(preview_lock);
|
||||
let retry = writer.save_directory(&root, entries(NEW)).unwrap();
|
||||
assert_manifest(&root, NEW);
|
||||
assert_eq!(retry.removed, 1);
|
||||
assert!(!root.join(stale).exists());
|
||||
assert!(!root.join(FULL_COMPOSITE).exists());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn cleanup_preserves_symlinks_and_does_not_traverse_linked_directories() {
|
||||
use std::os::unix::fs::symlink;
|
||||
let temp = Temporary::new();
|
||||
let root = temp.bundle();
|
||||
let outside = temp.0.join("outside");
|
||||
fs::create_dir(&outside).unwrap();
|
||||
let object_name = format!("{}.zst", "d".repeat(64));
|
||||
fs::write(outside.join(&object_name), b"outside object").unwrap();
|
||||
fs::write(outside.join("preview.png"), b"outside preview").unwrap();
|
||||
let mut writer = DirectoryBundleWriter::default();
|
||||
writer.save_directory(&root, entries(OLD)).unwrap();
|
||||
symlink(&outside, root.join("blobs")).unwrap();
|
||||
symlink(&outside, root.join("composite")).unwrap();
|
||||
symlink(outside.join("preview.png"), root.join(THUMB)).unwrap();
|
||||
fs::create_dir(root.join("tiles")).unwrap();
|
||||
let linked_object = root.join("tiles").join(&object_name);
|
||||
symlink(outside.join(&object_name), &linked_object).unwrap();
|
||||
let stats = writer.save_directory(&root, entries(NEW)).unwrap();
|
||||
assert_manifest(&root, NEW);
|
||||
assert_eq!(stats.removed, 0);
|
||||
assert_eq!(fs::read(outside.join(object_name)).unwrap(), b"outside object");
|
||||
assert_eq!(fs::read(outside.join("preview.png")).unwrap(), b"outside preview");
|
||||
for link in [root.join("blobs"), root.join("composite"), root.join(THUMB), linked_object] {
|
||||
assert!(fs::symlink_metadata(link).unwrap().file_type().is_symlink());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn required_symlink_objects_and_previews_still_fail_before_commit() {
|
||||
use std::os::unix::fs::symlink;
|
||||
let temp = Temporary::new();
|
||||
let root = temp.bundle();
|
||||
let outside = temp.0.join("outside-file");
|
||||
fs::write(&outside, b"unchanged outside bytes").unwrap();
|
||||
let mut writer = DirectoryBundleWriter::default();
|
||||
writer.save_directory(&root, entries(OLD)).unwrap();
|
||||
let mut changed = entries(NEW);
|
||||
let object = changed.add_object(ObjectKind::Tile, b"required object").unwrap();
|
||||
fs::create_dir(root.join("tiles")).unwrap();
|
||||
symlink(&outside, root.join(object)).unwrap();
|
||||
assert!(writer.save_directory(&root, changed).is_err());
|
||||
assert_manifest(&root, OLD);
|
||||
symlink(&outside, root.join(THUMB)).unwrap();
|
||||
let mut changed = entries(NEW);
|
||||
changed.set_preview(THUMB, b"new preview".to_vec()).unwrap();
|
||||
assert!(writer.save_directory(&root, changed).is_err());
|
||||
assert_manifest(&root, OLD);
|
||||
assert_eq!(fs::read(outside).unwrap(), b"unchanged outside bytes");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn cleanup_does_not_traverse_windows_directory_junctions() {
|
||||
use std::os::windows::process::CommandExt;
|
||||
struct Junction(PathBuf);
|
||||
impl Drop for Junction {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_dir(&self.0);
|
||||
}
|
||||
}
|
||||
fn junction(link: PathBuf, target: &Path) -> Junction {
|
||||
// Fixture-owned absolute paths only; junctions require no symlink privilege.
|
||||
assert!(link.is_absolute() && target.is_absolute());
|
||||
let output = std::process::Command::new("cmd")
|
||||
.args(["/C", "mklink", "/J"])
|
||||
.arg(&link)
|
||||
.arg(target)
|
||||
.creation_flags(0x0800_0000) // CREATE_NO_WINDOW
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr));
|
||||
Junction(link)
|
||||
}
|
||||
let temp = Temporary::new();
|
||||
let root = temp.bundle();
|
||||
let outside = temp.0.join("outside");
|
||||
fs::create_dir(&outside).unwrap();
|
||||
let object_name = format!("{}.zst", "e".repeat(64));
|
||||
fs::write(outside.join(&object_name), b"outside object").unwrap();
|
||||
fs::write(outside.join("preview.png"), b"outside preview").unwrap();
|
||||
fs::write(outside.join("full.png"), b"outside full preview").unwrap();
|
||||
let mut writer = DirectoryBundleWriter::default();
|
||||
writer.save_directory(&root, entries(OLD)).unwrap();
|
||||
let _objects = junction(root.join("blobs"), &outside);
|
||||
let _previews = junction(root.join("composite"), &outside);
|
||||
let stats = writer.save_directory(&root, entries(NEW)).unwrap();
|
||||
assert_manifest(&root, NEW);
|
||||
assert_eq!(stats.removed, 0);
|
||||
assert_eq!(fs::read(outside.join(object_name)).unwrap(), b"outside object");
|
||||
assert_eq!(fs::read(outside.join("preview.png")).unwrap(), b"outside preview");
|
||||
assert_eq!(fs::read(outside.join("full.png")).unwrap(), b"outside full preview");
|
||||
assert!(root.join("blobs").exists());
|
||||
assert!(root.join("composite").exists());
|
||||
}
|
||||
|
||||
#[cfg(any(unix, windows))]
|
||||
#[test]
|
||||
fn manifest_symlink_is_refused_before_writing_any_objects_or_previews() {
|
||||
let temp = Temporary::new();
|
||||
let root = temp.bundle();
|
||||
let mut writer = DirectoryBundleWriter::default();
|
||||
writer.save_directory(&root, entries(OLD)).unwrap();
|
||||
assert_manifest(&root, OLD); // Ordinary publication remains supported.
|
||||
let external = temp.0.join("external-manifest.json");
|
||||
fs::write(&external, b"unchanged external manifest").unwrap();
|
||||
fs::remove_file(root.join(MANIFEST)).unwrap();
|
||||
#[cfg(unix)]
|
||||
std::os::unix::fs::symlink(&external, root.join(MANIFEST)).unwrap();
|
||||
#[cfg(windows)]
|
||||
if let Err(error) = std::os::windows::fs::symlink_file(&external, root.join(MANIFEST)) {
|
||||
if error.raw_os_error() == Some(1314) || error.kind() == std::io::ErrorKind::Unsupported {
|
||||
eprintln!("SKIP manifest symlink fixture: {error}");
|
||||
return;
|
||||
}
|
||||
panic!("create manifest symlink fixture: {error}");
|
||||
}
|
||||
let mut changed = entries(NEW);
|
||||
changed.add_object(ObjectKind::Tile, b"must not write tile").unwrap();
|
||||
changed.add_object(ObjectKind::Blob, b"must not write blob").unwrap();
|
||||
changed.set_preview(THUMB, b"must not write thumbnail".to_vec()).unwrap();
|
||||
changed.set_preview(FULL_COMPOSITE, b"must not write composite".to_vec()).unwrap();
|
||||
assert!(writer.save_directory(&root, changed).is_err());
|
||||
assert_eq!(fs::read(&external).unwrap(), b"unchanged external manifest");
|
||||
assert!(fs::symlink_metadata(root.join(MANIFEST)).unwrap().file_type().is_symlink());
|
||||
for unwritten in ["tiles", "blobs", THUMB, "composite"] {
|
||||
assert!(!root.join(unwritten).exists(), "manifest refusal must precede {unwritten} publication");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn chosen_root_symlink_cleanup_works_with_search_only_root_and_composite_permissions() {
|
||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
struct Restore(PathBuf, fs::Permissions);
|
||||
impl Restore {
|
||||
fn restrict(path: &Path) -> Self {
|
||||
let old = fs::metadata(path).unwrap().permissions();
|
||||
fs::set_permissions(path, fs::Permissions::from_mode(0o300)).unwrap();
|
||||
Self(path.to_path_buf(), old)
|
||||
}
|
||||
}
|
||||
impl Drop for Restore {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::set_permissions(&self.0, self.1.clone());
|
||||
}
|
||||
}
|
||||
let temp = Temporary::new();
|
||||
let root = temp.bundle();
|
||||
let mut writer = DirectoryBundleWriter::default();
|
||||
let mut first = entries(OLD);
|
||||
let stale = first.add_object(ObjectKind::Blob, b"stale blob").unwrap();
|
||||
first.set_preview(THUMB, b"stale thumbnail".to_vec()).unwrap();
|
||||
first.set_preview(COMPOSITE, b"stale preview".to_vec()).unwrap();
|
||||
writer.save_directory(&root, first).unwrap();
|
||||
let chosen = temp.0.join("chosen.pcraft");
|
||||
symlink(&root, &chosen).unwrap();
|
||||
let composite_mode = Restore::restrict(&root.join("composite"));
|
||||
let root_mode = Restore::restrict(&root);
|
||||
let result = writer.save_directory(&chosen, entries(NEW));
|
||||
drop(root_mode);
|
||||
drop(composite_mode);
|
||||
let stats = result.expect("chosen root resolves before cleanup; traversal does not require directory read permission");
|
||||
assert_manifest(&root, NEW);
|
||||
assert_eq!(stats.removed, 1);
|
||||
assert!(!root.join(stale).exists());
|
||||
assert!(!root.join(THUMB).exists());
|
||||
assert!(!root.join(COMPOSITE).exists());
|
||||
assert!(fs::symlink_metadata(chosen).unwrap().file_type().is_symlink());
|
||||
}
|
||||
@@ -35,8 +35,9 @@ workspace = true
|
||||
# ticket (winprint.rs#12, storytold/pdfcraft#3823), and reads whitespace-padded integers in
|
||||
# PrintCapabilities (winprint.rs#3, storytold/pdfcraft#2430). Pinned to an immutable revision;
|
||||
# review before bumping. The settings chosen in a driver's own dialog become a ticket through
|
||||
# `PrintTicket::from_dev_mode`.
|
||||
winprint = { git = "https://github.com/storytold/winprint.rs", rev = "72951d7455ebcab569fbd9386c513d3d9f53fb02", default-features = false }
|
||||
# `PrintTicket::from_dev_mode` (winprint.rs#24, pinned at its merge commit on winprint.rs's
|
||||
# `main`).
|
||||
winprint = { git = "https://github.com/storytold/winprint.rs", rev = "8ae097ef941b432b0e7671b1ec3a506800e65e57", default-features = false }
|
||||
# GetDefaultPrinter (MIT).
|
||||
winsafe = { version = "0.0.29", default-features = false, features = ["winspool"] }
|
||||
# The job file's private folder.
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "craft-proto"
|
||||
description = "Mail protocols without sockets: IMAP4rev1 and SMTP clients over any Read + Write stream"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
base64 = "0.23"
|
||||
zeroize = "1"
|
||||
# Internationalization: error messages and user-facing text.
|
||||
log = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = { workspace = true }
|
||||
@@ -0,0 +1,246 @@
|
||||
// SPDX-FileCopyrightText: Craft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
|
||||
//! Error message handling for protocol errors.
|
||||
//!
|
||||
//! Provides the `said!` macro and related types for creating user-facing error messages.
|
||||
|
||||
pub use crate::said;
|
||||
use std::fmt;
|
||||
|
||||
/// A value that can be converted to an argument for error messages.
|
||||
pub trait ToArg {
|
||||
fn to_arg(&self) -> Arg;
|
||||
}
|
||||
|
||||
impl<T: fmt::Display> ToArg for T {
|
||||
fn to_arg(&self) -> Arg {
|
||||
Arg::Plain(format!("{}", self))
|
||||
}
|
||||
}
|
||||
|
||||
/// An argument for error messages.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Arg {
|
||||
Plain(String),
|
||||
}
|
||||
|
||||
impl fmt::Display for Arg {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Arg::Plain(s) => f.write_str(s),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Arg {
|
||||
/// Create an argument from a string.
|
||||
pub fn said(t: Text) -> Self {
|
||||
Arg::Plain(format!("{}", t))
|
||||
}
|
||||
}
|
||||
|
||||
/// A text with values: `said!("no folder named {name}", name)`, or
|
||||
/// `said!("{name} is larger than {mb} MB", name, mb = MAX >> 20)`.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Text {
|
||||
template: String,
|
||||
}
|
||||
|
||||
impl Text {
|
||||
/// Preserve text supplied by a server or host without interpreting it.
|
||||
pub fn verbatim(text: String) -> Self {
|
||||
Self { template: text }
|
||||
}
|
||||
|
||||
/// Create a new text with the given template and arguments.
|
||||
pub fn new(template: &str, args: Vec<(&str, Arg)>) -> Self {
|
||||
// Substitute only placeholders in the template, never text inserted by a server.
|
||||
let mut filled = String::new();
|
||||
let mut rest = template;
|
||||
while let Some(open) = rest.find('{') {
|
||||
filled.push_str(rest.get(..open).unwrap_or_default());
|
||||
let after = rest.get(open.saturating_add(1)..).unwrap_or_default();
|
||||
let Some(close) = after.find('}') else {
|
||||
filled.push_str(rest.get(open..).unwrap_or_default());
|
||||
rest = "";
|
||||
break;
|
||||
};
|
||||
let key = after.get(..close).unwrap_or_default();
|
||||
if let Some((_, arg)) = args.iter().find(|(name, _)| *name == key) {
|
||||
filled.push_str(&arg.to_string());
|
||||
} else {
|
||||
filled.push('{');
|
||||
filled.push_str(key);
|
||||
filled.push('}');
|
||||
}
|
||||
rest = after.get(close.saturating_add(1)..).unwrap_or_default();
|
||||
}
|
||||
filled.push_str(rest);
|
||||
Text { template: filled }
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Text {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.template)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for Text {
|
||||
fn from(s: &str) -> Self {
|
||||
Text { template: s.to_string() }
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for Text {
|
||||
fn from(s: String) -> Self {
|
||||
Self::verbatim(s)
|
||||
}
|
||||
}
|
||||
|
||||
/// The `said!` macro for creating error messages.
|
||||
#[macro_export]
|
||||
macro_rules! said {
|
||||
(@arg $k:ident) => { $crate::i18n::ToArg::to_arg(&$k) };
|
||||
(@arg $k:ident = $v:expr) => { $crate::i18n::ToArg::to_arg(&$v) };
|
||||
($t:literal $(, $k:ident $(= $v:expr)?)* $(,)?) => {
|
||||
$crate::i18n::Text::new($t, vec![$((stringify!($k), $crate::said!(@arg $k $(= $v)?))),*])
|
||||
};
|
||||
}
|
||||
|
||||
/// Trace only recognized protocol structure, never arbitrary reply text or command arguments.
|
||||
/// Uppercase text can be private too; capitalization must not determine what is safe to log.
|
||||
pub fn for_trace(line: &str) -> String {
|
||||
let line = line.trim_end();
|
||||
let mut words = line.split_ascii_whitespace();
|
||||
let first = words.next().unwrap_or_default();
|
||||
let known = |word: &str| {
|
||||
matches!(
|
||||
word,
|
||||
"OK" | "NO"
|
||||
| "BAD"
|
||||
| "BYE"
|
||||
| "PREAUTH"
|
||||
| "CAPABILITY"
|
||||
| "LOGIN"
|
||||
| "LOGOUT"
|
||||
| "AUTHENTICATE"
|
||||
| "SELECT"
|
||||
| "EXAMINE"
|
||||
| "CREATE"
|
||||
| "DELETE"
|
||||
| "RENAME"
|
||||
| "SUBSCRIBE"
|
||||
| "UNSUBSCRIBE"
|
||||
| "LIST"
|
||||
| "LSUB"
|
||||
| "STATUS"
|
||||
| "APPEND"
|
||||
| "CHECK"
|
||||
| "CLOSE"
|
||||
| "EXPUNGE"
|
||||
| "SEARCH"
|
||||
| "FETCH"
|
||||
| "STORE"
|
||||
| "COPY"
|
||||
| "MOVE"
|
||||
| "UID"
|
||||
| "IDLE"
|
||||
| "DONE"
|
||||
| "NOOP"
|
||||
| "EHLO"
|
||||
| "HELO"
|
||||
| "STARTTLS"
|
||||
| "AUTH"
|
||||
| "MAIL"
|
||||
| "RCPT"
|
||||
| "DATA"
|
||||
| "RSET"
|
||||
| "QUIT"
|
||||
| "EXISTS"
|
||||
| "RECENT"
|
||||
| "FLAGS"
|
||||
)
|
||||
};
|
||||
let smtp_code = first.get(..3).is_some_and(|code| code.bytes().all(|b| b.is_ascii_digit()))
|
||||
&& first.as_bytes().first().is_some_and(|b| matches!(b, b'1'..=b'5'));
|
||||
let tag = first.strip_prefix('A').is_some_and(|digits| !digits.is_empty() && digits.len() <= 16 && digits.bytes().all(|b| b.is_ascii_digit()));
|
||||
let end = if smtp_code && (first.len() == 3 || first.get(3..4) == Some("-")) {
|
||||
if first.len() > 3 { 4 } else { 3 }
|
||||
} else if known(first) {
|
||||
first.len()
|
||||
} else if first == "*" || tag {
|
||||
let second = words.next().unwrap_or_default();
|
||||
let last = if first == "*" && !second.is_empty() && second.bytes().all(|b| b.is_ascii_digit()) {
|
||||
words.next().filter(|word| known(word))
|
||||
} else {
|
||||
known(second).then_some(second)
|
||||
};
|
||||
last.and_then(|word| line.find(word)?.checked_add(word.len())).unwrap_or(first.len())
|
||||
} else if first == "+" {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let kept = line.get(..end).unwrap_or_default();
|
||||
let left = line.get(end..).unwrap_or_default().trim_start().len();
|
||||
match (kept.is_empty(), left) {
|
||||
(_, 0) => kept.to_owned(),
|
||||
(true, _) => format!("({left} bytes)"),
|
||||
(false, _) => format!("{kept} … ({left} bytes)"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Server text made fit for an error message: one line, no control characters, not too long.
|
||||
pub fn tidy(text: &str) -> String {
|
||||
const MAX_CHARS: usize = 300;
|
||||
let mut out = String::new();
|
||||
let mut space = false;
|
||||
for (n, c) in text.trim().chars().enumerate() {
|
||||
if n >= MAX_CHARS {
|
||||
out.push('…');
|
||||
break;
|
||||
}
|
||||
if c.is_control() || c.is_whitespace() {
|
||||
space = !out.is_empty();
|
||||
} else {
|
||||
if space {
|
||||
out.push(' ');
|
||||
space = false;
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn inserted_arguments_are_not_interpreted() {
|
||||
let msg = Text::new("{a} {b}", vec![("a", Arg::Plain("{b}".into())), ("b", Arg::Plain("secret".into()))]);
|
||||
assert_eq!(msg.to_string(), "{b} secret");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn simple_message() {
|
||||
let msg = said!("the server closed the connection");
|
||||
assert_eq!(msg.to_string(), "the server closed the connection");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn message_with_args() {
|
||||
let msg = said!("the server sent {n} bytes", n = 123);
|
||||
assert_eq!(msg.to_string(), "the server sent 123 bytes");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tidy_removes_control_chars() {
|
||||
assert_eq!(tidy("hello\x01world"), "hello world");
|
||||
assert_eq!(tidy("hello\nworld"), "hello world");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
//! Gmail's labels of a message (`X-GM-LABELS`, Google's "Gmail IMAP extensions", #1153).
|
||||
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use proptest::prelude::*;
|
||||
|
||||
use super::*;
|
||||
use crate::script::Script;
|
||||
|
||||
fn labels(server: &str) -> Vec<(u32, Option<Vec<String>>)> {
|
||||
let mut s = Script::new(server);
|
||||
let got = Imap::resume(&mut s).uid_fetch_flags_gmail("1:*").unwrap_or_default();
|
||||
got.into_iter().map(|f| (f.uid, f.gmail_labels)).collect()
|
||||
}
|
||||
|
||||
fn owned(labels: &[&str]) -> Option<Vec<String>> {
|
||||
Some(labels.iter().map(|l| l.to_string()).collect())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn labels_come_with_the_flags() {
|
||||
let mut server = String::new();
|
||||
// Atoms, quoted strings, Gmail's own with their backslash, an empty list.
|
||||
server += "* 1 FETCH (UID 1 FLAGS (\\Seen) X-GM-MSGID 11 X-GM-LABELS (\\Inbox \\Sent Important \"Muy Importante\"))\r\n";
|
||||
server += "* 2 FETCH (X-GM-LABELS (foo) UID 2 FLAGS ())\r\n";
|
||||
server += "* 3 FETCH (UID 3 FLAGS () X-GM-LABELS ())\r\n";
|
||||
// A literal, and modified UTF-7 as mailbox names are written.
|
||||
server += "* 4 FETCH (UID 4 FLAGS () X-GM-LABELS ({12}\r\nWork/Clients \"Caf&AOk-\" &AOk-t&AOk-))\r\n";
|
||||
// UTF-8 as it stands, and an ampersand that isn't the start of a shifted run.
|
||||
server += "* 5 FETCH (UID 5 FLAGS () X-GM-LABELS (\"Été\" \"Tom & Jerry\" \\Drafts))\r\n";
|
||||
// A message the server gave none for.
|
||||
server += "* 6 FETCH (UID 6 FLAGS ())\r\n";
|
||||
server += "A0001 OK done\r\n";
|
||||
let mut s = Script::new(server);
|
||||
let got = Imap::resume(&mut s).uid_fetch_flags_gmail("1:*").unwrap();
|
||||
assert_eq!(s.sent(), "A0001 UID FETCH 1:* (UID FLAGS X-GM-MSGID X-GM-LABELS)\r\n");
|
||||
let got: Vec<(u32, Option<Vec<String>>)> = got.into_iter().map(|f| (f.uid, f.gmail_labels)).collect();
|
||||
assert_eq!(
|
||||
got,
|
||||
[
|
||||
(1, owned(&["\\Inbox", "\\Sent", "Important", "Muy Importante"])),
|
||||
(2, owned(&["foo"])),
|
||||
(3, owned(&[])),
|
||||
(4, owned(&["Work/Clients", "Café", "été"])),
|
||||
(5, owned(&["Été", "Tom & Jerry", "\\Drafts"])),
|
||||
(6, None),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_later_word_counts_and_other_servers_are_not_asked() {
|
||||
let got = labels(
|
||||
"* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (a))\r\n* 1 FETCH (UID 1 X-GM-LABELS (b c))\r\n* 1 FETCH (UID 1 FLAGS ())\r\nA0001 OK done\r\n",
|
||||
);
|
||||
assert_eq!(got, [(1, owned(&["b", "c"]))]);
|
||||
// Without asking for them, labels the server gives anyway are not taken.
|
||||
let mut s = Script::new("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (a))\r\nA0001 OK done\r\n");
|
||||
let plain = Imap::resume(&mut s).uid_fetch_flags("1:*").unwrap();
|
||||
assert_eq!(plain.first().and_then(|f| f.gmail_labels.clone()), None);
|
||||
assert_eq!(s.sent(), "A0001 UID FETCH 1:* (UID FLAGS)\r\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_label_lists_are_kept_small() {
|
||||
// A huge list: only so many are kept.
|
||||
let many: String = (0..1000).map(|n| format!(" L{n}")).collect();
|
||||
let got = labels(&format!("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS ({many}))\r\nA0001 OK done\r\n"));
|
||||
assert_eq!(got.first().and_then(|(_, l)| l.as_ref()).map(Vec::len), Some(MAX_GMAIL_LABELS));
|
||||
// A long label is cut, at a character boundary.
|
||||
let long = "é".repeat(400);
|
||||
let got = labels(&format!("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (\"{long}\"))\r\nA0001 OK done\r\n"));
|
||||
let label = got.first().and_then(|(_, l)| l.as_ref()).and_then(|l| l.first()).cloned().unwrap_or_default();
|
||||
assert!(label.len() <= MAX_GMAIL_LABEL_LEN && !label.is_empty() && label.chars().all(|c| c == 'é'));
|
||||
// A NUL in a literal is carried as text for the model to clean; a list inside the list is
|
||||
// skipped; a list that isn't closed is read as far as it goes.
|
||||
let got = labels("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS ({3}\r\na\0b (nested list) ok))\r\nA0001 OK done\r\n");
|
||||
assert_eq!(got, [(1, owned(&["a\0b", "ok"]))]);
|
||||
let got = labels("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (a (b\r\nA0001 OK done\r\n");
|
||||
assert!(got.len() <= 1);
|
||||
// Something other than a list is no labels.
|
||||
let got = labels("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS NIL)\r\nA0001 OK done\r\n");
|
||||
assert_eq!(got, [(1, None)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fetch_keeps_no_more_label_bytes_than_its_budget() {
|
||||
// Labels as literals, which the cap on a command's text doesn't count: each message with as
|
||||
// many as a response may carry, of 256 bytes each. Past 32 MiB the rest come without.
|
||||
let label = "x".repeat(MAX_GMAIL_LABEL_LEN);
|
||||
let list: String = (0..MAX_LITERALS).map(|_| format!(" {{{}}}\r\n{label}", label.len())).collect();
|
||||
let count = MAX_GMAIL_LABEL_BYTES / (MAX_GMAIL_LABEL_LEN * MAX_LITERALS) + 4;
|
||||
let mut server = String::new();
|
||||
for uid in 1..=count {
|
||||
server += &format!("* {uid} FETCH (UID {uid} FLAGS () X-GM-LABELS ({list}))\r\n");
|
||||
}
|
||||
server += "A0001 OK done\r\n";
|
||||
let mut s = Script::new(server);
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
imap.selected = Selected { exists: u32::try_from(count).unwrap_or(u32::MAX), uid_validity: 1, uid_next: 0 };
|
||||
let got = imap.uid_fetch_flags_gmail("1:*").unwrap();
|
||||
assert_eq!(got.len(), count);
|
||||
let kept: usize = got.iter().filter_map(|f| f.gmail_labels.as_ref()).flatten().map(String::len).sum();
|
||||
assert!(kept <= MAX_GMAIL_LABEL_BYTES);
|
||||
assert!(got.last().is_some_and(|f| f.gmail_labels.is_none()));
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn arbitrary_label_lists_never_panic(inside in proptest::collection::vec(any::<u8>(), 0..400)) {
|
||||
let mut server = b"* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (".to_vec();
|
||||
server.extend(inside.iter().filter(|b| **b != b'\r' && **b != b'\n'));
|
||||
server.extend_from_slice(b"))\r\nA0001 OK done\r\n");
|
||||
let mut s = Script::new(server);
|
||||
if let Ok(got) = Imap::resume(&mut s).uid_fetch_flags_gmail("1:*") {
|
||||
for labels in got.iter().filter_map(|f| f.gmail_labels.as_ref()) {
|
||||
prop_assert!(labels.len() <= MAX_GMAIL_LABELS);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn arbitrary_server_bytes_never_panic_in_a_gmail_fetch(server in proptest::collection::vec(any::<u8>(), 0..600)) {
|
||||
let mut s = Script::new(server);
|
||||
let _ = Imap::resume(&mut s).uid_fetch_flags_gmail("1:*");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folder_other_than_the_inbox_asks_for_labels_alone() {
|
||||
let mut s = Script::new("* 1 FETCH (UID 1 FLAGS () X-GM-MSGID 5 X-GM-LABELS (a))\r\nA0001 OK done\r\n");
|
||||
let got = Imap::resume(&mut s).uid_fetch_flags_labels("1:*").unwrap();
|
||||
assert_eq!(s.sent(), "A0001 UID FETCH 1:* (UID FLAGS X-GM-LABELS)\r\n");
|
||||
// An id it wasn't asked for is not taken.
|
||||
assert_eq!(got.into_iter().map(|f| (f.uid, f.gmail_id, f.gmail_labels)).collect::<Vec<_>>(), [(1, None, owned(&["a"]))]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nil_in_a_list_is_no_label() {
|
||||
assert_eq!(labels("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (NIL))\r\nA0001 OK done\r\n"), [(1, owned(&[]))]);
|
||||
// A label called so comes quoted, and is one.
|
||||
assert_eq!(labels("* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (nil \"NIL\"))\r\nA0001 OK done\r\n"), [(1, owned(&["NIL"]))]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_labels_count_against_the_budget_too() {
|
||||
// Empty labels, as many as a message keeps: each costs its overhead, so the budget still ends.
|
||||
let list: String = (0..MAX_GMAIL_LABELS).map(|_| " \"\"").collect();
|
||||
let per_message = MAX_GMAIL_LABELS * LABEL_OVERHEAD;
|
||||
let count = MAX_GMAIL_LABEL_BYTES / per_message + 4;
|
||||
let mut server = String::new();
|
||||
for uid in 1..=count {
|
||||
server += &format!("* {uid} FETCH (UID {uid} FLAGS () X-GM-LABELS ({list}))\r\n");
|
||||
}
|
||||
server += "A0001 OK done\r\n";
|
||||
let mut s = Script::new(server);
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
imap.selected = Selected { exists: u32::try_from(count).unwrap_or(u32::MAX), uid_validity: 1, uid_next: 0 };
|
||||
let got = imap.uid_fetch_flags_labels("1:*").unwrap();
|
||||
assert_eq!(got.len(), count);
|
||||
let kept = got.iter().filter(|f| f.gmail_labels.is_some()).count();
|
||||
assert!(kept * per_message <= MAX_GMAIL_LABEL_BYTES && kept < count);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_big_folders_listing_is_not_cut_short_for_its_size() {
|
||||
// 300,000 messages with labels: more text than one command may otherwise bring (32 MiB).
|
||||
let count: u32 = 300_000;
|
||||
let mut server = String::with_capacity(count as usize * 200);
|
||||
for uid in 1..=count {
|
||||
server += &format!(
|
||||
"* {uid} FETCH (FLAGS (\\Seen) UID {uid} X-GM-LABELS (\\Inbox \\Important \"Work/Clients and partners\" \"Receipts 2026\" \"Travel/Japan\"))\r\n"
|
||||
);
|
||||
}
|
||||
assert!(server.len() > MAX_COMMAND_TEXT);
|
||||
server += "A0001 OK done\r\n";
|
||||
let mut s = Script::new(server);
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
imap.selected = Selected { exists: count, uid_validity: 1, uid_next: count + 1 };
|
||||
assert_eq!(imap.uid_fetch_flags_labels("1:*").map(|got| got.len()), Ok(count as usize));
|
||||
// The room is the listing's alone: the next command has the usual cap again.
|
||||
assert_eq!(imap.text_cap, MAX_COMMAND_TEXT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_mailbox_claimed_huge_cant_fill_memory_with_flags() {
|
||||
// Any server (no Gmail here): ten million messages claimed, then message after message with
|
||||
// every flag it may have. The text it may send is capped, and so are the flags kept: the
|
||||
// listing fails, and never hands back messages that seem to have lost their flags.
|
||||
let flags: Vec<String> = (0..MAX_FLAGS).map(|n| format!("f{n}")).collect();
|
||||
let flags = flags.join(" ");
|
||||
let mut server = String::new();
|
||||
let mut uid = 0u32;
|
||||
while server.len() < MAX_FETCH_TEXT / 4 {
|
||||
uid += 1;
|
||||
server += &format!("* {uid} FETCH (UID {uid} FLAGS ({flags}))\r\n");
|
||||
}
|
||||
server += "A0001 OK done\r\n";
|
||||
let mut s = Script::new(server);
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
imap.selected = Selected { exists: 10_000_000, uid_validity: 1, uid_next: 0 };
|
||||
let got = imap.uid_fetch_flags("1:*");
|
||||
assert!(matches!(&got, Err(ProtoError::Protocol(m)) if format!("{m:?}").contains("more flags")), "{:?}", got.as_ref().map(Vec::len));
|
||||
assert_eq!(imap.text_cap, MAX_COMMAND_TEXT);
|
||||
// And what it may send at all stays within the ceiling, however many messages are claimed.
|
||||
let mut s = Script::new(format!("* OK {}\r\n", "x".repeat(60_000)).repeat(MAX_FETCH_TEXT / 60_000 + 8));
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
imap.selected = Selected { exists: u32::MAX, uid_validity: 1, uid_next: 0 };
|
||||
assert!(matches!(imap.uid_fetch_flags("1:*"), Err(ProtoError::Protocol(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_honest_big_mailboxs_flags_fit_the_budget() {
|
||||
// A million messages with three flags each.
|
||||
const { assert!(1_000_000 * (ENTRY_OVERHEAD + 3 * (FLAG_OVERHEAD + 8)) <= MAX_FLAG_BYTES) };
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_small_folders_listing_still_has_its_cap() {
|
||||
// One message, and more than 32 MiB of other lines before it: refused as before.
|
||||
let junk = format!("* OK {}\r\n", "x".repeat(60_000));
|
||||
let mut server = junk.repeat(MAX_COMMAND_TEXT / 60_000 + 8);
|
||||
server += "* 1 FETCH (UID 1 FLAGS () X-GM-LABELS (a))\r\nA0001 OK done\r\n";
|
||||
let mut s = Script::new(server);
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
imap.selected = Selected { exists: 1, uid_validity: 1, uid_next: 2 };
|
||||
assert!(matches!(imap.uid_fetch_flags_labels("1:*"), Err(ProtoError::Protocol(_))));
|
||||
assert_eq!(imap.text_cap, MAX_COMMAND_TEXT);
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
//! Gmail's IMAP extensions (#911): `X-GM-MSGID` and `X-GM-RAW`.
|
||||
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use super::*;
|
||||
use crate::script::Script;
|
||||
|
||||
#[test]
|
||||
fn gmail_ids_come_with_the_flags() {
|
||||
let mut server = String::new();
|
||||
server += "* 1 FETCH (UID 4 FLAGS (\\Seen) X-GM-MSGID 1278455344230334865)\r\n";
|
||||
server += "* 2 FETCH (X-GM-MSGID 18446744073709551615 UID 7 FLAGS ())\r\n";
|
||||
// An id that is no number, or 0, is none; a later word about a message without one keeps it.
|
||||
server += "* 3 FETCH (UID 9 FLAGS () X-GM-MSGID 12ab)\r\n";
|
||||
server += "* 4 FETCH (UID 10 FLAGS () X-GM-MSGID 0)\r\n";
|
||||
server += "* 2 FETCH (UID 7 FLAGS (\\Flagged))\r\n";
|
||||
server += "A0001 OK done\r\n";
|
||||
let mut s = Script::new(server);
|
||||
let got = Imap::resume(&mut s).uid_fetch_flags_gmail("1:*").unwrap();
|
||||
let ids: Vec<(u32, Option<u64>)> = got.iter().map(|f| (f.uid, f.gmail_id)).collect();
|
||||
assert_eq!(ids, [(4, Some(1_278_455_344_230_334_865)), (7, Some(u64::MAX)), (9, None), (10, None)]);
|
||||
assert_eq!(got.get(1).map(|f| f.flags.clone()), Some(vec!["\\Flagged".to_string()]));
|
||||
assert_eq!(s.sent(), "A0001 UID FETCH 1:* (UID FLAGS X-GM-MSGID X-GM-LABELS)\r\n");
|
||||
// Without asking for them, an id the server gives anyway is not taken.
|
||||
let mut s = Script::new("* 1 FETCH (UID 4 FLAGS () X-GM-MSGID 5)\r\nA0001 OK done\r\n");
|
||||
assert_eq!(Imap::resume(&mut s).uid_fetch_flags("1:*").unwrap().first().and_then(|f| f.gmail_id), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gmail_search_is_quoted_and_kept_to_its_set() {
|
||||
let mut s = Script::new("* SEARCH 9 4 0 x 4 12 600\r\nA0001 OK done\r\nA0002 OK done\r\n");
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
// In UID order, once each, and only what the set names.
|
||||
assert_eq!(imap.uid_search_gmail("4:12", "category:social"), Ok(vec![4, 9, 12]));
|
||||
// Quotes are quoted; CR, LF and NUL never go out.
|
||||
assert_eq!(imap.uid_search_gmail("1:*", "category:\"x\"\r\nA9 DELETE INBOX\0"), Ok(vec![]));
|
||||
assert_eq!(
|
||||
s.sent(),
|
||||
"A0001 UID SEARCH UID 4:12 UNDELETED X-GM-RAW \"category:social\"\r\n\
|
||||
A0002 UID SEARCH UID 1:* UNDELETED X-GM-RAW \"category:\\\"x\\\"A9 DELETE INBOX\"\r\n"
|
||||
);
|
||||
// A refusal is said.
|
||||
let mut s = Script::new("A0001 BAD no such search\r\n");
|
||||
assert!(matches!(Imap::resume(&mut s).uid_search_gmail("1:*", "category:forums"), Err(ProtoError::Rejected(_))));
|
||||
// What can't be sent is refused unsent.
|
||||
for (set, query) in [("1:*", ""), ("1:*", "\r\n"), ("1:*", "kategorie:äöü"), ("1 2", "category:social"), ("", "category:social")] {
|
||||
let mut s = Script::new("A0001 OK done\r\n");
|
||||
assert!(matches!(Imap::resume(&mut s).uid_search_gmail(set, query), Err(ProtoError::Protocol(_))), "{set:?} {query:?}");
|
||||
assert_eq!(s.sent(), "", "{set:?} {query:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gmail_search_takes_no_more_than_its_cap() {
|
||||
let many: String = (1..=6000).map(|n| format!(" {n}")).collect();
|
||||
let mut s = Script::new(format!("* SEARCH{many}\r\nA0001 OK done\r\n"));
|
||||
assert_eq!(Imap::resume(&mut s).uid_search_gmail("1:*", "category:promotions").map(|uids| uids.len()), Ok(MAX_GMAIL_FOUND));
|
||||
// As many as the cap, at the longest a UID is written, fit in one line.
|
||||
let longest: String = (0..MAX_GMAIL_FOUND).map(|_| " 4294967295").collect();
|
||||
assert!(format!("* SEARCH{longest}").len() < crate::MAX_LINE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_gmail_answer_too_long_is_refused_and_the_session_goes_on() {
|
||||
let long: String = (0..20_000).map(|n| format!(" {}", 1_000_000_000u32 + n)).collect();
|
||||
assert!(long.len() > crate::MAX_LINE);
|
||||
let server = format!("* SEARCH{long}\r\n* SEARCH 3\r\nA0001 OK done\r\n* SEARCH 5\r\nA0002 OK done\r\n");
|
||||
let mut s = Script::new(server);
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
assert!(matches!(imap.uid_search_gmail("1:*", "category:social"), Err(ProtoError::Rejected(_))));
|
||||
// The next command is answered as if nothing had happened.
|
||||
assert_eq!(imap.uid_search_gmail("1:*", "category:social"), Ok(vec![5]));
|
||||
// Elsewhere a line that long still ends the session, and a tagged one is never read past.
|
||||
let mut s = Script::new(format!("* SEARCH{long}\r\nA0001 OK done\r\n"));
|
||||
assert!(matches!(Imap::resume(&mut s).uid_search_header("Message-ID", "x"), Err(ProtoError::Protocol(_))));
|
||||
let mut s = Script::new(format!("A0001 OK{long}\r\n"));
|
||||
assert!(matches!(Imap::resume(&mut s).uid_search_gmail("1:*", "category:social"), Err(ProtoError::Protocol(_))));
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
//! IDLE (RFC 2177): the server says when a mailbox changes, instead of being asked. The session
|
||||
//! is in one of two states, and this module only moves it between them: a mailbox is selected
|
||||
//! (EXAMINE), [`Imap::idle_begin`] puts the server to waiting, [`Imap::idle_poll`] hears what it
|
||||
//! says (a line a second at most, so that the caller can look at the clock and at whether it was
|
||||
//! told to stop), and [`Imap::idle_end`] says DONE and reads on to the command's end.
|
||||
//!
|
||||
//! The server's word is bounded as everywhere: a line over [`crate::MAX_LINE`] and a line that
|
||||
//! announces a literal end the session, and the caller counts what it hears.
|
||||
|
||||
use std::io::{Read, Write};
|
||||
|
||||
use super::Imap;
|
||||
use super::parse;
|
||||
use crate::i18n::{Arg, said};
|
||||
use crate::{ProtoError, for_trace};
|
||||
|
||||
/// Most lines [`Imap::idle_end`] reads on the way to the command's end.
|
||||
const MAX_ENDING: usize = 256;
|
||||
/// Most quiet reads [`Imap::idle_end`] waits through for one line.
|
||||
const MAX_QUIET: usize = 30;
|
||||
|
||||
/// What the server said while it was waiting.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum IdleEvent {
|
||||
/// `* n EXISTS`: the mailbox now has `n` messages.
|
||||
Exists(u32),
|
||||
/// `* n EXPUNGE`: the message with that sequence number is gone.
|
||||
Expunge(u32),
|
||||
/// `* n RECENT`: `n` messages are new since the mailbox was opened.
|
||||
Recent(u32),
|
||||
/// Anything else the server said unasked (a flag change, a status line): not news of mail.
|
||||
Other,
|
||||
/// The server ended the command itself (`tag OK`): IDLE has to be begun again.
|
||||
Ended,
|
||||
}
|
||||
|
||||
/// From the text after `* `: `3 EXISTS`.
|
||||
fn event_from(rest: &[u8], secrets: &crate::Secrets) -> Result<IdleEvent, ProtoError> {
|
||||
let text = String::from_utf8_lossy(rest);
|
||||
let mut words = text.split_whitespace();
|
||||
let (first, second) = (words.next().unwrap_or(""), words.next().unwrap_or(""));
|
||||
let count = first.parse::<u32>().ok();
|
||||
Ok(match (count, second.to_ascii_uppercase().as_str()) {
|
||||
(Some(n), "EXISTS") => IdleEvent::Exists(n),
|
||||
(Some(n), "EXPUNGE") => IdleEvent::Expunge(n),
|
||||
(Some(n), "RECENT") => IdleEvent::Recent(n),
|
||||
_ if first.eq_ignore_ascii_case("BYE") => {
|
||||
return Err(ProtoError::Io(said!("the server ended the session: {reason}", reason = crate::tidy(&secrets.hide(&text)))));
|
||||
}
|
||||
_ => IdleEvent::Other,
|
||||
})
|
||||
}
|
||||
|
||||
impl<S: Read + Write> Imap<S> {
|
||||
/// Send IDLE and wait for the server's go-ahead. The session has to have a mailbox selected;
|
||||
/// a server that refuses (NO, BAD) is an error. While idle, the only commands that may follow
|
||||
/// are [`Imap::idle_poll`] and [`Imap::idle_end`].
|
||||
pub fn idle_begin(&mut self) -> Result<(), ProtoError> {
|
||||
if self.idle_tag.is_some() {
|
||||
return Err(ProtoError::Protocol(said!("the server is already idling")));
|
||||
}
|
||||
self.idle_pending.clear();
|
||||
let tag = self.tag();
|
||||
self.text_seen = 0;
|
||||
log::trace!("C: {tag} IDLE");
|
||||
self.wire.write(format!("{tag} IDLE\r\n").as_bytes())?;
|
||||
self.wire.flush()?;
|
||||
self.idle_tag = Some(tag.clone());
|
||||
// Untagged lines may come before the `+`; the tagged answer is a refusal.
|
||||
for _ in 0..MAX_ENDING {
|
||||
match self.read_response()? {
|
||||
super::Response::GoAhead => return Ok(()),
|
||||
super::Response::Status { tag: Some(answered), line } => {
|
||||
self.idle_tag = None;
|
||||
if answered != tag {
|
||||
return Err(ProtoError::Protocol(said!("the server answered a command that wasn't sent")));
|
||||
}
|
||||
return Err(ProtoError::Rejected(self.refusal(&line)));
|
||||
}
|
||||
super::Response::Status { tag: None, line } => {
|
||||
if line.status == parse::Status::Bye {
|
||||
self.idle_tag = None;
|
||||
return Err(ProtoError::Io(said!("the server ended the session: {reason}", reason = Arg::said(self.refusal(&line)))));
|
||||
}
|
||||
self.note_status(&line);
|
||||
}
|
||||
super::Response::Data(values) => {
|
||||
self.note_data(&values);
|
||||
let event = match (values.first().and_then(parse::Value::number), values.get(1).and_then(parse::Value::atom)) {
|
||||
(Some(n), Some(word)) if word.eq_ignore_ascii_case("EXISTS") => IdleEvent::Exists(n),
|
||||
(Some(n), Some(word)) if word.eq_ignore_ascii_case("EXPUNGE") => IdleEvent::Expunge(n),
|
||||
(Some(n), Some(word)) if word.eq_ignore_ascii_case("RECENT") => IdleEvent::Recent(n),
|
||||
_ => IdleEvent::Other,
|
||||
};
|
||||
self.idle_pending.push_back(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.idle_tag = None;
|
||||
Err(ProtoError::Protocol(said!("the server sent too many responses to one command")))
|
||||
}
|
||||
|
||||
/// The next thing the server says while idle, or `None` after one read if the line is
|
||||
/// incomplete or no bytes arrived (see [`crate::MAX_LINE`] for the bound on a line).
|
||||
pub fn idle_poll(&mut self) -> Result<Option<IdleEvent>, ProtoError> {
|
||||
let Some(tag) = self.idle_tag.clone() else {
|
||||
return Err(ProtoError::Protocol(said!("the server isn't idling")));
|
||||
};
|
||||
if let Some(event) = self.idle_pending.pop_front() {
|
||||
return Ok(Some(event));
|
||||
}
|
||||
let Some(line) = self.wire.poll_line()? else { return Ok(None) };
|
||||
if log::log_enabled!(log::Level::Trace) {
|
||||
log::trace!("S: {}", for_trace(&self.secrets.hide(&String::from_utf8_lossy(&line))));
|
||||
}
|
||||
if let Some(rest) = line.strip_prefix(b"* ") {
|
||||
// A literal in what is said while idle is nothing a server has a reason to send.
|
||||
if parse::literal_marker(rest).is_some() {
|
||||
return Err(ProtoError::Protocol(said!("the server sent data it has no reason to send while idle")));
|
||||
}
|
||||
return event_from(rest, &self.secrets).map(Some);
|
||||
}
|
||||
let text = String::from_utf8_lossy(&line);
|
||||
if text.strip_prefix(tag.as_str()).is_some_and(|rest| rest.starts_with(' ')) {
|
||||
// The server ended IDLE by itself (a timeout of its own, say).
|
||||
self.idle_tag = None;
|
||||
let status = text.split_ascii_whitespace().nth(1).and_then(parse::Status::parse);
|
||||
return match status {
|
||||
Some(parse::Status::Ok) => Ok(Some(IdleEvent::Ended)),
|
||||
Some(parse::Status::No | parse::Status::Bad) => Err(ProtoError::Rejected(said!("the server didn't end IDLE properly"))),
|
||||
_ => Err(ProtoError::Protocol(said!("the server returned an invalid IDLE completion"))),
|
||||
};
|
||||
}
|
||||
Err(ProtoError::Protocol(said!("the server said something that is no news while idle")))
|
||||
}
|
||||
|
||||
/// Say DONE and read what the server still has to say, up to the end of the command. What was
|
||||
/// said on the way is given back. After this the session is as it was before
|
||||
/// [`Imap::idle_begin`].
|
||||
pub fn idle_end(&mut self) -> Result<Vec<IdleEvent>, ProtoError> {
|
||||
let Some(tag) = self.idle_tag.take() else {
|
||||
return Err(ProtoError::Protocol(said!("the server isn't idling")));
|
||||
};
|
||||
log::trace!("C: DONE");
|
||||
self.wire.write(b"DONE\r\n")?;
|
||||
self.wire.flush()?;
|
||||
let mut events: Vec<_> = self.idle_pending.drain(..).collect();
|
||||
for _ in 0..MAX_ENDING {
|
||||
// The server may take a moment to answer: a few polls of the stream's own wait.
|
||||
let mut polled = None;
|
||||
for _ in 0..MAX_QUIET {
|
||||
polled = self.wire.poll_line()?;
|
||||
if polled.is_some() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let Some(line) = polled else { return Err(ProtoError::Io(said!("the server took too long to answer"))) };
|
||||
if let Some(rest) = line.strip_prefix(b"* ") {
|
||||
if parse::literal_marker(rest).is_some() {
|
||||
return Err(ProtoError::Protocol(said!("the server sent data it has no reason to send while idle")));
|
||||
}
|
||||
events.push(event_from(rest, &self.secrets)?);
|
||||
continue;
|
||||
}
|
||||
let text = String::from_utf8_lossy(&line);
|
||||
let mut words = text.splitn(3, ' ');
|
||||
let (answered, status) = (words.next().unwrap_or(""), words.next().unwrap_or(""));
|
||||
if answered != tag {
|
||||
return Err(ProtoError::Protocol(said!("the server answered a command that wasn't sent")));
|
||||
}
|
||||
return match parse::Status::parse(status) {
|
||||
Some(parse::Status::Ok) => Ok(events),
|
||||
_ => Err(ProtoError::Rejected(said!("the server didn't end IDLE properly"))),
|
||||
};
|
||||
}
|
||||
Err(ProtoError::Protocol(said!("the server sent too many responses to one command")))
|
||||
}
|
||||
|
||||
/// Whether the server said it can do IDLE (upper-cased capability list as last said).
|
||||
pub fn offers_idle(&self) -> bool {
|
||||
self.offers("IDLE")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use proptest::prelude::*;
|
||||
|
||||
use super::*;
|
||||
use crate::script::Script;
|
||||
|
||||
// Polling now yields after each underlying read, including partial lines.
|
||||
fn next_event(imap: &mut Imap<&mut Script>) -> Result<Option<IdleEvent>, ProtoError> {
|
||||
for _ in 0..crate::MAX_LINE + 2 {
|
||||
match imap.idle_poll() {
|
||||
Ok(None) => continue,
|
||||
result => return result,
|
||||
}
|
||||
}
|
||||
panic!("script failed to produce an event within its bounded input");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_goes_ahead_hears_news_and_ends() {
|
||||
// The go-ahead is the first line; the rest is heard by polling.
|
||||
let mut s2 = Script::new("+ idling\r\n* 4 EXISTS\r\n* 1 RECENT\r\n* 2 FETCH (FLAGS (\\Seen))\r\n* 3 EXPUNGE\r\n");
|
||||
let mut imap2 = Imap::resume(&mut s2);
|
||||
imap2.idle_begin().unwrap();
|
||||
assert_eq!(next_event(&mut imap2).unwrap(), Some(IdleEvent::Exists(4)));
|
||||
assert_eq!(next_event(&mut imap2).unwrap(), Some(IdleEvent::Recent(1)));
|
||||
assert_eq!(next_event(&mut imap2).unwrap(), Some(IdleEvent::Other));
|
||||
assert_eq!(next_event(&mut imap2).unwrap(), Some(IdleEvent::Expunge(3)));
|
||||
assert_eq!(s2.sent(), "A0001 IDLE\r\n");
|
||||
|
||||
let mut s3 = Script::new("+ idling\r\n* 5 EXISTS\r\nA0001 OK IDLE terminated\r\n");
|
||||
let mut imap3 = Imap::resume(&mut s3);
|
||||
imap3.idle_begin().unwrap();
|
||||
assert_eq!(imap3.idle_end().unwrap(), vec![IdleEvent::Exists(5)]);
|
||||
assert_eq!(s3.sent(), "A0001 IDLE\r\nDONE\r\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_refusal_or_an_early_end_or_a_goodbye_is_said() {
|
||||
let mut s = Script::new("A0001 NO [CANNOT] not now\r\n");
|
||||
assert!(matches!(Imap::resume(&mut s).idle_begin(), Err(ProtoError::Rejected(_))));
|
||||
let mut s = Script::new("+ idling\r\nA0001 OK the server gave up waiting\r\n");
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
imap.idle_begin().unwrap();
|
||||
assert_eq!(next_event(&mut imap).unwrap(), Some(IdleEvent::Ended));
|
||||
// Not idling any more: nothing to poll or to end.
|
||||
assert!(matches!(next_event(&mut imap), Err(ProtoError::Protocol(_))));
|
||||
assert!(matches!(imap.idle_end(), Err(ProtoError::Protocol(_))));
|
||||
let mut s = Script::new("+ idling\r\n* BYE going away\r\n");
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
imap.idle_begin().unwrap();
|
||||
assert!(matches!(next_event(&mut imap), Err(ProtoError::Io(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_a_server_has_no_reason_to_send_while_idle_ends_the_session() {
|
||||
for hostile in ["* 1 FETCH (BODY[] {5}\r\n", "A0099 OK that was never asked\r\n", "garbage\r\n", "* 1 EXISTS\r\n".repeat(0).as_str()] {
|
||||
let mut s = Script::new(format!("+ idling\r\n{hostile}"));
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
imap.idle_begin().unwrap();
|
||||
let got = next_event(&mut imap);
|
||||
assert!(got.is_err() || hostile.is_empty(), "{hostile:?}: {got:?}");
|
||||
}
|
||||
// A line longer than any a server is allowed.
|
||||
let mut long = b"+ idling\r\n* ".to_vec();
|
||||
long.extend(std::iter::repeat_n(b'x', crate::MAX_LINE + 10));
|
||||
long.extend_from_slice(b"\r\n");
|
||||
let mut s = Script::whole(long);
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
imap.idle_begin().unwrap();
|
||||
assert!(matches!(next_event(&mut imap), Err(ProtoError::Protocol(_))));
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn arbitrary_server_bytes_never_panic_while_idle(server in proptest::collection::vec(any::<u8>(), 0..800), greeted in any::<bool>()) {
|
||||
let mut bytes = if greeted { b"+ idling\r\n".to_vec() } else { Vec::new() };
|
||||
bytes.extend_from_slice(&server);
|
||||
let mut s = Script::new(&bytes);
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
if imap.idle_begin().is_ok() {
|
||||
// However much is said, polling ends (the script runs out) and nothing is kept without bound.
|
||||
for _ in 0..2_000 {
|
||||
match next_event(&mut imap) {
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) | Err(_) => break,
|
||||
}
|
||||
}
|
||||
let _ = imap.idle_end();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_shaped_noise_never_panics(words in proptest::collection::vec(proptest::sample::select(&["* ", "1 ", "EXISTS", "EXPUNGE", "RECENT", " ", "\r\n", "A0001 OK ", "BYE ", "{9}", "FETCH (", ")"][..]), 0..60)) {
|
||||
let mut s = Script::new(format!("+ go\r\n{}", words.concat()));
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
if imap.idle_begin().is_ok() {
|
||||
for _ in 0..200 {
|
||||
if !matches!(next_event(&mut imap), Ok(Some(_))) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let _ = imap.idle_end();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notifications_before_continuation_are_delivered() {
|
||||
let mut s = Script::new("* 4 EXISTS\r\n* 2 EXPUNGE\r\n+ idling\r\nA0001 OK done\r\n");
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
imap.idle_begin().unwrap();
|
||||
assert_eq!(imap.idle_poll().unwrap(), Some(IdleEvent::Exists(4)));
|
||||
assert_eq!(imap.idle_end().unwrap(), vec![IdleEvent::Expunge(2)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_bye_errors_hide_registered_secrets() {
|
||||
for ending in [false, true] {
|
||||
let mut s = Script::new("+ idling\r\n* BYE test-token\r\n");
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
imap.secrets.add("test-token");
|
||||
imap.idle_begin().unwrap();
|
||||
let error = if ending { imap.idle_end().unwrap_err() } else { next_event(&mut imap).unwrap_err() };
|
||||
assert!(!error.to_string().contains("test-token"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_completion_requires_a_valid_success_status() {
|
||||
for (status, rejected) in [("NO", true), ("BAD", true), ("nonsense", false)] {
|
||||
let mut s = Script::new(format!("+ idling\r\nA0001 {status} failed\r\n"));
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
imap.idle_begin().unwrap();
|
||||
let error = next_event(&mut imap).unwrap_err();
|
||||
assert!(if rejected { matches!(error, ProtoError::Rejected(_)) } else { matches!(error, ProtoError::Protocol(_)) });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
//! What one command may make the client keep or read, however much the server sends.
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use proptest::prelude::*;
|
||||
|
||||
use super::*;
|
||||
use crate::script::Script;
|
||||
|
||||
/// A session whose open mailbox has `exists` messages and the next UID `uid_next`.
|
||||
fn opened(s: &mut Script, exists: u32, uid_next: u32) -> Imap<&mut Script> {
|
||||
let mut imap = Imap::resume(s);
|
||||
imap.selected = Selected { exists, uid_validity: 1, uid_next };
|
||||
imap
|
||||
}
|
||||
|
||||
/// `lines` lines made by `line(n)`.
|
||||
fn flood(lines: usize, line: impl Fn(usize) -> String) -> String {
|
||||
(0..lines).map(line).collect()
|
||||
}
|
||||
|
||||
fn padding() -> String {
|
||||
"x".repeat(60 * 1024)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_flood_of_long_lines_ends_the_command() {
|
||||
let pad = padding();
|
||||
// 600 lines of 60 KB are more than the 32 MB one command may bring.
|
||||
let server = flood(600, |n| format!("* {n} FETCH (UID {} FLAGS ({pad}))\r\n", n + 1)) + "A0001 OK done\r\n";
|
||||
let mut s = Script::new(server);
|
||||
let err = opened(&mut s, 1000, 2000).uid_fetch_flags("1:*").unwrap_err();
|
||||
assert!(matches!(err, ProtoError::Protocol(ref m) if m.to_string().contains("too much")), "{err:?}");
|
||||
// The same in a list.
|
||||
let server = flood(600, |n| format!("* LIST () \"/\" \"{n}{pad}\"\r\n")) + "A0001 OK done\r\n";
|
||||
let mut s = Script::new(server);
|
||||
assert!(matches!(opened(&mut s, 0, 0).list(), Err(ProtoError::Protocol(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_flood_of_responses_ends_the_command() {
|
||||
let server = "* OK still here\r\n".repeat(MAX_RESPONSES + 1) + "A0001 OK done\r\n";
|
||||
let mut s = Script::new(server);
|
||||
let err = opened(&mut s, 0, 0).uid_fetch_sizes("1:*").unwrap_err();
|
||||
assert!(matches!(err, ProtoError::Protocol(ref m) if m.to_string().contains("too many")), "{err:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_list_keeps_a_capped_number_of_mailboxes() {
|
||||
let server = flood(MAX_MAILBOXES + 500, |n| format!("* LIST () \"/\" f{n}\r\n")) + "A0001 OK done\r\n";
|
||||
let mut s = Script::new(server);
|
||||
let got = opened(&mut s, 0, 0).list().unwrap();
|
||||
assert_eq!(got.len(), MAX_MAILBOXES);
|
||||
assert_eq!(got.first().map(|m| m.raw.as_str()), Some("f0"));
|
||||
// #991: and how many were dropped is said; none, for a list kept whole.
|
||||
let server = flood(MAX_MAILBOXES + 500, |n| format!("* LIST () \"/\" f{n}\r\n")) + "A0001 OK done\r\n";
|
||||
let mut s = Script::new(server);
|
||||
assert_eq!(opened(&mut s, 0, 0).list_counted().unwrap().1, 500);
|
||||
let mut s = Script::new("* LIST () \"/\" a\r\n* LIST () \"/\" a\r\nA0001 OK done\r\n");
|
||||
assert_eq!(
|
||||
opened(&mut s, 0, 0).list_counted().unwrap(),
|
||||
(vec![Mailbox { name: "a".into(), raw: "a".into(), delimiter: Some('/'), flags: Vec::new() }], 0)
|
||||
);
|
||||
}
|
||||
|
||||
/// #991: a list keeps a bounded number of bytes too, however many mailboxes that is.
|
||||
#[test]
|
||||
fn a_list_keeps_a_capped_number_of_bytes() {
|
||||
let flags: Vec<String> = (0..MAX_FLAGS).map(|n| format!("\\F{n}{}", "x".repeat(MAX_FLAG_LEN - 8))).collect();
|
||||
let flags = flags.join(" ");
|
||||
let server = flood(1_500, |n| format!("* LIST ({flags}) \"/\" f{n}\r\n")) + "A0001 OK done\r\n";
|
||||
let mut s = Script::new(server);
|
||||
let (got, dropped) = opened(&mut s, 0, 0).list_counted().unwrap();
|
||||
assert_eq!(got.len() + dropped, 1_500);
|
||||
let kept: usize = got.iter().map(|m| m.raw.len() * 2 + m.name.len() + m.flags.iter().map(String::len).sum::<usize>()).sum();
|
||||
assert!(!got.is_empty() && got.len() < 1_500, "{}", got.len());
|
||||
// The last one kept may take it over by one mailbox's worth, never more.
|
||||
assert!(kept < MAX_LIST_BYTES + MAX_LINE, "{kept}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn names_and_flags_are_kept_short() {
|
||||
let long_flag = "é".repeat(400);
|
||||
let long_name = "n".repeat(MAX_MAILBOX_NAME + 1);
|
||||
let server = format!("* LIST ({long_flag} \\Sent) \"/\" Fine\r\n* LIST () \"/\" {long_name}\r\nA0001 OK done\r\n");
|
||||
let mut s = Script::new(server);
|
||||
let got = opened(&mut s, 0, 0).list().unwrap();
|
||||
// The mailbox with the oversized name is left out, not cut into another one.
|
||||
assert_eq!(got.len(), 1);
|
||||
assert!(got[0].flags[0].len() <= MAX_FLAG_LEN && got[0].flags[0].chars().all(|c| c == 'é'));
|
||||
assert_eq!(got[0].flags[1], "\\Sent");
|
||||
|
||||
let server = format!("* 1 FETCH (UID 1 FLAGS ({long_flag}))\r\nA0001 OK done\r\n");
|
||||
let mut s = Script::new(server);
|
||||
let got = opened(&mut s, 1, 2).uid_fetch_flags("1:*").unwrap();
|
||||
assert!(got[0].flags[0].len() <= MAX_FLAG_LEN);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_uids_that_were_asked_for_and_exist_are_kept() {
|
||||
let server = "* 1 FETCH (UID 3 FLAGS (\\Seen))\r\n* 2 FETCH (UID 7 FLAGS ())\r\n* 3 FETCH (UID 9 FLAGS ())\r\n* 4 FETCH (UID 4000000000 FLAGS ())\r\n\
|
||||
* 5 FETCH (UID 10 FLAGS ())\r\nA0001 OK done\r\n";
|
||||
// Not in the set.
|
||||
let mut s = Script::new(server);
|
||||
let got = opened(&mut s, 5, 11).uid_fetch_flags("3,8:9").unwrap();
|
||||
assert_eq!(got.iter().map(|f| f.uid).collect::<Vec<_>>(), [3, 9]);
|
||||
// UIDNEXT is only a snapshot: later deliveries must not be discarded.
|
||||
let sizes = server.replace("FLAGS (\\Seen)", "RFC822.SIZE 5").replace("FLAGS ()", "RFC822.SIZE 6");
|
||||
let mut s = Script::new(sizes);
|
||||
let got = opened(&mut s, 5, 10).uid_fetch_sizes("1:*").unwrap();
|
||||
assert_eq!(got, [(3, 5), (7, 6), (9, 6), (10, 6), (4000000000, 6)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn more_distinct_uids_than_the_mailbox_can_hold_fail_the_command() {
|
||||
// 3 messages: room for 2 * 3 + FETCH_SLACK distinct UIDs, and not one more.
|
||||
let room = 2 * 3 + FETCH_SLACK;
|
||||
let many = |n: usize| flood(n, |i| format!("* {i} FETCH (UID {} FLAGS ())\r\n", i + 1)) + "A0001 OK done\r\n";
|
||||
let mut s = Script::new(many(room));
|
||||
assert_eq!(opened(&mut s, 3, u32::MAX).uid_fetch_flags("1:*").unwrap().len(), room);
|
||||
let mut s = Script::new(many(room + 1));
|
||||
assert!(matches!(opened(&mut s, 3, u32::MAX).uid_fetch_flags("1:*"), Err(ProtoError::Protocol(_))));
|
||||
let sizes = flood(room + 1, |i| format!("* {i} FETCH (UID {} RFC822.SIZE 1)\r\n", i + 1)) + "A0001 OK done\r\n";
|
||||
let mut s = Script::new(sizes);
|
||||
assert!(matches!(opened(&mut s, 3, u32::MAX).uid_fetch_sizes("1:*"), Err(ProtoError::Protocol(_))));
|
||||
// The same UID again and again is one.
|
||||
let same = flood(room * 3, |_| "* 1 FETCH (UID 1 FLAGS ())\r\n".to_string()) + "A0001 OK done\r\n";
|
||||
let mut s = Script::new(same);
|
||||
assert_eq!(opened(&mut s, 3, 100).uid_fetch_flags("1:*").unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_ranges_read_what_is_written() {
|
||||
assert_eq!(set_ranges("1:*", 9), Some(vec![(1, 9)]));
|
||||
assert_eq!(set_ranges("5,9,20:12", 9), Some(vec![(5, 5), (9, 9), (12, 20)]));
|
||||
assert_eq!(set_ranges("1:x", 9), None);
|
||||
assert_eq!(set_ranges("", 9), None);
|
||||
}
|
||||
|
||||
proptest! {
|
||||
/// Whatever the server sends, the command ends, and what it keeps is bounded by the caps.
|
||||
#[test]
|
||||
fn arbitrary_responses_are_kept_within_bounds(lines in proptest::collection::vec(("[ -~]{0,200}", any::<u32>(), 0usize..4), 0..400)) {
|
||||
let server: String = lines
|
||||
.iter()
|
||||
.map(|(text, uid, kind)| match kind {
|
||||
0 => format!("* LIST (\\{text}) \"/\" {text}\r\n"),
|
||||
1 => format!("* 1 FETCH (UID {uid} FLAGS ({text}))\r\n"),
|
||||
2 => format!("* 1 FETCH (UID {uid} RFC822.SIZE {uid})\r\n"),
|
||||
_ => format!("* {text}\r\n"),
|
||||
})
|
||||
.collect::<String>()
|
||||
+ "A0001 OK done\r\n";
|
||||
let mut s = Script::new(server.clone());
|
||||
if let Ok(list) = opened(&mut s, 10, 50).list() {
|
||||
prop_assert!(list.len() <= MAX_MAILBOXES);
|
||||
prop_assert!(list.iter().all(|m| m.raw.len() <= MAX_MAILBOX_NAME && m.flags.len() <= MAX_FLAGS && m.flags.iter().all(|f| f.len() <= MAX_FLAG_LEN)));
|
||||
}
|
||||
let mut s = Script::new(server.clone());
|
||||
// A fixed range retains its UID bound even if the server sends newer deliveries.
|
||||
if let Ok(got) = opened(&mut s, 10, 50).uid_fetch_flags("1:49") {
|
||||
prop_assert!(got.len() <= 2 * 10 + FETCH_SLACK);
|
||||
prop_assert!(got.iter().all(|f| f.uid >= 1 && f.uid < 50 && f.flags.len() <= MAX_FLAGS && f.flags.iter().all(|x| x.len() <= MAX_FLAG_LEN)));
|
||||
}
|
||||
let mut s = Script::new(server.clone());
|
||||
// Wildcards admit deliveries newer than SELECT's UIDNEXT, but retain all storage caps.
|
||||
if let Ok(got) = opened(&mut s, 10, 50).uid_fetch_flags("1:*") {
|
||||
prop_assert!(got.len() <= 2 * 10 + FETCH_SLACK);
|
||||
prop_assert!(got.iter().all(|f| f.uid >= 1 && f.flags.len() <= MAX_FLAGS && f.flags.iter().all(|x| x.len() <= MAX_FLAG_LEN)));
|
||||
}
|
||||
let mut s = Script::new(server);
|
||||
if let Ok(got) = opened(&mut s, 10, 50).uid_fetch_sizes("5:20") {
|
||||
prop_assert!(got.iter().all(|(uid, _)| (5..=20).contains(uid)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folder_with_more_messages_than_other_commands_may_answer_still_reads() {
|
||||
let exists = 300_000u32;
|
||||
let lines = |n: u32| flood(n as usize, |i| format!("* {i} FETCH (UID {} FLAGS ())\r\n", i + 1)) + "A0001 OK done\r\n";
|
||||
let mut s = Script::new(lines(exists));
|
||||
assert_eq!(opened(&mut s, exists, exists + 1).uid_fetch_flags("1:*").unwrap().len(), exists as usize);
|
||||
let sizes = flood(exists as usize, |i| format!("* {i} FETCH (UID {} RFC822.SIZE 1)\r\n", i + 1)) + "A0001 OK done\r\n";
|
||||
let mut s = Script::new(sizes);
|
||||
assert_eq!(opened(&mut s, exists, exists + 1).uid_fetch_sizes("1:*").unwrap().len(), exists as usize);
|
||||
// Responses that are not messages still end at the cap for that folder, and the next command is back to the usual one.
|
||||
let noise = "* OK still here\r\n".repeat(2 * exists as usize + FETCH_SLACK + 1) + "A0001 OK done\r\n";
|
||||
let mut s = Script::new(noise);
|
||||
let mut imap = opened(&mut s, exists, exists + 1);
|
||||
assert!(matches!(imap.uid_fetch_flags("1:*"), Err(ProtoError::Protocol(_))));
|
||||
assert_eq!(imap.response_cap, MAX_RESPONSES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_flags_release_large_backing_allocations() {
|
||||
let flags = short_flags(vec![Value::Atom("A".repeat(1024 * 1024))]);
|
||||
assert_eq!(flags[0].len(), MAX_FLAG_LEN);
|
||||
assert!(flags[0].capacity() <= MAX_FLAG_LEN);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wildcard_fetch_accepts_deliveries_after_select() {
|
||||
let server = "* 1 FETCH (UID 10 FLAGS ())\r\nA0001 OK done\r\n";
|
||||
let mut s = Script::new(server);
|
||||
assert_eq!(opened(&mut s, 1, 10).uid_fetch_flags("*").unwrap()[0].uid, 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn brackets_in_flag_keywords_do_not_consume_body_sections() {
|
||||
for flag in ["x[", "BODY["] {
|
||||
let mut s = Script::new(format!("* 1 FETCH (UID 3 FLAGS ({flag}) BODY[] {{2}}\r\nok)\r\nA0001 OK done\r\n"));
|
||||
assert_eq!(opened(&mut s, 1, 4).uid_fetch_message(3).unwrap().unwrap().body, Some(b"ok".to_vec()));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,346 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
//! Reading IMAP responses: the pieces a response arrives in, the values in it, and status lines.
|
||||
|
||||
use crate::ProtoError;
|
||||
use crate::i18n::said;
|
||||
|
||||
/// Deepest that parenthesised lists may nest.
|
||||
const MAX_DEPTH: usize = 32;
|
||||
|
||||
/// A response as it came off the wire: lines of text with the literals between them.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum Piece {
|
||||
Text(Vec<u8>),
|
||||
Literal(Vec<u8>),
|
||||
}
|
||||
|
||||
/// One value of a response.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum Value {
|
||||
/// A bare word: a number, a flag, `NIL`, `BODY[]`.
|
||||
Atom(String),
|
||||
/// A quoted string or a literal.
|
||||
Str(Vec<u8>),
|
||||
List(Vec<Value>),
|
||||
}
|
||||
|
||||
impl Value {
|
||||
pub(crate) fn atom(&self) -> Option<&str> {
|
||||
match self {
|
||||
Value::Atom(a) => Some(a),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn number(&self) -> Option<u32> {
|
||||
self.atom()?.parse().ok()
|
||||
}
|
||||
|
||||
/// The text of a string or a bare word (what the grammar calls an `astring`).
|
||||
pub(crate) fn into_text(self) -> Option<String> {
|
||||
match self {
|
||||
Value::Atom(a) => Some(a),
|
||||
Value::Str(bytes) => Some(String::from_utf8_lossy(&bytes).into_owned()),
|
||||
Value::List(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a line ends in a literal marker (`{123}`, also `{123+}`) outside any quoted string: the
|
||||
/// marker's position and the size it names.
|
||||
pub(crate) fn literal_marker(line: &[u8]) -> Option<(usize, u64)> {
|
||||
let inner_end = line.len().checked_sub(1).filter(|last| line.get(*last) == Some(&b'}'))?;
|
||||
let open = line.iter().rposition(|b| *b == b'{')?;
|
||||
let inner = line.get(open + 1..inner_end)?;
|
||||
let digits = inner.strip_suffix(b"+").unwrap_or(inner);
|
||||
if digits.is_empty() || !digits.iter().all(u8::is_ascii_digit) {
|
||||
return None;
|
||||
}
|
||||
// A brace inside a quoted string is just text.
|
||||
let (mut quoted, mut escaped) = (false, false);
|
||||
for b in line.get(..open)? {
|
||||
if escaped {
|
||||
escaped = false;
|
||||
} else if quoted && *b == b'\\' {
|
||||
escaped = true;
|
||||
} else if *b == b'"' {
|
||||
quoted = !quoted;
|
||||
}
|
||||
}
|
||||
if quoted {
|
||||
return None;
|
||||
}
|
||||
// A size beyond u64 is as large as can be, and refused like any other that is too large.
|
||||
let size = digits.iter().try_fold(0u64, |n, d| n.checked_mul(10)?.checked_add(u64::from(d.wrapping_sub(b'0')))).unwrap_or(u64::MAX);
|
||||
Some((open, size))
|
||||
}
|
||||
|
||||
/// The values of a response. Unbalanced parentheses are forgiven; lists nested too deeply are not.
|
||||
pub(crate) fn values(pieces: Vec<Piece>) -> Result<Vec<Value>, ProtoError> {
|
||||
// The list being filled is on top; the response itself is at the bottom.
|
||||
let mut stack: Vec<Vec<Value>> = vec![Vec::new()];
|
||||
let mut pieces = pieces.into_iter().peekable();
|
||||
while let Some(piece) = pieces.next() {
|
||||
match piece {
|
||||
Piece::Literal(bytes) => push(&mut stack, Value::Str(bytes)),
|
||||
Piece::Text(text) => {
|
||||
// The marker of the literal that follows is not a value.
|
||||
let end = match (pieces.peek(), literal_marker(&text)) {
|
||||
(Some(Piece::Literal(_)), Some((at, _))) => at,
|
||||
_ => text.len(),
|
||||
};
|
||||
scan(text.get(..end).unwrap_or(&[]), &mut stack)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
while stack.len() > 1 {
|
||||
close(&mut stack);
|
||||
}
|
||||
Ok(stack.pop().unwrap_or_default())
|
||||
}
|
||||
|
||||
fn push(stack: &mut [Vec<Value>], value: Value) {
|
||||
if let Some(top) = stack.last_mut() {
|
||||
top.push(value);
|
||||
}
|
||||
}
|
||||
|
||||
fn close(stack: &mut Vec<Vec<Value>>) {
|
||||
if stack.len() > 1
|
||||
&& let Some(list) = stack.pop()
|
||||
{
|
||||
push(stack, Value::List(list));
|
||||
}
|
||||
}
|
||||
|
||||
fn scan(text: &[u8], stack: &mut Vec<Vec<Value>>) -> Result<(), ProtoError> {
|
||||
// A `[` only opens a section (`BODY[HEADER.FIELDS (TO)]`) when a `]` follows somewhere.
|
||||
let last_close = text.iter().rposition(|b| *b == b']');
|
||||
let mut i = 0;
|
||||
// Every arm moves `i` forward.
|
||||
while let Some(&b) = text.get(i) {
|
||||
match b {
|
||||
b' ' | b'\t' => i += 1,
|
||||
b'(' => {
|
||||
if stack.len() > MAX_DEPTH {
|
||||
return Err(ProtoError::Protocol(said!("the server's response is nested too deeply")));
|
||||
}
|
||||
stack.push(Vec::new());
|
||||
i += 1;
|
||||
}
|
||||
b')' => {
|
||||
close(stack);
|
||||
i += 1;
|
||||
}
|
||||
b'"' => {
|
||||
let (string, next) = quoted(text, i + 1);
|
||||
push(stack, Value::Str(string));
|
||||
i = next;
|
||||
}
|
||||
_ => {
|
||||
let fetch_attribute = stack.len() == 2
|
||||
&& stack.first().and_then(|root| root.get(1)).and_then(Value::atom).is_some_and(|word| word.eq_ignore_ascii_case("FETCH"));
|
||||
let end = atom_end(text, i, if fetch_attribute { last_close } else { None });
|
||||
push(stack, Value::Atom(String::from_utf8_lossy(text.get(i..end).unwrap_or(&[])).into_owned()));
|
||||
i = end;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The contents of the quoted string that starts at `start` (after its opening quote), and where
|
||||
/// the text goes on. A string the line ends in the middle of is taken as far as it goes.
|
||||
fn quoted(text: &[u8], start: usize) -> (Vec<u8>, usize) {
|
||||
let mut out = Vec::new();
|
||||
let mut i = start;
|
||||
while let Some(&b) = text.get(i) {
|
||||
match b {
|
||||
b'"' => return (out, i + 1),
|
||||
b'\\' => {
|
||||
out.extend(text.get(i + 1));
|
||||
i += 2;
|
||||
}
|
||||
_ => {
|
||||
out.push(b);
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
(out, i)
|
||||
}
|
||||
|
||||
/// Where the bare word starting at `start` ends: at a space or parenthesis outside `[…]`.
|
||||
fn atom_end(text: &[u8], start: usize, last_close: Option<usize>) -> usize {
|
||||
let mut i = start;
|
||||
while let Some(&b) = text.get(i) {
|
||||
match b {
|
||||
b' ' | b'\t' | b'(' | b')' if i > start => break,
|
||||
b'[' if last_close.is_some_and(|close| close > i)
|
||||
&& text.get(start..i).is_some_and(|prefix| prefix.eq_ignore_ascii_case(b"BODY") || prefix.eq_ignore_ascii_case(b"BODY.PEEK")) =>
|
||||
{
|
||||
let close = text.get(i..).and_then(|rest| rest.iter().position(|b| *b == b']')).unwrap_or(0);
|
||||
i += close + 1;
|
||||
}
|
||||
_ => i += 1,
|
||||
}
|
||||
}
|
||||
i
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum Status {
|
||||
Ok,
|
||||
No,
|
||||
Bad,
|
||||
Bye,
|
||||
Preauth,
|
||||
}
|
||||
|
||||
impl Status {
|
||||
pub(crate) fn parse(word: &str) -> Option<Status> {
|
||||
match word.to_ascii_uppercase().as_str() {
|
||||
"OK" => Some(Status::Ok),
|
||||
"NO" => Some(Status::No),
|
||||
"BAD" => Some(Status::Bad),
|
||||
"BYE" => Some(Status::Bye),
|
||||
"PREAUTH" => Some(Status::Preauth),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `OK [UIDVALIDITY 3857529045] UIDs valid`, tagged or not.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct StatusLine {
|
||||
pub(crate) status: Status,
|
||||
/// The name of the response code, upper-cased (`UIDVALIDITY`); empty when there is none.
|
||||
pub(crate) code: String,
|
||||
/// What follows the name inside the brackets.
|
||||
pub(crate) arg: String,
|
||||
/// Everything after the status word, code included.
|
||||
pub(crate) text: String,
|
||||
}
|
||||
|
||||
impl StatusLine {
|
||||
pub(crate) fn new(status: Status, rest: &str) -> StatusLine {
|
||||
let text = rest.trim();
|
||||
let inner = text.strip_prefix('[').and_then(|r| r.split_once(']')).map_or("", |(inner, _)| inner.trim());
|
||||
let (code, arg) = inner.split_once(' ').unwrap_or((inner, ""));
|
||||
StatusLine { status, code: code.to_ascii_uppercase(), arg: arg.trim().to_string(), text: text.to_string() }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use super::*;
|
||||
|
||||
fn text(s: &str) -> Piece {
|
||||
Piece::Text(s.as_bytes().to_vec())
|
||||
}
|
||||
|
||||
fn atom(s: &str) -> Value {
|
||||
Value::Atom(s.into())
|
||||
}
|
||||
|
||||
fn string(s: &str) -> Value {
|
||||
Value::Str(s.as_bytes().to_vec())
|
||||
}
|
||||
|
||||
fn parse(s: &str) -> Vec<Value> {
|
||||
values(vec![text(s)]).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atoms_strings_and_lists() {
|
||||
assert_eq!(
|
||||
parse(r#"LIST (\HasNoChildren \Sent) "/" "Sent \"old\" \\ mail""#),
|
||||
vec![atom("LIST"), Value::List(vec![atom("\\HasNoChildren"), atom("\\Sent")]), string("/"), string("Sent \"old\" \\ mail")]
|
||||
);
|
||||
assert_eq!(
|
||||
parse("12 FETCH (UID 7 FLAGS ())"),
|
||||
vec![atom("12"), atom("FETCH"), Value::List(vec![atom("UID"), atom("7"), atom("FLAGS"), Value::List(vec![])])]
|
||||
);
|
||||
assert_eq!(
|
||||
parse("a ((b) (c (d))) NIL"),
|
||||
vec![atom("a"), Value::List(vec![Value::List(vec![atom("b")]), Value::List(vec![atom("c"), Value::List(vec![atom("d")])])]), atom("NIL")]
|
||||
);
|
||||
assert_eq!(parse(""), vec![]);
|
||||
assert_eq!(parse(" "), vec![]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sections_are_part_of_the_word() {
|
||||
assert_eq!(
|
||||
parse("12 FETCH (BODY[HEADER.FIELDS (FROM TO)] x BODY[]<0> y)"),
|
||||
vec![atom("12"), atom("FETCH"), Value::List(vec![atom("BODY[HEADER.FIELDS (FROM TO)]"), atom("x"), atom("BODY[]<0>"), atom("y")])]
|
||||
);
|
||||
assert_eq!(parse(r#"LIST () "/" [Gmail]/Trash"#), vec![atom("LIST"), Value::List(vec![]), string("/"), atom("[Gmail]/Trash")]);
|
||||
// A bracket that never closes is an ordinary character.
|
||||
assert_eq!(parse("(a[b c)"), vec![Value::List(vec![atom("a[b"), atom("c")])]);
|
||||
assert_eq!(parse("[[[[ x"), vec![atom("[[[["), atom("x")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn broken_text_is_read_as_far_as_it_goes() {
|
||||
assert_eq!(parse("(a (b"), vec![Value::List(vec![atom("a"), Value::List(vec![atom("b")])])]);
|
||||
assert_eq!(parse("a) b))"), vec![atom("a"), atom("b")]);
|
||||
assert_eq!(parse(r#""never closed"#), vec![string("never closed")]);
|
||||
assert_eq!(parse(r#""ends in a backslash\"#), vec![string("ends in a backslash")]);
|
||||
assert_eq!(parse("a\"b c"), vec![atom("a\"b"), atom("c")]);
|
||||
let deep = "(".repeat(MAX_DEPTH + 5);
|
||||
assert!(matches!(values(vec![text(&deep)]), Err(ProtoError::Protocol(_))));
|
||||
assert!(values(vec![text(&"(".repeat(MAX_DEPTH))]).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn literals_take_the_place_of_their_marker() {
|
||||
let pieces =
|
||||
vec![text("1 FETCH (BODY[] {5}"), Piece::Literal(b"a (\"b".to_vec()), text(" UID 3 X {0}"), Piece::Literal(Vec::new()), text(")")];
|
||||
assert_eq!(
|
||||
values(pieces).unwrap(),
|
||||
vec![atom("1"), atom("FETCH"), Value::List(vec![atom("BODY[]"), string("a (\"b"), atom("UID"), atom("3"), atom("X"), string("")])]
|
||||
);
|
||||
// Without a literal after it, a marker is only a word.
|
||||
assert_eq!(parse("a {5}"), vec![atom("a"), atom("{5}")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn markers() {
|
||||
assert_eq!(literal_marker(b"* 1 FETCH (BODY[] {123}"), Some((18, 123)));
|
||||
assert_eq!(literal_marker(b"{0}"), Some((0, 0)));
|
||||
assert_eq!(literal_marker(b"x {7+}"), Some((2, 7)));
|
||||
assert_eq!(literal_marker(b"\"quoted\" {7}"), Some((9, 7)));
|
||||
assert_eq!(literal_marker(b"x {99999999999999999999999}"), Some((2, u64::MAX)));
|
||||
for not in ["", "}", "{}", "{+}", "{5} ", "{-5}", "{5 }", "\"in a string {5}", "\"a \\\" {5}", "{5}x", "{ 5}", "5}"] {
|
||||
assert_eq!(literal_marker(not.as_bytes()), None, "{not:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_lines() {
|
||||
let s = StatusLine::new(Status::Ok, " [UIDVALIDITY 3857529045] UIDs valid ");
|
||||
assert_eq!((s.code.as_str(), s.arg.as_str(), s.text.as_str()), ("UIDVALIDITY", "3857529045", "[UIDVALIDITY 3857529045] UIDs valid"));
|
||||
let s = StatusLine::new(Status::No, "[alert] disk full");
|
||||
assert_eq!((s.code.as_str(), s.arg.as_str()), ("ALERT", ""));
|
||||
let s = StatusLine::new(Status::Ok, "done [not a code]");
|
||||
assert_eq!((s.code.as_str(), s.text.as_str()), ("", "done [not a code]"));
|
||||
assert_eq!(StatusLine::new(Status::Bad, "[unclosed").code, "");
|
||||
assert_eq!(StatusLine::new(Status::Bad, "").text, "");
|
||||
assert_eq!(Status::parse("preauth"), Some(Status::Preauth));
|
||||
assert_eq!(Status::parse("FETCH"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn values_as_numbers_and_text() {
|
||||
assert_eq!(atom("42").number(), Some(42));
|
||||
assert_eq!(atom("4294967296").number(), None);
|
||||
assert_eq!(atom("-1").number(), None);
|
||||
assert_eq!(string("42").number(), None);
|
||||
assert_eq!(string("Entw&APw-rfe").into_text().as_deref(), Some("Entw&APw-rfe"));
|
||||
assert_eq!(Value::Str(vec![0xFF, b'a']).into_text().as_deref(), Some("\u{FFFD}a"));
|
||||
assert_eq!(Value::List(vec![]).into_text(), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
//! Password authentication from RFC 4616 and RFC 4959, over an already secured stream.
|
||||
|
||||
use super::{Imap, ProtoError, Read, Status, Write, ignore, said};
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
|
||||
impl<S: Read + Write> Imap<S> {
|
||||
/// Prefer the server's advertised PLAIN mechanism; otherwise use LOGIN. The host must
|
||||
/// secure the stream first. Unknown capabilities are queried here; known ones are reused.
|
||||
/// A refusal never triggers a second attempt with another mechanism.
|
||||
pub fn authenticate_password(&mut self, user: &str, password: &str) -> Result<(), ProtoError> {
|
||||
if password.is_empty() {
|
||||
return Err(ProtoError::Auth(said!("enter a password to sign in")));
|
||||
}
|
||||
if user.is_empty() {
|
||||
return Err(ProtoError::Auth(said!("enter a user name to sign in")));
|
||||
}
|
||||
if self.capabilities.is_none() {
|
||||
self.capabilities()?;
|
||||
}
|
||||
if !self.offers("AUTH=PLAIN") {
|
||||
return self.login(user, password);
|
||||
}
|
||||
if user.contains('\0') || password.contains('\0') {
|
||||
return Err(ProtoError::Auth(said!("PLAIN needs a user name and password without NUL characters")));
|
||||
}
|
||||
if user.len().saturating_add(password.len()) > 32 * 1024 {
|
||||
return Err(ProtoError::Auth(said!("the user name or password is too long")));
|
||||
}
|
||||
// Empty authorization identity: the server derives it from the authentication identity.
|
||||
let response = STANDARD.encode(format!("\0{user}\0{password}").as_bytes());
|
||||
self.secrets.add(password);
|
||||
self.secrets.add(&response);
|
||||
let inline = self.offers("SASL-IR");
|
||||
let tag = self.tag();
|
||||
let command = if inline { format!("{tag} AUTHENTICATE PLAIN {response}\r\n") } else { format!("{tag} AUTHENTICATE PLAIN\r\n") };
|
||||
log::trace!("C: {tag} AUTHENTICATE PLAIN{}", if inline { " ***" } else { "" });
|
||||
self.wire.write(command.as_bytes())?;
|
||||
self.wire.flush()?;
|
||||
let mut sent = inline;
|
||||
let mut cancelled = false;
|
||||
let verdict = loop {
|
||||
if let Some(verdict) = self.read_until(&tag, true, &mut ignore)? {
|
||||
break verdict;
|
||||
}
|
||||
if cancelled {
|
||||
return Err(ProtoError::Protocol(said!("the server continued a cancelled password authentication")));
|
||||
}
|
||||
let answer = if sent {
|
||||
cancelled = true;
|
||||
"*"
|
||||
} else {
|
||||
sent = true;
|
||||
response.as_str()
|
||||
};
|
||||
log::trace!("C: ***");
|
||||
self.wire.write(format!("{answer}\r\n").as_bytes())?;
|
||||
self.wire.flush()?;
|
||||
};
|
||||
match verdict.status {
|
||||
_ if cancelled => Err(ProtoError::Auth(said!("the server requested more than one PLAIN response"))),
|
||||
Status::Ok if !sent => Err(ProtoError::Protocol(said!("the server accepted a password that was not sent"))),
|
||||
Status::Ok => {
|
||||
self.authenticated = true;
|
||||
Ok(())
|
||||
}
|
||||
Status::No => Err(ProtoError::Auth(self.refusal(&verdict))),
|
||||
_ => Err(ProtoError::Rejected(self.refusal(&verdict))),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use super::*;
|
||||
use crate::script::Script;
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
|
||||
fn greeting(caps: &str, replies: &str) -> String {
|
||||
format!("* OK [CAPABILITY IMAP4rev1 {caps}] ready\r\n{replies}")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_uses_an_initial_response_only_when_advertised() {
|
||||
let payload = STANDARD.encode(b"\0user@example.org\0invented");
|
||||
for inline in [false, true] {
|
||||
let caps = if inline { "AUTH=PLAIN SASL-IR LOGINDISABLED" } else { "AUTH=PLAIN LOGINDISABLED" };
|
||||
let replies = if inline { "A0001 OK welcome\r\n" } else { "+ continue\r\nA0001 OK welcome\r\n" };
|
||||
let mut stream = Script::new(greeting(caps, replies));
|
||||
let mut imap = Imap::connect(&mut stream).unwrap();
|
||||
imap.authenticate_password("user@example.org", "invented").unwrap();
|
||||
assert!(imap.is_authenticated());
|
||||
let expected = if inline { format!("A0001 AUTHENTICATE PLAIN {payload}\r\n") } else { format!("A0001 AUTHENTICATE PLAIN\r\n{payload}\r\n") };
|
||||
assert_eq!(stream.sent(), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn login_remains_available_when_plain_is_not_advertised() {
|
||||
let mut stream = Script::new(greeting("SASL-IR", "A0001 OK welcome\r\n"));
|
||||
Imap::connect(&mut stream).unwrap().authenticate_password("user@example.org", "invented").unwrap();
|
||||
assert_eq!(stream.sent(), "A0001 LOGIN \"user@example.org\" \"invented\"\r\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_plain_refusal_is_redacted_and_never_retried_with_login() {
|
||||
let payload = STANDARD.encode(b"\0user@example.org\0invented");
|
||||
let mut stream = Script::new(greeting("AUTH=PLAIN SASL-IR", &format!("A0001 NO [AUTHENTICATIONFAILED] invented {payload}\r\n")));
|
||||
let mut imap = Imap::connect(&mut stream).unwrap();
|
||||
let error = imap.authenticate_password("user@example.org", "invented").unwrap_err();
|
||||
assert!(matches!(error, ProtoError::Auth(_)));
|
||||
assert!(!format!("{error:?} {imap:?}").contains("invented"));
|
||||
assert!(!format!("{error:?} {imap:?}").contains(&payload));
|
||||
assert!(!imap.is_authenticated());
|
||||
assert!(!stream.sent().contains(" LOGIN "));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_refusal_before_the_challenge_sends_no_password() {
|
||||
let mut stream = Script::new(greeting("AUTH=PLAIN", "A0001 NO unavailable\r\n"));
|
||||
assert!(Imap::connect(&mut stream).unwrap().authenticate_password("user@example.org", "invented").is_err());
|
||||
assert_eq!(stream.sent(), "A0001 AUTHENTICATE PLAIN\r\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_challenges_cancel_and_never_authenticate() {
|
||||
for replies in ["+ first\r\n+ second\r\nA0001 OK ignored\r\n", "+ first\r\n+ second\r\n+ third\r\n"] {
|
||||
let mut stream = Script::new(greeting("AUTH=PLAIN", replies));
|
||||
let mut imap = Imap::connect(&mut stream).unwrap();
|
||||
assert!(imap.authenticate_password("user@example.org", "invented").is_err());
|
||||
assert!(!imap.is_authenticated());
|
||||
assert!(stream.sent().ends_with("\r\n*\r\n"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn premature_success_and_invalid_credentials_are_refused() {
|
||||
let mut stream = Script::new(greeting("AUTH=PLAIN", "A0001 OK too early\r\n"));
|
||||
assert!(Imap::connect(&mut stream).unwrap().authenticate_password("user@example.org", "invented").is_err());
|
||||
for (user, password) in [("", "invented"), ("user@example.org", ""), ("bad\0user", "invented"), ("user@example.org", "bad\0password")] {
|
||||
let mut stream = Script::new(greeting("AUTH=PLAIN", ""));
|
||||
assert!(Imap::connect(&mut stream).unwrap().authenticate_password(user, password).is_err());
|
||||
assert!(stream.sent().is_empty());
|
||||
}
|
||||
let mut stream = Script::new(greeting("AUTH=PLAIN", ""));
|
||||
assert!(Imap::connect(&mut stream).unwrap().authenticate_password("user@example.org", &"x".repeat(32769)).is_err());
|
||||
assert!(stream.sent().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utf8_and_line_breaks_are_encoded_not_protocol_lines() {
|
||||
let mut stream = Script::new(greeting("auth=plain sasl-ir", "A0001 OK welcome\r\n"));
|
||||
Imap::connect(&mut stream).unwrap().authenticate_password("user@example.org", "päss\r\nword").unwrap();
|
||||
assert_eq!(stream.sent().lines().count(), 1);
|
||||
assert!(stream.sent().contains(&STANDARD.encode("\0user@example.org\0päss\r\nword".as_bytes())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_capabilities_are_queried_once_by_password_authentication() {
|
||||
let mut stream = Script::new("* OK ready\r\n* CAPABILITY IMAP4rev1 AUTH=PLAIN SASL-IR\r\nA0001 OK done\r\nA0002 OK welcome\r\n");
|
||||
Imap::connect(&mut stream).unwrap().authenticate_password("user@example.org", "invented").unwrap();
|
||||
assert!(stream.sent().starts_with("A0001 CAPABILITY\r\nA0002 AUTHENTICATE PLAIN "));
|
||||
assert_eq!(stream.sent().matches(" CAPABILITY").count(), 1);
|
||||
}
|
||||
#[test]
|
||||
fn a_missing_password_has_its_own_message_without_network_commands() {
|
||||
let mut stream = Script::new("* OK ready\r\n");
|
||||
let error = Imap::connect(&mut stream).unwrap().authenticate_password("user@example.org", "").unwrap_err();
|
||||
assert!(error.to_string().contains("enter a password"));
|
||||
assert!(!error.to_string().contains("NUL"));
|
||||
assert!(stream.sent().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_user_name_is_explained_before_any_commands() {
|
||||
for text in ["* OK ready\r\n".to_string(), greeting("AUTH=PLAIN", ""), greeting("", "")] {
|
||||
let mut stream = Script::new(text);
|
||||
let error = Imap::connect(&mut stream).unwrap().authenticate_password("", "invented").unwrap_err();
|
||||
assert!(error.to_string().contains("enter a user name"));
|
||||
assert!(!error.to_string().contains("NUL"));
|
||||
assert!(stream.sent().is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
//! STATUS (RFC 3501 §6.3.10): what a mailbox holds, asked without opening it, for a program to
|
||||
//! tell whether it has changed since it was last read (#276). Many mailboxes are asked in one
|
||||
//! go: the commands are written together and the answers read together (pipelining, RFC 3501
|
||||
//! §5.5), so a hundred mailboxes cost about one round trip, not a hundred.
|
||||
|
||||
use std::io::{Read, Write};
|
||||
|
||||
use super::parse::{Status, Value};
|
||||
use super::{Command, Imap, MAX_COMMAND_TEXT, MAX_QUOTED, Part, Response};
|
||||
use crate::i18n::said;
|
||||
use crate::{ProtoError, for_trace};
|
||||
|
||||
/// Most mailboxes asked in one go: their commands are a few kilobytes at most, so the server
|
||||
/// can't be left waiting to be read while we write.
|
||||
pub const MAX_STATUS_BATCH: usize = 50;
|
||||
|
||||
/// What STATUS says of a mailbox: all four counts, or nothing at all (a server that leaves one
|
||||
/// out gets its mailbox read, never passed over on a guess).
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub struct FolderStatus {
|
||||
/// How many messages there are.
|
||||
pub messages: u32,
|
||||
/// The UID the next message to arrive gets.
|
||||
pub uid_next: u32,
|
||||
pub uid_validity: u32,
|
||||
/// How many are not marked `\Seen`.
|
||||
pub unseen: u32,
|
||||
}
|
||||
|
||||
/// From `STATUS name (MESSAGES 3 UIDNEXT 9 …)`; `None` when it isn't one, or a count is missing.
|
||||
fn status_from(values: Vec<Value>) -> Option<FolderStatus> {
|
||||
let mut values = values.into_iter();
|
||||
if !values.next()?.atom()?.eq_ignore_ascii_case("STATUS") {
|
||||
return None;
|
||||
}
|
||||
values.next()?;
|
||||
let Value::List(pairs) = values.next()? else { return None };
|
||||
let (mut messages, mut uid_next, mut uid_validity, mut unseen) = (None, None, None, None);
|
||||
let mut pairs = pairs.into_iter();
|
||||
while let (Some(name), Some(value)) = (pairs.next(), pairs.next()) {
|
||||
let Some(number) = value.number() else { continue };
|
||||
match name.atom().map(str::to_ascii_uppercase).as_deref() {
|
||||
Some("MESSAGES") => messages = Some(number),
|
||||
Some("UIDNEXT") => uid_next = Some(number),
|
||||
Some("UIDVALIDITY") => uid_validity = Some(number),
|
||||
Some("UNSEEN") => unseen = Some(number),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Some(FolderStatus { messages: messages?, uid_next: uid_next?, uid_validity: uid_validity?, unseen: unseen? })
|
||||
}
|
||||
|
||||
fn command(raw_name: &str) -> Command<'_> {
|
||||
Command::new("STATUS").string(raw_name.as_bytes()).arg("(MESSAGES UIDNEXT UIDVALIDITY UNSEEN)")
|
||||
}
|
||||
|
||||
impl<S: Read + Write> Imap<S> {
|
||||
/// STATUS of one mailbox; `None` when the server refuses or leaves a count out.
|
||||
pub fn status(&mut self, raw_name: &str) -> Result<Option<FolderStatus>, ProtoError> {
|
||||
let mut found = None;
|
||||
let line = self.exchange(command(raw_name), &mut |values| {
|
||||
if found.is_none() {
|
||||
found = status_from(values);
|
||||
}
|
||||
})?;
|
||||
Ok(found.filter(|_| line.status == Status::Ok))
|
||||
}
|
||||
|
||||
/// STATUS of each of `raw_names`, in the same order, `None` for one the server refuses or
|
||||
/// leaves a count out of. At most [`MAX_STATUS_BATCH`] are asked in one go; a name that
|
||||
/// can't go in a quoted string is asked alone.
|
||||
///
|
||||
/// An error is the connection's: the answers read before it are lost with it.
|
||||
pub fn status_many(&mut self, raw_names: &[&str]) -> Result<Vec<Option<FolderStatus>>, ProtoError> {
|
||||
let mut out = Vec::with_capacity(raw_names.len());
|
||||
for batch in raw_names.chunks(MAX_STATUS_BATCH) {
|
||||
let plain = batch.iter().all(|name| name.len() <= MAX_QUOTED && name.bytes().all(|b| matches!(b, b' '..=b'~')));
|
||||
if plain && batch.len() > 1 {
|
||||
out.extend(self.status_batch(batch)?);
|
||||
} else {
|
||||
for name in batch {
|
||||
out.push(self.status(name)?);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// One batch, written together: the answers come in the order of the commands.
|
||||
fn status_batch(&mut self, batch: &[&str]) -> Result<Vec<Option<FolderStatus>>, ProtoError> {
|
||||
let mut tags = Vec::with_capacity(batch.len());
|
||||
let mut out = Vec::with_capacity(batch.len().saturating_mul(64));
|
||||
for name in batch {
|
||||
let command = command(name);
|
||||
if command.unsendable {
|
||||
return Err(ProtoError::Protocol(said!("the text has a character that can't be sent")));
|
||||
}
|
||||
let tag = self.tag();
|
||||
if log::log_enabled!(log::Level::Trace) {
|
||||
log::trace!("C: {tag} {}", for_trace(&self.secrets.hide(&command.for_log())));
|
||||
}
|
||||
out.extend_from_slice(tag.as_bytes());
|
||||
out.push(b' ');
|
||||
for part in &command.parts {
|
||||
if let Part::Text(text) = part {
|
||||
out.extend_from_slice(text);
|
||||
}
|
||||
}
|
||||
out.extend_from_slice(b"\r\n");
|
||||
tags.push(tag);
|
||||
}
|
||||
self.wire.write(&out)?;
|
||||
self.wire.flush()?;
|
||||
self.text_seen = 0;
|
||||
let mut found: Vec<Option<FolderStatus>> = vec![None; batch.len()];
|
||||
let mut answered = 0usize;
|
||||
// Each command brings one STATUS and its tagged answer; the rest is slack for what a
|
||||
// server says unasked.
|
||||
let cap = batch.len().saturating_mul(2).saturating_add(64);
|
||||
for _ in 0..cap {
|
||||
if answered >= tags.len() {
|
||||
break;
|
||||
}
|
||||
match self.read_response()? {
|
||||
Response::GoAhead => return Err(ProtoError::Protocol(said!("the server asked for more than the command had"))),
|
||||
Response::Status { tag: Some(tag), line } => {
|
||||
if tags.get(answered) != Some(&tag) || !matches!(line.status, Status::Ok | Status::No | Status::Bad) {
|
||||
return Err(ProtoError::Protocol(said!("the server answered a command that wasn't sent")));
|
||||
}
|
||||
self.note_status(&line);
|
||||
if line.status != Status::Ok
|
||||
&& let Some(slot) = found.get_mut(answered)
|
||||
{
|
||||
*slot = None;
|
||||
}
|
||||
answered += 1;
|
||||
}
|
||||
Response::Status { tag: None, line } => {
|
||||
if line.status == Status::Bye && !self.leaving {
|
||||
return Err(ProtoError::Io(said!("the server ended the session")));
|
||||
}
|
||||
self.note_status(&line);
|
||||
}
|
||||
Response::Data(values) => {
|
||||
if self.text_seen > MAX_COMMAND_TEXT {
|
||||
return Err(ProtoError::Protocol(said!("the server sent too much in answer to one command")));
|
||||
}
|
||||
self.note_data(&values);
|
||||
if let Some(slot) = found.get_mut(answered)
|
||||
&& slot.is_none()
|
||||
{
|
||||
*slot = status_from(values);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if answered < tags.len() {
|
||||
return Err(ProtoError::Protocol(said!("the server sent too many responses to one command")));
|
||||
}
|
||||
Ok(found)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use proptest::prelude::*;
|
||||
|
||||
use super::*;
|
||||
use crate::script::Script;
|
||||
|
||||
fn status(messages: u32, uid_next: u32, uid_validity: u32, unseen: u32) -> FolderStatus {
|
||||
FolderStatus { messages, uid_next, uid_validity, unseen }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_mailbox_is_asked_its_four_counts() {
|
||||
let mut s = Script::new("* STATUS \"Work\" (MESSAGES 3 UIDNEXT 9 UIDVALIDITY 77 UNSEEN 1)\r\nA0001 OK done\r\n");
|
||||
let got = Imap::resume(&mut s).status("Work").unwrap();
|
||||
assert_eq!(got, Some(status(3, 9, 77, 1)));
|
||||
assert_eq!(s.sent(), "A0001 STATUS \"Work\" (MESSAGES UIDNEXT UIDVALIDITY UNSEEN)\r\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_count_left_out_or_a_refusal_is_no_status() {
|
||||
let mut s = Script::new("* STATUS Work (MESSAGES 3 UIDNEXT 9 UIDVALIDITY 77)\r\nA0001 OK done\r\nA0002 NO [NONEXISTENT] no such mailbox\r\n");
|
||||
let mut imap = Imap::resume(&mut s);
|
||||
assert_eq!(imap.status("Work").unwrap(), None);
|
||||
assert_eq!(imap.status("Gone").unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn many_mailboxes_are_asked_in_one_write() {
|
||||
let mut s = Script::new(
|
||||
"* STATUS A (MESSAGES 1 UIDNEXT 2 UIDVALIDITY 5 UNSEEN 0)\r\nA0001 OK done\r\n\
|
||||
A0002 NO [NONEXISTENT] no\r\n\
|
||||
* STATUS \"C d\" (UNSEEN 4 UIDVALIDITY 7 UIDNEXT 8 MESSAGES 9 HIGHESTMODSEQ 12)\r\nA0003 OK done\r\n",
|
||||
);
|
||||
let got = Imap::resume(&mut s).status_many(&["A", "B", "C d"]).unwrap();
|
||||
assert_eq!(got, vec![Some(status(1, 2, 5, 0)), None, Some(status(9, 8, 7, 4))]);
|
||||
let sent = s.sent();
|
||||
assert_eq!(sent.matches("STATUS").count(), 3);
|
||||
assert!(sent.starts_with("A0001 STATUS \"A\" ") && sent.contains("A0003 STATUS \"C d\" "), "{sent}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn answers_out_of_turn_or_too_many_end_the_session_not_the_program() {
|
||||
let mut s = Script::new("A0002 OK done\r\n");
|
||||
assert!(matches!(Imap::resume(&mut s).status_many(&["A", "B"]), Err(ProtoError::Protocol(_))));
|
||||
let flood = "* 1 EXISTS\r\n".repeat(500);
|
||||
let mut s = Script::new(flood);
|
||||
assert!(matches!(Imap::resume(&mut s).status_many(&["A", "B"]), Err(ProtoError::Protocol(_))));
|
||||
let mut s = Script::new("* BYE going\r\n");
|
||||
assert!(matches!(Imap::resume(&mut s).status_many(&["A", "B"]), Err(ProtoError::Io(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_name_that_needs_a_literal_is_asked_alone() {
|
||||
let mut s = Script::new("+ go\r\n* STATUS x (MESSAGES 1 UIDNEXT 2 UIDVALIDITY 3 UNSEEN 0)\r\nA0001 OK done\r\n");
|
||||
let got = Imap::resume(&mut s).status_many(&["é", "A"]).unwrap_or_default();
|
||||
assert!(got.len() <= 2);
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn arbitrary_server_bytes_never_panic_in_status(server in proptest::collection::vec(any::<u8>(), 0..600), names in 0usize..120) {
|
||||
let list: Vec<String> = (0..names).map(|n| format!("F{n}")).collect();
|
||||
let refs: Vec<&str> = list.iter().map(String::as_str).collect();
|
||||
let mut s = Script::new(&server);
|
||||
let got = Imap::resume(&mut s).status_many(&refs);
|
||||
if let Ok(got) = got {
|
||||
prop_assert_eq!(got.len(), names);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn any_status_line_is_read_or_not(line in "[ -~]{0,80}") {
|
||||
let mut s = Script::new(format!("* STATUS {line}\r\nA0001 OK done\r\n"));
|
||||
let _ = Imap::resume(&mut s).status("x");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,175 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
//! Modified UTF-7, the encoding of mailbox names (RFC 3501 §5.1.3).
|
||||
//!
|
||||
//! Printable ASCII stands for itself, except `&`, which is written `&-`. Everything else is
|
||||
//! UTF-16 in base64 (with `,` in place of `/`, and no padding) between `&` and `-`.
|
||||
|
||||
const DIGITS: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,";
|
||||
|
||||
fn digit(six_bits: u32) -> char {
|
||||
DIGITS.get((six_bits & 0x3F) as usize).map_or('A', |b| char::from(*b))
|
||||
}
|
||||
|
||||
/// A mailbox name as people read it. Text that isn't valid modified UTF-7 is kept as it is.
|
||||
pub fn decode_utf7(name: &str) -> String {
|
||||
let mut out = String::with_capacity(name.len());
|
||||
let mut rest = name;
|
||||
// Every round takes at least the `&` off `rest`.
|
||||
while let Some(at) = rest.find('&') {
|
||||
out.push_str(rest.get(..at).unwrap_or(""));
|
||||
let after = rest.get(at + 1..).unwrap_or("");
|
||||
rest = match after.find('-') {
|
||||
Some(0) => {
|
||||
out.push('&');
|
||||
after.get(1..).unwrap_or("")
|
||||
}
|
||||
Some(end) => match shifted(after.get(..end).unwrap_or("")) {
|
||||
Some(text) => {
|
||||
out.push_str(&text);
|
||||
after.get(end + 1..).unwrap_or("")
|
||||
}
|
||||
// Not base64 after all: an ampersand like any other character.
|
||||
None => {
|
||||
out.push('&');
|
||||
after
|
||||
}
|
||||
},
|
||||
None => {
|
||||
out.push('&');
|
||||
after
|
||||
}
|
||||
};
|
||||
}
|
||||
out.push_str(rest);
|
||||
out
|
||||
}
|
||||
|
||||
/// The text of one base64 run, or `None` when it isn't one.
|
||||
fn shifted(encoded: &str) -> Option<String> {
|
||||
let mut units = Vec::with_capacity(encoded.len() * 3 / 8 + 1);
|
||||
let (mut bits, mut count) = (0u32, 0u32);
|
||||
for b in encoded.bytes() {
|
||||
let value = DIGITS.iter().position(|d| *d == b)? as u32;
|
||||
bits = (bits << 6) | value;
|
||||
count += 6;
|
||||
if count >= 16 {
|
||||
count -= 16;
|
||||
units.push((bits >> count) as u16);
|
||||
bits &= (1 << count) - 1;
|
||||
}
|
||||
}
|
||||
// What is left over is padding: less than one digit's worth.
|
||||
if count >= 6 || units.is_empty() {
|
||||
return None;
|
||||
}
|
||||
String::from_utf16(&units).ok()
|
||||
}
|
||||
|
||||
/// A mailbox name as a server wants it.
|
||||
pub fn encode_utf7(name: &str) -> String {
|
||||
let mut out = String::with_capacity(name.len());
|
||||
let mut pending: Vec<u16> = Vec::new();
|
||||
for c in name.chars() {
|
||||
if matches!(c, ' '..='~') {
|
||||
flush(&mut pending, &mut out);
|
||||
out.push(c);
|
||||
if c == '&' {
|
||||
out.push('-');
|
||||
}
|
||||
} else {
|
||||
let mut units = [0u16; 2];
|
||||
pending.extend_from_slice(c.encode_utf16(&mut units));
|
||||
}
|
||||
}
|
||||
flush(&mut pending, &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
/// Write the characters waiting to be encoded as one base64 run.
|
||||
fn flush(pending: &mut Vec<u16>, out: &mut String) {
|
||||
if pending.is_empty() {
|
||||
return;
|
||||
}
|
||||
out.push('&');
|
||||
let (mut bits, mut count) = (0u32, 0u32);
|
||||
for unit in pending.drain(..) {
|
||||
bits = (bits << 16) | u32::from(unit);
|
||||
count += 16;
|
||||
while count >= 6 {
|
||||
count -= 6;
|
||||
out.push(digit(bits >> count));
|
||||
}
|
||||
bits &= (1 << count) - 1;
|
||||
}
|
||||
if count > 0 {
|
||||
out.push(digit(bits << (6 - count)));
|
||||
}
|
||||
out.push('-');
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use proptest::prelude::*;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_examples_of_the_rfc() {
|
||||
// RFC 3501 §5.1.3.
|
||||
assert_eq!(decode_utf7("~peter/mail/&U,BTFw-/&ZeVnLIqe-"), "~peter/mail/台北/日本語");
|
||||
assert_eq!(encode_utf7("~peter/mail/台北/日本語"), "~peter/mail/&U,BTFw-/&ZeVnLIqe-");
|
||||
assert_eq!(decode_utf7("&Jjo-!"), "☺!");
|
||||
assert_eq!(encode_utf7("☺!"), "&Jjo-!");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_names() {
|
||||
assert_eq!(encode_utf7("INBOX"), "INBOX");
|
||||
assert_eq!(encode_utf7("Tom & Jerry"), "Tom &- Jerry");
|
||||
assert_eq!(decode_utf7("Tom &- Jerry"), "Tom & Jerry");
|
||||
assert_eq!(encode_utf7("Entwürfe"), "Entw&APw-rfe");
|
||||
assert_eq!(decode_utf7("Entw&APw-rfe"), "Entwürfe");
|
||||
assert_eq!(encode_utf7("Gelöschte Elemente"), "Gel&APY-schte Elemente");
|
||||
// Outside the basic plane: a surrogate pair.
|
||||
assert_eq!(decode_utf7(&encode_utf7("📬 Post")), "📬 Post");
|
||||
assert_eq!(encode_utf7("📬"), "&2D3c7A-");
|
||||
// Control characters are encoded too.
|
||||
assert_eq!(encode_utf7("a\tb"), "a&AAk-b");
|
||||
assert_eq!(encode_utf7(""), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn junk_is_kept_as_it_is() {
|
||||
for junk in ["&", "a&b", "&&&", "&!!!-", "&A-", "&AAAAA-x", "R&D", "&2D0-", "&-&", "é&é-"] {
|
||||
let decoded = decode_utf7(junk);
|
||||
assert!(!decoded.is_empty(), "{junk}");
|
||||
}
|
||||
assert_eq!(decode_utf7("R&D"), "R&D");
|
||||
assert_eq!(decode_utf7("&!!!-"), "&!!!-");
|
||||
// A lone surrogate is not text.
|
||||
assert_eq!(decode_utf7("&2D0-"), "&2D0-");
|
||||
assert_eq!(decode_utf7("AT&T-Mobile"), "AT&T-Mobile");
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn any_name_round_trips(name in "\\PC{0,40}") {
|
||||
let encoded = encode_utf7(&name);
|
||||
prop_assert!(encoded.bytes().all(|b| (0x20..=0x7E).contains(&b)), "{encoded}");
|
||||
prop_assert_eq!(decode_utf7(&encoded), name);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn any_text_round_trips_with_control_characters(name in proptest::collection::vec(any::<char>(), 0..40)) {
|
||||
let name: String = name.into_iter().collect();
|
||||
prop_assert_eq!(decode_utf7(&encode_utf7(&name)), name);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoding_never_panics(name in "[&A-Za-z0-9+,\\-é ]{0,60}") {
|
||||
let _ = decode_utf7(&name);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
// SPDX-FileCopyrightText: Craft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
|
||||
//! Mail protocols without sockets.
|
||||
//!
|
||||
//! [`smtp::Smtp`] and [`imap::Imap`] speak SMTP (RFC 5321, 3207, 4954) and IMAP4rev1 (RFC 3501,
|
||||
//! 4315, 4959, 6154, 6851) over any `Read + Write` stream: a TLS socket on the desktop, a scripted
|
||||
//! transcript in the tests. Everything a server sends is treated as hostile: lines, literals and
|
||||
//! response counts are capped, and nothing here panics on junk.
|
||||
//!
|
||||
//! Passwords and access tokens never reach a log or an error text, even when a server echoes
|
||||
//! them back.
|
||||
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
pub mod i18n;
|
||||
pub mod imap;
|
||||
mod line;
|
||||
pub mod smtp;
|
||||
|
||||
#[cfg(test)]
|
||||
mod script;
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use i18n::{Arg, Text};
|
||||
|
||||
pub(crate) use i18n::{for_trace, tidy};
|
||||
|
||||
/// Longest line a server may send.
|
||||
pub const MAX_LINE: usize = 64 * 1024;
|
||||
/// Largest counted block (an IMAP literal, so: the largest message) a server may send.
|
||||
pub const MAX_LITERAL: usize = 64 * 1024 * 1024;
|
||||
/// Longest user name and access token, together, that one sign-in may carry.
|
||||
const MAX_XOAUTH2: usize = 32 * 1024;
|
||||
|
||||
/// What went wrong. Each holds the reason as a [`Text`]: Craft's own words are a template
|
||||
/// displayed in its language ([`said!`]), and what a server or the system said is kept as
|
||||
/// it said it ([`Text::verbatim`]). The English ([`Display`](std::fmt::Display)) is what the CLI, MCP
|
||||
/// and the log read, and it is the same either way.
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ProtoError {
|
||||
Io(Text),
|
||||
Protocol(Text),
|
||||
/// The server refused the command (IMAP NO/BAD, SMTP 4xx/5xx); the text is the server's.
|
||||
Rejected(Text),
|
||||
Auth(Text),
|
||||
}
|
||||
|
||||
impl ProtoError {
|
||||
/// The underlying reason, without its error category prefix.
|
||||
pub fn reason(&self) -> &Text {
|
||||
match self {
|
||||
Self::Io(why) | Self::Protocol(why) | Self::Rejected(why) | Self::Auth(why) => why,
|
||||
}
|
||||
}
|
||||
|
||||
/// The error as the user is told it: what kind of trouble, and why.
|
||||
pub fn said(&self) -> Text {
|
||||
match self {
|
||||
ProtoError::Io(why) => said!("connection problem: {reason}", reason = Arg::said(why.clone())),
|
||||
ProtoError::Protocol(why) => said!("unexpected server response: {reason}", reason = Arg::said(why.clone())),
|
||||
ProtoError::Rejected(why) => said!("the server said: {reason}", reason = Arg::said(why.clone())),
|
||||
ProtoError::Auth(why) => said!("sign-in failed: {reason}", reason = Arg::said(why.clone())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ProtoError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
std::fmt::Display::fmt(&self.said(), f)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ProtoError {}
|
||||
|
||||
/// Secrets: passwords and tokens, never shown in logs or errors.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct Secrets(Vec<zeroize::Zeroizing<String>>);
|
||||
|
||||
impl Secrets {
|
||||
pub(crate) fn add(&mut self, secret: &str) {
|
||||
if !secret.is_empty() && !self.0.iter().any(|s| s.as_str() == secret) {
|
||||
self.0.push(zeroize::Zeroizing::new(secret.to_owned()));
|
||||
// Longer overlapping secrets must be hidden before their prefixes.
|
||||
self.0.sort_by_key(|s| std::cmp::Reverse(s.len()));
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn hide(&self, text: &str) -> String {
|
||||
self.0.iter().fold(text.to_owned(), |text, secret| text.replace(secret.as_str(), "***"))
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a bounded, base64-encoded SASL XOAUTH2 initial response.
|
||||
pub(crate) fn xoauth2(user: &str, token: &str) -> Result<String, ProtoError> {
|
||||
if user.is_empty() || token.is_empty() || user.chars().chain(token.chars()).any(char::is_control) {
|
||||
return Err(ProtoError::Auth(said!("the user name or token has a character that can't be sent")));
|
||||
}
|
||||
if user.len().saturating_add(token.len()) > MAX_XOAUTH2 {
|
||||
return Err(ProtoError::Auth(said!("the user name or token is too long")));
|
||||
}
|
||||
let plain = zeroize::Zeroizing::new(format!("user={user}\x01auth=Bearer {token}\x01\x01"));
|
||||
Ok(STANDARD.encode(plain.as_bytes()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn secrets_redact_overlapping_values() {
|
||||
let mut secrets = Secrets::default();
|
||||
secrets.add("token");
|
||||
secrets.add("token-long");
|
||||
assert_eq!(secrets.hide("token-long token"), "*** ***");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xoauth2_rejects_injection_and_oversized_values() {
|
||||
assert!(xoauth2("user\x01auth=Bearer injected", "token").is_err());
|
||||
assert!(xoauth2("user", &"x".repeat(MAX_XOAUTH2)).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_display() {
|
||||
let err = ProtoError::Io(said!("connection problem"));
|
||||
assert_eq!(err.to_string(), "connection problem: connection problem");
|
||||
assert_eq!(err.reason().to_string(), "connection problem");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_said() {
|
||||
let err = ProtoError::Auth(said!("invalid password"));
|
||||
assert_eq!(err.said().to_string(), "sign-in failed: invalid password");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xoauth2_format() {
|
||||
let result = xoauth2("user@example.com", "token123").unwrap();
|
||||
let result = String::from_utf8(STANDARD.decode(result).unwrap()).unwrap();
|
||||
assert!(result.contains("user=user@example.com"));
|
||||
assert!(result.contains("auth=Bearer token123"));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod source_regression_tests {
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_wire_trace_keeps_keywords_and_no_address_name_or_text() {
|
||||
// (Line, what the trace shows.)
|
||||
for (line, _old_shown) in [
|
||||
("A0001 OK ada@example.org authenticated (Success)\r\n", "A0001 OK … (39 bytes)"),
|
||||
("A0002 LOGIN ***", "A0002 LOGIN ***"),
|
||||
("MAIL FROM:<ada@example.org> SIZE=120", "MAIL FROM: … (26 bytes)"),
|
||||
("RCPT TO:<bob@example.org>", "RCPT TO: … (17 bytes)"),
|
||||
("550 5.1.1 <bob@example.org>: Recipient address rejected", "550 5.1.1 … (45 bytes)"),
|
||||
("250-mx.example.org at your service", "250- … (30 bytes)"),
|
||||
("250-AUTH LOGIN PLAIN XOAUTH2", "250-AUTH LOGIN PLAIN XOAUTH2"),
|
||||
("A0003 SELECT \"Letters from Grace\"", "A0003 SELECT … (20 bytes)"),
|
||||
("* LIST (\\HasNoChildren) \"/\" Grace", "* LIST … (26 bytes)"),
|
||||
("* 12 FETCH (UID 40 FLAGS (\\Seen) BODY[HEADER.FIELDS (SUBJECT)] \"Subject: Quarterly figures\")", "* 12 FETCH … (81 bytes)"),
|
||||
("* 3 EXISTS", "* 3 EXISTS"),
|
||||
("UID FETCH 1:40,42 (UID FLAGS)", "UID FETCH 1:40,42 … (11 bytes)"),
|
||||
("hello there", "(11 bytes)"),
|
||||
("", ""),
|
||||
] {
|
||||
let traced = for_trace(line);
|
||||
assert!(traced.len() <= line.len().saturating_add(32), "{line:?}");
|
||||
for private in ["ada", "bob", "Grace", "Quarterly", "mx.example"] {
|
||||
assert!(!traced.contains(private), "{line:?}: {traced}");
|
||||
}
|
||||
}
|
||||
assert!(for_trace(&format!("A0001 OK {}", "X ".repeat(500))).len() < 160);
|
||||
assert_eq!(for_trace("ünïcödé ä"), "(14 bytes)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uppercase_private_text_is_not_protocol_structure() {
|
||||
assert_eq!(for_trace("550 CUSTOMER ALICE PAYMENT OVERDUE"), "550 … (30 bytes)");
|
||||
assert_eq!(for_trace("A0003 SELECT PRIVATE_MAILBOX"), "A0003 SELECT … (15 bytes)");
|
||||
assert_eq!(for_trace("250-PRIVATE_CUSTOMER_DOMAIN"), "250- … (23 bytes)");
|
||||
assert_eq!(for_trace("A0001 OK PRIVATE CUSTOMER"), "A0001 OK … (16 bytes)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secrets_are_hidden_longest_first() {
|
||||
let mut s = Secrets::default();
|
||||
s.add("hunter2");
|
||||
s.add("hunter2-and-more");
|
||||
s.add("");
|
||||
assert_eq!(s.hide("no hunter2-and-more, no hunter2, nothing"), "no ***, no ***, nothing");
|
||||
assert_eq!(Secrets::default().hide("plain"), "plain");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_xoauth2_response_is_user_and_bearer_token_between_control_a() {
|
||||
let decoded = |payload: String| STANDARD.decode(payload).unwrap();
|
||||
assert_eq!(decoded(xoauth2("ada@example.org", "tok.en-123").unwrap()), b"user=ada@example.org\x01auth=Bearer tok.en-123\x01\x01");
|
||||
// The example of the mechanism's description.
|
||||
assert_eq!(
|
||||
xoauth2("someuser@example.com", "ya29.vF9dft4qmTc2Nvb3RlckBhdHRhdmlzdGEuY29tCg").unwrap(),
|
||||
"dXNlcj1zb21ldXNlckBleGFtcGxlLmNvbQFhdXRoPUJlYXJlciB5YTI5LnZGOWRmdDRxbVRjMk52YjNSbGNrQmhkSFJoZG1semRHRXVZMjl0Q2cBAQ=="
|
||||
);
|
||||
assert_eq!(decoded(xoauth2("ädä", "t").unwrap()), "user=ädä\x01auth=Bearer t\x01\x01".as_bytes());
|
||||
// Nothing that could end the name, the token or the line early.
|
||||
for (user, token) in [("ada\x01auth=Bearer other", "t"), ("ada", "t\x01"), ("ada\r\n", "t"), ("ada", "t\nQUIT"), ("a\0", "t"), ("ada", "\0")]
|
||||
{
|
||||
assert!(matches!(xoauth2(user, token), Err(ProtoError::Auth(_))), "{user:?} {token:?}");
|
||||
}
|
||||
assert!(matches!(xoauth2("ada", ""), Err(ProtoError::Auth(_))));
|
||||
assert!(xoauth2("ada", &"t".repeat(MAX_XOAUTH2 - 3)).is_ok());
|
||||
let err = xoauth2("ada", &"t".repeat(MAX_XOAUTH2)).unwrap_err();
|
||||
assert!(matches!(&err, ProtoError::Auth(why) if why.to_string().len() < 100), "{err:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn server_text_is_tidied() {
|
||||
assert_eq!(tidy(" 5.7.8\tBad\r\n credentials \u{7}"), "5.7.8 Bad credentials");
|
||||
let long = tidy(&"é".repeat(1000));
|
||||
assert_eq!(long.chars().count(), 301);
|
||||
assert!(long.ends_with('…'));
|
||||
assert_eq!(tidy(""), "");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
//! The bounded reader both protocols share: lines and counted blocks from a stream that may be hostile.
|
||||
|
||||
use std::io::{ErrorKind, Read, Write};
|
||||
|
||||
use crate::said;
|
||||
use crate::{MAX_LINE, MAX_LITERAL, ProtoError};
|
||||
|
||||
/// How much is asked of the stream in one read.
|
||||
const CHUNK: usize = 16 * 1024;
|
||||
/// Most bytes of one over-long untagged line read past ([`Wire::read_line_or_skip`]).
|
||||
const MAX_SKIPPED: usize = 4 * 1024 * 1024;
|
||||
/// Reads interrupted in a row before giving up.
|
||||
const MAX_INTERRUPTS: u32 = 1_000;
|
||||
|
||||
/// A stream with the bytes read ahead of what was asked for.
|
||||
pub(crate) struct Wire<S> {
|
||||
stream: S,
|
||||
/// Bytes read from the stream and not handed out yet (from `start`).
|
||||
buf: Vec<u8>,
|
||||
start: usize,
|
||||
// Bytes already searched for a newline in the current pending polling line.
|
||||
poll_scanned: usize,
|
||||
}
|
||||
|
||||
fn closed() -> ProtoError {
|
||||
ProtoError::Io(said!("the server closed the connection"))
|
||||
}
|
||||
|
||||
fn io_error(e: &std::io::Error) -> ProtoError {
|
||||
match e.kind() {
|
||||
ErrorKind::TimedOut | ErrorKind::WouldBlock => ProtoError::Io(said!("the server took too long to answer")),
|
||||
ErrorKind::UnexpectedEof | ErrorKind::ConnectionAborted | ErrorKind::ConnectionReset | ErrorKind::BrokenPipe => closed(),
|
||||
// The system's own words, as it said them.
|
||||
_ => ProtoError::Io(crate::tidy(&e.to_string()).into()),
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> Wire<S> {
|
||||
pub(crate) fn new(stream: S) -> Self {
|
||||
Wire { stream, buf: Vec::new(), start: 0, poll_scanned: 0 }
|
||||
}
|
||||
|
||||
/// Whether the server sent more than has been read so far.
|
||||
pub(crate) fn has_pending(&self) -> bool {
|
||||
self.start < self.buf.len()
|
||||
}
|
||||
|
||||
pub(crate) fn into_inner(self) -> S {
|
||||
self.stream
|
||||
}
|
||||
|
||||
/// The stream itself, for its owner to set up (what a socket waits for, say). Reading from
|
||||
/// it or writing to it here would take the session apart.
|
||||
pub(crate) fn get_mut(&mut self) -> &mut S {
|
||||
&mut self.stream
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: Read + Write> Wire<S> {
|
||||
/// Read some more into the buffer; the end of the stream is an error.
|
||||
fn fill(&mut self) -> Result<(), ProtoError> {
|
||||
self.poll_scanned = 0;
|
||||
if self.start > 0 {
|
||||
self.buf.drain(..self.start.min(self.buf.len()));
|
||||
self.start = 0;
|
||||
}
|
||||
let mut tmp = vec![0u8; CHUNK];
|
||||
let n = read_some(&mut self.stream, &mut tmp)?;
|
||||
self.buf.extend_from_slice(tmp.get(..n).unwrap_or(&[]));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The next line without its line ending (CRLF, or a bare LF).
|
||||
pub(crate) fn read_line(&mut self) -> Result<Vec<u8>, ProtoError> {
|
||||
self.next_line(false)?.ok_or_else(too_long)
|
||||
}
|
||||
|
||||
/// The next line, or `None` for an untagged one (`* …`) longer than [`MAX_LINE`], which is
|
||||
/// read past to its end (at most [`MAX_SKIPPED`] bytes of it) so that the session goes on.
|
||||
/// Any other line that long is an error, as for [`Wire::read_line`].
|
||||
pub(crate) fn read_line_or_skip(&mut self) -> Result<Option<Vec<u8>>, ProtoError> {
|
||||
self.next_line(true)
|
||||
}
|
||||
|
||||
fn next_line(&mut self, skip: bool) -> Result<Option<Vec<u8>>, ProtoError> {
|
||||
self.poll_scanned = 0;
|
||||
// How much of the pending bytes has been searched for a line feed already.
|
||||
let mut scanned = 0usize;
|
||||
// Bytes of an over-long line dropped so far, when one is being read past.
|
||||
let mut skipped: Option<usize> = None;
|
||||
loop {
|
||||
let pending = self.buf.get(self.start..).unwrap_or(&[]);
|
||||
if let Some(i) = pending.get(scanned..).and_then(|rest| rest.iter().position(|b| *b == b'\n')) {
|
||||
let end = scanned.saturating_add(i);
|
||||
let line = pending.get(..end).unwrap_or(&[]);
|
||||
let line = line.strip_suffix(b"\r").unwrap_or(line);
|
||||
let line = if skipped.is_some() {
|
||||
None
|
||||
} else if line.len() > MAX_LINE {
|
||||
if !(skip && line.starts_with(b"* ")) {
|
||||
return Err(too_long());
|
||||
}
|
||||
None
|
||||
} else {
|
||||
Some(line.to_vec())
|
||||
};
|
||||
self.start = self.start.saturating_add(end).saturating_add(1);
|
||||
return Ok(line);
|
||||
}
|
||||
scanned = pending.len();
|
||||
// The limit is on the line itself; a CR may still come on top.
|
||||
if scanned > MAX_LINE + 1 {
|
||||
let dropped = match skipped {
|
||||
Some(n) => n,
|
||||
None if skip && pending.starts_with(b"* ") => 0,
|
||||
None => return Err(too_long()),
|
||||
};
|
||||
let dropped = dropped.saturating_add(scanned);
|
||||
if dropped > MAX_SKIPPED {
|
||||
return Err(too_long());
|
||||
}
|
||||
skipped = Some(dropped);
|
||||
self.start = self.buf.len();
|
||||
scanned = 0;
|
||||
}
|
||||
// Each round reads at least one byte, so this ends at the line feed, the cap or the end of the stream.
|
||||
self.fill()?;
|
||||
}
|
||||
}
|
||||
|
||||
/// The next line after at most one underlying read; `None` for a partial line or when
|
||||
/// the stream times out (IDLE waits like this, a second at a time, so that its owner can
|
||||
/// look at the clock and at whether it was told to stop). Bytes of a line that came so far are
|
||||
/// kept for the next call. A line over [`MAX_LINE`] is an error, as for [`Wire::read_line`].
|
||||
pub(crate) fn poll_line(&mut self) -> Result<Option<Vec<u8>>, ProtoError> {
|
||||
let mut read_once = false;
|
||||
loop {
|
||||
let pending = self.buf.get(self.start..).unwrap_or(&[]);
|
||||
let scanned = self.poll_scanned.min(pending.len());
|
||||
if let Some(offset) = pending.get(scanned..).unwrap_or(&[]).iter().position(|b| *b == b'\n') {
|
||||
let end = scanned.saturating_add(offset);
|
||||
let line = pending.get(..end).unwrap_or(&[]);
|
||||
let line = line.strip_suffix(b"\r").unwrap_or(line);
|
||||
if line.len() > MAX_LINE {
|
||||
return Err(too_long());
|
||||
}
|
||||
let line = line.to_vec();
|
||||
self.start = self.start.saturating_add(end).saturating_add(1);
|
||||
self.poll_scanned = 0;
|
||||
return Ok(Some(line));
|
||||
}
|
||||
if pending.len() > MAX_LINE + 1 {
|
||||
return Err(too_long());
|
||||
}
|
||||
self.poll_scanned = pending.len();
|
||||
// One underlying read per poll lets the caller check cancellation even when
|
||||
// a hostile peer trickles bytes without ever triggering the read timeout.
|
||||
if read_once {
|
||||
return Ok(None);
|
||||
}
|
||||
if self.start > 0 {
|
||||
self.buf.drain(..self.start.min(self.buf.len()));
|
||||
self.start = 0;
|
||||
}
|
||||
let mut tmp = vec![0u8; CHUNK];
|
||||
read_once = true;
|
||||
match self.stream.read(&mut tmp) {
|
||||
Ok(0) => return Err(closed()),
|
||||
Ok(n) => self.buf.extend_from_slice(tmp.get(..n.min(tmp.len())).unwrap_or(&[])),
|
||||
Err(e) if matches!(e.kind(), ErrorKind::TimedOut | ErrorKind::WouldBlock | ErrorKind::Interrupted) => return Ok(None),
|
||||
Err(e) => return Err(io_error(&e)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Exactly `n` bytes (at most [`MAX_LITERAL`]).
|
||||
pub(crate) fn read_block(&mut self, n: usize) -> Result<Vec<u8>, ProtoError> {
|
||||
self.poll_scanned = 0;
|
||||
if n > MAX_LITERAL {
|
||||
return Err(ProtoError::Protocol(said!("the server sent {n} bytes in one piece, more than MailCraft accepts", n)));
|
||||
}
|
||||
// The size is the server's word: memory is taken as the bytes arrive.
|
||||
let mut out = Vec::with_capacity(n.min(CHUNK));
|
||||
let pending = self.buf.get(self.start..).unwrap_or(&[]);
|
||||
let take = pending.len().min(n);
|
||||
out.extend_from_slice(pending.get(..take).unwrap_or(&[]));
|
||||
self.start = self.start.saturating_add(take);
|
||||
let mut tmp = vec![0u8; CHUNK];
|
||||
while out.len() < n {
|
||||
let want = (n - out.len()).min(tmp.len());
|
||||
let got = read_some(&mut self.stream, tmp.get_mut(..want).unwrap_or(&mut []))?;
|
||||
out.extend_from_slice(tmp.get(..got.min(want)).unwrap_or(&[]));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub(crate) fn write(&mut self, mut bytes: &[u8]) -> Result<(), ProtoError> {
|
||||
// Not `write_all`: that retries an interrupted write for ever.
|
||||
let mut interrupts = 0;
|
||||
while !bytes.is_empty() {
|
||||
match self.stream.write(bytes) {
|
||||
Ok(0) => return Err(closed()),
|
||||
Ok(n) => bytes = bytes.get(n..).unwrap_or(&[]),
|
||||
Err(e) if e.kind() == ErrorKind::Interrupted && interrupts < MAX_INTERRUPTS => interrupts += 1,
|
||||
Err(e) => return Err(io_error(&e)),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn flush(&mut self) -> Result<(), ProtoError> {
|
||||
self.stream.flush().map_err(|e| io_error(&e))
|
||||
}
|
||||
}
|
||||
|
||||
fn too_long() -> ProtoError {
|
||||
ProtoError::Protocol(said!("the server sent a line longer than {kb} KB", kb = MAX_LINE / 1024))
|
||||
}
|
||||
|
||||
/// One read of at least one byte; the end of the stream is an error.
|
||||
fn read_some<S: Read>(stream: &mut S, buf: &mut [u8]) -> Result<usize, ProtoError> {
|
||||
let mut interrupts = 0;
|
||||
loop {
|
||||
match stream.read(buf) {
|
||||
Ok(0) => return Err(closed()),
|
||||
// A stream that claims more than the buffer holds is lying.
|
||||
Ok(n) => return Ok(n.min(buf.len())),
|
||||
Err(e) if e.kind() == ErrorKind::Interrupted && interrupts < MAX_INTERRUPTS => interrupts += 1,
|
||||
Err(e) => return Err(io_error(&e)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use super::*;
|
||||
use crate::script::{Broken, Script};
|
||||
|
||||
#[test]
|
||||
fn polling_reads_once_and_retains_linear_scan_progress() {
|
||||
struct ByteAtATime {
|
||||
bytes: std::io::Cursor<Vec<u8>>,
|
||||
reads: usize,
|
||||
}
|
||||
impl Read for ByteAtATime {
|
||||
fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
|
||||
self.reads += 1;
|
||||
self.bytes.read(&mut out[..1])
|
||||
}
|
||||
}
|
||||
impl Write for ByteAtATime {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
Ok(bytes.len())
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
let mut wire = Wire::new(ByteAtATime { bytes: std::io::Cursor::new(b"abc\n".to_vec()), reads: 0 });
|
||||
for scanned in 1..=3 {
|
||||
assert_eq!(wire.poll_line().unwrap(), None);
|
||||
assert_eq!(wire.poll_scanned, scanned);
|
||||
assert_eq!(wire.get_mut().reads, scanned);
|
||||
}
|
||||
assert_eq!(wire.poll_line().unwrap(), Some(b"abc".to_vec()));
|
||||
assert_eq!(wire.get_mut().reads, 4);
|
||||
assert_eq!(wire.poll_scanned, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lines_and_blocks_arrive_in_pieces() {
|
||||
let mut s = Script::new("first\r\nsecond\nab\r\ncd\r\nlast\r\n");
|
||||
let mut w = Wire::new(&mut s);
|
||||
assert_eq!(w.read_line().unwrap(), b"first");
|
||||
assert_eq!(w.read_line().unwrap(), b"second");
|
||||
assert_eq!(w.read_block(6).unwrap(), b"ab\r\ncd");
|
||||
assert_eq!(w.read_block(0).unwrap(), b"");
|
||||
assert_eq!(w.read_line().unwrap(), b"");
|
||||
assert_eq!(w.read_line().unwrap(), b"last");
|
||||
assert!(!w.has_pending());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_end_of_the_stream_is_an_error() {
|
||||
let mut s = Script::new("no line ending");
|
||||
let mut w = Wire::new(&mut s);
|
||||
assert_eq!(w.read_line(), Err(ProtoError::Io(said!("the server closed the connection"))));
|
||||
let mut s = Script::new("short");
|
||||
assert!(matches!(Wire::new(&mut s).read_block(10), Err(ProtoError::Io(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_lines_and_large_blocks_are_refused() {
|
||||
let mut s = Script::whole(vec![b'x'; MAX_LINE + CHUNK * 2]);
|
||||
assert!(matches!(Wire::new(&mut s).read_line(), Err(ProtoError::Protocol(_))));
|
||||
// A line of exactly the limit is fine.
|
||||
let mut ok = vec![b'x'; MAX_LINE];
|
||||
ok.extend_from_slice(b"\r\n");
|
||||
let mut s = Script::whole(ok);
|
||||
assert_eq!(Wire::new(&mut s).read_line().unwrap().len(), MAX_LINE);
|
||||
// The claim alone is refused: nothing is read or allocated.
|
||||
let mut s = Script::new("");
|
||||
assert!(matches!(Wire::new(&mut s).read_block(MAX_LITERAL + 1), Err(ProtoError::Protocol(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stream_errors_become_connection_problems() {
|
||||
let mut w = Wire::new(Broken(ErrorKind::TimedOut));
|
||||
assert_eq!(w.read_line(), Err(ProtoError::Io(said!("the server took too long to answer"))));
|
||||
assert!(matches!(w.write(b"x"), Err(ProtoError::Io(_))));
|
||||
// A stream that is interrupted for ever hangs neither the reader nor the writer.
|
||||
assert!(matches!(Wire::new(Broken(ErrorKind::Interrupted)).read_line(), Err(ProtoError::Io(_))));
|
||||
assert!(matches!(Wire::new(Broken(ErrorKind::Interrupted)).write(b"x"), Err(ProtoError::Io(_))));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
//! Test streams: a scripted server, and a stream that only fails.
|
||||
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use std::io::{Error, ErrorKind, Read, Write};
|
||||
|
||||
/// A canned server: reads hand out what the server "says" (a few bytes at a time, so every
|
||||
/// boundary gets split somewhere), writes are kept for the test to look at.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct Script {
|
||||
input: Vec<u8>,
|
||||
pos: usize,
|
||||
step: usize,
|
||||
trickle: bool,
|
||||
pub(crate) written: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Script {
|
||||
/// Hands the bytes out 1–7 at a time.
|
||||
pub(crate) fn new(server: impl AsRef<[u8]>) -> Script {
|
||||
Script { input: server.as_ref().to_vec(), pos: 0, step: 0, trickle: true, written: Vec::new() }
|
||||
}
|
||||
|
||||
/// Hands out as much as each read asks for.
|
||||
pub(crate) fn whole(server: impl AsRef<[u8]>) -> Script {
|
||||
Script { trickle: false, ..Script::new(server) }
|
||||
}
|
||||
|
||||
/// What the client wrote, as text.
|
||||
pub(crate) fn sent(&self) -> String {
|
||||
String::from_utf8_lossy(&self.written).into_owned()
|
||||
}
|
||||
|
||||
/// Whether the client read everything the script had.
|
||||
pub(crate) fn finished(&self) -> bool {
|
||||
self.pos >= self.input.len()
|
||||
}
|
||||
}
|
||||
|
||||
impl Read for Script {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
let left = &self.input[self.pos.min(self.input.len())..];
|
||||
let mut n = left.len().min(buf.len());
|
||||
if self.trickle {
|
||||
n = n.min([3, 1, 7, 2, 5, 4, 6][self.step % 7]);
|
||||
self.step += 1;
|
||||
}
|
||||
buf[..n].copy_from_slice(&left[..n]);
|
||||
self.pos += n;
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for Script {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
self.written.extend_from_slice(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// A stream whose every read and write fails.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct Broken(pub(crate) ErrorKind);
|
||||
|
||||
impl Read for Broken {
|
||||
fn read(&mut self, _: &mut [u8]) -> std::io::Result<usize> {
|
||||
Err(Error::from(self.0))
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for Broken {
|
||||
fn write(&mut self, _: &[u8]) -> std::io::Result<usize> {
|
||||
Err(Error::from(self.0))
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Err(Error::from(self.0))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,419 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
//! An SMTP client (RFC 5321) with STARTTLS (RFC 3207) and AUTH PLAIN / LOGIN / XOAUTH2 (RFC 4954).
|
||||
|
||||
use std::io::{Read, Write};
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
|
||||
use crate::i18n::{Arg, Text};
|
||||
use crate::line::Wire;
|
||||
use crate::said;
|
||||
use crate::{ProtoError, Secrets, for_trace, tidy, xoauth2};
|
||||
|
||||
/// Most lines one reply may have.
|
||||
const MAX_REPLY_LINES: usize = 256;
|
||||
/// Longest command line without its line ending (RFC 5321 §4.5.3.1.4).
|
||||
const MAX_COMMAND: usize = 510;
|
||||
/// Longest address (RFC 5321 §4.5.3.1.3 allows 256; some servers take more).
|
||||
const MAX_ADDRESS: usize = 320;
|
||||
/// Most recipients one message may have.
|
||||
const MAX_RECIPIENTS: usize = 10_000;
|
||||
|
||||
/// An SMTP session on a stream.
|
||||
pub struct Smtp<S> {
|
||||
wire: Wire<S>,
|
||||
ext: Extensions,
|
||||
secrets: Secrets,
|
||||
}
|
||||
|
||||
/// What the server offered in its answer to EHLO.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Extensions {
|
||||
pub starttls: bool,
|
||||
pub auth_plain: bool,
|
||||
pub auth_login: bool,
|
||||
/// The server takes an OAuth access token instead of a password (SASL XOAUTH2).
|
||||
pub auth_xoauth2: bool,
|
||||
/// The largest message the server takes; `Some(0)` when it names no limit.
|
||||
pub size: Option<u64>,
|
||||
pub eight_bit_mime: bool,
|
||||
}
|
||||
|
||||
/// Neither the stream nor anything secret is shown.
|
||||
impl<S> std::fmt::Debug for Smtp<S> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("Smtp").field("extensions", &self.ext).finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// One reply: `250-first`, `250 last`.
|
||||
struct Reply {
|
||||
code: u16,
|
||||
lines: Vec<String>,
|
||||
}
|
||||
|
||||
impl Reply {
|
||||
fn class(&self) -> u16 {
|
||||
self.code / 100
|
||||
}
|
||||
|
||||
/// `550 5.1.1 No such user`, as the server wrote it.
|
||||
fn text(&self) -> String {
|
||||
format!("{} {}", self.code, self.lines.join(" "))
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: Read + Write> Smtp<S> {
|
||||
/// Read the 220 greeting.
|
||||
pub fn connect(stream: S) -> Result<Self, ProtoError> {
|
||||
let mut smtp = Self::resume(stream);
|
||||
let greeting = smtp.reply()?;
|
||||
if greeting.code == 220 { Ok(smtp) } else { Err(smtp.refused(&greeting)) }
|
||||
}
|
||||
|
||||
/// Continue on a stream that already greeted (after the caller wrapped it in TLS).
|
||||
pub fn resume(stream: S) -> Self {
|
||||
Smtp { wire: Wire::new(stream), ext: Extensions::default(), secrets: Secrets::default() }
|
||||
}
|
||||
|
||||
/// What the last [`hello`](Self::hello) learned.
|
||||
pub fn extensions(&self) -> &Extensions {
|
||||
&self.ext
|
||||
}
|
||||
|
||||
/// The stream the session runs on, for its owner to set up between commands (how long a
|
||||
/// socket waits, say). Reading from it or writing to it would take the session apart.
|
||||
pub fn get_mut(&mut self) -> &mut S {
|
||||
self.wire.get_mut()
|
||||
}
|
||||
|
||||
/// EHLO (falling back to HELO); remembers and returns what the server offers.
|
||||
pub fn hello(&mut self, client_name: &str) -> Result<Extensions, ProtoError> {
|
||||
// A name that could break the command line is not sent.
|
||||
let name = if is_word(client_name) { client_name } else { "localhost" };
|
||||
let reply = self.command(&format!("EHLO {name}"))?;
|
||||
if reply.code == 250 {
|
||||
self.ext = extensions(&reply.lines);
|
||||
return Ok(self.ext.clone());
|
||||
}
|
||||
if reply.class() != 5 {
|
||||
return Err(self.refused(&reply));
|
||||
}
|
||||
// A server from before 1995 (or one hiding its extensions).
|
||||
let reply = self.command(&format!("HELO {name}"))?;
|
||||
if reply.code != 250 {
|
||||
return Err(self.refused(&reply));
|
||||
}
|
||||
self.ext = Extensions::default();
|
||||
Ok(self.ext.clone())
|
||||
}
|
||||
|
||||
/// Send STARTTLS and hand the stream back for the caller to wrap; call `resume` + `hello` afterwards.
|
||||
pub fn starttls(mut self) -> Result<S, ProtoError> {
|
||||
let reply = self.command("STARTTLS")?;
|
||||
if reply.code != 220 {
|
||||
return Err(self.refused(&reply));
|
||||
}
|
||||
// Anything after the go-ahead was slipped in before encryption started (RFC 3207 §6).
|
||||
if self.wire.has_pending() {
|
||||
return Err(ProtoError::Protocol(said!("the server sent data before the secure connection started")));
|
||||
}
|
||||
Ok(self.wire.into_inner())
|
||||
}
|
||||
|
||||
/// AUTH PLAIN when offered, else AUTH LOGIN. 535 and friends → `ProtoError::Auth`.
|
||||
pub fn auth(&mut self, user: &str, password: &str) -> Result<(), ProtoError> {
|
||||
if user.contains('\0') || password.contains('\0') {
|
||||
return Err(ProtoError::Auth(said!("the user name or password has a character that can't be sent")));
|
||||
}
|
||||
self.secrets.add(password);
|
||||
if self.ext.auth_plain {
|
||||
let payload = STANDARD.encode(format!("\0{user}\0{password}"));
|
||||
self.secrets.add(&payload);
|
||||
let line = format!("AUTH PLAIN {payload}");
|
||||
let reply = if line.len() <= MAX_COMMAND {
|
||||
self.secret_command(&line, "AUTH PLAIN ***")?
|
||||
} else {
|
||||
// Too long for one line: wait for the server to ask (RFC 4954 §4).
|
||||
let ask = self.command("AUTH PLAIN")?;
|
||||
if ask.code != 334 {
|
||||
return Err(self.sign_in_failed(&ask));
|
||||
}
|
||||
self.secret_command(&payload, "***")?
|
||||
};
|
||||
return self.signed_in(&reply);
|
||||
}
|
||||
if self.ext.auth_login {
|
||||
let ask = self.command("AUTH LOGIN")?;
|
||||
if ask.code != 334 {
|
||||
return Err(self.sign_in_failed(&ask));
|
||||
}
|
||||
let ask = self.secret_command(&STANDARD.encode(user), "***")?;
|
||||
if ask.code != 334 {
|
||||
return Err(self.sign_in_failed(&ask));
|
||||
}
|
||||
let encoded = STANDARD.encode(password);
|
||||
self.secrets.add(&encoded);
|
||||
let reply = self.secret_command(&encoded, "***")?;
|
||||
return self.signed_in(&reply);
|
||||
}
|
||||
Err(ProtoError::Auth(said!("the server offers no way to sign in with a password")))
|
||||
}
|
||||
|
||||
/// AUTH XOAUTH2: an OAuth access token in place of a password. A server that doesn't offer
|
||||
/// it isn't shown the token. 535 and friends → `ProtoError::Auth`.
|
||||
pub fn auth_xoauth2(&mut self, user: &str, token: &str) -> Result<(), ProtoError> {
|
||||
let payload = xoauth2(user, token)?;
|
||||
if !self.ext.auth_xoauth2 {
|
||||
return Err(ProtoError::Auth(said!("the server offers no way to sign in with an access token")));
|
||||
}
|
||||
self.secrets.add(token);
|
||||
self.secrets.add(&payload);
|
||||
let line = format!("AUTH XOAUTH2 {payload}");
|
||||
let reply = if line.len() <= MAX_COMMAND {
|
||||
self.secret_command(&line, "AUTH XOAUTH2 ***")?
|
||||
} else {
|
||||
// Too long for one line, as tokens mostly are: wait for the server to ask (RFC 4954 §4).
|
||||
let ask = self.command("AUTH XOAUTH2")?;
|
||||
if ask.code != 334 {
|
||||
return Err(self.sign_in_failed(&ask));
|
||||
}
|
||||
self.secret_command(&payload, "***")?
|
||||
};
|
||||
match reply.code {
|
||||
235 => Ok(()),
|
||||
334 => {
|
||||
// A refusal comes in two parts: the reasons (JSON in base64), and, once an empty
|
||||
// line has acknowledged them, the verdict.
|
||||
let verdict = self.command("")?;
|
||||
match verdict.code {
|
||||
235 => Ok(()),
|
||||
334 => {
|
||||
let _ = self.command("*");
|
||||
Err(ProtoError::Auth(said!("the server asked for more than a user name and access token")))
|
||||
}
|
||||
_ => Err(self.sign_in_failed(&verdict)),
|
||||
}
|
||||
}
|
||||
_ => Err(self.sign_in_failed(&reply)),
|
||||
}
|
||||
}
|
||||
|
||||
/// MAIL FROM, RCPT TO (every recipient must be accepted), DATA with CRLF normalisation and dot-stuffing.
|
||||
pub fn send(&mut self, from: &str, recipients: &[&str], message: &[u8]) -> Result<(), ProtoError> {
|
||||
// Nothing is sent unless every address is safe to put on a command line.
|
||||
if !from.is_empty() && !is_address(from) {
|
||||
return Err(bad_address(from));
|
||||
}
|
||||
if recipients.is_empty() {
|
||||
return Err(ProtoError::Protocol(said!("the message has no recipients")));
|
||||
}
|
||||
if recipients.len() > MAX_RECIPIENTS {
|
||||
return Err(ProtoError::Protocol(said!("the message has too many recipients")));
|
||||
}
|
||||
if let Some(bad) = recipients.iter().find(|r| !is_address(r)) {
|
||||
return Err(bad_address(bad));
|
||||
}
|
||||
let (data, size) = data_block(message);
|
||||
let mut mail = format!("MAIL FROM:<{from}>");
|
||||
if self.ext.size.is_some() {
|
||||
mail.push_str(&format!(" SIZE={size}"));
|
||||
}
|
||||
if self.ext.eight_bit_mime && !message.is_ascii() {
|
||||
mail.push_str(" BODY=8BITMIME");
|
||||
}
|
||||
let reply = self.command(&mail)?;
|
||||
if reply.class() != 2 {
|
||||
return Err(self.refused(&reply));
|
||||
}
|
||||
for rcpt in recipients {
|
||||
let reply = self.command(&format!("RCPT TO:<{rcpt}>"))?;
|
||||
if reply.class() != 2 {
|
||||
let refused = match self.refused(&reply) {
|
||||
// (The address, then what the server said of it: no words of MailCraft's to say.)
|
||||
ProtoError::Rejected(text) => {
|
||||
ProtoError::Rejected(Text::new("{rcpt}: {text}", vec![("rcpt", Arg::Plain(rcpt.to_string())), ("text", Arg::said(text))]))
|
||||
}
|
||||
other => other,
|
||||
};
|
||||
return Err(self.give_up(refused));
|
||||
}
|
||||
}
|
||||
let reply = self.command("DATA")?;
|
||||
if reply.code != 354 {
|
||||
let refused = self.refused(&reply);
|
||||
return Err(self.give_up(refused));
|
||||
}
|
||||
log::trace!("C: ({size} bytes)");
|
||||
self.wire.write(&data)?;
|
||||
self.wire.flush()?;
|
||||
let reply = self.reply()?;
|
||||
if reply.class() != 2 {
|
||||
return Err(self.refused(&reply));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// QUIT. A server that hangs up without answering has quit all the same.
|
||||
pub fn quit(mut self) -> Result<(), ProtoError> {
|
||||
match self.command("QUIT") {
|
||||
Ok(reply) if reply.class() == 2 => Ok(()),
|
||||
Ok(reply) => Err(self.refused(&reply)),
|
||||
// A server that hangs up instead of saying goodbye has quit all the same.
|
||||
Err(ProtoError::Io(_)) => Ok(()),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Forget the message being composed, so the session can go on; hands the error back.
|
||||
fn give_up(&mut self, error: ProtoError) -> ProtoError {
|
||||
let _ = self.command("RSET");
|
||||
error
|
||||
}
|
||||
|
||||
fn command(&mut self, line: &str) -> Result<Reply, ProtoError> {
|
||||
self.secret_command(line, line)
|
||||
}
|
||||
|
||||
/// Send `line`; the log gets `shown` instead.
|
||||
fn secret_command(&mut self, line: &str, shown: &str) -> Result<Reply, ProtoError> {
|
||||
log::trace!("C: {}", for_trace(&self.secrets.hide(shown)));
|
||||
let mut out = Vec::with_capacity(line.len() + 2);
|
||||
out.extend_from_slice(line.as_bytes());
|
||||
out.extend_from_slice(b"\r\n");
|
||||
self.wire.write(&out)?;
|
||||
self.wire.flush()?;
|
||||
self.reply()
|
||||
}
|
||||
|
||||
fn reply(&mut self) -> Result<Reply, ProtoError> {
|
||||
let mut reply = Reply { code: 0, lines: Vec::new() };
|
||||
for n in 0..MAX_REPLY_LINES {
|
||||
let raw = self.wire.read_line()?;
|
||||
let line = String::from_utf8_lossy(&raw);
|
||||
log::trace!("S: {}", for_trace(&self.secrets.hide(&line)));
|
||||
// Three digits, then `-` (more lines follow), a space or nothing (the last line).
|
||||
let code = line.get(..3).filter(|d| d.bytes().all(|b| b.is_ascii_digit())).and_then(|d| d.parse::<u16>().ok());
|
||||
let rest = line.get(3..).unwrap_or("");
|
||||
let more = match (code, rest.chars().next()) {
|
||||
(Some(code), Some('-')) if n == 0 || code == reply.code => true,
|
||||
(Some(code), Some(' ') | None) if n == 0 || code == reply.code => false,
|
||||
_ => return Err(ProtoError::Protocol(said!("\"{line}\" is not an SMTP reply", line = tidy(&self.secrets.hide(&line))))),
|
||||
};
|
||||
reply.code = code.unwrap_or(0);
|
||||
reply.lines.push(rest.get(1..).unwrap_or("").to_string());
|
||||
if !more {
|
||||
return Ok(reply);
|
||||
}
|
||||
}
|
||||
Err(ProtoError::Protocol(said!("the server's reply has too many lines")))
|
||||
}
|
||||
|
||||
/// The error for a reply that isn't the one hoped for.
|
||||
fn refused(&self, reply: &Reply) -> ProtoError {
|
||||
// Hidden before it is shortened: a secret cut in half would no longer be recognised.
|
||||
let text = tidy(&self.secrets.hide(&reply.text()));
|
||||
if matches!(reply.class(), 4 | 5) { ProtoError::Rejected(text.into()) } else { ProtoError::Protocol(text.into()) }
|
||||
}
|
||||
|
||||
fn sign_in_failed(&mut self, reply: &Reply) -> ProtoError {
|
||||
match self.refused(reply) {
|
||||
ProtoError::Rejected(text) => ProtoError::Auth(text),
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn signed_in(&mut self, reply: &Reply) -> Result<(), ProtoError> {
|
||||
match reply.code {
|
||||
235 => Ok(()),
|
||||
334 => {
|
||||
// Another question we have no answer to: cancel the exchange (RFC 4954 §4).
|
||||
let _ = self.command("*");
|
||||
Err(ProtoError::Auth(said!("the server asked for more than a user name and password")))
|
||||
}
|
||||
_ => Err(self.sign_in_failed(reply)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The extensions named in an EHLO reply (its first line is the server's name).
|
||||
fn extensions(lines: &[String]) -> Extensions {
|
||||
let mut ext = Extensions::default();
|
||||
for line in lines.iter().skip(1) {
|
||||
let upper = line.to_ascii_uppercase();
|
||||
// `AUTH=PLAIN LOGIN` is how some old servers spell `AUTH PLAIN LOGIN`.
|
||||
let mut words = upper.split([' ', '\t', '=']).filter(|w| !w.is_empty());
|
||||
match words.next() {
|
||||
Some("STARTTLS") => ext.starttls = true,
|
||||
Some("8BITMIME") => ext.eight_bit_mime = true,
|
||||
Some("SIZE") => ext.size = Some(words.next().and_then(|n| n.parse().ok()).unwrap_or(0)),
|
||||
Some("AUTH") => {
|
||||
for mechanism in words {
|
||||
match mechanism {
|
||||
"PLAIN" => ext.auth_plain = true,
|
||||
"LOGIN" => ext.auth_login = true,
|
||||
"XOAUTH2" => ext.auth_xoauth2 = true,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
ext
|
||||
}
|
||||
|
||||
/// Visible ASCII without spaces: safe as one word of a command line.
|
||||
fn is_word(s: &str) -> bool {
|
||||
!s.is_empty() && s.len() <= 255 && s.bytes().all(|b| b.is_ascii_graphic())
|
||||
}
|
||||
|
||||
/// An address that can go between `<` and `>` as it is.
|
||||
fn is_address(s: &str) -> bool {
|
||||
is_word(s) && s.len() <= MAX_ADDRESS && !s.contains(['<', '>'])
|
||||
}
|
||||
|
||||
fn bad_address(address: &str) -> ProtoError {
|
||||
ProtoError::Protocol(said!("\"{address}\" can't be used as an address", address = tidy(address)))
|
||||
}
|
||||
|
||||
/// The message as DATA wants it: every line ending CRLF, a leading dot doubled, the closing
|
||||
/// `.` line added. Also returns the size without the added dots and closing line (RFC 1870).
|
||||
fn data_block(message: &[u8]) -> (Vec<u8>, usize) {
|
||||
let mut out = Vec::with_capacity(message.len().saturating_add(message.len() / 32).saturating_add(8));
|
||||
let mut dots = 0usize;
|
||||
let mut line_start = true;
|
||||
let mut bytes = message.iter().copied().peekable();
|
||||
while let Some(b) = bytes.next() {
|
||||
match b {
|
||||
b'\r' => {
|
||||
bytes.next_if_eq(&b'\n');
|
||||
out.extend_from_slice(b"\r\n");
|
||||
line_start = true;
|
||||
}
|
||||
b'\n' => {
|
||||
out.extend_from_slice(b"\r\n");
|
||||
line_start = true;
|
||||
}
|
||||
b'.' if line_start => {
|
||||
out.extend_from_slice(b"..");
|
||||
dots += 1;
|
||||
line_start = false;
|
||||
}
|
||||
_ => {
|
||||
out.push(b);
|
||||
line_start = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !line_start {
|
||||
out.extend_from_slice(b"\r\n");
|
||||
}
|
||||
let size = out.len().saturating_sub(dots);
|
||||
out.extend_from_slice(b".\r\n");
|
||||
(out, size)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,455 @@
|
||||
// SPDX-FileCopyrightText: MailCraft contributors
|
||||
// SPDX-License-Identifier: MIT OR Apache-2.0
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use std::io::ErrorKind;
|
||||
|
||||
use proptest::prelude::*;
|
||||
|
||||
use super::*;
|
||||
use crate::script::{Broken, Script};
|
||||
|
||||
const EHLO: &str =
|
||||
"250-mail.example.org greets you\r\n250-SIZE 35882577\r\n250-8BITMIME\r\n250-STARTTLS\r\n250-AUTH LOGIN PLAIN XOAUTH2\r\n250 HELP\r\n";
|
||||
|
||||
fn rejected(r: Result<(), ProtoError>) -> String {
|
||||
match r {
|
||||
Err(ProtoError::Rejected(text)) => text.to_string(),
|
||||
other => panic!("expected a rejection, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_whole_session() {
|
||||
let mut s = Script::new(format!(
|
||||
"220-mail.example.org ESMTP\r\n220 ready\r\n{EHLO}235 2.7.0 Accepted\r\n250 OK\r\n250 OK\r\n251 will forward\r\n354 go ahead\r\n250 queued as 1\r\n221 bye\r\n"
|
||||
));
|
||||
let mut smtp = Smtp::connect(&mut s).unwrap();
|
||||
let ext = smtp.hello("[127.0.0.1]").unwrap();
|
||||
assert_eq!(
|
||||
ext,
|
||||
Extensions { starttls: true, auth_plain: true, auth_login: true, auth_xoauth2: true, size: Some(35_882_577), eight_bit_mime: true }
|
||||
);
|
||||
assert_eq!(smtp.extensions(), &ext);
|
||||
smtp.auth("ada@example.org", "hunter2").unwrap();
|
||||
smtp.send("ada@example.org", &["bob@example.org", "carol@example.org"], b"Subject: Hi\r\n\r\nHello.\r\n").unwrap();
|
||||
smtp.quit().unwrap();
|
||||
assert!(s.finished());
|
||||
assert_eq!(
|
||||
s.sent(),
|
||||
"EHLO [127.0.0.1]\r\n\
|
||||
AUTH PLAIN AGFkYUBleGFtcGxlLm9yZwBodW50ZXIy\r\n\
|
||||
MAIL FROM:<ada@example.org> SIZE=23\r\n\
|
||||
RCPT TO:<bob@example.org>\r\n\
|
||||
RCPT TO:<carol@example.org>\r\n\
|
||||
DATA\r\n\
|
||||
Subject: Hi\r\n\r\nHello.\r\n.\r\n\
|
||||
QUIT\r\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_plain_payload_is_nul_user_nul_password() {
|
||||
let mut s = Script::new("250-x\r\n250 AUTH PLAIN\r\n235 ok\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
smtp.auth("grace", "pässword \"1\"").unwrap();
|
||||
let sent = s.sent();
|
||||
let payload = sent.lines().nth(1).unwrap().strip_prefix("AUTH PLAIN ").unwrap();
|
||||
assert_eq!(STANDARD.decode(payload).unwrap(), "\0grace\0pässword \"1\"".as_bytes());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_plain_payload_waits_for_the_server_to_ask() {
|
||||
let mut s = Script::new("250-x\r\n250 AUTH PLAIN\r\n334 \r\n235 ok\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
let password = "p".repeat(600);
|
||||
smtp.auth("grace", &password).unwrap();
|
||||
let sent = s.sent();
|
||||
let lines: Vec<&str> = sent.lines().collect();
|
||||
assert_eq!(lines[1], "AUTH PLAIN");
|
||||
assert_eq!(STANDARD.decode(lines[2]).unwrap(), format!("\0grace\0{password}").as_bytes());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn login_is_used_when_plain_is_not_offered() {
|
||||
let mut s = Script::new("250-x\r\n250 AUTH=LOGIN CRAM-MD5\r\n334 VXNlcm5hbWU6\r\n334 UGFzc3dvcmQ6\r\n235 ok\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
let ext = smtp.hello("localhost").unwrap();
|
||||
assert!(ext.auth_login && !ext.auth_plain);
|
||||
smtp.auth("ada", "hunter2").unwrap();
|
||||
assert_eq!(s.sent(), "EHLO localhost\r\nAUTH LOGIN\r\nYWRh\r\naHVudGVyMg==\r\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_wrong_password_is_an_auth_error_that_never_shows_it() {
|
||||
// A careless server repeats what it was sent.
|
||||
let echo = "535-5.7.8 no such password: hunter2\r\n535 5.7.8 (AGFkYQBodW50ZXIy)\r\n";
|
||||
let mut s = Script::new(format!("250-x\r\n250 AUTH PLAIN\r\n{echo}"));
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
let err = smtp.auth("ada", "hunter2").unwrap_err();
|
||||
assert_eq!(err, ProtoError::Auth("535 5.7.8 no such password: *** 5.7.8 (***)".into()));
|
||||
assert!(!format!("{err} {err:?} {smtp:?}").contains("hunter2"));
|
||||
|
||||
// The same over LOGIN, where the password travels alone.
|
||||
let mut s = Script::new("250-x\r\n250 AUTH LOGIN\r\n334 VXNlcm5hbWU6\r\n334 UGFzc3dvcmQ6\r\n535 bad aHVudGVyMg== / hunter2\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
assert_eq!(smtp.auth("ada", "hunter2"), Err(ProtoError::Auth("535 bad *** / ***".into())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sign_in_needs_a_method_and_sendable_text() {
|
||||
let mut s = Script::new("250-x\r\n250 AUTH XOAUTH2\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
assert!(matches!(smtp.auth("ada", "hunter2"), Err(ProtoError::Auth(_))));
|
||||
assert!(matches!(smtp.auth("ada", "hun\0ter2"), Err(ProtoError::Auth(_))));
|
||||
// Neither attempt put anything on the wire.
|
||||
assert_eq!(s.sent(), "EHLO localhost\r\n");
|
||||
}
|
||||
|
||||
/// What an XOAUTH2 response says under its base64.
|
||||
fn bearer(payload: &str) -> String {
|
||||
String::from_utf8(STANDARD.decode(payload).unwrap()).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_token_signs_in_with_xoauth2() {
|
||||
let mut s = Script::new("250-x\r\n250 AUTH LOGIN XOAUTH2\r\n235 2.7.0 Authentication successful\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
assert!(smtp.hello("localhost").unwrap().auth_xoauth2);
|
||||
smtp.auth_xoauth2("ada@example.org", "tok.en-123").unwrap();
|
||||
assert!(s.finished());
|
||||
let sent = s.sent();
|
||||
let lines: Vec<&str> = sent.lines().collect();
|
||||
assert_eq!(lines.len(), 2);
|
||||
assert_eq!(bearer(lines[1].strip_prefix("AUTH XOAUTH2 ").unwrap()), "user=ada@example.org\x01auth=Bearer tok.en-123\x01\x01");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_token_waits_for_the_server_to_ask() {
|
||||
// Real tokens run to a couple of thousand characters.
|
||||
let token = "t".repeat(2_000);
|
||||
let mut s = Script::new("250-x\r\n250 AUTH XOAUTH2\r\n334 \r\n235 ok\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
smtp.auth_xoauth2("ada@example.org", &token).unwrap();
|
||||
let sent = s.sent();
|
||||
let lines: Vec<&str> = sent.lines().collect();
|
||||
assert_eq!(lines[1], "AUTH XOAUTH2");
|
||||
assert_eq!(bearer(lines[2]), format!("user=ada@example.org\x01auth=Bearer {token}\x01\x01"));
|
||||
|
||||
// The longest that still fits goes in one line.
|
||||
let token = "t".repeat(349);
|
||||
let mut s = Script::new("250-x\r\n250 AUTH XOAUTH2\r\n235 ok\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
smtp.auth_xoauth2("ada", &token).unwrap();
|
||||
let sent = s.sent();
|
||||
assert_eq!((sent.lines().count(), sent.lines().nth(1).unwrap().len()), (2, MAX_COMMAND - 1));
|
||||
|
||||
// A server that won't ask is not shown the token.
|
||||
let token = "t".repeat(2_000);
|
||||
let mut s = Script::new("250-x\r\n250 AUTH XOAUTH2\r\n504 5.5.4 not that way\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
assert_eq!(smtp.auth_xoauth2("ada", &token), Err(ProtoError::Auth("504 5.5.4 not that way".into())));
|
||||
assert_eq!(s.sent(), "EHLO localhost\r\nAUTH XOAUTH2\r\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_refused_token_is_an_auth_error_that_never_shows_it() {
|
||||
let payload = xoauth2("ada", "tok.en-123").unwrap();
|
||||
let reasons = STANDARD.encode(r#"{"status":"401","schemes":"bearer","scope":"https://mail.example.org/"}"#);
|
||||
// The reasons come first and the verdict after an empty line; a careless server repeats what it was sent.
|
||||
let mut s = Script::new(format!("250-x\r\n250 AUTH XOAUTH2\r\n334 {reasons}\r\n535-5.7.3 no good: tok.en-123\r\n535 5.7.3 ({payload})\r\n"));
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
let err = smtp.auth_xoauth2("ada", "tok.en-123").unwrap_err();
|
||||
assert_eq!(err, ProtoError::Auth("535 5.7.3 no good: *** 5.7.3 (***)".into()));
|
||||
assert!(!format!("{err} {err:?} {smtp:?}").contains("tok.en-123"));
|
||||
assert_eq!(s.sent(), format!("EHLO localhost\r\nAUTH XOAUTH2 {payload}\r\n\r\n"));
|
||||
assert!(s.finished());
|
||||
|
||||
// The same when the token went on a line of its own.
|
||||
let token = "t".repeat(2_000);
|
||||
let mut s = Script::new(format!("250-x\r\n250 AUTH XOAUTH2\r\n334 \r\n334 {reasons}\r\n535 5.7.3 Authentication unsuccessful\r\n"));
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
assert_eq!(smtp.auth_xoauth2("ada", &token), Err(ProtoError::Auth("535 5.7.3 Authentication unsuccessful".into())));
|
||||
assert_eq!(s.sent(), format!("EHLO localhost\r\nAUTH XOAUTH2\r\n{}\r\n\r\n", xoauth2("ada", &token).unwrap()));
|
||||
assert!(s.finished());
|
||||
|
||||
// Refused in one word, without reasons.
|
||||
let mut s = Script::new("250-x\r\n250 AUTH XOAUTH2\r\n535 5.7.3 Authentication unsuccessful\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
assert_eq!(smtp.auth_xoauth2("ada", "tok.en-123"), Err(ProtoError::Auth("535 5.7.3 Authentication unsuccessful".into())));
|
||||
assert_eq!(s.sent(), format!("EHLO localhost\r\nAUTH XOAUTH2 {payload}\r\n"));
|
||||
|
||||
// A question after the reasons has no answer: the exchange is cancelled.
|
||||
let mut s = Script::new(format!("250-x\r\n250 AUTH XOAUTH2\r\n334 {reasons}\r\n334 more?\r\n501 cancelled\r\n"));
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
assert!(matches!(smtp.auth_xoauth2("ada", "tok.en-123"), Err(ProtoError::Auth(_))));
|
||||
assert!(s.sent().ends_with("\r\n\r\n*\r\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_token_echoed_in_a_long_reply_is_hidden_whole() {
|
||||
// The reply is shortened for the error text: not before the token is out of it.
|
||||
let token = format!("head.{}.tail", "x".repeat(2_000));
|
||||
let mut s = Script::new(format!("250-x\r\n250 AUTH XOAUTH2\r\n334 \r\n535 5.7.3 {} bad token {token}\r\n", "no ".repeat(80)));
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
let err = smtp.auth_xoauth2("ada", &token).unwrap_err();
|
||||
let shown = format!("{err} {err:?}");
|
||||
assert!(matches!(err, ProtoError::Auth(_)));
|
||||
assert!(!shown.contains("head.") && !shown.contains("xxxx"), "{shown}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_token_needs_a_server_that_takes_one_and_sendable_text() {
|
||||
let mut s = Script::new("250-x\r\n250 AUTH PLAIN LOGIN\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
assert_eq!(smtp.auth_xoauth2("ada", "tok.en-123"), Err(ProtoError::Auth(said!("the server offers no way to sign in with an access token"))));
|
||||
assert_eq!(s.sent(), "EHLO localhost\r\n");
|
||||
|
||||
let mut s = Script::new("250-x\r\n250 AUTH XOAUTH2\r\n235 ok\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
for (user, token) in
|
||||
[("ada", ""), ("ada", "tok\r\nQUIT"), ("ada", "tok\x01"), ("ada\x01auth=Bearer x", "tok"), ("a\0da", "tok"), ("ada\n", "tok")]
|
||||
{
|
||||
assert!(matches!(smtp.auth_xoauth2(user, token), Err(ProtoError::Auth(_))), "{user:?} {token:?}");
|
||||
}
|
||||
assert_eq!(s.sent(), "EHLO localhost\r\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unexpected_second_question_cancels_the_sign_in() {
|
||||
let mut s = Script::new("250-x\r\n250 AUTH PLAIN\r\n334 more?\r\n501 cancelled\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
assert!(matches!(smtp.auth("ada", "hunter2"), Err(ProtoError::Auth(_))));
|
||||
assert!(s.sent().ends_with("\r\n*\r\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn helo_is_the_fallback() {
|
||||
let mut s = Script::new("220 old.example.org\r\n502 what?\r\n250 old.example.org\r\n");
|
||||
let mut smtp = Smtp::connect(&mut s).unwrap();
|
||||
assert_eq!(smtp.hello("localhost").unwrap(), Extensions::default());
|
||||
assert_eq!(s.sent(), "EHLO localhost\r\nHELO localhost\r\n");
|
||||
|
||||
// A server on its way out is not asked twice.
|
||||
let mut s = Script::new("421 shutting down\r\n");
|
||||
assert_eq!(Smtp::resume(&mut s).hello("localhost"), Err(ProtoError::Rejected("421 shutting down".into())));
|
||||
assert_eq!(s.sent(), "EHLO localhost\r\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_client_name_that_could_break_the_line_is_replaced() {
|
||||
for bad in ["", "two words", "evil\r\nMAIL FROM:<x>", "nön-ascii"] {
|
||||
let mut s = Script::new("250 ok\r\n");
|
||||
Smtp::resume(&mut s).hello(bad).unwrap();
|
||||
assert_eq!(s.sent(), "EHLO localhost\r\n", "{bad:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extensions_are_read_whatever_their_case() {
|
||||
let mut s = Script::new("250-x\r\n250-starttls\r\n250-Size\r\n250-auth plain\r\n250 8bitmime\r\n");
|
||||
let ext = Smtp::resume(&mut s).hello("localhost").unwrap();
|
||||
assert_eq!(ext, Extensions { starttls: true, auth_plain: true, auth_login: false, auth_xoauth2: false, size: Some(0), eight_bit_mime: true });
|
||||
let mut s = Script::new("250-x\r\n250 auth=login xoauth2 OAUTHBEARER\r\n");
|
||||
let ext = Smtp::resume(&mut s).hello("localhost").unwrap();
|
||||
assert_eq!(ext, Extensions { auth_login: true, auth_xoauth2: true, ..Default::default() });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn starttls_hands_the_stream_back() {
|
||||
let mut s = Script::new("220 hi\r\n250-x\r\n250 STARTTLS\r\n220 go ahead\r\n");
|
||||
let mut smtp = Smtp::connect(&mut s).unwrap();
|
||||
assert!(smtp.hello("localhost").unwrap().starttls);
|
||||
let stream = smtp.starttls().unwrap();
|
||||
assert_eq!(stream.sent(), "EHLO localhost\r\nSTARTTLS\r\n");
|
||||
|
||||
let mut s = Script::new("454 TLS not available\r\n");
|
||||
assert_eq!(Smtp::resume(&mut s).starttls().unwrap_err(), ProtoError::Rejected("454 TLS not available".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_slipped_in_before_tls_is_refused() {
|
||||
// Both lines arrive in one read, as an attacker in the middle would send them.
|
||||
let mut s = Script::whole("220 go ahead\r\n250 injected\r\n");
|
||||
assert!(matches!(Smtp::resume(&mut s).starttls(), Err(ProtoError::Protocol(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_is_normalised_and_dot_stuffed() {
|
||||
let message = b"Subject: dots\n\n.leading\r\n..two\rbare CR\n.\nno dot. here\n\nlast line without an ending";
|
||||
let mut s = Script::new("250 ok\r\n250 ok\r\n354 go\r\n250 ok\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.send("ada@example.org", &["bob@example.org"], message).unwrap();
|
||||
let sent = s.sent();
|
||||
let data = sent.split_once("DATA\r\n").unwrap().1;
|
||||
assert_eq!(data, "Subject: dots\r\n\r\n..leading\r\n...two\r\nbare CR\r\n..\r\nno dot. here\r\n\r\nlast line without an ending\r\n.\r\n");
|
||||
// No SIZE or BODY parameter for a server that didn't offer them.
|
||||
assert!(sent.starts_with("MAIL FROM:<ada@example.org>\r\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_blocks() {
|
||||
assert_eq!(data_block(b""), (b".\r\n".to_vec(), 0));
|
||||
assert_eq!(data_block(b"a"), (b"a\r\n.\r\n".to_vec(), 3));
|
||||
assert_eq!(data_block(b"."), (b"..\r\n.\r\n".to_vec(), 3));
|
||||
assert_eq!(data_block(b"\r\r\n\n"), (b"\r\n\r\n\r\n.\r\n".to_vec(), 6));
|
||||
assert_eq!(data_block(b"a\r"), (b"a\r\n.\r\n".to_vec(), 3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eight_bit_mail_says_so_when_the_server_takes_it() {
|
||||
let mut s = Script::new("250-x\r\n250 8BITMIME\r\n250 ok\r\n250 ok\r\n354 go\r\n250 ok\r\n250 ok\r\n250 ok\r\n354 go\r\n250 ok\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.hello("localhost").unwrap();
|
||||
smtp.send("ada@example.org", &["bob@example.org"], "Subject: é\r\n\r\n".as_bytes()).unwrap();
|
||||
smtp.send("", &["bob@example.org"], b"Subject: e\r\n\r\n").unwrap();
|
||||
let sent = s.sent();
|
||||
assert!(sent.contains("MAIL FROM:<ada@example.org> BODY=8BITMIME\r\n"));
|
||||
// An empty sender is the bounce address.
|
||||
assert!(sent.contains("MAIL FROM:<>\r\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_refused_recipient_stops_the_message_and_resets() {
|
||||
let mut s = Script::new("250 ok\r\n250 ok\r\n550 5.1.1 no such user\r\n250 reset\r\n");
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
let text = rejected(smtp.send("ada@example.org", &["bob@example.org", "nobody@example.org", "carol@example.org"], b"x"));
|
||||
assert_eq!(text, "nobody@example.org: 550 5.1.1 no such user");
|
||||
assert!(s.sent().ends_with("RCPT TO:<nobody@example.org>\r\nRSET\r\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refusals_at_each_step() {
|
||||
// MAIL FROM.
|
||||
let mut s = Script::new("552 5.3.4 too big\r\n");
|
||||
assert_eq!(rejected(Smtp::resume(&mut s).send("a@example.org", &["b@example.org"], b"x")), "552 5.3.4 too big");
|
||||
// DATA.
|
||||
let mut s = Script::new("250 ok\r\n250 ok\r\n554 no\r\n250 reset\r\n");
|
||||
assert_eq!(rejected(Smtp::resume(&mut s).send("a@example.org", &["b@example.org"], b"x")), "554 no");
|
||||
assert!(s.sent().ends_with("DATA\r\nRSET\r\n"));
|
||||
// After the message.
|
||||
let mut s = Script::new("250 ok\r\n250 ok\r\n354 go\r\n451 4.3.0 try later\r\n");
|
||||
assert_eq!(rejected(Smtp::resume(&mut s).send("a@example.org", &["b@example.org"], b"x")), "451 4.3.0 try later");
|
||||
// The greeting.
|
||||
let mut s = Script::new("554 no service for you\r\n");
|
||||
assert_eq!(Smtp::connect(&mut s).unwrap_err(), ProtoError::Rejected("554 no service for you".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsafe_addresses_are_never_sent() {
|
||||
let cases: [(&str, &[&str]); 6] = [
|
||||
("ada@example.org", &[]),
|
||||
("ada@example.org", &["bob@example.org>\r\nRCPT TO:<eve@example.org"]),
|
||||
("ada@example.org", &["bob smith@example.org"]),
|
||||
("ada@example.org", &[""]),
|
||||
("ada@example.org\r\nDATA", &["bob@example.org"]),
|
||||
("ada@example.org", &["bøb@example.org"]),
|
||||
];
|
||||
for (from, to) in cases {
|
||||
let mut s = Script::new("250 ok\r\n250 ok\r\n354 go\r\n250 ok\r\n");
|
||||
assert!(matches!(Smtp::resume(&mut s).send(from, to, b"x"), Err(ProtoError::Protocol(_))), "{from:?} {to:?}");
|
||||
assert_eq!(s.sent(), "", "{from:?} {to:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn junk_replies_are_protocol_errors() {
|
||||
for junk in ["hello there\r\n", "25 ok\r\n", "250ok\r\n", "250-first\r\n251 second\r\n", "\r\n", "2éé ok\r\n"] {
|
||||
let mut s = Script::new(junk);
|
||||
assert!(matches!(Smtp::resume(&mut s).hello("localhost"), Err(ProtoError::Protocol(_))), "{junk:?}");
|
||||
}
|
||||
// A reply that never ends.
|
||||
let mut s = Script::whole("250-more\r\n".repeat(MAX_REPLY_LINES + 1));
|
||||
assert_eq!(Smtp::resume(&mut s).hello("localhost"), Err(ProtoError::Protocol(said!("the server's reply has too many lines"))));
|
||||
// A greeting that is a success code but not 220.
|
||||
let mut s = Script::new("250 hello\r\n");
|
||||
assert!(matches!(Smtp::connect(&mut s), Err(ProtoError::Protocol(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_dead_connection_is_an_io_error() {
|
||||
let mut s = Script::new("250-half a rep");
|
||||
assert_eq!(Smtp::resume(&mut s).hello("localhost"), Err(ProtoError::Io(said!("the server closed the connection"))));
|
||||
assert!(matches!(Smtp::connect(Broken(ErrorKind::TimedOut)), Err(ProtoError::Io(_))));
|
||||
assert!(matches!(Smtp::resume(Broken(ErrorKind::ConnectionReset)).send("a@example.org", &["b@example.org"], b"x"), Err(ProtoError::Io(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quitting() {
|
||||
let mut s = Script::new("221 bye\r\n");
|
||||
assert_eq!(Smtp::resume(&mut s).quit(), Ok(()));
|
||||
// A server that just hangs up has quit too.
|
||||
let mut s = Script::new("");
|
||||
assert_eq!(Smtp::resume(&mut s).quit(), Ok(()));
|
||||
assert_eq!(s.sent(), "QUIT\r\n");
|
||||
let mut s = Script::new("500 what\r\n");
|
||||
assert!(matches!(Smtp::resume(&mut s).quit(), Err(ProtoError::Rejected(_))));
|
||||
}
|
||||
|
||||
/// Everything a client does, against whatever the "server" says.
|
||||
fn drive(server: &[u8]) {
|
||||
let mut s = Script::new(server);
|
||||
if let Ok(mut smtp) = Smtp::connect(&mut s) {
|
||||
let _ = smtp.hello("localhost");
|
||||
let _ = smtp.auth("ada@example.org", "hunter2");
|
||||
let _ = smtp.send("ada@example.org", &["bob@example.org"], b"Subject: x\r\n\r\n.\r\n");
|
||||
let _ = smtp.quit();
|
||||
}
|
||||
let mut s = Script::new(server);
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.ext = Extensions { auth_login: true, ..Default::default() };
|
||||
let _ = smtp.auth("ada", "hunter2");
|
||||
let _ = smtp.starttls();
|
||||
// With a token, on one line and on its own.
|
||||
for token in ["tok.en-123".to_string(), "t".repeat(700)] {
|
||||
let mut s = Script::new(server);
|
||||
let mut smtp = Smtp::resume(&mut s);
|
||||
smtp.ext = Extensions { auth_xoauth2: true, ..Default::default() };
|
||||
let _ = smtp.auth_xoauth2("ada@example.org", &token);
|
||||
let _ = smtp.send("ada@example.org", &["bob@example.org"], b"x");
|
||||
}
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn arbitrary_server_bytes_never_panic_or_hang(server in proptest::collection::vec(any::<u8>(), 0..600)) {
|
||||
drive(&server);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reply_shaped_noise_never_panics(lines in proptest::collection::vec(("[0-9]{3}", "[- ]?", "[ -~]{0,30}"), 0..40)) {
|
||||
let server: String = lines.iter().map(|(code, sep, text)| format!("{code}{sep}{text}\r\n")).collect();
|
||||
drive(server.as_bytes());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_blocks_never_end_early(message in proptest::collection::vec(prop_oneof![Just(b'.'), Just(b'\r'), Just(b'\n'), any::<u8>()], 0..300)) {
|
||||
let (data, size) = data_block(&message);
|
||||
prop_assert!(data.ends_with(b".\r\n"));
|
||||
let body = &data[..data.len() - 3];
|
||||
prop_assert!(body.is_empty() || body.ends_with(b"\r\n"));
|
||||
// No bare CR or LF, and no line that is just a dot before the end.
|
||||
let text = String::from_utf8_lossy(body);
|
||||
prop_assert_eq!(body.iter().filter(|b| **b == b'\r').count(), body.iter().filter(|b| **b == b'\n').count());
|
||||
prop_assert!(!text.contains("\r\n.\r\n") && !text.starts_with(".\r\n"));
|
||||
prop_assert_eq!(body.windows(2).filter(|w| w[0] == b'\r' && w[1] == b'\n').count(), body.iter().filter(|b| **b == b'\n').count());
|
||||
// Taking the stuffing out gives the size that was announced.
|
||||
let unstuffed: usize = body.split_inclusive(|b| *b == b'\n').map(|l| l.len() - usize::from(l.starts_with(b"."))).sum();
|
||||
prop_assert_eq!(unstuffed, size);
|
||||
}
|
||||
}
|
||||
+74
-5
@@ -629,6 +629,19 @@ fn raw_ifd(tiff: &Tiff) -> Option<&Ifd> {
|
||||
.max_by_key(|i| i.u64(t::IMAGE_WIDTH).unwrap_or(0).saturating_mul(i.u64(t::IMAGE_LENGTH).unwrap_or(0)))
|
||||
}
|
||||
|
||||
fn valid_arw2_curve(curve: &[u64]) -> bool {
|
||||
curve.len() == 4 && curve.iter().all(|&v| v <= 16383) && curve.windows(2).all(|v| (v[0] >> 2) < (v[1] >> 2))
|
||||
}
|
||||
|
||||
fn declared_arw2_curve(raw: &Ifd, compression: u16, one_byte_per_sample: bool) -> Result<Option<Vec<u64>>> {
|
||||
let curve = raw.u64s(TONE_CURVE);
|
||||
if compression == 32767 && one_byte_per_sample && curve.as_ref().is_some_and(|curve| !valid_arw2_curve(curve)) {
|
||||
// Reordering or clamping malformed breakpoints fabricates a different transfer curve.
|
||||
return Err(RawError::Unsupported("Sony ARW2 with an invalid tone curve".into()));
|
||||
}
|
||||
Ok(curve)
|
||||
}
|
||||
|
||||
pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawImage> {
|
||||
let tiff = Tiff::parse(bytes)?;
|
||||
let ifd0 = tiff.ifds.first().ok_or_else(|| RawError::Corrupt("TIFF without IFDs".into()))?;
|
||||
@@ -645,6 +658,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
// archive measure exactly width x height. A200/A230/A350 files carry a strip 4-23 % larger than that (and A290/A390 one
|
||||
// smaller), which is a different packing, so they are refused rather than read with the wrong layout.
|
||||
let one_byte_per_sample = chunks.len() == 1 && strip_len == samples;
|
||||
let declared_curve = declared_arw2_curve(raw, info.compression, one_byte_per_sample)?;
|
||||
let packed12 = info.compression == 32767 && is_packed12(&info, chunks.len(), strip_len);
|
||||
let (data, out_bits) = match info.compression {
|
||||
7 if linear_rgb => (RawData::U16(read_ycbcr_tiles(bytes, &info, raw, mode, limits)?), 14),
|
||||
@@ -720,13 +734,16 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
// agree too. Genuinely packed 12-bit strips retain their 12-bit scale.
|
||||
let scale_bits = if info.compression == 1 && out_bits < 14 && strip_len >= (w * h * 2) as u64 { 14 } else { out_bits };
|
||||
|
||||
let cfa = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
|
||||
let (cfa, assumed_cfa) = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
|
||||
(Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => {
|
||||
Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }
|
||||
(Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }, false)
|
||||
}
|
||||
_ => Cfa::bayer_static("RGGB"),
|
||||
_ => (Cfa::bayer_static("RGGB"), true),
|
||||
};
|
||||
let default_black = if scale_bits >= 14 { 512.0 } else { 128.0 };
|
||||
let curve_black = (info.compression == 32767 && one_byte_per_sample)
|
||||
.then(|| declared_curve.as_ref().and_then(|curve| code_curve(curve).get(256).copied()).map(f32::from))
|
||||
.flatten();
|
||||
let default_black = curve_black.unwrap_or(if scale_bits >= 14 { 512.0 } else { 128.0 });
|
||||
// the decrypted SR2SubIFD, read once and only when a plain black level or white balance is missing
|
||||
let sr2_cell = std::cell::OnceCell::new();
|
||||
let sr2 = || sr2_cell.get_or_init(|| crate::sr2::SubIfd::read(bytes, ifd0, tiff.order)).as_ref();
|
||||
@@ -741,7 +758,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
_ if linear_rgb => (BlackLevel::uniform(default_black + YCBCR_OFFSET), crate::LevelSource::Fallback),
|
||||
_ => match (scale_bits >= 14).then(|| sr2().and_then(sr2_black)).flatten() {
|
||||
Some(b) => (BlackLevel::uniform(b), crate::LevelSource::Metadata),
|
||||
None => (BlackLevel::uniform(default_black), crate::LevelSource::Fallback),
|
||||
None => (BlackLevel::uniform(default_black), curve_black.map_or(crate::LevelSource::Fallback, |_| crate::LevelSource::Metadata)),
|
||||
},
|
||||
};
|
||||
// ARW6 samples are in 2 x 14-bit units, so the file's 14-bit levels double; without the tag the white is the
|
||||
@@ -842,6 +859,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
cinema: Default::default(),
|
||||
lens_metadata: LensMetadata { sony: sony_lens },
|
||||
level_sources: crate::LevelSources { black: black_source, white: white_source },
|
||||
notices: crate::RawNotices { assumed_cfa: assumed_cfa && !linear_rgb, ..Default::default() },
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
@@ -873,6 +891,8 @@ mod tests {
|
||||
assert_eq!(header.info(), full.info());
|
||||
assert_eq!((full.width, full.height, full.cpp), (4, 4, 3));
|
||||
assert!(full.cfa.is_none());
|
||||
assert!(!header.notices.assumed_cfa);
|
||||
assert!(!full.notices.assumed_cfa);
|
||||
assert_eq!(full.data.len(), 48);
|
||||
assert_eq!(full.wb_multipliers, Some([1.0; 3]));
|
||||
// the YCbCr data sit 512 above the recorded sensor black level; white moves with it
|
||||
@@ -911,6 +931,10 @@ mod tests {
|
||||
|
||||
/// A 32767-compressed ARW whose single strip is `strip` bytes for a 32 x 4 image.
|
||||
fn arw_with_strip(strip: usize) -> Vec<u8> {
|
||||
arw_with_strip_and_cfa(strip, None)
|
||||
}
|
||||
|
||||
fn arw_with_strip_and_cfa(strip: usize, cfa: Option<(Vec<u16>, Vec<u8>)>) -> Vec<u8> {
|
||||
use craft_tiff::{IfdBuilder, ImageData, TiffWriter, Value};
|
||||
let mut raw = IfdBuilder::new();
|
||||
raw.set(t::MAKE, Value::Ascii("SONY".into()));
|
||||
@@ -921,11 +945,37 @@ mod tests {
|
||||
raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1]));
|
||||
raw.set(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA]));
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![32767]));
|
||||
if let Some((dimensions, pattern)) = cfa {
|
||||
raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(dimensions));
|
||||
raw.set(t::CFA_PATTERN_EP, Value::Byte(pattern));
|
||||
}
|
||||
raw.set(TONE_CURVE, Value::Short(vec![8000, 10400, 12900, 14100]));
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![vec![0u8; strip]] });
|
||||
TiffWriter::default().write(&[raw]).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cfa_fallback_notices_follow_missing_malformed_and_valid_source_tags() {
|
||||
for (tags, expected, assumed) in [
|
||||
(None, "RGGB", true),
|
||||
(Some((vec![1, 4], vec![0, 1, 1, 2])), "RGGB", true),
|
||||
(Some((vec![2, 2], vec![0, 1, 3, 2])), "RGGB", true),
|
||||
(Some((vec![2, 2], vec![0, 1, 2])), "RGGB", true),
|
||||
(Some((vec![2, 2], vec![1, 0, 2, 1])), "GRBG", false),
|
||||
] {
|
||||
let file = arw_with_strip_and_cfa(32 * 4, tags);
|
||||
let header = decode(&file, Mode::Header, &Limits::default()).unwrap();
|
||||
let full = decode(&file, Mode::Full, &Limits::default()).unwrap();
|
||||
assert_eq!(header.info(), full.info());
|
||||
for image in [&header, &full] {
|
||||
assert_eq!(image.cfa.as_ref().unwrap().name(), expected);
|
||||
assert_eq!(image.notices.assumed_cfa, assumed);
|
||||
}
|
||||
assert!(header.data.is_empty());
|
||||
assert_eq!(full.data.len(), 32 * 4);
|
||||
}
|
||||
}
|
||||
|
||||
/// Strip byte counts that add up past `u64::MAX` (LONG8 `[u64::MAX, 1]`) are an error or a picture, not an overflow.
|
||||
#[test]
|
||||
fn strip_byte_counts_past_u64_max_do_not_overflow() {
|
||||
@@ -1110,6 +1160,25 @@ mod tests {
|
||||
let _ = code_curve(&[60000, 1, 0, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn arw2_rejects_non_increasing_declared_curve_in_header_and_full_decode() {
|
||||
let valid = arw_with_strip(32 * 4);
|
||||
assert!(crate::decode(&valid).is_ok());
|
||||
let mut damaged = valid.clone();
|
||||
let tiff = Tiff::parse(&damaged).unwrap();
|
||||
let curve = raw_ifd(&tiff).unwrap().get(TONE_CURVE).unwrap();
|
||||
let at = curve.offset as usize;
|
||||
for (dst, value) in damaged[at..at + 8].as_chunks_mut::<2>().0.iter_mut().zip([9000u16, 8000, 12000, 13000]) {
|
||||
dst.copy_from_slice(&value.to_le_bytes());
|
||||
}
|
||||
for mode in [Mode::Header, Mode::Full] {
|
||||
assert!(matches!(decode(&damaged, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("curve")));
|
||||
let image = decode(&valid, mode, &Limits::default()).unwrap();
|
||||
assert_eq!(image.black.mean(), 512.0);
|
||||
assert_eq!(image.level_sources.black, crate::LevelSource::Metadata, "the declared curve maps the black code");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decipher_inverts_the_cube_substitution() {
|
||||
let mut seen = [false; 256];
|
||||
|
||||
+128
-2
@@ -10,8 +10,11 @@
|
||||
//! - Colour filter layout: IFD#3 tag `0xc5e0` (`CR2CFAPattern` in the ExifTool EXIF tag-name docs): 1 = RGGB,
|
||||
//! 2 = BGGR, 3 = GBRG, 4 = GRBG, anchored at the top-left of the full decoded sensor (masked borders included).
|
||||
//! It differs by model (e.g. 3 on the 50D/60D/7D/550D/5D Mark II, 1 on the 40D/5D Mark III/6D/5DS R), so it is
|
||||
//! read per file (no model table); files without the tag fall back to RGGB (storytold/lightcraft#85). The parity of the
|
||||
//! read per file first. Without a known tag, observed model IDs provide the fallback; scene-derived
|
||||
//! measurements are not authoritative enough to establish sensor phase. The parity of the
|
||||
//! `SensorInfo` borders does *not* predict it (RGGB files come with both even and odd top borders).
|
||||
//! Header-only probes do not decode pixels for this fallback: without the tag their reported CFA
|
||||
//! is assumed and can refine on full decode, as indicated by `RawNotices::assumed_cfa`.
|
||||
//! - `SensorInfo` (maker note `0x00e0`): sensor width/height and the left/top/right/bottom borders of the image
|
||||
//! area. Light already reaches some columns left of that border (EOS 6D: masked columns 0–70, border 84; 10–36
|
||||
//! such columns on every corpus body), so the masked columns that give the black level are measured from the
|
||||
@@ -38,6 +41,7 @@ const SRAW_TYPE: u16 = 0xc6c5;
|
||||
/// `SRAW_TYPE` value of the YCbCr frames of sRAW / mRAW (IFD2's preview image carries 3).
|
||||
const SRAW_YCC: u32 = 4;
|
||||
const CR2_CFA_PATTERN: u16 = 0xc5e0;
|
||||
const CANON_MODEL_ID: u16 = 0x0010;
|
||||
const SENSOR_INFO: u16 = 0x00e0;
|
||||
const COLOR_BALANCE: u16 = 0x4001;
|
||||
/// ShotInfo, whose value 7 is the white-balance setting.
|
||||
@@ -61,6 +65,13 @@ fn cfa_from_tag(v: Option<u64>) -> Option<Cfa> {
|
||||
Some(Cfa::bayer_static(name))
|
||||
}
|
||||
|
||||
/// Fallback for files without CR2CFAPattern. Image content cannot establish sensor phase safely.
|
||||
fn fallback_cfa(_data: &[u16], _width: usize, _area: Rect, _black: f32, _white: f32, model: Option<u64>) -> (Cfa, bool) {
|
||||
// Observed CC0 raw.pixls.us captures: 5D II, 7D, 550D, 600D and 1200D.
|
||||
let green_first = [0x8000_0218, 0x8000_0250, 0x8000_0270, 0x8000_0286, 0x8000_0327].into_iter().any(|id| model == Some(id));
|
||||
(Cfa::bayer_static(if green_first { "GBRG" } else { "RGGB" }), true)
|
||||
}
|
||||
|
||||
/// The 16-bit words of a maker-note array. Canon writes `ColorData` as SHORT in most models, but as UNDEFINED
|
||||
/// bytes (count = byte length, e.g. 5120) in the PowerShot / EOS M models with ColorData versions -3 and -4;
|
||||
/// those bytes are the same 16-bit words in the maker note's byte order, so they are paired up, not read one by
|
||||
@@ -198,6 +209,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
let chunk = craft_tiff::image::Chunk { index: 0, x: 0, y: 0, width: 0, height: 0, plane: 0, offset: off, len };
|
||||
let src = chunk_bytes(bytes, &chunk).ok_or_else(|| RawError::Corrupt("raw strip outside file".into()))?;
|
||||
let (fw, fh, nc, prec) = ljpeg::frame_info(src)?;
|
||||
let tagged_cfa = cfa_from_tag(raw.u64(CR2_CFA_PATTERN));
|
||||
let frame = match mode {
|
||||
Mode::Full => {
|
||||
let total = (fw as u64)
|
||||
@@ -270,12 +282,19 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
let color_data = mn.as_ref().and_then(|m| m.ifd.value(COLOR_BALANCE).map(|v| words(v, m.order)));
|
||||
let kelvin_presets = color_data.as_deref().and_then(kelvin_presets_from_color_data);
|
||||
let wb = color_data.as_deref().and_then(wb_from_color_balance).or_else(|| wb_from_wb_info(mn.as_ref()?));
|
||||
let cfa = cfa_from_tag(raw.u64(CR2_CFA_PATTERN)).unwrap_or_else(|| Cfa::bayer_static("RGGB"));
|
||||
let (white, white_source) = white_from_data_sourced(&active_rows(&data, width, active), prec as u32);
|
||||
let (black, black_source) = match masked_columns(&data, width, active, white) {
|
||||
cols if cols.len() >= 2 => black_from_columns_sourced(&data, width, cols, active.y..active.y + active.height, active),
|
||||
_ => (BlackLevel::uniform(0.0), crate::LevelSource::Fallback),
|
||||
};
|
||||
let model = mn.as_ref().and_then(|m| m.ifd.u64(CANON_MODEL_ID));
|
||||
let (cfa, assumed_cfa) = match tagged_cfa {
|
||||
Some(cfa) => (cfa, false),
|
||||
None => {
|
||||
let (cfa, guessed) = fallback_cfa(&data, width, active, black.mean(), white, model);
|
||||
(cfa, guessed || mode == Mode::Header)
|
||||
}
|
||||
};
|
||||
let img = RawImage {
|
||||
format: RawFormat::Cr2,
|
||||
width,
|
||||
@@ -298,6 +317,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: crate::LevelSources { black: black_source, white: white_source },
|
||||
notices: crate::RawNotices { assumed_cfa, ..Default::default() },
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
@@ -397,6 +417,112 @@ pub(crate) mod tests {
|
||||
assert_eq!(name(None), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_cfa_tag_uses_safe_phase_without_shifting_for_crop_parity() {
|
||||
let (w, h) = (192usize, 96usize);
|
||||
for red_row in [0usize, 1] {
|
||||
let samples: Vec<u16> = (0..w * h)
|
||||
.map(|i| {
|
||||
let (x, y) = (i % w, i / w);
|
||||
let texture = ((x / 2 + y / 2) % 17) as u16;
|
||||
if i % 97 == 0 {
|
||||
return 13000;
|
||||
}
|
||||
100 + if (y & 1) == red_row {
|
||||
if x & 1 == 0 { 500 + texture } else { 1000 + texture }
|
||||
} else if x & 1 == 0 {
|
||||
1000 + texture
|
||||
} else {
|
||||
667 + texture
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
for (left, top) in [(16, 4), (17, 4), (16, 5), (17, 5)] {
|
||||
let area = Rect::new(left, top, w - left, h - top);
|
||||
let (cfa, assumed) = fallback_cfa(&samples, w, area, 100.0, 4000.0, None);
|
||||
assert_eq!(cfa.name(), "RGGB");
|
||||
assert!(assumed);
|
||||
}
|
||||
let mut bytes = synthetic_cr2_of(&samples, w, h, None, None);
|
||||
let marker = [0xe0, 0xc5, 4, 0, 1, 0, 0, 0];
|
||||
let at = bytes.windows(marker.len()).position(|b| b == marker).unwrap();
|
||||
// The explicit tag remains authoritative even when the observed sensor phase disagrees.
|
||||
bytes[at + 8..at + 12].copy_from_slice(&4u32.to_le_bytes());
|
||||
let explicit = crate::decode(&bytes).unwrap();
|
||||
assert_eq!(explicit.cfa.as_ref().unwrap().name(), "GRBG");
|
||||
assert_eq!(explicit.info(), crate::probe_info(&bytes).unwrap());
|
||||
assert!(!explicit.notices.assumed_cfa);
|
||||
bytes[at..at + 2].copy_from_slice(&0xc5dfu16.to_le_bytes());
|
||||
let full = crate::decode(&bytes).unwrap();
|
||||
assert_eq!(full.cfa.as_ref().unwrap().name(), "RGGB");
|
||||
assert!(full.notices.assumed_cfa);
|
||||
let header = decode(&bytes, Mode::Header, &Limits::default()).unwrap();
|
||||
assert!(header.notices.assumed_cfa);
|
||||
assert!(matches!(header.data, RawData::U16(ref v) if v.is_empty()));
|
||||
assert_eq!(full.data, RawData::U16(samples));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inconclusive_cfa_measurement_keeps_observed_model_fallback_and_safe_geometry() {
|
||||
let data = vec![512; 64 * 32];
|
||||
let area = Rect::new(0, 0, 64, 32);
|
||||
assert_eq!(fallback_cfa(&data, 64, area, 512.0, 16000.0, Some(0x8000_0250)).0.name(), "GBRG");
|
||||
assert!(fallback_cfa(&data, 64, area, 512.0, 16000.0, Some(0x8000_0250)).1);
|
||||
assert_eq!(fallback_cfa(&data, 64, area, 512.0, 16000.0, None).0.name(), "RGGB");
|
||||
assert!(fallback_cfa(&data, 64, area, 512.0, 16000.0, None).1);
|
||||
let gradient: Vec<u16> = (0..192 * 96)
|
||||
.map(|i| {
|
||||
let (x, y) = (i % 192, i / 192);
|
||||
if y % 2 == 0 {
|
||||
if x % 2 == 0 { 500 } else { 1000 + 8 * x as u16 }
|
||||
} else if x % 2 == 0 {
|
||||
1000 + 8 * x as u16
|
||||
} else {
|
||||
500
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let (cfa, assumed) = fallback_cfa(&gradient, 192, Rect::new(0, 0, 192, 96), 0.0, 4000.0, None);
|
||||
assert_eq!(cfa.name(), "RGGB");
|
||||
assert!(assumed);
|
||||
let mut bytes = synthetic_cr2_of(&gradient, 192, 96, None, None);
|
||||
let marker = [0xe0, 0xc5, 4, 0, 1, 0, 0, 0];
|
||||
let at = bytes.windows(marker.len()).position(|b| b == marker).unwrap();
|
||||
// Exercise the decoder's missing-tag fallback, not the fixture's default CFA tag.
|
||||
bytes[at..at + 2].copy_from_slice(&0xc5dfu16.to_le_bytes());
|
||||
let decoded = crate::decode(&bytes).unwrap();
|
||||
assert_eq!(decoded.cfa.as_ref().unwrap().name(), "RGGB");
|
||||
assert!(decoded.notices.assumed_cfa);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_cfa_fallback_is_disclosed_in_header_and_full_decodes() {
|
||||
// A single inline LONG maker-note entry needs no external offsets.
|
||||
let mut note = Vec::new();
|
||||
note.extend_from_slice(&1u16.to_le_bytes());
|
||||
note.extend_from_slice(&CANON_MODEL_ID.to_le_bytes());
|
||||
note.extend_from_slice(&4u16.to_le_bytes());
|
||||
note.extend_from_slice(&1u32.to_le_bytes());
|
||||
note.extend_from_slice(&0x8000_0250u32.to_le_bytes());
|
||||
note.extend_from_slice(&0u32.to_le_bytes());
|
||||
let samples = vec![512u16; 64 * 32];
|
||||
let mut bytes = synthetic_cr2_of(&samples, 64, 32, None, Some(note));
|
||||
let marker = [0xe0, 0xc5, 4, 0, 1, 0, 0, 0];
|
||||
let at = bytes.windows(marker.len()).position(|b| b == marker).unwrap();
|
||||
// Remove only the CFA tag; the model ID and encoded samples stay intact.
|
||||
bytes[at..at + 2].copy_from_slice(&0xc5dfu16.to_le_bytes());
|
||||
let header = decode(&bytes, Mode::Header, &Limits::default()).unwrap();
|
||||
let full = decode(&bytes, Mode::Full, &Limits::default()).unwrap();
|
||||
for image in [&header, &full] {
|
||||
assert_eq!(image.cfa.as_ref().unwrap().name(), "GBRG");
|
||||
assert!(image.notices.assumed_cfa);
|
||||
}
|
||||
assert_eq!(header.info(), full.info());
|
||||
assert_eq!(full.data, RawData::U16(samples));
|
||||
assert!(header.data.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wb_probe() {
|
||||
let mut v = vec![0u64; 120];
|
||||
|
||||
@@ -393,6 +393,7 @@ pub(super) fn decode(bytes: &[u8], tiff: &Tiff, raw: &Ifd, mode: Mode, limits: &
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
notices: Default::default(),
|
||||
level_sources: Default::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
|
||||
+119
-1
@@ -274,6 +274,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
camera: ifd0.map(Camera::from_ifd0).unwrap_or_default(),
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
notices: Default::default(),
|
||||
level_sources: Default::default(),
|
||||
};
|
||||
// Canon's stated white level can sit below the sensor's true saturation (EOS RP: 14558
|
||||
@@ -404,6 +405,15 @@ mod tests {
|
||||
}
|
||||
|
||||
fn track(entry: &[u8], offset: u64, size: u32) -> Vec<u8> {
|
||||
let mapping = [0u32, 1, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>();
|
||||
track_with_mapping(entry, offset, size, &mapping)
|
||||
}
|
||||
|
||||
fn track_with_mapping(entry: &[u8], offset: u64, size: u32, mapping: &[u8]) -> Vec<u8> {
|
||||
track_with_table(entry, offset, size, b"stsc", mapping)
|
||||
}
|
||||
|
||||
fn track_with_table(entry: &[u8], offset: u64, size: u32, table: &[u8; 4], payload: &[u8]) -> Vec<u8> {
|
||||
let mut stsd = vec![0u8; 4];
|
||||
stsd.extend(1u32.to_be_bytes());
|
||||
stsd.extend(entry);
|
||||
@@ -413,7 +423,15 @@ mod tests {
|
||||
let mut co64 = vec![0u8; 4];
|
||||
co64.extend(1u32.to_be_bytes());
|
||||
co64.extend(offset.to_be_bytes());
|
||||
let tables = [box_bytes(b"stsd", &stsd), box_bytes(b"stsz", &stsz), box_bytes(b"co64", &co64)].concat();
|
||||
let mapping = [0u32, 1, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>();
|
||||
let offsets = if table == b"stco" { box_bytes(b"stco", payload) } else { box_bytes(b"co64", if table == b"co64" { payload } else { &co64 }) };
|
||||
let tables = [
|
||||
box_bytes(b"stsd", &stsd),
|
||||
box_bytes(b"stsc", if table == b"stsc" { payload } else { &mapping }),
|
||||
box_bytes(b"stsz", if table == b"stsz" { payload } else { &stsz }),
|
||||
offsets,
|
||||
]
|
||||
.concat();
|
||||
box_bytes(b"trak", &box_bytes(b"mdia", &box_bytes(b"minf", &box_bytes(b"stbl", &tables))))
|
||||
}
|
||||
|
||||
@@ -636,6 +654,105 @@ mod tests {
|
||||
assert!(matches!(decode(&missing, Mode::Header, &Limits::default()), Err(RawError::Corrupt(_))));
|
||||
}
|
||||
|
||||
/// Independently framed constant-value CRX: each 4x4 tile has four 2x2
|
||||
/// planes whose one-byte 0x1e entropy streams reconstruct sensor value8193.
|
||||
fn flat_raw_track(tiles: u32) -> (Vec<u8>, Vec<u8>) {
|
||||
let mut sample = Vec::new();
|
||||
let marker = |out: &mut Vec<u8>, code: u16, size: u32, flags: u32| {
|
||||
out.extend(code.to_be_bytes());
|
||||
out.extend(8u16.to_be_bytes());
|
||||
out.extend(size.to_be_bytes());
|
||||
out.extend(flags.to_be_bytes());
|
||||
};
|
||||
for tile in 0..tiles {
|
||||
marker(&mut sample, 0xff01, 4, tile << 16);
|
||||
for plane in 0..4 {
|
||||
marker(&mut sample, 0xff02, 1, (plane << 28) | 0x0800_0000);
|
||||
marker(&mut sample, 0xff03, 1, 0x0020_0000);
|
||||
}
|
||||
}
|
||||
let header_size = sample.len() as u32;
|
||||
sample.extend(std::iter::repeat_n(0x1e, tiles as usize * 4));
|
||||
let mut coding = vec![0u8; 52];
|
||||
coding[..2].copy_from_slice(&0xff00u16.to_be_bytes());
|
||||
coding[2..4].copy_from_slice(&48u16.to_be_bytes());
|
||||
coding[4..6].copy_from_slice(&0x100u16.to_be_bytes());
|
||||
for (offset, value) in [(8, tiles * 4), (12, 4), (16, 4), (20, 4), (28, header_size)] {
|
||||
coding[offset..offset + 4].copy_from_slice(&value.to_be_bytes());
|
||||
}
|
||||
coding[24] = 14;
|
||||
coding[25] = 0x40;
|
||||
let mut entry = vec![0u8; 82];
|
||||
entry[24..26].copy_from_slice(&((tiles * 4) as u16).to_be_bytes());
|
||||
entry[26..28].copy_from_slice(&4u16.to_be_bytes());
|
||||
entry.extend(box_bytes(b"CMP1", &coding));
|
||||
(box_bytes(b"CRAW", &entry), sample)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_main_sample_mapping_cannot_silently_select_a_valid_reduced_raw() {
|
||||
let mapping = [0u32, 1, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>();
|
||||
let (reduced_entry, reduced_sample) = flat_raw_track(1);
|
||||
let (main_entry, main_sample) = flat_raw_track(2);
|
||||
let main_offset = 4096 + reduced_sample.len();
|
||||
let file = |main_table: Option<(&[u8; 4], &[u8])>, main_first: bool| {
|
||||
let reduced = track_with_mapping(&reduced_entry, 4096, reduced_sample.len() as u32, &mapping);
|
||||
let tracks = if let Some((table, payload)) = main_table {
|
||||
let main = track_with_table(&main_entry, main_offset as u64, main_sample.len() as u32, table, payload);
|
||||
if main_first { [main, reduced].concat() } else { [reduced, main].concat() }
|
||||
} else {
|
||||
reduced
|
||||
};
|
||||
let mut bytes = box_bytes(b"ftyp", b"crx \0\0\0\x01crx isom");
|
||||
bytes.extend(box_bytes(b"moov", &tracks));
|
||||
assert!(bytes.len() < 4096);
|
||||
bytes.resize(4096, 0);
|
||||
bytes.extend_from_slice(&reduced_sample);
|
||||
bytes.extend_from_slice(&main_sample);
|
||||
bytes
|
||||
};
|
||||
let mut co64 = vec![0u8; 4];
|
||||
co64.extend(1u32.to_be_bytes());
|
||||
co64.extend((main_offset as u64).to_be_bytes());
|
||||
let valid_tables = [
|
||||
(b"stsc", mapping.clone()),
|
||||
(b"stsz", [0u32, main_sample.len() as u32, 1].into_iter().flat_map(u32::to_be_bytes).collect()),
|
||||
(b"co64", co64),
|
||||
(b"stco", [0u32, 1, main_offset as u32].into_iter().flat_map(u32::to_be_bytes).collect()),
|
||||
];
|
||||
for mode in [Mode::Header, Mode::Full] {
|
||||
// The smaller source is genuinely decodable, so dropping the larger
|
||||
// track would return success rather than merely a different failure.
|
||||
let reduced = decode(&file(None, false), mode, &Limits::default()).unwrap();
|
||||
assert_eq!((reduced.width, reduced.height), (4, 4));
|
||||
if mode == Mode::Full {
|
||||
assert_eq!(reduced.data, RawData::U16(vec![8193; 16]));
|
||||
}
|
||||
for main_first in [false, true] {
|
||||
for (table, payload) in &valid_tables {
|
||||
let valid = decode(&file(Some((table, payload)), main_first), mode, &Limits::default()).unwrap();
|
||||
assert_eq!((valid.width, valid.height), (8, 4));
|
||||
if mode == Mode::Full {
|
||||
assert_eq!(valid.data, RawData::U16(vec![8193; 32]));
|
||||
}
|
||||
for truncated in [vec![], vec![0; 4], vec![0, 0, 0, 0, 0, 0, 0, 1]] {
|
||||
let bytes = file(Some((table, &truncated)), main_first);
|
||||
let container = parse_cr3(&bytes).unwrap();
|
||||
assert_eq!(container.tracks.len(), 2, "mapping damage cannot erase the main track");
|
||||
let selected = main_track(&container.tracks).unwrap();
|
||||
assert!(matches!(selected.kind, Cr3TrackKind::Raw { width: 8, height: 4, .. }));
|
||||
assert!(selected.data.is_none());
|
||||
assert!(selected.compression(&bytes).is_some());
|
||||
assert!(
|
||||
matches!(decode(&bytes, mode, &Limits::default()), Err(RawError::Corrupt(message)) if message.contains("sample")),
|
||||
"{mode:?}/{table:?}: main mapping damage cannot decode the reduced source"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn color_data_uses_versioned_as_shot_offsets() {
|
||||
for (version, wb_at, black_at, white_at) in
|
||||
@@ -712,6 +829,7 @@ mod tests {
|
||||
camera: Camera::default(),
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
notices: Default::default(),
|
||||
level_sources: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,6 +65,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, caller: &Limits) -> Result<RawIma
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: Default::default(),
|
||||
notices: crate::RawNotices::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
+82
-35
@@ -5,8 +5,8 @@ use crate::gaintable::GainTableMap;
|
||||
use crate::profile::{HsvTable, ProfileLook, ToneCurve};
|
||||
use crate::tiffraw::{Packing, read_image_in};
|
||||
use crate::{
|
||||
BlackLevel, Camera, Cfa, CinemaTags, ColorData, Limits, Matrix3, Mode, Orientation, RawData, RawError, RawFormat, RawImage, Rect, Result,
|
||||
Timecode, opcodes,
|
||||
BlackLevel, Camera, Cfa, CinemaTags, ColorData, LevelSource, LevelSources, Limits, Matrix3, Mode, Orientation, RawData, RawError, RawFormat,
|
||||
RawImage, RawNotices, Rect, Result, Timecode, opcodes,
|
||||
};
|
||||
use craft_tiff::tags::{self as t, photometric};
|
||||
use craft_tiff::{ByteOrder, Ifd, Tiff, Value};
|
||||
@@ -178,7 +178,15 @@ pub(crate) fn is_plain_cfa_tiff(tiff: &Tiff, bytes: &[u8]) -> bool {
|
||||
if !raw.contains(t::AS_SHOT_NEUTRAL) && !tiff.ifds.first().is_some_and(|i| i.contains(t::AS_SHOT_NEUTRAL)) {
|
||||
return false;
|
||||
}
|
||||
let Ok(info) = raw.image() else { return false };
|
||||
let Ok(info) = raw.image() else {
|
||||
// Recognition must not turn a damaged, otherwise supported plain CFA
|
||||
// image into an unsupported container. Let decode_as report its layout
|
||||
// error; these tags alone never decompress or allocate pixel samples.
|
||||
return raw.u16(t::COMPRESSION).unwrap_or(1) == t::compression::NONE
|
||||
&& raw.u16(t::SAMPLES_PER_PIXEL).unwrap_or(1) == 1
|
||||
&& (1..=16).contains(&raw.u16(t::BITS_PER_SAMPLE).unwrap_or(1))
|
||||
&& raw.u16(t::SAMPLE_FORMAT).unwrap_or(1) != 3;
|
||||
};
|
||||
if info.samples_per_pixel != 1 || !(1..=16).contains(&info.bits()) || info.sample_format == 3 {
|
||||
return false;
|
||||
}
|
||||
@@ -274,8 +282,13 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
let bits = info.bits() as u32;
|
||||
// DNG 1.4 stores lossy JPEG at 8 bits per sample: the tiles are 8-bit baseline JPEGs whatever the tag says,
|
||||
// and reading them as a deeper sample would put the picture at the bottom of its range
|
||||
if info.compression == t::compression::LOSSY_JPEG && bits != 8 {
|
||||
return Err(RawError::Unsupported(format!("lossy JPEG DNG with {bits}-bit samples (DNG allows only 8)")));
|
||||
if info.compression == t::compression::LOSSY_JPEG {
|
||||
if bits != 8 {
|
||||
return Err(RawError::Unsupported(format!("lossy JPEG DNG with {bits}-bit samples (DNG allows only 8)")));
|
||||
}
|
||||
if !matches!(cpp, 1 | 3) {
|
||||
return Err(RawError::Unsupported(format!("lossy JPEG DNG with {cpp} samples per pixel")));
|
||||
}
|
||||
}
|
||||
let mut data = read_image_in(mode, bytes, &info, tiff.order, Packing::Msb, limits)?;
|
||||
|
||||
@@ -312,6 +325,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
// (a DOUBLE past `f32::MAX` casts to infinity, which would black the picture out: ignored like a zero one)
|
||||
let mut white: Vec<f32> =
|
||||
levels(t::WHITE_LEVEL).f64s(t::WHITE_LEVEL).unwrap_or_default().into_iter().map(|v| v as f32).filter(|v| v.is_finite() && *v > 0.0).collect();
|
||||
let white_source = if !white.is_empty() || table_max.is_some() { LevelSource::Metadata } else { LevelSource::Fallback };
|
||||
if white.is_empty() {
|
||||
white = vec![default_white];
|
||||
}
|
||||
@@ -333,7 +347,13 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
};
|
||||
// (a FLOAT / DOUBLE tag can hold NaN or infinity: such a level reads as 0, in its place)
|
||||
let level = |v: f64| Some(v as f32).filter(|v| v.is_finite()).unwrap_or(0.0);
|
||||
let mut values: Vec<f32> = levels(t::BLACK_LEVEL).f64s(t::BLACK_LEVEL).unwrap_or_default().into_iter().map(level).collect();
|
||||
let declared_black = levels(t::BLACK_LEVEL).f64s(t::BLACK_LEVEL).unwrap_or_default();
|
||||
let black_source = if !declared_black.is_empty() && declared_black.iter().all(|value| (*value as f32).is_finite()) {
|
||||
LevelSource::Metadata
|
||||
} else {
|
||||
LevelSource::Fallback
|
||||
};
|
||||
let mut values: Vec<f32> = declared_black.into_iter().map(level).collect();
|
||||
// A TIFF that is not a DNG may state four black levels without `BlackLevelRepeatDim`: one per site of its 2 × 2
|
||||
// CFA, row-major (PhotoCraft's reading of Hasselblad files). In a DNG the count has to match the repeat.
|
||||
let (br, bc) = if format != RawFormat::Dng && cpp == 1 && values.len() == 4 && !levels(t::BLACK_LEVEL).contains(t::BLACK_LEVEL_REPEAT_DIM) {
|
||||
@@ -364,7 +384,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
}
|
||||
_ => Rect::new(0, 0, full_w, full_h),
|
||||
};
|
||||
let crop = default_crop(raw, active_area);
|
||||
let (crop, default_crop_adjusted) = default_crop(raw, active_area, full_w, full_h);
|
||||
let (active_area, crop) = if (full_w, full_h) == (w, h) {
|
||||
(active_area, crop)
|
||||
} else {
|
||||
@@ -378,6 +398,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
(area, scale(crop).clipped(area.width, area.height))
|
||||
};
|
||||
|
||||
let mut assumed_cfa = false;
|
||||
let cfa = if info.photometric == photometric::CFA {
|
||||
if raw.u16(t::CFA_LAYOUT).unwrap_or(1) != 1 {
|
||||
return Err(RawError::Unsupported("non-rectangular CFA layout".into()));
|
||||
@@ -391,7 +412,10 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
let assumed = [0u8, 1, 1, 2];
|
||||
let pat = match raw.bytes(t::CFA_PATTERN_EP) {
|
||||
Some(p) => p,
|
||||
None if format != RawFormat::Dng && (rows, cols) == (2, 2) => &assumed,
|
||||
None if format != RawFormat::Dng && (rows, cols) == (2, 2) => {
|
||||
assumed_cfa = true;
|
||||
&assumed
|
||||
}
|
||||
None => return Err(RawError::Corrupt("missing CFAPattern".into())),
|
||||
};
|
||||
if pat.len() != rows * cols {
|
||||
@@ -422,6 +446,9 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
let opcodes =
|
||||
crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) };
|
||||
|
||||
let color = color_data(ifd0, raw, tiff.order);
|
||||
let gain_table_omitted = [raw, ifd0].iter().any(|ifd| ifd.contains(t::PROFILE_GAIN_TABLE_MAP) || ifd.contains(t::PROFILE_GAIN_TABLE_MAP_2))
|
||||
&& color.profile.gain_table_map.is_none();
|
||||
let img = RawImage {
|
||||
format,
|
||||
width: w,
|
||||
@@ -435,7 +462,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
active_area,
|
||||
crop,
|
||||
orientation: Orientation::from_exif(ifd0.u16(t::ORIENTATION).unwrap_or(1)),
|
||||
color: color_data(ifd0, raw, tiff.order),
|
||||
color,
|
||||
wb_multipliers: None,
|
||||
linearized,
|
||||
pixel_aspect: default_scale_aspect(raw.f64s(t::DEFAULT_SCALE).as_deref()),
|
||||
@@ -443,7 +470,8 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
camera: Camera::from_ifd0(ifd0),
|
||||
cinema: cinema_tags(ifd0, raw),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: Default::default(),
|
||||
level_sources: LevelSources { black: black_source, white: white_source },
|
||||
notices: RawNotices { default_crop_adjusted, gain_table_omitted, assumed_cfa },
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
@@ -472,33 +500,45 @@ fn cinema_tags(ifd0: &Ifd, raw: &Ifd) -> CinemaTags {
|
||||
}
|
||||
}
|
||||
|
||||
/// The default crop, relative to the active area. Each tag has its own default (DNG 1.7.1, `DefaultCropOrigin`
|
||||
/// and `DefaultCropSize`): the origin (0, 0), the size the whole area, so a file with only a size crops from the
|
||||
/// area's corner. A tag that is present must hold two finite, non-negative values, and the crop must start inside
|
||||
/// the area and be at least one pixel each way; otherwise the crop is the whole active area.
|
||||
fn default_crop(raw: &Ifd, active_area: Rect) -> Rect {
|
||||
/// The default crop is relative to ActiveArea. DNG 1.7.1 defines independent
|
||||
/// defaults: origin (0, 0), size ImageWidth/ImageLength. An unusable optional crop
|
||||
/// preserves the entire active image. Valid partly overlapping rectangles retain the
|
||||
/// existing clipped-crop policy, with an explicit notice rather than silent adjustment.
|
||||
fn default_crop(raw: &Ifd, active_area: Rect, full_width: usize, full_height: usize) -> (Rect, bool) {
|
||||
let whole = Rect::new(0, 0, active_area.width, active_area.height);
|
||||
let pair = |tag: u16, default: [f64; 2]| -> Option<[f64; 2]> {
|
||||
if !raw.contains(tag) {
|
||||
return Some(default);
|
||||
}
|
||||
match raw.f64s(tag).as_deref() {
|
||||
Some([a, b]) if a.is_finite() && b.is_finite() && *a >= 0.0 && *b >= 0.0 => Some([*a, *b]),
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
let (Some([x, y]), Some([cw, ch])) =
|
||||
(pair(t::DEFAULT_CROP_ORIGIN, [0.0, 0.0]), pair(t::DEFAULT_CROP_SIZE, [active_area.width as f64, active_area.height as f64]))
|
||||
else {
|
||||
return whole;
|
||||
};
|
||||
if cw < 1.0 || ch < 1.0 {
|
||||
return whole;
|
||||
if !raw.contains(t::DEFAULT_CROP_ORIGIN) && !raw.contains(t::DEFAULT_CROP_SIZE) {
|
||||
return (whole, false);
|
||||
}
|
||||
// (`as usize` saturates, and `clipped` holds the origin to the area)
|
||||
let crop =
|
||||
Rect::new(x.round() as usize, y.round() as usize, cw.round() as usize, ch.round() as usize).clipped(active_area.width, active_area.height);
|
||||
if crop.width == 0 || crop.height == 0 { whole } else { crop }
|
||||
let pair = |tag: u16, default: [f64; 2]| -> Option<[usize; 2]> {
|
||||
let values = if raw.contains(tag) {
|
||||
match raw.f64s(tag).as_deref() {
|
||||
Some([a, b]) => [*a, *b],
|
||||
_ => return None,
|
||||
}
|
||||
} else {
|
||||
default
|
||||
};
|
||||
// Float-to-integer casts saturate. Refuse nonfinite/unrepresentable numbers before
|
||||
// casting so a hostile origin cannot become a seemingly valid clipped rectangle.
|
||||
if values.iter().any(|value| !value.is_finite() || value.round() >= usize::MAX as f64) {
|
||||
return None;
|
||||
}
|
||||
if values.iter().any(|value| *value < 0.0) {
|
||||
return None;
|
||||
}
|
||||
Some(values.map(|value| value.round() as usize))
|
||||
};
|
||||
let (Some([x, y]), Some([width, height])) =
|
||||
(pair(t::DEFAULT_CROP_ORIGIN, [0.0, 0.0]), pair(t::DEFAULT_CROP_SIZE, [full_width as f64, full_height as f64]))
|
||||
else {
|
||||
return (whole, true);
|
||||
};
|
||||
let requested = Rect::new(x, y, width, height);
|
||||
let crop = requested.clipped(active_area.width, active_area.height);
|
||||
if crop.width == 0 || crop.height == 0 {
|
||||
return (whole, true);
|
||||
}
|
||||
(crop, crop != requested)
|
||||
}
|
||||
|
||||
/// Pixel height over width from `DefaultScale` (horizontal, vertical): only a clear 1.5 to 4 ratio counts (the rows stretch; a wider-than-tall pixel is left alone).
|
||||
@@ -509,6 +549,9 @@ fn default_scale_aspect(scale: Option<&[f64]>) -> f32 {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod compat_review_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
@@ -621,3 +664,7 @@ mod tests {
|
||||
assert!(deinterleave(&mut hostile, 8, 6, 1, 1 << 20, 1).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "hasselblad"))]
|
||||
#[path = "dng/probe_review_tests.rs"]
|
||||
mod probe_review_tests;
|
||||
@@ -0,0 +1,120 @@
|
||||
use super::*;
|
||||
use craft_tiff::{Entry, IfdBuilder, ImageData, TiffWriter};
|
||||
|
||||
fn ifd(tags: Vec<(u16, Value)>) -> Ifd {
|
||||
let mut entries: Vec<_> = tags.into_iter().map(|(tag, value)| Entry { tag, value, offset: 0 }).collect();
|
||||
entries.sort_by_key(|entry| entry.tag);
|
||||
Ifd { entries, ..Default::default() }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optional_crop_adjustments_are_explicit_and_valid_clipping_is_preserved() {
|
||||
let active = Rect::new(3, 2, 11, 8);
|
||||
let whole = Rect::new(0, 0, 11, 8);
|
||||
let clipped = ifd(vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![10, 0])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]);
|
||||
assert_eq!(default_crop(&clipped, active, 16, 12), (Rect::new(10, 0, 1, 4), true));
|
||||
assert_eq!(default_crop(&ifd(vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![1, 1]))]), active, 16, 12), (Rect::new(1, 1, 10, 7), true));
|
||||
for fields in [
|
||||
vec![(t::DEFAULT_CROP_SIZE, Value::Long(vec![4]))],
|
||||
vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![0, 0])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![0, 4]))],
|
||||
] {
|
||||
assert_eq!(default_crop(&ifd(fields), active, 16, 12), (whole, true));
|
||||
}
|
||||
assert_eq!(default_crop(&Ifd::default(), active, 16, 12), (whole, false));
|
||||
assert_eq!(default_crop(&ifd(vec![(t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]), active, 16, 12), (Rect::new(0, 0, 5, 4), false));
|
||||
assert_eq!(
|
||||
default_crop(&ifd(vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![2, 1])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]), active, 16, 12),
|
||||
(Rect::new(2, 1, 5, 4), false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_crop_coordinates_fall_back_explicitly_before_saturating_casts() {
|
||||
for bad in [1e300, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
|
||||
for tag in [t::DEFAULT_CROP_ORIGIN, t::DEFAULT_CROP_SIZE] {
|
||||
let raw = ifd(vec![(tag, Value::Double(vec![bad, 1.]))]);
|
||||
assert_eq!(default_crop(&raw, Rect::new(0, 0, 16, 12), 16, 12), (Rect::new(0, 0, 16, 12), true));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn linear(extra: &[(u16, Value)]) -> Vec<u8> {
|
||||
let mut raw = IfdBuilder::new();
|
||||
for (tag, value) in [
|
||||
(t::IMAGE_WIDTH, Value::Long(vec![8])),
|
||||
(t::IMAGE_LENGTH, Value::Long(vec![6])),
|
||||
(t::BITS_PER_SAMPLE, Value::Short(vec![16])),
|
||||
(t::SAMPLES_PER_PIXEL, Value::Short(vec![1])),
|
||||
(t::PHOTOMETRIC, Value::Short(vec![photometric::LINEAR_RAW])),
|
||||
(t::COMPRESSION, Value::Short(vec![1])),
|
||||
] {
|
||||
raw.set(tag, value);
|
||||
}
|
||||
for (tag, value) in extra {
|
||||
raw.set(*tag, value.clone());
|
||||
}
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: 6, strips: vec![(0u16..48).flat_map(u16::to_le_bytes).collect()] });
|
||||
TiffWriter::new(ByteOrder::Little, false).write(&[raw]).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dng_and_generic_cfa_tiff_report_actual_level_sources_instead_of_unknown() {
|
||||
for format in [RawFormat::Dng, RawFormat::CfaTiff] {
|
||||
let missing = decode_as(&linear(&[]), Mode::Full, format, &Limits::default()).unwrap();
|
||||
assert_eq!(missing.level_sources, LevelSources { black: LevelSource::Fallback, white: LevelSource::Fallback });
|
||||
let declared = decode_as(
|
||||
&linear(&[(t::BLACK_LEVEL, Value::Short(vec![17])), (t::WHITE_LEVEL, Value::Short(vec![1000]))]),
|
||||
Mode::Full,
|
||||
format,
|
||||
&Limits::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(declared.level_sources, LevelSources { black: LevelSource::Metadata, white: LevelSource::Metadata });
|
||||
assert_eq!(declared.black.values, vec![17.]);
|
||||
assert_eq!(declared.white, vec![1000.]);
|
||||
let invalid = decode_as(
|
||||
&linear(&[(t::BLACK_LEVEL, Value::Double(vec![f64::NAN])), (t::WHITE_LEVEL, Value::Double(vec![f64::INFINITY]))]),
|
||||
Mode::Full,
|
||||
format,
|
||||
&Limits::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(invalid.level_sources, LevelSources { black: LevelSource::Fallback, white: LevelSource::Fallback });
|
||||
}
|
||||
let table = decode_as(&linear(&[(t::LINEARIZATION_TABLE, Value::Short(vec![0, 1000]))]), Mode::Full, RawFormat::Dng, &Limits::default()).unwrap();
|
||||
assert_eq!(table.white, vec![1000.]);
|
||||
assert_eq!(table.level_sources.white, LevelSource::Metadata);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_notices_record_actual_crop_gain_table_and_cfa_decisions() {
|
||||
let plain = decode_as(&linear(&[]), Mode::Full, RawFormat::Dng, &Limits::default()).unwrap();
|
||||
assert_eq!(plain.notices, RawNotices::default());
|
||||
let adjusted = decode_as(
|
||||
&linear(&[(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![7, 0])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]),
|
||||
Mode::Full,
|
||||
RawFormat::Dng,
|
||||
&Limits::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(adjusted.crop, Rect::new(7, 0, 1, 4));
|
||||
assert!(adjusted.notices.default_crop_adjusted);
|
||||
let invalid_gain =
|
||||
decode_as(&linear(&[(t::PROFILE_GAIN_TABLE_MAP, Value::Undefined(vec![0; 16]))]), Mode::Full, RawFormat::Dng, &Limits::default()).unwrap();
|
||||
assert!(invalid_gain.notices.gain_table_omitted);
|
||||
assert!(invalid_gain.color.profile.gain_table_map.is_none());
|
||||
let assumed =
|
||||
decode_as(&linear(&[(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA]))]), Mode::Full, RawFormat::CfaTiff, &Limits::default()).unwrap();
|
||||
assert!(assumed.notices.assumed_cfa);
|
||||
assert_eq!(assumed.cfa.as_ref().unwrap().pattern, vec![0, 1, 1, 2]);
|
||||
let tagged = decode_as(
|
||||
&linear(&[(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])), (t::CFA_PATTERN_EP, Value::Byte(vec![0, 1, 1, 2]))]),
|
||||
Mode::Full,
|
||||
RawFormat::Dng,
|
||||
&Limits::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!tagged.notices.assumed_cfa);
|
||||
let limits = Limits { max_samples: 47, ..Default::default() };
|
||||
assert!(matches!(decode_as(&linear(&[]), Mode::Full, RawFormat::Dng, &limits), Err(RawError::Limit(_))));
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
use super::*;
|
||||
use craft_tiff::{IfdBuilder, ImageData, TiffWriter};
|
||||
|
||||
#[allow(clippy::unwrap_used)] // Test fixture construction must fail the test if its independent writer fails.
|
||||
fn plain() -> Vec<u8> {
|
||||
let mut raw = IfdBuilder::new();
|
||||
for (tag, value) in [
|
||||
(t::IMAGE_WIDTH, Value::Long(vec![2])),
|
||||
(t::IMAGE_LENGTH, Value::Long(vec![2])),
|
||||
(t::BITS_PER_SAMPLE, Value::Short(vec![16])),
|
||||
(t::SAMPLES_PER_PIXEL, Value::Short(vec![1])),
|
||||
(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])),
|
||||
(t::COMPRESSION, Value::Short(vec![1])),
|
||||
(t::AS_SHOT_NEUTRAL, Value::Rational(vec![(1, 2), (1, 1), (2, 3)])),
|
||||
] {
|
||||
raw.set(tag, value);
|
||||
}
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: 2, strips: vec![[17u16, 31, 53, 79].into_iter().flat_map(u16::to_le_bytes).collect()] });
|
||||
TiffWriter::new(ByteOrder::Little, false).write(&[raw]).unwrap()
|
||||
}
|
||||
|
||||
#[allow(clippy::unwrap_used)] // Only known tags in this module's synthetic TIFF are inspected.
|
||||
fn entry(bytes: &[u8], tag: u16) -> usize {
|
||||
let at = u32::from_le_bytes(bytes[4..8].try_into().unwrap()) as usize;
|
||||
let count = u16::from_le_bytes(bytes[at..at + 2].try_into().unwrap()) as usize;
|
||||
(0..count).map(|i| at + 2 + i * 12).find(|&i| u16::from_le_bytes(bytes[i..i + 2].try_into().unwrap()) == tag).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_cfa_probe_keeps_malformed_geometry_and_tables_in_the_supported_decoder() {
|
||||
let valid = plain();
|
||||
assert_eq!(crate::probe(&valid), Some(RawFormat::CfaTiff));
|
||||
let decoded = crate::decode(&valid).unwrap();
|
||||
assert_eq!(decoded.data, RawData::U16(vec![17, 31, 53, 79]));
|
||||
for tag in [t::IMAGE_WIDTH, t::IMAGE_LENGTH] {
|
||||
let mut damaged = valid.clone();
|
||||
let at = entry(&damaged, tag);
|
||||
damaged[at + 8..at + 12].copy_from_slice(&0u32.to_le_bytes());
|
||||
assert_eq!(crate::probe(&damaged), Some(RawFormat::CfaTiff));
|
||||
assert!(matches!(crate::probe_info(&damaged), Err(RawError::Tiff(_))), "zero geometry remains malformed before pixel decode");
|
||||
assert!(matches!(crate::decode(&damaged), Err(RawError::Tiff(_))));
|
||||
}
|
||||
let mut missing = valid.clone();
|
||||
let at = entry(&missing, t::STRIP_OFFSETS);
|
||||
missing[at..at + 2].copy_from_slice(&0xf123u16.to_le_bytes());
|
||||
assert_eq!(crate::probe(&missing), Some(RawFormat::CfaTiff));
|
||||
assert!(matches!(crate::probe_info(&missing), Err(RawError::Tiff(_))));
|
||||
assert!(matches!(crate::decode(&missing), Err(RawError::Tiff(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_cfa_probe_does_not_claim_unknown_tiff_codings_or_missing_color_contracts() {
|
||||
for tag in [t::AS_SHOT_NEUTRAL, t::COMPRESSION] {
|
||||
let mut unknown = plain();
|
||||
let at = entry(&unknown, tag);
|
||||
if tag == t::COMPRESSION {
|
||||
unknown[at + 8..at + 10].copy_from_slice(&65000u16.to_le_bytes());
|
||||
} else {
|
||||
unknown[at..at + 2].copy_from_slice(&0xf123u16.to_le_bytes());
|
||||
}
|
||||
assert_eq!(crate::probe(&unknown), Some(RawFormat::OtherTiff));
|
||||
assert!(matches!(crate::decode(&unknown), Err(RawError::Unsupported(_))));
|
||||
}
|
||||
}
|
||||
@@ -462,6 +462,7 @@ mod tests {
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: Default::default(),
|
||||
notices: crate::RawNotices::default(),
|
||||
};
|
||||
let opts = |tile: u32| DngWriteOptions { compression: DngCompression::Deflate { tile, half: false }, ..DngWriteOptions::default() };
|
||||
let hostile = write_dng(&raw, &opts(u32::MAX)).expect("the clamped tile writes");
|
||||
|
||||
@@ -128,6 +128,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: crate::LevelSources { black: black_source, white: white_source },
|
||||
notices: crate::RawNotices::default(),
|
||||
pixel_aspect: 1.0,
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
|
||||
@@ -121,6 +121,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: Default::default(),
|
||||
notices: crate::RawNotices::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
@@ -997,6 +997,20 @@ pub struct ColorData {
|
||||
pub kelvin_presets: Option<kelvin::KelvinPresets>,
|
||||
}
|
||||
|
||||
/// Bounded diagnostics for decisions actually made by a raw decoder. An absent optional tag
|
||||
/// does not itself set a notice; hosts can translate these flags into their own warning text
|
||||
/// without inferring a fallback from missing metadata or retaining source-controlled strings.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct RawNotices {
|
||||
/// Explicit default-crop metadata needed correction or a fallback to the active area.
|
||||
pub default_crop_adjusted: bool,
|
||||
/// A source profile gain table could not be used and was omitted from the decoded image.
|
||||
pub gain_table_omitted: bool,
|
||||
/// The CFA layout was selected by a decoder fallback instead of usable source metadata.
|
||||
pub assumed_cfa: bool,
|
||||
}
|
||||
|
||||
/// A decoded raw image.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct RawImage {
|
||||
@@ -1041,6 +1055,9 @@ pub struct RawImage {
|
||||
/// measured levels are placeholders.
|
||||
#[serde(default)]
|
||||
pub level_sources: LevelSources,
|
||||
/// Fixed-size decoder decisions, separate from ordinary missing optional metadata.
|
||||
#[serde(default)]
|
||||
pub notices: RawNotices,
|
||||
}
|
||||
|
||||
impl RawImage {
|
||||
@@ -1173,3 +1190,21 @@ mod checked_arithmetic_tests {
|
||||
assert_eq!(Orientation::from_exif(9), Orientation::Normal);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod raw_notices_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn missing_notices_default_to_no_claims_and_explicit_flags_are_independent() {
|
||||
type Error = serde::de::value::Error;
|
||||
let empty = serde::de::value::MapDeserializer::<_, Error>::new(std::iter::empty::<(&str, bool)>());
|
||||
assert_eq!(RawNotices::deserialize(empty).unwrap(), RawNotices::default());
|
||||
let one = serde::de::value::MapDeserializer::<_, Error>::new([("assumed_cfa", true)].into_iter());
|
||||
let notices = RawNotices::deserialize(one).unwrap();
|
||||
let copied = notices;
|
||||
assert_eq!(notices, copied);
|
||||
assert_eq!(notices, RawNotices { assumed_cfa: true, default_crop_adjusted: false, gain_table_omitted: false });
|
||||
assert!(!std::mem::needs_drop::<RawNotices>(), "decoder notices own no allocated strings");
|
||||
}
|
||||
}
|
||||
@@ -128,6 +128,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: crate::LevelSources { black: crate::LevelSource::Fallback, white: white_source },
|
||||
notices: crate::RawNotices::default(),
|
||||
pixel_aspect: 1.0,
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
|
||||
@@ -170,6 +170,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: crate::LevelSources { black: crate::LevelSource::Fallback, white: white_source },
|
||||
notices: crate::RawNotices::default(),
|
||||
pixel_aspect: 1.0,
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
|
||||
+144
-23
@@ -13,7 +13,7 @@
|
||||
//! refused.
|
||||
|
||||
use crate::nefc;
|
||||
use crate::shared::white_from_data;
|
||||
use crate::shared::white_from_data_sourced;
|
||||
use crate::tiffraw::{Packing, check_image, read_image};
|
||||
use crate::{BlackLevel, Camera, Cfa, ColorData, Limits, Mode, OpcodeLists, Orientation, RawData, RawError, RawFormat, RawImage, Rect, Result};
|
||||
use craft_tiff::image::ImageInfo;
|
||||
@@ -266,35 +266,44 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
// D7500 and Z 50 store 600 / 1008 / 400 / 400 / 1008 while their darkest samples are 150 / ~252 / 99 / 99 / ~250
|
||||
// (our measurements on CC0 raw.pixls.us samples; the same bodies' 14-bit files match the tag as is).
|
||||
let black_scale = if bits == 12 { 0.25 } else { 1.0 };
|
||||
let black = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_LEVEL)).as_deref() {
|
||||
Some([a, b, c, d]) if [a, b, c, d].iter().all(|v| **v < 16384.0) => BlackLevel {
|
||||
repeat_rows: 2,
|
||||
repeat_cols: 2,
|
||||
values: [a, b, c, d].iter().map(|v| (**v * black_scale) as f32).collect(),
|
||||
..Default::default()
|
||||
let (black, black_source) = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_LEVEL)).as_deref() {
|
||||
Some([a, b, c, d]) if [a, b, c, d].iter().all(|v| **v < 16384.0) => (
|
||||
BlackLevel {
|
||||
repeat_rows: 2,
|
||||
repeat_cols: 2,
|
||||
values: [a, b, c, d].iter().map(|v| (**v * black_scale) as f32).collect(),
|
||||
..Default::default()
|
||||
},
|
||||
crate::LevelSource::Metadata,
|
||||
),
|
||||
_ => match nrw_black(mn.as_ref(), bits) {
|
||||
Some(value) => (BlackLevel::uniform(value), crate::LevelSource::Metadata),
|
||||
None => {
|
||||
let (value, source) = be16_black(&info, bytes.len() as u64, mn.as_ref(), samples);
|
||||
(BlackLevel::uniform(value), source)
|
||||
}
|
||||
},
|
||||
_ => BlackLevel::uniform(nrw_black(mn.as_ref(), bits).unwrap_or_else(|| be16_black(&info, bytes.len() as u64, mn.as_ref(), samples))),
|
||||
};
|
||||
let wb = mn
|
||||
.as_ref()
|
||||
.and_then(|m| m.ifd.f64s(WB_RB_LEVELS))
|
||||
.filter(|v| v.len() >= 2 && v[0] > 0.1 && v[1] > 0.1 && v[0] < 10.0 && v[1] < 10.0)
|
||||
.map(|v| [v[0] as f32, 1.0, v[1] as f32]);
|
||||
let cfa = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
|
||||
let (cfa, assumed_cfa) = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
|
||||
(Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => {
|
||||
Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }
|
||||
(Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }, false)
|
||||
}
|
||||
_ => Cfa::bayer_static("RGGB"),
|
||||
_ => (Cfa::bayer_static("RGGB"), true),
|
||||
};
|
||||
let model = ifd0.string(t::MODEL).unwrap_or_default();
|
||||
let body = measured_body(&model, w, h, bits);
|
||||
let active_w = trailing_masked_columns(samples, w, bits);
|
||||
// the white level is measured from the whole image (or known for a measured body); the probe (no samples in its
|
||||
// result) takes the nominal one
|
||||
let white = match (body.and_then(|b| b.white), mode) {
|
||||
(Some(white), _) => white,
|
||||
(None, Mode::Full) => white_from_data(samples, bits),
|
||||
(None, Mode::Header) => ((1u32 << bits.clamp(1, 16)) - 1) as f32,
|
||||
let (white, white_source) = match (body.and_then(|b| b.white), mode) {
|
||||
(Some(white), _) => (white, crate::LevelSource::Fallback),
|
||||
(None, Mode::Full) => white_from_data_sourced(samples, bits),
|
||||
(None, Mode::Header) => (((1u32 << bits.clamp(1, 16)) - 1) as f32, crate::LevelSource::Fallback),
|
||||
};
|
||||
let crop = match body {
|
||||
Some(&MeasuredBody { crop: (x, y, cw, ch), .. }) if x + cw <= active_w && y + ch <= h => Rect::new(x, y, cw, ch),
|
||||
@@ -329,7 +338,8 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: Default::default(),
|
||||
level_sources: crate::LevelSources { black: black_source, white: white_source },
|
||||
notices: crate::RawNotices { assumed_cfa, ..Default::default() },
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
@@ -378,8 +388,8 @@ fn uncompressed(
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum PackedLayout {
|
||||
/// LSB-first bit stream of `bits` per sample (12: 3 bytes per 2 samples, 14: 7 bytes per 4), rows byte aligned.
|
||||
/// 12-bit rows are `3w/2` bytes (older bodies) or padded to 16 bytes (Z 6, D6); 14-bit rows are `7w/4` bytes
|
||||
/// padded to 16 bytes.
|
||||
/// Rows use `ceil(bits * width / 8)` bytes, optionally padded to 16 bytes. Without an explicit packed maker
|
||||
/// mode, recognition requires complete sample groups and 14-bit rows padded to 16 bytes.
|
||||
Lsb { stride: usize },
|
||||
/// D100: 16-byte chunks of fifteen data bytes holding ten MSB-first 12-bit samples, plus one zero pad byte. The
|
||||
/// rows carry more samples than the IFD width (3040 against 3034); the extra columns are not image.
|
||||
@@ -399,7 +409,11 @@ fn packed_layout(w: usize, h: usize, bits: u32, strip: u64, stated: bool) -> Opt
|
||||
}
|
||||
let stride = strip / h;
|
||||
let tight = w.checked_mul(bits as usize)?.div_ceil(8);
|
||||
let tight = if bits == 12 && w.is_multiple_of(2) || bits == 14 && w.is_multiple_of(4) { tight } else { return None };
|
||||
// Unmarked data must still match a measured complete-group layout: do not guess a packing for compressed
|
||||
// data. An explicit packed maker mode permits a final partial group, byte-aligned independently on each row.
|
||||
if !stated && !(bits == 12 && w.is_multiple_of(2) || bits == 14 && w.is_multiple_of(4)) {
|
||||
return None;
|
||||
}
|
||||
let padded = tight.checked_next_multiple_of(16)?;
|
||||
let lsb_ok = match bits {
|
||||
12 => stride == tight || stride == padded,
|
||||
@@ -436,14 +450,14 @@ fn be16_layout(w: usize, h: usize, bits: u32, strip: u64) -> Option<PackedLayout
|
||||
/// 223), and the other 12-bit Coolpix NRWs state 200 in their NRW block; so 200, never above the darkest sample.
|
||||
const BE16_BLACK: f32 = 200.0;
|
||||
|
||||
fn be16_black(info: &ImageInfo, file_len: u64, mn: Option<&makernote::MakerNote>, samples: &[u16]) -> f32 {
|
||||
fn be16_black(info: &ImageInfo, file_len: u64, mn: Option<&makernote::MakerNote>, samples: &[u16]) -> (f32, crate::LevelSource) {
|
||||
let has_table = mn.is_some_and(|m| m.ifd.bytes(LINEARIZATION_TABLE).is_some());
|
||||
let single = matches!(info.chunks(file_len).as_slice(), [c] if be16_layout(info.width as usize, info.height as usize, info.bits() as u32, c.len).is_some());
|
||||
if info.compression != t::compression::NIKON || has_table || info.samples_per_pixel != 1 || !single {
|
||||
return 0.0;
|
||||
return (0.0, crate::LevelSource::Fallback);
|
||||
}
|
||||
let darkest = samples.iter().copied().min().unwrap_or(0);
|
||||
BE16_BLACK.min(f32::from(darkest))
|
||||
(BE16_BLACK.min(f32::from(darkest)), crate::LevelSource::Inferred)
|
||||
}
|
||||
|
||||
/// Unpack a [`PackedLayout`] strip into `w * h` samples. Short input leaves the missing samples zero.
|
||||
@@ -524,7 +538,7 @@ fn compressed(bytes: &[u8], info: &ImageInfo, mn: Option<&makernote::MakerNote>,
|
||||
}
|
||||
|
||||
/// A "compressed" strip of packed samples: the strip size must match a measured layout exactly. `stated` is set
|
||||
/// when the maker note names the packed mode, which also allows 14-bit rows without padding.
|
||||
/// when the maker note names the packed mode, which also allows partial groups and 14-bit rows without padding.
|
||||
fn packed(bytes: &[u8], info: &ImageInfo, rows: usize, stated: bool, limits: &Limits) -> Result<RawData> {
|
||||
let (w, h) = (info.width as usize, info.height as usize);
|
||||
let bits = info.bits() as u32;
|
||||
@@ -663,6 +677,7 @@ fn small_raw(bytes: &[u8], tiff: &Tiff, info: &ImageInfo, strip: craft_tiff::ima
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: Default::default(),
|
||||
notices: Default::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
@@ -725,6 +740,21 @@ mod tests {
|
||||
TiffWriter::new(order, false).write(&[ifd0]).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn level_provenance_distinguishes_missing_black_from_recorded_black() {
|
||||
let strip: Vec<u8> = [1000u16; 16].into_iter().flat_map(u16::to_be_bytes).collect();
|
||||
let fallback = nef(1, 12, vec![strip.clone()], 4, 4, 4);
|
||||
let mut note = IfdBuilder::new();
|
||||
note.set(BLACK_LEVEL, Value::Short(vec![400; 4]));
|
||||
let recorded = nef_with_note(1, 12, vec![strip], 4, 4, 4, Some(note));
|
||||
for mode in [Mode::Header, Mode::Full] {
|
||||
let missing = decode(&fallback, mode, &Limits::default()).unwrap();
|
||||
assert_eq!((missing.black.mean(), missing.level_sources.black), (0.0, crate::LevelSource::Fallback));
|
||||
let stated = decode(&recorded, mode, &Limits::default()).unwrap();
|
||||
assert_eq!((stated.black.mean(), stated.level_sources.black), (100.0, crate::LevelSource::Metadata));
|
||||
}
|
||||
}
|
||||
|
||||
/// A small uncompressed NEF with a real NikonNEFInfo TIFF block, without binary fixtures.
|
||||
fn nikon_distortion_file(model: &str, mode: u8) -> Vec<u8> {
|
||||
use crate::nef_lens::tests::{distortion_block, nef_info_blob, offset_block};
|
||||
@@ -1527,4 +1557,95 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mod partial_row_tests {
|
||||
use super::*;
|
||||
|
||||
// Independent wire oracle: place each sample bit directly, leaving nonzero tail bits and padding.
|
||||
fn image(data: &[u16], w: usize, bits: u32, padded: bool) -> Vec<u8> {
|
||||
let tight = (w * bits as usize).div_ceil(8);
|
||||
let stride = if padded { tight.next_multiple_of(16) } else { tight };
|
||||
let mut bytes = vec![0xA5; stride * (data.len() / w)];
|
||||
for (i, &sample) in data.iter().enumerate() {
|
||||
for bit in 0..bits as usize {
|
||||
let at = (i % w) * bits as usize + bit;
|
||||
let byte = &mut bytes[(i / w) * stride + at / 8];
|
||||
*byte = (*byte & !(1 << (at % 8))) | (((sample >> bit) as u8 & 1) << (at % 8));
|
||||
}
|
||||
}
|
||||
bytes
|
||||
}
|
||||
|
||||
fn note(order: ByteOrder, modern: bool, mode: u16) -> IfdBuilder {
|
||||
let mut n = IfdBuilder::new();
|
||||
if modern {
|
||||
let mut record = b"01000400\0\0".to_vec();
|
||||
record.extend(if order == ByteOrder::Big { mode.to_be_bytes() } else { mode.to_le_bytes() });
|
||||
n.set(NEF_COMPRESSION_RECORD, Value::Undefined(record));
|
||||
} else {
|
||||
n.set(NEF_COMPRESSION, Value::Short(vec![mode]));
|
||||
}
|
||||
n
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn declared_packed_partial_rows_preserve_every_sample() {
|
||||
let h = 5;
|
||||
for (bits, mode) in [(12u32, 6), (14, 10)] {
|
||||
for w in [1, 3, 5, 22, 23] {
|
||||
let data = samples(w * h, bits);
|
||||
for padded in [false, true] {
|
||||
let wire = image(&data, w, bits, padded);
|
||||
if w == 22 && bits == 14 {
|
||||
assert_eq!(wire.len() / h, if padded { 48 } else { 39 });
|
||||
}
|
||||
for order in [ByteOrder::Big, ByteOrder::Little] {
|
||||
for modern in [false, true] {
|
||||
let file = nef_in(
|
||||
order,
|
||||
34713,
|
||||
bits as u16,
|
||||
vec![wire.clone()],
|
||||
w as u32,
|
||||
h as u32,
|
||||
h as u32,
|
||||
Some(note(order, modern, mode)),
|
||||
);
|
||||
let raw = crate::decode(&file).unwrap();
|
||||
assert_eq!((raw.width, raw.height, raw.cpp), (w, h, 1));
|
||||
assert!(!raw.notices.assumed_cfa, "valid CFA tag is retained");
|
||||
let RawData::U16(got) = raw.data else { panic!("integer samples") };
|
||||
assert_eq!(got, data, "{bits}-bit width {w}, padded {padded}, {order:?}, modern {modern}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_rows_keep_mode_depth_and_strip_guards() {
|
||||
let (w, h) = (22, 5);
|
||||
let data = samples(w * h, 14);
|
||||
for padded in [false, true] {
|
||||
let wire = image(&data, w, 14, padded);
|
||||
let file = |bytes, mode: Option<u16>| {
|
||||
nef_with_note(34713, 14, vec![bytes], w as u32, h as u32, h as u32, mode.map(|m| note(ByteOrder::Big, false, m)))
|
||||
};
|
||||
for mode in [None, Some(0), Some(6), Some(13), Some(14)] {
|
||||
assert!(matches!(crate::decode(&file(wire.clone(), mode)), Err(RawError::Unsupported(_))), "mode {mode:?}");
|
||||
}
|
||||
let good = file(wire.clone(), Some(10));
|
||||
assert!(crate::decode(&good).is_ok());
|
||||
let short = truncated_tail_strip(good, wire.len() - 1);
|
||||
assert!(matches!(crate::decode(&short), Err(RawError::Corrupt(_))));
|
||||
for len in [wire.len() - 1, wire.len() + 1] {
|
||||
assert!(matches!(crate::decode(&file(vec![0; len], Some(10))), Err(RawError::Unsupported(_))));
|
||||
}
|
||||
}
|
||||
for (w, h, bits) in [(0, 5, 14), (22, 0, 14), (22, 5, 13), (usize::MAX, 5, 14)] {
|
||||
assert!(packed_layout(w, h, bits, u64::MAX, true).is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -207,6 +207,19 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
let info = ifd0.image()?;
|
||||
let (w, h) = (info.width as usize, info.height as usize);
|
||||
let n = w.checked_mul(h).filter(|n| *n > 0 && *n <= limits.samples()).ok_or(RawError::Limit("image too large"))?;
|
||||
// A partial TIFF directory is damage, not evidence of an unsupported compressed layout.
|
||||
// Do not infer packing from an absent strip table or from clipped sensor bytes.
|
||||
if info.compression == 1
|
||||
&& (info.offsets.is_empty()
|
||||
|| info.offsets.len() != info.byte_counts.len()
|
||||
|| info
|
||||
.offsets
|
||||
.iter()
|
||||
.zip(&info.byte_counts)
|
||||
.any(|(&at, &len)| len == 0 || at.checked_add(len).is_none_or(|end| end > bytes.len() as u64)))
|
||||
{
|
||||
return Err(RawError::Corrupt("ORF strip table is missing or outside file".into()));
|
||||
}
|
||||
let chunks = info.chunks(bytes.len() as u64);
|
||||
let total = chunks.iter().fold(0u64, |a, c| a.saturating_add(c.len));
|
||||
let stated = cfa_from_exif(&tiff);
|
||||
@@ -302,6 +315,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
}
|
||||
_ => Rect::new(0, 0, w, h),
|
||||
};
|
||||
let assumed_cfa = stated.is_none();
|
||||
let cfa = stated.unwrap_or_else(|| cfa_from_data(samples, w, active));
|
||||
let (black, black_source) = match ip.as_ref().and_then(|i| i.f64s(BLACK)).as_deref() {
|
||||
Some(v @ [_, _, _, _]) => {
|
||||
@@ -348,6 +362,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
notices: crate::RawNotices { assumed_cfa, ..Default::default() },
|
||||
level_sources: crate::LevelSources { black: black_source, white: white_source },
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
@@ -416,6 +431,34 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_uncompressed_directory_and_strip_are_corrupt_not_compressed() {
|
||||
let mut bytes = orf_with(4, 4, 16, vec![0; 32], Some(&[2, 0, 2, 0, 0, 1, 1, 2]));
|
||||
// TiffWriter puts image chunks before its directory/metadata, so EOF is not a
|
||||
// required sensor byte. Relocate the declared strip to EOF without changing it.
|
||||
let directory = u32::from_le_bytes(bytes[4..8].try_into().unwrap()) as usize;
|
||||
let count = u16::from_le_bytes(bytes[directory..directory + 2].try_into().unwrap()) as usize;
|
||||
let entry = (0..count)
|
||||
.map(|i| directory + 2 + i * 12)
|
||||
.find(|&at| u16::from_le_bytes(bytes[at..at + 2].try_into().unwrap()) == t::STRIP_OFFSETS)
|
||||
.unwrap();
|
||||
let old_offset = u32::from_le_bytes(bytes[entry + 8..entry + 12].try_into().unwrap()) as usize;
|
||||
let pixels = bytes[old_offset..old_offset + 32].to_vec();
|
||||
let offset = bytes.len();
|
||||
bytes.extend(pixels);
|
||||
bytes[entry + 8..entry + 12].copy_from_slice(&(offset as u32).to_le_bytes());
|
||||
assert!(crate::decode(&bytes).is_ok());
|
||||
for cut in 0..bytes.len() {
|
||||
for result in [crate::decode(&bytes[..cut]).map(|_| ()), crate::probe_info(&bytes[..cut]).map(|_| ())] {
|
||||
assert!(!matches!(result, Err(RawError::Unsupported(ref why)) if why.contains("compressed")), "cut {cut}: {result:?}");
|
||||
}
|
||||
}
|
||||
let result = crate::decode(&bytes[..bytes.len() - 1]);
|
||||
assert!(matches!(result, Err(RawError::Corrupt(_))), "required final strip byte missing: {result:?}");
|
||||
let result = crate::probe_info(&bytes[..bytes.len() - 1]);
|
||||
assert!(matches!(result, Err(RawError::Corrupt(_))), "required final strip byte missing during probe: {result:?}");
|
||||
}
|
||||
|
||||
/// Strip byte counts that add up past `u64::MAX` (LONG8 `[u64::MAX, 1]`) are an error or a picture, not an overflow.
|
||||
#[test]
|
||||
fn strip_byte_counts_past_u64_max_do_not_overflow() {
|
||||
@@ -627,6 +670,28 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cfa_notice_distinguishes_missing_and_unusable_exif_from_stated_patterns() {
|
||||
let (w, h) = (160usize, 144usize);
|
||||
let strip = mosaic(w, h, |cx, cy| [600, 1000, 1000, 300].map(|value| value + shade(cx, cy)));
|
||||
let expected: Vec<u16> = strip.as_chunks::<2>().0.iter().map(|word| u16::from_le_bytes(*word)).collect();
|
||||
for (tag, assumed, pattern) in [
|
||||
(None, true, "RGGB"),
|
||||
(Some(&[2, 0, 2, 0, 0, 1][..]), true, "RGGB"),
|
||||
(Some(&[2, 0, 2, 0, 0, 1, 1, 3][..]), true, "RGGB"),
|
||||
(Some(BGGR), false, "BGGR"),
|
||||
(Some(GRBG), false, "GRBG"),
|
||||
] {
|
||||
let bytes = orf_with(w as u32, h as u32, 16, strip.clone(), tag);
|
||||
for mode in [Mode::Header, Mode::Full] {
|
||||
let image = decode(&bytes, mode, &Limits::default()).unwrap();
|
||||
assert_eq!(image.cfa, Some(Cfa::bayer_static(pattern)));
|
||||
assert_eq!(image.data, RawData::U16(expected.clone()), "word16 samples remain unchanged");
|
||||
assert_eq!(image.notices, crate::RawNotices { assumed_cfa: assumed, ..Default::default() });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn packed_12_bit_probe_agrees_with_decode() {
|
||||
let (w, h) = (16usize, 4usize);
|
||||
|
||||
+47
-1
@@ -194,6 +194,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
Some(r) => r,
|
||||
None => dark_trimmed(samples, w, hgt, black_point.as_deref().map(|v| v.iter().sum::<f64>() / 4.0), white),
|
||||
};
|
||||
let mut assumed_cfa = false;
|
||||
let cfa = match (ifd0.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), ifd0.bytes(t::CFA_PATTERN_EP)) {
|
||||
(Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => {
|
||||
Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }
|
||||
@@ -202,7 +203,10 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
// of the image area: the *ist D (origin 19, 11) and the 645D (80, 59) only fit unshifted. The 12-bit 10 MP
|
||||
// bodies and everything from the K-3 on say RGGB; the K-7 / K-5 / 645D generation says BGGR. Files without the
|
||||
// tag keep BGGR.
|
||||
_ => cfa_from_exif(&tiff).unwrap_or_else(|| Cfa::bayer_static("BGGR")),
|
||||
_ => cfa_from_exif(&tiff).unwrap_or_else(|| {
|
||||
assumed_cfa = true;
|
||||
Cfa::bayer_static("BGGR")
|
||||
}),
|
||||
};
|
||||
let black = match black_point.as_deref() {
|
||||
Some(v @ [_, _, _, _]) => {
|
||||
@@ -240,6 +244,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: Default::default(),
|
||||
notices: crate::RawNotices { assumed_cfa, ..Default::default() },
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
@@ -487,6 +492,47 @@ mod tests {
|
||||
assert_eq!(img.cfa, Some(Cfa::bayer_static("BGGR")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fixed_cfa_fallback_is_noticed_but_either_usable_metadata_form_is_not_assumed() {
|
||||
use craft_tiff::{IfdBuilder, ImageData, TiffWriter, Value};
|
||||
let expected: Vec<u16> = (0..16).map(|i| 100 + i * 7).collect();
|
||||
let valid_exif = &[2, 0, 2, 0, 1, 0, 2, 1][..];
|
||||
let invalid_exif = &[2, 0, 2, 0, 1, 0][..];
|
||||
for (primary, exif, assumed, pattern) in [
|
||||
(None, None, true, "BGGR"),
|
||||
(Some([0, 1, 1, 3]), None, true, "BGGR"),
|
||||
(None, Some(invalid_exif), true, "BGGR"),
|
||||
(Some([0, 1, 1, 3]), Some(invalid_exif), true, "BGGR"),
|
||||
(Some([0, 1, 1, 3]), Some(valid_exif), false, "GRBG"),
|
||||
(None, Some(valid_exif), false, "GRBG"),
|
||||
(Some([0, 1, 1, 2]), None, false, "RGGB"),
|
||||
(Some([0, 1, 1, 2]), Some(valid_exif), false, "RGGB"),
|
||||
] {
|
||||
let mut raw = IfdBuilder::new();
|
||||
raw.set(t::IMAGE_WIDTH, Value::Long(vec![4]));
|
||||
raw.set(t::IMAGE_LENGTH, Value::Long(vec![4]));
|
||||
raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![16]));
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![1]));
|
||||
raw.set(t::PHOTOMETRIC, Value::Short(vec![32803]));
|
||||
raw.set(t::MAKE, Value::Ascii("PENTAX Corporation".into()));
|
||||
if let Some(pattern) = primary {
|
||||
raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2, 2]));
|
||||
raw.set(t::CFA_PATTERN_EP, Value::Byte(pattern.to_vec()));
|
||||
}
|
||||
if let Some(pattern) = exif {
|
||||
raw.set_child(t::EXIF_IFD, IfdBuilder::new().with(crate::EXIF_CFA_PATTERN, Value::Undefined(pattern.to_vec())));
|
||||
}
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![expected.iter().flat_map(|value| value.to_be_bytes()).collect()] });
|
||||
let bytes = TiffWriter::new(ByteOrder::Big, false).write(&[raw]).unwrap();
|
||||
for mode in [Mode::Header, Mode::Full] {
|
||||
let image = decode(&bytes, mode, &Limits::default()).unwrap();
|
||||
assert_eq!(image.cfa, Some(Cfa::bayer_static(pattern)));
|
||||
assert_eq!(image.data, RawData::U16(expected.clone()));
|
||||
assert_eq!(image.notices, crate::RawNotices { assumed_cfa: assumed, ..Default::default() });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A `w x h` big-endian 14-bit PEF of plain 16-bit words without area tags, whose maker note has only a
|
||||
/// BlackPoint of `black`, with `px(x, y)` as the samples.
|
||||
fn pef_with_black_point(w: usize, h: usize, black: u16, px: impl Fn(usize, usize) -> u16) -> Vec<u8> {
|
||||
|
||||
@@ -643,9 +643,15 @@ fn cr3_preview(bytes: &[u8]) -> Option<&[u8]> {
|
||||
/// A CR3's full-size JPEG track, when it holds a DCT JPEG.
|
||||
#[cfg(feature = "cr3")]
|
||||
fn cr3_full_jpeg(bytes: &[u8]) -> Option<&[u8]> {
|
||||
let full = craft_metadata::cr3::parse_cr3(bytes)
|
||||
.and_then(|c| c.tracks.iter().find(|t| t.kind == craft_metadata::cr3::Cr3TrackKind::Jpeg).and_then(|t| t.data));
|
||||
full.and_then(|(at, len)| bytes.get(at..at.checked_add(len)?)).filter(|j| is_dct_jpeg(j))
|
||||
let cr3 = craft_metadata::cr3::parse_cr3(bytes)?;
|
||||
// An unusable preview track must not hide a later usable one. Share the marker budget
|
||||
// across candidates so overlapping hostile track ranges cannot cause quadratic walks.
|
||||
let mut work = CR3_MAX_WALK;
|
||||
cr3.tracks.iter().filter(|t| t.kind == craft_metadata::cr3::Cr3TrackKind::Jpeg).find_map(|t| {
|
||||
let (at, len) = t.data?;
|
||||
let jpeg = bytes.get(at..at.checked_add(len)?)?;
|
||||
first_frame_within(jpeg, &mut work).filter(|(m, _)| matches!(m, 0xc0..=0xc2)).map(|_| jpeg)
|
||||
})
|
||||
}
|
||||
|
||||
/// Marker-walk budget of the CR3 fallback: candidates can overlap (each claiming the rest of the file) and
|
||||
|
||||
@@ -582,7 +582,19 @@ fn cr3_prefers_the_full_size_jpeg_track() {
|
||||
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat());
|
||||
let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat());
|
||||
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
|
||||
let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat()))));
|
||||
let trak = bx(
|
||||
b"trak",
|
||||
&bx(
|
||||
b"mdia",
|
||||
&bx(
|
||||
b"minf",
|
||||
&bx(
|
||||
b"stbl",
|
||||
&[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>()), stsz, co64].concat(),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
file.extend(bx(b"moov", &trak));
|
||||
file.resize(at as usize, 0);
|
||||
file.extend_from_slice(&big);
|
||||
@@ -592,6 +604,48 @@ fn cr3_prefers_the_full_size_jpeg_track() {
|
||||
assert_eq!(embedded_preview(&file).unwrap(), small);
|
||||
}
|
||||
|
||||
#[cfg(feature = "cr3")]
|
||||
#[test]
|
||||
fn cr3_skips_bad_preview_tracks_but_never_uses_sensor_tracks_as_jpeg() {
|
||||
let bx = |kind: &[u8; 4], body: &[u8]| -> Vec<u8> { [&((body.len() + 8) as u32).to_be_bytes()[..], kind, body].concat() };
|
||||
let full = |kind: &[u8; 4], body: &[u8]| bx(kind, &[&[0u8; 4][..], body].concat());
|
||||
let track = |jpeg: bool, at: u64, len: u32| {
|
||||
let mut entry = vec![0u8; 82];
|
||||
if jpeg {
|
||||
entry.extend(bx(b"JPEG", &[0; 4]));
|
||||
}
|
||||
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &entry)].concat());
|
||||
let stsz = full(b"stsz", &[len.to_be_bytes(), 1u32.to_be_bytes()].concat());
|
||||
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
|
||||
bx(
|
||||
b"trak",
|
||||
&bx(
|
||||
b"mdia",
|
||||
&bx(
|
||||
b"minf",
|
||||
&bx(
|
||||
b"stbl",
|
||||
&[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>()), stsz, co64].concat(),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
};
|
||||
let good = fake_jpeg(100);
|
||||
for first in [track(true, u64::MAX, 32), track(true, 4096, 32)] {
|
||||
let mut bytes = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
|
||||
bytes.extend(bx(b"moov", &[first, track(true, 4128, good.len() as u32)].concat()));
|
||||
bytes.resize(4128, 0);
|
||||
bytes.extend(&good);
|
||||
assert_eq!(embedded_preview(&bytes).as_deref(), Some(good.as_slice()));
|
||||
}
|
||||
let mut bytes = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
|
||||
bytes.extend(bx(b"moov", &track(false, 4096, good.len() as u32)));
|
||||
bytes.resize(4096, 0);
|
||||
bytes.extend(&good);
|
||||
assert_eq!(embedded_preview(&bytes), None, "a sensor sample is not a rendered preview even if its bytes look like JPEG");
|
||||
}
|
||||
|
||||
#[cfg(feature = "cr3")]
|
||||
/// A CR3 whose `PRVW` box holds `prvw` and whose JPEG track holds `big`.
|
||||
fn cr3_with(prvw_jpeg: &[u8], big: &[u8]) -> Vec<u8> {
|
||||
@@ -608,7 +662,19 @@ fn cr3_with(prvw_jpeg: &[u8], big: &[u8]) -> Vec<u8> {
|
||||
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat());
|
||||
let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat());
|
||||
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
|
||||
let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat()))));
|
||||
let trak = bx(
|
||||
b"trak",
|
||||
&bx(
|
||||
b"mdia",
|
||||
&bx(
|
||||
b"minf",
|
||||
&bx(
|
||||
b"stbl",
|
||||
&[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>()), stsz, co64].concat(),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
file.extend(bx(b"moov", &trak));
|
||||
file.resize(at as usize, 0);
|
||||
file.extend_from_slice(big);
|
||||
@@ -656,7 +722,19 @@ fn the_smallest_preview_covering_the_size_is_chosen() {
|
||||
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat());
|
||||
let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat());
|
||||
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
|
||||
let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat()))));
|
||||
let trak = bx(
|
||||
b"trak",
|
||||
&bx(
|
||||
b"mdia",
|
||||
&bx(
|
||||
b"minf",
|
||||
&bx(
|
||||
b"stbl",
|
||||
&[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>()), stsz, co64].concat(),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
file.extend(bx(b"moov", &trak));
|
||||
file.resize(at as usize, 0);
|
||||
file.extend_from_slice(&big);
|
||||
@@ -769,7 +847,19 @@ fn a_cr3_with_two_prvw_boxes_offers_both() {
|
||||
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat());
|
||||
let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat());
|
||||
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
|
||||
let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat()))));
|
||||
let trak = bx(
|
||||
b"trak",
|
||||
&bx(
|
||||
b"mdia",
|
||||
&bx(
|
||||
b"minf",
|
||||
&bx(
|
||||
b"stbl",
|
||||
&[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>()), stsz, co64].concat(),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
file.extend(bx(b"moov", &trak));
|
||||
// (the track in the media data, as a camera writes it, after a `free` box up to it)
|
||||
let pad = at as usize - 16 - file.len();
|
||||
|
||||
+242
-17
@@ -18,7 +18,10 @@
|
||||
//! - CFA: the X-Trans layout record (36 bytes, 0 = R, 1 = G, 2 = B) is stored in reverse order and, reversed,
|
||||
//! is anchored at raw pixel (0, 0) — verified on X-Trans I and III samples by minimising the difference
|
||||
//! between horizontally / vertically adjacent "green" samples over all 72 orientations/phases, and by colour
|
||||
//! renders. Bayer bodies (X-A series) use RGGB at (0, 0), verified by colour renders.
|
||||
//! renders. Bayer FujiLayout (`0x0130`) gives the reversed 2×2 pattern in its low two bits
|
||||
//! (0 = R, 1/3 = G, 2 = B); absent layout records retain the assumed RGGB fallback.
|
||||
//! - RawExposureBias (`0x9650`) is a signed big-endian 16-bit fraction; its negative is the
|
||||
//! baseline exposure used to restore the camera's dynamic-range exposure reduction.
|
||||
//! - Striped predictive compression is decoded by `rafc`; see `docs/raf-compression.md`.
|
||||
|
||||
use crate::shared::white_from_data_sourced;
|
||||
@@ -87,7 +90,7 @@ fn pair(h: &Header, tag: u16) -> Option<(usize, usize)> {
|
||||
pub(crate) fn raw_ifd(raw: &[u8]) -> Result<Ifd> {
|
||||
let tiff = Tiff::parse(raw).map_err(|_| RawError::Unsupported("RAF without a raw IFD (older FinePix layout)".into()))?;
|
||||
let off = tiff.ifds.first().and_then(|i| i.u64(0xf000)).ok_or_else(|| RawError::Unsupported("RAF without a raw IFD".into()))?;
|
||||
let (ifd, _) = craft_tiff::parse_ifd_at(raw, off, ByteOrder::Little, 0, false, &Default::default())?;
|
||||
let (ifd, _) = craft_tiff::parse_ifd_at(raw, off, tiff.order, 0, false, &Default::default())?;
|
||||
Ok(ifd)
|
||||
}
|
||||
|
||||
@@ -122,14 +125,42 @@ fn camera_name(b: &[u8]) -> String {
|
||||
String::from_utf8_lossy(name.get(..end).unwrap_or_default()).trim().to_string()
|
||||
}
|
||||
|
||||
/// The CFA: X-Trans from the (reversed) layout record, else RGGB.
|
||||
fn cfa(h: &Header) -> Cfa {
|
||||
match record(h, XTRANS_LAYOUT) {
|
||||
Some(l) if l.len() == 36 && l.iter().all(|&c| c <= 2) => {
|
||||
Cfa { width: 6, height: 6, pattern: l.iter().rev().copied().collect(), origin_x: 0, origin_y: 0 }
|
||||
/// Layout records are reversed and anchored at sensor (0, 0). FujiLayout's low
|
||||
/// two bits encode Bayer colour; both 1 and 3 denote green.
|
||||
fn cfa(h: &Header) -> Result<Cfa> {
|
||||
if let Some(l) = record(h, XTRANS_LAYOUT) {
|
||||
if l.len() != 36 || l.iter().any(|&c| c > 2) {
|
||||
return Err(RawError::Unsupported("Fujifilm RAF with an unknown X-Trans layout".into()));
|
||||
}
|
||||
_ => Cfa::bayer_static("RGGB"),
|
||||
return Ok(Cfa { width: 6, height: 6, pattern: l.iter().rev().copied().collect(), origin_x: 0, origin_y: 0 });
|
||||
}
|
||||
let Some(l) = record(h, 0x0130) else { return Ok(Cfa::bayer_static("RGGB")) };
|
||||
let cfa = Cfa { width: 2, height: 2, pattern: l.iter().rev().map(|&c| if c & 3 == 3 { 1 } else { c & 3 }).collect(), origin_x: 0, origin_y: 0 };
|
||||
// Colour counts alone also accept adjacent green sites; the two greens
|
||||
// must occupy opposite corners of the 2x2 Bayer repeat.
|
||||
if l.len() != 4 || !matches!(cfa.pattern.as_slice(), [0, 1, 1, 2] | [2, 1, 1, 0] | [1, 0, 2, 1] | [1, 2, 0, 1]) {
|
||||
return Err(RawError::Unsupported("Fujifilm RAF with an unknown Bayer layout".into()));
|
||||
}
|
||||
Ok(cfa)
|
||||
}
|
||||
|
||||
/// Fuji RawExposureBias is a signed big-endian 16-bit fraction, undone like
|
||||
/// DNG BaselineExposure. Invalid or implausible metadata does not alter exposure.
|
||||
fn baseline_exposure(h: &Header) -> f64 {
|
||||
record(h, 0x9650)
|
||||
.and_then(|d| {
|
||||
let num = i16::from_be_bytes(d.get(..2)?.try_into().ok()?);
|
||||
let den = i16::from_be_bytes(d.get(2..4)?.try_into().ok()?);
|
||||
(den != 0).then(|| -f64::from(num) / f64::from(den))
|
||||
})
|
||||
.filter(|v| v.is_finite() && v.abs() <= 8.0)
|
||||
.unwrap_or(0.0)
|
||||
}
|
||||
|
||||
/// The codec fixes the green-site geometry; exchanging the two non-green
|
||||
/// planes only exchanges their colour labels, so RGGB and BGGR are both valid.
|
||||
fn compressed_layout_supported(cfa: &Cfa) -> bool {
|
||||
*cfa == Cfa::xtrans() || *cfa == Cfa::bayer_static("RGGB") || *cfa == Cfa::bayer_static("BGGR")
|
||||
}
|
||||
|
||||
/// Unpack one row of `bits`-bit samples in Fujifilm's packing. `words32`: the raw IFD's `0xf006` flag is set, the
|
||||
@@ -137,7 +168,7 @@ fn cfa(h: &Header) -> Cfa {
|
||||
/// LSB-first bit stream, or (other depths) the same 32-bit words.
|
||||
pub(crate) fn unpack_row(src: &[u8], bits: u32, words32: bool, out: &mut [u16]) {
|
||||
match bits {
|
||||
16 => out.iter_mut().zip(src.as_chunks::<2>().0).for_each(|(o, c)| *o = u16::from_le_bytes([c[0], c[1]])),
|
||||
16 if !words32 => out.iter_mut().zip(src.as_chunks::<2>().0).for_each(|(o, c)| *o = u16::from_le_bytes([c[0], c[1]])),
|
||||
12 if !words32 => unpack_lsb(src, 12, out),
|
||||
_ => unpack_words32_msb(src, ByteOrder::Little, bits, out),
|
||||
}
|
||||
@@ -171,7 +202,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
let src = raw
|
||||
.get(off..off.checked_add(len).ok_or_else(|| RawError::Corrupt("RAF strip size overflow".into()))?)
|
||||
.ok_or_else(|| RawError::Corrupt("RAF strip outside file".into()))?;
|
||||
let cfa = cfa(&h);
|
||||
let cfa = cfa(&h)?;
|
||||
let compression = be32(bytes, 108).unwrap_or(0);
|
||||
let compressed = match compression {
|
||||
0 => false,
|
||||
@@ -180,8 +211,13 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
};
|
||||
let mut data = Vec::new();
|
||||
if compressed {
|
||||
if cfa.width == 6 && cfa != Cfa::xtrans() {
|
||||
return Err(RawError::Unsupported("Fujifilm compressed RAF with an unknown X-Trans layout".into()));
|
||||
if !compressed_layout_supported(&cfa) {
|
||||
return Err(RawError::Unsupported("Fujifilm compressed RAF with an unsupported CFA layout".into()));
|
||||
}
|
||||
if !src.starts_with(b"IS") {
|
||||
// A recognized RAF compression mode can wrap an unknown packing. Do not
|
||||
// feed it to the IS codec; damage inside an identified IS stream stays Corrupt.
|
||||
return Err(RawError::Unsupported("Fujifilm compressed RAF without a supported IS signature".into()));
|
||||
}
|
||||
let flag = if compression == 2 { 1 } else { 0 };
|
||||
if src.get(2) != Some(&flag) {
|
||||
@@ -190,9 +226,18 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
data = crate::rafc::decode(src, w, hgt, bits, cfa.width == 6, mode, limits)?;
|
||||
} else {
|
||||
let words32 = ifd.u64(PACKED_WORDS).is_some_and(|v| v != 0);
|
||||
// Samples may use 16-bit words (e.g. 14-bit X-T20); otherwise they are packed at `bits`.
|
||||
let store = if len as u64 >= (n as u64) * 2 { 16 } else { bits };
|
||||
let packed_row = w.checked_mul(store as usize).map(|n| n.div_ceil(8)).ok_or(RawError::Limit("RAF row too large"))?;
|
||||
// An explicit packed-word flag wins over the size heuristic: padding in short
|
||||
// packed rows can occupy as many bytes as unpacked 16-bit samples.
|
||||
// Unflagged samples may use 16-bit words (e.g. 14-bit X-T20).
|
||||
let store = if !words32 && len as u64 >= (n as u64) * 2 { 16 } else { bits };
|
||||
let row_bits = w.checked_mul(store as usize).ok_or(RawError::Limit("RAF row too large"))?;
|
||||
// Word-packed rows store the high bits in the last little-endian word's final bytes.
|
||||
// Reading only ceil(bits / 8) bytes would lose those samples, not just padding.
|
||||
let packed_row = if words32 || (store != 16 && store != 12) {
|
||||
row_bits.div_ceil(32).checked_mul(4).ok_or(RawError::Limit("RAF row too large"))?
|
||||
} else {
|
||||
row_bits.div_ceil(8)
|
||||
};
|
||||
let stride = match get(0xf005)?.unwrap_or(0) {
|
||||
0 => packed_row,
|
||||
words => words.checked_mul(4).ok_or(RawError::Limit("RAF stride too large"))?.max(packed_row),
|
||||
@@ -206,7 +251,11 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
data.par_chunks_mut(w).enumerate().for_each(|(y, row)| {
|
||||
// `required <= src.len()` proves these row offsets fit, including the final row.
|
||||
let s = src.get(y * stride..y * stride + packed_row).unwrap_or(&[]);
|
||||
unpack_row(s, store, words32, row);
|
||||
if !words32 && store == 16 && raw.starts_with(b"MM") {
|
||||
crate::unpack::read_u16s(s, ByteOrder::Big, row);
|
||||
} else {
|
||||
unpack_row(s, store, words32, row);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -246,7 +295,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
active_area: active,
|
||||
crop: Rect::new(0, 0, active.width, active.height),
|
||||
orientation,
|
||||
color: ColorData { kelvin_presets, ..Default::default() },
|
||||
color: ColorData { kelvin_presets, baseline_exposure: baseline_exposure(&h), ..Default::default() },
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
pixel_aspect: 1.0,
|
||||
@@ -256,6 +305,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
.unwrap_or_else(|| Camera { model: Some(camera_name(bytes)).filter(|n| !n.is_empty()), ..Default::default() }),
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
notices: crate::RawNotices { assumed_cfa: record(&h, XTRANS_LAYOUT).is_none() && record(&h, 0x0130).is_none(), ..Default::default() },
|
||||
level_sources: crate::LevelSources { black: black_source, white: white_source },
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
@@ -480,6 +530,181 @@ pub(super) mod tests {
|
||||
assert!(matches!(crate::decode(&bytes), Err(RawError::Unsupported(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn word_packed_rows_keep_the_final_partial_word_and_reject_truncation() {
|
||||
for (w, bits, flag) in [(30u32, 14u32, 0u32), (22, 12, 1), (6, 12, 1), (5, 16, 1)] {
|
||||
let h = 6u32;
|
||||
let px: Vec<u16> = (0..w * h).map(|i| ((i * 977 + 101) % (1 << bits)) as u16).collect();
|
||||
// Independently place each sample bit into whole little-endian 32-bit row words.
|
||||
let strip: Vec<u8> = px
|
||||
.chunks(w as usize)
|
||||
.flat_map(|row| {
|
||||
let mut words = vec![0u32; (row.len() * bits as usize).div_ceil(32)];
|
||||
for (i, &sample) in row.iter().enumerate() {
|
||||
for bit in 0..bits as usize {
|
||||
let at = i * bits as usize + bit;
|
||||
words[at / 32] |= u32::from((sample >> (bits as usize - 1 - bit)) & 1) << (31 - at % 32);
|
||||
}
|
||||
}
|
||||
words.into_iter().flat_map(u32::to_le_bytes).collect::<Vec<_>>()
|
||||
})
|
||||
.collect();
|
||||
let bytes = raf_flagged(w, h, bits, strip, None, &[], flag);
|
||||
assert_eq!(crate::decode(&bytes).unwrap().data, RawData::U16(px));
|
||||
assert!(crate::decode(&bytes[..bytes.len() - 1]).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn big_endian_words_and_bayer_layout_preserve_exact_sensor_values() {
|
||||
use craft_tiff::{IfdBuilder, TiffWriter, Value};
|
||||
let (w, h) = (6u32, 4u32);
|
||||
let px: Vec<u16> = (0..w * h).map(|i| (i * 977 + 123) as u16).collect();
|
||||
for order in [ByteOrder::Little, ByteOrder::Big] {
|
||||
let build = |at| {
|
||||
let mut sensor = IfdBuilder::new();
|
||||
for (tag, value) in [(0xf001, w), (0xf002, h), (0xf003, 16), (0xf007, at), (0xf008, w * h * 2)] {
|
||||
sensor.set(tag, Value::Long(vec![value]));
|
||||
}
|
||||
let mut root = IfdBuilder::new();
|
||||
root.set_child(0xf000, sensor);
|
||||
TiffWriter::new(order, false).write(&[root]).unwrap()
|
||||
};
|
||||
let mut raw = build(build(0).len() as u32);
|
||||
raw.extend(px.iter().flat_map(|v| if order == ByteOrder::Big { v.to_be_bytes() } else { v.to_le_bytes() }));
|
||||
let mut bytes = raf(w, h, 16, vec![0; (w * h * 2) as usize], None, &[]);
|
||||
let raw_at = be32(&bytes, 100).unwrap();
|
||||
bytes.truncate(raw_at);
|
||||
bytes[104..108].copy_from_slice(&(raw.len() as u32).to_be_bytes());
|
||||
bytes.extend(raw);
|
||||
// A four-site Bayer record is reversed in storage, just like the X-Trans record.
|
||||
let dir_at = be32(&bytes, 92).unwrap();
|
||||
let mut records = bytes[dir_at..raw_at].to_vec();
|
||||
let n = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1;
|
||||
records[..4].copy_from_slice(&n.to_be_bytes());
|
||||
records.extend_from_slice(&[0x01, 0x30, 0, 4, 0x08, 0x09, 0x0b, 0x0a]);
|
||||
bytes.splice(dir_at..raw_at, records.clone());
|
||||
bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes());
|
||||
bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes());
|
||||
let image = crate::decode(&bytes).unwrap();
|
||||
assert_eq!(image.data, RawData::U16(px.clone()));
|
||||
assert_eq!(image.cfa.unwrap().name(), "BGGR");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_exposure_bias_is_preserved_as_baseline_exposure_and_invalid_values_are_ignored() {
|
||||
for (num, den, expected) in [(-72i16, 100i16, 0.72f64), (1, 0, 0.0), (32767, 1, 0.0)] {
|
||||
let mut bytes = raf(6, 4, 16, pack(&[1000; 24], 16), None, &[]);
|
||||
let (dir_at, raw_at) = (be32(&bytes, 92).unwrap(), be32(&bytes, 100).unwrap());
|
||||
let mut records = bytes[dir_at..raw_at].to_vec();
|
||||
let n = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1;
|
||||
records[..4].copy_from_slice(&n.to_be_bytes());
|
||||
records.extend_from_slice(&[0x96, 0x50, 0, 4]);
|
||||
records.extend(num.to_be_bytes());
|
||||
records.extend(den.to_be_bytes());
|
||||
bytes.splice(dir_at..raw_at, records.clone());
|
||||
bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes());
|
||||
bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes());
|
||||
for mode in [Mode::Header, Mode::Full] {
|
||||
assert!((decode(&bytes, mode, &Limits::default()).unwrap().color.baseline_exposure - expected).abs() < 1e-9);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_declared_cfa_layouts_are_not_silently_replaced() {
|
||||
for (tag, data) in [(0x0130, vec![8, 8, 10, 10]), (0x0130, vec![1, 2, 1, 0]), (0x0131, vec![3; 36])] {
|
||||
let header = Header { jpeg: None, records: vec![(tag, data.as_slice())], raw: None };
|
||||
assert!(matches!(cfa(&header), Err(RawError::Unsupported(_))));
|
||||
}
|
||||
}
|
||||
|
||||
fn with_bayer_record(mut bytes: Vec<u8>, pattern: [u8; 4]) -> Vec<u8> {
|
||||
let (dir_at, raw_at) = (be32(&bytes, 92).unwrap(), be32(&bytes, 100).unwrap());
|
||||
let mut records = bytes[dir_at..raw_at].to_vec();
|
||||
let count = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1;
|
||||
records[..4].copy_from_slice(&count.to_be_bytes());
|
||||
records.extend_from_slice(&[0x01, 0x30, 0, 4]);
|
||||
records.extend_from_slice(&pattern);
|
||||
bytes.splice(dir_at..raw_at, records.clone());
|
||||
bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes());
|
||||
bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes());
|
||||
bytes
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_bayer_geometry_is_rejected_in_header_and_full_modes() {
|
||||
// Reversal gives RGBG: correct colour counts, but both greens in one column.
|
||||
let bytes = with_bayer_record(raf(6, 4, 16, pack(&[1000; 24], 16), None, &[]), [1, 2, 1, 0]);
|
||||
for mode in [Mode::Header, Mode::Full] {
|
||||
assert!(matches!(decode(&bytes, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("Bayer layout")));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_four_bayer_geometries_preserve_the_stored_samples() {
|
||||
let pixels: Vec<u16> = (0..24).map(|i| 123 + i * 977).collect();
|
||||
for (stored, expected) in [([2, 1, 1, 0], "RGGB"), ([0, 1, 1, 2], "BGGR"), ([1, 2, 0, 1], "GRBG"), ([1, 0, 2, 1], "GBRG")] {
|
||||
let bytes = with_bayer_record(raf(6, 4, 16, pack(&pixels, 16), None, &[]), stored);
|
||||
for mode in [Mode::Header, Mode::Full] {
|
||||
let image = decode(&bytes, mode, &Limits::default()).unwrap();
|
||||
assert_eq!((image.width, image.height), (6, 4));
|
||||
assert_eq!(image.cfa.as_ref().unwrap().name(), expected);
|
||||
assert_eq!(image.data, RawData::U16(if mode == Mode::Full { pixels.clone() } else { Vec::new() }));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compressed_bayer_requires_the_layout_assumed_by_its_predictor() {
|
||||
let strip = b"IS\x01\x00\x0e\0\x12\x06\0\x03\x0c\x03\0\x02\0\x03".to_vec();
|
||||
let mut bytes = raf(780, 18, 14, strip, None, &[]);
|
||||
let (dir_at, raw_at) = (be32(&bytes, 92).unwrap(), be32(&bytes, 100).unwrap());
|
||||
let mut records = bytes[dir_at..raw_at].to_vec();
|
||||
let n = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1;
|
||||
records[..4].copy_from_slice(&n.to_be_bytes());
|
||||
// Reversed GRBG: a valid Bayer pattern, but not the RGGB order of this codec.
|
||||
records.extend_from_slice(&[0x01, 0x30, 0, 4, 0x0b, 0x0a, 0x08, 0x09]);
|
||||
bytes.splice(dir_at..raw_at, records.clone());
|
||||
bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes());
|
||||
bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes());
|
||||
for mode in [Mode::Header, Mode::Full] {
|
||||
assert!(matches!(decode(&bytes, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("layout")));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compressed_layout_accepts_both_red_blue_orders_without_accepting_wrong_green_phase() {
|
||||
for name in ["RGGB", "BGGR"] {
|
||||
assert!(compressed_layout_supported(&Cfa::bayer_static(name)), "{name}");
|
||||
}
|
||||
for name in ["GRBG", "GBRG"] {
|
||||
assert!(!compressed_layout_supported(&Cfa::bayer_static(name)), "{name}");
|
||||
}
|
||||
let mut other = Cfa::xtrans();
|
||||
assert!(compressed_layout_supported(&other));
|
||||
other.pattern.swap(0, 2);
|
||||
assert!(!compressed_layout_supported(&other));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_compressed_signature_is_unsupported_but_identified_stream_damage_is_corrupt() {
|
||||
let header = b"IS\x01\x00\x0e\0\x12\x06\0\x03\x0c\x03\0\x02\0\x03".to_vec();
|
||||
let known = raf(780, 18, 14, header, None, &[]);
|
||||
let raw_at = be32(&known, 100).unwrap();
|
||||
let strip_at = raw_at + raw_ifd(&known[raw_at..]).unwrap().u64(0xf007).unwrap() as usize;
|
||||
let mut unknown = known.clone();
|
||||
unknown[strip_at..strip_at + 2].copy_from_slice(b"XX");
|
||||
for mode in [Mode::Header, Mode::Full] {
|
||||
assert!(matches!(decode(&unknown, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("signature")));
|
||||
assert!(
|
||||
matches!(decode(&known, mode, &Limits::default()), Err(RawError::Corrupt(_))),
|
||||
"identified but truncated IS stream must remain corrupt"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Raw IFD flag `0xf006` selects the 32-bit-word packing at 12 bits; without it 12 bits are an LSB-first stream.
|
||||
#[test]
|
||||
fn flag_selects_word_packed_12_bit() {
|
||||
|
||||
+57
-5
@@ -615,11 +615,18 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
}
|
||||
_ => Rect::new(0, 0, active.width, active.height),
|
||||
};
|
||||
if !data.is_empty() {
|
||||
// Packed formats 2/5/7 are lossless sample containers: zero is an ordinary
|
||||
// code, not evidence of a mapped-out photosite. The documented zero repair
|
||||
// applies to the predictive format 4 and older word containers only.
|
||||
if !data.is_empty() && matches!(encoding, Encoding::Words | Encoding::Blocks { kind: Block::Format4, .. }) {
|
||||
fill_defects(&mut data, w, active);
|
||||
}
|
||||
|
||||
let cfa = cfa(ifd0.u16(CFA_PATTERN).unwrap_or(1));
|
||||
let cfa_code = ifd0.u16(CFA_PATTERN);
|
||||
// RGGB is still usable when metadata is absent or unknown, but callers
|
||||
// must be able to distinguish that fallback from a recorded sensor phase.
|
||||
let assumed_cfa = !matches!(cfa_code, Some(1..=4));
|
||||
let cfa = cfa(cfa_code.unwrap_or(1));
|
||||
let read = BLACK_RGB.map(|tag| ifd0.f64(tag).filter(|v| v.is_finite() && *v >= 0.0));
|
||||
let tags = read.map(|v| v.unwrap_or(0.0) as f32);
|
||||
let offset = match encoding {
|
||||
@@ -671,6 +678,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: crate::LevelSources { black: black_source, white: white_source },
|
||||
notices: crate::RawNotices { assumed_cfa, ..Default::default() },
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
@@ -1069,18 +1077,62 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cfa_fallback_provenance_preserves_pixels_in_header_and_full_modes() {
|
||||
let pixels: Vec<u16> = (0..72).map(|i| 17 + i * 113).collect();
|
||||
let payload = pack(&pixels, 14);
|
||||
for (code, expected, assumed) in [
|
||||
(None, "RGGB", true),
|
||||
(Some(0), "RGGB", true),
|
||||
(Some(5), "RGGB", true),
|
||||
(Some(u16::MAX), "RGGB", true),
|
||||
(Some(1), "RGGB", false),
|
||||
(Some(2), "GRBG", false),
|
||||
(Some(3), "GBRG", false),
|
||||
(Some(4), "BGGR", false),
|
||||
] {
|
||||
let mut tags = vec![short(BITS, 14), short(RAW_FORMAT, 7)];
|
||||
if let Some(code) = code {
|
||||
tags.push(short(CFA_PATTERN, code));
|
||||
}
|
||||
let bytes = file(18, 4, &tags, &payload, &|off| vec![(RAW_OFFSET, Value::Long(vec![off]))]);
|
||||
for mode in [Mode::Header, Mode::Full] {
|
||||
let image = decode(&bytes, mode, &Limits::default()).unwrap();
|
||||
assert_eq!((image.width, image.height), (18, 4));
|
||||
assert_eq!(image.cfa.as_ref().unwrap().name(), expected, "code {code:?}");
|
||||
assert_eq!(image.notices, crate::RawNotices { assumed_cfa: assumed, ..Default::default() }, "code {code:?}, {mode:?}");
|
||||
assert_eq!(samples(&image), if mode == Mode::Full { pixels.as_slice() } else { &[] });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lossless_packed_formats_preserve_zero_codes_inside_the_active_area() {
|
||||
let (w, h) = (90usize, 12usize);
|
||||
for (format, bits, rotated) in [(5u16, 12u32, true), (5, 14, true), (7, 14, false), (2, 12, false)] {
|
||||
let mut px = vec![1000u16; w * h];
|
||||
px[5 * w + 20] = 0;
|
||||
let payload = pack(&px, bits);
|
||||
let bytes = rw2(w as u16, h as u16, bits as u16, format, 100, &if rotated { rotate(payload) } else { payload });
|
||||
assert_eq!(samples(&crate::decode(&bytes).unwrap()), px, "format {format}/{bits}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn levels_crop_defects_and_cfa() {
|
||||
let (w, h) = (90usize, 60usize);
|
||||
let mut px: Vec<u16> = (0..w * h).map(|i| 300 + (i % 7) as u16).collect();
|
||||
// a saturation plateau sets the white level; zeros inside the active area are defects
|
||||
// A saturation plateau sets the white level. In the older word container,
|
||||
// isolated zeros inside the active area are documented mapped-out defects.
|
||||
px[10 * w..20 * w].fill(3900);
|
||||
px[30 * w + 40] = 0;
|
||||
px[1] = 0; // in the border: left alone
|
||||
let mut tags = vec![short(CFA_PATTERN, 2), short(BITS, 12), short(RAW_FORMAT, 5)];
|
||||
let mut tags = vec![short(CFA_PATTERN, 2), short(BITS, 12), short(COMPRESSION, 34828)];
|
||||
tags.extend([0x002f, 0x0030, 0x0031, 0x0032].iter().zip([12u16, 10, 52, 80]).map(|(&tag, v)| short(tag, v)));
|
||||
tags.extend(LINEARITY_LIMIT.map(|tag| short(tag, 2000))); // recorded limits are not used
|
||||
let bytes = file(w as u16, h as u16, &tags, &rotate(pack(&px, 12)), &|off| vec![(RAW_OFFSET, Value::Long(vec![off]))]);
|
||||
// Compression 34828 stores left-justified samples in little-endian u16 words.
|
||||
let words: Vec<u8> = px.iter().flat_map(|&v| (v << 4).to_le_bytes()).collect();
|
||||
let bytes = file(w as u16, h as u16, &tags, &words, &|off| vec![(t::STRIP_OFFSETS, Value::Long(vec![off]))]);
|
||||
let r = crate::decode(&bytes).unwrap();
|
||||
let d = samples(&r);
|
||||
assert_eq!(d[30 * w + 40], ((px[30 * w + 38] as u32 + px[30 * w + 42] as u32 + px[28 * w + 40] as u32 + px[32 * w + 40] as u32) / 4) as u16);
|
||||
|
||||
@@ -192,6 +192,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: crate::LevelSources { black: black_source, white: white_source },
|
||||
notices: crate::RawNotices::default(),
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
|
||||
@@ -49,6 +49,7 @@ fn an_active_area_that_wraps_past_the_image_is_corrupt() {
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: Default::default(),
|
||||
notices: crate::RawNotices::default(),
|
||||
pixel_aspect: 1.0,
|
||||
};
|
||||
assert!(raw.validate().is_err(), "`x + width` wrapped: the area is outside the image");
|
||||
|
||||
+72
-13
@@ -115,6 +115,7 @@ pub(crate) fn read_image_in(
|
||||
crate::Mode::Full => read_image(data, info, order, packing, limits),
|
||||
crate::Mode::Header => {
|
||||
check_image(data, info, limits)?;
|
||||
check_uncompressed_chunks(data, info, &info.chunks(data.len() as u64), packing)?;
|
||||
// a lossless-JPEG layout the decoder rejects (subsampled components, e.g. Sony's
|
||||
// lossless M/S sizes) must fail here too, as the full decode will; likewise a JPEG XL
|
||||
// chunk whose headers don't match the IFD
|
||||
@@ -151,6 +152,7 @@ pub(crate) fn read_image(data: &[u8], info: &ImageInfo, order: ByteOrder, packin
|
||||
let bits = info.bits() as u32;
|
||||
let float = info.sample_format == 3;
|
||||
let chunks = info.chunks(data.len() as u64);
|
||||
check_uncompressed_chunks(data, info, &chunks, packing)?;
|
||||
let planar = info.planar == 2 && cpp > 1;
|
||||
let ccpp = if planar { 1 } else { cpp };
|
||||
// The chunks are decoded and copied into the output in groups whose combined nominal size stays within
|
||||
@@ -222,6 +224,26 @@ pub(crate) fn read_image(data: &[u8], info: &ImageInfo, order: ByteOrder, packin
|
||||
Ok(if float { RawData::F32(out_f32) } else { RawData::U16(out_u16) })
|
||||
}
|
||||
|
||||
/// Framing precedes allocation even in header-only probes: missing source rows may not
|
||||
/// be reported as a decodable image and later silently filled with zero samples.
|
||||
fn check_uncompressed_chunks(data: &[u8], info: &ImageInfo, chunks: &[Chunk], packing: Packing) -> Result<()> {
|
||||
if info.compression != comp::NONE {
|
||||
return Ok(());
|
||||
}
|
||||
let cpp = if info.planar == 2 { 1 } else { info.samples_per_pixel as usize };
|
||||
for chunk in chunks {
|
||||
let src = chunk_bytes(data, chunk).ok_or_else(|| RawError::Corrupt("uncompressed chunk is outside the file".into()))?;
|
||||
let (cw, _, vw, vh) = chunk_dims(info, chunk);
|
||||
let row = packed_row_bytes(cw as u64 * cpp as u64, info.bits() as u32, packing, info.sample_format == 3);
|
||||
let last = packed_row_bytes(vw as u64 * cpp as u64, info.bits() as u32, packing, info.sample_format == 3);
|
||||
let need = row.saturating_mul(vh.saturating_sub(1) as u64).saturating_add(last);
|
||||
if (src.len() as u64) < need {
|
||||
return Err(RawError::Corrupt(format!("an uncompressed chunk has {} bytes, expected at least {need}", src.len())));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Bytes that `n` samples take in a row of an uncompressed or inflated chunk, laid out as
|
||||
/// [`unpack_chunk`] reads them: whole bytes or 16-bit words, or 32-bit words for `Words32Msb`.
|
||||
fn packed_row_bytes(n: u64, bits: u32, packing: Packing, float: bool) -> u64 {
|
||||
@@ -285,10 +307,12 @@ fn decode_chunk(
|
||||
}
|
||||
match info.compression {
|
||||
comp::NONE => {
|
||||
// uncompressed: missing rows are filled with zeros, but a chunk holding a small fraction of what its
|
||||
// samples need is no picture: refused before the nominal chunk is allocated (storytold/lightcraft#1512)
|
||||
let need = (n as u64).saturating_mul(u64::from(bits)) / 8;
|
||||
if (src.len() as u64).saturating_mul(64) < need {
|
||||
// Require every visible sample. Nominal edge-tile padding may be absent, but
|
||||
// missing image rows must not turn a truncated file into a zero-filled success.
|
||||
let row_bytes = packed_row_bytes(cw as u64 * cpp as u64, bits, packing, float);
|
||||
let last_row = packed_row_bytes(vw as u64 * cpp as u64, bits, packing, float);
|
||||
let need = row_bytes.saturating_mul(vh.saturating_sub(1) as u64).saturating_add(last_row);
|
||||
if (src.len() as u64) < need {
|
||||
return Err(RawError::Corrupt(format!("an uncompressed chunk of {n} samples has only {} bytes", src.len())));
|
||||
}
|
||||
unpack_chunk(src, order, packing, cw, ch, cpp, bits, float, 1)
|
||||
@@ -314,13 +338,7 @@ fn decode_chunk(
|
||||
let mcu = |v: usize| v.div_ceil(32).saturating_mul(32);
|
||||
let tile_budget =
|
||||
mcu(cw).saturating_mul(mcu(ch)).saturating_mul(if cpp == 1 { 1 } else { 3 }).clamp(1, limits.max_lossy_tile_samples.max(1));
|
||||
let (px, jw, jh, jc) = lossy_jpeg(src, cpp, tile_budget)?;
|
||||
// An edge tile's JPEG may be smaller than the nominal tile, but it has to cover the part of the tile
|
||||
// inside the image: one that doesn't would leave a black region in a picture reported as decoded
|
||||
// (PhotoCraft's and FilmCraft's readers refuse a JPEG smaller than its tile; this keeps the edge case).
|
||||
if jw < vw || jh < vh {
|
||||
return Err(RawError::Corrupt(format!("a lossy JPEG of {jw} × {jh} does not cover the {vw} × {vh} of its tile inside the image")));
|
||||
}
|
||||
let (px, jw, jh, jc) = lossy_jpeg_with_tile(src, cpp, tile_budget, Some((cw, ch, vw, vh)))?;
|
||||
// the buffer holds the rows the decoded JPEG covers (the caller zero-fills the tile's
|
||||
// remainder and reads only rows it fills): a tile-many-larger decoded JPEG does not
|
||||
// get the nominal tile's full buffer (storytold/lightcraft#1517)
|
||||
@@ -408,9 +426,21 @@ const MAX_PADDING_FACTOR: usize = 32;
|
||||
#[cfg(feature = "dng")]
|
||||
const MAX_WRITER_TILE_SAMPLES: usize = 2048 * 2048;
|
||||
|
||||
#[cfg(feature = "dng")]
|
||||
#[cfg(all(test, feature = "dng"))]
|
||||
/// Decode one baseline JPEG chunk to 8-bit samples: (samples, width, height, channels).
|
||||
pub(crate) fn lossy_jpeg(src: &[u8], cpp: usize, max_samples: usize) -> Result<(Vec<u8>, usize, usize, usize)> {
|
||||
lossy_jpeg_with_tile(src, cpp, max_samples, None)
|
||||
}
|
||||
|
||||
/// Validate the actual frame before decode allocates its output. Edge tiles may store
|
||||
/// a smaller frame covering all visible samples or bounded whole-MCU padding.
|
||||
#[cfg(feature = "dng")]
|
||||
fn lossy_jpeg_with_tile(
|
||||
src: &[u8],
|
||||
cpp: usize,
|
||||
max_samples: usize,
|
||||
tile: Option<(usize, usize, usize, usize)>,
|
||||
) -> Result<(Vec<u8>, usize, usize, usize)> {
|
||||
use zune_core::bytestream::ZCursor;
|
||||
use zune_core::colorspace::ColorSpace;
|
||||
use zune_core::options::DecoderOptions;
|
||||
@@ -422,7 +452,27 @@ pub(crate) fn lossy_jpeg(src: &[u8], cpp: usize, max_samples: usize) -> Result<(
|
||||
d.decode_headers().map_err(|e| RawError::Corrupt(format!("lossy JPEG tile: {e}")))?;
|
||||
let info = d.info().ok_or_else(|| RawError::Corrupt("lossy JPEG tile without a header".into()))?;
|
||||
let (w, h) = (info.width as usize, info.height as usize);
|
||||
let c = if cpp == 1 { 1 } else { 3 };
|
||||
if !matches!(cpp, 1 | 3) {
|
||||
return Err(RawError::Unsupported(format!("lossy JPEG with {cpp} samples per pixel")));
|
||||
}
|
||||
if usize::from(info.components) != cpp {
|
||||
return Err(RawError::Corrupt(format!("lossy JPEG has {} components, expected {cpp}", info.components)));
|
||||
}
|
||||
if let Some((cw, ch, vw, vh)) = tile {
|
||||
if w < vw || h < vh {
|
||||
return Err(RawError::Corrupt(format!("a lossy JPEG of {w} x {h} does not cover the {vw} x {vh} of its tile inside the image")));
|
||||
}
|
||||
// Writers can store a whole MCU around an edge tile. Keep the documented
|
||||
// 32-pixel padding allowance without letting one tiny tile decode an arbitrary frame.
|
||||
let mcu_extent = |value: usize| value.checked_add(31).map(|rounded| rounded / 32).and_then(|blocks| blocks.checked_mul(32));
|
||||
let (Some(max_w), Some(max_h)) = (mcu_extent(cw), mcu_extent(ch)) else {
|
||||
return Err(RawError::Corrupt("lossy JPEG tile dimensions overflow MCU padding".into()));
|
||||
};
|
||||
if w > max_w || h > max_h {
|
||||
return Err(RawError::Corrupt(format!("a lossy JPEG of {w} x {h} exceeds its {max_w} x {max_h} MCU-padded tile dimensions")));
|
||||
}
|
||||
}
|
||||
let c = cpp;
|
||||
let out = w as u64 * h as u64 * c as u64;
|
||||
if w == 0 || h == 0 || out > max_samples as u64 {
|
||||
return Err(RawError::Corrupt(format!("a lossy JPEG of {w} × {h} × {c} would exceed the {max_samples} output samples allowed")));
|
||||
@@ -519,6 +569,12 @@ fn unpack_chunk(
|
||||
}
|
||||
} else {
|
||||
undo_diff_u16(row, cpp * factor);
|
||||
// Horizontal differences wrap in the declared sample precision, not in
|
||||
// the u16 container used by the unpacker (TIFF 6.0 Predictor 2).
|
||||
if bits < 16 {
|
||||
let mask = ((1u32 << bits) - 1) as u16;
|
||||
row.iter_mut().for_each(|sample| *sample &= mask);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -982,3 +1038,6 @@ mod budget_tests {
|
||||
assert_eq!(decode_groups(&big.chunks(64), 1, 1 << 20), vec![0..1]);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "dng"))]
|
||||
mod compat_review_tests;
|
||||
@@ -0,0 +1,115 @@
|
||||
use super::*;
|
||||
use craft_tiff::image::Layout;
|
||||
|
||||
fn image(width: u32, height: u32, bits: u16, cpp: u16, compression: u16, len: usize) -> ImageInfo {
|
||||
ImageInfo {
|
||||
width,
|
||||
height,
|
||||
bits_per_sample: vec![bits; cpp as usize],
|
||||
samples_per_pixel: cpp,
|
||||
compression,
|
||||
photometric: 34892,
|
||||
planar: 1,
|
||||
predictor: 1,
|
||||
sample_format: 1,
|
||||
new_subfile_type: 0,
|
||||
layout: Layout::Strips { rows_per_strip: height },
|
||||
offsets: vec![0],
|
||||
byte_counts: vec![len as u64],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_uncompressed_rows_are_rejected_in_full_and_header_modes() {
|
||||
for (bits, packing, float, bytes) in [
|
||||
(8, Packing::Msb, false, 8),
|
||||
(12, Packing::Msb, false, 12),
|
||||
(12, Packing::Word16, false, 16),
|
||||
(16, Packing::Msb, false, 16),
|
||||
(32, Packing::Msb, true, 32),
|
||||
] {
|
||||
let data = vec![0u8; bytes];
|
||||
let mut info = image(4, 2, bits, 1, comp::NONE, bytes);
|
||||
info.sample_format = if float { 3 } else { 1 };
|
||||
assert!(read_image_in(crate::Mode::Full, &data, &info, ByteOrder::Little, packing, &Limits::default()).is_ok());
|
||||
info.byte_counts[0] -= 1;
|
||||
for mode in [crate::Mode::Full, crate::Mode::Header] {
|
||||
assert!(
|
||||
matches!(read_image_in(mode, &data[..bytes - 1], &info, ByteOrder::Little, packing, &Limits::default()), Err(RawError::Corrupt(message)) if message.contains("uncompressed chunk")),
|
||||
"{bits}/{float}/{mode:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edge_padding_may_be_absent_but_visible_uncompressed_samples_must_all_exist() {
|
||||
let data = vec![31u8; 16 * 7 + 8];
|
||||
let mut info = image(8, 8, 8, 1, comp::NONE, data.len());
|
||||
info.layout = Layout::Tiles { tile_width: 16, tile_height: 16 };
|
||||
let RawData::U16(values) = read_image(&data, &info, ByteOrder::Little, Packing::Msb, &Limits::default()).unwrap() else { panic!("integer") };
|
||||
assert_eq!(values, vec![31; 64]);
|
||||
info.byte_counts[0] -= 1;
|
||||
assert!(read_image(&data[..data.len() - 1], &info, ByteOrder::Little, Packing::Msb, &Limits::default()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn packed_predictor_two_wraps_at_declared_twelve_bits() {
|
||||
// Samples [250, 10, 4090, 20]; differences modulo4096 are 0FA,F10,FF0,01A.
|
||||
let packed = [0x0f, 0xaf, 0x10, 0xff, 0x00, 0x1a];
|
||||
let ChunkPx::U16(values) = unpack_chunk(&packed, ByteOrder::Little, Packing::Msb, 4, 1, 1, 12, false, 2).unwrap() else { panic!("integer") };
|
||||
assert_eq!(values, vec![250, 10, 4090, 20]);
|
||||
}
|
||||
|
||||
#[allow(clippy::unwrap_used)] // Test-only fixture encoder; failed construction must fail the test.
|
||||
fn jpeg(cpp: usize, width: u16, height: u16) -> Vec<u8> {
|
||||
let mut data = Vec::new();
|
||||
let color = if cpp == 1 { jpeg_encoder::ColorType::Luma } else { jpeg_encoder::ColorType::Rgb };
|
||||
jpeg_encoder::Encoder::new(&mut data, 100).encode(&vec![90; width as usize * height as usize * cpp], width, height, color).unwrap();
|
||||
data
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn baseline_jpeg_components_are_checked_before_forced_colorspace_conversion() {
|
||||
for (actual, expected) in [(3, 1), (1, 3)] {
|
||||
let data = jpeg(actual, 8, 8);
|
||||
let error = lossy_jpeg(&data, expected, 1024).unwrap_err();
|
||||
assert!(matches!(error, RawError::Corrupt(message) if message.contains(&format!("{actual} components"))));
|
||||
}
|
||||
assert!(matches!(lossy_jpeg(&jpeg(3, 8, 8), 4, 1024), Err(RawError::Unsupported(message)) if message.contains("4 samples")));
|
||||
for cpp in [1, 3] {
|
||||
assert_eq!(lossy_jpeg(&jpeg(cpp, 8, 8), cpp, 1024).unwrap().3, cpp);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jpeg_tile_geometry_and_limits_are_checked_at_the_header_with_valid_edge_controls() {
|
||||
let data = jpeg(3, 8, 8);
|
||||
assert!(lossy_jpeg_with_tile(&data, 3, 1024, Some((16, 16, 8, 8))).is_ok());
|
||||
assert!(
|
||||
matches!(lossy_jpeg_with_tile(&data, 3, 1024, Some((16, 16, 9, 8))), Err(RawError::Corrupt(message)) if message.contains("does not cover"))
|
||||
);
|
||||
assert!(matches!(lossy_jpeg_with_tile(&data, 3, 32, Some((16, 16, 8, 8))), Err(RawError::Corrupt(message)) if message.contains("would exceed")));
|
||||
// Mutate only SOF dimensions, retaining a valid header but no entropy for the claimed frame.
|
||||
// The geometry error proves refusal precedes decode's short/corrupt entropy failure.
|
||||
for (width, height) in [(33u16, 8u16), (8, 33), (256, 256)] {
|
||||
let mut oversized = data.clone();
|
||||
let sof = oversized.windows(2).position(|bytes| bytes == [0xff, 0xc0]).unwrap();
|
||||
oversized[sof + 5..sof + 7].copy_from_slice(&height.to_be_bytes());
|
||||
oversized[sof + 7..sof + 9].copy_from_slice(&width.to_be_bytes());
|
||||
assert!(
|
||||
matches!(lossy_jpeg_with_tile(&oversized, 3, 1 << 20, Some((16, 16, 8, 8))), Err(RawError::Corrupt(message)) if message.contains("exceeds its"))
|
||||
);
|
||||
}
|
||||
// A genuine full MCU remains valid even when only a 4x4 part of the nominal16 tile is visible.
|
||||
let padded = jpeg(3, 32, 32);
|
||||
assert!(lossy_jpeg_with_tile(&padded, 3, 32 * 32 * 3, Some((16, 16, 4, 4))).is_ok());
|
||||
assert!(
|
||||
matches!(lossy_jpeg_with_tile(&data, 3, 1 << 20, Some((usize::MAX, 16, 8, 8))), Err(RawError::Corrupt(message)) if message.contains("overflow MCU padding"))
|
||||
);
|
||||
let mut info = image(8, 8, 8, 3, comp::LOSSY_JPEG, data.len());
|
||||
info.layout = Layout::Tiles { tile_width: 16, tile_height: 16 };
|
||||
let RawData::U16(values) = read_image(&data, &info, ByteOrder::Little, Packing::Msb, &Limits::default()).unwrap() else { panic!("integer") };
|
||||
assert_eq!(values.len(), 8 * 8 * 3);
|
||||
assert!(values.iter().all(|sample| (i32::from(*sample) - 90).abs() <= 3));
|
||||
}
|
||||
@@ -64,6 +64,7 @@ fn synthetic(w: usize, h: usize, cfa: Option<Cfa>, cpp: usize) -> RawImage {
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: Default::default(),
|
||||
notices: craft_raw::RawNotices::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -948,18 +949,31 @@ fn malformed_semantic_masks_are_skipped() {
|
||||
let mut zero = mask_ifd(2, 2, "zero size", vec![0; 4]);
|
||||
zero.set(t::IMAGE_WIDTH, Value::Long(vec![0]));
|
||||
ifd0.add_sub_ifd(zero);
|
||||
// A 4x4 8-bit mask needs 16 visible samples; three bytes must not be
|
||||
// accepted as a synthetic mask with thirteen zero-filled samples.
|
||||
let mut short = mask_ifd(4, 4, "short strip", vec![200; 3]);
|
||||
short.set(t::MASK_SUB_AREA, Value::Long(vec![u32::MAX, u32::MAX, u32::MAX, u32::MAX]));
|
||||
ifd0.add_sub_ifd(short);
|
||||
let mut valid = mask_ifd(4, 2, "valid crop", vec![10, 20, 30, 40, 50, 60, 70, 255]);
|
||||
valid.set(t::MASK_SUB_AREA, Value::Long(vec![1, 2, 8, 4]));
|
||||
ifd0.add_sub_ifd(valid);
|
||||
let mut outside = mask_ifd(2, 2, "sub area too small", vec![255; 4]);
|
||||
outside.set(t::MASK_SUB_AREA, Value::Long(vec![3, 0, 2, 4]));
|
||||
ifd0.add_sub_ifd(outside);
|
||||
});
|
||||
assert_eq!(decode(&bytes).unwrap().cpp, 3);
|
||||
let raw = decode(&bytes).unwrap();
|
||||
let baseline = decode(&profile_dng(ByteOrder::Little, |_| {})).unwrap();
|
||||
assert_eq!((raw.width, raw.height, raw.cpp), (8, 6, 3));
|
||||
assert_eq!(raw.data, baseline.data, "optional damaged masks must not change base image samples");
|
||||
let masks = craft_raw::semantic_masks(&bytes);
|
||||
let names: Vec<&str> = masks.iter().map(|m| m.name.as_str()).collect();
|
||||
assert_eq!(names, ["short strip", "sub area too small"]);
|
||||
assert!(masks.iter().all(|m| m.sub_area.is_none()), "sub areas the mask doesn't fit are ignored");
|
||||
assert_eq!(names, ["valid crop", "sub area too small"], "the truncated mask is omitted, and later usable masks survive");
|
||||
assert_eq!(masks[0].data, [10u16, 20, 30, 40, 50, 60, 70, 255].map(|sample| sample * 257));
|
||||
assert_eq!(masks[0].sub_area, Some(craft_raw::semantic::MaskSubArea { top: 1, left: 2, full_width: 8, full_height: 4 }));
|
||||
assert!((masks[0].value_at(2.5, 1.5) - 10. / 255.).abs() < 1e-4);
|
||||
assert_eq!(masks[0].value_at(1.5, 1.5), 0.);
|
||||
assert!(masks[1].sub_area.is_none(), "the complete mask survives while a sub area it doesn't fit is ignored");
|
||||
assert_eq!(masks[1].data, vec![65535; 4]);
|
||||
for m in &masks {
|
||||
assert_eq!(m.data.len(), m.width * m.height);
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ fn sample(comp: DngCompression) -> Vec<u8> {
|
||||
cinema: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
level_sources: Default::default(),
|
||||
notices: craft_raw::RawNotices::default(),
|
||||
};
|
||||
craft_raw::write_dng(&raw, &DngWriteOptions { compression: comp, ..Default::default() }).unwrap()
|
||||
}
|
||||
|
||||
+31
-1
@@ -302,6 +302,7 @@ struct Used {
|
||||
raster_limit: AtomicUsize,
|
||||
invalid_svg: AtomicUsize,
|
||||
svg_limit: AtomicUsize,
|
||||
svg_resource_limit: AtomicUsize,
|
||||
}
|
||||
|
||||
/// What resolving one drawing's embedded images used or refused.
|
||||
@@ -323,6 +324,10 @@ pub struct ImageUsage {
|
||||
pub invalid_svg_images: usize,
|
||||
/// Embedded SVG data refused by byte or nesting limits.
|
||||
pub svg_limit_images: usize,
|
||||
/// Subset of `svg_limit_images` refused by unpacked byte or embedded-document-chain limits.
|
||||
/// XML/tree-depth omissions and invalid or corrupt data do not set this counter. Consumers
|
||||
/// can reject resource overruns while still allowing an optional, over-deep child to be omitted.
|
||||
pub svg_resource_limit_images: usize,
|
||||
}
|
||||
|
||||
/// A data-URL resolver and the accounting for the one drawing it serves.
|
||||
@@ -350,6 +355,7 @@ impl ImageResolver {
|
||||
raster_limit: AtomicUsize::new(0),
|
||||
invalid_svg: AtomicUsize::new(0),
|
||||
svg_limit: AtomicUsize::new(0),
|
||||
svg_resource_limit: AtomicUsize::new(0),
|
||||
}),
|
||||
}
|
||||
}
|
||||
@@ -376,6 +382,7 @@ impl ImageResolver {
|
||||
raster_limit_images: self.used.raster_limit.load(Ordering::Relaxed),
|
||||
invalid_svg_images: self.used.invalid_svg.load(Ordering::Relaxed),
|
||||
svg_limit_images: self.used.svg_limit.load(Ordering::Relaxed),
|
||||
svg_resource_limit_images: self.used.svg_resource_limit.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -438,6 +445,7 @@ fn raster_image(used: &Used, limits: &Limits, data: Arc<Vec<u8>>, format: craft_
|
||||
fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<'_>) -> Option<usvg::ImageKind> {
|
||||
if used.depth.load(Ordering::Relaxed) >= limits.embedded_depth {
|
||||
used.svg_limit.fetch_add(1, Ordering::Relaxed);
|
||||
used.svg_resource_limit.fetch_add(1, Ordering::Relaxed);
|
||||
return None;
|
||||
}
|
||||
let left = limits.embedded_bytes.saturating_sub(used.bytes.load(Ordering::Relaxed));
|
||||
@@ -445,15 +453,37 @@ fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<'
|
||||
Ok(text) => text,
|
||||
// What a refused document unpacked before it was refused was work too: it uses up what
|
||||
// is left, so the next image can't repeat it.
|
||||
Err(_) => {
|
||||
Err(error) => {
|
||||
used.bytes.fetch_add(left, Ordering::Relaxed);
|
||||
used.svg_limit.fetch_add(1, Ordering::Relaxed);
|
||||
if matches!(error, svgz::Error::TooLarge(_)) {
|
||||
used.svg_resource_limit.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
return None;
|
||||
}
|
||||
};
|
||||
// The raw/inflated bytes can fit while UTF-16/Latin-1 decoding expands
|
||||
// beyond the aggregate allowance. A refusal consumes the remaining work.
|
||||
if text.len() > left {
|
||||
used.bytes.fetch_add(left, Ordering::Relaxed);
|
||||
used.svg_limit.fetch_add(1, Ordering::Relaxed);
|
||||
used.svg_resource_limit.fetch_add(1, Ordering::Relaxed);
|
||||
return None;
|
||||
}
|
||||
used.bytes.fetch_add(text.len(), Ordering::Relaxed);
|
||||
let xml = match checked(&text, limits) {
|
||||
Ok(xml) => xml,
|
||||
// Decoding UTF-16 or Latin-1 can exceed the text cap after the
|
||||
// raw/inflated bytes fitted. This is a resource refusal, not invalid XML.
|
||||
Err(Error::Text(svgz::Error::TooLarge(_))) => {
|
||||
used.svg_limit.fetch_add(1, Ordering::Relaxed);
|
||||
used.svg_resource_limit.fetch_add(1, Ordering::Relaxed);
|
||||
return None;
|
||||
}
|
||||
Err(Error::TooDeep(_)) => {
|
||||
used.svg_limit.fetch_add(1, Ordering::Relaxed);
|
||||
return None;
|
||||
}
|
||||
Err(_) => {
|
||||
used.invalid_svg.fetch_add(1, Ordering::Relaxed);
|
||||
return None;
|
||||
|
||||
@@ -245,6 +245,195 @@ fn embedded_svgs_are_read_within_their_budgets() {
|
||||
assert_eq!(found(&svg(r#"<image width="5" height="5" xlink:href="/etc/passwd.svg"/>"#), shallow()), [false; 0]);
|
||||
}
|
||||
|
||||
mod resource_limit_usage {
|
||||
use super::*;
|
||||
|
||||
fn data_image(bytes: &[u8]) -> String {
|
||||
let encoded: String = bytes.iter().map(|byte| format!("%{byte:02X}")).collect();
|
||||
format!(r#"<image width="5" height="5" xlink:href="data:image/svg+xml,{encoded}"/>"#)
|
||||
}
|
||||
|
||||
fn packed(text: &str, gzip: bool) -> Vec<u8> {
|
||||
if gzip { svgz::compress(text.as_bytes()) } else { text.as_bytes().to_vec() }
|
||||
}
|
||||
|
||||
/// The outer document has its own generous text allowance, so tiny resolver limits
|
||||
/// test the public embedded admission contract rather than rejecting the outer XML.
|
||||
fn resolved(text: &str, limits: Limits, outer_depth: usize) -> (Vec<bool>, ImageUsage) {
|
||||
let resolver = ImageResolver::new(limits);
|
||||
resolver.set_document_depth(outer_depth);
|
||||
let options = usvg::Options { image_href_resolver: resolver.resolver(), ..Default::default() };
|
||||
let tree = parse(text.as_bytes(), &shallow(), &options).unwrap();
|
||||
let mut found = Vec::new();
|
||||
images(tree.root(), &mut found);
|
||||
(found, resolver.usage())
|
||||
}
|
||||
|
||||
fn expanding_encoded_children() -> Vec<(&'static str, Vec<u8>)> {
|
||||
let latin =
|
||||
format!("<?xml version='1.0' encoding='ISO-8859-1'?>{}", svg(&format!("<title>{}</title><rect width='1' height='1'/>", "é".repeat(100))));
|
||||
let latin = latin.chars().map(|c| u8::try_from(u32::from(c)).unwrap()).collect();
|
||||
let wide = svg(&format!("<title>{}</title><rect width='1' height='1'/>", "ࠀ".repeat(400)));
|
||||
let mut little = vec![0xff, 0xfe];
|
||||
little.extend(wide.encode_utf16().flat_map(u16::to_le_bytes));
|
||||
let mut big = vec![0xfe, 0xff];
|
||||
big.extend(wide.encode_utf16().flat_map(u16::to_be_bytes));
|
||||
vec![("Latin-1", latin), ("UTF-16LE", little), ("UTF-16BE", big)]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_expansion_is_a_resource_refusal_for_plain_and_gzip_children() {
|
||||
for (encoding, raw) in expanding_encoded_children() {
|
||||
let decoded = svgz::decode(&raw).unwrap();
|
||||
let cap = raw.len();
|
||||
assert!(decoded.len() > cap, "{encoding} fixture must expand after inflation");
|
||||
for gzip in [false, true] {
|
||||
let data = if gzip { svgz::compress(&raw) } else { raw.clone() };
|
||||
assert_eq!(svgz::inflate(&data, cap).unwrap().len(), cap, "raw bytes fit exactly");
|
||||
assert_eq!(svgz::text(&data, cap).unwrap().len(), decoded.len(), "decoding itself succeeds");
|
||||
let limits = Limits { bytes: cap, embedded_bytes: decoded.len(), ..shallow() };
|
||||
assert!(matches!(checked(&decoded, &limits), Err(Error::Text(svgz::Error::TooLarge(n))) if n == cap));
|
||||
let image = data_image(&data);
|
||||
let (found, usage) = resolved(&svg(&image), limits, 1);
|
||||
assert!(found.is_empty(), "{encoding}, gzip {gzip}");
|
||||
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (1, 1, 0));
|
||||
assert_eq!(usage.embedded_svg_bytes, decoded.len(), "spent decoded work remains charged");
|
||||
let (found, usage) = resolved(&svg(&format!("{image}{image}")), limits, 1);
|
||||
assert!(found.is_empty());
|
||||
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (2, 2, 0));
|
||||
assert_eq!(usage.embedded_svg_bytes, decoded.len(), "a refused sibling cannot repeat exhausted work");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_expansion_respects_aggregate_bytes_even_when_each_child_fits() {
|
||||
for (encoding, raw) in expanding_encoded_children() {
|
||||
let decoded = svgz::decode(&raw).unwrap();
|
||||
let limits = Limits { bytes: decoded.len(), embedded_bytes: raw.len(), ..shallow() };
|
||||
assert!(decoded.len() > limits.embedded_bytes);
|
||||
checked(&decoded, &limits).unwrap(); // The independent per-image cap allows this complete XML.
|
||||
for gzip in [false, true] {
|
||||
let data = if gzip { svgz::compress(&raw) } else { raw.clone() };
|
||||
assert_eq!(svgz::inflate(&data, limits.embedded_bytes).unwrap().len(), raw.len());
|
||||
let image = data_image(&data);
|
||||
let (found, usage) = resolved(&svg(&image), limits, 1);
|
||||
assert!(found.is_empty(), "{encoding}, gzip {gzip}");
|
||||
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (1, 1, 0));
|
||||
assert_eq!(usage.embedded_svg_bytes, limits.embedded_bytes, "refusal consumes exactly the remaining allowance");
|
||||
let (found, usage) = resolved(&svg(&format!("{image}{image}")), limits, 1);
|
||||
assert!(found.is_empty());
|
||||
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (2, 2, 0));
|
||||
assert_eq!(usage.embedded_svg_bytes, limits.embedded_bytes, "a later child cannot repeat exhausted work");
|
||||
let (found, usage) = resolved(&svg(&image), Limits { embedded_bytes: decoded.len(), ..limits }, 1);
|
||||
assert_eq!(found, [true]);
|
||||
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (0, 0, 0));
|
||||
assert_eq!(usage.embedded_svg_bytes, decoded.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_expansion_at_the_cap_is_valid_and_malformed_xml_stays_invalid() {
|
||||
for (encoding, raw) in expanding_encoded_children() {
|
||||
let decoded = svgz::decode(&raw).unwrap();
|
||||
for gzip in [false, true] {
|
||||
let data = if gzip { svgz::compress(&raw) } else { raw.clone() };
|
||||
let limits = Limits { bytes: decoded.len(), embedded_bytes: decoded.len(), ..shallow() };
|
||||
let (found, usage) = resolved(&svg(&data_image(&data)), limits, 1);
|
||||
assert_eq!(found, [true], "{encoding}, gzip {gzip}");
|
||||
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (0, 0, 0));
|
||||
assert_eq!(usage.embedded_svg_bytes, decoded.len());
|
||||
// Preserve the declared/BOM encoding when making the XML malformed;
|
||||
// re-encoding Latin-1 text as UTF-8 would test a second expansion instead.
|
||||
let closing_bytes = if encoding == "Latin-1" { 6 } else { 12 };
|
||||
let malformed = &raw[..raw.len() - closing_bytes];
|
||||
let data = if gzip { svgz::compress(malformed) } else { malformed.to_vec() };
|
||||
let (found, usage) = resolved(&svg(&data_image(&data)), limits, 1);
|
||||
assert!(found.is_empty());
|
||||
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (0, 0, 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embedded_chain_caps_are_explicit_resource_refusals_for_plain_and_gzip_children() {
|
||||
for gzip in [false, true] {
|
||||
let inner = svg("<rect width='1' height='1'/>");
|
||||
let middle = svg(&data_image(&packed(&inner, gzip)));
|
||||
let outer = svg(&data_image(&packed(&middle, gzip)));
|
||||
let (found, usage) = resolved(&outer, Limits { embedded_depth: 1, ..shallow() }, 1);
|
||||
assert_eq!(found, [true]);
|
||||
assert_eq!(usage.embedded_svg_depth, 1);
|
||||
assert_eq!(usage.svg_limit_images, 1, "strict consumers still see the existing limit flag");
|
||||
assert_eq!(usage.svg_resource_limit_images, 1, "a chain cap must not become a successful partial bounded preview");
|
||||
let (found, usage) = resolved(&outer, Limits { embedded_depth: 2, ..shallow() }, 1);
|
||||
assert_eq!(found, [true, true]);
|
||||
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embedded_per_image_byte_caps_are_resource_refusals_for_plain_and_gzip_children() {
|
||||
let inner = svg("<rect width='1' height='1'/>");
|
||||
for gzip in [false, true] {
|
||||
let outer = svg(&data_image(&packed(&inner, gzip)));
|
||||
let (_, usage) = resolved(&outer, Limits { bytes: inner.len() - 1, embedded_bytes: inner.len() * 3, ..shallow() }, 1);
|
||||
assert_eq!(usage.embedded_svg_count, 0);
|
||||
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (1, 1));
|
||||
let (found, usage) = resolved(&outer, Limits { bytes: inner.len(), ..shallow() }, 1);
|
||||
assert_eq!(found, [true]);
|
||||
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aggregate_embedded_bytes_are_resource_refusals_even_when_each_child_fits() {
|
||||
let inner = svg("<rect width='1' height='1'/>");
|
||||
for gzip in [false, true] {
|
||||
let image = data_image(&packed(&inner, gzip));
|
||||
let outer = svg(&format!("{image}{image}"));
|
||||
let cap = inner.len() * 2 - 1;
|
||||
let (found, usage) = resolved(&outer, Limits { embedded_bytes: cap, ..shallow() }, 1);
|
||||
assert_eq!(found, [true]);
|
||||
assert_eq!(usage.embedded_svg_bytes, cap, "a refused child still spends the remaining work allowance");
|
||||
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (1, 1));
|
||||
let (found, usage) = resolved(&outer, Limits { embedded_bytes: inner.len() * 2, ..shallow() }, 1);
|
||||
assert_eq!(found, [true, true]);
|
||||
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn combined_xml_depth_omissions_remain_distinct_from_resource_refusals() {
|
||||
let inner = svg("<rect width='1' height='1'/>");
|
||||
for gzip in [false, true] {
|
||||
let image = data_image(&packed(&inner, gzip));
|
||||
let outer = svg(&format!("{}{}{}", "<g>".repeat(4), image, "</g>".repeat(4)));
|
||||
let (found, usage) = resolved(&outer, Limits { tree_depth: 6, ..shallow() }, 5);
|
||||
assert!(found.is_empty());
|
||||
assert_eq!(usage.svg_limit_images, 1, "strict consumers must continue to reject this omission");
|
||||
assert_eq!(usage.svg_resource_limit_images, 0, "non-strict consumers may omit this child without disguising a byte/chain overrun");
|
||||
let (found, usage) = resolved(&svg(&image), Limits { tree_depth: 6, ..shallow() }, 1);
|
||||
assert_eq!(found, [true]);
|
||||
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_and_corrupt_children_do_not_claim_a_resource_limit() {
|
||||
let mut corrupt = svgz::compress(svg("<rect width='1' height='1'/>").as_bytes());
|
||||
let crc = corrupt.len() - 8;
|
||||
corrupt[crc] ^= 1;
|
||||
for data in [b"not XML".to_vec(), corrupt] {
|
||||
let (found, usage) = resolved(&svg(&data_image(&data)), shallow(), 1);
|
||||
assert!(found.is_empty());
|
||||
assert!(usage.invalid_svg_images > 0 || usage.svg_limit_images > 0, "strict consumers still observe unreadable content");
|
||||
assert_eq!(usage.svg_resource_limit_images, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mod draw_checks {
|
||||
use super::*;
|
||||
|
||||
|
||||
+124
-11
@@ -125,6 +125,16 @@ struct Asset {
|
||||
has_audio: bool,
|
||||
}
|
||||
|
||||
// Bound semantic resource chains separately from XML syntax: flat media resources can
|
||||
// reference one another without adding any XML nesting.
|
||||
const MAX_MEDIA_DEPTH: usize = 64;
|
||||
const MAX_XML_DEPTH: usize = 128;
|
||||
// Retrying truncated resources is useful at shallower depths, but must remain bounded.
|
||||
const MAX_REIMPORTS: usize = 4096;
|
||||
// A hostile document may reference the same large cached graph repeatedly. Bound total
|
||||
// reachability work, not just one walk, and refuse reuse when safety cannot be established.
|
||||
const MAX_CACHE_EDGES: usize = 1_000_000;
|
||||
|
||||
struct Imp<'a, 'c, 'r> {
|
||||
b: Builder<'c>,
|
||||
opts: &'a ImportOptions<'a>,
|
||||
@@ -135,6 +145,13 @@ struct Imp<'a, 'c, 'r> {
|
||||
media_nodes: HashMap<String, Node<'a>>,
|
||||
media_items: HashMap<String, ItemId>,
|
||||
media_active: HashSet<String>,
|
||||
depth_hits: usize,
|
||||
// Depths at which a cached resource was truncated. A retry may improve neither limit
|
||||
// at the expense of the other, and reuses the same item so earlier references stay valid.
|
||||
media_truncated_at: HashMap<String, (usize, usize)>,
|
||||
reimports_left: usize,
|
||||
cache_edges_left: usize,
|
||||
depth: usize,
|
||||
effects: HashMap<String, String>,
|
||||
}
|
||||
|
||||
@@ -186,6 +203,11 @@ pub fn import(text: &str, opts: &ImportOptions, report: &mut Report) -> Result<I
|
||||
media_nodes: HashMap::new(),
|
||||
media_items: HashMap::new(),
|
||||
media_active: HashSet::new(),
|
||||
depth_hits: 0,
|
||||
media_truncated_at: HashMap::new(),
|
||||
reimports_left: MAX_REIMPORTS,
|
||||
cache_edges_left: MAX_CACHE_EDGES,
|
||||
depth: 0,
|
||||
effects: HashMap::new(),
|
||||
};
|
||||
if let Some(res) = child(root, "resources") {
|
||||
@@ -229,7 +251,7 @@ pub fn import(text: &str, opts: &ImportOptions, report: &mut Report) -> Result<I
|
||||
"project" => {
|
||||
if let Some(s) = child(c, "sequence") {
|
||||
let name = c.attribute("name").unwrap_or("Project");
|
||||
let id = imp.sequence(s, name, Some(bin));
|
||||
let id = imp.sequence(s, name, Some(bin), None);
|
||||
imp.b.top.push(id);
|
||||
}
|
||||
}
|
||||
@@ -244,7 +266,7 @@ pub fn import(text: &str, opts: &ImportOptions, report: &mut Report) -> Result<I
|
||||
}
|
||||
for p in children(root, "project") {
|
||||
if let Some(s) = child(p, "sequence") {
|
||||
let id = imp.sequence(s, p.attribute("name").unwrap_or("Project"), None);
|
||||
let id = imp.sequence(s, p.attribute("name").unwrap_or("Project"), None, None);
|
||||
imp.b.top.push(id);
|
||||
}
|
||||
}
|
||||
@@ -308,34 +330,100 @@ impl<'a> Imp<'a, '_, '_> {
|
||||
Some(a)
|
||||
}
|
||||
|
||||
/// A rebuilt item retains its ID. A cached ancestor that already points to it must
|
||||
/// not be inserted underneath it, even when the resource names are not directly active.
|
||||
fn cache_reaches_active(&mut self, root: ItemId) -> bool {
|
||||
let active: HashSet<ItemId> = self.media_active.iter().filter_map(|id| self.media_items.get(id).copied()).collect();
|
||||
if active.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let mut seen = HashSet::from([root]);
|
||||
let mut pending = vec![root];
|
||||
while let Some(id) = pending.pop() {
|
||||
if active.contains(&id) {
|
||||
return true;
|
||||
}
|
||||
let Some(seq) = self.b.p.sequence(id) else { continue };
|
||||
for item in seq.all_tracks().flat_map(|track| &track.items) {
|
||||
if self.cache_edges_left == 0 {
|
||||
return true;
|
||||
}
|
||||
self.cache_edges_left -= 1;
|
||||
if active.contains(&item.item) {
|
||||
return true;
|
||||
}
|
||||
if seen.insert(item.item) {
|
||||
pending.push(item.item);
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Nested sequence from a `<media>` resource.
|
||||
fn media(&mut self, id: &str) -> Option<ItemId> {
|
||||
if let Some(i) = self.media_items.get(id) {
|
||||
return Some(*i);
|
||||
}
|
||||
let n = *self.media_nodes.get(id)?;
|
||||
let s = child(n, "sequence")?;
|
||||
if !self.media_active.insert(id.into()) {
|
||||
// Check before the cache: a truncated resource may already have an item while it
|
||||
// is being rebuilt, but a back-edge must never turn that item into a self-cycle.
|
||||
if self.media_active.contains(id) {
|
||||
self.report.warn(format!("recursive FCPXML media resource {id:?} ignored"));
|
||||
return None;
|
||||
}
|
||||
let depth = self.media_active.len();
|
||||
let mut again = None;
|
||||
if let Some(i) = self.media_items.get(id).copied() {
|
||||
// Imported with a nested one cut off: import it again (into the same item) where that
|
||||
// one now fits.
|
||||
let shallower = self
|
||||
.media_truncated_at
|
||||
.get(id)
|
||||
.is_some_and(|&(media, xml)| depth <= media && self.depth <= xml && (depth < media || self.depth < xml))
|
||||
&& !self.media_active.contains(id);
|
||||
if !shallower || self.reimports_left == 0 {
|
||||
if self.cache_reaches_active(i) {
|
||||
self.depth_hits += 1;
|
||||
self.report.warn(format!(
|
||||
"FCPXML cached media {id:?} could form a recursive graph or exceeds the cache traversal budget; imported as a gap"
|
||||
));
|
||||
return None;
|
||||
}
|
||||
return Some(i);
|
||||
}
|
||||
self.reimports_left -= 1;
|
||||
again = Some(i);
|
||||
}
|
||||
let n = *self.media_nodes.get(id)?;
|
||||
let s = child(n, "sequence")?;
|
||||
// The active set is the current nesting chain: a long acyclic chain of compound clips
|
||||
// would otherwise recurse once per resource.
|
||||
if depth >= MAX_MEDIA_DEPTH {
|
||||
self.depth_hits += 1;
|
||||
self.report.warn(format!("FCPXML media resource {id:?} is nested too deeply; imported as a gap"));
|
||||
return None;
|
||||
}
|
||||
self.media_active.insert(id.into());
|
||||
// Mark the resource before descending: a compound clip can reference itself through a
|
||||
// ref-clip before a completed sequence is available to cache.
|
||||
let name = n.attribute("name").unwrap_or("Compound Clip").to_string();
|
||||
let item = self.sequence(s, &name, None);
|
||||
let hits_before = self.depth_hits;
|
||||
let item = self.sequence(s, &name, None, again);
|
||||
self.media_active.remove(id);
|
||||
self.media_items.insert(id.into(), item);
|
||||
if self.depth_hits > hits_before {
|
||||
self.media_truncated_at.insert(id.into(), (depth, self.depth));
|
||||
} else {
|
||||
self.media_truncated_at.remove(id);
|
||||
}
|
||||
Some(item)
|
||||
}
|
||||
|
||||
fn sequence(&mut self, s: Node<'a>, name: &str, bin: Option<BinId>) -> ItemId {
|
||||
fn sequence(&mut self, s: Node<'a>, name: &str, bin: Option<BinId>, reuse: Option<ItemId>) -> ItemId {
|
||||
let (rate, w, h) = s.attribute("format").and_then(|f| self.formats.get(f)).copied().unwrap_or((FrameRate::default(), 1920, 1080));
|
||||
let df = s.attribute("tcFormat") == Some("DF");
|
||||
let mut settings = settings_for(rate, w, h, df);
|
||||
if let Some(sr) = s.attribute("audioRate").and_then(parse_audio_rate) {
|
||||
settings.sample_rate = sr;
|
||||
}
|
||||
let id = self.b.reserve_sequence(name, settings.clone(), bin);
|
||||
let id = reuse.unwrap_or_else(|| self.b.reserve_sequence(name, settings.clone(), bin));
|
||||
let mut seq = empty_sequence(settings);
|
||||
let tc_start = attr_time(s, "tcStart").unwrap_or(Tick::ZERO);
|
||||
seq.start_timecode = rate.frame_at(tc_start);
|
||||
@@ -353,7 +441,19 @@ impl<'a> Imp<'a, '_, '_> {
|
||||
|
||||
/// Lay out a storyline. `origin` maps the storyline's offsets to timeline time
|
||||
/// (`timeline = origin + offset`). `lane` is the storyline's lane (0 = primary).
|
||||
/// Refuses nesting past [`MAX_XML_DEPTH`].
|
||||
fn storyline(&mut self, seq: &mut Sequence, ctx: &Ctx, spine: Node<'a>, origin: Tick, lane: i32, primary: bool) {
|
||||
if self.depth >= MAX_XML_DEPTH {
|
||||
self.depth_hits += 1;
|
||||
self.report.warn(format!("XML storylines are nested too deeply (more than {MAX_XML_DEPTH}); deeper clips were skipped"));
|
||||
return;
|
||||
}
|
||||
self.depth += 1;
|
||||
self.storyline_inner(seq, ctx, spine, origin, lane, primary);
|
||||
self.depth -= 1;
|
||||
}
|
||||
|
||||
fn storyline_inner(&mut self, seq: &mut Sequence, ctx: &Ctx, spine: Node<'a>, origin: Tick, lane: i32, primary: bool) {
|
||||
// (timeline start, timeline end, video clip on this storyline's track)
|
||||
let mut laid: Vec<(Tick, Tick, Option<ClipId>)> = Vec::new();
|
||||
let mut transitions: Vec<(usize, Node)> = Vec::new();
|
||||
@@ -421,7 +521,20 @@ impl<'a> Imp<'a, '_, '_> {
|
||||
}
|
||||
|
||||
/// Lay out one storyline element at timeline time `t0`. Returns (timeline end, video clip).
|
||||
/// Refuses nesting past [`MAX_XML_DEPTH`].
|
||||
fn element(&mut self, seq: &mut Sequence, ctx: &Ctx, el: Node<'a>, t0: Tick, lane: i32, primary: bool) -> (Tick, Option<ClipId>) {
|
||||
if self.depth >= MAX_XML_DEPTH {
|
||||
self.depth_hits += 1;
|
||||
self.report.warn(format!("XML elements are nested too deeply (more than {MAX_XML_DEPTH}); deeper clips were skipped"));
|
||||
return (t0, None);
|
||||
}
|
||||
self.depth += 1;
|
||||
let out = self.element_inner(seq, ctx, el, t0, lane, primary);
|
||||
self.depth -= 1;
|
||||
out
|
||||
}
|
||||
|
||||
fn element_inner(&mut self, seq: &mut Sequence, ctx: &Ctx, el: Node<'a>, t0: Tick, lane: i32, primary: bool) -> (Tick, Option<ClipId>) {
|
||||
let tag = el.name();
|
||||
let dur = attr_time(el, "duration").unwrap_or(Tick::ZERO);
|
||||
let local_start = attr_time(el, "start").unwrap_or(Tick::ZERO);
|
||||
|
||||
@@ -503,3 +503,114 @@ fn relative_media_urls_are_percent_decoded() {
|
||||
let boats = &p.sequence(only_seq(&imp)).unwrap().video_tracks[0].items[1];
|
||||
assert_eq!(media_key(p, boats.item), "/proj/media/boats b.mov");
|
||||
}
|
||||
|
||||
// Parity regressions from FilmCraft e6df318 (including #4799), adapted to the neutral model.
|
||||
#[test]
|
||||
fn media_cut_off_by_the_depth_limit_is_complete_when_reached_higher_up() {
|
||||
// m0 … m64 nest one inside the next. Project A reaches m63 at the depth limit, so m64 is left
|
||||
// out of it; project B refers to m63 directly, where m64 fits, and must not get the cut-off copy.
|
||||
let mut media = String::new();
|
||||
for i in 0..64 {
|
||||
media.push_str(&format!(
|
||||
r#"<media id="m{i}" name="M{i}"><sequence format="r1"><spine><ref-clip ref="m{}" offset="0s" start="0s" duration="1s"/></spine></sequence></media>"#,
|
||||
i + 1
|
||||
));
|
||||
}
|
||||
media.push_str(r#"<media id="m64" name="M64"><sequence format="r1"><spine/></sequence></media>"#);
|
||||
let xml = format!(
|
||||
r#"<fcpxml version="1.10"><resources><format id="r1" frameDuration="1/24s" width="1920" height="1080"/>{media}</resources>
|
||||
<library><event>
|
||||
<project name="A"><sequence format="r1"><spine><ref-clip ref="m0" offset="0s" start="0s" duration="1s"/></spine></sequence></project>
|
||||
<project name="B"><sequence format="r1"><spine><ref-clip ref="m63" offset="0s" start="0s" duration="1s"/></spine></sequence></project>
|
||||
</event></library></fcpxml>"#
|
||||
);
|
||||
let (imp, rep) = import(xml.as_bytes(), Format::Fcpxml, None).unwrap();
|
||||
assert!(rep.mentions("nested too deeply"), "{rep:?}");
|
||||
let m63 = imp.project.items.values().find(|i| i.name == "M63").and_then(|i| imp.project.sequence(i.id)).unwrap();
|
||||
assert_eq!(m63.video_tracks[0].items.len(), 1, "M63 keeps its nested M64");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn media_cut_off_by_xml_layout_is_complete_when_revisited_at_the_same_media_depth() {
|
||||
// Both references have media-chain depth zero, but the first is reached through connected
|
||||
// clips. Its resource must not remain cached as complete after the XML-layout guard fires.
|
||||
for depth in [125, 126] {
|
||||
let nested = format!(
|
||||
"{}<ref-clip lane=\"1\" ref=\"m0\" offset=\"0s\" duration=\"1s\"/>{}",
|
||||
"<gap lane=\"1\" offset=\"0s\" duration=\"1s\">".repeat(depth),
|
||||
"</gap>".repeat(depth)
|
||||
);
|
||||
let xml = format!(
|
||||
r#"<fcpxml version="1.10"><resources>
|
||||
<format id="r1" frameDuration="1/24s" width="1920" height="1080"/>
|
||||
<media id="m0" name="M0"><sequence format="r1"><spine>
|
||||
<ref-clip ref="m1" offset="0s" duration="1s"/>
|
||||
</spine></sequence></media>
|
||||
<media id="m1" name="M1"><sequence format="r1"><spine/></sequence></media>
|
||||
</resources><library><event>
|
||||
<project name="A"><sequence format="r1"><spine>{nested}</spine></sequence></project>
|
||||
<project name="B"><sequence format="r1"><spine>
|
||||
<ref-clip ref="m0" offset="0s" duration="1s"/>
|
||||
</spine></sequence></project>
|
||||
</event></library></fcpxml>"#
|
||||
);
|
||||
let (imp, rep) = import(xml.as_bytes(), Format::Fcpxml, None).unwrap();
|
||||
assert!(rep.mentions("XML") && rep.mentions("nested"), "{rep:?}");
|
||||
let m0 = imp.project.items.values().find(|i| i.name == "M0").and_then(|i| imp.project.sequence(i.id)).unwrap();
|
||||
assert_eq!(m0.video_tracks[0].items.len(), 1, "M0 keeps its nested M1 after the shallower revisit");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flat_resource_chain_is_bounded_even_when_xml_is_shallow() {
|
||||
let mut media = String::new();
|
||||
for i in 0..512 {
|
||||
media.push_str(&format!(
|
||||
r#"<media id="m{i}" name="M{i}"><sequence><spine><ref-clip ref="m{}" duration="1s"/></spine></sequence></media>"#,
|
||||
i + 1
|
||||
));
|
||||
}
|
||||
let xml = format!(
|
||||
r#"<fcpxml version="1.10"><resources>{media}</resources><project name="P"><sequence><spine><ref-clip ref="m0" duration="1s"/></spine></sequence></project></fcpxml>"#
|
||||
);
|
||||
let (imp, rep) = import(xml.as_bytes(), Format::Fcpxml, None).unwrap();
|
||||
assert!(rep.mentions("nested too deeply"), "{rep:?}");
|
||||
assert!(imp.project.items.len() <= 65, "the flat resource graph is not expanded beyond the semantic limit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shallower_retry_does_not_link_a_cached_ancestor_back_to_the_rebuilt_item() {
|
||||
let mut media = String::new();
|
||||
for i in 0..64 {
|
||||
media.push_str(&format!(
|
||||
r#"<media id="m{i}" name="M{i}"><sequence><spine><ref-clip ref="m{}" duration="1s"/></spine></sequence></media>"#,
|
||||
(i + 1) % 64
|
||||
));
|
||||
}
|
||||
let xml = format!(
|
||||
r#"<fcpxml version="1.10"><resources>{media}</resources>
|
||||
<project name="A"><sequence><spine><ref-clip ref="m0" duration="1s"/></spine></sequence></project>
|
||||
<project name="B"><sequence><spine><ref-clip ref="m62" duration="1s"/></spine></sequence></project></fcpxml>"#
|
||||
);
|
||||
let (imp, rep) = import(xml.as_bytes(), Format::Fcpxml, None).unwrap();
|
||||
assert!(rep.mentions("cached media"), "{rep:?}");
|
||||
// Check the final graph, not just the resource-name recursion stack during import.
|
||||
fn visit(p: &Project, id: ItemId, active: &mut std::collections::HashSet<ItemId>, done: &mut std::collections::HashSet<ItemId>) {
|
||||
if done.contains(&id) {
|
||||
return;
|
||||
}
|
||||
assert!(active.insert(id), "cached media introduced a cycle at {id:?}");
|
||||
if let Some(seq) = p.sequence(id) {
|
||||
for item in seq.all_tracks().flat_map(|track| &track.items) {
|
||||
visit(p, item.item, active, done);
|
||||
}
|
||||
}
|
||||
active.remove(&id);
|
||||
done.insert(id);
|
||||
}
|
||||
let mut active = std::collections::HashSet::new();
|
||||
let mut done = std::collections::HashSet::new();
|
||||
for id in imp.project.items.keys() {
|
||||
visit(&imp.project, *id, &mut active, &mut done);
|
||||
}
|
||||
}
|
||||
@@ -170,6 +170,7 @@ fn scene(ui: &mut egui::Ui, ours: bool) {
|
||||
}
|
||||
|
||||
fn render(ours: bool, hover: Option<&str>, open: Option<&str>) -> image::RgbaImage {
|
||||
let _gpu = crate::gpu_test_lock();
|
||||
// Two of the combo boxes have no label: the scene compares looks, not names.
|
||||
let mut h = Harness::builder().with_size(egui::vec2(480.0, 700.0)).with_accessibility_check(false).build_ui(move |ui| scene(ui, ours));
|
||||
h.run();
|
||||
|
||||
@@ -27,3 +27,14 @@ pub mod text;
|
||||
pub mod theme;
|
||||
|
||||
pub use combo::ComboBox;
|
||||
|
||||
/// Lets one unit test at a time hold a wgpu renderer. Each renderer creates its own Vulkan instance
|
||||
/// and device. When one test uses its device while a test on another thread creates or destroys
|
||||
/// one, the Vulkan loader now and then crashes the test binary with SIGSEGV. Take the lock before
|
||||
/// building the harness (`let _gpu = crate::gpu_test_lock();`), so it is released after the harness
|
||||
/// is dropped. A test that failed while holding it does not block the others.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn gpu_test_lock() -> std::sync::MutexGuard<'static, ()> {
|
||||
static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||
LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
@@ -123,6 +123,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn selected_tab_erases_a_baseline_centered_on_its_bottom_edge() {
|
||||
let _gpu = crate::gpu_test_lock();
|
||||
let mut h = egui_kittest::Harness::builder().with_size(Vec2::new(120.0, 80.0)).wgpu().build_ui(|ui| {
|
||||
ui.painter().rect_filled(ui.max_rect(), 0.0, Color32::WHITE);
|
||||
let rect = Rect::from_min_size(egui::pos2(10.0, 10.0), Vec2::new(100.0, 26.0));
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
mod fixtures;
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Mutex, MutexGuard, PoisonError};
|
||||
|
||||
use egui::{Color32, Context, Event, FullOutput, Modifiers, PointerButton, Pos2, RawInput, Rect, Response, TextureId, Ui};
|
||||
use egui_kittest::{TestRenderer, wgpu::WgpuTestRenderer};
|
||||
@@ -28,6 +29,17 @@ struct Capture {
|
||||
meshes: Option<Vec<Mesh>>,
|
||||
}
|
||||
|
||||
/// Lets one test at a time hold a wgpu renderer. Each renderer creates its own Vulkan instance and
|
||||
/// device. When one test uses its device while a test on another thread creates or destroys one,
|
||||
/// the Vulkan loader now and then crashes the test binary with SIGSEGV.
|
||||
static GPU_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
/// Take this before creating a renderer (`let _gpu = gpu_lock();`), so it is released after the
|
||||
/// renderer is dropped. A test that failed while holding it does not block the others.
|
||||
fn gpu_lock() -> MutexGuard<'static, ()> {
|
||||
GPU_LOCK.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
fn context(theme: Theme, scale: f32) -> Context {
|
||||
let ctx = Context::default();
|
||||
ctx.set_theme(theme.egui());
|
||||
@@ -184,6 +196,7 @@ impl Artifacts {
|
||||
|
||||
#[test]
|
||||
fn rendered_controls_match_independent_reference_pixels_and_meshes() {
|
||||
let _gpu = gpu_lock();
|
||||
let mut renderer = WgpuTestRenderer::new();
|
||||
let mut artifacts = Artifacts::new("controls");
|
||||
let mut count = 0;
|
||||
@@ -213,6 +226,7 @@ fn rendered_controls_match_independent_reference_pixels_and_meshes() {
|
||||
|
||||
#[test]
|
||||
fn visual_comparisons_detect_color_spacing_and_stroke_mutations() {
|
||||
let _gpu = gpu_lock();
|
||||
let mut renderer = WgpuTestRenderer::new();
|
||||
let mut artifacts = Artifacts::new("negative-controls");
|
||||
let mut count = 0;
|
||||
@@ -241,6 +255,7 @@ fn toolbar_fitting_matches_literal_group_and_overflow_positions() {
|
||||
use craft_ui::layout::fit_toolbar;
|
||||
use egui::{Align, Layout, Stroke, StrokeKind, UiBuilder, pos2, vec2};
|
||||
|
||||
let _gpu = gpu_lock();
|
||||
let mut renderer = WgpuTestRenderer::new();
|
||||
let mut artifacts = Artifacts::new("toolbar");
|
||||
let mut count = 0;
|
||||
@@ -295,6 +310,7 @@ fn widget_palette_renders_all_five_states_with_existing_borders_and_expansion()
|
||||
use craft_ui::theme::WidgetPalette;
|
||||
use egui::{CornerRadius, Stroke, StrokeKind, pos2, vec2};
|
||||
|
||||
let _gpu = gpu_lock();
|
||||
let mut renderer = WgpuTestRenderer::new();
|
||||
let mut artifacts = Artifacts::new("widget-palette");
|
||||
for theme in [Theme::Light, Theme::Dark] {
|
||||
|
||||
+59
-43
@@ -56,48 +56,8 @@ pub fn animation(width: u32, height: u32, frames: &[AnimFrame], opts: &AnimOptio
|
||||
if frames.is_empty() {
|
||||
return Err(Error::Encode("an animation needs a frame"));
|
||||
}
|
||||
let mut anmf = Vec::new();
|
||||
let mut any_alpha = opts.alpha;
|
||||
for f in frames {
|
||||
if f.width == 0 || f.height == 0 || f.x % 2 != 0 || f.y % 2 != 0 {
|
||||
return Err(Error::Encode("a frame is empty or not at an even position"));
|
||||
}
|
||||
if f.x.checked_add(f.width).is_none_or(|r| r > width) || f.y.checked_add(f.height).is_none_or(|b| b > height) {
|
||||
return Err(Error::Encode("a frame reaches past the canvas"));
|
||||
}
|
||||
if f.duration_ms >= 1 << 24 {
|
||||
return Err(Error::Encode("a frame duration is 2^24 ms or more"));
|
||||
}
|
||||
let mut body = Vec::new();
|
||||
u24(&mut body, f.x / 2);
|
||||
u24(&mut body, f.y / 2);
|
||||
u24(&mut body, f.width - 1);
|
||||
u24(&mut body, f.height - 1);
|
||||
u24(&mut body, f.duration_ms);
|
||||
// Bit 1: do not blend; bit 0: dispose to the background.
|
||||
body.push(u8::from(!f.blend) << 1 | u8::from(f.dispose_to_background));
|
||||
match f.payload {
|
||||
Payload::Lossy { vp8, alpha } => {
|
||||
if f.width > MAX_VP8_DIMENSION || f.height > MAX_VP8_DIMENSION {
|
||||
return Err(Error::Encode("a lossy frame is at most 16383 pixels a side"));
|
||||
}
|
||||
if let Some(a) = alpha {
|
||||
if a.len() as u64 != u64::from(f.width) * u64::from(f.height) {
|
||||
return Err(Error::Encode("an alpha plane does not match its frame"));
|
||||
}
|
||||
any_alpha = true;
|
||||
let mut alph = Vec::with_capacity(a.len() + 1);
|
||||
// Header byte: no pre-processing, no filtering, no compression.
|
||||
alph.push(0);
|
||||
alph.extend_from_slice(a);
|
||||
chunk(&mut body, b"ALPH", &alph)?;
|
||||
}
|
||||
chunk(&mut body, b"VP8 ", vp8)?;
|
||||
}
|
||||
Payload::Lossless { vp8l } => chunk(&mut body, b"VP8L", vp8l)?,
|
||||
}
|
||||
chunk(&mut anmf, b"ANMF", &body)?;
|
||||
}
|
||||
// Determine the header flag before writing, without retaining another full animation.
|
||||
let any_alpha = opts.alpha || frames.iter().any(|f| matches!(f.payload, Payload::Lossy { alpha: Some(_), .. }));
|
||||
let mut body = b"WEBP".to_vec();
|
||||
let flags = [(opts.icc.is_some(), 0x20u8), (any_alpha, 0x10), (opts.exif.is_some(), 0x08), (opts.xmp.is_some(), 0x04), (true, 0x02)]
|
||||
.iter()
|
||||
@@ -113,7 +73,46 @@ pub fn animation(width: u32, height: u32, frames: &[AnimFrame], opts: &AnimOptio
|
||||
let mut anim = opts.background_bgra.to_vec();
|
||||
anim.extend_from_slice(&opts.loop_count.to_le_bytes());
|
||||
chunk(&mut body, b"ANIM", &anim)?;
|
||||
body.extend_from_slice(&anmf);
|
||||
|
||||
for f in frames {
|
||||
if f.width == 0 || f.height == 0 || f.x % 2 != 0 || f.y % 2 != 0 {
|
||||
return Err(Error::Encode("a frame is empty or not at an even position"));
|
||||
}
|
||||
if f.x.checked_add(f.width).is_none_or(|r| r > width) || f.y.checked_add(f.height).is_none_or(|b| b > height) {
|
||||
return Err(Error::Encode("a frame reaches past the canvas"));
|
||||
}
|
||||
if f.duration_ms >= 1 << 24 {
|
||||
return Err(Error::Encode("a frame duration is 2^24 ms or more"));
|
||||
}
|
||||
let mut frame_body = Vec::new();
|
||||
u24(&mut frame_body, f.x / 2);
|
||||
u24(&mut frame_body, f.y / 2);
|
||||
u24(&mut frame_body, f.width - 1);
|
||||
u24(&mut frame_body, f.height - 1);
|
||||
u24(&mut frame_body, f.duration_ms);
|
||||
// Bit 1: do not blend; bit 0: dispose to the background.
|
||||
frame_body.push(u8::from(!f.blend) << 1 | u8::from(f.dispose_to_background));
|
||||
match f.payload {
|
||||
Payload::Lossy { vp8, alpha } => {
|
||||
if f.width > MAX_VP8_DIMENSION || f.height > MAX_VP8_DIMENSION {
|
||||
return Err(Error::Encode("a lossy frame is at most 16383 pixels a side"));
|
||||
}
|
||||
if let Some(a) = alpha {
|
||||
if a.len() as u64 != u64::from(f.width) * u64::from(f.height) {
|
||||
return Err(Error::Encode("an alpha plane does not match its frame"));
|
||||
}
|
||||
let mut alph = Vec::with_capacity(a.len() + 1);
|
||||
// Header byte: no pre-processing, no filtering, no compression.
|
||||
alph.push(0);
|
||||
alph.extend_from_slice(a);
|
||||
chunk(&mut frame_body, b"ALPH", &alph)?;
|
||||
}
|
||||
chunk(&mut frame_body, b"VP8 ", vp8)?;
|
||||
}
|
||||
Payload::Lossless { vp8l } => chunk(&mut frame_body, b"VP8L", vp8l)?,
|
||||
}
|
||||
chunk(&mut body, b"ANMF", &frame_body)?;
|
||||
}
|
||||
if let Some(exif) = opts.exif {
|
||||
chunk(&mut body, b"EXIF", exif)?;
|
||||
}
|
||||
@@ -231,6 +230,23 @@ mod tests {
|
||||
assert_eq!(&file[anim + 8..anim + 14], &[1, 2, 3, 4, 0, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lossy_container_keeps_alpha_payload_padding_and_xmp() {
|
||||
// Container-only fixture: the writer must not reinterpret an encoder's VP8 bytes.
|
||||
let vp8 = [11, 22, 33];
|
||||
let alpha = [128];
|
||||
let f = frame(Payload::Lossy { vp8: &vp8, alpha: Some(&alpha) }, 0, 0, 1, 1, 17);
|
||||
let file = animation(1, 1, &[f], &AnimOptions { xmp: Some(b"xmp"), ..Default::default() }).unwrap();
|
||||
let info = scan(&file, 10).unwrap();
|
||||
assert!(info.alpha && info.complete);
|
||||
assert_eq!(info.frames[0].duration_ms, 17);
|
||||
let at = file.windows(4).position(|c| c == b"ANMF").unwrap();
|
||||
let payload = at + 8 + 16;
|
||||
assert_eq!(&file[payload..payload + 10], &[b'A', b'L', b'P', b'H', 2, 0, 0, 0, 0, 128]);
|
||||
assert_eq!(&file[payload + 10..payload + 22], &[b'V', b'P', b'8', b' ', 3, 0, 0, 0, 11, 22, 33, 0]);
|
||||
assert_eq!(&file[file.len() - 12..], &[b'X', b'M', b'P', b' ', 3, 0, 0, 0, b'x', b'm', b'p', 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_does_not_fit_is_refused() {
|
||||
let px = vp8l(2, 2, [0; 4]);
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
//! Input is treated as hostile: every read is bounds-checked, chunk sizes are checked against the
|
||||
//! file, and the number of frames recorded is capped by the caller.
|
||||
|
||||
pub mod anim;
|
||||
#[cfg(feature = "frames")]
|
||||
pub mod frames;
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
//! Guard the public module wiring: the implementation must be usable by consumers.
|
||||
#[test]
|
||||
fn public_animation_api_refuses_empty_movies() {
|
||||
let result = craft_webp::anim::animation(1, 1, &[], &craft_webp::anim::AnimOptions::default());
|
||||
assert!(result.is_err());
|
||||
}
|
||||
+348
-28
@@ -3,6 +3,7 @@
|
||||
//! This layer recognizes the document, stage, scene, layer and keyframe fields defined by XFL.
|
||||
//! It does not construct a consumer's animation model. More detailed payloads remain available
|
||||
//! through [`Document::root`] for the application adapter.
|
||||
use crate::shape::XflElement;
|
||||
use craft_xml::{Doctype, El, Error as XmlError, Limits, Local, Options, Pat, UnknownEntity};
|
||||
use std::borrow::Cow;
|
||||
|
||||
@@ -104,36 +105,102 @@ pub struct FrameModel {
|
||||
}
|
||||
|
||||
/// A timeline (`DOMTimeline`) listed under `DOMDocument/timelines`.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Timeline<'a> {
|
||||
element: &'a El,
|
||||
#[derive(Debug)]
|
||||
pub struct Timeline<'a, E: XflElement = El> {
|
||||
element: &'a E,
|
||||
}
|
||||
|
||||
impl<E: XflElement> Copy for Timeline<'_, E> {}
|
||||
|
||||
impl<E: XflElement> Clone for Timeline<'_, E> {
|
||||
fn clone(&self) -> Self {
|
||||
*self
|
||||
}
|
||||
}
|
||||
|
||||
/// A native `DOMLayer` element.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Layer<'a> {
|
||||
element: &'a El,
|
||||
#[derive(Debug)]
|
||||
pub struct Layer<'a, E: XflElement = El> {
|
||||
element: &'a E,
|
||||
}
|
||||
|
||||
impl<E: XflElement> Copy for Layer<'_, E> {}
|
||||
|
||||
impl<E: XflElement> Clone for Layer<'_, E> {
|
||||
fn clone(&self) -> Self {
|
||||
*self
|
||||
}
|
||||
}
|
||||
|
||||
/// A native `DOMFrame` element.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Frame<'a> {
|
||||
element: &'a El,
|
||||
#[derive(Debug)]
|
||||
pub struct Frame<'a, E: XflElement = El> {
|
||||
element: &'a E,
|
||||
}
|
||||
|
||||
fn attr_last<'a>(element: &'a El, name: &str) -> Option<&'a str> {
|
||||
element.attrs.iter().rev().find(|(key, _)| key == name).map(|(_, value)| value.as_str())
|
||||
/// An `Include` entry in the document's `<symbols>` library list.
|
||||
#[derive(Debug)]
|
||||
pub struct SymbolInclude<'a, E: XflElement = El> {
|
||||
element: &'a E,
|
||||
}
|
||||
|
||||
fn number_f32(element: &El, name: &str) -> Option<f32> {
|
||||
/// A folder entry in the document's `<folders>` library list.
|
||||
#[derive(Debug)]
|
||||
pub struct LibraryFolder<'a, E: XflElement = El> {
|
||||
element: &'a E,
|
||||
}
|
||||
|
||||
/// Native metadata stored on a library `DOMSymbolItem`.
|
||||
#[derive(Debug)]
|
||||
pub struct SymbolItem<'a, E: XflElement = El> {
|
||||
element: &'a E,
|
||||
}
|
||||
|
||||
impl<E: XflElement> Copy for Frame<'_, E> {}
|
||||
|
||||
impl<E: XflElement> Clone for Frame<'_, E> {
|
||||
fn clone(&self) -> Self {
|
||||
*self
|
||||
}
|
||||
}
|
||||
|
||||
impl<E: XflElement> Copy for SymbolInclude<'_, E> {}
|
||||
|
||||
impl<E: XflElement> Clone for SymbolInclude<'_, E> {
|
||||
fn clone(&self) -> Self {
|
||||
*self
|
||||
}
|
||||
}
|
||||
|
||||
impl<E: XflElement> Copy for LibraryFolder<'_, E> {}
|
||||
|
||||
impl<E: XflElement> Clone for LibraryFolder<'_, E> {
|
||||
fn clone(&self) -> Self {
|
||||
*self
|
||||
}
|
||||
}
|
||||
|
||||
impl<E: XflElement> Copy for SymbolItem<'_, E> {}
|
||||
|
||||
impl<E: XflElement> Clone for SymbolItem<'_, E> {
|
||||
fn clone(&self) -> Self {
|
||||
*self
|
||||
}
|
||||
}
|
||||
|
||||
fn attr_last<'a>(element: &'a impl XflElement, name: &str) -> Option<&'a str> {
|
||||
element.attr(name)
|
||||
}
|
||||
|
||||
fn number_f32(element: &impl XflElement, name: &str) -> Option<f32> {
|
||||
attr_last(element, name).and_then(|value| value.trim().parse::<f32>().ok()).filter(|value| value.is_finite())
|
||||
}
|
||||
|
||||
fn number_u32(element: &El, name: &str) -> Option<u32> {
|
||||
fn number_u32(element: &impl XflElement, name: &str) -> Option<u32> {
|
||||
attr_last(element, name).and_then(|value| value.trim().parse::<u32>().ok())
|
||||
}
|
||||
|
||||
fn boolean(element: &El, name: &str) -> Option<bool> {
|
||||
fn boolean(element: &impl XflElement, name: &str) -> Option<bool> {
|
||||
attr_last(element, name).map(|value| value.trim() == "true")
|
||||
}
|
||||
|
||||
@@ -187,7 +254,7 @@ impl Document {
|
||||
let profile = self
|
||||
.root
|
||||
.elements()
|
||||
.find(|element| Local("flash_profile").matches(&element.name) && boolean(element, "current") == Some(true))
|
||||
.find(|element| Local("flash_profile").matches(&element.name) && boolean(*element, "current") == Some(true))
|
||||
.or_else(|| self.root.elements().find(|element| Local("flash_profile").matches(&element.name)))
|
||||
.unwrap_or(&self.root);
|
||||
let properties = profile.child(Local("PublishFlashProperties")).unwrap_or(profile);
|
||||
@@ -237,23 +304,28 @@ impl Document {
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Timeline<'a> {
|
||||
impl<'a, E: XflElement> Timeline<'a, E> {
|
||||
/// Creates a borrowed XFL timeline view over a consumer's compatible XML element.
|
||||
pub fn new(element: &'a E) -> Self {
|
||||
Self { element }
|
||||
}
|
||||
|
||||
pub fn name(self) -> Option<&'a str> {
|
||||
attr_last(self.element, "name")
|
||||
}
|
||||
|
||||
/// Raw timeline XML for application-specific payload conversion.
|
||||
pub fn element(self) -> &'a El {
|
||||
pub fn element(self) -> &'a E {
|
||||
self.element
|
||||
}
|
||||
|
||||
/// Layers in native order. Non-`DOMLayer` extension nodes are skipped.
|
||||
pub fn layers(self) -> impl Iterator<Item = Layer<'a>> {
|
||||
pub fn layers(self) -> impl Iterator<Item = Layer<'a, E>> {
|
||||
self.element
|
||||
.child(Local("layers"))
|
||||
.child("layers")
|
||||
.into_iter()
|
||||
.flat_map(El::elements)
|
||||
.filter(|element| Local("DOMLayer").matches(&element.name))
|
||||
.flat_map(|element| element.elements())
|
||||
.filter(|element| element.local_name() == "DOMLayer")
|
||||
.map(|element| Layer { element })
|
||||
}
|
||||
|
||||
@@ -262,7 +334,12 @@ impl<'a> Timeline<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Layer<'a> {
|
||||
impl<'a, E: XflElement> Layer<'a, E> {
|
||||
/// Creates a borrowed XFL layer view over a consumer's compatible XML element.
|
||||
pub fn new(element: &'a E) -> Self {
|
||||
Self { element }
|
||||
}
|
||||
|
||||
pub fn name(self) -> Option<&'a str> {
|
||||
attr_last(self.element, "name")
|
||||
}
|
||||
@@ -296,12 +373,16 @@ impl<'a> Layer<'a> {
|
||||
}
|
||||
|
||||
/// Frames in native order. Non-`DOMFrame` extension nodes are skipped.
|
||||
pub fn frames(self) -> impl Iterator<Item = Frame<'a>> {
|
||||
pub fn element(self) -> &'a E {
|
||||
self.element
|
||||
.child(Local("frames"))
|
||||
}
|
||||
|
||||
pub fn frames(self) -> impl Iterator<Item = Frame<'a, E>> {
|
||||
self.element
|
||||
.child("frames")
|
||||
.into_iter()
|
||||
.flat_map(El::elements)
|
||||
.filter(|element| Local("DOMFrame").matches(&element.name))
|
||||
.flat_map(|element| element.elements())
|
||||
.filter(|element| element.local_name() == "DOMFrame")
|
||||
.map(|element| Frame { element })
|
||||
}
|
||||
|
||||
@@ -325,7 +406,12 @@ impl<'a> Layer<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Frame<'a> {
|
||||
impl<'a, E: XflElement> Frame<'a, E> {
|
||||
/// Creates a borrowed XFL frame view over a consumer's compatible XML element.
|
||||
pub fn new(element: &'a E) -> Self {
|
||||
Self { element }
|
||||
}
|
||||
|
||||
pub fn index(self) -> Option<u32> {
|
||||
number_u32(self.element, "index")
|
||||
}
|
||||
@@ -347,7 +433,7 @@ impl<'a> Frame<'a> {
|
||||
}
|
||||
|
||||
/// Raw frame XML for application-specific shape, action, sound and tween conversion.
|
||||
pub fn element(self) -> &'a El {
|
||||
pub fn element(self) -> &'a E {
|
||||
self.element
|
||||
}
|
||||
|
||||
@@ -367,6 +453,112 @@ impl<'a> Frame<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, E: XflElement> SymbolInclude<'a, E> {
|
||||
/// Creates a borrowed view over a native `Include` element.
|
||||
pub fn new(element: &'a E) -> Self {
|
||||
Self { element }
|
||||
}
|
||||
|
||||
/// The package-relative library XML path from `href`, kept in native string form.
|
||||
pub fn href(self) -> Option<&'a str> {
|
||||
attr_last(self.element, "href")
|
||||
}
|
||||
|
||||
/// The source element for application-specific path resolution.
|
||||
pub fn element(self) -> &'a E {
|
||||
self.element
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, E: XflElement> LibraryFolder<'a, E> {
|
||||
/// Creates a borrowed view over a native `DOMFolderItem` element.
|
||||
pub fn new(element: &'a E) -> Self {
|
||||
Self { element }
|
||||
}
|
||||
|
||||
/// The folder path from `name`, kept in native string form.
|
||||
pub fn name(self) -> Option<&'a str> {
|
||||
attr_last(self.element, "name")
|
||||
}
|
||||
|
||||
/// The source element for application-specific path validation.
|
||||
pub fn element(self) -> &'a E {
|
||||
self.element
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, E: XflElement> SymbolItem<'a, E> {
|
||||
/// Creates a borrowed view over a native `DOMSymbolItem` element.
|
||||
pub fn new(element: &'a E) -> Self {
|
||||
Self { element }
|
||||
}
|
||||
|
||||
pub fn name(self) -> Option<&'a str> {
|
||||
attr_last(self.element, "name")
|
||||
}
|
||||
|
||||
pub fn symbol_type(self) -> Option<&'a str> {
|
||||
attr_last(self.element, "symbolType")
|
||||
}
|
||||
|
||||
pub fn linkage_export_for_action_script(self) -> Option<&'a str> {
|
||||
attr_last(self.element, "linkageExportForAS")
|
||||
}
|
||||
|
||||
pub fn linkage_class_name(self) -> Option<&'a str> {
|
||||
attr_last(self.element, "linkageClassName")
|
||||
}
|
||||
|
||||
pub fn linkage_identifier(self) -> Option<&'a str> {
|
||||
attr_last(self.element, "linkageIdentifier")
|
||||
}
|
||||
|
||||
pub fn linkage_base_class(self) -> Option<&'a str> {
|
||||
attr_last(self.element, "linkageBaseClass")
|
||||
}
|
||||
|
||||
/// Whether the symbol declares any native 9-slice scale-grid attribute, even if malformed.
|
||||
pub fn has_scale_grid(self) -> bool {
|
||||
["scaleGridLeft", "scaleGridRight", "scaleGridTop", "scaleGridBottom"].iter().any(|name| attr_last(self.element, name).is_some())
|
||||
}
|
||||
|
||||
pub fn scale_grid_left(self) -> Option<f32> {
|
||||
number_f32(self.element, "scaleGridLeft")
|
||||
}
|
||||
|
||||
pub fn scale_grid_right(self) -> Option<f32> {
|
||||
number_f32(self.element, "scaleGridRight")
|
||||
}
|
||||
|
||||
pub fn scale_grid_top(self) -> Option<f32> {
|
||||
number_f32(self.element, "scaleGridTop")
|
||||
}
|
||||
|
||||
pub fn scale_grid_bottom(self) -> Option<f32> {
|
||||
number_f32(self.element, "scaleGridBottom")
|
||||
}
|
||||
|
||||
/// The symbol's optional timeline wrapper, for the consumer to map into its own model.
|
||||
pub fn timeline(self) -> Option<&'a E> {
|
||||
self.element.child("timeline")
|
||||
}
|
||||
|
||||
/// The source element for application-specific symbol validation and mapping.
|
||||
pub fn element(self) -> &'a E {
|
||||
self.element
|
||||
}
|
||||
}
|
||||
|
||||
/// Symbol library entries in source order, skipping non-`Include` extension nodes.
|
||||
pub fn symbol_includes<'a, E: XflElement>(root: &'a E) -> impl Iterator<Item = SymbolInclude<'a, E>> + 'a {
|
||||
root.child("symbols").into_iter().flat_map(XflElement::elements).filter(|element| element.local_name() == "Include").map(SymbolInclude::new)
|
||||
}
|
||||
|
||||
/// Library folders in source order, skipping non-`DOMFolderItem` extension nodes.
|
||||
pub fn library_folders<'a, E: XflElement>(root: &'a E) -> impl Iterator<Item = LibraryFolder<'a, E>> + 'a {
|
||||
root.child("folders").into_iter().flat_map(XflElement::elements).filter(|element| element.local_name() == "DOMFolderItem").map(LibraryFolder::new)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -408,6 +600,91 @@ mod tests {
|
||||
assert_eq!(frame.element().child("elements").unwrap().child("Shape").unwrap().name, "Shape");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn borrowed_timeline_views_accept_compatible_element_adapters() {
|
||||
#[derive(Debug)]
|
||||
struct AppElement {
|
||||
name: String,
|
||||
attributes: Vec<(String, String)>,
|
||||
children: Vec<AppElement>,
|
||||
}
|
||||
|
||||
impl XflElement for AppElement {
|
||||
type Elements<'a> = std::slice::Iter<'a, Self>;
|
||||
|
||||
fn local_name(&self) -> &str {
|
||||
self.name.rsplit(':').next().unwrap_or(&self.name)
|
||||
}
|
||||
|
||||
fn attr(&self, name: &str) -> Option<&str> {
|
||||
self.attributes.iter().rev().find(|(key, _)| key == name).map(|(_, value)| value.as_str())
|
||||
}
|
||||
|
||||
fn elements(&self) -> Self::Elements<'_> {
|
||||
self.children.iter()
|
||||
}
|
||||
|
||||
fn text_content(&self) -> Cow<'_, str> {
|
||||
Cow::Borrowed("")
|
||||
}
|
||||
}
|
||||
|
||||
fn element(name: &str, attributes: &[(&str, &str)], children: Vec<AppElement>) -> AppElement {
|
||||
AppElement {
|
||||
name: name.to_string(),
|
||||
attributes: attributes.iter().map(|(key, value)| ((*key).to_string(), (*value).to_string())).collect(),
|
||||
children,
|
||||
}
|
||||
}
|
||||
|
||||
let root = element(
|
||||
"x:DOMDocument",
|
||||
&[],
|
||||
vec![element(
|
||||
"x:timelines",
|
||||
&[],
|
||||
vec![element(
|
||||
"x:DOMTimeline",
|
||||
&[("name", "Old"), ("name", "Scene")],
|
||||
vec![element(
|
||||
"x:layers",
|
||||
&[],
|
||||
vec![element(
|
||||
"x:DOMLayer",
|
||||
&[("name", "Old"), ("name", "Layer"), ("color", "#abcdef"), ("visible", "false")],
|
||||
vec![element(
|
||||
"x:frames",
|
||||
&[],
|
||||
vec![element(
|
||||
"x:DOMFrame",
|
||||
&[
|
||||
("index", "999"),
|
||||
("index", "3"),
|
||||
("duration", "2"),
|
||||
("name", "Intro"),
|
||||
("labelType", "name"),
|
||||
("tweenType", "motion"),
|
||||
],
|
||||
vec![],
|
||||
)],
|
||||
)],
|
||||
)],
|
||||
)],
|
||||
)],
|
||||
)],
|
||||
);
|
||||
let timeline_element = root.child("timelines").unwrap().child("DOMTimeline").unwrap();
|
||||
let timeline = Timeline::new(timeline_element);
|
||||
assert_eq!(timeline.name(), Some("Scene"));
|
||||
let layer = timeline.layers().next().unwrap();
|
||||
assert_eq!(layer.name(), Some("Layer"));
|
||||
assert_eq!(layer.color(), Some("#abcdef"));
|
||||
assert_eq!(layer.visible(), Some(false));
|
||||
let frame = layer.frames().next().unwrap();
|
||||
assert_eq!((frame.index(), frame.duration()), (Some(3), Some(2)));
|
||||
assert_eq!((frame.name(), frame.label_type(), frame.tween_type()), (Some("Intro"), Some("name"), Some("motion")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uses_last_duplicate_attribute_like_the_legacy_xfl_adapter() {
|
||||
let document = Document::parse(br##"<DOMDocument width="1" width="2"><timelines/></DOMDocument>"##).unwrap();
|
||||
@@ -521,4 +798,47 @@ mod tests {
|
||||
assert_eq!(settings.action_script_version.as_deref(), Some("3"));
|
||||
assert_eq!(settings.document_class.as_deref(), Some("RootClass"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_symbol_library_entries_and_native_symbol_metadata() {
|
||||
let document = Document::parse(
|
||||
br#"<DOMDocument><symbols><Extension/><Include href="LIBRARY/Shape.xml"/><Include href="LIBRARY/Button.xml"/></symbols><folders><Extension/><DOMFolderItem name="Characters/Hero"/></folders></DOMDocument>"#,
|
||||
)
|
||||
.unwrap();
|
||||
let includes: Vec<_> = symbol_includes(document.root()).map(SymbolInclude::href).collect();
|
||||
assert_eq!(includes, [Some("LIBRARY/Shape.xml"), Some("LIBRARY/Button.xml")]);
|
||||
let include = symbol_includes(document.root()).next().unwrap();
|
||||
assert_eq!(include.element().local_name(), "Include");
|
||||
let folders: Vec<_> = library_folders(document.root()).map(LibraryFolder::name).collect();
|
||||
assert_eq!(folders, [Some("Characters/Hero")]);
|
||||
let folder = library_folders(document.root()).next().unwrap();
|
||||
assert_eq!(folder.element().local_name(), "DOMFolderItem");
|
||||
|
||||
let symbol = Document::parse(
|
||||
br#"<x:DOMSymbolItem xmlns:x="urn:adobe:xfl" name="Characters/Hero" name="Characters/Final" symbolType="graphic" linkageExportForAS="true" linkageClassName="example.Hero" linkageIdentifier="legacy.Hero" linkageBaseClass="flash.display.MovieClip" scaleGridLeft="10" scaleGridRight="90" scaleGridTop="5" scaleGridBottom="75"><x:timeline><x:DOMTimeline/></x:timeline></x:DOMSymbolItem>"#,
|
||||
)
|
||||
.unwrap();
|
||||
let item = SymbolItem::new(symbol.root());
|
||||
assert_eq!(item.element().local_name(), "DOMSymbolItem");
|
||||
assert_eq!(item.name(), Some("Characters/Final"), "the last duplicate attribute wins");
|
||||
assert_eq!(item.symbol_type(), Some("graphic"));
|
||||
assert_eq!(item.linkage_export_for_action_script(), Some("true"));
|
||||
assert_eq!(item.linkage_class_name(), Some("example.Hero"));
|
||||
assert_eq!(item.linkage_identifier(), Some("legacy.Hero"));
|
||||
assert_eq!(item.linkage_base_class(), Some("flash.display.MovieClip"));
|
||||
assert_eq!(
|
||||
(item.scale_grid_left(), item.scale_grid_right(), item.scale_grid_top(), item.scale_grid_bottom()),
|
||||
(Some(10.0), Some(90.0), Some(5.0), Some(75.0))
|
||||
);
|
||||
assert!(item.has_scale_grid());
|
||||
assert!(item.timeline().is_some());
|
||||
|
||||
let malformed = Document::parse(br#"<DOMSymbolItem scaleGridLeft="not-a-number"/>"#).unwrap();
|
||||
let malformed = SymbolItem::new(malformed.root());
|
||||
assert!(malformed.has_scale_grid(), "attribute presence remains observable when its value is malformed");
|
||||
assert_eq!(malformed.scale_grid_left(), None);
|
||||
|
||||
let absent = Document::parse(b"<DOMSymbolItem/>").unwrap();
|
||||
assert!(!SymbolItem::new(absent.root()).has_scale_grid());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
//! Borrowed views of native XFL filter records.
|
||||
//!
|
||||
//! This module parses the wire-level filter names, fields, and gradient-ramp entries. Consumers
|
||||
//! remain responsible for mapping those values into their own rendering/filter models.
|
||||
|
||||
use crate::shape::{self, Color, XflElement, boolean, integer, number};
|
||||
|
||||
/// Native filter variants understood by XFL.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum FilterKind {
|
||||
Blur,
|
||||
DropShadow,
|
||||
Glow,
|
||||
AdjustColor,
|
||||
Bevel,
|
||||
GradientGlow,
|
||||
GradientBevel,
|
||||
}
|
||||
|
||||
/// Bevel/glow placement recorded by the native `type` attribute.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum FilterSide {
|
||||
Inner,
|
||||
Outer,
|
||||
Full,
|
||||
}
|
||||
|
||||
/// The native `<filters>` list on an element, preserving source order and unknown records.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct FilterList<'a, E: XflElement> {
|
||||
wrapper: Option<&'a E>,
|
||||
}
|
||||
|
||||
impl<'a, E: XflElement> FilterList<'a, E> {
|
||||
pub fn new(element: &'a E) -> Self {
|
||||
Self { wrapper: element.child("filters") }
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> impl Iterator<Item = Filter<'a, E>> + 'a {
|
||||
let wrapper = self.wrapper;
|
||||
wrapper.into_iter().flat_map(XflElement::elements).map(|element| Filter { element })
|
||||
}
|
||||
}
|
||||
|
||||
/// One native filter record. Unknown records are retained so the consumer can report/skip them.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Filter<'a, E: XflElement> {
|
||||
element: &'a E,
|
||||
}
|
||||
|
||||
impl<'a, E: XflElement> Filter<'a, E> {
|
||||
pub fn kind(&self) -> Option<FilterKind> {
|
||||
match self.element.local_name() {
|
||||
"BlurFilter" => Some(FilterKind::Blur),
|
||||
"DropShadowFilter" => Some(FilterKind::DropShadow),
|
||||
"GlowFilter" => Some(FilterKind::Glow),
|
||||
"AdjustColorFilter" => Some(FilterKind::AdjustColor),
|
||||
"BevelFilter" => Some(FilterKind::Bevel),
|
||||
"GradientGlowFilter" => Some(FilterKind::GradientGlow),
|
||||
"GradientBevelFilter" => Some(FilterKind::GradientBevel),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// XFL treats filters as enabled unless explicitly set to false.
|
||||
pub fn enabled(&self) -> bool {
|
||||
boolean(self.element, "enabled") != Some(false)
|
||||
}
|
||||
|
||||
/// Quality is a native integer in the range 1 through 3, defaulting to 1.
|
||||
pub fn quality(&self) -> u8 {
|
||||
integer(self.element, "quality").unwrap_or(1).clamp(1, 3) as u8
|
||||
}
|
||||
|
||||
pub fn blur_x(&self) -> Option<f32> {
|
||||
number(self.element, "blurX")
|
||||
}
|
||||
|
||||
pub fn blur_y(&self) -> Option<f32> {
|
||||
number(self.element, "blurY")
|
||||
}
|
||||
|
||||
pub fn strength(&self) -> Option<f32> {
|
||||
number(self.element, "strength")
|
||||
}
|
||||
|
||||
pub fn angle(&self) -> Option<f32> {
|
||||
number(self.element, "angle")
|
||||
}
|
||||
|
||||
pub fn distance(&self) -> Option<f32> {
|
||||
number(self.element, "distance")
|
||||
}
|
||||
|
||||
pub fn brightness(&self) -> Option<f32> {
|
||||
number(self.element, "brightness")
|
||||
}
|
||||
|
||||
pub fn contrast(&self) -> Option<f32> {
|
||||
number(self.element, "contrast")
|
||||
}
|
||||
|
||||
pub fn saturation(&self) -> Option<f32> {
|
||||
number(self.element, "saturation")
|
||||
}
|
||||
|
||||
pub fn hue(&self) -> Option<f32> {
|
||||
number(self.element, "hue")
|
||||
}
|
||||
|
||||
pub fn inner(&self) -> Option<bool> {
|
||||
boolean(self.element, "inner")
|
||||
}
|
||||
|
||||
pub fn knockout(&self) -> Option<bool> {
|
||||
boolean(self.element, "knockout")
|
||||
}
|
||||
|
||||
pub fn hide_object(&self) -> Option<bool> {
|
||||
boolean(self.element, "hideObject")
|
||||
}
|
||||
|
||||
pub fn side(&self) -> Option<FilterSide> {
|
||||
match self.element.attr("type").map(str::trim) {
|
||||
Some("inner") => Some(FilterSide::Inner),
|
||||
Some("outer") => Some(FilterSide::Outer),
|
||||
Some("full") => Some(FilterSide::Full),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn color(&self) -> Color {
|
||||
shape::color(self.element, "color", "alpha")
|
||||
}
|
||||
|
||||
pub fn shadow_color(&self) -> Color {
|
||||
shape::color(self.element, "shadowColor", "shadowAlpha")
|
||||
}
|
||||
|
||||
/// Explicit highlight color, absent when the native attribute is missing so consumers can
|
||||
/// apply the format-specific white highlight default.
|
||||
pub fn highlight_color(&self) -> Option<Color> {
|
||||
self.element.attr("highlightColor").map(|_| shape::color(self.element, "highlightColor", "highlightAlpha"))
|
||||
}
|
||||
|
||||
pub fn highlight_alpha(&self) -> Option<f32> {
|
||||
number(self.element, "highlightAlpha")
|
||||
}
|
||||
|
||||
/// Native gradient entries, either direct children or one wrapper level below the filter.
|
||||
pub fn ramp_entries(&self) -> impl Iterator<Item = RampEntry<'a, E>> + 'a {
|
||||
self.element
|
||||
.elements()
|
||||
.flat_map(|child| std::iter::once(child).chain(child.elements()))
|
||||
.filter(|entry| entry.local_name() == "GradientEntry")
|
||||
.map(|element| RampEntry { element })
|
||||
}
|
||||
}
|
||||
|
||||
/// One XFL gradient-ramp entry.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct RampEntry<'a, E: XflElement> {
|
||||
element: &'a E,
|
||||
}
|
||||
|
||||
impl<'a, E: XflElement> RampEntry<'a, E> {
|
||||
pub fn ratio(&self) -> Option<f32> {
|
||||
number(self.element, "ratio")
|
||||
}
|
||||
|
||||
pub fn color(&self) -> Color {
|
||||
shape::color(self.element, "color", "alpha")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn parse(source: &[u8]) -> craft_xml::El {
|
||||
craft_xml::parse(source).expect("well-formed test XML")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recognizes_filter_kinds_and_preserves_unknown_entries_and_order() {
|
||||
let root = parse(br##"<DOMSymbolInstance><filters><BlurFilter/><x:GlowFilter xmlns:x="urn:xfl" enabled="false"/><FutureFilter/><GradientBevelFilter/></filters></DOMSymbolInstance>"##);
|
||||
let filters: Vec<_> = FilterList::new(&root).iter().collect();
|
||||
assert_eq!(filters.len(), 4);
|
||||
assert_eq!(filters[0].kind(), Some(FilterKind::Blur));
|
||||
assert_eq!(filters[1].kind(), Some(FilterKind::Glow));
|
||||
assert!(!filters[1].enabled());
|
||||
assert_eq!(filters[2].kind(), None);
|
||||
assert_eq!(filters[3].kind(), Some(FilterKind::GradientBevel));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_native_filter_values_colors_and_gradient_entries_in_order() {
|
||||
let root = parse(
|
||||
br##"<DOMSymbolInstance><filters><DropShadowFilter blurX="7.5" blurY="8" strength="1.25" quality="9" angle="45" distance="4" color="#123456" alpha="0.5" inner="true" knockout="false" hideObject="true"/><BevelFilter type="full" shadowColor="#010203" shadowAlpha="0.25" highlightColor="#aabbcc" highlightAlpha="0.75"/><GradientGlowFilter><GradientEntry ratio="255" color="#ff0000"/><Ramp><GradientEntry ratio="64" color="#00ff00"/></Ramp></GradientGlowFilter></filters></DOMSymbolInstance>"##,
|
||||
);
|
||||
let filters: Vec<_> = FilterList::new(&root).iter().collect();
|
||||
let shadow = &filters[0];
|
||||
assert_eq!(shadow.quality(), 3);
|
||||
assert_eq!(
|
||||
(shadow.blur_x(), shadow.blur_y(), shadow.strength(), shadow.angle(), shadow.distance()),
|
||||
(Some(7.5), Some(8.0), Some(1.25), Some(45.0), Some(4.0))
|
||||
);
|
||||
assert_eq!(shadow.color(), Color { rgb: 0x123456, alpha: 0.5 });
|
||||
assert_eq!(shadow.inner(), Some(true));
|
||||
assert_eq!(shadow.knockout(), Some(false));
|
||||
assert_eq!(shadow.hide_object(), Some(true));
|
||||
|
||||
let bevel = &filters[1];
|
||||
assert_eq!(bevel.side(), Some(FilterSide::Full));
|
||||
assert_eq!(bevel.shadow_color(), Color { rgb: 0x010203, alpha: 0.25 });
|
||||
assert_eq!(bevel.highlight_color(), Some(Color { rgb: 0xaabbcc, alpha: 0.75 }));
|
||||
|
||||
let ramp: Vec<_> = filters[2].ramp_entries().collect();
|
||||
assert_eq!(ramp.iter().map(RampEntry::ratio).collect::<Vec<_>>(), [Some(255.0), Some(64.0)]);
|
||||
assert_eq!(ramp[0].color(), Color { rgb: 0xff0000, alpha: 1.0 });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_lists_are_empty_and_values_use_shared_wire_validation() {
|
||||
let root = parse(b"<DOMSymbolInstance/>");
|
||||
assert_eq!(FilterList::new(&root).iter().count(), 0);
|
||||
|
||||
let root = parse(br##"<BlurFilter quality="bad" blurX="NaN" enabled="unexpected"/>"##);
|
||||
let filter = FilterList::new(&root).iter().next();
|
||||
assert_eq!(filter.map(|filter| filter.kind()), None);
|
||||
let parent =
|
||||
parse(br##"<DOMSymbolInstance><filters><BlurFilter quality="bad" blurX="NaN" enabled="unexpected"/></filters></DOMSymbolInstance>"##);
|
||||
let filter = FilterList::new(&parent).iter().next().expect("filter entry");
|
||||
assert_eq!(filter.kind(), Some(FilterKind::Blur));
|
||||
assert_eq!(filter.quality(), 1);
|
||||
assert_eq!(filter.blur_x(), None);
|
||||
assert!(!filter.enabled());
|
||||
}
|
||||
}
|
||||
@@ -27,10 +27,20 @@ use std::collections::HashMap;
|
||||
pub mod document;
|
||||
/// XFL's edge-attribute grammar, independent of any consuming application's geometry types.
|
||||
pub mod edges;
|
||||
/// Borrowed views of native XFL filter records.
|
||||
pub mod filters;
|
||||
/// Borrowed views of native XFL media library records.
|
||||
pub mod media;
|
||||
/// Borrowed views of native XFL shape and style elements.
|
||||
pub mod shape;
|
||||
/// Borrowed views of native XFL shape-hint records.
|
||||
pub mod shape_hints;
|
||||
/// Borrowed views of native XFL frame-sound and envelope records.
|
||||
pub mod sound;
|
||||
/// Borrowed views of native XFL text-field and text-run schema.
|
||||
pub mod text;
|
||||
/// Borrowed views of native XFL tween and custom-ease records.
|
||||
pub mod tween;
|
||||
|
||||
/// Largest single entry inflated (bytes).
|
||||
pub const MAX_ENTRY: usize = 256 * 1024 * 1024;
|
||||
|
||||
+90
-5
@@ -5,6 +5,7 @@
|
||||
//! compatible borrowed element adapters implemented by consumers.
|
||||
|
||||
use crate::edges::{self, Segment};
|
||||
use std::borrow::Cow;
|
||||
|
||||
/// An element source used by the borrowed XFL schema views.
|
||||
///
|
||||
@@ -27,6 +28,9 @@ pub trait XflElement {
|
||||
/// Direct child elements in document order.
|
||||
fn elements(&self) -> Self::Elements<'_>;
|
||||
|
||||
/// This element's direct character data, with XML references resolved.
|
||||
fn text_content(&self) -> Cow<'_, str>;
|
||||
|
||||
/// The first direct child whose local name matches.
|
||||
fn child(&self, name: &str) -> Option<&Self> {
|
||||
self.elements().find(|element| element.local_name() == name)
|
||||
@@ -68,6 +72,10 @@ impl XflElement for craft_xml::El {
|
||||
fn elements(&self) -> Self::Elements<'_> {
|
||||
XmlElements { nodes: self.children.iter() }
|
||||
}
|
||||
|
||||
fn text_content(&self) -> Cow<'_, str> {
|
||||
self.text()
|
||||
}
|
||||
}
|
||||
|
||||
/// XFL matrix coefficients, in the native units stored in XML.
|
||||
@@ -88,28 +96,78 @@ pub struct Color {
|
||||
pub alpha: f32,
|
||||
}
|
||||
|
||||
/// A native XFL instance color record (`<color><Color .../></color>`).
|
||||
///
|
||||
/// The view parses the authored tint, brightness, channel multipliers and offsets but does not
|
||||
/// choose a consumer's color-effect style or clamp values into an application's model. Offsets
|
||||
/// remain in XFL's native 0–255 scale. Invalid or non-finite channel multipliers use 1, and
|
||||
/// invalid or non-finite offsets use 0, matching the historical AniCraft importer.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct NativeColorEffect<'a, E: XflElement> {
|
||||
element: &'a E,
|
||||
}
|
||||
|
||||
impl Matrix {
|
||||
/// The matrix used when an XFL element omits its transform.
|
||||
pub const IDENTITY: Matrix = Matrix { a: 1.0, b: 0.0, c: 0.0, d: 1.0, tx: 0.0, ty: 0.0 };
|
||||
}
|
||||
|
||||
fn number(element: &impl XflElement, name: &str) -> Option<f32> {
|
||||
impl<'a, E: XflElement> NativeColorEffect<'a, E> {
|
||||
/// Views a native XFL `Color` element without converting it to an application's model.
|
||||
pub fn new(element: &'a E) -> Self {
|
||||
Self { element }
|
||||
}
|
||||
|
||||
/// The native tint amount, when authored.
|
||||
pub fn tint_multiplier(&self) -> Option<f32> {
|
||||
number(self.element, "tintMultiplier")
|
||||
}
|
||||
|
||||
/// The native tint color. Missing or malformed color values use black as other XFL colors do.
|
||||
pub fn tint_color(&self) -> Color {
|
||||
color(self.element, "tintColor", "__none")
|
||||
}
|
||||
|
||||
/// The native brightness value, when authored.
|
||||
pub fn brightness(&self) -> Option<f32> {
|
||||
number(self.element, "brightness")
|
||||
}
|
||||
|
||||
/// Red, green, blue, and alpha multipliers, with identity defaults for absent or invalid data.
|
||||
pub fn multipliers(&self) -> [f32; 4] {
|
||||
["redMultiplier", "greenMultiplier", "blueMultiplier", "alphaMultiplier"].map(|name| number(self.element, name).unwrap_or(1.0))
|
||||
}
|
||||
|
||||
/// Red, green, blue, and alpha offsets in XFL's native 0–255 scale.
|
||||
pub fn offsets(&self) -> [f32; 4] {
|
||||
["redOffset", "greenOffset", "blueOffset", "alphaOffset"].map(|name| number(self.element, name).unwrap_or(0.0))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn number(element: &impl XflElement, name: &str) -> Option<f32> {
|
||||
element.attr(name)?.trim().parse::<f32>().ok().filter(|value| value.is_finite())
|
||||
}
|
||||
|
||||
fn integer(element: &impl XflElement, name: &str) -> Option<u32> {
|
||||
pub(crate) fn integer(element: &impl XflElement, name: &str) -> Option<u32> {
|
||||
element.attr(name)?.trim().parse::<u32>().ok()
|
||||
}
|
||||
|
||||
fn boolean(element: &impl XflElement, name: &str) -> Option<bool> {
|
||||
pub(crate) fn boolean(element: &impl XflElement, name: &str) -> Option<bool> {
|
||||
element.attr(name).map(|value| value.trim() == "true")
|
||||
}
|
||||
|
||||
/// Parses XFL's hexadecimal color and alpha attributes. Missing or malformed colors use black;
|
||||
/// missing, malformed, or non-finite alpha uses full opacity, matching the legacy importer.
|
||||
pub fn color(element: &impl XflElement, color_attribute: &str, alpha_attribute: &str) -> Color {
|
||||
let rgb = element.attr(color_attribute).and_then(|value| u32::from_str_radix(value.trim().trim_start_matches('#'), 16).ok()).unwrap_or(0);
|
||||
let alpha = number(element, alpha_attribute).unwrap_or(1.0).clamp(0.0, 1.0);
|
||||
color_from_attributes(element.attr(color_attribute), element.attr(alpha_attribute))
|
||||
}
|
||||
|
||||
/// Parses native color and alpha strings after a consumer has read them from its own element
|
||||
/// adapter. This keeps wire-value interpretation shared even when an application already has a
|
||||
/// borrowed schema view for the containing element.
|
||||
pub fn color_from_attributes(color: Option<&str>, alpha: Option<&str>) -> Color {
|
||||
let rgb = color.and_then(|value| u32::from_str_radix(value.trim().trim_start_matches('#'), 16).ok()).unwrap_or(0);
|
||||
let alpha = alpha.and_then(|value| value.trim().parse::<f32>().ok()).filter(|value| value.is_finite()).unwrap_or(1.0).clamp(0.0, 1.0);
|
||||
Color { rgb, alpha }
|
||||
}
|
||||
|
||||
@@ -534,6 +592,14 @@ mod tests {
|
||||
assert_eq!(fill.color_value(), Color { rgb: 0, alpha: 1.0 });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn color_attribute_values_keep_native_defaults_and_clamping() {
|
||||
assert_eq!(color_from_attributes(Some(" #123456 "), Some("0.4")), Color { rgb: 0x123456, alpha: 0.4 });
|
||||
assert_eq!(color_from_attributes(Some("invalid"), Some("NaN")), Color { rgb: 0, alpha: 1.0 });
|
||||
assert_eq!(color_from_attributes(Some("#abcdef"), Some("2")), Color { rgb: 0xabcdef, alpha: 1.0 });
|
||||
assert_eq!(color_from_attributes(None, Some("-1")), Color { rgb: 0, alpha: 0.0 });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn style_table_views_keep_the_legacy_first_child_and_index_behavior() {
|
||||
let root = parse(b"<DOMShape><fills><LegacyFill index='5'><SolidColor color='#123456'/></LegacyFill></fills><strokes><LegacyStroke index='6'><SolidStroke weight='2'/></LegacyStroke></strokes></DOMShape>");
|
||||
@@ -565,6 +631,9 @@ mod tests {
|
||||
fn elements(&self) -> Self::Elements<'_> {
|
||||
self.children.iter()
|
||||
}
|
||||
fn text_content(&self) -> Cow<'_, str> {
|
||||
Cow::Borrowed("")
|
||||
}
|
||||
}
|
||||
let app = AppElement { name: "x:DOMShape".into(), attrs: vec![("isDrawingObject".into(), "true".into())], children: Vec::new() };
|
||||
assert_eq!(Shape::new(&app).unwrap().is_drawing_object(), Some(true));
|
||||
@@ -587,4 +656,20 @@ mod tests {
|
||||
assert_eq!(Primitive::new(&oval).unwrap().kind(), PrimitiveKind::Oval);
|
||||
assert!(Primitive::new(&parse(b"<DOMShape/>")).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_color_effect_keeps_xfl_values_and_safe_channel_defaults() {
|
||||
let element = parse(
|
||||
br##"<x:Color xmlns:x="urn:xfl" tintMultiplier=".25" tintColor="#123456"
|
||||
brightness="-.4" alphaMultiplier=".5" redMultiplier="Infinity" greenMultiplier="-2"
|
||||
blueMultiplier=".5" redOffset="255" greenOffset="-255" blueOffset="NaN" alphaOffset=".25"/>"##,
|
||||
);
|
||||
let effect = NativeColorEffect::new(&element);
|
||||
|
||||
assert_eq!(effect.tint_multiplier(), Some(0.25));
|
||||
assert_eq!(effect.tint_color(), Color { rgb: 0x123456, alpha: 1.0 });
|
||||
assert_eq!(effect.brightness(), Some(-0.4));
|
||||
assert_eq!(effect.multipliers(), [1.0, -2.0, 0.5, 0.5]);
|
||||
assert_eq!(effect.offsets(), [255.0, -255.0, 0.0, 0.25]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
//! Borrowed views of native XFL shape-hint records.
|
||||
//!
|
||||
//! These parse the native point encodings and hint ordering. Snapping points onto consumer
|
||||
//! geometry and pairing start/end keyframes remain application responsibilities.
|
||||
|
||||
use crate::shape::{XflElement, integer, number};
|
||||
|
||||
/// A native XFL point in document coordinates.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Point {
|
||||
pub x: f32,
|
||||
pub y: f32,
|
||||
}
|
||||
|
||||
/// The optional `<shapeHints>` wrapper on a keyframe.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct ShapeHints<'a, E: XflElement> {
|
||||
wrapper: Option<&'a E>,
|
||||
}
|
||||
|
||||
impl<'a, E: XflElement> ShapeHints<'a, E> {
|
||||
pub fn new(keyframe: &'a E) -> Self {
|
||||
Self { wrapper: keyframe.child("shapeHints") }
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.wrapper.is_none_or(|wrapper| wrapper.elements().next().is_none())
|
||||
}
|
||||
|
||||
/// Records in source order; unrecognized child names are retained because the legacy reader
|
||||
/// interpreted their point and ordering attributes too.
|
||||
pub fn iter(&self) -> impl Iterator<Item = ShapeHint<'a, E>> + 'a {
|
||||
let wrapper = self.wrapper;
|
||||
wrapper.into_iter().flat_map(XflElement::elements).map(|element| ShapeHint { element })
|
||||
}
|
||||
}
|
||||
|
||||
/// One native shape-hint record.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct ShapeHint<'a, E: XflElement> {
|
||||
element: &'a E,
|
||||
}
|
||||
|
||||
impl<'a, E: XflElement> ShapeHint<'a, E> {
|
||||
/// Native numeric order, or its single-letter `name` fallback (`a` is zero).
|
||||
pub fn index(&self) -> Option<usize> {
|
||||
integer(self.element, "index").map(|value| value as usize).or_else(|| self.element.attr("name").and_then(letter_index))
|
||||
}
|
||||
|
||||
/// The keyframe's point, in either joined or separate native coordinate attributes.
|
||||
pub fn start_point(&self) -> Option<Point> {
|
||||
read_point(self.element, "startPoint", "startX", "startY").or_else(|| read_point(self.element, "point", "x", "y"))
|
||||
}
|
||||
|
||||
/// The next keyframe's point, when the record contains one.
|
||||
pub fn end_point(&self) -> Option<Point> {
|
||||
read_point(self.element, "endPoint", "endX", "endY")
|
||||
}
|
||||
}
|
||||
|
||||
fn letter_index(name: &str) -> Option<usize> {
|
||||
let [byte] = name.as_bytes() else { return None };
|
||||
match byte.to_ascii_lowercase() {
|
||||
b'a'..=b'z' => Some((byte.to_ascii_lowercase() - b'a') as usize),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn read_point(element: &impl XflElement, joined: &str, x: &str, y: &str) -> Option<Point> {
|
||||
let point = match element.attr(joined) {
|
||||
Some(value) => {
|
||||
let mut values = value.split([',', ' ']).filter(|part| !part.trim().is_empty()).map(|part| part.trim().parse::<f32>().ok());
|
||||
Point { x: values.next()??, y: values.next()?? }
|
||||
}
|
||||
None => Point { x: number(element, x)?, y: number(element, y)? },
|
||||
};
|
||||
(point.x.is_finite() && point.y.is_finite()).then_some(point)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn parse(source: &[u8]) -> craft_xml::El {
|
||||
craft_xml::parse(source).expect("well-formed test XML")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_joined_and_split_native_points_in_source_order() {
|
||||
let keyframe = parse(br#"<DOMFrame><shapeHints><Hint name="B" startPoint="12.5, -2" endX="30" endY="40"/><Hint index="0" startX="1" startY="2" endPoint="3 4"/></shapeHints></DOMFrame>"#);
|
||||
let hints = ShapeHints::new(&keyframe);
|
||||
assert!(!hints.is_empty());
|
||||
let records: Vec<_> = hints.iter().collect();
|
||||
assert_eq!(records[0].index(), Some(1));
|
||||
assert_eq!(records[0].start_point(), Some(Point { x: 12.5, y: -2.0 }));
|
||||
assert_eq!(records[0].end_point(), Some(Point { x: 30.0, y: 40.0 }));
|
||||
assert_eq!(records[1].index(), Some(0));
|
||||
assert_eq!(records[1].start_point(), Some(Point { x: 1.0, y: 2.0 }));
|
||||
assert_eq!(records[1].end_point(), Some(Point { x: 3.0, y: 4.0 }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_joined_values_fall_back_to_the_other_native_point_field() {
|
||||
let keyframe = parse(br#"<DOMFrame><shapeHints><Any name="z" startPoint="bad" x="7" y="8" endPoint="nan, 2"/></shapeHints></DOMFrame>"#);
|
||||
let hint = ShapeHints::new(&keyframe).iter().next().expect("hint record");
|
||||
assert_eq!(hint.index(), Some(25));
|
||||
assert_eq!(hint.start_point(), Some(Point { x: 7.0, y: 8.0 }));
|
||||
assert_eq!(hint.end_point(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_lists_and_non_letter_names_remain_optional() {
|
||||
let keyframe = parse(b"<DOMFrame/>");
|
||||
assert!(ShapeHints::new(&keyframe).is_empty());
|
||||
let keyframe = parse(br#"<DOMFrame><shapeHints><Hint name="AB" startX="1" startY="2"/></shapeHints></DOMFrame>"#);
|
||||
let hint = ShapeHints::new(&keyframe).iter().next().expect("hint record");
|
||||
assert_eq!(hint.index(), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
//! Borrowed views of native XFL frame-sound and envelope records.
|
||||
//!
|
||||
//! These parse the frame fields, effect-name spelling, and envelope wire values. Resolving sound
|
||||
//! library items and mapping effects/envelopes into a consumer's audio model remain application
|
||||
//! responsibilities.
|
||||
|
||||
use crate::shape::{XflElement, integer, number};
|
||||
|
||||
/// Native frame-sound synchronization mode. Unknown values use Animate's event behavior.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum SyncMode {
|
||||
Start,
|
||||
Stop,
|
||||
Stream,
|
||||
#[default]
|
||||
Event,
|
||||
}
|
||||
|
||||
/// A native sound instruction stored on a `DOMFrame`.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct FrameSound<'a, E: XflElement> {
|
||||
frame: &'a E,
|
||||
}
|
||||
|
||||
impl<'a, E: XflElement> FrameSound<'a, E> {
|
||||
pub fn new(frame: &'a E) -> Self {
|
||||
Self { frame }
|
||||
}
|
||||
|
||||
pub fn name(&self) -> Option<&'a str> {
|
||||
self.frame.attr("soundName")
|
||||
}
|
||||
|
||||
pub fn sync_mode(&self) -> SyncMode {
|
||||
match self.frame.attr("soundSync").map(str::trim) {
|
||||
Some("start") => SyncMode::Start,
|
||||
Some("stop") => SyncMode::Stop,
|
||||
Some("stream") => SyncMode::Stream,
|
||||
_ => SyncMode::Event,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn looping(&self) -> bool {
|
||||
self.frame.attr("soundLoopMode").map(str::trim) == Some("loop")
|
||||
}
|
||||
|
||||
pub fn repeat(&self) -> Option<u32> {
|
||||
integer(self.frame, "soundLoop")
|
||||
}
|
||||
|
||||
pub fn has_in_point(&self) -> bool {
|
||||
self.frame.attr("inPoint44").is_some()
|
||||
}
|
||||
|
||||
pub fn has_out_point(&self) -> bool {
|
||||
self.frame.attr("outPoint44").is_some()
|
||||
}
|
||||
|
||||
/// The effect menu label normalized to lowercase ASCII letters and digits, matching the
|
||||
/// legacy reader's recognition of labels such as `Fade Left to Right`.
|
||||
pub fn effect_key(&self) -> String {
|
||||
self.frame
|
||||
.attr("soundEffect")
|
||||
.unwrap_or("")
|
||||
.chars()
|
||||
.filter(char::is_ascii_alphanumeric)
|
||||
.map(|character| character.to_ascii_lowercase())
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn envelope_points(&self) -> impl Iterator<Item = EnvelopePoint<'a, E>> + 'a {
|
||||
self.frame
|
||||
.child("SoundEnvelope")
|
||||
.into_iter()
|
||||
.flat_map(|envelope| envelope.elements_named("SoundEnvelopePoint"))
|
||||
.map(|element| EnvelopePoint { element })
|
||||
}
|
||||
}
|
||||
|
||||
/// One native `SoundEnvelopePoint` record. Levels are raw SWF/XFL integer units.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct EnvelopePoint<'a, E: XflElement> {
|
||||
element: &'a E,
|
||||
}
|
||||
|
||||
impl<E: XflElement> EnvelopePoint<'_, E> {
|
||||
pub fn mark44(&self) -> Option<f32> {
|
||||
number(self.element, "mark44")
|
||||
}
|
||||
|
||||
pub fn level0(&self) -> Option<f32> {
|
||||
number(self.element, "level0")
|
||||
}
|
||||
|
||||
pub fn level1(&self) -> Option<f32> {
|
||||
number(self.element, "level1")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn parse(source: &[u8]) -> craft_xml::El {
|
||||
craft_xml::parse(source).expect("well-formed test XML")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_frame_sound_modes_repeat_trim_and_effect_name() {
|
||||
let frame = parse(br#"<x:DOMFrame xmlns:x="urn:xfl" soundName="Music" soundSync=" stream " soundLoopMode="loop" soundLoop="12" inPoint44="2" outPoint44="100" soundEffect="Fade Left to Right!"/>"#);
|
||||
let sound = FrameSound::new(&frame);
|
||||
assert_eq!(sound.name(), Some("Music"));
|
||||
assert_eq!(sound.sync_mode(), SyncMode::Stream);
|
||||
assert!(sound.looping());
|
||||
assert_eq!(sound.repeat(), Some(12));
|
||||
assert!(sound.has_in_point());
|
||||
assert!(sound.has_out_point());
|
||||
assert_eq!(sound.effect_key(), "fadelefttoright");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn defaults_and_envelope_records_keep_native_optional_values() {
|
||||
let frame = parse(br#"<DOMFrame soundSync="unknown" soundLoopMode="repeat" soundEffect="CuStOm"><SoundEnvelope><SoundEnvelopePoint mark44="44100" level0="32768"/><Other/><SoundEnvelopePoint mark44="bad" level0="0" level1="nan"/></SoundEnvelope></DOMFrame>"#);
|
||||
let sound = FrameSound::new(&frame);
|
||||
assert_eq!(sound.sync_mode(), SyncMode::Event);
|
||||
assert!(!sound.looping());
|
||||
assert_eq!(sound.repeat(), None);
|
||||
assert!(!sound.has_in_point());
|
||||
assert!(!sound.has_out_point());
|
||||
assert_eq!(sound.effect_key(), "custom");
|
||||
let points: Vec<_> = sound.envelope_points().collect();
|
||||
assert_eq!(points.len(), 2);
|
||||
assert_eq!((points[0].mark44(), points[0].level0(), points[0].level1()), (Some(44100.0), Some(32768.0), None));
|
||||
assert_eq!((points[1].mark44(), points[1].level0(), points[1].level1()), (None, Some(0.0), None));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
//! Borrowed views of native XFL text-field, run and style records.
|
||||
//!
|
||||
//! The views expose format attributes and direct text only. Text layout, font selection,
|
||||
//! unsupported-style reporting and conversion into an application's text model stay with the
|
||||
//! consumer.
|
||||
|
||||
use crate::shape::{self, Color, XflElement};
|
||||
use std::borrow::Cow;
|
||||
|
||||
/// A native static, dynamic or input text field.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct TextField<'a, E: XflElement> {
|
||||
element: &'a E,
|
||||
kind: TextFieldKind,
|
||||
}
|
||||
|
||||
/// Kind of native XFL text field.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum TextFieldKind {
|
||||
Static,
|
||||
Dynamic,
|
||||
Input,
|
||||
}
|
||||
|
||||
impl<'a, E: XflElement> TextField<'a, E> {
|
||||
/// Views a recognized XFL text field.
|
||||
pub fn new(element: &'a E) -> Option<Self> {
|
||||
let kind = match element.local_name() {
|
||||
"DOMStaticText" => TextFieldKind::Static,
|
||||
"DOMDynamicText" => TextFieldKind::Dynamic,
|
||||
"DOMInputText" => TextFieldKind::Input,
|
||||
_ => return None,
|
||||
};
|
||||
Some(Self { element, kind })
|
||||
}
|
||||
|
||||
pub fn kind(&self) -> TextFieldKind {
|
||||
self.kind
|
||||
}
|
||||
|
||||
/// Native field orientation (`horizontal`, `vertical`, etc.).
|
||||
pub fn orientation(&self) -> Option<&'a str> {
|
||||
self.element.attr("orientation")
|
||||
}
|
||||
|
||||
/// Native `autoExpand` flag, if present and parseable.
|
||||
pub fn auto_expand(&self) -> Option<bool> {
|
||||
boolean(self.element, "autoExpand")
|
||||
}
|
||||
|
||||
pub fn width(&self) -> Option<f32> {
|
||||
number(self.element, "width")
|
||||
}
|
||||
|
||||
pub fn height(&self) -> Option<f32> {
|
||||
number(self.element, "height")
|
||||
}
|
||||
|
||||
pub fn left(&self) -> Option<f32> {
|
||||
number(self.element, "left")
|
||||
}
|
||||
|
||||
pub fn top(&self) -> Option<f32> {
|
||||
number(self.element, "top")
|
||||
}
|
||||
|
||||
pub fn line_type(&self) -> Option<&'a str> {
|
||||
self.element.attr("lineType")
|
||||
}
|
||||
|
||||
pub fn border(&self) -> Option<bool> {
|
||||
boolean(self.element, "border")
|
||||
}
|
||||
|
||||
pub fn is_selectable(&self) -> Option<bool> {
|
||||
boolean(self.element, "isSelectable")
|
||||
}
|
||||
|
||||
pub fn render_as_html(&self) -> Option<bool> {
|
||||
boolean(self.element, "renderAsHTML")
|
||||
}
|
||||
|
||||
pub fn max_characters(&self) -> Option<f32> {
|
||||
number(self.element, "maxCharacters")
|
||||
}
|
||||
|
||||
pub fn variable_name(&self) -> Option<&'a str> {
|
||||
self.element.attr("variableName")
|
||||
}
|
||||
|
||||
/// Whether the native field has any filter records, which consumers may need to report as
|
||||
/// unsupported while keeping the field.
|
||||
pub fn has_filters(&self) -> bool {
|
||||
self.element.child("filters").is_some_and(|wrapper| wrapper.elements().next().is_some())
|
||||
}
|
||||
|
||||
/// Text runs in document order. Non-`DOMTextRun` children are ignored.
|
||||
pub fn runs(&self) -> impl Iterator<Item = TextRun<'a, E>> + 'a {
|
||||
self.element.child("textRuns").into_iter().flat_map(|wrapper| wrapper.elements_named("DOMTextRun")).map(|element| TextRun { element })
|
||||
}
|
||||
|
||||
/// Number of element records inside the native `<textRuns>` wrapper, including unknown
|
||||
/// records. This lets consumers retain their prior malformed-record and resource-limit
|
||||
/// policy while parsing recognized runs through [`Self::runs`].
|
||||
pub fn run_entries_count(&self) -> usize {
|
||||
self.element.child("textRuns").map_or(0, |wrapper| wrapper.elements().count())
|
||||
}
|
||||
}
|
||||
|
||||
/// One native text run and its optional style record.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct TextRun<'a, E: XflElement> {
|
||||
element: &'a E,
|
||||
}
|
||||
|
||||
impl<'a, E: XflElement> TextRun<'a, E> {
|
||||
/// Direct character data of this run's `<characters>` element.
|
||||
pub fn characters(&self) -> Cow<'a, str> {
|
||||
self.element.child("characters").map_or(Cow::Borrowed(""), XflElement::text_content)
|
||||
}
|
||||
|
||||
/// The native text attributes nested under `<textAttrs><DOMTextAttrs .../>`.
|
||||
pub fn style(&self) -> Option<TextStyle<'a, E>> {
|
||||
self.element.child("textAttrs").and_then(|wrapper| wrapper.child("DOMTextAttrs")).map(|element| TextStyle { element })
|
||||
}
|
||||
}
|
||||
|
||||
/// Native `DOMTextAttrs` fields. Values are parsed from the wire schema but not laid out or
|
||||
/// normalized for any application.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct TextStyle<'a, E: XflElement> {
|
||||
element: &'a E,
|
||||
}
|
||||
|
||||
impl<'a, E: XflElement> TextStyle<'a, E> {
|
||||
pub fn face(&self) -> Option<&'a str> {
|
||||
self.element.attr("face")
|
||||
}
|
||||
|
||||
pub fn size(&self) -> Option<f32> {
|
||||
number(self.element, "size")
|
||||
}
|
||||
|
||||
pub fn bold(&self) -> Option<bool> {
|
||||
boolean(self.element, "bold")
|
||||
}
|
||||
|
||||
pub fn italic(&self) -> Option<bool> {
|
||||
boolean(self.element, "italic")
|
||||
}
|
||||
|
||||
pub fn fill_color(&self) -> Option<&'a str> {
|
||||
self.element.attr("fillColor")
|
||||
}
|
||||
|
||||
/// Parsed native fill color and alpha, with XFL's black/full-opacity defaults.
|
||||
pub fn color(&self) -> Color {
|
||||
shape::color(self.element, "fillColor", "alpha")
|
||||
}
|
||||
|
||||
pub fn alpha(&self) -> Option<f32> {
|
||||
number(self.element, "alpha")
|
||||
}
|
||||
|
||||
pub fn alignment(&self) -> Option<&'a str> {
|
||||
self.element.attr("alignment")
|
||||
}
|
||||
|
||||
pub fn indent(&self) -> Option<f32> {
|
||||
number(self.element, "indent")
|
||||
}
|
||||
|
||||
pub fn character_position(&self) -> Option<&'a str> {
|
||||
self.element.attr("characterPosition")
|
||||
}
|
||||
|
||||
pub fn letter_spacing(&self) -> Option<f32> {
|
||||
number(self.element, "letterSpacing")
|
||||
}
|
||||
|
||||
pub fn line_spacing(&self) -> Option<f32> {
|
||||
number(self.element, "lineSpacing")
|
||||
}
|
||||
|
||||
pub fn left_margin(&self) -> Option<f32> {
|
||||
number(self.element, "leftMargin")
|
||||
}
|
||||
|
||||
pub fn right_margin(&self) -> Option<f32> {
|
||||
number(self.element, "rightMargin")
|
||||
}
|
||||
}
|
||||
|
||||
fn number(element: &impl XflElement, name: &str) -> Option<f32> {
|
||||
element.attr(name)?.trim().parse::<f32>().ok().filter(|value| value.is_finite())
|
||||
}
|
||||
|
||||
fn boolean(element: &impl XflElement, name: &str) -> Option<bool> {
|
||||
element.attr(name).map(|value| value.trim() == "true")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn parse(source: &[u8]) -> craft_xml::El {
|
||||
craft_xml::parse(source).expect("well-formed test XML")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_prefixed_text_fields_runs_and_native_style_values() {
|
||||
let root = parse(
|
||||
br##"<x:DOMDynamicText xmlns:x="urn:xfl" orientation="horizontal" autoExpand="true" width="320" height="40" left="12" top="3" lineType="multiline" border="true" isSelectable="false" renderAsHTML="false" maxCharacters="45" variableName="message"><x:filters><x:BlurFilter/></x:filters><x:textRuns><x:DOMTextRun><x:characters>one & two</x:characters><x:textAttrs><x:DOMTextAttrs face="Arial" size="24" bold="true" italic="false" fillColor="#123456" alpha="0.5" alignment="center" indent="3" characterPosition="superscript" letterSpacing="1.25" lineSpacing="2" leftMargin="4" rightMargin="5"/></x:textAttrs></x:DOMTextRun><x:Other/><x:DOMTextRun><x:characters><![CDATA[third]]></x:characters></x:DOMTextRun></x:textRuns></x:DOMDynamicText>"##,
|
||||
);
|
||||
let field = TextField::new(&root).expect("dynamic text field");
|
||||
assert_eq!(field.kind(), TextFieldKind::Dynamic);
|
||||
assert_eq!(field.orientation(), Some("horizontal"));
|
||||
assert_eq!(field.auto_expand(), Some(true));
|
||||
assert_eq!((field.width(), field.height(), field.left(), field.top()), (Some(320.0), Some(40.0), Some(12.0), Some(3.0)));
|
||||
assert_eq!(field.line_type(), Some("multiline"));
|
||||
assert_eq!(field.border(), Some(true));
|
||||
assert_eq!(field.is_selectable(), Some(false));
|
||||
assert_eq!(field.render_as_html(), Some(false));
|
||||
assert_eq!(field.max_characters(), Some(45.0));
|
||||
assert_eq!(field.variable_name(), Some("message"));
|
||||
assert!(field.has_filters());
|
||||
|
||||
let mut runs = field.runs();
|
||||
let first = runs.next().expect("first run");
|
||||
assert_eq!(first.characters(), "one & two");
|
||||
let style = first.style().expect("text style");
|
||||
assert_eq!(style.face(), Some("Arial"));
|
||||
assert_eq!(style.size(), Some(24.0));
|
||||
assert_eq!(style.bold(), Some(true));
|
||||
assert_eq!(style.italic(), Some(false));
|
||||
assert_eq!(style.fill_color(), Some("#123456"));
|
||||
assert_eq!(style.alpha(), Some(0.5));
|
||||
assert_eq!(style.alignment(), Some("center"));
|
||||
assert_eq!(style.indent(), Some(3.0));
|
||||
assert_eq!(style.character_position(), Some("superscript"));
|
||||
assert_eq!(style.letter_spacing(), Some(1.25));
|
||||
assert_eq!(style.line_spacing(), Some(2.0));
|
||||
assert_eq!(style.left_margin(), Some(4.0));
|
||||
assert_eq!(style.right_margin(), Some(5.0));
|
||||
|
||||
let second = runs.next().expect("second run");
|
||||
assert_eq!(second.characters(), "third");
|
||||
assert!(second.style().is_none());
|
||||
assert!(runs.next().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recognizes_all_native_field_kinds_and_ignores_non_text_elements() {
|
||||
for (name, expected) in
|
||||
[("DOMStaticText", TextFieldKind::Static), ("DOMDynamicText", TextFieldKind::Dynamic), ("DOMInputText", TextFieldKind::Input)]
|
||||
{
|
||||
let xml = format!("<{name}/>");
|
||||
let root = craft_xml::parse(xml.as_bytes()).expect("test XML");
|
||||
assert_eq!(TextField::new(&root).map(|field| field.kind()), Some(expected));
|
||||
}
|
||||
let root = parse(b"<DOMShape/>");
|
||||
assert!(TextField::new(&root).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_and_non_finite_numbers_remain_absent() {
|
||||
let root = parse(b"<DOMStaticText><textRuns><DOMTextRun><textAttrs><DOMTextAttrs size='NaN' letterSpacing='bad'/></textAttrs></DOMTextRun></textRuns></DOMStaticText>");
|
||||
let field = TextField::new(&root).expect("text field");
|
||||
let style = field.runs().next().and_then(|run| run.style()).expect("style");
|
||||
assert_eq!(style.size(), None);
|
||||
assert_eq!(style.letter_spacing(), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
//! Borrowed views of native XFL frame-tween and custom-ease records.
|
||||
//!
|
||||
//! This parses the native attributes and point lists. Interpreting easing curves and applying
|
||||
//! them to an application's timeline model remain consumer responsibilities.
|
||||
|
||||
use crate::shape::{XflElement, boolean, number};
|
||||
|
||||
/// Native motion/shape tween settings on a `DOMFrame`.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct FrameTween<'a, E: XflElement> {
|
||||
frame: &'a E,
|
||||
}
|
||||
|
||||
impl<'a, E: XflElement> FrameTween<'a, E> {
|
||||
pub fn new(frame: &'a E) -> Self {
|
||||
Self { frame }
|
||||
}
|
||||
|
||||
/// The classic slider value, when present and finite.
|
||||
pub fn acceleration(&self) -> Option<f32> {
|
||||
number(self.frame, "acceleration")
|
||||
}
|
||||
|
||||
pub fn has_custom_ease(&self) -> Option<bool> {
|
||||
boolean(self.frame, "hasCustomEase")
|
||||
}
|
||||
|
||||
/// Animate's default is a single curve unless this is explicitly false.
|
||||
pub fn use_single_ease_curve(&self) -> Option<bool> {
|
||||
boolean(self.frame, "useSingleEaseCurve")
|
||||
}
|
||||
|
||||
pub fn orient_to_path(&self) -> Option<bool> {
|
||||
boolean(self.frame, "motionTweenOrientToPath")
|
||||
}
|
||||
|
||||
/// Raw native rotation setting; interpretation stays in the consuming application.
|
||||
pub fn rotate(&self) -> Option<&'a str> {
|
||||
self.frame.attr("motionTweenRotate")
|
||||
}
|
||||
|
||||
/// Custom ease curves in source order; unknown tween entries are skipped.
|
||||
pub fn custom_eases(&self) -> impl Iterator<Item = CustomEase<'a, E>> + 'a {
|
||||
self.frame.child("tweens").into_iter().flat_map(|wrapper| wrapper.elements_named("CustomEase")).map(|element| CustomEase { element })
|
||||
}
|
||||
}
|
||||
|
||||
/// A native custom-ease curve.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct CustomEase<'a, E: XflElement> {
|
||||
element: &'a E,
|
||||
}
|
||||
|
||||
impl<'a, E: XflElement> CustomEase<'a, E> {
|
||||
/// The property-group name, defaulting to `all` in XFL.
|
||||
pub fn target(&self) -> &'a str {
|
||||
self.element.attr("target").map(str::trim).unwrap_or("all")
|
||||
}
|
||||
|
||||
pub fn points(&self) -> impl Iterator<Item = EasePoint<'a, E>> + 'a {
|
||||
self.element.elements_named("Point").map(|element| EasePoint { element })
|
||||
}
|
||||
}
|
||||
|
||||
/// One native custom-ease point.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct EasePoint<'a, E: XflElement> {
|
||||
element: &'a E,
|
||||
}
|
||||
|
||||
impl<E: XflElement> EasePoint<'_, E> {
|
||||
pub fn x(&self) -> Option<f32> {
|
||||
number(self.element, "x")
|
||||
}
|
||||
|
||||
pub fn y(&self) -> Option<f32> {
|
||||
number(self.element, "y")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn parse(source: &[u8]) -> craft_xml::El {
|
||||
craft_xml::parse(source).expect("well-formed test XML")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_frame_tween_settings_and_custom_eases_in_order() {
|
||||
let frame = parse(
|
||||
br#"<x:DOMFrame xmlns:x="urn:xfl" acceleration="-45" hasCustomEase="true" useSingleEaseCurve="false" motionTweenOrientToPath="true" motionTweenRotate="clockwise"><x:tweens><x:Other/><x:CustomEase target="position"><x:Point x="0" y="0"/><x:Point x="0.5" y="0.25"/></x:CustomEase><x:CustomEase><x:Point x="1" y="1"/></x:CustomEase></x:tweens></x:DOMFrame>"#,
|
||||
);
|
||||
let tween = FrameTween::new(&frame);
|
||||
assert_eq!(tween.acceleration(), Some(-45.0));
|
||||
assert_eq!(tween.has_custom_ease(), Some(true));
|
||||
assert_eq!(tween.use_single_ease_curve(), Some(false));
|
||||
assert_eq!(tween.orient_to_path(), Some(true));
|
||||
assert_eq!(tween.rotate(), Some("clockwise"));
|
||||
let curves: Vec<_> = tween.custom_eases().collect();
|
||||
assert_eq!(curves.len(), 2);
|
||||
assert_eq!(curves[0].target(), "position");
|
||||
assert_eq!(curves[0].points().map(|point| (point.x(), point.y())).collect::<Vec<_>>(), [(Some(0.0), Some(0.0)), (Some(0.5), Some(0.25))]);
|
||||
assert_eq!(curves[1].target(), "all");
|
||||
assert_eq!(curves[1].points().next().map(|point| (point.x(), point.y())), Some((Some(1.0), Some(1.0))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_and_malformed_values_remain_optional() {
|
||||
let frame = parse(br#"<DOMFrame acceleration="NaN" hasCustomEase="bad" useSingleEaseCurve="false"/>"#);
|
||||
let tween = FrameTween::new(&frame);
|
||||
assert_eq!(tween.acceleration(), None);
|
||||
assert_eq!(tween.has_custom_ease(), Some(false));
|
||||
assert_eq!(tween.use_single_ease_curve(), Some(false));
|
||||
assert!(tween.custom_eases().next().is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check the Workspace workflow's pull-request-only cancellation policy.
|
||||
|
||||
The policy is the top-level `concurrency` block of .github/workflows/workspace.yml. This script
|
||||
reads those fields, requires the documented expression shapes, and checks which group each event
|
||||
would be given. The rest of the workflow is left to diff review, not frozen here.
|
||||
|
||||
Use --workflow PATH to read a baseline or a deliberately wrong variant without touching this
|
||||
checkout.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try: # cross-check only; the reader below is what a plain `python3` run needs
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
DEFAULT_WORKFLOW = Path(__file__).resolve().parent.parent / ".github" / "workflows" / "workspace.yml"
|
||||
PULL_REQUEST, PUSH, DISPATCH = "pull_request", "push", "workflow_dispatch"
|
||||
|
||||
# The two shapes this policy may use. GitHub reads `a && b || c` as "b, or c when a is false"; these
|
||||
# patterns are the whole of that expression language this script understands, so a reworded policy
|
||||
# has to be reworded here too, on purpose. github.ref, never github.head_ref or github.sha.
|
||||
GROUP_SHAPE = re.compile(
|
||||
r"(?P<prefix>[A-Za-z0-9._-]+)-\$\{\{\s*github\.event_name\s*==\s*'(?P<event>[a-z_]+)'\s*"
|
||||
r"&&\s*github\.ref\s*\|\|\s*github\.run_id\s*\}\}"
|
||||
)
|
||||
CANCEL_SHAPE = re.compile(r"\$\{\{\s*github\.event_name\s*==\s*'(?P<event>[a-z_]+)'\s*\}\}")
|
||||
|
||||
|
||||
def read_concurrency(text):
|
||||
"""The top-level `concurrency` mapping. A deliberately limited reader: two-space fields."""
|
||||
lines = text.splitlines()
|
||||
starts = [index for index, line in enumerate(lines) if line.startswith("concurrency:")]
|
||||
if len(starts) != 1:
|
||||
raise AssertionError(f"expected one top-level concurrency block, found {len(starts)}")
|
||||
header = lines[starts[0]].partition("#")[0].strip()
|
||||
if header != "concurrency:":
|
||||
# `concurrency: |` and `concurrency: some-string` are scalars, not a mapping. Reading the
|
||||
# lines under one as fields would let a workflow that is not this policy pass every check.
|
||||
raise AssertionError(f"concurrency must be a mapping, not a scalar: {lines[starts[0]]!r}")
|
||||
fields = {}
|
||||
for line in lines[starts[0] + 1:]:
|
||||
stripped = line.strip()
|
||||
if stripped and not line.startswith(" "):
|
||||
break # the next top-level key
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
if not line.startswith(" ") or line.startswith(" "):
|
||||
raise AssertionError(f"unexpected indentation under concurrency: {line!r}")
|
||||
key, separator, value = stripped.partition(":")
|
||||
if not separator:
|
||||
raise AssertionError(f"not a field of concurrency: {line!r}")
|
||||
fields[key.strip()] = value.strip()
|
||||
return fields
|
||||
|
||||
|
||||
def pull_request(number, run_id):
|
||||
# github.ref for a pull request is refs/pull/<n>/merge: the PR number, never the branch name.
|
||||
return (PULL_REQUEST, f"refs/pull/{number}/merge", run_id)
|
||||
|
||||
|
||||
def other_run(event_name, ref, run_id):
|
||||
return (event_name, ref, run_id)
|
||||
|
||||
|
||||
MAIN, TAG = ("refs/heads/main", "refs/tags/v1.2.3")
|
||||
SAME_GROUP = [
|
||||
("a newer commit on the same pull request", pull_request(1316, 43599251877), pull_request(1316, 43600899878)),
|
||||
("a rerun of one main run", other_run(PUSH, MAIN, 7001), other_run(PUSH, MAIN, 7001)),
|
||||
]
|
||||
DIFFERENT_GROUPS = [
|
||||
("two pull requests", pull_request(1316, 1), pull_request(1317, 1)),
|
||||
("the same branch name in two forks", pull_request(1316, 2), pull_request(1400, 2)),
|
||||
("two pushes to main", other_run(PUSH, MAIN, 3), other_run(PUSH, MAIN, 4)),
|
||||
("two release tag runs", other_run(PUSH, TAG, 5), other_run(PUSH, TAG, 6)),
|
||||
("two manual runs", other_run(DISPATCH, MAIN, 7), other_run(DISPATCH, MAIN, 8)),
|
||||
("a main push and a pull request", other_run(PUSH, MAIN, 9), pull_request(1316, 9)),
|
||||
("a release tag and a pull request", other_run(PUSH, TAG, 10), pull_request(1316, 10)),
|
||||
]
|
||||
CANCELLATION = [
|
||||
("a push supersedes the previous run of its pull request", pull_request(1316, 11), True),
|
||||
("a push to main is never cancelled", other_run(PUSH, MAIN, 12), False),
|
||||
("a release tag run is never cancelled", other_run(PUSH, TAG, 13), False),
|
||||
("a manual run is never cancelled", other_run(DISPATCH, MAIN, 14), False),
|
||||
]
|
||||
|
||||
|
||||
class ConcurrencyTests(unittest.TestCase):
|
||||
workflow = DEFAULT_WORKFLOW
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.text = cls.workflow.read_text()
|
||||
cls.fields = read_concurrency(cls.text)
|
||||
cls.group_shape = GROUP_SHAPE.fullmatch(cls.fields.get("group", ""))
|
||||
cls.cancel_shape = CANCEL_SHAPE.fullmatch(cls.fields.get("cancel-in-progress", ""))
|
||||
if not cls.group_shape or not cls.cancel_shape:
|
||||
raise AssertionError(f"the concurrency fields are not the documented shapes: {cls.fields}")
|
||||
|
||||
def cancels(self, event_name):
|
||||
return event_name == self.cancel_shape["event"]
|
||||
|
||||
def group(self, event_name, ref, run_id):
|
||||
return self.group_shape["prefix"] + "-" + (ref if event_name == self.group_shape["event"] else str(run_id))
|
||||
|
||||
def test_policy_is_the_two_documented_fields(self):
|
||||
self.assertEqual(sorted(self.fields), ["cancel-in-progress", "group"])
|
||||
self.assertEqual(self.group_shape["prefix"], "workspace")
|
||||
self.assertEqual(self.group_shape["event"], PULL_REQUEST)
|
||||
self.assertEqual(self.cancel_shape["event"], PULL_REQUEST)
|
||||
|
||||
def test_a_scalar_concurrency_value_is_not_read_as_a_mapping(self):
|
||||
cases = {
|
||||
"a block scalar": "concurrency: |\n group: workspace-x\n cancel-in-progress: true\n",
|
||||
"a plain scalar": "concurrency: workspace-x\njobs: {}\n",
|
||||
}
|
||||
for description, text in cases.items():
|
||||
with self.subTest(description):
|
||||
with self.assertRaises(AssertionError):
|
||||
read_concurrency(text)
|
||||
|
||||
def test_one_group_per_pull_request_and_run(self):
|
||||
for description, first, second in SAME_GROUP:
|
||||
with self.subTest(description):
|
||||
self.assertEqual(self.group(*first), self.group(*second), description)
|
||||
for description, first, second in DIFFERENT_GROUPS:
|
||||
with self.subTest(description):
|
||||
self.assertNotEqual(self.group(*first), self.group(*second), description)
|
||||
|
||||
def test_only_a_pull_request_cancels(self):
|
||||
for description, context, expected in CANCELLATION:
|
||||
with self.subTest(description):
|
||||
self.assertEqual(self.cancels(context[0]), expected, description)
|
||||
|
||||
def test_pyyaml_reads_the_same_fields(self):
|
||||
if yaml is None:
|
||||
self.skipTest("PyYAML is not installed; the limited reader above ran alone")
|
||||
document = yaml.load(self.text, Loader=yaml.BaseLoader) # BaseLoader keeps `on:` a string
|
||||
self.assertEqual(document["concurrency"], self.fields)
|
||||
|
||||
def test_no_per_job_concurrency_is_added(self):
|
||||
indented = [line for line in self.text.splitlines()
|
||||
if line.strip().startswith("concurrency:") and not line.startswith("concurrency:")]
|
||||
self.assertEqual(indented, [], "the policy belongs to the workflow, not to one job")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--workflow", type=Path, default=DEFAULT_WORKFLOW, help="workflow to read (default: %(default)s)")
|
||||
arguments, remaining = parser.parse_known_args()
|
||||
ConcurrencyTests.workflow = arguments.workflow
|
||||
unittest.main(argv=[sys.argv[0], *remaining])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -7,7 +7,11 @@ sonar.organization=storytold
|
||||
# *_tests.rs), which includes the browser fault-injection fixtures in crates/platform/tests.
|
||||
sonar.sources=.
|
||||
sonar.tests=.
|
||||
sonar.test.inclusions=**/tests/**,**/tests.rs,**/*_tests.rs
|
||||
# scripts/test-workspace-concurrency.py is a unit-test module for the Workspace workflow's
|
||||
# cancellation policy: its unittest cases read a workflow and assert on it, so it is test
|
||||
# code, analysed as a test and still linted, indexed and reported, not counted as production
|
||||
# code that needs coverage. Its name is listed exactly, so no other script is classified.
|
||||
sonar.test.inclusions=**/tests/**,**/tests.rs,**/*_tests.rs,scripts/test-workspace-concurrency.py
|
||||
|
||||
# Codec constant tables are transcribed from the format specifications; their repeated rows are
|
||||
# data, not copy-pasted logic.
|
||||
|
||||
Reference in new issue
Block a user