diff --git a/Cargo.lock b/Cargo.lock index 794e35b1ff..334bedc2fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4827,6 +4827,7 @@ dependencies = [ "photocraft-engine", "photocraft-format", "photocraft-io", + "photocraft-raster", "photocraft-raw", "rmcp", "rustix 1.1.5", diff --git a/crates/automation/Cargo.toml b/crates/automation/Cargo.toml index 5421640a84..d4e7d14e73 100644 --- a/crates/automation/Cargo.toml +++ b/crates/automation/Cargo.toml @@ -11,6 +11,7 @@ workspace = true [dependencies] photocraft-doc = { workspace = true } +photocraft-raster = { workspace = true } photocraft-compose = { workspace = true } photocraft-codecs = { workspace = true } photocraft-io = { workspace = true } diff --git a/crates/automation/src/files.rs b/crates/automation/src/files.rs index fc4526eedb..0b8ff103aa 100644 --- a/crates/automation/src/files.rs +++ b/crates/automation/src/files.rs @@ -33,9 +33,23 @@ pub fn open(path: &Path) -> Result { let document = photocraft_format::load_path(path)?; return Ok(Opened { warnings: photocraft_io::open_warnings(&document), document, source_read_only: false }); } - let bytes = photocraft_format::read_file(path).map_err(|e| AutomationError::Io(format!("{}: {e}", path.display())))?; let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); - open_bytes(&name, &bytes) + // A PSD/PSB is parsed from the open file (channel data read as it is decoded), so a CLI + // convert or info of a 2 GB file does not hold the file in memory next to the document. + #[cfg(any(unix, windows))] + { + let r = + photocraft_io::import_file(&name, path, &photocraft_io::ImportOptions::default(), &photocraft_raster::Interrupt::NONE).map_err(|e| match e { + photocraft_io::IoError::Read(msg) => AutomationError::Io(msg), + e => e.into(), + })?; + Ok(Opened { document: r.document, warnings: r.warnings, source_read_only: r.source_read_only }) + } + #[cfg(not(any(unix, windows)))] + { + let bytes = photocraft_format::read_file(path).map_err(|e| AutomationError::Io(format!("{}: {e}", path.display())))?; + open_bytes(&name, &bytes) + } } /// Previews for a `.pcraft` bundle. diff --git a/crates/engine/src/jobs.rs b/crates/engine/src/jobs.rs index 5c26e41cba..d106fc0faf 100644 --- a/crates/engine/src/jobs.rs +++ b/crates/engine/src/jobs.rs @@ -330,14 +330,25 @@ fn independent_of_document(id: &str) -> bool { } #[cfg(not(target_arch = "wasm32"))] -fn read_path(path: &str) -> Result> { - // Bounded reads, and a clear error for a file larger than memory (#375). - photocraft_format::read_file(std::path::Path::new(path)).map_err(|e| EngineError::Other(format!("{path}: {e}"))) +fn import_path( + name: &str, + path: &str, + opts: &photocraft_io::ImportOptions, + ctl: &photocraft_raster::Interrupt, +) -> std::result::Result<(photocraft_io::ImportResult, bool), photocraft_io::IoError> { + // Bounded reads, and a clear error for a file larger than memory (#375); a PSD/PSB is + // parsed from the open file instead of being read whole. + photocraft_io::import_file_for_open_with(name, std::path::Path::new(path), opts, ctl) } #[cfg(target_arch = "wasm32")] -fn read_path(path: &str) -> Result> { - Err(EngineError::Other(format!("cannot open paths on the web: {path}"))) +fn import_path( + _: &str, + path: &str, + _: &photocraft_io::ImportOptions, + _: &photocraft_raster::Interrupt, +) -> std::result::Result<(photocraft_io::ImportResult, bool), photocraft_io::IoError> { + Err(photocraft_io::IoError::Read(format!("cannot open paths on the web: {path}"))) } /// Run a job-capable command's heavy part. `work` runs on a worker thread when the command was @@ -553,17 +564,19 @@ impl Session { false, move |ctx| { ctx.progress(0.0, "Reading"); - let bytes = match source { - OpenSource::Bytes(b) => b, - OpenSource::Path(p) => Arc::new(read_path(&p)?), - }; - ctx.check()?; - ctx.progress(0.02, "Decoding"); - let imported = - ctx.stage(0.02, 0.95, "Decoding", |ctl| photocraft_io::import_for_open_with(&name_w, &bytes, &opts, ctl)).map_err(|e| match e { - photocraft_io::IoError::Cancelled => EngineError::Cancelled, - e => EngineError::Other(e.to_string()), - })?; + let imported = match source { + OpenSource::Bytes(bytes) => { + ctx.check()?; + ctx.progress(0.02, "Decoding"); + ctx.stage(0.02, 0.95, "Decoding", |ctl| photocraft_io::import_for_open_with(&name_w, &bytes, &opts, ctl)) + } + // A PSD/PSB is decoded straight from the file, which is never read whole. + OpenSource::Path(p) => ctx.stage(0.0, 0.95, "Decoding", |ctl| import_path(&name_w, &p, &opts, ctl)), + } + .map_err(|e| match e { + photocraft_io::IoError::Cancelled => EngineError::Cancelled, + e => EngineError::Other(e.to_string()), + })?; ctx.progress(0.95, "Preparing layers"); warm_layer_bounds(&imported.0.document, || ctx.check())?; // Type layers need the shared text engine, whose first use waits for the diff --git a/crates/io/examples/psd_open_mem.rs b/crates/io/examples/psd_open_mem.rs new file mode 100644 index 0000000000..903eb35af9 --- /dev/null +++ b/crates/io/examples/psd_open_mem.rs @@ -0,0 +1,56 @@ +//! Opens a file the way the app's background open does and reports the time. Run it under +//! `/usr/bin/time -l` for the peak memory. `--read` reads the whole file first and imports the +//! bytes, as opens did before `import_file` (to compare). +//! +//! `cargo run --release -p photocraft-io --example psd_open_mem -- [--read]` + +fn main() { + let Some(path) = std::env::args().nth(1).filter(|a| a != "--read") else { + eprintln!("usage: psd_open_mem "); + std::process::exit(2); + }; + if std::env::args().any(|a| a == "--read") { + read_then_import(&path); + return; + } + let t = std::time::Instant::now(); + let res = + match photocraft_io::import_file(&path, std::path::Path::new(&path), &photocraft_io::ImportOptions::default(), &photocraft_raster::Interrupt::NONE) { + Ok(r) => r, + Err(e) => { + eprintln!("{path}: open failed: {e}"); + std::process::exit(1); + } + }; + report(&path, &format!("open {:.2}s", t.elapsed().as_secs_f64()), &res); +} + +fn read_then_import(path: &str) { + let t = std::time::Instant::now(); + let bytes = match photocraft_format::read_file(std::path::Path::new(path)) { + Ok(b) => b, + Err(e) => { + eprintln!("{path}: read failed: {e}"); + std::process::exit(1); + } + }; + let read_s = t.elapsed().as_secs_f64(); + let t = std::time::Instant::now(); + let res = match photocraft_io::import_with(path, &bytes, &photocraft_raster::Interrupt::NONE) { + Ok(r) => r, + Err(e) => { + eprintln!("{path}: open failed: {e}"); + std::process::exit(1); + } + }; + drop(bytes); + report(path, &format!("read {read_s:.2}s, import {:.2}s", t.elapsed().as_secs_f64()), &res); +} + +fn report(path: &str, times: &str, res: &photocraft_io::ImportResult) { + let doc = &res.document; + println!("{path}: {times}, {} layers ({} top level)", doc.walk().len(), doc.layers.len()); + for w in &res.warnings { + println!(" warning: {w}"); + } +} diff --git a/crates/io/src/lib.rs b/crates/io/src/lib.rs index 44beb59d47..b666c47d02 100644 --- a/crates/io/src/lib.rs +++ b/crates/io/src/lib.rs @@ -125,6 +125,9 @@ pub enum IoError { /// An SVG that does not parse (or is too large to rasterise). #[error("SVG: {0}")] Svg(String), + /// The file could not be read. + #[error("{0}")] + Read(String), /// A background import or save was cancelled ([`import_with`], [`export_with`]). #[error("cancelled")] Cancelled, @@ -373,26 +376,82 @@ pub fn import_for_open_with(name: &str, bytes: &[u8], opts: &ImportOptions, ctl: Ok((result, damaged)) } -fn import_psd(name: &str, bytes: &[u8], ctl: &photocraft_raster::Interrupt, damaged: &mut bool) -> Result { +/// The PSD/PSB part of an import, from a parse that takes [`PsdFile::from_bytes_and_merged_recovering`]'s +/// `want` and `recover`: of the file's bytes, or of the file itself ([`import_file_for_open_with`]). +fn import_psd( + name: &str, + ctl: &photocraft_raster::Interrupt, + damaged: &mut bool, + parse: impl FnOnce(&dyn Fn(&PsdFile) -> bool, &dyn Fn(&PsdFile) -> bool) -> photocraft_psd::Result<(PsdFile, Option>>)>, +) -> Result { // A merged image the import reads is decoded with the parse: a ZIP one is inflated once. - let (file, merged) = PsdFile::from_bytes_and_merged_recovering( - bytes, - |f| psd_import::check_group_depth(f).is_ok() && psd_import::merged_needed(f), - psd_import::can_recover_layers, - )?; + // A failed read of the file (one parsed from its source) is a read error, as reading the + // whole file first would report it. + let (file, merged) = + parse(&|f| psd_import::check_group_depth(f).is_ok() && psd_import::merged_needed(f), &psd_import::can_recover_layers).map_err(|e| match e { + photocraft_psd::PsdError::Read(msg) => IoError::Read(msg), + e => e.into(), + })?; // Nesting past the document model's cap could never be saved; reject the file with the // actionable limit. psd_import::check_group_depth(&file)?; ctl.check().map_err(|_| IoError::Cancelled)?; ctl.progress(0.05); - let (mut document, warnings, psd_damaged) = psd_import::psd_to_document_decoded_integrity(&file, ctl, merged).ok_or(IoError::Cancelled)?; + // Read before the conversion, which moves the global blocks into the document. + let txt2 = file.global_blocks.iter().find(|b| &b.key == b"Txt2").and_then(|b| photocraft_text::psd::parse_txt2(&b.data)); + let (mut document, warnings, psd_damaged, read_error) = psd_import::psd_to_document_owned(file, ctl, merged).ok_or(IoError::Cancelled)?; + // The file became unreadable while its layers were read from it (`import_file_for_open_with`): + // the open fails, rather than opening with those layers empty. + if let Some(e) = read_error { + return Err(IoError::Read(e)); + } *damaged = psd_damaged; document.name = name.to_string(); - let txt2 = file.global_blocks.iter().find(|b| &b.key == b"Txt2").and_then(|b| photocraft_text::psd::parse_txt2(&b.data)); text_import::prepare(&mut document, txt2.as_ref()); Ok(ImportResult { document, warnings, source_read_only: false, preview_only: false }) } +/// [`import_for_open_with`] for the file at `path`. A PSD/PSB is parsed from the open file +/// ([`PsdFile::from_source`]) rather than read whole: its channel data is read layer by layer as +/// it is decoded, so opening it needs memory for the document, not for the document plus the +/// file. A read that fails meanwhile (the file is truncated or rewritten, its drive goes away) +/// fails the open. Other formats are read whole. +#[cfg(any(unix, windows))] +pub fn import_file_for_open_with( + name: &str, + path: &std::path::Path, + opts: &ImportOptions, + ctl: &photocraft_raster::Interrupt, +) -> Result<(ImportResult, bool), IoError> { + let read_failed = |e: std::io::Error| IoError::Read(format!("{}: {e}", path.display())); + ctl.check().map_err(|_| IoError::Cancelled)?; + let source = photocraft_psd::FileSource::open(path).map_err(read_failed)?; + let mut magic = [0u8; 4]; + let psd = !has_extension(name, photocraft_format::EXTENSION) && photocraft_psd::ReadAt::read_at(&source, 0, &mut magic).is_ok() && is_psd(&magic); + if !psd { + drop(source); + let bytes = photocraft_format::read_file(path).map_err(read_failed)?; + return import_for_open_with(name, &bytes, opts, ctl); + } + import_source_for_open_with(name, photocraft_psd::Source::new(source), ctl) +} + +/// [`import_file_for_open_with`] for a PSD/PSB in any [`photocraft_psd::ReadAt`] source. +pub fn import_source_for_open_with(name: &str, source: photocraft_psd::Source, ctl: &photocraft_raster::Interrupt) -> Result<(ImportResult, bool), IoError> { + ctl.check().map_err(|_| IoError::Cancelled)?; + let mut damaged = false; + let r = import_psd(name, ctl, &mut damaged, |want, recover| PsdFile::from_source_and_merged_recovering(source, want, recover))?; + ctl.check().map_err(|_| IoError::Cancelled)?; + ctl.progress(1.0); + Ok((r, damaged)) +} + +/// [`import_file_for_open_with`] without the damage flag. +#[cfg(any(unix, windows))] +pub fn import_file(name: &str, path: &std::path::Path, opts: &ImportOptions, ctl: &photocraft_raster::Interrupt) -> Result { + import_file_for_open_with(name, path, opts, ctl).map(|(result, _)| result) +} + /// Dispatches format-specific importers and propagates a structured PSD damage /// flag alongside the ordinary best-effort import result. fn import_stages(name: &str, bytes: &[u8], ctl: &photocraft_raster::Interrupt, opts: &ImportOptions, damaged: &mut bool) -> Result { @@ -404,7 +463,7 @@ fn import_stages(name: &str, bytes: &[u8], ctl: &photocraft_raster::Interrupt, o // Magic first: an Affinity file renamed .psd is still Affinity (#5258 corpus // preview_never_saves_back_to_its_native_or_renamed_source). if is_psd(bytes) { - return import_psd(name, bytes, ctl, damaged); + return import_psd(name, ctl, damaged, |want, recover| PsdFile::from_bytes_and_merged_recovering(bytes, want, recover)); } if affinity::is_affinity(bytes) || affinity::has_extension(name) { return affinity::import(name, bytes); @@ -437,7 +496,7 @@ fn import_stages(name: &str, bytes: &[u8], ctl: &photocraft_raster::Interrupt, o // A `.psd`/`.psb`/`.psdt` with no raster signature keeps the PSD error instead of // `codec: unrecognized image format` (#5258). Magic already returned above. if is_psd_input(name, bytes) { - return import_psd(name, bytes, ctl, damaged); + return import_psd(name, ctl, damaged, |want, recover| PsdFile::from_bytes_and_merged_recovering(bytes, want, recover)); } flat::import_flat(name, bytes) } diff --git a/crates/io/src/linked.rs b/crates/io/src/linked.rs index e200b875ea..dcf7ecd98d 100644 --- a/crates/io/src/linked.rs +++ b/crates/io/src/linked.rs @@ -118,7 +118,30 @@ pub fn find_in_blocks<'a>(blocks: impl IntoIterator(blocks: impl IntoIterator, uuid: &str) -> Option<(String, String, &'a [u8])> { + if uuid.is_empty() { + return None; + } + blocks.into_iter().filter(|(k, _)| matches!(*k, b"lnk2" | b"lnk3" | b"lnkD")).find_map(|(_, d)| { + let mut r = Rd { d, p: 0 }; + while let Some(len) = r.u64() { + let Ok(len) = usize::try_from(len) else { return None }; + let item = r.take(len)?; + let found = item_header(item).filter(|h| h.0 == uuid).and_then(|(id, name, start, n)| Some((id, name, item.get(start..start.checked_add(n)?)?))); + if found.is_some() { + return found; + } + r.p = r.p.next_multiple_of(4); + } + None + }) } /// The size of embedded file `uuid` among PSD global blocks given as (key, data), when it is there @@ -128,11 +151,11 @@ pub(crate) fn source_size_in_blocks<'a>( uuid: &str, ctl: &photocraft_raster::Interrupt, ) -> Option<(f64, f64)> { - let f = find_in_blocks(blocks, uuid)?; + let (_, _, bytes) = find_slice_in_blocks(blocks, uuid)?; if ctl.cancelled() { return None; } - source_size_probe(&f.bytes).filter(|(w, h)| *w > 0.0 && *h > 0.0) + source_size_probe(bytes).filter(|(w, h)| *w > 0.0 && *h > 0.0) } /// Pixel size of an embedded source file from its header: PSD/PSB, TIFF (the default page's diff --git a/crates/io/src/multichannel_map.rs b/crates/io/src/multichannel_map.rs index e9437deb6c..f5b2933041 100644 --- a/crates/io/src/multichannel_map.rs +++ b/crates/io/src/multichannel_map.rs @@ -116,7 +116,7 @@ pub(crate) fn document_to_psd(doc: &Document, force_psb: bool) -> (PsdFile, Vec< let header = Header::new(version, doc.size.width, doc.size.height, channels.len().max(1) as u16, psd_depth(sample), PsdMode::Multichannel); let image_data = ImageData::encode(Compression::Rle, &planes, &header) .or_else(|_| ImageData::encode(Compression::Raw, &planes, &header)) - .unwrap_or(ImageData { compression: Compression::Raw, data: planes }); + .unwrap_or(ImageData::new(Compression::Raw, planes)); let mut resources = vec![ ImageResource::new(ids::RESOLUTION_INFO, ResolutionInfo::from_dpi(f64::from(doc.resolution_dpi)).to_bytes()), ImageResource::new(ids::GLOBAL_ANGLE, (doc.global_light.angle.round() as i32).to_be_bytes().to_vec()), diff --git a/crates/io/src/palette_map.rs b/crates/io/src/palette_map.rs index 45b8168f1c..b4aa1ff0c6 100644 --- a/crates/io/src/palette_map.rs +++ b/crates/io/src/palette_map.rs @@ -45,7 +45,7 @@ pub(crate) fn document_to_psd(doc: &Document, force_psb: bool) -> Option<(PsdFil let header = Header::new(version, doc.size.width, doc.size.height, 1, depth, mode); let image_data = ImageData::encode(Compression::Rle, &plane, &header) .or_else(|_| ImageData::encode(Compression::Raw, &plane, &header)) - .unwrap_or(ImageData { compression: Compression::Raw, data: plane }); + .unwrap_or(ImageData::new(Compression::Raw, plane)); let mut resources = vec![ ImageResource::new(ids::RESOLUTION_INFO, ResolutionInfo::from_dpi(f64::from(doc.resolution_dpi)).to_bytes()), ImageResource::new(ids::GLOBAL_ANGLE, (doc.global_light.angle.round() as i32).to_be_bytes().to_vec()), diff --git a/crates/io/src/psd_export.rs b/crates/io/src/psd_export.rs index 4835397822..351e33331b 100644 --- a/crates/io/src/psd_export.rs +++ b/crates/io/src/psd_export.rs @@ -368,7 +368,7 @@ impl Ex { let compression = if w == 0 || h == 0 { Compression::Raw } else { self.compression() }; ChannelData::encode(id, compression, plane, w, h, depth, self.version) .or_else(|_| ChannelData::encode(id, Compression::Raw, plane, w, h, depth, self.version)) - .unwrap_or(ChannelData { id, compression: Some(Compression::Raw), data: plane.to_vec() }) + .unwrap_or(ChannelData::new(id, Some(Compression::Raw), plane.to_vec())) } fn empty_channels(&self) -> Vec { @@ -1833,7 +1833,7 @@ fn document_to_psd_nested(doc: &Document, opts: &PsdExportOptions, depth: u32, c let mcomp = if sample == SampleType::F32 { Compression::Raw } else { opts.merged_compression }; let image_data = ImageData::encode(mcomp, &planes, &header) .or_else(|_| ImageData::encode(Compression::Raw, &planes, &header)) - .unwrap_or(ImageData { compression: Compression::Raw, data: planes }); + .unwrap_or(ImageData::new(Compression::Raw, planes)); ctl.progress(1.0); // Resources. diff --git a/crates/io/src/psd_import.rs b/crates/io/src/psd_import.rs index 717f848716..f646e1f665 100644 --- a/crates/io/src/psd_import.rs +++ b/crates/io/src/psd_import.rs @@ -4,7 +4,8 @@ use std::sync::Arc; use photocraft_color::{BlendMode, ColorMode, PixelFormat, SampleType}; use photocraft_doc::{ - AlphaChannel, Document, Effects, FillCache, Group, Layer, LayerContent, LayerMask, ShapeLayer, SmartContentsId, SmartObject, SmartSource, TextLayer, + AlphaChannel, Document, Effects, FillCache, Group, Layer, LayerContent, LayerMask, PsdGlobalBlock, ShapeLayer, SmartContentsId, SmartObject, SmartSource, + TextLayer, }; use photocraft_geom::{Rect, Size, TILE_SIZE}; use photocraft_psd::layer::{CHANNEL_REAL_USER_MASK, CHANNEL_TRANSPARENCY, CHANNEL_USER_MASK}; @@ -46,6 +47,8 @@ pub(crate) const REGENERATED: [&[u8; 4]; 9] = [b"luni", b"lyid", b"lsct", b"lsdk pub(crate) struct Ctx<'a> { pub file: &'a PsdFile, + /// The file's global blocks, as the document keeps them. + pub global: Vec, pub fmt: PixelFormat, pub mask_fmt: PixelFormat, pub cc: usize, @@ -53,6 +56,9 @@ pub(crate) struct Ctx<'a> { pub warnings: Vec, /// Decoding encountered damaged PSD data; interactive opens ask whether to continue. pub damaged: bool, + /// Reading the file's channel data failed (it is read as it is decoded: see + /// `PsdFile::from_source`): the open fails with it, rather than leaving layers empty. + pub read_error: Option, /// Document resolution (type sizes are converted to points with it). pub dpi: f32, /// The parsed `Txt2` block (type settings EngineData lacks, e.g. optical kerning). @@ -74,9 +80,35 @@ pub(crate) struct Ctx<'a> { pub ctl: photocraft_raster::Interrupt<'a>, /// Layer records decoded so far, out of `total` (progress). pub done: usize, + /// Tiles of one pixel value, shared by every surface the import decodes. + pub uniform: photocraft_raster::TileInterner, pub total: usize, } +/// Bytes of interleaved pixels a band decoded into tiles at once aims at: whole layers up to this +/// size decode in one band, larger ones in bands of whole tile rows. +const BAND_BYTES: usize = 32 << 20; + +/// Row ranges (`0..h`, relative to a layer whose first row is at document row `top`) to decode a +/// layer in: each ends on a tile row boundary, so no tile is written by two bands, and holds +/// about [`BAND_BYTES`] of `row_bytes`-byte rows. +fn band_rows(top: i32, h: usize, row_bytes: usize) -> Vec<(usize, usize)> { + let tile = TILE_SIZE as usize; + let rows = (BAND_BYTES / row_bytes.max(1)).max(tile) / tile * tile; + // The first band ends at the first tile boundary at least `rows` rows in. + let into_tile = usize::try_from(top.rem_euclid(TILE_SIZE)).unwrap_or(0); + let mut out = Vec::new(); + let mut r0 = 0; + let mut r1 = rows - into_tile; + while r0 < h { + let end = r1.min(h); + out.push((r0, end)); + r0 = end; + r1 = end + rows; + } + out +} + fn doc_mode(m: PsdMode) -> Option { Some(match m { PsdMode::Bitmap => ColorMode::Bitmap, @@ -148,6 +180,17 @@ impl Ctx<'_> { self.warn(s); } + /// A channel that failed to decode: a failed read of the file is kept to fail the open + /// (`true`); damaged data is warned about, its layer left empty (`false`). + fn channel_failed(&mut self, name: &str, id: i16, e: &photocraft_psd::PsdError) -> bool { + if let photocraft_psd::PsdError::Read(msg) = e { + self.read_error.get_or_insert_with(|| format!("layer \"{name}\": {msg}")); + return true; + } + self.warn_unreadable_data(format!("layer \"{name}\": channel {id} could not be decoded ({e}); treated as empty")); + false + } + /// Source id, transform, (Distort/Perspective) projective placement and warp of a placed /// layer that has only a `PlLd` block, as Photoshop CS2 and older write it (#2465). `PlLd` /// stores the placed corners but not the source size, which comes from the embedded file. @@ -168,11 +211,11 @@ impl Ctx<'_> { return Placement::default(); } }; - let (file, ctl) = (self.file, self.ctl); + let (global, ctl) = (&self.global, self.ctl); let size = *self .plld_sizes .entry(id.clone()) - .or_insert_with(|| crate::linked::source_size_in_blocks(file.global_blocks.iter().map(|b| (&b.key, b.data.as_slice())), &id, &ctl)); + .or_insert_with(|| crate::linked::source_size_in_blocks(global.iter().map(|(_, k, d)| (k, d.as_slice())), &id, &ctl)); // Without the source (missing or unreadable), it is taken as placed at 100 %. let (transform, perspective) = blocks::plld_transform(&p, size, warp.as_ref()); Placement { id, transform, perspective, warp } @@ -181,72 +224,83 @@ impl Ctx<'_> { /// Decodes a PSD layer channel, marking the document damaged if its data is unreadable. fn channel_plane(&mut self, rec: &LayerRecord, id: i16, name: &str) -> Option> { rec.channel(id)?; + if self.read_error.is_some() { + return None; + } match rec.decode_channel(id, self.file.header.depth, self.file.header.version) { Ok(p) => Some(p), Err(e) => { - self.warn_unreadable_data(format!("layer \"{name}\": channel {id} could not be decoded ({e}); treated as empty")); + self.channel_failed(name, id, &e); None } } } - /// [`Self::channel_plane`] without the warnings sink, so a layer's channels decode - /// concurrently (their RLE rows are the bulk of the open time on full-canvas layers). - fn channel_plane_decoded(&self, rec: &LayerRecord, id: i16) -> Option> { - rec.channel(id)?; - rec.decode_channel(id, self.file.header.depth, self.file.header.version).ok() - } - /// Builds a raster surface from the layer channels, leaving the surface empty if /// any present channel fails to decode rather than inventing replacement pixels. fn record_surface(&mut self, rec: &LayerRecord, name: &str) -> Surface { let r = rec.rect; - if r.is_empty() || r.size().is_err() { + // Once the file couldn't be read the open fails: nothing more is decoded. + if r.is_empty() || r.size().is_err() || self.read_error.is_some() { return Surface::new(self.fmt); } let (w, h) = r.size().unwrap_or((0, 0)); let s = self.fmt.sample; + let (depth, version) = (self.file.header.depth, self.file.header.version); let ids: Vec = (0..self.cc as i16).chain([CHANNEL_TRANSPARENCY]).collect(); - let planes: Vec>> = if cfg!(target_arch = "wasm32") { - ids.iter().map(|&id| self.channel_plane(rec, id, name)).collect() - } else { - // The channels decode independently; their RLE rows are the bulk of the open - // time for full-canvas layers, so they run side by side and the warnings land - // in the same order afterwards. - // par_map reuses the crate's worker pool (one OS thread per channel per layer - // would spawn thousands on a file with many layers). - let decoded: Vec>> = crate::pixels::par_map(ids.to_vec(), |id| self.channel_plane_decoded(rec, id)); - let mut planes: Vec>> = Vec::with_capacity(decoded.len()); - for (&id, p) in ids.iter().zip(decoded) { - if p.is_none() && rec.channel(id).is_some() { - self.warn_unreadable_data(format!("layer \"{name}\": channel {id} could not be decoded; treated as empty")); - } - planes.push(p); + // A decoder per channel present (a ZIP channel is inflated here, the channels side by + // side); the rows are then decoded and interleaved a band at a time straight into the + // tiles, so no buffer of the whole layer (planes or interleaved) is ever held. + // par_map reuses the crate's worker pool (one OS thread per channel per layer would + // spawn thousands on a file with many layers). + let decoders: Vec>>> = + crate::pixels::par_map(ids.to_vec(), |id| rec.channel(id).map(|ch| ch.row_decoder(w, h, depth, version))); + let mut failed = false; + for (&id, d) in ids.iter().zip(&decoders) { + if let Some(Err(e)) = d { + self.channel_failed(name, id, e); + failed = true; } - planes - }; + } // A channel that is in the file but could not be decoded leaves the layer empty, as the // warning says. Filling in for it would paint an opaque black layer over the document. - if ids.iter().zip(planes.iter()).any(|(&id, p)| p.is_none() && rec.channel(id).is_some()) { + if failed { return Surface::new(self.fmt); } // Only decoded channel data justifies a buffer: the decoders bound their output by the // bytes in the file, but a bare rectangle can declare 300000² pixels in a few bytes (#755). - if planes.iter().all(Option::is_none) { + if decoders.iter().all(Option::is_none) { self.warn_unreadable_data(format!("layer \"{name}\": no channel data for its {w}x{h} bounds; treated as empty")); return Surface::new(self.fmt); } - let refs: Vec> = planes.iter().map(|p| p.as_deref()).collect(); let mut fill: Vec> = vec![zero_sample(s); self.cc]; fill.push(max_sample(s)); let mut invert = vec![self.cmyk; self.cc]; invert.push(false); - let mut bytes = interleave(&refs, &fill, w * h, s, &invert); - if self.fmt.mode == ColorMode::Lab && s == SampleType::U16 { - crate::pixels::lab16_chroma(&mut bytes, self.cc + 1, true); + let mut surf = Surface::new(self.fmt); + for (b0, b1) in band_rows(r.top, h, w * (self.cc + 1) * s.bytes()) { + let planes: Vec>>> = + crate::pixels::par_map(decoders.iter().collect(), |d| d.as_ref().and_then(|d| d.as_ref().ok()).map(|d| d.rows(b0, b1))); + let mut refs: Vec> = Vec::with_capacity(planes.len()); + for (&id, p) in ids.iter().zip(&planes) { + match p { + Some(Err(e)) => { + self.channel_failed(name, id, e); + return Surface::new(self.fmt); + } + Some(Ok(p)) => refs.push(Some(p)), + None => refs.push(None), + } + } + let mut bytes = interleave(&refs, &fill, w * (b1 - b0), s, &invert); + if self.fmt.mode == ColorMode::Lab && s == SampleType::U16 { + crate::pixels::lab16_chroma(&mut bytes, self.cc + 1, true); + } + surf.write_interleaved(Rect::new(r.left, r.top + b0 as i32, r.right, r.top + b1 as i32), &bytes); } - let mut surf = Surface::from_interleaved(self.fmt, Rect::new(r.left, r.top, r.right, r.bottom), &bytes); surf.prune(); + // A white background, a solid fill's cache: one tile per value, shared across layers. + surf.share_uniform_tiles(&mut self.uniform); surf } @@ -270,6 +324,7 @@ impl Ctx<'_> { let bytes = interleave(&[Some(&plane)], &[zero_sample(s)], w * h, s, &[false]); surface.write_interleaved(Rect::new(rect.left, rect.top, rect.right, rect.bottom), &bytes); surface.prune(); + surface.share_uniform_tiles(&mut self.uniform); } let params = m.parameters; Some(LayerMask { @@ -910,6 +965,38 @@ pub(crate) fn psd_to_document_decoded_integrity( ctl: &photocraft_raster::Interrupt, decoded: Option>>, ) -> Option<(Document, Vec, bool)> { + let global = file.global_blocks.iter().map(|b| (b.signature, b.key, Arc::new(b.data.clone()))).collect(); + psd_to_document_global(file, ctl, decoded, global).map(|(document, warnings, damaged, _)| (document, warnings, damaged)) +} + +/// [`psd_to_document_decoded_integrity`] taking the file: its global blocks move into the +/// document rather than being copied (an embedded smart object's `lnk2` block can be most of a +/// file). The file's channel data may still be in its source ([`PsdFile::from_source`]): the +/// fourth value is a failure to read it, which fails the open (a layer is never left empty +/// because its file became unreadable while it opened). +pub(crate) fn psd_to_document_owned( + mut file: PsdFile, + ctl: &photocraft_raster::Interrupt, + decoded: Option>>, +) -> Option<(Document, Vec, bool, Option)> { + let global = file + .global_blocks + .iter_mut() + .map(|b| { + // A layer block is still read by `unreadable_layer_block` (to say why it failed). + let data = if matches!(&b.key, b"Lr16" | b"Lr32" | b"Layr") { b.data.clone() } else { std::mem::take(&mut b.data) }; + (b.signature, b.key, Arc::new(data)) + }) + .collect(); + psd_to_document_global(&file, ctl, decoded, global) +} + +fn psd_to_document_global( + file: &PsdFile, + ctl: &photocraft_raster::Interrupt, + decoded: Option>>, + global: Vec, +) -> Option<(Document, Vec, bool, Option)> { let h = &file.header; let mut warnings = Vec::new(); let mode = doc_mode(h.color_mode).unwrap_or_else(|| { @@ -970,9 +1057,7 @@ pub(crate) fn psd_to_document_decoded_integrity( } } crate::guides_map::apply_colors(&mut doc); - for b in &file.global_blocks { - doc.metadata.psd_global_blocks.push((b.signature, b.key, Arc::new(b.data.clone()))); - } + doc.metadata.psd_global_blocks.extend(global.iter().cloned()); // A 16/32-bit file whose layer block can't be parsed would otherwise open as a flattened // file without a word (#2700). The raw block is kept, but a PSD save writes the document's // own layers in its place (see `psd_export`). @@ -998,14 +1083,16 @@ pub(crate) fn psd_to_document_decoded_integrity( let cc = fmt.mode.color_channels(); let mut cx = Ctx { file, + global: global.clone(), fmt, mask_fmt: PixelFormat::new(ColorMode::Grayscale, depth, false), cc, cmyk: fmt.mode == ColorMode::Cmyk, warnings, damaged, + read_error: None, dpi: doc.resolution_dpi, - txt2: file.global_blocks.iter().find(|b| &b.key == b"Txt2").and_then(|b| photocraft_text::psd::parse_txt2(&b.data)), + txt2: global.iter().find(|(_, k, _)| k == b"Txt2").and_then(|(_, _, d)| photocraft_text::psd::parse_txt2(d)), filter_effects: Vec::new(), filter_masks: Default::default(), unfiltered: Default::default(), @@ -1014,16 +1101,17 @@ pub(crate) fn psd_to_document_decoded_integrity( plld_sizes: Default::default(), ctl: *ctl, done: 0, + uniform: Default::default(), total: file.layers().len(), }; - for b in file.global_blocks.iter().filter(|b| matches!(&b.key, b"FEid" | b"FXid")) { + for (_, key, data) in cx.global.clone().iter().filter(|(_, k, _)| matches!(k, b"FEid" | b"FXid")) { // Smart-filter caches can be large: a cancelled open stops between blocks. if ctl.cancelled() { return None; } - match photocraft_psd::filter_effects::FilterEffects::parse(&b.data) { + match photocraft_psd::filter_effects::FilterEffects::parse(data) { Ok(fx) => cx.filter_effects.extend(fx.items), - Err(e) => cx.warn(format!("smart filter masks ({}) could not be read: {e}", b.key_str())), + Err(e) => cx.warn(format!("smart filter masks ({}) could not be read: {e}", String::from_utf8_lossy(key))), } } if ctl.cancelled() { @@ -1239,7 +1327,7 @@ pub(crate) fn psd_to_document_decoded_integrity( return None; } ctl.progress(1.0); - Some((doc, cx.warnings, cx.damaged)) + Some((doc, cx.warnings, cx.damaged, cx.read_error)) } #[cfg(test)] @@ -1349,4 +1437,25 @@ mod tests { assert_eq!(merged_plane(&all, 2, 2), None); assert_eq!(merged_plane(&all, usize::MAX, 2), None); } + + #[test] + fn layer_bands_end_on_tile_rows_and_cover_the_layer() { + let t = TILE_SIZE as usize; + for (top, h, row_bytes) in [(0, 2000, 8000), (-37, 15000, 80_000), (100, 5, 4), (255, 1, 4), (-256, 40_000, 1 << 20), (7, 0, 4)] { + let bands = band_rows(top, h, row_bytes); + let mut next = 0; + for &(a, b) in &bands { + assert_eq!(a, next); + assert!(b > a); + if b < h { + assert_eq!((top + b as i32).rem_euclid(TILE_SIZE), 0, "{top} {h}: band ends inside a tile row"); + assert!((b - a) * row_bytes <= BAND_BYTES + t * row_bytes || b - a <= t); + } + next = b; + } + assert_eq!(next, h); + } + // A small layer is one band. + assert_eq!(band_rows(0, 2000, 8000), vec![(0, 2000)]); + } } diff --git a/crates/io/src/tiff_layers.rs b/crates/io/src/tiff_layers.rs index 25c2df5010..cc833f1434 100644 --- a/crates/io/src/tiff_layers.rs +++ b/crates/io/src/tiff_layers.rs @@ -104,7 +104,7 @@ pub(crate) fn import_layered(name: &str, img: &Image, layers: &[u8]) -> Result Self { + let dir = std::env::temp_dir().join(format!("photocraft-import-file-{tag}-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + TempDir(dir) + } + + fn write(&self, name: &str, bytes: &[u8]) -> std::path::PathBuf { + let path = self.0.join(name); + std::fs::write(&path, bytes).unwrap(); + path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// The document's debug form with every `…Id(n)` number blanked. +fn without_ids(doc: &photocraft_doc::Document) -> String { + let s = format!("{doc:?}"); + let mut out = String::with_capacity(s.len()); + let mut rest = s.as_str(); + while let Some(at) = rest.find("Id(") { + out.push_str(&rest[..at + 3]); + rest = rest[at + 3..].trim_start_matches(|c: char| c.is_ascii_digit()); + } + out.push_str(rest); + out +} + +fn open(name: &str, path: &std::path::Path) -> Result { + import_file(name, path, &photocraft_io::ImportOptions::default(), &Interrupt::NONE) +} + +#[test] +fn import_file_opens_what_importing_the_bytes_opens() { + let dir = TempDir::new("same"); + let mut checked = 0; + for case in testgen::all_cases() { + let ext = if case.file.header.version.is_psb() { "psb" } else { "psd" }; + let name = format!("case.{ext}"); + let bytes = case.file.to_bytes().unwrap(); + let path = dir.write(&name, &bytes); + let from_bytes = import(&name, &bytes); + let from_file = open(&name, &path); + match (from_bytes, from_file) { + (Ok(a), Ok(b)) => { + // Ids come from a process-wide counter, so the two opens differ only there. + assert!(without_ids(&b.document) == without_ids(&a.document), "{}", case.name); + assert_eq!(b.warnings, a.warnings, "{}", case.name); + checked += 1; + } + (Err(_), Err(_)) => {} + (a, b) => panic!("{}: bytes {:?}, file {:?}", case.name, a.map(|r| r.warnings), b.map(|r| r.warnings)), + } + } + assert!(checked > 50, "only {checked} cases opened"); +} + +#[test] +fn import_file_reads_other_formats_whole() { + let dir = TempDir::new("other"); + let doc = photocraft_doc::Document::new("x", photocraft_geom::Size::new(3, 2), photocraft_color::ColorMode::Rgb, photocraft_color::SampleType::U8); + let png = photocraft_io::export(&doc, "x.png", &Default::default()).unwrap().bytes; + let path = dir.write("x.png", &png); + let r = open("x.png", &path).unwrap(); + assert_eq!(r.document.size, doc.size); +} + +#[test] +fn import_file_fails_cleanly() { + let dir = TempDir::new("fail"); + assert!(open("missing.psd", &dir.0.join("missing.psd")).is_err()); + let bytes = testgen::small(photocraft_psd::Version::Psd, photocraft_psd::Compression::Rle).to_bytes().unwrap(); + // Cut inside the layer records, inside the channel data, and inside the merged image. + for cut in [10, 60, bytes.len() / 2, bytes.len() - 5] { + let name = format!("cut{cut}.psd"); + let path = dir.write(&name, &bytes[..cut]); + let from_bytes = import(&name, &bytes[..cut]); + let from_file = open(&name, &path); + // The merged image is not read when the layers are what opens, so a file cut inside it + // can open from the file where the strict parse of its bytes refused it. + if from_bytes.is_ok() { + assert!(from_file.is_ok(), "cut at {cut}"); + } + } +} + +#[test] +fn a_cancelled_import_file_reports_cancelled() { + let dir = TempDir::new("cancel"); + let bytes = testgen::small(photocraft_psd::Version::Psd, photocraft_psd::Compression::Zip).to_bytes().unwrap(); + let path = dir.write("c.psd", &bytes); + let cancel = || true; + let ctl = Interrupt::cancel_only(&cancel); + assert!(matches!(import_file("c.psd", &path, &photocraft_io::ImportOptions::default(), &ctl), Err(photocraft_io::IoError::Cancelled))); +} + +/// A read that fails while the layers are decoded from the file (it is truncated or rewritten, +/// its drive goes away) fails the open: no layer is left empty for a save to lose. +#[test] +fn a_read_failing_while_layers_decode_fails_the_open() { + use photocraft_psd::ReadAt; + /// The file's bytes, failing every read of channel data (as a dropped network share or a + /// truncated file would) except the 2-byte compression fields the parse reads. + struct Failing { + bytes: Vec, + data: Vec<(u64, u64)>, + } + impl ReadAt for Failing { + fn len(&self) -> u64 { + self.bytes.len() as u64 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> photocraft_psd::Result<()> { + let end = offset + buf.len() as u64; + if self.data.iter().any(|&(a, b)| offset < b && end > a) { + return Err(photocraft_psd::PsdError::Read("the drive went away".into())); + } + self.bytes.as_slice().read_at(offset, buf) + } + } + for c in [photocraft_psd::Compression::Rle, photocraft_psd::Compression::Raw, photocraft_psd::Compression::Zip] { + let file = testgen::layered(photocraft_psd::Version::Psd, photocraft_psd::ColorMode::Rgb, 8, c); + let bytes = file.to_bytes().unwrap(); + let lazy = photocraft_psd::PsdFile::from_source(photocraft_psd::Source::new(bytes.clone())).unwrap(); + let data: Vec<(u64, u64)> = lazy + .layers() + .iter() + .flat_map(|l| &l.channels) + .filter_map(|ch| ch.stored.as_ref()) + .filter(|s| s.len > 0) + .map(|s| (s.offset, s.offset + s.len)) + .collect(); + assert!(!data.is_empty()); + // Opening the same file whole succeeds: only the failed reads fail the open. + assert!(import("x.psd", &bytes).is_ok()); + let r = photocraft_io::import_source_for_open_with("x.psd", photocraft_psd::Source::new(Failing { bytes, data }), &Interrupt::NONE); + match r { + Err(photocraft_io::IoError::Read(msg)) => assert!(msg.contains("the drive went away"), "{c:?}: {msg}"), + r => panic!("{c:?}: the open didn't fail: {:?}", r.map(|(r, damaged)| (r.warnings, damaged))), + } + } +} diff --git a/crates/io/tests/it/psd_structure.rs b/crates/io/tests/it/psd_structure.rs index a110fbd367..e263f83a6b 100644 --- a/crates/io/tests/it/psd_structure.rs +++ b/crates/io/tests/it/psd_structure.rs @@ -658,7 +658,7 @@ fn fill_layer_keeps_photoshop_pixels() { VersionedDescriptor::new(Descriptor::new("null").with("Clr ", Value::Descriptor(c))).to_bytes() }; // Photoshop-rendered pixels deliberately differ from the parametric fill. - let plane = |v: u8| ChannelData { id: 0, compression: Some(Compression::Raw), data: vec![v; 4] }; + let plane = |v: u8| ChannelData::new(0, Some(Compression::Raw), vec![v; 4]); let mut chans = vec![plane(255), plane(10), plane(20), plane(30)]; for (i, c) in chans.iter_mut().enumerate() { c.id = i as i16 - 1; @@ -762,7 +762,7 @@ fn merged_composite_unmattes_photoshop_white() { let mut f = testgen::merged_only(Version::Psd, ColorMode::Rgb, 8, Compression::Raw, 1, 1); f.header.channels = 4; // straight red at alpha 0.5, matted: r = 1, g = b = 0.5 - f.image_data = photocraft_psd::ImageData { compression: Compression::Raw, data: vec![255, 128, 128, 128] }; + f.image_data = photocraft_psd::ImageData::new(Compression::Raw, vec![255, 128, 128, 128]); let m = merged_composite(&f).unwrap(); assert!((m[0][0] - 1.0).abs() < 0.01 && m[0][1] < 0.01 && m[0][2] < 0.01, "{:?}", m[0]); let (d, _) = psd_to_document(&f); @@ -772,8 +772,7 @@ fn merged_composite_unmattes_photoshop_white() { // HDR and negative straight samples must survive the actual F32 PSD import boundary. let mut hdr = testgen::merged_only(Version::Psd, ColorMode::Rgb, 32, Compression::Raw, 1, 1); hdr.header.channels = 4; - hdr.image_data = - photocraft_psd::ImageData { compression: Compression::Raw, data: [1.5_f32, 0.25, 0.5, 0.5].into_iter().flat_map(f32::to_be_bytes).collect() }; + hdr.image_data = photocraft_psd::ImageData::new(Compression::Raw, [1.5_f32, 0.25, 0.5, 0.5].into_iter().flat_map(f32::to_be_bytes).collect()); assert_eq!(merged_composite(&hdr).unwrap(), vec![[2.0, -0.5, 0.0, 0.5]]); let mut deep = testgen::merged_only(Version::Psd, ColorMode::Rgb, 16, Compression::Raw, 1, 1); deep.header.channels = 4; @@ -793,7 +792,7 @@ fn vector_rendered_mask_not_doubled_on_shapes() { f.layers_mut().push(LayerRecord { name: b"shape".to_vec(), mask: photocraft_psd::MaskData::Mask(mask), - channels: vec![photocraft_psd::ChannelData { id: -2, compression: Some(Compression::Raw), data: vec![] }], + channels: vec![photocraft_psd::ChannelData::new(-2, Some(Compression::Raw), vec![])], blocks: vec![TaggedBlock::unicode_name("shape"), TaggedBlock::new(*b"SoCo", vec![]), TaggedBlock::new(*b"vmsk", vec![0; 8])], ..Default::default() }); @@ -805,7 +804,7 @@ fn vector_rendered_mask_not_doubled_on_shapes() { let record = f.layers_mut().last_mut().unwrap(); let photocraft_psd::MaskData::Mask(mask) = &mut record.mask else { panic!("fixture has a mask") }; mask.real = Some(photocraft_psd::RealMask { flags: 0, background: 255, rect: photocraft_psd::Rect { top: 0, left: 0, bottom: 1, right: 1 } }); - record.channels.push(photocraft_psd::ChannelData { id: -3, compression: Some(Compression::Raw), data: vec![64] }); + record.channels.push(photocraft_psd::ChannelData::new(-3, Some(Compression::Raw), vec![64])); let (imported, _) = psd_to_document(&f); assert_eq!(imported.layers.last().unwrap().content.kind_name(), "Shape"); let mask = imported.layers.last().unwrap().mask.as_ref().expect("selected real mask must survive synthetic cleanup"); diff --git a/crates/io/tests/it/psd_unknown_adjustment.rs b/crates/io/tests/it/psd_unknown_adjustment.rs index 8b1f0cbdd3..58bc909c57 100644 --- a/crates/io/tests/it/psd_unknown_adjustment.rs +++ b/crates/io/tests/it/psd_unknown_adjustment.rs @@ -77,7 +77,7 @@ fn pixel_layers_with_unknown_blocks_stay_pixel_layers() { // A normal pixel layer (pixels, flag clear) carrying an unknown block. f.layers_mut()[0].blocks.push(extra()); // Flagged but with pixels: never dropped for an adjustment. - let plane = |id: i16| ChannelData { id, compression: Some(Compression::Raw), data: vec![200; 4] }; + let plane = |id: i16| ChannelData::new(id, Some(Compression::Raw), vec![200; 4]); let mut painted = adjustment_record("painted", vec![extra()]); painted.rect = Rect::from_xywh(0, 0, 2, 2); painted.channels = (-1..3).map(plane).collect(); diff --git a/crates/psd/README.md b/crates/psd/README.md index 421d706ab7..71d12e9f12 100644 --- a/crates/psd/README.md +++ b/crates/psd/README.md @@ -13,6 +13,7 @@ A standalone reader and writer for Adobe Photoshop **PSD** (version 1) and **PSB | `PsdFile::from_bytes(b)?.to_bytes()? == b` for unmodified files | Channel data keeps its original encoding. Unknown blocks and resources are stored raw. Exact padding is recorded wherever it can vary: tagged blocks, layer info, section trailers. | | `from_bytes(to_bytes(f)) == f` for any model `f` | Canonical defaults: `padding: None` means "zero-pad to even", and the parser yields `None` whenever the file used that default. | | Lazy decoding | The parser only copies encoded bytes. Channels are decoded when asked (`ChannelData::decode`, `ImageData::decode`). | +| Lazy reading | `PsdFile::from_source` reads only the structure. Channel data and the merged image stay in the source as `Stored` ranges and are read when decoded, so opening a 2 GB file doesn't need 2 GB for the file itself. The model equals what `from_bytes` builds, and it writes back the same bytes. | | No compositing | `PsdBuilder` takes the merged image from the caller. Without one, it writes a white placeholder and sets `has_real_merged_data = false` in resource 1057. | ## Public API (overview) @@ -22,6 +23,8 @@ A standalone reader and writer for Adobe Photoshop **PSD** (version 1) and **PSB PsdFile::from_bytes(&[u8]) -> Result PsdFile::from_bytes_and_merged(&[u8], want) -> Result<(PsdFile, Option>>)> // + decode_merged() when want(&file): ZIP inflated once +PsdFile::from_source(Source) -> Result // channel data and merged image left in the source +PsdFile::from_source_and_merged_recovering(Source, want, recover) PsdFile::to_bytes(&self) -> Result> PsdFile::open(path) / save(path) // native only PsdFile { header, color_mode_data, resources, layer_info, layer_info_placement, @@ -36,6 +39,11 @@ PsdFile::composite_rgba8() -> Result PsdFile::resource(id), global_block(key), icc_profile(), resolution(), has_real_merged_data(), merged_has_alpha(), validate() +// Sources (lazy reading) +trait ReadAt { len(), read_at(offset, buf) } // [u8], Vec, Arc, FileSource (pread / seek_read) +Source::new(impl ReadAt) / read_vec(offset, len) // shared handle; equal when the same source +Stored { source, offset, len }::read() / read_part(at, len) + // Header Header { version: Version::{Psd,Psb}, channels, width, height, depth, color_mode, reserved } ColorMode::{Bitmap, Grayscale, Indexed, Rgb, Cmyk, Multichannel, Duotone, Lab, Unknown(u16)} @@ -47,7 +55,8 @@ LayerRecord { rect, channels: Vec, blend_mode, opacity, clipping, f LayerRecord::name() // prefers `luni` LayerRecord::section_divider(), section_type(), layer_id(), fill_opacity(), is_visible(), block(key), block_mut(key), channel(id), channel_rect(id), decode_channel(id, depth, version) -ChannelData { id, compression: Option, data } // encoded bytes kept verbatim +ChannelData { id, compression: Option, data, stored: Option } // encoded bytes kept verbatim +ChannelData::new(id, compression, data); encoded() -> Cow<[u8]>; encoded_len(); load() ChannelData::encode(id, compression, decoded, w, h, depth, version) ChannelData::decode(w, h, depth, version); set_decoded(...) MaskData::{None, Mask(LayerMask), Raw(Vec)} diff --git a/crates/psd/examples/bench_layer_block.rs b/crates/psd/examples/bench_layer_block.rs index 4d34f9b8f3..2b48de1d28 100644 --- a/crates/psd/examples/bench_layer_block.rs +++ b/crates/psd/examples/bench_layer_block.rs @@ -11,12 +11,12 @@ fn main() -> Result<()> { let plane_bytes = 6000 * 4000 * usize::from(depth / 8); let mut layer = LayerRecord { rect: Rect::from_xywh(0, 0, 6000, 4000), name: b"RGB".to_vec(), ..Default::default() }; for id in 0..3 { - layer.channels.push(ChannelData { id, compression: Some(Compression::Raw), data: vec![0; plane_bytes] }); + layer.channels.push(ChannelData::new(id, Some(Compression::Raw), vec![0; plane_bytes])); } file.layer_info = Some(LayerInfo { merged_alpha: false, layers: vec![layer], padding: None }); file.layer_info_placement = LayerInfoPlacement::GlobalBlock { index: 0, signature: *b"8BIM", key: if depth == 16 { *b"Lr16" } else { *b"Lr32" }, padding: None }; - file.image_data = ImageData { compression: Compression::Raw, data: vec![0; plane_bytes * 3] }; + file.image_data = ImageData::new(Compression::Raw, vec![0; plane_bytes * 3]); black_box(file.to_bytes()?); let mut samples = Vec::new(); let mut size = 0; diff --git a/crates/psd/src/error.rs b/crates/psd/src/error.rs index da519e6698..a1fc1a5c98 100644 --- a/crates/psd/src/error.rs +++ b/crates/psd/src/error.rs @@ -37,6 +37,9 @@ pub enum PsdError { /// A compressed stream could not be decoded. #[error("decompression failed: {0}")] Decompress(String), + /// Reading from a [`crate::source::ReadAt`] source failed. + #[error("read failed: {0}")] + Read(String), /// An operation is not supported for this data (e.g. unknown compression). #[error("unsupported: {0}")] Unsupported(String), diff --git a/crates/psd/src/file.rs b/crates/psd/src/file.rs index fab262732b..fe1bb0fda0 100644 --- a/crates/psd/src/file.rs +++ b/crates/psd/src/file.rs @@ -174,14 +174,27 @@ impl PsdFile { layer_mask_trailing = s.peek_rest().to_vec(); } } - let image_data = ImageData { compression: Compression::Raw, data: Vec::new() }; + let image_data = ImageData::new(Compression::Raw, Vec::new()); let mut file = PsdFile { header, color_mode_data, resources, layer_info, layer_info_placement, global_layer_mask, global_blocks, layer_mask_trailing, image_data }; + let want = want(&file); + let merged = file.read_merged_section(&mut r, want, recover)?; + Ok((file, merged)) + } + + /// Reads the merged image section from `rest` (the bytes after the layer and mask section) as + /// [`Self::from_bytes_and_merged_recovering`] does, decoding it. + pub(crate) fn read_merged_recovering(&mut self, rest: &[u8], recover: impl FnOnce(&PsdFile) -> bool) -> Result>>> { + self.read_merged_section(&mut Reader::new(rest), true, recover) + } + + fn read_merged_section(&mut self, r: &mut Reader<'_>, want: bool, recover: impl FnOnce(&PsdFile) -> bool) -> Result>>> { + let file = self; let mut encoded = r.clone(); - let merged = if want(&file) { - file.read_merged_decoded(&mut r).map(Some) + let merged = if want { + file.read_merged_decoded(r).map(Some) } else { - ImageData::read(&mut r, &file.header).map(|image| { + ImageData::read(r, &file.header).map(|image| { file.image_data = image; None }) @@ -189,7 +202,7 @@ impl PsdFile { let merged = match merged { Ok(merged) => merged, Err(error) => { - if matches!(error, PsdError::LimitExceeded(_)) || !recover(&file) { + if matches!(error, PsdError::LimitExceeded(_)) || !recover(file) { return Err(error); } let compression = Compression::from_u16(encoded.u16()?); @@ -197,11 +210,11 @@ impl PsdFile { let mut data = Vec::new(); data.try_reserve_exact(source.len()).map_err(|_| PsdError::LimitExceeded("not enough memory to retain the damaged merged image"))?; data.extend_from_slice(source); - file.image_data = ImageData { compression, data }; + file.image_data = ImageData::new(compression, data); Some(Err(error)) } }; - Ok((file, merged)) + Ok(merged) } /// Reads the merged image into `self.image_data` and decodes it. A ZIP stream within the @@ -224,7 +237,7 @@ impl PsdFile { if decoded.is_err() { validate_planes(compression, data, &layout)?; } - self.image_data = ImageData { compression, data: data.to_vec() }; + self.image_data = ImageData::new(compression, data.to_vec()); r.skip(data.len())?; Ok(decoded) } @@ -235,7 +248,11 @@ impl PsdFile { let psb = v.is_psb(); // Reserve the pixel data up front: appending it to an empty Vec reallocates and copies // repeatedly. Everything else (headers, resources, records) is small by comparison. - let hint = self.layer_info.as_ref().map_or(0, LayerInfo::channel_data_len).saturating_add(self.image_data.data.len()); + let hint = self + .layer_info + .as_ref() + .map_or(0, LayerInfo::channel_data_len) + .saturating_add(usize::try_from(self.image_data.encoded_len()).unwrap_or(usize::MAX)); let mut out = Vec::new(); out.try_reserve(hint).map_err(|_| PsdError::LimitExceeded("not enough memory to serialize the file"))?; self.header.write(&mut out); @@ -289,7 +306,7 @@ impl PsdFile { out.end_len(at, psb)?; out.put_u16(self.image_data.compression.as_u16()); - out.put(&self.image_data.data); + out.put(&self.image_data.encoded()?); Ok(out) } diff --git a/crates/psd/src/image_data.rs b/crates/psd/src/image_data.rs index 864486b4ac..9e27af2917 100644 --- a/crates/psd/src/image_data.rs +++ b/crates/psd/src/image_data.rs @@ -1,13 +1,20 @@ //! Merged (composite) image data section. +use std::borrow::Cow; + use crate::compression::{Compression, PlaneLayout, decode_planes, encode_planes, validate_planes}; use crate::error::Result; use crate::header::Header; use crate::io::Reader; +use crate::source::Stored; /// The merged image: compression plus the encoded bytes of all channels /// (to the end of the file), kept verbatim. -#[derive(Debug, Clone, PartialEq, Eq)] +/// +/// As with [`crate::ChannelData`], a file parsed by +/// [`PsdFile::from_source`](crate::PsdFile::from_source) may leave the bytes in the source +/// (`stored`, with `data` empty); equality compares the bytes wherever they are. +#[derive(Debug, Clone)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct ImageData { /// Compression method for the whole image. @@ -15,9 +22,50 @@ pub struct ImageData { /// Encoded data. For RLE: the byte-count table for every row of every /// channel followed by the rows. pub data: Vec, + /// Where the encoded bytes are when they were left in the source. (Not serialized.) + #[cfg_attr(feature = "serde", serde(skip))] + pub stored: Option, } +impl PartialEq for ImageData { + fn eq(&self, other: &Self) -> bool { + if self.compression != other.compression || self.encoded_len() != other.encoded_len() { + return false; + } + match (self.encoded(), other.encoded()) { + (Ok(a), Ok(b)) => a == b, + _ => false, + } + } +} + +impl Eq for ImageData {} + impl ImageData { + /// A merged image whose encoded bytes are in memory. + pub fn new(compression: Compression, data: Vec) -> Self { + ImageData { compression, data, stored: None } + } + + /// The encoded bytes: borrowed from `data`, or read from the source. + pub fn encoded(&self) -> Result> { + crate::source::encoded(&self.data, self.stored.as_ref()) + } + + /// Number of encoded bytes, wherever they are. + pub fn encoded_len(&self) -> u64 { + self.stored.as_ref().map_or(self.data.len() as u64, |s| s.len) + } + + /// Reads bytes left in the source into `data`. + pub fn load(&mut self) -> Result<()> { + if let Some(s) = &self.stored { + self.data = s.read()?; + self.stored = None; + } + Ok(()) + } + /// Plane layout for the merged image described by `header`. pub fn layout(header: &Header) -> PlaneLayout { PlaneLayout { @@ -31,12 +79,12 @@ impl ImageData { /// Encodes planar data (`channels × height × row_bytes`). pub fn encode(compression: Compression, decoded: &[u8], header: &Header) -> Result { - Ok(ImageData { compression, data: encode_planes(compression, decoded, &Self::layout(header))? }) + Ok(ImageData::new(compression, encode_planes(compression, decoded, &Self::layout(header))?)) } /// Decodes all channels into planar big-endian samples. pub fn decode(&self, header: &Header) -> Result> { - decode_planes(self.compression, &self.data, &Self::layout(header)) + decode_planes(self.compression, &self.encoded()?, &Self::layout(header)) } pub(crate) fn read(r: &mut Reader<'_>, header: &Header) -> Result { @@ -45,6 +93,6 @@ impl ImageData { validate_planes(compression, data, &Self::layout(header))?; let data = data.to_vec(); r.skip(data.len())?; - Ok(ImageData { compression, data }) + Ok(ImageData::new(compression, data)) } } diff --git a/crates/psd/src/layer.rs b/crates/psd/src/layer.rs index 5b11411dfb..407a92233d 100644 --- a/crates/psd/src/layer.rs +++ b/crates/psd/src/layer.rs @@ -1,10 +1,13 @@ //! Layer records, layer masks, blending ranges and per-layer channel data. +use std::borrow::Cow; + use crate::blend::BlendMode; use crate::compression::{Compression, PlaneLayout, decode_planes, encode_planes}; use crate::error::{PsdError, Result}; use crate::header::{MAX_DIMENSION, Version}; use crate::io::{Reader, WriteExt, decode_legacy_name, read_pascal, write_pascal}; +use crate::source::Stored; use crate::tagged::{BlockData, SectionDivider, SectionType, TaggedBlock, read_blocks, uses_long_length, write_blocks}; /// Channel id of the transparency mask. @@ -368,7 +371,12 @@ impl BlendingRanges { } /// Encoded image data for one layer channel. -#[derive(Debug, Clone, PartialEq, Eq)] +/// +/// The encoded bytes are in `data`, or, for a file parsed by +/// [`PsdFile::from_source`](crate::PsdFile::from_source), still in the source at `stored` (and +/// `data` is empty). Every method here reads them from wherever they are; two channels are equal +/// when their bytes are, wherever each one keeps them. +#[derive(Debug, Clone)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct ChannelData { /// Channel id: 0.. color channels, -1 transparency, -2 user mask, @@ -377,22 +385,65 @@ pub struct ChannelData { /// Compression; `None` when the stored length was 0 (no compression /// field at all). pub compression: Option, - /// Encoded bytes following the compression field, kept verbatim. + /// Encoded bytes following the compression field, kept verbatim (empty while `stored`). pub data: Vec, + /// Where the encoded bytes are when they were left in the source: see [`Self::load`]. + /// (Not serialized: load the channel first.) + #[cfg_attr(feature = "serde", serde(skip))] + pub stored: Option, } +impl PartialEq for ChannelData { + fn eq(&self, other: &Self) -> bool { + if self.id != other.id || self.compression != other.compression || self.encoded_len() != other.encoded_len() { + return false; + } + match (self.encoded(), other.encoded()) { + (Ok(a), Ok(b)) => a == b, + _ => false, + } + } +} + +impl Eq for ChannelData {} + impl ChannelData { /// Encodes planar samples (`height × row_bytes`). pub fn encode(id: i16, compression: Compression, decoded: &[u8], width: usize, height: usize, depth: u16, version: Version) -> Result { let layout = PlaneLayout { planes: 1, width, height, depth, version }; - Ok(ChannelData { id, compression: Some(compression), data: encode_planes(compression, decoded, &layout)? }) + Ok(ChannelData::new(id, Some(compression), encode_planes(compression, decoded, &layout)?)) + } + + /// A channel whose encoded bytes are in memory. + pub fn new(id: i16, compression: Option, data: Vec) -> Self { + ChannelData { id, compression, data, stored: None } + } + + /// The encoded bytes (after the compression field): borrowed from `data`, or read from the + /// source when they were left there. + pub fn encoded(&self) -> Result> { + crate::source::encoded(&self.data, self.stored.as_ref()) + } + + /// Number of encoded bytes, wherever they are. + pub fn encoded_len(&self) -> u64 { + self.stored.as_ref().map_or(self.data.len() as u64, |s| s.len) + } + + /// Reads bytes left in the source into `data`, so the channel no longer needs its source. + pub fn load(&mut self) -> Result<()> { + if let Some(s) = &self.stored { + self.data = s.read()?; + self.stored = None; + } + Ok(()) } /// Decodes into planar big-endian samples. pub fn decode(&self, width: usize, height: usize, depth: u16, version: Version) -> Result> { let layout = PlaneLayout { planes: 1, width, height, depth, version }; match self.compression { - Some(c) => decode_planes(c, &self.data, &layout), + Some(c) => decode_planes(c, &self.encoded()?, &layout), None => { if layout.decoded_len()? == 0 { Ok(Vec::new()) @@ -412,7 +463,7 @@ impl ChannelData { fn stored_len(&self) -> u64 { match self.compression { - Some(_) => 2 + self.data.len() as u64, + Some(_) => 2 + self.encoded_len(), None => 0, } } @@ -556,7 +607,7 @@ impl LayerRecord { ch.decode(w, h, depth, version) } - fn read(r: &mut Reader<'_>, version: Version) -> Result<(Self, Vec)> { + pub(crate) fn read(r: &mut Reader<'_>, version: Version) -> Result<(Self, Vec)> { let rect = Rect::read(r)?; let nch = r.u16()?; if nch > MAX_LAYER_CHANNELS { @@ -567,7 +618,7 @@ impl LayerRecord { for _ in 0..nch { let id = r.i16()?; let len = r.len_field(version.is_psb())?; - channels.push(ChannelData { id, compression: None, data: Vec::new() }); + channels.push(ChannelData::new(id, None, Vec::new())); lens.push(len); } let sig = r.array::<4>()?; @@ -716,7 +767,7 @@ impl LayerInfo { n = n.saturating_add(l.encoded_len(version)?); } for c in self.layers.iter().flat_map(|l| &l.channels).filter(|c| c.compression.is_some()) { - n = n.saturating_add(2).saturating_add(c.data.len() as u64); + n = n.saturating_add(2).saturating_add(c.encoded_len()); } Ok(n) } @@ -740,7 +791,11 @@ impl LayerInfo { /// Bytes of channel data and their compression fields: the part of the body worth reserving up /// front (the records are small next to the pixels). Saturates instead of overflowing. pub(crate) fn channel_data_len(&self) -> usize { - self.layers.iter().flat_map(|l| &l.channels).filter(|c| c.compression.is_some()).fold(0usize, |n, c| n.saturating_add(2).saturating_add(c.data.len())) + self.layers + .iter() + .flat_map(|l| &l.channels) + .filter(|c| c.compression.is_some()) + .fold(0usize, |n, c| n.saturating_add(2).saturating_add(usize::try_from(c.encoded_len()).unwrap_or(usize::MAX))) } /// Serializes the body without any trailing padding (the padding field is ignored). @@ -754,7 +809,7 @@ impl LayerInfo { for c in &l.channels { if let Some(comp) = c.compression { out.put_u16(comp.as_u16()); - out.put(&c.data); + out.put(&c.encoded()?); } } } @@ -835,7 +890,7 @@ mod tests { #[test] fn psb_channel_lengths_are_8_bytes() { let mut rec = LayerRecord::default(); - rec.channels.push(ChannelData { id: 0, compression: Some(Compression::Raw), data: vec![] }); + rec.channels.push(ChannelData::new(0, Some(Compression::Raw), vec![])); let info = LayerInfo { layers: vec![rec], ..Default::default() }; let psd = rt(&info, Version::Psd); let psb = rt(&info, Version::Psb); @@ -1075,7 +1130,7 @@ mod tests { #[test] fn zero_length_channel() { let mut rec = LayerRecord::default(); - rec.channels.push(ChannelData { id: 0, compression: None, data: vec![] }); + rec.channels.push(ChannelData::new(0, None, vec![])); let info = LayerInfo { layers: vec![rec], ..Default::default() }; rt(&info, Version::Psd); assert_eq!(info.layers[0].decode_channel(0, 8, Version::Psd).unwrap(), Vec::::new()); diff --git a/crates/psd/src/lazy.rs b/crates/psd/src/lazy.rs new file mode 100644 index 0000000000..d22661abb5 --- /dev/null +++ b/crates/psd/src/lazy.rs @@ -0,0 +1,378 @@ +//! Parsing from a [`Source`] without reading the whole file: [`PsdFile::from_source`]. +//! +//! The structure (header, resources, layer records, global blocks) is read into memory as +//! [`PsdFile::from_bytes`] would hold it. The bulk of a layered file is left in the source: every +//! channel's encoded bytes and the merged image become [`Stored`] ranges, read when they are +//! decoded. Apart from those ranges, the model is the one `from_bytes` builds from the same bytes +//! (the parse tests hold the two to that), with one difference: a merged image the caller does +//! not ask for is not read, so it is not checked either. + +use crate::compression::Compression; +use crate::error::{PsdError, Result}; +use crate::file::{GlobalLayerMask, LayerInfoPlacement, PsdFile}; +use crate::header::{Header, Version}; +use crate::image_data::ImageData; +use crate::io::Reader; +use crate::layer::{LayerInfo, LayerRecord}; +use crate::resources::ImageResource; +use crate::source::{Source, Stored}; +use crate::tagged::{TaggedBlock, is_sig, uses_long_length}; + +/// Bytes read per attempt at the layer records; doubled until they all fit. (Small in tests, so +/// they grow it.) +const RECORDS_CHUNK: u64 = if cfg!(test) { 64 } else { 1 << 20 }; + +/// A bounds-checked cursor over `start..end` of a source. +struct Cursor<'a> { + src: &'a Source, + pos: u64, + end: u64, +} + +impl<'a> Cursor<'a> { + fn new(src: &'a Source) -> Self { + Cursor { src, pos: 0, end: src.get().len() } + } + + fn remaining(&self) -> u64 { + self.end.saturating_sub(self.pos) + } + + fn is_empty(&self) -> bool { + self.remaining() == 0 + } + + fn eof(&self, needed: u64) -> PsdError { + PsdError::UnexpectedEof { + offset: usize::try_from(self.pos).unwrap_or(usize::MAX), + needed: usize::try_from(needed.saturating_sub(self.remaining())).unwrap_or(usize::MAX), + } + } + + fn need(&self, n: u64) -> Result<()> { + if n > self.remaining() { Err(self.eof(n)) } else { Ok(()) } + } + + /// Reads `n` bytes. + fn bytes(&mut self, n: u64) -> Result> { + self.need(n)?; + let v = self.src.read_vec(self.pos, n)?; + self.pos += n; + Ok(v) + } + + /// Up to `n` bytes from the current position, without consuming. + fn peek(&self, n: u64) -> Result> { + self.src.read_vec(self.pos, n.min(self.remaining())) + } + + fn array(&mut self) -> Result<[u8; N]> { + self.need(N as u64)?; + let mut b = [0u8; N]; + self.src.get().read_at(self.pos, &mut b)?; + self.pos += N as u64; + Ok(b) + } + + fn u16(&mut self) -> Result { + self.array().map(u16::from_be_bytes) + } + + fn i16(&mut self) -> Result { + self.array().map(i16::from_be_bytes) + } + + fn u32(&mut self) -> Result { + self.array().map(u32::from_be_bytes) + } + + fn len_field(&mut self, long: bool) -> Result { + if long { self.array().map(u64::from_be_bytes) } else { self.u32().map(u64::from) } + } + + /// The next `n` bytes as a cursor of their own; this one moves past them. + fn sub(&mut self, n: u64) -> Result> { + self.need(n)?; + let sub = Cursor { src: self.src, pos: self.pos, end: self.pos + n }; + self.pos += n; + Ok(sub) + } + + fn skip(&mut self, n: u64) -> Result<()> { + self.need(n)?; + self.pos += n; + Ok(()) + } + + fn rest(&mut self) -> Result> { + self.bytes(self.remaining()) + } + + /// The rest as a stored range; the cursor moves to the end. + fn stored_rest(&mut self) -> Stored { + let s = Stored { source: self.src.clone(), offset: self.pos, len: self.remaining() }; + self.pos = self.end; + s + } +} + +impl PsdFile { + /// Parses a file from `source`, leaving channel data and the merged image in it: see + /// [`crate::source`]. Decoding a channel then reads its bytes from the source, which must + /// stay unchanged while the model is in use. + pub fn from_source(source: Source) -> Result { + Self::from_source_and_merged_recovering(source, |_| false, |_| false).map(|(file, _)| file) + } + + /// [`Self::from_bytes_and_merged_recovering`] for a [`Source`]: the merged image is read only + /// when `want` asks for it (and is then decoded as there, with the same recovery); otherwise + /// it stays in the source, unread and unchecked. + pub fn from_source_and_merged_recovering( + source: Source, + want: impl FnOnce(&PsdFile) -> bool, + recover: impl FnOnce(&PsdFile) -> bool, + ) -> Result<(Self, Option>>)> { + let mut c = Cursor::new(&source); + let header = Header::read(&mut Reader::new(&c.bytes(26)?))?; + let version = header.version; + let psb = version.is_psb(); + let cmd_len = c.u32()?; + let color_mode_data = c.bytes(u64::from(cmd_len))?; + let res_len = c.u32()?; + let res = c.bytes(u64::from(res_len))?; + let mut rr = Reader::new(&res); + let mut resources = Vec::new(); + while !rr.is_empty() { + resources.push(ImageResource::read(&mut rr)?); + } + + let lm_len = c.len_field(psb)?; + let mut s = c.sub(lm_len)?; + let mut layer_info = None; + let mut global_layer_mask = None; + let mut global_blocks = Vec::new(); + let mut layer_mask_trailing = Vec::new(); + let mut layer_info_placement = LayerInfoPlacement::Section; + if !s.is_empty() { + let li_len = s.len_field(psb)?; + if li_len > 0 { + layer_info = Some(read_layer_info(s.sub(li_len)?, version)?); + } + if s.remaining() >= 4 { + let n = s.u32()?; + global_layer_mask = Some(GlobalLayerMask { data: s.bytes(u64::from(n))? }); + let target_key = match header.depth { + 16 => Some(b"Lr16"), + 32 => Some(b"Lr32"), + _ => None, + }; + let mut candidate_seen = match target_key { + Some(_) => false, + None => layer_info.as_ref().is_some_and(|i: &LayerInfo| !i.layers.is_empty()), + }; + let trailing = read_blocks( + &mut s, + version, + |index, signature, key, data, padding| { + let is_candidate = match target_key { + Some(k) => &key == k, + None => matches!(&key, b"Layr" | b"Lr16" | b"Lr32"), + }; + if candidate_seen || !is_candidate { + return Ok(None); + } + candidate_seen = true; + let Ok(info) = read_layer_info(data, version) else { + if target_key.is_some() { + layer_info = None; + } + return Ok(None); + }; + layer_info = Some(info); + layer_info_placement = LayerInfoPlacement::GlobalBlock { index, signature, key, padding: padding.clone() }; + Ok(Some(())) + }, + &mut global_blocks, + )?; + layer_mask_trailing = trailing; + } else { + layer_mask_trailing = s.rest()?; + } + } + let mut file = PsdFile { + header, + color_mode_data, + resources, + layer_info, + layer_info_placement, + global_layer_mask, + global_blocks, + layer_mask_trailing, + image_data: ImageData::new(Compression::Raw, Vec::new()), + }; + if !want(&file) { + let compression = Compression::from_u16(c.u16()?); + file.image_data = ImageData { compression, data: Vec::new(), stored: Some(c.stored_rest()) }; + return Ok((file, None)); + } + // The merged image is wanted: read it and decode it as `from_bytes` does. + let rest = c.rest()?; + let merged = file.read_merged_recovering(&rest, recover)?; + Ok((file, merged)) + } +} + +/// Reads tagged blocks as [`crate::tagged::read_blocks_with`] does, from a cursor: `consume` +/// sees each block's data as a cursor and returns `Some` when it took the block; the others are +/// read into memory and pushed to `blocks`. Returns the trailing bytes. +fn read_blocks<'a>( + r: &mut Cursor<'a>, + version: Version, + mut consume: impl FnMut(usize, [u8; 4], [u8; 4], Cursor<'a>, &Option>) -> Result>, + blocks: &mut Vec, +) -> Result> { + const ALIGNMENT: usize = 4; + loop { + let head = r.peek(12)?; + if head.len() < 12 || !is_sig(&head) { + return r.rest(); + } + let signature = r.array::<4>()?; + let key = r.array::<4>()?; + let len = r.len_field(uses_long_length(version, &key))?; + let data = r.sub(len)?; + // Padding as `read_blocks_with` detects it: the smallest k in 0..=3 such that the next + // signature (or the end of the region) follows k bytes later. + let rest = r.peek(7)?; + let mut pad = None; + for k in 0..=3usize { + if k == rest.len() || rest.get(k..).is_some_and(is_sig) { + pad = Some(k); + break; + } + } + let default = TaggedBlock::default_padding((len % ALIGNMENT as u64) as usize, ALIGNMENT); + let pad = pad.unwrap_or_else(|| if rest.get(..default).is_some_and(|p| p.iter().all(|&b| b == 0)) { default } else { 0 }); + let pad_bytes = rest.get(..pad).map(<[u8]>::to_vec).unwrap_or_default(); + r.skip(pad as u64)?; + let padding = if pad_bytes.len() == default && pad_bytes.iter().all(|&b| b == 0) { None } else { Some(pad_bytes) }; + let (start, end) = (data.pos, data.end); + if consume(blocks.len(), signature, key, data, &padding)?.is_none() { + let data = Cursor { src: r.src, pos: start, end }.rest()?; + blocks.push(TaggedBlock { signature, key, data, padding }); + } + } +} + +/// [`LayerInfo::read_body`] from a cursor over the body: the records are read into memory, and +/// each channel's encoded bytes become a [`Stored`] range after its compression field. +fn read_layer_info(mut r: Cursor<'_>, version: Version) -> Result { + let body_start = r.pos; + let count = r.i16()?; + let merged_alpha = count < 0; + let n = count.unsigned_abs(); + // Minimum record size, as `read_body` checks it. + if u64::from(n) * 34 > r.remaining() { + return Err(r.eof(u64::from(n) * 34)); + } + // The records' total size is only known once they are parsed: read a chunk and grow it until + // they all fit (or the body is exhausted, when their error is the body's). + let mut chunk = RECORDS_CHUNK.min(r.remaining()); + let (layers, all_lens, used) = loop { + let buf = r.peek(chunk)?; + let mut rr = Reader::new(&buf); + let mut layers = Vec::with_capacity(usize::from(n)); + let mut all_lens = Vec::with_capacity(usize::from(n)); + let parsed = (0..n).try_for_each(|_| { + let (rec, lens) = LayerRecord::read(&mut rr, version)?; + layers.push(rec); + all_lens.push(lens); + Ok::<_, PsdError>(()) + }); + match parsed { + Ok(()) => break (layers, all_lens, rr.pos() as u64), + Err(PsdError::UnexpectedEof { .. }) if chunk < r.remaining() => chunk = chunk.saturating_mul(2).min(r.remaining()), + Err(e) => return Err(e), + } + }; + r.skip(used)?; + let mut layers = layers; + for (rec, lens) in layers.iter_mut().zip(all_lens) { + for (ch, len) in rec.channels.iter_mut().zip(lens) { + if len == 0 { + continue; + } + if len == 1 { + return Err(PsdError::invalid("channel data length 1")); + } + ch.compression = Some(Compression::from_u16(r.u16()?)); + let data = r.sub(len - 2)?; + ch.stored = Some(Stored { source: data.src.clone(), offset: data.pos, len: data.remaining() }); + } + } + let body_len = r.pos - body_start; + let rest = r.rest()?; + let padding = if rest.len() as u64 == body_len % 2 && rest.iter().all(|&b| b == 0) { None } else { Some(rest) }; + Ok(LayerInfo { merged_alpha, layers, padding }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testgen; + + /// Every generated file, plus the layered 16/32-bit ones with their layers in a global block. + fn samples() -> Vec> { + let mut v = Vec::new(); + for case in testgen::all_cases() { + let mut file = case.file; + v.push(file.to_bytes().unwrap()); + let key = match file.header.depth { + 16 => *b"Lr16", + 32 => *b"Lr32", + _ => continue, + }; + if file.layer_info.as_ref().is_some_and(|i| !i.layers.is_empty()) { + file.global_layer_mask.get_or_insert_with(GlobalLayerMask::default); + file.layer_info_placement = LayerInfoPlacement::GlobalBlock { index: 0, signature: *b"8BIM", key, padding: None }; + v.push(file.to_bytes().unwrap()); + } + } + v + } + + fn same(bytes: Vec) { + let eager = PsdFile::from_bytes(&bytes).unwrap(); + let lazy = PsdFile::from_source(Source::new(bytes.clone())).unwrap(); + assert_eq!(lazy, eager); + assert_eq!(lazy.to_bytes().unwrap(), bytes); + assert!(lazy.layers().iter().flat_map(|l| &l.channels).all(|c| c.data.is_empty())); + assert!(lazy.image_data.data.is_empty() && lazy.image_data.stored.is_some()); + } + + #[test] + fn lazy_parse_matches_the_eager_one() { + for bytes in samples() { + same(bytes); + } + } + + #[test] + fn lazy_parse_fails_where_the_eager_one_does() { + for bytes in samples() { + for cut in [1, 2, 7, 26, 40, bytes.len() / 3, bytes.len() / 2, bytes.len() - 3] { + let Some(b) = bytes.get(..cut) else { continue }; + let eager = PsdFile::from_bytes(b); + let lazy = PsdFile::from_source(Source::new(b.to_vec())); + // A truncated merged image is only noticed when it is read. + match eager { + Ok(eager) => assert_eq!(lazy.unwrap(), eager, "cut at {cut}"), + Err(e) => { + let records_ok = PsdFile::from_bytes_and_merged_recovering(b, |_| false, |_| true).is_ok(); + assert!(lazy.is_err() || records_ok, "cut at {cut}: eager {e:?}"); + } + } + } + } + } +} diff --git a/crates/psd/src/lib.rs b/crates/psd/src/lib.rs index 5248456d52..f5ffbed8b7 100644 --- a/crates/psd/src/lib.rs +++ b/crates/psd/src/lib.rs @@ -61,13 +61,16 @@ pub mod header; pub mod image_data; mod io; pub mod layer; +mod lazy; pub mod metadata; pub mod path; pub mod patterns; pub mod pixels; pub mod resources; +pub mod rows; pub mod shc; pub mod slices; +pub mod source; pub mod tagged; #[cfg(any(test, feature = "testgen"))] // Test-data generator (tests and fuzz seeds only): failing loudly is the point. @@ -87,5 +90,9 @@ pub use image_data::ImageData; pub use layer::{BlendingRanges, ChannelData, LayerFlags, LayerInfo, LayerMask, LayerRecord, MaskData, MaskParameters, RealMask, Rect}; pub use pixels::{GrayImage, Layer, RgbaImage}; pub use resources::{ImageResource, ResolutionInfo, ResourceData}; +pub use rows::RowDecoder; +#[cfg(any(unix, windows))] +pub use source::FileSource; +pub use source::{ReadAt, Source, Stored}; pub use tagged::{BlockData, SectionDivider, SectionType, TaggedBlock}; pub use tree::LayerNode; diff --git a/crates/psd/src/rows.rs b/crates/psd/src/rows.rs new file mode 100644 index 0000000000..6413c801df --- /dev/null +++ b/crates/psd/src/rows.rs @@ -0,0 +1,234 @@ +//! Decoding a channel a band of rows at a time: [`ChannelData::row_decoder`]. +//! +//! [`ChannelData::decode`] returns the whole plane, so turning a layer into tiles holds every +//! channel's plane plus an interleaved copy at once: for a 20000×15000 RGBA layer that is 2.4 GB +//! on top of the tiles. A [`RowDecoder`] hands out any band of rows instead. RLE and raw channels +//! are read band by band (a channel still in its [`Stored`](crate::Stored) source reads only the +//! band's bytes: RLE's per-row byte counts give each row's offset); ZIP streams cannot be entered +//! part way, so a ZIP channel is inflated once, when the decoder is made. + +use std::borrow::Cow; + +use crate::compression::{Compression, MAX_DECODED_BYTES, PlaneLayout, decode_planes, packbits}; +use crate::error::{PsdError, Result}; +use crate::header::Version; +use crate::layer::ChannelData; +use crate::source::Stored; + +/// Where a channel's encoded bytes are. +enum Bytes<'a> { + Memory(&'a [u8]), + Source(&'a Stored), +} + +impl Bytes<'_> { + fn len(&self) -> u64 { + match self { + Bytes::Memory(b) => b.len() as u64, + Bytes::Source(s) => s.len, + } + } + + fn get(&self, at: u64, len: u64) -> Result> { + match self { + Bytes::Memory(b) => { + let range = usize::try_from(at).ok().zip(usize::try_from(len).ok()).and_then(|(a, n)| b.get(a..a.checked_add(n)?)); + range + .map(Cow::Borrowed) + .ok_or(PsdError::UnexpectedEof { offset: usize::try_from(at).unwrap_or(usize::MAX), needed: usize::try_from(len).unwrap_or(usize::MAX) }) + } + Bytes::Source(s) => { + if at.checked_add(len).is_none_or(|end| end > s.len) { + return Err(PsdError::UnexpectedEof { + offset: usize::try_from(at).unwrap_or(usize::MAX), + needed: usize::try_from(len).unwrap_or(usize::MAX), + }); + } + s.read_part(at, len).map(Cow::Owned) + } + } + } +} + +enum Kind<'a> { + /// Rows at `row * row_bytes`. + Raw(Bytes<'a>), + /// `offsets[r]..offsets[r + 1]` are row `r`'s PackBits bytes (offsets from the start of the + /// data, after the count table). + Rle { bytes: Bytes<'a>, offsets: Vec }, + /// The whole decoded plane. + Decoded(Vec), + /// No data and no rows. + Empty, +} + +/// Decodes bands of rows of one channel: see the [module docs](self). +pub struct RowDecoder<'a> { + kind: Kind<'a>, + row_bytes: usize, + height: usize, +} + +impl ChannelData { + /// A decoder of this channel's rows, for a `width × height` plane. Fails where + /// [`Self::decode`] would on everything it can check without decoding the rows (sizes, the + /// RLE count table, the whole of a ZIP stream); a damaged row fails when its band is decoded. + pub fn row_decoder(&self, width: usize, height: usize, depth: u16, version: Version) -> Result> { + let layout = PlaneLayout { planes: 1, width, height, depth, version }; + let total = layout.decoded_len()?; + let row_bytes = layout.row_bytes(); + let bytes = match &self.stored { + Some(s) => Bytes::Source(s), + None => Bytes::Memory(&self.data), + }; + let kind = match self.compression { + None if total == 0 => Kind::Empty, + None => return Err(PsdError::invalid("channel has no data")), + Some(Compression::Raw) => { + if bytes.len() < total as u64 { + return Err(PsdError::UnexpectedEof { offset: usize::try_from(bytes.len()).unwrap_or(usize::MAX), needed: total - bytes.len() as usize }); + } + Kind::Raw(bytes) + } + Some(Compression::Rle) => { + let size: u64 = if version.is_psb() { 4 } else { 2 }; + let table_len = (height as u64).saturating_mul(size); + if table_len > bytes.len() { + return Err(PsdError::UnexpectedEof { offset: 0, needed: usize::try_from(table_len - bytes.len()).unwrap_or(usize::MAX) }); + } + let table = bytes.get(0, table_len)?; + let mut offsets = Vec::new(); + offsets.try_reserve_exact(height + 1).map_err(|_| PsdError::LimitExceeded("not enough memory for the RLE row table"))?; + let mut at = table_len; + offsets.push(at); + for c in table.chunks_exact(size as usize) { + let n = match *c { + [a, b] => u64::from(u16::from_be_bytes([a, b])), + [a, b, c, d] => u64::from(u32::from_be_bytes([a, b, c, d])), + _ => 0, + }; + at = at.saturating_add(n); + offsets.push(at); + } + let data_len = bytes.len() - table_len; + // As `decode_planes` checks: PackBits expands at most 64x. + if (total as u64) > data_len.saturating_mul(64) { + return Err(PsdError::Decompress("RLE data too short for declared size".into())); + } + if at > bytes.len() { + return Err(PsdError::UnexpectedEof { + offset: usize::try_from(bytes.len()).unwrap_or(usize::MAX), + needed: usize::try_from(at - bytes.len()).unwrap_or(usize::MAX), + }); + } + Kind::Rle { bytes, offsets } + } + Some(c) => Kind::Decoded(decode_planes(c, &self.encoded()?, &layout)?), + }; + Ok(RowDecoder { kind, row_bytes, height }) + } +} + +impl RowDecoder<'_> { + /// Bytes per decoded row. + pub fn row_bytes(&self) -> usize { + self.row_bytes + } + + /// Number of rows. + pub fn height(&self) -> usize { + self.height + } + + /// Rows `r0..r1` as planar big-endian samples (`(r1 - r0) × row_bytes` bytes). + pub fn rows(&self, r0: usize, r1: usize) -> Result> { + if r0 > r1 || r1 > self.height { + return Err(PsdError::invalid(format!("rows {r0}..{r1} outside 0..{}", self.height))); + } + let n = (r1 - r0).checked_mul(self.row_bytes).ok_or(PsdError::LimitExceeded("band size overflow"))?; + if n as u64 > MAX_DECODED_BYTES { + return Err(PsdError::LimitExceeded("decoded channel data exceeds MAX_DECODED_BYTES")); + } + let mut out = Vec::new(); + out.try_reserve_exact(n).map_err(|_| PsdError::LimitExceeded("not enough memory for the decoded channel data"))?; + match &self.kind { + Kind::Empty => {} + Kind::Raw(bytes) => out.extend_from_slice(&bytes.get((r0 * self.row_bytes) as u64, n as u64)?), + Kind::Decoded(plane) => { + out.extend_from_slice(plane.get(r0 * self.row_bytes..r1 * self.row_bytes).ok_or_else(|| PsdError::invalid("decoded plane too short"))?) + } + Kind::Rle { bytes, offsets } => { + let (Some(&start), Some(&end)) = (offsets.get(r0), offsets.get(r1)) else { + return Err(PsdError::invalid("RLE row table too short")); + }; + let band = bytes.get(start, end - start)?; + for r in r0..r1 { + let (Some(&a), Some(&b)) = (offsets.get(r), offsets.get(r + 1)) else { + return Err(PsdError::invalid("RLE row table too short")); + }; + let row = band.get((a - start) as usize..(b - start) as usize).ok_or_else(|| PsdError::invalid("RLE row outside its band"))?; + packbits::decode_into(row, self.row_bytes, &mut out)?; + } + } + } + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use crate::source::Source; + use crate::{PsdFile, testgen}; + + /// Every channel of every generated file, decoded in uneven bands from memory and from a + /// source, is its whole-plane decode. + #[test] + fn bands_join_up_to_the_whole_plane() { + let mut checked = 0; + for case in testgen::all_cases() { + let bytes = case.file.to_bytes().unwrap(); + let eager = PsdFile::from_bytes(&bytes).unwrap(); + let lazy = PsdFile::from_source(Source::new(bytes)).unwrap(); + let (d, v) = (eager.header.depth, eager.header.version); + for file in [&eager, &lazy] { + for rec in file.layers() { + for ch in &rec.channels { + let (w, h) = rec.channel_rect(ch.id).size().unwrap(); + let whole = ch.decode(w, h, d, v); + let dec = ch.row_decoder(w, h, d, v); + let (Ok(whole), Ok(dec)) = (whole, dec) else { continue }; + let mut joined = Vec::new(); + let mut r = 0; + for step in [1, 3, 2, 7].iter().cycle() { + if r >= h { + break; + } + let r1 = (r + step).min(h); + joined.extend(dec.rows(r, r1).unwrap()); + r = r1; + } + assert_eq!(joined, whole, "{}", case.name); + assert!(dec.rows(0, h + 1).is_err()); + checked += 1; + } + } + } + } + assert!(checked > 100, "{checked}"); + } + + #[test] + fn a_damaged_rle_row_fails_its_band() { + use crate::Compression; + use crate::layer::ChannelData; + // Two rows of 4 bytes: a valid row, then one that ends early. + let mut data = vec![0, 2, 0, 2]; + data.extend([0xFD, 7]); // header -3: 1 - (-3) = 4 copies of 7 + data.extend([0x01, 9]); // a literal of 2 bytes, only 1 present + let ch = ChannelData::new(0, Some(Compression::Rle), data); + let dec = ch.row_decoder(4, 2, 8, crate::Version::Psd).unwrap(); + assert_eq!(dec.rows(0, 1).unwrap(), vec![7; 4]); + assert!(dec.rows(1, 2).is_err()); + assert!(ch.decode(4, 2, 8, crate::Version::Psd).is_err()); + } +} diff --git a/crates/psd/src/source.rs b/crates/psd/src/source.rs new file mode 100644 index 0000000000..2e7c8039dd --- /dev/null +++ b/crates/psd/src/source.rs @@ -0,0 +1,270 @@ +//! Random-access byte sources, for parsing a file without reading all of it. +//! +//! [`PsdFile::from_source`](crate::PsdFile::from_source) reads the structure of a file (header, +//! resources, layer records, tagged blocks) and leaves the bulk of it, the encoded channel data +//! and the merged image, where it is: each [`ChannelData`](crate::ChannelData) records a +//! [`Stored`] range instead of a copy, and the bytes are read when the channel is decoded. A +//! 2 GB layered file then never needs 2 GB of memory just to be opened. + +use std::borrow::Cow; +use std::fmt; +use std::sync::Arc; + +use crate::error::{PsdError, Result}; + +/// Bytes that can be read at any offset, from any thread. +/// +/// Implemented for byte buffers and, on native targets, for [`std::fs::File`] (as +/// [`FileSource`]). The source must not change while a file parsed from it is in use: a +/// [`Stored`] range is an offset into it. +pub trait ReadAt: Send + Sync { + /// Total length in bytes. + fn len(&self) -> u64; + + /// `true` when the source holds no bytes. + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Fills `buf` with the bytes at `offset..offset + buf.len()`. Fails, without a partial + /// result, when the range is not all inside the source or the read fails. + fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<()>; +} + +fn out_of_range(offset: u64, needed: usize, len: u64) -> PsdError { + PsdError::UnexpectedEof { + offset: usize::try_from(offset).unwrap_or(usize::MAX), + needed: needed.saturating_sub(usize::try_from(len.saturating_sub(offset)).unwrap_or(usize::MAX)), + } +} + +impl ReadAt for [u8] { + fn len(&self) -> u64 { + <[u8]>::len(self) as u64 + } + + fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<()> { + let src = usize::try_from(offset) + .ok() + .and_then(|start| self.get(start..start.checked_add(buf.len())?)) + .ok_or_else(|| out_of_range(offset, buf.len(), ReadAt::len(self)))?; + buf.copy_from_slice(src); + Ok(()) + } +} + +impl ReadAt for Vec { + fn len(&self) -> u64 { + self.as_slice().len() as u64 + } + + fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<()> { + self.as_slice().read_at(offset, buf) + } +} + +impl ReadAt for Arc { + fn len(&self) -> u64 { + (**self).len() + } + + fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<()> { + (**self).read_at(offset, buf) + } +} + +/// Largest single read from a [`FileSource`]. +#[cfg(any(unix, windows))] +const MAX_READ: usize = 64 << 20; + +/// An open file read at offsets (`pread` on Unix, `seek_read` on Windows), with its length taken +/// when it was opened. Reads from several threads at once need no lock. +#[cfg(any(unix, windows))] +#[derive(Debug)] +pub struct FileSource { + file: std::fs::File, + len: u64, +} + +#[cfg(any(unix, windows))] +impl FileSource { + /// Opens `path` for reading. + pub fn open(path: &std::path::Path) -> std::io::Result { + Self::new(std::fs::File::open(path)?) + } + + /// Wraps an open file. + pub fn new(file: std::fs::File) -> std::io::Result { + let len = file.metadata()?.len(); + Ok(FileSource { file, len }) + } + + /// The file. + pub fn file(&self) -> &std::fs::File { + &self.file + } +} + +#[cfg(any(unix, windows))] +impl ReadAt for FileSource { + fn len(&self) -> u64 { + self.len + } + + fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<()> { + if offset.checked_add(buf.len() as u64).is_none_or(|end| end > self.len) { + return Err(out_of_range(offset, buf.len(), self.len)); + } + let n = buf.len(); + let failed = |e: std::io::Error| PsdError::Read(format!("reading {n} bytes at offset {offset}: {e}")); + // Bounded reads, as `photocraft-format`'s `read_file` makes them: some Windows file + // systems reject a single read of gigabytes (#375). + let mut done = 0usize; + while done < n { + let Some(rest) = buf.get_mut(done..n.min(done + MAX_READ)) else { break }; + let at = offset + done as u64; + #[cfg(unix)] + let read = { + use std::os::unix::fs::FileExt; + self.file.read_at(rest, at) + }; + #[cfg(windows)] + let read = { + use std::os::windows::fs::FileExt; + self.file.seek_read(rest, at) + }; + match read { + Ok(0) => return Err(failed(std::io::ErrorKind::UnexpectedEof.into())), + Ok(k) => done += k, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Err(e) => return Err(failed(e)), + } + } + Ok(()) + } +} + +/// A shared [`ReadAt`]: what [`Stored`] ranges point into. Two handles are equal when they are +/// the same source. +#[derive(Clone)] +pub struct Source(Arc); + +impl Source { + /// Shares `source`. + pub fn new(source: impl ReadAt + 'static) -> Self { + Source(Arc::new(source)) + } + + /// Shares an already shared source. + pub fn from_arc(source: Arc) -> Self { + Source(source) + } + + /// The source. + pub fn get(&self) -> &dyn ReadAt { + &*self.0 + } + + /// Reads `len` bytes at `offset` into a new buffer. The buffer is reserved fallibly: a + /// range larger than memory is an error, not an abort. + pub fn read_vec(&self, offset: u64, len: u64) -> Result> { + let n = usize::try_from(len).map_err(|_| PsdError::LimitExceeded("length exceeds address space"))?; + if offset.checked_add(len).is_none_or(|end| end > self.0.len()) { + return Err(out_of_range(offset, n, self.0.len())); + } + let mut out = Vec::new(); + out.try_reserve_exact(n).map_err(|_| PsdError::LimitExceeded("not enough memory to read the data"))?; + out.resize(n, 0); + self.0.read_at(offset, &mut out)?; + Ok(out) + } +} + +impl fmt::Debug for Source { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Source({} bytes)", self.0.len()) + } +} + +impl PartialEq for Source { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } +} + +impl Eq for Source {} + +/// Encoded bytes left in their [`Source`] by a lazy parse: `len` bytes at `offset`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Stored { + /// Where the bytes are. + pub source: Source, + /// Offset of the first byte. + pub offset: u64, + /// Number of bytes. + pub len: u64, +} + +impl Stored { + /// Reads the bytes. + pub fn read(&self) -> Result> { + self.source.read_vec(self.offset, self.len) + } + + /// Reads `len` bytes starting `at` bytes into the range. + pub fn read_part(&self, at: u64, len: u64) -> Result> { + if at.checked_add(len).is_none_or(|end| end > self.len) { + return Err(PsdError::invalid("read past the end of stored data")); + } + self.source.read_vec(self.offset + at, len) + } +} + +/// The bytes of a value that is either in memory (`data`) or still in its source (`stored`). +pub(crate) fn encoded<'a>(data: &'a [u8], stored: Option<&Stored>) -> Result> { + match stored { + Some(s) => s.read().map(Cow::Owned), + None => Ok(Cow::Borrowed(data)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn slice_reads_are_bounds_checked() { + let v = vec![1u8, 2, 3, 4]; + let mut b = [0u8; 2]; + v.read_at(2, &mut b).unwrap(); + assert_eq!(b, [3, 4]); + assert!(v.read_at(3, &mut b).is_err()); + assert!(v.read_at(u64::MAX, &mut b).is_err()); + let s = Source::new(v); + assert_eq!(s.read_vec(1, 2).unwrap(), vec![2, 3]); + assert!(s.read_vec(1, u64::MAX).is_err()); + let st = Stored { source: s.clone(), offset: 1, len: 3 }; + assert_eq!(st.read().unwrap(), vec![2, 3, 4]); + assert_eq!(st.read_part(1, 2).unwrap(), vec![3, 4]); + assert!(st.read_part(2, 2).is_err()); + assert_eq!(st.source, s); + assert_ne!(st.source, Source::new(vec![1u8, 2, 3, 4])); + } + + #[cfg(any(unix, windows))] + #[test] + fn file_source_reads_at_offsets() { + use std::io::Write; + let dir = std::env::temp_dir().join(format!("photocraft-psd-source-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("f.bin"); + std::fs::File::create(&path).unwrap().write_all(&[9, 8, 7, 6, 5]).unwrap(); + let f = FileSource::open(&path).unwrap(); + assert_eq!(ReadAt::len(&f), 5); + let mut b = [0u8; 3]; + f.read_at(2, &mut b).unwrap(); + assert_eq!(b, [7, 6, 5]); + assert!(f.read_at(3, &mut b).is_err()); + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/crates/psd/src/tagged.rs b/crates/psd/src/tagged.rs index 7728d53fd1..1574b1faab 100644 --- a/crates/psd/src/tagged.rs +++ b/crates/psd/src/tagged.rs @@ -160,7 +160,7 @@ impl TaggedBlock { String::from_utf8_lossy(&self.key).into_owned() } - fn default_padding(len: usize, alignment: usize) -> usize { + pub(crate) fn default_padding(len: usize, alignment: usize) -> usize { (alignment - len % alignment) % alignment } @@ -436,7 +436,7 @@ fn parse_section(d: &[u8]) -> Result { /// `8BIM` / `8B64`, or the markers `MIB8` / `46B8` that [`crate::tiff`] gives a block kept /// verbatim from a little-endian TIFF when its layout is unknown (its data is in that order). -fn is_sig(b: &[u8]) -> bool { +pub(crate) fn is_sig(b: &[u8]) -> bool { b.starts_with(b"8BIM") || b.starts_with(b"8B64") || TaggedBlock::is_foreign_order_signature(b) } diff --git a/crates/psd/src/testgen.rs b/crates/psd/src/testgen.rs index d2d4784cb9..5912009d41 100644 --- a/crates/psd/src/testgen.rs +++ b/crates/psd/src/testgen.rs @@ -113,7 +113,7 @@ pub fn merged_only(version: Version, mode: ColorMode, depth: u16, compression: C for c in 0..channels { planes.extend(pattern_plane(w as usize, h as usize, depth, u32::from(c))); } - let image_data = ImageData::encode(compression, &planes, &header).unwrap_or(ImageData { compression, data: Vec::new() }); + let image_data = ImageData::encode(compression, &planes, &header).unwrap_or(ImageData::new(compression, Vec::new())); PsdFile { header, color_mode_data: color_mode_data(mode), @@ -179,7 +179,7 @@ fn raster_layer(g: LayerGen<'_>) -> LayerRecord { fn group_record(header: &Header, name: &str, kind: SectionType, blend: BlendMode, id: u32) -> LayerRecord { let cc = mode_channels(header.color_mode) as i16; - let channels = (-1..cc).map(|c| ChannelData { id: c, compression: Some(Compression::Raw), data: Vec::new() }).collect(); + let channels = (-1..cc).map(|c| ChannelData::new(c, Some(Compression::Raw), Vec::new())).collect(); let lsct = if kind == SectionType::BoundingDivider { TaggedBlock::section_divider(kind, None, None) } else { diff --git a/crates/raster/src/lib.rs b/crates/raster/src/lib.rs index f3778ccbe5..c6616d86d5 100644 --- a/crates/raster/src/lib.rs +++ b/crates/raster/src/lib.rs @@ -35,6 +35,24 @@ impl fmt::Display for AllocationError { impl std::error::Error for AllocationError {} +/// One shared tile per pixel value (and tile size), for [`Surface::share_uniform_tiles`]. +#[derive(Debug, Default)] +pub struct TileInterner { + tiles: std::collections::HashMap, Arc>>, +} + +impl TileInterner { + /// Number of distinct uniform tiles held. + pub fn len(&self) -> usize { + self.tiles.values().map(|m| m.len()).sum() + } + + /// `true` when no tile is held. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + /// Pixel storage for one tile: `TILE_SIZE² × bytes_per_pixel`, row-major, interleaved channels. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Tile { @@ -415,6 +433,28 @@ impl Surface { out } + /// Makes every tile that holds one pixel value throughout share the interner's tile for that + /// value: a white background, an opaque mask or a solid fill then costs one tile however + /// large it is, and across every surface that shares `interner`. Tiles are copy-on-write, so + /// editing one splits it off again. Returns the bytes no longer held by this surface alone. + pub fn share_uniform_tiles(&mut self, interner: &mut TileInterner) -> usize { + let bpp = self.format.bytes_per_pixel(); + let mut saved = 0; + for t in self.tiles.values_mut() { + let mut px = t.data.chunks_exact(bpp); + let Some(p0) = px.next() else { continue }; + if !px.all(|p| p == p0) { + continue; + } + let shared = interner.tiles.entry(t.data.len()).or_default().entry(p0.to_vec()).or_insert_with(|| t.clone()); + if !Arc::ptr_eq(shared, t) { + saved += t.data.len(); + *t = shared.clone(); + } + } + saved + } + /// Drop tiles that are entirely the default pixel. pub fn prune(&mut self) { let dp = self.default_pixel.as_ref(); @@ -1295,6 +1335,38 @@ mod tests { seen.len() } + /// Uniform tiles written pixel by pixel end up as one shared tile per value, across + /// surfaces, with every pixel unchanged; editing one splits it off. + #[test] + fn uniform_tiles_are_shared() { + let mut interner = TileInterner::default(); + let r = Rect::new(0, 0, 1024, 600); + let mut bytes = Vec::new(); + for y in 0..600 { + for x in 0..1024 { + // White, except a gradient in the tiles of column 3. + bytes.extend(if x >= 768 { [x as u8, y as u8, 0, 255] } else { [255, 255, 255, 255] }); + } + } + let mut a = Surface::from_interleaved(PixelFormat::RGBA8, r, &bytes); + let mut b = Surface::from_interleaved(PixelFormat::RGBA8, r, &bytes); + let before = a.to_interleaved(r); + assert_eq!(distinct_tiles(&a), 12); + // Rows 512..600 cover only part of their tiles: the rest is the default pixel, so those + // tiles are not uniform. + let saved = a.share_uniform_tiles(&mut interner); + assert_eq!(distinct_tiles(&a), 1 + 3 + 3); + assert_eq!(saved, 5 * TILE_SIZE as usize * TILE_SIZE as usize * 4); + b.share_uniform_tiles(&mut interner); + assert_eq!(interner.len(), 1); + assert!(Arc::ptr_eq(a.tile(TileCoord { tx: 0, ty: 0 }).unwrap(), b.tile(TileCoord { tx: 2, ty: 1 }).unwrap())); + assert_eq!(a.to_interleaved(r), before); + a.write_pixel(5, 5, &[0.0, 0.0, 0.0, 1.0]); + assert_eq!(b.pixel(5, 5), vec![1.0, 1.0, 1.0, 1.0]); + assert_eq!(a.pixel(5, 5), vec![0.0, 0.0, 0.0, 1.0]); + assert_eq!(a.pixel(300, 5), vec![1.0, 1.0, 1.0, 1.0]); + } + /// Converting or tone-mapping a surface keeps a shared tile shared (#4531) and gives the /// same pixels as converting each pixel: a filled background is one tile across the grid, /// plus some distinct tiles.