From 42bb9d35bba5c1074cce44fbc157785f3d29ad90 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 17 Nov 2026 09:58:18 -0500 Subject: [PATCH] tar: preserve current sparse, PAX size and CRC symlink behavior (#2012) * tar: port old GNU sparse extension traversal from Craft Anything Source: storytold/craft-anything#1973, 8ea2ec66a1bc80f0420f3695baac877a0cb9b9a5. Retains non-extractable sparse member kind and regression tests. * tar: preserve GNU CRC symlink checksum compatibility Source: storytold/craft-anything#1984, b226a475f7dce9710e3af7a069386fe7a9720df7; preserves nonzero checks and regular-file checksum validation. * tar: apply valid PAX sizes before legacy member size fields Source: storytold/craft-anything#2007, d29465c20b0d570ea4973be512a0cb4e212bb60f; metadata records still retain their own framing. * Record provenance of current TAR and CPIO compatibility fixes --------- Co-authored-by: RRFZ <247457381+Plastic-time@users.noreply.github.com> --- ATTRIBUTION.md | 5 ++ crates/tar/src/lib.rs | 31 ++++++-- crates/tar/src/tar_tests.rs | 152 ++++++++++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 7 deletions(-) diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 4c86afd61..1311506eb 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -119,9 +119,14 @@ The XML reader in `crates/plist/src/xml.rs` and its base64 compatibility helper `crates/palette` extracts the GPL text parser from AniCraft `crates/doc/src/swatch.rs` at `d875a7feb250e6ab06aa28a37fb78a0699e88330` (MIT OR Apache-2.0), introduced by Tanish Manku in `a64130a9da9bdde545463b852d95d6a6774b1dfd` (AniCraft #270). The import preserves Tanish as primary author. Neutral RGB/name adaptation separates application display-name sanitization and the `Untitled` policy; validation tests and integration are subsequent Dylan M. Taylor changes. +## TAR/CPIO compatibility updates + +The old GNU sparse-extension traversal, GNU CRC-symlink compatibility and PAX-size precedence fixes and their regression tests in `crates/tar` are ported from RRFZ's Craft Anything PRs #1973 (`8ea2ec66a1bc80f0420f3695baac877a0cb9b9a5`), #1984 (`b226a475f7dce9710e3af7a069386fe7a9720df7`) and #2007 (`d29465c20b0d570ea4973be512a0cb4e212bb60f`). Source is MIT OR Apache-2.0; RRFZ remains the primary Git author of each port. The PAX update adapts to the shared parser's existing `PaxOverrides` structure without changing the source policy. + ## Aligned stored ZIP entries The USDZ alignment requirement and compatibility fixture for `Writer::add_stored_aligned` come from TextureCraft `crates/formats/src/mesh_export/usd.rs`, introduced by bflatastic in `bdbb52c9` and maintained through `9eb20086` (MIT OR Apache-2.0). The shared API and alignment validation are subsequent Dylan M. Taylor work. Scene/material mapping remains in TextureCraft. + The raw gradient wire ordering in `craft-lottie::Gradient::flat_values` follows AniCraft's Lottie exporter by Tanish Manku (`63b39355`, `b9258b7b`), MIT OR Apache-2.0. Sanitization, animation fitting and numeric output policy remain caller-owned. diff --git a/crates/tar/src/lib.rs b/crates/tar/src/lib.rs index 7712fc2aa..af39d6a5e 100644 --- a/crates/tar/src/lib.rs +++ b/crates/tar/src/lib.rs @@ -175,17 +175,32 @@ fn tar_with_name_copy_limit(data: &[u8], max_members: usize, mut name_bytes_left break; } let typeflag = h.get(156).copied().unwrap_or(0); - let Some(mut size) = h.get(124..136).and_then(octal) else { + // Metadata records keep their own framing. For a member, PAX replaces the + // legacy size field entirely, so a valid override does not depend on its contents. + let override_size = match typeflag { + b'x' | b'g' | b'L' | b'K' => None, + _ => pax.size.take().or(pax.global_size), + }; + let Some(size) = override_size.or_else(|| h.get(124..136).and_then(octal)) else { out.warnings.push(format!("unreadable size at {pos:#x}")); out.length = pos as u64; break; }; - if !matches!(typeflag, b'x' | b'g' | b'L' | b'K') - && let Some(s) = pax.size.take().or(pax.global_size) - { - size = s; + let mut data_off = pos + BLOCK; + let old_gnu_sparse = typeflag == b'S' && h.get(257..265) == Some(b"ustar \0"); + if old_gnu_sparse { + // Old GNU sparse maps can continue in whole blocks before the stored extents. + let mut extended = h.get(482).is_some_and(|&flag| flag != 0); + while extended { + let Some(block) = r.slice(data_off, BLOCK) else { + out.warnings.push(format!("GNU sparse extension ends early at {data_off:#x}")); + out.length = data.len() as u64; + break 'headers; + }; + extended = block.get(504).is_some_and(|&flag| flag != 0); + data_off += BLOCK; + } } - let data_off = pos + BLOCK; let Some(next) = round_up(size, BLOCK as u64).and_then(|s| u64::try_from(data_off).ok()?.checked_add(s)).and_then(|n| usize::try_from(n).ok()) else { @@ -262,7 +277,7 @@ fn tar_with_name_copy_limit(data: &[u8], max_members: usize, mut name_bytes_left .or_else(|| Some(text(h.get(157..257).unwrap_or(&[])))) .filter(|l| !l.is_empty()); let mut kind = tar_kind(typeflag, &name); - if std::mem::take(&mut pax.sparse) || !pax.global_sparse.is_empty() { + if std::mem::take(&mut pax.sparse) || !pax.global_sparse.is_empty() || old_gnu_sparse { kind = MemberKind::Other; out.warnings.push(format!("{name}: GNU sparse reconstruction is unsupported; stored extents are not the complete file")); } @@ -523,6 +538,8 @@ pub fn cpio_with_checksum(data: &[u8], max_members: usize, checksum: &mut dyn Fn } if format == "cpio-crc" { match h.get(102..110).filter(|field| field.iter().all(u8::is_ascii_hexdigit)).and_then(hex_field).and_then(|n| u32::try_from(n).ok()) { + // GNU cpio writes the symlink target without a checksum. + Some(0) if kind == MemberKind::Symlink => {} Some(expected) => { if let Some(body) = r.slice(data_off, size) { match checksum(body) { diff --git a/crates/tar/src/tar_tests.rs b/crates/tar/src/tar_tests.rs index 38f72abd9..83dd18342 100644 --- a/crates/tar/src/tar_tests.rs +++ b/crates/tar/src/tar_tests.rs @@ -402,3 +402,155 @@ mod tar_global_pax { } } } + +// Ported from Craft Anything gnu_sparse_extensions.rs (RRFZ). +mod gnu_sparse_extensions { + use crate::testkit::tar_entries; + use crate::{MemberKind, member_data, tar}; + + /// Build an old GNU sparse header followed by the requested number of extension blocks. + fn archive(extension_blocks: usize) -> Vec { + let mut archive = tar_entries(&[("sparse.dat", b'S', &[b'x'; 512], ""), ("after.txt", b'0', b"after", "")]); + archive[257..265].copy_from_slice(b"ustar \0"); + archive[482] = u8::from(extension_blocks != 0); + archive[148..156].fill(b' '); + let sum: u32 = archive[..512].iter().map(|&byte| u32::from(byte)).sum(); + archive[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); + let mut extensions = vec![0; 512 * extension_blocks]; + for block in extensions.as_chunks_mut::<512>().0.iter_mut().take(extension_blocks.saturating_sub(1)) { + block[504] = 1; + } + archive.splice(512..512, extensions); + archive + } + + /// Actual file data starts after the entire sparse map. Later members remain readable. + #[test] + fn skips_old_gnu_sparse_extension_blocks() { + for blocks in [1, 2] { + let bytes = archive(blocks); + let listing = tar(&bytes, 8).unwrap(); + assert!(listing.complete, "{blocks} extensions: {:?}", listing.warnings); + assert_eq!(listing.members.len(), 2); + assert_eq!(listing.members[0].kind, MemberKind::Other); + assert_eq!(listing.members[0].data_offset, (512 * (blocks + 1)) as u64); + assert_eq!(listing.members[1].name, "after.txt"); + assert_eq!(member_data(&bytes, &listing.members[1]), Some(b"after".as_slice())); + } + } + + /// A missing sparse extension cannot turn padding into a complete archive. + #[test] + fn refuses_a_truncated_old_gnu_sparse_extension() { + let mut bytes = archive(2); + bytes.truncate(512 + 512 + 17); + let listing = tar(&bytes, 8).unwrap(); + assert!(!listing.complete); + assert!(listing.members.is_empty()); + assert!(listing.warnings.iter().any(|warning| warning.contains("sparse extension"))); + } + + /// A sparse member with no extensions still leaves the next header at its usual offset. + #[test] + fn keeps_the_next_member_when_no_sparse_extensions_follow() { + let bytes = archive(0); + let listing = tar(&bytes, 8).unwrap(); + assert!(listing.complete); + assert_eq!(listing.members[0].kind, MemberKind::Other); + assert_eq!(listing.members[1].name, "after.txt"); + } +} + +// Ported from Craft Anything cpio_symlink_crc.rs (RRFZ). +mod cpio_symlink_crc { + use crate::cpio; + use crate::testkit::cpio_newc; + + /// Convert a single-member newc fixture to CRC and set the member checksum. + fn archive(mode: u32, checksum: u32) -> Vec { + let mut bytes = cpio_newc(&[("entry", b"target", mode)]); + bytes[..6].copy_from_slice(b"070702"); + bytes[102..110].copy_from_slice(format!("{checksum:08x}").as_bytes()); + let trailer = (110 + "entry".len() + 1 + 3) & !3; + let trailer = (trailer + "target".len() + 3) & !3; + bytes[trailer..trailer + 6].copy_from_slice(b"070702"); + bytes + } + + /// GNU-style CRC symlinks carry a zero placeholder and keep the recorded target. + #[test] + fn accepts_a_symlink_with_no_stored_checksum() { + let listing = cpio(&archive(0o120777, 0), 10).unwrap(); + assert!(listing.complete); + assert_eq!(listing.members[0].link_target.as_deref(), Some("target")); + assert!(listing.warnings.is_empty(), "{:?}", listing.warnings); + } + + /// A nonzero symlink checksum and even a zero regular-file checksum are checked. + #[test] + fn still_checks_nonzero_symlink_and_zero_regular_file_checksums() { + assert!(cpio(&archive(0o120777, 647), 10).unwrap().warnings.is_empty()); + for bytes in [archive(0o120777, 646), archive(0o100644, 0)] { + assert!(cpio(&bytes, 10).unwrap().warnings.iter().any(|w| w.contains("checksum mismatch"))); + } + } +} + +// Ported from Craft Anything tar_pax_size_precedence.rs (RRFZ). +mod tar_pax_size_precedence { + use crate::testkit::tar_entries; + use crate::{member_data, tar}; + + /// Make an octal size unreadable while leaving the enclosing TAR header checksum valid. + fn corrupt_size(archive: &mut [u8], offset: usize) { + let header = &mut archive[offset..offset + 512]; + header[124] = b'9'; + header[148..156].fill(b' '); + let sum: u32 = header.iter().map(|&byte| u32::from(byte)).sum(); + header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); + } + + /// Local and inherited size records override the bad field and keep the following header aligned. + #[test] + fn valid_pax_size_controls_the_member_extent() { + for extension in *b"xg" { + let mut archive = tar_entries(&[("PaxHeader", extension, b"10 size=3\n", ""), ("file", b'0', b"abc", ""), ("next", b'0', b"def", "")]); + corrupt_size(&mut archive, 1024); + let list = tar(&archive, 10).unwrap(); + assert!(list.complete, "{:?}", list.warnings); + assert_eq!(list.members.len(), 2); + assert_eq!(member_data(&archive, &list.members[0]), Some(b"abc".as_slice())); + assert_eq!(member_data(&archive, &list.members[1]), Some(b"def".as_slice())); + } + } + + /// A local zero overrides a global size and does not consume the next header. + #[test] + fn a_local_zero_size_overrides_the_global_value() { + let mut archive = tar_entries(&[ + ("Global", b'g', b"12 size=600\n", ""), + ("Local", b'x', b"10 size=0\n", ""), + ("empty", b'0', b"", ""), + ("next", b'0', &[b'a'; 600], ""), + ]); + corrupt_size(&mut archive, 2048); + let list = tar(&archive, 10).unwrap(); + assert!(list.complete, "{:?}", list.warnings); + assert_eq!(list.members.iter().map(|m| m.size).collect::>(), [0, 600]); + } + + /// An ordinary header without an override and a PAX header itself still need readable sizes. + #[test] + fn rejects_bad_sizes_without_an_applicable_override() { + let mut plain = tar_entries(&[("file", b'0', b"abc", "")]); + corrupt_size(&mut plain, 0); + let mut with_global = tar_entries(&[("Global", b'g', b"10 size=3\n", ""), ("Local", b'x', b"10 size=0\n", ""), ("file", b'0', b"", "")]); + corrupt_size(&mut with_global, 1024); + for archive in [plain, with_global] { + let list = tar(&archive, 10).unwrap(); + assert!(!list.complete); + assert!(list.members.is_empty()); + assert!(list.warnings.iter().any(|warning| warning.contains("unreadable size"))); + } + } +}