tar: preserve current sparse, PAX size and CRC symlink behavior (#2012)

* tar: port old GNU sparse extension traversal from Craft Anything

Source: storytold/craft-anything#1973, 8ea2ec66a1bc80f0420f3695baac877a0cb9b9a5. Retains non-extractable sparse member kind and regression tests.

* tar: preserve GNU CRC symlink checksum compatibility

Source: storytold/craft-anything#1984, b226a475f7dce9710e3af7a069386fe7a9720df7; preserves nonzero checks and regular-file checksum validation.

* tar: apply valid PAX sizes before legacy member size fields

Source: storytold/craft-anything#2007, d29465c20b0d570ea4973be512a0cb4e212bb60f; metadata records still retain their own framing.

* Record provenance of current TAR and CPIO compatibility fixes

---------

Co-authored-by: RRFZ <247457381+Plastic-time@users.noreply.github.com>
This commit is contained in:
Dylan M. TaylorandRRFZ authored and GitHub committed 2026-11-17 09:58:18 -05:00
1 parent 90947c57d7
commit 42bb9d35bb
3 files changed
+181 -7

No files matched your search

+5
View File
@@ -119,9 +119,14 @@ The XML reader in `crates/plist/src/xml.rs` and its base64 compatibility helper
`crates/palette` extracts the GPL text parser from AniCraft `crates/doc/src/swatch.rs` at `d875a7feb250e6ab06aa28a37fb78a0699e88330` (MIT OR Apache-2.0), introduced by Tanish Manku in `a64130a9da9bdde545463b852d95d6a6774b1dfd` (AniCraft #270). The import preserves Tanish as primary author. Neutral RGB/name adaptation separates application display-name sanitization and the `Untitled` policy; validation tests and integration are subsequent Dylan M. Taylor changes.
## TAR/CPIO compatibility updates
The old GNU sparse-extension traversal, GNU CRC-symlink compatibility and PAX-size precedence fixes and their regression tests in `crates/tar` are ported from RRFZ's Craft Anything PRs #1973 (`8ea2ec66a1bc80f0420f3695baac877a0cb9b9a5`), #1984 (`b226a475f7dce9710e3af7a069386fe7a9720df7`) and #2007 (`d29465c20b0d570ea4973be512a0cb4e212bb60f`). Source is MIT OR Apache-2.0; RRFZ remains the primary Git author of each port. The PAX update adapts to the shared parser's existing `PaxOverrides` structure without changing the source policy.
## Aligned stored ZIP entries
The USDZ alignment requirement and compatibility fixture for `Writer::add_stored_aligned` come from TextureCraft `crates/formats/src/mesh_export/usd.rs`, introduced by bflatastic in `bdbb52c9` and maintained through `9eb20086` (MIT OR Apache-2.0). The shared API and alignment validation are subsequent Dylan M. Taylor work. Scene/material mapping remains in TextureCraft.
The raw gradient wire ordering in `craft-lottie::Gradient::flat_values` follows
AniCraft's Lottie exporter by Tanish Manku (`63b39355`, `b9258b7b`), MIT OR Apache-2.0.
Sanitization, animation fitting and numeric output policy remain caller-owned.
+24 -7
View File
@@ -175,17 +175,32 @@ fn tar_with_name_copy_limit(data: &[u8], max_members: usize, mut name_bytes_left
break;
}
let typeflag = h.get(156).copied().unwrap_or(0);
let Some(mut size) = h.get(124..136).and_then(octal) else {
// Metadata records keep their own framing. For a member, PAX replaces the
// legacy size field entirely, so a valid override does not depend on its contents.
let override_size = match typeflag {
b'x' | b'g' | b'L' | b'K' => None,
_ => pax.size.take().or(pax.global_size),
};
let Some(size) = override_size.or_else(|| h.get(124..136).and_then(octal)) else {
out.warnings.push(format!("unreadable size at {pos:#x}"));
out.length = pos as u64;
break;
};
if !matches!(typeflag, b'x' | b'g' | b'L' | b'K')
&& let Some(s) = pax.size.take().or(pax.global_size)
{
size = s;
let mut data_off = pos + BLOCK;
let old_gnu_sparse = typeflag == b'S' && h.get(257..265) == Some(b"ustar \0");
if old_gnu_sparse {
// Old GNU sparse maps can continue in whole blocks before the stored extents.
let mut extended = h.get(482).is_some_and(|&flag| flag != 0);
while extended {
let Some(block) = r.slice(data_off, BLOCK) else {
out.warnings.push(format!("GNU sparse extension ends early at {data_off:#x}"));
out.length = data.len() as u64;
break 'headers;
};
extended = block.get(504).is_some_and(|&flag| flag != 0);
data_off += BLOCK;
}
}
let data_off = pos + BLOCK;
let Some(next) =
round_up(size, BLOCK as u64).and_then(|s| u64::try_from(data_off).ok()?.checked_add(s)).and_then(|n| usize::try_from(n).ok())
else {
@@ -262,7 +277,7 @@ fn tar_with_name_copy_limit(data: &[u8], max_members: usize, mut name_bytes_left
.or_else(|| Some(text(h.get(157..257).unwrap_or(&[]))))
.filter(|l| !l.is_empty());
let mut kind = tar_kind(typeflag, &name);
if std::mem::take(&mut pax.sparse) || !pax.global_sparse.is_empty() {
if std::mem::take(&mut pax.sparse) || !pax.global_sparse.is_empty() || old_gnu_sparse {
kind = MemberKind::Other;
out.warnings.push(format!("{name}: GNU sparse reconstruction is unsupported; stored extents are not the complete file"));
}
@@ -523,6 +538,8 @@ pub fn cpio_with_checksum(data: &[u8], max_members: usize, checksum: &mut dyn Fn
}
if format == "cpio-crc" {
match h.get(102..110).filter(|field| field.iter().all(u8::is_ascii_hexdigit)).and_then(hex_field).and_then(|n| u32::try_from(n).ok()) {
// GNU cpio writes the symlink target without a checksum.
Some(0) if kind == MemberKind::Symlink => {}
Some(expected) => {
if let Some(body) = r.slice(data_off, size) {
match checksum(body) {
+152
View File
@@ -402,3 +402,155 @@ mod tar_global_pax {
}
}
}
// Ported from Craft Anything gnu_sparse_extensions.rs (RRFZ).
mod gnu_sparse_extensions {
use crate::testkit::tar_entries;
use crate::{MemberKind, member_data, tar};
/// Build an old GNU sparse header followed by the requested number of extension blocks.
fn archive(extension_blocks: usize) -> Vec<u8> {
let mut archive = tar_entries(&[("sparse.dat", b'S', &[b'x'; 512], ""), ("after.txt", b'0', b"after", "")]);
archive[257..265].copy_from_slice(b"ustar \0");
archive[482] = u8::from(extension_blocks != 0);
archive[148..156].fill(b' ');
let sum: u32 = archive[..512].iter().map(|&byte| u32::from(byte)).sum();
archive[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes());
let mut extensions = vec![0; 512 * extension_blocks];
for block in extensions.as_chunks_mut::<512>().0.iter_mut().take(extension_blocks.saturating_sub(1)) {
block[504] = 1;
}
archive.splice(512..512, extensions);
archive
}
/// Actual file data starts after the entire sparse map. Later members remain readable.
#[test]
fn skips_old_gnu_sparse_extension_blocks() {
for blocks in [1, 2] {
let bytes = archive(blocks);
let listing = tar(&bytes, 8).unwrap();
assert!(listing.complete, "{blocks} extensions: {:?}", listing.warnings);
assert_eq!(listing.members.len(), 2);
assert_eq!(listing.members[0].kind, MemberKind::Other);
assert_eq!(listing.members[0].data_offset, (512 * (blocks + 1)) as u64);
assert_eq!(listing.members[1].name, "after.txt");
assert_eq!(member_data(&bytes, &listing.members[1]), Some(b"after".as_slice()));
}
}
/// A missing sparse extension cannot turn padding into a complete archive.
#[test]
fn refuses_a_truncated_old_gnu_sparse_extension() {
let mut bytes = archive(2);
bytes.truncate(512 + 512 + 17);
let listing = tar(&bytes, 8).unwrap();
assert!(!listing.complete);
assert!(listing.members.is_empty());
assert!(listing.warnings.iter().any(|warning| warning.contains("sparse extension")));
}
/// A sparse member with no extensions still leaves the next header at its usual offset.
#[test]
fn keeps_the_next_member_when_no_sparse_extensions_follow() {
let bytes = archive(0);
let listing = tar(&bytes, 8).unwrap();
assert!(listing.complete);
assert_eq!(listing.members[0].kind, MemberKind::Other);
assert_eq!(listing.members[1].name, "after.txt");
}
}
// Ported from Craft Anything cpio_symlink_crc.rs (RRFZ).
mod cpio_symlink_crc {
use crate::cpio;
use crate::testkit::cpio_newc;
/// Convert a single-member newc fixture to CRC and set the member checksum.
fn archive(mode: u32, checksum: u32) -> Vec<u8> {
let mut bytes = cpio_newc(&[("entry", b"target", mode)]);
bytes[..6].copy_from_slice(b"070702");
bytes[102..110].copy_from_slice(format!("{checksum:08x}").as_bytes());
let trailer = (110 + "entry".len() + 1 + 3) & !3;
let trailer = (trailer + "target".len() + 3) & !3;
bytes[trailer..trailer + 6].copy_from_slice(b"070702");
bytes
}
/// GNU-style CRC symlinks carry a zero placeholder and keep the recorded target.
#[test]
fn accepts_a_symlink_with_no_stored_checksum() {
let listing = cpio(&archive(0o120777, 0), 10).unwrap();
assert!(listing.complete);
assert_eq!(listing.members[0].link_target.as_deref(), Some("target"));
assert!(listing.warnings.is_empty(), "{:?}", listing.warnings);
}
/// A nonzero symlink checksum and even a zero regular-file checksum are checked.
#[test]
fn still_checks_nonzero_symlink_and_zero_regular_file_checksums() {
assert!(cpio(&archive(0o120777, 647), 10).unwrap().warnings.is_empty());
for bytes in [archive(0o120777, 646), archive(0o100644, 0)] {
assert!(cpio(&bytes, 10).unwrap().warnings.iter().any(|w| w.contains("checksum mismatch")));
}
}
}
// Ported from Craft Anything tar_pax_size_precedence.rs (RRFZ).
mod tar_pax_size_precedence {
use crate::testkit::tar_entries;
use crate::{member_data, tar};
/// Make an octal size unreadable while leaving the enclosing TAR header checksum valid.
fn corrupt_size(archive: &mut [u8], offset: usize) {
let header = &mut archive[offset..offset + 512];
header[124] = b'9';
header[148..156].fill(b' ');
let sum: u32 = header.iter().map(|&byte| u32::from(byte)).sum();
header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes());
}
/// Local and inherited size records override the bad field and keep the following header aligned.
#[test]
fn valid_pax_size_controls_the_member_extent() {
for extension in *b"xg" {
let mut archive = tar_entries(&[("PaxHeader", extension, b"10 size=3\n", ""), ("file", b'0', b"abc", ""), ("next", b'0', b"def", "")]);
corrupt_size(&mut archive, 1024);
let list = tar(&archive, 10).unwrap();
assert!(list.complete, "{:?}", list.warnings);
assert_eq!(list.members.len(), 2);
assert_eq!(member_data(&archive, &list.members[0]), Some(b"abc".as_slice()));
assert_eq!(member_data(&archive, &list.members[1]), Some(b"def".as_slice()));
}
}
/// A local zero overrides a global size and does not consume the next header.
#[test]
fn a_local_zero_size_overrides_the_global_value() {
let mut archive = tar_entries(&[
("Global", b'g', b"12 size=600\n", ""),
("Local", b'x', b"10 size=0\n", ""),
("empty", b'0', b"", ""),
("next", b'0', &[b'a'; 600], ""),
]);
corrupt_size(&mut archive, 2048);
let list = tar(&archive, 10).unwrap();
assert!(list.complete, "{:?}", list.warnings);
assert_eq!(list.members.iter().map(|m| m.size).collect::<Vec<_>>(), [0, 600]);
}
/// An ordinary header without an override and a PAX header itself still need readable sizes.
#[test]
fn rejects_bad_sizes_without_an_applicable_override() {
let mut plain = tar_entries(&[("file", b'0', b"abc", "")]);
corrupt_size(&mut plain, 0);
let mut with_global = tar_entries(&[("Global", b'g', b"10 size=3\n", ""), ("Local", b'x', b"10 size=0\n", ""), ("file", b'0', b"", "")]);
corrupt_size(&mut with_global, 1024);
for archive in [plain, with_global] {
let list = tar(&archive, 10).unwrap();
assert!(!list.complete);
assert!(list.members.is_empty());
assert!(list.warnings.iter().any(|warning| warning.contains("unreadable size")));
}
}
}