mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-18 08:15:29 +00:00
tar: preserve current sparse, PAX size and CRC symlink behavior (#2012)
* tar: port old GNU sparse extension traversal from Craft Anything Source: storytold/craft-anything#1973, 8ea2ec66a1bc80f0420f3695baac877a0cb9b9a5. Retains non-extractable sparse member kind and regression tests. * tar: preserve GNU CRC symlink checksum compatibility Source: storytold/craft-anything#1984, b226a475f7dce9710e3af7a069386fe7a9720df7; preserves nonzero checks and regular-file checksum validation. * tar: apply valid PAX sizes before legacy member size fields Source: storytold/craft-anything#2007, d29465c20b0d570ea4973be512a0cb4e212bb60f; metadata records still retain their own framing. * Record provenance of current TAR and CPIO compatibility fixes --------- Co-authored-by: RRFZ <247457381+Plastic-time@users.noreply.github.com>
This commit is contained in:
1 parent
90947c57d7
commit
42bb9d35bb
3 files changed
+181
-7
No files matched your search
@@ -119,9 +119,14 @@ The XML reader in `crates/plist/src/xml.rs` and its base64 compatibility helper
|
||||
|
||||
`crates/palette` extracts the GPL text parser from AniCraft `crates/doc/src/swatch.rs` at `d875a7feb250e6ab06aa28a37fb78a0699e88330` (MIT OR Apache-2.0), introduced by Tanish Manku in `a64130a9da9bdde545463b852d95d6a6774b1dfd` (AniCraft #270). The import preserves Tanish as primary author. Neutral RGB/name adaptation separates application display-name sanitization and the `Untitled` policy; validation tests and integration are subsequent Dylan M. Taylor changes.
|
||||
|
||||
## TAR/CPIO compatibility updates
|
||||
|
||||
The old GNU sparse-extension traversal, GNU CRC-symlink compatibility and PAX-size precedence fixes and their regression tests in `crates/tar` are ported from RRFZ's Craft Anything PRs #1973 (`8ea2ec66a1bc80f0420f3695baac877a0cb9b9a5`), #1984 (`b226a475f7dce9710e3af7a069386fe7a9720df7`) and #2007 (`d29465c20b0d570ea4973be512a0cb4e212bb60f`). Source is MIT OR Apache-2.0; RRFZ remains the primary Git author of each port. The PAX update adapts to the shared parser's existing `PaxOverrides` structure without changing the source policy.
|
||||
|
||||
## Aligned stored ZIP entries
|
||||
|
||||
The USDZ alignment requirement and compatibility fixture for `Writer::add_stored_aligned` come from TextureCraft `crates/formats/src/mesh_export/usd.rs`, introduced by bflatastic in `bdbb52c9` and maintained through `9eb20086` (MIT OR Apache-2.0). The shared API and alignment validation are subsequent Dylan M. Taylor work. Scene/material mapping remains in TextureCraft.
|
||||
|
||||
The raw gradient wire ordering in `craft-lottie::Gradient::flat_values` follows
|
||||
AniCraft's Lottie exporter by Tanish Manku (`63b39355`, `b9258b7b`), MIT OR Apache-2.0.
|
||||
Sanitization, animation fitting and numeric output policy remain caller-owned.
|
||||
|
||||
+24
-7
@@ -175,17 +175,32 @@ fn tar_with_name_copy_limit(data: &[u8], max_members: usize, mut name_bytes_left
|
||||
break;
|
||||
}
|
||||
let typeflag = h.get(156).copied().unwrap_or(0);
|
||||
let Some(mut size) = h.get(124..136).and_then(octal) else {
|
||||
// Metadata records keep their own framing. For a member, PAX replaces the
|
||||
// legacy size field entirely, so a valid override does not depend on its contents.
|
||||
let override_size = match typeflag {
|
||||
b'x' | b'g' | b'L' | b'K' => None,
|
||||
_ => pax.size.take().or(pax.global_size),
|
||||
};
|
||||
let Some(size) = override_size.or_else(|| h.get(124..136).and_then(octal)) else {
|
||||
out.warnings.push(format!("unreadable size at {pos:#x}"));
|
||||
out.length = pos as u64;
|
||||
break;
|
||||
};
|
||||
if !matches!(typeflag, b'x' | b'g' | b'L' | b'K')
|
||||
&& let Some(s) = pax.size.take().or(pax.global_size)
|
||||
{
|
||||
size = s;
|
||||
let mut data_off = pos + BLOCK;
|
||||
let old_gnu_sparse = typeflag == b'S' && h.get(257..265) == Some(b"ustar \0");
|
||||
if old_gnu_sparse {
|
||||
// Old GNU sparse maps can continue in whole blocks before the stored extents.
|
||||
let mut extended = h.get(482).is_some_and(|&flag| flag != 0);
|
||||
while extended {
|
||||
let Some(block) = r.slice(data_off, BLOCK) else {
|
||||
out.warnings.push(format!("GNU sparse extension ends early at {data_off:#x}"));
|
||||
out.length = data.len() as u64;
|
||||
break 'headers;
|
||||
};
|
||||
extended = block.get(504).is_some_and(|&flag| flag != 0);
|
||||
data_off += BLOCK;
|
||||
}
|
||||
}
|
||||
let data_off = pos + BLOCK;
|
||||
let Some(next) =
|
||||
round_up(size, BLOCK as u64).and_then(|s| u64::try_from(data_off).ok()?.checked_add(s)).and_then(|n| usize::try_from(n).ok())
|
||||
else {
|
||||
@@ -262,7 +277,7 @@ fn tar_with_name_copy_limit(data: &[u8], max_members: usize, mut name_bytes_left
|
||||
.or_else(|| Some(text(h.get(157..257).unwrap_or(&[]))))
|
||||
.filter(|l| !l.is_empty());
|
||||
let mut kind = tar_kind(typeflag, &name);
|
||||
if std::mem::take(&mut pax.sparse) || !pax.global_sparse.is_empty() {
|
||||
if std::mem::take(&mut pax.sparse) || !pax.global_sparse.is_empty() || old_gnu_sparse {
|
||||
kind = MemberKind::Other;
|
||||
out.warnings.push(format!("{name}: GNU sparse reconstruction is unsupported; stored extents are not the complete file"));
|
||||
}
|
||||
@@ -523,6 +538,8 @@ pub fn cpio_with_checksum(data: &[u8], max_members: usize, checksum: &mut dyn Fn
|
||||
}
|
||||
if format == "cpio-crc" {
|
||||
match h.get(102..110).filter(|field| field.iter().all(u8::is_ascii_hexdigit)).and_then(hex_field).and_then(|n| u32::try_from(n).ok()) {
|
||||
// GNU cpio writes the symlink target without a checksum.
|
||||
Some(0) if kind == MemberKind::Symlink => {}
|
||||
Some(expected) => {
|
||||
if let Some(body) = r.slice(data_off, size) {
|
||||
match checksum(body) {
|
||||
|
||||
@@ -402,3 +402,155 @@ mod tar_global_pax {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ported from Craft Anything gnu_sparse_extensions.rs (RRFZ).
|
||||
mod gnu_sparse_extensions {
|
||||
use crate::testkit::tar_entries;
|
||||
use crate::{MemberKind, member_data, tar};
|
||||
|
||||
/// Build an old GNU sparse header followed by the requested number of extension blocks.
|
||||
fn archive(extension_blocks: usize) -> Vec<u8> {
|
||||
let mut archive = tar_entries(&[("sparse.dat", b'S', &[b'x'; 512], ""), ("after.txt", b'0', b"after", "")]);
|
||||
archive[257..265].copy_from_slice(b"ustar \0");
|
||||
archive[482] = u8::from(extension_blocks != 0);
|
||||
archive[148..156].fill(b' ');
|
||||
let sum: u32 = archive[..512].iter().map(|&byte| u32::from(byte)).sum();
|
||||
archive[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes());
|
||||
let mut extensions = vec![0; 512 * extension_blocks];
|
||||
for block in extensions.as_chunks_mut::<512>().0.iter_mut().take(extension_blocks.saturating_sub(1)) {
|
||||
block[504] = 1;
|
||||
}
|
||||
archive.splice(512..512, extensions);
|
||||
archive
|
||||
}
|
||||
|
||||
/// Actual file data starts after the entire sparse map. Later members remain readable.
|
||||
#[test]
|
||||
fn skips_old_gnu_sparse_extension_blocks() {
|
||||
for blocks in [1, 2] {
|
||||
let bytes = archive(blocks);
|
||||
let listing = tar(&bytes, 8).unwrap();
|
||||
assert!(listing.complete, "{blocks} extensions: {:?}", listing.warnings);
|
||||
assert_eq!(listing.members.len(), 2);
|
||||
assert_eq!(listing.members[0].kind, MemberKind::Other);
|
||||
assert_eq!(listing.members[0].data_offset, (512 * (blocks + 1)) as u64);
|
||||
assert_eq!(listing.members[1].name, "after.txt");
|
||||
assert_eq!(member_data(&bytes, &listing.members[1]), Some(b"after".as_slice()));
|
||||
}
|
||||
}
|
||||
|
||||
/// A missing sparse extension cannot turn padding into a complete archive.
|
||||
#[test]
|
||||
fn refuses_a_truncated_old_gnu_sparse_extension() {
|
||||
let mut bytes = archive(2);
|
||||
bytes.truncate(512 + 512 + 17);
|
||||
let listing = tar(&bytes, 8).unwrap();
|
||||
assert!(!listing.complete);
|
||||
assert!(listing.members.is_empty());
|
||||
assert!(listing.warnings.iter().any(|warning| warning.contains("sparse extension")));
|
||||
}
|
||||
|
||||
/// A sparse member with no extensions still leaves the next header at its usual offset.
|
||||
#[test]
|
||||
fn keeps_the_next_member_when_no_sparse_extensions_follow() {
|
||||
let bytes = archive(0);
|
||||
let listing = tar(&bytes, 8).unwrap();
|
||||
assert!(listing.complete);
|
||||
assert_eq!(listing.members[0].kind, MemberKind::Other);
|
||||
assert_eq!(listing.members[1].name, "after.txt");
|
||||
}
|
||||
}
|
||||
|
||||
// Ported from Craft Anything cpio_symlink_crc.rs (RRFZ).
|
||||
mod cpio_symlink_crc {
|
||||
use crate::cpio;
|
||||
use crate::testkit::cpio_newc;
|
||||
|
||||
/// Convert a single-member newc fixture to CRC and set the member checksum.
|
||||
fn archive(mode: u32, checksum: u32) -> Vec<u8> {
|
||||
let mut bytes = cpio_newc(&[("entry", b"target", mode)]);
|
||||
bytes[..6].copy_from_slice(b"070702");
|
||||
bytes[102..110].copy_from_slice(format!("{checksum:08x}").as_bytes());
|
||||
let trailer = (110 + "entry".len() + 1 + 3) & !3;
|
||||
let trailer = (trailer + "target".len() + 3) & !3;
|
||||
bytes[trailer..trailer + 6].copy_from_slice(b"070702");
|
||||
bytes
|
||||
}
|
||||
|
||||
/// GNU-style CRC symlinks carry a zero placeholder and keep the recorded target.
|
||||
#[test]
|
||||
fn accepts_a_symlink_with_no_stored_checksum() {
|
||||
let listing = cpio(&archive(0o120777, 0), 10).unwrap();
|
||||
assert!(listing.complete);
|
||||
assert_eq!(listing.members[0].link_target.as_deref(), Some("target"));
|
||||
assert!(listing.warnings.is_empty(), "{:?}", listing.warnings);
|
||||
}
|
||||
|
||||
/// A nonzero symlink checksum and even a zero regular-file checksum are checked.
|
||||
#[test]
|
||||
fn still_checks_nonzero_symlink_and_zero_regular_file_checksums() {
|
||||
assert!(cpio(&archive(0o120777, 647), 10).unwrap().warnings.is_empty());
|
||||
for bytes in [archive(0o120777, 646), archive(0o100644, 0)] {
|
||||
assert!(cpio(&bytes, 10).unwrap().warnings.iter().any(|w| w.contains("checksum mismatch")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ported from Craft Anything tar_pax_size_precedence.rs (RRFZ).
|
||||
mod tar_pax_size_precedence {
|
||||
use crate::testkit::tar_entries;
|
||||
use crate::{member_data, tar};
|
||||
|
||||
/// Make an octal size unreadable while leaving the enclosing TAR header checksum valid.
|
||||
fn corrupt_size(archive: &mut [u8], offset: usize) {
|
||||
let header = &mut archive[offset..offset + 512];
|
||||
header[124] = b'9';
|
||||
header[148..156].fill(b' ');
|
||||
let sum: u32 = header.iter().map(|&byte| u32::from(byte)).sum();
|
||||
header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes());
|
||||
}
|
||||
|
||||
/// Local and inherited size records override the bad field and keep the following header aligned.
|
||||
#[test]
|
||||
fn valid_pax_size_controls_the_member_extent() {
|
||||
for extension in *b"xg" {
|
||||
let mut archive = tar_entries(&[("PaxHeader", extension, b"10 size=3\n", ""), ("file", b'0', b"abc", ""), ("next", b'0', b"def", "")]);
|
||||
corrupt_size(&mut archive, 1024);
|
||||
let list = tar(&archive, 10).unwrap();
|
||||
assert!(list.complete, "{:?}", list.warnings);
|
||||
assert_eq!(list.members.len(), 2);
|
||||
assert_eq!(member_data(&archive, &list.members[0]), Some(b"abc".as_slice()));
|
||||
assert_eq!(member_data(&archive, &list.members[1]), Some(b"def".as_slice()));
|
||||
}
|
||||
}
|
||||
|
||||
/// A local zero overrides a global size and does not consume the next header.
|
||||
#[test]
|
||||
fn a_local_zero_size_overrides_the_global_value() {
|
||||
let mut archive = tar_entries(&[
|
||||
("Global", b'g', b"12 size=600\n", ""),
|
||||
("Local", b'x', b"10 size=0\n", ""),
|
||||
("empty", b'0', b"", ""),
|
||||
("next", b'0', &[b'a'; 600], ""),
|
||||
]);
|
||||
corrupt_size(&mut archive, 2048);
|
||||
let list = tar(&archive, 10).unwrap();
|
||||
assert!(list.complete, "{:?}", list.warnings);
|
||||
assert_eq!(list.members.iter().map(|m| m.size).collect::<Vec<_>>(), [0, 600]);
|
||||
}
|
||||
|
||||
/// An ordinary header without an override and a PAX header itself still need readable sizes.
|
||||
#[test]
|
||||
fn rejects_bad_sizes_without_an_applicable_override() {
|
||||
let mut plain = tar_entries(&[("file", b'0', b"abc", "")]);
|
||||
corrupt_size(&mut plain, 0);
|
||||
let mut with_global = tar_entries(&[("Global", b'g', b"10 size=3\n", ""), ("Local", b'x', b"10 size=0\n", ""), ("file", b'0', b"", "")]);
|
||||
corrupt_size(&mut with_global, 1024);
|
||||
for archive in [plain, with_global] {
|
||||
let list = tar(&archive, 10).unwrap();
|
||||
assert!(!list.complete);
|
||||
assert!(list.members.is_empty());
|
||||
assert!(list.warnings.iter().any(|warning| warning.contains("unreadable size")));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user