diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 1e6fd5f73a..2e0e718450 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -179,11 +179,18 @@ impl Document { log.push("the %PDF- header is missing; reading the file as PDF 1.4".into()); } let parsed = doc.read_xref_chain(&mut log); + let mut authenticated = false; let ok = match parsed { Ok((entries, trailer, revisions)) => { doc.entries = Arc::new(entries); doc.trailer = trailer; doc.revisions = Arc::new(revisions); + // Authenticate before checking the catalog: it may sit in an encrypted object + // stream, which reads as garbage until the key is known. + if let Some(enc) = doc.trailer.get(b"Encrypt").cloned() { + doc.authenticate(&enc, password)?; + authenticated = true; + } doc.root_is_catalog() } Err(e) => { @@ -198,7 +205,10 @@ impl Document { doc.reconstruct(&mut log)?; } doc.repair_log = Arc::new(log); - if let Some(enc) = doc.trailer.get(b"Encrypt").cloned() { + // Reconstruction may have found a different trailer: (re)authenticate against it. + if (!ok || !authenticated) + && let Some(enc) = doc.trailer.get(b"Encrypt").cloned() + { doc.authenticate(&enc, password)?; } let max = doc.entries.keys().next_back().copied().unwrap_or(0); diff --git a/crates/cos/src/writer.rs b/crates/cos/src/writer.rs index 14243b9c21..bcf7a27f9d 100644 --- a/crates/cos/src/writer.rs +++ b/crates/cos/src/writer.rs @@ -142,14 +142,31 @@ pub struct SaveOptions { pub mod_date: Option, /// Seed for a new file identifier when the document has none (deterministic in tests). pub id_seed: u64, + /// Full saves: pack non-stream objects into compressed object streams with a cross-reference + /// stream (PDF 1.5+, §7.5.7–7.5.8). Off writes a classic table that any reader accepts. + pub object_streams: bool, } impl Default for SaveOptions { fn default() -> Self { - Self { mod_date: None, id_seed: 0x5052_494E_5443_5241 } + Self { mod_date: None, id_seed: 0x5052_494E_5443_5241, object_streams: true } } } +/// Objects per object stream (Acrobat and qpdf use 100–200). +const OBJSTM_SIZE: usize = 100; + +/// Where an object lives, for the cross-reference section. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Row { + /// Free, with the generation the number gets when reused. + Free(u16), + /// At a byte offset. + InFile(u64, u16), + /// Inside object stream `stream`, at position `index`. + InStream(u32, u32), +} + /// Append the document's edits to its original bytes (incremental update, §7.5.6). /// /// A document that had to be reconstructed (no usable cross-reference chain) cannot be updated @@ -192,7 +209,8 @@ pub fn write_incremental(doc: &Document, opts: &SaveOptions) -> Result, } let size = doc.next_num().max(trailer.int(b"Size").unwrap_or(0) as u32); if as_stream { - write_xref_stream(&mut out, &mut trailer, &offsets, size, &[])?; + let rows = offsets.iter().map(|(n, (o, g, used))| (*n, if *used { Row::InFile(*o, *g) } else { Row::Free(*g) })).collect(); + write_xref_stream(&mut out, &mut trailer, rows, size); } else { trailer.set(b"Size".to_vec(), Object::Int(size as i64)); let xref_at = out.len(); @@ -231,20 +249,59 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro order.push(r); visit_refs(&o, &mut queue); } + let version = doc.version().chars().take(3).collect::(); + let version = if opts.object_streams && version.as_str() < "1.5" { "1.5".to_string() } else { version }; let mut out = Vec::new(); - let _ = writeln!(out, "%PDF-{}\n%\u{e2}\u{e3}\u{cf}\u{d3}", doc.version().chars().take(3).collect::()); - // The binary comment must be raw high bytes, not UTF-8. - out.truncate(out.len() - "\u{e2}\u{e3}\u{cf}\u{d3}\n".len()); - out.extend_from_slice(&[0xE2, 0xE3, 0xCF, 0xD3, b'\n']); - let mut offsets: BTreeMap = BTreeMap::new(); + let _ = writeln!(out, "%PDF-{version}"); + // A binary comment (raw high bytes) marks the file as binary for transfer tools. + out.extend_from_slice(&[b'%', 0xE2, 0xE3, 0xCF, 0xD3, b'\n']); + // The /Encrypt dictionary and everything it references (e.g. indirect /CF crypt-filter + // dictionaries) are needed before anything can be decrypted: never inside object streams. + let mut encryption_objects: std::collections::HashSet = std::collections::HashSet::new(); + let mut pending: Vec = trailer_in.reference(b"Encrypt").into_iter().collect(); + while let Some(r) = pending.pop() { + if encryption_objects.insert(r) { + collect_refs(&doc.get(r), &mut |x| pending.push(x)); + } + } + let mut rows: BTreeMap = BTreeMap::new(); + let mut packed: Vec<(u32, Object)> = Vec::new(); // objects bound for object streams for (i, r) in order.iter().enumerate() { let num = (i + 1) as u32; let o = renumber(&doc.get(*r), &map); - offsets.insert(num, (out.len() as u64, 0, true)); - write_indirect(num, 0, &prepared(&doc, r.num, num, 0, &o), &mut out); + let in_stream = opts.object_streams && !matches!(o, Object::Stream(_)) && !encryption_objects.contains(r); + if in_stream { + packed.push((num, o)); + } else { + rows.insert(num, Row::InFile(out.len() as u64, 0)); + write_indirect(num, 0, &prepared(&doc, r.num, num, 0, &o), &mut out); + } + } + // Object streams take the numbers after the last object. Their contents are not encrypted + // individually: the whole stream is (§7.5.7, §7.6.2). + let mut next = order.len() as u32 + 1; + for chunk in packed.chunks(OBJSTM_SIZE) { + let stm_num = next; + next += 1; + let mut header = Vec::new(); + let mut body = Vec::new(); + for (index, (num, o)) in chunk.iter().enumerate() { + let _ = write!(header, "{} {} ", num, body.len()); + serialize(o, &mut body); + body.push(b'\n'); + rows.insert(*num, Row::InStream(stm_num, index as u32)); + } + let first = header.len(); + header.extend_from_slice(&body); + let mut d = Dict::new(); + d.set(b"Type".to_vec(), Object::name("ObjStm")); + d.set(b"N".to_vec(), Object::Int(chunk.len() as i64)); + d.set(b"First".to_vec(), Object::Int(first as i64)); + let stm = Object::Stream(Stream::flate(d, &header)); + rows.insert(stm_num, Row::InFile(out.len() as u64, 0)); + write_indirect(stm_num, 0, &prepared(&doc, u32::MAX, stm_num, 0, &stm), &mut out); } let mut trailer = Dict::new(); - trailer.set(b"Size".to_vec(), Object::Int(order.len() as i64 + 1)); for key in [&b"Root"[..], b"Info", b"Encrypt"] { if let Some(r) = trailer_in.reference(key).and_then(|r| map.get(&r)) { trailer.set(key.to_vec(), Object::Ref(ObjRef::new(*r, 0))); @@ -259,11 +316,18 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro if doc.output_security().0.is_none() { ensure_id(&mut trailer, opts, doc.bytes()); } - let xref_at = out.len(); - write_xref_table(&mut out, &offsets, true); - out.extend_from_slice(b"trailer\n"); - write_dict(&trailer, &mut out); - let _ = write!(out, "\nstartxref\n{xref_at}\n%%EOF\n"); + if opts.object_streams { + write_xref_stream(&mut out, &mut trailer, rows, next); + } else { + trailer.set(b"Size".to_vec(), Object::Int(next as i64)); + let offsets: BTreeMap = + rows.iter().filter_map(|(n, r)| if let Row::InFile(o, g) = r { Some((*n, (*o, *g, true))) } else { None }).collect(); + let xref_at = out.len(); + write_xref_table(&mut out, &offsets, true); + out.extend_from_slice(b"trailer\n"); + write_dict(&trailer, &mut out); + let _ = write!(out, "\nstartxref\n{xref_at}\n%%EOF\n"); + } Ok(out) } @@ -309,20 +373,22 @@ fn write_xref_table(out: &mut Vec, offsets: &BTreeMap } } -/// A cross-reference stream section (for files whose last section is a stream, §7.5.8). -fn write_xref_stream( - out: &mut Vec, - trailer: &mut Dict, - offsets: &BTreeMap, - size: u32, - _extra: &[u32], -) -> Result<(), CosError> { - let xref_num = size; +/// A cross-reference stream section (§7.5.8) that takes object number `num`, ending the file. +/// Rows are Flate-compressed with the PNG Up predictor, and field widths fit the largest value. +fn write_xref_stream(out: &mut Vec, trailer: &mut Dict, mut rows: BTreeMap, num: u32) { let xref_at = out.len() as u64; - let mut rows: BTreeMap = - offsets.iter().map(|(n, (o, g, used))| (*n, if *used { (1, *o, *g) } else { (0, 0, *g) })).collect(); - rows.insert(xref_num, (1, xref_at, 0)); - let mut data = Vec::new(); + rows.insert(num, Row::InFile(xref_at, 0)); + if trailer.get(b"Prev").is_none() { + rows.entry(0).or_insert(Row::Free(65535)); + } + let fields = |r: &Row| match *r { + Row::Free(g) => (0u64, 0u64, u64::from(g)), + Row::InFile(o, g) => (1, o, u64::from(g)), + Row::InStream(s, i) => (2, u64::from(s), u64::from(i)), + }; + let width = |v: u64| ((64 - v.leading_zeros()).div_ceil(8)).max(1) as usize; + let (w2, w3) = rows.values().map(fields).fold((1, 1), |(a, b), (_, x, y)| (width(x).max(a), width(y).max(b))); + let cols = 1 + w2 + w3; let mut index = Vec::new(); let nums: Vec = rows.keys().copied().collect(); let mut i = 0; @@ -335,21 +401,30 @@ fn write_xref_stream( index.push(Object::Int((j - i + 1) as i64)); i = j + 1; } - for (t, a, b) in rows.values() { - data.push(*t); - data.extend_from_slice(&(*a as u32).to_be_bytes()); - data.extend_from_slice(&b.to_be_bytes()); + let mut data = Vec::with_capacity(rows.len() * (cols + 1)); + let mut prev = vec![0u8; cols]; + for r in rows.values() { + let (t, a, b) = fields(r); + let mut row = vec![t as u8]; + row.extend_from_slice(&a.to_be_bytes()[8 - w2..]); + row.extend_from_slice(&b.to_be_bytes()[8 - w3..]); + data.push(2); // PNG "Up" + data.extend(row.iter().zip(&prev).map(|(c, p)| c.wrapping_sub(*p))); + prev = row; } let mut d = trailer.clone(); d.set(b"Type".to_vec(), Object::name("XRef")); - d.set(b"Size".to_vec(), Object::Int(xref_num as i64 + 1)); - d.set(b"W".to_vec(), Object::Array(vec![Object::Int(1), Object::Int(4), Object::Int(2)])); + d.set(b"Size".to_vec(), Object::Int(num as i64 + 1)); + d.set(b"W".to_vec(), Object::Array(vec![Object::Int(1), Object::Int(w2 as i64), Object::Int(w3 as i64)])); d.set(b"Index".to_vec(), Object::Array(index)); - let stream = Stream::flate(d, &data); - write_indirect(xref_num, 0, &Object::Stream(stream), out); + let mut stream = Stream::flate(d, &data); + let mut parms = Dict::new(); + parms.set(b"Predictor".to_vec(), Object::Int(12)); + parms.set(b"Columns".to_vec(), Object::Int(cols as i64)); + stream.dict.set(b"DecodeParms".to_vec(), Object::Dict(parms)); + write_indirect(num, 0, &Object::Stream(stream), out); let _ = write!(out, "startxref\n{xref_at}\n%%EOF\n"); - trailer.set(b"Size".to_vec(), Object::Int(xref_num as i64 + 1)); - Ok(()) + trailer.set(b"Size".to_vec(), Object::Int(num as i64 + 1)); } fn stamp_mod_date(doc: &mut Document, opts: &SaveOptions) { diff --git a/crates/cos/tests/writer.rs b/crates/cos/tests/writer.rs new file mode 100644 index 0000000000..3c05e5afee --- /dev/null +++ b/crates/cos/tests/writer.rs @@ -0,0 +1,165 @@ +//! Full saves with object streams and cross-reference streams (M1.9), read back by our parser +//! and by hayro, and incremental saves stacked on top of them. + +use std::sync::Arc; + +use printcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, write_full, write_incremental}; + +/// Four pages, a shared font, document info with a non-ASCII title. +fn fixture() -> Vec { + let mut objs: Vec = vec!["<< /Type /Catalog /Pages 2 0 R >>".into()]; + let kids: Vec = (0..4).map(|i| format!("{} 0 R", 4 + 2 * i)).collect(); + objs.push(format!("<< /Type /Pages /Kids [{}] /Count 4 /MediaBox [0 0 200 300] >>", kids.join(" "))); + objs.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".into()); + for i in 0..4 { + objs.push(format!("<< /Type /Page /Parent 2 0 R /Contents {} 0 R /Resources << /Font << /F1 3 0 R >> >> >>", 5 + 2 * i)); + let body = format!("BT /F1 24 Tf 20 150 Td (Page {}) Tj ET", i + 1); + objs.push(format!("<< /Length {} >>\nstream\n{body}\nendstream", body.len())); + } + objs.push("<< /Title (Caf\\351 \\(draft\\)) >>".into()); + let mut out = b"%PDF-1.4\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice( + format!("trailer\n<< /Size {} /Root 1 0 R /Info {} 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1, objs.len()).as_bytes(), + ); + out +} + +fn title(doc: &Document) -> Vec { + let info = doc.resolve(doc.trailer().get(b"Info").unwrap()); + info.as_dict().unwrap().get(b"Title").unwrap().as_string().unwrap().bytes.clone() +} + +fn hayro_pages(bytes: &[u8]) -> usize { + hayro_syntax::Pdf::new(Arc::new(bytes.to_vec())).expect("hayro opens").pages().len() +} + +#[test] +fn full_save_packs_objects_into_object_streams() { + let doc = Document::open(Arc::new(fixture())).unwrap(); + let packed = write_full(&doc, &SaveOptions::default()).unwrap(); + let text = String::from_utf8_lossy(&packed); + assert!(packed.starts_with(b"%PDF-1.5"), "object streams need PDF 1.5"); + assert!(text.contains("/ObjStm") && text.contains("/XRef")); + assert!(!text.contains("\nxref\n"), "no classic table"); + assert!(!text.contains("/Type /Catalog") && !text.contains("/Type/Catalog"), "the catalog is compressed"); + + let back = Document::open(Arc::new(packed.clone())).unwrap(); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + assert_eq!(title(&back), b"Caf\xe9 (draft)"); + assert_eq!(hayro_pages(&packed), 4); + + let classic = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + assert!(classic.starts_with(b"%PDF-1.4")); + assert!(String::from_utf8_lossy(&classic).contains("\nxref\n")); + assert_eq!(hayro_pages(&classic), 4); +} + +#[test] +fn incremental_saves_stack_on_an_xref_stream_file() { + let doc = Document::open(Arc::new(fixture())).unwrap(); + let packed = Arc::new(write_full(&doc, &SaveOptions::default()).unwrap()); + let mut doc = Document::open(packed.clone()).unwrap(); + // Edit an object that lives in an object stream. + let info = doc.trailer().reference(b"Info").unwrap(); + doc.update_dict(info, |d: &mut Dict| { + d.set(b"Title".to_vec(), Object::String(printcraft_cos::PdfString { bytes: b"Second".to_vec(), hex: false })) + }) + .unwrap(); + let added = doc.add(Object::Int(42)); + let updated = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(updated.starts_with(&packed), "an incremental save only appends"); + let tail = String::from_utf8_lossy(&updated[packed.len()..]); + assert!(tail.contains("/XRef") && tail.contains("/Prev"), "the update uses an xref stream too"); + + let back = Document::open(Arc::new(updated.clone())).unwrap(); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + assert_eq!(title(&back), b"Second"); + assert_eq!(*back.get(ObjRef::new(added.num, 0)), Object::Int(42)); + assert_eq!(back.revisions().len(), 2); + assert_eq!(hayro_pages(&updated), 4); +} + +#[test] +fn qpdf_accepts_object_stream_output() { + let Ok(out) = std::process::Command::new("qpdf").arg("--version").output() else { + eprintln!("qpdf not installed: skipped"); + return; + }; + assert!(out.status.success()); + let doc = Document::open(Arc::new(fixture())).unwrap(); + let packed = write_full(&doc, &SaveOptions::default()).unwrap(); + let path = std::env::temp_dir().join(format!("printcraft-objstm-{}.pdf", std::process::id())); + std::fs::write(&path, &packed).unwrap(); + let check = std::process::Command::new("qpdf").arg("--check").arg(&path).output().unwrap(); + let _ = std::fs::remove_file(&path); + let report = String::from_utf8_lossy(&check.stdout); + assert!(check.status.success() && report.contains("No syntax or stream encoding errors"), "{report}"); +} + +#[test] +fn objects_referenced_by_the_encrypt_dictionary_stay_outside_object_streams() { + // Encrypt a document, then move its crypt filters into indirect objects, as some producers + // do (pdf.js issue7665). A full save must keep them readable before decryption. + let doc = Document::open(Arc::new(fixture())).unwrap(); + let mut enc = doc.clone(); + enc.set_encryption(&printcraft_cos::NewEncryption { + algorithm: printcraft_cos::Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: -4, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let bytes = Arc::new(write_full(&enc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap()); + let mut doc = Document::open(bytes).unwrap(); + let enc_ref = doc.trailer().reference(b"Encrypt").expect("indirect /Encrypt"); + let cf = doc.get(enc_ref).as_dict().unwrap().get(b"CF").cloned().unwrap(); + let cf_ref = doc.add(cf); + doc.update_dict(enc_ref, |d| d.set(b"CF".to_vec(), Object::Ref(cf_ref))).unwrap(); + let packed = write_full(&doc, &SaveOptions::default()).unwrap(); + let back = Document::open(Arc::new(packed.clone())).expect("reopens"); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + assert_eq!(title(&back), b"Caf\xe9 (draft)"); + assert_eq!(hayro_pages(&packed), 4); +} + +#[test] +fn page_tree_loops_in_object_streams_do_not_crash_readers() { + // pdf.js Pages-tree-refs.pdf: /Kids 3 → 4 → 5 → 3. Written faithfully into object streams, + // it used to overflow hayro-syntax's stack (vendored patch: cycle guard in `resolve_pages`). + let objs = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [6 0 R 3 0 R] /Count 2 /MediaBox [0 0 595 842] >>", + "<< /Type /Pages /Kids [4 0 R] /Count 1 >>", + "<< /Type /Pages /Kids [5 0 R] /Count 1 >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R >>", + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + let doc = Document::open(Arc::new(out)).unwrap(); + let packed = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(String::from_utf8_lossy(&packed).contains("/ObjStm")); + assert_eq!(hayro_pages(&packed), 1); +} diff --git a/crates/ui-egui/tests/control.rs b/crates/ui-egui/tests/control.rs index 4a027baaef..917f2d6567 100644 --- a/crates/ui-egui/tests/control.rs +++ b/crates/ui-egui/tests/control.rs @@ -158,7 +158,10 @@ fn loopback_transport_requires_the_token() { let mut r = BufReader::new(s).lines(); let mut out = Vec::new(); for l in lines { - writeln!(w, "{l}").unwrap(); + // The server closes the connection after a failed auth; later writes may fail. + if writeln!(w, "{l}").is_err() { + break; + } match r.next() { Some(Ok(reply)) => out.push(serde_json::from_str::(&reply).unwrap()), _ => break,