M1.9: full saves write object streams + xref stream (PNG Up predictor); encrypt-dict closure kept outside; reader authenticates before validating the catalog

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/cos/src/document.rs

@@ -179,11 +179,18 @@ impl Document {
             log.push("the %PDF- header is missing; reading the file as PDF 1.4".into());
         }
         let parsed = doc.read_xref_chain(&mut log);
+        let mut authenticated = false;
         let ok = match parsed {
             Ok((entries, trailer, revisions)) => {
                 doc.entries = Arc::new(entries);
                 doc.trailer = trailer;
                 doc.revisions = Arc::new(revisions);
+                // Authenticate before checking the catalog: it may sit in an encrypted object
+                // stream, which reads as garbage until the key is known.
+                if let Some(enc) = doc.trailer.get(b"Encrypt").cloned() {
+                    doc.authenticate(&enc, password)?;
+                    authenticated = true;
+                }
                 doc.root_is_catalog()
             }
             Err(e) => {
@@ -198,7 +205,10 @@ impl Document {
             doc.reconstruct(&mut log)?;
         }
         doc.repair_log = Arc::new(log);
-        if let Some(enc) = doc.trailer.get(b"Encrypt").cloned() {
+        // Reconstruction may have found a different trailer: (re)authenticate against it.
+        if (!ok || !authenticated)
+            && let Some(enc) = doc.trailer.get(b"Encrypt").cloned()
+        {
             doc.authenticate(&enc, password)?;
         }
         let max = doc.entries.keys().next_back().copied().unwrap_or(0);

crates/cos/src/writer.rs

@@ -142,14 +142,31 @@ pub struct SaveOptions {
     pub mod_date: Option<String>,
     /// Seed for a new file identifier when the document has none (deterministic in tests).
     pub id_seed: u64,
+    /// Full saves: pack non-stream objects into compressed object streams with a cross-reference
+    /// stream (PDF 1.5+, §7.5.7–7.5.8). Off writes a classic table that any reader accepts.
+    pub object_streams: bool,
 }
 
 impl Default for SaveOptions {
     fn default() -> Self {
-        Self { mod_date: None, id_seed: 0x5052_494E_5443_5241 }
+        Self { mod_date: None, id_seed: 0x5052_494E_5443_5241, object_streams: true }
     }
 }
 
+/// Objects per object stream (Acrobat and qpdf use 100–200).
+const OBJSTM_SIZE: usize = 100;
+
+/// Where an object lives, for the cross-reference section.
+#[derive(Clone, Copy, Debug, PartialEq)]
+enum Row {
+    /// Free, with the generation the number gets when reused.
+    Free(u16),
+    /// At a byte offset.
+    InFile(u64, u16),
+    /// Inside object stream `stream`, at position `index`.
+    InStream(u32, u32),
+}
+
 /// Append the document's edits to its original bytes (incremental update, §7.5.6).
 ///
 /// A document that had to be reconstructed (no usable cross-reference chain) cannot be updated
@@ -192,7 +209,8 @@ pub fn write_incremental(doc: &Document, opts: &SaveOptions) -> Result<Vec<u8>,
     }
     let size = doc.next_num().max(trailer.int(b"Size").unwrap_or(0) as u32);
     if as_stream {
-        write_xref_stream(&mut out, &mut trailer, &offsets, size, &[])?;
+        let rows = offsets.iter().map(|(n, (o, g, used))| (*n, if *used { Row::InFile(*o, *g) } else { Row::Free(*g) })).collect();
+        write_xref_stream(&mut out, &mut trailer, rows, size);
     } else {
         trailer.set(b"Size".to_vec(), Object::Int(size as i64));
         let xref_at = out.len();
@@ -231,20 +249,59 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result<Vec<u8>, CosErro
         order.push(r);
         visit_refs(&o, &mut queue);
     }
+    let version = doc.version().chars().take(3).collect::<String>();
+    let version = if opts.object_streams && version.as_str() < "1.5" { "1.5".to_string() } else { version };
     let mut out = Vec::new();
-    let _ = writeln!(out, "%PDF-{}\n%\u{e2}\u{e3}\u{cf}\u{d3}", doc.version().chars().take(3).collect::<String>());
-    // The binary comment must be raw high bytes, not UTF-8.
-    out.truncate(out.len() - "\u{e2}\u{e3}\u{cf}\u{d3}\n".len());
-    out.extend_from_slice(&[0xE2, 0xE3, 0xCF, 0xD3, b'\n']);
-    let mut offsets: BTreeMap<u32, (u64, u16, bool)> = BTreeMap::new();
+    let _ = writeln!(out, "%PDF-{version}");
+    // A binary comment (raw high bytes) marks the file as binary for transfer tools.
+    out.extend_from_slice(&[b'%', 0xE2, 0xE3, 0xCF, 0xD3, b'\n']);
+    // The /Encrypt dictionary and everything it references (e.g. indirect /CF crypt-filter
+    // dictionaries) are needed before anything can be decrypted: never inside object streams.
+    let mut encryption_objects: std::collections::HashSet<ObjRef> = std::collections::HashSet::new();
+    let mut pending: Vec<ObjRef> = trailer_in.reference(b"Encrypt").into_iter().collect();
+    while let Some(r) = pending.pop() {
+        if encryption_objects.insert(r) {
+            collect_refs(&doc.get(r), &mut |x| pending.push(x));
+        }
+    }
+    let mut rows: BTreeMap<u32, Row> = BTreeMap::new();
+    let mut packed: Vec<(u32, Object)> = Vec::new(); // objects bound for object streams
     for (i, r) in order.iter().enumerate() {
         let num = (i + 1) as u32;
         let o = renumber(&doc.get(*r), &map);
-        offsets.insert(num, (out.len() as u64, 0, true));
-        write_indirect(num, 0, &prepared(&doc, r.num, num, 0, &o), &mut out);
+        let in_stream = opts.object_streams && !matches!(o, Object::Stream(_)) && !encryption_objects.contains(r);
+        if in_stream {
+            packed.push((num, o));
+        } else {
+            rows.insert(num, Row::InFile(out.len() as u64, 0));
+            write_indirect(num, 0, &prepared(&doc, r.num, num, 0, &o), &mut out);
+        }
+    }
+    // Object streams take the numbers after the last object. Their contents are not encrypted
+    // individually: the whole stream is (§7.5.7, §7.6.2).
+    let mut next = order.len() as u32 + 1;
+    for chunk in packed.chunks(OBJSTM_SIZE) {
+        let stm_num = next;
+        next += 1;
+        let mut header = Vec::new();
+        let mut body = Vec::new();
+        for (index, (num, o)) in chunk.iter().enumerate() {
+            let _ = write!(header, "{} {} ", num, body.len());
+            serialize(o, &mut body);
+            body.push(b'\n');
+            rows.insert(*num, Row::InStream(stm_num, index as u32));
+        }
+        let first = header.len();
+        header.extend_from_slice(&body);
+        let mut d = Dict::new();
+        d.set(b"Type".to_vec(), Object::name("ObjStm"));
+        d.set(b"N".to_vec(), Object::Int(chunk.len() as i64));
+        d.set(b"First".to_vec(), Object::Int(first as i64));
+        let stm = Object::Stream(Stream::flate(d, &header));
+        rows.insert(stm_num, Row::InFile(out.len() as u64, 0));
+        write_indirect(stm_num, 0, &prepared(&doc, u32::MAX, stm_num, 0, &stm), &mut out);
     }
     let mut trailer = Dict::new();
-    trailer.set(b"Size".to_vec(), Object::Int(order.len() as i64 + 1));
     for key in [&b"Root"[..], b"Info", b"Encrypt"] {
         if let Some(r) = trailer_in.reference(key).and_then(|r| map.get(&r)) {
             trailer.set(key.to_vec(), Object::Ref(ObjRef::new(*r, 0)));
@@ -259,11 +316,18 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result<Vec<u8>, CosErro
     if doc.output_security().0.is_none() {
         ensure_id(&mut trailer, opts, doc.bytes());
     }
-    let xref_at = out.len();
-    write_xref_table(&mut out, &offsets, true);
-    out.extend_from_slice(b"trailer\n");
-    write_dict(&trailer, &mut out);
-    let _ = write!(out, "\nstartxref\n{xref_at}\n%%EOF\n");
+    if opts.object_streams {
+        write_xref_stream(&mut out, &mut trailer, rows, next);
+    } else {
+        trailer.set(b"Size".to_vec(), Object::Int(next as i64));
+        let offsets: BTreeMap<u32, (u64, u16, bool)> =
+            rows.iter().filter_map(|(n, r)| if let Row::InFile(o, g) = r { Some((*n, (*o, *g, true))) } else { None }).collect();
+        let xref_at = out.len();
+        write_xref_table(&mut out, &offsets, true);
+        out.extend_from_slice(b"trailer\n");
+        write_dict(&trailer, &mut out);
+        let _ = write!(out, "\nstartxref\n{xref_at}\n%%EOF\n");
+    }
     Ok(out)
 }
 
@@ -309,20 +373,22 @@ fn write_xref_table(out: &mut Vec<u8>, offsets: &BTreeMap<u32, (u64, u16, bool)>
     }
 }
 
-/// A cross-reference stream section (for files whose last section is a stream, §7.5.8).
-fn write_xref_stream(
-    out: &mut Vec<u8>,
-    trailer: &mut Dict,
-    offsets: &BTreeMap<u32, (u64, u16, bool)>,
-    size: u32,
-    _extra: &[u32],
-) -> Result<(), CosError> {
-    let xref_num = size;
+/// A cross-reference stream section (§7.5.8) that takes object number `num`, ending the file.
+/// Rows are Flate-compressed with the PNG Up predictor, and field widths fit the largest value.
+fn write_xref_stream(out: &mut Vec<u8>, trailer: &mut Dict, mut rows: BTreeMap<u32, Row>, num: u32) {
     let xref_at = out.len() as u64;
-    let mut rows: BTreeMap<u32, (u8, u64, u16)> =
-        offsets.iter().map(|(n, (o, g, used))| (*n, if *used { (1, *o, *g) } else { (0, 0, *g) })).collect();
-    rows.insert(xref_num, (1, xref_at, 0));
-    let mut data = Vec::new();
+    rows.insert(num, Row::InFile(xref_at, 0));
+    if trailer.get(b"Prev").is_none() {
+        rows.entry(0).or_insert(Row::Free(65535));
+    }
+    let fields = |r: &Row| match *r {
+        Row::Free(g) => (0u64, 0u64, u64::from(g)),
+        Row::InFile(o, g) => (1, o, u64::from(g)),
+        Row::InStream(s, i) => (2, u64::from(s), u64::from(i)),
+    };
+    let width = |v: u64| ((64 - v.leading_zeros()).div_ceil(8)).max(1) as usize;
+    let (w2, w3) = rows.values().map(fields).fold((1, 1), |(a, b), (_, x, y)| (width(x).max(a), width(y).max(b)));
+    let cols = 1 + w2 + w3;
     let mut index = Vec::new();
     let nums: Vec<u32> = rows.keys().copied().collect();
     let mut i = 0;
@@ -335,21 +401,30 @@ fn write_xref_stream(
         index.push(Object::Int((j - i + 1) as i64));
         i = j + 1;
     }
-    for (t, a, b) in rows.values() {
-        data.push(*t);
-        data.extend_from_slice(&(*a as u32).to_be_bytes());
-        data.extend_from_slice(&b.to_be_bytes());
+    let mut data = Vec::with_capacity(rows.len() * (cols + 1));
+    let mut prev = vec![0u8; cols];
+    for r in rows.values() {
+        let (t, a, b) = fields(r);
+        let mut row = vec![t as u8];
+        row.extend_from_slice(&a.to_be_bytes()[8 - w2..]);
+        row.extend_from_slice(&b.to_be_bytes()[8 - w3..]);
+        data.push(2); // PNG "Up"
+        data.extend(row.iter().zip(&prev).map(|(c, p)| c.wrapping_sub(*p)));
+        prev = row;
     }
     let mut d = trailer.clone();
     d.set(b"Type".to_vec(), Object::name("XRef"));
-    d.set(b"Size".to_vec(), Object::Int(xref_num as i64 + 1));
-    d.set(b"W".to_vec(), Object::Array(vec![Object::Int(1), Object::Int(4), Object::Int(2)]));
+    d.set(b"Size".to_vec(), Object::Int(num as i64 + 1));
+    d.set(b"W".to_vec(), Object::Array(vec![Object::Int(1), Object::Int(w2 as i64), Object::Int(w3 as i64)]));
     d.set(b"Index".to_vec(), Object::Array(index));
-    let stream = Stream::flate(d, &data);
-    write_indirect(xref_num, 0, &Object::Stream(stream), out);
+    let mut stream = Stream::flate(d, &data);
+    let mut parms = Dict::new();
+    parms.set(b"Predictor".to_vec(), Object::Int(12));
+    parms.set(b"Columns".to_vec(), Object::Int(cols as i64));
+    stream.dict.set(b"DecodeParms".to_vec(), Object::Dict(parms));
+    write_indirect(num, 0, &Object::Stream(stream), out);
     let _ = write!(out, "startxref\n{xref_at}\n%%EOF\n");
-    trailer.set(b"Size".to_vec(), Object::Int(xref_num as i64 + 1));
-    Ok(())
+    trailer.set(b"Size".to_vec(), Object::Int(num as i64 + 1));
 }
 
 fn stamp_mod_date(doc: &mut Document, opts: &SaveOptions) {

crates/cos/tests/writer.rs

@@ -0,0 +1,165 @@
+//! Full saves with object streams and cross-reference streams (M1.9), read back by our parser
+//! and by hayro, and incremental saves stacked on top of them.
+
+use std::sync::Arc;
+
+use printcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, write_full, write_incremental};
+
+/// Four pages, a shared font, document info with a non-ASCII title.
+fn fixture() -> Vec<u8> {
+    let mut objs: Vec<String> = vec!["<< /Type /Catalog /Pages 2 0 R >>".into()];
+    let kids: Vec<String> = (0..4).map(|i| format!("{} 0 R", 4 + 2 * i)).collect();
+    objs.push(format!("<< /Type /Pages /Kids [{}] /Count 4 /MediaBox [0 0 200 300] >>", kids.join(" ")));
+    objs.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".into());
+    for i in 0..4 {
+        objs.push(format!("<< /Type /Page /Parent 2 0 R /Contents {} 0 R /Resources << /Font << /F1 3 0 R >> >> >>", 5 + 2 * i));
+        let body = format!("BT /F1 24 Tf 20 150 Td (Page {}) Tj ET", i + 1);
+        objs.push(format!("<< /Length {} >>\nstream\n{body}\nendstream", body.len()));
+    }
+    objs.push("<< /Title (Caf\\351 \\(draft\\)) >>".into());
+    let mut out = b"%PDF-1.4\n".to_vec();
+    let mut offsets = Vec::new();
+    for (i, o) in objs.iter().enumerate() {
+        offsets.push(out.len());
+        out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes());
+    }
+    let xref = out.len();
+    out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes());
+    for o in offsets {
+        out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes());
+    }
+    out.extend_from_slice(
+        format!("trailer\n<< /Size {} /Root 1 0 R /Info {} 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1, objs.len()).as_bytes(),
+    );
+    out
+}
+
+fn title(doc: &Document) -> Vec<u8> {
+    let info = doc.resolve(doc.trailer().get(b"Info").unwrap());
+    info.as_dict().unwrap().get(b"Title").unwrap().as_string().unwrap().bytes.clone()
+}
+
+fn hayro_pages(bytes: &[u8]) -> usize {
+    hayro_syntax::Pdf::new(Arc::new(bytes.to_vec())).expect("hayro opens").pages().len()
+}
+
+#[test]
+fn full_save_packs_objects_into_object_streams() {
+    let doc = Document::open(Arc::new(fixture())).unwrap();
+    let packed = write_full(&doc, &SaveOptions::default()).unwrap();
+    let text = String::from_utf8_lossy(&packed);
+    assert!(packed.starts_with(b"%PDF-1.5"), "object streams need PDF 1.5");
+    assert!(text.contains("/ObjStm") && text.contains("/XRef"));
+    assert!(!text.contains("\nxref\n"), "no classic table");
+    assert!(!text.contains("/Type /Catalog") && !text.contains("/Type/Catalog"), "the catalog is compressed");
+
+    let back = Document::open(Arc::new(packed.clone())).unwrap();
+    assert!(back.repair_log().is_empty(), "{:?}", back.repair_log());
+    assert_eq!(title(&back), b"Caf\xe9 (draft)");
+    assert_eq!(hayro_pages(&packed), 4);
+
+    let classic = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap();
+    assert!(classic.starts_with(b"%PDF-1.4"));
+    assert!(String::from_utf8_lossy(&classic).contains("\nxref\n"));
+    assert_eq!(hayro_pages(&classic), 4);
+}
+
+#[test]
+fn incremental_saves_stack_on_an_xref_stream_file() {
+    let doc = Document::open(Arc::new(fixture())).unwrap();
+    let packed = Arc::new(write_full(&doc, &SaveOptions::default()).unwrap());
+    let mut doc = Document::open(packed.clone()).unwrap();
+    // Edit an object that lives in an object stream.
+    let info = doc.trailer().reference(b"Info").unwrap();
+    doc.update_dict(info, |d: &mut Dict| {
+        d.set(b"Title".to_vec(), Object::String(printcraft_cos::PdfString { bytes: b"Second".to_vec(), hex: false }))
+    })
+    .unwrap();
+    let added = doc.add(Object::Int(42));
+    let updated = write_incremental(&doc, &SaveOptions::default()).unwrap();
+    assert!(updated.starts_with(&packed), "an incremental save only appends");
+    let tail = String::from_utf8_lossy(&updated[packed.len()..]);
+    assert!(tail.contains("/XRef") && tail.contains("/Prev"), "the update uses an xref stream too");
+
+    let back = Document::open(Arc::new(updated.clone())).unwrap();
+    assert!(back.repair_log().is_empty(), "{:?}", back.repair_log());
+    assert_eq!(title(&back), b"Second");
+    assert_eq!(*back.get(ObjRef::new(added.num, 0)), Object::Int(42));
+    assert_eq!(back.revisions().len(), 2);
+    assert_eq!(hayro_pages(&updated), 4);
+}
+
+#[test]
+fn qpdf_accepts_object_stream_output() {
+    let Ok(out) = std::process::Command::new("qpdf").arg("--version").output() else {
+        eprintln!("qpdf not installed: skipped");
+        return;
+    };
+    assert!(out.status.success());
+    let doc = Document::open(Arc::new(fixture())).unwrap();
+    let packed = write_full(&doc, &SaveOptions::default()).unwrap();
+    let path = std::env::temp_dir().join(format!("printcraft-objstm-{}.pdf", std::process::id()));
+    std::fs::write(&path, &packed).unwrap();
+    let check = std::process::Command::new("qpdf").arg("--check").arg(&path).output().unwrap();
+    let _ = std::fs::remove_file(&path);
+    let report = String::from_utf8_lossy(&check.stdout);
+    assert!(check.status.success() && report.contains("No syntax or stream encoding errors"), "{report}");
+}
+
+#[test]
+fn objects_referenced_by_the_encrypt_dictionary_stay_outside_object_streams() {
+    // Encrypt a document, then move its crypt filters into indirect objects, as some producers
+    // do (pdf.js issue7665). A full save must keep them readable before decryption.
+    let doc = Document::open(Arc::new(fixture())).unwrap();
+    let mut enc = doc.clone();
+    enc.set_encryption(&printcraft_cos::NewEncryption {
+        algorithm: printcraft_cos::Algorithm::Aes256,
+        user_password: "",
+        owner_password: "owner",
+        permissions: -4,
+        encrypt_metadata: true,
+        seed: [7; 32],
+    })
+    .unwrap();
+    let bytes = Arc::new(write_full(&enc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap());
+    let mut doc = Document::open(bytes).unwrap();
+    let enc_ref = doc.trailer().reference(b"Encrypt").expect("indirect /Encrypt");
+    let cf = doc.get(enc_ref).as_dict().unwrap().get(b"CF").cloned().unwrap();
+    let cf_ref = doc.add(cf);
+    doc.update_dict(enc_ref, |d| d.set(b"CF".to_vec(), Object::Ref(cf_ref))).unwrap();
+    let packed = write_full(&doc, &SaveOptions::default()).unwrap();
+    let back = Document::open(Arc::new(packed.clone())).expect("reopens");
+    assert!(back.repair_log().is_empty(), "{:?}", back.repair_log());
+    assert_eq!(title(&back), b"Caf\xe9 (draft)");
+    assert_eq!(hayro_pages(&packed), 4);
+}
+
+#[test]
+fn page_tree_loops_in_object_streams_do_not_crash_readers() {
+    // pdf.js Pages-tree-refs.pdf: /Kids 3 → 4 → 5 → 3. Written faithfully into object streams,
+    // it used to overflow hayro-syntax's stack (vendored patch: cycle guard in `resolve_pages`).
+    let objs = [
+        "<< /Type /Catalog /Pages 2 0 R >>",
+        "<< /Type /Pages /Kids [6 0 R 3 0 R] /Count 2 /MediaBox [0 0 595 842] >>",
+        "<< /Type /Pages /Kids [4 0 R] /Count 1 >>",
+        "<< /Type /Pages /Kids [5 0 R] /Count 1 >>",
+        "<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+        "<< /Type /Page /Parent 2 0 R >>",
+    ];
+    let mut out = b"%PDF-1.7\n".to_vec();
+    let mut offsets = Vec::new();
+    for (i, o) in objs.iter().enumerate() {
+        offsets.push(out.len());
+        out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes());
+    }
+    let xref = out.len();
+    out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes());
+    for o in offsets {
+        out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes());
+    }
+    out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes());
+    let doc = Document::open(Arc::new(out)).unwrap();
+    let packed = write_full(&doc, &SaveOptions::default()).unwrap();
+    assert!(String::from_utf8_lossy(&packed).contains("/ObjStm"));
+    assert_eq!(hayro_pages(&packed), 1);
+}

crates/ui-egui/tests/control.rs

@@ -158,7 +158,10 @@ fn loopback_transport_requires_the_token() {
             let mut r = BufReader::new(s).lines();
             let mut out = Vec::new();
             for l in lines {
-                writeln!(w, "{l}").unwrap();
+                // The server closes the connection after a failed auth; later writes may fail.
+                if writeln!(w, "{l}").is_err() {
+                    break;
+                }
                 match r.next() {
                     Some(Ok(reply)) => out.push(serde_json::from_str::<Value>(&reply).unwrap()),
                     _ => break,