diff --git a/crates/csv/README.md b/crates/csv/README.md index c677ec9eb..6369ce8b4 100644 --- a/crates/csv/README.md +++ b/crates/csv/README.md @@ -11,7 +11,7 @@ mapping a SYLK sheet onto a document are the caller's work. | Function | What it does | |---|---| | `decode_text(bytes)` | UTF-8 and UTF-16 byte order marks; without one, valid UTF-8 is kept; other bytes are Windows-1256 when they read as Arabic, else Windows-1252 | -| `sniff_delimiter(text, quote)` | comma, semicolon or tab: the one with the most consistent count per line over the first 20 non-blank lines, outside quotes | +| `sniff_delimiter(text, quote)` | comma, semicolon or tab: the one with the most consistent field count over the first 20 non-blank records in a 64 KiB sample; quote boundaries must be coherent, and ties prefer tab, comma, then semicolon | | `parse_records(text, delim, quote, max_fields, on_record)` | records of fields, one callback per record (return `false` to stop); CR, LF and CRLF line ends; quoted fields with doubled quotes and line breaks; a quote in the middle of a field is text | | `records(text, delim, quote, max_records, max_fields)` | the same, collected | | `push_field(out, field, delim)`, `field_len(field, delim)` | one field as written, quoted only when it holds the delimiter, a quote or a line break; its exact length, to enforce a size cap before writing | diff --git a/crates/csv/src/lib.rs b/crates/csv/src/lib.rs index a773f066f..5cbdc28c5 100644 --- a/crates/csv/src/lib.rs +++ b/crates/csv/src/lib.rs @@ -29,44 +29,96 @@ pub mod sylk; pub use decode::decode_text; -/// Picks comma, semicolon or tab: the candidate with the most consistent count per line -/// (outside quotes) over the first non-blank lines. A candidate must appear on every such line, otherwise comma wins. +/// Picks comma, semicolon or tab by consistent field counts in the first non-blank records. +/// Quotes start only at a candidate field boundary; candidates with incoherent quoted fields +/// are discarded when another coherent candidate separates the records. Ties prefer tab, +/// then comma, then semicolon. Comma is the fallback when no candidate separates the text. pub fn sniff_delimiter(text: &str, quote: char) -> u8 { - let mut best = (b',', 0usize, 0usize); - for d in [',', ';', '\t'] { - let mut counts = vec![]; - let mut n = 0usize; - let mut in_q = false; - let mut blank = true; - for ch in text.chars().take(64 * 1024) { - if !in_q && ch == '\n' { - // Blank lines carry no delimiter evidence. - if !blank { - counts.push(n); - } - n = 0; - blank = true; - if counts.len() >= 20 { - break; + let mut candidates = [(b'\t', 0usize, 0usize, true), (b',', 0, 0, true), (b';', 0, 0, true)]; + const SAMPLE_BYTES: usize = 64 * 1024; + let mut sample_end = text.len().min(SAMPLE_BYTES); + while !text.is_char_boundary(sample_end) { + sample_end -= 1; + } + let sample = &text[..sample_end]; + let truncated = sample_end < text.len(); + for (delimiter, min_count, total, coherent) in &mut candidates { + let mut counts = Vec::new(); + let mut count = 0usize; + let mut quoted = false; + let mut after_quote = false; + let mut field_start = true; + let mut record_has_content = false; + let mut record_coherent = true; + let mut candidate_coherent = true; + let mut reached_record_limit = false; + let mut chars = sample.chars().peekable(); + while let Some(ch) = chars.next() { + if quoted { + if ch == quote { + if chars.peek() == Some("e) { + chars.next(); + record_has_content = true; + } else { + quoted = false; + after_quote = true; + } + } else if !ch.is_whitespace() { + record_has_content = true; } continue; } - if ch != '\r' { - blank = false; + if ch == '\r' || ch == '\n' { + if ch == '\r' && chars.peek() == Some(&'\n') { + chars.next(); + } + if record_has_content { + counts.push(count); + candidate_coherent &= record_coherent; + if counts.len() >= 20 { + reached_record_limit = true; + break; + } + } + count = 0; + quoted = false; + after_quote = false; + field_start = true; + record_has_content = false; + record_coherent = true; + continue; } - if ch == quote { - in_q = !in_q; - } else if !in_q && ch == d { - n += 1; + if ch == quote && field_start { + quoted = true; + field_start = false; + record_has_content = true; + } else if ch == char::from(*delimiter) { + count += 1; + field_start = true; + after_quote = false; + } else { + record_coherent &= !after_quote || ch.is_whitespace(); + record_has_content |= !ch.is_whitespace(); + field_start = false; } } - if !blank { - counts.push(n); + if !reached_record_limit && !truncated { + if quoted { + candidate_coherent = false; + } else if record_has_content { + candidate_coherent &= record_coherent; + counts.push(count); + } } - let min = counts.iter().copied().min().unwrap_or(0); - let total: usize = counts.iter().sum(); - if min > 0 && (min, total) > (best.1, best.2) { - best = (d as u8, min, total); + *coherent &= candidate_coherent; + *min_count = counts.iter().copied().min().unwrap_or(0); + *total = counts.iter().sum(); + } + let prefer_coherent = candidates.iter().any(|(_, min, _, coherent)| *min > 0 && *coherent); + let mut best = (b',', 0usize, 0usize); + for (delimiter, min, total, coherent) in candidates { + if min > 0 && (!prefer_coherent || coherent) && (min, total) > (best.1, best.2) { + best = (delimiter, min, total); } } best.0 @@ -204,6 +256,9 @@ mod tests { assert_eq!(sniff_delimiter("a\tb\n1\t2\n", '"'), b'\t'); assert_eq!(sniff_delimiter("a,b\n\"x;y;z\",2\n", '"'), b','); assert_eq!(sniff_delimiter("single", '"'), b','); + assert_eq!(sniff_delimiter("a,b\tc\n1,2\td\n", '"'), b'\t'); + assert_eq!(sniff_delimiter("a,b;c\n1,2;c\n", '"'), b','); + assert_eq!(sniff_delimiter("a;b\tc\n1;2\td\n", '"'), b'\t'); // A delimiter missing from some lines is text, not a separator (single-column export round trip). assert_eq!(sniff_delimiter("alpha;beta\r\ngamma\r\n", '"'), b','); assert_eq!(sniff_delimiter("alpha\tbeta\r\ngamma\r\n", '"'), b','); @@ -211,6 +266,48 @@ mod tests { assert_eq!(sniff_delimiter("x;y\n1,5;2", '"'), b';'); } + #[test] + fn delimiter_sniffing_preserves_candidate_quote_boundaries() { + for ending in ["\n", "\r\n", "\r"] { + for (text, expected) in [ + (format!("Name,\"preferred\tCity\"{ending}Ada\tLondon{ending}"), b'\t'), + (format!("\"Last, First\";City{ending}Ada;London{ending}"), b';'), + (format!("\"Name\";\"Last, First\"{ending}\"Ada\";\"Lovelace, Countess\"{ending}\"Grace\";\"Hopper, Admiral\"{ending}"), b';'), + (format!("\"Name\"suffix,Title\tCity{ending}\"Ada\"suffix,Analyst\tLondon\tignored{ending}"), b'\t'), + (format!("\r\n \r\n\"Name\";City{ending}Ada;London{ending}"), b';'), + ] { + assert_eq!(sniff_delimiter(&text, '"'), expected, "{text:?}"); + } + } + } + + #[test] + fn delimiter_sniffing_counts_empty_quoted_fields_and_ignores_whitespace_records() { + assert_eq!(sniff_delimiter("\"\";name\n\"\";Ada\n", '"'), b';'); + + let blank_rows = "\t \t\n".repeat(25); + assert_eq!(sniff_delimiter(&format!("{blank_rows}a;b\nc;d\n"), '"'), b';'); + } + + #[test] + fn delimiter_sniffing_does_not_count_the_twentieth_record_twice() { + let mut text = "a;b\tc\td\n".repeat(19); + text.push('a'); + text.push_str(&";".repeat(19)); + text.push_str("\tc\n"); + assert_eq!(sniff_delimiter(&text, '"'), b'\t'); + } + + #[test] + fn delimiter_sniffing_discards_an_incomplete_record_at_the_sample_boundary() { + let complete = "a\tb\tc\n1\t2\t3\n"; + let remaining = 64 * 1024 - complete.len(); + let partial = format!("{}x\t\"unfinished", "z".repeat(remaining - 4)); + assert!(partial.len() >= remaining); + let text = format!("{complete}{partial}\"\n"); + assert_eq!(sniff_delimiter(&text, '"'), b'\t'); + } + #[test] fn parser_stops_collecting_fields_at_the_cap() { let (max, extra) = (16_384usize, 50); diff --git a/crates/eps/src/embedded.rs b/crates/eps/src/embedded.rs new file mode 100644 index 000000000..47f73fb43 --- /dev/null +++ b/crates/eps/src/embedded.rs @@ -0,0 +1,131 @@ +//! Embedded Type 1 and CFF font programs in PostScript / EPS source. + +/// A font program embedded in the PostScript source of an EPS file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EmbeddedFontProgram { + /// The PostScript `/FontName` when one is present in the source. CFF `StartData` blocks store + /// their name inside the CFF Name INDEX, which the font parser reads from `program`. + pub name: Option, + /// The bytes of the Type 1 program or CFF font set. + pub program: Vec, + /// Whether `program` is a CFF font set rather than a Type 1 program. + pub cff: bool, +} + +/// Embedded font programs in a PostScript section, in source order within each format. +/// +/// Type 1 programs are recognized from their `eexec` and `cleartomark` delimiters and are named +/// by `/FontName`. CFF `StartData` blocks are returned without a name because the name is inside +/// the binary CFF data. Malformed or truncated blocks are skipped, as in the original EPS reader. +pub fn embedded_font_programs(ps: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut heads = positions(ps, b"%!PS-AdobeFont"); + heads.extend(positions(ps, b"%!FontType1")); + heads.sort_unstable(); + let names = positions(ps, b"/FontName"); + let mut from = 0; + let mut used = 0; + while let Some(ee) = find_from(ps, b"eexec", from) { + from = ee + 5; + let last_before = |v: &[usize]| v[..v.partition_point(|&p| p < ee)].last().copied().filter(|&p| p >= used); + // The fallback look-back stops at the previous font's end, so it cannot pick up its header. + let start = last_before(&heads).or_else(|| last_before(&names).map(|p| p.saturating_sub(512).max(used))); + let Some(start) = start else { continue }; + let end = find_from(ps, b"cleartomark", ee).map_or(ps.len(), |p| p + 11); + if let Some(name) = name_after(&ps[start..ee], b"/FontName") { + out.push(EmbeddedFontProgram { name: Some(name), program: ps[start..end].to_vec(), cff: false }); + } + from = end; + used = end; + } + + let mut from = 0; + while let Some(sd) = find_from(ps, b"StartData", from) { + from = sd + 9; + let head = String::from_utf8_lossy(&ps[sd.saturating_sub(64)..sd]).into_owned(); + let Some(len) = head.split_whitespace().last().and_then(|t| t.parse::().ok()) else { continue }; + let s = sd + 10; + let Some(data) = ps.get(s..s.saturating_add(len)) else { continue }; + out.push(EmbeddedFontProgram { name: None, program: data.to_vec(), cff: true }); + from = s + len; + } + out +} + +fn find_from(hay: &[u8], needle: &[u8], from: usize) -> Option { + hay.get(from..)?.windows(needle.len()).position(|w| w == needle).map(|p| p + from) +} + +fn positions(hay: &[u8], needle: &[u8]) -> Vec { + let mut out = vec![]; + let mut from = 0; + while let Some(p) = find_from(hay, needle, from) { + out.push(p); + from = p + needle.len(); + } + out +} + +fn name_after(b: &[u8], key: &[u8]) -> Option { + let p = find_from(b, key, 0)? + key.len(); + let rest = &b[p..]; + let s = rest.iter().position(|c| *c == b'/')? + 1; + let e = rest[s..].iter().position(|c| c.is_ascii_whitespace() || matches!(c, b'/' | b'[' | b'(' | b'{')).map_or(rest.len(), |q| s + q); + (e > s).then(|| String::from_utf8_lossy(&rest[s..e]).into_owned()) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::embedded_font_programs; + + #[test] + fn finds_type1_font_programs_by_name() { + let ps = b"%!PS-Adobe-3.0 EPSF-3.0\n%!PS-AdobeFont-1.0: Test\n/FontName /Test def\ncurrentfile eexec\n0000\ncleartomark\n"; + let fonts = embedded_font_programs(ps); + assert_eq!(fonts.len(), 1); + assert_eq!(fonts[0].name.as_deref(), Some("Test")); + assert!(!fonts[0].cff); + assert!(String::from_utf8_lossy(&fonts[0].program).contains("cleartomark")); + } + + #[test] + fn finds_cff_startdata_without_interpreting_its_name_index() { + let ps = b"%!PS-Adobe-3.0 EPSF-3.0\n10 StartData\n0123456789\n"; + let fonts = embedded_font_programs(ps); + assert_eq!(fonts.len(), 1); + assert_eq!(fonts[0].name, None); + assert!(fonts[0].cff); + assert_eq!(fonts[0].program, b"0123456789"); + } + + #[test] + fn ignores_truncated_startdata() { + let fonts = embedded_font_programs(b"%!PS-Adobe-3.0 EPSF-3.0\n100 StartData\nshort"); + assert!(fonts.is_empty()); + } + + #[test] + fn many_headerless_eexec_words_do_not_hide_later_font() { + let mut ps = b"%!PS-Adobe-3.0 EPSF-3.0\n".to_vec(); + ps.extend_from_slice(b"eexec ".repeat(20_000).as_slice()); + ps.extend_from_slice(b"\n%!PS-AdobeFont-1.0: Real\n/FontName /Real def\ncurrentfile eexec\n0000\ncleartomark\n"); + let fonts = embedded_font_programs(&ps); + assert_eq!(fonts.len(), 1); + assert_eq!(fonts[0].name.as_deref(), Some("Real")); + } + + #[test] + fn headerless_font_does_not_reuse_previous_font_header() { + let first = b"%!PS-AdobeFont-1.0: A\n/FontName /A def\ncurrentfile eexec\n0000\ncleartomark\n"; + let second = b"/FontName /B def\ncurrentfile eexec\n0000\ncleartomark\n"; + let mut ps = b"%!PS-Adobe-3.0 EPSF-3.0\n".to_vec(); + ps.extend_from_slice(first); + ps.extend_from_slice(second); + let fonts = embedded_font_programs(&ps); + assert_eq!(fonts.len(), 2); + let b = fonts.iter().find(|font| font.name.as_deref() == Some("B")).unwrap(); + assert!(!String::from_utf8_lossy(&b.program).contains("AdobeFont")); + } +} diff --git a/crates/eps/src/lib.rs b/crates/eps/src/lib.rs index e7a8cee01..4cf84b8c6 100644 --- a/crates/eps/src/lib.rs +++ b/crates/eps/src/lib.rs @@ -18,6 +18,7 @@ pub mod ascii85; mod container; mod dsc; +mod embedded; #[cfg(feature = "paths")] pub mod paths; mod preview; @@ -25,6 +26,7 @@ mod rectangles; pub use container::{DOS_HEADER, DOS_MAGIC, Sections, sections, wrap}; pub use dsc::{Comments, MAX_HEADER, MAX_TRAILER}; +pub use embedded::{EmbeddedFontProgram, embedded_font_programs}; pub use preview::{Bitmap, MAX_EPSI, epsi}; pub use rectangles::{MAX_SIMPLE_RECTANGLE_SAVE_DEPTH, MAX_SIMPLE_RECTANGLE_TOKENS, MAX_SIMPLE_RECTANGLES, Rectangle, simple_rectangles}; diff --git a/crates/gif/src/frames.rs b/crates/gif/src/frames.rs index ab57df9a6..243128809 100644 --- a/crates/gif/src/frames.rs +++ b/crates/gif/src/frames.rs @@ -300,6 +300,11 @@ mod tests { assert_eq!((a.frames.len(), a.stop), (2, Stop::ByteLimit)); assert!(matches!(read_frames(&bytes, &FrameLimits { max_canvas_bytes: 23, ..FrameLimits::default() }), Err(FrameError::LimitExceeded(_)))); assert!(read_frames(&bytes, &FrameLimits { max_canvas_bytes: 24, ..FrameLimits::default() }).is_ok()); + // Canvas and frame buffer each need 24 bytes; decoder allocations share this budget. + assert!(matches!( + read_frames(&bytes, &FrameLimits { max_decode_bytes: Some(24), ..FrameLimits::default() }), + Err(FrameError::LimitExceeded(_)) + )); assert!(read_frames(&bytes, &FrameLimits { max_decode_bytes: Some(1), ..FrameLimits::default() }).is_err()); assert!( matches!(read_frames(&bytes, &FrameLimits { max_frames: 0, ..FrameLimits::default() }), Err(FrameError::LimitExceeded(message)) if message.contains("frame limit is zero"))