From 991ea7dbd6d498bdb8d62a1a1637dba1a9218b85 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 18:07:00 -0500 Subject: [PATCH] Add craft-csv: delimited text and SYLK, from GridCraft (#889) Moved from GridCraft at 5cf52c5f: the encoding detection, delimiter sniffing, record parser and field quoting of crates/xlsx/src/csv.rs, and the SYLK reader of crates/engine/src/io/sylk.rs. Fields stay text and the SYLK reader returns its own Sheet; typed values, dates and the workbook stay in GridCraft. --- Cargo.lock | 7 + Cargo.toml | 1 + README.md | 1 + crates/csv/Cargo.toml | 14 ++ crates/csv/README.md | 36 ++++ crates/csv/src/decode.rs | 258 +++++++++++++++++++++++++++ crates/csv/src/lib.rs | 263 +++++++++++++++++++++++++++ crates/csv/src/sylk.rs | 371 +++++++++++++++++++++++++++++++++++++++ crates/csv/tests/sylk.rs | 117 ++++++++++++ 9 files changed, 1068 insertions(+) create mode 100644 crates/csv/Cargo.toml create mode 100644 crates/csv/README.md create mode 100644 crates/csv/src/decode.rs create mode 100644 crates/csv/src/lib.rs create mode 100644 crates/csv/src/sylk.rs create mode 100644 crates/csv/tests/sylk.rs diff --git a/Cargo.lock b/Cargo.lock index 0a7e71ea7..6d3120edf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -874,6 +874,13 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-csv" +version = "0.1.0" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "craft-dialogs" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index b17dbdd88..601a96e60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ craft-captions = { path = "crates/captions" } craft-cfb = { path = "crates/cfb" } craft-cms = { path = "crates/cms" } craft-crw = { path = "crates/crw" } +craft-csv = { path = "crates/csv" } craft-dialogs = { path = "crates/dialogs" } craft-dxf = { path = "crates/dxf" } craft-drawingml = { path = "crates/drawingml" } diff --git a/README.md b/README.md index ad38249e0..6bf26f3d0 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " Self { + let mut e = Self::default(); + for (i, &b) in bytes.iter().enumerate() { + if b < 0x80 { + continue; + } + let before = i.checked_sub(1).and_then(|j| bytes.get(j)).copied(); + let after = bytes.get(i + 1).copied(); + let next_to = |pred: fn(u8) -> bool| before.is_some_and(pred) || after.is_some_and(pred); + if same_in_cp1252_and_cp1256(b) { + if is_shared_latin_letter(b) && next_to(|x| x >= 0x80) { + e.shared_letters_in_runs += 1; + } + continue; + } + if is_invisible_mark(b) || is_ordinal_after_digit(b, before) { + continue; + } + e.differing += 1; + if is_cp1256_arabic(b) { + e.arabic += 1; + if next_to(|x| x.is_ascii_alphabetic()) { + e.arabic_touching_latin += 1; + } + } + } + e + } + + /// Thresholds chosen against month, day and language names from 95 glibc locales (Western in + /// Windows-1252, Arabic, Persian and Urdu in 1256, Hebrew and Yiddish in 1255) plus CSV samples. + fn is_arabic(&self) -> bool { + self.arabic >= 2 + && self.arabic * 10 >= self.differing * 9 + && self.arabic_touching_latin * 10 <= self.arabic * 3 + && self.shared_letters_in_runs * 20 <= self.arabic + } +} + +fn same_in_cp1252_and_cp1256(b: u8) -> bool { + matches!( + b, + 0x80 | 0x82..=0x89 | 0x8B..=0x8C | 0x91..=0x97 | 0x99 | 0x9B..=0x9C | 0xA0 | 0xA2..=0xA9 | 0xAB..=0xB9 | 0xBB..=0xBE | 0xD7 | 0xE0 | 0xE2 | 0xE7..=0xEB + | 0xEE..=0xEF | 0xF4 | 0xF7 | 0xF9 | 0xFB..=0xFC + ) +} + +/// ZWNJ, ZWJ, LRM and RLM: common inside Persian words and around Arabic cells, but no evidence +/// either way. +fn is_invisible_mark(b: u8) -> bool { + matches!(b, 0x9D | 0x9E | 0xFD | 0xFE) +} + +/// `º` and `ª` after a digit (`1º`, `2ª`) are Western ordinals, though Windows-1256 reads the bytes +/// as `؛` and `ھ`; either way they are no evidence. +fn is_ordinal_after_digit(b: u8, before: Option) -> bool { + matches!(b, 0xAA | 0xBA) && before.is_some_and(|x| x.is_ascii_digit()) +} + +fn is_shared_latin_letter(b: u8) -> bool { + matches!(b, 0xE0 | 0xE2 | 0xE7..=0xEB | 0xEE..=0xEF | 0xF4 | 0xF9 | 0xFB..=0xFC) +} + +fn is_cp1256_arabic(b: u8) -> bool { + matches!( + b, + 0x81 | 0x8A | 0x8D..=0x90 | 0x98 | 0x9A | 0x9F | 0xA1 | 0xAA | 0xBA | 0xBF..=0xD6 | 0xD8..=0xDB | 0xDD..=0xDF | 0xE1 | 0xE3..=0xE6 | 0xEC..=0xED | 0xFF + ) +} + +/// Decodes the bytes of a text file with BOM-aware UTF-8 and UTF-16 handling. Without a BOM, valid UTF-8 is kept. +/// Other bytes decode as Windows-1256 when they read as Arabic (see [`Cp1256Evidence`]), else as +/// Windows-1252. +pub fn decode_text(bytes: &[u8]) -> String { + if let Some(rest) = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]) { + return String::from_utf8_lossy(rest).into_owned(); + } + let utf16 = |b: &[u8], le: bool| -> String { + let units = b.as_chunks::<2>().0.iter().map(|c| if le { u16::from_le_bytes(*c) } else { u16::from_be_bytes(*c) }); + char::decode_utf16(units).map(|r| r.unwrap_or('\u{FFFD}')).collect() + }; + if let Some(rest) = bytes.strip_prefix(&[0xFF, 0xFE]) { + return utf16(rest, true); + } + if let Some(rest) = bytes.strip_prefix(&[0xFE, 0xFF]) { + return utf16(rest, false); + } + match std::str::from_utf8(bytes) { + Ok(s) => s.to_string(), + Err(_) => { + let use_windows_1256 = Cp1256Evidence::of(bytes).is_arabic(); + bytes + .iter() + .map(|&b| match b { + 0x80..=0xFF if use_windows_1256 => { + char::from_u32(CP1256.get((b - 0x80) as usize).copied().unwrap_or(0xFFFD)).unwrap_or('\u{FFFD}') + } + 0x80..=0x9F => char::from_u32(CP1252.get((b - 0x80) as usize).copied().unwrap_or(0xFFFD)).unwrap_or('\u{FFFD}'), + _ => b as char, + }) + .collect() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn encodings() { + let mut b = vec![0xEF, 0xBB, 0xBF]; + b.extend_from_slice("é,x".as_bytes()); + assert_eq!(decode_text(&b), "é,x"); + let mut le = vec![0xFF, 0xFE]; + for u in "ü;1".encode_utf16() { + le.extend_from_slice(&u.to_le_bytes()); + } + assert_eq!(decode_text(&le), "ü;1"); + let mut be = vec![0xFE, 0xFF]; + for u in "ß".encode_utf16() { + be.extend_from_slice(&u.to_be_bytes()); + } + assert_eq!(decode_text(&be), "ß"); + assert_eq!(decode_text(&[0x80, b'a', 0xE9]), "€aé"); + // Bytes that are no text at all still decode. + assert_eq!(decode_text(&[0xFF; 1000]).chars().count(), 1000); + assert_eq!(decode_text(&[0xFF, 0xFE, 0x00, 0xD8]), "\u{FFFD}", "a lone surrogate"); + } + + #[test] + fn windows_1256_table_matches_the_unicode_mapping() { + let at = |b: u8| CP1256.get(usize::from(b - 0x80)).and_then(|&c| char::from_u32(c)); + let expected = [ + (0x80, '€'), + (0x81, 'پ'), + (0x8D, 'چ'), + (0x98, 'ک'), + (0x9D, '\u{200C}'), + (0xA1, '،'), + (0xBA, '؛'), + (0xBF, '؟'), + (0xC7, 'ا'), + (0xC8, 'ب'), + (0xDC, 'ـ'), + (0xE1, 'ل'), + (0xE3, 'م'), + (0xE9, 'é'), + (0xFD, '\u{200E}'), + (0xFE, '\u{200F}'), + (0xFF, 'ے'), + ]; + for (b, c) in expected { + assert_eq!(at(b), Some(c), "0x{b:02X}"); + } + } + + #[test] + fn arabic_csv_in_windows_1256_reads_as_arabic() { + let bytes = [ + 0xC7, 0xE1, 0xC7, 0xD3, 0xE3, b',', 0xC7, 0xE1, 0xDE, 0xED, 0xE3, 0xC9, b'\n', 0xE3, 0xD1, 0xCD, 0xC8, 0xC7, b' ', b'1', b'2', b'3', + 0xA1, b' ', b'4', b'2', 0xBF, b',', 0xC8, 0xED, 0xC7, 0xE4, 0xC7, 0xCA, b' ', b'7', b'\n', + ]; + assert_eq!(decode_text(&bytes), "الاسم,القيمة\nمرحبا 123، 42؟,بيانات 7\n"); + } + + fn reads_as_arabic(bytes: &[u8]) -> bool { + decode_text(bytes).chars().any(|c| ('\u{0600}'..='\u{06FF}').contains(&c)) + } + + #[test] + fn western_1252_text_stays_latin() { + let french = [ + b'C', b'r', 0xE8, b'm', b'e', b' ', b'b', b'r', 0xFB, b'l', 0xE9, b'e', b',', b' ', 0xE0, b' ', b'c', 0xF4, b't', 0xE9, b',', b' ', 0xE9, + b't', 0xE9, + ]; + let german = [ + b'G', b'r', 0xF6, 0xDF, b'e', b',', b' ', 0xDC, b'b', b'e', b'r', b'm', b'a', 0xDF, b',', b' ', b'F', b'u', 0xDF, b'g', 0xE4, b'n', b'g', + b'e', b'r', + ]; + assert_eq!(decode_text(&french), "Crème brûlée, à côté, été"); + assert_eq!(decode_text(&german), "Größe, Übermaß, Fußgänger"); + let portuguese = [ + b'A', 0xC7, 0xC3, b'O', b',', b' ', b'i', b'n', b'f', b'o', b'r', b'm', b'a', 0xE7, 0xE3, b'o', b',', b' ', b's', b'i', b't', b'u', b'a', + 0xE7, 0xE3, b'o', b',', b' ', b'P', 0xE3, b'o', + ]; + assert_eq!(decode_text(&portuguese), "AÇÃO, informação, situação, Pão"); + let nordic = [ + 0xC6, b'r', 0xF8, b',', 0xD8, b's', b't', b',', 0xC5, b's', b'e', b',', 0xDE, 0xF3, b'r', 0xF0, b'u', b'r', b',', b'G', b'u', 0xF0, b'r', + 0xFA, b'n', b',', 0xC4, 0xD6, 0xDC, + ]; + assert_eq!(decode_text(&nordic), "Ærø,Øst,Åse,Þórður,Guðrún,ÄÖÜ"); + let ordinals = [b'1', 0xBA, b',', b'2', 0xBA, b',', b'3', 0xBA, b'\n', b'1', 0xAA, b',', b'2', 0xAA]; + assert_eq!(decode_text(&ordinals), "1º,2º,3º\n1ª,2ª"); + let inside_words = [b'K', 0xE4, b's', b'e', b',', b'B', 0xE4, b'c', b'k', b'e', b'r']; + assert_eq!(decode_text(&inside_words), "Käse,Bäcker"); + } + + #[test] + fn short_arabic_words_read_as_arabic() { + assert_eq!(decode_text(&[0xDD, 0xED, b',', 0xE3, 0xE4, b',', 0xDA, 0xE4, b'\n']), "في,من,عن\n"); + assert_eq!(decode_text(&[0xE4, 0xDA, 0xE3, b',', 0xE1, 0xC7, b'\n', 0xE1, 0xC7, b',', 0xE4, 0xDA, 0xE3, b'\n']), "نعم,لا\nلا,نعم\n"); + assert_eq!(decode_text(&[0x98, 0xCA, 0xC7, 0xC8, b' ', 0x81, 0xC7, 0xD1, 0xD3, 0xED]), "کتاب پارسي"); + let with_marks = [ + 0xDF, 0xCA, 0xC7, 0xC8, 0x9D, 0xE5, 0xC7, b',', 0xCE, 0xC7, 0xE4, 0xE5, 0x9D, 0xE5, 0xC7, b',', 0xE3, 0xCD, 0xE3, 0xCF, 0xFE, b',', 0xDA, + 0xE1, 0xED, 0xFE, + ]; + assert_eq!(decode_text(&with_marks), "كتاب\u{200C}ها,خانه\u{200C}ها,محمد\u{200F},علي\u{200F}"); + } + + #[test] + fn hebrew_1255_bytes_are_not_read_as_arabic() { + let plain = [0xF9, 0xEC, 0xE5, 0xED, b',', 0xF2, 0xE5, 0xEC, 0xED]; + let pointed = [0xF9, 0xC8, 0xD1, 0xEC, 0xE5, 0xC9, 0xED, b' ', 0xF2, 0xE5, 0xC9, 0xEC, 0xC8, 0xED]; + let maqaf = [0xFA, 0xEC, 0xCE, 0xE0, 0xE1, 0xE9, 0xE1, b',', 0xE1, 0xEF, 0xCE, 0xE2, 0xE5, 0xF8, 0xE9, 0xE5, 0xEF]; + let gershayim = [0xF6, 0xE4, 0xD8, 0xEC, b',', 0xE3, 0xD8, 0xF8]; + for bytes in [&plain[..], &pointed, &maqaf, &gershayim] { + assert!(!reads_as_arabic(bytes), "{:?}", decode_text(bytes)); + } + } +} diff --git a/crates/csv/src/lib.rs b/crates/csv/src/lib.rs new file mode 100644 index 000000000..a773f066f --- /dev/null +++ b/crates/csv/src/lib.rs @@ -0,0 +1,263 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! Delimited text (CSV, TSV and their relatives) and SYLK. +//! +//! - [`decode_text`] turns a file's bytes into a string: byte order marks, UTF-8, else +//! Windows-1256 when the bytes read as Arabic and Windows-1252 otherwise. +//! - [`sniff_delimiter`] picks comma, semicolon or tab. +//! - [`parse_records`] splits text into records of fields (RFC 4180 with the usual leniency), +//! handing each record to a callback, with a cap on the fields kept per record. [`records`] +//! collects them. +//! - [`push_field`] and [`field_len`] write one field, quoted when it has to be. +//! - [`sylk`] reads SYLK (`.slk`) files. +//! +//! Fields are text. Reading a field as a number or a date, and deciding how a value is written, +//! belong to the caller. +//! +//! ``` +//! let text = craft_csv::decode_text(b"name;note\r\nAda;\"said \"\"hi\"\"\"\r\n"); +//! let delimiter = char::from(craft_csv::sniff_delimiter(&text, '"')); +//! let rows = craft_csv::records(&text, delimiter, '"', 1000, 1000); +//! assert_eq!(rows, [["name", "note"], ["Ada", "said \"hi\""]]); +//! +//! let mut line = String::new(); +//! craft_csv::push_field(&mut line, "a;b", ';'); +//! assert_eq!(line, "\"a;b\""); +//! ``` + +mod decode; +pub mod sylk; + +pub use decode::decode_text; + +/// Picks comma, semicolon or tab: the candidate with the most consistent count per line +/// (outside quotes) over the first non-blank lines. A candidate must appear on every such line, otherwise comma wins. +pub fn sniff_delimiter(text: &str, quote: char) -> u8 { + let mut best = (b',', 0usize, 0usize); + for d in [',', ';', '\t'] { + let mut counts = vec![]; + let mut n = 0usize; + let mut in_q = false; + let mut blank = true; + for ch in text.chars().take(64 * 1024) { + if !in_q && ch == '\n' { + // Blank lines carry no delimiter evidence. + if !blank { + counts.push(n); + } + n = 0; + blank = true; + if counts.len() >= 20 { + break; + } + continue; + } + if ch != '\r' { + blank = false; + } + if ch == quote { + in_q = !in_q; + } else if !in_q && ch == d { + n += 1; + } + } + if !blank { + counts.push(n); + } + let min = counts.iter().copied().min().unwrap_or(0); + let total: usize = counts.iter().sum(); + if min > 0 && (min, total) > (best.1, best.2) { + best = (d as u8, min, total); + } + } + best.0 +} + +/// Splits delimited text into records of fields and hands each to `on_record`, which returns +/// whether to go on. A record ends at CR, LF or CRLF; a blank line is a record with one empty +/// field. A field that starts with `quote` runs to the closing quote (a doubled quote is one +/// quote character, and line ends inside are text); a quote elsewhere in a field is text, and an +/// unterminated quoted field runs to the end of the text. +/// +/// A record keeps at most `max_fields` fields; the rest are dropped as they are read, so a row +/// of endless delimiters cannot allocate a string per field. +pub fn parse_records(text: &str, delim: char, quote: char, max_fields: usize, mut on_record: impl FnMut(Vec) -> bool) { + let mut field = String::new(); + let mut record: Vec = vec![]; + let mut in_q = false; + let mut chars = text.chars().peekable(); + let mut field_started_quoted = false; + while let Some(c) = chars.next() { + if in_q { + if c == quote { + if chars.peek() == Some("e) { + field.push(quote); + chars.next(); + } else { + in_q = false; + } + } else { + field.push(c); + } + continue; + } + if c == quote && field.is_empty() && !field_started_quoted { + in_q = true; + field_started_quoted = true; + } else if c == delim { + let f = std::mem::take(&mut field); + if record.len() < max_fields { + record.push(f); + } + field_started_quoted = false; + } else if c == '\r' || c == '\n' { + if c == '\r' && chars.peek() == Some(&'\n') { + chars.next(); + } + let f = std::mem::take(&mut field); + if record.len() < max_fields { + record.push(f); + } + field_started_quoted = false; + if !on_record(std::mem::take(&mut record)) { + return; + } + } else { + field.push(c); + } + } + if !field.is_empty() || !record.is_empty() || field_started_quoted { + if record.len() < max_fields { + record.push(field); + } + on_record(record); + } +} + +/// The first `max_records` records of [`parse_records`], each with at most `max_fields` fields. +pub fn records(text: &str, delim: char, quote: char, max_records: usize, max_fields: usize) -> Vec> { + let mut out = Vec::new(); + parse_records(text, delim, quote, max_fields, |record| { + if out.len() < max_records { + out.push(record); + } + out.len() < max_records + }); + out +} + +/// Whether a field must be quoted when written with `delim`: it holds the delimiter, a double +/// quote or a line break. +fn needs_quotes(field: &str, delim: char) -> bool { + field.contains(delim) || field.contains('"') || field.contains('\n') || field.contains('\r') +} + +/// The bytes [`push_field`] appends for `field`, so a writer can enforce a size cap before it +/// writes (no quoted copy of the text is built). +pub fn field_len(field: &str, delim: char) -> usize { + if needs_quotes(field, delim) { field.len() + field.matches('"').count() + 2 } else { field.len() } +} + +/// Appends one field to `out`: as it is, or in double quotes with its quotes doubled when it +/// holds the delimiter, a quote or a line break. Delimiters between fields and the line end +/// (CRLF in RFC 4180) are the caller's. +pub fn push_field(out: &mut String, field: &str, delim: char) { + if !needs_quotes(field, delim) { + out.push_str(field); + return; + } + out.push('"'); + for ch in field.chars() { + if ch == '"' { + out.push('"'); + } + out.push(ch); + } + out.push('"'); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rows(text: &str) -> Vec> { + records(text, ',', '"', usize::MAX, usize::MAX) + } + + #[test] + fn quoting_and_newlines() { + let src = "a,\"b,c\",\"say \"\"hi\"\"\"\r\n\"multi\nline\",,3\n\nlast"; + assert_eq!(rows(src), [vec!["a", "b,c", "say \"hi\""], vec!["multi\nline", "", "3"], vec![""], vec!["last"]]); + // A quote inside an unquoted field is text; an unterminated quote runs to the end. + assert_eq!(rows("a\"b,c\r1"), [vec!["a\"b", "c"], vec!["1"]]); + assert_eq!(rows("\"unterminated,quote\n1,2"), [vec!["unterminated,quote\n1,2"]]); + // An empty quoted last field is still a field; a trailing line end adds no record. + assert_eq!(rows("a,\"\""), [vec!["a", ""]]); + assert_eq!(rows("a\n"), [vec!["a"]]); + assert!(rows("").is_empty()); + // Other delimiters and quotes. + assert_eq!(records("'a;b';c", ';', '\'', 9, 9), [vec!["a;b", "c"]]); + } + + #[test] + fn delimiter_sniffing() { + assert_eq!(sniff_delimiter("a;b;c\n1;2;3\n", '"'), b';'); + assert_eq!(sniff_delimiter("a\tb\n1\t2\n", '"'), b'\t'); + assert_eq!(sniff_delimiter("a,b\n\"x;y;z\",2\n", '"'), b','); + assert_eq!(sniff_delimiter("single", '"'), b','); + // A delimiter missing from some lines is text, not a separator (single-column export round trip). + assert_eq!(sniff_delimiter("alpha;beta\r\ngamma\r\n", '"'), b','); + assert_eq!(sniff_delimiter("alpha\tbeta\r\ngamma\r\n", '"'), b','); + assert_eq!(sniff_delimiter("a;b\r\n\r\nc;d\r\n", '"'), b';'); + assert_eq!(sniff_delimiter("x;y\n1,5;2", '"'), b';'); + } + + #[test] + fn parser_stops_collecting_fields_at_the_cap() { + let (max, extra) = (16_384usize, 50); + // max + extra fields on the first row (the last one filled), then a short second row. + let mut text = ",".repeat(max + extra - 1); + text.push_str("end\nx,y\n"); + let mut lens = vec![]; + parse_records(&text, ',', '"', max, |rec| { + lens.push(rec.len()); + true + }); + assert_eq!(lens, vec![max, 2]); + // An unterminated last row is capped the same way. + assert_eq!(records(&",".repeat(max + extra), ',', '"', 9, max).iter().map(Vec::len).collect::>(), [max]); + } + + #[test] + fn reading_stops_when_the_callback_says_so_and_at_the_record_cap() { + let mut seen = 0; + parse_records("1\n2\n3\n", ',', '"', 9, |_| { + seen += 1; + seen < 2 + }); + assert_eq!(seen, 2); + assert_eq!(records("1\n2\n3\n", ',', '"', 2, 9), [["1"], ["2"]]); + assert!(records("1\n2\n", ',', '"', 0, 9).is_empty()); + } + + #[test] + fn writing_quotes_only_what_needs_it() { + let line = |fields: &[&str], delim: char| { + let mut out = String::new(); + let mut len = 0; + for (i, f) in fields.iter().enumerate() { + if i > 0 { + out.push(delim); + } + len += field_len(f, delim); + push_field(&mut out, f, delim); + } + assert_eq!(out.len(), len + (fields.len() - 1) * delim.len_utf8(), "field_len is exact"); + out + }; + assert_eq!(line(&["", "a,b", "q\"t", "0.1", "x\ny", "r\rs"], ','), ",\"a,b\",\"q\"\"t\",0.1,\"x\ny\",\"r\rs\""); + assert_eq!(line(&["a,b", "c;d"], ';'), "a,b;\"c;d\""); + // Round trip. + let fields = ["a,b", "say \"hi\"", "", "x\r\ny", "plain"]; + assert_eq!(rows(&line(&fields, ',')), [fields]); + } +} diff --git a/crates/csv/src/sylk.rs b/crates/csv/src/sylk.rs new file mode 100644 index 000000000..f9a74405a --- /dev/null +++ b/crates/csv/src/sylk.rs @@ -0,0 +1,371 @@ +//! SYLK (`.slk`): the symbolic link text format older spreadsheets and some legacy systems +//! export. One record per line, fields separated by `;` (`;;` is a literal semicolon): +//! +//! - `ID` starts the file; `E` ends it. +//! - `C;Y;X;K;E` is a cell. `Y` and `X` may be left out to stay on the +//! current row or column. `K` is a number, `"text"`, `TRUE`/`FALSE` or an error; `E` is a +//! formula in R1C1 notation; `S;R;C` shares the formula of another cell. +//! - `P;P` adds a number format to the file's list; `F;P;Y;X` gives a cell the +//! n-th format (0-based), and `F;W ` sets column widths in characters. +//! +//! Other records (fonts, options, window settings) are skipped. Formulas are converted to A1 with +//! the same relative and absolute references, as Excel does when it opens the file. + +use std::collections::{BTreeMap, HashMap}; + +/// The largest SYLK file read: past it the file is refused. +pub const MAX_BYTES: usize = 512 << 20; +/// The grid cells are read into (Excel's): positions outside it make a record unreadable. +pub const MAX_ROWS: u32 = 1_048_576; +pub const MAX_COLS: u32 = 16_384; +/// Column-width records kept from one file; each can cover every column. +const MAX_WIDTH_RECORDS: usize = 1024; + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// The file is larger than [`MAX_BYTES`]. + #[error("the SYLK file is larger than {} MB", MAX_BYTES >> 20)] + TooLarge, + /// The text does not start with an `ID` record. + #[error("not a SYLK file (it does not start with an ID record)")] + NotSylk, +} + +/// A cell's value. +#[derive(Debug, Clone, Default, PartialEq)] +pub enum Value { + #[default] + Empty, + Number(f64), + Text(String), + Bool(bool), + /// An unquoted value that starts with `#`, as written (`#DIV/0!`). Which of these are error + /// values is the caller's decision. + Error(String), +} + +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Cell { + pub value: Value, + /// The formula in A1 notation, without a leading `=`. It is the file's text with its + /// references rewritten, not checked for syntax. + pub formula: Option, + /// The cell's number format code, when the file gives it one other than `General`. + pub format: Option, +} + +/// An `F;W` record: columns `first..=last` (zero-based) are `width` characters wide. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ColumnWidth { + pub first: u32, + pub last: u32, + pub width: f64, +} + +/// The one sheet a SYLK file holds. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Sheet { + /// Cells by zero-based (row, column). + pub cells: BTreeMap<(u32, u32), Cell>, + /// Column widths in file order (a later record overrides an earlier one). + pub column_widths: Vec, + /// Cell records that could not be read (a position outside the grid) and were skipped. + pub skipped: usize, +} + +/// One-letter field tag and its argument. +fn tagged(field: &str) -> (&str, &str) { + field.split_at(field.chars().next().map_or(0, char::len_utf8)) +} + +/// Reads a SYLK file. +pub fn read(bytes: &[u8]) -> Result { + if bytes.len() > MAX_BYTES { + return Err(Error::TooLarge); + } + let text = match std::str::from_utf8(bytes) { + Ok(s) => s.to_string(), + // Older exports are in a single-byte code page: keep each byte as its Latin-1 character. + Err(_) => bytes.iter().map(|&b| char::from(b)).collect(), + }; + let text = text.strip_prefix('\u{feff}').unwrap_or(&text); + if !text.starts_with("ID") { + return Err(Error::NotSylk); + } + let mut sheet = Sheet::default(); + let mut formats: Vec = Vec::new(); + // Cells' number formats by index into `formats`, applied once every format is known. + let mut cell_formats: Vec<((u32, u32), usize)> = Vec::new(); + // Each formula cell's R1C1 text, for the cells that share it. + let mut formulas: HashMap<(u32, u32), String> = HashMap::new(); + let (mut row, mut col) = (0u32, 0u32); + for line in text.lines() { + let fields = split_fields(line); + let Some((kind, rest)) = fields.split_first() else { continue }; + match kind.as_str() { + "C" => cell_record(rest, &mut sheet, &mut formulas, &mut row, &mut col), + "F" => { + let mut format = None; + for f in rest { + let (tag, arg) = tagged(f); + match tag { + "Y" => row = position(arg, MAX_ROWS).unwrap_or(row), + "X" => col = position(arg, MAX_COLS).unwrap_or(col), + "P" => format = arg.parse::().ok(), + "W" => { + let mut parts = arg.split_whitespace(); + if let (Some(first), Some(last), Some(width)) = (parts.next(), parts.next(), parts.next()) + && let (Some(first), Some(last), Ok(width)) = + (position(first, MAX_COLS), position(last, MAX_COLS), width.parse::()) + && first <= last + && (0.0..=255.0).contains(&width) + // A file of width records can't make its reader loop over every + // column again and again. + && sheet.column_widths.len() < MAX_WIDTH_RECORDS + { + sheet.column_widths.push(ColumnWidth { first, last, width }); + } + } + _ => {} + } + } + if let Some(format) = format { + cell_formats.push(((row, col), format)); + } + } + // Number formats only: `P;F…` and `P;E…` are fonts. + "P" => { + if let Some(code) = rest.first().and_then(|f| f.strip_prefix('P')) { + formats.push(code.to_string()); + } + } + "E" => break, + _ => {} + } + } + for (at, index) in cell_formats { + let Some(code) = formats.get(index).filter(|code| !code.is_empty() && *code != "General") else { continue }; + sheet.cells.entry(at).or_default().format = Some(code.clone()); + } + Ok(sheet) +} + +/// A `C` record: a cell's value and formula at the current or the given position. +fn cell_record(fields: &[String], sheet: &mut Sheet, formulas: &mut HashMap<(u32, u32), String>, row: &mut u32, col: &mut u32) { + let (mut value, mut formula, mut shared) = (None, None, None::<(Option, Option)>); + let mut bad = false; + for f in fields { + let (tag, arg) = tagged(f); + match tag { + "Y" => match position(arg, MAX_ROWS) { + Some(r) => *row = r, + None => bad = true, + }, + "X" => match position(arg, MAX_COLS) { + Some(c) => *col = c, + None => bad = true, + }, + "K" => value = Some(parse_value(arg)), + "E" => formula = Some(arg.to_string()), + "S" => shared = Some((None, None)), + // In a shared-formula cell, R and C name the cell whose formula it shares. + "R" if shared.is_some() => shared = shared.map(|(_, c)| (position(arg, MAX_ROWS), c)), + "C" if shared.is_some() => shared = shared.map(|(r, _)| (r, position(arg, MAX_COLS))), + _ => {} + } + } + if bad { + sheet.skipped += 1; + return; + } + let at = (*row, *col); + if let Some((Some(r), Some(c))) = shared { + formula = formulas.get(&(r, c)).cloned(); + } + let cell = sheet.cells.entry(at).or_default(); + if let Some(v) = value { + cell.value = v; + } + if let Some(r1c1) = formula { + cell.formula = Some(r1c1_formula_to_a1(&r1c1, at)); + formulas.insert(at, r1c1); + } +} + +/// A record's fields: `;` separates them and `;;` is a literal semicolon. +fn split_fields(line: &str) -> Vec { + let mut fields = vec![String::new()]; + let mut chars = line.trim_end_matches('\r').chars().peekable(); + while let Some(c) = chars.next() { + if c == ';' { + if chars.peek() == Some(&';') { + chars.next(); + if let Some(f) = fields.last_mut() { + f.push(';'); + } + } else { + fields.push(String::new()); + } + } else if let Some(f) = fields.last_mut() { + f.push(c); + } + } + fields +} + +/// A 1-based row or column number → 0-based, within `max`. +fn position(arg: &str, max: u32) -> Option { + arg.trim().parse::().ok().filter(|n| (1..=max).contains(n)).map(|n| n - 1) +} + +/// A `K` value: `"text"` (everything between the first and the last quote, as written), a +/// boolean, a number, or other text (an error value when it starts with `#`). +fn parse_value(arg: &str) -> Value { + if let Some(text) = arg.strip_prefix('"') { + let text = text.strip_suffix('"').unwrap_or(text); + return Value::Text(text.to_string()); + } + if arg.eq_ignore_ascii_case("TRUE") || arg.eq_ignore_ascii_case("FALSE") { + return Value::Bool(arg.eq_ignore_ascii_case("TRUE")); + } + if arg.starts_with('#') { + return Value::Error(arg.to_string()); + } + match arg.trim().parse::() { + Ok(n) if n.is_finite() => Value::Number(n), + _ => Value::Text(arg.to_string()), + } +} + +/// One part of an R1C1 reference: `R`/`C` followed by an absolute number, a bracketed offset or +/// nothing (the cell's own row or column). +#[derive(Clone, Copy, Debug, PartialEq)] +enum Part { + Absolute(u32), + Relative(i64), +} + +impl Part { + /// The 0-based index and whether it is absolute, or `None` when it falls off the sheet. + fn resolve(self, base: u32, max: u32) -> Option<(u32, bool)> { + match self { + Part::Absolute(n) => (1..=max).contains(&n).then(|| (n - 1, true)), + Part::Relative(k) => i64::from(base).checked_add(k).and_then(|v| u32::try_from(v).ok()).filter(|v| *v < max).map(|v| (v, false)), + } + } +} + +/// Parses the number after `R` or `C` at `i`: digits, `[offset]` or nothing. Returns the part and +/// the index after it, or `None` when the text isn't a reference part. +fn parse_part(chars: &[char], i: usize) -> Option<(Part, usize)> { + match chars.get(i) { + Some('[') => { + let close = chars[i..].iter().position(|&c| c == ']')? + i; + let inner: String = chars[i + 1..close].iter().collect(); + Some((Part::Relative(inner.parse().ok()?), close + 1)) + } + Some(c) if c.is_ascii_digit() => { + let end = chars[i..].iter().position(|c| !c.is_ascii_digit()).map_or(chars.len(), |p| p + i); + let n: String = chars[i..end].iter().collect(); + Some((Part::Absolute(n.parse().unwrap_or(u32::MAX)), end)) + } + _ => Some((Part::Relative(0), i)), + } +} + +fn is_word_char(c: char) -> bool { + c.is_alphanumeric() || matches!(c, '_' | '.' | '\\' | '?' | '$') +} + +/// Rewrites the R1C1 references in a SYLK formula as A1 references for the cell at `at` +/// (zero-based row and column): +/// `RC[-1]` → `B2`, `R1C1` → `$A$1`, `R[-2]C:R[-1]C` → `D2:D3`, a whole column `C4` → `$D:$D` and +/// a whole row `R3` → `$3:$3`. Text in quotes and quoted sheet names are left alone; a reference +/// off the sheet becomes `#REF!`. +pub fn r1c1_formula_to_a1(formula: &str, at: (u32, u32)) -> String { + let chars: Vec = formula.chars().collect(); + let mut out = String::with_capacity(formula.len()); + let mut i = 0; + while i < chars.len() { + let c = chars[i]; + if c == '"' || c == '\'' { + // Copy the quoted run as it is; a doubled quote stays inside it. + let mut j = i + 1; + while j < chars.len() && !(chars[j] == c && chars.get(j + 1) != Some(&c)) { + j += if chars[j] == c { 2 } else { 1 }; + } + let stop = (j + 1).min(chars.len()); + out.extend(&chars[i..stop]); + i = stop; + continue; + } + let starts_word = i == 0 || !is_word_char(chars[i - 1]); + if starts_word + && matches!(c, 'R' | 'r' | 'C' | 'c') + && let Some((text, next)) = reference_at(&chars, i, at) + { + out.push_str(&text); + i = next; + continue; + } + // Copy the rest of a word as it is, so `ROUND` or `Sheet2` isn't read as a reference. + if is_word_char(c) { + let end = (i..chars.len()).find(|&j| !is_word_char(chars[j])).unwrap_or(chars.len()); + out.extend(&chars[i..end]); + i = end; + } else { + out.push(c); + i += 1; + } + } + out +} + +/// The A1 text of the R1C1 reference starting at `i`, and the index after it, or `None` when the +/// word there isn't a reference (a function or a name). +fn reference_at(chars: &[char], i: usize, at: (u32, u32)) -> Option<(String, usize)> { + let mut j = i; + let mut row = None; + let mut col = None; + if matches!(chars.get(j), Some('R' | 'r')) { + let (part, next) = parse_part(chars, j + 1)?; + row = Some(part); + j = next; + } + if matches!(chars.get(j), Some('C' | 'c')) { + let (part, next) = parse_part(chars, j + 1)?; + col = Some(part); + j = next; + } + if row.is_none() && col.is_none() { + return None; + } + // A reference ends the word: `ROUND(`, `RATE` or `R2D2` are names. + if chars.get(j).is_some_and(|&c| is_word_char(c) || c == '(' || c == '[' || c == '!') { + return None; + } + let row_text = |p: Part| p.resolve(at.0, MAX_ROWS).map(|(r, abs)| format!("{}{}", if abs { "$" } else { "" }, r + 1)); + let col_text = |p: Part| p.resolve(at.1, MAX_COLS).map(|(c, abs)| format!("{}{}", if abs { "$" } else { "" }, col_letters(c))); + // One side of a range (`R2:R4`, `C1:C3`) is written alone; a lone row or column is the whole of it. + let prev = chars[..i].iter().rev().find(|c| !c.is_whitespace()); + let in_range = prev == Some(&':') || chars[j..].iter().find(|c| !c.is_whitespace()) == Some(&':'); + let text = match (row, col) { + (Some(r), Some(c)) => row_text(r).zip(col_text(c)).map(|(r, c)| format!("{c}{r}")), + (Some(r), None) => row_text(r).map(|r| if in_range { r } else { format!("{r}:{r}") }), + (None, Some(c)) => col_text(c).map(|c| if in_range { c } else { format!("{c}:{c}") }), + (None, None) => None, + }; + Some((text.unwrap_or_else(|| "#REF!".into()), j)) +} + +/// 0 → `A`, 25 → `Z`, 26 → `AA`. +fn col_letters(col: u32) -> String { + let mut n = u64::from(col) + 1; + let mut out = Vec::new(); + while n > 0 { + out.push(char::from(b'A' + ((n - 1) % 26) as u8)); + n = (n - 1) / 26; + } + out.iter().rev().collect() +} diff --git a/crates/csv/tests/sylk.rs b/crates/csv/tests/sylk.rs new file mode 100644 index 000000000..763508983 --- /dev/null +++ b/crates/csv/tests/sylk.rs @@ -0,0 +1,117 @@ +//! SYLK reading. The files are written here, the way spreadsheets export them. + +use craft_csv::sylk::{ColumnWidth, Error, Value, r1c1_formula_to_a1, read}; + +/// `B3` → (row, column), zero-based. +fn at(a1: &str) -> (u32, u32) { + let col = u32::from(a1.as_bytes()[0] - b'A'); + (a1[1..].parse::().unwrap() - 1, col) +} + +/// A small SYLK file: a header, number formats, column widths, cells that carry on from the last +/// row or column, formulas in R1C1 and a shared formula. +const SYLK: &str = "ID;PTEST;N;E\r\nP;PGeneral\r\nP;P0.00\r\nP;Pyyyy\\-mm\\-dd\r\nF;W1 1 18\r\nF;W2 3 9\r\nB;Y5;X7\r\n\ +C;Y1;X1;K\"Item\"\r\nC;X2;K\"Qty\"\r\nC;X4;K\"Total\"\r\n\ +C;Y2;X1;K\"Bolts;; small\"\r\nC;X2;K12\r\nF;P1;F$2G;X3\r\nC;X3;K0.35\r\nC;X4;K4.2;ERC[-2]*RC[-1]\r\n\ +C;X5;K52;ER2C2+R[1]C[-3]\r\nC;X6;KTRUE\r\nC;X7;K8.4;ERC[-3]*2\r\n\ +C;Y3;X1;K\"Nuts \"\"hex\"\"\"\r\nC;X2;K40\r\nC;X3;K0.12\r\nC;X4;K4.8;ERC[-2]*RC[-1]\r\nC;X5;K\"big\";EIF(R4C4>5,\"big\",\"small\")\r\nC;X7;K9.6;S;R2;C7\r\n\ +C;Y4;X1;K\"Sum\"\r\nC;X4;K9;ESUM(R[-2]C:R[-1]C)\r\nC;X5;K#DIV/0!;E1/0\r\nC;X7;K18;ESUM(C4)\r\n\ +F;P2;FG0G;Y5;X2\r\nC;Y5;X2;K45000\r\nC;X3;K\"R2C3\";E\"R2C3\"\r\nE\r\n"; + +#[test] +fn reads_values_formulas_formats_and_widths() { + let sh = read(SYLK.as_bytes()).unwrap(); + assert_eq!(sh.skipped, 0); + let value = |a: &str| sh.cells.get(&at(a)).map(|c| c.value.clone()).unwrap_or_default(); + let formula = |a: &str| sh.cells.get(&at(a)).and_then(|c| c.formula.clone()); + let text = |s: &str| Value::Text(s.into()); + assert_eq!(value("A1"), text("Item")); + assert_eq!(value("A2"), text("Bolts; small"), ";; is a semicolon"); + assert_eq!(value("A3"), text("Nuts \"\"hex\"\""), "text is kept between the outer quotes as written"); + assert_eq!(value("B2"), Value::Number(12.0)); + assert_eq!(value("F2"), Value::Bool(true)); + assert_eq!(value("E4"), Value::Error("#DIV/0!".into())); + assert_eq!(value("D4"), Value::Number(9.0), "a formula keeps the value the file cached"); + assert_eq!(formula("D2").as_deref(), Some("B2*C2")); + assert_eq!(formula("E2").as_deref(), Some("$B$2+B3")); + assert_eq!(formula("E3").as_deref(), Some("IF($D$4>5,\"big\",\"small\")")); + assert_eq!(formula("D4").as_deref(), Some("SUM(D2:D3)")); + assert_eq!(formula("G3").as_deref(), Some("D3*2"), "a shared formula is the other cell's, moved"); + assert_eq!(formula("G4").as_deref(), Some("SUM($D:$D)"), "C4 alone is the whole column"); + assert_eq!(formula("C5").as_deref(), Some("\"R2C3\""), "text in quotes isn't a reference"); + assert_eq!(formula("B2"), None); + // `F;P1;X3` formats the cell on the current row only. + let format = |a: &str| sh.cells.get(&at(a)).and_then(|c| c.format.clone()); + assert_eq!(format("C2").as_deref(), Some("0.00")); + assert_eq!(format("C3"), None); + assert_eq!(format("B5").as_deref(), Some("yyyy\\-mm\\-dd")); + // Widths are in characters; column D has none. + assert_eq!(sh.column_widths, [ColumnWidth { first: 0, last: 0, width: 18.0 }, ColumnWidth { first: 1, last: 2, width: 9.0 }]); +} + +#[test] +fn r1c1_formulas_become_a1() { + for (r1c1, a1) in [ + ("RC", "B2"), + ("R[1]C[-1]", "A3"), + ("R1C1:R2C2", "$A$1:$B$2"), + ("R3", "$3:$3"), + ("R1:R[2]", "$1:4"), + ("C[1]", "C:C"), + ("SUM(C3:C4)", "SUM($C:$D)"), + ("ROUND(RC[1],2)+RATE(1,2,3)", "ROUND(C2,2)+RATE(1,2,3)"), + ("'R1C1 sheet'!R1C1", "'R1C1 sheet'!$A$1"), + ("Sheet2!RC", "Sheet2!B2"), + ("R2D2+rc", "R2D2+B2"), + ("R[-5]C", "#REF!"), + ("\"RC\"&R1C1", "\"RC\"&$A$1"), + ("RC[26]+RC16384", "AB2+$XFD2"), + ("RC16385", "#REF!"), + ] { + assert_eq!(r1c1_formula_to_a1(r1c1, at("B2")), a1, "{r1c1}"); + } +} + +#[test] +fn rejects_other_files_and_survives_damage() { + assert_eq!(read(b"A,B\n1,2\n"), Err(Error::NotSylk), "a SYLK file starts with ID"); + assert_eq!(read(b""), Err(Error::NotSylk)); + // A byte order mark and a single-byte code page are accepted. + assert_eq!(read("\u{feff}ID;P\nC;Y1;X1;K\"café\"\n".as_bytes()).unwrap().cells[&(0, 0)].value, Value::Text("café".into())); + assert_eq!(read(b"ID;P\nC;Y1;X1;K\"caf\xE9\"\n").unwrap().cells[&(0, 0)].value, Value::Text("café".into())); + // Positions off the sheet, broken numbers, a missing end, a format past the list and a + // formula that is no formula (kept as text for the caller to judge). + let sh = read(b"ID;P\nC;Y0;X1;K1\nC;Y2;X99999;K2\nC;Y3;X1;K1e999\nF;P7;Y3;X1\nF;W5 2 9\nF;W1 2 256\nC;Y4;X1;K5;E(((\n").unwrap(); + assert_eq!(sh.cells.keys().copied().collect::>(), [(2, 0), (3, 0)]); + assert_eq!(sh.cells[&(2, 0)].value, Value::Text("1e999".into())); + assert_eq!((sh.cells[&(3, 0)].value.clone(), sh.cells[&(3, 0)].formula.as_deref()), (Value::Number(5.0), Some("((("))); + assert_eq!(sh.skipped, 2, "the two cells off the sheet"); + assert!(sh.column_widths.is_empty()); +} + +#[test] +fn survives_huge_relative_offsets_and_width_floods() { + // Relative offsets at the edge of i64 fall off the sheet instead of overflowing. + let text = b"ID;P\nC;Y1;X1;K0;ER[9223372036854775807]C\nC;Y2;X1;K0;ER[-9223372036854775808]C[-9223372036854775808]\nE\n"; + let sh = read(text).unwrap(); + assert_eq!(sh.cells[&(0, 0)].formula.as_deref(), Some("#REF!")); + // Width records over every column, many times over: a bounded number is kept. + let mut flood = b"ID;P\n".to_vec(); + for _ in 0..5000 { + flood.extend_from_slice(b"F;W1 16384 12\n"); + } + flood.extend_from_slice(b"C;Y1;X1;K1\nE\n"); + let sh = read(&flood).unwrap(); + assert_eq!(sh.column_widths.len(), 1024); + assert_eq!(sh.cells[&(0, 0)].value, Value::Number(1.0)); + // Truncation and byte damage anywhere never panic. + let good = SYLK.as_bytes().to_vec(); + for cut in 0..good.len() { + let _ = read(&good[..cut]); + let mut b = good.clone(); + b[cut] = b'['; + let _ = read(&b); + b[cut] = 0xFF; + let _ = read(&b); + } +}