mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-13 22:12:12 +00:00
Add craft-csv: delimited text and SYLK, from GridCraft (#889)
Moved from GridCraft at 5cf52c5f: the encoding detection, delimiter sniffing, record parser and field quoting of crates/xlsx/src/csv.rs, and the SYLK reader of crates/engine/src/io/sylk.rs. Fields stay text and the SYLK reader returns its own Sheet; typed values, dates and the workbook stay in GridCraft.
This commit is contained in:
1 parent
f86766e7d6
commit
991ea7dbd6
9 files changed
+1068
No files matched your search
Generated
+7
@@ -874,6 +874,13 @@ dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-csv"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-dialogs"
|
||||
version = "0.1.0"
|
||||
|
||||
@@ -30,6 +30,7 @@ craft-captions = { path = "crates/captions" }
|
||||
craft-cfb = { path = "crates/cfb" }
|
||||
craft-cms = { path = "crates/cms" }
|
||||
craft-crw = { path = "crates/crw" }
|
||||
craft-csv = { path = "crates/csv" }
|
||||
craft-dialogs = { path = "crates/dialogs" }
|
||||
craft-dxf = { path = "crates/dxf" }
|
||||
craft-drawingml = { path = "crates/drawingml" }
|
||||
|
||||
@@ -50,6 +50,7 @@ craft-<name> = { git = "https://github.com/storytold/craft-libs", rev = "<commit
|
||||
the heap container, the entropy coding derived by black-box analysis (frozen code tables), sensor
|
||||
black and white levels and as-shot white balance. The complete derivation record (plan, sources
|
||||
log, scripts, verification) is in `crates/crw/clean-room/`. Colour stays in each app.
|
||||
- [`craft-csv`](crates/csv/README.md): delimited text (CSV, TSV) and SYLK: encoding detection (UTF-8, UTF-16, Windows-1252 or Windows-1256), delimiter sniffing, a record parser with a cap on fields per record, field quoting for writers, and a SYLK (`.slk`) reader with R1C1 formulas rewritten to A1.
|
||||
- [`craft-dialogs`](crates/dialogs/README.md): native file and message dialogs that never block
|
||||
the event loop: queued requests shown at the end of the frame and answered on a later frame,
|
||||
free-standing Open and Folder panels and Save sheets on macOS, case-insensitive filters. What
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
[package]
|
||||
name = "craft-csv"
|
||||
description = "Delimited text (CSV, TSV) and SYLK: encoding detection, delimiter sniffing, a bounded record parser, field quoting, and a SYLK (.slk) reader."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
thiserror = { workspace = true }
|
||||
@@ -0,0 +1,36 @@
|
||||
# craft-csv
|
||||
|
||||
Delimited text (CSV, TSV and their relatives) and SYLK (`.slk`). No `unsafe`; the only
|
||||
dependency is `thiserror`.
|
||||
|
||||
Fields are text. Reading a field as a number or a date, deciding how a value is written, and
|
||||
mapping a SYLK sheet onto a document are the caller's work.
|
||||
|
||||
## Delimited text
|
||||
|
||||
| Function | What it does |
|
||||
|---|---|
|
||||
| `decode_text(bytes)` | UTF-8 and UTF-16 byte order marks; without one, valid UTF-8 is kept; other bytes are Windows-1256 when they read as Arabic, else Windows-1252 |
|
||||
| `sniff_delimiter(text, quote)` | comma, semicolon or tab: the one with the most consistent count per line over the first 20 non-blank lines, outside quotes |
|
||||
| `parse_records(text, delim, quote, max_fields, on_record)` | records of fields, one callback per record (return `false` to stop); CR, LF and CRLF line ends; quoted fields with doubled quotes and line breaks; a quote in the middle of a field is text |
|
||||
| `records(text, delim, quote, max_records, max_fields)` | the same, collected |
|
||||
| `push_field(out, field, delim)`, `field_len(field, delim)` | one field as written, quoted only when it holds the delimiter, a quote or a line break; its exact length, to enforce a size cap before writing |
|
||||
|
||||
`max_fields` bounds what one record keeps, so a row of endless delimiters cannot allocate a
|
||||
string per field. The delimiter between fields and the line end are the writer's own.
|
||||
|
||||
## SYLK
|
||||
|
||||
`sylk::read(bytes)` gives a `sylk::Sheet`: cells by (row, column) with a value, the formula in A1
|
||||
notation (the file's R1C1 text with its references rewritten, not checked for syntax) and the
|
||||
number format code; column widths in characters; and the number of cell records skipped because
|
||||
their position is outside the grid. `sylk::r1c1_formula_to_a1` is the rewrite on its own.
|
||||
|
||||
Files over 512 MB are refused, positions outside 1,048,576 rows by 16,384 columns are skipped,
|
||||
and at most 1,024 column-width records are kept.
|
||||
|
||||
## Origin
|
||||
|
||||
Moved from GridCraft: `crates/xlsx/src/csv.rs` (decoding, sniffing, the record parser and the
|
||||
quoting of its writer) and `crates/engine/src/io/sylk.rs`, at `5cf52c5f`. GridCraft keeps the
|
||||
parts that depend on its model: typed values and dates from fields, and the sheet it builds.
|
||||
@@ -0,0 +1,258 @@
|
||||
//! Bytes of a text file to a string: byte order marks, UTF-8, and a guess between Windows-1252
|
||||
//! and Windows-1256 for everything else.
|
||||
|
||||
/// Windows-1252 code points for bytes 0x80–0x9F (others map to Latin-1).
|
||||
const CP1252: [u32; 32] = [
|
||||
0x20AC, 0x81, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x8D, 0x017D, 0x8F, 0x90, 0x2018, 0x2019,
|
||||
0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x9D, 0x017E, 0x0178,
|
||||
];
|
||||
|
||||
/// Windows-1256 code points for bytes 0x80–0xFF (unicode.org `CP1256.TXT`).
|
||||
const CP1256: [u32; 128] = [
|
||||
0x20AC, 0x067E, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688, 0x06AF, 0x2018,
|
||||
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x06A9, 0x2122, 0x0691, 0x203A, 0x0153, 0x200C, 0x200D, 0x06BA, 0x00A0, 0x060C, 0x00A2, 0x00A3,
|
||||
0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x06BE, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5,
|
||||
0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x061B, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x061F, 0x06C1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627,
|
||||
0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F, 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00D7, 0x0637, 0x0638,
|
||||
0x0639, 0x063A, 0x0640, 0x0641, 0x0642, 0x0643, 0x00E0, 0x0644, 0x00E2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB,
|
||||
0x0649, 0x064A, 0x00EE, 0x00EF, 0x064B, 0x064C, 0x064D, 0x064E, 0x00F4, 0x064F, 0x0650, 0x00F7, 0x0651, 0x00F9, 0x0652, 0x00FB, 0x00FC, 0x200E,
|
||||
0x200F, 0x06D2,
|
||||
];
|
||||
|
||||
/// Evidence that bytes which aren't UTF-8 are Windows-1256 Arabic rather than Windows-1252.
|
||||
#[derive(Default)]
|
||||
struct Cp1256Evidence {
|
||||
/// High bytes that decode to different characters in the two code pages, apart from invisible
|
||||
/// marks and ordinals after a digit.
|
||||
differing: usize,
|
||||
/// Of those, Arabic letters and Arabic punctuation (، ؛ ؟) in Windows-1256.
|
||||
arabic: usize,
|
||||
/// Arabic bytes next to an ASCII letter: Western accents sit inside Latin words, Arabic doesn't.
|
||||
arabic_touching_latin: usize,
|
||||
/// Accented Latin bytes the two code pages share, next to another high byte: Hebrew
|
||||
/// Windows-1255 letters land there, Arabic and Western text rarely do.
|
||||
shared_letters_in_runs: usize,
|
||||
}
|
||||
|
||||
impl Cp1256Evidence {
|
||||
fn of(bytes: &[u8]) -> Self {
|
||||
let mut e = Self::default();
|
||||
for (i, &b) in bytes.iter().enumerate() {
|
||||
if b < 0x80 {
|
||||
continue;
|
||||
}
|
||||
let before = i.checked_sub(1).and_then(|j| bytes.get(j)).copied();
|
||||
let after = bytes.get(i + 1).copied();
|
||||
let next_to = |pred: fn(u8) -> bool| before.is_some_and(pred) || after.is_some_and(pred);
|
||||
if same_in_cp1252_and_cp1256(b) {
|
||||
if is_shared_latin_letter(b) && next_to(|x| x >= 0x80) {
|
||||
e.shared_letters_in_runs += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if is_invisible_mark(b) || is_ordinal_after_digit(b, before) {
|
||||
continue;
|
||||
}
|
||||
e.differing += 1;
|
||||
if is_cp1256_arabic(b) {
|
||||
e.arabic += 1;
|
||||
if next_to(|x| x.is_ascii_alphabetic()) {
|
||||
e.arabic_touching_latin += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
e
|
||||
}
|
||||
|
||||
/// Thresholds chosen against month, day and language names from 95 glibc locales (Western in
|
||||
/// Windows-1252, Arabic, Persian and Urdu in 1256, Hebrew and Yiddish in 1255) plus CSV samples.
|
||||
fn is_arabic(&self) -> bool {
|
||||
self.arabic >= 2
|
||||
&& self.arabic * 10 >= self.differing * 9
|
||||
&& self.arabic_touching_latin * 10 <= self.arabic * 3
|
||||
&& self.shared_letters_in_runs * 20 <= self.arabic
|
||||
}
|
||||
}
|
||||
|
||||
fn same_in_cp1252_and_cp1256(b: u8) -> bool {
|
||||
matches!(
|
||||
b,
|
||||
0x80 | 0x82..=0x89 | 0x8B..=0x8C | 0x91..=0x97 | 0x99 | 0x9B..=0x9C | 0xA0 | 0xA2..=0xA9 | 0xAB..=0xB9 | 0xBB..=0xBE | 0xD7 | 0xE0 | 0xE2 | 0xE7..=0xEB
|
||||
| 0xEE..=0xEF | 0xF4 | 0xF7 | 0xF9 | 0xFB..=0xFC
|
||||
)
|
||||
}
|
||||
|
||||
/// ZWNJ, ZWJ, LRM and RLM: common inside Persian words and around Arabic cells, but no evidence
|
||||
/// either way.
|
||||
fn is_invisible_mark(b: u8) -> bool {
|
||||
matches!(b, 0x9D | 0x9E | 0xFD | 0xFE)
|
||||
}
|
||||
|
||||
/// `º` and `ª` after a digit (`1º`, `2ª`) are Western ordinals, though Windows-1256 reads the bytes
|
||||
/// as `؛` and `ھ`; either way they are no evidence.
|
||||
fn is_ordinal_after_digit(b: u8, before: Option<u8>) -> bool {
|
||||
matches!(b, 0xAA | 0xBA) && before.is_some_and(|x| x.is_ascii_digit())
|
||||
}
|
||||
|
||||
fn is_shared_latin_letter(b: u8) -> bool {
|
||||
matches!(b, 0xE0 | 0xE2 | 0xE7..=0xEB | 0xEE..=0xEF | 0xF4 | 0xF9 | 0xFB..=0xFC)
|
||||
}
|
||||
|
||||
fn is_cp1256_arabic(b: u8) -> bool {
|
||||
matches!(
|
||||
b,
|
||||
0x81 | 0x8A | 0x8D..=0x90 | 0x98 | 0x9A | 0x9F | 0xA1 | 0xAA | 0xBA | 0xBF..=0xD6 | 0xD8..=0xDB | 0xDD..=0xDF | 0xE1 | 0xE3..=0xE6 | 0xEC..=0xED | 0xFF
|
||||
)
|
||||
}
|
||||
|
||||
/// Decodes the bytes of a text file with BOM-aware UTF-8 and UTF-16 handling. Without a BOM, valid UTF-8 is kept.
|
||||
/// Other bytes decode as Windows-1256 when they read as Arabic (see [`Cp1256Evidence`]), else as
|
||||
/// Windows-1252.
|
||||
pub fn decode_text(bytes: &[u8]) -> String {
|
||||
if let Some(rest) = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]) {
|
||||
return String::from_utf8_lossy(rest).into_owned();
|
||||
}
|
||||
let utf16 = |b: &[u8], le: bool| -> String {
|
||||
let units = b.as_chunks::<2>().0.iter().map(|c| if le { u16::from_le_bytes(*c) } else { u16::from_be_bytes(*c) });
|
||||
char::decode_utf16(units).map(|r| r.unwrap_or('\u{FFFD}')).collect()
|
||||
};
|
||||
if let Some(rest) = bytes.strip_prefix(&[0xFF, 0xFE]) {
|
||||
return utf16(rest, true);
|
||||
}
|
||||
if let Some(rest) = bytes.strip_prefix(&[0xFE, 0xFF]) {
|
||||
return utf16(rest, false);
|
||||
}
|
||||
match std::str::from_utf8(bytes) {
|
||||
Ok(s) => s.to_string(),
|
||||
Err(_) => {
|
||||
let use_windows_1256 = Cp1256Evidence::of(bytes).is_arabic();
|
||||
bytes
|
||||
.iter()
|
||||
.map(|&b| match b {
|
||||
0x80..=0xFF if use_windows_1256 => {
|
||||
char::from_u32(CP1256.get((b - 0x80) as usize).copied().unwrap_or(0xFFFD)).unwrap_or('\u{FFFD}')
|
||||
}
|
||||
0x80..=0x9F => char::from_u32(CP1252.get((b - 0x80) as usize).copied().unwrap_or(0xFFFD)).unwrap_or('\u{FFFD}'),
|
||||
_ => b as char,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn encodings() {
|
||||
let mut b = vec![0xEF, 0xBB, 0xBF];
|
||||
b.extend_from_slice("é,x".as_bytes());
|
||||
assert_eq!(decode_text(&b), "é,x");
|
||||
let mut le = vec![0xFF, 0xFE];
|
||||
for u in "ü;1".encode_utf16() {
|
||||
le.extend_from_slice(&u.to_le_bytes());
|
||||
}
|
||||
assert_eq!(decode_text(&le), "ü;1");
|
||||
let mut be = vec![0xFE, 0xFF];
|
||||
for u in "ß".encode_utf16() {
|
||||
be.extend_from_slice(&u.to_be_bytes());
|
||||
}
|
||||
assert_eq!(decode_text(&be), "ß");
|
||||
assert_eq!(decode_text(&[0x80, b'a', 0xE9]), "€aé");
|
||||
// Bytes that are no text at all still decode.
|
||||
assert_eq!(decode_text(&[0xFF; 1000]).chars().count(), 1000);
|
||||
assert_eq!(decode_text(&[0xFF, 0xFE, 0x00, 0xD8]), "\u{FFFD}", "a lone surrogate");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn windows_1256_table_matches_the_unicode_mapping() {
|
||||
let at = |b: u8| CP1256.get(usize::from(b - 0x80)).and_then(|&c| char::from_u32(c));
|
||||
let expected = [
|
||||
(0x80, '€'),
|
||||
(0x81, 'پ'),
|
||||
(0x8D, 'چ'),
|
||||
(0x98, 'ک'),
|
||||
(0x9D, '\u{200C}'),
|
||||
(0xA1, '،'),
|
||||
(0xBA, '؛'),
|
||||
(0xBF, '؟'),
|
||||
(0xC7, 'ا'),
|
||||
(0xC8, 'ب'),
|
||||
(0xDC, 'ـ'),
|
||||
(0xE1, 'ل'),
|
||||
(0xE3, 'م'),
|
||||
(0xE9, 'é'),
|
||||
(0xFD, '\u{200E}'),
|
||||
(0xFE, '\u{200F}'),
|
||||
(0xFF, 'ے'),
|
||||
];
|
||||
for (b, c) in expected {
|
||||
assert_eq!(at(b), Some(c), "0x{b:02X}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn arabic_csv_in_windows_1256_reads_as_arabic() {
|
||||
let bytes = [
|
||||
0xC7, 0xE1, 0xC7, 0xD3, 0xE3, b',', 0xC7, 0xE1, 0xDE, 0xED, 0xE3, 0xC9, b'\n', 0xE3, 0xD1, 0xCD, 0xC8, 0xC7, b' ', b'1', b'2', b'3',
|
||||
0xA1, b' ', b'4', b'2', 0xBF, b',', 0xC8, 0xED, 0xC7, 0xE4, 0xC7, 0xCA, b' ', b'7', b'\n',
|
||||
];
|
||||
assert_eq!(decode_text(&bytes), "الاسم,القيمة\nمرحبا 123، 42؟,بيانات 7\n");
|
||||
}
|
||||
|
||||
fn reads_as_arabic(bytes: &[u8]) -> bool {
|
||||
decode_text(bytes).chars().any(|c| ('\u{0600}'..='\u{06FF}').contains(&c))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn western_1252_text_stays_latin() {
|
||||
let french = [
|
||||
b'C', b'r', 0xE8, b'm', b'e', b' ', b'b', b'r', 0xFB, b'l', 0xE9, b'e', b',', b' ', 0xE0, b' ', b'c', 0xF4, b't', 0xE9, b',', b' ', 0xE9,
|
||||
b't', 0xE9,
|
||||
];
|
||||
let german = [
|
||||
b'G', b'r', 0xF6, 0xDF, b'e', b',', b' ', 0xDC, b'b', b'e', b'r', b'm', b'a', 0xDF, b',', b' ', b'F', b'u', 0xDF, b'g', 0xE4, b'n', b'g',
|
||||
b'e', b'r',
|
||||
];
|
||||
assert_eq!(decode_text(&french), "Crème brûlée, à côté, été");
|
||||
assert_eq!(decode_text(&german), "Größe, Übermaß, Fußgänger");
|
||||
let portuguese = [
|
||||
b'A', 0xC7, 0xC3, b'O', b',', b' ', b'i', b'n', b'f', b'o', b'r', b'm', b'a', 0xE7, 0xE3, b'o', b',', b' ', b's', b'i', b't', b'u', b'a',
|
||||
0xE7, 0xE3, b'o', b',', b' ', b'P', 0xE3, b'o',
|
||||
];
|
||||
assert_eq!(decode_text(&portuguese), "AÇÃO, informação, situação, Pão");
|
||||
let nordic = [
|
||||
0xC6, b'r', 0xF8, b',', 0xD8, b's', b't', b',', 0xC5, b's', b'e', b',', 0xDE, 0xF3, b'r', 0xF0, b'u', b'r', b',', b'G', b'u', 0xF0, b'r',
|
||||
0xFA, b'n', b',', 0xC4, 0xD6, 0xDC,
|
||||
];
|
||||
assert_eq!(decode_text(&nordic), "Ærø,Øst,Åse,Þórður,Guðrún,ÄÖÜ");
|
||||
let ordinals = [b'1', 0xBA, b',', b'2', 0xBA, b',', b'3', 0xBA, b'\n', b'1', 0xAA, b',', b'2', 0xAA];
|
||||
assert_eq!(decode_text(&ordinals), "1º,2º,3º\n1ª,2ª");
|
||||
let inside_words = [b'K', 0xE4, b's', b'e', b',', b'B', 0xE4, b'c', b'k', b'e', b'r'];
|
||||
assert_eq!(decode_text(&inside_words), "Käse,Bäcker");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_arabic_words_read_as_arabic() {
|
||||
assert_eq!(decode_text(&[0xDD, 0xED, b',', 0xE3, 0xE4, b',', 0xDA, 0xE4, b'\n']), "في,من,عن\n");
|
||||
assert_eq!(decode_text(&[0xE4, 0xDA, 0xE3, b',', 0xE1, 0xC7, b'\n', 0xE1, 0xC7, b',', 0xE4, 0xDA, 0xE3, b'\n']), "نعم,لا\nلا,نعم\n");
|
||||
assert_eq!(decode_text(&[0x98, 0xCA, 0xC7, 0xC8, b' ', 0x81, 0xC7, 0xD1, 0xD3, 0xED]), "کتاب پارسي");
|
||||
let with_marks = [
|
||||
0xDF, 0xCA, 0xC7, 0xC8, 0x9D, 0xE5, 0xC7, b',', 0xCE, 0xC7, 0xE4, 0xE5, 0x9D, 0xE5, 0xC7, b',', 0xE3, 0xCD, 0xE3, 0xCF, 0xFE, b',', 0xDA,
|
||||
0xE1, 0xED, 0xFE,
|
||||
];
|
||||
assert_eq!(decode_text(&with_marks), "كتاب\u{200C}ها,خانه\u{200C}ها,محمد\u{200F},علي\u{200F}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hebrew_1255_bytes_are_not_read_as_arabic() {
|
||||
let plain = [0xF9, 0xEC, 0xE5, 0xED, b',', 0xF2, 0xE5, 0xEC, 0xED];
|
||||
let pointed = [0xF9, 0xC8, 0xD1, 0xEC, 0xE5, 0xC9, 0xED, b' ', 0xF2, 0xE5, 0xC9, 0xEC, 0xC8, 0xED];
|
||||
let maqaf = [0xFA, 0xEC, 0xCE, 0xE0, 0xE1, 0xE9, 0xE1, b',', 0xE1, 0xEF, 0xCE, 0xE2, 0xE5, 0xF8, 0xE9, 0xE5, 0xEF];
|
||||
let gershayim = [0xF6, 0xE4, 0xD8, 0xEC, b',', 0xE3, 0xD8, 0xF8];
|
||||
for bytes in [&plain[..], &pointed, &maqaf, &gershayim] {
|
||||
assert!(!reads_as_arabic(bytes), "{:?}", decode_text(bytes));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
//! Delimited text (CSV, TSV and their relatives) and SYLK.
|
||||
//!
|
||||
//! - [`decode_text`] turns a file's bytes into a string: byte order marks, UTF-8, else
|
||||
//! Windows-1256 when the bytes read as Arabic and Windows-1252 otherwise.
|
||||
//! - [`sniff_delimiter`] picks comma, semicolon or tab.
|
||||
//! - [`parse_records`] splits text into records of fields (RFC 4180 with the usual leniency),
|
||||
//! handing each record to a callback, with a cap on the fields kept per record. [`records`]
|
||||
//! collects them.
|
||||
//! - [`push_field`] and [`field_len`] write one field, quoted when it has to be.
|
||||
//! - [`sylk`] reads SYLK (`.slk`) files.
|
||||
//!
|
||||
//! Fields are text. Reading a field as a number or a date, and deciding how a value is written,
|
||||
//! belong to the caller.
|
||||
//!
|
||||
//! ```
|
||||
//! let text = craft_csv::decode_text(b"name;note\r\nAda;\"said \"\"hi\"\"\"\r\n");
|
||||
//! let delimiter = char::from(craft_csv::sniff_delimiter(&text, '"'));
|
||||
//! let rows = craft_csv::records(&text, delimiter, '"', 1000, 1000);
|
||||
//! assert_eq!(rows, [["name", "note"], ["Ada", "said \"hi\""]]);
|
||||
//!
|
||||
//! let mut line = String::new();
|
||||
//! craft_csv::push_field(&mut line, "a;b", ';');
|
||||
//! assert_eq!(line, "\"a;b\"");
|
||||
//! ```
|
||||
|
||||
mod decode;
|
||||
pub mod sylk;
|
||||
|
||||
pub use decode::decode_text;
|
||||
|
||||
/// Picks comma, semicolon or tab: the candidate with the most consistent count per line
|
||||
/// (outside quotes) over the first non-blank lines. A candidate must appear on every such line, otherwise comma wins.
|
||||
pub fn sniff_delimiter(text: &str, quote: char) -> u8 {
|
||||
let mut best = (b',', 0usize, 0usize);
|
||||
for d in [',', ';', '\t'] {
|
||||
let mut counts = vec![];
|
||||
let mut n = 0usize;
|
||||
let mut in_q = false;
|
||||
let mut blank = true;
|
||||
for ch in text.chars().take(64 * 1024) {
|
||||
if !in_q && ch == '\n' {
|
||||
// Blank lines carry no delimiter evidence.
|
||||
if !blank {
|
||||
counts.push(n);
|
||||
}
|
||||
n = 0;
|
||||
blank = true;
|
||||
if counts.len() >= 20 {
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ch != '\r' {
|
||||
blank = false;
|
||||
}
|
||||
if ch == quote {
|
||||
in_q = !in_q;
|
||||
} else if !in_q && ch == d {
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
if !blank {
|
||||
counts.push(n);
|
||||
}
|
||||
let min = counts.iter().copied().min().unwrap_or(0);
|
||||
let total: usize = counts.iter().sum();
|
||||
if min > 0 && (min, total) > (best.1, best.2) {
|
||||
best = (d as u8, min, total);
|
||||
}
|
||||
}
|
||||
best.0
|
||||
}
|
||||
|
||||
/// Splits delimited text into records of fields and hands each to `on_record`, which returns
|
||||
/// whether to go on. A record ends at CR, LF or CRLF; a blank line is a record with one empty
|
||||
/// field. A field that starts with `quote` runs to the closing quote (a doubled quote is one
|
||||
/// quote character, and line ends inside are text); a quote elsewhere in a field is text, and an
|
||||
/// unterminated quoted field runs to the end of the text.
|
||||
///
|
||||
/// A record keeps at most `max_fields` fields; the rest are dropped as they are read, so a row
|
||||
/// of endless delimiters cannot allocate a string per field.
|
||||
pub fn parse_records(text: &str, delim: char, quote: char, max_fields: usize, mut on_record: impl FnMut(Vec<String>) -> bool) {
|
||||
let mut field = String::new();
|
||||
let mut record: Vec<String> = vec![];
|
||||
let mut in_q = false;
|
||||
let mut chars = text.chars().peekable();
|
||||
let mut field_started_quoted = false;
|
||||
while let Some(c) = chars.next() {
|
||||
if in_q {
|
||||
if c == quote {
|
||||
if chars.peek() == Some("e) {
|
||||
field.push(quote);
|
||||
chars.next();
|
||||
} else {
|
||||
in_q = false;
|
||||
}
|
||||
} else {
|
||||
field.push(c);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if c == quote && field.is_empty() && !field_started_quoted {
|
||||
in_q = true;
|
||||
field_started_quoted = true;
|
||||
} else if c == delim {
|
||||
let f = std::mem::take(&mut field);
|
||||
if record.len() < max_fields {
|
||||
record.push(f);
|
||||
}
|
||||
field_started_quoted = false;
|
||||
} else if c == '\r' || c == '\n' {
|
||||
if c == '\r' && chars.peek() == Some(&'\n') {
|
||||
chars.next();
|
||||
}
|
||||
let f = std::mem::take(&mut field);
|
||||
if record.len() < max_fields {
|
||||
record.push(f);
|
||||
}
|
||||
field_started_quoted = false;
|
||||
if !on_record(std::mem::take(&mut record)) {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
field.push(c);
|
||||
}
|
||||
}
|
||||
if !field.is_empty() || !record.is_empty() || field_started_quoted {
|
||||
if record.len() < max_fields {
|
||||
record.push(field);
|
||||
}
|
||||
on_record(record);
|
||||
}
|
||||
}
|
||||
|
||||
/// The first `max_records` records of [`parse_records`], each with at most `max_fields` fields.
|
||||
pub fn records(text: &str, delim: char, quote: char, max_records: usize, max_fields: usize) -> Vec<Vec<String>> {
|
||||
let mut out = Vec::new();
|
||||
parse_records(text, delim, quote, max_fields, |record| {
|
||||
if out.len() < max_records {
|
||||
out.push(record);
|
||||
}
|
||||
out.len() < max_records
|
||||
});
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether a field must be quoted when written with `delim`: it holds the delimiter, a double
|
||||
/// quote or a line break.
|
||||
fn needs_quotes(field: &str, delim: char) -> bool {
|
||||
field.contains(delim) || field.contains('"') || field.contains('\n') || field.contains('\r')
|
||||
}
|
||||
|
||||
/// The bytes [`push_field`] appends for `field`, so a writer can enforce a size cap before it
|
||||
/// writes (no quoted copy of the text is built).
|
||||
pub fn field_len(field: &str, delim: char) -> usize {
|
||||
if needs_quotes(field, delim) { field.len() + field.matches('"').count() + 2 } else { field.len() }
|
||||
}
|
||||
|
||||
/// Appends one field to `out`: as it is, or in double quotes with its quotes doubled when it
|
||||
/// holds the delimiter, a quote or a line break. Delimiters between fields and the line end
|
||||
/// (CRLF in RFC 4180) are the caller's.
|
||||
pub fn push_field(out: &mut String, field: &str, delim: char) {
|
||||
if !needs_quotes(field, delim) {
|
||||
out.push_str(field);
|
||||
return;
|
||||
}
|
||||
out.push('"');
|
||||
for ch in field.chars() {
|
||||
if ch == '"' {
|
||||
out.push('"');
|
||||
}
|
||||
out.push(ch);
|
||||
}
|
||||
out.push('"');
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn rows(text: &str) -> Vec<Vec<String>> {
|
||||
records(text, ',', '"', usize::MAX, usize::MAX)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quoting_and_newlines() {
|
||||
let src = "a,\"b,c\",\"say \"\"hi\"\"\"\r\n\"multi\nline\",,3\n\nlast";
|
||||
assert_eq!(rows(src), [vec!["a", "b,c", "say \"hi\""], vec!["multi\nline", "", "3"], vec![""], vec!["last"]]);
|
||||
// A quote inside an unquoted field is text; an unterminated quote runs to the end.
|
||||
assert_eq!(rows("a\"b,c\r1"), [vec!["a\"b", "c"], vec!["1"]]);
|
||||
assert_eq!(rows("\"unterminated,quote\n1,2"), [vec!["unterminated,quote\n1,2"]]);
|
||||
// An empty quoted last field is still a field; a trailing line end adds no record.
|
||||
assert_eq!(rows("a,\"\""), [vec!["a", ""]]);
|
||||
assert_eq!(rows("a\n"), [vec!["a"]]);
|
||||
assert!(rows("").is_empty());
|
||||
// Other delimiters and quotes.
|
||||
assert_eq!(records("'a;b';c", ';', '\'', 9, 9), [vec!["a;b", "c"]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delimiter_sniffing() {
|
||||
assert_eq!(sniff_delimiter("a;b;c\n1;2;3\n", '"'), b';');
|
||||
assert_eq!(sniff_delimiter("a\tb\n1\t2\n", '"'), b'\t');
|
||||
assert_eq!(sniff_delimiter("a,b\n\"x;y;z\",2\n", '"'), b',');
|
||||
assert_eq!(sniff_delimiter("single", '"'), b',');
|
||||
// A delimiter missing from some lines is text, not a separator (single-column export round trip).
|
||||
assert_eq!(sniff_delimiter("alpha;beta\r\ngamma\r\n", '"'), b',');
|
||||
assert_eq!(sniff_delimiter("alpha\tbeta\r\ngamma\r\n", '"'), b',');
|
||||
assert_eq!(sniff_delimiter("a;b\r\n\r\nc;d\r\n", '"'), b';');
|
||||
assert_eq!(sniff_delimiter("x;y\n1,5;2", '"'), b';');
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_stops_collecting_fields_at_the_cap() {
|
||||
let (max, extra) = (16_384usize, 50);
|
||||
// max + extra fields on the first row (the last one filled), then a short second row.
|
||||
let mut text = ",".repeat(max + extra - 1);
|
||||
text.push_str("end\nx,y\n");
|
||||
let mut lens = vec![];
|
||||
parse_records(&text, ',', '"', max, |rec| {
|
||||
lens.push(rec.len());
|
||||
true
|
||||
});
|
||||
assert_eq!(lens, vec![max, 2]);
|
||||
// An unterminated last row is capped the same way.
|
||||
assert_eq!(records(&",".repeat(max + extra), ',', '"', 9, max).iter().map(Vec::len).collect::<Vec<_>>(), [max]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reading_stops_when_the_callback_says_so_and_at_the_record_cap() {
|
||||
let mut seen = 0;
|
||||
parse_records("1\n2\n3\n", ',', '"', 9, |_| {
|
||||
seen += 1;
|
||||
seen < 2
|
||||
});
|
||||
assert_eq!(seen, 2);
|
||||
assert_eq!(records("1\n2\n3\n", ',', '"', 2, 9), [["1"], ["2"]]);
|
||||
assert!(records("1\n2\n", ',', '"', 0, 9).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writing_quotes_only_what_needs_it() {
|
||||
let line = |fields: &[&str], delim: char| {
|
||||
let mut out = String::new();
|
||||
let mut len = 0;
|
||||
for (i, f) in fields.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push(delim);
|
||||
}
|
||||
len += field_len(f, delim);
|
||||
push_field(&mut out, f, delim);
|
||||
}
|
||||
assert_eq!(out.len(), len + (fields.len() - 1) * delim.len_utf8(), "field_len is exact");
|
||||
out
|
||||
};
|
||||
assert_eq!(line(&["", "a,b", "q\"t", "0.1", "x\ny", "r\rs"], ','), ",\"a,b\",\"q\"\"t\",0.1,\"x\ny\",\"r\rs\"");
|
||||
assert_eq!(line(&["a,b", "c;d"], ';'), "a,b;\"c;d\"");
|
||||
// Round trip.
|
||||
let fields = ["a,b", "say \"hi\"", "", "x\r\ny", "plain"];
|
||||
assert_eq!(rows(&line(&fields, ',')), [fields]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
//! SYLK (`.slk`): the symbolic link text format older spreadsheets and some legacy systems
|
||||
//! export. One record per line, fields separated by `;` (`;;` is a literal semicolon):
|
||||
//!
|
||||
//! - `ID` starts the file; `E` ends it.
|
||||
//! - `C;Y<row>;X<col>;K<value>;E<formula>` is a cell. `Y` and `X` may be left out to stay on the
|
||||
//! current row or column. `K` is a number, `"text"`, `TRUE`/`FALSE` or an error; `E` is a
|
||||
//! formula in R1C1 notation; `S;R<row>;C<col>` shares the formula of another cell.
|
||||
//! - `P;P<code>` adds a number format to the file's list; `F;P<n>;Y<row>;X<col>` gives a cell the
|
||||
//! n-th format (0-based), and `F;W<first> <last> <width>` sets column widths in characters.
|
||||
//!
|
||||
//! Other records (fonts, options, window settings) are skipped. Formulas are converted to A1 with
|
||||
//! the same relative and absolute references, as Excel does when it opens the file.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
|
||||
/// The largest SYLK file read: past it the file is refused.
|
||||
pub const MAX_BYTES: usize = 512 << 20;
|
||||
/// The grid cells are read into (Excel's): positions outside it make a record unreadable.
|
||||
pub const MAX_ROWS: u32 = 1_048_576;
|
||||
pub const MAX_COLS: u32 = 16_384;
|
||||
/// Column-width records kept from one file; each can cover every column.
|
||||
const MAX_WIDTH_RECORDS: usize = 1024;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[non_exhaustive]
|
||||
pub enum Error {
|
||||
/// The file is larger than [`MAX_BYTES`].
|
||||
#[error("the SYLK file is larger than {} MB", MAX_BYTES >> 20)]
|
||||
TooLarge,
|
||||
/// The text does not start with an `ID` record.
|
||||
#[error("not a SYLK file (it does not start with an ID record)")]
|
||||
NotSylk,
|
||||
}
|
||||
|
||||
/// A cell's value.
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub enum Value {
|
||||
#[default]
|
||||
Empty,
|
||||
Number(f64),
|
||||
Text(String),
|
||||
Bool(bool),
|
||||
/// An unquoted value that starts with `#`, as written (`#DIV/0!`). Which of these are error
|
||||
/// values is the caller's decision.
|
||||
Error(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct Cell {
|
||||
pub value: Value,
|
||||
/// The formula in A1 notation, without a leading `=`. It is the file's text with its
|
||||
/// references rewritten, not checked for syntax.
|
||||
pub formula: Option<String>,
|
||||
/// The cell's number format code, when the file gives it one other than `General`.
|
||||
pub format: Option<String>,
|
||||
}
|
||||
|
||||
/// An `F;W` record: columns `first..=last` (zero-based) are `width` characters wide.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct ColumnWidth {
|
||||
pub first: u32,
|
||||
pub last: u32,
|
||||
pub width: f64,
|
||||
}
|
||||
|
||||
/// The one sheet a SYLK file holds.
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct Sheet {
|
||||
/// Cells by zero-based (row, column).
|
||||
pub cells: BTreeMap<(u32, u32), Cell>,
|
||||
/// Column widths in file order (a later record overrides an earlier one).
|
||||
pub column_widths: Vec<ColumnWidth>,
|
||||
/// Cell records that could not be read (a position outside the grid) and were skipped.
|
||||
pub skipped: usize,
|
||||
}
|
||||
|
||||
/// One-letter field tag and its argument.
|
||||
fn tagged(field: &str) -> (&str, &str) {
|
||||
field.split_at(field.chars().next().map_or(0, char::len_utf8))
|
||||
}
|
||||
|
||||
/// Reads a SYLK file.
|
||||
pub fn read(bytes: &[u8]) -> Result<Sheet, Error> {
|
||||
if bytes.len() > MAX_BYTES {
|
||||
return Err(Error::TooLarge);
|
||||
}
|
||||
let text = match std::str::from_utf8(bytes) {
|
||||
Ok(s) => s.to_string(),
|
||||
// Older exports are in a single-byte code page: keep each byte as its Latin-1 character.
|
||||
Err(_) => bytes.iter().map(|&b| char::from(b)).collect(),
|
||||
};
|
||||
let text = text.strip_prefix('\u{feff}').unwrap_or(&text);
|
||||
if !text.starts_with("ID") {
|
||||
return Err(Error::NotSylk);
|
||||
}
|
||||
let mut sheet = Sheet::default();
|
||||
let mut formats: Vec<String> = Vec::new();
|
||||
// Cells' number formats by index into `formats`, applied once every format is known.
|
||||
let mut cell_formats: Vec<((u32, u32), usize)> = Vec::new();
|
||||
// Each formula cell's R1C1 text, for the cells that share it.
|
||||
let mut formulas: HashMap<(u32, u32), String> = HashMap::new();
|
||||
let (mut row, mut col) = (0u32, 0u32);
|
||||
for line in text.lines() {
|
||||
let fields = split_fields(line);
|
||||
let Some((kind, rest)) = fields.split_first() else { continue };
|
||||
match kind.as_str() {
|
||||
"C" => cell_record(rest, &mut sheet, &mut formulas, &mut row, &mut col),
|
||||
"F" => {
|
||||
let mut format = None;
|
||||
for f in rest {
|
||||
let (tag, arg) = tagged(f);
|
||||
match tag {
|
||||
"Y" => row = position(arg, MAX_ROWS).unwrap_or(row),
|
||||
"X" => col = position(arg, MAX_COLS).unwrap_or(col),
|
||||
"P" => format = arg.parse::<usize>().ok(),
|
||||
"W" => {
|
||||
let mut parts = arg.split_whitespace();
|
||||
if let (Some(first), Some(last), Some(width)) = (parts.next(), parts.next(), parts.next())
|
||||
&& let (Some(first), Some(last), Ok(width)) =
|
||||
(position(first, MAX_COLS), position(last, MAX_COLS), width.parse::<f64>())
|
||||
&& first <= last
|
||||
&& (0.0..=255.0).contains(&width)
|
||||
// A file of width records can't make its reader loop over every
|
||||
// column again and again.
|
||||
&& sheet.column_widths.len() < MAX_WIDTH_RECORDS
|
||||
{
|
||||
sheet.column_widths.push(ColumnWidth { first, last, width });
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if let Some(format) = format {
|
||||
cell_formats.push(((row, col), format));
|
||||
}
|
||||
}
|
||||
// Number formats only: `P;F…` and `P;E…` are fonts.
|
||||
"P" => {
|
||||
if let Some(code) = rest.first().and_then(|f| f.strip_prefix('P')) {
|
||||
formats.push(code.to_string());
|
||||
}
|
||||
}
|
||||
"E" => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
for (at, index) in cell_formats {
|
||||
let Some(code) = formats.get(index).filter(|code| !code.is_empty() && *code != "General") else { continue };
|
||||
sheet.cells.entry(at).or_default().format = Some(code.clone());
|
||||
}
|
||||
Ok(sheet)
|
||||
}
|
||||
|
||||
/// A `C` record: a cell's value and formula at the current or the given position.
|
||||
fn cell_record(fields: &[String], sheet: &mut Sheet, formulas: &mut HashMap<(u32, u32), String>, row: &mut u32, col: &mut u32) {
|
||||
let (mut value, mut formula, mut shared) = (None, None, None::<(Option<u32>, Option<u32>)>);
|
||||
let mut bad = false;
|
||||
for f in fields {
|
||||
let (tag, arg) = tagged(f);
|
||||
match tag {
|
||||
"Y" => match position(arg, MAX_ROWS) {
|
||||
Some(r) => *row = r,
|
||||
None => bad = true,
|
||||
},
|
||||
"X" => match position(arg, MAX_COLS) {
|
||||
Some(c) => *col = c,
|
||||
None => bad = true,
|
||||
},
|
||||
"K" => value = Some(parse_value(arg)),
|
||||
"E" => formula = Some(arg.to_string()),
|
||||
"S" => shared = Some((None, None)),
|
||||
// In a shared-formula cell, R and C name the cell whose formula it shares.
|
||||
"R" if shared.is_some() => shared = shared.map(|(_, c)| (position(arg, MAX_ROWS), c)),
|
||||
"C" if shared.is_some() => shared = shared.map(|(r, _)| (r, position(arg, MAX_COLS))),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if bad {
|
||||
sheet.skipped += 1;
|
||||
return;
|
||||
}
|
||||
let at = (*row, *col);
|
||||
if let Some((Some(r), Some(c))) = shared {
|
||||
formula = formulas.get(&(r, c)).cloned();
|
||||
}
|
||||
let cell = sheet.cells.entry(at).or_default();
|
||||
if let Some(v) = value {
|
||||
cell.value = v;
|
||||
}
|
||||
if let Some(r1c1) = formula {
|
||||
cell.formula = Some(r1c1_formula_to_a1(&r1c1, at));
|
||||
formulas.insert(at, r1c1);
|
||||
}
|
||||
}
|
||||
|
||||
/// A record's fields: `;` separates them and `;;` is a literal semicolon.
|
||||
fn split_fields(line: &str) -> Vec<String> {
|
||||
let mut fields = vec![String::new()];
|
||||
let mut chars = line.trim_end_matches('\r').chars().peekable();
|
||||
while let Some(c) = chars.next() {
|
||||
if c == ';' {
|
||||
if chars.peek() == Some(&';') {
|
||||
chars.next();
|
||||
if let Some(f) = fields.last_mut() {
|
||||
f.push(';');
|
||||
}
|
||||
} else {
|
||||
fields.push(String::new());
|
||||
}
|
||||
} else if let Some(f) = fields.last_mut() {
|
||||
f.push(c);
|
||||
}
|
||||
}
|
||||
fields
|
||||
}
|
||||
|
||||
/// A 1-based row or column number → 0-based, within `max`.
|
||||
fn position(arg: &str, max: u32) -> Option<u32> {
|
||||
arg.trim().parse::<u32>().ok().filter(|n| (1..=max).contains(n)).map(|n| n - 1)
|
||||
}
|
||||
|
||||
/// A `K` value: `"text"` (everything between the first and the last quote, as written), a
|
||||
/// boolean, a number, or other text (an error value when it starts with `#`).
|
||||
fn parse_value(arg: &str) -> Value {
|
||||
if let Some(text) = arg.strip_prefix('"') {
|
||||
let text = text.strip_suffix('"').unwrap_or(text);
|
||||
return Value::Text(text.to_string());
|
||||
}
|
||||
if arg.eq_ignore_ascii_case("TRUE") || arg.eq_ignore_ascii_case("FALSE") {
|
||||
return Value::Bool(arg.eq_ignore_ascii_case("TRUE"));
|
||||
}
|
||||
if arg.starts_with('#') {
|
||||
return Value::Error(arg.to_string());
|
||||
}
|
||||
match arg.trim().parse::<f64>() {
|
||||
Ok(n) if n.is_finite() => Value::Number(n),
|
||||
_ => Value::Text(arg.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// One part of an R1C1 reference: `R`/`C` followed by an absolute number, a bracketed offset or
|
||||
/// nothing (the cell's own row or column).
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
enum Part {
|
||||
Absolute(u32),
|
||||
Relative(i64),
|
||||
}
|
||||
|
||||
impl Part {
|
||||
/// The 0-based index and whether it is absolute, or `None` when it falls off the sheet.
|
||||
fn resolve(self, base: u32, max: u32) -> Option<(u32, bool)> {
|
||||
match self {
|
||||
Part::Absolute(n) => (1..=max).contains(&n).then(|| (n - 1, true)),
|
||||
Part::Relative(k) => i64::from(base).checked_add(k).and_then(|v| u32::try_from(v).ok()).filter(|v| *v < max).map(|v| (v, false)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses the number after `R` or `C` at `i`: digits, `[offset]` or nothing. Returns the part and
|
||||
/// the index after it, or `None` when the text isn't a reference part.
|
||||
fn parse_part(chars: &[char], i: usize) -> Option<(Part, usize)> {
|
||||
match chars.get(i) {
|
||||
Some('[') => {
|
||||
let close = chars[i..].iter().position(|&c| c == ']')? + i;
|
||||
let inner: String = chars[i + 1..close].iter().collect();
|
||||
Some((Part::Relative(inner.parse().ok()?), close + 1))
|
||||
}
|
||||
Some(c) if c.is_ascii_digit() => {
|
||||
let end = chars[i..].iter().position(|c| !c.is_ascii_digit()).map_or(chars.len(), |p| p + i);
|
||||
let n: String = chars[i..end].iter().collect();
|
||||
Some((Part::Absolute(n.parse().unwrap_or(u32::MAX)), end))
|
||||
}
|
||||
_ => Some((Part::Relative(0), i)),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_word_char(c: char) -> bool {
|
||||
c.is_alphanumeric() || matches!(c, '_' | '.' | '\\' | '?' | '$')
|
||||
}
|
||||
|
||||
/// Rewrites the R1C1 references in a SYLK formula as A1 references for the cell at `at`
|
||||
/// (zero-based row and column):
|
||||
/// `RC[-1]` → `B2`, `R1C1` → `$A$1`, `R[-2]C:R[-1]C` → `D2:D3`, a whole column `C4` → `$D:$D` and
|
||||
/// a whole row `R3` → `$3:$3`. Text in quotes and quoted sheet names are left alone; a reference
|
||||
/// off the sheet becomes `#REF!`.
|
||||
pub fn r1c1_formula_to_a1(formula: &str, at: (u32, u32)) -> String {
|
||||
let chars: Vec<char> = formula.chars().collect();
|
||||
let mut out = String::with_capacity(formula.len());
|
||||
let mut i = 0;
|
||||
while i < chars.len() {
|
||||
let c = chars[i];
|
||||
if c == '"' || c == '\'' {
|
||||
// Copy the quoted run as it is; a doubled quote stays inside it.
|
||||
let mut j = i + 1;
|
||||
while j < chars.len() && !(chars[j] == c && chars.get(j + 1) != Some(&c)) {
|
||||
j += if chars[j] == c { 2 } else { 1 };
|
||||
}
|
||||
let stop = (j + 1).min(chars.len());
|
||||
out.extend(&chars[i..stop]);
|
||||
i = stop;
|
||||
continue;
|
||||
}
|
||||
let starts_word = i == 0 || !is_word_char(chars[i - 1]);
|
||||
if starts_word
|
||||
&& matches!(c, 'R' | 'r' | 'C' | 'c')
|
||||
&& let Some((text, next)) = reference_at(&chars, i, at)
|
||||
{
|
||||
out.push_str(&text);
|
||||
i = next;
|
||||
continue;
|
||||
}
|
||||
// Copy the rest of a word as it is, so `ROUND` or `Sheet2` isn't read as a reference.
|
||||
if is_word_char(c) {
|
||||
let end = (i..chars.len()).find(|&j| !is_word_char(chars[j])).unwrap_or(chars.len());
|
||||
out.extend(&chars[i..end]);
|
||||
i = end;
|
||||
} else {
|
||||
out.push(c);
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The A1 text of the R1C1 reference starting at `i`, and the index after it, or `None` when the
|
||||
/// word there isn't a reference (a function or a name).
|
||||
fn reference_at(chars: &[char], i: usize, at: (u32, u32)) -> Option<(String, usize)> {
|
||||
let mut j = i;
|
||||
let mut row = None;
|
||||
let mut col = None;
|
||||
if matches!(chars.get(j), Some('R' | 'r')) {
|
||||
let (part, next) = parse_part(chars, j + 1)?;
|
||||
row = Some(part);
|
||||
j = next;
|
||||
}
|
||||
if matches!(chars.get(j), Some('C' | 'c')) {
|
||||
let (part, next) = parse_part(chars, j + 1)?;
|
||||
col = Some(part);
|
||||
j = next;
|
||||
}
|
||||
if row.is_none() && col.is_none() {
|
||||
return None;
|
||||
}
|
||||
// A reference ends the word: `ROUND(`, `RATE` or `R2D2` are names.
|
||||
if chars.get(j).is_some_and(|&c| is_word_char(c) || c == '(' || c == '[' || c == '!') {
|
||||
return None;
|
||||
}
|
||||
let row_text = |p: Part| p.resolve(at.0, MAX_ROWS).map(|(r, abs)| format!("{}{}", if abs { "$" } else { "" }, r + 1));
|
||||
let col_text = |p: Part| p.resolve(at.1, MAX_COLS).map(|(c, abs)| format!("{}{}", if abs { "$" } else { "" }, col_letters(c)));
|
||||
// One side of a range (`R2:R4`, `C1:C3`) is written alone; a lone row or column is the whole of it.
|
||||
let prev = chars[..i].iter().rev().find(|c| !c.is_whitespace());
|
||||
let in_range = prev == Some(&':') || chars[j..].iter().find(|c| !c.is_whitespace()) == Some(&':');
|
||||
let text = match (row, col) {
|
||||
(Some(r), Some(c)) => row_text(r).zip(col_text(c)).map(|(r, c)| format!("{c}{r}")),
|
||||
(Some(r), None) => row_text(r).map(|r| if in_range { r } else { format!("{r}:{r}") }),
|
||||
(None, Some(c)) => col_text(c).map(|c| if in_range { c } else { format!("{c}:{c}") }),
|
||||
(None, None) => None,
|
||||
};
|
||||
Some((text.unwrap_or_else(|| "#REF!".into()), j))
|
||||
}
|
||||
|
||||
/// 0 → `A`, 25 → `Z`, 26 → `AA`.
|
||||
fn col_letters(col: u32) -> String {
|
||||
let mut n = u64::from(col) + 1;
|
||||
let mut out = Vec::new();
|
||||
while n > 0 {
|
||||
out.push(char::from(b'A' + ((n - 1) % 26) as u8));
|
||||
n = (n - 1) / 26;
|
||||
}
|
||||
out.iter().rev().collect()
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
//! SYLK reading. The files are written here, the way spreadsheets export them.
|
||||
|
||||
use craft_csv::sylk::{ColumnWidth, Error, Value, r1c1_formula_to_a1, read};
|
||||
|
||||
/// `B3` → (row, column), zero-based.
|
||||
fn at(a1: &str) -> (u32, u32) {
|
||||
let col = u32::from(a1.as_bytes()[0] - b'A');
|
||||
(a1[1..].parse::<u32>().unwrap() - 1, col)
|
||||
}
|
||||
|
||||
/// A small SYLK file: a header, number formats, column widths, cells that carry on from the last
|
||||
/// row or column, formulas in R1C1 and a shared formula.
|
||||
const SYLK: &str = "ID;PTEST;N;E\r\nP;PGeneral\r\nP;P0.00\r\nP;Pyyyy\\-mm\\-dd\r\nF;W1 1 18\r\nF;W2 3 9\r\nB;Y5;X7\r\n\
|
||||
C;Y1;X1;K\"Item\"\r\nC;X2;K\"Qty\"\r\nC;X4;K\"Total\"\r\n\
|
||||
C;Y2;X1;K\"Bolts;; small\"\r\nC;X2;K12\r\nF;P1;F$2G;X3\r\nC;X3;K0.35\r\nC;X4;K4.2;ERC[-2]*RC[-1]\r\n\
|
||||
C;X5;K52;ER2C2+R[1]C[-3]\r\nC;X6;KTRUE\r\nC;X7;K8.4;ERC[-3]*2\r\n\
|
||||
C;Y3;X1;K\"Nuts \"\"hex\"\"\"\r\nC;X2;K40\r\nC;X3;K0.12\r\nC;X4;K4.8;ERC[-2]*RC[-1]\r\nC;X5;K\"big\";EIF(R4C4>5,\"big\",\"small\")\r\nC;X7;K9.6;S;R2;C7\r\n\
|
||||
C;Y4;X1;K\"Sum\"\r\nC;X4;K9;ESUM(R[-2]C:R[-1]C)\r\nC;X5;K#DIV/0!;E1/0\r\nC;X7;K18;ESUM(C4)\r\n\
|
||||
F;P2;FG0G;Y5;X2\r\nC;Y5;X2;K45000\r\nC;X3;K\"R2C3\";E\"R2C3\"\r\nE\r\n";
|
||||
|
||||
#[test]
|
||||
fn reads_values_formulas_formats_and_widths() {
|
||||
let sh = read(SYLK.as_bytes()).unwrap();
|
||||
assert_eq!(sh.skipped, 0);
|
||||
let value = |a: &str| sh.cells.get(&at(a)).map(|c| c.value.clone()).unwrap_or_default();
|
||||
let formula = |a: &str| sh.cells.get(&at(a)).and_then(|c| c.formula.clone());
|
||||
let text = |s: &str| Value::Text(s.into());
|
||||
assert_eq!(value("A1"), text("Item"));
|
||||
assert_eq!(value("A2"), text("Bolts; small"), ";; is a semicolon");
|
||||
assert_eq!(value("A3"), text("Nuts \"\"hex\"\""), "text is kept between the outer quotes as written");
|
||||
assert_eq!(value("B2"), Value::Number(12.0));
|
||||
assert_eq!(value("F2"), Value::Bool(true));
|
||||
assert_eq!(value("E4"), Value::Error("#DIV/0!".into()));
|
||||
assert_eq!(value("D4"), Value::Number(9.0), "a formula keeps the value the file cached");
|
||||
assert_eq!(formula("D2").as_deref(), Some("B2*C2"));
|
||||
assert_eq!(formula("E2").as_deref(), Some("$B$2+B3"));
|
||||
assert_eq!(formula("E3").as_deref(), Some("IF($D$4>5,\"big\",\"small\")"));
|
||||
assert_eq!(formula("D4").as_deref(), Some("SUM(D2:D3)"));
|
||||
assert_eq!(formula("G3").as_deref(), Some("D3*2"), "a shared formula is the other cell's, moved");
|
||||
assert_eq!(formula("G4").as_deref(), Some("SUM($D:$D)"), "C4 alone is the whole column");
|
||||
assert_eq!(formula("C5").as_deref(), Some("\"R2C3\""), "text in quotes isn't a reference");
|
||||
assert_eq!(formula("B2"), None);
|
||||
// `F;P1;X3` formats the cell on the current row only.
|
||||
let format = |a: &str| sh.cells.get(&at(a)).and_then(|c| c.format.clone());
|
||||
assert_eq!(format("C2").as_deref(), Some("0.00"));
|
||||
assert_eq!(format("C3"), None);
|
||||
assert_eq!(format("B5").as_deref(), Some("yyyy\\-mm\\-dd"));
|
||||
// Widths are in characters; column D has none.
|
||||
assert_eq!(sh.column_widths, [ColumnWidth { first: 0, last: 0, width: 18.0 }, ColumnWidth { first: 1, last: 2, width: 9.0 }]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn r1c1_formulas_become_a1() {
|
||||
for (r1c1, a1) in [
|
||||
("RC", "B2"),
|
||||
("R[1]C[-1]", "A3"),
|
||||
("R1C1:R2C2", "$A$1:$B$2"),
|
||||
("R3", "$3:$3"),
|
||||
("R1:R[2]", "$1:4"),
|
||||
("C[1]", "C:C"),
|
||||
("SUM(C3:C4)", "SUM($C:$D)"),
|
||||
("ROUND(RC[1],2)+RATE(1,2,3)", "ROUND(C2,2)+RATE(1,2,3)"),
|
||||
("'R1C1 sheet'!R1C1", "'R1C1 sheet'!$A$1"),
|
||||
("Sheet2!RC", "Sheet2!B2"),
|
||||
("R2D2+rc", "R2D2+B2"),
|
||||
("R[-5]C", "#REF!"),
|
||||
("\"RC\"&R1C1", "\"RC\"&$A$1"),
|
||||
("RC[26]+RC16384", "AB2+$XFD2"),
|
||||
("RC16385", "#REF!"),
|
||||
] {
|
||||
assert_eq!(r1c1_formula_to_a1(r1c1, at("B2")), a1, "{r1c1}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_other_files_and_survives_damage() {
|
||||
assert_eq!(read(b"A,B\n1,2\n"), Err(Error::NotSylk), "a SYLK file starts with ID");
|
||||
assert_eq!(read(b""), Err(Error::NotSylk));
|
||||
// A byte order mark and a single-byte code page are accepted.
|
||||
assert_eq!(read("\u{feff}ID;P\nC;Y1;X1;K\"café\"\n".as_bytes()).unwrap().cells[&(0, 0)].value, Value::Text("café".into()));
|
||||
assert_eq!(read(b"ID;P\nC;Y1;X1;K\"caf\xE9\"\n").unwrap().cells[&(0, 0)].value, Value::Text("café".into()));
|
||||
// Positions off the sheet, broken numbers, a missing end, a format past the list and a
|
||||
// formula that is no formula (kept as text for the caller to judge).
|
||||
let sh = read(b"ID;P\nC;Y0;X1;K1\nC;Y2;X99999;K2\nC;Y3;X1;K1e999\nF;P7;Y3;X1\nF;W5 2 9\nF;W1 2 256\nC;Y4;X1;K5;E(((\n").unwrap();
|
||||
assert_eq!(sh.cells.keys().copied().collect::<Vec<_>>(), [(2, 0), (3, 0)]);
|
||||
assert_eq!(sh.cells[&(2, 0)].value, Value::Text("1e999".into()));
|
||||
assert_eq!((sh.cells[&(3, 0)].value.clone(), sh.cells[&(3, 0)].formula.as_deref()), (Value::Number(5.0), Some("(((")));
|
||||
assert_eq!(sh.skipped, 2, "the two cells off the sheet");
|
||||
assert!(sh.column_widths.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn survives_huge_relative_offsets_and_width_floods() {
|
||||
// Relative offsets at the edge of i64 fall off the sheet instead of overflowing.
|
||||
let text = b"ID;P\nC;Y1;X1;K0;ER[9223372036854775807]C\nC;Y2;X1;K0;ER[-9223372036854775808]C[-9223372036854775808]\nE\n";
|
||||
let sh = read(text).unwrap();
|
||||
assert_eq!(sh.cells[&(0, 0)].formula.as_deref(), Some("#REF!"));
|
||||
// Width records over every column, many times over: a bounded number is kept.
|
||||
let mut flood = b"ID;P\n".to_vec();
|
||||
for _ in 0..5000 {
|
||||
flood.extend_from_slice(b"F;W1 16384 12\n");
|
||||
}
|
||||
flood.extend_from_slice(b"C;Y1;X1;K1\nE\n");
|
||||
let sh = read(&flood).unwrap();
|
||||
assert_eq!(sh.column_widths.len(), 1024);
|
||||
assert_eq!(sh.cells[&(0, 0)].value, Value::Number(1.0));
|
||||
// Truncation and byte damage anywhere never panic.
|
||||
let good = SYLK.as_bytes().to_vec();
|
||||
for cut in 0..good.len() {
|
||||
let _ = read(&good[..cut]);
|
||||
let mut b = good.clone();
|
||||
b[cut] = b'[';
|
||||
let _ = read(&b);
|
||||
b[cut] = 0xFF;
|
||||
let _ = read(&b);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user