From ab87149855cf6cf4c16b23dc9a207929d21fd2ea Mon Sep 17 00:00:00 2001 From: Hamza Date: Mon, 16 Nov 2026 18:51:40 +0500 Subject: [PATCH] craft-csv: decode Excel's character escapes in SYLK text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Excel writes a character outside ASCII in a SYLK text value as ESC N and one byte, the character's ISO 6937 code less 0x80; a diacritic's code is followed by the letter it goes on. The reader kept those bytes, so Größe für Köln opened as GrNHoN{e fNHur KNHoln (storytold/gridcraft#4321). Text values now decode them: a diacritic and letter become the precomposed letter (or the letter and its combining mark when Unicode has no single character), a spacing code its character; a code ISO 6937 doesn't define is kept as written. --- crates/csv/src/sylk.rs | 146 ++++++++++++++++++++++++++++++++++++++- crates/csv/tests/sylk.rs | 25 +++++++ 2 files changed, 168 insertions(+), 3 deletions(-) diff --git a/crates/csv/src/sylk.rs b/crates/csv/src/sylk.rs index f9a74405a..d8c0ebec5 100644 --- a/crates/csv/src/sylk.rs +++ b/crates/csv/src/sylk.rs @@ -219,12 +219,13 @@ fn position(arg: &str, max: u32) -> Option { arg.trim().parse::().ok().filter(|n| (1..=max).contains(n)).map(|n| n - 1) } -/// A `K` value: `"text"` (everything between the first and the last quote, as written), a -/// boolean, a number, or other text (an error value when it starts with `#`). +/// A `K` value: `"text"` (everything between the first and the last quote, with Excel's +/// character escapes decoded), a boolean, a number, or other text (an error value when it starts +/// with `#`). fn parse_value(arg: &str) -> Value { if let Some(text) = arg.strip_prefix('"') { let text = text.strip_suffix('"').unwrap_or(text); - return Value::Text(text.to_string()); + return Value::Text(decode_escapes(text)); } if arg.eq_ignore_ascii_case("TRUE") || arg.eq_ignore_ascii_case("FALSE") { return Value::Bool(arg.eq_ignore_ascii_case("TRUE")); @@ -238,6 +239,145 @@ fn parse_value(arg: &str) -> Value { } } +/// ISO 6937's diacritics (codes 0xC1–0xCF): each code, its combining mark, and the letters it +/// composes with, as pairs of letter and precomposed letter (from Unicode's canonical composition). +const ISO6937_DIACRITICS: [(u8, char, &str); 13] = [ + (0xC1, '\u{0300}', "AÀaàEÈeèIÌiìNǸnǹOÒoòUÙuùWẀwẁYỲyỳ"), // grave + (0xC2, '\u{0301}', "AÁaáCĆcćEÉeéGǴgǵIÍiíKḰkḱLĹlĺMḾmḿNŃnńOÓoóPṔpṕRŔrŕSŚsśUÚuúWẂwẃYÝyýZŹzź"), // acute + (0xC3, '\u{0302}', "AÂaâCĈcĉEÊeêGĜgĝHĤhĥIÎiîJĴjĵOÔoôSŜsŝUÛuûWŴwŵYŶyŷZẐzẑ"), // circumflex + (0xC4, '\u{0303}', "AÃaãEẼeẽIĨiĩNÑnñOÕoõUŨuũVṼvṽYỸyỹ"), // tilde + (0xC5, '\u{0304}', "AĀaāEĒeēGḠgḡIĪiīOŌoōUŪuūYȲyȳ"), // macron + (0xC6, '\u{0306}', "AĂaăEĔeĕGĞgğIĬiĭOŎoŏUŬuŭ"), // breve + (0xC7, '\u{0307}', "AȦaȧBḂbḃCĊcċDḊdḋEĖeėFḞfḟGĠgġHḢhḣIİMṀmṁNṄnṅOȮoȯPṖpṗRṘrṙSṠsṡTṪtṫWẆwẇXẊxẋYẎyẏZŻzż"), // dot above + (0xC8, '\u{0308}', "AÄaäEËeëHḦhḧIÏiïOÖoötẗUÜuüWẄwẅXẌxẍYŸyÿ"), // diaeresis + (0xCA, '\u{030a}', "AÅaåUŮuůwẘyẙ"), // ring + (0xCB, '\u{0327}', "CÇcçDḐdḑEȨeȩGĢgģHḨhḩKĶkķLĻlļNŅnņRŖrŗSŞsşTŢtţ"), // cedilla + (0xCD, '\u{030b}', "OŐoőUŰuű"), // double acute + (0xCE, '\u{0328}', "AĄaąEĘeęIĮiįOǪoǫUŲuų"), // ogonek + (0xCF, '\u{030c}', "AǍaǎCČcčDĎdďEĚeěGǦgǧHȞhȟIǏiǐjǰKǨkǩLĽlľNŇnňOǑoǒRŘrřSŠsšTŤtťUǓuǔZŽzž"), // caron +]; + +/// ISO 6937's spacing characters above ASCII (codes 0xA1–0xFE that aren't diacritics). +const ISO6937_SPACING: [(u8, char); 71] = [ + (0xA1, '¡'), + (0xA2, '¢'), + (0xA3, '£'), + (0xA5, '¥'), + (0xA7, '§'), + (0xA8, '¤'), + (0xA9, '‘'), + (0xAA, '“'), + (0xAB, '«'), + (0xAC, '←'), + (0xAD, '↑'), + (0xAE, '→'), + (0xAF, '↓'), + (0xB0, '°'), + (0xB1, '±'), + (0xB2, '²'), + (0xB3, '³'), + (0xB4, '×'), + (0xB5, 'µ'), + (0xB6, '¶'), + (0xB7, '·'), + (0xB8, '÷'), + (0xB9, '’'), + (0xBA, '”'), + (0xBB, '»'), + (0xBC, '¼'), + (0xBD, '½'), + (0xBE, '¾'), + (0xBF, '¿'), + (0xD0, '―'), + (0xD1, '¹'), + (0xD2, '®'), + (0xD3, '©'), + (0xD4, '™'), + (0xD5, '♪'), + (0xD6, '¬'), + (0xD7, '¦'), + (0xDC, '⅛'), + (0xDD, '⅜'), + (0xDE, '⅝'), + (0xDF, '⅞'), + (0xE0, 'Ω'), + (0xE1, 'Æ'), + (0xE2, 'Đ'), + (0xE3, 'ª'), + (0xE4, 'Ħ'), + (0xE6, 'IJ'), + (0xE7, 'Ŀ'), + (0xE8, 'Ł'), + (0xE9, 'Ø'), + (0xEA, 'Œ'), + (0xEB, 'º'), + (0xEC, 'Þ'), + (0xED, 'Ŧ'), + (0xEE, 'Ŋ'), + (0xEF, 'ʼn'), + (0xF0, 'ĸ'), + (0xF1, 'æ'), + (0xF2, 'đ'), + (0xF3, 'ð'), + (0xF4, 'ħ'), + (0xF5, 'ı'), + (0xF6, 'ij'), + (0xF7, 'ŀ'), + (0xF8, 'ł'), + (0xF9, 'ø'), + (0xFA, 'œ'), + (0xFB, 'ß'), + (0xFC, 'þ'), + (0xFD, 'ŧ'), + (0xFE, 'ŋ'), +]; + +/// Text with Excel's SYLK character escapes decoded. Excel writes a character outside ASCII as +/// `ESC N` and one byte, the character's ISO 6937 code less 0x80. A diacritic's code is followed +/// by the letter it goes on (`ESC N H o` is `ö`, `ESC N {` is `ß`). A code ISO 6937 doesn't define, +/// or an `ESC` that isn't followed by `N` and a code, is kept as written. +fn decode_escapes(text: &str) -> String { + if !text.contains('\u{1b}') { + return text.to_string(); + } + let mut out = String::with_capacity(text.len()); + let mut chars = text.chars(); + while let Some(c) = chars.next() { + let mut ahead = chars.clone(); + let code = (c == '\u{1b}' && ahead.next() == Some('N')) + .then(|| ahead.next()) + .flatten() + .and_then(|b| u8::try_from(b).ok()) + .filter(|b| (0x20..0x80).contains(b)); + let Some(code) = code.map(|b| b + 0x80) else { + out.push(c); + continue; + }; + if let Some(&(_, mark, pairs)) = ISO6937_DIACRITICS.iter().find(|(d, _, _)| *d == code) { + // The letter it goes on: precomposed where Unicode has the pair, else letter and mark. + let Some(base) = ahead.next() else { + out.push(c); + continue; + }; + let letters: Vec = pairs.chars().collect(); + match letters.chunks(2).find(|p| p.first() == Some(&base)).and_then(|p| p.get(1)) { + Some(&composed) => out.push(composed), + None => { + out.push(base); + out.push(mark); + } + } + } else if let Some(&(_, ch)) = ISO6937_SPACING.iter().find(|(s, _)| *s == code) { + out.push(ch); + } else { + out.push(c); + continue; + } + chars = ahead; + } + out +} + /// One part of an R1C1 reference: `R`/`C` followed by an absolute number, a bracketed offset or /// nothing (the cell's own row or column). #[derive(Clone, Copy, Debug, PartialEq)] diff --git a/crates/csv/tests/sylk.rs b/crates/csv/tests/sylk.rs index 763508983..b91d07427 100644 --- a/crates/csv/tests/sylk.rs +++ b/crates/csv/tests/sylk.rs @@ -115,3 +115,28 @@ fn survives_huge_relative_offsets_and_width_floods() { let _ = read(&b); } } + +#[test] +fn decodes_excels_character_escapes_in_text() { + // storytold/gridcraft#4321: Excel writes `Größe für Köln` with `ESC N` escapes (ISO 6937 codes + // less 0x80: `H` is the diaeresis, put on the next letter; `{` is ß). + let file = "ID;PWXL;N;E\r\nC;Y1;X1;K\"Gr\u{1b}NHo\u{1b}N{e f\u{1b}NHur K\u{1b}NHoln\"\r\nC;Y2;X1;K\"Plain ASCII 123\"\r\n\ + C;Y3;X1;K\"\u{1b}NBe \u{1b}NAa \u{1b}NCa \u{1b}NKc \u{1b}NOs \u{1b}NMo \u{1b}NJa \u{1b}NDn\"\r\n\ + C;Y4;X1;K\"\u{1b}NAa \u{1b}NAZ\"\r\nC;Y5;X1;K\"\u{1b}Nq \u{1b}NyStra\u{1b}N{e \u{1b}Na\u{1b}Ni\"\r\n\ + C;Y6;X1;K\"\u{1b}NHq \u{1b}N\u{1b}NHo end\u{1b}N\"\r\nC;Y7;X1;K\"\u{1b}N\u{7f} \u{1b}NL \u{1b}X\"\r\nE\r\n"; + let sh = read(file.as_bytes()).unwrap(); + let value = |a: &str| sh.cells.get(&at(a)).map(|c| c.value.clone()).unwrap_or_default(); + let text = |s: &str| Value::Text(s.into()); + assert_eq!(value("A1"), text("Größe für Köln")); + assert_eq!(value("A2"), text("Plain ASCII 123"), "plain ASCII is unchanged"); + // Acute, grave, circumflex, cedilla, caron, double acute, ring, tilde. + assert_eq!(value("A3"), text("é à â ç š ő å ñ")); + // A letter the mark has no precomposed form with keeps the combining mark. + assert_eq!(value("A4"), text("à Z\u{300}")); + // Spacing characters, one code each: `q` æ, `y` ø, `{` ß, `a` Æ, `i` Ø. + assert_eq!(value("A5"), text("æ øStraße ÆØ")); + // A diacritic on a letter with no precomposed form, and a stray `ESC N` before an escape. + assert_eq!(value("A6"), text("q\u{308} \u{1b}Nö end\u{1b}N")); + // Codes ISO 6937 doesn't define (0xFF, 0xCC) and an `ESC` without `N` stay as written. + assert_eq!(value("A7"), text("\u{1b}N\u{7f} \u{1b}NL \u{1b}X")); +}