craft-csv: decode Excel's character escapes in SYLK text

Excel writes a character outside ASCII in a SYLK text value as ESC N and one byte, the character's ISO 6937 code less 0x80; a diacritic's code is followed by the letter it goes on. The reader kept those bytes, so Größe für Köln opened as Gr<ESC>NHo<ESC>N{e f<ESC>NHur K<ESC>NHoln (storytold/gridcraft#4321). Text values now decode them: a diacritic and letter become the precomposed letter (or the letter and its combining mark when Unicode has no single character), a spacing code its character; a code ISO 6937 doesn't define is kept as written.
This commit is contained in:
Hamza committed 2026-11-16 18:51:40 +05:00
1 parent e836b6fd89
commit ab87149855
2 files changed
+168 -3

No files matched your search

+143 -3
View File
@@ -219,12 +219,13 @@ fn position(arg: &str, max: u32) -> Option<u32> {
arg.trim().parse::<u32>().ok().filter(|n| (1..=max).contains(n)).map(|n| n - 1)
}
/// A `K` value: `"text"` (everything between the first and the last quote, as written), a
/// boolean, a number, or other text (an error value when it starts with `#`).
/// A `K` value: `"text"` (everything between the first and the last quote, with Excel's
/// character escapes decoded), a boolean, a number, or other text (an error value when it starts
/// with `#`).
fn parse_value(arg: &str) -> Value {
if let Some(text) = arg.strip_prefix('"') {
let text = text.strip_suffix('"').unwrap_or(text);
return Value::Text(text.to_string());
return Value::Text(decode_escapes(text));
}
if arg.eq_ignore_ascii_case("TRUE") || arg.eq_ignore_ascii_case("FALSE") {
return Value::Bool(arg.eq_ignore_ascii_case("TRUE"));
@@ -238,6 +239,145 @@ fn parse_value(arg: &str) -> Value {
}
}
/// ISO 6937's diacritics (codes 0xC1–0xCF): each code, its combining mark, and the letters it
/// composes with, as pairs of letter and precomposed letter (from Unicode's canonical composition).
const ISO6937_DIACRITICS: [(u8, char, &str); 13] = [
(0xC1, '\u{0300}', "AÀaàEÈeèIÌiìNǸnǹOÒoòUÙuùWẀwẁYỲyỳ"), // grave
(0xC2, '\u{0301}', "AÁaáCĆcćEÉeéGǴgǵIÍiíKḰkḱLĹlĺMḾmḿNŃnńOÓoóPṔpṕRŔrŕSŚsśUÚuúWẂwẃYÝyýZŹzź"), // acute
(0xC3, '\u{0302}', "AÂaâCĈcĉEÊeêGĜgĝHĤhĥIÎiîJĴjĵOÔoôSŜsŝUÛuûWŴwŵYŶyŷZẐzẑ"), // circumflex
(0xC4, '\u{0303}', "AÃaãEẼeẽIĨiĩNÑnñOÕoõUŨuũVṼvṽYỸyỹ"), // tilde
(0xC5, '\u{0304}', "AĀaāEĒeēGḠgḡIĪiīOŌoōUŪuūYȲyȳ"), // macron
(0xC6, '\u{0306}', "AĂaăEĔeĕGĞgğIĬiĭOŎoŏUŬuŭ"), // breve
(0xC7, '\u{0307}', "AȦaȧBḂbḃCĊcċDḊdḋEĖeėFḞfḟGĠgġHḢhḣIİMṀmṁNṄnṅOȮoȯPṖpṗRṘrṙSṠsṡTṪtṫWẆwẇXẊxẋYẎyẏZŻzż"), // dot above
(0xC8, '\u{0308}', "AÄaäEËeëHḦhḧIÏiïOÖoötẗUÜuüWẄwẅXẌxẍYŸyÿ"), // diaeresis
(0xCA, '\u{030a}', "AÅaåUŮuůwẘyẙ"), // ring
(0xCB, '\u{0327}', "CÇcçDḐdḑEȨeȩGĢgģHḨhḩKĶkķLĻlļNŅnņRŖrŗSŞsşTŢtţ"), // cedilla
(0xCD, '\u{030b}', "OŐoőUŰuű"), // double acute
(0xCE, '\u{0328}', "AĄaąEĘeęIĮiįOǪoǫUŲuų"), // ogonek
(0xCF, '\u{030c}', "AǍaǎCČcčDĎdďEĚeěGǦgǧHȞhȟIǏiǐjǰKǨkǩLĽlľNŇnňOǑoǒRŘrřSŠsšTŤtťUǓuǔZŽzž"), // caron
];
/// ISO 6937's spacing characters above ASCII (codes 0xA1–0xFE that aren't diacritics).
const ISO6937_SPACING: [(u8, char); 71] = [
(0xA1, '¡'),
(0xA2, '¢'),
(0xA3, '£'),
(0xA5, '¥'),
(0xA7, '§'),
(0xA8, '¤'),
(0xA9, '‘'),
(0xAA, '“'),
(0xAB, '«'),
(0xAC, '←'),
(0xAD, '↑'),
(0xAE, '→'),
(0xAF, '↓'),
(0xB0, '°'),
(0xB1, '±'),
(0xB2, '²'),
(0xB3, '³'),
(0xB4, '×'),
(0xB5, 'µ'),
(0xB6, '¶'),
(0xB7, '·'),
(0xB8, '÷'),
(0xB9, '’'),
(0xBA, '”'),
(0xBB, '»'),
(0xBC, '¼'),
(0xBD, '½'),
(0xBE, '¾'),
(0xBF, '¿'),
(0xD0, '―'),
(0xD1, '¹'),
(0xD2, '®'),
(0xD3, '©'),
(0xD4, '™'),
(0xD5, '♪'),
(0xD6, '¬'),
(0xD7, '¦'),
(0xDC, '⅛'),
(0xDD, '⅜'),
(0xDE, '⅝'),
(0xDF, '⅞'),
(0xE0, 'Ω'),
(0xE1, 'Æ'),
(0xE2, 'Đ'),
(0xE3, 'ª'),
(0xE4, 'Ħ'),
(0xE6, 'IJ'),
(0xE7, 'Ŀ'),
(0xE8, 'Ł'),
(0xE9, 'Ø'),
(0xEA, 'Œ'),
(0xEB, 'º'),
(0xEC, 'Þ'),
(0xED, 'Ŧ'),
(0xEE, 'Ŋ'),
(0xEF, 'ʼn'),
(0xF0, 'ĸ'),
(0xF1, 'æ'),
(0xF2, 'đ'),
(0xF3, 'ð'),
(0xF4, 'ħ'),
(0xF5, 'ı'),
(0xF6, 'ij'),
(0xF7, 'ŀ'),
(0xF8, 'ł'),
(0xF9, 'ø'),
(0xFA, 'œ'),
(0xFB, 'ß'),
(0xFC, 'þ'),
(0xFD, 'ŧ'),
(0xFE, 'ŋ'),
];
/// Text with Excel's SYLK character escapes decoded. Excel writes a character outside ASCII as
/// `ESC N` and one byte, the character's ISO 6937 code less 0x80. A diacritic's code is followed
/// by the letter it goes on (`ESC N H o` is `ö`, `ESC N {` is `ß`). A code ISO 6937 doesn't define,
/// or an `ESC` that isn't followed by `N` and a code, is kept as written.
fn decode_escapes(text: &str) -> String {
if !text.contains('\u{1b}') {
return text.to_string();
}
let mut out = String::with_capacity(text.len());
let mut chars = text.chars();
while let Some(c) = chars.next() {
let mut ahead = chars.clone();
let code = (c == '\u{1b}' && ahead.next() == Some('N'))
.then(|| ahead.next())
.flatten()
.and_then(|b| u8::try_from(b).ok())
.filter(|b| (0x20..0x80).contains(b));
let Some(code) = code.map(|b| b + 0x80) else {
out.push(c);
continue;
};
if let Some(&(_, mark, pairs)) = ISO6937_DIACRITICS.iter().find(|(d, _, _)| *d == code) {
// The letter it goes on: precomposed where Unicode has the pair, else letter and mark.
let Some(base) = ahead.next() else {
out.push(c);
continue;
};
let letters: Vec<char> = pairs.chars().collect();
match letters.chunks(2).find(|p| p.first() == Some(&base)).and_then(|p| p.get(1)) {
Some(&composed) => out.push(composed),
None => {
out.push(base);
out.push(mark);
}
}
} else if let Some(&(_, ch)) = ISO6937_SPACING.iter().find(|(s, _)| *s == code) {
out.push(ch);
} else {
out.push(c);
continue;
}
chars = ahead;
}
out
}
/// One part of an R1C1 reference: `R`/`C` followed by an absolute number, a bracketed offset or
/// nothing (the cell's own row or column).
#[derive(Clone, Copy, Debug, PartialEq)]
+25
View File
@@ -115,3 +115,28 @@ fn survives_huge_relative_offsets_and_width_floods() {
let _ = read(&b);
}
}
#[test]
fn decodes_excels_character_escapes_in_text() {
// storytold/gridcraft#4321: Excel writes `Größe für Köln` with `ESC N` escapes (ISO 6937 codes
// less 0x80: `H` is the diaeresis, put on the next letter; `{` is ß).
let file = "ID;PWXL;N;E\r\nC;Y1;X1;K\"Gr\u{1b}NHo\u{1b}N{e f\u{1b}NHur K\u{1b}NHoln\"\r\nC;Y2;X1;K\"Plain ASCII 123\"\r\n\
C;Y3;X1;K\"\u{1b}NBe \u{1b}NAa \u{1b}NCa \u{1b}NKc \u{1b}NOs \u{1b}NMo \u{1b}NJa \u{1b}NDn\"\r\n\
C;Y4;X1;K\"\u{1b}NAa \u{1b}NAZ\"\r\nC;Y5;X1;K\"\u{1b}Nq \u{1b}NyStra\u{1b}N{e \u{1b}Na\u{1b}Ni\"\r\n\
C;Y6;X1;K\"\u{1b}NHq \u{1b}N\u{1b}NHo end\u{1b}N\"\r\nC;Y7;X1;K\"\u{1b}N\u{7f} \u{1b}NL \u{1b}X\"\r\nE\r\n";
let sh = read(file.as_bytes()).unwrap();
let value = |a: &str| sh.cells.get(&at(a)).map(|c| c.value.clone()).unwrap_or_default();
let text = |s: &str| Value::Text(s.into());
assert_eq!(value("A1"), text("Größe für Köln"));
assert_eq!(value("A2"), text("Plain ASCII 123"), "plain ASCII is unchanged");
// Acute, grave, circumflex, cedilla, caron, double acute, ring, tilde.
assert_eq!(value("A3"), text("é à â ç š ő å ñ"));
// A letter the mark has no precomposed form with keeps the combining mark.
assert_eq!(value("A4"), text("à Z\u{300}"));
// Spacing characters, one code each: `q` æ, `y` ø, `{` ß, `a` Æ, `i` Ø.
assert_eq!(value("A5"), text("æ øStraße ÆØ"));
// A diacritic on a letter with no precomposed form, and a stray `ESC N` before an escape.
assert_eq!(value("A6"), text("q\u{308} \u{1b}Nö end\u{1b}N"));
// Codes ISO 6937 doesn't define (0xFF, 0xCC) and an `ESC` without `N` stay as written.
assert_eq!(value("A7"), text("\u{1b}N\u{7f} \u{1b}NL \u{1b}X"));
}