fix(csv): preserve quoted delimiter coherence

crates/csv/README.md

@@ -11,7 +11,7 @@ mapping a SYLK sheet onto a document are the caller's work.
 | Function | What it does |
 |---|---|
 | `decode_text(bytes)` | UTF-8 and UTF-16 byte order marks; without one, valid UTF-8 is kept; other bytes are Windows-1256 when they read as Arabic, else Windows-1252 |
-| `sniff_delimiter(text, quote)` | comma, semicolon or tab: the one with the most consistent count per line over the first 20 non-blank lines, outside quotes |
+| `sniff_delimiter(text, quote)` | comma, semicolon or tab: the one with the most consistent field count over the first 20 non-blank records in a 64 KiB sample; quote boundaries must be coherent, and ties prefer tab, comma, then semicolon |
 | `parse_records(text, delim, quote, max_fields, on_record)` | records of fields, one callback per record (return `false` to stop); CR, LF and CRLF line ends; quoted fields with doubled quotes and line breaks; a quote in the middle of a field is text |
 | `records(text, delim, quote, max_records, max_fields)` | the same, collected |
 | `push_field(out, field, delim)`, `field_len(field, delim)` | one field as written, quoted only when it holds the delimiter, a quote or a line break; its exact length, to enforce a size cap before writing |

crates/csv/src/lib.rs

@@ -29,44 +29,78 @@ pub mod sylk;
 
 pub use decode::decode_text;
 
-/// Picks comma, semicolon or tab: the candidate with the most consistent count per line
-/// (outside quotes) over the first non-blank lines. A candidate must appear on every such line, otherwise comma wins.
+/// Picks comma, semicolon or tab by consistent field counts in the first non-blank records.
+/// Quotes start only at a candidate field boundary; candidates with incoherent quoted fields
+/// are discarded when another coherent candidate separates the records. Ties prefer tab,
+/// then comma, then semicolon. Comma is the fallback when no candidate separates the text.
 pub fn sniff_delimiter(text: &str, quote: char) -> u8 {
-    let mut best = (b',', 0usize, 0usize);
-    for d in [',', ';', '\t'] {
-        let mut counts = vec![];
-        let mut n = 0usize;
-        let mut in_q = false;
-        let mut blank = true;
-        for ch in text.chars().take(64 * 1024) {
-            if !in_q && ch == '\n' {
-                // Blank lines carry no delimiter evidence.
-                if !blank {
-                    counts.push(n);
-                }
-                n = 0;
-                blank = true;
-                if counts.len() >= 20 {
-                    break;
+    let mut candidates = [(b'\t', 0usize, 0usize, true), (b',', 0, 0, true), (b';', 0, 0, true)];
+    for (delimiter, min_count, total, coherent) in &mut candidates {
+        let mut counts = Vec::new();
+        let mut count = 0usize;
+        let mut quoted = false;
+        let mut after_quote = false;
+        let mut field_start = true;
+        let mut nonempty = false;
+        let mut chars = text.chars().take(64 * 1024).peekable();
+        while let Some(ch) = chars.next() {
+            if quoted {
+                if ch == quote {
+                    if chars.peek() == Some(&quote) {
+                        chars.next();
+                        nonempty = true;
+                    } else {
+                        quoted = false;
+                        after_quote = true;
+                    }
+                } else if !ch.is_whitespace() {
+                    nonempty = true;
                 }
                 continue;
             }
-            if ch != '\r' {
-                blank = false;
+            if ch == '\r' || ch == '\n' {
+                if ch == '\r' && chars.peek() == Some(&'\n') {
+                    chars.next();
+                }
+                if nonempty {
+                    counts.push(count);
+                    if counts.len() >= 20 {
+                        break;
+                    }
+                }
+                count = 0;
+                quoted = false;
+                after_quote = false;
+                field_start = true;
+                nonempty = false;
+                continue;
             }
-            if ch == quote {
-                in_q = !in_q;
-            } else if !in_q && ch == d {
-                n += 1;
+            if ch == quote && field_start {
+                quoted = true;
+                field_start = false;
+            } else if ch == char::from(*delimiter) {
+                count += 1;
+                field_start = true;
+                after_quote = false;
+                nonempty = true;
+            } else {
+                *coherent &= !after_quote || ch.is_whitespace();
+                nonempty |= !ch.is_whitespace();
+                field_start = false;
             }
         }
-        if !blank {
-            counts.push(n);
+        if nonempty {
+            counts.push(count);
         }
-        let min = counts.iter().copied().min().unwrap_or(0);
-        let total: usize = counts.iter().sum();
-        if min > 0 && (min, total) > (best.1, best.2) {
-            best = (d as u8, min, total);
+        *min_count = counts.iter().copied().min().unwrap_or(0);
+        *total = counts.iter().sum();
+        *coherent &= !quoted;
+    }
+    let prefer_coherent = candidates.iter().any(|(_, min, _, coherent)| *min > 0 && *coherent);
+    let mut best = (b',', 0usize, 0usize);
+    for (delimiter, min, total, coherent) in candidates {
+        if min > 0 && (!prefer_coherent || coherent) && (min, total) > (best.1, best.2) {
+            best = (delimiter, min, total);
         }
     }
     best.0
@@ -211,6 +245,21 @@ mod tests {
         assert_eq!(sniff_delimiter("x;y\n1,5;2", '"'), b';');
     }
 
+    #[test]
+    fn delimiter_sniffing_preserves_candidate_quote_boundaries() {
+        for ending in ["\n", "\r\n", "\r"] {
+            for (text, expected) in [
+                (format!("Name,\"preferred\tCity\"{ending}Ada\tLondon{ending}"), b'\t'),
+                (format!("\"Last, First\";City{ending}Ada;London{ending}"), b';'),
+                (format!("\"Name\";\"Last, First\"{ending}\"Ada\";\"Lovelace, Countess\"{ending}\"Grace\";\"Hopper, Admiral\"{ending}"), b';'),
+                (format!("\"Name\"suffix,Title\tCity{ending}\"Ada\"suffix,Analyst\tLondon\tignored{ending}"), b'\t'),
+                (format!("\r\n \r\n\"Name\";City{ending}Ada;London{ending}"), b';'),
+            ] {
+                assert_eq!(sniff_delimiter(&text, '"'), expected, "{text:?}");
+            }
+        }
+    }
+
     #[test]
     fn parser_stops_collecting_fields_at_the_cap() {
         let (max, extra) = (16_384usize, 50);