From 976e4ef8ea3315f066bdfe65bdfa6e313caa196c Mon Sep 17 00:00:00 2001 From: dotCooCoo Date: Mon, 16 Nov 2026 19:01:57 -0800 Subject: [PATCH] craft-csv: rows of empty cells decide the delimiter when nothing else does sniff_delimiter skipped every record without a non-whitespace character other than the delimiter, so a record such as ";" or ";;;" was blank to the semicolon candidate. A semicolon export of a sheet whose only value is far down (B40001) starts with 40,000 ";" rows, which fill the 64 KiB sample, and the file was read as comma-separated with one column. Records that hold only a candidate's delimiters and whitespace are now kept apart, at most 20 per candidate. When no candidate separates the records with other text, the same rules pick among the candidates whose sample holds only such records. Every result that did not fall back to comma stays the same: a row of tabs alone is still blank to the other candidates and does not outweigh them, as #1209 set out. --- crates/csv/README.md | 2 +- crates/csv/src/lib.rs | 122 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 110 insertions(+), 14 deletions(-) diff --git a/crates/csv/README.md b/crates/csv/README.md index bfd525c8c..8fd264d4c 100644 --- a/crates/csv/README.md +++ b/crates/csv/README.md @@ -11,7 +11,7 @@ mapping a SYLK sheet onto a document are the caller's work. | Function | What it does | |---|---| | `decode_text(bytes)` | UTF-8 and UTF-16 byte order marks; without one, valid UTF-8 is kept; other bytes are Windows-1256 when they read as Arabic, else Windows-1252 | -| `sniff_delimiter(text, quote)` | comma, semicolon or tab: the one with the most consistent field count over the first 20 non-blank records in a 64 KiB sample; when no candidate finds coherent complete records with a delimiter in each, a candidate that finds no complete record counts the delimiters of the first record up to the end of the sample, or up to a quoted field still open there, and a quote inside a field that does not start with one makes that record incoherent; quote boundaries must be coherent, and ties prefer tab, comma, then semicolon | +| `sniff_delimiter(text, quote)` | comma, semicolon or tab: the one with the most consistent field count over the first 20 records with other text in a 64 KiB sample; when none separates those records, the candidates whose records hold only their delimiters and whitespace (rows of empty cells) are compared the same way, leaving out any candidate that found a record with other text; when no candidate finds coherent complete records with a delimiter in each, a candidate that finds no complete record with other text counts the delimiters of the first such record up to the end of the sample, or up to a quoted field still open there (after its rows of empty cells, a record without its delimiter is left out), and a quote inside a field that does not start with one makes that record incoherent; quote boundaries must be coherent, and ties prefer tab, comma, then semicolon | | `parse_records(text, delim, quote, max_fields, on_record)` | records of fields, one callback per record (return `false` to stop); CR, LF and CRLF line ends; quoted fields with doubled quotes and line breaks; a quote in the middle of a field is text | | `records(text, delim, quote, max_records, max_fields)` | the same, collected | | `push_field(out, field, delim)`, `field_len(field, delim)` | one field as written, quoted only when it holds the delimiter, a quote or a line break; its exact length, to enforce a size cap before writing | diff --git a/crates/csv/src/lib.rs b/crates/csv/src/lib.rs index 70f6148a6..395ad2152 100644 --- a/crates/csv/src/lib.rs +++ b/crates/csv/src/lib.rs @@ -33,13 +33,17 @@ pub use decode::{decode_text, decode_utf8}; /// Picks comma, semicolon or tab by consistent field counts in the first non-blank records. /// Quotes start only at a candidate field boundary; candidates with incoherent quoted fields /// are discarded when another coherent candidate separates the records. Ties prefer tab, -/// then comma, then semicolon. Comma is the fallback when no candidate separates the text. +/// then comma, then semicolon. When no candidate separates the records that hold other text, +/// the same rules pick among the candidates whose records hold only their delimiters and +/// whitespace, such as an exported row of empty cells; a candidate that finds a record with +/// other text is left out of that pick. Comma is the fallback when neither pick finds one. /// Only the first 64 KiB are read. A record cut off at the end of that sample is left out, /// except when no candidate finds coherent complete records with a delimiter in each. Then a candidate -/// that finds no complete record counts the delimiters read so far in the first record, and that -/// record is incoherent when it has a quote inside a field that does not start with one. When one -/// of those candidates ends the sample inside a quoted field, each of them counts only up to that -/// field's opening quote. +/// that finds no complete record with other text counts the delimiters read so far in the first +/// such record, unless it found rows of its empty cells and that record has none of its +/// delimiters. That record is incoherent when it has a quote inside a field that does not start +/// with one. When one of those candidates ends the sample inside a quoted field, each of them +/// counts only up to that field's opening quote. pub fn sniff_delimiter(text: &str, quote: char) -> u8 { const SAMPLE_BYTES: usize = 64 * 1024; let mut sample_end = text.len().min(SAMPLE_BYTES); @@ -64,7 +68,15 @@ pub fn sniff_delimiter(text: &str, quote: char) -> u8 { (scan.count, scan.record_has_content, scan.record_coherent, scan.record_bare_quote) = (before_cut.count, before_cut.record_has_content, before_cut.record_coherent, before_cut.record_bare_quote); } - if scan.record_has_content { + if scan.delimiter_only_counts.is_empty() { + if scan.record_has_content { + scan.coherent &= scan.record_coherent && !scan.record_bare_quote; + scan.counts.push(scan.count); + } else { + scan.push_delimiter_only_record(); + } + } else if scan.record_has_content && scan.count > 0 { + // After rows of empty cells, a cut record without this delimiter is left out. scan.coherent &= scan.record_coherent && !scan.record_bare_quote; scan.counts.push(scan.count); } @@ -79,6 +91,8 @@ pub fn sniff_delimiter(text: &str, quote: char) -> u8 { } else if scan.record_has_content { scan.coherent &= scan.record_coherent; scan.counts.push(scan.count); + } else { + scan.push_delimiter_only_record(); } } } @@ -86,23 +100,34 @@ pub fn sniff_delimiter(text: &str, quote: char) -> u8 { } fn pick_delimiter(scans: [DelimiterScan; 3]) -> u8 { - let candidates = - scans.map(|scan| (scan.delimiter, scan.counts.iter().copied().min().unwrap_or(0), scan.counts.iter().sum::(), scan.coherent)); + let with_text = scans.each_ref().map(|scan| (scan.delimiter, scan.counts.as_slice(), scan.coherent)); + // Records of delimiters alone have no quotes, so they are coherent. + let delimiters_only = + scans.each_ref().map(|scan| (scan.delimiter, if scan.counts.is_empty() { scan.delimiter_only_counts.as_slice() } else { &[] }, true)); + best_delimiter(with_text).or_else(|| best_delimiter(delimiters_only)).unwrap_or(b',') +} + +fn best_delimiter(candidates: [(u8, &[usize], bool); 3]) -> Option { + let candidates = candidates + .map(|(delimiter, counts, coherent)| (delimiter, counts.iter().copied().min().unwrap_or(0), counts.iter().sum::(), coherent)); let prefer_coherent = candidates.iter().any(|(_, min, _, coherent)| *min > 0 && *coherent); - let mut best = (b',', 0usize, 0usize); + let mut best: Option<(u8, usize, usize)> = None; for (delimiter, min, total, coherent) in candidates { - if min > 0 && (!prefer_coherent || coherent) && (min, total) > (best.1, best.2) { - best = (delimiter, min, total); + if min > 0 && (!prefer_coherent || coherent) && best.is_none_or(|(_, best_min, best_total)| (min, total) > (best_min, best_total)) { + best = Some((delimiter, min, total)); } } - best.0 + best.map(|(delimiter, _, _)| delimiter) } /// One candidate delimiter's pass over the sample in [`sniff_delimiter`]. struct DelimiterScan { delimiter: u8, - /// Delimiter counts of the complete non-blank records, at most 20. + /// Delimiter counts of the complete records that hold other text, at most 20. counts: Vec, + /// Delimiter counts of the complete records that hold only this candidate's delimiters and + /// whitespace, at most 20. They do not count toward the 20 records above. + delimiter_only_counts: Vec, /// Whether every counted record had coherent quoted fields. coherent: bool, reached_record_limit: bool, @@ -117,10 +142,21 @@ struct DelimiterScan { open_quote: Option, } +impl DelimiterScan { + /// Keeps the open record's delimiter count when the record has no other text and at least + /// one delimiter. + fn push_delimiter_only_record(&mut self) { + if !self.record_has_content && self.count > 0 && self.delimiter_only_counts.len() < 20 { + self.delimiter_only_counts.push(self.count); + } + } +} + fn scan_delimiter(sample: &str, delimiter: u8, quote: char) -> DelimiterScan { let mut scan = DelimiterScan { delimiter, counts: Vec::new(), + delimiter_only_counts: Vec::new(), coherent: true, reached_record_limit: false, count: 0, @@ -158,6 +194,8 @@ fn scan_delimiter(sample: &str, delimiter: u8, quote: char) -> DelimiterScan { scan.reached_record_limit = true; break; } + } else { + scan.push_delimiter_only_record(); } scan.count = 0; after_quote = false; @@ -403,6 +441,64 @@ mod tests { assert_eq!(sniff_delimiter(&format!("a,b,c;\"hello,\";d\n{}", "x;y;z\n".repeat(12_000)), '"'), b';'); } + // A sheet whose only value is far down exports rows of empty cells that fill the whole sample. + #[test] + fn delimiter_sniffing_reads_a_sparse_semicolon_export() { + assert_eq!(sniff_delimiter(&format!("{};7\n", ";\n".repeat(40_000)), '"'), b';'); + } + + #[test] + fn delimiter_sniffing_reads_a_sparse_semicolon_export_with_crlf() { + assert_eq!(sniff_delimiter(&format!("{};7\r\n", ";\r\n".repeat(40_000)), '"'), b';'); + } + + #[test] + fn delimiter_sniffing_reads_a_sparse_tab_export() { + assert_eq!(sniff_delimiter(&format!("{}\t7\r\n", "\t\r\n".repeat(40_000)), '"'), b'\t'); + } + + #[test] + fn delimiter_sniffing_reads_a_short_file_of_empty_semicolon_cells() { + assert_eq!(sniff_delimiter(";;\n;;\n", '"'), b';'); + assert_eq!(sniff_delimiter(";;\r\n;;", '"'), b';'); + } + + #[test] + fn delimiter_sniffing_reads_empty_semicolon_cells_padded_with_whitespace() { + assert_eq!(sniff_delimiter(&"; ; \n".repeat(20_000), '"'), b';'); + } + + #[test] + fn delimiter_sniffing_reads_a_first_record_of_empty_cells_longer_than_the_sample() { + assert_eq!(sniff_delimiter(&format!("{}\n1;2\n", ";".repeat(70_000)), '"'), b';'); + } + + #[test] + fn delimiter_sniffing_leaves_out_a_cut_record_after_a_row_of_empty_cells() { + assert_eq!(sniff_delimiter(&format!(";\n{};7\n", "x".repeat(70_000)), '"'), b';'); + } + + #[test] + fn delimiter_sniffing_counts_a_long_record_after_a_row_of_empty_cells() { + // Delimiters inside the long record's quoted field are text for every candidate. + assert_eq!(sniff_delimiter(&format!("\t\nname\t\"{}\"\n", "x,y".repeat(30_000)), '"'), b'\t'); + assert_eq!(sniff_delimiter(&format!(";\nname;\"{}\"\n", "x,y\t".repeat(20_000)), '"'), b';'); + } + + #[test] + fn delimiter_sniffing_lets_records_with_text_outweigh_records_of_delimiters_alone() { + // Rows of tabs alone are blank to the other candidates and do not outweigh them. + assert_eq!(sniff_delimiter(&format!("{}a;b\nc;d\n", "\t\n".repeat(25)), '"'), b';'); + assert_eq!(sniff_delimiter(&format!("{}a;b\nc;d\n", "\t\t\n".repeat(25)), '"'), b';'); + // The same holds for a record with text cut off at the end of the sample. + assert_eq!(sniff_delimiter(&format!("\t\n{}", "x;".repeat(40_000)), '"'), b';'); + // A candidate with a record of other text is judged on those records alone. + assert_eq!(sniff_delimiter(&format!("title\n{}", ";\n".repeat(25)), '"'), b','); + // Rows of empty cells in a comma file; comma is also the fallback. + assert_eq!(sniff_delimiter("a,b,c,d\n,,,\n,,,\n1,2,3,4\n", '"'), b','); + assert_eq!(sniff_delimiter(&format!("{}x;y\n", ",,,\n".repeat(20_000)), '"'), b','); + } + #[test] fn parser_stops_collecting_fields_at_the_cap() { let (max, extra) = (16_384usize, 50);