diff --git a/crates/csv/README.md b/crates/csv/README.md index 6369ce8b4..bfd525c8c 100644 --- a/crates/csv/README.md +++ b/crates/csv/README.md @@ -11,7 +11,7 @@ mapping a SYLK sheet onto a document are the caller's work. | Function | What it does | |---|---| | `decode_text(bytes)` | UTF-8 and UTF-16 byte order marks; without one, valid UTF-8 is kept; other bytes are Windows-1256 when they read as Arabic, else Windows-1252 | -| `sniff_delimiter(text, quote)` | comma, semicolon or tab: the one with the most consistent field count over the first 20 non-blank records in a 64 KiB sample; quote boundaries must be coherent, and ties prefer tab, comma, then semicolon | +| `sniff_delimiter(text, quote)` | comma, semicolon or tab: the one with the most consistent field count over the first 20 non-blank records in a 64 KiB sample; when no candidate finds coherent complete records with a delimiter in each, a candidate that finds no complete record counts the delimiters of the first record up to the end of the sample, or up to a quoted field still open there, and a quote inside a field that does not start with one makes that record incoherent; quote boundaries must be coherent, and ties prefer tab, comma, then semicolon | | `parse_records(text, delim, quote, max_fields, on_record)` | records of fields, one callback per record (return `false` to stop); CR, LF and CRLF line ends; quoted fields with doubled quotes and line breaks; a quote in the middle of a field is text | | `records(text, delim, quote, max_records, max_fields)` | the same, collected | | `push_field(out, field, delim)`, `field_len(field, delim)` | one field as written, quoted only when it holds the delimiter, a quote or a line break; its exact length, to enforce a size cap before writing | diff --git a/crates/csv/src/lib.rs b/crates/csv/src/lib.rs index 0aec971ac..70f6148a6 100644 --- a/crates/csv/src/lib.rs +++ b/crates/csv/src/lib.rs @@ -34,8 +34,13 @@ pub use decode::{decode_text, decode_utf8}; /// Quotes start only at a candidate field boundary; candidates with incoherent quoted fields /// are discarded when another coherent candidate separates the records. Ties prefer tab, /// then comma, then semicolon. Comma is the fallback when no candidate separates the text. +/// Only the first 64 KiB are read. A record cut off at the end of that sample is left out, +/// except when no candidate finds coherent complete records with a delimiter in each. Then a candidate +/// that finds no complete record counts the delimiters read so far in the first record, and that +/// record is incoherent when it has a quote inside a field that does not start with one. When one +/// of those candidates ends the sample inside a quoted field, each of them counts only up to that +/// field's opening quote. pub fn sniff_delimiter(text: &str, quote: char) -> u8 { - let mut candidates = [(b'\t', 0usize, 0usize, true), (b',', 0, 0, true), (b';', 0, 0, true)]; const SAMPLE_BYTES: usize = 64 * 1024; let mut sample_end = text.len().min(SAMPLE_BYTES); while !text.is_char_boundary(sample_end) { @@ -43,78 +48,46 @@ pub fn sniff_delimiter(text: &str, quote: char) -> u8 { } let sample = &text[..sample_end]; let truncated = sample_end < text.len(); - for (delimiter, min_count, total, coherent) in &mut candidates { - let mut counts = Vec::new(); - let mut count = 0usize; - let mut quoted = false; - let mut after_quote = false; - let mut field_start = true; - let mut record_has_content = false; - let mut record_coherent = true; - let mut candidate_coherent = true; - let mut reached_record_limit = false; - let mut chars = sample.chars().peekable(); - while let Some(ch) = chars.next() { - if quoted { - if ch == quote { - if chars.peek() == Some("e) { - chars.next(); - record_has_content = true; - } else { - quoted = false; - after_quote = true; - } - } else if !ch.is_whitespace() { - record_has_content = true; - } + let mut scans = b"\t,;".map(|delimiter| scan_delimiter(sample, delimiter, quote)); + if truncated { + if scans.iter().any(|scan| scan.coherent && scan.counts.iter().min().is_some_and(|&min| min > 0)) { + return pick_delimiter(scans); + } + // Every quote in the scans is at a char boundary, so the cut is one too. + let cut = scans.iter().filter(|scan| scan.counts.is_empty()).filter_map(|scan| scan.open_quote).min().unwrap_or(sample_end); + for scan in &mut scans { + if !scan.counts.is_empty() { continue; } - if ch == '\r' || ch == '\n' { - if ch == '\r' && chars.peek() == Some(&'\n') { - chars.next(); - } - if record_has_content { - counts.push(count); - candidate_coherent &= record_coherent; - if counts.len() >= 20 { - reached_record_limit = true; - break; - } - } - count = 0; - quoted = false; - after_quote = false; - field_start = true; - record_has_content = false; - record_coherent = true; + if cut < sample_end { + let before_cut = scan_delimiter(&sample[..cut], scan.delimiter, quote); + (scan.count, scan.record_has_content, scan.record_coherent, scan.record_bare_quote) = + (before_cut.count, before_cut.record_has_content, before_cut.record_coherent, before_cut.record_bare_quote); + } + if scan.record_has_content { + scan.coherent &= scan.record_coherent && !scan.record_bare_quote; + scan.counts.push(scan.count); + } + } + } else { + for scan in &mut scans { + if scan.reached_record_limit { continue; } - if ch == quote && field_start { - quoted = true; - field_start = false; - record_has_content = true; - } else if ch == char::from(*delimiter) { - count += 1; - field_start = true; - after_quote = false; - } else { - record_coherent &= !after_quote || ch.is_whitespace(); - record_has_content |= !ch.is_whitespace(); - field_start = false; + if scan.open_quote.is_some() { + scan.coherent = false; + } else if scan.record_has_content { + scan.coherent &= scan.record_coherent; + scan.counts.push(scan.count); } } - if !reached_record_limit && !truncated { - if quoted { - candidate_coherent = false; - } else if record_has_content { - candidate_coherent &= record_coherent; - counts.push(count); - } - } - *coherent &= candidate_coherent; - *min_count = counts.iter().copied().min().unwrap_or(0); - *total = counts.iter().sum(); } + pick_delimiter(scans) +} + +fn pick_delimiter(scans: [DelimiterScan; 3]) -> u8 { + let candidates = + scans.map(|scan| (scan.delimiter, scan.counts.iter().copied().min().unwrap_or(0), scan.counts.iter().sum::(), scan.coherent)); let prefer_coherent = candidates.iter().any(|(_, min, _, coherent)| *min > 0 && *coherent); let mut best = (b',', 0usize, 0usize); for (delimiter, min, total, coherent) in candidates { @@ -125,6 +98,93 @@ pub fn sniff_delimiter(text: &str, quote: char) -> u8 { best.0 } +/// One candidate delimiter's pass over the sample in [`sniff_delimiter`]. +struct DelimiterScan { + delimiter: u8, + /// Delimiter counts of the complete non-blank records, at most 20. + counts: Vec, + /// Whether every counted record had coherent quoted fields. + coherent: bool, + reached_record_limit: bool, + /// The record still open at the end of the sample: its delimiter count so far, whether it + /// has content, and whether its quoted fields so far are coherent. + count: usize, + record_has_content: bool, + record_coherent: bool, + /// Whether the open record has a quote inside a field that does not start with one. + record_bare_quote: bool, + /// The byte offset of the quote that opens a field still unterminated at the end. + open_quote: Option, +} + +fn scan_delimiter(sample: &str, delimiter: u8, quote: char) -> DelimiterScan { + let mut scan = DelimiterScan { + delimiter, + counts: Vec::new(), + coherent: true, + reached_record_limit: false, + count: 0, + record_has_content: false, + record_coherent: true, + record_bare_quote: false, + open_quote: None, + }; + let mut after_quote = false; + let mut field_start = true; + let mut chars = sample.char_indices().peekable(); + while let Some((at, ch)) = chars.next() { + if scan.open_quote.is_some() { + if ch == quote { + if chars.peek().map(|&(_, next)| next) == Some(quote) { + chars.next(); + scan.record_has_content = true; + } else { + scan.open_quote = None; + after_quote = true; + } + } else if !ch.is_whitespace() { + scan.record_has_content = true; + } + continue; + } + if ch == '\r' || ch == '\n' { + if ch == '\r' && chars.peek().map(|&(_, next)| next) == Some('\n') { + chars.next(); + } + if scan.record_has_content { + scan.counts.push(scan.count); + scan.coherent &= scan.record_coherent; + if scan.counts.len() >= 20 { + scan.reached_record_limit = true; + break; + } + } + scan.count = 0; + after_quote = false; + field_start = true; + scan.record_has_content = false; + scan.record_coherent = true; + scan.record_bare_quote = false; + continue; + } + if ch == quote && field_start { + scan.open_quote = Some(at); + field_start = false; + scan.record_has_content = true; + } else if ch == char::from(delimiter) { + scan.count += 1; + field_start = true; + after_quote = false; + } else { + scan.record_coherent &= !after_quote || ch.is_whitespace(); + scan.record_bare_quote |= ch == quote; + scan.record_has_content |= !ch.is_whitespace(); + field_start = false; + } + } + scan +} + /// Splits delimited text into records of fields and hands each to `on_record`, which returns /// whether to go on. A record ends at CR, LF or CRLF; a blank line is a record with one empty /// field. A field that starts with `quote` runs to the closing quote (a doubled quote is one @@ -309,6 +369,40 @@ mod tests { assert_eq!(sniff_delimiter(&text, '"'), b'\t'); } + #[test] + fn delimiter_sniffing_counts_a_first_record_longer_than_the_sample() { + let wide = |field: &str, delimiter: char| { + let mut record = String::new(); + while record.len() <= 64 * 1024 { + record.push_str(field); + record.push(delimiter); + } + record + }; + assert_eq!(sniff_delimiter(&format!("{}end\n1;2\n", wide("cell", ';')), '"'), b';'); + assert_eq!(sniff_delimiter(&format!("{}end\n1\t2\n", wide("cell", '\t')), '"'), b'\t'); + // Delimiters inside quoted fields of the long record are text. + assert_eq!(sniff_delimiter(&format!("{}end\n", wide("\"a,b\tc\"", ';')), '"'), b';'); + assert_eq!(sniff_delimiter(&format!("{}end\n", wide("\"a;b\tc\"", ',')), '"'), b','); + // A candidate that reads a quoted field as text inside an unquoted one is incoherent. + assert_eq!(sniff_delimiter(&format!("a,\"{}\",{}", "x;y".repeat(20_000), "z".repeat(10_000)), '"'), b','); + // The sample ends inside a quoted field: the delimiters before that field count. + let text = format!("a;b;c;\"{}\"\n1;2;3;4\n", "long text ".repeat(8_000)); + assert_eq!(sniff_delimiter(&text, '"'), b';'); + // Candidates without a complete record do not count delimiters after that quote, also + // when they read it as text. + assert_eq!(sniff_delimiter(&format!("a,\"{}\"\n1,2\n", "x;y".repeat(30_000)), '"'), b','); + assert_eq!(sniff_delimiter(&format!("a;b;\"{}\"\n1;2;3\n", "x,y\t".repeat(20_000)), '"'), b';'); + // A line break inside the open quoted field gives the other candidates a record without + // a delimiter, which does not stop the first record from counting. + assert_eq!(sniff_delimiter(&format!("a;b;\"hello\n{}\"\n", "x".repeat(70_000)), '"'), b';'); + assert_eq!(sniff_delimiter(&format!("\"a\";b;\"hello,\n{}\"\n", "x".repeat(70_000)), '"'), b';'); + // When a candidate finds coherent complete records with a delimiter in each, a record cut + // off at the end is left out. + assert_eq!(sniff_delimiter(&format!("a;\"hello,\";b\n{}", "x;y;z\n".repeat(12_000)), '"'), b';'); + assert_eq!(sniff_delimiter(&format!("a,b,c;\"hello,\";d\n{}", "x;y;z\n".repeat(12_000)), '"'), b';'); + } + #[test] fn parser_stops_collecting_fields_at_the_cap() { let (max, extra) = (16_384usize, 50);