diff --git a/crates/csv/src/decode.rs b/crates/csv/src/decode.rs index 238f7f18e..63b4bb590 100644 --- a/crates/csv/src/decode.rs +++ b/crates/csv/src/decode.rs @@ -19,6 +19,16 @@ const CP1256: [u32; 128] = [ 0x200F, 0x06D2, ]; +/// Decode strictly as UTF-8, stripping a leading UTF-8 BOM when present. +/// +/// Unlike [`decode_text`], this does not guess a legacy encoding or accept UTF-16. It is useful +/// when a format or application contract requires UTF-8 and malformed bytes must be rejected. +pub fn decode_utf8(bytes: &[u8]) -> Result { + let bytes = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]).unwrap_or(bytes); + let text = std::str::from_utf8(bytes)?; + Ok(text.strip_prefix('\u{FEFF}').unwrap_or(text).to_string()) +} + /// Evidence that bytes which aren't UTF-8 are Windows-1256 Arabic rather than Windows-1252. #[derive(Default)] struct Cp1256Evidence { @@ -165,6 +175,16 @@ mod tests { assert_eq!(decode_text(&[0xFF, 0xFE, 0x00, 0xD8]), "\u{FFFD}", "a lone surrogate"); } + #[test] + fn strict_utf8_strips_bom_and_rejects_legacy_or_malformed_bytes() { + assert_eq!(decode_utf8(b"\xEF\xBB\xBFName,City\nAda,Town\n").unwrap(), "Name,City\nAda,Town\n"); + assert_eq!(decode_utf8("café".as_bytes()).unwrap(), "café"); + assert!(decode_utf8(&[0xFF, 0xFE, 0x00]).is_err()); + assert!(decode_utf8(b"\xEF\xBB\xBF\xFF").is_err()); + // The permissive text decoder still accepts legacy encodings for callers that want it. + assert_eq!(decode_text(&[0x80, b'a', 0xE9]), "€aé"); + } + #[test] fn windows_1256_table_matches_the_unicode_mapping() { let at = |b: u8| CP1256.get(usize::from(b - 0x80)).and_then(|&c| char::from_u32(c)); diff --git a/crates/csv/src/lib.rs b/crates/csv/src/lib.rs index 5cbdc28c5..0aec971ac 100644 --- a/crates/csv/src/lib.rs +++ b/crates/csv/src/lib.rs @@ -3,6 +3,7 @@ //! //! - [`decode_text`] turns a file's bytes into a string: byte order marks, UTF-8, else //! Windows-1256 when the bytes read as Arabic and Windows-1252 otherwise. +//! - [`decode_utf8`] requires valid UTF-8 and strips a leading UTF-8 byte order mark. //! - [`sniff_delimiter`] picks comma, semicolon or tab. //! - [`parse_records`] splits text into records of fields (RFC 4180 with the usual leniency), //! handing each record to a callback, with a cap on the fields kept per record. [`records`] @@ -27,7 +28,7 @@ mod decode; pub mod sylk; -pub use decode::decode_text; +pub use decode::{decode_text, decode_utf8}; /// Picks comma, semicolon or tab by consistent field counts in the first non-blank records. /// Quotes start only at a candidate field boundary; candidates with incoherent quoted fields