Add strict UTF-8 text decoder

crates/csv/src/decode.rs

@@ -19,6 +19,16 @@ const CP1256: [u32; 128] = [
     0x200F, 0x06D2,
 ];
 
+/// Decode strictly as UTF-8, stripping a leading UTF-8 BOM when present.
+///
+/// Unlike [`decode_text`], this does not guess a legacy encoding or accept UTF-16. It is useful
+/// when a format or application contract requires UTF-8 and malformed bytes must be rejected.
+pub fn decode_utf8(bytes: &[u8]) -> Result<String, std::str::Utf8Error> {
+    let bytes = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]).unwrap_or(bytes);
+    let text = std::str::from_utf8(bytes)?;
+    Ok(text.strip_prefix('\u{FEFF}').unwrap_or(text).to_string())
+}
+
 /// Evidence that bytes which aren't UTF-8 are Windows-1256 Arabic rather than Windows-1252.
 #[derive(Default)]
 struct Cp1256Evidence {
@@ -165,6 +175,16 @@ mod tests {
         assert_eq!(decode_text(&[0xFF, 0xFE, 0x00, 0xD8]), "\u{FFFD}", "a lone surrogate");
     }
 
+    #[test]
+    fn strict_utf8_strips_bom_and_rejects_legacy_or_malformed_bytes() {
+        assert_eq!(decode_utf8(b"\xEF\xBB\xBFName,City\nAda,Town\n").unwrap(), "Name,City\nAda,Town\n");
+        assert_eq!(decode_utf8("café".as_bytes()).unwrap(), "café");
+        assert!(decode_utf8(&[0xFF, 0xFE, 0x00]).is_err());
+        assert!(decode_utf8(b"\xEF\xBB\xBF\xFF").is_err());
+        // The permissive text decoder still accepts legacy encodings for callers that want it.
+        assert_eq!(decode_text(&[0x80, b'a', 0xE9]), "€aé");
+    }
+
     #[test]
     fn windows_1256_table_matches_the_unicode_mapping() {
         let at = |b: u8| CP1256.get(usize::from(b - 0x80)).and_then(|&c| char::from_u32(c));

crates/csv/src/lib.rs

@@ -3,6 +3,7 @@
 //!
 //! - [`decode_text`] turns a file's bytes into a string: byte order marks, UTF-8, else
 //!   Windows-1256 when the bytes read as Arabic and Windows-1252 otherwise.
+//! - [`decode_utf8`] requires valid UTF-8 and strips a leading UTF-8 byte order mark.
 //! - [`sniff_delimiter`] picks comma, semicolon or tab.
 //! - [`parse_records`] splits text into records of fields (RFC 4180 with the usual leniency),
 //!   handing each record to a callback, with a cap on the fields kept per record. [`records`]
@@ -27,7 +28,7 @@
 mod decode;
 pub mod sylk;
 
-pub use decode::decode_text;
+pub use decode::{decode_text, decode_utf8};
 
 /// Picks comma, semicolon or tab by consistent field counts in the first non-blank records.
 /// Quotes start only at a candidate field boundary; candidates with incoherent quoted fields