From e5299a3be0535f7e0850cde71056d1cf81a16d49 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 18:54:48 -0500 Subject: [PATCH] craft-html: a tolerant tokenizer, character references, escaping and the declared charset (#944) The bottom layer every HTML reader in the Craft apps carried its own copy of. The tokenizer is MailCraft's (crates/html/src/tokenizer.rs), with what it dropped and capped turned into Options and Limits, a self-closing flag, and raw_content for callers that read a style sheet's text. The character references are WordCraft's (crates/formats/src/html/entity.rs at 06c2ae6f): its name table and the 94 legacy names that decode without a semicolon, with MailCraft's Latin-1 and Greek tables and its handling of numbers too long to be numbers. No tree, no CSS, no decoding of bytes. --- Cargo.lock | 7 + Cargo.toml | 1 + README.md | 1 + crates/html/Cargo.toml | 14 ++ crates/html/README.md | 68 ++++++ crates/html/src/entity.rs | 388 ++++++++++++++++++++++++++++++++ crates/html/src/entity/tests.rs | 132 +++++++++++ crates/html/src/lex.rs | 347 ++++++++++++++++++++++++++++ crates/html/src/lex/tests.rs | 177 +++++++++++++++ crates/html/src/lib.rs | 78 +++++++ 10 files changed, 1213 insertions(+) create mode 100644 crates/html/Cargo.toml create mode 100644 crates/html/README.md create mode 100644 crates/html/src/entity.rs create mode 100644 crates/html/src/entity/tests.rs create mode 100644 crates/html/src/lex.rs create mode 100644 crates/html/src/lex/tests.rs create mode 100644 crates/html/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 11b1a2c9d..ff7081810 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1150,6 +1150,13 @@ dependencies = [ "rayon", ] +[[package]] +name = "craft-html" +version = "0.1.0" +dependencies = [ + "proptest", +] + [[package]] name = "craft-i18n" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 249108320..9931dc81b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,7 @@ craft-gcode = { path = "crates/gcode" } craft-gif = { path = "crates/gif" } craft-hdr = { path = "crates/hdr" } craft-mesh = { path = "crates/mesh" } +craft-html = { path = "crates/html" } craft-heif = { path = "crates/heif" } craft-fig = { path = "crates/fig" } craft-fonts-build = { path = "crates/fonts-build" } diff --git a/README.md b/README.md index ab4347290..85a327ab2 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,7 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = "tcafé <3", opts); +let mut text = String::new(); +while let Some(token) = lexer.next() { + match token { + Token::Text(t) => text.push_str(&unescape(t)), + // A style sheet's text, for a caller that reads it. + Token::Start(tag) if tag.name == "style" => assert_eq!(lexer.raw_content("style"), "p{}"), + Token::Start(tag) => assert_eq!(tag.attr("class").as_deref(), Some("x")), + Token::End(name) => assert_eq!(name, "b"), + } +} +assert_eq!(text, "café <3"); +assert_eq!(escape(&text), "café <3"); +``` + +## The tokenizer + +`Lexer` is an iterator of `Token::Text`, `Token::Start(Tag)` and `Token::End(name)`, borrowed +from the input. Text and attribute values come as written; `unescape` and `Tag::attr` decode +them. + +- A `<` that starts no tag is text. A tag, comment or raw element that never ends runs to the + end of the input. +- Comments, doctypes, processing instructions and conditional comments never come out. +- `Options::drop`: elements skipped whole, content and end tag included. +- `Options::raw_text`: elements whose content is one text token, not markup. +- `Lexer::raw_content(name)`: after a start tag, take the element's raw content yourself. +- `Limits`: attributes kept per tag (64) and the longest tag name (64 bytes; a longer one has an + empty name). + +## Character references + +`unescape` decodes named, decimal and hex references. The named set is HTML 4's (Latin-1, Greek, +symbols, arrows, mathematics) and a few more. The 94 legacy names decode without their `;` +(`© 2024`), longest match first, as in browsers; `unescape_attr` leaves such a name alone +when a letter, a digit or `=` follows, so a query string keeps its `©=1`. A numeric reference +to 0x80-0x9F means the Windows-1252 character; 0, surrogates, and numbers past U+10FFFF or too +long to be numbers become U+FFFD. Names are looked at for 32 bytes at most, so the work is +linear. + +`escape` writes `& < > " '` as references; `escape_dq` leaves `'`, for content and attribute +values in double quotes. + +## Not here + +- The full HTML5 named reference table (about 2,200 names): only the names above. +- Decoding bytes to text. `declared_charset` gives the label a page names; a decoder for it is + the caller's (`craft-mime::charset` for the WHATWG labels). +- Tree construction, implied end tags, CSS. + +## Provenance + +The tokenizer is MailCraft's (`crates/html/src/tokenizer.rs`), with its policy (which elements to +drop, the caps) turned into options. The character references are WordCraft's +(`crates/formats/src/html/entity.rs`: the name table and the legacy names without `;`), with +MailCraft's Latin-1 and Greek tables and its handling of numbers too long to be numbers. +Written from the HTML Living Standard. diff --git a/crates/html/src/entity.rs b/crates/html/src/entity.rs new file mode 100644 index 000000000..8ece41926 --- /dev/null +++ b/crates/html/src/entity.rs @@ -0,0 +1,388 @@ +//! Character references (`&`, `é`, `é`) and escaping. + +/// The Latin-1 names in code point order, from U+00A0 (`nbsp`) to U+00FF (`yuml`). +const LATIN1: [&str; 96] = [ + "nbsp", "iexcl", "cent", "pound", "curren", "yen", "brvbar", "sect", "uml", "copy", "ordf", "laquo", "not", "shy", "reg", "macr", "deg", + "plusmn", "sup2", "sup3", "acute", "micro", "para", "middot", "cedil", "sup1", "ordm", "raquo", "frac14", "frac12", "frac34", "iquest", "Agrave", + "Aacute", "Acirc", "Atilde", "Auml", "Aring", "AElig", "Ccedil", "Egrave", "Eacute", "Ecirc", "Euml", "Igrave", "Iacute", "Icirc", "Iuml", "ETH", + "Ntilde", "Ograve", "Oacute", "Ocirc", "Otilde", "Ouml", "times", "Oslash", "Ugrave", "Uacute", "Ucirc", "Uuml", "Yacute", "THORN", "szlig", + "agrave", "aacute", "acirc", "atilde", "auml", "aring", "aelig", "ccedil", "egrave", "eacute", "ecirc", "euml", "igrave", "iacute", "icirc", + "iuml", "eth", "ntilde", "ograve", "oacute", "ocirc", "otilde", "ouml", "divide", "oslash", "ugrave", "uacute", "ucirc", "uuml", "yacute", + "thorn", "yuml", +]; + +/// Greek capitals from U+0391; U+03A2 is unassigned. +const GREEK_UPPER: [&str; 25] = [ + "Alpha", "Beta", "Gamma", "Delta", "Epsilon", "Zeta", "Eta", "Theta", "Iota", "Kappa", "Lambda", "Mu", "Nu", "Xi", "Omicron", "Pi", "Rho", "", + "Sigma", "Tau", "Upsilon", "Phi", "Chi", "Psi", "Omega", +]; + +/// Greek small letters from U+03B1. +const GREEK_LOWER: [&str; 25] = [ + "alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta", "iota", "kappa", "lambda", "mu", "nu", "xi", "omicron", "pi", "rho", + "sigmaf", "sigma", "tau", "upsilon", "phi", "chi", "psi", "omega", +]; + +/// What a numeric reference to 0x80–0x9F means (HTML Living Standard §13.2.5.80: the +/// Windows-1252 characters, by long tradition). +const C1: [u32; 32] = [ + 0x20AC, 0x81, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x8D, 0x017D, 0x8F, 0x90, 0x2018, 0x2019, + 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x9D, 0x017E, 0x0178, +]; + +/// Longest entity name looked at. +const MAX_NAME: usize = 32; + +/// The legacy names HTML also decodes without a `;`, sorted by bytes. +const BARE_NAMES: &[&str] = &[ + "AElig", "AMP", "Aacute", "Acirc", "Agrave", "Aring", "Atilde", "Auml", "COPY", "Ccedil", "Eacute", "Ecirc", "Egrave", "Euml", "GT", "Iacute", + "Icirc", "Igrave", "Iuml", "LT", "Ntilde", "Oacute", "Ocirc", "Ograve", "Oslash", "Otilde", "Ouml", "QUOT", "REG", "Uacute", "Ucirc", "Ugrave", + "Uuml", "Yacute", "aacute", "acirc", "aelig", "agrave", "amp", "aring", "atilde", "auml", "ccedil", "cent", "copy", "curren", "deg", "divide", + "eacute", "ecirc", "egrave", "euml", "frac12", "frac14", "frac34", "gt", "iacute", "icirc", "iexcl", "igrave", "iquest", "iuml", "laquo", "lt", + "micro", "middot", "nbsp", "ntilde", "oacute", "ocirc", "ograve", "oslash", "otilde", "ouml", "para", "plusmn", "pound", "quot", "raquo", "reg", + "sect", "shy", "sup1", "sup2", "sup3", "szlig", "times", "uacute", "ucirc", "ugrave", "uuml", "yacute", "yen", "yuml", +]; + +/// Text as HTML, for content and for attribute values in either kind of quote: `&`, `<`, `>`, +/// `"` and `'` become character references. +pub fn escape(text: &str) -> String { + let mut out = String::with_capacity(text.len().saturating_add(text.len() / 8)); + push_escaped(&mut out, text, true); + out +} + +/// [`escape`] that leaves `'` as it is: for content and for attribute values in double quotes. +pub fn escape_dq(text: &str) -> String { + let mut out = String::with_capacity(text.len().saturating_add(text.len() / 8)); + push_escaped(&mut out, text, false); + out +} + +fn push_escaped(out: &mut String, text: &str, apos: bool) { + for c in text.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' if apos => out.push_str("'"), + c => out.push(c), + } + } +} + +/// The character an entity body names (`amp`, `#38`, `#x26`: what stands between `&` and `;`). +pub fn decode_entity(body: &str) -> Option { + match body.strip_prefix('#') { + Some(_) => reference(body, false).filter(|(_, used)| *used == body.len()).map(|(c, _)| c), + None => named(body), + } +} + +/// Character references in text decoded: named (the HTML 4 set and more), decimal and hex. +/// Anything that is no reference stays as it is. A legacy name decodes without its `;` +/// (`© 2024`), as in browsers. +pub fn unescape(text: &str) -> String { + unescape_in(text, false) +} + +/// [`unescape`] for an attribute value: a legacy name without its `;` stays text when a letter, +/// a digit or `=` follows, so `?a=1©=2` is a query string and not a copyright sign. +pub fn unescape_attr(text: &str) -> String { + unescape_in(text, true) +} + +fn unescape_in(text: &str, attr: bool) -> String { + if !text.contains('&') { + return text.to_string(); + } + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(i) = rest.find('&') { + out.push_str(rest.get(..i).unwrap_or("")); + let after = rest.get(i + 1..).unwrap_or(""); + match reference(after, attr) { + Some((c, used)) => { + out.push(c); + rest = after.get(used..).unwrap_or(""); + } + None => { + out.push('&'); + rest = after; + } + } + } + out.push_str(rest); + out +} + +/// The character a reference names and the bytes it takes, given the text after its `&` +/// (`attr`: inside an attribute value, see [`unescape_attr`]). At most [`MAX_NAME`] bytes are +/// looked at for a name, so a text of stray `&`s is read in linear time. +pub fn reference(s: &str, attr: bool) -> Option<(char, usize)> { + if let Some(num) = s.strip_prefix('#') { + let (digits, radix, skip) = match num.strip_prefix(['x', 'X']) { + Some(hex) => (hex, 16, 2), + None => (num, 10, 1), + }; + let len = digits.bytes().take_while(|b| char::from(*b).is_digit(radix)).count(); + if len == 0 { + return None; + } + // Too many digits is not a character; neither is 0, a surrogate, or anything past U+10FFFF. + let mut code: u32 = 0; + for d in digits.bytes().take(len) { + let d = char::from(d).to_digit(radix).unwrap_or(0); + code = code.checked_mul(radix).and_then(|c| c.checked_add(d)).unwrap_or(u32::MAX); + } + let semicolon = usize::from(digits.as_bytes().get(len) == Some(&b';')); + return Some((numeric(code), skip + len + semicolon)); + } + let len = s.bytes().take(MAX_NAME).take_while(u8::is_ascii_alphanumeric).count(); + if s.as_bytes().get(len) == Some(&b';') + && let Some(c) = s.get(..len).and_then(named) + { + return Some((c, len + 1)); + } + // Without its `;`: the longest legacy name (2 to 6 letters) the text starts with. + let len = (2..=len.min(6)).rev().find(|&l| s.get(..l).is_some_and(|p| BARE_NAMES.binary_search(&p).is_ok()))?; + if attr && s.as_bytes().get(len).is_some_and(|b| b.is_ascii_alphanumeric() || *b == b'=') { + return None; + } + s.get(..len).and_then(named).map(|c| (c, len)) +} + +fn numeric(code: u32) -> char { + let code = match code { + 0x80..=0x9F => C1.get((code - 0x80) as usize).copied().unwrap_or(code), + c => c, + }; + match char::from_u32(code) { + Some('\0') | None => char::REPLACEMENT_CHARACTER, + Some(c) => c, + } +} + +fn named(name: &str) -> Option { + if name.is_empty() { + return None; + } + let c = match name { + "amp" | "AMP" => '&', + "lt" | "LT" => '<', + "gt" | "GT" => '>', + "quot" | "QUOT" => '"', + "apos" | "APOS" => '\'', + "nbsp" | "NBSP" => '\u{00A0}', + "ensp" => '\u{2002}', + "emsp" => '\u{2003}', + "thinsp" => '\u{2009}', + "shy" => '\u{00AD}', + "copy" | "COPY" => '©', + "reg" | "REG" => '®', + "trade" | "TRADE" => '™', + "hellip" => '…', + "mdash" => '—', + "ndash" => '–', + "minus" => '−', + "lsquo" => '‘', + "rsquo" => '’', + "sbquo" => '‚', + "ldquo" => '“', + "rdquo" => '”', + "bdquo" => '„', + "laquo" => '«', + "raquo" => '»', + "bull" => '•', + "middot" => '·', + "deg" => '°', + "plusmn" => '±', + "times" => '×', + "divide" => '÷', + "frac12" => '½', + "frac14" => '¼', + "frac34" => '¾', + "frac13" => '⅓', + "frac23" => '⅔', + "frac18" => '⅛', + "frac38" => '⅜', + "frac58" => '⅝', + "frac78" => '⅞', + "sup1" => '¹', + "sup2" => '²', + "sup3" => '³', + "micro" => 'µ', + "permil" => '‰', + "prime" => '′', + "Prime" => '″', + "oline" => '‾', + "frasl" => '⁄', + "sect" => '§', + "para" => '¶', + "euro" | "EURO" => '€', + "pound" | "POUND" => '£', + "yen" | "YEN" => '¥', + "cent" | "CENT" => '¢', + "curren" => '¤', + "dagger" => '†', + "Dagger" => '‡', + "larr" => '←', + "rarr" => '→', + "uarr" => '↑', + "darr" => '↓', + "harr" => '↔', + "lArr" => '⇐', + "rArr" => '⇒', + "uArr" => '⇑', + "dArr" => '⇓', + "hArr" => '⇔', + "crarr" => '↵', + "forall" => '∀', + "part" => '∂', + "exist" => '∃', + "empty" => '∅', + "nabla" => '∇', + "isin" => '∈', + "notin" => '∉', + "ni" => '∋', + "prod" => '∏', + "sum" => '∑', + "radic" => '√', + "prop" => '∝', + "ang" => '∠', + "and" => '∧', + "or" => '∨', + "cap" => '∩', + "cup" => '∪', + "int" => '∫', + "there4" => '∴', + "sim" => '∼', + "cong" => '≅', + "asymp" | "approx" => '≈', + "ne" => '≠', + "equiv" => '≡', + "le" => '≤', + "ge" => '≥', + "sub" => '⊂', + "sup" => '⊃', + "nsub" => '⊄', + "sube" => '⊆', + "supe" => '⊇', + "oplus" => '⊕', + "otimes" => '⊗', + "perp" => '⊥', + "sdot" => '⋅', + "infin" => '∞', + "loz" => '◊', + "spades" => '♠', + "clubs" => '♣', + "hearts" => '♥', + "diams" => '♦', + "check" => '✓', + "iexcl" => '¡', + "iquest" => '¿', + "szlig" => 'ß', + "auml" => 'ä', + "ouml" => 'ö', + "uuml" => 'ü', + "Auml" => 'Ä', + "Ouml" => 'Ö', + "Uuml" => 'Ü', + "eacute" => 'é', + "egrave" => 'è', + "ecirc" => 'ê', + "euml" => 'ë', + "Eacute" => 'É', + "Egrave" => 'È', + "Ecirc" => 'Ê', + "Euml" => 'Ë', + "aacute" => 'á', + "agrave" => 'à', + "acirc" => 'â', + "atilde" => 'ã', + "Agrave" => 'À', + "Aacute" => 'Á', + "Acirc" => 'Â', + "Atilde" => 'Ã', + "iacute" => 'í', + "igrave" => 'ì', + "icirc" => 'î', + "iuml" => 'ï', + "Iacute" => 'Í', + "Igrave" => 'Ì', + "Icirc" => 'Î', + "Iuml" => 'Ï', + "oacute" => 'ó', + "ograve" => 'ò', + "ocirc" => 'ô', + "otilde" => 'õ', + "Oacute" => 'Ó', + "Ograve" => 'Ò', + "Ocirc" => 'Ô', + "Otilde" => 'Õ', + "uacute" => 'ú', + "ugrave" => 'ù', + "ucirc" => 'û', + "Uacute" => 'Ú', + "Ugrave" => 'Ù', + "Ucirc" => 'Û', + "yacute" => 'ý', + "yuml" => 'ÿ', + "Yacute" => 'Ý', + "ntilde" => 'ñ', + "Ntilde" => 'Ñ', + "ccedil" => 'ç', + "Ccedil" => 'Ç', + "aring" => 'å', + "Aring" => 'Å', + "oslash" => 'ø', + "Oslash" => 'Ø', + "aelig" => 'æ', + "AElig" => 'Æ', + "alpha" => 'α', + "beta" => 'β', + "gamma" => 'γ', + "delta" => 'δ', + "epsilon" => 'ε', + "zeta" => 'ζ', + "eta" => 'η', + "theta" => 'θ', + "iota" => 'ι', + "kappa" => 'κ', + "lambda" => 'λ', + "mu" => 'μ', + "nu" => 'ν', + "xi" => 'ξ', + "pi" => 'π', + "rho" => 'ρ', + "sigma" => 'σ', + "tau" => 'τ', + "upsilon" => 'υ', + "phi" => 'φ', + "chi" => 'χ', + "psi" => 'ψ', + "omega" => 'ω', + "Gamma" => 'Γ', + "Delta" => 'Δ', + "Theta" => 'Θ', + "Lambda" => 'Λ', + "Xi" => 'Ξ', + "Pi" => 'Π', + "Sigma" => 'Σ', + "Upsilon" => 'Υ', + "Phi" => 'Φ', + "Psi" => 'Ψ', + "Omega" => 'Ω', + "zwj" => '\u{200D}', + "zwnj" => '\u{200C}', + "cross" => '\u{2717}', + _ => { + let at = |table: &[&str], base: u32| table.iter().position(|n| *n == name).and_then(|i| char::from_u32(base + i as u32)); + return at(&LATIN1, 0xA0).or_else(|| at(&GREEK_UPPER, 0x391)).or_else(|| at(&GREEK_LOWER, 0x3B1)); + } + }; + Some(c) +} + +#[cfg(test)] +mod tests; diff --git a/crates/html/src/entity/tests.rs b/crates/html/src/entity/tests.rs new file mode 100644 index 000000000..3abd35fdc --- /dev/null +++ b/crates/html/src/entity/tests.rs @@ -0,0 +1,132 @@ +use super::{BARE_NAMES, GREEK_LOWER, GREEK_UPPER, LATIN1, decode_entity, escape, escape_dq, named, reference, unescape, unescape_attr}; + +#[test] +fn escapes_the_five() { + assert_eq!(escape(r#"Tom & 'Jerry'"#), "<a href="x">Tom & 'Jerry'</a>"); + assert_eq!(escape_dq(r#"Tom & 'Jerry'"#), "<a href="x">Tom & 'Jerry'</a>"); + assert_eq!(escape("plain ünïcode"), "plain ünïcode"); + assert_eq!(escape(""), ""); +} + +#[test] +fn bare_names_are_sorted_and_known() { + assert_eq!(BARE_NAMES.len(), 94); + assert!(BARE_NAMES.windows(2).all(|w| w[0] < w[1])); + assert!(BARE_NAMES.iter().all(|n| (2..=6).contains(&n.len()) && named(n).is_some())); +} + +#[test] +fn every_latin_1_and_greek_name_is_its_code_point() { + for (table, base) in [(&LATIN1[..], 0xA0), (&GREEK_UPPER[..], 0x391), (&GREEK_LOWER[..], 0x3B1)] { + for (i, name) in table.iter().enumerate().filter(|(_, n)| !n.is_empty()) { + assert_eq!(named(name), char::from_u32(base + i as u32), "{name}"); + } + } +} + +#[test] +fn named_references() { + assert_eq!(unescape("<b> & "x" 'y'"), " & \"x\" 'y'"); + assert_eq!(unescape("a b"), "a\u{a0}b"); + assert_eq!(unescape("© ® ™ … — –"), "© ® ™ … — –"); + assert_eq!(unescape("‘a’ “b” €5"), "‘a’ “b” €5"); + assert_eq!(unescape("café naïve Über ß ÿ"), "café naïve Über ß ÿ"); + assert_eq!(unescape("αβΩς → ≤ ≠"), "αβΩς → ≤ ≠"); + assert_eq!(unescape("‌¦ÐþΑο✗✓♦"), "\u{200c}¦Ðþ\u{391}ο✗✓♦"); + assert_eq!(unescape("⅓∴⇔◊"), "⅓∴⇔◊"); +} + +#[test] +fn names_are_case_sensitive() { + assert_eq!(unescape("Éé"), "Éé"); + assert_eq!(unescape("&EACUTE;"), "&EACUTE;"); + assert_eq!(unescape("&<"), "&<"); +} + +#[test] +fn numeric_references() { + assert_eq!(unescape("ééé"), "ééé"); + assert_eq!(unescape("😀"), "😀"); + assert_eq!(unescape("😀"), "😀"); + // Without the semicolon. + assert_eq!(unescape("A B."), "A B."); + // Windows-1252 for the C1 range. + assert_eq!(unescape("“hi” € ™"), "“hi” € ™"); +} + +#[test] +fn bad_numbers_become_the_replacement_character() { + assert_eq!(unescape("�"), "\u{fffd}"); + assert_eq!(unescape("�"), "\u{fffd}"); + assert_eq!(unescape("�"), "\u{fffd}"); + assert_eq!(unescape("�"), "\u{fffd}"); + assert_eq!(unescape("�"), "\u{fffd}"); +} + +#[test] +fn invalid_references_are_left_alone() { + for s in ["&", "&&", "& ", "&;", "&#;", "&#x;", "&#xZZ;", "&bogus;", "AT&T", "a & b", "x&y=z", "&#", "&#x", "&eacut", "ünï&", "&Alpha", "&hellip"] + { + assert_eq!(unescape(s), s, "{s}"); + assert_eq!(unescape_attr(s), s, "{s}"); + } +} + +#[test] +fn legacy_names_work_without_a_semicolon() { + assert_eq!(unescape("a < b > c"), "a < b > c"); + assert_eq!(unescape("x "), "x\u{a0}"); + assert_eq!(unescape("&"), "&"); + assert_eq!(unescape("© 2024 café!"), "© 2024 café!"); + // The longest name wins: `∉` is one reference; `¥x;` is `¥` and `x;`. + assert_eq!(unescape("∉ ¥x;"), "∉ ¥x;"); + // In text even inside a word, as browsers read it. + assert_eq!(unescape("&lify ?a=1&=2"), "&lify ?a=1&=2"); +} + +#[test] +fn in_an_attribute_a_bare_name_before_a_letter_digit_or_equals_is_text() { + assert_eq!(unescape_attr("?a=1&=2<=3©=4®1"), "?a=1&=2<=3©=4®1"); + assert_eq!(unescape_attr("&lify"), "&lify"); + assert_eq!(unescape_attr("?a=1&b=2"), "?a=1&b=2"); + assert_eq!(unescape_attr("a < b©"), "a < b©"); + // Numeric references need no `;` anywhere. + assert_eq!(unescape_attr("A=1"), "A=1"); +} + +#[test] +fn an_entity_body_decodes_whole_or_not_at_all() { + assert_eq!(decode_entity("amp"), Some('&')); + assert_eq!(decode_entity("#38"), Some('&')); + assert_eq!(decode_entity("#x26"), Some('&')); + assert_eq!(decode_entity("#150"), Some('–')); + for none in ["", "#", "#x", "#38x", "#3 8", "bogus", "amp;"] { + assert_eq!(decode_entity(none), None, "{none}"); + } + assert_eq!(reference("amp;rest", false), Some(('&', 4))); + assert_eq!(reference("#x41;", true), Some(('A', 5))); +} + +#[test] +fn a_long_run_of_ampersands_is_quick() { + let s = "&".repeat(200_000); + assert_eq!(unescape(&s), s); + let s = "&#".repeat(100_000); + assert_eq!(unescape(&s), s); + let s = "&aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".repeat(20_000); + assert_eq!(unescape(&s), s); +} + +proptest::proptest! { + #[test] + fn unescape_undoes_escape(s in proptest::prelude::any::()) { + proptest::prop_assert_eq!(unescape(&escape(&s)), s.clone()); + proptest::prop_assert_eq!(unescape_attr(&escape_dq(&s)), s); + } + + #[test] + fn unescape_never_panics(s in "[&#xX0-9a-zA-Z;= é]{0,60}") { + let _ = unescape(&s); + let _ = unescape_attr(&s); + } +} diff --git a/crates/html/src/lex.rs b/crates/html/src/lex.rs new file mode 100644 index 000000000..bf6688ffb --- /dev/null +++ b/crates/html/src/lex.rs @@ -0,0 +1,347 @@ +//! A tolerant HTML tokenizer: text, start tags and end tags. +//! +//! It follows the general syntax of the HTML Living Standard closely enough for documents, mail +//! and clipboard fragments, and reads anything: a `<` that starts no tag is text, a tag or +//! comment that never ends runs to the end of the input. Comments, doctypes and processing +//! instructions never leave it. + +use crate::entity::unescape_attr; + +/// What a [`Lexer`] caps. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Limits { + /// Most attributes kept per tag; the rest are read and dropped. Default 64. + pub max_attrs: usize, + /// Longest tag name kept, in bytes; a longer one is no real element and its name is empty. + /// Default 64. + pub max_tag_name: usize, +} + +impl Limits { + pub const DEFAULT: Limits = Limits { max_attrs: 64, max_tag_name: 64 }; +} + +impl Default for Limits { + fn default() -> Limits { + Limits::DEFAULT + } +} + +/// How a [`Lexer`] reads. Element names are lower case. +#[derive(Clone, Copy, Debug, Default)] +pub struct Options<'o> { + pub limits: Limits, + /// Elements that never leave the lexer: the start tag, the content (raw text, not markup) + /// and the end tag are skipped. For `script`, `style`, `title` and the like, when the caller + /// has no use for them. A caller that wants a `style`'s text leaves it out of this list and + /// calls [`Lexer::raw_content`] on its start tag. + pub drop: &'o [&'o str], + /// Elements whose content is text, not markup, and is given: after the start tag comes one + /// [`Token::Text`] with everything up to the end tag. For `textarea`, `xmp`, `plaintext`. + pub raw_text: &'o [&'o str], +} + +/// A start tag. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Tag<'a> { + /// Lower case; empty for a name longer than [`Limits::max_tag_name`]. + pub name: String, + /// Names and values as written, in the order written. + pub attrs: Vec<(&'a str, &'a str)>, + /// The tag ends in `/>`. + pub self_close: bool, +} + +impl Tag<'_> { + /// An attribute's value with character references decoded ([`unescape_attr`]); the first + /// of a name wins, as in browsers. Names match in any letter case. + pub fn attr(&self, name: &str) -> Option { + self.raw_attr(name).map(unescape_attr) + } + + /// An attribute's value as written, character references and all. + pub fn raw_attr(&self, name: &str) -> Option<&str> { + self.attrs.iter().find(|(n, _)| n.eq_ignore_ascii_case(name)).map(|(_, v)| *v) + } + + /// Whether the tag has the attribute, with or without a value. + pub fn has(&self, name: &str) -> bool { + self.attrs.iter().any(|(n, _)| n.eq_ignore_ascii_case(name)) + } + + /// Every attribute in the order written: the name as written, the value decoded. + pub fn attributes(&self) -> impl Iterator + '_ { + self.attrs.iter().map(|(n, v)| (*n, unescape_attr(v))) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Token<'a> { + /// Character data as written: references are not decoded ([`crate::unescape`]). + Text(&'a str), + Start(Tag<'a>), + /// An end tag's name, lower case (empty when longer than [`Limits::max_tag_name`]). + End(String), +} + +/// The tokenizer: an iterator of [`Token`]s over a string. +pub struct Lexer<'a, 'o> { + src: &'a str, + pos: usize, + opts: Options<'o>, + /// Set after the start tag of a raw-text element: its content comes next. + raw: Option, +} + +impl<'a> Lexer<'a, 'static> { + /// A lexer with [`Options::default`]: nothing dropped, no raw-text elements. + pub fn new(src: &'a str) -> Lexer<'a, 'static> { + Lexer::with(src, Options::default()) + } +} + +impl<'a, 'o> Lexer<'a, 'o> { + pub fn with(src: &'a str, opts: Options<'o>) -> Lexer<'a, 'o> { + Lexer { src, pos: 0, opts, raw: None } + } + + /// How far into the input the tokens given so far go, in bytes. + pub fn pos(&self) -> usize { + self.pos + } + + /// The raw content of element `name`, whose start tag was the last token: everything up to + /// its end tag (any letter case), or to the end of the input when it has none. The end tag + /// is consumed. For `style` and `script`, whose content is not markup. + pub fn raw_content(&mut self, name: &str) -> &'a str { + self.raw = None; + let b = self.bytes(); + let end = find_end_tag(b, self.pos, name).unwrap_or(b.len()); + let text = self.slice(self.pos, end); + self.pos = find_byte(b, end, b'>').map_or(b.len(), |i| i + 1); + text + } + + fn bytes(&self) -> &'a [u8] { + self.src.as_bytes() + } + + fn slice(&self, start: usize, end: usize) -> &'a str { + self.src.get(start..end).unwrap_or("") + } + + /// A tag name in lower case; empty when it is too long to be a real element. + fn tag_name(&self, raw: &str) -> String { + if raw.len() > self.opts.limits.max_tag_name { String::new() } else { raw.to_ascii_lowercase() } + } + + /// Whether the `<` at `i` starts a tag, a comment or a declaration (rather than being text). + fn markup_at(&self, i: usize) -> bool { + let b = self.bytes(); + match b.get(i + 1) { + Some(c) if c.is_ascii_alphabetic() => true, + Some(b'!' | b'?') => true, + Some(b'/') => matches!(b.get(i + 2), Some(c) if c.is_ascii_alphabetic() || *c == b'>'), + _ => false, + } + } + + /// The end of the text run that starts at `self.pos`: the next `<` that starts markup. + fn text_end(&self) -> usize { + let b = self.bytes(); + let mut i = self.pos + 1; + loop { + match find_byte(b, i, b'<') { + Some(j) if self.markup_at(j) => return j, + Some(j) => i = j + 1, + None => return b.len(), + } + } + } + + /// Skip a comment, doctype or processing instruction at `self.pos`. + fn skip_declaration(&mut self) { + let b = self.bytes(); + let end = if b.get(self.pos..self.pos + 4) == Some(b"").map_or(b.len(), |i| i + 3) + } + } else { + find_byte(b, self.pos, b'>').map_or(b.len(), |i| i + 1) + }; + self.pos = end.min(b.len()); + } + + /// Read the end tag at `self.pos` (``). + fn close_tag(&mut self) -> Option> { + let b = self.bytes(); + let start = self.pos + 2; + let mut i = start; + while let Some(c) = b.get(i) { + if c.is_ascii_whitespace() || *c == b'/' || *c == b'>' { + break; + } + i += 1; + } + let name = self.slice(start, i); + self.pos = find_byte(b, i, b'>').map_or(b.len(), |j| j + 1); + // `` is nothing at all. + (!name.is_empty()).then(|| Token::End(self.tag_name(name))) + } + + /// Read the start tag at `self.pos` (``). + fn open_tag(&mut self) -> Tag<'a> { + let b = self.bytes(); + let start = self.pos + 1; + let mut i = start; + while let Some(c) = b.get(i) { + if c.is_ascii_whitespace() || *c == b'/' || *c == b'>' { + break; + } + i += 1; + } + let name = self.tag_name(self.slice(start, i)); + let mut attrs = Vec::new(); + let mut self_close = false; + loop { + while matches!(b.get(i), Some(c) if c.is_ascii_whitespace() || *c == b'/') { + self_close = b.get(i) == Some(&b'/') && b.get(i + 1) == Some(&b'>'); + i += 1; + } + match b.get(i) { + None => break, + Some(b'>') => { + i += 1; + break; + } + Some(_) => {} + } + // An attribute name runs to white space, `/`, `>` or `=`; a leading `=` is part of it. + let name_start = i; + i += 1; + while let Some(c) = b.get(i) { + if c.is_ascii_whitespace() || matches!(*c, b'/' | b'>' | b'=') { + break; + } + i += 1; + } + let attr = self.slice(name_start, i); + let (value, next) = self.attr_value(i); + i = next; + if attrs.len() < self.opts.limits.max_attrs { + attrs.push((attr, value)); + } + } + self.pos = i.min(b.len()); + Tag { name, attrs, self_close } + } + + /// The value after an attribute name that ends at `i` (empty when it has none), and where + /// reading goes on. + fn attr_value(&self, i: usize) -> (&'a str, usize) { + let b = self.bytes(); + let mut j = i; + while matches!(b.get(j), Some(c) if c.is_ascii_whitespace()) { + j += 1; + } + if b.get(j) != Some(&b'=') { + return ("", j); + } + j += 1; + while matches!(b.get(j), Some(c) if c.is_ascii_whitespace()) { + j += 1; + } + match b.get(j) { + Some(&q) if q == b'"' || q == b'\'' => { + let end = find_byte(b, j + 1, q).unwrap_or(b.len()); + (self.slice(j + 1, end), (end + 1).min(b.len())) + } + _ => { + let mut end = j; + while let Some(c) = b.get(end) { + if c.is_ascii_whitespace() || *c == b'>' { + break; + } + end += 1; + } + (self.slice(j, end), end) + } + } + } +} + +impl<'a> Iterator for Lexer<'a, '_> { + type Item = Token<'a>; + + fn next(&mut self) -> Option> { + loop { + let b = self.bytes(); + if self.pos >= b.len() { + return None; + } + if let Some(name) = self.raw.take() { + let end = find_end_tag(b, self.pos, &name).unwrap_or(b.len()); + let text = self.slice(self.pos, end); + self.pos = end; + if !text.is_empty() { + return Some(Token::Text(text)); + } + continue; + } + if b.get(self.pos) != Some(&b'<') || !self.markup_at(self.pos) { + let start = self.pos; + self.pos = self.text_end(); + return Some(Token::Text(self.slice(start, self.pos))); + } + match b.get(self.pos + 1) { + Some(b'!' | b'?') => self.skip_declaration(), + Some(b'/') => { + if let Some(token) = self.close_tag() { + return Some(token); + } + } + _ => { + let tag = self.open_tag(); + if self.opts.drop.contains(&tag.name.as_str()) { + self.raw_content(&tag.name); + continue; + } + if self.opts.raw_text.contains(&tag.name.as_str()) { + self.raw = Some(tag.name.clone()); + } + return Some(Token::Start(tag)); + } + } + } + } +} + +fn find_byte(b: &[u8], from: usize, byte: u8) -> Option { + b.get(from..)?.iter().position(|c| *c == byte).map(|i| i + from) +} + +fn find_sub(b: &[u8], from: usize, needle: &[u8]) -> Option { + b.get(from..)?.windows(needle.len()).position(|w| w == needle).map(|i| i + from) +} + +/// Where `` or the end), at or after `from`. +fn find_end_tag(b: &[u8], from: usize, name: &str) -> Option { + let mut i = from; + while let Some(lt) = find_byte(b, i, b'<') { + let after = lt + 2 + name.len(); + let named = b.get(lt + 1) == Some(&b'/') && b.get(lt + 2..after).is_some_and(|n| n.eq_ignore_ascii_case(name.as_bytes())); + if named && b.get(after).is_none_or(|c| c.is_ascii_whitespace() || matches!(*c, b'/' | b'>')) { + return Some(lt); + } + i = lt + 1; + } + None +} + +#[cfg(test)] +mod tests; diff --git a/crates/html/src/lex/tests.rs b/crates/html/src/lex/tests.rs new file mode 100644 index 000000000..4f19e5b7c --- /dev/null +++ b/crates/html/src/lex/tests.rs @@ -0,0 +1,177 @@ +use super::{Lexer, Limits, Options, Token}; + +/// As a mail reader sets it up. +const MAIL: Options<'static> = + Options { limits: Limits { max_attrs: 32, max_tag_name: 24 }, drop: &["script", "style", "title", "iframe"], raw_text: &["textarea", "xmp"] }; + +fn show<'a>(lexer: impl Iterator>) -> Vec { + lexer + .map(|t| match t { + Token::Text(t) => format!("T:{t}"), + Token::Start(tag) => { + let attrs: Vec = tag.attrs.iter().map(|(n, v)| format!("{n}={v}")).collect(); + if attrs.is_empty() { format!("<{}>", tag.name) } else { format!("<{} {}>", tag.name, attrs.join(" ")) } + } + Token::End(n) => format!(""), + }) + .collect() +} + +fn tokens(html: &str) -> Vec { + show(Lexer::with(html, MAIL)) +} + +#[test] +fn text_and_tags() { + assert_eq!(tokens("Hi there!"), ["T:Hi ", "", "T:there", "", "T:!"]); + assert_eq!(tokens(""), Vec::::new()); + assert_eq!(tokens("plain"), ["T:plain"]); + assert_eq!(tokens("
"), ["
", "
"]); +} + +#[test] +fn attributes_in_every_quoting_style() { + assert_eq!(tokens(r#""#), [r#""#]); + assert_eq!(tokens("t"), ["", "T:t"]); + assert_eq!(tokens(""), [""]); + assert_eq!(tokens("

"), ["

"]); + // A `>` inside a quoted value does not end the tag. + assert_eq!(tokens(r#"t"#), ["b>", "T:t", ""]); +} + +#[test] +fn attribute_values_are_decoded_on_demand() { + let mut t = Lexer::new(r#""#); + let Some(Token::Start(tag)) = t.next() else { panic!("no tag") }; + assert_eq!(tag.attr("href").as_deref(), Some("?a=1&b=2©=3")); + assert_eq!(tag.raw_attr("href"), Some("?a=1&b=2©=3")); + assert_eq!(tag.attr("missing"), None); + assert!(tag.has("checked") && !tag.has("check")); + assert_eq!(tag.attributes().map(|(n, _)| n).collect::>(), ["HREF", "href", "Checked"]); +} + +#[test] +fn a_tag_that_closes_itself_says_so() { + let closes = |html: &str| match Lexer::new(html).next() { + Some(Token::Start(tag)) => tag.self_close, + other => panic!("{other:?}"), + }; + for yes in ["
", "
", "", "", ""] { + assert!(closes(yes), "{yes}"); + } + // A `/` that ends an unquoted value, or stands anywhere but before the `>`, is not it. + for no in ["
", "", "", "", "
d"), ["T:a < b and c > d"]); + assert_eq!(tokens("1<2<3"), ["T:1<2<3"]); + assert_eq!(tokens("<"), ["T:<"]); + assert_eq!(tokens("x<"), ["T:x<"]); + assert_eq!(tokens("<3 <- b"), ["T:a", "T:b"]); +} + +#[test] +fn comments_and_declarations_vanish() { + assert_eq!(tokens("ab"), ["T:a", "T:b"]); + assert_eq!(tokens("

x"), ["

", "T:x"]); + assert_eq!(tokens("x"), ["T:x"]); + assert_eq!(tokens("abc"), ["T:a", "T:b", "T:c"]); + assert_eq!(tokens("shown"), ["T:shown"]); + assert_eq!(tokens("shown"), ["T:shown"]); + // An unfinished comment swallows the rest. + assert_eq!(tokens("a