diff --git a/Cargo.lock b/Cargo.lock index c7509a6ca..7643420ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1327,6 +1327,14 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "craft-xml" +version = "0.1.0" +dependencies = [ + "quick-xml 0.41.0", + "thiserror 2.0.21", +] + [[package]] name = "crc32fast" version = "1.5.2" diff --git a/Cargo.toml b/Cargo.toml index 821190328..01dfe9234 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,6 +59,7 @@ craft-tiff = { path = "crates/tiff" } craft-ui = { path = "crates/ui" } craft-update = { path = "crates/update" } craft-vectorcraft-format = { path = "crates/vectorcraft-format" } +craft-xml = { path = "crates/xml" } craft-video = { path = "crates/video" } craft-web-api = { path = "crates/web-api" } craft-window = { path = "crates/window" } diff --git a/README.md b/README.md index dd23658fe..351596d65 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = "` shape (`request`, `execute`, `commands`, `docInspect`, `uiInspect`, `screenshot`) over the app's JSON control channel. - [`craft-web-host`](crates/web-host/README.md): the host messages between a Craft app's web build and the page that hosts it (`?host=parent` or `data-host="page"`): one contract and one bridge for every app. +- [`craft-xml`](crates/xml/README.md): bounded XML for hostile input: an owned element tree with limits on size, depth, nodes and attributes, streaming of repeated elements, namespace handling (as written, by local name, or canonical prefixes by URI), escaping and a writer. No DTDs and no entity expansion. - [Shared UI component audit](docs/ui-components-audit.md): source inventory, extraction priorities and compatibility boundaries across the Craft suite. - [Theme infrastructure](docs/theme-infrastructure.md): eight-app comparison and the small shared diff --git a/crates/xml/Cargo.toml b/crates/xml/Cargo.toml new file mode 100644 index 000000000..f957fa625 --- /dev/null +++ b/crates/xml/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "craft-xml" +description = "Bounded XML for hostile input: an owned element tree, streaming of repeated elements, a pull reader, escaping and a writer. No DTDs, no entity expansion." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +quick-xml = { version = "0.41", default-features = false } +thiserror = { workspace = true } diff --git a/crates/xml/README.md b/crates/xml/README.md new file mode 100644 index 000000000..a15f514a0 --- /dev/null +++ b/crates/xml/README.md @@ -0,0 +1,242 @@ +# craft-xml + +Bounded XML for hostile input: an owned element tree, streaming of repeated elements, a pull +reader, escaping and a writer. It replaces the small XML helper each Craft app carried. + +It is built on `quick-xml`'s tokenizer (MIT). Everything above the tokens is here: limits, +references, namespaces, recovery, the tree and the writer. + +```rust +use craft_xml::{Local, Options}; + +let root = craft_xml::parse(b"one & two").unwrap(); +assert_eq!(root.attr_u32("id"), Some(5)); +assert_eq!(root.child(Local("t")).map(|t| t.deep_text()).as_deref(), Some("one & two")); + +// Rows go to the callback one at a time; the tree keeps the rest. +let part = b""; +let mut rows = 0; +let sheet = Options::local().parse_streaming(part, &["row"], |row| { + rows += row.attr_u32("r").unwrap_or(0); + Ok::<(), craft_xml::Error>(()) +}); +assert_eq!((rows, sheet.map(|s| s.children.len())), (3, Ok(0))); +``` + +## Reading + +`parse(bytes)` reads with `Options::default()`. `Options` is a plain struct; set what differs. + +| Option | Default | Meaning | +| --- | --- | --- | +| `limits` | see below | What the reader refuses to exceed. | +| `names` | `AsWritten` | `AsWritten` keeps `p:sp`. `Local` keeps `sp`. `Canonical(&c)` rewrites to the caller's prefix for the namespace URI. | +| `ns_decls` | `true` | Keep `xmlns` declarations as attributes. `Options::local()` and `Options::canonical(..)` turn it off. | +| `strict` | `false` | See below. `.strict()` also makes an undefined entity an error. | +| `unknown_entity` | `Drop` | `Drop`, `Literal` (keep `&name;` as text) or `Error`. | +| `doctype` | `Ignore` | `Ignore`, `RejectInternalSubset` or `Reject`. Never processed, whichever is chosen. | +| `cdata`, `pis`, `comments` | `false` | Keep these as their own nodes. CDATA content is otherwise joined to the text around it. | +| `drop_blank_text` | `false` | Drop the text of an element with child elements when all of it is whitespace (indentation). | +| `split_depth` | `2` | For streaming: how far below the root an element may be and still be handed out. | + +### Limits + +| Limit | Default | Error | +| --- | --- | --- | +| `max_input` | 256 MiB | `TooLarge` | +| `max_depth` | 256 levels (the root is 1) | `TooDeep` | +| `max_nodes` | 2,000,000 nodes kept in the tree: elements, text stretches, kept CDATA, instructions and comments | `TooManyNodes` | +| `max_attrs` | 1024 on one start tag, namespace declarations and malformed ones included; with canonical names, also the declarations in scope at once | `TooManyAttrs` | + +A limit exceeded is an error, never a panic and never a truncated tree. Elements handed to a +streaming callback leave the node count, so a streamed part may be of any length. Text is not +counted separately: nothing expands, so it is never more than three times the input (the worst +case is input that is not UTF-8, where each bad byte becomes U+FFFD). + +### What is never done + +- **No DTD processing.** A `` is skipped or refused; its entity and element declarations are not read. +- **No entity expansion** beyond `&` `<` `>` `"` `'` and numeric character references. A reference to a declared entity is an undefined entity. Nothing external is ever fetched. +- No validation, no XPath, no XInclude, no schema. +- No encodings other than UTF-8 and UTF-16. The encoding an XML declaration names is not consulted. +- Names are not checked on reading. The writer checks them (below). + +### Lenient and strict + +A lenient read keeps what a damaged document had: + +- an end tag closes the innermost open element, whatever its name; an end tag with nothing open is skipped; +- elements still open at the end of the input are closed (a truncated part); +- reading stops when the root element ends: nothing after it is looked at; +- a malformed attribute is skipped; bytes that are not UTF-8 become U+FFFD. + +A strict read makes each of those an error, and also a second root element and text outside the +root. Markup the tokenizer cannot read (`` escaped; CR written ` `. +- `esc_attr`, `push_esc_attr`: a double-quoted attribute value. Also `"`, and tab, LF, CR as ` ` ` ` ` `. +- Both drop characters XML 1.0 cannot carry (`is_xml_char`). +- `El::to_xml`, `write_xml`: one line. `El::to_xml_pretty(indent)`, `write_xml_pretty`: an element whose children are all elements has them one per line; any other element is on one line exactly as `to_xml` writes it. +- An element or attribute whose name is not an XML name (`is_name`) is not written, with everything inside it. A name can never inject markup. +- `Writer`: `Writer::document()` (with the `standalone="yes"` declaration), `Writer::new()` (a fragment), `Writer::pretty(indent)`; `decl`, `doctype`, `pi`; `tag(name).attr(k, v).attr_opt(k, v).open()` / `.empty()` / `.text(t)`; `open`, `empty`, `leaf` with a slice of attributes; `close()` (the writer knows the name); `text`, `cdata`, `raw`, `tree`; `finish` closes what is open. + +### Round trip + +Reading what `to_xml` wrote gives an equal tree. Reading a document and writing it keeps the +elements, attributes in order, text, and with the options on, CDATA sections, processing +instructions and comments. It does not keep: + +- the XML declaration, the DOCTYPE, and anything outside the root element; +- quoting and spacing inside tags, and how a character was written (`A` against `A`); +- `` against ``; +- comments, processing instructions and CDATA sections as sections, unless the options keep them; +- a literal CR; literal tab, LF and CR in attribute values (they were read as spaces); +- with `to_xml_pretty`, whitespace-only text between the children of an element-only element; +- namespace declarations, when names are read as `Local` or `Canonical`. + +## Migration + +### WordCraft `crates/docx/src/xml.rs` + +| WordCraft | craft-xml | +| --- | --- | +| `MAX_DEPTH`, `MAX_ELEMENTS`, `MAX_ATTRS` | `Limits`. `max_nodes` counts text nodes too: use twice the element count. | +| `NAMESPACES`, `OTHER_NAMESPACES`, `STRICT`, `prefix_for` | The tables stay in WordCraft, as `Canonical::namespaces`; `Canonical::prefix`. | +| `element_prefix`, `drawing_prefix` | `Canonical::element`. The rules stay in WordCraft. | +| `requirement_prefix` and the `Requires` rewrite | `Canonical::attr_value` with `Scope::uri`. | +| `parse`, `parse_ns(extra)`, `parse_strict` | `Options::canonical(&c).parse`; extra entries first in the table; `.strict()`. | +| `Node`, `El { name, attrs, kids }` | `Node`, `El { name, attrs, children }` | +| `attr` | `attr_or_plain` | +| `els`, `child`, `children`, `find`, `deep_text`, `local`, `to_xml` | `elements`, then the same names | +| `text` | `text` (a `Cow`) | +| `child_val(n)` | `child(n).and_then(\|c\| c.attr_or_plain("w:val"))` | +| `utf16_to_utf8` | `to_utf8` | +| `esc` | `esc` for text, `esc_attr` for attribute values | +| `W::new`, `open`, `empty`, `leaf`, `text`, `raw`, `into_bytes` | `Writer::document`, then the same names | +| `W::close(name)`, `W::val(name, v)`, `w.s` | `close()`, `empty(name, &[("w:val", v)])`, `as_str()` | +| `is_rel_attr`, `REL_*`, `DECORATIVE_*`, `body_ns` | Stay in WordCraft. | + +### GridCraft `crates/xlsx/src/xml.rs` + +| GridCraft | craft-xml | +| --- | --- | +| `MAX_DEPTH`, `MAX_XML_NODES` | `Limits` | +| `parse` | `Options { unknown_entity: Literal, ..Options::local() }.parse` | +| `parse_streaming(xml, split, cb)` | `.parse_streaming(xml, split, cb)`; `cb` returns any error that is `From` | +| `first_start(xml, name)` | `Options::local().first_start(xml, name)` | +| `El.children` (elements), `El.text` | `elements()`, `text()` | +| `attr`, `attr_u32`, `attr_i64`, `attr_f64`, `attr_bool`, `flag`, `child`, `path` | The same names | +| `kids(n)`, `child_val(n)` | `children(n)`, `child_attr(n, "val")` | +| `to_utf8`, `esc`, `push_esc`, `esc_attr` | The same names | +| `push_u64`, `push_a1`, `num`, `push_num`, `rich_text`, `phonetic_data`, `encode_xstring`, `decode_xstring`, `t_el`, `si_body` | Stay in GridCraft. | + +### DeckCraft `crates/pptx/src/xml.rs` + +| DeckCraft | craft-xml | +| --- | --- | +| `MAX_DEPTH`, `MAX_NODES` | `Limits` | +| `parse(bytes)?.root`, `Doc::ns_decls` | `craft_xml::parse(bytes)?`, `root.ns_decls()` (prefix and URI) | +| `attr(n)`, `has_attr(n)` | `attr_local(n)`, `attr_local(n).is_some()` | +| `i64(n)`, `i32(n)`, `u32(n)` | `attr_local(n).and_then(value::to_i64_lossy)` and `to_i32_lossy`, `to_u32_lossy` | +| `f64(n)`, `bool(n)` | `attr_local(n).and_then(value::to_f64)` and `to_bool` | +| `is(l)`, `child(l)`, `children_named(l)`, `path(ls)`, `find(l)` | `is(Local(l))`, `child(Local(l))`, `children(Local(l))`, `path(&[Local(..)])`, `find(Local(l))` | +| `find_all(l, &mut out)` | `out.extend(e.find_all(Local(l)))` | +| `text()` | `deep_text()` | +| `child_val(l)` | `child(Local(l)).and_then(\|c\| c.attr_local("val"))` | +| `local`, `prefix`, `set_attr`, `elements`, `elements_mut`, `to_xml`, `esc`, `esc_attr` | The same names | +| `W::new`, `W::frag`, `finish` | `Writer::document`, `Writer::new`, `into_bytes` | +| `A::new().a(k, v)`, `.o(k, opt)` | `tag(name).attr(k, v)`, `.attr_opt(k, opt)` | +| `A::b(k, opt_bool)`, `A::t(k, bool)` | `.attr_opt(k, opt.map(u8::from))`, `.attr_opt(k, flag.then_some(1))` | +| `open(name, a)`, `open0`, `empty(name, a)`, `empty0` | `tag(name)….open()` / `.empty()`, or `open(name, &[])`, `empty(name, &[])` | +| `close(name)`, `val(name, v)`, `elt(name, t)` | `close()`, `tag(name).attr("val", v).empty()`, `leaf(name, &[], t)` | + +### DesignCraft `crates/idml/src/xml.rs`, `crates/pptx/src/xml.rs`, `crates/engine/src/cmd/xml.rs` + +| DesignCraft | craft-xml | +| --- | --- | +| `parse` | `Options { strict: true, pis: true, ..Options::default() }.parse` | +| `Node::Pi(target, content)` | `Node::Pi { target, data }` | +| `El::new(n).attr(k, v).child(c).text(t)` | `El::new(n).with_attr(k, v).with_child(c).with_text(t)` | +| `set`, `push`, `get`, `local`, `elements` | `set_attr`, `push`, `attr`, `local`, `elements` | +| `find(l)`, `find_all(l)` (direct children) | `child(Local(l))`, `children(Local(l))` | +| `text_content` | `deep_text` | +| `write(out, depth)` | `write_xml_pretty(out, "\t")` | +| `escape_into(out, s, attr)` | `push_esc(out, s)`, `push_esc_attr(out, s)` | +| `document(root)`, `document_with_pi(root, pi)` | `Writer::pretty("\t")`, then `decl(true)`, `pi(target, data)` or `raw(pi)`, `tree(root)`, `into_bytes()` | +| `prop`, `prop_el`, `num`, `boolean` (IDML's `` form) | Stay in DesignCraft, on `attr` and `child`. | +| pptx `esc` | `esc` for text, `esc_attr` for attribute values. `emu`, `emu_len`, `angle`, `pct`, `hex` stay. | +| cmd `esc`, `valid_name` | `esc`/`esc_attr`; `is_name` (which also accepts `:`). The rest is the app's structure commands and stays. | + +### The others + +| App helper | craft-xml | +| --- | --- | +| WordCraft `formats/src/odt/xml.rs`: `events`, `events_at`, `Ev`, `Attrs`, `get`, `local` | `Options::events(utf8)`, `Events::next`, `Events::span`, `Event`. `Event::End` names the start tag. An error is returned, not swallowed. `length` stays. | +| AniCraft `crates/xml`: `parse` | `Options { strict: true, drop_blank_text: true, unknown_entity: Literal, ..Options::default() }.parse` | +| AniCraft `parse_mixed`, `TEXT_NODE` | Not needed: text nodes keep their place among the children. | +| AniCraft `Element::attr`, `u32`, `child`, `f32`, `list(n)` | `attr`, `attr_u32`, `child`, `attr_f64(n) as f32`, `child(n)` then `elements()` | +| LightCraft `lensdb/src/xml.rs`: `parse`, `Node`, `attr`, `num`, `kids`, `kid`, `kid_text`, `entity`, `unescape` | `Options::parse` with its limits (`max_depth: 32`), `El`, `attr`, `attr_f64`, `children`, `child`, `child_text` (trim it), `entity`, `unescape` | +| FilmCraft `interchange/src/xml.rs`: `escape`, `escape_attr`, `XmlWriter` | `esc`, `esc_attr`, `Writer::pretty(" ")` with `decl(false)` and `doctype(name)`; `text_attrs` is `leaf` | +| FilmCraft `parse`, `child`, `children`, `elements`, `text`, `path_text`, `child_text`, `child_i64`, `child_f64` | `Options { doctype: RejectInternalSubset, .. }.parse` and the tree's `child`, `children`, `elements`, `text`, `path`, `child_text`, `value::to_i64_lossy`, `value::to_f64`. The result is an `El`, not a `roxmltree::Document`. | +| FigCraft `vectordrawable/src/xml.rs`: `esc` | `esc_attr` | +| MailCraft `dav`, `ews` readers | `Options { doctype: Reject, .. }.strict()`, with `Canonical` for the DAV and EWS namespaces | + +### Behaviour that changes on migration + +- **Bytes written.** `esc` no longer escapes quotes in text; WordCraft, FilmCraft, FigCraft and DesignCraft's pptx writer escaped `'` and this crate never does (values are double-quoted); DesignCraft's IDML writer wrote ` ` where this writes ` `, and wrote a literal CR in text where this writes ` `. All read back the same, except the CR, which now survives. +- **Bytes that are not UTF-8** in text were an error in WordCraft, GridCraft and DesignCraft's lenient readers. Here a lenient read replaces them and a strict read refuses them. +- **GridCraft** kept an attribute value raw when it held an undefined entity; here the rest of the value is still resolved. It did not normalise line ends in CDATA; here they are. +- **DeckCraft** kept reading after the root element and could fail on what followed; a lenient read here stops. +- **Node counts** include text, so an element limit becomes about twice the number. +- **LightCraft** did not normalise literal tabs and line ends in attribute values to spaces; here they are. +- **FilmCraft** rounded a fractional `child_i64`; `value::to_i64_lossy` truncates. Its `child_f64` accepted `NaN` and infinities; `value::to_f64` does not. +- **AniCraft** kept attributes in a map (the last of a repeated name won); here they are a list and `attr` returns the first. + +### Left to the apps + +- FigCraft's `Out` layout (one attribute per line, as Android Studio formats drawables). +- EffectCraft's `xml_project.rs`: a project codec with its own reader. It writes control characters as `&#N;`, which XML 1.0 forbids and this crate does not read. +- VectorCraft's `xml_id.rs`: object names as XML IDs (`_xHH_` escapes), a naming rule and not XML syntax. +- PhotoCraft's `io/src/xml.rs` and FilmCraft's `parse` guard `roxmltree` and return its document. Code that moves to `El` gets the same guards from `Limits` and `Doctype`; code that stays on `roxmltree` keeps its guard. +- Booleans that are not `xsd:boolean`: FilmCraft's `TRUE`/`FALSE`, AniCraft's "anything but `true` is false". +- Number formatting (`num`), units, colours, and every format's own vocabulary. diff --git a/crates/xml/src/lib.rs b/crates/xml/src/lib.rs new file mode 100644 index 000000000..ca33c0224 --- /dev/null +++ b/crates/xml/src/lib.rs @@ -0,0 +1,78 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! Bounded XML for hostile input. +//! +//! - [`parse`] and [`Options::parse`] read bytes into an owned tree of [`El`] and [`Node`], with +//! limits on input size, nesting, node count and attributes per element ([`Limits`]). +//! - [`Options::parse_streaming`] hands repeated elements to a callback, so a large part never +//! becomes a whole tree. [`Options::events`] is the pull reader underneath. +//! - [`Names`] chooses how names are read: as written, by local name, or rewritten to the +//! caller's canonical prefixes by namespace URI. +//! - [`esc`], [`esc_attr`], [`El::to_xml`], [`El::to_xml_pretty`] and [`Writer`] write XML. +//! +//! No document type definition is processed, and no entity is expanded except the five +//! predefined ones (`&` `<` `>` `"` `'`) and numeric character references. +//! The text the reader produces is therefore never more than three times its input. +//! +//! ``` +//! let root = craft_xml::parse(b"one & two")?; +//! assert_eq!(root.attr_u32("x"), Some(1)); +//! assert_eq!(root.child("b").map(|b| b.text().into_owned()).as_deref(), Some("one & two")); +//! assert_eq!(root.to_xml(), r#"one & two"#); +//! # Ok::<(), craft_xml::Error>(()) +//! ``` + +#[cfg(doctest)] +#[doc = include_str!("../README.md")] +struct ReadmeExamples; + +mod read; +mod tree; +pub mod value; +mod write; + +pub use read::{ + AttrValueHook, Canonical, Doctype, ElementHook, Event, Events, Limits, Names, NsName, Options, Scope, UnknownEntity, XML_NS, entity, parse, + to_utf8, unescape, +}; +pub use tree::{Descendants, El, Local, Node, Pat}; +pub use write::{DECL, DECL_STANDALONE, Tag, Writer, esc, esc_attr, is_name, is_xml_char, push_esc, push_esc_attr}; + +/// Why a document was not read. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The input is longer than [`Limits::max_input`]. + #[error("the XML is larger than {0} bytes")] + TooLarge(usize), + /// Elements nest deeper than [`Limits::max_depth`]. + #[error("XML elements nest deeper than {0} levels")] + TooDeep(usize), + /// The tree would hold more than [`Limits::max_nodes`] nodes. + #[error("the XML holds more than {0} nodes")] + TooManyNodes(usize), + /// One start tag has more than [`Limits::max_attrs`] attributes. + #[error("an XML element has more than {0} attributes")] + TooManyAttrs(usize), + /// The document has a document type declaration that [`Options::doctype`] refuses. + #[error("the XML has a document type declaration, which is not accepted")] + Doctype, + /// A reference to an entity that is neither predefined nor numeric, with + /// [`UnknownEntity::Error`]. + #[error("the entity `&{name};` at byte {at} is not defined")] + UnknownEntity { at: usize, name: String }, + /// Text that is not UTF-8, in a strict parse. + #[error("the XML is not valid UTF-8 at byte {0}")] + Encoding(usize), + /// Markup that cannot be read. `at` is a byte offset into the UTF-8 text ([`to_utf8`]). + #[error("malformed XML at byte {at}: {msg}")] + Syntax { at: usize, msg: String }, + /// There is no root element. + #[error("the XML has no root element")] + NoRoot, +} + +impl Error { + /// Was a limit exceeded (as opposed to the markup being wrong)? + pub fn is_limit(&self) -> bool { + matches!(self, Error::TooLarge(_) | Error::TooDeep(_) | Error::TooManyNodes(_) | Error::TooManyAttrs(_)) + } +} diff --git a/crates/xml/src/read.rs b/crates/xml/src/read.rs new file mode 100644 index 000000000..bbe5c221c --- /dev/null +++ b/crates/xml/src/read.rs @@ -0,0 +1,758 @@ +//! Reading: the pull reader over `quick-xml`, the tree builder on top of it, and streaming. + +use std::borrow::Cow; +use std::ops::Range; + +use quick_xml::Reader; +use quick_xml::events::{BytesStart, Event as Q}; + +use crate::Error; +use crate::tree::{El, Node, Pat, local_of}; +use crate::write::is_xml_char; + +/// The namespace the `xml` prefix is always bound to. +pub const XML_NS: &str = "http://www.w3.org/XML/1998/namespace"; + +/// What the reader refuses to exceed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Limits { + /// Most bytes of input. Default 256 MiB. + pub max_input: usize, + /// Deepest element nesting: the root is at depth 1. Default 256. + pub max_depth: usize, + /// Most nodes in a tree: elements, text stretches and whatever else is kept. Elements + /// handed to a streaming callback leave the count with everything inside them. Default + /// 2,000,000: a tree node costs about 72 bytes before its strings, and text lies between + /// elements at most, so this is about a million elements of an indented document. + pub max_nodes: usize, + /// Most attributes on one start tag, namespace declarations and malformed ones included. + /// With [`Names::Canonical`] it is also the most namespace declarations in scope at once. + /// Default 1024. + pub max_attrs: usize, +} + +impl Limits { + pub const DEFAULT: Limits = Limits { max_input: 256 * 1024 * 1024, max_depth: 256, max_nodes: 2_000_000, max_attrs: 1024 }; +} + +impl Default for Limits { + fn default() -> Limits { + Limits::DEFAULT + } +} + +/// How element and attribute names are read. +#[derive(Clone, Copy, Default)] +pub enum Names<'a> { + /// The qualified name as written: `p:sp`, `r:id`. Match with a `&str` when the prefixes + /// are known, or with [`crate::Local`] and [`El::attr_local`] when they are not. + #[default] + AsWritten, + /// The local name only: `sp`, `id`. + Local, + /// The caller's prefix for the name's namespace and the local name: `w:p`, whatever + /// prefix the document used. See [`Canonical`]. + Canonical(&'a Canonical<'a>), +} + +/// An element name being given its canonical prefix. +#[derive(Clone, Copy, Debug)] +pub struct NsName<'s> { + /// The prefix [`Canonical::namespaces`] has for `uri`, or [`Canonical::unknown`]. + pub prefix: &'s str, + pub uri: &'s str, + pub local: &'s str, + /// The canonical name of the enclosing element; `None` at the root. + pub parent: Option<&'s str>, +} + +/// See [`Canonical::element`]. +pub type ElementHook<'a> = dyn Fn(&NsName<'_>) -> Option<&'static str> + 'a; +/// See [`Canonical::attr_value`]. +pub type AttrValueHook<'a> = dyn Fn(&str, &str, &str, &Scope<'_>) -> Option + 'a; + +/// Canonical prefixes by namespace URI. +/// +/// - An element in a listed namespace is named `prefix:local`; in an unlisted one, +/// `unknown:local`; in no namespace (or with a prefix nothing declares), `local`. +/// - A prefixed attribute is named the same way. An unprefixed attribute is in no namespace +/// and keeps its name. +/// - The `xml` prefix needs no entry: `xml:space` stays `xml:space`. +#[derive(Clone, Copy)] +pub struct Canonical<'a> { + /// (prefix, namespace URI). The first entry with a URI wins, so several URIs may share a + /// prefix (the Strict and Transitional forms of a vocabulary). + pub namespaces: &'a [(&'a str, &'a str)], + /// The prefix for a namespace that is not listed. `"?"` by default: it cannot be part of an + /// XML name, so such names never collide with real ones and [`El::write_xml`] leaves them + /// out. + pub unknown: &'a str, + /// Overrides the prefix of an element, for vocabularies where one namespace is read as + /// several depending on the element and its parent. `None` keeps [`NsName::prefix`]. The + /// prefixes it returns are constants of the caller's. + pub element: Option<&'a ElementHook<'a>>, + /// Rewrites an attribute value while the namespace declarations in scope are still known: + /// for values that hold prefixes (`mc:Choice/@Requires`, `xsi:type`). Called with the + /// element's name, the attribute's name, the value and the scope; `None` keeps the value. + pub attr_value: Option<&'a AttrValueHook<'a>>, +} + +impl<'a> Canonical<'a> { + pub const fn new(namespaces: &'a [(&'a str, &'a str)]) -> Canonical<'a> { + Canonical { namespaces, unknown: "?", element: None, attr_value: None } + } + /// The prefix for `uri`: the listed one, `xml` for [`XML_NS`], else [`Canonical::unknown`]. + pub fn prefix(&self, uri: &str) -> &'a str { + match self.namespaces.iter().find(|(_, u)| *u == uri) { + Some((p, _)) => p, + None if uri == XML_NS => "xml", + None => self.unknown, + } + } +} + +/// The namespace declarations in scope at an element. +pub struct Scope<'s> { + bindings: &'s [(String, String)], +} + +impl<'s> Scope<'s> { + /// The namespace URI `prefix` is bound to; `""` is the default namespace. + pub fn uri(&self, prefix: &str) -> Option<&'s str> { + if prefix == "xml" { + return Some(XML_NS); + } + self.bindings.iter().rev().find(|(p, _)| p == prefix).map(|(_, u)| u.as_str()).filter(|u| !u.is_empty()) + } +} + +/// What to do with a reference to an entity that is neither predefined nor numeric. Entities a +/// document declares are never expanded, so `&name;` for a declared entity is one of these too. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum UnknownEntity { + /// Leave it out. + #[default] + Drop, + /// Keep it as the text `&name;`. + Literal, + /// Refuse the document ([`Error::UnknownEntity`]). + Error, +} + +/// What to do with a document type declaration. Whatever is chosen, the declaration is never +/// processed. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Doctype { + /// Skip it. + #[default] + Ignore, + /// Refuse a declaration with an internal subset (``), skip any other. + RejectInternalSubset, + /// Refuse any declaration. + Reject, +} + +/// How to read a document. `Options::default()` keeps names as written and is lenient. +#[derive(Clone, Copy)] +pub struct Options<'a> { + pub limits: Limits, + pub names: Names<'a>, + /// Keep namespace declarations (`xmlns`, `xmlns:p`) as attributes, as written. On by + /// default; [`Options::local`] and [`Options::canonical`] turn it off. + pub ns_decls: bool, + /// Refuse what a lenient read repairs. Lenient: an end tag closes the innermost open + /// element whatever its name, an end tag with nothing open is skipped, elements still open + /// at the end of the input are closed, reading stops when the root element ends, a + /// malformed attribute is skipped, and bytes that are not UTF-8 become U+FFFD. Strict: each + /// of those is an error, as are a second root element and text outside the root. + pub strict: bool, + pub unknown_entity: UnknownEntity, + pub doctype: Doctype, + /// Keep CDATA sections as [`Node::CData`] instead of joining them to the text around them. + pub cdata: bool, + /// Keep processing instructions inside the root element as [`Node::Pi`]. + pub pis: bool, + /// Keep comments inside the root element as [`Node::Comment`]. + pub comments: bool, + /// Drop the text of an element that has child elements when all of it is whitespace: the + /// indentation of an indented document. + pub drop_blank_text: bool, + /// For [`Options::parse_streaming`]: how far below the root an element may be and still be + /// handed out. 2 by default: children and grandchildren of the root. + pub split_depth: usize, +} + +impl Default for Options<'_> { + fn default() -> Self { + Options { + limits: Limits::DEFAULT, + names: Names::AsWritten, + ns_decls: true, + strict: false, + unknown_entity: UnknownEntity::Drop, + doctype: Doctype::Ignore, + cdata: false, + pis: false, + comments: false, + drop_blank_text: false, + split_depth: 2, + } + } +} + +/// Reads `bytes` with [`Options::default`]. +pub fn parse(bytes: &[u8]) -> Result { + Options::default().parse(bytes) +} + +impl<'a> Options<'a> { + /// Names by local part, namespace declarations dropped. + pub fn local() -> Options<'static> { + Options { names: Names::Local, ns_decls: false, ..Options::default() } + } + /// Names with canonical prefixes, namespace declarations dropped. + pub fn canonical(canonical: &'a Canonical<'a>) -> Options<'a> { + Options { names: Names::Canonical(canonical), ns_decls: false, ..Options::default() } + } + /// [`Options::strict`] on, and an undefined entity an error. + #[must_use] + pub fn strict(self) -> Self { + Options { strict: true, unknown_entity: UnknownEntity::Error, ..self } + } + #[must_use] + pub fn with_limits(self, limits: Limits) -> Self { + Options { limits, ..self } + } + + /// Reads a document into its root element. + pub fn parse(&self, bytes: &[u8]) -> Result { + self.parse_streaming(bytes, &[], |_| Ok::<(), Error>(())) + } + + /// Reads a document, handing each complete element whose name is in `split` to `each` + /// instead of keeping it in the tree: worksheet rows, shared strings. Only elements at + /// most [`Options::split_depth`] levels below the root are handed out; a deeper element of + /// the same name stays in the tree. `split` names are matched against names as this reader + /// produces them. Elements handed out do not count toward [`Limits::max_nodes`] once they + /// are gone, so a part of any length can be read; each one is itself held to the limit. + /// + /// An error from `each` stops the read and is returned. + pub fn parse_streaming>(&self, bytes: &[u8], split: &[&str], mut each: impl FnMut(El) -> Result<(), E>) -> Result { + if bytes.len() > self.limits.max_input { + return Err(Error::TooLarge(self.limits.max_input).into()); + } + let text = to_utf8(bytes); + let mut events = Events::new(*self, &text, false); + let mut tree = Builder { opts: self, split, stack: Vec::new(), marks: Vec::new(), kept: 0, root: None, at: 0 }; + while let Some(ev) = events.next()? { + tree.at = events.span().start; + tree.event(ev, &mut each)?; + if !self.strict && tree.root.is_some() { + break; + } + } + // A lenient read keeps what a truncated document had; a strict one failed above. + while let Some(el) = tree.stack.pop() { + tree.finish(el, &mut each)?; + } + tree.root.ok_or_else(|| Error::NoRoot.into()) + } + + /// The first element that matches, from its start tag alone: its name and attributes, no + /// children. Nothing after the tag is read, so unbalanced markup around it neither hides + /// nor moves it. `None` when there is none or the markup before it cannot be read. + pub fn first_start(&self, bytes: &[u8], pat: impl Pat) -> Option { + if bytes.len() > self.limits.max_input { + return None; + } + let text = to_utf8(bytes); + // No tree is built, and markup that never closes its tags would otherwise look deep. + let opts = Options { limits: Limits { max_depth: usize::MAX, ..self.limits }, ..*self }; + let mut events = Events::new(opts, &text, false); + loop { + match events.next().ok()?? { + Event::Start { name, attrs } if pat.matches(&name) => return Some(El { name, attrs, children: Vec::new() }), + _ => {} + } + } + } + + /// A pull reader over `utf8`, which is UTF-8 text: pass other input through [`to_utf8`] + /// first. No tree is built, so [`Limits::max_nodes`] and [`Limits::max_input`] do not + /// apply; depth and attribute limits do. + pub fn events<'i>(&self, utf8: &'i [u8]) -> Events<'a, 'i> { + Events::new(*self, utf8, true) + } +} + +/// One step of a document. See [`Options::events`]. +#[derive(Clone, Debug, PartialEq)] +pub enum Event { + /// A start tag. An empty element (``) is a `Start` followed by its `End`. + Start { + name: String, + attrs: Vec<(String, String)>, + }, + /// The end of the element named `name` (the name of its start tag). + End { + name: String, + }, + /// Character data, references resolved. One stretch of text may arrive in several events. + Text(String), + /// The content of a CDATA section. + CData(String), + Pi { + target: String, + data: String, + }, + Comment(String), +} + +struct Open { + /// Where this element's namespace declarations start in the scope. + scope: usize, + /// The name as written, kept for a strict read to check the end tag against. + raw: Vec, + /// The name as produced, kept when an end event or a canonical parent needs it. + name: String, +} + +/// A pull reader. See [`Options::events`]. +pub struct Events<'o, 'i> { + reader: Reader<&'i [u8]>, + opts: Options<'o>, + /// Whether [`Event::End`] carries the name. + end_names: bool, + open: Vec, + scope: Vec<(String, String)>, + /// The end of an empty element, due after its start. + pending: bool, + span: Range, +} + +fn at(pos: u64) -> usize { + usize::try_from(pos).unwrap_or(usize::MAX) +} + +fn line_ends(s: Cow<'_, str>) -> String { + // XML 1.0 §2.11: a literal CR LF or CR is read as LF. + if s.contains('\r') { s.replace("\r\n", "\n").replace('\r', "\n") } else { s.into_owned() } +} + +fn is_ns_decl(key: &[u8]) -> bool { + key == b"xmlns" || key.starts_with(b"xmlns:") +} + +impl<'o, 'i> Events<'o, 'i> { + fn new(opts: Options<'o>, utf8: &'i [u8], end_names: bool) -> Self { + let mut reader = Reader::from_reader(utf8); + let cfg = reader.config_mut(); + // The checks a strict read wants are made here, where the answer can be lenient. + cfg.check_end_names = false; + cfg.allow_unmatched_ends = true; + cfg.allow_dangling_amp = true; + cfg.trim_text_start = false; + cfg.trim_text_end = false; + cfg.expand_empty_elements = false; + Events { reader, opts, end_names, open: Vec::new(), scope: Vec::new(), pending: false, span: 0..0 } + } + + /// The bytes of the input the last event was read from. Both events of an empty element + /// have the whole element. + pub fn span(&self) -> Range { + self.span.clone() + } + + fn syntax(&self, msg: impl Into) -> Result { + Err(Error::Syntax { at: self.span.start, msg: msg.into() }) + } + + fn decode<'b>(&self, bytes: &'b [u8]) -> Result, Error> { + match std::str::from_utf8(bytes) { + Ok(s) => Ok(Cow::Borrowed(s)), + Err(e) if self.opts.strict => Err(Error::Encoding(self.span.start.saturating_add(e.valid_up_to()))), + Err(_) => Ok(String::from_utf8_lossy(bytes)), + } + } + + fn unknown(&self, name: &str) -> Result, Error> { + match self.opts.unknown_entity { + UnknownEntity::Drop => Ok(None), + UnknownEntity::Literal => Ok(Some(format!("&{name};"))), + UnknownEntity::Error => Err(Error::UnknownEntity { at: self.span.start, name: name.to_string() }), + } + } + + /// The next event, or `None` at the end of the input. + #[allow(clippy::should_implement_trait)] + pub fn next(&mut self) -> Result, Error> { + if self.pending { + self.pending = false; + return Ok(Some(self.end())); + } + loop { + let from = at(self.reader.buffer_position()); + let ev = match self.reader.read_event() { + Ok(ev) => ev, + Err(e) => return Err(Error::Syntax { at: at(self.reader.error_position()), msg: e.to_string() }), + }; + self.span = from..at(self.reader.buffer_position()); + if let Some(out) = self.step(ev)? { + return Ok(out); + } + } + } + + /// `None` to read on, `Some(None)` at the end of the input. + fn step(&mut self, ev: Q<'i>) -> Result>, Error> { + let event = match ev { + Q::Start(e) => self.start(&e)?, + Q::Empty(e) => { + self.pending = true; + self.start(&e)? + } + Q::End(e) => { + let Some(top) = self.open.last() else { + return if self.opts.strict { self.syntax("an end tag has no start tag") } else { Ok(None) }; + }; + if self.opts.strict && top.raw != e.name().as_ref() { + return self.syntax(format!("the end tag `{}` does not match its start tag", String::from_utf8_lossy(e.name().as_ref()))); + } + self.end() + } + Q::Text(t) => { + if t.is_empty() { + return Ok(None); + } + Event::Text(line_ends(self.decode(&t)?)) + } + Q::CData(t) => Event::CData(line_ends(self.decode(&t)?)), + Q::GeneralRef(r) => { + let name = self.decode(&r)?; + match entity(&name) { + Some(c) => Event::Text(c.to_string()), + None => match self.unknown(&name)? { + Some(literal) => Event::Text(literal), + None => return Ok(None), + }, + } + } + Q::PI(p) => Event::Pi { target: self.decode(p.target())?.into_owned(), data: self.decode(p.content())?.trim_start().to_string() }, + Q::Comment(c) => Event::Comment(self.decode(&c)?.into_owned()), + Q::DocType(d) => { + let refuse = match self.opts.doctype { + Doctype::Ignore => false, + // A `[` in a quoted identifier is refused too, which fails safe. + Doctype::RejectInternalSubset => d.contains(&b'['), + Doctype::Reject => true, + }; + return if refuse { Err(Error::Doctype) } else { Ok(None) }; + } + Q::Decl(_) => return Ok(None), + Q::Eof => { + return if self.opts.strict && !self.open.is_empty() { self.syntax("an element is not closed") } else { Ok(Some(None)) }; + } + }; + Ok(Some(Some(event))) + } + + fn end(&mut self) -> Event { + let (scope, name) = self.open.pop().map_or((0, String::new()), |o| (o.scope, o.name)); + self.scope.truncate(scope); + Event::End { name } + } + + /// The URI `prefix` is bound to here; `""` is the default namespace. + fn uri(&self, prefix: &str) -> Option<&str> { + Scope { bindings: &self.scope }.uri(prefix) + } + + fn element_name(&self, raw: &str) -> String { + match self.opts.names { + Names::AsWritten => raw.to_string(), + Names::Local => local_of(raw).to_string(), + Names::Canonical(canon) => { + let (prefix, local) = raw.split_once(':').unwrap_or(("", raw)); + let Some(uri) = self.uri(prefix) else { return local.to_string() }; + let listed = canon.prefix(uri); + let parent = self.open.last().map(|o| o.name.as_str()); + let chosen = canon.element.and_then(|hook| hook(&NsName { prefix: listed, uri, local, parent })).unwrap_or(listed); + format!("{chosen}:{local}") + } + } + } + + fn attr_name(&self, raw: &str) -> String { + match self.opts.names { + Names::AsWritten => raw.to_string(), + Names::Local => local_of(raw).to_string(), + Names::Canonical(canon) => match raw.split_once(':') { + Some((prefix, local)) => match self.uri(prefix) { + Some(uri) => format!("{}:{local}", canon.prefix(uri)), + None => local.to_string(), + }, + None => raw.to_string(), + }, + } + } + + fn attr_value(&self, raw: &[u8]) -> Result { + match decode_refs(&self.decode(raw)?, true, self.opts.unknown_entity) { + Ok(v) => Ok(v), + Err(name) => Err(Error::UnknownEntity { at: self.span.start, name }), + } + } + + /// Reads the tag's namespace declarations into the scope, so that the tag's own name and + /// attributes resolve against them. + fn declare(&mut self, e: &BytesStart<'_>) -> Result<(), Error> { + let mut attrs = e.attributes(); + attrs.with_checks(false); + // The attribute pass that follows counts every attribute; this one only has to stop. + for a in attrs.take(self.opts.limits.max_attrs).flatten() { + let key = a.key.as_ref(); + if is_ns_decl(key) { + let prefix = self.decode(key.get(6..).unwrap_or_default())?.into_owned(); + let uri = self.attr_value(&a.value)?; + self.scope.push((prefix, uri)); + } + } + // Every name is resolved against the scope, so the scope as a whole is bounded too: + // declarations piled up over many levels would make each lookup long. + if self.scope.len() > self.opts.limits.max_attrs { + return Err(Error::TooManyAttrs(self.opts.limits.max_attrs)); + } + Ok(()) + } + + fn start(&mut self, e: &BytesStart<'_>) -> Result { + let limits = self.opts.limits; + if self.open.len() >= limits.max_depth { + return Err(Error::TooDeep(limits.max_depth)); + } + let canonical = match self.opts.names { + Names::Canonical(c) => Some(c), + _ => None, + }; + let scope = self.scope.len(); + if canonical.is_some() { + self.declare(e)?; + } + let raw = e.name(); + let name = self.element_name(&self.decode(raw.as_ref())?); + let mut out = Vec::new(); + let mut attrs = e.attributes(); + attrs.with_checks(false); + for (seen, a) in attrs.enumerate() { + if seen >= limits.max_attrs { + return Err(Error::TooManyAttrs(limits.max_attrs)); + } + let a = match a { + Ok(a) => a, + Err(err) if self.opts.strict => return self.syntax(err.to_string()), + Err(_) => continue, + }; + let key = a.key.as_ref(); + let decl = is_ns_decl(key); + if decl && !self.opts.ns_decls { + continue; + } + let key = self.decode(key)?; + let key = if decl { key.into_owned() } else { self.attr_name(&key) }; + let mut value = self.attr_value(&a.value)?; + if let Some(hook) = canonical.and_then(|c| c.attr_value) + && let Some(rewritten) = hook(&name, &key, &value, &Scope { bindings: &self.scope }) + { + value = rewritten; + } + out.push((key, value)); + } + let keep_name = self.end_names || canonical.is_some_and(|c| c.element.is_some()); + self.open.push(Open { + scope, + raw: if self.opts.strict { raw.as_ref().to_vec() } else { Vec::new() }, + name: if keep_name { name.clone() } else { String::new() }, + }); + Ok(Event::Start { name, attrs: out }) + } +} + +struct Builder<'b, 'o> { + opts: &'b Options<'o>, + split: &'b [&'b str], + stack: Vec, + /// The node count when each open element started. + marks: Vec, + kept: usize, + root: Option, + /// Where the event being handled starts, for errors. + at: usize, +} + +impl Builder<'_, '_> { + fn count(&mut self) -> Result<(), Error> { + self.kept = self.kept.saturating_add(1); + if self.kept > self.opts.limits.max_nodes { Err(Error::TooManyNodes(self.opts.limits.max_nodes)) } else { Ok(()) } + } + + fn push(&mut self, node: Node) -> Result<(), Error> { + if !self.stack.is_empty() { + self.count()?; + } + if let Some(top) = self.stack.last_mut() { + top.children.push(node); + } + Ok(()) + } + + fn text(&mut self, text: String) -> Result<(), Error> { + match self.stack.last_mut() { + Some(top) => { + if let Some(Node::Text(prev)) = top.children.last_mut() { + prev.push_str(&text); + return Ok(()); + } + self.push(Node::Text(text)) + } + None if self.opts.strict && !text.trim().is_empty() => Err(Error::Syntax { at: self.at, msg: "text outside the root element".into() }), + None => Ok(()), + } + } + + fn event>(&mut self, ev: Event, each: &mut impl FnMut(El) -> Result<(), E>) -> Result<(), E> { + match ev { + Event::Start { name, attrs } => { + if self.root.is_some() { + return Err(Error::Syntax { at: self.at, msg: "more than one root element".into() }.into()); + } + self.stack.push(El { name, attrs, children: Vec::new() }); + self.marks.push(self.kept); + } + Event::End { .. } => { + if let Some(el) = self.stack.pop() { + self.finish(el, each)?; + } + } + Event::Text(t) => self.text(t)?, + Event::CData(t) if self.opts.cdata => self.push(Node::CData(t))?, + Event::CData(t) => self.text(t)?, + Event::Pi { target, data } if self.opts.pis => self.push(Node::Pi { target, data })?, + Event::Comment(c) if self.opts.comments => self.push(Node::Comment(c))?, + Event::Pi { .. } | Event::Comment(_) => {} + } + Ok(()) + } + + /// Attaches a complete element to its parent, hands it out, or makes it the root. + fn finish>(&mut self, mut el: El, each: &mut impl FnMut(El) -> Result<(), E>) -> Result<(), E> { + let mark = self.marks.pop().unwrap_or(0); + if self.opts.drop_blank_text && el.children.iter().any(|n| matches!(n, Node::El(_))) { + let is_text = |n: &Node| matches!(n, Node::Text(_)); + if el.children.iter().all(|n| !is_text(n) || matches!(n, Node::Text(t) if t.trim().is_empty())) { + let before = el.children.len(); + el.children.retain(|n| !is_text(n)); + self.kept = self.kept.saturating_sub(before - el.children.len()); + } + } + let depth = self.stack.len(); + if depth == 0 { + self.count()?; + self.root = Some(el); + } else if depth <= self.opts.split_depth && self.split.contains(&el.name.as_str()) { + each(el)?; + // The element and everything inside it have left the tree. + self.kept = mark; + } else { + self.push(Node::El(el))?; + } + Ok(()) + } +} + +/// The input as UTF-8 text for the reader: a UTF-8 byte-order mark is removed, and UTF-16 (with +/// a byte-order mark, or recognisable without one by the zero bytes of its first two +/// characters) is transcoded. An unpaired surrogate becomes U+FFFD and a trailing odd byte is +/// dropped. The result is at most 1.5 times the input. Any other input is returned as it is; +/// the encoding an XML declaration names is not consulted. +pub fn to_utf8(bytes: &[u8]) -> Cow<'_, [u8]> { + let (little_endian, body) = match bytes { + [0xEF, 0xBB, 0xBF, rest @ ..] => return Cow::Borrowed(rest), + [0xFF, 0xFE, rest @ ..] => (true, rest), + [0xFE, 0xFF, rest @ ..] => (false, rest), + [a, 0, b, 0, ..] if *a != 0 && *b != 0 => (true, bytes), + [0, a, 0, b, ..] if *a != 0 && *b != 0 => (false, bytes), + _ => return Cow::Borrowed(bytes), + }; + let units = body.as_chunks::<2>().0.iter().map(|pair| if little_endian { u16::from_le_bytes(*pair) } else { u16::from_be_bytes(*pair) }); + Cow::Owned(char::decode_utf16(units).map(|c| c.unwrap_or(char::REPLACEMENT_CHARACTER)).collect::().into_bytes()) +} + +/// The character a reference names, without its `&` and `;`: one of the five predefined +/// entities (`amp`, `lt`, `gt`, `quot`, `apos`) or a numeric character reference (`#65`, +/// `#x41`, zero-padded or not). `None` for anything else, for a number that is not a character +/// XML can carry, and for a number with a sign. +pub fn entity(name: &str) -> Option { + let number = match name { + "amp" => return Some('&'), + "lt" => return Some('<'), + "gt" => return Some('>'), + "quot" => return Some('"'), + "apos" => return Some('\''), + _ => name.strip_prefix('#')?, + }; + let (digits, radix) = match number.strip_prefix(['x', 'X']) { + Some(hex) => (hex, 16), + None => (number, 10), + }; + if digits.is_empty() { + return None; + } + let code = digits.chars().try_fold(0u32, |n, c| n.checked_mul(radix)?.checked_add(c.to_digit(radix)?))?; + char::from_u32(code).filter(|c| is_xml_char(*c)) +} + +/// Resolves the references in `s`, and with `attr` turns literal whitespace into spaces as +/// attribute-value normalisation does. An `&` that starts no reference stays as it is. `Err` is +/// the name of an unknown entity under [`UnknownEntity::Error`]. +fn decode_refs(s: &str, attr: bool, unknown: UnknownEntity) -> Result { + let special = |c: char| c == '&' || (attr && matches!(c, '\t' | '\n' | '\r')); + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(i) = rest.find(special) { + out.push_str(rest.get(..i).unwrap_or_default()); + let tail = rest.get(i..).unwrap_or_default(); + let Some(after) = tail.strip_prefix('&') else { + out.push(' '); + rest = tail.strip_prefix("\r\n").or_else(|| tail.get(1..)).unwrap_or_default(); + continue; + }; + // Only the bytes a name or a number can have are scanned, so a run of `&` costs one + // look each however far away the next `;` is. + let len = after.bytes().take_while(|b| b.is_ascii_alphanumeric() || matches!(b, b'#' | b'_' | b'-' | b'.' | b':')).count(); + let name = after.get(..len).unwrap_or_default(); + if len == 0 || after.as_bytes().get(len) != Some(&b';') { + out.push('&'); + rest = after; + continue; + } + match (entity(name), unknown) { + (Some(c), _) => out.push(c), + (None, UnknownEntity::Drop) => {} + (None, UnknownEntity::Literal) => { + out.push('&'); + out.push_str(name); + out.push(';'); + } + (None, UnknownEntity::Error) => return Err(name.to_string()), + } + rest = after.get(len + 1..).unwrap_or_default(); + } + out.push_str(rest); + Ok(out) +} + +/// `s` with its predefined and numeric references resolved ([`entity`]). Anything else that +/// starts with `&` stays as it is. +pub fn unescape(s: &str) -> String { + decode_refs(s, false, UnknownEntity::Literal).unwrap_or_else(|_| s.to_string()) +} diff --git a/crates/xml/src/tree.rs b/crates/xml/src/tree.rs new file mode 100644 index 000000000..89dfabfee --- /dev/null +++ b/crates/xml/src/tree.rs @@ -0,0 +1,363 @@ +//! The owned element tree. + +use std::borrow::Cow; + +use crate::value; + +/// An element: its name, its attributes in document order and its children in document order. +/// +/// What `name` and the attribute names hold depends on [`crate::Names`]: the qualified name as +/// written (`p:sp`), the local name (`sp`), or a canonical prefix and the local name (`w:p`). +/// +/// A tree the reader built is never deeper than [`crate::Limits::max_depth`]. A tree built by +/// hand is as deep as its builder made it; the methods here do not recurse, but dropping a tree +/// does, once per level. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct El { + pub name: String, + /// Attributes in document order. A name the document repeats is kept each time. + pub attrs: Vec<(String, String)>, + pub children: Vec, +} + +/// A child of an element. +#[derive(Clone, Debug, PartialEq)] +pub enum Node { + El(El), + /// Character data, references resolved. Adjacent stretches are one node. + Text(String), + /// A CDATA section. The reader makes these only with [`crate::Options::cdata`]; otherwise a + /// section's content is part of the surrounding [`Node::Text`]. + CData(String), + /// A processing instruction (``), kept only with [`crate::Options::pis`]. + Pi { + target: String, + data: String, + }, + /// A comment, kept only with [`crate::Options::comments`]. + Comment(String), +} + +impl From for Node { + fn from(e: El) -> Node { + Node::El(e) + } +} + +impl From<&str> for Node { + fn from(t: &str) -> Node { + Node::Text(t.to_string()) + } +} + +impl From for Node { + fn from(t: String) -> Node { + Node::Text(t) + } +} + +/// What a name is matched against: a `&str` is the whole name, [`Local`] the part after the +/// prefix. +pub trait Pat: Copy { + fn matches(self, name: &str) -> bool; +} + +impl Pat for &str { + fn matches(self, name: &str) -> bool { + self == name + } +} + +impl Pat for &String { + fn matches(self, name: &str) -> bool { + self == name + } +} + +/// Matches a name by its local part, whatever its prefix: `Local("sp")` matches `sp` and `p:sp`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Local<'a>(pub &'a str); + +impl Pat for Local<'_> { + fn matches(self, name: &str) -> bool { + local_of(name) == self.0 + } +} + +pub(crate) fn local_of(name: &str) -> &str { + name.rsplit_once(':').map_or(name, |(_, l)| l) +} + +fn is_ns_decl(name: &str) -> bool { + name == "xmlns" || name.starts_with("xmlns:") +} + +impl El { + /// An element with no attributes and no children. + pub fn new(name: impl Into) -> El { + El { name: name.into(), attrs: Vec::new(), children: Vec::new() } + } + + // ---------------------------------------------------------------- building and changing + + /// [`El::set_attr`], for chaining. + #[must_use] + pub fn with_attr(mut self, name: &str, value: impl ToString) -> El { + self.set_attr(name, value); + self + } + /// Appends a child element. + #[must_use] + pub fn with_child(mut self, child: El) -> El { + self.children.push(Node::El(child)); + self + } + /// Appends text. + #[must_use] + pub fn with_text(mut self, text: impl Into) -> El { + self.children.push(Node::Text(text.into())); + self + } + /// Appends any node (a CDATA section, a processing instruction…). + #[must_use] + pub fn with_node(mut self, node: impl Into) -> El { + self.children.push(node.into()); + self + } + /// Sets the attribute named exactly `name`, replacing its first occurrence or appending it. + pub fn set_attr(&mut self, name: &str, value: impl ToString) { + let value = value.to_string(); + match self.attrs.iter_mut().find(|(k, _)| k == name) { + Some(slot) => slot.1 = value, + None => self.attrs.push((name.to_string(), value)), + } + } + /// Removes every attribute named exactly `name`; the value of the first, if there was one. + pub fn remove_attr(&mut self, name: &str) -> Option { + let first = self.attrs.iter().position(|(k, _)| k == name)?; + let (_, value) = self.attrs.remove(first); + self.attrs.retain(|(k, _)| k != name); + Some(value) + } + /// Appends a child node. + pub fn push(&mut self, node: impl Into) { + self.children.push(node.into()); + } + /// Child elements, for changing them. + pub fn elements_mut(&mut self) -> impl Iterator { + self.children.iter_mut().filter_map(|n| match n { + Node::El(e) => Some(e), + _ => None, + }) + } + /// The first child element that matches, for changing it. + pub fn child_mut(&mut self, pat: impl Pat) -> Option<&mut El> { + self.elements_mut().find(|e| pat.matches(&e.name)) + } + + // ---------------------------------------------------------------- names + + /// The name without its prefix. + pub fn local(&self) -> &str { + local_of(&self.name) + } + /// The prefix, or `""`. + pub fn prefix(&self) -> &str { + self.name.split_once(':').map_or("", |(p, _)| p) + } + /// Does the name match? + pub fn is(&self, pat: impl Pat) -> bool { + pat.matches(&self.name) + } + /// The namespace declarations on this element as (prefix, URI); the default namespace has + /// the prefix `""`. They are attributes, so the reader must have kept them + /// ([`crate::Options::ns_decls`]). + pub fn ns_decls(&self) -> impl Iterator { + self.attrs.iter().filter(|(k, _)| is_ns_decl(k)).map(|(k, v)| (k.strip_prefix("xmlns:").unwrap_or(""), v.as_str())) + } + + // ---------------------------------------------------------------- attributes + + /// The first attribute named exactly `name`. + pub fn attr(&self, name: &str) -> Option<&str> { + self.attrs.iter().find(|(k, _)| k == name).map(|(_, v)| v.as_str()) + } + /// [`El::attr`]; failing that, for a prefixed `name`, the unprefixed attribute with the same + /// local name. Some producers leave the prefix out (`val` for `w:val`). + pub fn attr_or_plain(&self, name: &str) -> Option<&str> { + self.attr(name).or_else(|| { + let local = local_of(name); + if local == name { None } else { self.attr(local) } + }) + } + /// An attribute by local name, for names kept as written. An unprefixed `name` is the + /// attribute of exactly that name, else the first one with that local name under any + /// prefix. A prefixed `name` (`r:id`) is the first *prefixed* attribute with that local + /// name, whatever its prefix. Namespace declarations never match. + pub fn attr_local(&self, name: &str) -> Option<&str> { + let by_local = |local: &str, prefixed_only: bool| { + self.attrs.iter().find(|(k, _)| !is_ns_decl(k) && local_of(k) == local && (!prefixed_only || k.contains(':'))).map(|(_, v)| v.as_str()) + }; + match name.split_once(':') { + Some((_, local)) => by_local(local, true), + None => self.attr(name).or_else(|| by_local(name, false)), + } + } + pub fn has_attr(&self, name: &str) -> bool { + self.attr(name).is_some() + } + /// [`El::attr`] as a number: see [`value::to_u32`]. + pub fn attr_u32(&self, name: &str) -> Option { + self.attr(name).and_then(value::to_u32) + } + /// See [`value::to_i32`]. + pub fn attr_i32(&self, name: &str) -> Option { + self.attr(name).and_then(value::to_i32) + } + /// See [`value::to_i64`]. + pub fn attr_i64(&self, name: &str) -> Option { + self.attr(name).and_then(value::to_i64) + } + /// See [`value::to_f64`]: finite values only. + pub fn attr_f64(&self, name: &str) -> Option { + self.attr(name).and_then(value::to_f64) + } + /// See [`value::to_bool`]. + pub fn attr_bool(&self, name: &str) -> Option { + self.attr(name).and_then(value::to_bool) + } + /// [`El::attr_bool`], or `default` when the attribute is missing or not a boolean. + pub fn flag(&self, name: &str, default: bool) -> bool { + self.attr_bool(name).unwrap_or(default) + } + + // ---------------------------------------------------------------- children + + /// Child elements in document order. + pub fn elements(&self) -> impl Iterator { + self.children.iter().filter_map(|n| match n { + Node::El(e) => Some(e), + _ => None, + }) + } + /// The first child element that matches. + pub fn child(&self, pat: impl Pat) -> Option<&El> { + self.elements().find(|e| pat.matches(&e.name)) + } + /// Every child element that matches. + pub fn children<'a, P: Pat + 'a>(&'a self, pat: P) -> impl Iterator + 'a { + self.elements().filter(move |e| pat.matches(&e.name)) + } + /// Follows a chain of child names, taking the first match at each step. + pub fn path(&self, names: &[P]) -> Option<&El> { + names.iter().try_fold(self, |e, n| e.child(*n)) + } + /// An attribute of the first matching child (``). + pub fn child_attr(&self, pat: impl Pat, attr: &str) -> Option<&str> { + self.child(pat).and_then(|c| c.attr(attr)) + } + /// [`El::text`] of the first matching child. + pub fn child_text(&self, pat: impl Pat) -> Option> { + self.child(pat).map(El::text) + } + /// Every descendant element, depth first in document order, without this element. + pub fn descendants(&self) -> Descendants<'_> { + Descendants { stack: vec![self.children.iter()] } + } + /// The first descendant that matches, depth first. + pub fn find(&self, pat: impl Pat) -> Option<&El> { + self.descendants().find(|e| pat.matches(&e.name)) + } + /// Every descendant that matches, depth first. A match inside a match is found too. + pub fn find_all<'a, P: Pat + 'a>(&'a self, pat: P) -> impl Iterator + 'a { + self.descendants().filter(move |e| pat.matches(&e.name)) + } + + // ---------------------------------------------------------------- text + + /// This element's own character data: its text and CDATA children joined, not its + /// descendants'. Borrowed when there is at most one such child. + pub fn text(&self) -> Cow<'_, str> { + let mut parts = self.children.iter().filter_map(|n| match n { + Node::Text(t) | Node::CData(t) => Some(t.as_str()), + _ => None, + }); + let first = parts.next().unwrap_or(""); + match parts.next() { + None => Cow::Borrowed(first), + Some(second) => { + let mut s = String::from(first); + s.push_str(second); + parts.for_each(|t| s.push_str(t)); + Cow::Owned(s) + } + } + } + /// The character data of this element and every descendant, in document order. + pub fn deep_text(&self) -> String { + let mut s = String::new(); + walk(self, |step| { + if let Step::Leaf(Node::Text(t) | Node::CData(t)) = step { + s.push_str(t); + } + true + }); + s + } +} + +/// See [`El::descendants`]. +#[derive(Clone, Debug)] +pub struct Descendants<'a> { + stack: Vec>, +} + +impl<'a> Iterator for Descendants<'a> { + type Item = &'a El; + + fn next(&mut self) -> Option<&'a El> { + loop { + match self.stack.last_mut()?.next() { + Some(Node::El(e)) => { + self.stack.push(e.children.iter()); + return Some(e); + } + Some(_) => {} + None => { + self.stack.pop(); + } + } + } + } +} + +pub(crate) enum Step<'a> { + Enter(&'a El), + Leave(&'a El), + Leaf(&'a Node), +} + +/// Walks `root` in document order without recursion. `f` returns whether to go into the element +/// it was just given; an element it declines gets no [`Step::Leave`]. +pub(crate) fn walk<'a>(root: &'a El, mut f: impl FnMut(Step<'a>) -> bool) { + if !f(Step::Enter(root)) { + return; + } + let mut stack = vec![(root, root.children.iter())]; + while let Some((el, rest)) = stack.last_mut() { + match rest.next() { + Some(Node::El(child)) => { + if f(Step::Enter(child)) { + stack.push((child, child.children.iter())); + } + } + Some(leaf) => { + f(Step::Leaf(leaf)); + } + None => { + f(Step::Leave(el)); + stack.pop(); + } + } + } +} diff --git a/crates/xml/src/value.rs b/crates/xml/src/value.rs new file mode 100644 index 000000000..6219c80df --- /dev/null +++ b/crates/xml/src/value.rs @@ -0,0 +1,79 @@ +//! Attribute and text values as numbers and booleans. Every function trims surrounding +//! whitespace and returns `None` for anything it does not read. + +/// A decimal `u32`. Signs other than `+`, fractions and out-of-range values are `None`. +pub fn to_u32(s: &str) -> Option { + s.trim().parse().ok() +} + +/// A decimal `i32`. +pub fn to_i32(s: &str) -> Option { + s.trim().parse().ok() +} + +/// A decimal `i64`. +pub fn to_i64(s: &str) -> Option { + s.trim().parse().ok() +} + +/// A finite `f64`. `NaN` and the infinities are `None`. +pub fn to_f64(s: &str) -> Option { + s.trim().parse::().ok().filter(|v| v.is_finite()) +} + +/// A boolean in any spelling the office formats use: `1`, `true`, `on`, `t` and `0`, `false`, +/// `off`, `f`. Case matters, as in `xsd:boolean`. +pub fn to_bool(s: &str) -> Option { + match s.trim() { + "1" | "true" | "on" | "t" => Some(true), + "0" | "false" | "off" | "f" => Some(false), + _ => None, + } +} + +/// An integer that may be written as a decimal fraction or lie out of range: a finite fraction +/// is truncated toward zero and anything beyond ±9.0e18 is clamped. +pub fn to_i64_lossy(s: &str) -> Option { + to_i64(s).or_else(|| to_f64(s).map(|f| f.clamp(-9.0e18, 9.0e18) as i64)) +} + +/// [`to_i64_lossy`], clamped to `i32`. +pub fn to_i32_lossy(s: &str) -> Option { + to_i64_lossy(s).map(|v| v.clamp(i64::from(i32::MIN), i64::from(i32::MAX)) as i32) +} + +/// [`to_i64_lossy`], clamped to `u32` (a negative value reads as 0). +pub fn to_u32_lossy(s: &str) -> Option { + to_i64_lossy(s).map(|v| v.clamp(0, i64::from(u32::MAX)) as u32) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strict_and_lossy_numbers() { + assert_eq!(to_u32(" 42 "), Some(42)); + assert_eq!((to_u32("-1"), to_u32("1.5"), to_u32("4294967296"), to_u32("")), (None, None, None, None)); + assert_eq!((to_i32("-7"), to_i64("-9000000000")), (Some(-7), Some(-9_000_000_000))); + assert_eq!((to_f64("2.5"), to_f64("NaN"), to_f64("inf"), to_f64("-inf"), to_f64("x")), (Some(2.5), None, None, None, None)); + assert_eq!( + (to_i64_lossy("12.9"), to_i64_lossy("-12.9"), to_i64_lossy("1e30"), to_i64_lossy("NaN")), + (Some(12), Some(-12), Some(9_000_000_000_000_000_000), None) + ); + assert_eq!((to_i32_lossy("99999999999"), to_u32_lossy("-5"), to_u32_lossy("99999999999")), (Some(i32::MAX), Some(0), Some(u32::MAX))); + } + + #[test] + fn boolean_spellings() { + for t in ["1", "true", "on", "t", " true "] { + assert_eq!(to_bool(t), Some(true), "{t}"); + } + for f in ["0", "false", "off", "f"] { + assert_eq!(to_bool(f), Some(false), "{f}"); + } + for other in ["", "TRUE", "yes", "2"] { + assert_eq!(to_bool(other), None, "{other}"); + } + } +} diff --git a/crates/xml/src/write.rs b/crates/xml/src/write.rs new file mode 100644 index 000000000..3c7a52b33 --- /dev/null +++ b/crates/xml/src/write.rs @@ -0,0 +1,458 @@ +//! Writing: escaping, serialising a tree, and a writer for documents built without a tree. + +use std::fmt::{self, Display, Write as _}; + +use crate::tree::{El, Node, Step, walk}; + +/// The XML declaration, with a newline. +pub const DECL: &str = "\n"; +/// The XML declaration the office formats use (`standalone="yes"`), with a newline. +pub const DECL_STANDALONE: &str = "\n"; + +/// May `c` appear in an XML 1.0 document? Control characters other than tab, LF and CR, and +/// U+FFFE and U+FFFF, may not, even as character references. +pub fn is_xml_char(c: char) -> bool { + matches!(c, '\t' | '\n' | '\r' | '\u{20}'..='\u{D7FF}' | '\u{E000}'..='\u{FFFD}' | '\u{10000}'..='\u{10FFFF}') +} + +fn is_name_start(c: char) -> bool { + matches!(c, + ':' | '_' | 'A'..='Z' | 'a'..='z' + | '\u{C0}'..='\u{D6}' | '\u{D8}'..='\u{F6}' | '\u{F8}'..='\u{2FF}' | '\u{370}'..='\u{37D}' | '\u{37F}'..='\u{1FFF}' + | '\u{200C}'..='\u{200D}' | '\u{2070}'..='\u{218F}' | '\u{2C00}'..='\u{2FEF}' | '\u{3001}'..='\u{D7FF}' + | '\u{F900}'..='\u{FDCF}' | '\u{FDF0}'..='\u{FFFD}' | '\u{10000}'..='\u{EFFFF}') +} + +/// Is `s` an XML 1.0 name (an element, attribute or processing-instruction target name)? +pub fn is_name(s: &str) -> bool { + let mut chars = s.chars(); + chars.next().is_some_and(is_name_start) + && chars.all(|c| is_name_start(c) || matches!(c, '-' | '.' | '0'..='9' | '\u{B7}' | '\u{300}'..='\u{36F}' | '\u{203F}'..='\u{2040}')) +} + +/// `s` as element content: `&`, `<` and `>` escaped, a carriage return written as ` ` (a +/// literal one would be read back as a line feed), characters XML cannot carry dropped. +pub fn esc(s: &str) -> String { + let mut o = String::with_capacity(s.len()); + push_esc(&mut o, s); + o +} + +/// Appends [`esc`]`(s)` to `o`. +pub fn push_esc(o: &mut String, s: &str) { + // Most text is plain ASCII with nothing to escape. + if s.bytes().all(|b| matches!(b, b'\t' | b'\n' | 0x20..=0x7E) && !matches!(b, b'&' | b'<' | b'>')) { + o.push_str(s); + return; + } + o.reserve(s.len()); + for c in s.chars() { + match c { + '&' => o.push_str("&"), + '<' => o.push_str("<"), + '>' => o.push_str(">"), + '\r' => o.push_str(" "), + c if is_xml_char(c) => o.push(c), + _ => {} + } + } +} + +/// `s` as a double-quoted attribute value: `&`, `<`, `>` and `"` escaped, tab, LF and CR written +/// as character references (literal ones would be read back as spaces), characters XML cannot +/// carry dropped. +pub fn esc_attr(s: &str) -> String { + let mut o = String::with_capacity(s.len()); + push_esc_attr(&mut o, s); + o +} + +/// Appends [`esc_attr`]`(s)` to `o`. +pub fn push_esc_attr(o: &mut String, s: &str) { + for c in s.chars() { + match c { + '&' => o.push_str("&"), + '<' => o.push_str("<"), + '>' => o.push_str(">"), + '"' => o.push_str("""), + '\t' => o.push_str(" "), + '\n' => o.push_str(" "), + '\r' => o.push_str(" "), + c if is_xml_char(c) => o.push(c), + _ => {} + } + } +} + +/// Escapes whatever is formatted into it as an attribute value. +struct AttrSink<'a>(&'a mut String); + +impl fmt::Write for AttrSink<'_> { + fn write_str(&mut self, s: &str) -> fmt::Result { + push_esc_attr(self.0, s); + Ok(()) + } +} + +fn push_attr(o: &mut String, name: &str, value: impl Display) { + o.push(' '); + o.push_str(name); + o.push_str("=\""); + let _ = write!(AttrSink(o), "{value}"); + o.push('"'); +} + +fn push_pi(o: &mut String, target: &str, data: &str) { + // A target that is not a name, the reserved target `xml`, or data that would end the + // instruction early cannot be written. + if !is_name(target) || target.eq_ignore_ascii_case("xml") || data.contains("?>") { + return; + } + o.push_str(""); +} + +fn push_cdata(o: &mut String, text: &str) { + o.push_str("` cannot appear in a section: end the section inside it and start another. + for (i, part) in text.split("]]>").enumerate() { + if i > 0 { + o.push_str("]]]]>"); + } + o.extend(part.chars().filter(|c| is_xml_char(*c))); + } + o.push_str("]]>"); +} + +fn push_leaf(o: &mut String, node: &Node) { + match node { + Node::El(_) => {} + Node::Text(t) => push_esc(o, t), + Node::CData(t) => push_cdata(o, t), + Node::Pi { target, data } => push_pi(o, target, data), + // `--` cannot appear in a comment and a comment cannot end with `-`. + Node::Comment(c) if c.contains("--") || c.ends_with('-') => {} + Node::Comment(c) => { + o.push_str(""); + } + } +} + +fn push_start(o: &mut String, e: &El) { + o.push('<'); + o.push_str(&e.name); + for (k, v) in e.attrs.iter().filter(|(k, _)| is_name(k)) { + push_attr(o, k, v); + } +} + +fn push_end(o: &mut String, e: &El) { + o.push_str("'); +} + +fn push_el(o: &mut String, root: &El) { + // Where each open element's content starts: one that wrote none is closed as ``, so + // that what is written reads back as the same text however much of it was left out. + let mut content = Vec::new(); + walk(root, |step| { + match step { + Step::Enter(e) => { + if !is_name(&e.name) { + return false; + } + push_start(o, e); + if e.children.is_empty() { + o.push_str("/>"); + return false; + } + o.push('>'); + content.push(o.len()); + } + Step::Leave(e) => { + if content.pop() == Some(o.len()) { + o.pop(); + o.push_str("/>"); + } else { + push_end(o, e); + } + } + Step::Leaf(n) => push_leaf(o, n), + } + true + }); +} + +fn is_blank(n: &Node) -> bool { + matches!(n, Node::Text(t) if t.chars().all(|c| matches!(c, ' ' | '\t' | '\n' | '\r'))) +} + +/// Children that are all elements (with at most blank text between them) go one per line. +fn is_block(e: &El) -> bool { + let mut any = false; + for n in &e.children { + match n { + Node::El(c) => any |= is_name(&c.name), + n if is_blank(n) => {} + _ => return false, + } + } + any +} + +fn push_pretty(o: &mut String, root: &El, indent: &str, base: usize) { + let mut depth = base; + let pad = |o: &mut String, depth: usize| (0..depth).for_each(|_| o.push_str(indent)); + walk(root, |step| { + match step { + Step::Enter(e) => { + if !is_name(&e.name) { + return false; + } + pad(o, depth); + if !is_block(e) { + push_el(o, e); + o.push('\n'); + return false; + } + push_start(o, e); + o.push_str(">\n"); + depth += 1; + } + Step::Leave(e) => { + depth = depth.saturating_sub(1); + pad(o, depth); + push_end(o, e); + o.push('\n'); + } + // Blank text between the children of a block. + Step::Leaf(_) => {} + } + true + }); +} + +impl El { + /// The element as XML on one line, with no declaration. See [`El::write_xml`]. + pub fn to_xml(&self) -> String { + let mut s = String::new(); + self.write_xml(&mut s); + s + } + + /// Appends the element as XML. Text is escaped with [`esc`] and attribute values with + /// [`esc_attr`]; an element without children is written ``. An element or attribute + /// whose name is not an XML name ([`is_name`]) is left out with everything inside it, so a + /// name can never inject markup. Reading the result back gives an equal tree, apart from + /// such names and characters XML cannot carry. + pub fn write_xml(&self, out: &mut String) { + push_el(out, self); + } + + /// The element as indented XML, ending with a newline. See [`El::write_xml_pretty`]. + pub fn to_xml_pretty(&self, indent: &str) -> String { + let mut s = String::new(); + self.write_xml_pretty(&mut s, indent); + s + } + + /// Appends the element as indented XML: an element whose children are all elements has them + /// one per line, `indent` deeper each level, and any other element is written on one line + /// exactly as [`El::write_xml`] writes it. Character data is therefore never changed, but + /// whitespace-only text between the children of an element-only element is not kept. + pub fn write_xml_pretty(&self, out: &mut String, indent: &str) { + push_pretty(out, self, indent, 0); + } +} + +/// Builds XML text without a tree. Element names are closed for the caller, attribute values and +/// text are escaped, and with [`Writer::pretty`] each element goes on its own line. +/// +/// Names are written as given: they are the caller's own constants, not checked. +/// +/// ``` +/// let mut w = craft_xml::Writer::document(); +/// w.tag("w:p").attr("w:id", 7).open(); +/// w.leaf("w:t", &[("xml:space", "preserve")], " a < b "); +/// w.empty("w:br", &[]); +/// assert_eq!( +/// w.finish(), +/// "\n a < b " +/// ); +/// ``` +#[derive(Clone, Debug, Default)] +pub struct Writer { + out: String, + /// The names of the open elements, one after another, and where each starts. + names: String, + open: Vec, + indent: Option, +} + +/// A start tag being written: add attributes, then end it with [`Tag::open`], [`Tag::empty`] or +/// [`Tag::text`]. +#[must_use = "end the tag with `open`, `empty` or `text`"] +pub struct Tag<'w> { + w: &'w mut Writer, + name: &'w str, +} + +impl Tag<'_> { + /// An attribute; `value` is formatted and escaped. + pub fn attr(self, name: &str, value: impl Display) -> Self { + push_attr(&mut self.w.out, name, value); + self + } + /// An attribute when there is a value. + pub fn attr_opt(self, name: &str, value: Option) -> Self { + match value { + Some(v) => self.attr(name, v), + None => self, + } + } + /// Several attributes. + pub fn attrs(self, attrs: &[(&str, &str)]) -> Self { + attrs.iter().fold(self, |tag, (k, v)| tag.attr(k, v)) + } + /// ``: the element stays open until [`Writer::close`]. + pub fn open(self) { + self.w.out.push('>'); + self.w.open.push(self.w.names.len()); + self.w.names.push_str(self.name); + self.w.newline(); + } + /// ``. + pub fn empty(self) { + self.w.out.push_str("/>"); + self.w.newline(); + } + /// `text`. + pub fn text(self, text: &str) { + self.w.out.push('>'); + push_esc(&mut self.w.out, text); + self.w.out.push_str("'); + self.w.newline(); + } +} + +impl Writer { + /// An empty writer, for a fragment. + pub fn new() -> Writer { + Writer::default() + } + /// A writer that starts with [`DECL_STANDALONE`]. + pub fn document() -> Writer { + Writer { out: String::from(DECL_STANDALONE), ..Writer::default() } + } + /// A writer that puts each element on its own line, `indent` deeper per open element. + /// Elements written with [`Writer::leaf`] or [`Tag::text`] stay on one line. Text written + /// with [`Writer::text`] is not indented. + pub fn pretty(indent: &str) -> Writer { + Writer { indent: Some(indent.to_string()), ..Writer::default() } + } + + fn pad(&mut self) { + if let Some(indent) = &self.indent { + (0..self.open.len()).for_each(|_| self.out.push_str(indent)); + } + } + fn newline(&mut self) { + if self.indent.is_some() { + self.out.push('\n'); + } + } + + /// Writes the XML declaration: [`DECL_STANDALONE`] or [`DECL`]. + pub fn decl(&mut self, standalone: bool) { + self.out.push_str(if standalone { DECL_STANDALONE } else { DECL }); + } + /// Writes `` and a newline. Nothing is written if `name` is not a name. + pub fn doctype(&mut self, name: &str) { + if is_name(name) { + let _ = writeln!(self.out, ""); + } + } + /// Writes a processing instruction, and a newline when no element is open. Nothing is + /// written if `target` is not a name or `data` holds `?>`. + pub fn pi(&mut self, target: &str, data: &str) { + let before = self.out.len(); + push_pi(&mut self.out, target, data); + if self.open.is_empty() && self.out.len() > before { + self.out.push('\n'); + } + } + + /// Starts a tag: `w.tag("a:off").attr("x", 1).attr("y", 2).empty()`. + pub fn tag<'w>(&'w mut self, name: &'w str) -> Tag<'w> { + self.pad(); + self.out.push('<'); + self.out.push_str(name); + Tag { w: self, name } + } + /// ``. + pub fn open(&mut self, name: &str, attrs: &[(&str, &str)]) { + self.tag(name).attrs(attrs).open(); + } + /// ``. + pub fn empty(&mut self, name: &str, attrs: &[(&str, &str)]) { + self.tag(name).attrs(attrs).empty(); + } + /// `text`. + pub fn leaf(&mut self, name: &str, attrs: &[(&str, &str)], text: &str) { + self.tag(name).attrs(attrs).text(text); + } + /// Closes the innermost open element. Nothing happens when none is open. + pub fn close(&mut self) { + let Some(start) = self.open.pop() else { return }; + self.pad(); + self.out.push_str("'); + self.names.truncate(start); + self.newline(); + } + /// Escaped text. + pub fn text(&mut self, text: &str) { + push_esc(&mut self.out, text); + } + /// A CDATA section. + pub fn cdata(&mut self, text: &str) { + push_cdata(&mut self.out, text); + } + /// Markup the caller has already written, as it is. + pub fn raw(&mut self, xml: &str) { + self.out.push_str(xml); + } + /// A tree: on one line, or indented to the current depth with [`Writer::pretty`]. + pub fn tree(&mut self, el: &El) { + match &self.indent { + Some(indent) => push_pretty(&mut self.out, el, indent, self.open.len()), + None => push_el(&mut self.out, el), + } + } + /// What has been written so far. + pub fn as_str(&self) -> &str { + &self.out + } + /// Closes every element still open and returns the text. + pub fn finish(mut self) -> String { + while !self.open.is_empty() { + self.close(); + } + self.out + } + /// [`Writer::finish`], as bytes. + pub fn into_bytes(self) -> Vec { + self.finish().into_bytes() + } +} diff --git a/crates/xml/tests/apps.rs b/crates/xml/tests/apps.rs new file mode 100644 index 000000000..6438cef59 --- /dev/null +++ b/crates/xml/tests/apps.rs @@ -0,0 +1,669 @@ +//! The tests the Craft apps had for their own XML helpers, against this crate: each app's +//! expectations, read the way that app reads (see the migration table in the README). + +use craft_xml::{Canonical, Doctype, El, Error, Event, Limits, Local, Node, NsName, Options, Scope, Writer, esc, esc_attr, parse, to_utf8, unescape}; + +// ------------------------------------------------------------------ WordCraft: canonical prefixes + +const W: &str = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"; +const WORD_NS: &[(&str, &str)] = &[ + ("w", W), + ("r", "http://schemas.openxmlformats.org/officeDocument/2006/relationships"), + ("wp", "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing"), + ("a", "http://schemas.openxmlformats.org/drawingml/2006/main"), + ("mc", "http://schemas.openxmlformats.org/markup-compatibility/2006"), + ("wpi", "http://schemas.microsoft.com/office/word/2010/wordprocessingInk"), + ("wps", "http://schemas.microsoft.com/office/word/2010/wordprocessingShape"), + // ISO/IEC 29500 Strict namespaces map onto the same prefixes. + ("w", "http://purl.oclc.org/ooxml/wordprocessingml/main"), + ("wp", "http://purl.oclc.org/ooxml/drawingml/wordprocessingDrawing"), + ("a", "http://purl.oclc.org/ooxml/drawingml/main"), +]; +const WORD: Canonical<'static> = Canonical::new(WORD_NS); + +fn word(bytes: &[u8]) -> Result { + Options::canonical(&WORD).with_limits(Limits { max_nodes: 8_000_000, ..Limits::DEFAULT }).parse(bytes) +} + +#[test] +fn word_parses_with_namespaces() { + let x = br#" hi A< "#; + let e = word(x).unwrap(); + assert_eq!(e.name, "w:document"); + assert!(e.attrs.is_empty(), "namespace declarations are not attributes of this tree"); + let p = e.child("w:p").unwrap(); + assert_eq!(p.attr("r:id"), Some("r1")); + assert_eq!(p.attr("w:val"), Some("a&b")); + let t = p.child("w:t").unwrap(); + assert_eq!(t.text(), " hi A< "); + assert_eq!(t.attr("xml:space"), Some("preserve")); + assert_eq!((t.local(), t.prefix()), ("t", "w")); + assert_eq!(e.child_attr("w:p", "w:val"), Some("a&b")); +} + +#[test] +fn word_attr_falls_back_to_the_unqualified_name() { + let x = format!(r#""#); + let p = word(x.as_bytes()).unwrap(); + let val = |name: &str| p.child(name).and_then(|c| c.attr_or_plain("w:val")); + assert_eq!((val("w:a"), val("w:b"), val("w:c")), (Some("1"), Some("2"), None)); + assert_eq!(p.child("w:b").and_then(|c| c.attr("w:val")), None, "the exact lookup does not fall back"); +} + +#[test] +fn word_unknown_namespaces_read_behind_a_question_mark_and_are_not_written() { + let x = format!( + r#"t"# + ); + let body = word(x.as_bytes()).unwrap(); + let p = body.child("w:p").unwrap(); + assert_eq!(p.attr("?:new"), Some("1")); + assert!(p.child("?:thing").is_some()); + assert!(body.child("x").is_some(), "a prefix nothing declares is no namespace"); + assert!(body.child("plain").is_some()); + assert_eq!(body.to_xml(), r#"t"#); +} + +/// WordCraft reads the one Strict WordprocessingDrawing namespace as Word 2010's several, by +/// element and parent: the element hook. +#[test] +fn word_strict_drawing_names_read_as_word_2010_names() { + const WNE: &str = "http://schemas.microsoft.com/office/word/2006/wordml"; + fn drawing(n: &NsName<'_>) -> Option<&'static str> { + match (n.prefix, n.parent, n.local) { + ("wp", _, "wsp") => Some("wps"), + ("wp", Some("wps:wsp"), "spPr" | "txbx" | "bodyPr") => Some("wps"), + ("wp", Some("wps:txbx"), "txbxContent") => Some("w"), + ("?", Some("wps:txbx"), "txbxContent") if n.uri == WNE => Some("w"), + _ => None, + } + } + let canon = Canonical { element: Some(&drawing), ..WORD }; + let x = br#" + +"#; + let root = Options::canonical(&canon).parse(x).unwrap(); + let names: Vec<&str> = std::iter::once(&root).chain(root.descendants()).map(|e| e.name.as_str()).collect(); + assert_eq!( + names.join(" "), + "w:body wp:anchor wp:docPr wps:wsp wps:spPr wps:txbx w:txbxContent w:p wps:bodyPr wps:wsp wps:txbx w:txbxContent ?:txbxContent wp:inline wp:bodyPr" + ); +} + +/// `mc:Choice/@Requires` holds prefixes, not names: WordCraft resolves them while the scope is +/// known. The attribute-value hook. +#[test] +fn word_choice_requirements_resolve_namespace_aliases_and_rebindings() { + fn requires(el: &str, attr: &str, value: &str, scope: &Scope<'_>) -> Option { + (el == "mc:Choice" && attr == "Requires").then(|| { + let prefixes = value.split_whitespace().map(|p| scope.uri(p).map_or("?", |uri| WORD.prefix(uri))); + prefixes.collect::>().join(" ") + }) + } + let canon = Canonical { attr_value: Some(&requires), ..WORD }; + let x = br#""#; + let e = Options::canonical(&canon).parse(x).unwrap(); + let values: Vec<_> = e.children("mc:Choice").map(|c| c.attr("Requires")).collect(); + assert_eq!(values, [Some("wpi"), Some("?"), Some("?"), Some("wpi mc")]); +} + +#[test] +fn word_strict_parsing_rejects_a_dropped_entity_and_trailing_markup() { + let strict = Options::canonical(&WORD).strict(); + let bad = [&b"before &unknown; after"[..], b"", b"", b"", b"", b"", b"text"]; + for xml in bad { + assert!(strict.parse(xml).is_err(), "{}", String::from_utf8_lossy(xml)); + } + assert!(strict.parse(b"x & y\n").is_ok()); + assert!(strict.parse(b"\n\n\n\n").is_ok()); + assert!(word(b"").is_ok(), "lenient parsing still stops at the root"); + assert_eq!(word(b"before &unknown; after").unwrap().text(), "before after", "and drops an undefined entity"); +} + +#[test] +fn word_utf16_parts_parse_and_odd_ones_never_panic() { + let src = r#""#; + for le in [true, false] { + let mut b = if le { vec![0xFF, 0xFE] } else { vec![0xFE, 0xFF] }; + b.extend(src.encode_utf16().flat_map(|u| if le { u.to_le_bytes() } else { u.to_be_bytes() })); + let root = word(&b).unwrap(); + assert_eq!(root.elements().next().and_then(|e| e.attr("Target")), Some("café😀.xml"), "le {le}"); + } + // Odd length, unpaired surrogates, a bare BOM: an error or a tree, never a panic. + for b in [&[0xFF, 0xFE][..], &[0xFE, 0xFF, 0x00], &[0xFF, 0xFE, b'<', 0, 0x00, 0xD8, b'a', 0, b'/', 0, b'>', 0], &[0xFF, 0xFE, 0x00, 0xDC, 0x00]] + { + let _ = word(b); + } + assert_eq!(to_utf8(&[0xFF, 0xFE, b'a', 0, 0x00, 0xD8, b'b']).as_ref(), "a\u{FFFD}".as_bytes()); + assert_eq!(to_utf8(b"").as_ref(), b""); +} + +/// XML 1.0 §2.11: CR LF and a lone CR in text and CDATA are read as LF; ` ` stays a CR. +#[test] +fn word_line_ends_are_normalised_but_a_cr_reference_stays() { + let x = format!("a\r\nb\rc d"); + assert_eq!(word(x.as_bytes()).unwrap().child("w:t").unwrap().text(), "a\nb\nc\rde\nf"); + // The writer escapes a CR, so a CR in the text survives a save. + let x = format!("{}", esc("a\r\nb\rc")); + assert_eq!(word(x.as_bytes()).unwrap().child("w:t").unwrap().text(), "a\r\nb\rc"); + // CDATA that isn't UTF-8 is decoded lossily, and its line ends are still normalised. + let mut x = format!(""); + assert_eq!(word(&x).unwrap().child("w:t").unwrap().text(), "a\u{FFFD}\nb\nc"); +} + +#[test] +fn word_deep_text_and_find() { + let x = format!(r#"abcd"#); + let p = word(x.as_bytes()).unwrap(); + assert_eq!(p.text(), "ad"); + assert_eq!(p.deep_text(), "abcd"); + assert_eq!(p.find("w:t").map(|t| t.text().into_owned()).as_deref(), Some("b")); + assert_eq!(p.find_all("w:t").count(), 2); + assert!(p.find("w:p").is_none(), "the element itself is not its own descendant"); +} + +#[test] +fn word_hostile_parts_are_errors() { + assert_eq!(word("".repeat(1000).as_bytes()), Err(Error::TooDeep(256))); + // Thousands of `xmlns:` declarations on one tag are refused, not bound (RUSTSEC-2026-0195). + let decls: String = (0..10_000).map(|i| format!(" xmlns:a{i}=\"urn:a{i}\"")).collect(); + assert_eq!(word(format!("").as_bytes()), Err(Error::TooManyAttrs(1024))); + // One start tag with 200,000 attributes, distinct or duplicate, is refused (wordcraft#3702). + let distinct: String = (0..200_000).map(|i| format!(" a{i}=\"\"")).collect(); + let duplicate = " a=\"\"".repeat(200_000); + for attrs in [distinct, duplicate] { + assert_eq!(word(format!("").as_bytes()), Err(Error::TooManyAttrs(1024))); + } + // Far more than any real producer writes still reads. + let many: String = (0..256).map(|i| format!(" a{i}=\"{i}\"")).collect(); + let p = word(format!("").as_bytes()).unwrap(); + assert_eq!((p.attrs.len(), p.attr("a255")), (256, Some("255"))); + assert!(word(b"").is_err()); + assert!(word(b"\xff\xfe<").is_err()); + let _ = word(b""); + assert_eq!(word(b"").unwrap().attrs.len(), 2, "a repeated attribute is kept each time"); +} + +#[test] +fn word_writer() { + assert_eq!(esc_attr("a"); + w.text("tail\r"); + w.close(); + let expect = "\nx & ytail "; + assert_eq!(String::from_utf8(w.into_bytes()).unwrap(), expect); +} + +// ------------------------------------------------------------------ GridCraft: local names, streaming + +fn grid(bytes: &[u8]) -> Result { + Options { unknown_entity: craft_xml::UnknownEntity::Literal, ..Options::local() }.parse(bytes) +} + +#[test] +fn grid_parse_basic() { + let x = br#"t < uA]]>"#; + let e = grid(x).unwrap(); + assert_eq!(e.name, "root"); + assert_eq!(e.attr("k"), Some("v & w")); + assert_eq!(e.attr("id"), Some("rId1"), "attributes are named by their local name"); + assert_eq!(e.attrs.len(), 2, "namespace declarations carry no data"); + assert_eq!(e.child("b").unwrap().text(), "t < uA"); + assert!(e.child("c").is_some()); + assert_eq!(e.text(), ""); + assert_eq!(grid(b"&unknown; &").unwrap().text(), "&unknown; &", "an undefined entity stays as written"); +} + +#[test] +fn grid_typed_attributes_paths_and_values() { + let x = br#""#; + let e = grid(x).unwrap(); + assert_eq!((e.attr_u32("n"), e.attr_u32("neg"), e.attr_u32("f"), e.attr_u32("missing")), (Some(7), None, None, None)); + assert_eq!((e.attr_i64("neg"), e.attr_i32("neg"), e.attr_f64("f"), e.attr_f64("inf")), (Some(-3), Some(-3), Some(1.5), None)); + assert_eq!((e.attr_bool("on"), e.attr_bool("t"), e.attr_bool("no"), e.attr_bool("odd")), (Some(true), Some(true), Some(false), None)); + assert!(e.flag("odd", true) && !e.flag("no", true) && !e.flag("missing", false)); + assert_eq!(e.path(&["a", "b", "c"]).and_then(|c| c.attr("val")), Some("deep")); + assert!(e.path(&["a", "c"]).is_none()); + assert_eq!(e.child_attr("v", "val"), Some("top")); + assert_eq!(e.children("v").count(), 1); +} + +#[test] +fn grid_streaming_split() { + let x = b""; + let mut seen = Vec::new(); + let root = Options::local() + .parse_streaming(x, &["row"], |e| { + assert_eq!(e.name, "row"); + seen.push(e.attr_u32("n")); + Ok::<(), Error>(()) + }) + .unwrap(); + assert_eq!(seen, [Some(1), Some(2), Some(3)], "children and grandchildren of the root are handed out"); + assert_eq!(root.elements().count(), 2); + assert_eq!(root.find("row").and_then(|r| r.attr("n")), Some("4"), "a deeper element of the same name stays in the tree"); +} + +#[test] +fn grid_a_streaming_error_stops_the_read() { + #[derive(Debug, PartialEq)] + enum AppError { + Xml(Error), + Row(u32), + } + impl From for AppError { + fn from(e: Error) -> Self { + AppError::Xml(e) + } + } + let mut rows = 0; + let read = |xml: &[u8], rows: &mut u32| { + Options::local().parse_streaming(xml, &["row"], |_| { + *rows += 1; + if *rows == 2 { Err(AppError::Row(2)) } else { Ok(()) } + }) + }; + assert_eq!(read(b"", &mut rows), Err(AppError::Row(2))); + assert_eq!(rows, 2); + assert_eq!(read(b"", &mut rows), Err(AppError::Xml(Error::NoRoot))); +} + +#[test] +fn grid_a_streamed_part_may_hold_more_elements_than_the_cap() { + // 501 rows of 10 cells is 10,020 `c` and `v` elements, more than this cap, but the rows are + // streamed out one by one and never pile up in the tree. Kept in a tree, the part is refused. + let opts = Options::local().with_limits(Limits { max_nodes: 10_000, ..Limits::DEFAULT }); + let row = format!("{}", "1".repeat(10)); + let xml = format!("{}", row.repeat(501)); + let mut rows = 0usize; + let root = opts + .parse_streaming(xml.as_bytes(), &["row"], |r| { + assert_eq!(r.elements().count(), 10); + rows += 1; + Ok::<(), Error>(()) + }) + .unwrap(); + assert_eq!(rows, 501); + assert_eq!(root.child("sheetData").map(|d| d.children.len()), Some(0)); + assert_eq!(opts.parse(xml.as_bytes()), Err(Error::TooManyNodes(10_000))); + // One row that is itself over the cap is refused while it is being read. + let huge = format!("{}", "".repeat(10_001)); + assert_eq!(opts.parse_streaming(huge.as_bytes(), &["row"], |_| Ok::<(), Error>(())), Err(Error::TooManyNodes(10_000))); +} + +#[test] +fn grid_first_start_reads_one_tag_and_nothing_after_it() { + let vml = b"
one
two
", "
".repeat(1000)); + assert!(Options::local().first_start(unclosed.as_bytes(), "hit").is_some()); +} + +#[test] +fn grid_utf16_without_a_byte_order_mark() { + let src = ""; + for le in [true, false] { + let b: Vec = src.encode_utf16().flat_map(|u| if le { u.to_le_bytes() } else { u.to_be_bytes() }).collect(); + assert_eq!(grid(&b).unwrap().attr("k"), Some("ü"), "le {le}"); + } + assert_eq!(grid(b"\xEF\xBB\xBF").unwrap().name, "a"); +} + +#[test] +fn grid_garbage_and_depth() { + assert!(grid("".repeat(10_000).as_bytes()).is_err()); + assert!(grid(b"").is_err()); + let _ = grid(b"\x00\x01\x02 not xml"); + let _ = grid(b""); + let _ = grid(b"&unknown;"); +} + +/// The appending escape writes what the character loop GridCraft checked it against writes. +#[test] +fn grid_escaping() { + assert_eq!(esc("a\rb"), "a b"); + assert_eq!(esc_attr("a\rb"), "a b"); + for t in ["plain", "", "a & > c \"q\"", "tab\tnew\nline\rcr", "\u{1}\u{8}ctl", "日本語 ü", "\u{FFFE}\u{FFFF}x", "~\u{7F}"] { + let mut reference = String::new(); + for c in t.chars() { + match c { + '&' => reference.push_str("&"), + '<' => reference.push_str("<"), + '>' => reference.push_str(">"), + '\r' => reference.push_str(" "), + c if craft_xml::is_xml_char(c) => reference.push(c), + _ => {} + } + } + let mut s = String::from("x"); + craft_xml::push_esc(&mut s, t); + assert_eq!(s, format!("x{reference}"), "{t:?}"); + assert_eq!(esc(t), reference); + } +} + +// ------------------------------------------------------------------ DeckCraft: as written, matched by local name + +#[test] +fn deck_parse_basic_and_entities() { + let root = + parse(b"A & B A <").unwrap(); + assert_eq!((root.name.as_str(), root.local(), root.prefix()), ("a:p", "p", "a")); + assert!(root.is(Local("p")) && root.is("a:p") && !root.is("p")); + assert_eq!(root.find(Local("t")).map(El::deep_text), Some("A & B A <".to_string())); + let x = root.child(Local("x")).unwrap(); + assert_eq!(x.attr_local("id"), Some("5")); + assert_eq!(x.attr_local("r:id"), Some("rId1")); + assert_eq!(x.attr_local("q:id"), Some("rId1"), "a qualified lookup takes the local name under any prefix"); + assert_eq!(root.ns_decls().collect::>(), [("a", "x")]); + assert_eq!(root.attr_local("a"), None, "a namespace declaration is not an attribute by local name"); + // Without a plain `id`, the unqualified lookup takes the prefixed one. + let only = parse(b"").unwrap(); + assert_eq!(only.attr_local("id"), Some("rId9")); + assert_eq!(only.ns_decls().collect::>(), [("", "urn:d")]); +} + +#[test] +fn deck_lossy_integers_and_booleans() { + use craft_xml::value::{to_bool, to_i32_lossy, to_i64_lossy, to_u32_lossy}; + let e = parse(b"").unwrap(); + assert_eq!(e.attr_local("x").and_then(to_i64_lossy), Some(12)); + assert_eq!(e.attr_local("y").and_then(to_u32_lossy), Some(0)); + assert_eq!(e.attr_local("big").and_then(to_i32_lossy), Some(i32::MAX)); + assert_eq!(e.attr_local("on").and_then(to_bool), Some(true)); + assert_eq!(e.attr_i64("x"), None, "the exact getter does not accept a fraction"); +} + +#[test] +fn deck_line_ends_and_carriage_returns() { + let root = parse(b"one\r\ntwo\rthree four").unwrap(); + let t: Vec = root.children(Local("t")).map(El::deep_text).collect(); + assert_eq!(t, ["one\ntwo\nthree\rfour", "x\ny"]); + let mut w = Writer::new(); + w.tag("a:t").text("one\rtwo\r\nthree"); + assert_eq!(parse(w.as_str().as_bytes()).unwrap().deep_text(), "one\rtwo\r\nthree"); +} + +#[test] +fn deck_truncated_and_garbage() { + let kept = parse(b"text").unwrap(); + assert_eq!(kept.path(&["b", "c"]).map(|c| c.text().into_owned()).as_deref(), Some("text"), "a truncated document keeps what it had"); + assert!(parse(b"").is_err()); + assert!(parse("".repeat(256 + 10).as_bytes()).is_err()); +} + +#[test] +fn deck_escape_drops_control_chars() { + assert_eq!(esc("a\u{b}b<\u{0}"), "ab<"); + assert_eq!(esc_attr("\"x\"\n"), ""x" "); +} + +#[test] +fn deck_roundtrip_and_mutation() { + let mut root = parse(b"x & y").unwrap(); + assert_eq!(root.to_xml(), "x & y"); + root.set_attr("a", "2"); + root.set_attr("b", 3); + for e in root.elements_mut() { + e.set_attr("seen", true); + } + if let Some(e) = root.child_mut(Local("e")) { + e.push("text"); + e.push(El::new("f")); + } + assert_eq!(root.remove_attr("xmlns:p").as_deref(), Some("urn:p")); + assert_eq!(root.remove_attr("xmlns:p"), None); + assert!(root.has_attr("b") && !root.has_attr("c")); + assert_eq!(root.to_xml(), "x & ytext"); +} + +#[test] +fn deck_writer_attributes() { + let name: Option<&str> = None; + let mut w = Writer::new(); + w.tag("p:sp").attr("id", 5).attr_opt("name", name).attr_opt("bold", Some(u8::from(true))).attr("t", "a\"b").open(); + w.tag("a:off").attr("x", -1.5).attr("y", 2).empty(); + w.tag("a:t").text("hi"); + assert_eq!(w.finish(), "hi"); +} + +// ------------------------------------------------------------------ DesignCraft: builder, PIs, pretty documents + +fn design() -> Options<'static> { + Options { strict: true, pis: true, ..Options::default() } +} + +/// designcraft#712: nesting past the budget is an error from the reader, not a stack overflow. +#[test] +fn design_nesting_past_the_budget_is_refused() { + let nested = |n: usize| format!("{}{}", "".repeat(n), "".repeat(n)).into_bytes(); + assert!(design().parse(&nested(256)).is_ok(), "the budget itself parses"); + assert_eq!(design().parse(&nested(257)), Err(Error::TooDeep(256))); + assert!(design().parse(&nested(100_000)).is_err()); + // An empty element is a level too. + let leaf = |n: usize| format!("{}{}", "".repeat(n), "".repeat(n)).into_bytes(); + assert!(design().parse(&leaf(255)).is_ok()); + assert_eq!(design().parse(&leaf(256)), Err(Error::TooDeep(256))); +} + +#[test] +fn design_roundtrip_text_pi_and_entities() { + let page = El::new("Content").with_text("Page ").with_node(Node::Pi { target: "ACE".into(), data: "18".into() }); + let root = El::new("Root").with_attr("A", "x\"<&\t").with_attr("N", 9).with_child(El::new("Content").with_text("a\tb & ")).with_child(page); + let mut w = Writer::pretty("\t"); + w.decl(true); + w.pi("aid", "style=\"50\" type=\"document\""); + w.tree(&root); + let text = w.finish(); + assert_eq!( + text, + "\n\n\n\ta\tb & <c>\n\tPage \n\n" + ); + let back = design().parse(text.as_bytes()).unwrap(); + assert_eq!(back.attr("A"), Some("x\"<&\t")); + let contents: Vec<&El> = back.children(Local("Content")).collect(); + assert_eq!(contents[0].deep_text(), "a\tb & "); + assert_eq!(contents[1].children[1], Node::Pi { target: "ACE".into(), data: "18".into() }); + assert_eq!(Options { drop_blank_text: true, ..design() }.parse(text.as_bytes()).unwrap(), root); +} + +#[test] +fn design_unbalanced_documents_are_errors() { + for bad in [&b""[..], b"", b"", b"", b""] { + assert!(design().parse(bad).is_err(), "{}", String::from_utf8_lossy(bad)); + } + assert_eq!(design().parse(b"x y").unwrap().text(), "xy", "an undefined entity is dropped"); +} + +// ------------------------------------------------------------------ AniCraft: strict, indentation dropped, mixed content + +fn ani() -> Options<'static> { + Options { strict: true, drop_blank_text: true, unknown_entity: craft_xml::UnknownEntity::Literal, ..Options::default() } +} + +#[test] +fn ani_elements_attributes_and_entities() { + let x = br#" + + + + ]]> + text is kept +"#; + let root = ani().parse(x).unwrap(); + assert_eq!(root.name, "DOMDocument"); + assert_eq!((root.attr_f64("width"), root.attr_u32("frameRate")), (Some(640.0), Some(30))); + let tl = root.child("timelines").and_then(|t| t.elements().next()).unwrap(); + assert_eq!(tl.attr("name"), Some("Scene & \"1\" AB")); + assert_eq!(tl.child("layers").map(|l| l.children.len()), Some(0)); + assert_eq!(root.elements().count(), 1, "CDATA and text are not elements"); + assert!(root.text().contains(" ") && root.text().contains("text is kept"), "{:?}", root.text()); +} + +#[test] +fn ani_character_data_is_kept_and_indentation_dropped() { + let x = b"\n a <b> line 2\n z\n"; + let root = ani().parse(x).unwrap(); + assert_eq!(root.text(), "", "whitespace between elements"); + assert_eq!(root.children.len(), 3); + let texts: Vec = root.elements().map(|c| c.text().into_owned()).collect(); + assert_eq!(texts, ["a \rline 2", " ", "xHello big world& more\n \n"; + let root = ani().parse(x).unwrap(); + let t = root.child("text").unwrap(); + let parts: Vec = t.children.clone(); + assert_eq!(parts, [Node::Text("Hello ".into()), Node::El(El::new("tspan").with_text("big")), Node::Text(" world& more".into())]); + assert_eq!(t.text(), "Hello world& more"); + assert_eq!(root.children.len(), 2, "indentation between elements is dropped"); + assert_eq!(root.child("g").map(|g| g.children.len()), Some(1), "an element without child elements keeps its whitespace"); +} + +#[test] +fn ani_malformed_input_is_an_error_not_a_panic() { + for bad in [&b""[..], b"<", b"", b"A").as_bytes()).unwrap(); + assert_eq!(e.attr("maker").map(str::len), Some(200_002)); + assert_eq!((e.attr("model"), e.text().as_ref()), (Some("Canon"), "A")); + assert_eq!(e.child_text("missing"), None); +} + +// ------------------------------------------------------------------ FilmCraft: pretty writer, DOCTYPE + +#[test] +fn film_writer_roundtrip() { + let mut w = Writer::pretty(" "); + w.decl(false); + w.doctype("xmeml"); + w.open("a", &[("x", "1 & 2")]); + w.leaf("b", &[], ""); + w.empty("c", &[]); + w.open("d", &[]); + w.tag("e").attr("n", 1.5).text("t"); + let s = w.finish(); + assert_eq!( + s, + "\n\n\n <hi>\n \n \n t\n \n\n" + ); + let film = Options { doctype: Doctype::RejectInternalSubset, drop_blank_text: true, ..Options::default() }; + let a = film.parse(s.as_bytes()).unwrap(); + assert_eq!(a.attr("x"), Some("1 & 2")); + assert_eq!(a.child_text("b").as_deref(), Some("")); + assert!(a.child("c").is_some()); +} + +#[test] +fn film_attr_whitespace_roundtrip() { + let mut w = Writer::pretty(" "); + w.open("a", &[("name", "x\ny\tz\r")]); + w.leaf("b", &[], "a\nb"); + let a = parse(w.finish().as_bytes()).unwrap(); + assert_eq!(a.attr("name"), Some("x\ny\tz\r")); + assert_eq!(a.child_text("b").as_deref(), Some("a\nb")); + // Literal whitespace in an attribute value is normalised to spaces, as XML says. + assert_eq!(parse(b"").unwrap().attr("v"), Some("x y z w")); +} + +#[test] +fn film_rejects_internal_dtd_entity_expansion() { + let film = Options { doctype: Doctype::RejectInternalSubset, ..Options::default() }; + assert_eq!(film.parse(b"]>&word;"), Err(Error::Doctype)); + assert!(film.parse(b"").is_ok()); + assert!(film.parse(b"").is_ok()); +} + +// ------------------------------------------------------------------ MailCraft: no DOCTYPE at all + +#[test] +fn mail_a_document_with_a_doctype_is_not_read() { + let mail = Options { doctype: Doctype::Reject, ..Options::default() }.strict(); + assert_eq!(mail.parse(b""), Err(Error::Doctype)); + assert_eq!( + mail.parse(b"]>&b;"), + Err(Error::Doctype) + ); + assert!(matches!(mail.parse(b"&b;"), Err(Error::UnknownEntity { name, .. }) if name == "b")); + assert!(mail.parse(b"/a&b").is_ok()); +} + +// ------------------------------------------------------------------ WordCraft ODT: events with byte ranges + +#[test] +fn odt_events_with_byte_ranges() { + let xml = b"a & b"; + let opts = Options::default(); + let mut events = opts.events(xml); + let mut seen = Vec::new(); + while let Some(ev) = events.next().unwrap() { + seen.push((ev, events.span())); + } + let start = |name: &str, attrs: &[(&str, &str)]| Event::Start { + name: name.into(), + attrs: attrs.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(), + }; + let end = |name: &str| Event::End { name: name.into() }; + let expect = vec![ + (start("office:text", &[]), 0..13), + (start("text:p", &[("text:style-name", "P1")]), 13..42), + (Event::Text("a ".into()), 42..44), + (Event::Text("&".into()), 44..49), + (Event::Text(" b".into()), 49..51), + (start("text:s", &[]), 51..60), + (end("text:s"), 51..60), + (end("text:p"), 60..69), + (end("office:text"), 69..83), + ]; + assert_eq!(seen, expect); + assert_eq!((&xml[51..60], xml.len()), (&b""[..], 83)); + // Malformed markup ends the stream with an error; what came before it was delivered. + let mut events = opts.events(b"t<"); + let mut count = 0; + let err = loop { + match events.next() { + Ok(Some(_)) => count += 1, + Ok(None) => break None, + Err(e) => break Some(e), + } + }; + assert_eq!(count, 4); + assert!(matches!(err, Some(Error::Syntax { .. })), "{err:?}"); +} diff --git a/crates/xml/tests/hostile.rs b/crates/xml/tests/hostile.rs new file mode 100644 index 000000000..1cbba111b --- /dev/null +++ b/crates/xml/tests/hostile.rs @@ -0,0 +1,330 @@ +//! Hostile input: every limit, entity expansion, truncation, wrong encodings, wrong nesting. +//! Nothing here may panic, and nothing may grow beyond a small multiple of its input. + +use craft_xml::{Canonical, Doctype, El, Error, Limits, Options, UnknownEntity, parse, to_utf8}; + +const CANON: Canonical<'static> = Canonical::new(&[("w", "urn:w"), ("r", "urn:r")]); + +/// Every way of reading, lenient and strict. +fn all_options() -> Vec> { + let full = + Options { cdata: true, pis: true, comments: true, drop_blank_text: true, unknown_entity: UnknownEntity::Literal, ..Options::default() }; + let base = [Options::default(), Options::local(), Options::canonical(&CANON), full, Options { doctype: Doctype::Reject, ..Options::default() }]; + base.iter().flat_map(|o| [*o, o.strict()]).collect() +} + +fn text_len(e: &El) -> usize { + let own: usize = e.attrs.iter().map(|(k, v)| k.len() + v.len()).sum::() + e.name.len() + e.text().len(); + own + e.descendants().map(|d| d.name.len() + d.text().len() + d.attrs.iter().map(|(k, v)| k.len() + v.len()).sum::()).sum::() +} + +#[test] +fn the_defaults_are_the_documented_ones() { + assert_eq!(Limits::default(), Limits { max_input: 256 * 1024 * 1024, max_depth: 256, max_nodes: 2_000_000, max_attrs: 1024 }); + for e in [Error::TooLarge(1), Error::TooDeep(1), Error::TooManyNodes(1), Error::TooManyAttrs(1)] { + assert!(e.is_limit(), "{e}"); + } + assert!(!Error::NoRoot.is_limit() && !Error::Doctype.is_limit()); +} + +#[test] +fn a_depth_bomb_is_refused_at_the_limit() { + let limits = Limits { max_depth: 8, ..Limits::DEFAULT }; + let nested = |n: usize| format!("{}{}", "".repeat(n), "".repeat(n)); + for opts in all_options() { + let opts = opts.with_limits(limits); + assert!(opts.parse(nested(8).as_bytes()).is_ok()); + assert_eq!(opts.parse(nested(9).as_bytes()), Err(Error::TooDeep(8))); + assert_eq!(opts.parse("".repeat(9).as_bytes()), Err(Error::TooDeep(8)), "unclosed tags are depth too"); + assert_eq!(opts.parse_streaming(nested(9).as_bytes(), &["a"], |_| Ok::<(), Error>(())), Err(Error::TooDeep(8))); + } + // A megabyte of open tags against the default limit. + assert_eq!(parse("".repeat(350_000).as_bytes()), Err(Error::TooDeep(256))); + // Text and deep traversals of the deepest tree the reader builds do not recurse. + let deepest = parse(format!("{}x{}", "".repeat(256), "".repeat(256)).as_bytes()).unwrap(); + assert_eq!((deepest.deep_text().as_str(), deepest.descendants().count()), ("x", 255)); + assert_eq!(parse(deepest.to_xml().as_bytes()).unwrap(), deepest); + assert_eq!(Options { drop_blank_text: true, ..Options::default() }.parse(deepest.to_xml_pretty(" ").as_bytes()).unwrap(), deepest); +} + +#[test] +fn a_node_count_bomb_is_refused_at_the_limit() { + let limits = Limits { max_nodes: 100, ..Limits::DEFAULT }; + for opts in all_options() { + let opts = opts.with_limits(limits); + // The root and 99 children are 100 nodes. + assert!(opts.parse(format!("{}", "".repeat(99)).as_bytes()).is_ok()); + assert_eq!(opts.parse(format!("{}", "".repeat(100)).as_bytes()), Err(Error::TooManyNodes(100))); + // Text between elements is nodes too. + assert_eq!(opts.parse(format!("{}", "t".repeat(50)).as_bytes()), Err(Error::TooManyNodes(100))); + assert_eq!(opts.parse(format!("{}", "".repeat(100)).as_bytes()).err().map(|e| e.is_limit() || opts.strict), Some(true)); + } + // Kept comments, processing instructions and CDATA sections count. + let keep = Options { cdata: true, pis: true, comments: true, ..Options::default() }.with_limits(limits); + for filler in ["", "", ""] { + assert_eq!(keep.parse(format!("{}", filler.repeat(100)).as_bytes()), Err(Error::TooManyNodes(100)), "{filler}"); + assert!(Options::default().with_limits(limits).parse(format!("{}", filler.repeat(40)).as_bytes()).is_ok(), "{filler}"); + } + // Indentation that is dropped does not count against the tree that is kept. + let indented = format!("{}", "\n ".repeat(99)); + assert!( + Options { drop_blank_text: true, ..Options::default() } + .with_limits(Limits { max_nodes: 200, ..Limits::DEFAULT }) + .parse(indented.as_bytes()) + .is_ok() + ); + // The default limit, on a part of endless empty elements. + let endless = format!("{}", "".repeat(2_000_000)); + assert_eq!(parse(endless.as_bytes()), Err(Error::TooManyNodes(2_000_000))); +} + +#[test] +fn an_attribute_flood_is_refused_at_the_limit() { + let limits = Limits { max_attrs: 16, ..Limits::DEFAULT }; + let tag = |n: usize, attr: &dyn Fn(usize) -> String| format!("", (0..n).map(attr).collect::()); + let kinds: [&dyn Fn(usize) -> String; 4] = + [&|i| format!(" a{i}='v'"), &|_| " a='v'".to_string(), &|i| format!(" xmlns:p{i}='urn:{i}'"), &|i| format!(" bad{i}")]; + for opts in all_options() { + let opts = opts.with_limits(limits); + for (k, kind) in kinds.iter().enumerate() { + let _ = opts.parse(tag(16, kind).as_bytes()); + assert_eq!(opts.parse(tag(17, kind).as_bytes()).err().map(|e| e.is_limit() || (opts.strict && k == 3)), Some(true), "kind {k}"); + assert!(opts.first_start(tag(17, kind).as_bytes(), "a").is_none()); + } + assert_eq!(opts.parse(tag(17, kinds[0]).as_bytes()), Err(Error::TooManyAttrs(16))); + } + assert_eq!(parse(tag(200_000, kinds[0]).as_bytes()), Err(Error::TooManyAttrs(1024))); + // Namespace declarations spread over many levels are bounded as a whole where names are + // resolved against them: 3 levels of 6 is over 16, though no tag has more than 6. + let level = |d: usize| format!("", (0..6).map(|i| format!(" xmlns:p{d}_{i}='urn:x'")).collect::()); + let spread = |levels: usize| format!("{}", (0..levels).map(level).collect::()); + let canonical = Options::canonical(&CANON).with_limits(limits); + assert_eq!(canonical.parse(spread(2).as_bytes()).map(|r| r.name), Ok("w:r".to_string())); + assert_eq!(canonical.parse(spread(3).as_bytes()), Err(Error::TooManyAttrs(16))); + assert!(Options::default().with_limits(limits).parse(spread(3).as_bytes()).is_ok(), "names as written resolve nothing"); +} + +#[test] +fn input_over_the_size_limit_is_refused_before_it_is_read() { + let opts = Options::default().with_limits(Limits { max_input: 1000, ..Limits::DEFAULT }); + let doc = |len: usize| format!("{}", "x".repeat(len - 7)); + assert!(opts.parse(doc(1000).as_bytes()).is_ok()); + assert_eq!(opts.parse(doc(1001).as_bytes()), Err(Error::TooLarge(1000))); + assert_eq!(opts.parse_streaming(doc(1001).as_bytes(), &[], |_| Ok::<(), Error>(())), Err(Error::TooLarge(1000))); + assert!(opts.first_start(doc(1001).as_bytes(), "a").is_none()); +} + +/// The "billion laughs": entities declared in a DTD are never expanded, whatever the options. +#[test] +fn declared_entities_are_never_expanded() { + let mut dtd = String::from(""); + for i in 1..10 { + let prev = if i == 1 { "lol".to_string() } else { format!("lol{}", i - 1) }; + dtd.push_str(&format!("", format!("&{prev};").repeat(10))); + } + let bomb = format!("{dtd}]>&lol9;"); + let quadratic = format!("]>{}", "a".repeat(50_000), "&a;".repeat(50_000)); + let external = + "%pe;]>&xxe;"; + for doc in [bomb.as_str(), quadratic.as_str(), external] { + for opts in all_options() { + match opts.parse(doc.as_bytes()) { + Ok(root) => { + assert!(text_len(&root) <= doc.len(), "the tree is not larger than the document"); + let text = root.text().into_owned() + root.attr("a").unwrap_or(""); + assert!(!text.contains("lol") || text.contains("&lol9;"), "{text}"); + assert!(!text.contains("root:"), "an external entity is never fetched"); + match opts.unknown_entity { + UnknownEntity::Drop => assert_eq!(text, ""), + UnknownEntity::Literal => assert!(text.starts_with('&') && text.ends_with(';'), "{text}"), + UnknownEntity::Error => panic!("an undefined entity was accepted"), + } + } + Err(Error::Doctype) => assert_eq!(opts.doctype, Doctype::Reject), + Err(Error::UnknownEntity { .. }) => assert_eq!(opts.unknown_entity, UnknownEntity::Error), + Err(e) => panic!("unexpected error {e}"), + } + } + } + let refuse = Options { doctype: Doctype::RejectInternalSubset, ..Options::default() }; + assert_eq!(refuse.parse(bomb.as_bytes()), Err(Error::Doctype)); + assert_eq!(refuse.parse(external.as_bytes()), Err(Error::Doctype)); + // Predefined entities and character references are the only expansion, and they shrink. + let refs = format!("{0}", "&􏿿A<".repeat(10_000)); + let root = parse(refs.as_bytes()).unwrap(); + assert_eq!(root.text().chars().count(), 40_000); + assert!(text_len(&root) < refs.len()); +} + +const DOCUMENTS: &[&str] = &[ + "\n a & b AA \n", + "]>\n ]]> e]]>&e;&bogus;text\r\n", + "\n Héllo 日本語 wörld😀\n \n", + "onetwo three", +]; + +/// Every prefix of a few documents, read every way: an error or a tree, never a panic. A strict +/// read of a cut document is an error unless only what follows the root was cut. +#[test] +fn input_truncated_at_every_byte_never_panics() { + for doc in DOCUMENTS { + let bytes = doc.as_bytes(); + let whole_strict_ok = Options::default().strict().parse(bytes).is_ok(); + for cut in 0..=bytes.len() { + let part = &bytes[..cut]; + for opts in all_options() { + let tree = opts.parse(part); + let _ = opts.parse_streaming(part, &["si", "w:p", "x", "y"], |_| Ok::<(), Error>(())); + let _ = opts.first_start(part, "t"); + let mut events = opts.events(part); + while let Ok(Some(_)) = events.next() {} + if let Ok(root) = &tree { + assert!(text_len(root) <= part.len() * 3 + 16); + let _ = (root.to_xml(), root.to_xml_pretty(" "), root.deep_text()); + } + } + let strict = Options::default().strict().parse(part); + let root_ended = doc.rfind("').is_some_and(|j| cut > i + j)); + if whole_strict_ok && !root_ended { + assert!(strict.is_err(), "a strict read accepted {:?}", String::from_utf8_lossy(part)); + } + } + } +} + +/// The same documents with bytes overwritten, deleted and doubled: a cheap deterministic fuzz. +#[test] +fn damaged_input_never_panics() { + let mut seed = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + let junk = b"<>&;'\"/=?![]-\x00\xff\xfe\xc3\xed\xa0\x80 :x#"; + for doc in DOCUMENTS { + for _ in 0..600 { + let mut bytes = doc.as_bytes().to_vec(); + for _ in 0..(next() % 4 + 1) { + let at = (next() % bytes.len() as u64) as usize; + match next() % 3 { + 0 => bytes[at] = junk[(next() % junk.len() as u64) as usize], + 1 => { + bytes.remove(at); + } + _ => bytes.insert(at, bytes[at]), + } + } + for opts in all_options() { + if let Ok(root) = opts.parse(&bytes) { + assert!(text_len(&root) <= bytes.len() * 3 + 16); + // Whatever was written reads back. (A damaged name that is no XML name is + // not written at all: the reader does not check names, the writer does.) + let written = root.to_xml(); + if !written.is_empty() { + let again = Options { names: craft_xml::Names::AsWritten, strict: false, ns_decls: true, ..opts }.parse(written.as_bytes()); + assert!(again.is_ok(), "{written:?}"); + assert_eq!(again.map(|e| e.to_xml()), Ok(written)); + } + } + let _ = opts.parse_streaming(&bytes, &["si", "w:p"], |_| Ok::<(), Error>(())); + let _ = opts.first_start(&bytes, "t"); + } + } + } +} + +#[test] +fn invalid_utf8_is_replaced_or_refused() { + let cases: [&[u8]; 6] = [ + b"\xff", + b"", + b"", + b"<\xff\xfe/>", + b"", + b"", + ]; + let keep = Options { cdata: true, pis: true, comments: true, ..Options::default() }; + for bytes in cases { + for opts in [Options::default(), Options::local(), Options::canonical(&CANON), keep] { + let lenient = opts.parse(bytes).unwrap(); + assert!(std::str::from_utf8(lenient.to_xml().as_bytes()).is_ok()); + assert!(matches!(opts.strict().parse(bytes), Err(Error::Encoding(_) | Error::Syntax { .. })), "{:?}", String::from_utf8_lossy(bytes)); + } + assert!(format!("{:?}", keep.parse(bytes).unwrap()).contains('\u{FFFD}'), "each bad byte is U+FFFD where it is kept"); + } + assert_eq!(parse(b"\xff").unwrap().text(), "\u{FFFD}"); + assert_eq!(Options::default().strict().parse(b"ok\xff"), Err(Error::Encoding(5))); +} + +#[test] +fn invalid_utf16_never_panics() { + let good: Vec = "t".encode_utf16().collect(); + let le = |units: &[u16]| [0xFF, 0xFE].into_iter().chain(units.iter().flat_map(|u| u.to_le_bytes())).collect::>(); + let be = |units: &[u16]| [0xFE, 0xFF].into_iter().chain(units.iter().flat_map(|u| u.to_be_bytes())).collect::>(); + assert_eq!(parse(&le(&good)).unwrap().attr("k"), Some("v")); + assert_eq!(parse(&be(&good)).unwrap().text(), "t"); + for at in 0..good.len() { + for bad in [0xD800u16, 0xDC00, 0xFFFF, 0x0000] { + let mut units = good.clone(); + units[at] = bad; + for bytes in [le(&units), be(&units)] { + for cut in [bytes.len(), bytes.len() - 1, bytes.len() / 2] { + for opts in all_options() { + let _ = opts.parse(&bytes[..cut]); + } + } + } + } + } + // An unpaired surrogate is U+FFFD; a trailing odd byte is dropped. + assert_eq!(parse(&le(&[0x3C, 0x61, 0x3E, 0xD800, 0x3C, 0x2F, 0x61, 0x3E])).unwrap().text(), "\u{FFFD}"); + assert_eq!(to_utf8(&[0xFE, 0xFF, 0x00, 0x41, 0x00]).as_ref(), b"A"); + for empty in [&[0xFF, 0xFE][..], &[0xFE, 0xFF], &[0xEF, 0xBB, 0xBF], &[0xFF], &[]] { + assert_eq!(parse(empty), Err(Error::NoRoot)); + } +} + +#[test] +fn mismatched_tags_are_repaired_or_refused() { + let cases = ["", "", "", "", "", "", "", "", ""]; + for xml in cases { + for opts in all_options() { + let got = opts.parse(xml.as_bytes()); + if opts.strict { + assert!(matches!(got, Err(Error::Syntax { .. })), "{xml}: {got:?}"); + } else { + assert_eq!(got.map(|e| e.local().to_string()), Ok("a".to_string()), "{xml}"); + } + } + } + // A lenient end tag closes the innermost open element, whatever its name. + let root = parse(b"xy").unwrap(); + assert_eq!(root.to_xml(), "xy"); + // Nothing after the root element is read by a lenient parse, however broken. + assert_eq!(parse(b"<<<\xff&&&").unwrap().name, "a"); +} + +#[test] +fn markup_that_is_not_xml_is_an_error() { + for bytes in [ + &b""[..], + b" ", + b"text", + b"<", + b"", + ] { + for opts in all_options() { + assert!(opts.parse(bytes).is_err(), "{:?}", String::from_utf8_lossy(bytes)); + assert!(opts.first_start(bytes, "a").is_none() || bytes.starts_with(b" Options<'static> { + Options { cdata: true, pis: true, comments: true, ..Options::default() } +} + +#[test] +fn escaping_text_and_attributes() { + assert_eq!(esc("a & b < c > d \"q\" 'r'"), "a & b < c > d \"q\" 'r'"); + assert_eq!(esc("tab\tlf\ncr\r]]>"), "tab\tlf\ncr ]]>"); + assert_eq!(esc_attr("a & b < c > d \"q\" 'r'"), "a & b < c > d "q" 'r'"); + assert_eq!(esc_attr("tab\tlf\ncr\r"), "tab lf cr "); + // Characters XML 1.0 cannot carry, even as references, are dropped. + let forbidden = "a\u{0}\u{1}\u{8}\u{B}\u{C}\u{E}\u{1F}\u{FFFE}\u{FFFF}b"; + assert_eq!((esc(forbidden), esc_attr(forbidden)), ("ab".to_string(), "ab".to_string())); + assert_eq!(esc("\u{7F}\u{85}é日😀\u{FFFD}"), "\u{7F}\u{85}é日😀\u{FFFD}"); + let mut s = String::from("k="); + push_esc_attr(&mut s, "\""); + assert_eq!(s, "k=""); + assert!(is_xml_char('\t') && is_xml_char('\u{10FFFF}') && !is_xml_char('\u{0}') && !is_xml_char('\u{FFFE}')); +} + +#[test] +fn every_character_survives_a_round_trip_or_is_dropped() { + let all: String = (0..0x3000u32).chain([0xD7FF, 0xE000, 0xFFFD, 0xFFFE, 0xFFFF, 0x1_0000, 0x10_FFFF]).filter_map(char::from_u32).collect(); + let kept: String = all.chars().filter(|c| is_xml_char(*c)).collect(); + let root = El::new("a").with_attr("v", &all).with_text(&all); + let back = parse(root.to_xml().as_bytes()).unwrap(); + assert_eq!(back.attr("v"), Some(kept.as_str())); + assert_eq!(back.text(), kept); + let mut w = Writer::new(); + w.tag("a").attr("v", &all).text(&all); + assert_eq!(parse(w.finish().as_bytes()).unwrap(), back); +} + +#[test] +fn names() { + for good in ["a", "A1", "_x", "w:p", ":odd", "a-b.c", "é", "日本", "a\u{B7}"] { + assert!(is_name(good), "{good}"); + } + for bad in ["", "1a", "-a", ".a", "a b", "a>", "a=\"", "?:x", "a/", "b".into() }) + .with_node(Node::Comment("a--b".into())) + .with_node(Node::Comment("ends-".into())); + assert_eq!(root.to_xml(), "kept"); + assert_eq!(root.to_xml_pretty(" "), "kept\n"); + assert_eq!(El::new("not a name").to_xml(), ""); + // An element whose content was all left out is written the way it reads back. + let hollow = El::new("r").with_child(El::new("a").with_child(El::new("1")).with_text("\u{1}")).with_child(El::new("b").with_child(El::new("2"))); + assert_eq!(hollow.to_xml(), ""); + assert_eq!(hollow.to_xml_pretty(" "), "\n \n \n\n"); +} + +#[test] +fn compact_round_trip_keeps_what_the_tree_holds() { + let docs = [ + "", + "x & y", + " leading\n\ttrailing mixed text ", + "&<> \n\t", + " & ]]]]> end]]>t", + "deep", + "<é:日本 ü=\"ö\">😀", + ]; + for doc in docs { + let tree = full().parse(doc.as_bytes()).unwrap(); + assert_eq!(tree.to_xml(), doc, "these are already in the form the writer uses"); + assert_eq!(full().parse(tree.to_xml().as_bytes()).unwrap(), tree); + } +} + +#[test] +fn what_a_round_trip_does_not_keep() { + let doc = "\n\n\n\nd\r\n\n"; + let tree = parse(doc.as_bytes()).unwrap(); + // The declaration, the DOCTYPE and everything outside the root are not part of the tree. + // Inside it: quoting and spacing in tags, how a character was written, `` against + // ``, CDATA sections as sections, comments and processing instructions (unless the + // options keep them) and a literal CR are not kept. + assert_eq!(tree.to_xml(), "cd\n"); + let kept = full().parse(doc.as_bytes()).unwrap(); + assert_eq!(kept.to_xml(), "d\n"); +} + +#[test] +fn pretty_output() { + let tree = parse(b"textmixed content ").unwrap(); + let pretty = tree.to_xml_pretty(" "); + assert_eq!(pretty, "\n \n \n text\n mixed content\n \n \n \n\n"); + // Character data is unchanged: reading it back without the indentation gives the tree. + let back = Options { drop_blank_text: true, ..Options::default() }.parse(pretty.as_bytes()).unwrap(); + assert_eq!(back, tree); + // Printing what was printed changes nothing. + assert_eq!(parse(pretty.as_bytes()).unwrap().to_xml_pretty(" "), pretty); + assert_eq!(tree.to_xml_pretty("\t").lines().nth(2), Some("\t\t")); + // An element with a comment or an instruction among its children is written on one line. + let noted = full().parse(b"").unwrap(); + assert_eq!(noted.to_xml_pretty(" "), "\n"); +} + +#[test] +fn builder_and_accessors() { + let mut root = El::new("root").with_attr("id", 7).with_child(El::new("p:item").with_attr("n", "1").with_text("one")).with_text("tail"); + root.push(El::new("q:item").with_node(Node::CData("two".into()))); + root.push(Node::Comment("c".into())); + assert_eq!(root.to_xml(), "onetail"); + assert_eq!(root.children(Local("item")).count(), 2); + assert_eq!(root.children("p:item").count(), 1); + assert_eq!(root.child_text(Local("item")).as_deref(), Some("one")); + assert_eq!(root.child("q:item").map(|e| e.text().into_owned()).as_deref(), Some("two"), "CDATA is character data"); + assert_eq!((root.text().as_ref(), root.deep_text().as_str()), ("tail", "onetailtwo")); + assert_eq!(root.path(&[Local("item")]).and_then(|e| e.attr("n")), Some("1")); + assert_eq!(root.path::<&str>(&[]).map(|e| e.name.as_str()), Some("root")); + let name = String::from("p:item"); + assert!(root.child(&name).is_some()); +} + +#[test] +fn writer_compact() { + let mut w = Writer::new(); + w.open("a", &[("k", "v\"")]); + w.open("b", &[]); + w.text("t & u"); + w.cdata("x]]>y"); + w.close(); + w.pi("p", "d"); + w.tree(&El::new("c").with_text("<")); + w.raw(""); + assert_eq!(w.as_str(), "t & uy]]><"); + w.close(); + w.close(); + w.close(); + assert_eq!(w.finish(), "t & uy]]><"); + assert_eq!(Writer::document().into_bytes(), craft_xml::DECL_STANDALONE.as_bytes()); + let mut plain = Writer::new(); + plain.decl(false); + plain.doctype("not a name"); + plain.pi("xml", "x"); + assert_eq!(plain.finish(), craft_xml::DECL); +} + +#[test] +fn writer_closes_names_in_order_and_what_is_left_open() { + let mut w = Writer::new(); + for name in ["a", "w:body", "日本", "d"] { + w.open(name, &[]); + } + w.close(); + w.empty("e", &[]); + assert_eq!(w.finish(), "<日本>"); +} + +#[test] +fn writer_pretty_nests_a_tree_at_the_current_depth() { + let mut w = Writer::pretty(" "); + w.open("a", &[]); + w.tree(&El::new("b").with_child(El::new("c"))); + assert_eq!(w.finish(), "\n \n \n \n\n"); +}