Files
craft-libs/crates/xml/src/dtd.rs
T

1064 lines
43 KiB
Rust

//! DTD content-model parsing and XML validation.
//! Document Type Definitions: element declarations (content models) and required attributes, and
//! validation of an XML document against them (Structure ▸ Validate).
use std::collections::{HashMap, HashSet};
/// How much text parameter entities may add during expansion (a pass count is not an output
/// bound: each pass replaces whole strings, so references can inflate multiplicatively).
const MAX_EXPANDED_BYTES: usize = 1024 * 1024;
const MAX_DTD_BYTES: usize = 1024 * 1024;
const MAX_VALIDATION_XML_BYTES: usize = 16 * 1024 * 1024;
const MAX_VALIDATION_NODES: usize = 100_000;
const MAX_VALIDATION_DEPTH: usize = 256;
const MAX_VALIDATION_ATTRIBUTES: usize = 1_000_000;
const MAX_MATCH_WORK: usize = 20_000_000;
const MAX_MATCH_POSITIONS: usize = 5_000_000;
const MAX_MATCH_STATES: usize = 100_000;
/// A content particle: a name, a sequence, a choice, or one of those repeated (`?`, `*`, `+`).
#[derive(Clone, Debug, PartialEq)]
enum Cp {
Name(String),
Seq(Vec<Cp>),
Choice(Vec<Cp>),
Rep(Box<Cp>, char),
}
#[derive(Clone, Debug, PartialEq)]
enum Model {
Empty,
Any,
/// `(#PCDATA | a | b)*`: text and these elements in any order.
Mixed(Vec<String>),
Children(Cp),
}
/// A parsed DTD.
#[derive(Clone, Debug, Default)]
pub struct Dtd {
elements: Vec<(String, Model)>,
/// Element → its `#REQUIRED` attributes.
required: HashMap<String, Vec<String>>,
}
/// A problem found by validation: where (an element path such as `Root/story[2]`) and what.
#[derive(Clone, Debug, PartialEq, serde::Serialize)]
pub struct Problem {
pub path: String,
pub message: String,
}
impl Dtd {
/// Declared element names, in order.
pub fn element_names(&self) -> impl Iterator<Item = &str> {
self.elements.iter().map(|(n, _)| n.as_str())
}
fn model(&self, name: &str) -> Option<&Model> {
self.elements.iter().find(|(n, _)| n == name).map(|(_, m)| m)
}
}
/// Markup declarations (`<!…>`) with comments removed and quoted strings kept whole.
fn declarations(src: &str) -> Result<Vec<String>, String> {
let mut out = Vec::new();
let mut rest = src;
while let Some(i) = rest.find("<!") {
rest = &rest[i..];
if rest.starts_with("<!--") {
let Some(j) = rest.find("-->") else {
return Err("unterminated DTD comment".into());
};
rest = &rest[j + 3..];
continue;
}
let mut quote = None;
let mut end = None;
for (k, c) in rest.char_indices().skip(2) {
match (quote, c) {
(Some(q), c) if c == q => quote = None,
(None, '"' | '\'') => quote = Some(c),
(None, '>') => {
end = Some(k);
break;
}
_ => {}
}
}
let Some(end) = end else {
return Err("unterminated DTD declaration".into());
};
out.push(rest[2..end].to_string());
rest = &rest[end + 1..];
}
Ok(out)
}
fn tokens(s: &str) -> Vec<String> {
let mut out = Vec::new();
let mut cur = String::new();
for c in s.chars() {
if c.is_whitespace() || "(),|?*+".contains(c) {
if !cur.is_empty() {
out.push(std::mem::take(&mut cur));
}
if !c.is_whitespace() {
out.push(c.to_string());
}
} else {
cur.push(c);
}
}
if !cur.is_empty() {
out.push(cur);
}
out
}
/// How deeply groups may nest in a content model; parsing, matching (`ends`) and printing (`show`)
/// recurse once per level.
const MAX_CONTENT_MODEL_DEPTH: usize = 64;
/// Parse a content particle from `t[*i..]`, inside `depth` enclosing groups.
fn particle(t: &[String], i: &mut usize, depth: usize) -> Result<Cp, String> {
let base = match t.get(*i).map(String::as_str) {
Some("(") => {
if depth >= MAX_CONTENT_MODEL_DEPTH {
return Err(format!("content model nested more than {MAX_CONTENT_MODEL_DEPTH} levels"));
}
*i += 1;
let mut items = vec![particle(t, i, depth + 1)?];
let mut sep = None;
loop {
match t.get(*i).map(String::as_str) {
Some(")") => {
*i += 1;
break;
}
Some(s @ ("," | "|")) => {
if sep.is_some_and(|p| p != s) {
return Err("mixed `,` and `|` in one group".into());
}
sep = Some(if s == "," { "," } else { "|" });
*i += 1;
items.push(particle(t, i, depth + 1)?);
}
other => return Err(format!("unexpected `{}` in a content model", other.unwrap_or("end"))),
}
}
if sep == Some("|") { Cp::Choice(items) } else { Cp::Seq(items) }
}
Some(n) if !"),|?*+".contains(n) => {
*i += 1;
Cp::Name(n.to_string())
}
other => return Err(format!("unexpected `{}` in a content model", other.unwrap_or("end"))),
};
Ok(match t.get(*i).map(String::as_str) {
Some(r @ ("?" | "*" | "+")) => {
*i += 1;
Cp::Rep(Box::new(base), r.chars().next().unwrap_or('*'))
}
_ => base,
})
}
fn parse_model(s: &str) -> Result<Model, String> {
let s = s.trim();
match s {
"EMPTY" => return Ok(Model::Empty),
"ANY" => return Ok(Model::Any),
_ => {}
}
if s.contains("#PCDATA") {
let t = tokens(s);
if t.first().map(String::as_str) != Some("(") || t.get(1).map(String::as_str) != Some("#PCDATA") {
return Err("malformed mixed content model".into());
}
let mut i = 2;
let mut allowed = Vec::new();
if t.get(i).map(String::as_str) == Some(")") {
i += 1;
if t.get(i).map(String::as_str) == Some("*") {
i += 1;
}
if i != t.len() {
return Err("unexpected token after mixed content model".into());
}
return Ok(Model::Mixed(allowed));
}
loop {
if t.get(i).map(String::as_str) != Some("|") {
return Err("expected `|` in mixed content model".into());
}
i += 1;
let Some(name) = t.get(i).filter(|name| !"(),|?*+".contains(name.as_str())) else {
return Err("expected element name in mixed content model".into());
};
if allowed.contains(name) {
return Err(format!("duplicate `{name}` in mixed content model"));
}
allowed.push(name.clone());
i += 1;
match t.get(i).map(String::as_str) {
Some("|") => continue,
Some(")") => {
i += 1;
break;
}
_ => return Err("expected `|` or `)` in mixed content model".into()),
}
}
if t.get(i).map(String::as_str) != Some("*") || i + 1 != t.len() {
return Err("mixed content with element names must end in `*`".into());
}
return Ok(Model::Mixed(allowed));
}
let t = tokens(s);
let mut i = 0;
let cp = particle(&t, &mut i, 0)?;
if i != t.len() {
return Err(format!("unexpected `{}` after the content model", t[i]));
}
Ok(Model::Children(cp))
}
/// How deeply parameter entities may nest: a reference in the DTD is level 1, a reference in its
/// value level 2, and so on.
const MAX_PARAMETER_ENTITY_DEPTH: usize = 64;
/// Expand `%name;` references, one level per round, until a round changes nothing.
fn expand_parameter_entities(src: &str, ents: &[(String, String)]) -> Result<String, String> {
let mut entities = HashMap::new();
for (name, value) in ents {
entities.entry(name.as_str()).or_insert(value.as_str());
}
let limit = src.len().saturating_add(MAX_EXPANDED_BYTES);
let append = |out: &mut String, value: &str| -> Result<(), String> {
if value.len() > limit.saturating_sub(out.len()) {
return Err(format!("parameter entities expand the DTD beyond {limit} bytes"));
}
out.push_str(value);
Ok(())
};
let too_deep = || format!("parameter entities refer to themselves or nest deeper than {MAX_PARAMETER_ENTITY_DEPTH} levels");
let mut text = src.to_string();
// The round after the deepest level changes nothing, hence one more round than levels.
for _ in 0..=MAX_PARAMETER_ENTITY_DEPTH {
let mut out = String::with_capacity(text.len());
let mut rest = text.as_str();
let mut changed = false;
loop {
let mut special = None;
for (i, c) in rest.char_indices() {
if c == '%' {
special = Some((i, false));
break;
}
if c == '<' && rest[i..].starts_with("<!--") {
special = Some((i, true));
break;
}
}
let Some((i, is_comment)) = special else {
append(&mut out, rest)?;
rest = "";
break;
};
if is_comment {
append(&mut out, &rest[..i])?;
let comment_tail = &rest[i..];
let Some(end) = comment_tail.find("-->") else {
return Err("unterminated DTD comment".into());
};
append(&mut out, &comment_tail[..end + 3])?;
rest = &comment_tail[end + 3..];
continue;
}
let (prefix, reference) = rest.split_at(i);
append(&mut out, prefix)?;
// `find` points to an ASCII `%`, so both split positions are UTF-8 boundaries.
rest = reference.split_at(1).1;
// A name has no whitespace and no `%` (`parse` ends a name at whitespace), whatever its
// length: the search for the closing `;` stops at the first of those, so the searches of
// a round never overlap and a round stays linear in the text.
let end =
rest.bytes().position(|b| matches!(b, b';' | b'%') || b.is_ascii_whitespace()).filter(|&k| rest.as_bytes().get(k) == Some(&b';'));
if let Some(end) = end
&& let Some(value) = rest.get(..end).and_then(|name| entities.get(name))
{
// A value that is its own reference (`<!ENTITY % a "%a;">`) never gets anywhere.
if reference.get(..end + 2) == Some(*value) {
return Err(too_deep());
}
append(&mut out, value)?;
changed = true;
rest = rest.split_at(end + 1).1;
continue;
}
append(&mut out, "%")?;
}
append(&mut out, rest)?;
text = out;
if !changed {
return Ok(text);
}
}
Err(too_deep())
}
/// Parse a DTD (its element and attribute-list declarations; parameter entities are expanded).
pub fn parse(src: &str) -> Result<Dtd, String> {
if src.len() > MAX_DTD_BYTES {
return Err(format!("DTD exceeds the {MAX_DTD_BYTES} byte parsing limit"));
}
// Parameter entities first, then expand references and read the declarations again.
let mut ents: Vec<(String, String)> = Vec::new();
for d in declarations(src)? {
if let Some(rest) = d.strip_prefix("ENTITY") {
let rest = rest.trim_start();
if let Some(rest) = rest.strip_prefix('%') {
let rest = rest.trim_start();
let name: String = rest.chars().take_while(|c| !c.is_whitespace()).collect();
let val = rest[name.len()..].trim();
let q = val.chars().next().unwrap_or('"');
if let Some(v) = val.strip_prefix(q).and_then(|v| v.find(q).map(|e| &v[..e])) {
ents.push((name, v.to_string()));
}
}
}
}
let text = expand_parameter_entities(src, &ents)?;
let mut dtd = Dtd::default();
for d in declarations(&text)? {
if let Some(rest) = d.strip_prefix("ELEMENT") {
let rest = rest.trim_start();
let name: String = rest.chars().take_while(|c| !c.is_whitespace()).collect();
let model = parse_model(&rest[name.len()..]).map_err(|e| format!("<!ELEMENT {name}>: {e}"))?;
dtd.elements.push((name, model));
} else if let Some(rest) = d.strip_prefix("ATTLIST") {
// name (attr type default)*; only `#REQUIRED` matters here.
let mut words = Vec::new();
let mut cur = String::new();
let mut quote = None;
let mut group = 0;
for c in rest.chars() {
match (quote, c) {
(Some(q), c) if c == q => {
quote = None;
cur.push(c);
}
(Some(_), c) => cur.push(c),
(None, '"' | '\'') => {
quote = Some(c);
cur.push(c);
}
(None, '(') => {
group += 1;
cur.push(c);
}
(None, ')') => {
group -= 1;
cur.push(c);
}
(None, c) if c.is_whitespace() && group == 0 => {
if !cur.is_empty() {
words.push(std::mem::take(&mut cur));
}
}
(None, c) => cur.push(c),
}
}
if !cur.is_empty() {
words.push(cur);
}
let Some((el, attrs)) = words.split_first() else { continue };
let mut k = 0;
while k < attrs.len() {
let name = &attrs[k];
let Some(ty) = attrs.get(k + 1) else {
return Err(format!("incomplete ATTLIST for `{el}`"));
};
k += 2;
// NOTATION is a two-token type: the keyword followed by its enumeration.
if ty == "NOTATION" {
if attrs.get(k).is_none_or(|group| !group.starts_with('(') || !group.ends_with(')')) {
return Err(format!("malformed NOTATION type for `{name}`"));
}
k += 1;
}
let Some(default) = attrs.get(k).map(String::as_str) else {
return Err(format!("missing default declaration for `{name}`"));
};
k += 1;
if default == "#REQUIRED" {
dtd.required.entry(el.clone()).or_default().push(name.clone());
} else if default == "#FIXED" {
if k >= attrs.len() {
return Err(format!("missing #FIXED value for `{name}`"));
}
k += 1;
}
}
}
}
if dtd.elements.is_empty() {
return Err("no element declarations".into());
}
Ok(dtd)
}
/// What `ends` has worked out for one sequence of children: the end positions of a group (by the
/// address of its node, which lives as long as the model) from a position.
type EndsMemo = HashMap<(usize, usize), Vec<usize>>;
#[derive(Default)]
struct MatchBudget {
work: usize,
positions: usize,
states: usize,
exhausted: bool,
reported: bool,
}
impl MatchBudget {
fn spend(&mut self, work: usize, positions: usize) -> Result<(), &'static str> {
if self.exhausted {
return Err("content model matching exceeded its resource budget");
}
self.work = self.work.saturating_add(work);
self.positions = self.positions.saturating_add(positions);
if self.work > MAX_MATCH_WORK {
self.exhausted = true;
return Err("content model matching exceeded its work limit");
}
if self.positions > MAX_MATCH_POSITIONS {
self.exhausted = true;
return Err("content model matching exceeded its memory limit");
}
Ok(())
}
fn retain_state(&mut self) -> Result<(), &'static str> {
self.states = self.states.saturating_add(1);
if self.states > MAX_MATCH_STATES {
self.exhausted = true;
return Err("content model matching exceeded its state limit");
}
Ok(())
}
}
/// End positions after `cp` matches a prefix of `seq[i..]`.
///
/// The positions of each group are worked out once per start position (`memo`): nested repetitions
/// such as `((a*)*)*` ask the inner group for the same positions again and again, which is
/// exponential in their depth without it.
fn ends(cp: &Cp, seq: &[&str], i: usize, memo: &mut EndsMemo, budget: &mut MatchBudget) -> Result<Vec<usize>, &'static str> {
budget.spend(1, 0)?;
if let Cp::Name(n) = cp {
if seq.get(i) == Some(&n.as_str()) {
budget.spend(0, 1)?;
return Ok(vec![i + 1]);
}
return Ok(vec![]);
}
let key = (std::ptr::from_ref(cp) as usize, i);
if let Some(done) = memo.get(&key) {
budget.spend(0, done.len())?;
return Ok(done.clone());
}
let mut out: Vec<usize> = match cp {
Cp::Name(_) => vec![],
// Keep the positions after each item a set: as a list of every path, `(a?, a?, …)` doubles per item.
Cp::Seq(items) => {
let mut at = vec![i];
for c in items {
let mut next = Vec::new();
for p in at {
let found = ends(c, seq, p, memo, budget)?;
budget.spend(found.len(), found.len())?;
next.extend(found);
}
next.sort_unstable();
next.dedup();
at = next;
}
at
}
Cp::Choice(items) => {
let mut all = Vec::new();
for c in items {
let found = ends(c, seq, i, memo, budget)?;
budget.spend(found.len(), found.len())?;
all.extend(found);
}
all
}
Cp::Rep(c, r) => {
let mut all = if *r == '+' { vec![] } else { vec![i] };
let mut seen: HashSet<usize> = all.iter().copied().collect();
let mut frontier = ends(c, seq, i, memo, budget)?;
budget.spend(frontier.len(), frontier.len())?;
if *r == '?' {
all.extend(frontier);
} else {
while !frontier.is_empty() {
let mut new = Vec::new();
for p in frontier.iter().copied() {
budget.spend(1, 0)?;
if seen.insert(p) {
budget.spend(0, 1)?;
new.push(p);
}
}
budget.spend(new.len(), new.len())?;
all.extend(&new);
let mut next = Vec::new();
let mut next_seen = HashSet::new();
for p in new.into_iter().filter(|&p| p > i) {
for found in ends(c, seq, p, memo, budget)? {
budget.spend(1, 0)?;
if !seen.contains(&found) && next_seen.insert(found) {
budget.spend(0, 1)?;
next.push(found);
}
}
}
// Two paths to one position are one position: kept apart they multiply per round.
next.sort_unstable();
next.dedup();
frontier = next;
}
}
all
}
};
out.sort_unstable();
out.dedup();
budget.spend(out.len(), out.len())?;
budget.retain_state()?;
memo.insert(key, out.clone());
Ok(out)
}
fn show(cp: &Cp) -> String {
match cp {
Cp::Name(n) => n.clone(),
Cp::Seq(v) => format!("({})", v.iter().map(show).collect::<Vec<_>>().join(", ")),
Cp::Choice(v) => format!("({})", v.iter().map(show).collect::<Vec<_>>().join(" | ")),
Cp::Rep(c, r) => format!("{}{r}", show(c)),
}
}
/// An element of the document being validated.
struct Node {
name: String,
attrs: Vec<String>,
children: Vec<Node>,
text: bool,
content: bool,
}
fn bounded_node(e: &quick_xml::events::BytesStart<'_>, depth: usize, nodes: &mut usize, attributes: &mut usize) -> Result<Node, String> {
if depth >= MAX_VALIDATION_DEPTH {
return Err(format!("XML nesting exceeds the {MAX_VALIDATION_DEPTH} level validation limit"));
}
*nodes = nodes.saturating_add(1);
if *nodes > MAX_VALIDATION_NODES {
return Err(format!("XML exceeds the {MAX_VALIDATION_NODES} node validation limit"));
}
let mut attrs = Vec::new();
for attr in e.attributes().with_checks(true) {
let attr = attr.map_err(|err| format!("invalid attribute: {err}"))?;
attrs.push(String::from_utf8_lossy(attr.key.as_ref()).to_string());
}
*attributes = attributes.saturating_add(attrs.len());
if *attributes > MAX_VALIDATION_ATTRIBUTES {
return Err(format!("XML exceeds the {MAX_VALIDATION_ATTRIBUTES} attribute validation limit"));
}
Ok(Node { name: String::from_utf8_lossy(e.name().as_ref()).to_string(), attrs, children: vec![], text: false, content: false })
}
fn read_tree(xml: &str) -> Result<Node, String> {
use quick_xml::events::Event;
if xml.len() > MAX_VALIDATION_XML_BYTES {
return Err(format!("XML exceeds the {} byte validation limit", MAX_VALIDATION_XML_BYTES));
}
let mut r = quick_xml::Reader::from_str(xml);
let mut stack: Vec<Node> = Vec::new();
let mut root = None;
let mut nodes = 0usize;
let mut attributes = 0usize;
loop {
match r.read_event().map_err(|e| e.to_string())? {
Event::Start(e) => {
if stack.is_empty() && root.is_some() {
return Err("multiple root elements".into());
}
let n = bounded_node(&e, stack.len(), &mut nodes, &mut attributes)?;
if let Some(parent) = stack.last_mut() {
parent.content = true;
}
stack.push(n);
}
Event::Empty(e) => {
if stack.is_empty() && root.is_some() {
return Err("multiple root elements".into());
}
let n = bounded_node(&e, stack.len(), &mut nodes, &mut attributes)?;
match stack.last_mut() {
Some(p) => {
p.content = true;
p.children.push(n);
}
None => root = Some(n),
}
}
Event::End(e) => {
let n = stack.pop().ok_or("unbalanced end tag")?;
if n.name.as_bytes() != e.name().as_ref() {
return Err("mismatched end tag".into());
}
match stack.last_mut() {
Some(p) => p.children.push(n),
None => root = Some(n),
}
}
Event::Text(t) => {
if let Some(p) = stack.last_mut() {
p.content = true;
if !t.iter().all(u8::is_ascii_whitespace) {
p.text = true;
}
} else if !t.iter().all(u8::is_ascii_whitespace) {
return Err("text outside the root element".into());
}
}
Event::CData(_) => {
if let Some(p) = stack.last_mut() {
p.text = true;
p.content = true;
}
}
Event::GeneralRef(reference) => {
let name = reference.decode().map_err(|e| e.to_string())?;
let accepted = if let Some(character) = reference.resolve_char_ref().map_err(|e| e.to_string())? {
crate::is_xml_char(character)
} else {
matches!(name.as_ref(), "amp" | "lt" | "gt" | "apos" | "quot")
};
if !accepted {
return Err(format!("unsupported general entity reference `&{name};`"));
}
if let Some(p) = stack.last_mut() {
p.text = true;
p.content = true;
} else {
return Err("reference outside the root element".into());
}
}
Event::Comment(_) | Event::PI(_) => {
if let Some(parent) = stack.last_mut() {
parent.content = true;
}
}
Event::Eof => {
if !stack.is_empty() {
return Err("unclosed element".into());
}
return root.ok_or_else(|| "no root element".into());
}
_ => {}
}
}
}
fn check(dtd: &Dtd, n: &Node, path: &str, out: &mut Vec<Problem>, budget: &mut MatchBudget) {
let mut problem = |m: String| out.push(Problem { path: path.to_string(), message: m });
match dtd.model(&n.name) {
None => problem(format!("`{}` isn't declared in the DTD", n.name)),
Some(Model::Any) => {}
Some(Model::Empty) => {
if n.content {
problem(format!("`{}` should be empty", n.name));
}
}
Some(Model::Mixed(allowed)) => {
for c in &n.children {
if !allowed.contains(&c.name) {
problem(format!("`{}` isn't allowed in `{}`", c.name, n.name));
}
}
}
Some(Model::Children(cp)) => {
if n.text {
problem(format!("`{}` can't contain text", n.name));
}
let seq: Vec<&str> = n.children.iter().map(|c| c.name.as_str()).collect();
let matched = ends(cp, &seq, 0, &mut EndsMemo::new(), budget);
match matched {
Ok(positions) if positions.contains(&seq.len()) => {}
Ok(_) => {
let got = if seq.is_empty() { "nothing".to_string() } else { seq.join(", ") };
problem(format!("`{}` contains {got}; the DTD asks for {}", n.name, show(cp)));
}
Err(error) => {
if !budget.reported {
problem(format!("`{}`: {error}", n.name));
budget.reported = true;
}
}
}
}
}
for a in dtd.required.get(&n.name).into_iter().flatten() {
if !n.attrs.contains(a) {
problem(format!("`{}` is missing the required attribute `{a}`", n.name));
}
}
let mut seen: HashMap<&str, usize> = HashMap::new();
for c in &n.children {
let k = seen.entry(c.name.as_str()).or_default();
*k += 1;
check(dtd, c, &format!("{path}/{}[{k}]", c.name), out, budget);
}
}
/// Validate `xml` against `dtd`; an empty list when it's valid.
///
/// Validation rejects input above the internal byte, depth, node, attribute and content-model
/// matching budgets. Only predefined entities and numeric character references are resolved; a
/// general entity reference is reported as unsupported rather than expanded or fetched.
pub fn validate(dtd: &Dtd, xml: &str) -> Vec<Problem> {
match read_tree(xml) {
Ok(root) => {
let mut out = Vec::new();
let path = root.name.clone();
check(dtd, &root, &path, &mut out, &mut MatchBudget::default());
out
}
Err(e) => vec![Problem { path: String::new(), message: format!("not well-formed XML: {e}") }],
}
}
#[cfg(test)]
mod tests {
use super::*;
const DTD: &str = r##"<!-- an article -->
<!ENTITY % inline "#PCDATA | em | term">
<!ELEMENT Root (story+, figure*)>
<!ELEMENT story (title, (para | list)*)>
<!ELEMENT title (%inline;)*>
<!ELEMENT para (%inline;)*>
<!ELEMENT list (para+)>
<!ELEMENT em (#PCDATA)>
<!ELEMENT term (#PCDATA)>
<!ELEMENT figure EMPTY>
<!ATTLIST figure href CDATA #REQUIRED alt CDATA "a > b" kind (photo|chart) "photo">"##;
#[test]
fn content_models_and_required_attributes() {
let dtd = parse(DTD).unwrap();
assert_eq!(dtd.element_names().count(), 8);
let ok = r#"<Root><story><title>T <em>x</em></title><para>a</para><list><para>b</para></list></story><figure href="a.png"/></Root>"#;
assert_eq!(validate(&dtd, ok), vec![]);
let bad = r#"<Root><story><para>no title</para></story><figure/><figure href="b"><em>x</em></figure><aside/></Root>"#;
let p = validate(&dtd, bad);
let msgs: Vec<String> = p.iter().map(|p| format!("{}: {}", p.path, p.message)).collect();
assert!(msgs.iter().any(|m| m.starts_with("Root/story[1]: `story` contains para; the DTD asks for (title, (para | list)*)")), "{msgs:?}");
assert!(msgs.iter().any(|m| m == "Root/figure[1]: `figure` is missing the required attribute `href`"), "{msgs:?}");
assert!(msgs.iter().any(|m| m == "Root/figure[2]: `figure` should be empty"), "{msgs:?}");
assert!(msgs.iter().any(|m| m == "Root/aside[1]: `aside` isn't declared in the DTD"), "{msgs:?}");
// The root's model fails too (aside isn't in it).
assert!(msgs.iter().any(|m| m.starts_with("Root: ")), "{msgs:?}");
// Text where only elements are allowed; an element not in a mixed model.
let p = validate(&dtd, "<Root><story>loose<title><list/></title></story></Root>");
assert!(p.iter().any(|p| p.message == "`story` can't contain text"), "{p:?}");
assert!(p.iter().any(|p| p.message == "`list` isn't allowed in `title`"), "{p:?}");
assert!(parse("<!ELEMENT a (b, c | d)>").is_err());
assert!(parse("just text").is_err());
}
/// A few hundred bytes of nested parameter entities used to inflate multiplicatively across
/// the eight wholly-string-replacing passes (#713); expansion is capped.
#[test]
fn parameter_entities_cannot_expand_without_a_budget() {
let mut src = String::from("<!ENTITY % e1 \"xx\">\n");
for i in 2..=6 {
src.push_str(&format!("<!ENTITY % e{i} \"{}\">\n", format!("%e{};", i - 1).repeat(16)));
}
src.push_str("<!ELEMENT r (%e6;)>");
assert!(src.len() < 500, "the payload is small: {} bytes", src.len());
let err = parse(&src).unwrap_err();
assert!(err.contains("expand the DTD beyond"), "{err}");
// Entities that expand a little still parse.
let dtd = parse("<!ENTITY % inline \"#PCDATA | em\">\n<!ELEMENT r (%inline;)*>").unwrap();
assert_eq!(dtd.element_names().collect::<Vec<_>>(), ["r"]);
}
#[test]
fn repetition_matches_like_a_regular_expression() {
let dtd = parse("<!ELEMENT r ((a, b?)+, c*)><!ELEMENT a EMPTY><!ELEMENT b EMPTY><!ELEMENT c EMPTY>").unwrap();
for (xml, ok) in [
("<r><a/></r>", true),
("<r><a/><b/><a/><c/><c/></r>", true),
("<r><a/><a/><b/></r>", true),
("<r><b/></r>", false),
("<r><a/><b/><b/></r>", false),
("<r><a/><c/><a/></r>", false),
("<r/>", false),
] {
assert_eq!(validate(&dtd, xml).is_empty(), ok, "{xml}");
}
}
#[test]
fn parameter_entity_bomb_is_rejected() {
let mut src = String::new();
for i in (0..10).rev() {
let value = if i == 0 { "x".to_string() } else { format!("%e{};", i - 1).repeat(8) };
src.push_str(&format!("<!ENTITY % e{i} \"{value}\">\n"));
}
src.push_str("<!ELEMENT root (%e9;)>");
match parse(&src) {
Err(error) => assert!(error.contains("parameter entities expand"), "{error}"),
Ok(_) => panic!("parameter entity expansion bomb was accepted"),
}
}
/// `n` parameter entities, `e{i}` referring to `e{i-1}` and `e0` being `x`, declared in ascending
/// or descending order, and an element `root` whose model uses the last one.
fn entity_chain(n: usize, ascending: bool) -> String {
let mut order: Vec<usize> = (0..n).collect();
if !ascending {
order.reverse();
}
let mut src = String::new();
for i in order {
let value = if i == 0 { "x".to_string() } else { format!("%e{};", i - 1) };
src.push_str(&format!("<!ENTITY % e{i} \"{value}\">\n"));
}
src.push_str(&format!("<!ELEMENT root (%e{};)><!ELEMENT x EMPTY>", n - 1));
src
}
#[test]
fn parameter_entity_reverse_chain_expands_completely() {
let dtd = parse(&entity_chain(20, false)).unwrap();
assert_eq!(validate(&dtd, "<root><x/></root>"), vec![]);
}
#[test]
fn parameter_entity_ascending_chain_expands_completely() {
// With `e0` declared first, replacing the entities in declaration order resolves one level per pass.
let dtd = parse(&entity_chain(20, true)).unwrap();
assert_eq!(validate(&dtd, "<root><x/></root>"), vec![]);
}
#[test]
fn parameter_entity_cycle_is_refused() {
// Two entities that refer to each other, and one whose value is its own reference.
for src in ["<!ENTITY % a \"%b;\"><!ENTITY % b \"%a;\"><!ELEMENT root (%a;)>", "<!ENTITY % a \"%a;\"><!ELEMENT root (%a;)>"] {
match parse(src) {
Err(error) => assert!(error.contains("refer to themselves"), "{src}: {error}"),
Ok(dtd) => panic!("a cycle was accepted ({src}): {:?}", dtd.model("root")),
}
}
}
#[test]
fn parameter_entity_chain_deeper_than_the_limit_is_refused() {
for ascending in [true, false] {
for n in [65, 70] {
match parse(&entity_chain(n, ascending)) {
Err(error) => assert!(error.contains("deeper than 64 levels"), "{n}, {ascending}: {error}"),
Ok(dtd) => panic!("a chain of {n} ({ascending}) was accepted: {:?}", dtd.model("root")),
}
}
let dtd = parse(&entity_chain(64, ascending)).unwrap();
assert_eq!(validate(&dtd, "<root><x/></root>"), vec![], "{ascending}");
}
}
#[test]
fn parameter_entity_name_scan_stops_at_the_next_percent() {
// A name never contains `%`, so the search for the closing `;` stops at the next one: the
// searches of a run of `%` don't overlap and a round stays linear. (Looking past the next
// `%` would expand this reference to an entity that is named `a%b`.)
let entities = [("a%b".to_string(), "X".to_string())];
assert_eq!(expand_parameter_entities("%a%b;", &entities), Ok("%a%b;".to_string()));
// A run of `%` around a real reference still finds it.
let entities = [("model".to_string(), "EMPTY".to_string())];
assert_eq!(expand_parameter_entities("%%%model;%%", &entities), Ok("%%EMPTY%%".to_string()));
}
#[test]
fn parameter_entity_with_a_long_name_expands() {
// A name can be any length: the search for the closing `;` has no window.
let name = "n".repeat(300);
let dtd = parse(&format!("<!ENTITY % {name} \"EMPTY\"><!ELEMENT root %{name};>")).unwrap();
assert_eq!(dtd.model("root"), Some(&Model::Empty));
}
#[test]
fn percent_in_text_without_a_reference_is_left_alone() {
// `100% sure` (a space ends the search) and a `%` followed by 1 MiB without a `;` are text.
let text = format!("100% sure, %{}", "a".repeat(1 << 20));
let entities = [("a".to_string(), "X".to_string())];
assert_eq!(expand_parameter_entities(&text, &entities), Ok(text.clone()));
}
/// `a` inside `levels` groups, each repeated: `((a)*)*…`.
fn nested_repetitions(levels: usize) -> String {
format!("{}a{}", "(".repeat(levels), ")*".repeat(levels))
}
#[test]
fn nested_repetitions_validate_without_going_exponential() {
// Ten levels over 100 children: each level asked the one below for the same positions again
// and again, about 10^14 steps without the memo of `ends`.
let dtd = parse(&format!("<!ELEMENT root {}><!ELEMENT a EMPTY><!ELEMENT b EMPTY>", nested_repetitions(10))).unwrap();
assert_eq!(validate(&dtd, &format!("<root>{}</root>", "<a/>".repeat(100))), vec![]);
let problems = validate(&dtd, &format!("<root>{}<b/></root>", "<a/>".repeat(100)));
assert_eq!(problems.len(), 1, "{problems:?}");
}
#[test]
fn repetition_of_a_sequence_of_optional_elements_validates() {
// `(a, a?, a?, a?, a?, a?)*` over 90 children: every way to split the children among the
// repetitions reaches the same positions, and the paths used to be kept apart in each round.
let dtd = parse("<!ELEMENT r (a, a?, a?, a?, a?, a?)*><!ELEMENT a EMPTY><!ELEMENT b EMPTY>").unwrap();
assert_eq!(validate(&dtd, &format!("<r>{}</r>", "<a/>".repeat(90))), vec![]);
let problems = validate(&dtd, &format!("<r>{}<b/></r>", "<a/>".repeat(90)));
assert_eq!(problems.len(), 1, "{problems:?}");
}
#[test]
fn parameter_entity_first_declaration_wins() {
let dtd = parse("<!ENTITY % model \"EMPTY\"><!ENTITY % model \"ANY\"><!ELEMENT root %model;>").unwrap();
assert_eq!(dtd.model("root"), Some(&Model::Empty));
}
#[test]
fn parameter_entity_unknown_and_lone_percent_are_preserved() {
let src = "%nothing; <!ELEMENT root EMPTY> %";
let dtd = parse(src).unwrap();
assert_eq!(dtd.model("root"), Some(&Model::Empty));
assert_eq!(expand_parameter_entities("%nothing; %", &[]), Ok("%nothing; %".to_string()));
}
#[test]
fn parameter_entity_many_flat_declarations_parse() {
let mut src = String::new();
for i in 0..5000 {
src.push_str(&format!("<!ENTITY % e{i} \"x\">\n"));
}
src.push_str("<!ELEMENT root (%e0;, %e2499;, %e4999;)><!ELEMENT x EMPTY>");
let dtd = parse(&src).unwrap();
assert_eq!(validate(&dtd, "<root><x/><x/><x/></root>"), vec![]);
}
/// An element `root` whose content model is `a` inside `depth` nested groups.
fn nested_model(depth: usize) -> String {
format!("<!ELEMENT root {}a{}><!ELEMENT a EMPTY>", "(".repeat(depth), ")".repeat(depth))
}
#[test]
fn content_model_nested_100000_deep_is_refused() {
// One recursion per `(` overflowed the stack: an abort, not a panic.
match parse(&nested_model(100_000)) {
Err(error) => assert!(error.contains("nested more than 64 levels"), "{error}"),
Ok(_) => panic!("a content model nested 100000 deep was accepted"),
}
}
#[test]
fn content_model_nesting_limit() {
let dtd = parse(&nested_model(64)).unwrap();
assert_eq!(validate(&dtd, "<root><a/></root>"), vec![]);
assert_eq!(validate(&dtd, "<root/>").len(), 1);
match parse(&nested_model(65)) {
Err(error) => assert_eq!(error, "<!ELEMENT root>: content model nested more than 64 levels"),
Ok(_) => panic!("a content model nested 65 deep was accepted"),
}
}
#[test]
fn sequence_of_40_optional_elements_validates() {
// Keeping every path through the sequence meant 2^40 positions here.
let dtd = parse(&format!("<!ELEMENT r ({})><!ELEMENT a EMPTY>", ["a?"; 40].join(", "))).unwrap();
assert_eq!(validate(&dtd, &format!("<r>{}</r>", "<a/>".repeat(40))), vec![]);
let problems = validate(&dtd, &format!("<r>{}</r>", "<a/>".repeat(41)));
assert_eq!(problems.len(), 1, "{problems:?}");
assert!(problems[0].message.starts_with("`r` contains a, a, a"), "{problems:?}");
}
#[test]
fn incomplete_declarations_and_malformed_mixed_models_are_rejected() {
assert!(parse("<!ELEMENT r EMPTY").is_err());
assert!(parse("<!-- unterminated").is_err());
for model in ["(#PCDATA | a)", "(#PCDATA, a)*", "(#PCDATA | )*", "(#PCDATA | a)* junk"] {
assert!(parse(&format!("<!ELEMENT r {model}>")).is_err(), "{model}");
}
assert!(parse("<!ELEMENT r (#PCDATA)>").is_ok());
assert!(parse("<!ELEMENT r (#PCDATA)*>").is_ok());
assert!(parse("<!ELEMENT r (#PCDATA | a)*>").is_ok());
}
#[test]
fn parameter_entities_are_not_expanded_inside_comments() {
let dtd = parse(
r#"<!ENTITY % e "--> <!ELEMENT r ANY> <!--">
<!-- %e; -->
<!ELEMENT r EMPTY>
<!ELEMENT a EMPTY>"#,
)
.unwrap();
assert_eq!(dtd.model("r"), Some(&Model::Empty));
assert!(!validate(&dtd, "<r><a/></r>").is_empty());
}
#[test]
fn notation_attribute_type_keeps_following_required_attribute_aligned() {
let dtd = parse("<!ELEMENT r ANY><!ATTLIST r kind NOTATION (png|jpeg) #IMPLIED id CDATA #REQUIRED>").unwrap();
assert!(validate(&dtd, "<r kind='png'/>").iter().any(|p| p.message.contains("required attribute `id`")));
assert!(validate(&dtd, "<r id='x'/>").is_empty());
}
#[test]
fn malformed_or_unbounded_xml_is_rejected_before_validation() {
let dtd = parse("<!ELEMENT r ANY><!ELEMENT a EMPTY>").unwrap();
for xml in ["<r/><broken", "<r></r><a/>", "<r a='1' a='2'/>"] {
assert!(!validate(&dtd, xml).is_empty(), "{xml}");
}
let deep = format!("{}{}", "<r>".repeat(MAX_VALIDATION_DEPTH + 1), "</r>".repeat(MAX_VALIDATION_DEPTH + 1));
let problems = validate(&dtd, &deep);
assert!(problems.iter().any(|p| p.message.contains("nesting exceeds")), "{problems:?}");
}
#[test]
fn references_must_be_declared_and_empty_elements_must_have_no_content() {
let dtd = parse("<!ELEMENT r ANY><!ELEMENT e EMPTY><!ENTITY known 'value'>").unwrap();
assert!(validate(&dtd, "<r>&missing;</r>").iter().any(|p| p.message.contains("unsupported general entity")));
assert!(validate(&dtd, "<r>&known;</r>").iter().any(|p| p.message.contains("unsupported general entity")));
assert!(validate(&dtd, "<r>&#x41;&amp;</r>").is_empty());
assert!(!validate(&dtd, "<r>&#0;</r>").is_empty());
for xml in ["<e> </e>", "<e><!--x--></e>", "<e><?work x?></e>", "<e>&amp;</e>"] {
assert!(validate(&dtd, xml).iter().any(|p| p.message.contains("should be empty")), "{xml}");
}
assert!(validate(&dtd, "<e/>").is_empty());
}
#[test]
fn ambiguous_content_matching_stops_at_its_budget() {
let dtd = parse("<!ELEMENT r ((a*)*)*><!ELEMENT a EMPTY>").unwrap();
let xml = format!("<r>{}</r>", "<a/>".repeat(10_000));
let problems = validate(&dtd, &xml);
assert!(problems.iter().any(|p| p.message.contains("matching exceeded its ")), "{}", problems.len());
}
}