diff --git a/Cargo.lock b/Cargo.lock index fbc4b8aec..a65be1290 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1379,6 +1379,15 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-markdown" +version = "0.1.0" +dependencies = [ + "craft-flow", + "craft-html", + "craft-wordmodel", +] + [[package]] name = "craft-math" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index d1ec4df26..f5a0c2fbb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,6 +70,7 @@ craft-assuan = { path = "crates/assuan" } craft-kra = { path = "crates/kra" } craft-audio = { path = "crates/audio" } # No default features, so craft-audio's `encode` builds craft-media without `isobmff`. +craft-markdown = { path = "crates/markdown" } craft-math = { path = "crates/math" } craft-media = { path = "crates/media", default-features = false } craft-media-testkit = { path = "crates/media-testkit" } diff --git a/README.md b/README.md index 27aa41ec9..2a604df2d 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,7 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = "` and `` anchors. +- Write (`export`): the same constructs; pictures become `![alt](name)`. + +```rust +let doc = craft_markdown::import("# Title\n\nSome *emphasis* and a [link](https://example.com).\n"); +assert_eq!(craft_markdown::export(&doc).trim_end(), "# Title\n\nSome *emphasis* and a [link](https://example.com)."); +``` + +Input is hostile. Nesting is bounded, and table cells are capped (`MAX_TABLE_CELLS` in all, plus +the model's row and column limits), with one warning per kind of thing left out. + +Moved from WordCraft (`crates/formats/src/markdown`, `07f97737`) with its tests; character +references come from craft-html. diff --git a/crates/markdown/src/blocks.rs b/crates/markdown/src/blocks.rs new file mode 100644 index 000000000..72a7aedfb --- /dev/null +++ b/crates/markdown/src/blocks.rs @@ -0,0 +1,727 @@ +use super::inlines::parse_inlines; +use super::{Align, Arc, Cell, Document, FBlock, FTable, Fmt, HashMap, Kind, ListInfo, MAX_COLS, MAX_ROWS, Para, model}; + +/// Nesting limit for block containers (quotes, list items) and inline links. +pub(super) const MAX_NEST: usize = 32; + +// --------------------------------------------------------------------------------------------- +// Import + +/// Parse Markdown text into a document. +pub fn import(text: &str) -> Document { + model::begin_import(); + model::to_doc(&parse(text)) +} + +/// [`import`] onto `page`: Markdown gives no page size (#1381). +pub fn import_on(text: &str, page: &craft_wordmodel::SectionProps) -> Document { + model::begin_import(); + model::to_doc_in(&parse(text), page.clone()) +} + +/// Link reference definitions: the normalised label to its destination. +/// A destination is shared by every reference to it, not copied into each (#3713). +pub type Refs = HashMap>; + +/// A label as CommonMark compares them: case-folded, and any run of white space one space +/// (§4.7, "matches"). `[FOO bar]` and `[foo BAR]` are the same label. +pub(super) fn ref_label(raw: &str) -> String { + raw.split_whitespace().collect::>().join(" ").to_lowercase() +} + +/// A link reference definition on its own line: `[label]: destination "optional title"`. The +/// destination may be wrapped in `<...>`. Returns the label and the destination. +/// +/// CommonMark allows the title on the following line and the label to span lines; a definition +/// written across lines is left as text here, which is what it was before these were read at all. +fn link_definition(line: &str) -> Option<(String, String)> { + let t = line.trim_start(); + // Up to three spaces of indentation; four makes it indented code. + if line.len() - t.len() > 3 || !t.starts_with('[') { + return None; + } + let bytes = t.as_bytes(); + // The label ends at the first unescaped `]`, and cannot be empty or hold an unescaped `[`. + let mut i = 1; + let mut close = None; + while i < bytes.len() { + match bytes[i] { + b'\\' => i += 1, + b'[' => return None, + b']' => { + close = Some(i); + break; + } + _ => {} + } + i += 1; + } + let close = close?; + let label = ref_label(t.get(1..close)?); + // `[^label]: ...` is a footnote definition, which this importer leaves as text. + if label.starts_with('^') { + return None; + } + if label.is_empty() || t.as_bytes().get(close + 1) != Some(&b':') { + return None; + } + let rest = t.get(close + 2..)?.trim_start(); + if rest.is_empty() { + return None; + } + // The destination: `<...>`, else up to the next space. What follows may only be a title. + let (dest, after) = match rest.strip_prefix('<') { + Some(inner) => { + let end = inner.find('>')?; + (inner.get(..end)?.to_string(), inner.get(end + 1..)?) + } + None => { + let end = rest.find(char::is_whitespace).unwrap_or(rest.len()); + (rest.get(..end)?.to_string(), rest.get(end..)?) + } + }; + if dest.is_empty() { + return None; + } + let title = after.trim(); + let titled = + title.is_empty() || (title.len() >= 2 && matches!((title.chars().next()?, title.chars().last()?), ('"', '"') | ('\'', '\'') | ('(', ')'))); + titled.then_some((label, dest)) +} + +/// Take the link reference definitions out of `lines`, leaving the rest. Lines inside a fenced +/// code block are left alone: a definition there is code, not a definition. Fences open and +/// close by the same rules [`parse_blocks`] applies, or a line it reads as code would be read +/// here as a definition and deleted. +/// +/// A definition has to start a block: CommonMark §4.7 forbids one from interrupting a paragraph, +/// and a line like `[Warning]: unstable` in the middle of one is the author's text. This pass +/// sees lines without the block structure around them, so it takes a definition only after a +/// blank line, at the start of the document, after a heading or thematic break, or after another +/// definition. A definition anywhere else stays in the document and reads as text, which is what +/// it did before definitions were understood at all; deleting a line that was part of a +/// paragraph would lose it silently. +fn take_definitions(lines: &mut Vec) -> Refs { + let mut refs = Refs::new(); + let mut open: Option<(char, usize)> = None; + // The start of the document begins a block. + let mut starts_block = true; + lines.retain(|line| { + let t = line.trim_start_matches([' ', '\t']); + // Four spaces of indentation is code, not a fence (`parse_blocks`), on both sides. + let marker = (indent(line) < 4).then(|| fence(t)).flatten(); + match open { + Some((c, n)) => { + // A closing fence holds nothing but its own run of fence characters. + if marker.is_some_and(|(fc, fl)| fc == c && fl >= n) && t.trim_start_matches(c).trim().is_empty() { + open = None; + starts_block = true; + } + return true; + } + None => { + if let Some(f) = marker { + open = Some(f); + return true; + } + } + } + if starts_block && let Some((label, dest)) = link_definition(line) { + // The first definition of a label wins, as CommonMark says, and definitions may + // follow one another. + refs.entry(label).or_insert(dest.into()); + return false; + } + starts_block = is_blank(line) || atx_heading(t).is_some() || thematic_break(t); + true + }); + refs +} + +/// Parse Markdown text into the flow model. +pub fn parse(text: &str) -> model::Flow { + let text = text.strip_prefix('\u{FEFF}').unwrap_or(text); + let mut lines: Vec = text.split('\n').map(|l| expand_tabs(l.trim_end_matches('\r'))).collect(); + let refs = take_definitions(&mut lines); + let budget = std::cell::Cell::new(super::MAX_TABLE_CELLS); + let mut blocks = parse_blocks(&lines, 0, 0, &refs, &budget); + // A leading "Title" from front matter isn't supported; a first-level heading stays a heading. + if blocks.is_empty() { + blocks.push(FBlock::Para(Para::default())); + } + model::Flow { blocks, meta: Default::default() } +} + +/// Leading tabs → spaces (tab stops of 4); the rest of the line is kept. +fn expand_tabs(l: &str) -> String { + if !l.starts_with([' ', '\t']) || !l.contains('\t') { + return l.to_string(); + } + let mut out = String::with_capacity(l.len() + 8); + let mut col = 0usize; + let mut rest = l; + while let Some(c) = rest.chars().next() { + match c { + ' ' => { + out.push(' '); + col += 1; + } + '\t' => { + let n = 4 - col % 4; + out.push_str(&" ".repeat(n)); + col += n; + } + _ => break, + } + rest = &rest[c.len_utf8()..]; + } + out.push_str(rest); + out +} + +fn indent(l: &str) -> usize { + l.len() - l.trim_start_matches(' ').len() +} + +fn is_blank(l: &str) -> bool { + l.trim().is_empty() +} + +/// Strip up to `n` leading spaces. +fn strip_indent(l: &str, n: usize) -> &str { + let k = indent(l).min(n); + l.get(k..).unwrap_or("") +} + +fn atx_heading(t: &str) -> Option<(u8, &str)> { + let n = t.chars().take_while(|c| *c == '#').count(); + if !(1..=6).contains(&n) { + return None; + } + let rest = t.get(n..)?; + if !rest.is_empty() && !rest.starts_with([' ', '\t']) { + return None; + } + let mut body = rest.trim(); + // Optional closing sequence. + let trimmed = body.trim_end_matches('#'); + if trimmed.len() != body.len() && (trimmed.is_empty() || trimmed.ends_with(' ')) { + body = trimmed.trim_end(); + } + Some((n as u8, body)) +} + +pub(super) fn thematic_break(t: &str) -> bool { + let mut ch = None; + let mut n = 0; + for c in t.chars() { + match c { + ' ' | '\t' => {} + '*' | '-' | '_' => { + if ch.is_some_and(|x| x != c) { + return false; + } + ch = Some(c); + n += 1; + } + _ => return false, + } + } + n >= 3 +} + +fn fence(t: &str) -> Option<(char, usize)> { + let c = t.chars().next().filter(|c| *c == '`' || *c == '~')?; + let n = t.chars().take_while(|x| *x == c).count(); + if n < 3 { + return None; + } + if c == '`' && t.get(n..).is_some_and(|info| info.contains('`')) { + return None; + } + Some((c, n)) +} + +#[derive(Clone, Copy, PartialEq)] +struct Marker { + ordered: bool, + /// Bullet char or ordered delimiter. + ch: char, + /// Bytes of the marker itself (after the indent). + len: usize, + start: u32, +} + +fn list_marker(t: &str) -> Option { + let first = t.chars().next()?; + let (m, len) = if matches!(first, '-' | '+' | '*') { + (Marker { ordered: false, ch: first, len: 1, start: 1 }, 1) + } else { + let digits = t.chars().take_while(char::is_ascii_digit).count(); + if !(1..=9).contains(&digits) { + return None; + } + let d = t.get(digits..)?.chars().next()?; + if d != '.' && d != ')' { + return None; + } + let start = t.get(..digits)?.parse::().unwrap_or(1); + (Marker { ordered: true, ch: d, len: digits + 1, start }, digits + 1) + }; + let rest = t.get(len..)?; + if rest.is_empty() || rest.starts_with([' ', '\t']) { Some(m) } else { None } +} + +/// A table row without its leading and trailing pipes (an escaped last pipe stays). +fn row_body(l: &str) -> &str { + let t = l.trim(); + let t = t.strip_prefix('|').unwrap_or(t); + if let Some(before) = t.strip_suffix('|') { + let num_bs = before.chars().rev().take_while(|&c| c == '\\').count(); + if num_bs % 2 == 0 { before } else { t } + } else { + t + } +} + +/// The first `keep` cells of a table row, and how many cells it has: the rest are only counted. +pub(super) fn split_row(l: &str, keep: usize) -> (Vec, usize) { + let mut cells = Vec::new(); + let mut count = 0usize; + let mut cur = String::new(); + let mut chars = row_body(l).chars().peekable(); + while let Some(c) = chars.next() { + let full = cells.len() >= keep; + match c { + '\\' => match chars.peek() { + Some(&'|') => { + if !full { + cur.push('|'); + } + chars.next(); + } + Some(&'\\') => { + if !full { + cur.push_str("\\\\"); + } + chars.next(); + } + _ if !full => cur.push('\\'), + _ => {} + }, + '|' => { + if !full { + cells.push(std::mem::take(&mut cur).trim().to_string()); + } + count += 1; + } + _ if !full => cur.push(c), + _ => {} + } + } + if cells.len() < keep { + cells.push(cur.trim().to_string()); + } + (cells, count + 1) +} + +/// The alignments of the first `keep` cells of a delimiter row, and how many cells it has; every +/// cell is checked, so a bad one past `keep` still means it isn't a delimiter row. +fn delimiter_row(l: &str, keep: usize) -> Option<(Vec>, usize)> { + if !l.contains('-') { + return None; + } + let mut out = Vec::new(); + let mut count = 0usize; + // A valid cell is only dashes, colons and spaces, so an escaped pipe can't be in one. + for c in row_body(l).split('|') { + let c = c.trim(); + let left = c.starts_with(':'); + let right = c.ends_with(':'); + let core = c.trim_matches(':'); + if core.is_empty() || !core.chars().all(|x| x == '-') { + return None; + } + if out.len() < keep { + out.push(match (left, right) { + (true, true) => Some(Align::Center), + (false, true) => Some(Align::Right), + (true, false) => Some(Align::Left), + _ => None, + }); + } + count += 1; + } + Some((out, count)) +} + +/// Does this line start a block that interrupts a paragraph? +fn interrupts(l: &str) -> bool { + if indent(l) >= 4 { + return false; + } + let t = l.trim_start_matches([' ', '\t']); + atx_heading(t).is_some() + || thematic_break(t) + || fence(t).is_some() + || t.starts_with('>') + || t.starts_with("") { + break; + } + } + continue; + } + if t.starts_with('>') { + flush_para(&mut para, &mut out, depth, refs); + parse_quote(lines, &mut i, depth, level, refs, budget, &mut out); + continue; + } + if para.is_empty() + && t.contains('|') + && let Some((aligns, width)) = lines.get(i + 1).and_then(|l| delimiter_row(l, MAX_COLS)) + { + let (head, head_width) = split_row(t, MAX_COLS); + if head_width == width { + let mut table = FTable::default(); + let cell = |s: &str, header: bool, a: Option| { + let p = Para { align: a, inlines: parse_inlines(s, &Fmt::default(), depth, refs), ..Default::default() }; + Cell { blocks: vec![FBlock::Para(p)], header, ..Default::default() } + }; + // The document keeps no more columns or rows than this, nor cells past its budget, + // so those aren't built; their lines are still the table's. + let cols = aligns.len(); + let fits = |rows: usize| rows < MAX_ROWS && budget.get() >= cols; + let mut cut = head_width > cols; + if fits(0) { + budget.set(budget.get() - cols); + table.rows.push(head.iter().zip(&aligns).map(|(s, a)| cell(s, true, *a)).collect()); + } else { + cut = true; + } + i += 2; + while let Some(l) = lines.get(i) { + if is_blank(l) || !l.contains('|') || interrupts(l) { + break; + } + if !table.rows.is_empty() && fits(table.rows.len()) { + budget.set(budget.get() - cols); + let (mut cells, _) = split_row(l, cols); + cells.resize(cols, String::new()); + table.rows.push(cells.iter().zip(&aligns).map(|(s, a)| cell(s, false, *a)).collect()); + } else { + cut = true; + } + i += 1; + } + if cut { + model::dropped(model::Dropped::TableCells); + } + if !table.rows.is_empty() { + out.push(FBlock::Table(table)); + } + continue; + } + } + if let Some(m) = list_marker(t) + && (para.is_empty() || ((!m.ordered || m.start == 1) && !is_blank(t.get(m.len..).unwrap_or("")))) + { + flush_para(&mut para, &mut out, depth, refs); + i = parse_list(lines, i, depth, level, &mut out, refs, budget); + continue; + } + para.push(line.clone()); + i += 1; + } + flush_para(&mut para, &mut out, depth, refs); + out +} + +fn parse_indented_code(lines: &[String], i: &mut usize, out: &mut Vec) { + let mut code: Vec<&str> = Vec::new(); + while let Some(line) = lines.get(*i) { + if is_blank(line) { + code.push(""); + } else if indent(line) >= 4 { + code.push(strip_indent(line, 4)); + } else { + break; + } + *i += 1; + } + while code.last().is_some_and(|line| line.is_empty()) { + code.pop(); + } + out.extend(code.into_iter().map(code_para)); +} + +fn parse_quote(lines: &[String], i: &mut usize, depth: usize, level: u8, refs: &Refs, budget: &std::cell::Cell, out: &mut Vec) { + let mut inner = Vec::new(); + while let Some(line) = lines.get(*i) { + let trimmed = line.trim_start_matches([' ', '\t']); + if indent(line) < 4 && trimmed.starts_with('>') { + let content = &trimmed[1..]; + inner.push(content.strip_prefix(' ').unwrap_or(content).to_string()); + } else if !is_blank(line) && !interrupts(line) && inner.last().is_some_and(|line| !is_blank(line)) { + inner.push(line.clone()); + } else { + break; + } + *i += 1; + } + if depth >= MAX_NEST { + let mut p = Para::new(Kind::Quote); + p.push_text(&inner.join(" "), &Fmt::default()); + out.push(FBlock::Para(p)); + return; + } + for mut block in parse_blocks(&inner, depth + 1, level, refs, budget) { + if let FBlock::Para(p) = &mut block + && p.kind == Kind::Normal + { + p.kind = Kind::Quote; + } + out.push(block); + } +} + +fn marker_space(ind: usize, marker_len: usize, after: &str) -> (usize, usize) { + let mut cols = 0; + let mut bytes = 0; + let mut rest = after; + while let Some(c) = rest.chars().next() { + match c { + ' ' => { + cols += 1; + bytes += 1; + rest = &rest[1..]; + if cols >= 4 { + break; + } + } + '\t' => { + let col = ind + marker_len + cols; + let n = 4 - col % 4; + cols += n; + bytes += 1; + break; + } + _ => break, + } + } + (cols, bytes) +} + +/// Parse a list starting at `lines[i]`; returns the index after it. +fn parse_list(lines: &[String], mut i: usize, depth: usize, level: u8, out: &mut Vec, refs: &Refs, budget: &std::cell::Cell) -> usize { + let Some(first) = lines.get(i).and_then(|l| list_marker(l.trim_start_matches([' ', '\t']))) else { return i + 1 }; + while let Some(line) = lines.get(i) { + let ind = indent(line); + if ind >= 4 { + break; + } + let t = line.trim_start_matches([' ', '\t']); + let Some(m) = list_marker(t) else { break }; + if m.ordered != first.ordered || m.ch != first.ch { + break; + } + let after = t.get(m.len..).unwrap_or(""); + let (spaces, skip) = if indent(after) > 4 { + (1, 1) + } else if after.starts_with([' ', '\t']) { + marker_space(ind, m.len, after) + } else { + (0, 0) + }; + let content_indent = if is_blank(after) { ind + m.len + 1 } else { ind + m.len + spaces }; + let item0 = if is_blank(after) { String::new() } else { after.get(skip.min(after.len())..).unwrap_or("").to_string() }; + let mut item: Vec = vec![item0]; + i += 1; + while let Some(l) = lines.get(i) { + if is_blank(l) { + item.push(String::new()); + } else if indent(l) >= content_indent { + item.push(strip_indent(l, content_indent).to_string()); + } else if item.last().is_some_and(|x| !is_blank(x)) && !interrupts(l) && list_marker(l.trim_start_matches([' ', '\t'])).is_none() { + // Lazy continuation of the item's paragraph. + item.push(l.trim_start_matches([' ', '\t']).to_string()); + } else { + break; + } + i += 1; + } + // Blank lines at the end of an item belong between items. + while item.len() > 1 && item.last().is_some_and(String::is_empty) { + item.pop(); + } + let li = ListInfo { ordered: first.ordered, level: level.min(8), start: first.start, ..Default::default() }; + let inner = if depth >= MAX_NEST { + let mut p = Para::default(); + p.push_text(&item.join(" "), &Fmt::default()); + vec![FBlock::Para(p)] + } else { + parse_blocks(&item, depth + 1, level.saturating_add(1).min(8), refs, budget) + }; + let mut placed = false; + if inner.is_empty() { + out.push(FBlock::Para(Para { list: Some(li), ..Default::default() })); + placed = true; + } + for b in inner { + match b { + FBlock::Para(mut p) if p.list.is_none() => { + if !placed { + p.list = Some(li); + placed = true; + out.push(FBlock::Para(p)); + } else if let Some(FBlock::Para(prev)) = out.last_mut() + && prev.list.is_some_and(|x| x.level == li.level) + { + // A continuation paragraph joins the item with a line break. + prev.push_text("\n", &Fmt::default()); + prev.inlines.extend(p.inlines); + } else { + p.list = Some(li); + out.push(FBlock::Para(p)); + } + } + FBlock::Para(p) => { + if !placed { + // The item starts with a nested list: give it an empty first line. + out.push(FBlock::Para(Para { list: Some(li), ..Default::default() })); + placed = true; + } + out.push(FBlock::Para(p)); + } + other => { + placed = true; + out.push(other); + } + } + } + // Blank lines between items. + while lines.get(i).is_some_and(|l| is_blank(l)) { + let next_is_item = lines.get(i + 1).is_some_and(|l| { + indent(l) < 4 && list_marker(l.trim_start_matches([' ', '\t'])).is_some_and(|m| m.ordered == first.ordered && m.ch == first.ch) + }); + if !next_is_item { + return i; + } + i += 1; + } + } + i +} diff --git a/crates/markdown/src/export.rs b/crates/markdown/src/export.rs new file mode 100644 index 000000000..82fd952e3 --- /dev/null +++ b/crates/markdown/src/export.rs @@ -0,0 +1,448 @@ +use super::blocks::thematic_break; +use super::inlines::{is_punct, within}; +use super::{Align, Arc, Cell, Document, FBlock, FTable, Fmt, Inline, Kind, model}; + +// --------------------------------------------------------------------------------------------- +// Export + +/// Escape text for Markdown (inline context). +pub fn escape(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 8); + for c in s.chars() { + match c { + '\\' | '`' | '*' | '_' | '[' | ']' | '<' | '>' | '~' | '|' | '&' => { + out.push('\\'); + out.push(c); + } + '\u{000C}' | '\u{000E}' => {} + _ => out.push(c), + } + } + out +} + +/// Escape what would start a block at the beginning of a line. Leading spaces are kept, but at +/// most three (a tab counts as four): four would turn the line into an indented code block. +pub(super) fn escape_line_start(s: String) -> String { + let t = s.trim_start_matches([' ', '\t']); + let lead = s.get(..s.len() - t.len()).unwrap_or(""); + let cols: usize = lead.chars().map(|c| if c == '\t' { 4 } else { 1 }).sum(); + let indent = " ".repeat(cols.min(3)); + let needs = t.starts_with('#') + || t.starts_with('>') + || t.starts_with("- ") + || t.starts_with("+ ") + || t.starts_with("-\t") + || t.starts_with("+\t") + || t == "-" + || t == "+" + || t.starts_with('=') + || (t.starts_with('-') && t.trim_end_matches([' ', '\t']).chars().all(|c| c == '-')) + || thematic_break(t) + || { + let d = t.chars().take_while(char::is_ascii_digit).count(); + d > 0 + && t.get(d..).is_some_and(|r| { + r.starts_with(". ") || r.starts_with(") ") || r.starts_with(".\t") || r.starts_with(")\t") || r == "." || r == ")" + }) + }; + if !needs { + return if cols > 3 || lead.contains('\t') { format!("{indent}{t}") } else { s }; + } + if let Some(d) = t.find(['.', ')']).filter(|_| t.starts_with(|c: char| c.is_ascii_digit())) { + return format!("{indent}{}\\{}", &t[..d], &t[d..]); + } + format!("{indent}\\{t}") +} + +/// [`escape_line_start`] for every line of a body, including those after a manual line break. +fn escape_lines(body: String) -> String { + if !body.contains("\\\n") { + return escape_line_start(body); + } + body.split("\\\n").map(|l| escape_line_start(l.to_string())).collect::>().join("\\\n") +} + +fn code_span(t: &str) -> String { + let t = t.replace("\r\n", " ").replace(['\r', '\n'], " "); + let mut longest = 0; + let mut cur = 0; + for c in t.chars() { + if c == '`' { + cur += 1; + longest = longest.max(cur); + } else { + cur = 0; + } + } + let ticks = "`".repeat(longest + 1); + let pad = if t.starts_with('`') || t.ends_with('`') || (t.starts_with(' ') && t.ends_with(' ') && !t.trim().is_empty()) { " " } else { "" }; + format!("{ticks}{pad}{t}{pad}{ticks}") +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +enum Mark { + Bold, + Italic, + Strike, +} + +/// Inline spans → Markdown with properly nested markers. +pub(super) fn inlines_md(inlines: &[Inline]) -> String { + inlines_md_impl(inlines, false) +} + +fn inlines_md_table(inlines: &[Inline]) -> String { + inlines_md_impl(inlines, true) +} + +fn inlines_md_impl(inlines: &[Inline], in_table: bool) -> String { + // Split into (text, fmt) pieces with whitespace moved outside emphasis. + let mut pieces: Vec<(String, Fmt, Option)> = Vec::new(); + let eq = craft_flow::equations_as_text(inlines); + // Markdown keeps no ruby: its base text goes through the text path below. + let inlines = craft_flow::ruby_as_text(&eq); + for i in inlines.iter() { + match i { + Inline::Text(t, f) => { + let has_emph = f.bold || f.italic || f.strike; + if has_emph && !f.code { + let core = t.trim_matches([' ', '\t', '\n']); + let lead = &t[..t.len() - t.trim_start_matches([' ', '\t', '\n']).len()]; + let trail = &t[t.trim_end_matches([' ', '\t', '\n']).len()..]; + let plain = Fmt { bold: false, italic: false, strike: false, ..f.clone() }; + if !lead.is_empty() { + pieces.push((lead.to_string(), plain.clone(), None)); + } + if !core.is_empty() { + pieces.push((core.to_string(), f.clone(), None)); + } + if !trail.is_empty() && !core.is_empty() { + pieces.push((trail.to_string(), plain, None)); + } + } else { + pieces.push((t.clone(), f.clone(), None)); + } + } + Inline::Image(img) => { + let name = format!("image.{}", img.ext); + pieces.push((String::new(), Fmt::default(), Some(format!("![{}]({})", escape(&img.alt), name)))); + } + Inline::Anchor(a) => { + let safe: String = a.chars().filter(|c| c.is_alphanumeric() || matches!(c, '_' | '-' | '.' | ':')).collect(); + pieces.push((String::new(), Fmt::default(), Some(format!("")))); + } + Inline::Figure(alt) | Inline::Chart(craft_flow::Chart { alt, .. }) => { + pieces.push((String::new(), Fmt::default(), Some(format!("![{}]()", escape(alt))))) + } + Inline::Equation { .. } => {} + // ASK and SET show nothing (see `model::shows_nothing`). + Inline::Field { instr, .. } if model::shows_nothing(instr) => {} + Inline::Field { result, fmt, .. } | Inline::Control { text: result, fmt, .. } => pieces.push((result.clone(), fmt.clone(), None)), + Inline::Note { .. } + | Inline::Comment { .. } + | Inline::CommentEnd(_) + | Inline::Shape { .. } + | Inline::Group { .. } + | Inline::FieldStart(_) + | Inline::FieldEnd + | Inline::AnchorEnd(_) => {} + // Read as its base text by `ruby_as_text` above. + Inline::Ruby { .. } => {} + } + } + let mut out = String::new(); + let mut stack: Vec = Vec::new(); + let mut link: Option> = None; + let close_all = |out: &mut String, stack: &mut Vec| { + while let Some(m) = stack.pop() { + out.push_str(marker(m)); + } + }; + for (text, f, raw) in pieces { + // Links are the outermost construct; a script or other non-web target is plain text. + let f_link = f.link.clone().filter(|l| craft_wordmodel::link::is_safe_href(l)); + if f_link != link || raw.is_some() { + close_all(&mut out, &mut stack); + if let Some(l) = &link { + out.push_str(&format!("]({})", link_target(l))); + } + link = None; + if let Some(r) = raw { + out.push_str(&r); + continue; + } + if let Some(l) = f_link { + out.push('['); + link = Some(l); + } + } + let want: Vec = + [(f.bold, Mark::Bold), (f.italic, Mark::Italic), (f.strike, Mark::Strike)].iter().filter(|x| x.0).map(|x| x.1).collect(); + let body = if f.code { + if in_table { code_span(&text.replace('|', "\\|")) } else { code_span(&text) } + } else { + escape(&text).replace('\n', "\\\n").replace('\t', " ") + }; + // Close until the stack is a prefix of what we want. + let mut close = String::new(); + while let Some(pos) = stack.iter().position(|m| !want.contains(m)) { + while stack.len() > pos { + if let Some(m) = stack.pop() { + close.push_str(marker(m)); + } + } + } + let mut open = String::new(); + for m in want { + if !stack.contains(&m) { + open.push_str(marker(m)); + stack.push(m); + } + } + // CommonMark only reads a closing run after punctuation as closing when whitespace or + // punctuation follows it, and an opening run before punctuation as opening when whitespace + // or punctuation precedes it. Where a letter sits there instead, or an opening run would + // directly follow a closing run (the two would merge into one run), an empty HTML comment (which readers drop) supplies + // the punctuation. + let prev = out.chars().next_back(); + let next = body.chars().next(); + let word = |c: Option| c.is_some_and(|c| !c.is_whitespace() && !is_punct(c)); + out.push_str(&close); + let close_needs = !close.is_empty() && prev.is_some_and(is_punct) && (!open.is_empty() || word(next)); + let open_needs = !open.is_empty() && next.is_some_and(is_punct) && (!close.is_empty() || word(prev)); + if close_needs || open_needs { + out.push_str(EMPTY_COMMENT); + } + out.push_str(&open); + out.push_str(&body); + } + close_all(&mut out, &mut stack); + if let Some(l) = &link { + out.push_str(&format!("]({})", link_target(l))); + } + out +} + +/// `l` as a Markdown link destination that reads back as exactly `l`, the target +/// `is_safe_href` passed. What Markdown would decode (a backslash escape, an entity reference) +/// or end the destination on (`<`, `>`) is backslash-escaped, never deleted: deleting `<` or `>` +/// could turn an accepted relative target into `javascript:…` or `//host`. Control characters +/// can't be written in a destination and are left out, as the safety check ignores them. +fn link_target(l: &str) -> String { + let l: String = l.chars().filter(|c| !c.is_ascii_control()).collect(); + let mut out = String::with_capacity(l.len() + 2); + for (i, c) in l.char_indices() { + let rest = l.get(i + c.len_utf8()..).unwrap_or(""); + match c { + '<' | '>' => out.push('\\'), + // A backslash is literal unless punctuation (or the closing `)`/`>`) follows it. + '\\' if rest.chars().next().is_none_or(|n| n.is_ascii_punctuation()) => out.push('\\'), + '&' if entity_like(rest) => out.push('\\'), + _ => {} + } + out.push(c); + } + if l.contains(|c: char| c.is_whitespace() || matches!(c, '(' | ')' | '<' | '>')) { format!("<{out}>") } else { out } +} + +/// Whether `rest` (the text after a `&`) starts like an entity reference (`amp;`, `#106;`, +/// `#x6A;`), which a Markdown reader would decode. +fn entity_like(rest: &str) -> bool { + let Some(end) = within(rest, 34).find(';') else { return false }; + let body = rest.get(..end).unwrap_or(""); + let body = body.strip_prefix('#').map_or(body, |n| n.strip_prefix(['x', 'X']).unwrap_or(n)); + !body.is_empty() && body.chars().all(|c| c.is_ascii_alphanumeric()) +} + +/// Inline HTML that Markdown readers drop, used to separate emphasis markers from letters. +const EMPTY_COMMENT: &str = ""; + +fn marker(m: Mark) -> &'static str { + match m { + Mark::Bold => "**", + Mark::Italic => "*", + Mark::Strike => "~~", + } +} + +pub(super) fn table_md(t: &FTable, out: &mut Vec) { + let cols = t.cols().max(1); + let cell_md = |c: &Cell| -> String { + let mut parts = Vec::new(); + for b in &c.blocks { + match b { + FBlock::Para(p) => parts.push(inlines_md_table(&p.inlines).replace("\\\n", "
")), + FBlock::Table(inner) => parts.push(escape(&inner.rows.iter().flat_map(|r| r.iter().map(Cell::text)).collect::>().join(" "))), + } + } + parts.join("
").replace('\n', " ") + }; + let align_of = |c: &Cell| c.blocks.iter().find_map(|b| if let FBlock::Para(p) = b { Some(p.align) } else { None }).flatten(); + let row_md = |r: &Vec| -> Vec { + let mut v: Vec = Vec::new(); + for c in r { + v.push(if c.covered { String::new() } else { cell_md(c) }); + for _ in 1..c.colspan.clamp(1, 63) { + v.push(String::new()); + } + } + v.resize(cols, String::new()); + v + }; + let Some(head) = t.rows.first() else { return }; + out.push(format!("| {} |", row_md(head).join(" | "))); + let mut aligns: Vec> = Vec::new(); + for c in head { + aligns.push(align_of(c)); + for _ in 1..c.colspan.clamp(1, 63) { + aligns.push(None); + } + } + aligns.resize(cols, None); + let delim: Vec<&str> = aligns + .iter() + .map(|a| match a { + Some(Align::Center) => ":---:", + Some(Align::Right) => "---:", + Some(Align::Left) => ":---", + _ => "---", + }) + .collect(); + out.push(format!("| {} |", delim.join(" | "))); + for r in t.rows.iter().skip(1) { + out.push(format!("| {} |", row_md(r).join(" | "))); + } +} + +/// Markdown text of a document. +pub fn export(doc: &Document) -> String { + let flow = model::from_doc(doc); + let mut out: Vec = Vec::new(); + write_blocks(&flow.blocks, &mut out); + let mut s = out.join("\n"); + while s.ends_with("\n\n") { + s.pop(); + } + if !s.ends_with('\n') { + s.push('\n'); + } + s +} + +/// A heading's text on one line, with a trailing run of '#' escaped so a reader does not take it +/// for the optional closing sequence (`## Item #` reads back as "Item"). +fn heading_text(body: &str) -> String { + let mut s = body.replace("\\\n", " "); + let t = s.trim_end(); + let core = t.trim_end_matches('#'); + if core.len() != t.len() && (core.is_empty() || core.ends_with([' ', '\t'])) { + s.insert(core.len(), '\\'); + } + s +} + +pub(super) fn write_blocks(blocks: &[FBlock], out: &mut Vec) { + let mut i = 0; + let mut counters = [0u32; 9]; + let mut prev_list = false; + let mut prev_top_ordered = false; + while let Some(b) = blocks.get(i) { + match b { + FBlock::Table(t) => { + table_md(t, out); + out.push(String::new()); + prev_list = false; + } + FBlock::Para(p) if p.kind == Kind::Code && p.list.is_none() => { + let mut lines = Vec::new(); + while let Some(FBlock::Para(q)) = blocks.get(i) { + if q.kind != Kind::Code || q.list.is_some() { + break; + } + lines.push(q.text()); + i += 1; + } + let longest = lines.iter().map(|l| l.chars().filter(|c| *c == '`').count()).max().unwrap_or(0); + // The fence must be longer than the longest run in the content, or a content + // line is read back as the closing fence (the `.min(64)` cap did exactly that). + let fence = "`".repeat(3.max(longest + 1)); + out.push(fence.clone()); + out.extend(lines); + out.push(fence); + out.push(String::new()); + prev_list = false; + continue; + } + FBlock::Para(p) => { + let body = inlines_md(&p.inlines); + match p.list { + Some(li) => { + let lv = li.level.min(8) as usize; + if !prev_list || (lv == 0 && prev_top_ordered != li.ordered) { + counters = [0; 9]; + } + if lv == 0 { + prev_top_ordered = li.ordered; + } + let n = counters.get_mut(lv).map(|c| { + if *c == 0 { + *c = li.start.saturating_sub(1); + } + *c = c.saturating_add(1); + *c + }); + for d in counters.iter_mut().skip(lv + 1) { + *d = 0; + } + let marker = if li.ordered { format!("{}.", n.unwrap_or(1)) } else { "-".to_string() }; + let pad = " ".repeat(4 * lv); + let cont = " ".repeat(4 * lv + marker.len() + 1); + let body = if p.kind == Kind::Code { + code_span(&p.text()) + } else { + let body = escape_lines(body).replace("\\\n", &format!("\\\n{cont}")); + // The bullet marker can turn a short run of hyphens into a rule. + if thematic_break(&format!("{marker} {body}")) { format!("\\{body}") } else { body } + }; + let line = format!("{pad}{marker} {body}").trim_end().to_string(); + // A list inside a block quote stays in it: every line of the item, its + // continuation lines included, takes the quote marker. + let line = if p.kind == Kind::Quote { format!("> {}", line.replace('\n', "\n> ")) } else { line }; + out.push(line); + prev_list = true; + // A list going into or out of a block quote is a new block: without the + // blank line, an unquoted item would be read back into the quote. + let quoted = |k: Kind| k == Kind::Quote; + let next_is_list = matches!(blocks.get(i + 1), Some(FBlock::Para(q)) if q.list.is_some() && quoted(q.kind) == quoted(p.kind)); + if !next_is_list { + out.push(String::new()); + } + } + None => { + prev_list = false; + let line = match p.kind { + Kind::Heading(n) => format!("{} {}", "#".repeat(n.clamp(1, 6) as usize), heading_text(&body)), + Kind::Title => format!("# {}", heading_text(&body)), + Kind::Rule => "---".to_string(), + Kind::Quote => { + let b = escape_lines(body); + format!("> {}", b.replace("\\\n", "\\\n> ")) + } + _ => { + if body.is_empty() { + i += 1; + continue; + } + escape_lines(body) + } + }; + out.push(line); + out.push(String::new()); + } + } + } + } + i += 1; + } +} diff --git a/crates/markdown/src/inlines.rs b/crates/markdown/src/inlines.rs new file mode 100644 index 000000000..99c397246 --- /dev/null +++ b/crates/markdown/src/inlines.rs @@ -0,0 +1,595 @@ +use super::blocks::{MAX_NEST, Refs, ref_label}; +use super::{Arc, Fmt, HashMap, Inline, decode_entity, make_img, model}; + +// ---- inlines ---- + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Emph { + Em, + Strong, + Strike, +} + +enum Node { + Text(String, Fmt), + Inl(Inline), + Delim { ch: char, orig: usize, left: usize, open: bool, close: bool, opens: Vec, closes: Vec }, +} + +pub(super) fn is_punct(c: char) -> bool { + c.is_ascii_punctuation() || (!c.is_alphanumeric() && !c.is_whitespace() && !c.is_control()) +} + +/// Parse inline Markdown into formatted spans (on top of `base`). +pub fn parse_inlines(s: &str, base: &Fmt, depth: usize, refs: &Refs) -> Vec { + let nodes = tokenize(s, base, depth, refs); + emit(nodes) +} + +fn push_text(nodes: &mut Vec, s: &str, f: &Fmt) { + if s.is_empty() { + return; + } + if let Some(Node::Text(t, pf)) = nodes.last_mut() + && pf == f + { + t.push_str(s); + return; + } + nodes.push(Node::Text(s.to_string(), f.clone())); +} + +/// The closing `]` matching the `[` at byte `open` (escapes and code spans respected). Gives up +/// after looking `max` bytes ahead. Records in `known` every bracket pair it passes, and every `[` +/// still open at the end of the text: a scan from one of those would take the same path. +fn matching_bracket(s: &str, open: usize, max: usize, ticks: &Backticks, known: &mut HashMap>) -> Option { + let b = s.as_bytes(); + let mut opens: Vec = Vec::new(); + let mut i = open; + while let Some(&c) = b.get(i) { + if i - open > max { + return None; + } + match c { + b'\\' => i += 1, + b'`' => { + let n = b.get(i..).map(|r| r.iter().take_while(|x| **x == b'`').count()).unwrap_or(1); + if let Some(end) = ticks.find(i + n, n) { + i = end + n - 1; + } else { + i += n - 1; + } + } + b'[' => opens.push(i), + b']' => { + if let Some(o) = opens.pop() { + known.insert(o, Some(i)); + } + if opens.is_empty() { + return Some(i); + } + } + _ => {} + } + i += 1; + } + for o in opens { + known.insert(o, None); + } + None +} + +/// Where a text's runs of backticks start, by run length. Finding the run that closes a code span +/// is then a lookup: scanning the rest of the text for it, again for every run nothing closes +/// (each of a different length) and from every link bracket over them, was superlinear. +struct Backticks(HashMap>); + +impl Backticks { + fn new(s: &str) -> Self { + let b = s.as_bytes(); + let mut runs: HashMap> = HashMap::new(); + let mut i = 0; + while let Some(&c) = b.get(i) { + if c == b'`' { + let k = b.get(i..).map(|r| r.iter().take_while(|x| **x == b'`').count()).unwrap_or(1); + runs.entry(k).or_default().push(i); + i += k; + } else { + i += 1; + } + } + Self(runs) + } + + /// Byte index of a run of exactly `n` backticks at or after `from` (which is not inside a run). + fn find(&self, from: usize, n: usize) -> Option { + let starts = self.0.get(&n)?; + starts.get(starts.partition_point(|&p| p < from)).copied() + } +} + +struct LinkScan<'a> { + text: &'a str, + refs: &'a Refs, + ticks: &'a Backticks, + known: HashMap>, + budget: usize, +} + +impl<'a> LinkScan<'a> { + fn parse_at(&mut self, open: usize) -> Option<(&'a str, Arc, usize)> { + let bytes = self.text.as_bytes(); + let close = match self.known.get(&open).copied() { + Some(found) => found, + None if self.budget > 0 => { + let found = matching_bracket(self.text, open, self.budget, self.ticks, &mut self.known); + self.budget = self.budget.saturating_sub(found.unwrap_or(self.text.len()).saturating_sub(open)); + found + } + None => None, + }?; + if self.budget == 0 { + return None; + } + let label_text = self.text.get(open + 1..close)?; + if bytes.get(close + 1) == Some(&b'(') { + let destination = link_dest(self.text, close + 1); + self.budget = self.budget.saturating_sub(destination.as_ref().map_or(self.text.len(), |d| d.1).saturating_sub(close)); + let (destination, end) = destination?; + return Some((label_text, Arc::from(destination), end)); + } + // Full and shortcut reference links use definitions collected before inline parsing. + if self.refs.is_empty() { + return None; + } + let (label, end) = match self.text.get(close + 1..).and_then(|rest| rest.strip_prefix('[')) { + Some(rest) => { + let ahead = within(rest, self.budget); + let shut = ahead.find(']'); + self.budget = self.budget.saturating_sub(shut.map_or(ahead.len(), |n| n + 1)); + let shut = shut?; + let inner = rest.get(..shut)?; + let label = if inner.trim().is_empty() { label_text } else { inner }; + (label, close + 2 + shut + 1) + } + None => (label_text, close + 1), + }; + // A cached bracket pair still costs a pass over its label. Without charging it, + // nested undefined references normalize every suffix again. + if label.len() > self.budget { + self.budget = 0; + return None; + } + self.budget -= label.len(); + Some((label_text, self.refs.get(&ref_label(label))?.clone(), end)) + } +} + +/// Parse `(dest "title")` at byte `at` (which holds `(`): (destination, index after `)`). +fn link_dest(s: &str, at: usize) -> Option<(String, usize)> { + let rest = s.get(at..)?.strip_prefix('(')?; + let mut i = at + 1; + let ws = rest.len() - rest.trim_start().len(); + i += ws; + let r = s.get(i..)?; + let (dest, used) = if let Some(inner) = r.strip_prefix('<') { + // The first `>` not escaped by a backslash ends it; a line break first means no link. + let b = inner.as_bytes(); + let mut k = 0; + let end = loop { + match b.get(k)? { + b'\\' => k += 1, + b'>' => break k, + b'\n' => return None, + _ => {} + } + k += 1; + }; + (inner.get(..end)?.to_string(), end + 2) + } else { + let mut depth = 0i32; + let mut end = r.len(); + let mut chars = r.char_indices().peekable(); + while let Some((k, c)) = chars.next() { + match c { + '\\' => { + chars.next(); + } + '(' => depth += 1, + ')' if depth == 0 => { + end = k; + break; + } + ')' => depth -= 1, + c if c.is_whitespace() => { + end = k; + break; + } + _ => {} + } + } + (r.get(..end)?.to_string(), end) + }; + i += used; + let r = s.get(i..)?; + let ws = r.len() - r.trim_start().len(); + i += ws; + let r = s.get(i..)?; + if let Some(q) = r.chars().next().filter(|c| matches!(c, '"' | '\'' | '(')) { + let close = if q == '(' { ')' } else { q }; + let end = r.get(1..)?.find(close)?; + i += end + 2; + let r = s.get(i..)?; + i += r.len() - r.trim_start().len(); + } + if s.get(i..)?.starts_with(')') { Some((unescape(&dest), i + 1)) } else { None } +} + +fn unescape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + if c == '\\' + && let Some(n) = chars.peek().copied().filter(char::is_ascii_punctuation) + { + out.push(n); + chars.next(); + } else { + out.push(c); + } + } + out +} + +/// The value of attribute `name` in an HTML tag's text. +fn tag_attr(tag: &str, name: &str) -> Option { + let lower = tag.to_ascii_lowercase(); + let mut from = 0; + while let Some(k) = lower.get(from..)?.find(name) { + let at = from + k; + let before = lower.get(..at)?.chars().next_back(); + let after = lower.get(at + name.len()..)?.trim_start(); + if before.is_some_and(char::is_whitespace) && after.starts_with('=') { + let v = after.get(1..)?.trim_start(); + let q = v.chars().next()?; + return Some(if q == '"' || q == '\'' { + let inner = v.get(1..)?; + let off = tag.len() - inner.len(); + let end = inner.find(q)?; + tag.get(off..off + end)?.to_string() + } else { + let end = v.find(|c: char| c.is_whitespace() || c == '>').unwrap_or(v.len()); + let off = tag.len() - v.len(); + tag.get(off..off + end)?.to_string() + }); + } + from = at + name.len(); + } + None +} + +fn tokenize(s: &str, base: &Fmt, depth: usize, refs: &Refs) -> Vec { + let mut nodes: Vec = Vec::new(); + // Bytes the link/image scans and reference-label lookups may read in total. Unmatched + // brackets and nested undefined references would otherwise rescan the rest of the text, + // which is quadratic on hostile input. Once the budget is gone, further brackets are plain + // text. A scan pays for what it reads; earlier bracket matches are looked up in `known`. + let ticks = Backticks::new(s); + let mut links = LinkScan { text: s, refs, ticks: &ticks, known: HashMap::new(), budget: s.len().saturating_mul(8).saturating_add(8192) }; + let b = s.as_bytes(); + let mut i = 0usize; + let mut text_start = 0usize; + macro_rules! flush { + () => { + if text_start < i { + push_text(&mut nodes, s.get(text_start..i).unwrap_or(""), base); + } + }; + } + while let Some(&c) = b.get(i) { + match c { + b'\\' => { + flush!(); + match s.get(i + 1..).and_then(|r| r.chars().next()) { + Some(n) if n.is_ascii_punctuation() => { + push_text(&mut nodes, &n.to_string(), base); + i += 2; + } + _ => { + push_text(&mut nodes, "\\", base); + i += 1; + } + } + text_start = i; + } + b'`' => { + flush!(); + let (code, fmt, end) = code_span(s, i, base, &ticks); + push_text(&mut nodes, &code, &fmt); + i = end; + text_start = i; + } + b'*' | b'_' | b'~' => { + flush!(); + let ch = c as char; + let n = b.get(i..).map(|r| r.iter().take_while(|x| **x == c).count()).unwrap_or(1); + let before = s.get(..i).and_then(|r| r.chars().next_back()).unwrap_or(' '); + let after = s.get(i + n..).and_then(|r| r.chars().next()).unwrap_or(' '); + let lf = !after.is_whitespace() && (!is_punct(after) || before.is_whitespace() || is_punct(before)); + let rf = !before.is_whitespace() && (!is_punct(before) || after.is_whitespace() || is_punct(after)); + let (open, close) = match ch { + '_' => (lf && (!rf || is_punct(before)), rf && (!lf || is_punct(after))), + '~' if n != 2 => (false, false), + _ => (lf, rf), + }; + if open || close { + nodes.push(Node::Delim { ch, orig: n, left: n, open, close, opens: Vec::new(), closes: Vec::new() }); + } else { + push_text(&mut nodes, s.get(i..i + n).unwrap_or(""), base); + } + i += n; + text_start = i; + } + b'!' | b'[' if depth < MAX_NEST => { + let img = c == b'!'; + let open = if img { i + 1 } else { i }; + let parsed = if b.get(open) == Some(&b'[') { links.parse_at(open) } else { None }; + match parsed { + Some((inner, dest, end)) if img => { + flush!(); + let alt: String = parse_inlines(inner, &Fmt::default(), depth + 1, refs) + .iter() + .filter_map(|x| if let Inline::Text(t, _) = x { Some(t.as_str()) } else { None }) + .collect(); + match model::data_uri(&dest).and_then(|d| make_img(d, None, None, &alt)) { + Some(im) => nodes.push(Node::Inl(Inline::Image(im))), + None => push_text(&mut nodes, &alt, base), + } + i = end; + text_start = i; + } + Some((inner, dest, end)) if base.link.is_none() => { + flush!(); + let mut f = base.clone(); + // A destination too long to keep leaves its text unlinked. + f.link = (dest.len() <= model::MAX_LINK).then_some(dest); + if f.link.is_none() { + model::dropped(model::Dropped::Link); + } + for x in parse_inlines(inner, &f, depth + 1, refs) { + nodes.push(Node::Inl(x)); + } + i = end; + text_start = i; + } + _ => i += 1, + } + } + b'<' => { + if let Some((end, parsed)) = angle_inlines(s, i, base) { + flush!(); + nodes.extend(parsed); + i = end; + text_start = end; + } else { + i += 1; + } + } + b'&' => { + let rest = s.get(i..).unwrap_or(""); + match within(rest, 33).find(';').and_then(|k| decode_entity(rest.get(1..k)?).map(|ch| (ch, k))) { + Some((ch, k)) => { + flush!(); + push_text(&mut nodes, &ch, base); + i += k + 1; + text_start = i; + } + None => i += 1, + } + } + _ => i += 1, + } + } + flush!(); + nodes +} + +fn angle_inlines(s: &str, start: usize, base: &Fmt) -> Option<(usize, Vec)> { + if let Some(pos) = s.get(start..).and_then(|rest| rest.strip_prefix("")) { + return Some((start + 4 + pos + 3, Vec::new())); + } + let rest = s.get(start + 1..).unwrap_or(""); + let tag_end = within(rest, MAX_TAG).find('>')?; + let inner = rest.get(..tag_end)?; + let lower = inner.to_ascii_lowercase(); + let end = start + inner.len() + 2; + let mut nodes = Vec::new(); + if !inner.is_empty() + && !inner.contains([' ', '<', '\n']) + && (inner.contains("://") || lower.starts_with("mailto:") || (inner.contains('@') && !inner.contains('/'))) + { + let mut fmt = base.clone(); + fmt.link = Some(if inner.contains('@') && !inner.contains(':') { format!("mailto:{inner}").into() } else { inner.into() }); + push_text(&mut nodes, inner, &fmt); + Some((end, nodes)) + } else if matches!(lower.trim_end_matches('/').trim(), "br") { + push_text(&mut nodes, "\n", base); + Some((end, nodes)) + } else if lower.starts_with("a ") && rest.get(inner.len() + 1..).is_some_and(|tail| tail.to_ascii_lowercase().starts_with("")) { + if let Some(name) = tag_attr(inner, "id").or_else(|| tag_attr(inner, "name")) { + nodes.push(Node::Inl(Inline::Anchor(name))); + } + Some((end + 4, nodes)) + } else { + None + } +} + +fn code_span(s: &str, start: usize, base: &Fmt, ticks: &Backticks) -> (String, Fmt, usize) { + let bytes = s.as_bytes(); + let count = bytes.get(start..).map(|tail| tail.iter().take_while(|byte| **byte == b'`').count()).unwrap_or(1); + let Some(end) = ticks.find(start + count, count) else { + return (s.get(start..start + count).unwrap_or("").to_string(), base.clone(), start + count); + }; + let mut code = s.get(start + count..end).unwrap_or("").replace('\n', " "); + if code.len() >= 2 && code.starts_with(' ') && code.ends_with(' ') && !code.trim().is_empty() { + code = code[1..code.len() - 1].to_string(); + } + let mut fmt = base.clone(); + fmt.code = true; + (code, fmt, end + count) +} + +/// Longest `<...>` tag or autolink taken as inline HTML. +const MAX_TAG: usize = 4096; + +/// The first `n` bytes of `s` (fewer if that would cut a character), so a search for a closing +/// character stays short however long the text is. +pub(super) fn within(s: &str, n: usize) -> &str { + let mut end = n.min(s.len()); + while !s.is_char_boundary(end) { + end -= 1; + } + s.get(..end).unwrap_or("") +} + +fn emph_index(ch: char) -> usize { + match ch { + '*' => 0, + '_' => 1, + _ => 2, + } +} + +/// The CommonMark "process emphasis" pass over the delimiter runs, then flattening into spans. +fn emit(mut nodes: Vec) -> Vec { + // Delimiter stack of node indexes; bottoms per (char, closer length % 3, closer can open). + let mut stack: Vec = Vec::new(); + let mut bottom = [[[None::; 2]; 3]; 3]; + for c in 0..nodes.len() { + let (ch, corig, copen, cclose) = match nodes.get(c) { + Some(Node::Delim { ch, orig, open, close, .. }) => (*ch, *orig, *open, *close), + _ => continue, + }; + if cclose { + loop { + let cleft = match nodes.get(c) { + Some(Node::Delim { left, .. }) => *left, + _ => 0, + }; + if cleft == 0 { + break; + } + let bot = bottom.get(emph_index(ch)).and_then(|x| x.get(corig % 3)).and_then(|x| x.get(copen as usize)).copied().flatten(); + let mut found = None; + for (k, &o) in stack.iter().enumerate().rev() { + if bot.is_some_and(|b| o <= b) { + break; + } + if let Some(Node::Delim { ch: och, orig: oorig, left: oleft, open: oopen, close: oclose, .. }) = nodes.get(o) { + if *och != ch || !*oopen || *oleft == 0 { + continue; + } + if ch == '~' && (*oleft < 2 || cleft < 2) { + continue; + } + let odd = (*oclose || copen) && (oorig + corig) % 3 == 0 && !(oorig % 3 == 0 && corig % 3 == 0); + if odd { + continue; + } + found = Some((k, o, *oleft)); + break; + } + } + let Some((k, o, oleft)) = found else { + if let Some(slot) = bottom.get_mut(emph_index(ch)).and_then(|x| x.get_mut(corig % 3)).and_then(|x| x.get_mut(copen as usize)) { + *slot = Some(c.saturating_sub(1)); + } + break; + }; + let (used, kind) = if ch == '~' { + (2, Emph::Strike) + } else if oleft >= 2 && cleft >= 2 { + (2, Emph::Strong) + } else { + (1, Emph::Em) + }; + if let Some(Node::Delim { left, opens, .. }) = nodes.get_mut(o) { + *left -= used; + opens.push(kind); + } + if let Some(Node::Delim { left, closes, .. }) = nodes.get_mut(c) { + *left -= used; + closes.push(kind); + } + // Delimiters between opener and closer can no longer match. + stack.truncate(k + 1); + if oleft - used == 0 { + stack.truncate(k); + } + } + } + let left = match nodes.get(c) { + Some(Node::Delim { left, .. }) => *left, + _ => 0, + }; + if copen && left > 0 { + stack.push(c); + } + } + // Flatten. + let mut out: Vec = Vec::new(); + let (mut em, mut strong, mut strike) = (0usize, 0usize, 0usize); + let push = |out: &mut Vec, t: &str, f: &Fmt, em: usize, strong: usize, strike: usize| { + if t.is_empty() { + return; + } + let mut f = f.clone(); + f.italic |= em > 0; + f.bold |= strong > 0; + f.strike |= strike > 0; + if let Some(Inline::Text(pt, pf)) = out.last_mut() + && *pf == f + { + pt.push_str(t); + return; + } + out.push(Inline::Text(t.to_string(), f)); + }; + // Delimiter literals take the formatting (link, …) of the text before them: the last text + // node, tracked here rather than searched for backwards from every delimiter (quadratic on + // a paragraph of delimiters). + let mut last_text: Option<&Fmt> = None; + for n in &nodes { + match n { + Node::Text(t, f) => { + push(&mut out, t, f, em, strong, strike); + last_text = Some(f); + } + Node::Inl(Inline::Text(t, f)) => push(&mut out, t, f, em, strong, strike), + Node::Inl(other) => out.push(other.clone()), + Node::Delim { ch, left, opens, closes, .. } => { + for e in closes { + match e { + Emph::Em => em = em.saturating_sub(1), + Emph::Strong => strong = strong.saturating_sub(1), + Emph::Strike => strike = strike.saturating_sub(1), + } + } + if *left > 0 { + let lit: String = std::iter::repeat_n(*ch, *left).collect(); + let f = last_text.map_or_else(Fmt::default, |f| Fmt { bold: false, italic: false, strike: false, ..f.clone() }); + push(&mut out, &lit, &f, em, strong, strike); + } + for e in opens { + match e { + Emph::Em => em += 1, + Emph::Strong => strong += 1, + Emph::Strike => strike += 1, + } + } + } + } + } + out +} diff --git a/crates/markdown/src/lib.rs b/crates/markdown/src/lib.rs new file mode 100644 index 000000000..e7fc9abfc --- /dev/null +++ b/crates/markdown/src/lib.rs @@ -0,0 +1,48 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +// The code came from an app whose lints allow this. +#![allow(clippy::type_complexity)] +//! Markdown: a CommonMark subset plus GitHub tables and strikethrough. +//! +//! Import: ATX and setext headings, paragraphs with soft/hard breaks, block quotes (→ Quote +//! style), fenced and indented code (→ one Code paragraph per line), thematic breaks (→ a +//! bottom-border paragraph), bullet and ordered lists (nested by indentation), GFM pipe tables, +//! and inlines: emphasis/strong (the CommonMark delimiter algorithm), `~~strike~~`, code spans, +//! inline links, autolinks, images (`data:` URIs become pictures, others their alt text), +//! backslash escapes, entities, HTML comments (``), `
` and `` anchors. +//! +//! Export writes the same constructs; pictures become `![alt](media-name)` placeholders. + +#[cfg(doctest)] +#[doc = include_str!("../README.md")] +struct ReadmeExamples; + +use std::collections::HashMap; +use std::sync::Arc; + +use craft_wordmodel::table::{MAX_COLS, MAX_ROWS}; +use craft_wordmodel::{Align, Document}; + +use craft_flow::{self as model, Cell, FBlock, FTable, Fmt, Inline, Kind, ListInfo, Para, make_img}; + +mod blocks; +mod export; +mod inlines; + +pub use craft_flow::MAX_TABLE_CELLS; + +/// The text of an entity body (`amp`, `#38`, `#x26`). +fn decode_entity(body: &str) -> Option { + craft_html::decode_entity(body).map(String::from) +} + +pub use blocks::{Refs, import, import_on, parse}; +pub use export::escape; +pub use export::export; +#[cfg(test)] +use export::{escape_line_start, inlines_md, table_md, write_blocks}; +pub use inlines::parse_inlines; + +#[cfg(test)] +use blocks::split_row; +#[cfg(test)] +mod tests; diff --git a/crates/markdown/src/tests.rs b/crates/markdown/src/tests.rs new file mode 100644 index 000000000..96ad1dc3a --- /dev/null +++ b/crates/markdown/src/tests.rs @@ -0,0 +1,795 @@ +use super::*; + +fn inl(s: &str) -> Vec { + parse_inlines(s, &Fmt::default(), 0, &Refs::new()) +} + +fn spans(s: &str) -> Vec<(String, bool, bool)> { + inl(s).into_iter().filter_map(|i| if let Inline::Text(t, f) = i { Some((t, f.bold, f.italic)) } else { None }).collect() +} + +#[test] +fn emphasis() { + assert_eq!(spans("a *b* c"), vec![("a ".into(), false, false), ("b".into(), false, true), (" c".into(), false, false)]); + assert_eq!(spans("**b**"), vec![("b".into(), true, false)]); + assert_eq!(spans("***x***"), vec![("x".into(), true, true)]); + assert_eq!(spans("*a **b** c*"), vec![("a ".into(), false, true), ("b".into(), true, true), (" c".into(), false, true)]); + assert_eq!(spans("snake_case_name"), vec![("snake_case_name".into(), false, false)]); + assert_eq!(spans("2 * 3 * 4"), vec![("2 * 3 * 4".into(), false, false)]); + assert_eq!(spans("**unclosed"), vec![("**unclosed".into(), false, false)]); + let s = inl("~~gone~~"); + assert!(matches!(&s[0], Inline::Text(t, f) if t == "gone" && f.strike)); +} + +#[test] +fn links_code_escapes() { + let s = inl("see [the *site*](http://x.org \"T\") and `a*b*` \\*lit\\* & "); + let link: Vec<_> = s + .iter() + .filter_map(|i| if let Inline::Text(t, f) = i { f.link.as_ref().map(|l| (t.clone(), l.to_string(), f.italic)) } else { None }) + .collect(); + assert_eq!(link[0], ("the ".into(), "http://x.org".into(), false)); + assert_eq!(link[1], ("site".into(), "http://x.org".into(), true)); + assert!(link.iter().any(|l| l.1 == "https://y.com")); + assert!(s.iter().any(|i| matches!(i, Inline::Text(t, f) if t == "a*b*" && f.code))); + let text: String = s.iter().filter_map(|i| if let Inline::Text(t, _) = i { Some(t.as_str()) } else { None }).collect(); + assert!(text.contains("*lit* & "), "{text}"); +} + +/// Reference links: `[text][label]`, `[text][]` and `[text]` take their destination from a +/// definition line, and the definition itself leaves the document. Before, none of the three +/// was a link and the definition stayed as a paragraph of its own (#3238). +#[test] +fn reference_links_resolve_and_their_definitions_leave_the_body() { + let links = |md: &str| -> Vec<(String, String)> { + match &*parse(md).blocks { + [FBlock::Para(p), ..] => p + .inlines + .iter() + .filter_map(|x| match x { + Inline::Text(t, f) => f.link.as_deref().map(|l| (t.clone(), l.to_string())), + _ => None, + }) + .collect(), + _ => vec![], + } + }; + let def = "\n\n[ref1]: https://example.com/ref\n"; + let to_ref = |text: &str| vec![(text.to_string(), "https://example.com/ref".to_string())]; + + assert_eq!(links(&format!("[click here][ref1]{def}")), to_ref("click here"), "full"); + assert_eq!(links(&format!("[ref1][]{def}")), to_ref("ref1"), "collapsed"); + assert_eq!(links(&format!("[ref1]{def}")), to_ref("ref1"), "shortcut"); + // Labels match case-folded with runs of white space collapsed (CommonMark §4.7). + assert_eq!(links(&format!("[click here][ REF1 ]{def}")), to_ref("click here"), "folded label"); + // An undefined label is not a link: an ordinary bracketed phrase stays text. + assert_eq!(links(&format!("[nope]{def}")), vec![], "undefined"); + // Inline links are unaffected. + assert_eq!(links("[x](https://inline.example)"), vec![("x".into(), "https://inline.example".into())], "inline"); + + // The definition is not a paragraph, and its text is nowhere in the body. + let doc = import(&format!("Reference link: [click here][ref1]{def}")); + let text = doc.plain_text(craft_wordmodel::StoryRef::Body); + assert_eq!(text, "Reference link: click here", "{text:?}"); + + // A definition inside a fence is code, not a definition. + let fenced = "```\n[ref1]: https://example.com/ref\n```\n\n[ref1]\n"; + let body = import(fenced).plain_text(craft_wordmodel::StoryRef::Body); + assert!(body.contains("[ref1]: https://example.com/ref"), "the fenced definition is still code: {body:?}"); + assert_eq!(links(fenced), vec![], "a fenced definition defines nothing"); +} + +/// Cached bracket pairs must not make undefined shortcut references normalize every nested +/// suffix again. One unrelated definition keeps this on the reference-lookup path. +#[test] +fn nested_undefined_reference_labels_have_bounded_work() { + let literal = format!("prefix {}x{} suffix", "[".repeat(128_000), "]".repeat(128_000)); + let src = format!("{literal}\n\n[other]: https://example.com/other\n"); + let started = std::time::Instant::now(); + let doc = import(&src); + let elapsed = started.elapsed(); + assert_eq!(doc.plain_text(craft_wordmodel::StoryRef::Body), literal); + assert!(elapsed < std::time::Duration::from_secs(10), "nested undefined references took {elapsed:?}"); +} + +#[test] +fn reference_lookup_keeps_long_unicode_labels_and_nested_text() { + // The work guard is not a label-length cap. Collapsed/full/shortcut references keep their + // case and whitespace matching, even when lowercasing changes the number of UTF-8 bytes. + let label = format!("{}İ END", "Écho ".repeat(1500)); + let key = label.to_lowercase(); + let src = format!("[{label}] [{label}][] [full][{label}]\n\n[{key}]: https://example.com/long\n"); + let flow = parse(&src); + let FBlock::Para(p) = &flow.blocks[0] else { panic!("paragraph") }; + let links: Vec<_> = p + .inlines + .iter() + .filter_map(|x| match x { + Inline::Text(t, f) if f.link.is_some() => Some((t.as_str(), f.link.as_deref())), + _ => None, + }) + .collect(); + assert_eq!( + links, + vec![ + (label.as_str(), Some("https://example.com/long")), + (label.as_str(), Some("https://example.com/long")), + ("full", Some("https://example.com/long")) + ] + ); + + let refs = Refs::from([("known".into(), Arc::from("https://example.com/known"))]); + let out = parse_inlines("[outer [known]] tail [inline](https://example.com/inline)", &Fmt::default(), 0, &refs); + let text: String = out.iter().filter_map(|x| if let Inline::Text(t, _) = x { Some(t.as_str()) } else { None }).collect(); + assert_eq!(text, "[outer known] tail inline"); + assert!(out.iter().any(|x| matches!(x, Inline::Text(t, f) if t == "known" && f.link.as_deref() == Some("https://example.com/known")))); + assert!(out.iter().any(|x| matches!(x, Inline::Text(t, f) if t == "inline" && f.link.as_deref() == Some("https://example.com/inline")))); +} + +/// A definition has to start a block (CommonMark §4.7), and a fence opens and closes by the +/// same rules the block parser uses. Reading a line without that context deleted text: a +/// definition-shaped line inside a paragraph, and every such line inside a code block whose +/// fence this pass thought had closed. +#[test] +fn a_definition_shaped_line_that_is_not_a_definition_stays_in_the_document() { + let body = |md: &str| import(md).plain_text(craft_wordmodel::StoryRef::Body); + + // A definition cannot interrupt a paragraph: this is the author's text, lines and all. + let para = "Status update\n[Warning]: unstable\nmore text\n"; + assert_eq!(body(para), "Status update [Warning]: unstable more text", "{:?}", body(para)); + + // `` ```rust `` holds an info string, so it does not close a fence: the definition-shaped + // line after it is still code. + let code = "```\n```rust\n[foo]: /bar\n```\n\nSee [foo].\n"; + assert!(body(code).contains("[foo]: /bar"), "{:?}", body(code)); + assert!(body(code).ends_with("See [foo]."), "nothing defines foo: {:?}", body(code)); + + // Four spaces of indentation is code, not a fence, so the definition that follows is + // taken and the reference resolves. + let indented = " ```\n\n[x]: /y\n\nSee [x].\n"; + let doc = import(indented); + let text = doc.plain_text(craft_wordmodel::StoryRef::Body); + assert!(text.contains("```") && text.ends_with("See x."), "{text:?}"); + + // A definition still gives up its line after a blank line, at the start of the document, + // after a heading, and after another definition. + let starts = "[a]: /a\n[b]: /b\n\n# Head\n[c]: /c\n\nSee [a], [b], [c].\n"; + assert_eq!(body(starts), "Head\nSee a, b, c.", "{:?}", body(starts)); + + // A footnote definition is not a link definition: it stays in the body. + let footnote = "x[^1]\n\n[^1]: Note\n"; + assert!(body(footnote).contains("[^1]: Note"), "{:?}", body(footnote)); +} + +#[test] +fn blocks() { + let f = parse( + "# Title\n\nPara one\ncontinues.\n\n> quoted\n\n- a\n- b\n 1. nested\n\n```\ncode *x*\n```\n\n---\n\n| A | B |\n|---|:-:|\n| 1 | 2 |\n", + ); + let kinds: Vec = f + .blocks + .iter() + .map(|b| match b { + FBlock::Para(p) => format!("{:?}{:?}:{}", p.kind, p.list.map(|l| (l.ordered, l.level)), p.text()), + FBlock::Table(t) => format!("table{}", t.rows.len()), + }) + .collect(); + assert_eq!( + kinds, + vec![ + "Heading(1)None:Title", + "NormalNone:Para one continues.", + "QuoteNone:quoted", + "NormalSome((false, 0)):a", + "NormalSome((false, 0)):b", + "NormalSome((true, 1)):nested", + "CodeNone:code *x*", + "RuleNone:", + "table2", + ] + ); +} + +#[test] +fn export_escapes_round_trip() { + let mut p = Para::default(); + p.push_text("1. not a list *really*", &Fmt::default()); + let md = escape_line_start(inlines_md(&p.inlines)); + let back = parse(&md); + let FBlock::Para(q) = &back.blocks[0] else { panic!() }; + assert_eq!(q.text(), "1. not a list *really*"); + assert!(q.list.is_none()); +} + +#[test] +fn export_emphasis_next_to_punctuation_round_trips() { + let marks: [(&str, fn(&mut Fmt)); 3] = [("bold", |f| f.bold = true), ("italic", |f| f.italic = true), ("strike", |f| f.strike = true)]; + let cases: [&[(&str, bool)]; 4] = [ + &[("注意：", true), ("请阅读", false)], + &[("a.", true), ("b", false)], + &[("a", false), ("!", true), ("b", false)], + &[("a", false), ("(x)", true), ("b", false)], + ]; + for (name, set) in marks { + for runs in cases { + let mut p = Para::default(); + for (t, on) in runs { + let mut f = Fmt::default(); + if *on { + set(&mut f); + } + p.push_text(t, &f); + } + let md = inlines_md(&p.inlines); + let back = parse(&md); + let FBlock::Para(q) = &back.blocks[0] else { panic!() }; + let want: String = runs.iter().map(|r| r.0).collect(); + assert_eq!(q.text(), want, "{name} exported as {md:?}"); + let got: Vec<(String, bool)> = q + .inlines + .iter() + .filter_map(|i| if let Inline::Text(t, f) = i { Some((t.clone(), f.bold || f.italic || f.strike)) } else { None }) + .collect(); + let expect: Vec<(String, bool)> = runs.iter().map(|(t, on)| (t.to_string(), *on)).collect(); + assert_eq!(got, expect, "{name} exported as {md:?}"); + } + } +} + +#[test] +fn export_adjacent_emphasis_next_to_punctuation_round_trips() { + let b = Fmt { bold: true, ..Fmt::default() }; + let i = Fmt { italic: true, ..Fmt::default() }; + let s = Fmt { strike: true, ..Fmt::default() }; + let cases: [&[(&str, &Fmt)]; 6] = [ + &[("ab", &b), ("!c", &i)], + &[("ab", &i), ("!c", &b)], + &[("ab", &b), ("!c", &s)], + &[("ab", &s), ("(c)", &b)], + &[("a.", &b), ("b", &i)], + &[("a.", &b), ("!b", &i)], + ]; + for runs in cases { + let mut p = Para::default(); + for (t, f) in runs { + p.push_text(t, f); + } + let md = inlines_md(&p.inlines); + let back = parse(&md); + let FBlock::Para(q) = &back.blocks[0] else { panic!() }; + let got: Vec<(String, bool, bool, bool)> = + q.inlines.iter().filter_map(|x| if let Inline::Text(t, f) = x { Some((t.clone(), f.bold, f.italic, f.strike)) } else { None }).collect(); + let expect: Vec<(String, bool, bool, bool)> = runs.iter().map(|(t, f)| (t.to_string(), f.bold, f.italic, f.strike)).collect(); + assert_eq!(got, expect, "exported as {md:?}"); + } +} + +#[test] +fn escape_line_start_preserves_indentation() { + assert_eq!(escape_line_start(" indented text".to_string()), " indented text"); + assert_eq!(escape_line_start(" # not a heading".to_string()), " \\# not a heading"); + assert_eq!(escape_line_start(" 1. not a list".to_string()), " 1\\. not a list"); + assert_eq!(escape_line_start(" > quote".to_string()), " \\> quote"); + // Four spaces or a tab would make an indented code block: the indent is capped at three. + assert_eq!(escape_line_start(" deep".to_string()), " deep"); + assert_eq!(escape_line_start("\tTabbed # text".to_string()), " Tabbed # text"); + assert_eq!(escape_line_start(" # not a heading".to_string()), " \\# not a heading"); + assert_eq!(escape_line_start(" \t1. not a list".to_string()), " 1\\. not a list"); +} + +#[test] +fn leading_unicode_whitespace_is_kept_and_does_not_start_blocks() { + let kinds_text = |md: &str| -> Vec<(Kind, bool, String)> { + parse(md).blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some((p.kind, p.list.is_some(), p.text())) } else { None }).collect() + }; + assert_eq!(kinds_text("\u{3000}\u{3000}日本語の段落\n"), vec![(Kind::Normal, false, "\u{3000}\u{3000}日本語の段落".to_string())]); + for src in ["\u{a0}# not a heading", "\u{3000}1. not a list", "\u{a0}- x", "\u{a0}> not a quote"] { + let mut p = Para::default(); + p.push_text(src, &Fmt::default()); + let md = escape_line_start(inlines_md(&p.inlines)); + assert_eq!(kinds_text(&md), vec![(Kind::Normal, false, src.to_string())], "{src:?} via {md:?}"); + } +} + +#[test] +fn single_backtick_code_span_escaped_pipe() { + let md = "| Command | Status |\n| --- | --- |\n| `cmd \\| grep` | pass |\n"; + let flow = parse(md); + let FBlock::Table(t) = &flow.blocks[0] else { panic!("expected table") }; + assert_eq!(t.rows.len(), 2); + assert_eq!(t.rows[1].len(), 2); + let FBlock::Para(p0) = &t.rows[1][0].blocks[0] else { panic!() }; + assert_eq!(p0.inlines.len(), 1); + assert!(matches!(&p0.inlines[0], Inline::Text(t, f) if t == "cmd | grep" && f.code)); + let FBlock::Para(p1) = &t.rows[1][1].blocks[0] else { panic!() }; + assert_eq!(p1.text(), "pass"); + + let mut out = Vec::new(); + table_md(t, &mut out); + let exported = out.join("\n"); + assert!(exported.contains("`cmd \\| grep`"), "Exported: {exported}"); +} + +#[test] +fn double_backtick_code_span_escaped_pipe() { + let md = "| Syntax | Note |\n| --- | --- |\n| ``a \\| b`` | second |\n"; + let flow = parse(md); + let FBlock::Table(t) = &flow.blocks[0] else { panic!("expected table") }; + assert_eq!(t.rows.len(), 2); + assert_eq!(t.rows[1].len(), 2); + let FBlock::Para(p0) = &t.rows[1][0].blocks[0] else { panic!() }; + assert_eq!(p0.inlines.len(), 1); + assert!(matches!(&p0.inlines[0], Inline::Text(t, f) if t == "a | b" && f.code)); + let FBlock::Para(p1) = &t.rows[1][1].blocks[0] else { panic!() }; + assert_eq!(p1.text(), "second"); +} + +#[test] +fn code_span_literal_backtick_and_escaped_pipe() { + let md = "| Expression | Detail |\n| --- | --- |\n| ``a`b\\|c`` | distinct |\n"; + let flow = parse(md); + let FBlock::Table(t) = &flow.blocks[0] else { panic!("expected table") }; + assert_eq!(t.rows.len(), 2); + assert_eq!(t.rows[1].len(), 2); + let FBlock::Para(p0) = &t.rows[1][0].blocks[0] else { panic!() }; + assert_eq!(p0.inlines.len(), 1); + assert!(matches!(&p0.inlines[0], Inline::Text(t, f) if t == "a`b|c" && f.code)); + let FBlock::Para(p1) = &t.rows[1][1].blocks[0] else { panic!() }; + assert_eq!(p1.text(), "distinct"); +} + +#[test] +fn three_column_table_with_distinctive_text() { + let md = "| Col1 | Col2 | Col3 |\n| --- | --- | --- |\n| `p1\\|q1` | Distinctive Two | Distinctive Three |\n"; + let flow = parse(md); + let FBlock::Table(t) = &flow.blocks[0] else { panic!("expected table") }; + assert_eq!(t.rows.len(), 2); + assert_eq!(t.rows[1].len(), 3); + assert_eq!(t.rows[1][0].text(), "p1|q1"); + assert_eq!(t.rows[1][1].text(), "Distinctive Two"); + assert_eq!(t.rows[1][2].text(), "Distinctive Three"); + let FBlock::Para(p0) = &t.rows[1][0].blocks[0] else { panic!() }; + assert!(matches!(&p0.inlines[0], Inline::Text(_, f) if f.code)); + let FBlock::Para(p1) = &t.rows[1][1].blocks[0] else { panic!() }; + assert!(matches!(&p1.inlines[0], Inline::Text(_, f) if !f.code)); + let FBlock::Para(p2) = &t.rows[1][2].blocks[0] else { panic!() }; + assert!(matches!(&p2.inlines[0], Inline::Text(_, f) if !f.code)); +} + +#[test] +fn table_cells_past_the_document_limits_are_not_built() { + // Every body row was padded to the header's width before any cap (#3701). + let wide = format!("{}|\n{}|\n|\n| b |\n", "|a".repeat(50_000), "|-".repeat(50_000)); + craft_flow::begin_import(); + let flow = parse(&wide); + let FBlock::Table(t) = &flow.blocks[0] else { panic!("expected table") }; + assert_eq!(t.rows.len(), 3); + assert!(t.rows.iter().all(|r| r.len() == MAX_COLS)); + assert_eq!(t.rows[2][0].text(), "b"); + assert!(craft_flow::Dropped::TableCells.reported(), "the columns left out are logged"); + // Rows past the limit still belong to the table, so the text after it reads as before. + let tall = format!("| a |\n| - |\n{}\nafter\n", "| x |\n".repeat(MAX_ROWS + 10)); + craft_flow::begin_import(); + let flow = parse(&tall); + let FBlock::Table(t) = &flow.blocks[0] else { panic!("expected table") }; + assert_eq!(t.rows.len(), MAX_ROWS); + let FBlock::Para(p) = &flow.blocks[1] else { panic!("expected paragraph") }; + assert_eq!(p.text(), "after"); + assert_eq!(flow.blocks.len(), 2); + assert!(craft_flow::Dropped::TableCells.reported(), "the rows left out are logged"); +} + +/// A short row is padded to its header's width, so a small file of one-cell rows under wide +/// headers made millions of cells (CodeRabbit on #3862). The document's tables share one cell +/// budget: rows past it are left out and logged, and their lines make no paragraphs. +#[test] +fn tables_share_one_cell_budget() { + let header = |n: usize| format!("|{}|\n|{}|\n", vec![" h "; n].join("|"), vec!["-"; n].join("|")); + let fits = crate::MAX_TABLE_CELLS / MAX_COLS; + let src = format!("{}{}\nafter\n\n{}| 1 | 2 |\n", header(MAX_COLS), "| x |\n".repeat(fits + 100), header(2)); + craft_flow::begin_import(); + let flow = parse(&src); + let tables: Vec<&FTable> = flow.blocks.iter().filter_map(|b| if let FBlock::Table(t) = b { Some(t) } else { None }).collect(); + let cells: usize = tables.iter().flat_map(|t| &t.rows).map(Vec::len).sum(); + assert!(cells <= crate::MAX_TABLE_CELLS, "{cells} cells"); + assert_eq!(tables.iter().map(|t| t.rows.len()).collect::>(), [fits], "the second table is past the budget"); + let texts: Vec = flow.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p.text()) } else { None }).collect(); + assert_eq!(texts, ["after"]); + assert!(craft_flow::Dropped::TableCells.reported()); + // Within the budget a table is whole and nothing is logged. + craft_flow::begin_import(); + let flow = parse(&format!("{}| 1 | 2 |\n", header(2))); + let [FBlock::Table(t)] = flow.blocks.as_slice() else { panic!("{:?}", flow.blocks) }; + assert_eq!(t.rows.len(), 2); + assert!(!craft_flow::Dropped::TableCells.reported()); +} + +#[test] +fn a_wide_table_still_checks_every_header_and_delimiter_cell() { + // Only the first 63 cells are kept, but every one is counted and checked. + let head = "|a".repeat(100) + "|\n"; + let delim = |bad: Option| (0..100).map(|i| if Some(i) == bad { "|x" } else { "|-" }).collect::() + "|\n"; + let is_table = |md: String| matches!(parse(&md).blocks.first(), Some(FBlock::Table(_))); + assert!(is_table(format!("{head}{}| b |\n", delim(None)))); + assert!(!is_table(format!("{head}{}| b |\n", delim(Some(80)))), "a bad delimiter cell past the kept ones"); + assert!(!is_table(format!("{}|\n{}| b |\n", "|a".repeat(99), delim(None))), "99 header cells, 100 delimiter cells"); + let (cells, n) = split_row(r"| a | b \| c | d |", 2); + assert_eq!((cells, n), (vec!["a".to_string(), "b | c".to_string()], 3)); +} + +#[test] +fn regular_table_cells_containing_escaped_pipes() { + let md = "| Option | Description |\n| --- | --- |\n| Regular \\| Pipe | Normal text cell |\n"; + let flow = parse(md); + let FBlock::Table(t) = &flow.blocks[0] else { panic!("expected table") }; + assert_eq!(t.rows.len(), 2); + assert_eq!(t.rows[1].len(), 2); + assert_eq!(t.rows[1][0].text(), "Regular | Pipe"); + assert_eq!(t.rows[1][1].text(), "Normal text cell"); + let FBlock::Para(p0) = &t.rows[1][0].blocks[0] else { panic!() }; + assert!(matches!(&p0.inlines[0], Inline::Text(_, f) if !f.code)); + + let mut out = Vec::new(); + table_md(t, &mut out); + let exported = out.join("\n"); + assert!(exported.contains("Regular \\| Pipe"), "Exported: {exported}"); + assert!(!exported.contains("Regular \\\\| Pipe"), "Exported: {exported}"); +} + +#[test] +fn unescaped_pipes_behave_as_gfm_column_separators() { + let md = "| Header 1 | Header 2 |\n| --- | --- |\n| `raw|pipe` | Extra Column |\n"; + let flow = parse(md); + let FBlock::Table(t) = &flow.blocks[0] else { panic!("expected table") }; + assert_eq!(t.rows.len(), 2); + assert_eq!(t.rows[1].len(), 2); + assert_eq!(t.rows[1][0].text(), "`raw"); + assert_eq!(t.rows[1][1].text(), "pipe`"); +} + +#[test] +fn ordinary_tables_and_standalone_code_spans_unchanged() { + let md = "| Alpha | Beta |\n| --- | --- |\n| 1 | 2 |\n"; + let flow = parse(md); + let FBlock::Table(t) = &flow.blocks[0] else { panic!() }; + let mut out = Vec::new(); + table_md(t, &mut out); + assert_eq!(out.join("\n"), "| Alpha | Beta |\n| --- | --- |\n| 1 | 2 |"); + + let mut p = Para::default(); + let f = Fmt { code: true, ..Default::default() }; + p.push_text("cmd | grep", &f); + let exported = inlines_md(&p.inlines); + assert_eq!(exported, "`cmd | grep`"); +} + +#[test] +fn code_span_line_endings_stay_literal_on_export() { + let cases = [ + ("a\n# b", "a # b"), + ("a\n
b
", "a
b
"), + ("a\n- b", "a - b"), + ("a\n```", "a ```"), + ("a\r# b", "a # b"), + ("a\r\n# b", "a # b"), + ("a\n\n> b", "a > b"), + ("\na\n", " a "), + ("`a\n# b`", "`a # b`"), + ("cmd | grep", "cmd | grep"), + ]; + let shapes = [ + (Kind::Normal, None), + (Kind::Normal, Some(ListInfo { ordered: false, level: 0, start: 1, ..Default::default() })), + (Kind::Code, Some(ListInfo { ordered: true, level: 0, start: 1, ..Default::default() })), + ]; + for (source, expected) in cases { + for (kind, list) in shapes { + let mut p = Para::new(kind); + p.list = list; + p.push_text(source, &Fmt { code: true, ..Default::default() }); + let mut doc = model::to_doc(&model::Flow { blocks: vec![FBlock::Para(p)], ..Default::default() }); + // The flow builder removes CR controls; keep the native document's raw input. + let p = doc.para_mut(craft_wordmodel::StoryRef::Body, &craft_wordmodel::Path::top(0)).unwrap(); + p.text = source.to_string(); + p.runs[0].len = source.len(); + let md = export(&doc); + assert_eq!(md.trim_end().lines().count(), 1, "{source:?} {kind:?} {list:?} -> {md:?}"); + assert!(!md.contains('\r'), "{md:?}"); + let back = import(&md); + let flow = model::from_doc(&back); + let [FBlock::Para(q)] = flow.blocks.as_slice() else { panic!("{md:?}") }; + assert_eq!(q.text(), expected, "{source:?} {kind:?} {list:?} -> {md:?}"); + assert_eq!(q.list, list, "{md:?}"); + assert!(q.inlines.iter().all(|i| matches!(i, Inline::Text(_, f) if f.code)), "{md:?}"); + } + } + let fenced = "```\na\n# b\n```"; + assert_eq!(export(&import(fenced)).trim_end(), fenced); +} + +#[test] +fn code_span_line_endings_in_table_cells_stay_literal() { + for ending in ["\n", "\r", "\r\n"] { + let source = format!("a{ending}# b|c"); + let mut p = Para::default(); + p.push_text(&source, &Fmt { code: true, ..Default::default() }); + let cell = Cell { blocks: vec![FBlock::Para(p)], ..Default::default() }; + let table = FTable { rows: vec![vec![cell, Cell::default()], vec![Cell::default(), Cell::default()]], ..Default::default() }; + let mut doc = model::to_doc(&model::Flow { blocks: vec![FBlock::Table(table)], ..Default::default() }); + let p = doc.para_mut(craft_wordmodel::StoryRef::Body, &craft_wordmodel::Path(vec![0, 0, 0, 0])).unwrap(); + p.text = source.clone(); + p.runs[0].len = source.len(); + let md = export(&doc); + let back = model::from_doc(&import(&md)); + let [FBlock::Table(table), ..] = back.blocks.as_slice() else { panic!("{md:?}") }; + assert!(back.blocks.iter().skip(1).all(|b| matches!(b, FBlock::Para(p) if p.text().is_empty())), "{md:?}"); + assert_eq!(table.rows.len(), 2, "{md:?}"); + assert_eq!(table.rows[0].len(), 2, "{md:?}"); + let [FBlock::Para(p)] = table.rows[0][0].blocks.as_slice() else { panic!("{md:?}") }; + assert_eq!(p.text(), "a # b|c", "{md:?}"); + assert!(p.inlines.iter().all(|i| matches!(i, Inline::Text(_, f) if f.code)), "{md:?}"); + } +} + +#[test] +fn table_code_pipe_export_import_round_trip() { + let mut table = FTable::default(); + let mut cell_a = Cell::default(); + let mut para_a = Para::default(); + let fmt_a = Fmt { code: true, ..Default::default() }; + para_a.push_text("a`b|c", &fmt_a); + cell_a.blocks.push(FBlock::Para(para_a)); + + let mut cell_b = Cell::default(); + let mut para_b = Para::default(); + para_b.push_text("CRITICAL_DISTINCTIVE_INFO", &Fmt::default()); + cell_b.blocks.push(FBlock::Para(para_b)); + + let mut head_a = Cell::default(); + let mut head_para_a = Para::default(); + head_para_a.push_text("Col A", &Fmt::default()); + head_a.blocks.push(FBlock::Para(head_para_a)); + + let mut head_b = Cell::default(); + let mut head_para_b = Para::default(); + head_para_b.push_text("Col B", &Fmt::default()); + head_b.blocks.push(FBlock::Para(head_para_b)); + + table.rows.push(vec![head_a, head_b]); + table.rows.push(vec![cell_a, cell_b]); + + let mut out = Vec::new(); + table_md(&table, &mut out); + let exported_md = out.join("\n"); + + assert!(exported_md.contains("``a`b\\|c``"), "Exported: {exported_md}"); + assert!(exported_md.contains("CRITICAL\\_DISTINCTIVE\\_INFO"), "Exported: {exported_md}"); + + let back_flow = parse(&exported_md); + let FBlock::Table(back_table) = &back_flow.blocks[0] else { panic!("expected table") }; + + assert_eq!(back_table.rows.len(), 2, "Both rows must exist"); + let row1 = &back_table.rows[1]; + assert_eq!(row1.len(), 2, "Both cells must exist in row 1"); + + let back_cell_a = &row1[0]; + let back_cell_b = &row1[1]; + assert_eq!(back_cell_a.text(), "a`b|c", "Cell A text must be preserved"); + assert_eq!(back_cell_b.text(), "CRITICAL_DISTINCTIVE_INFO", "Cell B text must be preserved"); + + let FBlock::Para(back_p_a) = &back_cell_a.blocks[0] else { panic!() }; + assert_eq!(back_p_a.inlines.len(), 1); + assert!( + matches!(&back_p_a.inlines[0], Inline::Text(t, f) if t == "a`b|c" && f.code), + "Code formatting must remain intact on Cell A: {:?}", + back_p_a.inlines + ); + + let FBlock::Para(back_p_b) = &back_cell_b.blocks[0] else { panic!() }; + assert_eq!(back_p_b.inlines.len(), 1); + assert!( + matches!(&back_p_b.inlines[0], Inline::Text(t, f) if t == "CRITICAL_DISTINCTIVE_INFO" && !f.code), + "Cell B formatting must remain plain text: {:?}", + back_p_b.inlines + ); + + assert_eq!(back_cell_a.text(), "a`b|c"); + assert_eq!(back_cell_b.text(), "CRITICAL_DISTINCTIVE_INFO"); + + assert!(!back_cell_a.text().contains("CRITICAL_DISTINCTIVE_INFO")); + assert!(!back_cell_b.text().contains("a`b|c")); + + let doc = import(&exported_md); + let re_exported = export(&doc); + assert!(re_exported.contains("``a`b\\|c``"), "Re-exported: {re_exported}"); + assert!(re_exported.contains("CRITICAL\\_DISTINCTIVE\\_INFO"), "Re-exported: {re_exported}"); + let doc2 = import(&re_exported); + let back_flow2 = model::from_doc(&doc2); + let FBlock::Table(back_table2) = &back_flow2.blocks[0] else { panic!() }; + assert_eq!(back_table2.rows[1][0].text(), "a`b|c"); + assert_eq!(back_table2.rows[1][1].text(), "CRITICAL_DISTINCTIVE_INFO"); +} + +#[test] +fn ordered_list_keeps_start_number() { + assert_eq!(export(&import("5. a\n6. b")).trim_end(), "5. a\n6. b"); +} + +/// A list inside a block quote stays in the quote: its items were imported as quoted list +/// paragraphs but written back without `>`, so the list moved out of the quote. +#[test] +fn a_list_inside_a_quote_stays_quoted() { + let src = "> Intro\n>\n> - one\n> - two\n> - nested\n> 1. first\n\nAfter.\n"; + let want = "> Intro\n\n> - one\n> - two\n> - nested\n> 1. first\n\nAfter."; + let md = export(&import(src)); + assert_eq!(md.trim_end(), want); + assert_eq!(export(&import(&md)).trim_end(), want, "and stays so when read back"); + // An item's continuation line is quoted too. + assert_eq!(export(&import("> - a\\\n> b\n")).trim_end(), "> - a\\\n> b"); + // A list outside a quote is unchanged. + assert_eq!(export(&import("- one\n- two\n")).trim_end(), "- one\n- two"); + // A list after a quoted list stays outside the quote. + let md = export(&import("> - a\n\n2. b\n")); + assert_eq!(md.trim_end(), "> - a\n\n2. b"); + assert_eq!(export(&import(&md)).trim_end(), "> - a\n\n2. b", "and stays so when read back"); +} + +#[test] +fn lines_after_line_break_stay_in_paragraph() { + use craft_wordmodel::{CharProps, Document, Paragraph, StoryRef, para_block}; + let cases = ["Shopping\n- milk\n# not a heading", "a\n1. x", "a\n2) x", "a\n+ x", "a\n---", "a\n***", "a\n==", "a\n--", "a\n> q"]; + for s in cases { + let mut d = Document::new(); + d.body = vec![para_block(Paragraph::with_text(s, CharProps::default()))]; + let md = export(&d); + let back = import(&md); + assert_eq!(back.body.len(), 1, "{s:?} -> {md:?}"); + assert_eq!(back.plain_text(StoryRef::Body), s, "{md:?}"); + } +} + +#[test] +fn lines_after_line_break_stay_in_list_item_and_quote() { + let cases = ["a\n- x", "a\n# h", "a\n1. x", "a\n--", "a\n---", "a\n==", "a\n> q"]; + let shapes: [(Kind, Option); 5] = [ + (Kind::Normal, Some(ListInfo { ordered: false, level: 0, start: 1, ..Default::default() })), + (Kind::Normal, Some(ListInfo { ordered: true, level: 0, start: 1, ..Default::default() })), + (Kind::Normal, Some(ListInfo { ordered: false, level: 1, start: 1, ..Default::default() })), + (Kind::Normal, Some(ListInfo { ordered: true, level: 1, start: 1, ..Default::default() })), + (Kind::Quote, None), + ]; + for s in cases { + for (kind, list) in shapes { + let mut blocks = Vec::new(); + if let Some(li) = list.filter(|l| l.level > 0) { + let mut parent = Para::new(Kind::Normal); + parent.list = Some(ListInfo { ordered: li.ordered, level: 0, start: 1, ..Default::default() }); + parent.push_text("top", &Fmt::default()); + blocks.push(FBlock::Para(parent)); + } + let mut p = Para::new(kind); + p.list = list; + p.push_text(s, &Fmt::default()); + blocks.push(FBlock::Para(p)); + let mut out = Vec::new(); + write_blocks(&blocks, &mut out); + let md = out.join("\n"); + let back = parse(&md); + assert_eq!(back.blocks.len(), blocks.len(), "{s:?} {kind:?} {list:?} -> {md:?}"); + let Some(FBlock::Para(q)) = back.blocks.last() else { panic!("{md:?}") }; + assert_eq!(q.kind, kind, "{md:?}"); + assert_eq!(q.list, list, "{md:?}"); + assert_eq!(q.text(), s, "{md:?}"); + } + } +} + +#[test] +fn table_column_alignments_import_and_export() { + let md = "| Left | Center | Right | Default |\n| :--- | :---: | ---: | --- |\n| 1 | 2 | 3 | 4 |\n"; + let flow = parse(md); + let FBlock::Table(t) = &flow.blocks[0] else { panic!("expected table") }; + assert_eq!(t.rows.len(), 2); + let aligns: Vec> = + t.rows[0].iter().map(|c| c.blocks.iter().find_map(|b| if let FBlock::Para(p) = b { Some(p.align) } else { None }).flatten()).collect(); + assert_eq!(aligns, vec![Some(Align::Left), Some(Align::Center), Some(Align::Right), None]); + + let mut out = Vec::new(); + table_md(t, &mut out); + let exported = out.join("\n"); + assert!(exported.contains("| :--- | :---: | ---: | --- |"), "Exported: {exported}"); +} + +#[test] +fn tab_separated_list_markers() { + let f = parse("-\tbullet item\n*\tstar item\n+\tplus item\n1.\tone item\n2)\ttwo item\n"); + assert_eq!(f.blocks.len(), 5); + for (i, b) in f.blocks.iter().enumerate() { + let FBlock::Para(p) = b else { panic!("block {i} not a para") }; + assert!(p.list.is_some(), "block {i} should be a list item: {:?}", p); + } + let FBlock::Para(p0) = &f.blocks[0] else { panic!() }; + assert_eq!(p0.text(), "bullet item"); + assert!(!p0.list.unwrap().ordered); + + let FBlock::Para(p3) = &f.blocks[3] else { panic!() }; + assert_eq!(p3.text(), "one item"); + assert!(p3.list.unwrap().ordered); +} + +#[test] +fn html_comments_are_ignored_in_blocks_and_inlines() { + let md = "\n\nHello world!\n\n\n\nAfter comment."; + let flow = parse(md); + assert_eq!(flow.blocks.len(), 2); + let texts: Vec = flow.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p.text()) } else { None }).collect(); + assert_eq!(texts, ["Hello world!", "After comment."]); +} + +#[test] +fn a_paragraph_of_delimiters_parses_in_linear_time() { + // #3714: each delimiter's literal looked back for the text before it, to the start of + // the paragraph when there was none (quadratic). + let src = "*_".repeat(200_000); + let t = std::time::Instant::now(); + let out = inl(&src); + // Under a second on a loaded machine; the backwards search ran past 6 minutes. + assert!(t.elapsed() < std::time::Duration::from_secs(10), "{:?}", t.elapsed()); + let text: String = out.iter().filter_map(|i| if let Inline::Text(t, _) = i { Some(t.as_str()) } else { None }).collect(); + assert!(text.chars().all(|c| matches!(c, '*' | '_')), "only delimiters come out"); + // Literal delimiters still take the link of the text before them. + let s = inl("[a](http://x.org) [b*](http://y.org)"); + assert!(s.iter().any(|i| matches!(i, Inline::Text(t, f) if t == "b*" && f.link.as_deref() == Some("http://y.org"))), "{s:?}"); +} + +#[test] +fn exported_link_destinations_read_back_unchanged() { + // Removing `<` and `>` from a destination after the safety check could make a relative + // target a script or network-path one (`java)"); + for l in [ + "javab c", + "javascript:alert(1)", + "javascript:x", + "&", + "javascript\\:x", + "dir\\", + "dir\\(1)\\", + "q?a=1&b=2", + ] { + let out = md(l); + let back = inl(&out); + let got: Vec<_> = back.iter().filter_map(|i| if let Inline::Text(_, f) = i { f.link.as_deref().map(str::to_string) } else { None }).collect(); + assert_eq!(got, [l.to_string()], "{l:?} → {out:?}"); + } + // Ordinary destinations are written as before. + assert_eq!(md("https://example.com/a?b=1&c=2"), "[x](https://example.com/a?b=1&c=2)"); + assert_eq!(md("https://example.com/a b"), "[x]()"); + assert_eq!(md("https://e.com/x\ny"), "[x](https://e.com/xy)"); +} + +#[test] +fn heading_trailing_hash_survives_round_trip() { + let heading = |md: &str| { + let f = parse(md); + let Some(FBlock::Para(p)) = f.blocks.first() else { panic!("no paragraph in {md:?}") }; + assert_eq!(p.kind, Kind::Heading(2), "{md:?}"); + p.text() + }; + for (src, text, want) in [ + ("## Item \\#\n", "Item #", "## Item \\#\n"), + ("## Issue \\##\n", "Issue ##", "## Issue \\##\n"), + ("## \\#\n", "#", "## \\#\n"), + ("## C#\n", "C#", "## C#\n"), + ] { + assert_eq!(heading(src), text, "{src:?}"); + let md = export(&import(src)); + assert_eq!(md, want, "first export of {src:?}"); + assert_eq!(heading(&md), text, "md was {md:?}"); + assert_eq!(export(&import(&md)), want, "second export of {src:?}"); + } +}