diff --git a/crates/latex/src/parser.rs b/crates/latex/src/parser.rs index 91e419a0f..12971f1d0 100644 --- a/crates/latex/src/parser.rs +++ b/crates/latex/src/parser.rs @@ -16,100 +16,111 @@ pub fn import(bytes: &[u8]) -> Document { d } -/// Commands dropped together with their arguments (layout and definitions, not text). -const DROP_WITH_ARGS: &[&str] = &[ - "documentclass", - "usepackage", - "RequirePackage", - "newcommand", - "renewcommand", - "providecommand", - "newcommand*", - "renewcommand*", - "providecommand*", - "newenvironment", - "renewenvironment", - "DeclareMathOperator", - "DeclareMathOperator*", - "def", - "let", - "setlength", - "addtolength", - "setcounter", - "addtocounter", - "stepcounter", - "refstepcounter", - "vspace", - "vspace*", - "hspace", - "hspace*", - "vskip", - "hskip", - "pagestyle", - "thispagestyle", - "pagenumbering", - "bibliographystyle", - "bibliography", - "addbibresource", - "printbibliography", - "input", - "include", - "includeonly", - "graphicspath", - "hypersetup", - "definecolor", - "geometry", - "numberwithin", - "newtheorem", - "newtheorem*", - "theoremstyle", - "phantom", - "hphantom", - "vphantom", - "index", - "nocite", - "addcontentsline", - "markboth", - "markright", - "linespread", - "setstretch", - "fancyhf", - "fancyhead", - "fancyfoot", - "lstset", - "captionsetup", - "setmainfont", - "setsansfont", - "setmonofont", - "newfontfamily", - "date", - "thanks", - "maketitle*", - "tableofcontents", - "listoffigures", - "listoftables", - "appendix", - "frontmatter", - "mainmatter", - "backmatter", - "cline", - "hline", - "toprule", - "midrule", - "bottomrule", - "cmidrule", - "addlinespace", - "endhead", - "endfirsthead", - "endfoot", - "endlastfoot", - "rowcolor", - "arraystretch", - "smallskip", - "medskip", - "bigskip", - "today", - "and", +/// Commands dropped together with their arguments (layout and definitions, not text), each with +/// the arguments it takes in order: `m` is one `{…}` group or command, and `o` is every `[…]` there. +/// Text after the last argument stays text. +pub(super) const DROP_WITH_ARGS: &[(&str, &str)] = &[ + ("documentclass", "omo"), + ("usepackage", "omo"), + ("RequirePackage", "omo"), + ("newcommand", "mom"), + ("renewcommand", "mom"), + ("providecommand", "mom"), + ("newcommand*", "mom"), + ("renewcommand*", "mom"), + ("providecommand*", "mom"), + ("newenvironment", "momm"), + ("renewenvironment", "momm"), + ("DeclareMathOperator", "mm"), + ("DeclareMathOperator*", "mm"), + ("def", "m"), + ("let", ""), + ("setlength", "mm"), + ("addtolength", "mm"), + ("setcounter", "mm"), + ("addtocounter", "mm"), + ("stepcounter", "m"), + ("refstepcounter", "m"), + ("vspace", "m"), + ("vspace*", "m"), + ("hspace", "m"), + ("hspace*", "m"), + ("vskip", "m"), + ("hskip", "m"), + ("pagestyle", "m"), + ("thispagestyle", "m"), + ("pagenumbering", "m"), + ("bibliographystyle", "m"), + ("bibliography", "m"), + ("addbibresource", "om"), + ("printbibliography", "o"), + ("input", "m"), + ("include", "m"), + ("includeonly", "m"), + ("graphicspath", "m"), + ("hypersetup", "m"), + ("definecolor", "ommm"), + ("geometry", "m"), + ("numberwithin", "omm"), + ("newtheorem", "momo"), + ("newtheorem*", "mm"), + ("theoremstyle", "m"), + ("phantom", "m"), + ("hphantom", "m"), + ("vphantom", "m"), + ("index", "om"), + ("nocite", "m"), + ("addcontentsline", "mmm"), + ("markboth", "mm"), + ("markright", "m"), + ("linespread", "m"), + ("setstretch", "m"), + ("fancyhf", "om"), + ("fancyhead", "om"), + ("fancyfoot", "om"), + ("lstset", "m"), + ("captionsetup", "om"), + ("setmainfont", "omo"), + ("setsansfont", "omo"), + ("setmonofont", "omo"), + ("newfontfamily", "momo"), + ("date", "m"), + ("thanks", "m"), + ("hyphenation", "m"), + ("addvspace", "m"), + ("enlargethispage", "m"), + ("enlargethispage*", "m"), + ("selectlanguage", "m"), + ("newcounter", "mo"), + ("settowidth", "mm"), + ("settoheight", "mm"), + ("settodepth", "mm"), + ("maketitle*", ""), + ("tableofcontents", ""), + ("listoffigures", ""), + ("listoftables", ""), + ("appendix", ""), + ("frontmatter", ""), + ("mainmatter", ""), + ("backmatter", ""), + ("cline", "m"), + ("hline", ""), + ("toprule", "o"), + ("midrule", "o"), + ("bottomrule", "o"), + ("cmidrule", "om"), + ("addlinespace", "o"), + ("endhead", ""), + ("endfirsthead", ""), + ("endfoot", ""), + ("endlastfoot", ""), + ("rowcolor", "omo"), + ("arraystretch", ""), + ("smallskip", ""), + ("medskip", ""), + ("bigskip", ""), + ("today", ""), + ("and", ""), ]; /// Paragraph-level context from the enclosing environments. @@ -128,8 +139,9 @@ pub(super) struct Parser<'a> { fmt: Fmt, /// Paragraph context of the enclosing environments. ctx: Ctx, - /// Text that followed an optional argument's `]` in the same token, emitted next. - pending: Option, + /// Text that followed an optional argument's `]` in the same token, emitted next. A slice + /// of the token, so reading `[]` after `[]` copies nothing. + pending: Option<&'a str>, /// Open lists: ordered? lists: Vec, page_break: bool, @@ -228,35 +240,47 @@ impl<'a> Parser<'a> { /// The tokens of an optional `[…]` argument, if one follows. pub(super) fn opt_arg(&mut self) -> Option { - let save = self.pos; - self.skip_spaces(); - let Some(Tok::Text(s)) = self.t.get(self.pos) else { - self.pos = save; - return None; - }; - if !s.starts_with('[') { - self.pos = save; - return None; + let t = self.t; + let mut out = String::new(); + if let Some(p) = self.pending.take() { + // Text left after the previous argument's `]` comes first: in `\makebox[3cm][l]` the + // second argument is in the first one's token and waits there. + let Some(body) = p.trim_start().strip_prefix('[') else { + self.pending = Some(p); + return None; + }; + if self.close_opt(body, &mut out) { + return Some(out); + } + } else { + let save = self.pos; + self.skip_spaces(); + let Some(Tok::Text(s)) = t.get(self.pos) else { + self.pos = save; + return None; + }; + // LaTeX skips spaces before an optional argument, and in `\\begin{figure} [h]` the + // space and the `[` share one text token. + let Some(body) = s.trim_start().strip_prefix('[') else { + self.pos = save; + return None; + }; + self.pos += 1; + if self.close_opt(body, &mut out) { + return Some(out); + } } // Collect raw text up to the matching `]` (no nested brackets across tokens). - let mut out = String::new(); - let mut first = true; - while let Some(tok) = self.t.get(self.pos) { + while let Some(tok) = t.get(self.pos) { + if matches!(tok, Tok::Par) { + break; + } + self.pos += 1; match tok { Tok::Text(s) => { - let s = if first { s.get(1..).unwrap_or("") } else { s.as_str() }; - first = false; - if let Some(end) = s.find(']') { - out.push_str(s.get(..end).unwrap_or("")); - let rest = s.get(end + 1..).unwrap_or("").to_string(); - self.pos += 1; - if !rest.trim().is_empty() { - // The text after `]` comes next (tokens are shared, so it waits here). - self.pending = Some(rest); - } + if self.close_opt(s, &mut out) { return Some(out); } - out.push_str(s); } Tok::Open => out.push('{'), Tok::Close => out.push('}'), @@ -267,14 +291,29 @@ impl<'a> Parser<'a> { out.push('\\'); out.push_str(c); } - Tok::Par => break, _ => {} } - self.pos += 1; } Some(out) } + /// Adds `s` to an optional argument's text up to its `]`. True when the `]` was there; the + /// text after it then comes next (tokens are shared, so it waits in `pending`). + fn close_opt(&mut self, s: &'a str, out: &mut String) -> bool { + let Some(end) = s.find(']') else { + out.push_str(s); + return false; + }; + out.push_str(s.get(..end).unwrap_or("")); + let rest = s.get(end + 1..).unwrap_or(""); + // Checked from the start only: a no-break space stays in its token, so the end can be a + // long run of whitespace that every argument in a row would read again. + if !rest.trim_start().is_empty() { + self.pending = Some(rest); + } + true + } + /// The tokens of a `{…}` argument (or a single token), consumed. pub(super) fn arg_tokens(&mut self) -> &'a [Tok] { self.skip_spaces(); @@ -358,45 +397,44 @@ impl<'a> Parser<'a> { self.fmt = saved_fmt; } - /// Skip every `[…]` and `{…}` argument that follows (and, for definitions, the macro name). + /// Skip a dropped command's arguments, as its `DROP_WITH_ARGS` entry lists them. fn drop_args(&mut self, name: &str) { - if matches!(name, "newcommand" | "renewcommand" | "providecommand" | "newcommand*" | "renewcommand*" | "providecommand*" | "def" | "let") { - self.skip_spaces(); - if matches!(self.t.get(self.pos), Some(Tok::Cmd(_))) { - self.pos += 1; - } - if name == "let" { + if name == "let" { + for _ in 0..2 { self.skip_spaces(); if matches!(self.t.get(self.pos), Some(Tok::Cmd(_))) { self.pos += 1; } - return; } - if name == "def" { - // `\def\x#1{…}`: the parameter text, then the body. - while matches!(self.t.get(self.pos), Some(Tok::Text(_))) { - self.pos += 1; - } + return; + } + if name == "def" { + // `\def\x#1{…}`: the macro name and its parameter text come before the body. + self.skip_spaces(); + if matches!(self.t.get(self.pos), Some(Tok::Cmd(_))) { + self.pos += 1; + } + while matches!(self.t.get(self.pos), Some(Tok::Text(_))) { + self.pos += 1; } } - loop { + let args = DROP_WITH_ARGS.iter().find(|(n, _)| *n == name).map_or("", |(_, a)| *a); + for kind in args.chars() { + if kind == 'o' { + // Text after a `]` waits in `pending`: another `[…]`, or the text that follows. + while self.opt_arg().is_some() {} + continue; + } + if self.pending.is_some() { + return; + } let save = self.pos; self.skip_spaces(); - match self.t.get(self.pos) { - Some(Tok::Open) => { - let _ = self.arg_tokens(); - } - Some(Tok::Text(s)) if s.starts_with('[') => { - let _ = self.opt_arg(); - if self.pending.is_some() { - return; - } - } - _ => { - self.pos = save; - return; - } + if !matches!(self.t.get(self.pos), Some(Tok::Open | Tok::Cmd(_))) { + self.pos = save; + return; } + let _ = self.arg_tokens(); } } @@ -412,7 +450,7 @@ impl<'a> Parser<'a> { /// Parse until `\end{env}` (when given), a closing brace at this level, or the end. fn flush_pending(&mut self) { if let Some(p) = self.pending.take() { - self.text(&ligatures(&p)); + self.text(&ligatures(p)); } } @@ -515,7 +553,7 @@ impl<'a> Parser<'a> { let mut chars = t.chars(); let first = chars.next().map(String::from).unwrap_or_default(); if !chars.as_str().is_empty() { - self.pending = Some(chars.as_str().to_string()); + self.pending = Some(chars.as_str()); } first } @@ -710,6 +748,36 @@ impl<'a> Parser<'a> { self.flush(); self.text(&format!("[{}] ", label.unwrap_or(key))); } + "parbox" | "raisebox" | "scalebox" | "resizebox" | "resizebox*" | "rotatebox" => { + // The box's position, size, scale or angle place it; only its last argument is + // text. + match name { + "parbox" => { + let _ = self.opt_arg(); + let _ = self.opt_arg(); + let _ = self.opt_arg(); + let _ = self.arg_raw(); + } + "raisebox" => { + let _ = self.arg_raw(); + let _ = self.opt_arg(); + let _ = self.opt_arg(); + } + "scalebox" => { + let _ = self.arg_raw(); + let _ = self.opt_arg(); + } + "rotatebox" => { + let _ = self.opt_arg(); + let _ = self.arg_raw(); + } + _ => { + let _ = self.arg_raw(); + let _ = self.arg_raw(); + } + } + self.with_arg(|_| {}); + } "multicolumn" | "multirow" => { // Outside a tabular: keep the content. let _ = self.arg_raw(); @@ -718,7 +786,7 @@ impl<'a> Parser<'a> { self.with_arg(|_| {}); } "begin" | "end" => {} - n if DROP_WITH_ARGS.contains(&n) => self.drop_args(n), + n if DROP_WITH_ARGS.iter().any(|(d, _)| *d == n) => self.drop_args(n), _ => { // Unknown: drop the command, keep its arguments' text. let _ = self.opt_arg(); @@ -818,6 +886,11 @@ impl<'a> Parser<'a> { | "proposition" => { // Floats and boxes: their content, in order. let _ = self.opt_arg(); + if name == "minipage" { + // `[position][height][inner position]`. + let _ = self.opt_arg(); + let _ = self.opt_arg(); + } if matches!(name, "minipage" | "subfigure" | "multicols" | "multicols*") { let _ = self.arg_raw(); } diff --git a/crates/latex/src/tests.rs b/crates/latex/src/tests.rs index 24c8c15b1..9bc8b6c72 100644 --- a/crates/latex/src/tests.rs +++ b/crates/latex/src/tests.rs @@ -340,6 +340,103 @@ fn leading_brackets_stay_text() { assert_eq!(back.plain_text(Default::default()), "[x] done"); } +#[test] +fn a_second_optional_argument_is_not_text() { + // Both optional arguments sit in one token (`[3cm][l]`), and the second was left over as text. + for src in ["a\\makebox[3cm][l]{x}b", "a\\framebox[3cm][r]{x}b", "a\\makebox[3cm] [l]{x}b"] { + assert_eq!(texts(&format!("\\begin{{document}}{src}\\end{{document}}")), vec!["axb".to_string()], "{src}"); + } + let t = texts("\\begin{document}\\begin{minipage}[t][2cm][c]{3cm}x\\end{minipage}\\end{document}"); + assert_eq!(t, vec!["x".to_string()]); +} + +#[test] +fn an_optional_argument_after_a_space_is_not_text() { + // A space before the `[` kept the whole argument as text. + for src in ["\\begin{figure} [h]x\\end{figure}", "\\begin{minipage} [t]{3cm}x\\end{minipage}"] { + assert_eq!(texts(&format!("\\begin{{document}}{src}\\end{{document}}")), vec!["x".to_string()], "{src}"); + } + assert_eq!(texts("a\\\\ [2pt]b"), ["a\nb"]); +} + +#[test] +fn a_definition_with_a_default_argument_shows_nothing() { + // `[2][x]`: the default was left over as text, and the definition's body with it. + assert_eq!(texts("\\begin{document}\\newcommand{\\foo}[2][x]{#1 and #2}word\\end{document}"), vec!["word".to_string()]); + assert_eq!(texts("\\begin{document}\\newcommand{\\foo}[2] [x]{#1 and #2}word\\end{document}"), vec!["word".to_string()]); +} + +#[test] +fn a_box_shows_only_its_text() { + // The box's position, size, scale or angle showed before its text. + for src in [ + "a\\parbox{3cm}{x}b", + "a\\parbox[t][2cm][s]{3cm}{x}b", + "a\\raisebox{2pt}{x}b", + "a\\raisebox{2pt}[0pt][0pt]{x}b", + "a\\raisebox{2pt} [0pt] [0pt]{x}b", + "a\\scalebox{2}{x}b", + "a\\scalebox{2}[1]{x}b", + "a\\scalebox{2} [1]{x}b", + "a\\resizebox{2cm}{!}{x}b", + "a\\resizebox*{2cm}{1cm}{x}b", + "a\\rotatebox{90}{x}b", + "a\\rotatebox[origin=c]{90}{x}b", + ] { + assert_eq!(texts(&format!("\\begin{{document}}{src}\\end{{document}}")), vec!["axb".to_string()], "{src}"); + } +} + +#[test] +fn settings_that_are_not_text_show_nothing() { + for src in [ + "\\hyphenation{ta-ble}word", + "\\addvspace{1ex}word", + "\\enlargethispage{2\\baselineskip}word", + "\\enlargethispage*{1cm}word", + "\\selectlanguage{german}word", + "\\newlength{\\gap}\\settowidth{\\gap}{wide}word", + "\\newcounter{step}word", + ] { + assert_eq!(texts(&format!("\\begin{{document}}{src}\\end{{document}}")), vec!["word".to_string()], "{src}"); + } +} + +/// A dropped command takes only its own arguments: `\settowidth{\gap}{wide}{Visible}` keeps `Visible`. +#[test] +fn dropped_commands_keep_the_text_after_their_arguments() { + for src in [ + "\\hyphenation{ta-ble}{word}", + "\\addvspace{1ex}{word}", + "\\enlargethispage{2\\baselineskip}{word}", + "\\enlargethispage*{1cm}{word}", + "\\selectlanguage{german}{word}", + "\\newlength{\\gap}\\settowidth{\\gap}{wide}{word}", + "\\settoheight{\\gap}{tall}{word}", + "\\settodepth{\\gap}{deep}{word}", + "\\newcounter{step}[section]{word}", + "\\date{today}{word}", + "\\thanks{A note}{word}", + "\\captionsetup[figure]{font=it}{word}", + "\\fancyhf[LH]{Hidden}word", + "\\setlength{\\parskip}{1em}{word}", + "\\setlength\\parindent{0pt}word", + "\\newcommand\\foo[1][x]{body}{word}", + "\\newenvironment{note}[1]{begin}{end}{word}", + "\\def\\foo#1{body}{word}", + "\\newtheorem{thm}{Theorem}[section]{word}", + "\\addcontentsline{toc}{section}{Intro}{word}", + "\\bigskip{word}", + ] { + assert_eq!(texts(&format!("\\begin{{document}}{src}\\end{{document}}")), vec!["word".to_string()], "{src}"); + } + for (name, args) in crate::parser::DROP_WITH_ARGS { + let given: String = args.chars().map(|k| if k == 'o' { "[a]" } else { "{a}" }).collect(); + let src = format!("\\{name}{given}{{word}}"); + assert_eq!(texts(&format!("\\begin{{document}}{src}\\end{{document}}")), vec!["word".to_string()], "{src}"); + } +} + #[test] fn special_characters_round_trip() { let text = "50% of $5 & #1_a {b} ~c^ \\d |f| -- ``g'' tab\there"; diff --git a/crates/latex/tests/hostile.rs b/crates/latex/tests/hostile.rs index a1f249336..d8bf8df70 100644 --- a/crates/latex/tests/hostile.rs +++ b/crates/latex/tests/hostile.rs @@ -86,6 +86,23 @@ fn latex_labels_in_a_row_are_read_in_linear_time() { assert_eq!(d.plain_text(craft_wordmodel::StoryRef::Body).replace('\u{FFFC}', ""), "First second third."); } +/// Empty optional arguments in a row after a definition are skipped in linear time: the text after +/// each `]` is read where it lies in the token, not copied, and a run of no-break spaces at the +/// token's end is not read again for each argument. +#[test] +fn latex_optional_arguments_in_a_row_are_read_in_linear_time() { + let cases: [fn(usize) -> String; 3] = [ + |n| "\\newcommand{\\foo}".to_string() + &"[]".repeat(n) + "x", + |n| "\\usepackage".to_string() + &"[] ".repeat(n) + "x", + |n| "\\newcommand{\\foo}".to_string() + &"[]".repeat(n) + &"\u{a0}".repeat(n) + "\n\nx", + ]; + for make in cases { + assert_eq!(text(&make(3)), "x"); + let (small, large) = (fastest(&make(50_000)), fastest(&make(200_000))); + assert!(large < small * 8 + Duration::from_millis(50), "{small:?} for 50,000, {large:?} for 200,000"); + } +} + proptest::proptest! { #[test] fn fuzz_latex(s in "(\\\\(begin|end)\\{(itemize|tabular|verbatim|quote|center|x)\\}|\\\\[a-zA-Z]{1,8}\\*?|\\\\[^a-zA-Z]|[{}\\[\\]$&%~^_#]|\n\n|[a-z ]){0,150}") {