From b974484e03425a0b09351aca9958b70aa59f7283 Mon Sep 17 00:00:00 2001 From: Simon Koch-Sultan <126534658+scks1@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:31:46 -0500 Subject: [PATCH] craft-latex: paragraph breaks survive comments, control words and CR line ends (wordcraft#6350, #6365, #6366) Three ways the lexer lost a blank line, so two paragraphs ran together: - A comment took its line end with it, so a blank line right after it read as a single line end (wordcraft#6350). - A control word took the spaces and one line end after it, so a blank line right after `\relax` at a line end read as a single line end (wordcraft#6366). - Only LF was a line end. TeX ends a line at LF, CR or CRLF. With CR line ends every paragraph break was lost, and a comment or a verbatim environment ran to the end of the file and dropped the rest of it; with CR or CRLF a control word kept the line end after it as a space (wordcraft#6365, #6373, #6374, #6375). The lexer now reads every line end as LF. Each symptom has its own test. The CR fix needs the control-word fix before it: CRLF files read a blank line after a control word correctly only because the CR stopped the control word from taking the line end. Port of storytold/wordcraft#6387, whose files moved here with the LaTeX reader. --- crates/latex/src/lex.rs | 26 +++++++++++++++- crates/latex/src/tests.rs | 65 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/crates/latex/src/lex.rs b/crates/latex/src/lex.rs index 711adb551..9349ee446 100644 --- a/crates/latex/src/lex.rs +++ b/crates/latex/src/lex.rs @@ -94,6 +94,16 @@ impl Lexer<'_> { } } } + /// After a comment or a control word has taken its line end: when the next line is blank, + /// skip the blank lines and end the paragraph, as TeX does. + fn blank_line_after_line_end(&mut self) { + if self.peek() == Some('\n') { + while matches!(self.peek(), Some(' ' | '\t' | '\n')) { + self.bump(); + } + self.out.push(Tok::Par); + } + } /// After `\begin` / `\end`: the `{name}`, if well formed. fn env_name(&mut self) -> Option { let save = self.i; @@ -186,10 +196,16 @@ impl Lexer<'_> { break; } } + if newline { + self.blank_line_after_line_end(); + } } } } - '%' => self.comment(), + '%' => { + self.comment(); + self.blank_line_after_line_end(); + } '{' => { self.bump(); self.out.push(Tok::Open); @@ -248,6 +264,14 @@ impl Lexer<'_> { } pub(super) fn tokenize(src: &str) -> Vec { + // TeX ends a line at LF, CR or CRLF alike; the lexer reads LF only. + let lf; + let src = if src.contains('\r') { + lf = src.replace("\r\n", "\n").replace('\r', "\n"); + lf.as_str() + } else { + src + }; Lexer { s: src, i: 0, out: Vec::new() }.run() } diff --git a/crates/latex/src/tests.rs b/crates/latex/src/tests.rs index ccf349c80..0f596665b 100644 --- a/crates/latex/src/tests.rs +++ b/crates/latex/src/tests.rs @@ -240,6 +240,71 @@ fn fragments_without_a_document_environment() { assert_eq!((ps[0].kind, ps[1].kind), (Kind::Heading(1), Kind::Heading(2))); } +#[test] +fn a_blank_line_after_a_comment_ends_the_paragraph() { + // The comment took its line end with it, so the blank line after it read as a single line + // end: the paragraphs ran together with two spaces between them. + let two = ["first", "second"]; + assert_eq!(texts("\\begin{document}first % note\n\nsecond\\end{document}"), two); + assert_eq!(texts("\\begin{document}first\n% note\n\nsecond\\end{document}"), two); + assert_eq!(texts("\\begin{document}first\n% note\n \t\n\nsecond\\end{document}"), two); + // A comment still joins its line to a next line that is not blank. + assert_eq!(texts("\\begin{document}first%\nsecond\\end{document}"), ["firstsecond"]); + assert_eq!(texts("\\begin{document}first % note\n second\\end{document}"), ["first second"]); +} + +#[test] +fn a_blank_line_after_a_control_word_at_a_line_end_ends_the_paragraph() { + // The control word took the line end after it, so the blank line read as a single line end. + assert_eq!(texts("\\begin{document}first \\relax\n\nsecond\\end{document}"), ["first", "second"]); + assert_eq!(texts("\\begin{document}first \\relax \n \n second\\end{document}"), ["first", "second"]); + assert_eq!(texts("\\begin{document}first \\relax\nsecond\\end{document}"), ["first second"]); +} + +/// Wraps `body` in a document whose line ends are all `nl`. +fn with_line_ends(body: &str, nl: &str) -> String { + format!("\\begin{{document}}{nl}{}{nl}\\end{{document}}{nl}", body.replace('\n', nl)) +} + +#[test] +fn a_blank_line_ends_the_paragraph_with_cr_and_crlf_line_ends() { + // #6365: only LF counted as a line end, so with CR line ends every paragraph break was lost. + for nl in ["\r", "\r\n"] { + assert_eq!(texts(&with_line_ends("first\n\nsecond", nl)), ["first", "second"], "{nl:?}"); + assert_eq!(texts(&with_line_ends("first\nsecond", nl)), ["first second"], "{nl:?}"); + assert_eq!(texts(&with_line_ends("first \\relax\n\nsecond", nl)), ["first", "second"], "{nl:?}"); + } +} + +#[test] +fn a_comment_ends_at_a_cr_line_end() { + // #6373: a comment ran to the next LF, so with CR line ends it took the rest of the file. + for nl in ["\r", "\r\n"] { + assert_eq!(texts(&with_line_ends("first\n% note\nsecond\n\nthird", nl)), ["first second", "third"], "{nl:?}"); + assert_eq!(texts(&with_line_ends("first\n% note\n\nsecond", nl)), ["first", "second"], "{nl:?}"); + } +} + +#[test] +fn a_verbatim_environment_starts_after_a_cr_line_end() { + // #6374: the rest of the `\begin{verbatim}` line ran to the next LF, so with CR line ends it took the rest of the file. + for nl in ["\r", "\r\n"] { + assert_eq!( + texts(&with_line_ends("before\n\n\\begin{verbatim}\ncode one\ncode two\n\\end{verbatim}\n\nafter", nl)), + ["before", "code one", "code two", "after"], + "{nl:?}" + ); + } +} + +#[test] +fn the_line_end_after_a_control_word_reads_as_nothing_with_cr_and_crlf_line_ends() { + // #6375: the control word took only an LF after it, so a CR there read as a space. + for nl in ["\r", "\r\n"] { + assert_eq!(texts(&with_line_ends("first \\TeX\nis", nl)), ["first TeXis"], "{nl:?}"); + } +} + #[test] fn equations_survive_a_round_trip() { let mut d = Document::new();