craft-latex: paragraph breaks survive comments, control words and CR line ends (wordcraft#6350, #6365, #6366)

Three ways the lexer lost a blank line, so two paragraphs ran together:

- A comment took its line end with it, so a blank line right after it
  read as a single line end (wordcraft#6350).
- A control word took the spaces and one line end after it, so a blank
  line right after `\relax` at a line end read as a single line end
  (wordcraft#6366).
- Only LF was a line end. TeX ends a line at LF, CR or CRLF. With CR line
  ends every paragraph break was lost, and a comment or a verbatim
  environment ran to the end of the file and dropped the rest of it; with
  CR or CRLF a control word kept the line end after it as a space
  (wordcraft#6365, #6373, #6374, #6375). The lexer now reads every line
  end as LF.

Each symptom has its own test. The CR fix needs the control-word fix
before it: CRLF files read a blank line after a control word correctly
only because the CR stopped the control word from taking the line end.

Port of storytold/wordcraft#6387, whose files moved here with the LaTeX
reader.
This commit is contained in:
Simon Koch-Sultan committed 2026-11-16 12:18:30 -05:00
1 parent a58989cae9
commit b974484e03
2 files changed
+90 -1

No files matched your search

+25 -1
View File
@@ -94,6 +94,16 @@ impl Lexer<'_> {
}
}
}
/// After a comment or a control word has taken its line end: when the next line is blank,
/// skip the blank lines and end the paragraph, as TeX does.
fn blank_line_after_line_end(&mut self) {
if self.peek() == Some('\n') {
while matches!(self.peek(), Some(' ' | '\t' | '\n')) {
self.bump();
}
self.out.push(Tok::Par);
}
}
/// After `\begin` / `\end`: the `{name}`, if well formed.
fn env_name(&mut self) -> Option<String> {
let save = self.i;
@@ -186,10 +196,16 @@ impl Lexer<'_> {
break;
}
}
if newline {
self.blank_line_after_line_end();
}
}
}
}
'%' => self.comment(),
'%' => {
self.comment();
self.blank_line_after_line_end();
}
'{' => {
self.bump();
self.out.push(Tok::Open);
@@ -248,6 +264,14 @@ impl Lexer<'_> {
}
pub(super) fn tokenize(src: &str) -> Vec<Tok> {
// TeX ends a line at LF, CR or CRLF alike; the lexer reads LF only.
let lf;
let src = if src.contains('\r') {
lf = src.replace("\r\n", "\n").replace('\r', "\n");
lf.as_str()
} else {
src
};
Lexer { s: src, i: 0, out: Vec::new() }.run()
}
+65
View File
@@ -240,6 +240,71 @@ fn fragments_without_a_document_environment() {
assert_eq!((ps[0].kind, ps[1].kind), (Kind::Heading(1), Kind::Heading(2)));
}
#[test]
fn a_blank_line_after_a_comment_ends_the_paragraph() {
// The comment took its line end with it, so the blank line after it read as a single line
// end: the paragraphs ran together with two spaces between them.
let two = ["first", "second"];
assert_eq!(texts("\\begin{document}first % note\n\nsecond\\end{document}"), two);
assert_eq!(texts("\\begin{document}first\n% note\n\nsecond\\end{document}"), two);
assert_eq!(texts("\\begin{document}first\n% note\n \t\n\nsecond\\end{document}"), two);
// A comment still joins its line to a next line that is not blank.
assert_eq!(texts("\\begin{document}first%\nsecond\\end{document}"), ["firstsecond"]);
assert_eq!(texts("\\begin{document}first % note\n second\\end{document}"), ["first second"]);
}
#[test]
fn a_blank_line_after_a_control_word_at_a_line_end_ends_the_paragraph() {
// The control word took the line end after it, so the blank line read as a single line end.
assert_eq!(texts("\\begin{document}first \\relax\n\nsecond\\end{document}"), ["first", "second"]);
assert_eq!(texts("\\begin{document}first \\relax \n \n second\\end{document}"), ["first", "second"]);
assert_eq!(texts("\\begin{document}first \\relax\nsecond\\end{document}"), ["first second"]);
}
/// Wraps `body` in a document whose line ends are all `nl`.
fn with_line_ends(body: &str, nl: &str) -> String {
format!("\\begin{{document}}{nl}{}{nl}\\end{{document}}{nl}", body.replace('\n', nl))
}
#[test]
fn a_blank_line_ends_the_paragraph_with_cr_and_crlf_line_ends() {
// #6365: only LF counted as a line end, so with CR line ends every paragraph break was lost.
for nl in ["\r", "\r\n"] {
assert_eq!(texts(&with_line_ends("first\n\nsecond", nl)), ["first", "second"], "{nl:?}");
assert_eq!(texts(&with_line_ends("first\nsecond", nl)), ["first second"], "{nl:?}");
assert_eq!(texts(&with_line_ends("first \\relax\n\nsecond", nl)), ["first", "second"], "{nl:?}");
}
}
#[test]
fn a_comment_ends_at_a_cr_line_end() {
// #6373: a comment ran to the next LF, so with CR line ends it took the rest of the file.
for nl in ["\r", "\r\n"] {
assert_eq!(texts(&with_line_ends("first\n% note\nsecond\n\nthird", nl)), ["first second", "third"], "{nl:?}");
assert_eq!(texts(&with_line_ends("first\n% note\n\nsecond", nl)), ["first", "second"], "{nl:?}");
}
}
#[test]
fn a_verbatim_environment_starts_after_a_cr_line_end() {
// #6374: the rest of the `\begin{verbatim}` line ran to the next LF, so with CR line ends it took the rest of the file.
for nl in ["\r", "\r\n"] {
assert_eq!(
texts(&with_line_ends("before\n\n\\begin{verbatim}\ncode one\ncode two\n\\end{verbatim}\n\nafter", nl)),
["before", "code one", "code two", "after"],
"{nl:?}"
);
}
}
#[test]
fn the_line_end_after_a_control_word_reads_as_nothing_with_cr_and_crlf_line_ends() {
// #6375: the control word took only an LF after it, so a CR there read as a space.
for nl in ["\r", "\r\n"] {
assert_eq!(texts(&with_line_ends("first \\TeX\nis", nl)), ["first TeXis"], "{nl:?}");
}
}
#[test]
fn equations_survive_a_round_trip() {
let mut d = Document::new();