mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-17 15:55:14 +00:00
craft-latex: paragraph breaks survive comments, control words and CR line ends (wordcraft#6350, #6365, #6366)
Three ways the lexer lost a blank line, so two paragraphs ran together: - A comment took its line end with it, so a blank line right after it read as a single line end (wordcraft#6350). - A control word took the spaces and one line end after it, so a blank line right after `\relax` at a line end read as a single line end (wordcraft#6366). - Only LF was a line end. TeX ends a line at LF, CR or CRLF. With CR line ends every paragraph break was lost, and a comment or a verbatim environment ran to the end of the file and dropped the rest of it; with CR or CRLF a control word kept the line end after it as a space (wordcraft#6365, #6373, #6374, #6375). The lexer now reads every line end as LF. Each symptom has its own test. The CR fix needs the control-word fix before it: CRLF files read a blank line after a control word correctly only because the CR stopped the control word from taking the line end. Port of storytold/wordcraft#6387, whose files moved here with the LaTeX reader.
This commit is contained in:
1 parent
a58989cae9
commit
b974484e03
2 files changed
+90
-1
No files matched your search
+25
-1
@@ -94,6 +94,16 @@ impl Lexer<'_> {
|
||||
}
|
||||
}
|
||||
}
|
||||
/// After a comment or a control word has taken its line end: when the next line is blank,
|
||||
/// skip the blank lines and end the paragraph, as TeX does.
|
||||
fn blank_line_after_line_end(&mut self) {
|
||||
if self.peek() == Some('\n') {
|
||||
while matches!(self.peek(), Some(' ' | '\t' | '\n')) {
|
||||
self.bump();
|
||||
}
|
||||
self.out.push(Tok::Par);
|
||||
}
|
||||
}
|
||||
/// After `\begin` / `\end`: the `{name}`, if well formed.
|
||||
fn env_name(&mut self) -> Option<String> {
|
||||
let save = self.i;
|
||||
@@ -186,10 +196,16 @@ impl Lexer<'_> {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if newline {
|
||||
self.blank_line_after_line_end();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'%' => self.comment(),
|
||||
'%' => {
|
||||
self.comment();
|
||||
self.blank_line_after_line_end();
|
||||
}
|
||||
'{' => {
|
||||
self.bump();
|
||||
self.out.push(Tok::Open);
|
||||
@@ -248,6 +264,14 @@ impl Lexer<'_> {
|
||||
}
|
||||
|
||||
pub(super) fn tokenize(src: &str) -> Vec<Tok> {
|
||||
// TeX ends a line at LF, CR or CRLF alike; the lexer reads LF only.
|
||||
let lf;
|
||||
let src = if src.contains('\r') {
|
||||
lf = src.replace("\r\n", "\n").replace('\r', "\n");
|
||||
lf.as_str()
|
||||
} else {
|
||||
src
|
||||
};
|
||||
Lexer { s: src, i: 0, out: Vec::new() }.run()
|
||||
}
|
||||
|
||||
|
||||
@@ -240,6 +240,71 @@ fn fragments_without_a_document_environment() {
|
||||
assert_eq!((ps[0].kind, ps[1].kind), (Kind::Heading(1), Kind::Heading(2)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_blank_line_after_a_comment_ends_the_paragraph() {
|
||||
// The comment took its line end with it, so the blank line after it read as a single line
|
||||
// end: the paragraphs ran together with two spaces between them.
|
||||
let two = ["first", "second"];
|
||||
assert_eq!(texts("\\begin{document}first % note\n\nsecond\\end{document}"), two);
|
||||
assert_eq!(texts("\\begin{document}first\n% note\n\nsecond\\end{document}"), two);
|
||||
assert_eq!(texts("\\begin{document}first\n% note\n \t\n\nsecond\\end{document}"), two);
|
||||
// A comment still joins its line to a next line that is not blank.
|
||||
assert_eq!(texts("\\begin{document}first%\nsecond\\end{document}"), ["firstsecond"]);
|
||||
assert_eq!(texts("\\begin{document}first % note\n second\\end{document}"), ["first second"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_blank_line_after_a_control_word_at_a_line_end_ends_the_paragraph() {
|
||||
// The control word took the line end after it, so the blank line read as a single line end.
|
||||
assert_eq!(texts("\\begin{document}first \\relax\n\nsecond\\end{document}"), ["first", "second"]);
|
||||
assert_eq!(texts("\\begin{document}first \\relax \n \n second\\end{document}"), ["first", "second"]);
|
||||
assert_eq!(texts("\\begin{document}first \\relax\nsecond\\end{document}"), ["first second"]);
|
||||
}
|
||||
|
||||
/// Wraps `body` in a document whose line ends are all `nl`.
|
||||
fn with_line_ends(body: &str, nl: &str) -> String {
|
||||
format!("\\begin{{document}}{nl}{}{nl}\\end{{document}}{nl}", body.replace('\n', nl))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_blank_line_ends_the_paragraph_with_cr_and_crlf_line_ends() {
|
||||
// #6365: only LF counted as a line end, so with CR line ends every paragraph break was lost.
|
||||
for nl in ["\r", "\r\n"] {
|
||||
assert_eq!(texts(&with_line_ends("first\n\nsecond", nl)), ["first", "second"], "{nl:?}");
|
||||
assert_eq!(texts(&with_line_ends("first\nsecond", nl)), ["first second"], "{nl:?}");
|
||||
assert_eq!(texts(&with_line_ends("first \\relax\n\nsecond", nl)), ["first", "second"], "{nl:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_comment_ends_at_a_cr_line_end() {
|
||||
// #6373: a comment ran to the next LF, so with CR line ends it took the rest of the file.
|
||||
for nl in ["\r", "\r\n"] {
|
||||
assert_eq!(texts(&with_line_ends("first\n% note\nsecond\n\nthird", nl)), ["first second", "third"], "{nl:?}");
|
||||
assert_eq!(texts(&with_line_ends("first\n% note\n\nsecond", nl)), ["first", "second"], "{nl:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_verbatim_environment_starts_after_a_cr_line_end() {
|
||||
// #6374: the rest of the `\begin{verbatim}` line ran to the next LF, so with CR line ends it took the rest of the file.
|
||||
for nl in ["\r", "\r\n"] {
|
||||
assert_eq!(
|
||||
texts(&with_line_ends("before\n\n\\begin{verbatim}\ncode one\ncode two\n\\end{verbatim}\n\nafter", nl)),
|
||||
["before", "code one", "code two", "after"],
|
||||
"{nl:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_line_end_after_a_control_word_reads_as_nothing_with_cr_and_crlf_line_ends() {
|
||||
// #6375: the control word took only an LF after it, so a CR there read as a space.
|
||||
for nl in ["\r", "\r\n"] {
|
||||
assert_eq!(texts(&with_line_ends("first \\TeX\nis", nl)), ["first TeXis"], "{nl:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn equations_survive_a_round_trip() {
|
||||
let mut d = Document::new();
|
||||
|
||||
Reference in new issue
Block a user