Merge pull request #1912 from scks1/fix/latex-control-space-eol

craft-latex: a backslash before a line end or a tab is a control space (wordcraft#6296)
This commit is contained in:
Dylan M. Taylor authored and GitHub committed 2026-11-16 19:39:23 -05:00
commit 02f3a449f4
3 files changed
+92 -10

No files matched your search

+32 -10
View File
@@ -68,6 +68,19 @@ impl Lexer<'_> {
self.out.push(Tok::Text(t.to_string()));
}
}
/// Skip spaces, tabs and line ends, returning the number of line ends skipped.
fn blanks(&mut self) -> usize {
let mut lines = 0;
while let Some(c) = self.peek() {
match c {
'\n' => lines += 1,
' ' | '\t' | '\r' => {}
_ => break,
}
self.bump();
}
lines
}
/// Skip the rest of a line (a comment), the line end and the next line's leading blanks.
fn comment(&mut self) {
while let Some(c) = self.bump() {
@@ -126,8 +139,26 @@ impl Lexer<'_> {
let r = self.rest();
let n = r.find(|c: char| !c.is_ascii_alphabetic()).unwrap_or(r.len());
if n == 0 {
// `\` before a line end is a control space, as TeX defines `\^^M`. TeX
// skips the next line's leading blanks, so no second space follows, but
// a blank line after it still ends the paragraph.
if matches!(self.peek(), Some('\n' | '\r')) {
self.out.push(Tok::Cmd(" ".to_string()));
if self.blanks() >= 2 {
self.out.push(Tok::Par);
}
continue;
}
let Some(sym) = self.bump() else { break };
match sym {
// `\ ` and `\^^I` (a tab) are control spaces too. TeX skips the blanks
// after them, but a blank line after one still ends the paragraph.
' ' | '\t' => {
self.out.push(Tok::Cmd(" ".to_string()));
if self.blanks() >= 2 {
self.out.push(Tok::Par);
}
}
'(' => {
let m = self.until("\\)");
self.out.push(Tok::Math(m, false));
@@ -219,16 +250,7 @@ impl Lexer<'_> {
}
'\n' | '\r' | ' ' | '\t' => {
// Whitespace: a blank line ends the paragraph, anything else is one space.
let mut lines = 0;
while let Some(c) = self.peek() {
match c {
'\n' => lines += 1,
' ' | '\t' | '\r' => {}
_ => break,
}
self.bump();
}
if lines >= 2 {
if self.blanks() >= 2 {
self.out.push(Tok::Par);
} else {
self.text(" ");
+3
View File
@@ -260,6 +260,9 @@ impl<'a> Parser<'a> {
}
Tok::Open => out.push('{'),
Tok::Close => out.push('}'),
// A control space (`\ `, or `\` before a tab or a line end) is a space, as TeX
// sets it in a label.
Tok::Cmd(c) if c == " " => out.push(' '),
Tok::Cmd(c) => {
out.push('\\');
out.push_str(c);
+57
View File
@@ -284,6 +284,63 @@ fn special_characters_round_trip() {
assert_eq!(back.plain_text(Default::default()).trim(), text.replace('\t', "\u{2003}"), "{tex}");
}
#[test]
fn a_backslash_before_a_line_end_or_a_tab_is_a_control_space() {
// The control symbol took the line end with it and read as nothing: the two words ran
// together, and a blank line after it no longer ended the paragraph.
assert_eq!(texts("\\begin{document}mid\\\nline\\end{document}"), vec!["mid line".to_string()]);
assert_eq!(texts("\\begin{document}mid\\\r\nline\\end{document}"), vec!["mid line".to_string()]);
assert_eq!(texts("\\begin{document}tab\\\there\\end{document}"), vec!["tab here".to_string()]);
assert_eq!(texts("\\begin{document}first\\\n\nsecond\\end{document}"), vec!["first".to_string(), "second".to_string()]);
assert_eq!(texts("\\begin{document}first\\\r\n\r\nsecond\\end{document}"), vec!["first".to_string(), "second".to_string()]);
}
#[test]
fn a_control_space_at_a_line_end_adds_no_second_space() {
// The line end after it was read as a space of its own, so text taken as written (a
// reference, a URL) had two spaces where `\ ` gives one.
assert_eq!(tokenize("a\\\nb"), vec![Tok::Text("a".into()), Tok::Cmd(" ".into()), Tok::Text("b".into())]);
assert_eq!(texts("\\begin{document}\\ref{one\\\ntwo}\\end{document}"), vec!["one two".to_string()]);
assert_eq!(texts("\\begin{document}\\ref{one\\\r\n two}\\end{document}"), vec!["one two".to_string()]);
}
#[test]
fn blanks_after_a_control_space_add_no_second_space() {
// TeX skips the blanks after `\ ` and after `\` and a tab, but they were read as a space of
// their own, so text taken as written had two spaces.
for cs in ["\\ ", "\\\t"] {
assert_eq!(tokenize(&format!("a{cs} \tb")), vec![Tok::Text("a".into()), Tok::Cmd(" ".into()), Tok::Text("b".into())], "{cs:?}");
assert_eq!(texts(&format!("\\begin{{document}}\\ref{{one{cs}\n two}}\\end{{document}}")), vec!["one two".to_string()], "{cs:?}");
assert_eq!(texts(&format!("\\begin{{document}}\\ref{{one{cs} two}}\\end{{document}}")), vec!["one two".to_string()], "{cs:?}");
let t = texts(&format!("\\begin{{document}}\\begin{{description}}\\item[one{cs} two] x\\end{{description}}\\end{{document}}"));
assert_eq!(t.iter().map(|s| s.trim()).collect::<Vec<_>>(), vec!["one two x"], "{cs:?}");
// A blank line after it still ends the paragraph.
assert_eq!(
texts(&format!("\\begin{{document}}first{cs}\n\nsecond\\end{{document}}")),
vec!["first".to_string(), "second".to_string()],
"{cs:?}"
);
}
}
#[test]
fn a_control_space_in_an_item_label_is_a_space() {
// The optional argument kept a control symbol as written, so the label read `one\ two`.
for src in ["one\\ two", "one\\\ttwo", "one\\\ntwo"] {
let t = texts(&format!("\\begin{{document}}\\begin{{description}}\\item[{src}] x\\end{{description}}\\end{{document}}"));
assert_eq!(t.iter().map(|s| s.trim()).collect::<Vec<_>>(), vec!["one two x"], "{src:?}");
}
}
#[test]
fn a_paragraph_that_ends_in_spaces_stays_apart_on_the_way_back() {
// Export writes the last of the spaces as a control space at the line end (`a \`), which
// import read as nothing, so the paragraph ran into the next one.
let tex = export(&Document::from_text("a \nb"));
let t: Vec<String> = texts(&tex).iter().map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect();
assert_eq!(t, vec!["a".to_string(), "b".to_string()], "{tex}");
}
#[test]
fn exported_tables_size_their_columns() {
let mut t = FTable { rows: Vec::new(), widths: vec![100.0, 200.0], borderless: false, extra: None, kept: None, wrap: None };