diff --git a/crates/latex/src/lex.rs b/crates/latex/src/lex.rs index 711adb551..82eabe15f 100644 --- a/crates/latex/src/lex.rs +++ b/crates/latex/src/lex.rs @@ -68,6 +68,19 @@ impl Lexer<'_> { self.out.push(Tok::Text(t.to_string())); } } + /// Skip spaces, tabs and line ends, returning the number of line ends skipped. + fn blanks(&mut self) -> usize { + let mut lines = 0; + while let Some(c) = self.peek() { + match c { + '\n' => lines += 1, + ' ' | '\t' | '\r' => {} + _ => break, + } + self.bump(); + } + lines + } /// Skip the rest of a line (a comment), the line end and the next line's leading blanks. fn comment(&mut self) { while let Some(c) = self.bump() { @@ -126,8 +139,20 @@ impl Lexer<'_> { let r = self.rest(); let n = r.find(|c: char| !c.is_ascii_alphabetic()).unwrap_or(r.len()); if n == 0 { + // `\` before a line end is a control space, as TeX defines `\^^M`. TeX + // skips the next line's leading blanks, so no second space follows, but + // a blank line after it still ends the paragraph. + if matches!(self.peek(), Some('\n' | '\r')) { + self.out.push(Tok::Cmd(" ".to_string())); + if self.blanks() >= 2 { + self.out.push(Tok::Par); + } + continue; + } let Some(sym) = self.bump() else { break }; match sym { + // `\^^I`, likewise. + '\t' => self.out.push(Tok::Cmd(" ".to_string())), '(' => { let m = self.until("\\)"); self.out.push(Tok::Math(m, false)); @@ -219,16 +244,7 @@ impl Lexer<'_> { } '\n' | '\r' | ' ' | '\t' => { // Whitespace: a blank line ends the paragraph, anything else is one space. - let mut lines = 0; - while let Some(c) = self.peek() { - match c { - '\n' => lines += 1, - ' ' | '\t' | '\r' => {} - _ => break, - } - self.bump(); - } - if lines >= 2 { + if self.blanks() >= 2 { self.out.push(Tok::Par); } else { self.text(" "); diff --git a/crates/latex/src/parser.rs b/crates/latex/src/parser.rs index 36552e07a..91e419a0f 100644 --- a/crates/latex/src/parser.rs +++ b/crates/latex/src/parser.rs @@ -260,6 +260,9 @@ impl<'a> Parser<'a> { } Tok::Open => out.push('{'), Tok::Close => out.push('}'), + // A control space (`\ `, or `\` before a tab or a line end) is a space, as TeX + // sets it in a label. + Tok::Cmd(c) if c == " " => out.push(' '), Tok::Cmd(c) => { out.push('\\'); out.push_str(c); diff --git a/crates/latex/src/tests.rs b/crates/latex/src/tests.rs index ccf349c80..5f6311094 100644 --- a/crates/latex/src/tests.rs +++ b/crates/latex/src/tests.rs @@ -284,6 +284,44 @@ fn special_characters_round_trip() { assert_eq!(back.plain_text(Default::default()).trim(), text.replace('\t', "\u{2003}"), "{tex}"); } +#[test] +fn a_backslash_before_a_line_end_or_a_tab_is_a_control_space() { + // The control symbol took the line end with it and read as nothing: the two words ran + // together, and a blank line after it no longer ended the paragraph. + assert_eq!(texts("\\begin{document}mid\\\nline\\end{document}"), vec!["mid line".to_string()]); + assert_eq!(texts("\\begin{document}mid\\\r\nline\\end{document}"), vec!["mid line".to_string()]); + assert_eq!(texts("\\begin{document}tab\\\there\\end{document}"), vec!["tab here".to_string()]); + assert_eq!(texts("\\begin{document}first\\\n\nsecond\\end{document}"), vec!["first".to_string(), "second".to_string()]); + assert_eq!(texts("\\begin{document}first\\\r\n\r\nsecond\\end{document}"), vec!["first".to_string(), "second".to_string()]); +} + +#[test] +fn a_control_space_at_a_line_end_adds_no_second_space() { + // The line end after it was read as a space of its own, so text taken as written (a + // reference, a URL) had two spaces where `\ ` gives one. + assert_eq!(tokenize("a\\\nb"), vec![Tok::Text("a".into()), Tok::Cmd(" ".into()), Tok::Text("b".into())]); + assert_eq!(texts("\\begin{document}\\ref{one\\\ntwo}\\end{document}"), vec!["one two".to_string()]); + assert_eq!(texts("\\begin{document}\\ref{one\\\r\n two}\\end{document}"), vec!["one two".to_string()]); +} + +#[test] +fn a_control_space_in_an_item_label_is_a_space() { + // The optional argument kept a control symbol as written, so the label read `one\ two`. + for src in ["one\\ two", "one\\\ttwo", "one\\\ntwo"] { + let t = texts(&format!("\\begin{{document}}\\begin{{description}}\\item[{src}] x\\end{{description}}\\end{{document}}")); + assert_eq!(t.iter().map(|s| s.trim()).collect::>(), vec!["one two x"], "{src:?}"); + } +} + +#[test] +fn a_paragraph_that_ends_in_spaces_stays_apart_on_the_way_back() { + // Export writes the last of the spaces as a control space at the line end (`a \`), which + // import read as nothing, so the paragraph ran into the next one. + let tex = export(&Document::from_text("a \nb")); + let t: Vec = texts(&tex).iter().map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect(); + assert_eq!(t, vec!["a".to_string(), "b".to_string()], "{tex}"); +} + #[test] fn exported_tables_size_their_columns() { let mut t = FTable { rows: Vec::new(), widths: vec![100.0, 200.0], borderless: false, extra: None, kept: None, wrap: None };