From a699e1330535bd0a7f6b8a79ea9bedefd97325bd Mon Sep 17 00:00:00 2001 From: oracola91 <118899664+oracola91@users.noreply.github.com> Date: Wed, 4 Nov 2026 23:00:11 +0200 Subject: [PATCH 1/4] Word and HTML exports name the document's language Word export named no language, so Word checked the spelling of every export as English and underlined a whole document in any other language. The PDF already says its language in the catalog's /Lang. When that is a well formed language tag, Word gets it as the default language of the text (w:docDefaults), in the attribute Word keeps for its script: w:eastAsia for Chinese, Japanese and Korean, w:bidi for Arabic, Hebrew, Persian and Urdu, w:val for the rest. Both HTML exports, with and without CSS, get . Without /Lang, or with a malformed one, the files are exactly as before. docx_with_language, html_with_language and html_simple_with_language take the language; docx, html and html_simple stay as they are and call them with none. The engine reads /Lang through the existing initial view. The new export and engine tests fail without the change. --- crates/engine/src/compare.rs | 8 +++++--- crates/engine/src/tests.rs | 29 +++++++++++++++++++++++++++ crates/export/src/as_docx.rs | 30 +++++++++++++++++++++++++--- crates/export/src/as_html.rs | 36 ++++++++++++++++++++++++++++++---- crates/export/src/lib.rs | 4 ++-- crates/export/src/tests.rs | 38 ++++++++++++++++++++++++++++++++++++ parity/acrobat-features.toml | 8 ++++---- 7 files changed, 137 insertions(+), 16 deletions(-) diff --git a/crates/engine/src/compare.rs b/crates/engine/src/compare.rs index 9be1954b09..0996ed76ee 100644 --- a/crates/engine/src/compare.rs +++ b/crates/engine/src/compare.rs @@ -300,11 +300,13 @@ impl crate::Document { let (pages, truncated_pages) = self.export_pages_checked(!matches!(format, OfficeFormat::Rtf | OfficeFormat::Markdown), MAX_EXPORT_IMAGE_BYTES); let title = self.info.title.clone().unwrap_or_else(|| self.name.trim_end_matches(".pdf").to_string()); + // Word and HTML name the document's language (`/Lang`), so spelling is checked in it. + let language = self.initial_view().language; let bytes = match format { - OfficeFormat::Docx => pdfcraft_export::docx(&pages, &title), + OfficeFormat::Docx => pdfcraft_export::docx_with_language(&pages, &title, language.as_deref()), OfficeFormat::Pptx => pdfcraft_export::pptx(&pages, &title), - OfficeFormat::Html => pdfcraft_export::html(&pages, &title).into_bytes(), - OfficeFormat::HtmlSimple => pdfcraft_export::html_simple(&pages, &title).into_bytes(), + OfficeFormat::Html => pdfcraft_export::html_with_language(&pages, &title, language.as_deref()).into_bytes(), + OfficeFormat::HtmlSimple => pdfcraft_export::html_simple_with_language(&pages, &title, language.as_deref()).into_bytes(), OfficeFormat::Rtf => pdfcraft_export::rtf(&pages).into_bytes(), OfficeFormat::Markdown => pdfcraft_export::markdown(&pages).into_bytes(), }; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 6c54131d02..9673f2beee 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -5912,6 +5912,35 @@ fn actions_fail_when_hidden_information_could_not_be_removed() { assert!(e.starts_with("Sanitize document:"), "{e}"); } +/// A PDF whose catalog names its language (`/Lang`) exports to Word and HTML in that language, so +/// Word checks the spelling in it, not in English; one that names none exports as before. +#[test] +fn office_exports_take_the_documents_language() { + let pdf = |catalog: &str| { + pdfcraft_testkit::Pdf::new() + .obj(catalog) + .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 >>") + .obj("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >> >>") + .stream("", b"BT /F1 12 Tf 20 200 Td (Hello) Tj ET") + .build() + }; + let mut s = Session::new(); + let bg = s.open("lang.pdf", None, Arc::new(pdf("<< /Type /Catalog /Pages 2 0 R /Lang (bg-BG) >>")), None).unwrap(); + let plain = s.open("plain.pdf", None, Arc::new(pdf("<< /Type /Catalog /Pages 2 0 R >>")), None).unwrap(); + let d = s.get(bg).unwrap(); + let docx = d.export_office(compare::OfficeFormat::Docx); + assert!(docx == pdfcraft_export::docx_with_language(&d.export_pages(), "lang", Some("bg-BG")), "Word names bg-BG"); + assert!(docx != pdfcraft_export::docx(&d.export_pages(), "lang"), "Word names a language"); + for format in [compare::OfficeFormat::Html, compare::OfficeFormat::HtmlSimple] { + let html = String::from_utf8(d.export_office(format)).unwrap(); + assert!(html.contains(""), "{html}"); + } + let d = s.get(plain).unwrap(); + assert!(d.export_office(compare::OfficeFormat::Docx) == pdfcraft_export::docx(&d.export_pages(), "plain"), "no language, as before"); + assert!(String::from_utf8(d.export_office(compare::OfficeFormat::Html)).unwrap().contains("\n")); + assert!(String::from_utf8(d.export_office(compare::OfficeFormat::HtmlSimple)).unwrap().contains("\n")); +} + #[test] fn exporting_office_files_keeps_images() { let mut s = Session::new(); diff --git a/crates/export/src/as_docx.rs b/crates/export/src/as_docx.rs index ab7451dc0e..107f3676e2 100644 --- a/crates/export/src/as_docx.rs +++ b/crates/export/src/as_docx.rs @@ -1,5 +1,5 @@ use super::Zip; -use super::as_html::esc; +use super::as_html::{esc, language_tag}; use super::layout::{Image, Item, Page, Table, body_size, items}; /// A colour's 8-bit sRGB components (NaN and out-of-range values clamp; NaN reads as 0). @@ -228,9 +228,28 @@ fn word_background(pages: &[Page]) -> Option { (same && first != [255, 255, 255]).then(|| format!("{:02X}{:02X}{:02X}", first[0], first[1], first[2])) } +/// The `w:lang` that makes `tag` the language of a Word document's text. Word keeps one language +/// per script: Chinese, Japanese and Korean are its East Asian text, Arabic, Hebrew, Persian and +/// Urdu its right-to-left text, and any other language its Latin, Greek and Cyrillic text. +fn word_lang(tag: &str) -> String { + let attribute = match tag.split('-').next().unwrap_or_default().to_ascii_lowercase().as_str() { + "zh" | "ja" | "ko" => "w:eastAsia", + "ar" | "he" | "fa" | "ur" => "w:bidi", + _ => "w:val", + }; + format!("", esc(tag)) +} + /// A Word document (.docx, Office Open XML). Ordinary pages flow as paragraphs and tables; /// pages with rotated text, or text that overlaps artwork, use floating shapes to retain their layout. pub fn docx(pages: &[Page], title: &str) -> Vec { + docx_with_language(pages, title, None) +} + +/// [`docx`] for a document in `language` (a BCP 47 tag such as `bg-BG`, as a PDF's `/Lang` gives +/// it): the default language of its text, so Word checks the spelling in that language rather +/// than in its own. A language that is not a well formed tag is left out. +pub fn docx_with_language(pages: &[Page], title: &str, language: Option<&str>) -> Vec { let mut body = String::new(); let mut media: Vec<(String, &Image)> = Vec::new(); // Word needs a paragraph between adjacent tables and after the last one in the body. @@ -338,11 +357,16 @@ xmlns:wp=\"http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawin doc_rels.push_str(""); } doc_rels.push_str(""); - let styles = "\n\ + let defaults = language_tag(language) + .map(|tag| format!("{}", word_lang(tag))) + .unwrap_or_default(); + let styles = format!( + "\n{defaults}\ \ \ \ -"; +" + ); let core = format!( "\n{}", esc(title) diff --git a/crates/export/src/as_html.rs b/crates/export/src/as_html.rs index 3bc6709e69..6ae767d09c 100644 --- a/crates/export/src/as_html.rs +++ b/crates/export/src/as_html.rs @@ -8,6 +8,20 @@ pub(crate) fn allowed(c: char) -> bool { matches!(c, '\t' | '\n' | '\r') || !(c.is_ascii_control() || c == '\u{FFFE}' || c == '\u{FFFF}') } +/// `lang` when it is a well formed language tag (BCP 47, as a PDF's `/Lang` holds one): a primary +/// language of two or three letters, then subtags of one to eight letters or digits, joined by +/// hyphens, 35 characters at most. Anything else names no language. +pub(crate) fn language_tag(lang: Option<&str>) -> Option<&str> { + let tag = lang?.trim(); + let mut subtags = tag.split('-'); + let primary = subtags.next()?; + let well_formed = tag.len() <= 35 + && (2..=3).contains(&primary.len()) + && primary.bytes().all(|b| b.is_ascii_alphabetic()) + && subtags.all(|s| (1..=8).contains(&s.len()) && s.bytes().all(|b| b.is_ascii_alphanumeric())); + well_formed.then_some(tag) +} + pub(crate) fn esc(s: &str) -> String { let mut o = String::with_capacity(s.len()); for c in s.chars().filter(|c| allowed(*c)) { @@ -38,17 +52,31 @@ pub(crate) fn base64(data: &[u8]) -> String { /// One HTML file: headings and paragraphs, tables as real `` elements, images inline, a /// rule between pages. pub fn html(pages: &[Page], title: &str) -> String { - html_with_mode(pages, title, false) + html_with_language(pages, title, None) } /// A simple HTML export with no CSS; page layout and colours are omitted. pub fn html_simple(pages: &[Page], title: &str) -> String { - html_with_mode(pages, title, true) + html_simple_with_language(pages, title, None) } -fn html_with_mode(pages: &[Page], title: &str, simple: bool) -> String { +/// [`html`] for a document in `language` (a BCP 47 tag such as `bg-BG`, as a PDF's `/Lang` gives +/// it), which the page names (``) for spell checking, speech and hyphenation. A +/// language that is not a well formed tag is left out. +pub fn html_with_language(pages: &[Page], title: &str, language: Option<&str>) -> String { + html_with_mode(pages, title, false, language) +} + +/// [`html_simple`] for a document in `language`, named as [`html_with_language`] names it: a +/// language is not styling, so the simple export keeps it. +pub fn html_simple_with_language(pages: &[Page], title: &str, language: Option<&str>) -> String { + html_with_mode(pages, title, true, language) +} + +fn html_with_mode(pages: &[Page], title: &str, simple: bool, language: Option<&str>) -> String { + let lang = language_tag(language).map(|tag| format!(" lang=\"{}\"", esc(tag))).unwrap_or_default(); let mut s = format!( - "\n\n\n\n{}\n{}\n\n\n", + "\n\n\n\n{}\n{}\n\n\n", esc(title), if simple { "" diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 2b82919bbf..b8f10e7b68 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -22,14 +22,14 @@ mod as_markdown; mod as_rtf; mod layout; -pub use as_docx::docx; pub(crate) use as_docx::hex; #[cfg(test)] use as_docx::{MAX_MEDIA, docx_table}; +pub use as_docx::{docx, docx_with_language}; pub(crate) use as_html::esc; #[cfg(test)] use as_html::{allowed, base64}; -pub use as_html::{html, html_simple}; +pub use as_html::{html, html_simple, html_simple_with_language, html_with_language}; pub use as_markdown::markdown; #[cfg(test)] use as_markdown::{emphasis, paragraph}; diff --git a/crates/export/src/tests.rs b/crates/export/src/tests.rs index f630d98d5d..d62148aaf2 100644 --- a/crates/export/src/tests.rs +++ b/crates/export/src/tests.rs @@ -245,6 +245,44 @@ pub(crate) fn part(zip: &[u8], name: &str) -> String { panic!("{name} missing") } +/// The PDF's language (`/Lang`) is the default language of the Word document's text, in the +/// attribute Word keeps for its script, and the HTML page's. No language, or one that is not a +/// well formed tag, leaves both files as they were. +#[test] +fn word_and_html_take_the_documents_language() { + let pages = [page()]; + let styles = |lang| part(&docx_with_language(&pages, "Report", lang), "word/styles.xml"); + let defaults = |lang: &str| format!("{lang}"); + assert!(styles(Some("bg-BG")).contains(&defaults("")), "{}", styles(Some("bg-BG"))); + assert!(styles(Some(" en-GB ")).contains(""), "the tag is trimmed"); + assert!(styles(Some("zh-Hant-TW")).contains(&defaults("")), "East Asian text"); + assert!(styles(Some("ja")).contains(""), "East Asian text"); + assert!(styles(Some("ar-EG")).contains(&defaults("")), "right-to-left text"); + assert!(styles(Some("HE")).contains(""), "right-to-left text"); + assert!(html_with_language(&pages, "Report", Some("bg-BG")).starts_with("\n\n")); + for lang in [ + None, + Some(""), + Some(" "), + Some("bg_BG"), + Some("b"), + Some("bulg-BG"), + Some("bg--BG"), + Some("bg-"), + Some("en-US-abcdefghi"), + Some("en-abcdefgh-abcdefgh-abcdefgh-abcdefgh"), + Some("en-\"x\""), + Some("1a-BG"), + Some("\u{431}\u{433}-BG"), + ] { + assert!(docx_with_language(&pages, "Report", lang) == docx(&pages, "Report"), "{lang:?}"); + assert_eq!(html_with_language(&pages, "Report", lang), html(&pages, "Report"), "{lang:?}"); + assert_eq!(html_simple_with_language(&pages, "Report", lang), html_simple(&pages, "Report"), "{lang:?}"); + } + // The simple export leaves out styling, not the language. + assert!(html_simple_with_language(&pages, "Report", Some("bg-BG")).starts_with("\n\n")); +} + #[test] fn word_opens_text_with_control_characters_and_long_pages() { // #72: text extracted without a usable /ToUnicode holds C0 controls, which XML forbids; diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 252fb43fd2..36c415ec6b 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -7131,8 +7131,8 @@ milestone = "M10" status = "partial" commands = ["export.docx"] tools = ["doc_export_office"] -evidence = ["crates/export/src/tests.rs::word_table_cell_widths_follow_the_columns_they_span", "crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/edit/src/tests.rs::reading_keeps_dotted_leader_entries_whole_and_in_order", "crates/engine/src/tests.rs::exporting_office_keeps_dotted_leader_entries_as_paragraphs", "crates/export/src/tests.rs::word_does_not_invent_borders_for_aligned_invoice_fields", "crates/export/src/tests.rs::word_overlap_does_not_consume_invoice_body_space", "crates/export/src/tests.rs::word_keeps_a_page_with_a_clear_image_and_big_title_in_flow", "crates/export/src/tests.rs::word_rotated_pages_keep_separate_page_sizes_and_unique_shapes", "crates/engine/src/tests.rs::office_export_positions_follow_the_displayed_crop_rotation_and_units", "crates/automation/tests/it/automation.rs::exporting_an_invoice_to_word_keeps_the_watermark_out_of_flow", "crates/export/src/tests.rs::docx_is_a_word_package", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/engine/src/tests.rs::exporting_office_files_keeps_text_colour", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/export/src/tests.rs::a_link_inside_a_sentence_is_not_a_table_column", "crates/export/src/tests.rs::sentences_that_run_on_into_links_are_not_a_table", "crates/export/src/tests.rs::a_superscript_beside_a_number_keeps_its_table", "crates/export/src/tests.rs::word_opens_text_with_control_characters_and_long_pages", "crates/engine/src/tests.rs::exporting_office_files_keeps_images", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/export/src/tests.rs::docx_keeps_a_shared_page_background", "crates/edit/src/tests.rs::a_page_painted_first_with_a_full_page_fill_has_that_background", "crates/engine/src/tests.rs::exporting_to_word_keeps_the_page_background", "crates/export/src/tests.rs::an_italic_word_stays_in_its_sentence", "crates/export/src/tests.rs::a_sentence_keeps_its_reading_order", "crates/export/src/tests.rs::runs_keep_the_spaces_the_source_had_between_them", "crates/export/src/tests.rs::a_paragraph_is_a_heading_only_when_its_text_is", "crates/automation/tests/it/automation.rs::exporting_to_word_keeps_an_italic_word_inside_its_sentence"] -notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Flowing Word document: paragraphs, Heading 1/2 from larger text, bold/italic, text colour (fill as sRGB; gray/CMYK converted naively, colour spaces via cs/sc read as black), borderless detected tables with spans/empty cells (cell widths sum the grid columns they occupy, including cells after spans, verified in the XML; a row whose blocks run on into each other, such as a link or superscript inside a sentence, or overlap, as a paragraph's lines do, is text and not a table row, #2806), per-page sections/sizes, the page colour (a full-page opaque fill painted first, such as Edit a PDF ▸ Background) as the document background when every page shares one non-white colour (Word has no per-page background, so differing colours are dropped; #2992; checked in the XML only, not yet opened in Word), and positioned editable text/artwork only for pages with rotated text or text overlapped by an image or watermark-sized text, other pages flowing with inline images (#773); synthetic invoice regression; dotted-leader contents entries keep the title, dots and page number together in separate paragraphs (#1757); the emitted package validates with Open XML SDK 2.20 (Office 2010 schema). Word/LibreOffice rendering of the positioned output remains unverified. Pictures on positioned pages are DrawingML anchors behind the text (a VML rectangle with an image showed no picture, #1936; not yet opened in Word); a Word file embeds at most 1000 pictures and 512 MiB of them. The earlier flowing output was checked with macOS textutil and opened in Microsoft Word (Windows). Characters XML forbids are dropped; the page size stays within Word's 22 in. A block that continues another on its line (an italic case name, a link, a superscript) is a run in its own size, bold/italic and colour inside that paragraph, keeping the source's word space between them, rather than a paragraph of its own; a lead-in in bold no longer makes its whole line a heading (#4158). Not yet: a style change that wraps onto the next line still splits its paragraph there (the reading blocks keep style boundaries, #4158), exact font matching, vector rules, arbitrary paint-order/transparency, general exact layout fidelity." +evidence = ["crates/export/src/tests.rs::word_table_cell_widths_follow_the_columns_they_span", "crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/edit/src/tests.rs::reading_keeps_dotted_leader_entries_whole_and_in_order", "crates/engine/src/tests.rs::exporting_office_keeps_dotted_leader_entries_as_paragraphs", "crates/export/src/tests.rs::word_does_not_invent_borders_for_aligned_invoice_fields", "crates/export/src/tests.rs::word_overlap_does_not_consume_invoice_body_space", "crates/export/src/tests.rs::word_keeps_a_page_with_a_clear_image_and_big_title_in_flow", "crates/export/src/tests.rs::word_rotated_pages_keep_separate_page_sizes_and_unique_shapes", "crates/engine/src/tests.rs::office_export_positions_follow_the_displayed_crop_rotation_and_units", "crates/automation/tests/it/automation.rs::exporting_an_invoice_to_word_keeps_the_watermark_out_of_flow", "crates/export/src/tests.rs::docx_is_a_word_package", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/engine/src/tests.rs::exporting_office_files_keeps_text_colour", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/export/src/tests.rs::a_link_inside_a_sentence_is_not_a_table_column", "crates/export/src/tests.rs::sentences_that_run_on_into_links_are_not_a_table", "crates/export/src/tests.rs::a_superscript_beside_a_number_keeps_its_table", "crates/export/src/tests.rs::word_opens_text_with_control_characters_and_long_pages", "crates/engine/src/tests.rs::exporting_office_files_keeps_images", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/export/src/tests.rs::docx_keeps_a_shared_page_background", "crates/edit/src/tests.rs::a_page_painted_first_with_a_full_page_fill_has_that_background", "crates/engine/src/tests.rs::exporting_to_word_keeps_the_page_background", "crates/export/src/tests.rs::an_italic_word_stays_in_its_sentence", "crates/export/src/tests.rs::a_sentence_keeps_its_reading_order", "crates/export/src/tests.rs::runs_keep_the_spaces_the_source_had_between_them", "crates/export/src/tests.rs::a_paragraph_is_a_heading_only_when_its_text_is", "crates/automation/tests/it/automation.rs::exporting_to_word_keeps_an_italic_word_inside_its_sentence", "crates/export/src/tests.rs::word_and_html_take_the_documents_language", "crates/engine/src/tests.rs::office_exports_take_the_documents_language"] +notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Flowing Word document: paragraphs, Heading 1/2 from larger text, bold/italic, text colour (fill as sRGB; gray/CMYK converted naively, colour spaces via cs/sc read as black), borderless detected tables with spans/empty cells (cell widths sum the grid columns they occupy, including cells after spans, verified in the XML; a row whose blocks run on into each other, such as a link or superscript inside a sentence, or overlap, as a paragraph's lines do, is text and not a table row, #2806), per-page sections/sizes, the page colour (a full-page opaque fill painted first, such as Edit a PDF ▸ Background) as the document background when every page shares one non-white colour (Word has no per-page background, so differing colours are dropped; #2992; checked in the XML only, not yet opened in Word), and positioned editable text/artwork only for pages with rotated text or text overlapped by an image or watermark-sized text, other pages flowing with inline images (#773); synthetic invoice regression; dotted-leader contents entries keep the title, dots and page number together in separate paragraphs (#1757); the emitted package validates with Open XML SDK 2.20 (Office 2010 schema). Word/LibreOffice rendering of the positioned output remains unverified. Pictures on positioned pages are DrawingML anchors behind the text (a VML rectangle with an image showed no picture, #1936; not yet opened in Word); a Word file embeds at most 1000 pictures and 512 MiB of them. The earlier flowing output was checked with macOS textutil and opened in Microsoft Word (Windows). Characters XML forbids are dropped; the page size stays within Word's 22 in. A block that continues another on its line (an italic case name, a link, a superscript) is a run in its own size, bold/italic and colour inside that paragraph, keeping the source's word space between them, rather than a paragraph of its own; a lead-in in bold no longer makes its whole line a heading (#4158). The PDF's language (`/Lang`, when it is a well formed BCP 47 tag) is the default language of the text, in the attribute Word keeps for its script (East Asian, right to left or other), so Word checks the spelling in it rather than in English (checked in the XML only, not yet opened in Word). Not yet: a style change that wraps onto the next line still splits its paragraph there (the reading blocks keep style boundaries, #4158), exact font matching, vector rules, arbitrary paint-order/transparency, general exact layout fidelity." [[feature]] id = "create.export-rtf" @@ -7255,8 +7255,8 @@ milestone = "M10" status = "partial" commands = ["export.html"] tools = ["doc_export_office"] -evidence = ["crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/export/src/tests.rs::html_has_headings_paragraphs_and_images_in_order", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/automation/tests/it/automation.rs::exporting_office_files_says_which_pages_left_form_text_out", "crates/edit/src/tests.rs::reading_follows_forms_as_far_as_the_renderer_and_says_where_it_stops"] -notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Single HTML file with headings, paragraphs (with text colour), inline images, and detected tables with spans/empty cells. Text and images in form XObjects are read 50 levels deep, as the renderer does, and up to 50 000 forms a page; the tool lists pages past that in truncated_pages, with a warning (#3120). Not yet: multi-page folder output, CSS from fonts." +evidence = ["crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/export/src/tests.rs::html_has_headings_paragraphs_and_images_in_order", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/automation/tests/it/automation.rs::exporting_office_files_says_which_pages_left_form_text_out", "crates/edit/src/tests.rs::reading_follows_forms_as_far_as_the_renderer_and_says_where_it_stops", "crates/export/src/tests.rs::word_and_html_take_the_documents_language", "crates/engine/src/tests.rs::office_exports_take_the_documents_language"] +notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Single HTML file with headings, paragraphs (with text colour), inline images, and detected tables with spans/empty cells. The page names the PDF's language (``, from a well formed `/Lang`). Text and images in form XObjects are read 50 levels deep, as the renderer does, and up to 50 000 forms a page; the tool lists pages past that in truncated_pages, with a warning (#3120). Not yet: multi-page folder output, CSS from fonts." [[feature]] id = "create.export-text" From 0cae1c518b863ab660272b375710a3261e1a3c34 Mon Sep 17 00:00:00 2001 From: oracola91 <118899664+oracola91@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:10:15 +0200 Subject: [PATCH 2/4] A language tag longer than 35 characters is still a tag BCP 47 sets no maximum length; 35 characters is only the least a protocol must hold. A well formed tag such as en-US-u-ca-gregory-nu-latn-co-phonebk was left out of both exports. The bound is now 255, kept only so a PDF's /Lang can't put an unbounded string in the file. --- crates/export/src/as_html.rs | 8 ++++++-- crates/export/src/tests.rs | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/export/src/as_html.rs b/crates/export/src/as_html.rs index 6ae767d09c..45f6ecafca 100644 --- a/crates/export/src/as_html.rs +++ b/crates/export/src/as_html.rs @@ -8,14 +8,18 @@ pub(crate) fn allowed(c: char) -> bool { matches!(c, '\t' | '\n' | '\r') || !(c.is_ascii_control() || c == '\u{FFFE}' || c == '\u{FFFF}') } +/// The longest language tag the exports write. BCP 47 sets no maximum (35 characters is only +/// the least a protocol must hold), so this bounds only what a PDF's `/Lang` can put in a file. +const LONGEST_TAG: usize = 255; + /// `lang` when it is a well formed language tag (BCP 47, as a PDF's `/Lang` holds one): a primary /// language of two or three letters, then subtags of one to eight letters or digits, joined by -/// hyphens, 35 characters at most. Anything else names no language. +/// hyphens, [`LONGEST_TAG`] characters at most. Anything else names no language. pub(crate) fn language_tag(lang: Option<&str>) -> Option<&str> { let tag = lang?.trim(); let mut subtags = tag.split('-'); let primary = subtags.next()?; - let well_formed = tag.len() <= 35 + let well_formed = tag.len() <= LONGEST_TAG && (2..=3).contains(&primary.len()) && primary.bytes().all(|b| b.is_ascii_alphabetic()) && subtags.all(|s| (1..=8).contains(&s.len()) && s.bytes().all(|b| b.is_ascii_alphanumeric())); diff --git a/crates/export/src/tests.rs b/crates/export/src/tests.rs index d62148aaf2..ba5f95f66d 100644 --- a/crates/export/src/tests.rs +++ b/crates/export/src/tests.rs @@ -260,6 +260,11 @@ fn word_and_html_take_the_documents_language() { assert!(styles(Some("ar-EG")).contains(&defaults("")), "right-to-left text"); assert!(styles(Some("HE")).contains(""), "right-to-left text"); assert!(html_with_language(&pages, "Report", Some("bg-BG")).starts_with("\n\n")); + // A tag longer than 35 characters is still a tag. + let long = "en-US-u-ca-gregory-nu-latn-co-phonebk"; + assert!(styles(Some(long)).contains(&format!("")), "{long}"); + assert!(html_with_language(&pages, "Report", Some(long)).contains(&format!("")), "{long}"); + let too_long = format!("en{}", "-abcdefgh".repeat(29)); for lang in [ None, Some(""), @@ -270,7 +275,7 @@ fn word_and_html_take_the_documents_language() { Some("bg--BG"), Some("bg-"), Some("en-US-abcdefghi"), - Some("en-abcdefgh-abcdefgh-abcdefgh-abcdefgh"), + Some(too_long.as_str()), Some("en-\"x\""), Some("1a-BG"), Some("\u{431}\u{433}-BG"), From 561cc0d7079fef5d2f8b2a3af4715c91b3c1cb92 Mon Sep 17 00:00:00 2001 From: oracola91 <118899664+oracola91@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:10:33 +0200 Subject: [PATCH 3/4] A language tag can't end in a singleton A singleton (one letter or digit) starts an extension, or with x private use, and BCP 47 needs a subtag after it. en-x and en-a were written as the document's language. They and a singleton straight after another are now left out; one-character subtags inside private use stay allowed. --- crates/export/src/as_html.rs | 29 +++++++++++++++++++++++------ crates/export/src/tests.rs | 6 ++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/crates/export/src/as_html.rs b/crates/export/src/as_html.rs index 45f6ecafca..8c250ca484 100644 --- a/crates/export/src/as_html.rs +++ b/crates/export/src/as_html.rs @@ -14,16 +14,33 @@ const LONGEST_TAG: usize = 255; /// `lang` when it is a well formed language tag (BCP 47, as a PDF's `/Lang` holds one): a primary /// language of two or three letters, then subtags of one to eight letters or digits, joined by -/// hyphens, [`LONGEST_TAG`] characters at most. Anything else names no language. +/// hyphens, [`LONGEST_TAG`] characters at most. A singleton (one letter or digit) starts an +/// extension, or with `x` private use, and needs a subtag after it. Anything else names no +/// language. pub(crate) fn language_tag(lang: Option<&str>) -> Option<&str> { let tag = lang?.trim(); let mut subtags = tag.split('-'); let primary = subtags.next()?; - let well_formed = tag.len() <= LONGEST_TAG - && (2..=3).contains(&primary.len()) - && primary.bytes().all(|b| b.is_ascii_alphabetic()) - && subtags.all(|s| (1..=8).contains(&s.len()) && s.bytes().all(|b| b.is_ascii_alphanumeric())); - well_formed.then_some(tag) + if tag.len() > LONGEST_TAG || !(2..=3).contains(&primary.len()) || !primary.bytes().all(|b| b.is_ascii_alphabetic()) { + return None; + } + // In private use every subtag is data, one character long or not. + let (mut private, mut singleton) = (false, false); + for s in subtags { + if !(1..=8).contains(&s.len()) || !s.bytes().all(|b| b.is_ascii_alphanumeric()) { + return None; + } + if s.len() == 1 && !private { + if singleton { + return None; + } + private = s.eq_ignore_ascii_case("x"); + singleton = true; + } else { + singleton = false; + } + } + (!singleton).then_some(tag) } pub(crate) fn esc(s: &str) -> String { diff --git a/crates/export/src/tests.rs b/crates/export/src/tests.rs index ba5f95f66d..5820fc57bc 100644 --- a/crates/export/src/tests.rs +++ b/crates/export/src/tests.rs @@ -265,6 +265,8 @@ fn word_and_html_take_the_documents_language() { assert!(styles(Some(long)).contains(&format!("")), "{long}"); assert!(html_with_language(&pages, "Report", Some(long)).contains(&format!("")), "{long}"); let too_long = format!("en{}", "-abcdefgh".repeat(29)); + // Private use takes one-character subtags. + assert!(html_with_language(&pages, "Report", Some("en-x-a-b")).contains("")); for lang in [ None, Some(""), @@ -274,6 +276,10 @@ fn word_and_html_take_the_documents_language() { Some("bulg-BG"), Some("bg--BG"), Some("bg-"), + Some("en-x"), + Some("en-a"), + Some("en-a-x-private"), + Some("en-US-u-ca-x"), Some("en-US-abcdefghi"), Some(too_long.as_str()), Some("en-\"x\""), From a427cd2de59e981449ebdb8072493d69d15b5320 Mon Sep 17 00:00:00 2001 From: oracola91 <118899664+oracola91@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:10:57 +0200 Subject: [PATCH 4/4] Word takes the script a language tag names pa-Arab-PK is Punjabi in Arabic script, which Word keeps as right-to-left text, but word_lang looked only at the language and wrote w:val. A script subtag (Arab, Hebr, Syrc, Thaa; Hans, Hant, Hani, Jpan, Kore, Hang, Hira, Kana; Latn, Cyrl, Grek) now decides over the language. It is read only in its own place, after the language and any extended languages, so private use such as en-x-arab names no script. --- crates/export/src/as_docx.rs | 20 ++++++++++++++++---- crates/export/src/tests.rs | 6 ++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/crates/export/src/as_docx.rs b/crates/export/src/as_docx.rs index 107f3676e2..4fa7edfce1 100644 --- a/crates/export/src/as_docx.rs +++ b/crates/export/src/as_docx.rs @@ -230,11 +230,23 @@ fn word_background(pages: &[Page]) -> Option { /// The `w:lang` that makes `tag` the language of a Word document's text. Word keeps one language /// per script: Chinese, Japanese and Korean are its East Asian text, Arabic, Hebrew, Persian and -/// Urdu its right-to-left text, and any other language its Latin, Greek and Cyrillic text. +/// Urdu its right-to-left text, and any other language its Latin, Greek and Cyrillic text. A +/// script the tag names (`pa-Arab`, `zh-Latn`) decides over its language. fn word_lang(tag: &str) -> String { - let attribute = match tag.split('-').next().unwrap_or_default().to_ascii_lowercase().as_str() { - "zh" | "ja" | "ko" => "w:eastAsia", - "ar" | "he" | "fa" | "ur" => "w:bidi", + let mut subtags = tag.split('-'); + let primary = subtags.next().unwrap_or_default().to_ascii_lowercase(); + // The script comes straight after the language and any three-letter extended languages; a + // singleton before it starts an extension or private use, which names no script. + let script = subtags + .find(|s| s.len() != 3 || !s.bytes().all(|b| b.is_ascii_alphabetic())) + .filter(|s| s.len() == 4 && s.bytes().all(|b| b.is_ascii_alphabetic())) + .map(str::to_ascii_lowercase); + let attribute = match (script.as_deref(), primary.as_str()) { + (Some("arab" | "hebr" | "syrc" | "thaa"), _) => "w:bidi", + (Some("hans" | "hant" | "hani" | "jpan" | "kore" | "hang" | "hira" | "kana"), _) => "w:eastAsia", + (Some("latn" | "cyrl" | "grek"), _) => "w:val", + (_, "zh" | "ja" | "ko") => "w:eastAsia", + (_, "ar" | "he" | "fa" | "ur") => "w:bidi", _ => "w:val", }; format!("", esc(tag)) diff --git a/crates/export/src/tests.rs b/crates/export/src/tests.rs index 5820fc57bc..baaf323117 100644 --- a/crates/export/src/tests.rs +++ b/crates/export/src/tests.rs @@ -259,6 +259,12 @@ fn word_and_html_take_the_documents_language() { assert!(styles(Some("ja")).contains(""), "East Asian text"); assert!(styles(Some("ar-EG")).contains(&defaults("")), "right-to-left text"); assert!(styles(Some("HE")).contains(""), "right-to-left text"); + // A script subtag decides over the language; private use names no script. + assert!(styles(Some("pa-Arab-PK")).contains(""), "Punjabi in Arabic script"); + assert!(styles(Some("sr-Latn")).contains(""), "Serbian in Latin script"); + assert!(styles(Some("zh-Latn")).contains(""), "Chinese in Latin script"); + assert!(styles(Some("en-x-arab")).contains(""), "private use"); + assert!(styles(Some("az-Latn-x-arab")).contains(""), "private use"); assert!(html_with_language(&pages, "Report", Some("bg-BG")).starts_with("\n\n")); // A tag longer than 35 characters is still a tag. let long = "en-US-u-ca-gregory-nu-latn-co-phonebk";