diff --git a/crates/engine/src/cmd/tools/compare.rs b/crates/engine/src/cmd/tools/compare.rs index d275d73652..3709cb6ca0 100644 --- a/crates/engine/src/cmd/tools/compare.rs +++ b/crates/engine/src/cmd/tools/compare.rs @@ -5,7 +5,7 @@ use super::{ use wordcraft_doc::para::OBJ; /// Compare and Combine take the same settings. -pub(super) const COMPARE_PARAMS: &str = r#"{"path"?: string (revised document), "text"?: string (revised text), "original"?: string (original document; default: this one), "author"?: string (label changes with), "caseChanges"?: bool, "whiteSpace"?: bool, "formatting"?: bool (default true: differences count), "level"?: "word|character", "showIn"?: "original|revised|new" (revised/new: an untitled document), "moves"|"comments"|"tables"|"headersFooters"|"footnotes"|"textBoxes"|"fields"?: bool (accepted; not honoured yet except tables, always compared)}"#; +pub(super) const COMPARE_PARAMS: &str = r#"{"path"?: string (revised document), "text"?: string (revised text), "original"?: string (original document; default: this one), "author"?: string (label changes with), "caseChanges"?: bool, "whiteSpace"?: bool, "formatting"?: bool (default true: differences count), "level"?: "word|character" (character: paragraphs over 2000 characters fall back to word; see characterFallback/notHonoured), "showIn"?: "original|revised|new" (revised/new: an untitled document), "moves"|"comments"|"tables"|"headersFooters"|"footnotes"|"textBoxes"|"fields"?: bool (accepted; not honoured yet except tables, always compared)}"#; /// What Compare / Combine compares (the More >> settings of Word's dialog). struct CompareOptions { @@ -27,6 +27,15 @@ impl CompareOptions { } } +/// Context and tracking counters for diffing paragraph contents. +struct DiffCtx<'a> { + ins: u32, + del: u32, + opts: &'a CompareOptions, + budget: &'a mut u64, + char_fallbacks: &'a mut usize, +} + /// Changed paragraphs longer than this (in characters) are compared word by word even at /// character level, so the comparison table stays small. const MAX_CHAR_DIFF: usize = 2000; @@ -35,6 +44,9 @@ const MAX_CHAR_DIFF: usize = 2000; /// result shows the differences as tracked insertions, deletions and formatting changes, labelled /// with `author`. Options Word offers that the comparison can't honour yet (moves, comments, /// headers and footers, footnotes, text boxes, fields) are accepted and listed in the result. +/// When character-level comparison is requested (`"level": "character"`) but any paragraph exceeds +/// [`MAX_CHAR_DIFF`] characters, it falls back to word-level diff for that paragraph, surfaces the +/// fallback count in `"characterFallback"`, and includes `"level"` in `"notHonoured"`. pub(super) fn compare(s: &mut Session, v: &Value) -> CmdResult { let revised = if let Some(t) = p::str(v, "text") { Document::from_text(t) @@ -61,7 +73,7 @@ pub(super) fn compare(s: &mut Session, v: &Value) -> CmdResult { formatting: p::bool(v, "formatting").unwrap_or(true), characters: matches!(level, "character" | "char"), }; - let not_honoured: Vec<&str> = + let mut not_honoured: Vec<&str> = ["moves", "comments", "headersFooters", "footnotes", "textBoxes", "fields"].into_iter().filter(|k| v.get(*k).is_some()).collect(); let author = p::str(v, "author").map(str::trim).filter(|a| !a.is_empty()).map(str::to_string).unwrap_or_else(|| s.author.clone()); let date = crate::cmd::now_iso(); @@ -83,7 +95,9 @@ pub(super) fn compare(s: &mut Session, v: &Value) -> CmdResult { Block::Table(_) => format!("\u{0}table\u{0}{}", opts.key(&block_text(b, 0))), }; let mut budget = MAX_COMPARE_CELLS; + let mut char_fallbacks = 0usize; let ops = lcs_ops(&old.iter().map(key).collect::>(), &new.iter().map(key).collect::>(), &mut budget); + let mut ctx = DiffCtx { ins, del, opts: &opts, budget: &mut budget, char_fallbacks: &mut char_fallbacks }; let mut out: Vec> = Vec::new(); let (mut i, mut j) = (0usize, 0usize); let mut changes = 0usize; @@ -97,7 +111,7 @@ pub(super) fn compare(s: &mut Session, v: &Value) -> CmdResult { if !same_objects(&s.doc, pa, pb) { // Same text, but objects (text boxes, pictures…) differ: diffed, so the // original's objects aren't lost. - out.push(para_block(word_diff(&s.doc, pa, pb, ins, del, &opts, &mut budget))); + out.push(para_block(word_diff(&s.doc, pa, pb, &mut ctx))); changes += 1; i += 1; j += 1; @@ -129,9 +143,7 @@ pub(super) fn compare(s: &mut Session, v: &Value) -> CmdResult { for x in 0..dels.max(inss) { let (a, b) = (if x < dels { old.get(i + x) } else { None }, if x < inss { new.get(j + x) } else { None }); match (a.map(|x| &**x), b.map(|x| &**x)) { - (Some(Block::Para(pa)), Some(Block::Para(pb))) => { - out.push(para_block(word_diff(&s.doc, pa, pb, ins, del, &opts, &mut budget))) - } + (Some(Block::Para(pa)), Some(Block::Para(pb))) => out.push(para_block(word_diff(&s.doc, pa, pb, &mut ctx))), _ => { if let Some(x) = a { out.push(Arc::new(mark_block(x, &|c| c.del = Some(del), 0))); @@ -162,8 +174,21 @@ pub(super) fn compare(s: &mut Session, v: &Value) -> CmdResult { // Compare creates an untitled result from the current document. s.clear_versions(); } + if char_fallbacks > 0 { + log::warn!( + "compare: {char_fallbacks} paragraph(s) exceeded character diff limit of {MAX_CHAR_DIFF}; fell back from character to word level diff" + ); + if !not_honoured.contains(&"level") { + not_honoured.push("level"); + } + } s.sel = Selection::caret(s.doc.start_of(StoryRef::Body)); - Ok(json!({"changes": changes, "author": author, "notHonoured": not_honoured})) + Ok(json!({ + "changes": changes, + "author": author, + "notHonoured": not_honoured, + "characterFallback": char_fallbacks, + })) } /// A block's text (a table's cells in order), for matching blocks. @@ -410,15 +435,39 @@ fn put(p: &mut Paragraph, t: Option<&Tok<'_>>, props: &CharProps) { } } -fn word_diff(doc: &Document, a: &Paragraph, b: &Paragraph, ins: u32, del: u32, opts: &CompareOptions, budget: &mut u64) -> Paragraph { - let (at, bt) = (a.plain_text(), b.plain_text()); - let chars = opts.characters && at.chars().count().max(bt.chars().count()) <= MAX_CHAR_DIFF; +/// Returns `true` if paragraph `p` contains more characters than `limit`. +/// +/// Bounded to scan at most `limit + 1` characters without allocating unless inline objects +/// are present, avoiding O(n) traversal of large paragraphs. +fn para_exceeds_char_limit(p: &Paragraph, limit: usize) -> bool { + if p.objects.is_empty() { + p.text.chars().take(limit.saturating_add(1)).count() > limit + } else { + p.plain_text().chars().take(limit.saturating_add(1)).count() > limit + } +} + +/// Diffs two matched paragraphs either at word granularity or at character granularity according +/// to `ctx.opts.characters`. When character-level diff is requested but either paragraph exceeds +/// [`MAX_CHAR_DIFF`] characters, falls back to word-level diff and increments `ctx.char_fallbacks` +/// so callers can discover the fallback. +fn word_diff(doc: &Document, a: &Paragraph, b: &Paragraph, ctx: &mut DiffCtx<'_>) -> Paragraph { + let chars = if ctx.opts.characters { + if para_exceeds_char_limit(a, MAX_CHAR_DIFF) || para_exceeds_char_limit(b, MAX_CHAR_DIFF) { + *ctx.char_fallbacks = ctx.char_fallbacks.saturating_add(1); + false + } else { + true + } + } else { + false + }; let (ta, tb) = (tokens(a, chars), tokens(b, chars)); let key = |t: &Tok<'_>| match t { - Tok::Text(t) => Key::Text(opts.key(t)), + Tok::Text(t) => Key::Text(ctx.opts.key(t)), Tok::Obj(o) => obj_key(doc, o), }; - let ops = lcs_ops(&ta.iter().map(key).collect::>(), &tb.iter().map(key).collect::>(), budget); + let ops = lcs_ops(&ta.iter().map(key).collect::>(), &tb.iter().map(key).collect::>(), ctx.budget); // Only at character granularity: a matched whole word almost never happens by coincidence, // but a matched single character very often does (digits, punctuation), so only there does // a short interior match need folding back into its surrounding edit. @@ -436,11 +485,11 @@ fn word_diff(doc: &Document, a: &Paragraph, b: &Paragraph, ins: u32, del: u32, o j += 1; } Op::Del => { - put(&mut p, ta.get(i), &CharProps { del: Some(del), ..base.clone() }); + put(&mut p, ta.get(i), &CharProps { del: Some(ctx.del), ..base.clone() }); i += 1; } Op::Ins => { - put(&mut p, tb.get(j), &CharProps { ins: Some(ins), ..base.clone() }); + put(&mut p, tb.get(j), &CharProps { ins: Some(ctx.ins), ..base.clone() }); j += 1; } } diff --git a/crates/engine/src/cmd/tools/tests.rs b/crates/engine/src/cmd/tools/tests.rs index 3d2de2d844..74f10197ae 100644 --- a/crates/engine/src/cmd/tools/tests.rs +++ b/crates/engine/src/cmd/tools/tests.rs @@ -80,6 +80,145 @@ fn compare_at_word_level_still_matches_a_whole_unchanged_word() { ); } +#[test] +fn compare_character_level_surfaces_fallback_on_long_paragraphs() { + // Under MAX_CHAR_DIFF (2,000 characters), character-level diff engages normally: + // sub-word fragment is marked, characterFallback is 0, and notHonoured does not contain "level". + let original_short = "The quick brown fox jumps over the lazy dog. ".repeat(10); + let revised_short = original_short.replacen("jumps", "jumped", 1); + assert!(original_short.chars().count() < 2000); + let mut s = Session::new(Document::from_text(&original_short)); + let r = s.run("review.compare", &json!({"text": revised_short, "level": "character"})).unwrap(); + assert_eq!(r["characterFallback"].as_u64(), Some(0)); + let not_honoured = r["notHonoured"].as_array().expect("notHonoured array"); + assert!(!not_honoured.iter().any(|v| v == "level"), "level should be honoured for short paragraphs"); + let p = s.doc.body[0].as_para().expect("body paragraph"); + let deleted: String = p.run_ranges().filter(|(_, c)| c.del.is_some()).map(|(r, _)| p.text[r].to_string()).collect(); + let inserted: String = p.run_ranges().filter(|(_, c)| c.ins.is_some()).map(|(r, _)| p.text[r].to_string()).collect(); + assert_eq!(deleted, "s", "character level marks only the sub-word suffix for short paragraphs"); + assert_eq!(inserted, "ed", "character level marks only the sub-word suffix for short paragraphs"); + + // Above MAX_CHAR_DIFF (2,000 characters), character-level diff falls back to word-level diff. + // The fallback is surfaced in result metadata: characterFallback is incremented and + // notHonoured reports "level" rather than silently degrading without signal. + // In the output document, the fallback marks the whole word rather than sub-word fragments. + let original_long = "The quick brown fox jumps over the lazy dog. ".repeat(50); + let revised_long = original_long.replacen("jumps", "jumped", 1); + assert!(original_long.chars().count() > 2000); + let mut s = Session::new(Document::from_text(&original_long)); + let r = s.run("review.compare", &json!({"text": revised_long, "level": "character"})).unwrap(); + assert_eq!(r["characterFallback"].as_u64(), Some(1)); + let not_honoured = r["notHonoured"].as_array().expect("notHonoured array"); + assert!(not_honoured.iter().any(|v| v == "level"), "level should be listed in notHonoured when fallback occurs"); + let p = s.doc.body[0].as_para().expect("body paragraph"); + let deleted: String = p.run_ranges().filter(|(_, c)| c.del.is_some()).map(|(r, _)| p.text[r].to_string()).collect(); + let inserted: String = p.run_ranges().filter(|(_, c)| c.ins.is_some()).map(|(r, _)| p.text[r].to_string()).collect(); + assert_eq!(deleted, "jumps ", "long paragraph fallback marks the whole word"); + assert_eq!(inserted, "jumped ", "long paragraph fallback marks the whole word"); +} + +#[test] +fn compare_character_level_exact_boundaries_at_limit() { + let prefix = "The quick brown fox jumps over the lazy dog. ".repeat(44); // 1980 chars + + // Exactly MAX_CHAR_DIFF (2000) characters: character-level diff engages without fallback. + let original_2000 = format!("{prefix}1234567890123456789x"); + let revised_2000 = format!("{prefix}1234567890123456789y"); + assert_eq!(original_2000.chars().count(), 2000); + assert_eq!(revised_2000.chars().count(), 2000); + let mut s = Session::new(Document::from_text(&original_2000)); + let r = s.run("review.compare", &json!({"text": revised_2000, "level": "character"})).unwrap(); + assert_eq!(r["characterFallback"].as_u64(), Some(0)); + let not_honoured = r["notHonoured"].as_array().expect("notHonoured array"); + assert!(!not_honoured.iter().any(|v| v == "level"), "2000 chars should not fall back"); + let p = s.doc.body[0].as_para().expect("body paragraph"); + let deleted: String = p.run_ranges().filter(|(_, c)| c.del.is_some()).map(|(r, _)| p.text[r].to_string()).collect(); + let inserted: String = p.run_ranges().filter(|(_, c)| c.ins.is_some()).map(|(r, _)| p.text[r].to_string()).collect(); + assert_eq!(deleted, "x", "character level marks single changed character at exactly 2000 chars"); + assert_eq!(inserted, "y", "character level marks single changed character at exactly 2000 chars"); + + // Exactly 2001 characters (MAX_CHAR_DIFF + 1): triggers fallback to word-level diff. + let original_2001 = format!("{prefix}12345678901234567890x"); + let revised_2001 = format!("{prefix}12345678901234567890y"); + assert_eq!(original_2001.chars().count(), 2001); + assert_eq!(revised_2001.chars().count(), 2001); + let mut s = Session::new(Document::from_text(&original_2001)); + let r = s.run("review.compare", &json!({"text": revised_2001, "level": "character"})).unwrap(); + assert_eq!(r["characterFallback"].as_u64(), Some(1)); + let not_honoured = r["notHonoured"].as_array().expect("notHonoured array"); + assert!(not_honoured.iter().any(|v| v == "level"), "2001 chars must trigger fallback"); + let p = s.doc.body[0].as_para().expect("body paragraph"); + let deleted: String = p.run_ranges().filter(|(_, c)| c.del.is_some()).map(|(r, _)| p.text[r].to_string()).collect(); + let inserted: String = p.run_ranges().filter(|(_, c)| c.ins.is_some()).map(|(r, _)| p.text[r].to_string()).collect(); + assert_eq!(deleted, "12345678901234567890x", "fallback marks whole word at 2001 chars"); + assert_eq!(inserted, "12345678901234567890y", "fallback marks whole word at 2001 chars"); +} + +#[test] +fn compare_character_level_multiple_paragraphs_and_asymmetric_lengths() { + let short_a = "Short original text here."; + let short_b = "Short revised text here."; + let long_a = "The quick brown fox jumps over the lazy dog. ".repeat(50); + let long_b = long_a.replacen("jumps", "jumped", 1); + let long_c = "Another long paragraph that exceeds two thousand chars. ".repeat(40); + let long_d = long_c.replacen("Another", "Different", 1); + + // Document with 1 short modified paragraph and 1 long modified paragraph: fallback count is 1. + let doc_mixed_a = format!("{short_a}\n{long_a}"); + let doc_mixed_b = format!("{short_b}\n{long_b}"); + let mut s = Session::new(Document::from_text(&doc_mixed_a)); + let r = s.run("review.compare", &json!({"text": doc_mixed_b, "level": "character"})).unwrap(); + assert_eq!(r["characterFallback"].as_u64(), Some(1)); + + // Document with 2 long modified paragraphs: fallback count is 2. + let doc_2long_a = format!("{long_a}\n{long_c}"); + let doc_2long_b = format!("{long_b}\n{long_d}"); + let mut s = Session::new(Document::from_text(&doc_2long_a)); + let r = s.run("review.compare", &json!({"text": doc_2long_b, "level": "character"})).unwrap(); + assert_eq!(r["characterFallback"].as_u64(), Some(2)); + + // Asymmetric lengths: original short, revised long (> 2000 chars) -> fallback count is 1. + let mut s = Session::new(Document::from_text(short_a)); + let r = s.run("review.compare", &json!({"text": long_b, "level": "character"})).unwrap(); + assert_eq!(r["characterFallback"].as_u64(), Some(1)); + + // Asymmetric lengths: original long (> 2000 chars), revised short -> fallback count is 1. + let mut s = Session::new(Document::from_text(&long_a)); + let r = s.run("review.compare", &json!({"text": short_b, "level": "character"})).unwrap(); + assert_eq!(r["characterFallback"].as_u64(), Some(1)); +} + +#[test] +fn compare_word_level_default_and_explicit_never_flag_fallback() { + let original_long = "The quick brown fox jumps over the lazy dog. ".repeat(50); + let revised_long = original_long.replacen("jumps", "jumped", 1); + assert!(original_long.chars().count() > 2000); + + // Explicit "level": "word" on long paragraph: characterFallback is 0 and "level" not in notHonoured. + let mut s = Session::new(Document::from_text(&original_long)); + let r = s.run("review.compare", &json!({"text": revised_long, "level": "word"})).unwrap(); + assert_eq!(r["characterFallback"].as_u64(), Some(0)); + let not_honoured = r["notHonoured"].as_array().expect("notHonoured array"); + assert!(!not_honoured.iter().any(|v| v == "level")); + let p = s.doc.body[0].as_para().expect("body paragraph"); + let deleted: String = p.run_ranges().filter(|(_, c)| c.del.is_some()).map(|(r, _)| p.text[r].to_string()).collect(); + let inserted: String = p.run_ranges().filter(|(_, c)| c.ins.is_some()).map(|(r, _)| p.text[r].to_string()).collect(); + assert_eq!(deleted, "jumps "); + assert_eq!(inserted, "jumped "); + + // Default (no "level" param provided): behaves as word level, characterFallback is 0 and "level" not in notHonoured. + let mut s = Session::new(Document::from_text(&original_long)); + let r = s.run("review.compare", &json!({"text": revised_long})).unwrap(); + assert_eq!(r["characterFallback"].as_u64(), Some(0)); + let not_honoured = r["notHonoured"].as_array().expect("notHonoured array"); + assert!(!not_honoured.iter().any(|v| v == "level")); + let p = s.doc.body[0].as_para().expect("body paragraph"); + let deleted: String = p.run_ranges().filter(|(_, c)| c.del.is_some()).map(|(r, _)| p.text[r].to_string()).collect(); + let inserted: String = p.run_ranges().filter(|(_, c)| c.ins.is_some()).map(|(r, _)| p.text[r].to_string()).collect(); + assert_eq!(deleted, "jumps "); + assert_eq!(inserted, "jumped "); +} + #[test] fn compare_keeps_text_boxes_of_the_original() { // Text boxes are anchored in body paragraphs: one after changed text, one alone in an