fix(engine): surface character-level compare fallback on long paragraphs (#6111) (#6120)

crates/engine/src/cmd/tools/compare.rs

@@ -5,7 +5,7 @@ use super::{
 use wordcraft_doc::para::OBJ;
 
 /// Compare and Combine take the same settings.
-pub(super) const COMPARE_PARAMS: &str = r#"{"path"?: string (revised document), "text"?: string (revised text), "original"?: string (original document; default: this one), "author"?: string (label changes with), "caseChanges"?: bool, "whiteSpace"?: bool, "formatting"?: bool (default true: differences count), "level"?: "word|character", "showIn"?: "original|revised|new" (revised/new: an untitled document), "moves"|"comments"|"tables"|"headersFooters"|"footnotes"|"textBoxes"|"fields"?: bool (accepted; not honoured yet except tables, always compared)}"#;
+pub(super) const COMPARE_PARAMS: &str = r#"{"path"?: string (revised document), "text"?: string (revised text), "original"?: string (original document; default: this one), "author"?: string (label changes with), "caseChanges"?: bool, "whiteSpace"?: bool, "formatting"?: bool (default true: differences count), "level"?: "word|character" (character: paragraphs over 2000 characters fall back to word; see characterFallback/notHonoured), "showIn"?: "original|revised|new" (revised/new: an untitled document), "moves"|"comments"|"tables"|"headersFooters"|"footnotes"|"textBoxes"|"fields"?: bool (accepted; not honoured yet except tables, always compared)}"#;
 
 /// What Compare / Combine compares (the More >> settings of Word's dialog).
 struct CompareOptions {
@@ -27,6 +27,15 @@ impl CompareOptions {
     }
 }
 
+/// Context and tracking counters for diffing paragraph contents.
+struct DiffCtx<'a> {
+    ins: u32,
+    del: u32,
+    opts: &'a CompareOptions,
+    budget: &'a mut u64,
+    char_fallbacks: &'a mut usize,
+}
+
 /// Changed paragraphs longer than this (in characters) are compared word by word even at
 /// character level, so the comparison table stays small.
 const MAX_CHAR_DIFF: usize = 2000;
@@ -35,6 +44,9 @@ const MAX_CHAR_DIFF: usize = 2000;
 /// result shows the differences as tracked insertions, deletions and formatting changes, labelled
 /// with `author`. Options Word offers that the comparison can't honour yet (moves, comments,
 /// headers and footers, footnotes, text boxes, fields) are accepted and listed in the result.
+/// When character-level comparison is requested (`"level": "character"`) but any paragraph exceeds
+/// [`MAX_CHAR_DIFF`] characters, it falls back to word-level diff for that paragraph, surfaces the
+/// fallback count in `"characterFallback"`, and includes `"level"` in `"notHonoured"`.
 pub(super) fn compare(s: &mut Session, v: &Value) -> CmdResult {
     let revised = if let Some(t) = p::str(v, "text") {
         Document::from_text(t)
@@ -61,7 +73,7 @@ pub(super) fn compare(s: &mut Session, v: &Value) -> CmdResult {
         formatting: p::bool(v, "formatting").unwrap_or(true),
         characters: matches!(level, "character" | "char"),
     };
-    let not_honoured: Vec<&str> =
+    let mut not_honoured: Vec<&str> =
         ["moves", "comments", "headersFooters", "footnotes", "textBoxes", "fields"].into_iter().filter(|k| v.get(*k).is_some()).collect();
     let author = p::str(v, "author").map(str::trim).filter(|a| !a.is_empty()).map(str::to_string).unwrap_or_else(|| s.author.clone());
     let date = crate::cmd::now_iso();
@@ -83,7 +95,9 @@ pub(super) fn compare(s: &mut Session, v: &Value) -> CmdResult {
         Block::Table(_) => format!("\u{0}table\u{0}{}", opts.key(&block_text(b, 0))),
     };
     let mut budget = MAX_COMPARE_CELLS;
+    let mut char_fallbacks = 0usize;
     let ops = lcs_ops(&old.iter().map(key).collect::<Vec<_>>(), &new.iter().map(key).collect::<Vec<_>>(), &mut budget);
+    let mut ctx = DiffCtx { ins, del, opts: &opts, budget: &mut budget, char_fallbacks: &mut char_fallbacks };
     let mut out: Vec<Arc<Block>> = Vec::new();
     let (mut i, mut j) = (0usize, 0usize);
     let mut changes = 0usize;
@@ -97,7 +111,7 @@ pub(super) fn compare(s: &mut Session, v: &Value) -> CmdResult {
                     if !same_objects(&s.doc, pa, pb) {
                         // Same text, but objects (text boxes, pictures…) differ: diffed, so the
                         // original's objects aren't lost.
-                        out.push(para_block(word_diff(&s.doc, pa, pb, ins, del, &opts, &mut budget)));
+                        out.push(para_block(word_diff(&s.doc, pa, pb, &mut ctx)));
                         changes += 1;
                         i += 1;
                         j += 1;
@@ -129,9 +143,7 @@ pub(super) fn compare(s: &mut Session, v: &Value) -> CmdResult {
                 for x in 0..dels.max(inss) {
                     let (a, b) = (if x < dels { old.get(i + x) } else { None }, if x < inss { new.get(j + x) } else { None });
                     match (a.map(|x| &**x), b.map(|x| &**x)) {
-                        (Some(Block::Para(pa)), Some(Block::Para(pb))) => {
-                            out.push(para_block(word_diff(&s.doc, pa, pb, ins, del, &opts, &mut budget)))
-                        }
+                        (Some(Block::Para(pa)), Some(Block::Para(pb))) => out.push(para_block(word_diff(&s.doc, pa, pb, &mut ctx))),
                         _ => {
                             if let Some(x) = a {
                                 out.push(Arc::new(mark_block(x, &|c| c.del = Some(del), 0)));
@@ -162,8 +174,21 @@ pub(super) fn compare(s: &mut Session, v: &Value) -> CmdResult {
         // Compare creates an untitled result from the current document.
         s.clear_versions();
     }
+    if char_fallbacks > 0 {
+        log::warn!(
+            "compare: {char_fallbacks} paragraph(s) exceeded character diff limit of {MAX_CHAR_DIFF}; fell back from character to word level diff"
+        );
+        if !not_honoured.contains(&"level") {
+            not_honoured.push("level");
+        }
+    }
     s.sel = Selection::caret(s.doc.start_of(StoryRef::Body));
-    Ok(json!({"changes": changes, "author": author, "notHonoured": not_honoured}))
+    Ok(json!({
+        "changes": changes,
+        "author": author,
+        "notHonoured": not_honoured,
+        "characterFallback": char_fallbacks,
+    }))
 }
 
 /// A block's text (a table's cells in order), for matching blocks.
@@ -410,15 +435,39 @@ fn put(p: &mut Paragraph, t: Option<&Tok<'_>>, props: &CharProps) {
     }
 }
 
-fn word_diff(doc: &Document, a: &Paragraph, b: &Paragraph, ins: u32, del: u32, opts: &CompareOptions, budget: &mut u64) -> Paragraph {
-    let (at, bt) = (a.plain_text(), b.plain_text());
-    let chars = opts.characters && at.chars().count().max(bt.chars().count()) <= MAX_CHAR_DIFF;
+/// Returns `true` if paragraph `p` contains more characters than `limit`.
+///
+/// Bounded to scan at most `limit + 1` characters without allocating unless inline objects
+/// are present, avoiding O(n) traversal of large paragraphs.
+fn para_exceeds_char_limit(p: &Paragraph, limit: usize) -> bool {
+    if p.objects.is_empty() {
+        p.text.chars().take(limit.saturating_add(1)).count() > limit
+    } else {
+        p.plain_text().chars().take(limit.saturating_add(1)).count() > limit
+    }
+}
+
+/// Diffs two matched paragraphs either at word granularity or at character granularity according
+/// to `ctx.opts.characters`. When character-level diff is requested but either paragraph exceeds
+/// [`MAX_CHAR_DIFF`] characters, falls back to word-level diff and increments `ctx.char_fallbacks`
+/// so callers can discover the fallback.
+fn word_diff(doc: &Document, a: &Paragraph, b: &Paragraph, ctx: &mut DiffCtx<'_>) -> Paragraph {
+    let chars = if ctx.opts.characters {
+        if para_exceeds_char_limit(a, MAX_CHAR_DIFF) || para_exceeds_char_limit(b, MAX_CHAR_DIFF) {
+            *ctx.char_fallbacks = ctx.char_fallbacks.saturating_add(1);
+            false
+        } else {
+            true
+        }
+    } else {
+        false
+    };
     let (ta, tb) = (tokens(a, chars), tokens(b, chars));
     let key = |t: &Tok<'_>| match t {
-        Tok::Text(t) => Key::Text(opts.key(t)),
+        Tok::Text(t) => Key::Text(ctx.opts.key(t)),
         Tok::Obj(o) => obj_key(doc, o),
     };
-    let ops = lcs_ops(&ta.iter().map(key).collect::<Vec<_>>(), &tb.iter().map(key).collect::<Vec<_>>(), budget);
+    let ops = lcs_ops(&ta.iter().map(key).collect::<Vec<_>>(), &tb.iter().map(key).collect::<Vec<_>>(), ctx.budget);
     // Only at character granularity: a matched whole word almost never happens by coincidence,
     // but a matched single character very often does (digits, punctuation), so only there does
     // a short interior match need folding back into its surrounding edit.
@@ -436,11 +485,11 @@ fn word_diff(doc: &Document, a: &Paragraph, b: &Paragraph, ins: u32, del: u32, o
                 j += 1;
             }
             Op::Del => {
-                put(&mut p, ta.get(i), &CharProps { del: Some(del), ..base.clone() });
+                put(&mut p, ta.get(i), &CharProps { del: Some(ctx.del), ..base.clone() });
                 i += 1;
             }
             Op::Ins => {
-                put(&mut p, tb.get(j), &CharProps { ins: Some(ins), ..base.clone() });
+                put(&mut p, tb.get(j), &CharProps { ins: Some(ctx.ins), ..base.clone() });
                 j += 1;
             }
         }

crates/engine/src/cmd/tools/tests.rs

@@ -80,6 +80,145 @@ fn compare_at_word_level_still_matches_a_whole_unchanged_word() {
     );
 }
 
+#[test]
+fn compare_character_level_surfaces_fallback_on_long_paragraphs() {
+    // Under MAX_CHAR_DIFF (2,000 characters), character-level diff engages normally:
+    // sub-word fragment is marked, characterFallback is 0, and notHonoured does not contain "level".
+    let original_short = "The quick brown fox jumps over the lazy dog. ".repeat(10);
+    let revised_short = original_short.replacen("jumps", "jumped", 1);
+    assert!(original_short.chars().count() < 2000);
+    let mut s = Session::new(Document::from_text(&original_short));
+    let r = s.run("review.compare", &json!({"text": revised_short, "level": "character"})).unwrap();
+    assert_eq!(r["characterFallback"].as_u64(), Some(0));
+    let not_honoured = r["notHonoured"].as_array().expect("notHonoured array");
+    assert!(!not_honoured.iter().any(|v| v == "level"), "level should be honoured for short paragraphs");
+    let p = s.doc.body[0].as_para().expect("body paragraph");
+    let deleted: String = p.run_ranges().filter(|(_, c)| c.del.is_some()).map(|(r, _)| p.text[r].to_string()).collect();
+    let inserted: String = p.run_ranges().filter(|(_, c)| c.ins.is_some()).map(|(r, _)| p.text[r].to_string()).collect();
+    assert_eq!(deleted, "s", "character level marks only the sub-word suffix for short paragraphs");
+    assert_eq!(inserted, "ed", "character level marks only the sub-word suffix for short paragraphs");
+
+    // Above MAX_CHAR_DIFF (2,000 characters), character-level diff falls back to word-level diff.
+    // The fallback is surfaced in result metadata: characterFallback is incremented and
+    // notHonoured reports "level" rather than silently degrading without signal.
+    // In the output document, the fallback marks the whole word rather than sub-word fragments.
+    let original_long = "The quick brown fox jumps over the lazy dog. ".repeat(50);
+    let revised_long = original_long.replacen("jumps", "jumped", 1);
+    assert!(original_long.chars().count() > 2000);
+    let mut s = Session::new(Document::from_text(&original_long));
+    let r = s.run("review.compare", &json!({"text": revised_long, "level": "character"})).unwrap();
+    assert_eq!(r["characterFallback"].as_u64(), Some(1));
+    let not_honoured = r["notHonoured"].as_array().expect("notHonoured array");
+    assert!(not_honoured.iter().any(|v| v == "level"), "level should be listed in notHonoured when fallback occurs");
+    let p = s.doc.body[0].as_para().expect("body paragraph");
+    let deleted: String = p.run_ranges().filter(|(_, c)| c.del.is_some()).map(|(r, _)| p.text[r].to_string()).collect();
+    let inserted: String = p.run_ranges().filter(|(_, c)| c.ins.is_some()).map(|(r, _)| p.text[r].to_string()).collect();
+    assert_eq!(deleted, "jumps ", "long paragraph fallback marks the whole word");
+    assert_eq!(inserted, "jumped ", "long paragraph fallback marks the whole word");
+}
+
+#[test]
+fn compare_character_level_exact_boundaries_at_limit() {
+    let prefix = "The quick brown fox jumps over the lazy dog. ".repeat(44); // 1980 chars
+
+    // Exactly MAX_CHAR_DIFF (2000) characters: character-level diff engages without fallback.
+    let original_2000 = format!("{prefix}1234567890123456789x");
+    let revised_2000 = format!("{prefix}1234567890123456789y");
+    assert_eq!(original_2000.chars().count(), 2000);
+    assert_eq!(revised_2000.chars().count(), 2000);
+    let mut s = Session::new(Document::from_text(&original_2000));
+    let r = s.run("review.compare", &json!({"text": revised_2000, "level": "character"})).unwrap();
+    assert_eq!(r["characterFallback"].as_u64(), Some(0));
+    let not_honoured = r["notHonoured"].as_array().expect("notHonoured array");
+    assert!(!not_honoured.iter().any(|v| v == "level"), "2000 chars should not fall back");
+    let p = s.doc.body[0].as_para().expect("body paragraph");
+    let deleted: String = p.run_ranges().filter(|(_, c)| c.del.is_some()).map(|(r, _)| p.text[r].to_string()).collect();
+    let inserted: String = p.run_ranges().filter(|(_, c)| c.ins.is_some()).map(|(r, _)| p.text[r].to_string()).collect();
+    assert_eq!(deleted, "x", "character level marks single changed character at exactly 2000 chars");
+    assert_eq!(inserted, "y", "character level marks single changed character at exactly 2000 chars");
+
+    // Exactly 2001 characters (MAX_CHAR_DIFF + 1): triggers fallback to word-level diff.
+    let original_2001 = format!("{prefix}12345678901234567890x");
+    let revised_2001 = format!("{prefix}12345678901234567890y");
+    assert_eq!(original_2001.chars().count(), 2001);
+    assert_eq!(revised_2001.chars().count(), 2001);
+    let mut s = Session::new(Document::from_text(&original_2001));
+    let r = s.run("review.compare", &json!({"text": revised_2001, "level": "character"})).unwrap();
+    assert_eq!(r["characterFallback"].as_u64(), Some(1));
+    let not_honoured = r["notHonoured"].as_array().expect("notHonoured array");
+    assert!(not_honoured.iter().any(|v| v == "level"), "2001 chars must trigger fallback");
+    let p = s.doc.body[0].as_para().expect("body paragraph");
+    let deleted: String = p.run_ranges().filter(|(_, c)| c.del.is_some()).map(|(r, _)| p.text[r].to_string()).collect();
+    let inserted: String = p.run_ranges().filter(|(_, c)| c.ins.is_some()).map(|(r, _)| p.text[r].to_string()).collect();
+    assert_eq!(deleted, "12345678901234567890x", "fallback marks whole word at 2001 chars");
+    assert_eq!(inserted, "12345678901234567890y", "fallback marks whole word at 2001 chars");
+}
+
+#[test]
+fn compare_character_level_multiple_paragraphs_and_asymmetric_lengths() {
+    let short_a = "Short original text here.";
+    let short_b = "Short revised text here.";
+    let long_a = "The quick brown fox jumps over the lazy dog. ".repeat(50);
+    let long_b = long_a.replacen("jumps", "jumped", 1);
+    let long_c = "Another long paragraph that exceeds two thousand chars. ".repeat(40);
+    let long_d = long_c.replacen("Another", "Different", 1);
+
+    // Document with 1 short modified paragraph and 1 long modified paragraph: fallback count is 1.
+    let doc_mixed_a = format!("{short_a}\n{long_a}");
+    let doc_mixed_b = format!("{short_b}\n{long_b}");
+    let mut s = Session::new(Document::from_text(&doc_mixed_a));
+    let r = s.run("review.compare", &json!({"text": doc_mixed_b, "level": "character"})).unwrap();
+    assert_eq!(r["characterFallback"].as_u64(), Some(1));
+
+    // Document with 2 long modified paragraphs: fallback count is 2.
+    let doc_2long_a = format!("{long_a}\n{long_c}");
+    let doc_2long_b = format!("{long_b}\n{long_d}");
+    let mut s = Session::new(Document::from_text(&doc_2long_a));
+    let r = s.run("review.compare", &json!({"text": doc_2long_b, "level": "character"})).unwrap();
+    assert_eq!(r["characterFallback"].as_u64(), Some(2));
+
+    // Asymmetric lengths: original short, revised long (> 2000 chars) -> fallback count is 1.
+    let mut s = Session::new(Document::from_text(short_a));
+    let r = s.run("review.compare", &json!({"text": long_b, "level": "character"})).unwrap();
+    assert_eq!(r["characterFallback"].as_u64(), Some(1));
+
+    // Asymmetric lengths: original long (> 2000 chars), revised short -> fallback count is 1.
+    let mut s = Session::new(Document::from_text(&long_a));
+    let r = s.run("review.compare", &json!({"text": short_b, "level": "character"})).unwrap();
+    assert_eq!(r["characterFallback"].as_u64(), Some(1));
+}
+
+#[test]
+fn compare_word_level_default_and_explicit_never_flag_fallback() {
+    let original_long = "The quick brown fox jumps over the lazy dog. ".repeat(50);
+    let revised_long = original_long.replacen("jumps", "jumped", 1);
+    assert!(original_long.chars().count() > 2000);
+
+    // Explicit "level": "word" on long paragraph: characterFallback is 0 and "level" not in notHonoured.
+    let mut s = Session::new(Document::from_text(&original_long));
+    let r = s.run("review.compare", &json!({"text": revised_long, "level": "word"})).unwrap();
+    assert_eq!(r["characterFallback"].as_u64(), Some(0));
+    let not_honoured = r["notHonoured"].as_array().expect("notHonoured array");
+    assert!(!not_honoured.iter().any(|v| v == "level"));
+    let p = s.doc.body[0].as_para().expect("body paragraph");
+    let deleted: String = p.run_ranges().filter(|(_, c)| c.del.is_some()).map(|(r, _)| p.text[r].to_string()).collect();
+    let inserted: String = p.run_ranges().filter(|(_, c)| c.ins.is_some()).map(|(r, _)| p.text[r].to_string()).collect();
+    assert_eq!(deleted, "jumps ");
+    assert_eq!(inserted, "jumped ");
+
+    // Default (no "level" param provided): behaves as word level, characterFallback is 0 and "level" not in notHonoured.
+    let mut s = Session::new(Document::from_text(&original_long));
+    let r = s.run("review.compare", &json!({"text": revised_long})).unwrap();
+    assert_eq!(r["characterFallback"].as_u64(), Some(0));
+    let not_honoured = r["notHonoured"].as_array().expect("notHonoured array");
+    assert!(!not_honoured.iter().any(|v| v == "level"));
+    let p = s.doc.body[0].as_para().expect("body paragraph");
+    let deleted: String = p.run_ranges().filter(|(_, c)| c.del.is_some()).map(|(r, _)| p.text[r].to_string()).collect();
+    let inserted: String = p.run_ranges().filter(|(_, c)| c.ins.is_some()).map(|(r, _)| p.text[r].to_string()).collect();
+    assert_eq!(deleted, "jumps ");
+    assert_eq!(inserted, "jumped ");
+}
+
 #[test]
 fn compare_keeps_text_boxes_of_the_original() {
     // Text boxes are anchored in body paragraphs: one after changed text, one alone in an