diff --git a/crates/core/src/value.rs b/crates/core/src/value.rs index 0000028e9..a0b4fcad6 100644 --- a/crates/core/src/value.rs +++ b/crates/core/src/value.rs @@ -346,19 +346,91 @@ pub fn compare_numbers(a: f64, b: f64) -> Ordering { round15(a).partial_cmp(&round15(b)).unwrap_or(Ordering::Equal) } -/// Case-insensitive text order (Unicode lowercase). +/// Case-insensitive text order in three levels, as Excel's collation sorts: first by letters and +/// digits alone (Arabic marks, tatweel and invisible format characters skipped, Arabic-Indic digits +/// read as digits); then, letter by letter, by the marks on each letter (none first, then in +/// Excel's mark order) and the digit script (ASCII, Arabic-Indic, Extended Arabic-Indic); then, to +/// break a remaining tie, by every character (Unicode lowercase). Only text that is equal at the +/// last level compares equal. pub fn compare_text(a: &str, b: &str) -> Ordering { - let mut ai = a.chars().flat_map(char::to_lowercase); - let mut bi = b.chars().flat_map(char::to_lowercase); - loop { - match (ai.next(), bi.next()) { - (None, None) => return Ordering::Equal, - (None, Some(_)) => return Ordering::Less, - (Some(_), None) => return Ordering::Greater, - (Some(x), Some(y)) if x != y => return x.cmp(&y), - _ => {} + if !(a.is_ascii() && b.is_ascii()) { + let primary = primary_chars(a).cmp(primary_chars(b)); + if primary != Ordering::Equal { + return primary; + } + let secondary = secondary_weights(a).cmp(secondary_weights(b)); + if secondary != Ordering::Equal { + return secondary; } } + lowercase_chars(a).cmp(lowercase_chars(b)) +} + +fn lowercase_chars(s: &str) -> impl Iterator + '_ { + s.chars().flat_map(char::to_lowercase) +} + +/// Characters Excel's text order skips entirely: tatweel and invisible format characters (soft +/// hyphen, combining grapheme joiner, zero-width (non-)joiner, bidi marks, embeddings, overrides +/// and isolates, word joiner, byte order mark). The zero-width space is not one. +fn ignorable(c: char) -> bool { + matches!( + c, + '\u{00AD}' + | '\u{034F}' + | '\u{061C}' + | '\u{0640}' + | '\u{200C}'..='\u{200F}' + | '\u{202A}'..='\u{202E}' + | '\u{2060}' + | '\u{2066}'..='\u{2069}' + | '\u{FEFF}' + ) +} + +/// Arabic combining marks in the order Excel sorts a letter carrying them; an unmarked letter +/// sorts before all of them. Measured in Excel 16.0 (build 20527, en-US) with one mark on a letter. +/// U+06D6 and U+06D7 sort as letters in Excel, so they are not listed. +const ARABIC_MARK_ORDER: [char; 48] = [ + '\u{06EC}', '\u{0611}', '\u{0612}', '\u{0613}', '\u{0614}', '\u{06DB}', '\u{0656}', '\u{0657}', '\u{0610}', '\u{06EB}', '\u{0615}', '\u{06D9}', + '\u{06E4}', '\u{06E1}', '\u{06DA}', '\u{06DC}', '\u{06EA}', '\u{06E3}', '\u{06D8}', '\u{0655}', '\u{065F}', '\u{0654}', '\u{06E7}', '\u{0653}', + '\u{06E8}', '\u{06ED}', '\u{06E2}', '\u{0658}', '\u{064B}', '\u{064C}', '\u{064D}', '\u{064E}', '\u{064F}', '\u{0650}', '\u{0652}', '\u{0651}', + '\u{0670}', '\u{0659}', '\u{065A}', '\u{065B}', '\u{065C}', '\u{065D}', '\u{065E}', '\u{0616}', '\u{0617}', '\u{0618}', '\u{0619}', '\u{061A}', +]; + +/// Rank of an Arabic mark that letter order skips (its index in `ARABIC_MARK_ORDER`). +fn arabic_mark_rank(c: char) -> Option { + ARABIC_MARK_ORDER.iter().position(|&m| m == c).and_then(|i| u16::try_from(i).ok()) +} + +/// Digit script of `c` (0 ASCII, 1 Arabic-Indic, 2 Extended Arabic-Indic) and its ASCII digit. +fn digit(c: char) -> Option<(u16, char)> { + let (script, zero) = match c { + '0'..='9' => return Some((0, c)), + '\u{0660}'..='\u{0669}' => (1, 0x0660), + '\u{06F0}'..='\u{06F9}' => (2, 0x06F0), + _ => return None, + }; + char::from_u32(u32::from(c) - zero + u32::from('0')).map(|d| (script, d)) +} + +/// The characters that decide text order before any tie-break. +fn primary_chars(s: &str) -> impl Iterator + '_ { + lowercase_chars(s).filter_map(|c| if ignorable(c) || arabic_mark_rank(c).is_some() { None } else { Some(digit(c).map_or(c, |(_, d)| d)) }) +} + +/// The second level, for text whose primary characters are equal: per letter, its digit script and +/// then its marks, the letters' lists closed by 0 so that an unmarked letter sorts first. Marks +/// before the first letter form a list of their own. +fn secondary_weights(s: &str) -> impl Iterator + '_ { + lowercase_chars(s) + .filter(|&c| !ignorable(c)) + .flat_map(|c| match arabic_mark_rank(c) { + Some(rank) => [Some(rank.saturating_add(1)), None], + None => [Some(0), Some(digit(c).map_or(0, |(script, _)| script))], + }) + .flatten() + .chain(std::iter::once(0)) } /// Sorting order used by Sort (ascending): numbers, text, booleans, errors, then blanks last. @@ -457,6 +529,182 @@ mod tests { assert_eq!(sort_compare(&Value::Empty, &Value::Number(1.0)), Ordering::Greater); } + #[test] + fn arabic_diacritics_sort_with_their_base_letter() { + assert_eq!(compare_text("بَاب", "بيت"), Ordering::Less); + assert_eq!(compare_text("بيت", "بَاب"), Ordering::Greater); + assert_eq!(compare_text("كِتاب", "كتب"), Ordering::Less); + assert_eq!(compare_text("رحمٰن", "رحمي"), Ordering::Less); + assert_eq!(compare(&Value::from("بَاب"), &Value::from("بيت")), Ordering::Less); + } + + #[test] + fn arabic_indic_digits_sort_with_digits() { + assert_eq!(compare_text("١٢", "ب"), Ordering::Less); + assert_eq!(compare_text("۱۲", "ب"), Ordering::Less); + assert_eq!(compare_text("١", "a"), Ordering::Less); + assert_eq!(compare_text("١٢", "13"), Ordering::Less); + assert_eq!(compare_text("۱۴", "13"), Ordering::Greater); + } + + #[test] + fn tatweel_and_bidi_marks_do_not_move_sort_order() { + assert_eq!(compare_text("بـاب", "بت"), Ordering::Less); + assert_eq!(compare_text("\u{200F}apple", "banana"), Ordering::Less); + assert_eq!(compare_text("\u{200E}zebra", "apple"), Ordering::Greater); + assert_eq!(compare_text("\u{061C}a", "b"), Ordering::Less); + assert_eq!(compare_text("a\u{200C}b", "ac"), Ordering::Less); + assert_eq!(compare_text("a\u{200D}b", "ac"), Ordering::Less); + } + + #[test] + fn primary_ties_keep_equality_unchanged() { + assert_eq!(compare_text("ABC", "abc"), Ordering::Equal); + assert_eq!(compare_text("باب", "بَاب"), Ordering::Less); + assert_eq!(compare_text("12", "١٢"), Ordering::Less); + assert_eq!(compare_text("a", "\u{200F}a"), Ordering::Less); + let samples = [ + "", + "abc", + "ABC", + "abd", + "ab", + "باب", + "بَاب", + "بِاب", + "بـاب", + "بيت", + "١٢", + "۱۲", + "12", + "\u{200F}12", + "\u{200F}a", + "a", + "A\u{200C}", + "a\u{200F}", + "b", + "Straße", + "STRASSE", + "ΣΑΣ", + "σας", + "İ", + "i̇", + ]; + let lowered = |s: &str| s.chars().flat_map(char::to_lowercase).collect::(); + for a in samples { + for b in samples { + let order = compare_text(a, b); + assert_eq!(order.is_eq(), lowered(a) == lowered(b), "{a:?} vs {b:?}"); + assert_eq!(order, compare_text(b, a).reverse(), "{a:?} vs {b:?}"); + for c in samples { + if order.is_le() && compare_text(b, c).is_le() { + assert!(compare_text(a, c).is_le(), "{a:?} <= {b:?} <= {c:?}"); + } + } + } + } + } + + /// Sorts `expected` from reversed and rotated starts and checks each comes back as `expected`. + #[track_caller] + fn assert_sorts_to(expected: &[&str]) { + let mut reversed: Vec<&str> = expected.iter().rev().copied().collect(); + reversed.sort_by(|a, b| compare_text(a, b)); + assert_eq!(reversed, expected); + let mut rotated: Vec<&str> = expected.to_vec(); + rotated.rotate_left(expected.len() / 2); + rotated.sort_by(|a, b| compare_text(a, b)); + assert_eq!(rotated, expected); + } + + // The expected orders below are Excel's, observed with Range.Sort, =SORT() and the `<` + // operator in Excel 16.0 (build 20527, en-US). Strings Excel ranks equal are left out. + + #[test] + fn excel_orders_a_letter_by_its_mark() { + assert_sorts_to(&[ + "باب", + "ب\u{0611}اب", + "ب\u{0610}اب", + "ب\u{0654}اب", + "ب\u{064B}اب", + "ب\u{064C}اب", + "ب\u{064D}اب", + "ب\u{064E}اب", + "ب\u{064F}اب", + "ب\u{0650}اب", + "ب\u{0652}اب", + "ب\u{0651}اب", + "ب\u{0670}اب", + "ب\u{065E}اب", + "ب\u{061A}اب", + ]); + // Quranic marks are skipped by letter order like harakat (hamza sorts before alef). + assert_eq!(compare_text("ب\u{0610}ء", "باب"), Ordering::Less); + assert_eq!(compare_text("ب\u{06E1}ء", "باب"), Ordering::Less); + } + + #[test] + fn excel_compares_marks_letter_by_letter() { + assert_sorts_to(&[ + "بت", + "بت\u{064E}", + "بت\u{064F}", + "بت\u{0650}", + "ب\u{064E}ت", + "ب\u{064E}ت\u{064E}", + "ب\u{064E}ت\u{064F}", + "ب\u{064F}ت", + "ب\u{064F}ت\u{064E}", + "ب\u{0650}ت\u{0650}", + ]); + assert_sorts_to(&["مـحـمـد", "محم\u{0651}د", "م\u{064E}حمد", "م\u{064F}ح\u{064E}م\u{064E}\u{0651}د", "محمدي"]); + } + + #[test] + fn excel_weighs_marks_and_digit_scripts_by_position() { + // A mark on an earlier letter outranks the digit script or letter after it, whatever the + // code point of that next character. + assert_sorts_to(&[ + "ب", + "ب\u{064E}", + "ب1", + "ب١", + "ب۱", + "ب\u{064E}1", + "ب\u{064E}١", + "ب\u{064E}۱", + "بa", + "ب\u{064E}a", + "با", + "ب\u{064E}ا", + "ب\u{067E}", + "ب\u{064E}\u{067E}", + "ب\u{06CC}", + "ب\u{064E}\u{06CC}", + ]); + // And an earlier digit script outranks a later mark. + assert_sorts_to(&["12", "1٢", "١2", "١٢", "1ب", "1ب\u{064E}", "1ب\u{0650}", "١ب", "١ب\u{064E}", "١ب\u{0650}", "۱ب", "۱ب\u{064E}"]); + assert_sorts_to(&["ب1ب", "ب1ب\u{064E}", "ب١ب", "ب\u{064E}ب", "بب1", "بب١", "ب\u{064E}ب1"]); + } + + #[test] + fn excel_skips_invisible_format_characters() { + for c in ['\u{00AD}', '\u{034F}', '\u{061C}', '\u{200C}', '\u{200D}', '\u{200E}', '\u{200F}', '\u{202A}', '\u{202B}'] + .into_iter() + .chain(['\u{202C}', '\u{202D}', '\u{202E}', '\u{2060}', '\u{2066}', '\u{2067}', '\u{2068}', '\u{2069}', '\u{FEFF}']) + { + let s = format!("{c}b"); + assert_eq!(compare_text(&s, "a"), Ordering::Greater, "{s:?}"); + assert_eq!(compare_text(&s, "ba"), Ordering::Less, "{s:?}"); + assert_eq!(compare_text(&format!("ب{c}\u{064E}اب"), "باب\u{064B}"), Ordering::Greater, "{c:?}"); + } + // Not even as a tie-break: a leading mark doesn't push text past marked or other-script forms. + assert_sorts_to(&["\u{200F}باب", "ب\u{064B}اب", "ب\u{064E}اب", "ب\u{0670}اب"]); + assert_sorts_to(&["\u{200F}12", "١٢", "۱۲"]); + assert_sorts_to(&["ب\u{200F}ب\u{064E}", "\u{200F}ب\u{064E}ب"]); + } + #[test] fn errors() { for e in CellError::ALL { diff --git a/crates/engine/src/cmd/data/filter.rs b/crates/engine/src/cmd/data/filter.rs index 437bd7887..cda2160bf 100644 --- a/crates/engine/src/cmd/data/filter.rs +++ b/crates/engine/src/cmd/data/filter.rs @@ -317,5 +317,9 @@ pub fn filter_values(wb: &Workbook, sheet: usize, col: u32) -> Vec<(String, bool let key = if blank { sort_key(&Value::Empty) } else { sort_key(&v) }; seen.entry((key, t)).or_insert(on); } - seen.into_iter().map(|((_, t), on)| (t, on)).collect() + let mut values: Vec<_> = seen.into_iter().collect(); + values.sort_by(|((a, ta), _), ((b, tb), _)| { + a.0.cmp(&b.0).then(a.1.cmp(&b.1)).then_with(|| gridcraft_core::compare_text(&a.2, &b.2)).then_with(|| ta.cmp(tb)) + }); + values.into_iter().map(|((_, t), on)| (t, on)).collect() } diff --git a/crates/engine/src/cmd/data/tests.rs b/crates/engine/src/cmd/data/tests.rs index 3c2782444..e073e23a5 100644 --- a/crates/engine/src/cmd/data/tests.rs +++ b/crates/engine/src/cmd/data/tests.rs @@ -53,6 +53,17 @@ fn subtotal_puts_summary_rows_below_their_detail() { assert_eq!(sh.value(CellRef::new(3, 0)).display(), "West"); } +#[test] +fn filter_list_orders_arabic_like_sort() { + let mut s = Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [["H"], ["بج"], ["بُت"], ["\u{200F}بب"]]})).unwrap(); + s.execute("data.filter", json!({"range": "A1:A4"})).unwrap(); + let doc = s.doc().unwrap(); + let list: Vec = filter_values(&doc.wb, 0, 0).into_iter().map(|(t, _)| t).collect(); + assert_eq!(list, ["\u{200F}بب", "بُت", "بج"]); +} + #[test] fn filter_by_huge_numeric_column_is_rejected_not_overflowed() { let mut s = Session::new(); diff --git a/crates/engine/src/pivot/items.rs b/crates/engine/src/pivot/items.rs index 79b74da1a..6dee1dc5a 100644 --- a/crates/engine/src/pivot/items.rs +++ b/crates/engine/src/pivot/items.rs @@ -21,7 +21,7 @@ pub(super) struct FieldItems { } fn cmp_items(a: &Item, b: &Item) -> Ordering { - a.class.cmp(&b.class).then(a.num.total_cmp(&b.num)).then_with(|| a.text.cmp(&b.text)) + a.class.cmp(&b.class).then(a.num.total_cmp(&b.num)).then_with(|| gridcraft_core::compare_text(&a.text, &b.text)) } /// (identity key, item) for a source value under a field's date grouping. diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index f97e1bc10..bf36bd15b 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1459,6 +1459,28 @@ fn case_sensitive_sort_puts_lowercase_first() { assert_eq!(col, ["a", "A", "aB", "Ab", "b", "B"].map(Value::from)); } +#[test] +fn data_sort_orders_vowelled_arabic_by_letters() { + let mut s = s(); + s.execute("range.setValues", json!({"range": "A1", "values": [["بيت"], ["بَاب"], ["تمر"], ["\u{200F}بحر"]]})).unwrap(); + s.execute("data.sort", json!({"range": "A1:A4", "header": false, "keys": [{"column": "A"}]})).unwrap(); + let col: Vec = (1..=4).map(|r| v(&s, &format!("A{r}"))).collect(); + assert_eq!(col, ["بَاب", "\u{200F}بحر", "بيت", "تمر"].map(Value::from)); +} + +#[test] +fn data_sort_orders_marks_and_digit_scripts_like_excel() { + // Excel 16.0 (build 20527, en-US) sorts this column, as text, into exactly this order. + let want = ["١٢", "۱۲", "ب\u{064E}1", "ب\u{064E}١", "بa", "\u{200F}باب", "ب\u{064B}اب", "ب\u{064E}اب", "ب\u{0652}اب", "ب\u{0651}اب"]; + let mut s = s(); + s.execute("home.numberFormat", json!({"range": "A1:A10", "code": "@"})).unwrap(); + let input: Vec> = want.iter().rev().map(|t| vec![*t]).collect(); + s.execute("range.setValues", json!({"range": "A1", "values": input})).unwrap(); + s.execute("data.sort", json!({"range": "A1:A10", "header": false, "keys": [{"column": "A"}]})).unwrap(); + let col: Vec = (1..=10).map(|r| v(&s, &format!("A{r}"))).collect(); + assert_eq!(col, want.map(Value::from)); +} + #[test] fn sort_with_a_sheet_prefix_sorts_the_named_sheet() { let col = |s: &Session, sheet: &str| -> Vec { diff --git a/docs/localization-parity.md b/docs/localization-parity.md index 39a9b3f91..7b5dda2c0 100644 --- a/docs/localization-parity.md +++ b/docs/localization-parity.md @@ -1,6 +1,6 @@ # Localization parity -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-23 · **Change:** minor (missing translations are listed by `cargo xtask i18n-report` and never fail a check) · **Target:** Microsoft Excel (Microsoft 365), Excel for Mac 16.113.4 +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-23 · **Change:** minor (missing translations are listed by `cargo xtask i18n-report` and never fail a check; Arabic text sorts by its letters, not its vowel marks) · **Target:** Microsoft Excel (Microsoft 365), Excel for Mac 16.113.4 Per-language status against Excel, which ships its UI in dozens of languages (Windows language packs cover 100+), with localized function names, argument separators, number and date formats, @@ -117,6 +117,7 @@ language (none so far) and of localized function names. | 2026-10-20 | minor | Dutch (nl) interface catalog (`i18n/nl.rs`, 518 labels, ~24%): the shared key set every language must cover (ribbon, context menus, File view, dialog titles, Options page), matching the German, French and Czech catalogs; `nl-NL`, `nl-BE` locale tags resolve to Dutch | | 2026-10-20 | minor | XLSX: a theme's per-script typefaces (``, Hebr, Jpan…) are kept through open + save, so Arabic Excel's theme fonts for shapes, charts and theme-font cells survive a GridCraft save | | 2026-10-20 | trivial | XLSX: a workbook theme's major/minor East Asian (``) and complex-script (``) typefaces are kept through open + save, so a Japanese workbook keeps its theme's East Asian font; rendering does not resolve them yet (follow-up) | +| 2026-10-19 | minor | Text order (Sort, SORT/SORTBY, `<`/`>`, approximate MATCH/XLOOKUP, `"