From 86e59ebc082109bc5fd765926a45283701345c69 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 18:44:12 -0500 Subject: [PATCH] craft-captions: ten caption fixes ported from open FilmCraft pull requests (#932) * craft-captions: TTML cue text collapses only XML whitespace and keeps edge line breaks Ported from storytold/filmcraft#3145. char::is_whitespace also matched U+00A0, U+2009 and U+3000, so no-break spaces became ordinary spaces on import, and a leading or trailing
was trimmed away. * craft-captions: TTML/DFXP export drops characters XML 1.0 forbids Ported from storytold/filmcraft#4467. A control character in cue text made the exported file unreadable. The filter also covers U+FFFE and U+FFFF, which the review of that pull request asked for. * craft-captions: WebVTT cue-text arrow escape is reversible Ported from storytold/filmcraft#4437. A literal --> in cue text came back as --> after a write and parse; the writer now adds one amp; level and the reader removes exactly one. * craft-captions: a WebVTT speaker name with > no longer spills into the cue text Ported from storytold/filmcraft#4470. The writer puts the lookalike U+203A in place of > inside the voice tag. Also from that pull request's review: a line break in a name is written as a space, and a name ending in - is kept apart from the tag's closing >. * craft-captions: an SRT text line that looks like a timing line survives a round trip Ported from storytold/filmcraft#4448. The writer marks such a line with a leading zero-width space and the reader removes it, so the cue is not split. * craft-captions: back-to-back SCC captions with the same text stay separate cues Ported from storytold/filmcraft#3808. Every End of Caption now closes the open cue. MCC's CEA-608 path uses the same decoder. * craft-captions: an SCC pop-on load may run across the previous caption's erase Ported from storytold/filmcraft#1837 (filmcraft#1791). A caption that closely follows another popped on about a second late, because its load had to be one unbroken run of free frames. Doubled codes stay adjacent; pauses are allowed between them. * craft-captions: CEA-708 export starts each caption at its window's top left Ported from storytold/filmcraft#4043 (filmcraft#4042). Each window load now sends DeleteWindows before DefineWindow in place of the ClearWindows after it: a window made anew is empty with the pen at its top left in any decoder, for the same number of bytes, so cue timing is unchanged. * craft-captions: TTML/DFXP export keeps literal < and > text Ported from storytold/filmcraft#2830. The writer treated every < with a later > as a tag and wrote nothing for unknown names, so "1 < 2 and 3 > 2" lost its middle. Only a well-formed tag is markup now; a > inside a quoted attribute value does not end the tag. * craft-captions: TTML/DFXP export keeps cue IDs unique and valid Unicode IDs as written Ported from the TTML commit of storytold/filmcraft#1839. A cue named s1 could collide with the style ID, and a generated c1 could take the ID of a later cue. Valid explicit IDs are reserved first; missing, invalid or repeated ones get a replacement that is in no one's way. IDs are checked as XML NCNames. * craft-captions: compare the craft-xml and roxmltree views of TTML documents The TTML reader moved from roxmltree to craft-xml. This dev-only test checks that both hand it the same elements, attributes and text, by namespace, on TTML documents and on the writer's output. roxmltree becomes a dev-dependency, which the cue ID tests use too. --------- Co-authored-by: NetterSilver <133535268+Silvertree2010@users.noreply.github.com> Co-authored-by: SeeRay11 Co-authored-by: jnohclee-rgb <336857373+jnohclee-rgb@users.noreply.github.com> Co-authored-by: JonRC Co-authored-by: sanyamsingh001 <314051937+sanyamsingh001@users.noreply.github.com> --- Cargo.lock | 1 + crates/captions/Cargo.toml | 2 + crates/captions/README.md | 11 + crates/captions/src/mcc.rs | 108 +++++- crates/captions/src/scc.rs | 66 +++- crates/captions/src/srt.rs | 39 +- crates/captions/src/ttml.rs | 167 ++++++-- crates/captions/src/vtt.rs | 80 +++- crates/captions/tests/it/main.rs | 5 + crates/captions/tests/it/mcc_708_timing.rs | 27 ++ crates/captions/tests/it/roundtrip.rs | 47 +++ .../captions/tests/it/ttml_control_chars.rs | 19 + crates/captions/tests/it/ttml_cue_ids.rs | 362 ++++++++++++++++++ .../captions/tests/it/ttml_literal_angle.rs | 76 ++++ crates/captions/tests/it/ttml_xml_views.rs | 119 ++++++ 15 files changed, 1078 insertions(+), 51 deletions(-) create mode 100644 crates/captions/tests/it/mcc_708_timing.rs create mode 100644 crates/captions/tests/it/ttml_control_chars.rs create mode 100644 crates/captions/tests/it/ttml_cue_ids.rs create mode 100644 crates/captions/tests/it/ttml_literal_angle.rs create mode 100644 crates/captions/tests/it/ttml_xml_views.rs diff --git a/Cargo.lock b/Cargo.lock index 460fdfd92..1b7bfd666 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -903,6 +903,7 @@ version = "0.1.0" dependencies = [ "craft-xml", "proptest", + "roxmltree 0.21.2", "thiserror 2.0.21", ] diff --git a/crates/captions/Cargo.toml b/crates/captions/Cargo.toml index 81f0e6575..70293e2d3 100644 --- a/crates/captions/Cargo.toml +++ b/crates/captions/Cargo.toml @@ -16,3 +16,5 @@ thiserror = { workspace = true } [dev-dependencies] proptest = { workspace = true } +# A second XML reader for the tests: checks of written TTML, and the comparison in tests/it/ttml_xml_views.rs. +roxmltree = "0.21" diff --git a/crates/captions/README.md b/crates/captions/README.md index 927a5dc44..6fd2a4005 100644 --- a/crates/captions/README.md +++ b/crates/captions/README.md @@ -86,6 +86,17 @@ This mapping follows [W3C TTML1, third edition (2018), §§12.1.5–12.2.1](http The [April 2006 TTaf1 draft, §§7.2.1 and 12.1.5–12.2.1](https://www.w3.org/TR/2006/WD-ttaf1-dfxp-20060427/) defines its earlier metadata namespace and agent ID form. +### TTML cue identifiers + +TTML and DFXP writers keep valid explicit cue IDs verbatim unless a style, region or earlier cue +already uses them. Missing, invalid or repeated IDs receive deterministic replacements that also +avoid valid IDs on later cues. Emitted cue, style, region and speaker IDs are unique across the +document. Unicode spelling is preserved without normalization. + +Identifier validation follows the `NCName` production in [Namespaces in XML 1.0, third edition +(2009), §3](https://www.w3.org/TR/2009/REC-xml-names-20091208/#NT-NCName), using the character +ranges in [XML 1.0, fifth edition (2008), §2.3](https://www.w3.org/TR/2008/REC-xml-20081126/#NT-NameStartChar). + ### SCC writer Each caption is sent as a pop-on load (RCL, ENM, then per row a preamble address code and tab diff --git a/crates/captions/src/mcc.rs b/crates/captions/src/mcc.rs index 0b780ccc1..7169a4ec7 100644 --- a/crates/captions/src/mcc.rs +++ b/crates/captions/src/mcc.rs @@ -23,9 +23,9 @@ //! carries a CDP with 20 cc_data triplets; empty frames are omitted. Each CDP contains field 1 CEA-608 //! pop-on data (the same schedule as [SCC](crate::scc)), a null field-2 pair, CEA-708 //! service 1 data and padding. The CEA-708 captions emulate pop-on with two windows: each -//! caption is defined (hidden) and loaded into one window while the other shows, then a -//! DisplayWindows / HideWindows pair switches them on the caption's first frame; HideWindows -//! clears it at its end. +//! caption goes into one window, deleted and defined anew first (hidden, empty, the pen at its +//! top left), while the other shows, then a DisplayWindows / HideWindows pair switches them on +//! the caption's first frame; HideWindows clears it at its end. //! - **Reading:** every CDP's `cc_data`; channel 1 CEA-608 field-1 data is decoded with the SCC //! decoder when present, otherwise CEA-708 service 1 (windows, text, CR, clear / display / //! hide / toggle / delete windows, DefineWindow visibility); a cue starts whenever the visible @@ -240,15 +240,18 @@ pub fn shown_as_708(c: char) -> char { if char_atom(c).is_some() { c } else { '?' } } -/// Atoms that define window `win` hidden and load `rows` into it. +/// Atoms that make window `win` anew, hidden, and load `rows` into it. fn load_atoms(win: u8, rows: &[String]) -> Vec> { let rc = rows.len().clamp(1, 12) as u8 - 1; let cols = rows.iter().map(|r| r.chars().count()).max().unwrap_or(1).clamp(1, 42) as u8 - 1; let mut v = vec![ + // DeleteWindows: the window holds the caption before last. Decoders differ on whether + // redefining it or ClearWindows moves the pen back (CCExtractor keeps it), but a new + // window starts empty with the pen at its top left, for the same bytes as ClearWindows. + vec![0x8c, 1 << win], // DefineWindow: hidden, row/column lock, relative position 90 % down / 50 % across, // anchored at its bottom centre, rows / columns, window style 1, pen style 1 vec![0x98 + win, 0x18, 0x80 | 90, 50, 7 << 4 | rc, cols, 1 << 3 | 1], - vec![0x88, 1 << win], // ClearWindows ]; for (i, r) in rows.iter().enumerate() { if i > 0 { @@ -788,6 +791,13 @@ mod tests { assert!(parse("garbage").is_err()); } + /// Keep back-to-back captions with the same text separate on the CEA-608 path (the SCC decoder). + #[test] + fn identical_back_to_back_cues_stay_separate_on_608() { + let doc = Document { cues: vec![cue(90, 135, "Ht"), cue(135, 180, "Ht")], ..Default::default() }; + assert_eq!(parse(&write_with(&doc, false, true, false, "test")).unwrap().cues, doc.cues); + } + /// Reject malformed fields instead of silently dropping them from the timecode. #[test] fn rejects_timecodes_with_unparseable_extra_fields() { @@ -891,4 +901,92 @@ mod tests { assert!(decoder.cues.iter().map(|c| c.2.len()).sum::() <= MAX_708_TEXT_BYTES); assert!(decoder.open.is_none()); } + + /// Start every CEA-708 caption at the top left of its window, also for a decoder that sets + /// the pen only when a window is created and moves it only on SetPenLocation, carriage return + /// and text. CCExtractor keeps the pen through ClearWindows and a redefined window, so the + /// third caption below once started at the end of the first one, past its smaller window. + #[test] + fn each_708_caption_starts_at_its_window_top_left() { + let doc = Document { + cues: vec![ + cue(60, 120, "Por que você não admite que\nminha mão robótica te assusta?"), + cue(150, 200, "Você partiu meu coração!"), + cue(230, 250, "Eu sei."), + cue(260, 300, "Você me esqueceu na Terra."), + ], + ..Default::default() + }; + // Service 1 bytes in order, out of the scheduled DTVCC packets. + let mut packets: Vec> = Vec::new(); + for (start, a, b) in schedule_708(&doc).into_values().flatten() { + if start { + packets.push(Vec::new()); + } + packets.last_mut().unwrap().extend([a, b]); + } + let mut s = Vec::new(); + for p in &packets { + let mut body = &p[1..]; + while let [h, rest @ ..] = body { + let n = (h & 0x1f) as usize; + if h >> 5 == 1 { + s.extend_from_slice(&rest[..n]); + } + body = &rest[n..]; + } + } + // Pen (row, column), size (rows, columns) and whether defined, per window; where each + // caption's text starts (its first character after a DefineWindow). + let (mut pen, mut size, mut defined, mut cur) = ([(0usize, 0usize); 8], [(0usize, 0usize); 8], [false; 8], 0usize); + let (mut starts, mut loaded, mut i) = (Vec::new(), false, 0); + while let Some(&c) = s.get(i) { + let mut text = false; + i += match c { + 0x80..=0x87 => { + cur = (c - 0x80) as usize; + 1 + } + 0x8c => { + (0..8).filter(|w| s[i + 1] & 1 << w != 0).for_each(|w| defined[w] = false); + 2 + } + 0x88..=0x8b => 2, + 0x92 => { + pen[cur] = (s[i + 1] as usize, s[i + 2] as usize); + 3 + } + 0x98..=0x9f => { + cur = (c - 0x98) as usize; + if !std::mem::replace(&mut defined[cur], true) { + pen[cur] = (0, 0); + } + size[cur] = ((s[i + 4] & 0x0f) as usize + 1, (s[i + 5] & 0x3f) as usize + 1); + loaded = true; + 7 + } + 0x0d => { + pen[cur] = (pen[cur].0 + 1, 0); + 1 + } + 0x10 => { + text = true; + 2 + } + 0x20..=0x7f | 0xa0..=0xff => { + text = true; + 1 + } + _ => 1, + }; + if text { + if std::mem::take(&mut loaded) { + starts.push((cur, pen[cur])); + } + assert!(pen[cur].0 < size[cur].0 && pen[cur].1 < size[cur].1, "window {cur} {:?}: text at {:?}", size[cur], pen[cur]); + pen[cur].1 += 1; + } + } + assert_eq!(starts, [(0, (0, 0)), (1, (0, 0)), (0, (0, 0)), (1, (0, 0))]); + } } diff --git a/crates/captions/src/scc.rs b/crates/captions/src/scc.rs index 3b285ead0..e0000cff9 100644 --- a/crates/captions/src/scc.rs +++ b/crates/captions/src/scc.rs @@ -11,7 +11,8 @@ //! A tab separates the timecode from the code words. Each line's pairs are sent one per frame //! starting at its timecode. **Reading** runs a channel-1 //! CEA-608 decoder (pop-on, roll-up, paint-on; redundant control codes ignored) and emits a cue -//! whenever the displayed memory changes. **Writing** produces pop-on captions: for each cue the +//! whenever the displayed memory changes or an End of Caption swaps it, so neighbours with the +//! same text stay separate cues. **Writing** produces pop-on captions: for each cue the //! load (RCL, ENM, PAC + tab offset per row, text) is scheduled into free frames before the cue so //! its End of Caption lands exactly on the cue start; an Erase Displayed Memory clears it at the //! cue end (omitted when the next caption replaces it). Rows are bottom-aligned (row 15 last), @@ -116,14 +117,19 @@ impl Decoder { if now == cur { return; } + self.close(f); + if !now.is_empty() { + self.open = Some((f, now)); + } + } + + /// End the cue being shown at frame `f` (dropped if it never reached a frame). + fn close(&mut self, f: i64) { if let Some((s, text)) = self.open.take() && f > s { self.cues.push((s, f, text)); } - if !now.is_empty() { - self.open = Some((f, now)); - } } fn pair(&mut self, f: i64, b1: u8, b2: u8) { @@ -186,6 +192,8 @@ impl Decoder { misc::EOC => { std::mem::swap(&mut self.displayed, &mut self.hidden); self.mode = Mode::PopOn; + // each swap presents its own caption, even one reading the same as the last + self.close(f); } misc::CR => { if let Mode::RollUp(n) = self.mode { @@ -438,14 +446,32 @@ pub fn schedule(doc: &Document) -> BTreeMap { } let load = load_pairs(&c.text); let n = load.len() as i64; - // Doubled codes must stay adjacent, so the load is one contiguous run of free frames: - // the latest run ending by `s`, else the first run after `cursor` (delaying the caption). - let run_free = |occ: &BTreeMap, b: i64| (b..b + n).all(|f| !occ.contains_key(&f)); - let b = (cursor..=s - n).rev().find(|&b| run_free(&occ, b)).unwrap_or_else(|| (cursor..).find(|&b| run_free(&occ, b)).unwrap_or(cursor)); - for (k, p) in load.into_iter().enumerate() { - occ.insert(b + k as i64, p); + // Erasing displayed memory can interrupt loading non-displayed memory. Keep each + // doubled control/character code adjacent, but allow pauses between those groups. + let plan_load = |mut f: i64| { + let mut plan = Vec::with_capacity(load.len()); + let mut k = 0; + while let Some(&p) = load.get(k) { + let doubled = (0x10..=0x1f).contains(&p.0) && load.get(k + 1) == Some(&p); + let width = if doubled { 2 } else { 1 }; + while (f..f + width).any(|at| occ.contains_key(&at)) { + f += 1; + } + for offset in 0..width { + plan.push((f + offset, p)); + } + f += width; + k += width as usize; + } + (plan, f) + }; + // Prefer the latest load that finishes by the requested start; if it cannot fit, + // use the earliest available frames rather than waiting until the prior cue ends. + let (plan, end) = (cursor..=s - n).rev().map(plan_load).find(|(_, end)| *end <= s).unwrap_or_else(|| plan_load(cursor)); + for (f, p) in plan { + occ.insert(f, p); } - let mut d = s.max(b + n); + let mut d = s.max(end); while occ.contains_key(&d) { d += 1; } @@ -561,6 +587,24 @@ mod tests { assert_eq!(got, vec![(60, 120, "Hello, world!"), (120, 200, "Second caption\nwith two lines"), (260, 300, "Ñandú ♪ {x} * café “q”")]); } + /// Replaying the `scc_back_to_back` proptest failure: neighbours with the same text merged. + #[test] + fn identical_back_to_back_cues_stay_separate() { + let doc = Document { + cues: vec![cue(90, 135, "Ht"), cue(135, 180, "Ht"), cue(180, 225, "Ht"), cue(225, 270, "Other"), cue(270, 315, "Ht")], + ..Default::default() + }; + // The stream carries each boundary: every cue has its own End of Caption on its start frame. + let occ = schedule(&doc); + for start in [90, 135, 180, 225, 270] { + assert_eq!(occ.get(&start), Some(&(0x14, misc::EOC)), "frame {start}"); + } + let s = write(&doc, false); + let back = parse(&s).unwrap(); + assert_eq!(back.cues, doc.cues); + assert_eq!(write(&back, false), s); + } + #[test] fn wraps_long_lines() { let rows = wrap_rows("the quick brown fox jumps over the lazy dog and keeps running far away"); diff --git a/crates/captions/src/srt.rs b/crates/captions/src/srt.rs index cab7fa8f6..0c51eede7 100644 --- a/crates/captions/src/srt.rs +++ b/crates/captions/src/srt.rs @@ -34,6 +34,21 @@ fn is_index(line: &str) -> bool { !l.is_empty() && l.len() <= 10 && l.bytes().all(|b| b.is_ascii_digit()) } +const ZWSP: char = '\u{200B}'; + +/// Keep a text line from reading as a timing line: such a line gains a leading zero-width space. +fn escape(line: &str) -> String { + if parse_timing_line(line.trim_start_matches(ZWSP)).is_some() { format!("{ZWSP}{line}") } else { line.to_string() } +} + +/// Reverse [`escape`]: drop one leading zero-width space when the rest still looks like a timing line. +fn unescape_line(line: &str) -> &str { + match line.strip_prefix(ZWSP) { + Some(r) if parse_timing_line(r.trim_start_matches(ZWSP)).is_some() => r, + _ => line, + } +} + pub fn parse(text: &str) -> Result { let text = text.strip_prefix('\u{feff}').unwrap_or(text); let lines: Vec<&str> = text.lines().collect(); @@ -63,7 +78,7 @@ pub fn parse(text: &str) -> Result { if is_index(l) && lines.get(i + 1).is_some_and(|n| parse_timing_line(n).is_some()) { break; } - text_lines.push(l.trim_end()); + text_lines.push(unescape_line(l.trim_end())); i += 1; } if end <= start { @@ -84,7 +99,7 @@ pub fn write(doc: &Document) -> String { out.push_str(&format!("{}\n{} --> {}\n", n + 1, format_clock(c.start, ','), format_clock(c.end, ','))); // blank lines inside text would end the cue: drop them for l in c.text.lines().filter(|l| !l.trim().is_empty()) { - out.push_str(l); + out.push_str(&escape(l)); out.push('\n'); } out.push('\n'); @@ -120,6 +135,26 @@ mod tests { assert_eq!(got, vec![(1500, 2250, "A"), (3000, 4000, "B"), (5000, 6000, "42")]); } + #[test] + fn round_trips_text_line_that_looks_like_timing() { + let mut d = parse("1\n00:00:01,000 --> 00:00:02,000\nx\n").unwrap(); + d.cues[0].text = "Shift:\n9:00 --> 10:00".into(); + let back = parse(&write(&d)).unwrap(); + assert_eq!(back.cues.len(), 1); + assert_eq!(back.cues[0].text, "Shift:\n9:00 --> 10:00"); + } + + #[test] + fn round_trips_arrow_and_marker_text() { + for text in ["a --> b", "a --&gt; b", "a --> b", "Shift:\n9:00 --> 10:00\n\u{200B}9:00 --> 10:00", "5\n9:00 --> 10:00"] { + let mut d = parse("1\n00:00:01,000 --> 00:00:02,000\nx\n").unwrap(); + d.cues[0].text = text.into(); + let back = parse(&write(&d)).unwrap(); + assert_eq!(back.cues.len(), 1, "{text:?}"); + assert_eq!(back.cues[0].text, text); + } + } + #[test] fn ignores_byte_order_mark() { let d = parse("\u{feff}00:00:01,000 --> 00:00:02,000\nHi\n").unwrap(); diff --git a/crates/captions/src/ttml.rs b/crates/captions/src/ttml.rs index 797d86c9c..c439c2ff4 100644 --- a/crates/captions/src/ttml.rs +++ b/crates/captions/src/ttml.rs @@ -88,6 +88,8 @@ fn esc(s: &str) -> String { '>' => o.push_str(">"), '"' => o.push_str("""), '\'' => o.push_str("'"), + // XML 1.0 forbids these (C0 controls, U+FFFE, U+FFFF), even escaped + c if !craft_xml::is_xml_char(c) => {} c => o.push(c), } } @@ -106,6 +108,75 @@ fn time_expr(t: Tick, rate: Option) -> String { } } +/// Memoised "next occurrence of a quote byte" searches, so unterminated quotes are not rescanned. +#[derive(Default)] +struct QuoteCache { + found: [Option<(usize, Option)>; 2], +} + +impl QuoteCache { + /// Index of the next `quote` byte at or after `from`. + fn next(&mut self, bytes: &[u8], quote: u8, from: usize) -> Option { + let slot = usize::from(quote == b'\''); + if let Some((searched, result)) = self.found[slot] + && searched <= from + && result.is_none_or(|r| r >= from) + { + return result; + } + let result = bytes.get(from..).and_then(|t| t.iter().position(|&c| c == quote)).map(|i| from + i); + self.found[slot] = Some((from, result)); + result + } +} + +/// Parses a tag starting at `bytes[start] == b'<'`: optional `/` (closing), a name of ASCII +/// letters/digits starting with a letter immediately after, then whitespace-separated +/// `name="value"` / `name='value'` attributes, optional `/` before `>`. A `>` inside a quoted +/// value does not end the tag. Returns `(index after '>', closing, lowercase name)`; the name is empty for a self-closing tag. +fn parse_tag(bytes: &[u8], start: usize, cache: &mut QuoteCache) -> Option<(usize, bool, String)> { + let mut i = start + 1; + let closing = bytes.get(i) == Some(&b'/'); + if closing { + i += 1; + } + let name_start = i; + if !bytes.get(i)?.is_ascii_alphabetic() { + return None; + } + while bytes.get(i).is_some_and(u8::is_ascii_alphanumeric) { + i += 1; + } + let name = String::from_utf8_lossy(bytes.get(name_start..i)?).to_ascii_lowercase(); + loop { + let ws_start = i; + while bytes.get(i).is_some_and(u8::is_ascii_whitespace) { + i += 1; + } + match bytes.get(i)? { + b'>' => return Some((i + 1, closing, name)), + // self-closing: no markup + b'/' if !closing && bytes.get(i + 1) == Some(&b'>') => return Some((i + 2, closing, String::new())), + _ => {} + } + if i == ws_start { + return None; + } + let key_start = i; + while bytes.get(i).is_some_and(|c| c.is_ascii_alphanumeric() || matches!(c, b'-' | b':' | b'_')) { + i += 1; + } + if i == key_start || bytes.get(i) != Some(&b'=') { + return None; + } + let quote = *bytes.get(i + 1)?; + if quote != b'"' && quote != b'\'' { + return None; + } + i = cache.next(bytes, quote, i + 2)? + 1; + } +} + /// Caption text (`\n` lines, ``/``/`` tags) → TTML inline content. fn inline(text: &str) -> String { let mut out = String::new(); @@ -127,25 +198,12 @@ fn inline(text: &str) -> String { } }; let mut rest = text; - // Once no `>` remains, no later `<` can start a tag either. Remember that instead of searching - // the shrinking suffix again after every literal `<`, which made `<<<<…` quadratic. - let mut closer_remains = true; + let mut cache = QuoteCache::default(); while !rest.is_empty() { - let tag_end = match rest.strip_prefix('<') { - Some(after) if closer_remains => { - let end = after.find('>'); - closer_remains = end.is_some(); - end - } - _ => None, - }; - if let Some(end) = tag_end { - let after = &rest[1..]; - let tag = after[..end].trim().to_ascii_lowercase(); - let (closing, name) = match tag.strip_prefix('/') { - Some(n) => (true, n.trim().to_string()), - None => (false, tag.split_whitespace().next().unwrap_or("").to_string()), - }; + // Only a well-formed tag is markup; "1 < 2 and 3 > 2" and "ad" are text. + if rest.starts_with('<') + && let Some((end, closing, name)) = parse_tag(text.as_bytes(), text.len() - rest.len(), &mut cache) + { let attr = match name.as_str() { "i" => Some("tts:fontStyle=\"italic\""), "b" => Some("tts:fontWeight=\"bold\""), @@ -159,7 +217,7 @@ fn inline(text: &str) -> String { active.push(a); } } - rest = &after[end + 1..]; + rest = text.get(end..).unwrap_or(""); continue; } let Some(ch) = rest.chars().next() else { break }; @@ -177,13 +235,34 @@ fn inline(text: &str) -> String { out } +/// An XML name without a colon (Namespaces in XML, NCName). fn is_ncname(s: &str) -> bool { - let mut c = s.chars(); - c.next().is_some_and(|f| f.is_alphabetic() || f == '_') && c.all(|x| x.is_alphanumeric() || matches!(x, '_' | '-' | '.')) + craft_xml::is_name(s) && !s.contains(':') } -fn cue_id(cue: &Cue, index: usize) -> String { - cue.id.as_deref().filter(|x| is_ncname(x)).map(str::to_string).unwrap_or_else(|| format!("c{}", index + 1)) +fn cue_ids(doc: &Document) -> Vec { + // Reserve every valid explicit ID before allocating fallbacks, including IDs on later cues. + let reserved: HashSet<&str> = + ["s1", "r1"].into_iter().chain(doc.cues.iter().filter_map(|cue| cue.id.as_deref()).filter(|id| is_ncname(id))).collect(); + let mut used: HashSet = ["s1".into(), "r1".into()].into_iter().collect(); + doc.cues + .iter() + .enumerate() + .map(|(index, cue)| { + if let Some(id) = cue.id.as_deref().filter(|id| is_ncname(id)) + && used.insert(id.to_string()) + { + return id.to_string(); + } + let mut id = format!("c{}", index.saturating_add(1)); + // Each collision consumes a distinct reserved or used ID, so this loop is bounded by their combined size. + while reserved.contains(id.as_str()) || used.contains(&id) { + id.push('_'); + } + used.insert(id.clone()); + id + }) + .collect() } /// Write `doc` as TTML. `rate` (IMSC1 only) makes times exact frame offsets; `lang` is the @@ -205,8 +284,9 @@ pub fn write(doc: &Document, flavor: Flavor, rate: Option, lang: &str } o.push_str(&format!(" xml:lang=\"{}\">\n", esc(if lang.is_empty() { "en" } else { lang }))); // Keep agent identifiers distinct from the IDs already emitted for cues, style and region. + let cue_ids = cue_ids(doc); let mut ids: HashSet = ["s1".into(), "r1".into()].into_iter().collect(); - ids.extend(doc.cues.iter().enumerate().map(|(i, c)| cue_id(c, i))); + ids.extend(cue_ids.iter().cloned()); let mut speakers = Vec::new(); let mut speaker_ids = HashMap::new(); for speaker in doc.cues.iter().filter_map(|c| c.speaker.as_deref()) { @@ -235,8 +315,7 @@ pub fn write(doc: &Document, flavor: Flavor, rate: Option, lang: &str " \n \n \n", ); o.push_str(" \n \n
\n"); - for (i, c) in doc.cues.iter().enumerate() { - let id = cue_id(c, i); + for (c, id) in doc.cues.iter().zip(cue_ids) { let agent = c.speaker.as_deref().and_then(|s| speaker_ids.get(s)).map(|id| format!(" ttm:role=\"x-speaker\" ttm:agent=\"{id}\"")).unwrap_or_default(); o.push_str(&format!( @@ -541,7 +620,7 @@ fn collect(n: &El, styles: &std::collections::HashMap, fmt: Fmt, ou for c in &n.children { if let Node::Text(text) | Node::CData(text) = c { for ch in text.chars() { - if ch.is_whitespace() { + if is_xml_space(ch) { out.pending = true; } else { out.flush(); @@ -578,8 +657,14 @@ fn collect(n: &El, styles: &std::collections::HashMap, fmt: Fmt, ou } } +/// XML whitespace: only these characters collapse (U+00A0 and other Unicode spaces are content). +fn is_xml_space(c: char) -> bool { + matches!(c, ' ' | '\t' | '\n' | '\r') +} + fn clean(s: &str) -> String { - s.lines().map(str::trim).collect::>().join("\n").trim().to_string() + // `split` (not `lines`) keeps the empty edge lines produced by a leading or trailing `
`. + s.split('\n').map(|l| l.trim_matches(is_xml_space)).collect::>().join("\n") } pub fn parse(text: &str) -> Result { @@ -718,7 +803,7 @@ fn walk(n: &El, parent: Scope, tm: &Timing, styles: &HashMap, speak o.s.push_str(&format!("")); } let text = clean(&o.s); - if e > b && !text.is_empty() { + if e > b && !text.trim_matches(is_xml_space).is_empty() { doc.cues.push(Cue { start: b, end: e, text, speaker: cue_speaker, id, ..Default::default() }); } } @@ -732,7 +817,7 @@ fn walk(n: &El, parent: Scope, tm: &Timing, styles: &HashMap, speak let mut o = Out { s: String::new(), pending: false }; collect(sp, styles, style_of(sp, styles, f), &mut o); let text = clean(&o.s); - if se > sb && !text.is_empty() { + if se > sb && !text.trim_matches(is_xml_space).is_empty() { doc.cues.push(Cue { start: sb, end: se, @@ -759,6 +844,28 @@ mod tests { Timing { rate, base: rate.timecode_base(), sub_frame_rate: 1, tick_rate: FrameRate::new(1, 1) } } + #[test] + fn only_xml_whitespace_collapses_in_cue_text() { + let xml = "

\t 10\u{a0}km\u{2009}a\u{3000}b \t\r\n c \n

"; + let doc = parse(xml).unwrap(); + assert_eq!(doc.cues[0].text, "10\u{a0}km\u{2009}a\u{3000}b c"); + let xml = "

\u{a0}x
y\u{a0}

"; + let doc = parse(xml).unwrap(); + assert_eq!(doc.cues[0].text, "\u{a0}x\ny\u{a0}"); + } + + #[test] + fn edge_line_breaks_survive_round_trip() { + for (body, want) in [("
text", "\ntext"), ("text
", "text\n"), ("
text
", "\ntext\n")] { + let xml = format!("

{body}

"); + let doc = parse(&xml).unwrap(); + assert_eq!(doc.cues[0].text, want); + let out = write(&doc, Flavor::Dfxp, None, "en"); + let again = parse(&out).unwrap(); + assert_eq!(again.cues[0].text, want); + } + } + #[test] fn invalid_timing_parameters_return_an_error() { for params in [ diff --git a/crates/captions/src/vtt.rs b/crates/captions/src/vtt.rs index e173f925a..e7f770679 100644 --- a/crates/captions/src/vtt.rs +++ b/crates/captions/src/vtt.rs @@ -52,7 +52,53 @@ pub fn split_voice(text: &str) -> (Option, String) { /// as a timing line to anything scanning the block, so it is written `-->`. Other entities /// (`&`, `<`, …) are cue markup and stay verbatim, as `parse` has always kept them. fn unescape_text(text: &str) -> String { - text.replace("-->", "-->") + rewrite_arrows(text, false) +} + +/// Apply [`write`]'s cue-text escape. `-->` becomes `-->`, and a literal `-->` (or the same +/// with more `&` in front of `gt;`) gains one `&`, so the escape is reversible. +fn escape_text(text: &str) -> String { + rewrite_arrows(text, true) +} + +/// Treat `--&` + `amp;`×k + `gt;` as level k, and `-->` as level -1. Escaping raises the level by +/// one, unescaping lowers it by one; anything else is copied unchanged. +fn rewrite_arrows(text: &str, escape: bool) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(c) = rest.chars().next() { + if let Some(after) = rest.strip_prefix("--") { + if escape && after.starts_with('>') { + out.push_str("-->"); + rest = &after[1..]; + continue; + } + let mut tail = after.strip_prefix('&').unwrap_or("\0"); + let mut k = 0usize; + while let Some(t) = tail.strip_prefix("amp;") { + tail = t; + k += 1; + } + if let Some(t) = tail.strip_prefix("gt;") { + if escape { + out.push_str("--&"); + out.push_str(&"amp;".repeat(k + 1)); + out.push_str("gt;"); + } else if k == 0 { + out.push_str("-->"); + } else { + out.push_str("--&"); + out.push_str(&"amp;".repeat(k - 1)); + out.push_str("gt;"); + } + rest = t; + continue; + } + } + out.push(c); + rest = &rest[c.len_utf8()..]; + } + out } pub fn parse(text: &str) -> Result { @@ -142,11 +188,15 @@ pub fn write(doc: &Document) -> String { out.push('\n'); let mut text = String::new(); if let Some(s) = c.speaker.as_deref().filter(|s| !s.is_empty()) { - text.push_str(&format!("")); + // `>` would end the voice tag early and spill the rest of the name into the cue text, so it + // is written as the lookalike `›` (U+203A), and a line break as a space. A name ending + // in `-` gets a space before the closing `>`, so the two never read as an arrow. + let name = s.replace('>', "\u{203a}").replace(['\r', '\n'], " "); + text.push_str(&format!("", if name.ends_with('-') { " " } else { "" })); } text.push_str(&c.text); for l in text.lines().filter(|l| !l.trim().is_empty()) { - out.push_str(&l.replace("-->", "-->")); + out.push_str(&escape_text(l)); out.push('\n'); } out.push('\n'); @@ -196,6 +246,19 @@ mod tests { assert_eq!(parse(&w).unwrap().cues, d.cues); } + #[test] + fn speaker_with_gt_keeps_cue_text_intact() { + for (name, back_name) in [("A > B", "A \u{203a} B"), ("Tom & Jerry", "Tom & Jerry"), ("A > --", "A \u{203a} --"), ("A\nB", "A B")] { + let mut doc = Document::default(); + doc.cues.push(Cue { start: Tick(0), end: Tick(TICKS_PER_MS), text: "Hi".into(), speaker: Some(name.into()), ..Default::default() }); + let written = write(&doc); + assert!(!written.contains("-->"), "the voice tag keeps its own `>`: {written}"); + let back = parse(&written).unwrap(); + assert_eq!(back.cues[0].speaker.as_deref(), Some(back_name)); + assert_eq!(back.cues[0].text, "Hi"); + } + } + #[test] fn newline_in_cue_id_round_trips() { let mut doc = Document::default(); @@ -267,6 +330,17 @@ mod tests { assert_eq!(write(&back), w, "stable from there on"); } + /// A literal `-->` in cue text must not turn into `-->` on a round trip. + #[test] + fn a_literal_escaped_arrow_survives_a_round_trip() { + for text in ["a --> b", "a --> b", "--&gt;", "--->-->", "x --&amp;gt; y"] { + let mut d = Document::default(); + d.cues.push(Cue { start: Tick(0), end: Tick(1000 * TICKS_PER_MS), text: text.into(), ..Cue::default() }); + let back = parse(&write(&d)).unwrap(); + assert_eq!(back.cues[0].text, text); + } + } + /// Only the writer's own `-->` escape is reversed: cue markup keeps its entities verbatim, /// as `parse` has always done. #[test] diff --git a/crates/captions/tests/it/main.rs b/crates/captions/tests/it/main.rs index f1b66ae62..92560a1b2 100644 --- a/crates/captions/tests/it/main.rs +++ b/crates/captions/tests/it/main.rs @@ -1,6 +1,11 @@ mod limits; +mod mcc_708_timing; mod roundtrip; mod ttml_body_timing; mod ttml_clock_overflow; +mod ttml_control_chars; +mod ttml_cue_ids; mod ttml_language; +mod ttml_literal_angle; mod ttml_speakers; +mod ttml_xml_views; diff --git a/crates/captions/tests/it/mcc_708_timing.rs b/crates/captions/tests/it/mcc_708_timing.rs new file mode 100644 index 000000000..77289f0b0 --- /dev/null +++ b/crates/captions/tests/it/mcc_708_timing.rs @@ -0,0 +1,27 @@ +//! CEA-708 timing of back-to-back one-frame captions in MCC. + +use craft_captions::{Cue, Document, Format, mcc, parse}; + +/// A frame carries 18 DTVCC pairs, and with one-frame captions back to back it holds one +/// caption's display (3 pairs) and the next one's load. A load of up to 19 characters (15 pairs) +/// fits beside it. A SetPenLocation in every load made it 16 pairs from 17 characters on, and +/// the display a frame late. +#[test] +fn back_to_back_one_frame_708_captions_keep_their_frames() { + for len in [1, 16, 17, 18, 19] { + let cues: Vec = (0..4i64) + .map(|k| Cue { + start: mcc::RATE.tick_of(60 + k), + end: mcc::RATE.tick_of(61 + k), + // a different text each time, so the reader sees every change + text: ('a'..='z').cycle().skip(k as usize).take(len).collect(), + ..Default::default() + }) + .collect(); + let doc = Document { cues, ..Default::default() }; + let back = parse(mcc::write_with(&doc, true, false, true, "test").as_bytes(), Format::Mcc).unwrap(); + let frames: Vec<(i64, i64)> = back.cues.iter().map(|c| (mcc::RATE.frame_at(c.start), mcc::RATE.frame_at(c.end))).collect(); + assert_eq!(frames, [(60, 61), (61, 62), (62, 63), (63, 64)], "{len} characters"); + assert_eq!(back.cues, doc.cues, "{len} characters"); + } +} diff --git a/crates/captions/tests/it/roundtrip.rs b/crates/captions/tests/it/roundtrip.rs index cbb7bdcd7..effb81e9a 100644 --- a/crates/captions/tests/it/roundtrip.rs +++ b/crates/captions/tests/it/roundtrip.rs @@ -260,3 +260,50 @@ fn srt_to_vtt_to_scc_keeps_text_and_frames() { assert_eq!(scc::frame_of(doc3.cues[0].start), 60); assert_eq!(scc::frame_of(doc3.cues[1].end), 210); } + +#[test] +fn tight_pop_on_load_can_cross_the_previous_caption_end() { + let text = "A second caption that needs more time to load"; + let doc = Document { + cues: vec![ + Cue { start: scc::RATE.tick_of(30), end: scc::RATE.tick_of(60), text: "First".into(), ..Default::default() }, + Cue { start: scc::RATE.tick_of(67), end: scc::RATE.tick_of(150), text: text.into(), ..Default::default() }, + ], + ..Default::default() + }; + for format in [Format::Scc, Format::Mcc] { + let back = parse(&write(&doc, format, WriteOptions::default()), format).unwrap(); + assert_eq!(back.cues.len(), 2); + assert_eq!(scc::frame_of(back.cues[0].end), 60); + assert_eq!(scc::frame_of(back.cues[1].start), 67, "{format:?}"); + assert_eq!(scc::frame_of(back.cues[1].end), 150); + } +} + +#[test] +fn interrupted_load_keeps_extended_characters_and_repeated_basic_pairs() { + for text in + ["A second caption that needs more time to load", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "ÀÀÀÀÀÀÀÀÀÀÀÀÀÀÀÀÀÀ\nÀÀÀÀÀÀÀÀÀÀÀÀÀÀÀÀÀÀ"] + { + let doc = Document { + cues: vec![ + Cue { start: scc::RATE.tick_of(30), end: scc::RATE.tick_of(60), text: "First".into(), ..Default::default() }, + Cue { start: scc::RATE.tick_of(67), end: scc::RATE.tick_of(180), text: text.into(), ..Default::default() }, + ], + ..Default::default() + }; + for format in [Format::Scc, Format::Mcc] { + let back = parse(&write(&doc, format, WriteOptions::default()), format).unwrap(); + assert_eq!(back.cues.len(), 2); + assert_eq!(back.cues[1].text.replace(['\n', ' '], ""), text.replace(['\n', ' '], ""), "{format:?}"); + assert_eq!(scc::frame_of(back.cues[0].end), 60); + assert_eq!(scc::frame_of(back.cues[1].end), 180); + } + let pairs = scc::schedule(&doc); + for (&frame, &pair) in &pairs { + if (0x10..=0x1f).contains(&pair.0) { + assert!(pairs.get(&(frame - 1)) == Some(&pair) || pairs.get(&(frame + 1)) == Some(&pair), "split doubled code at {frame}"); + } + } + } +} diff --git a/crates/captions/tests/it/ttml_control_chars.rs b/crates/captions/tests/it/ttml_control_chars.rs new file mode 100644 index 000000000..bb2eb8437 --- /dev/null +++ b/crates/captions/tests/it/ttml_control_chars.rs @@ -0,0 +1,19 @@ +use craft_captions::{Cue, Document, Format, WriteOptions, parse, write}; +use craft_captions::{TICKS_PER_SECOND, Tick}; + +/// Control characters that XML 1.0 forbids must not make the export unreadable. +#[test] +fn ttml_export_drops_xml_illegal_control_chars() { + // every forbidden character, each between two letters that must survive + let forbidden = (0u32..0x20).filter(|c| ![0x9, 0xA, 0xD].contains(c)).chain([0xFFFE, 0xFFFF]).filter_map(char::from_u32); + let text: String = forbidden.flat_map(|c| ['a', c]).chain(['e']).collect(); + let want = format!("{}e", "a".repeat(31)); + for format in [Format::Ttml, Format::Dfxp] { + let cue = Cue { start: Tick(TICKS_PER_SECOND), end: Tick(2 * TICKS_PER_SECOND), text: text.clone(), ..Default::default() }; + let doc = Document { cues: vec![cue], ..Default::default() }; + let xml = String::from_utf8(write(&doc, format, WriteOptions::default())).unwrap(); + let back = parse(xml.as_bytes(), format); + assert!(back.is_ok(), "{format:?}: {xml}"); + assert_eq!(back.unwrap().cues[0].text, want, "{format:?}"); + } +} diff --git a/crates/captions/tests/it/ttml_cue_ids.rs b/crates/captions/tests/it/ttml_cue_ids.rs new file mode 100644 index 000000000..ea2238955 --- /dev/null +++ b/crates/captions/tests/it/ttml_cue_ids.rs @@ -0,0 +1,362 @@ +use std::collections::BTreeSet; + +use craft_captions::{Cue, Document, Format, WriteOptions, parse, write}; +use craft_captions::{FrameRate, TICKS_PER_SECOND, Tick}; + +const XML: &str = "http://www.w3.org/XML/1998/namespace"; +const TT: &str = "http://www.w3.org/ns/ttml"; + +fn timeline(doc: &Document) -> Vec<(Tick, Tick, &str)> { + doc.cues.iter().map(|cue| (cue.start, cue.end, cue.text.as_str())).collect() +} + +fn distinct_vtt_ids_do_not_collide_with_document_ids(format: Format) { + // Every WebVTT identifier is distinct. Spaces are legal in WebVTT identifiers. + // The fourth identifier requires an XML-name fallback, which must not reuse c4. + let vtt = br#"WEBVTT + +s1 +00:00:01.000 --> 00:00:02.000 +Opening + +r1 +00:00:03.000 --> 00:00:04.000 +Reply + +c4 +00:00:05.000 --> 00:00:06.000 +Explicit identifier + +display label +00:00:07.000 --> 00:00:08.000 +Display identifier + +keep +00:00:09.000 --> 00:00:10.000 +Closing +"#; + let doc = parse(vtt, Format::WebVtt).unwrap(); + assert!(doc.warnings.is_empty(), "{:?}", doc.warnings); + let input_ids: Vec<_> = doc.cues.iter().map(|cue| cue.id.as_deref().unwrap()).collect(); + assert_eq!(input_ids, ["s1", "r1", "c4", "display label", "keep"]); + assert_eq!(input_ids.iter().collect::>().len(), input_ids.len(), "the source identifiers are unique"); + let expected = [ + (Tick(TICKS_PER_SECOND), Tick(2 * TICKS_PER_SECOND), "Opening"), + (Tick(3 * TICKS_PER_SECOND), Tick(4 * TICKS_PER_SECOND), "Reply"), + (Tick(5 * TICKS_PER_SECOND), Tick(6 * TICKS_PER_SECOND), "Explicit identifier"), + (Tick(7 * TICKS_PER_SECOND), Tick(8 * TICKS_PER_SECOND), "Display identifier"), + (Tick(9 * TICKS_PER_SECOND), Tick(10 * TICKS_PER_SECOND), "Closing"), + ]; + assert_eq!(timeline(&doc), expected); + + // Check all document IDs independently of the production allocator. + let options = WriteOptions { rate: Some(FrameRate::FPS_25), ..Default::default() }; + let bytes = write(&doc, format, options); + assert_eq!(bytes, write(&doc, format, options), "stable export for {format:?}"); + let text = std::str::from_utf8(&bytes).unwrap(); + let xml = roxmltree::Document::parse(text).unwrap(); + let paragraphs: Vec<_> = xml.descendants().filter(|node| node.has_tag_name((TT, "p"))).collect(); + assert_eq!(paragraphs.len(), expected.len()); + assert!(paragraphs.iter().all(|node| node.attribute((XML, "id")).is_some_and(|id| !id.is_empty()))); + for paragraph in ¶graphs { + assert_ncname(paragraph.attribute((XML, "id")).unwrap()); + } + assert_eq!(paragraphs.last().unwrap().attribute((XML, "id")), Some("keep"), "an unrelated valid ID stays intact"); + assert_eq!(paragraphs[2].attribute((XML, "id")), Some("c4"), "the existing explicit collision ID stays intact"); + assert_ne!( + paragraphs[3].attribute((XML, "id")), + Some("display label"), + "{format:?}: do not emit the space-containing display label verbatim as xml:id" + ); + let mut seen = BTreeSet::new(); + let repeated: Vec<_> = + xml.descendants().filter_map(|node| node.attribute((XML, "id"))).filter(|id| !seen.insert(*id)).map(str::to_owned).collect(); + let back = parse(&bytes, format).unwrap(); + assert_eq!(timeline(&back), expected, "{format:?}: renaming IDs must preserve every cue and its literal timing"); + assert!(repeated.is_empty(), "{format:?}: all document xml:id values must be unique: {repeated:?}"); +} + +// Validate the public writer's observed ID independently of its allocator and +// without requiring ASCII: an NCName is an XML name with no colon. +fn assert_ncname(id: &str) { + assert!(!id.contains(':'), "an xml:id must not contain a colon: {id:?}"); + let witness = format!("<{id}/>"); + let xml = roxmltree::Document::parse(&witness).expect("the generated ID must be a valid XML name"); + assert_eq!(xml.root_element().tag_name().name(), id, "the entire generated ID must be the XML name"); +} + +fn valid_ascii_and_unicode_cue_ids_survive_export(format: Format) { + // U+00E9 followed by ASCII pisode_1 is a valid Unicode NCName. + let ids = ["c4", "épisode_1"]; + for id in ids { + assert_ncname(id); + } + let doc = Document { + cues: vec![ + Cue { + start: Tick(TICKS_PER_SECOND), + end: Tick(2 * TICKS_PER_SECOND), + text: "ASCII identifier".into(), + id: Some(ids[0].into()), + ..Default::default() + }, + Cue { + start: Tick(3 * TICKS_PER_SECOND), + end: Tick(4 * TICKS_PER_SECOND), + text: "Unicode identifier".into(), + id: Some(ids[1].into()), + ..Default::default() + }, + ], + ..Default::default() + }; + let expected = [ + (Tick(TICKS_PER_SECOND), Tick(2 * TICKS_PER_SECOND), "ASCII identifier"), + (Tick(3 * TICKS_PER_SECOND), Tick(4 * TICKS_PER_SECOND), "Unicode identifier"), + ]; + assert_eq!(timeline(&doc), expected); + let expected_ids: Vec<_> = ids.into_iter().map(|id| Some(id.to_string())).collect(); + let options = WriteOptions { rate: Some(FrameRate::FPS_25), ..Default::default() }; + let bytes = write(&doc, format, options); + assert_eq!(bytes, write(&doc, format, options), "stable export for {format:?}"); + let xml = roxmltree::Document::parse(std::str::from_utf8(&bytes).unwrap()).unwrap(); + let exported_ids: Vec<_> = + xml.descendants().filter(|node| node.has_tag_name((TT, "p"))).map(|node| node.attribute((XML, "id")).map(str::to_string)).collect(); + let mut seen = BTreeSet::new(); + let duplicates: Vec<_> = + xml.descendants().filter_map(|node| node.attribute((XML, "id"))).filter(|id| !seen.insert(*id)).map(str::to_string).collect(); + let back = parse(&bytes, format).unwrap(); + assert_eq!(timeline(&back), expected, "{format:?}: valid IDs must preserve literal cue timing and text"); + let imported_ids: Vec<_> = back.cues.iter().map(|cue| cue.id.clone()).collect(); + assert_eq!(exported_ids, expected_ids, "{format:?}: valid ASCII and Unicode IDs must survive verbatim in XML"); + assert_eq!(imported_ids, expected_ids, "{format:?}: valid ASCII and Unicode IDs must survive owned-reader import"); + assert!(duplicates.is_empty(), "{format:?}: all document xml:id values must be unique: {duplicates:?}"); +} + +fn a_generated_id_does_not_take_a_later_explicit_cue_id(format: Format) { + let earlier = Cue { start: Tick(TICKS_PER_SECOND), end: Tick(2 * TICKS_PER_SECOND), text: "Earlier unlabeled cue".into(), ..Default::default() }; + assert!(earlier.id.is_none()); + let probe = Document { cues: vec![earlier.clone()], ..Default::default() }; + let expected = [ + (Tick(TICKS_PER_SECOND), Tick(2 * TICKS_PER_SECOND), "Earlier unlabeled cue"), + (Tick(3 * TICKS_PER_SECOND), Tick(4 * TICKS_PER_SECOND), "Later explicit cue"), + ]; + assert_eq!(timeline(&probe), expected[..1]); + let options = WriteOptions { rate: Some(FrameRate::FPS_25), ..Default::default() }; + // The first ordinary export selects a collision input, not the expected + // replacement spelling. An unsuitable generated ID is a failure, not a skip. + let probe_bytes = write(&probe, format, options); + let probe_xml = roxmltree::Document::parse(std::str::from_utf8(&probe_bytes).unwrap()).unwrap(); + let probe_paragraphs: Vec<_> = probe_xml.descendants().filter(|node| node.has_tag_name((TT, "p"))).collect(); + assert_eq!(probe_paragraphs.len(), 1); + let observed = probe_paragraphs[0].attribute((XML, "id")).expect("an unlabeled cue must receive an ID").to_string(); + assert_ncname(&observed); + let probe_back = parse(&probe_bytes, format).unwrap(); + assert_eq!(timeline(&probe_back), expected[..1], "{format:?}: the ordinary probe must preserve literal timing and text"); + + let doc = Document { + cues: vec![ + earlier.clone(), + Cue { + start: Tick(3 * TICKS_PER_SECOND), + end: Tick(4 * TICKS_PER_SECOND), + text: "Later explicit cue".into(), + id: Some(observed.clone()), + ..Default::default() + }, + ], + ..Default::default() + }; + assert_eq!(timeline(&doc), expected); + assert!(doc.cues[0].id.is_none()); + assert_eq!(doc.cues[1].id.as_deref(), Some(observed.as_str())); + let bytes = write(&doc, format, options); + assert_eq!(bytes, write(&doc, format, options), "stable export for {format:?}"); + let xml = roxmltree::Document::parse(std::str::from_utf8(&bytes).unwrap()).unwrap(); + let paragraphs: Vec<_> = xml.descendants().filter(|node| node.has_tag_name((TT, "p"))).collect(); + assert_eq!(paragraphs.len(), expected.len()); + let exported_ids: Vec<_> = paragraphs.iter().map(|node| node.attribute((XML, "id")).expect("each cue must have an ID").to_string()).collect(); + for id in &exported_ids { + assert_ncname(id); + } + let mut seen = BTreeSet::new(); + let duplicates: Vec<_> = + xml.descendants().filter_map(|node| node.attribute((XML, "id"))).filter(|id| !seen.insert(*id)).map(str::to_string).collect(); + let back = parse(&bytes, format).unwrap(); + assert_eq!(timeline(&back), expected, "{format:?}: reserving the later ID must preserve every cue and its literal timing"); + assert_eq!(exported_ids[1], observed, "{format:?}: the later explicit ID must survive verbatim"); + assert_ne!(exported_ids[0], observed, "{format:?}: the earlier generated ID must not take the later explicit ID"); + assert_eq!(back.cues[1].id.as_deref(), Some(observed.as_str()), "{format:?}: the later explicit ID must survive owned-reader import"); + assert!(duplicates.is_empty(), "{format:?}: all document xml:id values must be unique: {duplicates:?}"); +} + +#[test] +fn ttml_distinct_vtt_ids_do_not_collide_with_document_ids() { + distinct_vtt_ids_do_not_collide_with_document_ids(Format::Ttml); +} + +#[test] +fn dfxp_distinct_vtt_ids_do_not_collide_with_document_ids() { + distinct_vtt_ids_do_not_collide_with_document_ids(Format::Dfxp); +} + +#[test] +fn ttml_valid_ascii_and_unicode_cue_ids_survive_export() { + valid_ascii_and_unicode_cue_ids_survive_export(Format::Ttml); +} + +#[test] +fn dfxp_valid_ascii_and_unicode_cue_ids_survive_export() { + valid_ascii_and_unicode_cue_ids_survive_export(Format::Dfxp); +} + +#[test] +fn ttml_generated_id_reserves_later_explicit_cue_id() { + a_generated_id_does_not_take_a_later_explicit_cue_id(Format::Ttml); +} + +#[test] +fn dfxp_generated_id_reserves_later_explicit_cue_id() { + a_generated_id_does_not_take_a_later_explicit_cue_id(Format::Dfxp); +} + +fn duplicate_parser_accepted_cue_ids_keep_first_id(format: Format) { + // FilmCraft tolerates duplicate input IDs; WebVTT itself requires unique identifiers. + // Keep the first usable ID and give the later cue a distinct ID when writing XML. + let vtt = br#"WEBVTT + +shared +00:00:01.000 --> 00:00:02.000 +First duplicate identifier + +shared +00:00:03.000 --> 00:00:04.000 +Second duplicate identifier +"#; + assert_ncname("shared"); + let doc = parse(vtt, Format::WebVtt).unwrap(); + let imported_ids: Vec<_> = doc.cues.iter().map(|cue| cue.id.as_deref()).collect(); + assert_eq!(imported_ids, [Some("shared"), Some("shared")], "both input IDs must reach the writer unchanged"); + let expected = [ + (Tick(TICKS_PER_SECOND), Tick(2 * TICKS_PER_SECOND), "First duplicate identifier"), + (Tick(3 * TICKS_PER_SECOND), Tick(4 * TICKS_PER_SECOND), "Second duplicate identifier"), + ]; + assert_eq!(timeline(&doc), expected); + + let options = WriteOptions { rate: Some(FrameRate::FPS_25), ..Default::default() }; + let bytes = write(&doc, format, options); + assert_eq!(bytes, write(&doc, format, options), "stable export for {format:?}"); + let xml = roxmltree::Document::parse(std::str::from_utf8(&bytes).unwrap()).unwrap(); + let paragraphs: Vec<_> = xml.descendants().filter(|node| node.has_tag_name((TT, "p"))).collect(); + assert_eq!(paragraphs.len(), expected.len()); + let exported_ids: Vec<_> = paragraphs.iter().map(|node| node.attribute((XML, "id")).expect("each cue must have an ID")).collect(); + for id in &exported_ids { + assert_ncname(id); + } + assert_eq!(exported_ids[0], "shared", "{format:?}: preserve the first usable explicit ID verbatim"); + assert_ne!(exported_ids[1], "shared", "{format:?}: the second explicit ID must become distinct"); + let mut seen = BTreeSet::new(); + let duplicates: Vec<_> = + xml.descendants().filter_map(|node| node.attribute((XML, "id"))).filter(|id| !seen.insert(*id)).map(str::to_owned).collect(); + let back = parse(&bytes, format).unwrap(); + assert_eq!(timeline(&back), expected, "{format:?}: duplicate-ID tolerance must preserve literal cue timing and text"); + assert_eq!(back.cues[0].id.as_deref(), Some("shared"), "{format:?}: the first explicit ID must survive owned-reader import"); + assert!(duplicates.is_empty(), "{format:?}: all document xml:id values must be unique: {duplicates:?}"); +} + +#[test] +fn ttml_duplicate_parser_accepted_cue_ids_keep_first_id() { + duplicate_parser_accepted_cue_ids_keep_first_id(Format::Ttml); +} + +#[test] +fn dfxp_duplicate_parser_accepted_cue_ids_keep_first_id() { + duplicate_parser_accepted_cue_ids_keep_first_id(Format::Dfxp); +} + +fn decomposed_and_composed_cue_ids_stay_distinct(format: Format) { + // XML 1.0 Fifth Edition NameChar permits U+0301 after e. Do not normalize either ID. + let ids = ["e\u{301}_1", "é_1"]; + assert_ne!(ids[0], ids[1]); + for id in ids { + assert_ncname(id); + } + let vtt = format!( + "WEBVTT\n\n{}\n00:00:01.000 --> 00:00:02.000\nDecomposed identifier\n\n{}\n00:00:03.000 --> 00:00:04.000\nComposed identifier\n", + ids[0], ids[1] + ); + let doc = parse(vtt.as_bytes(), Format::WebVtt).unwrap(); + assert!(doc.warnings.is_empty(), "{:?}", doc.warnings); + let input_ids: Vec<_> = doc.cues.iter().map(|cue| cue.id.as_deref()).collect(); + assert_eq!(input_ids, [Some(ids[0]), Some(ids[1])], "the parser must preserve both exact input IDs"); + let expected = [ + (Tick(TICKS_PER_SECOND), Tick(2 * TICKS_PER_SECOND), "Decomposed identifier"), + (Tick(3 * TICKS_PER_SECOND), Tick(4 * TICKS_PER_SECOND), "Composed identifier"), + ]; + assert_eq!(timeline(&doc), expected); + let options = WriteOptions { rate: Some(FrameRate::FPS_25), ..Default::default() }; + let bytes = write(&doc, format, options); + assert_eq!(bytes, write(&doc, format, options), "stable export for {format:?}"); + let xml = roxmltree::Document::parse(std::str::from_utf8(&bytes).unwrap()).unwrap(); + let exported_ids: Vec<_> = xml.descendants().filter(|node| node.has_tag_name((TT, "p"))).map(|node| node.attribute((XML, "id"))).collect(); + assert_eq!(exported_ids, input_ids, "{format:?}: preserve the decomposed and composed IDs verbatim without normalization"); + let mut seen = BTreeSet::new(); + assert!(xml.descendants().filter_map(|node| node.attribute((XML, "id"))).all(|id| seen.insert(id)), "all document xml:id values must be unique"); + let back = parse(&bytes, format).unwrap(); + assert!(back.warnings.is_empty(), "{:?}", back.warnings); + assert_eq!(timeline(&back), expected, "{format:?}: preserve literal cue timing and text"); + let imported_ids: Vec<_> = back.cues.iter().map(|cue| cue.id.as_deref()).collect(); + assert_eq!(imported_ids, input_ids, "{format:?}: both exact IDs must survive owned-reader import"); +} + +fn non_xml_numeric_cue_id_uses_fallback(format: Format) { + // Leading combining marks and colons are unchanged invalid controls. + // U+00B2 is numeric in Rust, but is not XML 1.0 Fifth Edition NameChar. + for id in ["\u{301}_1", "a:b", "a\u{b2}"] { + let vtt = format!("WEBVTT\n\n{id}\n00:00:01.000 --> 00:00:02.000\nIdentifier fallback\n"); + let doc = parse(vtt.as_bytes(), Format::WebVtt).unwrap(); + assert!(doc.warnings.is_empty(), "{:?}", doc.warnings); + assert_eq!(doc.cues.len(), 1); + assert_eq!(doc.cues[0].id.as_deref(), Some(id)); + let expected = [(Tick(TICKS_PER_SECOND), Tick(2 * TICKS_PER_SECOND), "Identifier fallback")]; + assert_eq!(timeline(&doc), expected); + let options = WriteOptions { rate: Some(FrameRate::FPS_25), ..Default::default() }; + let bytes = write(&doc, format, options); + assert_eq!(bytes, write(&doc, format, options), "stable export for {format:?}: {id:?}"); + let xml = roxmltree::Document::parse(std::str::from_utf8(&bytes).unwrap()).unwrap(); + let paragraphs: Vec<_> = xml.descendants().filter(|node| node.has_tag_name((TT, "p"))).collect(); + assert_eq!(paragraphs.len(), 1); + let output_id = paragraphs[0].attribute((XML, "id")).expect("the cue must have an ID"); + assert_ne!(output_id, id, "{format:?}: an invalid XML NCName must receive a fallback: {id:?}"); + assert_ncname(output_id); + let mut seen = BTreeSet::new(); + assert!( + xml.descendants().filter_map(|node| node.attribute((XML, "id"))).all(|id| seen.insert(id)), + "all document xml:id values must be unique" + ); + let back = parse(&bytes, format).unwrap(); + assert!(back.warnings.is_empty(), "{:?}", back.warnings); + assert_eq!(timeline(&back), expected, "{format:?}: fallback must preserve literal cue timing and text"); + assert_eq!(back.cues[0].id.as_deref(), Some(output_id)); + } +} + +#[test] +fn ttml_decomposed_and_composed_cue_ids_stay_distinct() { + decomposed_and_composed_cue_ids_stay_distinct(Format::Ttml); +} + +#[test] +fn dfxp_decomposed_and_composed_cue_ids_stay_distinct() { + decomposed_and_composed_cue_ids_stay_distinct(Format::Dfxp); +} + +#[test] +fn ttml_non_xml_numeric_cue_id_uses_fallback() { + non_xml_numeric_cue_id_uses_fallback(Format::Ttml); +} + +#[test] +fn dfxp_non_xml_numeric_cue_id_uses_fallback() { + non_xml_numeric_cue_id_uses_fallback(Format::Dfxp); +} diff --git a/crates/captions/tests/it/ttml_literal_angle.rs b/crates/captions/tests/it/ttml_literal_angle.rs new file mode 100644 index 000000000..0e4843901 --- /dev/null +++ b/crates/captions/tests/it/ttml_literal_angle.rs @@ -0,0 +1,76 @@ +//! TTML / DFXP export keeps literal `<` and `>` text that is not markup. + +use craft_captions::FrameRate; +use craft_captions::{Cue, Document, Format, WriteOptions, write}; + +fn export(text: &str, format: Format) -> String { + let rate = FrameRate::FPS_25; + let doc = Document { + cues: vec![Cue { start: rate.tick_of(25), end: rate.tick_of(75), text: text.into(), ..Default::default() }], + ..Default::default() + }; + String::from_utf8(write(&doc, format, WriteOptions::default())).unwrap() +} + +#[test] +fn literal_angle_brackets_keep_middle_text() { + for format in [Format::Ttml, Format::Dfxp] { + for (input, expected) in [("1 < 2 and 3 > 2", "1 < 2 and 3 > 2"), ("ad", "a<b and c>d")] { + let text = export(input, format); + assert!(text.contains(expected), "{format:?}: {text}"); + } + } +} + +#[test] +fn real_tags_still_produce_markup() { + for format in [Format::Ttml, Format::Dfxp] { + let text = export("text x z it", format); + assert!(text.contains("text"), "{format:?}: {text}"); + assert!(text.contains("it"), "{format:?}: {text}"); + assert!(text.contains(" x z "), "{format:?}: {text}"); + } +} + +#[test] +fn self_closing_tags_are_still_stripped() { + for format in [Format::Ttml, Format::Dfxp] { + let text = export("a
b c", format); + assert!(text.contains(">ab c

"), "{format:?}: {text}"); + } +} + +#[test] +fn quoted_gt_in_attribute_does_not_end_tag() { + for format in [Format::Ttml, Format::Dfxp] { + let text = export("\">text", format); + assert!(text.contains("text"), "{format:?}: {text}"); + } +} + +#[test] +fn single_quoted_attributes_are_stripped() { + for format in [Format::Ttml, Format::Dfxp] { + let text = export("word", format); + assert!(text.contains("word"), "{format:?}: {text}"); + assert!(!text.contains("caption"), "{format:?}: {text}"); + } +} + +#[test] +fn whitespace_after_lt_is_literal_text() { + for format in [Format::Ttml, Format::Dfxp] { + let text = export("1 < b > 2", format); + assert!(text.contains("1 < b > 2"), "{format:?}: {text}"); + assert!(!text.contains("fontWeight"), "{format:?}: {text}"); + } +} + +#[test] +fn many_unmatched_lt_is_fast() { + let input = format!("{}>", "<".repeat(256_000)); + let start = std::time::Instant::now(); + let text = export(&input, Format::Ttml); + assert!(start.elapsed() < std::time::Duration::from_secs(2), "took {:?}", start.elapsed()); + assert!(text.contains(&format!("{}>", "<".repeat(256_000)))); +} diff --git a/crates/captions/tests/it/ttml_xml_views.rs b/crates/captions/tests/it/ttml_xml_views.rs new file mode 100644 index 000000000..8173039e7 --- /dev/null +++ b/crates/captions/tests/it/ttml_xml_views.rs @@ -0,0 +1,119 @@ +//! The TTML reader used to sit on `roxmltree` and now sits on `craft-xml`. Its logic is the same; +//! what changed is the tree it is handed. This compares the two trees on TTML documents: element +//! names by namespace, attributes by namespace, and character data, in document order. + +use craft_captions::{Cue, Document, Format, FrameRate, WriteOptions, write}; + +const NAMESPACES: [(&str, &str); 6] = [ + ("tt", "http://www.w3.org/ns/ttml"), + ("tt10", "http://www.w3.org/2006/10/ttaf1"), + ("tt04", "http://www.w3.org/2006/04/ttaf1"), + ("ttm", "http://www.w3.org/ns/ttml#metadata"), + ("ttm10", "http://www.w3.org/2006/10/ttaf1#metadata"), + ("ttm04", "http://www.w3.org/2006/04/ttaf1#metadata"), +]; + +/// A name the way the reader's canonical names spell it. +fn name(namespace: Option<&str>, local: &str) -> String { + match namespace { + None => local.to_string(), + Some("http://www.w3.org/XML/1998/namespace") => format!("xml:{local}"), + Some(uri) => format!("{}:{local}", NAMESPACES.iter().find(|(_, u)| *u == uri).map_or("?", |(p, _)| p)), + } +} + +fn dump_rox(n: roxmltree::Node, out: &mut Vec) { + let mut attrs: Vec = n.attributes().map(|a| format!("@{}={}", name(a.namespace(), a.name()), a.value())).collect(); + attrs.sort(); + out.push(format!("<{} {}", name(n.tag_name().namespace(), n.tag_name().name()), attrs.join(" "))); + let mut text = String::new(); + for c in n.children() { + if c.is_text() { + text.push_str(c.text().unwrap_or("")); + } else if c.is_element() { + out.push(format!("#{}", std::mem::take(&mut text))); + dump_rox(c, out); + } + } + out.push(format!("#{text}")); + out.push(">".into()); +} + +fn dump_craft(n: &craft_xml::El, out: &mut Vec) { + let mut attrs: Vec = n.attrs.iter().map(|(k, v)| format!("@{k}={v}")).collect(); + attrs.sort(); + out.push(format!("<{} {}", n.name, attrs.join(" "))); + let mut text = String::new(); + for c in &n.children { + match c { + craft_xml::Node::Text(t) | craft_xml::Node::CData(t) => text.push_str(t), + craft_xml::Node::El(e) => { + out.push(format!("#{}", std::mem::take(&mut text))); + dump_craft(e, out); + } + _ => {} + } + } + out.push(format!("#{text}")); + out.push(">".into()); +} + +fn documents() -> Vec { + let mut docs: Vec = [ + r#"

Hi

"#, + // prefixed content, styling and metadata; xml:lang, xml:id, xml:space + r#" + + Ann Lee + + onetwo three four +"#, + // the 2006 drafts + r#" +Bo

x

"#, + r#"Bo +

x

"#, + // no namespace at all, an unknown namespace, and a foreign element inside + r#"

x

"#, + r#"

x

"#, + // character data: references, CDATA, comments, processing instructions, mixed whitespace + "

\n a <b> & © 😀 & ]]>tail\u{a0}more end\r\n

", + // a namespace redeclared in an inner scope + r#"no"#, + ] + .into_iter() + .map(str::to_string) + .collect(); + // and what the writer produces, in both flavours + let rate = FrameRate::FPS_25; + let cue = |f: i64, text: &str, speaker: Option<&str>| Cue { + start: rate.tick_of(f), + end: rate.tick_of(f + 25), + text: text.into(), + speaker: speaker.map(str::to_string), + ..Default::default() + }; + let doc = Document { + cues: vec![cue(25, "Olá & tchau\n1 < 2", Some("Ana \"A\" ")), cue(75, "second\u{a0}line", None)], + language: Some("pt".into()), + ..Default::default() + }; + for format in [Format::Ttml, Format::Dfxp] { + docs.push(String::from_utf8(write(&doc, format, WriteOptions { rate: Some(rate), ..Default::default() })).unwrap()); + } + docs +} + +#[test] +fn craft_xml_hands_the_ttml_reader_the_tree_roxmltree_did() { + let canonical = craft_xml::Canonical::new(&NAMESPACES); + for text in documents() { + let rox = roxmltree::Document::parse(&text).unwrap(); + let craft = craft_xml::Options::canonical(&canonical).strict().parse(text.as_bytes()).unwrap(); + let (mut a, mut b) = (Vec::new(), Vec::new()); + dump_rox(rox.root_element(), &mut a); + dump_craft(&craft, &mut b); + assert_eq!(a, b, "{text}"); + } +}