diff --git a/crates/ocr/src/lib.rs b/crates/ocr/src/lib.rs index aac95f18c6..d702bca055 100644 --- a/crates/ocr/src/lib.rs +++ b/crates/ocr/src/lib.rs @@ -23,9 +23,11 @@ pub mod confidence; pub mod merge; mod ocrs; +mod pitch; pub mod preprocess; #[cfg(not(target_arch = "wasm32"))] pub mod tesseract; +mod tiles; pub use confidence::{ConfidenceBand, band_for, is_suspect, low_confidence}; pub use merge::{MergeStrategy, StrategyOutcome, mean_confidence, recognize_with_strategy, rover_merge}; @@ -867,6 +869,38 @@ mod tests { assert!(word.rect[0] >= 10.0 && word.rect[0] < 60.0, "{word:?}"); } + /// A large page with small print: shrunk whole to the detector's input, its letters are too + /// small to find; in tiles they are found. The words come back in page pixels, once each, + /// also where they lie in the overlap of two tiles or across a cut. + #[test] + fn finds_small_print_on_a_large_page() { + let Some(models) = Models::find() else { + eprintln!("skipped: OCR models not installed"); + return; + }; + let ocr = Ocr::load(&models).unwrap(); + assert_eq!(ocr.detector_input, Some((600, 800)), "the tiling rule assumes the ocrs detector"); + let (w, h, dot) = (2480u32, 3508u32, DOT); + // An A4 page at 300 dpi is cut at x = 1240 and y = 1754, its tiles overlapping by 99 and + // 140 pixels around the cuts. The lines lie: in one tile only; across the vertical cut; + // half in the overlap left of it; across the horizontal cut. + let at = [(300, 400), (1170, 1000), (1100, 2500), (300, 1748)]; + let px = test_image::hello_on(w, h, dot, &at); + let lines = ocr.recognize(&px, w, h).unwrap(); + let words: Vec<&Word> = lines.iter().flat_map(|l| &l.words).collect(); + // Two words per line, each once: none lost, none doubled where tiles overlap. The letters + // are tiny and blocky, so the text itself may be a letter off ("HFLLO"). + assert_eq!(words.len(), 2 * at.len(), "{lines:?}"); + for (x, y) in at { + let near = |w: &&&Word| (w.rect[0] - x as f32).abs() < 4.0 * dot as f32 && (w.rect[1] - y as f32).abs() < 4.0 * dot as f32; + assert_eq!(words.iter().filter(near).count(), 1, "no first word at ({x}, {y}): {lines:?}"); + } + assert!(words.iter().any(|w| w.text == "HELLO"), "{lines:?}"); + } + + /// The letter size of [`finds_small_print_on_a_large_page`], in pixels per dot of a 5×7 letter. + const DOT: u32 = 2; + /// The recognised words name their engine. #[test] fn ocrs_recognizer_names_its_words() { @@ -994,6 +1028,14 @@ mod tests { mod test_image { /// "HELLO WORLD" drawn with blocky 5×7 letters, 4 pixels per dot, black on white. pub fn hello() -> (u32, u32, Vec) { + let (dot, x0, y0) = (4u32, 40u32, 40u32); + let (w, h) = (x0 * 2 + 11 * 6 * dot, y0 * 2 + 7 * dot); + (w, h, hello_on(w, h, dot, &[(x0, y0)])) + } + + /// A white `w` × `h` page with "HELLO WORLD" at each of `at` (top left corner), `dot` + /// pixels per dot of the letters. + pub fn hello_on(w: u32, h: u32, dot: u32, at: &[(u32, u32)]) -> Vec { const GLYPHS: &[(char, [&str; 7])] = &[ ('H', ["X...X", "X...X", "X...X", "XXXXX", "X...X", "X...X", "X...X"]), ('E', ["XXXXX", "X....", "X....", "XXXX.", "X....", "X....", "XXXXX"]), @@ -1004,10 +1046,8 @@ mod tests { ('D', ["XXXX.", "X...X", "X...X", "X...X", "X...X", "X...X", "XXXX."]), ]; let text = "HELLO WORLD"; - let (dot, x0, y0) = (4u32, 40u32, 40u32); - let (w, h) = (x0 * 2 + text.len() as u32 * 6 * dot, y0 * 2 + 7 * dot); let mut px = vec![255u8; (w * h * 4) as usize]; - for (i, c) in text.chars().enumerate() { + for ((i, c), &(x0, y0)) in at.iter().flat_map(|p| text.chars().enumerate().map(move |c| (c, p))) { let Some((_, rows)) = GLYPHS.iter().find(|g| g.0 == c) else { continue }; for (ry, row) in rows.iter().enumerate() { for (rx, b) in row.bytes().enumerate() { @@ -1025,7 +1065,7 @@ mod tests { } } } - (w, h, px) + px } } } diff --git a/crates/ocr/src/ocrs.rs b/crates/ocr/src/ocrs.rs index 53070d90c5..03bbfef808 100644 --- a/crates/ocr/src/ocrs.rs +++ b/crates/ocr/src/ocrs.rs @@ -131,6 +131,9 @@ fn clip(rect: [f32; 4], to: Option) -> [f32; 4] { /// A loaded recogniser. Loading takes a moment; keep one and reuse it. pub struct Ocr { engine: ocrs::OcrEngine, + /// The detection model's fixed input size, (width, height) pixels, when it declares one: a + /// page much larger than that is detected in tiles. + pub(crate) detector_input: Option<(u32, u32)>, } impl Ocr { @@ -152,6 +155,7 @@ impl Ocr { } fn with_models(detection: rten::Model, recognition: rten::Model) -> Result { + let detector_input = detector_input(&detection); let params = ocrs::OcrEngineParams { detection_model: Some(detection), recognition_model: Some(recognition), @@ -159,7 +163,7 @@ impl Ocr { ..Default::default() }; let engine = ocrs::OcrEngine::new(params).map_err(|e| OcrError::Load("ocr engine".into(), e.to_string()))?; - Ok(Ocr { engine }) + Ok(Ocr { engine, detector_input }) } /// Load the models found by [`Models::find`]. @@ -182,7 +186,7 @@ impl Ocr { let err = |e: &dyn std::fmt::Display| OcrError::Recognize(e.to_string()); let source = ocrs::ImageSource::from_bytes(&rgb, (width, height)).map_err(|e| err(&e))?; let input = self.engine.prepare_input(source).map_err(|e| err(&e))?; - let found = self.engine.detect_words(&input).map_err(|e| err(&e))?; + let found = self.detect_words(&rgb, width, height, &input)?; let mut lines = self.engine.find_text_lines(&input, &found); let before: Vec> = lines.iter_mut().map(|l| pad_line(l)).collect(); let read = self.engine.recognize_text(&input, &lines).map_err(|e| err(&e))?; @@ -204,6 +208,51 @@ impl Ocr { .filter(|l| !l.words.is_empty()) .collect()) } + + /// The words in a page, `rgb` (1 or 3 bytes per pixel) prepared as `page`. A page at least + /// twice the detector's input along an axis whose text is small once shrunk to that input + /// ([`crate::pitch`]) is detected in overlapping tiles ([`crate::tiles`]), each word kept by the + /// tile whose core holds its centre and moved back to page pixels. + fn detect_words(&self, rgb: &[u8], width: u32, height: u32, page: &ocrs::OcrInput) -> Result, OcrError> { + let err = |e: &dyn std::fmt::Display| OcrError::Recognize(e.to_string()); + let tiles = self.detector_input.map(|input| crate::tiles::tiles(width, height, input)).unwrap_or_default(); + let area = (width as usize).checked_mul(height as usize).filter(|&a| a > 0); + let channels = area.and_then(|a| rgb.len().is_multiple_of(a).then(|| rgb.len() / a)).filter(|c| matches!(c, 1 | 3)); + let (Some(channels), Some(input), true) = (channels, self.detector_input, tiles.len() > 1) else { + return self.engine.detect_words(page).map_err(|e| err(&e)); + }; + // Large print reads as well whole, at a quarter of the detection time. + if !crate::pitch::lines(rgb, width, height, channels, input).wants_tiles() { + return self.engine.detect_words(page).map_err(|e| err(&e)); + } + let mut words = Vec::new(); + for tile in &tiles { + let pixels = crate::tiles::crop(rgb, width, channels, tile.crop) + .ok_or_else(|| OcrError::Recognize("a detection tile lies outside the page".into()))?; + let source = ocrs::ImageSource::from_bytes(&pixels, (tile.width(), tile.height())).map_err(|e| err(&e))?; + let input = self.engine.prepare_input(source).map_err(|e| err(&e))?; + let (dx, dy) = (tile.crop[0] as f32, tile.crop[1] as f32); + for word in self.engine.detect_words(&input).map_err(|e| err(&e))? { + let centre = word.center(); + let (x, y) = (centre.x + dx, centre.y + dy); + if tile.owns(x, y) { + words.push(RotatedRect::new(rten_imageproc::PointF::from_yx(y, x), word.up_axis(), word.width(), word.height())); + } + } + } + Ok(words) + } +} + +/// The detection model's input size, (width, height) pixels, if its input is an NCHW image of a +/// fixed size. +fn detector_input(model: &rten::Model) -> Option<(u32, u32)> { + let id = *model.input_ids().first()?; + let shape = model.node_info(id)?.shape()?; + match shape.as_slice() { + [_, _, rten::Dimension::Fixed(h), rten::Dimension::Fixed(w)] => Some((u32::try_from(*w).ok()?, u32::try_from(*h).ok()?)), + _ => None, + } } /// The ocrs engine as a [`Recognizer`] (the default engine). diff --git a/crates/ocr/src/pitch.rs b/crates/ocr/src/pitch.rs new file mode 100644 index 0000000000..316d914e06 --- /dev/null +++ b/crates/ocr/src/pitch.rs @@ -0,0 +1,363 @@ +//! How tall the text on a page is, measured cheaply from its pixels, to decide whether detecting +//! it in tiles is worth the cost. +//! +//! What decides whether tiling helps is the height of the text once the page is shrunk to the +//! detector's input, not how full the page is. Lines of text repeat at a regular distance, the +//! line pitch, so the ink summed per pixel row rises and falls with that period. The page is +//! shrunk to a grey copy of at most [`WORK`] pixels a side and cut into a [`GRID`] × [`GRID`] +//! grid of blocks. In each block the ink is summed per row, the slow trend (shading, pictures) is +//! taken out with a moving average, and the autocorrelation of what remains is taken. The +//! autocorrelations of the blocks are averaged: columns of text whose lines do not line up still +//! agree on the period, and blocks of blank paper are left out. The period is the first strong +//! peak of the average. The same is done with the ink summed per pixel column, for pages that lie +//! sideways, and the axis with the stronger peak wins. +//! +//! The constants were chosen on MDPBench (de/es/fr, 480 pages; see the PR that added this file): +//! tiling a page whose lines repeat every fewer than [`SMALL_PITCH`] detector pixels, or whose +//! lines show no regular spacing at all, scored better than tiling every page, at about half the +//! cost. + +/// The largest side of the grey copy the pitch is measured on, in pixels. +const WORK: u32 = 1600; +/// The page is cut into `GRID` × `GRID` blocks. +const GRID: u32 = 6; +/// The longest line pitch looked for, in detector pixels. +const MAX_PITCH: f64 = 30.0; +/// A peak of the averaged, normalised autocorrelation weaker than this is noise. +const PEAK_FLOOR: f64 = 0.05; +/// The first peak at least this share of the strongest one is the pitch; the later ones are its +/// multiples (every second or third line). +const PEAK_SHARE: f64 = 0.6; +/// Lines closer together than this, in detector pixels, are small print. +pub(crate) const SMALL_PITCH: f64 = 9.0; + +/// What the pixels of a page say about its lines of text. +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) enum Lines { + /// Nothing to measure: blank or uniform paper, a page too narrow to show a pitch, or pixels + /// that do not form the image they claim to. + Blank, + /// Ink without a regular line spacing: pictures, blur, a skewed or curved photo, a few + /// scattered lines. + Unclear, + /// Lines repeat every so many pixels of the detector's input. + Pitch(f64), +} + +impl Lines { + /// Whether the page is worth detecting in tiles: its lines are close together once shrunk to + /// the detector's input, or the pixels cannot tell. A blank page is not. + pub fn wants_tiles(self) -> bool { + match self { + Lines::Blank => false, + Lines::Unclear => true, + Lines::Pitch(p) => p < SMALL_PITCH, + } + } +} + +/// The lines of a `width` × `height` image of `channels` bytes per pixel (grey, grey and alpha, +/// RGB or RGBA), for a detector that reads `input` = (width, height) pixels. Reads every pixel +/// once; the rest works on at most [`WORK`] × [`WORK`] values. +pub(crate) fn lines(pixels: &[u8], width: u32, height: u32, channels: usize, input: (u32, u32)) -> Lines { + let size = (width as usize).checked_mul(height as usize).and_then(|a| a.checked_mul(channels)); + if !(1..=4).contains(&channels) || size != Some(pixels.len()) || pixels.is_empty() || input.0 == 0 || input.1 == 0 { + return Lines::Blank; + } + let Some(grey) = Grey::shrink(pixels, width, height, channels) else { return Lines::Blank }; + // The longest pitch looked for, in pixels of the grey copy, along each axis. + let reach = |len: u32, input: u32| ((MAX_PITCH * f64::from(len) / f64::from(input)) as usize).max(4); + let down = grey.curve(Axis::Down, reach(grey.height, input.1)); + let across = grey.curve(Axis::Across, reach(grey.width, input.0)); + if !down.inked && !across.inked { + return Lines::Blank; + } + let to_input = |lag: usize, len: u32, input: u32| lag as f64 * f64::from(grey.step) * f64::from(input) / f64::from(len); + match (peak(&down.correlation), peak(&across.correlation)) { + (None, None) => Lines::Unclear, + (Some((lag, d)), Some((_, a))) if d >= a => Lines::Pitch(to_input(lag, height, input.1)), + (Some((lag, _)), None) => Lines::Pitch(to_input(lag, height, input.1)), + (_, Some((lag, _))) => Lines::Pitch(to_input(lag, width, input.0)), + } +} + +/// A shrunk copy of a page as ink: 0 for white, 255 for black. +struct Grey { + ink: Vec, + width: u32, + height: u32, + /// Page pixels per pixel of the copy, along each axis. + step: u32, +} + +#[derive(Clone, Copy)] +enum Axis { + /// Ink summed per row: lines of text lying the normal way. + Down, + /// Ink summed per column: lines lying sideways. + Across, +} + +/// The averaged autocorrelation of one axis, and whether any block had ink. +struct Curve { + correlation: Vec, + inked: bool, +} + +impl Grey { + /// Box-average the page down by a whole factor until neither side exceeds [`WORK`]. The + /// caller has checked that `pixels` holds `width` × `height` pixels of `channels` bytes. + fn shrink(pixels: &[u8], width: u32, height: u32, channels: usize) -> Option { + let step = width.max(height).div_ceil(WORK).max(1); + let (w, h) = (width / step, height / step); + if w == 0 || h == 0 { + return None; + } + // At most WORK × WORK values: `step` is chosen so that each side fits. + let mut sum = vec![0f32; (w as usize).checked_mul(h as usize)?]; + let row_bytes = (width as usize).checked_mul(channels)?; + let (step_px, w_px) = (step as usize, w as usize); + for (y, row) in pixels.chunks_exact(row_bytes).take(h as usize * step_px).enumerate() { + let start = (y / step_px).checked_mul(w_px)?; + let Some(out) = sum.get_mut(start..start.checked_add(w_px)?) else { continue }; + for (x, px) in row.chunks_exact(channels).take(w_px * step_px).enumerate() { + if let Some(o) = out.get_mut(x / step_px) { + *o += luma(px); + } + } + } + let area = (step as f32) * (step as f32); + let ink = sum.into_iter().map(|s| 255.0 - s / area).collect(); + Some(Grey { ink, width: w, height: h, step }) + } + + /// The ink of the copy at `(x, y)`, 0 outside it. + fn at(&self, x: usize, y: usize) -> f32 { + if x >= self.width as usize { + return 0.0; + } + y.checked_mul(self.width as usize).and_then(|i| i.checked_add(x)).and_then(|i| self.ink.get(i)).copied().unwrap_or(0.0) + } + + /// The autocorrelation of the ink profile along `axis`, for lags `0..=reach`, averaged over + /// the blocks that have ink. Empty when no block has, or the blocks are too short to show + /// `reach`. + fn curve(&self, axis: Axis, reach: usize) -> Curve { + let (along, across) = match axis { + Axis::Down => (self.height as usize, self.width as usize), + Axis::Across => (self.width as usize, self.height as usize), + }; + let (block, side) = (along / GRID as usize, across / GRID as usize); + let mut total = vec![0f64; reach + 1]; + let (mut blocks, mut inked) = (0usize, false); + if side == 0 || block < 2 * reach + 2 { + return Curve { correlation: Vec::new(), inked: false }; + } + let mut profile = vec![0f64; block]; + for i in 0..GRID as usize { + for j in 0..GRID as usize { + // The block's ink per row (or column), averaged across the block. + for (k, p) in profile.iter_mut().enumerate() { + let a = i * block + k; + let sum: f32 = (j * side..(j + 1) * side) + .map(|b| match axis { + Axis::Down => self.at(b, a), + Axis::Across => self.at(a, b), + }) + .sum(); + *p = f64::from(sum) / side as f64; + } + let detrended = detrend(&profile, reach); + let energy: f64 = detrended.iter().map(|d| d * d).sum(); + // Blank paper: next to no variation along the block. + if energy <= 1e-3 * block as f64 { + continue; + } + inked = true; + blocks += 1; + for (lag, t) in total.iter_mut().enumerate() { + let c: f64 = detrended.iter().zip(detrended.iter().skip(lag)).map(|(a, b)| a * b).sum(); + *t += c / energy; + } + } + } + if blocks == 0 { + return Curve { correlation: Vec::new(), inked }; + } + total.iter_mut().for_each(|t| *t /= blocks as f64); + Curve { correlation: total, inked } + } +} + +/// The grey value of a pixel of 1 to 4 bytes (grey, grey and alpha, RGB, RGBA), 0 to 255. +fn luma(px: &[u8]) -> f32 { + match *px { + [r, g, b, ..] => 0.299 * f32::from(r) + 0.587 * f32::from(g) + 0.114 * f32::from(b), + [v, ..] => f32::from(v), + [] => 255.0, + } +} + +/// `profile` minus its moving average over `window` samples, the ends held at the edge values. +fn detrend(profile: &[f64], window: usize) -> Vec { + let n = profile.len(); + let window = window.max(1); + let half = window / 2; + (0..n) + .map(|i| { + let sum: f64 = (0..window).filter_map(|j| profile.get((i + j).saturating_sub(half).min(n.saturating_sub(1)))).sum(); + profile.get(i).copied().unwrap_or(0.0) - sum / window as f64 + }) + .collect() +} + +/// The line pitch in a normalised autocorrelation, as `(lag, strength)`: past the first zero +/// crossing, the first local maximum at least [`PEAK_SHARE`] of the strongest one, counting only +/// maxima above [`PEAK_FLOOR`]. `None` without such a peak. +fn peak(correlation: &[f64]) -> Option<(usize, f64)> { + let zero = correlation.iter().position(|&c| c < 0.0)?; + let peaks: Vec<(usize, f64)> = correlation + .windows(3) + .enumerate() + .filter_map(|(i, w)| match *w { + [a, b, c] if i + 1 >= zero.max(2) && b >= a && b >= c && b >= PEAK_FLOOR => Some((i + 1, b)), + _ => None, + }) + .collect(); + let strongest = peaks.iter().map(|p| p.1).fold(f64::NEG_INFINITY, f64::max); + peaks.into_iter().find(|p| p.1 >= PEAK_SHARE * strongest) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The detector of the ocrs models reads 600 × 800 pixels. + const INPUT: (u32, u32) = (600, 800); + /// A4 at 300 dpi. + const A4: (u32, u32) = (2480, 3508); + + /// A white grey page with lines of "words" (dark bars) every `pitch` pixels, 40 % of the + /// pitch tall, within 200-pixel margins; `sideways` turns the lines a quarter turn. + fn print(pitch: u32, sideways: bool) -> Vec { + let (w, h) = A4; + let mut px = vec![255u8; (w * h) as usize]; + for y in 200..h - 200 { + for x in 200..w - 200 { + let (along, line) = if sideways { (y, x) } else { (x, y) }; + // Words of 4 × 15 pixels with a space of 15 between them. + if line % pitch < pitch * 2 / 5 && (along / 15) % 5 != 4 { + px[(y * w + x) as usize] = 0; + } + } + } + px + } + + fn pitch_of(lines: Lines) -> f64 { + match lines { + Lines::Pitch(p) => p, + other => panic!("no pitch: {other:?}"), + } + } + + #[test] + fn a_blank_page_is_not_tiled() { + let (w, h) = A4; + let lines = lines(&vec![255u8; (w * h) as usize], w, h, 1, INPUT); + assert_eq!(lines, Lines::Blank); + assert!(!lines.wants_tiles()); + } + + #[test] + fn large_print_is_not_tiled() { + // Lines every 100 pixels of an A4 page at 300 dpi (8 mm, a 20-point font) repeat every + // 100 × 800 / 3508 = 22.8 detector pixels. + let (w, h) = A4; + let lines = lines(&print(100, false), w, h, 1, INPUT); + let p = pitch_of(lines); + assert!((21.0..24.0).contains(&p), "{p}"); + assert!(!lines.wants_tiles()); + } + + #[test] + fn small_print_is_tiled() { + // Lines every 30 pixels (2.5 mm, a 7-point font): 6.8 detector pixels. + let (w, h) = A4; + let lines = lines(&print(30, false), w, h, 1, INPUT); + let p = pitch_of(lines); + assert!((6.0..7.5).contains(&p), "{p}"); + assert!(lines.wants_tiles()); + // The same page in RGB and RGBA. + for channels in [3, 4] { + let px: Vec = print(30, false).into_iter().flat_map(|v| std::iter::repeat_n(v, channels)).collect(); + assert_eq!(super::lines(&px, w, h, channels, INPUT), lines, "{channels} channels"); + } + } + + #[test] + fn sideways_small_print_is_tiled() { + // Lines every 30 pixels across a 2480-pixel width: 30 × 600 / 2480 = 7.3 detector pixels. + let (w, h) = A4; + let lines = lines(&print(30, true), w, h, 1, INPUT); + let p = pitch_of(lines); + assert!((6.5..8.0).contains(&p), "{p}"); + assert!(lines.wants_tiles()); + } + + #[test] + fn ink_without_lines_is_tiled() { + // Noise (a picture, a blurred photo): the pixels cannot tell how tall the text is. + let (w, h) = A4; + let mut seed = 0x2545_f491_4f6c_dd1du64; + let px: Vec = (0..w * h) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + if seed & 1 == 0 { 0 } else { 255 } + }) + .collect(); + let lines = lines(&px, w, h, 1, INPUT); + assert_eq!(lines, Lines::Unclear); + assert!(lines.wants_tiles()); + } + + #[test] + fn odd_inputs_give_an_answer_without_panicking() { + assert_eq!(lines(&[], 0, 0, 1, INPUT), Lines::Blank); + assert_eq!(lines(&[0], 1, 1, 1, INPUT), Lines::Blank); + assert_eq!(lines(&[0, 0, 0], 1, 1, 3, INPUT), Lines::Blank); + // Fewer or more bytes than the size claims, no channels, too many, a detector of no size. + assert_eq!(lines(&[0; 7], 2, 2, 2, INPUT), Lines::Blank); + assert_eq!(lines(&[0; 4], 2, 2, 0, INPUT), Lines::Blank); + assert_eq!(lines(&[0; 20], 2, 2, 5, INPUT), Lines::Blank); + assert_eq!(lines(&[0; 4], u32::MAX, u32::MAX, 4, INPUT), Lines::Blank); + assert_eq!(lines(&print(30, false), A4.0, A4.1, 1, (0, 0)), Lines::Blank); + // A sliver: one side shrinks to nothing, or too short to show a pitch. + assert_eq!(lines(&[0; 5000], 1, 5000, 1, INPUT), Lines::Blank); + let tall = (0..3 * 40_000u32).map(|i| if (i / 3) % 30 < 12 { 0 } else { 255 }).collect::>(); + let _ = lines(&tall, 3, 40_000, 1, INPUT); + // A black page: ink everywhere, but nothing that varies. + assert_eq!(lines(&vec![0u8; 2000 * 2000], 2000, 2000, 1, INPUT), Lines::Blank); + } + + #[test] + fn the_first_strong_peak_is_the_pitch() { + // Peaks at 5 (0.5), 10 (0.8), 15 (0.3): 5 is past the share of the strongest, so it is + // the pitch and 10 its double. + let mut c = vec![0.0; 18]; + c[0] = 1.0; + c[2] = -0.2; + c[5] = 0.5; + c[10] = 0.8; + c[15] = 0.3; + assert_eq!(peak(&c), Some((5, 0.5))); + c[5] = 0.4; + assert_eq!(peak(&c), Some((10, 0.8))); + // No zero crossing, or nothing above the floor: no pitch. + assert_eq!(peak(&[1.0, 0.5, 0.4, 0.5, 0.3]), None); + assert_eq!(peak(&[1.0, -0.1, 0.01, 0.0, 0.02, 0.0]), None); + assert_eq!(peak(&[]), None); + } +} diff --git a/crates/ocr/src/tiles.rs b/crates/ocr/src/tiles.rs new file mode 100644 index 0000000000..a20f5d5f12 --- /dev/null +++ b/crates/ocr/src/tiles.rs @@ -0,0 +1,196 @@ +//! Tiled text detection for large pages. +//! +//! The detection model reads a fixed-size image, so ocrs shrinks a larger page to that size first. +//! On a dense page (a newspaper, small print, a blurred photo) the shrunk letters become too small +//! to find. A page at least twice the model's input along an axis is therefore cut in two along it +//! (2 × 2 tiles for a full page at 300 dpi), each tile reaching a little into its neighbours so +//! that a word on a cut is whole in one of them. Words are detected tile by tile; a word is kept +//! only by the tile whose core cell holds its centre, so a word seen twice in an overlap counts +//! once. Lines are then formed and recognised on the whole page, as without tiles. Only a page +//! whose text is small once shrunk is tiled; [`crate::pitch`] decides. + +/// How far a tile reaches past its core cell into each neighbour, as a fraction of the page along +/// that axis. MDPBench (de/es/fr) was measured with 4 %. +const OVERLAP: f64 = 0.04; + +/// One tile of a page. +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) struct Tile { + /// The pixels the tile crops from the page, `[left, top, right, bottom]`, right and bottom + /// exclusive. Always within the page and never empty. + pub crop: [u32; 4], + /// The cell the tile owns, in page pixels, `[left, top, right, bottom]`, right and bottom + /// exclusive. The cells of the page's edge tiles reach to infinity, so the cells cover the + /// plane without gaps or overlaps: every finite point lies in exactly one. + pub core: [f32; 4], +} + +impl Tile { + /// Whether a word centred at `(x, y)` (page pixels) belongs to this tile. Never true for a + /// non-finite point. + pub fn owns(&self, x: f32, y: f32) -> bool { + let [l, t, r, b] = self.core; + x.is_finite() && y.is_finite() && l <= x && x < r && t <= y && y < b + } + + pub fn width(&self) -> u32 { + self.crop[2].saturating_sub(self.crop[0]) + } + + pub fn height(&self) -> u32 { + self.crop[3].saturating_sub(self.crop[1]) + } +} + +/// How many pieces an axis of `len` pixels is cut into for a detector that reads `input` pixels +/// along it: two once each half still fills the detector's input (the detector would shrink the +/// page by at least half), otherwise one, so a smaller page is read as before at no extra cost. +fn pieces(len: u32, input: u32) -> u32 { + if input > 0 && len / 2 >= input { 2 } else { 1 } +} + +/// The cells and crops of one axis of `len` pixels cut into `n` pieces: `(crop start, crop end, +/// core start, core end)` for each piece. +fn axis(len: u32, n: u32) -> Vec<(u32, u32, f32, f32)> { + let total = f64::from(len); + let margin = (OVERLAP * total).max(0.0); + (0..n) + .map(|i| { + let c0 = total * f64::from(i) / f64::from(n); + let c1 = total * f64::from(i + 1) / f64::from(n); + // `as` saturates for floats; both values lie in 0..=len. + let from = (c0 - margin).max(0.0).floor() as u32; + let to = ((c1 + margin).min(total).ceil() as u32).min(len); + let core0 = if i == 0 { f32::NEG_INFINITY } else { c0 as f32 }; + let core1 = if i + 1 == n { f32::INFINITY } else { c1 as f32 }; + (from, to, core0, core1) + }) + .collect() +} + +/// The tiles to detect text in on a `width` × `height` page, for a detector whose input is +/// `input` = (width, height) pixels. One tile covering the whole page when no axis is worth +/// cutting. +pub(crate) fn tiles(width: u32, height: u32, input: (u32, u32)) -> Vec { + let columns = axis(width, pieces(width, input.0)); + let rows = axis(height, pieces(height, input.1)); + rows.iter() + .flat_map(|&(top, bottom, core_top, core_bottom)| { + columns.iter().map(move |&(left, right, core_left, core_right)| Tile { + crop: [left, top, right, bottom], + core: [core_left, core_top, core_right, core_bottom], + }) + }) + .filter(|t| t.width() > 0 && t.height() > 0) + .collect() +} + +/// The pixels of `crop` from an image `width` pixels wide with `channels` bytes per pixel, row +/// by row. `None` if the crop reaches outside `pixels`. +pub(crate) fn crop(pixels: &[u8], width: u32, channels: usize, crop: [u32; 4]) -> Option> { + let [left, top, right, bottom] = crop.map(|v| v as usize); + let width = width as usize; + if right > width || left >= right || top >= bottom { + return None; + } + let row_len = (right - left).checked_mul(channels)?; + let mut out = Vec::with_capacity(row_len.checked_mul(bottom - top)?); + for y in top..bottom { + let start = y.checked_mul(width)?.checked_add(left)?.checked_mul(channels)?; + out.extend_from_slice(pixels.get(start..start.checked_add(row_len)?)?); + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The detector of the ocrs models reads 600 × 800 pixels. + const INPUT: (u32, u32) = (600, 800); + + #[test] + fn small_pages_are_read_whole() { + // US Letter at 72 and 150 dpi, a screenshot, a page twice as wide as the input but not + // twice as tall: no axis is cut, and the one tile is the page. + for (w, h) in [(612, 792), (1275, 1599), (1199, 3000), (1, 1)] { + let t = tiles(w, h, INPUT); + let expect_cols = if w >= 1200 { 2 } else { 1 }; + let expect_rows = if h >= 1600 { 2 } else { 1 }; + assert_eq!(t.len(), expect_cols * expect_rows, "{w}x{h}: {t:?}"); + } + let one = tiles(612, 792, INPUT); + assert_eq!(one, vec![Tile { crop: [0, 0, 612, 792], core: [f32::NEG_INFINITY, f32::NEG_INFINITY, f32::INFINITY, f32::INFINITY] }]); + } + + #[test] + fn a_page_at_300_dpi_is_cut_into_four_overlapping_tiles() { + // A4 at 300 dpi. + let (w, h) = (2480, 3508); + let t = tiles(w, h, INPUT); + assert_eq!(t.len(), 4, "{t:?}"); + // 4 % of 2480 is 99.2 pixels, of 3508 140.32. + assert_eq!(t[0].crop, [0, 0, 1340, 1895]); + assert_eq!(t[1].crop, [1140, 0, 2480, 1895]); + assert_eq!(t[2].crop, [0, 1613, 1340, 3508]); + assert_eq!(t[3].crop, [1140, 1613, 2480, 3508]); + // Each tile still fills the detector's input. + assert!(t.iter().all(|t| t.width() >= INPUT.0 && t.height() >= INPUT.1), "{t:?}"); + // The cores meet at the middle of the page. + assert_eq!(t[0].core[2], 1240.0); + assert_eq!(t[3].core[0], 1240.0); + assert_eq!(t[0].core[3], 1754.0); + // Each core lies inside its crop, wherever it does not run off the page. + for tile in &t { + let [l, tp, r, b] = tile.core.map(|v| v.clamp(0.0, 4000.0)); + let [cl, ct, cr, cb] = tile.crop.map(|v| v as f32); + assert!(cl <= l && ct <= tp && r.min(w as f32) <= cr && b.min(h as f32) <= cb, "{tile:?}"); + } + } + + /// Each axis is cut at most once, so no page costs more than four detector runs: A0 at + /// 300 dpi, a long receipt and the largest page the types allow are cut like A4. + #[test] + fn no_page_is_cut_into_more_than_four_tiles() { + for (w, h, expect) in [(9933, 14043, 4), (14043, 9933, 4), (800, 60_000, 2), (u32::MAX, u32::MAX, 4)] { + let t = tiles(w, h, INPUT); + assert_eq!(t.len(), expect, "{w}x{h}: {t:?}"); + assert!(t.iter().all(|t| t.width() > 0 && t.height() > 0 && t.crop[2] <= w && t.crop[3] <= h), "{w}x{h}: {t:?}"); + } + } + + /// A word in an overlap is seen by two tiles; exactly one keeps it. Every point of the page, + /// on the cuts and on the edges included, belongs to exactly one tile. + #[test] + fn every_word_centre_belongs_to_exactly_one_tile() { + let (w, h) = (2480, 3508); + let t = tiles(w, h, INPUT); + let points = [ + (0.0, 0.0), + (1239.9, 1753.9), + (1240.0, 1754.0), + (1240.0, 0.0), + (1200.0, 1700.0), // in the overlap of all four crops + (w as f32, h as f32), + (-3.0, 5000.0), // a box that runs off the page + ]; + for (x, y) in points { + let owners = t.iter().filter(|tile| tile.owns(x, y)).count(); + assert_eq!(owners, 1, "({x}, {y})"); + } + assert!(t.iter().all(|tile| !tile.owns(f32::NAN, 10.0) && !tile.owns(10.0, f32::INFINITY))); + } + + #[test] + fn crops_rows_of_pixels() { + // 3 × 2 pixels, 2 bytes each. + let px: Vec = (0..12).collect(); + assert_eq!(crop(&px, 3, 2, [1, 0, 3, 2]), Some(vec![2, 3, 4, 5, 8, 9, 10, 11])); + assert_eq!(crop(&px, 3, 2, [0, 1, 1, 2]), Some(vec![6, 7])); + // Outside the image, empty, or more rows than there are: no crop, no panic. + assert_eq!(crop(&px, 3, 2, [2, 0, 4, 1]), None); + assert_eq!(crop(&px, 3, 2, [1, 1, 1, 2]), None); + assert_eq!(crop(&px, 3, 2, [0, 1, 3, 3]), None); + assert_eq!(crop(&px, u32::MAX, usize::MAX, [0, 1, u32::MAX, 2]), None); + } +} diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 045083de08..25688c90a9 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -6707,8 +6707,8 @@ milestone = "M10" status = "shipped" commands = ["ocr.recognize"] tools = ["ocr_recognize", "ocr_status"] -evidence = ["crates/ocr/src/ocrs.rs::model_discovery_prefers_the_complete_onnx_pair", "crates/ocr/src/ocrs.rs::model_discovery_falls_back_to_legacy_when_the_onnx_pair_is_incomplete", "crates/ocr/src/ocrs.rs::model_discovery_refuses_mixed_format_pairs", "crates/ocr/src/ocrs.rs::model_discovery_keeps_legacy_only_installations", "crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable", "crates/automation/tests/it/automation.rs::ocr_tools_make_a_scan_searchable", "crates/ui-egui/tests/it/ocr.rs::recognize_text_dialog_adds_searchable_text"] -notes = "ocrs engine (MIT/Apache) with CC-BY-SA ONNX models fetched and SHA-256 checked by `cargo xtask models`; within each search directory, a complete ONNX pair takes precedence over a complete legacy RTEN pair, incomplete and mixed-format pairs are not combined, and directory precedence is unchanged; the desktop release packages ship them (#103; packaging/env.sh stage_models), and the web zip serves them for download (ocr.ocr-web); runs on a worker thread with progress; one undoable step; pages with text are skipped unless asked (ocr.skip-renderable-text)." +evidence = ["crates/ocr/src/ocrs.rs::model_discovery_prefers_the_complete_onnx_pair", "crates/ocr/src/ocrs.rs::model_discovery_falls_back_to_legacy_when_the_onnx_pair_is_incomplete", "crates/ocr/src/ocrs.rs::model_discovery_refuses_mixed_format_pairs", "crates/ocr/src/ocrs.rs::model_discovery_keeps_legacy_only_installations", "crates/ocr/src/tiles.rs::every_word_centre_belongs_to_exactly_one_tile", "crates/ocr/src/pitch.rs::small_print_is_tiled", "crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable", "crates/automation/tests/it/automation.rs::ocr_tools_make_a_scan_searchable", "crates/ui-egui/tests/it/ocr.rs::recognize_text_dialog_adds_searchable_text"] +notes = "ocrs engine (MIT/Apache) with CC-BY-SA ONNX models fetched and SHA-256 checked by `cargo xtask models`; within each search directory, a complete ONNX pair takes precedence over a complete legacy RTEN pair, incomplete and mixed-format pairs are not combined, and directory precedence is unchanged; the desktop release packages ship them (#103; packaging/env.sh stage_models), and the web zip serves them for download (ocr.ocr-web); runs on a worker thread with progress; one undoable step; pages with text are skipped unless asked (ocr.skip-renderable-text); a page at least twice the detector's 600 x 800 pixel input along an axis (any full page at 300 dpi) is detected in overlapping tiles, 2 x 2 for a full page, so small print on dense pages (newspapers, blurred photos) is not shrunk away, but only when its text is small once shrunk (line pitch under 9 detector pixels, measured from the ink profile per row or per column) or the pixels show no regular line spacing; blank pages and large print are detected whole, and its lines are then read on the whole page (the model-backed test finds_small_print_on_a_large_page runs only where the models are installed, not in CI)." [[feature]] id = "ocr.recognize-range"