From 9121f65564d8bedb9cbffc15c9ea817442abf59c Mon Sep 17 00:00:00 2001 From: bmh129 <11478682+bmh129@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:48:12 -0600 Subject: [PATCH 1/8] M10: OCR models load from bytes; recognised lines travel as JSON (ocr.ocr-web) The web build has no model files: it downloads them. `Ocr::from_bytes` loads the pair only when each file's SHA-256 matches ATTRIBUTION.toml (rten's ONNX parser can spin forever on bytes that aren't a model), and `OcrsRecognizer::from_ocr` wraps the result as an engine that stays available without files. The web build's workers send what they read back as JSON: `lines_to_json` / `lines_from_json` carry each word's box and, when it has one, its confidence; the reader credits the words to the engine the worker ran, and refuses anything else (a missing or non-finite number, too many words) with an error. --- Cargo.lock | 12 ++++ crates/ocr/Cargo.toml | 2 + crates/ocr/src/lib.rs | 139 ++++++++++++++++++++++++++++++++++++++++- crates/ocr/src/ocrs.rs | 42 +++++++++++-- 4 files changed, 190 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6710dddeea..9cfe8c9c00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4540,10 +4540,22 @@ dependencies = [ "pdfcraft-fonts", "png 0.18.1", "rten", + "serde_json", + "sha2 0.11.1", "tempfile", "thiserror 2.0.21", ] +[[package]] +name = "pdfcraft-ocr-worker" +version = "0.18.1" +dependencies = [ + "js-sys", + "pdfcraft-ocr", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "pdfcraft-optimize" version = "0.18.1" diff --git a/crates/ocr/Cargo.toml b/crates/ocr/Cargo.toml index 1b04dbb522..60e66d46c9 100644 --- a/crates/ocr/Cargo.toml +++ b/crates/ocr/Cargo.toml @@ -14,6 +14,8 @@ workspace = true pdfcraft-fonts = { workspace = true } ocrs = { version = "0.13.1", default-features = false, features = ["rten"] } rten = { version = "0.26.0", default-features = false, features = ["onnx_format", "rten_format"] } +serde_json.workspace = true +sha2 = "0.11" thiserror.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] diff --git a/crates/ocr/src/lib.rs b/crates/ocr/src/lib.rs index ff5b920dca..aac95f18c6 100644 --- a/crates/ocr/src/lib.rs +++ b/crates/ocr/src/lib.rs @@ -29,7 +29,9 @@ pub mod tesseract; pub use confidence::{ConfidenceBand, band_for, is_suspect, low_confidence}; pub use merge::{MergeStrategy, StrategyOutcome, mean_confidence, recognize_with_strategy, rover_merge}; -pub use ocrs::{ALPHABET, DETECTION_MODEL, LANGUAGES, Models, Ocr, OcrsRecognizer, RECOGNITION_MODEL}; +pub use ocrs::{ + ALPHABET, DETECTION_MODEL, DETECTION_MODEL_SHA256, LANGUAGES, Models, Ocr, OcrsRecognizer, RECOGNITION_MODEL, RECOGNITION_MODEL_SHA256, +}; pub use pdfcraft_fonts::TextPlan; pub use pdfcraft_fonts::helvetica_width; @@ -464,6 +466,58 @@ pub fn text_layer(words: &[PlacedWord], plan: &pdfcraft_fonts::TextPlan) -> Vec< out } +/// The recognised lines as JSON, `[[["word", left, top, right, bottom], …], …]`, with the +/// word's confidence as a sixth item when it has one: how the web build's OCR workers send what +/// they found back to the page. +pub fn lines_to_json(lines: &[Line]) -> String { + let word = |w: &Word| { + let mut items = vec![serde_json::json!(w.text), w.rect[0].into(), w.rect[1].into(), w.rect[2].into(), w.rect[3].into()]; + items.extend(w.confidence.map(serde_json::Value::from)); + serde_json::Value::Array(items) + }; + serde_json::Value::Array(lines.iter().map(|l| serde_json::Value::Array(l.words.iter().map(word).collect())).collect()).to_string() +} + +/// At most this many words come back for one page (a dense page has a few thousand). +const MAX_JSON_WORDS: usize = 100_000; + +/// [`lines_to_json`] read back, the words credited to `source` (the engine the worker ran). +/// Anything else (a word without its box, a box or confidence that isn't a finite number, too +/// many words) is an error rather than a guess. +pub fn lines_from_json(json: &str, source: &str) -> Result, OcrError> { + let bad = |why: &str| OcrError::Recognize(format!("unreadable result: {why}")); + let value: serde_json::Value = serde_json::from_str(json).map_err(|e| bad(&e.to_string()))?; + let lines = value.as_array().ok_or_else(|| bad("not a list of lines"))?; + let number = |v: &serde_json::Value| v.as_f64().map(|v| v as f32).filter(|v| v.is_finite()); + let mut count = 0usize; + let mut out = Vec::with_capacity(lines.len().min(MAX_JSON_WORDS)); + for line in lines { + let mut words = Vec::new(); + for word in line.as_array().ok_or_else(|| bad("a line is not a list of words"))? { + count += 1; + if count > MAX_JSON_WORDS { + return Err(bad("too many words")); + } + let (text, rect, confidence) = match word.as_array().map(Vec::as_slice) { + Some([serde_json::Value::String(text), l, t, r, b]) => (text, [l, t, r, b], None), + Some([serde_json::Value::String(text), l, t, r, b, c]) => (text, [l, t, r, b], Some(c)), + _ => return Err(bad("a word is not [text, left, top, right, bottom, confidence?]")), + }; + let mut r = [0f32; 4]; + for (slot, v) in r.iter_mut().zip(rect) { + *slot = number(v).ok_or_else(|| bad("a word's box is not finite numbers"))?; + } + let mut w = Word::new(text.clone(), r, source); + if let Some(c) = confidence { + w = w.with_confidence(number(c).ok_or_else(|| bad("a word's confidence is not a finite number"))?); + } + words.push(w); + } + out.push(Line { words }); + } + Ok(out) +} + #[cfg(test)] mod tests { use super::*; @@ -854,6 +908,89 @@ mod tests { assert_eq!(ok.gray(-1, -1), Some(255)); } + /// The web build loads the models from downloaded bytes; they read the same text. + #[test] + fn reads_rendered_text_with_models_from_bytes() { + let Some(models) = Models::find().filter(|m| m.detection.ends_with(DETECTION_MODEL)) else { + eprintln!("skipped: ONNX OCR models not installed"); + return; + }; + let ocr = Ocr::from_bytes(std::fs::read(&models.detection).unwrap(), std::fs::read(&models.recognition).unwrap()).unwrap(); + let rec = OcrsRecognizer::from_ocr(ocr); + assert!(rec.available(), "a recogniser loaded from bytes needs no model files"); + let (w, h, px) = test_image::hello(); + let lines = rec.recognize(&OcrImage::new(w, h, px).unwrap(), &RecognizeOptions::default()).unwrap(); + let text: Vec = lines.iter().map(Line::text).collect(); + assert!(text.iter().any(|t| t.contains("HELL") && t.contains("WOR")), "{text:?}"); + } + + /// The web build's workers send lines back as JSON; they arrive unchanged, credited to the + /// engine the worker ran. + #[test] + fn lines_survive_json() { + let lines = vec![ + Line { + words: vec![ + Word::new("Total \"€250\"", [1.5, 2.0, 30.25, 12.0], "ocrs"), + Word::new("due", [32.0, 2.0, 50.0, 12.0], "ocrs").with_confidence(87.5), + ], + }, + Line::default(), + Line { words: vec![Word::new("日本語", [0.0, 20.0, 9.0, 30.0], "ocrs")] }, + ]; + assert_eq!(lines_from_json(&lines_to_json(&lines), "ocrs").unwrap(), lines); + assert_eq!(lines_from_json(&lines_to_json(&[]), "ocrs").unwrap(), Vec::::new()); + let other = lines_from_json(&lines_to_json(&lines), "tesseract").unwrap(); + assert!(other.iter().flat_map(|l| &l.words).all(|w| w.source == "tesseract")); + } + + /// A result that isn't what a worker sends is an error, not a crash or a guess. + #[test] + fn malformed_json_lines_are_an_error() { + for json in [ + "", + "{}", + "[1]", + "[[1]]", + "[[[\"a\", 1, 2, 3]]]", + "[[[\"a\", 1, 2, 3, 4, 5, 6]]]", + "[[[1, 1, 2, 3, 4]]]", + "[[[\"a\", \"1\", 2, 3, 4]]]", + "[[[\"a\", 1e999, 2, 3, 4]]]", + "[[[\"a\", null, 2, 3, 4]]]", + "[[[\"a\", 1, 2, 3, 4, null]]]", + "[[[\"a\", 1, 2, 3, 4, \"90\"]]]", + ] { + assert!(matches!(lines_from_json(json, "ocrs"), Err(OcrError::Recognize(_))), "{json}"); + } + let many = format!("[[{}]]", vec!["[\"a\",0,0,1,1]"; MAX_JSON_WORDS + 1].join(",")); + assert!(lines_from_json(&many, "ocrs").is_err()); + let deep = format!("{}{}", "[".repeat(10_000), "]".repeat(10_000)); + assert!(lines_from_json(&deep, "ocrs").is_err()); + } + + /// A download that isn't a model (an HTML error page, a truncated file) is an error, not a + /// crash or a hang. + #[test] + fn models_from_bad_bytes_are_an_error() { + for (detection, recognition) in [(Vec::new(), Vec::new()), (b"".to_vec(), b"RTEN".to_vec()), (vec![0xff; 64], vec![0; 64])] { + assert!(matches!(Ocr::from_bytes(detection, recognition), Err(OcrError::Load(..)))); + } + } + + /// The hashes [`Ocr::from_bytes`] checks are the ones `cargo xtask models` verifies. + #[test] + fn model_hashes_match_attribution() { + let manifest = include_str!("../../../ATTRIBUTION.toml"); + for (file, sha256) in [(DETECTION_MODEL, DETECTION_MODEL_SHA256), (RECOGNITION_MODEL, RECOGNITION_MODEL_SHA256)] { + let entry = manifest + .split("[[fetched]]") + .find(|e| e.contains(&format!("file = \"{file}\""))) + .unwrap_or_else(|| panic!("{file} is not in ATTRIBUTION.toml")); + assert!(entry.lines().any(|l| l == format!("sha256 = \"{sha256}\"")), "{file}: SHA-256 differs from ATTRIBUTION.toml"); + } + } + mod test_image { /// "HELLO WORLD" drawn with blocky 5×7 letters, 4 pixels per dot, black on white. pub fn hello() -> (u32, u32, Vec) { diff --git a/crates/ocr/src/ocrs.rs b/crates/ocr/src/ocrs.rs index 7f8feb47c4..94f2abfa18 100644 --- a/crates/ocr/src/ocrs.rs +++ b/crates/ocr/src/ocrs.rs @@ -32,6 +32,15 @@ pub const LANGUAGES: &[(&str, &str)] = &[("en", "English")]; /// The model files, as named in ATTRIBUTION.toml. pub const DETECTION_MODEL: &str = "text-detection.onnx"; pub const RECOGNITION_MODEL: &str = "text-recognition.onnx"; +/// Their SHA-256s, as in ATTRIBUTION.toml. [`Ocr::from_bytes`] loads nothing else: rten's ONNX +/// parser can spin forever on bytes that aren't a model (64 × 0xff does). +pub const DETECTION_MODEL_SHA256: &str = "a917b23dbd9524b465df7e922641b3ff2981623df4ded5a0234004ef2fee7cfe"; +pub const RECOGNITION_MODEL_SHA256: &str = "86c145c2edb96c8caed5b1ebb8f44d706408922211451c309c625157dd6061c5"; + +fn sha256_hex(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + Sha256::digest(bytes).iter().map(|b| format!("{b:02x}")).collect() +} pub const ALPHABET: &str = " 0123456789!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~€ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; @@ -84,9 +93,25 @@ pub struct Ocr { impl Ocr { pub fn load(models: &Models) -> Result { let load = |p: &Path| rten::Model::load_file(p).map_err(|e| OcrError::Load(p.display().to_string(), e.to_string())); + Self::with_models(load(&models.detection)?, load(&models.recognition)?) + } + + /// Load the models from the contents of their files, where there is no file system (the web + /// build downloads them). Bytes that aren't the files ATTRIBUTION.toml names are an error. + pub fn from_bytes(detection: Vec, recognition: Vec) -> Result { + let load = |name: &str, sha256: &str, data: Vec| { + if sha256_hex(&data) != sha256 { + return Err(OcrError::Load(name.into(), "not the expected model file (SHA-256 differs)".into())); + } + rten::Model::load(data).map_err(|e| OcrError::Load(name.into(), e.to_string())) + }; + Self::with_models(load(DETECTION_MODEL, DETECTION_MODEL_SHA256, detection)?, load(RECOGNITION_MODEL, RECOGNITION_MODEL_SHA256, recognition)?) + } + + fn with_models(detection: rten::Model, recognition: rten::Model) -> Result { let params = ocrs::OcrEngineParams { - detection_model: Some(load(&models.detection)?), - recognition_model: Some(load(&models.recognition)?), + detection_model: Some(detection), + recognition_model: Some(recognition), alphabet: Some(ALPHABET.into()), ..Default::default() }; @@ -139,12 +164,21 @@ impl Ocr { /// The ocrs engine as a [`Recognizer`] (the default engine). pub struct OcrsRecognizer { engine: Ocr, + /// Loaded from bytes ([`OcrsRecognizer::from_ocr`]), not from model files: it stays + /// available without them. + in_memory: bool, } impl OcrsRecognizer { /// Load the models from `models`. pub fn load(models: &Models) -> Result { - Ok(OcrsRecognizer { engine: Ocr::load(models)? }) + Ok(OcrsRecognizer { engine: Ocr::load(models)?, in_memory: false }) + } + + /// A recogniser loaded some other way than from model files: the web build downloads the + /// models and loads them with [`Ocr::from_bytes`]. + pub fn from_ocr(engine: Ocr) -> OcrsRecognizer { + OcrsRecognizer { engine, in_memory: true } } /// Load the models found by [`Models::find`]. @@ -159,7 +193,7 @@ impl Recognizer for OcrsRecognizer { } fn available(&self) -> bool { - Models::find().is_some() + self.in_memory || Models::find().is_some() } fn languages(&self) -> Vec<(String, String)> { From 0bfb50a853fa75aebe6814d2ca2de350c3b4a55e Mon Sep 17 00:00:00 2001 From: bmh129 <11478682+bmh129@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:48:12 -0600 Subject: [PATCH 2/8] M10: read OCR jobs a page at a time; install a downloaded engine (ocr.ocr-web) `OcrJob::steps` splits `run` where the engine reads: `OcrSteps::next_page` renders the page, crops it to the region and preprocesses it, an engine reads `OcrShot::image`, and `OcrShot::place` maps the words back through the preprocessing and the crop. `run` is the same loop with `OcrShot::read`, so desktop results don't change. With only one engine, every strategy is that engine's reading, which is what the web build's workers return. `FileJob` splits `recognize_file` the same way (open, read, finish), keeping the skip-files probe on opening. `ocr::install` puts a recogniser loaded from downloaded bytes in the engine cache, so `recognizers`, `available`, `engine_unavailable_reason` and `engines_status` see it. --- crates/engine/src/ocr.rs | 539 ++++++++++++++++++++++++++----------- crates/engine/src/tests.rs | 29 ++ 2 files changed, 409 insertions(+), 159 deletions(-) diff --git a/crates/engine/src/ocr.rs b/crates/engine/src/ocr.rs index c4854d7e9f..2977406bdf 100644 --- a/crates/engine/src/ocr.rs +++ b/crates/engine/src/ocr.rs @@ -15,7 +15,8 @@ use pdfcraft_render::text::TextFilter; use pdfcraft_render::{PageInfo, PageRenderer, RenderConfig, RenderRequest, RequestKind}; pub use pdfcraft_ocr::{ - ConfidenceBand, LANGUAGES, MergeStrategy, Models, Ocr, OcrError, PlacedWord, Recognizer, band_for, is_suspect, low_confidence, + ConfidenceBand, DETECTION_MODEL, LANGUAGES, Line, MergeStrategy, Models, Ocr, OcrError, OcrImage, PlacedWord, RECOGNITION_MODEL, Recognizer, + StrategyOutcome, Word, band_for, is_suspect, lines_from_json, low_confidence, }; use crate::{DocId, Edit, EditError, Session}; @@ -257,6 +258,23 @@ struct EngineCache { ocrs: Option>, } +/// The engines built so far (see [`recognizers`] and [`install`]). +static ENGINES: Mutex> = Mutex::new(None); + +/// Share an ocrs recogniser loaded some other way than from the model files [`Models::find`] +/// looks for: the web build downloads the models and loads them with [`Ocr::from_bytes`]. From +/// then on [`recognizers`] hands it out and the engine counts as available. +pub fn install(ocr: Ocr) -> Arc { + let ocrs: Arc = Arc::new(pdfcraft_ocr::OcrsRecognizer::from_ocr(ocr)); + ENGINES.lock().unwrap_or_else(|e| e.into_inner()).get_or_insert_with(|| EngineCache { ocrs: None }).ocrs = Some(ocrs.clone()); + ocrs +} + +/// Whether an ocrs recogniser has been loaded (from the model files, or by [`install`]). +fn ocrs_loaded() -> bool { + ENGINES.lock().unwrap_or_else(|e| e.into_inner()).as_ref().is_some_and(|c| c.ocrs.is_some()) +} + /// How long a negative engine probe (no program there) is remembered. A short TTL, not /// forever: a tesseract installed while the app runs is found by the next job, without a /// restart. An engine that was found is remembered for the session. @@ -408,10 +426,11 @@ fn models_installed_cached() -> bool { found } -/// The web build has no disk search worth caching. +/// The web build has no model files: the models count as installed once [`install`] has +/// loaded the ones it downloaded. #[cfg(target_arch = "wasm32")] fn models_installed_cached() -> bool { - Models::find().is_some() + ocrs_loaded() } /// The program's `--version` line, remembered per program path for the session (`ocr_status` @@ -473,7 +492,7 @@ pub fn engines_status() -> Vec { /// takes a location, because an agent must not choose which program runs; the tools report the /// default settings.) pub fn engines_status_for(settings: &OcrSettings) -> Vec { - let ocrs = Models::find().is_some(); + let ocrs = ocrs_loaded() || Models::find().is_some(); let mut out = vec![EngineStatus { id: "ocrs", available: ocrs, why_not: (!ocrs).then(models_missing), program: None, version: None }]; #[cfg(not(target_arch = "wasm32"))] { @@ -524,7 +543,6 @@ pub fn recognizers(settings: &OcrSettings) -> Result { let configured = configured_tesseract(settings); let tesseract = if matches!(settings.engine, EngineChoice::Tesseract | EngineChoice::Auto) { tesseract_engine(configured.as_deref()) } else { None }; - static ENGINES: Mutex> = Mutex::new(None); let mut slot = ENGINES.lock().unwrap_or_else(|e| e.into_inner()); let cache = slot.get_or_insert_with(|| EngineCache { ocrs: None }); if cache.ocrs.is_none() { @@ -567,7 +585,7 @@ fn models_missing() -> String { /// never waits. pub fn available() -> bool { let configured = std::env::var_os("PDFCRAFT_TESSERACT").map(|v| v.to_string_lossy().into_owned()); - Models::find().is_some() || tesseract_engine(configured.as_deref()).is_some() + ocrs_loaded() || Models::find().is_some() || tesseract_engine(configured.as_deref()).is_some() } /// Why `engine` cannot run right now, named for the UI's tooltips (the Engine dropdown @@ -703,138 +721,226 @@ impl OcrJob { /// Read the pages. `progress(done, total)` is called before each page; returning `false` /// stops (the pages read so far are returned). pub fn run(self, recognizers: &Recognizers, mut progress: impl FnMut(usize, usize) -> bool) -> Vec { - let config = - RenderConfig { password: self.password.as_deref().map(Arc::from), hide_comments: self.settings.native_text_only, ..Default::default() }; - let mut r = PageRenderer::new(self.bytes.clone(), config); - let mut out = Vec::new(); let total = self.pages.len(); - let options = pdfcraft_ocr::RecognizeOptions { language: self.settings.language.clone() }; - for (done, &page) in self.pages.iter().enumerate() { - if !progress(done, total) { - break; + let mut steps = self.steps(); + let mut out = Vec::new(); + while progress(steps.done(), total) { + match steps.next_page() { + None => break, + Some(Err(skipped)) => out.push(skipped), + Some(Ok(shot)) => out.push(shot.read(recognizers)), } - let Some(info) = self.infos.get(page) else { continue }; - let skip = |why: &str| OcrPage { page, words: Vec::new(), already_on_page: 0, skipped: Some(why.into()), notes: Vec::new() }; - // The boxes of all text on the page, in view space: an earlier OCR layer's, and words - // only /ActualText names, so neither is read again. - let existing: Vec<[f32; 4]> = r - .render(RenderRequest { page, kind: RequestKind::Text, scale: 1.0, ..Default::default() }) - .text - .map(|t| t.glyphs.iter().filter(|g| !g.text.trim().is_empty()).map(|g| g.rect).collect()) - .unwrap_or_default(); - if self.settings.skip_text_pages && !self.settings.force_ocr && !existing.is_empty() && has_own_text(&mut r, page) { - out.push(skip("the page already contains text")); - continue; - } - // A scan is read at its own resolution (enlarging it only blurs the letters), at - // most the chosen one. - let scale = self.render_scale(page); - let shot = r.render(RenderRequest { page, scale, ..Default::default() }); - if let Some(e) = shot.error { - out.push(skip(&e)); - continue; - } - let full = match pdfcraft_ocr::OcrImage::new(shot.width, shot.height, shot.rgba.to_vec()) { - Ok(img) => img, - Err(e) => { - out.push(skip(&e.to_string())); - continue; - } - }; - // The optional region: crop the rendered page, clamped; an empty region skips the - // page with a named reason (nothing was asked of it). - let (image, offset) = match self.settings.region { - None => (full, [0.0f32; 2]), - Some(region) => match clamp_region(region, shot.width, shot.height) { - Some([x0, y0, x1, y1]) => match crop(&full, x0, y0, x1, y1) { - Some(c) => (c, [x0 as f32, y0 as f32]), - None => { - out.push(skip("the selected region is empty")); - continue; - } - }, - None => { - out.push(skip("the selected region is outside the page")); - continue; - } - }, - }; - // Preprocessing sits between the page raster and recognition; its word boxes come - // back in processed coordinates and are mapped to the ORIGINAL raster's before - // placement. The render dpi is the real one (pixels per point × 72), never a - // fixed value. - let pre = pdfcraft_ocr::preprocess::preprocess( - image, - scale * 72.0, - &pdfcraft_ocr::preprocess::PreprocessOptions { - auto_rotate: self.settings.auto_rotate, - deskew: self.settings.deskew, - denoise: self.settings.denoise, - binarize: self.settings.binarize, - }, - ); - let (image, inverse) = match pre { - Ok(p) => (p.image, p.inverse_transform), - Err(e) => { - out.push(skip(&e.to_string())); - continue; - } - }; - let outcome = match pdfcraft_ocr::recognize_with_strategy( - recognizers.primary.as_ref(), - recognizers.secondary.as_deref(), - self.settings.strategy, - &image, - &options, - ) { - Ok(o) => o, - Err(e) => { - out.push(skip(&e.to_string())); - continue; - } - }; - let scale = shot.width as f32 / info.width.max(1e-3); - let to_user = |x: f32, y: f32| { - // processed pixels → original raster pixels (the exact inverse) → past the - // region crop → page view space. - let (ox, oy) = inverse.apply(f64::from(x), f64::from(y)); - let [u, v] = info.view_to_user((ox as f32 + offset[0]) / scale, (oy as f32 + offset[1]) / scale); - [u as f64, v as f64] - }; - // Validate recognition geometry and compare against existing text in view space, - // undoing preprocessing and region offsets before testing overlap. - let (fresh, already): (Vec<_>, Vec<_>) = outcome.lines.iter().flat_map(|l| &l.words).filter(|w| box_writable(w)).partition(|w| { - let [l, t, r, b] = w.rect; - let bounds = [(l, t), (r, t), (r, b), (l, b)].into_iter().fold( - [f32::INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::NEG_INFINITY], - |mut bounds, (x, y)| { - let (x, y) = inverse.apply(f64::from(x), f64::from(y)); - let x = (x as f32 + offset[0]) / scale; - let y = (y as f32 + offset[1]) / scale; - bounds[0] = bounds[0].min(x); - bounds[1] = bounds[1].min(y); - bounds[2] = bounds[2].max(x); - bounds[3] = bounds[3].max(y); - bounds - }, - ); - covered(bounds, &existing) < ALREADY_ON_PAGE - }); - let words = fresh.into_iter().map(|w| PlacedWord::place(w, to_user)).collect::>(); - let mut notes = outcome.notes; - // Only visible editable output needs display glyphs. Searchable output preserves - // every Unicode character with empty Type3 programs and ToUnicode maps. - if self.settings.output_mode == OutputMode::EditableText && !words.is_empty() { - let missing = pdfcraft_fonts::TextPlan::for_words(words.iter().map(|w| w.text.as_str())).missing(); - if missing > 0 { - notes.push(format!("{missing} characters had no font that could show them and appear as ? in the output")); - } - } - out.push(OcrPage { page, words, already_on_page: already.len(), skipped: None, notes }); } progress(total, total); out } + + /// Read the pages one at a time, preparing each page's image and recognising it separately: + /// the web build prepares pages on the page and recognises them in Web Workers. + pub fn steps(self) -> OcrSteps { + let config = + RenderConfig { password: self.password.as_deref().map(Arc::from), hide_comments: self.settings.native_text_only, ..Default::default() }; + let renderer = PageRenderer::new(self.bytes.clone(), config); + OcrSteps { job: self, renderer, next: 0 } + } +} + +/// [`OcrJob`] a page at a time: [`OcrSteps::next_page`] renders the next page, crops it to the +/// region and preprocesses it; an engine reads [`OcrShot::image`], and [`OcrShot::place`] puts +/// the words found on the page ([`OcrShot::read`] does both with the job's engines). +pub struct OcrSteps { + job: OcrJob, + renderer: PageRenderer, + next: usize, +} + +impl OcrSteps { + /// The number of pages to read. + pub fn total(&self) -> usize { + self.job.pages.len() + } + + /// The number of pages prepared (or skipped) so far. + pub fn done(&self) -> usize { + self.next + } + + /// The settings the job reads with. + pub fn settings(&self) -> &OcrSettings { + &self.job.settings + } + + /// Prepare the next page for recognition. `Err` is a page that isn't read (it already has + /// text, or couldn't be rendered or prepared), with the reason; `None` once every page is + /// done. + pub fn next_page(&mut self) -> Option> { + loop { + let page = *self.job.pages.get(self.next)?; + self.next += 1; + let Some(info) = self.job.infos.get(page).cloned() else { continue }; + return Some(self.prepare(page, info)); + } + } + + fn prepare(&mut self, page: usize, info: PageInfo) -> Result { + let settings = &self.job.settings; + let skip = |why: &str| OcrPage { page, words: Vec::new(), already_on_page: 0, skipped: Some(why.into()), notes: Vec::new() }; + let r = &mut self.renderer; + // The boxes of all text on the page, in view space: an earlier OCR layer's, and words + // only /ActualText names, so neither is read again. + let existing: Vec<[f32; 4]> = r + .render(RenderRequest { page, kind: RequestKind::Text, scale: 1.0, ..Default::default() }) + .text + .map(|t| t.glyphs.iter().filter(|g| !g.text.trim().is_empty()).map(|g| g.rect).collect()) + .unwrap_or_default(); + if settings.skip_text_pages && !settings.force_ocr && !existing.is_empty() && has_own_text(r, page) { + return Err(skip("the page already contains text")); + } + // A scan is read at its own resolution (enlarging it only blurs the letters), at + // most the chosen one. + let scale = self.job.render_scale(page); + let shot = r.render(RenderRequest { page, scale, ..Default::default() }); + if let Some(e) = shot.error { + return Err(skip(&e)); + } + let full = pdfcraft_ocr::OcrImage::new(shot.width, shot.height, shot.rgba.to_vec()).map_err(|e| skip(&e.to_string()))?; + // The optional region: crop the rendered page, clamped; an empty region skips the + // page with a named reason (nothing was asked of it). + let (image, offset) = match settings.region { + None => (full, [0.0f32; 2]), + Some(region) => match clamp_region(region, shot.width, shot.height) { + Some([x0, y0, x1, y1]) => match crop(&full, x0, y0, x1, y1) { + Some(c) => (c, [x0 as f32, y0 as f32]), + None => return Err(skip("the selected region is empty")), + }, + None => return Err(skip("the selected region is outside the page")), + }, + }; + // Preprocessing sits between the page raster and recognition; its word boxes come + // back in processed coordinates and are mapped to the ORIGINAL raster's before + // placement. The render dpi is the real one (pixels per point × 72), never a + // fixed value. + let pre = pdfcraft_ocr::preprocess::preprocess( + image, + scale * 72.0, + &pdfcraft_ocr::preprocess::PreprocessOptions { + auto_rotate: settings.auto_rotate, + deskew: settings.deskew, + denoise: settings.denoise, + binarize: settings.binarize, + }, + ) + .map_err(|e| skip(&e.to_string()))?; + Ok(OcrShot { + page, + image: Some(pre.image), + inverse: pre.inverse_transform, + offset, + pixels_per_point: shot.width as f32 / info.width.max(1e-3), + info, + existing, + strategy: settings.strategy, + options: pdfcraft_ocr::RecognizeOptions { language: settings.language.clone() }, + output_mode: settings.output_mode, + }) + } +} + +/// A page prepared for recognition: the image an engine reads, and what placing the words +/// found in it needs. +pub struct OcrShot { + pub page: usize, + /// The page's pixels as the engines read them: rendered, cropped to the region and + /// preprocessed (`None` once [`OcrShot::take_image`] has handed them on). + image: Option, + /// Processed pixels → the rendered raster's (the exact inverse of preprocessing). + inverse: pdfcraft_ocr::preprocess::InverseTransform, + /// Where the region crop starts in the rendered raster. + offset: [f32; 2], + /// The rendered raster's pixels per point of page view space. + pixels_per_point: f32, + info: PageInfo, + /// The boxes of the page's own text, in view space. + existing: Vec<[f32; 4]>, + strategy: MergeStrategy, + options: pdfcraft_ocr::RecognizeOptions, + output_mode: OutputMode, +} + +impl OcrShot { + /// What the engines read (`None` once taken). + pub fn image(&self) -> Option<&OcrImage> { + self.image.as_ref() + } + + /// Hand the image on (the web build sends it to a worker), keeping what placing the words + /// needs: the page's pixels need not stay in memory while the worker reads them. + pub fn take_image(&mut self) -> Option { + self.image.take() + } + + /// Recognise the image with `recognizers` (per the job's strategy) and place the words. + pub fn read(&self, recognizers: &Recognizers) -> OcrPage { + let Some(image) = &self.image else { return self.place(Err("the page's image was handed to another reader".into())) }; + let outcome = pdfcraft_ocr::recognize_with_strategy( + recognizers.primary.as_ref(), + recognizers.secondary.as_deref(), + self.strategy, + image, + &self.options, + ); + self.place(outcome.map_err(|e| e.to_string())) + } + + /// Put the lines recognised in [`Self::image`] on the page (or skip the page with the + /// error). A single engine's reading, which is all the web build's workers run, is + /// `StrategyOutcome { lines, ..Default::default() }`: every strategy reads that way when + /// there is no second engine. + pub fn place(&self, outcome: Result) -> OcrPage { + let page = self.page; + let outcome = match outcome { + Ok(o) => o, + Err(e) => return OcrPage { page, words: Vec::new(), already_on_page: 0, skipped: Some(e), notes: Vec::new() }, + }; + let (info, inverse, offset, scale) = (&self.info, self.inverse, self.offset, self.pixels_per_point); + let to_user = |x: f32, y: f32| { + // processed pixels → original raster pixels (the exact inverse) → past the + // region crop → page view space. + let (ox, oy) = inverse.apply(f64::from(x), f64::from(y)); + let [u, v] = info.view_to_user((ox as f32 + offset[0]) / scale, (oy as f32 + offset[1]) / scale); + [u as f64, v as f64] + }; + // Validate recognition geometry and compare against existing text in view space, + // undoing preprocessing and region offsets before testing overlap. + let (fresh, already): (Vec<_>, Vec<_>) = outcome.lines.iter().flat_map(|l| &l.words).filter(|w| box_writable(w)).partition(|w| { + let [l, t, r, b] = w.rect; + let bounds = [(l, t), (r, t), (r, b), (l, b)].into_iter().fold( + [f32::INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::NEG_INFINITY], + |mut bounds, (x, y)| { + let (x, y) = inverse.apply(f64::from(x), f64::from(y)); + let x = (x as f32 + offset[0]) / scale; + let y = (y as f32 + offset[1]) / scale; + bounds[0] = bounds[0].min(x); + bounds[1] = bounds[1].min(y); + bounds[2] = bounds[2].max(x); + bounds[3] = bounds[3].max(y); + bounds + }, + ); + covered(bounds, &self.existing) < ALREADY_ON_PAGE + }); + let words = fresh.into_iter().map(|w| PlacedWord::place(w, to_user)).collect::>(); + let mut notes = outcome.notes; + // Only visible editable output needs display glyphs. Searchable output preserves + // every Unicode character with empty Type3 programs and ToUnicode maps. + if self.output_mode == OutputMode::EditableText && !words.is_empty() { + let missing = pdfcraft_fonts::TextPlan::for_words(words.iter().map(|w| w.text.as_str())).missing(); + if missing > 0 { + notes.push(format!("{missing} characters had no font that could show them and appear as ? in the output")); + } + } + OcrPage { page, words, already_on_page: already.len(), skipped: None, notes } + } } /// Whether a word's pixel box may reach the content stream: width and height (right − left, @@ -1095,36 +1201,70 @@ pub fn recognize_file( recognizers: &Recognizers, progress: impl FnMut(usize, usize) -> bool, ) -> Result { - let mut s = Session::new(); - let id = s.open(name, None, bytes.clone(), password).map_err(|e| e.to_string())?; - if let Some(why) = s.get(id).and_then(|d| d.read_only_reason.clone()) { - return Err(why); + match FileJob::open(name, bytes, password, settings)? { + Opened::Skipped(result) => Ok(result), + Opened::Read(mut file) => { + let job = file.job.take().ok_or("the document could not be read")?; + let pages = job.run(recognizers, progress); + file.finish(pages) + } } - let job = s.ocr_job(id, &[], settings.clone()).ok_or("the document could not be read")?; - // The skip-files probe runs before any engine work: a file with text needs no engine at - // all, and the original bytes come back unchanged. - if settings.skip_text_files && !settings.force_ocr && job.has_text() { - let pages = job - .pages - .iter() - .map(|&page| OcrPage { - already_on_page: 0, - page, - words: Vec::new(), - skipped: Some("the file already contains text".into()), - notes: Vec::new(), - }) - .collect(); - return Ok(FileResult { bytes, pages, file_skipped: Some("the file already contains text".into()) }); +} + +/// [`recognize_file`] in two halves, for callers that read the pages themselves (the web build +/// reads them in Web Workers): [`FileJob::open`], read `job`, then [`FileJob::finish`]. +pub struct FileJob { + session: Session, + id: DocId, + bytes: Arc>, + /// The pages to read (taken by the caller). + pub job: Option, +} + +/// What [`FileJob::open`] found: a file to read, or one the skip-files option passes over +/// (its result is final, and no engine is needed). +pub enum Opened { + Read(Box), + Skipped(FileResult), +} + +impl FileJob { + pub fn open(name: &str, bytes: Arc>, password: Option<&str>, settings: OcrSettings) -> Result { + let mut session = Session::new(); + let id = session.open(name, None, bytes.clone(), password).map_err(|e| e.to_string())?; + if let Some(why) = session.get(id).and_then(|d| d.read_only_reason.clone()) { + return Err(why); + } + let job = session.ocr_job(id, &[], settings.clone()).ok_or("the document could not be read")?; + // The skip-files probe runs before any engine work: a file with text needs no engine at + // all, and the original bytes come back unchanged. + if settings.skip_text_files && !settings.force_ocr && job.has_text() { + let pages = job + .pages + .iter() + .map(|&page| OcrPage { + already_on_page: 0, + page, + words: Vec::new(), + skipped: Some("the file already contains text".into()), + notes: Vec::new(), + }) + .collect(); + return Ok(Opened::Skipped(FileResult { bytes, pages, file_skipped: Some("the file already contains text".into()) })); + } + Ok(Opened::Read(Box::new(FileJob { session, id, bytes, job: Some(job) }))) } - let pages = job.run(recognizers, progress); - if s.apply_ocr(id, &pages).map_err(|e| e.to_string())? == 0 { - return Ok(FileResult { bytes, pages, file_skipped: None }); + + /// Add the words found and save: the searchable file, or the original when nothing was found. + pub fn finish(mut self, pages: Vec) -> Result { + if self.session.apply_ocr(self.id, &pages).map_err(|e| e.to_string())? == 0 { + return Ok(FileResult { bytes: self.bytes, pages, file_skipped: None }); + } + // Checked like a save before the caller writes it (#2166). + let bytes = self.session.save_bytes(self.id).map_err(|e| e.to_string())?; + self.session.check_save(self.id, &bytes).map_err(|e| e.to_string())?; + Ok(FileResult { bytes, pages, file_skipped: None }) } - // Checked like a save before the caller writes it (#2166). - let bytes = s.save_bytes(id).map_err(|e| e.to_string())?; - s.check_save(id, &bytes).map_err(|e| e.to_string())?; - Ok(FileResult { bytes, pages, file_skipped: None }) } #[cfg(test)] @@ -1639,6 +1779,87 @@ mod tests { assert_eq!(again.pages[0].skipped.as_deref(), Some("the page already contains text")); } + /// The web build prepares pages with [`OcrJob::steps`] and has a worker read each image, + /// its lines coming back as JSON: the pages it ends with are the ones [`OcrJob::run`] gives, + /// with a region crop and preprocessing between the render and the engine. + #[test] + fn reading_page_by_page_matches_a_run() { + let mut s = Session::new(); + let bytes = s.create_from_text("t", "One page\nanother page").unwrap(); + let id = s.open("t.pdf", None, bytes, None).unwrap(); + let settings = OcrSettings { force_ocr: true, deskew: true, binarize: true, region: Some([40.0, 30.0, 900.0, 700.0]), ..Default::default() }; + let (r, fake) = fake_recognizers(); + let run = s.ocr_job(id, &[], settings.clone()).unwrap().run(&r, |_, _| true); + assert!(run.iter().all(|p| p.skipped.is_none() && p.words.len() == 1), "{run:?}"); + let mut steps = s.ocr_job(id, &[], settings).unwrap().steps(); + assert_eq!((steps.done(), steps.total()), (0, run.len())); + let mut stepped = Vec::new(); + while let Some(next) = steps.next_page() { + let shot = next.unwrap(); + let json = pdfcraft_ocr::lines_to_json(&fake.recognize(shot.image().unwrap(), &pdfcraft_ocr::RecognizeOptions::default()).unwrap()); + let lines = lines_from_json(&json, "fake").map_err(|e| e.to_string()); + stepped.push(shot.place(lines.map(|lines| StrategyOutcome { lines, ..Default::default() }))); + } + assert_eq!(steps.done(), run.len()); + assert_eq!(stepped, run); + // An engine's failure skips the page with the reason, as in a run. + let mut steps = s.ocr_job(id, &[0], OcrSettings { force_ocr: true, ..Default::default() }).unwrap().steps(); + let mut shot = steps.next_page().unwrap().unwrap(); + let page = shot.place(Err("worker gone".into())); + assert_eq!(page.skipped.as_deref(), Some("worker gone")); + // Once the image is handed on, it can't be read here as well. + assert!(shot.take_image().is_some() && shot.image().is_none()); + assert!(shot.read(&r).skipped.is_some()); + assert!(steps.next_page().is_none()); + } + + /// A page the job doesn't read comes out of [`OcrSteps::next_page`] as skipped, with its + /// reason, before any engine is involved. + #[test] + fn page_by_page_skips_text_pages_without_an_engine() { + let mut s = Session::new(); + let bytes = s.create_from_text("t", "A page with text").unwrap(); + let id = s.open("t.pdf", None, bytes, None).unwrap(); + let mut steps = s.ocr_job(id, &[], OcrSettings::default()).unwrap().steps(); + let Some(Err(page)) = steps.next_page() else { panic!("the text page should be skipped") }; + assert_eq!(page.skipped.as_deref(), Some("the page already contains text")); + } + + /// The web build has no model files: a recogniser loaded from downloaded bytes is installed + /// and from then on is the engine every job reads with. + #[test] + fn an_installed_recognizer_is_available_and_handed_out() { + let Some(models) = Models::find().filter(|m| m.detection.ends_with(DETECTION_MODEL)) else { + eprintln!("skipped: ONNX OCR models not installed"); + return; + }; + let ocr = Ocr::from_bytes(std::fs::read(&models.detection).unwrap(), std::fs::read(&models.recognition).unwrap()).unwrap(); + let installed = install(ocr); + assert!(installed.available()); + assert!(available()); + let r = recognizers(&OcrSettings::default()).unwrap(); + assert!(Arc::ptr_eq(&r.primary, &installed), "the installed recogniser is the one handed out"); + assert!(engines_status().iter().any(|e| e.id == "ocrs" && e.available)); + } + + /// The batch's halves: a file to read opens with its job, and [`FileJob::finish`] saves + /// the searchable copy; a file the skip-files option passes over is final on opening. + #[test] + fn a_file_job_reads_in_two_halves() { + let s = Session::new(); + let bytes = s.create_from_text("t", "Already typed here").unwrap(); + let (r, fake) = fake_recognizers(); + let skip = OcrSettings { skip_text_files: true, ..Default::default() }; + let Ok(Opened::Skipped(out)) = FileJob::open("t.pdf", bytes.clone(), None, skip) else { panic!("the file should be skipped") }; + assert_eq!(out.file_skipped.as_deref(), Some("the file already contains text")); + let force = OcrSettings { force_ocr: true, ..Default::default() }; + let Ok(Opened::Read(mut file)) = FileJob::open("t.pdf", bytes.clone(), None, force.clone()) else { panic!("the file should be read") }; + let pages = file.job.take().unwrap().run(&r, |_, _| true); + let out = file.finish(pages).unwrap(); + assert_eq!((fake.calls(), out.words(), out.file_skipped.clone()), (1, 1, None)); + assert_eq!(out, recognize_file("t.pdf", bytes, None, force, &r, |_, _| true).unwrap(), "the same as reading it in one call"); + } + /// Two inputs of one name (from different folders, or differing only in case) would /// write the same output: both are found, each once. #[test] diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index d230d098de..41912df9c5 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -5681,6 +5681,35 @@ fn backgrounds_and_watermarks_from_files() { assert!(s.apply(id, Edit::AddBackground { pages: vec![0], settings: Background::default(), replace: true, file: Some(junk) }).is_err()); } +/// The web build reads a page at a time (prepared on the page, recognised in a worker, the +/// lines sent back as JSON); with the real ocrs engine that finds the same words as reading in +/// one go (needs the models: `cargo xtask models`; skipped without them). +#[test] +fn reading_a_page_at_a_time_finds_the_same_words() { + let settings = ocr::OcrSettings::default(); + let Ok(engines) = ocr::recognizers(&settings) else { + eprintln!("skipped: OCR models not installed"); + return; + }; + let mut s = Session::new().with_clock(|| 1_700_000_000); + let text = s.create_from_text("t", "Invoice 42: pay by Friday").unwrap(); + let typed = s.open("text.pdf", None, text, None).unwrap(); + let png = export::Exporter::new(s.get(typed).unwrap()).png(0, 150.0).unwrap(); + let scan = s.create_from_images(&[("scan.png".into(), png.clone()), ("scan2.png".into(), png)]).unwrap(); + let id = s.open("scan.pdf", None, scan, None).unwrap(); + let at_once = s.ocr_job(id, &[], settings.clone()).unwrap().run(&engines, |_, _| true); + let mut steps = s.ocr_job(id, &[], settings).unwrap().steps(); + let mut one_by_one = Vec::new(); + while let Some(next) = steps.next_page() { + let shot = next.unwrap(); + let lines = engines.primary.recognize(shot.image().unwrap(), &pdfcraft_ocr::RecognizeOptions::default()).unwrap(); + let sent = ocr::lines_from_json(&pdfcraft_ocr::lines_to_json(&lines), engines.primary.id()).map_err(|e| e.to_string()); + one_by_one.push(shot.place(sent.map(|lines| ocr::StrategyOutcome { lines, ..Default::default() }))); + } + assert_eq!(one_by_one, at_once); + assert!(at_once.len() == 2 && at_once.iter().all(|p| p.skipped.is_none() && !p.words.is_empty()), "{at_once:?}"); +} + /// Scan & OCR ▸ Recognize text on a page that is only a picture of text (needs the models: /// `cargo xtask models`; skipped without them). #[test] From a10906e99101cd25eebdac69c05ff9ae5344ef64 Mon Sep 17 00:00:00 2001 From: bmh129 <11478682+bmh129@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:48:12 -0600 Subject: [PATCH 3/8] M10: Recognize Text in the browser, read in Web Workers (ocr.ocr-web) The web build downloads the OCR models the first time Recognize Text (or Edit on a scanned page) needs them, installs the engine, and reads pages in up to two Web Workers (apps/pdfcraft-ocr-worker) built with WebAssembly SIMD, so the page keeps responding. The page prepares each page and sends its image to an idle worker, which runs ocrs and sends the lines back. Where no worker can start, the page reads with the installed engine. Answers go back to the run that sent the page (ocr_route), so a single run and a batch can overlap. Recognize Text in Multiple Files on the web keeps main's skipped-files bucket, low-confidence notes and cancel between files. Files chosen before the models are in wait for them. A scan Edit finds while no engine can read it waits for the models instead of counting as no scan. Trunk builds the worker beside the app; `cargo xtask web-worker-link` hashes its files and names it in index.html. The web zip ships the models in models/. --- .cargo/config.toml | 5 + apps/pdfcraft-ocr-worker/Cargo.toml | 18 + apps/pdfcraft-ocr-worker/src/main.rs | 101 +++++ apps/pdfcraft-web/Trunk.toml | 7 + apps/pdfcraft-web/index.html | 2 + crates/ui-egui/Cargo.toml | 2 +- crates/ui-egui/src/app.rs | 4 + crates/ui-egui/src/canvas.rs | 5 + crates/ui-egui/src/lib.rs | 11 + crates/ui-egui/src/workflows.rs | 4 + crates/ui-egui/src/workflows/ocr_route.rs | 139 ++++++ crates/ui-egui/src/workflows/ocr_ui.rs | 270 ++++++++++- crates/ui-egui/src/workflows/ocr_web.rs | 521 ++++++++++++++++++++++ packaging/web/package.sh | 9 +- xtask/src/layers.rs | 3 +- xtask/src/main.rs | 6 + xtask/src/web.rs | 145 ++++++ 17 files changed, 1224 insertions(+), 28 deletions(-) create mode 100644 apps/pdfcraft-ocr-worker/Cargo.toml create mode 100644 apps/pdfcraft-ocr-worker/src/main.rs create mode 100644 crates/ui-egui/src/workflows/ocr_route.rs create mode 100644 crates/ui-egui/src/workflows/ocr_web.rs create mode 100644 xtask/src/web.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index 5592118f6c..a1caaeaa38 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,7 @@ [alias] xtask = "run -p xtask --" + +# WebAssembly SIMD (every current browser has it; there's no runtime detection on wasm). The OCR +# engine's kernels (rten-simd) and other vectorised code only use it when compiled in. +[target.wasm32-unknown-unknown] +rustflags = ["-C", "target-feature=+simd128"] diff --git a/apps/pdfcraft-ocr-worker/Cargo.toml b/apps/pdfcraft-ocr-worker/Cargo.toml new file mode 100644 index 0000000000..7360c13440 --- /dev/null +++ b/apps/pdfcraft-ocr-worker/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "pdfcraft-ocr-worker" +description = "PdfCraft in the browser: the Web Worker that recognises text (wasm32; trunk builds it with pdfcraft-web)" +version.workspace = true +edition.workspace = true +license.workspace = true +publish = false + +[lints] +workspace = true + +[dependencies] +pdfcraft-ocr = { workspace = true } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +wasm-bindgen = "0.2" +js-sys = "0.3" +web-sys = { version = "0.3", features = ["DedicatedWorkerGlobalScope", "MessageEvent"] } diff --git a/apps/pdfcraft-ocr-worker/src/main.rs b/apps/pdfcraft-ocr-worker/src/main.rs new file mode 100644 index 0000000000..a6a41e24eb --- /dev/null +++ b/apps/pdfcraft-ocr-worker/src/main.rs @@ -0,0 +1,101 @@ +//! PdfCraft in the browser: the Web Worker that recognises text, so reading a scanned page doesn't +//! stop the page responding and several pages are read at once. Trunk builds it alongside +//! `pdfcraft-web` (see apps/pdfcraft-web/index.html); the page starts and feeds the workers +//! (crates/ui-egui/src/workflows/ocr_ui.rs). +//! +//! Messages are plain objects with a `type`: +//! +//! - `ready` (to the page): the worker is listening; send the models now. +//! - `models` (from the page): `{ detection, recognition }`, each a `Uint8Array` holding a model +//! file. Answered with `loaded`, or `failed` with an `error`. +//! - `page` (from the page): `{ id, width, height, pixels }`, the RGBA pixels of a rendered page +//! as a `Uint8Array`. Answered with `lines` `{ id, lines }`, the recognised lines as +//! `pdfcraft_ocr::lines_to_json` writes them, or `failed` `{ id, error }`. + +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +#[cfg(target_arch = "wasm32")] +fn main() { + use wasm_bindgen::{JsCast, closure::Closure}; + + let Ok(scope) = js_sys::global().dyn_into::() else { + return; + }; + let reply_to = scope.clone(); + let mut ocr = None; + let on_message = Closure::::new(move |e: web_sys::MessageEvent| { + let _ = reply_to.post_message(&worker::handle(&mut ocr, &e.data())); + }); + scope.set_onmessage(Some(on_message.as_ref().unchecked_ref())); + // The handler lives as long as the worker. + on_message.forget(); + let _ = scope.post_message(&worker::message(&[("type", "ready".into())])); +} + +#[cfg(not(target_arch = "wasm32"))] +fn main() { + eprintln!("pdfcraft-ocr-worker runs in a browser: trunk builds it with pdfcraft-web"); +} + +#[cfg(target_arch = "wasm32")] +mod worker { + use pdfcraft_ocr::Ocr; + use wasm_bindgen::{JsCast, JsValue}; + + /// The models are about 12 MB together. + const MAX_MODEL_BYTES: u32 = 64 << 20; + /// The renderer keeps pages far smaller; anything larger isn't a page. + const MAX_PIXEL_BYTES: u64 = 1 << 30; + + pub fn message(fields: &[(&str, JsValue)]) -> JsValue { + let o = js_sys::Object::new(); + for (k, v) in fields { + let _ = js_sys::Reflect::set(&o, &(*k).into(), v); + } + o.into() + } + + /// Answer one message from the page. + pub fn handle(ocr: &mut Option, data: &JsValue) -> JsValue { + let get = |k: &str| js_sys::Reflect::get(data, &k.into()).unwrap_or(JsValue::UNDEFINED); + let id = get("id"); + let reply = match get("type").as_string().as_deref() { + Some("models") => load(&get("detection"), &get("recognition")).map(|o| { + *ocr = Some(o); + message(&[("type", "loaded".into())]) + }), + Some("page") => match ocr.as_ref() { + None => Err("the text recognition models aren't loaded".into()), + Some(o) => recognize(o, &get("width"), &get("height"), &get("pixels")) + .map(|lines| message(&[("type", "lines".into()), ("id", id.clone()), ("lines", lines.into())])), + }, + _ => Err("unknown message".into()), + }; + reply.unwrap_or_else(|error| message(&[("type", "failed".into()), ("id", id), ("error", error.into())])) + } + + fn bytes(v: &JsValue, max: u64) -> Result, String> { + let a = v.dyn_ref::().ok_or("expected a Uint8Array")?; + if u64::from(a.length()) > max { + return Err(format!("{} bytes is too large", a.length())); + } + Ok(a.to_vec()) + } + + fn load(detection: &JsValue, recognition: &JsValue) -> Result { + let max = u64::from(MAX_MODEL_BYTES); + Ocr::from_bytes(bytes(detection, max)?, bytes(recognition, max)?).map_err(|e| e.to_string()) + } + + fn recognize(ocr: &Ocr, width: &JsValue, height: &JsValue, pixels: &JsValue) -> Result { + let side = |v: &JsValue| v.as_f64().filter(|v| v.fract() == 0.0 && (1.0..=f64::from(u32::MAX)).contains(v)).map(|v| v as u32); + let (Some(w), Some(h)) = (side(width), side(height)) else { + return Err("the page size is not a positive whole number".into()); + }; + let px = bytes(pixels, MAX_PIXEL_BYTES)?; + if px.len() as u64 != u64::from(w) * u64::from(h) * 4 { + return Err("the pixels don't match the page size".into()); + } + ocr.recognize(&px, w, h).map(|lines| pdfcraft_ocr::lines_to_json(&lines)).map_err(|e| e.to_string()) + } +} diff --git a/apps/pdfcraft-web/Trunk.toml b/apps/pdfcraft-web/Trunk.toml index 563c2267c1..f03ccc6ea1 100644 --- a/apps/pdfcraft-web/Trunk.toml +++ b/apps/pdfcraft-web/Trunk.toml @@ -9,3 +9,10 @@ filehash = true [serve] addresses = ["127.0.0.1"] port = 8766 + +# Trunk leaves the OCR worker's files unhashed and unnamed in index.html: hash them and name the +# worker for the page (xtask/src/web.rs). +[[hooks]] +stage = "post_build" +command = "cargo" +command_arguments = ["xtask", "web-worker-link"] diff --git a/apps/pdfcraft-web/index.html b/apps/pdfcraft-web/index.html index 46b3a25e21..f93bb0812d 100644 --- a/apps/pdfcraft-web/index.html +++ b/apps/pdfcraft-web/index.html @@ -5,6 +5,8 @@ PdfCraft + +