diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml new file mode 100644 index 00000000..19156ec0 --- /dev/null +++ b/.github/workflows/doc.yml @@ -0,0 +1,34 @@ +name: Craft documents + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/doc/**', 'crates/pcraft-format/**', 'crates/vectorcraft-format/**', 'crates/ecproj-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/doc.yml'] + pull_request: + paths: ['crates/doc/**', 'crates/pcraft-format/**', 'crates/vectorcraft-format/**', 'crates/ecproj-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/doc.yml'] + +permissions: + contents: read + +jobs: + doc: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + with: + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - run: cargo fmt --check + - run: cargo clippy -p craft-doc --all-targets --locked -- -D warnings + - run: cargo test -p craft-doc --locked -- --nocapture + - run: cargo check -p craft-doc --all-features --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/ecproj-format.yml b/.github/workflows/ecproj-format.yml new file mode 100644 index 00000000..bd01cbfa --- /dev/null +++ b/.github/workflows/ecproj-format.yml @@ -0,0 +1,34 @@ +name: EffectCraft project format + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/ecproj-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/ecproj-format.yml'] + pull_request: + paths: ['crates/ecproj-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/ecproj-format.yml'] + +permissions: + contents: read + +jobs: + ecproj-format: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + with: + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - run: cargo fmt --check + - run: cargo clippy -p craft-ecproj-format --all-targets --locked -- -D warnings + - run: cargo test -p craft-ecproj-format --locked -- --nocapture + - run: cargo check -p craft-ecproj-format --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/font-store.yml b/.github/workflows/font-store.yml new file mode 100644 index 00000000..5a9d2d73 --- /dev/null +++ b/.github/workflows/font-store.yml @@ -0,0 +1,34 @@ +name: Font store + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/font-store/**', 'crates/font-catalog/**', 'crates/atomic/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/font-store.yml'] + pull_request: + paths: ['crates/font-store/**', 'crates/font-catalog/**', 'crates/atomic/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/font-store.yml'] + +permissions: + contents: read + +jobs: + font-store: + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + with: + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - run: cargo fmt --check + - run: cargo clippy -p craft-font-store --all-targets --locked -- -D warnings + - run: cargo test -p craft-font-store --locked + - name: Check the web build + if: runner.os == 'Linux' + run: cargo check -p craft-font-store --target wasm32-unknown-unknown --locked diff --git a/.github/workflows/media.yml b/.github/workflows/media.yml index 5d519b0a..940431f2 100644 --- a/.github/workflows/media.yml +++ b/.github/workflows/media.yml @@ -3,9 +3,9 @@ name: Shared media on: push: branches: [main] - paths: ["crates/media/**", "Cargo.toml", "Cargo.lock", ".github/workflows/media.yml"] + paths: ["crates/media/**", "crates/media-testkit/**", "Cargo.toml", "Cargo.lock", ".github/workflows/media.yml"] pull_request: - paths: ["crates/media/**", "Cargo.toml", "Cargo.lock", ".github/workflows/media.yml"] + paths: ["crates/media/**", "crates/media-testkit/**", "Cargo.toml", "Cargo.lock", ".github/workflows/media.yml"] workflow_dispatch: permissions: @@ -37,6 +37,8 @@ jobs: - run: cargo clippy -p craft-media --all-targets --locked -- -D warnings - run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features matroska -- -D warnings - run: cargo clippy -p craft-media --all-targets --locked --all-features -- -D warnings + - run: cargo clippy -p craft-media-testkit --all-targets --locked -- -D warnings + - run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features serde -- -D warnings - run: cargo check -p craft-media --all-features --locked --target wasm32-unknown-unknown test: timeout-minutes: 15 diff --git a/.github/workflows/pcraft-format.yml b/.github/workflows/pcraft-format.yml new file mode 100644 index 00000000..e79cebc2 --- /dev/null +++ b/.github/workflows/pcraft-format.yml @@ -0,0 +1,35 @@ +name: PhotoCraft format + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/pcraft-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/pcraft-format.yml'] + pull_request: + paths: ['crates/pcraft-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/pcraft-format.yml'] + +permissions: + contents: read + +jobs: + pcraft-format: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + with: + toolchain: "1.95" + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - run: cargo fmt --check + - run: cargo clippy -p craft-pcraft-format --all-targets --locked -- -D warnings + - run: cargo test -p craft-pcraft-format --locked -- --nocapture + - run: cargo check -p craft-pcraft-format --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/vectorcraft-format.yml b/.github/workflows/vectorcraft-format.yml new file mode 100644 index 00000000..55049d1b --- /dev/null +++ b/.github/workflows/vectorcraft-format.yml @@ -0,0 +1,35 @@ +name: VectorCraft format + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/vectorcraft-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/vectorcraft-format.yml'] + pull_request: + paths: ['crates/vectorcraft-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/vectorcraft-format.yml'] + +permissions: + contents: read + +jobs: + vectorcraft-format: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + with: + toolchain: "1.95" + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - run: cargo fmt --check + - run: cargo clippy -p craft-vectorcraft-format --all-targets --locked -- -D warnings + - run: cargo test -p craft-vectorcraft-format --locked -- --nocapture + - run: cargo check -p craft-vectorcraft-format --locked --target wasm32-unknown-unknown diff --git a/Cargo.lock b/Cargo.lock index 6f73792f..1af6fd17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -798,10 +798,33 @@ dependencies = [ "rfd", ] +[[package]] +name = "craft-doc" +version = "0.1.0" +dependencies = [ + "craft-ecproj-format", + "craft-pcraft-format", + "craft-vectorcraft-format", + "flate2", + "proptest", + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "craft-docpath" version = "0.1.0" +[[package]] +name = "craft-ecproj-format" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "craft-font-catalog" version = "0.1.0" @@ -837,11 +860,15 @@ dependencies = [ "craft-atomic", "craft-font-catalog", "flate2", + "log", + "ruzstd", "serde", "serde_json", "sha2", "skrifa 0.47.0", + "tar", "thiserror 2.0.21", + "toml", "ureq", "write-fonts 0.54.1", ] @@ -914,11 +941,17 @@ dependencies = [ name = "craft-media" version = "0.1.0" dependencies = [ + "craft-media-testkit", "proptest", + "serde", "serde_json", "thiserror 2.0.21", ] +[[package]] +name = "craft-media-testkit" +version = "0.1.0" + [[package]] name = "craft-menu" version = "0.1.0" @@ -931,6 +964,15 @@ dependencies = [ "objc2-app-kit 0.3.2", ] +[[package]] +name = "craft-pcraft-format" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "craft-pdf" version = "0.1.0" @@ -1104,6 +1146,16 @@ dependencies = [ "zip", ] +[[package]] +name = "craft-vectorcraft-format" +version = "0.1.0" +dependencies = [ + "flate2", + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "craft-window" version = "0.1.0" @@ -4657,6 +4709,12 @@ dependencies = [ "unicode-script", ] +[[package]] +name = "ruzstd" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e4ce4a039f42bf25920b95667e4d789e98749ab11bbb8d2c2936f788421bb5" + [[package]] name = "ryu" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 3946fd50..24e75777 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,9 @@ craft-atomic = { path = "crates/atomic" } craft-cms = { path = "crates/cms" } craft-crw = { path = "crates/crw" } craft-dialogs = { path = "crates/dialogs" } +craft-doc = { path = "crates/doc" } craft-docpath = { path = "crates/docpath" } +craft-ecproj-format = { path = "crates/ecproj-format" } craft-font-catalog = { path = "crates/font-catalog" } craft-font-manager = { path = "crates/font-manager" } craft-font-store = { path = "crates/font-store" } @@ -36,7 +38,9 @@ craft-i18n = { path = "crates/i18n" } craft-jpeg = { path = "crates/jpeg" } craft-keymap = { path = "crates/keymap" } craft-media = { path = "crates/media" } +craft-media-testkit = { path = "crates/media-testkit" } craft-menu = { path = "crates/menu" } +craft-pcraft-format = { path = "crates/pcraft-format" } craft-prepress = { path = "crates/prepress" } craft-print = { path = "crates/print" } craft-proof = { path = "crates/proof" } @@ -45,6 +49,7 @@ craft-settings = { path = "crates/settings" } craft-text = { path = "crates/text" } craft-ui = { path = "crates/ui" } craft-update = { path = "crates/update" } +craft-vectorcraft-format = { path = "crates/vectorcraft-format" } craft-window = { path = "crates/window" } egui = { version = "0.37", default-features = false } rayon = "1" diff --git a/README.md b/README.md index 7c1db128..7cf4ff91 100644 --- a/README.md +++ b/README.md @@ -42,9 +42,15 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " = { git = "https://github.com/storytold/craft-libs", rev = "` with the document as a type parameter, `TileRef`, the entry paths, the + manifest decode limits and the migration runner. The document model stays in PhotoCraft. - [`craft-print`](crates/print/README.md): PDF submission to CUPS and Windows, printer discovery, duplex/copies/colour and driver preferences. PDF generation stays in each app. - [`craft-proof`](crates/proof/README.md): proofing per language (spelling, suggestions, grammar, @@ -96,6 +105,9 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " Self { - Self { max_samples: 1 << 23 } + Self { max_samples: 1 << 23, max_stream_end_fill: 64 } } } @@ -179,6 +183,9 @@ pub struct Image { pub black: BlackLevel, /// White level: the saturation plateau, the sensor's clip level, or the full range. pub white: f32, + /// Samples filled after the entropy stream ended, at their predictor values (the stream-end + /// rule); at most [`Limits::max_stream_end_fill`]. Zero on every real file measured. + pub stream_end_fill: usize, } /// Result alias for this crate. @@ -368,13 +375,18 @@ fn parse(bytes: &[u8], limits: &Limits, full: bool) -> Result { let has_low = header_after_plane || (plane_fits && head.windows(2).any(|p| p[0] == 0xff && p[1] != 0x00)); let bits = if has_low { 12 } else { 10 }; let low_len = if has_low { q } else { 0 }; + let mut stream_end_fill = 0; let samples = if full { let entropy = payload.get(low_len.saturating_add(ent_off)..).ok_or_else(|| corrupt("payload shorter than its header"))?; let low = has_low.then(|| payload.get(..q)).flatten(); if has_low && low.is_none() { return Err(corrupt("payload shorter than its low-bits plane")); } - let hi = decode_entropy(entropy, w, h, &tables)?; + let (hi, tail) = decode_entropy(entropy, w, h, &tables)?; + if tail > limits.max_stream_end_fill { + return Err(Error::Corrupt(format!("entropy stream ends {tail} samples before the frame does"))); + } + stream_end_fill = tail; let mut samples = vec![0u16; n]; // `samples` is documented as `bits` deep, so a damaged stream whose chain value runs past // the range is clamped here: a consumer indexing a `1 << bits` LUT must not see more. @@ -429,7 +441,7 @@ fn parse(bytes: &[u8], limits: &Limits, full: bool) -> Result { exif_orientation, wb_multipliers: wb_from_color_balance(bytes, &rec, o), }; - Ok(Image { header, samples, black, white }) + Ok(Image { header, samples, black, white, stream_end_fill }) } /// As-shot white-balance multipliers from the ColorBalance record (0x10a9, D60/10D/300D): a @@ -645,8 +657,9 @@ fn build_lut(codes: &[(u16, u8, u8, u8)]) -> Vec { /// model (`clean-room/02-crw-format.md` §2): 64-sample blocks, first/main code tables, /// (run, size) leaves with JPEG-rule magnitudes, EOB, block-DC carry, and two parity DPCM chains /// initialised to 512 and reset at every row start. A stream that ends early leaves the remaining -/// samples at their chain values (the frozen stream-end rule). -fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables) -> Result> { +/// samples at their chain values (the frozen stream-end rule); the second value returned is how +/// many, so [`decode`] can refuse a stream cut short ([`Limits::max_stream_end_fill`]). +fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables) -> Result<(Vec, usize)> { let d = unstuff(entropy); let nbits = d.len().saturating_mul(8); let peek16 = |pos: usize| -> u16 { @@ -661,12 +674,15 @@ fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables let mut out = Chains { dec: vec![0u16; n], w, qi: 0, ce: 512, co: 512 }; let mut carry: i32 = 0; let mut pos = 0usize; + // samples filled after the stream ended (the stream-end rule) + let mut tail = 0usize; while out.qi < n { let block_end = n.min((out.qi / 64 + 1) * 64); let mut tok = 0u32; while out.qi < block_end { // stream exhausted: every remaining sample keeps its chain value if pos >= nbits.saturating_sub(1) { + tail = n - out.qi; out.fill(n); break; } @@ -676,6 +692,7 @@ fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables } let (len, mut rr, s) = ((e >> 16) as usize, ((e >> 8) & 0xff) as usize, e & 0xff); if pos.saturating_add(len).saturating_add(s as usize) > nbits { + tail = n - out.qi; out.fill(n); break; } @@ -713,7 +730,7 @@ fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables } } } - Ok(out.dec) + Ok((out.dec, tail)) } /// Drop the stuffing byte: an 0x00 after an 0xff. diff --git a/crates/crw/src/tests.rs b/crates/crw/src/tests.rs index a858cef9..428c70fc 100644 --- a/crates/crw/src/tests.rs +++ b/crates/crw/src/tests.rs @@ -266,7 +266,7 @@ fn header_reads_geometry_without_samples() { fn limits_cap_the_sample_count() { let hi = test_hi(128); let f = roundtrip_file(16, 8, "Canon EOS D60", None, &hi, Order::Le); - let mut limits = Limits { max_samples: 127 }; + let mut limits = Limits { max_samples: 127, ..Limits::default() }; assert_eq!(decode(&f, &limits), Err(Error::LimitExceeded("image too large"))); assert!(header(&f, &limits).is_err()); limits.max_samples = 128; @@ -365,7 +365,7 @@ fn black_level_ignores_clamped_rows() { } /// The frozen stream-end rule: a truncated entropy stream leaves the remaining samples at -/// their chain values instead of failing the file. +/// their chain values instead of failing the file, when the limit allows that many. #[test] fn truncated_stream_keeps_chain_values() { let (w, h) = (16u16, 8u16); @@ -378,9 +378,117 @@ fn truncated_stream_keeps_chain_values() { Order::Le, ); let f = ciff(&[(RAW_DATA, payload(None, &entropy[..cut])), (0x300a, props)], Order::Le); - let img = full(&f).expect("truncated stream still decodes"); + let lenient = Limits { max_stream_end_fill: 128, ..Limits::default() }; + let img = decode(&f, &lenient).expect("truncated stream still decodes"); assert_eq!(img.samples.len(), 128); assert_eq!(&img.samples[..32], &hi[..32], "the leading samples decode exactly"); + assert!(img.stream_end_fill > 0, "the samples after the cut are filled"); + let tail_start = img.samples.len() - img.stream_end_fill; + for i in tail_start..img.samples.len() { + let expected = if i % (w as usize) < 2 { 512 } else { img.samples[i - 2] }; + assert_eq!(img.samples[i], expected, "filled sample at {i} keeps its predictor value"); + } +} + +/// A stream cut short decoded as a smeared image: the samples after its end take their chain +/// values. Past one block that is `Corrupt` by default; a complete stream fills none, and a fill +/// within the limit is accepted. +#[test] +fn a_stream_cut_short_is_corrupt() { + let (w, h) = (64u16, 32u16); + let hi = test_hi(2048); + let tables = tables::tables(0).expect("selector 0"); + let entropy = encode_hi(&hi, 64, &tables); + let file = |entropy: &[u8], with_trailer: bool| { + let mut p = Vec::with_capacity(514 + entropy.len() + 2); + p.extend_from_slice(&[0; 514]); + p.extend_from_slice(entropy); + if with_trailer { + p.extend_from_slice(&[0, 0]); + } + let props = heap( + &[ + (SENSOR_INFO, sensor_info(w, h, None, Order::Le)), + (DECODER_TABLE, decoder_table(0, Order::Le)), + (MAKE_MODEL, make_model("Canon EOS D60")), + ], + Order::Le, + ); + ciff(&[(RAW_DATA, p), (0x300a, props)], Order::Le) + }; + let whole = full(&file(&entropy, true)).expect("a complete stream decodes"); + assert_eq!(whole.stream_end_fill, 0); + let cut = file(&entropy[..entropy.len() / 2], false); + let Err(Error::Corrupt(why)) = full(&cut) else { panic!("a stream cut in half must be refused") }; + assert!(why.contains("samples before the frame does"), "{why}"); + let filled = decode(&cut, &Limits { max_stream_end_fill: usize::MAX, ..Limits::default() }).expect("lenient").stream_end_fill; + assert_eq!(filled, 1024, "half the 2048-sample frame is 1024 filled samples"); + let at_limit = Limits { max_stream_end_fill: 1024, ..Limits::default() }; + assert_eq!(decode(&cut, &at_limit).map(|i| i.stream_end_fill), Ok(1024), "a fill at the limit is accepted"); +} + +/// Truncated entropy streams at 1 sample, 64 samples (at the default limit), 65 samples (past +/// the default limit, returning Corrupt), and a complete file; and verifying the limit can be raised +/// by callers wanting partial recovery. +#[test] +fn truncated_stream_fill_limits_and_recovery() { + let (w, h) = (16u16, 8u16); + let n = (w * h) as usize; // 128 + // Construct hi where every sample difference has magnitude 16 (size 5), encoding to exactly 8 bits + // (1 byte) per sample across both first and main tables. + let mut hi = Vec::with_capacity(n); + let (mut ce, mut co) = (512i32, 512i32); + for qi in 0..n { + if qi % w as usize == 0 { + ce = 512; + co = 512; + } + let c = if qi % 2 == 0 { &mut ce } else { &mut co }; + *c += 16; + hi.push(*c as u16); + } + let tables = tables::tables(0).expect("selector 0"); + let entropy = encode_hi(&hi, w as usize, &tables); + assert_eq!(entropy.len(), n, "each sample encodes to exactly 1 byte"); + let file = |entropy_bytes: &[u8]| { + let mut p = Vec::with_capacity(514 + entropy_bytes.len()); + p.extend_from_slice(&[0; 514]); + p.extend_from_slice(entropy_bytes); + let props = heap( + &[ + (SENSOR_INFO, sensor_info(w, h, None, Order::Le)), + (DECODER_TABLE, decoder_table(0, Order::Le)), + (MAKE_MODEL, make_model("Canon EOS D60")), + ], + Order::Le, + ); + ciff(&[(RAW_DATA, p), (0x300a, props)], Order::Le) + }; + + // 1. Complete file: fills 0 samples, decodes cleanly with default limits + let complete = file(&entropy); + let img = full(&complete).expect("complete file decodes"); + assert_eq!(img.stream_end_fill, 0); + + // 2. Truncated 1 sample before end: fills 1 sample, accepted under default limit (64) + let cut_1 = file(&entropy[..n - 1]); + let img_1 = full(&cut_1).expect("1-sample truncation accepted by default limits"); + assert_eq!(img_1.stream_end_fill, 1); + + // 3. Truncated 64 samples before end: fills 64 samples, accepted at the boundary of default limit (64) + let cut_64 = file(&entropy[..n - 64]); + let img_64 = full(&cut_64).expect("64-sample truncation accepted at boundary by default limits"); + assert_eq!(img_64.stream_end_fill, 64); + + // 4. Truncated 65 samples before end: fills 65 samples, refused by default limit (64) as Corrupt + let cut_65 = file(&entropy[..n - 65]); + let err_65 = full(&cut_65).expect_err("65-sample truncation must be refused by default limits"); + assert_eq!(err_65, Error::Corrupt("entropy stream ends 65 samples before the frame does".into())); + + // 5. Check the limit can be raised by callers who want partial recovery + let lenient = Limits { max_stream_end_fill: 65, ..Limits::default() }; + let img_65_recovered = decode(&cut_65, &lenient).expect("decodes when limit is raised"); + assert_eq!(img_65_recovered.stream_end_fill, 65); } /// Every record the decoder needs missing, bogus selectors, dimensions and offsets: errors, diff --git a/crates/doc/Cargo.toml b/crates/doc/Cargo.toml new file mode 100644 index 00000000..4cd92ee1 --- /dev/null +++ b/crates/doc/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "craft-doc" +description = "Read-only access to PhotoCraft, VectorCraft and EffectCraft documents: format detection, version, embedded previews and PDF, and the layer and page outline." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +craft-ecproj-format = { workspace = true } +craft-pcraft-format = { workspace = true } +craft-vectorcraft-format = { workspace = true } +flate2 = { version = "1", default-features = false, features = ["rust_backend"] } +serde = { workspace = true } +# `unbounded_depth`: PhotoCraft manifests nest deeper than serde_json's 128 levels; the crate +# checks the nesting against `Limits::max_depth` before parsing. +serde_json = { workspace = true, features = ["unbounded_depth"] } +thiserror = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/doc/README.md b/crates/doc/README.md new file mode 100644 index 00000000..20d29ec5 --- /dev/null +++ b/crates/doc/README.md @@ -0,0 +1,113 @@ +# craft-doc + +Reads PhotoCraft `.pcraft`, VectorCraft `.vectorcraft` and EffectCraft `.ecproj` documents +without the apps: which app wrote a file, its format version, the previews and PDF it carries, +and its pages and layers. It takes bytes and returns plain data. It does not open files, render +or composite. No `unsafe`, no panics on malformed input, and it builds for +`wasm32-unknown-unknown`. Each format's envelope is read through its crate in this repository: +[`craft-pcraft-format`](../pcraft-format/README.md) (the `Manifest` members and checks, and +`check_manifest`), +[`craft-vectorcraft-format`](../vectorcraft-format/README.md) (`Head`, `Image`, `sniff`, the +version and the marker) and [`craft-ecproj-format`](../ecproj-format/README.md) (`Head`, the +schema check). The outlines are read with craft-doc's own read-only views of the documents. The +other dependencies are `serde`, `serde_json`, `thiserror` and `flate2` (its pure-Rust backend). + +```toml +craft-doc = { git = "https://github.com/storytold/craft-libs", rev = "" } +``` + +## Use + +```rust +use craft_doc::{Kind, Limits, RenditionKind}; + +let limits = Limits::default(); +match craft_doc::sniff(&bytes) { + Some(Kind::VectorCraft) => { + let pdf = craft_doc::renditions(&bytes, &limits)?.into_iter().find(|r| r.kind == RenditionKind::Pdf); + } + Some(kind) => { /* ... */ } + None => { /* not a Craft document */ } +} +let outline = craft_doc::outline(&bytes, &limits)?; +``` + +- `sniff(head) -> Option` reads the first bytes. The first 64 KiB are enough for every file + the apps write. A re-zipped `.pcraft` whose `manifest.json` entry starts after the head is + recognized only when `head` holds the whole file. +- `info(bytes, &Limits) -> Result`: the kind, the version the file declares, the + writer it names, and `supported`, which is `false` when the version is newer than the crate + reads. +- `renditions(bytes, &Limits) -> Result, Error>`: each `Rendition` has a `kind` + (`Pdf`, `Thumbnail`, `Composite`), a `mime` and the `bytes`. +- `outline(bytes, &Limits) -> Result`: `canvas`, `pages` and `layers`. +- `Error` is `Unrecognized`, `Malformed`, `TooNew`, `Unsupported` or `LimitExceeded`. A file + whose version is newer than the crate reads gives `TooNew` from `renditions` and `outline`. + +## What each format gives + +| | `.vectorcraft` | `.pcraft` | `.ecproj` | +|---|---|---|---| +| Container | JSON, plain or gzip; since version 4, large members after the JSON (`\n\0VCBLOBS\0`) | ZIP of STORE or DEFLATE entries | JSON | +| Version read | `version` up to 4 (`"format": "vectorcraft"` or `"drawcraft"`) | `format_version` 1 | `schema` up to 2 | +| Writer | `generator` | `generator` | `savedBy` | +| Renditions | `preview` (Thumbnail) and `pdf` (Pdf), when the file was saved with them | the `thumbnail` and `composite` the manifest names | none | +| Pages | artboards, in points | none | compositions by item id, in pixels, with frame rate and duration | +| Layers | top-level layers | the layer tree with groups | each composition's layers | + +Layers are listed bottom first in every format. A layer's `kind` is the name the file uses: +`layer` in VectorCraft; `raster`, `group`, `adjustment`, `fill`, `text`, `shape` or `smart` in +PhotoCraft; the source type in EffectCraft (`Footage`, `Comp`, `Solid`, `Text` and others). +EffectCraft layers have no `opacity`, because opacity is an animated property there. + +A `.pcraft` bundle saved as a folder, ZIP64 archives and encrypted entries are not read. The +`.ecprojx` XML copies are not read. + +## Limits + +`Limits` (fields `max_json`, `max_entry`, `max_total`, `max_depth`) bounds each call: + +- `max_json` (256 MiB): the JSON read, which is the VectorCraft file's JSON, the PhotoCraft + manifest or the EffectCraft project. The names and other strings returned are parts of it. +- `max_entry` (256 MiB): one rendition or one ZIP entry. The size a file states is checked before + any of it is copied or inflated. +- `max_total` (1 GiB): every byte one call decompresses from gzip, reads out of ZIP entries or + returns as a rendition, together. +- `max_depth` (400, also the largest value it takes, `MAX_DEPTH`): the nesting of JSON arrays and + objects, checked before parsing. PhotoCraft nests up to 100 groups of layers, three levels each. + +A PhotoCraft manifest with more than 4,194,304 JSON items +(`craft_pcraft_format::MAX_MANIFEST_ITEMS`) is refused with `Error::LimitExceeded`, as PhotoCraft +reads it. + +On a compressed `.vectorcraft` file, `info` and `outline` decompress only as far as the end of +the JSON. `renditions` checks the size of each member after the JSON against `max_entry` +before it decompresses any of that data, then decompresses only as far as the members end. The +struct is `#[non_exhaustive]`: start from the defaults and set the fields you +change. + +```rust +let mut limits = craft_doc::Limits::default(); +limits.max_entry = 32 << 20; +``` + +## Testing + +`cargo test -p craft-doc` runs: + +- `tests/formats.rs`: documents of each format written by the tests (JSON by hand, PNG and ZIP + bytes assembled in the test, gzip through `flate2`), read back through each call: plain and + gzip VectorCraft files, members inline as base64 and after the JSON, `.pcraft` bundles with + STORE and DEFLATE entries and the manifest last, `.ecproj` compositions, newer versions, and + files `sniff` must not claim (PDF, PNG, other ZIP and JSON files). +- `tests/hostile.rs`: every truncation of each sample, seeded mutations, proptest on arbitrary + bytes, and inputs that pass each limit: a gzip bomb, a DEFLATE entry larger than its stated + size, members longer than `max_entry`, JSON nested past `max_depth`, ZIP64, encryption, other + compression methods and CRC mismatches. +- `tests/envelopes.rs`: files whose envelope is written with the format crates' types read + through each call, and envelopes those crates refuse (a manifest without `generator`, a preview + that names another entry, a `format` member inside another member) are refused. A preview or + PDF member without `mime` gives no rendition. + +`.github/workflows/doc.yml` runs fmt, clippy and the tests on Linux and Windows, and a +`wasm32-unknown-unknown` check. diff --git a/crates/doc/src/ecproj.rs b/crates/doc/src/ecproj.rs new file mode 100644 index 00000000..03c437fa --- /dev/null +++ b/crates/doc/src/ecproj.rs @@ -0,0 +1,221 @@ +//! EffectCraft `.ecproj` projects: one UTF-8 JSON object, uncompressed: +//! +//! ```text +//! { "savedBy": "0.12.0", "schema": 2, "settings": {...}, +//! "items": { "1": { "id": 1, "name": "Comp 1", "kind": { "type": "Comp", "width": 1920, ... } } }, +//! ... } +//! ``` +//! +//! Projects saved before `savedBy` existed start with `schema`. A project holds no rendered +//! frames: footage is referenced by path. `savedBy` and `schema` are read with +//! [`craft_ecproj_format::Head`]. + +use std::collections::BTreeMap; + +use craft_ecproj_format::{Head, ReadError}; +use serde::Deserialize; + +use crate::{Budget, Error, Kind, Layer, Outline, Page, Rendition, Timing, check_depth, counted_map, counted_option, malformed, parse, too_new}; + +/// EffectCraft's time unit. +const TICKS_PER_SECOND: f64 = 254_016_000_000.0; + +/// Calls `stop` with each member name of the JSON object `text` starts with, in order, as far as +/// `text` goes, until it returns `true`. Members of nested objects are skipped. A name is given as +/// written, escapes included. +fn top_level_keys<'a>(text: &'a [u8], mut stop: impl FnMut(&'a [u8]) -> bool) { + let Some(body) = text.trim_ascii_start().strip_prefix(b"{") else { return }; + let (mut depth, mut want_key, mut i) = (0u32, true, 0); + while let Some(&b) = body.get(i) { + match b { + b'"' => { + let start = i + 1; + let mut end = start; + loop { + match body.get(end) { + None => return, + Some(b'\\') => end += 2, + Some(b'"') => break, + Some(_) => end += 1, + } + } + if depth == 0 + && want_key + && let Some(key) = body.get(start..end) + && stop(key) + { + return; + } + want_key = false; + i = end; + } + b'{' | b'[' => depth += 1, + b'}' | b']' if depth == 0 => return, + b'}' | b']' => depth -= 1, + b',' if depth == 0 => want_key = true, + _ => {} + } + i += 1; + } +} + +/// A project starts with `savedBy` followed by `schema`, or, saved before `savedBy` existed, with +/// `schema` followed by `items` or `savedBy`, all top-level members. +pub(crate) fn sniff(head: &[u8]) -> bool { + let (mut first, mut found) = (None::<&[u8]>, false); + top_level_keys(head, |key| match first { + None => { + first = Some(key); + !matches!(key, b"savedBy" | b"schema") + } + Some(b"savedBy") => { + found = key == b"schema"; + found + } + Some(_) => { + found = key == b"items" || key == b"savedBy"; + found + } + }); + found +} + +fn json<'a>(bytes: &'a [u8], budget: &Budget) -> Result<&'a [u8], Error> { + budget.check_json(bytes.len() as u64, "the project")?; + Ok(bytes) +} + +/// The project's [`Head`] read with [`Head::read`]; [`ReadError::TooNew`] when its schema is +/// newer than this crate reads. +fn read_head(json: &[u8], budget: &Budget) -> Result, Error> { + check_depth(json, budget.limits.max_depth)?; + let text = std::str::from_utf8(json).map_err(|e| malformed(format!("the project is not UTF-8: {e}")))?; + Ok(Head::read(text)) +} + +/// The project's [`Head`], refused when its schema is newer than this crate reads. +fn head(json: &[u8], budget: &Budget) -> Result { + read_head(json, budget)?.map_err(|e| match e { + ReadError::TooNew { too_new: e, .. } => too_new(Kind::EffectCraft, e.schema), + e => malformed(e), + }) +} + +pub(crate) fn info(bytes: &[u8], budget: &mut Budget) -> Result<(u32, Option), Error> { + match read_head(json(bytes, budget)?, budget)? { + Ok(head) => Ok((head.schema, head.saved_by)), + Err(ReadError::TooNew { too_new, saved_by, .. }) => Ok((too_new.schema, saved_by)), + Err(e) => Err(malformed(e)), + } +} + +pub(crate) fn renditions(bytes: &[u8], budget: &mut Budget) -> Result, Error> { + head(json(bytes, budget)?, budget)?; + Ok(vec![]) +} + +#[derive(Deserialize)] +struct Project { + #[serde(default, deserialize_with = "counted_map")] + items: BTreeMap, +} + +#[derive(Deserialize)] +struct ItemM { + id: u64, + #[serde(default)] + name: String, + kind: ItemKindM, +} + +/// The members of an item's kind that a composition has. Footage and solids share some of them +/// with the same types; folders have none. +#[derive(Deserialize)] +struct ItemKindM { + #[serde(rename = "type")] + kind: String, + #[serde(default)] + width: Option, + #[serde(default)] + height: Option, + #[serde(default)] + frame_rate: Option, + /// In ticks. + #[serde(default)] + duration: Option, + /// A composition's layers, the top one first. + #[serde(default, deserialize_with = "counted_option")] + layers: Option>, +} + +#[derive(Deserialize)] +struct FrameRateM { + num: i64, + den: i64, +} + +fn yes() -> bool { + true +} + +#[derive(Deserialize)] +struct LayerM { + #[serde(default)] + name: String, + #[serde(default)] + source: Option, + #[serde(default)] + switches: Option, + #[serde(default)] + blend_mode: Option, +} + +#[derive(Deserialize)] +struct SourceM { + #[serde(rename = "type")] + kind: String, +} + +#[derive(Deserialize)] +struct SwitchesM { + #[serde(default = "yes")] + video: bool, +} + +pub(crate) fn outline(bytes: &[u8], budget: &mut Budget) -> Result { + let json = json(bytes, budget)?; + head(json, budget)?; + let p: Project = parse(json, budget.limits)?; + let mut comps: Vec = p.items.into_values().filter(|i| i.kind.kind == "Comp").collect(); + comps.sort_by_key(|i| i.id); + let pages = comps + .into_iter() + .map(|i| { + let k = i.kind; + let (Some(width), Some(height)) = (k.width, k.height) else { + return Err(malformed(format!("composition `{}` has no size", i.name))); + }; + let timing = k.frame_rate.zip(k.duration).map(|(r, duration)| Timing { + frame_rate_num: r.num, + frame_rate_den: r.den, + duration_seconds: duration as f64 / TICKS_PER_SECOND, + }); + let layers = k + .layers + .unwrap_or_default() + .into_iter() + .rev() + .map(|l| Layer { + name: l.name, + kind: l.source.map(|s| s.kind).unwrap_or_default(), + visible: l.switches.is_none_or(|s| s.video), + opacity: None, + blend: l.blend_mode, + children: vec![], + }) + .collect(); + Ok(Page { name: i.name, x: 0.0, y: 0.0, width: f64::from(width), height: f64::from(height), timing, layers }) + }) + .collect::>()?; + Ok(Outline { canvas: None, pages, layers: vec![] }) +} diff --git a/crates/doc/src/lib.rs b/crates/doc/src/lib.rs new file mode 100644 index 00000000..5d27cf83 --- /dev/null +++ b/crates/doc/src/lib.rs @@ -0,0 +1,536 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! Read-only access to the Craft apps' native documents, without the apps: PhotoCraft +//! `.pcraft`, VectorCraft `.vectorcraft` (and `.drawcraft`) and EffectCraft `.ecproj`. +//! +//! - [`sniff`] tells which app wrote a file, from its first bytes. +//! - [`info`] reads the format version and the writer the file names. +//! - [`renditions`] returns the previews and the PDF a file carries. +//! - [`outline`] returns the canvas, the pages (artboards or compositions) and the layers. +//! +//! Bytes go in and plain data comes out: no file system access, no rendering and no app crate. +//! Each format's envelope is read through its craft-libs crate: `craft-pcraft-format`, +//! `craft-vectorcraft-format` and `craft-ecproj-format`. +//! A file whose format version is newer than this crate reads gives [`Info::supported`] `false` +//! from [`info`] and [`Error::TooNew`] from the other calls. +//! +//! Nothing panics on input. Every length a file states is checked against [`Limits`] before +//! anything is allocated for it, and every decompressed byte is counted against them. +//! +//! ``` +//! let file = br#"{"format":"vectorcraft","version":4,"generator":"VectorCraft 0.1.0","document":{"artboards":[],"layers":[]}}"#; +//! assert_eq!(craft_doc::sniff(file), Some(craft_doc::Kind::VectorCraft)); +//! let info = craft_doc::info(file, &craft_doc::Limits::default()).unwrap(); +//! assert_eq!(info.version, 4); +//! assert!(info.supported); +//! ``` + +mod ecproj; +mod pcraft; +mod vectorcraft; +mod zip; + +use std::cell::Cell; +use std::collections::BTreeMap; +use std::fmt; +use std::marker::PhantomData; + +use serde::Deserialize; +use serde::de::{self, DeserializeOwned, DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor}; + +/// The app that wrote a document. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Kind { + /// A PhotoCraft `.pcraft` bundle (a ZIP archive). + PhotoCraft, + /// A VectorCraft `.vectorcraft` document (JSON, plain or gzip), or one saved under the + /// earlier name `.drawcraft`. + VectorCraft, + /// An EffectCraft `.ecproj` project (JSON). + EffectCraft, +} + +impl Kind { + /// The file extension the app saves with, without the dot. + pub fn extension(self) -> &'static str { + match self { + Kind::PhotoCraft => "pcraft", + Kind::VectorCraft => "vectorcraft", + Kind::EffectCraft => "ecproj", + } + } + + /// The newest format version this crate reads: PhotoCraft's `format_version`, VectorCraft's + /// `version` or EffectCraft's `schema`. + pub fn supported_version(self) -> u32 { + match self { + Kind::PhotoCraft => craft_pcraft_format::FORMAT_VERSION, + Kind::VectorCraft => craft_vectorcraft_format::VERSION, + Kind::EffectCraft => craft_ecproj_format::SCHEMA_VERSION, + } + } +} + +/// What a document says about itself. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct Info { + pub kind: Kind, + /// The format version the file declares. + pub version: u32, + /// The writer, as the file names it: VectorCraft's `generator` (`"VectorCraft 0.13.1"`), + /// PhotoCraft's `generator` (`"photocraft-format 0.1.0"`) or EffectCraft's `savedBy` (the + /// app version, `"0.12.0"`). `None` when the file has none. + pub generator: Option, + /// `false` when [`Info::version`] is newer than [`Kind::supported_version`]; the other calls + /// then return [`Error::TooNew`]. + pub supported: bool, +} + +/// Why a document could not be read. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// [`sniff`] does not recognize the bytes as a Craft document. + #[error("not a PhotoCraft, VectorCraft or EffectCraft document")] + Unrecognized, + /// The file is damaged or does not follow its format. + #[error("malformed document: {0}")] + Malformed(String), + /// The file's format version is newer than this crate reads. + #[error("{} version {version} is newer than this reader supports ({supported})", kind.extension())] + TooNew { kind: Kind, version: u32, supported: u32 }, + /// A valid container feature this reader does not implement (ZIP64, encryption, a ZIP + /// compression method other than STORE or DEFLATE). + #[error("{0}")] + Unsupported(String), + /// Reading the file would exceed the caller's [`Limits`]. + #[error("limit exceeded: {0}")] + LimitExceeded(String), +} + +/// Bounds on the work and memory one call may use. Each is checked before the bytes it covers +/// are allocated. +/// +/// ``` +/// let mut limits = craft_doc::Limits::default(); +/// limits.max_entry = 16 << 20; +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct Limits { + /// Most bytes of JSON read: the VectorCraft file's JSON, the PhotoCraft manifest or the + /// EffectCraft project. The names and other strings returned are parts of it, so this also + /// bounds them. Default 256 MiB. + pub max_json: u64, + /// Most bytes of one rendition or one ZIP entry. Default 256 MiB. + pub max_entry: u64, + /// Most bytes one call decompresses from gzip, reads out of ZIP entries and returns as + /// renditions, together. Default 1 GiB. + pub max_total: u64, + /// Deepest nesting of JSON arrays and objects, at most [`MAX_DEPTH`] (a larger value counts + /// as [`MAX_DEPTH`]). Default [`MAX_DEPTH`]. + pub max_depth: u32, + /// Most list elements one call reads, together: EffectCraft project items and composition + /// layers, VectorCraft artboards and layers, and PhotoCraft layers at every level. Each is + /// counted before it is read. Default 1,048,576. + pub max_items: u64, +} + +/// The deepest JSON nesting any call reads: 400 levels, which holds PhotoCraft's 100 nested +/// groups of layers at three levels each. +pub const MAX_DEPTH: u32 = 400; + +impl Default for Limits { + fn default() -> Self { + Limits { max_json: 256 << 20, max_entry: 256 << 20, max_total: 1 << 30, max_depth: MAX_DEPTH, max_items: 1 << 20 } + } +} + +/// What a rendition shows. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum RenditionKind { + /// A PDF of the document (VectorCraft's `pdf`, saved with Create PDF-Compatible File). + Pdf, + /// A small preview image: VectorCraft's `preview` (the first artboard) or PhotoCraft's + /// `thumbnail`. + Thumbnail, + /// PhotoCraft's flattened `composite` preview. + Composite, +} + +/// An image or PDF the document carries. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct Rendition { + pub kind: RenditionKind, + /// The MIME type: the one a VectorCraft file gives (a member without one gives no + /// rendition), or `image/png` for PhotoCraft's previews. + pub mime: String, + pub bytes: Vec, +} + +/// The structure of a document, without its content. +#[derive(Debug, Clone, PartialEq, Default)] +#[non_exhaustive] +pub struct Outline { + /// PhotoCraft's canvas; `None` for the other formats. + pub canvas: Option, + /// VectorCraft's artboards (in points) or EffectCraft's compositions (in pixels), in file + /// order; empty for PhotoCraft. + pub pages: Vec, + /// VectorCraft's top-level layers or PhotoCraft's layer tree, bottom first; empty for + /// EffectCraft, whose layers are in each [`Page`]. + pub layers: Vec, +} + +/// A PhotoCraft canvas. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub struct Canvas { + pub width: u32, + pub height: u32, + pub resolution_dpi: f32, + /// The color mode as the manifest writes it: `Rgb`, `Cmyk`, `Grayscale`, `Lab`, `Indexed`, + /// `Bitmap`, `Duotone` or `Multichannel`. + pub mode: String, + /// The sample type as the manifest writes it: `U8`, `U16` or `F32`. + pub depth: String, +} + +/// A VectorCraft artboard or an EffectCraft composition. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub struct Page { + pub name: String, + /// The left edge (0 for a composition). + pub x: f64, + /// The top edge (0 for a composition). + pub y: f64, + pub width: f64, + pub height: f64, + /// A composition's frame rate and duration; `None` for an artboard, and for a composition + /// that gives either one without the other. + pub timing: Option, + /// A composition's layers, bottom first; empty for an artboard. + pub layers: Vec, +} + +/// A composition's timing. +#[derive(Debug, Clone, Copy, PartialEq)] +#[non_exhaustive] +pub struct Timing { + /// Frames per second as `frame_rate_num / frame_rate_den`. + pub frame_rate_num: i64, + pub frame_rate_den: i64, + pub duration_seconds: f64, +} + +/// One layer. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub struct Layer { + pub name: String, + /// What the layer holds, as the file names it: `layer` for VectorCraft; `raster`, `group`, + /// `adjustment`, `fill`, `text`, `shape` or `smart` for PhotoCraft; the source type for + /// EffectCraft (`Footage`, `Comp`, `Solid`, `Text`, `Shape`, `Null`, `Camera`, `Light`, + /// `Model`, `Primitive`). + pub kind: String, + pub visible: bool, + /// Opacity from 0 to 1; `None` for EffectCraft, where opacity is an animated property. + pub opacity: Option, + /// The blend mode as the file names it (`Normal`, `Multiply`…). + pub blend: Option, + /// A PhotoCraft group's layers, bottom first. + pub children: Vec, +} + +/// Which app wrote the document whose first bytes are `head`, or `None`. The first 64 KiB are +/// enough for every file the apps write. A re-zipped `.pcraft` whose `manifest.json` entry +/// starts past `head` is recognized only when `head` holds the whole file. +pub fn sniff(head: &[u8]) -> Option { + if pcraft::sniff(head) { + Some(Kind::PhotoCraft) + } else if vectorcraft::sniff(head) { + Some(Kind::VectorCraft) + } else if ecproj::sniff(head) { + Some(Kind::EffectCraft) + } else { + None + } +} + +/// The document's kind, version and writer. A version newer than this crate reads is not an +/// error here: [`Info::supported`] is `false`. +pub fn info(bytes: &[u8], limits: &Limits) -> Result { + let mut budget = Budget::new(limits); + let kind = sniff(bytes).ok_or(Error::Unrecognized)?; + let (version, generator) = match kind { + Kind::PhotoCraft => pcraft::info(bytes, &mut budget)?, + Kind::VectorCraft => vectorcraft::info(bytes, &mut budget)?, + Kind::EffectCraft => ecproj::info(bytes, &mut budget)?, + }; + Ok(Info { kind, version, generator, supported: version <= kind.supported_version() }) +} + +/// The previews and the PDF the document carries, the thumbnail first. A +/// `.vectorcraft` file has a `preview` only when saved with one and a `pdf` only when saved with +/// Create PDF-Compatible File; a `.pcraft` bundle has the `thumbnail` and `composite` its +/// manifest names; an `.ecproj` project has none. +pub fn renditions(bytes: &[u8], limits: &Limits) -> Result, Error> { + let mut budget = Budget::new(limits); + match sniff(bytes).ok_or(Error::Unrecognized)? { + Kind::PhotoCraft => pcraft::renditions(bytes, &mut budget), + Kind::VectorCraft => vectorcraft::renditions(bytes, &mut budget), + Kind::EffectCraft => ecproj::renditions(bytes, &mut budget), + } +} + +/// The document's canvas, pages and layers (see [`Outline`]). +pub fn outline(bytes: &[u8], limits: &Limits) -> Result { + let mut budget = Budget::new(limits); + match sniff(bytes).ok_or(Error::Unrecognized)? { + Kind::PhotoCraft => pcraft::outline(bytes, &mut budget), + Kind::VectorCraft => vectorcraft::outline(bytes, &mut budget), + Kind::EffectCraft => ecproj::outline(bytes, &mut budget), + } +} + +fn malformed(m: impl std::fmt::Display) -> Error { + Error::Malformed(m.to_string()) +} + +/// The bytes one call may still decompress, read out of ZIP entries or return as renditions. +pub(crate) struct Budget<'a> { + pub(crate) limits: &'a Limits, + left: u64, +} + +impl<'a> Budget<'a> { + fn new(limits: &'a Limits) -> Self { + Budget { limits, left: limits.max_total } + } + + /// Bytes still allowed. + pub(crate) fn left(&self) -> u64 { + self.left + } + + /// Count `n` bytes of `what` against the total. + pub(crate) fn take(&mut self, n: u64, what: &str) -> Result<(), Error> { + self.left = self + .left + .checked_sub(n) + .ok_or_else(|| Error::LimitExceeded(format!("{what}: more than max_total ({} bytes) in all", self.limits.max_total)))?; + Ok(()) + } + + /// Check one rendition or ZIP entry of `n` bytes against `max_entry`, then count it. + pub(crate) fn take_entry(&mut self, n: u64, what: &str) -> Result<(), Error> { + if n > self.limits.max_entry { + return Err(Error::LimitExceeded(format!("{what} is {n} bytes, more than max_entry ({})", self.limits.max_entry))); + } + self.take(n, what) + } + + /// Check `n` bytes of JSON against `max_json`. + pub(crate) fn check_json(&self, n: u64, what: &str) -> Result<(), Error> { + if n > self.limits.max_json { + return Err(Error::LimitExceeded(format!("{what} is more than max_json ({} bytes)", self.limits.max_json))); + } + Ok(()) + } +} + +/// Is the JSON text nested at most `max` levels deep (arrays and objects), and at most +/// [`MAX_DEPTH`]? +pub(crate) fn check_depth(json: &[u8], max: u32) -> Result<(), Error> { + let max = max.min(MAX_DEPTH); + let (mut depth, mut in_str, mut escaped) = (0u32, false, false); + for &b in json { + if in_str { + match b { + _ if escaped => escaped = false, + b'\\' => escaped = true, + b'"' => in_str = false, + _ => {} + } + continue; + } + match b { + b'"' => in_str = true, + b'[' | b'{' => { + depth += 1; + if depth > max { + return Err(Error::LimitExceeded(format!("JSON nested deeper than max_depth ({max} levels)"))); + } + } + b']' | b'}' => depth = depth.saturating_sub(1), + _ => {} + } + } + Ok(()) +} + +thread_local! { + /// The elements [`counted`] lists may still read in the current [`parse`], and whether a + /// list went past them. + static ITEMS: Cell<(u64, bool)> = const { Cell::new((u64::MAX, false)) }; +} + +/// Run `f` with `max` elements for [`counted`] lists; when `f` fails after a list went past +/// them, the error is [`Error::LimitExceeded`]. +fn counting(max: u64, f: impl FnOnce() -> Result) -> Result { + let saved = ITEMS.with(|c| c.replace((max, false))); + let r = f(); + let (_, over) = ITEMS.with(|c| c.replace(saved)); + match r { + Err(_) if over => Err(Error::LimitExceeded(format!("the file lists more than max_items ({max}) items, layers and pages"))), + r => r, + } +} + +/// Count one list element against [`Limits::max_items`]. +fn take_item() -> Result<(), E> { + ITEMS.with(|c| match c.get().0.checked_sub(1) { + Some(left) => { + c.set((left, false)); + Ok(()) + } + None => { + c.set((0, true)); + Err(E::custom("more than max_items list elements")) + } + }) +} + +/// A `T` read after one element is counted with [`take_item`]. +struct Item(PhantomData); + +impl<'de, T: Deserialize<'de>> DeserializeSeed<'de> for Item { + type Value = T; + + fn deserialize>(self, d: D) -> Result { + take_item()?; + T::deserialize(d) + } +} + +/// A list read by [`parse`] (as `#[serde(deserialize_with = "counted")]`), each element counted +/// against [`Limits::max_items`] before it is read. +pub(crate) fn counted<'de, D: Deserializer<'de>, T: Deserialize<'de>>(d: D) -> Result, D::Error> { + struct List(PhantomData); + impl<'de, T: Deserialize<'de>> Visitor<'de> for List { + type Value = Vec; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a list") + } + + fn visit_seq>(self, mut seq: A) -> Result, A::Error> { + let mut out = Vec::new(); + while let Some(v) = seq.next_element_seed(Item(PhantomData))? { + out.push(v); + } + Ok(out) + } + } + d.deserialize_seq(List(PhantomData)) +} + +/// [`counted`] for a list that may be `null`. +pub(crate) fn counted_option<'de, D: Deserializer<'de>, T: Deserialize<'de>>(d: D) -> Result>, D::Error> { + struct Counted(Vec); + impl<'de, T: Deserialize<'de>> Deserialize<'de> for Counted { + fn deserialize>(d: D) -> Result { + counted(d).map(Counted) + } + } + Ok(Option::>::deserialize(d)?.map(|c| c.0)) +} + +/// [`counted`] for a map, each value counted before it is read. +pub(crate) fn counted_map<'de, D: Deserializer<'de>, T: Deserialize<'de>>(d: D) -> Result, D::Error> { + struct Map(PhantomData); + impl<'de, T: Deserialize<'de>> Visitor<'de> for Map { + type Value = BTreeMap; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a map") + } + + fn visit_map>(self, mut map: A) -> Result, A::Error> { + let mut out = BTreeMap::new(); + while let Some(k) = map.next_key::()? { + let v = map.next_value_seed(Item(PhantomData))?; + out.insert(k, v); + } + Ok(out) + } + } + d.deserialize_map(Map(PhantomData)) +} + +/// Parse one JSON value of type `T` from `json`, after checking its nesting against the limits. +pub(crate) fn parse(json: &[u8], limits: &Limits) -> Result { + check_depth(json, limits.max_depth)?; + counting(limits.max_items, || parse_checked(json).map_err(malformed)) +} + +/// Parse JSON whose nesting [`check_depth`] has accepted. +fn parse_checked(json: &[u8]) -> Result { + let mut de = serde_json::Deserializer::from_slice(json); + de.disable_recursion_limit(); + let v = T::deserialize(&mut de)?; + de.end()?; + Ok(v) +} + +/// A document's version number: VectorCraft's `version`, PhotoCraft's `format_version` or +/// EffectCraft's `schema`. +#[derive(serde::Deserialize)] +struct Version { + #[serde(alias = "format_version", alias = "schema")] + version: u32, +} + +/// Parse `T` from `json`; when that fails, report [`Error::TooNew`] instead if the file's version +/// (its top-level `version`, `format_version` or `schema`) is newer than `kind` supports, since a +/// newer format may change what `T` expects. +pub(crate) fn parse_versioned(json: &[u8], limits: &Limits, kind: Kind) -> Result { + check_depth(json, limits.max_depth)?; + counting(limits.max_items, || { + parse_checked(json).map_err(|e| match parse_checked::(json) { + Ok(v) if v.version > kind.supported_version() => too_new(kind, v.version), + _ => malformed(e), + }) + }) +} + +pub(crate) fn too_new(kind: Kind, version: u32) -> Error { + Error::TooNew { kind, version, supported: kind.supported_version() } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn depth_counts_brackets_outside_strings() { + assert!(check_depth(br#"[[["[[[[[[[["]]]"#, 3).is_ok()); + assert!(check_depth(br#"[[[["\"[["]]]]"#, 3).is_err()); + assert!(check_depth(br#"{"a":{"b":[1]}}"#, 3).is_ok()); + } + + #[test] + fn the_budget_refuses_past_its_limits() { + let limits = Limits { max_json: 10, max_entry: 5, max_total: 8, max_depth: 4, max_items: 2 }; + let mut b = Budget::new(&limits); + assert!(b.take_entry(6, "x").is_err()); + assert!(b.take_entry(5, "x").is_ok()); + assert!(matches!(b.take_entry(4, "x"), Err(Error::LimitExceeded(_)))); + assert_eq!(b.left(), 3); + assert!(b.check_json(11, "x").is_err() && b.check_json(10, "x").is_ok()); + } +} diff --git a/crates/doc/src/pcraft.rs b/crates/doc/src/pcraft.rs new file mode 100644 index 00000000..dd079869 --- /dev/null +++ b/crates/doc/src/pcraft.rs @@ -0,0 +1,158 @@ +//! PhotoCraft `.pcraft` bundles: a ZIP archive holding `manifest.json` (the document tree, with +//! `format_version`), tiles and blobs, and optionally the previews the manifest names +//! (`thumbnail`, `composite`). Only the ZIP form is read; a bundle saved as a folder is not. The +//! manifest's envelope is [`craft_pcraft_format::Manifest`]'s. + +use craft_pcraft_format::{self as format, COMPOSITE, FORMAT_VERSION, MANIFEST, THUMB}; +use serde::Deserialize; +use serde::de::{DeserializeOwned, IgnoredAny}; + +use crate::zip::{Zip, local_names_include}; +use crate::{Budget, Canvas, Error, Kind, Layer, MAX_DEPTH, Outline, Rendition, RenditionKind, counted, malformed, parse, parse_versioned, too_new}; + +pub(crate) fn sniff(head: &[u8]) -> bool { + head.starts_with(b"PK\x03\x04") && (local_names_include(head, MANIFEST) || Zip::open(head).is_ok_and(|z| z.find(MANIFEST).is_some())) +} + +/// The archive and its `manifest.json`. +fn manifest<'a>(bytes: &'a [u8], budget: &mut Budget) -> Result<(Zip<'a>, Vec), Error> { + let zip = Zip::open(bytes)?; + let entry = zip.find(MANIFEST).ok_or_else(|| malformed("the bundle has no manifest.json"))?; + let max = budget.limits.max_json.min(budget.limits.max_entry); + let json = zip.read(entry, max, budget, MANIFEST)?; + Ok((zip, json)) +} + +/// The version and the writer, read without the rest of the manifest, so that a manifest of a +/// newer format still reports them. +#[derive(Deserialize)] +struct Stamp { + format_version: u32, + #[serde(default)] + generator: Option, +} + +fn checked(version: u32) -> Result { + if version == 0 { + return Err(malformed("format_version 0 is invalid")); + } + Ok(version) +} + +/// The members of [`craft_pcraft_format::Manifest`], with the document as `D`, read without +/// building the manifest as a `serde_json::Value` first. +#[derive(Deserialize)] +struct Envelope { + format_version: u32, + /// Required, and a string, as in the manifest; [`info`] reports it. + #[serde(rename = "generator")] + _generator: String, + document: D, + #[serde(default)] + thumbnail: Option, + #[serde(default)] + composite: Option, +} + +/// The manifest's envelope, checked as [`craft_pcraft_format::read_manifest`] checks it: the +/// size, nesting and item count with [`craft_pcraft_format::check_manifest`], a version of 0 or +/// newer than [`FORMAT_VERSION`] refused, and a preview that names an entry other than [`THUMB`] +/// or [`COMPOSITE`] refused. A member that appears twice is refused. +fn read(json: &[u8], budget: &Budget) -> Result, Error> { + let mut limits = format::Limits::default(); + limits.max_manifest_bytes = json.len(); + limits.max_depth = budget.limits.max_depth.min(MAX_DEPTH) as usize; + format::check_manifest(json, &limits).map_err(|e| match e { + format::Error::LimitExceeded(m) => Error::LimitExceeded(m), + e => malformed(e), + })?; + let m: Envelope = parse_versioned(json, budget.limits, Kind::PhotoCraft)?; + let version = checked(m.format_version)?; + if version > FORMAT_VERSION { + return Err(too_new(Kind::PhotoCraft, version)); + } + for (member, entry, expected) in [("thumbnail", &m.thumbnail, THUMB), ("composite", &m.composite, COMPOSITE)] { + if let Some(entry) = entry + && entry != expected + { + return Err(malformed(format!("`{member}` names {entry:?}, not {expected:?}"))); + } + } + Ok(m) +} + +pub(crate) fn info(bytes: &[u8], budget: &mut Budget) -> Result<(u32, Option), Error> { + let (_, json) = manifest(bytes, budget)?; + let stamp: Stamp = parse(&json, budget.limits)?; + Ok((checked(stamp.format_version)?, stamp.generator)) +} + +pub(crate) fn renditions(bytes: &[u8], budget: &mut Budget) -> Result, Error> { + let (zip, json) = manifest(bytes, budget)?; + let m: Envelope = read(&json, budget)?; + let mut out = vec![]; + for (kind, path) in [(RenditionKind::Thumbnail, m.thumbnail), (RenditionKind::Composite, m.composite)] { + let Some(path) = path else { continue }; + let entry = zip.find(&path).ok_or_else(|| malformed(format!("the manifest names `{path}`, which the bundle does not have")))?; + let max = budget.limits.max_entry; + let bytes = zip.read(entry, max, budget, &path)?; + out.push(Rendition { kind, mime: "image/png".into(), bytes }); + } + Ok(out) +} + +#[derive(Deserialize)] +struct DocM { + size: SizeM, + resolution_dpi: f32, + mode: String, + depth: String, + /// Bottom to top. + #[serde(deserialize_with = "counted")] + layers: Vec, +} + +#[derive(Deserialize)] +struct SizeM { + width: u32, + height: u32, +} + +#[derive(Deserialize)] +struct LayerM { + name: String, + visible: bool, + blend: String, + opacity: f32, + content: ContentM, +} + +#[derive(Deserialize)] +struct ContentM { + kind: String, + #[serde(default, deserialize_with = "counted")] + children: Vec, +} + +impl From for Layer { + fn from(l: LayerM) -> Self { + Layer { + name: l.name, + kind: l.content.kind, + visible: l.visible, + opacity: Some(l.opacity), + blend: Some(l.blend), + children: l.content.children.into_iter().map(Layer::from).collect(), + } + } +} + +pub(crate) fn outline(bytes: &[u8], budget: &mut Budget) -> Result { + let (_, json) = manifest(bytes, budget)?; + let d = read::(&json, budget)?.document; + Ok(Outline { + canvas: Some(Canvas { width: d.size.width, height: d.size.height, resolution_dpi: d.resolution_dpi, mode: d.mode, depth: d.depth }), + pages: vec![], + layers: d.layers.into_iter().map(Layer::from).collect(), + }) +} diff --git a/crates/doc/src/vectorcraft.rs b/crates/doc/src/vectorcraft.rs new file mode 100644 index 00000000..5be99eec --- /dev/null +++ b/crates/doc/src/vectorcraft.rs @@ -0,0 +1,302 @@ +//! VectorCraft `.vectorcraft` documents: one JSON object, gzip-compressed when saved with Use +//! Compression: +//! +//! ```text +//! { "format": "vectorcraft", "version": 4, "generator": "VectorCraft 0.1.0", +//! "preview": { "mime": "image/png", "data": "" }, +//! "pdf": { "mime": "application/pdf", "blob": [offset, length] }, +//! "document": { "artboards": [...], "layers": [...], ... }, ... } +//! ``` +//! +//! Since version 4, large members may follow the JSON after the marker `\n\0VCBLOBS\0`; such a +//! member gives `"blob": [offset, length]` (from the end of the marker) instead of `data`. In a +//! compressed file, the marker and the data after it are inside the gzip stream too. Files +//! written before the rename say `"format": "drawcraft"`. The envelope's members are +//! [`craft_vectorcraft_format::Head`] and [`craft_vectorcraft_format::Image`]. + +use std::borrow::Cow; +use std::io::Read; + +use craft_vectorcraft_format::{BLOB_MAGIC, Head, Image, is_compressed}; +use flate2::read::GzDecoder; +use serde::Deserialize; + +use crate::{Budget, Error, Kind, Layer, Outline, Page, Rendition, RenditionKind, counted, malformed, parse, parse_versioned, too_new}; + +/// Bytes decompressed at a time. +const CHUNK: u64 = 64 << 10; + +pub(crate) fn sniff(head: &[u8]) -> bool { + craft_vectorcraft_format::sniff(head) +} + +/// Where the JSON ends and the data after the marker starts, when `text` holds a NUL. JSON holds +/// no raw NUL and the marker's second byte is one, so the JSON ends right before the first NUL; +/// a NUL that is not the marker's makes the file malformed. +fn marker(text: &[u8], nul: usize) -> Result<(usize, usize), Error> { + match nul.checked_sub(1) { + Some(at) if text.get(at..).is_some_and(|r| r.starts_with(BLOB_MAGIC)) => Ok((at, at + BLOB_MAGIC.len())), + _ => Err(malformed("the file holds a NUL byte outside the marker after its JSON")), + } +} + +fn damaged(e: std::io::Error) -> Error { + malformed(format!("damaged compressed data: {e}")) +} + +/// A file's JSON, and the data after it. +struct Unpacked<'a> { + json: Cow<'a, [u8]>, + tail: Tail<'a>, +} + +/// The data after the marker. +enum Tail<'a> { + /// In an uncompressed file. + Slice(&'a [u8]), + /// Still in the gzip stream: `read` holds what was decompressed past the marker already, + /// of which `counted` bytes are counted against the budget. + Stream { dec: Box>, read: Vec, counted: usize }, +} + +impl Tail<'_> { + /// The data up to `end` (less where the file ends first), decompressing only that far. + fn up_to(&mut self, end: u64, budget: &mut Budget) -> Result<&[u8], Error> { + match self { + Tail::Slice(s) => Ok(s), + Tail::Stream { dec, read, counted } => { + let end = usize::try_from(end).map_err(|_| Error::LimitExceeded("the data after the JSON does not fit in memory".into()))?; + let had = read.len().min(end); + if had > *counted { + budget.take((had - *counted) as u64, "the decompressed document")?; + *counted = had; + } + if read.len() < end { + let want = ((end - read.len()) as u64).min(budget.left().saturating_add(1)); + let n = (&mut **dec).take(want).read_to_end(read).map_err(damaged)?; + budget.take(n as u64, "the decompressed document")?; + *counted += n; + } + Ok(read.as_slice()) + } + } + } +} + +/// A file's JSON and where the data after it is. A compressed file is decompressed only as far +/// as the end of the marker, every byte used counted against the budget, and its JSON is +/// checked against `max_json` as it grows. +fn unpack<'a>(bytes: &'a [u8], budget: &mut Budget) -> Result, Error> { + const JSON: &str = "the document's JSON"; + let too_long = |budget: &Budget| Error::LimitExceeded(format!("{JSON} is more than max_json ({} bytes)", budget.limits.max_json)); + if !is_compressed(bytes) { + let (json, tail) = match bytes.iter().position(|&b| b == 0) { + Some(nul) => { + let (end, tail) = marker(bytes, nul)?; + (bytes.get(..end).unwrap_or_default(), bytes.get(tail..).unwrap_or_default()) + } + None => (bytes, &[][..]), + }; + budget.check_json(json.len() as u64, JSON)?; + return Ok(Unpacked { json: Cow::Borrowed(json), tail: Tail::Slice(tail) }); + } + let mut dec = Box::new(GzDecoder::new(bytes)); + let mut buf = Vec::new(); + let mut split: Option<(usize, usize)> = None; + let mut nul: Option = None; + loop { + let start = buf.len(); + let want = CHUNK.min(budget.left().saturating_add(1)); + let n = (&mut *dec).take(want).read_to_end(&mut buf).map_err(damaged)?; + // The JSON ends before the first NUL: until there is one, all of it is JSON. The marker + // may end in a later chunk than its first NUL. + if nul.is_none() { + nul = buf.get(start..).and_then(|b| b.iter().position(|&c| c == 0)).map(|p| p + start); + } + if nul.unwrap_or(buf.len()).saturating_sub(1) as u64 > budget.limits.max_json { + return Err(too_long(budget)); + } + let mut used = n; + if let Some(nul) = nul + && buf.len() >= nul + BLOB_MAGIC.len() - 1 + { + let s = marker(&buf, nul)?; + split = Some(s); + // The data after the marker in this chunk is counted when it is read. + used = s.1.saturating_sub(start).min(n); + } + budget.take(used as u64, "the decompressed document")?; + if (n as u64) < want || split.is_some() { + break; + } + } + let (json_len, tail_at) = match (split, nul) { + (Some(s), _) => s, + // The stream ended inside the marker. + (None, Some(nul)) => marker(&buf, nul)?, + (None, None) => (buf.len(), buf.len()), + }; + budget.check_json(json_len as u64, JSON)?; + let read = if tail_at <= buf.len() { buf.split_off(tail_at) } else { vec![] }; + buf.truncate(json_len); + Ok(Unpacked { json: Cow::Owned(buf), tail: Tail::Stream { dec, read, counted: 0 } }) +} + +/// The format name, the version and the writer, read without the rest of the file, so that a file +/// of a newer version still reports them. +#[derive(Deserialize)] +struct Stamp { + format: String, + version: u32, + #[serde(default)] + generator: Option, +} + +/// The file's [`Stamp`], refused when the format is not VectorCraft's, and when the version is +/// newer than this crate reads unless `newer` is true. +fn stamp(json: &[u8], budget: &Budget, newer: bool) -> Result { + let s: Stamp = if newer { parse(json, budget.limits)? } else { parse_versioned(json, budget.limits, Kind::VectorCraft)? }; + match craft_vectorcraft_format::check(&s.format, s.version) { + Ok(()) => Ok(s), + Err(craft_vectorcraft_format::Error::Format(format)) => Err(malformed(format!("format is `{format}`"))), + Err(craft_vectorcraft_format::Error::TooNew(_)) if newer => Ok(s), + Err(craft_vectorcraft_format::Error::TooNew(version)) => Err(too_new(Kind::VectorCraft, version)), + Err(e) => Err(malformed(e)), + } +} + +pub(crate) fn info(bytes: &[u8], budget: &mut Budget) -> Result<(u32, Option), Error> { + let json = unpack(bytes, budget)?.json; + let s = stamp(&json, budget, true)?; + Ok((s.version, s.generator)) +} + +/// Check the entry's size against `max_entry` and count it; for an entry after the JSON, the +/// offset where it ends. +fn take(image: &Image, budget: &mut Budget, what: &str) -> Result, Error> { + budget.take_entry(image.len(), what)?; + Ok(image.blob.map(|[offset, len]| offset.saturating_add(len))) +} + +/// The entry's bytes, after [`take`] has checked its size. +fn entry_bytes(image: &Image, tail: &[u8], what: &str) -> Result, Error> { + image.bytes_at_most(tail, u64::MAX).ok_or_else(|| match image.blob { + Some(_) => malformed(format!("the file ends before its {what}")), + None => malformed(format!("the {what} is not valid base64")), + }) +} + +pub(crate) fn renditions(bytes: &[u8], budget: &mut Budget) -> Result, Error> { + let Unpacked { json, mut tail } = unpack(bytes, budget)?; + stamp(&json, budget, false)?; + let head: Head = parse(&json, budget.limits)?; + // A member without a MIME type gives no rendition: a preview may be PNG or JPEG. + let members: Vec<_> = [(RenditionKind::Thumbnail, head.preview, "preview"), (RenditionKind::Pdf, head.pdf, "PDF")] + .into_iter() + .filter_map(|(kind, m, what)| Some((kind, m.filter(|m| !m.mime.is_empty())?, what))) + .collect(); + // Every size is checked before any data after the JSON is decompressed. + let mut end = 0u64; + for (_, m, what) in &members { + end = end.max(take(m, budget, what)?.unwrap_or(0)); + } + let tail = if end > 0 { tail.up_to(end, budget)? } else { &[] }; + members + .into_iter() + .map(|(kind, m, what)| { + let bytes = entry_bytes(&m, tail, what)?; + Ok(Rendition { kind, mime: m.mime, bytes }) + }) + .collect() +} + +#[derive(Deserialize)] +struct WithDocument { + document: DocM, +} + +#[derive(Deserialize)] +struct DocM { + #[serde(default, deserialize_with = "counted")] + artboards: Vec, + /// Bottom first. + #[serde(default, deserialize_with = "counted")] + layers: Vec, +} + +#[derive(Deserialize)] +struct ArtboardM { + #[serde(default)] + name: String, + rect: RectM, +} + +#[derive(Deserialize)] +struct RectM { + x0: f64, + y0: f64, + x1: f64, + y1: f64, +} + +fn yes() -> bool { + true +} + +fn one() -> f32 { + 1.0 +} + +/// A node, written without the fields at their defaults. +#[derive(Deserialize)] +struct NodeM { + #[serde(default)] + name: Option, + #[serde(default = "yes")] + visible: bool, + #[serde(default = "one")] + opacity: f32, + #[serde(default)] + blend: Option, + kind: KindM, +} + +#[derive(Deserialize)] +struct KindM { + #[serde(rename = "type")] + kind: String, +} + +pub(crate) fn outline(bytes: &[u8], budget: &mut Budget) -> Result { + let json = unpack(bytes, budget)?.json; + stamp(&json, budget, false)?; + let f: WithDocument = parse(&json, budget.limits)?; + let pages = f + .document + .artboards + .into_iter() + .map(|a| Page { + name: a.name, + x: a.rect.x0.min(a.rect.x1), + y: a.rect.y0.min(a.rect.y1), + width: (a.rect.x1 - a.rect.x0).abs(), + height: (a.rect.y1 - a.rect.y0).abs(), + timing: None, + layers: vec![], + }) + .collect(); + let layers = f + .document + .layers + .into_iter() + .map(|n| Layer { + name: n.name.unwrap_or_default(), + kind: n.kind.kind, + visible: n.visible, + opacity: Some(n.opacity), + blend: Some(n.blend.unwrap_or_else(|| "Normal".into())), + children: vec![], + }) + .collect(); + Ok(Outline { canvas: None, pages, layers }) +} diff --git a/crates/doc/src/zip.rs b/crates/doc/src/zip.rs new file mode 100644 index 00000000..ec0a4469 --- /dev/null +++ b/crates/doc/src/zip.rs @@ -0,0 +1,183 @@ +//! A ZIP reader for `.pcraft` bundles: STORE and DEFLATE entries, found through the central +//! directory, CRC-checked. No ZIP64, no encryption, no archives split across disks. + +use std::io::Read; +use std::ops::Range; + +use crate::{Budget, Error, malformed}; + +const LOCAL_SIG: u32 = 0x0403_4b50; +const CENTRAL_SIG: u32 = 0x0201_4b50; +const EOCD_SIG: u32 = 0x0605_4b50; +const ZIP64_LOCATOR_SIG: u32 = 0x0706_4b50; +const EOCD_LEN: usize = 22; +const CENTRAL_LEN: usize = 46; +const LOCAL_LEN: usize = 30; + +fn u16_at(b: &[u8], at: usize) -> Option { + let s = b.get(at..at.checked_add(2)?)?; + Some(u16::from_le_bytes([*s.first()?, *s.get(1)?])) +} + +fn u32_at(b: &[u8], at: usize) -> Option { + let s = b.get(at..at.checked_add(4)?)?; + Some(u32::from_le_bytes([*s.first()?, *s.get(1)?, *s.get(2)?, *s.get(3)?])) +} + +fn truncated() -> Error { + malformed("truncated ZIP archive") +} + +/// One entry of the central directory. +pub(crate) struct Entry { + name: Range, + flags: u16, + method: u16, + crc: u32, + compressed: usize, + size: u64, + /// Where the entry's data starts. + data: usize, +} + +pub(crate) struct Zip<'a> { + bytes: &'a [u8], + entries: Vec, +} + +impl<'a> Zip<'a> { + pub(crate) fn open(bytes: &'a [u8]) -> Result { + let last = bytes.len().checked_sub(EOCD_LEN).ok_or_else(|| malformed("not a ZIP archive (too short)"))?; + // The comment may itself hold the signature: accept only a record whose comment ends the + // file and whose central directory ends where the record starts. + let eocd = (last.saturating_sub(65535)..=last) + .rev() + .find(|&i| { + let comment_end = u16_at(bytes, i + 20).and_then(|n| i.checked_add(EOCD_LEN)?.checked_add(usize::from(n))); + let dir_end = u32_at(bytes, i + 16) + .zip(u32_at(bytes, i + 12)) + .and_then(|(off, size)| usize::try_from(off).ok()?.checked_add(usize::try_from(size).ok()?)); + u32_at(bytes, i) == Some(EOCD_SIG) && comment_end == Some(bytes.len()) && dir_end == Some(i) + }) + .ok_or_else(|| { + let zip64 = (last.saturating_sub(65535 + 20)..=last.saturating_sub(20)).any(|i| u32_at(bytes, i) == Some(ZIP64_LOCATOR_SIG)); + if zip64 { + Error::Unsupported("ZIP64 archives are not supported".into()) + } else { + malformed("ZIP end of central directory not found") + } + })?; + let disk = u16_at(bytes, eocd + 4).ok_or_else(truncated)?; + let dir_disk = u16_at(bytes, eocd + 6).ok_or_else(truncated)?; + let count = usize::from(u16_at(bytes, eocd + 10).ok_or_else(truncated)?); + if disk != 0 || dir_disk != 0 { + return Err(Error::Unsupported("ZIP archives split across disks are not supported".into())); + } + let dir_size = u32_at(bytes, eocd + 12).ok_or_else(truncated)? as usize; + let dir = eocd - dir_size; + let mut entries = Vec::with_capacity(count.min(dir_size / CENTRAL_LEN)); + let mut p = dir; + for _ in 0..count { + if u32_at(bytes, p) != Some(CENTRAL_SIG) || p + CENTRAL_LEN > eocd { + return Err(malformed("bad ZIP central directory entry")); + } + let field16 = |o: usize| u16_at(bytes, p + o).ok_or_else(truncated); + let field32 = |o: usize| u32_at(bytes, p + o).ok_or_else(truncated); + let (flags, method, crc) = (field16(8)?, field16(10)?, field32(16)?); + let (compressed, size) = (field32(20)?, field32(24)?); + let (name_len, extra_len, comment_len) = (usize::from(field16(28)?), usize::from(field16(30)?), usize::from(field16(32)?)); + let local = field32(42)? as usize; + if compressed == u32::MAX || size == u32::MAX || local == u32::MAX as usize { + return Err(Error::Unsupported("ZIP64 entries are not supported".into())); + } + let name_start = p + CENTRAL_LEN; + let name_end = name_start + name_len; + let next = name_end + extra_len + comment_len; + if next > eocd { + return Err(malformed("ZIP central directory entry out of range")); + } + if u32_at(bytes, local) != Some(LOCAL_SIG) { + return Err(malformed("bad ZIP local header")); + } + let local_name = usize::from(u16_at(bytes, local + 26).ok_or_else(truncated)?); + let local_extra = usize::from(u16_at(bytes, local + 28).ok_or_else(truncated)?); + let data = local.checked_add(LOCAL_LEN + local_name + local_extra).ok_or_else(truncated)?; + let compressed = compressed as usize; + if data.checked_add(compressed).is_none_or(|end| end > bytes.len()) { + return Err(malformed("ZIP entry data out of range")); + } + entries.push(Entry { name: name_start..name_end, flags, method, crc, compressed, size: u64::from(size), data }); + p = next; + } + Ok(Zip { bytes, entries }) + } + + /// The first entry named `name`. + pub(crate) fn find(&self, name: &str) -> Option<&Entry> { + self.entries.iter().find(|e| self.bytes.get(e.name.clone()) == Some(name.as_bytes())) + } + + /// The entry's bytes, decompressed and CRC-checked; their size is checked against `max` and + /// counted against `budget` first. + pub(crate) fn read(&self, e: &Entry, max: u64, budget: &mut Budget, what: &str) -> Result, Error> { + if e.flags & 1 != 0 { + return Err(Error::Unsupported(format!("{what} is encrypted"))); + } + if !matches!(e.method, 0 | 8) { + return Err(Error::Unsupported(format!("{what} uses ZIP compression method {}", e.method))); + } + if e.size > max { + return Err(Error::LimitExceeded(format!("{what} is {} bytes, more than the limit ({max})", e.size))); + } + budget.take(e.size, what)?; + let raw = self.bytes.get(e.data..e.data + e.compressed).ok_or_else(truncated)?; + let size = usize::try_from(e.size).map_err(|_| Error::LimitExceeded(format!("{what} does not fit in memory")))?; + let data = if e.method == 0 { + if e.compressed != size { + return Err(malformed(format!("{what} has the wrong size"))); + } + raw.to_vec() + } else { + // DEFLATE expands at most 1032 times. + let mut out = Vec::with_capacity(size.min(raw.len().saturating_mul(1032))); + flate2::read::DeflateDecoder::new(raw).take(e.size + 1).read_to_end(&mut out).map_err(|err| malformed(format!("{what}: {err}")))?; + if out.len() != size { + return Err(malformed(format!("{what} has the wrong size"))); + } + out + }; + let mut crc = flate2::Crc::new(); + crc.update(&data); + if crc.sum() != e.crc { + return Err(malformed(format!("{what} failed its CRC check"))); + } + Ok(data) + } +} + +/// Does a local header in `head` name `name`? Walks the local headers from the start, for a +/// head too short to reach the central directory; stops at an entry whose size is only in a +/// data descriptor. +pub(crate) fn local_names_include(head: &[u8], name: &str) -> bool { + let mut p = 0usize; + while u32_at(head, p) == Some(LOCAL_SIG) { + let (Some(flags), Some(compressed), Some(name_len), Some(extra_len)) = + (u16_at(head, p + 6), u32_at(head, p + 18), u16_at(head, p + 26), u16_at(head, p + 28)) + else { + return false; + }; + let start = p + LOCAL_LEN; + let end = start + usize::from(name_len); + if head.get(start..end) == Some(name.as_bytes()) { + return true; + } + if flags & 8 != 0 { + return false; + } + p = match end.checked_add(usize::from(extra_len)).and_then(|q| q.checked_add(compressed as usize)) { + Some(q) => q, + None => return false, + }; + } + false +} diff --git a/crates/doc/tests/common/mod.rs b/crates/doc/tests/common/mod.rs new file mode 100644 index 00000000..727379d2 --- /dev/null +++ b/crates/doc/tests/common/mod.rs @@ -0,0 +1,222 @@ +//! Synthetic documents built from scratch for the tests: hand-written JSON, a small literal PDF, +//! PNG files encoded here, ZIP archives and gzip streams assembled here. +#![allow(dead_code)] + +use std::io::Write; + +/// A one-page PDF with an empty page. +pub const PDF: &[u8] = b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 72 72] >>\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF\n"; + +pub const BLOB_MAGIC: &[u8] = b"\n\0VCBLOBS\0"; + +pub fn gzip(bytes: &[u8]) -> Vec { + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(bytes).unwrap(); + gz.finish().unwrap() +} + +pub fn deflate(bytes: &[u8]) -> Vec { + let mut d = flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default()); + d.write_all(bytes).unwrap(); + d.finish().unwrap() +} + +pub fn crc32(bytes: &[u8]) -> u32 { + let mut c = flate2::Crc::new(); + c.update(bytes); + c.sum() +} + +pub fn base64(data: &[u8]) -> String { + const A: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::new(); + for c in data.chunks(3) { + let n = (u32::from(c[0]) << 16) | (u32::from(*c.get(1).unwrap_or(&0)) << 8) | u32::from(*c.get(2).unwrap_or(&0)); + out.push(A[(n >> 18) as usize & 63] as char); + out.push(A[(n >> 12) as usize & 63] as char); + out.push(if c.len() > 1 { A[(n >> 6) as usize & 63] as char } else { '=' }); + out.push(if c.len() > 2 { A[n as usize & 63] as char } else { '=' }); + } + out +} + +/// A gray `w` × `h` PNG with every pixel `value`. +pub fn png(w: u32, h: u32, value: u8) -> Vec { + fn chunk(out: &mut Vec, kind: &[u8; 4], data: &[u8]) { + out.extend((data.len() as u32).to_be_bytes()); + let mut body = kind.to_vec(); + body.extend(data); + out.extend(&body); + out.extend(crc32(&body).to_be_bytes()); + } + let mut ihdr = w.to_be_bytes().to_vec(); + ihdr.extend(h.to_be_bytes()); + ihdr.extend([8, 0, 0, 0, 0]); + let mut raw = vec![]; + for _ in 0..h { + raw.push(0); + raw.extend(std::iter::repeat_n(value, w as usize)); + } + let mut z = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + z.write_all(&raw).unwrap(); + let mut out = b"\x89PNG\r\n\x1a\n".to_vec(); + chunk(&mut out, b"IHDR", &ihdr); + chunk(&mut out, b"IDAT", &z.finish().unwrap()); + chunk(&mut out, b"IEND", &[]); + out +} + +/// One ZIP entry: its name, its bytes, and whether it is DEFLATE-compressed. +pub struct Entry<'a> { + pub name: &'a str, + pub data: &'a [u8], + pub deflate: bool, +} + +pub fn store<'a>(name: &'a str, data: &'a [u8]) -> Entry<'a> { + Entry { name, data, deflate: false } +} + +pub fn deflated<'a>(name: &'a str, data: &'a [u8]) -> Entry<'a> { + Entry { name, data, deflate: true } +} + +/// A ZIP archive of `entries`, in order, with a central directory and an end record. +pub fn zip(entries: &[Entry]) -> Vec { + let mut out = vec![]; + let mut central = vec![]; + for e in entries { + let payload = if e.deflate { deflate(e.data) } else { e.data.to_vec() }; + let crc = crc32(e.data); + let method: u16 = if e.deflate { 8 } else { 0 }; + let offset = out.len() as u32; + let fields = |out: &mut Vec| { + out.extend(20u16.to_le_bytes()); + out.extend(0x0800u16.to_le_bytes()); + out.extend(method.to_le_bytes()); + out.extend(0u16.to_le_bytes()); + out.extend(0x21u16.to_le_bytes()); + out.extend(crc.to_le_bytes()); + out.extend((payload.len() as u32).to_le_bytes()); + out.extend((e.data.len() as u32).to_le_bytes()); + out.extend((e.name.len() as u16).to_le_bytes()); + out.extend(0u16.to_le_bytes()); + }; + out.extend(0x0403_4b50u32.to_le_bytes()); + fields(&mut out); + out.extend(e.name.as_bytes()); + out.extend(&payload); + central.extend(0x0201_4b50u32.to_le_bytes()); + central.extend(20u16.to_le_bytes()); + fields(&mut central); + // Comment length, disk, internal and external attributes. + central.extend([0u8; 10]); + central.extend(offset.to_le_bytes()); + central.extend(e.name.as_bytes()); + } + let dir_offset = out.len() as u32; + out.extend(¢ral); + out.extend(0x0605_4b50u32.to_le_bytes()); + out.extend([0u8; 4]); + out.extend((entries.len() as u16).to_le_bytes()); + out.extend((entries.len() as u16).to_le_bytes()); + out.extend((central.len() as u32).to_le_bytes()); + out.extend(dir_offset.to_le_bytes()); + out.extend(0u16.to_le_bytes()); + out +} + +/// A VectorCraft document member holding `bytes` as base64. +pub fn inline(mime: &str, bytes: &[u8]) -> String { + format!(r#"{{"mime":"{mime}","data":"{}"}}"#, base64(bytes)) +} + +/// A VectorCraft document member pointing at the data after the JSON. +pub fn blob(mime: &str, offset: usize, len: usize) -> String { + format!(r#"{{"mime":"{mime}","blob":[{offset},{len}]}}"#) +} + +/// The JSON of a VectorCraft file: `members` go between `generator` and `document`. +pub fn vectorcraft_json(version: u32, members: &str) -> String { + let document = r#"{"version":1,"title":"","units":"Points","color_mode":"Rgb", + "artboards":[{"id":1,"name":"Front","rect":{"x0":0.0,"y0":0.0,"x1":612.0,"y1":792.0},"show_center_mark":false,"show_cross_hairs":false}, + {"id":2,"name":"Back","rect":{"x0":700.0,"y0":800.0,"x1":650.0,"y1":700.0},"show_center_mark":false,"show_cross_hairs":false}], + "layers":[{"id":1,"name":"Art","kind":{"type":"layer","color":{"Preset":0},"template":false,"printable":true, + "children":[{"id":3,"kind":{"type":"group","children":[{"id":4,"kind":{"type":"path","path":{"subpaths":[]}}}]}}]}}, + {"id":2,"name":"Notes","visible":false,"opacity":0.5,"blend":"Multiply","kind":{"type":"layer","color":{"Preset":1},"template":false,"printable":false,"children":[]}}], + "next_id":5}"#; + let sep = if members.is_empty() { "" } else { "," }; + format!(r#"{{"format":"vectorcraft","version":{version},"generator":"VectorCraft 0.1.0"{sep}{members},"document":{document},"images":{{}}}}"#) +} + +/// A PhotoCraft layer as the manifest writes it. +pub fn pcraft_layer(name: &str, visible: bool, blend: &str, opacity: f32, content: &str) -> String { + format!( + r#"{{"id":1,"name":"{name}","visible":{visible},"locks":{{}},"blend":"{blend}","opacity":{opacity},"fill_opacity":1.0,"clipped":false,"mask":null, + "effects":{{"enabled":true,"items":[],"psd_raw":null}},"label":"None","content":{content},"psd_blocks":[],"psd_id":null,"fill_cache":null}}"# + ) +} + +pub fn raster() -> &'static str { + r#"{"kind":"raster","surface":{"format":{"mode":"Rgb","sample":"U8","alpha":true},"default":"00000000","tiles":[]}}"# +} + +/// A PhotoCraft manifest with `layers` (a JSON array) and the preview paths given. +pub fn pcraft_manifest(version: u32, layers: &str, previews: &str) -> String { + format!( + r#"{{"format_version":{version},"generator":"photocraft-format 0.1.0", + "document":{{"id":1,"name":"Untitled","size":{{"width":640,"height":480}},"resolution_dpi":300.0,"mode":"Rgb","depth":"U16", + "icc_profile":null,"layers":{layers},"channels":[],"guides":{{}},"selection":null,"metadata":{{}},"global_light":{{}}}}{previews}}}"# + ) +} + +/// A `.pcraft` bundle: a tile, the previews, and the manifest last. +pub fn pcraft(thumb: &[u8], composite: &[u8]) -> Vec { + let layers = format!( + "[{},{}]", + pcraft_layer("Background", true, "Normal", 1.0, raster()), + pcraft_layer( + "Group", + false, + "Screen", + 0.25, + &format!(r#"{{"kind":"group","expanded":true,"children":[{}]}}"#, pcraft_layer("Inner", true, "Multiply", 0.5, raster())) + ) + ); + let manifest = pcraft_manifest(1, &layers, r#","thumbnail":"thumb.png","composite":"composite/preview.png""#); + zip(&[ + store("tiles/0123.zst", b"not really zstd"), + store("thumb.png", thumb), + deflated("composite/preview.png", composite), + deflated("manifest.json", manifest.as_bytes()), + ]) +} + +/// An EffectCraft project with a folder, a footage item and two compositions. +pub fn ecproj(schema: u32) -> String { + format!( + r#"{{ + "savedBy": "0.12.0", + "activeComp": 10, + "schema": {schema}, + "settings": {{}}, + "items": {{ + "1": {{"id": 1, "name": "Footage", "label": 0, "kind": {{"type": "Folder"}}}}, + "2": {{"id": 2, "name": "clip.mov", "label": 0, "parent": 1, + "kind": {{"type": "Footage", "path": "clip.mov", "kind": "Video", "width": 640, "height": 360, "pixel_aspect": 1.0, + "frame_rate": {{"num": 30, "den": 1}}, "duration": 2540160000000}}}}, + "10": {{"id": 10, "name": "Main", "label": 0, + "kind": {{"type": "Comp", "width": 1920, "height": 1080, "pixel_aspect": 1.0, "frame_rate": {{"num": 24000, "den": 1001}}, + "duration": 1270080000000, "display_start": 0, "background": [0.0, 0.0, 0.0], "work_area": [0, 1270080000000], + "layers": [ + {{"id": 1, "name": "Title", "source": {{"type": "Text"}}, "switches": {{"video": true, "audio": false}}, "blend_mode": "Add"}}, + {{"id": 2, "name": "Plate", "source": {{"type": "Footage", "item": 2}}, "switches": {{"video": false, "audio": true}}, "blend_mode": "Normal"}} + ]}}}}, + "3": {{"id": 3, "name": "Intro", "label": 0, + "kind": {{"type": "Comp", "width": 640, "height": 360, "pixel_aspect": 1.0, "frame_rate": {{"num": 30, "den": 1}}, + "duration": 254016000000, "layers": []}}}} + }}, + "next_id": 11 +}}"# + ) +} diff --git a/crates/doc/tests/envelopes.rs b/crates/doc/tests/envelopes.rs new file mode 100644 index 00000000..15828919 --- /dev/null +++ b/crates/doc/tests/envelopes.rs @@ -0,0 +1,138 @@ +//! Documents whose envelope is written with the format crates' own types read through craft-doc, +//! and envelopes those crates refuse are refused here too. + +mod common; + +use std::collections::BTreeMap; + +use common::*; +use craft_doc::{Error, Kind, Limits, RenditionKind, info, outline, renditions, sniff}; + +fn limits() -> Limits { + Limits::default() +} + +fn vectorcraft_document() -> serde_json::Value { + serde_json::json!({ + "artboards": [{"id": 1, "name": "Front", "rect": {"x0": 0.0, "y0": 0.0, "x1": 612.0, "y1": 792.0}}], + "layers": [{"id": 1, "name": "Art", "kind": {"type": "layer", "children": []}}], + }) +} + +#[test] +fn a_vectorcraft_file_written_with_the_envelope_types_reads() { + let preview = png(16, 16, 3); + let pdf: Vec = PDF.iter().copied().cycle().take(craft_vectorcraft_format::INLINE_MAX + 1).collect(); + let mut tail = craft_vectorcraft_format::Tail::new(craft_vectorcraft_format::VERSION); + let mut file = craft_vectorcraft_format::File::new(vectorcraft_document()); + file.generator = Some("VectorCraft 0.1.0".into()); + file.preview = Some(tail.image("image/png", &preview, false)); + file.pdf = Some(tail.image("application/pdf", &pdf, false)); + assert!(file.pdf.as_ref().unwrap().blob.is_some() && file.preview.as_ref().unwrap().blob.is_none()); + let mut bytes = serde_json::to_vec(&file).unwrap(); + tail.write_to(&mut bytes).unwrap(); + + for bytes in [bytes.clone(), gzip(&bytes)] { + assert_eq!(sniff(&bytes), Some(Kind::VectorCraft)); + let i = info(&bytes, &limits()).unwrap(); + assert_eq!((i.version, i.generator.as_deref()), (craft_vectorcraft_format::VERSION, Some("VectorCraft 0.1.0"))); + let r = renditions(&bytes, &limits()).unwrap(); + assert_eq!((r[0].kind, r[0].bytes.as_slice()), (RenditionKind::Thumbnail, preview.as_slice())); + assert_eq!((r[1].kind, r[1].bytes.as_slice()), (RenditionKind::Pdf, pdf.as_slice())); + let o = outline(&bytes, &limits()).unwrap(); + assert_eq!((o.pages[0].name.as_str(), o.pages[0].width, o.layers[0].name.as_str()), ("Front", 612.0, "Art")); + } +} + +#[test] +fn a_pcraft_manifest_written_with_the_envelope_type_reads() { + let document = serde_json::json!({ + "size": {"width": 64, "height": 32}, "resolution_dpi": 72.0, "mode": "Rgb", "depth": "U8", + "layers": [{"name": "Background", "visible": true, "blend": "Normal", "opacity": 1.0, "content": {"kind": "raster"}}], + }); + let mut manifest = craft_pcraft_format::Manifest::new("photocraft-format 0.1.0", document); + manifest.thumbnail = Some(craft_pcraft_format::THUMB.into()); + let json = serde_json::to_vec(&manifest).unwrap(); + let thumb = png(4, 4, 1); + let bundle = zip(&[store(craft_pcraft_format::MANIFEST, &json), store(craft_pcraft_format::THUMB, &thumb)]); + + assert_eq!(sniff(&bundle), Some(Kind::PhotoCraft)); + assert_eq!(info(&bundle, &limits()).unwrap().generator.as_deref(), Some("photocraft-format 0.1.0")); + let r = renditions(&bundle, &limits()).unwrap(); + assert_eq!((r.len(), r[0].kind, r[0].bytes.as_slice()), (1, RenditionKind::Thumbnail, thumb.as_slice())); + let o = outline(&bundle, &limits()).unwrap(); + assert_eq!((o.canvas.unwrap().width, o.layers[0].name.as_str()), (64, "Background")); +} + +#[test] +fn an_ecproj_file_written_with_the_envelope_type_reads() { + let view = BTreeMap::from([("activeComp", 1)]); + let project = serde_json::json!({ + "schema": craft_ecproj_format::SCHEMA_VERSION, + "items": {"1": {"id": 1, "name": "Main", "kind": {"type": "Comp", "width": 1280, "height": 720, "layers": []}}}, + }); + let text = craft_ecproj_format::File::new("0.12.0", &view, &project).to_json().unwrap(); + + assert_eq!(sniff(text.as_bytes()), Some(Kind::EffectCraft)); + let i = info(text.as_bytes(), &limits()).unwrap(); + assert_eq!((i.version, i.generator.as_deref()), (craft_ecproj_format::SCHEMA_VERSION, Some("0.12.0"))); + let o = outline(text.as_bytes(), &limits()).unwrap(); + assert_eq!((o.pages[0].name.as_str(), o.pages[0].width), ("Main", 1280.0)); +} + +#[test] +fn a_pcraft_manifest_without_a_generator_is_malformed() { + let manifest = pcraft_manifest(1, "[]", "").replacen(r#""generator":"photocraft-format 0.1.0","#, "", 1); + let bundle = zip(&[store("manifest.json", manifest.as_bytes())]); + assert_eq!(info(&bundle, &limits()).unwrap().generator, None); + assert!(matches!(renditions(&bundle, &limits()), Err(Error::Malformed(_)))); + assert!(matches!(outline(&bundle, &limits()), Err(Error::Malformed(_)))); +} + +#[test] +fn a_pcraft_preview_naming_another_entry_is_malformed() { + let manifest = pcraft_manifest(1, "[]", r#","thumbnail":"../thumb.png""#); + let bundle = zip(&[store("manifest.json", manifest.as_bytes()), store("../thumb.png", &png(1, 1, 0))]); + assert!(matches!(renditions(&bundle, &limits()), Err(Error::Malformed(_)))); + assert!(matches!(outline(&bundle, &limits()), Err(Error::Malformed(_)))); +} + +#[test] +fn an_ecproj_nested_to_max_depth_reads() { + let deep = |levels: usize| format!(r#"{{"savedBy":"0.12.0","schema":2,"items":{{}},"x":{}{}}}"#, "[".repeat(levels), "]".repeat(levels)); + let levels = craft_doc::MAX_DEPTH as usize - 1; + let objects = format!(r#"{{"savedBy":"0.12.0","schema":2,"items":{{}},"x":{}1{}}}"#, r#"{"a":"#.repeat(levels), "}".repeat(levels)); + for ok in [deep(levels), objects] { + assert_eq!(info(ok.as_bytes(), &limits()).unwrap().version, 2); + assert_eq!(renditions(ok.as_bytes(), &limits()).unwrap(), vec![]); + assert_eq!(outline(ok.as_bytes(), &limits()).unwrap().pages.len(), 0); + } +} + +#[test] +fn a_pcraft_manifest_with_a_repeated_member_or_a_late_newer_version_is_refused() { + let repeated = pcraft_manifest(1, "[]", r#","thumbnail":"thumb.png","thumbnail":"thumb.png""#); + let bundle = zip(&[store("manifest.json", repeated.as_bytes())]); + assert!(matches!(renditions(&bundle, &limits()), Err(Error::Malformed(_)))); + let late = r#"{"generator":"photocraft-format 9","document":"elsewhere","thumbnail":7,"format_version":2}"#; + let bundle = zip(&[store("manifest.json", late.as_bytes())]); + let refused = Error::TooNew { kind: Kind::PhotoCraft, version: 2, supported: Kind::PhotoCraft.supported_version() }; + assert_eq!(renditions(&bundle, &limits()), Err(refused.clone())); + assert_eq!(outline(&bundle, &limits()), Err(refused)); +} + +#[test] +fn a_vectorcraft_member_without_a_mime_type_is_skipped() { + let file = vectorcraft_json(4, &format!(r#""preview":{{"data":"{}"}}"#, base64(&png(1, 1, 0)))); + assert!(renditions(file.as_bytes(), &limits()).unwrap().is_empty()); + let empty = vectorcraft_json(4, &format!(r#""preview":{},"pdf":{}"#, inline("", &png(1, 1, 0)), inline("", PDF))); + assert!(renditions(empty.as_bytes(), &limits()).unwrap().is_empty()); +} + +#[test] +fn a_format_member_inside_another_member_is_not_vectorcraft() { + let nested = br#"{"meta":{"format":"vectorcraft"},"version":4}"#; + assert_eq!(sniff(nested), None); + let second = br#"{"version":4,"format":"vectorcraft","document":{}}"#; + assert_eq!(sniff(second), Some(Kind::VectorCraft)); +} diff --git a/crates/doc/tests/formats.rs b/crates/doc/tests/formats.rs new file mode 100644 index 00000000..d18b888c --- /dev/null +++ b/crates/doc/tests/formats.rs @@ -0,0 +1,264 @@ +//! Each format's documents, built from scratch: what `sniff`, `info`, `renditions` and `outline` +//! return for them. + +mod common; + +use common::*; +use craft_doc::{Error, Kind, Limits, RenditionKind, info, outline, renditions, sniff}; + +fn limits() -> Limits { + Limits::default() +} + +#[test] +fn sniff_tells_the_three_formats() { + let vc = vectorcraft_json(4, ""); + assert_eq!(sniff(vc.as_bytes()), Some(Kind::VectorCraft)); + assert_eq!(sniff(&gzip(vc.as_bytes())), Some(Kind::VectorCraft)); + let legacy = vc.replacen("\"vectorcraft\"", "\"drawcraft\"", 1); + assert_eq!(sniff(legacy.as_bytes()), Some(Kind::VectorCraft)); + // A gzip stream cut short still shows its start. + let gz = gzip(vc.as_bytes()); + assert_eq!(sniff(gz.get(..gz.len() - 8).unwrap()), Some(Kind::VectorCraft)); + + let pc = pcraft(&png(4, 4, 9), &png(8, 8, 9)); + assert_eq!(sniff(&pc), Some(Kind::PhotoCraft), "the manifest last, found through the central directory"); + let first = zip(&[store("manifest.json", b"{}"), store("thumb.png", &png(1, 1, 0))]); + assert_eq!(sniff(first.get(..60).unwrap()), Some(Kind::PhotoCraft), "the manifest first, in a short head"); + let second = zip(&[store("tiles/a.zst", b"x"), store("manifest.json", b"{}")]); + assert_eq!(sniff(second.get(..100).unwrap()), Some(Kind::PhotoCraft), "a later local header in the head"); + + assert_eq!(sniff(ecproj(2).as_bytes()), Some(Kind::EffectCraft)); + let old = r#"{"schema":1,"settings":{},"items":{},"next_id":1}"#; + assert_eq!(sniff(old.as_bytes()), Some(Kind::EffectCraft), "saved before savedBy"); +} + +#[test] +fn sniff_refuses_other_files() { + let plain_zip = zip(&[store("word/document.xml", b""), deflated("readme.txt", b"hello hello hello")]); + let behind = zip(&[store("tiles/a.zst", &[7; 4096]), store("manifest.json", b"{}")]); + for (what, bytes) in [ + ("empty", &b""[..]), + ("a PDF", PDF), + ("a PNG", &png(2, 2, 1)), + ("a ZIP without a manifest", &plain_zip), + ("a manifest past the head", behind.get(..100).unwrap()), + ("other JSON", br#"{"name":"x","items":[],"schema":"y"}"#), + ("JSON naming the format in a value", br#"{"title":"vectorcraft"}"#), + ("another format", br#"{"format":"vectorcraftx","version":4}"#), + ("a JSON array", br#"["vectorcraft"]"#), + ("savedBy without schema", br#"{"savedBy":"1.0","items":{}}"#), + ("savedBy with schema only in a nested object", br#"{"savedBy":"1.0","metadata":{"schema":1}}"#), + ("savedBy with schema only in a value", br#"{"savedBy":"1.0","note":"\"schema\""}"#), + ("schema with items only in a nested object", br#"{"schema":1,"meta":{"items":[]}}"#), + ("gzip of other JSON", &gzip(br#"{"format":"svg"}"#)), + ("PK alone", b"PK\x03\x04"), + ] { + assert_eq!(sniff(bytes), None, "{what}"); + } +} + +#[test] +fn info_reads_version_and_writer() { + let i = info(vectorcraft_json(3, "").as_bytes(), &limits()).unwrap(); + assert_eq!((i.kind, i.version, i.generator.as_deref(), i.supported), (Kind::VectorCraft, 3, Some("VectorCraft 0.1.0"), true)); + let i = info(&gzip(vectorcraft_json(4, "").as_bytes()), &limits()).unwrap(); + assert_eq!((i.version, i.supported), (4, true)); + + let i = info(&pcraft(&png(1, 1, 0), &png(1, 1, 0)), &limits()).unwrap(); + assert_eq!((i.kind, i.version, i.generator.as_deref(), i.supported), (Kind::PhotoCraft, 1, Some("photocraft-format 0.1.0"), true)); + + let i = info(ecproj(2).as_bytes(), &limits()).unwrap(); + assert_eq!((i.kind, i.version, i.generator.as_deref(), i.supported), (Kind::EffectCraft, 2, Some("0.12.0"), true)); + let i = info(br#"{"schema":1,"settings":{},"items":{},"next_id":1}"#, &limits()).unwrap(); + assert_eq!((i.version, i.generator), (1, None)); + + assert_eq!(info(PDF, &limits()), Err(Error::Unrecognized)); +} + +#[test] +fn too_new_files_are_reported_and_refused() { + // Newer versions may change everything after the version: a member of a new shape. + let vc = vectorcraft_json(5, r#""pdf":"elsewhere""#); + let manifest = r#"{"format_version":2,"generator":"photocraft-format 9","document":"elsewhere","thumbnail":7}"#; + let pc = zip(&[store("manifest.json", manifest.as_bytes())]); + let ec = r#"{"savedBy":"9.0.0","schema":3,"items":[]}"#.to_string(); + for (bytes, kind, version) in [(vc.into_bytes(), Kind::VectorCraft, 5), (pc, Kind::PhotoCraft, 2), (ec.into_bytes(), Kind::EffectCraft, 3)] { + let i = info(&bytes, &limits()).unwrap(); + assert_eq!((i.kind, i.version, i.supported), (kind, version, false)); + let refused = Error::TooNew { kind, version, supported: kind.supported_version() }; + assert_eq!(renditions(&bytes, &limits()), Err(refused.clone()), "{kind:?}"); + assert_eq!(outline(&bytes, &limits()), Err(refused), "{kind:?}"); + } +} + +#[test] +fn a_pcraft_format_version_of_zero_is_malformed() { + let pc = zip(&[store("manifest.json", pcraft_manifest(0, "[]", "").as_bytes())]); + assert!(matches!(info(&pc, &limits()), Err(Error::Malformed(_)))); + assert!(matches!(outline(&pc, &limits()), Err(Error::Malformed(_)))); +} + +#[test] +fn vectorcraft_members_inline_as_base64() { + let preview = png(16, 8, 200); + let members = format!(r#""preview":{},"pdf":{}"#, inline("image/png", &preview), inline("application/pdf", PDF)); + let json = vectorcraft_json(3, &members); + for bytes in [json.clone().into_bytes(), gzip(json.as_bytes())] { + let r = renditions(&bytes, &limits()).unwrap(); + assert_eq!(r.len(), 2); + assert_eq!((r[0].kind, r[0].mime.as_str(), r[0].bytes.as_slice()), (RenditionKind::Thumbnail, "image/png", preview.as_slice())); + assert_eq!((r[1].kind, r[1].mime.as_str(), r[1].bytes.as_slice()), (RenditionKind::Pdf, "application/pdf", PDF)); + } +} + +#[test] +fn vectorcraft_members_after_the_json() { + // Version 4: the PDF after the marker, behind an image the document uses; the preview inline. + let image = vec![0xAB; 9000]; + let preview = png(3, 3, 1); + let members = format!(r#""preview":{},"pdf":{}"#, inline("image/png", &preview), blob("application/pdf", image.len(), PDF.len())); + let mut file = vectorcraft_json(4, &members).into_bytes(); + file.extend(BLOB_MAGIC); + file.extend(&image); + file.extend(PDF); + for bytes in [file.clone(), gzip(&file)] { + let r = renditions(&bytes, &limits()).unwrap(); + assert_eq!(r.iter().map(|r| r.kind).collect::>(), [RenditionKind::Thumbnail, RenditionKind::Pdf]); + assert_eq!(r[0].bytes, preview); + assert_eq!(r[1].bytes, PDF); + // The JSON alone answers info and outline. + assert_eq!(info(&bytes, &limits()).unwrap().version, 4); + assert_eq!(outline(&bytes, &limits()).unwrap().pages.len(), 2); + } +} + +#[test] +fn a_compressed_marker_across_the_reader_s_64_kib_chunks() { + // The marker starts at every offset around the first 64 KiB boundary of the unpacked stream. + let base = vectorcraft_json(4, &format!(r#""pdf":{},"pad":"""#, blob("application/pdf", 0, PDF.len()))); + for json_len in (64 << 10) - 12..(64 << 10) + 2 { + let json = base.replacen(r#""pad":"""#, &format!(r#""pad":"{}""#, "x".repeat(json_len - base.len())), 1); + assert_eq!(json.len(), json_len); + let mut file = json.into_bytes(); + file.extend(BLOB_MAGIC); + file.extend(PDF); + let gz = gzip(&file); + assert_eq!(info(&gz, &limits()).unwrap().version, 4, "{json_len}"); + assert_eq!(outline(&gz, &limits()).unwrap().pages.len(), 2, "{json_len}"); + assert_eq!(renditions(&gz, &limits()).unwrap()[0].bytes, PDF, "{json_len}"); + } +} + +#[test] +fn a_padded_base64_member_fits_a_limit_of_its_own_size() { + for len in 1..=3 { + let data = vec![7u8; len]; + let bytes = vectorcraft_json(3, &format!(r#""preview":{}"#, inline("image/png", &data))).into_bytes(); + let mut l = limits(); + l.max_entry = len as u64; + assert_eq!(renditions(&bytes, &l).unwrap()[0].bytes, data); + } +} + +#[test] +fn vectorcraft_without_members_has_no_renditions() { + let json = vectorcraft_json(4, ""); + assert_eq!(renditions(json.as_bytes(), &limits()).unwrap(), vec![]); + assert_eq!(renditions(&gzip(json.as_bytes()), &limits()).unwrap(), vec![]); +} + +#[test] +fn vectorcraft_outline_has_artboards_and_top_level_layers() { + for bytes in [vectorcraft_json(4, "").into_bytes(), gzip(vectorcraft_json(4, "").as_bytes())] { + let o = outline(&bytes, &limits()).unwrap(); + assert_eq!(o.canvas, None); + let pages: Vec<_> = o.pages.iter().map(|p| (p.name.as_str(), p.x, p.y, p.width, p.height)).collect(); + assert_eq!(pages, [("Front", 0.0, 0.0, 612.0, 792.0), ("Back", 650.0, 700.0, 50.0, 100.0)]); + assert!(o.pages.iter().all(|p| p.timing.is_none() && p.layers.is_empty())); + let layers: Vec<_> = o.layers.iter().map(|l| (l.name.as_str(), l.kind.as_str(), l.visible, l.opacity, l.blend.as_deref())).collect(); + assert_eq!(layers, [("Art", "layer", true, Some(1.0), Some("Normal")), ("Notes", "layer", false, Some(0.5), Some("Multiply"))]); + assert!(o.layers.iter().all(|l| l.children.is_empty())); + } +} + +#[test] +fn pcraft_previews_stored_and_deflated_with_the_manifest_last() { + let (thumb, composite) = (png(4, 3, 10), png(40, 30, 20)); + let r = renditions(&pcraft(&thumb, &composite), &limits()).unwrap(); + assert_eq!(r.len(), 2); + assert_eq!((r[0].kind, r[0].mime.as_str(), &r[0].bytes), (RenditionKind::Thumbnail, "image/png", &thumb)); + assert_eq!((r[1].kind, r[1].mime.as_str(), &r[1].bytes), (RenditionKind::Composite, "image/png", &composite)); +} + +#[test] +fn pcraft_without_previews_has_none() { + let pc = zip(&[store("manifest.json", pcraft_manifest(1, "[]", "").as_bytes())]); + assert_eq!(renditions(&pc, &limits()).unwrap(), vec![]); + let missing = zip(&[store("manifest.json", pcraft_manifest(1, "[]", r#","thumbnail":"thumb.png""#).as_bytes())]); + assert!(matches!(renditions(&missing, &limits()), Err(Error::Malformed(_)))); +} + +#[test] +fn pcraft_outline_has_the_canvas_and_layer_tree() { + let o = outline(&pcraft(&png(1, 1, 0), &png(1, 1, 0)), &limits()).unwrap(); + let c = o.canvas.unwrap(); + assert_eq!((c.width, c.height, c.resolution_dpi, c.mode.as_str(), c.depth.as_str()), (640, 480, 300.0, "Rgb", "U16")); + assert!(o.pages.is_empty()); + let layers: Vec<_> = o.layers.iter().map(|l| (l.name.as_str(), l.kind.as_str(), l.visible, l.opacity, l.blend.as_deref())).collect(); + assert_eq!(layers, [("Background", "raster", true, Some(1.0), Some("Normal")), ("Group", "group", false, Some(0.25), Some("Screen"))]); + let inner = &o.layers[1].children; + assert_eq!(inner.len(), 1); + assert_eq!((inner[0].name.as_str(), inner[0].blend.as_deref(), inner[0].opacity), ("Inner", Some("Multiply"), Some(0.5))); +} + +#[test] +fn pcraft_groups_nest_to_the_depth_cap() { + // 3 levels to the layer list, 3 per group and 4 for the leaf: 397 of the 400 levels. + let mut layer = pcraft_layer("Leaf", true, "Normal", 1.0, raster()); + for i in 0..130 { + layer = pcraft_layer(&format!("G{i}"), true, "Normal", 1.0, &format!(r#"{{"kind":"group","expanded":false,"children":[{layer}]}}"#)); + } + let pc = zip(&[deflated("manifest.json", pcraft_manifest(1, &format!("[{layer}]"), "").as_bytes())]); + let o = outline(&pc, &limits()).unwrap(); + let mut depth = 0; + let mut l = &o.layers[0]; + while let Some(c) = l.children.first() { + depth += 1; + l = c; + } + assert_eq!((depth, l.name.as_str()), (130, "Leaf")); +} + +#[test] +fn ecproj_has_no_renditions_and_lists_its_comps() { + let p = ecproj(2); + assert_eq!(renditions(p.as_bytes(), &limits()).unwrap(), vec![]); + let o = outline(p.as_bytes(), &limits()).unwrap(); + assert!(o.canvas.is_none() && o.layers.is_empty()); + // By item id (3 before 10), the folder and the footage left out. + let pages: Vec<_> = o.pages.iter().map(|p| (p.name.as_str(), p.width, p.height)).collect(); + assert_eq!(pages, [("Intro", 640.0, 360.0), ("Main", 1920.0, 1080.0)]); + let t = o.pages[1].timing.unwrap(); + assert_eq!((t.frame_rate_num, t.frame_rate_den, t.duration_seconds), (24000, 1001, 5.0)); + assert_eq!(o.pages[0].timing.unwrap().duration_seconds, 1.0); + // Bottom first: the file lists the top layer first. + let layers: Vec<_> = o.pages[1].layers.iter().map(|l| (l.name.as_str(), l.kind.as_str(), l.visible, l.opacity, l.blend.as_deref())).collect(); + assert_eq!(layers, [("Plate", "Footage", false, None, Some("Normal")), ("Title", "Text", true, None, Some("Add"))]); +} + +#[test] +fn an_ecproj_comp_without_a_duration_has_no_timing() { + let p = ecproj(2).replacen(r#""duration": 254016000000, "#, "", 1); + let o = outline(p.as_bytes(), &limits()).unwrap(); + assert_eq!((o.pages[0].name.as_str(), o.pages[0].timing), ("Intro", None)); + assert!(o.pages[1].timing.is_some()); +} + +#[test] +fn kinds_name_their_extension_and_version() { + assert_eq!( + [Kind::PhotoCraft, Kind::VectorCraft, Kind::EffectCraft].map(|k| (k.extension(), k.supported_version())), + [("pcraft", 1), ("vectorcraft", 4), ("ecproj", 2)] + ); +} diff --git a/crates/doc/tests/hostile.rs b/crates/doc/tests/hostile.rs new file mode 100644 index 00000000..a5d22eb4 --- /dev/null +++ b/crates/doc/tests/hostile.rs @@ -0,0 +1,353 @@ +//! Truncated, oversized and hostile input: the reader returns an error within its `Limits` and +//! never panics. + +mod common; + +use common::*; +use craft_doc::{Error, Limits, info, outline, renditions, sniff}; +use proptest::prelude::*; + +fn small() -> Limits { + let mut l = Limits::default(); + l.max_json = 64 << 10; + l.max_entry = 64 << 10; + l.max_total = 256 << 10; + l +} + +fn limit_exceeded(r: Result) -> bool { + matches!(r, Err(Error::LimitExceeded(_))) +} + +/// Every call on `bytes`; none may panic. +fn exercise(bytes: &[u8], limits: &Limits) { + let _ = sniff(bytes); + let _ = info(bytes, limits); + let _ = renditions(bytes, limits); + let _ = outline(bytes, limits); +} + +fn samples() -> Vec> { + let members = format!(r#""preview":{},"pdf":{}"#, inline("image/png", &png(2, 2, 3)), blob("application/pdf", 0, PDF.len())); + let mut v4 = vectorcraft_json(4, &members).into_bytes(); + v4.extend(BLOB_MAGIC); + v4.extend(PDF); + vec![gzip(&v4), v4, pcraft(&png(3, 3, 1), &png(6, 6, 2)), ecproj(2).into_bytes()] +} + +#[test] +fn every_truncation_is_an_error_not_a_panic() { + for s in samples() { + for n in 0..s.len() { + exercise(&s[..n], &small()); + } + } +} + +#[test] +fn a_blob_longer_than_max_entry_is_refused_before_it_is_copied() { + let mut bytes = vectorcraft_json(4, &format!(r#""pdf":{}"#, blob("application/pdf", 0, 1 << 40))).into_bytes(); + bytes.extend(BLOB_MAGIC); + bytes.extend(PDF); + assert!(limit_exceeded(renditions(&bytes, &small()))); +} + +#[test] +fn a_blob_past_the_end_of_the_file_is_malformed() { + for at in [[PDF.len() as u64, 1], [u64::MAX, 1], [1, PDF.len() as u64]] { + let members = format!(r#""pdf":{{"mime":"application/pdf","blob":[{},{}]}}"#, at[0], at[1]); + let mut bytes = vectorcraft_json(4, &members).into_bytes(); + bytes.extend(BLOB_MAGIC); + bytes.extend(PDF); + assert!(matches!(renditions(&bytes, &small()), Err(Error::Malformed(_))), "{at:?}"); + } +} + +#[test] +fn base64_past_max_entry_is_refused() { + let big = vec![1u8; 100 << 10]; + let mut limits = small(); + limits.max_json = 1 << 20; + let bytes = vectorcraft_json(3, &format!(r#""pdf":{}"#, inline("application/pdf", &big))).into_bytes(); + assert!(limit_exceeded(renditions(&bytes, &limits))); + limits.max_entry = 1 << 20; + assert_eq!(renditions(&bytes, &limits).unwrap()[0].bytes, big); + let bad = vectorcraft_json(3, r#""pdf":{"mime":"application/pdf","data":"QU*D"}"#).into_bytes(); + assert!(matches!(renditions(&bad, &limits), Err(Error::Malformed(_)))); +} + +#[test] +fn a_gzip_bomb_stops_at_the_limits() { + // 8 MiB of padding in the JSON, compressed to a few KiB. + let pad = "x".repeat(8 << 20); + let json = vectorcraft_json(4, &format!(r#""pad":"{pad}""#)); + let gz = gzip(json.as_bytes()); + assert!(gz.len() < 64 << 10); + assert!(limit_exceeded(info(&gz, &small()))); + assert!(limit_exceeded(outline(&gz, &small()))); + assert!(limit_exceeded(renditions(&gz, &small()))); + // The data after the JSON counts against max_total as far as it is read. + let mut tail = vectorcraft_json(4, &format!(r#""pdf":{}"#, blob("application/pdf", 8 << 20, 1))).into_bytes(); + tail.extend(BLOB_MAGIC); + tail.extend(vec![0u8; (8 << 20) + 1]); + let gz = gzip(&tail); + assert!(limit_exceeded(renditions(&gz, &small()))); + let mut more = small(); + more.max_total = 9 << 20; + assert_eq!(renditions(&gz, &more).unwrap()[0].bytes, [0]); + // info stops at the marker and reads the JSON only. + assert_eq!(info(&gz, &small()).unwrap().version, 4); +} + +#[test] +fn a_compressed_member_past_max_entry_is_refused_before_the_tail_is_inflated() { + // A member declares 8 MiB; the data after the JSON is 8 MiB that compress to a few KiB. + let mut file = vectorcraft_json(4, &format!(r#""pdf":{}"#, blob("application/pdf", 0, 8 << 20))).into_bytes(); + file.extend(BLOB_MAGIC); + file.extend(vec![1u8; 8 << 20]); + let gz = gzip(&file); + let mut limits = small(); + // Enough for the tail, not for the member. + limits.max_total = 64 << 20; + match renditions(&gz, &limits) { + Err(Error::LimitExceeded(m)) => assert!(m.contains("max_entry"), "{m}"), + other => panic!("{other:?}"), + } +} + +#[test] +fn a_nul_outside_the_marker_is_malformed_without_reading_on() { + // A NUL where the marker is not, then 8 MiB that compress to almost nothing. + let mut file = vectorcraft_json(4, "").into_bytes(); + file.extend(b"\n\0NOTBLOBS\0"); + file.extend(vec![0u8; 8 << 20]); + let mut limits = small(); + limits.max_total = 1 << 20; + for bytes in [gzip(&file), file] { + assert!(matches!(info(&bytes, &limits), Err(Error::Malformed(_)))); + assert!(matches!(renditions(&bytes, &limits), Err(Error::Malformed(_)))); + assert!(matches!(outline(&bytes, &limits), Err(Error::Malformed(_)))); + } + // A NUL as the very first byte of the decompressed stream. + let mut first = b"\0".to_vec(); + first.extend(vectorcraft_json(4, "").as_bytes()); + exercise(&gzip(&first), &limits); +} + +#[test] +fn a_manifest_past_max_entry_is_refused() { + let pc = pcraft(&png(1, 1, 0), &png(1, 1, 0)); + let mut limits = small(); + limits.max_entry = 64; + assert!(limit_exceeded(info(&pc, &limits))); +} + +#[test] +fn an_uncompressed_document_past_max_json_is_refused() { + let json = vectorcraft_json(4, &format!(r#""pad":"{}""#, "x".repeat(100 << 10))); + assert!(limit_exceeded(info(json.as_bytes(), &small()))); + let ec = ecproj(2).replacen(r#""settings": {}"#, &format!(r#""settings": {{"pad":"{}"}}"#, "x".repeat(100 << 10)), 1); + assert!(limit_exceeded(outline(ec.as_bytes(), &small()))); +} + +#[test] +fn json_nested_past_max_depth_is_refused() { + let deep = format!("{}{}", "[".repeat(2000), "]".repeat(2000)); + let vc = vectorcraft_json(4, &format!(r#""deep":{deep}"#)); + assert!(limit_exceeded(outline(vc.as_bytes(), &Limits::default()))); + let ec = ecproj(2).replacen(r#""settings": {}"#, &format!(r#""settings": {deep}"#), 1); + assert!(limit_exceeded(info(ec.as_bytes(), &Limits::default()))); + let pc = zip(&[store("manifest.json", pcraft_manifest(1, &deep, "").as_bytes())]); + assert!(limit_exceeded(outline(&pc, &Limits::default()))); + // At the limit (the file's object is one level), the document still reads. + let n = Limits::default().max_depth as usize - 1; + let at_limit = vectorcraft_json(4, &format!(r#""deep":{}{}"#, "[".repeat(n), "]".repeat(n))); + assert_eq!(outline(at_limit.as_bytes(), &Limits::default()).unwrap().pages.len(), 2); + let past = vectorcraft_json(4, &format!(r#""deep":{}{}"#, "[".repeat(n + 1), "]".repeat(n + 1))); + assert!(limit_exceeded(outline(past.as_bytes(), &Limits::default()))); + // A raised limit stops at MAX_DEPTH: 2000 nested groups are refused, not parsed. + let mut raised = Limits::default(); + raised.max_depth = u32::MAX; + assert!(limit_exceeded(outline(past.as_bytes(), &raised))); + let mut layer = pcraft_layer("Leaf", true, "Normal", 1.0, raster()); + for _ in 0..2000 { + layer = pcraft_layer("G", true, "Normal", 1.0, &format!(r#"{{"kind":"group","expanded":false,"children":[{layer}]}}"#)); + } + let pc = zip(&[deflated("manifest.json", pcraft_manifest(1, &format!("[{layer}]"), "").as_bytes())]); + raised.max_json = 1 << 30; + raised.max_entry = 1 << 30; + assert!(limit_exceeded(outline(&pc, &raised))); +} + +#[test] +fn lists_past_max_items_are_refused() { + let with_max = |n: u64| { + let mut l = Limits::default(); + l.max_items = n; + l + }; + // Four project items and two layers; layers of `{}` count as well. + let ec = ecproj(2); + assert_eq!(outline(ec.as_bytes(), &with_max(6)).unwrap().pages.len(), 2); + assert!(limit_exceeded(outline(ec.as_bytes(), &with_max(5)))); + let empty = ec.replacen(r#""layers": []"#, &format!(r#""layers": [{}{{}}]"#, "{},".repeat(9)), 1); + assert_eq!(outline(empty.as_bytes(), &with_max(16)).unwrap().pages[0].layers.len(), 10); + assert!(limit_exceeded(outline(empty.as_bytes(), &with_max(15)))); + let null = ec.replacen(r#""layers": []"#, r#""layers": null"#, 1); + assert!(outline(null.as_bytes(), &with_max(6)).unwrap().pages[0].layers.is_empty()); + // Two artboards and two layers. + let vc = vectorcraft_json(4, ""); + assert_eq!(outline(vc.as_bytes(), &with_max(4)).unwrap().layers.len(), 2); + assert!(limit_exceeded(outline(vc.as_bytes(), &with_max(3)))); + // An element past the limit is refused before it is read. + let bad = vc.replacen(r#""artboards":["#, r#""artboards":[{"rect":0},"#, 1); + assert!(matches!(outline(bad.as_bytes(), &with_max(4)), Err(Error::Malformed(_)))); + assert!(limit_exceeded(outline(bad.as_bytes(), &with_max(0)))); + // A group and its one layer. + let leaf = pcraft_layer("Leaf", true, "Normal", 1.0, raster()); + let group = pcraft_layer("G", true, "Normal", 1.0, &format!(r#"{{"kind":"group","expanded":false,"children":[{leaf}]}}"#)); + let pc = zip(&[store("manifest.json", pcraft_manifest(1, &format!("[{group}]"), "").as_bytes())]); + assert_eq!(outline(&pc, &with_max(2)).unwrap().layers[0].children.len(), 1); + assert!(limit_exceeded(outline(&pc, &with_max(1)))); +} + +#[test] +fn only_the_json_of_a_compressed_file_is_counted_when_no_member_follows_it() { + let mut file = vectorcraft_json(4, "").into_bytes(); + let json_len = file.len() as u64; + file.extend(BLOB_MAGIC); + file.extend((0..1u32 << 20).map(|i| (i % 251) as u8)); + let gz = gzip(&file); + let mut limits = small(); + limits.max_total = json_len + BLOB_MAGIC.len() as u64; + assert_eq!(info(&gz, &limits).unwrap().version, 4); + assert_eq!(outline(&gz, &limits).unwrap().pages.len(), 2); + assert_eq!(renditions(&gz, &limits).unwrap(), vec![]); +} + +#[test] +fn a_zip_entry_past_max_entry_is_refused_before_it_is_inflated() { + let big = vec![0u8; 1 << 20]; + let pc = zip(&[ + store("manifest.json", pcraft_manifest(1, "[]", r#","composite":"composite/preview.png""#).as_bytes()), + deflated("composite/preview.png", &big), + ]); + assert!(limit_exceeded(renditions(&pc, &small()))); + let mut limits = small(); + limits.max_entry = 2 << 20; + limits.max_total = 4 << 20; + assert_eq!(renditions(&pc, &limits).unwrap()[0].bytes.len(), big.len()); +} + +/// Where the central directory entry of the `index`th entry starts. +fn central(zip: &[u8], index: usize) -> usize { + zip.windows(4).enumerate().filter(|(_, w)| *w == [0x50, 0x4b, 0x01, 0x02]).nth(index).unwrap().0 +} + +#[test] +fn a_deflate_entry_that_inflates_past_its_size_is_malformed() { + let manifest = pcraft_manifest(1, "[]", r#","composite":"c.png""#); + let mut pc = zip(&[store("manifest.json", manifest.as_bytes()), deflated("c.png", &vec![5u8; 1 << 20])]); + // Declare 16 bytes in the central directory. + let c = central(&pc, 1); + pc[c + 24..c + 28].copy_from_slice(&16u32.to_le_bytes()); + assert!(matches!(renditions(&pc, &small()), Err(Error::Malformed(_)))); +} + +#[test] +fn zip_features_outside_the_reader_are_unsupported() { + let manifest = pcraft_manifest(1, "[]", ""); + let base = zip(&[store("manifest.json", manifest.as_bytes())]); + let c = central(&base, 0); + let mut encrypted = base.clone(); + encrypted[c + 8] |= 1; + assert!(matches!(info(&encrypted, &small()), Err(Error::Unsupported(_)))); + let mut method = base.clone(); + method[c + 10] = 12; + assert!(matches!(info(&method, &small()), Err(Error::Unsupported(_)))); + let mut zip64 = base.clone(); + zip64[c + 20..c + 24].copy_from_slice(&u32::MAX.to_le_bytes()); + assert!(matches!(info(&zip64, &small()), Err(Error::Unsupported(_)))); + // A ZIP64 end locator in front of the end record. + let eocd = base.len() - 22; + let mut locator = base[..eocd].to_vec(); + locator.extend(0x0706_4b50u32.to_le_bytes()); + locator.extend([0u8; 16]); + locator.extend(&base[eocd..]); + assert!(matches!(info(&locator, &small()), Err(Error::Unsupported(_)))); + let mut crc = base.clone(); + crc[c + 16] ^= 0xff; + assert!(matches!(info(&crc, &small()), Err(Error::Malformed(_)))); +} + +#[test] +fn a_comment_holding_an_end_signature_does_not_hide_the_archive() { + let mut pc = pcraft(&png(1, 1, 0), &png(1, 1, 0)); + let comment = b"PK\x05\x06 not the end record"; + let n = pc.len(); + pc[n - 2..].copy_from_slice(&(comment.len() as u16).to_le_bytes()); + pc.extend(comment); + assert_eq!(info(&pc, &small()).unwrap().version, 1); +} + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + // xorshift64* + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + self.0.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + fn below(&mut self, n: usize) -> usize { + (self.next() % n.max(1) as u64) as usize + } +} + +#[test] +fn seeded_mutations_never_panic() { + let mut rng = Rng(0x5eed); + for s in samples() { + for _ in 0..1500 { + let mut d = s.clone(); + for _ in 0..1 + rng.below(4) { + let at = rng.below(d.len()); + match rng.below(4) { + 0 => d[at] ^= 1 << rng.below(8), + 1 => d[at] = rng.next() as u8, + 2 => { + let v = [0u32, 1, u32::MAX, 0x8000_0000][rng.below(4)]; + let end = (at + 4).min(d.len()); + d[at..end].copy_from_slice(&v.to_le_bytes()[..end - at]); + } + _ => { + let end = (at + 1 + rng.below(64)).min(d.len()); + d.drain(at..end); + if d.is_empty() { + break; + } + } + } + } + exercise(&d, &small()); + } + } +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(256))] + + #[test] + fn arbitrary_bytes_never_panic(bytes in proptest::collection::vec(any::(), 0..512)) { + exercise(&bytes, &small()); + } + + #[test] + fn arbitrary_bytes_behind_each_magic_never_panic(prefix in 0usize..4, rest in proptest::collection::vec(any::(), 0..512)) { + let heads: [&[u8]; 4] = [b"PK\x03\x04", &[0x1f, 0x8b, 8, 0], br#"{"format":"vectorcraft","#, br#"{"savedBy":"1","schema":"#]; + let mut bytes = heads[prefix].to_vec(); + bytes.extend(rest); + exercise(&bytes, &small()); + } +} diff --git a/crates/ecproj-format/Cargo.toml b/crates/ecproj-format/Cargo.toml new file mode 100644 index 00000000..367ddcf0 --- /dev/null +++ b/crates/ecproj-format/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "craft-ecproj-format" +description = "The envelope of EffectCraft .ecproj project files: savedBy, the schema version and its check, and the file object with the project and its saved view as type parameters." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/ecproj-format/README.md b/crates/ecproj-format/README.md new file mode 100644 index 00000000..55426f21 --- /dev/null +++ b/crates/ecproj-format/README.md @@ -0,0 +1,49 @@ +# craft-ecproj-format + +The envelope of EffectCraft `.ecproj` project files: the members around the project that +EffectCraft writes and other apps read. The project model, the saved view and the schema 1 +conversion stay in EffectCraft. No `unsafe`, no panics, and it builds for +`wasm32-unknown-unknown`. The dependencies are `serde`, `serde_json` and `thiserror`. + +```toml +craft-ecproj-format = { git = "https://github.com/storytold/craft-libs", rev = "" } +``` + +## The file + +A project file is one UTF-8 JSON object with no newline at the end: `savedBy`, then the saved +view's members (`activeComp`, `openComps`, `compTimes`, each left out when empty), then the +project's members, starting with `schema`. Projects saved before `savedBy` was added start with +`schema`. EffectCraft indents the object with two spaces, or writes it on one line when Settings ▸ +Project ▸ Save Compact Project Files is on. The crate writes and reads both forms. + +## Use + +```rust +use craft_ecproj_format::{File, Head, check_schema}; + +// Writing +check_schema(project.schema)?; +let text = File::new(APP_VERSION, &view, &project).to_json()?; + +// Reading +let head = Head::read(&text)?; +``` + +- `SCHEMA_VERSION` is the newest project schema, 2. +- `check_schema(schema) -> Result<(), TooNew>` refuses a schema newer than `SCHEMA_VERSION`. + `TooNew` reads "project schema 3 is newer than this version supports (2)". It is + `#[non_exhaustive]`: other crates read its `schema` and `supported` fields. +- `File { saved_by, view, project }` serializes as one object, with `view` and `project` + flattened into it. `to_json()` gives the file's text indented with two spaces, and + `to_json_compact()` gives it on one line. Neither checks the values: `serde_json` writes NaN and + infinity as `null`, so the caller checks its values before saving. +- `Head::read(text) -> Result` reads `savedBy` and `schema` from one JSON + object, and skips other members. `savedBy` is `None` when it is absent, is not a string, or + appears more than once. Text that is not a JSON object, has no `schema` that is a `u32`, or has + two `schema` members gives `ReadError::Json`. A schema newer than `SCHEMA_VERSION` gives + `ReadError::TooNew { too_new, saved_by, .. }`, which reads as `TooNew` does and holds the file's + `savedBy`, so a message can name the version that wrote it. `ReadError` and its `TooNew` + variant are `#[non_exhaustive]`: match them with a wildcard arm and `..`. + +`.ecprojx` files hold the same object encoded as XML, which this crate does not read or write. diff --git a/crates/ecproj-format/src/lib.rs b/crates/ecproj-format/src/lib.rs new file mode 100644 index 00000000..344d14ed --- /dev/null +++ b/crates/ecproj-format/src/lib.rs @@ -0,0 +1,311 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! The envelope of EffectCraft `.ecproj` project files, for EffectCraft, which writes them, and +//! for the apps that read them. +//! +//! A project file is one UTF-8 JSON object. EffectCraft indents it with two spaces, or writes it +//! on one line when Settings ▸ Project ▸ Save Compact Project Files is on: +//! +//! ```text +//! { +//! "savedBy": "0.12.0", +//! "activeComp": 1, +//! "openComps": [1], +//! "schema": 2, +//! "settings": { ... }, +//! "items": { "1": { ... } }, +//! "next_id": 2 +//! } +//! ``` +//! +//! - `savedBy` is the version of EffectCraft that wrote the file. Projects saved before it was +//! added start with `schema`. +//! - The saved view (`activeComp`, `openComps`, `compTimes`) and the project's own members follow +//! in the same object. Both are type parameters of [`File`]; their types stay in EffectCraft. +//! - `schema` is the project schema. [`check_schema`] refuses a schema newer than +//! [`SCHEMA_VERSION`]. +//! +//! [`File`] writes the object: [`File::to_json`] indented, [`File::to_json_compact`] on one line. +//! [`Head::read`] reads `savedBy` and `schema` from either, and refuses a schema newer than +//! [`SCHEMA_VERSION`]. +//! +//! ``` +//! use std::collections::BTreeMap; +//! +//! #[derive(serde::Serialize)] +//! struct Project { +//! schema: u32, +//! items: BTreeMap, +//! } +//! +//! let project = Project { schema: craft_ecproj_format::SCHEMA_VERSION, items: BTreeMap::from([(1, "Comp 1".to_string())]) }; +//! craft_ecproj_format::check_schema(project.schema)?; +//! let view = BTreeMap::::new(); +//! let file = craft_ecproj_format::File::new("0.12.0", &view, &project); +//! let text = file.to_json()?; +//! assert_eq!(text, "{\n \"savedBy\": \"0.12.0\",\n \"schema\": 2,\n \"items\": {\n \"1\": \"Comp 1\"\n }\n}"); +//! assert_eq!(file.to_json_compact()?, r#"{"savedBy":"0.12.0","schema":2,"items":{"1":"Comp 1"}}"#); +//! +//! let head = craft_ecproj_format::Head::read(&text)?; +//! assert_eq!((head.saved_by.as_deref(), head.schema), (Some("0.12.0"), 2)); +//! # Ok::<(), Box>(()) +//! ``` +//! +//! An `.ecprojx` file holds the same object encoded as XML. This crate reads and writes the JSON +//! form only. + +use std::borrow::Cow; +use std::fmt; + +use serde::de::{self, IgnoredAny, MapAccess, SeqAccess, Visitor}; +use serde::{Deserialize, Deserializer, Serialize}; + +/// The newest project schema. Schema 2 measures the positions of effects on layers without a +/// source rectangle (shape, text) from the top-left of their comp-sized bounds. EffectCraft +/// converts schema 1 projects when it opens them. +pub const SCHEMA_VERSION: u32 = 2; + +/// A project's schema is newer than [`SCHEMA_VERSION`]. +/// +/// Other crates read its fields but do not build one: +/// +/// ```compile_fail +/// let _ = craft_ecproj_format::TooNew { schema: 3, supported: 2 }; +/// ``` +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[error("project schema {schema} is newer than this version supports ({supported})")] +#[non_exhaustive] +pub struct TooNew { + /// The schema the project declares. + pub schema: u32, + /// [`SCHEMA_VERSION`]. + pub supported: u32, +} + +/// Refuses a project whose `schema` is newer than [`SCHEMA_VERSION`], before it is read or saved. +pub fn check_schema(schema: u32) -> Result<(), TooNew> { + if schema > SCHEMA_VERSION { Err(TooNew { schema, supported: SCHEMA_VERSION }) } else { Ok(()) } +} + +/// The object a project file holds: `savedBy`, then the members of `view`, then the members of +/// `project`. +/// +/// `view` and `project` must each serialize as a map or a struct, and their member names must not +/// repeat `savedBy` or each other. +/// +/// The values are written as they are, without a check: JSON has no NaN or infinity, and +/// `serde_json` writes either as `null`, which may not read back as the project. The caller +/// checks its values before saving. +#[derive(Debug, Serialize)] +pub struct File<'a, V, P> { + /// The version of the app that writes the file. + #[serde(rename = "savedBy")] + pub saved_by: &'a str, + /// The compositions that were open, and their times. + #[serde(flatten)] + pub view: &'a V, + /// The project. + #[serde(flatten)] + pub project: &'a P, +} + +impl<'a, V: Serialize, P: Serialize> File<'a, V, P> { + pub fn new(saved_by: &'a str, view: &'a V, project: &'a P) -> Self { + File { saved_by, view, project } + } + + /// The file's text, indented with two spaces, with no newline at the end. + pub fn to_json(&self) -> Result { + serde_json::to_string_pretty(self) + } + + /// The file's text on one line, with no spaces between tokens and no newline at the end. + pub fn to_json_compact(&self) -> Result { + serde_json::to_string(self) + } +} + +/// `savedBy` and `schema`, read from a project file by [`Head::read`]. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Head { + /// The version of the app that wrote the file. `None` for projects saved before `savedBy` was + /// added, and when `savedBy` is not a string or appears more than once. + pub saved_by: Option, + /// The project schema, no newer than [`SCHEMA_VERSION`]. + pub schema: u32, +} + +impl Head { + /// Reads `text`, a project file: one JSON object with a `schema` that is a `u32`. Other + /// members are skipped, and two `schema` members are refused. A schema newer than + /// [`SCHEMA_VERSION`] is refused with [`ReadError::TooNew`], which holds the file's `savedBy`. + pub fn read(text: &str) -> Result { + let mut deserializer = serde_json::Deserializer::from_str(text); + let head = deserializer.deserialize_map(HeadVisitor)?; + deserializer.end()?; + if let Err(too_new) = check_schema(head.schema) { + return Err(ReadError::TooNew { too_new, saved_by: head.saved_by }); + } + Ok(head) + } +} + +/// [`Head::read`] could not read a project file. +/// +/// Other crates match it with a wildcard arm, and match [`ReadError::TooNew`] with `..`: +/// +/// ``` +/// use craft_ecproj_format::{Head, ReadError}; +/// +/// let text = r#"{"savedBy": "0.13.0", "schema": 3}"#; +/// let message = match Head::read(text) { +/// Ok(_) => String::new(), +/// Err(ReadError::TooNew { too_new, saved_by, .. }) => format!("saved by {}: {too_new}", saved_by.unwrap_or_default()), +/// Err(e) => e.to_string(), +/// }; +/// assert_eq!(message, "saved by 0.13.0: project schema 3 is newer than this version supports (2)"); +/// ``` +/// +/// ```compile_fail +/// fn name(e: craft_ecproj_format::ReadError) -> &'static str { +/// match e { +/// craft_ecproj_format::ReadError::Json(_) => "json", +/// craft_ecproj_format::ReadError::TooNew { .. } => "too new", +/// } +/// } +/// ``` +/// +/// ```compile_fail +/// fn saved_by(e: craft_ecproj_format::ReadError) -> Option { +/// match e { +/// craft_ecproj_format::ReadError::TooNew { too_new: _, saved_by } => saved_by, +/// _ => None, +/// } +/// } +/// ``` +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum ReadError { + /// The text is not one JSON object with a `schema` that is a `u32`. + #[error(transparent)] + Json(#[from] serde_json::Error), + /// The schema is newer than [`SCHEMA_VERSION`]. Reads as [`TooNew`] does. + #[error("{too_new}")] + #[non_exhaustive] + TooNew { + /// The schema the file declares, and [`SCHEMA_VERSION`]. + too_new: TooNew, + /// The file's `savedBy`, read as [`Head::saved_by`] is. + saved_by: Option, + }, +} + +/// Reads a [`Head`] from a JSON object. Other members are skipped. +struct HeadVisitor; + +impl<'de> Visitor<'de> for HeadVisitor { + type Value = Head; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a project file object") + } + + fn visit_map>(self, mut map: A) -> Result { + let mut saved_by = None; + let mut seen_saved_by = false; + let mut schema = None; + while let Some(key) = map.next_key::>()? { + match &*key { + "savedBy" if seen_saved_by => { + map.next_value::()?; + saved_by = None; + } + "savedBy" => { + saved_by = map.next_value::()?.0; + seen_saved_by = true; + } + "schema" if schema.is_some() => return Err(de::Error::duplicate_field("schema")), + "schema" => schema = Some(map.next_value()?), + _ => { + map.next_value::()?; + } + } + } + let schema = schema.ok_or_else(|| de::Error::missing_field("schema"))?; + Ok(Head { saved_by, schema }) + } +} + +/// A `savedBy` value: the string, or `None` for any other value. Arrays and objects are skipped +/// element by element with [`IgnoredAny`], so a large or deeply nested value is not kept. +struct SavedBy(Option); + +impl<'de> Deserialize<'de> for SavedBy { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(SavedByVisitor) + } +} + +struct SavedByVisitor; + +impl<'de> Visitor<'de> for SavedByVisitor { + type Value = SavedBy; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("any JSON value") + } + + fn visit_str(self, v: &str) -> Result { + Ok(SavedBy(Some(v.to_owned()))) + } + + fn visit_string(self, v: String) -> Result { + Ok(SavedBy(Some(v))) + } + + fn visit_bool(self, _: bool) -> Result { + Ok(SavedBy(None)) + } + + fn visit_i64(self, _: i64) -> Result { + Ok(SavedBy(None)) + } + + fn visit_i128(self, _: i128) -> Result { + Ok(SavedBy(None)) + } + + fn visit_u64(self, _: u64) -> Result { + Ok(SavedBy(None)) + } + + fn visit_u128(self, _: u128) -> Result { + Ok(SavedBy(None)) + } + + fn visit_f64(self, _: f64) -> Result { + Ok(SavedBy(None)) + } + + fn visit_unit(self) -> Result { + Ok(SavedBy(None)) + } + + fn visit_none(self) -> Result { + Ok(SavedBy(None)) + } + + fn visit_some>(self, deserializer: D) -> Result { + deserializer.deserialize_ignored_any(IgnoredAny)?; + Ok(SavedBy(None)) + } + + fn visit_seq>(self, mut seq: A) -> Result { + while seq.next_element::()?.is_some() {} + Ok(SavedBy(None)) + } + + fn visit_map>(self, mut map: A) -> Result { + while map.next_entry::()?.is_some() {} + Ok(SavedBy(None)) + } +} diff --git a/crates/ecproj-format/tests/envelope.rs b/crates/ecproj-format/tests/envelope.rs new file mode 100644 index 00000000..b6443d4c --- /dev/null +++ b/crates/ecproj-format/tests/envelope.rs @@ -0,0 +1,209 @@ +use std::collections::BTreeMap; + +use craft_ecproj_format::{File, Head, ReadError, SCHEMA_VERSION, check_schema}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct View { + #[serde(skip_serializing_if = "Option::is_none")] + active_comp: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + open_comps: Vec, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + comp_times: BTreeMap, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +struct Settings { + bit_depth: String, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +struct Item { + id: u64, + name: String, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +struct Project { + schema: u32, + settings: Settings, + items: BTreeMap, + next_id: u64, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + render_queue: Vec, +} + +fn project() -> Project { + Project { + schema: 2, + settings: Settings { bit_depth: "Bpc8".into() }, + items: BTreeMap::from([(1, Item { id: 1, name: "Comp 1".into() }), (2, Item { id: 2, name: "clip.mov".into() })]), + next_id: 3, + render_queue: vec![], + } +} + +const PROJECT_MEMBERS: &str = r#" "schema": 2, + "settings": { + "bit_depth": "Bpc8" + }, + "items": { + "1": { + "id": 1, + "name": "Comp 1" + }, + "2": { + "id": 2, + "name": "clip.mov" + } + }, + "next_id": 3 +}"#; + +#[test] +fn a_file_with_a_saved_view_holds_saved_by_then_the_view_then_the_project() { + let view = View { active_comp: Some(1), open_comps: vec![1, 2], comp_times: BTreeMap::from([(1, 254_016_000_000)]) }; + let p = project(); + let text = File::new("0.12.0", &view, &p).to_json().unwrap(); + let expected = format!( + r#"{{ + "savedBy": "0.12.0", + "activeComp": 1, + "openComps": [ + 1, + 2 + ], + "compTimes": {{ + "1": 254016000000 + }}, +{PROJECT_MEMBERS}"# + ); + assert_eq!(text, expected); +} + +#[test] +fn a_file_with_an_empty_view_holds_saved_by_then_the_project() { + let p = project(); + let text = File::new("0.12.0", &View::default(), &p).to_json().unwrap(); + assert_eq!(text, format!("{{\n \"savedBy\": \"0.12.0\",\n{PROJECT_MEMBERS}")); +} + +#[test] +fn a_compact_file_holds_the_same_members_on_one_line() { + let view = View { active_comp: Some(1), open_comps: vec![1, 2], comp_times: BTreeMap::from([(1, 254_016_000_000)]) }; + let p = project(); + let text = File::new("0.12.0", &view, &p).to_json_compact().unwrap(); + assert_eq!( + text, + r#"{"savedBy":"0.12.0","activeComp":1,"openComps":[1,2],"compTimes":{"1":254016000000},"schema":2,"settings":{"bit_depth":"Bpc8"},"items":{"1":{"id":1,"name":"Comp 1"},"2":{"id":2,"name":"clip.mov"}},"next_id":3}"# + ); + assert_eq!(serde_json::from_str::(&text).unwrap(), p); + assert_eq!(Head::read(&text).unwrap(), Head { saved_by: Some("0.12.0".into()), schema: 2 }); + let pretty = File::new("0.12.0", &view, &p).to_json().unwrap(); + assert_eq!(serde_json::from_str::(&text).unwrap(), serde_json::from_str::(&pretty).unwrap()); +} + +#[test] +fn a_written_file_reads_back_as_the_project_and_its_head() { + let p = project(); + let view = View { active_comp: Some(1), ..View::default() }; + let text = File::new("0.12.0", &view, &p).to_json().unwrap(); + assert_eq!(serde_json::from_str::(&text).unwrap(), p); + assert_eq!(Head::read(&text).unwrap(), Head { saved_by: Some("0.12.0".into()), schema: 2 }); +} + +#[test] +fn a_project_saved_before_saved_by_has_a_head_without_it() { + let text = r#"{"schema": 1, "settings": {}, "items": {}, "next_id": 1}"#; + assert_eq!(Head::read(text).unwrap(), Head { saved_by: None, schema: 1 }); +} + +#[test] +fn a_head_needs_a_schema() { + assert!(Head::read(r#"{"savedBy": "0.12.0", "items": {}}"#).is_err()); + assert!(Head::read(r#"{"savedBy": "0.12.0", "schema": "2"}"#).is_err()); + assert!(Head::read(r#"{"savedBy": "0.13.0", "schema": null, "schema": "3"}"#).is_err()); + assert!(Head::read("[]").is_err()); + assert!(Head::read(r#"["0.12.0", 2]"#).is_err()); + assert!(Head::read(r#"{"schema": 2, "schema": 2}"#).is_err()); + assert!(Head::read(r#"{"schema": 2} {}"#).is_err()); + assert!(Head::read(r#"{"savedBy": "0.12.0", "schema": 2"#).is_err()); + assert!(Head::read("").is_err()); + assert!(Head::read("2").is_err()); +} + +#[test] +fn a_head_reads_a_saved_by_that_is_not_one_string_as_none() { + assert_eq!(Head::read(r#"{"savedBy": 12, "schema": 2}"#).unwrap(), Head { saved_by: None, schema: 2 }); + assert_eq!(Head::read(r#"{"savedBy": ["0.12.0"], "schema": 2}"#).unwrap(), Head { saved_by: None, schema: 2 }); + assert_eq!(Head::read(r#"{"savedBy": {"v": "0.12.0"}, "schema": 2}"#).unwrap(), Head { saved_by: None, schema: 2 }); + assert_eq!(Head::read(r#"{"savedBy": "a", "savedBy": "b", "schema": 2}"#).unwrap(), Head { saved_by: None, schema: 2 }); + assert_eq!(Head::read(r#"{"savedBy": "a", "schema": 2, "savedBy": 1, "savedBy": "c"}"#).unwrap(), Head { saved_by: None, schema: 2 }); + assert_eq!(Head::read(r#"{"schema": 2, "savedBy": "0.13.0-dev"}"#).unwrap(), Head { saved_by: Some("0.13.0-dev".into()), schema: 2 }); +} + +#[test] +fn a_head_reads_the_object_form_only() { + assert!(matches!(Head::read(r#"[2, {}, {}, 1]"#), Err(ReadError::Json(_)))); + assert!(matches!(Head::read("[1]"), Err(ReadError::Json(_)))); + assert!(matches!(Head::read(r#"["savedBy", "0.12.0"]"#), Err(ReadError::Json(_)))); +} + +#[test] +fn a_head_refuses_a_newer_schema() { + let Err(ReadError::TooNew { too_new, saved_by, .. }) = Head::read(r#"{"savedBy": "0.13.0", "schema": 3}"#) else { + panic!("schema 3 is newer than 2") + }; + assert_eq!((too_new.schema, too_new.supported), (3, 2)); + assert_eq!(saved_by.as_deref(), Some("0.13.0")); + let Err(e) = Head::read(r#"{"schema": 4294967295, "savedBy": 1}"#) else { panic!("schema u32::MAX is newer than 2") }; + assert_eq!(e.to_string(), "project schema 4294967295 is newer than this version supports (2)"); + let ReadError::TooNew { saved_by, .. } = e else { panic!("{e:?} is not TooNew") }; + assert_eq!(saved_by, None); + assert_eq!(Head::read(r#"{"savedBy": "0.12.0", "schema": 2}"#).unwrap(), Head { saved_by: Some("0.12.0".into()), schema: 2 }); +} + +#[test] +fn a_head_skips_other_members_of_any_type() { + let text = r#"{"items": [1, {"schema": 9}], "savedBy": null, "schema": 2, "next_id": 1.5}"#; + assert_eq!(Head::read(text).unwrap(), Head { saved_by: None, schema: 2 }); +} + +#[test] +fn a_deeply_nested_saved_by_is_skipped_like_other_members() { + let deep = format!("{}{}", "[".repeat(10_000), "]".repeat(10_000)); + let text = format!(r#"{{"savedBy": {deep}, "schema": 2, "items": {deep}}}"#); + assert_eq!(Head::read(&text).unwrap(), Head { saved_by: None, schema: 2 }); + let deep_map = format!("{}0{}", r#"{"v": "#.repeat(10_000), "}".repeat(10_000)); + let text = format!(r#"{{"savedBy": "0.12.0", "savedBy": {deep_map}, "schema": 2}}"#); + assert_eq!(Head::read(&text).unwrap(), Head { saved_by: None, schema: 2 }); +} + +#[test] +fn a_wide_saved_by_array_reads_as_none() { + let wide = format!("[{}0]", "0,".repeat(1_000_000)); + let text = format!(r#"{{"savedBy": {wide}, "schema": 2, "savedBy": {wide}}}"#); + assert_eq!(Head::read(&text).unwrap(), Head { saved_by: None, schema: 2 }); + let text = format!(r#"{{"schema": 2, "savedBy": "0.12.0", "savedBy": "{}"}}"#, "9".repeat(1_000_000)); + assert_eq!(Head::read(&text).unwrap(), Head { saved_by: None, schema: 2 }); +} + +#[test] +fn check_schema_refuses_only_a_newer_schema() { + assert_eq!(SCHEMA_VERSION, 2); + assert_eq!(check_schema(0), Ok(())); + assert_eq!(check_schema(1), Ok(())); + assert_eq!(check_schema(2), Ok(())); + for schema in [3, u32::MAX] { + let Err(e) = check_schema(schema) else { panic!("schema {schema} is newer than 2") }; + assert_eq!((e.schema, e.supported), (schema, 2)); + } +} + +#[test] +fn too_new_reads_as_effectcraft_words_it() { + let Err(e) = check_schema(3) else { panic!("schema 3 is newer than 2") }; + assert_eq!(e.to_string(), "project schema 3 is newer than this version supports (2)"); +} diff --git a/crates/font-store/Cargo.toml b/crates/font-store/Cargo.toml index 5e3ed270..5d78609f 100644 --- a/crates/font-store/Cargo.toml +++ b/crates/font-store/Cargo.toml @@ -25,6 +25,11 @@ thiserror = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] craft-atomic = { workspace = true } craft-font-catalog = { workspace = true } +log = "0.4" +# Language packs (`language_packs`): `.tar.zst` archives, unpacked in pure Rust. +ruzstd = { version = "0.9", default-features = false, features = ["std"] } +toml = { version = "1", default-features = false, features = ["parse", "serde", "std"] } +tar = { version = "0.4", default-features = false } # Blocking HTTPS with rustls and webpki roots: no async runtime, no OpenSSL. ureq = { version = "3", default-features = false, features = ["rustls", "gzip"] } diff --git a/crates/font-store/README.md b/crates/font-store/README.md index 12216302..520b0e0b 100644 --- a/crates/font-store/README.md +++ b/crates/font-store/README.md @@ -94,6 +94,8 @@ settings.json the network setting and the sources the user added manifest.json what is installed indexes/ the last indexes fetched: sources.json, .json.gz, user- / the font files and the licence +language-packs/ installed Language packs, one folder per locale +language-packs.json which Language packs are installed .staging/ downloads being installed .lock held while the store changes ``` @@ -113,6 +115,33 @@ A Google Fonts family's folder is google/fonts' own folder name (`lato/`); anoth The catalog scans the store folder. With `with_catalog(catalog)`, every install and removal ends with `catalog.rescan()`, which reads only new and changed files, so the app sees the fonts at once. Other running apps notice through `Catalog::rescan_if_changed` (when their window is activated) or `Catalog::watch`. +## Language packs + +The store's second item kind, `language-pack`: a language's spelling dictionary, hyphenation patterns and licence texts, published by [storytold/craft-i18n](https://github.com/storytold/craft-i18n) as `.tar.zst` release assets listed in an `index.json`, and read at runtime by `craft-proof` (`craft_proof::packs`). It reuses the store's folder, consent, HTTP client, staging and sha256 checks; there is no second downloader. + +```rust +const PACKS: PackIndexPin = PackIndexPin { url: "https://github.com/storytold/craft-i18n/releases/download/language-packs-1/index.json", sha256: "" }; +let index = store.fetch_language_pack_index(&PACKS, &Cancel::new())?; // or cached_language_pack_index(&PACKS) +let nl = index.pack("nl").unwrap(); +store.install_language_pack(&index, nl, &Cancel::new(), &mut |p| ())?; // network on, like fonts +craft_proof::packs::set_packs_dir(Some(store.language_packs_dir())); // then packs::reload() after each change +store.remove_language_pack("nl")?; +let installed = store.installed_language_packs(); +``` + +| Call | What it does | +|---|---| +| `fetch_language_pack_index(pin, cancel)` | Downloads the index, checks it against the pin's sha256, saves it as `indexes/language-packs.json`. Needs the network on. | +| `cached_language_pack_index(pin)` | The saved index, when it is the pinned one. | +| `install_language_pack(index, pack, cancel, progress)` | Downloads the archive (at most 128 MiB), checks size and sha256, unpacks it into `.staging/` and moves it into `language-packs//` with one rename (an older pack is replaced the same way). Records it in `language-packs.json`. | +| `remove_language_pack(locale)` | Moves the folder out at once, updates the record, deletes the files. No network needed. | +| `installed_language_packs()` | The recorded packs whose folder is there. | +| `language_packs_dir()` | `/language-packs`, for `craft_proof::packs::set_packs_dir`. | + +**Unpacking is hostile-input safe:** pure-Rust zstd (`ruzstd`, MIT; every frame is read, and nothing but tar padding may follow the archive) and `tar` (MIT OR Apache-2.0); regular files and the `LICENCES` folder only (no links or devices). Only the contract's names are allowed: `pack.toml`, `spelling.aff`, `spelling.dic`, `hyphenation.pat` at the root and `LICENCES/` (`[A-Za-z0-9._-]`, no leading or trailing dot, no Windows device names such as `CON` or `nul.txt`). Each file at most 64 MiB, 256 MiB and 256 entries per pack. `pack.toml` is required and must name the entry's locale; the files the index promises (`spelling`, `hyphenation`) must be there. Index entries need a lowercase BCP 47 locale (listed once, not a Windows device name), a sha256 and an `https://` URL; names, licences, versions, classes and the release lose control and bidi/format characters before they are stored or listed; an optional `source` must be an `https://` URL. At most 4096 zstd frames per archive. An unreadable `language-packs.json` is copied to `language-packs.json.bad` and rebuilt from the pack folders (each `pack.toml`) atomically in place, as is a missing one when pack folders are there, so no installed pack drops out of the list. Only an install or removal (under the store lock) rewrites it; listing reads the folders meanwhile and writes nothing. + +**Developer override (testing only — bypasses the pin):** `CRAFT_LANGUAGE_PACK_INDEX=` replaces the pinned index URL with an `http://` URL on a loopback host (`127.0.0.1`, `localhost`, `[::1]`), an `https://` URL or a local file path (`file://` URLs are not taken), and **skips the index's sha256 pin**. Archives are still checked against the sha256 and size in that index, and may then also be loopback `http://` URLs or local paths. Off unless the variable is set; it works in release builds too (for end-to-end tests of a real build), a warning is logged whenever it is used, and `language_pack_index_overridden() -> Option` returns the value so apps can show that the pin is bypassed. Never set it for users. + ## Web The index, the lookups, `verify_font` and `IndexPin` build for `wasm32-unknown-unknown`. The store folder and the HTTP client are native only; the web keeps fonts elsewhere. diff --git a/crates/font-store/src/http.rs b/crates/font-store/src/http.rs index 94a6bdd8..3c6d007b 100644 --- a/crates/font-store/src/http.rs +++ b/crates/font-store/src/http.rs @@ -111,6 +111,12 @@ impl Client { } } + /// A client that also allows plain `http://`, used only for the developer override of the + /// Language pack index (`CRAFT_LANGUAGE_PACK_INDEX`), which accepts only loopback hosts. + pub(crate) fn allowing_http(t: Timeouts) -> Self { + Self::build(t, false) + } + /// Plain `http://` to a local test server, with `https://` prefixes mapped onto it. #[cfg(test)] pub(crate) fn for_tests(t: Timeouts, rewrite: Vec<(String, String)>) -> Self { diff --git a/crates/font-store/src/index.rs b/crates/font-store/src/index.rs index 96585aa1..aad4bc45 100644 --- a/crates/font-store/src/index.rs +++ b/crates/font-store/src/index.rs @@ -24,25 +24,25 @@ pub const LICENSES: [&str; 3] = ["OFL", "APACHE2", "UFL"]; #[derive(Debug, thiserror::Error)] pub enum IndexError { /// More than [`MAX_INDEX_BYTES`]. - #[error("the font index is larger than {max} bytes")] + #[error("the index is larger than {max} bytes")] TooLarge { /// The limit. max: usize, }, /// Not JSON, or a field has the wrong type. - #[error("the font index can't be read: {0}")] + #[error("the index can't be read: {0}")] Json(String), /// A major version this crate doesn't know. - #[error("the font index has version {0}; this build reads version {MAJOR_VERSION}.x")] + #[error("the index has version {0}; this build reads version {MAJOR_VERSION}.x")] Version(String), /// An entry breaks the format's rules. - #[error("the font index has an invalid entry: {0}")] + #[error("the index has an invalid entry: {0}")] Invalid(String), /// A gzipped index that doesn't decompress. - #[error("the gzipped font index can't be read: {0}")] + #[error("the gzipped index can't be read: {0}")] Gzip(String), /// The bytes aren't the pinned ones. - #[error("the font index doesn't have the sha256 this build expects")] + #[error("the index doesn't have the sha256 this build expects")] Checksum, } diff --git a/crates/font-store/src/language_packs.rs b/crates/font-store/src/language_packs.rs new file mode 100644 index 00000000..282d5909 --- /dev/null +++ b/crates/font-store/src/language_packs.rs @@ -0,0 +1,722 @@ +//! The store's second item kind, `language-pack`: a language's spelling dictionary, hyphenation +//! patterns and licence texts, published by storytold/craft-i18n as `.tar.zst` release +//! assets listed in an `index.json`, and read at runtime by `craft-proof` (`craft_proof::packs`). +//! +//! It reuses the store's folder, consent ([`FontStore::enable_network`]), HTTP client and sha256 +//! checks: the app pins the index by URL and sha256 ([`PackIndexPin`]), each archive is checked +//! against the size and sha256 the index gives, then unpacked into `.staging/` and moved into +//! `language-packs//` with one rename. Packs have no update checks of their own: a new +//! pin is a new index. +//! +//! ```text +//! /language-packs// pack.toml, spelling.aff, spelling.dic, hyphenation.pat, LICENCES/… +//! /language-packs.json which packs are installed, from which index entry +//! /indexes/language-packs.json the last pack index fetched +//! ``` +//! +//! **Developer override (testing only):** `CRAFT_LANGUAGE_PACK_INDEX=` replaces the +//! pinned index URL with an `http://` URL on a loopback host (`127.0.0.1`, `localhost`, `[::1]`), +//! an `https://` URL or a local file path, and skips the index's sha256 pin. Archives are still +//! checked against the sha256 and size in that index, and may then also be loopback `http://` +//! URLs or local paths. It is off unless the variable is set, and logged as a warning when used. + +use std::io::Read; +use std::path::{Component, Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::http::{Cancel, Client, Timeouts}; +use crate::index::IndexError; +use crate::store::{Error, FontStore, INDEX_CACHE, Progress, Stage, io, read_limited}; +use crate::verify::{VerifyError, sha256_hex}; + +/// The store's folder of installed Language packs. +pub const LANGUAGE_PACKS: &str = "language-packs"; +/// The developer override of the pinned pack index (see the module docs). Testing only. +pub const LANGUAGE_PACK_INDEX_ENV: &str = "CRAFT_LANGUAGE_PACK_INDEX"; +/// Largest pack index read. +pub const MAX_PACK_INDEX_BYTES: u64 = 4 << 20; +/// Largest pack archive downloaded. +pub const MAX_PACK_ARCHIVE_BYTES: u64 = 128 << 20; +/// Largest file unpacked from a pack (craft-proof's dictionary cap). +pub const MAX_PACK_FILE_BYTES: u64 = 64 << 20; +/// Largest total unpacked size of a pack. +pub const MAX_PACK_UNPACKED_BYTES: u64 = 256 << 20; +/// Most files and folders in a pack archive. +pub const MAX_PACK_ENTRIES: usize = 256; +/// Most packs in an index. +const MAX_PACKS: usize = 1024; +const INSTALLED: &str = "language-packs.json"; +const INDEX_CACHE_NAME: &str = "language-packs.json"; + +/// The pack index an app trusts: `index.json` of a craft-i18n `language-packs-` release, by +/// URL and sha256 (the app's Pin). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PackIndexPin { + /// `https://github.com/storytold/craft-i18n/releases/download/language-packs-/index.json`. + pub url: &'static str, + /// The sha256 of that file, lowercase hex. + pub sha256: &'static str, +} + +/// `index.json` of a pack release. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PackIndex { + /// The format; 1. + pub format: u32, + /// The release (`language-packs-1`). + pub release: String, + /// The packs, by locale. + pub packs: Vec, +} + +/// One pack of a [`PackIndex`]. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PackEntry { + /// Lowercase BCP 47 tag (`nl`, `pt-br`); the pack's folder name. + pub locale: String, + /// The language's own name (`Nederlands`). + pub name: String, + /// The craft-i18n commit it was built from. + #[serde(default)] + pub version: String, + /// The archive (`.tar.zst`). + pub url: String, + /// The archive's sha256, lowercase hex. + pub sha256: String, + /// The archive's size in bytes. + pub size: u64, + /// SPDX licence expression. + pub licence: String, + /// The licence class: `Embeddable` or `Download-only`. + #[serde(default)] + pub class: String, + /// Whether it has a spelling dictionary. + #[serde(default)] + pub spelling: bool, + /// Whether it has hyphenation patterns. + #[serde(default)] + pub hyphenation: bool, + /// Where the data comes from (an `https://` page), for display; optional. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, +} + +/// An installed pack, as recorded when it was installed. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct InstalledLanguagePack { + /// Lowercase BCP 47 tag; the folder under [`LANGUAGE_PACKS`]. + pub locale: String, + /// The language's own name. + pub name: String, + /// The release it came from. + pub release: String, + /// The craft-i18n commit it was built from. + pub version: String, + /// SPDX licence expression. + pub licence: String, + /// The archive's sha256. + pub sha256: String, + /// The archive's size in bytes. + pub size: u64, +} + +#[derive(Default, Serialize, Deserialize)] +struct Installed { + version: u32, + packs: Vec, +} + +impl PackIndex { + /// Parses and checks an index: format 1, every locale a lowercase tag and listed once, every + /// archive at most [`MAX_PACK_ARCHIVE_BYTES`] with a sha256. Archive URLs must be `https://` + /// (with `local`, the developer override, also loopback `http://` or a path). + pub fn from_json(bytes: &[u8]) -> Result { + Self::parse(bytes, false) + } + + fn parse(bytes: &[u8], local: bool) -> Result { + let bad = |why: String| Error::Index(IndexError::Invalid(format!("Language pack index: {why}"))); + let mut index: PackIndex = serde_json::from_slice(bytes).map_err(|e| Error::Index(IndexError::Json(format!("Language pack index: {e}"))))?; + if index.format != 1 { + return Err(bad(format!("format {} is not supported", index.format))); + } + if index.packs.len() > MAX_PACKS { + return Err(bad("too many packs".into())); + } + for (i, p) in index.packs.iter().enumerate() { + if !is_locale(&p.locale) || index.packs.iter().take(i).any(|q| q.locale == p.locale) { + return Err(bad(format!("pack {:?}: not a lowercase language tag, or listed twice", p.locale.chars().take(40).collect::()))); + } + if p.size > MAX_PACK_ARCHIVE_BYTES || !is_sha256(&p.sha256) { + return Err(bad(format!("pack {}: bad size or sha256", p.locale))); + } + if !(p.url.starts_with("https://") || local && local_source(&p.url).is_some()) { + return Err(bad(format!("pack {}: {} is not an https URL", p.locale, p.url.chars().take(200).collect::()))); + } + if let Some(src) = &p.source + && !(src.starts_with("https://") && src.len() <= 2048 && !src.chars().any(|c| c.is_control() || is_format(c) || c == ' ')) + { + return Err(bad(format!("pack {}: source {:?} is not an https URL", p.locale, src.chars().take(200).collect::()))); + } + } + // Text shown to people loses control and bidi/format characters. + index.release = plain_text(&index.release); + for p in &mut index.packs { + for t in [&mut p.name, &mut p.version, &mut p.licence, &mut p.class] { + *t = plain_text(t); + } + } + Ok(index) + } + + /// The pack for `locale`. + pub fn pack(&self, locale: &str) -> Option<&PackEntry> { + self.packs.iter().find(|p| p.locale == locale) + } +} + +/// Where the pack index comes from: the pin, or the developer override. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum IndexSource { + /// The pinned URL, checked against the pinned sha256. + Pinned(PackIndexPin), + /// `CRAFT_LANGUAGE_PACK_INDEX`: no index pin; loopback `http://` and paths allowed. + Override(String), +} + +impl IndexSource { + /// The pin, unless [`LANGUAGE_PACK_INDEX_ENV`] is set. + fn from_env(pin: &PackIndexPin) -> Self { + match language_pack_index_overridden() { + Some(v) => { + log::warn!( + "{LANGUAGE_PACK_INDEX_ENV}={v}: the Language pack index is NOT the pinned one and its sha256 is not checked (testing only)" + ); + Self::Override(v) + } + None => Self::Pinned(*pin), + } + } + + fn is_override(&self) -> bool { + matches!(self, Self::Override(_)) + } +} + +/// The developer override of the pack index, when [`LANGUAGE_PACK_INDEX_ENV`] is set: the URL or +/// path that replaces the pinned index, whose sha256 is then **not** checked. Apps show a warning +/// in their Language pack interface while it is set. It works in release builds too, for +/// end-to-end testing against a local index. +pub fn language_pack_index_overridden() -> Option { + std::env::var(LANGUAGE_PACK_INDEX_ENV).ok().map(|v| v.trim().to_string()).filter(|v| !v.is_empty()) +} + +/// A source the developer override may read: `Ok(url)` for an `https://` URL or an `http://` +/// one on a loopback host, `Err(path)` for a local file path; `None` for anything else. +fn local_source(s: &str) -> Option> { + if s.starts_with("https://") { + return Some(Ok(s)); + } + if let Some(rest) = s.strip_prefix("http://") { + let host = rest.split(['/', '?', '#']).next().unwrap_or_default(); + let host = host.rsplit_once('@').map_or(host, |(_, h)| h); + let name = if host.starts_with('[') { + host.split(']').next().map(|h| format!("{h}]")).unwrap_or_default() + } else { + host.split(':').next().unwrap_or_default().to_string() + }; + return ["127.0.0.1", "localhost", "[::1]"].contains(&name.as_str()).then_some(Ok(s)); + } + if s.contains("://") { + return None; + } + Some(Err(PathBuf::from(s))) +} + +/// A lowercase BCP 47 tag as packs are named (craft-proof's rule): ASCII lowercase letters and +/// digits in parts of 1 to 8 joined by `-`, at most 35 characters. +fn is_locale(s: &str) -> bool { + !windows_reserved(s) + && !s.is_empty() + && s.len() <= 35 + && s.split('-').all(|p| !p.is_empty() && p.len() <= 8 && p.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit())) +} + +fn is_sha256(s: &str) -> bool { + s.len() == 64 && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +impl FontStore { + /// The folder of installed Language packs (`/language-packs`): give it to + /// `craft_proof::packs::set_packs_dir`. + pub fn language_packs_dir(&self) -> PathBuf { + self.dir().join(LANGUAGE_PACKS) + } + + /// The pack index saved by the last [`FontStore::fetch_language_pack_index`], when it is + /// `pin`'s (or the developer override is set). + pub fn cached_language_pack_index(&self, pin: &PackIndexPin) -> Option { + self.cached_pack_index_from(&IndexSource::from_env(pin)) + } + + pub(crate) fn cached_pack_index_from(&self, source: &IndexSource) -> Option { + let bytes = read_limited(&self.dir().join(INDEX_CACHE).join(INDEX_CACHE_NAME), MAX_PACK_INDEX_BYTES)?; + match source { + IndexSource::Pinned(pin) if sha256_hex(&bytes) != pin.sha256 => None, + _ => PackIndex::parse(&bytes, source.is_override()).ok(), + } + } + + /// Downloads the pack index `pin` names, checks it against the pin and saves it for + /// [`FontStore::cached_language_pack_index`]. Needs the network on. + pub fn fetch_language_pack_index(&self, pin: &PackIndexPin, cancel: &Cancel) -> Result { + self.fetch_pack_index_from(&IndexSource::from_env(pin), cancel) + } + + pub(crate) fn fetch_pack_index_from(&self, source: &IndexSource, cancel: &Cancel) -> Result { + self.require_network()?; + let bytes = match source { + IndexSource::Pinned(pin) => { + let check = |b: &[u8]| if sha256_hex(b) == pin.sha256 { Ok(()) } else { Err(VerifyError::Sha256) }; + self.fetch(&[pin.url.to_string()], MAX_PACK_INDEX_BYTES, "the Language pack index", cancel, &mut |_| {}, &check)? + } + IndexSource::Override(at) => self.get_local(at, MAX_PACK_INDEX_BYTES, "the Language pack index", cancel, &mut |_| {}, &|_| Ok(()))?, + }; + let index = PackIndex::parse(&bytes, source.is_override())?; + std::fs::create_dir_all(self.dir().join(INDEX_CACHE)).map_err(io("creating the index folder"))?; + self.write_file(&format!("{INDEX_CACHE}/{INDEX_CACHE_NAME}"), &bytes)?; + Ok(index) + } + + /// A download under the developer override: an https or loopback http URL, or a file. + fn get_local( + &self, + at: &str, + max: u64, + what: &str, + cancel: &Cancel, + on_bytes: &mut dyn FnMut(u64), + check: &dyn Fn(&[u8]) -> Result<(), VerifyError>, + ) -> Result, Error> { + let bytes = match local_source(at) { + Some(Ok(url)) if url.starts_with("https://") => return self.fetch(&[url.to_string()], max, what, cancel, on_bytes, check), + Some(Ok(url)) => Client::allowing_http(Timeouts::default()) + .get(url, max, cancel, on_bytes) + .map_err(|e| Error::Download { what: what.to_string(), reasons: vec![e.to_string()] })?, + Some(Err(path)) => { + let mut buf = Vec::new(); + std::fs::File::open(&path) + .and_then(|f| f.take(max.saturating_add(1)).read_to_end(&mut buf)) + .map_err(io(format!("reading {}", path.display())))?; + if buf.len() as u64 > max { + return Err(Error::Download { what: what.to_string(), reasons: vec![format!("larger than {max} bytes")] }); + } + on_bytes(buf.len() as u64); + buf + } + None => return Err(Error::BadUrl(at.chars().take(200).collect())), + }; + check(&bytes).map_err(|error| Error::Verify { file: what.to_string(), error })?; + Ok(bytes) + } + + /// Downloads `pack` from `index`, checks its size and sha256, unpacks it and moves it into + /// `language-packs//` in one rename, replacing an older one. Needs the network on. + /// Then call `craft_proof::packs::reload()`. + pub fn install_language_pack( + &self, + index: &PackIndex, + pack: &PackEntry, + cancel: &Cancel, + progress: &mut dyn FnMut(Progress), + ) -> Result { + let local = language_pack_index_overridden().is_some(); + self.install_pack_from(local, index, pack, cancel, progress) + } + + pub(crate) fn install_pack_from( + &self, + local: bool, + index: &PackIndex, + pack: &PackEntry, + cancel: &Cancel, + progress: &mut dyn FnMut(Progress), + ) -> Result { + self.require_network()?; + // The entry is checked again here: callers may build one by hand. + if !is_locale(&pack.locale) || !is_sha256(&pack.sha256) || pack.size > MAX_PACK_ARCHIVE_BYTES { + let locale: String = pack.locale.chars().take(40).collect(); + return Err(Error::Index(IndexError::Invalid(format!("Language pack {locale:?}: bad locale, sha256 or size")))); + } + let what = format!("{}.tar.zst", pack.locale); + let (size, sha) = (pack.size, pack.sha256.clone()); + let check = move |b: &[u8]| { + if b.len() as u64 != size { + Err(VerifyError::Size { expected: size, actual: b.len() as u64 }) + } else if sha256_hex(b) != sha { + Err(VerifyError::Sha256) + } else { + Ok(()) + } + }; + let mut on_bytes = |done: u64| progress(Progress { file: 0, files: 1, done, total: size }); + let bytes = if local { + self.get_local(&pack.url, size, &what, cancel, &mut on_bytes, &check)? + } else if pack.url.starts_with("https://") { + self.fetch(std::slice::from_ref(&pack.url), size, &what, cancel, &mut on_bytes, &check)? + } else { + return Err(Error::BadUrl(pack.url.chars().take(200).collect())); + }; + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + self.sweep_staging(); + let stage = Stage::new(self.dir())?; + let unpacked = stage.dir.join(&pack.locale); + unpack(&bytes, &unpacked, pack).map_err(|e| Error::Io { what: format!("unpacking {what}"), source: std::io::Error::other(e) })?; + let record = InstalledLanguagePack { + locale: pack.locale.clone(), + name: plain_text(&pack.name), + release: plain_text(&index.release), + version: plain_text(&pack.version), + licence: plain_text(&pack.licence), + sha256: pack.sha256.clone(), + size: pack.size, + }; + let _lock = self.lock()?; + // The record first: rebuilding a damaged one must not count the pack being installed. + let mut installed = self.read_installed()?; + let packs = self.language_packs_dir(); + std::fs::create_dir_all(&packs).map_err(io("creating the Language packs folder"))?; + let dest = packs.join(&pack.locale); + let old = stage.dir.join(format!("{}.old", pack.locale)); + let had = std::fs::symlink_metadata(&dest).is_ok(); + if had { + std::fs::rename(&dest, &old).map_err(io(format!("replacing the {} pack", pack.locale)))?; + } + if let Err(e) = std::fs::rename(&unpacked, &dest) { + if had { + let _ = std::fs::rename(&old, &dest); + } + return Err(io(format!("installing the {} pack", pack.locale))(e)); + } + installed.packs.retain(|p| p.locale != record.locale); + installed.packs.push(record.clone()); + installed.packs.sort_by(|a, b| a.locale.cmp(&b.locale)); + if let Err(e) = self.write_installed(&installed) { + // Put the old pack back, so folder and record agree. + let _ = std::fs::remove_dir_all(&dest); + if had { + let _ = std::fs::rename(&old, &dest); + } + return Err(e); + } + Ok(record) + } + + /// The installed Language packs whose folder is there, by locale. + pub fn installed_language_packs(&self) -> Vec { + let dir = self.language_packs_dir(); + // Read only: without the store lock this never writes the record (an install or removal + // may be under way); a record that can't be read is listed from the folders and repaired + // by the next install or removal, which hold the lock. + let installed = self.read_record().unwrap_or_else(|| self.installed_from_disk()); + installed + .packs + .into_iter() + .filter(|p| is_locale(&p.locale) && dir.join(&p.locale).is_dir()) + .map(|p| InstalledLanguagePack { + name: plain_text(&p.name), + release: plain_text(&p.release), + version: plain_text(&p.version), + licence: plain_text(&p.licence), + ..p + }) + .collect() + } + + /// Removes the pack for `locale`: its folder disappears at once (one rename) and is deleted + /// after. No network needed. Then call `craft_proof::packs::reload()`. + pub fn remove_language_pack(&self, locale: &str) -> Result<(), Error> { + if !is_locale(locale) { + return Err(Error::NotInstalled(locale.chars().take(40).collect())); + } + let stage = Stage::new(self.dir())?; + let _lock = self.lock()?; + let dest = self.language_packs_dir().join(locale); + let mut installed = self.read_installed()?; + let known = installed.packs.iter().any(|p| p.locale == locale); + if !known && std::fs::symlink_metadata(&dest).is_err() { + return Err(Error::NotInstalled(locale.to_string())); + } + if std::fs::symlink_metadata(&dest).is_ok() { + std::fs::rename(&dest, stage.dir.join(locale)).map_err(io(format!("removing the {locale} pack")))?; + } + installed.packs.retain(|p| p.locale != locale); + self.write_installed(&installed) + // `stage` deletes the old folder when dropped. + } + + /// The record of installed packs. One that can't be read is copied aside as + /// `language-packs.json.bad` and rebuilt from the pack folders on disk (written atomically + /// over it), so no installed pack is ever dropped from it; a missing one is rebuilt the same + /// way when pack folders are there (none: empty). + fn read_installed(&self) -> Result { + let path = self.dir().join(INSTALLED); + let there = std::fs::symlink_metadata(&path).is_ok(); + if let Some(i) = self.read_record() { + return Ok(i); + } + let rebuilt = self.installed_from_disk(); + if !there && rebuilt.packs.is_empty() { + return Ok(Installed::default()); + } + log::warn!("{}: {}; rebuilt from the Language pack folders", path.display(), if there { "can't be read" } else { "missing" }); + if there { + let _ = std::fs::copy(&path, self.dir().join(format!("{INSTALLED}.bad"))); + } + self.write_installed(&rebuilt)?; + Ok(rebuilt) + } + + /// The record as written, if there is one that can be read. + fn read_record(&self) -> Option { + read_limited(&self.dir().join(INSTALLED), MAX_PACK_INDEX_BYTES).and_then(|b| serde_json::from_slice::(&b).ok()) + } + + /// The installed packs as their folders say: each folder named by a locale, with the name + /// and licence from its `pack.toml` (release, version and sha256 unknown). + fn installed_from_disk(&self) -> Installed { + let mut packs: Vec = std::fs::read_dir(self.language_packs_dir()) + .into_iter() + .flatten() + .flatten() + .filter_map(|e| e.file_name().into_string().ok()) + .filter(|l| is_locale(l)) + .take(MAX_PACKS) + .map(|locale| { + let info = read_limited(&self.language_packs_dir().join(&locale).join("pack.toml"), MAX_PACK_TOML_BYTES) + .and_then(|b| String::from_utf8(b).ok()) + .and_then(|t| t.parse::().ok()); + let field = |k: &str| info.as_ref().and_then(|t| t.get(k)).and_then(toml::Value::as_str).map(plain_text); + InstalledLanguagePack { + name: field("name").unwrap_or_else(|| locale.clone()), + licence: field("licence").unwrap_or_default(), + locale, + release: String::new(), + version: String::new(), + sha256: String::new(), + size: 0, + } + }) + .collect(); + packs.sort_by(|a, b| a.locale.cmp(&b.locale)); + Installed { version: 1, packs } + } + + fn write_installed(&self, installed: &Installed) -> Result<(), Error> { + let bytes = serde_json::to_vec_pretty(&Installed { version: 1, packs: installed.packs.clone() }) + .map_err(|e| io("saving the Language pack list")(std::io::Error::other(e)))?; + self.write_file(INSTALLED, &bytes) + } +} + +/// The zstd frames of `archive` one after another, as one stream (`zstd` writes one frame, but +/// a concatenation of frames is a valid `.zst` too, and must not stop after the first). +struct Frames<'a> { + decoder: Option>, + /// Frames started so far, empty ones included (at most [`MAX_PACK_FRAMES`]). + frames: usize, +} + +/// Most zstd frames in a pack archive (`zstd` writes one; empty frames count too). +const MAX_PACK_FRAMES: usize = 4096; + +impl<'a> Frames<'a> { + fn new(archive: &'a [u8]) -> Result { + Ok(Self { decoder: Some(ruzstd::decoding::StreamingDecoder::new(archive).map_err(|e| format!("not zstd: {e}"))?), frames: 1 }) + } +} + +impl Read for Frames<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + loop { + let Some(d) = self.decoder.as_mut() else { return Ok(0) }; + let n = d.read(buf)?; + if n > 0 || buf.is_empty() { + return Ok(n); + } + let rest: &[u8] = d.get_ref(); + if !rest.is_empty() { + self.frames += 1; + if self.frames > MAX_PACK_FRAMES { + return Err(std::io::Error::other(format!("more than {MAX_PACK_FRAMES} zstd frames"))); + } + } + self.decoder = if rest.is_empty() { None } else { Some(ruzstd::decoding::StreamingDecoder::new(rest).map_err(std::io::Error::other)?) }; + } + } +} + +/// Unpacks the `.tar.zst` of `pack` into `dest` (created). Only the contract's files are allowed: +/// `pack.toml`, `spelling.aff`, `spelling.dic`, `hyphenation.pat` at the root and +/// `LICENCES/` ([`licence_name`]); regular files and that folder only, each file at most +/// [`MAX_PACK_FILE_BYTES`], all of them [`MAX_PACK_UNPACKED_BYTES`], at most [`MAX_PACK_ENTRIES`]. +/// Every zstd frame is read, and nothing but tar padding may follow the archive's end. `pack.toml` +/// must name `pack`'s locale, and the files the index promises (`spelling`, `hyphenation`) must +/// be there. +pub(crate) fn unpack(archive: &[u8], dest: &Path, pack: &PackEntry) -> Result<(), String> { + // The decompressed stream is capped too: tar padding and headers beyond the files' bytes. + let cap = MAX_PACK_UNPACKED_BYTES.saturating_add((MAX_PACK_ENTRIES as u64 + 2) * 1024 + 20 * 512); + let mut tar = tar::Archive::new(Frames::new(archive)?.take(cap)); + std::fs::create_dir_all(dest).map_err(|e| e.to_string())?; + let (mut total, mut count) = (0u64, 0usize); + let mut written: Vec = Vec::new(); + for entry in tar.entries().map_err(|e| format!("not a tar archive: {e}"))? { + let mut entry = entry.map_err(|e| format!("damaged archive: {e}"))?; + count += 1; + if count > MAX_PACK_ENTRIES { + return Err("too many files".into()); + } + let raw = entry.path().map_err(|e| format!("bad file name: {e}"))?.into_owned(); + let rel = safe_entry_path(&raw).ok_or_else(|| format!("{} is outside the pack", raw.display()))?; + let parts: Vec<&str> = rel.iter().filter_map(|c| c.to_str()).collect(); + let kind = entry.header().entry_type(); + if kind.is_dir() { + match parts.as_slice() { + [] => {} + [LICENCES_DIR] => std::fs::create_dir_all(dest.join(LICENCES_DIR)).map_err(|e| e.to_string())?, + _ => return Err(format!("{}: not a folder a pack has", rel.display())), + } + continue; + } + if !kind.is_file() { + return Err(format!("{}: only files and folders are allowed", rel.display())); + } + let allowed = match parts.as_slice() { + [name] => ROOT_FILES.contains(name), + [LICENCES_DIR, name] => licence_name(name), + _ => false, + }; + let key = parts.join("/"); + if !allowed || written.contains(&key) { + return Err(format!("{}: not a file a pack has, or listed twice", rel.display())); + } + let size = entry.header().size().map_err(|e| e.to_string())?; + if size > MAX_PACK_FILE_BYTES { + return Err(format!("{}: larger than {MAX_PACK_FILE_BYTES} bytes", rel.display())); + } + total = total.saturating_add(size); + if total > MAX_PACK_UNPACKED_BYTES { + return Err(format!("larger than {MAX_PACK_UNPACKED_BYTES} bytes unpacked")); + } + let path = dest.join(&rel); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| e.to_string())?; + } + let mut out = std::fs::File::options().write(true).create_new(true).open(&path).map_err(|e| format!("{}: {e}", rel.display()))?; + let copied = std::io::copy(&mut (&mut entry).take(MAX_PACK_FILE_BYTES), &mut out).map_err(|e| format!("{}: {e}", rel.display()))?; + if copied != size { + return Err(format!("{}: truncated", rel.display())); + } + written.push(key); + } + // Only zeros (the tar end blocks and padding) may follow: not a second archive, not a frame + // that wasn't read. + let mut rest = tar.into_inner(); + let mut buf = [0u8; 8192]; + loop { + let n = rest.read(&mut buf).map_err(|e| format!("damaged archive: {e}"))?; + if n == 0 { + break; + } + if buf.get(..n).is_some_and(|b| b.iter().any(|&x| x != 0)) { + return Err("data after the end of the archive".into()); + } + } + let has = |f: &str| written.iter().any(|w| w == f); + if !has("pack.toml") { + return Err("no pack.toml".into()); + } + let toml = std::fs::read_to_string(dest.join("pack.toml")).map_err(|e| format!("pack.toml: {e}"))?; + let named = toml.parse::().ok().and_then(|t| t.get("locale").and_then(toml::Value::as_str).map(str::to_string)); + if named.as_deref() != Some(pack.locale.as_str()) { + return Err(format!("pack.toml names locale {named:?}, not {:?}", pack.locale)); + } + if pack.spelling && !(has("spelling.aff") && has("spelling.dic")) { + return Err("the index promises a spelling dictionary the archive doesn't have".into()); + } + if pack.hyphenation && !has("hyphenation.pat") { + return Err("the index promises hyphenation patterns the archive doesn't have".into()); + } + Ok(()) +} + +/// The files a pack holds at its root. +const ROOT_FILES: [&str; 4] = ["pack.toml", "spelling.aff", "spelling.dic", "hyphenation.pat"]; +/// The folder of a pack's licence texts. +const LICENCES_DIR: &str = "LICENCES"; +/// Largest `pack.toml` read back. +const MAX_PACK_TOML_BYTES: u64 = 1 << 20; + +/// A licence file's name: `[A-Za-z0-9._-]`, at most 100 characters, not starting with a dot or +/// ending with one, and not a name Windows reserves. +fn licence_name(n: &str) -> bool { + !n.is_empty() + && n.len() <= 100 + && !n.starts_with('.') + && !n.ends_with('.') + && n.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) + && !windows_reserved(n) +} + +/// A name Windows reserves for a device (`CON`, `nul.txt`, `COM1`, `lpt9.md`), with or without an +/// extension, in any case. +fn windows_reserved(n: &str) -> bool { + let stem = n.split('.').next().unwrap_or_default().trim_end_matches(' ').to_ascii_lowercase(); + matches!(stem.as_str(), "con" | "prn" | "aux" | "nul") + || (stem.len() == 4 + && (stem.starts_with("com") || stem.starts_with("lpt")) + && stem.as_bytes().get(3).is_some_and(|d| (b'1'..=b'9').contains(d))) +} + +/// Text from an index or `pack.toml` shown to people (a pack's name): without control, +/// bidirectional or other format characters (U+202E would reverse what follows), trimmed, at most +/// 200 characters. +fn plain_text(s: &str) -> String { + s.chars().filter(|&c| !c.is_control() && !is_format(c)).take(200).collect::().trim().to_string() +} + +/// Unicode format characters (general category Cf): bidirectional controls, zero-width ones, the +/// soft hyphen, tags. +fn is_format(c: char) -> bool { + matches!(c, + '\u{ad}' | '\u{600}'..='\u{605}' | '\u{61c}' | '\u{6dd}' | '\u{70f}' | '\u{890}'..='\u{891}' | '\u{8e2}' | '\u{180e}' + | '\u{200b}'..='\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2060}'..='\u{2064}' | '\u{2066}'..='\u{206f}' | '\u{feff}' + | '\u{fff9}'..='\u{fffb}' | '\u{110bd}' | '\u{110cd}' | '\u{13430}'..='\u{1343f}' | '\u{1bca0}'..='\u{1bca3}' + | '\u{1d173}'..='\u{1d17a}' | '\u{e0001}' | '\u{e0020}'..='\u{e007f}') +} + +/// `p` as a plain relative path (`./` prefixes dropped), or `None` for a root, a drive, `..` or a +/// name that isn't UTF-8 or holds a backslash, colon or control character. +fn safe_entry_path(p: &Path) -> Option { + let mut out = PathBuf::new(); + for c in p.components() { + match c { + Component::CurDir => {} + Component::Normal(n) => { + let n = n.to_str()?; + if n.is_empty() || n.contains(['\\', ':']) || n.chars().any(char::is_control) { + return None; + } + out.push(n); + } + Component::ParentDir | Component::RootDir | Component::Prefix(_) => return None, + } + } + Some(out) +} + +#[cfg(test)] +mod tests; diff --git a/crates/font-store/src/language_packs/tests.rs b/crates/font-store/src/language_packs/tests.rs new file mode 100644 index 00000000..813ee1c1 --- /dev/null +++ b/crates/font-store/src/language_packs/tests.rs @@ -0,0 +1,472 @@ +use std::collections::HashMap; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use super::*; +use crate::store::Caller; + +const RELEASE: &str = "https://github.com/storytold/craft-i18n/releases/download/language-packs-1/"; + +/// A local HTTP server answering 200 with the body set for a path, else 404. +struct Server { + base: String, + routes: Arc>>>, +} + +impl Server { + fn start() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}/", listener.local_addr().unwrap()); + let routes: Arc>>> = Arc::default(); + let r = routes.clone(); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let r = r.clone(); + std::thread::spawn(move || answer(stream, &r)); + } + }); + Self { base, routes } + } + + fn serve(&self, path: &str, body: &[u8]) { + self.routes.lock().unwrap().insert(format!("/{path}"), body.to_vec()); + } + + /// A client that maps the release URL onto this server. + fn client(&self) -> Client { + let t = Timeouts { connect: Duration::from_secs(2), response: Duration::from_secs(5), overall: Duration::from_secs(10) }; + Client::for_tests(t, vec![(RELEASE.into(), format!("{}release/", self.base))]) + } +} + +fn answer(mut s: TcpStream, routes: &Mutex>>) { + let mut head = Vec::new(); + let mut b = [0u8; 1]; + while !head.ends_with(b"\r\n\r\n") { + match s.read(&mut b) { + Ok(1) => head.push(b[0]), + _ => return, + } + } + let path = String::from_utf8_lossy(&head).split_whitespace().nth(1).unwrap_or("/").to_string(); + let body = routes.lock().unwrap().get(&path).cloned(); + match body { + Some(body) => { + let _ = write!(s, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + let _ = s.write_all(&body); + } + None => { + let _ = write!(s, "HTTP/1.1 404 Nope\r\nContent-Length: 4\r\nConnection: close\r\n\r\nnope"); + } + } +} + +fn temp(name: &str) -> PathBuf { + static N: AtomicUsize = AtomicUsize::new(0); + let d = std::env::temp_dir().join(format!("craft-language-packs-{name}-{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed))); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +/// A tar of `files` (name, kind, bytes), with raw header names so hostile ones can be written. +fn tar_of(files: &[(&str, tar::EntryType, &[u8])]) -> Vec { + let mut b = tar::Builder::new(Vec::new()); + for (name, kind, bytes) in files { + let mut h = tar::Header::new_old(); + let field = &mut h.as_old_mut().name; + field[..name.len()].copy_from_slice(name.as_bytes()); + h.set_entry_type(*kind); + h.set_size(bytes.len() as u64); + h.set_mode(0o644); + if *kind == tar::EntryType::Symlink || *kind == tar::EntryType::Link { + h.set_link_name("/etc/passwd").unwrap(); + } + h.set_cksum(); + b.append(&h, *bytes).unwrap(); + } + b.into_inner().unwrap() +} + +fn zst(bytes: &[u8]) -> Vec { + ruzstd::encoding::compress_to_vec(bytes, ruzstd::encoding::CompressionLevel::Fastest) +} + +const FILE: tar::EntryType = tar::EntryType::Regular; + +/// A good pack archive for `locale`, with `marker` in its word list. +fn pack_archive(locale: &str, marker: &str) -> Vec { + let toml = format!("locale = \"{locale}\"\nname = \"Test\"\nlicence = \"MIT\"\nclass = \"Embeddable\"\n"); + let dic = format!("2\nhuis\n{marker}\n"); + zst(&tar_of(&[ + ("pack.toml", FILE, toml.as_bytes()), + ("spelling.aff", FILE, b"SET UTF-8\n"), + ("spelling.dic", FILE, dic.as_bytes()), + ("hyphenation.pat", FILE, b"1na n2at\n"), + ("LICENCES/", tar::EntryType::Directory, b""), + ("LICENCES/COPYING", FILE, b"Test licence."), + ])) +} + +fn entry(locale: &str, url: &str, archive: &[u8]) -> serde_json::Value { + serde_json::json!({"locale": locale, "name": "Nederlands", "version": "abc", "url": url, "sha256": sha256_hex(archive), + "size": archive.len(), "licence": "MIT", "class": "Embeddable", "spelling": true, "hyphenation": true}) +} + +/// The index entry for `archive` as `locale`. +fn pack_entry(locale: &str, archive: &[u8]) -> PackEntry { + serde_json::from_value(entry(locale, &format!("{RELEASE}{locale}.tar.zst"), archive)).unwrap() +} + +fn index_json(packs: &[serde_json::Value]) -> Vec { + serde_json::to_vec(&serde_json::json!({"format": 1, "release": "language-packs-1", "packs": packs})).unwrap() +} + +/// A store whose network is on, a server with the index and the `nl` pack, and the pin. +fn setup(name: &str) -> (FontStore, Server, PackIndexPin, Vec) { + let server = Server::start(); + let archive = pack_archive("nl", "zzmarker"); + let index = index_json(&[entry("nl", &format!("{RELEASE}nl.tar.zst"), &archive)]); + server.serve("release/nl.tar.zst", &archive); + server.serve("release/index.json", &index); + let sha: &'static str = Box::leak(sha256_hex(&index).into_boxed_str()); + let pin = PackIndexPin { url: "https://github.com/storytold/craft-i18n/releases/download/language-packs-1/index.json", sha256: sha }; + let store = FontStore::new(temp(name), Caller::User).with_client(server.client()); + store.enable_network().unwrap(); + (store, server, pin, archive) +} + +#[test] +fn install_list_replace_and_remove_a_pack() { + let (store, server, pin, _) = setup("install"); + let source = IndexSource::Pinned(pin); + let index = store.fetch_pack_index_from(&source, &Cancel::new()).unwrap(); + assert_eq!(store.cached_pack_index_from(&source), Some(index.clone())); + let nl = index.pack("nl").unwrap().clone(); + let mut seen = Vec::new(); + let got = store.install_pack_from(false, &index, &nl, &Cancel::new(), &mut |p| seen.push(p)).unwrap(); + assert_eq!((got.locale.as_str(), got.release.as_str(), got.licence.as_str()), ("nl", "language-packs-1", "MIT")); + assert!(seen.last().is_some_and(|p| p.done == nl.size && p.total == nl.size)); + let dir = store.language_packs_dir().join("nl"); + assert_eq!(store.language_packs_dir(), store.dir().join("language-packs")); + for f in ["pack.toml", "spelling.aff", "spelling.dic", "hyphenation.pat", "LICENCES/COPYING"] { + assert!(dir.join(f).is_file(), "{f}"); + } + assert!(std::fs::read_to_string(dir.join("spelling.dic")).unwrap().contains("zzmarker")); + assert_eq!(store.installed_language_packs(), vec![got]); + // A newer archive replaces the old one in one move. + let newer = pack_archive("nl", "zznewer"); + server.serve("release/nl.tar.zst", &newer); + let newer_entry = PackEntry { sha256: sha256_hex(&newer), size: newer.len() as u64, ..nl.clone() }; + store.install_pack_from(false, &index, &newer_entry, &Cancel::new(), &mut |_| {}).unwrap(); + assert!(std::fs::read_to_string(dir.join("spelling.dic")).unwrap().contains("zznewer")); + assert_eq!(store.installed_language_packs().len(), 1); + assert!(std::fs::read_dir(store.dir().join(".staging")).into_iter().flatten().flatten().next().is_none(), "nothing left staged"); + store.remove_language_pack("nl").unwrap(); + assert!(!dir.exists() && store.installed_language_packs().is_empty()); + assert!(matches!(store.remove_language_pack("nl"), Err(Error::NotInstalled(_)))); + assert!(matches!(store.remove_language_pack("../x"), Err(Error::NotInstalled(_)))); +} + +#[test] +fn network_off_pins_and_checksums_are_enforced() { + let (store, server, pin, archive) = setup("pins"); + let source = IndexSource::Pinned(pin); + store.disable_network().unwrap(); + assert!(matches!(store.fetch_pack_index_from(&source, &Cancel::new()), Err(Error::NetworkOff))); + store.enable_network().unwrap(); + // Another index than the pinned one. + let wrong = IndexSource::Pinned(PackIndexPin { sha256: "0000000000000000000000000000000000000000000000000000000000000000", ..pin }); + assert!(matches!(store.fetch_pack_index_from(&wrong, &Cancel::new()), Err(Error::Verify { error: VerifyError::Sha256, .. }))); + let index = store.fetch_pack_index_from(&source, &Cancel::new()).unwrap(); + assert_eq!(store.cached_pack_index_from(&wrong), None, "the cache is the pinned index only"); + let nl = index.pack("nl").unwrap().clone(); + // An archive that isn't the one the index names. + server.serve("release/nl.tar.zst", &pack_archive("nl", "tampered")); + assert!(store.install_pack_from(false, &index, &nl, &Cancel::new(), &mut |_| {}).is_err()); + server.serve("release/nl.tar.zst", &archive); + let bad_sha = PackEntry { sha256: sha256_hex(b"other"), ..nl.clone() }; + assert!(matches!( + store.install_pack_from(false, &index, &bad_sha, &Cancel::new(), &mut |_| {}), + Err(Error::Verify { error: VerifyError::Sha256, .. }) + )); + assert!(store.installed_language_packs().is_empty() && !store.language_packs_dir().join("nl").exists()); + // Hand-made entries are checked again. + for e in [PackEntry { locale: "../nl".into(), ..nl.clone() }, PackEntry { url: "http://example.org/nl.tar.zst".into(), ..nl.clone() }] { + assert!(store.install_pack_from(false, &index, &e, &Cancel::new(), &mut |_| {}).is_err()); + } + store.disable_network().unwrap(); + assert!(matches!(store.install_pack_from(false, &index, &nl, &Cancel::new(), &mut |_| {}), Err(Error::NetworkOff))); +} + +#[test] +fn indexes_are_checked() { + let a = pack_archive("nl", "x"); + let ok = index_json(&[entry("nl", &format!("{RELEASE}nl.tar.zst"), &a)]); + assert_eq!(PackIndex::from_json(&ok).unwrap().packs.len(), 1); + let bad = [ + serde_json::to_vec(&serde_json::json!({"format": 2, "release": "r", "packs": []})).unwrap(), + index_json(&[entry("NL", &format!("{RELEASE}nl.tar.zst"), &a)]), + index_json(&[entry("../x", &format!("{RELEASE}nl.tar.zst"), &a)]), + index_json(&[entry("nl", &format!("{RELEASE}nl.tar.zst"), &a), entry("nl", &format!("{RELEASE}nl.tar.zst"), &a)]), + index_json(&[entry("nl", "http://127.0.0.1/nl.tar.zst", &a)]), + index_json(&[entry("nl", "/tmp/nl.tar.zst", &a)]), + b"not json".to_vec(), + ]; + for b in bad { + assert!(PackIndex::from_json(&b).is_err(), "{}", String::from_utf8_lossy(&b)); + } + // The developer override also takes loopback http and paths, never other http hosts. + assert!(PackIndex::parse(&index_json(&[entry("nl", "http://127.0.0.1:9/nl.tar.zst", &a)]), true).is_ok()); + assert!(PackIndex::parse(&index_json(&[entry("nl", "/tmp/nl.tar.zst", &a)]), true).is_ok()); + assert!(PackIndex::parse(&index_json(&[entry("nl", "http://example.org/nl.tar.zst", &a)]), true).is_err()); + for (s, ok) in [ + ("http://127.0.0.1:8000/i.json", true), + ("http://localhost/i.json", true), + ("http://[::1]:8/i.json", true), + ("http://127.0.0.1.evil.org/i.json", false), + ("http://evil.org@127.0.0.1/i.json", true), + ("http://127.0.0.1@evil.org/i.json", false), + ("ftp://127.0.0.1/i.json", false), + ("file:///tmp/i.json", false), + ] { + assert_eq!(local_source(s).is_some(), ok, "{s}"); + } +} + +/// The developer override (`CRAFT_LANGUAGE_PACK_INDEX`): an index from a local path or a +/// loopback http server, with no index pin, while archives are still checked against its sha256. +#[test] +fn developer_override_reads_a_local_index_without_the_pin() { + let (store, server, _, archive) = setup("override"); + // A local file index pointing at a local archive. + let dir = temp("override-files"); + std::fs::write(dir.join("nl.tar.zst"), &archive).unwrap(); + let index = index_json(&[entry("nl", &dir.join("nl.tar.zst").display().to_string(), &archive)]); + std::fs::write(dir.join("index.json"), &index).unwrap(); + let source = IndexSource::Override(dir.join("index.json").display().to_string()); + let got = store.fetch_pack_index_from(&source, &Cancel::new()).unwrap(); + assert_eq!(store.cached_pack_index_from(&source), Some(got.clone())); + store.install_pack_from(true, &got, got.pack("nl").unwrap(), &Cancel::new(), &mut |_| {}).unwrap(); + assert!(store.language_packs_dir().join("nl/spelling.dic").is_file()); + // A loopback http index and archive. + let http_index = index_json(&[entry("nl", &format!("{}local/nl.tar.zst", server.base), &archive)]); + server.serve("local/index.json", &http_index); + server.serve("local/nl.tar.zst", &archive); + let source = IndexSource::Override(format!("{}local/index.json", server.base)); + let got = store.fetch_pack_index_from(&source, &Cancel::new()).unwrap(); + store.install_pack_from(true, &got, got.pack("nl").unwrap(), &Cancel::new(), &mut |_| {}).unwrap(); + // The archive is still verified. + server.serve("local/nl.tar.zst", &pack_archive("nl", "tampered")); + assert!(store.install_pack_from(true, &got, got.pack("nl").unwrap(), &Cancel::new(), &mut |_| {}).is_err()); + // Only loopback http. + let source = IndexSource::Override("http://example.org/index.json".into()); + assert!(matches!(store.fetch_pack_index_from(&source, &Cancel::new()), Err(Error::BadUrl(_)))); + // Without the variable the pin is used. + let pin = PackIndexPin { url: "https://example.org/index.json", sha256: "00" }; + if std::env::var_os(LANGUAGE_PACK_INDEX_ENV).is_none() { + assert_eq!(IndexSource::from_env(&pin), IndexSource::Pinned(pin)); + } +} + +#[test] +fn hostile_archives_are_refused() { + let toml: &[u8] = b"locale = \"nl\"\nlicence = \"MIT\"\n"; + let nl = PackEntry { spelling: false, hyphenation: false, ..pack_entry("nl", b"") }; + let big = vec![0u8; 10]; + let cases: Vec<(&str, Vec)> = vec![ + ("traversal", zst(&tar_of(&[("pack.toml", FILE, toml), ("../evil", FILE, b"x")]))), + ("deep traversal", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/../../evil", FILE, b"x")]))), + ("absolute", zst(&tar_of(&[("pack.toml", FILE, toml), ("/tmp/evil", FILE, b"x")]))), + ("symlink", zst(&tar_of(&[("pack.toml", FILE, toml), ("spelling.dic", tar::EntryType::Symlink, b"")]))), + ("hard link", zst(&tar_of(&[("pack.toml", FILE, toml), ("spelling.dic", tar::EntryType::Link, b"")]))), + ("too deep", zst(&tar_of(&[("pack.toml", FILE, toml), ("a/b/c", FILE, b"x")]))), + ("duplicate", zst(&tar_of(&[("pack.toml", FILE, toml), ("pack.toml", FILE, toml)]))), + ("backslash", zst(&tar_of(&[("pack.toml", FILE, toml), ("..\\evil", FILE, b"x")]))), + ("no pack.toml", zst(&tar_of(&[("spelling.dic", FILE, b"1\nx\n")]))), + ("not zstd", tar_of(&[("pack.toml", FILE, toml)])), + ("not tar", zst(b"hello, this is not a tar archive at all")), + ("empty", Vec::new()), + // Only the contract's names. + ("other root file", zst(&tar_of(&[("pack.toml", FILE, toml), ("readme.txt", FILE, b"x")]))), + ("other folder", zst(&tar_of(&[("pack.toml", FILE, toml), ("extra/", tar::EntryType::Directory, b"")]))), + ("nested licence", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/sub/x", FILE, b"x")]))), + ("hidden licence", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/.x", FILE, b"x")]))), + ("space in licence", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/a b", FILE, b"x")]))), + ("trailing dot", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/COPYING.", FILE, b"x")]))), + ("reserved CON", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/CON", FILE, b"x")]))), + ("reserved nul.txt", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/nul.txt", FILE, b"x")]))), + ("reserved com1", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/com1.md", FILE, b"x")]))), + // pack.toml must name the entry's locale. + ("other locale", zst(&tar_of(&[("pack.toml", FILE, b"locale = \"de\"\n")]))), + ("no locale", zst(&tar_of(&[("pack.toml", FILE, b"licence = \"MIT\"\n")]))), + ("bad toml", zst(&tar_of(&[("pack.toml", FILE, b"locale = \n")]))), + // Nothing may follow the archive: not another frame, not raw bytes. + ("second archive", [zst(&tar_of(&[("pack.toml", FILE, toml)])), zst(&tar_of(&[("spelling.dic", FILE, b"1\nx\n")]))].concat()), + ("trailing bytes", [zst(&tar_of(&[("pack.toml", FILE, toml)])), b"garbage".to_vec()].concat()), + ("truncated", zst(&tar_of(&[("pack.toml", FILE, toml), ("spelling.dic", FILE, &big)]))[..20].to_vec()), + ]; + for (what, archive) in cases { + let root = temp("hostile"); + let dest = root.join("nl"); + assert!(unpack(&archive, &dest, &nl).is_err(), "{what}"); + assert!(!root.join("evil").exists() && !root.parent().unwrap().join("evil").exists(), "{what}"); + } + // Too many entries. + let names: Vec = (0..=MAX_PACK_ENTRIES).map(|i| format!("LICENCES/f{i}")).collect(); + let mut files: Vec<(&str, tar::EntryType, &[u8])> = vec![("pack.toml", FILE, toml)]; + files.extend(names.iter().map(|n| (n.as_str(), FILE, &b""[..]))); + assert!(unpack(&zst(&tar_of(&files)), &temp("many").join("nl"), &nl).is_err()); + // A header claiming a file larger than the cap is refused before anything is read. + let mut t = tar::Header::new_gnu(); + t.set_path("pack.toml").unwrap(); + t.set_size(toml.len() as u64); + t.set_cksum(); + let mut b = tar::Builder::new(Vec::new()); + b.append(&t, toml).unwrap(); + let mut tarball = b.into_inner().unwrap(); + tarball.truncate(tarball.len() - 1024); // drop the end marker + let mut h = tar::Header::new_gnu(); + h.set_path("spelling.dic").unwrap(); + h.set_size(MAX_PACK_FILE_BYTES + 1); + h.set_cksum(); + tarball.extend_from_slice(h.as_bytes()); + tarball.extend_from_slice(&[0u8; 1024]); + let err = unpack(&zst(&tarball), &temp("huge").join("nl"), &nl).unwrap_err(); + assert!(err.contains("larger than"), "{err}"); + // A good one unpacks. + let dest = temp("good").join("nl"); + let good = pack_archive("nl", "x"); + unpack(&good, &dest, &pack_entry("nl", &good)).unwrap(); + assert!(dest.join("LICENCES/COPYING").is_file()); + // A tar split over several zstd frames is read to the end. + let tarball = tar_of(&[("pack.toml", FILE, toml), ("LICENCES/", tar::EntryType::Directory, b""), ("LICENCES/COPYING", FILE, b"t")]); + let (a, b) = tarball.split_at(700); + let dest = temp("frames").join("nl"); + unpack(&[zst(a), zst(b)].concat(), &dest, &nl).unwrap(); + assert!(dest.join("LICENCES/COPYING").is_file()); + // Empty frames are read past, but only so many. + let one = zst(&tar_of(&[("pack.toml", FILE, toml)])); + let empty = zst(b""); + unpack(&[one.clone(), empty.clone(), empty.clone()].concat(), &temp("empties").join("nl"), &nl).unwrap(); + let many: Vec = std::iter::once(one).chain(std::iter::repeat_n(empty, MAX_PACK_FRAMES)).flatten().collect(); + let err = unpack(&many, &temp("frames-cap").join("nl"), &nl).unwrap_err(); + assert!(err.contains("frames"), "{err}"); + // The index's promises are checked against the files. + let bare = zst(&tar_of(&[("pack.toml", FILE, toml)])); + for e in [PackEntry { spelling: true, ..nl.clone() }, PackEntry { hyphenation: true, ..nl.clone() }] { + let err = unpack(&bare, &temp("promise").join("nl"), &e).unwrap_err(); + assert!(err.contains("promises"), "{err}"); + } +} + +#[test] +fn names_are_plain_text_and_reserved_locales_refused() { + let a = pack_archive("nl", "x"); + let mut e = entry("nl", &format!("{RELEASE}nl.tar.zst"), &a); + e["name"] = serde_json::json!(" Neder\u{202e}lands\u{7}\u{200b}\n"); + assert_eq!(PackIndex::from_json(&index_json(&[e])).unwrap().packs[0].name, "Nederlands"); + for l in ["con", "nul", "aux", "prn", "com1", "lpt9"] { + assert!(!is_locale(l), "{l}"); + assert!(PackIndex::from_json(&index_json(&[entry(l, &format!("{RELEASE}{l}.tar.zst"), &a)])).is_err(), "{l}"); + } + assert!(is_locale("com") && is_locale("lpt") && is_locale("com10")); +} + +#[test] +fn a_corrupt_record_is_rebuilt_from_the_folders() { + let (store, server, pin, _) = setup("corrupt"); + let index = store.fetch_pack_index_from(&IndexSource::Pinned(pin), &Cancel::new()).unwrap(); + let nl = index.pack("nl").unwrap().clone(); + store.install_pack_from(false, &index, &nl, &Cancel::new(), &mut |_| {}).unwrap(); + let record = store.dir().join("language-packs.json"); + std::fs::write(&record, b"{ not json").unwrap(); + // Still listed, with the name from its pack.toml. + let listed = store.installed_language_packs(); + assert_eq!(listed.iter().map(|p| (p.locale.as_str(), p.name.as_str())).collect::>(), [("nl", "Test")]); + // craft-libs#486 review: listing holds no store lock, so it never writes the record (it + // could race an install); the next install or removal repairs it. + assert_eq!(std::fs::read(&record).unwrap(), b"{ not json"); + assert!(!store.dir().join("language-packs.json.bad").exists()); + // Installing another keeps it. + std::fs::write(&record, b"{ not json").unwrap(); + let de = pack_archive("de", "zzde"); + server.serve("release/de.tar.zst", &de); + let de_entry = PackEntry { url: format!("{RELEASE}de.tar.zst"), ..pack_entry("de", &de) }; + store.install_pack_from(false, &index, &de_entry, &Cancel::new(), &mut |_| {}).unwrap(); + let mut locales: Vec = store.installed_language_packs().into_iter().map(|p| p.locale).collect(); + locales.sort(); + assert_eq!(locales, ["de", "nl"]); + assert_eq!(std::fs::read(store.dir().join("language-packs.json.bad")).unwrap(), b"{ not json", "the unreadable record is kept aside"); + assert!(serde_json::from_slice::(&std::fs::read(&record).unwrap()).is_ok(), "rebuilt in place"); + // A missing record with pack folders is rebuilt too. + std::fs::remove_file(&record).unwrap(); + assert_eq!(store.installed_language_packs().len(), 2); + assert!(!record.exists(), "listing writes nothing"); + let nl_again = PackEntry { url: format!("{RELEASE}nl.tar.zst"), ..nl.clone() }; + store.install_pack_from(false, &index, &nl_again, &Cancel::new(), &mut |_| {}).unwrap(); + assert_eq!(store.installed_language_packs().len(), 2, "de kept when the record was missing"); + // Removing works on a rebuilt record too. + std::fs::write(&record, b"\xff").unwrap(); + store.remove_language_pack("nl").unwrap(); + assert_eq!(store.installed_language_packs().into_iter().map(|p| p.locale).collect::>(), ["de"]); +} + +#[test] +fn index_text_is_plain_and_sources_are_checked() { + let a = pack_archive("nl", "x"); + let mut e = entry("nl", &format!("{RELEASE}nl.tar.zst"), &a); + e["licence"] = serde_json::json!("MIT\u{202e}"); + e["version"] = serde_json::json!("ab\u{0}c"); + e["class"] = serde_json::json!("\u{200f}Embeddable"); + e["source"] = serde_json::json!("https://example.org/nl"); + let json = serde_json::to_vec(&serde_json::json!({"format": 1, "release": "r1\u{2066}", "packs": [e.clone()]})).unwrap(); + let index = PackIndex::from_json(&json).unwrap(); + let p = &index.packs[0]; + assert_eq!((index.release.as_str(), p.licence.as_str(), p.version.as_str(), p.class.as_str()), ("r1", "MIT", "abc", "Embeddable")); + assert_eq!(p.source.as_deref(), Some("https://example.org/nl")); + assert_eq!(PackIndex::from_json(&index_json(&[entry("nl", &format!("{RELEASE}nl.tar.zst"), &a)])).unwrap().packs[0].source, None); + for bad in ["javascript:alert(1)", "http://example.org", "https://a b", "https://a\u{202e}b"] { + e["source"] = serde_json::json!(bad); + let err = PackIndex::from_json(&index_json(std::slice::from_ref(&e))).unwrap_err(); + assert!(!err.to_string().contains("font"), "{err}"); + } +} + +#[test] +fn bad_hand_made_entries_are_index_errors() { + let (store, _server, pin, _) = setup("variant"); + let index = store.fetch_pack_index_from(&IndexSource::Pinned(pin), &Cancel::new()).unwrap(); + let nl = index.pack("nl").unwrap().clone(); + for e in [PackEntry { locale: "../nl".into(), ..nl.clone() }, PackEntry { sha256: "x".into(), ..nl.clone() }] { + let got = store.install_pack_from(false, &index, &e, &Cancel::new(), &mut |_| {}); + assert!(matches!(got, Err(Error::Index(IndexError::Invalid(_)))), "{got:?}"); + } +} + +#[test] +fn the_override_is_visible() { + let set = std::env::var(LANGUAGE_PACK_INDEX_ENV).ok().map(|v| v.trim().to_string()).filter(|v| !v.is_empty()); + assert_eq!(language_pack_index_overridden(), set); +} + +/// A staging folder of the same name (an earlier process, or another one in a PID namespace) is +/// never used or deleted: the next name is taken, so an unpack can't mix in its files. +#[test] +fn a_stale_stage_folder_is_cleared() { + let store = temp("stale-stage"); + let staging = store.join(".staging"); + for n in 0..512 { + let d = staging.join(format!("{}-{n}", std::process::id())); + std::fs::create_dir_all(d.join("nl")).unwrap(); + std::fs::write(d.join("nl/pack.toml"), b"old").unwrap(); + } + let stage = Stage::new(&store).unwrap(); + assert_eq!(std::fs::read_dir(&stage.dir).unwrap().count(), 0, "{}", stage.dir.display()); + // The others are left alone: they may be another process's (PID namespaces). + let left = std::fs::read_dir(&staging).unwrap().flatten().filter(|e| e.path().join("nl/pack.toml").is_file()).count(); + assert_eq!(left, 512); +} diff --git a/crates/font-store/src/lib.rs b/crates/font-store/src/lib.rs index bd79a3ab..ac43dc09 100644 --- a/crates/font-store/src/lib.rs +++ b/crates/font-store/src/lib.rs @@ -18,8 +18,9 @@ //! # } //! ``` //! -//! The index and its lookups, [`verify_font`] and [`IndexPin`] build on every target. The store -//! and the HTTP client are native only. +//! The index and its lookups, [`verify_font`] and [`IndexPin`] build on every target. The store, +//! the HTTP client and Language packs (the store's second item kind, `language-pack`: see +//! [`FontStore::install_language_pack`]) are native only. mod index; mod pin; @@ -30,6 +31,8 @@ mod verify; #[cfg(not(target_arch = "wasm32"))] mod http; #[cfg(not(target_arch = "wasm32"))] +mod language_packs; +#[cfg(not(target_arch = "wasm32"))] mod store; pub use index::{ @@ -43,6 +46,11 @@ pub use verify::{VerifyError, is_font, sha256_hex, verify_font}; #[cfg(not(target_arch = "wasm32"))] pub use http::{Cancel, Client, FetchError, Timeouts, USER_AGENT}; #[cfg(not(target_arch = "wasm32"))] +pub use language_packs::{ + InstalledLanguagePack, LANGUAGE_PACK_INDEX_ENV, LANGUAGE_PACKS, MAX_PACK_ARCHIVE_BYTES, MAX_PACK_ENTRIES, MAX_PACK_FILE_BYTES, + MAX_PACK_INDEX_BYTES, MAX_PACK_UNPACKED_BYTES, PackEntry, PackIndex, PackIndexPin, language_pack_index_overridden, +}; +#[cfg(not(target_arch = "wasm32"))] pub use store::{Caller, Error, FontStore, InstalledFamily, InstalledFile, Network, Problem, Progress, UserSource}; #[cfg(test)] diff --git a/crates/font-store/src/store.rs b/crates/font-store/src/store.rs index 880d9857..4dabaad8 100644 --- a/crates/font-store/src/store.rs +++ b/crates/font-store/src/store.rs @@ -7,6 +7,8 @@ //! manifest.json what is installed //! indexes/ the last indexes fetched: sources.json, one file per source //! / google/fonts' folder name, or /: the fonts and the licence +//! language-packs/ installed Language packs, one folder per locale (`crate::language_packs`) +//! language-packs.json which Language packs are installed, from which index entry //! .staging/ downloads being installed; nothing a font scan picks up //! .lock held while the store changes //! ``` @@ -30,7 +32,7 @@ use crate::verify::{VerifyError, is_font, sha256_hex, verify_font}; const SETTINGS: &str = "settings.json"; const MANIFEST: &str = "manifest.json"; -const INDEX_CACHE: &str = "indexes"; +pub(crate) const INDEX_CACHE: &str = "indexes"; const SOURCES_CACHE: &str = "sources.json"; const STAGING: &str = ".staging"; const LOCK: &str = ".lock"; @@ -111,7 +113,7 @@ pub enum Error { }, } -fn io(what: impl Into) -> impl FnOnce(std::io::Error) -> Error { +pub(crate) fn io(what: impl Into) -> impl FnOnce(std::io::Error) -> Error { let what = what.into(); move |source| Error::Io { what, source } } @@ -336,7 +338,7 @@ impl FontStore { self.dir.join(INDEX_CACHE).join(format!("user-{key}")) } - fn require_network(&self) -> Result<(), Error> { + pub(crate) fn require_network(&self) -> Result<(), Error> { if self.network() == Network::On { Ok(()) } else { Err(Error::NetworkOff) } } @@ -545,7 +547,7 @@ impl FontStore { } /// Downloads from each URL in turn until one gives bytes that pass `check`. - fn fetch( + pub(crate) fn fetch( &self, urls: &[String], max: u64, @@ -712,7 +714,7 @@ impl FontStore { } } - fn lock(&self) -> Result { + pub(crate) fn lock(&self) -> Result { std::fs::create_dir_all(&self.dir).map_err(io("creating the font store folder"))?; let file = File::options().create(true).truncate(false).write(true).open(self.dir.join(LOCK)).map_err(io("locking the font store"))?; file.lock().map_err(io("locking the font store"))?; @@ -740,12 +742,12 @@ impl FontStore { self.write_file(MANIFEST, &bytes) } - fn write_file(&self, name: &str, bytes: &[u8]) -> Result<(), Error> { + pub(crate) fn write_file(&self, name: &str, bytes: &[u8]) -> Result<(), Error> { std::fs::create_dir_all(&self.dir).map_err(io("creating the font store folder"))?; craft_atomic::atomic_write(&self.dir.join(name), bytes).map_err(io(format!("saving {name}"))) } - fn sweep_staging(&self) { + pub(crate) fn sweep_staging(&self) { for e in std::fs::read_dir(self.dir.join(STAGING)).into_iter().flatten().flatten() { let old = e.metadata().and_then(|m| m.modified()).is_ok_and(|t| SystemTime::now().duration_since(t).is_ok_and(|d| d > STALE)); if old { @@ -818,7 +820,7 @@ fn user_index(bytes: &[u8], user: &UserSource) -> Result { Ok(index) } -fn read_limited(path: &Path, max: u64) -> Option> { +pub(crate) fn read_limited(path: &Path, max: u64) -> Option> { if std::fs::metadata(path).ok()?.len() > max { return None; } @@ -830,16 +832,28 @@ fn host(url: &str) -> &str { } /// A folder of downloads under `.staging`, deleted when dropped. -struct Stage { - dir: PathBuf, +pub(crate) struct Stage { + pub(crate) dir: PathBuf, } impl Stage { - fn new(store: &Path) -> Result { + pub(crate) fn new(store: &Path) -> Result { static N: AtomicU64 = AtomicU64::new(0); - let dir = store.join(STAGING).join(format!("{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed))); - std::fs::create_dir_all(&dir).map_err(io("creating a download folder"))?; - Ok(Self { dir }) + let staging = store.join(STAGING); + std::fs::create_dir_all(&staging).map_err(io("creating a download folder"))?; + // A folder of that name may be another process's (PID namespaces in Flatpak or + // containers repeat ids) or a stale one: never touched here, the next name is taken + // (stale ones go in `sweep_staging`). + let mut last = None; + for _ in 0..1000 { + let dir = staging.join(format!("{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed))); + match std::fs::create_dir(&dir) { + Ok(()) => return Ok(Self { dir }), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => last = Some(e), + Err(e) => return Err(io("creating a download folder")(e)), + } + } + Err(io("creating a download folder")(last.unwrap_or_else(|| std::io::Error::other("no free name")))) } /// Writes a verified download as `.part`, which no font scan picks up. diff --git a/crates/font-store/src/verify.rs b/crates/font-store/src/verify.rs index ba976df1..5ae297d0 100644 --- a/crates/font-store/src/verify.rs +++ b/crates/font-store/src/verify.rs @@ -20,7 +20,7 @@ pub enum VerifyError { actual: u64, }, /// The sha256 differs. - #[error("the file's sha256 doesn't match the font index")] + #[error("the file's sha256 doesn't match the index")] Sha256, /// It doesn't parse as a font with outlines. #[error("the file is not a font, or a damaged one")] diff --git a/crates/media-testkit/Cargo.toml b/crates/media-testkit/Cargo.toml new file mode 100644 index 00000000..827d2ae6 --- /dev/null +++ b/crates/media-testkit/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "craft-media-testkit" +description = "Oracle-test support (ffmpeg discovery, fixture locations) for the craft-libs media crates. Test support only." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true +publish = false + +[lints] +workspace = true diff --git a/crates/media-testkit/src/lib.rs b/crates/media-testkit/src/lib.rs new file mode 100644 index 00000000..36f609f7 --- /dev/null +++ b/crates/media-testkit/src/lib.rs @@ -0,0 +1,159 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! Oracle-test support for the craft-libs media crates (craft-media, craft-audio, and the video +//! crate to come): ffmpeg/ffprobe discovery, the skip-or-fail rule, and fixture locations. +//! ffmpeg is used only as an external fixture generator and oracle (never linked). Test support +//! only: unpublished, and used as a dev-dependency. +//! +//! Tool search order: `CRAFT_MEDIA_FFMPEG` / `CRAFT_MEDIA_FFPROBE` (an explicit path; a wrong +//! path fails the test), then `PATH`, then well-known install directories. A missing tool, a +//! failed fixture generator or a failed ffprobe run prints `SKIPPED` and the test passes, unless +//! `CRAFT_MEDIA_REQUIRE_ORACLES=1` is set: then each of these fails the test. +//! Fixtures are generated into `$CRAFT_MEDIA_FIXTURES_DIR` or `/target/fixtures`, one +//! directory per crate or format, and never committed. +//! +//! Failing loudly is what a test helper is for: a wrong `CRAFT_MEDIA_*` setting or an unwritable +//! fixture directory panics, which fails the test that asked. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicUsize, Ordering}; + +const REQUIRE_ENV: &str = "CRAFT_MEDIA_REQUIRE_ORACLES"; + +const KNOWN_DIRS: &[&str] = &[ + "/opt/homebrew/bin", + "/usr/local/bin", + "/usr/bin", + "/opt/local/bin", + "/snap/bin", + r"C:\ffmpeg\bin", + r"C:\Program Files\ffmpeg\bin", + r"C:\ProgramData\chocolatey\bin", +]; + +fn env_var(name: &str) -> String { + format!("CRAFT_MEDIA_{}", name.to_ascii_uppercase()) +} + +#[allow(clippy::panic)] +fn find_tool(name: &str) -> Option { + if let Some(e) = std::env::var_os(env_var(name)).filter(|e| !e.is_empty()) { + let p = PathBuf::from(e); + assert!(p.is_file(), "{} is set but `{}` is not a file", env_var(name), p.display()); + return Some(p); + } + let names = if cfg!(windows) { [format!("{name}.exe"), name.to_string()] } else { [name.to_string(), format!("{name}.exe")] }; + let path_dirs = std::env::var_os("PATH").map(|p| std::env::split_paths(&p).collect::>()).unwrap_or_default(); + let dirs = path_dirs.into_iter().chain(KNOWN_DIRS.iter().map(PathBuf::from)); + for dir in dirs { + for n in &names { + let p = dir.join(n); + if p.is_file() { + return Some(p); + } + } + } + None +} + +/// Find a tool silently (cached for the process for ffmpeg and ffprobe). +pub fn tool(name: &str) -> Option { + static FFMPEG: OnceLock> = OnceLock::new(); + static FFPROBE: OnceLock> = OnceLock::new(); + match name { + "ffmpeg" => FFMPEG.get_or_init(|| find_tool("ffmpeg")).clone(), + "ffprobe" => FFPROBE.get_or_init(|| find_tool("ffprobe")).clone(), + _ => find_tool(name), + } +} + +pub fn oracles_required() -> bool { + std::env::var(REQUIRE_ENV).is_ok_and(|v| matches!(v.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on")) +} + +/// Give up on an oracle check: print `SKIPPED` and return `None`, or fail when oracles are +/// required. +pub fn skip(reason: &str) -> Option { + skip_with(oracles_required(), reason) +} + +#[allow(clippy::panic)] +fn skip_with(required: bool, reason: &str) -> Option { + assert!(!required, "oracle: {reason}, and {REQUIRE_ENV}=1 forbids skipping oracle tests"); + eprintln!("SKIPPED (oracle): {reason}"); + None +} + +/// Run a fixture generator writing `tmp`, then rename it to `out`. A generator that cannot start +/// or fails skips (see [`skip`]); `required` is explicit so tests can set it without touching +/// the process environment. +pub fn run_generator(cmd: &mut Command, tmp: &Path, out: &Path, required: bool) -> Option { + let reason = match cmd.output() { + Ok(o) if o.status.success() => match std::fs::rename(tmp, out) { + Ok(()) => return Some(out.to_path_buf()), + Err(e) => format!("could not move {} into place: {e}", tmp.display()), + }, + Ok(o) => format!("generator failed for {} ({}): {}", out.display(), o.status, String::from_utf8_lossy(&o.stderr).trim()), + Err(e) => format!("could not start generator for {}: {e}", out.display()), + }; + let _ = std::fs::remove_file(tmp); + skip_with(required, &format!("{reason} (encoder unavailable?)")) +} + +fn tool_or_skip(name: &str) -> Option { + tool(name).or_else(|| skip(&format!("{name} not found (set {} or put {name} on PATH)", env_var(name)))) +} + +/// ffmpeg, or `None` after printing `SKIPPED` (a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`). +pub fn ffmpeg() -> Option { + tool_or_skip("ffmpeg") +} + +/// ffprobe, or `None` after printing `SKIPPED` (a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`). +pub fn ffprobe() -> Option { + tool_or_skip("ffprobe") +} + +/// `$CRAFT_MEDIA_FIXTURES_DIR/` or `/target/fixtures/`, created if missing. +#[allow(clippy::unwrap_used)] +pub fn fixture_subdir(name: &str) -> PathBuf { + let d = match std::env::var_os("CRAFT_MEDIA_FIXTURES_DIR").filter(|v| !v.is_empty()) { + Some(d) => PathBuf::from(d), + None => { + let here = Path::new(env!("CARGO_MANIFEST_DIR")); + let root = here.parent().and_then(Path::parent).unwrap_or(here); + root.join("target").join("fixtures") + } + } + .join(name); + std::fs::create_dir_all(&d).unwrap(); + d +} + +/// A temporary sibling of `out` that no other thread or process will pick; the extension is kept +/// because ffmpeg chooses the muxer from it. +pub fn temp_path(out: &Path) -> PathBuf { + static N: AtomicUsize = AtomicUsize::new(0); + let tid = format!("{:?}", std::thread::current().id()).replace(|c: char| !c.is_ascii_alphanumeric(), ""); + let n = N.fetch_add(1, Ordering::Relaxed); + let stem = out.file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_default(); + let name = match out.extension() { + Some(ext) => format!("{stem}.tmp-{}-{tid}-{n}.{}", std::process::id(), ext.to_string_lossy()), + None => format!("{stem}.tmp-{}-{tid}-{n}", std::process::id()), + }; + out.with_file_name(name) +} + +/// Run a fixture generator and print `FIXTURE `. +pub fn generate_and_report(name: &str, outputs: &[PathBuf], generate: impl FnOnce() -> Option) -> Option { + let existed = !outputs.is_empty() && outputs.iter().all(|p| std::fs::metadata(p).is_ok_and(|m| m.len() > 0)); + let r = generate(); + let status = match (&r, existed) { + (None, _) => "skipped", + (Some(_), true) => "cached", + (Some(_), false) => "made", + }; + println!("FIXTURE {status} {name}"); + r +} diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index dd1db2c6..0d5cbf35 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -7,6 +7,9 @@ license.workspace = true rust-version.workspace = true repository.workspace = true +[package.metadata.docs.rs] +all-features = true + [lints] workspace = true @@ -15,11 +18,15 @@ default = ["isobmff"] bitstream = [] isobmff = ["bitstream", "dep:thiserror"] matroska = [] +# Serialize and Deserialize for `Container`, `CodecId`, `CodecParams` and `PcmFormat`. +serde = ["dep:serde"] [dependencies] thiserror = { workspace = true, optional = true } +serde = { workspace = true, optional = true } [dev-dependencies] +craft-media-testkit = { workspace = true } proptest = { workspace = true } serde_json = { workspace = true } diff --git a/crates/media/README.md b/crates/media/README.md index a408f04d..bee14288 100644 --- a/crates/media/README.md +++ b/crates/media/README.md @@ -1,9 +1,13 @@ # craft-media -Media containers and bit-level I/O for the Craft apps: an MSB-first bit reader and writer, and -clean-room ISO BMFF (MP4), QuickTime (MOV) and Matroska (MKV, WebM) demuxers and muxers written -from the public specifications. No `unsafe`, no panics on malformed input, and it builds for -`wasm32-unknown-unknown`. The only dependency is `thiserror` (for `isobmff`). +Media containers and bit-level I/O for the Craft apps: an MSB-first bit reader and writer, +clean-room ISO BMFF (MP4), QuickTime (MOV) and Matroska (MKV, WebM) demuxers and muxers, written from the public specifications. No `unsafe`, no panics on malformed input, builds +for `wasm32-unknown-unknown`. Dependencies: `thiserror` (for `isobmff`), `serde` (optional). + +Codecs go in separate crates that depend on this one: `craft-audio` for audio (in a follow-up), +and a video codec crate later. What both need lives here: the containers, the +bit I/O, one `ByteSource` trait, and the [shared identifiers](#containers-codecs-and-codec-parameters) +that codec crates use to describe a file and configure a decoder. ## Features @@ -12,10 +16,12 @@ from the public specifications. No `unsafe`, no panics on malformed input, and i | `bitstream` | `craft_media::bitstream` | none | | `isobmff` | `craft_media::isobmff` | `bitstream`, `thiserror` | | `matroska` | `craft_media::matroska` | none | +| (none) | `Container`, `CodecId`, `CodecParams`, `PcmFormat`, `xiph_lace`, `ByteSource`, `ReadSeekSource` | none | +| `serde` | `Serialize`/`Deserialize` on `Container`, `CodecId`, `CodecParams` and `PcmFormat` | `serde` | `default = ["isobmff"]`, so `bitstream` and `isobmff` build. `matroska` is opt-in: it is about as much code as `isobmff`, and most users of MP4/MOV (LightCraft, FigCraft, SoundCraft, EffectCraft) -do not read Matroska. The features are independent; `matroska` does not need the other two. +do not read Matroska. The features are independent. ```toml craft-media = { git = "https://github.com/storytold/craft-libs", rev = "", features = ["matroska"] } @@ -30,9 +36,12 @@ craft-media = { git = "https://github.com/storytold/craft-libs", rev = " ## Boundaries - In scope: bit-level reading and writing, and container demuxing and muxing (boxes, sample - tables, sample entries and codec configuration records). -- Out of scope for now: codecs (H.264, HEVC, ProRes, AAC and others). They stay in the apps. - Ogg and other containers can join later as their own features. + tables, sample entries and codec configuration records) for audio and video tracks. +- Out of scope: codecs. Audio codecs are in `craft-audio`; video codecs (H.264, HEVC, ProRes and + others) stay in the apps until a video crate exists. Containers only FilmCraft reads (MXF, + MPEG-TS/PS, AVI) are not here yet. +- Anything both the audio and the video codec crates need belongs here, so the two never depend + on each other. ## Origin and API @@ -98,9 +107,14 @@ Other differences: accepted any entry. - `isobmff::Mp4Writer::write_sample` returns an error when a track would pass 2³² − 1 samples (PCM: frames), the most `stsz` can count; FilmCraft clamped the count. -- `matroska::ByteSource` stays a separate trait from `isobmff::ByteSource`, with the same shape. - FilmCraft implements both for one type, which a single shared trait would reject as conflicting - impls. +- `isobmff::ByteSource` and `matroska::ByteSource` are one trait, + `craft_media::ByteSource` (with `ReadSeekSource`, which was Matroska's). A type FilmCraft + implements both traits for keeps one of its two impls. +- `isobmff::Track` and `matroska::Track` have `codec_params()`, both returning a + `CodecParams` (`CodecId::Unknown` for a codec without an identifier). The MP4 one describes + the first sample's sample description; `codec_params_for(description_index)` gives any other, + and `has_mixed_descriptions()` says whether the samples use more than one. + `isobmff::CodecConfig` has `codec_id()` and `extradata()`, and `OpusConfig::opus_head()`. ## `bitstream` @@ -170,7 +184,7 @@ Boxes with 64-bit sizes and with size 0 (runs to the end of the file) are suppor | `jpeg mjpa mjpb` | `Jpeg { fourcc }` | | `AVdn AVdh` | `Dnx { fourcc }` | | `mp4a` + `esds` (in `wave` too) | `Aac(AacConfig)`: ASC bytes + decoded object type/rate/channels; `Mp3` for OTI 0x69/0x6B | -| `.mp3`/`.mp2` | `Mp3`: MPEG-1/2 audio of any layer (each frame header names it) | +| `.mp3`/`.mp2` | `Mp3`: MPEG-1/2 audio of any layer (each frame header names it; `Track::codec_params()` reports `Mp1`, `Mp2` or `Mp3` from the first frame) | | PCM fourccs | `Pcm(PcmConfig)`: bits, float, endianness (`enda`, lpcm flags, `pcmC`), channels, rate | | `alac` | `Alac { cookie }` | | `Opus` + `dOps` | `Opus(OpusConfig)` | @@ -383,6 +397,38 @@ first frame (AV1 from `rav1e`) creates the writer after that frame. from codec parsers). - Muxer: no CRC-32, Chapters, Tags, Attachments, content encodings or block additions. +## Containers, codecs and codec parameters + +Always built, with no dependencies, so codec crates share them: + +- `Container`: the file format (`Wav`, `Aiff`, `Flac`, `Mp4`, `QuickTime`, `Matroska`, `WebM`, + `Ogg`, `Caf`, `Adts`, `MpegAudio`, `Ac3`), with `name()`. +- `CodecId`: audio and video codecs (`Aac`, `HeAac`, `Opus`, `Eac3`, `Speex`, …, `H264`, `Hevc`, + `Av1`, `ProRes`, `Dnxhd`, `Apv`, `Theora`, …, `Unknown`), with `name()`, `is_audio()`, + `is_video()`. +- `PcmFormat`: the layout of uncompressed samples, built with `PcmFormat::int(bits, signed, + big_endian)` or `PcmFormat::float(bits, big_endian)`. +- `CodecParams`: what a decoder needs about a track, whatever its container: `codec`, + `extradata`, `tag` (the container's four-character code, such as a ProRes profile), the audio + fields `sample_rate`, `channels` and `pcm` (`PcmFormat`), and the video fields `width` and + `height`. Built with `CodecParams::new(codec)` and the `with_*` methods, or by the demuxers' + `codec_params()`, which every container returns in the same form. `extradata` follows + Matroska's `CodecPrivate`, as ffmpeg's extradata does: the `AudioSpecificConfig` for AAC, the + `OpusHead` for Opus (MP4's `dOps` is converted), Xiph-laced headers for Vorbis (`xiph_lace`, + `None` for more than 256 packets), `fLaC` + metadata blocks for FLAC, the + `avcC`/`hvcC`/`av1C`/`apvC` record for video. + +All are `#[non_exhaustive]`: new formats and fields are minor changes. + +## `ByteSource` + +`craft_media::ByteSource` (`len`, `read_at(offset, buf)`) is what every demuxer reads from. It is +always built, so a crate that uses only `bitstream` can name it too. `isobmff::ByteSource` and +`matroska::ByteSource` are the same trait. It is implemented for `[u8]`, +`Vec`, `&T`, `Arc`, `Box` and `std::fs::File` (not on wasm). `ReadSeekSource` adapts any +`Read + Seek`; it is `Send + Sync` when the reader is `Send`. Implement the trait once for a +custom source (a web `Blob`, a network range reader) to use every container with it. + ## Tests - Unit tests cover the bit reader and writer, the box reader (64-bit, size-0, `uuid`, truncated and @@ -406,9 +452,10 @@ first frame (AV1 from `rav1e`) creates the writer after that frame. `fuzz.rs` also walks every box of a mutated file as a container and checks that each payload range stays inside its parent. -ffmpeg and ffprobe are test oracles only, never linked. They are looked up in +ffmpeg and ffprobe are test oracles only, never linked. The helpers are in `craft-media-testkit` +(an unpublished crate, shared with craft-audio and the video crate to come). They are looked up in `$CRAFT_MEDIA_FFMPEG` / `$CRAFT_MEDIA_FFPROBE`, then `PATH`, then common install directories. When a tool is missing, the oracle tests print `SKIPPED` and pass; `CRAFT_MEDIA_REQUIRE_ORACLES=1` turns a skip into a failure. CI (`.github/workflows/media.yml`) sets it and installs Ubuntu's ffmpeg. -Generate every fixture ahead of time with +craft-audio's oracle tests use the same variables. Generate every fixture ahead of time with `cargo test -p craft-media --all-features -- --ignored generate_fixtures`. diff --git a/crates/media/src/id.rs b/crates/media/src/id.rs new file mode 100644 index 00000000..6ee5f9fd --- /dev/null +++ b/crates/media/src/id.rs @@ -0,0 +1,376 @@ +//! Container and codec identifiers and codec parameters shared by the codec crates (craft-audio, +//! and the video crate to come), so a file's container and codec are named the same way +//! everywhere, and a codec is configured the same way whatever container its track came from. +//! +//! Both enums are `#[non_exhaustive]`: new formats are added as minor changes, so a `match` on them +//! needs a `_` arm. + +#![warn(missing_docs)] + +/// The file or stream format that holds the codec data. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Container { + /// RIFF WAVE, including BWF and RF64/BW64. + Wav, + /// AIFF and AIFF-C. + Aiff, + /// A native FLAC stream (`fLaC`). + Flac, + /// ISO base media file format: MP4, M4A, M4B. + Mp4, + /// QuickTime (MOV). + QuickTime, + /// Matroska (MKV, MKA). + Matroska, + /// WebM (the Matroska subset). + WebM, + /// Ogg. + Ogg, + /// Apple Core Audio Format. + Caf, + /// A raw ADTS AAC stream. + Adts, + /// A raw MPEG audio stream (MP1, MP2 or MP3 frames, optionally behind ID3v2 tags). + MpegAudio, + /// A raw AC-3 or E-AC-3 stream. + Ac3, +} + +impl Container { + /// A short display name: `"WAV"`, `"MP4"`, `"Matroska"`. + pub fn name(self) -> &'static str { + match self { + Container::Wav => "WAV", + Container::Aiff => "AIFF", + Container::Flac => "FLAC", + Container::Mp4 => "MP4", + Container::QuickTime => "QuickTime", + Container::Matroska => "Matroska", + Container::WebM => "WebM", + Container::Ogg => "Ogg", + Container::Caf => "CAF", + Container::Adts => "ADTS", + Container::MpegAudio => "MPEG audio", + Container::Ac3 => "AC-3", + } + } +} + +/// A codec, audio or video. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum CodecId { + /// Uncompressed PCM, integer or float. + Pcm, + /// ADPCM (IMA, Microsoft, G.722, G.726). + Adpcm, + /// G.711 A-law. + ALaw, + /// G.711 µ-law. + MuLaw, + /// FLAC. + Flac, + /// Apple Lossless. + Alac, + /// AAC (LC). Containers report every AAC stream as `Aac`; a decoder that finds spectral band + /// replication reports [`CodecId::HeAac`]. + Aac, + /// HE-AAC (AAC with spectral band replication, v1 or v2). + HeAac, + /// MPEG-1/2 Audio Layer I. + Mp1, + /// MPEG-1/2 Audio Layer II. + Mp2, + /// MPEG-1/2 Audio Layer III. + Mp3, + /// AC-3 (Dolby Digital). + Ac3, + /// E-AC-3 (Dolby Digital Plus). + Eac3, + /// Opus. + Opus, + /// Vorbis. + Vorbis, + /// Speex. + Speex, + /// H.264 / AVC. + H264, + /// H.265 / HEVC. + Hevc, + /// AV1. + Av1, + /// VP8. + Vp8, + /// VP9. + Vp9, + /// Apple ProRes. + ProRes, + /// Motion JPEG. + Mjpeg, + /// MPEG-2 video. + Mpeg2Video, + /// MPEG-4 Part 2 video. + Mpeg4Video, + /// Theora. + Theora, + /// Avid DNxHD / DNxHR. + Dnxhd, + /// APV (Advanced Professional Video). + Apv, + /// A codec this crate has no identifier for. + Unknown, +} + +impl CodecId { + /// A short display name: `"AAC"`, `"HE-AAC"`, `"E-AC-3"`, `"H.264"`. + pub fn name(self) -> &'static str { + match self { + CodecId::Pcm => "PCM", + CodecId::Adpcm => "ADPCM", + CodecId::ALaw => "A-law", + CodecId::MuLaw => "µ-law", + CodecId::Flac => "FLAC", + CodecId::Alac => "ALAC", + CodecId::Aac => "AAC", + CodecId::HeAac => "HE-AAC", + CodecId::Mp1 => "MP1", + CodecId::Mp2 => "MP2", + CodecId::Mp3 => "MP3", + CodecId::Ac3 => "AC-3", + CodecId::Eac3 => "E-AC-3", + CodecId::Opus => "Opus", + CodecId::Vorbis => "Vorbis", + CodecId::Speex => "Speex", + CodecId::H264 => "H.264", + CodecId::Hevc => "HEVC", + CodecId::Av1 => "AV1", + CodecId::Vp8 => "VP8", + CodecId::Vp9 => "VP9", + CodecId::ProRes => "ProRes", + CodecId::Mjpeg => "Motion JPEG", + CodecId::Mpeg2Video => "MPEG-2", + CodecId::Mpeg4Video => "MPEG-4", + CodecId::Theora => "Theora", + CodecId::Dnxhd => "DNxHD", + CodecId::Apv => "APV", + CodecId::Unknown => "unknown", + } + } + + /// Whether this is an audio codec. + pub fn is_audio(self) -> bool { + matches!( + self, + CodecId::Pcm + | CodecId::Adpcm + | CodecId::ALaw + | CodecId::MuLaw + | CodecId::Flac + | CodecId::Alac + | CodecId::Aac + | CodecId::HeAac + | CodecId::Mp1 + | CodecId::Mp2 + | CodecId::Mp3 + | CodecId::Ac3 + | CodecId::Eac3 + | CodecId::Opus + | CodecId::Vorbis + | CodecId::Speex + ) + } + + /// Whether this is a video codec. + pub fn is_video(self) -> bool { + matches!( + self, + CodecId::H264 + | CodecId::Hevc + | CodecId::Av1 + | CodecId::Vp8 + | CodecId::Vp9 + | CodecId::ProRes + | CodecId::Mjpeg + | CodecId::Mpeg2Video + | CodecId::Mpeg4Video + | CodecId::Theora + | CodecId::Dnxhd + | CodecId::Apv + ) + } +} + +/// The layout of uncompressed PCM samples. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct PcmFormat { + /// Bits per sample (8, 16, 24, 32 for integers; 32 or 64 for floats). + pub bits: u16, + /// IEEE float samples. + pub float: bool, + /// Big-endian byte order. + pub big_endian: bool, + /// Signed integers (8-bit WAV and `raw ` QuickTime PCM are unsigned). + pub signed: bool, +} + +impl PcmFormat { + /// Integer samples of `bits` bits. + pub fn int(bits: u16, signed: bool, big_endian: bool) -> PcmFormat { + PcmFormat { bits, float: false, big_endian, signed } + } + + /// IEEE float samples of `bits` bits (32 or 64). + pub fn float(bits: u16, big_endian: bool) -> PcmFormat { + PcmFormat { bits, float: true, big_endian, signed: true } + } +} + +/// What a decoder needs to know about a track, whatever container it came from. +/// +/// `extradata` is the codec configuration in Matroska's `CodecPrivate` form, which ffmpeg's +/// extradata also follows: +/// +/// | Codec | `extradata` | +/// |---|---| +/// | AAC | `AudioSpecificConfig` | +/// | Opus | the `OpusHead` identification header (RFC 7845 §5.1) | +/// | Vorbis | the three headers, Xiph-laced | +/// | FLAC | `fLaC` followed by the metadata blocks (STREAMINFO first) | +/// | ALAC | the magic cookie (`ALACSpecificConfig`) | +/// | AC-3 / E-AC-3 | the MP4 `dac3` / `dec3` payload, or empty | +/// | MP1/MP2/MP3, PCM | empty | +/// | H.264 / HEVC / AV1 | the `avcC` / `hvcC` / `av1C` record | +/// | VP9 | the MP4 `vpcC` payload, or Matroska's codec feature metadata | +/// | APV | the `apvC` record | +/// +/// `sample_rate` and `channels` are what the container declares (for HE-AAC with implicit +/// signalling, usually the core rate), except that Opus is always 48 kHz, the rate it decodes at; +/// `width` and `height` are the coded picture size of a video +/// track. `tag` is the container's four-character code for the codec when it has one (the MP4 +/// sample entry format, Matroska's ProRes or VfW FourCC), which is how a ProRes profile (`apcn`, +/// `ap4h`, …) or a DNx variant is told apart. +/// +/// `#[non_exhaustive]`: build it with [`CodecParams::new`] and the `with_*` methods. +#[non_exhaustive] +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct CodecParams { + /// The codec. + pub codec: CodecId, + /// The codec configuration (see the table above). + pub extradata: Vec, + /// The container's four-character code for the codec, if it has one. + pub tag: Option<[u8; 4]>, + /// Audio: samples per second. + pub sample_rate: Option, + /// Audio: channel count. + pub channels: Option, + /// Audio: the sample layout of uncompressed PCM. + pub pcm: Option, + /// Video: coded picture width in pixels. + pub width: Option, + /// Video: coded picture height in pixels. + pub height: Option, +} + +impl CodecParams { + /// Parameters for `codec`, with nothing else known yet. + pub fn new(codec: CodecId) -> CodecParams { + CodecParams { codec, extradata: Vec::new(), tag: None, sample_rate: None, channels: None, pcm: None, width: None, height: None } + } + + /// Sets `tag`. + pub fn with_tag(mut self, tag: [u8; 4]) -> CodecParams { + self.tag = Some(tag); + self + } + + /// The coded picture size; a zero dimension leaves both unknown. + pub fn with_size(mut self, width: u32, height: u32) -> CodecParams { + (self.width, self.height) = if width > 0 && height > 0 { (Some(width), Some(height)) } else { (None, None) }; + self + } + + /// Sets `extradata`. + pub fn with_extradata(mut self, extradata: impl Into>) -> CodecParams { + self.extradata = extradata.into(); + self + } + + /// Sets `sample_rate`; 0 leaves it unknown. + pub fn with_sample_rate(mut self, sample_rate: u32) -> CodecParams { + self.sample_rate = (sample_rate > 0).then_some(sample_rate); + self + } + + /// Sets `channels`; 0 leaves it unknown. + pub fn with_channels(mut self, channels: u16) -> CodecParams { + self.channels = (channels > 0).then_some(channels); + self + } + + /// Sets `pcm`. + pub fn with_pcm(mut self, pcm: PcmFormat) -> CodecParams { + self.pcm = Some(pcm); + self + } +} + +/// Xiph lacing of `packets` (Matroska `CodecPrivate` for Vorbis): the count less one, each +/// length but the last as a run of 255s and a remainder, then the packets. `None` for no packets +/// or more than 256, which the one-byte count cannot hold. +pub fn xiph_lace(packets: &[Vec]) -> Option> { + let mut out = vec![u8::try_from(packets.len().checked_sub(1)?).ok()?]; + for p in packets.iter().take(packets.len().saturating_sub(1)) { + let mut n = p.len(); + while n >= 255 { + out.push(255); + n -= 255; + } + out.push(n as u8); + } + for p in packets { + out.extend_from_slice(p); + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn audio_and_video_are_disjoint() { + for c in [CodecId::Aac, CodecId::Opus, CodecId::Pcm, CodecId::Eac3, CodecId::Speex] { + assert!(c.is_audio() && !c.is_video(), "{c:?}"); + } + for c in [CodecId::H264, CodecId::Av1, CodecId::ProRes, CodecId::Theora, CodecId::Dnxhd, CodecId::Apv] { + assert!(c.is_video() && !c.is_audio(), "{c:?}"); + } + assert!(!CodecId::Unknown.is_audio() && !CodecId::Unknown.is_video()); + } + + #[test] + fn params_builder() { + let p = CodecParams::new(CodecId::Aac).with_extradata([0x12, 0x10]).with_sample_rate(48_000).with_channels(2).with_sample_rate(0); + assert_eq!((p.codec, p.extradata.as_slice(), p.sample_rate, p.channels), (CodecId::Aac, &[0x12, 0x10][..], None, Some(2))); + let v = CodecParams::new(CodecId::ProRes).with_tag(*b"apcn").with_size(1920, 1080); + assert_eq!((v.tag, v.width, v.height), (Some(*b"apcn"), Some(1920), Some(1080))); + assert_eq!(v.with_size(0, 1080).width, None); + } + + #[test] + fn xiph_lacing() { + let laced = xiph_lace(&[vec![1; 30], vec![2; 300], vec![3; 5]]).unwrap(); + assert_eq!(&laced[..4], &[2, 30, 255, 45]); + assert_eq!(laced.len(), 4 + 335); + assert_eq!(xiph_lace(&[]), None); + assert_eq!(xiph_lace(&vec![vec![0]; 257]), None); + assert!(xiph_lace(&vec![vec![0]; 256]).is_some()); + } +} diff --git a/crates/media/src/isobmff/codec.rs b/crates/media/src/isobmff/codec.rs index b98fec6c..2a9f3fa9 100644 --- a/crates/media/src/isobmff/codec.rs +++ b/crates/media/src/isobmff/codec.rs @@ -1,5 +1,6 @@ //! Sample entries (`stsd` children) and codec configuration records: parsing and serialization. +use crate::CodecId; use crate::bitstream::BitReader; use crate::isobmff::bytes::{BoxBuf, Cur, FourCc, boxes}; use crate::isobmff::error::{Error, Result}; @@ -182,7 +183,8 @@ pub enum CodecConfig { Apv(ApvConfig), Aac(AacConfig), /// MPEG-1/2 audio (esds object type 0x69/0x6B, or QuickTime `.mp3` / `.mp2`). Usually layer - /// III, but these name no layer: ffmpeg stores MP2 as `mp4a` + 0x6B. Each frame header has it. + /// III, but these name no layer: ffmpeg stores MP2 as `mp4a` + 0x6B. Each frame header has it, + /// and [`crate::isobmff::Track::codec_params`] reports the layer the first frame names. Mp3, Pcm(PcmConfig), /// ALAC magic cookie (ALACSpecificConfig, 24 bytes, optionally followed by a channel layout). @@ -208,6 +210,51 @@ pub enum CodecConfig { } impl CodecConfig { + /// The codec this configuration is for. `Mp3` covers every MPEG audio layer: these entries + /// name none, and each frame header has it (ffmpeg stores MP2 as `mp4a` + 0x6B); + /// [`crate::isobmff::Track::codec_params`] reads it from the first frame. + pub fn codec_id(&self) -> CodecId { + match self { + CodecConfig::Avc(_) => CodecId::H264, + CodecConfig::Hevc(_) => CodecId::Hevc, + CodecConfig::Av1(_) => CodecId::Av1, + CodecConfig::Vp9(_) => CodecId::Vp9, + CodecConfig::ProRes { .. } => CodecId::ProRes, + CodecConfig::Jpeg { .. } => CodecId::Mjpeg, + CodecConfig::Aac(_) => CodecId::Aac, + CodecConfig::Mp3 => CodecId::Mp3, + CodecConfig::Pcm(_) => CodecId::Pcm, + CodecConfig::Alac { .. } => CodecId::Alac, + CodecConfig::Opus(_) => CodecId::Opus, + CodecConfig::Flac(_) => CodecId::Flac, + CodecConfig::Ac3 { .. } => CodecId::Ac3, + CodecConfig::Eac3 { .. } => CodecId::Eac3, + CodecConfig::Dnx { .. } => CodecId::Dnxhd, + CodecConfig::Apv(_) => CodecId::Apv, + CodecConfig::Timecode(_) | CodecConfig::Unknown { .. } => CodecId::Unknown, + } + } + + /// The codec configuration in Matroska's `CodecPrivate` form (see [`crate::CodecParams`]): + /// the `OpusHead` built from `dOps`, `fLaC` + the `dfLa` metadata blocks, the other records + /// as stored. + pub fn extradata(&self) -> Vec { + match self { + CodecConfig::Avc(c) => c.to_bytes(), + CodecConfig::Hevc(c) => c.to_bytes(), + CodecConfig::Av1(c) => c.to_bytes(), + CodecConfig::Vp9(c) => c.to_bytes(), + CodecConfig::Apv(c) => c.record_bytes(), + CodecConfig::Aac(a) => a.asc.clone(), + CodecConfig::Alac { cookie } => cookie.clone(), + CodecConfig::Opus(o) => o.opus_head(), + CodecConfig::Flac(f) => [b"fLaC".as_slice(), &f.metadata_blocks].concat(), + CodecConfig::Ac3 { dac3 } => dac3.clone(), + CodecConfig::Eac3 { dec3 } => dec3.clone(), + _ => Vec::new(), + } + } + /// A short human-readable codec name. pub fn name(&self) -> &'static str { match self { @@ -633,6 +680,23 @@ pub struct OpusConfig { } impl OpusConfig { + /// The Ogg Opus identification header (RFC 7845 §5.1) this `dOps` box describes: the same + /// fields, little-endian, behind the `OpusHead` magic. + pub fn opus_head(&self) -> Vec { + let mut h = Vec::with_capacity(19 + self.channel_mapping.len()); + h.extend_from_slice(b"OpusHead"); + h.push(1); + h.push(self.output_channels); + h.extend_from_slice(&self.pre_skip.to_le_bytes()); + h.extend_from_slice(&self.input_sample_rate.to_le_bytes()); + h.extend_from_slice(&self.output_gain.to_le_bytes()); + h.push(self.channel_mapping_family); + if self.channel_mapping_family != 0 { + h.extend_from_slice(&self.channel_mapping); + } + h + } + pub fn parse(data: &[u8]) -> Result { let mut c = Cur::labeled(data, "dOps"); let _version = c.u8()?; @@ -808,12 +872,15 @@ impl ApvConfig { /// Serialize the full `apvC` box payload (4-byte `FullBox` header + `APVDecoderConfigurationRecord`). pub fn to_bytes(&self) -> Vec { - let mut b = BoxBuf::new(); - b.u32(0); // FullBox version = 0, flags = 0 + [&[0u8; 4][..], &self.record_bytes()].concat() // FullBox version = 0, flags = 0 + } + + /// Serialize the bare `APVDecoderConfigurationRecord` (Matroska's `CodecPrivate` for `V_APV`). + pub fn record_bytes(&self) -> Vec { if !self.raw_record.is_empty() { - b.bytes(&self.raw_record); - return b.buf; + return self.raw_record.clone(); } + let mut b = BoxBuf::new(); b.u8(1); // configurationVersion = 1 b.u8(1); // number_of_configuration_entry = 1 b.u8(1); // pbu_type = 1 (primary frame) @@ -1840,3 +1907,51 @@ mod tests { assert_eq!(parse_asc(&[]), None); } } + +#[cfg(test)] +mod params_tests { + use super::*; + + /// `dOps` is big-endian; the `OpusHead` it becomes is little-endian, with the mapping table + /// only for families other than 0. + #[test] + fn dops_becomes_an_opus_head() { + let o = OpusConfig { + output_channels: 6, + pre_skip: 312, + input_sample_rate: 48_000, + output_gain: -256, + channel_mapping_family: 1, + channel_mapping: vec![4, 2, 0, 4, 1, 2, 3, 5], + }; + let h = o.opus_head(); + assert_eq!(&h[..8], b"OpusHead"); + assert_eq!(&h[8..19], &[1, 6, 0x38, 0x01, 0x80, 0xBB, 0, 0, 0x00, 0xFF, 1]); + assert_eq!(&h[19..], &o.channel_mapping[..]); + let stereo = OpusConfig { output_channels: 2, channel_mapping_family: 0, channel_mapping: vec![9, 9], ..o }; + assert_eq!(stereo.opus_head().len(), 19); + assert_eq!(CodecConfig::Opus(stereo).codec_id(), CodecId::Opus); + } + + #[test] + fn apv_extradata_is_the_bare_record() { + let cfg = ApvConfig { + profile_idc: 33, + level_idc: 60, + frame_width: 1920, + frame_height: 1080, + chroma_format_idc: 2, + bit_depth_minus8: 2, + ..Default::default() + }; + let record = cfg.record_bytes(); + assert_eq!(record[0], 1, "configurationVersion"); + assert_eq!(cfg.to_bytes(), [&[0u8; 4][..], &record].concat()); + // Matroska's V_APV CodecPrivate carries the record as is; an MP4 track must match it + let parsed = ApvConfig::parse(&cfg.to_bytes()).unwrap(); + assert_eq!(parsed.raw_record, record); + assert_eq!(CodecConfig::Apv(parsed.clone()).extradata(), record); + assert_eq!(parsed.to_bytes(), cfg.to_bytes()); + assert_eq!(CodecConfig::Apv(cfg).extradata(), record); + } +} diff --git a/crates/media/src/isobmff/demux.rs b/crates/media/src/isobmff/demux.rs index 8f7c413d..3c7a1c88 100644 --- a/crates/media/src/isobmff/demux.rs +++ b/crates/media/src/isobmff/demux.rs @@ -3,7 +3,7 @@ use crate::isobmff::bytes::{BoxHeader, Cur, FourCc, boxes, find}; use crate::isobmff::codec::{CodecConfig, SampleEntry, parse_sample_entry}; use crate::isobmff::error::{Error, Result, invalid}; -use crate::isobmff::source::ByteSource; +use crate::source::ByteSource; use std::collections::HashMap; /// Hard cap on the sample count of the whole file, all tracks and fragments (after PCM chunk merging). @@ -140,6 +140,9 @@ pub struct Track { pub pcm_chunked: bool, sync_index: Vec, pts_order: Vec, + /// Per entry: the MPEG audio codec its first sample's frame header names, for the + /// [`CodecConfig::Mp3`] entries, which name no layer themselves. + mpeg_audio: Vec>, } impl Track { @@ -147,6 +150,52 @@ impl Track { pub fn codec(&self) -> Option<&CodecConfig> { self.entries.first().map(|e| &e.codec) } + + /// What a decoder needs for this track, in the container-neutral form the codec crates take: + /// [`Track::codec_params_for`] the first sample's description (the first entry for a track + /// without samples, or when that index names no entry). A track whose samples use more than + /// one description ([`Track::has_mixed_descriptions`]) needs `codec_params_for` per sample. + pub fn codec_params(&self) -> crate::CodecParams { + let first = self.samples.first().map(|s| s.description_index); + self.codec_params_for(first.filter(|&i| i >= 1 && i as usize <= self.entries.len()).unwrap_or(1)) + } + + /// What a decoder needs for the samples whose [`Sample::description_index`] is + /// `description_index` (1-based, into [`Track::entries`]); [`crate::CodecId::Unknown`] when + /// there is no such entry. + pub fn codec_params_for(&self, description_index: u32) -> crate::CodecParams { + let Some(entry) = description_index.checked_sub(1).and_then(|i| self.entries.get(i as usize)) else { + return crate::CodecParams::new(crate::CodecId::Unknown); + }; + let codec = &entry.codec; + let id = self.mpeg_audio.get(description_index as usize - 1).copied().flatten().unwrap_or(codec.codec_id()); + let mut p = crate::CodecParams::new(id).with_extradata(codec.extradata()).with_tag(entry.format.0); + if let Some(v) = &entry.video { + p = p.with_size(u32::from(v.width), u32::from(v.height)); + } + if let Some(a) = &entry.audio { + p = p.with_sample_rate(a.sample_rate.round() as u32).with_channels(u16::try_from(a.channels).unwrap_or(0)); + } + match codec { + CodecConfig::Pcm(c) => { + p = p.with_pcm(if c.float { + crate::PcmFormat::float(c.bits, c.big_endian) + } else { + crate::PcmFormat::int(c.bits, c.signed, c.big_endian) + }); + p = p.with_sample_rate(c.sample_rate.round() as u32).with_channels(u16::try_from(c.channels).unwrap_or(0)); + } + CodecConfig::Opus(o) => p = p.with_sample_rate(48_000).with_channels(u16::from(o.output_channels)), + _ => {} + } + p + } + + /// True when the samples use more than one sample description, so no single + /// [`Track::codec_params`] describes them all. + pub fn has_mixed_descriptions(&self) -> bool { + self.samples.windows(2).any(|w| w[0].description_index != w[1].description_index) + } pub fn video(&self) -> Option<&crate::isobmff::codec::VideoParams> { self.entries.first().and_then(|e| e.video.as_ref()) } @@ -402,10 +451,39 @@ pub fn open(src: S) -> Result { tc.start_frame = Some(u32::from_be_bytes(b)); } } + t.mpeg_audio = mpeg_audio_layers(t, &src); } Ok(file) } +/// For each [`CodecConfig::Mp3`] entry, the layer its first sample's frame header names (these +/// entries name none: ffmpeg stores MP2 as `mp4a` + 0x6B), else what the sample entry says +/// (QuickTime `.mp2`, `.mp3`). +fn mpeg_audio_layers(t: &Track, src: &S) -> Vec> { + use crate::CodecId; + let from_header = |index: u32| { + let s = t.samples.iter().find(|s| s.description_index == index && s.size >= 4)?; + let mut b = [0u8; 4]; + src.read_at(s.offset, &mut b).ok()?; + if b[0] != 0xFF || b[1] & 0xE0 != 0xE0 { + return None; + } + match (b[1] >> 1) & 3 { + 3 => Some(CodecId::Mp1), + 2 => Some(CodecId::Mp2), + 1 => Some(CodecId::Mp3), + _ => None, + } + }; + (1..) + .zip(&t.entries) + .map(|(index, e)| match e.codec { + CodecConfig::Mp3 => from_header(index).or(Some(if e.format == b".mp2" { CodecId::Mp2 } else { CodecId::Mp3 })), + _ => None, + }) + .collect() +} + #[derive(Clone, Copy, Debug, Default)] struct Trex { description_index: u32, @@ -457,6 +535,7 @@ fn parse_trak(p: &[u8], file: &Mp4File, file_len: u64, budget: &mut SampleBudget pcm_chunked: false, sync_index: Vec::new(), pts_order: Vec::new(), + mpeg_audio: Vec::new(), }; let mut mdia = None; for b in boxes(p) { @@ -1082,6 +1161,60 @@ mod tests { assert_eq!(c.next_value(), 0); } + #[test] + fn codec_params_follow_the_sample_description() { + use crate::isobmff::codec::SampleEntry; + use crate::isobmff::mux::{Brand, Mp4Writer, TrackConfig, WriteSample, WriterOptions}; + let mut w = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap(); + let a = w.add_track(TrackConfig::new(SampleEntry::aac(vec![0x12, 0x10], 2, 48_000), 48_000)).unwrap(); + for _ in 0..2 { + w.write_sample(a, WriteSample { data: &[0; 8], duration: 1024, composition_offset: 0, is_sync: true }).unwrap(); + } + let out = w.finish().unwrap().into_inner(); + let mut t = open(out.as_slice()).unwrap().tracks.remove(0); + assert!(!t.has_mixed_descriptions()); + assert_eq!(t.codec_params(), t.codec_params_for(1)); + // entry 1 unused, every sample on entry 2 (a 44.1 kHz mono AudioSpecificConfig) + t.entries.push(SampleEntry::aac(vec![0x12, 0x08], 1, 44_100)); + t.samples.iter_mut().for_each(|s| s.description_index = 2); + let p = t.codec_params(); + assert_eq!((p.extradata.as_slice(), p.sample_rate, p.channels), (&[0x12, 0x08][..], Some(44_100), Some(1))); + assert_eq!(t.codec_params_for(1).extradata, [0x12, 0x10]); + assert_eq!(t.codec_params_for(0).codec, crate::CodecId::Unknown); + assert_eq!(t.codec_params_for(3).codec, crate::CodecId::Unknown); + assert!(!t.has_mixed_descriptions()); + t.samples[1].description_index = 1; + assert!(t.has_mixed_descriptions()); + t.samples[0].description_index = 0; // malformed: falls back to the first entry + assert_eq!(t.codec_params().extradata, [0x12, 0x10]); + } + + #[test] + fn mpeg_audio_layer_comes_from_the_first_frame() { + use crate::CodecId; + use crate::isobmff::codec::SampleEntry; + use crate::isobmff::mux::{Brand, Mp4Writer, TrackConfig, WriteSample, WriterOptions}; + // MPEG-1 frame headers: layer I, II, III; then no frame header at all + let cases: [(&[u8; 4], Brand, [u8; 2], CodecId); 5] = [ + (b"mp4a", Brand::Mp4, [0xFF, 0xFF], CodecId::Mp1), + (b"mp4a", Brand::Mp4, [0xFF, 0xFD], CodecId::Mp2), + (b"mp4a", Brand::Mp4, [0xFF, 0xFB], CodecId::Mp3), + (b"mp4a", Brand::Mp4, [0, 0], CodecId::Mp3), + (b".mp2", Brand::Mov, [0, 0], CodecId::Mp2), + ]; + for (format, brand, header, want) in cases { + let mut w = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(brand)).unwrap(); + let entry = SampleEntry::audio(FourCc(*format), CodecConfig::Mp3, 2, 48_000.0, 16); + let a = w.add_track(TrackConfig::new(entry, 48_000)).unwrap(); + let frame = [&header[..], &[0x94, 0x00, 0, 0, 0, 0]].concat(); + w.write_sample(a, WriteSample { data: &frame, duration: 1152, composition_offset: 0, is_sync: true }).unwrap(); + let out = w.finish().unwrap().into_inner(); + let t = &open(out.as_slice()).unwrap().tracks[0]; + assert_eq!(t.codec(), Some(&CodecConfig::Mp3)); + assert_eq!(t.codec_params().codec, want, "{format:?} {header:02X?}"); + } + } + #[test] fn rejects_no_moov_and_cmov() { let mut b = crate::isobmff::bytes::BoxBuf::new(); diff --git a/crates/media/src/isobmff/mod.rs b/crates/media/src/isobmff/mod.rs index b9627878..a971f144 100644 --- a/crates/media/src/isobmff/mod.rs +++ b/crates/media/src/isobmff/mod.rs @@ -16,8 +16,8 @@ mod codec; mod demux; mod error; mod mux; -mod source; +pub use crate::source::{ByteSource, ReadSeekSource}; pub use bytes::FourCc; pub use codec::{ AacConfig, ApvConfig, AudioParams, Av1Config, AvcConfig, BitRate, CleanAperture, CodecConfig, ColorInfo, FieldInfo, FlacConfig, HevcConfig, @@ -26,4 +26,3 @@ pub use codec::{ pub use demux::{Edit, Metadata, Mp4File, Sample, SampleGroup, Track, TrackKind, open}; pub use error::{Error, Result}; pub use mux::{Brand, FragmentedWriter, Mp4Writer, TrackConfig, WriteSample, WriterOptions}; -pub use source::ByteSource; diff --git a/crates/media/src/isobmff/source.rs b/crates/media/src/isobmff/source.rs deleted file mode 100644 index 0dad82b2..00000000 --- a/crates/media/src/isobmff/source.rs +++ /dev/null @@ -1,123 +0,0 @@ -//! Random-access byte sources for the demuxer. - -use std::io; -use std::sync::Arc; - -/// Random-access, read-only byte source (file, in-memory buffer, web Blob…). -/// -/// Implementations must be able to serve reads at arbitrary offsets through `&self`. -pub trait ByteSource { - /// Total length in bytes. - fn len(&self) -> u64; - /// Fill `buf` entirely with bytes starting at `offset`; fail with `UnexpectedEof` if past the end. - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()>; - /// True if the source holds no bytes. - fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -fn eof() -> io::Error { - io::Error::new(io::ErrorKind::UnexpectedEof, "read past end of byte source") -} - -impl ByteSource for [u8] { - fn len(&self) -> u64 { - <[u8]>::len(self) as u64 - } - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - let start = usize::try_from(offset).map_err(|_| eof())?; - let end = start.checked_add(buf.len()).ok_or_else(eof)?; - let src = self.get(start..end).ok_or_else(eof)?; - buf.copy_from_slice(src); - Ok(()) - } -} - -impl ByteSource for Vec { - fn len(&self) -> u64 { - self.as_slice().len() as u64 - } - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - self.as_slice().read_at(offset, buf) - } -} - -impl ByteSource for &T { - fn len(&self) -> u64 { - (**self).len() - } - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - (**self).read_at(offset, buf) - } -} - -impl ByteSource for Arc { - fn len(&self) -> u64 { - (**self).len() - } - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - (**self).read_at(offset, buf) - } -} - -impl ByteSource for Box { - fn len(&self) -> u64 { - (**self).len() - } - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - (**self).read_at(offset, buf) - } -} - -#[cfg(not(target_arch = "wasm32"))] -impl ByteSource for std::fs::File { - fn len(&self) -> u64 { - self.metadata().map(|m| m.len()).unwrap_or(0) - } - #[cfg(unix)] - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - std::os::unix::fs::FileExt::read_exact_at(self, buf, offset) - } - #[cfg(windows)] - fn read_at(&self, mut offset: u64, mut buf: &mut [u8]) -> io::Result<()> { - use std::os::windows::fs::FileExt; - while !buf.is_empty() { - let n = self.seek_read(buf, offset)?; - if n == 0 { - return Err(eof()); - } - offset += n as u64; - buf = &mut buf[n..]; - } - Ok(()) - } - #[cfg(not(any(unix, windows)))] - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - use std::io::{Read, Seek, SeekFrom}; - let mut f = self; - f.seek(SeekFrom::Start(offset))?; - f.read_exact(buf) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn slice_reads() { - let v = vec![1u8, 2, 3, 4]; - let mut b = [0u8; 2]; - v.read_at(1, &mut b).unwrap(); - assert_eq!(b, [2, 3]); - assert!(v.read_at(3, &mut b).is_err()); - assert!(v.read_at(u64::MAX, &mut b).is_err()); - let a: Arc<[u8]> = Arc::from(v.clone()); - a.read_at(2, &mut b).unwrap(); - assert_eq!(b, [3, 4]); - assert_eq!(ByteSource::len(&a), 4); - let s: &[u8] = &v; - assert_eq!(ByteSource::len(&s), 4); - } -} diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 86cf06b6..5dfe2e7e 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -1,18 +1,26 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] -//! Media containers and bit-level I/O for the Craft apps. +//! Media containers and bit-level I/O for the Craft apps. Codecs are in other crates +//! (`craft-audio`; video later), which depend on this one. //! //! - `bitstream` (feature `bitstream`): MSB-first bit reader and writer, Exp-Golomb codes, //! emulation-prevention escaping and NAL unit splitting. //! - `isobmff` (feature `isobmff`, enables `bitstream`): a clean-room ISO BMFF (MP4) and //! QuickTime (MOV) demuxer and muxer, and a box reader for other ISO BMFF formats. //! - `matroska` (feature `matroska`): a clean-room Matroska (MKV) and WebM demuxer and muxer. +//! - [`Container`] and [`CodecId`] (no feature): the identifiers the codec crates report a file's +//! container and codec with. //! //! `bitstream` and `isobmff` are on by default; `matroska` is opt-in. A crate that needs only bit -//! I/O uses `default-features = false, features = ["bitstream"]`. Codecs stay in the apps. +//! I/O uses `default-features = false, features = ["bitstream"]`. #[cfg(feature = "bitstream")] pub mod bitstream; +mod id; #[cfg(feature = "isobmff")] pub mod isobmff; #[cfg(feature = "matroska")] pub mod matroska; +pub mod source; + +pub use id::{CodecId, CodecParams, Container, PcmFormat, xiph_lace}; +pub use source::{ByteSource, ReadSeekSource}; diff --git a/crates/media/src/matroska/demux.rs b/crates/media/src/matroska/demux.rs index 253f613c..1d22705f 100644 --- a/crates/media/src/matroska/demux.rs +++ b/crates/media/src/matroska/demux.rs @@ -14,8 +14,8 @@ use crate::matroska::meta::{ Attachment, ClusterInfo, CuePoint, Edition, SeekEntry, SegmentInfo, Tag, parse_chapters, parse_cues, parse_info, parse_seekhead, parse_tags, parse_tracks, }; -use crate::matroska::source::{ByteSource, ReadSeekSource}; use crate::matroska::track::{Sample, Track, TrackKind, gcd}; +use crate::source::{ByteSource, ReadSeekSource}; const WIN: usize = 64 * 1024; /// Upper bound for level-1 metadata elements read into memory (Info, Tracks, Cues, Tags…). diff --git a/crates/media/src/matroska/mod.rs b/crates/media/src/matroska/mod.rs index 515e99e3..c0d7dae2 100644 --- a/crates/media/src/matroska/mod.rs +++ b/crates/media/src/matroska/mod.rs @@ -17,11 +17,11 @@ mod error; mod ids; mod meta; mod mux; -mod source; #[cfg(test)] mod synthetic_tests; mod track; +pub use crate::source::{ByteSource, ReadSeekSource}; pub use codec::Codec; pub use demux::{Demuxer, Keyframe, MkvFile, OpenOptions, Packet, SeekPoint, open, open_with}; pub use error::{Error, Result}; @@ -29,7 +29,6 @@ pub use meta::{ Attachment, Chapter, ChapterDisplay, ClusterInfo, CuePoint, CuePosition, Edition, SeekEntry, SegmentInfo, SimpleTag, Tag, TagTargets, }; pub use mux::{LacingMode, MkvWriter, MuxOptions, TrackSpec}; -pub use source::{ByteSource, ReadSeekSource}; pub use track::{AudioInfo, Color, ContentEncoding, MasteringMetadata, Projection, Sample, Track, TrackKind, VideoInfo}; /// Element IDs used by the demuxer (marker bits included), for callers inspecting raw elements. diff --git a/crates/media/src/matroska/source.rs b/crates/media/src/matroska/source.rs deleted file mode 100644 index 267c4271..00000000 --- a/crates/media/src/matroska/source.rs +++ /dev/null @@ -1,174 +0,0 @@ -//! Random-access byte sources for the demuxer. - -use std::cell::RefCell; -use std::io::{self, Read, Seek, SeekFrom}; -use std::sync::Arc; - -/// Random-access, read-only byte source (file, in-memory buffer, web Blob…). -/// -/// Same shape as `craft_media::isobmff::ByteSource`, so one adapter type can implement both. They -/// are separate traits so that a type can implement both without conflicting impls. -pub trait ByteSource { - /// Total length in bytes. - fn len(&self) -> u64; - /// Fill `buf` entirely with bytes starting at `offset`; fail with `UnexpectedEof` if past the end. - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()>; - /// True if the source holds no bytes. - fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -fn eof() -> io::Error { - io::Error::new(io::ErrorKind::UnexpectedEof, "read past end of byte source") -} - -impl ByteSource for [u8] { - fn len(&self) -> u64 { - <[u8]>::len(self) as u64 - } - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - let start = usize::try_from(offset).map_err(|_| eof())?; - let end = start.checked_add(buf.len()).ok_or_else(eof)?; - let src = self.get(start..end).ok_or_else(eof)?; - buf.copy_from_slice(src); - Ok(()) - } -} - -impl ByteSource for Vec { - fn len(&self) -> u64 { - self.as_slice().len() as u64 - } - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - self.as_slice().read_at(offset, buf) - } -} - -impl ByteSource for &T { - fn len(&self) -> u64 { - (**self).len() - } - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - (**self).read_at(offset, buf) - } -} - -impl ByteSource for Arc { - fn len(&self) -> u64 { - (**self).len() - } - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - (**self).read_at(offset, buf) - } -} - -impl ByteSource for Box { - fn len(&self) -> u64 { - (**self).len() - } - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - (**self).read_at(offset, buf) - } -} - -#[cfg(not(target_arch = "wasm32"))] -impl ByteSource for std::fs::File { - fn len(&self) -> u64 { - self.metadata().map(|m| m.len()).unwrap_or(0) - } - #[cfg(unix)] - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - std::os::unix::fs::FileExt::read_exact_at(self, buf, offset) - } - #[cfg(windows)] - fn read_at(&self, mut offset: u64, mut buf: &mut [u8]) -> io::Result<()> { - use std::os::windows::fs::FileExt; - while !buf.is_empty() { - let n = self.seek_read(buf, offset)?; - if n == 0 { - return Err(eof()); - } - offset += n as u64; - buf = &mut buf[n..]; - } - Ok(()) - } - #[cfg(not(any(unix, windows)))] - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - let mut f = self; - f.seek(SeekFrom::Start(offset))?; - f.read_exact(buf) - } -} - -/// Adapts any `Read + Seek` (e.g. `BufReader`, `Cursor>`) into a [`ByteSource`]. -/// -/// The length is taken once at construction; the stream position is tracked so sequential reads -/// do not issue redundant seeks. -pub struct ReadSeekSource { - inner: RefCell<(R, u64)>, - len: u64, -} - -impl ReadSeekSource { - /// Wrap a reader; seeks to the end once to learn the length. - pub fn new(mut r: R) -> io::Result { - let len = r.seek(SeekFrom::End(0))?; - Ok(Self { inner: RefCell::new((r, len)), len }) - } - /// Unwrap the reader. - pub fn into_inner(self) -> R { - self.inner.into_inner().0 - } -} - -impl ByteSource for ReadSeekSource { - fn len(&self) -> u64 { - self.len - } - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - if offset.checked_add(buf.len() as u64).is_none_or(|e| e > self.len) { - return Err(eof()); - } - let mut g = self.inner.borrow_mut(); - let (r, pos) = &mut *g; - if *pos != offset { - r.seek(SeekFrom::Start(offset))?; - *pos = offset; - } - let res = r.read_exact(buf); - match res { - Ok(()) => { - *pos = offset + buf.len() as u64; - Ok(()) - } - Err(e) => { - *pos = u64::MAX; // unknown; force a seek next time - Err(e) - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn slice_and_reader_sources() { - let v = vec![1u8, 2, 3, 4]; - let mut b = [0u8; 2]; - v.read_at(1, &mut b).unwrap(); - assert_eq!(b, [2, 3]); - assert!(v.read_at(3, &mut b).is_err()); - assert!(v.read_at(u64::MAX, &mut b).is_err()); - let r = ReadSeekSource::new(io::Cursor::new(v.clone())).unwrap(); - assert_eq!(r.len(), 4); - r.read_at(2, &mut b).unwrap(); - assert_eq!(b, [3, 4]); - r.read_at(0, &mut b).unwrap(); - assert_eq!(b, [1, 2]); - assert!(r.read_at(3, &mut b).is_err()); - } -} diff --git a/crates/media/src/matroska/track.rs b/crates/media/src/matroska/track.rs index 9a5dfd7a..4f1db134 100644 --- a/crates/media/src/matroska/track.rs +++ b/crates/media/src/matroska/track.rs @@ -234,6 +234,56 @@ pub struct Track { } impl Track { + /// What a decoder needs for this track, in the container-neutral form the codec crates take + /// (`CodecPrivate` is already in that form). + pub fn codec_params(&self) -> crate::CodecParams { + use crate::CodecId as Id; + let (id, extradata) = match &self.codec { + Codec::Avc { avcc } => (Id::H264, avcc.clone()), + Codec::Hevc { hvcc } => (Id::Hevc, hvcc.clone()), + Codec::Vp8 => (Id::Vp8, Vec::new()), + Codec::Vp9 { private } => (Id::Vp9, private.clone()), + Codec::Av1 { av1c } => (Id::Av1, av1c.clone()), + Codec::ProRes { .. } => (Id::ProRes, Vec::new()), + Codec::Apv { apvc } => (Id::Apv, apvc.clone()), + Codec::Mjpeg => (Id::Mjpeg, Vec::new()), + Codec::Aac { asc } => (Id::Aac, asc.clone()), + Codec::Opus { head } => (Id::Opus, head.clone()), + Codec::Vorbis { headers } => (Id::Vorbis, crate::xiph_lace(headers).unwrap_or_default()), + Codec::Flac { private } => (Id::Flac, private.clone()), + Codec::Pcm { .. } => (Id::Pcm, Vec::new()), + Codec::Ac3 => (Id::Ac3, Vec::new()), + Codec::Eac3 => (Id::Eac3, Vec::new()), + Codec::Mp3 => (Id::Mp3, Vec::new()), + Codec::Mp2 => (Id::Mp2, Vec::new()), + Codec::Mp1 => (Id::Mp1, Vec::new()), + // the ALACSpecificConfig, as MP4's `alac` box holds it after its FullBox header + Codec::Other(id) if id == "A_ALAC" => (Id::Alac, self.codec_private.clone()), + Codec::VfwFourcc { fourcc: [b'A', b'V', b'd', b'n' | b'h'], .. } => (Id::Dnxhd, Vec::new()), + _ => (Id::Unknown, Vec::new()), + }; + let mut p = crate::CodecParams::new(id).with_extradata(extradata); + match &self.codec { + Codec::ProRes { fourcc: Some(tag) } | Codec::VfwFourcc { fourcc: tag, .. } => p = p.with_tag(*tag), + _ => {} + } + if let Some(v) = &self.video { + p = p.with_size(v.pixel_width, v.pixel_height); + } + if let Some(a) = &self.audio { + p = p.with_sample_rate(a.sampling_frequency.round() as u32).with_channels(u16::try_from(a.channels).unwrap_or(0)); + } + if let Codec::Opus { .. } = self.codec { + // Opus always decodes at 48 kHz; a muxer may have declared the input rate instead + p = p.with_sample_rate(48_000); + } + if let Codec::Pcm { float, big_endian, bits } = self.codec { + // 8-bit integer PCM is unsigned in Matroska, as in WAV. + p = p.with_pcm(if float { crate::PcmFormat::float(bits, big_endian) } else { crate::PcmFormat::int(bits, bits > 8, big_endian) }); + } + p + } + /// Bytes prepended to every frame by header-stripping compression, if any. pub fn stripped_header(&self) -> Option<&[u8]> { self.encodings.iter().find_map(|e| match e { @@ -381,3 +431,36 @@ mod aspect_overflow_tests { assert_eq!(video.pixel_aspect(), (3, 4)); } } + +#[cfg(test)] +mod codec_params_tests { + use crate::CodecId; + use crate::matroska::mux::{MkvWriter, MuxOptions, TrackSpec}; + use crate::matroska::{Demuxer, TrackKind}; + + #[test] + fn alac_dnx_and_opus_match_mp4() { + let cookie: Vec = (0..24).collect(); + let mut alac = TrackSpec::new(TrackKind::Audio, "A_ALAC"); + alac.codec_private = cookie.clone(); + alac.audio = Some((44_100.0, 2, Some(16))); + let mut bih = vec![0u8; 40]; + bih[16..20].copy_from_slice(b"AVdn"); + let mut dnx = TrackSpec::new(TrackKind::Video, "V_MS/VFW/FOURCC"); + dnx.codec_private = bih; + dnx.video_size = Some((1920, 1080)); + let mut opus = TrackSpec::new(TrackKind::Audio, "A_OPUS"); + opus.codec_private = [b"OpusHead".as_slice(), &[1, 2, 0x38, 0x01, 0x44, 0xAC, 0, 0, 0, 0, 0]].concat(); + opus.audio = Some((44_100.0, 2, None)); // the input rate, as some muxers write it + let mut m = MkvWriter::new(std::io::Cursor::new(Vec::new()), vec![alac, dnx, opus], MuxOptions::default()).unwrap(); + for t in 0..3 { + m.write_frame(t, 0, true, &[0; 4], None).unwrap(); + } + let bytes = m.finish().unwrap().into_inner(); + let d = Demuxer::from_slice(&bytes).unwrap(); + let p: Vec<_> = d.tracks().iter().map(|t| t.codec_params()).collect(); + assert_eq!((p[0].codec, p[0].extradata.as_slice()), (CodecId::Alac, cookie.as_slice())); + assert_eq!((p[1].codec, p[1].tag), (CodecId::Dnxhd, Some(*b"AVdn"))); + assert_eq!((p[2].codec, p[2].sample_rate, p[2].channels), (CodecId::Opus, Some(48_000), Some(2))); + } +} diff --git a/crates/media/src/source.rs b/crates/media/src/source.rs new file mode 100644 index 00000000..44fde153 --- /dev/null +++ b/crates/media/src/source.rs @@ -0,0 +1,269 @@ +//! Random-access byte sources: what the demuxers read from. One trait for every container, so a +//! file, buffer or custom source implements it once. + +#![warn(missing_docs)] + +use std::io::{self, Read, Seek, SeekFrom}; +use std::sync::{Arc, Mutex}; + +/// Random-access, read-only byte source (file, in-memory buffer, web Blob…). +/// +/// Implementations serve reads at arbitrary offsets through `&self`. `isobmff::ByteSource` and +/// `matroska::ByteSource` are this trait. +pub trait ByteSource { + /// Total length in bytes. + fn len(&self) -> u64; + /// Fill `buf` entirely with bytes starting at `offset`; fail with `UnexpectedEof` if past the end. + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()>; + /// True if the source holds no bytes. + fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +fn eof() -> io::Error { + io::Error::new(io::ErrorKind::UnexpectedEof, "read past end of byte source") +} + +impl ByteSource for [u8] { + fn len(&self) -> u64 { + <[u8]>::len(self) as u64 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + let start = usize::try_from(offset).map_err(|_| eof())?; + let end = start.checked_add(buf.len()).ok_or_else(eof)?; + let src = self.get(start..end).ok_or_else(eof)?; + buf.copy_from_slice(src); + Ok(()) + } +} + +impl ByteSource for Vec { + fn len(&self) -> u64 { + self.as_slice().len() as u64 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + self.as_slice().read_at(offset, buf) + } +} + +impl ByteSource for &T { + fn len(&self) -> u64 { + (**self).len() + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + (**self).read_at(offset, buf) + } +} + +impl ByteSource for Arc { + fn len(&self) -> u64 { + (**self).len() + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + (**self).read_at(offset, buf) + } +} + +impl ByteSource for Box { + fn len(&self) -> u64 { + (**self).len() + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + (**self).read_at(offset, buf) + } +} + +/// Past the largest offset a file can have (`i64::MAX`), where the OS would fail with +/// `InvalidInput` instead of reporting the end of the file. +#[cfg(not(target_arch = "wasm32"))] +fn past_file_offsets(offset: u64, len: usize) -> bool { + offset.checked_add(len as u64).is_none_or(|e| e > i64::MAX as u64) +} + +/// A file read in place. `len` asks the file system each time, and a failure there reads as an +/// empty file; [`ReadSeekSource`] takes the length once and reports a failure when it is made. +#[cfg(not(target_arch = "wasm32"))] +impl ByteSource for std::fs::File { + fn len(&self) -> u64 { + self.metadata().map(|m| m.len()).unwrap_or(0) + } + #[cfg(unix)] + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + if past_file_offsets(offset, buf.len()) { + return Err(eof()); + } + std::os::unix::fs::FileExt::read_exact_at(self, buf, offset) + } + #[cfg(windows)] + fn read_at(&self, mut offset: u64, mut buf: &mut [u8]) -> io::Result<()> { + use std::os::windows::fs::FileExt; + if past_file_offsets(offset, buf.len()) { + return Err(eof()); + } + while !buf.is_empty() { + let n = self.seek_read(buf, offset)?; + if n == 0 { + return Err(eof()); + } + offset += n as u64; + buf = &mut buf[n..]; + } + Ok(()) + } + #[cfg(not(any(unix, windows)))] + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + if past_file_offsets(offset, buf.len()) { + return Err(eof()); + } + let mut f = self; + f.seek(SeekFrom::Start(offset))?; + f.read_exact(buf) + } +} + +/// Adapts any `Read + Seek` (e.g. `BufReader`, `Cursor>`) into a [`ByteSource`]. +/// +/// The length is taken once at construction; the stream position is tracked so sequential reads +/// do not issue redundant seeks. The reader sits behind a mutex, so the source is `Send + Sync` +/// whenever `R` is `Send` and can be shared across threads. +pub struct ReadSeekSource { + inner: Mutex<(R, u64)>, + len: u64, +} + +impl ReadSeekSource { + /// Wrap a reader; seeks to the end once to learn the length. + pub fn new(mut r: R) -> io::Result { + let len = r.seek(SeekFrom::End(0))?; + Ok(Self { inner: Mutex::new((r, len)), len }) + } + /// Unwrap the reader. + pub fn into_inner(self) -> R { + // A poisoned lock only means a read panicked in another thread; the reader is still usable. + self.inner.into_inner().unwrap_or_else(|e| e.into_inner()).0 + } +} + +impl ByteSource for ReadSeekSource { + fn len(&self) -> u64 { + self.len + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + if offset.checked_add(buf.len() as u64).is_none_or(|e| e > self.len) { + return Err(eof()); + } + let mut g = self.inner.lock().unwrap_or_else(|e| { + // a read panicked in another thread and may have moved the reader; force a seek + let mut g = e.into_inner(); + g.1 = u64::MAX; + self.inner.clear_poison(); + g + }); + let (r, pos) = &mut *g; + if *pos != offset { + *pos = u64::MAX; // unknown until the seek succeeds + r.seek(SeekFrom::Start(offset))?; + *pos = offset; + } + let res = r.read_exact(buf); + match res { + Ok(()) => { + *pos = offset + buf.len() as u64; + Ok(()) + } + Err(e) => { + *pos = u64::MAX; // unknown; force a seek next time + Err(e) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn slice_and_reader_sources() { + let v = vec![1u8, 2, 3, 4]; + let mut b = [0u8; 2]; + v.read_at(1, &mut b).unwrap(); + assert_eq!(b, [2, 3]); + assert!(v.read_at(3, &mut b).is_err()); + assert!(v.read_at(u64::MAX, &mut b).is_err()); + let r = ReadSeekSource::new(io::Cursor::new(v.clone())).unwrap(); + assert_eq!(r.len(), 4); + r.read_at(2, &mut b).unwrap(); + assert_eq!(b, [3, 4]); + r.read_at(0, &mut b).unwrap(); + assert_eq!(b, [1, 2]); + assert!(r.read_at(3, &mut b).is_err()); + let a: Arc<[u8]> = Arc::from(v.clone()); + a.read_at(2, &mut b).unwrap(); + assert_eq!(b, [3, 4]); + assert_eq!(ByteSource::len(&a), 4); + let s: &[u8] = &v; + assert_eq!(ByteSource::len(&s), 4); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn file_sources_read_like_a_buffer() { + let v: Vec = (0..=255).collect(); + let path = std::env::temp_dir().join(format!("craft-media-source-{}.bin", std::process::id())); + std::fs::write(&path, &v).unwrap(); + let f = std::fs::File::open(&path).unwrap(); + let r = ReadSeekSource::new(std::fs::File::open(&path).unwrap()).unwrap(); + let sources: [&dyn ByteSource; 3] = [&v, &f, &r]; + for s in sources { + assert_eq!(s.len(), 256); + // out of order, so ReadSeekSource has to seek back + for (offset, n) in [(0, 0), (10, 16), (0, 256), (200, 56), (255, 1), (3, 7)] { + let mut b = vec![0u8; n]; + s.read_at(offset as u64, &mut b).unwrap(); + assert_eq!(b, v[offset..offset + n], "offset {offset}, {n} bytes"); + } + for (offset, n) in [(200, 57), (256, 1), (300, 1), (u64::MAX, 1), (u64::MAX - 1, 4)] { + let err = s.read_at(offset, &mut vec![0u8; n]).unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof, "offset {offset}, {n} bytes"); + } + } + drop((f, r)); // Windows deletes no open file + std::fs::remove_file(&path).ok(); + } + + #[test] + fn read_seek_source_seeks_after_a_panicked_read() { + struct PanicOnce(io::Cursor>, bool); + impl Read for PanicOnce { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let n = self.0.read(&mut buf[..1])?; + if std::mem::take(&mut self.1) { + panic!("read failed halfway"); + } + Ok(n) + } + } + impl Seek for PanicOnce { + fn seek(&mut self, to: SeekFrom) -> io::Result { + self.0.seek(to) + } + } + let r = ReadSeekSource::new(PanicOnce(io::Cursor::new(vec![1, 2, 3, 4]), false)).unwrap(); + let mut b = [0u8; 2]; + r.read_at(0, &mut b).unwrap(); + r.inner.lock().unwrap().0.1 = true; + // the reader moves one byte, then panics with the tracked position at 2 + assert!(std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| r.read_at(2, &mut b))).is_err()); + r.read_at(2, &mut b).unwrap(); + assert_eq!(b, [3, 4]); + assert!(!r.inner.is_poisoned(), "recovered once, not on every read"); + } + + #[test] + fn read_seek_source_is_send_and_sync() { + fn shareable(_: &T) {} + shareable(&ReadSeekSource::new(io::Cursor::new(vec![0u8; 4])).unwrap()); + } +} diff --git a/crates/media/tests/common/mod.rs b/crates/media/tests/common/mod.rs index e0a2a3c7..d4aa5e9b 100644 --- a/crates/media/tests/common/mod.rs +++ b/crates/media/tests/common/mod.rs @@ -1,167 +1,23 @@ -//! Shared helpers for oracle tests: ffmpeg/ffprobe discovery and fixture generation. -//! ffmpeg is used only as an external fixture generator and oracle (never linked). -//! -//! Tool search order: `CRAFT_MEDIA_FFMPEG` / `CRAFT_MEDIA_FFPROBE` (an explicit path; a wrong -//! path fails the test), then `PATH`, then well-known install directories. A missing tool, a -//! failed fixture generator or a failed ffprobe run prints `SKIPPED` and the test passes, unless -//! `CRAFT_MEDIA_REQUIRE_ORACLES=1` is set: then each of these fails the test. -//! Fixtures are generated into `$CRAFT_MEDIA_FIXTURES_DIR` or `/target/fixtures`, one -//! directory per container (`isobmff`, `matroska`), and never committed. +//! craft-media's oracle-test helpers: the generic ones from `craft-media-testkit`, and the MP4 +//! fixtures and ffprobe queries the container tests share. #![allow(dead_code)] +pub use craft_media_testkit::*; use serde_json::Value; use std::path::{Path, PathBuf}; use std::process::Command; -use std::sync::OnceLock; -use std::sync::atomic::{AtomicUsize, Ordering}; - -const REQUIRE_ENV: &str = "CRAFT_MEDIA_REQUIRE_ORACLES"; - -const KNOWN_DIRS: &[&str] = &[ - "/opt/homebrew/bin", - "/usr/local/bin", - "/usr/bin", - "/opt/local/bin", - "/snap/bin", - r"C:\ffmpeg\bin", - r"C:\Program Files\ffmpeg\bin", - r"C:\ProgramData\chocolatey\bin", -]; - -fn env_var(name: &str) -> String { - format!("CRAFT_MEDIA_{}", name.to_ascii_uppercase()) -} - -fn find_tool(name: &str) -> Option { - if let Some(e) = std::env::var_os(env_var(name)).filter(|e| !e.is_empty()) { - let p = PathBuf::from(e); - assert!(p.is_file(), "{} is set but `{}` is not a file", env_var(name), p.display()); - return Some(p); - } - let names = if cfg!(windows) { [format!("{name}.exe"), name.to_string()] } else { [name.to_string(), format!("{name}.exe")] }; - let path_dirs = std::env::var_os("PATH").map(|p| std::env::split_paths(&p).collect::>()).unwrap_or_default(); - let dirs = path_dirs.into_iter().chain(KNOWN_DIRS.iter().map(PathBuf::from)); - for dir in dirs { - for n in &names { - let p = dir.join(n); - if p.is_file() { - return Some(p); - } - } - } - None -} - -/// Find a tool silently (cached for the process for ffmpeg and ffprobe). -pub fn tool(name: &str) -> Option { - static FFMPEG: OnceLock> = OnceLock::new(); - static FFPROBE: OnceLock> = OnceLock::new(); - match name { - "ffmpeg" => FFMPEG.get_or_init(|| find_tool("ffmpeg")).clone(), - "ffprobe" => FFPROBE.get_or_init(|| find_tool("ffprobe")).clone(), - _ => find_tool(name), - } -} - -pub fn oracles_required() -> bool { - std::env::var(REQUIRE_ENV).is_ok_and(|v| matches!(v.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on")) -} - -/// Give up on an oracle check: print `SKIPPED` and return `None`, or fail when oracles are -/// required. -pub fn skip(reason: &str) -> Option { - skip_with(oracles_required(), reason) -} - -fn skip_with(required: bool, reason: &str) -> Option { - assert!(!required, "craft-media oracle: {reason}, and {REQUIRE_ENV}=1 forbids skipping oracle tests"); - eprintln!("SKIPPED (craft-media oracle): {reason}"); - None -} - -/// Run a fixture generator writing `tmp`, then rename it to `out`. A generator that cannot start -/// or fails skips (see [`skip`]); `required` is explicit so tests can set it without touching -/// the process environment. -pub fn run_generator(cmd: &mut Command, tmp: &Path, out: &Path, required: bool) -> Option { - let reason = match cmd.output() { - Ok(o) if o.status.success() => match std::fs::rename(tmp, out) { - Ok(()) => return Some(out.to_path_buf()), - Err(e) => format!("could not move {} into place: {e}", tmp.display()), - }, - Ok(o) => format!("generator failed for {} ({}): {}", out.display(), o.status, String::from_utf8_lossy(&o.stderr).trim()), - Err(e) => format!("could not start generator for {}: {e}", out.display()), - }; - let _ = std::fs::remove_file(tmp); - skip_with(required, &format!("{reason} (encoder unavailable?)")) -} - -fn tool_or_skip(name: &str) -> Option { - tool(name).or_else(|| skip(&format!("{name} not found (set {} or put {name} on PATH)", env_var(name)))) -} - -/// ffmpeg, or `None` after printing `SKIPPED` (a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`). -pub fn ffmpeg() -> Option { - tool_or_skip("ffmpeg") -} - -/// ffprobe, or `None` after printing `SKIPPED` (a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`). -pub fn ffprobe() -> Option { - tool_or_skip("ffprobe") -} /// Fixtures of the `isobmff` tests. pub fn fixture_dir() -> PathBuf { fixture_subdir("isobmff") } -/// `$CRAFT_MEDIA_FIXTURES_DIR/` or `/target/fixtures/`, created if missing. -pub fn fixture_subdir(name: &str) -> PathBuf { - let d = match std::env::var_os("CRAFT_MEDIA_FIXTURES_DIR").filter(|v| !v.is_empty()) { - Some(d) => PathBuf::from(d), - None => { - let here = Path::new(env!("CARGO_MANIFEST_DIR")); - let root = here.parent().and_then(Path::parent).unwrap_or(here); - root.join("target").join("fixtures") - } - } - .join(name); - std::fs::create_dir_all(&d).unwrap(); - d -} - pub fn out_dir() -> PathBuf { let d = fixture_dir().join("out"); std::fs::create_dir_all(&d).unwrap(); d } -/// A temporary sibling of `out` that no other thread or process will pick; the extension is kept -/// because ffmpeg chooses the muxer from it. -pub fn temp_path(out: &Path) -> PathBuf { - static N: AtomicUsize = AtomicUsize::new(0); - let tid = format!("{:?}", std::thread::current().id()).replace(|c: char| !c.is_ascii_alphanumeric(), ""); - let n = N.fetch_add(1, Ordering::Relaxed); - let stem = out.file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_default(); - let name = match out.extension() { - Some(ext) => format!("{stem}.tmp-{}-{tid}-{n}.{}", std::process::id(), ext.to_string_lossy()), - None => format!("{stem}.tmp-{}-{tid}-{n}", std::process::id()), - }; - out.with_file_name(name) -} - -/// Run a fixture generator and print `FIXTURE `. -pub fn generate_and_report(name: &str, outputs: &[PathBuf], generate: impl FnOnce() -> Option) -> Option { - let existed = !outputs.is_empty() && outputs.iter().all(|p| std::fs::metadata(p).is_ok_and(|m| m.len() > 0)); - let r = generate(); - let status = match (&r, existed) { - (None, _) => "skipped", - (Some(_), true) => "cached", - (Some(_), false) => "made", - }; - println!("FIXTURE {status} {name}"); - r -} - const V: &[&str] = &["-f", "lavfi", "-i", "testsrc2=size=160x120:rate=25"]; const A: &[&str] = &["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000"]; diff --git a/crates/pcraft-format/Cargo.toml b/crates/pcraft-format/Cargo.toml new file mode 100644 index 00000000..ce1d6873 --- /dev/null +++ b/crates/pcraft-format/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "craft-pcraft-format" +description = "The envelope of PhotoCraft's .pcraft bundle: the manifest with the document as a type parameter, tile references, entry paths, decode limits and the migration runner." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +serde = { workspace = true } +# `unbounded_depth`: manifests nest deeper than serde_json's 128 levels; `parse_manifest` checks +# the nesting against `Limits::max_depth` before parsing. +serde_json = { workspace = true, features = ["unbounded_depth"] } +thiserror = { workspace = true } diff --git a/crates/pcraft-format/README.md b/crates/pcraft-format/README.md new file mode 100644 index 00000000..29321df8 --- /dev/null +++ b/crates/pcraft-format/README.md @@ -0,0 +1,116 @@ +# craft-pcraft-format + +The envelope of PhotoCraft's `.pcraft` bundle: the manifest around the document, tile +references, the entry paths, the limits for reading a manifest, and the runner for manifest +migrations. The document model stays in PhotoCraft and is a type parameter here. The crate +does not read ZIP archives, open files or decode tiles. No `unsafe`, no panics on malformed +input, and it builds for `wasm32-unknown-unknown`. The dependencies are `serde`, `serde_json` +and `thiserror`. + +```toml +craft-pcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "" } +``` + +## Bundle layout + +A bundle is a ZIP archive or a directory with these entries: + +| Entry | Constant or function | Contents | +|---|---|---| +| `manifest.json` | `MANIFEST` | the versioned manifest | +| `tiles/.zst` | `tile_path(hash)` | a 256×256 tile, zstd, little-endian samples | +| `blobs/.zst` | `blob_path(hash)` | binary data, zstd | +| `thumb.png` | `THUMB` | optional thumbnail | +| `composite/preview.png` | `COMPOSITE` | optional flattened preview | +| `save.lock` | `SAVE_LOCK` | directory bundles only: held while a save runs (empty) | + +`` is the BLAKE3 hash of the uncompressed bytes as 64 lowercase hex digits. +`is_valid_hash` checks that form. `tile_path` and `blob_path` return `None` for any other +string, so a hash read from a file never becomes an entry name such as `tiles/../x.zst`. +`TILES_DIR` (`tiles`), `BLOBS_DIR` (`blobs`) and `COMPOSITE_DIR` (`composite`) name the +directories, and `OBJECT_SUFFIX` (`.zst`) ends every tile and blob entry. `EXTENSION` is +`pcraft` and `FORMAT_VERSION` is 1. + +## Use + +```rust +use craft_pcraft_format::{FORMAT_VERSION, Limits, Manifest, Step}; + +const STEPS: &[Step] = &[]; + +let manifest: Manifest = craft_pcraft_format::read_manifest(&manifest_json, &Limits::default(), STEPS, FORMAT_VERSION)?; +``` + +- `read_manifest(json, &Limits, steps, target)` is the reader of a manifest. It runs + `parse_manifest`, then `migrate`, which refuses a version newer than `target`, then reads the + members, the document as `D`. `read_manifest_with` takes a function that reads the + document from its `serde_json::Value` instead of `D`'s `Deserialize`. +- `Manifest` has `format_version`, `generator`, `document: D`, `thumbnail` and `composite`, + and serializes them in that order. Absent previews serialize as `null`. On read, missing + previews are `None`, members the type does not name are skipped, and a `thumbnail` other + than `thumb.png` or a `composite` other than `composite/preview.png` gives `Corrupt`. + `Manifest` has no `Deserialize` and is `#[non_exhaustive]`: `Manifest::new(generator, + document)` builds one at `FORMAT_VERSION` without previews. +- `TileRef` has `tx`, `ty` and `hash`, and is `#[non_exhaustive]`: `TileRef::new(tx, ty, hash)` + builds one. `TileRef::path` gives its entry, or `None` when the hash is not valid. +- `check_manifest(json, &Limits)` checks the size, the nesting and the item count in one pass + without recursion. PhotoCraft's own check of a manifest it saves tests the nesting and the + item count only. `parse_manifest(json, &Limits)` runs that check and then parses into a + `serde_json::Value` with serde_json's 128-level recursion limit turned off. +- `migrate(&mut value, steps, target)` upgrades the JSON in place, where `steps[i]` upgrades + version `i + 1` to `i + 2`, and returns the version the file had. `Step` is + `fn(&mut Value) -> Result<(), E>`; with `E: From`, the steps return the caller's own + error type. +- `Error` is `Corrupt`, `Unsupported`, `TooNew`, `LimitExceeded` or `Json`. A manifest whose + `format_version` is newer than `target` gives `TooNew`, and one with no step to the next + version gives `Unsupported`. A missing or invalid `format_version` gives `Corrupt`, and any + other missing or mistyped member gives `Json`. `TooNew` holds `found`, `supported` and the manifest's + `generator` (`None` when it is missing or not a string), and is matched with `..`. + +## Limits + +`Limits` (fields `max_manifest_bytes`, `max_blob_bytes`, `max_total_bytes`, `max_depth`, +`max_items`) holds the limits PhotoCraft loads with: + +- `max_manifest_bytes` (256 MiB): the size of `manifest.json`. +- `max_blob_bytes` (1 GiB): one decompressed blob. +- `max_total_bytes` (16 GiB): every decompressed tile and blob in one bundle. +- `max_depth` (367, `manifest_depth(MAX_GROUP_DEPTH)`): the nesting of JSON arrays and objects. + PhotoCraft nests up to `MAX_GROUP_DEPTH` (100, a copy of PhotoCraft's own constant) layer + groups, three levels each, and allows 64 more levels for the document and leaf layers. A + value above `MAX_DEPTH` (400) is treated as 400, so parsing stays within the stack. +- `max_items` (`MAX_MANIFEST_ITEMS`, 4,194,304): arrays and objects opened, plus elements and + members after a comma. + +`check_manifest`, `parse_manifest` and `read_manifest` apply the manifest limits. The crate +does not read tiles or blobs, so it does not apply `max_blob_bytes` or `max_total_bytes`: the +caller that decompresses the entries enforces them. The struct is `#[non_exhaustive]`: start from the +defaults and set the fields you change. + +```rust +let mut limits = craft_pcraft_format::Limits::default(); +limits.max_manifest_bytes = 16 << 20; +``` + +## Testing + +`cargo test -p craft-pcraft-format` runs: + +- `tests/manifest.rs`: a manifest with and without previews serializes with + `serde_json::to_vec_pretty` to JSON written by hand in the test, byte for byte, and reads + back equal; missing previews and unknown members; entry names, hash checks, and no entry name + for an invalid hash. +- `tests/read.rs`: `read_manifest` and `read_manifest_with` read the envelope and the document, + refuse a newer version with the writer's `generator`, refuse a manifest past the limits, run + the steps first, and report missing or mistyped members as a struct with a derived + `Deserialize` reports them, in the same order; a missing or invalid `format_version` is + `Corrupt`, and so is a preview that names another entry, such as `../x` or an absolute path. +- Doc tests: `Manifest` and `TileRef` are not built by struct literal, `Manifest` has no + `Deserialize`, and `Error::TooNew` is matched with `..`. +- `tests/limits.rs`: the default limits, nesting at and past `max_depth`, items at and past + `max_items`, brackets inside strings, the manifest size, JSON errors and the messages. +- `tests/migrate.rs`: steps in order, partial chains, the current version, newer, missing, + zero and out-of-range versions, a missing step, and steps that return the caller's error type. + +`.github/workflows/pcraft-format.yml` runs fmt, clippy and the tests on Linux and Windows, and +a `wasm32-unknown-unknown` check. diff --git a/crates/pcraft-format/src/lib.rs b/crates/pcraft-format/src/lib.rs new file mode 100644 index 00000000..eb6482aa --- /dev/null +++ b/crates/pcraft-format/src/lib.rs @@ -0,0 +1,195 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! The envelope of PhotoCraft's `.pcraft` bundle, shared by PhotoCraft and by readers in other +//! apps. +//! +//! ```text +//! manifest.json versioned manifest (see [`Manifest`]) +//! tiles/.zst content-addressed 256×256 tiles (zstd, little-endian samples) +//! blobs/.zst content-addressed binary data +//! thumb.png optional thumbnail +//! composite/preview.png optional flattened preview +//! save.lock directory bundles only: held while a save runs (empty) +//! ``` +//! +//! A bundle is a ZIP archive or a directory with this layout. [`Manifest`] holds the format +//! version, the writer and the preview paths, with the document as the type parameter `D`, so +//! the document model stays in the app. [`TileRef`] is a tile's place in a surface. +//! [`read_manifest`] reads manifest JSON: it checks the JSON against [`Limits`], refuses a version +//! newer than the one the caller reads, upgrades older versions with the caller's steps, and reads +//! the envelope and the document. +//! +//! The crate does not read ZIP archives, open files or decode tiles. It builds for +//! `wasm32-unknown-unknown`. +//! +//! ``` +//! use craft_pcraft_format::{FORMAT_VERSION, Limits, Manifest, Step}; +//! +//! let raw = br#"{"format_version":1,"generator":"photocraft-format 0.1.0","document":{"name":"Untitled"}}"#; +//! let steps: &[Step] = &[]; +//! let manifest: Manifest = craft_pcraft_format::read_manifest(raw, &Limits::default(), steps, FORMAT_VERSION).unwrap(); +//! assert_eq!(manifest.document["name"], "Untitled"); +//! assert_eq!(manifest.thumbnail, None); +//! ``` + +mod limits; +mod migrate; +mod read; + +use serde::{Deserialize, Serialize}; + +pub use limits::{Limits, MAX_DEPTH, MAX_GROUP_DEPTH, MAX_MANIFEST_ITEMS, check_manifest, manifest_depth, parse_manifest}; +pub use migrate::{Step, migrate}; +pub use read::{read_manifest, read_manifest_with}; + +/// The manifest version PhotoCraft writes. +pub const FORMAT_VERSION: u32 = 1; + +/// File extension of the bundle, without the dot. +pub const EXTENSION: &str = "pcraft"; + +/// The directory of the tiles. +pub const TILES_DIR: &str = "tiles"; +/// The directory of the blobs. +pub const BLOBS_DIR: &str = "blobs"; +/// The directory of the flattened preview, [`COMPOSITE`]. +pub const COMPOSITE_DIR: &str = "composite"; +/// The suffix of every tile and blob entry, after the hash. +pub const OBJECT_SUFFIX: &str = ".zst"; + +/// The manifest entry. +pub const MANIFEST: &str = "manifest.json"; +/// The thumbnail entry, a PNG. +pub const THUMB: &str = "thumb.png"; +/// The flattened preview entry, a PNG, in [`COMPOSITE_DIR`]. +pub const COMPOSITE: &str = "composite/preview.png"; +/// The file a directory save holds an exclusive lock on while it runs. ZIP bundles do not have it. +pub const SAVE_LOCK: &str = "save.lock"; + +/// Hex-encoded BLAKE3 hash (64 lowercase hex digits) of an object's uncompressed bytes. +pub type Hash = String; + +/// The entry of the tile with hash `hash`: `tiles/.zst`. `None` when [`is_valid_hash`] +/// refuses `hash`. +pub fn tile_path(hash: &str) -> Option { + is_valid_hash(hash).then(|| format!("{TILES_DIR}/{hash}{OBJECT_SUFFIX}")) +} + +/// The entry of the blob with hash `hash`: `blobs/.zst`. `None` when [`is_valid_hash`] +/// refuses `hash`. +pub fn blob_path(hash: &str) -> Option { + is_valid_hash(hash).then(|| format!("{BLOBS_DIR}/{hash}{OBJECT_SUFFIX}")) +} + +/// `true` when `hash` is 64 lowercase hex digits, the only form a valid manifest gives. +/// [`tile_path`] and [`blob_path`] build no entry name for any other string. +pub fn is_valid_hash(hash: &str) -> bool { + hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +/// `manifest.json`: the envelope around the app's document `D`. +/// +/// The fields serialize in this order, and `thumbnail` and `composite` serialize as `null` when +/// absent. [`read_manifest`] reads one: a missing `thumbnail` or `composite` reads as `None`, +/// members the type does not name are skipped, and a preview that names any entry other than +/// [`THUMB`] or [`COMPOSITE`] is refused. +/// +/// The struct is `#[non_exhaustive]`: build one with [`Manifest::new`] and set the previews. +/// +/// ```compile_fail +/// let _ = craft_pcraft_format::Manifest { format_version: 1, generator: String::new(), document: (), thumbnail: None, composite: None }; +/// ``` +/// +/// It has no `Deserialize`, so a manifest is read only through [`read_manifest`]: +/// +/// ```compile_fail +/// let _: craft_pcraft_format::Manifest = serde_json::from_str("{}").unwrap(); +/// ``` +#[derive(Debug, Clone, PartialEq, Serialize)] +#[non_exhaustive] +pub struct Manifest { + pub format_version: u32, + /// Free-form writer identification, for example `photocraft-format 0.1.0`. + pub generator: String, + pub document: D, + /// The thumbnail entry, [`THUMB`]. [`read_manifest`] refuses any other name. + pub thumbnail: Option, + /// The flattened preview entry, [`COMPOSITE`]. [`read_manifest`] refuses any other name. + pub composite: Option, +} + +impl Manifest { + /// A manifest at [`FORMAT_VERSION`] written by `generator`, holding `document`, with no + /// previews. + pub fn new(generator: impl Into, document: D) -> Self { + Manifest { format_version: FORMAT_VERSION, generator: generator.into(), document, thumbnail: None, composite: None } + } +} + +/// One tile of a surface: its column `tx`, its row `ty` and the hash of its samples, stored at +/// [`tile_path`]. `hash` is read as written; [`TileRef::path`] checks it. +/// +/// The struct is `#[non_exhaustive]`: build one with [`TileRef::new`]. +/// +/// ```compile_fail +/// let _ = craft_pcraft_format::TileRef { tx: 0, ty: 0, hash: String::new() }; +/// ``` +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[non_exhaustive] +pub struct TileRef { + pub tx: i32, + pub ty: i32, + pub hash: Hash, +} + +impl TileRef { + /// The tile at column `tx` and row `ty` whose samples hash to `hash`. + pub fn new(tx: i32, ty: i32, hash: Hash) -> Self { + TileRef { tx, ty, hash } + } + + /// The tile's entry, [`tile_path`] of its `hash`: `None` when the hash is not valid. + pub fn path(&self) -> Option { + tile_path(&self.hash) + } +} + +/// Why a manifest could not be read. +/// +/// [`Error::TooNew`] is `#[non_exhaustive]`: match it with `..`. +/// +/// ```compile_fail +/// fn found(e: craft_pcraft_format::Error) -> u32 { +/// match e { +/// craft_pcraft_format::Error::TooNew { found, supported: _, generator: _ } => found, +/// _ => 0, +/// } +/// } +/// ``` +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// The manifest breaks the format. + #[error("corrupt .pcraft bundle: {0}")] + Corrupt(String), + /// The manifest needs a migration step the caller did not pass. + #[error("unsupported .pcraft feature: {0}")] + Unsupported(String), + /// The manifest's `format_version` is newer than the version the caller reads. + #[error("bundle written by a newer version (format {found}, this build reads up to {supported})")] + #[non_exhaustive] + TooNew { + /// The manifest's `format_version`. + found: u32, + /// The version the caller reads. + supported: u32, + /// The manifest's `generator`, so a message can name the writer. `None` when it is + /// missing or not a string. + generator: Option, + }, + /// The manifest passes one of the caller's [`Limits`]. + #[error("limit exceeded: {0}")] + LimitExceeded(String), + /// The manifest is not valid JSON, or a member does not have the type the reader expects. + #[error("manifest JSON: {0}")] + Json(#[from] serde_json::Error), +} diff --git a/crates/pcraft-format/src/limits.rs b/crates/pcraft-format/src/limits.rs new file mode 100644 index 00000000..62c4ad10 --- /dev/null +++ b/crates/pcraft-format/src/limits.rs @@ -0,0 +1,114 @@ +use crate::Error; + +/// Deepest nesting of layer groups PhotoCraft saves. The value is a copy of PhotoCraft's +/// `photocraft_doc::MAX_GROUP_DEPTH`; change both together. +pub const MAX_GROUP_DEPTH: usize = 100; + +/// Default [`Limits::max_items`]: the most JSON items (arrays and objects opened, plus elements +/// and members after a comma) a manifest may hold. +pub const MAX_MANIFEST_ITEMS: usize = 4 << 20; + +/// The largest [`Limits::max_depth`] the checks apply. A larger value is treated as this one, +/// so parsing stays within the stack. +pub const MAX_DEPTH: usize = 400; + +/// The deepest JSON nesting of a manifest whose layer groups nest `groups` deep: three levels per +/// group (the layer object, its content object and the `children` array) and 64 for the document +/// and the leaf layers' fields. +pub const fn manifest_depth(groups: usize) -> usize { + 3usize.saturating_mul(groups.saturating_add(1)).saturating_add(64) +} + +/// Limits applied while reading a bundle from an untrusted source. The defaults are the ones +/// PhotoCraft loads with. [`check_manifest`], [`parse_manifest`] and [`crate::read_manifest`] +/// apply `max_manifest_bytes`, `max_depth` and `max_items`; the caller applies `max_blob_bytes` +/// and `max_total_bytes`. +/// +/// The struct is `#[non_exhaustive]`: start from [`Limits::default`] and set the fields you +/// change. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct Limits { + /// Largest `manifest.json`, in bytes (256 MiB). + pub max_manifest_bytes: usize, + /// Largest decompressed blob, in bytes (1 GiB). The crate does not read blobs or apply this + /// limit: the caller that decompresses them enforces it. + pub max_blob_bytes: usize, + /// Most decompressed tile and blob bytes in one bundle (16 GiB). The crate does not read + /// tiles or blobs or apply this limit: the caller that decompresses them enforces it. + pub max_total_bytes: u64, + /// Deepest JSON nesting of the manifest (`manifest_depth(MAX_GROUP_DEPTH)`, 367). Values + /// above [`MAX_DEPTH`] are treated as [`MAX_DEPTH`]. + pub max_depth: usize, + /// Most JSON items in the manifest ([`MAX_MANIFEST_ITEMS`]). + pub max_items: usize, +} + +impl Default for Limits { + fn default() -> Self { + Limits { + max_manifest_bytes: 256 << 20, + max_blob_bytes: 1 << 30, + max_total_bytes: 16 << 30, + max_depth: manifest_depth(MAX_GROUP_DEPTH), + max_items: MAX_MANIFEST_ITEMS, + } + } +} + +/// Checks manifest JSON against [`Limits::max_manifest_bytes`], [`Limits::max_depth`] (at most +/// [`MAX_DEPTH`]) and +/// [`Limits::max_items`] in one pass without recursion. Brackets and commas inside strings are +/// not counted. Returns [`Error::LimitExceeded`] for the first limit passed. +/// +/// PhotoCraft's own check of a manifest it is about to save tests the depth and the items only. +/// This check also refuses JSON larger than [`Limits::max_manifest_bytes`], so a writer that runs +/// it before saving refuses a manifest of more than that size. +pub fn check_manifest(json: &[u8], limits: &Limits) -> Result<(), Error> { + if json.len() > limits.max_manifest_bytes { + return Err(Error::LimitExceeded(format!("`{}` exceeds {} bytes", crate::MANIFEST, limits.max_manifest_bytes))); + } + let max_depth = limits.max_depth.min(MAX_DEPTH); + let (mut depth, mut items, mut in_str, mut escaped) = (0usize, 0usize, false, false); + for &b in json { + if in_str { + match b { + _ if escaped => escaped = false, + b'\\' => escaped = true, + b'"' => in_str = false, + _ => {} + } + continue; + } + match b { + b'"' => in_str = true, + b'[' | b'{' => { + depth += 1; + if depth > max_depth { + return Err(Error::LimitExceeded(format!("manifest JSON nested deeper than {max_depth} levels"))); + } + } + b']' | b'}' => depth = depth.saturating_sub(1), + _ => {} + } + if matches!(b, b'[' | b'{' | b',') { + items += 1; + if items > limits.max_items { + return Err(Error::LimitExceeded(format!("manifest JSON holds more than {} values", limits.max_items))); + } + } + } + Ok(()) +} + +/// Checks manifest JSON with [`check_manifest`], then parses it into a [`serde_json::Value`] +/// with serde_json's recursion limit turned off, so a manifest nested deeper than 128 levels +/// and within [`Limits::max_depth`] parses. +pub fn parse_manifest(json: &[u8], limits: &Limits) -> Result { + check_manifest(json, limits)?; + let mut de = serde_json::Deserializer::from_slice(json); + de.disable_recursion_limit(); + let value = ::deserialize(&mut de)?; + de.end()?; + Ok(value) +} diff --git a/crates/pcraft-format/src/migrate.rs b/crates/pcraft-format/src/migrate.rs new file mode 100644 index 00000000..2751c38f --- /dev/null +++ b/crates/pcraft-format/src/migrate.rs @@ -0,0 +1,41 @@ +use serde_json::Value; + +use crate::Error; + +/// One migration step: upgrades manifest JSON from version `n` to `n + 1` in place. The runner +/// sets `format_version` after each step. +pub type Step = fn(&mut Value) -> Result<(), E>; + +/// Upgrades the manifest JSON `v` in place to version `target`, where `steps[i]` upgrades +/// version `i + 1` to `i + 2`. Returns the version the manifest had. +/// +/// - A missing or non-integer `format_version`, one past `u32`, or `0` is [`Error::Corrupt`]. +/// - A version newer than `target` is [`Error::TooNew`], holding the manifest's `generator` when +/// it is a string. +/// - A version with no step to the next one is [`Error::Unsupported`]. +/// +/// A step's error is returned as it is, and the steps before it have already changed `v`. A step +/// that leaves `v` something other than a JSON object is [`Error::Corrupt`]. +pub fn migrate>(v: &mut Value, steps: &[Step], target: u32) -> Result { + let found = v.get("format_version").and_then(Value::as_u64).ok_or_else(|| corrupt("manifest has no format_version"))?; + let found = u32::try_from(found).map_err(|_| corrupt("format_version out of range"))?; + if found == 0 { + return Err(corrupt("format_version 0 is invalid").into()); + } + if found > target { + let generator = v.get("generator").and_then(Value::as_str).map(str::to_owned); + return Err(Error::TooNew { found, supported: target, generator }.into()); + } + for ver in found..target { + let step = + usize::try_from(ver - 1).ok().and_then(|i| steps.get(i)).ok_or_else(|| Error::Unsupported(format!("no migration from format {ver}")))?; + step(v)?; + let object = v.as_object_mut().ok_or_else(|| corrupt("a migration step left the manifest without an object"))?; + object.insert("format_version".to_owned(), Value::from(ver + 1)); + } + Ok(found) +} + +fn corrupt(msg: &str) -> Error { + Error::Corrupt(msg.to_owned()) +} diff --git a/crates/pcraft-format/src/read.rs b/crates/pcraft-format/src/read.rs new file mode 100644 index 00000000..ab9b5167 --- /dev/null +++ b/crates/pcraft-format/src/read.rs @@ -0,0 +1,73 @@ +use serde::Deserialize; +use serde::de::DeserializeOwned; +use serde_json::Value; + +use crate::{COMPOSITE, Error, Limits, Manifest, Step, THUMB, migrate, parse_manifest}; + +/// Reads manifest JSON into a [`Manifest`] whose document is `D`. +/// +/// 1. [`parse_manifest`] checks `json` against `limits` and parses it. +/// 2. [`migrate`] refuses a `format_version` newer than `target` with [`Error::TooNew`], which +/// holds the manifest's `generator`, and upgrades an older one with `steps`. +/// 3. The members are read in the order the parsed JSON object holds them, the document as `D`. +/// +/// A missing `thumbnail` or `composite` reads as `None`, and members the envelope does not name +/// are skipped. A missing or invalid `format_version` is [`Error::Corrupt`], from [`migrate`]. +/// Any other member of the wrong type is [`Error::Json`], and so is a missing `generator` or +/// `document`, which is reported after the members that are present, in the order serde's derived +/// `Deserialize` reports them. A `thumbnail` other than [`THUMB`] or a `composite` other than +/// [`COMPOSITE`] is [`Error::Corrupt`], checked after the members are read. +pub fn read_manifest>(json: &[u8], limits: &Limits, steps: &[Step], target: u32) -> Result, E> { + read_manifest_with(json, limits, steps, target, |document| serde_json::from_value(document).map_err(|e| E::from(Error::Json(e)))) +} + +/// [`read_manifest`], with the document read by `document` from its JSON value instead of by +/// `D`'s `Deserialize`, for example to decode a deep tree a level at a time. +pub fn read_manifest_with>( + json: &[u8], + limits: &Limits, + steps: &[Step], + target: u32, + document: impl FnOnce(Value) -> Result, +) -> Result, E> { + let mut read_document = Some(document); + let mut value = parse_manifest(json, limits)?; + migrate(&mut value, steps, target)?; + let Value::Object(members) = value else { + return Err(Error::Corrupt("the manifest is not a JSON object".to_owned()).into()); + }; + let (mut format_version, mut generator, mut doc, mut thumbnail, mut composite) = (None, None, None, None, None); + for (name, member) in members { + match name.as_str() { + "format_version" => format_version = Some(u32::deserialize(member).map_err(Error::Json)?), + "generator" => generator = Some(String::deserialize(member).map_err(Error::Json)?), + "document" => { + if let Some(read) = read_document.take() { + doc = Some(read(member)?); + } + } + "thumbnail" => thumbnail = Option::::deserialize(member).map_err(Error::Json)?, + "composite" => composite = Option::::deserialize(member).map_err(Error::Json)?, + _ => {} + } + } + let missing = |name| Error::Json(serde::de::Error::missing_field(name)); + let manifest = Manifest { + format_version: format_version.ok_or_else(|| missing("format_version"))?, + generator: generator.ok_or_else(|| missing("generator"))?, + document: doc.ok_or_else(|| missing("document"))?, + thumbnail, + composite, + }; + check_preview("thumbnail", manifest.thumbnail.as_deref(), THUMB)?; + check_preview("composite", manifest.composite.as_deref(), COMPOSITE)?; + Ok(manifest) +} + +/// Refuses a preview member that names an entry other than `expected`. +fn check_preview(member: &str, entry: Option<&str>, expected: &str) -> Result<(), Error> { + match entry { + Some(entry) if entry != expected => Err(Error::Corrupt(format!("`{member}` names {entry:?}, not {expected:?}"))), + _ => Ok(()), + } +} diff --git a/crates/pcraft-format/tests/limits.rs b/crates/pcraft-format/tests/limits.rs new file mode 100644 index 00000000..a7caf6f1 --- /dev/null +++ b/crates/pcraft-format/tests/limits.rs @@ -0,0 +1,101 @@ +use craft_pcraft_format::{Error, Limits, MAX_DEPTH, MAX_GROUP_DEPTH, MAX_MANIFEST_ITEMS, check_manifest, manifest_depth, parse_manifest}; + +fn nested(depth: usize) -> Vec { + let mut json = "[".repeat(depth).into_bytes(); + json.extend(std::iter::repeat_n(b']', depth)); + json +} + +fn small(max_depth: usize, max_items: usize) -> Limits { + let mut limits = Limits::default(); + limits.max_depth = max_depth; + limits.max_items = max_items; + limits +} + +#[test] +fn defaults_are_photocraft_s_load_limits() { + let limits = Limits::default(); + assert_eq!(limits.max_manifest_bytes, 256 << 20); + assert_eq!(limits.max_blob_bytes, 1 << 30); + assert_eq!(limits.max_total_bytes, 16 << 30); + assert_eq!(MAX_GROUP_DEPTH, 100); + assert_eq!(limits.max_depth, 367); + assert_eq!(limits.max_depth, manifest_depth(MAX_GROUP_DEPTH)); + assert_eq!(limits.max_items, MAX_MANIFEST_ITEMS); + assert_eq!(MAX_MANIFEST_ITEMS, 4 << 20); +} + +#[test] +fn manifest_depth_saturates() { + assert_eq!(manifest_depth(0), 67); + assert_eq!(manifest_depth(usize::MAX), usize::MAX); +} + +#[test] +fn nesting_up_to_max_depth_passes_and_one_more_level_does_not() { + let limits = Limits::default(); + let at = limits.max_depth; + assert!(check_manifest(&nested(at), &limits).is_ok()); + assert!(matches!(check_manifest(&nested(at + 1), &limits), Err(Error::LimitExceeded(_)))); +} + +#[test] +fn a_manifest_deeper_than_serde_json_s_128_levels_parses() { + let value = parse_manifest(&nested(300), &Limits::default()).unwrap(); + assert!(value.is_array()); +} + +#[test] +fn a_raised_max_depth_stops_at_max_depth_cap() { + let limits = small(100_000, MAX_MANIFEST_ITEMS); + assert!(parse_manifest(&nested(MAX_DEPTH), &limits).unwrap().is_array()); + let deeper = parse_manifest(&nested(MAX_DEPTH + 1), &limits).err().map(|e| e.to_string()); + assert_eq!(deeper, Some(format!("limit exceeded: manifest JSON nested deeper than {MAX_DEPTH} levels"))); + assert!(matches!(parse_manifest(&nested(100_000), &limits), Err(Error::LimitExceeded(_)))); +} + +#[test] +fn items_up_to_max_items_pass_and_one_more_does_not() { + let limits = small(8, 4); + assert!(check_manifest(b"[1,2,3,4]", &limits).is_ok()); + assert!(matches!(check_manifest(b"[1,2,3,4,5]", &limits), Err(Error::LimitExceeded(_)))); + assert!(matches!(check_manifest(b"[[[[[]]]]]", &limits), Err(Error::LimitExceeded(_)))); +} + +#[test] +fn brackets_and_commas_inside_strings_are_not_counted() { + let limits = small(2, 3); + assert!(check_manifest(br#"{"a":"[[[{{{,,,,"}"#, &limits).is_ok()); + assert!(check_manifest(br#"{"a":"\"[[[,,,\\"}"#, &limits).is_ok()); + assert!(matches!(check_manifest(br#"{"a":"\\",[[[]]]}"#, &limits), Err(Error::LimitExceeded(_)))); +} + +#[test] +fn a_manifest_larger_than_max_manifest_bytes_is_refused() { + let mut limits = Limits::default(); + limits.max_manifest_bytes = 8; + assert!(check_manifest(b"{\"a\":12}", &limits).is_ok()); + assert!(matches!(check_manifest(b"{\"a\":123}", &limits), Err(Error::LimitExceeded(_)))); + assert!(matches!(parse_manifest(b"{\"a\":123}", &limits), Err(Error::LimitExceeded(_)))); +} + +#[test] +fn invalid_or_trailing_json_is_a_json_error() { + let limits = Limits::default(); + assert!(matches!(parse_manifest(b"{\"a\":", &limits), Err(Error::Json(_)))); + assert!(matches!(parse_manifest(b"{} {}", &limits), Err(Error::Json(_)))); + assert!(matches!(parse_manifest(b"", &limits), Err(Error::Json(_)))); +} + +#[test] +fn messages_read_as_photocraft_s() { + let none: &[craft_pcraft_format::Step] = &[]; + let too_new = craft_pcraft_format::migrate(&mut serde_json::json!({"format_version": 2, "generator": "g"}), none, 1).err().map(|e| e.to_string()); + assert_eq!(too_new.as_deref(), Some("bundle written by a newer version (format 2, this build reads up to 1)")); + assert_eq!(Error::Corrupt("x".into()).to_string(), "corrupt .pcraft bundle: x"); + assert_eq!(Error::Unsupported("x".into()).to_string(), "unsupported .pcraft feature: x"); + assert_eq!(Error::LimitExceeded("x".into()).to_string(), "limit exceeded: x"); + let deep = check_manifest(&nested(368), &Limits::default()).err().map(|e| e.to_string()); + assert_eq!(deep.as_deref(), Some("limit exceeded: manifest JSON nested deeper than 367 levels")); +} diff --git a/crates/pcraft-format/tests/manifest.rs b/crates/pcraft-format/tests/manifest.rs new file mode 100644 index 00000000..550c829b --- /dev/null +++ b/crates/pcraft-format/tests/manifest.rs @@ -0,0 +1,152 @@ +use craft_pcraft_format::{ + BLOBS_DIR, COMPOSITE, COMPOSITE_DIR, EXTENSION, FORMAT_VERSION, Limits, MANIFEST, Manifest, OBJECT_SUFFIX, SAVE_LOCK, Step, THUMB, TILES_DIR, + TileRef, blob_path, is_valid_hash, tile_path, +}; +use serde::{Deserialize, Serialize}; + +fn read(text: &str) -> Result, craft_pcraft_format::Error> { + let none: &[Step] = &[]; + craft_pcraft_format::read_manifest(text.as_bytes(), &Limits::default(), none, FORMAT_VERSION) +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +struct Doc { + id: u64, + name: String, + tiles: Vec, +} + +const HASH: &str = "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"; + +fn doc() -> Doc { + Doc { id: 7, name: "Untitled-1".into(), tiles: vec![TileRef::new(0, -1, HASH.into())] } +} + +fn manifest(previews: bool) -> Manifest { + let mut manifest = Manifest::new("photocraft-format 0.1.0", doc()); + manifest.thumbnail = previews.then(|| THUMB.to_owned()); + manifest.composite = previews.then(|| COMPOSITE.to_owned()); + manifest +} + +const WITH_PREVIEWS: &str = r#"{ + "format_version": 1, + "generator": "photocraft-format 0.1.0", + "document": { + "id": 7, + "name": "Untitled-1", + "tiles": [ + { + "tx": 0, + "ty": -1, + "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" + } + ] + }, + "thumbnail": "thumb.png", + "composite": "composite/preview.png" +}"#; + +const WITHOUT_PREVIEWS: &str = r#"{ + "format_version": 1, + "generator": "photocraft-format 0.1.0", + "document": { + "id": 7, + "name": "Untitled-1", + "tiles": [ + { + "tx": 0, + "ty": -1, + "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" + } + ] + }, + "thumbnail": null, + "composite": null +}"#; + +#[test] +fn a_manifest_with_previews_writes_photocraft_s_pretty_json_byte_for_byte() { + let bytes = serde_json::to_vec_pretty(&manifest(true)).unwrap(); + assert_eq!(String::from_utf8(bytes).unwrap(), WITH_PREVIEWS); +} + +#[test] +fn a_manifest_without_previews_writes_null_members() { + let bytes = serde_json::to_vec_pretty(&manifest(false)).unwrap(); + assert_eq!(String::from_utf8(bytes).unwrap(), WITHOUT_PREVIEWS); +} + +#[test] +fn written_manifests_read_back_equal() { + for (text, previews) in [(WITH_PREVIEWS, true), (WITHOUT_PREVIEWS, false)] { + let read: Manifest = read(text).unwrap(); + assert_eq!(read, manifest(previews)); + } +} + +#[test] +fn a_new_manifest_is_at_the_current_version_without_previews() { + let m = Manifest::new(String::from("g"), 5); + assert_eq!((m.format_version, m.generator.as_str(), m.document, m.thumbnail, m.composite), (FORMAT_VERSION, "g", 5, None, None)); +} + +#[test] +fn missing_preview_members_read_as_none_and_unknown_members_are_skipped() { + let text = + r#"{"format_version":1,"generator":"g","document":{"id":1,"name":"a","tiles":[]},"full_composite":"composite/full.png","later":{"x":[1,2]}}"#; + let read: Manifest = read(text).unwrap(); + assert_eq!(read.thumbnail, None); + assert_eq!(read.composite, None); + assert_eq!(read.document.name, "a"); +} + +#[test] +fn a_manifest_without_a_document_or_generator_does_not_read() { + assert!(read::(r#"{"format_version":1,"generator":"g"}"#).is_err()); + assert!(read::(r#"{"format_version":1,"document":{"id":1,"name":"a","tiles":[]}}"#).is_err()); +} + +#[test] +fn the_document_can_be_any_json_value() { + let read: Manifest = read(WITH_PREVIEWS).unwrap(); + assert_eq!(read.document["tiles"][0]["ty"], -1); + assert_eq!(read.composite.as_deref(), Some(COMPOSITE)); +} + +#[test] +fn entry_names_match_the_bundle_layout() { + assert_eq!(EXTENSION, "pcraft"); + assert_eq!(MANIFEST, "manifest.json"); + assert_eq!(THUMB, "thumb.png"); + assert_eq!(COMPOSITE, "composite/preview.png"); + assert_eq!(SAVE_LOCK, "save.lock"); + assert_eq!((TILES_DIR, BLOBS_DIR, COMPOSITE_DIR, OBJECT_SUFFIX), ("tiles", "blobs", "composite", ".zst")); + assert!(COMPOSITE.starts_with(&format!("{COMPOSITE_DIR}/"))); + assert_eq!(tile_path(HASH), Some(format!("tiles/{HASH}.zst"))); + assert_eq!(blob_path(HASH), Some(format!("blobs/{HASH}.zst"))); +} + +#[test] +fn no_entry_name_is_built_from_an_invalid_hash() { + for hash in ["../../etc/passwd", "../x", "", &HASH.to_uppercase(), &HASH[1..]] { + assert_eq!(tile_path(hash), None, "{hash}"); + assert_eq!(blob_path(hash), None, "{hash}"); + } + let tile: TileRef = serde_json::from_str(r#"{"tx":0,"ty":0,"hash":"../x"}"#).unwrap(); + assert_eq!(tile.path(), None); + let tile = TileRef::new(1, -1, HASH.into()); + assert_eq!(tile.path(), Some(format!("tiles/{HASH}.zst"))); +} + +#[test] +fn only_64_lowercase_hex_digits_are_a_valid_hash() { + assert!(is_valid_hash(HASH)); + assert!(is_valid_hash(&"0".repeat(64))); + assert!(!is_valid_hash(&HASH.to_uppercase())); + assert!(!is_valid_hash(&HASH[1..])); + assert!(!is_valid_hash(&format!("{HASH}0"))); + assert!(!is_valid_hash(&format!("../{}", &HASH[3..]))); + assert!(!is_valid_hash(&format!("{}g", &HASH[1..]))); + assert!(!is_valid_hash("")); +} diff --git a/crates/pcraft-format/tests/migrate.rs b/crates/pcraft-format/tests/migrate.rs new file mode 100644 index 00000000..8aae1430 --- /dev/null +++ b/crates/pcraft-format/tests/migrate.rs @@ -0,0 +1,114 @@ +use craft_pcraft_format::{Error, FORMAT_VERSION, Step, migrate}; +use serde_json::{Value, json}; + +fn rename_title(v: &mut Value) -> Result<(), Error> { + let document = v.get_mut("document").and_then(Value::as_object_mut).ok_or(Error::Corrupt("no document".into()))?; + let title = document.remove("title").unwrap_or_default(); + document.insert("name".into(), title); + Ok(()) +} + +fn add_flag(v: &mut Value) -> Result<(), Error> { + v["document"]["flag"] = json!(true); + Ok(()) +} + +fn replace_with_array(v: &mut Value) -> Result<(), Error> { + *v = json!([]); + Ok(()) +} + +const STEPS: &[Step] = &[rename_title, add_flag]; + +#[test] +fn steps_run_in_order_from_the_file_s_version() { + let mut v = json!({"format_version": 1, "document": {"title": "old"}}); + assert_eq!(migrate(&mut v, STEPS, 3).unwrap(), 1); + assert_eq!(v, json!({"format_version": 3, "document": {"name": "old", "flag": true}})); +} + +#[test] +fn a_file_at_a_later_version_runs_only_the_later_steps() { + let mut v = json!({"format_version": 2, "document": {"title": "kept"}}); + assert_eq!(migrate(&mut v, STEPS, 3).unwrap(), 2); + assert_eq!(v, json!({"format_version": 3, "document": {"title": "kept", "flag": true}})); +} + +#[test] +fn the_current_version_is_left_unchanged() { + let mut v = json!({"format_version": FORMAT_VERSION, "document": {}}); + let before = v.clone(); + let none: &[Step] = &[]; + assert_eq!(migrate(&mut v, none, FORMAT_VERSION).unwrap(), FORMAT_VERSION); + assert_eq!(v, before); +} + +#[test] +fn a_newer_version_is_too_new() { + let mut v = json!({"format_version": 4}); + assert!(matches!(migrate(&mut v, STEPS, 3), Err(Error::TooNew { found: 4, supported: 3, generator: None, .. }))); + let mut v = json!({"format_version": 4, "generator": "photocraft-format 2.0.0"}); + assert!(matches!(migrate(&mut v, STEPS, 3), Err(Error::TooNew { generator: Some(g), .. }) if g == "photocraft-format 2.0.0")); +} + +#[test] +fn a_missing_zero_or_out_of_range_version_is_corrupt() { + for v in [ + json!({}), + json!({"format_version": 0}), + json!({"format_version": "1"}), + json!({"format_version": -1}), + json!({"format_version": 1.5}), + json!({"format_version": 4_294_967_296_u64}), + json!([1]), + json!(null), + ] { + let mut v = v; + assert!(matches!(migrate(&mut v, STEPS, 3), Err(Error::Corrupt(_))), "{v}"); + } +} + +#[test] +fn a_version_with_no_step_is_unsupported() { + let mut v = json!({"format_version": 1, "document": {"title": "t"}}); + let one: &[Step] = &[rename_title]; + assert!(matches!(migrate(&mut v, one, 3), Err(Error::Unsupported(m)) if m == "no migration from format 2")); + assert_eq!(v["format_version"], 2); +} + +#[test] +fn a_step_that_leaves_no_object_is_corrupt() { + let mut v = json!({"format_version": 1}); + let steps: &[Step] = &[replace_with_array]; + assert!(matches!(migrate(&mut v, steps, 2), Err(Error::Corrupt(_)))); +} + +#[derive(Debug)] +enum AppError { + Format(Error), + Step(&'static str), +} + +impl From for AppError { + fn from(e: Error) -> Self { + AppError::Format(e) + } +} + +fn failing(_: &mut Value) -> Result<(), AppError> { + Err(AppError::Step("refused")) +} + +fn app_add_flag(v: &mut Value) -> Result<(), AppError> { + add_flag(v).map_err(AppError::from) +} + +#[test] +fn steps_return_the_caller_s_error_type() { + let steps: &[Step] = &[app_add_flag, failing]; + let mut v = json!({"format_version": 1, "document": {}}); + assert!(matches!(migrate(&mut v, steps, 3), Err(AppError::Step("refused")))); + assert_eq!(v, json!({"format_version": 2, "document": {"flag": true}})); + let mut newer = json!({"format_version": 9}); + assert!(matches!(migrate(&mut newer, steps, 3), Err(AppError::Format(Error::TooNew { found: 9, supported: 3, .. })))); +} diff --git a/crates/pcraft-format/tests/read.rs b/crates/pcraft-format/tests/read.rs new file mode 100644 index 00000000..ab3a218e --- /dev/null +++ b/crates/pcraft-format/tests/read.rs @@ -0,0 +1,191 @@ +use craft_pcraft_format::{COMPOSITE, Error, FORMAT_VERSION, Limits, Manifest, Step, THUMB, TileRef, read_manifest, read_manifest_with}; +use serde::Deserialize; +use serde_json::{Value, json}; + +#[derive(Debug, Clone, PartialEq, Deserialize)] +struct Doc { + name: String, + #[serde(default)] + tiles: Vec, +} + +const NONE: &[Step] = &[]; + +fn read(json: &[u8]) -> Result, Error> { + read_manifest(json, &Limits::default(), NONE, FORMAT_VERSION) +} + +fn nested_document(depth: usize) -> Vec { + let mut json = br#"{"format_version":1,"generator":"g","document":{"name":"a","deep":"#.to_vec(); + json.extend("[".repeat(depth).bytes()); + json.extend("]".repeat(depth).bytes()); + json.extend(b"}}"); + json +} + +#[test] +fn a_manifest_reads_with_its_previews_and_document() { + let json = br#"{"format_version":1,"generator":"photocraft-format 0.1.0","document":{"name":"a","tiles":[{"tx":0,"ty":-1,"hash":"x"}]},"thumbnail":"thumb.png","composite":null,"later":[1]}"#; + let m = read(json).unwrap(); + assert_eq!(m.format_version, 1); + assert_eq!(m.generator, "photocraft-format 0.1.0"); + assert_eq!(m.document.name, "a"); + assert_eq!(m.document.tiles, vec![TileRef::new(0, -1, "x".into())]); + assert_eq!(m.thumbnail.as_deref(), Some("thumb.png")); + assert_eq!(m.composite, None); +} + +#[test] +fn a_newer_manifest_is_too_new_and_names_its_writer() { + let json = br#"{"format_version":2,"generator":"photocraft-format 9.0.0","document":{"name":"a"}}"#; + let Err(Error::TooNew { found, supported, generator, .. }) = read(json) else { panic!("not TooNew") }; + assert_eq!((found, supported), (2, 1)); + assert_eq!(generator.as_deref(), Some("photocraft-format 9.0.0")); + let message = read(json).err().map(|e| e.to_string()); + assert_eq!(message.as_deref(), Some("bundle written by a newer version (format 2, this build reads up to 1)")); +} + +#[test] +fn a_newer_manifest_without_a_string_generator_is_too_new_without_one() { + for json in [&br#"{"format_version":3,"document":{}}"#[..], br#"{"format_version":3,"generator":7}"#] { + assert!(matches!(read(json), Err(Error::TooNew { found: 3, generator: None, .. }))); + } +} + +#[test] +fn a_manifest_past_the_limits_is_refused_before_it_is_read() { + let limits = Limits::default(); + let deep = nested_document(limits.max_depth); + assert!(matches!(read(&deep), Err(Error::LimitExceeded(m)) if m == "manifest JSON nested deeper than 367 levels")); + let within = nested_document(limits.max_depth - 2); + let m: Manifest = read_manifest(&within, &limits, NONE, FORMAT_VERSION).unwrap(); + assert_eq!(m.document["name"], "a"); + + let mut small = Limits::default(); + small.max_manifest_bytes = 10; + let json = br#"{"format_version":1,"generator":"g","document":{"name":"a"}}"#; + let large: Result, Error> = read_manifest(json, &small, NONE, FORMAT_VERSION); + assert!(matches!(large, Err(Error::LimitExceeded(m)) if m == "`manifest.json` exceeds 10 bytes")); + + let mut few = Limits::default(); + few.max_items = 2; + let many: Result, Error> = read_manifest(br#"{"format_version":1,"generator":"g","document":{"name":"a"}}"#, &few, NONE, 1); + assert!(matches!(many, Err(Error::LimitExceeded(_)))); +} + +#[test] +fn the_steps_run_before_the_document_is_read() { + fn rename(v: &mut Value) -> Result<(), Error> { + v["document"]["name"] = v["document"]["title"].take(); + Ok(()) + } + let steps: &[Step] = &[rename]; + let json = br#"{"format_version":1,"generator":"g","document":{"title":"old"}}"#; + let m: Manifest = read_manifest(json, &Limits::default(), steps, 2).unwrap(); + assert_eq!(m.format_version, 2); + assert_eq!(m.document.name, "old"); +} + +#[test] +fn a_missing_or_invalid_envelope_member_is_a_json_error() { + for (json, message) in [ + (&br#"{"format_version":1,"generator":"g"}"#[..], "missing field `document`"), + (br#"{"format_version":1,"document":{"name":"a"}}"#, "missing field `generator`"), + (br#"{"format_version":1,"generator":"g","document":{"name":"a"},"thumbnail":5}"#, "invalid type: integer `5`, expected a string"), + (br#"{"format_version":1,"generator":"g","document":{"title":"a"}}"#, "missing field `name`"), + ] { + let e = read(json).err(); + assert!(matches!(&e, Some(Error::Json(_))), "{e:?}"); + assert_eq!(e.map(|e| e.to_string()).as_deref(), Some(format!("manifest JSON: {message}").as_str())); + } +} + +/// The manifest as a struct with a derived `Deserialize`, for comparing errors. +#[derive(Debug, Deserialize)] +#[allow(dead_code)] +struct Derived { + format_version: u32, + generator: String, + document: Doc, + #[serde(default)] + thumbnail: Option, + #[serde(default)] + composite: Option, +} + +#[test] +fn a_manifest_with_several_faults_reports_the_one_a_derived_struct_reports() { + for json in [ + json!({"format_version": 1, "composite": 1, "document": {"title": "a"}}), + json!({"format_version": 1, "generator": 2, "document": {"title": "a"}}), + json!({"format_version": 1, "document": {"title": "a"}, "thumbnail": 3}), + json!({"format_version": 1, "document": {"name": "a"}, "thumbnail": 3}), + json!({"format_version": 1, "thumbnail": 3}), + json!({"format_version": 1}), + json!({"format_version": 1, "generator": "g"}), + json!({"format_version": 1, "composite": [], "generator": {}}), + ] { + let want = serde_json::from_value::(json.clone()).err().map(|e| format!("manifest JSON: {e}")); + let got = read(json.to_string().as_bytes()).err().map(|e| e.to_string()); + assert!(want.is_some()); + assert_eq!(got, want, "{json}"); + } +} + +#[test] +fn a_missing_or_invalid_version_is_corrupt() { + assert!(matches!(read(br#"{"generator":"g","document":{"name":"a"}}"#), Err(Error::Corrupt(_)))); + assert!(matches!(read(br#"{"format_version":"1","generator":"g","document":{"name":"a"}}"#), Err(Error::Corrupt(_)))); + assert!(matches!(read(br#"{"format_version":1.5,"generator":"g","document":{"name":"a"}}"#), Err(Error::Corrupt(_)))); + assert!(matches!(read(b"[1]"), Err(Error::Corrupt(_)))); +} + +#[test] +fn a_preview_other_than_its_entry_is_corrupt() { + for (member, entry) in [ + ("thumbnail", "../x"), + ("thumbnail", "/etc/passwd"), + ("composite", "../x"), + ("composite", "C:\\x.png"), + ("thumbnail", "composite/preview.png"), + ("composite", "thumb.png"), + ] { + let json = json!({"format_version": 1, "generator": "g", "document": {"name": "a"}, member: entry}); + let e = read(json.to_string().as_bytes()).err(); + assert!(matches!(&e, Some(Error::Corrupt(_))), "{json}: {e:?}"); + } + let json = + json!({"format_version": 1, "generator": "g", "document": {"name": "a"}, "thumbnail": "thumb.png", "composite": "composite/preview.png"}); + let m = read(json.to_string().as_bytes()).unwrap(); + assert_eq!((m.thumbnail.as_deref(), m.composite.as_deref()), (Some(THUMB), Some(COMPOSITE))); +} + +#[derive(Debug)] +enum AppError { + Format(Error), + Document(String), +} + +impl From for AppError { + fn from(e: Error) -> Self { + AppError::Format(e) + } +} + +#[test] +fn read_manifest_with_reads_the_document_with_the_caller_s_function() { + let json = br#"{"format_version":1,"generator":"g","document":{"name":"a","layers":[1,2,3]}}"#; + let none: &[Step] = &[]; + let m = read_manifest_with(json, &Limits::default(), none, 1, |mut document: Value| { + let layers = document.get_mut("layers").map(Value::take).unwrap_or_default(); + Ok::<_, AppError>((document, layers)) + }) + .unwrap(); + assert_eq!(m.document.0, json!({"name": "a", "layers": null})); + assert_eq!(m.document.1, json!([1, 2, 3])); + + let refused = read_manifest_with(json, &Limits::default(), none, 1, |_| Err::<(), _>(AppError::Document("no".into()))); + assert!(matches!(refused, Err(AppError::Document(m)) if m == "no")); + let newer = read_manifest_with(br#"{"format_version":2}"#, &Limits::default(), none, 1, |_| Ok::<(), AppError>(())); + assert!(matches!(newer, Err(AppError::Format(Error::TooNew { found: 2, .. })))); +} diff --git a/crates/proof/README.md b/crates/proof/README.md index 9ad76062..97067014 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -13,7 +13,7 @@ craft-proof = { git = "https://github.com/storytold/craft-libs", rev = " | Language | Spelling | Grammar | Hyphenation | |---|---|---|---| -| English (en-US, en-CA, en-GB and the British-spelling Englishes such as en-AU, en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic | +| English (en-US, en-CA, en-AU, en-GB and the British-spelling Englishes such as en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic | | Polish, Portuguese, Italian | the installed Hunspell dictionary | yes | TeX patterns (embedded, or a Language pack) | | Croatian, Turkish, Russian, Czech, Indonesian, Spanish (es, es-MX, es-419, es-UY) | the installed Hunspell dictionary (Russian also `ru-extra.txt`) | yes | Turkish and Russian rules; English fallback otherwise | | German | — | — | TeX patterns (embedded, or a Language pack) | @@ -79,7 +79,7 @@ pub fn packs_dir() -> Option; // ＋ pub fn installed() -> Vec; // InstalledPack { locale, name, licence, path }, by locale pub fn licence_texts() -> Vec<(String, String)>; // (title, text): embedded English notices, then each pack's LICENCES/* pub fn load_errors() -> Vec<(String, String)>; // ＋ (locale or folder, reason): packs skipped, pattern files that failed -pub fn embedded_locales() -> &'static [&'static str]; // ＋ what this build embeds ("en-us", "de", "pl", "pt", "it" by default; "en-us" with CRAFT_PROOF_EMBED=en) +pub fn embedded_locales() -> &'static [&'static str]; // ＋ locales with embedded upstream data, not our supplements ("en-us", "en-gb", "en-ca", "en-au", "pl", "pt", "it", "de" by default; the four English ones with CRAFT_PROOF_EMBED=en) // craft_proof (crate root) pub enum ProofLang { …, Pack(PackLocale), Other } // ＋ Pack: any language without tools of its own (nl, fr, sr-Latn…) @@ -151,14 +151,16 @@ English spelling uses the first of these that is there; `english_source(lang)` s or in WordCraft's own `dictionaries` folder. The user's choice comes first. 2. **Installed English pack:** reserved for Language packs (craft-libs#485, CP4.2). 3. **Embedded SCOWL:** the English's own SCOWL `rel-2026.02.25` Hunspell dictionary from - `CRAFT_I18N_DIR` (`language-data/en-us`, `en-gb` (British `-ise`) or `en-ca`, `spelling.aff` + - `spelling.dic`), read by the affix engine. en-AU, en-NZ, en-IE and the other British-spelling - Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own + `CRAFT_I18N_DIR` (`language-data/en-us`, `en-gb` (British `-ise`), `en-au` or `en-ca`, + `spelling.aff` + `spelling.dic`), read by the affix engine. Australian English reads en-AU + (Australian words such as `billabong`, with en-GB's spelling policy); en-NZ, en-IE and the + other British-spelling Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own dictionary knows it (or `en-extra.txt` does), never because another English's does - (`colourblind` is wrong in en-US, `colorblind` in en-GB); suggestions come from the same - dictionary. SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so + (`colourblind` is wrong in en-US, `colorblind` in en-GB); the one exception is Australian + English, which also takes en-GB's words (`enquiry`: en-AU's list keeps `inquiry` only). + Suggestions come from the English's own dictionary (en-AU's for Australian). SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so British English also takes the `-ize` spellings of its `-ise` words that American or Canadian - English spell so (`organize`, `realization`; never `wize` or `advize`), and `en-variants.txt` flags the other dialect's spellings and accepts this + English spell so (`organize`, `realization`; never `wize` or `advize`, and never `-yze`: `reanalyze` stays American), and `en-variants.txt` flags the other dialect's spellings and accepts this dialect's other spellings of known words (`judgement` in en-US). English with no dialect (`is_correct`) takes a word any of the three knows. Suggestions offer a missing space only between two lower-case words of three letters or more, after every one-word candidate, and no @@ -199,7 +201,7 @@ craft-i18n's copy. The SCOWL notice (SCOWL's licence, the Ispell BSD licence of the affix files and Benjamin Titze's notice) is embedded with the dictionaries: `craft_proof::scowl::notice()`, for an About window. -Embedding SCOWL adds 1,542,396 bytes of Language data (the en-US, en-GB and en-CA `.aff`/`.dic` +Embedding SCOWL adds 2,056,533 bytes of Language data (the en-US, en-GB, en-CA and en-AU `.aff`/`.dic` and the notice); the Moby list (432,905 bytes) stays for hyphenation. ## Check diff --git a/crates/proof/build.rs b/crates/proof/build.rs index 391de460..2922baa0 100644 --- a/crates/proof/build.rs +++ b/crates/proof/build.rs @@ -16,9 +16,8 @@ //! `include_str!` takes a string, so a `CRAFT_I18N_DIR` that isn't valid UTF-8 can't be embedded: //! it is treated like a folder without the data. //! -//! English: SCOWL's en-US, en-GB (-ise) and en-CA dictionaries; en-AU is not embedded, as -//! Australian English is checked as British-spelling English (`ProofLang::EnGb`). The SCOWL -//! notice is embedded once (the four upstream READMEs differ only in their title and build line). +//! English: SCOWL's en-US, en-GB (-ise), en-CA and en-AU dictionaries. The SCOWL notice is +//! embedded once (the four upstream READMEs differ only in their title and build line). use std::fmt::Write as _; use std::path::{Path, PathBuf}; @@ -34,6 +33,8 @@ const FILES: &[(&str, &str, bool)] = &[ ("EN_GB_SPELLING", "en-gb/spelling.dic", false), ("EN_CA_AFF", "en-ca/spelling.aff", false), ("EN_CA_SPELLING", "en-ca/spelling.dic", false), + ("EN_AU_AFF", "en-au/spelling.aff", false), + ("EN_AU_SPELLING", "en-au/spelling.dic", false), ("SCOWL_NOTICE", "en-us/LICENCE-SCOWL.txt", false), ("EN_US_PAT", "en-us/hyphenation.pat", false), ("EXTRA_EN", "en-us/en-extra.txt", false), diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 1c32affc..20ce6fda 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -79,6 +79,14 @@ const MAX_ALIAS_BYTES: usize = 4 << 20; /// Most flags all distinct flag sets (stems' and continuation classes') may hold together: a lookup /// tests each distinct continuation set it meets at most once, so this bounds its cost. const MAX_SET_DATA: usize = 1 << 23; +/// Most flags all stems may carry together before their sets are merged and interned (the same +/// set on many stems counts each time; an `AF` alias counts once). Checked while reading, so a +/// huge `.dic` is refused before its flags fill memory, not after (#495: a 760 MB `.dic` once +/// held 3 GB before [`MAX_SET_DATA`] refused it). +const MAX_STEM_FLAGS: usize = 4 * MAX_SET_DATA; +/// Most bytes of flag text per flag of [`MAX_STEM_FLAGS`] the stem lines may hold in all while +/// they are read (`FLAG num`: `65535,` is six). +const FLAG_TEXT_PER_FLAG: usize = 6; /// Most distinct flags a dictionary may use: each gets a `u16` id. const MAX_DISTINCT_FLAGS: usize = 1 << 16; @@ -1184,12 +1192,19 @@ impl AffixDict { } fn from_hunspell_in(aff: &str, dic: &str, comments: &[&str], english: bool) -> Result { + AffixDict::from_hunspell_capped(aff, dic, comments, english, MAX_STEM_FLAGS) + } + + /// [`AffixDict::from_hunspell_in`] with at most `max_flags` stem flags in all + /// ([`MAX_STEM_FLAGS`]; tests lower it). + fn from_hunspell_capped(aff: &str, dic: &str, comments: &[&str], english: bool, max_flags: usize) -> Result { // Some dictionaries open with a UTF-8 BOM; it must not glue itself to the first // directive or turn the `.dic` count line into a stem. let aff = aff.strip_prefix('\u{feff}').unwrap_or(aff); let dic = dic.strip_prefix('\u{feff}').unwrap_or(dic); let rules = parse_aff(aff, english)?; let mut stems: Vec<(String, String)> = Vec::new(); + let mut flag_text = FlagText::new(max_flags); for (n, line) in dic.lines().enumerate() { // The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it. // Leading blanks belong to it, as in Hunspell (` lab` is no `lab`). @@ -1225,14 +1240,16 @@ impl AffixDict { } // With an `AF` table the field stays an alias number here: build() reads each alias // once, and every stem using it shares those flags. + flag_text.add(&flags)?; stems.push((word, flags)); } - AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect(), MAX_FLAGS_PER_ENTRY) + AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect(), MAX_FLAGS_PER_ENTRY, max_flags) } /// Build from parsed rules and stems whose flag fields carry at most `entry_cap` flags each /// ([`MAX_FLAGS_PER_ENTRY`] for `.dic` entries, [`MAX_FLAGS_PER_STEM`] for compact data). - fn build(rules: Rules, stems: Vec<(String, String)>, comments: Vec, entry_cap: usize) -> Result { + /// The stems' flags may number `max_flags` in all ([`MAX_STEM_FLAGS`]), counted as they are read. + fn build(rules: Rules, stems: Vec<(String, String)>, comments: Vec, entry_cap: usize, max_flags: usize) -> Result { let mode = rules.mode; // A `.dic` entry's malformed flag items read as Hunspell reads them ([`FlagMode`]); an // entry with more than MAX_FLAGS_PER_STEM flags, or compact data whose flags don't read, @@ -1261,20 +1278,24 @@ impl AffixDict { } Some((start, stem_keys.len(), word)) }; - let mut stems: Vec<(String, usize, usize, bool)> = stems - .into_iter() - .filter_map(|(w, f)| { - let range = match rules.alias_index(&f) { - Some(i) => *alias_ranges.entry(i).or_insert_with(|| read(rules.aliases.get(i).map_or("", String::as_str), &mut stem_keys)), - None if rules.aliases_on => Some((0, 0, true)), - None => read(&f, &mut stem_keys), - }; - if range.is_none_or(|r| !r.2) { - skipped += 1; - } - range.map(|(a, b, word)| (w, a, b, word)) - }) - .collect(); + let mut read_stems: Vec<(String, usize, usize, bool)> = Vec::with_capacity(stems.len()); + for (w, f) in stems { + let range = match rules.alias_index(&f) { + Some(i) => *alias_ranges.entry(i).or_insert_with(|| read(rules.aliases.get(i).map_or("", String::as_str), &mut stem_keys)), + None if rules.aliases_on => Some((0, 0, true)), + None => read(&f, &mut stem_keys), + }; + if stem_keys.len() > max_flags { + return Err(format!("the stems carry more than {max_flags} flags in all")); + } + if range.is_none_or(|r| !r.2) { + skipped += 1; + } + if let Some((a, b, word)) = range { + read_stems.push((w, a, b, word)); + } + } + let mut stems = read_stems; if skipped > 0 { log::debug!("affix dictionary: {skipped} entries with unreadable or too many flags skipped"); } @@ -2664,6 +2685,7 @@ impl AffixDict { } let body = raw.get(words_at + 7..).unwrap_or_default(); let mut stems: Vec<(String, String)> = Vec::new(); + let mut flag_text = FlagText::new(MAX_STEM_FLAGS); let mut cur: Vec = Vec::new(); let mut i = 0; while let Some(&p) = body.get(i) { @@ -2686,9 +2708,32 @@ impl AffixDict { if word.is_empty() || word.len() > MAX_WORD_BYTES { return Err("affix dictionary: bad stem".into()); } + flag_text.add(flags).map_err(|e| format!("affix dictionary: {e}"))?; stems.push((word.to_string(), flags.to_string())); } - AffixDict::build(rules, stems, comments, MAX_FLAGS_PER_STEM) + AffixDict::build(rules, stems, comments, MAX_FLAGS_PER_STEM, MAX_STEM_FLAGS) + } +} + +/// A running count of the stems' flag text while a dictionary is read: past +/// [`FLAG_TEXT_PER_FLAG`] bytes for each of `max_flags` flags, reading stops with an error, before +/// the rest of a huge `.dic` is held ([`MAX_STEM_FLAGS`]). +struct FlagText { + bytes: usize, + cap: usize, +} + +impl FlagText { + fn new(max_flags: usize) -> Self { + FlagText { bytes: 0, cap: max_flags.saturating_mul(FLAG_TEXT_PER_FLAG) } + } + + fn add(&mut self, flags: &str) -> Result<(), String> { + self.bytes = self.bytes.saturating_add(flags.len()); + if self.bytes > self.cap { + return Err(format!("the stems' flags take more than {} bytes", self.cap)); + } + Ok(()) } } @@ -2738,6 +2783,30 @@ pub(crate) mod tests { AffixDict::from_hunspell(AFF, DIC, &["test data"]).unwrap() } + /// #495 review: the stems' flags are counted as they are read, so a `.dic` whose flags pass + /// the budget is refused while reading (its flag text) or while its flags are read (the + /// keys), before they are all held; one within it loads. + #[test] + fn stem_flags_are_budgeted_while_reading() { + let aff = "SET UTF-8\nSFX A Y 1\nSFX A 0 s .\nSFX B Y 1\nSFX B 0 x .\n"; + let many: String = (0..100).map(|i| format!("w{i}/AB\n")).collect(); + let dic = format!("100\n{many}"); + // 200 flags: over a budget of 50 keys (whose flag text, 300 bytes, still fits). + let e = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 50).unwrap_err(); + assert!(e.contains("more than 50 flags"), "{e}"); + let d = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 200).unwrap(); + assert!(d.check("w7s") && d.check("w99x")); + // Flag text past the budget (entries over the per-entry cap hold no keys, but their text + // still counts): refused while reading the `.dic`. + let long = "A".repeat(MAX_FLAGS_PER_ENTRY + 1); + let dic = format!("3\na/{long}\nb/{long}\nc/{long}\n"); + let e = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 2000).unwrap_err(); + assert!(e.contains("more than 12000 bytes"), "{e}"); + assert!(AffixDict::from_hunspell(aff, &dic, &[]).is_ok(), "within the real budget: skipped entries, no error"); + let mut text = FlagText::new(1); + assert!(text.add("ABCDEF").is_ok() && text.add("G").is_err()); + } + #[test] fn affix_headers_allow_comments_but_rule_conditions_stay_literal() { let aff = "SET UTF-8\nPFX X Y 1 # prefix comment\nPFX X 0 ne . # rule comment\nSFX y Y 1 # suffix comment\nSFX y 0 y . # rule comment\n"; diff --git a/crates/proof/src/cs/grammar.rs b/crates/proof/src/cs/grammar.rs index 8a2590e6..e2d29954 100644 --- a/crates/proof/src/cs/grammar.rs +++ b/crates/proof/src/cs/grammar.rs @@ -62,11 +62,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Mezera před interpunkcí".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Dvojitá mezera".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Dvojitá mezera".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs index 2be5e3f0..b1695140 100644 --- a/crates/proof/src/cs/mod.rs +++ b/crates/proof/src/cs/mod.rs @@ -63,7 +63,8 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { return true; }; if w.contains('-') { - return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + // The whole word first, as Hunspell does: a hyphenated entry or user word. + return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } known(d, w) } @@ -142,6 +143,14 @@ mod tests { } } + /// craft-libs#496 review: a hyphenated dictionary entry is a word even when its parts aren't. + #[test] + fn a_hyphenated_entry_is_looked_up_whole() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap(); + assert!(is_correct_with(&d, "tic-tac")); + assert!(!is_correct_with(&d, "tic-tacc")); + } + #[test] fn missing_dictionary_and_ignored_inputs_are_quiet() { let empty = AffixDict::default(); diff --git a/crates/proof/src/cs/suggest.rs b/crates/proof/src/cs/suggest.rs index 05eba74a..1e31496e 100644 --- a/crates/proof/src/cs/suggest.rs +++ b/crates/proof/src/cs/suggest.rs @@ -75,4 +75,13 @@ mod tests { assert!(!suggest("město", 4).contains(&"město".into())); assert!(!suggest("zqzakázane", 8).contains(&"zqzakázané".into())); } + + #[test] + fn a_letter_that_lowercases_to_two_chars_is_still_suggested_for() { + // wordcraft#4683: "İx" lowercases to three chars ("i", U+0307, "x") but has two typed + // letters; the one-letter edits stay in range and dropping the dot still finds "ix". + let d = AffixDict::from_hunspell("", "1\nix\n", &[]).unwrap(); + let suggestions = suggest_in(&d, "İx", 4); + assert!(suggestions.contains(&"Ix".into()), "{suggestions:?}"); + } } diff --git a/crates/proof/src/en_variants.rs b/crates/proof/src/en_variants.rs index 07c16bb1..c8a63fd8 100644 --- a/crates/proof/src/en_variants.rs +++ b/crates/proof/src/en_variants.rs @@ -1,5 +1,5 @@ //! American and British spellings of the same English word (#1281, #1045): which forms each -//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`]) accepts, which it flags, +//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`] and [`ProofLang::EnAu`]) accepts, which it flags, //! and what it suggests instead (`colour` → `color` in American English). //! //! The pairs come from craft-i18n's `language-data/en-us/en-variants.txt`, our own list of base forms with a @@ -77,7 +77,8 @@ fn column(lang: ProofLang) -> Option { match lang { ProofLang::En => Some(0), ProofLang::EnCa => Some(1), - ProofLang::EnGb => Some(2), + // Australian English follows the British policy (`colour`, `centre`, `organise`). + ProofLang::EnGb | ProofLang::EnAu => Some(2), _ => None, } } diff --git a/crates/proof/src/english.rs b/crates/proof/src/english.rs index 0c2c2e13..e821e801 100644 --- a/crates/proof/src/english.rs +++ b/crates/proof/src/english.rs @@ -4,10 +4,11 @@ //! user keeps for WordCraft ([`crate::hunspell::user_dirs`]: the folder set in the options, //! `WORDCRAFT_DICTIONARIES`, WordCraft's own `dictionaries` folder). //! 2. **Installed English pack**: reserved for Language packs (craft-libs#485, CP4.2). -//! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB or en-CA) embedded +//! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB, en-AU or en-CA) embedded //! from craft-i18n at build time ([`crate::scowl`]); a word it knows (British English: also //! the `-ize` spelling of one it knows, [`ise_spelling`]), or the supplement ([`EXTRA`]), is -//! right; never one only another English knows. When it is there, the system locations aren't +//! right; never one only another English knows, except that Australian English also takes +//! en-GB's words (its suggestions still come from en-AU). When it is there, the system locations aren't //! searched. //! 4. **System Hunspell**: a pair for the run's own English in the system locations (none on the //! web). `COMPOUNDRULE` (ordinals) and `OCONV` are read; flag-based compounding is ignored @@ -18,8 +19,9 @@ //! embedded from craft-i18n is the same list; it is used for hyphenation only.) //! //! Hunspell dictionaries are found by [`crate::hunspell`] by their own English's names only -//! (`en_US` for American, `en_GB` for British, `en_CA` for Canadian English), each name in every -//! folder before the next name. Another English's dictionary is never used: it would mark this +//! (`en_US` for American, `en_GB` for British, `en_CA` for Canadian English; `en_AU`, then +//! `en_GB`, for Australian), each name in every folder before the next name. Another English's +//! dictionary is never used: it would mark this //! English's spellings (`colour` in an `en_US` one). A file that can't be read is skipped for the //! next name or folder, and one file is read once for every English that uses it. `COMPOUNDRULE` //! (ordinals) and `OCONV` (suggestions are written as the dictionary says: `don’t`) are read; @@ -90,15 +92,47 @@ fn names(lang: ProofLang) -> &'static [&'static str] { match lang { ProofLang::EnGb => &["en_GB", "en-GB"], ProofLang::EnCa => &["en_CA", "en-CA"], + // Australian English's own, else a British one (a system en_GB on a build without SCOWL): + // [`exact_candidates`] reads every en_AU source before any en_GB one. + ProofLang::EnAu => &["en_AU", "en-AU", "en_GB", "en-GB"], _ => &["en_US", "en-US"], } } +/// The Hunspell pairs of `lang` without SCOWL, from `find` (a folder search for some names, +/// [`crate::hunspell::candidates_exact`]): its own names first, and only then Australian English's +/// British fallback, so every en_AU source (a dictionaries folder, a pack, the system) comes before +/// any en_GB one. +fn exact_candidates(lang: ProofLang, find: impl Fn(&[&str]) -> Vec<(PathBuf, PathBuf)>) -> Vec<(PathBuf, PathBuf)> { + let mut out = find(own_names(lang)); + if lang == ProofLang::EnAu { + for c in find(&names(lang)[own_names(lang).len()..]) { + if !out.iter().any(|(_, d)| *d == c.1) { + out.push(c); + } + } + } + out +} + +/// The names [`names`] gives that outrank the dialect's own embedded SCOWL dictionary: its own pair +/// only, not Australian English's British fallback (a user's `en_GB` is for British text). +fn own_names(lang: ProofLang) -> &'static [&'static str] { + match lang { + ProofLang::EnAu => &["en_AU", "en-AU"], + _ => names(lang), + } +} + +/// How many Englishes have a slot of their own ([`slot`]). +const ENGLISHES: usize = 4; + /// The slot of each English in [`State::slots`]. fn slot(lang: ProofLang) -> usize { match lang { ProofLang::EnGb => 1, ProofLang::EnCa => 2, + ProofLang::EnAu => 3, _ => 0, } } @@ -109,7 +143,7 @@ type Loaded = Option<(Arc, PathBuf)>; /// The English Hunspell dictionaries, under one lock. struct State { /// Per English (see [`slot`]): `None` until first use, then what was found. - slots: [Option; 3], + slots: [Option; ENGLISHES], /// Every `.dic` file read, so Englishes that use the same file share one copy. files: Vec<(PathBuf, Arc)>, /// Counts reloads, so a load that started before one is not kept. @@ -118,7 +152,7 @@ struct State { impl State { const fn new() -> Self { - State { slots: [None, None, None], files: Vec::new(), reloads: 0 } + State { slots: [None, None, None, None], files: Vec::new(), reloads: 0 } } } @@ -181,7 +215,7 @@ fn already_read(state: &RwLock, dic: &Path) -> Option> { /// Forget what `state` holds, so the next check looks again. fn forget(state: &RwLock) { let mut s = state.write().unwrap_or_else(PoisonError::into_inner); - s.slots = [None, None, None]; + s.slots = [None, None, None, None]; s.files.clear(); s.reloads = s.reloads.wrapping_add(1); } @@ -198,9 +232,9 @@ fn key(lang: ProofLang) -> &'static str { /// would rather use), for [`note_error`]. fn load(lang: ProofLang) -> (Loaded, Option<(PathBuf, String)>) { let candidates = if crate::scowl::dict(lang).is_some() { - crate::hunspell::user_candidates(names(lang)) + crate::hunspell::user_candidates(own_names(lang)) } else { - crate::hunspell::candidates_exact(names(lang)) + exact_candidates(lang, crate::hunspell::candidates_exact) }; if candidates.is_empty() { log::info!("English spelling: no Hunspell {} dictionary to read first; using SCOWL or the built-in word list", key(lang)); @@ -240,7 +274,7 @@ fn note_error(lang: ProofLang, error: Option<(PathBuf, String)>) { /// folder changed). pub(crate) fn reload() { forget(&HUNSPELL); - for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::EnAu] { crate::hunspell::clear_load_error(key(lang)); } } @@ -283,10 +317,16 @@ fn pick(found: Loaded, lang: ProofLang) -> Speller { if let Some((d, dic)) = found { return Speller::Hunspell(d, dic); } - match crate::scowl::dict(lang) { - Some(d) => Speller::Scowl(vec![d], crate::scowl::is_british(lang)), - None => Speller::List, + let Some(d) = crate::scowl::dict(lang) else { return Speller::List }; + let mut ds = vec![d]; + // SCOWL's en-AU list keeps one spelling per word (`inquiry`; `enquiry` is only in the en-GB + // list), and Australian English takes both. Suggestions come from the first, en-AU. + if lang == ProofLang::EnAu + && let Some(gb) = crate::scowl::dict(ProofLang::EnGb) + { + ds.push(gb); } + Speller::Scowl(ds, crate::scowl::is_british(lang)) } /// [`speller`] for English with no dialect ([`crate::is_correct`]): a SCOWL word of any English @@ -348,11 +388,12 @@ pub(crate) fn is_correct(word: &str, lang: ProofLang, opts: &Options, policy: bo || (policy && !parts.is_empty() && parts.iter().all(|p| s.knows_lower(p) || crate::en_variants::known_variant(p, lang, |x| s.knows_lower(x)))) } -/// `w` with its `-ize`/`-yze` endings (and their forms: `-ization`, `-izing`…) spelled `-ise`/ -/// `-yse`, when it has any: British English takes both (Word accepts `organize` and `organise` in -/// en-GB), and SCOWL's en-GB dictionary has only `-ise`. A `z` between `i`/`y` and a vowel is -/// swapped; the `-ize` form counts only when American or Canadian English spells the word so -/// ([`ize_elsewhere`]), or `wize` would pass as `wise`. +/// `w` with its `-ize` endings (and their forms: `-ization`, `-izing`…) spelled `-ise`, when it +/// has any: British English takes both (Word accepts `organize` and `organise` in en-GB), and +/// SCOWL's en-GB dictionary has only `-ise`. A `z` between `i` and a vowel is swapped; the `-ize` +/// form counts only when American or Canadian English spells the word so ([`ize_elsewhere`]), or +/// `wize` would pass as `wise`. Never `-yze`: Oxford spelling has `analyse`, `paralyse`, so +/// `reanalyze` or `breathalyzer` is American in British English, as Hunspell's en_GB says. pub(crate) fn ise_spelling(w: &str) -> Option { let b = w.as_bytes(); let mut out = b.to_vec(); @@ -360,7 +401,7 @@ pub(crate) fn ise_spelling(w: &str) -> Option { for i in 1..b.len() { let after = b.get(i + 1).is_some_and(|c| b"aeiAEI".contains(c)); if after - && matches!(b.get(i - 1), Some(b'i' | b'y' | b'I' | b'Y')) + && matches!(b.get(i - 1), Some(b'i' | b'I')) && let Some(c) = out.get_mut(i) { match *c { @@ -512,6 +553,18 @@ fn list_suggest(word: &str, max: usize) -> Vec { mod tests { use super::*; + /// Review follow-up: only `-ize` is swapped for `-ise`, never `-yze` for `-yse` (Oxford + /// spelling has `analyse`), so `reanalyze`, `breathalyzer`, `paralyzingly` stay American. + #[test] + fn ise_spelling_swaps_ize_only() { + assert_eq!(ise_spelling("organize").as_deref(), Some("organise")); + assert_eq!(ise_spelling("Realizations").as_deref(), Some("Realisations")); + assert_eq!(ise_spelling("ORGANIZING").as_deref(), Some("ORGANISING")); + for w in ["reanalyze", "reanalyzed", "breathalyze", "breathalyzer", "paralyzingly", "catalyze", "Analyzer", "ANALYZE"] { + assert_eq!(ise_spelling(w), None, "{w}"); + } + } + #[test] fn built_in_list_is_the_whole_moby_list() { let d = words(); @@ -596,16 +649,16 @@ mod tests { fn lookup_order() { let tiny = Arc::new(AffixDict::from_hunspell("", "1\nzzyzx\n", &[]).unwrap()); let file = PathBuf::from("/folder/en_US.dic"); - for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { assert!(matches!(pick(Some((tiny.clone(), file.clone())), lang), Speller::Hunspell(_, p) if p == file), "{lang:?}"); let fallback = pick(None, lang); if crate::scowl::dict(lang).is_some() { assert!( - matches!(&fallback, Speller::Scowl(ds, _) if ds.len() == 1 && std::ptr::eq(ds[0], crate::scowl::dict(lang).unwrap())), + matches!(&fallback, Speller::Scowl(ds, _) if ds.len() == if lang == ProofLang::EnAu { 2 } else { 1 } && std::ptr::eq(ds[0], crate::scowl::dict(lang).unwrap())), "{lang:?}" ); assert!(fallback.knows("21st") && !fallback.knows("becuase"), "{lang:?}"); - assert_eq!(fallback.knows("colour"), lang != ProofLang::En, "{lang:?}: its own dictionary only"); + assert_eq!(fallback.knows("colour"), lang != ProofLang::En, "{lang:?}: its own dictionary (en-AU: and en-GB's)"); } else { assert!(matches!(fallback, Speller::List), "{lang:?}"); } @@ -625,6 +678,44 @@ mod tests { assert_eq!(found(names(ProofLang::EnGb)), ["en_GB"], "a user's en_US doesn't stand in for SCOWL's en-GB"); assert_eq!(found(names(ProofLang::EnCa)), ["en_CA"]); assert_eq!(found(names(ProofLang::En)), ["en_US"]); + // Australian English: its own, else the British one it read before it had its own. + assert_eq!(found(names(ProofLang::EnAu)), ["en_GB"]); + std::fs::write(dir.join("en_AU.aff"), "SET UTF-8\n").unwrap(); + std::fs::write(dir.join("en_AU.dic"), "1\nzzyzx\n").unwrap(); + assert_eq!(found(names(ProofLang::EnAu)), ["en_AU", "en_GB"]); + // craft-libs#496 review: with SCOWL embedded, a user's en_GB must not outrank its en-AU. + assert_eq!(found(own_names(ProofLang::EnAu)), ["en_AU"]); + std::fs::remove_file(dir.join("en_AU.aff")).unwrap(); + std::fs::remove_file(dir.join("en_AU.dic")).unwrap(); + assert!(found(own_names(ProofLang::EnAu)).is_empty(), "an en_GB alone is not Australian English's own"); + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + assert_eq!(own_names(lang), names(lang)); + } + let _ = std::fs::remove_dir_all(&dir); + } + + /// Without SCOWL, every en_AU source comes before any en_GB one (a dictionaries folder's, an + /// installed pack's, the system's): a user's en_GB doesn't outrank a system en_AU. + #[test] + fn australian_english_reads_every_en_au_before_any_en_gb() { + let dir = std::env::temp_dir().join(format!("wc-proof-en-au-order-{}", std::process::id())); + for sub in ["user", "system"] { + std::fs::create_dir_all(dir.join(sub)).unwrap(); + } + let write = |sub: &str, name: &str| { + std::fs::write(dir.join(sub).join(format!("{name}.aff")), "SET UTF-8\n").unwrap(); + std::fs::write(dir.join(sub).join(format!("{name}.dic")), "1\nzzyzx\n").unwrap(); + }; + write("user", "en_GB"); + write("system", "en_AU"); + let (user, system) = (vec![dir.join("user")], vec![dir.join("system")]); + let find = |n: &[&str]| crate::hunspell::ordered(n, &user, &[], &system, &[]); + let stems = |v: Vec<(PathBuf, PathBuf)>| -> Vec { + v.iter().filter_map(|(_, d)| d.file_stem().map(|s| s.to_string_lossy().into_owned())).collect() + }; + assert_eq!(stems(exact_candidates(ProofLang::EnAu, find)), ["en_AU", "en_GB"]); + assert_eq!(stems(exact_candidates(ProofLang::EnGb, find)), ["en_GB"]); + assert_eq!(stems(exact_candidates(ProofLang::En, find)), Vec::::new()); let _ = std::fs::remove_dir_all(&dir); } diff --git a/crates/proof/src/es/grammar.rs b/crates/proof/src/es/grammar.rs index 42eede50..00b20789 100644 --- a/crates/proof/src/es/grammar.rs +++ b/crates/proof/src/es/grammar.rs @@ -75,11 +75,8 @@ fn spaces(text: &str, lang: ProofLang, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Espacio antes de un signo de puntuación".into(), vec![p.to_string()], lang)); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Espacio doble".into(), vec![" ".into()], lang)); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Espacio doble".into(), vec![" ".into()], lang)); } } @@ -194,6 +191,16 @@ mod tests { .collect() } + /// craft-libs#496 review: a run of spaces is one issue over the whole run. + #[test] + fn a_run_of_spaces_is_reported_whole() { + for (text, run) in [("casa roja", " "), ("casa roja", " "), ("casa roja", " "), ("casa roja", " ")] { + let m: Vec<_> = messages(text).into_iter().filter(|(_, msg)| msg == "Espacio doble").collect(); + assert_eq!(m, [(run.to_string(), "Espacio doble".to_string())], "{text}"); + } + assert!(messages("casa roja").is_empty()); + } + #[test] fn repeated_words_and_spaces_have_spanish_messages() { let text = "Es es la casa , dijo."; diff --git a/crates/proof/src/hr/grammar.rs b/crates/proof/src/hr/grammar.rs index 2605ae88..f4997e9d 100644 --- a/crates/proof/src/hr/grammar.rs +++ b/crates/proof/src/hr/grammar.rs @@ -105,11 +105,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Razmak ispred interpunkcijskog znaka".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Dvostruki razmak".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Dvostruki razmak".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/hr/mod.rs b/crates/proof/src/hr/mod.rs index 53edf067..e0bda809 100644 --- a/crates/proof/src/hr/mod.rs +++ b/crates/proof/src/hr/mod.rs @@ -65,7 +65,8 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { return true; }; if w.contains('-') { - return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + // The whole word first, as Hunspell does: a hyphenated entry or user word. + return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } known(d, w) } @@ -105,6 +106,14 @@ mod tests { } } + /// craft-libs#496 review: a hyphenated dictionary entry is a word even when its parts aren't. + #[test] + fn a_hyphenated_entry_is_looked_up_whole() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap(); + assert!(is_correct_with(&d, "tic-tac")); + assert!(!is_correct_with(&d, "tic-tacc")); + } + #[test] fn missing_dictionary_is_silent() { assert!(is_correct_with(&AffixDict::default(), "nepostojećariječ")); diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 274af9eb..966ded72 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -196,19 +196,22 @@ pub(crate) fn ordered( let first = names.first().map(|n| norm(n)); let (region, rest): (Vec<&str>, Vec<&str>) = names.iter().copied().partition(|n| Some(norm(n)) == first); let pack: &[&str] = &[crate::packs::SPELLING]; + // (folder, names, a Language pack's: never read through a symbolic link, as the pack scan + // checks; a system folder's dictionaries often are links, `en_GB.dic` → `en_GB-ise.dic`). let tiers = dirs .iter() - .map(|d| (d, names)) - .chain(region_packs.iter().map(|d| (d, pack))) - .chain(system.iter().map(|d| (d, ®ion[..]))) - .chain(other_packs.iter().map(|d| (d, pack))) - .chain(system.iter().map(|d| (d, &rest[..]))); + .map(|d| (d, names, false)) + .chain(region_packs.iter().map(|d| (d, pack, true))) + .chain(system.iter().map(|d| (d, ®ion[..], false))) + .chain(other_packs.iter().map(|d| (d, pack, true))) + .chain(system.iter().map(|d| (d, &rest[..], false))); + let file = |p: &Path, pack: bool| if pack { std::fs::symlink_metadata(p).is_ok_and(|m| m.is_file()) } else { p.is_file() }; let mut out: Vec<(PathBuf, PathBuf)> = Vec::new(); - for (dir, names) in tiers { + for (dir, names, in_pack) in tiers { for n in names { let aff = dir.join(format!("{n}.aff")); let dic = dir.join(format!("{n}.dic")); - if aff.is_file() && dic.is_file() && !out.iter().any(|(_, d)| *d == dic) { + if file(&aff, in_pack) && file(&dic, in_pack) && !out.iter().any(|(_, d)| *d == dic) { out.push((aff, dic)); } } @@ -696,6 +699,34 @@ mod tests { } } + /// Review follow-up (craft-libs#485): a pack's `spelling.aff`/`.dic` that is a symbolic link + /// (swapped in after the pack scan) is never a candidate; one in a system folder still is. + #[cfg(unix)] + #[test] + fn pack_files_are_never_read_through_a_link() { + let root = std::env::temp_dir().join(format!("wc-proof-pack-links-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let (pack, system, elsewhere) = (root.join("packs").join("nl"), root.join("system"), root.join("elsewhere")); + for d in [&pack, &system, &elsewhere] { + std::fs::create_dir_all(d).unwrap(); + } + std::fs::write(elsewhere.join("x.aff"), "SET UTF-8\n").unwrap(); + std::fs::write(elsewhere.join("x.dic"), "1\nx\n").unwrap(); + std::fs::write(pack.join("spelling.aff"), "SET UTF-8\n").unwrap(); + std::os::unix::fs::symlink(elsewhere.join("x.dic"), pack.join("spelling.dic")).unwrap(); + std::os::unix::fs::symlink(elsewhere.join("x.aff"), system.join("nl.aff")).unwrap(); + std::os::unix::fs::symlink(elsewhere.join("x.dic"), system.join("nl.dic")).unwrap(); + let got: Vec = + ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect(); + assert_eq!(got, [system.join("nl.dic")], "the system's link is followed, the pack's never"); + std::fs::remove_file(pack.join("spelling.dic")).unwrap(); + std::fs::write(pack.join("spelling.dic"), "1\nx\n").unwrap(); + let got: Vec = + ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect(); + assert_eq!(got, [pack.join("spelling.dic"), system.join("nl.dic")]); + let _ = std::fs::remove_dir_all(&root); + } + /// WordCraft's own dictionaries folder under the user's application data is searched on every /// desktop system, after the setting and before the shared system locations (#1063). #[test] diff --git a/crates/proof/src/id/grammar.rs b/crates/proof/src/id/grammar.rs index 72ca10c9..6c13b7f1 100644 --- a/crates/proof/src/id/grammar.rs +++ b/crates/proof/src/id/grammar.rs @@ -56,11 +56,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Spasi sebelum tanda baca".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Spasi ganda".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Spasi ganda".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/id/mod.rs b/crates/proof/src/id/mod.rs index c78e25d4..85dc825a 100644 --- a/crates/proof/src/id/mod.rs +++ b/crates/proof/src/id/mod.rs @@ -76,7 +76,7 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { || w.split('-').all(|p| { p.chars().count() <= 1 || CLITICS.iter().any(|c| p.eq_ignore_ascii_case(c)) - || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= 6) + || (opts.ignore_uppercase && crate::spelling::capitals_part(p, 6)) || known(d, p) }); } @@ -141,6 +141,19 @@ mod tests { } } + /// Review follow-up (craft-libs#485's cased-letter rule): a part in a caseless script is no + /// abbreviation, so `buku-日本` is checked part by part; a part in capitals or without letters + /// still is one (`KTP-nya`, `COVID-19` with numbers checked). + #[test] + fn caseless_parts_are_not_abbreviations() { + test_dict::install(); + let d = dictionary(); + let defaults = crate::Options::default(); + assert!(!check(&d, "buku-日本", &defaults) && !check(&d, "buku-ไทย", &defaults)); + let numbers = crate::Options { ignore_numbers: false, ..defaults }; + assert!(check(&d, "KTP-nya", &defaults) && check(&d, "buku-19", &numbers)); + } + #[test] fn forbidden_words_are_rejected_in_every_case() { test_dict::install(); diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs index 57c62e28..9019218c 100644 --- a/crates/proof/src/it/grammar.rs +++ b/crates/proof/src/it/grammar.rs @@ -150,10 +150,12 @@ fn abbreviation_or_number(prev: &str) -> bool { || ABBREVIATIONS.contains(&prev.to_lowercase().as_str()) } +const CLOSING: [char; 8] = ['"', '”', '’', '\'', ')', '»', ']', '\u{1}']; + /// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after /// an abbreviation), then space, with nothing but quotes and brackets around? fn ends_sentence(gap: &str, prev: &str) -> bool { - let t = gap.trim_start_matches(['"', '”', '’', '\'', ')', '»', ']', '\u{1}']); + let t = gap.trim_start_matches(CLOSING); let Some(end) = t.chars().next() else { return false }; if !matches!(end, '.' | '!' | '?') || t.starts_with("..") { return false; @@ -161,8 +163,16 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { if end == '.' && abbreviation_or_number(prev) { return false; } - let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(['"', '”', '’', '\'', ')', '»', ']']); - rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—')) + let after_marks = t.trim_start_matches(['.', '!', '?']); + let rest = after_marks.trim_start_matches(CLOSING); + // (Not the object placeholder `\u{1}`: `Chi era?\u{1} era Marco.` still starts a sentence.) + // `«Vai?» disse lui.` and `—Vai? —disse lui.`: the narration after a quoted question or + // exclamation (not after a period: `Finì. —domani` starts a sentence), or after a dialogue dash, goes on in lower case. + if end != '.' && (rest.contains('—') || after_marks.trim_start_matches('\u{1}').starts_with(CLOSING)) { + return false; + } + rest.starts_with(char::is_whitespace) + && rest.trim_start().chars().all(|c| c.is_whitespace() || matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—')) } fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { @@ -203,11 +213,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Spazio prima della punteggiatura".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Spazio doppio".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Spazio doppio".into(), vec![" ".into()])); } } @@ -403,6 +410,23 @@ mod tests { } } + /// craft-libs#496 review: narration after a quoted or dashed question or exclamation goes on in + /// lower case. + #[test] + fn narration_after_a_quoted_question_is_not_a_new_sentence() { + for t in ["«Vai?» disse lui.", "«Vieni!» gridò Anna.", "\"Vai?\" disse lui.", "—Vai? —disse lui.", "— Vai! — disse lui."] { + assert!(fixes(t).is_empty(), "{t}: {:?}", fixes(t)); + } + assert!(fixes("Vado a Roma? sì.").contains(&pair("sì", "Sì")), "a bare question still ends the sentence"); + // After a period the dash opens the next line of speech: a sentence starts. + assert!(fixes("Finì. —domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. —domani partiamo.")); + assert!(fixes("Finì. — domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. — domani partiamo.")); + // An inline object (`\u{1}`) before a real closing quote is skipped. + assert!(fixes("«Vai?\u{1}» disse lui.").is_empty(), "{:?}", fixes("«Vai?\u{1}» disse lui.")); + // An inline object (`\u{1}`) is not a closing quote. + assert!(fixes("Chi era?\u{1} era Marco.").contains(&pair("era", "Era")), "{:?}", fixes("Chi era?\u{1} era Marco.")); + } + #[test] fn po_takes_an_apostrophe() { assert_eq!(fixes("Dammi un pò di pane"), [pair("pò", "po'")]); diff --git a/crates/proof/src/it/mod.rs b/crates/proof/src/it/mod.rs index a742eaa2..44550501 100644 --- a/crates/proof/src/it/mod.rs +++ b/crates/proof/src/it/mod.rs @@ -97,8 +97,14 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { } // The whole word first, as Hunspell does: a hyphenated entry or user word. if w.contains('-') { - return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known_elided(d, p)); } + known_elided(d, w) +} + +/// [`known`] for a word that may start with an elided proclitic (`l'amico`, `l'ex`): the word after +/// the apostrophe is what gets checked. +fn known_elided(d: &AffixDict, w: &str) -> bool { if let Some((head, tail)) = w.split_once(['\'', '’']) && PROCLITICS.contains(&head.to_lowercase().as_str()) && !tail.is_empty() @@ -148,6 +154,16 @@ mod tests { assert!(!is_correct_with(&d, "tic-tacc")); } + /// craft-libs#496 review: an elision inside a hyphenated word (`l'ex-amico`) is read per part. + #[test] + fn an_elision_in_a_hyphenated_part_is_read() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "2\nex\namico\n", &[]).unwrap(); + for w in ["l'ex-amico", "dell’ex-amico", "ex-amico"] { + assert!(is_correct_with(&d, w), "{w}"); + } + assert!(!is_correct_with(&d, "l'exx-amico")); + } + #[test] fn without_a_dictionary_nothing_is_flagged() { let none = AffixDict::default(); diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs index 993e4030..463406a9 100644 --- a/crates/proof/src/lang.rs +++ b/crates/proof/src/lang.rs @@ -5,7 +5,7 @@ /// A language WordCraft can proof, or [`ProofLang::Other`] for one it can't. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum ProofLang { - /// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`] or + /// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`], [`ProofLang::EnAu`] or /// [`ProofLang::EnCa`]): SCOWL's en-US dictionary ([`crate::scowl`]), the English grammar rules and the /// English hyphenation dictionary and patterns. British spellings (`colour`, `centre`, /// `organise`) are flagged, with the American form suggested ([`crate::en_variants`]). Also @@ -13,10 +13,14 @@ pub enum ProofLang { /// Word's). #[default] En, - /// English with British spelling (`en-GB`, `en-AU`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…): + /// English with British spelling (`en-GB`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…): /// [`ProofLang::En`]'s tools, but American spellings (`color`, `center`, `analyze`) are /// flagged and `-ize` and `-ise` are both right, as in Oxford spelling. EnGb, + /// Australian English (`en-AU`): [`ProofLang::EnGb`]'s spelling policy, against SCOWL's own + /// en-AU dictionary, which knows Australian words (`billabong`, `brumby`, `larrikin`) and + /// spells some words the Australian way (`program`, `inquiry`). + EnAu, /// Canadian English (`en-CA`): British `-our`, `-re` and `-ence` (`colour`, `centre`, /// `defence`) with American `-ize` (`organize`), `tire` and `program`. EnCa, @@ -86,9 +90,10 @@ pub enum ProofLang { impl ProofLang { /// Every language with a spelling dictionary, in display order. - pub const PROOFED: [ProofLang; 15] = [ + pub const PROOFED: [ProofLang; 16] = [ ProofLang::En, ProofLang::EnGb, + ProofLang::EnAu, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, @@ -155,12 +160,13 @@ impl ProofLang { } } - /// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], regions - /// that spell the British way are [`ProofLang::EnGb`], and the United States, any other + /// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], Australia + /// [`ProofLang::EnAu`], other regions that spell the British way are [`ProofLang::EnGb`], and the United States, any other /// region and no region are [`ProofLang::En`]. fn english(tag: &str) -> ProofLang { match region(tag) { Some(r) if r.eq_ignore_ascii_case("CA") => ProofLang::EnCa, + Some(r) if r.eq_ignore_ascii_case("AU") => ProofLang::EnAu, Some(r) if BRITISH_SPELLING.iter().any(|b| b.eq_ignore_ascii_case(r)) => ProofLang::EnGb, _ => ProofLang::En, } @@ -189,9 +195,10 @@ impl ProofLang { } } - /// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnCa`])? + /// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnAu`], + /// [`ProofLang::EnCa`])? pub fn is_english(self) -> bool { - matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) + matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa) } /// Is this one of the Spanishes ([`ProofLang::Es`], [`ProofLang::EsMx`], [`ProofLang::Es419`], @@ -208,7 +215,7 @@ impl ProofLang { pub fn code(self) -> &'static str { match self { ProofLang::Pack(_) => "", - ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => "en", + ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => "en", ProofLang::Pl => "pl", ProofLang::Bg => "bg", ProofLang::Pt => "pt", @@ -371,12 +378,12 @@ pub(crate) fn region(tag: &str) -> Option<&str> { const LATIN_AMERICA: [&str; 22] = ["419", "005", "013", "029", "AR", "BO", "CL", "CO", "CR", "CU", "DO", "EC", "GT", "HN", "NI", "PA", "PE", "PR", "PY", "SV", "US", "VE"]; -/// Regions whose English uses British spelling ([`ProofLang::EnGb`]): the United Kingdom (and -/// the non-standard `UK`), Ireland, Australia, New Zealand, South Africa, India and other +/// Regions whose English uses British spelling ([`ProofLang::EnGb`]; Australia has its own, +/// [`ProofLang::EnAu`]): the United Kingdom (and the non-standard `UK`), Ireland, New Zealand, South Africa, India and other /// Commonwealth countries, the Crown dependencies and Europe (`150`). -const BRITISH_SPELLING: [&str; 26] = [ - "GB", "UK", "IE", "AU", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH", - "PK", "150", +const BRITISH_SPELLING: [&str; 25] = [ + "GB", "UK", "IE", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH", "PK", + "150", ]; /// A BCP 47 tag in canonical case (`pl-PL`, `en-GB`, `zh-Hant-TW`, `x-none`), or `None` when @@ -469,26 +476,12 @@ mod tests { fn tags_resolve_english_regions() { assert_tags(&["en", "en-US", "EN_us", "en-Latn-US", "en-PH", "en-001", "eng", "en_US.UTF-8", "en-x-gb"], ProofLang::En); assert_tags( - &[ - "en-GB", - "EN_gb", - "en_GB.UTF-8", - "en-AU", - "en_AU", - "en-NZ", - "en-IE", - "en-ZA", - "en-IN", - "en-Latn-GB", - "en-UK", - "en-150", - "eng-GB", - "en_GB@euro", - ], + &["en-GB", "EN_gb", "en_GB.UTF-8", "en-NZ", "en-IE", "en-ZA", "en-IN", "en-Latn-GB", "en-UK", "en-150", "eng-GB", "en_GB@euro"], ProofLang::EnGb, ); assert_tags(&["en-CA", "en_CA", "EN-ca", "en_CA.UTF-8", "en-Latn-CA"], ProofLang::EnCa); - for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + assert_tags(&["en-AU", "en_AU", "EN-au", "en_AU.UTF-8", "en-Latn-AU"], ProofLang::EnAu); + for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { assert!(language.is_english() && language.is_proofed() && ProofLang::PROOFED.contains(&language), "{language:?}"); assert_eq!(language.code(), "en"); } diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 5b58768c..2e804097 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -152,6 +152,28 @@ fn dictionary_key(w: &str) -> String { composed(&w.to_lowercase()).into_owned() } +/// Each run of two or more spaces in `text`, once, over the whole run (a fix replaces it with one +/// space). A run of three is one issue over three, not a pair and a leftover. +pub(crate) fn space_runs(text: &str) -> Vec> { + let bytes = text.as_bytes(); + let mut runs = Vec::new(); + let mut i = 0; + while i < bytes.len() { + if bytes.get(i) != Some(&b' ') { + i += 1; + continue; + } + let start = i; + while bytes.get(i) == Some(&b' ') { + i += 1; + } + if i - start >= 2 { + runs.push(start..i); + } + } + runs +} + /// Add a word to the user dictionary ("Add to Dictionary"). It applies to every language. pub fn add_word(w: &str) { if user_words().write().unwrap_or_else(std::sync::PoisonError::into_inner).insert(dictionary_key(w)) { @@ -340,7 +362,7 @@ pub fn installed_dictionaries() -> Vec<(ProofLang, Option)> /// checked yet reports `None` (#2830). For an English, its embedded SCOWL dictionary or the /// English Hunspell dictionary that failed; English then uses the next source ([`english_source`]). pub fn dictionary_load_error(lang: ProofLang) -> Option { - if matches!(lang, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) { + if lang.is_english() { return english::load_error(lang); } if let Some(l) = lang.spelling_locale() { @@ -669,7 +691,7 @@ pub(crate) fn accepted_by_options(word: &str, opts: &Options) -> bool { || !w.chars().any(char::is_alphabetic) || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) // Only a word with a cased letter is in capitals: a Hebrew or Thai word has none. - || (opts.ignore_uppercase && w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase())) + || (opts.ignore_uppercase && spelling::in_capitals(w)) // The whole word: the languages split a hyphenated word into its parts, and an added // `Unigranrio-RJ` is none of them (#2082). || in_user_dictionary(&w.to_lowercase()) @@ -953,7 +975,7 @@ pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec { fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec { match lang { ProofLang::En => grammar_en(text, capital_first), - ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(), + ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(), // `pt-pt` shares Portuguese's rules, its issues in its own language. _ => language(lang.built_in()) .map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect()), @@ -1098,13 +1120,10 @@ fn grammar_en(text: &str, capital_first: bool) -> Vec { lang: ProofLang::En, }); } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } + for run in space_runs(text) { v.push(Issue { - start: i, - end: i + 2, + start: run.start, + end: run.end, kind: IssueKind::Grammar, message: "Extra space".into(), suggestions: vec![" ".into()], @@ -1197,7 +1216,7 @@ mod tests { } // Every proofed language other than English has a Hunspell dictionary to report. for l in ProofLang::PROOFED { - let english = matches!(l, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa); + let english = l.is_english(); assert_eq!(dictionary_needed(l, false).is_some(), !english, "{l:?}"); } for l in [ProofLang::En, ProofLang::Other] { @@ -1709,6 +1728,31 @@ mod tests { let _ = std::fs::remove_dir_all(&root); } + /// Review follow-up (craft-libs#485): a word in another script than a pack language's + /// dictionary (a Russian name in a Dutch run, a Dutch word in a Greek one) isn't marked; a + /// misspelling in the dictionary's own script still is. + #[test] + fn pack_languages_leave_other_scripts_alone() { + crate::hold_dictionary_tests(); + set_dictionaries_folder(Some(shared_test_folder())); + let root = std::env::temp_dir().join(format!("wc-proof-other-script-packs-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + write_pack(&root, "nl", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", "2\nhuis\nkat\n".as_bytes())]); + write_pack(&root, "el", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", "1\nσπίτι\n".as_bytes())]); + packs::set_packs_dir(Some(root.clone())); + let (nl, el) = (ProofLang::from_tag(Some("nl-NL")), ProofLang::from_tag(Some("el-GR"))); + assert!(is_correct_in("huis", nl) && !is_correct_in("huiss", nl)); + assert!(!is_correct_in("zzzz", nl), "a misspelling in the dictionary's script, whatever its letters"); + assert!(is_correct_in("Пушкин", nl) && is_correct_in("σπίτι", nl)); + assert!(is_correct_in("σπίτι", el) && !is_correct_in("σπίτιι", el) && is_correct_in("huis", el)); + let text = "Пушкин huiss"; + let issues = check_text(text, &[(0..text.len(), nl)]); + assert_eq!(issues.iter().map(|i| &text[i.start..i.end]).collect::>(), ["huiss"], "{issues:?}"); + packs::set_packs_dir(None); + set_dictionaries_folder(Some(shared_test_folder())); + let _ = std::fs::remove_dir_all(&root); + } + /// The `.dic` file `lang`'s spelling is read from with the installed packs and `system` as the /// system's only dictionary folder. fn chosen_dictionary(lang: ProofLang, system: &std::path::Path) -> Option { @@ -2465,6 +2509,49 @@ mod tests { assert_eq!(suggest_in("programmes", 5, us).first().map(String::as_str), Some("programs")); } + /// Review follow-up: British English takes `-ize` (Oxford spelling) but never `-yze`, also in + /// forms en-variants.txt doesn't list. Hunspell 1.7.5 with SCOWL's en_GB rejects these too. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn british_english_rejects_unlisted_yze_forms() { + crate::hold_dictionary_tests(); + let gb = ProofLang::EnGb; + for w in ["reanalyze", "reanalyzed", "breathalyze", "breathalyzer", "paralyzingly", "catalyze", "analyzer"] { + assert!(!is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_in(w, ProofLang::En), "en-US: {w}"); + } + for w in ["reanalyse", "breathalyser", "paralysingly", "catalyse", "organize", "realization"] { + assert!(is_correct_in(w, gb), "en-GB: {w}"); + } + } + + /// Review follow-up: Australian English reads SCOWL's own en-AU dictionary (craft-i18n ships + /// it), not en-GB's: its Australian words are right and spelled as Hunspell's en_AU spells + /// them; British English doesn't take them. The British policy still applies: American + /// spellings are wrong, `-ise` and `-ize` both right. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn australian_english_reads_its_own_scowl() { + crate::hold_dictionary_tests(); + let (au, gb) = (ProofLang::EnAu, ProofLang::EnGb); + assert!(scowl::is_british(au)); + assert_eq!(english_source(au), Some(EnglishSource::Embedded)); + for w in ["billabong", "brumby", "larrikinism", "bunyip", "corroboree", "Bundaberg", "Rockhampton"] { + assert!(is_correct_in(w, au), "en-AU: {w}"); + assert!(!is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_for(w, Some("en-AU")), "en-AU tag: {w}"); + } + for w in ["colour", "centre", "organise", "organize", "analyse", "travelled", "defence"] { + assert!(is_correct_in(w, au), "en-AU: {w}"); + } + for w in ["color", "center", "analyze", "reanalyze", "defense"] { + assert!(!is_correct_in(w, au), "en-AU: {w}"); + } + let text = "The larrikin saw a brumby by the billabong and analysed the colour of the sky."; + assert!(check_spelling_in(text, au).is_empty(), "{:?}", check_spelling_in(text, au)); + assert_eq!(check_spelling_in(text, gb).len(), 3, "larrikin, brumby, billabong: {:?}", check_spelling_in(text, gb)); + } + /// More British spellings that must stay right in en-GB and en-AU (#1281): -re, single l, ae, /// with their inflections. #[test] @@ -2472,7 +2559,7 @@ mod tests { // English results depend on the dictionaries other tests install. crate::hold_dictionary_tests(); let gb = ProofLang::EnGb; - assert_eq!(ProofLang::from_tag(Some("en-AU")), gb); + assert_eq!(ProofLang::from_tag(Some("en-AU")), ProofLang::EnAu); for w in [ "goitre", "goitres", @@ -2497,11 +2584,66 @@ mod tests { "encyclopaedias", ] { assert!(is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_in(w, ProofLang::EnAu), "en-AU: {w}"); } let text = "The wilful knight took the sceptre to fulfil his enrolment, paid the instalment and read the encyclopaedia."; assert!(check_spelling_in(text, gb).is_empty(), "{:?}", check_spelling_in(text, gb)); } + /// Review follow-up: SCOWL's en-AU list keeps one spelling per word, so Australian English also + /// takes a word the en-GB list has (`enquiry`, `guerrilla`, `liquorice`); American-only + /// spellings, wrong in both, stay wrong. Suggestions still come from the en-AU list. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn australian_english_also_takes_british_words() { + crate::hold_dictionary_tests(); + let (au, gb) = (ProofLang::EnAu, ProofLang::EnGb); + for w in ["enquiry", "enquire", "guerrilla", "guerrillas", "euthanise", "liveable", "liquorice", "veranda", "burka", "Anthea"] { + assert!(is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_in(w, au), "en-AU: {w}"); + } + for w in ["analyze", "catalyze", "color", "center", "defense", "zzyzxq"] { + assert!(!is_correct_in(w, gb), "en-GB: {w}"); + assert!(!is_correct_in(w, au), "en-AU: {w}"); + } + assert!(is_correct_in("billabong", au) && !is_correct_in("billabong", gb), "en-AU still has its own"); + assert!(suggest_in("billabon", 5, au).iter().any(|s| s == "billabong"), "{:?}", suggest_in("billabon", 5, au)); + } + + /// Review follow-up: a run of spaces is one issue over the whole run, in every language with a + /// double-space rule (a run of three used to leave two after the fix, four three). + #[test] + fn a_run_of_spaces_is_one_issue_in_every_language() { + assert_eq!(space_runs("a b c d "), [3..5, 6..9, 10..14]); + assert!(space_runs("a b c").is_empty() && space_runs("").is_empty() && space_runs(" ").is_empty()); + let langs = [ + ProofLang::En, + ProofLang::EnGb, + ProofLang::Es, + ProofLang::It, + ProofLang::Pt, + ProofLang::Cs, + ProofLang::Hr, + ProofLang::Id, + ProofLang::Pl, + ProofLang::Ru, + ProofLang::Tr, + ProofLang::EnAu, + ProofLang::EnCa, + ] + .into_iter() + .chain(PackLocale::new("pt-PT").map(ProofLang::Pack)); + for lang in langs { + for n in 2..=5 { + let text = format!("Aaa{}bbb", " ".repeat(n)); + let hits: Vec<_> = + check_grammar_in(&text, lang).into_iter().filter(|i| i.start <= 3 && i.end > 3 && i.suggestions == [" "]).collect(); + assert_eq!(hits.len(), 1, "{lang:?} {n}: {hits:?}"); + assert_eq!((hits[0].start, hits[0].end), (3, 3 + n), "{lang:?} {n}"); + } + } + } + /// English spelling reads SCOWL 2026 through the affix engine: one dictionary per English, /// its notice embedded, and its ordinals (`COMPOUNDRULE`) understood. #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] @@ -2513,7 +2655,8 @@ mod tests { assert_eq!(english_source(l), Some(EnglishSource::Embedded)); assert_eq!(dictionary_load_error(l), None); } - assert!(std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::from_tag(Some("en-AU"))).unwrap()), "en-AU reads en-GB"); + assert!(!std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::EnAu).unwrap()), "en-AU reads its own"); + assert!(std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::from_tag(Some("en-NZ"))).unwrap()), "en-NZ reads en-GB"); let notice = scowl::notice().unwrap(); for credit in ["Kevin Atkinson", "Geoff Kuenning", "clause 4 removed", "Benjamin Titze"] { assert!(notice.contains(credit), "{credit}"); diff --git a/crates/proof/src/no_data.rs b/crates/proof/src/no_data.rs index 2f429985..cdd9025d 100644 --- a/crates/proof/src/no_data.rs +++ b/crates/proof/src/no_data.rs @@ -7,9 +7,10 @@ use crate::hyphen::{self, Limits, Source}; use crate::{EnglishSource, Options, ProofLang, dict, en_variants, patterns}; -const LANGS: [ProofLang; 18] = [ +const LANGS: [ProofLang; 19] = [ ProofLang::En, ProofLang::EnGb, + ProofLang::EnAu, ProofLang::EnCa, ProofLang::Pl, ProofLang::Bg, @@ -45,9 +46,14 @@ const SAMPLES: [&str; 10] = [ #[test] fn nothing_is_embedded() { assert!(crate::data::EN_US_DIC.is_empty()); - assert!(crate::data::EN_US_SPELLING.is_empty() && crate::data::EN_GB_SPELLING.is_empty() && crate::data::EN_CA_SPELLING.is_empty()); + assert!( + crate::data::EN_US_SPELLING.is_empty() + && crate::data::EN_GB_SPELLING.is_empty() + && crate::data::EN_CA_SPELLING.is_empty() + && crate::data::EN_AU_SPELLING.is_empty() + ); assert!(crate::scowl::notice().is_none()); - for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { assert!(crate::scowl::dict(l).is_none() && crate::scowl::load_error(l).is_none()); } assert!(dict::Dictionary::en_us().is_empty()); @@ -63,7 +69,7 @@ fn nothing_is_embedded() { #[test] fn english_spelling_still_marks_misspellings() { crate::hold_dictionary_tests(); - for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { let source = crate::english_source(l); assert!(matches!(source, Some(EnglishSource::Hunspell(_) | EnglishSource::CoreList)), "{l:?}"); // Any English dictionary marks junk and knows `the`; the word-by-word results below are diff --git a/crates/proof/src/pack_lang.rs b/crates/proof/src/pack_lang.rs index 6c116653..7b4d1931 100644 --- a/crates/proof/src/pack_lang.rs +++ b/crates/proof/src/pack_lang.rs @@ -6,6 +6,10 @@ //! files); the one used least recently goes first, but never one used during the current //! [`in_one_check`] call, so a paragraph in many languages loads each dictionary once. Locales //! with no dictionary are remembered apart and take no room. +//! +//! A word none of whose letters is in a script the dictionary's words are written in (a Cyrillic +//! word in a Dutch run) isn't this language's: it is never marked, as Word leaves a word in another +//! script alone. A word in the dictionary's script is always checked, whatever its letters. use std::cell::Cell; use std::path::PathBuf; @@ -20,7 +24,7 @@ use crate::lang::PackLocale; pub(crate) const MAX_LOADED_BYTES: u64 = 256 << 20; /// Estimated memory per byte of `.aff` and `.dic` file. const BYTES_PER_FILE_BYTE: u64 = 6; -/// Most locales remembered as having no dictionary (the list is cleared when full). +/// Most locales remembered as having no dictionary (the oldest is forgotten first when full). const MAX_MISSING: usize = 256; /// The budget in force (tests lower it). @@ -36,6 +40,8 @@ thread_local! { struct Loaded { locale: PackLocale, dict: Arc, + /// The scripts its words are written in ([`scripts`]), sorted. + scripts: Arc<[u32]>, /// The `.dic` it came from. file: Option, /// Estimated memory. @@ -66,15 +72,88 @@ pub(crate) fn reload() { /// over the budget. pub(crate) fn in_one_check(f: impl FnOnce() -> R) -> R { let outer = CHECK_FROM.with(Cell::get); - if outer.is_none() { + let _end = outer.is_none().then(|| { let now = CACHE.read().unwrap_or_else(PoisonError::into_inner).clock; CHECK_FROM.with(|c| c.set(Some(now.wrapping_add(1)))); - } - let r = f(); - if outer.is_none() { + EndCheck + }); + f() +} + +/// Ends the outermost [`in_one_check`] when dropped: also when `f` panics (`Session::run` catches +/// it), or every dictionary would stay pinned on this thread and the budget never hold again. +struct EndCheck; + +impl Drop for EndCheck { + fn drop(&mut self) { CHECK_FROM.with(|c| c.set(None)); } - r +} + +/// Remember that `locale` has no dictionary; when [`MAX_MISSING`] are remembered, the oldest is +/// forgotten (clearing them all would look every other one up again). +fn remember_missing(missing: &mut Vec, locale: PackLocale) { + if missing.len() >= MAX_MISSING { + missing.remove(0); + } + missing.push(locale); +} + +/// The script of the letter `c`, as a number: the main alphabetic scripts by their Unicode blocks +/// (Latin with its extensions, Greek, Cyrillic, Armenian, Hebrew, Arabic, Georgian, Hangul, kana, +/// Han, Ethiopic); any other letter by its 128-character block. Only told apart, never named. +fn script_of(c: char) -> u32 { + const SCRIPTS: &[(u32, u32, u32)] = &[ + (0x0041, 0x024F, 1), + (0x0250, 0x02AF, 1), + (0x1E00, 0x1EFF, 1), + (0x2C60, 0x2C7F, 1), + (0xA720, 0xA7FF, 1), + (0xAB30, 0xAB6F, 1), + (0xFF21, 0xFF5A, 1), + (0x0370, 0x03FF, 2), + (0x1F00, 0x1FFF, 2), + (0x0400, 0x052F, 3), + (0x1C80, 0x1C8F, 3), + (0x2DE0, 0x2DFF, 3), + (0xA640, 0xA69F, 3), + (0x0530, 0x058F, 4), + (0xFB13, 0xFB17, 4), + (0x0590, 0x05FF, 5), + (0xFB1D, 0xFB4F, 5), + (0x0600, 0x06FF, 6), + (0x0750, 0x077F, 6), + (0x08A0, 0x08FF, 6), + (0xFB50, 0xFDFF, 6), + (0xFE70, 0xFEFF, 6), + (0x10A0, 0x10FF, 7), + (0x1C90, 0x1CBF, 7), + (0x2D00, 0x2D2F, 7), + (0x1100, 0x11FF, 8), + (0x3130, 0x318F, 8), + (0xAC00, 0xD7AF, 8), + (0x3040, 0x30FF, 9), + (0x3400, 0x4DBF, 10), + (0x4E00, 0x9FFF, 10), + (0x1200, 0x139F, 11), + ]; + let u = u32::from(c); + SCRIPTS.iter().find(|(lo, hi, _)| (*lo..=*hi).contains(&u)).map_or(0x100 + (u >> 7), |(_, _, s)| *s) +} + +/// The scripts the words of `d` are written in ([`script_of`] of their letters), sorted. +fn scripts(d: &AffixDict) -> Arc<[u32]> { + let mut seen = std::collections::BTreeSet::new(); + for c in d.stems().flat_map(str::chars).filter(|c| c.is_alphabetic()) { + seen.insert(script_of(c)); + } + seen.into_iter().collect() +} + +/// Is `word` in another script than the dictionary's (`scripts`): none of its letters is in one +/// of them? A word in the dictionary's script is never, whatever its letters (`zzzz` in Dutch). +fn other_script(word: &str, scripts: &[u32]) -> bool { + !scripts.is_empty() && !word.chars().filter(|c| c.is_alphabetic()).any(|c| scripts.binary_search(&script_of(c)).is_ok()) } #[cfg(test)] @@ -149,30 +228,34 @@ pub(crate) fn names(locale: PackLocale) -> Vec { /// `locale`'s dictionary, loaded on first use; an empty one when there is none. fn get(locale: PackLocale) -> (Arc, Option) { + let (d, file, _) = get_with_scripts(locale); + (d, file) +} + +/// [`get`], with the dictionary's [`scripts`]. +fn get_with_scripts(locale: PackLocale) -> (Arc, Option, Arc<[u32]>) { let reloads = { let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner); c.clock = c.clock.wrapping_add(1); let now = c.clock; if let Some(l) = c.loaded.iter_mut().find(|l| l.locale == locale) { l.used = now; - return (l.dict.clone(), l.file.clone()); + return (l.dict.clone(), l.file.clone(), l.scripts.clone()); } if c.missing.contains(&locale) { - return (Arc::default(), None); + return (Arc::default(), None, Arc::from([])); } c.reloads }; let (dict, file, bytes) = load(locale); + let scripts = scripts(&dict); let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner); if c.reloads != reloads || c.loaded.iter().any(|l| l.locale == locale) || c.missing.contains(&locale) { - return (dict, file); + return (dict, file, scripts); } if dict.is_empty() { - if c.missing.len() >= MAX_MISSING { - c.missing.clear(); - } - c.missing.push(locale); - return (dict, file); + remember_missing(&mut c.missing, locale); + return (dict, file, scripts); } LOADS.fetch_add(1, Ordering::Relaxed); let budget = BUDGET.load(Ordering::Relaxed); @@ -187,8 +270,8 @@ fn get(locale: PackLocale) -> (Arc, Option) { total = total.saturating_sub(gone.bytes); } let used = c.clock; - c.loaded.push(Loaded { locale, dict: dict.clone(), file: file.clone(), bytes, used }); - (dict, file) + c.loaded.push(Loaded { locale, dict: dict.clone(), scripts: scripts.clone(), file: file.clone(), bytes, used }); + (dict, file, scripts) } /// Load `locale`'s dictionary, with its estimated memory. @@ -229,11 +312,14 @@ pub(crate) fn load_error(locale: PackLocale) -> Option { /// Is `word` right in `locale`? Every word is when there is no dictionary. pub(crate) fn is_correct(word: &str, locale: PackLocale, opts: &crate::Options) -> bool { - let (d, _) = get(locale); + let (d, _, scripts) = get_with_scripts(locale); if d.is_empty() { return true; } let Some(w) = crate::spelling::word_to_check_in(&d, word, opts) else { return true }; + if other_script(w, &scripts) { + return true; + } let known = |w: &str| d.check(w) || crate::in_user_dictionary(&w.to_lowercase()); if w.contains('-') && !known(w) { return w.split('-').all(|p| p.chars().count() <= 1 || known(p)); @@ -264,6 +350,57 @@ mod tests { PackLocale::new(t).unwrap() } + /// Review follow-up (craft-libs#485): a full list of locales with no dictionary forgets its + /// oldest one, not all of them, so a document in more languages doesn't look every one up again. + #[test] + fn missing_locales_forget_the_oldest_first() { + let tag = |i: usize| l(&format!("q{}{}", (b'a' + (i / 26 % 26) as u8) as char, (b'a' + (i % 26) as u8) as char)); + let mut missing = Vec::new(); + for i in 0..MAX_MISSING + 10 { + remember_missing(&mut missing, tag(i)); + assert!(missing.len() <= MAX_MISSING); + } + assert_eq!(missing.len(), MAX_MISSING); + assert!(!missing.contains(&tag(0)) && !missing.contains(&tag(9)), "the oldest went first"); + assert!(missing.contains(&tag(10)) && missing.contains(&tag(MAX_MISSING + 9)), "the rest stay"); + } + + /// Review follow-up (craft-libs#485): a panic inside a check (caught by the app) doesn't leave + /// this thread's dictionaries pinned past the budget for good. + #[test] + fn a_panic_inside_a_check_ends_it() { + let caught = std::panic::catch_unwind(|| in_one_check(|| panic!("a bug inside a check"))); + assert!(caught.is_err()); + assert_eq!(CHECK_FROM.with(Cell::get), None); + in_one_check(|| { + assert!(CHECK_FROM.with(Cell::get).is_some()); + in_one_check(|| ()); + assert!(CHECK_FROM.with(Cell::get).is_some(), "an inner check doesn't end the outer one"); + }); + assert_eq!(CHECK_FROM.with(Cell::get), None); + } + + /// Review follow-up (craft-libs#485): a word with no letter in the dictionary's scripts is + /// in another script (a Cyrillic word in a Dutch run); a word in its script is always checked, + /// even with letters no dictionary word uses (CodeRabbit on #496: `zzzz`). + #[test] + fn words_in_another_script_are_not_the_dictionarys() { + let nl = AffixDict::from_hunspell("SET UTF-8\n", "3\nhuis\nkat\nÉén\n", &[]).unwrap(); + let latin = scripts(&nl); + assert_eq!(latin.len(), 1); + for w in ["привет", "Привет", "αβγ", "שלום", "日本"] { + assert!(other_script(w, &latin), "{w}"); + } + for w in ["huus", "kät", "huisп", "ÉÉN", "zzzz", "qxqw", "ŋʒ"] { + assert!(!other_script(w, &latin), "{w}"); + } + assert!(!other_script("привет", &[]), "no scripts: every word is checked"); + let ru = AffixDict::from_hunspell("SET UTF-8\n", "1\nдом\n", &[]).unwrap(); + assert!(!other_script("щщщ", &scripts(&ru)) && other_script("dom", &scripts(&ru))); + assert_ne!(script_of('ა'), script_of('a')); + assert_eq!(script_of('ă'), script_of('Ạ')); + } + #[test] fn locales_and_file_names() { assert_eq!(l("nl-NL").as_str(), "nl-nl"); diff --git a/crates/proof/src/packs.rs b/crates/proof/src/packs.rs index 9d5ea1fa..2e558393 100644 --- a/crates/proof/src/packs.rs +++ b/crates/proof/src/packs.rs @@ -34,6 +34,13 @@ pub(crate) const SPELLING: &str = "spelling"; const HYPHENATION: &str = "hyphenation.pat"; /// Most languages whose pack patterns are remembered (a document may tag thousands). const MAX_HYPHENATION_LOOKUPS: usize = 256; +/// The memory budget of the remembered pack patterns, in estimated bytes ([`pattern_bytes`]): +/// a pack's patterns may be as large as a dictionary (64 MB of file), and up to +/// [`MAX_HYPHENATION_LOOKUPS`] of them are kept, so they get a budget of their own, as the +/// dictionaries do (`pack_lang::MAX_LOADED_BYTES`). +const MAX_HYPHENATION_BYTES: u64 = 256 << 20; +/// Estimated memory per pattern (its packed key, its values and the map's own room). +const BYTES_PER_PATTERN: u64 = 64; /// A pack's description. const PACK_TOML: &str = "pack.toml"; /// A pack's licence texts. @@ -80,15 +87,18 @@ struct State { dir: Option, scan: Option>, /// Each language's pack patterns once looked up (`None`: no pack has them), at most - /// [`MAX_HYPHENATION_LOOKUPS`] languages (cleared when full). + /// [`MAX_HYPHENATION_LOOKUPS`] languages and [`MAX_HYPHENATION_BYTES`] (cleared when full). hyphenation: BTreeMap>>, + /// The estimated memory of the patterns in [`State::hyphenation`]. + hyphenation_bytes: u64, /// (locale, reason) of pack hyphenation files that failed to load. hyphenation_errors: Vec<(String, String)>, /// Counts changes, so a scan or load that started before one is not kept. changes: u64, } -static STATE: RwLock = RwLock::new(State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_errors: Vec::new(), changes: 0 }); +static STATE: RwLock = + RwLock::new(State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_bytes: 0, hyphenation_errors: Vec::new(), changes: 0 }); fn read() -> RwLockReadGuard<'static, State> { STATE.read().unwrap_or_else(PoisonError::into_inner) @@ -121,6 +131,7 @@ pub fn reload() { fn forget(s: &mut State) { s.scan = None; s.hyphenation.clear(); + s.hyphenation_bytes = 0; s.hyphenation_errors.clear(); s.changes = s.changes.wrapping_add(1); } @@ -532,7 +543,7 @@ fn hyphenation_locales(lang: ProofLang) -> Vec { ProofLang::EsUy => &["es-uy", "es-419", "es"], ProofLang::De => &["de", "de-1996", "de-de"], ProofLang::Pack(l) => return crate::pack_lang::hyphenation_locales(l), - ProofLang::En | ProofLang::EnGb | ProofLang::EnCa | ProofLang::Other => &[], + ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa | ProofLang::Other => &[], }; names.iter().map(|n| (*n).to_string()).collect() } @@ -567,10 +578,8 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option> { let mut s = write(); if s.changes == changes && !s.hyphenation.contains_key(&lang) { // ponytail: cleared when full, an LRU if many languages each need their patterns. - if s.hyphenation.len() >= MAX_HYPHENATION_LOOKUPS { - s.hyphenation.clear(); - } - s.hyphenation.insert(lang, found.clone()); + let bytes = found.as_deref().map_or(0, pattern_bytes); + remember_patterns(&mut s, lang, found.clone(), bytes, MAX_HYPHENATION_BYTES); for e in errors { if !s.hyphenation_errors.contains(&e) { s.hyphenation_errors.push(e); @@ -580,6 +589,23 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option> { found } +/// Estimated memory of `p`. +fn pattern_bytes(p: &Patterns) -> u64 { + (p.len() as u64).saturating_add(p.exception_count() as u64).saturating_mul(BYTES_PER_PATTERN) +} + +/// Remember `lang`'s pack patterns (`bytes` of them): over [`MAX_HYPHENATION_LOOKUPS`] languages +/// or `budget` bytes, the ones remembered so far are forgotten first. Patterns larger than the +/// whole budget are still kept alone, or they would be read from disk for every word. +fn remember_patterns(s: &mut State, lang: ProofLang, found: Option>, bytes: u64, budget: u64) { + if s.hyphenation.len() >= MAX_HYPHENATION_LOOKUPS || s.hyphenation_bytes.saturating_add(bytes) > budget { + s.hyphenation.clear(); + s.hyphenation_bytes = 0; + } + s.hyphenation.insert(lang, found); + s.hyphenation_bytes = s.hyphenation_bytes.saturating_add(bytes); +} + /// How many languages' pack patterns are remembered now. #[cfg(test)] pub(crate) fn hyphenation_lookups() -> usize { @@ -601,6 +627,29 @@ fn load_patterns(path: &Path) -> Result { mod tests { use super::*; + /// Review follow-up (craft-libs#485): the remembered pack patterns have a memory budget, not + /// only a count, so 256 languages of large patterns can't each stay in memory. + #[test] + fn remembered_patterns_keep_to_a_memory_budget() { + let mut s = State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_bytes: 0, hyphenation_errors: Vec::new(), changes: 0 }; + let p = Arc::new(Patterns::parse("hy3ph he2n hena4 hen5at 1na n2at 1tio 2io o2n")); + let bytes = pattern_bytes(&p); + assert_eq!(bytes, 9 * BYTES_PER_PATTERN); + let lang = |t: &str| ProofLang::from_tag(Some(t)); + let budget = bytes * 2; + remember_patterns(&mut s, lang("nl"), Some(p.clone()), bytes, budget); + remember_patterns(&mut s, lang("fr"), Some(p.clone()), bytes, budget); + remember_patterns(&mut s, lang("sv"), None, 0, budget); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (3, budget), "a language with no patterns takes no room"); + remember_patterns(&mut s, lang("da"), Some(p.clone()), bytes, budget); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (1, bytes), "over the budget: the others are forgotten"); + assert!(s.hyphenation.contains_key(&lang("da"))); + remember_patterns(&mut s, lang("fi"), Some(p.clone()), budget * 4, budget); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (1, budget * 4), "patterns over the whole budget are kept alone"); + forget(&mut s); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (0, 0)); + } + /// Review of #485: a pack's name loses bidirectional and other format characters. #[test] fn pack_names_are_plain_text() { diff --git a/crates/proof/src/pl/grammar.rs b/crates/proof/src/pl/grammar.rs index d514429e..2fd93f86 100644 --- a/crates/proof/src/pl/grammar.rs +++ b/crates/proof/src/pl/grammar.rs @@ -304,11 +304,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Zbędna spacja przed znakiem interpunkcyjnym".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Podwójna spacja".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Podwójna spacja".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/pt/grammar.rs b/crates/proof/src/pt/grammar.rs index 303929de..eb0f89a6 100644 --- a/crates/proof/src/pt/grammar.rs +++ b/crates/proof/src/pt/grammar.rs @@ -62,7 +62,12 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { if end == '.' && abbreviation_or_number(prev) { return false; } - let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING); + let after_marks = t.trim_start_matches(['.', '!', '?']); + let rest = after_marks.trim_start_matches(CLOSING); + // `"Vai?" disse ele.` (not the object placeholder `\u{1}`): narration after a quoted question or exclamation goes on in lower case. + if end != '.' && after_marks.trim_start_matches('\u{1}').starts_with(CLOSING) { + return false; + } // A dialogue dash after the mark brings back the narration (`—Vai? —disse ele.`), which goes // on in lower case. rest.starts_with(char::is_whitespace) && !rest.contains('—') && rest.trim_start().chars().all(|c| OPENING.contains(&c)) @@ -124,11 +129,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Espaço antes da pontuação".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Espaço duplo".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Espaço duplo".into(), vec![" ".into()])); } } @@ -225,6 +227,14 @@ mod tests { // craft-libs#488 review: no space after the dash. clean("—Vai? —disse ele."); clean("— Vai? — disse ele."); + // craft-libs#496 review: a closing quote after the mark. + clean("\"Vai?\" disse ele."); + clean("«Vai!» gritou ela."); + assert!(!messages("Vai? \"sim\".").is_empty(), "a bare question still ends the sentence"); + // An inline object (`\u{1}`) before a real closing quote is skipped. + clean("\"Vai?\u{1}\" disse ele."); + // An inline object (`\u{1}`) is not a closing quote. + assert!(!messages("Quem era?\u{1} era Marco.").is_empty()); clean("Ele disse: \"Volto amanhã\". E saiu."); // A list item may start in lower case. clean("maçãs,"); diff --git a/crates/proof/src/ru/grammar.rs b/crates/proof/src/ru/grammar.rs index ff8711fb..4d1ce3c2 100644 --- a/crates/proof/src/ru/grammar.rs +++ b/crates/proof/src/ru/grammar.rs @@ -503,11 +503,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Лишний пробел перед знаком препинания".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Двойной пробел".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Двойной пробел".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index 0bed8dfc..4a7cd3e0 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -105,7 +105,7 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) { return true; } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS { + if opts.ignore_uppercase && crate::spelling::in_capitals(w) && n <= ABBREVIATION_LETTERS { return true; } // Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word. @@ -333,7 +333,7 @@ fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool { let last = parts.len() - 1; parts.iter().enumerate().all(|(i, p)| { // An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word. - if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) { + if known(d, p) || (opts.ignore_uppercase && crate::spelling::capitals_part(p, ABBREVIATION_LETTERS)) { return true; } // A first part with a linking vowel: the stem (`северо` → `север`) or its adjective @@ -485,5 +485,7 @@ mod tests { assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too"); assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults"); + // The cased-letter rule (craft-libs#485): capitals and letterless parts are abbreviations. + assert!(crate::is_correct_in_with("ИИ-2", lang, &check_digits) && crate::is_correct_in_with("ТУ-154М", lang, &check_digits)); } } diff --git a/crates/proof/src/scowl.rs b/crates/proof/src/scowl.rs index 9a71188f..cb101cd8 100644 --- a/crates/proof/src/scowl.rs +++ b/crates/proof/src/scowl.rs @@ -2,10 +2,15 @@ //! dictionaries of craft-i18n's `language-data/en-us`, `en-gb` (British `-ise`) and `en-ca`, //! embedded unmodified at build time and read by the affix engine ([`crate::affix`]) on first use. //! -//! Each English has its own: American English reads en-US, the British-spelling Englishes -//! (en-GB, en-AU, en-NZ, en-IE…) read en-GB and Canadian English reads en-CA. Its suggestions +//! Each English has its own: American English reads en-US, Australian English en-AU (SCOWL's +//! Australian words: `billabong`, `brumby`; and `program`, `inquiry`), the other British-spelling +//! Englishes (en-GB, en-NZ, en-IE…) read en-GB and Canadian English reads en-CA. SCOWL has no +//! en-NZ: New Zealand English keeps `programme` and has few words of its own in SCOWL's en-AU +//! data, so it stays on en-GB. Its suggestions //! come from its own dictionary, and a word is right when its own dictionary knows it: the other -//! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's). SCOWL +//! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's), except +//! that en-AU's list keeps one spelling per word, so Australian English also takes en-GB's words +//! (`enquiry`; its suggestions still come from en-AU). SCOWL //! picks one spelling per dialect (en-GB `-ise` has no `organize`, en-US no `judgement`), and Word //! accepts more, so British English also takes the `-ize` spellings of its `-ise` words that //! en-US or en-CA spells so, and `en-variants.txt` widens each dialect by its pairs @@ -19,16 +24,17 @@ use std::sync::OnceLock; use crate::ProofLang; use crate::affix::AffixDict; -use crate::data::{EN_CA_AFF, EN_CA_SPELLING, EN_GB_AFF, EN_GB_SPELLING, EN_US_AFF, EN_US_SPELLING, SCOWL_NOTICE}; +use crate::data::{EN_AU_AFF, EN_AU_SPELLING, EN_CA_AFF, EN_CA_SPELLING, EN_GB_AFF, EN_GB_SPELLING, EN_US_AFF, EN_US_SPELLING, SCOWL_NOTICE}; -/// Per English (American, British, Canadian): the loaded dictionary, or why it isn't there. -static DICTS: [OnceLock>; 3] = [OnceLock::new(), OnceLock::new(), OnceLock::new()]; +/// Per English (American, British, Canadian, Australian): the loaded dictionary, or why it isn't there. +static DICTS: [OnceLock>; 4] = [OnceLock::new(), OnceLock::new(), OnceLock::new(), OnceLock::new()]; /// The slot, `.aff` and `.dic` text of `lang`'s dictionary. fn source(lang: ProofLang) -> (usize, &'static str, &'static str) { match lang { ProofLang::EnGb => (1, EN_GB_AFF, EN_GB_SPELLING), ProofLang::EnCa => (2, EN_CA_AFF, EN_CA_SPELLING), + ProofLang::EnAu => (3, EN_AU_AFF, EN_AU_SPELLING), _ => (0, EN_US_AFF, EN_US_SPELLING), } } @@ -50,15 +56,17 @@ pub(crate) fn dict(lang: ProofLang) -> Option<&'static AffixDict> { loaded.as_ref().ok() } -/// Does `lang` read the British `-ise` dictionary (en-GB, en-AU, en-NZ…)? +/// Does `lang` read a British `-ise` dictionary (en-GB, and en-NZ… through it; en-AU)? pub(crate) fn is_british(lang: ProofLang) -> bool { - source(lang).0 == 1 + matches!(source(lang).0, 1 | 3) } /// Why `lang`'s embedded SCOWL dictionary failed to load, once something tried to load it. pub(crate) fn load_error(lang: ProofLang) -> Option { let (i, ..) = source(lang); - DICTS.get(i)?.get()?.as_ref().err().map(|e| format!("embedded SCOWL dictionary: {e}")) + let own = DICTS.get(i).and_then(|d| d.get()).and_then(|r| r.as_ref().err()).map(|e| format!("embedded SCOWL dictionary: {e}")); + // Australian English also reads the en-GB list (`crate::english`): its failure is reported too. + own.or_else(|| if lang == ProofLang::EnAu { load_error(ProofLang::EnGb).map(|e| format!("{e} (en-GB, also read for en-AU)")) } else { None }) } /// The SCOWL copyright and licence notices (the upstream README, with the Ispell BSD licence of diff --git a/crates/proof/src/spelling.rs b/crates/proof/src/spelling.rs index c80f42fe..1b42f5ff 100644 --- a/crates/proof/src/spelling.rs +++ b/crates/proof/src/spelling.rs @@ -32,17 +32,43 @@ pub(crate) fn word_to_check_in<'a>(d: &AffixDict, word: &'a str, opts: &Options) if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { return None; } - // Only a word with a cased letter can be in capitals: Hebrew, Arabic, Thai… words have none. - if opts.ignore_uppercase && w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { + if opts.ignore_uppercase && in_capitals(w) && n <= 6 { return None; } Some(w) } +/// Is `w` in capitals: a cased letter and no lower-case one? Only a word with a cased letter can +/// be: Hebrew, Arabic, Thai, Chinese… words have none, so they are never taken for acronyms. +pub(crate) fn in_capitals(w: &str) -> bool { + w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase()) +} + +/// Is `part` of a hyphenated word an abbreviation Ignore words in UPPERCASE skips: at most `max` +/// characters, [`in_capitals`] or without letters (`SMS` in `SMS-ка`, `19` in `COVID-19`)? A +/// part in a caseless script is no abbreviation. +pub(crate) fn capitals_part(part: &str, max: usize) -> bool { + part.chars().count() <= max && (in_capitals(part) || !part.chars().any(char::is_alphabetic)) +} + #[cfg(test)] mod tests { use super::*; + /// craft-libs#485's cased-letter rule, shared by every language: a word or part is in + /// capitals only with a cased letter; a part without letters counts as an abbreviation. + #[test] + fn capitals_need_a_cased_letter() { + for w in ["PKO", "SMS", "ÉTÉ", "ВКП", "B2B"] { + assert!(in_capitals(w) && capitals_part(w, 6), "{w}"); + } + for w in ["日本", "שלום", "ไทย", "Pko", "pko", "123", ""] { + assert!(!in_capitals(w), "{w}"); + } + assert!(capitals_part("19", 6) && capitals_part("", 6)); + assert!(!capitals_part("日本", 6) && !capitals_part("ไทย", 6) && !capitals_part("ABCDEFG", 6)); + } + #[test] fn words_needing_no_lookup() { let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap(); diff --git a/crates/proof/src/tr/grammar.rs b/crates/proof/src/tr/grammar.rs index b2391000..4dcdc84b 100644 --- a/crates/proof/src/tr/grammar.rs +++ b/crates/proof/src/tr/grammar.rs @@ -211,23 +211,17 @@ pub fn check_with(text: &str, capital_first: bool) -> Vec { }); } } - // Double space - if c == ' ' && i + 1 < chars.len() && chars[i + 1].1 == ' ' && (i == 0 || chars[i - 1].1 != ' ') { - let mut end = idx + 1; - while end < text.len() && text.as_bytes().get(end) == Some(&b' ') { - end += 1; - } - v.push(Issue { - start: idx, - end, - kind: IssueKind::Grammar, - message: "Fazladan boşluk".into(), - suggestions: vec![" ".into()], - lang: ProofLang::Tr, - }); - } } - + for run in crate::space_runs(text) { + v.push(Issue { + start: run.start, + end: run.end, + kind: IssueKind::Grammar, + message: "Fazladan boşluk".into(), + suggestions: vec![" ".into()], + lang: ProofLang::Tr, + }); + } // 3. Comma before or after conjunctions (`ali, ve veli` or `ve, veli`) for (i, &(w_start, w_end)) in ws.iter().enumerate() { let word = match text.get(w_start..w_end) { diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs index 0bb705b8..9dbff6f1 100644 --- a/crates/proof/src/tr/suggest.rs +++ b/crates/proof/src/tr/suggest.rs @@ -97,8 +97,8 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { } } - // 3. Detached suffix check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü` - for suffix in ["de", "da", "te", "ta", "mi", "mı", "mu", "mü"] { + // 3. Detached particle check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü` + for suffix in ["de", "da", "mi", "mı", "mu", "mü"] { if lower.ends_with(suffix) && lower.len() > suffix.len() + 1 { let stem = &lower[..lower.len() - suffix.len()]; if d.is_empty() || d.contains(stem) { @@ -333,6 +333,19 @@ mod tests { assert!(sugs.contains(&"geldin mi".to_string())); } + /// craft-libs#496 review: the particle is `de`/`da` whatever consonant comes before it. `te` + /// and `ta` are only the locative suffix after a voiceless consonant (`kitapta`), written + /// attached, so `kitapte` is never offered as `kitap te`. + #[test] + fn the_detached_particle_is_never_te_or_ta() { + let d = TrDict::from_hunspell("SET UTF-8\n", "2\nkitap\nsonra\n").unwrap(); + assert!(suggest_with(&d, "kitapde", 8).contains(&"kitap de".to_string())); + for w in ["kitapte", "kitapta"] { + let sugs = suggest_with(&d, w, 8); + assert!(!sugs.iter().any(|s| s == "kitap te" || s == "kitap ta"), "{w}: {sugs:?}"); + } + } + #[test] fn overlong_words_get_no_suggestions_promptly() { let d = TrDict::from_hunspell("SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n", "2\nkitap/1\nsonra\n").unwrap(); diff --git a/crates/ui/README.md b/crates/ui/README.md index 3906429e..16bb8d95 100644 --- a/crates/ui/README.md +++ b/crates/ui/README.md @@ -11,6 +11,45 @@ The current delivery covers nine creative apps (DesignCraft, VectorCraft, PhotoC records their scope and acceptance limits. Assets (font files included), complete themes, localization and command handling remain in each application. No icons, fonts or application engines are bundled. +## Host-owned docking chrome + +`DockContent` can replace an iconic column or a complete tab group without a +second layout tree. `column_rect` reports the fitted column before painting; +`owns_iconic_column`/`iconic_column` and `custom_group` suppress the corresponding +stock chrome, body and input when the host takes ownership. Docked custom groups +also skip stock body top/bottom drop bands; column-edge targets remain available. +These hooks are also +forwarded by `show_customized`'s label/permissions/limits adapter. + +`resize_handle_style` and `split_separator` let the host paint one seam while +retaining stock resize input. `owns_floating_width` delegates horizontal floating +frame-edge handles to the host; internal dividers and vertical handles remain available. +`owns_floating_split_width` separately delegates internal horizontal dividers when +the host supplies their replacement input. Presentation-only +`group_collapsed` does not change the selected tab in the shared model. + +All ownership hooks default to false/no-op. `DockStyle::dock_gap` defaults to +`None`, preserving the ordinary split gap; setting it affects side-dock seams +only. `FloatingFrame::resize_axes` defaults to both axes and preserves an enabled +axis at corners when the other is disabled. A host with its own title bar can +disable the separate top gutter through `reserve_top_resize_band(false)`. + +Compatibility and target-size audit: before these hooks, resizable frames already +reserved a 4 pt top band. That remains the default; `false` is the new host opt-out. +The unchanged stock edge hit bands are 4 pt thick; corners are at most 12 pt. +They do **not** meet a 24-by-24 logical-pixel target at 1x. This PR preserves +those hit rectangles and does not claim accessibility compliance. Focused +resize handles also support arrow keys (8 pt steps). Enlarging pointer targets +needs separate overlap, title-bar and viewport-edge acceptance checks. + +`chrome::compatibility_tests::defaults_preserve_pre_host_hook_frame_geometry` +locks down the pre-hook hit rectangles/defaults. `host_hooks_tests` exercises all +four axis settings with actual corner and side pointer drags, minimum/maximum +sizes, the viewport edge, and top-gutter opt-out geometry. + +Exhaustive `DockStyle` struct literals need `dock_gap: None`; constructors such +as `from_ui` and `from_visuals` supply the compatible default. + ## SVG catalogs ```rust diff --git a/crates/ui/src/docking/chrome.rs b/crates/ui/src/docking/chrome.rs index 9e3aa2bc..3841c256 100644 --- a/crates/ui/src/docking/chrome.rs +++ b/crates/ui/src/docking/chrome.rs @@ -116,8 +116,10 @@ pub struct FloatingFrame<'a> { bounds: Option, limits: PanelLimits, resizable: bool, + resize_axes: [bool; 2], resize_label: &'a str, opacity: f32, + reserve_top_resize_band: bool, } /// What a [`FloatingFrame`] did this frame. @@ -137,7 +139,17 @@ pub struct FrameResponse { impl<'a> FloatingFrame<'a> { pub fn new(id: Id, rect: Rect) -> Self { - Self { id, rect, bounds: None, limits: PanelLimits::default(), resizable: true, resize_label: "Resize window", opacity: 1.0 } + Self { + id, + rect, + bounds: None, + limits: PanelLimits::default(), + resizable: true, + resize_axes: [true, true], + resize_label: "Resize window", + opacity: 1.0, + reserve_top_resize_band: true, + } } /// The frame stays inside `bounds`; the default is the viewport's content rect. @@ -153,11 +165,23 @@ impl<'a> FloatingFrame<'a> { self } + /// Reserve a separate top resize gutter; disable when contents already supply a header. + pub fn reserve_top_resize_band(mut self, reserve: bool) -> Self { + self.reserve_top_resize_band = reserve; + self + } + pub fn resizable(mut self, resizable: bool) -> Self { self.resizable = resizable; self } + /// Restrict the stock edge handles when a host owns resizing on one axis. + pub fn resize_axes(mut self, axes: [bool; 2]) -> Self { + self.resize_axes = axes; + self + } + /// Accessibility label of the resize handles. pub fn resize_label(mut self, label: &'a str) -> Self { self.resize_label = label; @@ -192,8 +216,11 @@ impl<'a> FloatingFrame<'a> { ui.set_min_size(rect.size()); ui.allocate_rect(rect, Sense::hover()); ui.painter().rect_filled(rect, 0.0, style.background); - let contents = - if self.resizable { Rect::from_min_max(pos2(rect.left(), (rect.top() + RESIZE_BAND).min(rect.bottom())), rect.max) } else { rect }; + let contents = if self.resizable && self.reserve_top_resize_band { + Rect::from_min_max(pos2(rect.left(), (rect.top() + RESIZE_BAND).min(rect.bottom())), rect.max) + } else { + rect + }; let inner = add_contents(ui, contents); ui.painter().rect_stroke(rect, 0.0, style.stroke, StrokeKind::Outside); let (resized, resize_started, handles) = if self.resizable { self.handles(ui, rect, bounds, style) } else { (None, None, Vec::new()) }; @@ -217,7 +244,14 @@ impl<'a> FloatingFrame<'a> { let mut responses = Vec::new(); let limits = PanelLimits { min: self.limits.min.max(vec2(40.0, RESIZE_BAND + TITLE_HEIGHT + 20.0)).min(self.limits.max), ..self.limits }; let canceled = ui.input(|input| input.key_pressed(Key::Escape)) || crate::focus::lost(ui.ctx()); - for (name, handle, direction) in handles(rect) { + for (name, mut handle, mut direction) in handles(rect) { + if !self.reserve_top_resize_band && direction.y < 0.0 { + handle.max.y = (rect.top() + ui.ctx().pixels_per_point().recip()).min(rect.bottom()); + } + direction = vec2(if self.resize_axes[0] { direction.x } else { 0.0 }, if self.resize_axes[1] { direction.y } else { 0.0 }); + if direction == egui::Vec2::ZERO { + continue; + } if !super::targets::usable(handle) { continue; } @@ -486,3 +520,33 @@ pub fn live_move(ctx: &Context, bar: &Response, rect: Rect, bounds: Rect) -> Opt forget(); None } + +#[cfg(test)] +mod compatibility_tests { + use super::*; + + #[test] + fn defaults_preserve_pre_host_hook_frame_geometry() { + let rect = Rect::from_min_size(pos2(100.0, 100.0), vec2(300.0, 200.0)); + let frame = FloatingFrame::new(Id::unique("defaults"), rect); + assert_eq!(frame.resize_axes, [true, true]); + assert!(frame.reserve_top_resize_band); + assert!(frame.resizable); + // Literal pre-PR hit rectangles: preserve these rather than silently enlarging them. + let expected = [ + [388.0, 288.0, 400.0, 300.0], + [100.0, 112.0, 104.0, 288.0], + [396.0, 112.0, 400.0, 288.0], + [112.0, 296.0, 388.0, 300.0], + [100.0, 288.0, 112.0, 300.0], + [112.0, 100.0, 388.0, 104.0], + [100.0, 100.0, 112.0, 104.0], + [388.0, 100.0, 400.0, 104.0], + ]; + for ((_, hit, _), [x0, y0, x1, y1]) in handles(rect).into_iter().zip(expected) { + assert_eq!(hit, Rect::from_min_max(pos2(x0, y0), pos2(x1, y1))); + // Audit result: legacy targets do NOT meet a 24-by-24 logical-pixel target. + assert!(hit.width() < 24.0 || hit.height() < 24.0); + } + } +} diff --git a/crates/ui/src/docking/host_hooks_tests.rs b/crates/ui/src/docking/host_hooks_tests.rs new file mode 100644 index 00000000..8415b76a --- /dev/null +++ b/crates/ui/src/docking/host_hooks_tests.rs @@ -0,0 +1,425 @@ +//! Host-owned chrome uses the same fitted geometry and preserves stock defaults. +use super::*; +use egui::{Context, Event, Id, Modifiers, PointerButton, RawInput, Rect, pos2, vec2}; + +fn input(events: Vec) -> RawInput { + RawInput { screen_rect: Some(Rect::from_min_size(pos2(0.0, 0.0), vec2(800.0, 600.0))), focused: true, events, ..Default::default() } +} +fn button(at: egui::Pos2, pressed: bool) -> Event { + Event::PointerButton { pos: at, button: PointerButton::Primary, pressed, modifiers: Modifiers::NONE } +} + +#[test] +fn corner_resize_keeps_each_enabled_axis_including_single_axis_hosts() { + for axes in [[true, true], [false, true], [true, false], [false, false]] { + let ctx = Context::default(); + let rect = Rect::from_min_size(pos2(100.0, 100.0), vec2(300.0, 200.0)); + let frame = |events| { + let mut resized = None; + ctx.run_ui(input(events), |ui| { + let style = chrome::ChromeStyle::from_ui(ui); + resized = chrome::FloatingFrame::new(Id::unique("axes"), rect).resize_axes(axes).show(ui.ctx(), &style, |_, _| ()).resized; + }) + .drop_without_applying_deltas(); + resized + }; + frame(vec![]); + frame(vec![]); + let from = rect.right_bottom() - vec2(2.0, 2.0); + frame(vec![Event::PointerMoved(from), button(from, true)]); + frame(vec![Event::PointerMoved(from + vec2(10.0, 10.0))]); + let result = frame(vec![Event::PointerMoved(from + vec2(40.0, 35.0))]); + if axes == [false, false] { + assert!(result.is_none()); + } else { + let expected = rect.size() + vec2(if axes[0] { 40.0 } else { 0.0 }, if axes[1] { 35.0 } else { 0.0 }); + assert_eq!(result, Some(Rect::from_min_size(rect.min, expected)), "{axes:?}"); + } + } +} + +#[test] +fn the_top_gutter_is_reserved_by_default_and_can_be_owned_by_the_contents() { + for reserve in [true, false] { + let ctx = Context::default(); + let rect = Rect::from_min_size(pos2(100.0, 100.0), vec2(300.0, 200.0)); + let mut inner = Rect::NOTHING; + ctx.run_ui(input(vec![]), |ui| { + let style = chrome::ChromeStyle::from_ui(ui); + inner = chrome::FloatingFrame::new(Id::unique("gutter"), rect).reserve_top_resize_band(reserve).show(ui.ctx(), &style, |_, r| r).inner; + }) + .drop_without_applying_deltas(); + assert_eq!(inner.left(), rect.left()); + assert_eq!(inner.max, rect.max); + assert_eq!(inner.top(), rect.top() + if reserve { chrome::RESIZE_BAND } else { 0.0 }); + } +} + +#[derive(Default)] +struct Host { + exclusive: bool, + stock_group: Option, + bodies: usize, + columns: Vec, + iconic: Vec, + groups: std::collections::HashMap, + collapsed: bool, + protected: bool, + frame_width: bool, + split_width: bool, + space: Option>, + style_calls: std::cell::Cell, +} +impl DockContent for Host { + fn group_collapsed(&self, _: &[u32]) -> bool { + self.collapsed + } + fn permissions(&self, _: &u32) -> Permissions { + if self.protected { Permissions::PROTECTED } else { Permissions::default() } + } + fn owns_floating_split_width(&self, _: &u32) -> bool { + self.split_width + } + fn owns_floating_width(&self, _: &u32) -> bool { + self.frame_width + } + fn resize_handle_style(&self, _: crate::layout::ResizeHandleStyle) -> crate::layout::ResizeHandleStyle { + self.style_calls.set(self.style_calls.get() + 1); + crate::layout::ResizeHandleStyle { + hovered: egui::Color32::TRANSPARENT, + dragged: egui::Color32::TRANSPARENT, + focused: egui::Color32::TRANSPARENT, + } + } + fn body(&mut self, _: &mut egui::Ui, _: &u32) { + self.bodies += 1; + } + fn custom_group(&mut self, _: &mut egui::Ui, _: &[u32], selected: &u32, rect: Rect, _: bool) -> bool { + self.groups.insert(*selected, rect); + self.exclusive && self.stock_group.as_ref() != Some(selected) + } + fn owns_iconic_column(&self, _: &u32) -> bool { + self.exclusive + } + fn column_rect(&mut self, _: &Context, _: &Column, rect: Rect, _: Side) { + self.columns.push(rect); + } + fn iconic_column(&mut self, _: &mut egui::Ui, _: &Column, rect: Rect, _: Side) { + self.iconic.push(rect); + } +} +fn render(layout: &Layout, host: &mut Host) -> Vec<(Hit, Rect)> { + let ctx = Context::default(); + for _ in 0..3 { + ctx.run_ui(input(vec![]), |ui| { + host.bodies = 0; + host.columns.clear(); + host.iconic.clear(); + let mut style = DockStyle::from_ui(ui); + super::floating_tests::frame_style(&mut style); + let protected = host.protected; + let shown = DockArea::new(Id::unique("host")).show_customized( + ui, + layout, + &style, + |id| format!("Pane {id}"), + |_| if protected { Permissions::PROTECTED } else { Permissions::default() }, + |_| PanelLimits::default(), + host, + ); + assert!(shown.error.is_none(), "{:?}", shown.error); + }) + .drop_without_applying_deltas(); + } + host.space = float_space(&ctx, Id::unique("host")); + DockArea::hit_rects(&ctx, Id::unique("host")) +} + +#[test] +fn an_exclusive_group_replaces_the_stock_body_and_tab_targets_through_overrides() { + let layout = Layout { root: Some(Node::Tabs { panels: vec![1, 2], active: 0 }), floating: vec![] }; + for exclusive in [false, true] { + let mut host = Host { exclusive, ..Default::default() }; + let hits = render(&layout, &mut host); + assert_eq!(host.bodies, usize::from(!exclusive)); + assert_eq!(hits.iter().any(|(hit, _)| matches!(hit, Hit::Tab(_))), !exclusive); + } +} + +#[test] +fn an_iconic_host_receives_the_fitted_column_rect_without_a_second_stock_body() { + let mut column = Column::new(vec![Group::new(vec![1, 2], 0)], 900.0); + column.iconic = true; + let layout = Layout { root: Some(Node::Dock { side: Side::Start, columns: vec![column] }), floating: vec![] }; + let mut host = Host { exclusive: true, ..Default::default() }; + render(&layout, &mut host); + assert_eq!(host.columns.len(), 1); + assert_eq!(host.columns, host.iconic); + assert_eq!(host.bodies, 0); + let rect = host.columns[0]; + assert!(rect.width() > 0.0 && rect.right() <= 800.0 && rect.bottom() <= 600.0); +} + +#[test] +fn invalid_side_dock_gaps_are_rejected_before_host_painting() { + for gap in [f32::NAN, f32::INFINITY, -1.0, 4097.0] { + let ctx = Context::default(); + let layout = Layout { root: Some(Node::Tabs { panels: vec![1], active: 0 }), floating: vec![] }; + let mut host = Host::default(); + ctx.run_ui(input(vec![]), |ui| { + let mut style = DockStyle::from_ui(ui); + style.dock_gap = Some(gap); + let shown = DockArea::new(Id::unique("invalid-gap")).show_content(ui, &layout, &style, &mut host); + assert!(shown.error.is_some(), "{gap}"); + assert_eq!(host.bodies, 0); + }) + .drop_without_applying_deltas(); + } +} + +#[test] +fn host_owned_surfaces_record_fitted_float_geometry() { + let mut host = Host { exclusive: true, ..Default::default() }; + let layout = Layout { root: Some(Node::Tabs { panels: vec![1, 2], active: 0 }), floating: vec![] }; + render(&layout, &mut host); + let space = host.space.as_ref().unwrap(); + assert!(space.panel(&1).is_some_and(|s| s.x > 700.0 && s.y > 0.0)); + assert!(space.group(&2).is_some_and(|s| s.x > 700.0 && s.y > 0.0)); +} + +#[test] +fn host_owned_iconic_columns_record_fitted_float_geometry() { + let mut host = Host { exclusive: true, ..Default::default() }; + let mut column = Column::new(vec![Group::new(vec![1, 2], 0)], 90.0); + column.iconic = true; + let layout = Layout { root: Some(Node::Dock { side: Side::Start, columns: vec![column] }), floating: vec![] }; + render(&layout, &mut host); + let size = host.space.as_ref().unwrap().column(&1).expect("host column is recorded"); + assert_eq!(size.x, host.columns[0].width()); + assert!(size.y > 0.0); +} + +#[test] +fn a_host_collapsed_protected_group_keeps_its_selection_without_drawing_a_body() { + let layout = Layout { root: Some(Node::Tabs { panels: vec![1], active: 0 }), floating: vec![] }; + let mut host = Host { protected: true, collapsed: true, ..Default::default() }; + render(&layout, &mut host); + assert_eq!(host.bodies, 0); + assert!(matches!(layout.root, Some(Node::Tabs { active: 0, .. }))); + assert!(host.space.as_ref().unwrap().panel(&1).is_none()); +} + +#[test] +fn frame_width_ownership_keeps_internal_split_input_and_host_style() { + use crate::layout::{SplitAxis, SplitSize}; + let ctx = Context::default(); + let mut window = Floating::single(vec![1], 0, [100.0, 100.0, 500.0, 350.0]); + window.root = Node::Split { + axis: SplitAxis::Horizontal, + size: SplitSize::Ratio(0.5), + first: Box::new(Node::Tabs { panels: vec![1], active: 0 }), + second: Box::new(Node::Tabs { panels: vec![2], active: 0 }), + }; + let layout = Layout { root: None, floating: vec![window] }; + let mut host = Host { frame_width: true, ..Default::default() }; + let mut step = |events| { + let mut out = DockOutput::default(); + ctx.run_ui(input(events), |ui| { + let mut style = DockStyle::from_ui(ui); + super::floating_tests::frame_style(&mut style); + out = DockArea::new(Id::unique("owned-width")).show_content(ui, &layout, &style, &mut host); + }) + .drop_without_applying_deltas(); + out + }; + step(vec![]); + step(vec![]); + step(vec![]); + let hit = Hit::Divider(Divider::WindowSplit(1, vec![])); + let hits = DockArea::::hit_rects(&ctx, Id::unique("owned-width")); + let at = hits.iter().find(|(h, _)| *h == hit).expect("internal divider remains interactive").1.center(); + step(vec![Event::PointerMoved(at), button(at, true)]); + step(vec![Event::PointerMoved(at + vec2(10.0, 0.0))]); + let out = step(vec![Event::PointerMoved(at + vec2(40.0, 0.0))]); + assert!(out.actions.iter().any(|a| matches!(a, Action::ResizeWindowSplit { .. })), "{:?}", out.actions); + assert!(host.style_calls.get() > 0); +} + +#[test] +fn internal_width_input_is_suppressed_only_when_explicitly_owned_through_overrides() { + use crate::layout::{SplitAxis, SplitSize}; + let mut window = Floating::single(vec![1], 0, [100.0, 100.0, 500.0, 350.0]); + window.root = Node::Split { + axis: SplitAxis::Horizontal, + size: SplitSize::Ratio(0.5), + first: Box::new(Node::Tabs { panels: vec![1], active: 0 }), + second: Box::new(Node::Tabs { panels: vec![2], active: 0 }), + }; + let layout = Layout { root: None, floating: vec![window] }; + for owned in [false, true] { + let mut host = Host { frame_width: true, split_width: owned, ..Default::default() }; + let hits = render(&layout, &mut host); + assert_eq!(hits.iter().any(|(hit, _)| *hit == Hit::Divider(Divider::WindowSplit(1, vec![]))), !owned); + } +} + +#[test] +fn exclusive_docked_groups_skip_stock_body_bands_but_keep_column_edges() { + for owned in [false, true] { + for (zone, anchor, inset) in [ + (Zone::Top, egui::Align2::CENTER_TOP, vec2(0.0, 50.0)), + (Zone::Bottom, egui::Align2::CENTER_BOTTOM, vec2(0.0, -20.0)), + (Zone::Left, egui::Align2::LEFT_CENTER, vec2(20.0, 0.0)), + (Zone::Right, egui::Align2::RIGHT_CENTER, vec2(-20.0, 0.0)), + ] { + let ctx = Context::default(); + let area = Id::unique("owned-dock-targets"); + let layout = Layout { + root: Some(Node::Dock { side: Side::Start, columns: vec![Column::new(vec![Group::new(vec![1, 2], 0)], 240.0)] }), + floating: vec![Floating::single(vec![99], 0, [500.0, 100.0, 200.0, 150.0])], + }; + let mut host = Host { exclusive: owned, stock_group: Some(99), ..Default::default() }; + let mut step = |events, start: bool| { + let mut out = DockOutput::default(); + ctx.run_ui(input(events), |ui| { + let mut style = DockStyle::from_ui(ui); + super::floating_tests::frame_style(&mut style); + out = DockArea::new(area).show_content(ui, &layout, &style, &mut host); + if start { + assert!(begin_drag(ui.ctx(), area, 99, Rect::from_min_size(pos2(500.0, 100.0), vec2(200.0, 150.0)))); + } + }) + .drop_without_applying_deltas(); + (out, host.groups.get(&1).copied()) + }; + step(vec![], false); + step(vec![], false); + let (_, target) = step(vec![], false); + let target = target.expect("the docked group is drawn"); + let from = pos2(550.0, 140.0); + step(vec![Event::PointerMoved(from), button(from, true)], true); + let to = anchor.pos_in_rect(&target) + inset; + step(vec![Event::PointerMoved(to)], false); + let (out, _) = step(vec![Event::PointerMoved(to), button(to, false)], false); + let group_drop = out.actions.iter().any(|a| { + matches!( + a, + Action::MovePanel { to: Target::At { placement: Placement::Group { .. }, .. }, .. } + | Action::MoveGroup { to: Target::At { placement: Placement::Group { .. }, .. }, .. } + ) + }); + if matches!(zone, Zone::Top | Zone::Bottom) { + assert_eq!(group_drop, !owned, "{owned} {zone:?}: {:?}", out.actions); + } else { + assert!(!out.actions.is_empty(), "column edges still work: {owned} {zone:?}"); + assert!(!group_drop, "column edges must not become group bands"); + } + } + } +} + +#[test] +fn owned_minimized_groups_skip_the_trailing_stock_target() { + for owned in [false, true] { + let ctx = Context::default(); + let area = Id::unique("owned-dock-targets"); + let mut layout = Layout { + root: Some(Node::Dock { side: Side::Start, columns: vec![Column::new(vec![Group::new(vec![1, 2], 0)], 240.0)] }), + floating: vec![Floating::single(vec![99], 0, [500.0, 100.0, 200.0, 150.0])], + }; + if let Some(Node::Dock { columns, .. }) = &mut layout.root { + columns[0].groups[0].minimized = true; + } + let mut host = Host { exclusive: owned, stock_group: Some(99), ..Default::default() }; + let mut step = |events, start: bool| { + let mut out = DockOutput::default(); + ctx.run_ui(input(events), |ui| { + let mut style = DockStyle::from_ui(ui); + super::floating_tests::frame_style(&mut style); + out = DockArea::new(area).show_content(ui, &layout, &style, &mut host); + if start { + assert!(begin_drag(ui.ctx(), area, 99, Rect::from_min_size(pos2(500.0, 100.0), vec2(200.0, 150.0)))); + } + }) + .drop_without_applying_deltas(); + (out, host.groups.get(&1).copied()) + }; + step(vec![], false); + step(vec![], false); + let (_, target) = step(vec![], false); + let target = target.expect("the docked group is drawn"); + let from = pos2(550.0, 140.0); + step(vec![Event::PointerMoved(from), button(from, true)], true); + let to = pos2(target.center().x, target.bottom() + 60.0); + step(vec![Event::PointerMoved(to)], false); + let (out, _) = step(vec![Event::PointerMoved(to), button(to, false)], false); + let group_drop = out.actions.iter().any(|a| { + matches!( + a, + Action::MovePanel { to: Target::At { placement: Placement::Group { .. }, .. }, .. } + | Action::MoveGroup { to: Target::At { placement: Placement::Group { .. }, .. }, .. } + ) + }); + assert_eq!(group_drop, !owned, "owned={owned}: {:?}", out.actions); + } +} + +#[test] +fn disabled_edge_axes_ignore_pointer_drags_and_enabled_edges_obey_limits() { + for axes in [[true, true], [true, false], [false, true], [false, false]] { + for horizontal in [true, false] { + for grow in [true, false] { + let ctx = Context::default(); + let rect = Rect::from_min_size(pos2(400.0, 300.0), vec2(300.0, 200.0)); + let bounds = Rect::from_min_max(pos2(0.0, 0.0), pos2(720.0, 520.0)); + let frame = |events| { + let mut resized = None; + ctx.run_ui(input(events), |ui| { + let style = chrome::ChromeStyle::from_ui(ui); + resized = chrome::FloatingFrame::new(Id::unique("edge-axis"), rect) + .resize_axes(axes) + .bounds(bounds) + .limits(PanelLimits { min: vec2(160.0, 120.0), max: vec2(310.0, 210.0) }) + .show(ui.ctx(), &style, |_, _| ()) + .resized; + }) + .drop_without_applying_deltas(); + resized + }; + frame(vec![]); + frame(vec![]); + let from = if horizontal { pos2(rect.right() - 2.0, rect.center().y) } else { pos2(rect.center().x, rect.bottom() - 2.0) }; + frame(vec![Event::PointerMoved(from), button(from, true)]); + let delta = if horizontal { vec2(10.0, 0.0) } else { vec2(0.0, 10.0) }; + frame(vec![Event::PointerMoved(from + delta)]); + let result = frame(vec![Event::PointerMoved(from + delta * if grow { 100.0 } else { -100.0 })]); + let enabled = if horizontal { axes[0] } else { axes[1] }; + let expected = if horizontal { vec2(if grow { 310.0 } else { 160.0 }, 200.0) } else { vec2(300.0, if grow { 210.0 } else { 120.0 }) }; + assert_eq!(result, enabled.then(|| Rect::from_min_size(rect.min, expected)), "{axes:?} horizontal={horizontal} grow={grow}"); + } + } + } +} + +#[test] +fn frame_at_viewport_edge_cannot_resize_beyond_it() { + let ctx = Context::default(); + let rect = Rect::from_min_max(pos2(400.0, 300.0), pos2(800.0, 600.0)); + let frame = |events| { + let mut result = None; + ctx.run_ui(input(events), |ui| { + let style = chrome::ChromeStyle::from_ui(ui); + let response = chrome::FloatingFrame::new(Id::unique("viewport-edge"), rect).show(ui.ctx(), &style, |_, _| ()); + assert_eq!(response.rect, rect); + result = response.resized; + }) + .drop_without_applying_deltas(); + result + }; + frame(vec![]); + frame(vec![]); + let from = rect.max - vec2(2.0, 2.0); + frame(vec![Event::PointerMoved(from), button(from, true)]); + frame(vec![Event::PointerMoved(from + vec2(10.0, 10.0))]); + assert_eq!(frame(vec![Event::PointerMoved(from + vec2(100.0, 100.0))]), None); +} diff --git a/crates/ui/src/docking/mod.rs b/crates/ui/src/docking/mod.rs index b7985b82..df3349c2 100644 --- a/crates/ui/src/docking/mod.rs +++ b/crates/ui/src/docking/mod.rs @@ -53,6 +53,8 @@ mod floating_tests; #[cfg(test)] mod group_tests; #[cfg(test)] +mod host_hooks_tests; +#[cfg(test)] mod iconic_tests; #[cfg(all(test, feature = "native-windows"))] mod ime_tests; diff --git a/crates/ui/src/docking/render/dock.rs b/crates/ui/src/docking/render/dock.rs index 878d909e..7805ff0d 100644 --- a/crates/ui/src/docking/render/dock.rs +++ b/crates/ui/src/docking/render/dock.rs @@ -77,7 +77,7 @@ pub(in crate::docking) fn dock_rects

( style: &DockStyle, content: &dyn crate::docking::DockContent

, ) -> (Rect, Rect, f32) { - let gap = style.gap; + let gap = style.dock_gap.unwrap_or(style.gap); let room = (rect.width() - gap - super::node_limits(other, style, content).min.x).clamp(0.0, rect.width()); let width = columns.iter().map(|c| column_width(c, style)).sum::().clamp(0.0, room); let (dock, rest) = match edge_of(side, style.right_to_left) { @@ -266,10 +266,12 @@ impl Renderer<'_, P> { _ => return false, }; let (dock, rest, room) = dock_rects(side, columns, other, rect, self.style, &*self.content); - self.dock(ui, side, columns, dock, room); path.push(dock_first); self.node(ui, other, rest, path, root); path.pop(); + // The divider's hit margin reaches into the sibling (often the canvas). Register + // the dock after that sibling so a seam press belongs to the resize handle. + self.dock(ui, side, columns, dock, room); true } @@ -328,6 +330,9 @@ impl Renderer<'_, P> { } // Registered after the columns, so the narrow hit areas win at the edges. for (index, column, first, width, inner) in drawn { + if column.iconic && self.content.owns_iconic_column(first) { + continue; + } if width < 1.0 { continue; } @@ -412,7 +417,7 @@ impl Renderer<'_, P> { } fn handle_style(&self) -> ResizeHandleStyle { - ResizeHandleStyle { hovered: self.style.accent, dragged: self.style.accent, focused: self.style.accent } + self.content.resize_handle_style(ResizeHandleStyle { hovered: self.style.accent, dragged: self.style.accent, focused: self.style.accent }) } /// Width bounds of an expanded column from its panels' limits. @@ -452,7 +457,15 @@ impl Renderer<'_, P> { if !usable(rect) { return; } + let side = side_at(edge, self.style.right_to_left); + self.content.column_rect(ui.ctx(), column, rect, side); self.record_column_float(column.groups.iter().flat_map(|g| g.panels.iter().cloned()).collect(), rect); + if column.iconic && self.content.owns_iconic_column(first) { + let mut child = ui.new_child(UiBuilder::new().id_salt(("host-iconic", first)).max_rect(rect)); + child.set_clip_rect(rect.intersect(ui.clip_rect())); + self.content.iconic_column(&mut child, column, rect, side); + return; + } let (bar, body, heights) = self.column_layout(column, rect); self.column_bar(ui, column, first, bar, rect, edge); if column.iconic && usable(body) { @@ -496,9 +509,12 @@ impl Renderer<'_, P> { /// seams and bands clipped to the visible `view`, and the height dividers. fn column_groups(&mut self, ui: &mut Ui, column: &Column

, heights: &[f32], area: Rect, view: Rect) { let rects = stacked(area, heights); - for (group, rect) in column.groups.iter().zip(&rects) { - self.group(ui, &group.panels, group.active, group.minimized, *rect, GroupKind::Dock); - } + let owned: Vec<_> = column + .groups + .iter() + .zip(&rects) + .map(|(group, rect)| self.group(ui, &group.panels, group.active, group.minimized, *rect, GroupKind::Dock)) + .collect(); let top = rects.last().map_or(area.top(), |r| r.bottom()); let xs = view.x_range(); let half = self.style.seam_band * 0.5; @@ -512,7 +528,7 @@ impl Renderer<'_, P> { } }; let mut previous: Option<&P> = None; - for (group, rect) in column.groups.iter().zip(&rects) { + for ((group, rect), owned) in column.groups.iter().zip(&rects).zip(&owned) { let Some(anchor) = group.panels.first() else { continue }; if let Some(above) = previous { let band = Rect::from_x_y_ranges(xs, (rect.top() - half)..=(rect.top() + half)); @@ -528,13 +544,14 @@ impl Renderer<'_, P> { self.dock_groups.push(DockGroup { body, center, - sides: [None, None, Some((above, line(rect.top()))), Some((below, line(rect.bottom())))], + sides: [None, None, (!*owned).then(|| (above, line(rect.top()))), (!*owned).then(|| (below, line(rect.bottom())))], }); } previous = Some(anchor); } // The room under the last group, when every group is minimized, stacks below it. if let Some(last) = previous + && !owned.last().copied().unwrap_or(false) && top < view.bottom() { let band = Rect::from_x_y_ranges(xs, top..=view.bottom()); diff --git a/crates/ui/src/docking/render/floating.rs b/crates/ui/src/docking/render/floating.rs index 256b128f..3164a462 100644 --- a/crates/ui/src/docking/render/floating.rs +++ b/crates/ui/src/docking/render/floating.rs @@ -54,7 +54,7 @@ impl Renderer<'_, P> { /// The title bar and, for a movable (resizable) window, the resize band above it. fn window_chrome(&self, window: &Floating

) -> f32 { - let band = if window.panels().all(|panel| self.permissions(panel).movable) { RESIZE_BAND } else { 0.0 }; + let band = if self.style.title_height > 0.0 && window.panels().all(|panel| self.permissions(panel).movable) { RESIZE_BAND } else { 0.0 }; band + self.style.title_height } @@ -81,12 +81,15 @@ impl Renderer<'_, P> { let style = ChromeStyle::from_dock(self.style); let resize_label = self.style.resize_window_label.clone(); let opacity = self.drag_opacity(ctx, first); - let response = - FloatingFrame::new(id, rect).bounds(screen).limits(limits).resizable(movable).resize_label(&resize_label).opacity(opacity).show( - ctx, - &style, - |ui, contents| self.window_contents(ui, window, contents, rect, &style), - ); + let response = FloatingFrame::new(id, rect) + .bounds(screen) + .limits(limits) + .reserve_top_resize_band(self.style.title_height > 0.0) + .resizable(movable) + .resize_axes([!self.content.owns_floating_width(first), true]) + .resize_label(&resize_label) + .opacity(opacity) + .show(ctx, &style, |ui, contents| self.window_contents(ui, window, contents, rect, &style)); for handle in &response.handles { self.register_interaction(ctx, handle.id); } @@ -132,7 +135,9 @@ impl Renderer<'_, P> { pub(super) fn floating_body(&mut self, ui: &mut Ui, window: &Floating

, body: Rect, frame: Rect) { let Some(key) = window.panels().next().cloned() else { return }; match (&window.root, window.minimized) { - (Node::Tabs { panels, active }, true) => self.group(ui, panels, *active, true, body, GroupKind::Floating { window: frame }), + (Node::Tabs { panels, active }, true) => { + self.group(ui, panels, *active, true, body, GroupKind::Floating { window: frame }); + } (_, true) => {} (root, false) => { let outer = self.tree.replace((key, frame)); diff --git a/crates/ui/src/docking/render/group.rs b/crates/ui/src/docking/render/group.rs index e4e04cda..7259b534 100644 --- a/crates/ui/src/docking/render/group.rs +++ b/crates/ui/src/docking/render/group.rs @@ -36,7 +36,7 @@ const OVERFLOW_WIDTH: f32 = 22.0; const WHEEL_NOTCH: f32 = 40.0; impl Renderer<'_, P> { - /// Draw one tab group at `rect`. + /// Draw one tab group at `rect`, returning whether the host took exclusive ownership. /// /// An `active` index outside `panels` is a no-op. A lone protected panel is drawn stripless: /// its whole rect is the body and its only tab has `Rect::NOTHING`. Side splits are offered @@ -45,14 +45,23 @@ impl Renderer<'_, P> { /// hidden overflow tabs still appear in the drop target with `Rect::NOTHING`. The host's extra /// tabs follow the panel tabs and fit with them ([`fit_with_extra`]). `minimized` hides the body /// but keeps the strip. Tabs, strip, header and drag bar are mirrored in right-to-left layout. - pub(super) fn group(&mut self, ui: &mut Ui, panels: &[P], active: usize, minimized: bool, rect: Rect, kind: GroupKind) { - let Some(selected) = panels.get(active) else { return }; - self.record_float(panels, rect.size(), rect.size()); + pub(super) fn group(&mut self, ui: &mut Ui, panels: &[P], active: usize, minimized: bool, rect: Rect, kind: GroupKind) -> bool { + let Some(selected) = panels.get(active) else { return false }; + let host_collapsed = self.content.group_collapsed(panels); + let minimized = minimized || host_collapsed; + if !host_collapsed { + self.record_float(panels, rect.size(), rect.size()); + } + if self.content.custom_group(ui, panels, selected, rect, matches!(kind, GroupKind::Floating { .. })) { + return true; + } let policy = self.permissions(selected); let splits = matches!(kind, GroupKind::Frame | GroupKind::Floating { .. }) && policy.accepts_splits; let rtl = self.style.right_to_left; if stripless(panels, &*self.content) { - self.contents(ui, selected, rect); + if !minimized { + self.contents(ui, selected, rect); + } self.targets.push(DropTarget { anchor: selected.clone(), rect, @@ -62,7 +71,7 @@ impl Renderer<'_, P> { accepts_tabs: policy.accepts_tabs, right_to_left: rtl, }); - return; + return false; } let movable = panels.iter().all(|p| self.permissions(p).movable); let strip = Rect::from_min_max(rect.min, pos2(rect.right(), (rect.top() + self.style.tab_height).min(rect.bottom()))); @@ -166,6 +175,7 @@ impl Renderer<'_, P> { self.contents(ui, selected, Rect::from_min_max(pos2(rect.left(), strip.bottom()), rect.max)); } ui.painter().rect_stroke(rect, 0.0, self.style.border, StrokeKind::Inside); + false } /// A part of the strip the host draws (`header` or `extra-tabs`) at `rect`. A press in it @@ -201,7 +211,21 @@ impl Renderer<'_, P> { if self.minimizes(owner) && response.double_clicked() { self.output.actions.push(Action::SetMinimized { panel: owner.selected.clone(), minimized: !owner.minimized }); } - let active = response.has_focus() || response.hovered(); + let modality = self.id.with("keyboard-chrome-focus"); + let keyboard_input = ui.input(|i| i.events.iter().any(|e| matches!(e, egui::Event::Key { pressed: true, .. }))); + let pointer_input = ui.input(|i| i.pointer.any_pressed()); + let keyboard_focus = ui.ctx().data_mut(|d| { + let on = if pointer_input { + false + } else if keyboard_input { + true + } else { + d.get_temp::(modality).unwrap_or(false) + }; + d.insert_temp(modality, on); + on + }); + let active = response.has_focus() && keyboard_focus; if active { ui.painter().with_clip_rect(hit).rect_stroke( hit.shrink(1.0), @@ -392,17 +416,22 @@ impl Renderer<'_, P> { let minimized = owner.filter(|owner| self.minimizes(owner)).map(|owner| owner.minimized); let policy = self.permissions(panel); let label = self.label_of(panel); - let color = if active { self.style.text } else { self.style.inactive_text }; + let visual_active = active && !owner.is_some_and(|owner| self.content.group_collapsed(owner.panels)); + let color = if visual_active { self.style.text } else { self.style.inactive_text }; let galley = self.tab_label(ui, panel, &label, color, (rect.width() - TAB_PADDING).max(0.0)); let elided = galley.elided; let tab_id = self.id.with(("tab", panel)); let pointer_focus = self.pointer_focus(ui, tab_id); - let response = tabs::Tab::new(tab_id, &label, active) + let response = tabs::Tab::new(tab_id, &label, visual_active) .sense(if policy.movable { Sense::click_and_drag() } else { Sense::click() }) .focus_stroke(if pointer_focus { Stroke::NONE } else { Stroke::new(1.0, self.style.accent) }) .show_at(ui, rect, |ui, response| { let painter = ui.painter().with_clip_rect(rect.intersect(ui.clip_rect())); - painter.rect_filled(rect, 0.0, if active || response.hovered() { self.style.active_background } else { self.style.tab_background }); + painter.rect_filled( + rect, + 0.0, + if visual_active || response.hovered() { self.style.active_background } else { self.style.tab_background }, + ); let at = pos2(rect.left() + TAB_PADDING * 0.5, rect.center().y - galley.size().y * 0.5); painter.galley(at, galley.clone(), color); }); diff --git a/crates/ui/src/docking/render/mod.rs b/crates/ui/src/docking/render/mod.rs index 751e17b4..256edccd 100644 --- a/crates/ui/src/docking/render/mod.rs +++ b/crates/ui/src/docking/render/mod.rs @@ -212,13 +212,23 @@ impl Renderer<'_, P> { // The divider's inner-edge hit margin must win over child scrollbars, just // as accordion dividers win over the adjacent panel's content. // Named by its layout path, which `ResizeSplit` takes and hits report. + self.content.split_separator(ui, split.divider); + let host_width = + *axis == SplitAxis::Horizontal && self.tree.as_ref().is_some_and(|(panel, _)| self.content.owns_floating_split_width(panel)); + if host_width { + return; + } let layout_path = self.original_path(path); let response = ResizeHandle::new( self.split_id(&layout_path), *axis, split.divider, &self.style.resize_label, - ResizeHandleStyle { hovered: self.style.accent, dragged: self.style.accent, focused: self.style.accent }, + self.content.resize_handle_style(ResizeHandleStyle { + hovered: self.style.accent, + dragged: self.style.accent, + focused: self.style.accent, + }), ) .value(split.first_size(), (split.first_size() + lo)..=(split.first_size() + hi)) .show(ui); @@ -490,11 +500,15 @@ pub(in crate::docking) fn node_limits

(node: &Node

, style: &DockStyle, cont Node::Split { axis, first, second, .. } => { let a = node_limits(first, style, content); let b = node_limits(second, style, content); + let gap = if matches!(first.as_ref(), Node::Dock { .. }) || matches!(second.as_ref(), Node::Dock { .. }) { + style.dock_gap.unwrap_or(style.gap) + } else { + style.gap + }; let limits = match axis { - SplitAxis::Horizontal => PanelLimits { - min: vec2(a.min.x + b.min.x + style.gap, a.min.y.max(b.min.y)), - max: vec2(a.max.x + b.max.x + style.gap, a.max.y.min(b.max.y)), - }, + SplitAxis::Horizontal => { + PanelLimits { min: vec2(a.min.x + b.min.x + gap, a.min.y.max(b.min.y)), max: vec2(a.max.x + b.max.x + gap, a.max.y.min(b.max.y)) } + } SplitAxis::Vertical => PanelLimits { min: vec2(a.min.x.max(b.min.x), a.min.y + b.min.y + style.gap), max: vec2(a.max.x.min(b.max.x), a.max.y + b.max.y + style.gap), diff --git a/crates/ui/src/docking/surface.rs b/crates/ui/src/docking/surface.rs index 8d2a1444..2dc5b7e9 100644 --- a/crates/ui/src/docking/surface.rs +++ b/crates/ui/src/docking/surface.rs @@ -13,6 +13,8 @@ pub(super) const TAB_PADDING: f32 = 24.0; pub struct DockStyle { pub tab_height: f32, pub gap: f32, + /// Optional gap beside attached side docks. Other split/row dividers retain `gap`. + pub dock_gap: Option, pub min_pane: f32, pub background: Color32, pub tab_background: Color32, @@ -175,6 +177,7 @@ impl DockStyle { Self { tab_height: 26.0, gap: 4.0, + dock_gap: None, min_pane: 40.0, background: visuals.panel_fill, tab_background: visuals.extreme_bg_color, @@ -304,6 +307,7 @@ impl DockStyle { && self.font.size > 0.0 && self.drop_line_width > 0.0 && self.side_zone_fraction <= 0.5 + && self.dock_gap.is_none_or(|gap| gap.is_finite() && (0.0..=4096.0).contains(&gap)) } } @@ -832,6 +836,45 @@ pub fn send_keys(ctx: &Context, viewport: egui::ViewportId, events: Vec { fn body(&mut self, ui: &mut Ui, panel: &P); + + /// Observe the fitted rectangle of each attached column, before its chrome is drawn. + fn column_rect(&mut self, _ctx: &Context, _column: &super::Column

, _rect: Rect, _side: super::Side) {} + + /// An exclusive host surface replaces stock iconic painting, focus, drag and resize. + /// Placement and column-edge drop targets still belong to the workspace renderer. + fn owns_iconic_column(&self, _first: &P) -> bool { + false + } + + /// Draw the complete iconic column inside its fitted rectangle when owned by the host. + fn iconic_column(&mut self, _ui: &mut Ui, _column: &super::Column

, _rect: Rect, _side: super::Side) {} + + /// Return true after painting an exclusive group surface; stock tabs/body/targets are skipped. + /// The host supplies that group's size limits through `limits`. + fn custom_group(&mut self, _ui: &mut Ui, _panels: &[P], _active: &P, _rect: Rect, _floating: bool) -> bool { + false + } + + /// Hosts which paint shared seams can keep the stock resize input without a second fill. + fn resize_handle_style(&self, default: crate::layout::ResizeHandleStyle) -> crate::layout::ResizeHandleStyle { + default + } + /// Presentation-only collapse of a group; its selected tab remains model state. + fn group_collapsed(&self, _panels: &[P]) -> bool { + false + } + /// Paint a fitted split gap in the owning surface, without another resize target. + fn split_separator(&mut self, _ui: &mut Ui, _rect: Rect) {} + /// The host supplies horizontal resize input at the floating frame edges. + /// Internal split dividers retain stock input unless `owns_floating_split_width` is true. + fn owns_floating_width(&self, _panel: &P) -> bool { + false + } + /// The host supplies horizontal resize input for every internal split in this floating tree. + /// This is separate from frame-edge ownership; only return true with replacement input. + fn owns_floating_split_width(&self, _panel: &P) -> bool { + false + } /// Plain text name for accessibility, menus and the default visual label. fn logical_label(&self, panel: &P) -> String where @@ -968,6 +1011,33 @@ impl + ?Sized> DockContent

for Overrides<'_, P, C> { fn body(&mut self, ui: &mut Ui, panel: &P) { self.inner.body(ui, panel); } + fn column_rect(&mut self, ctx: &Context, column: &super::Column

, rect: Rect, side: super::Side) { + self.inner.column_rect(ctx, column, rect, side); + } + fn owns_iconic_column(&self, first: &P) -> bool { + self.inner.owns_iconic_column(first) + } + fn iconic_column(&mut self, ui: &mut Ui, column: &super::Column

, rect: Rect, side: super::Side) { + self.inner.iconic_column(ui, column, rect, side); + } + fn custom_group(&mut self, ui: &mut Ui, panels: &[P], active: &P, rect: Rect, floating: bool) -> bool { + self.inner.custom_group(ui, panels, active, rect, floating) + } + fn resize_handle_style(&self, default: crate::layout::ResizeHandleStyle) -> crate::layout::ResizeHandleStyle { + self.inner.resize_handle_style(default) + } + fn group_collapsed(&self, panels: &[P]) -> bool { + self.inner.group_collapsed(panels) + } + fn split_separator(&mut self, ui: &mut Ui, rect: Rect) { + self.inner.split_separator(ui, rect); + } + fn owns_floating_width(&self, panel: &P) -> bool { + self.inner.owns_floating_width(panel) + } + fn owns_floating_split_width(&self, panel: &P) -> bool { + self.inner.owns_floating_split_width(panel) + } fn logical_label(&self, panel: &P) -> String { (self.label)(panel) } diff --git a/crates/vectorcraft-format/Cargo.toml b/crates/vectorcraft-format/Cargo.toml new file mode 100644 index 00000000..e85d9088 --- /dev/null +++ b/crates/vectorcraft-format/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "craft-vectorcraft-format" +description = "The envelope of VectorCraft's .vectorcraft file: its members, version, embedded images and the data after the JSON, with the document as a type parameter." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +flate2 = { version = "1", default-features = false, features = ["rust_backend"] } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/vectorcraft-format/README.md b/crates/vectorcraft-format/README.md new file mode 100644 index 00000000..e69a3ac9 --- /dev/null +++ b/crates/vectorcraft-format/README.md @@ -0,0 +1,112 @@ +# craft-vectorcraft-format + +The envelope of VectorCraft's `.vectorcraft` file, for VectorCraft and for the apps that read its +files. The document is a type parameter: VectorCraft supplies its own document type, and a reader +that needs only the envelope uses `serde::de::IgnoredAny`. The crate takes and returns bytes; it +does not open files. No `unsafe`, no panics on malformed input, and it builds for +`wasm32-unknown-unknown`. The dependencies are `serde`, `serde_json`, `thiserror` and `flate2` +(its pure-Rust backend). + +```toml +craft-vectorcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "" } +``` + +## The file + +A `.vectorcraft` file is UTF-8 JSON, gzip-compressed when saved with Use Compression. Since +version 4, entries of `INLINE_MAX` (8 KiB) or more follow the JSON after `BLOB_MAGIC` +(`\n\0VCBLOBS\0`) and give `"blob": [offset, length]` instead of base64 `data`. + +| Member | Type | Written | +|---|---|---| +| `format` | `String` | `"vectorcraft"`, or `"drawcraft"` in versions 1 and 2 | +| `version` | `u32` | 1 to `VERSION` (4) | +| `generator` | `Option` | `"VectorCraft "` | +| `preview` | `Option` | a PNG of the first artboard, at most `PREVIEW_MAX` (256) pixels on its longer side | +| `profiles` | `BTreeMap` | ICC files by profile name, when there are any | +| `pdf` | `Option` | a PDF of every artboard | +| `document` | `D` | the document | +| `images` | `BTreeMap` | the images the document uses, always written | + +`File` holds these members in this order. `serde_json::to_vec` and `to_vec_pretty` of a +`File` give the bytes VectorCraft writes before the data after the JSON. `File::read(json)` +reads one, and `Contents::read(json)` reads `format`, `version`, `document`, `images` and +`profiles` and skips `generator`, `preview` and `pdf` as the JSON is read, so their base64 is not +buffered or kept. Both read the files VectorCraft opens: + +- They return `ReadError::Envelope(Error::Format)` for another format name and + `ReadError::Envelope(Error::TooNew)` for a version newer than `VERSION`, after the JSON reads; + `ReadError::Json` otherwise. +- A repeated `generator`, `preview` or `pdf` is accepted, and `File` keeps the last value, as + `serde_json` reads a repeated key into a `Value`. Any other repeated member is refused. +- A top-level JSON array holds `format`, `version`, `document`, `images` and `profiles`, in that + order. + +`Head` reads only `preview` and `pdf`. `Image` has `mime`, `data` (base64), `blob` and `proxy` +(a linked image's low-resolution preview). `File`, `Contents`, `Head`, `Image`, `Part` and +`ReadError` are `#[non_exhaustive]`: other crates build a `File` with `File::new` and an `Image` +with `Image::inline`, `Tail::image` or `Image::default()`. + +## Use + +```rust +use craft_vectorcraft_format::{Entries, File, split_tail}; + +// Writing +let mut file = File::new(document); +let mut tail = file.tail(); +file.generator = Some(format!("VectorCraft {version}")); +file.pdf = Some(tail.image("application/pdf", &pdf, false)); +let mut out = serde_json::to_vec(&file)?; +tail.write_to(&mut out)?; + +// Reading (decompress first when `is_compressed(&bytes)`) +let (json, after) = split_tail(&bytes); +let file = File::::read(json)?; +let mut entries = Entries::new(after); +let pdf = file.pdf.map(|p| entries.bytes(&p)).transpose()?.flatten(); +``` + +- `sniff(head)` recognizes a file, plain or gzip, from its first 256 bytes after decompression: + they start with `{` and the first `"format"` member is `"vectorcraft"` or `"drawcraft"`, with + JSON escapes in the key and value decoded. +- `is_compressed(bytes)` checks for the gzip magic bytes. +- `split_tail(bytes)` and `json_part(bytes)` find the JSON and the data after it in an + uncompressed or decompressed file. `marker_at(bytes)` gives the offset of `BLOB_MAGIC` when it + follows the JSON as VectorCraft writes it, and works on the first bytes of a file that is being + decompressed. +- `File::check` and `check` return `Error::Format` for another format name and `Error::TooNew` + for a version newer than `VERSION`. +- A `generator`, `preview` or `pdf` of the wrong shape reads as `None` in `File` and `Head`. + `File` and `Head` hold each of these values in memory while checking its shape; `Contents` + skips them. +- `Image::len` gives the size an entry states without decoding it; `Image::bytes(after)` decodes + `data` or copies the `blob` range, and returns `None` for an entry of more than `MAX_ENTRY` + (1 GiB, VectorCraft's limit on one entry of a compressed file). `Image::bytes_at_most` takes + another limit. `entry(after, [offset, length])` returns the range. +- `Entries::new(after)` reads a file's entries: `Entries::bytes` and `Entries::blob` return each + entry's bytes as an `Arc>`, give entries that name the same `[offset, length]` one + shared buffer, and return `Error::TooBig` once the `blob` ranges copied add up to more than + `after` holds, as when many different ranges overlap. `Entries` has no limit on one entry. +- `base64::decode` does not check the unused bits before the padding: `Zh==` decodes like `Zg==`, + as in VectorCraft's own decoder. +- `File::tail()`, or `Tail::new(version)`, places each entry with `Tail::image`: after the JSON + for version 4 and entries of at least `INLINE_MAX` bytes, inline otherwise. `Tail::write_to` + writes the marker and the entries. +- `base64::encode`, `decode` and `decoded_len` handle the `data` members. + +Size limits on the JSON and on decompression stay with each reader. + +## Testing + +`cargo test -p craft-vectorcraft-format` runs `tests/envelope.rs`, which compares files written +through `File` and `Tail` with JSON written out by hand in VectorCraft's layout (compact and +pretty, version 4 with data after the JSON, version 2 under the former name), reads them back, +and checks `split_tail`, `marker_at`, `sniff` on plain, gzip and truncated gzip input, the limits +`Entries` and `Image::bytes` put on the bytes copied, members of the wrong shape or repeated, the +array form, the format and version checks of `File::read` and `Contents::read`, and the base64 +vectors of RFC 4648. The doctests check that the public structs and `ReadError` cannot be built or +matched exhaustively from another crate. + +`.github/workflows/vectorcraft-format.yml` runs fmt, clippy and the tests on Linux and Windows, +and a `wasm32-unknown-unknown` check. diff --git a/crates/vectorcraft-format/src/base64.rs b/crates/vectorcraft-format/src/base64.rs new file mode 100644 index 00000000..2af25a36 --- /dev/null +++ b/crates/vectorcraft-format/src/base64.rs @@ -0,0 +1,130 @@ +//! Standard base64 (RFC 4648 §4, with `=` padding), as VectorCraft writes members in the JSON. +//! The decoder skips ASCII whitespace. + +const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +/// Sextet value of each byte; 64 for bytes outside the alphabet. +const VALUE: [u8; 256] = { + let mut t = [64u8; 256]; + let mut i = 0; + while i < 64 { + t[ALPHABET[i] as usize] = i as u8; + i += 1; + } + t +}; + +fn symbol(sextet: u32) -> char { + ALPHABET.get((sextet & 63) as usize).map_or('A', |&b| char::from(b)) +} + +/// `data` as base64, padded with `=`. +pub fn encode(data: &[u8]) -> String { + let mut out = String::with_capacity(data.len().div_ceil(3) * 4); + for c in data.chunks(3) { + let byte = |i: usize| u32::from(c.get(i).copied().unwrap_or(0)); + let n = byte(0) << 16 | byte(1) << 8 | byte(2); + out.push(symbol(n >> 18)); + out.push(symbol(n >> 12)); + out.push(if c.len() > 1 { symbol(n >> 6) } else { '=' }); + out.push(if c.len() > 2 { symbol(n) } else { '=' }); + } + out +} + +/// The number of bytes `s` decodes to when it is valid: 3 for every 4 characters other than +/// whitespace, less one for each trailing `=` (at most 2). +pub fn decoded_len(s: &str) -> u64 { + let clean = s.bytes().filter(|b| !b.is_ascii_whitespace()); + let n = clean.clone().count() as u64; + let pad = clean.rev().take(2).take_while(|&b| b == b'=').count() as u64; + (n / 4 * 3).saturating_sub(pad) +} + +/// The bytes `s` encodes; `None` when it is not valid padded base64. Padding is allowed only at +/// the end. The unused bits before the padding are not checked, so a non-canonical group decodes +/// like its canonical form (`Zh==` gives the bytes of `Zg==`, and `Zm9=` those of `Zm8=`), as +/// VectorCraft's own decoder reads them. +pub fn decode(s: &str) -> Option> { + let clean = s.bytes().filter(|b| !b.is_ascii_whitespace()); + let n = clean.clone().count(); + if !n.is_multiple_of(4) { + return None; + } + let mut out = Vec::with_capacity(n / 4 * 3); + let (mut group, mut len, mut pad, mut ended) = (0u32, 0usize, 0usize, false); + for b in clean { + if ended { + return None; + } + group <<= 6; + if b == b'=' { + pad += 1; + } else { + let v = *VALUE.get(usize::from(b))?; + if v == 64 || pad > 0 { + return None; + } + group |= u32::from(v); + } + len += 1; + if len == 4 { + if pad > 2 { + return None; + } + let [_, a, b, c] = group.to_be_bytes(); + out.push(a); + if pad < 2 { + out.push(b); + } + if pad < 1 { + out.push(c); + } + ended = pad > 0; + (group, len) = (0, 0); + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_rfc_vectors_encode_and_decode() { + for (dec, enc) in + [("", ""), ("f", "Zg=="), ("fo", "Zm8="), ("foo", "Zm9v"), ("foob", "Zm9vYg=="), ("fooba", "Zm9vYmE="), ("foobar", "Zm9vYmFy")] + { + assert_eq!(encode(dec.as_bytes()), enc); + assert_eq!(decode(enc).as_deref(), Some(dec.as_bytes()), "{enc}"); + assert_eq!(decoded_len(enc), dec.len() as u64, "{enc}"); + } + assert_eq!(decode("Zm9v\nYmFy").as_deref(), Some(&b"foobar"[..])); + } + + #[test] + fn every_byte_value_round_trips() { + let all: Vec = (0..=255).collect(); + for n in 0..all.len() { + assert_eq!(decode(&encode(&all[..n])).as_deref(), Some(&all[..n])); + } + } + + #[test] + fn nonzero_bits_before_the_padding_are_ignored() { + for (loose, canonical) in [("Zh==", "Zg=="), ("Zm9=", "Zm8=")] { + assert_eq!(decode(loose), decode(canonical), "{loose}"); + assert_eq!(decoded_len(loose), decoded_len(canonical), "{loose}"); + } + assert_eq!(decode("Zh==").as_deref(), Some(&b"f"[..])); + assert_eq!(decode("Zm9=").as_deref(), Some(&b"fo"[..])); + } + + #[test] + fn misplaced_padding_and_foreign_characters_are_refused() { + for s in ["====", "A===", "AA=A", "=AAA", "QQ==QUJD", "QUI=QUJD", "QUJ", "QU*D", "QUJDé"] { + assert_eq!(decode(s), None, "{s}"); + } + } +} diff --git a/crates/vectorcraft-format/src/lib.rs b/crates/vectorcraft-format/src/lib.rs new file mode 100644 index 00000000..8b4892f2 --- /dev/null +++ b/crates/vectorcraft-format/src/lib.rs @@ -0,0 +1,708 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +//! The envelope of VectorCraft's `.vectorcraft` file, shared by VectorCraft and by the apps that +//! read its files. The document is a type parameter: VectorCraft supplies its own document type, +//! and a reader that needs only the envelope uses [`serde::de::IgnoredAny`]. +//! +//! A `.vectorcraft` file is UTF-8 JSON, gzip-compressed when saved with Use Compression: +//! +//! ```text +//! { "format": "vectorcraft", "version": 4, "generator": "VectorCraft 0.17.0", +//! "preview": { "mime": "image/png", "data": "" }, +//! "profiles": { "": { "mime": "application/vnd.iccprofile", "blob": [0, 9000] } }, +//! "pdf": { "mime": "application/pdf", "blob": [9000, 52000] }, +//! "document": { ... }, +//! "images": { "": { "mime": "image/png", "data": "" } } } +//! ``` +//! +//! Since version 4, entries of [`INLINE_MAX`] bytes or more follow the JSON after [`BLOB_MAGIC`], +//! and such an entry gives `"blob": [offset, length]` (from the end of the marker) instead of +//! `data`. In a compressed file, the marker and the data after it are inside the gzip stream. +//! Files written under the former name say `"format": "drawcraft"`. +//! +//! - [`File`], [`Head`] and [`Image`] are the members, in the order VectorCraft writes them. +//! [`File::read`] reads all of them, and [`Contents::read`] reads a file's document, images and +//! profiles and skips the preview and PDF. Both refuse another format and a newer version. +//! - [`split_tail`] separates a file's JSON from the data after it, and [`marker_at`] finds the +//! marker in a file's first bytes; [`Entries`] reads a file's entries from either place, copying +//! at most the size of that data in all; [`Tail`] places entries when writing. +//! - [`sniff`] and [`is_compressed`] recognize a file from its first bytes. +//! - [`base64`] encodes and decodes the `data` members. +//! +//! The crate takes and returns bytes. It does not open files or decompress past the first bytes +//! [`sniff`] reads: a reader of a compressed file decompresses it with its own limits first. + +pub mod base64; + +use std::collections::{BTreeMap, HashMap}; +use std::fmt; +use std::io::{self, Read, Write}; +use std::marker::PhantomData; +use std::sync::Arc; + +use serde::de::value::SeqAccessDeserializer; +use serde::de::{self, IgnoredAny, MapAccess, SeqAccess, Visitor}; +use serde::{Deserialize, Serialize}; + +/// The newest version: data after the JSON. Versions 1 to 3 still load. +pub const VERSION: u32 = 4; +/// The oldest version VectorCraft writes. +pub const MIN_VERSION: u32 = 1; +/// The first version whose readers open compressed files. +pub const COMPRESSED_SINCE: u32 = 3; +/// The first version that keeps entries after the JSON. +pub const TAIL_SINCE: u32 = 4; +/// Entries this large or larger go after the JSON (from [`TAIL_SINCE`]), not into it as base64. +pub const INLINE_MAX: usize = 8 << 10; +/// What separates the JSON from the data after it. +pub const BLOB_MAGIC: &[u8] = b"\n\0VCBLOBS\0"; +/// The most pixels on the longer side of the embedded `preview`. +pub const PREVIEW_MAX: u32 = 256; +/// The file extension, and the `format` name since version 3. +pub const EXTENSION: &str = "vectorcraft"; +/// The extension and `format` name from before the rename (DrawCraft), still read; VectorCraft +/// writes it for versions 1 and 2. +pub const LEGACY_EXTENSION: &str = "drawcraft"; +/// The MIME type of embedded ICC profiles. +pub const ICC_MIME: &str = "application/vnd.iccprofile"; +/// The most bytes [`Image::bytes`] gives for one entry: VectorCraft's limit on one entry of the +/// data after the JSON of a compressed file (1 GiB). +pub const MAX_ENTRY: u64 = 1 << 30; + +/// Why a file's envelope is refused. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + #[error("not a VectorCraft file: format is `{0}`")] + Format(String), + #[error("file version {0} is newer than this reader supports ({VERSION})")] + TooNew(u32), + #[error("the entries name more bytes than the data after the JSON holds")] + TooBig, +} + +/// [`File::read`] or [`Contents::read`] could not read a file's JSON. +/// +/// Other crates match it with a wildcard arm: +/// +/// ```compile_fail +/// fn name(e: craft_vectorcraft_format::ReadError) -> &'static str { +/// match e { +/// craft_vectorcraft_format::ReadError::Json(_) => "json", +/// craft_vectorcraft_format::ReadError::Envelope(_) => "envelope", +/// } +/// } +/// ``` +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum ReadError { + /// The JSON is not a file's top-level object, or the document `D` does not read. + #[error(transparent)] + Json(#[from] serde_json::Error), + /// The JSON reads, but the format is another one ([`Error::Format`]) or the version is newer + /// than [`VERSION`] ([`Error::TooNew`]). + #[error(transparent)] + Envelope(#[from] Error), +} + +/// The top-level object of a file, with the document as `D`. Serialized with `serde_json`, it +/// gives the bytes VectorCraft writes, compact or pretty: the members in this order, `generator`, +/// `preview`, `profiles` and `pdf` left out when absent, `images` always written. [`File::read`] +/// reads one. A reader that does not use `generator`, `preview` and `pdf` reads [`Contents`] +/// instead. +/// +/// Other crates start one with [`File::new`] and set its members: +/// +/// ```compile_fail +/// let _ = craft_vectorcraft_format::File { +/// format: "vectorcraft".into(), version: 4, generator: None, preview: None, +/// profiles: Default::default(), pdf: None, document: 0, images: Default::default(), +/// }; +/// ``` +#[derive(Debug, Clone, PartialEq, Serialize)] +#[non_exhaustive] +pub struct File { + /// [`EXTENSION`], or [`LEGACY_EXTENSION`] in versions 1 and 2. + pub format: String, + pub version: u32, + /// `"VectorCraft "`. + #[serde(skip_serializing_if = "Option::is_none")] + pub generator: Option, + /// A PNG of the first artboard, at most [`PREVIEW_MAX`] pixels on its longer side. + #[serde(skip_serializing_if = "Option::is_none")] + pub preview: Option, + /// ICC files by profile name (Embed ICC Profiles). + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub profiles: BTreeMap, + /// A PDF of every artboard (Create PDF-Compatible File). + #[serde(skip_serializing_if = "Option::is_none")] + pub pdf: Option, + pub document: D, + /// The images the document uses, by key. + pub images: BTreeMap, +} + +impl File { + /// A file of the current [`VERSION`] holding `document`, with no other members. + pub fn new(document: D) -> Self { + Self { + format: EXTENSION.into(), + version: VERSION, + generator: None, + preview: None, + profiles: BTreeMap::new(), + pdf: None, + document, + images: BTreeMap::new(), + } + } + + /// Is `format` a VectorCraft name, and `version` one this crate reads? + pub fn check(&self) -> Result<(), Error> { + check(&self.format, self.version) + } + + /// Written under the former name (`"format": "drawcraft"`). + pub fn is_legacy(&self) -> bool { + self.format == LEGACY_EXTENSION + } + + /// An empty [`Tail`] for this file's `version`. Set `version` before calling it. + pub fn tail<'a>(&self) -> Tail<'a> { + Tail::new(self.version) + } + + /// Reads a file's JSON (from [`split_tail`]), and refuses another format or a version newer + /// than [`VERSION`] ([`ReadError::Envelope`]), as VectorCraft does after reading it. A file + /// reads as VectorCraft opens it: + /// + /// - A `generator`, `preview` or `pdf` of the wrong shape reads as `None`. One that appears + /// more than once reads as its last value, as `serde_json` reads a repeated key into a + /// `serde_json::Value`. + /// - A missing or repeated `format`, `version` or `document`, or a repeated `images` or + /// `profiles`, is refused. Other members are skipped. + /// - A JSON array holds `format`, `version`, `document`, `images` and `profiles` in that + /// order; the last two may be left out. + pub fn read<'de>(json: &'de [u8]) -> Result + where + D: Deserialize<'de>, + { + read(json, true) + } +} + +/// [`File::read`] and [`Contents::read`]; `keep` is false for [`Contents`], which skips +/// `generator`, `preview` and `pdf` without keeping them. +fn read<'de, D: Deserialize<'de>>(json: &'de [u8], keep: bool) -> Result, ReadError> { + let mut de = serde_json::Deserializer::from_slice(json); + let file = serde::Deserializer::deserialize_struct(&mut de, "File", MEMBERS, FileVisitor { keep, document: PhantomData })?; + de.end()?; + file.check()?; + Ok(file) +} + +/// The members of a file, in the order of the array form. +const MEMBERS: &[&str] = &["format", "version", "document", "images", "profiles"]; + +/// The array form of a file, read as VectorCraft reads it. +#[derive(Deserialize)] +#[serde(rename = "File")] +struct Members { + format: String, + version: u32, + document: D, + #[serde(default)] + images: BTreeMap, + #[serde(default)] + profiles: BTreeMap, +} + +/// Reads a [`File`] from a JSON object or array. +struct FileVisitor { + keep: bool, + document: PhantomData, +} + +impl<'de, D: Deserialize<'de>> Visitor<'de> for FileVisitor { + type Value = File; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("struct File") + } + + fn visit_seq>(self, seq: A) -> Result, A::Error> { + let Members { format, version, document, images, profiles } = Members::deserialize(SeqAccessDeserializer::new(seq))?; + Ok(File { format, version, generator: None, preview: None, profiles, pdf: None, document, images }) + } + + fn visit_map>(self, mut map: A) -> Result, A::Error> { + let (mut format, mut version, mut document, mut images, mut profiles) = (None, None, None, None, None); + let (mut generator, mut preview, mut pdf) = (None, None, None); + while let Some(key) = map.next_key::()? { + match key.as_str() { + "format" => once(&mut map, &mut format, "format")?, + "version" => once(&mut map, &mut version, "version")?, + "document" => once(&mut map, &mut document, "document")?, + "images" => once(&mut map, &mut images, "images")?, + "profiles" => once(&mut map, &mut profiles, "profiles")?, + "generator" if self.keep => generator = map.next_value::>()?.0, + "preview" if self.keep => preview = map.next_value::>()?.0, + "pdf" if self.keep => pdf = map.next_value::>()?.0, + _ => { + map.next_value::()?; + } + } + } + Ok(File { + format: format.ok_or_else(|| de::Error::missing_field("format"))?, + version: version.ok_or_else(|| de::Error::missing_field("version"))?, + generator, + preview, + profiles: profiles.unwrap_or_default(), + pdf, + document: document.ok_or_else(|| de::Error::missing_field("document"))?, + images: images.unwrap_or_default(), + }) + } +} + +/// Reads the value of member `name` into `slot`, refusing a second one. +fn once<'de, A: MapAccess<'de>, T: Deserialize<'de>>(map: &mut A, slot: &mut Option, name: &'static str) -> Result<(), A::Error> { + if slot.is_some() { + return Err(de::Error::duplicate_field(name)); + } + *slot = Some(map.next_value()?); + Ok(()) +} + +/// An optional member read as `None` when its value has the wrong shape. +struct Lenient(Option); + +impl<'de, T: Deserialize<'de>> Deserialize<'de> for Lenient { + fn deserialize>(d: De) -> Result { + #[derive(Deserialize)] + #[serde(untagged)] + enum Shape { + Value(T), + Other(IgnoredAny), + } + Ok(Lenient(match Shape::deserialize(d)? { + Shape::Value(v) => Some(v), + Shape::Other(_) => None, + })) + } +} + +/// [`Lenient`] as a `deserialize_with` function. +fn lenient<'de, De: serde::Deserializer<'de>, T: Deserialize<'de>>(d: De) -> Result, De::Error> { + Lenient::deserialize(d).map(|l| l.0) +} + +/// [`File::check`] for a `format` and `version` read on their own. Every version up to +/// [`VERSION`] passes, 0 included, as VectorCraft opens them; [`MIN_VERSION`] limits only the +/// versions VectorCraft writes. +pub fn check(format: &str, version: u32) -> Result<(), Error> { + if format != EXTENSION && format != LEGACY_EXTENSION { + return Err(Error::Format(format.into())); + } + if version > VERSION { + return Err(Error::TooNew(version)); + } + Ok(()) +} + +/// The members of a file that a reader of its document needs: [`File`] without `generator`, +/// `preview` and `pdf`, in the order of the array form. Only [`Contents::read`] makes one: +/// +/// ```compile_fail +/// let _ = craft_vectorcraft_format::Contents { +/// format: "vectorcraft".into(), version: 4, document: 0, +/// images: Default::default(), profiles: Default::default(), +/// }; +/// ``` +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub struct Contents { + pub format: String, + pub version: u32, + pub document: D, + pub images: BTreeMap, + pub profiles: BTreeMap, +} + +impl Contents { + /// Reads a file's JSON as [`File::read`] does, and skips `generator`, `preview` and `pdf` as + /// it reads them, so their data is neither buffered nor kept. Any value of those three is + /// accepted, however many times it appears. + pub fn read<'de>(json: &'de [u8]) -> Result + where + D: Deserialize<'de>, + { + let File { format, version, document, images, profiles, .. } = read(json, false)?; + Ok(Self { format, version, document, images, profiles }) + } + + /// [`File::is_legacy`]. + pub fn is_legacy(&self) -> bool { + self.format == LEGACY_EXTENSION + } +} + +/// The members of a file besides its document, which is skipped without being decoded. A +/// `preview` or `pdf` of the wrong shape reads as `None`, as in [`File`]; a repeated `preview` or +/// `pdf` is refused. +/// +/// ```compile_fail +/// let _ = craft_vectorcraft_format::Head { preview: None, pdf: None }; +/// ``` +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)] +#[non_exhaustive] +pub struct Head { + #[serde(default, deserialize_with = "lenient")] + pub preview: Option, + #[serde(default, deserialize_with = "lenient")] + pub pdf: Option, +} + +/// An entry holding bytes: an image, an ICC profile, the preview or the PDF. Other crates make +/// one with [`Image::inline`], [`Tail::image`] or `Image::default()`: +/// +/// ```compile_fail +/// let _ = craft_vectorcraft_format::Image { mime: String::new(), data: String::new(), blob: None, proxy: false }; +/// ``` +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct Image { + pub mime: String, + /// The bytes as base64, unless they come after the JSON (`blob`). + #[serde(default, skip_serializing_if = "String::is_empty")] + pub data: String, + /// Where the bytes are in the data after the JSON: `[offset, length]`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub blob: Option<[u64; 2]>, + /// `data` is a linked image's low-resolution preview, not the linked file's bytes. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub proxy: bool, +} + +impl Image { + /// An entry with `bytes` in the JSON, as base64. + pub fn inline(mime: &str, bytes: &[u8]) -> Self { + Self { mime: mime.into(), data: base64::encode(bytes), blob: None, proxy: false } + } + + /// The number of bytes the entry states it holds, without decoding it: the `blob` length, or + /// the length `data` decodes to. + pub fn len(&self) -> u64 { + match self.blob { + Some([_, len]) => len, + None => base64::decoded_len(&self.data), + } + } + + /// Whether [`Image::len`] is 0. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// The entry's bytes: `data` decoded, or the range `blob` names in `tail` (the data after the + /// JSON, from [`split_tail`]). `None` when the entry states more than [`MAX_ENTRY`] bytes, + /// when `data` is not valid base64, or when `tail` ends before the range does. Each call + /// copies the range again; read the entries of one file through [`Entries`], which limits the + /// bytes copied in all. + pub fn bytes(&self, tail: &[u8]) -> Option> { + self.bytes_at_most(tail, MAX_ENTRY) + } + + /// [`Image::bytes`] with a limit of `max` bytes instead of [`MAX_ENTRY`]: `None` when + /// [`Image::len`] is more than `max`, checked before anything is decoded or copied. + pub fn bytes_at_most(&self, tail: &[u8], max: u64) -> Option> { + if self.len() > max { + return None; + } + match self.blob { + Some(at) => entry(tail, at).map(<[u8]>::to_vec), + None => base64::decode(&self.data), + } + } +} + +/// The bytes `[offset, length]` names in `tail`; `None` when `tail` ends before them. +pub fn entry(tail: &[u8], [offset, len]: [u64; 2]) -> Option<&[u8]> { + let start = usize::try_from(offset).ok()?; + let end = start.checked_add(usize::try_from(len).ok()?)?; + tail.get(start..end) +} + +/// Reads the entries of one file, copying at most as many bytes from the data after the JSON as +/// that data holds, in all. A file VectorCraft writes names each of those bytes once, so all of +/// its entries read. Entries that name the same `[offset, length]` share one copy, which counts +/// once. Entries whose different ranges overlap copy the shared bytes once each, and are refused +/// once their copies pass that size, as VectorCraft's reader refuses them. Unlike +/// [`Image::bytes`], `Entries` puts no limit on one entry: VectorCraft opens an uncompressed file +/// whose entry is larger than [`MAX_ENTRY`]. +#[derive(Debug, Clone)] +pub struct Entries<'a> { + tail: &'a [u8], + copied: u64, + done: HashMap<[u64; 2], Arc>>, +} + +impl<'a> Entries<'a> { + /// A reader of entries over `tail`, the data after the JSON (from [`split_tail`]). + pub fn new(tail: &'a [u8]) -> Self { + Self { tail, copied: 0, done: HashMap::new() } + } + + /// The entry's bytes: `data` decoded (`None` when it is not valid base64), or + /// [`Entries::blob`] of its range. + pub fn bytes(&mut self, image: &Image) -> Result>>, Error> { + match image.blob { + Some(at) => self.blob(at), + None => Ok(base64::decode(&image.data).map(Arc::new)), + } + } + + /// The bytes `[offset, length]` names in `tail` (`None` when `tail` ends before them). A range + /// this reader has already read gets the same buffer. [`Error::TooBig`] when the bytes this + /// reader has copied from `tail`, these included, would be more than `tail` holds; a refused + /// range is not counted. + pub fn blob(&mut self, at: [u64; 2]) -> Result>>, Error> { + if let Some(bytes) = self.done.get(&at) { + return Ok(Some(Arc::clone(bytes))); + } + let Some(bytes) = entry(self.tail, at) else { + return Ok(None); + }; + let copied = self.copied.saturating_add(bytes.len() as u64); + if copied > self.tail.len() as u64 { + return Err(Error::TooBig); + } + self.copied = copied; + let bytes = Arc::new(bytes.to_vec()); + self.done.insert(at, Arc::clone(&bytes)); + Ok(Some(bytes)) + } + + /// The bytes copied from `tail` so far. + pub fn copied(&self) -> u64 { + self.copied + } +} + +/// Is this gzip data (a file saved with Use Compression)? +pub fn is_compressed(bytes: &[u8]) -> bool { + bytes.starts_with(&[0x1f, 0x8b]) +} + +/// Does `head`, the first bytes of a file (compressed or not), start a `.vectorcraft` file? It +/// must hold a JSON object whose top-level `format` member is `"vectorcraft"` or `"drawcraft"` +/// within the first 256 bytes after decompression. VectorCraft writes `format` first. +pub fn sniff(head: &[u8]) -> bool { + const HEAD: usize = 256; + let mut unpacked = Vec::new(); + let head = if is_compressed(head) { + // A head cut short still decompresses its start. + let _ = flate2::read::GzDecoder::new(head).take(HEAD as u64).read_to_end(&mut unpacked); + unpacked.as_slice() + } else { + head.get(..HEAD).unwrap_or(head) + }; + format_member(head).is_some_and(|f| f == EXTENSION || f == LEGACY_EXTENSION) +} + +/// The string value of the top-level `format` member of the object `text` starts, with keys and +/// values compared as JSON reads them (escapes decoded). `None` when `text` ends before it, or the +/// value is not a string. +fn format_member(text: &[u8]) -> Option { + let mut s = Scan { text, at: 0 }; + s.space(); + s.expect(b'{')?; + loop { + s.space(); + let key = s.decoded()?; + s.space(); + s.expect(b':')?; + s.space(); + if key == "format" { + return s.decoded(); + } + s.skip_value()?; + s.space(); + s.expect(b',')?; + } +} + +/// A position in the first bytes of a JSON text. +struct Scan<'a> { + text: &'a [u8], + at: usize, +} + +impl<'a> Scan<'a> { + fn space(&mut self) { + while self.text.get(self.at).is_some_and(u8::is_ascii_whitespace) { + self.at += 1; + } + } + + fn expect(&mut self, b: u8) -> Option<()> { + (*self.text.get(self.at)? == b).then(|| self.at += 1) + } + + /// The contents of the string that starts here, between its quotes. + fn string(&mut self) -> Option<&'a [u8]> { + self.expect(b'"')?; + let start = self.at; + loop { + match *self.text.get(self.at)? { + b'"' => { + self.at += 1; + return self.text.get(start..self.at - 1); + } + b'\\' => self.at += 2, + _ => self.at += 1, + } + } + } + + /// The string that starts here, as JSON reads it. + fn decoded(&mut self) -> Option { + let start = self.at; + self.string()?; + serde_json::from_slice(self.text.get(start..self.at)?).ok() + } + + /// Move past the value that starts here: a string, an object or array with everything in it, + /// or a number or literal up to the `,` or `}` after it. + fn skip_value(&mut self) -> Option<()> { + let mut depth = 0usize; + loop { + match *self.text.get(self.at)? { + b'"' => { + self.string()?; + if depth == 0 { + return Some(()); + } + } + b'{' | b'[' => { + depth += 1; + self.at += 1; + } + b'}' | b']' if depth == 0 => return Some(()), + b'}' | b']' => { + depth -= 1; + self.at += 1; + if depth == 0 { + return Some(()); + } + } + b',' if depth == 0 => return Some(()), + _ => self.at += 1, + } + } + } +} + +/// The JSON of an uncompressed (or decompressed) file, without the data after it. +pub fn json_part(file: &[u8]) -> &[u8] { + split_tail(file).0 +} + +/// A file's JSON and the data after it, split at [`BLOB_MAGIC`] where it follows the first JSON +/// value. Without the marker there, all of `file` is the JSON and the data after it is empty. +pub fn split_tail(file: &[u8]) -> (&[u8], &[u8]) { + if !file.contains(&0) { + return (file, &[]); + } + if let Some(at) = marker_at(file) { + return (file.get(..at).unwrap_or(file), file.get(at + BLOB_MAGIC.len()..).unwrap_or_default()); + } + let mut values = serde_json::Deserializer::from_slice(file).into_iter::(); + if values.next().is_some_and(|v| v.is_ok()) { + let end = values.byte_offset(); + if let Some(rest) = file.get(end..) + && rest.starts_with(BLOB_MAGIC) + { + return (file.get(..end).unwrap_or(file), rest.get(BLOB_MAGIC.len()..).unwrap_or_default()); + } + } + (file, &[]) +} + +/// Where [`BLOB_MAGIC`] starts in `file` when it is where VectorCraft writes it: right after a +/// JSON object, array or string, and holding the first NUL of `file`, which JSON never contains. +/// `file` may be the first bytes of a file, up to the end of the marker. [`split_tail`] splits +/// there first; a reader that unpacks a compressed file as a stream finds the end of its JSON with +/// it. +pub fn marker_at(file: &[u8]) -> Option { + let at = file.iter().position(|&b| b == 0)?.checked_sub(1)?; + (file.get(at..)?.starts_with(BLOB_MAGIC) && matches!(file.get(at.checked_sub(1)?)?, b'}' | b']' | b'"')).then_some(at) +} + +/// One entry placed after the JSON by [`Tail::image`]. +/// +/// ```compile_fail +/// let _ = craft_vectorcraft_format::Part { mime: String::new(), bytes: &[] }; +/// ``` +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct Part<'a> { + pub mime: String, + pub bytes: &'a [u8], +} + +/// The data after the JSON of a file being written. Entries go after the JSON, each at the +/// offset where the one before ends, when the version keeps them there and they are at least +/// [`INLINE_MAX`] bytes; other entries go into the JSON as base64. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Tail<'a> { + parts: Vec>, + len: u64, + after: bool, +} + +impl<'a> Tail<'a> { + /// An empty tail for a file of `version`, which is the file's own `version`; [`File::tail`] + /// takes it from the file. + pub fn new(version: u32) -> Self { + Self { parts: vec![], len: 0, after: version >= TAIL_SINCE } + } + + /// The entry for `bytes`, placed after the JSON or inline. + pub fn image(&mut self, mime: &str, bytes: &'a [u8], proxy: bool) -> Image { + if self.after && bytes.len() >= INLINE_MAX { + let len = bytes.len() as u64; + let at = self.len; + self.len = self.len.saturating_add(len); + self.parts.push(Part { mime: mime.into(), bytes }); + return Image { mime: mime.into(), data: String::new(), blob: Some([at, len]), proxy }; + } + Image { proxy, ..Image::inline(mime, bytes) } + } + + /// The entries placed after the JSON, in order. + pub fn parts(&self) -> &[Part<'a>] { + &self.parts + } + + /// The bytes of the entries placed after the JSON, together. + pub fn len(&self) -> u64 { + self.len + } + + /// No entry is placed after the JSON. + pub fn is_empty(&self) -> bool { + self.parts.is_empty() + } + + /// Write what follows the JSON: [`BLOB_MAGIC`] and the entries, or nothing when there are none. + pub fn write_to(&self, mut w: impl Write) -> io::Result<()> { + if self.parts.is_empty() { + return Ok(()); + } + w.write_all(BLOB_MAGIC)?; + self.parts.iter().try_for_each(|p| w.write_all(p.bytes)) + } +} diff --git a/crates/vectorcraft-format/tests/envelope.rs b/crates/vectorcraft-format/tests/envelope.rs new file mode 100644 index 00000000..1712a99b --- /dev/null +++ b/crates/vectorcraft-format/tests/envelope.rs @@ -0,0 +1,445 @@ +use std::io::Write; + +use craft_vectorcraft_format::{ + BLOB_MAGIC, Contents, EXTENSION, Entries, Error, File, Head, INLINE_MAX, Image, LEGACY_EXTENSION, MAX_ENTRY, ReadError, Tail, VERSION, base64, + check, entry, is_compressed, json_part, marker_at, sniff, split_tail, +}; +use serde::de::IgnoredAny; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +struct Layer { + name: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +struct Doc { + layers: Vec, +} + +const PNG: &[u8] = &[0x89, b'P', b'N', b'G', b'\r', b'\n', 0x1a, b'\n']; + +fn doc() -> Doc { + Doc { layers: vec![Layer { name: "Layer 1".into() }] } +} + +fn pdf() -> Vec { + let mut pdf = b"%PDF-1.7\n".to_vec(); + pdf.resize(INLINE_MAX, b' '); + pdf +} + +/// A PNG entry at `[offset, length]` in the data after the JSON. +fn blob(at: [u64; 2]) -> Image { + let mut image = Image::default(); + image.mime = "image/png".into(); + image.blob = Some(at); + image +} + +fn gzip(bytes: &[u8]) -> Vec { + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(bytes).unwrap(); + gz.finish().unwrap() +} + +fn written(file: &File, tail: &Tail<'_>, pretty: bool) -> Vec { + let mut out = if pretty { serde_json::to_vec_pretty(file) } else { serde_json::to_vec(file) }.unwrap(); + tail.write_to(&mut out).unwrap(); + out +} + +fn full_file(pdf: &[u8]) -> (File, Tail<'_>) { + let mut tail = Tail::new(VERSION); + let mut file = File::new(doc()); + file.generator = Some("VectorCraft 0.17.0".into()); + file.preview = Some(Image::inline("image/png", PNG)); + file.pdf = Some(tail.image("application/pdf", pdf, false)); + file.images.insert("k".into(), tail.image("image/png", &[1, 2, 3, 250], false)); + (file, tail) +} + +#[test] +fn a_version_4_file_with_members_after_the_json_is_written_as_the_app_writes_it() { + let pdf = pdf(); + let (file, tail) = full_file(&pdf); + let mut expected = br#"{"format":"vectorcraft","version":4,"generator":"VectorCraft 0.17.0","preview":{"mime":"image/png","data":"iVBORw0KGgo="},"pdf":{"mime":"application/pdf","blob":[0,8192]},"document":{"layers":[{"name":"Layer 1"}]},"images":{"k":{"mime":"image/png","data":"AQID+g=="}}}"#.to_vec(); + expected.extend(b"\n\0VCBLOBS\0"); + expected.extend(&pdf); + assert_eq!(written(&file, &tail, false), expected); + assert_eq!(tail.len(), INLINE_MAX as u64); + assert_eq!(tail.parts().len(), 1); +} + +#[test] +fn a_pretty_file_without_optional_members_is_written_as_the_app_writes_it() { + let mut file = File::new(doc()); + file.generator = Some("VectorCraft 0.17.0".into()); + let expected = r#"{ + "format": "vectorcraft", + "version": 4, + "generator": "VectorCraft 0.17.0", + "document": { + "layers": [ + { + "name": "Layer 1" + } + ] + }, + "images": {} +}"#; + assert_eq!(String::from_utf8(written(&file, &Tail::new(VERSION), true)).unwrap(), expected); +} + +#[test] +fn a_version_2_file_keeps_large_entries_inline_under_the_former_name() { + let pdf = pdf(); + let mut tail = Tail::new(2); + let mut file = File::new(doc()); + file.format = LEGACY_EXTENSION.into(); + file.version = 2; + file.profiles.insert("Gray".into(), tail.image(craft_vectorcraft_format::ICC_MIME, &pdf, false)); + let bytes = written(&file, &tail, false); + assert!(tail.is_empty()); + assert!(!bytes.contains(&0)); + let text = String::from_utf8(bytes).unwrap(); + assert!(text.starts_with(r#"{"format":"drawcraft","version":2,"profiles":{"Gray":{"mime":"application/vnd.iccprofile","data":"JVBERi0xLjcK"#)); + let back = File::::read(text.as_bytes()).unwrap(); + assert!(back.is_legacy()); + assert_eq!(back.profiles["Gray"].bytes(&[]).unwrap(), pdf); +} + +#[test] +fn a_written_file_reads_back_with_its_entries() { + let pdf = pdf(); + let (file, tail) = full_file(&pdf); + for bytes in [written(&file, &tail, false), written(&file, &tail, true)] { + let (json, after) = split_tail(&bytes); + assert_eq!(json_part(&bytes), json); + assert_eq!(after, pdf.as_slice()); + let back = File::::read(json).unwrap(); + assert_eq!(back, file); + back.check().unwrap(); + assert!(!back.is_legacy()); + assert_eq!(back.pdf.as_ref().unwrap().bytes(after).unwrap(), pdf); + assert_eq!(back.pdf.as_ref().unwrap().len(), pdf.len() as u64); + assert_eq!(back.preview.as_ref().unwrap().bytes(after).unwrap(), PNG); + assert_eq!(back.images["k"].bytes(after).unwrap(), [1, 2, 3, 250]); + assert_eq!(back.images["k"].len(), 4); + let head: Head = serde_json::from_slice(json).unwrap(); + assert_eq!(head.preview, file.preview); + assert_eq!(head.pdf, file.pdf); + let skipped = File::::read(json).unwrap(); + assert_eq!(skipped.generator.as_deref(), Some("VectorCraft 0.17.0")); + } +} + +#[test] +fn a_linked_image_preview_is_marked_proxy() { + let mut tail = Tail::new(VERSION); + let image = tail.image("image/png", PNG, true); + assert_eq!(serde_json::to_string(&image).unwrap(), r#"{"mime":"image/png","data":"iVBORw0KGgo=","proxy":true}"#); + let back: Image = serde_json::from_str(r#"{"mime":"image/png","data":"iVBORw0KGgo=","proxy":true}"#).unwrap(); + assert_eq!(back, image); +} + +#[test] +fn entries_outside_the_data_or_with_bad_base64_give_none() { + let tail = b"abcd"; + assert_eq!(entry(tail, [1, 3]), Some(&b"bcd"[..])); + assert_eq!(entry(tail, [1, 4]), None); + assert_eq!(entry(tail, [u64::MAX, 1]), None); + assert_eq!(entry(tail, [1, u64::MAX]), None); + assert_eq!(blob([2, 8]).bytes(tail), None); + let mut bad = Image::default(); + bad.data = "QU*D".into(); + assert_eq!(bad.bytes(tail), None); + assert!(Image::default().is_empty()); +} + +#[test] +fn image_bytes_refuses_an_entry_past_the_limit_before_copying_it() { + assert_eq!(MAX_ENTRY, 1 << 30); + let tail = b"abcd"; + let inline = Image::inline("image/png", tail); + for image in [blob([0, 4]), inline.clone()] { + assert_eq!(image.bytes_at_most(tail, 4).as_deref(), Some(&tail[..])); + assert_eq!(image.bytes_at_most(tail, 3), None); + assert_eq!(image.bytes(tail).as_deref(), Some(&tail[..])); + } + // The stated length is checked, so a range past the limit is refused even where the data + // holds it. + assert_eq!(blob([0, MAX_ENTRY + 1]).bytes_at_most(&[0; 64], MAX_ENTRY), None); + assert_eq!(blob([0, 2]).bytes_at_most(tail, 2).as_deref(), Some(&b"ab"[..])); +} + +#[test] +fn entries_copy_at_most_the_data_after_the_json_in_all() { + let tail = vec![7u8; 1 << 20]; + let len = tail.len() as u64; + let whole = blob([0, len]); + // Many entries naming the same range share one copy, which counts once. + let mut entries = Entries::new(&tail); + let first = entries.bytes(&whole).ok().flatten(); + assert_eq!(first.as_deref().map(Vec::as_slice), Some(&tail[..])); + assert!(entries.blob([0, len]).ok().flatten().zip(first.as_ref()).is_some_and(|(a, b)| std::sync::Arc::ptr_eq(&a, b))); + for _ in 0..3 { + let again = entries.bytes(&whole).ok().flatten(); + assert!(again.as_ref().zip(first.as_ref()).is_some_and(|(a, b)| std::sync::Arc::ptr_eq(a, b))); + } + assert_eq!(entries.copied(), len); + // Overlapping ranges that differ copy the shared bytes once each, and pass the data's size. + let from = |at: u64| blob([at, len - at]); + assert!(matches!(entries.bytes(&from(1)), Err(Error::TooBig))); + // A refused entry is not counted, so the entries after it still read. + assert_eq!(entries.copied(), len); + let mut entries = Entries::new(&tail); + let part = |at: u64, n: u64| blob([at, n]); + assert!(entries.bytes(&part(0, len / 2)).is_ok_and(|b| b.is_some())); + assert!(matches!(entries.bytes(&part(1, len - 1)), Err(Error::TooBig))); + assert_eq!(entries.copied(), len / 2); + assert!(entries.bytes(&part(len / 2, len / 2)).is_ok_and(|b| b.is_some())); + assert_eq!(entries.copied(), len); + // Entries that name each byte once read whole, with inline and missing entries beside them. + let mut entries = Entries::new(&tail); + let half = |at: u64| blob([at, len / 2]); + assert_eq!(entries.bytes(&half(0)).unwrap().map(|b| b.len()), Some(tail.len() / 2)); + assert_eq!(entries.bytes(&Image::inline("image/png", PNG)).unwrap().as_deref().map(Vec::as_slice), Some(PNG)); + assert_eq!(entries.bytes(&half(len)), Ok(None)); + assert_eq!(entries.bytes(&half(len / 2)).unwrap().map(|b| b.len()), Some(tail.len() / 2)); + assert_eq!(entries.copied(), len); + assert_eq!(Entries::new(&[]).bytes(&whole), Ok(None)); +} + +#[test] +fn optional_members_of_the_wrong_shape_read_as_none() { + let json = br#"{"format":"vectorcraft","version":4,"generator":7,"preview":"x","pdf":{"blob":"y"},"document":{"layers":[]}}"#; + let file = File::::read(json).unwrap(); + assert_eq!((file.generator, file.preview, file.pdf), (None, None, None)); + let skipped = File::::read(json).unwrap(); + assert!(skipped.preview.is_none()); + let head: Head = serde_json::from_slice(json).unwrap(); + assert_eq!(head, Head::default()); + let good = br#"{"preview":{"mime":"image/png","data":"iVBORw0KGgo="},"pdf":null}"#; + let head: Head = serde_json::from_slice(good).unwrap(); + assert_eq!(head.preview, Some(Image::inline("image/png", PNG))); + assert!(head.pdf.is_none()); +} + +#[test] +fn contents_read_the_document_images_and_profiles_without_the_preview_and_pdf() { + let big = vec![b'%'; 64 << 10]; + let mut file = File::new(doc()); + file.version = 2; + file.generator = Some("VectorCraft 0.17.0".into()); + file.preview = Some(Image::inline("image/png", &big)); + file.pdf = Some(Image::inline("application/pdf", &big)); + file.profiles.insert("Gray".into(), Image::inline(craft_vectorcraft_format::ICC_MIME, &[4, 5])); + file.images.insert("k".into(), Image::inline("image/png", PNG)); + for json in [serde_json::to_vec(&file).unwrap(), serde_json::to_vec_pretty(&file).unwrap()] { + let contents = Contents::::read(&json).unwrap(); + let Contents { format, version, profiles, document, images, .. } = contents.clone(); + assert_eq!((format, version, profiles, document, images), (file.format.clone(), 2, file.profiles.clone(), doc(), file.images.clone())); + assert!(contents.is_legacy() == file.is_legacy()); + } + // Members of the wrong shape are skipped as `File` reads them as `None`; a missing document + // is refused as in `File`. + let odd = br#"{"format":"vectorcraft","version":4,"generator":7,"preview":"x","pdf":{"blob":"y"},"document":{"layers":[]},"z":[1]}"#; + let contents = Contents::::read(odd).unwrap(); + assert!(contents.document.layers.is_empty() && contents.images.is_empty()); + let no_document = br#"{"format":"vectorcraft","version":4}"#; + assert!(File::::read(no_document).is_err()); + assert!(Contents::::read(no_document).is_err()); +} + +#[test] +fn reading_refuses_another_format_and_a_newer_version() { + let newer = br#"{"format":"vectorcraft","version":5,"document":0}"#; + let other = br#"{"format":"other","version":1,"document":0}"#; + for (json, error) in [(&newer[..], Error::TooNew(5)), (other, Error::Format("other".into()))] { + let file = File::::read(json); + assert!(matches!(&file, Err(ReadError::Envelope(e)) if *e == error), "{file:?}"); + let contents = Contents::::read(json); + assert!(matches!(&contents, Err(ReadError::Envelope(e)) if *e == error), "{contents:?}"); + assert_eq!(contents.err().map(|e| e.to_string()), Some(error.to_string())); + } + for version in [0, 1, VERSION] { + let json = format!(r#"{{"format":"drawcraft","version":{version},"document":0}}"#); + assert!(File::::read(json.as_bytes()).is_ok() && Contents::::read(json.as_bytes()).is_ok()); + } + // A document that does not read is a JSON error, as before the version is looked at. + let bad = br#"{"format":"vectorcraft","version":5,"document":7}"#; + assert!(matches!(Contents::::read(bad), Err(ReadError::Json(_)))); + assert!(Contents::::read(b"\"x\"").is_err_and(|e| e.to_string().contains("expected struct File"))); +} + +#[test] +fn file_and_contents_read_a_repeated_generator_preview_or_pdf_and_refuse_other_repeated_members() { + // VectorCraft opens a file whose generator, preview or PDF is repeated, and refuses one whose + // format, version, document, images or profiles are. `File` keeps the last value, as + // serde_json reads a repeated key into a `Value`. + let png = r#"{"mime":"image/png","data":"iVBORw0KGgo="}"#; + let pdf = r#"{"mime":"application/pdf","data":"JQ=="}"#; + let file = |extra: &str| format!(r#"{{"format":"vectorcraft","version":4,{extra}"document":{{"layers":[]}},"images":{{}}}}"#); + let value = |json: &str, key: &str| serde_json::from_str::(json).unwrap()[key].clone(); + for extra in [ + r#""generator":"a","generator":"b","#.to_string(), + format!(r#""preview":{png},"preview":{png},"#), + format!(r#""pdf":null,"pdf":{pdf},"#), + format!(r#""pdf":{pdf},"pdf":"x","#), + format!(r#""generator":"a","preview":{png},"generator":7,"preview":"x","pdf":1,"pdf":{pdf},"#), + r#""generator":7,"preview":"x","pdf":1,"generator":7,"preview":"x","pdf":1,"#.to_string(), + ] { + let json = file(&extra); + let contents = Contents::::read(json.as_bytes()).unwrap_or_else(|e| panic!("{extra}: {e}")); + assert!(contents.document.layers.is_empty(), "{extra}"); + let read = File::::read(json.as_bytes()).unwrap_or_else(|e| panic!("{extra}: {e}")); + let last = |key: &str| value(&json, key); + assert_eq!(read.generator, last("generator").as_str().map(String::from), "{extra}"); + assert_eq!(read.preview, serde_json::from_value(last("preview")).ok(), "{extra}"); + assert_eq!(read.pdf, serde_json::from_value(last("pdf")).ok(), "{extra}"); + } + let read = File::::read(file(r#""generator":"a","generator":"b","#).as_bytes()).unwrap(); + assert_eq!(read.generator.as_deref(), Some("b")); + for extra in + [r#""format":"vectorcraft","#, r#""version":4,"#, r#""document":{"layers":[]},"#, r#""images":{},"#, r#""profiles":{},"profiles":{},"#] + { + let json = file(extra); + assert!(File::::read(json.as_bytes()).is_err_and(|e| e.to_string().starts_with("duplicate field")), "{extra}"); + assert!(Contents::::read(json.as_bytes()).is_err_and(|e| e.to_string().starts_with("duplicate field")), "{extra}"); + } +} + +#[test] +fn an_array_holds_format_version_document_images_and_profiles_in_that_order() { + // VectorCraft reads a top-level array by position, in this order. + let png = r#"{"mime":"image/png","data":"iVBORw0KGgo="}"#; + let images = format!(r#"{{"k":{png}}}"#); + let profiles = format!(r#"{{"Gray":{png}}}"#); + let full = format!(r#"["vectorcraft",3,{{"layers":[]}},{images},{profiles}]"#); + for json in [full.clone(), format!(r#"["vectorcraft",3,{{"layers":[]}},{images}]"#), r#"["vectorcraft",3,{"layers":[]}]"#.into()] { + let contents = Contents::::read(json.as_bytes()).unwrap_or_else(|e| panic!("{json}: {e}")); + let file = File::::read(json.as_bytes()).unwrap_or_else(|e| panic!("{json}: {e}")); + assert_eq!( + (&file.format, file.version, &file.document, &file.images, &file.profiles), + (&contents.format, 3, &contents.document, &contents.images, &contents.profiles) + ); + assert!(file.generator.is_none() && file.preview.is_none() && file.pdf.is_none()); + } + let contents = Contents::::read(full.as_bytes()).unwrap(); + assert_eq!((contents.images.len(), contents.profiles.len()), (1, 1)); + assert!(contents.images.contains_key("k") && contents.profiles.contains_key("Gray")); + for json in [r#"["vectorcraft",3,{},{"layers":[]},{}]"#, r#"["vectorcraft",3]"#, r#"["vectorcraft",5,{"layers":[]}]"#] { + assert!(Contents::::read(json.as_bytes()).is_err(), "{json}"); + assert!(File::::read(json.as_bytes()).is_err(), "{json}"); + } +} + +#[test] +fn a_files_tail_follows_its_version() { + let mut file = File::new(doc()); + let pdf = pdf(); + assert_eq!(file.tail().image("application/pdf", &pdf, false).blob, Some([0, pdf.len() as u64])); + file.version = 2; + assert!(file.tail().image("application/pdf", &pdf, false).blob.is_none()); +} + +#[test] +fn the_json_is_split_only_at_a_marker_right_after_it() { + let with = |json: &[u8], rest: &[u8]| { + let mut f = json.to_vec(); + f.extend(BLOB_MAGIC); + f.extend(rest); + f + }; + for json in [&b"{}"[..], b"[1]", b"\"s\"", b"1", b"{\"a\": [true, null]}"] { + let f = with(json, b"xyz"); + assert_eq!(split_tail(&f), (json, &b"xyz"[..]), "{json:?}"); + } + let empty: &[u8] = &[]; + let far = with(b"{} x", b"xyz"); + assert_eq!(split_tail(&far), (far.as_slice(), empty)); + let no_marker = b"{}\0rest".to_vec(); + assert_eq!(split_tail(&no_marker), (no_marker.as_slice(), empty)); + assert_eq!(split_tail(b"{}"), (&b"{}"[..], empty)); + let bad = with(b"{", b"x"); + assert_eq!(split_tail(&bad), (bad.as_slice(), empty)); +} + +#[test] +fn the_marker_is_found_where_the_writer_puts_it_in_the_first_bytes() { + let mut file = br#"{"a":[1]}"#.to_vec(); + file.extend(BLOB_MAGIC); + file.extend(b"\0rest"); + // The first bytes up to the end of the marker are enough. + assert_eq!(marker_at(&file), Some(9)); + assert_eq!(marker_at(&file[..9 + BLOB_MAGIC.len()]), Some(9)); + assert_eq!(marker_at(&file[..9 + BLOB_MAGIC.len() - 1]), None); + assert_eq!(split_tail(&file), (&file[..9], &b"\0rest"[..])); + for json in [&b"\"s\""[..], b"[]"] { + let mut f = json.to_vec(); + f.extend(BLOB_MAGIC); + assert_eq!(marker_at(&f), Some(json.len()), "{json:?}"); + } + // After a number, after a space, without its newline or with a NUL before it, the marker is + // not where the writer puts it. + for f in [&b"1\n\0VCBLOBS\0"[..], b"{} \n\0VCBLOBS\0", b"{}\0VCBLOBS\0", b"{}\0\n\0VCBLOBS\0", b"\n\0VCBLOBS\0", b"{}"] { + assert_eq!(marker_at(f), None, "{:?}", String::from_utf8_lossy(f)); + } +} + +#[test] +fn check_refuses_other_formats_and_newer_versions() { + assert_eq!(check(EXTENSION, VERSION), Ok(())); + assert_eq!(check(LEGACY_EXTENSION, 1), Ok(())); + assert_eq!(check("other", 1), Err(Error::Format("other".into()))); + assert_eq!(check(EXTENSION, VERSION + 1), Err(Error::TooNew(VERSION + 1))); + assert!(File::::read(br#"{"format":"vectorcraft","version":4}"#).is_err()); + let minimal = File::::read(br#"{"format":"vectorcraft","version":4,"document":{"layers":[]}}"#).unwrap(); + assert!(minimal.images.is_empty() && minimal.generator.is_none()); +} + +#[test] +fn sniff_needs_a_vectorcraft_format_member() { + let pdf = pdf(); + let (file, tail) = full_file(&pdf); + let plain = written(&file, &tail, false); + let pretty = written(&file, &tail, true); + assert!(sniff(&plain) && sniff(&pretty)); + assert!(sniff(b" \n{\"format\" : \"drawcraft\"}")); + // Keys and values are compared as JSON reads them, escapes decoded. + let esc = |s: &str| s.replace('|', "\\"); + let escaped = esc(r#"{"|u0066ormat":"vector|u0063raft","version":4,"document":{"layers":[]}}"#); + assert!(sniff(escaped.as_bytes())); + assert_eq!(File::::read(escaped.as_bytes()).unwrap().format, EXTENSION); + assert!(sniff(esc(r#"{"format":"draw|u0063raft"}"#).as_bytes())); + for no in [r#"{"form|u0061t":"other","format":"vectorcraft"}"#, r#"{"format":"vectorcraft|u0021"}"#, r#"{"format":"vectorcraft|"}"#] { + assert!(!sniff(esc(no).as_bytes()), "{no}"); + } + assert!(sniff(br#"{"generator":"format","x":{"format":"other","a":[1,"]"]},"n":-1.5e3,"t":true,"s":"\"format\":","format":"vectorcraft"}"#)); + assert!(is_compressed(&gzip(&plain)) && !is_compressed(&plain)); + assert!(sniff(&gzip(&plain))); + let gz = gzip(&plain); + assert!(sniff(&gz[..gz.len() / 2])); + for no in [ + &br#"{"name":"vectorcraft"}"#[..], + br#"{"format":"other","vectorcraft":1}"#, + br#"["format","vectorcraft"]"#, + br#"{"metadata":{"format":"vectorcraft"},"format":"other"}"#, + br#"{"name":"\"format\":\"vectorcraft\""}"#, + br#"{"format":["vectorcraft"]}"#, + br#"{"format":"vectorcraft"#, + b"%PDF-1.7", + b"", + b"\x1f\x8b", + PNG, + ] { + assert!(!sniff(no), "{:?}", String::from_utf8_lossy(no)); + } +} + +#[test] +fn base64_is_the_encoding_of_data_members() { + assert_eq!(base64::encode(PNG), "iVBORw0KGgo="); + assert_eq!(base64::decode("AQID+g==").unwrap(), [1, 2, 3, 250]); +}