daniel543x/craft-libs
1
0 Fork 0
mirror of https://github.com/storytold/craft-libs.git synced 2026-11-08 19:42:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge origin/main into fix/497-coderabbit

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Martin BallandClaude Opus 5.5 committed 2026-11-03 00:46:59 +00:00
2 parents 5eb1710914 b047c2f7a6
commit f87f08a87a
105 files changed
+10093 -752
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/doc.yml
+34
View File
@@ -0,0 +1,34 @@
name: Craft documents
on:
workflow_dispatch:
push:
branches: [main]
paths: ['crates/doc/**', 'crates/pcraft-format/**', 'crates/vectorcraft-format/**', 'crates/ecproj-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/doc.yml']
pull_request:
paths: ['crates/doc/**', 'crates/pcraft-format/**', 'crates/vectorcraft-format/**', 'crates/ecproj-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/doc.yml']
permissions:
contents: read
jobs:
doc:
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable
with:
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-doc --all-targets --locked -- -D warnings
- run: cargo test -p craft-doc --locked -- --nocapture
- run: cargo check -p craft-doc --all-features --locked --target wasm32-unknown-unknown
.github/workflows/ecproj-format.yml
+34
View File
@@ -0,0 +1,34 @@
name: EffectCraft project format
on:
workflow_dispatch:
push:
branches: [main]
paths: ['crates/ecproj-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/ecproj-format.yml']
pull_request:
paths: ['crates/ecproj-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/ecproj-format.yml']
permissions:
contents: read
jobs:
ecproj-format:
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable
with:
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-ecproj-format --all-targets --locked -- -D warnings
- run: cargo test -p craft-ecproj-format --locked -- --nocapture
- run: cargo check -p craft-ecproj-format --locked --target wasm32-unknown-unknown
.github/workflows/font-store.yml
+34
View File
@@ -0,0 +1,34 @@
name: Font store
on:
workflow_dispatch:
push:
branches: [main]
paths: ['crates/font-store/**', 'crates/font-catalog/**', 'crates/atomic/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/font-store.yml']
pull_request:
paths: ['crates/font-store/**', 'crates/font-catalog/**', 'crates/atomic/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/font-store.yml']
permissions:
contents: read
jobs:
font-store:
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable
with:
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-font-store --all-targets --locked -- -D warnings
- run: cargo test -p craft-font-store --locked
- name: Check the web build
if: runner.os == 'Linux'
run: cargo check -p craft-font-store --target wasm32-unknown-unknown --locked
.github/workflows/media.yml
+4 -2
View File
@@ -3,9 +3,9 @@ name: Shared media
on:
push:
branches: [main]
paths: ["crates/media/**", "Cargo.toml", "Cargo.lock", ".github/workflows/media.yml"]
paths: ["crates/media/**", "crates/media-testkit/**", "Cargo.toml", "Cargo.lock", ".github/workflows/media.yml"]
pull_request:
paths: ["crates/media/**", "Cargo.toml", "Cargo.lock", ".github/workflows/media.yml"]
paths: ["crates/media/**", "crates/media-testkit/**", "Cargo.toml", "Cargo.lock", ".github/workflows/media.yml"]
workflow_dispatch:
permissions:
@@ -37,6 +37,8 @@ jobs:
- run: cargo clippy -p craft-media --all-targets --locked -- -D warnings
- run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features matroska -- -D warnings
- run: cargo clippy -p craft-media --all-targets --locked --all-features -- -D warnings
- run: cargo clippy -p craft-media-testkit --all-targets --locked -- -D warnings
- run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features serde -- -D warnings
- run: cargo check -p craft-media --all-features --locked --target wasm32-unknown-unknown
test:
timeout-minutes: 15
.github/workflows/pcraft-format.yml
+35
View File
@@ -0,0 +1,35 @@
name: PhotoCraft format
on:
workflow_dispatch:
push:
branches: [main]
paths: ['crates/pcraft-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/pcraft-format.yml']
pull_request:
paths: ['crates/pcraft-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/pcraft-format.yml']
permissions:
contents: read
jobs:
pcraft-format:
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable
with:
toolchain: "1.95"
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-pcraft-format --all-targets --locked -- -D warnings
- run: cargo test -p craft-pcraft-format --locked -- --nocapture
- run: cargo check -p craft-pcraft-format --locked --target wasm32-unknown-unknown
.github/workflows/vectorcraft-format.yml
+35
View File
@@ -0,0 +1,35 @@
name: VectorCraft format
on:
workflow_dispatch:
push:
branches: [main]
paths: ['crates/vectorcraft-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/vectorcraft-format.yml']
pull_request:
paths: ['crates/vectorcraft-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/vectorcraft-format.yml']
permissions:
contents: read
jobs:
vectorcraft-format:
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable
with:
toolchain: "1.95"
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-vectorcraft-format --all-targets --locked -- -D warnings
- run: cargo test -p craft-vectorcraft-format --locked -- --nocapture
- run: cargo check -p craft-vectorcraft-format --locked --target wasm32-unknown-unknown
Cargo.lock
Generated
+58
View File
@@ -798,10 +798,33 @@ dependencies = [
"rfd",
]
[[package]]
name = "craft-doc"
version = "0.1.0"
dependencies = [
"craft-ecproj-format",
"craft-pcraft-format",
"craft-vectorcraft-format",
"flate2",
"proptest",
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-docpath"
version = "0.1.0"
[[package]]
name = "craft-ecproj-format"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-font-catalog"
version = "0.1.0"
@@ -837,11 +860,15 @@ dependencies = [
"craft-atomic",
"craft-font-catalog",
"flate2",
"log",
"ruzstd",
"serde",
"serde_json",
"sha2",
"skrifa 0.47.0",
"tar",
"thiserror 2.0.21",
"toml",
"ureq",
"write-fonts 0.54.1",
]
@@ -914,11 +941,17 @@ dependencies = [
name = "craft-media"
version = "0.1.0"
dependencies = [
"craft-media-testkit",
"proptest",
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-media-testkit"
version = "0.1.0"
[[package]]
name = "craft-menu"
version = "0.1.0"
@@ -931,6 +964,15 @@ dependencies = [
"objc2-app-kit 0.3.2",
]
[[package]]
name = "craft-pcraft-format"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-pdf"
version = "0.1.0"
@@ -1104,6 +1146,16 @@ dependencies = [
"zip",
]
[[package]]
name = "craft-vectorcraft-format"
version = "0.1.0"
dependencies = [
"flate2",
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-window"
version = "0.1.0"
@@ -4657,6 +4709,12 @@ dependencies = [
"unicode-script",
]
[[package]]
name = "ruzstd"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69e4ce4a039f42bf25920b95667e4d789e98749ab11bbb8d2c2936f788421bb5"
[[package]]
name = "ryu"
version = "1.0.23"
Cargo.toml
+5
View File
@@ -27,7 +27,9 @@ craft-atomic = { path = "crates/atomic" }
craft-cms = { path = "crates/cms" }
craft-crw = { path = "crates/crw" }
craft-dialogs = { path = "crates/dialogs" }
craft-doc = { path = "crates/doc" }
craft-docpath = { path = "crates/docpath" }
craft-ecproj-format = { path = "crates/ecproj-format" }
craft-font-catalog = { path = "crates/font-catalog" }
craft-font-manager = { path = "crates/font-manager" }
craft-font-store = { path = "crates/font-store" }
@@ -36,7 +38,9 @@ craft-i18n = { path = "crates/i18n" }
craft-jpeg = { path = "crates/jpeg" }
craft-keymap = { path = "crates/keymap" }
craft-media = { path = "crates/media" }
craft-media-testkit = { path = "crates/media-testkit" }
craft-menu = { path = "crates/menu" }
craft-pcraft-format = { path = "crates/pcraft-format" }
craft-prepress = { path = "crates/prepress" }
craft-print = { path = "crates/print" }
craft-proof = { path = "crates/proof" }
@@ -45,6 +49,7 @@ craft-settings = { path = "crates/settings" }
craft-text = { path = "crates/text" }
craft-ui = { path = "crates/ui" }
craft-update = { path = "crates/update" }
craft-vectorcraft-format = { path = "crates/vectorcraft-format" }
craft-window = { path = "crates/window" }
egui = { version = "0.37", default-features = false }
rayon = "1"
README.md
+12
View File
@@ -42,9 +42,15 @@ craft-<name> = { git = "https://github.com/storytold/craft-libs", rev = "<commit
the event loop: queued requests shown at the end of the frame and answered on a later frame,
free-standing Open and Folder panels and Save sheets on macOS, case-insensitive filters. What
to do with the answer stays in each app.
- [`craft-doc`](crates/doc/README.md): reads PhotoCraft `.pcraft`, VectorCraft `.vectorcraft`
and EffectCraft `.ecproj` documents without the apps: the kind and version, the previews and
PDF they carry, and their pages and layers. Rendering and editing stay in each app.
- [`craft-docpath`](crates/docpath/src/lib.rs): decides whether a path or `file:` URL stored in a
document may be opened. Paths stored in documents are followed only when they are local or on the
document's own network share; other network paths, device paths and non-file URLs are refused.
- [`craft-ecproj-format`](crates/ecproj-format/README.md): the envelope of EffectCraft `.ecproj`
project files: `savedBy`, the schema version and its check, and the file object with the
project and its saved view as type parameters. The project model stays in EffectCraft.
- [`craft-font-catalog`](crates/font-catalog/README.md): the installed fonts on every platform
(font folders, fontconfig and Flatpak host folders, CoreText, DirectWrite, the font store),
lookup by family, style, localized, legacy, full and PostScript names, family loads shared
@@ -79,6 +85,9 @@ craft-<name> = { git = "https://github.com/storytold/craft-libs", rev = "<commit
(app menu, Window, Help, system-key clashes), a platform backend trait, and key equivalents
fed back to egui as the input egui-winit would have made. Commands and their enabled state
stay in each app.
- [`craft-pcraft-format`](crates/pcraft-format/README.md): the envelope of PhotoCraft's `.pcraft`
bundle: `Manifest<D>` with the document as a type parameter, `TileRef`, the entry paths, the
manifest decode limits and the migration runner. The document model stays in PhotoCraft.
- [`craft-print`](crates/print/README.md): PDF submission to CUPS and Windows, printer discovery,
duplex/copies/colour and driver preferences. PDF generation stays in each app.
- [`craft-proof`](crates/proof/README.md): proofing per language (spelling, suggestions, grammar,
@@ -96,6 +105,9 @@ craft-<name> = { git = "https://github.com/storytold/craft-libs", rev = "<commit
for 47 more languages (one cargo feature each).
- [`craft-ui`](crates/ui/README.md): SVG catalogs, icon buttons, tabs, pane controls, transactional docking, print settings and theme helpers for egui frontends;
assets, themes, localization and command handling stay in each app. Requires Rust 1.95 or later.
- [`craft-vectorcraft-format`](crates/vectorcraft-format/README.md): the envelope of VectorCraft's
`.vectorcraft` file: its members, versions, embedded images and the data after the JSON, with the
document as a type parameter. The document and its migrations stay in VectorCraft.
- [Shared UI component audit](docs/ui-components-audit.md): source inventory, extraction priorities
and compatibility boundaries across the Craft suite.
- [Theme infrastructure](docs/theme-infrastructure.md): eight-app comparison and the small shared
crates/crw/README.md
+8 -3
View File
@@ -57,7 +57,7 @@ if craft_crw::probe(&bytes) {
### Output
`Image { header, samples, black, white }`:
`Image { header, samples, black, white, stream_end_fill }`:
- `samples`: `width × height` `u16`s, row-major, the whole sensor including the masked borders,
`bits` (10 or 12) deep, black not subtracted. Rows are in sensor order; apply
@@ -79,6 +79,7 @@ if craft_crw::probe(&bytes) {
offset. Convert with chrono (`DateTime::from_timestamp`), never by hand.
- `header.make` / `header.model`: the MakeModel record, e.g. `Canon` / `Canon EOS D60`. CRW
files carry no Exif block, so this is where the camera name comes from.
- `stream_end_fill`: how many samples were filled after the entropy stream ended (below).
### Errors and limits
@@ -91,8 +92,12 @@ by depth (16) and by a per-file record budget (4,096), and `width × height` is
buffers at its peak. `Limits` is `#[non_exhaustive]`: start from `Limits::default()` and set the
fields you change.
A stream that ends early is not an error: the remaining samples keep their predictor values (the
frozen stream-end rule; real files end 1–3 samples short).
When the entropy stream ends before the frame does, the remaining samples keep their predictor
values (the frozen stream-end rule) and `Image::stream_end_fill` says how many. Up to
`Limits::max_stream_end_fill` (default 64, one block) that is accepted; more is `Corrupt`, a stream
cut short, which would otherwise decode as a smeared image. Every real file measured fills none:
1,318 files of all seven verified bodies (1,308 of them EOS D60), whose last samples come from an
end-of-block code.
### Performance
crates/crw/src/lib.rs
+23 -6
View File
@@ -63,11 +63,15 @@ pub struct Limits {
/// `u16` buffers of this size at its peak. Default: 2²³ (8.4 million) on every target; real
/// CRW frames hold at most 7.4 million samples.
pub max_samples: usize,
/// Most samples [`decode`] may fill after the entropy stream ends (the stream-end rule: they
/// take their predictor values); more is [`Error::Corrupt`], a stream cut short. Default: 64,
/// one block; every real file measured (1,318 files, all seven verified bodies) fills none.
pub max_stream_end_fill: usize,
}
impl Default for Limits {
fn default() -> Self {
Self { max_samples: 1 << 23 }
Self { max_samples: 1 << 23, max_stream_end_fill: 64 }
}
}
@@ -179,6 +183,9 @@ pub struct Image {
pub black: BlackLevel,
/// White level: the saturation plateau, the sensor's clip level, or the full range.
pub white: f32,
/// Samples filled after the entropy stream ended, at their predictor values (the stream-end
/// rule); at most [`Limits::max_stream_end_fill`]. Zero on every real file measured.
pub stream_end_fill: usize,
}
/// Result alias for this crate.
@@ -368,13 +375,18 @@ fn parse(bytes: &[u8], limits: &Limits, full: bool) -> Result<Image> {
let has_low = header_after_plane || (plane_fits && head.windows(2).any(|p| p[0] == 0xff && p[1] != 0x00));
let bits = if has_low { 12 } else { 10 };
let low_len = if has_low { q } else { 0 };
let mut stream_end_fill = 0;
let samples = if full {
let entropy = payload.get(low_len.saturating_add(ent_off)..).ok_or_else(|| corrupt("payload shorter than its header"))?;
let low = has_low.then(|| payload.get(..q)).flatten();
if has_low && low.is_none() {
return Err(corrupt("payload shorter than its low-bits plane"));
}
let hi = decode_entropy(entropy, w, h, &tables)?;
let (hi, tail) = decode_entropy(entropy, w, h, &tables)?;
if tail > limits.max_stream_end_fill {
return Err(Error::Corrupt(format!("entropy stream ends {tail} samples before the frame does")));
}
stream_end_fill = tail;
let mut samples = vec![0u16; n];
// `samples` is documented as `bits` deep, so a damaged stream whose chain value runs past
// the range is clamped here: a consumer indexing a `1 << bits` LUT must not see more.
@@ -429,7 +441,7 @@ fn parse(bytes: &[u8], limits: &Limits, full: bool) -> Result<Image> {
exif_orientation,
wb_multipliers: wb_from_color_balance(bytes, &rec, o),
};
Ok(Image { header, samples, black, white })
Ok(Image { header, samples, black, white, stream_end_fill })
}
/// As-shot white-balance multipliers from the ColorBalance record (0x10a9, D60/10D/300D): a
@@ -645,8 +657,9 @@ fn build_lut(codes: &[(u16, u8, u8, u8)]) -> Vec<u32> {
/// model (`clean-room/02-crw-format.md` §2): 64-sample blocks, first/main code tables,
/// (run, size) leaves with JPEG-rule magnitudes, EOB, block-DC carry, and two parity DPCM chains
/// initialised to 512 and reset at every row start. A stream that ends early leaves the remaining
/// samples at their chain values (the frozen stream-end rule).
fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables) -> Result<Vec<u16>> {
/// samples at their chain values (the frozen stream-end rule); the second value returned is how
/// many, so [`decode`] can refuse a stream cut short ([`Limits::max_stream_end_fill`]).
fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables) -> Result<(Vec<u16>, usize)> {
let d = unstuff(entropy);
let nbits = d.len().saturating_mul(8);
let peek16 = |pos: usize| -> u16 {
@@ -661,12 +674,15 @@ fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables
let mut out = Chains { dec: vec![0u16; n], w, qi: 0, ce: 512, co: 512 };
let mut carry: i32 = 0;
let mut pos = 0usize;
// samples filled after the stream ended (the stream-end rule)
let mut tail = 0usize;
while out.qi < n {
let block_end = n.min((out.qi / 64 + 1) * 64);
let mut tok = 0u32;
while out.qi < block_end {
// stream exhausted: every remaining sample keeps its chain value
if pos >= nbits.saturating_sub(1) {
tail = n - out.qi;
out.fill(n);
break;
}
@@ -676,6 +692,7 @@ fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables
}
let (len, mut rr, s) = ((e >> 16) as usize, ((e >> 8) & 0xff) as usize, e & 0xff);
if pos.saturating_add(len).saturating_add(s as usize) > nbits {
tail = n - out.qi;
out.fill(n);
break;
}
@@ -713,7 +730,7 @@ fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables
}
}
}
Ok(out.dec)
Ok((out.dec, tail))
}
/// Drop the stuffing byte: an 0x00 after an 0xff.
crates/crw/src/tests.rs
+111 -3
View File
@@ -266,7 +266,7 @@ fn header_reads_geometry_without_samples() {
fn limits_cap_the_sample_count() {
let hi = test_hi(128);
let f = roundtrip_file(16, 8, "Canon EOS D60", None, &hi, Order::Le);
let mut limits = Limits { max_samples: 127 };
let mut limits = Limits { max_samples: 127, ..Limits::default() };
assert_eq!(decode(&f, &limits), Err(Error::LimitExceeded("image too large")));
assert!(header(&f, &limits).is_err());
limits.max_samples = 128;
@@ -365,7 +365,7 @@ fn black_level_ignores_clamped_rows() {
}
/// The frozen stream-end rule: a truncated entropy stream leaves the remaining samples at
/// their chain values instead of failing the file.
/// their chain values instead of failing the file, when the limit allows that many.
#[test]
fn truncated_stream_keeps_chain_values() {
let (w, h) = (16u16, 8u16);
@@ -378,9 +378,117 @@ fn truncated_stream_keeps_chain_values() {
Order::Le,
);
let f = ciff(&[(RAW_DATA, payload(None, &entropy[..cut])), (0x300a, props)], Order::Le);
let img = full(&f).expect("truncated stream still decodes");
let lenient = Limits { max_stream_end_fill: 128, ..Limits::default() };
let img = decode(&f, &lenient).expect("truncated stream still decodes");
assert_eq!(img.samples.len(), 128);
assert_eq!(&img.samples[..32], &hi[..32], "the leading samples decode exactly");
assert!(img.stream_end_fill > 0, "the samples after the cut are filled");
let tail_start = img.samples.len() - img.stream_end_fill;
for i in tail_start..img.samples.len() {
let expected = if i % (w as usize) < 2 { 512 } else { img.samples[i - 2] };
assert_eq!(img.samples[i], expected, "filled sample at {i} keeps its predictor value");
}
}
/// A stream cut short decoded as a smeared image: the samples after its end take their chain
/// values. Past one block that is `Corrupt` by default; a complete stream fills none, and a fill
/// within the limit is accepted.
#[test]
fn a_stream_cut_short_is_corrupt() {
let (w, h) = (64u16, 32u16);
let hi = test_hi(2048);
let tables = tables::tables(0).expect("selector 0");
let entropy = encode_hi(&hi, 64, &tables);
let file = |entropy: &[u8], with_trailer: bool| {
let mut p = Vec::with_capacity(514 + entropy.len() + 2);
p.extend_from_slice(&[0; 514]);
p.extend_from_slice(entropy);
if with_trailer {
p.extend_from_slice(&[0, 0]);
}
let props = heap(
&[
(SENSOR_INFO, sensor_info(w, h, None, Order::Le)),
(DECODER_TABLE, decoder_table(0, Order::Le)),
(MAKE_MODEL, make_model("Canon EOS D60")),
],
Order::Le,
);
ciff(&[(RAW_DATA, p), (0x300a, props)], Order::Le)
};
let whole = full(&file(&entropy, true)).expect("a complete stream decodes");
assert_eq!(whole.stream_end_fill, 0);
let cut = file(&entropy[..entropy.len() / 2], false);
let Err(Error::Corrupt(why)) = full(&cut) else { panic!("a stream cut in half must be refused") };
assert!(why.contains("samples before the frame does"), "{why}");
let filled = decode(&cut, &Limits { max_stream_end_fill: usize::MAX, ..Limits::default() }).expect("lenient").stream_end_fill;
assert_eq!(filled, 1024, "half the 2048-sample frame is 1024 filled samples");
let at_limit = Limits { max_stream_end_fill: 1024, ..Limits::default() };
assert_eq!(decode(&cut, &at_limit).map(|i| i.stream_end_fill), Ok(1024), "a fill at the limit is accepted");
}
/// Truncated entropy streams at 1 sample, 64 samples (at the default limit), 65 samples (past
/// the default limit, returning Corrupt), and a complete file; and verifying the limit can be raised
/// by callers wanting partial recovery.
#[test]
fn truncated_stream_fill_limits_and_recovery() {
let (w, h) = (16u16, 8u16);
let n = (w * h) as usize; // 128
// Construct hi where every sample difference has magnitude 16 (size 5), encoding to exactly 8 bits
// (1 byte) per sample across both first and main tables.
let mut hi = Vec::with_capacity(n);
let (mut ce, mut co) = (512i32, 512i32);
for qi in 0..n {
if qi % w as usize == 0 {
ce = 512;
co = 512;
}
let c = if qi % 2 == 0 { &mut ce } else { &mut co };
*c += 16;
hi.push(*c as u16);
}
let tables = tables::tables(0).expect("selector 0");
let entropy = encode_hi(&hi, w as usize, &tables);
assert_eq!(entropy.len(), n, "each sample encodes to exactly 1 byte");
let file = |entropy_bytes: &[u8]| {
let mut p = Vec::with_capacity(514 + entropy_bytes.len());
p.extend_from_slice(&[0; 514]);
p.extend_from_slice(entropy_bytes);
let props = heap(
&[
(SENSOR_INFO, sensor_info(w, h, None, Order::Le)),
(DECODER_TABLE, decoder_table(0, Order::Le)),
(MAKE_MODEL, make_model("Canon EOS D60")),
],
Order::Le,
);
ciff(&[(RAW_DATA, p), (0x300a, props)], Order::Le)
};
// 1. Complete file: fills 0 samples, decodes cleanly with default limits
let complete = file(&entropy);
let img = full(&complete).expect("complete file decodes");
assert_eq!(img.stream_end_fill, 0);
// 2. Truncated 1 sample before end: fills 1 sample, accepted under default limit (64)
let cut_1 = file(&entropy[..n - 1]);
let img_1 = full(&cut_1).expect("1-sample truncation accepted by default limits");
assert_eq!(img_1.stream_end_fill, 1);
// 3. Truncated 64 samples before end: fills 64 samples, accepted at the boundary of default limit (64)
let cut_64 = file(&entropy[..n - 64]);
let img_64 = full(&cut_64).expect("64-sample truncation accepted at boundary by default limits");
assert_eq!(img_64.stream_end_fill, 64);
// 4. Truncated 65 samples before end: fills 65 samples, refused by default limit (64) as Corrupt
let cut_65 = file(&entropy[..n - 65]);
let err_65 = full(&cut_65).expect_err("65-sample truncation must be refused by default limits");
assert_eq!(err_65, Error::Corrupt("entropy stream ends 65 samples before the frame does".into()));
// 5. Check the limit can be raised by callers who want partial recovery
let lenient = Limits { max_stream_end_fill: 65, ..Limits::default() };
let img_65_recovered = decode(&cut_65, &lenient).expect("decodes when limit is raised");
assert_eq!(img_65_recovered.stream_end_fill, 65);
}
/// Every record the decoder needs missing, bogus selectors, dimensions and offsets: errors,
crates/doc/Cargo.toml
+25
View File
@@ -0,0 +1,25 @@
[package]
name = "craft-doc"
description = "Read-only access to PhotoCraft, VectorCraft and EffectCraft documents: format detection, version, embedded previews and PDF, and the layer and page outline."
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
craft-ecproj-format = { workspace = true }
craft-pcraft-format = { workspace = true }
craft-vectorcraft-format = { workspace = true }
flate2 = { version = "1", default-features = false, features = ["rust_backend"] }
serde = { workspace = true }
# `unbounded_depth`: PhotoCraft manifests nest deeper than serde_json's 128 levels; the crate
# checks the nesting against `Limits::max_depth` before parsing.
serde_json = { workspace = true, features = ["unbounded_depth"] }
thiserror = { workspace = true }
[dev-dependencies]
proptest = { workspace = true }
crates/doc/README.md
+113
View File
@@ -0,0 +1,113 @@
# craft-doc
Reads PhotoCraft `.pcraft`, VectorCraft `.vectorcraft` and EffectCraft `.ecproj` documents
without the apps: which app wrote a file, its format version, the previews and PDF it carries,
and its pages and layers. It takes bytes and returns plain data. It does not open files, render
or composite. No `unsafe`, no panics on malformed input, and it builds for
`wasm32-unknown-unknown`. Each format's envelope is read through its crate in this repository:
[`craft-pcraft-format`](../pcraft-format/README.md) (the `Manifest` members and checks, and
`check_manifest`),
[`craft-vectorcraft-format`](../vectorcraft-format/README.md) (`Head`, `Image`, `sniff`, the
version and the marker) and [`craft-ecproj-format`](../ecproj-format/README.md) (`Head`, the
schema check). The outlines are read with craft-doc's own read-only views of the documents. The
other dependencies are `serde`, `serde_json`, `thiserror` and `flate2` (its pure-Rust backend).
```toml
craft-doc = { git = "https://github.com/storytold/craft-libs", rev = "<commit>" }
```
## Use
```rust
use craft_doc::{Kind, Limits, RenditionKind};
let limits = Limits::default();
match craft_doc::sniff(&bytes) {
Some(Kind::VectorCraft) => {
let pdf = craft_doc::renditions(&bytes, &limits)?.into_iter().find(|r| r.kind == RenditionKind::Pdf);
}
Some(kind) => { /* ... */ }
None => { /* not a Craft document */ }
}
let outline = craft_doc::outline(&bytes, &limits)?;
```
- `sniff(head) -> Option<Kind>` reads the first bytes. The first 64 KiB are enough for every file
the apps write. A re-zipped `.pcraft` whose `manifest.json` entry starts after the head is
recognized only when `head` holds the whole file.
- `info(bytes, &Limits) -> Result<Info, Error>`: the kind, the version the file declares, the
writer it names, and `supported`, which is `false` when the version is newer than the crate
reads.
- `renditions(bytes, &Limits) -> Result<Vec<Rendition>, Error>`: each `Rendition` has a `kind`
(`Pdf`, `Thumbnail`, `Composite`), a `mime` and the `bytes`.
- `outline(bytes, &Limits) -> Result<Outline, Error>`: `canvas`, `pages` and `layers`.
- `Error` is `Unrecognized`, `Malformed`, `TooNew`, `Unsupported` or `LimitExceeded`. A file
whose version is newer than the crate reads gives `TooNew` from `renditions` and `outline`.
## What each format gives
| | `.vectorcraft` | `.pcraft` | `.ecproj` |
|---|---|---|---|
| Container | JSON, plain or gzip; since version 4, large members after the JSON (`\n\0VCBLOBS\0`) | ZIP of STORE or DEFLATE entries | JSON |
| Version read | `version` up to 4 (`"format": "vectorcraft"` or `"drawcraft"`) | `format_version` 1 | `schema` up to 2 |
| Writer | `generator` | `generator` | `savedBy` |
| Renditions | `preview` (Thumbnail) and `pdf` (Pdf), when the file was saved with them | the `thumbnail` and `composite` the manifest names | none |
| Pages | artboards, in points | none | compositions by item id, in pixels, with frame rate and duration |
| Layers | top-level layers | the layer tree with groups | each composition's layers |
Layers are listed bottom first in every format. A layer's `kind` is the name the file uses:
`layer` in VectorCraft; `raster`, `group`, `adjustment`, `fill`, `text`, `shape` or `smart` in
PhotoCraft; the source type in EffectCraft (`Footage`, `Comp`, `Solid`, `Text` and others).
EffectCraft layers have no `opacity`, because opacity is an animated property there.
A `.pcraft` bundle saved as a folder, ZIP64 archives and encrypted entries are not read. The
`.ecprojx` XML copies are not read.
## Limits
`Limits` (fields `max_json`, `max_entry`, `max_total`, `max_depth`) bounds each call:
- `max_json` (256 MiB): the JSON read, which is the VectorCraft file's JSON, the PhotoCraft
manifest or the EffectCraft project. The names and other strings returned are parts of it.
- `max_entry` (256 MiB): one rendition or one ZIP entry. The size a file states is checked before
any of it is copied or inflated.
- `max_total` (1 GiB): every byte one call decompresses from gzip, reads out of ZIP entries or
returns as a rendition, together.
- `max_depth` (400, also the largest value it takes, `MAX_DEPTH`): the nesting of JSON arrays and
objects, checked before parsing. PhotoCraft nests up to 100 groups of layers, three levels each.
A PhotoCraft manifest with more than 4,194,304 JSON items
(`craft_pcraft_format::MAX_MANIFEST_ITEMS`) is refused with `Error::LimitExceeded`, as PhotoCraft
reads it.
On a compressed `.vectorcraft` file, `info` and `outline` decompress only as far as the end of
the JSON. `renditions` checks the size of each member after the JSON against `max_entry`
before it decompresses any of that data, then decompresses only as far as the members end. The
struct is `#[non_exhaustive]`: start from the defaults and set the fields you
change.
```rust
let mut limits = craft_doc::Limits::default();
limits.max_entry = 32 << 20;
```
## Testing
`cargo test -p craft-doc` runs:
- `tests/formats.rs`: documents of each format written by the tests (JSON by hand, PNG and ZIP
bytes assembled in the test, gzip through `flate2`), read back through each call: plain and
gzip VectorCraft files, members inline as base64 and after the JSON, `.pcraft` bundles with
STORE and DEFLATE entries and the manifest last, `.ecproj` compositions, newer versions, and
files `sniff` must not claim (PDF, PNG, other ZIP and JSON files).
- `tests/hostile.rs`: every truncation of each sample, seeded mutations, proptest on arbitrary
bytes, and inputs that pass each limit: a gzip bomb, a DEFLATE entry larger than its stated
size, members longer than `max_entry`, JSON nested past `max_depth`, ZIP64, encryption, other
compression methods and CRC mismatches.
- `tests/envelopes.rs`: files whose envelope is written with the format crates' types read
through each call, and envelopes those crates refuse (a manifest without `generator`, a preview
that names another entry, a `format` member inside another member) are refused. A preview or
PDF member without `mime` gives no rendition.
`.github/workflows/doc.yml` runs fmt, clippy and the tests on Linux and Windows, and a
`wasm32-unknown-unknown` check.
crates/doc/src/ecproj.rs
+221
View File
@@ -0,0 +1,221 @@
//! EffectCraft `.ecproj` projects: one UTF-8 JSON object, uncompressed:
//!
//! ```text
//! { "savedBy": "0.12.0", "schema": 2, "settings": {...},
//! "items": { "1": { "id": 1, "name": "Comp 1", "kind": { "type": "Comp", "width": 1920, ... } } },
//! ... }
//! ```
//!
//! Projects saved before `savedBy` existed start with `schema`. A project holds no rendered
//! frames: footage is referenced by path. `savedBy` and `schema` are read with
//! [`craft_ecproj_format::Head`].
use std::collections::BTreeMap;
use craft_ecproj_format::{Head, ReadError};
use serde::Deserialize;
use crate::{Budget, Error, Kind, Layer, Outline, Page, Rendition, Timing, check_depth, counted_map, counted_option, malformed, parse, too_new};
/// EffectCraft's time unit.
const TICKS_PER_SECOND: f64 = 254_016_000_000.0;
/// Calls `stop` with each member name of the JSON object `text` starts with, in order, as far as
/// `text` goes, until it returns `true`. Members of nested objects are skipped. A name is given as
/// written, escapes included.
fn top_level_keys<'a>(text: &'a [u8], mut stop: impl FnMut(&'a [u8]) -> bool) {
let Some(body) = text.trim_ascii_start().strip_prefix(b"{") else { return };
let (mut depth, mut want_key, mut i) = (0u32, true, 0);
while let Some(&b) = body.get(i) {
match b {
b'"' => {
let start = i + 1;
let mut end = start;
loop {
match body.get(end) {
None => return,
Some(b'\\') => end += 2,
Some(b'"') => break,
Some(_) => end += 1,
}
}
if depth == 0
&& want_key
&& let Some(key) = body.get(start..end)
&& stop(key)
{
return;
}
want_key = false;
i = end;
}
b'{' | b'[' => depth += 1,
b'}' | b']' if depth == 0 => return,
b'}' | b']' => depth -= 1,
b',' if depth == 0 => want_key = true,
_ => {}
}
i += 1;
}
}
/// A project starts with `savedBy` followed by `schema`, or, saved before `savedBy` existed, with
/// `schema` followed by `items` or `savedBy`, all top-level members.
pub(crate) fn sniff(head: &[u8]) -> bool {
let (mut first, mut found) = (None::<&[u8]>, false);
top_level_keys(head, |key| match first {
None => {
first = Some(key);
!matches!(key, b"savedBy" | b"schema")
}
Some(b"savedBy") => {
found = key == b"schema";
found
}
Some(_) => {
found = key == b"items" || key == b"savedBy";
found
}
});
found
}
fn json<'a>(bytes: &'a [u8], budget: &Budget) -> Result<&'a [u8], Error> {
budget.check_json(bytes.len() as u64, "the project")?;
Ok(bytes)
}
/// The project's [`Head`] read with [`Head::read`]; [`ReadError::TooNew`] when its schema is
/// newer than this crate reads.
fn read_head(json: &[u8], budget: &Budget) -> Result<Result<Head, ReadError>, Error> {
check_depth(json, budget.limits.max_depth)?;
let text = std::str::from_utf8(json).map_err(|e| malformed(format!("the project is not UTF-8: {e}")))?;
Ok(Head::read(text))
}
/// The project's [`Head`], refused when its schema is newer than this crate reads.
fn head(json: &[u8], budget: &Budget) -> Result<Head, Error> {
read_head(json, budget)?.map_err(|e| match e {
ReadError::TooNew { too_new: e, .. } => too_new(Kind::EffectCraft, e.schema),
e => malformed(e),
})
}
pub(crate) fn info(bytes: &[u8], budget: &mut Budget) -> Result<(u32, Option<String>), Error> {
match read_head(json(bytes, budget)?, budget)? {
Ok(head) => Ok((head.schema, head.saved_by)),
Err(ReadError::TooNew { too_new, saved_by, .. }) => Ok((too_new.schema, saved_by)),
Err(e) => Err(malformed(e)),
}
}
pub(crate) fn renditions(bytes: &[u8], budget: &mut Budget) -> Result<Vec<Rendition>, Error> {
head(json(bytes, budget)?, budget)?;
Ok(vec![])
}
#[derive(Deserialize)]
struct Project {
#[serde(default, deserialize_with = "counted_map")]
items: BTreeMap<String, ItemM>,
}
#[derive(Deserialize)]
struct ItemM {
id: u64,
#[serde(default)]
name: String,
kind: ItemKindM,
}
/// The members of an item's kind that a composition has. Footage and solids share some of them
/// with the same types; folders have none.
#[derive(Deserialize)]
struct ItemKindM {
#[serde(rename = "type")]
kind: String,
#[serde(default)]
width: Option<u32>,
#[serde(default)]
height: Option<u32>,
#[serde(default)]
frame_rate: Option<FrameRateM>,
/// In ticks.
#[serde(default)]
duration: Option<i64>,
/// A composition's layers, the top one first.
#[serde(default, deserialize_with = "counted_option")]
layers: Option<Vec<LayerM>>,
}
#[derive(Deserialize)]
struct FrameRateM {
num: i64,
den: i64,
}
fn yes() -> bool {
true
}
#[derive(Deserialize)]
struct LayerM {
#[serde(default)]
name: String,
#[serde(default)]
source: Option<SourceM>,
#[serde(default)]
switches: Option<SwitchesM>,
#[serde(default)]
blend_mode: Option<String>,
}
#[derive(Deserialize)]
struct SourceM {
#[serde(rename = "type")]
kind: String,
}
#[derive(Deserialize)]
struct SwitchesM {
#[serde(default = "yes")]
video: bool,
}
pub(crate) fn outline(bytes: &[u8], budget: &mut Budget) -> Result<Outline, Error> {
let json = json(bytes, budget)?;
head(json, budget)?;
let p: Project = parse(json, budget.limits)?;
let mut comps: Vec<ItemM> = p.items.into_values().filter(|i| i.kind.kind == "Comp").collect();
comps.sort_by_key(|i| i.id);
let pages = comps
.into_iter()
.map(|i| {
let k = i.kind;
let (Some(width), Some(height)) = (k.width, k.height) else {
return Err(malformed(format!("composition `{}` has no size", i.name)));
};
let timing = k.frame_rate.zip(k.duration).map(|(r, duration)| Timing {
frame_rate_num: r.num,
frame_rate_den: r.den,
duration_seconds: duration as f64 / TICKS_PER_SECOND,
});
let layers = k
.layers
.unwrap_or_default()
.into_iter()
.rev()
.map(|l| Layer {
name: l.name,
kind: l.source.map(|s| s.kind).unwrap_or_default(),
visible: l.switches.is_none_or(|s| s.video),
opacity: None,
blend: l.blend_mode,
children: vec![],
})
.collect();
Ok(Page { name: i.name, x: 0.0, y: 0.0, width: f64::from(width), height: f64::from(height), timing, layers })
})
.collect::<Result<_, Error>>()?;
Ok(Outline { canvas: None, pages, layers: vec![] })
}
crates/doc/src/lib.rs
+536
View File
@@ -0,0 +1,536 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
//! Read-only access to the Craft apps' native documents, without the apps: PhotoCraft
//! `.pcraft`, VectorCraft `.vectorcraft` (and `.drawcraft`) and EffectCraft `.ecproj`.
//!
//! - [`sniff`] tells which app wrote a file, from its first bytes.
//! - [`info`] reads the format version and the writer the file names.
//! - [`renditions`] returns the previews and the PDF a file carries.
//! - [`outline`] returns the canvas, the pages (artboards or compositions) and the layers.
//!
//! Bytes go in and plain data comes out: no file system access, no rendering and no app crate.
//! Each format's envelope is read through its craft-libs crate: `craft-pcraft-format`,
//! `craft-vectorcraft-format` and `craft-ecproj-format`.
//! A file whose format version is newer than this crate reads gives [`Info::supported`] `false`
//! from [`info`] and [`Error::TooNew`] from the other calls.
//!
//! Nothing panics on input. Every length a file states is checked against [`Limits`] before
//! anything is allocated for it, and every decompressed byte is counted against them.
//!
//! ```
//! let file = br#"{"format":"vectorcraft","version":4,"generator":"VectorCraft 0.1.0","document":{"artboards":[],"layers":[]}}"#;
//! assert_eq!(craft_doc::sniff(file), Some(craft_doc::Kind::VectorCraft));
//! let info = craft_doc::info(file, &craft_doc::Limits::default()).unwrap();
//! assert_eq!(info.version, 4);
//! assert!(info.supported);
//! ```
mod ecproj;
mod pcraft;
mod vectorcraft;
mod zip;
use std::cell::Cell;
use std::collections::BTreeMap;
use std::fmt;
use std::marker::PhantomData;
use serde::Deserialize;
use serde::de::{self, DeserializeOwned, DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor};
/// The app that wrote a document.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum Kind {
/// A PhotoCraft `.pcraft` bundle (a ZIP archive).
PhotoCraft,
/// A VectorCraft `.vectorcraft` document (JSON, plain or gzip), or one saved under the
/// earlier name `.drawcraft`.
VectorCraft,
/// An EffectCraft `.ecproj` project (JSON).
EffectCraft,
}
impl Kind {
/// The file extension the app saves with, without the dot.
pub fn extension(self) -> &'static str {
match self {
Kind::PhotoCraft => "pcraft",
Kind::VectorCraft => "vectorcraft",
Kind::EffectCraft => "ecproj",
}
}
/// The newest format version this crate reads: PhotoCraft's `format_version`, VectorCraft's
/// `version` or EffectCraft's `schema`.
pub fn supported_version(self) -> u32 {
match self {
Kind::PhotoCraft => craft_pcraft_format::FORMAT_VERSION,
Kind::VectorCraft => craft_vectorcraft_format::VERSION,
Kind::EffectCraft => craft_ecproj_format::SCHEMA_VERSION,
}
}
}
/// What a document says about itself.
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct Info {
pub kind: Kind,
/// The format version the file declares.
pub version: u32,
/// The writer, as the file names it: VectorCraft's `generator` (`"VectorCraft 0.13.1"`),
/// PhotoCraft's `generator` (`"photocraft-format 0.1.0"`) or EffectCraft's `savedBy` (the
/// app version, `"0.12.0"`). `None` when the file has none.
pub generator: Option<String>,
/// `false` when [`Info::version`] is newer than [`Kind::supported_version`]; the other calls
/// then return [`Error::TooNew`].
pub supported: bool,
}
/// Why a document could not be read.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[non_exhaustive]
pub enum Error {
/// [`sniff`] does not recognize the bytes as a Craft document.
#[error("not a PhotoCraft, VectorCraft or EffectCraft document")]
Unrecognized,
/// The file is damaged or does not follow its format.
#[error("malformed document: {0}")]
Malformed(String),
/// The file's format version is newer than this crate reads.
#[error("{} version {version} is newer than this reader supports ({supported})", kind.extension())]
TooNew { kind: Kind, version: u32, supported: u32 },
/// A valid container feature this reader does not implement (ZIP64, encryption, a ZIP
/// compression method other than STORE or DEFLATE).
#[error("{0}")]
Unsupported(String),
/// Reading the file would exceed the caller's [`Limits`].
#[error("limit exceeded: {0}")]
LimitExceeded(String),
}
/// Bounds on the work and memory one call may use. Each is checked before the bytes it covers
/// are allocated.
///
/// ```
/// let mut limits = craft_doc::Limits::default();
/// limits.max_entry = 16 << 20;
/// ```
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct Limits {
/// Most bytes of JSON read: the VectorCraft file's JSON, the PhotoCraft manifest or the
/// EffectCraft project. The names and other strings returned are parts of it, so this also
/// bounds them. Default 256 MiB.
pub max_json: u64,
/// Most bytes of one rendition or one ZIP entry. Default 256 MiB.
pub max_entry: u64,
/// Most bytes one call decompresses from gzip, reads out of ZIP entries and returns as
/// renditions, together. Default 1 GiB.
pub max_total: u64,
/// Deepest nesting of JSON arrays and objects, at most [`MAX_DEPTH`] (a larger value counts
/// as [`MAX_DEPTH`]). Default [`MAX_DEPTH`].
pub max_depth: u32,
/// Most list elements one call reads, together: EffectCraft project items and composition
/// layers, VectorCraft artboards and layers, and PhotoCraft layers at every level. Each is
/// counted before it is read. Default 1,048,576.
pub max_items: u64,
}
/// The deepest JSON nesting any call reads: 400 levels, which holds PhotoCraft's 100 nested
/// groups of layers at three levels each.
pub const MAX_DEPTH: u32 = 400;
impl Default for Limits {
fn default() -> Self {
Limits { max_json: 256 << 20, max_entry: 256 << 20, max_total: 1 << 30, max_depth: MAX_DEPTH, max_items: 1 << 20 }
}
}
/// What a rendition shows.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum RenditionKind {
/// A PDF of the document (VectorCraft's `pdf`, saved with Create PDF-Compatible File).
Pdf,
/// A small preview image: VectorCraft's `preview` (the first artboard) or PhotoCraft's
/// `thumbnail`.
Thumbnail,
/// PhotoCraft's flattened `composite` preview.
Composite,
}
/// An image or PDF the document carries.
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct Rendition {
pub kind: RenditionKind,
/// The MIME type: the one a VectorCraft file gives (a member without one gives no
/// rendition), or `image/png` for PhotoCraft's previews.
pub mime: String,
pub bytes: Vec<u8>,
}
/// The structure of a document, without its content.
#[derive(Debug, Clone, PartialEq, Default)]
#[non_exhaustive]
pub struct Outline {
/// PhotoCraft's canvas; `None` for the other formats.
pub canvas: Option<Canvas>,
/// VectorCraft's artboards (in points) or EffectCraft's compositions (in pixels), in file
/// order; empty for PhotoCraft.
pub pages: Vec<Page>,
/// VectorCraft's top-level layers or PhotoCraft's layer tree, bottom first; empty for
/// EffectCraft, whose layers are in each [`Page`].
pub layers: Vec<Layer>,
}
/// A PhotoCraft canvas.
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct Canvas {
pub width: u32,
pub height: u32,
pub resolution_dpi: f32,
/// The color mode as the manifest writes it: `Rgb`, `Cmyk`, `Grayscale`, `Lab`, `Indexed`,
/// `Bitmap`, `Duotone` or `Multichannel`.
pub mode: String,
/// The sample type as the manifest writes it: `U8`, `U16` or `F32`.
pub depth: String,
}
/// A VectorCraft artboard or an EffectCraft composition.
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct Page {
pub name: String,
/// The left edge (0 for a composition).
pub x: f64,
/// The top edge (0 for a composition).
pub y: f64,
pub width: f64,
pub height: f64,
/// A composition's frame rate and duration; `None` for an artboard, and for a composition
/// that gives either one without the other.
pub timing: Option<Timing>,
/// A composition's layers, bottom first; empty for an artboard.
pub layers: Vec<Layer>,
}
/// A composition's timing.
#[derive(Debug, Clone, Copy, PartialEq)]
#[non_exhaustive]
pub struct Timing {
/// Frames per second as `frame_rate_num / frame_rate_den`.
pub frame_rate_num: i64,
pub frame_rate_den: i64,
pub duration_seconds: f64,
}
/// One layer.
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct Layer {
pub name: String,
/// What the layer holds, as the file names it: `layer` for VectorCraft; `raster`, `group`,
/// `adjustment`, `fill`, `text`, `shape` or `smart` for PhotoCraft; the source type for
/// EffectCraft (`Footage`, `Comp`, `Solid`, `Text`, `Shape`, `Null`, `Camera`, `Light`,
/// `Model`, `Primitive`).
pub kind: String,
pub visible: bool,
/// Opacity from 0 to 1; `None` for EffectCraft, where opacity is an animated property.
pub opacity: Option<f32>,
/// The blend mode as the file names it (`Normal`, `Multiply`…).
pub blend: Option<String>,
/// A PhotoCraft group's layers, bottom first.
pub children: Vec<Layer>,
}
/// Which app wrote the document whose first bytes are `head`, or `None`. The first 64 KiB are
/// enough for every file the apps write. A re-zipped `.pcraft` whose `manifest.json` entry
/// starts past `head` is recognized only when `head` holds the whole file.
pub fn sniff(head: &[u8]) -> Option<Kind> {
if pcraft::sniff(head) {
Some(Kind::PhotoCraft)
} else if vectorcraft::sniff(head) {
Some(Kind::VectorCraft)
} else if ecproj::sniff(head) {
Some(Kind::EffectCraft)
} else {
None
}
}
/// The document's kind, version and writer. A version newer than this crate reads is not an
/// error here: [`Info::supported`] is `false`.
pub fn info(bytes: &[u8], limits: &Limits) -> Result<Info, Error> {
let mut budget = Budget::new(limits);
let kind = sniff(bytes).ok_or(Error::Unrecognized)?;
let (version, generator) = match kind {
Kind::PhotoCraft => pcraft::info(bytes, &mut budget)?,
Kind::VectorCraft => vectorcraft::info(bytes, &mut budget)?,
Kind::EffectCraft => ecproj::info(bytes, &mut budget)?,
};
Ok(Info { kind, version, generator, supported: version <= kind.supported_version() })
}
/// The previews and the PDF the document carries, the thumbnail first. A
/// `.vectorcraft` file has a `preview` only when saved with one and a `pdf` only when saved with
/// Create PDF-Compatible File; a `.pcraft` bundle has the `thumbnail` and `composite` its
/// manifest names; an `.ecproj` project has none.
pub fn renditions(bytes: &[u8], limits: &Limits) -> Result<Vec<Rendition>, Error> {
let mut budget = Budget::new(limits);
match sniff(bytes).ok_or(Error::Unrecognized)? {
Kind::PhotoCraft => pcraft::renditions(bytes, &mut budget),
Kind::VectorCraft => vectorcraft::renditions(bytes, &mut budget),
Kind::EffectCraft => ecproj::renditions(bytes, &mut budget),
}
}
/// The document's canvas, pages and layers (see [`Outline`]).
pub fn outline(bytes: &[u8], limits: &Limits) -> Result<Outline, Error> {
let mut budget = Budget::new(limits);
match sniff(bytes).ok_or(Error::Unrecognized)? {
Kind::PhotoCraft => pcraft::outline(bytes, &mut budget),
Kind::VectorCraft => vectorcraft::outline(bytes, &mut budget),
Kind::EffectCraft => ecproj::outline(bytes, &mut budget),
}
}
fn malformed(m: impl std::fmt::Display) -> Error {
Error::Malformed(m.to_string())
}
/// The bytes one call may still decompress, read out of ZIP entries or return as renditions.
pub(crate) struct Budget<'a> {
pub(crate) limits: &'a Limits,
left: u64,
}
impl<'a> Budget<'a> {
fn new(limits: &'a Limits) -> Self {
Budget { limits, left: limits.max_total }
}
/// Bytes still allowed.
pub(crate) fn left(&self) -> u64 {
self.left
}
/// Count `n` bytes of `what` against the total.
pub(crate) fn take(&mut self, n: u64, what: &str) -> Result<(), Error> {
self.left = self
.left
.checked_sub(n)
.ok_or_else(|| Error::LimitExceeded(format!("{what}: more than max_total ({} bytes) in all", self.limits.max_total)))?;
Ok(())
}
/// Check one rendition or ZIP entry of `n` bytes against `max_entry`, then count it.
pub(crate) fn take_entry(&mut self, n: u64, what: &str) -> Result<(), Error> {
if n > self.limits.max_entry {
return Err(Error::LimitExceeded(format!("{what} is {n} bytes, more than max_entry ({})", self.limits.max_entry)));
}
self.take(n, what)
}
/// Check `n` bytes of JSON against `max_json`.
pub(crate) fn check_json(&self, n: u64, what: &str) -> Result<(), Error> {
if n > self.limits.max_json {
return Err(Error::LimitExceeded(format!("{what} is more than max_json ({} bytes)", self.limits.max_json)));
}
Ok(())
}
}
/// Is the JSON text nested at most `max` levels deep (arrays and objects), and at most
/// [`MAX_DEPTH`]?
pub(crate) fn check_depth(json: &[u8], max: u32) -> Result<(), Error> {
let max = max.min(MAX_DEPTH);
let (mut depth, mut in_str, mut escaped) = (0u32, false, false);
for &b in json {
if in_str {
match b {
_ if escaped => escaped = false,
b'\\' => escaped = true,
b'"' => in_str = false,
_ => {}
}
continue;
}
match b {
b'"' => in_str = true,
b'[' | b'{' => {
depth += 1;
if depth > max {
return Err(Error::LimitExceeded(format!("JSON nested deeper than max_depth ({max} levels)")));
}
}
b']' | b'}' => depth = depth.saturating_sub(1),
_ => {}
}
}
Ok(())
}
thread_local! {
/// The elements [`counted`] lists may still read in the current [`parse`], and whether a
/// list went past them.
static ITEMS: Cell<(u64, bool)> = const { Cell::new((u64::MAX, false)) };
}
/// Run `f` with `max` elements for [`counted`] lists; when `f` fails after a list went past
/// them, the error is [`Error::LimitExceeded`].
fn counting<T>(max: u64, f: impl FnOnce() -> Result<T, Error>) -> Result<T, Error> {
let saved = ITEMS.with(|c| c.replace((max, false)));
let r = f();
let (_, over) = ITEMS.with(|c| c.replace(saved));
match r {
Err(_) if over => Err(Error::LimitExceeded(format!("the file lists more than max_items ({max}) items, layers and pages"))),
r => r,
}
}
/// Count one list element against [`Limits::max_items`].
fn take_item<E: de::Error>() -> Result<(), E> {
ITEMS.with(|c| match c.get().0.checked_sub(1) {
Some(left) => {
c.set((left, false));
Ok(())
}
None => {
c.set((0, true));
Err(E::custom("more than max_items list elements"))
}
})
}
/// A `T` read after one element is counted with [`take_item`].
struct Item<T>(PhantomData<T>);
impl<'de, T: Deserialize<'de>> DeserializeSeed<'de> for Item<T> {
type Value = T;
fn deserialize<D: Deserializer<'de>>(self, d: D) -> Result<T, D::Error> {
take_item()?;
T::deserialize(d)
}
}
/// A list read by [`parse`] (as `#[serde(deserialize_with = "counted")]`), each element counted
/// against [`Limits::max_items`] before it is read.
pub(crate) fn counted<'de, D: Deserializer<'de>, T: Deserialize<'de>>(d: D) -> Result<Vec<T>, D::Error> {
struct List<T>(PhantomData<T>);
impl<'de, T: Deserialize<'de>> Visitor<'de> for List<T> {
type Value = Vec<T>;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("a list")
}
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Vec<T>, A::Error> {
let mut out = Vec::new();
while let Some(v) = seq.next_element_seed(Item(PhantomData))? {
out.push(v);
}
Ok(out)
}
}
d.deserialize_seq(List(PhantomData))
}
/// [`counted`] for a list that may be `null`.
pub(crate) fn counted_option<'de, D: Deserializer<'de>, T: Deserialize<'de>>(d: D) -> Result<Option<Vec<T>>, D::Error> {
struct Counted<T>(Vec<T>);
impl<'de, T: Deserialize<'de>> Deserialize<'de> for Counted<T> {
fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
counted(d).map(Counted)
}
}
Ok(Option::<Counted<T>>::deserialize(d)?.map(|c| c.0))
}
/// [`counted`] for a map, each value counted before it is read.
pub(crate) fn counted_map<'de, D: Deserializer<'de>, T: Deserialize<'de>>(d: D) -> Result<BTreeMap<String, T>, D::Error> {
struct Map<T>(PhantomData<T>);
impl<'de, T: Deserialize<'de>> Visitor<'de> for Map<T> {
type Value = BTreeMap<String, T>;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("a map")
}
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<BTreeMap<String, T>, A::Error> {
let mut out = BTreeMap::new();
while let Some(k) = map.next_key::<String>()? {
let v = map.next_value_seed(Item(PhantomData))?;
out.insert(k, v);
}
Ok(out)
}
}
d.deserialize_map(Map(PhantomData))
}
/// Parse one JSON value of type `T` from `json`, after checking its nesting against the limits.
pub(crate) fn parse<T: DeserializeOwned>(json: &[u8], limits: &Limits) -> Result<T, Error> {
check_depth(json, limits.max_depth)?;
counting(limits.max_items, || parse_checked(json).map_err(malformed))
}
/// Parse JSON whose nesting [`check_depth`] has accepted.
fn parse_checked<T: DeserializeOwned>(json: &[u8]) -> Result<T, serde_json::Error> {
let mut de = serde_json::Deserializer::from_slice(json);
de.disable_recursion_limit();
let v = T::deserialize(&mut de)?;
de.end()?;
Ok(v)
}
/// A document's version number: VectorCraft's `version`, PhotoCraft's `format_version` or
/// EffectCraft's `schema`.
#[derive(serde::Deserialize)]
struct Version {
#[serde(alias = "format_version", alias = "schema")]
version: u32,
}
/// Parse `T` from `json`; when that fails, report [`Error::TooNew`] instead if the file's version
/// (its top-level `version`, `format_version` or `schema`) is newer than `kind` supports, since a
/// newer format may change what `T` expects.
pub(crate) fn parse_versioned<T: DeserializeOwned>(json: &[u8], limits: &Limits, kind: Kind) -> Result<T, Error> {
check_depth(json, limits.max_depth)?;
counting(limits.max_items, || {
parse_checked(json).map_err(|e| match parse_checked::<Version>(json) {
Ok(v) if v.version > kind.supported_version() => too_new(kind, v.version),
_ => malformed(e),
})
})
}
pub(crate) fn too_new(kind: Kind, version: u32) -> Error {
Error::TooNew { kind, version, supported: kind.supported_version() }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn depth_counts_brackets_outside_strings() {
assert!(check_depth(br#"[[["[[[[[[[["]]]"#, 3).is_ok());
assert!(check_depth(br#"[[[["\"[["]]]]"#, 3).is_err());
assert!(check_depth(br#"{"a":{"b":[1]}}"#, 3).is_ok());
}
#[test]
fn the_budget_refuses_past_its_limits() {
let limits = Limits { max_json: 10, max_entry: 5, max_total: 8, max_depth: 4, max_items: 2 };
let mut b = Budget::new(&limits);
assert!(b.take_entry(6, "x").is_err());
assert!(b.take_entry(5, "x").is_ok());
assert!(matches!(b.take_entry(4, "x"), Err(Error::LimitExceeded(_))));
assert_eq!(b.left(), 3);
assert!(b.check_json(11, "x").is_err() && b.check_json(10, "x").is_ok());
}
}
crates/doc/src/pcraft.rs
+158
View File
@@ -0,0 +1,158 @@
//! PhotoCraft `.pcraft` bundles: a ZIP archive holding `manifest.json` (the document tree, with
//! `format_version`), tiles and blobs, and optionally the previews the manifest names
//! (`thumbnail`, `composite`). Only the ZIP form is read; a bundle saved as a folder is not. The
//! manifest's envelope is [`craft_pcraft_format::Manifest`]'s.
use craft_pcraft_format::{self as format, COMPOSITE, FORMAT_VERSION, MANIFEST, THUMB};
use serde::Deserialize;
use serde::de::{DeserializeOwned, IgnoredAny};
use crate::zip::{Zip, local_names_include};
use crate::{Budget, Canvas, Error, Kind, Layer, MAX_DEPTH, Outline, Rendition, RenditionKind, counted, malformed, parse, parse_versioned, too_new};
pub(crate) fn sniff(head: &[u8]) -> bool {
head.starts_with(b"PK\x03\x04") && (local_names_include(head, MANIFEST) || Zip::open(head).is_ok_and(|z| z.find(MANIFEST).is_some()))
}
/// The archive and its `manifest.json`.
fn manifest<'a>(bytes: &'a [u8], budget: &mut Budget) -> Result<(Zip<'a>, Vec<u8>), Error> {
let zip = Zip::open(bytes)?;
let entry = zip.find(MANIFEST).ok_or_else(|| malformed("the bundle has no manifest.json"))?;
let max = budget.limits.max_json.min(budget.limits.max_entry);
let json = zip.read(entry, max, budget, MANIFEST)?;
Ok((zip, json))
}
/// The version and the writer, read without the rest of the manifest, so that a manifest of a
/// newer format still reports them.
#[derive(Deserialize)]
struct Stamp {
format_version: u32,
#[serde(default)]
generator: Option<String>,
}
fn checked(version: u32) -> Result<u32, Error> {
if version == 0 {
return Err(malformed("format_version 0 is invalid"));
}
Ok(version)
}
/// The members of [`craft_pcraft_format::Manifest`], with the document as `D`, read without
/// building the manifest as a `serde_json::Value` first.
#[derive(Deserialize)]
struct Envelope<D> {
format_version: u32,
/// Required, and a string, as in the manifest; [`info`] reports it.
#[serde(rename = "generator")]
_generator: String,
document: D,
#[serde(default)]
thumbnail: Option<String>,
#[serde(default)]
composite: Option<String>,
}
/// The manifest's envelope, checked as [`craft_pcraft_format::read_manifest`] checks it: the
/// size, nesting and item count with [`craft_pcraft_format::check_manifest`], a version of 0 or
/// newer than [`FORMAT_VERSION`] refused, and a preview that names an entry other than [`THUMB`]
/// or [`COMPOSITE`] refused. A member that appears twice is refused.
fn read<D: DeserializeOwned>(json: &[u8], budget: &Budget) -> Result<Envelope<D>, Error> {
let mut limits = format::Limits::default();
limits.max_manifest_bytes = json.len();
limits.max_depth = budget.limits.max_depth.min(MAX_DEPTH) as usize;
format::check_manifest(json, &limits).map_err(|e| match e {
format::Error::LimitExceeded(m) => Error::LimitExceeded(m),
e => malformed(e),
})?;
let m: Envelope<D> = parse_versioned(json, budget.limits, Kind::PhotoCraft)?;
let version = checked(m.format_version)?;
if version > FORMAT_VERSION {
return Err(too_new(Kind::PhotoCraft, version));
}
for (member, entry, expected) in [("thumbnail", &m.thumbnail, THUMB), ("composite", &m.composite, COMPOSITE)] {
if let Some(entry) = entry
&& entry != expected
{
return Err(malformed(format!("`{member}` names {entry:?}, not {expected:?}")));
}
}
Ok(m)
}
pub(crate) fn info(bytes: &[u8], budget: &mut Budget) -> Result<(u32, Option<String>), Error> {
let (_, json) = manifest(bytes, budget)?;
let stamp: Stamp = parse(&json, budget.limits)?;
Ok((checked(stamp.format_version)?, stamp.generator))
}
pub(crate) fn renditions(bytes: &[u8], budget: &mut Budget) -> Result<Vec<Rendition>, Error> {
let (zip, json) = manifest(bytes, budget)?;
let m: Envelope<IgnoredAny> = read(&json, budget)?;
let mut out = vec![];
for (kind, path) in [(RenditionKind::Thumbnail, m.thumbnail), (RenditionKind::Composite, m.composite)] {
let Some(path) = path else { continue };
let entry = zip.find(&path).ok_or_else(|| malformed(format!("the manifest names `{path}`, which the bundle does not have")))?;
let max = budget.limits.max_entry;
let bytes = zip.read(entry, max, budget, &path)?;
out.push(Rendition { kind, mime: "image/png".into(), bytes });
}
Ok(out)
}
#[derive(Deserialize)]
struct DocM {
size: SizeM,
resolution_dpi: f32,
mode: String,
depth: String,
/// Bottom to top.
#[serde(deserialize_with = "counted")]
layers: Vec<LayerM>,
}
#[derive(Deserialize)]
struct SizeM {
width: u32,
height: u32,
}
#[derive(Deserialize)]
struct LayerM {
name: String,
visible: bool,
blend: String,
opacity: f32,
content: ContentM,
}
#[derive(Deserialize)]
struct ContentM {
kind: String,
#[serde(default, deserialize_with = "counted")]
children: Vec<LayerM>,
}
impl From<LayerM> for Layer {
fn from(l: LayerM) -> Self {
Layer {
name: l.name,
kind: l.content.kind,
visible: l.visible,
opacity: Some(l.opacity),
blend: Some(l.blend),
children: l.content.children.into_iter().map(Layer::from).collect(),
}
}
}
pub(crate) fn outline(bytes: &[u8], budget: &mut Budget) -> Result<Outline, Error> {
let (_, json) = manifest(bytes, budget)?;
let d = read::<DocM>(&json, budget)?.document;
Ok(Outline {
canvas: Some(Canvas { width: d.size.width, height: d.size.height, resolution_dpi: d.resolution_dpi, mode: d.mode, depth: d.depth }),
pages: vec![],
layers: d.layers.into_iter().map(Layer::from).collect(),
})
}
crates/doc/src/vectorcraft.rs
+302
View File
@@ -0,0 +1,302 @@
//! VectorCraft `.vectorcraft` documents: one JSON object, gzip-compressed when saved with Use
//! Compression:
//!
//! ```text
//! { "format": "vectorcraft", "version": 4, "generator": "VectorCraft 0.1.0",
//! "preview": { "mime": "image/png", "data": "<base64>" },
//! "pdf": { "mime": "application/pdf", "blob": [offset, length] },
//! "document": { "artboards": [...], "layers": [...], ... }, ... }
//! ```
//!
//! Since version 4, large members may follow the JSON after the marker `\n\0VCBLOBS\0`; such a
//! member gives `"blob": [offset, length]` (from the end of the marker) instead of `data`. In a
//! compressed file, the marker and the data after it are inside the gzip stream too. Files
//! written before the rename say `"format": "drawcraft"`. The envelope's members are
//! [`craft_vectorcraft_format::Head`] and [`craft_vectorcraft_format::Image`].
use std::borrow::Cow;
use std::io::Read;
use craft_vectorcraft_format::{BLOB_MAGIC, Head, Image, is_compressed};
use flate2::read::GzDecoder;
use serde::Deserialize;
use crate::{Budget, Error, Kind, Layer, Outline, Page, Rendition, RenditionKind, counted, malformed, parse, parse_versioned, too_new};
/// Bytes decompressed at a time.
const CHUNK: u64 = 64 << 10;
pub(crate) fn sniff(head: &[u8]) -> bool {
craft_vectorcraft_format::sniff(head)
}
/// Where the JSON ends and the data after the marker starts, when `text` holds a NUL. JSON holds
/// no raw NUL and the marker's second byte is one, so the JSON ends right before the first NUL;
/// a NUL that is not the marker's makes the file malformed.
fn marker(text: &[u8], nul: usize) -> Result<(usize, usize), Error> {
match nul.checked_sub(1) {
Some(at) if text.get(at..).is_some_and(|r| r.starts_with(BLOB_MAGIC)) => Ok((at, at + BLOB_MAGIC.len())),
_ => Err(malformed("the file holds a NUL byte outside the marker after its JSON")),
}
}
fn damaged(e: std::io::Error) -> Error {
malformed(format!("damaged compressed data: {e}"))
}
/// A file's JSON, and the data after it.
struct Unpacked<'a> {
json: Cow<'a, [u8]>,
tail: Tail<'a>,
}
/// The data after the marker.
enum Tail<'a> {
/// In an uncompressed file.
Slice(&'a [u8]),
/// Still in the gzip stream: `read` holds what was decompressed past the marker already,
/// of which `counted` bytes are counted against the budget.
Stream { dec: Box<GzDecoder<&'a [u8]>>, read: Vec<u8>, counted: usize },
}
impl Tail<'_> {
/// The data up to `end` (less where the file ends first), decompressing only that far.
fn up_to(&mut self, end: u64, budget: &mut Budget) -> Result<&[u8], Error> {
match self {
Tail::Slice(s) => Ok(s),
Tail::Stream { dec, read, counted } => {
let end = usize::try_from(end).map_err(|_| Error::LimitExceeded("the data after the JSON does not fit in memory".into()))?;
let had = read.len().min(end);
if had > *counted {
budget.take((had - *counted) as u64, "the decompressed document")?;
*counted = had;
}
if read.len() < end {
let want = ((end - read.len()) as u64).min(budget.left().saturating_add(1));
let n = (&mut **dec).take(want).read_to_end(read).map_err(damaged)?;
budget.take(n as u64, "the decompressed document")?;
*counted += n;
}
Ok(read.as_slice())
}
}
}
}
/// A file's JSON and where the data after it is. A compressed file is decompressed only as far
/// as the end of the marker, every byte used counted against the budget, and its JSON is
/// checked against `max_json` as it grows.
fn unpack<'a>(bytes: &'a [u8], budget: &mut Budget) -> Result<Unpacked<'a>, Error> {
const JSON: &str = "the document's JSON";
let too_long = |budget: &Budget| Error::LimitExceeded(format!("{JSON} is more than max_json ({} bytes)", budget.limits.max_json));
if !is_compressed(bytes) {
let (json, tail) = match bytes.iter().position(|&b| b == 0) {
Some(nul) => {
let (end, tail) = marker(bytes, nul)?;
(bytes.get(..end).unwrap_or_default(), bytes.get(tail..).unwrap_or_default())
}
None => (bytes, &[][..]),
};
budget.check_json(json.len() as u64, JSON)?;
return Ok(Unpacked { json: Cow::Borrowed(json), tail: Tail::Slice(tail) });
}
let mut dec = Box::new(GzDecoder::new(bytes));
let mut buf = Vec::new();
let mut split: Option<(usize, usize)> = None;
let mut nul: Option<usize> = None;
loop {
let start = buf.len();
let want = CHUNK.min(budget.left().saturating_add(1));
let n = (&mut *dec).take(want).read_to_end(&mut buf).map_err(damaged)?;
// The JSON ends before the first NUL: until there is one, all of it is JSON. The marker
// may end in a later chunk than its first NUL.
if nul.is_none() {
nul = buf.get(start..).and_then(|b| b.iter().position(|&c| c == 0)).map(|p| p + start);
}
if nul.unwrap_or(buf.len()).saturating_sub(1) as u64 > budget.limits.max_json {
return Err(too_long(budget));
}
let mut used = n;
if let Some(nul) = nul
&& buf.len() >= nul + BLOB_MAGIC.len() - 1
{
let s = marker(&buf, nul)?;
split = Some(s);
// The data after the marker in this chunk is counted when it is read.
used = s.1.saturating_sub(start).min(n);
}
budget.take(used as u64, "the decompressed document")?;
if (n as u64) < want || split.is_some() {
break;
}
}
let (json_len, tail_at) = match (split, nul) {
(Some(s), _) => s,
// The stream ended inside the marker.
(None, Some(nul)) => marker(&buf, nul)?,
(None, None) => (buf.len(), buf.len()),
};
budget.check_json(json_len as u64, JSON)?;
let read = if tail_at <= buf.len() { buf.split_off(tail_at) } else { vec![] };
buf.truncate(json_len);
Ok(Unpacked { json: Cow::Owned(buf), tail: Tail::Stream { dec, read, counted: 0 } })
}
/// The format name, the version and the writer, read without the rest of the file, so that a file
/// of a newer version still reports them.
#[derive(Deserialize)]
struct Stamp {
format: String,
version: u32,
#[serde(default)]
generator: Option<String>,
}
/// The file's [`Stamp`], refused when the format is not VectorCraft's, and when the version is
/// newer than this crate reads unless `newer` is true.
fn stamp(json: &[u8], budget: &Budget, newer: bool) -> Result<Stamp, Error> {
let s: Stamp = if newer { parse(json, budget.limits)? } else { parse_versioned(json, budget.limits, Kind::VectorCraft)? };
match craft_vectorcraft_format::check(&s.format, s.version) {
Ok(()) => Ok(s),
Err(craft_vectorcraft_format::Error::Format(format)) => Err(malformed(format!("format is `{format}`"))),
Err(craft_vectorcraft_format::Error::TooNew(_)) if newer => Ok(s),
Err(craft_vectorcraft_format::Error::TooNew(version)) => Err(too_new(Kind::VectorCraft, version)),
Err(e) => Err(malformed(e)),
}
}
pub(crate) fn info(bytes: &[u8], budget: &mut Budget) -> Result<(u32, Option<String>), Error> {
let json = unpack(bytes, budget)?.json;
let s = stamp(&json, budget, true)?;
Ok((s.version, s.generator))
}
/// Check the entry's size against `max_entry` and count it; for an entry after the JSON, the
/// offset where it ends.
fn take(image: &Image, budget: &mut Budget, what: &str) -> Result<Option<u64>, Error> {
budget.take_entry(image.len(), what)?;
Ok(image.blob.map(|[offset, len]| offset.saturating_add(len)))
}
/// The entry's bytes, after [`take`] has checked its size.
fn entry_bytes(image: &Image, tail: &[u8], what: &str) -> Result<Vec<u8>, Error> {
image.bytes_at_most(tail, u64::MAX).ok_or_else(|| match image.blob {
Some(_) => malformed(format!("the file ends before its {what}")),
None => malformed(format!("the {what} is not valid base64")),
})
}
pub(crate) fn renditions(bytes: &[u8], budget: &mut Budget) -> Result<Vec<Rendition>, Error> {
let Unpacked { json, mut tail } = unpack(bytes, budget)?;
stamp(&json, budget, false)?;
let head: Head = parse(&json, budget.limits)?;
// A member without a MIME type gives no rendition: a preview may be PNG or JPEG.
let members: Vec<_> = [(RenditionKind::Thumbnail, head.preview, "preview"), (RenditionKind::Pdf, head.pdf, "PDF")]
.into_iter()
.filter_map(|(kind, m, what)| Some((kind, m.filter(|m| !m.mime.is_empty())?, what)))
.collect();
// Every size is checked before any data after the JSON is decompressed.
let mut end = 0u64;
for (_, m, what) in &members {
end = end.max(take(m, budget, what)?.unwrap_or(0));
}
let tail = if end > 0 { tail.up_to(end, budget)? } else { &[] };
members
.into_iter()
.map(|(kind, m, what)| {
let bytes = entry_bytes(&m, tail, what)?;
Ok(Rendition { kind, mime: m.mime, bytes })
})
.collect()
}
#[derive(Deserialize)]
struct WithDocument {
document: DocM,
}
#[derive(Deserialize)]
struct DocM {
#[serde(default, deserialize_with = "counted")]
artboards: Vec<ArtboardM>,
/// Bottom first.
#[serde(default, deserialize_with = "counted")]
layers: Vec<NodeM>,
}
#[derive(Deserialize)]
struct ArtboardM {
#[serde(default)]
name: String,
rect: RectM,
}
#[derive(Deserialize)]
struct RectM {
x0: f64,
y0: f64,
x1: f64,
y1: f64,
}
fn yes() -> bool {
true
}
fn one() -> f32 {
1.0
}
/// A node, written without the fields at their defaults.
#[derive(Deserialize)]
struct NodeM {
#[serde(default)]
name: Option<String>,
#[serde(default = "yes")]
visible: bool,
#[serde(default = "one")]
opacity: f32,
#[serde(default)]
blend: Option<String>,
kind: KindM,
}
#[derive(Deserialize)]
struct KindM {
#[serde(rename = "type")]
kind: String,
}
pub(crate) fn outline(bytes: &[u8], budget: &mut Budget) -> Result<Outline, Error> {
let json = unpack(bytes, budget)?.json;
stamp(&json, budget, false)?;
let f: WithDocument = parse(&json, budget.limits)?;
let pages = f
.document
.artboards
.into_iter()
.map(|a| Page {
name: a.name,
x: a.rect.x0.min(a.rect.x1),
y: a.rect.y0.min(a.rect.y1),
width: (a.rect.x1 - a.rect.x0).abs(),
height: (a.rect.y1 - a.rect.y0).abs(),
timing: None,
layers: vec![],
})
.collect();
let layers = f
.document
.layers
.into_iter()
.map(|n| Layer {
name: n.name.unwrap_or_default(),
kind: n.kind.kind,
visible: n.visible,
opacity: Some(n.opacity),
blend: Some(n.blend.unwrap_or_else(|| "Normal".into())),
children: vec![],
})
.collect();
Ok(Outline { canvas: None, pages, layers })
}
crates/doc/src/zip.rs
+183
View File
@@ -0,0 +1,183 @@
//! A ZIP reader for `.pcraft` bundles: STORE and DEFLATE entries, found through the central
//! directory, CRC-checked. No ZIP64, no encryption, no archives split across disks.
use std::io::Read;
use std::ops::Range;
use crate::{Budget, Error, malformed};
const LOCAL_SIG: u32 = 0x0403_4b50;
const CENTRAL_SIG: u32 = 0x0201_4b50;
const EOCD_SIG: u32 = 0x0605_4b50;
const ZIP64_LOCATOR_SIG: u32 = 0x0706_4b50;
const EOCD_LEN: usize = 22;
const CENTRAL_LEN: usize = 46;
const LOCAL_LEN: usize = 30;
fn u16_at(b: &[u8], at: usize) -> Option<u16> {
let s = b.get(at..at.checked_add(2)?)?;
Some(u16::from_le_bytes([*s.first()?, *s.get(1)?]))
}
fn u32_at(b: &[u8], at: usize) -> Option<u32> {
let s = b.get(at..at.checked_add(4)?)?;
Some(u32::from_le_bytes([*s.first()?, *s.get(1)?, *s.get(2)?, *s.get(3)?]))
}
fn truncated() -> Error {
malformed("truncated ZIP archive")
}
/// One entry of the central directory.
pub(crate) struct Entry {
name: Range<usize>,
flags: u16,
method: u16,
crc: u32,
compressed: usize,
size: u64,
/// Where the entry's data starts.
data: usize,
}
pub(crate) struct Zip<'a> {
bytes: &'a [u8],
entries: Vec<Entry>,
}
impl<'a> Zip<'a> {
pub(crate) fn open(bytes: &'a [u8]) -> Result<Self, Error> {
let last = bytes.len().checked_sub(EOCD_LEN).ok_or_else(|| malformed("not a ZIP archive (too short)"))?;
// The comment may itself hold the signature: accept only a record whose comment ends the
// file and whose central directory ends where the record starts.
let eocd = (last.saturating_sub(65535)..=last)
.rev()
.find(|&i| {
let comment_end = u16_at(bytes, i + 20).and_then(|n| i.checked_add(EOCD_LEN)?.checked_add(usize::from(n)));
let dir_end = u32_at(bytes, i + 16)
.zip(u32_at(bytes, i + 12))
.and_then(|(off, size)| usize::try_from(off).ok()?.checked_add(usize::try_from(size).ok()?));
u32_at(bytes, i) == Some(EOCD_SIG) && comment_end == Some(bytes.len()) && dir_end == Some(i)
})
.ok_or_else(|| {
let zip64 = (last.saturating_sub(65535 + 20)..=last.saturating_sub(20)).any(|i| u32_at(bytes, i) == Some(ZIP64_LOCATOR_SIG));
if zip64 {
Error::Unsupported("ZIP64 archives are not supported".into())
} else {
malformed("ZIP end of central directory not found")
}
})?;
let disk = u16_at(bytes, eocd + 4).ok_or_else(truncated)?;
let dir_disk = u16_at(bytes, eocd + 6).ok_or_else(truncated)?;
let count = usize::from(u16_at(bytes, eocd + 10).ok_or_else(truncated)?);
if disk != 0 || dir_disk != 0 {
return Err(Error::Unsupported("ZIP archives split across disks are not supported".into()));
}
let dir_size = u32_at(bytes, eocd + 12).ok_or_else(truncated)? as usize;
let dir = eocd - dir_size;
let mut entries = Vec::with_capacity(count.min(dir_size / CENTRAL_LEN));
let mut p = dir;
for _ in 0..count {
if u32_at(bytes, p) != Some(CENTRAL_SIG) || p + CENTRAL_LEN > eocd {
return Err(malformed("bad ZIP central directory entry"));
}
let field16 = |o: usize| u16_at(bytes, p + o).ok_or_else(truncated);
let field32 = |o: usize| u32_at(bytes, p + o).ok_or_else(truncated);
let (flags, method, crc) = (field16(8)?, field16(10)?, field32(16)?);
let (compressed, size) = (field32(20)?, field32(24)?);
let (name_len, extra_len, comment_len) = (usize::from(field16(28)?), usize::from(field16(30)?), usize::from(field16(32)?));
let local = field32(42)? as usize;
if compressed == u32::MAX || size == u32::MAX || local == u32::MAX as usize {
return Err(Error::Unsupported("ZIP64 entries are not supported".into()));
}
let name_start = p + CENTRAL_LEN;
let name_end = name_start + name_len;
let next = name_end + extra_len + comment_len;
if next > eocd {
return Err(malformed("ZIP central directory entry out of range"));
}
if u32_at(bytes, local) != Some(LOCAL_SIG) {
return Err(malformed("bad ZIP local header"));
}
let local_name = usize::from(u16_at(bytes, local + 26).ok_or_else(truncated)?);
let local_extra = usize::from(u16_at(bytes, local + 28).ok_or_else(truncated)?);
let data = local.checked_add(LOCAL_LEN + local_name + local_extra).ok_or_else(truncated)?;
let compressed = compressed as usize;
if data.checked_add(compressed).is_none_or(|end| end > bytes.len()) {
return Err(malformed("ZIP entry data out of range"));
}
entries.push(Entry { name: name_start..name_end, flags, method, crc, compressed, size: u64::from(size), data });
p = next;
}
Ok(Zip { bytes, entries })
}
/// The first entry named `name`.
pub(crate) fn find(&self, name: &str) -> Option<&Entry> {
self.entries.iter().find(|e| self.bytes.get(e.name.clone()) == Some(name.as_bytes()))
}
/// The entry's bytes, decompressed and CRC-checked; their size is checked against `max` and
/// counted against `budget` first.
pub(crate) fn read(&self, e: &Entry, max: u64, budget: &mut Budget, what: &str) -> Result<Vec<u8>, Error> {
if e.flags & 1 != 0 {
return Err(Error::Unsupported(format!("{what} is encrypted")));
}
if !matches!(e.method, 0 | 8) {
return Err(Error::Unsupported(format!("{what} uses ZIP compression method {}", e.method)));
}
if e.size > max {
return Err(Error::LimitExceeded(format!("{what} is {} bytes, more than the limit ({max})", e.size)));
}
budget.take(e.size, what)?;
let raw = self.bytes.get(e.data..e.data + e.compressed).ok_or_else(truncated)?;
let size = usize::try_from(e.size).map_err(|_| Error::LimitExceeded(format!("{what} does not fit in memory")))?;
let data = if e.method == 0 {
if e.compressed != size {
return Err(malformed(format!("{what} has the wrong size")));
}
raw.to_vec()
} else {
// DEFLATE expands at most 1032 times.
let mut out = Vec::with_capacity(size.min(raw.len().saturating_mul(1032)));
flate2::read::DeflateDecoder::new(raw).take(e.size + 1).read_to_end(&mut out).map_err(|err| malformed(format!("{what}: {err}")))?;
if out.len() != size {
return Err(malformed(format!("{what} has the wrong size")));
}
out
};
let mut crc = flate2::Crc::new();
crc.update(&data);
if crc.sum() != e.crc {
return Err(malformed(format!("{what} failed its CRC check")));
}
Ok(data)
}
}
/// Does a local header in `head` name `name`? Walks the local headers from the start, for a
/// head too short to reach the central directory; stops at an entry whose size is only in a
/// data descriptor.
pub(crate) fn local_names_include(head: &[u8], name: &str) -> bool {
let mut p = 0usize;
while u32_at(head, p) == Some(LOCAL_SIG) {
let (Some(flags), Some(compressed), Some(name_len), Some(extra_len)) =
(u16_at(head, p + 6), u32_at(head, p + 18), u16_at(head, p + 26), u16_at(head, p + 28))
else {
return false;
};
let start = p + LOCAL_LEN;
let end = start + usize::from(name_len);
if head.get(start..end) == Some(name.as_bytes()) {
return true;
}
if flags & 8 != 0 {
return false;
}
p = match end.checked_add(usize::from(extra_len)).and_then(|q| q.checked_add(compressed as usize)) {
Some(q) => q,
None => return false,
};
}
false
}
crates/doc/tests/common/mod.rs
+222
View File
@@ -0,0 +1,222 @@
//! Synthetic documents built from scratch for the tests: hand-written JSON, a small literal PDF,
//! PNG files encoded here, ZIP archives and gzip streams assembled here.
#![allow(dead_code)]
use std::io::Write;
/// A one-page PDF with an empty page.
pub const PDF: &[u8] = b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 72 72] >>\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF\n";
pub const BLOB_MAGIC: &[u8] = b"\n\0VCBLOBS\0";
pub fn gzip(bytes: &[u8]) -> Vec<u8> {
let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
gz.write_all(bytes).unwrap();
gz.finish().unwrap()
}
pub fn deflate(bytes: &[u8]) -> Vec<u8> {
let mut d = flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default());
d.write_all(bytes).unwrap();
d.finish().unwrap()
}
pub fn crc32(bytes: &[u8]) -> u32 {
let mut c = flate2::Crc::new();
c.update(bytes);
c.sum()
}
pub fn base64(data: &[u8]) -> String {
const A: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
let mut out = String::new();
for c in data.chunks(3) {
let n = (u32::from(c[0]) << 16) | (u32::from(*c.get(1).unwrap_or(&0)) << 8) | u32::from(*c.get(2).unwrap_or(&0));
out.push(A[(n >> 18) as usize & 63] as char);
out.push(A[(n >> 12) as usize & 63] as char);
out.push(if c.len() > 1 { A[(n >> 6) as usize & 63] as char } else { '=' });
out.push(if c.len() > 2 { A[n as usize & 63] as char } else { '=' });
}
out
}
/// A gray `w` × `h` PNG with every pixel `value`.
pub fn png(w: u32, h: u32, value: u8) -> Vec<u8> {
fn chunk(out: &mut Vec<u8>, kind: &[u8; 4], data: &[u8]) {
out.extend((data.len() as u32).to_be_bytes());
let mut body = kind.to_vec();
body.extend(data);
out.extend(&body);
out.extend(crc32(&body).to_be_bytes());
}
let mut ihdr = w.to_be_bytes().to_vec();
ihdr.extend(h.to_be_bytes());
ihdr.extend([8, 0, 0, 0, 0]);
let mut raw = vec![];
for _ in 0..h {
raw.push(0);
raw.extend(std::iter::repeat_n(value, w as usize));
}
let mut z = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default());
z.write_all(&raw).unwrap();
let mut out = b"\x89PNG\r\n\x1a\n".to_vec();
chunk(&mut out, b"IHDR", &ihdr);
chunk(&mut out, b"IDAT", &z.finish().unwrap());
chunk(&mut out, b"IEND", &[]);
out
}
/// One ZIP entry: its name, its bytes, and whether it is DEFLATE-compressed.
pub struct Entry<'a> {
pub name: &'a str,
pub data: &'a [u8],
pub deflate: bool,
}
pub fn store<'a>(name: &'a str, data: &'a [u8]) -> Entry<'a> {
Entry { name, data, deflate: false }
}
pub fn deflated<'a>(name: &'a str, data: &'a [u8]) -> Entry<'a> {
Entry { name, data, deflate: true }
}
/// A ZIP archive of `entries`, in order, with a central directory and an end record.
pub fn zip(entries: &[Entry]) -> Vec<u8> {
let mut out = vec![];
let mut central = vec![];
for e in entries {
let payload = if e.deflate { deflate(e.data) } else { e.data.to_vec() };
let crc = crc32(e.data);
let method: u16 = if e.deflate { 8 } else { 0 };
let offset = out.len() as u32;
let fields = |out: &mut Vec<u8>| {
out.extend(20u16.to_le_bytes());
out.extend(0x0800u16.to_le_bytes());
out.extend(method.to_le_bytes());
out.extend(0u16.to_le_bytes());
out.extend(0x21u16.to_le_bytes());
out.extend(crc.to_le_bytes());
out.extend((payload.len() as u32).to_le_bytes());
out.extend((e.data.len() as u32).to_le_bytes());
out.extend((e.name.len() as u16).to_le_bytes());
out.extend(0u16.to_le_bytes());
};
out.extend(0x0403_4b50u32.to_le_bytes());
fields(&mut out);
out.extend(e.name.as_bytes());
out.extend(&payload);
central.extend(0x0201_4b50u32.to_le_bytes());
central.extend(20u16.to_le_bytes());
fields(&mut central);
// Comment length, disk, internal and external attributes.
central.extend([0u8; 10]);
central.extend(offset.to_le_bytes());
central.extend(e.name.as_bytes());
}
let dir_offset = out.len() as u32;
out.extend(&central);
out.extend(0x0605_4b50u32.to_le_bytes());
out.extend([0u8; 4]);
out.extend((entries.len() as u16).to_le_bytes());
out.extend((entries.len() as u16).to_le_bytes());
out.extend((central.len() as u32).to_le_bytes());
out.extend(dir_offset.to_le_bytes());
out.extend(0u16.to_le_bytes());
out
}
/// A VectorCraft document member holding `bytes` as base64.
pub fn inline(mime: &str, bytes: &[u8]) -> String {
format!(r#"{{"mime":"{mime}","data":"{}"}}"#, base64(bytes))
}
/// A VectorCraft document member pointing at the data after the JSON.
pub fn blob(mime: &str, offset: usize, len: usize) -> String {
format!(r#"{{"mime":"{mime}","blob":[{offset},{len}]}}"#)
}
/// The JSON of a VectorCraft file: `members` go between `generator` and `document`.
pub fn vectorcraft_json(version: u32, members: &str) -> String {
let document = r#"{"version":1,"title":"","units":"Points","color_mode":"Rgb",
"artboards":[{"id":1,"name":"Front","rect":{"x0":0.0,"y0":0.0,"x1":612.0,"y1":792.0},"show_center_mark":false,"show_cross_hairs":false},
{"id":2,"name":"Back","rect":{"x0":700.0,"y0":800.0,"x1":650.0,"y1":700.0},"show_center_mark":false,"show_cross_hairs":false}],
"layers":[{"id":1,"name":"Art","kind":{"type":"layer","color":{"Preset":0},"template":false,"printable":true,
"children":[{"id":3,"kind":{"type":"group","children":[{"id":4,"kind":{"type":"path","path":{"subpaths":[]}}}]}}]}},
{"id":2,"name":"Notes","visible":false,"opacity":0.5,"blend":"Multiply","kind":{"type":"layer","color":{"Preset":1},"template":false,"printable":false,"children":[]}}],
"next_id":5}"#;
let sep = if members.is_empty() { "" } else { "," };
format!(r#"{{"format":"vectorcraft","version":{version},"generator":"VectorCraft 0.1.0"{sep}{members},"document":{document},"images":{{}}}}"#)
}
/// A PhotoCraft layer as the manifest writes it.
pub fn pcraft_layer(name: &str, visible: bool, blend: &str, opacity: f32, content: &str) -> String {
format!(
r#"{{"id":1,"name":"{name}","visible":{visible},"locks":{{}},"blend":"{blend}","opacity":{opacity},"fill_opacity":1.0,"clipped":false,"mask":null,
"effects":{{"enabled":true,"items":[],"psd_raw":null}},"label":"None","content":{content},"psd_blocks":[],"psd_id":null,"fill_cache":null}}"#
)
}
pub fn raster() -> &'static str {
r#"{"kind":"raster","surface":{"format":{"mode":"Rgb","sample":"U8","alpha":true},"default":"00000000","tiles":[]}}"#
}
/// A PhotoCraft manifest with `layers` (a JSON array) and the preview paths given.
pub fn pcraft_manifest(version: u32, layers: &str, previews: &str) -> String {
format!(
r#"{{"format_version":{version},"generator":"photocraft-format 0.1.0",
"document":{{"id":1,"name":"Untitled","size":{{"width":640,"height":480}},"resolution_dpi":300.0,"mode":"Rgb","depth":"U16",
"icc_profile":null,"layers":{layers},"channels":[],"guides":{{}},"selection":null,"metadata":{{}},"global_light":{{}}}}{previews}}}"#
)
}
/// A `.pcraft` bundle: a tile, the previews, and the manifest last.
pub fn pcraft(thumb: &[u8], composite: &[u8]) -> Vec<u8> {
let layers = format!(
"[{},{}]",
pcraft_layer("Background", true, "Normal", 1.0, raster()),
pcraft_layer(
"Group",
false,
"Screen",
0.25,
&format!(r#"{{"kind":"group","expanded":true,"children":[{}]}}"#, pcraft_layer("Inner", true, "Multiply", 0.5, raster()))
)
);
let manifest = pcraft_manifest(1, &layers, r#","thumbnail":"thumb.png","composite":"composite/preview.png""#);
zip(&[
store("tiles/0123.zst", b"not really zstd"),
store("thumb.png", thumb),
deflated("composite/preview.png", composite),
deflated("manifest.json", manifest.as_bytes()),
])
}
/// An EffectCraft project with a folder, a footage item and two compositions.
pub fn ecproj(schema: u32) -> String {
format!(
r#"{{
"savedBy": "0.12.0",
"activeComp": 10,
"schema": {schema},
"settings": {{}},
"items": {{
"1": {{"id": 1, "name": "Footage", "label": 0, "kind": {{"type": "Folder"}}}},
"2": {{"id": 2, "name": "clip.mov", "label": 0, "parent": 1,
"kind": {{"type": "Footage", "path": "clip.mov", "kind": "Video", "width": 640, "height": 360, "pixel_aspect": 1.0,
"frame_rate": {{"num": 30, "den": 1}}, "duration": 2540160000000}}}},
"10": {{"id": 10, "name": "Main", "label": 0,
"kind": {{"type": "Comp", "width": 1920, "height": 1080, "pixel_aspect": 1.0, "frame_rate": {{"num": 24000, "den": 1001}},
"duration": 1270080000000, "display_start": 0, "background": [0.0, 0.0, 0.0], "work_area": [0, 1270080000000],
"layers": [
{{"id": 1, "name": "Title", "source": {{"type": "Text"}}, "switches": {{"video": true, "audio": false}}, "blend_mode": "Add"}},
{{"id": 2, "name": "Plate", "source": {{"type": "Footage", "item": 2}}, "switches": {{"video": false, "audio": true}}, "blend_mode": "Normal"}}
]}}}},
"3": {{"id": 3, "name": "Intro", "label": 0,
"kind": {{"type": "Comp", "width": 640, "height": 360, "pixel_aspect": 1.0, "frame_rate": {{"num": 30, "den": 1}},
"duration": 254016000000, "layers": []}}}}
}},
"next_id": 11
}}"#
)
}
crates/doc/tests/envelopes.rs
+138
View File
@@ -0,0 +1,138 @@
//! Documents whose envelope is written with the format crates' own types read through craft-doc,
//! and envelopes those crates refuse are refused here too.
mod common;
use std::collections::BTreeMap;
use common::*;
use craft_doc::{Error, Kind, Limits, RenditionKind, info, outline, renditions, sniff};
fn limits() -> Limits {
Limits::default()
}
fn vectorcraft_document() -> serde_json::Value {
serde_json::json!({
"artboards": [{"id": 1, "name": "Front", "rect": {"x0": 0.0, "y0": 0.0, "x1": 612.0, "y1": 792.0}}],
"layers": [{"id": 1, "name": "Art", "kind": {"type": "layer", "children": []}}],
})
}
#[test]
fn a_vectorcraft_file_written_with_the_envelope_types_reads() {
let preview = png(16, 16, 3);
let pdf: Vec<u8> = PDF.iter().copied().cycle().take(craft_vectorcraft_format::INLINE_MAX + 1).collect();
let mut tail = craft_vectorcraft_format::Tail::new(craft_vectorcraft_format::VERSION);
let mut file = craft_vectorcraft_format::File::new(vectorcraft_document());
file.generator = Some("VectorCraft 0.1.0".into());
file.preview = Some(tail.image("image/png", &preview, false));
file.pdf = Some(tail.image("application/pdf", &pdf, false));
assert!(file.pdf.as_ref().unwrap().blob.is_some() && file.preview.as_ref().unwrap().blob.is_none());
let mut bytes = serde_json::to_vec(&file).unwrap();
tail.write_to(&mut bytes).unwrap();
for bytes in [bytes.clone(), gzip(&bytes)] {
assert_eq!(sniff(&bytes), Some(Kind::VectorCraft));
let i = info(&bytes, &limits()).unwrap();
assert_eq!((i.version, i.generator.as_deref()), (craft_vectorcraft_format::VERSION, Some("VectorCraft 0.1.0")));
let r = renditions(&bytes, &limits()).unwrap();
assert_eq!((r[0].kind, r[0].bytes.as_slice()), (RenditionKind::Thumbnail, preview.as_slice()));
assert_eq!((r[1].kind, r[1].bytes.as_slice()), (RenditionKind::Pdf, pdf.as_slice()));
let o = outline(&bytes, &limits()).unwrap();
assert_eq!((o.pages[0].name.as_str(), o.pages[0].width, o.layers[0].name.as_str()), ("Front", 612.0, "Art"));
}
}
#[test]
fn a_pcraft_manifest_written_with_the_envelope_type_reads() {
let document = serde_json::json!({
"size": {"width": 64, "height": 32}, "resolution_dpi": 72.0, "mode": "Rgb", "depth": "U8",
"layers": [{"name": "Background", "visible": true, "blend": "Normal", "opacity": 1.0, "content": {"kind": "raster"}}],
});
let mut manifest = craft_pcraft_format::Manifest::new("photocraft-format 0.1.0", document);
manifest.thumbnail = Some(craft_pcraft_format::THUMB.into());
let json = serde_json::to_vec(&manifest).unwrap();
let thumb = png(4, 4, 1);
let bundle = zip(&[store(craft_pcraft_format::MANIFEST, &json), store(craft_pcraft_format::THUMB, &thumb)]);
assert_eq!(sniff(&bundle), Some(Kind::PhotoCraft));
assert_eq!(info(&bundle, &limits()).unwrap().generator.as_deref(), Some("photocraft-format 0.1.0"));
let r = renditions(&bundle, &limits()).unwrap();
assert_eq!((r.len(), r[0].kind, r[0].bytes.as_slice()), (1, RenditionKind::Thumbnail, thumb.as_slice()));
let o = outline(&bundle, &limits()).unwrap();
assert_eq!((o.canvas.unwrap().width, o.layers[0].name.as_str()), (64, "Background"));
}
#[test]
fn an_ecproj_file_written_with_the_envelope_type_reads() {
let view = BTreeMap::from([("activeComp", 1)]);
let project = serde_json::json!({
"schema": craft_ecproj_format::SCHEMA_VERSION,
"items": {"1": {"id": 1, "name": "Main", "kind": {"type": "Comp", "width": 1280, "height": 720, "layers": []}}},
});
let text = craft_ecproj_format::File::new("0.12.0", &view, &project).to_json().unwrap();
assert_eq!(sniff(text.as_bytes()), Some(Kind::EffectCraft));
let i = info(text.as_bytes(), &limits()).unwrap();
assert_eq!((i.version, i.generator.as_deref()), (craft_ecproj_format::SCHEMA_VERSION, Some("0.12.0")));
let o = outline(text.as_bytes(), &limits()).unwrap();
assert_eq!((o.pages[0].name.as_str(), o.pages[0].width), ("Main", 1280.0));
}
#[test]
fn a_pcraft_manifest_without_a_generator_is_malformed() {
let manifest = pcraft_manifest(1, "[]", "").replacen(r#""generator":"photocraft-format 0.1.0","#, "", 1);
let bundle = zip(&[store("manifest.json", manifest.as_bytes())]);
assert_eq!(info(&bundle, &limits()).unwrap().generator, None);
assert!(matches!(renditions(&bundle, &limits()), Err(Error::Malformed(_))));
assert!(matches!(outline(&bundle, &limits()), Err(Error::Malformed(_))));
}
#[test]
fn a_pcraft_preview_naming_another_entry_is_malformed() {
let manifest = pcraft_manifest(1, "[]", r#","thumbnail":"../thumb.png""#);
let bundle = zip(&[store("manifest.json", manifest.as_bytes()), store("../thumb.png", &png(1, 1, 0))]);
assert!(matches!(renditions(&bundle, &limits()), Err(Error::Malformed(_))));
assert!(matches!(outline(&bundle, &limits()), Err(Error::Malformed(_))));
}
#[test]
fn an_ecproj_nested_to_max_depth_reads() {
let deep = |levels: usize| format!(r#"{{"savedBy":"0.12.0","schema":2,"items":{{}},"x":{}{}}}"#, "[".repeat(levels), "]".repeat(levels));
let levels = craft_doc::MAX_DEPTH as usize - 1;
let objects = format!(r#"{{"savedBy":"0.12.0","schema":2,"items":{{}},"x":{}1{}}}"#, r#"{"a":"#.repeat(levels), "}".repeat(levels));
for ok in [deep(levels), objects] {
assert_eq!(info(ok.as_bytes(), &limits()).unwrap().version, 2);
assert_eq!(renditions(ok.as_bytes(), &limits()).unwrap(), vec![]);
assert_eq!(outline(ok.as_bytes(), &limits()).unwrap().pages.len(), 0);
}
}
#[test]
fn a_pcraft_manifest_with_a_repeated_member_or_a_late_newer_version_is_refused() {
let repeated = pcraft_manifest(1, "[]", r#","thumbnail":"thumb.png","thumbnail":"thumb.png""#);
let bundle = zip(&[store("manifest.json", repeated.as_bytes())]);
assert!(matches!(renditions(&bundle, &limits()), Err(Error::Malformed(_))));
let late = r#"{"generator":"photocraft-format 9","document":"elsewhere","thumbnail":7,"format_version":2}"#;
let bundle = zip(&[store("manifest.json", late.as_bytes())]);
let refused = Error::TooNew { kind: Kind::PhotoCraft, version: 2, supported: Kind::PhotoCraft.supported_version() };
assert_eq!(renditions(&bundle, &limits()), Err(refused.clone()));
assert_eq!(outline(&bundle, &limits()), Err(refused));
}
#[test]
fn a_vectorcraft_member_without_a_mime_type_is_skipped() {
let file = vectorcraft_json(4, &format!(r#""preview":{{"data":"{}"}}"#, base64(&png(1, 1, 0))));
assert!(renditions(file.as_bytes(), &limits()).unwrap().is_empty());
let empty = vectorcraft_json(4, &format!(r#""preview":{},"pdf":{}"#, inline("", &png(1, 1, 0)), inline("", PDF)));
assert!(renditions(empty.as_bytes(), &limits()).unwrap().is_empty());
}
#[test]
fn a_format_member_inside_another_member_is_not_vectorcraft() {
let nested = br#"{"meta":{"format":"vectorcraft"},"version":4}"#;
assert_eq!(sniff(nested), None);
let second = br#"{"version":4,"format":"vectorcraft","document":{}}"#;
assert_eq!(sniff(second), Some(Kind::VectorCraft));
}
crates/doc/tests/formats.rs
+264
View File
@@ -0,0 +1,264 @@
//! Each format's documents, built from scratch: what `sniff`, `info`, `renditions` and `outline`
//! return for them.
mod common;
use common::*;
use craft_doc::{Error, Kind, Limits, RenditionKind, info, outline, renditions, sniff};
fn limits() -> Limits {
Limits::default()
}
#[test]
fn sniff_tells_the_three_formats() {
let vc = vectorcraft_json(4, "");
assert_eq!(sniff(vc.as_bytes()), Some(Kind::VectorCraft));
assert_eq!(sniff(&gzip(vc.as_bytes())), Some(Kind::VectorCraft));
let legacy = vc.replacen("\"vectorcraft\"", "\"drawcraft\"", 1);
assert_eq!(sniff(legacy.as_bytes()), Some(Kind::VectorCraft));
// A gzip stream cut short still shows its start.
let gz = gzip(vc.as_bytes());
assert_eq!(sniff(gz.get(..gz.len() - 8).unwrap()), Some(Kind::VectorCraft));
let pc = pcraft(&png(4, 4, 9), &png(8, 8, 9));
assert_eq!(sniff(&pc), Some(Kind::PhotoCraft), "the manifest last, found through the central directory");
let first = zip(&[store("manifest.json", b"{}"), store("thumb.png", &png(1, 1, 0))]);
assert_eq!(sniff(first.get(..60).unwrap()), Some(Kind::PhotoCraft), "the manifest first, in a short head");
let second = zip(&[store("tiles/a.zst", b"x"), store("manifest.json", b"{}")]);
assert_eq!(sniff(second.get(..100).unwrap()), Some(Kind::PhotoCraft), "a later local header in the head");
assert_eq!(sniff(ecproj(2).as_bytes()), Some(Kind::EffectCraft));
let old = r#"{"schema":1,"settings":{},"items":{},"next_id":1}"#;
assert_eq!(sniff(old.as_bytes()), Some(Kind::EffectCraft), "saved before savedBy");
}
#[test]
fn sniff_refuses_other_files() {
let plain_zip = zip(&[store("word/document.xml", b"<w/>"), deflated("readme.txt", b"hello hello hello")]);
let behind = zip(&[store("tiles/a.zst", &[7; 4096]), store("manifest.json", b"{}")]);
for (what, bytes) in [
("empty", &b""[..]),
("a PDF", PDF),
("a PNG", &png(2, 2, 1)),
("a ZIP without a manifest", &plain_zip),
("a manifest past the head", behind.get(..100).unwrap()),
("other JSON", br#"{"name":"x","items":[],"schema":"y"}"#),
("JSON naming the format in a value", br#"{"title":"vectorcraft"}"#),
("another format", br#"{"format":"vectorcraftx","version":4}"#),
("a JSON array", br#"["vectorcraft"]"#),
("savedBy without schema", br#"{"savedBy":"1.0","items":{}}"#),
("savedBy with schema only in a nested object", br#"{"savedBy":"1.0","metadata":{"schema":1}}"#),
("savedBy with schema only in a value", br#"{"savedBy":"1.0","note":"\"schema\""}"#),
("schema with items only in a nested object", br#"{"schema":1,"meta":{"items":[]}}"#),
("gzip of other JSON", &gzip(br#"{"format":"svg"}"#)),
("PK alone", b"PK\x03\x04"),
] {
assert_eq!(sniff(bytes), None, "{what}");
}
}
#[test]
fn info_reads_version_and_writer() {
let i = info(vectorcraft_json(3, "").as_bytes(), &limits()).unwrap();
assert_eq!((i.kind, i.version, i.generator.as_deref(), i.supported), (Kind::VectorCraft, 3, Some("VectorCraft 0.1.0"), true));
let i = info(&gzip(vectorcraft_json(4, "").as_bytes()), &limits()).unwrap();
assert_eq!((i.version, i.supported), (4, true));
let i = info(&pcraft(&png(1, 1, 0), &png(1, 1, 0)), &limits()).unwrap();
assert_eq!((i.kind, i.version, i.generator.as_deref(), i.supported), (Kind::PhotoCraft, 1, Some("photocraft-format 0.1.0"), true));
let i = info(ecproj(2).as_bytes(), &limits()).unwrap();
assert_eq!((i.kind, i.version, i.generator.as_deref(), i.supported), (Kind::EffectCraft, 2, Some("0.12.0"), true));
let i = info(br#"{"schema":1,"settings":{},"items":{},"next_id":1}"#, &limits()).unwrap();
assert_eq!((i.version, i.generator), (1, None));
assert_eq!(info(PDF, &limits()), Err(Error::Unrecognized));
}
#[test]
fn too_new_files_are_reported_and_refused() {
// Newer versions may change everything after the version: a member of a new shape.
let vc = vectorcraft_json(5, r#""pdf":"elsewhere""#);
let manifest = r#"{"format_version":2,"generator":"photocraft-format 9","document":"elsewhere","thumbnail":7}"#;
let pc = zip(&[store("manifest.json", manifest.as_bytes())]);
let ec = r#"{"savedBy":"9.0.0","schema":3,"items":[]}"#.to_string();
for (bytes, kind, version) in [(vc.into_bytes(), Kind::VectorCraft, 5), (pc, Kind::PhotoCraft, 2), (ec.into_bytes(), Kind::EffectCraft, 3)] {
let i = info(&bytes, &limits()).unwrap();
assert_eq!((i.kind, i.version, i.supported), (kind, version, false));
let refused = Error::TooNew { kind, version, supported: kind.supported_version() };
assert_eq!(renditions(&bytes, &limits()), Err(refused.clone()), "{kind:?}");
assert_eq!(outline(&bytes, &limits()), Err(refused), "{kind:?}");
}
}
#[test]
fn a_pcraft_format_version_of_zero_is_malformed() {
let pc = zip(&[store("manifest.json", pcraft_manifest(0, "[]", "").as_bytes())]);
assert!(matches!(info(&pc, &limits()), Err(Error::Malformed(_))));
assert!(matches!(outline(&pc, &limits()), Err(Error::Malformed(_))));
}
#[test]
fn vectorcraft_members_inline_as_base64() {
let preview = png(16, 8, 200);
let members = format!(r#""preview":{},"pdf":{}"#, inline("image/png", &preview), inline("application/pdf", PDF));
let json = vectorcraft_json(3, &members);
for bytes in [json.clone().into_bytes(), gzip(json.as_bytes())] {
let r = renditions(&bytes, &limits()).unwrap();
assert_eq!(r.len(), 2);
assert_eq!((r[0].kind, r[0].mime.as_str(), r[0].bytes.as_slice()), (RenditionKind::Thumbnail, "image/png", preview.as_slice()));
assert_eq!((r[1].kind, r[1].mime.as_str(), r[1].bytes.as_slice()), (RenditionKind::Pdf, "application/pdf", PDF));
}
}
#[test]
fn vectorcraft_members_after_the_json() {
// Version 4: the PDF after the marker, behind an image the document uses; the preview inline.
let image = vec![0xAB; 9000];
let preview = png(3, 3, 1);
let members = format!(r#""preview":{},"pdf":{}"#, inline("image/png", &preview), blob("application/pdf", image.len(), PDF.len()));
let mut file = vectorcraft_json(4, &members).into_bytes();
file.extend(BLOB_MAGIC);
file.extend(&image);
file.extend(PDF);
for bytes in [file.clone(), gzip(&file)] {
let r = renditions(&bytes, &limits()).unwrap();
assert_eq!(r.iter().map(|r| r.kind).collect::<Vec<_>>(), [RenditionKind::Thumbnail, RenditionKind::Pdf]);
assert_eq!(r[0].bytes, preview);
assert_eq!(r[1].bytes, PDF);
// The JSON alone answers info and outline.
assert_eq!(info(&bytes, &limits()).unwrap().version, 4);
assert_eq!(outline(&bytes, &limits()).unwrap().pages.len(), 2);
}
}
#[test]
fn a_compressed_marker_across_the_reader_s_64_kib_chunks() {
// The marker starts at every offset around the first 64 KiB boundary of the unpacked stream.
let base = vectorcraft_json(4, &format!(r#""pdf":{},"pad":"""#, blob("application/pdf", 0, PDF.len())));
for json_len in (64 << 10) - 12..(64 << 10) + 2 {
let json = base.replacen(r#""pad":"""#, &format!(r#""pad":"{}""#, "x".repeat(json_len - base.len())), 1);
assert_eq!(json.len(), json_len);
let mut file = json.into_bytes();
file.extend(BLOB_MAGIC);
file.extend(PDF);
let gz = gzip(&file);
assert_eq!(info(&gz, &limits()).unwrap().version, 4, "{json_len}");
assert_eq!(outline(&gz, &limits()).unwrap().pages.len(), 2, "{json_len}");
assert_eq!(renditions(&gz, &limits()).unwrap()[0].bytes, PDF, "{json_len}");
}
}
#[test]
fn a_padded_base64_member_fits_a_limit_of_its_own_size() {
for len in 1..=3 {
let data = vec![7u8; len];
let bytes = vectorcraft_json(3, &format!(r#""preview":{}"#, inline("image/png", &data))).into_bytes();
let mut l = limits();
l.max_entry = len as u64;
assert_eq!(renditions(&bytes, &l).unwrap()[0].bytes, data);
}
}
#[test]
fn vectorcraft_without_members_has_no_renditions() {
let json = vectorcraft_json(4, "");
assert_eq!(renditions(json.as_bytes(), &limits()).unwrap(), vec![]);
assert_eq!(renditions(&gzip(json.as_bytes()), &limits()).unwrap(), vec![]);
}
#[test]
fn vectorcraft_outline_has_artboards_and_top_level_layers() {
for bytes in [vectorcraft_json(4, "").into_bytes(), gzip(vectorcraft_json(4, "").as_bytes())] {
let o = outline(&bytes, &limits()).unwrap();
assert_eq!(o.canvas, None);
let pages: Vec<_> = o.pages.iter().map(|p| (p.name.as_str(), p.x, p.y, p.width, p.height)).collect();
assert_eq!(pages, [("Front", 0.0, 0.0, 612.0, 792.0), ("Back", 650.0, 700.0, 50.0, 100.0)]);
assert!(o.pages.iter().all(|p| p.timing.is_none() && p.layers.is_empty()));
let layers: Vec<_> = o.layers.iter().map(|l| (l.name.as_str(), l.kind.as_str(), l.visible, l.opacity, l.blend.as_deref())).collect();
assert_eq!(layers, [("Art", "layer", true, Some(1.0), Some("Normal")), ("Notes", "layer", false, Some(0.5), Some("Multiply"))]);
assert!(o.layers.iter().all(|l| l.children.is_empty()));
}
}
#[test]
fn pcraft_previews_stored_and_deflated_with_the_manifest_last() {
let (thumb, composite) = (png(4, 3, 10), png(40, 30, 20));
let r = renditions(&pcraft(&thumb, &composite), &limits()).unwrap();
assert_eq!(r.len(), 2);
assert_eq!((r[0].kind, r[0].mime.as_str(), &r[0].bytes), (RenditionKind::Thumbnail, "image/png", &thumb));
assert_eq!((r[1].kind, r[1].mime.as_str(), &r[1].bytes), (RenditionKind::Composite, "image/png", &composite));
}
#[test]
fn pcraft_without_previews_has_none() {
let pc = zip(&[store("manifest.json", pcraft_manifest(1, "[]", "").as_bytes())]);
assert_eq!(renditions(&pc, &limits()).unwrap(), vec![]);
let missing = zip(&[store("manifest.json", pcraft_manifest(1, "[]", r#","thumbnail":"thumb.png""#).as_bytes())]);
assert!(matches!(renditions(&missing, &limits()), Err(Error::Malformed(_))));
}
#[test]
fn pcraft_outline_has_the_canvas_and_layer_tree() {
let o = outline(&pcraft(&png(1, 1, 0), &png(1, 1, 0)), &limits()).unwrap();
let c = o.canvas.unwrap();
assert_eq!((c.width, c.height, c.resolution_dpi, c.mode.as_str(), c.depth.as_str()), (640, 480, 300.0, "Rgb", "U16"));
assert!(o.pages.is_empty());
let layers: Vec<_> = o.layers.iter().map(|l| (l.name.as_str(), l.kind.as_str(), l.visible, l.opacity, l.blend.as_deref())).collect();
assert_eq!(layers, [("Background", "raster", true, Some(1.0), Some("Normal")), ("Group", "group", false, Some(0.25), Some("Screen"))]);
let inner = &o.layers[1].children;
assert_eq!(inner.len(), 1);
assert_eq!((inner[0].name.as_str(), inner[0].blend.as_deref(), inner[0].opacity), ("Inner", Some("Multiply"), Some(0.5)));
}
#[test]
fn pcraft_groups_nest_to_the_depth_cap() {
// 3 levels to the layer list, 3 per group and 4 for the leaf: 397 of the 400 levels.
let mut layer = pcraft_layer("Leaf", true, "Normal", 1.0, raster());
for i in 0..130 {
layer = pcraft_layer(&format!("G{i}"), true, "Normal", 1.0, &format!(r#"{{"kind":"group","expanded":false,"children":[{layer}]}}"#));
}
let pc = zip(&[deflated("manifest.json", pcraft_manifest(1, &format!("[{layer}]"), "").as_bytes())]);
let o = outline(&pc, &limits()).unwrap();
let mut depth = 0;
let mut l = &o.layers[0];
while let Some(c) = l.children.first() {
depth += 1;
l = c;
}
assert_eq!((depth, l.name.as_str()), (130, "Leaf"));
}
#[test]
fn ecproj_has_no_renditions_and_lists_its_comps() {
let p = ecproj(2);
assert_eq!(renditions(p.as_bytes(), &limits()).unwrap(), vec![]);
let o = outline(p.as_bytes(), &limits()).unwrap();
assert!(o.canvas.is_none() && o.layers.is_empty());
// By item id (3 before 10), the folder and the footage left out.
let pages: Vec<_> = o.pages.iter().map(|p| (p.name.as_str(), p.width, p.height)).collect();
assert_eq!(pages, [("Intro", 640.0, 360.0), ("Main", 1920.0, 1080.0)]);
let t = o.pages[1].timing.unwrap();
assert_eq!((t.frame_rate_num, t.frame_rate_den, t.duration_seconds), (24000, 1001, 5.0));
assert_eq!(o.pages[0].timing.unwrap().duration_seconds, 1.0);
// Bottom first: the file lists the top layer first.
let layers: Vec<_> = o.pages[1].layers.iter().map(|l| (l.name.as_str(), l.kind.as_str(), l.visible, l.opacity, l.blend.as_deref())).collect();
assert_eq!(layers, [("Plate", "Footage", false, None, Some("Normal")), ("Title", "Text", true, None, Some("Add"))]);
}
#[test]
fn an_ecproj_comp_without_a_duration_has_no_timing() {
let p = ecproj(2).replacen(r#""duration": 254016000000, "#, "", 1);
let o = outline(p.as_bytes(), &limits()).unwrap();
assert_eq!((o.pages[0].name.as_str(), o.pages[0].timing), ("Intro", None));
assert!(o.pages[1].timing.is_some());
}
#[test]
fn kinds_name_their_extension_and_version() {
assert_eq!(
[Kind::PhotoCraft, Kind::VectorCraft, Kind::EffectCraft].map(|k| (k.extension(), k.supported_version())),
[("pcraft", 1), ("vectorcraft", 4), ("ecproj", 2)]
);
}
crates/doc/tests/hostile.rs
+353
View File
@@ -0,0 +1,353 @@
//! Truncated, oversized and hostile input: the reader returns an error within its `Limits` and
//! never panics.
mod common;
use common::*;
use craft_doc::{Error, Limits, info, outline, renditions, sniff};
use proptest::prelude::*;
fn small() -> Limits {
let mut l = Limits::default();
l.max_json = 64 << 10;
l.max_entry = 64 << 10;
l.max_total = 256 << 10;
l
}
fn limit_exceeded<T: std::fmt::Debug>(r: Result<T, Error>) -> bool {
matches!(r, Err(Error::LimitExceeded(_)))
}
/// Every call on `bytes`; none may panic.
fn exercise(bytes: &[u8], limits: &Limits) {
let _ = sniff(bytes);
let _ = info(bytes, limits);
let _ = renditions(bytes, limits);
let _ = outline(bytes, limits);
}
fn samples() -> Vec<Vec<u8>> {
let members = format!(r#""preview":{},"pdf":{}"#, inline("image/png", &png(2, 2, 3)), blob("application/pdf", 0, PDF.len()));
let mut v4 = vectorcraft_json(4, &members).into_bytes();
v4.extend(BLOB_MAGIC);
v4.extend(PDF);
vec![gzip(&v4), v4, pcraft(&png(3, 3, 1), &png(6, 6, 2)), ecproj(2).into_bytes()]
}
#[test]
fn every_truncation_is_an_error_not_a_panic() {
for s in samples() {
for n in 0..s.len() {
exercise(&s[..n], &small());
}
}
}
#[test]
fn a_blob_longer_than_max_entry_is_refused_before_it_is_copied() {
let mut bytes = vectorcraft_json(4, &format!(r#""pdf":{}"#, blob("application/pdf", 0, 1 << 40))).into_bytes();
bytes.extend(BLOB_MAGIC);
bytes.extend(PDF);
assert!(limit_exceeded(renditions(&bytes, &small())));
}
#[test]
fn a_blob_past_the_end_of_the_file_is_malformed() {
for at in [[PDF.len() as u64, 1], [u64::MAX, 1], [1, PDF.len() as u64]] {
let members = format!(r#""pdf":{{"mime":"application/pdf","blob":[{},{}]}}"#, at[0], at[1]);
let mut bytes = vectorcraft_json(4, &members).into_bytes();
bytes.extend(BLOB_MAGIC);
bytes.extend(PDF);
assert!(matches!(renditions(&bytes, &small()), Err(Error::Malformed(_))), "{at:?}");
}
}
#[test]
fn base64_past_max_entry_is_refused() {
let big = vec![1u8; 100 << 10];
let mut limits = small();
limits.max_json = 1 << 20;
let bytes = vectorcraft_json(3, &format!(r#""pdf":{}"#, inline("application/pdf", &big))).into_bytes();
assert!(limit_exceeded(renditions(&bytes, &limits)));
limits.max_entry = 1 << 20;
assert_eq!(renditions(&bytes, &limits).unwrap()[0].bytes, big);
let bad = vectorcraft_json(3, r#""pdf":{"mime":"application/pdf","data":"QU*D"}"#).into_bytes();
assert!(matches!(renditions(&bad, &limits), Err(Error::Malformed(_))));
}
#[test]
fn a_gzip_bomb_stops_at_the_limits() {
// 8 MiB of padding in the JSON, compressed to a few KiB.
let pad = "x".repeat(8 << 20);
let json = vectorcraft_json(4, &format!(r#""pad":"{pad}""#));
let gz = gzip(json.as_bytes());
assert!(gz.len() < 64 << 10);
assert!(limit_exceeded(info(&gz, &small())));
assert!(limit_exceeded(outline(&gz, &small())));
assert!(limit_exceeded(renditions(&gz, &small())));
// The data after the JSON counts against max_total as far as it is read.
let mut tail = vectorcraft_json(4, &format!(r#""pdf":{}"#, blob("application/pdf", 8 << 20, 1))).into_bytes();
tail.extend(BLOB_MAGIC);
tail.extend(vec![0u8; (8 << 20) + 1]);
let gz = gzip(&tail);
assert!(limit_exceeded(renditions(&gz, &small())));
let mut more = small();
more.max_total = 9 << 20;
assert_eq!(renditions(&gz, &more).unwrap()[0].bytes, [0]);
// info stops at the marker and reads the JSON only.
assert_eq!(info(&gz, &small()).unwrap().version, 4);
}
#[test]
fn a_compressed_member_past_max_entry_is_refused_before_the_tail_is_inflated() {
// A member declares 8 MiB; the data after the JSON is 8 MiB that compress to a few KiB.
let mut file = vectorcraft_json(4, &format!(r#""pdf":{}"#, blob("application/pdf", 0, 8 << 20))).into_bytes();
file.extend(BLOB_MAGIC);
file.extend(vec![1u8; 8 << 20]);
let gz = gzip(&file);
let mut limits = small();
// Enough for the tail, not for the member.
limits.max_total = 64 << 20;
match renditions(&gz, &limits) {
Err(Error::LimitExceeded(m)) => assert!(m.contains("max_entry"), "{m}"),
other => panic!("{other:?}"),
}
}
#[test]
fn a_nul_outside_the_marker_is_malformed_without_reading_on() {
// A NUL where the marker is not, then 8 MiB that compress to almost nothing.
let mut file = vectorcraft_json(4, "").into_bytes();
file.extend(b"\n\0NOTBLOBS\0");
file.extend(vec![0u8; 8 << 20]);
let mut limits = small();
limits.max_total = 1 << 20;
for bytes in [gzip(&file), file] {
assert!(matches!(info(&bytes, &limits), Err(Error::Malformed(_))));
assert!(matches!(renditions(&bytes, &limits), Err(Error::Malformed(_))));
assert!(matches!(outline(&bytes, &limits), Err(Error::Malformed(_))));
}
// A NUL as the very first byte of the decompressed stream.
let mut first = b"\0".to_vec();
first.extend(vectorcraft_json(4, "").as_bytes());
exercise(&gzip(&first), &limits);
}
#[test]
fn a_manifest_past_max_entry_is_refused() {
let pc = pcraft(&png(1, 1, 0), &png(1, 1, 0));
let mut limits = small();
limits.max_entry = 64;
assert!(limit_exceeded(info(&pc, &limits)));
}
#[test]
fn an_uncompressed_document_past_max_json_is_refused() {
let json = vectorcraft_json(4, &format!(r#""pad":"{}""#, "x".repeat(100 << 10)));
assert!(limit_exceeded(info(json.as_bytes(), &small())));
let ec = ecproj(2).replacen(r#""settings": {}"#, &format!(r#""settings": {{"pad":"{}"}}"#, "x".repeat(100 << 10)), 1);
assert!(limit_exceeded(outline(ec.as_bytes(), &small())));
}
#[test]
fn json_nested_past_max_depth_is_refused() {
let deep = format!("{}{}", "[".repeat(2000), "]".repeat(2000));
let vc = vectorcraft_json(4, &format!(r#""deep":{deep}"#));
assert!(limit_exceeded(outline(vc.as_bytes(), &Limits::default())));
let ec = ecproj(2).replacen(r#""settings": {}"#, &format!(r#""settings": {deep}"#), 1);
assert!(limit_exceeded(info(ec.as_bytes(), &Limits::default())));
let pc = zip(&[store("manifest.json", pcraft_manifest(1, &deep, "").as_bytes())]);
assert!(limit_exceeded(outline(&pc, &Limits::default())));
// At the limit (the file's object is one level), the document still reads.
let n = Limits::default().max_depth as usize - 1;
let at_limit = vectorcraft_json(4, &format!(r#""deep":{}{}"#, "[".repeat(n), "]".repeat(n)));
assert_eq!(outline(at_limit.as_bytes(), &Limits::default()).unwrap().pages.len(), 2);
let past = vectorcraft_json(4, &format!(r#""deep":{}{}"#, "[".repeat(n + 1), "]".repeat(n + 1)));
assert!(limit_exceeded(outline(past.as_bytes(), &Limits::default())));
// A raised limit stops at MAX_DEPTH: 2000 nested groups are refused, not parsed.
let mut raised = Limits::default();
raised.max_depth = u32::MAX;
assert!(limit_exceeded(outline(past.as_bytes(), &raised)));
let mut layer = pcraft_layer("Leaf", true, "Normal", 1.0, raster());
for _ in 0..2000 {
layer = pcraft_layer("G", true, "Normal", 1.0, &format!(r#"{{"kind":"group","expanded":false,"children":[{layer}]}}"#));
}
let pc = zip(&[deflated("manifest.json", pcraft_manifest(1, &format!("[{layer}]"), "").as_bytes())]);
raised.max_json = 1 << 30;
raised.max_entry = 1 << 30;
assert!(limit_exceeded(outline(&pc, &raised)));
}
#[test]
fn lists_past_max_items_are_refused() {
let with_max = |n: u64| {
let mut l = Limits::default();
l.max_items = n;
l
};
// Four project items and two layers; layers of `{}` count as well.
let ec = ecproj(2);
assert_eq!(outline(ec.as_bytes(), &with_max(6)).unwrap().pages.len(), 2);
assert!(limit_exceeded(outline(ec.as_bytes(), &with_max(5))));
let empty = ec.replacen(r#""layers": []"#, &format!(r#""layers": [{}{{}}]"#, "{},".repeat(9)), 1);
assert_eq!(outline(empty.as_bytes(), &with_max(16)).unwrap().pages[0].layers.len(), 10);
assert!(limit_exceeded(outline(empty.as_bytes(), &with_max(15))));
let null = ec.replacen(r#""layers": []"#, r#""layers": null"#, 1);
assert!(outline(null.as_bytes(), &with_max(6)).unwrap().pages[0].layers.is_empty());
// Two artboards and two layers.
let vc = vectorcraft_json(4, "");
assert_eq!(outline(vc.as_bytes(), &with_max(4)).unwrap().layers.len(), 2);
assert!(limit_exceeded(outline(vc.as_bytes(), &with_max(3))));
// An element past the limit is refused before it is read.
let bad = vc.replacen(r#""artboards":["#, r#""artboards":[{"rect":0},"#, 1);
assert!(matches!(outline(bad.as_bytes(), &with_max(4)), Err(Error::Malformed(_))));
assert!(limit_exceeded(outline(bad.as_bytes(), &with_max(0))));
// A group and its one layer.
let leaf = pcraft_layer("Leaf", true, "Normal", 1.0, raster());
let group = pcraft_layer("G", true, "Normal", 1.0, &format!(r#"{{"kind":"group","expanded":false,"children":[{leaf}]}}"#));
let pc = zip(&[store("manifest.json", pcraft_manifest(1, &format!("[{group}]"), "").as_bytes())]);
assert_eq!(outline(&pc, &with_max(2)).unwrap().layers[0].children.len(), 1);
assert!(limit_exceeded(outline(&pc, &with_max(1))));
}
#[test]
fn only_the_json_of_a_compressed_file_is_counted_when_no_member_follows_it() {
let mut file = vectorcraft_json(4, "").into_bytes();
let json_len = file.len() as u64;
file.extend(BLOB_MAGIC);
file.extend((0..1u32 << 20).map(|i| (i % 251) as u8));
let gz = gzip(&file);
let mut limits = small();
limits.max_total = json_len + BLOB_MAGIC.len() as u64;
assert_eq!(info(&gz, &limits).unwrap().version, 4);
assert_eq!(outline(&gz, &limits).unwrap().pages.len(), 2);
assert_eq!(renditions(&gz, &limits).unwrap(), vec![]);
}
#[test]
fn a_zip_entry_past_max_entry_is_refused_before_it_is_inflated() {
let big = vec![0u8; 1 << 20];
let pc = zip(&[
store("manifest.json", pcraft_manifest(1, "[]", r#","composite":"composite/preview.png""#).as_bytes()),
deflated("composite/preview.png", &big),
]);
assert!(limit_exceeded(renditions(&pc, &small())));
let mut limits = small();
limits.max_entry = 2 << 20;
limits.max_total = 4 << 20;
assert_eq!(renditions(&pc, &limits).unwrap()[0].bytes.len(), big.len());
}
/// Where the central directory entry of the `index`th entry starts.
fn central(zip: &[u8], index: usize) -> usize {
zip.windows(4).enumerate().filter(|(_, w)| *w == [0x50, 0x4b, 0x01, 0x02]).nth(index).unwrap().0
}
#[test]
fn a_deflate_entry_that_inflates_past_its_size_is_malformed() {
let manifest = pcraft_manifest(1, "[]", r#","composite":"c.png""#);
let mut pc = zip(&[store("manifest.json", manifest.as_bytes()), deflated("c.png", &vec![5u8; 1 << 20])]);
// Declare 16 bytes in the central directory.
let c = central(&pc, 1);
pc[c + 24..c + 28].copy_from_slice(&16u32.to_le_bytes());
assert!(matches!(renditions(&pc, &small()), Err(Error::Malformed(_))));
}
#[test]
fn zip_features_outside_the_reader_are_unsupported() {
let manifest = pcraft_manifest(1, "[]", "");
let base = zip(&[store("manifest.json", manifest.as_bytes())]);
let c = central(&base, 0);
let mut encrypted = base.clone();
encrypted[c + 8] |= 1;
assert!(matches!(info(&encrypted, &small()), Err(Error::Unsupported(_))));
let mut method = base.clone();
method[c + 10] = 12;
assert!(matches!(info(&method, &small()), Err(Error::Unsupported(_))));
let mut zip64 = base.clone();
zip64[c + 20..c + 24].copy_from_slice(&u32::MAX.to_le_bytes());
assert!(matches!(info(&zip64, &small()), Err(Error::Unsupported(_))));
// A ZIP64 end locator in front of the end record.
let eocd = base.len() - 22;
let mut locator = base[..eocd].to_vec();
locator.extend(0x0706_4b50u32.to_le_bytes());
locator.extend([0u8; 16]);
locator.extend(&base[eocd..]);
assert!(matches!(info(&locator, &small()), Err(Error::Unsupported(_))));
let mut crc = base.clone();
crc[c + 16] ^= 0xff;
assert!(matches!(info(&crc, &small()), Err(Error::Malformed(_))));
}
#[test]
fn a_comment_holding_an_end_signature_does_not_hide_the_archive() {
let mut pc = pcraft(&png(1, 1, 0), &png(1, 1, 0));
let comment = b"PK\x05\x06 not the end record";
let n = pc.len();
pc[n - 2..].copy_from_slice(&(comment.len() as u16).to_le_bytes());
pc.extend(comment);
assert_eq!(info(&pc, &small()).unwrap().version, 1);
}
struct Rng(u64);
impl Rng {
fn next(&mut self) -> u64 {
// xorshift64*
self.0 ^= self.0 >> 12;
self.0 ^= self.0 << 25;
self.0 ^= self.0 >> 27;
self.0.wrapping_mul(0x2545_F491_4F6C_DD1D)
}
fn below(&mut self, n: usize) -> usize {
(self.next() % n.max(1) as u64) as usize
}
}
#[test]
fn seeded_mutations_never_panic() {
let mut rng = Rng(0x5eed);
for s in samples() {
for _ in 0..1500 {
let mut d = s.clone();
for _ in 0..1 + rng.below(4) {
let at = rng.below(d.len());
match rng.below(4) {
0 => d[at] ^= 1 << rng.below(8),
1 => d[at] = rng.next() as u8,
2 => {
let v = [0u32, 1, u32::MAX, 0x8000_0000][rng.below(4)];
let end = (at + 4).min(d.len());
d[at..end].copy_from_slice(&v.to_le_bytes()[..end - at]);
}
_ => {
let end = (at + 1 + rng.below(64)).min(d.len());
d.drain(at..end);
if d.is_empty() {
break;
}
}
}
}
exercise(&d, &small());
}
}
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(256))]
#[test]
fn arbitrary_bytes_never_panic(bytes in proptest::collection::vec(any::<u8>(), 0..512)) {
exercise(&bytes, &small());
}
#[test]
fn arbitrary_bytes_behind_each_magic_never_panic(prefix in 0usize..4, rest in proptest::collection::vec(any::<u8>(), 0..512)) {
let heads: [&[u8]; 4] = [b"PK\x03\x04", &[0x1f, 0x8b, 8, 0], br#"{"format":"vectorcraft","#, br#"{"savedBy":"1","schema":"#];
let mut bytes = heads[prefix].to_vec();
bytes.extend(rest);
exercise(&bytes, &small());
}
}
crates/ecproj-format/Cargo.toml
+16
View File
@@ -0,0 +1,16 @@
[package]
name = "craft-ecproj-format"
description = "The envelope of EffectCraft .ecproj project files: savedBy, the schema version and its check, and the file object with the project and its saved view as type parameters."
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
crates/ecproj-format/README.md
+49
View File
@@ -0,0 +1,49 @@
# craft-ecproj-format
The envelope of EffectCraft `.ecproj` project files: the members around the project that
EffectCraft writes and other apps read. The project model, the saved view and the schema 1
conversion stay in EffectCraft. No `unsafe`, no panics, and it builds for
`wasm32-unknown-unknown`. The dependencies are `serde`, `serde_json` and `thiserror`.
```toml
craft-ecproj-format = { git = "https://github.com/storytold/craft-libs", rev = "<commit>" }
```
## The file
A project file is one UTF-8 JSON object with no newline at the end: `savedBy`, then the saved
view's members (`activeComp`, `openComps`, `compTimes`, each left out when empty), then the
project's members, starting with `schema`. Projects saved before `savedBy` was added start with
`schema`. EffectCraft indents the object with two spaces, or writes it on one line when Settings ▸
Project ▸ Save Compact Project Files is on. The crate writes and reads both forms.
## Use
```rust
use craft_ecproj_format::{File, Head, check_schema};
// Writing
check_schema(project.schema)?;
let text = File::new(APP_VERSION, &view, &project).to_json()?;
// Reading
let head = Head::read(&text)?;
```
- `SCHEMA_VERSION` is the newest project schema, 2.
- `check_schema(schema) -> Result<(), TooNew>` refuses a schema newer than `SCHEMA_VERSION`.
`TooNew` reads "project schema 3 is newer than this version supports (2)". It is
`#[non_exhaustive]`: other crates read its `schema` and `supported` fields.
- `File { saved_by, view, project }` serializes as one object, with `view` and `project`
flattened into it. `to_json()` gives the file's text indented with two spaces, and
`to_json_compact()` gives it on one line. Neither checks the values: `serde_json` writes NaN and
infinity as `null`, so the caller checks its values before saving.
- `Head::read(text) -> Result<Head, ReadError>` reads `savedBy` and `schema` from one JSON
object, and skips other members. `savedBy` is `None` when it is absent, is not a string, or
appears more than once. Text that is not a JSON object, has no `schema` that is a `u32`, or has
two `schema` members gives `ReadError::Json`. A schema newer than `SCHEMA_VERSION` gives
`ReadError::TooNew { too_new, saved_by, .. }`, which reads as `TooNew` does and holds the file's
`savedBy`, so a message can name the version that wrote it. `ReadError` and its `TooNew`
variant are `#[non_exhaustive]`: match them with a wildcard arm and `..`.
`.ecprojx` files hold the same object encoded as XML, which this crate does not read or write.
crates/ecproj-format/src/lib.rs
+311
View File
@@ -0,0 +1,311 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
//! The envelope of EffectCraft `.ecproj` project files, for EffectCraft, which writes them, and
//! for the apps that read them.
//!
//! A project file is one UTF-8 JSON object. EffectCraft indents it with two spaces, or writes it
//! on one line when Settings ▸ Project ▸ Save Compact Project Files is on:
//!
//! ```text
//! {
//! "savedBy": "0.12.0",
//! "activeComp": 1,
//! "openComps": [1],
//! "schema": 2,
//! "settings": { ... },
//! "items": { "1": { ... } },
//! "next_id": 2
//! }
//! ```
//!
//! - `savedBy` is the version of EffectCraft that wrote the file. Projects saved before it was
//! added start with `schema`.
//! - The saved view (`activeComp`, `openComps`, `compTimes`) and the project's own members follow
//! in the same object. Both are type parameters of [`File`]; their types stay in EffectCraft.
//! - `schema` is the project schema. [`check_schema`] refuses a schema newer than
//! [`SCHEMA_VERSION`].
//!
//! [`File`] writes the object: [`File::to_json`] indented, [`File::to_json_compact`] on one line.
//! [`Head::read`] reads `savedBy` and `schema` from either, and refuses a schema newer than
//! [`SCHEMA_VERSION`].
//!
//! ```
//! use std::collections::BTreeMap;
//!
//! #[derive(serde::Serialize)]
//! struct Project {
//! schema: u32,
//! items: BTreeMap<u64, String>,
//! }
//!
//! let project = Project { schema: craft_ecproj_format::SCHEMA_VERSION, items: BTreeMap::from([(1, "Comp 1".to_string())]) };
//! craft_ecproj_format::check_schema(project.schema)?;
//! let view = BTreeMap::<String, u64>::new();
//! let file = craft_ecproj_format::File::new("0.12.0", &view, &project);
//! let text = file.to_json()?;
//! assert_eq!(text, "{\n \"savedBy\": \"0.12.0\",\n \"schema\": 2,\n \"items\": {\n \"1\": \"Comp 1\"\n }\n}");
//! assert_eq!(file.to_json_compact()?, r#"{"savedBy":"0.12.0","schema":2,"items":{"1":"Comp 1"}}"#);
//!
//! let head = craft_ecproj_format::Head::read(&text)?;
//! assert_eq!((head.saved_by.as_deref(), head.schema), (Some("0.12.0"), 2));
//! # Ok::<(), Box<dyn std::error::Error>>(())
//! ```
//!
//! An `.ecprojx` file holds the same object encoded as XML. This crate reads and writes the JSON
//! form only.
use std::borrow::Cow;
use std::fmt;
use serde::de::{self, IgnoredAny, MapAccess, SeqAccess, Visitor};
use serde::{Deserialize, Deserializer, Serialize};
/// The newest project schema. Schema 2 measures the positions of effects on layers without a
/// source rectangle (shape, text) from the top-left of their comp-sized bounds. EffectCraft
/// converts schema 1 projects when it opens them.
pub const SCHEMA_VERSION: u32 = 2;
/// A project's schema is newer than [`SCHEMA_VERSION`].
///
/// Other crates read its fields but do not build one:
///
/// ```compile_fail
/// let _ = craft_ecproj_format::TooNew { schema: 3, supported: 2 };
/// ```
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
#[error("project schema {schema} is newer than this version supports ({supported})")]
#[non_exhaustive]
pub struct TooNew {
/// The schema the project declares.
pub schema: u32,
/// [`SCHEMA_VERSION`].
pub supported: u32,
}
/// Refuses a project whose `schema` is newer than [`SCHEMA_VERSION`], before it is read or saved.
pub fn check_schema(schema: u32) -> Result<(), TooNew> {
if schema > SCHEMA_VERSION { Err(TooNew { schema, supported: SCHEMA_VERSION }) } else { Ok(()) }
}
/// The object a project file holds: `savedBy`, then the members of `view`, then the members of
/// `project`.
///
/// `view` and `project` must each serialize as a map or a struct, and their member names must not
/// repeat `savedBy` or each other.
///
/// The values are written as they are, without a check: JSON has no NaN or infinity, and
/// `serde_json` writes either as `null`, which may not read back as the project. The caller
/// checks its values before saving.
#[derive(Debug, Serialize)]
pub struct File<'a, V, P> {
/// The version of the app that writes the file.
#[serde(rename = "savedBy")]
pub saved_by: &'a str,
/// The compositions that were open, and their times.
#[serde(flatten)]
pub view: &'a V,
/// The project.
#[serde(flatten)]
pub project: &'a P,
}
impl<'a, V: Serialize, P: Serialize> File<'a, V, P> {
pub fn new(saved_by: &'a str, view: &'a V, project: &'a P) -> Self {
File { saved_by, view, project }
}
/// The file's text, indented with two spaces, with no newline at the end.
pub fn to_json(&self) -> Result<String, serde_json::Error> {
serde_json::to_string_pretty(self)
}
/// The file's text on one line, with no spaces between tokens and no newline at the end.
pub fn to_json_compact(&self) -> Result<String, serde_json::Error> {
serde_json::to_string(self)
}
}
/// `savedBy` and `schema`, read from a project file by [`Head::read`].
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Head {
/// The version of the app that wrote the file. `None` for projects saved before `savedBy` was
/// added, and when `savedBy` is not a string or appears more than once.
pub saved_by: Option<String>,
/// The project schema, no newer than [`SCHEMA_VERSION`].
pub schema: u32,
}
impl Head {
/// Reads `text`, a project file: one JSON object with a `schema` that is a `u32`. Other
/// members are skipped, and two `schema` members are refused. A schema newer than
/// [`SCHEMA_VERSION`] is refused with [`ReadError::TooNew`], which holds the file's `savedBy`.
pub fn read(text: &str) -> Result<Head, ReadError> {
let mut deserializer = serde_json::Deserializer::from_str(text);
let head = deserializer.deserialize_map(HeadVisitor)?;
deserializer.end()?;
if let Err(too_new) = check_schema(head.schema) {
return Err(ReadError::TooNew { too_new, saved_by: head.saved_by });
}
Ok(head)
}
}
/// [`Head::read`] could not read a project file.
///
/// Other crates match it with a wildcard arm, and match [`ReadError::TooNew`] with `..`:
///
/// ```
/// use craft_ecproj_format::{Head, ReadError};
///
/// let text = r#"{"savedBy": "0.13.0", "schema": 3}"#;
/// let message = match Head::read(text) {
/// Ok(_) => String::new(),
/// Err(ReadError::TooNew { too_new, saved_by, .. }) => format!("saved by {}: {too_new}", saved_by.unwrap_or_default()),
/// Err(e) => e.to_string(),
/// };
/// assert_eq!(message, "saved by 0.13.0: project schema 3 is newer than this version supports (2)");
/// ```
///
/// ```compile_fail
/// fn name(e: craft_ecproj_format::ReadError) -> &'static str {
/// match e {
/// craft_ecproj_format::ReadError::Json(_) => "json",
/// craft_ecproj_format::ReadError::TooNew { .. } => "too new",
/// }
/// }
/// ```
///
/// ```compile_fail
/// fn saved_by(e: craft_ecproj_format::ReadError) -> Option<String> {
/// match e {
/// craft_ecproj_format::ReadError::TooNew { too_new: _, saved_by } => saved_by,
/// _ => None,
/// }
/// }
/// ```
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum ReadError {
/// The text is not one JSON object with a `schema` that is a `u32`.
#[error(transparent)]
Json(#[from] serde_json::Error),
/// The schema is newer than [`SCHEMA_VERSION`]. Reads as [`TooNew`] does.
#[error("{too_new}")]
#[non_exhaustive]
TooNew {
/// The schema the file declares, and [`SCHEMA_VERSION`].
too_new: TooNew,
/// The file's `savedBy`, read as [`Head::saved_by`] is.
saved_by: Option<String>,
},
}
/// Reads a [`Head`] from a JSON object. Other members are skipped.
struct HeadVisitor;
impl<'de> Visitor<'de> for HeadVisitor {
type Value = Head;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("a project file object")
}
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<Head, A::Error> {
let mut saved_by = None;
let mut seen_saved_by = false;
let mut schema = None;
while let Some(key) = map.next_key::<Cow<'de, str>>()? {
match &*key {
"savedBy" if seen_saved_by => {
map.next_value::<IgnoredAny>()?;
saved_by = None;
}
"savedBy" => {
saved_by = map.next_value::<SavedBy>()?.0;
seen_saved_by = true;
}
"schema" if schema.is_some() => return Err(de::Error::duplicate_field("schema")),
"schema" => schema = Some(map.next_value()?),
_ => {
map.next_value::<IgnoredAny>()?;
}
}
}
let schema = schema.ok_or_else(|| de::Error::missing_field("schema"))?;
Ok(Head { saved_by, schema })
}
}
/// A `savedBy` value: the string, or `None` for any other value. Arrays and objects are skipped
/// element by element with [`IgnoredAny`], so a large or deeply nested value is not kept.
struct SavedBy(Option<String>);
impl<'de> Deserialize<'de> for SavedBy {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
deserializer.deserialize_any(SavedByVisitor)
}
}
struct SavedByVisitor;
impl<'de> Visitor<'de> for SavedByVisitor {
type Value = SavedBy;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
fn visit_str<E: de::Error>(self, v: &str) -> Result<SavedBy, E> {
Ok(SavedBy(Some(v.to_owned())))
}
fn visit_string<E: de::Error>(self, v: String) -> Result<SavedBy, E> {
Ok(SavedBy(Some(v)))
}
fn visit_bool<E: de::Error>(self, _: bool) -> Result<SavedBy, E> {
Ok(SavedBy(None))
}
fn visit_i64<E: de::Error>(self, _: i64) -> Result<SavedBy, E> {
Ok(SavedBy(None))
}
fn visit_i128<E: de::Error>(self, _: i128) -> Result<SavedBy, E> {
Ok(SavedBy(None))
}
fn visit_u64<E: de::Error>(self, _: u64) -> Result<SavedBy, E> {
Ok(SavedBy(None))
}
fn visit_u128<E: de::Error>(self, _: u128) -> Result<SavedBy, E> {
Ok(SavedBy(None))
}
fn visit_f64<E: de::Error>(self, _: f64) -> Result<SavedBy, E> {
Ok(SavedBy(None))
}
fn visit_unit<E: de::Error>(self) -> Result<SavedBy, E> {
Ok(SavedBy(None))
}
fn visit_none<E: de::Error>(self) -> Result<SavedBy, E> {
Ok(SavedBy(None))
}
fn visit_some<D: Deserializer<'de>>(self, deserializer: D) -> Result<SavedBy, D::Error> {
deserializer.deserialize_ignored_any(IgnoredAny)?;
Ok(SavedBy(None))
}
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<SavedBy, A::Error> {
while seq.next_element::<IgnoredAny>()?.is_some() {}
Ok(SavedBy(None))
}
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<SavedBy, A::Error> {
while map.next_entry::<IgnoredAny, IgnoredAny>()?.is_some() {}
Ok(SavedBy(None))
}
}
crates/ecproj-format/tests/envelope.rs
+209
View File
@@ -0,0 +1,209 @@
use std::collections::BTreeMap;
use craft_ecproj_format::{File, Head, ReadError, SCHEMA_VERSION, check_schema};
use serde::{Deserialize, Serialize};
#[derive(Debug, Default, Serialize)]
#[serde(rename_all = "camelCase")]
struct View {
#[serde(skip_serializing_if = "Option::is_none")]
active_comp: Option<u64>,
#[serde(skip_serializing_if = "Vec::is_empty")]
open_comps: Vec<u64>,
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
comp_times: BTreeMap<u64, i64>,
}
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Settings {
bit_depth: String,
}
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Item {
id: u64,
name: String,
}
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Project {
schema: u32,
settings: Settings,
items: BTreeMap<u64, Item>,
next_id: u64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
render_queue: Vec<u64>,
}
fn project() -> Project {
Project {
schema: 2,
settings: Settings { bit_depth: "Bpc8".into() },
items: BTreeMap::from([(1, Item { id: 1, name: "Comp 1".into() }), (2, Item { id: 2, name: "clip.mov".into() })]),
next_id: 3,
render_queue: vec![],
}
}
const PROJECT_MEMBERS: &str = r#" "schema": 2,
"settings": {
"bit_depth": "Bpc8"
},
"items": {
"1": {
"id": 1,
"name": "Comp 1"
},
"2": {
"id": 2,
"name": "clip.mov"
}
},
"next_id": 3
}"#;
#[test]
fn a_file_with_a_saved_view_holds_saved_by_then_the_view_then_the_project() {
let view = View { active_comp: Some(1), open_comps: vec![1, 2], comp_times: BTreeMap::from([(1, 254_016_000_000)]) };
let p = project();
let text = File::new("0.12.0", &view, &p).to_json().unwrap();
let expected = format!(
r#"{{
"savedBy": "0.12.0",
"activeComp": 1,
"openComps": [
1,
2
],
"compTimes": {{
"1": 254016000000
}},
{PROJECT_MEMBERS}"#
);
assert_eq!(text, expected);
}
#[test]
fn a_file_with_an_empty_view_holds_saved_by_then_the_project() {
let p = project();
let text = File::new("0.12.0", &View::default(), &p).to_json().unwrap();
assert_eq!(text, format!("{{\n \"savedBy\": \"0.12.0\",\n{PROJECT_MEMBERS}"));
}
#[test]
fn a_compact_file_holds_the_same_members_on_one_line() {
let view = View { active_comp: Some(1), open_comps: vec![1, 2], comp_times: BTreeMap::from([(1, 254_016_000_000)]) };
let p = project();
let text = File::new("0.12.0", &view, &p).to_json_compact().unwrap();
assert_eq!(
text,
r#"{"savedBy":"0.12.0","activeComp":1,"openComps":[1,2],"compTimes":{"1":254016000000},"schema":2,"settings":{"bit_depth":"Bpc8"},"items":{"1":{"id":1,"name":"Comp 1"},"2":{"id":2,"name":"clip.mov"}},"next_id":3}"#
);
assert_eq!(serde_json::from_str::<Project>(&text).unwrap(), p);
assert_eq!(Head::read(&text).unwrap(), Head { saved_by: Some("0.12.0".into()), schema: 2 });
let pretty = File::new("0.12.0", &view, &p).to_json().unwrap();
assert_eq!(serde_json::from_str::<serde_json::Value>(&text).unwrap(), serde_json::from_str::<serde_json::Value>(&pretty).unwrap());
}
#[test]
fn a_written_file_reads_back_as_the_project_and_its_head() {
let p = project();
let view = View { active_comp: Some(1), ..View::default() };
let text = File::new("0.12.0", &view, &p).to_json().unwrap();
assert_eq!(serde_json::from_str::<Project>(&text).unwrap(), p);
assert_eq!(Head::read(&text).unwrap(), Head { saved_by: Some("0.12.0".into()), schema: 2 });
}
#[test]
fn a_project_saved_before_saved_by_has_a_head_without_it() {
let text = r#"{"schema": 1, "settings": {}, "items": {}, "next_id": 1}"#;
assert_eq!(Head::read(text).unwrap(), Head { saved_by: None, schema: 1 });
}
#[test]
fn a_head_needs_a_schema() {
assert!(Head::read(r#"{"savedBy": "0.12.0", "items": {}}"#).is_err());
assert!(Head::read(r#"{"savedBy": "0.12.0", "schema": "2"}"#).is_err());
assert!(Head::read(r#"{"savedBy": "0.13.0", "schema": null, "schema": "3"}"#).is_err());
assert!(Head::read("[]").is_err());
assert!(Head::read(r#"["0.12.0", 2]"#).is_err());
assert!(Head::read(r#"{"schema": 2, "schema": 2}"#).is_err());
assert!(Head::read(r#"{"schema": 2} {}"#).is_err());
assert!(Head::read(r#"{"savedBy": "0.12.0", "schema": 2"#).is_err());
assert!(Head::read("").is_err());
assert!(Head::read("2").is_err());
}
#[test]
fn a_head_reads_a_saved_by_that_is_not_one_string_as_none() {
assert_eq!(Head::read(r#"{"savedBy": 12, "schema": 2}"#).unwrap(), Head { saved_by: None, schema: 2 });
assert_eq!(Head::read(r#"{"savedBy": ["0.12.0"], "schema": 2}"#).unwrap(), Head { saved_by: None, schema: 2 });
assert_eq!(Head::read(r#"{"savedBy": {"v": "0.12.0"}, "schema": 2}"#).unwrap(), Head { saved_by: None, schema: 2 });
assert_eq!(Head::read(r#"{"savedBy": "a", "savedBy": "b", "schema": 2}"#).unwrap(), Head { saved_by: None, schema: 2 });
assert_eq!(Head::read(r#"{"savedBy": "a", "schema": 2, "savedBy": 1, "savedBy": "c"}"#).unwrap(), Head { saved_by: None, schema: 2 });
assert_eq!(Head::read(r#"{"schema": 2, "savedBy": "0.13.0-dev"}"#).unwrap(), Head { saved_by: Some("0.13.0-dev".into()), schema: 2 });
}
#[test]
fn a_head_reads_the_object_form_only() {
assert!(matches!(Head::read(r#"[2, {}, {}, 1]"#), Err(ReadError::Json(_))));
assert!(matches!(Head::read("[1]"), Err(ReadError::Json(_))));
assert!(matches!(Head::read(r#"["savedBy", "0.12.0"]"#), Err(ReadError::Json(_))));
}
#[test]
fn a_head_refuses_a_newer_schema() {
let Err(ReadError::TooNew { too_new, saved_by, .. }) = Head::read(r#"{"savedBy": "0.13.0", "schema": 3}"#) else {
panic!("schema 3 is newer than 2")
};
assert_eq!((too_new.schema, too_new.supported), (3, 2));
assert_eq!(saved_by.as_deref(), Some("0.13.0"));
let Err(e) = Head::read(r#"{"schema": 4294967295, "savedBy": 1}"#) else { panic!("schema u32::MAX is newer than 2") };
assert_eq!(e.to_string(), "project schema 4294967295 is newer than this version supports (2)");
let ReadError::TooNew { saved_by, .. } = e else { panic!("{e:?} is not TooNew") };
assert_eq!(saved_by, None);
assert_eq!(Head::read(r#"{"savedBy": "0.12.0", "schema": 2}"#).unwrap(), Head { saved_by: Some("0.12.0".into()), schema: 2 });
}
#[test]
fn a_head_skips_other_members_of_any_type() {
let text = r#"{"items": [1, {"schema": 9}], "savedBy": null, "schema": 2, "next_id": 1.5}"#;
assert_eq!(Head::read(text).unwrap(), Head { saved_by: None, schema: 2 });
}
#[test]
fn a_deeply_nested_saved_by_is_skipped_like_other_members() {
let deep = format!("{}{}", "[".repeat(10_000), "]".repeat(10_000));
let text = format!(r#"{{"savedBy": {deep}, "schema": 2, "items": {deep}}}"#);
assert_eq!(Head::read(&text).unwrap(), Head { saved_by: None, schema: 2 });
let deep_map = format!("{}0{}", r#"{"v": "#.repeat(10_000), "}".repeat(10_000));
let text = format!(r#"{{"savedBy": "0.12.0", "savedBy": {deep_map}, "schema": 2}}"#);
assert_eq!(Head::read(&text).unwrap(), Head { saved_by: None, schema: 2 });
}
#[test]
fn a_wide_saved_by_array_reads_as_none() {
let wide = format!("[{}0]", "0,".repeat(1_000_000));
let text = format!(r#"{{"savedBy": {wide}, "schema": 2, "savedBy": {wide}}}"#);
assert_eq!(Head::read(&text).unwrap(), Head { saved_by: None, schema: 2 });
let text = format!(r#"{{"schema": 2, "savedBy": "0.12.0", "savedBy": "{}"}}"#, "9".repeat(1_000_000));
assert_eq!(Head::read(&text).unwrap(), Head { saved_by: None, schema: 2 });
}
#[test]
fn check_schema_refuses_only_a_newer_schema() {
assert_eq!(SCHEMA_VERSION, 2);
assert_eq!(check_schema(0), Ok(()));
assert_eq!(check_schema(1), Ok(()));
assert_eq!(check_schema(2), Ok(()));
for schema in [3, u32::MAX] {
let Err(e) = check_schema(schema) else { panic!("schema {schema} is newer than 2") };
assert_eq!((e.schema, e.supported), (schema, 2));
}
}
#[test]
fn too_new_reads_as_effectcraft_words_it() {
let Err(e) = check_schema(3) else { panic!("schema 3 is newer than 2") };
assert_eq!(e.to_string(), "project schema 3 is newer than this version supports (2)");
}
crates/font-store/Cargo.toml
+5
View File
@@ -25,6 +25,11 @@ thiserror = { workspace = true }
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
craft-atomic = { workspace = true }
craft-font-catalog = { workspace = true }
log = "0.4"
# Language packs (`language_packs`): `<locale>.tar.zst` archives, unpacked in pure Rust.
ruzstd = { version = "0.9", default-features = false, features = ["std"] }
toml = { version = "1", default-features = false, features = ["parse", "serde", "std"] }
tar = { version = "0.4", default-features = false }
# Blocking HTTPS with rustls and webpki roots: no async runtime, no OpenSSL.
ureq = { version = "3", default-features = false, features = ["rustls", "gzip"] }
crates/font-store/README.md
+29
View File
@@ -94,6 +94,8 @@ settings.json the network setting and the sources the user added
manifest.json what is installed
indexes/ the last indexes fetched: sources.json, <source>.json.gz, user-<hash>
<family folder>/ the font files and the licence
language-packs/ installed Language packs, one folder per locale
language-packs.json which Language packs are installed
.staging/ downloads being installed
.lock held while the store changes
```
@@ -113,6 +115,33 @@ A Google Fonts family's folder is google/fonts' own folder name (`lato/`); anoth
The catalog scans the store folder. With `with_catalog(catalog)`, every install and removal ends with `catalog.rescan()`, which reads only new and changed files, so the app sees the fonts at once. Other running apps notice through `Catalog::rescan_if_changed` (when their window is activated) or `Catalog::watch`.
## Language packs
The store's second item kind, `language-pack`: a language's spelling dictionary, hyphenation patterns and licence texts, published by [storytold/craft-i18n](https://github.com/storytold/craft-i18n) as `<locale>.tar.zst` release assets listed in an `index.json`, and read at runtime by `craft-proof` (`craft_proof::packs`). It reuses the store's folder, consent, HTTP client, staging and sha256 checks; there is no second downloader.
```rust
const PACKS: PackIndexPin = PackIndexPin { url: "https://github.com/storytold/craft-i18n/releases/download/language-packs-1/index.json", sha256: "<hex>" };
let index = store.fetch_language_pack_index(&PACKS, &Cancel::new())?; // or cached_language_pack_index(&PACKS)
let nl = index.pack("nl").unwrap();
store.install_language_pack(&index, nl, &Cancel::new(), &mut |p| ())?; // network on, like fonts
craft_proof::packs::set_packs_dir(Some(store.language_packs_dir())); // then packs::reload() after each change
store.remove_language_pack("nl")?;
let installed = store.installed_language_packs();
```
| Call | What it does |
|---|---|
| `fetch_language_pack_index(pin, cancel)` | Downloads the index, checks it against the pin's sha256, saves it as `indexes/language-packs.json`. Needs the network on. |
| `cached_language_pack_index(pin)` | The saved index, when it is the pinned one. |
| `install_language_pack(index, pack, cancel, progress)` | Downloads the archive (at most 128 MiB), checks size and sha256, unpacks it into `.staging/` and moves it into `language-packs/<locale>/` with one rename (an older pack is replaced the same way). Records it in `language-packs.json`. |
| `remove_language_pack(locale)` | Moves the folder out at once, updates the record, deletes the files. No network needed. |
| `installed_language_packs()` | The recorded packs whose folder is there. |
| `language_packs_dir()` | `<store>/language-packs`, for `craft_proof::packs::set_packs_dir`. |
**Unpacking is hostile-input safe:** pure-Rust zstd (`ruzstd`, MIT; every frame is read, and nothing but tar padding may follow the archive) and `tar` (MIT OR Apache-2.0); regular files and the `LICENCES` folder only (no links or devices). Only the contract's names are allowed: `pack.toml`, `spelling.aff`, `spelling.dic`, `hyphenation.pat` at the root and `LICENCES/<name>` (`[A-Za-z0-9._-]`, no leading or trailing dot, no Windows device names such as `CON` or `nul.txt`). Each file at most 64 MiB, 256 MiB and 256 entries per pack. `pack.toml` is required and must name the entry's locale; the files the index promises (`spelling`, `hyphenation`) must be there. Index entries need a lowercase BCP 47 locale (listed once, not a Windows device name), a sha256 and an `https://` URL; names, licences, versions, classes and the release lose control and bidi/format characters before they are stored or listed; an optional `source` must be an `https://` URL. At most 4096 zstd frames per archive. An unreadable `language-packs.json` is copied to `language-packs.json.bad` and rebuilt from the pack folders (each `pack.toml`) atomically in place, as is a missing one when pack folders are there, so no installed pack drops out of the list. Only an install or removal (under the store lock) rewrites it; listing reads the folders meanwhile and writes nothing.
**Developer override (testing only — bypasses the pin):** `CRAFT_LANGUAGE_PACK_INDEX=<url or path>` replaces the pinned index URL with an `http://` URL on a loopback host (`127.0.0.1`, `localhost`, `[::1]`), an `https://` URL or a local file path (`file://` URLs are not taken), and **skips the index's sha256 pin**. Archives are still checked against the sha256 and size in that index, and may then also be loopback `http://` URLs or local paths. Off unless the variable is set; it works in release builds too (for end-to-end tests of a real build), a warning is logged whenever it is used, and `language_pack_index_overridden() -> Option<String>` returns the value so apps can show that the pin is bypassed. Never set it for users.
## Web
The index, the lookups, `verify_font` and `IndexPin` build for `wasm32-unknown-unknown`. The store folder and the HTTP client are native only; the web keeps fonts elsewhere.
crates/font-store/src/http.rs
+6
View File
@@ -111,6 +111,12 @@ impl Client {
}
}
/// A client that also allows plain `http://`, used only for the developer override of the
/// Language pack index (`CRAFT_LANGUAGE_PACK_INDEX`), which accepts only loopback hosts.
pub(crate) fn allowing_http(t: Timeouts) -> Self {
Self::build(t, false)
}
/// Plain `http://` to a local test server, with `https://` prefixes mapped onto it.
#[cfg(test)]
pub(crate) fn for_tests(t: Timeouts, rewrite: Vec<(String, String)>) -> Self {
crates/font-store/src/index.rs
+6 -6
View File
@@ -24,25 +24,25 @@ pub const LICENSES: [&str; 3] = ["OFL", "APACHE2", "UFL"];
#[derive(Debug, thiserror::Error)]
pub enum IndexError {
/// More than [`MAX_INDEX_BYTES`].
#[error("the font index is larger than {max} bytes")]
#[error("the index is larger than {max} bytes")]
TooLarge {
/// The limit.
max: usize,
},
/// Not JSON, or a field has the wrong type.
#[error("the font index can't be read: {0}")]
#[error("the index can't be read: {0}")]
Json(String),
/// A major version this crate doesn't know.
#[error("the font index has version {0}; this build reads version {MAJOR_VERSION}.x")]
#[error("the index has version {0}; this build reads version {MAJOR_VERSION}.x")]
Version(String),
/// An entry breaks the format's rules.
#[error("the font index has an invalid entry: {0}")]
#[error("the index has an invalid entry: {0}")]
Invalid(String),
/// A gzipped index that doesn't decompress.
#[error("the gzipped font index can't be read: {0}")]
#[error("the gzipped index can't be read: {0}")]
Gzip(String),
/// The bytes aren't the pinned ones.
#[error("the font index doesn't have the sha256 this build expects")]
#[error("the index doesn't have the sha256 this build expects")]
Checksum,
}
crates/font-store/src/language_packs.rs
+722
View File
@@ -0,0 +1,722 @@
//! The store's second item kind, `language-pack`: a language's spelling dictionary, hyphenation
//! patterns and licence texts, published by storytold/craft-i18n as `<locale>.tar.zst` release
//! assets listed in an `index.json`, and read at runtime by `craft-proof` (`craft_proof::packs`).
//!
//! It reuses the store's folder, consent ([`FontStore::enable_network`]), HTTP client and sha256
//! checks: the app pins the index by URL and sha256 ([`PackIndexPin`]), each archive is checked
//! against the size and sha256 the index gives, then unpacked into `.staging/` and moved into
//! `language-packs/<locale>/` with one rename. Packs have no update checks of their own: a new
//! pin is a new index.
//!
//! ```text
//! <store>/language-packs/<locale>/ pack.toml, spelling.aff, spelling.dic, hyphenation.pat, LICENCES/…
//! <store>/language-packs.json which packs are installed, from which index entry
//! <store>/indexes/language-packs.json the last pack index fetched
//! ```
//!
//! **Developer override (testing only):** `CRAFT_LANGUAGE_PACK_INDEX=<url or path>` replaces the
//! pinned index URL with an `http://` URL on a loopback host (`127.0.0.1`, `localhost`, `[::1]`),
//! an `https://` URL or a local file path, and skips the index's sha256 pin. Archives are still
//! checked against the sha256 and size in that index, and may then also be loopback `http://`
//! URLs or local paths. It is off unless the variable is set, and logged as a warning when used.
use std::io::Read;
use std::path::{Component, Path, PathBuf};
use serde::{Deserialize, Serialize};
use crate::http::{Cancel, Client, Timeouts};
use crate::index::IndexError;
use crate::store::{Error, FontStore, INDEX_CACHE, Progress, Stage, io, read_limited};
use crate::verify::{VerifyError, sha256_hex};
/// The store's folder of installed Language packs.
pub const LANGUAGE_PACKS: &str = "language-packs";
/// The developer override of the pinned pack index (see the module docs). Testing only.
pub const LANGUAGE_PACK_INDEX_ENV: &str = "CRAFT_LANGUAGE_PACK_INDEX";
/// Largest pack index read.
pub const MAX_PACK_INDEX_BYTES: u64 = 4 << 20;
/// Largest pack archive downloaded.
pub const MAX_PACK_ARCHIVE_BYTES: u64 = 128 << 20;
/// Largest file unpacked from a pack (craft-proof's dictionary cap).
pub const MAX_PACK_FILE_BYTES: u64 = 64 << 20;
/// Largest total unpacked size of a pack.
pub const MAX_PACK_UNPACKED_BYTES: u64 = 256 << 20;
/// Most files and folders in a pack archive.
pub const MAX_PACK_ENTRIES: usize = 256;
/// Most packs in an index.
const MAX_PACKS: usize = 1024;
const INSTALLED: &str = "language-packs.json";
const INDEX_CACHE_NAME: &str = "language-packs.json";
/// The pack index an app trusts: `index.json` of a craft-i18n `language-packs-<n>` release, by
/// URL and sha256 (the app's Pin).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct PackIndexPin {
/// `https://github.com/storytold/craft-i18n/releases/download/language-packs-<n>/index.json`.
pub url: &'static str,
/// The sha256 of that file, lowercase hex.
pub sha256: &'static str,
}
/// `index.json` of a pack release.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PackIndex {
/// The format; 1.
pub format: u32,
/// The release (`language-packs-1`).
pub release: String,
/// The packs, by locale.
pub packs: Vec<PackEntry>,
}
/// One pack of a [`PackIndex`].
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PackEntry {
/// Lowercase BCP 47 tag (`nl`, `pt-br`); the pack's folder name.
pub locale: String,
/// The language's own name (`Nederlands`).
pub name: String,
/// The craft-i18n commit it was built from.
#[serde(default)]
pub version: String,
/// The archive (`<locale>.tar.zst`).
pub url: String,
/// The archive's sha256, lowercase hex.
pub sha256: String,
/// The archive's size in bytes.
pub size: u64,
/// SPDX licence expression.
pub licence: String,
/// The licence class: `Embeddable` or `Download-only`.
#[serde(default)]
pub class: String,
/// Whether it has a spelling dictionary.
#[serde(default)]
pub spelling: bool,
/// Whether it has hyphenation patterns.
#[serde(default)]
pub hyphenation: bool,
/// Where the data comes from (an `https://` page), for display; optional.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub source: Option<String>,
}
/// An installed pack, as recorded when it was installed.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct InstalledLanguagePack {
/// Lowercase BCP 47 tag; the folder under [`LANGUAGE_PACKS`].
pub locale: String,
/// The language's own name.
pub name: String,
/// The release it came from.
pub release: String,
/// The craft-i18n commit it was built from.
pub version: String,
/// SPDX licence expression.
pub licence: String,
/// The archive's sha256.
pub sha256: String,
/// The archive's size in bytes.
pub size: u64,
}
#[derive(Default, Serialize, Deserialize)]
struct Installed {
version: u32,
packs: Vec<InstalledLanguagePack>,
}
impl PackIndex {
/// Parses and checks an index: format 1, every locale a lowercase tag and listed once, every
/// archive at most [`MAX_PACK_ARCHIVE_BYTES`] with a sha256. Archive URLs must be `https://`
/// (with `local`, the developer override, also loopback `http://` or a path).
pub fn from_json(bytes: &[u8]) -> Result<Self, Error> {
Self::parse(bytes, false)
}
fn parse(bytes: &[u8], local: bool) -> Result<Self, Error> {
let bad = |why: String| Error::Index(IndexError::Invalid(format!("Language pack index: {why}")));
let mut index: PackIndex = serde_json::from_slice(bytes).map_err(|e| Error::Index(IndexError::Json(format!("Language pack index: {e}"))))?;
if index.format != 1 {
return Err(bad(format!("format {} is not supported", index.format)));
}
if index.packs.len() > MAX_PACKS {
return Err(bad("too many packs".into()));
}
for (i, p) in index.packs.iter().enumerate() {
if !is_locale(&p.locale) || index.packs.iter().take(i).any(|q| q.locale == p.locale) {
return Err(bad(format!("pack {:?}: not a lowercase language tag, or listed twice", p.locale.chars().take(40).collect::<String>())));
}
if p.size > MAX_PACK_ARCHIVE_BYTES || !is_sha256(&p.sha256) {
return Err(bad(format!("pack {}: bad size or sha256", p.locale)));
}
if !(p.url.starts_with("https://") || local && local_source(&p.url).is_some()) {
return Err(bad(format!("pack {}: {} is not an https URL", p.locale, p.url.chars().take(200).collect::<String>())));
}
if let Some(src) = &p.source
&& !(src.starts_with("https://") && src.len() <= 2048 && !src.chars().any(|c| c.is_control() || is_format(c) || c == ' '))
{
return Err(bad(format!("pack {}: source {:?} is not an https URL", p.locale, src.chars().take(200).collect::<String>())));
}
}
// Text shown to people loses control and bidi/format characters.
index.release = plain_text(&index.release);
for p in &mut index.packs {
for t in [&mut p.name, &mut p.version, &mut p.licence, &mut p.class] {
*t = plain_text(t);
}
}
Ok(index)
}
/// The pack for `locale`.
pub fn pack(&self, locale: &str) -> Option<&PackEntry> {
self.packs.iter().find(|p| p.locale == locale)
}
}
/// Where the pack index comes from: the pin, or the developer override.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum IndexSource {
/// The pinned URL, checked against the pinned sha256.
Pinned(PackIndexPin),
/// `CRAFT_LANGUAGE_PACK_INDEX`: no index pin; loopback `http://` and paths allowed.
Override(String),
}
impl IndexSource {
/// The pin, unless [`LANGUAGE_PACK_INDEX_ENV`] is set.
fn from_env(pin: &PackIndexPin) -> Self {
match language_pack_index_overridden() {
Some(v) => {
log::warn!(
"{LANGUAGE_PACK_INDEX_ENV}={v}: the Language pack index is NOT the pinned one and its sha256 is not checked (testing only)"
);
Self::Override(v)
}
None => Self::Pinned(*pin),
}
}
fn is_override(&self) -> bool {
matches!(self, Self::Override(_))
}
}
/// The developer override of the pack index, when [`LANGUAGE_PACK_INDEX_ENV`] is set: the URL or
/// path that replaces the pinned index, whose sha256 is then **not** checked. Apps show a warning
/// in their Language pack interface while it is set. It works in release builds too, for
/// end-to-end testing against a local index.
pub fn language_pack_index_overridden() -> Option<String> {
std::env::var(LANGUAGE_PACK_INDEX_ENV).ok().map(|v| v.trim().to_string()).filter(|v| !v.is_empty())
}
/// A source the developer override may read: `Ok(url)` for an `https://` URL or an `http://`
/// one on a loopback host, `Err(path)` for a local file path; `None` for anything else.
fn local_source(s: &str) -> Option<Result<&str, PathBuf>> {
if s.starts_with("https://") {
return Some(Ok(s));
}
if let Some(rest) = s.strip_prefix("http://") {
let host = rest.split(['/', '?', '#']).next().unwrap_or_default();
let host = host.rsplit_once('@').map_or(host, |(_, h)| h);
let name = if host.starts_with('[') {
host.split(']').next().map(|h| format!("{h}]")).unwrap_or_default()
} else {
host.split(':').next().unwrap_or_default().to_string()
};
return ["127.0.0.1", "localhost", "[::1]"].contains(&name.as_str()).then_some(Ok(s));
}
if s.contains("://") {
return None;
}
Some(Err(PathBuf::from(s)))
}
/// A lowercase BCP 47 tag as packs are named (craft-proof's rule): ASCII lowercase letters and
/// digits in parts of 1 to 8 joined by `-`, at most 35 characters.
fn is_locale(s: &str) -> bool {
!windows_reserved(s)
&& !s.is_empty()
&& s.len() <= 35
&& s.split('-').all(|p| !p.is_empty() && p.len() <= 8 && p.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit()))
}
fn is_sha256(s: &str) -> bool {
s.len() == 64 && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
impl FontStore {
/// The folder of installed Language packs (`<store>/language-packs`): give it to
/// `craft_proof::packs::set_packs_dir`.
pub fn language_packs_dir(&self) -> PathBuf {
self.dir().join(LANGUAGE_PACKS)
}
/// The pack index saved by the last [`FontStore::fetch_language_pack_index`], when it is
/// `pin`'s (or the developer override is set).
pub fn cached_language_pack_index(&self, pin: &PackIndexPin) -> Option<PackIndex> {
self.cached_pack_index_from(&IndexSource::from_env(pin))
}
pub(crate) fn cached_pack_index_from(&self, source: &IndexSource) -> Option<PackIndex> {
let bytes = read_limited(&self.dir().join(INDEX_CACHE).join(INDEX_CACHE_NAME), MAX_PACK_INDEX_BYTES)?;
match source {
IndexSource::Pinned(pin) if sha256_hex(&bytes) != pin.sha256 => None,
_ => PackIndex::parse(&bytes, source.is_override()).ok(),
}
}
/// Downloads the pack index `pin` names, checks it against the pin and saves it for
/// [`FontStore::cached_language_pack_index`]. Needs the network on.
pub fn fetch_language_pack_index(&self, pin: &PackIndexPin, cancel: &Cancel) -> Result<PackIndex, Error> {
self.fetch_pack_index_from(&IndexSource::from_env(pin), cancel)
}
pub(crate) fn fetch_pack_index_from(&self, source: &IndexSource, cancel: &Cancel) -> Result<PackIndex, Error> {
self.require_network()?;
let bytes = match source {
IndexSource::Pinned(pin) => {
let check = |b: &[u8]| if sha256_hex(b) == pin.sha256 { Ok(()) } else { Err(VerifyError::Sha256) };
self.fetch(&[pin.url.to_string()], MAX_PACK_INDEX_BYTES, "the Language pack index", cancel, &mut |_| {}, &check)?
}
IndexSource::Override(at) => self.get_local(at, MAX_PACK_INDEX_BYTES, "the Language pack index", cancel, &mut |_| {}, &|_| Ok(()))?,
};
let index = PackIndex::parse(&bytes, source.is_override())?;
std::fs::create_dir_all(self.dir().join(INDEX_CACHE)).map_err(io("creating the index folder"))?;
self.write_file(&format!("{INDEX_CACHE}/{INDEX_CACHE_NAME}"), &bytes)?;
Ok(index)
}
/// A download under the developer override: an https or loopback http URL, or a file.
fn get_local(
&self,
at: &str,
max: u64,
what: &str,
cancel: &Cancel,
on_bytes: &mut dyn FnMut(u64),
check: &dyn Fn(&[u8]) -> Result<(), VerifyError>,
) -> Result<Vec<u8>, Error> {
let bytes = match local_source(at) {
Some(Ok(url)) if url.starts_with("https://") => return self.fetch(&[url.to_string()], max, what, cancel, on_bytes, check),
Some(Ok(url)) => Client::allowing_http(Timeouts::default())
.get(url, max, cancel, on_bytes)
.map_err(|e| Error::Download { what: what.to_string(), reasons: vec![e.to_string()] })?,
Some(Err(path)) => {
let mut buf = Vec::new();
std::fs::File::open(&path)
.and_then(|f| f.take(max.saturating_add(1)).read_to_end(&mut buf))
.map_err(io(format!("reading {}", path.display())))?;
if buf.len() as u64 > max {
return Err(Error::Download { what: what.to_string(), reasons: vec![format!("larger than {max} bytes")] });
}
on_bytes(buf.len() as u64);
buf
}
None => return Err(Error::BadUrl(at.chars().take(200).collect())),
};
check(&bytes).map_err(|error| Error::Verify { file: what.to_string(), error })?;
Ok(bytes)
}
/// Downloads `pack` from `index`, checks its size and sha256, unpacks it and moves it into
/// `language-packs/<locale>/` in one rename, replacing an older one. Needs the network on.
/// Then call `craft_proof::packs::reload()`.
pub fn install_language_pack(
&self,
index: &PackIndex,
pack: &PackEntry,
cancel: &Cancel,
progress: &mut dyn FnMut(Progress),
) -> Result<InstalledLanguagePack, Error> {
let local = language_pack_index_overridden().is_some();
self.install_pack_from(local, index, pack, cancel, progress)
}
pub(crate) fn install_pack_from(
&self,
local: bool,
index: &PackIndex,
pack: &PackEntry,
cancel: &Cancel,
progress: &mut dyn FnMut(Progress),
) -> Result<InstalledLanguagePack, Error> {
self.require_network()?;
// The entry is checked again here: callers may build one by hand.
if !is_locale(&pack.locale) || !is_sha256(&pack.sha256) || pack.size > MAX_PACK_ARCHIVE_BYTES {
let locale: String = pack.locale.chars().take(40).collect();
return Err(Error::Index(IndexError::Invalid(format!("Language pack {locale:?}: bad locale, sha256 or size"))));
}
let what = format!("{}.tar.zst", pack.locale);
let (size, sha) = (pack.size, pack.sha256.clone());
let check = move |b: &[u8]| {
if b.len() as u64 != size {
Err(VerifyError::Size { expected: size, actual: b.len() as u64 })
} else if sha256_hex(b) != sha {
Err(VerifyError::Sha256)
} else {
Ok(())
}
};
let mut on_bytes = |done: u64| progress(Progress { file: 0, files: 1, done, total: size });
let bytes = if local {
self.get_local(&pack.url, size, &what, cancel, &mut on_bytes, &check)?
} else if pack.url.starts_with("https://") {
self.fetch(std::slice::from_ref(&pack.url), size, &what, cancel, &mut on_bytes, &check)?
} else {
return Err(Error::BadUrl(pack.url.chars().take(200).collect()));
};
if cancel.is_cancelled() {
return Err(Error::Cancelled);
}
self.sweep_staging();
let stage = Stage::new(self.dir())?;
let unpacked = stage.dir.join(&pack.locale);
unpack(&bytes, &unpacked, pack).map_err(|e| Error::Io { what: format!("unpacking {what}"), source: std::io::Error::other(e) })?;
let record = InstalledLanguagePack {
locale: pack.locale.clone(),
name: plain_text(&pack.name),
release: plain_text(&index.release),
version: plain_text(&pack.version),
licence: plain_text(&pack.licence),
sha256: pack.sha256.clone(),
size: pack.size,
};
let _lock = self.lock()?;
// The record first: rebuilding a damaged one must not count the pack being installed.
let mut installed = self.read_installed()?;
let packs = self.language_packs_dir();
std::fs::create_dir_all(&packs).map_err(io("creating the Language packs folder"))?;
let dest = packs.join(&pack.locale);
let old = stage.dir.join(format!("{}.old", pack.locale));
let had = std::fs::symlink_metadata(&dest).is_ok();
if had {
std::fs::rename(&dest, &old).map_err(io(format!("replacing the {} pack", pack.locale)))?;
}
if let Err(e) = std::fs::rename(&unpacked, &dest) {
if had {
let _ = std::fs::rename(&old, &dest);
}
return Err(io(format!("installing the {} pack", pack.locale))(e));
}
installed.packs.retain(|p| p.locale != record.locale);
installed.packs.push(record.clone());
installed.packs.sort_by(|a, b| a.locale.cmp(&b.locale));
if let Err(e) = self.write_installed(&installed) {
// Put the old pack back, so folder and record agree.
let _ = std::fs::remove_dir_all(&dest);
if had {
let _ = std::fs::rename(&old, &dest);
}
return Err(e);
}
Ok(record)
}
/// The installed Language packs whose folder is there, by locale.
pub fn installed_language_packs(&self) -> Vec<InstalledLanguagePack> {
let dir = self.language_packs_dir();
// Read only: without the store lock this never writes the record (an install or removal
// may be under way); a record that can't be read is listed from the folders and repaired
// by the next install or removal, which hold the lock.
let installed = self.read_record().unwrap_or_else(|| self.installed_from_disk());
installed
.packs
.into_iter()
.filter(|p| is_locale(&p.locale) && dir.join(&p.locale).is_dir())
.map(|p| InstalledLanguagePack {
name: plain_text(&p.name),
release: plain_text(&p.release),
version: plain_text(&p.version),
licence: plain_text(&p.licence),
..p
})
.collect()
}
/// Removes the pack for `locale`: its folder disappears at once (one rename) and is deleted
/// after. No network needed. Then call `craft_proof::packs::reload()`.
pub fn remove_language_pack(&self, locale: &str) -> Result<(), Error> {
if !is_locale(locale) {
return Err(Error::NotInstalled(locale.chars().take(40).collect()));
}
let stage = Stage::new(self.dir())?;
let _lock = self.lock()?;
let dest = self.language_packs_dir().join(locale);
let mut installed = self.read_installed()?;
let known = installed.packs.iter().any(|p| p.locale == locale);
if !known && std::fs::symlink_metadata(&dest).is_err() {
return Err(Error::NotInstalled(locale.to_string()));
}
if std::fs::symlink_metadata(&dest).is_ok() {
std::fs::rename(&dest, stage.dir.join(locale)).map_err(io(format!("removing the {locale} pack")))?;
}
installed.packs.retain(|p| p.locale != locale);
self.write_installed(&installed)
// `stage` deletes the old folder when dropped.
}
/// The record of installed packs. One that can't be read is copied aside as
/// `language-packs.json.bad` and rebuilt from the pack folders on disk (written atomically
/// over it), so no installed pack is ever dropped from it; a missing one is rebuilt the same
/// way when pack folders are there (none: empty).
fn read_installed(&self) -> Result<Installed, Error> {
let path = self.dir().join(INSTALLED);
let there = std::fs::symlink_metadata(&path).is_ok();
if let Some(i) = self.read_record() {
return Ok(i);
}
let rebuilt = self.installed_from_disk();
if !there && rebuilt.packs.is_empty() {
return Ok(Installed::default());
}
log::warn!("{}: {}; rebuilt from the Language pack folders", path.display(), if there { "can't be read" } else { "missing" });
if there {
let _ = std::fs::copy(&path, self.dir().join(format!("{INSTALLED}.bad")));
}
self.write_installed(&rebuilt)?;
Ok(rebuilt)
}
/// The record as written, if there is one that can be read.
fn read_record(&self) -> Option<Installed> {
read_limited(&self.dir().join(INSTALLED), MAX_PACK_INDEX_BYTES).and_then(|b| serde_json::from_slice::<Installed>(&b).ok())
}
/// The installed packs as their folders say: each folder named by a locale, with the name
/// and licence from its `pack.toml` (release, version and sha256 unknown).
fn installed_from_disk(&self) -> Installed {
let mut packs: Vec<InstalledLanguagePack> = std::fs::read_dir(self.language_packs_dir())
.into_iter()
.flatten()
.flatten()
.filter_map(|e| e.file_name().into_string().ok())
.filter(|l| is_locale(l))
.take(MAX_PACKS)
.map(|locale| {
let info = read_limited(&self.language_packs_dir().join(&locale).join("pack.toml"), MAX_PACK_TOML_BYTES)
.and_then(|b| String::from_utf8(b).ok())
.and_then(|t| t.parse::<toml::Table>().ok());
let field = |k: &str| info.as_ref().and_then(|t| t.get(k)).and_then(toml::Value::as_str).map(plain_text);
InstalledLanguagePack {
name: field("name").unwrap_or_else(|| locale.clone()),
licence: field("licence").unwrap_or_default(),
locale,
release: String::new(),
version: String::new(),
sha256: String::new(),
size: 0,
}
})
.collect();
packs.sort_by(|a, b| a.locale.cmp(&b.locale));
Installed { version: 1, packs }
}
fn write_installed(&self, installed: &Installed) -> Result<(), Error> {
let bytes = serde_json::to_vec_pretty(&Installed { version: 1, packs: installed.packs.clone() })
.map_err(|e| io("saving the Language pack list")(std::io::Error::other(e)))?;
self.write_file(INSTALLED, &bytes)
}
}
/// The zstd frames of `archive` one after another, as one stream (`zstd` writes one frame, but
/// a concatenation of frames is a valid `.zst` too, and must not stop after the first).
struct Frames<'a> {
decoder: Option<ruzstd::decoding::StreamingDecoder<&'a [u8], ruzstd::decoding::FrameDecoder>>,
/// Frames started so far, empty ones included (at most [`MAX_PACK_FRAMES`]).
frames: usize,
}
/// Most zstd frames in a pack archive (`zstd` writes one; empty frames count too).
const MAX_PACK_FRAMES: usize = 4096;
impl<'a> Frames<'a> {
fn new(archive: &'a [u8]) -> Result<Self, String> {
Ok(Self { decoder: Some(ruzstd::decoding::StreamingDecoder::new(archive).map_err(|e| format!("not zstd: {e}"))?), frames: 1 })
}
}
impl Read for Frames<'_> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
loop {
let Some(d) = self.decoder.as_mut() else { return Ok(0) };
let n = d.read(buf)?;
if n > 0 || buf.is_empty() {
return Ok(n);
}
let rest: &[u8] = d.get_ref();
if !rest.is_empty() {
self.frames += 1;
if self.frames > MAX_PACK_FRAMES {
return Err(std::io::Error::other(format!("more than {MAX_PACK_FRAMES} zstd frames")));
}
}
self.decoder = if rest.is_empty() { None } else { Some(ruzstd::decoding::StreamingDecoder::new(rest).map_err(std::io::Error::other)?) };
}
}
}
/// Unpacks the `.tar.zst` of `pack` into `dest` (created). Only the contract's files are allowed:
/// `pack.toml`, `spelling.aff`, `spelling.dic`, `hyphenation.pat` at the root and
/// `LICENCES/<name>` ([`licence_name`]); regular files and that folder only, each file at most
/// [`MAX_PACK_FILE_BYTES`], all of them [`MAX_PACK_UNPACKED_BYTES`], at most [`MAX_PACK_ENTRIES`].
/// Every zstd frame is read, and nothing but tar padding may follow the archive's end. `pack.toml`
/// must name `pack`'s locale, and the files the index promises (`spelling`, `hyphenation`) must
/// be there.
pub(crate) fn unpack(archive: &[u8], dest: &Path, pack: &PackEntry) -> Result<(), String> {
// The decompressed stream is capped too: tar padding and headers beyond the files' bytes.
let cap = MAX_PACK_UNPACKED_BYTES.saturating_add((MAX_PACK_ENTRIES as u64 + 2) * 1024 + 20 * 512);
let mut tar = tar::Archive::new(Frames::new(archive)?.take(cap));
std::fs::create_dir_all(dest).map_err(|e| e.to_string())?;
let (mut total, mut count) = (0u64, 0usize);
let mut written: Vec<String> = Vec::new();
for entry in tar.entries().map_err(|e| format!("not a tar archive: {e}"))? {
let mut entry = entry.map_err(|e| format!("damaged archive: {e}"))?;
count += 1;
if count > MAX_PACK_ENTRIES {
return Err("too many files".into());
}
let raw = entry.path().map_err(|e| format!("bad file name: {e}"))?.into_owned();
let rel = safe_entry_path(&raw).ok_or_else(|| format!("{} is outside the pack", raw.display()))?;
let parts: Vec<&str> = rel.iter().filter_map(|c| c.to_str()).collect();
let kind = entry.header().entry_type();
if kind.is_dir() {
match parts.as_slice() {
[] => {}
[LICENCES_DIR] => std::fs::create_dir_all(dest.join(LICENCES_DIR)).map_err(|e| e.to_string())?,
_ => return Err(format!("{}: not a folder a pack has", rel.display())),
}
continue;
}
if !kind.is_file() {
return Err(format!("{}: only files and folders are allowed", rel.display()));
}
let allowed = match parts.as_slice() {
[name] => ROOT_FILES.contains(name),
[LICENCES_DIR, name] => licence_name(name),
_ => false,
};
let key = parts.join("/");
if !allowed || written.contains(&key) {
return Err(format!("{}: not a file a pack has, or listed twice", rel.display()));
}
let size = entry.header().size().map_err(|e| e.to_string())?;
if size > MAX_PACK_FILE_BYTES {
return Err(format!("{}: larger than {MAX_PACK_FILE_BYTES} bytes", rel.display()));
}
total = total.saturating_add(size);
if total > MAX_PACK_UNPACKED_BYTES {
return Err(format!("larger than {MAX_PACK_UNPACKED_BYTES} bytes unpacked"));
}
let path = dest.join(&rel);
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
}
let mut out = std::fs::File::options().write(true).create_new(true).open(&path).map_err(|e| format!("{}: {e}", rel.display()))?;
let copied = std::io::copy(&mut (&mut entry).take(MAX_PACK_FILE_BYTES), &mut out).map_err(|e| format!("{}: {e}", rel.display()))?;
if copied != size {
return Err(format!("{}: truncated", rel.display()));
}
written.push(key);
}
// Only zeros (the tar end blocks and padding) may follow: not a second archive, not a frame
// that wasn't read.
let mut rest = tar.into_inner();
let mut buf = [0u8; 8192];
loop {
let n = rest.read(&mut buf).map_err(|e| format!("damaged archive: {e}"))?;
if n == 0 {
break;
}
if buf.get(..n).is_some_and(|b| b.iter().any(|&x| x != 0)) {
return Err("data after the end of the archive".into());
}
}
let has = |f: &str| written.iter().any(|w| w == f);
if !has("pack.toml") {
return Err("no pack.toml".into());
}
let toml = std::fs::read_to_string(dest.join("pack.toml")).map_err(|e| format!("pack.toml: {e}"))?;
let named = toml.parse::<toml::Table>().ok().and_then(|t| t.get("locale").and_then(toml::Value::as_str).map(str::to_string));
if named.as_deref() != Some(pack.locale.as_str()) {
return Err(format!("pack.toml names locale {named:?}, not {:?}", pack.locale));
}
if pack.spelling && !(has("spelling.aff") && has("spelling.dic")) {
return Err("the index promises a spelling dictionary the archive doesn't have".into());
}
if pack.hyphenation && !has("hyphenation.pat") {
return Err("the index promises hyphenation patterns the archive doesn't have".into());
}
Ok(())
}
/// The files a pack holds at its root.
const ROOT_FILES: [&str; 4] = ["pack.toml", "spelling.aff", "spelling.dic", "hyphenation.pat"];
/// The folder of a pack's licence texts.
const LICENCES_DIR: &str = "LICENCES";
/// Largest `pack.toml` read back.
const MAX_PACK_TOML_BYTES: u64 = 1 << 20;
/// A licence file's name: `[A-Za-z0-9._-]`, at most 100 characters, not starting with a dot or
/// ending with one, and not a name Windows reserves.
fn licence_name(n: &str) -> bool {
!n.is_empty()
&& n.len() <= 100
&& !n.starts_with('.')
&& !n.ends_with('.')
&& n.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-'))
&& !windows_reserved(n)
}
/// A name Windows reserves for a device (`CON`, `nul.txt`, `COM1`, `lpt9.md`), with or without an
/// extension, in any case.
fn windows_reserved(n: &str) -> bool {
let stem = n.split('.').next().unwrap_or_default().trim_end_matches(' ').to_ascii_lowercase();
matches!(stem.as_str(), "con" | "prn" | "aux" | "nul")
|| (stem.len() == 4
&& (stem.starts_with("com") || stem.starts_with("lpt"))
&& stem.as_bytes().get(3).is_some_and(|d| (b'1'..=b'9').contains(d)))
}
/// Text from an index or `pack.toml` shown to people (a pack's name): without control,
/// bidirectional or other format characters (U+202E would reverse what follows), trimmed, at most
/// 200 characters.
fn plain_text(s: &str) -> String {
s.chars().filter(|&c| !c.is_control() && !is_format(c)).take(200).collect::<String>().trim().to_string()
}
/// Unicode format characters (general category Cf): bidirectional controls, zero-width ones, the
/// soft hyphen, tags.
fn is_format(c: char) -> bool {
matches!(c,
'\u{ad}' | '\u{600}'..='\u{605}' | '\u{61c}' | '\u{6dd}' | '\u{70f}' | '\u{890}'..='\u{891}' | '\u{8e2}' | '\u{180e}'
| '\u{200b}'..='\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2060}'..='\u{2064}' | '\u{2066}'..='\u{206f}' | '\u{feff}'
| '\u{fff9}'..='\u{fffb}' | '\u{110bd}' | '\u{110cd}' | '\u{13430}'..='\u{1343f}' | '\u{1bca0}'..='\u{1bca3}'
| '\u{1d173}'..='\u{1d17a}' | '\u{e0001}' | '\u{e0020}'..='\u{e007f}')
}
/// `p` as a plain relative path (`./` prefixes dropped), or `None` for a root, a drive, `..` or a
/// name that isn't UTF-8 or holds a backslash, colon or control character.
fn safe_entry_path(p: &Path) -> Option<PathBuf> {
let mut out = PathBuf::new();
for c in p.components() {
match c {
Component::CurDir => {}
Component::Normal(n) => {
let n = n.to_str()?;
if n.is_empty() || n.contains(['\\', ':']) || n.chars().any(char::is_control) {
return None;
}
out.push(n);
}
Component::ParentDir | Component::RootDir | Component::Prefix(_) => return None,
}
}
Some(out)
}
#[cfg(test)]
mod tests;
crates/font-store/src/language_packs/tests.rs
+472
View File
@@ -0,0 +1,472 @@
use std::collections::HashMap;
use std::io::{Read, Write};
use std::net::{TcpListener, TcpStream};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use super::*;
use crate::store::Caller;
const RELEASE: &str = "https://github.com/storytold/craft-i18n/releases/download/language-packs-1/";
/// A local HTTP server answering 200 with the body set for a path, else 404.
struct Server {
base: String,
routes: Arc<Mutex<HashMap<String, Vec<u8>>>>,
}
impl Server {
fn start() -> Self {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let base = format!("http://{}/", listener.local_addr().unwrap());
let routes: Arc<Mutex<HashMap<String, Vec<u8>>>> = Arc::default();
let r = routes.clone();
std::thread::spawn(move || {
for stream in listener.incoming().flatten() {
let r = r.clone();
std::thread::spawn(move || answer(stream, &r));
}
});
Self { base, routes }
}
fn serve(&self, path: &str, body: &[u8]) {
self.routes.lock().unwrap().insert(format!("/{path}"), body.to_vec());
}
/// A client that maps the release URL onto this server.
fn client(&self) -> Client {
let t = Timeouts { connect: Duration::from_secs(2), response: Duration::from_secs(5), overall: Duration::from_secs(10) };
Client::for_tests(t, vec![(RELEASE.into(), format!("{}release/", self.base))])
}
}
fn answer(mut s: TcpStream, routes: &Mutex<HashMap<String, Vec<u8>>>) {
let mut head = Vec::new();
let mut b = [0u8; 1];
while !head.ends_with(b"\r\n\r\n") {
match s.read(&mut b) {
Ok(1) => head.push(b[0]),
_ => return,
}
}
let path = String::from_utf8_lossy(&head).split_whitespace().nth(1).unwrap_or("/").to_string();
let body = routes.lock().unwrap().get(&path).cloned();
match body {
Some(body) => {
let _ = write!(s, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len());
let _ = s.write_all(&body);
}
None => {
let _ = write!(s, "HTTP/1.1 404 Nope\r\nContent-Length: 4\r\nConnection: close\r\n\r\nnope");
}
}
}
fn temp(name: &str) -> PathBuf {
static N: AtomicUsize = AtomicUsize::new(0);
let d = std::env::temp_dir().join(format!("craft-language-packs-{name}-{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed)));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
/// A tar of `files` (name, kind, bytes), with raw header names so hostile ones can be written.
fn tar_of(files: &[(&str, tar::EntryType, &[u8])]) -> Vec<u8> {
let mut b = tar::Builder::new(Vec::new());
for (name, kind, bytes) in files {
let mut h = tar::Header::new_old();
let field = &mut h.as_old_mut().name;
field[..name.len()].copy_from_slice(name.as_bytes());
h.set_entry_type(*kind);
h.set_size(bytes.len() as u64);
h.set_mode(0o644);
if *kind == tar::EntryType::Symlink || *kind == tar::EntryType::Link {
h.set_link_name("/etc/passwd").unwrap();
}
h.set_cksum();
b.append(&h, *bytes).unwrap();
}
b.into_inner().unwrap()
}
fn zst(bytes: &[u8]) -> Vec<u8> {
ruzstd::encoding::compress_to_vec(bytes, ruzstd::encoding::CompressionLevel::Fastest)
}
const FILE: tar::EntryType = tar::EntryType::Regular;
/// A good pack archive for `locale`, with `marker` in its word list.
fn pack_archive(locale: &str, marker: &str) -> Vec<u8> {
let toml = format!("locale = \"{locale}\"\nname = \"Test\"\nlicence = \"MIT\"\nclass = \"Embeddable\"\n");
let dic = format!("2\nhuis\n{marker}\n");
zst(&tar_of(&[
("pack.toml", FILE, toml.as_bytes()),
("spelling.aff", FILE, b"SET UTF-8\n"),
("spelling.dic", FILE, dic.as_bytes()),
("hyphenation.pat", FILE, b"1na n2at\n"),
("LICENCES/", tar::EntryType::Directory, b""),
("LICENCES/COPYING", FILE, b"Test licence."),
]))
}
fn entry(locale: &str, url: &str, archive: &[u8]) -> serde_json::Value {
serde_json::json!({"locale": locale, "name": "Nederlands", "version": "abc", "url": url, "sha256": sha256_hex(archive),
"size": archive.len(), "licence": "MIT", "class": "Embeddable", "spelling": true, "hyphenation": true})
}
/// The index entry for `archive` as `locale`.
fn pack_entry(locale: &str, archive: &[u8]) -> PackEntry {
serde_json::from_value(entry(locale, &format!("{RELEASE}{locale}.tar.zst"), archive)).unwrap()
}
fn index_json(packs: &[serde_json::Value]) -> Vec<u8> {
serde_json::to_vec(&serde_json::json!({"format": 1, "release": "language-packs-1", "packs": packs})).unwrap()
}
/// A store whose network is on, a server with the index and the `nl` pack, and the pin.
fn setup(name: &str) -> (FontStore, Server, PackIndexPin, Vec<u8>) {
let server = Server::start();
let archive = pack_archive("nl", "zzmarker");
let index = index_json(&[entry("nl", &format!("{RELEASE}nl.tar.zst"), &archive)]);
server.serve("release/nl.tar.zst", &archive);
server.serve("release/index.json", &index);
let sha: &'static str = Box::leak(sha256_hex(&index).into_boxed_str());
let pin = PackIndexPin { url: "https://github.com/storytold/craft-i18n/releases/download/language-packs-1/index.json", sha256: sha };
let store = FontStore::new(temp(name), Caller::User).with_client(server.client());
store.enable_network().unwrap();
(store, server, pin, archive)
}
#[test]
fn install_list_replace_and_remove_a_pack() {
let (store, server, pin, _) = setup("install");
let source = IndexSource::Pinned(pin);
let index = store.fetch_pack_index_from(&source, &Cancel::new()).unwrap();
assert_eq!(store.cached_pack_index_from(&source), Some(index.clone()));
let nl = index.pack("nl").unwrap().clone();
let mut seen = Vec::new();
let got = store.install_pack_from(false, &index, &nl, &Cancel::new(), &mut |p| seen.push(p)).unwrap();
assert_eq!((got.locale.as_str(), got.release.as_str(), got.licence.as_str()), ("nl", "language-packs-1", "MIT"));
assert!(seen.last().is_some_and(|p| p.done == nl.size && p.total == nl.size));
let dir = store.language_packs_dir().join("nl");
assert_eq!(store.language_packs_dir(), store.dir().join("language-packs"));
for f in ["pack.toml", "spelling.aff", "spelling.dic", "hyphenation.pat", "LICENCES/COPYING"] {
assert!(dir.join(f).is_file(), "{f}");
}
assert!(std::fs::read_to_string(dir.join("spelling.dic")).unwrap().contains("zzmarker"));
assert_eq!(store.installed_language_packs(), vec![got]);
// A newer archive replaces the old one in one move.
let newer = pack_archive("nl", "zznewer");
server.serve("release/nl.tar.zst", &newer);
let newer_entry = PackEntry { sha256: sha256_hex(&newer), size: newer.len() as u64, ..nl.clone() };
store.install_pack_from(false, &index, &newer_entry, &Cancel::new(), &mut |_| {}).unwrap();
assert!(std::fs::read_to_string(dir.join("spelling.dic")).unwrap().contains("zznewer"));
assert_eq!(store.installed_language_packs().len(), 1);
assert!(std::fs::read_dir(store.dir().join(".staging")).into_iter().flatten().flatten().next().is_none(), "nothing left staged");
store.remove_language_pack("nl").unwrap();
assert!(!dir.exists() && store.installed_language_packs().is_empty());
assert!(matches!(store.remove_language_pack("nl"), Err(Error::NotInstalled(_))));
assert!(matches!(store.remove_language_pack("../x"), Err(Error::NotInstalled(_))));
}
#[test]
fn network_off_pins_and_checksums_are_enforced() {
let (store, server, pin, archive) = setup("pins");
let source = IndexSource::Pinned(pin);
store.disable_network().unwrap();
assert!(matches!(store.fetch_pack_index_from(&source, &Cancel::new()), Err(Error::NetworkOff)));
store.enable_network().unwrap();
// Another index than the pinned one.
let wrong = IndexSource::Pinned(PackIndexPin { sha256: "0000000000000000000000000000000000000000000000000000000000000000", ..pin });
assert!(matches!(store.fetch_pack_index_from(&wrong, &Cancel::new()), Err(Error::Verify { error: VerifyError::Sha256, .. })));
let index = store.fetch_pack_index_from(&source, &Cancel::new()).unwrap();
assert_eq!(store.cached_pack_index_from(&wrong), None, "the cache is the pinned index only");
let nl = index.pack("nl").unwrap().clone();
// An archive that isn't the one the index names.
server.serve("release/nl.tar.zst", &pack_archive("nl", "tampered"));
assert!(store.install_pack_from(false, &index, &nl, &Cancel::new(), &mut |_| {}).is_err());
server.serve("release/nl.tar.zst", &archive);
let bad_sha = PackEntry { sha256: sha256_hex(b"other"), ..nl.clone() };
assert!(matches!(
store.install_pack_from(false, &index, &bad_sha, &Cancel::new(), &mut |_| {}),
Err(Error::Verify { error: VerifyError::Sha256, .. })
));
assert!(store.installed_language_packs().is_empty() && !store.language_packs_dir().join("nl").exists());
// Hand-made entries are checked again.
for e in [PackEntry { locale: "../nl".into(), ..nl.clone() }, PackEntry { url: "http://example.org/nl.tar.zst".into(), ..nl.clone() }] {
assert!(store.install_pack_from(false, &index, &e, &Cancel::new(), &mut |_| {}).is_err());
}
store.disable_network().unwrap();
assert!(matches!(store.install_pack_from(false, &index, &nl, &Cancel::new(), &mut |_| {}), Err(Error::NetworkOff)));
}
#[test]
fn indexes_are_checked() {
let a = pack_archive("nl", "x");
let ok = index_json(&[entry("nl", &format!("{RELEASE}nl.tar.zst"), &a)]);
assert_eq!(PackIndex::from_json(&ok).unwrap().packs.len(), 1);
let bad = [
serde_json::to_vec(&serde_json::json!({"format": 2, "release": "r", "packs": []})).unwrap(),
index_json(&[entry("NL", &format!("{RELEASE}nl.tar.zst"), &a)]),
index_json(&[entry("../x", &format!("{RELEASE}nl.tar.zst"), &a)]),
index_json(&[entry("nl", &format!("{RELEASE}nl.tar.zst"), &a), entry("nl", &format!("{RELEASE}nl.tar.zst"), &a)]),
index_json(&[entry("nl", "http://127.0.0.1/nl.tar.zst", &a)]),
index_json(&[entry("nl", "/tmp/nl.tar.zst", &a)]),
b"not json".to_vec(),
];
for b in bad {
assert!(PackIndex::from_json(&b).is_err(), "{}", String::from_utf8_lossy(&b));
}
// The developer override also takes loopback http and paths, never other http hosts.
assert!(PackIndex::parse(&index_json(&[entry("nl", "http://127.0.0.1:9/nl.tar.zst", &a)]), true).is_ok());
assert!(PackIndex::parse(&index_json(&[entry("nl", "/tmp/nl.tar.zst", &a)]), true).is_ok());
assert!(PackIndex::parse(&index_json(&[entry("nl", "http://example.org/nl.tar.zst", &a)]), true).is_err());
for (s, ok) in [
("http://127.0.0.1:8000/i.json", true),
("http://localhost/i.json", true),
("http://[::1]:8/i.json", true),
("http://127.0.0.1.evil.org/i.json", false),
("http://evil.org@127.0.0.1/i.json", true),
("http://127.0.0.1@evil.org/i.json", false),
("ftp://127.0.0.1/i.json", false),
("file:///tmp/i.json", false),
] {
assert_eq!(local_source(s).is_some(), ok, "{s}");
}
}
/// The developer override (`CRAFT_LANGUAGE_PACK_INDEX`): an index from a local path or a
/// loopback http server, with no index pin, while archives are still checked against its sha256.
#[test]
fn developer_override_reads_a_local_index_without_the_pin() {
let (store, server, _, archive) = setup("override");
// A local file index pointing at a local archive.
let dir = temp("override-files");
std::fs::write(dir.join("nl.tar.zst"), &archive).unwrap();
let index = index_json(&[entry("nl", &dir.join("nl.tar.zst").display().to_string(), &archive)]);
std::fs::write(dir.join("index.json"), &index).unwrap();
let source = IndexSource::Override(dir.join("index.json").display().to_string());
let got = store.fetch_pack_index_from(&source, &Cancel::new()).unwrap();
assert_eq!(store.cached_pack_index_from(&source), Some(got.clone()));
store.install_pack_from(true, &got, got.pack("nl").unwrap(), &Cancel::new(), &mut |_| {}).unwrap();
assert!(store.language_packs_dir().join("nl/spelling.dic").is_file());
// A loopback http index and archive.
let http_index = index_json(&[entry("nl", &format!("{}local/nl.tar.zst", server.base), &archive)]);
server.serve("local/index.json", &http_index);
server.serve("local/nl.tar.zst", &archive);
let source = IndexSource::Override(format!("{}local/index.json", server.base));
let got = store.fetch_pack_index_from(&source, &Cancel::new()).unwrap();
store.install_pack_from(true, &got, got.pack("nl").unwrap(), &Cancel::new(), &mut |_| {}).unwrap();
// The archive is still verified.
server.serve("local/nl.tar.zst", &pack_archive("nl", "tampered"));
assert!(store.install_pack_from(true, &got, got.pack("nl").unwrap(), &Cancel::new(), &mut |_| {}).is_err());
// Only loopback http.
let source = IndexSource::Override("http://example.org/index.json".into());
assert!(matches!(store.fetch_pack_index_from(&source, &Cancel::new()), Err(Error::BadUrl(_))));
// Without the variable the pin is used.
let pin = PackIndexPin { url: "https://example.org/index.json", sha256: "00" };
if std::env::var_os(LANGUAGE_PACK_INDEX_ENV).is_none() {
assert_eq!(IndexSource::from_env(&pin), IndexSource::Pinned(pin));
}
}
#[test]
fn hostile_archives_are_refused() {
let toml: &[u8] = b"locale = \"nl\"\nlicence = \"MIT\"\n";
let nl = PackEntry { spelling: false, hyphenation: false, ..pack_entry("nl", b"") };
let big = vec![0u8; 10];
let cases: Vec<(&str, Vec<u8>)> = vec![
("traversal", zst(&tar_of(&[("pack.toml", FILE, toml), ("../evil", FILE, b"x")]))),
("deep traversal", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/../../evil", FILE, b"x")]))),
("absolute", zst(&tar_of(&[("pack.toml", FILE, toml), ("/tmp/evil", FILE, b"x")]))),
("symlink", zst(&tar_of(&[("pack.toml", FILE, toml), ("spelling.dic", tar::EntryType::Symlink, b"")]))),
("hard link", zst(&tar_of(&[("pack.toml", FILE, toml), ("spelling.dic", tar::EntryType::Link, b"")]))),
("too deep", zst(&tar_of(&[("pack.toml", FILE, toml), ("a/b/c", FILE, b"x")]))),
("duplicate", zst(&tar_of(&[("pack.toml", FILE, toml), ("pack.toml", FILE, toml)]))),
("backslash", zst(&tar_of(&[("pack.toml", FILE, toml), ("..\\evil", FILE, b"x")]))),
("no pack.toml", zst(&tar_of(&[("spelling.dic", FILE, b"1\nx\n")]))),
("not zstd", tar_of(&[("pack.toml", FILE, toml)])),
("not tar", zst(b"hello, this is not a tar archive at all")),
("empty", Vec::new()),
// Only the contract's names.
("other root file", zst(&tar_of(&[("pack.toml", FILE, toml), ("readme.txt", FILE, b"x")]))),
("other folder", zst(&tar_of(&[("pack.toml", FILE, toml), ("extra/", tar::EntryType::Directory, b"")]))),
("nested licence", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/sub/x", FILE, b"x")]))),
("hidden licence", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/.x", FILE, b"x")]))),
("space in licence", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/a b", FILE, b"x")]))),
("trailing dot", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/COPYING.", FILE, b"x")]))),
("reserved CON", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/CON", FILE, b"x")]))),
("reserved nul.txt", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/nul.txt", FILE, b"x")]))),
("reserved com1", zst(&tar_of(&[("pack.toml", FILE, toml), ("LICENCES/com1.md", FILE, b"x")]))),
// pack.toml must name the entry's locale.
("other locale", zst(&tar_of(&[("pack.toml", FILE, b"locale = \"de\"\n")]))),
("no locale", zst(&tar_of(&[("pack.toml", FILE, b"licence = \"MIT\"\n")]))),
("bad toml", zst(&tar_of(&[("pack.toml", FILE, b"locale = \n")]))),
// Nothing may follow the archive: not another frame, not raw bytes.
("second archive", [zst(&tar_of(&[("pack.toml", FILE, toml)])), zst(&tar_of(&[("spelling.dic", FILE, b"1\nx\n")]))].concat()),
("trailing bytes", [zst(&tar_of(&[("pack.toml", FILE, toml)])), b"garbage".to_vec()].concat()),
("truncated", zst(&tar_of(&[("pack.toml", FILE, toml), ("spelling.dic", FILE, &big)]))[..20].to_vec()),
];
for (what, archive) in cases {
let root = temp("hostile");
let dest = root.join("nl");
assert!(unpack(&archive, &dest, &nl).is_err(), "{what}");
assert!(!root.join("evil").exists() && !root.parent().unwrap().join("evil").exists(), "{what}");
}
// Too many entries.
let names: Vec<String> = (0..=MAX_PACK_ENTRIES).map(|i| format!("LICENCES/f{i}")).collect();
let mut files: Vec<(&str, tar::EntryType, &[u8])> = vec![("pack.toml", FILE, toml)];
files.extend(names.iter().map(|n| (n.as_str(), FILE, &b""[..])));
assert!(unpack(&zst(&tar_of(&files)), &temp("many").join("nl"), &nl).is_err());
// A header claiming a file larger than the cap is refused before anything is read.
let mut t = tar::Header::new_gnu();
t.set_path("pack.toml").unwrap();
t.set_size(toml.len() as u64);
t.set_cksum();
let mut b = tar::Builder::new(Vec::new());
b.append(&t, toml).unwrap();
let mut tarball = b.into_inner().unwrap();
tarball.truncate(tarball.len() - 1024); // drop the end marker
let mut h = tar::Header::new_gnu();
h.set_path("spelling.dic").unwrap();
h.set_size(MAX_PACK_FILE_BYTES + 1);
h.set_cksum();
tarball.extend_from_slice(h.as_bytes());
tarball.extend_from_slice(&[0u8; 1024]);
let err = unpack(&zst(&tarball), &temp("huge").join("nl"), &nl).unwrap_err();
assert!(err.contains("larger than"), "{err}");
// A good one unpacks.
let dest = temp("good").join("nl");
let good = pack_archive("nl", "x");
unpack(&good, &dest, &pack_entry("nl", &good)).unwrap();
assert!(dest.join("LICENCES/COPYING").is_file());
// A tar split over several zstd frames is read to the end.
let tarball = tar_of(&[("pack.toml", FILE, toml), ("LICENCES/", tar::EntryType::Directory, b""), ("LICENCES/COPYING", FILE, b"t")]);
let (a, b) = tarball.split_at(700);
let dest = temp("frames").join("nl");
unpack(&[zst(a), zst(b)].concat(), &dest, &nl).unwrap();
assert!(dest.join("LICENCES/COPYING").is_file());
// Empty frames are read past, but only so many.
let one = zst(&tar_of(&[("pack.toml", FILE, toml)]));
let empty = zst(b"");
unpack(&[one.clone(), empty.clone(), empty.clone()].concat(), &temp("empties").join("nl"), &nl).unwrap();
let many: Vec<u8> = std::iter::once(one).chain(std::iter::repeat_n(empty, MAX_PACK_FRAMES)).flatten().collect();
let err = unpack(&many, &temp("frames-cap").join("nl"), &nl).unwrap_err();
assert!(err.contains("frames"), "{err}");
// The index's promises are checked against the files.
let bare = zst(&tar_of(&[("pack.toml", FILE, toml)]));
for e in [PackEntry { spelling: true, ..nl.clone() }, PackEntry { hyphenation: true, ..nl.clone() }] {
let err = unpack(&bare, &temp("promise").join("nl"), &e).unwrap_err();
assert!(err.contains("promises"), "{err}");
}
}
#[test]
fn names_are_plain_text_and_reserved_locales_refused() {
let a = pack_archive("nl", "x");
let mut e = entry("nl", &format!("{RELEASE}nl.tar.zst"), &a);
e["name"] = serde_json::json!(" Neder\u{202e}lands\u{7}\u{200b}\n");
assert_eq!(PackIndex::from_json(&index_json(&[e])).unwrap().packs[0].name, "Nederlands");
for l in ["con", "nul", "aux", "prn", "com1", "lpt9"] {
assert!(!is_locale(l), "{l}");
assert!(PackIndex::from_json(&index_json(&[entry(l, &format!("{RELEASE}{l}.tar.zst"), &a)])).is_err(), "{l}");
}
assert!(is_locale("com") && is_locale("lpt") && is_locale("com10"));
}
#[test]
fn a_corrupt_record_is_rebuilt_from_the_folders() {
let (store, server, pin, _) = setup("corrupt");
let index = store.fetch_pack_index_from(&IndexSource::Pinned(pin), &Cancel::new()).unwrap();
let nl = index.pack("nl").unwrap().clone();
store.install_pack_from(false, &index, &nl, &Cancel::new(), &mut |_| {}).unwrap();
let record = store.dir().join("language-packs.json");
std::fs::write(&record, b"{ not json").unwrap();
// Still listed, with the name from its pack.toml.
let listed = store.installed_language_packs();
assert_eq!(listed.iter().map(|p| (p.locale.as_str(), p.name.as_str())).collect::<Vec<_>>(), [("nl", "Test")]);
// craft-libs#486 review: listing holds no store lock, so it never writes the record (it
// could race an install); the next install or removal repairs it.
assert_eq!(std::fs::read(&record).unwrap(), b"{ not json");
assert!(!store.dir().join("language-packs.json.bad").exists());
// Installing another keeps it.
std::fs::write(&record, b"{ not json").unwrap();
let de = pack_archive("de", "zzde");
server.serve("release/de.tar.zst", &de);
let de_entry = PackEntry { url: format!("{RELEASE}de.tar.zst"), ..pack_entry("de", &de) };
store.install_pack_from(false, &index, &de_entry, &Cancel::new(), &mut |_| {}).unwrap();
let mut locales: Vec<String> = store.installed_language_packs().into_iter().map(|p| p.locale).collect();
locales.sort();
assert_eq!(locales, ["de", "nl"]);
assert_eq!(std::fs::read(store.dir().join("language-packs.json.bad")).unwrap(), b"{ not json", "the unreadable record is kept aside");
assert!(serde_json::from_slice::<serde_json::Value>(&std::fs::read(&record).unwrap()).is_ok(), "rebuilt in place");
// A missing record with pack folders is rebuilt too.
std::fs::remove_file(&record).unwrap();
assert_eq!(store.installed_language_packs().len(), 2);
assert!(!record.exists(), "listing writes nothing");
let nl_again = PackEntry { url: format!("{RELEASE}nl.tar.zst"), ..nl.clone() };
store.install_pack_from(false, &index, &nl_again, &Cancel::new(), &mut |_| {}).unwrap();
assert_eq!(store.installed_language_packs().len(), 2, "de kept when the record was missing");
// Removing works on a rebuilt record too.
std::fs::write(&record, b"\xff").unwrap();
store.remove_language_pack("nl").unwrap();
assert_eq!(store.installed_language_packs().into_iter().map(|p| p.locale).collect::<Vec<_>>(), ["de"]);
}
#[test]
fn index_text_is_plain_and_sources_are_checked() {
let a = pack_archive("nl", "x");
let mut e = entry("nl", &format!("{RELEASE}nl.tar.zst"), &a);
e["licence"] = serde_json::json!("MIT\u{202e}");
e["version"] = serde_json::json!("ab\u{0}c");
e["class"] = serde_json::json!("\u{200f}Embeddable");
e["source"] = serde_json::json!("https://example.org/nl");
let json = serde_json::to_vec(&serde_json::json!({"format": 1, "release": "r1\u{2066}", "packs": [e.clone()]})).unwrap();
let index = PackIndex::from_json(&json).unwrap();
let p = &index.packs[0];
assert_eq!((index.release.as_str(), p.licence.as_str(), p.version.as_str(), p.class.as_str()), ("r1", "MIT", "abc", "Embeddable"));
assert_eq!(p.source.as_deref(), Some("https://example.org/nl"));
assert_eq!(PackIndex::from_json(&index_json(&[entry("nl", &format!("{RELEASE}nl.tar.zst"), &a)])).unwrap().packs[0].source, None);
for bad in ["javascript:alert(1)", "http://example.org", "https://a b", "https://a\u{202e}b"] {
e["source"] = serde_json::json!(bad);
let err = PackIndex::from_json(&index_json(std::slice::from_ref(&e))).unwrap_err();
assert!(!err.to_string().contains("font"), "{err}");
}
}
#[test]
fn bad_hand_made_entries_are_index_errors() {
let (store, _server, pin, _) = setup("variant");
let index = store.fetch_pack_index_from(&IndexSource::Pinned(pin), &Cancel::new()).unwrap();
let nl = index.pack("nl").unwrap().clone();
for e in [PackEntry { locale: "../nl".into(), ..nl.clone() }, PackEntry { sha256: "x".into(), ..nl.clone() }] {
let got = store.install_pack_from(false, &index, &e, &Cancel::new(), &mut |_| {});
assert!(matches!(got, Err(Error::Index(IndexError::Invalid(_)))), "{got:?}");
}
}
#[test]
fn the_override_is_visible() {
let set = std::env::var(LANGUAGE_PACK_INDEX_ENV).ok().map(|v| v.trim().to_string()).filter(|v| !v.is_empty());
assert_eq!(language_pack_index_overridden(), set);
}
/// A staging folder of the same name (an earlier process, or another one in a PID namespace) is
/// never used or deleted: the next name is taken, so an unpack can't mix in its files.
#[test]
fn a_stale_stage_folder_is_cleared() {
let store = temp("stale-stage");
let staging = store.join(".staging");
for n in 0..512 {
let d = staging.join(format!("{}-{n}", std::process::id()));
std::fs::create_dir_all(d.join("nl")).unwrap();
std::fs::write(d.join("nl/pack.toml"), b"old").unwrap();
}
let stage = Stage::new(&store).unwrap();
assert_eq!(std::fs::read_dir(&stage.dir).unwrap().count(), 0, "{}", stage.dir.display());
// The others are left alone: they may be another process's (PID namespaces).
let left = std::fs::read_dir(&staging).unwrap().flatten().filter(|e| e.path().join("nl/pack.toml").is_file()).count();
assert_eq!(left, 512);
}
crates/font-store/src/lib.rs
+10 -2
View File
@@ -18,8 +18,9 @@
//! # }
//! ```
//!
//! The index and its lookups, [`verify_font`] and [`IndexPin`] build on every target. The store
//! and the HTTP client are native only.
//! The index and its lookups, [`verify_font`] and [`IndexPin`] build on every target. The store,
//! the HTTP client and Language packs (the store's second item kind, `language-pack`: see
//! [`FontStore::install_language_pack`]) are native only.
mod index;
mod pin;
@@ -30,6 +31,8 @@ mod verify;
#[cfg(not(target_arch = "wasm32"))]
mod http;
#[cfg(not(target_arch = "wasm32"))]
mod language_packs;
#[cfg(not(target_arch = "wasm32"))]
mod store;
pub use index::{
@@ -43,6 +46,11 @@ pub use verify::{VerifyError, is_font, sha256_hex, verify_font};
#[cfg(not(target_arch = "wasm32"))]
pub use http::{Cancel, Client, FetchError, Timeouts, USER_AGENT};
#[cfg(not(target_arch = "wasm32"))]
pub use language_packs::{
InstalledLanguagePack, LANGUAGE_PACK_INDEX_ENV, LANGUAGE_PACKS, MAX_PACK_ARCHIVE_BYTES, MAX_PACK_ENTRIES, MAX_PACK_FILE_BYTES,
MAX_PACK_INDEX_BYTES, MAX_PACK_UNPACKED_BYTES, PackEntry, PackIndex, PackIndexPin, language_pack_index_overridden,
};
#[cfg(not(target_arch = "wasm32"))]
pub use store::{Caller, Error, FontStore, InstalledFamily, InstalledFile, Network, Problem, Progress, UserSource};
#[cfg(test)]
crates/font-store/src/store.rs
+28 -14
View File
@@ -7,6 +7,8 @@
//! manifest.json what is installed
//! indexes/ the last indexes fetched: sources.json, one file per source
//! <family dir>/ google/fonts' folder name, or <source>/<family>: the fonts and the licence
//! language-packs/ installed Language packs, one folder per locale (`crate::language_packs`)
//! language-packs.json which Language packs are installed, from which index entry
//! .staging/ downloads being installed; nothing a font scan picks up
//! .lock held while the store changes
//! ```
@@ -30,7 +32,7 @@ use crate::verify::{VerifyError, is_font, sha256_hex, verify_font};
const SETTINGS: &str = "settings.json";
const MANIFEST: &str = "manifest.json";
const INDEX_CACHE: &str = "indexes";
pub(crate) const INDEX_CACHE: &str = "indexes";
const SOURCES_CACHE: &str = "sources.json";
const STAGING: &str = ".staging";
const LOCK: &str = ".lock";
@@ -111,7 +113,7 @@ pub enum Error {
},
}
fn io(what: impl Into<String>) -> impl FnOnce(std::io::Error) -> Error {
pub(crate) fn io(what: impl Into<String>) -> impl FnOnce(std::io::Error) -> Error {
let what = what.into();
move |source| Error::Io { what, source }
}
@@ -336,7 +338,7 @@ impl FontStore {
self.dir.join(INDEX_CACHE).join(format!("user-{key}"))
}
fn require_network(&self) -> Result<(), Error> {
pub(crate) fn require_network(&self) -> Result<(), Error> {
if self.network() == Network::On { Ok(()) } else { Err(Error::NetworkOff) }
}
@@ -545,7 +547,7 @@ impl FontStore {
}
/// Downloads from each URL in turn until one gives bytes that pass `check`.
fn fetch(
pub(crate) fn fetch(
&self,
urls: &[String],
max: u64,
@@ -712,7 +714,7 @@ impl FontStore {
}
}
fn lock(&self) -> Result<File, Error> {
pub(crate) fn lock(&self) -> Result<File, Error> {
std::fs::create_dir_all(&self.dir).map_err(io("creating the font store folder"))?;
let file = File::options().create(true).truncate(false).write(true).open(self.dir.join(LOCK)).map_err(io("locking the font store"))?;
file.lock().map_err(io("locking the font store"))?;
@@ -740,12 +742,12 @@ impl FontStore {
self.write_file(MANIFEST, &bytes)
}
fn write_file(&self, name: &str, bytes: &[u8]) -> Result<(), Error> {
pub(crate) fn write_file(&self, name: &str, bytes: &[u8]) -> Result<(), Error> {
std::fs::create_dir_all(&self.dir).map_err(io("creating the font store folder"))?;
craft_atomic::atomic_write(&self.dir.join(name), bytes).map_err(io(format!("saving {name}")))
}
fn sweep_staging(&self) {
pub(crate) fn sweep_staging(&self) {
for e in std::fs::read_dir(self.dir.join(STAGING)).into_iter().flatten().flatten() {
let old = e.metadata().and_then(|m| m.modified()).is_ok_and(|t| SystemTime::now().duration_since(t).is_ok_and(|d| d > STALE));
if old {
@@ -818,7 +820,7 @@ fn user_index(bytes: &[u8], user: &UserSource) -> Result<Index, IndexError> {
Ok(index)
}
fn read_limited(path: &Path, max: u64) -> Option<Vec<u8>> {
pub(crate) fn read_limited(path: &Path, max: u64) -> Option<Vec<u8>> {
if std::fs::metadata(path).ok()?.len() > max {
return None;
}
@@ -830,16 +832,28 @@ fn host(url: &str) -> &str {
}
/// A folder of downloads under `.staging`, deleted when dropped.
struct Stage {
dir: PathBuf,
pub(crate) struct Stage {
pub(crate) dir: PathBuf,
}
impl Stage {
fn new(store: &Path) -> Result<Self, Error> {
pub(crate) fn new(store: &Path) -> Result<Self, Error> {
static N: AtomicU64 = AtomicU64::new(0);
let dir = store.join(STAGING).join(format!("{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed)));
std::fs::create_dir_all(&dir).map_err(io("creating a download folder"))?;
Ok(Self { dir })
let staging = store.join(STAGING);
std::fs::create_dir_all(&staging).map_err(io("creating a download folder"))?;
// A folder of that name may be another process's (PID namespaces in Flatpak or
// containers repeat ids) or a stale one: never touched here, the next name is taken
// (stale ones go in `sweep_staging`).
let mut last = None;
for _ in 0..1000 {
let dir = staging.join(format!("{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed)));
match std::fs::create_dir(&dir) {
Ok(()) => return Ok(Self { dir }),
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => last = Some(e),
Err(e) => return Err(io("creating a download folder")(e)),
}
}
Err(io("creating a download folder")(last.unwrap_or_else(|| std::io::Error::other("no free name"))))
}
/// Writes a verified download as `<name>.part`, which no font scan picks up.
crates/font-store/src/verify.rs
+1 -1
View File
@@ -20,7 +20,7 @@ pub enum VerifyError {
actual: u64,
},
/// The sha256 differs.
#[error("the file's sha256 doesn't match the font index")]
#[error("the file's sha256 doesn't match the index")]
Sha256,
/// It doesn't parse as a font with outlines.
#[error("the file is not a font, or a damaged one")]
crates/media-testkit/Cargo.toml
+12
View File
@@ -0,0 +1,12 @@
[package]
name = "craft-media-testkit"
description = "Oracle-test support (ffmpeg discovery, fixture locations) for the craft-libs media crates. Test support only."
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
publish = false
[lints]
workspace = true
crates/media-testkit/src/lib.rs
+159
View File
@@ -0,0 +1,159 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
//! Oracle-test support for the craft-libs media crates (craft-media, craft-audio, and the video
//! crate to come): ffmpeg/ffprobe discovery, the skip-or-fail rule, and fixture locations.
//! ffmpeg is used only as an external fixture generator and oracle (never linked). Test support
//! only: unpublished, and used as a dev-dependency.
//!
//! Tool search order: `CRAFT_MEDIA_FFMPEG` / `CRAFT_MEDIA_FFPROBE` (an explicit path; a wrong
//! path fails the test), then `PATH`, then well-known install directories. A missing tool, a
//! failed fixture generator or a failed ffprobe run prints `SKIPPED` and the test passes, unless
//! `CRAFT_MEDIA_REQUIRE_ORACLES=1` is set: then each of these fails the test.
//! Fixtures are generated into `$CRAFT_MEDIA_FIXTURES_DIR` or `<workspace>/target/fixtures`, one
//! directory per crate or format, and never committed.
//!
//! Failing loudly is what a test helper is for: a wrong `CRAFT_MEDIA_*` setting or an unwritable
//! fixture directory panics, which fails the test that asked.
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicUsize, Ordering};
const REQUIRE_ENV: &str = "CRAFT_MEDIA_REQUIRE_ORACLES";
const KNOWN_DIRS: &[&str] = &[
"/opt/homebrew/bin",
"/usr/local/bin",
"/usr/bin",
"/opt/local/bin",
"/snap/bin",
r"C:\ffmpeg\bin",
r"C:\Program Files\ffmpeg\bin",
r"C:\ProgramData\chocolatey\bin",
];
fn env_var(name: &str) -> String {
format!("CRAFT_MEDIA_{}", name.to_ascii_uppercase())
}
#[allow(clippy::panic)]
fn find_tool(name: &str) -> Option<PathBuf> {
if let Some(e) = std::env::var_os(env_var(name)).filter(|e| !e.is_empty()) {
let p = PathBuf::from(e);
assert!(p.is_file(), "{} is set but `{}` is not a file", env_var(name), p.display());
return Some(p);
}
let names = if cfg!(windows) { [format!("{name}.exe"), name.to_string()] } else { [name.to_string(), format!("{name}.exe")] };
let path_dirs = std::env::var_os("PATH").map(|p| std::env::split_paths(&p).collect::<Vec<_>>()).unwrap_or_default();
let dirs = path_dirs.into_iter().chain(KNOWN_DIRS.iter().map(PathBuf::from));
for dir in dirs {
for n in &names {
let p = dir.join(n);
if p.is_file() {
return Some(p);
}
}
}
None
}
/// Find a tool silently (cached for the process for ffmpeg and ffprobe).
pub fn tool(name: &str) -> Option<PathBuf> {
static FFMPEG: OnceLock<Option<PathBuf>> = OnceLock::new();
static FFPROBE: OnceLock<Option<PathBuf>> = OnceLock::new();
match name {
"ffmpeg" => FFMPEG.get_or_init(|| find_tool("ffmpeg")).clone(),
"ffprobe" => FFPROBE.get_or_init(|| find_tool("ffprobe")).clone(),
_ => find_tool(name),
}
}
pub fn oracles_required() -> bool {
std::env::var(REQUIRE_ENV).is_ok_and(|v| matches!(v.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on"))
}
/// Give up on an oracle check: print `SKIPPED` and return `None`, or fail when oracles are
/// required.
pub fn skip<T>(reason: &str) -> Option<T> {
skip_with(oracles_required(), reason)
}
#[allow(clippy::panic)]
fn skip_with<T>(required: bool, reason: &str) -> Option<T> {
assert!(!required, "oracle: {reason}, and {REQUIRE_ENV}=1 forbids skipping oracle tests");
eprintln!("SKIPPED (oracle): {reason}");
None
}
/// Run a fixture generator writing `tmp`, then rename it to `out`. A generator that cannot start
/// or fails skips (see [`skip`]); `required` is explicit so tests can set it without touching
/// the process environment.
pub fn run_generator(cmd: &mut Command, tmp: &Path, out: &Path, required: bool) -> Option<PathBuf> {
let reason = match cmd.output() {
Ok(o) if o.status.success() => match std::fs::rename(tmp, out) {
Ok(()) => return Some(out.to_path_buf()),
Err(e) => format!("could not move {} into place: {e}", tmp.display()),
},
Ok(o) => format!("generator failed for {} ({}): {}", out.display(), o.status, String::from_utf8_lossy(&o.stderr).trim()),
Err(e) => format!("could not start generator for {}: {e}", out.display()),
};
let _ = std::fs::remove_file(tmp);
skip_with(required, &format!("{reason} (encoder unavailable?)"))
}
fn tool_or_skip(name: &str) -> Option<PathBuf> {
tool(name).or_else(|| skip(&format!("{name} not found (set {} or put {name} on PATH)", env_var(name))))
}
/// ffmpeg, or `None` after printing `SKIPPED` (a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`).
pub fn ffmpeg() -> Option<PathBuf> {
tool_or_skip("ffmpeg")
}
/// ffprobe, or `None` after printing `SKIPPED` (a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`).
pub fn ffprobe() -> Option<PathBuf> {
tool_or_skip("ffprobe")
}
/// `$CRAFT_MEDIA_FIXTURES_DIR/<name>` or `<workspace>/target/fixtures/<name>`, created if missing.
#[allow(clippy::unwrap_used)]
pub fn fixture_subdir(name: &str) -> PathBuf {
let d = match std::env::var_os("CRAFT_MEDIA_FIXTURES_DIR").filter(|v| !v.is_empty()) {
Some(d) => PathBuf::from(d),
None => {
let here = Path::new(env!("CARGO_MANIFEST_DIR"));
let root = here.parent().and_then(Path::parent).unwrap_or(here);
root.join("target").join("fixtures")
}
}
.join(name);
std::fs::create_dir_all(&d).unwrap();
d
}
/// A temporary sibling of `out` that no other thread or process will pick; the extension is kept
/// because ffmpeg chooses the muxer from it.
pub fn temp_path(out: &Path) -> PathBuf {
static N: AtomicUsize = AtomicUsize::new(0);
let tid = format!("{:?}", std::thread::current().id()).replace(|c: char| !c.is_ascii_alphanumeric(), "");
let n = N.fetch_add(1, Ordering::Relaxed);
let stem = out.file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_default();
let name = match out.extension() {
Some(ext) => format!("{stem}.tmp-{}-{tid}-{n}.{}", std::process::id(), ext.to_string_lossy()),
None => format!("{stem}.tmp-{}-{tid}-{n}", std::process::id()),
};
out.with_file_name(name)
}
/// Run a fixture generator and print `FIXTURE <made|cached|skipped> <name>`.
pub fn generate_and_report<T>(name: &str, outputs: &[PathBuf], generate: impl FnOnce() -> Option<T>) -> Option<T> {
let existed = !outputs.is_empty() && outputs.iter().all(|p| std::fs::metadata(p).is_ok_and(|m| m.len() > 0));
let r = generate();
let status = match (&r, existed) {
(None, _) => "skipped",
(Some(_), true) => "cached",
(Some(_), false) => "made",
};
println!("FIXTURE {status} {name}");
r
}
crates/media/Cargo.toml
+7
View File
@@ -7,6 +7,9 @@ license.workspace = true
rust-version.workspace = true
repository.workspace = true
[package.metadata.docs.rs]
all-features = true
[lints]
workspace = true
@@ -15,11 +18,15 @@ default = ["isobmff"]
bitstream = []
isobmff = ["bitstream", "dep:thiserror"]
matroska = []
# Serialize and Deserialize for `Container`, `CodecId`, `CodecParams` and `PcmFormat`.
serde = ["dep:serde"]
[dependencies]
thiserror = { workspace = true, optional = true }
serde = { workspace = true, optional = true }
[dev-dependencies]
craft-media-testkit = { workspace = true }
proptest = { workspace = true }
serde_json = { workspace = true }
crates/media/README.md
+61 -14
View File
@@ -1,9 +1,13 @@
# craft-media
Media containers and bit-level I/O for the Craft apps: an MSB-first bit reader and writer, and
clean-room ISO BMFF (MP4), QuickTime (MOV) and Matroska (MKV, WebM) demuxers and muxers written
from the public specifications. No `unsafe`, no panics on malformed input, and it builds for
`wasm32-unknown-unknown`. The only dependency is `thiserror` (for `isobmff`).
Media containers and bit-level I/O for the Craft apps: an MSB-first bit reader and writer,
clean-room ISO BMFF (MP4), QuickTime (MOV) and Matroska (MKV, WebM) demuxers and muxers, written from the public specifications. No `unsafe`, no panics on malformed input, builds
for `wasm32-unknown-unknown`. Dependencies: `thiserror` (for `isobmff`), `serde` (optional).
Codecs go in separate crates that depend on this one: `craft-audio` for audio (in a follow-up),
and a video codec crate later. What both need lives here: the containers, the
bit I/O, one `ByteSource` trait, and the [shared identifiers](#containers-codecs-and-codec-parameters)
that codec crates use to describe a file and configure a decoder.
## Features
@@ -12,10 +16,12 @@ from the public specifications. No `unsafe`, no panics on malformed input, and i
| `bitstream` | `craft_media::bitstream` | none |
| `isobmff` | `craft_media::isobmff` | `bitstream`, `thiserror` |
| `matroska` | `craft_media::matroska` | none |
| (none) | `Container`, `CodecId`, `CodecParams`, `PcmFormat`, `xiph_lace`, `ByteSource`, `ReadSeekSource` | none |
| `serde` | `Serialize`/`Deserialize` on `Container`, `CodecId`, `CodecParams` and `PcmFormat` | `serde` |
`default = ["isobmff"]`, so `bitstream` and `isobmff` build. `matroska` is opt-in: it is about as
much code as `isobmff`, and most users of MP4/MOV (LightCraft, FigCraft, SoundCraft, EffectCraft)
do not read Matroska. The features are independent; `matroska` does not need the other two.
do not read Matroska. The features are independent.
```toml
craft-media = { git = "https://github.com/storytold/craft-libs", rev = "<commit>", features = ["matroska"] }
@@ -30,9 +36,12 @@ craft-media = { git = "https://github.com/storytold/craft-libs", rev = "<commit>
## Boundaries
- In scope: bit-level reading and writing, and container demuxing and muxing (boxes, sample
tables, sample entries and codec configuration records).
- Out of scope for now: codecs (H.264, HEVC, ProRes, AAC and others). They stay in the apps.
Ogg and other containers can join later as their own features.
tables, sample entries and codec configuration records) for audio and video tracks.
- Out of scope: codecs. Audio codecs are in `craft-audio`; video codecs (H.264, HEVC, ProRes and
others) stay in the apps until a video crate exists. Containers only FilmCraft reads (MXF,
MPEG-TS/PS, AVI) are not here yet.
- Anything both the audio and the video codec crates need belongs here, so the two never depend
on each other.
## Origin and API
@@ -98,9 +107,14 @@ Other differences:
accepted any entry.
- `isobmff::Mp4Writer::write_sample` returns an error when a track would pass 2³² − 1 samples
(PCM: frames), the most `stsz` can count; FilmCraft clamped the count.
- `matroska::ByteSource` stays a separate trait from `isobmff::ByteSource`, with the same shape.
FilmCraft implements both for one type, which a single shared trait would reject as conflicting
impls.
- `isobmff::ByteSource` and `matroska::ByteSource` are one trait,
`craft_media::ByteSource` (with `ReadSeekSource`, which was Matroska's). A type FilmCraft
implements both traits for keeps one of its two impls.
- `isobmff::Track` and `matroska::Track` have `codec_params()`, both returning a
`CodecParams` (`CodecId::Unknown` for a codec without an identifier). The MP4 one describes
the first sample's sample description; `codec_params_for(description_index)` gives any other,
and `has_mixed_descriptions()` says whether the samples use more than one.
`isobmff::CodecConfig` has `codec_id()` and `extradata()`, and `OpusConfig::opus_head()`.
## `bitstream`
@@ -170,7 +184,7 @@ Boxes with 64-bit sizes and with size 0 (runs to the end of the file) are suppor
| `jpeg mjpa mjpb` | `Jpeg { fourcc }` |
| `AVdn AVdh` | `Dnx { fourcc }` |
| `mp4a` + `esds` (in `wave` too) | `Aac(AacConfig)`: ASC bytes + decoded object type/rate/channels; `Mp3` for OTI 0x69/0x6B |
| `.mp3`/`.mp2` | `Mp3`: MPEG-1/2 audio of any layer (each frame header names it) |
| `.mp3`/`.mp2` | `Mp3`: MPEG-1/2 audio of any layer (each frame header names it; `Track::codec_params()` reports `Mp1`, `Mp2` or `Mp3` from the first frame) |
| PCM fourccs | `Pcm(PcmConfig)`: bits, float, endianness (`enda`, lpcm flags, `pcmC`), channels, rate |
| `alac` | `Alac { cookie }` |
| `Opus` + `dOps` | `Opus(OpusConfig)` |
@@ -383,6 +397,38 @@ first frame (AV1 from `rav1e`) creates the writer after that frame.
from codec parsers).
- Muxer: no CRC-32, Chapters, Tags, Attachments, content encodings or block additions.
## Containers, codecs and codec parameters
Always built, with no dependencies, so codec crates share them:
- `Container`: the file format (`Wav`, `Aiff`, `Flac`, `Mp4`, `QuickTime`, `Matroska`, `WebM`,
`Ogg`, `Caf`, `Adts`, `MpegAudio`, `Ac3`), with `name()`.
- `CodecId`: audio and video codecs (`Aac`, `HeAac`, `Opus`, `Eac3`, `Speex`, …, `H264`, `Hevc`,
`Av1`, `ProRes`, `Dnxhd`, `Apv`, `Theora`, …, `Unknown`), with `name()`, `is_audio()`,
`is_video()`.
- `PcmFormat`: the layout of uncompressed samples, built with `PcmFormat::int(bits, signed,
big_endian)` or `PcmFormat::float(bits, big_endian)`.
- `CodecParams`: what a decoder needs about a track, whatever its container: `codec`,
`extradata`, `tag` (the container's four-character code, such as a ProRes profile), the audio
fields `sample_rate`, `channels` and `pcm` (`PcmFormat`), and the video fields `width` and
`height`. Built with `CodecParams::new(codec)` and the `with_*` methods, or by the demuxers'
`codec_params()`, which every container returns in the same form. `extradata` follows
Matroska's `CodecPrivate`, as ffmpeg's extradata does: the `AudioSpecificConfig` for AAC, the
`OpusHead` for Opus (MP4's `dOps` is converted), Xiph-laced headers for Vorbis (`xiph_lace`,
`None` for more than 256 packets), `fLaC` + metadata blocks for FLAC, the
`avcC`/`hvcC`/`av1C`/`apvC` record for video.
All are `#[non_exhaustive]`: new formats and fields are minor changes.
## `ByteSource`
`craft_media::ByteSource` (`len`, `read_at(offset, buf)`) is what every demuxer reads from. It is
always built, so a crate that uses only `bitstream` can name it too. `isobmff::ByteSource` and
`matroska::ByteSource` are the same trait. It is implemented for `[u8]`,
`Vec<u8>`, `&T`, `Arc<T>`, `Box<T>` and `std::fs::File` (not on wasm). `ReadSeekSource` adapts any
`Read + Seek`; it is `Send + Sync` when the reader is `Send`. Implement the trait once for a
custom source (a web `Blob`, a network range reader) to use every container with it.
## Tests
- Unit tests cover the bit reader and writer, the box reader (64-bit, size-0, `uuid`, truncated and
@@ -406,9 +452,10 @@ first frame (AV1 from `rav1e`) creates the writer after that frame.
`fuzz.rs` also walks every box of a mutated file as a container and checks that each payload
range stays inside its parent.
ffmpeg and ffprobe are test oracles only, never linked. They are looked up in
ffmpeg and ffprobe are test oracles only, never linked. The helpers are in `craft-media-testkit`
(an unpublished crate, shared with craft-audio and the video crate to come). They are looked up in
`$CRAFT_MEDIA_FFMPEG` / `$CRAFT_MEDIA_FFPROBE`, then `PATH`, then common install directories. When
a tool is missing, the oracle tests print `SKIPPED` and pass; `CRAFT_MEDIA_REQUIRE_ORACLES=1` turns
a skip into a failure. CI (`.github/workflows/media.yml`) sets it and installs Ubuntu's ffmpeg.
Generate every fixture ahead of time with
craft-audio's oracle tests use the same variables. Generate every fixture ahead of time with
`cargo test -p craft-media --all-features -- --ignored generate_fixtures`.
crates/media/src/id.rs
+376
View File
@@ -0,0 +1,376 @@
//! Container and codec identifiers and codec parameters shared by the codec crates (craft-audio,
//! and the video crate to come), so a file's container and codec are named the same way
//! everywhere, and a codec is configured the same way whatever container its track came from.
//!
//! Both enums are `#[non_exhaustive]`: new formats are added as minor changes, so a `match` on them
//! needs a `_` arm.
#![warn(missing_docs)]
/// The file or stream format that holds the codec data.
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum Container {
/// RIFF WAVE, including BWF and RF64/BW64.
Wav,
/// AIFF and AIFF-C.
Aiff,
/// A native FLAC stream (`fLaC`).
Flac,
/// ISO base media file format: MP4, M4A, M4B.
Mp4,
/// QuickTime (MOV).
QuickTime,
/// Matroska (MKV, MKA).
Matroska,
/// WebM (the Matroska subset).
WebM,
/// Ogg.
Ogg,
/// Apple Core Audio Format.
Caf,
/// A raw ADTS AAC stream.
Adts,
/// A raw MPEG audio stream (MP1, MP2 or MP3 frames, optionally behind ID3v2 tags).
MpegAudio,
/// A raw AC-3 or E-AC-3 stream.
Ac3,
}
impl Container {
/// A short display name: `"WAV"`, `"MP4"`, `"Matroska"`.
pub fn name(self) -> &'static str {
match self {
Container::Wav => "WAV",
Container::Aiff => "AIFF",
Container::Flac => "FLAC",
Container::Mp4 => "MP4",
Container::QuickTime => "QuickTime",
Container::Matroska => "Matroska",
Container::WebM => "WebM",
Container::Ogg => "Ogg",
Container::Caf => "CAF",
Container::Adts => "ADTS",
Container::MpegAudio => "MPEG audio",
Container::Ac3 => "AC-3",
}
}
}
/// A codec, audio or video.
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum CodecId {
/// Uncompressed PCM, integer or float.
Pcm,
/// ADPCM (IMA, Microsoft, G.722, G.726).
Adpcm,
/// G.711 A-law.
ALaw,
/// G.711 µ-law.
MuLaw,
/// FLAC.
Flac,
/// Apple Lossless.
Alac,
/// AAC (LC). Containers report every AAC stream as `Aac`; a decoder that finds spectral band
/// replication reports [`CodecId::HeAac`].
Aac,
/// HE-AAC (AAC with spectral band replication, v1 or v2).
HeAac,
/// MPEG-1/2 Audio Layer I.
Mp1,
/// MPEG-1/2 Audio Layer II.
Mp2,
/// MPEG-1/2 Audio Layer III.
Mp3,
/// AC-3 (Dolby Digital).
Ac3,
/// E-AC-3 (Dolby Digital Plus).
Eac3,
/// Opus.
Opus,
/// Vorbis.
Vorbis,
/// Speex.
Speex,
/// H.264 / AVC.
H264,
/// H.265 / HEVC.
Hevc,
/// AV1.
Av1,
/// VP8.
Vp8,
/// VP9.
Vp9,
/// Apple ProRes.
ProRes,
/// Motion JPEG.
Mjpeg,
/// MPEG-2 video.
Mpeg2Video,
/// MPEG-4 Part 2 video.
Mpeg4Video,
/// Theora.
Theora,
/// Avid DNxHD / DNxHR.
Dnxhd,
/// APV (Advanced Professional Video).
Apv,
/// A codec this crate has no identifier for.
Unknown,
}
impl CodecId {
/// A short display name: `"AAC"`, `"HE-AAC"`, `"E-AC-3"`, `"H.264"`.
pub fn name(self) -> &'static str {
match self {
CodecId::Pcm => "PCM",
CodecId::Adpcm => "ADPCM",
CodecId::ALaw => "A-law",
CodecId::MuLaw => "µ-law",
CodecId::Flac => "FLAC",
CodecId::Alac => "ALAC",
CodecId::Aac => "AAC",
CodecId::HeAac => "HE-AAC",
CodecId::Mp1 => "MP1",
CodecId::Mp2 => "MP2",
CodecId::Mp3 => "MP3",
CodecId::Ac3 => "AC-3",
CodecId::Eac3 => "E-AC-3",
CodecId::Opus => "Opus",
CodecId::Vorbis => "Vorbis",
CodecId::Speex => "Speex",
CodecId::H264 => "H.264",
CodecId::Hevc => "HEVC",
CodecId::Av1 => "AV1",
CodecId::Vp8 => "VP8",
CodecId::Vp9 => "VP9",
CodecId::ProRes => "ProRes",
CodecId::Mjpeg => "Motion JPEG",
CodecId::Mpeg2Video => "MPEG-2",
CodecId::Mpeg4Video => "MPEG-4",
CodecId::Theora => "Theora",
CodecId::Dnxhd => "DNxHD",
CodecId::Apv => "APV",
CodecId::Unknown => "unknown",
}
}
/// Whether this is an audio codec.
pub fn is_audio(self) -> bool {
matches!(
self,
CodecId::Pcm
| CodecId::Adpcm
| CodecId::ALaw
| CodecId::MuLaw
| CodecId::Flac
| CodecId::Alac
| CodecId::Aac
| CodecId::HeAac
| CodecId::Mp1
| CodecId::Mp2
| CodecId::Mp3
| CodecId::Ac3
| CodecId::Eac3
| CodecId::Opus
| CodecId::Vorbis
| CodecId::Speex
)
}
/// Whether this is a video codec.
pub fn is_video(self) -> bool {
matches!(
self,
CodecId::H264
| CodecId::Hevc
| CodecId::Av1
| CodecId::Vp8
| CodecId::Vp9
| CodecId::ProRes
| CodecId::Mjpeg
| CodecId::Mpeg2Video
| CodecId::Mpeg4Video
| CodecId::Theora
| CodecId::Dnxhd
| CodecId::Apv
)
}
}
/// The layout of uncompressed PCM samples.
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct PcmFormat {
/// Bits per sample (8, 16, 24, 32 for integers; 32 or 64 for floats).
pub bits: u16,
/// IEEE float samples.
pub float: bool,
/// Big-endian byte order.
pub big_endian: bool,
/// Signed integers (8-bit WAV and `raw ` QuickTime PCM are unsigned).
pub signed: bool,
}
impl PcmFormat {
/// Integer samples of `bits` bits.
pub fn int(bits: u16, signed: bool, big_endian: bool) -> PcmFormat {
PcmFormat { bits, float: false, big_endian, signed }
}
/// IEEE float samples of `bits` bits (32 or 64).
pub fn float(bits: u16, big_endian: bool) -> PcmFormat {
PcmFormat { bits, float: true, big_endian, signed: true }
}
}
/// What a decoder needs to know about a track, whatever container it came from.
///
/// `extradata` is the codec configuration in Matroska's `CodecPrivate` form, which ffmpeg's
/// extradata also follows:
///
/// | Codec | `extradata` |
/// |---|---|
/// | AAC | `AudioSpecificConfig` |
/// | Opus | the `OpusHead` identification header (RFC 7845 §5.1) |
/// | Vorbis | the three headers, Xiph-laced |
/// | FLAC | `fLaC` followed by the metadata blocks (STREAMINFO first) |
/// | ALAC | the magic cookie (`ALACSpecificConfig`) |
/// | AC-3 / E-AC-3 | the MP4 `dac3` / `dec3` payload, or empty |
/// | MP1/MP2/MP3, PCM | empty |
/// | H.264 / HEVC / AV1 | the `avcC` / `hvcC` / `av1C` record |
/// | VP9 | the MP4 `vpcC` payload, or Matroska's codec feature metadata |
/// | APV | the `apvC` record |
///
/// `sample_rate` and `channels` are what the container declares (for HE-AAC with implicit
/// signalling, usually the core rate), except that Opus is always 48 kHz, the rate it decodes at;
/// `width` and `height` are the coded picture size of a video
/// track. `tag` is the container's four-character code for the codec when it has one (the MP4
/// sample entry format, Matroska's ProRes or VfW FourCC), which is how a ProRes profile (`apcn`,
/// `ap4h`, …) or a DNx variant is told apart.
///
/// `#[non_exhaustive]`: build it with [`CodecParams::new`] and the `with_*` methods.
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct CodecParams {
/// The codec.
pub codec: CodecId,
/// The codec configuration (see the table above).
pub extradata: Vec<u8>,
/// The container's four-character code for the codec, if it has one.
pub tag: Option<[u8; 4]>,
/// Audio: samples per second.
pub sample_rate: Option<u32>,
/// Audio: channel count.
pub channels: Option<u16>,
/// Audio: the sample layout of uncompressed PCM.
pub pcm: Option<PcmFormat>,
/// Video: coded picture width in pixels.
pub width: Option<u32>,
/// Video: coded picture height in pixels.
pub height: Option<u32>,
}
impl CodecParams {
/// Parameters for `codec`, with nothing else known yet.
pub fn new(codec: CodecId) -> CodecParams {
CodecParams { codec, extradata: Vec::new(), tag: None, sample_rate: None, channels: None, pcm: None, width: None, height: None }
}
/// Sets `tag`.
pub fn with_tag(mut self, tag: [u8; 4]) -> CodecParams {
self.tag = Some(tag);
self
}
/// The coded picture size; a zero dimension leaves both unknown.
pub fn with_size(mut self, width: u32, height: u32) -> CodecParams {
(self.width, self.height) = if width > 0 && height > 0 { (Some(width), Some(height)) } else { (None, None) };
self
}
/// Sets `extradata`.
pub fn with_extradata(mut self, extradata: impl Into<Vec<u8>>) -> CodecParams {
self.extradata = extradata.into();
self
}
/// Sets `sample_rate`; 0 leaves it unknown.
pub fn with_sample_rate(mut self, sample_rate: u32) -> CodecParams {
self.sample_rate = (sample_rate > 0).then_some(sample_rate);
self
}
/// Sets `channels`; 0 leaves it unknown.
pub fn with_channels(mut self, channels: u16) -> CodecParams {
self.channels = (channels > 0).then_some(channels);
self
}
/// Sets `pcm`.
pub fn with_pcm(mut self, pcm: PcmFormat) -> CodecParams {
self.pcm = Some(pcm);
self
}
}
/// Xiph lacing of `packets` (Matroska `CodecPrivate` for Vorbis): the count less one, each
/// length but the last as a run of 255s and a remainder, then the packets. `None` for no packets
/// or more than 256, which the one-byte count cannot hold.
pub fn xiph_lace(packets: &[Vec<u8>]) -> Option<Vec<u8>> {
let mut out = vec![u8::try_from(packets.len().checked_sub(1)?).ok()?];
for p in packets.iter().take(packets.len().saturating_sub(1)) {
let mut n = p.len();
while n >= 255 {
out.push(255);
n -= 255;
}
out.push(n as u8);
}
for p in packets {
out.extend_from_slice(p);
}
Some(out)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn audio_and_video_are_disjoint() {
for c in [CodecId::Aac, CodecId::Opus, CodecId::Pcm, CodecId::Eac3, CodecId::Speex] {
assert!(c.is_audio() && !c.is_video(), "{c:?}");
}
for c in [CodecId::H264, CodecId::Av1, CodecId::ProRes, CodecId::Theora, CodecId::Dnxhd, CodecId::Apv] {
assert!(c.is_video() && !c.is_audio(), "{c:?}");
}
assert!(!CodecId::Unknown.is_audio() && !CodecId::Unknown.is_video());
}
#[test]
fn params_builder() {
let p = CodecParams::new(CodecId::Aac).with_extradata([0x12, 0x10]).with_sample_rate(48_000).with_channels(2).with_sample_rate(0);
assert_eq!((p.codec, p.extradata.as_slice(), p.sample_rate, p.channels), (CodecId::Aac, &[0x12, 0x10][..], None, Some(2)));
let v = CodecParams::new(CodecId::ProRes).with_tag(*b"apcn").with_size(1920, 1080);
assert_eq!((v.tag, v.width, v.height), (Some(*b"apcn"), Some(1920), Some(1080)));
assert_eq!(v.with_size(0, 1080).width, None);
}
#[test]
fn xiph_lacing() {
let laced = xiph_lace(&[vec![1; 30], vec![2; 300], vec![3; 5]]).unwrap();
assert_eq!(&laced[..4], &[2, 30, 255, 45]);
assert_eq!(laced.len(), 4 + 335);
assert_eq!(xiph_lace(&[]), None);
assert_eq!(xiph_lace(&vec![vec![0]; 257]), None);
assert!(xiph_lace(&vec![vec![0]; 256]).is_some());
}
}
crates/media/src/isobmff/codec.rs
+120 -5
View File
@@ -1,5 +1,6 @@
//! Sample entries (`stsd` children) and codec configuration records: parsing and serialization.
use crate::CodecId;
use crate::bitstream::BitReader;
use crate::isobmff::bytes::{BoxBuf, Cur, FourCc, boxes};
use crate::isobmff::error::{Error, Result};
@@ -182,7 +183,8 @@ pub enum CodecConfig {
Apv(ApvConfig),
Aac(AacConfig),
/// MPEG-1/2 audio (esds object type 0x69/0x6B, or QuickTime `.mp3` / `.mp2`). Usually layer
/// III, but these name no layer: ffmpeg stores MP2 as `mp4a` + 0x6B. Each frame header has it.
/// III, but these name no layer: ffmpeg stores MP2 as `mp4a` + 0x6B. Each frame header has it,
/// and [`crate::isobmff::Track::codec_params`] reports the layer the first frame names.
Mp3,
Pcm(PcmConfig),
/// ALAC magic cookie (ALACSpecificConfig, 24 bytes, optionally followed by a channel layout).
@@ -208,6 +210,51 @@ pub enum CodecConfig {
}
impl CodecConfig {
/// The codec this configuration is for. `Mp3` covers every MPEG audio layer: these entries
/// name none, and each frame header has it (ffmpeg stores MP2 as `mp4a` + 0x6B);
/// [`crate::isobmff::Track::codec_params`] reads it from the first frame.
pub fn codec_id(&self) -> CodecId {
match self {
CodecConfig::Avc(_) => CodecId::H264,
CodecConfig::Hevc(_) => CodecId::Hevc,
CodecConfig::Av1(_) => CodecId::Av1,
CodecConfig::Vp9(_) => CodecId::Vp9,
CodecConfig::ProRes { .. } => CodecId::ProRes,
CodecConfig::Jpeg { .. } => CodecId::Mjpeg,
CodecConfig::Aac(_) => CodecId::Aac,
CodecConfig::Mp3 => CodecId::Mp3,
CodecConfig::Pcm(_) => CodecId::Pcm,
CodecConfig::Alac { .. } => CodecId::Alac,
CodecConfig::Opus(_) => CodecId::Opus,
CodecConfig::Flac(_) => CodecId::Flac,
CodecConfig::Ac3 { .. } => CodecId::Ac3,
CodecConfig::Eac3 { .. } => CodecId::Eac3,
CodecConfig::Dnx { .. } => CodecId::Dnxhd,
CodecConfig::Apv(_) => CodecId::Apv,
CodecConfig::Timecode(_) | CodecConfig::Unknown { .. } => CodecId::Unknown,
}
}
/// The codec configuration in Matroska's `CodecPrivate` form (see [`crate::CodecParams`]):
/// the `OpusHead` built from `dOps`, `fLaC` + the `dfLa` metadata blocks, the other records
/// as stored.
pub fn extradata(&self) -> Vec<u8> {
match self {
CodecConfig::Avc(c) => c.to_bytes(),
CodecConfig::Hevc(c) => c.to_bytes(),
CodecConfig::Av1(c) => c.to_bytes(),
CodecConfig::Vp9(c) => c.to_bytes(),
CodecConfig::Apv(c) => c.record_bytes(),
CodecConfig::Aac(a) => a.asc.clone(),
CodecConfig::Alac { cookie } => cookie.clone(),
CodecConfig::Opus(o) => o.opus_head(),
CodecConfig::Flac(f) => [b"fLaC".as_slice(), &f.metadata_blocks].concat(),
CodecConfig::Ac3 { dac3 } => dac3.clone(),
CodecConfig::Eac3 { dec3 } => dec3.clone(),
_ => Vec::new(),
}
}
/// A short human-readable codec name.
pub fn name(&self) -> &'static str {
match self {
@@ -633,6 +680,23 @@ pub struct OpusConfig {
}
impl OpusConfig {
/// The Ogg Opus identification header (RFC 7845 §5.1) this `dOps` box describes: the same
/// fields, little-endian, behind the `OpusHead` magic.
pub fn opus_head(&self) -> Vec<u8> {
let mut h = Vec::with_capacity(19 + self.channel_mapping.len());
h.extend_from_slice(b"OpusHead");
h.push(1);
h.push(self.output_channels);
h.extend_from_slice(&self.pre_skip.to_le_bytes());
h.extend_from_slice(&self.input_sample_rate.to_le_bytes());
h.extend_from_slice(&self.output_gain.to_le_bytes());
h.push(self.channel_mapping_family);
if self.channel_mapping_family != 0 {
h.extend_from_slice(&self.channel_mapping);
}
h
}
pub fn parse(data: &[u8]) -> Result<Self> {
let mut c = Cur::labeled(data, "dOps");
let _version = c.u8()?;
@@ -808,12 +872,15 @@ impl ApvConfig {
/// Serialize the full `apvC` box payload (4-byte `FullBox` header + `APVDecoderConfigurationRecord`).
pub fn to_bytes(&self) -> Vec<u8> {
let mut b = BoxBuf::new();
b.u32(0); // FullBox version = 0, flags = 0
[&[0u8; 4][..], &self.record_bytes()].concat() // FullBox version = 0, flags = 0
}
/// Serialize the bare `APVDecoderConfigurationRecord` (Matroska's `CodecPrivate` for `V_APV`).
pub fn record_bytes(&self) -> Vec<u8> {
if !self.raw_record.is_empty() {
b.bytes(&self.raw_record);
return b.buf;
return self.raw_record.clone();
}
let mut b = BoxBuf::new();
b.u8(1); // configurationVersion = 1
b.u8(1); // number_of_configuration_entry = 1
b.u8(1); // pbu_type = 1 (primary frame)
@@ -1840,3 +1907,51 @@ mod tests {
assert_eq!(parse_asc(&[]), None);
}
}
#[cfg(test)]
mod params_tests {
use super::*;
/// `dOps` is big-endian; the `OpusHead` it becomes is little-endian, with the mapping table
/// only for families other than 0.
#[test]
fn dops_becomes_an_opus_head() {
let o = OpusConfig {
output_channels: 6,
pre_skip: 312,
input_sample_rate: 48_000,
output_gain: -256,
channel_mapping_family: 1,
channel_mapping: vec![4, 2, 0, 4, 1, 2, 3, 5],
};
let h = o.opus_head();
assert_eq!(&h[..8], b"OpusHead");
assert_eq!(&h[8..19], &[1, 6, 0x38, 0x01, 0x80, 0xBB, 0, 0, 0x00, 0xFF, 1]);
assert_eq!(&h[19..], &o.channel_mapping[..]);
let stereo = OpusConfig { output_channels: 2, channel_mapping_family: 0, channel_mapping: vec![9, 9], ..o };
assert_eq!(stereo.opus_head().len(), 19);
assert_eq!(CodecConfig::Opus(stereo).codec_id(), CodecId::Opus);
}
#[test]
fn apv_extradata_is_the_bare_record() {
let cfg = ApvConfig {
profile_idc: 33,
level_idc: 60,
frame_width: 1920,
frame_height: 1080,
chroma_format_idc: 2,
bit_depth_minus8: 2,
..Default::default()
};
let record = cfg.record_bytes();
assert_eq!(record[0], 1, "configurationVersion");
assert_eq!(cfg.to_bytes(), [&[0u8; 4][..], &record].concat());
// Matroska's V_APV CodecPrivate carries the record as is; an MP4 track must match it
let parsed = ApvConfig::parse(&cfg.to_bytes()).unwrap();
assert_eq!(parsed.raw_record, record);
assert_eq!(CodecConfig::Apv(parsed.clone()).extradata(), record);
assert_eq!(parsed.to_bytes(), cfg.to_bytes());
assert_eq!(CodecConfig::Apv(cfg).extradata(), record);
}
}
crates/media/src/isobmff/demux.rs
+134 -1
View File
@@ -3,7 +3,7 @@
use crate::isobmff::bytes::{BoxHeader, Cur, FourCc, boxes, find};
use crate::isobmff::codec::{CodecConfig, SampleEntry, parse_sample_entry};
use crate::isobmff::error::{Error, Result, invalid};
use crate::isobmff::source::ByteSource;
use crate::source::ByteSource;
use std::collections::HashMap;
/// Hard cap on the sample count of the whole file, all tracks and fragments (after PCM chunk merging).
@@ -140,6 +140,9 @@ pub struct Track {
pub pcm_chunked: bool,
sync_index: Vec<u32>,
pts_order: Vec<u32>,
/// Per entry: the MPEG audio codec its first sample's frame header names, for the
/// [`CodecConfig::Mp3`] entries, which name no layer themselves.
mpeg_audio: Vec<Option<crate::CodecId>>,
}
impl Track {
@@ -147,6 +150,52 @@ impl Track {
pub fn codec(&self) -> Option<&CodecConfig> {
self.entries.first().map(|e| &e.codec)
}
/// What a decoder needs for this track, in the container-neutral form the codec crates take:
/// [`Track::codec_params_for`] the first sample's description (the first entry for a track
/// without samples, or when that index names no entry). A track whose samples use more than
/// one description ([`Track::has_mixed_descriptions`]) needs `codec_params_for` per sample.
pub fn codec_params(&self) -> crate::CodecParams {
let first = self.samples.first().map(|s| s.description_index);
self.codec_params_for(first.filter(|&i| i >= 1 && i as usize <= self.entries.len()).unwrap_or(1))
}
/// What a decoder needs for the samples whose [`Sample::description_index`] is
/// `description_index` (1-based, into [`Track::entries`]); [`crate::CodecId::Unknown`] when
/// there is no such entry.
pub fn codec_params_for(&self, description_index: u32) -> crate::CodecParams {
let Some(entry) = description_index.checked_sub(1).and_then(|i| self.entries.get(i as usize)) else {
return crate::CodecParams::new(crate::CodecId::Unknown);
};
let codec = &entry.codec;
let id = self.mpeg_audio.get(description_index as usize - 1).copied().flatten().unwrap_or(codec.codec_id());
let mut p = crate::CodecParams::new(id).with_extradata(codec.extradata()).with_tag(entry.format.0);
if let Some(v) = &entry.video {
p = p.with_size(u32::from(v.width), u32::from(v.height));
}
if let Some(a) = &entry.audio {
p = p.with_sample_rate(a.sample_rate.round() as u32).with_channels(u16::try_from(a.channels).unwrap_or(0));
}
match codec {
CodecConfig::Pcm(c) => {
p = p.with_pcm(if c.float {
crate::PcmFormat::float(c.bits, c.big_endian)
} else {
crate::PcmFormat::int(c.bits, c.signed, c.big_endian)
});
p = p.with_sample_rate(c.sample_rate.round() as u32).with_channels(u16::try_from(c.channels).unwrap_or(0));
}
CodecConfig::Opus(o) => p = p.with_sample_rate(48_000).with_channels(u16::from(o.output_channels)),
_ => {}
}
p
}
/// True when the samples use more than one sample description, so no single
/// [`Track::codec_params`] describes them all.
pub fn has_mixed_descriptions(&self) -> bool {
self.samples.windows(2).any(|w| w[0].description_index != w[1].description_index)
}
pub fn video(&self) -> Option<&crate::isobmff::codec::VideoParams> {
self.entries.first().and_then(|e| e.video.as_ref())
}
@@ -402,10 +451,39 @@ pub fn open<S: ByteSource>(src: S) -> Result<Mp4File> {
tc.start_frame = Some(u32::from_be_bytes(b));
}
}
t.mpeg_audio = mpeg_audio_layers(t, &src);
}
Ok(file)
}
/// For each [`CodecConfig::Mp3`] entry, the layer its first sample's frame header names (these
/// entries name none: ffmpeg stores MP2 as `mp4a` + 0x6B), else what the sample entry says
/// (QuickTime `.mp2`, `.mp3`).
fn mpeg_audio_layers<S: ByteSource>(t: &Track, src: &S) -> Vec<Option<crate::CodecId>> {
use crate::CodecId;
let from_header = |index: u32| {
let s = t.samples.iter().find(|s| s.description_index == index && s.size >= 4)?;
let mut b = [0u8; 4];
src.read_at(s.offset, &mut b).ok()?;
if b[0] != 0xFF || b[1] & 0xE0 != 0xE0 {
return None;
}
match (b[1] >> 1) & 3 {
3 => Some(CodecId::Mp1),
2 => Some(CodecId::Mp2),
1 => Some(CodecId::Mp3),
_ => None,
}
};
(1..)
.zip(&t.entries)
.map(|(index, e)| match e.codec {
CodecConfig::Mp3 => from_header(index).or(Some(if e.format == b".mp2" { CodecId::Mp2 } else { CodecId::Mp3 })),
_ => None,
})
.collect()
}
#[derive(Clone, Copy, Debug, Default)]
struct Trex {
description_index: u32,
@@ -457,6 +535,7 @@ fn parse_trak(p: &[u8], file: &Mp4File, file_len: u64, budget: &mut SampleBudget
pcm_chunked: false,
sync_index: Vec::new(),
pts_order: Vec::new(),
mpeg_audio: Vec::new(),
};
let mut mdia = None;
for b in boxes(p) {
@@ -1082,6 +1161,60 @@ mod tests {
assert_eq!(c.next_value(), 0);
}
#[test]
fn codec_params_follow_the_sample_description() {
use crate::isobmff::codec::SampleEntry;
use crate::isobmff::mux::{Brand, Mp4Writer, TrackConfig, WriteSample, WriterOptions};
let mut w = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap();
let a = w.add_track(TrackConfig::new(SampleEntry::aac(vec![0x12, 0x10], 2, 48_000), 48_000)).unwrap();
for _ in 0..2 {
w.write_sample(a, WriteSample { data: &[0; 8], duration: 1024, composition_offset: 0, is_sync: true }).unwrap();
}
let out = w.finish().unwrap().into_inner();
let mut t = open(out.as_slice()).unwrap().tracks.remove(0);
assert!(!t.has_mixed_descriptions());
assert_eq!(t.codec_params(), t.codec_params_for(1));
// entry 1 unused, every sample on entry 2 (a 44.1 kHz mono AudioSpecificConfig)
t.entries.push(SampleEntry::aac(vec![0x12, 0x08], 1, 44_100));
t.samples.iter_mut().for_each(|s| s.description_index = 2);
let p = t.codec_params();
assert_eq!((p.extradata.as_slice(), p.sample_rate, p.channels), (&[0x12, 0x08][..], Some(44_100), Some(1)));
assert_eq!(t.codec_params_for(1).extradata, [0x12, 0x10]);
assert_eq!(t.codec_params_for(0).codec, crate::CodecId::Unknown);
assert_eq!(t.codec_params_for(3).codec, crate::CodecId::Unknown);
assert!(!t.has_mixed_descriptions());
t.samples[1].description_index = 1;
assert!(t.has_mixed_descriptions());
t.samples[0].description_index = 0; // malformed: falls back to the first entry
assert_eq!(t.codec_params().extradata, [0x12, 0x10]);
}
#[test]
fn mpeg_audio_layer_comes_from_the_first_frame() {
use crate::CodecId;
use crate::isobmff::codec::SampleEntry;
use crate::isobmff::mux::{Brand, Mp4Writer, TrackConfig, WriteSample, WriterOptions};
// MPEG-1 frame headers: layer I, II, III; then no frame header at all
let cases: [(&[u8; 4], Brand, [u8; 2], CodecId); 5] = [
(b"mp4a", Brand::Mp4, [0xFF, 0xFF], CodecId::Mp1),
(b"mp4a", Brand::Mp4, [0xFF, 0xFD], CodecId::Mp2),
(b"mp4a", Brand::Mp4, [0xFF, 0xFB], CodecId::Mp3),
(b"mp4a", Brand::Mp4, [0, 0], CodecId::Mp3),
(b".mp2", Brand::Mov, [0, 0], CodecId::Mp2),
];
for (format, brand, header, want) in cases {
let mut w = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(brand)).unwrap();
let entry = SampleEntry::audio(FourCc(*format), CodecConfig::Mp3, 2, 48_000.0, 16);
let a = w.add_track(TrackConfig::new(entry, 48_000)).unwrap();
let frame = [&header[..], &[0x94, 0x00, 0, 0, 0, 0]].concat();
w.write_sample(a, WriteSample { data: &frame, duration: 1152, composition_offset: 0, is_sync: true }).unwrap();
let out = w.finish().unwrap().into_inner();
let t = &open(out.as_slice()).unwrap().tracks[0];
assert_eq!(t.codec(), Some(&CodecConfig::Mp3));
assert_eq!(t.codec_params().codec, want, "{format:?} {header:02X?}");
}
}
#[test]
fn rejects_no_moov_and_cmov() {
let mut b = crate::isobmff::bytes::BoxBuf::new();
crates/media/src/isobmff/mod.rs
+1 -2
View File
@@ -16,8 +16,8 @@ mod codec;
mod demux;
mod error;
mod mux;
mod source;
pub use crate::source::{ByteSource, ReadSeekSource};
pub use bytes::FourCc;
pub use codec::{
AacConfig, ApvConfig, AudioParams, Av1Config, AvcConfig, BitRate, CleanAperture, CodecConfig, ColorInfo, FieldInfo, FlacConfig, HevcConfig,
@@ -26,4 +26,3 @@ pub use codec::{
pub use demux::{Edit, Metadata, Mp4File, Sample, SampleGroup, Track, TrackKind, open};
pub use error::{Error, Result};
pub use mux::{Brand, FragmentedWriter, Mp4Writer, TrackConfig, WriteSample, WriterOptions};
pub use source::ByteSource;
crates/media/src/isobmff/source.rs
-123
View File
@@ -1,123 +0,0 @@
//! Random-access byte sources for the demuxer.
use std::io;
use std::sync::Arc;
/// Random-access, read-only byte source (file, in-memory buffer, web Blob…).
///
/// Implementations must be able to serve reads at arbitrary offsets through `&self`.
pub trait ByteSource {
/// Total length in bytes.
fn len(&self) -> u64;
/// Fill `buf` entirely with bytes starting at `offset`; fail with `UnexpectedEof` if past the end.
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()>;
/// True if the source holds no bytes.
fn is_empty(&self) -> bool {
self.len() == 0
}
}
fn eof() -> io::Error {
io::Error::new(io::ErrorKind::UnexpectedEof, "read past end of byte source")
}
impl ByteSource for [u8] {
fn len(&self) -> u64 {
<[u8]>::len(self) as u64
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
let start = usize::try_from(offset).map_err(|_| eof())?;
let end = start.checked_add(buf.len()).ok_or_else(eof)?;
let src = self.get(start..end).ok_or_else(eof)?;
buf.copy_from_slice(src);
Ok(())
}
}
impl ByteSource for Vec<u8> {
fn len(&self) -> u64 {
self.as_slice().len() as u64
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
self.as_slice().read_at(offset, buf)
}
}
impl<T: ByteSource + ?Sized> ByteSource for &T {
fn len(&self) -> u64 {
(**self).len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
(**self).read_at(offset, buf)
}
}
impl<T: ByteSource + ?Sized> ByteSource for Arc<T> {
fn len(&self) -> u64 {
(**self).len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
(**self).read_at(offset, buf)
}
}
impl<T: ByteSource + ?Sized> ByteSource for Box<T> {
fn len(&self) -> u64 {
(**self).len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
(**self).read_at(offset, buf)
}
}
#[cfg(not(target_arch = "wasm32"))]
impl ByteSource for std::fs::File {
fn len(&self) -> u64 {
self.metadata().map(|m| m.len()).unwrap_or(0)
}
#[cfg(unix)]
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
std::os::unix::fs::FileExt::read_exact_at(self, buf, offset)
}
#[cfg(windows)]
fn read_at(&self, mut offset: u64, mut buf: &mut [u8]) -> io::Result<()> {
use std::os::windows::fs::FileExt;
while !buf.is_empty() {
let n = self.seek_read(buf, offset)?;
if n == 0 {
return Err(eof());
}
offset += n as u64;
buf = &mut buf[n..];
}
Ok(())
}
#[cfg(not(any(unix, windows)))]
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
use std::io::{Read, Seek, SeekFrom};
let mut f = self;
f.seek(SeekFrom::Start(offset))?;
f.read_exact(buf)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn slice_reads() {
let v = vec![1u8, 2, 3, 4];
let mut b = [0u8; 2];
v.read_at(1, &mut b).unwrap();
assert_eq!(b, [2, 3]);
assert!(v.read_at(3, &mut b).is_err());
assert!(v.read_at(u64::MAX, &mut b).is_err());
let a: Arc<[u8]> = Arc::from(v.clone());
a.read_at(2, &mut b).unwrap();
assert_eq!(b, [3, 4]);
assert_eq!(ByteSource::len(&a), 4);
let s: &[u8] = &v;
assert_eq!(ByteSource::len(&s), 4);
}
}
crates/media/src/lib.rs
+10 -2
View File
@@ -1,18 +1,26 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
//! Media containers and bit-level I/O for the Craft apps.
//! Media containers and bit-level I/O for the Craft apps. Codecs are in other crates
//! (`craft-audio`; video later), which depend on this one.
//!
//! - `bitstream` (feature `bitstream`): MSB-first bit reader and writer, Exp-Golomb codes,
//! emulation-prevention escaping and NAL unit splitting.
//! - `isobmff` (feature `isobmff`, enables `bitstream`): a clean-room ISO BMFF (MP4) and
//! QuickTime (MOV) demuxer and muxer, and a box reader for other ISO BMFF formats.
//! - `matroska` (feature `matroska`): a clean-room Matroska (MKV) and WebM demuxer and muxer.
//! - [`Container`] and [`CodecId`] (no feature): the identifiers the codec crates report a file's
//! container and codec with.
//!
//! `bitstream` and `isobmff` are on by default; `matroska` is opt-in. A crate that needs only bit
//! I/O uses `default-features = false, features = ["bitstream"]`. Codecs stay in the apps.
//! I/O uses `default-features = false, features = ["bitstream"]`.
#[cfg(feature = "bitstream")]
pub mod bitstream;
mod id;
#[cfg(feature = "isobmff")]
pub mod isobmff;
#[cfg(feature = "matroska")]
pub mod matroska;
pub mod source;
pub use id::{CodecId, CodecParams, Container, PcmFormat, xiph_lace};
pub use source::{ByteSource, ReadSeekSource};
crates/media/src/matroska/demux.rs
+1 -1
View File
@@ -14,8 +14,8 @@ use crate::matroska::meta::{
Attachment, ClusterInfo, CuePoint, Edition, SeekEntry, SegmentInfo, Tag, parse_chapters, parse_cues, parse_info, parse_seekhead, parse_tags,
parse_tracks,
};
use crate::matroska::source::{ByteSource, ReadSeekSource};
use crate::matroska::track::{Sample, Track, TrackKind, gcd};
use crate::source::{ByteSource, ReadSeekSource};
const WIN: usize = 64 * 1024;
/// Upper bound for level-1 metadata elements read into memory (Info, Tracks, Cues, Tags…).
crates/media/src/matroska/mod.rs
+1 -2
View File
@@ -17,11 +17,11 @@ mod error;
mod ids;
mod meta;
mod mux;
mod source;
#[cfg(test)]
mod synthetic_tests;
mod track;
pub use crate::source::{ByteSource, ReadSeekSource};
pub use codec::Codec;
pub use demux::{Demuxer, Keyframe, MkvFile, OpenOptions, Packet, SeekPoint, open, open_with};
pub use error::{Error, Result};
@@ -29,7 +29,6 @@ pub use meta::{
Attachment, Chapter, ChapterDisplay, ClusterInfo, CuePoint, CuePosition, Edition, SeekEntry, SegmentInfo, SimpleTag, Tag, TagTargets,
};
pub use mux::{LacingMode, MkvWriter, MuxOptions, TrackSpec};
pub use source::{ByteSource, ReadSeekSource};
pub use track::{AudioInfo, Color, ContentEncoding, MasteringMetadata, Projection, Sample, Track, TrackKind, VideoInfo};
/// Element IDs used by the demuxer (marker bits included), for callers inspecting raw elements.
crates/media/src/matroska/source.rs
-174
View File
@@ -1,174 +0,0 @@
//! Random-access byte sources for the demuxer.
use std::cell::RefCell;
use std::io::{self, Read, Seek, SeekFrom};
use std::sync::Arc;
/// Random-access, read-only byte source (file, in-memory buffer, web Blob…).
///
/// Same shape as `craft_media::isobmff::ByteSource`, so one adapter type can implement both. They
/// are separate traits so that a type can implement both without conflicting impls.
pub trait ByteSource {
/// Total length in bytes.
fn len(&self) -> u64;
/// Fill `buf` entirely with bytes starting at `offset`; fail with `UnexpectedEof` if past the end.
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()>;
/// True if the source holds no bytes.
fn is_empty(&self) -> bool {
self.len() == 0
}
}
fn eof() -> io::Error {
io::Error::new(io::ErrorKind::UnexpectedEof, "read past end of byte source")
}
impl ByteSource for [u8] {
fn len(&self) -> u64 {
<[u8]>::len(self) as u64
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
let start = usize::try_from(offset).map_err(|_| eof())?;
let end = start.checked_add(buf.len()).ok_or_else(eof)?;
let src = self.get(start..end).ok_or_else(eof)?;
buf.copy_from_slice(src);
Ok(())
}
}
impl ByteSource for Vec<u8> {
fn len(&self) -> u64 {
self.as_slice().len() as u64
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
self.as_slice().read_at(offset, buf)
}
}
impl<T: ByteSource + ?Sized> ByteSource for &T {
fn len(&self) -> u64 {
(**self).len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
(**self).read_at(offset, buf)
}
}
impl<T: ByteSource + ?Sized> ByteSource for Arc<T> {
fn len(&self) -> u64 {
(**self).len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
(**self).read_at(offset, buf)
}
}
impl<T: ByteSource + ?Sized> ByteSource for Box<T> {
fn len(&self) -> u64 {
(**self).len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
(**self).read_at(offset, buf)
}
}
#[cfg(not(target_arch = "wasm32"))]
impl ByteSource for std::fs::File {
fn len(&self) -> u64 {
self.metadata().map(|m| m.len()).unwrap_or(0)
}
#[cfg(unix)]
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
std::os::unix::fs::FileExt::read_exact_at(self, buf, offset)
}
#[cfg(windows)]
fn read_at(&self, mut offset: u64, mut buf: &mut [u8]) -> io::Result<()> {
use std::os::windows::fs::FileExt;
while !buf.is_empty() {
let n = self.seek_read(buf, offset)?;
if n == 0 {
return Err(eof());
}
offset += n as u64;
buf = &mut buf[n..];
}
Ok(())
}
#[cfg(not(any(unix, windows)))]
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
let mut f = self;
f.seek(SeekFrom::Start(offset))?;
f.read_exact(buf)
}
}
/// Adapts any `Read + Seek` (e.g. `BufReader<File>`, `Cursor<Vec<u8>>`) into a [`ByteSource`].
///
/// The length is taken once at construction; the stream position is tracked so sequential reads
/// do not issue redundant seeks.
pub struct ReadSeekSource<R> {
inner: RefCell<(R, u64)>,
len: u64,
}
impl<R: Read + Seek> ReadSeekSource<R> {
/// Wrap a reader; seeks to the end once to learn the length.
pub fn new(mut r: R) -> io::Result<Self> {
let len = r.seek(SeekFrom::End(0))?;
Ok(Self { inner: RefCell::new((r, len)), len })
}
/// Unwrap the reader.
pub fn into_inner(self) -> R {
self.inner.into_inner().0
}
}
impl<R: Read + Seek> ByteSource for ReadSeekSource<R> {
fn len(&self) -> u64 {
self.len
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
if offset.checked_add(buf.len() as u64).is_none_or(|e| e > self.len) {
return Err(eof());
}
let mut g = self.inner.borrow_mut();
let (r, pos) = &mut *g;
if *pos != offset {
r.seek(SeekFrom::Start(offset))?;
*pos = offset;
}
let res = r.read_exact(buf);
match res {
Ok(()) => {
*pos = offset + buf.len() as u64;
Ok(())
}
Err(e) => {
*pos = u64::MAX; // unknown; force a seek next time
Err(e)
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn slice_and_reader_sources() {
let v = vec![1u8, 2, 3, 4];
let mut b = [0u8; 2];
v.read_at(1, &mut b).unwrap();
assert_eq!(b, [2, 3]);
assert!(v.read_at(3, &mut b).is_err());
assert!(v.read_at(u64::MAX, &mut b).is_err());
let r = ReadSeekSource::new(io::Cursor::new(v.clone())).unwrap();
assert_eq!(r.len(), 4);
r.read_at(2, &mut b).unwrap();
assert_eq!(b, [3, 4]);
r.read_at(0, &mut b).unwrap();
assert_eq!(b, [1, 2]);
assert!(r.read_at(3, &mut b).is_err());
}
}
crates/media/src/matroska/track.rs
+83
View File
@@ -234,6 +234,56 @@ pub struct Track {
}
impl Track {
/// What a decoder needs for this track, in the container-neutral form the codec crates take
/// (`CodecPrivate` is already in that form).
pub fn codec_params(&self) -> crate::CodecParams {
use crate::CodecId as Id;
let (id, extradata) = match &self.codec {
Codec::Avc { avcc } => (Id::H264, avcc.clone()),
Codec::Hevc { hvcc } => (Id::Hevc, hvcc.clone()),
Codec::Vp8 => (Id::Vp8, Vec::new()),
Codec::Vp9 { private } => (Id::Vp9, private.clone()),
Codec::Av1 { av1c } => (Id::Av1, av1c.clone()),
Codec::ProRes { .. } => (Id::ProRes, Vec::new()),
Codec::Apv { apvc } => (Id::Apv, apvc.clone()),
Codec::Mjpeg => (Id::Mjpeg, Vec::new()),
Codec::Aac { asc } => (Id::Aac, asc.clone()),
Codec::Opus { head } => (Id::Opus, head.clone()),
Codec::Vorbis { headers } => (Id::Vorbis, crate::xiph_lace(headers).unwrap_or_default()),
Codec::Flac { private } => (Id::Flac, private.clone()),
Codec::Pcm { .. } => (Id::Pcm, Vec::new()),
Codec::Ac3 => (Id::Ac3, Vec::new()),
Codec::Eac3 => (Id::Eac3, Vec::new()),
Codec::Mp3 => (Id::Mp3, Vec::new()),
Codec::Mp2 => (Id::Mp2, Vec::new()),
Codec::Mp1 => (Id::Mp1, Vec::new()),
// the ALACSpecificConfig, as MP4's `alac` box holds it after its FullBox header
Codec::Other(id) if id == "A_ALAC" => (Id::Alac, self.codec_private.clone()),
Codec::VfwFourcc { fourcc: [b'A', b'V', b'd', b'n' | b'h'], .. } => (Id::Dnxhd, Vec::new()),
_ => (Id::Unknown, Vec::new()),
};
let mut p = crate::CodecParams::new(id).with_extradata(extradata);
match &self.codec {
Codec::ProRes { fourcc: Some(tag) } | Codec::VfwFourcc { fourcc: tag, .. } => p = p.with_tag(*tag),
_ => {}
}
if let Some(v) = &self.video {
p = p.with_size(v.pixel_width, v.pixel_height);
}
if let Some(a) = &self.audio {
p = p.with_sample_rate(a.sampling_frequency.round() as u32).with_channels(u16::try_from(a.channels).unwrap_or(0));
}
if let Codec::Opus { .. } = self.codec {
// Opus always decodes at 48 kHz; a muxer may have declared the input rate instead
p = p.with_sample_rate(48_000);
}
if let Codec::Pcm { float, big_endian, bits } = self.codec {
// 8-bit integer PCM is unsigned in Matroska, as in WAV.
p = p.with_pcm(if float { crate::PcmFormat::float(bits, big_endian) } else { crate::PcmFormat::int(bits, bits > 8, big_endian) });
}
p
}
/// Bytes prepended to every frame by header-stripping compression, if any.
pub fn stripped_header(&self) -> Option<&[u8]> {
self.encodings.iter().find_map(|e| match e {
@@ -381,3 +431,36 @@ mod aspect_overflow_tests {
assert_eq!(video.pixel_aspect(), (3, 4));
}
}
#[cfg(test)]
mod codec_params_tests {
use crate::CodecId;
use crate::matroska::mux::{MkvWriter, MuxOptions, TrackSpec};
use crate::matroska::{Demuxer, TrackKind};
#[test]
fn alac_dnx_and_opus_match_mp4() {
let cookie: Vec<u8> = (0..24).collect();
let mut alac = TrackSpec::new(TrackKind::Audio, "A_ALAC");
alac.codec_private = cookie.clone();
alac.audio = Some((44_100.0, 2, Some(16)));
let mut bih = vec![0u8; 40];
bih[16..20].copy_from_slice(b"AVdn");
let mut dnx = TrackSpec::new(TrackKind::Video, "V_MS/VFW/FOURCC");
dnx.codec_private = bih;
dnx.video_size = Some((1920, 1080));
let mut opus = TrackSpec::new(TrackKind::Audio, "A_OPUS");
opus.codec_private = [b"OpusHead".as_slice(), &[1, 2, 0x38, 0x01, 0x44, 0xAC, 0, 0, 0, 0, 0]].concat();
opus.audio = Some((44_100.0, 2, None)); // the input rate, as some muxers write it
let mut m = MkvWriter::new(std::io::Cursor::new(Vec::new()), vec![alac, dnx, opus], MuxOptions::default()).unwrap();
for t in 0..3 {
m.write_frame(t, 0, true, &[0; 4], None).unwrap();
}
let bytes = m.finish().unwrap().into_inner();
let d = Demuxer::from_slice(&bytes).unwrap();
let p: Vec<_> = d.tracks().iter().map(|t| t.codec_params()).collect();
assert_eq!((p[0].codec, p[0].extradata.as_slice()), (CodecId::Alac, cookie.as_slice()));
assert_eq!((p[1].codec, p[1].tag), (CodecId::Dnxhd, Some(*b"AVdn")));
assert_eq!((p[2].codec, p[2].sample_rate, p[2].channels), (CodecId::Opus, Some(48_000), Some(2)));
}
}
crates/media/src/source.rs
+269
View File
@@ -0,0 +1,269 @@
//! Random-access byte sources: what the demuxers read from. One trait for every container, so a
//! file, buffer or custom source implements it once.
#![warn(missing_docs)]
use std::io::{self, Read, Seek, SeekFrom};
use std::sync::{Arc, Mutex};
/// Random-access, read-only byte source (file, in-memory buffer, web Blob…).
///
/// Implementations serve reads at arbitrary offsets through `&self`. `isobmff::ByteSource` and
/// `matroska::ByteSource` are this trait.
pub trait ByteSource {
/// Total length in bytes.
fn len(&self) -> u64;
/// Fill `buf` entirely with bytes starting at `offset`; fail with `UnexpectedEof` if past the end.
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()>;
/// True if the source holds no bytes.
fn is_empty(&self) -> bool {
self.len() == 0
}
}
fn eof() -> io::Error {
io::Error::new(io::ErrorKind::UnexpectedEof, "read past end of byte source")
}
impl ByteSource for [u8] {
fn len(&self) -> u64 {
<[u8]>::len(self) as u64
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
let start = usize::try_from(offset).map_err(|_| eof())?;
let end = start.checked_add(buf.len()).ok_or_else(eof)?;
let src = self.get(start..end).ok_or_else(eof)?;
buf.copy_from_slice(src);
Ok(())
}
}
impl ByteSource for Vec<u8> {
fn len(&self) -> u64 {
self.as_slice().len() as u64
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
self.as_slice().read_at(offset, buf)
}
}
impl<T: ByteSource + ?Sized> ByteSource for &T {
fn len(&self) -> u64 {
(**self).len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
(**self).read_at(offset, buf)
}
}
impl<T: ByteSource + ?Sized> ByteSource for Arc<T> {
fn len(&self) -> u64 {
(**self).len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
(**self).read_at(offset, buf)
}
}
impl<T: ByteSource + ?Sized> ByteSource for Box<T> {
fn len(&self) -> u64 {
(**self).len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
(**self).read_at(offset, buf)
}
}
/// Past the largest offset a file can have (`i64::MAX`), where the OS would fail with
/// `InvalidInput` instead of reporting the end of the file.
#[cfg(not(target_arch = "wasm32"))]
fn past_file_offsets(offset: u64, len: usize) -> bool {
offset.checked_add(len as u64).is_none_or(|e| e > i64::MAX as u64)
}
/// A file read in place. `len` asks the file system each time, and a failure there reads as an
/// empty file; [`ReadSeekSource`] takes the length once and reports a failure when it is made.
#[cfg(not(target_arch = "wasm32"))]
impl ByteSource for std::fs::File {
fn len(&self) -> u64 {
self.metadata().map(|m| m.len()).unwrap_or(0)
}
#[cfg(unix)]
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
if past_file_offsets(offset, buf.len()) {
return Err(eof());
}
std::os::unix::fs::FileExt::read_exact_at(self, buf, offset)
}
#[cfg(windows)]
fn read_at(&self, mut offset: u64, mut buf: &mut [u8]) -> io::Result<()> {
use std::os::windows::fs::FileExt;
if past_file_offsets(offset, buf.len()) {
return Err(eof());
}
while !buf.is_empty() {
let n = self.seek_read(buf, offset)?;
if n == 0 {
return Err(eof());
}
offset += n as u64;
buf = &mut buf[n..];
}
Ok(())
}
#[cfg(not(any(unix, windows)))]
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
if past_file_offsets(offset, buf.len()) {
return Err(eof());
}
let mut f = self;
f.seek(SeekFrom::Start(offset))?;
f.read_exact(buf)
}
}
/// Adapts any `Read + Seek` (e.g. `BufReader<File>`, `Cursor<Vec<u8>>`) into a [`ByteSource`].
///
/// The length is taken once at construction; the stream position is tracked so sequential reads
/// do not issue redundant seeks. The reader sits behind a mutex, so the source is `Send + Sync`
/// whenever `R` is `Send` and can be shared across threads.
pub struct ReadSeekSource<R> {
inner: Mutex<(R, u64)>,
len: u64,
}
impl<R: Read + Seek> ReadSeekSource<R> {
/// Wrap a reader; seeks to the end once to learn the length.
pub fn new(mut r: R) -> io::Result<Self> {
let len = r.seek(SeekFrom::End(0))?;
Ok(Self { inner: Mutex::new((r, len)), len })
}
/// Unwrap the reader.
pub fn into_inner(self) -> R {
// A poisoned lock only means a read panicked in another thread; the reader is still usable.
self.inner.into_inner().unwrap_or_else(|e| e.into_inner()).0
}
}
impl<R: Read + Seek> ByteSource for ReadSeekSource<R> {
fn len(&self) -> u64 {
self.len
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
if offset.checked_add(buf.len() as u64).is_none_or(|e| e > self.len) {
return Err(eof());
}
let mut g = self.inner.lock().unwrap_or_else(|e| {
// a read panicked in another thread and may have moved the reader; force a seek
let mut g = e.into_inner();
g.1 = u64::MAX;
self.inner.clear_poison();
g
});
let (r, pos) = &mut *g;
if *pos != offset {
*pos = u64::MAX; // unknown until the seek succeeds
r.seek(SeekFrom::Start(offset))?;
*pos = offset;
}
let res = r.read_exact(buf);
match res {
Ok(()) => {
*pos = offset + buf.len() as u64;
Ok(())
}
Err(e) => {
*pos = u64::MAX; // unknown; force a seek next time
Err(e)
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn slice_and_reader_sources() {
let v = vec![1u8, 2, 3, 4];
let mut b = [0u8; 2];
v.read_at(1, &mut b).unwrap();
assert_eq!(b, [2, 3]);
assert!(v.read_at(3, &mut b).is_err());
assert!(v.read_at(u64::MAX, &mut b).is_err());
let r = ReadSeekSource::new(io::Cursor::new(v.clone())).unwrap();
assert_eq!(r.len(), 4);
r.read_at(2, &mut b).unwrap();
assert_eq!(b, [3, 4]);
r.read_at(0, &mut b).unwrap();
assert_eq!(b, [1, 2]);
assert!(r.read_at(3, &mut b).is_err());
let a: Arc<[u8]> = Arc::from(v.clone());
a.read_at(2, &mut b).unwrap();
assert_eq!(b, [3, 4]);
assert_eq!(ByteSource::len(&a), 4);
let s: &[u8] = &v;
assert_eq!(ByteSource::len(&s), 4);
}
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn file_sources_read_like_a_buffer() {
let v: Vec<u8> = (0..=255).collect();
let path = std::env::temp_dir().join(format!("craft-media-source-{}.bin", std::process::id()));
std::fs::write(&path, &v).unwrap();
let f = std::fs::File::open(&path).unwrap();
let r = ReadSeekSource::new(std::fs::File::open(&path).unwrap()).unwrap();
let sources: [&dyn ByteSource; 3] = [&v, &f, &r];
for s in sources {
assert_eq!(s.len(), 256);
// out of order, so ReadSeekSource has to seek back
for (offset, n) in [(0, 0), (10, 16), (0, 256), (200, 56), (255, 1), (3, 7)] {
let mut b = vec![0u8; n];
s.read_at(offset as u64, &mut b).unwrap();
assert_eq!(b, v[offset..offset + n], "offset {offset}, {n} bytes");
}
for (offset, n) in [(200, 57), (256, 1), (300, 1), (u64::MAX, 1), (u64::MAX - 1, 4)] {
let err = s.read_at(offset, &mut vec![0u8; n]).unwrap_err();
assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof, "offset {offset}, {n} bytes");
}
}
drop((f, r)); // Windows deletes no open file
std::fs::remove_file(&path).ok();
}
#[test]
fn read_seek_source_seeks_after_a_panicked_read() {
struct PanicOnce(io::Cursor<Vec<u8>>, bool);
impl Read for PanicOnce {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
let n = self.0.read(&mut buf[..1])?;
if std::mem::take(&mut self.1) {
panic!("read failed halfway");
}
Ok(n)
}
}
impl Seek for PanicOnce {
fn seek(&mut self, to: SeekFrom) -> io::Result<u64> {
self.0.seek(to)
}
}
let r = ReadSeekSource::new(PanicOnce(io::Cursor::new(vec![1, 2, 3, 4]), false)).unwrap();
let mut b = [0u8; 2];
r.read_at(0, &mut b).unwrap();
r.inner.lock().unwrap().0.1 = true;
// the reader moves one byte, then panics with the tracked position at 2
assert!(std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| r.read_at(2, &mut b))).is_err());
r.read_at(2, &mut b).unwrap();
assert_eq!(b, [3, 4]);
assert!(!r.inner.is_poisoned(), "recovered once, not on every read");
}
#[test]
fn read_seek_source_is_send_and_sync() {
fn shareable<T: ByteSource + Send + Sync + 'static>(_: &T) {}
shareable(&ReadSeekSource::new(io::Cursor::new(vec![0u8; 4])).unwrap());
}
}
crates/media/tests/common/mod.rs
+3 -147
View File
@@ -1,167 +1,23 @@
//! Shared helpers for oracle tests: ffmpeg/ffprobe discovery and fixture generation.
//! ffmpeg is used only as an external fixture generator and oracle (never linked).
//!
//! Tool search order: `CRAFT_MEDIA_FFMPEG` / `CRAFT_MEDIA_FFPROBE` (an explicit path; a wrong
//! path fails the test), then `PATH`, then well-known install directories. A missing tool, a
//! failed fixture generator or a failed ffprobe run prints `SKIPPED` and the test passes, unless
//! `CRAFT_MEDIA_REQUIRE_ORACLES=1` is set: then each of these fails the test.
//! Fixtures are generated into `$CRAFT_MEDIA_FIXTURES_DIR` or `<workspace>/target/fixtures`, one
//! directory per container (`isobmff`, `matroska`), and never committed.
//! craft-media's oracle-test helpers: the generic ones from `craft-media-testkit`, and the MP4
//! fixtures and ffprobe queries the container tests share.
#![allow(dead_code)]
pub use craft_media_testkit::*;
use serde_json::Value;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicUsize, Ordering};
const REQUIRE_ENV: &str = "CRAFT_MEDIA_REQUIRE_ORACLES";
const KNOWN_DIRS: &[&str] = &[
"/opt/homebrew/bin",
"/usr/local/bin",
"/usr/bin",
"/opt/local/bin",
"/snap/bin",
r"C:\ffmpeg\bin",
r"C:\Program Files\ffmpeg\bin",
r"C:\ProgramData\chocolatey\bin",
];
fn env_var(name: &str) -> String {
format!("CRAFT_MEDIA_{}", name.to_ascii_uppercase())
}
fn find_tool(name: &str) -> Option<PathBuf> {
if let Some(e) = std::env::var_os(env_var(name)).filter(|e| !e.is_empty()) {
let p = PathBuf::from(e);
assert!(p.is_file(), "{} is set but `{}` is not a file", env_var(name), p.display());
return Some(p);
}
let names = if cfg!(windows) { [format!("{name}.exe"), name.to_string()] } else { [name.to_string(), format!("{name}.exe")] };
let path_dirs = std::env::var_os("PATH").map(|p| std::env::split_paths(&p).collect::<Vec<_>>()).unwrap_or_default();
let dirs = path_dirs.into_iter().chain(KNOWN_DIRS.iter().map(PathBuf::from));
for dir in dirs {
for n in &names {
let p = dir.join(n);
if p.is_file() {
return Some(p);
}
}
}
None
}
/// Find a tool silently (cached for the process for ffmpeg and ffprobe).
pub fn tool(name: &str) -> Option<PathBuf> {
static FFMPEG: OnceLock<Option<PathBuf>> = OnceLock::new();
static FFPROBE: OnceLock<Option<PathBuf>> = OnceLock::new();
match name {
"ffmpeg" => FFMPEG.get_or_init(|| find_tool("ffmpeg")).clone(),
"ffprobe" => FFPROBE.get_or_init(|| find_tool("ffprobe")).clone(),
_ => find_tool(name),
}
}
pub fn oracles_required() -> bool {
std::env::var(REQUIRE_ENV).is_ok_and(|v| matches!(v.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on"))
}
/// Give up on an oracle check: print `SKIPPED` and return `None`, or fail when oracles are
/// required.
pub fn skip<T>(reason: &str) -> Option<T> {
skip_with(oracles_required(), reason)
}
fn skip_with<T>(required: bool, reason: &str) -> Option<T> {
assert!(!required, "craft-media oracle: {reason}, and {REQUIRE_ENV}=1 forbids skipping oracle tests");
eprintln!("SKIPPED (craft-media oracle): {reason}");
None
}
/// Run a fixture generator writing `tmp`, then rename it to `out`. A generator that cannot start
/// or fails skips (see [`skip`]); `required` is explicit so tests can set it without touching
/// the process environment.
pub fn run_generator(cmd: &mut Command, tmp: &Path, out: &Path, required: bool) -> Option<PathBuf> {
let reason = match cmd.output() {
Ok(o) if o.status.success() => match std::fs::rename(tmp, out) {
Ok(()) => return Some(out.to_path_buf()),
Err(e) => format!("could not move {} into place: {e}", tmp.display()),
},
Ok(o) => format!("generator failed for {} ({}): {}", out.display(), o.status, String::from_utf8_lossy(&o.stderr).trim()),
Err(e) => format!("could not start generator for {}: {e}", out.display()),
};
let _ = std::fs::remove_file(tmp);
skip_with(required, &format!("{reason} (encoder unavailable?)"))
}
fn tool_or_skip(name: &str) -> Option<PathBuf> {
tool(name).or_else(|| skip(&format!("{name} not found (set {} or put {name} on PATH)", env_var(name))))
}
/// ffmpeg, or `None` after printing `SKIPPED` (a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`).
pub fn ffmpeg() -> Option<PathBuf> {
tool_or_skip("ffmpeg")
}
/// ffprobe, or `None` after printing `SKIPPED` (a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`).
pub fn ffprobe() -> Option<PathBuf> {
tool_or_skip("ffprobe")
}
/// Fixtures of the `isobmff` tests.
pub fn fixture_dir() -> PathBuf {
fixture_subdir("isobmff")
}
/// `$CRAFT_MEDIA_FIXTURES_DIR/<name>` or `<workspace>/target/fixtures/<name>`, created if missing.
pub fn fixture_subdir(name: &str) -> PathBuf {
let d = match std::env::var_os("CRAFT_MEDIA_FIXTURES_DIR").filter(|v| !v.is_empty()) {
Some(d) => PathBuf::from(d),
None => {
let here = Path::new(env!("CARGO_MANIFEST_DIR"));
let root = here.parent().and_then(Path::parent).unwrap_or(here);
root.join("target").join("fixtures")
}
}
.join(name);
std::fs::create_dir_all(&d).unwrap();
d
}
pub fn out_dir() -> PathBuf {
let d = fixture_dir().join("out");
std::fs::create_dir_all(&d).unwrap();
d
}
/// A temporary sibling of `out` that no other thread or process will pick; the extension is kept
/// because ffmpeg chooses the muxer from it.
pub fn temp_path(out: &Path) -> PathBuf {
static N: AtomicUsize = AtomicUsize::new(0);
let tid = format!("{:?}", std::thread::current().id()).replace(|c: char| !c.is_ascii_alphanumeric(), "");
let n = N.fetch_add(1, Ordering::Relaxed);
let stem = out.file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_default();
let name = match out.extension() {
Some(ext) => format!("{stem}.tmp-{}-{tid}-{n}.{}", std::process::id(), ext.to_string_lossy()),
None => format!("{stem}.tmp-{}-{tid}-{n}", std::process::id()),
};
out.with_file_name(name)
}
/// Run a fixture generator and print `FIXTURE <made|cached|skipped> <name>`.
pub fn generate_and_report<T>(name: &str, outputs: &[PathBuf], generate: impl FnOnce() -> Option<T>) -> Option<T> {
let existed = !outputs.is_empty() && outputs.iter().all(|p| std::fs::metadata(p).is_ok_and(|m| m.len() > 0));
let r = generate();
let status = match (&r, existed) {
(None, _) => "skipped",
(Some(_), true) => "cached",
(Some(_), false) => "made",
};
println!("FIXTURE {status} {name}");
r
}
const V: &[&str] = &["-f", "lavfi", "-i", "testsrc2=size=160x120:rate=25"];
const A: &[&str] = &["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000"];
crates/pcraft-format/Cargo.toml
+18
View File
@@ -0,0 +1,18 @@
[package]
name = "craft-pcraft-format"
description = "The envelope of PhotoCraft's .pcraft bundle: the manifest with the document as a type parameter, tile references, entry paths, decode limits and the migration runner."
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
serde = { workspace = true }
# `unbounded_depth`: manifests nest deeper than serde_json's 128 levels; `parse_manifest` checks
# the nesting against `Limits::max_depth` before parsing.
serde_json = { workspace = true, features = ["unbounded_depth"] }
thiserror = { workspace = true }
crates/pcraft-format/README.md
+116
View File
@@ -0,0 +1,116 @@
# craft-pcraft-format
The envelope of PhotoCraft's `.pcraft` bundle: the manifest around the document, tile
references, the entry paths, the limits for reading a manifest, and the runner for manifest
migrations. The document model stays in PhotoCraft and is a type parameter here. The crate
does not read ZIP archives, open files or decode tiles. No `unsafe`, no panics on malformed
input, and it builds for `wasm32-unknown-unknown`. The dependencies are `serde`, `serde_json`
and `thiserror`.
```toml
craft-pcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "<commit>" }
```
## Bundle layout
A bundle is a ZIP archive or a directory with these entries:
| Entry | Constant or function | Contents |
|---|---|---|
| `manifest.json` | `MANIFEST` | the versioned manifest |
| `tiles/<blake3>.zst` | `tile_path(hash)` | a 256×256 tile, zstd, little-endian samples |
| `blobs/<blake3>.zst` | `blob_path(hash)` | binary data, zstd |
| `thumb.png` | `THUMB` | optional thumbnail |
| `composite/preview.png` | `COMPOSITE` | optional flattened preview |
| `save.lock` | `SAVE_LOCK` | directory bundles only: held while a save runs (empty) |
`<blake3>` is the BLAKE3 hash of the uncompressed bytes as 64 lowercase hex digits.
`is_valid_hash` checks that form. `tile_path` and `blob_path` return `None` for any other
string, so a hash read from a file never becomes an entry name such as `tiles/../x.zst`.
`TILES_DIR` (`tiles`), `BLOBS_DIR` (`blobs`) and `COMPOSITE_DIR` (`composite`) name the
directories, and `OBJECT_SUFFIX` (`.zst`) ends every tile and blob entry. `EXTENSION` is
`pcraft` and `FORMAT_VERSION` is 1.
## Use
```rust
use craft_pcraft_format::{FORMAT_VERSION, Limits, Manifest, Step};
const STEPS: &[Step] = &[];
let manifest: Manifest<MyDocument> = craft_pcraft_format::read_manifest(&manifest_json, &Limits::default(), STEPS, FORMAT_VERSION)?;
```
- `read_manifest(json, &Limits, steps, target)` is the reader of a manifest. It runs
`parse_manifest`, then `migrate`, which refuses a version newer than `target`, then reads the
members, the document as `D`. `read_manifest_with` takes a function that reads the
document from its `serde_json::Value` instead of `D`'s `Deserialize`.
- `Manifest<D>` has `format_version`, `generator`, `document: D`, `thumbnail` and `composite`,
and serializes them in that order. Absent previews serialize as `null`. On read, missing
previews are `None`, members the type does not name are skipped, and a `thumbnail` other
than `thumb.png` or a `composite` other than `composite/preview.png` gives `Corrupt`.
`Manifest` has no `Deserialize` and is `#[non_exhaustive]`: `Manifest::new(generator,
document)` builds one at `FORMAT_VERSION` without previews.
- `TileRef` has `tx`, `ty` and `hash`, and is `#[non_exhaustive]`: `TileRef::new(tx, ty, hash)`
builds one. `TileRef::path` gives its entry, or `None` when the hash is not valid.
- `check_manifest(json, &Limits)` checks the size, the nesting and the item count in one pass
without recursion. PhotoCraft's own check of a manifest it saves tests the nesting and the
item count only. `parse_manifest(json, &Limits)` runs that check and then parses into a
`serde_json::Value` with serde_json's 128-level recursion limit turned off.
- `migrate(&mut value, steps, target)` upgrades the JSON in place, where `steps[i]` upgrades
version `i + 1` to `i + 2`, and returns the version the file had. `Step<E>` is
`fn(&mut Value) -> Result<(), E>`; with `E: From<Error>`, the steps return the caller's own
error type.
- `Error` is `Corrupt`, `Unsupported`, `TooNew`, `LimitExceeded` or `Json`. A manifest whose
`format_version` is newer than `target` gives `TooNew`, and one with no step to the next
version gives `Unsupported`. A missing or invalid `format_version` gives `Corrupt`, and any
other missing or mistyped member gives `Json`. `TooNew` holds `found`, `supported` and the manifest's
`generator` (`None` when it is missing or not a string), and is matched with `..`.
## Limits
`Limits` (fields `max_manifest_bytes`, `max_blob_bytes`, `max_total_bytes`, `max_depth`,
`max_items`) holds the limits PhotoCraft loads with:
- `max_manifest_bytes` (256 MiB): the size of `manifest.json`.
- `max_blob_bytes` (1 GiB): one decompressed blob.
- `max_total_bytes` (16 GiB): every decompressed tile and blob in one bundle.
- `max_depth` (367, `manifest_depth(MAX_GROUP_DEPTH)`): the nesting of JSON arrays and objects.
PhotoCraft nests up to `MAX_GROUP_DEPTH` (100, a copy of PhotoCraft's own constant) layer
groups, three levels each, and allows 64 more levels for the document and leaf layers. A
value above `MAX_DEPTH` (400) is treated as 400, so parsing stays within the stack.
- `max_items` (`MAX_MANIFEST_ITEMS`, 4,194,304): arrays and objects opened, plus elements and
members after a comma.
`check_manifest`, `parse_manifest` and `read_manifest` apply the manifest limits. The crate
does not read tiles or blobs, so it does not apply `max_blob_bytes` or `max_total_bytes`: the
caller that decompresses the entries enforces them. The struct is `#[non_exhaustive]`: start from the
defaults and set the fields you change.
```rust
let mut limits = craft_pcraft_format::Limits::default();
limits.max_manifest_bytes = 16 << 20;
```
## Testing
`cargo test -p craft-pcraft-format` runs:
- `tests/manifest.rs`: a manifest with and without previews serializes with
`serde_json::to_vec_pretty` to JSON written by hand in the test, byte for byte, and reads
back equal; missing previews and unknown members; entry names, hash checks, and no entry name
for an invalid hash.
- `tests/read.rs`: `read_manifest` and `read_manifest_with` read the envelope and the document,
refuse a newer version with the writer's `generator`, refuse a manifest past the limits, run
the steps first, and report missing or mistyped members as a struct with a derived
`Deserialize` reports them, in the same order; a missing or invalid `format_version` is
`Corrupt`, and so is a preview that names another entry, such as `../x` or an absolute path.
- Doc tests: `Manifest` and `TileRef` are not built by struct literal, `Manifest` has no
`Deserialize`, and `Error::TooNew` is matched with `..`.
- `tests/limits.rs`: the default limits, nesting at and past `max_depth`, items at and past
`max_items`, brackets inside strings, the manifest size, JSON errors and the messages.
- `tests/migrate.rs`: steps in order, partial chains, the current version, newer, missing,
zero and out-of-range versions, a missing step, and steps that return the caller's error type.
`.github/workflows/pcraft-format.yml` runs fmt, clippy and the tests on Linux and Windows, and
a `wasm32-unknown-unknown` check.
crates/pcraft-format/src/lib.rs
+195
View File
@@ -0,0 +1,195 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
//! The envelope of PhotoCraft's `.pcraft` bundle, shared by PhotoCraft and by readers in other
//! apps.
//!
//! ```text
//! manifest.json versioned manifest (see [`Manifest`])
//! tiles/<blake3>.zst content-addressed 256×256 tiles (zstd, little-endian samples)
//! blobs/<blake3>.zst content-addressed binary data
//! thumb.png optional thumbnail
//! composite/preview.png optional flattened preview
//! save.lock directory bundles only: held while a save runs (empty)
//! ```
//!
//! A bundle is a ZIP archive or a directory with this layout. [`Manifest`] holds the format
//! version, the writer and the preview paths, with the document as the type parameter `D`, so
//! the document model stays in the app. [`TileRef`] is a tile's place in a surface.
//! [`read_manifest`] reads manifest JSON: it checks the JSON against [`Limits`], refuses a version
//! newer than the one the caller reads, upgrades older versions with the caller's steps, and reads
//! the envelope and the document.
//!
//! The crate does not read ZIP archives, open files or decode tiles. It builds for
//! `wasm32-unknown-unknown`.
//!
//! ```
//! use craft_pcraft_format::{FORMAT_VERSION, Limits, Manifest, Step};
//!
//! let raw = br#"{"format_version":1,"generator":"photocraft-format 0.1.0","document":{"name":"Untitled"}}"#;
//! let steps: &[Step] = &[];
//! let manifest: Manifest<serde_json::Value> = craft_pcraft_format::read_manifest(raw, &Limits::default(), steps, FORMAT_VERSION).unwrap();
//! assert_eq!(manifest.document["name"], "Untitled");
//! assert_eq!(manifest.thumbnail, None);
//! ```
mod limits;
mod migrate;
mod read;
use serde::{Deserialize, Serialize};
pub use limits::{Limits, MAX_DEPTH, MAX_GROUP_DEPTH, MAX_MANIFEST_ITEMS, check_manifest, manifest_depth, parse_manifest};
pub use migrate::{Step, migrate};
pub use read::{read_manifest, read_manifest_with};
/// The manifest version PhotoCraft writes.
pub const FORMAT_VERSION: u32 = 1;
/// File extension of the bundle, without the dot.
pub const EXTENSION: &str = "pcraft";
/// The directory of the tiles.
pub const TILES_DIR: &str = "tiles";
/// The directory of the blobs.
pub const BLOBS_DIR: &str = "blobs";
/// The directory of the flattened preview, [`COMPOSITE`].
pub const COMPOSITE_DIR: &str = "composite";
/// The suffix of every tile and blob entry, after the hash.
pub const OBJECT_SUFFIX: &str = ".zst";
/// The manifest entry.
pub const MANIFEST: &str = "manifest.json";
/// The thumbnail entry, a PNG.
pub const THUMB: &str = "thumb.png";
/// The flattened preview entry, a PNG, in [`COMPOSITE_DIR`].
pub const COMPOSITE: &str = "composite/preview.png";
/// The file a directory save holds an exclusive lock on while it runs. ZIP bundles do not have it.
pub const SAVE_LOCK: &str = "save.lock";
/// Hex-encoded BLAKE3 hash (64 lowercase hex digits) of an object's uncompressed bytes.
pub type Hash = String;
/// The entry of the tile with hash `hash`: `tiles/<hash>.zst`. `None` when [`is_valid_hash`]
/// refuses `hash`.
pub fn tile_path(hash: &str) -> Option<String> {
is_valid_hash(hash).then(|| format!("{TILES_DIR}/{hash}{OBJECT_SUFFIX}"))
}
/// The entry of the blob with hash `hash`: `blobs/<hash>.zst`. `None` when [`is_valid_hash`]
/// refuses `hash`.
pub fn blob_path(hash: &str) -> Option<String> {
is_valid_hash(hash).then(|| format!("{BLOBS_DIR}/{hash}{OBJECT_SUFFIX}"))
}
/// `true` when `hash` is 64 lowercase hex digits, the only form a valid manifest gives.
/// [`tile_path`] and [`blob_path`] build no entry name for any other string.
pub fn is_valid_hash(hash: &str) -> bool {
hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
/// `manifest.json`: the envelope around the app's document `D`.
///
/// The fields serialize in this order, and `thumbnail` and `composite` serialize as `null` when
/// absent. [`read_manifest`] reads one: a missing `thumbnail` or `composite` reads as `None`,
/// members the type does not name are skipped, and a preview that names any entry other than
/// [`THUMB`] or [`COMPOSITE`] is refused.
///
/// The struct is `#[non_exhaustive]`: build one with [`Manifest::new`] and set the previews.
///
/// ```compile_fail
/// let _ = craft_pcraft_format::Manifest { format_version: 1, generator: String::new(), document: (), thumbnail: None, composite: None };
/// ```
///
/// It has no `Deserialize`, so a manifest is read only through [`read_manifest`]:
///
/// ```compile_fail
/// let _: craft_pcraft_format::Manifest<serde_json::Value> = serde_json::from_str("{}").unwrap();
/// ```
#[derive(Debug, Clone, PartialEq, Serialize)]
#[non_exhaustive]
pub struct Manifest<D> {
pub format_version: u32,
/// Free-form writer identification, for example `photocraft-format 0.1.0`.
pub generator: String,
pub document: D,
/// The thumbnail entry, [`THUMB`]. [`read_manifest`] refuses any other name.
pub thumbnail: Option<String>,
/// The flattened preview entry, [`COMPOSITE`]. [`read_manifest`] refuses any other name.
pub composite: Option<String>,
}
impl<D> Manifest<D> {
/// A manifest at [`FORMAT_VERSION`] written by `generator`, holding `document`, with no
/// previews.
pub fn new(generator: impl Into<String>, document: D) -> Self {
Manifest { format_version: FORMAT_VERSION, generator: generator.into(), document, thumbnail: None, composite: None }
}
}
/// One tile of a surface: its column `tx`, its row `ty` and the hash of its samples, stored at
/// [`tile_path`]. `hash` is read as written; [`TileRef::path`] checks it.
///
/// The struct is `#[non_exhaustive]`: build one with [`TileRef::new`].
///
/// ```compile_fail
/// let _ = craft_pcraft_format::TileRef { tx: 0, ty: 0, hash: String::new() };
/// ```
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[non_exhaustive]
pub struct TileRef {
pub tx: i32,
pub ty: i32,
pub hash: Hash,
}
impl TileRef {
/// The tile at column `tx` and row `ty` whose samples hash to `hash`.
pub fn new(tx: i32, ty: i32, hash: Hash) -> Self {
TileRef { tx, ty, hash }
}
/// The tile's entry, [`tile_path`] of its `hash`: `None` when the hash is not valid.
pub fn path(&self) -> Option<String> {
tile_path(&self.hash)
}
}
/// Why a manifest could not be read.
///
/// [`Error::TooNew`] is `#[non_exhaustive]`: match it with `..`.
///
/// ```compile_fail
/// fn found(e: craft_pcraft_format::Error) -> u32 {
/// match e {
/// craft_pcraft_format::Error::TooNew { found, supported: _, generator: _ } => found,
/// _ => 0,
/// }
/// }
/// ```
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum Error {
/// The manifest breaks the format.
#[error("corrupt .pcraft bundle: {0}")]
Corrupt(String),
/// The manifest needs a migration step the caller did not pass.
#[error("unsupported .pcraft feature: {0}")]
Unsupported(String),
/// The manifest's `format_version` is newer than the version the caller reads.
#[error("bundle written by a newer version (format {found}, this build reads up to {supported})")]
#[non_exhaustive]
TooNew {
/// The manifest's `format_version`.
found: u32,
/// The version the caller reads.
supported: u32,
/// The manifest's `generator`, so a message can name the writer. `None` when it is
/// missing or not a string.
generator: Option<String>,
},
/// The manifest passes one of the caller's [`Limits`].
#[error("limit exceeded: {0}")]
LimitExceeded(String),
/// The manifest is not valid JSON, or a member does not have the type the reader expects.
#[error("manifest JSON: {0}")]
Json(#[from] serde_json::Error),
}
crates/pcraft-format/src/limits.rs
+114
View File
@@ -0,0 +1,114 @@
use crate::Error;
/// Deepest nesting of layer groups PhotoCraft saves. The value is a copy of PhotoCraft's
/// `photocraft_doc::MAX_GROUP_DEPTH`; change both together.
pub const MAX_GROUP_DEPTH: usize = 100;
/// Default [`Limits::max_items`]: the most JSON items (arrays and objects opened, plus elements
/// and members after a comma) a manifest may hold.
pub const MAX_MANIFEST_ITEMS: usize = 4 << 20;
/// The largest [`Limits::max_depth`] the checks apply. A larger value is treated as this one,
/// so parsing stays within the stack.
pub const MAX_DEPTH: usize = 400;
/// The deepest JSON nesting of a manifest whose layer groups nest `groups` deep: three levels per
/// group (the layer object, its content object and the `children` array) and 64 for the document
/// and the leaf layers' fields.
pub const fn manifest_depth(groups: usize) -> usize {
3usize.saturating_mul(groups.saturating_add(1)).saturating_add(64)
}
/// Limits applied while reading a bundle from an untrusted source. The defaults are the ones
/// PhotoCraft loads with. [`check_manifest`], [`parse_manifest`] and [`crate::read_manifest`]
/// apply `max_manifest_bytes`, `max_depth` and `max_items`; the caller applies `max_blob_bytes`
/// and `max_total_bytes`.
///
/// The struct is `#[non_exhaustive]`: start from [`Limits::default`] and set the fields you
/// change.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct Limits {
/// Largest `manifest.json`, in bytes (256 MiB).
pub max_manifest_bytes: usize,
/// Largest decompressed blob, in bytes (1 GiB). The crate does not read blobs or apply this
/// limit: the caller that decompresses them enforces it.
pub max_blob_bytes: usize,
/// Most decompressed tile and blob bytes in one bundle (16 GiB). The crate does not read
/// tiles or blobs or apply this limit: the caller that decompresses them enforces it.
pub max_total_bytes: u64,
/// Deepest JSON nesting of the manifest (`manifest_depth(MAX_GROUP_DEPTH)`, 367). Values
/// above [`MAX_DEPTH`] are treated as [`MAX_DEPTH`].
pub max_depth: usize,
/// Most JSON items in the manifest ([`MAX_MANIFEST_ITEMS`]).
pub max_items: usize,
}
impl Default for Limits {
fn default() -> Self {
Limits {
max_manifest_bytes: 256 << 20,
max_blob_bytes: 1 << 30,
max_total_bytes: 16 << 30,
max_depth: manifest_depth(MAX_GROUP_DEPTH),
max_items: MAX_MANIFEST_ITEMS,
}
}
}
/// Checks manifest JSON against [`Limits::max_manifest_bytes`], [`Limits::max_depth`] (at most
/// [`MAX_DEPTH`]) and
/// [`Limits::max_items`] in one pass without recursion. Brackets and commas inside strings are
/// not counted. Returns [`Error::LimitExceeded`] for the first limit passed.
///
/// PhotoCraft's own check of a manifest it is about to save tests the depth and the items only.
/// This check also refuses JSON larger than [`Limits::max_manifest_bytes`], so a writer that runs
/// it before saving refuses a manifest of more than that size.
pub fn check_manifest(json: &[u8], limits: &Limits) -> Result<(), Error> {
if json.len() > limits.max_manifest_bytes {
return Err(Error::LimitExceeded(format!("`{}` exceeds {} bytes", crate::MANIFEST, limits.max_manifest_bytes)));
}
let max_depth = limits.max_depth.min(MAX_DEPTH);
let (mut depth, mut items, mut in_str, mut escaped) = (0usize, 0usize, false, false);
for &b in json {
if in_str {
match b {
_ if escaped => escaped = false,
b'\\' => escaped = true,
b'"' => in_str = false,
_ => {}
}
continue;
}
match b {
b'"' => in_str = true,
b'[' | b'{' => {
depth += 1;
if depth > max_depth {
return Err(Error::LimitExceeded(format!("manifest JSON nested deeper than {max_depth} levels")));
}
}
b']' | b'}' => depth = depth.saturating_sub(1),
_ => {}
}
if matches!(b, b'[' | b'{' | b',') {
items += 1;
if items > limits.max_items {
return Err(Error::LimitExceeded(format!("manifest JSON holds more than {} values", limits.max_items)));
}
}
}
Ok(())
}
/// Checks manifest JSON with [`check_manifest`], then parses it into a [`serde_json::Value`]
/// with serde_json's recursion limit turned off, so a manifest nested deeper than 128 levels
/// and within [`Limits::max_depth`] parses.
pub fn parse_manifest(json: &[u8], limits: &Limits) -> Result<serde_json::Value, Error> {
check_manifest(json, limits)?;
let mut de = serde_json::Deserializer::from_slice(json);
de.disable_recursion_limit();
let value = <serde_json::Value as serde::Deserialize>::deserialize(&mut de)?;
de.end()?;
Ok(value)
}
crates/pcraft-format/src/migrate.rs
+41
View File
@@ -0,0 +1,41 @@
use serde_json::Value;
use crate::Error;
/// One migration step: upgrades manifest JSON from version `n` to `n + 1` in place. The runner
/// sets `format_version` after each step.
pub type Step<E = Error> = fn(&mut Value) -> Result<(), E>;
/// Upgrades the manifest JSON `v` in place to version `target`, where `steps[i]` upgrades
/// version `i + 1` to `i + 2`. Returns the version the manifest had.
///
/// - A missing or non-integer `format_version`, one past `u32`, or `0` is [`Error::Corrupt`].
/// - A version newer than `target` is [`Error::TooNew`], holding the manifest's `generator` when
/// it is a string.
/// - A version with no step to the next one is [`Error::Unsupported`].
///
/// A step's error is returned as it is, and the steps before it have already changed `v`. A step
/// that leaves `v` something other than a JSON object is [`Error::Corrupt`].
pub fn migrate<E: From<Error>>(v: &mut Value, steps: &[Step<E>], target: u32) -> Result<u32, E> {
let found = v.get("format_version").and_then(Value::as_u64).ok_or_else(|| corrupt("manifest has no format_version"))?;
let found = u32::try_from(found).map_err(|_| corrupt("format_version out of range"))?;
if found == 0 {
return Err(corrupt("format_version 0 is invalid").into());
}
if found > target {
let generator = v.get("generator").and_then(Value::as_str).map(str::to_owned);
return Err(Error::TooNew { found, supported: target, generator }.into());
}
for ver in found..target {
let step =
usize::try_from(ver - 1).ok().and_then(|i| steps.get(i)).ok_or_else(|| Error::Unsupported(format!("no migration from format {ver}")))?;
step(v)?;
let object = v.as_object_mut().ok_or_else(|| corrupt("a migration step left the manifest without an object"))?;
object.insert("format_version".to_owned(), Value::from(ver + 1));
}
Ok(found)
}
fn corrupt(msg: &str) -> Error {
Error::Corrupt(msg.to_owned())
}
crates/pcraft-format/src/read.rs
+73
View File
@@ -0,0 +1,73 @@
use serde::Deserialize;
use serde::de::DeserializeOwned;
use serde_json::Value;
use crate::{COMPOSITE, Error, Limits, Manifest, Step, THUMB, migrate, parse_manifest};
/// Reads manifest JSON into a [`Manifest`] whose document is `D`.
///
/// 1. [`parse_manifest`] checks `json` against `limits` and parses it.
/// 2. [`migrate`] refuses a `format_version` newer than `target` with [`Error::TooNew`], which
/// holds the manifest's `generator`, and upgrades an older one with `steps`.
/// 3. The members are read in the order the parsed JSON object holds them, the document as `D`.
///
/// A missing `thumbnail` or `composite` reads as `None`, and members the envelope does not name
/// are skipped. A missing or invalid `format_version` is [`Error::Corrupt`], from [`migrate`].
/// Any other member of the wrong type is [`Error::Json`], and so is a missing `generator` or
/// `document`, which is reported after the members that are present, in the order serde's derived
/// `Deserialize` reports them. A `thumbnail` other than [`THUMB`] or a `composite` other than
/// [`COMPOSITE`] is [`Error::Corrupt`], checked after the members are read.
pub fn read_manifest<D: DeserializeOwned, E: From<Error>>(json: &[u8], limits: &Limits, steps: &[Step<E>], target: u32) -> Result<Manifest<D>, E> {
read_manifest_with(json, limits, steps, target, |document| serde_json::from_value(document).map_err(|e| E::from(Error::Json(e))))
}
/// [`read_manifest`], with the document read by `document` from its JSON value instead of by
/// `D`'s `Deserialize`, for example to decode a deep tree a level at a time.
pub fn read_manifest_with<D, E: From<Error>>(
json: &[u8],
limits: &Limits,
steps: &[Step<E>],
target: u32,
document: impl FnOnce(Value) -> Result<D, E>,
) -> Result<Manifest<D>, E> {
let mut read_document = Some(document);
let mut value = parse_manifest(json, limits)?;
migrate(&mut value, steps, target)?;
let Value::Object(members) = value else {
return Err(Error::Corrupt("the manifest is not a JSON object".to_owned()).into());
};
let (mut format_version, mut generator, mut doc, mut thumbnail, mut composite) = (None, None, None, None, None);
for (name, member) in members {
match name.as_str() {
"format_version" => format_version = Some(u32::deserialize(member).map_err(Error::Json)?),
"generator" => generator = Some(String::deserialize(member).map_err(Error::Json)?),
"document" => {
if let Some(read) = read_document.take() {
doc = Some(read(member)?);
}
}
"thumbnail" => thumbnail = Option::<String>::deserialize(member).map_err(Error::Json)?,
"composite" => composite = Option::<String>::deserialize(member).map_err(Error::Json)?,
_ => {}
}
}
let missing = |name| Error::Json(serde::de::Error::missing_field(name));
let manifest = Manifest {
format_version: format_version.ok_or_else(|| missing("format_version"))?,
generator: generator.ok_or_else(|| missing("generator"))?,
document: doc.ok_or_else(|| missing("document"))?,
thumbnail,
composite,
};
check_preview("thumbnail", manifest.thumbnail.as_deref(), THUMB)?;
check_preview("composite", manifest.composite.as_deref(), COMPOSITE)?;
Ok(manifest)
}
/// Refuses a preview member that names an entry other than `expected`.
fn check_preview(member: &str, entry: Option<&str>, expected: &str) -> Result<(), Error> {
match entry {
Some(entry) if entry != expected => Err(Error::Corrupt(format!("`{member}` names {entry:?}, not {expected:?}"))),
_ => Ok(()),
}
}
crates/pcraft-format/tests/limits.rs
+101
View File
@@ -0,0 +1,101 @@
use craft_pcraft_format::{Error, Limits, MAX_DEPTH, MAX_GROUP_DEPTH, MAX_MANIFEST_ITEMS, check_manifest, manifest_depth, parse_manifest};
fn nested(depth: usize) -> Vec<u8> {
let mut json = "[".repeat(depth).into_bytes();
json.extend(std::iter::repeat_n(b']', depth));
json
}
fn small(max_depth: usize, max_items: usize) -> Limits {
let mut limits = Limits::default();
limits.max_depth = max_depth;
limits.max_items = max_items;
limits
}
#[test]
fn defaults_are_photocraft_s_load_limits() {
let limits = Limits::default();
assert_eq!(limits.max_manifest_bytes, 256 << 20);
assert_eq!(limits.max_blob_bytes, 1 << 30);
assert_eq!(limits.max_total_bytes, 16 << 30);
assert_eq!(MAX_GROUP_DEPTH, 100);
assert_eq!(limits.max_depth, 367);
assert_eq!(limits.max_depth, manifest_depth(MAX_GROUP_DEPTH));
assert_eq!(limits.max_items, MAX_MANIFEST_ITEMS);
assert_eq!(MAX_MANIFEST_ITEMS, 4 << 20);
}
#[test]
fn manifest_depth_saturates() {
assert_eq!(manifest_depth(0), 67);
assert_eq!(manifest_depth(usize::MAX), usize::MAX);
}
#[test]
fn nesting_up_to_max_depth_passes_and_one_more_level_does_not() {
let limits = Limits::default();
let at = limits.max_depth;
assert!(check_manifest(&nested(at), &limits).is_ok());
assert!(matches!(check_manifest(&nested(at + 1), &limits), Err(Error::LimitExceeded(_))));
}
#[test]
fn a_manifest_deeper_than_serde_json_s_128_levels_parses() {
let value = parse_manifest(&nested(300), &Limits::default()).unwrap();
assert!(value.is_array());
}
#[test]
fn a_raised_max_depth_stops_at_max_depth_cap() {
let limits = small(100_000, MAX_MANIFEST_ITEMS);
assert!(parse_manifest(&nested(MAX_DEPTH), &limits).unwrap().is_array());
let deeper = parse_manifest(&nested(MAX_DEPTH + 1), &limits).err().map(|e| e.to_string());
assert_eq!(deeper, Some(format!("limit exceeded: manifest JSON nested deeper than {MAX_DEPTH} levels")));
assert!(matches!(parse_manifest(&nested(100_000), &limits), Err(Error::LimitExceeded(_))));
}
#[test]
fn items_up_to_max_items_pass_and_one_more_does_not() {
let limits = small(8, 4);
assert!(check_manifest(b"[1,2,3,4]", &limits).is_ok());
assert!(matches!(check_manifest(b"[1,2,3,4,5]", &limits), Err(Error::LimitExceeded(_))));
assert!(matches!(check_manifest(b"[[[[[]]]]]", &limits), Err(Error::LimitExceeded(_))));
}
#[test]
fn brackets_and_commas_inside_strings_are_not_counted() {
let limits = small(2, 3);
assert!(check_manifest(br#"{"a":"[[[{{{,,,,"}"#, &limits).is_ok());
assert!(check_manifest(br#"{"a":"\"[[[,,,\\"}"#, &limits).is_ok());
assert!(matches!(check_manifest(br#"{"a":"\\",[[[]]]}"#, &limits), Err(Error::LimitExceeded(_))));
}
#[test]
fn a_manifest_larger_than_max_manifest_bytes_is_refused() {
let mut limits = Limits::default();
limits.max_manifest_bytes = 8;
assert!(check_manifest(b"{\"a\":12}", &limits).is_ok());
assert!(matches!(check_manifest(b"{\"a\":123}", &limits), Err(Error::LimitExceeded(_))));
assert!(matches!(parse_manifest(b"{\"a\":123}", &limits), Err(Error::LimitExceeded(_))));
}
#[test]
fn invalid_or_trailing_json_is_a_json_error() {
let limits = Limits::default();
assert!(matches!(parse_manifest(b"{\"a\":", &limits), Err(Error::Json(_))));
assert!(matches!(parse_manifest(b"{} {}", &limits), Err(Error::Json(_))));
assert!(matches!(parse_manifest(b"", &limits), Err(Error::Json(_))));
}
#[test]
fn messages_read_as_photocraft_s() {
let none: &[craft_pcraft_format::Step] = &[];
let too_new = craft_pcraft_format::migrate(&mut serde_json::json!({"format_version": 2, "generator": "g"}), none, 1).err().map(|e| e.to_string());
assert_eq!(too_new.as_deref(), Some("bundle written by a newer version (format 2, this build reads up to 1)"));
assert_eq!(Error::Corrupt("x".into()).to_string(), "corrupt .pcraft bundle: x");
assert_eq!(Error::Unsupported("x".into()).to_string(), "unsupported .pcraft feature: x");
assert_eq!(Error::LimitExceeded("x".into()).to_string(), "limit exceeded: x");
let deep = check_manifest(&nested(368), &Limits::default()).err().map(|e| e.to_string());
assert_eq!(deep.as_deref(), Some("limit exceeded: manifest JSON nested deeper than 367 levels"));
}
crates/pcraft-format/tests/manifest.rs
+152
View File
@@ -0,0 +1,152 @@
use craft_pcraft_format::{
BLOBS_DIR, COMPOSITE, COMPOSITE_DIR, EXTENSION, FORMAT_VERSION, Limits, MANIFEST, Manifest, OBJECT_SUFFIX, SAVE_LOCK, Step, THUMB, TILES_DIR,
TileRef, blob_path, is_valid_hash, tile_path,
};
use serde::{Deserialize, Serialize};
fn read<D: serde::de::DeserializeOwned>(text: &str) -> Result<Manifest<D>, craft_pcraft_format::Error> {
let none: &[Step] = &[];
craft_pcraft_format::read_manifest(text.as_bytes(), &Limits::default(), none, FORMAT_VERSION)
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
struct Doc {
id: u64,
name: String,
tiles: Vec<TileRef>,
}
const HASH: &str = "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262";
fn doc() -> Doc {
Doc { id: 7, name: "Untitled-1".into(), tiles: vec![TileRef::new(0, -1, HASH.into())] }
}
fn manifest(previews: bool) -> Manifest<Doc> {
let mut manifest = Manifest::new("photocraft-format 0.1.0", doc());
manifest.thumbnail = previews.then(|| THUMB.to_owned());
manifest.composite = previews.then(|| COMPOSITE.to_owned());
manifest
}
const WITH_PREVIEWS: &str = r#"{
"format_version": 1,
"generator": "photocraft-format 0.1.0",
"document": {
"id": 7,
"name": "Untitled-1",
"tiles": [
{
"tx": 0,
"ty": -1,
"hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"
}
]
},
"thumbnail": "thumb.png",
"composite": "composite/preview.png"
}"#;
const WITHOUT_PREVIEWS: &str = r#"{
"format_version": 1,
"generator": "photocraft-format 0.1.0",
"document": {
"id": 7,
"name": "Untitled-1",
"tiles": [
{
"tx": 0,
"ty": -1,
"hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"
}
]
},
"thumbnail": null,
"composite": null
}"#;
#[test]
fn a_manifest_with_previews_writes_photocraft_s_pretty_json_byte_for_byte() {
let bytes = serde_json::to_vec_pretty(&manifest(true)).unwrap();
assert_eq!(String::from_utf8(bytes).unwrap(), WITH_PREVIEWS);
}
#[test]
fn a_manifest_without_previews_writes_null_members() {
let bytes = serde_json::to_vec_pretty(&manifest(false)).unwrap();
assert_eq!(String::from_utf8(bytes).unwrap(), WITHOUT_PREVIEWS);
}
#[test]
fn written_manifests_read_back_equal() {
for (text, previews) in [(WITH_PREVIEWS, true), (WITHOUT_PREVIEWS, false)] {
let read: Manifest<Doc> = read(text).unwrap();
assert_eq!(read, manifest(previews));
}
}
#[test]
fn a_new_manifest_is_at_the_current_version_without_previews() {
let m = Manifest::new(String::from("g"), 5);
assert_eq!((m.format_version, m.generator.as_str(), m.document, m.thumbnail, m.composite), (FORMAT_VERSION, "g", 5, None, None));
}
#[test]
fn missing_preview_members_read_as_none_and_unknown_members_are_skipped() {
let text =
r#"{"format_version":1,"generator":"g","document":{"id":1,"name":"a","tiles":[]},"full_composite":"composite/full.png","later":{"x":[1,2]}}"#;
let read: Manifest<Doc> = read(text).unwrap();
assert_eq!(read.thumbnail, None);
assert_eq!(read.composite, None);
assert_eq!(read.document.name, "a");
}
#[test]
fn a_manifest_without_a_document_or_generator_does_not_read() {
assert!(read::<Doc>(r#"{"format_version":1,"generator":"g"}"#).is_err());
assert!(read::<Doc>(r#"{"format_version":1,"document":{"id":1,"name":"a","tiles":[]}}"#).is_err());
}
#[test]
fn the_document_can_be_any_json_value() {
let read: Manifest<serde_json::Value> = read(WITH_PREVIEWS).unwrap();
assert_eq!(read.document["tiles"][0]["ty"], -1);
assert_eq!(read.composite.as_deref(), Some(COMPOSITE));
}
#[test]
fn entry_names_match_the_bundle_layout() {
assert_eq!(EXTENSION, "pcraft");
assert_eq!(MANIFEST, "manifest.json");
assert_eq!(THUMB, "thumb.png");
assert_eq!(COMPOSITE, "composite/preview.png");
assert_eq!(SAVE_LOCK, "save.lock");
assert_eq!((TILES_DIR, BLOBS_DIR, COMPOSITE_DIR, OBJECT_SUFFIX), ("tiles", "blobs", "composite", ".zst"));
assert!(COMPOSITE.starts_with(&format!("{COMPOSITE_DIR}/")));
assert_eq!(tile_path(HASH), Some(format!("tiles/{HASH}.zst")));
assert_eq!(blob_path(HASH), Some(format!("blobs/{HASH}.zst")));
}
#[test]
fn no_entry_name_is_built_from_an_invalid_hash() {
for hash in ["../../etc/passwd", "../x", "", &HASH.to_uppercase(), &HASH[1..]] {
assert_eq!(tile_path(hash), None, "{hash}");
assert_eq!(blob_path(hash), None, "{hash}");
}
let tile: TileRef = serde_json::from_str(r#"{"tx":0,"ty":0,"hash":"../x"}"#).unwrap();
assert_eq!(tile.path(), None);
let tile = TileRef::new(1, -1, HASH.into());
assert_eq!(tile.path(), Some(format!("tiles/{HASH}.zst")));
}
#[test]
fn only_64_lowercase_hex_digits_are_a_valid_hash() {
assert!(is_valid_hash(HASH));
assert!(is_valid_hash(&"0".repeat(64)));
assert!(!is_valid_hash(&HASH.to_uppercase()));
assert!(!is_valid_hash(&HASH[1..]));
assert!(!is_valid_hash(&format!("{HASH}0")));
assert!(!is_valid_hash(&format!("../{}", &HASH[3..])));
assert!(!is_valid_hash(&format!("{}g", &HASH[1..])));
assert!(!is_valid_hash(""));
}
crates/pcraft-format/tests/migrate.rs
+114
View File
@@ -0,0 +1,114 @@
use craft_pcraft_format::{Error, FORMAT_VERSION, Step, migrate};
use serde_json::{Value, json};
fn rename_title(v: &mut Value) -> Result<(), Error> {
let document = v.get_mut("document").and_then(Value::as_object_mut).ok_or(Error::Corrupt("no document".into()))?;
let title = document.remove("title").unwrap_or_default();
document.insert("name".into(), title);
Ok(())
}
fn add_flag(v: &mut Value) -> Result<(), Error> {
v["document"]["flag"] = json!(true);
Ok(())
}
fn replace_with_array(v: &mut Value) -> Result<(), Error> {
*v = json!([]);
Ok(())
}
const STEPS: &[Step] = &[rename_title, add_flag];
#[test]
fn steps_run_in_order_from_the_file_s_version() {
let mut v = json!({"format_version": 1, "document": {"title": "old"}});
assert_eq!(migrate(&mut v, STEPS, 3).unwrap(), 1);
assert_eq!(v, json!({"format_version": 3, "document": {"name": "old", "flag": true}}));
}
#[test]
fn a_file_at_a_later_version_runs_only_the_later_steps() {
let mut v = json!({"format_version": 2, "document": {"title": "kept"}});
assert_eq!(migrate(&mut v, STEPS, 3).unwrap(), 2);
assert_eq!(v, json!({"format_version": 3, "document": {"title": "kept", "flag": true}}));
}
#[test]
fn the_current_version_is_left_unchanged() {
let mut v = json!({"format_version": FORMAT_VERSION, "document": {}});
let before = v.clone();
let none: &[Step] = &[];
assert_eq!(migrate(&mut v, none, FORMAT_VERSION).unwrap(), FORMAT_VERSION);
assert_eq!(v, before);
}
#[test]
fn a_newer_version_is_too_new() {
let mut v = json!({"format_version": 4});
assert!(matches!(migrate(&mut v, STEPS, 3), Err(Error::TooNew { found: 4, supported: 3, generator: None, .. })));
let mut v = json!({"format_version": 4, "generator": "photocraft-format 2.0.0"});
assert!(matches!(migrate(&mut v, STEPS, 3), Err(Error::TooNew { generator: Some(g), .. }) if g == "photocraft-format 2.0.0"));
}
#[test]
fn a_missing_zero_or_out_of_range_version_is_corrupt() {
for v in [
json!({}),
json!({"format_version": 0}),
json!({"format_version": "1"}),
json!({"format_version": -1}),
json!({"format_version": 1.5}),
json!({"format_version": 4_294_967_296_u64}),
json!([1]),
json!(null),
] {
let mut v = v;
assert!(matches!(migrate(&mut v, STEPS, 3), Err(Error::Corrupt(_))), "{v}");
}
}
#[test]
fn a_version_with_no_step_is_unsupported() {
let mut v = json!({"format_version": 1, "document": {"title": "t"}});
let one: &[Step] = &[rename_title];
assert!(matches!(migrate(&mut v, one, 3), Err(Error::Unsupported(m)) if m == "no migration from format 2"));
assert_eq!(v["format_version"], 2);
}
#[test]
fn a_step_that_leaves_no_object_is_corrupt() {
let mut v = json!({"format_version": 1});
let steps: &[Step] = &[replace_with_array];
assert!(matches!(migrate(&mut v, steps, 2), Err(Error::Corrupt(_))));
}
#[derive(Debug)]
enum AppError {
Format(Error),
Step(&'static str),
}
impl From<Error> for AppError {
fn from(e: Error) -> Self {
AppError::Format(e)
}
}
fn failing(_: &mut Value) -> Result<(), AppError> {
Err(AppError::Step("refused"))
}
fn app_add_flag(v: &mut Value) -> Result<(), AppError> {
add_flag(v).map_err(AppError::from)
}
#[test]
fn steps_return_the_caller_s_error_type() {
let steps: &[Step<AppError>] = &[app_add_flag, failing];
let mut v = json!({"format_version": 1, "document": {}});
assert!(matches!(migrate(&mut v, steps, 3), Err(AppError::Step("refused"))));
assert_eq!(v, json!({"format_version": 2, "document": {"flag": true}}));
let mut newer = json!({"format_version": 9});
assert!(matches!(migrate(&mut newer, steps, 3), Err(AppError::Format(Error::TooNew { found: 9, supported: 3, .. }))));
}
crates/pcraft-format/tests/read.rs
+191
View File
@@ -0,0 +1,191 @@
use craft_pcraft_format::{COMPOSITE, Error, FORMAT_VERSION, Limits, Manifest, Step, THUMB, TileRef, read_manifest, read_manifest_with};
use serde::Deserialize;
use serde_json::{Value, json};
#[derive(Debug, Clone, PartialEq, Deserialize)]
struct Doc {
name: String,
#[serde(default)]
tiles: Vec<TileRef>,
}
const NONE: &[Step] = &[];
fn read(json: &[u8]) -> Result<Manifest<Doc>, Error> {
read_manifest(json, &Limits::default(), NONE, FORMAT_VERSION)
}
fn nested_document(depth: usize) -> Vec<u8> {
let mut json = br#"{"format_version":1,"generator":"g","document":{"name":"a","deep":"#.to_vec();
json.extend("[".repeat(depth).bytes());
json.extend("]".repeat(depth).bytes());
json.extend(b"}}");
json
}
#[test]
fn a_manifest_reads_with_its_previews_and_document() {
let json = br#"{"format_version":1,"generator":"photocraft-format 0.1.0","document":{"name":"a","tiles":[{"tx":0,"ty":-1,"hash":"x"}]},"thumbnail":"thumb.png","composite":null,"later":[1]}"#;
let m = read(json).unwrap();
assert_eq!(m.format_version, 1);
assert_eq!(m.generator, "photocraft-format 0.1.0");
assert_eq!(m.document.name, "a");
assert_eq!(m.document.tiles, vec![TileRef::new(0, -1, "x".into())]);
assert_eq!(m.thumbnail.as_deref(), Some("thumb.png"));
assert_eq!(m.composite, None);
}
#[test]
fn a_newer_manifest_is_too_new_and_names_its_writer() {
let json = br#"{"format_version":2,"generator":"photocraft-format 9.0.0","document":{"name":"a"}}"#;
let Err(Error::TooNew { found, supported, generator, .. }) = read(json) else { panic!("not TooNew") };
assert_eq!((found, supported), (2, 1));
assert_eq!(generator.as_deref(), Some("photocraft-format 9.0.0"));
let message = read(json).err().map(|e| e.to_string());
assert_eq!(message.as_deref(), Some("bundle written by a newer version (format 2, this build reads up to 1)"));
}
#[test]
fn a_newer_manifest_without_a_string_generator_is_too_new_without_one() {
for json in [&br#"{"format_version":3,"document":{}}"#[..], br#"{"format_version":3,"generator":7}"#] {
assert!(matches!(read(json), Err(Error::TooNew { found: 3, generator: None, .. })));
}
}
#[test]
fn a_manifest_past_the_limits_is_refused_before_it_is_read() {
let limits = Limits::default();
let deep = nested_document(limits.max_depth);
assert!(matches!(read(&deep), Err(Error::LimitExceeded(m)) if m == "manifest JSON nested deeper than 367 levels"));
let within = nested_document(limits.max_depth - 2);
let m: Manifest<Value> = read_manifest(&within, &limits, NONE, FORMAT_VERSION).unwrap();
assert_eq!(m.document["name"], "a");
let mut small = Limits::default();
small.max_manifest_bytes = 10;
let json = br#"{"format_version":1,"generator":"g","document":{"name":"a"}}"#;
let large: Result<Manifest<Doc>, Error> = read_manifest(json, &small, NONE, FORMAT_VERSION);
assert!(matches!(large, Err(Error::LimitExceeded(m)) if m == "`manifest.json` exceeds 10 bytes"));
let mut few = Limits::default();
few.max_items = 2;
let many: Result<Manifest<Doc>, Error> = read_manifest(br#"{"format_version":1,"generator":"g","document":{"name":"a"}}"#, &few, NONE, 1);
assert!(matches!(many, Err(Error::LimitExceeded(_))));
}
#[test]
fn the_steps_run_before_the_document_is_read() {
fn rename(v: &mut Value) -> Result<(), Error> {
v["document"]["name"] = v["document"]["title"].take();
Ok(())
}
let steps: &[Step] = &[rename];
let json = br#"{"format_version":1,"generator":"g","document":{"title":"old"}}"#;
let m: Manifest<Doc> = read_manifest(json, &Limits::default(), steps, 2).unwrap();
assert_eq!(m.format_version, 2);
assert_eq!(m.document.name, "old");
}
#[test]
fn a_missing_or_invalid_envelope_member_is_a_json_error() {
for (json, message) in [
(&br#"{"format_version":1,"generator":"g"}"#[..], "missing field `document`"),
(br#"{"format_version":1,"document":{"name":"a"}}"#, "missing field `generator`"),
(br#"{"format_version":1,"generator":"g","document":{"name":"a"},"thumbnail":5}"#, "invalid type: integer `5`, expected a string"),
(br#"{"format_version":1,"generator":"g","document":{"title":"a"}}"#, "missing field `name`"),
] {
let e = read(json).err();
assert!(matches!(&e, Some(Error::Json(_))), "{e:?}");
assert_eq!(e.map(|e| e.to_string()).as_deref(), Some(format!("manifest JSON: {message}").as_str()));
}
}
/// The manifest as a struct with a derived `Deserialize`, for comparing errors.
#[derive(Debug, Deserialize)]
#[allow(dead_code)]
struct Derived {
format_version: u32,
generator: String,
document: Doc,
#[serde(default)]
thumbnail: Option<String>,
#[serde(default)]
composite: Option<String>,
}
#[test]
fn a_manifest_with_several_faults_reports_the_one_a_derived_struct_reports() {
for json in [
json!({"format_version": 1, "composite": 1, "document": {"title": "a"}}),
json!({"format_version": 1, "generator": 2, "document": {"title": "a"}}),
json!({"format_version": 1, "document": {"title": "a"}, "thumbnail": 3}),
json!({"format_version": 1, "document": {"name": "a"}, "thumbnail": 3}),
json!({"format_version": 1, "thumbnail": 3}),
json!({"format_version": 1}),
json!({"format_version": 1, "generator": "g"}),
json!({"format_version": 1, "composite": [], "generator": {}}),
] {
let want = serde_json::from_value::<Derived>(json.clone()).err().map(|e| format!("manifest JSON: {e}"));
let got = read(json.to_string().as_bytes()).err().map(|e| e.to_string());
assert!(want.is_some());
assert_eq!(got, want, "{json}");
}
}
#[test]
fn a_missing_or_invalid_version_is_corrupt() {
assert!(matches!(read(br#"{"generator":"g","document":{"name":"a"}}"#), Err(Error::Corrupt(_))));
assert!(matches!(read(br#"{"format_version":"1","generator":"g","document":{"name":"a"}}"#), Err(Error::Corrupt(_))));
assert!(matches!(read(br#"{"format_version":1.5,"generator":"g","document":{"name":"a"}}"#), Err(Error::Corrupt(_))));
assert!(matches!(read(b"[1]"), Err(Error::Corrupt(_))));
}
#[test]
fn a_preview_other_than_its_entry_is_corrupt() {
for (member, entry) in [
("thumbnail", "../x"),
("thumbnail", "/etc/passwd"),
("composite", "../x"),
("composite", "C:\\x.png"),
("thumbnail", "composite/preview.png"),
("composite", "thumb.png"),
] {
let json = json!({"format_version": 1, "generator": "g", "document": {"name": "a"}, member: entry});
let e = read(json.to_string().as_bytes()).err();
assert!(matches!(&e, Some(Error::Corrupt(_))), "{json}: {e:?}");
}
let json =
json!({"format_version": 1, "generator": "g", "document": {"name": "a"}, "thumbnail": "thumb.png", "composite": "composite/preview.png"});
let m = read(json.to_string().as_bytes()).unwrap();
assert_eq!((m.thumbnail.as_deref(), m.composite.as_deref()), (Some(THUMB), Some(COMPOSITE)));
}
#[derive(Debug)]
enum AppError {
Format(Error),
Document(String),
}
impl From<Error> for AppError {
fn from(e: Error) -> Self {
AppError::Format(e)
}
}
#[test]
fn read_manifest_with_reads_the_document_with_the_caller_s_function() {
let json = br#"{"format_version":1,"generator":"g","document":{"name":"a","layers":[1,2,3]}}"#;
let none: &[Step<AppError>] = &[];
let m = read_manifest_with(json, &Limits::default(), none, 1, |mut document: Value| {
let layers = document.get_mut("layers").map(Value::take).unwrap_or_default();
Ok::<_, AppError>((document, layers))
})
.unwrap();
assert_eq!(m.document.0, json!({"name": "a", "layers": null}));
assert_eq!(m.document.1, json!([1, 2, 3]));
let refused = read_manifest_with(json, &Limits::default(), none, 1, |_| Err::<(), _>(AppError::Document("no".into())));
assert!(matches!(refused, Err(AppError::Document(m)) if m == "no"));
let newer = read_manifest_with(br#"{"format_version":2}"#, &Limits::default(), none, 1, |_| Ok::<(), AppError>(()));
assert!(matches!(newer, Err(AppError::Format(Error::TooNew { found: 2, .. }))));
}
crates/proof/README.md
+11 -9
View File
@@ -13,7 +13,7 @@ craft-proof = { git = "https://github.com/storytold/craft-libs", rev = "<commit>
| Language | Spelling | Grammar | Hyphenation |
|---|---|---|---|
| English (en-US, en-CA, en-GB and the British-spelling Englishes such as en-AU, en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic |
| English (en-US, en-CA, en-AU, en-GB and the British-spelling Englishes such as en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic |
| Polish, Portuguese, Italian | the installed Hunspell dictionary | yes | TeX patterns (embedded, or a Language pack) |
| Croatian, Turkish, Russian, Czech, Indonesian, Spanish (es, es-MX, es-419, es-UY) | the installed Hunspell dictionary (Russian also `ru-extra.txt`) | yes | Turkish and Russian rules; English fallback otherwise |
| German | — | — | TeX patterns (embedded, or a Language pack) |
@@ -79,7 +79,7 @@ pub fn packs_dir() -> Option<PathBuf>; // ＋
pub fn installed() -> Vec<InstalledPack>; // InstalledPack { locale, name, licence, path }, by locale
pub fn licence_texts() -> Vec<(String, String)>; // (title, text): embedded English notices, then each pack's LICENCES/*
pub fn load_errors() -> Vec<(String, String)>; // ＋ (locale or folder, reason): packs skipped, pattern files that failed
pub fn embedded_locales() -> &'static [&'static str]; // ＋ what this build embeds ("en-us", "de", "pl", "pt", "it" by default; "en-us" with CRAFT_PROOF_EMBED=en)
pub fn embedded_locales() -> &'static [&'static str]; // ＋ locales with embedded upstream data, not our supplements ("en-us", "en-gb", "en-ca", "en-au", "pl", "pt", "it", "de" by default; the four English ones with CRAFT_PROOF_EMBED=en)
// craft_proof (crate root)
pub enum ProofLang { …, Pack(PackLocale), Other } // ＋ Pack: any language without tools of its own (nl, fr, sr-Latn…)
@@ -151,14 +151,16 @@ English spelling uses the first of these that is there; `english_source(lang)` s
or in WordCraft's own `dictionaries` folder. The user's choice comes first.
2. **Installed English pack:** reserved for Language packs (craft-libs#485, CP4.2).
3. **Embedded SCOWL:** the English's own SCOWL `rel-2026.02.25` Hunspell dictionary from
`CRAFT_I18N_DIR` (`language-data/en-us`, `en-gb` (British `-ise`) or `en-ca`, `spelling.aff` +
`spelling.dic`), read by the affix engine. en-AU, en-NZ, en-IE and the other British-spelling
Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own
`CRAFT_I18N_DIR` (`language-data/en-us`, `en-gb` (British `-ise`), `en-au` or `en-ca`,
`spelling.aff` + `spelling.dic`), read by the affix engine. Australian English reads en-AU
(Australian words such as `billabong`, with en-GB's spelling policy); en-NZ, en-IE and the
other British-spelling Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own
dictionary knows it (or `en-extra.txt` does), never because another English's does
(`colourblind` is wrong in en-US, `colorblind` in en-GB); suggestions come from the same
dictionary. SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so
(`colourblind` is wrong in en-US, `colorblind` in en-GB); the one exception is Australian
English, which also takes en-GB's words (`enquiry`: en-AU's list keeps `inquiry` only).
Suggestions come from the English's own dictionary (en-AU's for Australian). SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so
British English also takes the `-ize` spellings of its `-ise` words that American or Canadian
English spell so (`organize`, `realization`; never `wize` or `advize`), and `en-variants.txt` flags the other dialect's spellings and accepts this
English spell so (`organize`, `realization`; never `wize` or `advize`, and never `-yze`: `reanalyze` stays American), and `en-variants.txt` flags the other dialect's spellings and accepts this
dialect's other spellings of known words (`judgement` in en-US). English with no dialect
(`is_correct`) takes a word any of the three knows. Suggestions offer a missing space only
between two lower-case words of three letters or more, after every one-word candidate, and no
@@ -199,7 +201,7 @@ craft-i18n's copy.
The SCOWL notice (SCOWL's licence, the Ispell BSD licence of the affix files and Benjamin Titze's
notice) is embedded with the dictionaries: `craft_proof::scowl::notice()`, for an About window.
Embedding SCOWL adds 1,542,396 bytes of Language data (the en-US, en-GB and en-CA `.aff`/`.dic`
Embedding SCOWL adds 2,056,533 bytes of Language data (the en-US, en-GB, en-CA and en-AU `.aff`/`.dic`
and the notice); the Moby list (432,905 bytes) stays for hyphenation.
## Check
crates/proof/build.rs
+4 -3
View File
@@ -16,9 +16,8 @@
//! `include_str!` takes a string, so a `CRAFT_I18N_DIR` that isn't valid UTF-8 can't be embedded:
//! it is treated like a folder without the data.
//!
//! English: SCOWL's en-US, en-GB (-ise) and en-CA dictionaries; en-AU is not embedded, as
//! Australian English is checked as British-spelling English (`ProofLang::EnGb`). The SCOWL
//! notice is embedded once (the four upstream READMEs differ only in their title and build line).
//! English: SCOWL's en-US, en-GB (-ise), en-CA and en-AU dictionaries. The SCOWL notice is
//! embedded once (the four upstream READMEs differ only in their title and build line).
use std::fmt::Write as _;
use std::path::{Path, PathBuf};
@@ -34,6 +33,8 @@ const FILES: &[(&str, &str, bool)] = &[
("EN_GB_SPELLING", "en-gb/spelling.dic", false),
("EN_CA_AFF", "en-ca/spelling.aff", false),
("EN_CA_SPELLING", "en-ca/spelling.dic", false),
("EN_AU_AFF", "en-au/spelling.aff", false),
("EN_AU_SPELLING", "en-au/spelling.dic", false),
("SCOWL_NOTICE", "en-us/LICENCE-SCOWL.txt", false),
("EN_US_PAT", "en-us/hyphenation.pat", false),
("EXTRA_EN", "en-us/en-extra.txt", false),
crates/proof/src/affix.rs
+86 -17
View File
@@ -79,6 +79,14 @@ const MAX_ALIAS_BYTES: usize = 4 << 20;
/// Most flags all distinct flag sets (stems' and continuation classes') may hold together: a lookup
/// tests each distinct continuation set it meets at most once, so this bounds its cost.
const MAX_SET_DATA: usize = 1 << 23;
/// Most flags all stems may carry together before their sets are merged and interned (the same
/// set on many stems counts each time; an `AF` alias counts once). Checked while reading, so a
/// huge `.dic` is refused before its flags fill memory, not after (#495: a 760 MB `.dic` once
/// held 3 GB before [`MAX_SET_DATA`] refused it).
const MAX_STEM_FLAGS: usize = 4 * MAX_SET_DATA;
/// Most bytes of flag text per flag of [`MAX_STEM_FLAGS`] the stem lines may hold in all while
/// they are read (`FLAG num`: `65535,` is six).
const FLAG_TEXT_PER_FLAG: usize = 6;
/// Most distinct flags a dictionary may use: each gets a `u16` id.
const MAX_DISTINCT_FLAGS: usize = 1 << 16;
@@ -1184,12 +1192,19 @@ impl AffixDict {
}
fn from_hunspell_in(aff: &str, dic: &str, comments: &[&str], english: bool) -> Result<AffixDict, String> {
AffixDict::from_hunspell_capped(aff, dic, comments, english, MAX_STEM_FLAGS)
}
/// [`AffixDict::from_hunspell_in`] with at most `max_flags` stem flags in all
/// ([`MAX_STEM_FLAGS`]; tests lower it).
fn from_hunspell_capped(aff: &str, dic: &str, comments: &[&str], english: bool, max_flags: usize) -> Result<AffixDict, String> {
// Some dictionaries open with a UTF-8 BOM; it must not glue itself to the first
// directive or turn the `.dic` count line into a stem.
let aff = aff.strip_prefix('\u{feff}').unwrap_or(aff);
let dic = dic.strip_prefix('\u{feff}').unwrap_or(dic);
let rules = parse_aff(aff, english)?;
let mut stems: Vec<(String, String)> = Vec::new();
let mut flag_text = FlagText::new(max_flags);
for (n, line) in dic.lines().enumerate() {
// The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it.
// Leading blanks belong to it, as in Hunspell (` lab` is no `lab`).
@@ -1225,14 +1240,16 @@ impl AffixDict {
}
// With an `AF` table the field stays an alias number here: build() reads each alias
// once, and every stem using it shares those flags.
flag_text.add(&flags)?;
stems.push((word, flags));
}
AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect(), MAX_FLAGS_PER_ENTRY)
AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect(), MAX_FLAGS_PER_ENTRY, max_flags)
}
/// Build from parsed rules and stems whose flag fields carry at most `entry_cap` flags each
/// ([`MAX_FLAGS_PER_ENTRY`] for `.dic` entries, [`MAX_FLAGS_PER_STEM`] for compact data).
fn build(rules: Rules, stems: Vec<(String, String)>, comments: Vec<String>, entry_cap: usize) -> Result<AffixDict, String> {
/// The stems' flags may number `max_flags` in all ([`MAX_STEM_FLAGS`]), counted as they are read.
fn build(rules: Rules, stems: Vec<(String, String)>, comments: Vec<String>, entry_cap: usize, max_flags: usize) -> Result<AffixDict, String> {
let mode = rules.mode;
// A `.dic` entry's malformed flag items read as Hunspell reads them ([`FlagMode`]); an
// entry with more than MAX_FLAGS_PER_STEM flags, or compact data whose flags don't read,
@@ -1261,20 +1278,24 @@ impl AffixDict {
}
Some((start, stem_keys.len(), word))
};
let mut stems: Vec<(String, usize, usize, bool)> = stems
.into_iter()
.filter_map(|(w, f)| {
let range = match rules.alias_index(&f) {
Some(i) => *alias_ranges.entry(i).or_insert_with(|| read(rules.aliases.get(i).map_or("", String::as_str), &mut stem_keys)),
None if rules.aliases_on => Some((0, 0, true)),
None => read(&f, &mut stem_keys),
};
if range.is_none_or(|r| !r.2) {
skipped += 1;
}
range.map(|(a, b, word)| (w, a, b, word))
})
.collect();
let mut read_stems: Vec<(String, usize, usize, bool)> = Vec::with_capacity(stems.len());
for (w, f) in stems {
let range = match rules.alias_index(&f) {
Some(i) => *alias_ranges.entry(i).or_insert_with(|| read(rules.aliases.get(i).map_or("", String::as_str), &mut stem_keys)),
None if rules.aliases_on => Some((0, 0, true)),
None => read(&f, &mut stem_keys),
};
if stem_keys.len() > max_flags {
return Err(format!("the stems carry more than {max_flags} flags in all"));
}
if range.is_none_or(|r| !r.2) {
skipped += 1;
}
if let Some((a, b, word)) = range {
read_stems.push((w, a, b, word));
}
}
let mut stems = read_stems;
if skipped > 0 {
log::debug!("affix dictionary: {skipped} entries with unreadable or too many flags skipped");
}
@@ -2664,6 +2685,7 @@ impl AffixDict {
}
let body = raw.get(words_at + 7..).unwrap_or_default();
let mut stems: Vec<(String, String)> = Vec::new();
let mut flag_text = FlagText::new(MAX_STEM_FLAGS);
let mut cur: Vec<u8> = Vec::new();
let mut i = 0;
while let Some(&p) = body.get(i) {
@@ -2686,9 +2708,32 @@ impl AffixDict {
if word.is_empty() || word.len() > MAX_WORD_BYTES {
return Err("affix dictionary: bad stem".into());
}
flag_text.add(flags).map_err(|e| format!("affix dictionary: {e}"))?;
stems.push((word.to_string(), flags.to_string()));
}
AffixDict::build(rules, stems, comments, MAX_FLAGS_PER_STEM)
AffixDict::build(rules, stems, comments, MAX_FLAGS_PER_STEM, MAX_STEM_FLAGS)
}
}
/// A running count of the stems' flag text while a dictionary is read: past
/// [`FLAG_TEXT_PER_FLAG`] bytes for each of `max_flags` flags, reading stops with an error, before
/// the rest of a huge `.dic` is held ([`MAX_STEM_FLAGS`]).
struct FlagText {
bytes: usize,
cap: usize,
}
impl FlagText {
fn new(max_flags: usize) -> Self {
FlagText { bytes: 0, cap: max_flags.saturating_mul(FLAG_TEXT_PER_FLAG) }
}
fn add(&mut self, flags: &str) -> Result<(), String> {
self.bytes = self.bytes.saturating_add(flags.len());
if self.bytes > self.cap {
return Err(format!("the stems' flags take more than {} bytes", self.cap));
}
Ok(())
}
}
@@ -2738,6 +2783,30 @@ pub(crate) mod tests {
AffixDict::from_hunspell(AFF, DIC, &["test data"]).unwrap()
}
/// #495 review: the stems' flags are counted as they are read, so a `.dic` whose flags pass
/// the budget is refused while reading (its flag text) or while its flags are read (the
/// keys), before they are all held; one within it loads.
#[test]
fn stem_flags_are_budgeted_while_reading() {
let aff = "SET UTF-8\nSFX A Y 1\nSFX A 0 s .\nSFX B Y 1\nSFX B 0 x .\n";
let many: String = (0..100).map(|i| format!("w{i}/AB\n")).collect();
let dic = format!("100\n{many}");
// 200 flags: over a budget of 50 keys (whose flag text, 300 bytes, still fits).
let e = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 50).unwrap_err();
assert!(e.contains("more than 50 flags"), "{e}");
let d = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 200).unwrap();
assert!(d.check("w7s") && d.check("w99x"));
// Flag text past the budget (entries over the per-entry cap hold no keys, but their text
// still counts): refused while reading the `.dic`.
let long = "A".repeat(MAX_FLAGS_PER_ENTRY + 1);
let dic = format!("3\na/{long}\nb/{long}\nc/{long}\n");
let e = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 2000).unwrap_err();
assert!(e.contains("more than 12000 bytes"), "{e}");
assert!(AffixDict::from_hunspell(aff, &dic, &[]).is_ok(), "within the real budget: skipped entries, no error");
let mut text = FlagText::new(1);
assert!(text.add("ABCDEF").is_ok() && text.add("G").is_err());
}
#[test]
fn affix_headers_allow_comments_but_rule_conditions_stay_literal() {
let aff = "SET UTF-8\nPFX X Y 1 # prefix comment\nPFX X 0 ne . # rule comment\nSFX y Y 1 # suffix comment\nSFX y 0 y . # rule comment\n";
crates/proof/src/cs/grammar.rs
+2 -5
View File
@@ -62,11 +62,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Mezera před interpunkcí".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Dvojitá mezera".into(), vec![" ".into()]));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Dvojitá mezera".into(), vec![" ".into()]));
}
}
crates/proof/src/cs/mod.rs
+10 -1
View File
@@ -63,7 +63,8 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
return true;
};
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
// The whole word first, as Hunspell does: a hyphenated entry or user word.
return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
}
known(d, w)
}
@@ -142,6 +143,14 @@ mod tests {
}
}
/// craft-libs#496 review: a hyphenated dictionary entry is a word even when its parts aren't.
#[test]
fn a_hyphenated_entry_is_looked_up_whole() {
let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap();
assert!(is_correct_with(&d, "tic-tac"));
assert!(!is_correct_with(&d, "tic-tacc"));
}
#[test]
fn missing_dictionary_and_ignored_inputs_are_quiet() {
let empty = AffixDict::default();
crates/proof/src/cs/suggest.rs
+9
View File
@@ -75,4 +75,13 @@ mod tests {
assert!(!suggest("město", 4).contains(&"město".into()));
assert!(!suggest("zqzakázane", 8).contains(&"zqzakázané".into()));
}
#[test]
fn a_letter_that_lowercases_to_two_chars_is_still_suggested_for() {
// wordcraft#4683: "İx" lowercases to three chars ("i", U+0307, "x") but has two typed
// letters; the one-letter edits stay in range and dropping the dot still finds "ix".
let d = AffixDict::from_hunspell("", "1\nix\n", &[]).unwrap();
let suggestions = suggest_in(&d, "İx", 4);
assert!(suggestions.contains(&"Ix".into()), "{suggestions:?}");
}
}
crates/proof/src/en_variants.rs
+3 -2
View File
@@ -1,5 +1,5 @@
//! American and British spellings of the same English word (#1281, #1045): which forms each
//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`]) accepts, which it flags,
//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`] and [`ProofLang::EnAu`]) accepts, which it flags,
//! and what it suggests instead (`colour` → `color` in American English).
//!
//! The pairs come from craft-i18n's `language-data/en-us/en-variants.txt`, our own list of base forms with a
@@ -77,7 +77,8 @@ fn column(lang: ProofLang) -> Option<usize> {
match lang {
ProofLang::En => Some(0),
ProofLang::EnCa => Some(1),
ProofLang::EnGb => Some(2),
// Australian English follows the British policy (`colour`, `centre`, `organise`).
ProofLang::EnGb | ProofLang::EnAu => Some(2),
_ => None,
}
}
crates/proof/src/english.rs
+113 -22
View File
@@ -4,10 +4,11 @@
//! user keeps for WordCraft ([`crate::hunspell::user_dirs`]: the folder set in the options,
//! `WORDCRAFT_DICTIONARIES`, WordCraft's own `dictionaries` folder).
//! 2. **Installed English pack**: reserved for Language packs (craft-libs#485, CP4.2).
//! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB or en-CA) embedded
//! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB, en-AU or en-CA) embedded
//! from craft-i18n at build time ([`crate::scowl`]); a word it knows (British English: also
//! the `-ize` spelling of one it knows, [`ise_spelling`]), or the supplement ([`EXTRA`]), is
//! right; never one only another English knows. When it is there, the system locations aren't
//! right; never one only another English knows, except that Australian English also takes
//! en-GB's words (its suggestions still come from en-AU). When it is there, the system locations aren't
//! searched.
//! 4. **System Hunspell**: a pair for the run's own English in the system locations (none on the
//! web). `COMPOUNDRULE` (ordinals) and `OCONV` are read; flag-based compounding is ignored
@@ -18,8 +19,9 @@
//! embedded from craft-i18n is the same list; it is used for hyphenation only.)
//!
//! Hunspell dictionaries are found by [`crate::hunspell`] by their own English's names only
//! (`en_US` for American, `en_GB` for British, `en_CA` for Canadian English), each name in every
//! folder before the next name. Another English's dictionary is never used: it would mark this
//! (`en_US` for American, `en_GB` for British, `en_CA` for Canadian English; `en_AU`, then
//! `en_GB`, for Australian), each name in every folder before the next name. Another English's
//! dictionary is never used: it would mark this
//! English's spellings (`colour` in an `en_US` one). A file that can't be read is skipped for the
//! next name or folder, and one file is read once for every English that uses it. `COMPOUNDRULE`
//! (ordinals) and `OCONV` (suggestions are written as the dictionary says: `don’t`) are read;
@@ -90,15 +92,47 @@ fn names(lang: ProofLang) -> &'static [&'static str] {
match lang {
ProofLang::EnGb => &["en_GB", "en-GB"],
ProofLang::EnCa => &["en_CA", "en-CA"],
// Australian English's own, else a British one (a system en_GB on a build without SCOWL):
// [`exact_candidates`] reads every en_AU source before any en_GB one.
ProofLang::EnAu => &["en_AU", "en-AU", "en_GB", "en-GB"],
_ => &["en_US", "en-US"],
}
}
/// The Hunspell pairs of `lang` without SCOWL, from `find` (a folder search for some names,
/// [`crate::hunspell::candidates_exact`]): its own names first, and only then Australian English's
/// British fallback, so every en_AU source (a dictionaries folder, a pack, the system) comes before
/// any en_GB one.
fn exact_candidates(lang: ProofLang, find: impl Fn(&[&str]) -> Vec<(PathBuf, PathBuf)>) -> Vec<(PathBuf, PathBuf)> {
let mut out = find(own_names(lang));
if lang == ProofLang::EnAu {
for c in find(&names(lang)[own_names(lang).len()..]) {
if !out.iter().any(|(_, d)| *d == c.1) {
out.push(c);
}
}
}
out
}
/// The names [`names`] gives that outrank the dialect's own embedded SCOWL dictionary: its own pair
/// only, not Australian English's British fallback (a user's `en_GB` is for British text).
fn own_names(lang: ProofLang) -> &'static [&'static str] {
match lang {
ProofLang::EnAu => &["en_AU", "en-AU"],
_ => names(lang),
}
}
/// How many Englishes have a slot of their own ([`slot`]).
const ENGLISHES: usize = 4;
/// The slot of each English in [`State::slots`].
fn slot(lang: ProofLang) -> usize {
match lang {
ProofLang::EnGb => 1,
ProofLang::EnCa => 2,
ProofLang::EnAu => 3,
_ => 0,
}
}
@@ -109,7 +143,7 @@ type Loaded = Option<(Arc<AffixDict>, PathBuf)>;
/// The English Hunspell dictionaries, under one lock.
struct State {
/// Per English (see [`slot`]): `None` until first use, then what was found.
slots: [Option<Loaded>; 3],
slots: [Option<Loaded>; ENGLISHES],
/// Every `.dic` file read, so Englishes that use the same file share one copy.
files: Vec<(PathBuf, Arc<AffixDict>)>,
/// Counts reloads, so a load that started before one is not kept.
@@ -118,7 +152,7 @@ struct State {
impl State {
const fn new() -> Self {
State { slots: [None, None, None], files: Vec::new(), reloads: 0 }
State { slots: [None, None, None, None], files: Vec::new(), reloads: 0 }
}
}
@@ -181,7 +215,7 @@ fn already_read(state: &RwLock<State>, dic: &Path) -> Option<Arc<AffixDict>> {
/// Forget what `state` holds, so the next check looks again.
fn forget(state: &RwLock<State>) {
let mut s = state.write().unwrap_or_else(PoisonError::into_inner);
s.slots = [None, None, None];
s.slots = [None, None, None, None];
s.files.clear();
s.reloads = s.reloads.wrapping_add(1);
}
@@ -198,9 +232,9 @@ fn key(lang: ProofLang) -> &'static str {
/// would rather use), for [`note_error`].
fn load(lang: ProofLang) -> (Loaded, Option<(PathBuf, String)>) {
let candidates = if crate::scowl::dict(lang).is_some() {
crate::hunspell::user_candidates(names(lang))
crate::hunspell::user_candidates(own_names(lang))
} else {
crate::hunspell::candidates_exact(names(lang))
exact_candidates(lang, crate::hunspell::candidates_exact)
};
if candidates.is_empty() {
log::info!("English spelling: no Hunspell {} dictionary to read first; using SCOWL or the built-in word list", key(lang));
@@ -240,7 +274,7 @@ fn note_error(lang: ProofLang, error: Option<(PathBuf, String)>) {
/// folder changed).
pub(crate) fn reload() {
forget(&HUNSPELL);
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::EnAu] {
crate::hunspell::clear_load_error(key(lang));
}
}
@@ -283,10 +317,16 @@ fn pick(found: Loaded, lang: ProofLang) -> Speller {
if let Some((d, dic)) = found {
return Speller::Hunspell(d, dic);
}
match crate::scowl::dict(lang) {
Some(d) => Speller::Scowl(vec![d], crate::scowl::is_british(lang)),
None => Speller::List,
let Some(d) = crate::scowl::dict(lang) else { return Speller::List };
let mut ds = vec![d];
// SCOWL's en-AU list keeps one spelling per word (`inquiry`; `enquiry` is only in the en-GB
// list), and Australian English takes both. Suggestions come from the first, en-AU.
if lang == ProofLang::EnAu
&& let Some(gb) = crate::scowl::dict(ProofLang::EnGb)
{
ds.push(gb);
}
Speller::Scowl(ds, crate::scowl::is_british(lang))
}
/// [`speller`] for English with no dialect ([`crate::is_correct`]): a SCOWL word of any English
@@ -348,11 +388,12 @@ pub(crate) fn is_correct(word: &str, lang: ProofLang, opts: &Options, policy: bo
|| (policy && !parts.is_empty() && parts.iter().all(|p| s.knows_lower(p) || crate::en_variants::known_variant(p, lang, |x| s.knows_lower(x))))
}
/// `w` with its `-ize`/`-yze` endings (and their forms: `-ization`, `-izing`…) spelled `-ise`/
/// `-yse`, when it has any: British English takes both (Word accepts `organize` and `organise` in
/// en-GB), and SCOWL's en-GB dictionary has only `-ise`. A `z` between `i`/`y` and a vowel is
/// swapped; the `-ize` form counts only when American or Canadian English spells the word so
/// ([`ize_elsewhere`]), or `wize` would pass as `wise`.
/// `w` with its `-ize` endings (and their forms: `-ization`, `-izing`…) spelled `-ise`, when it
/// has any: British English takes both (Word accepts `organize` and `organise` in en-GB), and
/// SCOWL's en-GB dictionary has only `-ise`. A `z` between `i` and a vowel is swapped; the `-ize`
/// form counts only when American or Canadian English spells the word so ([`ize_elsewhere`]), or
/// `wize` would pass as `wise`. Never `-yze`: Oxford spelling has `analyse`, `paralyse`, so
/// `reanalyze` or `breathalyzer` is American in British English, as Hunspell's en_GB says.
pub(crate) fn ise_spelling(w: &str) -> Option<String> {
let b = w.as_bytes();
let mut out = b.to_vec();
@@ -360,7 +401,7 @@ pub(crate) fn ise_spelling(w: &str) -> Option<String> {
for i in 1..b.len() {
let after = b.get(i + 1).is_some_and(|c| b"aeiAEI".contains(c));
if after
&& matches!(b.get(i - 1), Some(b'i' | b'y' | b'I' | b'Y'))
&& matches!(b.get(i - 1), Some(b'i' | b'I'))
&& let Some(c) = out.get_mut(i)
{
match *c {
@@ -512,6 +553,18 @@ fn list_suggest(word: &str, max: usize) -> Vec<String> {
mod tests {
use super::*;
/// Review follow-up: only `-ize` is swapped for `-ise`, never `-yze` for `-yse` (Oxford
/// spelling has `analyse`), so `reanalyze`, `breathalyzer`, `paralyzingly` stay American.
#[test]
fn ise_spelling_swaps_ize_only() {
assert_eq!(ise_spelling("organize").as_deref(), Some("organise"));
assert_eq!(ise_spelling("Realizations").as_deref(), Some("Realisations"));
assert_eq!(ise_spelling("ORGANIZING").as_deref(), Some("ORGANISING"));
for w in ["reanalyze", "reanalyzed", "breathalyze", "breathalyzer", "paralyzingly", "catalyze", "Analyzer", "ANALYZE"] {
assert_eq!(ise_spelling(w), None, "{w}");
}
}
#[test]
fn built_in_list_is_the_whole_moby_list() {
let d = words();
@@ -596,16 +649,16 @@ mod tests {
fn lookup_order() {
let tiny = Arc::new(AffixDict::from_hunspell("", "1\nzzyzx\n", &[]).unwrap());
let file = PathBuf::from("/folder/en_US.dic");
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] {
assert!(matches!(pick(Some((tiny.clone(), file.clone())), lang), Speller::Hunspell(_, p) if p == file), "{lang:?}");
let fallback = pick(None, lang);
if crate::scowl::dict(lang).is_some() {
assert!(
matches!(&fallback, Speller::Scowl(ds, _) if ds.len() == 1 && std::ptr::eq(ds[0], crate::scowl::dict(lang).unwrap())),
matches!(&fallback, Speller::Scowl(ds, _) if ds.len() == if lang == ProofLang::EnAu { 2 } else { 1 } && std::ptr::eq(ds[0], crate::scowl::dict(lang).unwrap())),
"{lang:?}"
);
assert!(fallback.knows("21st") && !fallback.knows("becuase"), "{lang:?}");
assert_eq!(fallback.knows("colour"), lang != ProofLang::En, "{lang:?}: its own dictionary only");
assert_eq!(fallback.knows("colour"), lang != ProofLang::En, "{lang:?}: its own dictionary (en-AU: and en-GB's)");
} else {
assert!(matches!(fallback, Speller::List), "{lang:?}");
}
@@ -625,6 +678,44 @@ mod tests {
assert_eq!(found(names(ProofLang::EnGb)), ["en_GB"], "a user's en_US doesn't stand in for SCOWL's en-GB");
assert_eq!(found(names(ProofLang::EnCa)), ["en_CA"]);
assert_eq!(found(names(ProofLang::En)), ["en_US"]);
// Australian English: its own, else the British one it read before it had its own.
assert_eq!(found(names(ProofLang::EnAu)), ["en_GB"]);
std::fs::write(dir.join("en_AU.aff"), "SET UTF-8\n").unwrap();
std::fs::write(dir.join("en_AU.dic"), "1\nzzyzx\n").unwrap();
assert_eq!(found(names(ProofLang::EnAu)), ["en_AU", "en_GB"]);
// craft-libs#496 review: with SCOWL embedded, a user's en_GB must not outrank its en-AU.
assert_eq!(found(own_names(ProofLang::EnAu)), ["en_AU"]);
std::fs::remove_file(dir.join("en_AU.aff")).unwrap();
std::fs::remove_file(dir.join("en_AU.dic")).unwrap();
assert!(found(own_names(ProofLang::EnAu)).is_empty(), "an en_GB alone is not Australian English's own");
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
assert_eq!(own_names(lang), names(lang));
}
let _ = std::fs::remove_dir_all(&dir);
}
/// Without SCOWL, every en_AU source comes before any en_GB one (a dictionaries folder's, an
/// installed pack's, the system's): a user's en_GB doesn't outrank a system en_AU.
#[test]
fn australian_english_reads_every_en_au_before_any_en_gb() {
let dir = std::env::temp_dir().join(format!("wc-proof-en-au-order-{}", std::process::id()));
for sub in ["user", "system"] {
std::fs::create_dir_all(dir.join(sub)).unwrap();
}
let write = |sub: &str, name: &str| {
std::fs::write(dir.join(sub).join(format!("{name}.aff")), "SET UTF-8\n").unwrap();
std::fs::write(dir.join(sub).join(format!("{name}.dic")), "1\nzzyzx\n").unwrap();
};
write("user", "en_GB");
write("system", "en_AU");
let (user, system) = (vec![dir.join("user")], vec![dir.join("system")]);
let find = |n: &[&str]| crate::hunspell::ordered(n, &user, &[], &system, &[]);
let stems = |v: Vec<(PathBuf, PathBuf)>| -> Vec<String> {
v.iter().filter_map(|(_, d)| d.file_stem().map(|s| s.to_string_lossy().into_owned())).collect()
};
assert_eq!(stems(exact_candidates(ProofLang::EnAu, find)), ["en_AU", "en_GB"]);
assert_eq!(stems(exact_candidates(ProofLang::EnGb, find)), ["en_GB"]);
assert_eq!(stems(exact_candidates(ProofLang::En, find)), Vec::<String>::new());
let _ = std::fs::remove_dir_all(&dir);
}
crates/proof/src/es/grammar.rs
+12 -5
View File
@@ -75,11 +75,8 @@ fn spaces(text: &str, lang: ProofLang, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Espacio antes de un signo de puntuación".into(), vec![p.to_string()], lang));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Espacio doble".into(), vec![" ".into()], lang));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Espacio doble".into(), vec![" ".into()], lang));
}
}
@@ -194,6 +191,16 @@ mod tests {
.collect()
}
/// craft-libs#496 review: a run of spaces is one issue over the whole run.
#[test]
fn a_run_of_spaces_is_reported_whole() {
for (text, run) in [("casa roja", " "), ("casa roja", " "), ("casa roja", " "), ("casa roja", " ")] {
let m: Vec<_> = messages(text).into_iter().filter(|(_, msg)| msg == "Espacio doble").collect();
assert_eq!(m, [(run.to_string(), "Espacio doble".to_string())], "{text}");
}
assert!(messages("casa roja").is_empty());
}
#[test]
fn repeated_words_and_spaces_have_spanish_messages() {
let text = "Es es la casa , dijo.";
crates/proof/src/hr/grammar.rs
+2 -5
View File
@@ -105,11 +105,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Razmak ispred interpunkcijskog znaka".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Dvostruki razmak".into(), vec![" ".into()]));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Dvostruki razmak".into(), vec![" ".into()]));
}
}
crates/proof/src/hr/mod.rs
+10 -1
View File
@@ -65,7 +65,8 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
return true;
};
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
// The whole word first, as Hunspell does: a hyphenated entry or user word.
return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
}
known(d, w)
}
@@ -105,6 +106,14 @@ mod tests {
}
}
/// craft-libs#496 review: a hyphenated dictionary entry is a word even when its parts aren't.
#[test]
fn a_hyphenated_entry_is_looked_up_whole() {
let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap();
assert!(is_correct_with(&d, "tic-tac"));
assert!(!is_correct_with(&d, "tic-tacc"));
}
#[test]
fn missing_dictionary_is_silent() {
assert!(is_correct_with(&AffixDict::default(), "nepostojećariječ"));
crates/proof/src/hunspell.rs
+38 -7
View File
@@ -196,19 +196,22 @@ pub(crate) fn ordered(
let first = names.first().map(|n| norm(n));
let (region, rest): (Vec<&str>, Vec<&str>) = names.iter().copied().partition(|n| Some(norm(n)) == first);
let pack: &[&str] = &[crate::packs::SPELLING];
// (folder, names, a Language pack's: never read through a symbolic link, as the pack scan
// checks; a system folder's dictionaries often are links, `en_GB.dic` → `en_GB-ise.dic`).
let tiers = dirs
.iter()
.map(|d| (d, names))
.chain(region_packs.iter().map(|d| (d, pack)))
.chain(system.iter().map(|d| (d, &region[..])))
.chain(other_packs.iter().map(|d| (d, pack)))
.chain(system.iter().map(|d| (d, &rest[..])));
.map(|d| (d, names, false))
.chain(region_packs.iter().map(|d| (d, pack, true)))
.chain(system.iter().map(|d| (d, &region[..], false)))
.chain(other_packs.iter().map(|d| (d, pack, true)))
.chain(system.iter().map(|d| (d, &rest[..], false)));
let file = |p: &Path, pack: bool| if pack { std::fs::symlink_metadata(p).is_ok_and(|m| m.is_file()) } else { p.is_file() };
let mut out: Vec<(PathBuf, PathBuf)> = Vec::new();
for (dir, names) in tiers {
for (dir, names, in_pack) in tiers {
for n in names {
let aff = dir.join(format!("{n}.aff"));
let dic = dir.join(format!("{n}.dic"));
if aff.is_file() && dic.is_file() && !out.iter().any(|(_, d)| *d == dic) {
if file(&aff, in_pack) && file(&dic, in_pack) && !out.iter().any(|(_, d)| *d == dic) {
out.push((aff, dic));
}
}
@@ -696,6 +699,34 @@ mod tests {
}
}
/// Review follow-up (craft-libs#485): a pack's `spelling.aff`/`.dic` that is a symbolic link
/// (swapped in after the pack scan) is never a candidate; one in a system folder still is.
#[cfg(unix)]
#[test]
fn pack_files_are_never_read_through_a_link() {
let root = std::env::temp_dir().join(format!("wc-proof-pack-links-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
let (pack, system, elsewhere) = (root.join("packs").join("nl"), root.join("system"), root.join("elsewhere"));
for d in [&pack, &system, &elsewhere] {
std::fs::create_dir_all(d).unwrap();
}
std::fs::write(elsewhere.join("x.aff"), "SET UTF-8\n").unwrap();
std::fs::write(elsewhere.join("x.dic"), "1\nx\n").unwrap();
std::fs::write(pack.join("spelling.aff"), "SET UTF-8\n").unwrap();
std::os::unix::fs::symlink(elsewhere.join("x.dic"), pack.join("spelling.dic")).unwrap();
std::os::unix::fs::symlink(elsewhere.join("x.aff"), system.join("nl.aff")).unwrap();
std::os::unix::fs::symlink(elsewhere.join("x.dic"), system.join("nl.dic")).unwrap();
let got: Vec<PathBuf> =
ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect();
assert_eq!(got, [system.join("nl.dic")], "the system's link is followed, the pack's never");
std::fs::remove_file(pack.join("spelling.dic")).unwrap();
std::fs::write(pack.join("spelling.dic"), "1\nx\n").unwrap();
let got: Vec<PathBuf> =
ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect();
assert_eq!(got, [pack.join("spelling.dic"), system.join("nl.dic")]);
let _ = std::fs::remove_dir_all(&root);
}
/// WordCraft's own dictionaries folder under the user's application data is searched on every
/// desktop system, after the setting and before the shared system locations (#1063).
#[test]
crates/proof/src/id/grammar.rs
+2 -5
View File
@@ -56,11 +56,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Spasi sebelum tanda baca".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Spasi ganda".into(), vec![" ".into()]));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Spasi ganda".into(), vec![" ".into()]));
}
}
crates/proof/src/id/mod.rs
+14 -1
View File
@@ -76,7 +76,7 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|| w.split('-').all(|p| {
p.chars().count() <= 1
|| CLITICS.iter().any(|c| p.eq_ignore_ascii_case(c))
|| (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= 6)
|| (opts.ignore_uppercase && crate::spelling::capitals_part(p, 6))
|| known(d, p)
});
}
@@ -141,6 +141,19 @@ mod tests {
}
}
/// Review follow-up (craft-libs#485's cased-letter rule): a part in a caseless script is no
/// abbreviation, so `buku-日本` is checked part by part; a part in capitals or without letters
/// still is one (`KTP-nya`, `COVID-19` with numbers checked).
#[test]
fn caseless_parts_are_not_abbreviations() {
test_dict::install();
let d = dictionary();
let defaults = crate::Options::default();
assert!(!check(&d, "buku-日本", &defaults) && !check(&d, "buku-ไทย", &defaults));
let numbers = crate::Options { ignore_numbers: false, ..defaults };
assert!(check(&d, "KTP-nya", &defaults) && check(&d, "buku-19", &numbers));
}
#[test]
fn forbidden_words_are_rejected_in_every_case() {
test_dict::install();
crates/proof/src/it/grammar.rs
+32 -8
View File
@@ -150,10 +150,12 @@ fn abbreviation_or_number(prev: &str) -> bool {
|| ABBREVIATIONS.contains(&prev.to_lowercase().as_str())
}
const CLOSING: [char; 8] = ['"', '”', '’', '\'', ')', '»', ']', '\u{1}'];
/// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after
/// an abbreviation), then space, with nothing but quotes and brackets around?
fn ends_sentence(gap: &str, prev: &str) -> bool {
let t = gap.trim_start_matches(['"', '”', '’', '\'', ')', '»', ']', '\u{1}']);
let t = gap.trim_start_matches(CLOSING);
let Some(end) = t.chars().next() else { return false };
if !matches!(end, '.' | '!' | '?') || t.starts_with("..") {
return false;
@@ -161,8 +163,16 @@ fn ends_sentence(gap: &str, prev: &str) -> bool {
if end == '.' && abbreviation_or_number(prev) {
return false;
}
let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(['"', '”', '’', '\'', ')', '»', ']']);
rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—'))
let after_marks = t.trim_start_matches(['.', '!', '?']);
let rest = after_marks.trim_start_matches(CLOSING);
// (Not the object placeholder `\u{1}`: `Chi era?\u{1} era Marco.` still starts a sentence.)
// `«Vai?» disse lui.` and `—Vai? —disse lui.`: the narration after a quoted question or
// exclamation (not after a period: `Finì. —domani` starts a sentence), or after a dialogue dash, goes on in lower case.
if end != '.' && (rest.contains('—') || after_marks.trim_start_matches('\u{1}').starts_with(CLOSING)) {
return false;
}
rest.starts_with(char::is_whitespace)
&& rest.trim_start().chars().all(|c| c.is_whitespace() || matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—'))
}
fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
@@ -203,11 +213,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Spazio prima della punteggiatura".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Spazio doppio".into(), vec![" ".into()]));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Spazio doppio".into(), vec![" ".into()]));
}
}
@@ -403,6 +410,23 @@ mod tests {
}
}
/// craft-libs#496 review: narration after a quoted or dashed question or exclamation goes on in
/// lower case.
#[test]
fn narration_after_a_quoted_question_is_not_a_new_sentence() {
for t in ["«Vai?» disse lui.", "«Vieni!» gridò Anna.", "\"Vai?\" disse lui.", "—Vai? —disse lui.", "— Vai! — disse lui."] {
assert!(fixes(t).is_empty(), "{t}: {:?}", fixes(t));
}
assert!(fixes("Vado a Roma? sì.").contains(&pair("sì", "Sì")), "a bare question still ends the sentence");
// After a period the dash opens the next line of speech: a sentence starts.
assert!(fixes("Finì. —domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. —domani partiamo."));
assert!(fixes("Finì. — domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. — domani partiamo."));
// An inline object (`\u{1}`) before a real closing quote is skipped.
assert!(fixes("«Vai?\u{1}» disse lui.").is_empty(), "{:?}", fixes("«Vai?\u{1}» disse lui."));
// An inline object (`\u{1}`) is not a closing quote.
assert!(fixes("Chi era?\u{1} era Marco.").contains(&pair("era", "Era")), "{:?}", fixes("Chi era?\u{1} era Marco."));
}
#[test]
fn po_takes_an_apostrophe() {
assert_eq!(fixes("Dammi un pò di pane"), [pair("pò", "po'")]);
crates/proof/src/it/mod.rs
+17 -1
View File
@@ -97,8 +97,14 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
}
// The whole word first, as Hunspell does: a hyphenated entry or user word.
if w.contains('-') {
return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known_elided(d, p));
}
known_elided(d, w)
}
/// [`known`] for a word that may start with an elided proclitic (`l'amico`, `l'ex`): the word after
/// the apostrophe is what gets checked.
fn known_elided(d: &AffixDict, w: &str) -> bool {
if let Some((head, tail)) = w.split_once(['\'', '’'])
&& PROCLITICS.contains(&head.to_lowercase().as_str())
&& !tail.is_empty()
@@ -148,6 +154,16 @@ mod tests {
assert!(!is_correct_with(&d, "tic-tacc"));
}
/// craft-libs#496 review: an elision inside a hyphenated word (`l'ex-amico`) is read per part.
#[test]
fn an_elision_in_a_hyphenated_part_is_read() {
let d = AffixDict::from_hunspell("SET UTF-8\n", "2\nex\namico\n", &[]).unwrap();
for w in ["l'ex-amico", "dell’ex-amico", "ex-amico"] {
assert!(is_correct_with(&d, w), "{w}");
}
assert!(!is_correct_with(&d, "l'exx-amico"));
}
#[test]
fn without_a_dictionary_nothing_is_flagged() {
let none = AffixDict::default();
crates/proof/src/lang.rs
+23 -30
View File
@@ -5,7 +5,7 @@
/// A language WordCraft can proof, or [`ProofLang::Other`] for one it can't.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum ProofLang {
/// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`] or
/// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`], [`ProofLang::EnAu`] or
/// [`ProofLang::EnCa`]): SCOWL's en-US dictionary ([`crate::scowl`]), the English grammar rules and the
/// English hyphenation dictionary and patterns. British spellings (`colour`, `centre`,
/// `organise`) are flagged, with the American form suggested ([`crate::en_variants`]). Also
@@ -13,10 +13,14 @@ pub enum ProofLang {
/// Word's).
#[default]
En,
/// English with British spelling (`en-GB`, `en-AU`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…):
/// English with British spelling (`en-GB`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…):
/// [`ProofLang::En`]'s tools, but American spellings (`color`, `center`, `analyze`) are
/// flagged and `-ize` and `-ise` are both right, as in Oxford spelling.
EnGb,
/// Australian English (`en-AU`): [`ProofLang::EnGb`]'s spelling policy, against SCOWL's own
/// en-AU dictionary, which knows Australian words (`billabong`, `brumby`, `larrikin`) and
/// spells some words the Australian way (`program`, `inquiry`).
EnAu,
/// Canadian English (`en-CA`): British `-our`, `-re` and `-ence` (`colour`, `centre`,
/// `defence`) with American `-ize` (`organize`), `tire` and `program`.
EnCa,
@@ -86,9 +90,10 @@ pub enum ProofLang {
impl ProofLang {
/// Every language with a spelling dictionary, in display order.
pub const PROOFED: [ProofLang; 15] = [
pub const PROOFED: [ProofLang; 16] = [
ProofLang::En,
ProofLang::EnGb,
ProofLang::EnAu,
ProofLang::EnCa,
ProofLang::Pl,
ProofLang::Pt,
@@ -155,12 +160,13 @@ impl ProofLang {
}
}
/// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], regions
/// that spell the British way are [`ProofLang::EnGb`], and the United States, any other
/// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], Australia
/// [`ProofLang::EnAu`], other regions that spell the British way are [`ProofLang::EnGb`], and the United States, any other
/// region and no region are [`ProofLang::En`].
fn english(tag: &str) -> ProofLang {
match region(tag) {
Some(r) if r.eq_ignore_ascii_case("CA") => ProofLang::EnCa,
Some(r) if r.eq_ignore_ascii_case("AU") => ProofLang::EnAu,
Some(r) if BRITISH_SPELLING.iter().any(|b| b.eq_ignore_ascii_case(r)) => ProofLang::EnGb,
_ => ProofLang::En,
}
@@ -189,9 +195,10 @@ impl ProofLang {
}
}
/// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnCa`])?
/// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnAu`],
/// [`ProofLang::EnCa`])?
pub fn is_english(self) -> bool {
matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa)
matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa)
}
/// Is this one of the Spanishes ([`ProofLang::Es`], [`ProofLang::EsMx`], [`ProofLang::Es419`],
@@ -208,7 +215,7 @@ impl ProofLang {
pub fn code(self) -> &'static str {
match self {
ProofLang::Pack(_) => "",
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => "en",
ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => "en",
ProofLang::Pl => "pl",
ProofLang::Bg => "bg",
ProofLang::Pt => "pt",
@@ -371,12 +378,12 @@ pub(crate) fn region(tag: &str) -> Option<&str> {
const LATIN_AMERICA: [&str; 22] =
["419", "005", "013", "029", "AR", "BO", "CL", "CO", "CR", "CU", "DO", "EC", "GT", "HN", "NI", "PA", "PE", "PR", "PY", "SV", "US", "VE"];
/// Regions whose English uses British spelling ([`ProofLang::EnGb`]): the United Kingdom (and
/// the non-standard `UK`), Ireland, Australia, New Zealand, South Africa, India and other
/// Regions whose English uses British spelling ([`ProofLang::EnGb`]; Australia has its own,
/// [`ProofLang::EnAu`]): the United Kingdom (and the non-standard `UK`), Ireland, New Zealand, South Africa, India and other
/// Commonwealth countries, the Crown dependencies and Europe (`150`).
const BRITISH_SPELLING: [&str; 26] = [
"GB", "UK", "IE", "AU", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH",
"PK", "150",
const BRITISH_SPELLING: [&str; 25] = [
"GB", "UK", "IE", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH", "PK",
"150",
];
/// A BCP 47 tag in canonical case (`pl-PL`, `en-GB`, `zh-Hant-TW`, `x-none`), or `None` when
@@ -469,26 +476,12 @@ mod tests {
fn tags_resolve_english_regions() {
assert_tags(&["en", "en-US", "EN_us", "en-Latn-US", "en-PH", "en-001", "eng", "en_US.UTF-8", "en-x-gb"], ProofLang::En);
assert_tags(
&[
"en-GB",
"EN_gb",
"en_GB.UTF-8",
"en-AU",
"en_AU",
"en-NZ",
"en-IE",
"en-ZA",
"en-IN",
"en-Latn-GB",
"en-UK",
"en-150",
"eng-GB",
"en_GB@euro",
],
&["en-GB", "EN_gb", "en_GB.UTF-8", "en-NZ", "en-IE", "en-ZA", "en-IN", "en-Latn-GB", "en-UK", "en-150", "eng-GB", "en_GB@euro"],
ProofLang::EnGb,
);
assert_tags(&["en-CA", "en_CA", "EN-ca", "en_CA.UTF-8", "en-Latn-CA"], ProofLang::EnCa);
for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
assert_tags(&["en-AU", "en_AU", "EN-au", "en_AU.UTF-8", "en-Latn-AU"], ProofLang::EnAu);
for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] {
assert!(language.is_english() && language.is_proofed() && ProofLang::PROOFED.contains(&language), "{language:?}");
assert_eq!(language.code(), "en");
}
crates/proof/src/lib.rs
+155 -12
View File
@@ -152,6 +152,28 @@ fn dictionary_key(w: &str) -> String {
composed(&w.to_lowercase()).into_owned()
}
/// Each run of two or more spaces in `text`, once, over the whole run (a fix replaces it with one
/// space). A run of three is one issue over three, not a pair and a leftover.
pub(crate) fn space_runs(text: &str) -> Vec<Range<usize>> {
let bytes = text.as_bytes();
let mut runs = Vec::new();
let mut i = 0;
while i < bytes.len() {
if bytes.get(i) != Some(&b' ') {
i += 1;
continue;
}
let start = i;
while bytes.get(i) == Some(&b' ') {
i += 1;
}
if i - start >= 2 {
runs.push(start..i);
}
}
runs
}
/// Add a word to the user dictionary ("Add to Dictionary"). It applies to every language.
pub fn add_word(w: &str) {
if user_words().write().unwrap_or_else(std::sync::PoisonError::into_inner).insert(dictionary_key(w)) {
@@ -340,7 +362,7 @@ pub fn installed_dictionaries() -> Vec<(ProofLang, Option<std::path::PathBuf>)>
/// checked yet reports `None` (#2830). For an English, its embedded SCOWL dictionary or the
/// English Hunspell dictionary that failed; English then uses the next source ([`english_source`]).
pub fn dictionary_load_error(lang: ProofLang) -> Option<String> {
if matches!(lang, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) {
if lang.is_english() {
return english::load_error(lang);
}
if let Some(l) = lang.spelling_locale() {
@@ -669,7 +691,7 @@ pub(crate) fn accepted_by_options(word: &str, opts: &Options) -> bool {
|| !w.chars().any(char::is_alphabetic)
|| (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit()))
// Only a word with a cased letter is in capitals: a Hebrew or Thai word has none.
|| (opts.ignore_uppercase && w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase()))
|| (opts.ignore_uppercase && spelling::in_capitals(w))
// The whole word: the languages split a hyphenated word into its parts, and an added
// `Unigranrio-RJ` is none of them (#2082).
|| in_user_dictionary(&w.to_lowercase())
@@ -953,7 +975,7 @@ pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec<Issue> {
fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec<Issue> {
match lang {
ProofLang::En => grammar_en(text, capital_first),
ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(),
ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(),
// `pt-pt` shares Portuguese's rules, its issues in its own language.
_ => language(lang.built_in())
.map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect()),
@@ -1098,13 +1120,10 @@ fn grammar_en(text: &str, capital_first: bool) -> Vec<Issue> {
lang: ProofLang::En,
});
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
for run in space_runs(text) {
v.push(Issue {
start: i,
end: i + 2,
start: run.start,
end: run.end,
kind: IssueKind::Grammar,
message: "Extra space".into(),
suggestions: vec![" ".into()],
@@ -1197,7 +1216,7 @@ mod tests {
}
// Every proofed language other than English has a Hunspell dictionary to report.
for l in ProofLang::PROOFED {
let english = matches!(l, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa);
let english = l.is_english();
assert_eq!(dictionary_needed(l, false).is_some(), !english, "{l:?}");
}
for l in [ProofLang::En, ProofLang::Other] {
@@ -1709,6 +1728,31 @@ mod tests {
let _ = std::fs::remove_dir_all(&root);
}
/// Review follow-up (craft-libs#485): a word in another script than a pack language's
/// dictionary (a Russian name in a Dutch run, a Dutch word in a Greek one) isn't marked; a
/// misspelling in the dictionary's own script still is.
#[test]
fn pack_languages_leave_other_scripts_alone() {
crate::hold_dictionary_tests();
set_dictionaries_folder(Some(shared_test_folder()));
let root = std::env::temp_dir().join(format!("wc-proof-other-script-packs-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
write_pack(&root, "nl", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", "2\nhuis\nkat\n".as_bytes())]);
write_pack(&root, "el", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", "1\nσπίτι\n".as_bytes())]);
packs::set_packs_dir(Some(root.clone()));
let (nl, el) = (ProofLang::from_tag(Some("nl-NL")), ProofLang::from_tag(Some("el-GR")));
assert!(is_correct_in("huis", nl) && !is_correct_in("huiss", nl));
assert!(!is_correct_in("zzzz", nl), "a misspelling in the dictionary's script, whatever its letters");
assert!(is_correct_in("Пушкин", nl) && is_correct_in("σπίτι", nl));
assert!(is_correct_in("σπίτι", el) && !is_correct_in("σπίτιι", el) && is_correct_in("huis", el));
let text = "Пушкин huiss";
let issues = check_text(text, &[(0..text.len(), nl)]);
assert_eq!(issues.iter().map(|i| &text[i.start..i.end]).collect::<Vec<_>>(), ["huiss"], "{issues:?}");
packs::set_packs_dir(None);
set_dictionaries_folder(Some(shared_test_folder()));
let _ = std::fs::remove_dir_all(&root);
}
/// The `.dic` file `lang`'s spelling is read from with the installed packs and `system` as the
/// system's only dictionary folder.
fn chosen_dictionary(lang: ProofLang, system: &std::path::Path) -> Option<std::path::PathBuf> {
@@ -2465,6 +2509,49 @@ mod tests {
assert_eq!(suggest_in("programmes", 5, us).first().map(String::as_str), Some("programs"));
}
/// Review follow-up: British English takes `-ize` (Oxford spelling) but never `-yze`, also in
/// forms en-variants.txt doesn't list. Hunspell 1.7.5 with SCOWL's en_GB rejects these too.
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn british_english_rejects_unlisted_yze_forms() {
crate::hold_dictionary_tests();
let gb = ProofLang::EnGb;
for w in ["reanalyze", "reanalyzed", "breathalyze", "breathalyzer", "paralyzingly", "catalyze", "analyzer"] {
assert!(!is_correct_in(w, gb), "en-GB: {w}");
assert!(is_correct_in(w, ProofLang::En), "en-US: {w}");
}
for w in ["reanalyse", "breathalyser", "paralysingly", "catalyse", "organize", "realization"] {
assert!(is_correct_in(w, gb), "en-GB: {w}");
}
}
/// Review follow-up: Australian English reads SCOWL's own en-AU dictionary (craft-i18n ships
/// it), not en-GB's: its Australian words are right and spelled as Hunspell's en_AU spells
/// them; British English doesn't take them. The British policy still applies: American
/// spellings are wrong, `-ise` and `-ize` both right.
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn australian_english_reads_its_own_scowl() {
crate::hold_dictionary_tests();
let (au, gb) = (ProofLang::EnAu, ProofLang::EnGb);
assert!(scowl::is_british(au));
assert_eq!(english_source(au), Some(EnglishSource::Embedded));
for w in ["billabong", "brumby", "larrikinism", "bunyip", "corroboree", "Bundaberg", "Rockhampton"] {
assert!(is_correct_in(w, au), "en-AU: {w}");
assert!(!is_correct_in(w, gb), "en-GB: {w}");
assert!(is_correct_for(w, Some("en-AU")), "en-AU tag: {w}");
}
for w in ["colour", "centre", "organise", "organize", "analyse", "travelled", "defence"] {
assert!(is_correct_in(w, au), "en-AU: {w}");
}
for w in ["color", "center", "analyze", "reanalyze", "defense"] {
assert!(!is_correct_in(w, au), "en-AU: {w}");
}
let text = "The larrikin saw a brumby by the billabong and analysed the colour of the sky.";
assert!(check_spelling_in(text, au).is_empty(), "{:?}", check_spelling_in(text, au));
assert_eq!(check_spelling_in(text, gb).len(), 3, "larrikin, brumby, billabong: {:?}", check_spelling_in(text, gb));
}
/// More British spellings that must stay right in en-GB and en-AU (#1281): -re, single l, ae,
/// with their inflections.
#[test]
@@ -2472,7 +2559,7 @@ mod tests {
// English results depend on the dictionaries other tests install.
crate::hold_dictionary_tests();
let gb = ProofLang::EnGb;
assert_eq!(ProofLang::from_tag(Some("en-AU")), gb);
assert_eq!(ProofLang::from_tag(Some("en-AU")), ProofLang::EnAu);
for w in [
"goitre",
"goitres",
@@ -2497,11 +2584,66 @@ mod tests {
"encyclopaedias",
] {
assert!(is_correct_in(w, gb), "en-GB: {w}");
assert!(is_correct_in(w, ProofLang::EnAu), "en-AU: {w}");
}
let text = "The wilful knight took the sceptre to fulfil his enrolment, paid the instalment and read the encyclopaedia.";
assert!(check_spelling_in(text, gb).is_empty(), "{:?}", check_spelling_in(text, gb));
}
/// Review follow-up: SCOWL's en-AU list keeps one spelling per word, so Australian English also
/// takes a word the en-GB list has (`enquiry`, `guerrilla`, `liquorice`); American-only
/// spellings, wrong in both, stay wrong. Suggestions still come from the en-AU list.
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn australian_english_also_takes_british_words() {
crate::hold_dictionary_tests();
let (au, gb) = (ProofLang::EnAu, ProofLang::EnGb);
for w in ["enquiry", "enquire", "guerrilla", "guerrillas", "euthanise", "liveable", "liquorice", "veranda", "burka", "Anthea"] {
assert!(is_correct_in(w, gb), "en-GB: {w}");
assert!(is_correct_in(w, au), "en-AU: {w}");
}
for w in ["analyze", "catalyze", "color", "center", "defense", "zzyzxq"] {
assert!(!is_correct_in(w, gb), "en-GB: {w}");
assert!(!is_correct_in(w, au), "en-AU: {w}");
}
assert!(is_correct_in("billabong", au) && !is_correct_in("billabong", gb), "en-AU still has its own");
assert!(suggest_in("billabon", 5, au).iter().any(|s| s == "billabong"), "{:?}", suggest_in("billabon", 5, au));
}
/// Review follow-up: a run of spaces is one issue over the whole run, in every language with a
/// double-space rule (a run of three used to leave two after the fix, four three).
#[test]
fn a_run_of_spaces_is_one_issue_in_every_language() {
assert_eq!(space_runs("a b c d "), [3..5, 6..9, 10..14]);
assert!(space_runs("a b c").is_empty() && space_runs("").is_empty() && space_runs(" ").is_empty());
let langs = [
ProofLang::En,
ProofLang::EnGb,
ProofLang::Es,
ProofLang::It,
ProofLang::Pt,
ProofLang::Cs,
ProofLang::Hr,
ProofLang::Id,
ProofLang::Pl,
ProofLang::Ru,
ProofLang::Tr,
ProofLang::EnAu,
ProofLang::EnCa,
]
.into_iter()
.chain(PackLocale::new("pt-PT").map(ProofLang::Pack));
for lang in langs {
for n in 2..=5 {
let text = format!("Aaa{}bbb", " ".repeat(n));
let hits: Vec<_> =
check_grammar_in(&text, lang).into_iter().filter(|i| i.start <= 3 && i.end > 3 && i.suggestions == [" "]).collect();
assert_eq!(hits.len(), 1, "{lang:?} {n}: {hits:?}");
assert_eq!((hits[0].start, hits[0].end), (3, 3 + n), "{lang:?} {n}");
}
}
}
/// English spelling reads SCOWL 2026 through the affix engine: one dictionary per English,
/// its notice embedded, and its ordinals (`COMPOUNDRULE`) understood.
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
@@ -2513,7 +2655,8 @@ mod tests {
assert_eq!(english_source(l), Some(EnglishSource::Embedded));
assert_eq!(dictionary_load_error(l), None);
}
assert!(std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::from_tag(Some("en-AU"))).unwrap()), "en-AU reads en-GB");
assert!(!std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::EnAu).unwrap()), "en-AU reads its own");
assert!(std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::from_tag(Some("en-NZ"))).unwrap()), "en-NZ reads en-GB");
let notice = scowl::notice().unwrap();
for credit in ["Kevin Atkinson", "Geoff Kuenning", "clause 4 removed", "Benjamin Titze"] {
assert!(notice.contains(credit), "{credit}");
crates/proof/src/no_data.rs
+10 -4
View File
@@ -7,9 +7,10 @@
use crate::hyphen::{self, Limits, Source};
use crate::{EnglishSource, Options, ProofLang, dict, en_variants, patterns};
const LANGS: [ProofLang; 18] = [
const LANGS: [ProofLang; 19] = [
ProofLang::En,
ProofLang::EnGb,
ProofLang::EnAu,
ProofLang::EnCa,
ProofLang::Pl,
ProofLang::Bg,
@@ -45,9 +46,14 @@ const SAMPLES: [&str; 10] = [
#[test]
fn nothing_is_embedded() {
assert!(crate::data::EN_US_DIC.is_empty());
assert!(crate::data::EN_US_SPELLING.is_empty() && crate::data::EN_GB_SPELLING.is_empty() && crate::data::EN_CA_SPELLING.is_empty());
assert!(
crate::data::EN_US_SPELLING.is_empty()
&& crate::data::EN_GB_SPELLING.is_empty()
&& crate::data::EN_CA_SPELLING.is_empty()
&& crate::data::EN_AU_SPELLING.is_empty()
);
assert!(crate::scowl::notice().is_none());
for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] {
assert!(crate::scowl::dict(l).is_none() && crate::scowl::load_error(l).is_none());
}
assert!(dict::Dictionary::en_us().is_empty());
@@ -63,7 +69,7 @@ fn nothing_is_embedded() {
#[test]
fn english_spelling_still_marks_misspellings() {
crate::hold_dictionary_tests();
for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] {
let source = crate::english_source(l);
assert!(matches!(source, Some(EnglishSource::Hunspell(_) | EnglishSource::CoreList)), "{l:?}");
// Any English dictionary marks junk and knows `the`; the word-by-word results below are
crates/proof/src/pack_lang.rs
+154 -17
View File
@@ -6,6 +6,10 @@
//! files); the one used least recently goes first, but never one used during the current
//! [`in_one_check`] call, so a paragraph in many languages loads each dictionary once. Locales
//! with no dictionary are remembered apart and take no room.
//!
//! A word none of whose letters is in a script the dictionary's words are written in (a Cyrillic
//! word in a Dutch run) isn't this language's: it is never marked, as Word leaves a word in another
//! script alone. A word in the dictionary's script is always checked, whatever its letters.
use std::cell::Cell;
use std::path::PathBuf;
@@ -20,7 +24,7 @@ use crate::lang::PackLocale;
pub(crate) const MAX_LOADED_BYTES: u64 = 256 << 20;
/// Estimated memory per byte of `.aff` and `.dic` file.
const BYTES_PER_FILE_BYTE: u64 = 6;
/// Most locales remembered as having no dictionary (the list is cleared when full).
/// Most locales remembered as having no dictionary (the oldest is forgotten first when full).
const MAX_MISSING: usize = 256;
/// The budget in force (tests lower it).
@@ -36,6 +40,8 @@ thread_local! {
struct Loaded {
locale: PackLocale,
dict: Arc<AffixDict>,
/// The scripts its words are written in ([`scripts`]), sorted.
scripts: Arc<[u32]>,
/// The `.dic` it came from.
file: Option<PathBuf>,
/// Estimated memory.
@@ -66,15 +72,88 @@ pub(crate) fn reload() {
/// over the budget.
pub(crate) fn in_one_check<R>(f: impl FnOnce() -> R) -> R {
let outer = CHECK_FROM.with(Cell::get);
if outer.is_none() {
let _end = outer.is_none().then(|| {
let now = CACHE.read().unwrap_or_else(PoisonError::into_inner).clock;
CHECK_FROM.with(|c| c.set(Some(now.wrapping_add(1))));
}
let r = f();
if outer.is_none() {
EndCheck
});
f()
}
/// Ends the outermost [`in_one_check`] when dropped: also when `f` panics (`Session::run` catches
/// it), or every dictionary would stay pinned on this thread and the budget never hold again.
struct EndCheck;
impl Drop for EndCheck {
fn drop(&mut self) {
CHECK_FROM.with(|c| c.set(None));
}
r
}
/// Remember that `locale` has no dictionary; when [`MAX_MISSING`] are remembered, the oldest is
/// forgotten (clearing them all would look every other one up again).
fn remember_missing(missing: &mut Vec<PackLocale>, locale: PackLocale) {
if missing.len() >= MAX_MISSING {
missing.remove(0);
}
missing.push(locale);
}
/// The script of the letter `c`, as a number: the main alphabetic scripts by their Unicode blocks
/// (Latin with its extensions, Greek, Cyrillic, Armenian, Hebrew, Arabic, Georgian, Hangul, kana,
/// Han, Ethiopic); any other letter by its 128-character block. Only told apart, never named.
fn script_of(c: char) -> u32 {
const SCRIPTS: &[(u32, u32, u32)] = &[
(0x0041, 0x024F, 1),
(0x0250, 0x02AF, 1),
(0x1E00, 0x1EFF, 1),
(0x2C60, 0x2C7F, 1),
(0xA720, 0xA7FF, 1),
(0xAB30, 0xAB6F, 1),
(0xFF21, 0xFF5A, 1),
(0x0370, 0x03FF, 2),
(0x1F00, 0x1FFF, 2),
(0x0400, 0x052F, 3),
(0x1C80, 0x1C8F, 3),
(0x2DE0, 0x2DFF, 3),
(0xA640, 0xA69F, 3),
(0x0530, 0x058F, 4),
(0xFB13, 0xFB17, 4),
(0x0590, 0x05FF, 5),
(0xFB1D, 0xFB4F, 5),
(0x0600, 0x06FF, 6),
(0x0750, 0x077F, 6),
(0x08A0, 0x08FF, 6),
(0xFB50, 0xFDFF, 6),
(0xFE70, 0xFEFF, 6),
(0x10A0, 0x10FF, 7),
(0x1C90, 0x1CBF, 7),
(0x2D00, 0x2D2F, 7),
(0x1100, 0x11FF, 8),
(0x3130, 0x318F, 8),
(0xAC00, 0xD7AF, 8),
(0x3040, 0x30FF, 9),
(0x3400, 0x4DBF, 10),
(0x4E00, 0x9FFF, 10),
(0x1200, 0x139F, 11),
];
let u = u32::from(c);
SCRIPTS.iter().find(|(lo, hi, _)| (*lo..=*hi).contains(&u)).map_or(0x100 + (u >> 7), |(_, _, s)| *s)
}
/// The scripts the words of `d` are written in ([`script_of`] of their letters), sorted.
fn scripts(d: &AffixDict) -> Arc<[u32]> {
let mut seen = std::collections::BTreeSet::new();
for c in d.stems().flat_map(str::chars).filter(|c| c.is_alphabetic()) {
seen.insert(script_of(c));
}
seen.into_iter().collect()
}
/// Is `word` in another script than the dictionary's (`scripts`): none of its letters is in one
/// of them? A word in the dictionary's script is never, whatever its letters (`zzzz` in Dutch).
fn other_script(word: &str, scripts: &[u32]) -> bool {
!scripts.is_empty() && !word.chars().filter(|c| c.is_alphabetic()).any(|c| scripts.binary_search(&script_of(c)).is_ok())
}
#[cfg(test)]
@@ -149,30 +228,34 @@ pub(crate) fn names(locale: PackLocale) -> Vec<String> {
/// `locale`'s dictionary, loaded on first use; an empty one when there is none.
fn get(locale: PackLocale) -> (Arc<AffixDict>, Option<PathBuf>) {
let (d, file, _) = get_with_scripts(locale);
(d, file)
}
/// [`get`], with the dictionary's [`scripts`].
fn get_with_scripts(locale: PackLocale) -> (Arc<AffixDict>, Option<PathBuf>, Arc<[u32]>) {
let reloads = {
let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner);
c.clock = c.clock.wrapping_add(1);
let now = c.clock;
if let Some(l) = c.loaded.iter_mut().find(|l| l.locale == locale) {
l.used = now;
return (l.dict.clone(), l.file.clone());
return (l.dict.clone(), l.file.clone(), l.scripts.clone());
}
if c.missing.contains(&locale) {
return (Arc::default(), None);
return (Arc::default(), None, Arc::from([]));
}
c.reloads
};
let (dict, file, bytes) = load(locale);
let scripts = scripts(&dict);
let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner);
if c.reloads != reloads || c.loaded.iter().any(|l| l.locale == locale) || c.missing.contains(&locale) {
return (dict, file);
return (dict, file, scripts);
}
if dict.is_empty() {
if c.missing.len() >= MAX_MISSING {
c.missing.clear();
}
c.missing.push(locale);
return (dict, file);
remember_missing(&mut c.missing, locale);
return (dict, file, scripts);
}
LOADS.fetch_add(1, Ordering::Relaxed);
let budget = BUDGET.load(Ordering::Relaxed);
@@ -187,8 +270,8 @@ fn get(locale: PackLocale) -> (Arc<AffixDict>, Option<PathBuf>) {
total = total.saturating_sub(gone.bytes);
}
let used = c.clock;
c.loaded.push(Loaded { locale, dict: dict.clone(), file: file.clone(), bytes, used });
(dict, file)
c.loaded.push(Loaded { locale, dict: dict.clone(), scripts: scripts.clone(), file: file.clone(), bytes, used });
(dict, file, scripts)
}
/// Load `locale`'s dictionary, with its estimated memory.
@@ -229,11 +312,14 @@ pub(crate) fn load_error(locale: PackLocale) -> Option<String> {
/// Is `word` right in `locale`? Every word is when there is no dictionary.
pub(crate) fn is_correct(word: &str, locale: PackLocale, opts: &crate::Options) -> bool {
let (d, _) = get(locale);
let (d, _, scripts) = get_with_scripts(locale);
if d.is_empty() {
return true;
}
let Some(w) = crate::spelling::word_to_check_in(&d, word, opts) else { return true };
if other_script(w, &scripts) {
return true;
}
let known = |w: &str| d.check(w) || crate::in_user_dictionary(&w.to_lowercase());
if w.contains('-') && !known(w) {
return w.split('-').all(|p| p.chars().count() <= 1 || known(p));
@@ -264,6 +350,57 @@ mod tests {
PackLocale::new(t).unwrap()
}
/// Review follow-up (craft-libs#485): a full list of locales with no dictionary forgets its
/// oldest one, not all of them, so a document in more languages doesn't look every one up again.
#[test]
fn missing_locales_forget_the_oldest_first() {
let tag = |i: usize| l(&format!("q{}{}", (b'a' + (i / 26 % 26) as u8) as char, (b'a' + (i % 26) as u8) as char));
let mut missing = Vec::new();
for i in 0..MAX_MISSING + 10 {
remember_missing(&mut missing, tag(i));
assert!(missing.len() <= MAX_MISSING);
}
assert_eq!(missing.len(), MAX_MISSING);
assert!(!missing.contains(&tag(0)) && !missing.contains(&tag(9)), "the oldest went first");
assert!(missing.contains(&tag(10)) && missing.contains(&tag(MAX_MISSING + 9)), "the rest stay");
}
/// Review follow-up (craft-libs#485): a panic inside a check (caught by the app) doesn't leave
/// this thread's dictionaries pinned past the budget for good.
#[test]
fn a_panic_inside_a_check_ends_it() {
let caught = std::panic::catch_unwind(|| in_one_check(|| panic!("a bug inside a check")));
assert!(caught.is_err());
assert_eq!(CHECK_FROM.with(Cell::get), None);
in_one_check(|| {
assert!(CHECK_FROM.with(Cell::get).is_some());
in_one_check(|| ());
assert!(CHECK_FROM.with(Cell::get).is_some(), "an inner check doesn't end the outer one");
});
assert_eq!(CHECK_FROM.with(Cell::get), None);
}
/// Review follow-up (craft-libs#485): a word with no letter in the dictionary's scripts is
/// in another script (a Cyrillic word in a Dutch run); a word in its script is always checked,
/// even with letters no dictionary word uses (CodeRabbit on #496: `zzzz`).
#[test]
fn words_in_another_script_are_not_the_dictionarys() {
let nl = AffixDict::from_hunspell("SET UTF-8\n", "3\nhuis\nkat\nÉén\n", &[]).unwrap();
let latin = scripts(&nl);
assert_eq!(latin.len(), 1);
for w in ["привет", "Привет", "αβγ", "שלום", "日本"] {
assert!(other_script(w, &latin), "{w}");
}
for w in ["huus", "kät", "huisп", "ÉÉN", "zzzz", "qxqw", "ŋʒ"] {
assert!(!other_script(w, &latin), "{w}");
}
assert!(!other_script("привет", &[]), "no scripts: every word is checked");
let ru = AffixDict::from_hunspell("SET UTF-8\n", "1\nдом\n", &[]).unwrap();
assert!(!other_script("щщщ", &scripts(&ru)) && other_script("dom", &scripts(&ru)));
assert_ne!(script_of('ა'), script_of('a'));
assert_eq!(script_of('ă'), script_of('Ạ'));
}
#[test]
fn locales_and_file_names() {
assert_eq!(l("nl-NL").as_str(), "nl-nl");
crates/proof/src/packs.rs
+56 -7
View File
@@ -34,6 +34,13 @@ pub(crate) const SPELLING: &str = "spelling";
const HYPHENATION: &str = "hyphenation.pat";
/// Most languages whose pack patterns are remembered (a document may tag thousands).
const MAX_HYPHENATION_LOOKUPS: usize = 256;
/// The memory budget of the remembered pack patterns, in estimated bytes ([`pattern_bytes`]):
/// a pack's patterns may be as large as a dictionary (64 MB of file), and up to
/// [`MAX_HYPHENATION_LOOKUPS`] of them are kept, so they get a budget of their own, as the
/// dictionaries do (`pack_lang::MAX_LOADED_BYTES`).
const MAX_HYPHENATION_BYTES: u64 = 256 << 20;
/// Estimated memory per pattern (its packed key, its values and the map's own room).
const BYTES_PER_PATTERN: u64 = 64;
/// A pack's description.
const PACK_TOML: &str = "pack.toml";
/// A pack's licence texts.
@@ -80,15 +87,18 @@ struct State {
dir: Option<PathBuf>,
scan: Option<Arc<Scan>>,
/// Each language's pack patterns once looked up (`None`: no pack has them), at most
/// [`MAX_HYPHENATION_LOOKUPS`] languages (cleared when full).
/// [`MAX_HYPHENATION_LOOKUPS`] languages and [`MAX_HYPHENATION_BYTES`] (cleared when full).
hyphenation: BTreeMap<ProofLang, Option<Arc<Patterns>>>,
/// The estimated memory of the patterns in [`State::hyphenation`].
hyphenation_bytes: u64,
/// (locale, reason) of pack hyphenation files that failed to load.
hyphenation_errors: Vec<(String, String)>,
/// Counts changes, so a scan or load that started before one is not kept.
changes: u64,
}
static STATE: RwLock<State> = RwLock::new(State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_errors: Vec::new(), changes: 0 });
static STATE: RwLock<State> =
RwLock::new(State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_bytes: 0, hyphenation_errors: Vec::new(), changes: 0 });
fn read() -> RwLockReadGuard<'static, State> {
STATE.read().unwrap_or_else(PoisonError::into_inner)
@@ -121,6 +131,7 @@ pub fn reload() {
fn forget(s: &mut State) {
s.scan = None;
s.hyphenation.clear();
s.hyphenation_bytes = 0;
s.hyphenation_errors.clear();
s.changes = s.changes.wrapping_add(1);
}
@@ -532,7 +543,7 @@ fn hyphenation_locales(lang: ProofLang) -> Vec<String> {
ProofLang::EsUy => &["es-uy", "es-419", "es"],
ProofLang::De => &["de", "de-1996", "de-de"],
ProofLang::Pack(l) => return crate::pack_lang::hyphenation_locales(l),
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa | ProofLang::Other => &[],
ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa | ProofLang::Other => &[],
};
names.iter().map(|n| (*n).to_string()).collect()
}
@@ -567,10 +578,8 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option<Arc<Patterns>> {
let mut s = write();
if s.changes == changes && !s.hyphenation.contains_key(&lang) {
// ponytail: cleared when full, an LRU if many languages each need their patterns.
if s.hyphenation.len() >= MAX_HYPHENATION_LOOKUPS {
s.hyphenation.clear();
}
s.hyphenation.insert(lang, found.clone());
let bytes = found.as_deref().map_or(0, pattern_bytes);
remember_patterns(&mut s, lang, found.clone(), bytes, MAX_HYPHENATION_BYTES);
for e in errors {
if !s.hyphenation_errors.contains(&e) {
s.hyphenation_errors.push(e);
@@ -580,6 +589,23 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option<Arc<Patterns>> {
found
}
/// Estimated memory of `p`.
fn pattern_bytes(p: &Patterns) -> u64 {
(p.len() as u64).saturating_add(p.exception_count() as u64).saturating_mul(BYTES_PER_PATTERN)
}
/// Remember `lang`'s pack patterns (`bytes` of them): over [`MAX_HYPHENATION_LOOKUPS`] languages
/// or `budget` bytes, the ones remembered so far are forgotten first. Patterns larger than the
/// whole budget are still kept alone, or they would be read from disk for every word.
fn remember_patterns(s: &mut State, lang: ProofLang, found: Option<Arc<Patterns>>, bytes: u64, budget: u64) {
if s.hyphenation.len() >= MAX_HYPHENATION_LOOKUPS || s.hyphenation_bytes.saturating_add(bytes) > budget {
s.hyphenation.clear();
s.hyphenation_bytes = 0;
}
s.hyphenation.insert(lang, found);
s.hyphenation_bytes = s.hyphenation_bytes.saturating_add(bytes);
}
/// How many languages' pack patterns are remembered now.
#[cfg(test)]
pub(crate) fn hyphenation_lookups() -> usize {
@@ -601,6 +627,29 @@ fn load_patterns(path: &Path) -> Result<Patterns, String> {
mod tests {
use super::*;
/// Review follow-up (craft-libs#485): the remembered pack patterns have a memory budget, not
/// only a count, so 256 languages of large patterns can't each stay in memory.
#[test]
fn remembered_patterns_keep_to_a_memory_budget() {
let mut s = State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_bytes: 0, hyphenation_errors: Vec::new(), changes: 0 };
let p = Arc::new(Patterns::parse("hy3ph he2n hena4 hen5at 1na n2at 1tio 2io o2n"));
let bytes = pattern_bytes(&p);
assert_eq!(bytes, 9 * BYTES_PER_PATTERN);
let lang = |t: &str| ProofLang::from_tag(Some(t));
let budget = bytes * 2;
remember_patterns(&mut s, lang("nl"), Some(p.clone()), bytes, budget);
remember_patterns(&mut s, lang("fr"), Some(p.clone()), bytes, budget);
remember_patterns(&mut s, lang("sv"), None, 0, budget);
assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (3, budget), "a language with no patterns takes no room");
remember_patterns(&mut s, lang("da"), Some(p.clone()), bytes, budget);
assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (1, bytes), "over the budget: the others are forgotten");
assert!(s.hyphenation.contains_key(&lang("da")));
remember_patterns(&mut s, lang("fi"), Some(p.clone()), budget * 4, budget);
assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (1, budget * 4), "patterns over the whole budget are kept alone");
forget(&mut s);
assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (0, 0));
}
/// Review of #485: a pack's name loses bidirectional and other format characters.
#[test]
fn pack_names_are_plain_text() {
crates/proof/src/pl/grammar.rs
+2 -5
View File
@@ -304,11 +304,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Zbędna spacja przed znakiem interpunkcyjnym".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Podwójna spacja".into(), vec![" ".into()]));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Podwójna spacja".into(), vec![" ".into()]));
}
}
crates/proof/src/pt/grammar.rs
+16 -6
View File
@@ -62,7 +62,12 @@ fn ends_sentence(gap: &str, prev: &str) -> bool {
if end == '.' && abbreviation_or_number(prev) {
return false;
}
let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING);
let after_marks = t.trim_start_matches(['.', '!', '?']);
let rest = after_marks.trim_start_matches(CLOSING);
// `"Vai?" disse ele.` (not the object placeholder `\u{1}`): narration after a quoted question or exclamation goes on in lower case.
if end != '.' && after_marks.trim_start_matches('\u{1}').starts_with(CLOSING) {
return false;
}
// A dialogue dash after the mark brings back the narration (`—Vai? —disse ele.`), which goes
// on in lower case.
rest.starts_with(char::is_whitespace) && !rest.contains('—') && rest.trim_start().chars().all(|c| OPENING.contains(&c))
@@ -124,11 +129,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Espaço antes da pontuação".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Espaço duplo".into(), vec![" ".into()]));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Espaço duplo".into(), vec![" ".into()]));
}
}
@@ -225,6 +227,14 @@ mod tests {
// craft-libs#488 review: no space after the dash.
clean("—Vai? —disse ele.");
clean("— Vai? — disse ele.");
// craft-libs#496 review: a closing quote after the mark.
clean("\"Vai?\" disse ele.");
clean("«Vai!» gritou ela.");
assert!(!messages("Vai? \"sim\".").is_empty(), "a bare question still ends the sentence");
// An inline object (`\u{1}`) before a real closing quote is skipped.
clean("\"Vai?\u{1}\" disse ele.");
// An inline object (`\u{1}`) is not a closing quote.
assert!(!messages("Quem era?\u{1} era Marco.").is_empty());
clean("Ele disse: \"Volto amanhã\". E saiu.");
// A list item may start in lower case.
clean("maçãs,");
crates/proof/src/ru/grammar.rs
+2 -5
View File
@@ -503,11 +503,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Лишний пробел перед знаком препинания".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Двойной пробел".into(), vec![" ".into()]));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Двойной пробел".into(), vec![" ".into()]));
}
}
crates/proof/src/ru/mod.rs
+4 -2
View File
@@ -105,7 +105,7 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS {
if opts.ignore_uppercase && crate::spelling::in_capitals(w) && n <= ABBREVIATION_LETTERS {
return true;
}
// Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word.
@@ -333,7 +333,7 @@ fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool {
let last = parts.len() - 1;
parts.iter().enumerate().all(|(i, p)| {
// An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word.
if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) {
if known(d, p) || (opts.ignore_uppercase && crate::spelling::capitals_part(p, ABBREVIATION_LETTERS)) {
return true;
}
// A first part with a linking vowel: the stem (`северо` → `север`) or its adjective
@@ -485,5 +485,7 @@ mod tests {
assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too");
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults");
// The cased-letter rule (craft-libs#485): capitals and letterless parts are abbreviations.
assert!(crate::is_correct_in_with("ИИ-2", lang, &check_digits) && crate::is_correct_in_with("ТУ-154М", lang, &check_digits));
}
}
crates/proof/src/scowl.rs
+17 -9
View File
@@ -2,10 +2,15 @@
//! dictionaries of craft-i18n's `language-data/en-us`, `en-gb` (British `-ise`) and `en-ca`,
//! embedded unmodified at build time and read by the affix engine ([`crate::affix`]) on first use.
//!
//! Each English has its own: American English reads en-US, the British-spelling Englishes
//! (en-GB, en-AU, en-NZ, en-IE…) read en-GB and Canadian English reads en-CA. Its suggestions
//! Each English has its own: American English reads en-US, Australian English en-AU (SCOWL's
//! Australian words: `billabong`, `brumby`; and `program`, `inquiry`), the other British-spelling
//! Englishes (en-GB, en-NZ, en-IE…) read en-GB and Canadian English reads en-CA. SCOWL has no
//! en-NZ: New Zealand English keeps `programme` and has few words of its own in SCOWL's en-AU
//! data, so it stays on en-GB. Its suggestions
//! come from its own dictionary, and a word is right when its own dictionary knows it: the other
//! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's). SCOWL
//! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's), except
//! that en-AU's list keeps one spelling per word, so Australian English also takes en-GB's words
//! (`enquiry`; its suggestions still come from en-AU). SCOWL
//! picks one spelling per dialect (en-GB `-ise` has no `organize`, en-US no `judgement`), and Word
//! accepts more, so British English also takes the `-ize` spellings of its `-ise` words that
//! en-US or en-CA spells so, and `en-variants.txt` widens each dialect by its pairs
@@ -19,16 +24,17 @@ use std::sync::OnceLock;
use crate::ProofLang;
use crate::affix::AffixDict;
use crate::data::{EN_CA_AFF, EN_CA_SPELLING, EN_GB_AFF, EN_GB_SPELLING, EN_US_AFF, EN_US_SPELLING, SCOWL_NOTICE};
use crate::data::{EN_AU_AFF, EN_AU_SPELLING, EN_CA_AFF, EN_CA_SPELLING, EN_GB_AFF, EN_GB_SPELLING, EN_US_AFF, EN_US_SPELLING, SCOWL_NOTICE};
/// Per English (American, British, Canadian): the loaded dictionary, or why it isn't there.
static DICTS: [OnceLock<Result<AffixDict, String>>; 3] = [OnceLock::new(), OnceLock::new(), OnceLock::new()];
/// Per English (American, British, Canadian, Australian): the loaded dictionary, or why it isn't there.
static DICTS: [OnceLock<Result<AffixDict, String>>; 4] = [OnceLock::new(), OnceLock::new(), OnceLock::new(), OnceLock::new()];
/// The slot, `.aff` and `.dic` text of `lang`'s dictionary.
fn source(lang: ProofLang) -> (usize, &'static str, &'static str) {
match lang {
ProofLang::EnGb => (1, EN_GB_AFF, EN_GB_SPELLING),
ProofLang::EnCa => (2, EN_CA_AFF, EN_CA_SPELLING),
ProofLang::EnAu => (3, EN_AU_AFF, EN_AU_SPELLING),
_ => (0, EN_US_AFF, EN_US_SPELLING),
}
}
@@ -50,15 +56,17 @@ pub(crate) fn dict(lang: ProofLang) -> Option<&'static AffixDict> {
loaded.as_ref().ok()
}
/// Does `lang` read the British `-ise` dictionary (en-GB, en-AU, en-NZ…)?
/// Does `lang` read a British `-ise` dictionary (en-GB, and en-NZ… through it; en-AU)?
pub(crate) fn is_british(lang: ProofLang) -> bool {
source(lang).0 == 1
matches!(source(lang).0, 1 | 3)
}
/// Why `lang`'s embedded SCOWL dictionary failed to load, once something tried to load it.
pub(crate) fn load_error(lang: ProofLang) -> Option<String> {
let (i, ..) = source(lang);
DICTS.get(i)?.get()?.as_ref().err().map(|e| format!("embedded SCOWL dictionary: {e}"))
let own = DICTS.get(i).and_then(|d| d.get()).and_then(|r| r.as_ref().err()).map(|e| format!("embedded SCOWL dictionary: {e}"));
// Australian English also reads the en-GB list (`crate::english`): its failure is reported too.
own.or_else(|| if lang == ProofLang::EnAu { load_error(ProofLang::EnGb).map(|e| format!("{e} (en-GB, also read for en-AU)")) } else { None })
}
/// The SCOWL copyright and licence notices (the upstream README, with the Ispell BSD licence of
crates/proof/src/spelling.rs
+28 -2
View File
@@ -32,17 +32,43 @@ pub(crate) fn word_to_check_in<'a>(d: &AffixDict, word: &'a str, opts: &Options)
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
return None;
}
// Only a word with a cased letter can be in capitals: Hebrew, Arabic, Thai… words have none.
if opts.ignore_uppercase && w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
if opts.ignore_uppercase && in_capitals(w) && n <= 6 {
return None;
}
Some(w)
}
/// Is `w` in capitals: a cased letter and no lower-case one? Only a word with a cased letter can
/// be: Hebrew, Arabic, Thai, Chinese… words have none, so they are never taken for acronyms.
pub(crate) fn in_capitals(w: &str) -> bool {
w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase())
}
/// Is `part` of a hyphenated word an abbreviation Ignore words in UPPERCASE skips: at most `max`
/// characters, [`in_capitals`] or without letters (`SMS` in `SMS-ка`, `19` in `COVID-19`)? A
/// part in a caseless script is no abbreviation.
pub(crate) fn capitals_part(part: &str, max: usize) -> bool {
part.chars().count() <= max && (in_capitals(part) || !part.chars().any(char::is_alphabetic))
}
#[cfg(test)]
mod tests {
use super::*;
/// craft-libs#485's cased-letter rule, shared by every language: a word or part is in
/// capitals only with a cased letter; a part without letters counts as an abbreviation.
#[test]
fn capitals_need_a_cased_letter() {
for w in ["PKO", "SMS", "ÉTÉ", "ВКП", "B2B"] {
assert!(in_capitals(w) && capitals_part(w, 6), "{w}");
}
for w in ["日本", "שלום", "ไทย", "Pko", "pko", "123", ""] {
assert!(!in_capitals(w), "{w}");
}
assert!(capitals_part("19", 6) && capitals_part("", 6));
assert!(!capitals_part("日本", 6) && !capitals_part("ไทย", 6) && !capitals_part("ABCDEFG", 6));
}
#[test]
fn words_needing_no_lookup() {
let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap();
crates/proof/src/tr/grammar.rs
+10 -16
View File
@@ -211,23 +211,17 @@ pub fn check_with(text: &str, capital_first: bool) -> Vec<Issue> {
});
}
}
// Double space
if c == ' ' && i + 1 < chars.len() && chars[i + 1].1 == ' ' && (i == 0 || chars[i - 1].1 != ' ') {
let mut end = idx + 1;
while end < text.len() && text.as_bytes().get(end) == Some(&b' ') {
end += 1;
}
v.push(Issue {
start: idx,
end,
kind: IssueKind::Grammar,
message: "Fazladan boşluk".into(),
suggestions: vec![" ".into()],
lang: ProofLang::Tr,
});
}
}
for run in crate::space_runs(text) {
v.push(Issue {
start: run.start,
end: run.end,
kind: IssueKind::Grammar,
message: "Fazladan boşluk".into(),
suggestions: vec![" ".into()],
lang: ProofLang::Tr,
});
}
// 3. Comma before or after conjunctions (`ali, ve veli` or `ve, veli`)
for (i, &(w_start, w_end)) in ws.iter().enumerate() {
let word = match text.get(w_start..w_end) {
crates/proof/src/tr/suggest.rs
+15 -2
View File
@@ -97,8 +97,8 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec<String> {
}
}
// 3. Detached suffix check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü`
for suffix in ["de", "da", "te", "ta", "mi", "mı", "mu", "mü"] {
// 3. Detached particle check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü`
for suffix in ["de", "da", "mi", "mı", "mu", "mü"] {
if lower.ends_with(suffix) && lower.len() > suffix.len() + 1 {
let stem = &lower[..lower.len() - suffix.len()];
if d.is_empty() || d.contains(stem) {
@@ -333,6 +333,19 @@ mod tests {
assert!(sugs.contains(&"geldin mi".to_string()));
}
/// craft-libs#496 review: the particle is `de`/`da` whatever consonant comes before it. `te`
/// and `ta` are only the locative suffix after a voiceless consonant (`kitapta`), written
/// attached, so `kitapte` is never offered as `kitap te`.
#[test]
fn the_detached_particle_is_never_te_or_ta() {
let d = TrDict::from_hunspell("SET UTF-8\n", "2\nkitap\nsonra\n").unwrap();
assert!(suggest_with(&d, "kitapde", 8).contains(&"kitap de".to_string()));
for w in ["kitapte", "kitapta"] {
let sugs = suggest_with(&d, w, 8);
assert!(!sugs.iter().any(|s| s == "kitap te" || s == "kitap ta"), "{w}: {sugs:?}");
}
}
#[test]
fn overlong_words_get_no_suggestions_promptly() {
let d = TrDict::from_hunspell("SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n", "2\nkitap/1\nsonra\n").unwrap();
crates/ui/README.md
+39
View File
@@ -11,6 +11,45 @@ The current delivery covers nine creative apps (DesignCraft, VectorCraft, PhotoC
records their scope and acceptance limits. Assets (font files included), complete themes, localization
and command handling remain in each application. No icons, fonts or application engines are bundled.
## Host-owned docking chrome
`DockContent` can replace an iconic column or a complete tab group without a
second layout tree. `column_rect` reports the fitted column before painting;
`owns_iconic_column`/`iconic_column` and `custom_group` suppress the corresponding
stock chrome, body and input when the host takes ownership. Docked custom groups
also skip stock body top/bottom drop bands; column-edge targets remain available.
These hooks are also
forwarded by `show_customized`'s label/permissions/limits adapter.
`resize_handle_style` and `split_separator` let the host paint one seam while
retaining stock resize input. `owns_floating_width` delegates horizontal floating
frame-edge handles to the host; internal dividers and vertical handles remain available.
`owns_floating_split_width` separately delegates internal horizontal dividers when
the host supplies their replacement input. Presentation-only
`group_collapsed` does not change the selected tab in the shared model.
All ownership hooks default to false/no-op. `DockStyle::dock_gap` defaults to
`None`, preserving the ordinary split gap; setting it affects side-dock seams
only. `FloatingFrame::resize_axes` defaults to both axes and preserves an enabled
axis at corners when the other is disabled. A host with its own title bar can
disable the separate top gutter through `reserve_top_resize_band(false)`.
Compatibility and target-size audit: before these hooks, resizable frames already
reserved a 4 pt top band. That remains the default; `false` is the new host opt-out.
The unchanged stock edge hit bands are 4 pt thick; corners are at most 12 pt.
They do **not** meet a 24-by-24 logical-pixel target at 1x. This PR preserves
those hit rectangles and does not claim accessibility compliance. Focused
resize handles also support arrow keys (8 pt steps). Enlarging pointer targets
needs separate overlap, title-bar and viewport-edge acceptance checks.
`chrome::compatibility_tests::defaults_preserve_pre_host_hook_frame_geometry`
locks down the pre-hook hit rectangles/defaults. `host_hooks_tests` exercises all
four axis settings with actual corner and side pointer drags, minimum/maximum
sizes, the viewport edge, and top-gutter opt-out geometry.
Exhaustive `DockStyle` struct literals need `dock_gap: None`; constructors such
as `from_ui` and `from_visuals` supply the compatible default.
## SVG catalogs
```rust
crates/ui/src/docking/chrome.rs
+68 -4
View File
@@ -116,8 +116,10 @@ pub struct FloatingFrame<'a> {
bounds: Option<Rect>,
limits: PanelLimits,
resizable: bool,
resize_axes: [bool; 2],
resize_label: &'a str,
opacity: f32,
reserve_top_resize_band: bool,
}
/// What a [`FloatingFrame`] did this frame.
@@ -137,7 +139,17 @@ pub struct FrameResponse<R> {
impl<'a> FloatingFrame<'a> {
pub fn new(id: Id, rect: Rect) -> Self {
Self { id, rect, bounds: None, limits: PanelLimits::default(), resizable: true, resize_label: "Resize window", opacity: 1.0 }
Self {
id,
rect,
bounds: None,
limits: PanelLimits::default(),
resizable: true,
resize_axes: [true, true],
resize_label: "Resize window",
opacity: 1.0,
reserve_top_resize_band: true,
}
}
/// The frame stays inside `bounds`; the default is the viewport's content rect.
@@ -153,11 +165,23 @@ impl<'a> FloatingFrame<'a> {
self
}
/// Reserve a separate top resize gutter; disable when contents already supply a header.
pub fn reserve_top_resize_band(mut self, reserve: bool) -> Self {
self.reserve_top_resize_band = reserve;
self
}
pub fn resizable(mut self, resizable: bool) -> Self {
self.resizable = resizable;
self
}
/// Restrict the stock edge handles when a host owns resizing on one axis.
pub fn resize_axes(mut self, axes: [bool; 2]) -> Self {
self.resize_axes = axes;
self
}
/// Accessibility label of the resize handles.
pub fn resize_label(mut self, label: &'a str) -> Self {
self.resize_label = label;
@@ -192,8 +216,11 @@ impl<'a> FloatingFrame<'a> {
ui.set_min_size(rect.size());
ui.allocate_rect(rect, Sense::hover());
ui.painter().rect_filled(rect, 0.0, style.background);
let contents =
if self.resizable { Rect::from_min_max(pos2(rect.left(), (rect.top() + RESIZE_BAND).min(rect.bottom())), rect.max) } else { rect };
let contents = if self.resizable && self.reserve_top_resize_band {
Rect::from_min_max(pos2(rect.left(), (rect.top() + RESIZE_BAND).min(rect.bottom())), rect.max)
} else {
rect
};
let inner = add_contents(ui, contents);
ui.painter().rect_stroke(rect, 0.0, style.stroke, StrokeKind::Outside);
let (resized, resize_started, handles) = if self.resizable { self.handles(ui, rect, bounds, style) } else { (None, None, Vec::new()) };
@@ -217,7 +244,14 @@ impl<'a> FloatingFrame<'a> {
let mut responses = Vec::new();
let limits = PanelLimits { min: self.limits.min.max(vec2(40.0, RESIZE_BAND + TITLE_HEIGHT + 20.0)).min(self.limits.max), ..self.limits };
let canceled = ui.input(|input| input.key_pressed(Key::Escape)) || crate::focus::lost(ui.ctx());
for (name, handle, direction) in handles(rect) {
for (name, mut handle, mut direction) in handles(rect) {
if !self.reserve_top_resize_band && direction.y < 0.0 {
handle.max.y = (rect.top() + ui.ctx().pixels_per_point().recip()).min(rect.bottom());
}
direction = vec2(if self.resize_axes[0] { direction.x } else { 0.0 }, if self.resize_axes[1] { direction.y } else { 0.0 });
if direction == egui::Vec2::ZERO {
continue;
}
if !super::targets::usable(handle) {
continue;
}
@@ -486,3 +520,33 @@ pub fn live_move(ctx: &Context, bar: &Response, rect: Rect, bounds: Rect) -> Opt
forget();
None
}
#[cfg(test)]
mod compatibility_tests {
use super::*;
#[test]
fn defaults_preserve_pre_host_hook_frame_geometry() {
let rect = Rect::from_min_size(pos2(100.0, 100.0), vec2(300.0, 200.0));
let frame = FloatingFrame::new(Id::unique("defaults"), rect);
assert_eq!(frame.resize_axes, [true, true]);
assert!(frame.reserve_top_resize_band);
assert!(frame.resizable);
// Literal pre-PR hit rectangles: preserve these rather than silently enlarging them.
let expected = [
[388.0, 288.0, 400.0, 300.0],
[100.0, 112.0, 104.0, 288.0],
[396.0, 112.0, 400.0, 288.0],
[112.0, 296.0, 388.0, 300.0],
[100.0, 288.0, 112.0, 300.0],
[112.0, 100.0, 388.0, 104.0],
[100.0, 100.0, 112.0, 104.0],
[388.0, 100.0, 400.0, 104.0],
];
for ((_, hit, _), [x0, y0, x1, y1]) in handles(rect).into_iter().zip(expected) {
assert_eq!(hit, Rect::from_min_max(pos2(x0, y0), pos2(x1, y1)));
// Audit result: legacy targets do NOT meet a 24-by-24 logical-pixel target.
assert!(hit.width() < 24.0 || hit.height() < 24.0);
}
}
}
crates/ui/src/docking/host_hooks_tests.rs
+425
View File
@@ -0,0 +1,425 @@
//! Host-owned chrome uses the same fitted geometry and preserves stock defaults.
use super::*;
use egui::{Context, Event, Id, Modifiers, PointerButton, RawInput, Rect, pos2, vec2};
fn input(events: Vec<Event>) -> RawInput {
RawInput { screen_rect: Some(Rect::from_min_size(pos2(0.0, 0.0), vec2(800.0, 600.0))), focused: true, events, ..Default::default() }
}
fn button(at: egui::Pos2, pressed: bool) -> Event {
Event::PointerButton { pos: at, button: PointerButton::Primary, pressed, modifiers: Modifiers::NONE }
}
#[test]
fn corner_resize_keeps_each_enabled_axis_including_single_axis_hosts() {
for axes in [[true, true], [false, true], [true, false], [false, false]] {
let ctx = Context::default();
let rect = Rect::from_min_size(pos2(100.0, 100.0), vec2(300.0, 200.0));
let frame = |events| {
let mut resized = None;
ctx.run_ui(input(events), |ui| {
let style = chrome::ChromeStyle::from_ui(ui);
resized = chrome::FloatingFrame::new(Id::unique("axes"), rect).resize_axes(axes).show(ui.ctx(), &style, |_, _| ()).resized;
})
.drop_without_applying_deltas();
resized
};
frame(vec![]);
frame(vec![]);
let from = rect.right_bottom() - vec2(2.0, 2.0);
frame(vec![Event::PointerMoved(from), button(from, true)]);
frame(vec![Event::PointerMoved(from + vec2(10.0, 10.0))]);
let result = frame(vec![Event::PointerMoved(from + vec2(40.0, 35.0))]);
if axes == [false, false] {
assert!(result.is_none());
} else {
let expected = rect.size() + vec2(if axes[0] { 40.0 } else { 0.0 }, if axes[1] { 35.0 } else { 0.0 });
assert_eq!(result, Some(Rect::from_min_size(rect.min, expected)), "{axes:?}");
}
}
}
#[test]
fn the_top_gutter_is_reserved_by_default_and_can_be_owned_by_the_contents() {
for reserve in [true, false] {
let ctx = Context::default();
let rect = Rect::from_min_size(pos2(100.0, 100.0), vec2(300.0, 200.0));
let mut inner = Rect::NOTHING;
ctx.run_ui(input(vec![]), |ui| {
let style = chrome::ChromeStyle::from_ui(ui);
inner = chrome::FloatingFrame::new(Id::unique("gutter"), rect).reserve_top_resize_band(reserve).show(ui.ctx(), &style, |_, r| r).inner;
})
.drop_without_applying_deltas();
assert_eq!(inner.left(), rect.left());
assert_eq!(inner.max, rect.max);
assert_eq!(inner.top(), rect.top() + if reserve { chrome::RESIZE_BAND } else { 0.0 });
}
}
#[derive(Default)]
struct Host {
exclusive: bool,
stock_group: Option<u32>,
bodies: usize,
columns: Vec<Rect>,
iconic: Vec<Rect>,
groups: std::collections::HashMap<u32, Rect>,
collapsed: bool,
protected: bool,
frame_width: bool,
split_width: bool,
space: Option<FloatSpace<u32>>,
style_calls: std::cell::Cell<usize>,
}
impl DockContent<u32> for Host {
fn group_collapsed(&self, _: &[u32]) -> bool {
self.collapsed
}
fn permissions(&self, _: &u32) -> Permissions {
if self.protected { Permissions::PROTECTED } else { Permissions::default() }
}
fn owns_floating_split_width(&self, _: &u32) -> bool {
self.split_width
}
fn owns_floating_width(&self, _: &u32) -> bool {
self.frame_width
}
fn resize_handle_style(&self, _: crate::layout::ResizeHandleStyle) -> crate::layout::ResizeHandleStyle {
self.style_calls.set(self.style_calls.get() + 1);
crate::layout::ResizeHandleStyle {
hovered: egui::Color32::TRANSPARENT,
dragged: egui::Color32::TRANSPARENT,
focused: egui::Color32::TRANSPARENT,
}
}
fn body(&mut self, _: &mut egui::Ui, _: &u32) {
self.bodies += 1;
}
fn custom_group(&mut self, _: &mut egui::Ui, _: &[u32], selected: &u32, rect: Rect, _: bool) -> bool {
self.groups.insert(*selected, rect);
self.exclusive && self.stock_group.as_ref() != Some(selected)
}
fn owns_iconic_column(&self, _: &u32) -> bool {
self.exclusive
}
fn column_rect(&mut self, _: &Context, _: &Column<u32>, rect: Rect, _: Side) {
self.columns.push(rect);
}
fn iconic_column(&mut self, _: &mut egui::Ui, _: &Column<u32>, rect: Rect, _: Side) {
self.iconic.push(rect);
}
}
fn render(layout: &Layout<u32>, host: &mut Host) -> Vec<(Hit<u32>, Rect)> {
let ctx = Context::default();
for _ in 0..3 {
ctx.run_ui(input(vec![]), |ui| {
host.bodies = 0;
host.columns.clear();
host.iconic.clear();
let mut style = DockStyle::from_ui(ui);
super::floating_tests::frame_style(&mut style);
let protected = host.protected;
let shown = DockArea::new(Id::unique("host")).show_customized(
ui,
layout,
&style,
|id| format!("Pane {id}"),
|_| if protected { Permissions::PROTECTED } else { Permissions::default() },
|_| PanelLimits::default(),
host,
);
assert!(shown.error.is_none(), "{:?}", shown.error);
})
.drop_without_applying_deltas();
}
host.space = float_space(&ctx, Id::unique("host"));
DockArea::hit_rects(&ctx, Id::unique("host"))
}
#[test]
fn an_exclusive_group_replaces_the_stock_body_and_tab_targets_through_overrides() {
let layout = Layout { root: Some(Node::Tabs { panels: vec![1, 2], active: 0 }), floating: vec![] };
for exclusive in [false, true] {
let mut host = Host { exclusive, ..Default::default() };
let hits = render(&layout, &mut host);
assert_eq!(host.bodies, usize::from(!exclusive));
assert_eq!(hits.iter().any(|(hit, _)| matches!(hit, Hit::Tab(_))), !exclusive);
}
}
#[test]
fn an_iconic_host_receives_the_fitted_column_rect_without_a_second_stock_body() {
let mut column = Column::new(vec![Group::new(vec![1, 2], 0)], 900.0);
column.iconic = true;
let layout = Layout { root: Some(Node::Dock { side: Side::Start, columns: vec![column] }), floating: vec![] };
let mut host = Host { exclusive: true, ..Default::default() };
render(&layout, &mut host);
assert_eq!(host.columns.len(), 1);
assert_eq!(host.columns, host.iconic);
assert_eq!(host.bodies, 0);
let rect = host.columns[0];
assert!(rect.width() > 0.0 && rect.right() <= 800.0 && rect.bottom() <= 600.0);
}
#[test]
fn invalid_side_dock_gaps_are_rejected_before_host_painting() {
for gap in [f32::NAN, f32::INFINITY, -1.0, 4097.0] {
let ctx = Context::default();
let layout = Layout { root: Some(Node::Tabs { panels: vec![1], active: 0 }), floating: vec![] };
let mut host = Host::default();
ctx.run_ui(input(vec![]), |ui| {
let mut style = DockStyle::from_ui(ui);
style.dock_gap = Some(gap);
let shown = DockArea::new(Id::unique("invalid-gap")).show_content(ui, &layout, &style, &mut host);
assert!(shown.error.is_some(), "{gap}");
assert_eq!(host.bodies, 0);
})
.drop_without_applying_deltas();
}
}
#[test]
fn host_owned_surfaces_record_fitted_float_geometry() {
let mut host = Host { exclusive: true, ..Default::default() };
let layout = Layout { root: Some(Node::Tabs { panels: vec![1, 2], active: 0 }), floating: vec![] };
render(&layout, &mut host);
let space = host.space.as_ref().unwrap();
assert!(space.panel(&1).is_some_and(|s| s.x > 700.0 && s.y > 0.0));
assert!(space.group(&2).is_some_and(|s| s.x > 700.0 && s.y > 0.0));
}
#[test]
fn host_owned_iconic_columns_record_fitted_float_geometry() {
let mut host = Host { exclusive: true, ..Default::default() };
let mut column = Column::new(vec![Group::new(vec![1, 2], 0)], 90.0);
column.iconic = true;
let layout = Layout { root: Some(Node::Dock { side: Side::Start, columns: vec![column] }), floating: vec![] };
render(&layout, &mut host);
let size = host.space.as_ref().unwrap().column(&1).expect("host column is recorded");
assert_eq!(size.x, host.columns[0].width());
assert!(size.y > 0.0);
}
#[test]
fn a_host_collapsed_protected_group_keeps_its_selection_without_drawing_a_body() {
let layout = Layout { root: Some(Node::Tabs { panels: vec![1], active: 0 }), floating: vec![] };
let mut host = Host { protected: true, collapsed: true, ..Default::default() };
render(&layout, &mut host);
assert_eq!(host.bodies, 0);
assert!(matches!(layout.root, Some(Node::Tabs { active: 0, .. })));
assert!(host.space.as_ref().unwrap().panel(&1).is_none());
}
#[test]
fn frame_width_ownership_keeps_internal_split_input_and_host_style() {
use crate::layout::{SplitAxis, SplitSize};
let ctx = Context::default();
let mut window = Floating::single(vec![1], 0, [100.0, 100.0, 500.0, 350.0]);
window.root = Node::Split {
axis: SplitAxis::Horizontal,
size: SplitSize::Ratio(0.5),
first: Box::new(Node::Tabs { panels: vec![1], active: 0 }),
second: Box::new(Node::Tabs { panels: vec![2], active: 0 }),
};
let layout = Layout { root: None, floating: vec![window] };
let mut host = Host { frame_width: true, ..Default::default() };
let mut step = |events| {
let mut out = DockOutput::default();
ctx.run_ui(input(events), |ui| {
let mut style = DockStyle::from_ui(ui);
super::floating_tests::frame_style(&mut style);
out = DockArea::new(Id::unique("owned-width")).show_content(ui, &layout, &style, &mut host);
})
.drop_without_applying_deltas();
out
};
step(vec![]);
step(vec![]);
step(vec![]);
let hit = Hit::Divider(Divider::WindowSplit(1, vec![]));
let hits = DockArea::<u32>::hit_rects(&ctx, Id::unique("owned-width"));
let at = hits.iter().find(|(h, _)| *h == hit).expect("internal divider remains interactive").1.center();
step(vec![Event::PointerMoved(at), button(at, true)]);
step(vec![Event::PointerMoved(at + vec2(10.0, 0.0))]);
let out = step(vec![Event::PointerMoved(at + vec2(40.0, 0.0))]);
assert!(out.actions.iter().any(|a| matches!(a, Action::ResizeWindowSplit { .. })), "{:?}", out.actions);
assert!(host.style_calls.get() > 0);
}
#[test]
fn internal_width_input_is_suppressed_only_when_explicitly_owned_through_overrides() {
use crate::layout::{SplitAxis, SplitSize};
let mut window = Floating::single(vec![1], 0, [100.0, 100.0, 500.0, 350.0]);
window.root = Node::Split {
axis: SplitAxis::Horizontal,
size: SplitSize::Ratio(0.5),
first: Box::new(Node::Tabs { panels: vec![1], active: 0 }),
second: Box::new(Node::Tabs { panels: vec![2], active: 0 }),
};
let layout = Layout { root: None, floating: vec![window] };
for owned in [false, true] {
let mut host = Host { frame_width: true, split_width: owned, ..Default::default() };
let hits = render(&layout, &mut host);
assert_eq!(hits.iter().any(|(hit, _)| *hit == Hit::Divider(Divider::WindowSplit(1, vec![]))), !owned);
}
}
#[test]
fn exclusive_docked_groups_skip_stock_body_bands_but_keep_column_edges() {
for owned in [false, true] {
for (zone, anchor, inset) in [
(Zone::Top, egui::Align2::CENTER_TOP, vec2(0.0, 50.0)),
(Zone::Bottom, egui::Align2::CENTER_BOTTOM, vec2(0.0, -20.0)),
(Zone::Left, egui::Align2::LEFT_CENTER, vec2(20.0, 0.0)),
(Zone::Right, egui::Align2::RIGHT_CENTER, vec2(-20.0, 0.0)),
] {
let ctx = Context::default();
let area = Id::unique("owned-dock-targets");
let layout = Layout {
root: Some(Node::Dock { side: Side::Start, columns: vec![Column::new(vec![Group::new(vec![1, 2], 0)], 240.0)] }),
floating: vec![Floating::single(vec![99], 0, [500.0, 100.0, 200.0, 150.0])],
};
let mut host = Host { exclusive: owned, stock_group: Some(99), ..Default::default() };
let mut step = |events, start: bool| {
let mut out = DockOutput::default();
ctx.run_ui(input(events), |ui| {
let mut style = DockStyle::from_ui(ui);
super::floating_tests::frame_style(&mut style);
out = DockArea::new(area).show_content(ui, &layout, &style, &mut host);
if start {
assert!(begin_drag(ui.ctx(), area, 99, Rect::from_min_size(pos2(500.0, 100.0), vec2(200.0, 150.0))));
}
})
.drop_without_applying_deltas();
(out, host.groups.get(&1).copied())
};
step(vec![], false);
step(vec![], false);
let (_, target) = step(vec![], false);
let target = target.expect("the docked group is drawn");
let from = pos2(550.0, 140.0);
step(vec![Event::PointerMoved(from), button(from, true)], true);
let to = anchor.pos_in_rect(&target) + inset;
step(vec![Event::PointerMoved(to)], false);
let (out, _) = step(vec![Event::PointerMoved(to), button(to, false)], false);
let group_drop = out.actions.iter().any(|a| {
matches!(
a,
Action::MovePanel { to: Target::At { placement: Placement::Group { .. }, .. }, .. }
| Action::MoveGroup { to: Target::At { placement: Placement::Group { .. }, .. }, .. }
)
});
if matches!(zone, Zone::Top | Zone::Bottom) {
assert_eq!(group_drop, !owned, "{owned} {zone:?}: {:?}", out.actions);
} else {
assert!(!out.actions.is_empty(), "column edges still work: {owned} {zone:?}");
assert!(!group_drop, "column edges must not become group bands");
}
}
}
}
#[test]
fn owned_minimized_groups_skip_the_trailing_stock_target() {
for owned in [false, true] {
let ctx = Context::default();
let area = Id::unique("owned-dock-targets");
let mut layout = Layout {
root: Some(Node::Dock { side: Side::Start, columns: vec![Column::new(vec![Group::new(vec![1, 2], 0)], 240.0)] }),
floating: vec![Floating::single(vec![99], 0, [500.0, 100.0, 200.0, 150.0])],
};
if let Some(Node::Dock { columns, .. }) = &mut layout.root {
columns[0].groups[0].minimized = true;
}
let mut host = Host { exclusive: owned, stock_group: Some(99), ..Default::default() };
let mut step = |events, start: bool| {
let mut out = DockOutput::default();
ctx.run_ui(input(events), |ui| {
let mut style = DockStyle::from_ui(ui);
super::floating_tests::frame_style(&mut style);
out = DockArea::new(area).show_content(ui, &layout, &style, &mut host);
if start {
assert!(begin_drag(ui.ctx(), area, 99, Rect::from_min_size(pos2(500.0, 100.0), vec2(200.0, 150.0))));
}
})
.drop_without_applying_deltas();
(out, host.groups.get(&1).copied())
};
step(vec![], false);
step(vec![], false);
let (_, target) = step(vec![], false);
let target = target.expect("the docked group is drawn");
let from = pos2(550.0, 140.0);
step(vec![Event::PointerMoved(from), button(from, true)], true);
let to = pos2(target.center().x, target.bottom() + 60.0);
step(vec![Event::PointerMoved(to)], false);
let (out, _) = step(vec![Event::PointerMoved(to), button(to, false)], false);
let group_drop = out.actions.iter().any(|a| {
matches!(
a,
Action::MovePanel { to: Target::At { placement: Placement::Group { .. }, .. }, .. }
| Action::MoveGroup { to: Target::At { placement: Placement::Group { .. }, .. }, .. }
)
});
assert_eq!(group_drop, !owned, "owned={owned}: {:?}", out.actions);
}
}
#[test]
fn disabled_edge_axes_ignore_pointer_drags_and_enabled_edges_obey_limits() {
for axes in [[true, true], [true, false], [false, true], [false, false]] {
for horizontal in [true, false] {
for grow in [true, false] {
let ctx = Context::default();
let rect = Rect::from_min_size(pos2(400.0, 300.0), vec2(300.0, 200.0));
let bounds = Rect::from_min_max(pos2(0.0, 0.0), pos2(720.0, 520.0));
let frame = |events| {
let mut resized = None;
ctx.run_ui(input(events), |ui| {
let style = chrome::ChromeStyle::from_ui(ui);
resized = chrome::FloatingFrame::new(Id::unique("edge-axis"), rect)
.resize_axes(axes)
.bounds(bounds)
.limits(PanelLimits { min: vec2(160.0, 120.0), max: vec2(310.0, 210.0) })
.show(ui.ctx(), &style, |_, _| ())
.resized;
})
.drop_without_applying_deltas();
resized
};
frame(vec![]);
frame(vec![]);
let from = if horizontal { pos2(rect.right() - 2.0, rect.center().y) } else { pos2(rect.center().x, rect.bottom() - 2.0) };
frame(vec![Event::PointerMoved(from), button(from, true)]);
let delta = if horizontal { vec2(10.0, 0.0) } else { vec2(0.0, 10.0) };
frame(vec![Event::PointerMoved(from + delta)]);
let result = frame(vec![Event::PointerMoved(from + delta * if grow { 100.0 } else { -100.0 })]);
let enabled = if horizontal { axes[0] } else { axes[1] };
let expected = if horizontal { vec2(if grow { 310.0 } else { 160.0 }, 200.0) } else { vec2(300.0, if grow { 210.0 } else { 120.0 }) };
assert_eq!(result, enabled.then(|| Rect::from_min_size(rect.min, expected)), "{axes:?} horizontal={horizontal} grow={grow}");
}
}
}
}
#[test]
fn frame_at_viewport_edge_cannot_resize_beyond_it() {
let ctx = Context::default();
let rect = Rect::from_min_max(pos2(400.0, 300.0), pos2(800.0, 600.0));
let frame = |events| {
let mut result = None;
ctx.run_ui(input(events), |ui| {
let style = chrome::ChromeStyle::from_ui(ui);
let response = chrome::FloatingFrame::new(Id::unique("viewport-edge"), rect).show(ui.ctx(), &style, |_, _| ());
assert_eq!(response.rect, rect);
result = response.resized;
})
.drop_without_applying_deltas();
result
};
frame(vec![]);
frame(vec![]);
let from = rect.max - vec2(2.0, 2.0);
frame(vec![Event::PointerMoved(from), button(from, true)]);
frame(vec![Event::PointerMoved(from + vec2(10.0, 10.0))]);
assert_eq!(frame(vec![Event::PointerMoved(from + vec2(100.0, 100.0))]), None);
}
crates/ui/src/docking/mod.rs
+2
View File
@@ -53,6 +53,8 @@ mod floating_tests;
#[cfg(test)]
mod group_tests;
#[cfg(test)]
mod host_hooks_tests;
#[cfg(test)]
mod iconic_tests;
#[cfg(all(test, feature = "native-windows"))]
mod ime_tests;
crates/ui/src/docking/render/dock.rs
+25 -8
View File
@@ -77,7 +77,7 @@ pub(in crate::docking) fn dock_rects<P>(
style: &DockStyle,
content: &dyn crate::docking::DockContent<P>,
) -> (Rect, Rect, f32) {
let gap = style.gap;
let gap = style.dock_gap.unwrap_or(style.gap);
let room = (rect.width() - gap - super::node_limits(other, style, content).min.x).clamp(0.0, rect.width());
let width = columns.iter().map(|c| column_width(c, style)).sum::<f32>().clamp(0.0, room);
let (dock, rest) = match edge_of(side, style.right_to_left) {
@@ -266,10 +266,12 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
_ => return false,
};
let (dock, rest, room) = dock_rects(side, columns, other, rect, self.style, &*self.content);
self.dock(ui, side, columns, dock, room);
path.push(dock_first);
self.node(ui, other, rest, path, root);
path.pop();
// The divider's hit margin reaches into the sibling (often the canvas). Register
// the dock after that sibling so a seam press belongs to the resize handle.
self.dock(ui, side, columns, dock, room);
true
}
@@ -328,6 +330,9 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
}
// Registered after the columns, so the narrow hit areas win at the edges.
for (index, column, first, width, inner) in drawn {
if column.iconic && self.content.owns_iconic_column(first) {
continue;
}
if width < 1.0 {
continue;
}
@@ -412,7 +417,7 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
}
fn handle_style(&self) -> ResizeHandleStyle {
ResizeHandleStyle { hovered: self.style.accent, dragged: self.style.accent, focused: self.style.accent }
self.content.resize_handle_style(ResizeHandleStyle { hovered: self.style.accent, dragged: self.style.accent, focused: self.style.accent })
}
/// Width bounds of an expanded column from its panels' limits.
@@ -452,7 +457,15 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
if !usable(rect) {
return;
}
let side = side_at(edge, self.style.right_to_left);
self.content.column_rect(ui.ctx(), column, rect, side);
self.record_column_float(column.groups.iter().flat_map(|g| g.panels.iter().cloned()).collect(), rect);
if column.iconic && self.content.owns_iconic_column(first) {
let mut child = ui.new_child(UiBuilder::new().id_salt(("host-iconic", first)).max_rect(rect));
child.set_clip_rect(rect.intersect(ui.clip_rect()));
self.content.iconic_column(&mut child, column, rect, side);
return;
}
let (bar, body, heights) = self.column_layout(column, rect);
self.column_bar(ui, column, first, bar, rect, edge);
if column.iconic && usable(body) {
@@ -496,9 +509,12 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
/// seams and bands clipped to the visible `view`, and the height dividers.
fn column_groups(&mut self, ui: &mut Ui, column: &Column<P>, heights: &[f32], area: Rect, view: Rect) {
let rects = stacked(area, heights);
for (group, rect) in column.groups.iter().zip(&rects) {
self.group(ui, &group.panels, group.active, group.minimized, *rect, GroupKind::Dock);
}
let owned: Vec<_> = column
.groups
.iter()
.zip(&rects)
.map(|(group, rect)| self.group(ui, &group.panels, group.active, group.minimized, *rect, GroupKind::Dock))
.collect();
let top = rects.last().map_or(area.top(), |r| r.bottom());
let xs = view.x_range();
let half = self.style.seam_band * 0.5;
@@ -512,7 +528,7 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
}
};
let mut previous: Option<&P> = None;
for (group, rect) in column.groups.iter().zip(&rects) {
for ((group, rect), owned) in column.groups.iter().zip(&rects).zip(&owned) {
let Some(anchor) = group.panels.first() else { continue };
if let Some(above) = previous {
let band = Rect::from_x_y_ranges(xs, (rect.top() - half)..=(rect.top() + half));
@@ -528,13 +544,14 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
self.dock_groups.push(DockGroup {
body,
center,
sides: [None, None, Some((above, line(rect.top()))), Some((below, line(rect.bottom())))],
sides: [None, None, (!*owned).then(|| (above, line(rect.top()))), (!*owned).then(|| (below, line(rect.bottom())))],
});
}
previous = Some(anchor);
}
// The room under the last group, when every group is minimized, stacks below it.
if let Some(last) = previous
&& !owned.last().copied().unwrap_or(false)
&& top < view.bottom()
{
let band = Rect::from_x_y_ranges(xs, top..=view.bottom());
crates/ui/src/docking/render/floating.rs
+13 -8
View File
@@ -54,7 +54,7 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
/// The title bar and, for a movable (resizable) window, the resize band above it.
fn window_chrome(&self, window: &Floating<P>) -> f32 {
let band = if window.panels().all(|panel| self.permissions(panel).movable) { RESIZE_BAND } else { 0.0 };
let band = if self.style.title_height > 0.0 && window.panels().all(|panel| self.permissions(panel).movable) { RESIZE_BAND } else { 0.0 };
band + self.style.title_height
}
@@ -81,12 +81,15 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
let style = ChromeStyle::from_dock(self.style);
let resize_label = self.style.resize_window_label.clone();
let opacity = self.drag_opacity(ctx, first);
let response =
FloatingFrame::new(id, rect).bounds(screen).limits(limits).resizable(movable).resize_label(&resize_label).opacity(opacity).show(
ctx,
&style,
|ui, contents| self.window_contents(ui, window, contents, rect, &style),
);
let response = FloatingFrame::new(id, rect)
.bounds(screen)
.limits(limits)
.reserve_top_resize_band(self.style.title_height > 0.0)
.resizable(movable)
.resize_axes([!self.content.owns_floating_width(first), true])
.resize_label(&resize_label)
.opacity(opacity)
.show(ctx, &style, |ui, contents| self.window_contents(ui, window, contents, rect, &style));
for handle in &response.handles {
self.register_interaction(ctx, handle.id);
}
@@ -132,7 +135,9 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
pub(super) fn floating_body(&mut self, ui: &mut Ui, window: &Floating<P>, body: Rect, frame: Rect) {
let Some(key) = window.panels().next().cloned() else { return };
match (&window.root, window.minimized) {
(Node::Tabs { panels, active }, true) => self.group(ui, panels, *active, true, body, GroupKind::Floating { window: frame }),
(Node::Tabs { panels, active }, true) => {
self.group(ui, panels, *active, true, body, GroupKind::Floating { window: frame });
}
(_, true) => {}
(root, false) => {
let outer = self.tree.replace((key, frame));
crates/ui/src/docking/render/group.rs
+39 -10
View File
@@ -36,7 +36,7 @@ const OVERFLOW_WIDTH: f32 = 22.0;
const WHEEL_NOTCH: f32 = 40.0;
impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
/// Draw one tab group at `rect`.
/// Draw one tab group at `rect`, returning whether the host took exclusive ownership.
///
/// An `active` index outside `panels` is a no-op. A lone protected panel is drawn stripless:
/// its whole rect is the body and its only tab has `Rect::NOTHING`. Side splits are offered
@@ -45,14 +45,23 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
/// hidden overflow tabs still appear in the drop target with `Rect::NOTHING`. The host's extra
/// tabs follow the panel tabs and fit with them ([`fit_with_extra`]). `minimized` hides the body
/// but keeps the strip. Tabs, strip, header and drag bar are mirrored in right-to-left layout.
pub(super) fn group(&mut self, ui: &mut Ui, panels: &[P], active: usize, minimized: bool, rect: Rect, kind: GroupKind) {
let Some(selected) = panels.get(active) else { return };
self.record_float(panels, rect.size(), rect.size());
pub(super) fn group(&mut self, ui: &mut Ui, panels: &[P], active: usize, minimized: bool, rect: Rect, kind: GroupKind) -> bool {
let Some(selected) = panels.get(active) else { return false };
let host_collapsed = self.content.group_collapsed(panels);
let minimized = minimized || host_collapsed;
if !host_collapsed {
self.record_float(panels, rect.size(), rect.size());
}
if self.content.custom_group(ui, panels, selected, rect, matches!(kind, GroupKind::Floating { .. })) {
return true;
}
let policy = self.permissions(selected);
let splits = matches!(kind, GroupKind::Frame | GroupKind::Floating { .. }) && policy.accepts_splits;
let rtl = self.style.right_to_left;
if stripless(panels, &*self.content) {
self.contents(ui, selected, rect);
if !minimized {
self.contents(ui, selected, rect);
}
self.targets.push(DropTarget {
anchor: selected.clone(),
rect,
@@ -62,7 +71,7 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
accepts_tabs: policy.accepts_tabs,
right_to_left: rtl,
});
return;
return false;
}
let movable = panels.iter().all(|p| self.permissions(p).movable);
let strip = Rect::from_min_max(rect.min, pos2(rect.right(), (rect.top() + self.style.tab_height).min(rect.bottom())));
@@ -166,6 +175,7 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
self.contents(ui, selected, Rect::from_min_max(pos2(rect.left(), strip.bottom()), rect.max));
}
ui.painter().rect_stroke(rect, 0.0, self.style.border, StrokeKind::Inside);
false
}
/// A part of the strip the host draws (`header` or `extra-tabs`) at `rect`. A press in it
@@ -201,7 +211,21 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
if self.minimizes(owner) && response.double_clicked() {
self.output.actions.push(Action::SetMinimized { panel: owner.selected.clone(), minimized: !owner.minimized });
}
let active = response.has_focus() || response.hovered();
let modality = self.id.with("keyboard-chrome-focus");
let keyboard_input = ui.input(|i| i.events.iter().any(|e| matches!(e, egui::Event::Key { pressed: true, .. })));
let pointer_input = ui.input(|i| i.pointer.any_pressed());
let keyboard_focus = ui.ctx().data_mut(|d| {
let on = if pointer_input {
false
} else if keyboard_input {
true
} else {
d.get_temp::<bool>(modality).unwrap_or(false)
};
d.insert_temp(modality, on);
on
});
let active = response.has_focus() && keyboard_focus;
if active {
ui.painter().with_clip_rect(hit).rect_stroke(
hit.shrink(1.0),
@@ -392,17 +416,22 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
let minimized = owner.filter(|owner| self.minimizes(owner)).map(|owner| owner.minimized);
let policy = self.permissions(panel);
let label = self.label_of(panel);
let color = if active { self.style.text } else { self.style.inactive_text };
let visual_active = active && !owner.is_some_and(|owner| self.content.group_collapsed(owner.panels));
let color = if visual_active { self.style.text } else { self.style.inactive_text };
let galley = self.tab_label(ui, panel, &label, color, (rect.width() - TAB_PADDING).max(0.0));
let elided = galley.elided;
let tab_id = self.id.with(("tab", panel));
let pointer_focus = self.pointer_focus(ui, tab_id);
let response = tabs::Tab::new(tab_id, &label, active)
let response = tabs::Tab::new(tab_id, &label, visual_active)
.sense(if policy.movable { Sense::click_and_drag() } else { Sense::click() })
.focus_stroke(if pointer_focus { Stroke::NONE } else { Stroke::new(1.0, self.style.accent) })
.show_at(ui, rect, |ui, response| {
let painter = ui.painter().with_clip_rect(rect.intersect(ui.clip_rect()));
painter.rect_filled(rect, 0.0, if active || response.hovered() { self.style.active_background } else { self.style.tab_background });
painter.rect_filled(
rect,
0.0,
if visual_active || response.hovered() { self.style.active_background } else { self.style.tab_background },
);
let at = pos2(rect.left() + TAB_PADDING * 0.5, rect.center().y - galley.size().y * 0.5);
painter.galley(at, galley.clone(), color);
});
crates/ui/src/docking/render/mod.rs
+19 -5
View File
@@ -212,13 +212,23 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
// The divider's inner-edge hit margin must win over child scrollbars, just
// as accordion dividers win over the adjacent panel's content.
// Named by its layout path, which `ResizeSplit` takes and hits report.
self.content.split_separator(ui, split.divider);
let host_width =
*axis == SplitAxis::Horizontal && self.tree.as_ref().is_some_and(|(panel, _)| self.content.owns_floating_split_width(panel));
if host_width {
return;
}
let layout_path = self.original_path(path);
let response = ResizeHandle::new(
self.split_id(&layout_path),
*axis,
split.divider,
&self.style.resize_label,
ResizeHandleStyle { hovered: self.style.accent, dragged: self.style.accent, focused: self.style.accent },
self.content.resize_handle_style(ResizeHandleStyle {
hovered: self.style.accent,
dragged: self.style.accent,
focused: self.style.accent,
}),
)
.value(split.first_size(), (split.first_size() + lo)..=(split.first_size() + hi))
.show(ui);
@@ -490,11 +500,15 @@ pub(in crate::docking) fn node_limits<P>(node: &Node<P>, style: &DockStyle, cont
Node::Split { axis, first, second, .. } => {
let a = node_limits(first, style, content);
let b = node_limits(second, style, content);
let gap = if matches!(first.as_ref(), Node::Dock { .. }) || matches!(second.as_ref(), Node::Dock { .. }) {
style.dock_gap.unwrap_or(style.gap)
} else {
style.gap
};
let limits = match axis {
SplitAxis::Horizontal => PanelLimits {
min: vec2(a.min.x + b.min.x + style.gap, a.min.y.max(b.min.y)),
max: vec2(a.max.x + b.max.x + style.gap, a.max.y.min(b.max.y)),
},
SplitAxis::Horizontal => {
PanelLimits { min: vec2(a.min.x + b.min.x + gap, a.min.y.max(b.min.y)), max: vec2(a.max.x + b.max.x + gap, a.max.y.min(b.max.y)) }
}
SplitAxis::Vertical => PanelLimits {
min: vec2(a.min.x.max(b.min.x), a.min.y + b.min.y + style.gap),
max: vec2(a.max.x.min(b.max.x), a.max.y + b.max.y + style.gap),
crates/ui/src/docking/surface.rs
+70
View File
@@ -13,6 +13,8 @@ pub(super) const TAB_PADDING: f32 = 24.0;
pub struct DockStyle {
pub tab_height: f32,
pub gap: f32,
/// Optional gap beside attached side docks. Other split/row dividers retain `gap`.
pub dock_gap: Option<f32>,
pub min_pane: f32,
pub background: Color32,
pub tab_background: Color32,
@@ -175,6 +177,7 @@ impl DockStyle {
Self {
tab_height: 26.0,
gap: 4.0,
dock_gap: None,
min_pane: 40.0,
background: visuals.panel_fill,
tab_background: visuals.extreme_bg_color,
@@ -304,6 +307,7 @@ impl DockStyle {
&& self.font.size > 0.0
&& self.drop_line_width > 0.0
&& self.side_zone_fraction <= 0.5
&& self.dock_gap.is_none_or(|gap| gap.is_finite() && (0.0..=4096.0).contains(&gap))
}
}
@@ -832,6 +836,45 @@ pub fn send_keys(ctx: &Context, viewport: egui::ViewportId, events: Vec<egui::Ev
/// `show*` functions override `logical_label`, `permissions` and `limits`.
pub trait DockContent<P> {
fn body(&mut self, ui: &mut Ui, panel: &P);
/// Observe the fitted rectangle of each attached column, before its chrome is drawn.
fn column_rect(&mut self, _ctx: &Context, _column: &super::Column<P>, _rect: Rect, _side: super::Side) {}
/// An exclusive host surface replaces stock iconic painting, focus, drag and resize.
/// Placement and column-edge drop targets still belong to the workspace renderer.
fn owns_iconic_column(&self, _first: &P) -> bool {
false
}
/// Draw the complete iconic column inside its fitted rectangle when owned by the host.
fn iconic_column(&mut self, _ui: &mut Ui, _column: &super::Column<P>, _rect: Rect, _side: super::Side) {}
/// Return true after painting an exclusive group surface; stock tabs/body/targets are skipped.
/// The host supplies that group's size limits through `limits`.
fn custom_group(&mut self, _ui: &mut Ui, _panels: &[P], _active: &P, _rect: Rect, _floating: bool) -> bool {
false
}
/// Hosts which paint shared seams can keep the stock resize input without a second fill.
fn resize_handle_style(&self, default: crate::layout::ResizeHandleStyle) -> crate::layout::ResizeHandleStyle {
default
}
/// Presentation-only collapse of a group; its selected tab remains model state.
fn group_collapsed(&self, _panels: &[P]) -> bool {
false
}
/// Paint a fitted split gap in the owning surface, without another resize target.
fn split_separator(&mut self, _ui: &mut Ui, _rect: Rect) {}
/// The host supplies horizontal resize input at the floating frame edges.
/// Internal split dividers retain stock input unless `owns_floating_split_width` is true.
fn owns_floating_width(&self, _panel: &P) -> bool {
false
}
/// The host supplies horizontal resize input for every internal split in this floating tree.
/// This is separate from frame-edge ownership; only return true with replacement input.
fn owns_floating_split_width(&self, _panel: &P) -> bool {
false
}
/// Plain text name for accessibility, menus and the default visual label.
fn logical_label(&self, panel: &P) -> String
where
@@ -968,6 +1011,33 @@ impl<P, C: DockContent<P> + ?Sized> DockContent<P> for Overrides<'_, P, C> {
fn body(&mut self, ui: &mut Ui, panel: &P) {
self.inner.body(ui, panel);
}
fn column_rect(&mut self, ctx: &Context, column: &super::Column<P>, rect: Rect, side: super::Side) {
self.inner.column_rect(ctx, column, rect, side);
}
fn owns_iconic_column(&self, first: &P) -> bool {
self.inner.owns_iconic_column(first)
}
fn iconic_column(&mut self, ui: &mut Ui, column: &super::Column<P>, rect: Rect, side: super::Side) {
self.inner.iconic_column(ui, column, rect, side);
}
fn custom_group(&mut self, ui: &mut Ui, panels: &[P], active: &P, rect: Rect, floating: bool) -> bool {
self.inner.custom_group(ui, panels, active, rect, floating)
}
fn resize_handle_style(&self, default: crate::layout::ResizeHandleStyle) -> crate::layout::ResizeHandleStyle {
self.inner.resize_handle_style(default)
}
fn group_collapsed(&self, panels: &[P]) -> bool {
self.inner.group_collapsed(panels)
}
fn split_separator(&mut self, ui: &mut Ui, rect: Rect) {
self.inner.split_separator(ui, rect);
}
fn owns_floating_width(&self, panel: &P) -> bool {
self.inner.owns_floating_width(panel)
}
fn owns_floating_split_width(&self, panel: &P) -> bool {
self.inner.owns_floating_split_width(panel)
}
fn logical_label(&self, panel: &P) -> String {
(self.label)(panel)
}
Loaded 100 of 105 files, more files were not shown because too many files have changed in this diff. Show more