Compare commits
82
Commits
No files matched your search
@@ -27,6 +27,12 @@ jobs:
|
||||
timeout-minutes: 300
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# Optional font build input, so the Japanese font tests run (they skip without it).
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: abb83316d96aa59c1cf64784289e378fe9fa5695
|
||||
path: craft-fonts
|
||||
- name: Build and test in a FreeBSD VM
|
||||
uses: vmactions/freebsd-vm@c46abacb49f09938ca4e1702d15d836285d694cc # v1
|
||||
with:
|
||||
@@ -44,5 +50,6 @@ jobs:
|
||||
# The checkout is synced in under another owner; let tests that call git read it.
|
||||
git config --global --add safe.directory "*"
|
||||
rustc --version
|
||||
export CRAFT_FONTS_DIR="$PWD/craft-fonts" CRAFT_FONTS_REQUIRED=1
|
||||
CARGO_PROFILE_DEV_DEBUG=0 cargo build --workspace --all-targets --locked
|
||||
CARGO_PROFILE_DEV_DEBUG=0 cargo test --workspace --locked
|
||||
@@ -70,6 +70,12 @@ jobs:
|
||||
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: abb83316d96aa59c1cf64784289e378fe9fa5695 # bump deliberately
|
||||
path: craft-fonts
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: aarch64-apple-darwin, x86_64-apple-darwin
|
||||
@@ -87,6 +93,8 @@ jobs:
|
||||
run: packaging/macos/import-cert.sh
|
||||
- name: Build, sign, notarize
|
||||
env:
|
||||
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
|
||||
CRAFT_FONTS_REQUIRED: "1"
|
||||
APPLE_ID: ${{ secrets.APPLE_ID }}
|
||||
APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
|
||||
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
|
||||
@@ -112,11 +120,20 @@ jobs:
|
||||
include:
|
||||
- { arch: x64, target: x86_64-pc-windows-msvc }
|
||||
- { arch: x86, target: i686-pc-windows-msvc }
|
||||
# Windows on ARM: cross-compiled on the x64 runner, so signing and WiX run as above.
|
||||
# .github/workflows/windows-arm64.yml installs and runs it on ARM64 hardware.
|
||||
- { arch: arm64, target: aarch64-pc-windows-msvc }
|
||||
env:
|
||||
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
|
||||
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: abb83316d96aa59c1cf64784289e378fe9fa5695 # bump deliberately
|
||||
path: craft-fonts
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
@@ -133,6 +150,8 @@ jobs:
|
||||
- name: Build, sign, package
|
||||
shell: pwsh
|
||||
env:
|
||||
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
|
||||
CRAFT_FONTS_REQUIRED: "1"
|
||||
WINDOWS_CERTIFICATE: ${{ secrets.WINDOWS_CERTIFICATE }}
|
||||
WINDOWS_CERTIFICATE_PASSWORD: ${{ secrets.WINDOWS_CERTIFICATE_PASSWORD }}
|
||||
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
|
||||
@@ -167,6 +186,12 @@ jobs:
|
||||
NFPM_VERSION: 2.47.0
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: abb83316d96aa59c1cf64784289e378fe9fa5695 # bump deliberately
|
||||
path: craft-fonts
|
||||
- name: Install build and packaging tools
|
||||
run: |
|
||||
sudo apt-get update
|
||||
@@ -184,6 +209,9 @@ jobs:
|
||||
if: inputs.version != ''
|
||||
run: cargo xtask version set "$LIGHTCRAFT_VERSION"
|
||||
- name: Build and package (AppImage, deb, rpm, tar.gz)
|
||||
env:
|
||||
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
|
||||
CRAFT_FONTS_REQUIRED: "1"
|
||||
run: packaging/linux/package.sh
|
||||
- name: Check the packages
|
||||
run: |
|
||||
@@ -212,6 +240,12 @@ jobs:
|
||||
WASM_BINDGEN_VERSION: 0.2.129
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: abb83316d96aa59c1cf64784289e378fe9fa5695 # bump deliberately
|
||||
path: craft-fonts
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: wasm32-unknown-unknown
|
||||
@@ -229,6 +263,9 @@ jobs:
|
||||
if: inputs.version != ''
|
||||
run: cargo xtask version set "$LIGHTCRAFT_VERSION"
|
||||
- name: Build and zip
|
||||
env:
|
||||
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
|
||||
CRAFT_FONTS_REQUIRED: "1"
|
||||
run: packaging/web/package.sh
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
# Windows on ARM: package the ARM64 release the way release.yml does (cross-compiled on an x64
|
||||
# runner, unsigned here), then install that MSI on a Windows 11 ARM64 runner, check the installed
|
||||
# binaries are ARM64, run the CLI natively, and uninstall. (A native ARM64 `cargo test` job is left
|
||||
# out for now: PrintCraft saw an intermittent access violation in egui UI tests there.)
|
||||
name: Windows ARM64
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
paths:
|
||||
- Cargo.toml
|
||||
- Cargo.lock
|
||||
- "**/Cargo.toml"
|
||||
- "**/build.rs"
|
||||
- .github/workflows/windows-arm64.yml
|
||||
- packaging/windows/**
|
||||
schedule:
|
||||
- cron: "41 4 * * 1" # weekly, so drift in the runner images shows up
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: windows-arm64-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
CARGO_INCREMENTAL: 0
|
||||
|
||||
jobs:
|
||||
package:
|
||||
name: package (cross-compiled on x64)
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable
|
||||
with:
|
||||
targets: aarch64-pc-windows-msvc
|
||||
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
|
||||
with:
|
||||
key: windows-arm64-package
|
||||
- name: Install WiX
|
||||
shell: pwsh
|
||||
run: dotnet tool install --global wix --version 5.0.2
|
||||
- name: Build and package (unsigned)
|
||||
shell: pwsh
|
||||
env:
|
||||
DIST: ${{ github.workspace }}\dist\arm64
|
||||
run: ./packaging/windows/package.ps1 -Arch arm64
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: lightcraft-windows-arm64
|
||||
path: dist/arm64/*
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
install:
|
||||
name: install and run on ARM64
|
||||
needs: package
|
||||
runs-on: windows-11-arm
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: lightcraft-windows-arm64
|
||||
path: dist
|
||||
- name: Install the MSI, check it, run it, uninstall it
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$msi = (Get-ChildItem dist/*.msi | Select-Object -First 1).FullName
|
||||
$p = Start-Process msiexec.exe -ArgumentList '/i', "`"$msi`"", '/qn', '/l*v', 'install.log' -Wait -PassThru
|
||||
if ($p.ExitCode -ne 0) { Get-Content install.log -Tail 60; throw "msiexec /i exited $($p.ExitCode)" }
|
||||
$dir = Join-Path $env:ProgramFiles 'LightCraft'
|
||||
foreach ($exe in 'lightcraft.exe', 'lightcraft-cli.exe') {
|
||||
$path = Join-Path $dir $exe
|
||||
if (-not (Test-Path $path)) { throw "$exe not installed in $dir" }
|
||||
$bytes = [System.IO.File]::ReadAllBytes($path)
|
||||
$machine = [BitConverter]::ToUInt16($bytes, [BitConverter]::ToInt32($bytes, 0x3C) + 4)
|
||||
if ($machine -ne 0xAA64) { throw "$exe is for machine 0x$('{0:X}' -f $machine), not ARM64" }
|
||||
Write-Output "ok $path (ARM64)"
|
||||
}
|
||||
# The installed CLI runs natively on ARM64.
|
||||
$cli = Join-Path $dir 'lightcraft-cli.exe'
|
||||
$out = & $cli --version 2>&1 | Out-String
|
||||
if ($LASTEXITCODE -ne 0) { Write-Output $out; throw "lightcraft-cli --version exited $LASTEXITCODE" }
|
||||
Write-Output "ok lightcraft-cli --version: $($out.Trim())"
|
||||
$p = Start-Process msiexec.exe -ArgumentList '/x', "`"$msi`"", '/qn' -Wait -PassThru
|
||||
if ($p.ExitCode -ne 0) { throw "msiexec /x exited $($p.ExitCode)" }
|
||||
if (Test-Path (Join-Path $dir 'lightcraft.exe')) { throw 'uninstall left lightcraft.exe behind' }
|
||||
Write-Output 'ok uninstalled'
|
||||
@@ -7,3 +7,7 @@ plan/
|
||||
/log
|
||||
.DS_Store
|
||||
.claude/
|
||||
/craft-fonts
|
||||
# model weights are never committed (SAM 3: docs/ai-masks.md)
|
||||
*.safetensors
|
||||
*.safetensors.part
|
||||
@@ -62,10 +62,21 @@ of it. Full standard: `../craftrules/standards/never-crash.md`
|
||||
## Assets: icons, images, fonts (ABSOLUTE RULE — never violate)
|
||||
- **Never use any iconography, image, artwork, font, sound or other asset from Adobe products** (no Lightroom/Creative Cloud icons, no screenshots, no presets/profiles/LUTs, no UI bitmaps — not even as a temporary placeholder or "reference copy"). Observing Adobe's UI to imitate *layout and behaviour* is allowed; copying or tracing its assets is not.
|
||||
- **This includes Adobe's open-licensed assets**: no Source Sans/Serif/Code or Source Han fonts, no Adobe Fonts, no
|
||||
Adobe-published icon sets, sample photos, colour profiles or LUTs — even when OFL/MIT. The UI font is Inter (OFL).
|
||||
Adobe-published icon sets, sample photos, colour profiles or LUTs — even when OFL/MIT. The UI font is Inter (OFL);
|
||||
Japanese fonts come from craft-fonts (below).
|
||||
- Every asset in the repository must be one of: **our own original work** (e.g. icons drawn in code as vectors, procedurally generated demo photos), **public domain / CC0**, **Creative Commons** (CC-BY / CC-BY-SA with attribution honoured), **OFL** (fonts), or **permissive open-source** (MIT/Apache-2.0/BSD/ISC) — or contributed by a person who created the asset and licenses it openly.
|
||||
- **Exception: `docs/brand/`.** The ArtCraft name, wordmark and logos there are ArtCraft Team trademarks, not open source and not covered by LightCraft's MIT OR Apache-2.0 licence (`LICENSE-MIT`, `LICENSE-APACHE`, `NOTICE`); their terms are in `docs/brand/LICENSE-brand.txt`. Use them only unmodified and never redraw, recolour or derive from them.
|
||||
- **Every asset must have an entry in `assets/ATTRIBUTION.md`** (path, title, author/creator, source URL or "original work", licence, date added, modifications) and its licence text when required (e.g. `assets/fonts/OFL-*.txt`). Add the entry in the same commit as the asset. Assets without an attribution entry must not be committed.
|
||||
- **Fonts live in [storytold/craft-fonts](https://github.com/storytold/craft-fonts), never in this repo.** Don't commit
|
||||
font files here (Inter, already in `assets/fonts/`, is the one exception); add new fonts to craft-fonts. LightCraft
|
||||
uses it as the optional build input `CRAFT_FONTS_DIR`: `git clone https://github.com/storytold/craft-fonts ../craft-fonts`
|
||||
then `CRAFT_FONTS_DIR=../craft-fonts cargo run -p lightcraft` (or any cargo/xtask command). `crates/engine/build.rs`
|
||||
embeds the manifest's fonts as `lightcraft_engine::CRAFT_FONTS` (wasm32: BIZ UDPGothic Regular only); the UI
|
||||
(`theme::font_definitions`) and the export watermark renderer use its Japanese faces as fallbacks after Inter. Unset,
|
||||
`CRAFT_FONTS` is empty: everything builds, tests and runs, but Japanese text has no glyphs. Releases always build
|
||||
with it (`release.yml`, `CRAFT_FONTS_REQUIRED=1`) and ship the fonts' OFL licences. Tests that need these fonts skip
|
||||
without it; the FreeBSD CI job runs them with it. Rules: craftrules
|
||||
[`standards/fonts.md`](https://github.com/storytold/craftrules/blob/main/standards/fonts.md).
|
||||
- Icons drawn in code (e.g. `crates/ui-egui/src/icons.rs`) are original work and are recorded in `assets/ATTRIBUTION.md` as such; do not trace them from Adobe icons.
|
||||
- Demo/test images: generated procedurally by `lightcraft-scenes`, or CC0 downloads kept in the gitignored `corpus/` with their source recorded. Screenshots of Adobe apps live only in the gitignored `plan/` and are never committed or published.
|
||||
- **Enforced:** `cargo xtask assets` (in `ci`) fails when an image/icon/font/sound/video/raw/ICC/XMP file is not matched
|
||||
|
||||
@@ -62,10 +62,21 @@ of it. Full standard: `../craftrules/standards/never-crash.md`
|
||||
## Assets: icons, images, fonts (ABSOLUTE RULE — never violate)
|
||||
- **Never use any iconography, image, artwork, font, sound or other asset from Adobe products** (no Lightroom/Creative Cloud icons, no screenshots, no presets/profiles/LUTs, no UI bitmaps — not even as a temporary placeholder or "reference copy"). Observing Adobe's UI to imitate *layout and behaviour* is allowed; copying or tracing its assets is not.
|
||||
- **This includes Adobe's open-licensed assets**: no Source Sans/Serif/Code or Source Han fonts, no Adobe Fonts, no
|
||||
Adobe-published icon sets, sample photos, colour profiles or LUTs — even when OFL/MIT. The UI font is Inter (OFL).
|
||||
Adobe-published icon sets, sample photos, colour profiles or LUTs — even when OFL/MIT. The UI font is Inter (OFL);
|
||||
Japanese fonts come from craft-fonts (below).
|
||||
- Every asset in the repository must be one of: **our own original work** (e.g. icons drawn in code as vectors, procedurally generated demo photos), **public domain / CC0**, **Creative Commons** (CC-BY / CC-BY-SA with attribution honoured), **OFL** (fonts), or **permissive open-source** (MIT/Apache-2.0/BSD/ISC) — or contributed by a person who created the asset and licenses it openly.
|
||||
- **Exception: `docs/brand/`.** The ArtCraft name, wordmark and logos there are ArtCraft Team trademarks, not open source and not covered by LightCraft's MIT OR Apache-2.0 licence (`LICENSE-MIT`, `LICENSE-APACHE`, `NOTICE`); their terms are in `docs/brand/LICENSE-brand.txt`. Use them only unmodified and never redraw, recolour or derive from them.
|
||||
- **Every asset must have an entry in `assets/ATTRIBUTION.md`** (path, title, author/creator, source URL or "original work", licence, date added, modifications) and its licence text when required (e.g. `assets/fonts/OFL-*.txt`). Add the entry in the same commit as the asset. Assets without an attribution entry must not be committed.
|
||||
- **Fonts live in [storytold/craft-fonts](https://github.com/storytold/craft-fonts), never in this repo.** Don't commit
|
||||
font files here (Inter, already in `assets/fonts/`, is the one exception); add new fonts to craft-fonts. LightCraft
|
||||
uses it as the optional build input `CRAFT_FONTS_DIR`: `git clone https://github.com/storytold/craft-fonts ../craft-fonts`
|
||||
then `CRAFT_FONTS_DIR=../craft-fonts cargo run -p lightcraft` (or any cargo/xtask command). `crates/engine/build.rs`
|
||||
embeds the manifest's fonts as `lightcraft_engine::CRAFT_FONTS` (wasm32: BIZ UDPGothic Regular only); the UI
|
||||
(`theme::font_definitions`) and the export watermark renderer use its Japanese faces as fallbacks after Inter. Unset,
|
||||
`CRAFT_FONTS` is empty: everything builds, tests and runs, but Japanese text has no glyphs. Releases always build
|
||||
with it (`release.yml`, `CRAFT_FONTS_REQUIRED=1`) and ship the fonts' OFL licences. Tests that need these fonts skip
|
||||
without it; the FreeBSD CI job runs them with it. Rules: craftrules
|
||||
[`standards/fonts.md`](https://github.com/storytold/craftrules/blob/main/standards/fonts.md).
|
||||
- Icons drawn in code (e.g. `crates/ui-egui/src/icons.rs`) are original work and are recorded in `assets/ATTRIBUTION.md` as such; do not trace them from Adobe icons.
|
||||
- Demo/test images: generated procedurally by `lightcraft-scenes`, or CC0 downloads kept in the gitignored `corpus/` with their source recorded. Screenshots of Adobe apps live only in the gitignored `plan/` and are never committed or published.
|
||||
- **Enforced:** `cargo xtask assets` (in `ci`) fails when an image/icon/font/sound/video/raw/ICC/XMP file is not matched
|
||||
|
||||
Generated
+1453
-8
File diff suppressed because it is too large.
Load diff
@@ -29,6 +29,7 @@ lightcraft-merge = { path = "crates/merge" }
|
||||
lightcraft-engine = { path = "crates/engine" }
|
||||
lightcraft-ui-egui = { path = "crates/ui-egui" }
|
||||
lightcraft-mcp = { path = "crates/mcp" }
|
||||
lightcraft-segment = { path = "crates/segment" }
|
||||
|
||||
# foundation
|
||||
thiserror = "2"
|
||||
@@ -62,6 +63,10 @@ jxl-oxide = { version = "0.12", default-features = false }
|
||||
ravif = { version = "0.13", default-features = false, features = ["threading"] } # no "asm": it needs nasm (non-Rust)
|
||||
|
||||
# gpu (crates/gpu; native backends only, no GL: everything stays pure Rust)
|
||||
# SAM 3 inference (lightcraft-segment). 0.9.2: later releases pull `tokenizers` with the
|
||||
# Oniguruma C library into candle-core, and the product is pure Rust.
|
||||
candle-core = { version = "=0.9.2", default-features = false }
|
||||
candle-nn = { version = "=0.9.2", default-features = false }
|
||||
wgpu = { version = "30", default-features = false, features = ["std", "parking_lot", "metal", "vulkan", "dx12", "wgsl"] }
|
||||
pollster = "1"
|
||||
|
||||
|
||||
@@ -15,6 +15,20 @@ list (author, source, license) is in assets/ATTRIBUTION.md.
|
||||
Inter (assets/fonts/Inter-*.ttf) — SIL Open Font License 1.1. See assets/fonts/OFL-Inter.txt.
|
||||
Showcase photographs by Ansel Adams, Dorothea Lange and NASA (docs/images/) — public domain. See assets/ATTRIBUTION.md.
|
||||
|
||||
BIZ UDMincho: Copyright 2022 The BIZ UDMincho Project Authors
|
||||
(https://github.com/googlefonts/morisawa-biz-ud-mincho). SIL Open Font License 1.1;
|
||||
see assets/fonts/OFL-BIZUDMincho.txt.
|
||||
crates/segment (lightcraft-segment) ports the SAM 3 model code of Hugging Face Transformers
|
||||
(https://github.com/huggingface/transformers, src/transformers/models/sam3 and sam3_tracker),
|
||||
Copyright The HuggingFace Team and Meta Platforms, Inc., licensed under the Apache License 2.0.
|
||||
The port was modified by the LightCraft contributors (2026): translated from Python/PyTorch to
|
||||
Rust on candle and restructured; each ported file says so. Because of this, crates/segment is
|
||||
licensed under the Apache License 2.0 only (LICENSE-APACHE), not under the MIT option.
|
||||
|
||||
The SAM 3 weights are not part of LightCraft and are never distributed with it. A user downloads
|
||||
the facebook/sam3 checkpoint only when they choose to (in the app, or by hand), under Meta's SAM
|
||||
License (https://github.com/facebookresearch/sam3/blob/main/LICENSE), a separate, non-OSI
|
||||
licence. See docs/ai-masks.md.
|
||||
|
||||
Builds made with craft-fonts (the optional CRAFT_FONTS_DIR build input; all official releases)
|
||||
also embed the fonts listed in https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md
|
||||
(the Japanese fonts BIZ UDPGothic, BIZ UDMincho and Shippori Mincho; the web build embeds only BIZ
|
||||
UDPGothic Regular), under the SIL Open Font License 1.1. Release packages include each font's
|
||||
licence as OFL-<family>.txt.
|
||||
@@ -334,6 +334,19 @@ cargo run --release -p lightcraft-cli -- render photo.jpg -o out.jpg --set light
|
||||
cargo xtask ci # fmt, clippy, tests, layering, wasm checks
|
||||
```
|
||||
|
||||
**Chinese and Japanese text** need the shared font repo, an optional build input (official releases always include it):
|
||||
|
||||
```sh
|
||||
git clone https://github.com/storytold/craft-fonts ../craft-fonts
|
||||
CRAFT_FONTS_DIR=../craft-fonts cargo run --release -p lightcraft
|
||||
```
|
||||
|
||||
Without it LightCraft builds and runs the same, but Chinese and Japanese text have no glyphs. Fonts are never committed to this
|
||||
repo; see [craftrules `standards/fonts.md`](https://github.com/storytold/craftrules/blob/main/standards/fonts.md).
|
||||
|
||||
**Interface language:** **Edit → Language** (English, 简体中文, 繁體中文(台灣), 日本語) or **Settings → General →
|
||||
Language**; the choice applies immediately and persists. See [docs/localization.md](docs/localization.md).
|
||||
|
||||
The web build needs the `wasm32-unknown-unknown` target and the matching `wasm-bindgen` CLI
|
||||
(`cargo xtask web` prints the exact install command); see [docs/web.md](docs/web.md).
|
||||
|
||||
@@ -356,7 +369,8 @@ Humans and agents follow the same rules, so read [AGENTS.md](AGENTS.md) first. T
|
||||
work from public specs and black-box observation.
|
||||
- **No Adobe assets, ever:** no icons, screenshots, presets, profiles, LUTs or fonts from Adobe products. Every
|
||||
image, icon and font in the repo is original, public domain, Creative Commons, OFL or permissively licensed, and has
|
||||
an entry in [assets/ATTRIBUTION.md](assets/ATTRIBUTION.md) added in the same commit.
|
||||
an entry in [assets/ATTRIBUTION.md](assets/ATTRIBUTION.md) added in the same commit. New fonts go to
|
||||
[storytold/craft-fonts](https://github.com/storytold/craft-fonts), not here.
|
||||
- **Pure Rust**, enforced crate layering, everything is a command, and `cargo xtask ci` green before every commit
|
||||
(one task id per commit).
|
||||
- **Never crash.** Non-test code returns errors instead of panicking: no `unwrap()`, `expect()`, `panic!` or
|
||||
@@ -420,7 +434,9 @@ with its author, source and license in [assets/ATTRIBUTION.md](assets/ATTRIBUTIO
|
||||
Showcase photographs are public-domain works, used via Wikimedia Commons: Ansel Adams, *The Tetons and the Snake River*
|
||||
(1942, U.S. National Archives); Dorothea Lange, *Migrant Mother* (1936, Library of Congress); Bill Anders / NASA,
|
||||
*Earthrise* (1968); NASA, *The Blue Marble* (1972). The demo library is procedurally generated by LightCraft. UI font:
|
||||
Inter (SIL OFL 1.1). All icons are original.
|
||||
Inter (SIL OFL 1.1). Builds made with [craft-fonts](https://github.com/storytold/craft-fonts) (all official releases)
|
||||
also embed its Chinese and Japanese fonts (Noto Sans CJK SC, BIZ UDPGothic, BIZ UDMincho, Shippori Mincho; SIL OFL 1.1), listed in its
|
||||
[ATTRIBUTION.md](https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md). All icons are original.
|
||||
|
||||
The ArtCraft name, wordmark and logos in [`docs/brand/`](docs/brand/) are trademarks of the
|
||||
ArtCraft Team and are not covered by this license. They may be used only unmodified, and only as
|
||||
@@ -433,3 +449,7 @@ Forks and modified versions must remove them.
|
||||
<a href="https://getartcraft.com/"><img alt="ArtCraft" src="docs/brand/artcraft-mark.svg" width="28"></a><br>
|
||||
<sub>Made by the <a href="https://getartcraft.com/">ArtCraft</a> team and community.</sub>
|
||||
</p>
|
||||
|
||||
## Star history
|
||||
|
||||
[](https://www.star-history.com/?repos=storytold%2Flightcraft&type=date&legend=top-left)
|
||||
+22
-16
@@ -23,31 +23,31 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78).
|
||||
| Dimension | Estimate | What's true today | Biggest gaps |
|
||||
|---|---:|---|---|
|
||||
| **Feature checklist** | 79% | P0 core and P1 nearly complete: import (Add / Copy / Move, templates, devices), library, grid/loupe/compare/survey, every Edit slider, curves, colour grading, masking tools, crop/Upright, heal/clone, presets/profiles, versions/history, sync, export, menus, shortcuts | P1: lens-profile database, content-aware fill (patch synthesis), video playback/trim |
|
||||
| **RAW coverage** (formats people shoot) | ~50% | DNG (all kinds), CR2, ARW, NEF (uncompressed + Huffman lossless/lossy), uncompressed RAF/ORF, packed RW2, PEF; every container's embedded preview (incl. CR3) | **CR3** (every Canon since ~2018), compressed RAF/ORF, RW2 v4, Nikon lossy-after-split, Canon sRAW, HEIC/AVIF. Per-model verification is thin (~40 corpus files vs >1,000 models) |
|
||||
| **Colour & image quality** | ~55–65% | Pipeline is complete and fast; GPU path CPU-exact within 1/255 | **No measured camera calibration database**: ARW has a guarded per-file embedded-JPEG colour estimate (docs/camera-preview-colour.md); other non-DNG raws and rejected estimates use a neutral matrix. Colour fidelity remains incomplete. No lens-profile database. No measured fidelity against Lightroom (tone, highlights, texture/clarity, NR, sharpening are tuned by eye) |
|
||||
| **RAW coverage** (formats people shoot) | ~50% | DNG (all kinds), CR2, ARW, NEF (uncompressed + Huffman lossless/lossy), uncompressed RAF/ORF, packed RW2, PEF; every container's embedded preview (incl. CR3) | **CR3** (every Canon since ~2018), compressed RAF/ORF, RW2 v4, Nikon lossy-after-split, Canon sRAW, HEIC/AVIF. NEFs labelled compressed but stored uncompressed (Z 6 packed 14-bit, D850 12-bit uncompressed). Per-model verification is thin (~40 corpus files vs >1,000 models) |
|
||||
| **Colour & image quality** | ~55–65% | Pipeline is complete and fast; GPU path CPU-exact within 1/255 | **No measured camera calibration database**: ARW and NEF have a guarded per-file embedded-JPEG colour estimate (docs/camera-preview-colour.md); other non-DNG raws and rejected estimates use a neutral matrix. Colour fidelity remains incomplete. No lens-profile database. No measured fidelity against Lightroom (tone, highlights, texture/clarity, NR, sharpening are tuned by eye) |
|
||||
| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics | Real segmentation masks (subject, sky, people, objects, landscape, depth), AI denoise, super resolution, lens blur, generative remove, faces/people, natural-language search. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) |
|
||||
| **Workflow & library** | ~85% (single machine) | Robust catalog (journal + snapshots, background compaction, crash-tested), 85k-photo libraries stay responsive, Local browsing with automatic cleanup, XMP interop, keywords, smart albums, Move import | Opening an 85k library takes 1.7–4.7 s; no cloud sync (out of scope), no tablet companion (#74, roadmap), shared albums, publish services, tethering |
|
||||
| **Classic modules** | ~30% | Geotagging from GPX track logs, soft proofing (partial), slideshow (basic) | **Map view, Book, Print, Slideshow module, Web, publish services**: ~40 tracker rows ⬜ |
|
||||
| **HDR & video** | 0% | | HDR edit/display/export; video play/trim/edit/export |
|
||||
| **Platform & robustness** | ~70% | macOS native; Windows/Linux builds; web via WASM; no-panic lints workspace-wide, `unsafe` confined to `crates/sysmem`; failed saves are reported; GPU errors fall back to CPU | Windows installer UI unverified on Windows (PR #79); GPU path proven only on Apple + user reports; Japanese/English UI (see docs/localization-ja.md); remaining technical errors and other languages; accessibility partial; headless UI tests time out under machine load |
|
||||
| **Platform & robustness** | ~70% | macOS native; Windows/Linux builds; web via WASM; no-panic lints workspace-wide, `unsafe` confined to `crates/sysmem`; failed saves are reported; GPU errors fall back to CPU | Windows installer UI unverified on Windows (PR #79); GPU path proven only on Apple + user reports; English, Simplified/Traditional Chinese and Japanese UI (see docs/localization.md); remaining technical errors; Traditional Chinese uses the Simplified Chinese font until craft-fonts has a TC face; accessibility partial; headless UI tests time out under machine load |
|
||||
|
||||
### By kind of user
|
||||
|
||||
| User | Readiness | What blocks them |
|
||||
|---|---:|---|
|
||||
| JPEG / DNG shooter, single machine | ~85% | Fidelity polish, AI masks |
|
||||
| Nikon / Sony / older-Canon raw shooter | ~65% | Camera colour fidelity and coverage (ARW preview estimates are only a starting point) |
|
||||
| Canon CR3 / Fujifilm / Olympus shooter | ~35% | Their raws open as embedded previews only |
|
||||
| Nikon / Sony / older-Canon raw shooter | ~65% | Camera colour fidelity and coverage (ARW and NEF preview estimates are only a starting point) |
|
||||
| Canon CR3 / Fujifilm / Olympus shooter | ~35% | Their raws open as embedded previews only (CR3: the full-size JPEG with full metadata) |
|
||||
| Lightroom Classic power user | ~45% | Print, Book, Map, publish, tethering |
|
||||
| Relies on AI (masks, denoise) | ~25% | No segmentation or AI denoise models |
|
||||
| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); no AI denoise |
|
||||
|
||||
## Where we're going
|
||||
|
||||
Priorities, in order. Each points at tracker rows in [`docs/parity.md`](docs/parity.md) → *Top gaps*.
|
||||
|
||||
1. **Camera colour calibration of our own** (LR-PROF-CAMERACOLOR, P0): fit each camera to its own embedded JPEG, use
|
||||
matrices the files carry themselves, then chart shots. Sony ARW's file-local fit (matrix + tone curve from its own
|
||||
JPEG) is the first step; generalise it to the other makes' raws (NEF, RW2, PEF, ORF…), then validate fidelity.
|
||||
matrices the files carry themselves, then chart shots. Sony ARW and Nikon NEF have the file-local fit (matrix + tone
|
||||
curve from their own JPEG); generalise it to the other makes' raws (RW2, PEF, ORF…), then validate fidelity.
|
||||
2. **Raw formats, clean-room** (LR-IMP-FORMATS, P0): **CR3** first, then compressed RAF / ORF, RW2 v4, NEF
|
||||
lossy-after-split, sRAW. Decided 2026-10-05: write our own decoders from prose descriptions (never decoder source,
|
||||
no LGPL dependency); compressed NEF (#86) is the template.
|
||||
@@ -57,7 +57,9 @@ Priorities, in order. Each points at tracker rows in [`docs/parity.md`](docs/par
|
||||
(references stay in the local `plan/`), then tune against the numbers.
|
||||
5. **Lens profiles of our own** (LR-EDIT-OPTICS-PROFILE, P1).
|
||||
6. **AI model strategy** (maintainer decision): which permissively licensed models (or our own training) for
|
||||
segmentation masks and denoise; then pure-Rust inference. Unblocks M12 and Enhance.
|
||||
segmentation masks and denoise; then pure-Rust inference. Unblocks M12 and Enhance. Object / Describe masks now
|
||||
run Meta's SAM 3 in pure Rust (optional, user-consented download under the non-OSI SAM License; CDN mirrors still
|
||||
to be set up, docs/ai-masks.md); Subject / Sky / People and Enhance are still open.
|
||||
7. **Then:** HDR (Q), the Classic output modules (Print first, then Map view, Book, Slideshow), video (R), localisation
|
||||
and accessibility.
|
||||
|
||||
@@ -82,12 +84,12 @@ hardening (#78), copyright metadata (#51), GPX geotagging (#60), import tag help
|
||||
| M8 | Heal / Remove | content-aware remove (PatchMatch), heal, clone, brush spots, visualize spots, red/pet eye | 6–10 | 🚧 (heal, clone, auto source, visualize spots, red/pet eye ✅; PatchMatch remove ⬜) |
|
||||
| M9 | Presets, profiles, versions, sync | preset browser + amount, create/import presets, profile browser, versions, history, copy/paste/sync settings | 5–8 | ✅ |
|
||||
| M10 | Export & share | export dialog (JPEG/PNG/TIFF/DNG/AVIF/JXL/original), sizing, sharpening, metadata, watermark, naming, batch jobs, XMP sidecars, HDR export | 6–10 | 🚧 (all formats incl. DNG/original, sizing, presets, background jobs ✅; JXL encode, HDR export ⬜) |
|
||||
| M11 | RAW II | CR3, RAF (X-Trans), ORF, RW2, PEF, SRW, 3FR, IIQ + long tail; camera calibration DB; HEIC/AVIF/JXL import | 20–35 | 🚧 (RAF uncompressed, RW2 packed, PEF, ORF uncompressed ✅; **camera colour calibration** 🚧 (guarded ARW preview fitting; measured database still missing), CR3, compressed ORF/RAF ⬜) |
|
||||
| M11 | RAW II | CR3, RAF (X-Trans), ORF, RW2, PEF, SRW, 3FR, IIQ + long tail; camera calibration DB; HEIC/AVIF/JXL import | 20–35 | 🚧 (RAF uncompressed, RW2 packed, PEF, ORF uncompressed ✅; **camera colour calibration** 🚧 (guarded ARW and NEF preview fitting; measured database still missing), CR3, compressed ORF/RAF ⬜) |
|
||||
| M12 | AI & smart features | subject/sky/background/people/object masks, semantic search, faces/People (permissively licensed models, pure-Rust inference) | 20–40 | ⬜ |
|
||||
| M13 | Merge | HDR merge (deghost), panorama (projections, boundary warp, fill edges), HDR panorama | 10–15 | ✅ |
|
||||
| M14 | Video | import/playback/trim via FilmCraft crates, global edits + presets on video, video export | 6–10 | ⬜ |
|
||||
| M15 | Classic modules | Map, Book, Slideshow, Print, Web; smart collections, stacks, virtual copies, publish services, tethering | 25–40 | 🚧 (smart albums, stacks, virtual copies, compare/survey ✅; Map/Book/Slideshow/Print/Web ⬜) |
|
||||
| M16 | 1.0 polish | preferences, shortcut editor, accessibility, localization, packaging (dmg/msi/AppImage/web), hardening | 10–20 | 🚧 (settings, keyboard shortcuts sheet, packaging basics ✅; Japanese/English localisation 🟡; accessibility and other locales ⬜) |
|
||||
| M16 | 1.0 polish | preferences, shortcut editor, accessibility, localization, packaging (dmg/msi/AppImage/web), hardening | 10–20 | 🚧 (settings, keyboard shortcuts sheet, packaging basics ✅; English/Chinese (Simplified, Traditional)/Japanese localisation 🟡; accessibility and further locales ⬜) |
|
||||
|
||||
## Parity estimate (feature count updated 2026-10-05; effort estimate from 2026-10-02)
|
||||
|
||||
@@ -161,7 +163,8 @@ The milestone estimates in the table above were made before work started and are
|
||||
## Raw format coverage and known gaps
|
||||
|
||||
Decoded (CC0 corpus from raw.pixls.us, `cargo xtask corpus --download`, `crates/raw/tests/corpus.rs`): DNG (uncompressed,
|
||||
LJ92, lossy JPEG / Smart Previews, Deflate, float, linear), CR2, ARW (uncompressed, ARW2, LJ92), NEF/NRW uncompressed and Huffman-compressed (lossless, lossy type 1/2, 12/14-bit), RAF uncompressed (Bayer and
|
||||
LJ92, lossy JPEG / Smart Previews, Deflate, float, linear), CR2, ARW (uncompressed, ARW2, LJ92; as-shot white balance and black level of pre-2017 bodies from the enciphered
|
||||
maker-note `Tag2010` and the encrypted `SR2SubIFD`, both recovered by black-box analysis, `crates/raw/src/vendor/arw.rs`), NEF/NRW uncompressed and Huffman-compressed (lossless, lossy type 1/2, 12/14-bit), RAF uncompressed (Bayer and
|
||||
X-Trans), RW2 packed 12/14-bit, PEF (uncompressed and Huffman), ORF uncompressed (16-bit and 12-bit packed). Every
|
||||
supported container also yields its embedded JPEG preview (CR3 too), and the engine shows that preview for raw variants
|
||||
it can't decode yet.
|
||||
@@ -191,11 +194,14 @@ own chart-based calibration (M11.4). Adobe matrices are never used.
|
||||
export hardening, clippy 1.99. Added the honest *Where we stand* assessment and *Where we're going* priorities; added
|
||||
tracker rows for camera colour, camera coverage and render fidelity.
|
||||
|
||||
## Japanese interface and text watermarks
|
||||
## Chinese and Japanese interfaces, and text watermarks
|
||||
|
||||
English/Japanese interface language is persisted in UI state. Core menus have Japanese
|
||||
translations; untranslated panels and dialogs retain English. BIZ UDMincho is bundled
|
||||
under OFL for Japanese glyph coverage without system fonts. Text watermarks now accept
|
||||
English, Simplified Chinese, Traditional Chinese (Taiwan) and Japanese interface languages are persisted in UI state, and the
|
||||
language table (`crates/ui-egui/src/i18n.rs`) drives the menus, settings and fonts, so another
|
||||
language is a table entry plus two catalogs (docs/localization.md). Core menus and panels are
|
||||
translated; untranslated text retains English. CJK glyphs (Chinese: Noto Sans CJK SC; Japanese UI:
|
||||
BIZ UDPGothic; watermarks: BIZ UDMincho) come from storytold/craft-fonts, embedded by builds made
|
||||
with the optional `CRAFT_FONTS_DIR` input (all releases), so no system fonts are needed. Text watermarks now accept
|
||||
`vertical: true` in export JSON/presets and expose an orientation selector. Japanese
|
||||
characters stay upright in top-to-bottom columns, with newlines starting columns to the
|
||||
left. This is basic lettering, without tate-chu-yoko, ruby, kinsoku, or general vertical
|
||||
|
||||
+153
-31
@@ -9,6 +9,7 @@
|
||||
//! lightcraft-cli synth-merge hdr|panorama -o DIR
|
||||
//! lightcraft-cli commands [--json]
|
||||
//! lightcraft-cli controls [--json]
|
||||
//! lightcraft-cli calibrate [--max N] [--out DIR] FOLDERS/FILES…
|
||||
//! ```
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
@@ -84,6 +85,12 @@ USAGE:
|
||||
Write synthetic merge inputs (procedural scene; bracketed DNGs or overlapping PNG views).
|
||||
lightcraft-cli commands [--json] list every command id with its parameters
|
||||
lightcraft-cli controls [--json] list every develop control id with its range
|
||||
lightcraft-cli calibrate [--max N] [--out DIR] FOLDERS/FILES…
|
||||
Fit a colour profile per camera model from raw files and their embedded camera JPEGs
|
||||
(Sony ARW, Nikon NEF): up to N files spread over the folders (default 300; 0 = all), pooled per
|
||||
model, written as <model>.json to DIR (default: the profiles folder LightCraft reads,
|
||||
<config>/camera-profiles, or $LIGHTCRAFT_CAMERA_PROFILES). Raws of a profiled model then
|
||||
take their colour from the profile and only their tone from their own JPEG.
|
||||
lightcraft-cli --version | --help
|
||||
";
|
||||
|
||||
@@ -125,6 +132,7 @@ fn main() -> ExitCode {
|
||||
Some("merge") => merge(&args[1..]),
|
||||
Some("synth-merge") => synth_merge(&args[1..]),
|
||||
Some("controls") => controls(&args[1..]),
|
||||
Some("calibrate") => calibrate(&args[1..]),
|
||||
Some("--version" | "-V" | "version") => {
|
||||
println!("lightcraft-cli {}", env!("CARGO_PKG_VERSION"));
|
||||
Ok(())
|
||||
@@ -144,6 +152,114 @@ fn main() -> ExitCode {
|
||||
}
|
||||
}
|
||||
|
||||
/// Raw files below `path` (or `path` itself), skipping hidden and NAS metadata folders.
|
||||
fn raw_files(path: &Path, out: &mut Vec<std::path::PathBuf>, depth: usize) {
|
||||
let is_raw = |p: &Path| p.extension().and_then(|e| e.to_str()).is_some_and(|e| ["arw", "nef", "nrw"].iter().any(|x| e.eq_ignore_ascii_case(x)));
|
||||
if path.is_file() {
|
||||
if is_raw(path) {
|
||||
out.push(path.to_path_buf());
|
||||
}
|
||||
return;
|
||||
}
|
||||
let Ok(entries) = std::fs::read_dir(path) else { return };
|
||||
if depth > 32 {
|
||||
return;
|
||||
}
|
||||
for entry in entries.flatten() {
|
||||
let name = entry.file_name();
|
||||
if name.to_string_lossy().starts_with(['.', '@']) {
|
||||
continue;
|
||||
}
|
||||
raw_files(&entry.path(), out, depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
fn calibrate(args: &[String]) -> Result<(), String> {
|
||||
let mut max = 300usize;
|
||||
let mut out: Option<std::path::PathBuf> = None;
|
||||
let mut inputs = Vec::new();
|
||||
let mut i = 0;
|
||||
while i < args.len() {
|
||||
match args[i].as_str() {
|
||||
"--max" => max = take_value(args, &mut i, "--max")?.parse().map_err(|_| "--max needs a number".to_string())?,
|
||||
"--out" => out = Some(take_value(args, &mut i, "--out")?.into()),
|
||||
a if a.starts_with("--") => return Err(format!("unknown option `{a}`")),
|
||||
f => inputs.push(f.to_string()),
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
if inputs.is_empty() {
|
||||
return Err("calibrate needs folders or raw files".into());
|
||||
}
|
||||
let mut files = Vec::new();
|
||||
for input in &inputs {
|
||||
raw_files(Path::new(input), &mut files, 0);
|
||||
}
|
||||
files.sort();
|
||||
files.dedup();
|
||||
let found = files.len();
|
||||
if max > 0 && files.len() > max {
|
||||
// spread over all folders (dates, scenes) rather than the first N
|
||||
files = (0..max).filter_map(|k| files.get(k * found / max).cloned()).collect();
|
||||
}
|
||||
eprintln!("calibrate: {} of {found} raw files", files.len());
|
||||
let dir = out.or_else(lightcraft_engine::camera_profiles::dir).ok_or("no profiles folder: pass --out DIR")?;
|
||||
let workers = std::thread::available_parallelism().map_or(4, |n| n.get()).clamp(1, 6);
|
||||
let next = std::sync::atomic::AtomicUsize::new(0);
|
||||
let done = std::sync::atomic::AtomicUsize::new(0);
|
||||
let pools: Vec<lightcraft_engine::camera_profiles::Pool> = std::thread::scope(|scope| {
|
||||
let handles: Vec<_> = (0..workers)
|
||||
.map(|_| {
|
||||
scope.spawn(|| {
|
||||
let mut pool = lightcraft_engine::camera_profiles::Pool::default();
|
||||
while let Some(path) = files.get(next.fetch_add(1, std::sync::atomic::Ordering::Relaxed)) {
|
||||
let result = std::fs::read(path).map_err(|e| e.to_string()).and_then(|bytes| pool.add(&bytes));
|
||||
let n = done.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
|
||||
match result {
|
||||
Ok(Some(model)) => eprintln!("[{n}/{}] {} ({model})", files.len(), path.display()),
|
||||
Ok(None) => eprintln!("[{n}/{}] {}: skipped (no usable camera JPEG or colour)", files.len(), path.display()),
|
||||
Err(e) => eprintln!("[{n}/{}] {}: {e}", files.len(), path.display()),
|
||||
}
|
||||
}
|
||||
pool
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
handles.into_iter().filter_map(|h| h.join().ok()).collect()
|
||||
});
|
||||
let mut pool = lightcraft_engine::camera_profiles::Pool::default();
|
||||
for p in pools {
|
||||
pool.merge(p);
|
||||
}
|
||||
let mut written = 0;
|
||||
for result in pool.fit(5) {
|
||||
match result {
|
||||
Ok(profile) => {
|
||||
let path = lightcraft_engine::camera_profiles::save(&profile, &dir)?;
|
||||
println!(
|
||||
"{}: {} photos, {} colour pairs, hue/saturation table {} → {}",
|
||||
profile.model,
|
||||
profile.files,
|
||||
profile.samples,
|
||||
profile.hue_sat.is_some(),
|
||||
path.display()
|
||||
);
|
||||
written += 1;
|
||||
}
|
||||
Err(e) => eprintln!("calibrate: {e}"),
|
||||
}
|
||||
}
|
||||
for (model, n) in pool.files() {
|
||||
if n < 5 {
|
||||
eprintln!("calibrate: {model}: only {n} usable photo(s), at least 5 needed");
|
||||
}
|
||||
}
|
||||
if written == 0 {
|
||||
return Err("no profile written".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn mcp(args: &[String]) -> Result<(), String> {
|
||||
let mut connect: Option<String> = None;
|
||||
let mut demo = false;
|
||||
@@ -416,39 +532,39 @@ fn run(args: &[String]) -> Result<(), String> {
|
||||
if steps.is_empty() {
|
||||
return Err("run: no command given (see `lightcraft-cli commands`)".into());
|
||||
}
|
||||
let mut backend: Box<dyn Backend> = match connect {
|
||||
Some(addr) => {
|
||||
if demo || library.is_some() || !imports.is_empty() {
|
||||
return Err("--demo, --library and --import apply to headless mode only".into());
|
||||
}
|
||||
Box::new(
|
||||
Remote::connect(&addr)
|
||||
.map_err(|e| format!("LightCraft is not reachable at {addr} ({e}); start it with `lightcraft --control 7980`"))?,
|
||||
)
|
||||
}
|
||||
None => {
|
||||
let mut h = match &library {
|
||||
Some(dir) => {
|
||||
let mut h = Headless::default();
|
||||
h.session.open_library(dir, demo).map_err(|e| library_error(dir, e))?;
|
||||
h
|
||||
}
|
||||
None if demo => Headless::demo(),
|
||||
None => Headless::default(),
|
||||
};
|
||||
if !imports.is_empty() {
|
||||
let paths = expand_paths(&imports);
|
||||
let r = h.session.execute("library.import", &json!({"paths": paths})).map_err(|e| e.to_string())?;
|
||||
// what follows acts on the imported photos (already-known files are reported as duplicates)
|
||||
let mut ids: Vec<Value> = r["imported"].as_array().cloned().unwrap_or_default();
|
||||
ids.extend(r["duplicates"].as_array().into_iter().flatten().filter_map(|d| d.get("existing").filter(|v| v.is_u64()).cloned()));
|
||||
if let Some(first) = ids.first().cloned() {
|
||||
h.session.execute("library.select", &json!({"ids": ids, "active": first})).map_err(|e| e.to_string())?;
|
||||
let mut backend: Box<dyn Backend> =
|
||||
match connect {
|
||||
Some(addr) => {
|
||||
if demo || library.is_some() || !imports.is_empty() {
|
||||
return Err("--demo, --library and --import apply to headless mode only".into());
|
||||
}
|
||||
Box::new(Remote::connect(&addr).map_err(|e| {
|
||||
format!("LightCraft is not reachable at {addr} ({e}); start it with `lightcraft --control {}`", connect_port(&addr))
|
||||
})?)
|
||||
}
|
||||
Box::new(h)
|
||||
}
|
||||
};
|
||||
None => {
|
||||
let mut h = match &library {
|
||||
Some(dir) => {
|
||||
let mut h = Headless::default();
|
||||
h.session.open_library(dir, demo).map_err(|e| library_error(dir, e))?;
|
||||
h
|
||||
}
|
||||
None if demo => Headless::demo(),
|
||||
None => Headless::default(),
|
||||
};
|
||||
if !imports.is_empty() {
|
||||
let paths = expand_paths(&imports);
|
||||
let r = h.session.execute("library.import", &json!({"paths": paths})).map_err(|e| e.to_string())?;
|
||||
// what follows acts on the imported photos (already-known files are reported as duplicates)
|
||||
let mut ids: Vec<Value> = r["imported"].as_array().cloned().unwrap_or_default();
|
||||
ids.extend(r["duplicates"].as_array().into_iter().flatten().filter_map(|d| d.get("existing").filter(|v| v.is_u64()).cloned()));
|
||||
if let Some(first) = ids.first().cloned() {
|
||||
h.session.execute("library.select", &json!({"ids": ids, "active": first})).map_err(|e| e.to_string())?;
|
||||
}
|
||||
}
|
||||
Box::new(h)
|
||||
}
|
||||
};
|
||||
let mut out = std::io::stdout().lock();
|
||||
let mut failed = 0;
|
||||
for s in &steps {
|
||||
@@ -474,6 +590,12 @@ fn take_value<'a>(args: &'a [String], i: &mut usize, flag: &str) -> Result<&'a s
|
||||
args.get(*i).map(String::as_str).ok_or_else(|| format!("{flag} needs a value"))
|
||||
}
|
||||
|
||||
/// The port an `--connect` address names, for the recovery hint. The whole address when it has
|
||||
/// no port, so a hint is never built from a guess.
|
||||
fn connect_port(addr: &str) -> &str {
|
||||
addr.rsplit_once(':').map_or(addr, |(_, port)| port)
|
||||
}
|
||||
|
||||
fn render(args: &[String]) -> Result<(), String> {
|
||||
let mut input = None;
|
||||
let mut output = None;
|
||||
|
||||
@@ -183,6 +183,42 @@ fn snapshot_subcommand_renders_the_ui_headlessly() {
|
||||
assert_eq!((d.width, d.height), (640, 480));
|
||||
}
|
||||
|
||||
/// Issue #136: with the GPU switched off from the environment the UI starts and renders on the CPU,
|
||||
/// without creating a GPU device (no driver is loaded), and says why.
|
||||
#[test]
|
||||
fn snapshot_starts_without_a_gpu() {
|
||||
let script = tmp("nogpu.jsonl");
|
||||
std::fs::write(
|
||||
&script,
|
||||
format!(
|
||||
"{}\n{}\n{}\n",
|
||||
json!({"method": "ui.set", "params": {"view": "photoGrid"}}),
|
||||
json!({"method": "engine.execute", "params": {"command": "app.gpu"}}),
|
||||
json!({"method": "ui.screenshot"}),
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
for (var, value, reason) in [("LIGHTCRAFT_GPU", "0", "LIGHTCRAFT_GPU=0"), ("LIGHTCRAFT_GPU_BACKEND", "off", "LIGHTCRAFT_GPU_BACKEND=off")] {
|
||||
let out = tmp(&format!("nogpu-{var}.png"));
|
||||
let o = Command::new(BIN)
|
||||
.env_remove("LIGHTCRAFT_GPU")
|
||||
.env_remove("LIGHTCRAFT_GPU_BACKEND")
|
||||
.env(var, value)
|
||||
.args(["snapshot", "--demo", "--script", script.to_str().unwrap(), "-o", out.to_str().unwrap(), "--size", "480x320"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(o.status.success(), "{var}: {}", String::from_utf8_lossy(&o.stderr));
|
||||
let replies: Vec<Value> = String::from_utf8_lossy(&o.stdout).lines().map(|l| serde_json::from_str(l).unwrap()).collect();
|
||||
assert!(replies.iter().all(|r| r["ok"] == true), "{replies:?}");
|
||||
let gpu = &replies[1]["result"];
|
||||
assert_eq!(gpu["available"], false, "{var}: {gpu}");
|
||||
assert_eq!(gpu["adapter"], Value::Null, "{var}: no device was created: {gpu}");
|
||||
assert!(gpu["reason"].as_str().is_some_and(|r| r.contains(reason)), "{var}: {gpu}");
|
||||
let d = lightcraft_codecs::decode(&std::fs::read(&out).unwrap(), Default::default()).unwrap();
|
||||
assert_eq!((d.width, d.height), (480, 320));
|
||||
}
|
||||
}
|
||||
|
||||
fn run_cli(args: &[&str], stdin: Option<&str>) -> (bool, Vec<Value>, String) {
|
||||
let mut child = Command::new(BIN).arg("run").args(args).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap();
|
||||
if let Some(text) = stdin {
|
||||
@@ -288,3 +324,13 @@ fn a_library_open_in_another_process_is_refused() {
|
||||
assert_eq!(lines[0]["ok"], true);
|
||||
let _ = std::fs::remove_dir_all(&lib);
|
||||
}
|
||||
|
||||
/// Issue #167: when `--connect ADDR` cannot reach the app, the recovery hint must name the port
|
||||
/// ADDR actually names — following the old fixed "7980" left the retry connecting elsewhere.
|
||||
#[test]
|
||||
fn connect_failure_hint_names_the_attempted_port() {
|
||||
let (ok, _lines, err) = run_cli(&["--connect", "127.0.0.1:18437", "ui.inspect"], None);
|
||||
assert!(!ok);
|
||||
assert!(err.contains("--control 18437"), "the hint must name the port it tried: {err}");
|
||||
assert!(!err.contains("--control 7980"), "{err}");
|
||||
}
|
||||
@@ -10,7 +10,7 @@ default-run = "lightcraft"
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
lightcraft-engine = { workspace = true }
|
||||
lightcraft-engine = { workspace = true, features = ["sam"] }
|
||||
lightcraft-sysmem = { workspace = true }
|
||||
lightcraft-ui-egui = { workspace = true }
|
||||
lightcraft-codecs = { workspace = true }
|
||||
|
||||
+121
-11
@@ -71,17 +71,45 @@ impl eframe::App for App {
|
||||
}
|
||||
}
|
||||
|
||||
fn config_dir() -> Option<std::path::PathBuf> {
|
||||
if cfg!(target_os = "macos") {
|
||||
std::env::var_os("HOME").map(|h| std::path::PathBuf::from(h).join("Library/Application Support/LightCraft"))
|
||||
} else if cfg!(windows) {
|
||||
std::env::var_os("APPDATA").map(|a| std::path::PathBuf::from(a).join("LightCraft"))
|
||||
} else {
|
||||
std::env::var_os("XDG_CONFIG_HOME")
|
||||
.map(std::path::PathBuf::from)
|
||||
.or_else(|| std::env::var_os("HOME").map(|h| std::path::PathBuf::from(h).join(".config")))
|
||||
.map(|c| c.join("lightcraft"))
|
||||
/// The window's renderer (egui-wgpu): eframe's defaults, with LightCraft's backend choice — DX12
|
||||
/// alone on Windows unless `LIGHTCRAFT_GPU_BACKEND` / `WGPU_BACKEND` say otherwise (issue #136:
|
||||
/// with Vulkan in the set, wgpu loads the Vulkan driver even when it then picks DX12).
|
||||
fn window_wgpu_options() -> eframe::egui_wgpu::WgpuConfiguration {
|
||||
let mut c = eframe::egui_wgpu::WgpuConfiguration::default();
|
||||
if let eframe::egui_wgpu::WgpuSetup::CreateNew(n) = &mut c.wgpu_setup {
|
||||
n.instance_descriptor.backends = lightcraft_engine::gpu::backend::window_backends();
|
||||
}
|
||||
c
|
||||
}
|
||||
|
||||
/// Written while the GPU compute device is created, removed once that returned
|
||||
/// (`lightcraft_gpu::backend::set_init_marker`). Not with `LIGHTCRAFT_NO_PREFS` (tests, scripts).
|
||||
fn gpu_marker_path() -> Option<std::path::PathBuf> {
|
||||
if std::env::var_os("LIGHTCRAFT_NO_PREFS").is_some() {
|
||||
return None;
|
||||
}
|
||||
config_dir().map(|d| d.join("gpu-init.marker"))
|
||||
}
|
||||
|
||||
/// Arm the GPU crash sentinel; if the previous launch left it behind (it died inside the GPU
|
||||
/// driver while creating the device), the notice to show — GPU rendering then starts off.
|
||||
fn gpu_crash_check() -> Option<String> {
|
||||
let marker = gpu_marker_path();
|
||||
let left = marker.as_deref().and_then(lightcraft_engine::gpu::backend::take_init_marker);
|
||||
lightcraft_engine::gpu::backend::set_init_marker(marker);
|
||||
left.map(|what| gpu_crash_notice(&what))
|
||||
}
|
||||
|
||||
fn gpu_crash_notice(what: &str) -> String {
|
||||
format!(
|
||||
"LightCraft closed unexpectedly while starting the GPU last time ({what}), so GPU rendering is now off and photos render on the CPU. \
|
||||
To try the GPU again, turn on Settings ▸ Performance ▸ Use the GPU for rendering; to try another graphics backend, start LightCraft with \
|
||||
LIGHTCRAFT_GPU_BACKEND=dx12, vulkan or off (see docs/gpu-pipeline.md → Troubleshooting)."
|
||||
)
|
||||
}
|
||||
|
||||
fn config_dir() -> Option<std::path::PathBuf> {
|
||||
lightcraft_engine::camera_profiles::config_dir()
|
||||
}
|
||||
|
||||
/// The saved UI state and app settings (`<config>/ui.json`), if any, and a warning for the user
|
||||
@@ -374,10 +402,46 @@ OPTIONS:
|
||||
--control PORT serve the JSON-lines control channel on 127.0.0.1:PORT (env LIGHTCRAFT_CONTROL_PORT;
|
||||
see docs/control-protocol.md)
|
||||
--version, --help
|
||||
|
||||
ENVIRONMENT:
|
||||
LIGHTCRAFT_GPU_BACKEND=dx12|vulkan|metal|auto|off graphics backend (default: DX12 on Windows, Metal on macOS,
|
||||
Vulkan on Linux; off = render on the CPU); else WGPU_BACKEND. LIGHTCRAFT_GPU=0: CPU rendering.
|
||||
LIGHTCRAFT_SAM3_DIR=DIR the SAM 3 model for Object / Describe masks (default: <settings folder>/models/sam3;
|
||||
optional: LightCraft offers to download it when first needed)
|
||||
LIGHTCRAFT_SAM3_MIRRORS=URL,… where to download the SAM 3 model from (base URLs, tried in order)
|
||||
";
|
||||
|
||||
/// Warnings and errors (failed commands, AI mask analysis) on stderr; `LIGHTCRAFT_LOG=info`
|
||||
/// (or `debug`) shows more.
|
||||
struct StderrLog(log::LevelFilter);
|
||||
|
||||
impl log::Log for StderrLog {
|
||||
fn enabled(&self, m: &log::Metadata) -> bool {
|
||||
m.level() <= self.0 && (m.level() <= log::Level::Warn || m.target().starts_with("lightcraft"))
|
||||
}
|
||||
fn log(&self, r: &log::Record) {
|
||||
if self.enabled(r.metadata()) {
|
||||
eprintln!("lightcraft: {} {}: {}", r.level(), r.target(), r.args());
|
||||
}
|
||||
}
|
||||
fn flush(&self) {}
|
||||
}
|
||||
|
||||
fn install_log() {
|
||||
let level = match std::env::var("LIGHTCRAFT_LOG").unwrap_or_default().as_str() {
|
||||
"debug" => log::LevelFilter::Debug,
|
||||
"info" => log::LevelFilter::Info,
|
||||
_ => log::LevelFilter::Warn,
|
||||
};
|
||||
static LOGGER: std::sync::OnceLock<StderrLog> = std::sync::OnceLock::new();
|
||||
if log::set_logger(LOGGER.get_or_init(|| StderrLog(level))).is_ok() {
|
||||
log::set_max_level(level);
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> eframe::Result {
|
||||
lightcraft_engine::guard::install_hook(std::env::temp_dir().join("lightcraft-panics.log"));
|
||||
install_log();
|
||||
alloc_release::install();
|
||||
let mut control_port: Option<u16> = std::env::var("LIGHTCRAFT_CONTROL_PORT").ok().and_then(|p| p.parse().ok());
|
||||
let mut files = Vec::new();
|
||||
@@ -416,6 +480,11 @@ fn main() -> eframe::Result {
|
||||
.map(Into::into)
|
||||
});
|
||||
let library_dir = library_dir.or_else(lightcraft_engine::library::default_dir);
|
||||
// GPU compute: off if the preference says so, or if the last launch died creating the device
|
||||
// (issue #136) — before anything can create it
|
||||
let gpu_crash = gpu_crash_check();
|
||||
let gpu_on = prefs.as_ref().is_none_or(|u| u.settings.gpu) && gpu_crash.is_none();
|
||||
lightcraft_engine::gpu::set_enabled(gpu_on);
|
||||
let options = eframe::NativeOptions {
|
||||
viewport: egui::ViewportBuilder::default()
|
||||
.with_title("LightCraft")
|
||||
@@ -428,13 +497,20 @@ fn main() -> eframe::Result {
|
||||
.with_icon(app_icon())
|
||||
// Wayland matches the window to packaging/linux/ai.storyteller.lightcraft.desktop by this id
|
||||
.with_app_id(APP_ID),
|
||||
wgpu_options: window_wgpu_options(),
|
||||
..Default::default()
|
||||
};
|
||||
eframe::run_native(
|
||||
"LightCraft",
|
||||
options,
|
||||
Box::new(move |cc| {
|
||||
let (session, problem) = open_session(in_memory, library_dir, seed_demo && files.is_empty());
|
||||
let (mut session, problem) = open_session(in_memory, library_dir, seed_demo && files.is_empty());
|
||||
// AI masks: the SAM 3 checkpoint (facebook/sam3) in <config>/models/sam3, or LIGHTCRAFT_SAM3_DIR
|
||||
// (never required: without it, AI masks offer to download it; see docs/ai-masks.md)
|
||||
session.segmenter.dir =
|
||||
std::env::var_os("LIGHTCRAFT_SAM3_DIR").map(std::path::PathBuf::from).or_else(|| config_dir().map(|d| d.join("models").join("sam3")));
|
||||
// the user's own download locations, one base URL per line (LIGHTCRAFT_SAM3_MIRRORS too)
|
||||
session.segmenter.mirrors_file = config_dir().map(|d| d.join("models").join("sam3-mirrors.txt"));
|
||||
let mut app = LightcraftApp::new(session, services());
|
||||
if let Some(ui) = prefs {
|
||||
app.ui = ui;
|
||||
@@ -449,6 +525,11 @@ fn main() -> eframe::Result {
|
||||
keep_file: keep_prefs_file,
|
||||
..Default::default()
|
||||
};
|
||||
// after the snapshot above, so the writer saves the switched-off preference
|
||||
if let Some(notice) = gpu_crash {
|
||||
app.ui.settings.gpu = false;
|
||||
app.notices.push(notice);
|
||||
}
|
||||
if let Some(port) = control_port {
|
||||
let rx = control_server::start(port, cc.egui_ctx.clone());
|
||||
app = app.with_control(rx);
|
||||
@@ -506,6 +587,35 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// Issue #136: a GPU init marker left by a crashed launch is reported once and removed.
|
||||
#[test]
|
||||
fn gpu_crash_marker_is_reported_once() {
|
||||
let d = dir("gpu-marker");
|
||||
let m = d.join("gpu-init.marker");
|
||||
assert_eq!(lightcraft_engine::gpu::backend::take_init_marker(&m), None);
|
||||
std::fs::write(&m, "GPU device creation started (backends VULKAN | DX12)").unwrap();
|
||||
let what = lightcraft_engine::gpu::backend::take_init_marker(&m).unwrap();
|
||||
let notice = gpu_crash_notice(&what);
|
||||
assert!(notice.contains("GPU rendering is now off") && notice.contains("Use the GPU for rendering"), "{notice}");
|
||||
assert!(!m.exists());
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// Issue #136: Windows windows render with DX12 alone (no Vulkan driver loaded) by default.
|
||||
#[test]
|
||||
fn window_backends_follow_the_platform_default() {
|
||||
let eframe::egui_wgpu::WgpuSetup::CreateNew(n) = window_wgpu_options().wgpu_setup else { panic!("expected CreateNew") };
|
||||
let b = n.instance_descriptor.backends;
|
||||
if std::env::var_os("LIGHTCRAFT_GPU_BACKEND").is_none() && std::env::var_os("WGPU_BACKEND").is_none() {
|
||||
if cfg!(windows) {
|
||||
assert_eq!(b, eframe::wgpu::Backends::DX12);
|
||||
} else if cfg!(target_os = "macos") {
|
||||
assert_eq!(b, eframe::wgpu::Backends::METAL);
|
||||
}
|
||||
}
|
||||
assert!(!b.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ui_json_is_written_atomically() {
|
||||
let d = dir("atomic");
|
||||
|
||||
@@ -171,7 +171,7 @@ fn structure_of(bar: &[(String, Vec<MenuNode>)]) -> String {
|
||||
out.push(';');
|
||||
}
|
||||
}
|
||||
let mut s = format!("{};", lightcraft_ui_egui::i18n::is_japanese());
|
||||
let mut s = format!("{};", lightcraft_ui_egui::i18n::language().code());
|
||||
for (t, items) in bar {
|
||||
s.push_str(t);
|
||||
walk(items, &mut s);
|
||||
@@ -419,6 +419,27 @@ impl NativeMenu {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The menu model's titles stay English, so only the language can tell the native menu to
|
||||
/// rebuild its titles (and the macOS app menu): every language switch — also between two CJK
|
||||
/// languages — must change the structure key.
|
||||
#[test]
|
||||
fn switching_language_rebuilds_native_menu_structure() {
|
||||
use lightcraft_ui_egui::i18n::{Locale, set_language};
|
||||
let app = LightcraftApp::new(lightcraft_engine::Session::with_demo(), Default::default());
|
||||
set_language(Locale::En);
|
||||
let bar = menu_bar(&app);
|
||||
let mut seen = std::collections::BTreeSet::new();
|
||||
for language in Locale::ALL {
|
||||
set_language(*language);
|
||||
assert!(seen.insert(structure_of(&bar)), "{language:?} shares another language's menu structure key");
|
||||
}
|
||||
set_language(Locale::ZhHant);
|
||||
assert_eq!(label_text("File"), "檔案");
|
||||
assert_eq!(label_text("Settings…"), "設定…");
|
||||
assert_eq!(label_text("Quit LightCraft"), "結束 LightCraft");
|
||||
set_language(Locale::En);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shortcuts_map_to_accelerators() {
|
||||
assert_eq!(accelerator("Cmd+Shift+Z"), Some(Accelerator::new(Modifiers::META | Modifiers::SHIFT, Code::KeyZ)));
|
||||
|
||||
@@ -9,9 +9,8 @@ covers a whole directory), or if a licence file referenced in the Licence column
|
||||
| Path | Asset | Creator | Source | Licence | Added | Modifications |
|
||||
|---|---|---|---|---|---|---|
|
||||
| `assets/fonts/Inter-*.ttf` (Regular, Medium, SemiBold) | Inter | Rasmus Andersson | https://github.com/rsms/inter | SIL Open Font License 1.1 (`assets/fonts/OFL-Inter.txt`) | 2026-09-30 | none |
|
||||
| `assets/fonts/BIZUDMincho-Regular.ttf` | BIZ UDMincho Regular | The BIZ UDMincho Project Authors (Morisawa Inc., design TypeBank Co., Ltd.) | https://github.com/google/fonts/tree/63833b7d10bb1f86a8f0b91cba2d3ae1f68d1aa3/ofl/bizudmincho | OFL-1.1 (`assets/fonts/OFL-BIZUDMincho.txt`) | 2026-10-05 | none |
|
||||
| `assets/fonts/OFL-BIZUDMincho.txt` | BIZ UDMincho licence | The BIZ UDMincho Project Authors | https://github.com/google/fonts/blob/63833b7d10bb1f86a8f0b91cba2d3ae1f68d1aa3/ofl/bizudmincho/OFL.txt | OFL-1.1 | 2026-10-05 | none |
|
||||
| `assets/app-icon/` (`lightcraft.svg` and the PNG/ICNS/ICO/SVG files derived from it) | LightCraft app icon (lynx) | LightCraft project owner (drawn in ArtCraft) | original work (vectorised from the owner's drawing; see `assets/app-icon/README.md`) | MIT OR Apache-2.0 (`assets/app-icon/LICENSE.txt`) | 2026-10-02 | vectorised to SVG; rendered to PNG/ICNS/ICO by `packaging/icons.sh` |
|
||||
| `assets/camera-profiles/*.json` | Camera colour profiles (matrix + hue/saturation/value table), built into LightCraft | ILCE-7M4: Harald Hoyer | original work: fitted by `lightcraft-cli calibrate` to the contributor's own raw photos and their in-camera JPEGs (597 ILCE-7M4 photos, 2026); aggregate colour statistics only, no image content; no Adobe or third-party profile data | MIT OR Apache-2.0 | 2026-10-07 | none |
|
||||
| `crates/scenes/` (generated images) | Procedural demo photographs | LightCraft contributors | original work (generated at runtime, no source imagery) | MIT OR Apache-2.0 | 2026-09-30 | n/a |
|
||||
| `crates/ui-egui/src/icons.rs` | UI icons drawn as vector paths in code | LightCraft contributors | original work | MIT OR Apache-2.0 | 2026-09-30 | n/a |
|
||||
| `docs/images/*-tetons.jpg`, `docs/images/grid-pd.jpg` (screenshots containing it) | "The Tetons and the Snake River" | Ansel Adams (1942), U.S. National Archives | https://commons.wikimedia.org/wiki/File:Adams_The_Tetons_and_the_Snake_River.jpg | Public domain (U.S. federal government work) | 2026-09-30 | developed in LightCraft; shown inside app screenshots |
|
||||
@@ -20,9 +19,12 @@ covers a whole directory), or if a licence file referenced in the Licence column
|
||||
| `docs/images/*-blue-marble.jpg`, `docs/images/grid-pd.jpg` | "The Blue Marble" (AS17-148-22727) | NASA, Apollo 17 crew (1972) | https://commons.wikimedia.org/wiki/File:The_Earth_seen_from_Apollo_17.jpg | Public domain (NASA) | 2026-09-30 | developed in LightCraft; shown inside app screenshots |
|
||||
| `docs/images/*.jpg` (UI) | LightCraft application screenshots | LightCraft contributors | original work (captured with `docs/showcase/`) | MIT OR Apache-2.0 | 2026-09-30 | n/a |
|
||||
| `docs/brand/` (artcraft-logo and artcraft-mark, SVG and PNG) | ArtCraft wordmark and mark | ArtCraft Team | original work (https://getartcraft.com/) | ArtCraft trademark, see `docs/brand/LICENSE-brand.txt` (not open source) | 2026-10-01 | none |
|
||||
|
||||
| `assets/fonts/BIZUDPGothic-*.ttf` | BIZ UDPGothic (Regular, Bold) | Morisawa / BIZ UDGothic Project Authors | https://github.com/googlefonts/morisawa-biz-ud-gothic/tree/18934af56b9c003ca58c54bffbf226848cb11032 | SIL Open Font License 1.1 (`assets/fonts/OFL-BIZUDGothic.txt`) | 2026-10-05 | none |
|
||||
| craft-fonts: optional build input, not files in this repository | BIZ UDPGothic (Regular, Bold), BIZ UDMincho Regular, Shippori Mincho Regular | Morisawa Inc. / The BIZ UDGothic and BIZ UDMincho Project Authors; The Shippori Mincho Project Authors (per craft-fonts' attribution) | https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md | SIL Open Font License 1.1 (each font's OFL.txt in craft-fonts; release packages ship it as OFL-<family>.txt) | 2026-10-06 | none; embedded only in builds made with `CRAFT_FONTS_DIR` (all official releases); the web (wasm32) build embeds BIZ UDPGothic Regular only |
|
||||
|
||||
Notes
|
||||
- Fonts authored or published by Adobe (Source Sans/Serif/Code, Source Han, …) are not used, even though some are
|
||||
OFL-licensed: the asset rule excludes anything from Adobe. Source Sans 3 was removed on 2026-09-30 and replaced by Inter.
|
||||
- Japanese fonts (BIZ UDPGothic, BIZ UDMincho, Shippori Mincho) are not in this repository. They live in
|
||||
[storytold/craft-fonts](https://github.com/storytold/craft-fonts), with their licences and attribution in its
|
||||
`ATTRIBUTION.md`, and are embedded only in builds made with the optional `CRAFT_FONTS_DIR` build input (all official
|
||||
releases). Never commit font files here; add new fonts to craft-fonts (craftrules `standards/fonts.md`).
|
||||
@@ -0,0 +1 @@
|
||||
{"version":1,"model":"ILCE-7M4","files":597,"samples":2182876,"matrix":[[1.06975,-0.0709886,-0.000770444],[-0.0169822,1.08012,-0.0651276],[-0.114835,-0.601525,1.74804]],"hue_sat":{"hue_divisions":72,"sat_divisions":5,"val_divisions":5,"data":[[0.0,1.0,1.0],[-0.819008,1.14375,1.0],[-6.07446,1.10974,1.0],[-4.0478,1.04285,1.0],[-0.546764,1.00329,1.0],[0.0,1.0,1.0],[-1.0412,1.10597,1.0],[-7.22645,1.05878,1.0],[-7.30635,0.989387,1.0],[-5.18589,0.974522,1.0],[0.0,1.0,1.0],[-1.02364,1.06303,1.0],[-6.94669,1.01579,1.0],[-7.47614,0.955435,1.0],[-7.02579,0.940854,1.0],[0.0,1.0,1.0],[-0.29852,1.04349,1.0],[-6.37202,1.00021,1.0],[-7.52098,0.935457,1.0],[-7.45702,0.922239,1.0],[0.0,1.0,1.0],[0.367464,1.03266,1.0],[-4.96951,0.996822,1.0],[-7.41706,0.948709,1.0],[-8.04735,0.91672,1.0],[0.0,1.0,1.0],[0.0388419,1.02587,1.0],[-4.47266,1.01282,1.0],[-7.26834,0.987279,1.0],[-9.02713,0.921453,1.0],[0.0,1.0,1.0],[-0.297476,1.02333,1.0],[-4.6323,1.00993,1.0],[-8.57248,0.962973,1.0],[-12.2375,0.872088,1.0],[0.0,1.0,1.0],[-0.785867,1.02786,1.0],[-4.73124,1.00876,1.0],[-9.09856,0.955343,1.0],[-12.4324,0.875472,1.0],[0.0,1.0,1.0],[-0.444072,1.03863,1.0],[-3.94773,1.01784,1.0],[-7.97996,0.975996,1.0],[-10.673,0.917401,1.0],[0.0,1.0,1.0],[1.22651,1.0559,1.0],[-1.89459,1.02934,1.0],[-5.54545,0.993046,1.0],[-8.48877,0.941628,1.0],[0.0,1.0,1.0],[3.24516,1.08292,1.0],[1.20522,1.0577,1.0],[-0.826224,1.01668,1.0],[-3.71884,0.946733,1.0],[0.0,1.0,1.0],[4.68802,1.12039,1.0],[3.23147,1.08619,1.0],[1.84476,1.04,1.0],[0.0970375,0.962298,1.0],[0.0,1.0,1.0],[5.36041,1.16313,1.0],[4.5937,1.10933,1.0],[3.83314,1.05056,1.0],[3.122,0.959876,1.0],[0.0,1.0,1.0],[5.22222,1.17824,1.0],[5.09129,1.1142,1.0],[4.99294,1.04813,1.0],[5.01485,0.957528,1.0],[0.0,1.0,1.0],[4.34382,1.18446,1.0],[4.9457,1.11351,1.0],[5.56715,1.04462,1.0],[6.27579,0.961892,1.0],[0.0,1.0,1.0],[3.5428,1.18985,1.0],[4.9116,1.11394,1.0],[6.11507,1.04098,1.0],[7.03638,0.964331,1.0],[0.0,1.0,1.0],[2.92641,1.20663,1.0],[4.57132,1.1281,1.0],[5.96964,1.04895,1.0],[6.98579,0.96261,1.0],[0.0,1.0,1.0],[2.07123,1.23257,1.0],[3.51736,1.15454,1.0],[4.85051,1.06995,1.0],[5.97805,0.971597,1.0],[0.0,1.0,1.0],[0.869122,1.26762,1.0],[2.22009,1.19021,1.0],[3.54297,1.10536,1.0],[4.31783,1.00254,1.0],[0.0,1.0,1.0],[-0.409355,1.31042,1.0],[0.778039,1.23845,1.0],[1.87845,1.15455,1.0],[1.53759,1.02961,1.0],[0.0,1.0,1.0],[-1.82067,1.35265,1.0],[-0.910751,1.28551,1.0],[-0.100378,1.18585,1.0],[0.0425375,1.01463,1.0],[0.0,1.0,1.0],[-3.26659,1.391,1.0],[-2.50117,1.33263,1.0],[-1.55789,1.18102,1.0],[-0.0853484,1.00594,1.0],[0.0,1.0,1.0],[-4.92446,1.43632,1.0],[-3.92066,1.35798,1.0],[-1.66274,1.11605,1.0],[-0.0358972,1.00178,1.0],[0.0,1.0,1.0],[-5.82474,1.46137,1.0],[-4.22466,1.34716,1.0],[-0.64525,1.04813,1.0],[-0.00357121,1.00023,1.0],[0.0,1.0,1.0],[-7.0672,1.47242,1.0],[-5.38667,1.32663,1.0],[-0.553542,1.02668,1.0],[-0.00188478,1.00008,1.0],[0.0,1.0,1.0],[-8.37522,1.43136,1.0],[-6.13631,1.28174,1.0],[-0.515332,1.01808,1.0],[-0.0015762,1.00005,1.0],[0.0,1.0,1.0],[-10.5196,1.39935,1.0],[-7.49316,1.24442,1.0],[-0.517621,1.01282,1.0],[-0.001445,1.00003,1.0],[0.0,1.0,1.0],[-10.5952,1.3534,1.0],[-7.81548,1.20414,1.0],[-0.63473,1.0111,1.0],[-0.00208267,1.00003,1.0],[0.0,1.0,1.0],[-10.879,1.32051,1.0],[-8.3374,1.18517,1.0],[-0.91817,1.01139,1.0],[-0.00474972,1.00003,1.0],[0.0,1.0,1.0],[-11.8094,1.29525,1.0],[-9.18818,1.17522,1.0],[-0.968552,1.01157,1.0],[-0.00411555,1.00003,1.0],[0.0,1.0,1.0],[-12.2038,1.27777,1.0],[-8.97374,1.15856,1.0],[-0.751018,1.00872,1.0],[-0.00250012,1.00002,1.0],[0.0,1.0,1.0],[-12.0564,1.2709,1.0],[-8.38288,1.15926,1.0],[-0.643806,1.00878,1.0],[-0.00202184,1.00002,1.0],[0.0,1.0,1.0],[-12.8403,1.25618,1.0],[-8.85664,1.14517,1.0],[-0.646111,1.00756,1.0],[-0.00195373,1.00001,1.0],[0.0,1.0,1.0],[-13.2604,1.25094,1.0],[-8.74013,1.11989,1.0],[-0.556009,1.00457,1.0],[-0.00173404,1.00001,1.0],[0.0,1.0,1.0],[-13.3939,1.24388,1.0],[-8.23916,1.11659,1.0],[-0.376536,1.00355,1.0],[-0.000916476,1.00001,1.0],[0.0,1.0,1.0],[-13.6578,1.24212,1.0],[-7.97883,1.11035,1.0],[-0.409859,1.00287,1.0],[-0.0018203,1.0,1.0],[0.0,1.0,1.0],[-15.7225,1.22414,1.0],[-10.2345,1.10217,1.0],[-0.968177,1.00427,1.0],[-0.00650675,1.00001,1.0],[0.0,1.0,1.0],[-16.1557,1.16423,1.0],[-12.721,1.06603,1.0],[-2.02337,1.00334,1.0],[-0.0134173,1.00001,1.0],[0.0,1.0,1.0],[-17.253,1.09714,1.0],[-15.785,1.02184,1.0],[-4.02264,0.996649,1.0],[-0.0279292,0.999956,1.0],[0.0,1.0,1.0],[-17.6206,1.04331,1.0],[-16.2039,0.99401,1.0],[-4.09728,0.990246,1.0],[-0.0269801,0.999896,1.0],[0.0,1.0,1.0],[-17.2302,0.987569,1.0],[-15.1656,0.964114,1.0],[-3.21398,0.983725,1.0],[-0.0202065,0.999807,1.0],[0.0,1.0,1.0],[-16.2184,0.943678,1.0],[-13.4633,0.938141,1.0],[-3.40639,0.971571,1.0],[-0.00923182,0.999563,1.0],[0.0,1.0,1.0],[-14.5653,0.909699,1.0],[-11.6873,0.915733,1.0],[-5.4654,0.942337,1.0],[-0.124331,0.998194,1.0],[0.0,1.0,1.0],[-11.5122,0.906891,1.0],[-9.31547,0.923874,1.0],[-6.12222,0.939832,1.0],[-0.337538,0.995963,1.0],[0.0,1.0,1.0],[-7.77619,0.92294,1.0]Line truncated
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,93 +0,0 @@
|
||||
Copyright 2022 The BIZ UDGothic Project Authors (https://github.com/googlefonts/morisawa-biz-ud-mincho)
|
||||
|
||||
This Font Software is licensed under the SIL Open Font License, Version 1.1.
|
||||
This license is copied below, and is also available with a FAQ at:
|
||||
https://scripts.sil.org/OFL
|
||||
|
||||
|
||||
-----------------------------------------------------------
|
||||
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
|
||||
-----------------------------------------------------------
|
||||
|
||||
PREAMBLE
|
||||
The goals of the Open Font License (OFL) are to stimulate worldwide
|
||||
development of collaborative font projects, to support the font creation
|
||||
efforts of academic and linguistic communities, and to provide a free and
|
||||
open framework in which fonts may be shared and improved in partnership
|
||||
with others.
|
||||
|
||||
The OFL allows the licensed fonts to be used, studied, modified and
|
||||
redistributed freely as long as they are not sold by themselves. The
|
||||
fonts, including any derivative works, can be bundled, embedded,
|
||||
redistributed and/or sold with any software provided that any reserved
|
||||
names are not used by derivative works. The fonts and derivatives,
|
||||
however, cannot be released under any other type of license. The
|
||||
requirement for fonts to remain under this license does not apply
|
||||
to any document created using the fonts or their derivatives.
|
||||
|
||||
DEFINITIONS
|
||||
"Font Software" refers to the set of files released by the Copyright
|
||||
Holder(s) under this license and clearly marked as such. This may
|
||||
include source files, build scripts and documentation.
|
||||
|
||||
"Reserved Font Name" refers to any names specified as such after the
|
||||
copyright statement(s).
|
||||
|
||||
"Original Version" refers to the collection of Font Software components as
|
||||
distributed by the Copyright Holder(s).
|
||||
|
||||
"Modified Version" refers to any derivative made by adding to, deleting,
|
||||
or substituting -- in part or in whole -- any of the components of the
|
||||
Original Version, by changing formats or by porting the Font Software to a
|
||||
new environment.
|
||||
|
||||
"Author" refers to any designer, engineer, programmer, technical
|
||||
writer or other person who contributed to the Font Software.
|
||||
|
||||
PERMISSION & CONDITIONS
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of the Font Software, to use, study, copy, merge, embed, modify,
|
||||
redistribute, and sell modified and unmodified copies of the Font
|
||||
Software, subject to the following conditions:
|
||||
|
||||
1) Neither the Font Software nor any of its individual components,
|
||||
in Original or Modified Versions, may be sold by itself.
|
||||
|
||||
2) Original or Modified Versions of the Font Software may be bundled,
|
||||
redistributed and/or sold with any software, provided that each copy
|
||||
contains the above copyright notice and this license. These can be
|
||||
included either as stand-alone text files, human-readable headers or
|
||||
in the appropriate machine-readable metadata fields within text or
|
||||
binary files as long as those fields can be easily viewed by the user.
|
||||
|
||||
3) No Modified Version of the Font Software may use the Reserved Font
|
||||
Name(s) unless explicit written permission is granted by the corresponding
|
||||
Copyright Holder. This restriction only applies to the primary font name as
|
||||
presented to the users.
|
||||
|
||||
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
|
||||
Software shall not be used to promote, endorse or advertise any
|
||||
Modified Version, except to acknowledge the contribution(s) of the
|
||||
Copyright Holder(s) and the Author(s) or with their explicit written
|
||||
permission.
|
||||
|
||||
5) The Font Software, modified or unmodified, in part or in whole,
|
||||
must be distributed entirely under this license, and must not be
|
||||
distributed under any other license. The requirement for fonts to
|
||||
remain under this license does not apply to any document created
|
||||
using the Font Software.
|
||||
|
||||
TERMINATION
|
||||
This license becomes null and void if any of the above conditions are
|
||||
not met.
|
||||
|
||||
DISCLAIMER
|
||||
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
|
||||
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
|
||||
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
|
||||
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
|
||||
OTHER DEALINGS IN THE FONT SOFTWARE.
|
||||
@@ -1,93 +0,0 @@
|
||||
Copyright 2022 The BIZ UDMincho Project Authors (https://github.com/googlefonts/morisawa-biz-ud-mincho)
|
||||
|
||||
This Font Software is licensed under the SIL Open Font License, Version 1.1.
|
||||
This license is copied below, and is also available with a FAQ at:
|
||||
https://scripts.sil.org/OFL
|
||||
|
||||
|
||||
-----------------------------------------------------------
|
||||
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
|
||||
-----------------------------------------------------------
|
||||
|
||||
PREAMBLE
|
||||
The goals of the Open Font License (OFL) are to stimulate worldwide
|
||||
development of collaborative font projects, to support the font creation
|
||||
efforts of academic and linguistic communities, and to provide a free and
|
||||
open framework in which fonts may be shared and improved in partnership
|
||||
with others.
|
||||
|
||||
The OFL allows the licensed fonts to be used, studied, modified and
|
||||
redistributed freely as long as they are not sold by themselves. The
|
||||
fonts, including any derivative works, can be bundled, embedded,
|
||||
redistributed and/or sold with any software provided that any reserved
|
||||
names are not used by derivative works. The fonts and derivatives,
|
||||
however, cannot be released under any other type of license. The
|
||||
requirement for fonts to remain under this license does not apply
|
||||
to any document created using the fonts or their derivatives.
|
||||
|
||||
DEFINITIONS
|
||||
"Font Software" refers to the set of files released by the Copyright
|
||||
Holder(s) under this license and clearly marked as such. This may
|
||||
include source files, build scripts and documentation.
|
||||
|
||||
"Reserved Font Name" refers to any names specified as such after the
|
||||
copyright statement(s).
|
||||
|
||||
"Original Version" refers to the collection of Font Software components as
|
||||
distributed by the Copyright Holder(s).
|
||||
|
||||
"Modified Version" refers to any derivative made by adding to, deleting,
|
||||
or substituting -- in part or in whole -- any of the components of the
|
||||
Original Version, by changing formats or by porting the Font Software to a
|
||||
new environment.
|
||||
|
||||
"Author" refers to any designer, engineer, programmer, technical
|
||||
writer or other person who contributed to the Font Software.
|
||||
|
||||
PERMISSION & CONDITIONS
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of the Font Software, to use, study, copy, merge, embed, modify,
|
||||
redistribute, and sell modified and unmodified copies of the Font
|
||||
Software, subject to the following conditions:
|
||||
|
||||
1) Neither the Font Software nor any of its individual components,
|
||||
in Original or Modified Versions, may be sold by itself.
|
||||
|
||||
2) Original or Modified Versions of the Font Software may be bundled,
|
||||
redistributed and/or sold with any software, provided that each copy
|
||||
contains the above copyright notice and this license. These can be
|
||||
included either as stand-alone text files, human-readable headers or
|
||||
in the appropriate machine-readable metadata fields within text or
|
||||
binary files as long as those fields can be easily viewed by the user.
|
||||
|
||||
3) No Modified Version of the Font Software may use the Reserved Font
|
||||
Name(s) unless explicit written permission is granted by the corresponding
|
||||
Copyright Holder. This restriction only applies to the primary font name as
|
||||
presented to the users.
|
||||
|
||||
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
|
||||
Software shall not be used to promote, endorse or advertise any
|
||||
Modified Version, except to acknowledge the contribution(s) of the
|
||||
Copyright Holder(s) and the Author(s) or with their explicit written
|
||||
permission.
|
||||
|
||||
5) The Font Software, modified or unmodified, in part or in whole,
|
||||
must be distributed entirely under this license, and must not be
|
||||
distributed under any other license. The requirement for fonts to
|
||||
remain under this license does not apply to any document created
|
||||
using the Font Software.
|
||||
|
||||
TERMINATION
|
||||
This license becomes null and void if any of the above conditions are
|
||||
not met.
|
||||
|
||||
DISCLAIMER
|
||||
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
|
||||
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
|
||||
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
|
||||
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
|
||||
OTHER DEALINGS IN THE FONT SOFTWARE.
|
||||
@@ -10,6 +10,7 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
lightcraft-develop = { workspace = true }
|
||||
lightcraft-meta = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
@@ -35,7 +35,7 @@ use lightcraft_develop::DevelopSettings;
|
||||
pub use local::{DEFAULT_FORGET_DAYS, ForgetPlan, folder_of};
|
||||
pub use lock::{LibraryLock, LockError, LockOwner};
|
||||
pub use model::*;
|
||||
pub use query::{DateGroup, Filter, RatingOp, Sort, SortKey};
|
||||
pub use query::{DateGroup, Filter, Person, RatingOp, Sort, SortKey};
|
||||
pub use rules::{Match, Rule, RuleSet};
|
||||
use serde::{Deserialize, Serialize};
|
||||
pub use store::{FsStore, MemStore, Store};
|
||||
|
||||
@@ -183,6 +183,11 @@ pub struct Meta {
|
||||
pub copyright_url: String,
|
||||
pub creator: String,
|
||||
pub keywords: Vec<String>,
|
||||
/// Face/pet/focus regions read from XMP (MWG-RS), on the upright (EXIF-oriented) photo.
|
||||
/// Removing or resizing one edits the catalog only; LightCraft never writes regions to XMP.
|
||||
/// Left out of the catalog JSON when empty (most photos), so older catalogs read unchanged.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub regions: Vec<lightcraft_meta::Region>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
@@ -331,10 +336,11 @@ impl Photo {
|
||||
pub fn develops_raw(&self) -> bool {
|
||||
self.kind == MediaKind::Raw && self.preview_only.is_none()
|
||||
}
|
||||
/// The current ARW reader has vendor WB multipliers but no measured camera illuminant.
|
||||
/// Use adjustments relative to the camera's as-shot look, as for rendered photographs.
|
||||
/// The current ARW and NEF readers have vendor WB multipliers but no measured camera
|
||||
/// illuminant. Use adjustments relative to the camera's as-shot look, as for rendered
|
||||
/// photographs (the engine's `camera_preview::file_local_look` covers the same formats).
|
||||
pub fn relative_wb(&self) -> bool {
|
||||
self.develops_raw() && self.format.eq_ignore_ascii_case("ARW")
|
||||
self.develops_raw() && ["ARW", "NEF", "NRW"].iter().any(|f| self.format.eq_ignore_ascii_case(f))
|
||||
}
|
||||
/// The develop settings import gave this photo: [`Photo::camera_defaults`], or the user's
|
||||
/// default preset applied on top of them ([`Photo::import_look`]).
|
||||
@@ -421,4 +427,19 @@ mod edited_tests {
|
||||
p.develop = Arc::new(DevelopSettings::default());
|
||||
assert!(!p.is_edited(), "a full reset is unedited too");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sony_and_nikon_raws_use_relative_white_balance() {
|
||||
for (name, format, relative) in [("a.arw", "ARW", true), ("a.nef", "NEF", true), ("a.nrw", "nrw", true), ("a.dng", "DNG", false)] {
|
||||
let mut p = Photo::new(PhotoId(1), Source::Demo { scene: 0 }, name, format, 10, 10, "2026-10-01T00:00:00");
|
||||
p.kind = MediaKind::Raw;
|
||||
p.as_shot_wb = Some((5200.0, 4.0));
|
||||
assert_eq!(p.relative_wb(), relative, "{format}");
|
||||
let wb = p.camera_defaults().wb;
|
||||
assert_eq!((wb.temp, wb.tint), if relative { (6500.0, 0.0) } else { (5200.0, 4.0) }, "{format}");
|
||||
// shown from the embedded preview: a rendered image, not a relative-WB raw
|
||||
p.preview_only = Some("unsupported".into());
|
||||
assert!(!p.relative_wb());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,8 @@ pub struct Filter {
|
||||
pub date: Option<String>,
|
||||
/// A keyword; hierarchical keywords match their children too (`travel` finds `travel|italy`).
|
||||
pub keyword: Option<String>,
|
||||
/// A person: photos with a named face region of this name (case-insensitive), as read from XMP.
|
||||
pub person: Option<String>,
|
||||
pub camera: Option<String>,
|
||||
/// Lens (case-insensitive substring).
|
||||
pub lens: Option<String>,
|
||||
@@ -113,6 +115,7 @@ fn token_matches(p: &Photo, tok: &str) -> bool {
|
||||
"camera" => p.meta.camera.to_lowercase().contains(val),
|
||||
"lens" => p.meta.lens.to_lowercase().contains(val),
|
||||
"keyword" | "kw" => p.meta.keywords.iter().any(|k| crate::keywords::is_under(k, val)),
|
||||
"person" | "who" => has_person(p, val),
|
||||
"type" | "kind" => format!("{:?}", p.kind).eq_ignore_ascii_case(val),
|
||||
"edited" => (val == "true" || val == "yes") == p.is_edited(),
|
||||
"date" => p.date().starts_with(val),
|
||||
@@ -125,6 +128,12 @@ fn token_matches(p: &Photo, tok: &str) -> bool {
|
||||
hay.iter().any(|h| h.to_lowercase().contains(&t)) || p.meta.keywords.iter().any(|k| k.to_lowercase().contains(&t))
|
||||
}
|
||||
|
||||
/// Whether `p` has a named face region called `name` (case-insensitive).
|
||||
fn has_person(p: &Photo, name: &str) -> bool {
|
||||
let name = name.trim().to_lowercase();
|
||||
p.meta.regions.iter().any(|r| r.kind == lightcraft_meta::RegionKind::Face && r.name.as_deref().is_some_and(|n| n.to_lowercase() == name))
|
||||
}
|
||||
|
||||
impl Filter {
|
||||
/// Human-readable summary of the active rules (smart album tooltips, `album.list`).
|
||||
pub fn describe(&self) -> String {
|
||||
@@ -152,9 +161,14 @@ impl Filter {
|
||||
if let Some(e) = self.edited {
|
||||
v.push(if e { "edited".into() } else { "unedited".into() });
|
||||
}
|
||||
for (name, val) in
|
||||
[("keyword", &self.keyword), ("camera", &self.camera), ("lens", &self.lens), ("date", &self.date), ("imported", &self.imported)]
|
||||
{
|
||||
for (name, val) in [
|
||||
("keyword", &self.keyword),
|
||||
("person", &self.person),
|
||||
("camera", &self.camera),
|
||||
("lens", &self.lens),
|
||||
("date", &self.date),
|
||||
("imported", &self.imported),
|
||||
] {
|
||||
if let Some(x) = val {
|
||||
v.push(format!("{name} {x}"));
|
||||
}
|
||||
@@ -267,6 +281,11 @@ impl Filter {
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if let Some(n) = &self.person
|
||||
&& !has_person(p, n)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if let Some(c) = &self.camera
|
||||
&& !p.meta.camera.eq_ignore_ascii_case(c)
|
||||
{
|
||||
@@ -330,6 +349,54 @@ impl Catalog {
|
||||
}
|
||||
m.into_iter().collect()
|
||||
}
|
||||
|
||||
/// The people named on faces in the library (MWG regions read from XMP): how many photos each
|
||||
/// appears in and the photo showing their largest face (for a card's picture). Most photos first,
|
||||
/// then by name. Names that differ only in case are one person, shown as first seen; a person
|
||||
/// twice in one photo counts once.
|
||||
pub fn people(&self) -> Vec<Person> {
|
||||
self.people_in(&Filter::default())
|
||||
}
|
||||
|
||||
/// [`Self::people`] among the photos `filter` lets through (its `person` is ignored): what the
|
||||
/// People view offers while other filters (a date, a rating, an album…) are active, so picking
|
||||
/// a person never ends in an empty grid.
|
||||
pub fn people_in(&self, filter: &Filter) -> Vec<Person> {
|
||||
let filter = Filter { person: None, ..filter.clone() };
|
||||
let mut m: std::collections::HashMap<String, (Person, f64)> = Default::default();
|
||||
for p in self.photos().filter(|p| filter.matches(p, self)) {
|
||||
let mut seen: Vec<String> = Vec::new();
|
||||
for r in p.meta.regions.iter().filter(|r| r.kind == lightcraft_meta::RegionKind::Face) {
|
||||
let Some(name) = r.name.as_deref().map(str::trim).filter(|n| !n.is_empty()) else { continue };
|
||||
let key = name.to_lowercase();
|
||||
// the face's size in pixels of the photo
|
||||
let area = (r.rect.x1 - r.rect.x0) * (r.rect.y1 - r.rect.y0) * p.width as f64 * p.height as f64;
|
||||
let (person, best) =
|
||||
m.entry(key.clone()).or_insert_with(|| (Person { name: name.to_string(), count: 0, photo: p.id, face: r.rect }, area));
|
||||
if !seen.contains(&key) {
|
||||
person.count += 1;
|
||||
seen.push(key);
|
||||
}
|
||||
if area > *best || (area == *best && p.id < person.photo) {
|
||||
(person.photo, person.face, *best) = (p.id, r.rect, area);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut v: Vec<Person> = m.into_values().map(|(p, _)| p).collect();
|
||||
v.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.name.to_lowercase().cmp(&b.name.to_lowercase())));
|
||||
v
|
||||
}
|
||||
}
|
||||
|
||||
/// A person named on faces in the library ([`Catalog::people`]).
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Person {
|
||||
pub name: String,
|
||||
/// Photos they appear in.
|
||||
pub count: usize,
|
||||
/// The photo with their largest face, and that face (normalized, in the photo's upright frame).
|
||||
pub photo: PhotoId,
|
||||
pub face: lightcraft_meta::Rect,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
|
||||
@@ -1,11 +1,23 @@
|
||||
//! Crash-safe writes of user-visible files: exports, renders, DNG conversions, smart previews and
|
||||
//! XMP sidecars all go through here (the catalog's own files use [`crate::store::FsStore`]).
|
||||
//!
|
||||
//! **Write policy** (issue #134): what can't be recreated — originals, XMP sidecars, the catalog,
|
||||
//! settings, DNGs that replace raws, smart previews (the only stand-in while an original is
|
||||
//! offline), an edit copy that becomes a library photo — is written **durably**: synced to disk
|
||||
//! before it counts as written. What can always be made again from the originals — exports,
|
||||
//! renders, screenshots, merge previews — is written **atomically** only: a half-written file
|
||||
//! never replaces a good one, but there is no per-file sync (on a USB drive or a NAS that sync
|
||||
//! dominates a large export).
|
||||
//!
|
||||
//! - [`write_atomic`] replaces a file so that after a crash, a full disk or an unplugged drive
|
||||
//! either the old or the complete new content is there — never a truncated mix. The content
|
||||
//! goes to a new temp file in the same folder (unique name, `create_new`), is synced to disk and
|
||||
//! renamed over the target; then the folder is synced (POSIX). Any failure removes the temp
|
||||
//! file and leaves the target as it was.
|
||||
//! - [`write_atomic_nosync`] is the same temp file + rename without the two syncs, for
|
||||
//! recreatable outputs only: a failed write still removes the temp file and leaves the target as
|
||||
//! it was, but after a power cut the new file may be lost (on some file systems left empty) —
|
||||
//! it is then simply exported again.
|
||||
//! - [`write_new`] / [`write_new_unique`] write a file that must not exist yet: the same temp file
|
||||
//! and sync, plus a read-back check, and the file then appears under its final name **without
|
||||
//! replacing anything** (a hard link, which fails on a taken name; a check-then-rename where the
|
||||
@@ -21,6 +33,19 @@ use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
thread_local! {
|
||||
static FAIL_AFTER: Cell<Option<u64>> = const { Cell::new(None) };
|
||||
static SYNCS: Cell<u64> = const { Cell::new(0) };
|
||||
}
|
||||
|
||||
/// How many file and folder syncs this module made **on the calling thread** (for tests: a
|
||||
/// durable write path syncs, a recreatable output doesn't).
|
||||
#[doc(hidden)]
|
||||
pub fn syncs_on_this_thread() -> u64 {
|
||||
SYNCS.with(Cell::get)
|
||||
}
|
||||
|
||||
fn sync_file(f: &File) -> io::Result<()> {
|
||||
SYNCS.with(|c| c.set(c.get() + 1));
|
||||
f.sync_all()
|
||||
}
|
||||
|
||||
/// Fault injection for tests (in any crate): while the returned guard lives, every write made by
|
||||
@@ -78,7 +103,7 @@ fn parent_of(path: &Path) -> &Path {
|
||||
/// where NTFS journals the rename itself, so errors are ignored.
|
||||
pub fn sync_dir(dir: &Path) {
|
||||
if let Ok(d) = File::open(dir) {
|
||||
let _ = d.sync_all();
|
||||
let _ = sync_file(&d);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,9 +125,9 @@ fn create_temp(path: &Path) -> io::Result<(PathBuf, File)> {
|
||||
Err(io::Error::new(io::ErrorKind::AlreadyExists, format!("no free temp file name in {}", dir.display())))
|
||||
}
|
||||
|
||||
/// Write a temp file next to `path` with what `fill` writes, synced to disk. On failure the temp
|
||||
/// file is removed. Returns its path and length.
|
||||
fn write_temp(path: &Path, fill: &mut dyn FnMut(&mut dyn Write) -> io::Result<()>) -> io::Result<(PathBuf, u64)> {
|
||||
/// Write a temp file next to `path` with what `fill` writes, synced to disk when `sync`. On
|
||||
/// failure the temp file is removed. Returns its path and length.
|
||||
fn write_temp(path: &Path, sync: bool, fill: &mut dyn FnMut(&mut dyn Write) -> io::Result<()>) -> io::Result<(PathBuf, u64)> {
|
||||
let (tmp, file) = create_temp(path)?;
|
||||
let r = (|| {
|
||||
let limit = FAIL_AFTER.with(Cell::get);
|
||||
@@ -111,7 +136,9 @@ fn write_temp(path: &Path, fill: &mut dyn FnMut(&mut dyn Write) -> io::Result<()
|
||||
w.flush()?;
|
||||
let n = w.n;
|
||||
let f = w.inner.into_inner().map_err(|e| e.into_error())?;
|
||||
f.sync_all()?;
|
||||
if sync {
|
||||
sync_file(&f)?;
|
||||
}
|
||||
Ok(n)
|
||||
})();
|
||||
match r {
|
||||
@@ -132,12 +159,26 @@ pub fn write_atomic(path: &Path, data: &[u8]) -> io::Result<()> {
|
||||
/// [`write_atomic`] with streamed content: `fill` writes it (an error aborts and leaves the old
|
||||
/// file). Returns the length written.
|
||||
pub fn write_atomic_with(path: &Path, fill: &mut dyn FnMut(&mut dyn Write) -> io::Result<()>) -> io::Result<u64> {
|
||||
let (tmp, n) = write_temp(path, fill)?;
|
||||
replace_with(path, true, fill)
|
||||
}
|
||||
|
||||
/// Replace (or create) `path` with `data` atomically but **without syncing** it to disk — only for
|
||||
/// outputs that can always be recreated (exports, renders, screenshots); see the module docs.
|
||||
/// Anything that can't be made again goes through [`write_atomic`].
|
||||
pub fn write_atomic_nosync(path: &Path, data: &[u8]) -> io::Result<()> {
|
||||
replace_with(path, false, &mut |w| w.write_all(data)).map(|_| ())
|
||||
}
|
||||
|
||||
/// Temp file (synced when `sync`) renamed over `path`, then the folder synced when `sync`.
|
||||
fn replace_with(path: &Path, sync: bool, fill: &mut dyn FnMut(&mut dyn Write) -> io::Result<()>) -> io::Result<u64> {
|
||||
let (tmp, n) = write_temp(path, sync, fill)?;
|
||||
if let Err(e) = fs::rename(&tmp, path) {
|
||||
let _ = fs::remove_file(&tmp);
|
||||
return Err(e);
|
||||
}
|
||||
sync_dir(parent_of(path));
|
||||
if sync {
|
||||
sync_dir(parent_of(path));
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
@@ -155,8 +196,11 @@ pub fn write_new_unique(candidates: &mut dyn Iterator<Item = PathBuf>, data: &[u
|
||||
let Some(first) = candidates.peek().cloned() else {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "no file name to write"));
|
||||
};
|
||||
let (tmp, _) = write_temp(&first, &mut |w| w.write_all(data))?;
|
||||
let (tmp, _) = write_temp(&first, true, &mut |w| w.write_all(data))?;
|
||||
let r = (|| {
|
||||
// Kept (issue #134 point 4): these files replace raws (Convert / Copy as DNG) or become
|
||||
// library photos (merges). On a local disk the read usually comes from the cache, but on a
|
||||
// network volume it can go back to the server, and next to encoding a DNG it costs little.
|
||||
if !same_content(&tmp, data)? {
|
||||
return Err(io::Error::other("the written file reads back different (failing drive or connection?)"));
|
||||
}
|
||||
@@ -287,6 +331,33 @@ mod tests {
|
||||
let _ = fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recreatable_outputs_are_atomic_without_a_sync() {
|
||||
let d = temp("nosync");
|
||||
let p = d.join("export.jpg");
|
||||
let before = syncs_on_this_thread();
|
||||
write_atomic_nosync(&p, b"one").unwrap();
|
||||
write_atomic_nosync(&p, b"two two").unwrap();
|
||||
assert_eq!(syncs_on_this_thread(), before, "no file or folder sync");
|
||||
assert_eq!(fs::read(&p).unwrap(), b"two two");
|
||||
assert_eq!(names(&d), vec!["export.jpg"], "no temp file left");
|
||||
// still atomic: a failure part-way leaves the previous file and no temp file
|
||||
{
|
||||
let _f = fail_writes_after(3);
|
||||
assert!(write_atomic_nosync(&p, &[9u8; 10_000]).is_err());
|
||||
}
|
||||
assert_eq!(fs::read(&p).unwrap(), b"two two");
|
||||
assert_eq!(names(&d), vec!["export.jpg"]);
|
||||
// the durable writers do sync (the file; the folder too where it can be opened)
|
||||
let before = syncs_on_this_thread();
|
||||
write_atomic(&p, b"three").unwrap();
|
||||
let mid = syncs_on_this_thread();
|
||||
assert!(mid > before);
|
||||
write_new(&d.join("new.dng"), b"dng").unwrap();
|
||||
assert!(syncs_on_this_thread() > mid);
|
||||
let _ = fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_files_never_replace_anything() {
|
||||
let d = temp("new");
|
||||
|
||||
@@ -297,3 +297,86 @@ fn folder_identity_ignores_spelling() {
|
||||
assert!(folder_within("/a/b", "/"));
|
||||
assert!(folder_within("C:\\x", "c:\\"));
|
||||
}
|
||||
|
||||
/// People come from named *face* regions: counted once per photo, case-insensitive, most photos
|
||||
/// first; pets and unnamed faces are not people; the `person` filter and `person:` token match.
|
||||
#[test]
|
||||
fn people_from_named_face_regions() {
|
||||
use lightcraft_meta::{Rect, Region, RegionKind};
|
||||
let region = |name: Option<&str>, kind: RegionKind| Region {
|
||||
rect: Rect { x0: 0.4, y0: 0.4, x1: 0.6, y1: 0.6 },
|
||||
kind,
|
||||
name: name.map(str::to_string),
|
||||
description: None,
|
||||
};
|
||||
let mut c = Catalog::new();
|
||||
let mut add = |name: &str, regions: Vec<Region>| {
|
||||
let id = c.alloc_photo_id();
|
||||
let mut p = Photo::new(id, Source::Demo { scene: 1 }, name, "JPEG", 6000, 4000, "2026-09-30T10:00:00");
|
||||
p.meta.regions = regions;
|
||||
c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
|
||||
id
|
||||
};
|
||||
let a = add(
|
||||
"a.jpg",
|
||||
vec![region(Some("Jane Doe"), RegionKind::Face), region(Some("jane doe"), RegionKind::Face), region(Some("Rex"), RegionKind::Pet)],
|
||||
);
|
||||
let b =
|
||||
add("b.jpg", vec![region(Some("JANE DOE"), RegionKind::Face), region(Some("John Roe"), RegionKind::Face), region(None, RegionKind::Face)]);
|
||||
let big = Rect { x0: 0.2, y0: 0.2, x1: 0.8, y1: 0.8 };
|
||||
let d = add("d.jpg", vec![Region { rect: big, ..region(Some("John Roe"), RegionKind::Face) }]);
|
||||
let e = add("e.jpg", vec![region(Some("Sam"), RegionKind::Face)]);
|
||||
add("f.jpg", vec![]);
|
||||
let people = c.people();
|
||||
let summary: Vec<(&str, usize)> = people.iter().map(|p| (p.name.as_str(), p.count)).collect();
|
||||
assert_eq!(summary, vec![("Jane Doe", 2), ("John Roe", 2), ("Sam", 1)], "once per photo, pets and unnamed faces left out");
|
||||
// the picture is the person's largest face; ties go to the lower photo id
|
||||
assert_eq!((people[0].photo, people[0].face), (a, Rect { x0: 0.4, y0: 0.4, x1: 0.6, y1: 0.6 }));
|
||||
assert_eq!((people[1].photo, people[1].face), (d, big), "the larger face wins over an earlier, smaller one");
|
||||
|
||||
let q = |f: Filter| c.query(&f, &Sort::default());
|
||||
assert_eq!(q(Filter { person: Some("jane doe".into()), ..Default::default() }).len(), 2);
|
||||
let mut got = q(Filter { person: Some("John Roe".into()), ..Default::default() });
|
||||
got.sort();
|
||||
assert_eq!(got, vec![b, d]);
|
||||
assert!(q(Filter { person: Some("Rex".into()), ..Default::default() }).is_empty(), "a pet is not a person");
|
||||
assert_eq!(q(Filter { text: "person:SAM".into(), ..Default::default() }), vec![e], "the search token finds a person, any case");
|
||||
assert!(q(Filter { person: Some("Jane Doe".into()), ..Default::default() }).contains(&a));
|
||||
|
||||
// other filters narrow who is offered (so picking a person never ends in an empty grid); the
|
||||
// `person` filter itself does not
|
||||
c.apply(Op::SetRating { id: b, rating: 3 }).unwrap();
|
||||
let names = |f: Filter| c.people_in(&f).into_iter().map(|p| (p.name, p.count)).collect::<Vec<_>>();
|
||||
let rated = Filter { rating: 1, ..Default::default() };
|
||||
assert_eq!(
|
||||
names(rated.clone()),
|
||||
vec![("JANE DOE".to_string(), 1), ("John Roe".to_string(), 1)],
|
||||
"only the rated photo's people, counted within it, spelled as first seen there"
|
||||
);
|
||||
assert_eq!(
|
||||
names(Filter { person: Some("Sam".into()), ..rated }),
|
||||
names(Filter { rating: 1, ..Default::default() }),
|
||||
"the person filter is ignored"
|
||||
);
|
||||
assert_eq!(names(Filter { rating: 5, ..Default::default() }), vec![], "nobody in the filtered photos");
|
||||
}
|
||||
|
||||
/// Most photos have no regions: the field is left out of the catalog JSON then, and a catalog written
|
||||
/// before regions existed (no `regions` key) reads with none.
|
||||
#[test]
|
||||
fn empty_regions_are_not_serialized_and_default_when_missing() {
|
||||
let m = Meta { keywords: vec!["k".into()], ..Default::default() };
|
||||
let v = serde_json::to_value(&m).unwrap();
|
||||
assert!(v.get("regions").is_none(), "{v}");
|
||||
let back: Meta = serde_json::from_value(v).unwrap();
|
||||
assert!(back.regions.is_empty());
|
||||
let mut with = m.clone();
|
||||
with.regions.push(lightcraft_meta::Region {
|
||||
rect: lightcraft_meta::Rect { x0: 0.1, y0: 0.1, x1: 0.2, y1: 0.2 },
|
||||
kind: lightcraft_meta::RegionKind::Face,
|
||||
name: Some("A".into()),
|
||||
description: None,
|
||||
});
|
||||
let back: Meta = serde_json::from_str(&serde_json::to_string(&with).unwrap()).unwrap();
|
||||
assert_eq!(back, with);
|
||||
}
|
||||
@@ -13,6 +13,7 @@ lightcraft-geom = { workspace = true }
|
||||
lightcraft-color = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
miniz_oxide = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = { workspace = true }
|
||||
@@ -8,10 +8,12 @@
|
||||
|
||||
pub mod controls;
|
||||
pub mod presets;
|
||||
pub mod segmask;
|
||||
pub mod settings;
|
||||
|
||||
pub use controls::{CONTROLS, ControlSpec, Section, Track};
|
||||
pub use presets::{Preset, SettingsGroup, apply_partial, extract_groups};
|
||||
pub use segmask::SegMask;
|
||||
pub use settings::*;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
//! Model segmentations stored with a mask ([`SegMask`]), so rendering never needs the model.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Logits are stored in steps of 1/`SCALE`, clamped to ±127/`SCALE` (≈ ±15.9: a sigmoid of
|
||||
/// 1 − 1.2e-7, i.e. fully in or out).
|
||||
const SCALE: f32 = 8.0;
|
||||
/// Largest grid a stored segmentation may claim (hostile input caps the allocation).
|
||||
pub const MAX_SIDE: usize = 1024;
|
||||
|
||||
/// A segmentation computed by a model (SAM 3 in `lightcraft-segment`): `side × side` logits
|
||||
/// over the uncropped, oriented image stretched to a square (row `y`, column `x` covers
|
||||
/// normalized `((x + 0.5) / side, (y + 0.5) / side)`), quantized to `i8`, deflated and base64
|
||||
/// encoded. Coverage at a point is the sigmoid of the bilinearly sampled logit, so edges stay
|
||||
/// smooth at any output size.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SegMask {
|
||||
pub side: u32,
|
||||
pub data: String,
|
||||
/// The part of the image the grid covers, normalized `[x0, y0, x1, y1]` (`None` = all of
|
||||
/// it). A zoomed-in pass over one object covers just its surroundings, at a higher
|
||||
/// resolution than a pass over the whole photo.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub rect: Option<[f64; 4]>,
|
||||
}
|
||||
|
||||
impl SegMask {
|
||||
/// From `side × side` logits over the whole image.
|
||||
pub fn from_logits(side: usize, logits: &[f32]) -> SegMask {
|
||||
let q: Vec<u8> = logits.iter().take(side * side).map(|l| ((l * SCALE).round().clamp(-127.0, 127.0) as i8) as u8).collect();
|
||||
let z = miniz_oxide::deflate::compress_to_vec(&q, 8);
|
||||
SegMask { side: side as u32, data: base64_encode(&z), rect: None }
|
||||
}
|
||||
|
||||
/// From `side × side` logits over the part `rect` of the image.
|
||||
pub fn from_logits_in(side: usize, logits: &[f32], rect: [f64; 4]) -> SegMask {
|
||||
SegMask { rect: Some(rect), ..SegMask::from_logits(side, logits) }
|
||||
}
|
||||
|
||||
/// The covered part, normalized (the whole image without a `rect`); `None` when damaged.
|
||||
pub fn bounds(&self) -> Option<[f64; 4]> {
|
||||
let r = self.rect.unwrap_or([0.0, 0.0, 1.0, 1.0]);
|
||||
(r.iter().all(|v| v.is_finite()) && r[2] > r[0] && r[3] > r[1]).then_some(r)
|
||||
}
|
||||
|
||||
/// The logits, or `None` when the data is damaged or claims an absurd size.
|
||||
pub fn logits(&self) -> Option<Vec<f32>> {
|
||||
let side = self.side as usize;
|
||||
if side == 0 || side > MAX_SIDE {
|
||||
return None;
|
||||
}
|
||||
let z = base64_decode(&self.data)?;
|
||||
let q = miniz_oxide::inflate::decompress_to_vec_with_limit(&z, side * side).ok()?;
|
||||
(q.len() == side * side).then(|| q.iter().map(|b| f32::from(*b as i8) / SCALE).collect())
|
||||
}
|
||||
}
|
||||
|
||||
/// Most zoomed-in detail patches one AI mask component keeps (the app makes one; each costs a
|
||||
/// decode per render, so a hostile document can't make rendering explode).
|
||||
pub const MAX_DETAIL: usize = 4;
|
||||
|
||||
/// Deserialize a component's detail patches, keeping at most [`MAX_DETAIL`] (the rest are
|
||||
/// skipped without being decoded).
|
||||
pub(crate) fn de_detail<'de, D: serde::Deserializer<'de>>(d: D) -> Result<Vec<SegMask>, D::Error> {
|
||||
struct Capped;
|
||||
impl<'de> serde::de::Visitor<'de> for Capped {
|
||||
type Value = Vec<SegMask>;
|
||||
fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
f.write_str("a list of segmentations")
|
||||
}
|
||||
fn visit_seq<A: serde::de::SeqAccess<'de>>(self, mut seq: A) -> Result<Vec<SegMask>, A::Error> {
|
||||
let mut out = Vec::new();
|
||||
while out.len() < MAX_DETAIL {
|
||||
match seq.next_element::<SegMask>()? {
|
||||
Some(m) => out.push(m),
|
||||
None => return Ok(out),
|
||||
}
|
||||
}
|
||||
while seq.next_element::<serde::de::IgnoredAny>()?.is_some() {}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
d.deserialize_seq(Capped)
|
||||
}
|
||||
|
||||
const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
|
||||
fn base64_encode(bytes: &[u8]) -> String {
|
||||
let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4);
|
||||
for chunk in bytes.chunks(3) {
|
||||
let b = [chunk.first().copied().unwrap_or(0), chunk.get(1).copied().unwrap_or(0), chunk.get(2).copied().unwrap_or(0)];
|
||||
let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
|
||||
for i in 0..4 {
|
||||
if i <= chunk.len() {
|
||||
out.push(char::from(B64[((n >> (18 - 6 * i)) & 63) as usize]));
|
||||
} else {
|
||||
out.push('=');
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn base64_decode(s: &str) -> Option<Vec<u8>> {
|
||||
let mut out = Vec::with_capacity(s.len() / 4 * 3);
|
||||
let (mut acc, mut bits) = (0u32, 0u32);
|
||||
for c in s.bytes().filter(|c| !c.is_ascii_whitespace()) {
|
||||
if c == b'=' {
|
||||
break;
|
||||
}
|
||||
let v = B64.iter().position(|b| *b == c)? as u32;
|
||||
acc = (acc << 6) | v;
|
||||
bits += 6;
|
||||
if bits >= 8 {
|
||||
bits -= 8;
|
||||
out.push((acc >> bits) as u8);
|
||||
acc &= (1 << bits) - 1;
|
||||
}
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn base64_round_trips() {
|
||||
for n in 0..10 {
|
||||
let v: Vec<u8> = (0..n).map(|i| (i * 37 + 11) as u8).collect();
|
||||
assert_eq!(base64_decode(&base64_encode(&v)).unwrap(), v);
|
||||
}
|
||||
assert_eq!(base64_encode(b"Man"), "TWFu");
|
||||
assert_eq!(base64_encode(b"Ma"), "TWE=");
|
||||
assert!(base64_decode("TW!u").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logits_round_trip_quantized() {
|
||||
let side = 16;
|
||||
let l: Vec<f32> = (0..side * side).map(|i| (i as f32 - 128.0) / 4.0).collect();
|
||||
let m = SegMask::from_logits(side, &l);
|
||||
let back = m.logits().unwrap();
|
||||
for (a, b) in l.iter().zip(&back) {
|
||||
assert!((a.clamp(-127.0 / SCALE, 127.0 / SCALE) - b).abs() <= 0.5 / SCALE + 1e-6);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_hostile_document_keeps_at_most_max_detail_patches() {
|
||||
let patch = serde_json::to_value(SegMask::from_logits(4, &[1.0; 16])).unwrap();
|
||||
let shape = serde_json::json!({"kind": "object", "hint": [{"x": 0.5, "y": 0.5}], "detail": vec![patch; 10_000]});
|
||||
let s: crate::MaskShape = serde_json::from_value(shape).unwrap();
|
||||
let crate::MaskShape::Object { detail, .. } = s else { panic!("an Object") };
|
||||
assert_eq!(detail.len(), MAX_DETAIL);
|
||||
// and a whole develop document with them still loads
|
||||
let mut d = crate::DevelopSettings::default();
|
||||
d.masks.push(crate::Mask {
|
||||
id: 1,
|
||||
components: vec![crate::MaskComponent {
|
||||
name: None,
|
||||
op: crate::MaskOp::Add,
|
||||
invert: false,
|
||||
shape: crate::MaskShape::Prompt { text: "x".into(), seg: None, detail: vec![SegMask::from_logits(4, &[1.0; 16]); 50], edge: 0.0 },
|
||||
}],
|
||||
..Default::default()
|
||||
});
|
||||
let back: crate::DevelopSettings = serde_json::from_str(&serde_json::to_string(&d).unwrap()).unwrap();
|
||||
let crate::MaskShape::Prompt { detail, .. } = &back.masks[0].components[0].shape else { panic!("a Prompt") };
|
||||
assert_eq!(detail.len(), MAX_DETAIL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn damaged_data_is_none() {
|
||||
let mut m = SegMask::from_logits(8, &[1.0; 64]);
|
||||
m.side = 9;
|
||||
assert!(m.logits().is_none(), "size mismatch");
|
||||
m.side = 100_000;
|
||||
assert!(m.logits().is_none(), "absurd size");
|
||||
assert!(SegMask { side: 8, data: "not base64 !".into(), rect: None }.logits().is_none());
|
||||
assert!(SegMask { rect: Some([0.5, 0.0, 0.2, 1.0]), ..SegMask::from_logits(2, &[0.0; 4]) }.bounds().is_none());
|
||||
}
|
||||
}
|
||||
@@ -772,8 +772,33 @@ pub enum MaskShape {
|
||||
Subject,
|
||||
Sky,
|
||||
Background,
|
||||
/// One object picked by clicks (SAM 3 point prompts): `hint` holds the clicks that include,
|
||||
/// `exclude` the ones that exclude; `seg` the segmentation computed from them.
|
||||
Object {
|
||||
hint: Vec<Point>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
exclude: Vec<Point>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
seg: Option<crate::SegMask>,
|
||||
/// Zoomed-in passes over parts of the image (higher resolution than `seg`), used
|
||||
/// inside their rectangles.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty", deserialize_with = "crate::segmask::de_detail")]
|
||||
detail: Vec<crate::SegMask>,
|
||||
/// Edge −100..100: below 0 harder (a steeper transition), above 0 softer (feathered).
|
||||
#[serde(default, skip_serializing_if = "is_zero")]
|
||||
edge: f64,
|
||||
},
|
||||
/// Everything a description names ("sky", "the red car": SAM 3 concept prompts); `seg` is
|
||||
/// the segmentation computed for it.
|
||||
Prompt {
|
||||
text: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
seg: Option<crate::SegMask>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty", deserialize_with = "crate::segmask::de_detail")]
|
||||
detail: Vec<crate::SegMask>,
|
||||
/// Edge −100..100, as for `Object`.
|
||||
#[serde(default, skip_serializing_if = "is_zero")]
|
||||
edge: f64,
|
||||
},
|
||||
People {
|
||||
person: u32,
|
||||
|
||||
@@ -30,6 +30,11 @@ serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
log = { workspace = true }
|
||||
ab_glyph = { workspace = true }
|
||||
lightcraft-segment = { workspace = true, optional = true }
|
||||
|
||||
[features]
|
||||
# AI masks with SAM 3 (Object / Describe). Off for the web and CLI builds.
|
||||
sam = ["dep:lightcraft-segment"]
|
||||
|
||||
[dev-dependencies]
|
||||
libc = { workspace = true }
|
||||
@@ -0,0 +1,59 @@
|
||||
//! Optional build input: with `CRAFT_FONTS_DIR=<storytold/craft-fonts checkout>`, embed every font
|
||||
//! in its `fonts/manifest.txt` as `CRAFT_FONTS` (see src/fonts.rs). Unset, `CRAFT_FONTS` is empty.
|
||||
//! Recipe from craft-fonts' docs/integration.md; craft-fonts is never a Cargo dependency, and this
|
||||
//! script only reads a local directory (no network).
|
||||
//!
|
||||
//! wasm32 (the web build) embeds only the UI font, BIZ UDPGothic Regular (~4.5 MB), to keep the
|
||||
//! `.wasm` within the web size budget (Cloudflare serves at most 25 MiB per file); the export
|
||||
//! watermark falls back to it there. Native builds embed every font.
|
||||
use std::fmt::Write as _;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn main() {
|
||||
println!("cargo::rerun-if-changed=build.rs");
|
||||
println!("cargo::rerun-if-env-changed=CRAFT_FONTS_DIR");
|
||||
println!("cargo::rerun-if-env-changed=CRAFT_FONTS_REQUIRED");
|
||||
let mut src = String::from("pub static CRAFT_FONTS: &[CraftFont] = &[\n");
|
||||
if let Some(dir) = std::env::var_os("CRAFT_FONTS_DIR").map(PathBuf::from) {
|
||||
match craft_fonts(&dir) {
|
||||
Ok(entries) => src.push_str(&entries),
|
||||
Err(e) if std::env::var_os("CRAFT_FONTS_REQUIRED").is_some() => {
|
||||
println!("cargo::error=CRAFT_FONTS_DIR={}: {e}", dir.display());
|
||||
}
|
||||
Err(e) => println!("cargo::warning=building without craft-fonts: CRAFT_FONTS_DIR={}: {e}", dir.display()),
|
||||
}
|
||||
}
|
||||
src.push_str("];\n");
|
||||
let out = PathBuf::from(std::env::var_os("OUT_DIR").unwrap_or_default()).join("craft_fonts.rs");
|
||||
if let Err(e) = std::fs::write(&out, src) {
|
||||
println!("cargo::error=writing {}: {e}", out.display());
|
||||
}
|
||||
}
|
||||
|
||||
/// One `CraftFont { .. }` initialiser per manifest line.
|
||||
fn craft_fonts(dir: &std::path::Path) -> Result<String, String> {
|
||||
let manifest = dir.join("fonts/manifest.txt");
|
||||
println!("cargo::rerun-if-changed={}", manifest.display());
|
||||
let text = std::fs::read_to_string(&manifest).map_err(|e| format!("{}: {e}", manifest.display()))?;
|
||||
let wasm = std::env::var("CARGO_CFG_TARGET_ARCH").is_ok_and(|a| a == "wasm32");
|
||||
let mut out = String::new();
|
||||
for line in text.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')) {
|
||||
let f: Vec<&str> = line.split(" | ").map(str::trim).collect();
|
||||
let [family, style, file, scripts, ..] = f.as_slice() else {
|
||||
return Err(format!("malformed manifest line: {line}"));
|
||||
};
|
||||
if wasm && (*family, *style) != ("BIZ UDPGothic", "Regular") {
|
||||
continue;
|
||||
}
|
||||
let path = dir.join(file).canonicalize().map_err(|e| format!("{file}: {e}"))?;
|
||||
println!("cargo::rerun-if-changed={}", path.display());
|
||||
let scripts: Vec<String> = scripts.split(',').map(|s| format!("{:?}", s.trim())).collect();
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" CraftFont {{ family: {family:?}, style: {style:?}, scripts: &[{}], bytes: include_bytes!({:?}) }},",
|
||||
scripts.join(", "),
|
||||
path.display().to_string(),
|
||||
);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
@@ -1,25 +1,68 @@
|
||||
//! Estimate an ARW starting look from its own JPEG. Colour and luminance are fitted separately;
|
||||
//! the JPEG supplies correspondences only, never output pixels or a replacement for RAW editing.
|
||||
use lightcraft_color::{Mat3, luminance_2020};
|
||||
//! Estimate the starting look of a raw without a camera colour matrix (Sony ARW, Nikon NEF) from its
|
||||
//! own JPEG. Colour and luminance are fitted separately; the JPEG supplies correspondences only,
|
||||
//! never output pixels or a replacement for RAW editing.
|
||||
//! A global matrix can't follow the camera's hue-dependent rendering (the best matrix rendered a
|
||||
//! lime shirt olive that the camera kept lime): a hue/saturation table fitted to the residuals
|
||||
//! (applied like a DNG `ProfileHueSatMap`) corrects that when it also improves the held-out pixels.
|
||||
use lightcraft_color::{D50, D65, Mat3, PROPHOTO, REC2020, bradford, luminance_2020};
|
||||
use lightcraft_pipeline::tone::{CameraTone, ToneMap};
|
||||
use lightcraft_raster::{
|
||||
Rgb32f,
|
||||
resample::{Filter, fit},
|
||||
};
|
||||
use lightcraft_raw::{RawImage, color::CameraTransform};
|
||||
use lightcraft_raw::{RawFormat, RawImage, color::CameraTransform, profile::HsvTable};
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct CameraLook {
|
||||
pub matrix: Mat3,
|
||||
pub tone: CameraTone,
|
||||
/// Hue/saturation correction after `matrix`, in linear ProPhoto RGB (DNG `ProfileHueSatMap`).
|
||||
pub hue_sat: Option<HsvTable>,
|
||||
}
|
||||
|
||||
/// Long edge of the sensor/JPEG proxy a single photo's look is fitted on (the acceptance gates
|
||||
/// below were set at this size).
|
||||
const PROXY: usize = 96;
|
||||
/// Long edge of the proxies pooled for a camera profile: small objects (a shirt) get 4× the samples.
|
||||
pub(crate) const PROFILE_PROXY: usize = 192;
|
||||
|
||||
/// Raw formats whose decoder supplies vendor white-balance multipliers but no camera colour matrix:
|
||||
/// their starting look is fitted to the file's own JPEG, and white balance is relative to the
|
||||
/// as-shot look (`docs/camera-preview-colour.md`). The catalog's `Photo::relative_wb` matches the
|
||||
/// same formats by file extension.
|
||||
pub(crate) fn file_local_look(format: RawFormat) -> bool {
|
||||
matches!(format, RawFormat::Arw | RawFormat::Nef | RawFormat::Nrw)
|
||||
}
|
||||
|
||||
pub(crate) fn fit_preview(raw: &RawImage, bytes: &[u8], transform: &CameraTransform) -> Option<CameraLook> {
|
||||
if !transform.matrix_is_fallback || raw.format != lightcraft_raw::RawFormat::Arw {
|
||||
if !transform.matrix_is_fallback || !file_local_look(raw.format) {
|
||||
return None;
|
||||
}
|
||||
let (sensor, reference) = proxies(raw, bytes, transform, PROXY)?;
|
||||
// A camera profile pooled from many photos knows colours this photo shows too little of;
|
||||
// only the tone and chroma curves are fitted per photo (DRO and picture styles vary).
|
||||
let profile = raw.metadata.model.as_deref().and_then(crate::camera_profiles::get);
|
||||
let colour = profile.as_ref().and_then(|p| Some((p.matrix().mul(&transform.matrix.inverse()?), p.hue_sat.clone())));
|
||||
let look = fit_pairs_with(&sensor, &reference, colour)?;
|
||||
if lightcraft_pipeline::profiling() {
|
||||
eprintln!(
|
||||
"[profile] {:?} camera look: {:?}, {:?}, hue/sat table {}, camera profile {}",
|
||||
raw.format,
|
||||
look.matrix.0,
|
||||
look.tone,
|
||||
look.hue_sat.is_some(),
|
||||
profile.is_some()
|
||||
);
|
||||
}
|
||||
Some(look)
|
||||
}
|
||||
|
||||
/// Same-size proxies of the sensor (white-balanced, baseline exposure, through `transform`'s
|
||||
/// matrix: the generic camera ≈ sRGB model) and of the file's embedded camera JPEG (linear Rec.2020).
|
||||
fn proxies(raw: &RawImage, bytes: &[u8], transform: &CameraTransform, size: usize) -> Option<(Rgb32f, Rgb32f)> {
|
||||
let jpeg = lightcraft_raw::embedded_preview(bytes)?;
|
||||
let decoded = lightcraft_codecs::decode(&jpeg, lightcraft_codecs::DecodeOptions { max_size: Some((384, 384)), max_pixels: 64_000_000 }).ok()?;
|
||||
let edge = (2 * size).max(384) as u32;
|
||||
let decoded = lightcraft_codecs::decode(&jpeg, lightcraft_codecs::DecodeOptions { max_size: Some((edge, edge)), max_pixels: 64_000_000 }).ok()?;
|
||||
let reference = decoded.to_working();
|
||||
let crop = raw.crop.clipped(raw.active_area.width, raw.active_area.height);
|
||||
if crop.width == 0 || crop.height == 0 || reference.width == 0 || reference.height == 0 {
|
||||
@@ -30,17 +73,34 @@ pub(crate) fn fit_preview(raw: &RawImage, bytes: &[u8], transform: &CameraTransf
|
||||
return None;
|
||||
}
|
||||
// Fixed, bounded proxy: the selected look cannot depend on thumbnail/export resolution.
|
||||
let k = (crop.width.max(crop.height).div_ceil(384).max(2)).div_ceil(2) * 2;
|
||||
let k = (crop.width.max(crop.height).div_ceil(edge as usize).max(2)).div_ceil(2) * 2;
|
||||
let sensor = raw.develop_binned(k, 0.99).ok()??;
|
||||
let mut sensor = fit(&sensor, 96, 96, Filter::Box);
|
||||
let mut sensor = fit(&sensor, size, size, Filter::Box);
|
||||
let reference = fit(&reference, sensor.width, sensor.height, Filter::Box);
|
||||
let gain = 2f32.powf(transform.baseline_exposure as f32);
|
||||
sensor.map_in_place(|p| transform.matrix.apply_f32(std::array::from_fn(|i| p[i] * transform.wb[i] * gain)));
|
||||
let look = fit_pairs(&sensor, &reference)?;
|
||||
if lightcraft_pipeline::profiling() {
|
||||
eprintln!("[profile] ARW camera look: {:?}, {:?}", look.matrix.0, look.tone);
|
||||
Some((sensor, reference))
|
||||
}
|
||||
|
||||
/// Colour training pairs of one raw for a camera profile: white-balanced camera RGB (with the
|
||||
/// baseline exposure) → its camera JPEG (linear Rec.2020). `None` for formats with their own
|
||||
/// colour matrices, other files or unusable previews.
|
||||
pub(crate) fn profile_pairs(raw: &RawImage, bytes: &[u8]) -> Option<Vec<([f64; 3], [f64; 3])>> {
|
||||
if !file_local_look(raw.format) || lightcraft_raw::color::has_matrix(&raw.color) {
|
||||
return None;
|
||||
}
|
||||
Some(look)
|
||||
let transform = lightcraft_raw::color::camera_transform(raw, lightcraft_raw::color::as_shot_white_xy(raw));
|
||||
let (sensor, reference) = proxies(raw, bytes, &transform, PROFILE_PROXY)?;
|
||||
let to_camera = transform.matrix.inverse()?;
|
||||
let (pairs, _) = collect_pairs(&sensor, &reference)?;
|
||||
Some(pairs.into_iter().map(|(x, y)| (to_camera.apply(x), y)).collect())
|
||||
}
|
||||
|
||||
/// A camera profile's colour model (matrix from white-balanced camera RGB, hue/saturation table)
|
||||
/// fitted to pairs pooled from many photos.
|
||||
pub(crate) fn fit_profile(pairs: &[([f64; 3], [f64; 3])]) -> Option<(Mat3, Option<HsvTable>)> {
|
||||
let matrix = fit_matrix(pairs)?;
|
||||
Some((matrix, fit_hue_sat(pairs, &matrix)))
|
||||
}
|
||||
|
||||
/// A fit must cut the held-out squared error to below this share of the fallback's.
|
||||
@@ -55,27 +115,43 @@ fn luma(p: [f64; 3]) -> f64 {
|
||||
p[0] * 0.2627 + p[1] * 0.6780 + p[2] * 0.0593
|
||||
}
|
||||
|
||||
/// The finish stage's tone map and chroma curve (`lightcraft_pipeline::finish`).
|
||||
fn displayed(scene: [f64; 3], tone: &ToneMap) -> [f64; 3] {
|
||||
let scene = scene.map(|v| v.max(0.0));
|
||||
let y = luma(scene);
|
||||
if y <= 0.0 {
|
||||
return [0.0; 3];
|
||||
}
|
||||
scene.map(|v| v * f64::from(tone.apply(y as f32)) / y)
|
||||
let o = f64::from(tone.apply(y as f32));
|
||||
let k = f64::from(tone.chroma_scale(o as f32));
|
||||
scene.map(|v| o + (v * o / y - o) * k)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn fit_pairs(sensor: &Rgb32f, reference: &Rgb32f) -> Option<CameraLook> {
|
||||
fit_pairs_with(sensor, reference, None)
|
||||
}
|
||||
|
||||
/// Training pairs (sensor → JPEG, unclipped midtones) and the wider set including highlights
|
||||
/// (for the chroma curve); `None` when too few, or the photo has too little colour.
|
||||
type Pairs = Vec<([f64; 3], [f64; 3])>;
|
||||
fn collect_pairs(sensor: &Rgb32f, reference: &Rgb32f) -> Option<(Pairs, Pairs)> {
|
||||
if (sensor.width, sensor.height) != (reference.width, reference.height) || sensor.data.len() != reference.data.len() {
|
||||
return None;
|
||||
}
|
||||
let mut pairs = Vec::new();
|
||||
// Highlights too (camera JPEGs bleach colours toward white there), for the chroma curve only.
|
||||
let mut bright = Vec::new();
|
||||
let mut colour = 0;
|
||||
for (input, output) in sensor.data.iter().zip(&reference.data) {
|
||||
let y = luminance_2020(*output);
|
||||
if !input.iter().all(|v| v.is_finite() && *v > 0.001 && *v < 1.5)
|
||||
|| !output.iter().all(|v| v.is_finite() && *v > 0.004 && *v < 0.98)
|
||||
|| !(0.015..0.85).contains(&y)
|
||||
{
|
||||
if !input.iter().all(|v| v.is_finite() && *v > 0.001 && *v < 1.5) || !output.iter().all(|v| v.is_finite() && *v >= 0.0) {
|
||||
continue;
|
||||
}
|
||||
if (0.015..=1.0).contains(&y) {
|
||||
bright.push((input.map(f64::from), output.map(f64::from)));
|
||||
}
|
||||
if !output.iter().all(|v| *v > 0.004 && *v < 0.98) || !(0.015..0.85).contains(&y) {
|
||||
continue;
|
||||
}
|
||||
let min = output.iter().copied().fold(f32::INFINITY, f32::min);
|
||||
@@ -83,9 +159,11 @@ fn fit_pairs(sensor: &Rgb32f, reference: &Rgb32f) -> Option<CameraLook> {
|
||||
colour += usize::from(max - min > 0.05);
|
||||
pairs.push((input.map(f64::from), output.map(f64::from)));
|
||||
}
|
||||
if pairs.len() < 256 || colour < pairs.len() / 20 {
|
||||
return None;
|
||||
}
|
||||
(pairs.len() >= 256 && colour >= pairs.len() / 20).then_some((pairs, bright))
|
||||
}
|
||||
|
||||
/// Ridge-regularised 3×3 chromaticity matrix (luminance-normalised RGB) on the training pairs.
|
||||
fn fit_matrix(pairs: &[([f64; 3], [f64; 3])]) -> Option<Mat3> {
|
||||
let mut gram = [[0.0; 3]; 3];
|
||||
let mut cross = [[0.0; 3]; 3];
|
||||
for (i, (x, y)) in pairs.iter().enumerate() {
|
||||
@@ -93,6 +171,9 @@ fn fit_pairs(sensor: &Rgb32f, reference: &Rgb32f) -> Option<CameraLook> {
|
||||
continue;
|
||||
}
|
||||
let (lx, ly) = (luma(*x), luma(*y));
|
||||
if lx <= 0.0 || ly <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
for row in 0..3 {
|
||||
for col in 0..3 {
|
||||
// Normalising by luminance prevents a camera S-curve from corrupting colour.
|
||||
@@ -113,36 +194,274 @@ fn fit_pairs(sensor: &Rgb32f, reference: &Rgb32f) -> Option<CameraLook> {
|
||||
cross[i][i] += regularization;
|
||||
}
|
||||
let matrix = Mat3(cross).mul(&Mat3(gram).inverse()?);
|
||||
if !matrix.0.iter().flatten().all(|v| v.is_finite() && v.abs() < 8.0) {
|
||||
return None;
|
||||
matrix.0.iter().flatten().all(|v| v.is_finite() && v.abs() < 8.0).then_some(matrix)
|
||||
}
|
||||
|
||||
/// The photo's look: its own matrix (with and without a hue/saturation table), or the given
|
||||
/// colour model (a camera profile's, in the sensor proxy's space), each completed with a tone
|
||||
/// and chroma curve fitted to this photo.
|
||||
fn fit_pairs_with(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>) -> Option<CameraLook> {
|
||||
let (pairs, bright) = collect_pairs(sensor, reference)?;
|
||||
let candidates = match colour {
|
||||
Some(given) => vec![given],
|
||||
None => {
|
||||
let matrix = fit_matrix(&pairs)?;
|
||||
vec![(matrix, fit_hue_sat(&pairs, &matrix)), (matrix, None)]
|
||||
}
|
||||
};
|
||||
// Matrix + table when the table helps the held-out pixels, else the matrix alone. Chosen by
|
||||
// error in gamma-encoded display values (closer to what is seen: in linear values a slightly
|
||||
// missed bright rock outweighs a clearly wrong dark shirt); the acceptance gates below stay linear.
|
||||
let mut best: Option<(f64, f64, usize, CameraLook)> = None;
|
||||
for (matrix, hue_sat) in candidates {
|
||||
let correction = hue_sat.as_ref().and_then(HueSat::new);
|
||||
let colour = |x: [f64; 3]| {
|
||||
let p = matrix.apply(x);
|
||||
correction.as_ref().map_or(p, |c| c.apply(p.map(|v| v as f32)).map(f64::from))
|
||||
};
|
||||
let tone_pairs: Vec<_> =
|
||||
pairs.iter().enumerate().filter(|(i, _)| i % 3 != 0).map(|(_, (x, y))| (luma(colour(*x).map(|v| v.max(0.0))), luma(*y))).collect();
|
||||
let Some(curve) = fit_tone(tone_pairs) else { continue };
|
||||
let mut look = CameraLook { matrix, tone: curve, hue_sat: hue_sat.clone() };
|
||||
if let Some(tone) = fit_chroma(&bright, &look) {
|
||||
look.tone = tone;
|
||||
}
|
||||
let tone = ToneMap::camera(&look.tone, 0.0, 0.0, 0.0);
|
||||
let (mut linear, mut perceptual, mut samples) = (0.0, 0.0, 0);
|
||||
for (x, target) in pairs.iter().step_by(3) {
|
||||
let corrected = displayed(colour(*x), &tone);
|
||||
for c in 0..3 {
|
||||
linear += (corrected[c] - target[c]).powi(2);
|
||||
perceptual += (corrected[c].max(0.0).powf(1.0 / 2.2) - target[c].max(0.0).powf(1.0 / 2.2)).powi(2);
|
||||
samples += 1;
|
||||
}
|
||||
}
|
||||
if linear.is_finite() && perceptual.is_finite() && best.as_ref().is_none_or(|b| perceptual < b.0) {
|
||||
best = Some((perceptual, linear, samples, look));
|
||||
}
|
||||
}
|
||||
let tone_pairs: Vec<_> =
|
||||
pairs.iter().enumerate().filter(|(i, _)| i % 3 != 0).map(|(_, (x, y))| (luma(matrix.apply(*x).map(|v| v.max(0.0))), luma(*y))).collect();
|
||||
let curve = fit_tone(tone_pairs)?;
|
||||
let tone = ToneMap::camera(&curve, 0.0, 0.0, 0.0);
|
||||
let (_, after, samples, look) = best?;
|
||||
let original_tone = ToneMap::new(0.0, 0.0, 0.0);
|
||||
let mut before = 0.0;
|
||||
let mut after = 0.0;
|
||||
let mut samples = 0;
|
||||
for (i, (x, target)) in pairs.iter().enumerate() {
|
||||
if i % 3 != 0 {
|
||||
continue;
|
||||
}
|
||||
let corrected = displayed(matrix.apply(*x), &tone);
|
||||
let original = displayed(*x, &original_tone);
|
||||
for c in 0..3 {
|
||||
before += (original[c] - target[c]).powi(2);
|
||||
after += (corrected[c] - target[c]).powi(2);
|
||||
samples += 1;
|
||||
}
|
||||
}
|
||||
let before: f64 = pairs
|
||||
.iter()
|
||||
.step_by(3)
|
||||
.map(|(x, target)| {
|
||||
let original = displayed(*x, &original_tone);
|
||||
(0..3).map(|c| (original[c] - target[c]).powi(2)).sum::<f64>()
|
||||
})
|
||||
.sum();
|
||||
if lightcraft_pipeline::profiling() {
|
||||
eprintln!("[profile] ARW holdout RMS {:.5} -> {:.5} ({samples} channels)", (before / samples as f64).sqrt(), (after / samples as f64).sqrt());
|
||||
eprintln!(
|
||||
"[profile] camera look holdout RMS {:.5} -> {:.5} ({samples} channels)",
|
||||
(before / samples as f64).sqrt(),
|
||||
(after / samples as f64).sqrt()
|
||||
);
|
||||
}
|
||||
if !after.is_finite() || samples == 0 || after >= before * MIN_IMPROVEMENT || after / samples as f64 > MAX_HOLDOUT_RMS.powi(2) {
|
||||
if samples == 0 || after >= before * MIN_IMPROVEMENT || after / samples as f64 > MAX_HOLDOUT_RMS.powi(2) {
|
||||
return None;
|
||||
}
|
||||
Some(CameraLook { matrix, tone: curve })
|
||||
Some(look)
|
||||
}
|
||||
|
||||
/// Linear Rec.2020 D65 → linear ProPhoto RGB D50, the space DNG hue/saturation tables work in.
|
||||
fn to_prophoto() -> Mat3 {
|
||||
PROPHOTO.from_xyz().mul(&bradford(D65, D50)).mul(&REC2020.to_xyz())
|
||||
}
|
||||
|
||||
/// A fitted [`CameraLook::hue_sat`] table, ready to apply to linear Rec.2020 pixels. It changes
|
||||
/// hue and saturation only: luminance is restored, the camera tone curve owns it.
|
||||
pub(crate) struct HueSat<'a> {
|
||||
table: &'a HsvTable,
|
||||
to: [[f32; 3]; 3],
|
||||
from: [[f32; 3]; 3],
|
||||
}
|
||||
|
||||
impl<'a> HueSat<'a> {
|
||||
pub fn new(table: &'a HsvTable) -> Option<HueSat<'a>> {
|
||||
let to = to_prophoto();
|
||||
Some(HueSat { table, to: to.to_f32(), from: to.inverse()?.to_f32() })
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn apply(&self, rgb: [f32; 3]) -> [f32; 3] {
|
||||
let mul = |m: &[[f32; 3]; 3], v: [f32; 3]| -> [f32; 3] { std::array::from_fn(|i| m[i][0] * v[0] + m[i][1] * v[1] + m[i][2] * v[2]) };
|
||||
let out = mul(&self.from, self.table.apply(mul(&self.to, rgb)));
|
||||
let (before, after) = (luminance_2020(rgb), luminance_2020(out));
|
||||
if before > 0.0 && after > 0.0 && out.iter().all(|v| v.is_finite()) { out.map(|v| v * before / after) } else { rgb }
|
||||
}
|
||||
}
|
||||
|
||||
/// HSV hue (degrees) and saturation; `None` for black or non-finite colours.
|
||||
fn hue_saturation(p: [f64; 3]) -> Option<(f64, f64)> {
|
||||
let max = p[0].max(p[1]).max(p[2]);
|
||||
let min = p[0].min(p[1]).min(p[2]);
|
||||
if !(max.is_finite() && min.is_finite()) || max <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
let d = max - min;
|
||||
if d <= 0.0 {
|
||||
return Some((0.0, 0.0));
|
||||
}
|
||||
let h = if max == p[0] {
|
||||
((p[1] - p[2]) / d).rem_euclid(6.0)
|
||||
} else if max == p[1] {
|
||||
(p[2] - p[0]) / d + 2.0
|
||||
} else {
|
||||
(p[0] - p[1]) / d + 4.0
|
||||
};
|
||||
Some((h * 60.0, d / max))
|
||||
}
|
||||
|
||||
/// Table resolution: 5° hue steps (a coarser grid blurred the lime shirt into neighbouring browns
|
||||
/// that want the opposite shift), saturation 0, 0.25 … 1. Value is not an axis: tone is fitted separately.
|
||||
const TABLE_HUES: usize = 72;
|
||||
const TABLE_SATS: usize = 5;
|
||||
/// Value levels (sRGB-encoded, as DNG tables with encoding 1): cameras turn dark yellows toward
|
||||
/// orange but bright yellow-greens toward green, which one shift per hue averages away.
|
||||
const TABLE_VALS: usize = 5;
|
||||
/// Kernel widths around each table node (hue in degrees, saturation, encoded value).
|
||||
const KERNEL_HUE: f64 = 2.5;
|
||||
const KERNEL_SAT: f64 = 0.15;
|
||||
const KERNEL_VAL: f64 = 0.12;
|
||||
/// Kernel weight at which a node keeps half of its estimate; sparse nodes shrink to identity.
|
||||
const SHRINK_WEIGHT: f64 = 5.0;
|
||||
|
||||
/// Fit hue shifts and saturation scales of the training pairs left after `matrix`, per (hue,
|
||||
/// saturation) node, kernel-weighted and shrunk toward identity where the photo has few samples.
|
||||
/// Saturation 0 stays identity, so neutrals are never tinted.
|
||||
fn fit_hue_sat(pairs: &[([f64; 3], [f64; 3])], matrix: &Mat3) -> Option<HsvTable> {
|
||||
let to = to_prophoto();
|
||||
let samples: Vec<(f64, f64, f64, f64, f64)> = pairs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(i, _)| i % 3 != 0)
|
||||
.filter_map(|(_, (x, y))| {
|
||||
let p = to.apply(matrix.apply(*x));
|
||||
let (hp, sp) = hue_saturation(p)?;
|
||||
let value = f64::from(lightcraft_color::transfer::linear_to_srgb(p[0].max(p[1]).max(p[2]).min(1.0) as f32));
|
||||
let (ht, st) = hue_saturation(to.apply(*y))?;
|
||||
// hue is meaningless near neutral
|
||||
if sp < 0.08 || st < 0.02 {
|
||||
return None;
|
||||
}
|
||||
let shift = ((ht - hp + 180.0).rem_euclid(360.0) - 180.0).clamp(-30.0, 30.0);
|
||||
let log_scale = (st / sp).ln().clamp(-0.7, 0.7);
|
||||
Some((hp, sp.min(1.0), value, shift, log_scale))
|
||||
})
|
||||
.collect();
|
||||
if samples.len() < 64 {
|
||||
return None;
|
||||
}
|
||||
// Samples by table hue step: a node only looks at hues within its kernel's reach.
|
||||
let step = 360.0 / TABLE_HUES as f64;
|
||||
let mut by_hue: Vec<Vec<(f64, f64, f64, f64, f64)>> = vec![Vec::new(); TABLE_HUES];
|
||||
for sample in samples {
|
||||
if let Some(bin) = by_hue.get_mut(((sample.0.rem_euclid(360.0) / step) as usize).min(TABLE_HUES - 1)) {
|
||||
bin.push(sample);
|
||||
}
|
||||
}
|
||||
let reach = (4.0 * KERNEL_HUE / step).ceil() as usize + 1;
|
||||
use rayon::prelude::*;
|
||||
let data: Vec<[f32; 3]> = (0..TABLE_VALS * TABLE_HUES * TABLE_SATS)
|
||||
.into_par_iter()
|
||||
.map(|index| {
|
||||
let (v, h, s) = (index / (TABLE_HUES * TABLE_SATS), index / TABLE_SATS % TABLE_HUES, index % TABLE_SATS);
|
||||
if s == 0 {
|
||||
return [0.0, 1.0, 1.0];
|
||||
}
|
||||
let (val, hue, sat) = (v as f64 / (TABLE_VALS - 1) as f64, h as f64 * step, s as f64 / (TABLE_SATS - 1) as f64);
|
||||
let (mut weight, mut shift, mut log_scale) = (0.0, 0.0, 0.0);
|
||||
for offset in 0..=2 * reach {
|
||||
let Some(bin) = by_hue.get((h + TABLE_HUES * 2 + offset - reach) % TABLE_HUES) else { continue };
|
||||
for &(hp, sp, vp, dh, ls) in bin {
|
||||
let dhue = (hp - hue + 180.0).rem_euclid(360.0) - 180.0;
|
||||
if dhue.abs() > 4.0 * KERNEL_HUE {
|
||||
continue;
|
||||
}
|
||||
let d2 = (dhue / KERNEL_HUE).powi(2) + ((sp - sat) / KERNEL_SAT).powi(2) + ((vp - val) / KERNEL_VAL).powi(2);
|
||||
let k = (-0.5 * d2).exp() * sp;
|
||||
weight += k;
|
||||
shift += k * dh;
|
||||
log_scale += k * ls;
|
||||
}
|
||||
}
|
||||
if weight <= 0.0 {
|
||||
return [0.0, 1.0, 1.0];
|
||||
}
|
||||
let shrink = 1.0 / (weight + SHRINK_WEIGHT);
|
||||
[(shift * shrink) as f32, (log_scale * shrink).exp() as f32, 1.0]
|
||||
})
|
||||
.collect();
|
||||
data.iter().all(|e| e.iter().all(|v| v.is_finite())).then_some(HsvTable {
|
||||
hue_divisions: TABLE_HUES,
|
||||
sat_divisions: TABLE_SATS,
|
||||
val_divisions: TABLE_VALS,
|
||||
data,
|
||||
srgb_value: true,
|
||||
})
|
||||
}
|
||||
|
||||
/// Kernel width of the chroma curve's nodes (display luminance) and the weight at which a node
|
||||
/// keeps half of its estimate.
|
||||
const CHROMA_KERNEL: f64 = 0.08;
|
||||
const CHROMA_SHRINK: f64 = 2.0;
|
||||
|
||||
/// The look's tone curve with a chroma-by-display-luminance curve fitted to `pairs` (two thirds
|
||||
/// train, one third held out), when that lowers the held-out error. Colourfulness is compared as
|
||||
/// the distance from neutral of luminance-normalised RGB.
|
||||
fn fit_chroma(pairs: &[([f64; 3], [f64; 3])], look: &CameraLook) -> Option<CameraTone> {
|
||||
let correction = look.hue_sat.as_ref().and_then(HueSat::new);
|
||||
let scene = |x: &[f64; 3]| {
|
||||
let p = look.matrix.apply(*x);
|
||||
correction.as_ref().map_or(p, |c| c.apply(p.map(|v| v as f32)).map(f64::from))
|
||||
};
|
||||
let tone = ToneMap::camera(&look.tone, 0.0, 0.0, 0.0);
|
||||
let predict = |x: &[f64; 3]| displayed(scene(x), &tone);
|
||||
let chroma = |p: [f64; 3]| {
|
||||
let y = luma(p);
|
||||
(y > 0.0).then(|| p.iter().map(|v| (v / y - 1.0).powi(2)).sum::<f64>().sqrt())
|
||||
};
|
||||
let samples: Vec<(f64, f64, f64)> = pairs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(i, _)| i % 3 != 0)
|
||||
.filter_map(|(_, (x, y))| {
|
||||
let p = predict(x);
|
||||
let (cp, cy) = (chroma(p)?, chroma(*y)?);
|
||||
(cp > 0.05).then(|| (luma(p), cp, (cy.max(1e-3) / cp).ln().clamp(0.05f64.ln(), 2.5f64.ln())))
|
||||
})
|
||||
.collect();
|
||||
if samples.len() < 64 {
|
||||
return None;
|
||||
}
|
||||
let mut curve = [1.0f32; lightcraft_pipeline::tone::CHROMA_N];
|
||||
for (j, node) in curve.iter_mut().enumerate() {
|
||||
let at = j as f64 / (lightcraft_pipeline::tone::CHROMA_N - 1) as f64;
|
||||
let (mut weight, mut sum) = (0.0, 0.0);
|
||||
for &(o, cp, log_ratio) in &samples {
|
||||
let k = (-0.5 * ((o - at) / CHROMA_KERNEL).powi(2)).exp() * cp;
|
||||
weight += k;
|
||||
sum += k * log_ratio;
|
||||
}
|
||||
*node = (sum / (weight + CHROMA_SHRINK)).exp() as f32;
|
||||
}
|
||||
let fitted = look.tone.with_chroma(curve)?;
|
||||
let with = ToneMap::camera(&fitted, 0.0, 0.0, 0.0);
|
||||
let error = |map: &ToneMap| -> f64 {
|
||||
pairs
|
||||
.iter()
|
||||
.step_by(3)
|
||||
.map(|(x, y)| {
|
||||
let p = displayed(scene(x), map);
|
||||
(0..3).map(|c| (p[c] - y[c]).powi(2)).sum::<f64>()
|
||||
})
|
||||
.sum()
|
||||
};
|
||||
let (before, after) = (error(&tone), error(&with));
|
||||
if lightcraft_pipeline::profiling() {
|
||||
eprintln!("[profile] ARW chroma curve {curve:?}: held-out error {before:.4} -> {after:.4}");
|
||||
}
|
||||
(after.is_finite() && after < before).then_some(fitted)
|
||||
}
|
||||
|
||||
fn median(values: &mut [f64]) -> Option<f64> {
|
||||
@@ -248,6 +567,192 @@ mod tests {
|
||||
assert!(fit_pairs(&sensor, &reference).is_some());
|
||||
}
|
||||
|
||||
/// A camera that rotates saturated yellow-greens toward green (like Sony's "Standard" on a lime
|
||||
/// shirt) can't be followed by a matrix alone: the fitted hue/saturation table must close most
|
||||
/// of the gap, keep neutrals neutral and keep luminance.
|
||||
#[test]
|
||||
fn hue_table_follows_a_hue_dependent_camera_rendering() {
|
||||
let to = to_prophoto();
|
||||
let from = to.inverse().unwrap();
|
||||
let mut sensor = Rgb32f::new(96, 64);
|
||||
let mut reference = sensor.clone();
|
||||
let mut lime = Vec::new();
|
||||
for (i, (src, dst)) in sensor.data.iter_mut().zip(&mut reference.data).enumerate() {
|
||||
let ev = 0.05 + (i % 29) as f32 * 0.02;
|
||||
// mostly greys and mild colours, plus a patch of saturated yellow-green
|
||||
*src = if i % 7 == 0 {
|
||||
[ev * 0.75, ev, ev * 0.2]
|
||||
} else {
|
||||
[ev * (0.85 + (i % 11) as f32 * 0.03), ev, ev * (0.85 + (i % 13) as f32 * 0.025)]
|
||||
};
|
||||
let (h, s) = hue_saturation(to.apply(src.map(f64::from))).unwrap();
|
||||
// the camera turns hues in 50°..110° (ProPhoto) by up to +15° and boosts their saturation
|
||||
let k = (1.0 - ((h - 80.0) / 30.0).powi(2)).max(0.0) * s.min(1.0);
|
||||
let mut p = to.apply(src.map(f64::from)).map(|v| v as f32);
|
||||
let table = HsvTable {
|
||||
hue_divisions: 1,
|
||||
sat_divisions: 2,
|
||||
val_divisions: 1,
|
||||
data: vec![[0.0, 1.0, 1.0], [15.0 * k as f32, 1.0 + 0.3 * k as f32, 1.0]],
|
||||
srgb_value: false,
|
||||
};
|
||||
p = table.apply(p);
|
||||
let p = from.apply(p.map(f64::from)).map(|v| v as f32);
|
||||
let scale = luminance_2020(*src) / luminance_2020(p);
|
||||
let p = p.map(|v| v * scale);
|
||||
let y = luminance_2020(p);
|
||||
*dst = p.map(|v| v * (1.0 - (-2.5 * y).exp()) / y);
|
||||
if i % 7 == 0 {
|
||||
lime.push(i);
|
||||
}
|
||||
}
|
||||
let fit = fit_pairs(&sensor, &reference).unwrap();
|
||||
let table = fit.hue_sat.as_ref().expect("a hue/saturation table is fitted");
|
||||
let hue_sat = HueSat::new(table).unwrap();
|
||||
let hue_error = |with_table: bool| {
|
||||
lime.iter()
|
||||
.map(|&i| {
|
||||
let p = fit.matrix.apply(sensor.data[i].map(f64::from)).map(|v| v as f32);
|
||||
let p = if with_table { hue_sat.apply(p) } else { p };
|
||||
let (h, _) = hue_saturation(to.apply(p.map(f64::from))).unwrap();
|
||||
let (t, _) = hue_saturation(to.apply(reference.data[i].map(f64::from))).unwrap();
|
||||
((h - t + 180.0).rem_euclid(360.0) - 180.0).abs()
|
||||
})
|
||||
.sum::<f64>()
|
||||
/ lime.len() as f64
|
||||
};
|
||||
let (before, after) = (hue_error(false), hue_error(true));
|
||||
assert!(after < before * 0.5, "lime hue error {before:.2}° -> {after:.2}°");
|
||||
// neutrals pass through and luminance is kept
|
||||
let grey = [0.3, 0.3, 0.3];
|
||||
assert!(hue_sat.apply(grey).iter().all(|v| (v - 0.3).abs() < 1e-4), "{:?}", hue_sat.apply(grey));
|
||||
let green = [0.2, 0.4, 0.05];
|
||||
assert!((luminance_2020(hue_sat.apply(green)) - luminance_2020(green)).abs() < 1e-5);
|
||||
}
|
||||
|
||||
/// A camera that saturates shadows and bleaches highlights toward white (a per-channel curve),
|
||||
/// which a luminance tone curve can't: the fitted chroma curve must follow it, so bright
|
||||
/// warm-tinted rock renders white instead of cream.
|
||||
#[test]
|
||||
fn chroma_curve_follows_highlight_bleaching() {
|
||||
let known = Mat3([[1.8, -0.4, -0.1], [-0.2, 1.5, -0.1], [-0.05, -0.3, 1.7]]);
|
||||
let camera_chroma = |o: f32| 1.3 - 1.1 * o;
|
||||
let mut sensor = Rgb32f::new(96, 64);
|
||||
let mut reference = sensor.clone();
|
||||
for (i, (src, dst)) in sensor.data.iter_mut().zip(&mut reference.data).enumerate() {
|
||||
let ev = 0.02 + (i % 37) as f32 * 0.03;
|
||||
*src = [ev * (0.75 + (i % 11) as f32 * 0.05), ev, ev * (0.7 + (i % 13) as f32 * 0.05)];
|
||||
let p = known.apply_f32(*src);
|
||||
let y = luminance_2020(p);
|
||||
let o = 1.0 - (-2.5 * y).exp();
|
||||
let k = camera_chroma(o);
|
||||
*dst = p.map(|v| (o + (v * o / y - o) * k).clamp(0.0, 0.999));
|
||||
}
|
||||
let fit = fit_pairs(&sensor, &reference).unwrap();
|
||||
// relative to the matrix, which already carries the average colourfulness
|
||||
let chroma = fit.tone.chroma();
|
||||
assert!(chroma[6] < 0.5 * chroma[1], "highlights bleach relative to shadows: {chroma:?}");
|
||||
// and the rendered highlights land on the camera's, far closer than without the curve
|
||||
let with = ToneMap::camera(&fit.tone, 0.0, 0.0, 0.0);
|
||||
let without = ToneMap::camera(&fit.tone.with_chroma([1.0; lightcraft_pipeline::tone::CHROMA_N]).unwrap(), 0.0, 0.0, 0.0);
|
||||
let highlight_error = |map: &ToneMap| -> f64 {
|
||||
sensor
|
||||
.data
|
||||
.iter()
|
||||
.zip(&reference.data)
|
||||
.filter(|(_, y)| luminance_2020(**y) > 0.7)
|
||||
.map(|(x, y)| {
|
||||
let p = displayed(fit.matrix.apply(x.map(f64::from)), map);
|
||||
(0..3).map(|c| (p[c] - f64::from(y[c])).powi(2)).sum::<f64>()
|
||||
})
|
||||
.sum()
|
||||
};
|
||||
let (a, b) = (highlight_error(&without), highlight_error(&with));
|
||||
assert!(b < 0.5 * a, "highlight error {a:.4} -> {b:.4}");
|
||||
}
|
||||
|
||||
/// Pooled pairs of several "photos" recover the camera's matrix, and a photo given that
|
||||
/// colour model (a camera profile) keeps it, fitting only its own tone.
|
||||
#[test]
|
||||
fn profile_colour_is_pooled_and_then_used_as_given() {
|
||||
let known = Mat3([[1.8, -0.4, -0.1], [-0.2, 1.5, -0.1], [-0.05, -0.3, 1.7]]);
|
||||
let photo = |seed: usize, strength: f32| {
|
||||
let mut sensor = Rgb32f::new(64, 48);
|
||||
let mut reference = sensor.clone();
|
||||
for (i, (src, dst)) in sensor.data.iter_mut().zip(&mut reference.data).enumerate() {
|
||||
let i = i + seed * 7;
|
||||
let ev = 0.04 + (i % 23) as f32 * 0.02;
|
||||
*src = [ev * (0.7 + (i % 11) as f32 * 0.05), ev, ev * (0.7 + (i % 13) as f32 * 0.05)];
|
||||
let p = known.apply_f32(*src);
|
||||
let y = luminance_2020(p);
|
||||
// each photo has its own tone (DRO, picture style)
|
||||
*dst = p.map(|v| v * (1.0 - (-strength * y).exp()) / y);
|
||||
}
|
||||
(sensor, reference)
|
||||
};
|
||||
let mut pool = Vec::new();
|
||||
for seed in 0..4 {
|
||||
let (sensor, reference) = photo(seed, 2.0 + seed as f32 * 0.5);
|
||||
pool.extend(collect_pairs(&sensor, &reference).unwrap().0);
|
||||
}
|
||||
let (matrix, _) = fit_profile(&pool).unwrap();
|
||||
// the camera's colours (luminance-normalised: the tone curve sets brightness)
|
||||
for x in [[0.8, 1.0, 0.75], [1.1, 1.0, 0.8], [0.75, 1.0, 1.2], [1.0; 3]] {
|
||||
let (a, b) = (matrix.apply(x), known.apply(x));
|
||||
let (la, lb) = (luma(a), luma(b));
|
||||
assert!((0..3).all(|c| (a[c] / la - b[c] / lb).abs() < 0.01), "{x:?}: {a:?} vs {b:?}");
|
||||
}
|
||||
let (sensor, reference) = photo(9, 3.5);
|
||||
let look = fit_pairs_with(&sensor, &reference, Some((matrix, None))).unwrap();
|
||||
assert_eq!(look.matrix, matrix);
|
||||
assert!(look.hue_sat.is_none());
|
||||
let tone = ToneMap::camera(&look.tone, 0.0, 0.0, 0.0);
|
||||
let error = sensor
|
||||
.data
|
||||
.iter()
|
||||
.zip(&reference.data)
|
||||
.map(|(x, y)| {
|
||||
let p = displayed(look.matrix.apply(x.map(f64::from)), &tone);
|
||||
(0..3).map(|c| (p[c] - f64::from(y[c])).powi(2)).sum::<f64>() / 3.0
|
||||
})
|
||||
.sum::<f64>()
|
||||
/ sensor.data.len() as f64;
|
||||
assert!(error.sqrt() < 0.02, "this photo's own tone is followed: RMS {}", error.sqrt());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hue_sat_passes_black_and_non_finite_pixels_through() {
|
||||
let table = HsvTable { hue_divisions: 4, sat_divisions: 2, val_divisions: 1, data: vec![[10.0, 1.5, 1.0]; 8], srgb_value: false };
|
||||
let hue_sat = HueSat::new(&table).unwrap();
|
||||
assert_eq!(hue_sat.apply([0.0; 3]), [0.0; 3]);
|
||||
let nan = hue_sat.apply([f32::NAN, 0.2, 0.1]);
|
||||
assert!(nan[0].is_nan() && nan[1] == 0.2 && nan[2] == 0.1);
|
||||
assert!(fit_hue_sat(&[], &Mat3::IDENTITY).is_none(), "too few samples");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sony_and_nikon_raws_get_a_file_local_look() {
|
||||
assert!([RawFormat::Arw, RawFormat::Nef, RawFormat::Nrw].into_iter().all(file_local_look));
|
||||
assert!(![RawFormat::Dng, RawFormat::Cr2, RawFormat::Raf].into_iter().any(file_local_look));
|
||||
}
|
||||
|
||||
/// A public D7500 NEF (skipped without the corpus): its look is fitted to its own JPEG and white
|
||||
/// balance is relative to the as-shot look.
|
||||
#[test]
|
||||
fn corpus_nef_gets_a_camera_look() {
|
||||
let path = std::env::var_os("LIGHTCRAFT_CORPUS")
|
||||
.map(std::path::PathBuf::from)
|
||||
.unwrap_or_else(|| std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus"))
|
||||
.join("raw/nef-nikon-d7500-lossless14.nef");
|
||||
let Ok(bytes) = std::fs::read(&path) else {
|
||||
eprintln!("skip: {} absent", path.display());
|
||||
return;
|
||||
};
|
||||
let (_, info) = crate::files::load_bytes(&bytes, 400).unwrap();
|
||||
assert!(info.camera_tone.is_some(), "no camera look fitted");
|
||||
assert!(info.relative_wb && info.as_shot_temp == 6500.0 && info.as_shot_tint == 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_monochrome_invalid_and_unrelated_previews() {
|
||||
let mut sensor = Rgb32f::new(32, 32);
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
//! Camera colour profiles of our own, pooled from many photos of one camera model.
|
||||
//!
|
||||
//! A raw file without colour matrices (Sony ARW, Nikon NEF) gets its look fitted to its own embedded camera
|
||||
//! JPEG (`camera_preview`), but one photo shows too little of some colours: a lime shirt covering a
|
||||
//! few dozen proxy pixels next to a hillside of foliage at the same hue. `lightcraft-cli calibrate`
|
||||
//! pools the colour pairs of many photos per model and fits one matrix and hue/saturation/value
|
||||
//! table; photos of that model then only fit their tone and chroma curves.
|
||||
//!
|
||||
//! Profiles are JSON files (`<model>.json`) in [`dir`]: `LIGHTCRAFT_CAMERA_PROFILES`, else
|
||||
//! `<config>/camera-profiles`; a local profile replaces the one built in ([`BUNDLED`], from
|
||||
//! `assets/camera-profiles/`). They are read once per process; a damaged or hostile file is
|
||||
//! ignored with a warning. They hold aggregate colour statistics only, never image content.
|
||||
use lightcraft_color::Mat3;
|
||||
use lightcraft_raw::profile::HsvTable;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex, OnceLock, PoisonError};
|
||||
|
||||
/// Format version of the profile files.
|
||||
const VERSION: u32 = 1;
|
||||
/// Largest profile file read (a 5 × 72 × 5 table is ~60 KB of JSON).
|
||||
const MAX_FILE: u64 = 4 << 20;
|
||||
|
||||
/// Profiles built into LightCraft (`assets/camera-profiles/`, see `assets/ATTRIBUTION.md`):
|
||||
/// `(model, JSON)`.
|
||||
pub const BUNDLED: &[(&str, &str)] = &[("ILCE-7M4", include_str!("../../../assets/camera-profiles/ILCE-7M4.json"))];
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct CameraProfile {
|
||||
pub version: u32,
|
||||
/// The camera model as the files report it (Exif `Model`), e.g. `ILCE-7M4`.
|
||||
pub model: String,
|
||||
/// Photos and colour pairs the profile was fitted on.
|
||||
pub files: usize,
|
||||
pub samples: usize,
|
||||
/// White-balanced camera RGB (with the baseline exposure) → linear Rec.2020, rows.
|
||||
matrix: [[f64; 3]; 3],
|
||||
/// Hue/saturation/value correction after `matrix` (linear ProPhoto RGB).
|
||||
pub hue_sat: Option<HsvTable>,
|
||||
}
|
||||
|
||||
impl CameraProfile {
|
||||
pub fn new(model: &str, files: usize, samples: usize, matrix: Mat3, hue_sat: Option<HsvTable>) -> CameraProfile {
|
||||
CameraProfile { version: VERSION, model: model.to_owned(), files, samples, matrix: matrix.0, hue_sat }
|
||||
}
|
||||
|
||||
pub fn matrix(&self) -> Mat3 {
|
||||
Mat3(self.matrix)
|
||||
}
|
||||
|
||||
/// Whether the data is usable (bounded matrix, table shape matching its data).
|
||||
fn valid(&self) -> bool {
|
||||
let matrix = self.matrix.iter().flatten().all(|v| v.is_finite() && v.abs() < 8.0) && Mat3(self.matrix).inverse().is_some();
|
||||
let table = self.hue_sat.as_ref().is_none_or(|t| {
|
||||
let dims = (1..=4096).contains(&t.hue_divisions) && (2..=4096).contains(&t.sat_divisions) && (1..=4096).contains(&t.val_divisions);
|
||||
let len = t.hue_divisions.checked_mul(t.sat_divisions).and_then(|n| n.checked_mul(t.val_divisions));
|
||||
dims && len == Some(t.data.len()) && t.data.iter().flatten().all(|v| v.is_finite()) && t.data.iter().all(|e| e[1] >= 0.0 && e[2] >= 0.0)
|
||||
});
|
||||
self.version == VERSION && !self.model.is_empty() && matrix && table
|
||||
}
|
||||
}
|
||||
|
||||
/// LightCraft's configuration folder (settings, GPU marker, camera profiles).
|
||||
pub fn config_dir() -> Option<PathBuf> {
|
||||
if cfg!(target_os = "macos") {
|
||||
std::env::var_os("HOME").map(|h| PathBuf::from(h).join("Library/Application Support/LightCraft"))
|
||||
} else if cfg!(windows) {
|
||||
std::env::var_os("APPDATA").map(|a| PathBuf::from(a).join("LightCraft"))
|
||||
} else {
|
||||
std::env::var_os("XDG_CONFIG_HOME")
|
||||
.map(PathBuf::from)
|
||||
.or_else(|| std::env::var_os("HOME").map(|h| PathBuf::from(h).join(".config")))
|
||||
.map(|c| c.join("lightcraft"))
|
||||
}
|
||||
}
|
||||
|
||||
/// Where camera profiles are read from and written to.
|
||||
pub fn dir() -> Option<PathBuf> {
|
||||
std::env::var_os("LIGHTCRAFT_CAMERA_PROFILES").map(PathBuf::from).or_else(|| config_dir().map(|d| d.join("camera-profiles")))
|
||||
}
|
||||
|
||||
/// File name of `model`'s profile: letters, digits, `-` and `_` kept, anything else `_`.
|
||||
pub fn file_name(model: &str) -> String {
|
||||
let name: String = model.trim().chars().map(|c| if c.is_ascii_alphanumeric() || c == '-' || c == '_' { c } else { '_' }).collect();
|
||||
format!("{name}.json")
|
||||
}
|
||||
|
||||
/// Read and validate a profile file.
|
||||
pub fn load(path: &Path) -> Result<CameraProfile, String> {
|
||||
let size = std::fs::metadata(path).map_err(|e| format!("{}: {e}", path.display()))?.len();
|
||||
if size > MAX_FILE {
|
||||
return Err(format!("{}: {size} bytes is too large for a camera profile", path.display()));
|
||||
}
|
||||
let bytes = std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
parse(&bytes, &path.display().to_string())
|
||||
}
|
||||
|
||||
/// Parse and validate a profile's JSON (`origin` names it in errors).
|
||||
fn parse(json: &[u8], origin: &str) -> Result<CameraProfile, String> {
|
||||
let profile: CameraProfile = serde_json::from_slice(json).map_err(|e| format!("{origin}: {e}"))?;
|
||||
if !profile.valid() {
|
||||
return Err(format!("{origin}: not a usable camera profile (version {}, model {:?})", profile.version, profile.model));
|
||||
}
|
||||
Ok(profile)
|
||||
}
|
||||
|
||||
/// The built-in profile for `model`, if any.
|
||||
fn bundled(model: &str) -> Option<CameraProfile> {
|
||||
let (_, json) = BUNDLED.iter().find(|(m, _)| *m == model)?;
|
||||
parse(json.as_bytes(), &format!("built-in profile {model}")).inspect_err(|e| eprintln!("lightcraft: ignoring camera profile {e}")).ok()
|
||||
}
|
||||
|
||||
/// Write `profile` to `dir` (created if needed) as `<model>.json`; returns the path.
|
||||
pub fn save(profile: &CameraProfile, dir: &Path) -> Result<PathBuf, String> {
|
||||
std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
|
||||
let path = dir.join(file_name(&profile.model));
|
||||
let json = serde_json::to_vec_pretty(profile).map_err(|e| e.to_string())?;
|
||||
let tmp = path.with_extension("json.tmp");
|
||||
std::fs::write(&tmp, json).map_err(|e| format!("{}: {e}", tmp.display()))?;
|
||||
std::fs::rename(&tmp, &path).map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
fn cache() -> &'static Mutex<HashMap<String, Option<Arc<CameraProfile>>>> {
|
||||
static CACHE: OnceLock<Mutex<HashMap<String, Option<Arc<CameraProfile>>>>> = OnceLock::new();
|
||||
CACHE.get_or_init(Default::default)
|
||||
}
|
||||
|
||||
/// The profile for camera `model`: from [`dir`], else built in; read once per process.
|
||||
pub fn get(model: &str) -> Option<Arc<CameraProfile>> {
|
||||
let model = model.trim();
|
||||
if model.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut cache = cache().lock().unwrap_or_else(PoisonError::into_inner);
|
||||
if let Some(hit) = cache.get(model) {
|
||||
return hit.clone();
|
||||
}
|
||||
let path = dir().map(|d| d.join(file_name(model)));
|
||||
let profile = match path {
|
||||
Some(path) if path.is_file() => match load(&path) {
|
||||
Ok(p) if p.model.trim() == model => Some(Arc::new(p)),
|
||||
Ok(p) => {
|
||||
eprintln!("lightcraft: ignoring camera profile {}: it is for {:?}, not {model:?}", path.display(), p.model);
|
||||
None
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("lightcraft: ignoring camera profile {e}");
|
||||
None
|
||||
}
|
||||
},
|
||||
_ => None,
|
||||
};
|
||||
let profile = profile.or_else(|| bundled(model).map(Arc::new));
|
||||
cache.insert(model.to_owned(), profile.clone());
|
||||
profile
|
||||
}
|
||||
|
||||
/// Changes when the profiles folder's contents change (per process): part of the render cache
|
||||
/// keys, so thumbnails rendered before a profile existed are not reused after.
|
||||
pub fn cache_key() -> u64 {
|
||||
static KEY: OnceLock<u64> = OnceLock::new();
|
||||
*KEY.get_or_init(|| {
|
||||
let entries = dir().and_then(|d| std::fs::read_dir(d).ok());
|
||||
let mut files: Vec<(String, u64, u64)> = entries
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.flatten()
|
||||
.filter(|e| e.path().extension().is_some_and(|x| x == "json"))
|
||||
.filter_map(|e| {
|
||||
let meta = e.metadata().ok()?;
|
||||
let modified = meta.modified().ok()?.duration_since(std::time::UNIX_EPOCH).ok()?.as_secs();
|
||||
Some((e.file_name().to_string_lossy().into_owned(), meta.len(), modified))
|
||||
})
|
||||
.collect();
|
||||
files.sort();
|
||||
let mut h = lightcraft_preview::Hasher128::new();
|
||||
for (name, len, modified) in &files {
|
||||
h.str(name).u64(*len).u64(*modified);
|
||||
}
|
||||
for (model, json) in BUNDLED {
|
||||
h.str(model).str(json);
|
||||
}
|
||||
h.finish().0 as u64
|
||||
})
|
||||
}
|
||||
|
||||
/// Per camera model: the photos read and their pooled colour pairs.
|
||||
#[derive(Default)]
|
||||
pub struct Pool {
|
||||
models: HashMap<String, (usize, Vec<([f64; 3], [f64; 3])>)>,
|
||||
}
|
||||
|
||||
/// Most colour pairs kept per photo, so a few busy photos can't dominate a profile.
|
||||
const PAIRS_PER_FILE: usize = 4000;
|
||||
|
||||
impl Pool {
|
||||
/// Add one raw file's colour pairs. `Ok(None)` when the file can't contribute (not an ARW or
|
||||
/// NEF without colour matrices, no usable camera JPEG, too little colour).
|
||||
pub fn add(&mut self, bytes: &[u8]) -> Result<Option<String>, String> {
|
||||
let raw = lightcraft_raw::decode(bytes).map_err(|e| e.to_string())?;
|
||||
let Some(model) = raw.metadata.model.as_deref().map(str::trim).filter(|m| !m.is_empty()) else { return Ok(None) };
|
||||
let Some(pairs) = crate::camera_preview::profile_pairs(&raw, bytes) else { return Ok(None) };
|
||||
let step = pairs.len().div_ceil(PAIRS_PER_FILE).max(1);
|
||||
let entry = self.models.entry(model.to_owned()).or_default();
|
||||
entry.0 += 1;
|
||||
entry.1.extend(pairs.into_iter().step_by(step));
|
||||
Ok(Some(model.to_owned()))
|
||||
}
|
||||
|
||||
/// Fit a profile per model with at least `min_files` photos.
|
||||
pub fn fit(&self, min_files: usize) -> Vec<Result<CameraProfile, String>> {
|
||||
let mut models: Vec<_> = self.models.iter().collect();
|
||||
models.sort_by(|a, b| a.0.cmp(b.0));
|
||||
models
|
||||
.into_iter()
|
||||
.filter(|(_, (files, _))| *files >= min_files)
|
||||
.map(|(model, (files, pairs))| {
|
||||
let (matrix, hue_sat) = crate::camera_preview::fit_profile(pairs).ok_or_else(|| format!("{model}: no usable colour fit"))?;
|
||||
Ok(CameraProfile::new(model, *files, pairs.len(), matrix, hue_sat))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Add another pool's photos and pairs (pools filled on separate threads).
|
||||
pub fn merge(&mut self, other: Pool) {
|
||||
for (model, (files, pairs)) in other.models {
|
||||
let entry = self.models.entry(model).or_default();
|
||||
entry.0 += files;
|
||||
entry.1.extend(pairs);
|
||||
}
|
||||
}
|
||||
|
||||
/// Photos read per model.
|
||||
pub fn files(&self) -> Vec<(String, usize)> {
|
||||
let mut v: Vec<_> = self.models.iter().map(|(m, (n, _))| (m.clone(), *n)).collect();
|
||||
v.sort();
|
||||
v
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn profile() -> CameraProfile {
|
||||
let table = HsvTable { hue_divisions: 4, sat_divisions: 2, val_divisions: 1, data: vec![[5.0, 1.2, 1.0]; 8], srgb_value: false };
|
||||
CameraProfile::new("ILCE-7M4", 12, 3456, Mat3([[1.6, -0.5, -0.1], [-0.2, 1.3, -0.1], [-0.1, -0.2, 1.3]]), Some(table))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profiles_round_trip_and_reject_bad_files() {
|
||||
let dir = std::env::temp_dir().join(format!("lc-camera-profiles-{}", std::process::id()));
|
||||
let path = save(&profile(), &dir).unwrap();
|
||||
assert_eq!(path.file_name().unwrap(), "ILCE-7M4.json");
|
||||
assert_eq!(load(&path).unwrap(), profile());
|
||||
let bad = |p: CameraProfile| {
|
||||
let path = dir.join("bad.json");
|
||||
std::fs::write(&path, serde_json::to_vec(&p).unwrap()).unwrap();
|
||||
load(&path).is_err()
|
||||
};
|
||||
let mut p = profile();
|
||||
p.matrix[0][0] = f64::NAN;
|
||||
assert!(bad(p), "non-finite matrix");
|
||||
let mut p = profile();
|
||||
p.matrix = [[0.0; 3]; 3];
|
||||
assert!(bad(p), "singular matrix");
|
||||
let mut p = profile();
|
||||
if let Some(t) = p.hue_sat.as_mut() {
|
||||
t.data.truncate(3);
|
||||
}
|
||||
assert!(bad(p), "table data doesn't match its shape");
|
||||
let mut p = profile();
|
||||
p.version = 99;
|
||||
assert!(bad(p), "unknown version");
|
||||
std::fs::write(dir.join("junk.json"), b"{not json").unwrap();
|
||||
assert!(load(&dir.join("junk.json")).is_err());
|
||||
assert!(load(&dir.join("missing.json")).is_err());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bundled_profiles_are_valid_and_named_after_their_model() {
|
||||
assert!(!BUNDLED.is_empty());
|
||||
for (model, json) in BUNDLED {
|
||||
let p = parse(json.as_bytes(), model).unwrap();
|
||||
assert_eq!(p.model, *model);
|
||||
assert!(p.files >= 5 && p.hue_sat.is_some(), "{model}: {} photos", p.files);
|
||||
assert_eq!(bundled(model), Some(p));
|
||||
}
|
||||
assert!(bundled("No Such Camera").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_names_are_safe() {
|
||||
assert_eq!(file_name("ILCE-7M4"), "ILCE-7M4.json");
|
||||
assert_eq!(file_name(" DSC-RX100M3 "), "DSC-RX100M3.json");
|
||||
assert_eq!(file_name("../../etc/passwd"), "______etc_passwd.json");
|
||||
assert_eq!(file_name("Ω 1/2"), "__1_2.json");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_skips_files_that_cannot_contribute() {
|
||||
let mut pool = Pool::default();
|
||||
assert!(pool.add(b"not a raw file").is_err());
|
||||
assert!(pool.fit(1).is_empty());
|
||||
assert!(pool.files().is_empty());
|
||||
}
|
||||
}
|
||||
@@ -149,7 +149,8 @@ fn edit_external(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
};
|
||||
let mut written = Vec::new();
|
||||
let mut write = |path: &str, bytes: &[u8]| -> std::result::Result<(), String> {
|
||||
crate::export::write_file(path, bytes)?;
|
||||
// durable: the TIFF becomes a library photo the external editor then changes
|
||||
crate::export::write_file_durable(path, bytes)?;
|
||||
written.push(path.to_string());
|
||||
Ok(())
|
||||
};
|
||||
@@ -182,6 +183,33 @@ pub(crate) fn content_op(id: lightcraft_catalog::PhotoId, ph: &lightcraft_catalo
|
||||
})
|
||||
}
|
||||
|
||||
/// Ops that fill a photo's empty camera fields (camera, lens, exposure, GPS, capture time) from a fresh probe.
|
||||
fn fill_missing_meta(id: lightcraft_catalog::PhotoId, ph: &lightcraft_catalog::Photo, info: &crate::media::ProbeInfo) -> Vec<Op> {
|
||||
let (mut m, src) = (ph.meta.clone(), &info.meta);
|
||||
let before = m.clone();
|
||||
if m.camera.is_empty() {
|
||||
m.camera = src.camera.clone();
|
||||
}
|
||||
if m.lens.is_empty() {
|
||||
m.lens = src.lens.clone();
|
||||
}
|
||||
if m.shutter.is_empty() {
|
||||
m.shutter = src.shutter.clone();
|
||||
}
|
||||
m.focal_mm = m.focal_mm.or(src.focal_mm);
|
||||
m.aperture = m.aperture.or(src.aperture);
|
||||
m.iso = m.iso.or(src.iso);
|
||||
m.gps = m.gps.or(src.gps);
|
||||
let mut ops = Vec::new();
|
||||
if m != before {
|
||||
ops.push(Op::SetMeta { id, meta: Box::new(m) });
|
||||
}
|
||||
if ph.captured.is_none() && info.captured.is_some() {
|
||||
ops.push(Op::SetCaptured { id, captured: info.captured.clone() });
|
||||
}
|
||||
ops
|
||||
}
|
||||
|
||||
/// Re-read photos whose files changed on disk (an external editor saved them): new size,
|
||||
/// dimensions and content hash, cached sources dropped. → {reloaded: [ids]}
|
||||
fn reload(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
@@ -198,8 +226,18 @@ fn reload(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let mut reloaded = Vec::new();
|
||||
for ((id, _), info) in paths.into_iter().zip(probed) {
|
||||
let (Ok(info), Some(ph)) = (info, s.catalog.photo(id)) else { continue };
|
||||
let Some(op) = content_op(id, ph, info.clone()) else { continue };
|
||||
// camera fields the catalog lacks (e.g. a raw imported before its format was read) are filled in;
|
||||
// nothing already set is overwritten
|
||||
let meta_ops = fill_missing_meta(id, ph, &info);
|
||||
let Some(op) = content_op(id, ph, info.clone()) else {
|
||||
if !meta_ops.is_empty() {
|
||||
ops.extend(meta_ops);
|
||||
reloaded.push(id);
|
||||
}
|
||||
continue;
|
||||
};
|
||||
ops.push(op);
|
||||
ops.extend(meta_ops);
|
||||
// virtual copies share the file
|
||||
for c in s.catalog.photos().filter(|c| c.copy_of == Some(id)) {
|
||||
ops.extend(content_op(c.id, c, info.clone()));
|
||||
@@ -276,9 +314,9 @@ pub fn edit_specs() -> Vec<CommandSpec> {
|
||||
cmd!(
|
||||
"photo.reload",
|
||||
"Reload from Disk",
|
||||
[],
|
||||
["Photo"],
|
||||
None,
|
||||
"{ids?} — re-read photos whose files changed on disk (e.g. saved by an external editor), or raws shown from their embedded preview that can be decoded now → {reloaded}",
|
||||
"{ids?} — re-read photos whose files changed on disk (e.g. saved by an external editor), or raws shown from their embedded preview that can be decoded now; camera fields the catalog lacks are filled in → {reloaded}",
|
||||
has_selection,
|
||||
reload
|
||||
),
|
||||
@@ -293,3 +331,32 @@ pub fn edit_specs() -> Vec<CommandSpec> {
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use lightcraft_catalog::{Photo, PhotoId, Source};
|
||||
|
||||
/// Reload fills camera fields a photo lacks (a CR3 imported before CR3 metadata was read) and keeps
|
||||
/// everything already set.
|
||||
#[test]
|
||||
fn reload_fills_only_missing_camera_fields() {
|
||||
let mut ph = Photo::new(PhotoId(1), Source::File { path: "/x.cr3".into() }, "x.cr3", "CR3", 1620, 1080, "2026-10-06T00:00:00");
|
||||
ph.meta.title = "Mine".into();
|
||||
ph.meta.iso = Some(800);
|
||||
let mut info = crate::media::ProbeInfo { captured: Some("2026-10-04T08:16:11".into()), ..Default::default() };
|
||||
info.meta.camera = "Canon EOS R6 Mark III".into();
|
||||
info.meta.lens = "RF24-105mm".into();
|
||||
info.meta.iso = Some(400);
|
||||
info.meta.aperture = Some(6.3);
|
||||
let ops = fill_missing_meta(PhotoId(1), &ph, &info);
|
||||
let Some(Op::SetMeta { meta, .. }) = ops.first() else { panic!("{ops:?}") };
|
||||
assert_eq!((meta.camera.as_str(), meta.lens.as_str(), meta.aperture), ("Canon EOS R6 Mark III", "RF24-105mm", Some(6.3)));
|
||||
assert_eq!((meta.title.as_str(), meta.iso), ("Mine", Some(800)), "set fields are kept");
|
||||
assert!(matches!(ops.get(1), Some(Op::SetCaptured { captured: Some(_), .. })));
|
||||
// nothing missing → nothing to do
|
||||
ph.meta = (**meta).clone();
|
||||
ph.captured = Some("2026-10-04T08:16:11".into());
|
||||
assert!(fill_missing_meta(PhotoId(1), &ph, &info).is_empty());
|
||||
}
|
||||
}
|
||||
@@ -263,7 +263,7 @@ pub fn specs() -> Vec<CommandSpec> {
|
||||
"Filter",
|
||||
[],
|
||||
None,
|
||||
"partial Filter: {text?, rating?, ratingOp?: atLeast|exactly|atMost, flag?: pick|reject|none|null, label?, kind?, merged?: hdr|panorama|hdrPanorama|any, edited?, date?, keyword?, camera?}",
|
||||
"partial Filter: {text?, rating?, ratingOp?: atLeast|exactly|atMost, flag?: pick|reject|none|null, label?, kind?, merged?: hdr|panorama|hdrPanorama|any, edited?, date?, keyword?, person?, camera?}",
|
||||
always,
|
||||
|s, p| {
|
||||
let mut v = serde_json::to_value(&s.filter).unwrap_or_default();
|
||||
@@ -446,6 +446,66 @@ pub fn specs() -> Vec<CommandSpec> {
|
||||
cmd!("photo.rotateRight", "Rotate Right", ["Photo"], Some("Cmd+]"), "{ids?}", has_selection, |s, p| rotate(s, p, true)),
|
||||
cmd!("photo.flipHorizontal", "Flip Horizontal", ["Photo"], None, "{ids?}", has_selection, |s, p| flip(s, p, true)),
|
||||
cmd!("photo.flipVertical", "Flip Vertical", ["Photo"], None, "{ids?}", has_selection, |s, p| flip(s, p, false)),
|
||||
// ---- face / pet regions
|
||||
cmd!(
|
||||
"photo.removeRegion",
|
||||
"Remove Face Box",
|
||||
[],
|
||||
None,
|
||||
"{id?, index} — remove one face / pet region (by its position in the photo's regions) from the photo in the catalog; undoable. The XMP sidecar is never rewritten for this, even with auto-write on, so reading the metadata from the file brings the region back",
|
||||
always,
|
||||
|s, p| {
|
||||
let id =
|
||||
p.get("id").and_then(Value::as_u64).map(PhotoId).or_else(|| s.active()).ok_or_else(|| bad("photo.removeRegion", "no photo"))?;
|
||||
let index = p
|
||||
.get("index")
|
||||
.and_then(Value::as_u64)
|
||||
.and_then(|i| usize::try_from(i).ok())
|
||||
.ok_or_else(|| bad("photo.removeRegion", "missing or invalid `index`"))?;
|
||||
let mut meta = s.catalog.photo(id).ok_or_else(|| bad("photo.removeRegion", "no such photo"))?.meta.clone();
|
||||
if index >= meta.regions.len() {
|
||||
return Err(bad("photo.removeRegion", "no such region"));
|
||||
}
|
||||
let gone = meta.regions.remove(index);
|
||||
s.commit("Remove Face Box", Op::SetMeta { id, meta: Box::new(meta) })?;
|
||||
s.skip_auto_write = true;
|
||||
Ok(json!({"removed": gone.name}))
|
||||
}
|
||||
),
|
||||
cmd!(
|
||||
"photo.setRegion",
|
||||
"Resize Face Box",
|
||||
[],
|
||||
None,
|
||||
"{id?, index, rect: {x0, y0, x1, y1}} — set one face / pet region's box (normalized, in the photo's upright frame; clamped to the photo, at least 0.5 % each way) in the catalog; undoable. Like photo.removeRegion it never rewrites the XMP sidecar",
|
||||
always,
|
||||
|s, p| {
|
||||
const C: &str = "photo.setRegion";
|
||||
// a drag is one undo step however many frames it took: the caller commits once, on release
|
||||
let id = p.get("id").and_then(Value::as_u64).map(PhotoId).or_else(|| s.active()).ok_or_else(|| bad(C, "no photo"))?;
|
||||
let index = p
|
||||
.get("index")
|
||||
.and_then(Value::as_u64)
|
||||
.and_then(|i| usize::try_from(i).ok())
|
||||
.ok_or_else(|| bad(C, "missing or invalid `index`"))?;
|
||||
let r = p.get("rect").ok_or_else(|| bad(C, "missing `rect`"))?;
|
||||
let num = |k: &str| {
|
||||
r.get(k).and_then(Value::as_f64).filter(|v| v.is_finite()).ok_or_else(|| bad(C, format!("`rect.{k}` must be a finite number")))
|
||||
};
|
||||
let (a, b, c, d) = (num("x0")?, num("y0")?, num("x1")?, num("y1")?);
|
||||
let (x0, x1) = (a.min(c).clamp(0.0, 1.0), a.max(c).clamp(0.0, 1.0));
|
||||
let (y0, y1) = (b.min(d).clamp(0.0, 1.0), b.max(d).clamp(0.0, 1.0));
|
||||
if x1 - x0 < 0.005 || y1 - y0 < 0.005 {
|
||||
return Err(bad(C, "the box would be too small"));
|
||||
}
|
||||
let mut meta = s.catalog.photo(id).ok_or_else(|| bad(C, "no such photo"))?.meta.clone();
|
||||
let region = meta.regions.get_mut(index).ok_or_else(|| bad(C, "no such region"))?;
|
||||
region.rect = lightcraft_geom::Rect { x0, y0, x1, y1 };
|
||||
s.commit("Resize Face Box", Op::SetMeta { id, meta: Box::new(meta) })?;
|
||||
s.skip_auto_write = true;
|
||||
Ok(json!({"rect": {"x0": x0, "y0": y0, "x1": x1, "y1": y1}}))
|
||||
}
|
||||
),
|
||||
// ---- delete / restore
|
||||
cmd!("photo.delete", "Delete Photo", ["Photo"], Some("Delete"), "{ids?} — moves to Recently Deleted", has_selection, |s, p| {
|
||||
let v = for_targets(s, p, "Delete", |id| Some(Op::SetDeleted { id, deleted: true }))?;
|
||||
@@ -453,11 +513,11 @@ pub fn specs() -> Vec<CommandSpec> {
|
||||
s.selection = vis.first().map(|f| Selection::single(*f)).unwrap_or_default();
|
||||
Ok(v)
|
||||
}),
|
||||
cmd!("photo.restore", "Restore", [], None, "{ids?}", has_selection, |s, p| for_targets(s, p, "Restore", |id| Some(Op::SetDeleted {
|
||||
cmd!("photo.restore", "Restore", ["Photo"], None, "{ids?}", has_selection, |s, p| for_targets(s, p, "Restore", |id| Some(Op::SetDeleted {
|
||||
id,
|
||||
deleted: false
|
||||
}))),
|
||||
cmd!("photo.deletePermanently", "Delete Permanently", [], None, "{ids?}", has_selection, |s, p| {
|
||||
cmd!("photo.deletePermanently", "Delete Permanently", ["Photo"], None, "{ids?}", has_selection, |s, p| {
|
||||
let t = s.targets(p);
|
||||
let ops = t.iter().map(|id| s.catalog.delete_permanently_ops(*id)).collect::<Vec<_>>();
|
||||
s.commit("Delete Permanently", Op::Batch { ops })?;
|
||||
|
||||
@@ -5,6 +5,7 @@ use lightcraft_geom::Point;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::{CommandSpec, bad, bool_or, cmd, f64_or, has_active, ok, point, str_param};
|
||||
use crate::segment::{Click, MAX_CLICKS};
|
||||
use crate::{Result, Session};
|
||||
|
||||
fn shape_from(kind: &str, p: &Value, c: &str) -> Result<MaskShape> {
|
||||
@@ -38,10 +39,100 @@ fn shape_from(kind: &str, p: &Value, c: &str) -> Result<MaskShape> {
|
||||
.unwrap_or_default();
|
||||
MaskShape::ColorRange { samples, refine: f64_or(p, "refine", 50.0) }
|
||||
}
|
||||
other => return Err(bad(c, format!("unknown mask kind `{other}` (brush|linear|radial|sky|subject|background|luminanceRange|colorRange)"))),
|
||||
"object" => MaskShape::Object { hint: points(p, "points"), exclude: points(p, "exclude"), seg: seg_param(p), detail: vec![], edge: 0.0 },
|
||||
"prompt" => {
|
||||
MaskShape::Prompt { text: str_param(p, "text").unwrap_or_default().trim().to_string(), seg: seg_param(p), detail: vec![], edge: 0.0 }
|
||||
}
|
||||
other => {
|
||||
return Err(bad(
|
||||
c,
|
||||
format!("unknown mask kind `{other}` (brush|linear|radial|sky|subject|background|luminanceRange|colorRange|object|prompt)"),
|
||||
));
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// `[[x, y], …]` normalized points under `key`.
|
||||
fn points(p: &Value, key: &str) -> Vec<Point> {
|
||||
p.get(key)
|
||||
.and_then(Value::as_array)
|
||||
.map(|a| a.iter().filter_map(|q| Some(Point::new(q.get(0)?.as_f64()?, q.get(1)?.as_f64()?))).collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// A segmentation passed in (`seg`: a stored or replayed AI mask; the model isn't needed).
|
||||
fn seg_param(p: &Value) -> Option<lightcraft_develop::SegMask> {
|
||||
p.get("seg").and_then(|v| serde_json::from_value(v.clone()).ok())
|
||||
}
|
||||
|
||||
/// What a new AI shape still needs once it is added.
|
||||
enum Later {
|
||||
Nothing,
|
||||
/// Background mode: these Object clicks are segmented on the worker.
|
||||
Clicks(Vec<Point>, Vec<Point>),
|
||||
/// Background mode: the Describe selection is computed first; the mask is added when found.
|
||||
Text(String),
|
||||
}
|
||||
|
||||
/// Compute the segmentation of a new AI shape (Object with clicks, Prompt) — or, in background
|
||||
/// mode, say what to queue. An Object without clicks stays empty until the photo is clicked
|
||||
/// (the photo is analyzed meanwhile); a Prompt that matches nothing is an error. Shapes given
|
||||
/// with their `seg` need no model.
|
||||
fn resolve_ai(s: &mut Session, shape: &mut MaskShape, c: &str) -> Result<Later> {
|
||||
let id = s.active().ok_or_else(|| bad(c, "no active photo"))?;
|
||||
let background = s.segmenter.background;
|
||||
match shape {
|
||||
MaskShape::Object { seg: Some(_), .. } | MaskShape::Prompt { seg: Some(_), .. } => {}
|
||||
MaskShape::Object { hint, exclude, .. } if !hint.is_empty() && background => {
|
||||
s.segmenter.model_dir().map_err(ai_err)?;
|
||||
return Ok(Later::Clicks(std::mem::take(hint), std::mem::take(exclude)));
|
||||
}
|
||||
MaskShape::Object { hint, exclude, seg, .. } if !hint.is_empty() => {
|
||||
*seg = Some(s.segment_clicks(id, &clicks_of(hint, exclude)).map_err(ai_err)?);
|
||||
}
|
||||
MaskShape::Object { .. } => {
|
||||
// the clicks come later: refuse an Object mask that could never be computed (no
|
||||
// model) rather than leave an empty one; in the app, analyze the photo meanwhile
|
||||
s.segmenter.model_dir().map_err(ai_err)?;
|
||||
if background {
|
||||
s.segment_prepare(id).map_err(ai_err)?;
|
||||
}
|
||||
}
|
||||
MaskShape::Prompt { text, .. } if background => {
|
||||
s.segmenter.model_dir().map_err(ai_err)?;
|
||||
if text.trim().is_empty() {
|
||||
return Err(ai_err("describe what to select"));
|
||||
}
|
||||
return Ok(Later::Text(text.clone()));
|
||||
}
|
||||
MaskShape::Prompt { text, seg, .. } => {
|
||||
let found = s.segment_text(id, text).map_err(ai_err)?;
|
||||
*seg = Some(found.ok_or_else(|| ai_err(format!("nothing matching “{text}” was found in this photo")))?);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok(Later::Nothing)
|
||||
}
|
||||
|
||||
/// Queue what a new AI component (component `comp` of mask `mask`) still needs.
|
||||
fn queue_later(s: &mut Session, later: Later, mask: u32, comp: usize, mut out: Value) -> Result<Value> {
|
||||
if let Later::Clicks(hint, exclude) = later {
|
||||
let id = s.active().ok_or_else(|| ai_err("no active photo"))?;
|
||||
s.segment_clicks_later(id, mask, comp, hint, exclude).map_err(ai_err)?;
|
||||
out["pending"] = json!(true);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// AI mask failures are shown as they are (not as "invalid parameters").
|
||||
fn ai_err(msg: impl Into<String>) -> crate::EngineError {
|
||||
crate::EngineError::Other(msg.into())
|
||||
}
|
||||
|
||||
pub(crate) fn clicks_of(include: &[Point], exclude: &[Point]) -> Vec<Click> {
|
||||
include.iter().map(|p| Click { at: *p, include: true }).chain(exclude.iter().map(|p| Click { at: *p, include: false })).collect()
|
||||
}
|
||||
|
||||
fn masks_edit(s: &mut Session, c: &str, label: &str, f: impl FnOnce(&mut Vec<Mask>, &mut Option<u32>) -> Result<()>) -> Result<Value> {
|
||||
let id = s.active().ok_or_else(|| bad(c, "no active photo"))?;
|
||||
let mut d = (*s.develop_of(id).unwrap_or_default()).clone();
|
||||
@@ -67,14 +158,20 @@ pub fn specs() -> Vec<CommandSpec> {
|
||||
"Create New Mask",
|
||||
[],
|
||||
None,
|
||||
"{kind: brush|linear|radial|sky|subject|background|luminanceRange|colorRange, ...shape params (start/end, center/rx/ry/angle/feather, lo/hi…), name?}",
|
||||
"{kind: brush|linear|radial|sky|subject|background|luminanceRange|colorRange|object|prompt, ...shape params (start/end, center/rx/ry/angle/feather, lo/hi…; object: points/exclude [[x,y],…]; prompt: text; object/prompt: seg? a stored segmentation, no model needed), name?} — AI kinds need the SAM 3 model; in the app a prompt returns {pending} and the mask appears when found",
|
||||
has_active,
|
||||
|s, p| {
|
||||
let kind = str_param(p, "kind").unwrap_or("radial").to_string();
|
||||
let shape = shape_from(&kind, p, "mask.add")?;
|
||||
let mut shape = shape_from(&kind, p, "mask.add")?;
|
||||
let later = resolve_ai(s, &mut shape, "mask.add")?;
|
||||
let name = str_param(p, "name").map(str::to_string);
|
||||
if let Later::Text(text) = &later {
|
||||
let id = s.active().ok_or_else(|| bad("mask.add", "no active photo"))?;
|
||||
s.segment_text_later(id, None, "add", name, text).map_err(ai_err)?;
|
||||
return Ok(json!({"pending": true}));
|
||||
}
|
||||
let next = s.active().and_then(|id| s.develop_of(id)).map(|d| d.next_mask_id()).unwrap_or(1);
|
||||
masks_edit(s, "mask.add", "Add Mask", |masks, active| {
|
||||
let out = masks_edit(s, "mask.add", "Add Mask", |masks, active| {
|
||||
masks.push(Mask {
|
||||
id: next,
|
||||
name: name.unwrap_or_else(|| format!("Mask {next}")),
|
||||
@@ -83,7 +180,8 @@ pub fn specs() -> Vec<CommandSpec> {
|
||||
});
|
||||
*active = Some(next);
|
||||
Ok(())
|
||||
})
|
||||
})?;
|
||||
queue_later(s, later, next, 0, out)
|
||||
}
|
||||
),
|
||||
cmd!(
|
||||
@@ -95,16 +193,29 @@ pub fn specs() -> Vec<CommandSpec> {
|
||||
has_active,
|
||||
|s, p| {
|
||||
let kind = str_param(p, "kind").unwrap_or("brush").to_string();
|
||||
let shape = shape_from(&kind, p, "mask.addComponent")?;
|
||||
let op: MaskOp =
|
||||
serde_json::from_value(p.get("op").cloned().unwrap_or(json!("add"))).map_err(|e| bad("mask.addComponent", e.to_string()))?;
|
||||
let active = s.active_mask;
|
||||
let mid = mask_id(p, active, "mask.addComponent")?;
|
||||
masks_edit(s, "mask.addComponent", "Edit Mask", |masks, _| {
|
||||
let id = s.active().ok_or_else(|| bad("mask.addComponent", "no active photo"))?;
|
||||
if !s.develop_of(id).is_some_and(|d| d.masks.iter().any(|m| m.id == mid)) {
|
||||
return Err(bad("mask.addComponent", format!("no mask {mid}")));
|
||||
}
|
||||
let mut shape = shape_from(&kind, p, "mask.addComponent")?;
|
||||
let later = resolve_ai(s, &mut shape, "mask.addComponent")?;
|
||||
if let Later::Text(text) = &later {
|
||||
let op = p.get("op").and_then(Value::as_str).unwrap_or("add");
|
||||
s.segment_text_later(id, Some(mid), op, None, text).map_err(ai_err)?;
|
||||
return Ok(json!({"pending": true}));
|
||||
}
|
||||
let mut comp = 0;
|
||||
let out = masks_edit(s, "mask.addComponent", "Edit Mask", |masks, _| {
|
||||
let i = find(masks, mid, "mask.addComponent")?;
|
||||
masks[i].components.push(MaskComponent { name: None, op, invert: bool_or(p, "invert", false), shape });
|
||||
comp = masks[i].components.len() - 1;
|
||||
Ok(())
|
||||
})
|
||||
})?;
|
||||
queue_later(s, later, mid, comp, out)
|
||||
}
|
||||
),
|
||||
cmd!(
|
||||
@@ -213,6 +324,158 @@ pub fn specs() -> Vec<CommandSpec> {
|
||||
Ok(json!({"samples": out}))
|
||||
}
|
||||
),
|
||||
cmd!(
|
||||
"mask.objectPoint",
|
||||
"Add Object Click",
|
||||
[],
|
||||
None,
|
||||
"{x, y: normalized image point, exclude?: bool (⌥-click: leave this part out), id?: maskId} — a click on the selected mask's Object selection (its last Object component); SAM 3 re-segments the object → {include, exclude}",
|
||||
has_active,
|
||||
|s, p| {
|
||||
let c = "mask.objectPoint";
|
||||
let at = Point::new(super::f64_req(p, "x", c)?, super::f64_req(p, "y", c)?);
|
||||
if !(0.0..=1.0).contains(&at.x) || !(0.0..=1.0).contains(&at.y) {
|
||||
return Err(bad(c, "the point is outside the photo"));
|
||||
}
|
||||
let exclude = bool_or(p, "exclude", false);
|
||||
let id = s.active().ok_or_else(|| bad(c, "no active photo"))?;
|
||||
let mid = mask_id(p, s.active_mask, c)?;
|
||||
let d = s.develop_of(id).unwrap_or_default();
|
||||
let m = d.masks.iter().find(|m| m.id == mid).ok_or_else(|| bad(c, format!("no mask {mid}")))?;
|
||||
let k = m
|
||||
.components
|
||||
.iter()
|
||||
.rposition(|x| matches!(x.shape, MaskShape::Object { .. }))
|
||||
.ok_or_else(|| bad(c, "the mask has no Object selection (add one first)"))?;
|
||||
let Some(MaskShape::Object { hint, exclude: ex, edge, .. }) = m.components.get(k).map(|x| &x.shape) else {
|
||||
return Err(bad(c, "not an Object component"));
|
||||
};
|
||||
let (mut hint, mut ex, edge) = (hint.clone(), ex.clone(), *edge);
|
||||
// clicks still on their way to the model count too
|
||||
if let Some(pending) = s.segmenter.pending_clicks().filter(|q| (q.photo, q.mask, q.comp) == (id, mid, k)) {
|
||||
(hint, ex) = (pending.hint.clone(), pending.exclude.clone());
|
||||
}
|
||||
if hint.len() + ex.len() >= MAX_CLICKS {
|
||||
return Err(ai_err(format!("an Object selection takes at most {MAX_CLICKS} clicks; start a new one")));
|
||||
}
|
||||
if exclude {
|
||||
ex.push(at)
|
||||
} else {
|
||||
hint.push(at)
|
||||
}
|
||||
if s.segmenter.background && !hint.is_empty() {
|
||||
let out = json!({"include": hint.len(), "exclude": ex.len(), "pending": true});
|
||||
s.segment_clicks_later(id, mid, k, hint, ex).map_err(ai_err)?;
|
||||
return Ok(out);
|
||||
}
|
||||
let seg = if hint.is_empty() { None } else { Some(s.segment_clicks(id, &clicks_of(&hint, &ex)).map_err(ai_err)?) };
|
||||
let out = json!({"include": hint.len(), "exclude": ex.len()});
|
||||
masks_edit(s, c, "Object Mask", |masks, _| {
|
||||
let i = find(masks, mid, c)?;
|
||||
if let Some(comp) = masks[i].components.get_mut(k) {
|
||||
comp.shape = MaskShape::Object { hint, exclude: ex, seg, detail: vec![], edge };
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(out)
|
||||
}
|
||||
),
|
||||
cmd!(
|
||||
"mask.refineDetail",
|
||||
"Refine AI Mask Detail",
|
||||
[],
|
||||
None,
|
||||
"{id?: maskId, component?} — a zoomed-in SAM 3 pass over an Object/Describe selection (default: the mask's last one): the photo around it is analyzed again at a higher resolution, in the background; the mask updates when it's done → {started}",
|
||||
has_active,
|
||||
|s, p| {
|
||||
let c = "mask.refineDetail";
|
||||
let id = s.active().ok_or_else(|| bad(c, "no active photo"))?;
|
||||
let mid = mask_id(p, s.active_mask, c)?;
|
||||
let d = s.develop_of(id).unwrap_or_default();
|
||||
let m = d.masks.iter().find(|m| m.id == mid).ok_or_else(|| bad(c, format!("no mask {mid}")))?;
|
||||
let k = match p.get("component").and_then(Value::as_u64) {
|
||||
Some(k) => k as usize,
|
||||
None => m
|
||||
.components
|
||||
.iter()
|
||||
.rposition(|x| matches!(x.shape, MaskShape::Object { .. } | MaskShape::Prompt { .. }))
|
||||
.ok_or_else(|| bad(c, "the mask has no Object or Describe selection"))?,
|
||||
};
|
||||
let started = s.segment_detail(id, mid, k).map_err(ai_err)?;
|
||||
Ok(json!({"started": started}))
|
||||
}
|
||||
),
|
||||
cmd!(
|
||||
query "segment.prepare",
|
||||
"Prepare AI Masks",
|
||||
[],
|
||||
None,
|
||||
"{} — load SAM 3 and analyze the active photo in the background, so Object clicks are instant → {busy}",
|
||||
has_active,
|
||||
|s, _| {
|
||||
let c = "segment.prepare";
|
||||
let id = s.active().ok_or_else(|| bad(c, "no active photo"))?;
|
||||
s.segment_prepare(id).map_err(ai_err)?;
|
||||
Ok(json!({"busy": s.segmenter.busy()}))
|
||||
}
|
||||
),
|
||||
cmd!(
|
||||
query "segment.model.status",
|
||||
"AI Mask Model Status",
|
||||
[],
|
||||
None,
|
||||
"{} — whether this build has AI masks, whether the SAM 3 model is installed (and where), loaded, busy, and the download's progress → {available, installed, dir, loaded, busy, analyzing, sizeBytes, license, licenseUrl, mirrors, download: {running, done, total, file, error, finished}}",
|
||||
super::always,
|
||||
|s, _| {
|
||||
let g = &s.segmenter;
|
||||
Ok(json!({
|
||||
"available": crate::segment::Segmenter::AVAILABLE,
|
||||
"installed": g.installed(),
|
||||
"dir": g.dir.as_ref().map(|d| d.display().to_string()),
|
||||
"loaded": g.loaded(),
|
||||
"busy": g.busy(),
|
||||
"analyzing": g.analyzing(),
|
||||
"sizeBytes": crate::segment::MODEL_BYTES,
|
||||
"license": crate::segment::LICENSE_NAME,
|
||||
"licenseUrl": crate::segment::LICENSE_URL,
|
||||
"mirrors": g.mirrors().len(),
|
||||
"download": g.download_status(),
|
||||
}))
|
||||
}
|
||||
),
|
||||
cmd!(
|
||||
query "segment.model.download",
|
||||
"Download AI Mask Model",
|
||||
[],
|
||||
None,
|
||||
"{acknowledged: true} — download the SAM 3 model (about 3.4 GB, Meta's SAM License, not LightCraft's) in the background, from the configured mirrors; only after the user agreed to it. Watch segment.model.status; segment.model.cancel stops it (it resumes later) → {started, installed, downloading}",
|
||||
super::always,
|
||||
|s, p| {
|
||||
let c = "segment.model.download";
|
||||
if p.get("acknowledged").and_then(Value::as_bool) != Some(true) {
|
||||
return Err(bad(
|
||||
c,
|
||||
format!(
|
||||
"the SAM 3 model is a {:.1} GB download under Meta's {} ({}): pass `acknowledged: true` once the user has agreed to download it",
|
||||
crate::segment::MODEL_BYTES as f64 / 1e9,
|
||||
crate::segment::LICENSE_NAME,
|
||||
crate::segment::LICENSE_URL
|
||||
),
|
||||
));
|
||||
}
|
||||
let started = s.segmenter.start_download().map_err(ai_err)?;
|
||||
Ok(json!({"started": started, "installed": s.segmenter.installed(), "downloading": s.segmenter.download_status().running}))
|
||||
}
|
||||
),
|
||||
cmd!(
|
||||
query "segment.model.cancel",
|
||||
"Cancel AI Mask Model Download",
|
||||
[],
|
||||
None,
|
||||
"{} — stop the SAM 3 download (what has arrived is kept, and a new download resumes from it) → {cancelled}",
|
||||
super::always,
|
||||
|s, _| Ok(json!({"cancelled": s.segmenter.cancel_download()}))
|
||||
),
|
||||
cmd!(
|
||||
"mask.refine",
|
||||
"Refine Mask Edges",
|
||||
|
||||
@@ -160,7 +160,9 @@ fn smart_run(
|
||||
if damaged && crate::smart::is_valid(&path) {
|
||||
n.built += 1;
|
||||
} else {
|
||||
// atomic: a failed write leaves no partial proxy that would pass for a built one
|
||||
// atomic: a failed write leaves no partial proxy that would pass for a built one;
|
||||
// and synced (issue #134): built so the photo can be edited while its original
|
||||
// is offline, when it is the only copy — the sync is small next to the decode
|
||||
let r = source
|
||||
.ok_or_else(|| "nothing to build from".to_string())
|
||||
.and_then(|s| s.load_source())
|
||||
|
||||
+84
-29
@@ -244,14 +244,24 @@ impl Default for Watermark {
|
||||
|
||||
/// Inter SemiBold (OFL, see assets/ATTRIBUTION.md).
|
||||
static WATERMARK_FONT: &[u8] = include_bytes!("../../../assets/fonts/Inter-SemiBold.ttf");
|
||||
/// BIZ UDMincho (OFL), Japanese fallback on both native and web.
|
||||
/// Bundled OFL Japanese font bytes, shared by watermarks and UI glyph fallback.
|
||||
pub static WATERMARK_JAPANESE_FONT: &[u8] = include_bytes!("../../../assets/fonts/BIZUDMincho-Regular.ttf");
|
||||
|
||||
/// The watermark faces: Inter first, then the craft-fonts CJK faces (Mincho first, the watermark's
|
||||
/// serif look; then any other CJK face — a watermark has no UI language, so every CJK script is
|
||||
/// offered). Without craft-fonts that is Inter alone, and CJK characters draw as Inter's
|
||||
/// missing-glyph box.
|
||||
fn watermark_fonts(craft: &'static [crate::fonts::CraftFont]) -> Vec<ab_glyph::FontRef<'static>> {
|
||||
let mut cjk: Vec<_> = crate::fonts::cjk(craft).collect();
|
||||
cjk.sort_by_key(|f| !f.is_mincho());
|
||||
std::iter::once(WATERMARK_FONT)
|
||||
.chain(cjk.into_iter().map(|f| f.bytes))
|
||||
.filter_map(|bytes| ab_glyph::FontRef::try_from_slice(bytes).ok())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Draw `wm` onto `img` (straight alpha blending of the encoded values).
|
||||
pub fn draw_watermark(img: &mut Rgba8, wm: &Watermark) {
|
||||
let width = img.width;
|
||||
watermark_coverage(img.width, img.height, wm, |x, y, k, col| {
|
||||
watermark_coverage(img.width, img.height, wm, crate::fonts::CRAFT_FONTS, |x, y, k, col| {
|
||||
let p = &mut img.data[y * width + x];
|
||||
for c in 0..3 {
|
||||
p[c] = (p[c] as f32 + (col[c] as f32 - p[c] as f32) * k).round() as u8;
|
||||
@@ -264,14 +274,14 @@ pub fn draw_watermark(img: &mut Rgba8, wm: &Watermark) {
|
||||
pub fn draw_watermark_deep(img: &mut DeepImage, wm: &Watermark) {
|
||||
let (width, trc) = (img.width, img.space.trc());
|
||||
match &mut img.samples {
|
||||
DeepSamples::U16(v) => watermark_coverage(img.width, img.height, wm, |x, y, k, col| {
|
||||
DeepSamples::U16(v) => watermark_coverage(img.width, img.height, wm, crate::fonts::CRAFT_FONTS, |x, y, k, col| {
|
||||
let i = (y * width + x) * 3;
|
||||
for c in 0..3 {
|
||||
let p = v[i + c] as f32;
|
||||
v[i + c] = (p + (col[c] as f32 * 257.0 - p) * k).round().clamp(0.0, 65535.0) as u16;
|
||||
}
|
||||
}),
|
||||
DeepSamples::F32(v) => watermark_coverage(img.width, img.height, wm, |x, y, k, col| {
|
||||
DeepSamples::F32(v) => watermark_coverage(img.width, img.height, wm, crate::fonts::CRAFT_FONTS, |x, y, k, col| {
|
||||
let i = (y * width + x) * 3;
|
||||
for c in 0..3 {
|
||||
let target = trc.decode(col[c] as f32 / 255.0);
|
||||
@@ -282,9 +292,16 @@ pub fn draw_watermark_deep(img: &mut DeepImage, wm: &Watermark) {
|
||||
}
|
||||
|
||||
/// Lay out `wm` on a `width × height` image and call `blend(x, y, coverage × opacity, colour)` for
|
||||
/// every covered pixel (shadow pass first).
|
||||
fn watermark_coverage(width: usize, height: usize, wm: &Watermark, mut blend_px: impl FnMut(usize, usize, f32, [u8; 3])) {
|
||||
use ab_glyph::{Font, FontRef, PxScale, ScaleFont, point};
|
||||
/// every covered pixel (shadow pass first). `craft` is [`crate::fonts::CRAFT_FONTS`] (a parameter
|
||||
/// so tests can render without it).
|
||||
fn watermark_coverage(
|
||||
width: usize,
|
||||
height: usize,
|
||||
wm: &Watermark,
|
||||
craft: &'static [crate::fonts::CraftFont],
|
||||
mut blend_px: impl FnMut(usize, usize, f32, [u8; 3]),
|
||||
) {
|
||||
use ab_glyph::{Font, PxScale, ScaleFont, point};
|
||||
if !wm.image.trim().is_empty() {
|
||||
logo_coverage(width, height, wm, blend_px);
|
||||
return;
|
||||
@@ -293,9 +310,8 @@ fn watermark_coverage(width: usize, height: usize, wm: &Watermark, mut blend_px:
|
||||
if text.is_empty() || width == 0 || height == 0 {
|
||||
return;
|
||||
}
|
||||
let Ok(font) = FontRef::try_from_slice(WATERMARK_FONT) else { return };
|
||||
let Ok(japanese) = FontRef::try_from_slice(WATERMARK_JAPANESE_FONT) else { return };
|
||||
let fonts = [font, japanese];
|
||||
let fonts = watermark_fonts(craft);
|
||||
let Some(latin) = fonts.first() else { return };
|
||||
let short = width.min(height) as f32;
|
||||
let px = (wm.size.clamp(0.005, 0.5) * short).max(6.0);
|
||||
let mut glyphs = Vec::new();
|
||||
@@ -333,9 +349,11 @@ fn watermark_coverage(width: usize, height: usize, wm: &Watermark, mut blend_px:
|
||||
'…' => '︙',
|
||||
other => other,
|
||||
};
|
||||
let ch = if wm.vertical && fonts[1].glyph_id(vertical_form).0 != 0 { vertical_form } else { original };
|
||||
let face = usize::from(fonts[0].glyph_id(ch).0 == 0);
|
||||
let sf = fonts[face].as_scaled(PxScale::from(px));
|
||||
let has = |font: &ab_glyph::FontRef<'_>, c: char| font.glyph_id(c).0 != 0;
|
||||
let ch = if wm.vertical && fonts.iter().skip(1).any(|f| has(f, vertical_form)) { vertical_form } else { original };
|
||||
// The first face with the glyph; Inter (its missing-glyph box) when none has it.
|
||||
let face = fonts.iter().position(|f| has(f, ch)).unwrap_or(0);
|
||||
let sf = fonts.get(face).unwrap_or(latin).as_scaled(PxScale::from(px));
|
||||
let id = sf.glyph_id(ch);
|
||||
if wm.vertical {
|
||||
let left = columns.saturating_sub(column.saturating_add(1)) as f32 * px;
|
||||
@@ -1131,15 +1149,27 @@ pub fn export_batch(
|
||||
}
|
||||
|
||||
/// Write an exported or rendered file on disk: its folder is created if needed, and the file is
|
||||
/// replaced atomically ([`lightcraft_catalog::safe_file::write_atomic`]: a temp file, synced, then
|
||||
/// renamed), so a failure part-way leaves any previous file intact and no truncated one. The
|
||||
/// native writer behind exports, renders and screenshots (app, CLI, MCP).
|
||||
/// replaced atomically ([`lightcraft_catalog::safe_file::write_atomic_nosync`]: a temp file
|
||||
/// renamed into place), so a failure part-way leaves any previous file intact and no truncated
|
||||
/// one. Not synced to disk (issue #134): an export can always be made again from the original.
|
||||
/// The native writer behind exports, renders and screenshots (app, CLI, MCP).
|
||||
pub fn write_file(path: &str, bytes: &[u8]) -> Result<(), String> {
|
||||
write_with(path, bytes, false)
|
||||
}
|
||||
|
||||
/// [`write_file`], synced to disk: for a written file that becomes a library photo (Edit a Copy)
|
||||
/// and so can't simply be exported again once the catalog points at it.
|
||||
pub fn write_file_durable(path: &str, bytes: &[u8]) -> Result<(), String> {
|
||||
write_with(path, bytes, true)
|
||||
}
|
||||
|
||||
fn write_with(path: &str, bytes: &[u8], durable: bool) -> Result<(), String> {
|
||||
use lightcraft_catalog::safe_file::{write_atomic, write_atomic_nosync};
|
||||
let p = std::path::Path::new(path);
|
||||
if let Some(dir) = p.parent().filter(|d| !d.as_os_str().is_empty()) {
|
||||
std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
|
||||
}
|
||||
lightcraft_catalog::safe_file::write_atomic(p, bytes).map_err(|e| format!("{path}: {e}"))
|
||||
if durable { write_atomic(p, bytes) } else { write_atomic_nosync(p, bytes) }.map_err(|e| format!("{path}: {e}"))
|
||||
}
|
||||
|
||||
/// The path of the sidecar with extension `ext` of the exported file `main`.
|
||||
@@ -1334,21 +1364,46 @@ mod tests {
|
||||
assert!(ExportOptions::from_json(&serde_json::json!({"watermark": ""})).watermark.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn japanese_watermarks_support_vertical_columns_and_legacy_defaults() {
|
||||
use ab_glyph::Font;
|
||||
let font = ab_glyph::FontRef::try_from_slice(WATERMARK_JAPANESE_FONT).unwrap();
|
||||
for c in "日本語".chars() {
|
||||
assert_ne!(font.glyph_id(c).0, 0);
|
||||
}
|
||||
let old = ExportOptions::from_json(&json!({"watermark": {"text": "日本語"}}));
|
||||
assert!(!old.watermark.unwrap().vertical);
|
||||
fn vertical_japanese() -> Watermark {
|
||||
let options =
|
||||
ExportOptions::from_json(&json!({"watermark": {"text": "日本語", "vertical": true, "size": 0.1, "shadow": false, "opacity": 1.0}}));
|
||||
let wm = options.watermark.unwrap();
|
||||
options.watermark.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn japanese_watermark_options_and_legacy_defaults() {
|
||||
let old = ExportOptions::from_json(&json!({"watermark": {"text": "日本語"}}));
|
||||
assert!(!old.watermark.unwrap().vertical);
|
||||
let wm = vertical_japanese();
|
||||
assert!(wm.vertical);
|
||||
let round: Watermark = serde_json::from_value(serde_json::to_value(&wm).unwrap()).unwrap();
|
||||
assert!(round.vertical);
|
||||
}
|
||||
|
||||
/// Built without craft-fonts, a Japanese watermark still lays out and draws (Inter's
|
||||
/// missing-glyph boxes), and Latin text is unaffected.
|
||||
#[test]
|
||||
fn watermarks_work_without_craft_fonts() {
|
||||
assert_eq!(watermark_fonts(&[]).len(), 1, "Inter only");
|
||||
for wm in [vertical_japanese(), Watermark { text: "LightCraft 日本語".into(), ..Watermark::default() }] {
|
||||
let mut covered = 0usize;
|
||||
watermark_coverage(400, 300, &wm, &[], |_, _, k, _| covered += usize::from(k > 0.0));
|
||||
assert!(covered > 0, "{:?} draws something", wm.text);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn japanese_watermarks_support_vertical_columns() {
|
||||
use ab_glyph::Font;
|
||||
let fonts = watermark_fonts(crate::fonts::CRAFT_FONTS);
|
||||
if fonts.len() < 2 {
|
||||
eprintln!("skipped: built without CRAFT_FONTS_DIR, so there is no Japanese watermark face");
|
||||
return;
|
||||
}
|
||||
for c in "日本語の文字".chars() {
|
||||
assert!(fonts[1..].iter().any(|f| f.glyph_id(c).0 != 0), "a craft-fonts face has {c}");
|
||||
}
|
||||
let wm = vertical_japanese();
|
||||
let mut img = Rgba8::new(400, 300);
|
||||
img.data.fill([0, 0, 0, 255]);
|
||||
draw_watermark(&mut img, &wm);
|
||||
|
||||
+66
-12
@@ -38,6 +38,7 @@ fn meta_of(m: &lightcraft_meta::Metadata) -> (Meta, Option<String>) {
|
||||
copyright_url: m.copyright_url.clone().unwrap_or_default(),
|
||||
creator: m.artist.clone().unwrap_or_default(),
|
||||
keywords: m.keywords.clone(),
|
||||
regions: m.regions.clone(),
|
||||
};
|
||||
(meta, m.capture_time.as_ref().map(|d| d.to_iso()))
|
||||
}
|
||||
@@ -132,7 +133,7 @@ pub fn probe_bytes(name: &str, bytes: &[u8]) -> Result<ProbeInfo, String> {
|
||||
}
|
||||
let (t, tint) = xy_to_temp_tint(lightcraft_raw::color::as_shot_white_xy_of(&raw));
|
||||
// Vendor RGB multipliers do not identify an absolute illuminant without camera calibration.
|
||||
let relative = raw.format == lightcraft_raw::RawFormat::Arw && !lightcraft_raw::color::has_matrix(&raw.color);
|
||||
let relative = crate::camera_preview::file_local_look(raw.format) && !lightcraft_raw::color::has_matrix(&raw.color);
|
||||
let as_shot_wb = Some(if relative { (6500.0, 0.0) } else { (t.round(), tint.round()) });
|
||||
let embedded_lens = embedded_lens(&raw);
|
||||
return Ok(ProbeInfo {
|
||||
@@ -250,16 +251,25 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<
|
||||
stages.push(("transform", t0.elapsed()));
|
||||
lightcraft_raw::highlight::reconstruct(&mut img, t.wb, HIGHLIGHT_CLIP);
|
||||
stages.push(("highlights", t0.elapsed()));
|
||||
let m = camera_look.map(|p| p.matrix.mul(&t.matrix)).unwrap_or(t.matrix).to_f32();
|
||||
let m = camera_look.as_ref().map(|p| p.matrix.mul(&t.matrix)).unwrap_or(t.matrix).to_f32();
|
||||
let hue_sat = camera_look.as_ref().and_then(|p| p.hue_sat.as_ref()).and_then(crate::camera_preview::HueSat::new);
|
||||
let gain = 2f32.powf(t.baseline_exposure as f32);
|
||||
let wb = t.wb;
|
||||
// A DNG's own profile look (hue/saturation map, look table), DNG spec chapter 6.
|
||||
let tables = lightcraft_raw::profile::ProfileTables::new(&raw.color.profile, lightcraft_raw::color::illuminant_weight(&raw.color, xy));
|
||||
img.map_in_place(|p| {
|
||||
let c = [p[0] * wb[0], p[1] * wb[1], p[2] * wb[2]];
|
||||
[
|
||||
((m[0][0] * c[0] + m[0][1] * c[1] + m[0][2] * c[2]) * gain).max(0.0),
|
||||
((m[1][0] * c[0] + m[1][1] * c[1] + m[1][2] * c[2]) * gain).max(0.0),
|
||||
((m[2][0] * c[0] + m[2][1] * c[1] + m[2][2] * c[2]) * gain).max(0.0),
|
||||
]
|
||||
let rgb = [
|
||||
m[0][0] * c[0] + m[0][1] * c[1] + m[0][2] * c[2],
|
||||
m[1][0] * c[0] + m[1][1] * c[1] + m[1][2] * c[2],
|
||||
m[2][0] * c[0] + m[2][1] * c[1] + m[2][2] * c[2],
|
||||
];
|
||||
let rgb = match &tables {
|
||||
Some(tables) => tables.apply(rgb, gain),
|
||||
None => rgb.map(|v| v * gain),
|
||||
};
|
||||
// after the baseline exposure, as when it was fitted
|
||||
hue_sat.as_ref().map_or(rgb, |h| h.apply(rgb)).map(|v| v.max(0.0))
|
||||
});
|
||||
stages.push(("colour", t0.elapsed()));
|
||||
let img = fit(&img, max_edge, max_edge, Filter::Box);
|
||||
@@ -279,12 +289,10 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<
|
||||
eprintln!("[profile] raw source {}×{} (max {max_edge}, ms after decode): {}", img.width, img.height, parts.join(", "));
|
||||
}
|
||||
let (temp, tint) = xy_to_temp_tint(xy);
|
||||
let relative = raw.format == lightcraft_raw::RawFormat::Arw && t.matrix_is_fallback;
|
||||
let relative = crate::camera_preview::file_local_look(raw.format) && t.matrix_is_fallback;
|
||||
let camera_tone = camera_look.as_ref().map(|p| p.tone).or_else(|| raw.color.profile.tone_curve.as_ref().and_then(dng_tone_curve));
|
||||
let (temp, tint) = if relative { (6500.0, 0.0) } else { (temp.round(), tint.round()) };
|
||||
return Ok((
|
||||
img,
|
||||
SourceInfo { raw: true, as_shot_temp: temp, as_shot_tint: tint, lens, relative_wb: relative, camera_tone: camera_look.map(|p| p.tone) },
|
||||
));
|
||||
return Ok((img, SourceInfo { raw: true, as_shot_temp: temp, as_shot_tint: tint, lens, relative_wb: relative, camera_tone }));
|
||||
}
|
||||
let d = lightcraft_codecs::decode(&bytes, lightcraft_codecs::DecodeOptions::fit(max_edge as u32, max_edge as u32)).map_err(|e| e.to_string())?;
|
||||
drop(bytes);
|
||||
@@ -293,6 +301,17 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<
|
||||
Ok((img.into_oriented(Orientation::from_exif(d.orientation)), SourceInfo::default()))
|
||||
}
|
||||
|
||||
/// A DNG `ProfileToneCurve` (linear in, linear out, 1.0 = white after exposure compensation) as the
|
||||
/// finish stage's camera tone curve: 32 knots, log-spaced over 12 stops below white; above white
|
||||
/// the camera tone's shoulder continues it.
|
||||
pub(crate) fn dng_tone_curve(curve: &lightcraft_raw::profile::ToneCurve) -> Option<lightcraft_pipeline::tone::CameraTone> {
|
||||
let knots: [[f32; 2]; 32] = std::array::from_fn(|i| {
|
||||
let x = 2f32.powf(-12.0 + 12.0 * i as f32 / 31.0);
|
||||
[x, curve.eval(x).min(0.9995)]
|
||||
});
|
||||
lightcraft_pipeline::tone::CameraTone::new(knots)
|
||||
}
|
||||
|
||||
/// Orientation for an embedded preview: its own EXIF orientation when it has one, else the raw file's.
|
||||
fn preview_orientation(raw_bytes: &[u8], jpeg_orientation: u16) -> Orientation {
|
||||
if jpeg_orientation > 1 {
|
||||
@@ -398,6 +417,41 @@ mod tests {
|
||||
b
|
||||
}
|
||||
|
||||
/// Issue #138: a DNG's own profile look (hue/saturation map, look table, tone curve) is
|
||||
/// applied when it's developed — Lightroom-converted DNGs rendered flat and muted without it.
|
||||
#[test]
|
||||
fn dng_profile_look_is_applied() {
|
||||
use lightcraft_raw::profile::{HsvTable, ProfileLook, ToneCurve};
|
||||
let plain = crate::tests_xmp::synthetic_dng_with(None, Default::default());
|
||||
let mut raw = lightcraft_raw::decode(&plain).unwrap();
|
||||
let sat = |img: &Rgb32f| {
|
||||
img.data.iter().map(|p| (p[0].max(p[1]).max(p[2]) - p[0].min(p[1]).min(p[2])) / p[0].max(p[1]).max(p[2]).max(1e-6)).sum::<f32>()
|
||||
/ img.data.len() as f32
|
||||
};
|
||||
let (before, info) = load_bytes(&plain, 64).unwrap();
|
||||
assert!(sat(&before) > 0.05, "the synthetic scene is coloured");
|
||||
assert!(info.camera_tone.is_none());
|
||||
// a map that removes all saturation, and a tone curve
|
||||
let grey = HsvTable { hue_divisions: 4, sat_divisions: 2, val_divisions: 1, data: vec![[0.0, 0.0, 1.0]; 8], srgb_value: false };
|
||||
raw.color.profile =
|
||||
ProfileLook { hue_sat_map: [Some(grey), None], look_table: None, tone_curve: ToneCurve::from_tag(&[0.0, 0.0, 0.18, 0.3, 1.0, 1.0]) };
|
||||
let with = lightcraft_raw::write_dng(&raw, &Default::default()).unwrap();
|
||||
let (after, info) = load_bytes(&with, 64).unwrap();
|
||||
assert!(sat(&after) < 1e-3, "saturation {} → {}", sat(&before), sat(&after));
|
||||
// a saturation-only map keeps brightness roughly (HSV value is kept in ProPhoto RGB)
|
||||
let mean = |img: &Rgb32f| img.data.iter().map(|p| p[0].max(p[1]).max(p[2])).sum::<f32>() / img.data.len() as f32;
|
||||
assert!((0.8..1.1).contains(&(mean(&after) / mean(&before))), "{} vs {}", mean(&after), mean(&before));
|
||||
let tone = info.camera_tone.expect("the DNG tone curve becomes the camera tone");
|
||||
assert!((tone.apply(0.18) - 0.3).abs() < 0.01, "{}", tone.apply(0.18));
|
||||
// a look table alone changes the render too (applied after exposure)
|
||||
raw.color.profile = ProfileLook {
|
||||
look_table: Some(HsvTable { hue_divisions: 1, sat_divisions: 2, val_divisions: 1, data: vec![[0.0, 1.0, 0.5]; 2], srgb_value: false }),
|
||||
..Default::default()
|
||||
};
|
||||
let (dim, _) = load_bytes(&lightcraft_raw::write_dng(&raw, &Default::default()).unwrap(), 64).unwrap();
|
||||
assert!((mean(&dim) / mean(&before) - 0.5).abs() < 0.02, "{} vs {}", mean(&dim), mean(&before));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_raw_falls_back_to_embedded_preview() {
|
||||
let b = cr3_with_preview(48, 32);
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
//! Fonts from the optional craft-fonts build input (storytold/craft-fonts).
|
||||
//!
|
||||
//! Built with `CRAFT_FONTS_DIR=<craft-fonts checkout>`, [`CRAFT_FONTS`] holds every font in its
|
||||
//! manifest (today: the Japanese and Simplified-Chinese UI and document fonts); otherwise it is
|
||||
//! empty and LightCraft uses only its own bundled fonts (Inter). Both the egui UI and the export
|
||||
//! watermark renderer read it from here. See craftrules `standards/fonts.md`.
|
||||
|
||||
/// A font from the optional craft-fonts build input (empty unless built with `CRAFT_FONTS_DIR`).
|
||||
pub struct CraftFont {
|
||||
pub family: &'static str,
|
||||
pub style: &'static str,
|
||||
/// ISO 15924 scripts the font is for, e.g. `"Jpan"`.
|
||||
pub scripts: &'static [&'static str],
|
||||
pub bytes: &'static [u8],
|
||||
}
|
||||
|
||||
include!(concat!(env!("OUT_DIR"), "/craft_fonts.rs"));
|
||||
|
||||
impl CraftFont {
|
||||
/// Whether the font is meant for `script` (ISO 15924, e.g. `"Jpan"`).
|
||||
pub fn covers(&self, script: &str) -> bool {
|
||||
self.scripts.contains(&script)
|
||||
}
|
||||
|
||||
/// Mincho (serif) faces, the preferred Japanese faces for document-like text.
|
||||
pub fn is_mincho(&self) -> bool {
|
||||
self.family.contains("Mincho")
|
||||
}
|
||||
|
||||
/// Whether the font can stand in for `script`: it is meant for it, or — until craft-fonts has a
|
||||
/// Traditional Chinese face — it is a Simplified Chinese face, which carries the Traditional
|
||||
/// characters too (with mainland glyph forms; still closer than the Japanese faces).
|
||||
pub fn serves(&self, script: &str) -> bool {
|
||||
self.covers(script) || (script == "Hant" && self.covers("Hans"))
|
||||
}
|
||||
}
|
||||
|
||||
/// The craft-fonts faces for Japanese, in `fonts`' order (empty without craft-fonts).
|
||||
pub fn japanese(fonts: &'static [CraftFont]) -> impl Iterator<Item = &'static CraftFont> {
|
||||
fonts.iter().filter(|f| f.covers("Jpan"))
|
||||
}
|
||||
|
||||
/// The craft-fonts faces for any CJK script: what any CJK text needs, regardless of language.
|
||||
pub fn cjk(fonts: &'static [CraftFont]) -> impl Iterator<Item = &'static CraftFont> {
|
||||
fonts.iter().filter(|f| f.scripts.iter().any(|script| is_cjk_script(script)))
|
||||
}
|
||||
|
||||
/// Whether an ISO 15924 script is written with Han, kana or hangul characters (Unicode's CJK
|
||||
/// scripts), which Inter has no glyphs for.
|
||||
pub fn is_cjk_script(script: &str) -> bool {
|
||||
matches!(script, "Hans" | "Hant" | "Jpan" | "Kore" | "Hang" | "Hani" | "Bopo" | "Yiii" | "Nshu" | "Tang")
|
||||
}
|
||||
|
||||
/// The craft-fonts faces to fall back on for `script`, in preference order: the faces of that script
|
||||
/// first (for `style` when asked), then every other CJK face. One CJK face list serves every
|
||||
/// language, but Han characters are shared between them: a Chinese reader must get the Chinese
|
||||
/// forms and a Japanese reader the Japanese ones, so the active language's own script comes first.
|
||||
/// Empty without craft-fonts.
|
||||
pub fn cjk_fallback<'a>(fonts: &'a [CraftFont], script: &str, style: &str) -> Vec<&'a CraftFont> {
|
||||
let mut faces: Vec<&CraftFont> = fonts.iter().filter(|font| covers_cjk(font)).collect();
|
||||
faces.sort_by_key(|font| {
|
||||
(
|
||||
// The language's own script first. A face dedicated to the script is preferred over a
|
||||
// multi-script face (Noto Sans CJK SC claims `Hans,Latn`; a Japanese face claims `Jpan`).
|
||||
!font.serves(script),
|
||||
font.scripts.len() != 1,
|
||||
font.family != "BIZ UDPGothic",
|
||||
font.style != style,
|
||||
font.is_mincho(),
|
||||
)
|
||||
});
|
||||
faces
|
||||
}
|
||||
|
||||
/// Whether the font is for any CJK script.
|
||||
fn covers_cjk(font: &CraftFont) -> bool {
|
||||
font.scripts.iter().any(|script| is_cjk_script(script))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn craft_fonts_are_well_formed_or_absent() {
|
||||
if CRAFT_FONTS.is_empty() {
|
||||
eprintln!("built without CRAFT_FONTS_DIR: no craft-fonts to check");
|
||||
return;
|
||||
}
|
||||
for f in CRAFT_FONTS {
|
||||
assert!(!f.family.is_empty() && !f.style.is_empty() && !f.scripts.is_empty(), "{}", f.family);
|
||||
assert!(ab_glyph::FontRef::try_from_slice(f.bytes).is_ok(), "{} {} parses", f.family, f.style);
|
||||
}
|
||||
assert!(japanese(CRAFT_FONTS).next().is_some(), "craft-fonts carries Japanese faces");
|
||||
}
|
||||
|
||||
/// Each language's own faces come first, so shared Han keeps the language's forms, and Latin
|
||||
/// text still reaches Inter before any CJK fallback.
|
||||
#[test]
|
||||
fn a_language_gets_its_own_script_first() {
|
||||
static FACES: &[CraftFont] = &[
|
||||
CraftFont { family: "BIZ UDPGothic", style: "Regular", scripts: &["Jpan", "Latn"], bytes: &[] },
|
||||
CraftFont { family: "Noto Sans CJK SC", style: "Regular", scripts: &["Hans", "Latn"], bytes: &[] },
|
||||
CraftFont { family: "BIZ UDMincho", style: "Regular", scripts: &["Jpan", "Latn"], bytes: &[] },
|
||||
];
|
||||
let first = |script: &str| cjk_fallback(FACES, script, "Regular").first().map(|f| f.family);
|
||||
assert_eq!(first("Hans"), Some("Noto Sans CJK SC"));
|
||||
assert_eq!(first("Jpan"), Some("BIZ UDPGothic"));
|
||||
// No Traditional Chinese face yet: the Chinese face stands in before the Japanese ones.
|
||||
assert_eq!(first("Hant"), Some("Noto Sans CJK SC"));
|
||||
// Every CJK face is still offered to every language.
|
||||
assert_eq!(cjk_fallback(FACES, "Hans", "Regular").len(), 3);
|
||||
assert_eq!(cjk_fallback(&[], "Hans", "Regular").len(), 0);
|
||||
}
|
||||
}
|
||||
@@ -375,13 +375,16 @@ fn probe_all(probe: Option<&crate::media::FileProbe>, paths: &[String], progress
|
||||
|
||||
/// Copy `src` into `root`/`folders` as `name` (default: its own name), made unique with -1, -2…;
|
||||
/// returns the new path. The copy is verified ([`crate::import_move::copy_new`]: a new file,
|
||||
/// synced and compared byte for byte); a bad one is removed and the photo reported as failed.
|
||||
fn copy_into(root: &Path, src: &str, folders: &[String], name: Option<&str>) -> Result<String, String> {
|
||||
/// synced and checked against `probe_hash`, the content hash the probe computed — or compared
|
||||
/// byte for byte with the source when there is none); a bad one is removed and the photo
|
||||
/// reported as failed.
|
||||
fn copy_into(root: &Path, src: &str, folders: &[String], name: Option<&str>, probe_hash: Option<&str>) -> Result<String, String> {
|
||||
let dir = folders.iter().fold(root.to_path_buf(), |d, f| d.join(f));
|
||||
let name = name
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| Path::new(src).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "photo".into()));
|
||||
crate::import_move::copy_new(Path::new(src), &dir, &name).map(|p| p.to_string_lossy().to_string())
|
||||
let expect = probe_hash.and_then(lightcraft_preview::Hash128::parse);
|
||||
crate::import_move::copy_new(Path::new(src), &dir, &name, expect).map(|p| p.to_string_lossy().to_string())
|
||||
}
|
||||
|
||||
/// What a [`scan`] needs from the session, so it can run on another thread (a folder on a network
|
||||
@@ -784,7 +787,7 @@ impl ImportJob {
|
||||
let reason = "a symbolic link: the file it points to was copied, the link and its target stay";
|
||||
out.items.push(PreparedItem::Kept(Kept { path: path.clone(), reason: reason.into() }));
|
||||
}
|
||||
match copy_into(root, &path, &folders, name.as_deref()) {
|
||||
match copy_into(root, &path, &folders, name.as_deref(), info.content_hash.as_deref()) {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
out.items.push(PreparedItem::Failed(path, e));
|
||||
|
||||
@@ -171,9 +171,15 @@ pub(crate) fn place(src: &Path, dir: &Path, name: &str) -> Result<Placed, String
|
||||
|
||||
/// Import → **Copy**: copy `src` into `dir` as `name` (-1, -2… when taken) with the same checks as
|
||||
/// a move's copy — a new file (never overwriting one, even one that appears meanwhile), synced to
|
||||
/// disk and compared byte for byte with the source; a bad copy is removed and reported, so a card
|
||||
/// is never wiped on the strength of it. Returns the new path.
|
||||
pub(crate) fn copy_new(src: &Path, dir: &Path, name: &str) -> Result<PathBuf, String> {
|
||||
/// disk and verified; a bad copy is removed and reported, so a card is never wiped on the strength
|
||||
/// of it. Returns the new path.
|
||||
///
|
||||
/// `expect` is the content hash the import's probe computed from its full read of the source
|
||||
/// ([`lightcraft_preview::hash_bytes`]): the copy is read back and checked against it, which spares
|
||||
/// a third read of the card (issue #134). It also catches a source that changed since the probe
|
||||
/// (whose recorded metadata and hash would be stale). Without one the copy is compared byte for
|
||||
/// byte with the source.
|
||||
pub(crate) fn copy_new(src: &Path, dir: &Path, name: &str, expect: Option<lightcraft_preview::Hash128>) -> Result<PathBuf, String> {
|
||||
fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
|
||||
let (stem, ext) = match name.rsplit_once('.') {
|
||||
Some((s, e)) if !s.is_empty() => (s.to_string(), format!(".{e}")),
|
||||
@@ -181,7 +187,7 @@ pub(crate) fn copy_new(src: &Path, dir: &Path, name: &str) -> Result<PathBuf, St
|
||||
};
|
||||
for i in 0..100_000u32 {
|
||||
let d = if i == 0 { dir.join(name) } else { dir.join(format!("{stem}-{i}{ext}")) };
|
||||
match copy_verified(src, &d) {
|
||||
match copy_verified(src, &d, expect) {
|
||||
Ok(()) => return Ok(d),
|
||||
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue,
|
||||
Err(e) => return Err(format!("copy {}: {e} (nothing was imported from it; the original is untouched)", src.display())),
|
||||
@@ -262,12 +268,14 @@ fn link_or_copy(src: &Path, dst: &Path) -> io::Result<()> {
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
copy_verified(src, dst)
|
||||
// Move keeps the byte-for-byte comparison: the source is deleted on the strength of it
|
||||
copy_verified(src, dst, None)
|
||||
}
|
||||
|
||||
/// Copy into a new file, sync it to disk and compare it with the original; on any failure the
|
||||
/// partial copy is removed.
|
||||
fn copy_verified(src: &Path, dst: &Path) -> io::Result<()> {
|
||||
/// Copy into a new file, sync it to disk and verify it: against `expect` (the probe's hash of the
|
||||
/// source) when given, else byte for byte against the source. On any failure the partial copy is
|
||||
/// removed.
|
||||
pub(crate) fn copy_verified(src: &Path, dst: &Path, expect: Option<lightcraft_preview::Hash128>) -> io::Result<()> {
|
||||
let mut out = OpenOptions::new().write(true).create_new(true).open(dst)?;
|
||||
let r = (|| {
|
||||
let mut inp = File::open(src)?;
|
||||
@@ -283,10 +291,14 @@ fn copy_verified(src: &Path, dst: &Path) -> io::Result<()> {
|
||||
if let Ok(m) = fs::metadata(src).and_then(|m| m.modified()) {
|
||||
let _ = out.set_modified(m);
|
||||
}
|
||||
if !same_bytes(src, dst)? {
|
||||
return Err(io::Error::other("the copy differs from the original"));
|
||||
match expect {
|
||||
Some(h) if hash_file(dst)? != h => {
|
||||
Err(io::Error::other("the copy differs from the original as it was read for the import (a bad copy, or the file changed meanwhile)"))
|
||||
}
|
||||
Some(_) => Ok(()),
|
||||
None if !same_bytes(src, dst)? => Err(io::Error::other("the copy differs from the original")),
|
||||
None => Ok(()),
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
drop(out);
|
||||
if r.is_err() {
|
||||
@@ -295,6 +307,20 @@ fn copy_verified(src: &Path, dst: &Path) -> io::Result<()> {
|
||||
r
|
||||
}
|
||||
|
||||
/// [`lightcraft_preview::hash_bytes`] of a file's content, read in chunks.
|
||||
fn hash_file(path: &Path) -> io::Result<lightcraft_preview::Hash128> {
|
||||
let mut f = File::open(path)?;
|
||||
let mut h = lightcraft_preview::Hasher128::new();
|
||||
let mut buf = vec![0u8; 1 << 20];
|
||||
loop {
|
||||
let n = fill(&mut f, &mut buf)?;
|
||||
if n == 0 {
|
||||
return Ok(h.finish());
|
||||
}
|
||||
h.update(buf.get(..n).unwrap_or(&[]));
|
||||
}
|
||||
}
|
||||
|
||||
/// Do two files hold the same bytes?
|
||||
pub(crate) fn same_bytes(a: &Path, b: &Path) -> io::Result<bool> {
|
||||
if fs::metadata(a)?.len() != fs::metadata(b)?.len() {
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
|
||||
pub mod availability;
|
||||
mod camera_preview;
|
||||
pub mod camera_profiles;
|
||||
pub mod cmd;
|
||||
pub mod crs;
|
||||
pub mod crs_masks;
|
||||
@@ -19,6 +20,7 @@ pub mod demo;
|
||||
pub mod devices;
|
||||
pub mod export;
|
||||
pub mod files;
|
||||
pub mod fonts;
|
||||
pub mod guard;
|
||||
pub mod import;
|
||||
mod import_move;
|
||||
@@ -31,6 +33,7 @@ pub mod preset_import;
|
||||
pub mod preset_luminar;
|
||||
pub mod presets;
|
||||
pub mod rename;
|
||||
pub mod segment;
|
||||
pub mod sidecar;
|
||||
pub mod smart;
|
||||
mod view;
|
||||
@@ -38,6 +41,7 @@ mod view;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub use cmd::{CommandInfo, CommandSpec, command_specs, find_command};
|
||||
pub use fonts::{CRAFT_FONTS, CraftFont};
|
||||
use lightcraft_catalog::{Catalog, Filter, Op, PhotoId, Sort};
|
||||
use lightcraft_develop::DevelopSettings;
|
||||
pub use media::{RenderJob, SourceLevel};
|
||||
@@ -123,6 +127,9 @@ pub struct Session {
|
||||
pub undo: Vec<UndoEntry>,
|
||||
pub redo: Vec<UndoEntry>,
|
||||
pub interaction: Option<Interaction>,
|
||||
/// Set by a command whose change must not rewrite the photo's XMP sidecar even with auto-write on
|
||||
/// (a catalog-only edit of data the sidecar writer does not emit); consumed when the command ends.
|
||||
pub(crate) skip_auto_write: bool,
|
||||
/// Copied develop settings (partial JSON) for Paste.
|
||||
pub clipboard: Option<Value>,
|
||||
/// The folder on disk the [`LibrarySource::Folder`] view browses.
|
||||
@@ -148,6 +155,8 @@ pub struct Session {
|
||||
depth: u32,
|
||||
/// Selected mask (Masking panel), by mask id.
|
||||
pub active_mask: Option<u32>,
|
||||
/// AI masks (SAM 3): the model and the last photo prepared for it.
|
||||
pub segmenter: segment::Segmenter,
|
||||
/// Selected spot (Remove panel), by index into the active photo's spots.
|
||||
pub active_spot: Option<usize>,
|
||||
/// The persistent library this session writes to (`None` = in-memory only).
|
||||
@@ -221,6 +230,7 @@ impl Session {
|
||||
undo: Vec::new(),
|
||||
redo: Vec::new(),
|
||||
interaction: None,
|
||||
skip_auto_write: false,
|
||||
clipboard: None,
|
||||
meta_clipboard: None,
|
||||
browse: None,
|
||||
@@ -235,6 +245,7 @@ impl Session {
|
||||
clock: Box::new(|| "2026-09-30T12:00:00".to_string()),
|
||||
depth: 0,
|
||||
active_mask: None,
|
||||
segmenter: segment::Segmenter::default(),
|
||||
active_spot: None,
|
||||
library: None,
|
||||
xmp: sidecar::XmpPrefs::default(),
|
||||
@@ -306,8 +317,11 @@ impl Session {
|
||||
self.journal.drain(..1000);
|
||||
}
|
||||
}
|
||||
if r.is_ok() && self.depth == 0 && self.xmp.auto_write && self.interaction.is_none() && self.pending_log.len() > log_start {
|
||||
self.auto_write_sidecars(&self.pending_log[log_start..]);
|
||||
if self.depth == 0 {
|
||||
let skip = std::mem::take(&mut self.skip_auto_write);
|
||||
if r.is_ok() && !skip && self.xmp.auto_write && self.interaction.is_none() && self.pending_log.len() > log_start {
|
||||
self.auto_write_sidecars(&self.pending_log[log_start..]);
|
||||
}
|
||||
}
|
||||
if self.depth == 0 && self.library.is_some() {
|
||||
// Make the command durable before reporting success. When the command's own records
|
||||
@@ -713,6 +727,8 @@ mod tests_persist;
|
||||
#[cfg(test)]
|
||||
mod tests_prefs;
|
||||
#[cfg(test)]
|
||||
mod tests_segment;
|
||||
#[cfg(test)]
|
||||
mod tests_settings_files;
|
||||
#[cfg(test)]
|
||||
mod tests_spots;
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
//! ```text
|
||||
//! LightCraft Library/
|
||||
//! catalog.snap catalog.log (lightcraft-catalog journal)
|
||||
//! presets.json view.json (user presets + favourites; last source/filter/sort/selection)
|
||||
//! presets.json view.json (user presets + favourites; last source/sort/selection)
|
||||
//! prefs.json (library preferences: XMP sidecars, import defaults, cache size, last export)
|
||||
//! thumbs/ (rendered thumbnail cache, safe to delete)
|
||||
//! Originals/ (photos imported with "copy into library")
|
||||
@@ -102,7 +102,8 @@ struct PresetsFile {
|
||||
struct ViewFile {
|
||||
source: LibrarySource,
|
||||
browse: Option<crate::Browse>,
|
||||
filter: lightcraft_catalog::Filter,
|
||||
// No filter: a library opens unfiltered. A date, keyword or person left over from the last session
|
||||
// would silently hide photos, with only a small badge to say so.
|
||||
sort: lightcraft_catalog::Sort,
|
||||
selection: Selection,
|
||||
}
|
||||
@@ -346,7 +347,6 @@ impl Session {
|
||||
if let Some(v) = settings.read::<ViewFile>(files.as_mut(), "view.json") {
|
||||
self.source = v.source;
|
||||
self.browse = v.browse;
|
||||
self.filter = v.filter;
|
||||
self.sort = v.sort;
|
||||
self.selection = v.selection;
|
||||
self.selection.ids.retain(|id| self.catalog.photo(*id).is_some());
|
||||
@@ -494,17 +494,11 @@ impl Session {
|
||||
}
|
||||
|
||||
fn view_json(&self) -> Vec<u8> {
|
||||
let view = ViewFile {
|
||||
source: self.source,
|
||||
browse: self.browse.clone(),
|
||||
filter: self.filter.clone(),
|
||||
sort: self.sort,
|
||||
selection: self.selection.clone(),
|
||||
};
|
||||
let view = ViewFile { source: self.source, browse: self.browse.clone(), sort: self.sort, selection: self.selection.clone() };
|
||||
serde_json::to_vec_pretty(&view).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Save the view state (source, filter, sort, selection) if it changed since it was last
|
||||
/// Save the view state (source, sort, selection) if it changed since it was last
|
||||
/// written. Native hosts get this from [`Session::close_library`]; the browser host calls it
|
||||
/// periodically, since a tab can be closed without notice.
|
||||
pub fn save_view(&mut self) {
|
||||
|
||||
@@ -26,7 +26,7 @@ use lightcraft_raster::{Histogram, Rgb32f, Rgba8};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Bump when the pipeline's output changes, to invalidate cached thumbnails.
|
||||
pub const RENDER_CACHE_VERSION: u64 = 6;
|
||||
pub const RENDER_CACHE_VERSION: u64 = 11;
|
||||
|
||||
/// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache).
|
||||
pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512];
|
||||
@@ -647,7 +647,13 @@ impl crate::Session {
|
||||
^ id.0.wrapping_mul(0x9e37_79b9_7f4a_7c15)
|
||||
^ content.rotate_left(17);
|
||||
let cache = thumb_bucket.map(|b| {
|
||||
let k = Hasher128::new().str(&content_key(&p)).u64(settings.hash64()).u64(b as u64).u64(RENDER_CACHE_VERSION).finish();
|
||||
let k = Hasher128::new()
|
||||
.str(&content_key(&p))
|
||||
.u64(settings.hash64())
|
||||
.u64(b as u64)
|
||||
.u64(RENDER_CACHE_VERSION)
|
||||
.u64(crate::camera_profiles::cache_key())
|
||||
.finish();
|
||||
(self.media.rendered.clone(), k)
|
||||
});
|
||||
Some(RenderJob {
|
||||
@@ -676,7 +682,14 @@ impl crate::Session {
|
||||
|
||||
/// Cache key of a variant thumbnail ([`Self::variant_job`]).
|
||||
pub fn variant_key(p: &Photo, settings: &DevelopSettings, edge: usize) -> Hash128 {
|
||||
Hasher128::new().str(&content_key(p)).str("variant").u64(settings.hash64()).u64(edge as u64).u64(RENDER_CACHE_VERSION).finish()
|
||||
Hasher128::new()
|
||||
.str(&content_key(p))
|
||||
.str("variant")
|
||||
.u64(settings.hash64())
|
||||
.u64(edge as u64)
|
||||
.u64(RENDER_CACHE_VERSION)
|
||||
.u64(crate::camera_profiles::cache_key())
|
||||
.finish()
|
||||
}
|
||||
|
||||
/// A thumbnail of `id` rendered with `settings` instead of its own (profile and preset
|
||||
@@ -705,9 +718,58 @@ impl crate::Session {
|
||||
})
|
||||
}
|
||||
|
||||
/// A square close-up of `face` (normalized, in the photo's upright frame) for a People card: the
|
||||
/// photo's own look with its crop replaced by a square around the face (room for hair and chin),
|
||||
/// `edge` pixels across (≤ 512). The source level is the smallest that keeps the face sharp, up
|
||||
/// to the preview. Cached like [`Self::variant_job`], and `key` likewise follows the content.
|
||||
pub fn face_job(&mut self, id: PhotoId, face: lightcraft_geom::Rect, edge: usize) -> Option<RenderJob> {
|
||||
let p = self.catalog.photo(id)?.clone();
|
||||
// `face` is on the upright (EXIF-oriented) photo; the crop below is in the frame after the
|
||||
// user's Rotate Left/Right, so carry the box (and the photo's size) over to it.
|
||||
let orient = p.develop.orientation;
|
||||
let face = orient.map_norm_rect(face);
|
||||
let (w, h) = (f64::from(p.width.max(1)), f64::from(p.height.max(1)));
|
||||
let (w, h) = if orient.swaps_axes() { (h, w) } else { (w, h) };
|
||||
let side = ((face.x1 - face.x0) * w).max((face.y1 - face.y0) * h) * 1.7;
|
||||
// finite, within the photo, and never wider than its short side (so the clamps below are valid)
|
||||
let side = if side.is_finite() { side.clamp(1.0, w.min(h)) } else { w.min(h) };
|
||||
let (cx, cy) = ((face.x0 + face.x1) / 2.0 * w, (face.y0 + face.y1) / 2.0 * h);
|
||||
let (cx, cy) = (if cx.is_finite() { cx } else { w / 2.0 }, if cy.is_finite() { cy } else { h / 2.0 });
|
||||
let (x0, y0) = (cx.clamp(side / 2.0, w - side / 2.0) - side / 2.0, cy.clamp(side / 2.0, h - side / 2.0) - side / 2.0);
|
||||
let rect = lightcraft_geom::Rect { x0: x0 / w, y0: y0 / h, x1: (x0 + side) / w, y1: (y0 + side) / h };
|
||||
let mut settings = (*p.develop).clone();
|
||||
settings.crop = lightcraft_develop::Crop { geometry: lightcraft_geom::CropGeometry { rect, angle: 0.0 }, ..Default::default() };
|
||||
let edge = edge.clamp(16, SourceLevel::Thumb.max_edge());
|
||||
// pixels along the long edge that leave `edge` across the crop
|
||||
let needed = (edge as f64 * w.max(h) / side).ceil().min(SourceLevel::Preview.max_edge() as f64);
|
||||
let level = SourceLevel::for_size(needed as usize);
|
||||
let source = self.media.source_ref(&p, level);
|
||||
let ck = Self::variant_key(&p, &settings, edge);
|
||||
Some(RenderJob {
|
||||
photo: id,
|
||||
level,
|
||||
source,
|
||||
origin: p.source.clone(),
|
||||
info: source_info(&p),
|
||||
settings: Arc::new(settings),
|
||||
request: RenderRequest { apply_crop: true, ..RenderRequest::fit(edge, edge) },
|
||||
key: (ck.0 as u64) ^ ((ck.0 >> 64) as u64),
|
||||
cache: Some((self.media.rendered.clone(), ck)),
|
||||
stages: None,
|
||||
view_cache: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Size-independent cache key of a photo's view render (loupe) for its current settings.
|
||||
fn view_key(p: &Photo, apply_crop: bool) -> Hash128 {
|
||||
Hasher128::new().str(&content_key(p)).str("view").u64(p.develop.hash64()).u64(apply_crop as u64).u64(RENDER_CACHE_VERSION).finish()
|
||||
Hasher128::new()
|
||||
.str(&content_key(p))
|
||||
.str("view")
|
||||
.u64(p.develop.hash64())
|
||||
.u64(apply_crop as u64)
|
||||
.u64(RENDER_CACHE_VERSION)
|
||||
.u64(crate::camera_profiles::cache_key())
|
||||
.finish()
|
||||
}
|
||||
|
||||
/// The loupe's render job: like [`Self::render_job`], and a full-quality result is kept as the
|
||||
@@ -830,7 +892,9 @@ pub struct ProbeInfo {
|
||||
pub captured: Option<String>,
|
||||
pub meta: lightcraft_catalog::Meta,
|
||||
pub as_shot_wb: Option<(f64, f64)>,
|
||||
/// Hash of the file's bytes (hex), for duplicate detection.
|
||||
/// Hash of the file's bytes (hex), for duplicate detection. When it is a 32-digit
|
||||
/// [`lightcraft_preview::hash_bytes`] of the whole file (as the native probe computes it),
|
||||
/// Import → Copy verifies each copy against it instead of reading the source again.
|
||||
pub content_hash: Option<String>,
|
||||
/// Lens corrections embedded in the file (DNG opcodes).
|
||||
pub embedded_lens: Option<lightcraft_develop::EmbeddedLens>,
|
||||
|
||||
@@ -15,6 +15,8 @@ use lightcraft_catalog::{Catalog, Source};
|
||||
pub struct OriginalGuard {
|
||||
/// lower-case file name → (path on disk, what it is: "the original of IMG_1.CR3", …)
|
||||
by_name: HashMap<String, Vec<(PathBuf, String)>>,
|
||||
/// The entries of `by_name` that are symlinks, found on the first check that needs them.
|
||||
links: std::sync::OnceLock<Vec<(PathBuf, String)>>,
|
||||
}
|
||||
|
||||
impl OriginalGuard {
|
||||
@@ -32,6 +34,7 @@ impl OriginalGuard {
|
||||
/// Protect `path` (an original named `label`) and its sidecars (`IMG_1.xmp`, `IMG_1.CR3.xmp`).
|
||||
pub fn add(&mut self, path: &Path, label: &str) {
|
||||
let Some(name) = path.file_name().map(|n| n.to_string_lossy().to_string()) else { return };
|
||||
self.links = std::sync::OnceLock::new();
|
||||
let mut put = |file: String, p: PathBuf, what: String| self.by_name.entry(file.to_lowercase()).or_default().push((p, what));
|
||||
put(name.clone(), path.to_path_buf(), format!("the original of {label}"));
|
||||
let stem = path.file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_default();
|
||||
@@ -58,6 +61,19 @@ impl OriginalGuard {
|
||||
}
|
||||
}
|
||||
}
|
||||
// An imported symlink can have a different name from its original. A write to
|
||||
// that original would change what the library reads through the link. Only do
|
||||
// this fallback for existing targets that the name index did not protect; the links are
|
||||
// looked up once per guard, not once per target.
|
||||
let links = self.links.get_or_init(|| self.by_name.values().flatten().filter(|(p, _)| std::fs::read_link(p).is_ok()).cloned().collect());
|
||||
for (p, what) in links {
|
||||
if same_file(p, target) {
|
||||
return Err(format!(
|
||||
"{} is {what} in the library: LightCraft never writes over an original (choose another folder or file name)",
|
||||
target.display()
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -68,9 +84,14 @@ impl crate::Session {
|
||||
OriginalGuard::new(&self.catalog)
|
||||
}
|
||||
|
||||
/// `Err` when writing `path` would replace a catalogued photo's original or sidecar.
|
||||
/// `Err` when writing `path` would replace a catalogued photo's original or sidecar. A free
|
||||
/// path (the usual case) is answered without building the guard over the whole library.
|
||||
pub fn check_write_target(&self, path: &str) -> Result<(), String> {
|
||||
self.original_guard().check(Path::new(path))
|
||||
let path = Path::new(path);
|
||||
if std::fs::symlink_metadata(path).is_err() {
|
||||
return Ok(()); // nothing there to replace
|
||||
}
|
||||
self.original_guard().check(path)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
//! The model download as a background task the session starts, watches and cancels without
|
||||
//! ever blocking: progress lives in atomics, messages behind a mutex only ever `try_lock`ed
|
||||
//! by the session (the download thread holds it for a few instructions at a time).
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use lightcraft_segment::fetch::{self, Options, Progress};
|
||||
|
||||
/// What the session sees of a download.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
||||
pub struct Status {
|
||||
pub running: bool,
|
||||
pub done: u64,
|
||||
pub total: u64,
|
||||
/// The file being fetched.
|
||||
pub file: String,
|
||||
/// The last download's error (cleared when a new one starts).
|
||||
pub error: Option<String>,
|
||||
/// The last download finished and the files are in place.
|
||||
pub finished: bool,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Text {
|
||||
file: String,
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Shared {
|
||||
running: AtomicBool,
|
||||
cancel: AtomicBool,
|
||||
finished: AtomicBool,
|
||||
done: AtomicU64,
|
||||
total: AtomicU64,
|
||||
text: Mutex<Text>,
|
||||
}
|
||||
|
||||
/// Clears `running` when the download thread ends, however it ends.
|
||||
struct Running(Arc<Shared>);
|
||||
|
||||
impl Drop for Running {
|
||||
fn drop(&mut self) {
|
||||
self.0.running.store(false, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Downloader {
|
||||
shared: Arc<Shared>,
|
||||
/// The last text seen (when the download thread held the lock at that moment).
|
||||
last: Mutex<Text>,
|
||||
}
|
||||
|
||||
fn lock(m: &Mutex<Text>) -> std::sync::MutexGuard<'_, Text> {
|
||||
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
|
||||
impl Downloader {
|
||||
pub fn status(&self) -> Status {
|
||||
let s = &self.shared;
|
||||
if let Ok(t) = s.text.try_lock()
|
||||
&& let Ok(mut last) = self.last.try_lock()
|
||||
{
|
||||
last.file.clone_from(&t.file);
|
||||
last.error.clone_from(&t.error);
|
||||
}
|
||||
// (only the session's thread locks `last`)
|
||||
let (file, error) = self.last.try_lock().map(|l| (l.file.clone(), l.error.clone())).unwrap_or_default();
|
||||
Status {
|
||||
running: s.running.load(Ordering::SeqCst),
|
||||
done: s.done.load(Ordering::SeqCst),
|
||||
total: s.total.load(Ordering::SeqCst),
|
||||
file,
|
||||
error,
|
||||
finished: s.finished.load(Ordering::SeqCst),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn running(&self) -> bool {
|
||||
self.shared.running.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Ask a running download to stop (it keeps its partial files to resume later).
|
||||
pub fn cancel(&self) -> bool {
|
||||
let running = self.running();
|
||||
if running {
|
||||
self.shared.cancel.store(true, Ordering::SeqCst);
|
||||
}
|
||||
running
|
||||
}
|
||||
|
||||
/// Start downloading `files` from `mirrors` into `dir` on a background thread. False when
|
||||
/// one is already running.
|
||||
pub fn start(&self, files: &'static [fetch::FileSpec], mirrors: Vec<String>, dir: PathBuf, opts: Options) -> Result<bool, String> {
|
||||
let s = self.shared.clone();
|
||||
if s.running.swap(true, Ordering::SeqCst) {
|
||||
return Ok(false);
|
||||
}
|
||||
s.cancel.store(false, Ordering::SeqCst);
|
||||
s.finished.store(false, Ordering::SeqCst);
|
||||
s.done.store(0, Ordering::SeqCst);
|
||||
s.total.store(files.iter().filter_map(|f| f.size).sum(), Ordering::SeqCst);
|
||||
// nothing else runs while `running` was false
|
||||
*lock(&s.text) = Text::default();
|
||||
*lock(&self.last) = Text::default();
|
||||
let guard = Running(s.clone());
|
||||
let spawned = std::thread::Builder::new().name("sam3-download".into()).spawn(move || {
|
||||
let s = guard.0.clone();
|
||||
let mut last_file = String::new();
|
||||
let mut on_progress = |p: &Progress| {
|
||||
s.done.store(p.done, Ordering::SeqCst);
|
||||
s.total.store(p.total, Ordering::SeqCst);
|
||||
if p.file != last_file {
|
||||
last_file.clone_from(&p.file);
|
||||
lock(&s.text).file.clone_from(&p.file);
|
||||
}
|
||||
};
|
||||
let r =
|
||||
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| fetch::download(files, &mirrors, &dir, &opts, &s.cancel, &mut on_progress)));
|
||||
let error = match r {
|
||||
Ok(Ok(())) => {
|
||||
s.finished.store(true, Ordering::SeqCst);
|
||||
log::info!("SAM 3 model downloaded to {}", dir.display());
|
||||
None
|
||||
}
|
||||
Ok(Err(e)) => Some(e.to_string()),
|
||||
Err(_) => Some("the download failed unexpectedly".to_string()),
|
||||
};
|
||||
if let Some(e) = &error {
|
||||
log::warn!("SAM 3 download: {e}");
|
||||
}
|
||||
lock(&s.text).error = error;
|
||||
drop(guard);
|
||||
});
|
||||
match spawned {
|
||||
Ok(_) => Ok(true),
|
||||
Err(e) => {
|
||||
// the closure (and its guard) was dropped: `running` is false again
|
||||
Err(format!("could not start the download: {e}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,715 @@
|
||||
//! AI masks: Object (clicks) and Describe (text) selections computed with SAM 3
|
||||
//! (`lightcraft-segment`, cargo feature `sam`; the desktop app enables it).
|
||||
//!
|
||||
//! **Nothing requires the model.** It is not part of LightCraft (SAM License); the user
|
||||
//! downloads it when they first use an AI mask and agree to (`segment.model.download`), or
|
||||
//! puts the files in the model folder themselves. Without it, AI mask requests fail with a
|
||||
//! clear "not installed" error; renders, exports and every other feature never touch it
|
||||
//! (stored segmentations are plain data in the develop settings).
|
||||
//!
|
||||
//! The model sees the photo as developed, uncropped and without masks, at 1008 px — the frame
|
||||
//! mask coordinates live in — so a stored segmentation lines up with the photo at any size.
|
||||
//! Everything that touches the model (loading it, rendering and encoding the photo, clicks,
|
||||
//! descriptions, detail passes) runs on one worker thread ([`worker`]); the session talks to it
|
||||
//! over channels and never waits on a lock. With [`Segmenter::background`] (the desktop app)
|
||||
//! commands return at once and [`Session::segment_poll`] applies the results; without it (CLI,
|
||||
//! MCP, tests) they wait for the worker.
|
||||
|
||||
#[cfg(feature = "sam")]
|
||||
mod download;
|
||||
#[cfg(feature = "sam")]
|
||||
mod worker;
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use lightcraft_catalog::PhotoId;
|
||||
use lightcraft_develop::{MaskShape, SegMask};
|
||||
use lightcraft_geom::Point;
|
||||
|
||||
use crate::Session;
|
||||
|
||||
/// Long edge of the image the model sees.
|
||||
pub const INPUT_EDGE: usize = 1008;
|
||||
/// Download size of the model (`model.safetensors`; the tokenizer files add ~2 MB).
|
||||
pub const MODEL_BYTES: u64 = 3_439_938_512;
|
||||
/// The model's licence (not LightCraft's): shown before downloading.
|
||||
pub const LICENSE_NAME: &str = "SAM License (Meta)";
|
||||
pub const LICENSE_URL: &str = "https://github.com/facebookresearch/sam3/blob/main/LICENSE";
|
||||
/// How errors about a missing model start (the UI offers the download on it).
|
||||
pub const NOT_INSTALLED: &str = "The SAM 3 model is not installed";
|
||||
/// Longest a waiting command (CLI, MCP) waits for the model (loading on a CPU can take minutes).
|
||||
#[cfg_attr(not(feature = "sam"), allow(dead_code))]
|
||||
const WAIT: std::time::Duration = std::time::Duration::from_secs(15 * 60);
|
||||
/// Most clicks one Object selection takes (the model's prompt grows with each).
|
||||
pub const MAX_CLICKS: usize = 64;
|
||||
|
||||
/// One click of an Object selection (normalized image coordinates).
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Click {
|
||||
pub at: Point,
|
||||
pub include: bool,
|
||||
}
|
||||
|
||||
/// What a request's result is for (how [`Session::segment_poll`] applies it).
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
#[cfg_attr(not(feature = "sam"), allow(dead_code))]
|
||||
pub(crate) enum Tag {
|
||||
/// A command waiting for the result itself.
|
||||
Wait,
|
||||
/// Load the model and encode the photo ahead of the first click.
|
||||
Prepare,
|
||||
/// Object clicks on component `comp` of mask `mask`.
|
||||
Clicks { photo: PhotoId, mask: u32, comp: usize, hint: Vec<Point>, exclude: Vec<Point> },
|
||||
/// A Describe selection: a new mask (`mask: None`) or a component combined by `op`.
|
||||
NewPrompt { photo: PhotoId, mask: Option<u32>, op: String, name: Option<String>, text: String },
|
||||
/// A zoomed-in pass for a component, computed from `shape` (without its detail).
|
||||
Detail { photo: PhotoId, mask: u32, comp: usize, shape: MaskShape },
|
||||
}
|
||||
|
||||
/// The model input: 8-bit RGB.
|
||||
#[cfg_attr(not(feature = "sam"), allow(dead_code))]
|
||||
pub(crate) struct Input {
|
||||
rgb: Vec<u8>,
|
||||
w: usize,
|
||||
h: usize,
|
||||
}
|
||||
|
||||
/// What a detail pass re-runs.
|
||||
#[cfg_attr(not(feature = "sam"), allow(dead_code))]
|
||||
pub(crate) enum Prompt {
|
||||
Clicks(Vec<Click>),
|
||||
Text(String),
|
||||
}
|
||||
|
||||
/// Object clicks sent to the worker and not applied yet (background mode): further clicks add
|
||||
/// to these, and the loupe shows them.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct PendingClicks {
|
||||
pub photo: PhotoId,
|
||||
pub mask: u32,
|
||||
pub comp: usize,
|
||||
pub hint: Vec<Point>,
|
||||
pub exclude: Vec<Point>,
|
||||
}
|
||||
|
||||
/// What [`Session::segment_poll`] did.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Polled {
|
||||
/// A mask changed.
|
||||
pub changed: bool,
|
||||
/// Errors and notices from background work, for the user.
|
||||
pub messages: Vec<String>,
|
||||
/// A mask that got a new selection (worth a detail pass).
|
||||
pub refine: Option<u32>,
|
||||
}
|
||||
|
||||
/// The model download (`segment.model.*`).
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
||||
pub struct DownloadStatus {
|
||||
pub running: bool,
|
||||
pub done: u64,
|
||||
pub total: u64,
|
||||
pub file: String,
|
||||
pub error: Option<String>,
|
||||
pub finished: bool,
|
||||
}
|
||||
|
||||
/// The model (on its worker thread), the download, and AI mask requests in flight.
|
||||
pub struct Segmenter {
|
||||
/// Where the `facebook/sam3` files live (set by the app; `None`: no AI masks).
|
||||
pub dir: Option<PathBuf>,
|
||||
/// The user's list of download mirrors (one base URL per line), if any.
|
||||
pub mirrors_file: Option<PathBuf>,
|
||||
/// Run requests in the background and apply them in [`Session::segment_poll`] (the
|
||||
/// desktop app); otherwise commands wait for their result (CLI, MCP, tests).
|
||||
pub background: bool,
|
||||
pending: Option<PendingClicks>,
|
||||
messages: Vec<String>,
|
||||
#[cfg(feature = "sam")]
|
||||
worker: worker::Worker,
|
||||
#[cfg(feature = "sam")]
|
||||
download: download::Downloader,
|
||||
#[cfg(feature = "sam")]
|
||||
results: (std::sync::mpsc::Sender<worker::Outcome>, std::sync::mpsc::Receiver<worker::Outcome>),
|
||||
}
|
||||
|
||||
// (derivable only without the `sam` feature: the results channel has no Default)
|
||||
#[allow(clippy::derivable_impls)]
|
||||
impl Default for Segmenter {
|
||||
fn default() -> Self {
|
||||
Segmenter {
|
||||
dir: None,
|
||||
mirrors_file: None,
|
||||
background: false,
|
||||
pending: None,
|
||||
messages: Vec::new(),
|
||||
#[cfg(feature = "sam")]
|
||||
worker: worker::Worker::default(),
|
||||
#[cfg(feature = "sam")]
|
||||
download: download::Downloader::default(),
|
||||
#[cfg(feature = "sam")]
|
||||
results: std::sync::mpsc::channel(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Segmenter {
|
||||
/// Whether this build can compute AI masks at all.
|
||||
pub const AVAILABLE: bool = cfg!(feature = "sam");
|
||||
|
||||
/// Whether the model's files are in place.
|
||||
pub fn installed(&self) -> bool {
|
||||
#[cfg(feature = "sam")]
|
||||
if let Some(dir) = &self.dir {
|
||||
return lightcraft_segment::is_model_dir(dir);
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Whether requests are queued or running (loading, analyzing the photo, a click…).
|
||||
pub fn busy(&self) -> bool {
|
||||
#[cfg(feature = "sam")]
|
||||
return self.worker.pending() > 0;
|
||||
#[cfg(not(feature = "sam"))]
|
||||
false
|
||||
}
|
||||
|
||||
/// Whether the slow part runs now: loading the model or analyzing a photo.
|
||||
pub fn analyzing(&self) -> bool {
|
||||
#[cfg(feature = "sam")]
|
||||
return self.worker.encoding();
|
||||
#[cfg(not(feature = "sam"))]
|
||||
false
|
||||
}
|
||||
|
||||
/// Whether a zoomed-in detail pass is queued or running.
|
||||
pub fn detail_busy(&self) -> bool {
|
||||
#[cfg(feature = "sam")]
|
||||
return self.worker.detail() > 0;
|
||||
#[cfg(not(feature = "sam"))]
|
||||
false
|
||||
}
|
||||
|
||||
/// Whether the model is in memory (it is unloaded after a while without use).
|
||||
pub fn loaded(&self) -> bool {
|
||||
#[cfg(feature = "sam")]
|
||||
return self.worker.loaded();
|
||||
#[cfg(not(feature = "sam"))]
|
||||
false
|
||||
}
|
||||
|
||||
/// Clicks waiting for their segmentation.
|
||||
pub fn pending_clicks(&self) -> Option<&PendingClicks> {
|
||||
self.pending.as_ref()
|
||||
}
|
||||
|
||||
/// The model download's state.
|
||||
pub fn download_status(&self) -> DownloadStatus {
|
||||
#[cfg(feature = "sam")]
|
||||
{
|
||||
let s = self.download.status();
|
||||
DownloadStatus { running: s.running, done: s.done, total: s.total, file: s.file, error: s.error, finished: s.finished }
|
||||
}
|
||||
#[cfg(not(feature = "sam"))]
|
||||
DownloadStatus::default()
|
||||
}
|
||||
|
||||
/// Where the model is downloaded from, in order (the user's mirrors first).
|
||||
pub fn mirrors(&self) -> Vec<String> {
|
||||
#[cfg(feature = "sam")]
|
||||
{
|
||||
let env = std::env::var(lightcraft_segment::fetch::MIRRORS_ENV).ok();
|
||||
lightcraft_segment::fetch::mirrors(env.as_deref(), self.mirrors_file.as_deref())
|
||||
}
|
||||
#[cfg(not(feature = "sam"))]
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// The model folder, if the model is there; otherwise an error that says what to do.
|
||||
pub fn model_dir(&self) -> Result<PathBuf, String> {
|
||||
if !Self::AVAILABLE {
|
||||
return Err("AI masks are not available in this build".into());
|
||||
}
|
||||
let dir = self.dir.clone().ok_or("no folder is set for the SAM 3 model")?;
|
||||
if self.installed() {
|
||||
return Ok(dir);
|
||||
}
|
||||
let d = self.download_status();
|
||||
if d.running {
|
||||
let pct = d.done.saturating_mul(100).checked_div(d.total).unwrap_or(0);
|
||||
return Err(format!("{NOT_INSTALLED} yet: it is downloading ({pct} %)."));
|
||||
}
|
||||
Err(format!(
|
||||
"{NOT_INSTALLED}. Download it (about {:.1} GB, {LICENSE_NAME}) when LightCraft offers it, with `segment.model.download {{\"acknowledged\": true}}`, or put model.safetensors, vocab.json and merges.txt in {}.",
|
||||
MODEL_BYTES as f64 / 1e9,
|
||||
dir.display()
|
||||
))
|
||||
}
|
||||
|
||||
/// Start downloading the model (on a background thread; see `segment.model.download`).
|
||||
/// `Ok(false)` when it is installed or downloading already.
|
||||
pub fn start_download(&self) -> Result<bool, String> {
|
||||
if !Self::AVAILABLE {
|
||||
return Err("AI masks are not available in this build".into());
|
||||
}
|
||||
let dir = self.dir.clone().ok_or("no folder is set for the SAM 3 model")?;
|
||||
if self.installed() {
|
||||
return Ok(false);
|
||||
}
|
||||
#[cfg(feature = "sam")]
|
||||
{
|
||||
let mirrors = self.mirrors();
|
||||
if mirrors.is_empty() {
|
||||
return Err(lightcraft_segment::fetch::DownloadError::NoMirrors.to_string());
|
||||
}
|
||||
self.download.start(lightcraft_segment::fetch::SAM3_FILES, mirrors, dir, lightcraft_segment::fetch::Options::default())
|
||||
}
|
||||
#[cfg(not(feature = "sam"))]
|
||||
{
|
||||
let _ = dir;
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop a running download (its partial files stay, to resume). False when none runs.
|
||||
pub fn cancel_download(&self) -> bool {
|
||||
#[cfg(feature = "sam")]
|
||||
return self.download.cancel();
|
||||
#[cfg(not(feature = "sam"))]
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Mix `v` into a 64-bit hash.
|
||||
#[cfg_attr(not(feature = "sam"), allow(dead_code))]
|
||||
fn mix(h: u64, v: u64) -> u64 {
|
||||
(h ^ v).wrapping_mul(0x0100_0000_01b3).rotate_left(29)
|
||||
}
|
||||
|
||||
/// Objects covering more than this much of the photo gain little from a zoomed pass.
|
||||
const DETAIL_MAX_AREA: f64 = 0.4;
|
||||
|
||||
/// The part of the photo to zoom into for `seg`'s selection: its bounding box with a margin of
|
||||
/// a quarter of its size, at least 12 % of the long edge, at most 2:1, inside the photo
|
||||
/// (`aspect` = width / height). `None` when nothing is selected or it is most of the photo.
|
||||
pub fn detail_region(seg: &SegMask, aspect: f64) -> Option<[f64; 4]> {
|
||||
let logits = seg.logits()?;
|
||||
let r = seg.bounds()?;
|
||||
let side = seg.side as usize;
|
||||
let (mut x0, mut y0, mut x1, mut y1) = (usize::MAX, usize::MAX, 0, 0);
|
||||
for (i, l) in logits.iter().enumerate() {
|
||||
if *l > 0.0 {
|
||||
let (x, y) = (i % side, i / side);
|
||||
(x0, y0, x1, y1) = (x0.min(x), y0.min(y), x1.max(x + 1), y1.max(y + 1));
|
||||
}
|
||||
}
|
||||
if x0 == usize::MAX {
|
||||
return None;
|
||||
}
|
||||
let (rw, rh) = (r[2] - r[0], r[3] - r[1]);
|
||||
let nx = |c: usize| r[0] + c as f64 / side as f64 * rw;
|
||||
let ny = |c: usize| r[1] + c as f64 / side as f64 * rh;
|
||||
let (bx0, by0, bx1, by1) = (nx(x0), ny(y0), nx(x1), ny(y1));
|
||||
if (bx1 - bx0) * (by1 - by0) > DETAIL_MAX_AREA {
|
||||
return None;
|
||||
}
|
||||
// in units of the photo's height: x × aspect
|
||||
let a = if aspect.is_finite() { aspect.clamp(0.1, 10.0) } else { 1.0 };
|
||||
let (cx, cy) = ((bx0 + bx1) / 2.0 * a, (by0 + by1) / 2.0);
|
||||
let size = ((bx1 - bx0) * a).max(by1 - by0);
|
||||
let m = 0.25 * size;
|
||||
let mut w = ((bx1 - bx0) * a + 2.0 * m).max(0.12 * a.max(1.0));
|
||||
let mut h = ((by1 - by0) + 2.0 * m).max(0.12 * a.max(1.0));
|
||||
w = w.max(h / 2.0).min(a);
|
||||
h = h.max(w / 2.0).min(1.0);
|
||||
let x = (cx - w / 2.0).clamp(0.0, (a - w).max(0.0));
|
||||
let y = (cy - h / 2.0).clamp(0.0, (1.0 - h).max(0.0));
|
||||
let out = [x / a, y, (x + w) / a, y + h];
|
||||
// a region nearly the whole photo isn't worth a second pass
|
||||
((out[2] - out[0]) * (out[3] - out[1]) < 0.8).then_some(out)
|
||||
}
|
||||
|
||||
/// `shape` without its detail patches (what a detail pass was computed from)
|
||||
#[cfg_attr(not(feature = "sam"), allow(dead_code))]
|
||||
fn strip_detail(mut shape: MaskShape) -> MaskShape {
|
||||
// (nor its edge setting: moving the slider meanwhile keeps the pass)
|
||||
if let MaskShape::Object { detail, edge, .. } | MaskShape::Prompt { detail, edge, .. } = &mut shape {
|
||||
detail.clear();
|
||||
*edge = 0.0;
|
||||
}
|
||||
shape
|
||||
}
|
||||
|
||||
fn clicks_of(include: &[Point], exclude: &[Point]) -> Vec<Click> {
|
||||
include.iter().map(|p| Click { at: *p, include: true }).chain(exclude.iter().map(|p| Click { at: *p, include: false })).collect()
|
||||
}
|
||||
|
||||
impl Session {
|
||||
/// The key and settings of photo `id` as the model sees it: its look without masks, uncropped.
|
||||
#[cfg_attr(not(feature = "sam"), allow(dead_code))]
|
||||
fn segment_key(&self, id: PhotoId) -> Option<(u64, lightcraft_develop::DevelopSettings)> {
|
||||
let p = self.catalog.photo(id)?;
|
||||
let mut d = (*p.develop).clone();
|
||||
d.masks.clear();
|
||||
let content = crate::media::content_key(p);
|
||||
let mut h = mix(0xcbf2_9ce4_8422_2325, d.hash64());
|
||||
h = mix(h, id.0);
|
||||
for b in content.bytes() {
|
||||
h = mix(h, u64::from(b));
|
||||
}
|
||||
Some((h, d))
|
||||
}
|
||||
|
||||
/// Clicks (in normalized photo coordinates) for the model.
|
||||
#[cfg(feature = "sam")]
|
||||
fn model_clicks(clicks: &[Click]) -> Vec<lightcraft_segment::Click> {
|
||||
clicks.iter().map(|c| lightcraft_segment::Click { x: c.at.x as f32, y: c.at.y as f32, positive: c.include }).collect()
|
||||
}
|
||||
|
||||
/// A worker request about photo `id` (the render job for its model input goes along; the
|
||||
/// worker runs it only when its cache doesn't hold the photo). `region` renders that part
|
||||
/// of the photo, from a source sharp enough for it (detail passes).
|
||||
#[cfg(feature = "sam")]
|
||||
fn segment_job(
|
||||
&mut self,
|
||||
id: PhotoId,
|
||||
kind: worker::Kind,
|
||||
tag: Tag,
|
||||
reply: std::sync::mpsc::Sender<worker::Outcome>,
|
||||
region: Option<[f64; 4]>,
|
||||
) -> Result<worker::Job, String> {
|
||||
let dir = self.segmenter.model_dir()?;
|
||||
let (key, mut d) = self.segment_key(id).ok_or("no such photo")?;
|
||||
let p = self.catalog.photo(id).ok_or("no such photo")?.clone();
|
||||
let missing = match &p.source {
|
||||
lightcraft_catalog::Source::File { path } if !std::path::Path::new(path).exists() => Some(path.clone()),
|
||||
_ => None,
|
||||
};
|
||||
let render = match region {
|
||||
None => self.preview_job(id, INPUT_EDGE, INPUT_EDGE, false, &d).ok_or("no such photo")?,
|
||||
Some(region) => {
|
||||
d.crop.geometry =
|
||||
lightcraft_geom::CropGeometry { rect: lightcraft_geom::Rect::new(region[0], region[1], region[2], region[3]), angle: 0.0 };
|
||||
d.crop.flip_h = false;
|
||||
d.crop.flip_v = false;
|
||||
d.geometry.constrain_crop = false;
|
||||
let mut job = self.preview_job(id, INPUT_EDGE, INPUT_EDGE, true, &d).ok_or("no such photo")?;
|
||||
let frac = (region[2] - region[0]).max(region[3] - region[1]).max(0.01);
|
||||
job.level = crate::media::SourceLevel::for_size((INPUT_EDGE as f64 / frac) as usize);
|
||||
job.source = self.media.source_ref(&p, job.level);
|
||||
job
|
||||
}
|
||||
};
|
||||
Ok(worker::Job { dir, key: if region.is_some() { 0 } else { key }, render: Some(render), missing, kind, tag, reply })
|
||||
}
|
||||
|
||||
/// Send a request and wait for its result (CLI, MCP, tests; or when not in background mode).
|
||||
#[cfg(feature = "sam")]
|
||||
fn segment_wait(&mut self, id: PhotoId, kind: worker::Kind, region: Option<[f64; 4]>) -> Result<Option<SegMask>, String> {
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let job = self.segment_job(id, kind, Tag::Wait, tx, region)?;
|
||||
self.segmenter.worker.submit(job)?;
|
||||
match rx.recv_timeout(WAIT) {
|
||||
Ok(o) => o.result,
|
||||
Err(_) => Err("the AI model took too long; try again".into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Send a request whose result [`Session::segment_poll`] applies.
|
||||
#[cfg(feature = "sam")]
|
||||
fn segment_queue(&mut self, id: PhotoId, kind: worker::Kind, tag: Tag, region: Option<[f64; 4]>) -> Result<(), String> {
|
||||
let reply = self.segmenter.results.0.clone();
|
||||
let job = self.segment_job(id, kind, tag, reply, region)?;
|
||||
self.segmenter.worker.submit(job)
|
||||
}
|
||||
|
||||
/// Start preparing photo `id` for AI masks (load the model, encode the image), so the first
|
||||
/// click is fast. In background mode it returns at once.
|
||||
pub fn segment_prepare(&mut self, id: PhotoId) -> Result<(), String> {
|
||||
self.segmenter.model_dir()?;
|
||||
#[cfg(feature = "sam")]
|
||||
{
|
||||
if self.segmenter.background {
|
||||
return self.segment_queue(id, worker::Kind::Prepare, Tag::Prepare, None);
|
||||
}
|
||||
self.segment_wait(id, worker::Kind::Prepare, None)?;
|
||||
}
|
||||
#[cfg(not(feature = "sam"))]
|
||||
let _ = id;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Segment the object `clicks` pick on photo `id`, waiting for the model.
|
||||
pub fn segment_clicks(&mut self, id: PhotoId, clicks: &[Click]) -> Result<SegMask, String> {
|
||||
self.segmenter.model_dir()?;
|
||||
if !clicks.iter().any(|c| c.include) {
|
||||
return Err("click the object to include it first".into());
|
||||
}
|
||||
#[cfg(feature = "sam")]
|
||||
{
|
||||
self.segment_wait(id, worker::Kind::Clicks(Self::model_clicks(clicks)), None)?.ok_or_else(|| "nothing was selected".to_string())
|
||||
}
|
||||
#[cfg(not(feature = "sam"))]
|
||||
{
|
||||
let _ = id;
|
||||
Err("AI masks are not available in this build".into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Segment everything `text` describes on photo `id`, waiting for the model; `Ok(None)`
|
||||
/// when nothing matches.
|
||||
pub fn segment_text(&mut self, id: PhotoId, text: &str) -> Result<Option<SegMask>, String> {
|
||||
self.segmenter.model_dir()?;
|
||||
let text = text.trim();
|
||||
if text.is_empty() {
|
||||
return Err("describe what to select".into());
|
||||
}
|
||||
#[cfg(feature = "sam")]
|
||||
{
|
||||
self.segment_wait(id, worker::Kind::Text(text.to_string()), None)
|
||||
}
|
||||
#[cfg(not(feature = "sam"))]
|
||||
{
|
||||
let _ = id;
|
||||
Err("AI masks are not available in this build".into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Background mode: queue Object clicks for component `comp` of mask `mask`; the result
|
||||
/// (clicks and segmentation, one undo step) is applied by [`Session::segment_poll`].
|
||||
pub(crate) fn segment_clicks_later(&mut self, id: PhotoId, mask: u32, comp: usize, hint: Vec<Point>, exclude: Vec<Point>) -> Result<(), String> {
|
||||
self.segmenter.model_dir()?;
|
||||
if hint.is_empty() {
|
||||
return Err("click the object to include it first".into());
|
||||
}
|
||||
#[cfg(feature = "sam")]
|
||||
{
|
||||
let clicks = Self::model_clicks(&clicks_of(&hint, &exclude));
|
||||
let tag = Tag::Clicks { photo: id, mask, comp, hint: hint.clone(), exclude: exclude.clone() };
|
||||
self.segment_queue(id, worker::Kind::Clicks(clicks), tag, None)?;
|
||||
self.segmenter.pending = Some(PendingClicks { photo: id, mask, comp, hint, exclude });
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(not(feature = "sam"))]
|
||||
{
|
||||
let _ = (id, mask, comp, exclude);
|
||||
Err("AI masks are not available in this build".into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Background mode: queue a Describe selection; the mask (or component, combined by `op`
|
||||
/// into mask `mask`) is created by [`Session::segment_poll`] when something matches.
|
||||
pub(crate) fn segment_text_later(&mut self, id: PhotoId, mask: Option<u32>, op: &str, name: Option<String>, text: &str) -> Result<(), String> {
|
||||
self.segmenter.model_dir()?;
|
||||
let text = text.trim();
|
||||
if text.is_empty() {
|
||||
return Err("describe what to select".into());
|
||||
}
|
||||
#[cfg(feature = "sam")]
|
||||
{
|
||||
let tag = Tag::NewPrompt { photo: id, mask, op: op.to_string(), name, text: text.to_string() };
|
||||
self.segment_queue(id, worker::Kind::Text(text.to_string()), tag, None)
|
||||
}
|
||||
#[cfg(not(feature = "sam"))]
|
||||
{
|
||||
let _ = (id, mask, op, name);
|
||||
Err("AI masks are not available in this build".into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Start a zoomed-in pass for component `comp` of mask `mask` on photo `id` (an Object or
|
||||
/// Describe selection): the photo around the selection is rendered at a higher resolution
|
||||
/// and segmented again. In background mode [`Session::segment_poll`] applies the result;
|
||||
/// otherwise it is applied before this returns. `Ok(false)` when there is nothing to
|
||||
/// refine (no selection yet, or it is most of the photo) or a pass is running already.
|
||||
pub fn segment_detail(&mut self, id: PhotoId, mask: u32, comp: usize) -> Result<bool, String> {
|
||||
self.segmenter.model_dir()?;
|
||||
let d = self.develop_of(id).ok_or("no such photo")?;
|
||||
let shape = d.masks.iter().find(|m| m.id == mask).and_then(|m| m.components.get(comp)).map(|c| c.shape.clone()).ok_or("no such mask")?;
|
||||
let (seg, prompt) = match &shape {
|
||||
// (a document with absurdly many clicks isn't run through the model)
|
||||
MaskShape::Object { hint, exclude, seg: Some(seg), .. } if !hint.is_empty() && hint.len() + exclude.len() <= MAX_CLICKS => {
|
||||
(seg.clone(), Prompt::Clicks(clicks_of(hint, exclude)))
|
||||
}
|
||||
MaskShape::Prompt { text, seg: Some(seg), .. } => (seg.clone(), Prompt::Text(text.clone())),
|
||||
_ => return Ok(false),
|
||||
};
|
||||
let p = self.catalog.photo(id).ok_or("no such photo")?;
|
||||
let (pw, ph) = (f64::from(p.width.max(1)), f64::from(p.height.max(1)));
|
||||
let aspect = if d.orientation.swaps_axes() { ph / pw } else { pw / ph };
|
||||
let Some(region) = detail_region(&seg, aspect) else { return Ok(false) };
|
||||
#[cfg(feature = "sam")]
|
||||
{
|
||||
let kind = worker::Kind::Detail { prompt, region };
|
||||
let tag = Tag::Detail { photo: id, mask, comp, shape: strip_detail(shape) };
|
||||
if self.segmenter.background {
|
||||
if self.segmenter.detail_busy() {
|
||||
return Ok(false);
|
||||
}
|
||||
self.segment_queue(id, kind, tag, Some(region))?;
|
||||
} else {
|
||||
let patch = self.segment_wait(id, kind, Some(region))?;
|
||||
let outcome = worker::Outcome { tag, result: Ok(patch), superseded: false };
|
||||
let mut polled = Polled::default();
|
||||
self.apply_outcome(outcome, &mut polled);
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
#[cfg(not(feature = "sam"))]
|
||||
{
|
||||
let _ = (prompt, region, mask);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply finished background requests (to masks that haven't changed since) and collect
|
||||
/// their messages. Cheap when nothing is pending: call it every frame.
|
||||
pub fn segment_poll(&mut self) -> Polled {
|
||||
#[cfg_attr(not(feature = "sam"), allow(unused_mut))]
|
||||
let mut polled = Polled { messages: std::mem::take(&mut self.segmenter.messages), ..Default::default() };
|
||||
#[cfg(feature = "sam")]
|
||||
{
|
||||
// (before draining: a worker that is idle now has sent everything)
|
||||
let idle = !self.segmenter.busy();
|
||||
let done: Vec<worker::Outcome> = self.segmenter.results.1.try_iter().collect();
|
||||
for o in done {
|
||||
self.apply_outcome(o, &mut polled);
|
||||
}
|
||||
// clicks whose request was lost (the worker restarted) aren't pending forever
|
||||
if idle {
|
||||
self.segmenter.pending = None;
|
||||
}
|
||||
}
|
||||
polled
|
||||
}
|
||||
|
||||
#[cfg(feature = "sam")]
|
||||
fn apply_outcome(&mut self, o: worker::Outcome, polled: &mut Polled) {
|
||||
if o.superseded {
|
||||
return;
|
||||
}
|
||||
match o.tag {
|
||||
Tag::Wait => {}
|
||||
Tag::Prepare => {
|
||||
if let Err(e) = o.result {
|
||||
polled.messages.push(e);
|
||||
}
|
||||
}
|
||||
Tag::Clicks { photo, mask, comp, hint, exclude } => {
|
||||
let latest = PendingClicks { photo, mask, comp, hint, exclude };
|
||||
if self.segmenter.pending.as_ref() != Some(&latest) {
|
||||
return; // newer clicks are on their way
|
||||
}
|
||||
self.segmenter.pending = None;
|
||||
let seg = match o.result {
|
||||
Ok(Some(seg)) => seg,
|
||||
Ok(None) => return,
|
||||
Err(e) => return polled.messages.push(e),
|
||||
};
|
||||
let Some(d) = self.develop_of(photo) else { return };
|
||||
let mut d = (*d).clone();
|
||||
let Some(c) = d.masks.iter_mut().find(|m| m.id == mask).and_then(|m| m.components.get_mut(comp)) else { return };
|
||||
let MaskShape::Object { edge, .. } = c.shape else { return };
|
||||
c.shape = MaskShape::Object { hint: latest.hint, exclude: latest.exclude, seg: Some(seg), detail: vec![], edge };
|
||||
match self.set_develop(photo, d, "Object Mask") {
|
||||
Ok(()) => {
|
||||
polled.changed = true;
|
||||
polled.refine = Some(mask);
|
||||
}
|
||||
Err(e) => polled.messages.push(e.to_string()),
|
||||
}
|
||||
}
|
||||
Tag::NewPrompt { photo, mask, op, name, text } => {
|
||||
let seg = match o.result {
|
||||
Ok(Some(seg)) => seg,
|
||||
Ok(None) => return polled.messages.push(format!("Nothing matching “{text}” was found in this photo.")),
|
||||
Err(e) => return polled.messages.push(e),
|
||||
};
|
||||
if self.active() != Some(photo) {
|
||||
return polled.messages.push(format!("The photo changed before “{text}” was found; try again."));
|
||||
}
|
||||
let Ok(seg) = serde_json::to_value(seg) else { return };
|
||||
let mut p = serde_json::json!({"kind": "prompt", "text": text, "seg": seg});
|
||||
if let Some(n) = name {
|
||||
p["name"] = serde_json::json!(n);
|
||||
}
|
||||
let r = match mask {
|
||||
None => self.execute("mask.add", &p),
|
||||
Some(m) => {
|
||||
if !self.develop_of(photo).is_some_and(|d| d.masks.iter().any(|x| x.id == m)) {
|
||||
return polled.messages.push("The mask was removed before the selection was found.".into());
|
||||
}
|
||||
self.active_mask = Some(m);
|
||||
p["op"] = serde_json::json!(op);
|
||||
self.execute("mask.addComponent", &p)
|
||||
}
|
||||
};
|
||||
match r {
|
||||
Ok(_) => {
|
||||
polled.changed = true;
|
||||
polled.refine = self.active_mask;
|
||||
}
|
||||
Err(e) => polled.messages.push(e.to_string()),
|
||||
}
|
||||
}
|
||||
Tag::Detail { photo, mask, comp, shape } => {
|
||||
let patch = match o.result {
|
||||
Ok(Some(p)) => p,
|
||||
Ok(None) => return,
|
||||
Err(e) => return log::warn!("SAM 3 detail pass: {e}"),
|
||||
};
|
||||
let Some(d) = self.develop_of(photo) else { return };
|
||||
let mut d = (*d).clone();
|
||||
let Some(c) = d.masks.iter_mut().find(|m| m.id == mask).and_then(|m| m.components.get_mut(comp)) else { return };
|
||||
if strip_detail(c.shape.clone()) != shape {
|
||||
return; // clicked again meanwhile: a newer pass will follow
|
||||
}
|
||||
match &mut c.shape {
|
||||
MaskShape::Object { detail, .. } | MaskShape::Prompt { detail, .. } => *detail = vec![patch],
|
||||
_ => return,
|
||||
}
|
||||
match self.set_develop(photo, d, "AI Mask Detail") {
|
||||
Ok(()) => polled.changed = true,
|
||||
Err(e) => log::warn!("AI mask detail: {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn seg_box(side: usize, x0: usize, y0: usize, x1: usize, y1: usize) -> SegMask {
|
||||
let l: Vec<f32> =
|
||||
(0..side * side).map(|i| if (x0..x1).contains(&(i % side)) && (y0..y1).contains(&(i / side)) { 8.0 } else { -8.0 }).collect();
|
||||
SegMask::from_logits(side, &l)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detail_region_frames_the_selection_with_a_margin() {
|
||||
// a small object in the middle of a square photo
|
||||
let r = detail_region(&seg_box(100, 40, 45, 50, 55), 1.0).unwrap();
|
||||
assert!(r[0] < 0.4 && r[2] > 0.5 && r[1] < 0.45 && r[3] > 0.55, "contains it: {r:?}");
|
||||
assert!(r[2] - r[0] < 0.25 && r[3] - r[1] < 0.25, "zoomed in: {r:?}");
|
||||
assert!(r.iter().all(|v| (0.0..=1.0).contains(v)));
|
||||
// at the corner: shifted inside the photo, not cut
|
||||
let r = detail_region(&seg_box(100, 0, 0, 5, 5), 1.5).unwrap();
|
||||
assert!(r[0] >= 0.0 && r[1] >= 0.0 && r[0] < 1e-9 && r[1] < 1e-9);
|
||||
// a thin object stays at most 2:1 (in photo pixels, 1.5:1 photo)
|
||||
let r = detail_region(&seg_box(100, 10, 50, 90, 52), 1.5).unwrap();
|
||||
let (w, h) = ((r[2] - r[0]) * 1.5, r[3] - r[1]);
|
||||
assert!(w / h <= 2.0 + 1e-9, "{w} × {h}");
|
||||
// a hostile aspect ratio stays sane
|
||||
assert!(detail_region(&seg_box(100, 40, 45, 50, 55), f64::NAN).is_some_and(|r| r.iter().all(|v| v.is_finite())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_region_for_nothing_or_most_of_the_photo() {
|
||||
assert!(detail_region(&seg_box(50, 0, 0, 0, 0), 1.0).is_none());
|
||||
assert!(detail_region(&seg_box(50, 2, 2, 48, 48), 1.0).is_none());
|
||||
let damaged = SegMask { side: 50, data: "?".into(), rect: None };
|
||||
assert!(detail_region(&damaged, 1.0).is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,426 @@
|
||||
//! The SAM 3 worker thread: it owns the model and the last encoded photo, and runs every
|
||||
//! request (load, render the model input, encode, clicks, descriptions, detail passes) in
|
||||
//! turn. The session only sends jobs and receives outcomes over channels, so no thread ever
|
||||
//! waits on a lock the model holds, and the UI thread never runs the model.
|
||||
//!
|
||||
//! Every job runs under `catch_unwind`: a panic inside candle becomes an error outcome, drops
|
||||
//! the (possibly inconsistent) model and leaves the worker ready for the next job. The busy
|
||||
//! counters are decremented by drop guards, so they can't stay stuck. The model is unloaded
|
||||
//! after [`IDLE_UNLOAD`] without requests (it holds several GB).
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::panic::AssertUnwindSafe;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::mpsc::{self, RecvTimeoutError};
|
||||
use std::time::Duration;
|
||||
|
||||
use lightcraft_develop::SegMask;
|
||||
use lightcraft_segment::{Encoded, Sam3};
|
||||
|
||||
use super::{Input, Prompt, Tag};
|
||||
use crate::media::RenderJob;
|
||||
|
||||
/// Unload the model after this long without a request.
|
||||
pub const IDLE_UNLOAD: Duration = Duration::from_secs(10 * 60);
|
||||
|
||||
/// What to compute.
|
||||
pub enum Kind {
|
||||
/// Load the model and encode the photo (so the first click is fast).
|
||||
Prepare,
|
||||
Clicks(Vec<lightcraft_segment::Click>),
|
||||
Text(String),
|
||||
/// A zoomed-in pass over `region` (normalized) of the photo, rendered by the job's `render`.
|
||||
Detail {
|
||||
prompt: Prompt,
|
||||
region: [f64; 4],
|
||||
},
|
||||
/// Panics (tests of the worker's recovery).
|
||||
#[cfg(test)]
|
||||
Panic,
|
||||
}
|
||||
|
||||
/// One request.
|
||||
pub struct Job {
|
||||
/// The model folder.
|
||||
pub dir: PathBuf,
|
||||
/// Which photo and look the request is about (the encode cache's key).
|
||||
pub key: u64,
|
||||
/// Renders the model input when the cache doesn't hold `key` (always for a detail pass).
|
||||
pub render: Option<RenderJob>,
|
||||
/// The photo's file, when it is missing on disk (for a clearer error if rendering fails).
|
||||
pub missing: Option<String>,
|
||||
pub kind: Kind,
|
||||
pub tag: Tag,
|
||||
pub reply: mpsc::Sender<Outcome>,
|
||||
}
|
||||
|
||||
/// A finished request: the segmentation (`None`: nothing selected / nothing to report).
|
||||
pub struct Outcome {
|
||||
pub tag: Tag,
|
||||
pub result: Result<Option<SegMask>, String>,
|
||||
/// A newer request for the same selection came in; this one was skipped.
|
||||
pub superseded: bool,
|
||||
}
|
||||
|
||||
/// Counters the session reads without locking.
|
||||
#[derive(Default)]
|
||||
pub struct Shared {
|
||||
/// Requests other than detail passes, queued or running.
|
||||
pub pending: AtomicUsize,
|
||||
/// Detail passes queued or running.
|
||||
pub detail: AtomicUsize,
|
||||
/// The model is in memory.
|
||||
pub loaded: AtomicBool,
|
||||
/// Loading the model or encoding a photo right now (the slow parts).
|
||||
pub encoding: AtomicBool,
|
||||
}
|
||||
|
||||
/// Decrements a counter when dropped (also when unwinding).
|
||||
struct Done<'a>(&'a AtomicUsize);
|
||||
|
||||
impl Drop for Done<'_> {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.try_update(Ordering::SeqCst, Ordering::SeqCst, |n| Some(n.saturating_sub(1)));
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets a flag while alive.
|
||||
struct Flag<'a>(&'a AtomicBool);
|
||||
|
||||
impl<'a> Flag<'a> {
|
||||
fn raise(f: &'a AtomicBool) -> Self {
|
||||
f.store(true, Ordering::SeqCst);
|
||||
Flag(f)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Flag<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(false, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
/// The session's handle on the worker thread (started on the first request, restarted if it
|
||||
/// ever ends).
|
||||
#[derive(Default)]
|
||||
pub struct Worker {
|
||||
tx: Option<mpsc::Sender<Job>>,
|
||||
thread: Option<std::thread::JoinHandle<()>>,
|
||||
pub shared: Arc<Shared>,
|
||||
}
|
||||
|
||||
impl Worker {
|
||||
fn counter(&self, job: &Job) -> &AtomicUsize {
|
||||
if matches!(job.kind, Kind::Detail { .. }) { &self.shared.detail } else { &self.shared.pending }
|
||||
}
|
||||
|
||||
/// Whether the thread has ended (it never should); its counters are then meaningless.
|
||||
fn dead(&self) -> bool {
|
||||
self.thread.as_ref().is_some_and(|t| t.is_finished())
|
||||
}
|
||||
|
||||
pub fn pending(&self) -> usize {
|
||||
if self.dead() { 0 } else { self.shared.pending.load(Ordering::SeqCst) }
|
||||
}
|
||||
|
||||
pub fn detail(&self) -> usize {
|
||||
if self.dead() { 0 } else { self.shared.detail.load(Ordering::SeqCst) }
|
||||
}
|
||||
|
||||
pub fn encoding(&self) -> bool {
|
||||
!self.dead() && self.shared.encoding.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub fn loaded(&self) -> bool {
|
||||
!self.dead() && self.shared.loaded.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Queue `job`; returns at once.
|
||||
pub fn submit(&mut self, job: Job) -> Result<(), String> {
|
||||
if self.dead() {
|
||||
// a thread that ended took its counters' meaning with it
|
||||
self.tx = None;
|
||||
self.thread = None;
|
||||
self.shared = Arc::new(Shared::default());
|
||||
}
|
||||
self.counter(&job).fetch_add(1, Ordering::SeqCst);
|
||||
let job = match &self.tx {
|
||||
Some(tx) => match tx.send(job) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(mpsc::SendError(job)) => job,
|
||||
},
|
||||
None => job,
|
||||
};
|
||||
// (re)start the thread
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let shared = self.shared.clone();
|
||||
match std::thread::Builder::new().name("sam3".into()).spawn(move || run(&rx, &shared, IDLE_UNLOAD)) {
|
||||
Ok(t) => {
|
||||
self.thread = Some(t);
|
||||
self.tx = Some(tx.clone());
|
||||
tx.send(job).map_err(|_| "the AI mask worker stopped".to_string())
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = self.counter(&job).try_update(Ordering::SeqCst, Ordering::SeqCst, |n| Some(n.saturating_sub(1)));
|
||||
Err(format!("could not start the AI mask worker: {e}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The model and the last encoded photo.
|
||||
#[derive(Default)]
|
||||
struct State {
|
||||
model: Option<(PathBuf, Sam3)>,
|
||||
cache: Option<(u64, Encoded)>,
|
||||
}
|
||||
|
||||
/// Whether `later` makes `job` pointless (it asks for the same selection again).
|
||||
fn supersedes(later: &Job, job: &Job) -> bool {
|
||||
match (&later.tag, &job.tag) {
|
||||
(Tag::Prepare, Tag::Prepare) => later.key == job.key,
|
||||
(Tag::Clicks { photo: a, mask: am, comp: ac, .. }, Tag::Clicks { photo: b, mask: bm, comp: bc, .. }) => (a, am, ac) == (b, bm, bc),
|
||||
(Tag::Detail { photo: a, mask: am, comp: ac, .. }, Tag::Detail { photo: b, mask: bm, comp: bc, .. }) => (a, am, ac) == (b, bm, bc),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn run(rx: &mpsc::Receiver<Job>, shared: &Shared, idle: Duration) {
|
||||
let mut state = State::default();
|
||||
let mut queue: VecDeque<Job> = VecDeque::new();
|
||||
loop {
|
||||
if queue.is_empty() {
|
||||
match rx.recv_timeout(idle) {
|
||||
Ok(j) => queue.push_back(j),
|
||||
Err(RecvTimeoutError::Timeout) => {
|
||||
if state.model.is_some() {
|
||||
state = State::default();
|
||||
shared.loaded.store(false, Ordering::SeqCst);
|
||||
log::info!("SAM 3 unloaded after {} minutes without use", idle.as_secs() / 60);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(RecvTimeoutError::Disconnected) => return,
|
||||
}
|
||||
}
|
||||
while let Ok(j) = rx.try_recv() {
|
||||
queue.push_back(j);
|
||||
}
|
||||
let Some(mut job) = queue.pop_front() else { continue };
|
||||
let is_detail = matches!(job.kind, Kind::Detail { .. });
|
||||
let _done = Done(if is_detail { &shared.detail } else { &shared.pending });
|
||||
if queue.iter().any(|later| supersedes(later, &job)) {
|
||||
let _ = job.reply.send(Outcome { tag: job.tag, result: Ok(None), superseded: true });
|
||||
continue;
|
||||
}
|
||||
let result = match std::panic::catch_unwind(AssertUnwindSafe(|| run_job(&mut state, shared, &mut job))) {
|
||||
Ok(r) => r,
|
||||
Err(p) => {
|
||||
let why = p.downcast_ref::<&str>().map(|s| s.to_string()).or_else(|| p.downcast_ref::<String>().cloned()).unwrap_or_default();
|
||||
log::error!("SAM 3 panicked: {why}");
|
||||
// don't trust what the model kept: load it again next time
|
||||
state = State::default();
|
||||
Err("the AI model failed unexpectedly (it will be reloaded on the next try)".to_string())
|
||||
}
|
||||
};
|
||||
if result.is_err() && !is_detail {
|
||||
state.cache = None;
|
||||
}
|
||||
shared.loaded.store(state.model.is_some(), Ordering::SeqCst);
|
||||
let _ = job.reply.send(Outcome { tag: job.tag, result, superseded: false });
|
||||
}
|
||||
}
|
||||
|
||||
/// The model for `dir`, loading it when needed.
|
||||
fn model<'a>(state: &'a mut State, shared: &Shared, dir: &std::path::Path) -> Result<&'a mut Sam3, String> {
|
||||
if state.model.as_ref().is_none_or(|(d, _)| d != dir) {
|
||||
state.model = None;
|
||||
state.cache = None;
|
||||
let _busy = Flag::raise(&shared.encoding);
|
||||
let t = web_time::Instant::now();
|
||||
let m = Sam3::load(dir).map_err(|e| e.to_string())?;
|
||||
log::info!("SAM 3 loaded in {:?}", t.elapsed());
|
||||
state.model = Some((dir.to_path_buf(), m));
|
||||
shared.loaded.store(true, Ordering::SeqCst);
|
||||
}
|
||||
state.model.as_mut().map(|(_, m)| m).ok_or_else(|| "the model did not load".to_string())
|
||||
}
|
||||
|
||||
/// Render the model input.
|
||||
fn render(job: &mut Job) -> Result<Input, String> {
|
||||
let r = job.render.take().ok_or("the photo is not available")?.run();
|
||||
let img = match r.rendered {
|
||||
Ok(r) => r.image,
|
||||
Err(e) => {
|
||||
return Err(match &job.missing {
|
||||
Some(path) => format!(
|
||||
"This photo's file is missing (moved or deleted), so it can't be analyzed for AI masks: {path}. Library ▸ Find Missing Photos can relink it."
|
||||
),
|
||||
None => format!("couldn't render the photo for AI masks: {e}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
let rgb = img.data.iter().flat_map(|p| [p[0], p[1], p[2]]).collect();
|
||||
Ok(Input { rgb, w: img.width, h: img.height })
|
||||
}
|
||||
|
||||
/// The model and the encoded photo `job.key` (rendering and encoding it when needed).
|
||||
fn encoded<'a>(state: &'a mut State, shared: &Shared, job: &mut Job) -> Result<(&'a mut Sam3, &'a mut Encoded), String> {
|
||||
model(state, shared, &job.dir)?;
|
||||
if state.cache.as_ref().is_none_or(|(k, _)| *k != job.key) {
|
||||
state.cache = None;
|
||||
let input = render(job)?;
|
||||
let _busy = Flag::raise(&shared.encoding);
|
||||
let m = state.model.as_mut().map(|(_, m)| m).ok_or("the model did not load")?;
|
||||
let t = web_time::Instant::now();
|
||||
let mut enc = m.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?;
|
||||
m.prepare_clicks(&mut enc).map_err(|e| e.to_string())?;
|
||||
log::info!("SAM 3 encoded {}×{} in {:?}", input.w, input.h, t.elapsed());
|
||||
state.cache = Some((job.key, enc));
|
||||
}
|
||||
let State { model, cache } = state;
|
||||
match (model.as_mut(), cache.as_mut()) {
|
||||
(Some((_, m)), Some((_, e))) => Ok((m, e)),
|
||||
_ => Err("the model is not ready".into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Every phrase of `text` ("car, road"), merged (the per-pixel maximum); `None` when nothing
|
||||
/// matches.
|
||||
fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> Result<Option<Vec<f32>>, String> {
|
||||
let mut merged: Option<Vec<f32>> = None;
|
||||
for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) {
|
||||
if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? {
|
||||
merged = Some(match merged {
|
||||
None => p.logits,
|
||||
Some(m) => m.iter().zip(&p.logits).map(|(a, b)| a.max(*b)).collect(),
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(merged)
|
||||
}
|
||||
|
||||
fn run_job(state: &mut State, shared: &Shared, job: &mut Job) -> Result<Option<SegMask>, String> {
|
||||
let side = lightcraft_segment::MASK_SIDE;
|
||||
match std::mem::replace(&mut job.kind, Kind::Prepare) {
|
||||
Kind::Prepare => {
|
||||
encoded(state, shared, job)?;
|
||||
Ok(None)
|
||||
}
|
||||
Kind::Clicks(clicks) => {
|
||||
let (m, enc) = encoded(state, shared, job)?;
|
||||
let p = m.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?;
|
||||
Ok(Some(SegMask::from_logits(side, &p.logits)))
|
||||
}
|
||||
Kind::Text(text) => {
|
||||
let (m, enc) = encoded(state, shared, job)?;
|
||||
Ok(text_logits(m, enc, &text)?.map(|l| SegMask::from_logits(side, &l)))
|
||||
}
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::panic)]
|
||||
Kind::Panic => panic!("a test panic"),
|
||||
Kind::Detail { prompt, region } => {
|
||||
let input = render(job)?;
|
||||
let m = model(state, shared, &job.dir)?;
|
||||
let t = web_time::Instant::now();
|
||||
let mut enc = m.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?;
|
||||
let (rw, rh) = (region[2] - region[0], region[3] - region[1]);
|
||||
if !(rw > 0.0 && rh > 0.0) {
|
||||
return Ok(None);
|
||||
}
|
||||
let logits = match prompt {
|
||||
Prompt::Clicks(clicks) => {
|
||||
let inside: Vec<lightcraft_segment::Click> = clicks
|
||||
.iter()
|
||||
.map(|c| ((c.at.x - region[0]) / rw, (c.at.y - region[1]) / rh, c.include))
|
||||
.filter(|(x, y, _)| (0.0..=1.0).contains(x) && (0.0..=1.0).contains(y))
|
||||
.map(|(x, y, positive)| lightcraft_segment::Click { x: x as f32, y: y as f32, positive })
|
||||
.collect();
|
||||
if !inside.iter().any(|c| c.positive) {
|
||||
return Ok(None);
|
||||
}
|
||||
Some(m.segment_clicks(&mut enc, &inside).map_err(|e| e.to_string())?.logits)
|
||||
}
|
||||
Prompt::Text(text) => text_logits(m, &mut enc, &text)?,
|
||||
};
|
||||
log::info!("SAM 3 detail pass ({}×{}) in {:?}", input.w, input.h, t.elapsed());
|
||||
Ok(logits.map(|l| SegMask::from_logits_in(side, &l, region)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn job(tag: Tag, reply: &mpsc::Sender<Outcome>) -> Job {
|
||||
// an empty folder: loading fails with an error (no model files)
|
||||
let dir = std::env::temp_dir().join(format!("lc-sam3-worker-none-{}", std::process::id()));
|
||||
Job { dir, key: 1, render: None, missing: None, kind: Kind::Prepare, tag, reply: reply.clone() }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn errors_come_back_and_the_counters_return_to_zero() {
|
||||
let mut w = Worker::default();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
for _ in 0..3 {
|
||||
w.submit(job(Tag::Wait, &tx)).unwrap();
|
||||
}
|
||||
for _ in 0..3 {
|
||||
let o = rx.recv_timeout(Duration::from_secs(30)).unwrap();
|
||||
assert!(o.result.unwrap_err().contains("not found"));
|
||||
}
|
||||
let t = web_time::Instant::now();
|
||||
while w.pending() > 0 && t.elapsed() < Duration::from_secs(5) {
|
||||
std::thread::yield_now();
|
||||
}
|
||||
assert_eq!(w.pending(), 0);
|
||||
assert!(!w.encoding() && !w.loaded());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_panic_in_a_job_is_an_error_and_the_worker_carries_on() {
|
||||
let shared = Shared::default();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let (jtx, jrx) = mpsc::channel();
|
||||
let mut j = job(Tag::Wait, &tx);
|
||||
j.kind = Kind::Panic;
|
||||
jtx.send(j).unwrap();
|
||||
jtx.send(job(Tag::Wait, &tx)).unwrap();
|
||||
shared.pending.store(2, Ordering::SeqCst);
|
||||
drop(jtx);
|
||||
run(&jrx, &shared, Duration::from_secs(60));
|
||||
let first = rx.recv().unwrap();
|
||||
assert!(first.result.unwrap_err().contains("failed unexpectedly"));
|
||||
let second = rx.recv().unwrap();
|
||||
assert!(second.result.is_err(), "the next job still runs");
|
||||
assert_eq!(shared.pending.load(Ordering::SeqCst), 0, "counters reset");
|
||||
assert!(!shared.encoding.load(Ordering::SeqCst));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_newer_request_for_the_same_selection_supersedes_a_queued_one() {
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let (jtx, jrx) = mpsc::channel();
|
||||
let clicks = |n: usize| Tag::Clicks {
|
||||
photo: lightcraft_catalog::PhotoId(1),
|
||||
mask: 1,
|
||||
comp: 0,
|
||||
hint: vec![lightcraft_geom::Point::new(0.5, 0.5); n],
|
||||
exclude: vec![],
|
||||
};
|
||||
for n in 1..=3 {
|
||||
jtx.send(job(clicks(n), &tx)).unwrap();
|
||||
}
|
||||
drop(jtx);
|
||||
let shared = Shared::default();
|
||||
shared.pending.store(3, Ordering::SeqCst);
|
||||
run(&jrx, &shared, Duration::from_secs(60));
|
||||
let out: Vec<Outcome> = rx.try_iter().collect();
|
||||
assert_eq!(out.len(), 3);
|
||||
assert!(out[0].superseded && out[1].superseded && !out[2].superseded);
|
||||
assert_eq!(shared.pending.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
@@ -99,6 +99,7 @@ pub struct SidecarData {
|
||||
pub alt_text: Option<String>,
|
||||
pub extended_description: Option<String>,
|
||||
pub keywords: Option<Vec<String>>,
|
||||
pub regions: Option<Vec<lightcraft_meta::Region>>,
|
||||
/// Capture time (ISO 8601) from `exif:DateTimeOriginal`, `photoshop:DateCreated` or
|
||||
/// `xmp:CreateDate` (first found). Used only when the file itself has no capture time.
|
||||
pub captured: Option<String>,
|
||||
@@ -133,6 +134,12 @@ pub fn parse_sidecar(xmp: &str, raw: bool) -> std::result::Result<SidecarData, S
|
||||
alt_text: m.alt_text.clone(),
|
||||
extended_description: m.extended_description.clone(),
|
||||
keywords: (!m.keywords.is_empty()).then(|| m.keywords.clone()),
|
||||
// A sidecar that has `mwg-rs:Regions` at all (even an empty list) was written by an app that
|
||||
// knows about regions, so it's authoritative: its list, empty or not, replaces the catalog's.
|
||||
// One without it (most writers, LightCraft's own included, which keeps another app's
|
||||
// `mwg-rs:Regions` byte for byte but never writes one) says nothing about regions, and the
|
||||
// photo's are kept.
|
||||
regions: d.values.contains_key("mwg-rs:Regions").then(|| m.regions.clone()),
|
||||
captured: m.capture_time.map(|d| d.to_iso()),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -202,6 +209,9 @@ pub fn merge_into(p: &mut Photo, sc: &SidecarData, now: &str) -> bool {
|
||||
if let Some(k) = &sc.keywords {
|
||||
m.keywords = k.clone();
|
||||
}
|
||||
if let Some(r) = &sc.regions {
|
||||
m.regions = r.clone();
|
||||
}
|
||||
let develop = match &sc.develop {
|
||||
Some(DevelopPatch::Full(s)) => (**s).clone(),
|
||||
Some(DevelopPatch::Partial(v)) => {
|
||||
|
||||
@@ -727,3 +727,93 @@ fn curve_reset_by_channel_and_whole() {
|
||||
assert_eq!(active_dev(&s).curve, lightcraft_develop::ToneCurve::default());
|
||||
assert!(s.execute("curve.reset", &json!({"channel": "alpha"})).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ai_masks_without_the_model_explain_themselves() {
|
||||
let mut s = demo();
|
||||
let id = s.active().unwrap();
|
||||
// an Object mask that could never be computed is refused, not left empty
|
||||
let e = s.execute("mask.add", &json!({"kind": "object"})).unwrap_err().to_string();
|
||||
assert!(e.contains("SAM 3") || e.contains("not available"), "{e}");
|
||||
assert!(!e.contains("invalid parameters"), "shown as is: {e}");
|
||||
assert!(s.develop_of(id).unwrap().masks.is_empty());
|
||||
// an empty Object component (as an older document may hold) still takes clicks
|
||||
let shape = lightcraft_develop::MaskShape::Object { hint: vec![], exclude: vec![], seg: None, detail: vec![], edge: 0.0 };
|
||||
let mut d = (*s.develop_of(id).unwrap()).clone();
|
||||
d.masks.push(lightcraft_develop::Mask {
|
||||
id: 1,
|
||||
components: vec![lightcraft_develop::MaskComponent { name: None, op: lightcraft_develop::MaskOp::Add, invert: false, shape }],
|
||||
..Default::default()
|
||||
});
|
||||
s.set_develop(id, d, "test").unwrap();
|
||||
s.active_mask = Some(1);
|
||||
// clicks and descriptions need the model: an error that says what's missing, nothing changed
|
||||
let e = s.execute("mask.objectPoint", &json!({"x": 0.5, "y": 0.5})).unwrap_err().to_string();
|
||||
assert!(e.contains("SAM 3") || e.contains("not available"), "{e}");
|
||||
let e = s.execute("mask.add", &json!({"kind": "prompt", "text": "sky"})).unwrap_err().to_string();
|
||||
assert!(e.contains("SAM 3") || e.contains("not available"), "{e}");
|
||||
let empty = std::env::temp_dir().join(format!("lc-no-sam3-{}", std::process::id()));
|
||||
s.segmenter.dir = Some(empty.clone());
|
||||
let e = s.execute("mask.objectPoint", &json!({"x": 0.5, "y": 0.5})).unwrap_err().to_string();
|
||||
assert!(e.contains(&empty.display().to_string()) || e.contains("not available"), "names the folder: {e}");
|
||||
assert_eq!(s.develop_of(id).unwrap().masks.len(), 1);
|
||||
// bad input is an error, not a crash
|
||||
assert!(s.execute("mask.objectPoint", &json!({"x": 1.5, "y": 0.5})).is_err());
|
||||
assert!(s.execute("mask.add", &json!({"kind": "prompt", "text": " "})).is_err());
|
||||
}
|
||||
|
||||
/// A People card's close-up is a square (in pixels) inside the photo, around the face, whatever the
|
||||
/// face rectangle says: corners, oversized and degenerate rectangles never leave the frame or panic.
|
||||
#[test]
|
||||
fn face_job_crops_a_square_inside_the_photo() {
|
||||
use lightcraft_geom::Rect;
|
||||
let mut s = demo();
|
||||
let id = s.active().unwrap();
|
||||
let (w, h) = {
|
||||
let p = s.catalog.photo(id).unwrap();
|
||||
(f64::from(p.width), f64::from(p.height))
|
||||
};
|
||||
let faces = [
|
||||
Rect { x0: 0.40, y0: 0.30, x1: 0.50, y1: 0.45 },
|
||||
Rect { x0: 0.0, y0: 0.0, x1: 0.05, y1: 0.05 },
|
||||
Rect { x0: 0.95, y0: 0.95, x1: 1.0, y1: 1.0 },
|
||||
Rect { x0: -5.0, y0: -5.0, x1: 5.0, y1: 5.0 },
|
||||
Rect { x0: 0.5, y0: 0.5, x1: 0.5, y1: 0.5 },
|
||||
Rect { x0: f64::NAN, y0: 0.5, x1: 0.6, y1: f64::INFINITY },
|
||||
];
|
||||
for face in faces {
|
||||
let job = s.face_job(id, face, 256).expect("a job for a photo in the library");
|
||||
let r = job.settings.crop.geometry.rect;
|
||||
assert!((0.0..=1.0).contains(&r.x0) && (0.0..=1.0).contains(&r.y0) && r.x1 <= 1.0 + 1e-9 && r.y1 <= 1.0 + 1e-9, "{face:?} → {r:?}");
|
||||
assert!(r.x1 > r.x0 && r.y1 > r.y0, "{face:?} → {r:?}");
|
||||
let (pw, ph) = ((r.x1 - r.x0) * w, (r.y1 - r.y0) * h);
|
||||
assert!((pw - ph).abs() < 1e-6 * w.max(h), "square in pixels: {face:?} → {pw} × {ph}");
|
||||
assert_eq!(job.settings.crop.geometry.angle, 0.0);
|
||||
}
|
||||
// a face around (0.45, 0.375) is centred in its crop (room to spare on every side)
|
||||
let r = s.face_job(id, faces[0], 256).unwrap().settings.crop.geometry.rect;
|
||||
assert!((((r.x0 + r.x1) / 2.0) - 0.45).abs() < 1e-9 && (((r.y0 + r.y1) / 2.0) - 0.375).abs() < 1e-9, "{r:?}");
|
||||
assert!(s.face_job(lightcraft_catalog::PhotoId(u64::MAX), faces[0], 256).is_none());
|
||||
}
|
||||
|
||||
/// Regions are on the upright photo; after Rotate Right the close-up still frames the face: the box
|
||||
/// is carried into the rotated frame the crop lives in, and the crop stays square in rotated pixels.
|
||||
#[test]
|
||||
fn face_job_follows_rotate_right() {
|
||||
use lightcraft_geom::Rect;
|
||||
let mut s = demo();
|
||||
let id = s.active().unwrap();
|
||||
let (w, h) = {
|
||||
let p = s.catalog.photo(id).unwrap();
|
||||
(f64::from(p.width), f64::from(p.height))
|
||||
};
|
||||
// a small face left of centre on the upright photo
|
||||
let face = Rect { x0: 0.30, y0: 0.45, x1: 0.35, y1: 0.50 };
|
||||
s.execute("photo.rotateRight", &json!({})).unwrap();
|
||||
let r = s.face_job(id, face, 256).unwrap().settings.crop.geometry.rect;
|
||||
// a quarter turn clockwise puts the left of the photo at the top: centre (0.325, 0.475) → (0.525, 0.325)
|
||||
assert!((((r.x0 + r.x1) / 2.0) - 0.525).abs() < 1e-9 && (((r.y0 + r.y1) / 2.0) - 0.325).abs() < 1e-9, "{r:?}");
|
||||
// the rotated photo is h × w pixels
|
||||
let (pw, ph) = ((r.x1 - r.x0) * h, (r.y1 - r.y0) * w);
|
||||
assert!((pw - ph).abs() < 1e-6 * w.max(h), "square in rotated pixels: {pw} × {ph}");
|
||||
}
|
||||
@@ -323,3 +323,50 @@ fn a_failed_export_write_keeps_the_previous_file() {
|
||||
assert_ne!(std::fs::read(&prev).unwrap(), b"last week's export");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn export_protects_the_target_of_an_imported_symlink() {
|
||||
use crate::export::{Conflict, Destination};
|
||||
let (original_session, _, dir, src, original) = library_with_jpeg("export-symlink-original");
|
||||
drop(original_session);
|
||||
let alias = dir.join("alias.jpg");
|
||||
std::os::unix::fs::symlink(&src, &alias).unwrap();
|
||||
let sidecar = dir.join("metadata.xmp");
|
||||
std::fs::write(&sidecar, b"<x/>").unwrap();
|
||||
std::os::unix::fs::symlink(&sidecar, alias.with_extension("xmp")).unwrap();
|
||||
let mut s = Session::new().with_fs();
|
||||
s.execute("library.import", &json!({"paths": [alias.to_string_lossy()]})).unwrap();
|
||||
let id = s.catalog.photos().next().unwrap().id;
|
||||
s.execute("develop.set", &json!({"ids": [id.0], "control": "light.exposure", "value": 1.0})).unwrap();
|
||||
let o = ExportOptions { conflict: Conflict::Overwrite, ..Default::default() };
|
||||
let to = Destination { dir: String::new(), exact: Some(src.to_string_lossy().to_string()) };
|
||||
let result = disk_batch(&mut s, id, &o, &to);
|
||||
assert!(result.is_err(), "export replaced the original through its imported alias: {result:?}");
|
||||
assert_eq!(std::fs::read(&src).unwrap(), original);
|
||||
assert!(s.execute("export.checkTarget", &json!({"path": src.to_string_lossy()})).is_err());
|
||||
assert!(s.execute("export.checkTarget", &json!({"path": sidecar.to_string_lossy()})).is_err());
|
||||
assert_eq!(std::fs::read(&sidecar).unwrap(), b"<x/>");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// Issue #134: exports can always be made again, so they are written atomically but not synced to
|
||||
/// disk file by file; an edit copy that becomes a library photo still is.
|
||||
#[test]
|
||||
fn exports_are_atomic_without_a_sync() {
|
||||
use crate::export::{Conflict, Destination};
|
||||
use lightcraft_catalog::safe_file::syncs_on_this_thread;
|
||||
let (mut s, id, dir, _src, _) = library_with_jpeg("export-nosync");
|
||||
let out = dir.join("out");
|
||||
let to = Destination { dir: out.to_string_lossy().to_string(), exact: None };
|
||||
let o = ExportOptions { conflict: Conflict::Unique, ..Default::default() };
|
||||
let before = syncs_on_this_thread();
|
||||
for _ in 0..3 {
|
||||
disk_batch(&mut s, id, &o, &to).unwrap();
|
||||
}
|
||||
assert_eq!(syncs_on_this_thread(), before, "no per-file sync for exports");
|
||||
assert_eq!(std::fs::read_dir(&out).unwrap().count(), 3, "three exports, no temp files");
|
||||
crate::export::write_file_durable(&out.join("edit.tif").to_string_lossy(), b"tiff").unwrap();
|
||||
assert!(syncs_on_this_thread() > before, "the durable writer syncs");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
@@ -207,6 +207,67 @@ fn copy_import_verifies_each_copy() {
|
||||
let _ = std::fs::remove_dir_all(&base);
|
||||
}
|
||||
|
||||
/// Copy verifies against the probe's content hash (issue #134): the hash, not a re-read of the
|
||||
/// source, is the reference — a copy that matches the source but not the hash is refused, a
|
||||
/// corrupted copy is caught, and the file hash agrees with `hash_bytes` across read chunks.
|
||||
#[test]
|
||||
fn copy_is_verified_against_the_probe_hash() {
|
||||
use crate::import_move::copy_verified;
|
||||
let base = temp_dir("copyhash");
|
||||
let src = base.join("src.bin");
|
||||
// larger than one 1 MB read, not a multiple of it
|
||||
let bytes: Vec<u8> = (0..(3 << 20) + 12_345u32).map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8).collect();
|
||||
std::fs::write(&src, &bytes).unwrap();
|
||||
let good = lightcraft_preview::hash_bytes(&bytes);
|
||||
copy_verified(&src, &base.join("ok.bin"), Some(good)).unwrap();
|
||||
assert_eq!(std::fs::read(base.join("ok.bin")).unwrap(), bytes);
|
||||
// a hash that doesn't match: refused (a byte compare with the source would have passed)
|
||||
let stale = lightcraft_preview::hash_bytes(b"what the probe read earlier");
|
||||
let e = copy_verified(&src, &base.join("stale.bin"), Some(stale)).unwrap_err();
|
||||
assert!(e.to_string().contains("differs"), "{e}");
|
||||
assert!(!base.join("stale.bin").exists(), "the refused copy is removed");
|
||||
// a copy that comes out different: caught by the hash
|
||||
inject(Fault::Corrupt);
|
||||
let r = copy_verified(&src, &base.join("bad.bin"), Some(good));
|
||||
inject(Fault::None);
|
||||
assert!(r.is_err());
|
||||
assert!(!base.join("bad.bin").exists());
|
||||
assert_eq!(std::fs::read(&src).unwrap(), bytes, "the source is untouched");
|
||||
let _ = std::fs::remove_dir_all(&base);
|
||||
}
|
||||
|
||||
/// A file replaced after Review Import probed it is not imported on the strength of the old probe:
|
||||
/// its copy no longer matches the probe's hash, so it's reported as failed and nothing is left.
|
||||
#[test]
|
||||
fn copy_import_refuses_a_file_changed_since_the_review() {
|
||||
let base = temp_dir("copychanged");
|
||||
let (card, dest) = (base.join("card"), base.join("out"));
|
||||
write_png(&card.join("a.png"), 3);
|
||||
let mut s = session();
|
||||
let r = s.execute("library.importPreview", &json!({"paths": [card.to_string_lossy()]})).unwrap();
|
||||
assert_eq!(r["scanned"], 1, "{r}");
|
||||
write_png(&card.join("a.png"), 9); // other pixels
|
||||
let r = s
|
||||
.execute(
|
||||
"library.import",
|
||||
&json!({"paths": [card.to_string_lossy()], "mode": "copy", "destination": dest.to_string_lossy(), "organize": "flat"}),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(len(&r, "imported"), 0, "{r}");
|
||||
assert_eq!(len(&r, "failed"), 1, "{r}");
|
||||
assert!(r.to_string().contains("changed meanwhile"), "{r}");
|
||||
assert_eq!(files_under(&dest), Vec::<String>::new());
|
||||
// importing again probes afresh and succeeds
|
||||
let r = s
|
||||
.execute(
|
||||
"library.import",
|
||||
&json!({"paths": [card.to_string_lossy()], "mode": "copy", "destination": dest.to_string_lossy(), "organize": "flat"}),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(len(&r, "imported"), 1, "{r}");
|
||||
let _ = std::fs::remove_dir_all(&base);
|
||||
}
|
||||
|
||||
/// A source that can't be removed (a read-only card) stays and is reported; the photo is
|
||||
/// imported from its copy.
|
||||
#[test]
|
||||
@@ -320,3 +381,55 @@ fn move_into_the_library_is_saved_before_sources_go() {
|
||||
assert_eq!(photo_paths(&again), [want.to_string_lossy()]);
|
||||
let _ = std::fs::remove_dir_all(&base);
|
||||
}
|
||||
|
||||
/// Timing for issue #134 (not a pass/fail test): `cargo test -p lightcraft-engine --release
|
||||
/// write_policy_timing -- --ignored --nocapture`. Writes N export-sized files durably (temp +
|
||||
/// sync + rename) and atomically only, and copies N files verified byte for byte and against the
|
||||
/// probe hash, alternating so a loaded machine affects both alike. `LIGHTCRAFT_BENCH_DIR` puts
|
||||
/// the files on another volume (a USB drive or a NAS is where the difference shows).
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn write_policy_timing() {
|
||||
use crate::import_move::copy_verified;
|
||||
use lightcraft_catalog::safe_file::{write_atomic, write_atomic_nosync};
|
||||
use std::time::{Duration, Instant};
|
||||
let root = std::env::var_os("LIGHTCRAFT_BENCH_DIR").map(PathBuf::from).unwrap_or_else(std::env::temp_dir);
|
||||
let base = root.join(format!("lc-write-policy-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&base);
|
||||
std::fs::create_dir_all(&base).unwrap();
|
||||
let data = |n: usize, seed: u32| -> Vec<u8> { (0..n as u32).map(|i| (i.wrapping_add(seed).wrapping_mul(2_654_435_761) >> 11) as u8).collect() };
|
||||
let (n, size) = (200usize, 300 << 10);
|
||||
let (mut synced, mut unsynced) = (Duration::ZERO, Duration::ZERO);
|
||||
for i in 0..n {
|
||||
let b = data(size, i as u32);
|
||||
let t = Instant::now();
|
||||
write_atomic(&base.join(format!("sync-{i}.jpg")), &b).unwrap();
|
||||
synced += t.elapsed();
|
||||
let t = Instant::now();
|
||||
write_atomic_nosync(&base.join(format!("nosync-{i}.jpg")), &b).unwrap();
|
||||
unsynced += t.elapsed();
|
||||
}
|
||||
println!("export {n} × {} KB: write_atomic (synced) {synced:?}, write_atomic_nosync {unsynced:?}", size >> 10);
|
||||
let (n, size) = (40usize, 8 << 20);
|
||||
let card = base.join("card");
|
||||
std::fs::create_dir_all(&card).unwrap();
|
||||
let hashes: Vec<_> = (0..n)
|
||||
.map(|i| {
|
||||
let b = data(size, 7 * i as u32);
|
||||
std::fs::write(card.join(format!("{i}.raw")), &b).unwrap();
|
||||
lightcraft_preview::hash_bytes(&b)
|
||||
})
|
||||
.collect();
|
||||
let (mut bytewise, mut hashed) = (Duration::ZERO, Duration::ZERO);
|
||||
for (i, h) in hashes.iter().enumerate() {
|
||||
let src = card.join(format!("{i}.raw"));
|
||||
let t = Instant::now();
|
||||
copy_verified(&src, &base.join(format!("bytes-{i}.raw")), None).unwrap();
|
||||
bytewise += t.elapsed();
|
||||
let t = Instant::now();
|
||||
copy_verified(&src, &base.join(format!("hash-{i}.raw")), Some(*h)).unwrap();
|
||||
hashed += t.elapsed();
|
||||
}
|
||||
println!("copy {n} × {} MB: byte compare with the source {bytewise:?}, probe hash {hashed:?}", size >> 20);
|
||||
let _ = std::fs::remove_dir_all(&base);
|
||||
}
|
||||
@@ -201,6 +201,25 @@ fn album_source_reports_in_library_state_and_persists_the_view() {
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// A library opens unfiltered: the source and sort come back, a filter from the last session does not
|
||||
/// (it would hide photos behind a small badge).
|
||||
#[test]
|
||||
fn filters_are_not_restored_when_a_library_reopens() {
|
||||
let dir = temp_dir("filter-not-restored");
|
||||
let mut s = open(&dir);
|
||||
let all = s.visible_cloned().len();
|
||||
s.execute("library.sort", &json!({"key": "fileName"})).unwrap();
|
||||
s.execute("library.filter", &json!({"rating": 5, "date": "2026-01-16", "person": "Jane Doe"})).unwrap();
|
||||
assert!(s.visible_cloned().len() < all);
|
||||
s.close_library().unwrap();
|
||||
drop(s);
|
||||
let mut s2 = open(&dir);
|
||||
assert_eq!(s2.filter, lightcraft_catalog::Filter::default());
|
||||
assert_eq!(s2.visible_cloned().len(), all);
|
||||
assert_eq!(serde_json::to_value(s2.sort).unwrap()["key"], "fileName", "the rest of the view is restored");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_chips_clear_one_filter_and_total_ignores_the_filter() {
|
||||
let dir = temp_dir("chips");
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
//! AI masks without the SAM 3 model (it is never required): every request is a clear error that
|
||||
//! comes back at once, nothing waits on the network or the model, a damaged model is an error
|
||||
//! (not a crash or a stuck "busy"), and the download command needs consent and fails cleanly.
|
||||
//! No test touches the internet: downloads go to a local server.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
use crate::Session;
|
||||
use crate::segment::{NOT_INSTALLED, Segmenter};
|
||||
|
||||
fn tmp(name: &str) -> std::path::PathBuf {
|
||||
let d = std::env::temp_dir().join(format!("lc-sam3-engine-{name}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
d
|
||||
}
|
||||
|
||||
/// A folder that looks installed but holds no usable model.
|
||||
fn damaged_model(name: &str) -> std::path::PathBuf {
|
||||
let d = tmp(name);
|
||||
std::fs::create_dir_all(&d).unwrap();
|
||||
std::fs::write(d.join("model.safetensors"), b"\x10\x00\x00\x00\x00\x00\x00\x00{not json at all}").unwrap();
|
||||
std::fs::write(d.join("vocab.json"), b"{}").unwrap();
|
||||
std::fs::write(d.join("merges.txt"), b"#version: 0.2\n").unwrap();
|
||||
d
|
||||
}
|
||||
|
||||
/// Poll until the worker is idle (or `limit` passes), collecting messages.
|
||||
fn settle(s: &mut Session, limit: Duration) -> Vec<String> {
|
||||
let t = Instant::now();
|
||||
let mut messages = Vec::new();
|
||||
loop {
|
||||
let p = s.segment_poll();
|
||||
messages.extend(p.messages);
|
||||
if !s.segmenter.busy() && !s.segmenter.detail_busy() {
|
||||
messages.extend(s.segment_poll().messages);
|
||||
return messages;
|
||||
}
|
||||
assert!(t.elapsed() < limit, "the worker never finished");
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_and_model_commands_without_the_model() {
|
||||
let mut s = Session::with_demo();
|
||||
let st = s.execute("segment.model.status", &json!({})).unwrap();
|
||||
assert_eq!(st["available"], Segmenter::AVAILABLE);
|
||||
assert_eq!(st["installed"], false);
|
||||
assert_eq!(st["download"]["running"], false);
|
||||
assert!(st["license"].as_str().unwrap().contains("SAM License"));
|
||||
// downloading needs the user's consent, and names the size and the licence
|
||||
let e = s.execute("segment.model.download", &json!({})).unwrap_err().to_string();
|
||||
assert!(e.contains("acknowledged") && e.contains("SAM License") && e.contains("GB"), "{e}");
|
||||
let e = s.execute("segment.model.download", &json!({"acknowledged": "yes"})).unwrap_err().to_string();
|
||||
assert!(e.contains("acknowledged"), "{e}");
|
||||
assert_eq!(s.execute("segment.model.cancel", &json!({})).unwrap()["cancelled"], false);
|
||||
if !Segmenter::AVAILABLE {
|
||||
assert!(s.execute("segment.model.download", &json!({"acknowledged": true})).is_err());
|
||||
return;
|
||||
}
|
||||
// no folder, then no download location: clear errors, nothing started
|
||||
let e = s.execute("segment.model.download", &json!({"acknowledged": true})).unwrap_err().to_string();
|
||||
assert!(e.contains("no folder"), "{e}");
|
||||
let dir = tmp("nomirrors");
|
||||
s.segmenter.dir = Some(dir.clone());
|
||||
s.segmenter.mirrors_file = Some(dir.join("none.txt"));
|
||||
if s.segmenter.mirrors().is_empty() {
|
||||
let e = s.execute("segment.model.download", &json!({"acknowledged": true})).unwrap_err().to_string();
|
||||
assert!(e.contains("LIGHTCRAFT_SAM3_MIRRORS"), "{e}");
|
||||
}
|
||||
assert!(!s.segmenter.download_status().running);
|
||||
}
|
||||
|
||||
#[cfg(feature = "sam")]
|
||||
#[test]
|
||||
fn a_failing_download_ends_with_an_error_and_never_blocks() {
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
// a local mirror that has nothing
|
||||
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let base = format!("http://{}/sam3", l.local_addr().unwrap());
|
||||
std::thread::spawn(move || {
|
||||
for mut c in l.incoming().flatten() {
|
||||
let mut r = BufReader::new(c.try_clone().unwrap());
|
||||
let mut line = String::new();
|
||||
while r.read_line(&mut line).unwrap_or(0) > 2 {
|
||||
line.clear();
|
||||
}
|
||||
let _ = c.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 13\r\nConnection: close\r\n\r\n<h1>404</h1>\n");
|
||||
}
|
||||
});
|
||||
let dir = tmp("download404");
|
||||
let mirrors = dir.with_extension("mirrors.txt");
|
||||
std::fs::write(&mirrors, format!("# test mirror\n{base}\n")).unwrap();
|
||||
let mut s = Session::with_demo();
|
||||
s.segmenter.dir = Some(dir.clone());
|
||||
s.segmenter.mirrors_file = Some(mirrors.clone());
|
||||
let t = Instant::now();
|
||||
let r = s.execute("segment.model.download", &json!({"acknowledged": true})).unwrap();
|
||||
assert_eq!(r["started"], true);
|
||||
assert!(t.elapsed() < Duration::from_secs(1), "starting a download returns at once");
|
||||
// a second request while it runs starts nothing
|
||||
let again = s.execute("segment.model.download", &json!({"acknowledged": true})).unwrap();
|
||||
assert!(again["started"] == false || again["downloading"] == false);
|
||||
let t = Instant::now();
|
||||
while s.segmenter.download_status().running {
|
||||
assert!(t.elapsed() < Duration::from_secs(30), "the download never ended");
|
||||
// AI mask requests meanwhile: an error at once, never a wait
|
||||
let q = Instant::now();
|
||||
let e = s.execute("mask.add", &json!({"kind": "prompt", "text": "sky"})).unwrap_err().to_string();
|
||||
assert!(e.contains(NOT_INSTALLED), "{e}");
|
||||
assert!(q.elapsed() < Duration::from_millis(500));
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
let st = s.segmenter.download_status();
|
||||
assert!(!st.finished);
|
||||
let e = st.error.unwrap();
|
||||
assert!(e.contains("not found"), "{e}");
|
||||
assert!(!s.segmenter.installed());
|
||||
// nothing was left behind under a real name, and no error page was saved
|
||||
for f in ["model.safetensors", "vocab.json", "merges.txt", "vocab.json.part"] {
|
||||
assert!(!dir.join(f).exists(), "{f}");
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
let _ = std::fs::remove_file(&mirrors);
|
||||
}
|
||||
|
||||
/// The app (background mode) never waits: with the model missing every request fails at once;
|
||||
/// with a damaged model the requests are queued, fail on the worker, and come back as messages.
|
||||
#[test]
|
||||
fn background_requests_never_wait_and_failures_come_back_as_messages() {
|
||||
let mut s = Session::with_demo();
|
||||
s.segmenter.background = true;
|
||||
let id = s.active().unwrap();
|
||||
s.segmenter.dir = Some(tmp("missing"));
|
||||
for (cmd, p) in [("mask.add", json!({"kind": "object"})), ("mask.add", json!({"kind": "prompt", "text": "sky"})), ("segment.prepare", json!({}))]
|
||||
{
|
||||
let t = Instant::now();
|
||||
let e = s.execute(cmd, &p).unwrap_err().to_string();
|
||||
assert!(t.elapsed() < Duration::from_millis(500), "{cmd}");
|
||||
if Segmenter::AVAILABLE {
|
||||
assert!(e.starts_with(NOT_INSTALLED), "{cmd}: {e}");
|
||||
} else {
|
||||
assert!(e.contains("not available"), "{cmd}: {e}");
|
||||
}
|
||||
}
|
||||
assert!(s.develop_of(id).unwrap().masks.is_empty());
|
||||
if !Segmenter::AVAILABLE {
|
||||
return;
|
||||
}
|
||||
// a model folder whose weights are damaged
|
||||
let dir = damaged_model("damaged");
|
||||
s.segmenter.dir = Some(dir.clone());
|
||||
assert!(s.segmenter.installed());
|
||||
let t = Instant::now();
|
||||
s.execute("mask.add", &json!({"kind": "object"})).unwrap();
|
||||
let r = s.execute("mask.objectPoint", &json!({"x": 0.5, "y": 0.5})).unwrap();
|
||||
assert_eq!(r["pending"], true);
|
||||
assert!(s.segmenter.pending_clicks().is_some_and(|p| p.hint.len() == 1));
|
||||
// a second click while the first is on its way builds on it
|
||||
let r = s.execute("mask.objectPoint", &json!({"x": 0.6, "y": 0.5, "exclude": true})).unwrap();
|
||||
assert_eq!((r["include"].as_u64(), r["exclude"].as_u64()), (Some(1), Some(1)));
|
||||
let r = s.execute("mask.add", &json!({"kind": "prompt", "text": "the sky"})).unwrap();
|
||||
assert_eq!(r["pending"], true);
|
||||
assert!(t.elapsed() < Duration::from_secs(1), "nothing waited for the model: {:?}", t.elapsed());
|
||||
let messages = settle(&mut s, Duration::from_secs(60));
|
||||
assert!(!messages.is_empty() && messages.iter().all(|m| !m.is_empty()), "{messages:?}");
|
||||
assert!(messages.iter().any(|m| m.contains("model.safetensors") || m.contains("SAM 3")), "{messages:?}");
|
||||
assert!(s.segmenter.pending_clicks().is_none());
|
||||
// the Object mask is there, without a selection; no Describe mask was made from nothing
|
||||
let d = s.develop_of(id).unwrap();
|
||||
assert_eq!(d.masks.len(), 1);
|
||||
assert!(matches!(&d.masks[0].components[0].shape, lightcraft_develop::MaskShape::Object { seg: None, .. }));
|
||||
assert!(!s.segmenter.busy() && !s.segmenter.loaded());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// CLI / MCP (waiting mode): a damaged model is an error from the command, quickly.
|
||||
#[test]
|
||||
fn waiting_requests_with_a_damaged_model_are_errors() {
|
||||
if !Segmenter::AVAILABLE {
|
||||
return;
|
||||
}
|
||||
let mut s = Session::with_demo();
|
||||
let id = s.active().unwrap();
|
||||
let dir = damaged_model("damaged-wait");
|
||||
s.segmenter.dir = Some(dir.clone());
|
||||
let e = s.execute("mask.add", &json!({"kind": "prompt", "text": "sky"})).unwrap_err().to_string();
|
||||
assert!(e.contains("model.safetensors") || e.contains("SAM 3"), "{e}");
|
||||
let e = s.execute("mask.add", &json!({"kind": "object", "points": [[0.5, 0.5]]})).unwrap_err().to_string();
|
||||
assert!(!e.is_empty());
|
||||
assert!(s.develop_of(id).unwrap().masks.is_empty());
|
||||
assert!(!s.segmenter.busy());
|
||||
// a stored segmentation needs no model at all (replayed or pasted masks)
|
||||
let seg = serde_json::to_value(lightcraft_develop::SegMask::from_logits(4, &[5.0; 16])).unwrap();
|
||||
s.segmenter.dir = None;
|
||||
s.execute("mask.add", &json!({"kind": "prompt", "text": "sky", "seg": seg})).unwrap();
|
||||
assert_eq!(s.develop_of(id).unwrap().masks.len(), 1);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// Too many clicks on one Object selection are refused, not sent to the model.
|
||||
#[test]
|
||||
fn object_clicks_are_capped() {
|
||||
if !Segmenter::AVAILABLE {
|
||||
return;
|
||||
}
|
||||
let mut s = Session::with_demo();
|
||||
s.segmenter.background = true;
|
||||
let dir = damaged_model("clicks");
|
||||
s.segmenter.dir = Some(dir.clone());
|
||||
let id = s.active().unwrap();
|
||||
let mut d = (*s.develop_of(id).unwrap()).clone();
|
||||
let many = vec![lightcraft_geom::Point::new(0.5, 0.5); crate::segment::MAX_CLICKS];
|
||||
d.masks.push(lightcraft_develop::Mask {
|
||||
id: 1,
|
||||
components: vec![lightcraft_develop::MaskComponent {
|
||||
name: None,
|
||||
op: lightcraft_develop::MaskOp::Add,
|
||||
invert: false,
|
||||
shape: lightcraft_develop::MaskShape::Object { hint: many, exclude: vec![], seg: None, detail: vec![], edge: 0.0 },
|
||||
}],
|
||||
..Default::default()
|
||||
});
|
||||
s.set_develop(id, d, "test").unwrap();
|
||||
s.active_mask = Some(1);
|
||||
let e = s.execute("mask.objectPoint", &json!({"x": 0.5, "y": 0.5})).unwrap_err().to_string();
|
||||
assert!(e.contains("at most"), "{e}");
|
||||
assert!(!s.segmenter.busy());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
@@ -173,6 +173,162 @@ fn foreign_crs_sidecar_on_import_and_read_from_file() {
|
||||
let _ = std::fs::remove_dir_all(&src);
|
||||
}
|
||||
|
||||
const MWG_REGIONS: &str = r#"<?xpacket begin="" id="W5M0MpCehiHzreSzNTczkc9d"?>
|
||||
<x:xmpmeta xmlns:x="adobe:ns:meta/">
|
||||
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
|
||||
<rdf:Description rdf:about=""
|
||||
xmlns:mwg-rs="http://www.metadataworkinggroup.com/schemas/regions/"
|
||||
xmlns:stArea="http://ns.adobe.com/xmp/sType/Area#"
|
||||
xmlns:stDim="http://ns.adobe.com/xap/1.0/sType/Dimensions#">
|
||||
<mwg-rs:Regions>
|
||||
<rdf:Description>
|
||||
<mwg-rs:AppliedToDimensions stDim:w="100" stDim:h="100" stDim:unit="pixel"/>
|
||||
<mwg-rs:RegionList>
|
||||
<rdf:Bag>
|
||||
<rdf:li>
|
||||
<rdf:Description mwg-rs:Name="Jane Doe" mwg-rs:Type="Face">
|
||||
<mwg-rs:Area stArea:x="0.5" stArea:y="0.5" stArea:w="0.3" stArea:h="0.4" stArea:unit="normalized"/>
|
||||
</rdf:Description>
|
||||
</rdf:li>
|
||||
</rdf:Bag>
|
||||
</mwg-rs:RegionList>
|
||||
</rdf:Description>
|
||||
</mwg-rs:Regions>
|
||||
</rdf:Description>
|
||||
</rdf:RDF>
|
||||
</x:xmpmeta>
|
||||
<?xpacket end="w"?>"#;
|
||||
|
||||
/// A face region written by Lightroom (MWG-RS) into a sidecar reaches `Photo.meta.regions` on
|
||||
/// import — the same path real catalogs use, exercised end to end rather than just at the parser.
|
||||
#[test]
|
||||
fn regions_from_sidecar_are_read_on_import() {
|
||||
let src = temp_dir("regions");
|
||||
write_png(&src.join("portrait.png"), 9);
|
||||
std::fs::write(src.join("portrait.xmp"), MWG_REGIONS).unwrap();
|
||||
let mut s = Session::new().with_fs();
|
||||
let r = s.execute("library.import", &json!({"paths": [src.to_string_lossy()]})).unwrap();
|
||||
assert_eq!(r["sidecars"], 1);
|
||||
let p = s.catalog.photos().next().unwrap().clone();
|
||||
assert_eq!(p.meta.regions.len(), 1);
|
||||
let region = &p.meta.regions[0];
|
||||
assert_eq!(region.name.as_deref(), Some("Jane Doe"));
|
||||
assert_eq!(region.kind, lightcraft_meta::RegionKind::Face);
|
||||
assert!((region.rect.width() - 0.3).abs() < 1e-9, "{:?}", region.rect);
|
||||
let _ = std::fs::remove_dir_all(&src);
|
||||
}
|
||||
|
||||
/// Removing a face box is a catalog-only, undoable edit: it never rewrites the sidecar (Lightroom's
|
||||
/// regions and everything else in it stay byte for byte), even with auto-write on.
|
||||
#[test]
|
||||
fn removing_a_region_is_undoable_and_leaves_the_sidecar_alone() {
|
||||
let src = temp_dir("region-remove");
|
||||
write_png(&src.join("portrait.png"), 9);
|
||||
std::fs::write(src.join("portrait.xmp"), MWG_REGIONS).unwrap();
|
||||
let mut s = Session::new().with_fs();
|
||||
s.execute("library.import", &json!({"paths": [src.to_string_lossy()]})).unwrap();
|
||||
let id = s.active().unwrap();
|
||||
s.execute("library.xmpPreferences", &json!({"autoWrite": true})).unwrap();
|
||||
let before = std::fs::read(src.join("portrait.xmp")).unwrap();
|
||||
let n = s.catalog.photo(id).unwrap().meta.regions.len();
|
||||
assert!(n >= 1);
|
||||
assert!(s.execute("photo.removeRegion", &json!({"index": n})).is_err(), "out of range");
|
||||
assert!(s.execute("photo.removeRegion", &json!({})).is_err(), "no index");
|
||||
let r = s.execute("photo.removeRegion", &json!({"index": 0})).unwrap();
|
||||
assert_eq!(r["removed"], "Jane Doe");
|
||||
assert_eq!(s.catalog.photo(id).unwrap().meta.regions.len(), n - 1);
|
||||
assert_eq!(std::fs::read(src.join("portrait.xmp")).unwrap(), before, "the sidecar is untouched");
|
||||
// an ordinary edit still auto-writes (the guard is per command)
|
||||
s.execute("photo.rate", &json!({"rating": 4})).unwrap();
|
||||
assert_ne!(std::fs::read(src.join("portrait.xmp")).unwrap(), before, "auto-write still works for other edits");
|
||||
// undo brings the region back
|
||||
s.execute("edit.undo", &json!({})).unwrap();
|
||||
s.execute("edit.undo", &json!({})).unwrap();
|
||||
assert_eq!(s.catalog.photo(id).unwrap().meta.regions.len(), n);
|
||||
let _ = std::fs::remove_dir_all(&src);
|
||||
}
|
||||
|
||||
/// Resizing a face box is a catalog-only, undoable edit with strict input handling; it leaves the
|
||||
/// sidecar alone like removal does.
|
||||
#[test]
|
||||
fn resizing_a_region_validates_clamps_undoes_and_leaves_the_sidecar_alone() {
|
||||
let src = temp_dir("region-resize");
|
||||
write_png(&src.join("portrait.png"), 9);
|
||||
std::fs::write(src.join("portrait.xmp"), MWG_REGIONS).unwrap();
|
||||
let mut s = Session::new().with_fs();
|
||||
s.execute("library.import", &json!({"paths": [src.to_string_lossy()]})).unwrap();
|
||||
let id = s.active().unwrap();
|
||||
s.execute("library.xmpPreferences", &json!({"autoWrite": true})).unwrap();
|
||||
let before = std::fs::read(src.join("portrait.xmp")).unwrap();
|
||||
let original = s.catalog.photo(id).unwrap().meta.regions[0].rect;
|
||||
let set = |s: &mut Session, rect: serde_json::Value| s.execute("photo.setRegion", &json!({"index": 0, "rect": rect}));
|
||||
// hostile input is an error and changes nothing
|
||||
for bad in [
|
||||
json!({"x0": 0.1, "y0": 0.1, "x1": 0.1, "y1": 0.5}),
|
||||
json!({"x0": 0.1, "y0": 0.1, "x1": 0.1001, "y1": 0.5}),
|
||||
json!({"x0": 0.1, "y0": 0.1, "x1": 0.5}),
|
||||
json!({"x0": "a", "y0": 0.1, "x1": 0.5, "y1": 0.5}),
|
||||
json!({"x0": null, "y0": 0.1, "x1": 0.5, "y1": 0.5}),
|
||||
] {
|
||||
assert!(set(&mut s, bad.clone()).is_err(), "{bad}");
|
||||
}
|
||||
assert!(s.execute("photo.setRegion", &json!({"index": 9, "rect": {"x0": 0.1, "y0": 0.1, "x1": 0.5, "y1": 0.5}})).is_err(), "no such region");
|
||||
assert!(s.execute("photo.setRegion", &json!({"index": 0})).is_err(), "no rect");
|
||||
assert_eq!(s.catalog.photo(id).unwrap().meta.regions[0].rect, original);
|
||||
// corners given in any order and outside the photo are clamped into it
|
||||
let r = set(&mut s, json!({"x0": 0.6, "y0": 1.5, "x1": -0.2, "y1": 0.2})).unwrap();
|
||||
assert_eq!(r["rect"], json!({"x0": 0.0, "y0": 0.2, "x1": 0.6, "y1": 1.0}));
|
||||
let now = s.catalog.photo(id).unwrap().meta.regions[0].rect;
|
||||
assert_eq!((now.x0, now.y0, now.x1, now.y1), (0.0, 0.2, 0.6, 1.0));
|
||||
assert_eq!(std::fs::read(src.join("portrait.xmp")).unwrap(), before, "the sidecar is untouched");
|
||||
s.execute("edit.undo", &json!({})).unwrap();
|
||||
assert_eq!(s.catalog.photo(id).unwrap().meta.regions[0].rect, original);
|
||||
let _ = std::fs::remove_dir_all(&src);
|
||||
}
|
||||
|
||||
/// Re-reading a sidecar: one that has `mwg-rs:Regions` is authoritative, so an emptied region list
|
||||
/// clears the stale regions; one without `mwg-rs:Regions` (an app that doesn't do regions) leaves the
|
||||
/// photo's regions alone.
|
||||
#[test]
|
||||
fn rereading_a_sidecar_clears_regions_only_when_it_states_them() {
|
||||
let src = temp_dir("region-reread");
|
||||
write_png(&src.join("portrait.png"), 9);
|
||||
std::fs::write(src.join("portrait.xmp"), MWG_REGIONS).unwrap();
|
||||
let mut s = Session::new().with_fs();
|
||||
s.execute("library.import", &json!({"paths": [src.to_string_lossy()]})).unwrap();
|
||||
let id = s.active().unwrap();
|
||||
assert_eq!(s.catalog.photo(id).unwrap().meta.regions.len(), 1);
|
||||
// another app rewrote the sidecar without any notion of regions: they stay
|
||||
std::fs::write(
|
||||
src.join("portrait.xmp"),
|
||||
r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
|
||||
<rdf:Description rdf:about="" xmlns:xmp="http://ns.adobe.com/xap/1.0/" xmp:Rating="3"/>
|
||||
</rdf:RDF></x:xmpmeta>"#,
|
||||
)
|
||||
.unwrap();
|
||||
s.execute("photo.readMetadataFromFile", &json!({})).unwrap();
|
||||
let p = s.catalog.photo(id).unwrap();
|
||||
assert_eq!(p.rating, 3, "the sidecar was read");
|
||||
assert_eq!(p.meta.regions.len(), 1, "a sidecar without mwg-rs:Regions keeps the photo's regions");
|
||||
// a region-aware app removed every region: the sidecar says so, and the catalog follows
|
||||
let emptied = MWG_REGIONS.replace(
|
||||
r#"<rdf:li>
|
||||
<rdf:Description mwg-rs:Name="Jane Doe" mwg-rs:Type="Face">
|
||||
<mwg-rs:Area stArea:x="0.5" stArea:y="0.5" stArea:w="0.3" stArea:h="0.4" stArea:unit="normalized"/>
|
||||
</rdf:Description>
|
||||
</rdf:li>"#,
|
||||
"",
|
||||
);
|
||||
assert_ne!(emptied, MWG_REGIONS);
|
||||
std::fs::write(src.join("portrait.xmp"), emptied).unwrap();
|
||||
s.execute("photo.readMetadataFromFile", &json!({})).unwrap();
|
||||
assert!(s.catalog.photo(id).unwrap().meta.regions.is_empty(), "an emptied region list clears stale regions");
|
||||
// undo brings them back
|
||||
s.execute("edit.undo", &json!({})).unwrap();
|
||||
assert_eq!(s.catalog.photo(id).unwrap().meta.regions.len(), 1);
|
||||
let _ = std::fs::remove_dir_all(&src);
|
||||
}
|
||||
|
||||
fn synthetic_dng(xmp: &str) -> Vec<u8> {
|
||||
synthetic_dng_with(Some(xmp), lightcraft_meta::Metadata::default())
|
||||
}
|
||||
|
||||
@@ -163,7 +163,9 @@ pub fn filter_chips(f: &Filter, cat: &Catalog) -> Vec<FilterChip> {
|
||||
if let Some(e) = f.edited {
|
||||
add(if e { "Edited".into() } else { "Unedited".into() }, json!({"edited": Null}));
|
||||
}
|
||||
for (name, key, val) in [("Camera", "camera", &f.camera), ("Lens", "lens", &f.lens), ("Keyword", "keyword", &f.keyword)] {
|
||||
for (name, key, val) in
|
||||
[("Camera", "camera", &f.camera), ("Lens", "lens", &f.lens), ("Keyword", "keyword", &f.keyword), ("Person", "person", &f.person)]
|
||||
{
|
||||
if let Some(x) = val {
|
||||
add(format!("{name}: {x}"), json!({key: Null}));
|
||||
}
|
||||
|
||||
@@ -332,6 +332,23 @@ impl Orientation {
|
||||
// flipH ∘ rot(t) = rot(-t) ∘ flipH
|
||||
Orientation::from_parts(!f, (4 - t) % 4)
|
||||
}
|
||||
/// The orientation that undoes this one (`o.inverse().map(o.map(p)) == p`). Flips and half turns
|
||||
/// are their own inverses; quarter turns swap.
|
||||
pub fn inverse(self) -> Orientation {
|
||||
match self {
|
||||
Orientation::Rotate90 => Orientation::Rotate270,
|
||||
Orientation::Rotate270 => Orientation::Rotate90,
|
||||
o => o,
|
||||
}
|
||||
}
|
||||
/// Map a rectangle in normalized (0..1) coordinates of the stored image to normalized coordinates
|
||||
/// of the oriented one (e.g. a face region given on the upright photo → the frame after the user's
|
||||
/// Rotate Left/Right).
|
||||
pub fn map_norm_rect(self, r: Rect) -> Rect {
|
||||
let (ax, ay) = self.map(r.x0, r.y0, 1.0, 1.0);
|
||||
let (bx, by) = self.map(r.x1, r.y1, 1.0, 1.0);
|
||||
Rect::from_points(Point::new(ax, ay), Point::new(bx, by))
|
||||
}
|
||||
/// Map a point in stored pixel space (w×h stored dims) to oriented space.
|
||||
pub fn map(self, x: f64, y: f64, w: f64, h: f64) -> (f64, f64) {
|
||||
let (flip, t) = self.to_parts();
|
||||
@@ -417,6 +434,19 @@ mod tests {
|
||||
assert_eq!(Orientation::Rotate90.flipped_h().flipped_h(), Orientation::Rotate90);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orientation_inverse_and_norm_rect() {
|
||||
let r = Rect::new(0.1, 0.2, 0.3, 0.6);
|
||||
for v in 1..=8 {
|
||||
let o = Orientation::from_exif(v);
|
||||
let back = o.inverse().map_norm_rect(o.map_norm_rect(r));
|
||||
assert!([back.x0 - r.x0, back.y0 - r.y0, back.x1 - r.x1, back.y1 - r.y1].iter().all(|d| d.abs() < 1e-12), "{o:?}: {back:?}");
|
||||
}
|
||||
// a quarter turn clockwise: the left of the upright photo becomes its top
|
||||
let q = Orientation::Rotate90.map_norm_rect(r);
|
||||
assert!([q.x0 - 0.4, q.y0 - 0.1, q.x1 - 0.8, q.y1 - 0.3].iter().all(|d| d.abs() < 1e-12), "{q:?}");
|
||||
}
|
||||
|
||||
proptest::proptest! {
|
||||
#[test]
|
||||
fn affine_inverse_prop(a in -3.0f64..3.0, sx in 0.1f64..5.0, sy in 0.1f64..5.0, tx in -50.0f64..50.0, px in -20.0f64..20.0, py in -20.0f64..20.0) {
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
//! Which wgpu backends LightCraft lets wgpu use — for its compute device and for the desktop
|
||||
//! window (eframe) — and the crash sentinel around device creation (issue #136).
|
||||
//!
|
||||
//! wgpu loads the system driver of *every* backend in the instance's set while it enumerates
|
||||
//! adapters, even when it then picks another one: with Vulkan in the set, a broken Vulkan driver
|
||||
//! can crash the process (an access violation inside the driver DLL, which nothing can catch)
|
||||
//! although DX12 would have been used. So Windows defaults to DX12 alone. macOS uses Metal, Linux
|
||||
//! Vulkan (plus GL for the window when the build has it).
|
||||
//!
|
||||
//! Overrides, read once per process:
|
||||
//! - `LIGHTCRAFT_GPU_BACKEND` = `dx12` | `vulkan` | `metal` | `gl` (comma lists allowed), `auto`
|
||||
//! (the platform default), or `off` (no GPU compute; the window still needs one backend and
|
||||
//! keeps the platform default);
|
||||
//! - else `WGPU_BACKEND` (wgpu's own variable, same names) for both the window and compute.
|
||||
//!
|
||||
//! Backends this build doesn't contain (e.g. `gl`: the GL backend isn't compiled in) are ignored
|
||||
//! with a warning and the platform default is used.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Mutex;
|
||||
|
||||
pub use wgpu::Backends;
|
||||
|
||||
/// What the environment asks for.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum BackendChoice {
|
||||
/// The platform default.
|
||||
Auto,
|
||||
/// No GPU compute.
|
||||
Off,
|
||||
/// These backends.
|
||||
Use(Backends),
|
||||
}
|
||||
|
||||
/// Parse a backend list (`"dx12"`, `"vulkan,gl"`, `"auto"`, `"off"`…). `None`: nothing usable in
|
||||
/// it (empty or unknown names).
|
||||
pub fn parse_backends(s: &str) -> Option<BackendChoice> {
|
||||
let s = s.trim().to_ascii_lowercase();
|
||||
match s.as_str() {
|
||||
"" => None,
|
||||
"auto" | "default" => Some(BackendChoice::Auto),
|
||||
"off" | "none" | "cpu" | "0" | "false" | "no" => Some(BackendChoice::Off),
|
||||
_ => {
|
||||
let b = Backends::from_comma_list(&s);
|
||||
(!b.is_empty()).then_some(BackendChoice::Use(b))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The choice from `LIGHTCRAFT_GPU_BACKEND` (`lightcraft`) or else `WGPU_BACKEND` (`wgpu`): the
|
||||
/// first one that parses wins; an unknown value is skipped (with a warning).
|
||||
pub fn choose(lightcraft: Option<&str>, wgpu: Option<&str>) -> BackendChoice {
|
||||
for (name, v) in [("LIGHTCRAFT_GPU_BACKEND", lightcraft), ("WGPU_BACKEND", wgpu)] {
|
||||
let Some(v) = v else { continue };
|
||||
match parse_backends(v) {
|
||||
// WGPU_BACKEND is wgpu's variable: it selects backends, it never turns the GPU off
|
||||
Some(BackendChoice::Off) if name == "WGPU_BACKEND" => log::warn!("gpu: WGPU_BACKEND={v} ignored (use LIGHTCRAFT_GPU_BACKEND=off)"),
|
||||
Some(c) => return c,
|
||||
None => log::warn!("gpu: {name}={v} names no known backend; ignored"),
|
||||
}
|
||||
}
|
||||
BackendChoice::Auto
|
||||
}
|
||||
|
||||
/// The backends compute devices use when nothing is asked for on `os` (`std::env::consts::OS`).
|
||||
pub fn default_compute_backends(os: &str) -> Backends {
|
||||
match os {
|
||||
// never touch the Vulkan driver unless asked: see the module docs (issue #136)
|
||||
"windows" => Backends::DX12,
|
||||
"macos" | "ios" => Backends::METAL,
|
||||
_ => Backends::PRIMARY,
|
||||
}
|
||||
}
|
||||
|
||||
/// The backends the window uses when nothing is asked for on `os`.
|
||||
pub fn default_window_backends(os: &str) -> Backends {
|
||||
match os {
|
||||
"windows" => Backends::DX12,
|
||||
"macos" | "ios" => Backends::METAL,
|
||||
// eframe's default (GL only when the build has it)
|
||||
_ => Backends::PRIMARY | Backends::GL,
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve `choice` against the backends this build contains (`compiled`): `None` = GPU off.
|
||||
/// Asked-for backends that aren't compiled in fall back to `default`.
|
||||
pub fn resolve(choice: BackendChoice, default: Backends, compiled: Backends) -> Option<Backends> {
|
||||
match choice {
|
||||
BackendChoice::Off => None,
|
||||
BackendChoice::Auto => Some(default),
|
||||
BackendChoice::Use(b) if b.intersects(compiled) => Some(b & compiled),
|
||||
BackendChoice::Use(b) => {
|
||||
log::warn!("gpu: backend {b:?} is not in this build ({compiled:?}); using {default:?}");
|
||||
Some(default)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn env_choice() -> BackendChoice {
|
||||
static C: std::sync::OnceLock<BackendChoice> = std::sync::OnceLock::new();
|
||||
*C.get_or_init(|| {
|
||||
let lc = std::env::var("LIGHTCRAFT_GPU_BACKEND").ok();
|
||||
let wg = std::env::var("WGPU_BACKEND").ok();
|
||||
choose(lc.as_deref(), wg.as_deref())
|
||||
})
|
||||
}
|
||||
|
||||
/// The backends of the compute device (`None`: `LIGHTCRAFT_GPU_BACKEND=off`).
|
||||
pub fn compute_backends() -> Option<Backends> {
|
||||
resolve(env_choice(), default_compute_backends(std::env::consts::OS), wgpu::Instance::enabled_backend_features())
|
||||
}
|
||||
|
||||
/// The backends for the desktop window's renderer (eframe/egui-wgpu). The window always needs
|
||||
/// one: `off` keeps the platform default.
|
||||
pub fn window_backends() -> Backends {
|
||||
let default = default_window_backends(std::env::consts::OS);
|
||||
let choice = match env_choice() {
|
||||
BackendChoice::Off => BackendChoice::Auto,
|
||||
c => c,
|
||||
};
|
||||
resolve(choice, default, wgpu::Instance::enabled_backend_features()).unwrap_or(default)
|
||||
}
|
||||
|
||||
/// `LIGHTCRAFT_GPU_BACKEND=off`.
|
||||
pub(crate) fn env_off() -> bool {
|
||||
env_choice() == BackendChoice::Off
|
||||
}
|
||||
|
||||
static MARKER: Mutex<Option<PathBuf>> = Mutex::new(None);
|
||||
|
||||
/// Write this file before the compute device is created and remove it once creation returned
|
||||
/// (successfully or not). A file still there at the next launch means the process died inside
|
||||
/// the driver: see [`take_init_marker`]. `None` (the default) writes nothing.
|
||||
pub fn set_init_marker(path: Option<PathBuf>) {
|
||||
*MARKER.lock().unwrap_or_else(|e| e.into_inner()) = path;
|
||||
}
|
||||
|
||||
/// Run device creation `f` between writing and removing the init marker (if one is set).
|
||||
pub(crate) fn with_init_marker<T>(backends: Backends, f: impl FnOnce() -> T) -> T {
|
||||
let marker = MARKER.lock().unwrap_or_else(|e| e.into_inner()).clone();
|
||||
if let Some(m) = &marker {
|
||||
if let Some(d) = m.parent() {
|
||||
let _ = std::fs::create_dir_all(d);
|
||||
}
|
||||
let secs = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0);
|
||||
let _ = std::fs::write(m, format!("GPU device creation started (backends {backends:?}, unix time {secs})\n"));
|
||||
}
|
||||
let r = f();
|
||||
if let Some(m) = &marker {
|
||||
let _ = std::fs::remove_file(m);
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
/// If the init marker `path` is present — the last process died while creating the GPU device —
|
||||
/// remove it and return what it recorded.
|
||||
pub fn take_init_marker(path: &std::path::Path) -> Option<String> {
|
||||
let text = std::fs::read_to_string(path).ok()?;
|
||||
let _ = std::fs::remove_file(path);
|
||||
Some(text.trim().to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_backend_names() {
|
||||
assert_eq!(parse_backends("dx12"), Some(BackendChoice::Use(Backends::DX12)));
|
||||
assert_eq!(parse_backends(" D3D12 "), Some(BackendChoice::Use(Backends::DX12)));
|
||||
assert_eq!(parse_backends("vulkan"), Some(BackendChoice::Use(Backends::VULKAN)));
|
||||
assert_eq!(parse_backends("vk,gl"), Some(BackendChoice::Use(Backends::VULKAN | Backends::GL)));
|
||||
assert_eq!(parse_backends("metal"), Some(BackendChoice::Use(Backends::METAL)));
|
||||
assert_eq!(parse_backends("auto"), Some(BackendChoice::Auto));
|
||||
for off in ["off", "OFF", "none", "cpu", "0"] {
|
||||
assert_eq!(parse_backends(off), Some(BackendChoice::Off), "{off}");
|
||||
}
|
||||
assert_eq!(parse_backends(""), None);
|
||||
assert_eq!(parse_backends("directx9"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lightcraft_variable_wins_over_wgpu_backend() {
|
||||
assert_eq!(choose(None, None), BackendChoice::Auto);
|
||||
assert_eq!(choose(None, Some("dx12")), BackendChoice::Use(Backends::DX12));
|
||||
assert_eq!(choose(Some("vulkan"), Some("dx12")), BackendChoice::Use(Backends::VULKAN));
|
||||
assert_eq!(choose(Some("off"), Some("dx12")), BackendChoice::Off);
|
||||
// unknown LightCraft value: WGPU_BACKEND still applies
|
||||
assert_eq!(choose(Some("bogus"), Some("gl")), BackendChoice::Use(Backends::GL));
|
||||
// WGPU_BACKEND never turns the GPU off
|
||||
assert_eq!(choose(None, Some("off")), BackendChoice::Auto);
|
||||
}
|
||||
|
||||
/// Issue #136: Windows must not load the Vulkan driver unless asked to.
|
||||
#[test]
|
||||
fn windows_defaults_to_dx12_only() {
|
||||
assert_eq!(default_compute_backends("windows"), Backends::DX12);
|
||||
assert_eq!(default_window_backends("windows"), Backends::DX12);
|
||||
assert!(!default_window_backends("windows").contains(Backends::VULKAN));
|
||||
assert_eq!(default_compute_backends("macos"), Backends::METAL);
|
||||
assert!(default_compute_backends("linux").contains(Backends::VULKAN));
|
||||
assert!(default_window_backends("linux").contains(Backends::VULKAN | Backends::GL));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_against_the_build() {
|
||||
let compiled = Backends::DX12 | Backends::VULKAN;
|
||||
let def = Backends::DX12;
|
||||
assert_eq!(resolve(BackendChoice::Auto, def, compiled), Some(def));
|
||||
assert_eq!(resolve(BackendChoice::Off, def, compiled), None);
|
||||
assert_eq!(resolve(BackendChoice::Use(Backends::VULKAN), def, compiled), Some(Backends::VULKAN));
|
||||
assert_eq!(resolve(BackendChoice::Use(Backends::VULKAN | Backends::GL), def, compiled), Some(Backends::VULKAN));
|
||||
// GL isn't compiled in: the default, not an empty set (no adapter at all)
|
||||
assert_eq!(resolve(BackendChoice::Use(Backends::GL), def, compiled), Some(def));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn init_marker_is_written_during_creation_and_removed_after() {
|
||||
let dir = std::env::temp_dir().join(format!("lc-gpu-marker-{}", std::process::id()));
|
||||
let m = dir.join("gpu-init.marker");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
set_init_marker(Some(m.clone()));
|
||||
let seen = with_init_marker(Backends::DX12, || m.exists());
|
||||
set_init_marker(None);
|
||||
assert!(seen, "the marker exists while the device is created");
|
||||
assert!(!m.exists(), "and is removed afterwards");
|
||||
// a marker left behind by a crashed process is reported once
|
||||
std::fs::write(&m, "GPU device creation started (backends DX12)").unwrap();
|
||||
assert!(take_init_marker(&m).unwrap().contains("DX12"));
|
||||
assert_eq!(take_init_marker(&m), None);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
@@ -233,14 +233,16 @@ pub struct Gpu {
|
||||
/// WGSL constants shared by every module (generated from the CPU pipeline's values).
|
||||
fn constants() -> String {
|
||||
use lightcraft_pipeline::finish::{GRAIN_HASH, MASK_SUMS, MASK_TERMS, SRGB_LUT_N};
|
||||
use lightcraft_pipeline::tone::{LUT_MAX_EV, LUT_MIN_EV, LUT_N};
|
||||
use lightcraft_pipeline::tone::{CHROMA_N, LUT_MAX_EV, LUT_MIN_EV, LUT_N};
|
||||
let mut s = String::new();
|
||||
let [to_lms, from_lms, to_lab, from_lab] = lightcraft_color::perceptual::oklab_matrices();
|
||||
for (name, m) in [("OK_TO_LMS", to_lms), ("OK_FROM_LMS", from_lms), ("OK_TO_LAB", to_lab), ("OK_FROM_LAB", from_lab)] {
|
||||
let rows: Vec<String> = m.iter().map(|r| format!("vec3<f32>({:?}, {:?}, {:?})", r[0], r[1], r[2])).collect();
|
||||
s += &format!("const {name} = array<vec3<f32>, 3>({});\n", rows.join(", "));
|
||||
}
|
||||
s += &format!("const TONE_MIN_EV: f32 = {LUT_MIN_EV:?};\nconst TONE_MAX_EV: f32 = {LUT_MAX_EV:?};\nconst TONE_N: u32 = {LUT_N}u;\n");
|
||||
s += &format!(
|
||||
"const TONE_MIN_EV: f32 = {LUT_MIN_EV:?};\nconst TONE_MAX_EV: f32 = {LUT_MAX_EV:?};\nconst TONE_N: u32 = {LUT_N}u;\nconst CHROMA_N: u32 = {CHROMA_N}u;\n"
|
||||
);
|
||||
s += &format!("const TONE_MIN_GAIN: f32 = {:?};\n", 2f32.powf(LUT_MIN_EV));
|
||||
let b = lightcraft_pipeline::geometry::BLANK;
|
||||
s += &format!("const BLANK_R: f32 = {:?};\nconst BLANK_G: f32 = {:?};\nconst BLANK_B: f32 = {:?};\n", b[0], b[1], b[2]);
|
||||
@@ -276,14 +278,15 @@ fn module_source(m: &Module, consts: &str) -> String {
|
||||
|
||||
impl Gpu {
|
||||
/// Create a device on the best available adapter (no software fallback), or why there is none.
|
||||
pub fn new() -> Result<Gpu, String> {
|
||||
pub fn new(backends: wgpu::Backends) -> Result<Gpu, String> {
|
||||
let mut desc = wgpu::InstanceDescriptor::new_without_display_handle();
|
||||
desc.backends = wgpu::Backends::PRIMARY;
|
||||
// only these drivers are loaded (`crate::backend`: never Vulkan on Windows unless asked)
|
||||
desc.backends = backends;
|
||||
let instance = wgpu::Instance::new(desc);
|
||||
let adapter = pollster::block_on(
|
||||
instance.request_adapter(&wgpu::RequestAdapterOptions { power_preference: wgpu::PowerPreference::HighPerformance, ..Default::default() }),
|
||||
)
|
||||
.map_err(|e| format!("no GPU adapter ({e})"))?;
|
||||
.map_err(|e| format!("no GPU adapter among {backends:?} ({e})"))?;
|
||||
let info = adapter.get_info();
|
||||
if info.device_type == wgpu::DeviceType::Cpu {
|
||||
log::info!("gpu: only a software adapter ({}), using the CPU pipeline", info.name);
|
||||
|
||||
+33
-9
@@ -6,8 +6,11 @@
|
||||
//! same settings on both and bound the difference in 8-bit sRGB. Stages without a kernel run on the
|
||||
//! CPU inside the same render (per-stage hybrid); see `docs/gpu-pipeline.md`.
|
||||
//!
|
||||
//! Use [`render`]: it returns `None` when the GPU is unavailable, disabled (`LIGHTCRAFT_GPU=0` or
|
||||
//! [`set_enabled`]), the render does not fit the device, or the device reported an error, ran out
|
||||
//! Which backends wgpu may load (DX12 only on Windows; `LIGHTCRAFT_GPU_BACKEND`, `WGPU_BACKEND`) and
|
||||
//! the crash sentinel around device creation: [`backend`] (issue #136).
|
||||
//!
|
||||
//! Use [`render`]: it returns `None` when the GPU is unavailable, disabled (`LIGHTCRAFT_GPU=0`,
|
||||
//! `LIGHTCRAFT_GPU_BACKEND=off` or [`set_enabled`]), the render does not fit the device, or the device reported an error, ran out
|
||||
//! of memory or returned an incomplete image — callers then render on the CPU. [`unavailable_reason`]
|
||||
//! and [`last_fallback`] say why (`ui.inspect` → `perf.gpuReason` / `perf.gpuFallback`).
|
||||
//! The browser build has no GPU path yet (WebGPU device creation is asynchronous): everything here
|
||||
@@ -22,6 +25,8 @@ use lightcraft_develop::DevelopSettings;
|
||||
use lightcraft_pipeline::{RenderRequest, Rendered, SourceInfo, StageCache};
|
||||
use lightcraft_raster::Rgb32f;
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod backend;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
mod ctx;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
@@ -61,6 +66,10 @@ fn record_fallback(reason: String) {
|
||||
/// Never blocks.
|
||||
pub fn unavailable_reason() -> Option<String> {
|
||||
if env_disabled() {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
if backend::env_off() {
|
||||
return Some("disabled by LIGHTCRAFT_GPU_BACKEND=off".into());
|
||||
}
|
||||
return Some("disabled by LIGHTCRAFT_GPU=0".into());
|
||||
}
|
||||
if !ENABLED.load(Ordering::Relaxed) {
|
||||
@@ -136,9 +145,15 @@ pub fn enabled() -> bool {
|
||||
ENABLED.load(Ordering::Relaxed) && !BROKEN.load(Ordering::Relaxed) && !env_disabled()
|
||||
}
|
||||
|
||||
/// `LIGHTCRAFT_GPU=0` (or `LIGHTCRAFT_GPU_BACKEND=off`): no GPU for the whole process.
|
||||
fn env_disabled() -> bool {
|
||||
static OFF: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
|
||||
*OFF.get_or_init(|| std::env::var("LIGHTCRAFT_GPU").is_ok_and(|v| matches!(v.trim(), "0" | "off" | "false" | "no")))
|
||||
*OFF.get_or_init(|| {
|
||||
let off = std::env::var("LIGHTCRAFT_GPU").is_ok_and(|v| matches!(v.trim(), "0" | "off" | "false" | "no"));
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
let off = off || backend::env_off();
|
||||
off
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
@@ -146,10 +161,18 @@ static GPU: std::sync::OnceLock<Result<ctx::Gpu, String>> = std::sync::OnceLock:
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub(crate) fn device() -> Option<&'static ctx::Gpu> {
|
||||
if env_disabled() {
|
||||
return None;
|
||||
// turned off (preference, environment): don't even load the driver (issue #136)
|
||||
if env_disabled() || !ENABLED.load(Ordering::Relaxed) {
|
||||
return existing_device();
|
||||
}
|
||||
GPU.get_or_init(|| std::panic::catch_unwind(ctx::Gpu::new).unwrap_or_else(|_| Err("device creation panicked".into()))).as_ref().ok()
|
||||
GPU.get_or_init(|| {
|
||||
let Some(backends) = backend::compute_backends() else { return Err("disabled by LIGHTCRAFT_GPU_BACKEND=off".into()) };
|
||||
backend::with_init_marker(backends, || {
|
||||
std::panic::catch_unwind(|| ctx::Gpu::new(backends)).unwrap_or_else(|_| Err("device creation panicked".into()))
|
||||
})
|
||||
})
|
||||
.as_ref()
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// The device if it has been created (never creates it).
|
||||
@@ -158,11 +181,12 @@ pub(crate) fn existing_device() -> Option<&'static ctx::Gpu> {
|
||||
GPU.get().and_then(|g| g.as_ref().ok())
|
||||
}
|
||||
|
||||
/// Create the device and compile the kernels on a background thread now (app start), so the
|
||||
/// first render doesn't wait ~0.3–0.4 s for it and the UI thread never does.
|
||||
/// Create the device and compile the kernels on a background thread now (once the window is up),
|
||||
/// so the first render doesn't wait ~0.3–0.4 s for it and the UI thread never does. Does nothing
|
||||
/// while GPU rendering is turned off.
|
||||
pub fn warm_up() {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
if !env_disabled() && GPU.get().is_none() {
|
||||
if !env_disabled() && ENABLED.load(Ordering::Relaxed) && GPU.get().is_none() {
|
||||
let _ = std::thread::Builder::new().name("lc-gpu-init".into()).spawn(|| {
|
||||
let _ = device();
|
||||
});
|
||||
|
||||
@@ -136,10 +136,11 @@ pub struct Present {
|
||||
pub chroma: bool,
|
||||
}
|
||||
|
||||
/// The `finish` kernel's parameter block and auxiliary table (tone LUT | sRGB LUT | curve LUTs |
|
||||
/// The `finish` kernel's parameter block and auxiliary table (tone LUT | chroma curve | sRGB LUT | curve LUTs |
|
||||
/// mask terms) for `fp` with `masks` (their local adjustments' terms).
|
||||
pub fn finish_block(fp: &FinishParams, masks: &[[f32; MASK_TERMS]], present: &Present) -> (Vec<u32>, Vec<f32>) {
|
||||
let mut aux: Vec<f32> = fp.tone.lut().to_vec();
|
||||
aux.extend_from_slice(fp.tone.chroma_lut());
|
||||
let srgb_off = aux.len();
|
||||
aux.extend_from_slice(&srgb_lut()[..]);
|
||||
let curve_off = aux.len();
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// The per-pixel stage: a straight port of `lightcraft_pipeline::finish` (keep in step with it).
|
||||
// Bindings: img (rgb, pre-exposure), log_l, base, clar, tex, dark, masks (NMASK planes, then the
|
||||
// blurred chromaticity when HAS_CHROMA), aux
|
||||
// (tone LUT | sRGB LUT | curve LUTs | mask terms), out (packed RGBA8).
|
||||
// (tone LUT | chroma curve | sRGB LUT | curve LUTs | mask terms), out (packed RGBA8).
|
||||
|
||||
fn tone_apply(y: f32) -> f32 {
|
||||
if (y <= 0.0) {
|
||||
@@ -18,6 +18,19 @@ fn tone_apply(y: f32) -> f32 {
|
||||
return v;
|
||||
}
|
||||
|
||||
// The camera chroma curve follows the tone LUT in `aux` (`ToneMap::chroma_scale`).
|
||||
fn chroma_scale(o: f32) -> f32 {
|
||||
let f = clamp(o, 0.0, 1.0) * f32(CHROMA_N - 1u);
|
||||
let i = min(u32(f), CHROMA_N - 2u);
|
||||
let t = f - f32(i);
|
||||
let a = aux[TONE_N + i];
|
||||
let b = aux[TONE_N + i + 1u];
|
||||
if (a == b) {
|
||||
return a;
|
||||
}
|
||||
return a + (b - a) * t;
|
||||
}
|
||||
|
||||
fn encode_srgb(v: f32) -> f32 {
|
||||
let o = pu(F_SRGB_OFF);
|
||||
let f = clamp(v, 0.0, 1.0) * f32(SRGB_N);
|
||||
@@ -400,6 +413,10 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
||||
if (yl > 1e-9) {
|
||||
d = c * o / yl;
|
||||
}
|
||||
let k = chroma_scale(o);
|
||||
if (k != 1.0) {
|
||||
d = vec3<f32>(o) + (d - vec3<f32>(o)) * k;
|
||||
}
|
||||
let mx = max(d.x, max(d.y, d.z));
|
||||
if (mx > 1.0) {
|
||||
let t = clamp((mx - 1.0) / max(mx - o, 1e-6), 0.0, 1.0);
|
||||
|
||||
@@ -41,6 +41,11 @@ fn camera_tone_and_relative_wb() {
|
||||
s.wb.mode = WbMode::Custom;
|
||||
s.wb.temp = 8000.0;
|
||||
check("camera tone edited", &src, &info, &s, &RenderRequest::fit(320, 240));
|
||||
// a camera chroma curve: richer shadows, highlights bleached toward white
|
||||
let curve = curve.with_chroma([1.4, 1.3, 1.1, 1.0, 0.7, 0.4, 0.25, 0.2]).unwrap();
|
||||
let info = SourceInfo { camera_tone: Some(curve), ..info };
|
||||
check("camera chroma curve", &src, &info, &DevelopSettings::default(), &RenderRequest::fit(320, 240));
|
||||
check("camera chroma curve edited", &src, &info, &s, &RenderRequest::fit(320, 240));
|
||||
}
|
||||
|
||||
fn scene(i: usize, w: usize, h: usize) -> Arc<Rgb32f> {
|
||||
|
||||
@@ -45,6 +45,7 @@ pub fn color_data(color: &FrameColor) -> ColorData {
|
||||
as_shot_neutral: Some([1.0, 1.0, 1.0]),
|
||||
as_shot_white_xy: None,
|
||||
baseline_exposure: 0.0,
|
||||
profile: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@
|
||||
//! - WebP (RIFF container spec): `EXIF`, `XMP `, `ICCP` chunks.
|
||||
//! - TIFF-based files (TIFF, DNG and most raws): the whole file is the Exif block; XMP (700), ICC (34675) and
|
||||
//! IPTC (33723) from IFD0.
|
||||
//! - Canon CR3 (ISO base media file, see [`crate::cr3`]): `CMT1`/`CMT2`/`CMT4` merged into one Exif block; the
|
||||
//! XMP `uuid` box.
|
||||
|
||||
use lightcraft_tiff::{Tiff, tags};
|
||||
|
||||
@@ -34,6 +36,9 @@ pub fn embedded(bytes: &[u8]) -> Embedded {
|
||||
_ => Embedded::default(),
|
||||
};
|
||||
}
|
||||
if let Some(c) = crate::cr3::parse_cr3(bytes) {
|
||||
return Embedded { exif: crate::cr3::merged_exif(&c), xmp: c.xmp.map(|b| String::from_utf8_lossy(b).into_owned()), ..Default::default() };
|
||||
}
|
||||
if bytes.starts_with(&[0xff, 0xd8]) {
|
||||
jpeg_segments(bytes)
|
||||
} else if bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
|
||||
|
||||
@@ -0,0 +1,344 @@
|
||||
//! Canon CR3: the ISO base media file container (ISO/IEC 14496-12 box structure; CR3 layout from Laurent
|
||||
//! Clévy's public CR3 format notes, confirmed black-box on EOS R6 Mark III files).
|
||||
//!
|
||||
//! - `ftyp` with major brand `crx `.
|
||||
//! - `moov` holds a Canon `uuid` box (85c0b687-820f-11e0-8111-f4ce462b6a48) whose children are `CMT1` (a TIFF
|
||||
//! stream: IFD0), `CMT2` (TIFF: the Exif IFD), `CMT3` (TIFF: the Canon maker note), `CMT4` (TIFF: GPS) and the
|
||||
//! `THMB` thumbnail, then one `trak` per stream: a full-size JPEG, a reduced raw, the full raw (`CRAW` sample
|
||||
//! entries; raws carry a `CMP1` coding header) and timed metadata (`CTMD`).
|
||||
//! - A top-level `uuid` box with the XMP uuid (be7acfcb-97a9-42e8-9c71-999491e3afac, XMP spec part 3) holds the
|
||||
//! XMP packet.
|
||||
//!
|
||||
//! Every size and offset is checked against the buffer; nesting and box counts are bounded.
|
||||
|
||||
/// A track's sample-entry kind.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Cr3TrackKind {
|
||||
/// `CRAW` with a `JPEG` child: the full-size JPEG.
|
||||
Jpeg,
|
||||
/// `CRAW` raw image: dimensions and the `CMP1` coding header (byte range in the file).
|
||||
Raw { width: u16, height: u16, cmp1: Option<(usize, usize)> },
|
||||
/// Anything else (`CTMD` timed metadata, unknown entries).
|
||||
Other([u8; 4]),
|
||||
}
|
||||
|
||||
/// One track's single sample: where it is in the file.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Cr3Track {
|
||||
pub kind: Cr3TrackKind,
|
||||
/// Byte offset and length of the sample (the first one) in the file, when the tables give them.
|
||||
pub data: Option<(usize, usize)>,
|
||||
}
|
||||
|
||||
/// The parts of a CR3 file LightCraft reads.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Cr3<'a> {
|
||||
/// `CMT1`…`CMT4` (index 0…3).
|
||||
pub cmt: [Option<&'a [u8]>; 4],
|
||||
pub thumbnail: Option<&'a [u8]>,
|
||||
pub xmp: Option<&'a [u8]>,
|
||||
pub tracks: Vec<Cr3Track>,
|
||||
}
|
||||
|
||||
const CANON_UUID: [u8; 16] = [0x85, 0xc0, 0xb6, 0x87, 0x82, 0x0f, 0x11, 0xe0, 0x81, 0x11, 0xf4, 0xce, 0x46, 0x2b, 0x6a, 0x48];
|
||||
const XMP_UUID: [u8; 16] = [0xbe, 0x7a, 0xcf, 0xcb, 0x97, 0xa9, 0x42, 0xe8, 0x9c, 0x71, 0x99, 0x94, 0x91, 0xe3, 0xaf, 0xac];
|
||||
const MAX_DEPTH: usize = 12;
|
||||
const MAX_BOXES: usize = 4096;
|
||||
/// A `VisualSampleEntry` (78 bytes after the box header) plus 4 bytes Canon adds before the child boxes.
|
||||
const CRAW_CHILDREN_AT: usize = 82;
|
||||
|
||||
/// Does this look like a CR3 file (`ftyp` box with major brand `crx `)?
|
||||
pub fn is_cr3(bytes: &[u8]) -> bool {
|
||||
bytes.get(4..12) == Some(b"ftypcrx ".as_slice())
|
||||
}
|
||||
|
||||
/// Parse a CR3 file's boxes; `None` when it is not a CR3 file.
|
||||
pub fn parse_cr3(bytes: &[u8]) -> Option<Cr3<'_>> {
|
||||
if !is_cr3(bytes) {
|
||||
return None;
|
||||
}
|
||||
let mut out = Cr3::default();
|
||||
let mut budget = MAX_BOXES;
|
||||
for b in boxes(bytes, 0, bytes.len(), &mut budget) {
|
||||
match &b.kind {
|
||||
b"moov" => moov(bytes, &b, &mut out, &mut budget),
|
||||
b"uuid" if b.uuid == Some(XMP_UUID) => out.xmp = bytes.get(b.body..b.end),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
struct BoxRef {
|
||||
kind: [u8; 4],
|
||||
/// Start of the payload (after the header and, for `uuid`, the 16-byte uuid).
|
||||
body: usize,
|
||||
end: usize,
|
||||
uuid: Option<[u8; 16]>,
|
||||
}
|
||||
|
||||
/// The boxes directly inside `[from, to)`; stops at the first malformed header.
|
||||
fn boxes(bytes: &[u8], from: usize, to: usize, budget: &mut usize) -> Vec<BoxRef> {
|
||||
let mut out = Vec::new();
|
||||
let mut at = from;
|
||||
let to = to.min(bytes.len());
|
||||
while at.saturating_add(8) <= to && *budget > 0 {
|
||||
*budget -= 1;
|
||||
let (Some(size), Some(kind)) = (be32(bytes, at), bytes.get(at + 4..at + 8)) else { break };
|
||||
let mut kind4 = [0u8; 4];
|
||||
kind4.copy_from_slice(kind);
|
||||
let (len, mut hdr) = match size {
|
||||
0 => (to - at, 8),
|
||||
1 => match be64(bytes, at + 8).and_then(|v| usize::try_from(v).ok()) {
|
||||
Some(l) => (l, 16),
|
||||
None => break,
|
||||
},
|
||||
n => (n as usize, 8),
|
||||
};
|
||||
let Some(end) = at.checked_add(len).filter(|&e| e <= to && len >= hdr) else { break };
|
||||
let mut uuid = None;
|
||||
if &kind4 == b"uuid" {
|
||||
let Some(u) = bytes.get(at + hdr..at + hdr + 16).filter(|_| at + hdr + 16 <= end) else { break };
|
||||
let mut a = [0u8; 16];
|
||||
a.copy_from_slice(u);
|
||||
uuid = Some(a);
|
||||
hdr += 16;
|
||||
}
|
||||
out.push(BoxRef { kind: kind4, body: at + hdr, end, uuid });
|
||||
at = end;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn moov<'a>(bytes: &'a [u8], m: &BoxRef, out: &mut Cr3<'a>, budget: &mut usize) {
|
||||
for b in boxes(bytes, m.body, m.end, budget) {
|
||||
match &b.kind {
|
||||
b"uuid" if b.uuid == Some(CANON_UUID) => {
|
||||
for c in boxes(bytes, b.body, b.end, budget) {
|
||||
let slot = match &c.kind {
|
||||
b"CMT1" => 0,
|
||||
b"CMT2" => 1,
|
||||
b"CMT3" => 2,
|
||||
b"CMT4" => 3,
|
||||
b"THMB" => {
|
||||
out.thumbnail = bytes.get(c.body..c.end);
|
||||
continue;
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
out.cmt[slot] = bytes.get(c.body..c.end);
|
||||
}
|
||||
}
|
||||
b"trak" => {
|
||||
if let Some(t) = trak(bytes, &b, budget) {
|
||||
out.tracks.push(t);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Descend `trak` → `mdia` → `minf` → `stbl` and read its sample entry and first sample's location.
|
||||
fn trak(bytes: &[u8], t: &BoxRef, budget: &mut usize) -> Option<Cr3Track> {
|
||||
let mut stbl = None;
|
||||
let mut stack = vec![(t.body, t.end, 0usize)];
|
||||
while let Some((from, to, depth)) = stack.pop() {
|
||||
if depth > MAX_DEPTH {
|
||||
continue;
|
||||
}
|
||||
for b in boxes(bytes, from, to, budget) {
|
||||
match &b.kind {
|
||||
b"mdia" | b"minf" => stack.push((b.body, b.end, depth + 1)),
|
||||
b"stbl" => stbl = Some(b),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
let stbl = stbl?;
|
||||
let (mut kind, mut size, mut offset) = (None, None, None);
|
||||
for b in boxes(bytes, stbl.body, stbl.end, budget) {
|
||||
match &b.kind {
|
||||
// full box: version/flags, entry count, entries
|
||||
b"stsd" => kind = boxes(bytes, b.body + 8, b.end, budget).first().map(|e| sample_entry(bytes, e, budget)),
|
||||
// version/flags, sample_size (0 = per-sample table), count, sizes
|
||||
b"stsz" => {
|
||||
size = match be32(bytes, b.body + 4)? {
|
||||
0 if be32(bytes, b.body + 8)? > 0 => be32(bytes, b.body + 12),
|
||||
0 => None,
|
||||
n => Some(n),
|
||||
}
|
||||
}
|
||||
b"co64" if be32(bytes, b.body + 4)? > 0 => offset = be64(bytes, b.body + 8).and_then(|v| usize::try_from(v).ok()),
|
||||
b"stco" if be32(bytes, b.body + 4)? > 0 => offset = be32(bytes, b.body + 8).map(|v| v as usize),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let data = match (offset, size) {
|
||||
(Some(o), Some(s)) if o.checked_add(s as usize).is_some_and(|e| e <= bytes.len()) => Some((o, s as usize)),
|
||||
_ => None,
|
||||
};
|
||||
Some(Cr3Track { kind: kind?, data })
|
||||
}
|
||||
|
||||
fn sample_entry(bytes: &[u8], e: &BoxRef, budget: &mut usize) -> Cr3TrackKind {
|
||||
if &e.kind != b"CRAW" {
|
||||
return Cr3TrackKind::Other(e.kind);
|
||||
}
|
||||
let (Some(width), Some(height)) = (be16(bytes, e.body + 24), be16(bytes, e.body + 26)) else {
|
||||
return Cr3TrackKind::Other(e.kind);
|
||||
};
|
||||
let mut cmp1 = None;
|
||||
for c in boxes(bytes, e.body.saturating_add(CRAW_CHILDREN_AT), e.end, budget) {
|
||||
match &c.kind {
|
||||
b"JPEG" => return Cr3TrackKind::Jpeg,
|
||||
b"CMP1" => cmp1 = Some((c.body, c.end - c.body)),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Cr3TrackKind::Raw { width, height, cmp1 }
|
||||
}
|
||||
|
||||
fn be16(b: &[u8], at: usize) -> Option<u16> {
|
||||
b.get(at..at.checked_add(2)?).map(|s| u16::from_be_bytes([s[0], s[1]]))
|
||||
}
|
||||
fn be32(b: &[u8], at: usize) -> Option<u32> {
|
||||
b.get(at..at.checked_add(4)?).map(|s| u32::from_be_bytes([s[0], s[1], s[2], s[3]]))
|
||||
}
|
||||
fn be64(b: &[u8], at: usize) -> Option<u64> {
|
||||
let s = b.get(at..at.checked_add(8)?)?;
|
||||
let mut a = [0u8; 8];
|
||||
a.copy_from_slice(s);
|
||||
Some(u64::from_be_bytes(a))
|
||||
}
|
||||
|
||||
/// Merge `CMT1` (IFD0), `CMT2` (Exif) and `CMT4` (GPS) into one TIFF stream, so the Exif readers see a CR3
|
||||
/// like any TIFF-based raw. The maker note (`CMT3`) is left out: its offsets are relative to its own block.
|
||||
pub fn merged_exif(c: &Cr3<'_>) -> Option<Vec<u8>> {
|
||||
use lightcraft_tiff::{IfdBuilder, Tiff, TiffWriter, tags as t};
|
||||
let ifd0 = Tiff::parse(c.cmt[0]?).ok()?;
|
||||
let order = ifd0.order;
|
||||
let first = |i: usize| c.cmt[i].and_then(|b| Tiff::parse(b).ok()).and_then(|t| t.ifds.into_iter().next());
|
||||
// pointers and offsets that would dangle once the IFDs move
|
||||
const SKIP: [u16; 9] = [t::EXIF_IFD, t::GPS_IFD, t::INTEROP_IFD, t::SUB_IFDS, t::MAKER_NOTE, 273, 279, 513, 514];
|
||||
let build = |ifd: &lightcraft_tiff::Ifd| {
|
||||
let mut b = IfdBuilder::new();
|
||||
for e in ifd.entries.iter().filter(|e| !SKIP.contains(&e.tag)) {
|
||||
b.set(e.tag, e.value.clone());
|
||||
}
|
||||
b
|
||||
};
|
||||
let mut root = build(ifd0.ifds.first()?);
|
||||
if let Some(exif) = first(1) {
|
||||
root.set_child(t::EXIF_IFD, build(&exif));
|
||||
}
|
||||
if let Some(gps) = first(3) {
|
||||
root.set_child(t::GPS_IFD, build(&gps));
|
||||
}
|
||||
TiffWriter::new(order, false).write(&[root]).ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn bx(kind: &[u8; 4], body: &[u8]) -> Vec<u8> {
|
||||
let mut v = ((body.len() + 8) as u32).to_be_bytes().to_vec();
|
||||
v.extend_from_slice(kind);
|
||||
v.extend_from_slice(body);
|
||||
v
|
||||
}
|
||||
fn uuid(u: &[u8; 16], body: &[u8]) -> Vec<u8> {
|
||||
let mut b = u.to_vec();
|
||||
b.extend_from_slice(body);
|
||||
bx(b"uuid", &b)
|
||||
}
|
||||
fn full(kind: &[u8; 4], body: &[u8]) -> Vec<u8> {
|
||||
let mut b = vec![0u8; 4];
|
||||
b.extend_from_slice(body);
|
||||
bx(kind, &b)
|
||||
}
|
||||
fn craw(w: u16, h: u16, children: &[u8]) -> Vec<u8> {
|
||||
let mut b = vec![0u8; CRAW_CHILDREN_AT];
|
||||
b[24..26].copy_from_slice(&w.to_be_bytes());
|
||||
b[26..28].copy_from_slice(&h.to_be_bytes());
|
||||
b.extend_from_slice(children);
|
||||
bx(b"CRAW", &b)
|
||||
}
|
||||
fn trak(entry: Vec<u8>, offset: u64, size: u32) -> Vec<u8> {
|
||||
let mut stsd = 1u32.to_be_bytes().to_vec();
|
||||
stsd.extend(entry);
|
||||
let mut stsz = 0u32.to_be_bytes().to_vec();
|
||||
stsz.extend(1u32.to_be_bytes());
|
||||
stsz.extend(size.to_be_bytes());
|
||||
let mut co64 = 1u32.to_be_bytes().to_vec();
|
||||
co64.extend(offset.to_be_bytes());
|
||||
let stbl = [full(b"stsd", &stsd), full(b"stsz", &stsz), full(b"co64", &co64)].concat();
|
||||
bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &stbl))))
|
||||
}
|
||||
fn tiff_with(tag: u16, v: lightcraft_tiff::Value) -> Vec<u8> {
|
||||
let b = lightcraft_tiff::IfdBuilder::new().with(tag, v);
|
||||
lightcraft_tiff::TiffWriter::new(lightcraft_tiff::ByteOrder::Little, false).write(&[b]).unwrap()
|
||||
}
|
||||
|
||||
/// A small synthetic CR3: Canon uuid with CMT1/CMT2/CMT4, a JPEG track, a raw track, XMP.
|
||||
pub(crate) fn sample() -> Vec<u8> {
|
||||
use lightcraft_tiff::{Value, tags as t};
|
||||
let cmt1 = tiff_with(t::MODEL, Value::Ascii("Canon EOS Test".into()));
|
||||
let cmt2 = tiff_with(t::ISO_SPEED, Value::Short(vec![400]));
|
||||
let cmt4 = tiff_with(1, Value::Ascii("N".into()));
|
||||
let canon = uuid(&CANON_UUID, &[bx(b"CMT1", &cmt1), bx(b"CMT2", &cmt2), bx(b"CMT4", &cmt4), bx(b"THMB", b"thumb")].concat());
|
||||
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
|
||||
let payload_at = 4096u64;
|
||||
let jpeg_t = trak(craw(64, 48, &bx(b"JPEG", &[0; 4])), payload_at, 16);
|
||||
let raw_t = trak(craw(60, 40, &bx(b"CMP1", &[0xff, 0x10, 0, 0x30])), payload_at + 16, 32);
|
||||
file.extend(bx(b"moov", &[canon, jpeg_t, raw_t].concat()));
|
||||
file.extend(uuid(&XMP_UUID, b"<x:xmpmeta/>"));
|
||||
file.resize(payload_at as usize + 64, 0);
|
||||
file
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_canon_boxes_tracks_and_xmp() {
|
||||
let f = sample();
|
||||
assert!(is_cr3(&f));
|
||||
let c = parse_cr3(&f).unwrap();
|
||||
assert!(c.cmt[0].is_some() && c.cmt[1].is_some() && c.cmt[2].is_none() && c.cmt[3].is_some());
|
||||
assert_eq!(c.thumbnail, Some(b"thumb".as_slice()));
|
||||
assert_eq!(c.xmp, Some(b"<x:xmpmeta/>".as_slice()));
|
||||
assert_eq!(c.tracks.len(), 2);
|
||||
assert_eq!(c.tracks[0], Cr3Track { kind: Cr3TrackKind::Jpeg, data: Some((4096, 16)) });
|
||||
let Cr3TrackKind::Raw { width, height, cmp1: Some((at, len)) } = c.tracks[1].kind else { panic!("{:?}", c.tracks[1]) };
|
||||
assert_eq!((width, height, len), (60, 40, 4));
|
||||
assert_eq!(&f[at..at + 2], &[0xff, 0x10]);
|
||||
assert_eq!(c.tracks[1].data, Some((4112, 32)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merged_exif_reads_like_a_tiff_raw() {
|
||||
let f = sample();
|
||||
let m = crate::read_exif(&merged_exif(&parse_cr3(&f).unwrap()).unwrap());
|
||||
assert_eq!(m.model.as_deref(), Some("Canon EOS Test"));
|
||||
assert_eq!(m.iso, Some(400));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_input_never_panics() {
|
||||
let f = sample();
|
||||
// every truncation and single-byte corruption parses to something or nothing
|
||||
for n in 0..f.len().min(800) {
|
||||
let _ = parse_cr3(&f[..n]).map(|c| merged_exif(&c));
|
||||
}
|
||||
for i in 0..f.len().min(800) {
|
||||
for v in [0u8, 1, 0x7f, 0xff] {
|
||||
let mut g = f.clone();
|
||||
g[i] = v;
|
||||
let _ = parse_cr3(&g).map(|c| merged_exif(&c));
|
||||
}
|
||||
}
|
||||
// a sample offset past the end is not reported
|
||||
let mut g = f.clone();
|
||||
g.truncate(4100);
|
||||
assert!(parse_cr3(&g).unwrap().tracks.iter().all(|t| t.data.is_none()));
|
||||
}
|
||||
}
|
||||
+37
-2
@@ -7,10 +7,12 @@
|
||||
//! - [`parse_xmp`] / [`write_xmp`] — XMP packets (interchange fields + the opaque `lc:settings` JSON).
|
||||
//! - [`parse_iptc`] — IPTC-IIM record 2 datasets.
|
||||
//! - [`parse_gpx`] — GPS track logs (GPX) for geotagging by capture time.
|
||||
//! - [`extract`] — all of the above for a whole file (JPEG, PNG, WebP, TIFF/DNG/raw).
|
||||
//! - [`extract`] — all of the above for a whole file (JPEG, PNG, WebP, TIFF/DNG/raw, Canon CR3).
|
||||
//! - [`cr3`] — the Canon CR3 container (metadata blocks, XMP, the JPEG and raw tracks).
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
mod container;
|
||||
pub mod cr3;
|
||||
mod datetime;
|
||||
mod exif;
|
||||
mod gpx;
|
||||
@@ -24,7 +26,7 @@ pub use datetime::DateTime;
|
||||
pub use exif::{from_tiff, read_exif, strip_exif_header, try_read_exif, write_exif};
|
||||
pub use gpx::{GpxError, Match, TrackPoint, Tracklog, parse_gpx};
|
||||
pub use iptc::parse_iptc;
|
||||
pub use lightcraft_geom::Orientation;
|
||||
pub use lightcraft_geom::{Orientation, Rect};
|
||||
pub use tags::{TagRow, file_tag_rows, tag_rows};
|
||||
pub use xmp::{CRS_NS, LC_NS, XmpData, XmpError, XmpValue, parse_xmp, write_xmp, write_xmp_lc};
|
||||
pub use xmp_merge::{MergeRules, merge_xmp};
|
||||
@@ -48,6 +50,31 @@ pub struct Flash {
|
||||
pub raw: u16,
|
||||
}
|
||||
|
||||
/// A named region of interest on a photo (MWG Region Guidelines, `mwg-rs:Regions`): most often a face,
|
||||
/// drawn by Lightroom, digiKam, Picasa or similar tools. Read-only for now — LightCraft does not write
|
||||
/// regions yet (see `docs/xmp-interop.md`).
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Region {
|
||||
/// Normalized to the image's full, oriented frame (`Metadata::width`/`height`), y-down, 0..1 on
|
||||
/// both axes — MWG's own convention, and what `AppliedToDimensions` areas are converted to.
|
||||
pub rect: Rect,
|
||||
pub kind: RegionKind,
|
||||
/// `mwg-rs:Name` (e.g. the person's name).
|
||||
pub name: Option<String>,
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
/// `mwg-rs:Type`. Unrecognised values are kept verbatim rather than dropped.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum RegionKind {
|
||||
Face,
|
||||
Pet,
|
||||
Focus,
|
||||
BarCode,
|
||||
/// `mwg-rs:Type` was missing, empty, or a value we don't recognise.
|
||||
Other(String),
|
||||
}
|
||||
|
||||
/// Everything LightCraft shows or searches about a photo. All fields are optional; unknown = `None`/empty.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Metadata {
|
||||
@@ -113,6 +140,8 @@ pub struct Metadata {
|
||||
pub rating: Option<i8>,
|
||||
/// Colour label name (e.g. "Red").
|
||||
pub label: Option<String>,
|
||||
/// Face/pet/focus/barcode regions (`mwg-rs:Regions`), read from XMP only.
|
||||
pub regions: Vec<Region>,
|
||||
}
|
||||
|
||||
macro_rules! fill {
|
||||
@@ -173,6 +202,9 @@ impl Metadata {
|
||||
if self.hierarchical_keywords.is_empty() {
|
||||
self.hierarchical_keywords = other.hierarchical_keywords.clone();
|
||||
}
|
||||
if self.regions.is_empty() {
|
||||
self.regions = other.regions.clone();
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace the user-editable fields (rating, label, title, caption, artist, copyright, keywords, GPS) with
|
||||
@@ -185,6 +217,9 @@ impl Metadata {
|
||||
if !other.hierarchical_keywords.is_empty() {
|
||||
self.hierarchical_keywords = other.hierarchical_keywords.clone();
|
||||
}
|
||||
if !other.regions.is_empty() {
|
||||
self.regions = other.regions.clone();
|
||||
}
|
||||
}
|
||||
|
||||
/// Exposure time formatted like cameras show it: `1/250`, `0.5`, `2″`.
|
||||
|
||||
+277
-1
@@ -8,7 +8,8 @@
|
||||
//! Writing emits the interchange subset in standard namespaces (dc, xmp, photoshop, exif, exifEX, tiff, lr) and
|
||||
//! LightCraft's full develop settings as an opaque JSON string in `lc:settings`.
|
||||
|
||||
use crate::{DateTime, Flash, Gps, Metadata, Orientation, parse_number};
|
||||
use crate::{DateTime, Flash, Gps, Metadata, Orientation, Region, RegionKind, parse_number};
|
||||
use lightcraft_geom::{Point, Rect};
|
||||
use quick_xml::escape::{escape, partial_escape};
|
||||
use quick_xml::events::Event;
|
||||
use std::collections::BTreeMap;
|
||||
@@ -32,6 +33,11 @@ const NAMESPACES: &[(&str, &str)] = &[
|
||||
("lc", LC_NS),
|
||||
// Read-only: develop settings written by other raw developers (interchange; see docs/xmp-interop.md).
|
||||
("crs", CRS_NS),
|
||||
// Read-only: face/pet/focus regions written by Lightroom, digiKam, Picasa and others (MWG Region
|
||||
// Guidelines v2.0). Canonicalising these three lets a packet use any prefix for them.
|
||||
("mwg-rs", "http://www.metadataworkinggroup.com/schemas/regions/"),
|
||||
("stArea", "http://ns.adobe.com/xmp/sType/Area#"),
|
||||
("stDim", "http://ns.adobe.com/xap/1.0/sType/Dimensions#"),
|
||||
];
|
||||
|
||||
/// The camera-raw-settings namespace URI (prefix `crs`), read for interchange only.
|
||||
@@ -374,6 +380,72 @@ fn parse_gps_coord(s: &str) -> Option<f64> {
|
||||
v.is_finite().then_some(v)
|
||||
}
|
||||
|
||||
/// `mwg-rs:Regions` → every region in its `RegionList`, converted to the full-image normalized frame.
|
||||
/// Malformed or out-of-range entries are skipped rather than rejecting the whole list: one bad region
|
||||
/// (from Lightroom, digiKam, Picasa, or a hand-edited file) shouldn't hide the rest.
|
||||
fn parse_regions(regions: &XmpValue) -> Vec<Region> {
|
||||
// `AppliedToDimensions` documents the pixel size regions were authored against. It's only consulted
|
||||
// for the rare region whose own Area is in pixel units instead of MWG's normalized default below.
|
||||
let px_dims = regions.field("mwg-rs:AppliedToDimensions").and_then(|d| {
|
||||
let w = d.field("stDim:w").and_then(XmpValue::text).and_then(parse_number)?;
|
||||
let h = d.field("stDim:h").and_then(XmpValue::text).and_then(parse_number)?;
|
||||
(w > 0.0 && h > 0.0).then_some((w, h))
|
||||
});
|
||||
let Some(list) = regions.field("mwg-rs:RegionList") else { return Vec::new() };
|
||||
list.items().iter().filter_map(|item| parse_one_region(item, px_dims)).collect()
|
||||
}
|
||||
|
||||
fn parse_one_region(item: &XmpValue, px_dims: Option<(f64, f64)>) -> Option<Region> {
|
||||
let area = item.field("mwg-rs:Area")?;
|
||||
let num_attr = |k: &str| area.field(k).and_then(XmpValue::text).and_then(parse_number);
|
||||
// stArea:x/y are the region's CENTER, not its top-left corner (MWG Region Guidelines §Area).
|
||||
let (cx, cy, w, h) = (num_attr("stArea:x")?, num_attr("stArea:y")?, num_attr("stArea:w")?, num_attr("stArea:h")?);
|
||||
let finite_and_sane = [cx, cy, w, h].iter().all(|v| v.is_finite() && v.abs() <= 1.0e6);
|
||||
if !finite_and_sane || w <= 0.0 || h <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
// `stArea:unit` is "normalized" (a fraction of AppliedToDimensions) in every writer we know of, and
|
||||
// that needs no cross-referencing at all. Only a region that explicitly says "pixel" gets divided
|
||||
// down, and only if AppliedToDimensions is there to divide by; otherwise it's dropped rather than
|
||||
// stored as a nonsense fraction.
|
||||
let is_pixels = area.field("stArea:unit").and_then(XmpValue::text).is_some_and(|u| u.eq_ignore_ascii_case("pixel"));
|
||||
let (cx, cy, w, h) = match (is_pixels, px_dims) {
|
||||
(false, _) => (cx, cy, w, h),
|
||||
(true, Some((dw, dh))) => (cx / dw, cy / dh, w / dw, h / dh),
|
||||
(true, None) => return None,
|
||||
};
|
||||
// Lightroom Classic stores the box of a photo with Exif orientation 3 / 6 / 8 in the sensor's own
|
||||
// frame and writes the orientation as the region's `mwg-rs:Rotation` (π for 3, −π/2 for 6, +π/2 for
|
||||
// 8; 0 for 1) while `AppliedToDimensions` names the upright size. Turn it into the upright frame.
|
||||
// MWG has no way to say "mirrored", so orientations 2/4/5/7 can't be told apart from their unmirrored
|
||||
// twins here; boxes on mirrored photos are taken as written (see LR-LIB-PEOPLE in docs/parity.md).
|
||||
let rotation = item.field("mwg-rs:Rotation").and_then(XmpValue::text).and_then(parse_number).filter(|r| r.is_finite());
|
||||
let (quarter, half) = (std::f64::consts::FRAC_PI_2, std::f64::consts::PI);
|
||||
let (cx, cy, w, h) = match rotation {
|
||||
Some(r) if (r + quarter).abs() < 0.05 => (1.0 - cy, cx, h, w),
|
||||
Some(r) if (r - quarter).abs() < 0.05 => (cy, 1.0 - cx, h, w),
|
||||
Some(r) if (r.abs() - half).abs() < 0.05 => (1.0 - cx, 1.0 - cy, w, h),
|
||||
_ => (cx, cy, w, h),
|
||||
};
|
||||
// Keep only the part of the box inside the photo: a region hanging off an edge (common for faces
|
||||
// cut by the frame) is clipped, and one entirely outside it (hostile or stale) is dropped, so every
|
||||
// stored rect is within 0..1 and non-empty.
|
||||
let rect = Rect::from_center(Point::new(cx, cy), w, h).intersect(&Rect::UNIT);
|
||||
if rect.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let name = item.field("mwg-rs:Name").and_then(XmpValue::text).map(str::to_string).filter(|s| !s.is_empty());
|
||||
let description = item.field("mwg-rs:Description").and_then(XmpValue::text).map(str::to_string).filter(|s| !s.is_empty());
|
||||
let kind = match item.field("mwg-rs:Type").and_then(XmpValue::text).unwrap_or("") {
|
||||
t if t.eq_ignore_ascii_case("face") => RegionKind::Face,
|
||||
t if t.eq_ignore_ascii_case("pet") => RegionKind::Pet,
|
||||
t if t.eq_ignore_ascii_case("focus") => RegionKind::Focus,
|
||||
t if t.eq_ignore_ascii_case("barcode") => RegionKind::BarCode,
|
||||
t => RegionKind::Other(t.to_string()),
|
||||
};
|
||||
Some(Region { rect, kind, name, description })
|
||||
}
|
||||
|
||||
fn fmt_gps_coord(v: f64, pos: char, neg: char) -> String {
|
||||
let a = v.abs();
|
||||
let d = a.trunc();
|
||||
@@ -470,6 +542,7 @@ pub fn parse_xmp(s: &str) -> Result<XmpData, XmpError> {
|
||||
let altitude = num("exif:GPSAltitude").map(|a| if first("exif:GPSAltitudeRef").as_deref() == Some("1") { -a } else { a });
|
||||
m.gps = Some(Gps { latitude: lat, longitude: lon, altitude });
|
||||
}
|
||||
m.regions = values.get("mwg-rs:Regions").map(parse_regions).unwrap_or_default();
|
||||
let lc_settings = props.get("lc:settings").and_then(|v| v.first()).cloned();
|
||||
Ok(XmpData { metadata: m, lc_settings, properties: props, values })
|
||||
}
|
||||
@@ -610,6 +683,9 @@ mod tests {
|
||||
|
||||
fn full() -> Metadata {
|
||||
Metadata {
|
||||
// write_xmp doesn't emit regions yet (read-only interchange; see docs/xmp-interop.md), so the
|
||||
// roundtrip fixture must leave this empty or `roundtrip_all_fields` can't round-trip it.
|
||||
regions: Vec::new(),
|
||||
make: Some("Maker & Sons".into()),
|
||||
model: Some("X <1>".into()),
|
||||
serial_number: Some("SN1".into()),
|
||||
@@ -764,4 +840,204 @@ mod tests {
|
||||
assert_eq!(parse_gps_coord("é"), None);
|
||||
assert_eq!(fmt_gps_coord(-45.51, 'N', 'S'), "45,30.600000S");
|
||||
}
|
||||
|
||||
/// MWG Region Guidelines v2.0, element form (`rdf:parseType="Resource"`, self-closing `Area`) — how
|
||||
/// Lightroom typically writes it. One named face, one untyped/unnamed region (still usable), one
|
||||
/// region with a description. `AppliedToDimensions` is present but irrelevant: normalized areas
|
||||
/// (the default, and the only unit real files use) don't need it.
|
||||
#[test]
|
||||
fn mwg_regions_element_form() {
|
||||
let x = r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
|
||||
<rdf:Description rdf:about=""
|
||||
xmlns:mwg-rs="http://www.metadataworkinggroup.com/schemas/regions/"
|
||||
xmlns:stArea="http://ns.adobe.com/xmp/sType/Area#"
|
||||
xmlns:stDim="http://ns.adobe.com/xap/1.0/sType/Dimensions#">
|
||||
<mwg-rs:Regions rdf:parseType="Resource">
|
||||
<mwg-rs:AppliedToDimensions rdf:parseType="Resource" stDim:w="4000" stDim:h="3000" stDim:unit="pixel"/>
|
||||
<mwg-rs:RegionList>
|
||||
<rdf:Bag>
|
||||
<rdf:li rdf:parseType="Resource">
|
||||
<mwg-rs:Area rdf:parseType="Resource" stArea:x="0.5" stArea:y="0.4" stArea:w="0.2" stArea:h="0.3" stArea:unit="normalized"/>
|
||||
<mwg-rs:Name>Jane Doe</mwg-rs:Name>
|
||||
<mwg-rs:Type>Face</mwg-rs:Type>
|
||||
</rdf:li>
|
||||
<rdf:li rdf:parseType="Resource">
|
||||
<mwg-rs:Area rdf:parseType="Resource" stArea:x="0.1" stArea:y="0.1" stArea:w="0.05" stArea:h="0.05" stArea:unit="normalized"/>
|
||||
</rdf:li>
|
||||
<rdf:li rdf:parseType="Resource">
|
||||
<mwg-rs:Area rdf:parseType="Resource" stArea:x="0.8" stArea:y="0.2" stArea:w="0.1" stArea:h="0.1" stArea:unit="normalized"/>
|
||||
<mwg-rs:Type>Pet</mwg-rs:Type>
|
||||
<mwg-rs:Name>Rex</mwg-rs:Name>
|
||||
<mwg-rs:Description>Good boy</mwg-rs:Description>
|
||||
</rdf:li>
|
||||
</rdf:Bag>
|
||||
</mwg-rs:RegionList>
|
||||
</mwg-rs:Regions>
|
||||
</rdf:Description></rdf:RDF></x:xmpmeta>"#;
|
||||
let regions = parse_xmp(x).unwrap().metadata.regions;
|
||||
assert_eq!(regions.len(), 3, "{regions:?}");
|
||||
|
||||
assert_eq!(regions[0].kind, RegionKind::Face);
|
||||
assert_eq!(regions[0].name.as_deref(), Some("Jane Doe"));
|
||||
assert_eq!(regions[0].description, None);
|
||||
let r = regions[0].rect;
|
||||
assert!((r.x0 - 0.4).abs() < 1e-9 && (r.x1 - 0.6).abs() < 1e-9, "{r:?}");
|
||||
assert!((r.y0 - 0.25).abs() < 1e-9 && (r.y1 - 0.55).abs() < 1e-9, "{r:?}");
|
||||
|
||||
// No mwg-rs:Type or mwg-rs:Name at all: still a usable region, kind falls back to `Other("")`.
|
||||
assert_eq!(regions[1].kind, RegionKind::Other(String::new()));
|
||||
assert_eq!(regions[1].name, None);
|
||||
|
||||
assert_eq!(regions[2].kind, RegionKind::Pet);
|
||||
assert_eq!(regions[2].name.as_deref(), Some("Rex"));
|
||||
assert_eq!(regions[2].description.as_deref(), Some("Good boy"));
|
||||
}
|
||||
|
||||
/// Lightroom Classic writes a rotated photo's (Exif orientation 3 / 6 / 8) box in the sensor's frame
|
||||
/// and records the orientation as `mwg-rs:Rotation` (±π / −π/2 / +π/2); the reader returns the
|
||||
/// upright frame.
|
||||
#[test]
|
||||
fn mwg_regions_quarter_turn_rotation_is_mapped_to_the_upright_frame() {
|
||||
let x = r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
|
||||
<rdf:Description rdf:about=""
|
||||
xmlns:mwg-rs="http://www.metadataworkinggroup.com/schemas/regions/"
|
||||
xmlns:stArea="http://ns.adobe.com/xmp/sType/Area#">
|
||||
<mwg-rs:Regions rdf:parseType="Resource">
|
||||
<mwg-rs:RegionList>
|
||||
<rdf:Bag>
|
||||
<rdf:li rdf:parseType="Resource" mwg-rs:Rotation="-1.57080" mwg-rs:Type="Face">
|
||||
<mwg-rs:Area rdf:parseType="Resource" stArea:x="0.3" stArea:y="0.4" stArea:w="0.2" stArea:h="0.3"/>
|
||||
</rdf:li>
|
||||
<rdf:li rdf:parseType="Resource" mwg-rs:Rotation="1.57080" mwg-rs:Type="Face">
|
||||
<mwg-rs:Area rdf:parseType="Resource" stArea:x="0.3" stArea:y="0.4" stArea:w="0.2" stArea:h="0.3"/>
|
||||
</rdf:li>
|
||||
<rdf:li rdf:parseType="Resource" mwg-rs:Rotation="0.00000" mwg-rs:Type="Face">
|
||||
<mwg-rs:Area rdf:parseType="Resource" stArea:x="0.3" stArea:y="0.4" stArea:w="0.2" stArea:h="0.3"/>
|
||||
</rdf:li>
|
||||
<rdf:li rdf:parseType="Resource" mwg-rs:Rotation="3.14159" mwg-rs:Type="Face">
|
||||
<mwg-rs:Area rdf:parseType="Resource" stArea:x="0.3" stArea:y="0.4" stArea:w="0.2" stArea:h="0.3"/>
|
||||
</rdf:li>
|
||||
<rdf:li rdf:parseType="Resource" mwg-rs:Rotation="-3.14159" mwg-rs:Type="Face">
|
||||
<mwg-rs:Area rdf:parseType="Resource" stArea:x="0.3" stArea:y="0.4" stArea:w="0.2" stArea:h="0.3"/>
|
||||
</rdf:li>
|
||||
</rdf:Bag>
|
||||
</mwg-rs:RegionList>
|
||||
</mwg-rs:Regions>
|
||||
</rdf:Description></rdf:RDF></x:xmpmeta>"#;
|
||||
let regions = parse_xmp(x).unwrap().metadata.regions;
|
||||
assert_eq!(regions.len(), 5, "{regions:?}");
|
||||
let near = |r: Rect, b: [f64; 4]| [r.x0 - b[0], r.y0 - b[1], r.x1 - b[2], r.y1 - b[3]].iter().all(|d| d.abs() < 1e-9);
|
||||
assert!(near(regions[0].rect, [0.45, 0.2, 0.75, 0.4]), "−π/2 (orientation 6): {:?}", regions[0].rect);
|
||||
assert!(near(regions[1].rect, [0.25, 0.6, 0.55, 0.8]), "+π/2 (orientation 8): {:?}", regions[1].rect);
|
||||
assert!(near(regions[2].rect, [0.2, 0.25, 0.4, 0.55]), "0 is untouched: {:?}", regions[2].rect);
|
||||
assert!(near(regions[3].rect, [0.6, 0.45, 0.8, 0.75]), "π (orientation 3): {:?}", regions[3].rect);
|
||||
assert!(near(regions[4].rect, [0.6, 0.45, 0.8, 0.75]), "−π (orientation 3): {:?}", regions[4].rect);
|
||||
}
|
||||
|
||||
/// A second, differently-shaped encoding: each region is its own `rdf:Description` (rather than an
|
||||
/// `rdf:li rdf:parseType="Resource"`), `mwg-rs:Type` as an attribute instead of an element, and an
|
||||
/// unrecognised `mwg-rs:Type` value kept verbatim.
|
||||
#[test]
|
||||
fn mwg_regions_attribute_form_and_unknown_type() {
|
||||
let x = r#"<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
|
||||
<rdf:Description rdf:about=""
|
||||
xmlns:mwg-rs="http://www.metadataworkinggroup.com/schemas/regions/"
|
||||
xmlns:stArea="http://ns.adobe.com/xmp/sType/Area#">
|
||||
<mwg-rs:Regions rdf:parseType="Resource">
|
||||
<mwg-rs:RegionList>
|
||||
<rdf:Bag>
|
||||
<rdf:li>
|
||||
<rdf:Description mwg-rs:Type="BarCode">
|
||||
<mwg-rs:Area stArea:x="0.3" stArea:y="0.3" stArea:w="0.1" stArea:h="0.1"/>
|
||||
</rdf:Description>
|
||||
</rdf:li>
|
||||
<rdf:li>
|
||||
<rdf:Description mwg-rs:Type="Sticker">
|
||||
<mwg-rs:Area stArea:x="0.6" stArea:y="0.6" stArea:w="0.1" stArea:h="0.1"/>
|
||||
</rdf:Description>
|
||||
</rdf:li>
|
||||
</rdf:Bag>
|
||||
</mwg-rs:RegionList>
|
||||
</mwg-rs:Regions>
|
||||
</rdf:Description></rdf:RDF>"#;
|
||||
let regions = parse_xmp(x).unwrap().metadata.regions;
|
||||
assert_eq!(regions.len(), 2, "{regions:?}");
|
||||
assert_eq!(regions[0].kind, RegionKind::BarCode);
|
||||
assert_eq!(regions[1].kind, RegionKind::Other("Sticker".to_string()));
|
||||
}
|
||||
|
||||
/// Hostile/malformed regions are dropped individually, never panic, and never poison the rest of
|
||||
/// the list: missing Area, non-finite or absurd numbers, zero/negative size, and a pixel-unit area
|
||||
/// with no `AppliedToDimensions` to convert it (so it can't be stored as a nonsense fraction).
|
||||
#[test]
|
||||
fn mwg_regions_hostile_inputs_are_dropped_not_panicking() {
|
||||
let x = r#"<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
|
||||
<rdf:Description rdf:about=""
|
||||
xmlns:mwg-rs="http://www.metadataworkinggroup.com/schemas/regions/"
|
||||
xmlns:stArea="http://ns.adobe.com/xmp/sType/Area#">
|
||||
<mwg-rs:Regions rdf:parseType="Resource">
|
||||
<mwg-rs:RegionList>
|
||||
<rdf:Bag>
|
||||
<rdf:li rdf:parseType="Resource"><mwg-rs:Name>No Area</mwg-rs:Name></rdf:li>
|
||||
<rdf:li rdf:parseType="Resource"><mwg-rs:Area rdf:parseType="Resource" stArea:x="NaN" stArea:y="0.1" stArea:w="0.1" stArea:h="0.1"/></rdf:li>
|
||||
<rdf:li rdf:parseType="Resource"><mwg-rs:Area rdf:parseType="Resource" stArea:x="1e300" stArea:y="0.1" stArea:w="0.1" stArea:h="0.1"/></rdf:li>
|
||||
<rdf:li rdf:parseType="Resource"><mwg-rs:Area rdf:parseType="Resource" stArea:x="0.5" stArea:y="0.5" stArea:w="0" stArea:h="0.1"/></rdf:li>
|
||||
<rdf:li rdf:parseType="Resource"><mwg-rs:Area rdf:parseType="Resource" stArea:x="0.5" stArea:y="0.5" stArea:w="-0.1" stArea:h="0.1"/></rdf:li>
|
||||
<rdf:li rdf:parseType="Resource"><mwg-rs:Area rdf:parseType="Resource" stArea:x="100" stArea:y="100" stArea:w="50" stArea:h="50" stArea:unit="pixel"/></rdf:li>
|
||||
<rdf:li rdf:parseType="Resource"><mwg-rs:Area rdf:parseType="Resource" stArea:x="0.5" stArea:y="0.5" stArea:w="0.2" stArea:h="0.2"/><mwg-rs:Type>Face</mwg-rs:Type></rdf:li>
|
||||
</rdf:Bag>
|
||||
</mwg-rs:RegionList>
|
||||
</mwg-rs:Regions>
|
||||
</rdf:Description></rdf:RDF>"#;
|
||||
let regions = parse_xmp(x).unwrap().metadata.regions;
|
||||
assert_eq!(regions.len(), 1, "only the last, well-formed region should survive: {regions:?}");
|
||||
assert_eq!(regions[0].kind, RegionKind::Face);
|
||||
}
|
||||
|
||||
/// Regions are clipped to the photo frame, and ones that miss the photo entirely are dropped: a
|
||||
/// hostile sidecar can't park a box at x=1e5 (or a 1e5-wide one) that the loupe or the People view
|
||||
/// would then draw or crop far outside the image.
|
||||
#[test]
|
||||
fn mwg_regions_are_clipped_to_the_frame_and_off_photo_ones_dropped() {
|
||||
let x = r#"<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
|
||||
<rdf:Description rdf:about=""
|
||||
xmlns:mwg-rs="http://www.metadataworkinggroup.com/schemas/regions/"
|
||||
xmlns:stArea="http://ns.adobe.com/xmp/sType/Area#">
|
||||
<mwg-rs:Regions rdf:parseType="Resource">
|
||||
<mwg-rs:RegionList>
|
||||
<rdf:Bag>
|
||||
<rdf:li rdf:parseType="Resource"><mwg-rs:Area rdf:parseType="Resource" stArea:x="1e5" stArea:y="0.5" stArea:w="0.1" stArea:h="0.1"/><mwg-rs:Name>Far right</mwg-rs:Name></rdf:li>
|
||||
<rdf:li rdf:parseType="Resource"><mwg-rs:Area rdf:parseType="Resource" stArea:x="-3" stArea:y="-3" stArea:w="0.5" stArea:h="0.5"/><mwg-rs:Name>Far left</mwg-rs:Name></rdf:li>
|
||||
<rdf:li rdf:parseType="Resource"><mwg-rs:Area rdf:parseType="Resource" stArea:x="1.2" stArea:y="0.5" stArea:w="0.2" stArea:h="0.2"/><mwg-rs:Name>Touching edge</mwg-rs:Name></rdf:li>
|
||||
<rdf:li rdf:parseType="Resource"><mwg-rs:Area rdf:parseType="Resource" stArea:x="0.5" stArea:y="0.5" stArea:w="1e5" stArea:h="1e5"/><mwg-rs:Name>Huge</mwg-rs:Name></rdf:li>
|
||||
<rdf:li rdf:parseType="Resource"><mwg-rs:Area rdf:parseType="Resource" stArea:x="0.95" stArea:y="0.05" stArea:w="0.2" stArea:h="0.2"/><mwg-rs:Name>Corner</mwg-rs:Name></rdf:li>
|
||||
</rdf:Bag>
|
||||
</mwg-rs:RegionList>
|
||||
</mwg-rs:Regions>
|
||||
</rdf:Description></rdf:RDF>"#;
|
||||
let regions = parse_xmp(x).unwrap().metadata.regions;
|
||||
let names: Vec<_> = regions.iter().map(|r| r.name.as_deref().unwrap_or("")).collect();
|
||||
assert_eq!(names, ["Huge", "Corner"], "{regions:?}");
|
||||
for r in ®ions {
|
||||
let q = r.rect;
|
||||
assert!(q.x0 >= 0.0 && q.y0 >= 0.0 && q.x1 <= 1.0 && q.y1 <= 1.0 && !q.is_empty(), "{q:?}");
|
||||
}
|
||||
assert_eq!(regions[0].rect, Rect::UNIT);
|
||||
let c = regions[1].rect;
|
||||
assert!((c.x0 - 0.85).abs() < 1e-9 && c.x1 == 1.0 && c.y0 == 0.0 && (c.y1 - 0.15).abs() < 1e-9, "{c:?}");
|
||||
}
|
||||
|
||||
/// `extract()`'s precedence (EXIF/IPTC never carry regions; XMP is read and filled in like keywords).
|
||||
#[test]
|
||||
fn regions_merge_like_keywords() {
|
||||
let with_region = Region { rect: Rect::from_center(Point::new(0.5, 0.5), 0.2, 0.2), kind: RegionKind::Face, name: None, description: None };
|
||||
let mut a = Metadata::default();
|
||||
let b = Metadata { regions: vec![with_region.clone()], ..Default::default() };
|
||||
a.fill_missing(&b);
|
||||
assert_eq!(a.regions, vec![with_region.clone()]);
|
||||
|
||||
let mut a = Metadata::default();
|
||||
a.overlay_user_fields(&b);
|
||||
assert_eq!(a.regions, vec![with_region]);
|
||||
}
|
||||
}
|
||||
@@ -483,6 +483,10 @@ pub(crate) fn finish_with<T: Copy + Default + Send>(
|
||||
let yl = luminance_2020(c);
|
||||
let o = tone.apply(yl);
|
||||
let mut d = if yl > 1e-9 { c.map(|v| v * o / yl) } else { [0.0; 3] };
|
||||
let k = tone.chroma_scale(o);
|
||||
if k != 1.0 {
|
||||
d = d.map(|v| o + (v - o) * k);
|
||||
}
|
||||
let mx = d[0].max(d[1]).max(d[2]);
|
||||
if mx > 1.0 {
|
||||
let t = ((mx - 1.0) / (mx - o).max(1e-6)).clamp(0.0, 1.0);
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! Shapes are defined in normalized oriented-image coordinates; sizes are in "long-edge units".
|
||||
//! Components combine with Add (max), Subtract (a·(1−c)) and Intersect (a·c).
|
||||
|
||||
use lightcraft_develop::{BrushStroke, LocalAdjustments, Mask, MaskOp, MaskShape};
|
||||
use lightcraft_develop::{BrushStroke, LocalAdjustments, Mask, MaskOp, MaskShape, SegMask};
|
||||
use lightcraft_geom::Point;
|
||||
use lightcraft_raster::{Plane, Rgb32f};
|
||||
|
||||
@@ -153,6 +153,19 @@ pub fn shape_alpha(shape: &MaskShape, frame: &Frame, w: usize, h: usize, img: &R
|
||||
}
|
||||
smooth_plane(&mut out, 0.01 * frame_px(frame, w));
|
||||
}
|
||||
MaskShape::Object { seg, detail, edge, .. } | MaskShape::Prompt { seg, detail, edge, .. } if seg.is_some() || !detail.is_empty() => {
|
||||
// Edge: below 0 a steeper transition (up to 8× the logits), above 0 feathered (up to
|
||||
// 2 % of the long edge, so previews and exports match)
|
||||
let e = (*edge / 100.0).clamp(-1.0, 1.0) as f32;
|
||||
let gain = if e < 0.0 { 1.0 - 7.0 * e } else { 1.0 };
|
||||
sample_seg(seg.as_ref(), detail, gain, frame, w, h, &mut out);
|
||||
if e > 0.0 {
|
||||
smooth_plane(&mut out, e * 0.02 * frame_px(frame, w));
|
||||
}
|
||||
}
|
||||
// not computed yet (no clicks, or a build without the model): nothing selected
|
||||
MaskShape::Object { hint, .. } if hint.is_empty() => {}
|
||||
MaskShape::Prompt { .. } => {}
|
||||
MaskShape::Subject | MaskShape::Object { .. } | MaskShape::People { .. } => {
|
||||
// Saliency heuristic: centre-weighted local contrast (replaced by the segmenter in M12).
|
||||
let m = frame.out_to_norm(w, h);
|
||||
@@ -176,6 +189,61 @@ pub fn shape_alpha(shape: &MaskShape, frame: &Frame, w: usize, h: usize, img: &R
|
||||
out
|
||||
}
|
||||
|
||||
/// A decoded [`SegMask`]: its logits and the normalized rectangle they cover.
|
||||
struct SegGrid {
|
||||
logits: Vec<f32>,
|
||||
side: usize,
|
||||
r: [f64; 4],
|
||||
}
|
||||
|
||||
impl SegGrid {
|
||||
fn new(seg: &SegMask) -> Option<SegGrid> {
|
||||
Some(SegGrid { logits: seg.logits()?, side: seg.side as usize, r: seg.bounds()? })
|
||||
}
|
||||
|
||||
fn contains(&self, n: Point) -> bool {
|
||||
n.x >= self.r[0] && n.x <= self.r[2] && n.y >= self.r[1] && n.y <= self.r[3]
|
||||
}
|
||||
|
||||
/// The bilinearly sampled logit at normalized image point `n` (cell centres at
|
||||
/// `(i + 0.5) / side` of the rectangle).
|
||||
fn logit(&self, n: Point) -> f32 {
|
||||
let side = self.side;
|
||||
let at = |x: usize, y: usize| self.logits.get(y.min(side - 1) * side + x.min(side - 1)).copied().unwrap_or(-16.0);
|
||||
let u = (n.x - self.r[0]) / (self.r[2] - self.r[0]);
|
||||
let v = (n.y - self.r[1]) / (self.r[3] - self.r[1]);
|
||||
let (fx, fy) = ((u * side as f64 - 0.5) as f32, (v * side as f64 - 0.5) as f32);
|
||||
let (fx, fy) = (fx.clamp(0.0, (side - 1) as f32), fy.clamp(0.0, (side - 1) as f32));
|
||||
let (x0, y0) = (fx.floor() as usize, fy.floor() as usize);
|
||||
let (tx, ty) = (fx - x0 as f32, fy - y0 as f32);
|
||||
let top = at(x0, y0) * (1.0 - tx) + at(x0 + 1, y0) * tx;
|
||||
let bot = at(x0, y0 + 1) * (1.0 - tx) + at(x0 + 1, y0 + 1) * tx;
|
||||
top * (1.0 - ty) + bot * ty
|
||||
}
|
||||
}
|
||||
|
||||
/// Stored model segmentations at each output pixel: the zoomed-in `detail` passes inside
|
||||
/// their rectangles (the highest of them where they overlap), the whole-photo `seg`
|
||||
/// elsewhere; the logit (times `gain`: > 1 is a harder edge) through a sigmoid. Damaged
|
||||
/// data selects nothing.
|
||||
fn sample_seg(seg: Option<&SegMask>, detail: &[SegMask], gain: f32, frame: &Frame, w: usize, h: usize, out: &mut Plane) {
|
||||
let coarse = seg.and_then(SegGrid::new);
|
||||
// (capped: a hostile document can hold many)
|
||||
let patches: Vec<SegGrid> = detail.iter().take(lightcraft_develop::segmask::MAX_DETAIL).filter_map(SegGrid::new).collect();
|
||||
if coarse.is_none() && patches.is_empty() {
|
||||
return;
|
||||
}
|
||||
let m = frame.out_to_norm(w, h);
|
||||
for_rows(&mut out.data, w, |y, row| {
|
||||
for (x, v) in row.iter_mut().enumerate() {
|
||||
let n = m.apply(Point::new(x as f64 + 0.5, y as f64 + 0.5));
|
||||
let fine = patches.iter().filter(|p| p.contains(n)).map(|p| p.logit(n)).reduce(f32::max);
|
||||
let l = fine.or_else(|| coarse.as_ref().map(|c| c.logit(n))).unwrap_or(-16.0);
|
||||
*v = 1.0 / (1.0 + (-l * gain).exp());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn frame_px(frame: &Frame, w: usize) -> f32 {
|
||||
frame.px_per_long(w) as f32
|
||||
}
|
||||
@@ -356,6 +424,93 @@ mod tests {
|
||||
assert!((a.get(50, 25) - 0.5).abs() < 0.05);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stored_segmentations_render_at_any_size_and_follow_the_crop() {
|
||||
// the left half selected, on a 16² logit grid
|
||||
let side = 16;
|
||||
let logits: Vec<f32> = (0..side * side).map(|i| if i % side < side / 2 { 12.0 } else { -12.0 }).collect();
|
||||
let seg = SegMask::from_logits(side, &logits);
|
||||
for shape in [
|
||||
MaskShape::Object { hint: vec![Point::new(0.25, 0.5)], exclude: vec![], seg: Some(seg.clone()), detail: vec![], edge: 0.0 },
|
||||
MaskShape::Prompt { text: "left".into(), seg: Some(seg.clone()), detail: vec![], edge: 0.0 },
|
||||
] {
|
||||
for (w, h) in [(40, 20), (400, 200)] {
|
||||
let a = shape_alpha(&shape, &frame(w, h), w, h, &Rgb32f::new(w, h), &Plane::new(w, h), 0.0);
|
||||
assert!(a.get(1, h / 2) > 0.99 && a.get(w - 2, h / 2) < 0.01, "{w}×{h}");
|
||||
if w >= 400 {
|
||||
let soft = (0..w).filter(|x| (0.05..0.95).contains(&a.get(*x, h / 2))).count();
|
||||
assert!((2..30).contains(&soft), "an anti-aliased edge, not a ramp: {soft} px");
|
||||
}
|
||||
}
|
||||
}
|
||||
// cropped to the right half: nothing left selected
|
||||
let mut d = DevelopSettings::default();
|
||||
d.crop.geometry.rect = lightcraft_geom::Rect::from_center(Point::new(0.75, 0.5), 0.5, 1.0);
|
||||
let f = Frame::new(40, 20, &d, true);
|
||||
let shape = MaskShape::Prompt { text: "left".into(), seg: Some(seg), detail: vec![], edge: 0.0 };
|
||||
let a = shape_alpha(&shape, &f, 20, 20, &Rgb32f::new(20, 20), &Plane::new(20, 20), 0.0);
|
||||
assert!(a.data.iter().all(|v| *v < 0.05));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detail_patches_replace_the_coarse_mask_inside_their_rectangle() {
|
||||
// coarse: nothing; a patch over the right half selects its own left half (x 0.5..0.75)
|
||||
let coarse = SegMask::from_logits(4, &[-12.0; 16]);
|
||||
let side = 16;
|
||||
let l: Vec<f32> = (0..side * side).map(|i| if i % side < side / 2 { 12.0 } else { -12.0 }).collect();
|
||||
let patch = SegMask::from_logits_in(side, &l, [0.5, 0.0, 1.0, 1.0]);
|
||||
let shape = MaskShape::Object { hint: vec![Point::new(0.6, 0.5)], exclude: vec![], seg: Some(coarse), detail: vec![patch], edge: 0.0 };
|
||||
let (w, h) = (200, 100);
|
||||
let a = shape_alpha(&shape, &frame(w, h), w, h, &Rgb32f::new(w, h), &Plane::new(w, h), 0.0);
|
||||
assert!(a.get(20, 50) < 0.01, "outside the patch: the coarse mask");
|
||||
assert!(a.get(120, 50) > 0.99, "inside the patch, its selected half");
|
||||
assert!(a.get(190, 50) < 0.01, "inside the patch, its unselected half");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_hostile_number_of_detail_patches_is_capped() {
|
||||
// built in memory (a deserialized document is capped already): only the first
|
||||
// MAX_DETAIL are decoded and sampled, so this needs 4 × 4 MB, not 20 000 × 4 MB
|
||||
let side = 1024;
|
||||
let l: Vec<f32> = vec![12.0; side * side];
|
||||
let patch = SegMask::from_logits_in(side, &l, [0.0, 0.0, 0.5, 1.0]);
|
||||
let shape = MaskShape::Object { hint: vec![Point::new(0.2, 0.5)], exclude: vec![], seg: None, detail: vec![patch; 20_000], edge: 0.0 };
|
||||
let (w, h) = (64, 32);
|
||||
let t = std::time::Instant::now();
|
||||
let a = shape_alpha(&shape, &frame(w, h), w, h, &Rgb32f::new(w, h), &Plane::new(w, h), 0.0);
|
||||
assert!(a.get(5, 16) > 0.99 && a.get(60, 16) < 0.01);
|
||||
assert!(t.elapsed() < std::time::Duration::from_secs(20), "{:?}", t.elapsed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_edge_setting_hardens_or_feathers_the_transition() {
|
||||
let side = 16;
|
||||
let l: Vec<f32> = (0..side * side).map(|i| ((i % side) as f32 - 7.5) * -0.6).collect(); // a gentle ramp
|
||||
let seg = SegMask::from_logits(side, &l);
|
||||
let (w, h) = (400, 40);
|
||||
let soft_px = |edge: f64| {
|
||||
let shape = MaskShape::Prompt { text: "x".into(), seg: Some(seg.clone()), detail: vec![], edge };
|
||||
let a = shape_alpha(&shape, &frame(w, h), w, h, &Rgb32f::new(w, h), &Plane::new(w, h), 0.0);
|
||||
assert!(a.get(2, h / 2) > 0.9 && a.get(w - 3, h / 2) < 0.1, "still the same selection at {edge}");
|
||||
(0..w).filter(|x| (0.1..0.9).contains(&a.get(*x, h / 2))).count()
|
||||
};
|
||||
let (hard, plain, soft) = (soft_px(-100.0), soft_px(0.0), soft_px(100.0));
|
||||
assert!(hard < plain && plain < soft, "transition widths {hard} < {plain} < {soft}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ai_shapes_without_a_segmentation_select_nothing() {
|
||||
let f = frame(30, 30);
|
||||
let (img, l) = (Rgb32f::new(30, 30), Plane::new(30, 30));
|
||||
for shape in [
|
||||
MaskShape::Object { hint: vec![], exclude: vec![], seg: None, detail: vec![], edge: 0.0 },
|
||||
MaskShape::Prompt { text: "sky".into(), seg: None, detail: vec![], edge: 0.0 },
|
||||
MaskShape::Prompt { text: "sky".into(), seg: Some(SegMask { side: 4, data: "damaged!".into(), rect: None }), detail: vec![], edge: 0.0 },
|
||||
] {
|
||||
assert!(shape_alpha(&shape, &f, 30, 30, &img, &l, 0.0).data.iter().all(|v| *v == 0.0));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn radial_inside_and_invert() {
|
||||
let f = frame(100, 100);
|
||||
|
||||
@@ -13,13 +13,19 @@ pub const GREY: f32 = 0.18;
|
||||
pub const LUT_MIN_EV: f32 = -14.0;
|
||||
pub const LUT_MAX_EV: f32 = 10.0;
|
||||
pub const LUT_N: usize = 4096;
|
||||
/// Nodes of a camera chroma curve, evenly spaced over display luminance 0..=1.
|
||||
pub const CHROMA_N: usize = 8;
|
||||
const NO_CHROMA: [f32; CHROMA_N] = [1.0; CHROMA_N];
|
||||
|
||||
/// A file-local camera look, fitted independently of the scene-linear colour transform.
|
||||
/// Knots are scene/display-linear luminance pairs. Keeping this in the finish stage preserves
|
||||
/// RAW exposure and highlight headroom; it is never baked into the decoded sensor pixels.
|
||||
/// `chroma` scales colourfulness by display luminance after the curve (a camera's per-channel
|
||||
/// curve saturates shadows and bleaches highlights toward white, which a luminance curve can't).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize)]
|
||||
pub struct CameraTone {
|
||||
knots: [[f32; 2]; 32],
|
||||
chroma: [f32; CHROMA_N],
|
||||
}
|
||||
|
||||
impl<'de> serde::Deserialize<'de> for CameraTone {
|
||||
@@ -27,9 +33,16 @@ impl<'de> serde::Deserialize<'de> for CameraTone {
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Wire {
|
||||
knots: [[f32; 2]; 32],
|
||||
// absent in smart previews written before the chroma curve existed
|
||||
#[serde(default)]
|
||||
chroma: Option<[f32; CHROMA_N]>,
|
||||
}
|
||||
let w = Wire::deserialize(d)?;
|
||||
Self::new(w.knots).ok_or_else(|| serde::de::Error::custom("invalid camera tone curve"))
|
||||
let tone = Self::new(w.knots).ok_or_else(|| serde::de::Error::custom("invalid camera tone curve"))?;
|
||||
match w.chroma {
|
||||
Some(c) => tone.with_chroma(c).ok_or_else(|| serde::de::Error::custom("invalid camera chroma curve")),
|
||||
None => Ok(tone),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,7 +55,16 @@ impl CameraTone {
|
||||
}
|
||||
previous = p;
|
||||
}
|
||||
Some(Self { knots })
|
||||
Some(Self { knots, chroma: NO_CHROMA })
|
||||
}
|
||||
|
||||
/// The curve with chroma scales at display luminance 0, 1/7 … 1 (each finite, 0..=4).
|
||||
pub fn with_chroma(self, chroma: [f32; CHROMA_N]) -> Option<Self> {
|
||||
chroma.iter().all(|k| k.is_finite() && (0.0..=4.0).contains(k)).then_some(Self { chroma, ..self })
|
||||
}
|
||||
|
||||
pub fn chroma(&self) -> &[f32; CHROMA_N] {
|
||||
&self.chroma
|
||||
}
|
||||
|
||||
pub fn apply(&self, y: f32) -> f32 {
|
||||
@@ -68,6 +90,7 @@ impl CameraTone {
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ToneMap {
|
||||
lut: Vec<f32>,
|
||||
chroma: [f32; CHROMA_N],
|
||||
}
|
||||
|
||||
impl ToneMap {
|
||||
@@ -81,7 +104,7 @@ impl ToneMap {
|
||||
if neutral { y } else { adjustment.apply(y) }
|
||||
})
|
||||
.collect();
|
||||
ToneMap { lut }
|
||||
ToneMap { lut, chroma: curve.chroma }
|
||||
}
|
||||
/// `contrast`, `whites`, `blacks` in −100..100 (Lightroom slider units).
|
||||
pub fn new(contrast: f64, whites: f64, blacks: f64) -> ToneMap {
|
||||
@@ -113,7 +136,7 @@ impl ToneMap {
|
||||
o.clamp(0.0, 1.0)
|
||||
})
|
||||
.collect();
|
||||
ToneMap { lut }
|
||||
ToneMap { lut, chroma: NO_CHROMA }
|
||||
}
|
||||
|
||||
/// Tone map for display-referred sources: identity at neutral settings.
|
||||
@@ -147,7 +170,7 @@ impl ToneMap {
|
||||
o.clamp(0.0, 1.0)
|
||||
})
|
||||
.collect();
|
||||
ToneMap { lut }
|
||||
ToneMap { lut, chroma: NO_CHROMA }
|
||||
}
|
||||
|
||||
/// The table (`LUT_N` entries, see [`ToneMap::apply`]).
|
||||
@@ -155,6 +178,24 @@ impl ToneMap {
|
||||
&self.lut
|
||||
}
|
||||
|
||||
/// The chroma curve (`CHROMA_N` entries, see [`ToneMap::chroma_scale`]).
|
||||
pub fn chroma_lut(&self) -> &[f32] {
|
||||
&self.chroma
|
||||
}
|
||||
|
||||
/// Chroma scale at display luminance `o` (exactly 1 everywhere unless a camera look sets it).
|
||||
#[inline]
|
||||
pub fn chroma_scale(&self, o: f32) -> f32 {
|
||||
if !o.is_finite() {
|
||||
return 1.0;
|
||||
}
|
||||
let f = o.clamp(0.0, 1.0) * (CHROMA_N - 1) as f32;
|
||||
let i = (f as usize).min(CHROMA_N - 2);
|
||||
let t = f - i as f32;
|
||||
let (a, b) = (self.chroma.get(i).copied().unwrap_or(1.0), self.chroma.get(i + 1).copied().unwrap_or(1.0));
|
||||
if a == b { a } else { a + (b - a) * t }
|
||||
}
|
||||
|
||||
/// Scene luminance → display-linear luminance.
|
||||
#[inline]
|
||||
pub fn apply(&self, y: f32) -> f32 {
|
||||
@@ -179,6 +220,36 @@ fn smooth(e0: f32, e1: f32, x: f32) -> f32 {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn knots() -> [[f32; 2]; 32] {
|
||||
std::array::from_fn(|i| {
|
||||
let x = 0.004 * 1.18f32.powi(i as i32);
|
||||
[x, 1.0 - (-2.0 * x).exp()]
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chroma_curve_is_identity_unless_set_and_survives_serde() {
|
||||
let plain = CameraTone::new(knots()).unwrap();
|
||||
let map = ToneMap::camera(&plain, 0.0, 0.0, 0.0);
|
||||
assert!([0.0, 0.3, 0.77, 1.0, 2.0].iter().all(|o| map.chroma_scale(*o) == 1.0));
|
||||
assert!([0.0, 0.5, 1.0].iter().all(|o| ToneMap::new(0.0, 0.0, 0.0).chroma_scale(*o) == 1.0));
|
||||
// smart previews written before the chroma curve existed still load (identity)
|
||||
let old = serde_json::json!({ "knots": knots() });
|
||||
assert_eq!(serde_json::from_value::<CameraTone>(old).unwrap(), plain);
|
||||
let tone = plain.with_chroma([1.4, 1.3, 1.1, 1.0, 0.7, 0.4, 0.25, 0.2]).unwrap();
|
||||
let back: CameraTone = serde_json::from_value(serde_json::to_value(tone).unwrap()).unwrap();
|
||||
assert_eq!(back, tone);
|
||||
let map = ToneMap::camera(&tone, 0.0, 0.0, 0.0);
|
||||
assert!((map.chroma_scale(0.0) - 1.4).abs() < 1e-6 && (map.chroma_scale(1.0) - 0.2).abs() < 1e-6);
|
||||
assert!((map.chroma_scale(0.5 / 7.0) - 1.35).abs() < 1e-5, "interpolates between nodes");
|
||||
assert_eq!(map.chroma_scale(f32::NAN), 1.0);
|
||||
// hostile values are rejected, also when deserialized
|
||||
assert!(plain.with_chroma([f32::NAN; CHROMA_N]).is_none());
|
||||
assert!(plain.with_chroma([-1.0; CHROMA_N]).is_none());
|
||||
let bad = serde_json::json!({ "knots": knots(), "chroma": [9.0, 1, 1, 1, 1, 1, 1, 1] });
|
||||
assert!(serde_json::from_value::<CameraTone>(bad).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn camera_curve_preserves_black_and_extends_headroom() {
|
||||
let knots = std::array::from_fn(|i| {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! DNG (Adobe Digital Negative Specification 1.7): raw IFD selection, pixel data (via [`crate::tiffraw`]),
|
||||
//! linearization, black/white levels, active area, default crop, CFA description, colour tags, opcode lists.
|
||||
|
||||
use crate::profile::{HsvTable, ProfileLook, ToneCurve};
|
||||
use crate::tiffraw::{Packing, read_image_in};
|
||||
use crate::{BlackLevel, Cfa, ColorData, Mat3, Mode, RawData, RawError, RawFormat, RawImage, Rect, Result, opcodes};
|
||||
use lightcraft_color::Xy;
|
||||
@@ -45,6 +46,27 @@ pub(crate) fn color_data(ifd0: &Ifd, raw: &Ifd) -> ColorData {
|
||||
as_shot_white_xy: get(t::AS_SHOT_WHITE_XY).filter(|v| v.len() == 2 && v[0] > 0.0 && v[1] > 0.0).map(|v| Xy::new(v[0], v[1])),
|
||||
baseline_exposure: get(t::BASELINE_EXPOSURE).and_then(|v| v.first().copied()).filter(|v| v.is_finite()).unwrap_or(0.0)
|
||||
+ get(t::BASELINE_EXPOSURE_OFFSET).and_then(|v| v.first().copied()).filter(|v| v.is_finite()).unwrap_or(0.0),
|
||||
profile: profile_look(ifd0, raw),
|
||||
}
|
||||
}
|
||||
|
||||
/// The profile look tags (`ProfileHueSatMap*`, `ProfileLookTable*`, `ProfileToneCurve`); malformed
|
||||
/// ones are ignored. Like the colour tags, read from the raw IFD first, else IFD 0.
|
||||
pub(crate) fn profile_look(ifd0: &Ifd, raw: &Ifd) -> ProfileLook {
|
||||
let pick = |tag: u16| if raw.contains(tag) { raw } else { ifd0 };
|
||||
let table = |dims: u16, data: u16, enc: u16| {
|
||||
let ifd = pick(dims);
|
||||
let dims = ifd.u64s(dims)?;
|
||||
let data = ifd.f64s(data)?;
|
||||
HsvTable::from_tags(&dims, &data, ifd.u64s(enc).and_then(|v| v.first().copied()).unwrap_or(0))
|
||||
};
|
||||
ProfileLook {
|
||||
hue_sat_map: [
|
||||
table(t::PROFILE_HUE_SAT_MAP_DIMS, t::PROFILE_HUE_SAT_MAP_DATA_1, t::PROFILE_HUE_SAT_MAP_ENCODING),
|
||||
table(t::PROFILE_HUE_SAT_MAP_DIMS, t::PROFILE_HUE_SAT_MAP_DATA_2, t::PROFILE_HUE_SAT_MAP_ENCODING),
|
||||
],
|
||||
look_table: table(t::PROFILE_LOOK_TABLE_DIMS, t::PROFILE_LOOK_TABLE_DATA, t::PROFILE_LOOK_TABLE_ENCODING),
|
||||
tone_curve: pick(t::PROFILE_TONE_CURVE).f64s(t::PROFILE_TONE_CURVE).and_then(|v| ToneCurve::from_tag(&v)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -179,6 +179,38 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result<Vec<u8>> {
|
||||
ifd.set(t::AS_SHOT_WHITE_XY, rat_vec(&[xy.x, xy.y]));
|
||||
}
|
||||
ifd.set(t::BASELINE_EXPOSURE, Value::SRational(vec![srational(c.baseline_exposure)]));
|
||||
// the source's own profile look travels with its data (conversions, smart previews, merges)
|
||||
let p = &c.profile;
|
||||
let table_tags = |ifd: &mut IfdBuilder, table: &crate::profile::HsvTable, dims: u16, enc: u16| {
|
||||
ifd.set(dims, Value::Long(vec![table.hue_divisions as u32, table.sat_divisions as u32, table.val_divisions as u32]));
|
||||
if table.srgb_value {
|
||||
ifd.set(enc, Value::Long(vec![1]));
|
||||
}
|
||||
};
|
||||
let flat = |table: &crate::profile::HsvTable| Value::Float(table.data.iter().flatten().copied().collect());
|
||||
match &p.hue_sat_map {
|
||||
[Some(a), b] => {
|
||||
table_tags(&mut ifd, a, t::PROFILE_HUE_SAT_MAP_DIMS, t::PROFILE_HUE_SAT_MAP_ENCODING);
|
||||
ifd.set(t::PROFILE_HUE_SAT_MAP_DATA_1, flat(a));
|
||||
if let Some(b) =
|
||||
b.as_ref().filter(|b| (b.hue_divisions, b.sat_divisions, b.val_divisions) == (a.hue_divisions, a.sat_divisions, a.val_divisions))
|
||||
{
|
||||
ifd.set(t::PROFILE_HUE_SAT_MAP_DATA_2, flat(b));
|
||||
}
|
||||
}
|
||||
[None, Some(b)] => {
|
||||
table_tags(&mut ifd, b, t::PROFILE_HUE_SAT_MAP_DIMS, t::PROFILE_HUE_SAT_MAP_ENCODING);
|
||||
ifd.set(t::PROFILE_HUE_SAT_MAP_DATA_1, flat(b));
|
||||
}
|
||||
[None, None] => {}
|
||||
}
|
||||
if let Some(l) = &p.look_table {
|
||||
table_tags(&mut ifd, l, t::PROFILE_LOOK_TABLE_DIMS, t::PROFILE_LOOK_TABLE_ENCODING);
|
||||
ifd.set(t::PROFILE_LOOK_TABLE_DATA, flat(l));
|
||||
}
|
||||
if let Some(curve) = &p.tone_curve {
|
||||
ifd.set(t::PROFILE_TONE_CURVE, Value::Float(curve.points.iter().flatten().copied().collect()));
|
||||
}
|
||||
for (list, tag) in [(&raw.opcodes.list1, t::OPCODE_LIST_1), (&raw.opcodes.list2, t::OPCODE_LIST_2), (&raw.opcodes.list3, t::OPCODE_LIST_3)] {
|
||||
if !list.is_empty() {
|
||||
ifd.set(tag, Value::Undefined(opcodes::write_list(list)));
|
||||
|
||||
@@ -8,7 +8,8 @@
|
||||
//! [`RawImage::develop`] does all of it plus `OpcodeList3` and the default crop; [`RawImage::develop_binned`]
|
||||
//! produces the same at 1/k of the size straight from the mosaic (previews, thumbnails).
|
||||
//! - [`color`] implements the DNG colour model (dual-illuminant interpolation, forward matrices, white balance)
|
||||
//! and produces camera → linear Rec.2020 D65 matrices.
|
||||
//! and produces camera → linear Rec.2020 D65 matrices; [`profile`] reads and applies a DNG's own profile
|
||||
//! look tables and tone curve.
|
||||
//!
|
||||
//! Formats: DNG (uncompressed, lossless JPEG, lossy JPEG (Smart Previews), Deflate incl. floating point, tiled/stripped, CFA and LinearRaw),
|
||||
//! Canon CR2, Nikon NEF/NRW (uncompressed, Huffman lossless / lossy compressed), Sony ARW (uncompressed, ARW2, lossless), Fujifilm RAF (uncompressed Bayer
|
||||
@@ -29,6 +30,7 @@ pub mod highlight;
|
||||
pub mod ljpeg;
|
||||
pub mod opcodes;
|
||||
mod preview;
|
||||
pub mod profile;
|
||||
mod tiffraw;
|
||||
mod unpack;
|
||||
mod vendor;
|
||||
@@ -393,6 +395,10 @@ pub struct ColorData {
|
||||
pub as_shot_white_xy: Option<Xy>,
|
||||
/// EV to add for a "normal" rendering (`BaselineExposure` + `BaselineExposureOffset`).
|
||||
pub baseline_exposure: f64,
|
||||
/// The file's own camera-profile look (`ProfileHueSatMap*`, `ProfileLookTable*`,
|
||||
/// `ProfileToneCurve`), applied by [`color`]'s users at render time.
|
||||
#[serde(default)]
|
||||
pub profile: profile::ProfileLook,
|
||||
}
|
||||
|
||||
/// A decoded raw image.
|
||||
|
||||
@@ -96,10 +96,20 @@ pub fn embedded_preview(bytes: &[u8]) -> Option<Vec<u8>> {
|
||||
found.into_iter().filter(|s| is_dct_jpeg(s)).max_by_key(|s| s.len()).map(|s| trim_eoi(s).to_vec())
|
||||
}
|
||||
|
||||
/// Canon CR3: the full-size JPEG track (see [`lightcraft_meta::cr3`]), else the `PRVW` / `THMB` boxes.
|
||||
fn cr3_preview(bytes: &[u8]) -> Option<&[u8]> {
|
||||
let full = lightcraft_meta::cr3::parse_cr3(bytes)
|
||||
.and_then(|c| c.tracks.iter().find(|t| t.kind == lightcraft_meta::cr3::Cr3TrackKind::Jpeg).and_then(|t| t.data));
|
||||
if let Some(j) = full.and_then(|(at, len)| bytes.get(at..at.checked_add(len)?)).filter(|j| is_dct_jpeg(j)) {
|
||||
return Some(j);
|
||||
}
|
||||
cr3_preview_boxes(bytes)
|
||||
}
|
||||
|
||||
/// Canon CR3 (ISO base media file): the `PRVW` box (Laurent Clévy's CR3 notes; layout confirmed on a CC0
|
||||
/// sample) is `u32 size, "PRVW", u32 0, u16 ?, u16 width, u16 height, u16 ?, u32 jpeg length, JPEG`; the smaller
|
||||
/// `THMB` box has the same shape. Returns the larger valid one.
|
||||
fn cr3_preview(bytes: &[u8]) -> Option<&[u8]> {
|
||||
fn cr3_preview_boxes(bytes: &[u8]) -> Option<&[u8]> {
|
||||
let mut best: Option<&[u8]> = None;
|
||||
for tag in [b"PRVW", b"THMB"] {
|
||||
let mut from = 0;
|
||||
@@ -180,4 +190,32 @@ mod tests {
|
||||
let _ = embedded_preview(&cr3[..n]);
|
||||
}
|
||||
}
|
||||
|
||||
/// CR3 with a full-size JPEG track: that JPEG wins over the smaller `PRVW` box.
|
||||
#[test]
|
||||
fn cr3_prefers_the_full_size_jpeg_track() {
|
||||
let bx = |kind: &[u8; 4], body: &[u8]| -> Vec<u8> { [&((body.len() + 8) as u32).to_be_bytes()[..], kind, body].concat() };
|
||||
let full = |kind: &[u8; 4], body: &[u8]| bx(kind, &[&[0u8; 4][..], body].concat());
|
||||
let (small, big) = (fake_jpeg(300), fake_jpeg(3000));
|
||||
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
|
||||
let mut prvw = b"\0\0\0\0\0\x01\x06\x54\x04\x38\0\x01".to_vec();
|
||||
prvw.extend_from_slice(&(small.len() as u32).to_be_bytes());
|
||||
prvw.extend_from_slice(&small);
|
||||
file.extend(bx(b"PRVW", &prvw));
|
||||
// CRAW sample entry: 82 bytes, then a JPEG child box; one sample at `at`
|
||||
let mut craw = vec![0u8; 82];
|
||||
craw.extend(bx(b"JPEG", &[0; 4]));
|
||||
let at = 2048u64;
|
||||
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat());
|
||||
let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat());
|
||||
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
|
||||
let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat()))));
|
||||
file.extend(bx(b"moov", &trak));
|
||||
file.resize(at as usize, 0);
|
||||
file.extend_from_slice(&big);
|
||||
assert_eq!(embedded_preview(&file).unwrap(), big);
|
||||
// a track pointing past the end falls back to PRVW
|
||||
file.truncate(at as usize + 100);
|
||||
assert_eq!(embedded_preview(&file).unwrap(), small);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
//! DNG camera-profile look data carried by the file itself (Adobe DNG Specification 1.6/1.7,
|
||||
//! chapter 6 and the `ProfileHueSatMap*`, `ProfileLookTable*` and `ProfileToneCurve` tags).
|
||||
//!
|
||||
//! - A hue/saturation/value table holds, for a grid of (value, hue, saturation) inputs, a hue shift
|
||||
//! in degrees, a saturation scale and a value scale (value-major, hue-middle, saturation-minor).
|
||||
//! Hue samples are spread evenly around the circle with wrap-around, saturation and value samples
|
||||
//! evenly over 0..=1; a dimension with one division is constant. Lookups are tri-linear.
|
||||
//! - Tables are applied in HSV of linear ProPhoto RGB (D50). With encoding 1 ("sRGB") the value
|
||||
//! axis is indexed through the sRGB transfer curve (3D tables only).
|
||||
//! - `ProfileHueSatMapData1/2` follow the colour matrices (interpolated by the same illuminant
|
||||
//! weight); `ProfileLookTableData` comes after exposure compensation and before any tone curve.
|
||||
//! - `ProfileToneCurve`: (input, output) pairs in linear gamma from (0, 0) to (1, 1).
|
||||
//!
|
||||
//! The data is read from the user's own file at run time; nothing here ships profile data.
|
||||
//! Values above 1.0 (highlight headroom) keep their headroom: the table's value scale only ever
|
||||
//! lowers them, so a lookup can't clip what the scene-referred pipeline still needs.
|
||||
|
||||
use lightcraft_color::transfer::{linear_to_srgb, srgb_to_linear};
|
||||
use lightcraft_color::{D50, D65, Mat3, PROPHOTO, REC2020, bradford};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Upper bound on table entries (Adobe profiles use ≤ 90 × 30 × 16; guards header-driven sizes).
|
||||
const MAX_ENTRIES: usize = 1 << 20;
|
||||
|
||||
/// A DNG hue/saturation/value mapping table (`ProfileHueSatMapData*` / `ProfileLookTableData`).
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct HsvTable {
|
||||
pub hue_divisions: usize,
|
||||
pub sat_divisions: usize,
|
||||
pub val_divisions: usize,
|
||||
/// `(hue shift °, saturation scale, value scale)`, value-major, hue-middle, saturation-minor.
|
||||
pub data: Vec<[f32; 3]>,
|
||||
/// The value axis is indexed through the sRGB encoding curve (`…Encoding` = 1).
|
||||
pub srgb_value: bool,
|
||||
}
|
||||
|
||||
impl HsvTable {
|
||||
/// A table from its `…Dims` (hue, saturation, value divisions), data floats and encoding.
|
||||
/// `None` when the dimensions are out of range, the data length doesn't match or a value is
|
||||
/// not finite.
|
||||
pub fn from_tags(dims: &[u64], data: &[f64], encoding: u64) -> Option<HsvTable> {
|
||||
let [h, s, v] = <[u64; 3]>::try_from(dims).ok()?;
|
||||
let (h, s, v) = (usize::try_from(h).ok()?, usize::try_from(s).ok()?, usize::try_from(v).ok()?);
|
||||
if h < 1 || s < 2 || v < 1 || h > 4096 || s > 4096 || v > 4096 {
|
||||
return None;
|
||||
}
|
||||
let n = h.checked_mul(s)?.checked_mul(v)?;
|
||||
if n > MAX_ENTRIES || data.len() != n * 3 || data.iter().any(|x| !x.is_finite()) {
|
||||
return None;
|
||||
}
|
||||
let data = data.as_chunks::<3>().0.iter().map(|c| [c[0] as f32, (c[1] as f32).max(0.0), (c[2] as f32).max(0.0)]).collect();
|
||||
Some(HsvTable { hue_divisions: h, sat_divisions: s, val_divisions: v, data, srgb_value: encoding == 1 && v > 1 })
|
||||
}
|
||||
|
||||
/// The entry-wise blend `g · a + (1 − g) · b` of two tables of the same shape (`None` otherwise).
|
||||
pub fn blend(a: &HsvTable, b: &HsvTable, g: f64) -> Option<HsvTable> {
|
||||
if (a.hue_divisions, a.sat_divisions, a.val_divisions) != (b.hue_divisions, b.sat_divisions, b.val_divisions) {
|
||||
return None;
|
||||
}
|
||||
let g = g.clamp(0.0, 1.0) as f32;
|
||||
let data = a.data.iter().zip(&b.data).map(|(x, y)| std::array::from_fn(|i| g * x[i] + (1.0 - g) * y[i])).collect();
|
||||
Some(HsvTable { data, ..a.clone() })
|
||||
}
|
||||
|
||||
fn entry(&self, v: usize, h: usize, s: usize) -> [f32; 3] {
|
||||
self.data.get((v * self.hue_divisions + h) * self.sat_divisions + s).copied().unwrap_or([0.0, 1.0, 1.0])
|
||||
}
|
||||
|
||||
/// Tri-linear lookup at hue `h` (degrees), saturation `s` and value `v` (table coordinates).
|
||||
pub fn lookup(&self, h: f32, s: f32, v: f32) -> [f32; 3] {
|
||||
let hf = (h.rem_euclid(360.0) / 360.0 * self.hue_divisions as f32).clamp(0.0, self.hue_divisions as f32);
|
||||
let h0 = (hf as usize).min(self.hue_divisions - 1);
|
||||
let h1 = (h0 + 1) % self.hue_divisions;
|
||||
let th = (hf - h0 as f32).clamp(0.0, 1.0);
|
||||
let axis = |x: f32, n: usize| {
|
||||
let f = x.clamp(0.0, 1.0) * (n - 1) as f32;
|
||||
let i0 = (f as usize).min(n.saturating_sub(2));
|
||||
let i1 = (i0 + 1).min(n - 1);
|
||||
(i0, i1, (f - i0 as f32).clamp(0.0, 1.0))
|
||||
};
|
||||
let (s0, s1, ts) = axis(s, self.sat_divisions);
|
||||
let (v0, v1, tv) = if self.val_divisions > 1 { axis(v, self.val_divisions) } else { (0, 0, 0.0) };
|
||||
let lerp = |a: [f32; 3], b: [f32; 3], t: f32| -> [f32; 3] { std::array::from_fn(|i| a[i] + (b[i] - a[i]) * t) };
|
||||
let plane = |vi: usize| {
|
||||
let lo = lerp(self.entry(vi, h0, s0), self.entry(vi, h0, s1), ts);
|
||||
let hi = lerp(self.entry(vi, h1, s0), self.entry(vi, h1, s1), ts);
|
||||
lerp(lo, hi, th)
|
||||
};
|
||||
if v0 == v1 { plane(v0) } else { lerp(plane(v0), plane(v1), tv) }
|
||||
}
|
||||
|
||||
/// Apply the table to one linear ProPhoto RGB colour.
|
||||
pub fn apply(&self, rgb: [f32; 3]) -> [f32; 3] {
|
||||
let max = rgb[0].max(rgb[1]).max(rgb[2]);
|
||||
if max.is_nan() || max <= 0.0 || !max.is_finite() {
|
||||
return rgb;
|
||||
}
|
||||
let min = rgb[0].min(rgb[1]).min(rgb[2]);
|
||||
let d = max - min;
|
||||
let hue = hue_of(rgb, max, d);
|
||||
let sat = d / max;
|
||||
let v_index = if self.srgb_value { linear_to_srgb(max.min(1.0)) } else { max };
|
||||
let [shift, ss, vs] = self.lookup(hue, sat, v_index);
|
||||
// saturation and value: scaled and clipped to 1.0 (DNG chapter 6), except that inputs
|
||||
// already above 1 (out-of-ProPhoto colours, highlight headroom) are only ever lowered.
|
||||
let s2 = limited(sat, ss);
|
||||
let v2 = if self.srgb_value && max <= 1.0 {
|
||||
srgb_to_linear((linear_to_srgb(max) * vs).min(1.0))
|
||||
} else if self.srgb_value {
|
||||
max * srgb_to_linear(vs.min(1.0)).min(1.0)
|
||||
} else {
|
||||
limited(max, vs)
|
||||
};
|
||||
hsv_to_rgb(hue + shift, s2, v2)
|
||||
}
|
||||
}
|
||||
|
||||
/// `x · k` clipped to 1, for `x ≤ 1`; above 1, only scale factors below 1 apply (continuous at 1).
|
||||
fn limited(x: f32, k: f32) -> f32 {
|
||||
if x <= 1.0 { (x * k).min(1.0) } else { x * k.min(1.0) }
|
||||
}
|
||||
|
||||
fn hue_of(c: [f32; 3], max: f32, d: f32) -> f32 {
|
||||
if d <= 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
let h = if max == c[0] {
|
||||
((c[1] - c[2]) / d).rem_euclid(6.0)
|
||||
} else if max == c[1] {
|
||||
(c[2] - c[0]) / d + 2.0
|
||||
} else {
|
||||
(c[0] - c[1]) / d + 4.0
|
||||
};
|
||||
h * 60.0
|
||||
}
|
||||
|
||||
/// HSV → RGB for any saturation ≥ 0 (above 1 yields negative components, the inverse of
|
||||
/// [`hue_of`] for out-of-gamut inputs).
|
||||
fn hsv_to_rgb(h: f32, s: f32, v: f32) -> [f32; 3] {
|
||||
let hp = h.rem_euclid(360.0) / 60.0;
|
||||
let sector = (hp as u32).min(5);
|
||||
let f = hp - sector as f32;
|
||||
let (p, q, t) = (v * (1.0 - s), v * (1.0 - s * f), v * (1.0 - s * (1.0 - f)));
|
||||
match sector {
|
||||
0 => [v, t, p],
|
||||
1 => [q, v, p],
|
||||
2 => [p, v, t],
|
||||
3 => [p, q, v],
|
||||
4 => [t, p, v],
|
||||
_ => [v, p, q],
|
||||
}
|
||||
}
|
||||
|
||||
/// A DNG tone curve (`ProfileToneCurve`): linear-gamma (input, output) pairs, increasing input.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ToneCurve {
|
||||
pub points: Vec<[f32; 2]>,
|
||||
}
|
||||
|
||||
impl ToneCurve {
|
||||
/// From the tag's flat float pairs. `None` unless there are 2..=65536 finite pairs within
|
||||
/// 0..=1 with strictly increasing inputs.
|
||||
pub fn from_tag(v: &[f64]) -> Option<ToneCurve> {
|
||||
if !v.len().is_multiple_of(2) || v.len() < 4 || v.len() > 131_072 {
|
||||
return None;
|
||||
}
|
||||
let points: Vec<[f32; 2]> = v.as_chunks::<2>().0.iter().map(|p| [p[0] as f32, p[1] as f32]).collect();
|
||||
let ok = points.iter().all(|p| p.iter().all(|x| x.is_finite() && (0.0..=1.0).contains(x))) && points.windows(2).all(|w| w[1][0] > w[0][0]);
|
||||
ok.then_some(ToneCurve { points })
|
||||
}
|
||||
|
||||
/// The curve at `x` (0..=1): monotone cubic (Fritsch–Carlson) through the samples.
|
||||
pub fn eval(&self, x: f32) -> f32 {
|
||||
let p = &self.points;
|
||||
let n = p.len();
|
||||
let x = x.clamp(p[0][0], p[n - 1][0]);
|
||||
let i = p.partition_point(|q| q[0] <= x).clamp(1, n - 1) - 1;
|
||||
let slope = |k: usize| (p[k + 1][1] - p[k][1]) / (p[k + 1][0] - p[k][0]);
|
||||
let tangent = |k: usize| -> f32 {
|
||||
if k == 0 {
|
||||
return slope(0);
|
||||
}
|
||||
if k == n - 1 {
|
||||
return slope(n - 2);
|
||||
}
|
||||
let (a, b) = (slope(k - 1), slope(k));
|
||||
if a * b <= 0.0 { 0.0 } else { 2.0 / (1.0 / a + 1.0 / b) }
|
||||
};
|
||||
let (x0, x1) = (p[i][0], p[i + 1][0]);
|
||||
let h = x1 - x0;
|
||||
let t = (x - x0) / h;
|
||||
let (t2, t3) = (t * t, t * t * t);
|
||||
let y = (2.0 * t3 - 3.0 * t2 + 1.0) * p[i][1]
|
||||
+ (t3 - 2.0 * t2 + t) * h * tangent(i)
|
||||
+ (-2.0 * t3 + 3.0 * t2) * p[i + 1][1]
|
||||
+ (t3 - t2) * h * tangent(i + 1);
|
||||
y.clamp(0.0, 1.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// The DNG profile look tags of a file (all optional).
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ProfileLook {
|
||||
/// `ProfileHueSatMapData1/2` (same shape, `ProfileHueSatMapDims`).
|
||||
pub hue_sat_map: [Option<HsvTable>; 2],
|
||||
/// `ProfileLookTableData`.
|
||||
pub look_table: Option<HsvTable>,
|
||||
/// `ProfileToneCurve`.
|
||||
pub tone_curve: Option<ToneCurve>,
|
||||
}
|
||||
|
||||
impl ProfileLook {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.hue_sat_map.iter().all(Option::is_none) && self.look_table.is_none() && self.tone_curve.is_none()
|
||||
}
|
||||
|
||||
/// The hue/saturation map for calibration-1 weight `g` (one table: used for any white).
|
||||
pub fn hue_sat_for(&self, g: f64) -> Option<HsvTable> {
|
||||
match &self.hue_sat_map {
|
||||
[Some(a), Some(b)] => HsvTable::blend(a, b, g).or_else(|| Some(a.clone())),
|
||||
[Some(a), None] | [None, Some(a)] => Some(a.clone()),
|
||||
[None, None] => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The tables resolved for one white balance, applied to linear Rec.2020 D65 pixels.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ProfileTables {
|
||||
hue_sat: Option<HsvTable>,
|
||||
look: Option<HsvTable>,
|
||||
to_prophoto: [[f32; 3]; 3],
|
||||
from_prophoto: [[f32; 3]; 3],
|
||||
}
|
||||
|
||||
impl ProfileTables {
|
||||
/// `None` when the profile has no hue/saturation map and no look table.
|
||||
pub fn new(look: &ProfileLook, illuminant_weight: f64) -> Option<ProfileTables> {
|
||||
let hue_sat = look.hue_sat_for(illuminant_weight);
|
||||
if hue_sat.is_none() && look.look_table.is_none() {
|
||||
return None;
|
||||
}
|
||||
let to: Mat3 = PROPHOTO.from_xyz().mul(&bradford(D65, D50)).mul(&REC2020.to_xyz());
|
||||
let from = to.inverse()?;
|
||||
Some(ProfileTables { hue_sat, look: look.look_table.clone(), to_prophoto: to.to_f32(), from_prophoto: from.to_f32() })
|
||||
}
|
||||
|
||||
/// Hue/saturation map, then `gain` (exposure compensation), then the look table:
|
||||
/// linear Rec.2020 D65 in, linear Rec.2020 D65 out.
|
||||
#[inline]
|
||||
pub fn apply(&self, rgb: [f32; 3], gain: f32) -> [f32; 3] {
|
||||
let m = &self.to_prophoto;
|
||||
let mut p: [f32; 3] = std::array::from_fn(|i| m[i][0] * rgb[0] + m[i][1] * rgb[1] + m[i][2] * rgb[2]);
|
||||
if let Some(t) = &self.hue_sat {
|
||||
p = t.apply(p);
|
||||
}
|
||||
p = p.map(|v| v * gain);
|
||||
if let Some(t) = &self.look {
|
||||
p = t.apply(p);
|
||||
}
|
||||
let m = &self.from_prophoto;
|
||||
std::array::from_fn(|i| m[i][0] * p[0] + m[i][1] * p[1] + m[i][2] * p[2])
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn identity(h: usize, s: usize, v: usize) -> HsvTable {
|
||||
HsvTable { hue_divisions: h, sat_divisions: s, val_divisions: v, data: vec![[0.0, 1.0, 1.0]; h * s * v], srgb_value: false }
|
||||
}
|
||||
|
||||
fn close(a: [f32; 3], b: [f32; 3], tol: f32) -> bool {
|
||||
a.iter().zip(&b).all(|(x, y)| (x - y).abs() <= tol)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_table_is_identity() {
|
||||
let t = identity(6, 3, 1);
|
||||
for c in [[0.2, 0.5, 0.1], [0.9, 0.9, 0.9], [0.0, 0.0, 0.4], [3.0, 1.0, 0.5], [0.3, -0.05, 0.2]] {
|
||||
assert!(close(t.apply(c), c, 1e-5), "{c:?} → {:?}", t.apply(c));
|
||||
}
|
||||
assert_eq!(t.apply([0.0; 3]), [0.0; 3]);
|
||||
assert_eq!(t.apply([-1.0, -2.0, -0.5]), [-1.0, -2.0, -0.5]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_tags_and_rejects_bad_shapes() {
|
||||
let data: Vec<f64> = (0..4).flat_map(|_| [0.0, 1.0, 1.0]).collect();
|
||||
assert!(HsvTable::from_tags(&[2, 2, 1], &data, 0).is_some());
|
||||
assert!(HsvTable::from_tags(&[2, 2], &data, 0).is_none());
|
||||
assert!(HsvTable::from_tags(&[2, 1, 2], &data, 0).is_none(), "saturation needs ≥ 2 divisions");
|
||||
assert!(HsvTable::from_tags(&[2, 2, 2], &data, 0).is_none(), "length mismatch");
|
||||
assert!(HsvTable::from_tags(&[u64::MAX, 2, 2], &data, 0).is_none());
|
||||
let mut nan = data.clone();
|
||||
nan[1] = f64::NAN;
|
||||
assert!(HsvTable::from_tags(&[2, 2, 1], &nan, 0).is_none());
|
||||
// the sRGB encoding only applies to 3D tables
|
||||
assert!(!HsvTable::from_tags(&[2, 2, 1], &data, 1).unwrap().srgb_value);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn saturation_scale_follows_the_table_layout() {
|
||||
// 4 hues (0°, 90°, 180°, 270°), 2 saturations; saturation-minor: entry (h, s) at h·2 + s.
|
||||
let mut t = identity(4, 2, 1);
|
||||
t.data[1] = [0.0, 1.5, 1.0]; // hue 0°, saturation 1 → ×1.5
|
||||
let red = [0.5, 0.4, 0.4]; // hue 0°, saturation 0.2
|
||||
let out = t.apply(red);
|
||||
// half-way along the saturation axis is ×1.1 (0.2 → 0.22 sat), value kept
|
||||
let sat = (out[0] - out[1].min(out[2])) / out[0];
|
||||
assert!((sat - 0.2 * 1.1).abs() < 1e-4, "{out:?}");
|
||||
assert!((out[0] - 0.5).abs() < 1e-6);
|
||||
// other hues are untouched (hue 180°)
|
||||
let cyan = [0.3, 0.4, 0.4];
|
||||
assert!(close(t.apply(cyan), cyan, 1e-6));
|
||||
// saturation clips to 1.0
|
||||
assert!(t.apply([1.0, 0.1, 0.1]).iter().all(|v| *v >= -1e-6));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hue_wraps_and_value_scales() {
|
||||
// 2 hues: entries at 0° and 180°, hue shift +20° at 0°, value ×0.5 at 180°.
|
||||
let mut t = identity(2, 2, 1);
|
||||
t.data = vec![[20.0, 1.0, 1.0], [20.0, 1.0, 1.0], [0.0, 1.0, 0.5], [0.0, 1.0, 0.5]];
|
||||
// 350° sits between the last sample (180°) and the first (0° = 360°), 10/180 from 360
|
||||
assert!((t.lookup(350.0, 0.5, 0.5)[0] - 20.0 * (170.0 / 180.0)).abs() < 1e-3);
|
||||
assert!((t.lookup(180.0, 0.5, 0.5)[2] - 0.5).abs() < 1e-6);
|
||||
// headroom above 1 only goes down
|
||||
let bright = [2.0, 2.4, 2.4];
|
||||
let out = t.apply(bright);
|
||||
assert!((out[1] - 1.2).abs() < 1e-5, "{out:?}");
|
||||
let mut up = identity(1, 2, 1);
|
||||
up.data = vec![[0.0, 1.0, 2.0]; 2];
|
||||
assert!((up.apply([0.4, 0.4, 0.4])[0] - 0.8).abs() < 1e-6);
|
||||
assert!((up.apply([0.8, 0.8, 0.8])[0] - 1.0).abs() < 1e-6, "clipped to 1");
|
||||
assert!((up.apply([3.0, 3.0, 3.0])[0] - 3.0).abs() < 1e-6, "headroom kept");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn value_axis_and_srgb_encoding() {
|
||||
// 1 hue, 2 sats, 2 values: saturation ×2 at value 1 only
|
||||
let mut t = identity(1, 2, 2);
|
||||
t.data[3] = [0.0, 2.0, 1.0];
|
||||
let sat = |c: [f32; 3]| (c[0] - c[2]) / c[0];
|
||||
let dark = [0.25, 0.2, 0.2]; // V 0.25, S 0.2
|
||||
// linear: 1/4 of the way → S scale at s=0.2: lerp(1, lerp(1,2,0.2)=1.2, 0.25) = 1.05
|
||||
assert!((sat(t.apply(dark)) - 0.2 * 1.05).abs() < 1e-4);
|
||||
t.srgb_value = true;
|
||||
let k = linear_to_srgb(0.25); // ≈ 0.537
|
||||
let want = 0.2 * (1.0 + 0.2 * k);
|
||||
assert!((sat(t.apply(dark)) - want).abs() < 1e-4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blend_and_tables_round_trip_through_prophoto() {
|
||||
let a = identity(2, 2, 1);
|
||||
let mut b = identity(2, 2, 1);
|
||||
b.data[1] = [10.0, 2.0, 1.0];
|
||||
let m = HsvTable::blend(&a, &b, 0.25).unwrap();
|
||||
assert_eq!(m.data[1], [7.5, 1.75, 1.0]);
|
||||
assert!(HsvTable::blend(&a, &identity(3, 2, 1), 0.5).is_none());
|
||||
let look = ProfileLook { hue_sat_map: [Some(a.clone()), None], look_table: Some(a), tone_curve: None };
|
||||
let t = ProfileTables::new(&look, 0.5).unwrap();
|
||||
let c = [0.3, 0.2, 0.1];
|
||||
assert!(close(t.apply(c, 2.0), [0.6, 0.4, 0.2], 1e-4), "{:?}", t.apply(c, 2.0));
|
||||
assert!(ProfileTables::new(&ProfileLook::default(), 0.5).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tone_curve_interpolates_monotonically() {
|
||||
assert!(ToneCurve::from_tag(&[0.0, 0.0]).is_none());
|
||||
assert!(ToneCurve::from_tag(&[0.0, 0.0, 0.5, 0.6, 0.4, 0.7, 1.0, 1.0]).is_none(), "inputs must increase");
|
||||
assert!(ToneCurve::from_tag(&[0.0, 0.0, 1.0, 1.5]).is_none());
|
||||
let c = ToneCurve::from_tag(&[0.0, 0.0, 0.25, 0.4, 0.5, 0.7, 1.0, 1.0]).unwrap();
|
||||
assert_eq!(c.eval(0.0), 0.0);
|
||||
assert!((c.eval(0.25) - 0.4).abs() < 1e-6);
|
||||
assert!((c.eval(1.0) - 1.0).abs() < 1e-6);
|
||||
let mut last = 0.0;
|
||||
for i in 0..=100 {
|
||||
let y = c.eval(i as f32 / 100.0);
|
||||
assert!(y >= last - 1e-6, "monotone at {i}");
|
||||
last = y;
|
||||
}
|
||||
assert!(c.eval(0.1) > 0.1 && c.eval(0.1) < 0.4);
|
||||
}
|
||||
}
|
||||
Vendored
+282
-9
@@ -14,6 +14,12 @@
|
||||
//! a 128-bit little-endian block: 11-bit max, 11-bit min, 4-bit index of the max, 4-bit index of the min and 14
|
||||
//! seven-bit deltas above the min, scaled by the smallest shift that fits `max − min` into 7 bits.
|
||||
//!
|
||||
//! White balance: the raw IFD's `0x7313` levels when present (bodies from about 2017 on); otherwise the
|
||||
//! `WB_RGBLevels` of the maker note's enciphered `Tag2010` block (see [`DECIPHER`] and [`TAG2010_WB`]). Without
|
||||
//! either, an ARW opened with unit multipliers, i.e. a strong green cast (issue #148). Black level: the raw IFD's
|
||||
//! `0x7310`, else the level stored in the encrypted `SR2SubIFD` (see [`SR2_BLACK_AT`]; 800 rather than the
|
||||
//! default 512 on 1″-sensor bodies such as the RX100 series), else 512 (14-bit) / 128 (12-bit).
|
||||
//!
|
||||
//! Also: uncompressed 16-bit ARW, and lossless-compressed ARW (Compression 7, ILCE-7M4 and later): LJ92 tiles whose
|
||||
//! frames hold one 2×2 CFA cell per four-component sample ([`read_quad_tiles`]). Other lossless-JPEG layouts go
|
||||
//! through the generic TIFF path.
|
||||
@@ -23,7 +29,7 @@ use crate::{BlackLevel, Cfa, ColorData, Mode, OpcodeLists, RawData, RawError, Ra
|
||||
use lightcraft_geom::Orientation;
|
||||
use lightcraft_tiff::image::{Chunk, ImageInfo, Layout, chunk_bytes};
|
||||
use lightcraft_tiff::tags::{self as t, photometric};
|
||||
use lightcraft_tiff::{Ifd, Tiff};
|
||||
use lightcraft_tiff::{Ifd, Tiff, makernote};
|
||||
use rayon::prelude::*;
|
||||
|
||||
const TONE_CURVE: u16 = 0x7010;
|
||||
@@ -31,6 +37,137 @@ const BLACK_LEVEL: u16 = 0x7310;
|
||||
const WB_RGGB: u16 = 0x7313;
|
||||
const CROP_TOP_LEFT: u16 = 0x74c7;
|
||||
const CROP_SIZE: u16 = 0x74c8;
|
||||
/// Maker-note tags (ExifTool Sony tag names): the enciphered `Tag2010` block and `FullImageSize` (height, width).
|
||||
const MN_TAG2010: u16 = 0x2010;
|
||||
const MN_FULL_IMAGE_SIZE: u16 = 0xb02b;
|
||||
|
||||
/// Inverse of Sony's maker-note byte substitution. ExifTool's Sony tag documentation states that the data of
|
||||
/// tags `0x2010`, `0x9050` and `0x94xx` "is encrypted by a simple substitution cipher" (no decoder source was
|
||||
/// consulted). The substitution, checked black-box on CC0 raw.pixls.us samples: every byte `p < 249` is stored as
|
||||
/// `p³ mod 249` (a bijection on 0..249, as 3 is coprime to φ(249) = 164) and bytes 249–255 are stored unchanged.
|
||||
/// Verified on 15 files from 13 bodies covering seven `Tag2010` layouts: the deciphered `SonyISO` field encodes
|
||||
/// the Exif ISO (`100 · 2^(16 − v/256)`), the green level is 255–256 throughout, and on the ILCE-7M3 the deciphered
|
||||
/// `WB_RGBLevels` equal the plain raw-IFD `0x7313` levels.
|
||||
const DECIPHER: [u8; 256] = {
|
||||
let mut inv = [0u8; 256];
|
||||
let mut p = 0usize;
|
||||
while p < 256 {
|
||||
let c = if p < 249 { p * p % 249 * p % 249 } else { p };
|
||||
inv[c] = p as u8;
|
||||
p += 1;
|
||||
}
|
||||
inv
|
||||
};
|
||||
|
||||
/// `Tag2010` layouts (ExifTool "Sony Tag2010a" … "Tag2010i" tables): the models each is documented for and the
|
||||
/// byte offset of `WB_RGBLevels` (three `u16`, the as-shot white balance as R, G, B gains; Tag2010g and h share
|
||||
/// the offset). Newer bodies (Tag2010i) also write the levels in plain form in the raw IFD (`0x7313`), which takes
|
||||
/// precedence.
|
||||
const TAG2010_WB: &[(&[&str], usize)] = &[
|
||||
(&["NEX-5N"], 4476),
|
||||
(&["SLT-A65", "SLT-A77", "NEX-7", "NEX-VG20E"], 4480),
|
||||
(&["SLT-A37", "SLT-A57", "NEX-F3"], 4444),
|
||||
(&["DSC-HX10V", "DSC-HX20V", "DSC-HX200V", "DSC-TX66", "DSC-TX200V", "DSC-TX300V", "DSC-WX50", "DSC-WX100", "DSC-WX150"], 4568),
|
||||
(
|
||||
&[
|
||||
"SLT-A58",
|
||||
"SLT-A99",
|
||||
"ILCE-3000",
|
||||
"ILCE-3500",
|
||||
"NEX-3N",
|
||||
"NEX-5R",
|
||||
"NEX-5T",
|
||||
"NEX-6",
|
||||
"NEX-VG30E",
|
||||
"NEX-VG900",
|
||||
"DSC-RX100",
|
||||
"DSC-RX1",
|
||||
"DSC-RX1R",
|
||||
"DSC-HX300",
|
||||
"DSC-HX50V",
|
||||
"DSC-TX30",
|
||||
"DSC-WX60",
|
||||
"DSC-WX200",
|
||||
"DSC-WX300",
|
||||
],
|
||||
4532,
|
||||
),
|
||||
(&["DSC-RX100M2", "DSC-QX10", "DSC-QX100"], 4204),
|
||||
(
|
||||
&[
|
||||
"DSC-HX60V",
|
||||
"DSC-HX350",
|
||||
"DSC-HX400V",
|
||||
"DSC-QX30",
|
||||
"DSC-RX10",
|
||||
"DSC-RX100M3",
|
||||
"DSC-WX220",
|
||||
"DSC-WX350",
|
||||
"ILCE-7",
|
||||
"ILCE-7R",
|
||||
"ILCE-7S",
|
||||
"ILCE-7M2",
|
||||
"ILCE-5000",
|
||||
"ILCE-5100",
|
||||
"ILCE-6000",
|
||||
"ILCE-QX1",
|
||||
"ILCA-68",
|
||||
"ILCA-77M2",
|
||||
"DSC-HX80",
|
||||
"DSC-HX90V",
|
||||
"DSC-RX0",
|
||||
"DSC-RX1RM2",
|
||||
"DSC-RX10M2",
|
||||
"DSC-RX10M3",
|
||||
"DSC-RX100M4",
|
||||
"DSC-RX100M5",
|
||||
"DSC-WX500",
|
||||
"ILCE-6300",
|
||||
"ILCE-6500",
|
||||
"ILCE-7RM2",
|
||||
"ILCE-7SM2",
|
||||
"ILCA-99M2",
|
||||
],
|
||||
612,
|
||||
),
|
||||
(
|
||||
&[
|
||||
"ILCE-6100",
|
||||
"ILCE-6400",
|
||||
"ILCE-6600",
|
||||
"ILCE-7C",
|
||||
"ILCE-7M3",
|
||||
"ILCE-7RM3",
|
||||
"ILCE-7RM4",
|
||||
"ILCE-9",
|
||||
"ILCE-9M2",
|
||||
"DSC-RX0M2",
|
||||
"DSC-RX10M4",
|
||||
"DSC-RX100M6",
|
||||
"DSC-RX100M5A",
|
||||
"DSC-RX100M7",
|
||||
"DSC-HX99",
|
||||
],
|
||||
594,
|
||||
),
|
||||
];
|
||||
|
||||
/// As-shot white balance from the maker note's enciphered `Tag2010` block (bodies that do not write `0x7313` in
|
||||
/// the raw IFD: most ARWs before 2017). `model` is the Exif model; Sony appends a regional "V" to some names
|
||||
/// (SLT-A77V), which the documented lists omit.
|
||||
fn tag2010_wb(model: &str, block: &[u8], order: lightcraft_tiff::ByteOrder) -> Option<[f32; 3]> {
|
||||
let model = model.trim();
|
||||
let &(_, offset) = TAG2010_WB.iter().find(|(models, _)| models.iter().any(|m| model == *m || model.strip_suffix('V') == Some(*m)))?;
|
||||
let bytes: Vec<u8> = block.get(offset..offset + 6)?.iter().map(|&b| DECIPHER[b as usize]).collect();
|
||||
let level = |i: usize| order.read_u16(&bytes, 2 * i).map(f32::from);
|
||||
let (r, g, b) = (level(0)?, level(1)?, level(2)?);
|
||||
// levels are fixed-point gains (green 255–256 on every sample seen); reject implausible values from an unknown layout
|
||||
if !(16.0..=16384.0).contains(&g) {
|
||||
return None;
|
||||
}
|
||||
let (r, b) = (r / g, b / g);
|
||||
((0.2..=8.0).contains(&r) && (0.2..=8.0).contains(&b)).then_some([r, 1.0, b])
|
||||
}
|
||||
|
||||
/// The inverse tone curve: 11-bit code → 14-bit sensor value.
|
||||
pub(crate) fn code_curve(thresholds: &[u64]) -> Vec<u16> {
|
||||
@@ -163,6 +300,73 @@ fn read_quad_tiles(bytes: &[u8], info: &ImageInfo) -> Result<Vec<u16>> {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// `SR2Private` tags (ExifTool "Sony SR2Private" table; the IFD is referenced by IFD0's `DNGPrivateData`).
|
||||
const SR2_SUBIFD_OFFSET: u16 = 0x7200;
|
||||
const SR2_SUBIFD_LENGTH: u16 = 0x7201;
|
||||
const SR2_SUBIFD_KEY: u16 = 0x7221;
|
||||
/// Black level inside the encrypted `SR2SubIFD` (bodies that do not write `0x7310` in the raw IFD), recovered by
|
||||
/// black-box known-plaintext analysis of CC0 raw.pixls.us samples; no description of Sony's cipher was used.
|
||||
///
|
||||
/// Every Sony ARW we examined (16 files, 15 bodies, 2012–2021) stores the same key (`11 22 33 44`), so its
|
||||
/// keystream is the same in every file. The DSC-RX100M6 and ILCE-7M3 also write their black levels in plain form
|
||||
/// (`0x7310`: 800 and 512); XOR-ing their `SR2SubIFD`s against each other matched `pack(512×4) ^ pack(800×4)` at
|
||||
/// exactly one position, byte 2510, which gave the eight keystream bytes there. Applied to the other files at that
|
||||
/// position they decode to four equal levels that agree with each sensor's dark-pixel floor: 800 on every 1″-sensor
|
||||
/// body (RX10, RX100 … RX100M5), 512 on the APS-C/full-frame ones (NEX-5N/6, SLT-A77, ILCE-6000/7/7M2/7RM2), in
|
||||
/// both `SR2SubIFD` sizes seen (29252 and 56958 bytes). Newer layouts (ILCE-7M4, 35398 bytes) put other data there
|
||||
/// and are rejected by the plausibility checks; those bodies write `0x7310` anyway.
|
||||
const SR2_BLACK_AT: usize = 2510;
|
||||
const SR2_BLACK_KEYSTREAM: [u8; 8] = [0x43, 0xcb, 0x86, 0xb6, 0x11, 0xd2, 0x1a, 0x73];
|
||||
const SR2_KEY: [u8; 4] = [0x11, 0x22, 0x33, 0x44];
|
||||
|
||||
/// The per-channel black level from the encrypted `SR2SubIFD` (see [`SR2_BLACK_AT`]), in 14-bit units; `None`
|
||||
/// unless the file uses the known key and the four decoded levels are equal and plausible.
|
||||
fn sr2_black(bytes: &[u8], ifd0: &Ifd, order: lightcraft_tiff::ByteOrder) -> Option<f32> {
|
||||
let private = ifd0.bytes(t::DNG_PRIVATE_DATA)?;
|
||||
let at = order.read_u32(private, 0)? as u64;
|
||||
let opts = lightcraft_tiff::ParseOptions { max_ifds: 1, max_depth: 0, follow_children: false, ..Default::default() };
|
||||
let (sr2, _) = lightcraft_tiff::parse_ifd_at(bytes, at, order, 0, false, &opts).ok()?;
|
||||
if sr2.bytes(SR2_SUBIFD_KEY)? != SR2_KEY {
|
||||
return None;
|
||||
}
|
||||
let (start, len) = (usize::try_from(sr2.u64(SR2_SUBIFD_OFFSET)?).ok()?, usize::try_from(sr2.u64(SR2_SUBIFD_LENGTH)?).ok()?);
|
||||
if len < SR2_BLACK_AT + 8 {
|
||||
return None;
|
||||
}
|
||||
sr2_black_levels(bytes.get(start.checked_add(SR2_BLACK_AT)?..start.checked_add(SR2_BLACK_AT + 8)?)?, order)
|
||||
}
|
||||
|
||||
/// Decipher the eight `SR2SubIFD` bytes at [`SR2_BLACK_AT`] into one black level (the mean of four near-equal,
|
||||
/// plausible per-channel levels).
|
||||
fn sr2_black_levels(cipher: &[u8], order: lightcraft_tiff::ByteOrder) -> Option<f32> {
|
||||
let plain: Vec<u8> = cipher.iter().zip(SR2_BLACK_KEYSTREAM).map(|(c, k)| c ^ k).collect();
|
||||
let levels: Vec<u16> = (0..4).filter_map(|i| order.read_u16(&plain, 2 * i)).collect();
|
||||
let (&lo, &hi) = (levels.iter().min()?, levels.iter().max()?);
|
||||
(levels.len() == 4 && (64..=4096).contains(&lo) && hi - lo <= 64).then(|| levels.iter().map(|&v| f32::from(v)).sum::<f32>() / 4.0)
|
||||
}
|
||||
|
||||
/// The image area for files without Sony's crop tags (`0x74c7/0x74c8`, written since about 2017): the DNG-style
|
||||
/// default crop when the raw IFD has one, else the maker note's `FullImageSize` (the camera JPEG's size) anchored
|
||||
/// at the top-left. Older bodies store a few columns of padding at the right edge of the raw frame (constant
|
||||
/// values, 8–32 columns on the samples we checked) inside a frame 16–48 pixels wider than `FullImageSize`, so
|
||||
/// the anchored crop removes them while keeping the CFA phase.
|
||||
fn default_crop(raw: &Ifd, mn: Option<&makernote::MakerNote>, w: usize, h: usize) -> Rect {
|
||||
let full = Rect::new(0, 0, w, h);
|
||||
if let (Some([x, y]), Some([cw, ch])) = (raw.u64s(t::DEFAULT_CROP_ORIGIN).as_deref(), raw.u64s(t::DEFAULT_CROP_SIZE).as_deref())
|
||||
&& *cw > 0
|
||||
&& *ch > 0
|
||||
{
|
||||
return Rect::new(*x as usize, *y as usize, *cw as usize, *ch as usize).clipped(w, h);
|
||||
}
|
||||
match mn.and_then(|m| m.ifd.u64s(MN_FULL_IMAGE_SIZE)).as_deref() {
|
||||
// only a plausible trim: never more than 64 pixels per side, never an enlargement
|
||||
Some([fh, fw]) if *fw as usize <= w && *fh as usize <= h && *fw as usize + 64 >= w && *fh as usize + 64 >= h => {
|
||||
Rect::new(0, 0, *fw as usize, *fh as usize)
|
||||
}
|
||||
_ => full,
|
||||
}
|
||||
}
|
||||
|
||||
fn raw_ifd(tiff: &Tiff) -> Option<&Ifd> {
|
||||
tiff.all_ifds()
|
||||
.into_iter()
|
||||
@@ -215,26 +419,43 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
_ => (read_image_in(mode, bytes, &info, tiff.order, Packing::Msb)?, bits),
|
||||
};
|
||||
let RawData::U16(ref samples) = data else { return Err(RawError::Unsupported("float ARW".into())) };
|
||||
// "12-bit uncompressed" files (e.g. ILCE-7RM2) say BitsPerSample 12 but store 16-bit words on the 14-bit scale
|
||||
// of the other modes (black 512, peaks near 16383): black and white follow the data, `bits` keeps the tag
|
||||
let scale_bits = if info.compression == 1 && out_bits < 14 && strip_len >= (w * h * 2) as u64 && samples.iter().any(|&v| v >> (out_bits + 1) != 0)
|
||||
{
|
||||
14
|
||||
} else {
|
||||
out_bits
|
||||
};
|
||||
|
||||
let cfa = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
|
||||
(Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => Cfa { width: 2, height: 2, pattern: p.to_vec() },
|
||||
_ => Cfa::bayer_static("RGGB"),
|
||||
};
|
||||
let default_black = if out_bits >= 14 { 512.0 } else { 128.0 };
|
||||
let default_black = if scale_bits >= 14 { 512.0 } else { 128.0 };
|
||||
let black = match raw.f64s(BLACK_LEVEL).as_deref() {
|
||||
Some([a, b, c, d]) => {
|
||||
BlackLevel { repeat_rows: 2, repeat_cols: 2, values: vec![*a as f32, *b as f32, *c as f32, *d as f32], ..Default::default() }
|
||||
}
|
||||
_ => BlackLevel::uniform(default_black),
|
||||
_ => BlackLevel::uniform(sr2_black(bytes, ifd0, tiff.order).filter(|_| scale_bits >= 14).unwrap_or(default_black)),
|
||||
};
|
||||
let white = raw.f64(t::WHITE_LEVEL).map(|v| v as f32).filter(|v| *v > 0.0).unwrap_or_else(|| super::white_from_data(samples, out_bits));
|
||||
let wb = raw.f64s(WB_RGGB).filter(|v| v.len() == 4 && v[1] > 0.0 && v[0] > 0.0 && v[3] > 0.0).map(|v| {
|
||||
let g = (v[1] + v[2]) / 2.0;
|
||||
[(v[0] / g) as f32, 1.0, (v[3] / g) as f32]
|
||||
});
|
||||
let white = raw.f64(t::WHITE_LEVEL).map(|v| v as f32).filter(|v| *v > 0.0).unwrap_or_else(|| super::white_from_data(samples, scale_bits));
|
||||
let model = ifd0.string(t::MODEL).unwrap_or_default();
|
||||
let mn = tiff
|
||||
.exif()
|
||||
.and_then(|e| e.get(t::MAKER_NOTE))
|
||||
.and_then(|e| makernote::parse_makernote(bytes, e.offset, e.count() as u64, tiff.order, &ifd0.string(t::MAKE).unwrap_or_default()));
|
||||
let wb = raw
|
||||
.f64s(WB_RGGB)
|
||||
.filter(|v| v.len() == 4 && v[1] > 0.0 && v[0] > 0.0 && v[3] > 0.0)
|
||||
.map(|v| {
|
||||
let g = (v[1] + v[2]) / 2.0;
|
||||
[(v[0] / g) as f32, 1.0, (v[3] / g) as f32]
|
||||
})
|
||||
.or_else(|| mn.as_ref().and_then(|m| tag2010_wb(&model, m.ifd.bytes(MN_TAG2010)?, m.order)));
|
||||
let crop = match (raw.u64s(CROP_TOP_LEFT).as_deref(), raw.u64s(CROP_SIZE).as_deref()) {
|
||||
(Some([x, y]), Some([cw, ch])) if *cw > 0 && *ch > 0 => Rect::new(*x as usize, *y as usize, *cw as usize, *ch as usize).clipped(w, h),
|
||||
_ => Rect::new(0, 0, w, h),
|
||||
_ => default_crop(raw, mn.as_ref(), w, h),
|
||||
};
|
||||
let mut metadata = lightcraft_meta::from_tiff(&tiff);
|
||||
metadata.width = Some(crop.width as u32);
|
||||
@@ -329,6 +550,58 @@ mod tests {
|
||||
let _ = code_curve(&[60000, 1, 0, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decipher_inverts_the_cube_substitution() {
|
||||
let mut seen = [false; 256];
|
||||
for p in 0..=255u8 {
|
||||
let c = if p < 249 { (p as u32).pow(3) % 249 } else { p as u32 } as u8;
|
||||
assert_eq!(DECIPHER[c as usize], p);
|
||||
seen[DECIPHER[p as usize] as usize] = true;
|
||||
}
|
||||
assert!(seen.iter().all(|&s| s), "not a bijection");
|
||||
assert_eq!((DECIPHER[8], DECIPHER[27], DECIPHER[94]), (2, 3, 7)); // 7³ = 343 ≡ 94
|
||||
}
|
||||
|
||||
fn encipher(plain: &[u8]) -> Vec<u8> {
|
||||
plain.iter().map(|&p| if p < 249 { ((p as u32).pow(3) % 249) as u8 } else { p }).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tag2010_white_balance_by_model() {
|
||||
use lightcraft_tiff::ByteOrder::Little;
|
||||
let mut plain = vec![0u8; 700];
|
||||
for (i, v) in [669u16, 256, 441].iter().enumerate() {
|
||||
plain[612 + 2 * i..614 + 2 * i].copy_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
let block = encipher(&plain);
|
||||
let wb = tag2010_wb("DSC-RX100M3", &block, Little).unwrap();
|
||||
assert!((wb[0] - 669.0 / 256.0).abs() < 1e-6 && wb[1] == 1.0 && (wb[2] - 441.0 / 256.0).abs() < 1e-6, "{wb:?}");
|
||||
// same layout (Tag2010h), regional "V" suffix, other layouts, unknown models, truncated or implausible data
|
||||
assert!(tag2010_wb("ILCE-7RM2", &block, Little).is_some());
|
||||
assert!(tag2010_wb("DSC-RX100M3V", &block, Little).is_some());
|
||||
assert!(tag2010_wb("SLT-A77V", &block, Little).is_none()); // Tag2010b: offset 4480 is past the block
|
||||
assert!(tag2010_wb("DSC-RX100M3X", &block, Little).is_none());
|
||||
assert!(tag2010_wb("ILCE-1", &block, Little).is_none());
|
||||
assert!(tag2010_wb("DSC-RX100M3", &block[..615], Little).is_none());
|
||||
assert!(tag2010_wb("DSC-RX100M3", &encipher(&[0u8; 700]), Little).is_none());
|
||||
let mut odd = plain.clone();
|
||||
odd[612..614].copy_from_slice(&9000u16.to_le_bytes()); // R/G ≈ 35
|
||||
assert!(tag2010_wb("DSC-RX100M3", &encipher(&odd), Little).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sr2_black_level_decodes_and_rejects_garbage() {
|
||||
use lightcraft_tiff::ByteOrder::Little;
|
||||
let cipher =
|
||||
|levels: [u16; 4]| -> Vec<u8> { levels.iter().flat_map(|v| v.to_le_bytes()).zip(SR2_BLACK_KEYSTREAM).map(|(p, k)| p ^ k).collect() };
|
||||
assert_eq!(sr2_black_levels(&cipher([800; 4]), Little), Some(800.0));
|
||||
assert_eq!(sr2_black_levels(&cipher([512, 512, 514, 512]), Little), Some(512.5));
|
||||
assert_eq!(sr2_black_levels(&cipher([65320, 64928, 65260, 488]), Little), None); // ILCE-7M4's layout
|
||||
assert_eq!(sr2_black_levels(&cipher([0; 4]), Little), None);
|
||||
assert_eq!(sr2_black_levels(&cipher([800; 4])[..5], Little), None);
|
||||
assert_eq!(sr2_black_levels(&[], Little), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quad_tiles_place_cfa_cells() {
|
||||
// 14×12 mosaic in 8×8 tiles (the right and bottom tiles overhang), each tile a 4×4 frame of 2×2 cells,
|
||||
|
||||
Vendored
+36
-3
@@ -56,10 +56,17 @@ pub(crate) fn decode(bytes: &[u8]) -> Result<RawImage> {
|
||||
if info.compression == t::compression::NIKON { compressed(bytes, &info, mn.as_ref())? } else { uncompressed(bytes, &info, tiff.order)? };
|
||||
let RawData::U16(ref samples) = data else { return Err(RawError::Unsupported("float NEF".into())) };
|
||||
|
||||
// Maker note 0x003d is in 14-bit units whatever the sample depth: 12-bit files from the D750, D780, D850,
|
||||
// D7500 and Z 50 store 600 / 1008 / 400 / 400 / 1008 while their darkest samples are 150 / ~252 / 99 / 99 / ~250
|
||||
// (our measurements on CC0 raw.pixls.us samples; the same bodies' 14-bit files match the tag as is).
|
||||
let black_scale = if bits == 12 { 0.25 } else { 1.0 };
|
||||
let black = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_LEVEL)).as_deref() {
|
||||
Some([a, b, c, d]) if [a, b, c, d].iter().all(|v| **v < 16384.0) => {
|
||||
BlackLevel { repeat_rows: 2, repeat_cols: 2, values: vec![*a as f32, *b as f32, *c as f32, *d as f32], ..Default::default() }
|
||||
}
|
||||
Some([a, b, c, d]) if [a, b, c, d].iter().all(|v| **v < 16384.0) => BlackLevel {
|
||||
repeat_rows: 2,
|
||||
repeat_cols: 2,
|
||||
values: [a, b, c, d].iter().map(|v| (**v * black_scale) as f32).collect(),
|
||||
..Default::default()
|
||||
},
|
||||
_ => BlackLevel::uniform(0.0),
|
||||
};
|
||||
let wb = mn
|
||||
@@ -145,6 +152,11 @@ mod tests {
|
||||
use lightcraft_tiff::{ByteOrder, IfdBuilder, ImageData, TiffWriter, Value};
|
||||
|
||||
fn nef(compression: u16, bits: u16, strips: Vec<Vec<u8>>, w: u32, h: u32, rps: u32) -> Vec<u8> {
|
||||
nef_with_note(compression, bits, strips, w, h, rps, None)
|
||||
}
|
||||
|
||||
/// [`nef`] with an optional maker note (`Nikon\0` v2 header + embedded big-endian TIFF holding `note`).
|
||||
fn nef_with_note(compression: u16, bits: u16, strips: Vec<Vec<u8>>, w: u32, h: u32, rps: u32, note: Option<IfdBuilder>) -> Vec<u8> {
|
||||
let mut raw = IfdBuilder::new();
|
||||
raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0]));
|
||||
raw.set(t::IMAGE_WIDTH, Value::Long(vec![w]));
|
||||
@@ -158,10 +170,31 @@ mod tests {
|
||||
let mut ifd0 = IfdBuilder::new();
|
||||
ifd0.set(t::MAKE, Value::Ascii("NIKON CORPORATION".into()));
|
||||
ifd0.set(t::MODEL, Value::Ascii("NIKON TEST".into()));
|
||||
if let Some(mn) = note {
|
||||
let mut bytes = b"Nikon\0\x02\x10\0\0".to_vec();
|
||||
bytes.extend(TiffWriter::new(ByteOrder::Big, false).write(&[mn]).unwrap());
|
||||
let mut exif = IfdBuilder::new();
|
||||
exif.set(t::MAKER_NOTE, Value::Undefined(bytes));
|
||||
ifd0.set_child(t::EXIF_IFD, exif);
|
||||
}
|
||||
ifd0.add_sub_ifd(raw);
|
||||
TiffWriter::new(ByteOrder::Big, false).write(&[ifd0]).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn black_level_tag_is_in_14_bit_units() {
|
||||
let (w, h) = (16usize, 4usize);
|
||||
let words: Vec<u8> = (0..w * h).flat_map(|i| (100 + i as u16).to_be_bytes()).collect();
|
||||
let note = || {
|
||||
let mut mn = IfdBuilder::new();
|
||||
mn.set(BLACK_LEVEL, Value::Short(vec![400, 404, 408, 412]));
|
||||
mn
|
||||
};
|
||||
let black = |bits| crate::decode(&nef_with_note(1, bits, vec![words.clone()], w as u32, h as u32, h as u32, Some(note()))).unwrap().black;
|
||||
assert_eq!(black(12).values, vec![100.0, 101.0, 102.0, 103.0]);
|
||||
assert_eq!(black(14).values, vec![400.0, 404.0, 408.0, 412.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uncompressed_word16_and_packed12() {
|
||||
let (w, h) = (16usize, 6usize);
|
||||
|
||||
@@ -193,3 +193,99 @@ fn corpus_nef_compressed_matches_uncompressed() {
|
||||
assert!(hi <= 4095.0 && hi - lo > 500.0, "D7000 block means {lo}..{hi}");
|
||||
}
|
||||
}
|
||||
|
||||
/// raw.pixls.us has the same D7500 scene as 12- and 14-bit lossless compressed NEF. Both store black level 400 in
|
||||
/// maker note 0x003d (14-bit units): after subtracting the black level and scaling to white, the two must agree.
|
||||
#[test]
|
||||
fn corpus_nef_12_bit_black_level_matches_14_bit() {
|
||||
let dir = corpus_root().join("raw");
|
||||
let (Ok(a), Ok(b)) = (std::fs::read(dir.join("nef-nikon-d7500-lossless12.nef")), std::fs::read(dir.join("nef-nikon-d7500-lossless14.nef")))
|
||||
else {
|
||||
eprintln!("skip: D7500 NEF pair absent");
|
||||
return;
|
||||
};
|
||||
let (a, b) = (decode(&a).unwrap(), decode(&b).unwrap());
|
||||
assert_eq!((a.bits, b.bits), (12, 14));
|
||||
assert!((a.black.values[0] - 100.0).abs() < 1.0 && (b.black.values[0] - 400.0).abs() < 1.0, "{:?} {:?}", a.black, b.black);
|
||||
let normalized = |img: &lightcraft_raw::RawImage| {
|
||||
let (black, white) = (img.black.values[0] as f64, img.white[0] as f64);
|
||||
block_means(img).into_iter().map(|v| (v - black) / (white - black)).collect::<Vec<_>>()
|
||||
};
|
||||
// the two shots aren't pixel-aligned (handheld): compare the distributions, not block by block
|
||||
let (mut na, mut nb) = (normalized(&a), normalized(&b));
|
||||
na.sort_by(f64::total_cmp);
|
||||
nb.sort_by(f64::total_cmp);
|
||||
for q in [0.1, 0.5, 0.9] {
|
||||
let (x, y) = (na[(na.len() as f64 * q) as usize], nb[(nb.len() as f64 * q) as usize]);
|
||||
eprintln!("D7500 12- vs 14-bit: normalized quantile {q}: {x:.4} vs {y:.4}");
|
||||
// with the tag read as 12-bit units, the 12-bit values would sit below black (negative)
|
||||
assert!(x > 0.0 && (0.8..1.25).contains(&(x / y)), "quantile {q}: {x} vs {y}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Issue #138: DNGs converted by Adobe software carry their camera profile's hue/saturation map and
|
||||
/// look table; we read them (and render with them). Camera-written DNGs here carry none.
|
||||
#[test]
|
||||
fn corpus_adobe_dngs_carry_profile_looks() {
|
||||
let dir = corpus_root().join("raw");
|
||||
let Ok(rd) = std::fs::read_dir(&dir) else {
|
||||
eprintln!("skip: {} absent", dir.display());
|
||||
return;
|
||||
};
|
||||
let mut seen = 0;
|
||||
for p in rd.flatten().map(|e| e.path()) {
|
||||
let name = p.file_name().unwrap().to_string_lossy().to_lowercase();
|
||||
if !name.starts_with("dng-") || !name.ends_with(".dng") {
|
||||
continue;
|
||||
}
|
||||
let info = probe_info(&std::fs::read(&p).unwrap()).unwrap_or_else(|e| panic!("{name}: {e}"));
|
||||
let look = &info.color.profile;
|
||||
if name.starts_with("dng-adobe-") {
|
||||
seen += 1;
|
||||
let hsm = look.hue_sat_map[0].as_ref().unwrap_or_else(|| panic!("{name}: no hue/sat map"));
|
||||
assert!(hsm.hue_divisions > 1 && hsm.sat_divisions > 1, "{name}");
|
||||
assert!(look.look_table.is_some(), "{name}: no look table");
|
||||
// a profile applied to a mid grey keeps it (close to) neutral
|
||||
let t = lightcraft_raw::profile::ProfileTables::new(look, 0.5).unwrap();
|
||||
let g = t.apply([0.18; 3], 1.0);
|
||||
assert!(g.iter().all(|v| (v - g[0]).abs() < 0.01 * g[0].max(0.01)), "{name}: grey → {g:?}");
|
||||
}
|
||||
eprintln!(
|
||||
"{name:44} profile look: hsm {} look {} tone {}",
|
||||
look.hue_sat_map[0].is_some(),
|
||||
look.look_table.is_some(),
|
||||
look.tone_curve.is_some()
|
||||
);
|
||||
}
|
||||
eprintln!("{seen} Adobe-converted DNGs checked");
|
||||
}
|
||||
|
||||
/// Issue #148: Sony ARWs from before ~2017 carry no plain white-balance, black-level or crop tags in the raw IFD.
|
||||
/// White balance comes from the maker note's enciphered `Tag2010`, the black level from the encrypted `SR2SubIFD`
|
||||
/// and the crop from `FullImageSize`; without them the RX100 III opened bright green. Expected values: the black
|
||||
/// levels agree with each sensor's dark-pixel floor, the gains with the neutral sky of the camera JPEG.
|
||||
#[test]
|
||||
fn corpus_sony_pre2017_colour_metadata() {
|
||||
let dir = corpus_root().join("raw");
|
||||
// (file, black, approximate R and B gains, crop width × height)
|
||||
let cases = [
|
||||
("arw-sony-rx100m3.arw", 800.0, [2.61, 1.72], (5472, 3648)),
|
||||
("arw-sony-rx100.arw", 800.0, [2.23, 2.00], (5472, 3648)),
|
||||
("arw-sony-a7rm2-12bit-uncompressed.arw", 512.0, [2.58, 1.46], (7952, 5304)),
|
||||
];
|
||||
let mut seen = 0;
|
||||
for (name, black, [r, b], (cw, ch)) in cases {
|
||||
let Ok(bytes) = std::fs::read(dir.join(name)) else {
|
||||
eprintln!("skip: {name} absent");
|
||||
continue;
|
||||
};
|
||||
let img = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}"));
|
||||
assert_eq!(img.black.mean(), black, "{name}: black level");
|
||||
let wb = img.wb_multipliers.unwrap_or_else(|| panic!("{name}: no as-shot white balance"));
|
||||
assert!((wb[0] - r).abs() < 0.01 && wb[1] == 1.0 && (wb[2] - b).abs() < 0.01, "{name}: white balance {wb:?}");
|
||||
assert_eq!((img.crop.width, img.crop.height), (cw, ch), "{name}: crop");
|
||||
assert!(img.white_at(0) > 16000.0, "{name}: white {} (14-bit scale)", img.white_at(0));
|
||||
seen += 1;
|
||||
}
|
||||
eprintln!("pre-2017 Sony ARW colour metadata checked on {seen} files");
|
||||
}
|
||||
@@ -182,6 +182,11 @@ fn base_ifd(w: usize, h: usize, bits: u16, cpp: usize, cfa: bool) -> IfdBuilder
|
||||
}
|
||||
|
||||
fn dng(raw: IfdBuilder, order: ByteOrder) -> Vec<u8> {
|
||||
dng_with(raw, order, |_| {})
|
||||
}
|
||||
|
||||
/// [`dng`] with extra IFD 0 tags (where DNG writers put camera-profile tags).
|
||||
fn dng_with(raw: IfdBuilder, order: ByteOrder, extra: impl FnOnce(&mut IfdBuilder)) -> Vec<u8> {
|
||||
let mut ifd0 = IfdBuilder::new();
|
||||
ifd0.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![1]));
|
||||
ifd0.set(t::DNG_VERSION, Value::Byte(vec![1, 6, 0, 0]));
|
||||
@@ -200,6 +205,7 @@ fn dng(raw: IfdBuilder, order: ByteOrder) -> Vec<u8> {
|
||||
exif.set(t::ISO_SPEED, Value::Short(vec![200]));
|
||||
ifd0.set_child(t::EXIF_IFD, exif);
|
||||
ifd0.add_sub_ifd(raw);
|
||||
extra(&mut ifd0);
|
||||
TiffWriter::new(order, false).write(&[ifd0]).unwrap()
|
||||
}
|
||||
|
||||
@@ -461,3 +467,68 @@ fn rejects_unsupported_and_broken() {
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: 8, strips: vec![vec![0; 128]] });
|
||||
assert!(decode(&dng(raw, ByteOrder::Little)).is_err());
|
||||
}
|
||||
|
||||
/// A Lightroom-style DNG: profile look tags in IFD 0, LinearRaw data in a SubIFD (issue #138).
|
||||
fn profile_dng(order: ByteOrder, extra: impl FnOnce(&mut IfdBuilder)) -> Vec<u8> {
|
||||
let (w, h) = (8, 6);
|
||||
let px = pattern(w, h, 3, 16);
|
||||
let mut raw = base_ifd(w, h, 16, 3, false);
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![1]));
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: h as u32, strips: vec![u16_bytes(&px, order)] });
|
||||
dng_with(raw, order, extra)
|
||||
}
|
||||
|
||||
fn hsv_floats(h: usize, s: usize, v: usize, f: impl Fn(usize, usize, usize) -> [f32; 3]) -> Vec<f32> {
|
||||
(0..v).flat_map(|vi| (0..h).flat_map(move |hi| (0..s).map(move |si| (vi, hi, si)))).flat_map(|(vi, hi, si)| f(vi, hi, si)).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_look_tags_are_read_from_ifd0() {
|
||||
for order in [ByteOrder::Little, ByteOrder::Big] {
|
||||
let bytes = profile_dng(order, |ifd0| {
|
||||
ifd0.set(t::PROFILE_HUE_SAT_MAP_DIMS, Value::Long(vec![6, 3, 1]));
|
||||
ifd0.set(t::PROFILE_HUE_SAT_MAP_DATA_1, Value::Float(hsv_floats(6, 3, 1, |_, h, s| [h as f32, 1.0 + s as f32 * 0.1, 1.0])));
|
||||
ifd0.set(t::PROFILE_HUE_SAT_MAP_DATA_2, Value::Float(hsv_floats(6, 3, 1, |_, _, _| [0.0, 1.0, 1.0])));
|
||||
ifd0.set(t::PROFILE_LOOK_TABLE_DIMS, Value::Long(vec![4, 2, 3]));
|
||||
ifd0.set(t::PROFILE_LOOK_TABLE_DATA, Value::Float(hsv_floats(4, 2, 3, |v, _, _| [0.0, 1.0, 1.0 - v as f32 * 0.05])));
|
||||
ifd0.set(t::PROFILE_LOOK_TABLE_ENCODING, Value::Long(vec![1]));
|
||||
ifd0.set(t::PROFILE_TONE_CURVE, Value::Float(vec![0.0, 0.0, 0.25, 0.15, 0.5, 0.55, 1.0, 1.0]));
|
||||
});
|
||||
let img = decode(&bytes).unwrap();
|
||||
assert_eq!(img.cpp, 3);
|
||||
let p = &img.color.profile;
|
||||
let hsm = p.hue_sat_map[0].as_ref().unwrap();
|
||||
assert_eq!((hsm.hue_divisions, hsm.sat_divisions, hsm.val_divisions), (6, 3, 1));
|
||||
// value-major, hue-middle, saturation-minor: entry (h = 2, s = 1) is at 2·3 + 1
|
||||
assert_eq!(hsm.data[7], [2.0, 1.1, 1.0]);
|
||||
assert!(p.hue_sat_map[1].is_some());
|
||||
let look = p.look_table.as_ref().unwrap();
|
||||
assert_eq!((look.hue_divisions, look.sat_divisions, look.val_divisions, look.srgb_value), (4, 2, 3, true));
|
||||
assert_eq!(look.data[2 * 4 * 2], [0.0, 1.0, 0.9]);
|
||||
assert_eq!(p.tone_curve.as_ref().unwrap().points.len(), 4);
|
||||
// the header-only probe sees the same profile
|
||||
assert_eq!(lightcraft_raw::probe_info(&bytes).unwrap().color.profile, *p);
|
||||
// our DNG writer keeps the look with the data (conversions, smart previews)
|
||||
let again = decode(&write_dng(&img, &DngWriteOptions::default()).unwrap()).unwrap();
|
||||
assert_eq!(again.color.profile, *p);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_profile_look_tags_are_ignored() {
|
||||
let bytes = profile_dng(ByteOrder::Little, |ifd0| {
|
||||
// data count doesn't match the dimensions
|
||||
ifd0.set(t::PROFILE_HUE_SAT_MAP_DIMS, Value::Long(vec![6, 3, 1]));
|
||||
ifd0.set(t::PROFILE_HUE_SAT_MAP_DATA_1, Value::Float(vec![0.0, 1.0, 1.0]));
|
||||
// one saturation division is not allowed
|
||||
ifd0.set(t::PROFILE_LOOK_TABLE_DIMS, Value::Long(vec![2, 1, 1]));
|
||||
ifd0.set(t::PROFILE_LOOK_TABLE_DATA, Value::Float(vec![0.0, 1.0, 1.0, 0.0, 1.0, 1.0]));
|
||||
// decreasing inputs and a NaN
|
||||
ifd0.set(t::PROFILE_TONE_CURVE, Value::Float(vec![0.0, 0.0, 0.6, 0.5, 0.4, f32::NAN, 1.0, 1.0]));
|
||||
});
|
||||
let img = decode(&bytes).unwrap();
|
||||
assert!(img.color.profile.is_empty(), "{:?}", img.color.profile);
|
||||
// and without any of the tags there is nothing to apply
|
||||
let plain = decode(&profile_dng(ByteOrder::Little, |_| {})).unwrap();
|
||||
assert!(plain.color.profile.is_empty());
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
[package]
|
||||
name = "lightcraft-segment"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license = "Apache-2.0"
|
||||
description = "LightCraft segmentation: SAM 3 (Segment Anything with Concepts) inference in pure Rust (L3)"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
log = { workspace = true }
|
||||
|
||||
# the model runs natively only (the web build has no AI masks)
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
||||
candle-core = { workspace = true }
|
||||
candle-nn = { workspace = true }
|
||||
rustls = { version = "0.23", default-features = false, features = ["std", "tls12"] }
|
||||
rustls-rustcrypto = "0.0.2-alpha"
|
||||
webpki-roots = "1"
|
||||
sha2 = { version = "0.10", default-features = false, features = ["std"] }
|
||||
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
candle-core = { workspace = true, features = ["metal"] }
|
||||
candle-nn = { workspace = true, features = ["metal"] }
|
||||
@@ -0,0 +1,25 @@
|
||||
//! Times SAM 3 on this machine (a synthetic gradient image; the first run includes compiling
|
||||
//! the GPU kernels): `cargo run -p lightcraft-segment --release --example sam3_bench -- <model dir>`.
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut args = std::env::args().skip(1);
|
||||
let dir = std::path::PathBuf::from(args.next().ok_or("usage: sam3_bench <model dir>")?);
|
||||
let t = std::time::Instant::now();
|
||||
let mut model = lightcraft_segment::Sam3::load(&dir)?;
|
||||
eprintln!("load: {:?} on {:?}", t.elapsed(), model.device());
|
||||
let (w, h) = (1200, 800);
|
||||
let rgb: Vec<u8> = (0..w * h * 3).map(|i| ((i / 3 % w) * 255 / w) as u8).collect();
|
||||
for run in 0..3 {
|
||||
let t = std::time::Instant::now();
|
||||
let mut enc = model.encode(&rgb, w, h)?;
|
||||
lightcraft_segment::sync(&model)?;
|
||||
let e = t.elapsed();
|
||||
let t = std::time::Instant::now();
|
||||
let p = model.segment_clicks(&mut enc, &[lightcraft_segment::Click { x: 0.5, y: 0.5, positive: true }])?;
|
||||
let c = t.elapsed();
|
||||
let t = std::time::Instant::now();
|
||||
let d = model.segment_text(&mut enc, "sky", 0.5)?;
|
||||
eprintln!("run {run}: encode {e:?}, click {c:?} (score {:.2}), text {:?} ({})", p.score, t.elapsed(), d.is_some());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
//! The CLIP text encoder (24 layers, 1024 wide, causal) that turns a prompt into the token
|
||||
//! features SAM 3's detector attends to.
|
||||
//!
|
||||
//! Modified work (Apache License 2.0, §4(b)): ported by the LightCraft contributors in 2026 from
|
||||
//! the Python/PyTorch SAM 3 code of Hugging Face Transformers (`models/sam3/modeling_sam3.py`, CLIP text model), Copyright The HuggingFace
|
||||
//! Team and Meta Platforms, Inc.; translated to Rust on candle and restructured. See NOTICE.
|
||||
|
||||
use candle_core::{Device, Module, Tensor};
|
||||
use candle_nn::{Embedding, LayerNorm, VarBuilder};
|
||||
|
||||
use crate::Result;
|
||||
use crate::nn::{Attention, Mlp, ln};
|
||||
use crate::tokenizer::CONTEXT;
|
||||
|
||||
const HIDDEN: usize = 1024;
|
||||
const LAYERS: usize = 24;
|
||||
const HEADS: usize = 16;
|
||||
const EPS: f64 = 1e-5;
|
||||
|
||||
struct Layer {
|
||||
ln1: LayerNorm,
|
||||
attn: Attention,
|
||||
ln2: LayerNorm,
|
||||
mlp: Mlp,
|
||||
}
|
||||
|
||||
pub struct TextEncoder {
|
||||
tokens: Embedding,
|
||||
positions: Tensor,
|
||||
layers: Vec<Layer>,
|
||||
final_ln: LayerNorm,
|
||||
device: Device,
|
||||
}
|
||||
|
||||
impl TextEncoder {
|
||||
/// `vb` at `detector_model.text_encoder.text_model`.
|
||||
pub fn new(vb: VarBuilder) -> Result<Self> {
|
||||
let e = vb.pp("embeddings");
|
||||
let mut layers = Vec::with_capacity(LAYERS);
|
||||
for i in 0..LAYERS {
|
||||
let l = vb.pp(format!("encoder.layers.{i}"));
|
||||
layers.push(Layer {
|
||||
ln1: ln(HIDDEN, EPS, l.pp("layer_norm1"))?,
|
||||
attn: Attention::named(HIDDEN, HIDDEN, HEADS, "out_proj", l.pp("self_attn"))?,
|
||||
ln2: ln(HIDDEN, EPS, l.pp("layer_norm2"))?,
|
||||
mlp: Mlp::new(HIDDEN, 4 * HIDDEN, true, l.pp("mlp"))?,
|
||||
});
|
||||
}
|
||||
Ok(Self {
|
||||
tokens: candle_nn::embedding(49408, HIDDEN, e.pp("token_embedding"))?,
|
||||
positions: e.get((CONTEXT, HIDDEN), "position_embedding.weight")?.unsqueeze(0)?,
|
||||
layers,
|
||||
final_ln: ln(HIDDEN, EPS, vb.pp("final_layer_norm"))?,
|
||||
device: vb.device().clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// `[1, 32, 1024]` features for token `ids` (`valid` marks real tokens).
|
||||
pub fn forward(&self, ids: &[u32], valid: &[bool]) -> Result<Tensor> {
|
||||
let ids = Tensor::from_slice(ids, (1, ids.len()), &self.device)?;
|
||||
let mut x = self.tokens.forward(&ids)?.broadcast_add(&self.positions)?;
|
||||
// causal, and padding keys masked
|
||||
let n = valid.len();
|
||||
let mut bias = vec![0f32; n * n];
|
||||
for (q, row) in bias.chunks_mut(n).enumerate() {
|
||||
for (k, v) in row.iter_mut().enumerate() {
|
||||
if k > q || !valid.get(k).copied().unwrap_or(false) {
|
||||
*v = f32::MIN;
|
||||
}
|
||||
}
|
||||
}
|
||||
let bias = Tensor::from_vec(bias, (1, 1, n, n), &self.device)?;
|
||||
for l in &self.layers {
|
||||
let h = l.ln1.forward(&x)?;
|
||||
x = (&x + l.attn.forward(&h, &h, &h, Some(&bias))?)?;
|
||||
let h = l.ln2.forward(&x)?;
|
||||
x = (&x + l.mlp.forward(&h)?)?;
|
||||
}
|
||||
Ok(self.final_ln.forward(&x)?)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
//! Text prompts (SAM 3's concept detector): the prompt's CLIP features fused into the image
|
||||
//! features by a DETR encoder, 200 object queries refined by a DETR decoder (with box
|
||||
//! relative-position bias and a presence token), per-query masks from a pixel decoder.
|
||||
//!
|
||||
//! Modified work (Apache License 2.0, §4(b)): ported by the LightCraft contributors in 2026 from
|
||||
//! the Python/PyTorch SAM 3 code of Hugging Face Transformers (`models/sam3/modeling_sam3.py`), Copyright The HuggingFace
|
||||
//! Team and Meta Platforms, Inc.; translated to Rust on candle and restructured. See NOTICE.
|
||||
|
||||
use candle_core::{D, Device, IndexOp, Module, Tensor};
|
||||
use candle_nn::{Conv2d, Conv2dConfig, GroupNorm, LayerNorm, Linear, VarBuilder};
|
||||
|
||||
use crate::clip::TextEncoder;
|
||||
use crate::neck::{FPN, Neck};
|
||||
use crate::nn::{Attention, DecoderMlp, Mlp, TORCH_LN_EPS, cxcywh_to_xyxy, inverse_sigmoid, linear, ln, padding_bias, sine_grid, to_vec};
|
||||
use crate::{Error, Prediction, Result};
|
||||
|
||||
const HIDDEN: usize = FPN;
|
||||
const HEADS: usize = 8;
|
||||
const QUERIES: usize = 200;
|
||||
const LAYERS: usize = 6;
|
||||
|
||||
/// Detector image features, computed once per image.
|
||||
pub struct Features {
|
||||
/// Levels 0..=2 (288, 144, 72 per side).
|
||||
levels: Vec<Tensor>,
|
||||
/// Level 2 flattened `[1, 5184, 256]` and its sine position encoding.
|
||||
flat: Tensor,
|
||||
pos: Tensor,
|
||||
side: usize,
|
||||
}
|
||||
|
||||
/// Everything the detector predicts for one prompt.
|
||||
pub struct Output {
|
||||
/// Per-query mask logits `[200, 288, 288]` flattened.
|
||||
pub masks: Tensor,
|
||||
/// Per-query logits (before the presence score).
|
||||
pub logits: Vec<f32>,
|
||||
/// Whether the concept is in the image at all (logit).
|
||||
pub presence: f32,
|
||||
/// Boxes (x0, y0, x1, y1), normalized.
|
||||
pub boxes: Vec<[f32; 4]>,
|
||||
/// The text features `[1, 32, 256]` (tests).
|
||||
pub text: Tensor,
|
||||
}
|
||||
|
||||
struct EncoderLayer {
|
||||
ln1: LayerNorm,
|
||||
self_attn: Attention,
|
||||
ln2: LayerNorm,
|
||||
cross_attn: Attention,
|
||||
ln3: LayerNorm,
|
||||
mlp: Mlp,
|
||||
}
|
||||
|
||||
struct DecoderLayer {
|
||||
self_attn: Attention,
|
||||
self_ln: LayerNorm,
|
||||
text_attn: Attention,
|
||||
text_ln: LayerNorm,
|
||||
vision_attn: Attention,
|
||||
vision_ln: LayerNorm,
|
||||
mlp: Mlp,
|
||||
mlp_ln: LayerNorm,
|
||||
}
|
||||
|
||||
pub struct Detector {
|
||||
device: Device,
|
||||
neck: Neck,
|
||||
text: TextEncoder,
|
||||
text_proj: Linear,
|
||||
encoder: Vec<EncoderLayer>,
|
||||
decoder: Vec<DecoderLayer>,
|
||||
out_ln: LayerNorm,
|
||||
box_head: DecoderMlp,
|
||||
query_embed: Tensor,
|
||||
reference_points: Tensor,
|
||||
presence_token: Tensor,
|
||||
presence_head: DecoderMlp,
|
||||
presence_ln: LayerNorm,
|
||||
ref_point_head: DecoderMlp,
|
||||
rpb_x: DecoderMlp,
|
||||
rpb_y: DecoderMlp,
|
||||
// dot-product scoring
|
||||
score_text_mlp: DecoderMlp,
|
||||
score_text_ln: LayerNorm,
|
||||
score_text_proj: Linear,
|
||||
score_query_proj: Linear,
|
||||
// mask decoder
|
||||
prompt_attn: Attention,
|
||||
prompt_ln: LayerNorm,
|
||||
pixel_convs: Vec<Conv2d>,
|
||||
pixel_norms: Vec<GroupNorm>,
|
||||
instance_proj: Conv2d,
|
||||
mask_embedder: Vec<Linear>,
|
||||
}
|
||||
|
||||
impl Detector {
|
||||
/// `vb` at `detector_model`.
|
||||
pub fn new(vb: VarBuilder) -> Result<Self> {
|
||||
let e = vb.pp("detr_encoder");
|
||||
let mut encoder = Vec::with_capacity(LAYERS);
|
||||
for i in 0..LAYERS {
|
||||
let l = e.pp(format!("layers.{i}"));
|
||||
encoder.push(EncoderLayer {
|
||||
ln1: ln(HIDDEN, TORCH_LN_EPS, l.pp("layer_norm1"))?,
|
||||
self_attn: Attention::new(HIDDEN, HIDDEN, HEADS, l.pp("self_attn"))?,
|
||||
ln2: ln(HIDDEN, TORCH_LN_EPS, l.pp("layer_norm2"))?,
|
||||
cross_attn: Attention::new(HIDDEN, HIDDEN, HEADS, l.pp("cross_attn"))?,
|
||||
ln3: ln(HIDDEN, TORCH_LN_EPS, l.pp("layer_norm3"))?,
|
||||
mlp: Mlp::new(HIDDEN, 2048, false, l.pp("mlp"))?,
|
||||
});
|
||||
}
|
||||
let d = vb.pp("detr_decoder");
|
||||
let mut decoder = Vec::with_capacity(LAYERS);
|
||||
for i in 0..LAYERS {
|
||||
let l = d.pp(format!("layers.{i}"));
|
||||
decoder.push(DecoderLayer {
|
||||
self_attn: Attention::new(HIDDEN, HIDDEN, HEADS, l.pp("self_attn"))?,
|
||||
self_ln: ln(HIDDEN, TORCH_LN_EPS, l.pp("self_attn_layer_norm"))?,
|
||||
text_attn: Attention::new(HIDDEN, HIDDEN, HEADS, l.pp("text_cross_attn"))?,
|
||||
text_ln: ln(HIDDEN, TORCH_LN_EPS, l.pp("text_cross_attn_layer_norm"))?,
|
||||
vision_attn: Attention::new(HIDDEN, HIDDEN, HEADS, l.pp("vision_cross_attn"))?,
|
||||
vision_ln: ln(HIDDEN, TORCH_LN_EPS, l.pp("vision_cross_attn_layer_norm"))?,
|
||||
mlp: Mlp::new(HIDDEN, 2048, false, l.pp("mlp"))?,
|
||||
mlp_ln: ln(HIDDEN, TORCH_LN_EPS, l.pp("mlp_layer_norm"))?,
|
||||
});
|
||||
}
|
||||
let s = vb.pp("dot_product_scoring");
|
||||
let m = vb.pp("mask_decoder");
|
||||
let p = m.pp("pixel_decoder");
|
||||
let conv3 = Conv2dConfig { padding: 1, ..Default::default() };
|
||||
Ok(Self {
|
||||
device: vb.device().clone(),
|
||||
neck: Neck::new(vb.pp("vision_encoder.neck"))?,
|
||||
text: TextEncoder::new(vb.pp("text_encoder.text_model"))?,
|
||||
text_proj: linear(1024, HIDDEN, vb.pp("text_projection"))?,
|
||||
encoder,
|
||||
decoder,
|
||||
out_ln: ln(HIDDEN, TORCH_LN_EPS, d.pp("output_layer_norm"))?,
|
||||
box_head: DecoderMlp::new(HIDDEN, HIDDEN, 4, 3, d.pp("box_head"))?,
|
||||
query_embed: d.get((QUERIES, HIDDEN), "query_embed.weight")?.unsqueeze(0)?,
|
||||
reference_points: d.get((QUERIES, 4), "reference_points.weight")?.unsqueeze(0)?,
|
||||
presence_token: d.get((1, HIDDEN), "presence_token.weight")?.unsqueeze(0)?,
|
||||
presence_head: DecoderMlp::new(HIDDEN, HIDDEN, 1, 3, d.pp("presence_head"))?,
|
||||
presence_ln: ln(HIDDEN, TORCH_LN_EPS, d.pp("presence_layer_norm"))?,
|
||||
ref_point_head: DecoderMlp::new(2 * HIDDEN, HIDDEN, HIDDEN, 2, d.pp("ref_point_head"))?,
|
||||
rpb_x: DecoderMlp::new(2, HIDDEN, HEADS, 2, d.pp("box_rpb_embed_x"))?,
|
||||
rpb_y: DecoderMlp::new(2, HIDDEN, HEADS, 2, d.pp("box_rpb_embed_y"))?,
|
||||
score_text_mlp: DecoderMlp::new(HIDDEN, 2048, HIDDEN, 2, s.pp("text_mlp"))?,
|
||||
score_text_ln: ln(HIDDEN, TORCH_LN_EPS, s.pp("text_mlp_out_norm"))?,
|
||||
score_text_proj: linear(HIDDEN, HIDDEN, s.pp("text_proj"))?,
|
||||
score_query_proj: linear(HIDDEN, HIDDEN, s.pp("query_proj"))?,
|
||||
prompt_attn: Attention::new(HIDDEN, HIDDEN, HEADS, m.pp("prompt_cross_attn"))?,
|
||||
prompt_ln: ln(HIDDEN, TORCH_LN_EPS, m.pp("prompt_cross_attn_norm"))?,
|
||||
pixel_convs: (0..3).map(|i| Ok(candle_nn::conv2d(HIDDEN, HIDDEN, 3, conv3, p.pp(format!("conv_layers.{i}")))?)).collect::<Result<_>>()?,
|
||||
pixel_norms: (0..3).map(|i| Ok(candle_nn::group_norm(8, HIDDEN, 1e-5, p.pp(format!("norms.{i}")))?)).collect::<Result<_>>()?,
|
||||
instance_proj: candle_nn::conv2d(HIDDEN, HIDDEN, 1, Conv2dConfig::default(), m.pp("instance_projection"))?,
|
||||
mask_embedder: (0..3).map(|i| linear(HIDDEN, HIDDEN, m.pp(format!("mask_embedder.layers.{i}")))).collect::<Result<_>>()?,
|
||||
})
|
||||
}
|
||||
|
||||
/// The detector's feature pyramid for the backbone output `[1, 1024, 72, 72]`.
|
||||
pub fn features(&self, backbone: &Tensor) -> Result<Features> {
|
||||
let levels = self.neck.forward(backbone, 3)?;
|
||||
let last = levels.last().ok_or_else(|| Error::Model("detector neck".into()))?;
|
||||
let (_, _, side, _) = last.dims4()?;
|
||||
let flat = last.flatten_from(2)?.transpose(1, 2)?.contiguous()?;
|
||||
let pos = sine_grid(side, side, HIDDEN / 2, &self.device)?.flatten_from(2)?.transpose(1, 2)?.contiguous()?;
|
||||
Ok(Features { levels, flat, pos, side })
|
||||
}
|
||||
|
||||
/// Box relative-position bias `[1, heads, 1 + 200, side²]` for reference boxes (cx, cy, w, h).
|
||||
fn rpb(&self, boxes: &Tensor, side: usize) -> Result<Tensor> {
|
||||
let xyxy = cxcywh_to_xyxy(boxes)?.squeeze(0)?; // [200, 4]
|
||||
let coords = Tensor::arange(0u32, side as u32, &self.device)?.to_dtype(candle_core::DType::F32)?.affine(1.0 / side as f64, 0.0)?;
|
||||
let coords = coords.reshape((1, side, 1))?;
|
||||
let log_scale = |d: Tensor| -> Result<Tensor> {
|
||||
let d = (d * 8.0)?;
|
||||
Ok(d.sign()?.mul(&((d.abs()? + 1.0)?.log()? / (8f64.ln()))?)?)
|
||||
};
|
||||
let pick = |a: usize, b: usize| -> Result<Tensor> { Ok(Tensor::cat(&[xyxy.narrow(1, a, 1)?, xyxy.narrow(1, b, 1)?], 1)?.unsqueeze(1)?) };
|
||||
let dy = log_scale(coords.broadcast_sub(&pick(1, 3)?)?)?; // [200, side, 2]
|
||||
let dx = log_scale(coords.broadcast_sub(&pick(0, 2)?)?)?;
|
||||
let ey = self.rpb_y.forward(&dy)?; // [200, side, heads]
|
||||
let ex = self.rpb_x.forward(&dx)?;
|
||||
let m = ey.unsqueeze(2)?.broadcast_add(&ex.unsqueeze(1)?)?; // [200, side, side, heads]
|
||||
let m = m.reshape((QUERIES, side * side, HEADS))?.permute((2, 0, 1))?; // [heads, 200, side²]
|
||||
let m = m.pad_with_zeros(1, 1, 0)?; // the presence token attends everywhere
|
||||
Ok(m.unsqueeze(0)?.contiguous()?)
|
||||
}
|
||||
|
||||
/// Sine embedding of boxes (cx, cy, w, h) `[1, 200, 512]` (y, x, w, h features).
|
||||
fn box_sine(&self, boxes: &Tensor) -> Result<Tensor> {
|
||||
let feats = HIDDEN / 2;
|
||||
let dim_t: Vec<f32> = (0..feats).map(|i| 10000f64.powf(2.0 * (i / 2) as f64 / feats as f64) as f32).collect();
|
||||
let dim_t = Tensor::from_vec(dim_t, (1, 1, feats), &self.device)?;
|
||||
let even: Vec<f32> = (0..feats).map(|i| if i % 2 == 0 { 1.0 } else { 0.0 }).collect();
|
||||
let even = Tensor::from_vec(even, (1, 1, feats), &self.device)?;
|
||||
let one = |i: usize| -> Result<Tensor> {
|
||||
let p = (boxes.narrow(D::Minus1, i, 1)? * (2.0 * std::f64::consts::PI))?.broadcast_div(&dim_t)?;
|
||||
Ok((p.sin()?.broadcast_mul(&even)? + p.cos()?.broadcast_mul(&(1.0 - &even)?)?)?)
|
||||
};
|
||||
Ok(Tensor::cat(&[one(1)?, one(0)?, one(2)?, one(3)?], D::Minus1)?)
|
||||
}
|
||||
|
||||
/// Run the detector for one tokenized prompt.
|
||||
pub fn predict(&self, f: &Features, ids: &[u32], valid: &[bool]) -> Result<Output> {
|
||||
let text = self.text_proj.forward(&self.text.forward(ids, valid)?)?; // [1, 32, 256]
|
||||
let text_bias = padding_bias(valid, &self.device)?;
|
||||
|
||||
// DETR encoder: image tokens attend to themselves and to the prompt
|
||||
let mut x = f.flat.clone();
|
||||
for l in &self.encoder {
|
||||
let h = l.ln1.forward(&x)?;
|
||||
let hp = (&h + &f.pos)?;
|
||||
x = (&x + l.self_attn.forward(&hp, &hp, &h, None)?)?;
|
||||
let h = l.ln2.forward(&x)?;
|
||||
x = (&x + l.cross_attn.forward(&h, &text, &text, Some(&text_bias))?)?;
|
||||
let h = l.ln3.forward(&x)?;
|
||||
x = (&x + l.mlp.forward(&h)?)?;
|
||||
}
|
||||
let memory = x;
|
||||
|
||||
// DETR decoder: presence token + 200 queries with iterative box refinement
|
||||
let mut boxes = candle_nn::ops::sigmoid(&self.reference_points)?;
|
||||
let mut h = Tensor::cat(&[self.presence_token.clone(), self.query_embed.clone()], 1)?;
|
||||
let key_pos = (&memory + &f.pos)?;
|
||||
let mut queries = None;
|
||||
let mut presence = None;
|
||||
for l in &self.decoder {
|
||||
let qpos = self.ref_point_head.forward(&self.box_sine(&boxes)?)?.pad_with_zeros(1, 1, 0)?;
|
||||
let bias = self.rpb(&boxes, f.side)?;
|
||||
let q = (&h + &qpos)?;
|
||||
h = l.self_ln.forward(&(&h + l.self_attn.forward(&q, &q, &h, None)?)?)?;
|
||||
let q = (&h + &qpos)?;
|
||||
h = l.text_ln.forward(&(&h + l.text_attn.forward(&q, &text, &text, Some(&text_bias))?)?)?;
|
||||
let q = (&h + &qpos)?;
|
||||
h = l.vision_ln.forward(&(&h + l.vision_attn.forward(&q, &key_pos, &memory, Some(&bias))?)?)?;
|
||||
h = l.mlp_ln.forward(&(&h + l.mlp.forward(&h)?)?)?;
|
||||
let qh = self.out_ln.forward(&h.narrow(1, 1, QUERIES)?)?;
|
||||
boxes = candle_nn::ops::sigmoid(&(self.box_head.forward(&qh)? + inverse_sigmoid(&boxes)?)?)?;
|
||||
let p = self.presence_head.forward(&self.presence_ln.forward(&h.narrow(1, 0, 1)?)?)?;
|
||||
presence = Some(p.clamp(-10f32, 10f32)?);
|
||||
queries = Some(qh);
|
||||
}
|
||||
let (Some(queries), Some(presence)) = (queries, presence) else { return Err(Error::Model("decoder has no layers".into())) };
|
||||
|
||||
// dot-product scoring against the mean of the (refined) prompt tokens
|
||||
let t = self.score_text_ln.forward(&(self.score_text_mlp.forward(&text)? + &text)?)?;
|
||||
let w: Vec<f32> = valid.iter().map(|v| if *v { 1.0 } else { 0.0 }).collect();
|
||||
let count = w.iter().sum::<f32>().max(1.0);
|
||||
let w = Tensor::from_vec(w, (1, valid.len(), 1), &self.device)?;
|
||||
let pooled = (t.broadcast_mul(&w)?.sum(1)? / f64::from(count))?; // [1, 256]
|
||||
let pt = self.score_text_proj.forward(&pooled)?; // [1, 256]
|
||||
let pq = self.score_query_proj.forward(&queries)?; // [1, 200, 256]
|
||||
let logits = (pq.matmul(&pt.unsqueeze(2)?)? / (HIDDEN as f64).sqrt())?.clamp(-12f32, 12f32)?;
|
||||
|
||||
// masks: the encoder memory attends to the prompt, then a pixel decoder up to 288²
|
||||
let mem = (&memory + self.prompt_attn.forward(&self.prompt_ln.forward(&memory)?, &text, &text, Some(&text_bias))?)?;
|
||||
let [l0, l1, _] = &f.levels[..] else { return Err(Error::Model("detector levels".into())) };
|
||||
let mut prev = mem.transpose(1, 2)?.reshape((1, HIDDEN, f.side, f.side))?;
|
||||
for (i, skip) in [l1, l0].into_iter().enumerate() {
|
||||
let (_, _, sh, sw) = skip.dims4()?;
|
||||
prev = (prev.upsample_nearest2d(sh, sw)? + skip)?;
|
||||
let (Some(conv), Some(norm)) = (self.pixel_convs.get(i), self.pixel_norms.get(i)) else {
|
||||
return Err(Error::Model("pixel decoder".into()));
|
||||
};
|
||||
prev = norm.forward(&conv.forward(&prev)?)?.relu()?;
|
||||
}
|
||||
let inst = self.instance_proj.forward(&prev)?; // [1, 256, 288, 288]
|
||||
let (_, c, mh, mw) = inst.dims4()?;
|
||||
let mut e = queries.clone();
|
||||
for (i, l) in self.mask_embedder.iter().enumerate() {
|
||||
e = l.forward(&e)?;
|
||||
if i + 1 < self.mask_embedder.len() {
|
||||
e = e.relu()?;
|
||||
}
|
||||
}
|
||||
let masks = e.matmul(&inst.reshape((1, c, mh * mw))?)?.reshape((QUERIES, mh, mw))?;
|
||||
let boxes = cxcywh_to_xyxy(&boxes)?;
|
||||
let bv = to_vec(&boxes)?;
|
||||
Ok(Output {
|
||||
masks,
|
||||
logits: to_vec(&logits)?,
|
||||
presence: to_vec(&presence)?.first().copied().unwrap_or(-10.0),
|
||||
boxes: bv.as_chunks::<4>().0.to_vec(),
|
||||
text,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The instances of `out` scoring above `threshold` (sigmoid(logit) × sigmoid(presence)),
|
||||
/// merged into one mask (the per-pixel maximum of their probabilities, as logits).
|
||||
pub fn merge(out: &Output, threshold: f32) -> Option<Prediction> {
|
||||
let s = |x: f32| 1.0 / (1.0 + (-x).exp());
|
||||
let presence = s(out.presence);
|
||||
let keep: Vec<(usize, f32)> = out.logits.iter().enumerate().map(|(i, l)| (i, s(*l) * presence)).filter(|(_, p)| *p > threshold).collect();
|
||||
let best = keep.iter().map(|k| k.1).fold(0f32, f32::max);
|
||||
let mut merged: Option<Vec<f32>> = None;
|
||||
for (i, _) in &keep {
|
||||
let m = out.masks.i(*i).ok().and_then(|t| to_vec(&t).ok())?;
|
||||
merged = Some(match merged {
|
||||
None => m,
|
||||
Some(acc) => acc.iter().zip(&m).map(|(a, b)| a.max(*b)).collect(),
|
||||
});
|
||||
}
|
||||
merged.map(|logits| Prediction { logits, score: best })
|
||||
}
|
||||
@@ -0,0 +1,541 @@
|
||||
//! A small HTTP/1.1 GET client for model downloads, in pure Rust: `std::net` sockets and, for
|
||||
//! `https://`, rustls with the RustCrypto provider (`rustls-rustcrypto`; no `ring` or
|
||||
//! `aws-lc-rs`, so nothing is compiled from C or assembly) and the Mozilla root certificates
|
||||
//! (`webpki-roots`).
|
||||
//!
|
||||
//! Only what a large-file download needs: one request per connection (`Connection: close`),
|
||||
//! `Range` requests for resuming, redirects (followed by the caller), `Content-Length`,
|
||||
//! chunked and read-to-close bodies. Every read waits at most `stall` for data and checks the
|
||||
//! cancel flag at least twice a second, so a dead server or a cancelled download never hangs
|
||||
//! the downloading thread. No proxies (see docs/ai-masks.md).
|
||||
|
||||
use std::io::{ErrorKind, Read, Write};
|
||||
use std::net::{TcpStream, ToSocketAddrs};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Why a request failed.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum HttpError {
|
||||
/// The URL can't be used (scheme, host, characters).
|
||||
BadUrl(String),
|
||||
/// Name resolution or connecting failed (or took longer than the connect timeout).
|
||||
Connect(String),
|
||||
/// No data arrived for the stall timeout.
|
||||
Stalled,
|
||||
/// The cancel flag was raised.
|
||||
Cancelled,
|
||||
/// TLS failed (certificate, handshake).
|
||||
Tls(String),
|
||||
/// The server's response could not be understood.
|
||||
Protocol(String),
|
||||
/// Any other I/O error.
|
||||
Io(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for HttpError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
HttpError::BadUrl(e) => write!(f, "unusable URL: {e}"),
|
||||
HttpError::Connect(e) => write!(f, "could not connect: {e}"),
|
||||
HttpError::Stalled => write!(f, "the server stopped sending data"),
|
||||
HttpError::Cancelled => write!(f, "cancelled"),
|
||||
HttpError::Tls(e) => write!(f, "secure connection failed: {e}"),
|
||||
HttpError::Protocol(e) => write!(f, "unexpected response: {e}"),
|
||||
HttpError::Io(e) => write!(f, "{e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A parsed `http://` or `https://` URL.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Url {
|
||||
pub tls: bool,
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
/// Path and query, starting with `/`.
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
impl Url {
|
||||
pub fn parse(s: &str) -> Result<Url, HttpError> {
|
||||
let bad = |why: &str| HttpError::BadUrl(format!("{s}: {why}"));
|
||||
let s = s.trim();
|
||||
if s.chars().any(|c| c.is_control() || c == ' ') {
|
||||
return Err(bad("contains spaces or control characters"));
|
||||
}
|
||||
let (tls, rest) = if let Some(r) = s.strip_prefix("https://") {
|
||||
(true, r)
|
||||
} else if let Some(r) = s.strip_prefix("http://") {
|
||||
(false, r)
|
||||
} else {
|
||||
return Err(bad("only http:// and https:// are supported"));
|
||||
};
|
||||
let rest = rest.split('#').next().unwrap_or_default();
|
||||
let (authority, path) = match rest.find(['/', '?']) {
|
||||
Some(i) => (rest.get(..i).unwrap_or_default(), rest.get(i..).unwrap_or_default()),
|
||||
None => (rest, ""),
|
||||
};
|
||||
let path = if path.starts_with('/') { path.to_string() } else { format!("/{path}") };
|
||||
if authority.contains('@') {
|
||||
return Err(bad("user names and passwords in URLs are not supported"));
|
||||
}
|
||||
let (host, port) = if let Some(v6) = authority.strip_prefix('[') {
|
||||
let end = v6.find(']').ok_or_else(|| bad("unterminated IPv6 address"))?;
|
||||
let host = v6.get(..end).unwrap_or_default();
|
||||
let after = v6.get(end + 1..).unwrap_or_default();
|
||||
(host.to_string(), after.strip_prefix(':'))
|
||||
} else {
|
||||
match authority.rsplit_once(':') {
|
||||
Some((h, p)) => (h.to_string(), Some(p)),
|
||||
None => (authority.to_string(), None),
|
||||
}
|
||||
};
|
||||
let port = match port {
|
||||
Some(p) => p.parse::<u16>().ok().filter(|p| *p != 0).ok_or_else(|| bad("bad port"))?,
|
||||
None if tls => 443,
|
||||
None => 80,
|
||||
};
|
||||
if host.is_empty() || !host.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '.' | ':' | '_')) {
|
||||
return Err(bad("bad host name"));
|
||||
}
|
||||
Ok(Url { tls, host: host.to_ascii_lowercase(), port, path })
|
||||
}
|
||||
|
||||
/// `Location` of a redirect, resolved against this URL.
|
||||
pub fn join(&self, location: &str) -> Result<Url, HttpError> {
|
||||
let l = location.trim();
|
||||
if l.starts_with("http://") || l.starts_with("https://") {
|
||||
return Url::parse(l);
|
||||
}
|
||||
// another scheme (`ftp:`, `file:`, `data:`…)
|
||||
if let Some((scheme, _)) = l.split_once(':')
|
||||
&& !scheme.is_empty()
|
||||
&& scheme.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
|
||||
{
|
||||
return Err(HttpError::BadUrl(format!("redirect to an unsupported location `{scheme}:`")));
|
||||
}
|
||||
let scheme = if self.tls { "https" } else { "http" };
|
||||
let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() };
|
||||
let base = format!("{scheme}://{host}:{}", self.port);
|
||||
if let Some(rest) = l.strip_prefix("//") {
|
||||
return Url::parse(&format!("{scheme}://{rest}"));
|
||||
}
|
||||
if l.starts_with('/') {
|
||||
return Url::parse(&format!("{base}{l}"));
|
||||
}
|
||||
// relative to the current directory
|
||||
let dir = self.path.split('?').next().unwrap_or("/");
|
||||
let dir = dir.rsplit_once('/').map_or("", |(d, _)| d);
|
||||
Url::parse(&format!("{base}{dir}/{l}"))
|
||||
}
|
||||
|
||||
fn host_header(&self) -> String {
|
||||
let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() };
|
||||
let default = if self.tls { 443 } else { 80 };
|
||||
if self.port == default { host } else { format!("{host}:{}", self.port) }
|
||||
}
|
||||
|
||||
/// The URL without its query (for messages: a query may hold a signed token).
|
||||
pub fn display(&self) -> String {
|
||||
let scheme = if self.tls { "https" } else { "http" };
|
||||
format!("{scheme}://{}{}", self.host_header(), self.path.split('?').next().unwrap_or("/"))
|
||||
}
|
||||
}
|
||||
|
||||
/// Timeouts and the cancel flag for one request.
|
||||
pub struct Limits<'a> {
|
||||
pub connect: Duration,
|
||||
pub stall: Duration,
|
||||
pub cancel: &'a AtomicBool,
|
||||
}
|
||||
|
||||
/// How often a blocked read wakes up to check the cancel flag.
|
||||
const TICK: Duration = Duration::from_millis(250);
|
||||
/// Longest status line + headers accepted.
|
||||
const MAX_HEAD: usize = 64 * 1024;
|
||||
|
||||
enum Stream {
|
||||
Plain(TcpStream),
|
||||
Tls(Box<rustls::ClientConnection>, TcpStream),
|
||||
}
|
||||
|
||||
fn is_timeout(e: &std::io::Error) -> bool {
|
||||
matches!(e.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut | ErrorKind::Interrupted)
|
||||
}
|
||||
|
||||
fn tls_config() -> Result<Arc<rustls::ClientConfig>, HttpError> {
|
||||
let roots = rustls::RootCertStore { roots: webpki_roots::TLS_SERVER_ROOTS.to_vec() };
|
||||
let config = rustls::ClientConfig::builder_with_provider(Arc::new(rustls_rustcrypto::provider()))
|
||||
.with_safe_default_protocol_versions()
|
||||
.map_err(|e| HttpError::Tls(e.to_string()))?
|
||||
.with_root_certificates(roots)
|
||||
.with_no_client_auth();
|
||||
Ok(Arc::new(config))
|
||||
}
|
||||
|
||||
impl Stream {
|
||||
fn connect(url: &Url, limits: &Limits) -> Result<Stream, HttpError> {
|
||||
let addrs: Vec<_> = (url.host.as_str(), url.port).to_socket_addrs().map_err(|e| HttpError::Connect(format!("{}: {e}", url.host)))?.collect();
|
||||
if addrs.is_empty() {
|
||||
return Err(HttpError::Connect(format!("{}: no address", url.host)));
|
||||
}
|
||||
let mut last = String::new();
|
||||
let mut sock = None;
|
||||
for a in addrs {
|
||||
if limits.cancel.load(Ordering::Relaxed) {
|
||||
return Err(HttpError::Cancelled);
|
||||
}
|
||||
match TcpStream::connect_timeout(&a, limits.connect) {
|
||||
Ok(s) => {
|
||||
sock = Some(s);
|
||||
break;
|
||||
}
|
||||
Err(e) => last = format!("{}: {e}", url.host),
|
||||
}
|
||||
}
|
||||
let sock = sock.ok_or(HttpError::Connect(last))?;
|
||||
sock.set_read_timeout(Some(TICK)).map_err(|e| HttpError::Io(e.to_string()))?;
|
||||
sock.set_write_timeout(Some(TICK)).map_err(|e| HttpError::Io(e.to_string()))?;
|
||||
let _ = sock.set_nodelay(true);
|
||||
if !url.tls {
|
||||
return Ok(Stream::Plain(sock));
|
||||
}
|
||||
let name = rustls::pki_types::ServerName::try_from(url.host.clone()).map_err(|e| HttpError::Tls(format!("{}: {e}", url.host)))?;
|
||||
let conn = rustls::ClientConnection::new(tls_config()?, name).map_err(|e| HttpError::Tls(e.to_string()))?;
|
||||
let mut s = Stream::Tls(Box::new(conn), sock);
|
||||
s.handshake(limits)?;
|
||||
Ok(s)
|
||||
}
|
||||
|
||||
/// Wait for `f` to make progress, retrying timeouts until `stall` passes without any.
|
||||
fn retry<T>(limits: &Limits, mut f: impl FnMut() -> std::io::Result<T>) -> Result<T, HttpError> {
|
||||
let start = Instant::now();
|
||||
loop {
|
||||
if limits.cancel.load(Ordering::Relaxed) {
|
||||
return Err(HttpError::Cancelled);
|
||||
}
|
||||
match f() {
|
||||
Ok(v) => return Ok(v),
|
||||
Err(e) if is_timeout(&e) => {
|
||||
if start.elapsed() >= limits.stall {
|
||||
return Err(HttpError::Stalled);
|
||||
}
|
||||
}
|
||||
Err(e) if e.kind() == ErrorKind::InvalidData => return Err(HttpError::Tls(e.to_string())),
|
||||
Err(e) => return Err(HttpError::Io(e.to_string())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn handshake(&mut self, limits: &Limits) -> Result<(), HttpError> {
|
||||
let Stream::Tls(conn, sock) = self else { return Ok(()) };
|
||||
while conn.is_handshaking() {
|
||||
Self::retry(limits, || conn.complete_io(sock))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_all(&mut self, data: &[u8], limits: &Limits) -> Result<(), HttpError> {
|
||||
match self {
|
||||
Stream::Plain(sock) => {
|
||||
let mut rest = data;
|
||||
while !rest.is_empty() {
|
||||
let n = Self::retry(limits, || sock.write(rest))?;
|
||||
if n == 0 {
|
||||
return Err(HttpError::Io("connection closed while sending the request".into()));
|
||||
}
|
||||
rest = rest.get(n..).unwrap_or_default();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Stream::Tls(conn, sock) => {
|
||||
// buffered by rustls, then sent as TLS records
|
||||
conn.writer().write_all(data).map_err(|e| HttpError::Tls(e.to_string()))?;
|
||||
while conn.wants_write() {
|
||||
Self::retry(limits, || conn.write_tls(sock))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read some bytes (0 at the end of the stream), waiting at most `stall` for them.
|
||||
fn read(&mut self, buf: &mut [u8], limits: &Limits) -> Result<usize, HttpError> {
|
||||
match self {
|
||||
Stream::Plain(sock) => Self::retry(limits, || sock.read(buf)),
|
||||
Stream::Tls(conn, sock) => {
|
||||
let start = Instant::now();
|
||||
loop {
|
||||
if limits.cancel.load(Ordering::Relaxed) {
|
||||
return Err(HttpError::Cancelled);
|
||||
}
|
||||
match conn.reader().read(buf) {
|
||||
Ok(n) => return Ok(n),
|
||||
// closed without close_notify: the caller checks the length it got
|
||||
Err(e) if e.kind() == ErrorKind::UnexpectedEof => return Ok(0),
|
||||
Err(e) if e.kind() == ErrorKind::WouldBlock => {}
|
||||
Err(e) => return Err(HttpError::Tls(e.to_string())),
|
||||
}
|
||||
// no plaintext buffered: more TLS records from the socket
|
||||
match conn.read_tls(sock) {
|
||||
Ok(0) => return Ok(0),
|
||||
Ok(_) => {
|
||||
conn.process_new_packets().map_err(|e| HttpError::Tls(e.to_string()))?;
|
||||
while conn.wants_write() {
|
||||
Self::retry(limits, || conn.write_tls(sock))?;
|
||||
}
|
||||
}
|
||||
Err(e) if is_timeout(&e) => {
|
||||
if start.elapsed() >= limits.stall {
|
||||
return Err(HttpError::Stalled);
|
||||
}
|
||||
}
|
||||
Err(e) => return Err(HttpError::Io(e.to_string())),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The connection with a read buffer (headers and chunk sizes are read line by line).
|
||||
struct Conn {
|
||||
stream: Stream,
|
||||
buf: Vec<u8>,
|
||||
pos: usize,
|
||||
}
|
||||
|
||||
impl Conn {
|
||||
fn fill(&mut self, limits: &Limits) -> Result<usize, HttpError> {
|
||||
if self.pos >= self.buf.len() {
|
||||
self.buf.clear();
|
||||
self.pos = 0;
|
||||
}
|
||||
let mut tmp = [0u8; 64 * 1024];
|
||||
let n = self.stream.read(&mut tmp, limits)?;
|
||||
self.buf.extend_from_slice(tmp.get(..n).unwrap_or_default());
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// One line without its CRLF; at most `max` bytes.
|
||||
fn line(&mut self, max: usize, limits: &Limits) -> Result<String, HttpError> {
|
||||
loop {
|
||||
let avail = self.buf.get(self.pos..).unwrap_or_default();
|
||||
if let Some(i) = avail.iter().position(|b| *b == b'\n') {
|
||||
let line = avail.get(..i).unwrap_or_default();
|
||||
let line = line.strip_suffix(b"\r").unwrap_or(line);
|
||||
let s = String::from_utf8_lossy(line).into_owned();
|
||||
self.pos = self.pos.saturating_add(i + 1);
|
||||
return Ok(s);
|
||||
}
|
||||
if avail.len() > max {
|
||||
return Err(HttpError::Protocol("header line too long".into()));
|
||||
}
|
||||
// keep the partial line, read more after it
|
||||
if self.pos > 0 {
|
||||
self.buf.drain(..self.pos.min(self.buf.len()));
|
||||
self.pos = 0;
|
||||
}
|
||||
let mut tmp = [0u8; 16 * 1024];
|
||||
let n = self.stream.read(&mut tmp, limits)?;
|
||||
if n == 0 {
|
||||
return Err(HttpError::Protocol("connection closed in the middle of the response".into()));
|
||||
}
|
||||
self.buf.extend_from_slice(tmp.get(..n).unwrap_or_default());
|
||||
}
|
||||
}
|
||||
|
||||
/// Up to `out.len()` body bytes (0 at the end of the stream).
|
||||
fn read(&mut self, out: &mut [u8], limits: &Limits) -> Result<usize, HttpError> {
|
||||
if self.pos >= self.buf.len() && self.fill(limits)? == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
let avail = self.buf.get(self.pos..).unwrap_or_default();
|
||||
let n = avail.len().min(out.len());
|
||||
if let (Some(dst), Some(src)) = (out.get_mut(..n), avail.get(..n)) {
|
||||
dst.copy_from_slice(src);
|
||||
}
|
||||
self.pos = self.pos.saturating_add(n);
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
|
||||
/// A response's status, headers and body.
|
||||
pub struct Response {
|
||||
pub status: u16,
|
||||
headers: Vec<(String, String)>,
|
||||
conn: Conn,
|
||||
body: BodyMode,
|
||||
}
|
||||
|
||||
enum BodyMode {
|
||||
Length(u64),
|
||||
Chunked { left: u64, done: bool },
|
||||
Close,
|
||||
Done,
|
||||
}
|
||||
|
||||
impl Response {
|
||||
/// The first header named `name` (case-insensitive).
|
||||
pub fn header(&self, name: &str) -> Option<&str> {
|
||||
self.headers.iter().find(|(k, _)| k.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_str())
|
||||
}
|
||||
|
||||
/// The body's length when the server announced it.
|
||||
pub fn content_length(&self) -> Option<u64> {
|
||||
match self.body {
|
||||
BodyMode::Length(n) => Some(n),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Read body bytes into `out` (0 at the end of the body).
|
||||
pub fn read(&mut self, out: &mut [u8], limits: &Limits) -> Result<usize, HttpError> {
|
||||
match &mut self.body {
|
||||
BodyMode::Done => Ok(0),
|
||||
BodyMode::Length(left) => {
|
||||
if *left == 0 {
|
||||
self.body = BodyMode::Done;
|
||||
return Ok(0);
|
||||
}
|
||||
let want = usize::try_from(*left).unwrap_or(usize::MAX).min(out.len());
|
||||
let n = self.conn.read(out.get_mut(..want).unwrap_or_default(), limits)?;
|
||||
if n == 0 {
|
||||
return Err(HttpError::Protocol("the connection closed before the end of the file".into()));
|
||||
}
|
||||
*left = left.saturating_sub(n as u64);
|
||||
Ok(n)
|
||||
}
|
||||
BodyMode::Close => {
|
||||
let n = self.conn.read(out, limits)?;
|
||||
if n == 0 {
|
||||
self.body = BodyMode::Done;
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
BodyMode::Chunked { left, done } => {
|
||||
if *done {
|
||||
return Ok(0);
|
||||
}
|
||||
if *left == 0 {
|
||||
let line = self.conn.line(1024, limits)?;
|
||||
let hex = line.split(';').next().unwrap_or_default().trim();
|
||||
let size = u64::from_str_radix(hex, 16).map_err(|_| HttpError::Protocol(format!("bad chunk size `{hex}`")))?;
|
||||
if size == 0 {
|
||||
// trailers, up to the empty line
|
||||
let mut n = 0;
|
||||
while !self.conn.line(MAX_HEAD, limits)?.is_empty() {
|
||||
n += 1;
|
||||
if n > 100 {
|
||||
return Err(HttpError::Protocol("too many trailers".into()));
|
||||
}
|
||||
}
|
||||
*done = true;
|
||||
return Ok(0);
|
||||
}
|
||||
*left = size;
|
||||
}
|
||||
let want = usize::try_from(*left).unwrap_or(usize::MAX).min(out.len());
|
||||
let n = self.conn.read(out.get_mut(..want).unwrap_or_default(), limits)?;
|
||||
if n == 0 {
|
||||
return Err(HttpError::Protocol("the connection closed in the middle of a chunk".into()));
|
||||
}
|
||||
*left = left.saturating_sub(n as u64);
|
||||
if *left == 0 && !self.conn.line(16, limits)?.is_empty() {
|
||||
return Err(HttpError::Protocol("missing CRLF after a chunk".into()));
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Send `GET url` with extra `headers` and read the status line and headers.
|
||||
pub fn get(url: &Url, headers: &[(&str, String)], limits: &Limits) -> Result<Response, HttpError> {
|
||||
let mut req = format!(
|
||||
"GET {} HTTP/1.1\r\nHost: {}\r\nUser-Agent: LightCraft/{}\r\nAccept: */*\r\nAccept-Encoding: identity\r\nConnection: close\r\n",
|
||||
url.path,
|
||||
url.host_header(),
|
||||
env!("CARGO_PKG_VERSION")
|
||||
);
|
||||
for (k, v) in headers {
|
||||
if v.chars().any(|c| c == '\r' || c == '\n') || k.chars().any(|c| c == '\r' || c == '\n' || c == ':') {
|
||||
return Err(HttpError::BadUrl("header with a line break".into()));
|
||||
}
|
||||
req.push_str(&format!("{k}: {v}\r\n"));
|
||||
}
|
||||
req.push_str("\r\n");
|
||||
let mut stream = Stream::connect(url, limits)?;
|
||||
stream.write_all(req.as_bytes(), limits)?;
|
||||
let mut conn = Conn { stream, buf: Vec::new(), pos: 0 };
|
||||
let status_line = conn.line(MAX_HEAD, limits)?;
|
||||
let mut parts = status_line.split_whitespace();
|
||||
let version = parts.next().unwrap_or_default();
|
||||
if !version.starts_with("HTTP/1.") {
|
||||
return Err(HttpError::Protocol(format!("not an HTTP/1.x response: {}", status_line.chars().take(60).collect::<String>())));
|
||||
}
|
||||
let status = parts.next().and_then(|s| s.parse::<u16>().ok()).ok_or_else(|| HttpError::Protocol("bad status line".into()))?;
|
||||
let mut headers = Vec::new();
|
||||
let mut total = status_line.len();
|
||||
loop {
|
||||
let line = conn.line(MAX_HEAD, limits)?;
|
||||
if line.is_empty() {
|
||||
break;
|
||||
}
|
||||
total = total.saturating_add(line.len());
|
||||
if total > MAX_HEAD || headers.len() > 200 {
|
||||
return Err(HttpError::Protocol("response headers too large".into()));
|
||||
}
|
||||
if let Some((k, v)) = line.split_once(':') {
|
||||
headers.push((k.trim().to_string(), v.trim().to_string()));
|
||||
}
|
||||
}
|
||||
let find = |name: &str| headers.iter().find(|(k, _)| k.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_str());
|
||||
let body = if status == 204 || status == 304 || (100..200).contains(&status) {
|
||||
BodyMode::Done
|
||||
} else if find("transfer-encoding").is_some_and(|v| v.to_ascii_lowercase().contains("chunked")) {
|
||||
BodyMode::Chunked { left: 0, done: false }
|
||||
} else if let Some(n) = find("content-length") {
|
||||
BodyMode::Length(n.trim().parse::<u64>().map_err(|_| HttpError::Protocol(format!("bad Content-Length `{n}`")))?)
|
||||
} else {
|
||||
BodyMode::Close
|
||||
};
|
||||
Ok(Response { status, headers, conn, body })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_urls() {
|
||||
let u = Url::parse("https://cdn.example.com/models/sam3/model.safetensors?sig=1#x").unwrap();
|
||||
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "cdn.example.com", 443, "/models/sam3/model.safetensors?sig=1"));
|
||||
assert_eq!(u.display(), "https://cdn.example.com/models/sam3/model.safetensors");
|
||||
let u = Url::parse("http://127.0.0.1:8080").unwrap();
|
||||
assert_eq!((u.tls, u.port, u.path.as_str()), (false, 8080, "/"));
|
||||
let u = Url::parse("http://[::1]:9/a").unwrap();
|
||||
assert_eq!((u.host.as_str(), u.port), ("::1", 9));
|
||||
for bad in ["ftp://x/y", "https://", "https://a b/", "https://u:p@h/", "http://h:0/", "http://h:99999/", "https://h/\r\nX: y", "file:///etc"]
|
||||
{
|
||||
assert!(Url::parse(bad).is_err(), "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_redirects() {
|
||||
let u = Url::parse("https://a.example/x/y/file?q").unwrap();
|
||||
assert_eq!(u.join("https://b.example/z").unwrap().host, "b.example");
|
||||
assert_eq!(u.join("/root").unwrap().path, "/root");
|
||||
assert_eq!(u.join("other").unwrap().path, "/x/y/other");
|
||||
assert_eq!(u.join("//c.example/p").unwrap().host, "c.example");
|
||||
assert!(u.join("ftp://nope").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_tls_config_builds_with_the_pure_rust_provider() {
|
||||
assert!(tls_config().is_ok());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,424 @@
|
||||
//! Downloading the SAM 3 model, only when the user asked for it: from an ordered list of mirrors
|
||||
//! (the next one when a mirror fails), resuming partial files, with connect and stall timeouts,
|
||||
//! a size cap, progress and cancellation, and checked (exact size, SHA-256 where pinned) before
|
||||
//! each file is moved into the model folder. Runs on whatever thread calls [`download`] (the
|
||||
//! engine uses a background thread; the UI thread never waits for the network).
|
||||
//!
|
||||
//! Files are written to `<name>.part` next to their final place and renamed when verified, so a
|
||||
//! model folder never holds a half-written file under a real name, and an interrupted download
|
||||
//! resumes from where it stopped. A file that fails its check is deleted, never kept.
|
||||
//!
|
||||
//! The weights are never part of LightCraft (SAM License, see docs/ai-masks.md); where they are
|
||||
//! downloaded from is configured by [`DEFAULT_MIRRORS`] and the user's own list.
|
||||
|
||||
pub mod http;
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use http::{HttpError, Limits, Url};
|
||||
|
||||
/// One file of the model.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct FileSpec {
|
||||
pub name: &'static str,
|
||||
/// Exact size in bytes, when pinned (then the download resumes and is checked against it).
|
||||
pub size: Option<u64>,
|
||||
/// SHA-256 (lower-case hex), when pinned.
|
||||
pub sha256: Option<&'static str>,
|
||||
/// Largest size accepted when `size` is not pinned.
|
||||
pub max: u64,
|
||||
}
|
||||
|
||||
/// The `facebook/sam3` checkpoint files [`crate::Sam3::load`] needs.
|
||||
///
|
||||
/// `model.safetensors` is pinned to the official checkpoint (Hugging Face LFS SHA-256). The two
|
||||
/// tokenizer files are small text files without a pinned hash yet: they are only parsed by the
|
||||
/// tokenizer (never executed) and capped in size.
|
||||
// TODO(maintainer): pin `vocab.json` and `merges.txt` (size + SHA-256) when the files are
|
||||
// uploaded to LightCraft's CDN.
|
||||
pub const SAM3_FILES: &[FileSpec] = &[
|
||||
FileSpec { name: "vocab.json", size: None, sha256: None, max: 16 << 20 },
|
||||
FileSpec { name: "merges.txt", size: None, sha256: None, max: 16 << 20 },
|
||||
FileSpec {
|
||||
name: crate::WEIGHTS_FILE,
|
||||
size: Some(SAM3_WEIGHTS_SIZE),
|
||||
sha256: Some("6d06f0a5f84e435071fe6603e61d0b4cc7b40e0d39d487cfd4d67d8cc11cc14a"),
|
||||
max: SAM3_WEIGHTS_SIZE,
|
||||
},
|
||||
];
|
||||
|
||||
/// Size of the official `model.safetensors` (3.44 GB).
|
||||
pub const SAM3_WEIGHTS_SIZE: u64 = 3_439_938_512;
|
||||
|
||||
/// Where LightCraft downloads the model from, in order: each is a base URL, and file `f` is at
|
||||
/// `<base>/<f>`. **Empty for now**: LightCraft has no download location of its own yet, so the
|
||||
/// in-app download needs the user's list (`LIGHTCRAFT_SAM3_MIRRORS` or the mirrors file, see
|
||||
/// [`mirrors`] and docs/ai-masks.md) until this is filled in.
|
||||
///
|
||||
/// Hugging Face's `facebook/sam3` can't be a default: it is gated (each user must accept the
|
||||
/// SAM License there and download with their own token).
|
||||
// TODO(maintainer): add LightCraft's CDN locations (https, primary first), e.g.
|
||||
// "https://<cdn-host>/models/sam3/<revision>" — the files there must match `SAM3_FILES`.
|
||||
pub const DEFAULT_MIRRORS: &[&str] = &[];
|
||||
|
||||
/// Environment variable with extra mirrors (base URLs separated by commas, spaces or newlines),
|
||||
/// tried before the defaults.
|
||||
pub const MIRRORS_ENV: &str = "LIGHTCRAFT_SAM3_MIRRORS";
|
||||
|
||||
/// Most mirrors used (a hostile list can't make a download loop forever).
|
||||
const MAX_MIRRORS: usize = 16;
|
||||
/// Redirects followed per request.
|
||||
const MAX_REDIRECTS: usize = 8;
|
||||
|
||||
/// The mirrors to try, in order: the environment variable's, then the mirrors file's (one base
|
||||
/// URL per line, `#` comments), then [`DEFAULT_MIRRORS`]; duplicates and unusable URLs dropped.
|
||||
pub fn mirrors(env: Option<&str>, file: Option<&Path>) -> Vec<String> {
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
let mut add = |s: &str| {
|
||||
let s = s.trim().trim_end_matches('/');
|
||||
if !s.is_empty() && Url::parse(s).is_ok() && !out.iter().any(|m| m == s) && out.len() < MAX_MIRRORS {
|
||||
out.push(s.to_string());
|
||||
}
|
||||
};
|
||||
if let Some(env) = env {
|
||||
env.split([',', ' ', '\n', '\t']).for_each(&mut add);
|
||||
}
|
||||
// a small text file; anything larger isn't a mirror list
|
||||
if let Some(text) = file.and_then(|f| std::fs::metadata(f).ok().filter(|m| m.len() < 64 * 1024).and_then(|_| std::fs::read_to_string(f).ok())) {
|
||||
text.lines().map(|l| l.split('#').next().unwrap_or_default()).for_each(&mut add);
|
||||
}
|
||||
DEFAULT_MIRRORS.iter().for_each(|m| add(m));
|
||||
out
|
||||
}
|
||||
|
||||
/// Download settings.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Options {
|
||||
/// Longest wait to connect to a mirror.
|
||||
pub connect_timeout: Duration,
|
||||
/// Longest wait for the next bytes once connected.
|
||||
pub stall_timeout: Duration,
|
||||
/// Tries per mirror and file (each resumes where the last one stopped).
|
||||
pub attempts: u32,
|
||||
}
|
||||
|
||||
impl Default for Options {
|
||||
fn default() -> Self {
|
||||
Options { connect_timeout: Duration::from_secs(15), stall_timeout: Duration::from_secs(30), attempts: 3 }
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a download is.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Progress {
|
||||
pub file: String,
|
||||
/// Bytes of all files on disk so far, and the total expected (pinned sizes + announced ones).
|
||||
pub done: u64,
|
||||
pub total: u64,
|
||||
/// The mirror in use (without query).
|
||||
pub mirror: String,
|
||||
}
|
||||
|
||||
/// Why a download failed.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum DownloadError {
|
||||
/// No mirror is configured.
|
||||
NoMirrors,
|
||||
Cancelled,
|
||||
/// A local file problem (folder not writable, disk full…).
|
||||
Disk(String),
|
||||
/// Every mirror failed for `file`; the reason for each.
|
||||
AllMirrorsFailed {
|
||||
file: String,
|
||||
errors: Vec<String>,
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Display for DownloadError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
DownloadError::NoMirrors => write!(
|
||||
f,
|
||||
"no download location is configured for the SAM 3 model in this build (set {MIRRORS_ENV}, or put the files in the model folder yourself; see docs/ai-masks.md)"
|
||||
),
|
||||
DownloadError::Cancelled => write!(f, "the download was cancelled"),
|
||||
DownloadError::Disk(e) => write!(f, "{e}"),
|
||||
DownloadError::AllMirrorsFailed { file, errors } => write!(f, "could not download {file}: {}", errors.join("; ")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One mirror's failure for one file; `Retry` resumes on the same mirror.
|
||||
enum Fail {
|
||||
Retry(String),
|
||||
NextMirror(String),
|
||||
Cancelled,
|
||||
Disk(String),
|
||||
}
|
||||
|
||||
/// Download `files` into `dir` from `mirrors` (in order; see [`mirrors`]). Files already there
|
||||
/// and verified are kept. `progress` is called as bytes arrive (often: throttle in it).
|
||||
pub fn download(
|
||||
files: &[FileSpec],
|
||||
mirrors: &[String],
|
||||
dir: &Path,
|
||||
opts: &Options,
|
||||
cancel: &AtomicBool,
|
||||
progress: &mut dyn FnMut(&Progress),
|
||||
) -> Result<(), DownloadError> {
|
||||
std::fs::create_dir_all(dir).map_err(|e| DownloadError::Disk(format!("can't create {}: {e}", dir.display())))?;
|
||||
// what's left to fetch
|
||||
let mut todo = Vec::new();
|
||||
for f in files {
|
||||
if !verified(f, &dir.join(f.name))? {
|
||||
todo.push(f);
|
||||
}
|
||||
}
|
||||
if todo.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if mirrors.is_empty() {
|
||||
return Err(DownloadError::NoMirrors);
|
||||
}
|
||||
let total: u64 = files.iter().map(|f| f.size.unwrap_or(0)).sum();
|
||||
let mut before: u64 = files.iter().filter(|f| !todo.contains(f)).filter_map(|f| f.size).sum();
|
||||
for f in todo {
|
||||
let mut errors = Vec::new();
|
||||
let mut ok = false;
|
||||
'mirrors: for m in mirrors {
|
||||
let url = format!("{m}/{}", f.name);
|
||||
for _ in 0..opts.attempts.max(1) {
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
return Err(DownloadError::Cancelled);
|
||||
}
|
||||
let mut report = |n: u64, announced: u64| {
|
||||
let mirror = Url::parse(m).map(|u| u.display()).unwrap_or_default();
|
||||
progress(&Progress {
|
||||
file: f.name.to_string(),
|
||||
done: before.saturating_add(n),
|
||||
total: total.max(before.saturating_add(announced)),
|
||||
mirror,
|
||||
})
|
||||
};
|
||||
match fetch_file(f, &url, dir, opts, cancel, &mut report) {
|
||||
Ok(()) => {
|
||||
ok = true;
|
||||
break 'mirrors;
|
||||
}
|
||||
Err(Fail::Cancelled) => return Err(DownloadError::Cancelled),
|
||||
Err(Fail::Disk(e)) => return Err(DownloadError::Disk(e)),
|
||||
Err(Fail::Retry(e)) => {
|
||||
log::warn!("SAM 3 download of {url}: {e} (retrying)");
|
||||
errors.push(format!("{}: {e}", short(m)));
|
||||
}
|
||||
Err(Fail::NextMirror(e)) => {
|
||||
log::warn!("SAM 3 download of {url}: {e}");
|
||||
errors.push(format!("{}: {e}", short(m)));
|
||||
continue 'mirrors;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
return Err(DownloadError::AllMirrorsFailed { file: f.name.to_string(), errors });
|
||||
}
|
||||
before = before.saturating_add(std::fs::metadata(dir.join(f.name)).map(|m| m.len()).unwrap_or(0));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A mirror for messages (no query string: it may hold a token).
|
||||
fn short(m: &str) -> String {
|
||||
Url::parse(m).map(|u| u.display()).unwrap_or_else(|_| "mirror".into())
|
||||
}
|
||||
|
||||
/// Whether `path` is already the right file (exact size and hash when pinned).
|
||||
fn verified(f: &FileSpec, path: &Path) -> Result<bool, DownloadError> {
|
||||
let Ok(meta) = std::fs::metadata(path) else { return Ok(false) };
|
||||
if !meta.is_file() || meta.len() == 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
if f.size.is_some_and(|s| s != meta.len()) || meta.len() > f.max {
|
||||
return Ok(false);
|
||||
}
|
||||
match f.sha256 {
|
||||
Some(want) => Ok(sha256_file(path).map_err(DownloadError::Disk)? == want),
|
||||
None => Ok(true),
|
||||
}
|
||||
}
|
||||
|
||||
fn sha256_file(path: &Path) -> Result<String, String> {
|
||||
let mut file = File::open(path).map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
let mut h = Sha256::new();
|
||||
let mut buf = vec![0u8; 1 << 20];
|
||||
loop {
|
||||
let n = file.read(&mut buf).map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
h.update(buf.get(..n).unwrap_or_default());
|
||||
}
|
||||
Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect())
|
||||
}
|
||||
|
||||
fn part_path(dir: &Path, name: &str) -> PathBuf {
|
||||
dir.join(format!("{name}.part"))
|
||||
}
|
||||
|
||||
/// Fetch one file from one mirror into `<name>.part` (resuming it when its size is pinned),
|
||||
/// check it and move it into place.
|
||||
fn fetch_file(f: &FileSpec, url: &str, dir: &Path, opts: &Options, cancel: &AtomicBool, report: &mut dyn FnMut(u64, u64)) -> Result<(), Fail> {
|
||||
let part = part_path(dir, f.name);
|
||||
let limit = f.size.unwrap_or(f.max);
|
||||
let disk = |e: std::io::Error| Fail::Disk(format!("{}: {e}", part.display()));
|
||||
// resume only a pinned file (an unpinned one can't be checked after resuming)
|
||||
let mut have = match std::fs::metadata(&part) {
|
||||
Ok(m) if f.size.is_some() && m.len() <= limit => m.len(),
|
||||
Ok(_) => {
|
||||
std::fs::remove_file(&part).map_err(disk)?;
|
||||
0
|
||||
}
|
||||
Err(_) => 0,
|
||||
};
|
||||
let fetched = f.size != Some(have) || have == 0;
|
||||
if fetched {
|
||||
let limits = Limits { connect: opts.connect_timeout, stall: opts.stall_timeout, cancel };
|
||||
let mut url = Url::parse(url).map_err(|e| Fail::NextMirror(e.to_string()))?;
|
||||
let origin = url.host.clone();
|
||||
let mut redirects = 0;
|
||||
let mut resp = loop {
|
||||
let mut headers = vec![];
|
||||
if have > 0 {
|
||||
headers.push(("Range", format!("bytes={have}-")));
|
||||
}
|
||||
// a token for the configured host only (never sent on to a redirect's host)
|
||||
if let Ok(t) = std::env::var("LIGHTCRAFT_SAM3_TOKEN")
|
||||
&& !t.trim().is_empty()
|
||||
&& url.tls
|
||||
&& url.host == origin
|
||||
{
|
||||
headers.push(("Authorization", format!("Bearer {}", t.trim())));
|
||||
}
|
||||
let r = http::get(&url, &headers, &limits).map_err(http_fail)?;
|
||||
match r.status {
|
||||
301 | 302 | 303 | 307 | 308 => {
|
||||
redirects += 1;
|
||||
if redirects > MAX_REDIRECTS {
|
||||
return Err(Fail::NextMirror("too many redirects".into()));
|
||||
}
|
||||
let loc = r.header("location").ok_or_else(|| Fail::NextMirror("redirect without a location".into()))?;
|
||||
url = url.join(loc).map_err(|e| Fail::NextMirror(e.to_string()))?;
|
||||
}
|
||||
_ => break r,
|
||||
}
|
||||
};
|
||||
let mut file = match resp.status {
|
||||
200 => {
|
||||
// the whole file (from the start, also when a range was asked for)
|
||||
if resp.content_length().is_some_and(|n| n > limit || f.size.is_some_and(|s| s != n)) {
|
||||
return Err(Fail::NextMirror(format!("the server's file has the wrong size ({:?} bytes)", resp.content_length())));
|
||||
}
|
||||
have = 0;
|
||||
File::create(&part).map_err(disk)?
|
||||
}
|
||||
206 if have > 0 => {
|
||||
let range = resp.header("content-range").unwrap_or_default();
|
||||
let (start, total) = parse_content_range(range).ok_or_else(|| Fail::NextMirror(format!("bad Content-Range `{range}`")))?;
|
||||
if start != have || total.is_some_and(|t| Some(t) != f.size) {
|
||||
return Err(Fail::NextMirror(format!("the server resumed at the wrong place ({range})")));
|
||||
}
|
||||
OpenOptions::new().append(true).open(&part).map_err(disk)?
|
||||
}
|
||||
416 => {
|
||||
// the partial file doesn't fit what the server has: start over
|
||||
std::fs::remove_file(&part).map_err(disk)?;
|
||||
return Err(Fail::Retry("the partial download didn't match; starting over".into()));
|
||||
}
|
||||
401 | 403 => return Err(Fail::NextMirror(format!("access denied (HTTP {})", resp.status))),
|
||||
404 | 410 => return Err(Fail::NextMirror(format!("not found (HTTP {})", resp.status))),
|
||||
s if s >= 500 => return Err(Fail::Retry(format!("server error (HTTP {s})"))),
|
||||
s => return Err(Fail::NextMirror(format!("HTTP {s}"))),
|
||||
};
|
||||
let announced = f.size.unwrap_or_else(|| resp.content_length().unwrap_or(0));
|
||||
report(have, announced);
|
||||
let mut buf = vec![0u8; 256 * 1024];
|
||||
loop {
|
||||
let n = match resp.read(&mut buf, &limits) {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
let _ = file.flush();
|
||||
if f.size.is_none() {
|
||||
drop(file);
|
||||
let _ = std::fs::remove_file(&part);
|
||||
}
|
||||
return Err(http_fail(e));
|
||||
}
|
||||
};
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
if have.saturating_add(n as u64) > limit {
|
||||
drop(file);
|
||||
let _ = std::fs::remove_file(&part);
|
||||
return Err(Fail::NextMirror(format!("the server sent more than the expected {limit} bytes")));
|
||||
}
|
||||
file.write_all(buf.get(..n).unwrap_or_default()).map_err(disk)?;
|
||||
have = have.saturating_add(n as u64);
|
||||
report(have, announced);
|
||||
}
|
||||
file.sync_all().map_err(disk)?;
|
||||
}
|
||||
// checks: exact size, hash
|
||||
let len = std::fs::metadata(&part).map_err(disk)?.len();
|
||||
if f.size.is_some_and(|s| s != len) {
|
||||
return Err(Fail::Retry(format!("incomplete ({len} of {} bytes)", f.size.unwrap_or(0))));
|
||||
}
|
||||
if len == 0 {
|
||||
let _ = std::fs::remove_file(&part);
|
||||
return Err(Fail::NextMirror("the server sent an empty file".into()));
|
||||
}
|
||||
if let Some(want) = f.sha256 {
|
||||
let got = sha256_file(&part).map_err(Fail::Disk)?;
|
||||
if got != want {
|
||||
std::fs::remove_file(&part).map_err(disk)?;
|
||||
let why = format!("the file is damaged or not the official one (SHA-256 {got})");
|
||||
// a damaged partial file from an earlier run isn't this mirror's fault
|
||||
return Err(if fetched { Fail::NextMirror(why) } else { Fail::Retry(why) });
|
||||
}
|
||||
}
|
||||
std::fs::rename(&part, dir.join(f.name)).map_err(disk)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn http_fail(e: HttpError) -> Fail {
|
||||
match e {
|
||||
HttpError::Cancelled => Fail::Cancelled,
|
||||
HttpError::Stalled | HttpError::Io(_) | HttpError::Protocol(_) => Fail::Retry(e.to_string()),
|
||||
HttpError::BadUrl(_) | HttpError::Connect(_) | HttpError::Tls(_) => Fail::NextMirror(e.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// `bytes a-b/total` → (a, total) (`*` total → None).
|
||||
fn parse_content_range(v: &str) -> Option<(u64, Option<u64>)> {
|
||||
let v = v.trim().strip_prefix("bytes")?.trim();
|
||||
let (range, total) = v.split_once('/')?;
|
||||
let (a, b) = range.split_once('-')?;
|
||||
let (a, b) = (a.trim().parse::<u64>().ok()?, b.trim().parse::<u64>().ok()?);
|
||||
if b < a {
|
||||
return None;
|
||||
}
|
||||
let total = match total.trim() {
|
||||
"*" => None,
|
||||
t => Some(t.parse::<u64>().ok()?),
|
||||
};
|
||||
Some((a, total))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,314 @@
|
||||
//! The downloader against a local HTTP server (std `TcpListener`; never the internet): mirror
|
||||
//! fallback, redirects, resuming, chunked bodies, stalls, hash mismatches, size caps, cancel.
|
||||
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::net::{TcpListener, TcpStream};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use super::*;
|
||||
|
||||
/// The served file: 300 kB of a deterministic pattern.
|
||||
fn payload() -> Vec<u8> {
|
||||
(0..300_000u32).map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8).collect()
|
||||
}
|
||||
|
||||
fn sha_hex(b: &[u8]) -> String {
|
||||
Sha256::digest(b).iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
fn leak(s: String) -> &'static str {
|
||||
Box::leak(s.into_boxed_str())
|
||||
}
|
||||
|
||||
fn pinned() -> FileSpec {
|
||||
let p = payload();
|
||||
FileSpec { name: "model.bin", size: Some(p.len() as u64), sha256: Some(leak(sha_hex(&p))), max: p.len() as u64 }
|
||||
}
|
||||
|
||||
struct Server {
|
||||
base: String,
|
||||
hits: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
/// Serve on 127.0.0.1; the first path segment picks the behaviour.
|
||||
fn server() -> Server {
|
||||
let l = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let base = format!("http://{}", l.local_addr().unwrap());
|
||||
let hits = Arc::new(AtomicUsize::new(0));
|
||||
let h = hits.clone();
|
||||
std::thread::spawn(move || {
|
||||
for s in l.incoming().flatten() {
|
||||
let h = h.clone();
|
||||
std::thread::spawn(move || handle(s, &h));
|
||||
}
|
||||
});
|
||||
Server { base, hits }
|
||||
}
|
||||
|
||||
fn handle(mut s: TcpStream, hits: &AtomicUsize) {
|
||||
let n = hits.fetch_add(1, Ordering::SeqCst);
|
||||
let mut r = BufReader::new(s.try_clone().unwrap());
|
||||
let mut first = String::new();
|
||||
if r.read_line(&mut first).is_err() {
|
||||
return;
|
||||
}
|
||||
let mut range: Option<u64> = None;
|
||||
loop {
|
||||
let mut l = String::new();
|
||||
if r.read_line(&mut l).unwrap_or(0) == 0 || l.trim().is_empty() {
|
||||
break;
|
||||
}
|
||||
if let Some(v) = l.to_ascii_lowercase().strip_prefix("range: bytes=") {
|
||||
range = v.trim().trim_end_matches('-').parse().ok();
|
||||
}
|
||||
}
|
||||
let path = first.split_whitespace().nth(1).unwrap_or("/").to_string();
|
||||
let mode = path.trim_start_matches('/').split('/').next().unwrap_or("").to_string();
|
||||
let body = payload();
|
||||
let send = |s: &mut TcpStream, status: &str, headers: &str, data: &[u8]| {
|
||||
let _ = s.write_all(format!("HTTP/1.1 {status}\r\n{headers}Connection: close\r\n\r\n").as_bytes());
|
||||
let _ = s.write_all(data);
|
||||
};
|
||||
match mode.as_str() {
|
||||
"missing" => send(&mut s, "404 Not Found", "Content-Length: 9\r\n", b"not found"),
|
||||
"error" => send(&mut s, "500 Oops", "Content-Length: 25\r\n", b"<html>server error</html>"),
|
||||
"redirect" => send(&mut s, "302 Found", &format!("Location: {}\r\nContent-Length: 0\r\n", path.replacen("/redirect", "/good", 1)), b""),
|
||||
"wrong" => {
|
||||
let mut b = body.clone();
|
||||
b[1000] ^= 0xff;
|
||||
send(&mut s, "200 OK", &format!("Content-Length: {}\r\n", b.len()), &b)
|
||||
}
|
||||
"toolong" => {
|
||||
// no length announced, more bytes than the file can have
|
||||
let mut b = body.clone();
|
||||
b.extend_from_slice(&[7u8; 5000]);
|
||||
send(&mut s, "200 OK", "", &b)
|
||||
}
|
||||
"chunked" => {
|
||||
let _ = s.write_all(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n");
|
||||
for c in body.chunks(70_001) {
|
||||
let _ = s.write_all(format!("{:x};ext=1\r\n", c.len()).as_bytes());
|
||||
let _ = s.write_all(c);
|
||||
let _ = s.write_all(b"\r\n");
|
||||
}
|
||||
let _ = s.write_all(b"0\r\nX-Trailer: 1\r\n\r\n");
|
||||
}
|
||||
"stall" => {
|
||||
// half the file, then nothing
|
||||
let _ = s.write_all(format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len()).as_bytes());
|
||||
let _ = s.write_all(&body[..body.len() / 2]);
|
||||
std::thread::sleep(Duration::from_secs(5));
|
||||
}
|
||||
"slow" => {
|
||||
let _ = s.write_all(format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len()).as_bytes());
|
||||
for c in body.chunks(1000) {
|
||||
if s.write_all(c).is_err() {
|
||||
return;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
}
|
||||
"flaky" if n.is_multiple_of(2) && range.is_none() => {
|
||||
// the first try breaks off after a third
|
||||
let _ = s.write_all(format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len()).as_bytes());
|
||||
let _ = s.write_all(&body[..body.len() / 3]);
|
||||
}
|
||||
"range416" if range.is_some() => send(&mut s, "416 Range Not Satisfiable", "Content-Length: 0\r\n", b""),
|
||||
"norange" | "range416" => send(&mut s, "200 OK", &format!("Content-Length: {}\r\n", body.len()), &body),
|
||||
_ => match range {
|
||||
Some(from) if from as usize >= body.len() => send(&mut s, "416 Range Not Satisfiable", "Content-Length: 0\r\n", b""),
|
||||
Some(from) => send(
|
||||
&mut s,
|
||||
"206 Partial Content",
|
||||
&format!("Content-Range: bytes {from}-{}/{}\r\nContent-Length: {}\r\n", body.len() - 1, body.len(), body.len() - from as usize),
|
||||
&body[from as usize..],
|
||||
),
|
||||
None => send(&mut s, "200 OK", &format!("Content-Length: {}\r\n", body.len()), &body),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn tmp(name: &str) -> PathBuf {
|
||||
let d = std::env::temp_dir().join(format!("lc-sam3-fetch-{name}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
d
|
||||
}
|
||||
|
||||
fn quick() -> Options {
|
||||
Options { connect_timeout: Duration::from_secs(2), stall_timeout: Duration::from_millis(600), attempts: 2 }
|
||||
}
|
||||
|
||||
fn run(files: &[FileSpec], mirrors: &[String], dir: &Path, cancel: &AtomicBool) -> (Result<(), DownloadError>, Vec<Progress>) {
|
||||
let mut seen = Vec::new();
|
||||
let r = download(files, mirrors, dir, &quick(), cancel, &mut |p| seen.push(p.clone()));
|
||||
(r, seen)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn falls_back_to_the_next_mirror_and_verifies() {
|
||||
let srv = server();
|
||||
let dir = tmp("fallback");
|
||||
let mirrors = vec![
|
||||
format!("{}/missing", srv.base),
|
||||
format!("{}/error", srv.base),
|
||||
"http://127.0.0.1:1/unreachable".to_string(),
|
||||
format!("{}/redirect", srv.base),
|
||||
];
|
||||
let (r, seen) = run(&[pinned()], &mirrors, &dir, &AtomicBool::new(false));
|
||||
r.unwrap();
|
||||
assert_eq!(std::fs::read(dir.join("model.bin")).unwrap(), payload());
|
||||
assert!(!part_path(&dir, "model.bin").exists());
|
||||
let last = seen.last().unwrap();
|
||||
assert_eq!((last.done, last.total), (300_000, 300_000));
|
||||
// an error page never ends up in the file: nothing left over from the failed mirrors
|
||||
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1);
|
||||
// installed: nothing is fetched again
|
||||
let hits = srv.hits.load(Ordering::SeqCst);
|
||||
run(&[pinned()], &mirrors, &dir, &AtomicBool::new(false)).0.unwrap();
|
||||
assert_eq!(srv.hits.load(Ordering::SeqCst), hits);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_hash_mismatch_is_deleted_and_the_next_mirror_used() {
|
||||
let srv = server();
|
||||
let dir = tmp("hash");
|
||||
let (r, _) = run(&[pinned()], &[format!("{}/wrong", srv.base)], &dir, &AtomicBool::new(false));
|
||||
let e = r.unwrap_err().to_string();
|
||||
assert!(e.contains("SHA-256"), "{e}");
|
||||
assert!(!dir.join("model.bin").exists() && !part_path(&dir, "model.bin").exists(), "a damaged file is never kept");
|
||||
run(&[pinned()], &[format!("{}/wrong", srv.base), format!("{}/good", srv.base)], &dir, &AtomicBool::new(false)).0.unwrap();
|
||||
assert_eq!(std::fs::read(dir.join("model.bin")).unwrap(), payload());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resumes_partial_files_and_restarts_bad_ones() {
|
||||
let srv = server();
|
||||
let dir = tmp("resume");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
// a third already there
|
||||
std::fs::write(part_path(&dir, "model.bin"), &payload()[..100_000]).unwrap();
|
||||
run(&[pinned()], &[format!("{}/good", srv.base)], &dir, &AtomicBool::new(false)).0.unwrap();
|
||||
assert_eq!(std::fs::read(dir.join("model.bin")).unwrap(), payload());
|
||||
// a partial file with the wrong bytes: the hash fails, it's deleted, the next try starts over
|
||||
std::fs::remove_file(dir.join("model.bin")).unwrap();
|
||||
std::fs::write(part_path(&dir, "model.bin"), vec![0u8; 100_000]).unwrap();
|
||||
run(&[pinned()], &[format!("{}/good", srv.base), format!("{}/good", srv.base)], &dir, &AtomicBool::new(false)).0.unwrap();
|
||||
assert_eq!(std::fs::read(dir.join("model.bin")).unwrap(), payload());
|
||||
// a server ignoring Range: the whole file again, not appended
|
||||
std::fs::remove_file(dir.join("model.bin")).unwrap();
|
||||
std::fs::write(part_path(&dir, "model.bin"), &payload()[..5]).unwrap();
|
||||
run(&[pinned()], &[format!("{}/norange", srv.base)], &dir, &AtomicBool::new(false)).0.unwrap();
|
||||
assert_eq!(std::fs::read(dir.join("model.bin")).unwrap(), payload());
|
||||
// a connection breaking off is resumed on the same mirror
|
||||
std::fs::remove_file(dir.join("model.bin")).unwrap();
|
||||
run(&[pinned()], &[format!("{}/flaky", srv.base)], &dir, &AtomicBool::new(false)).0.unwrap();
|
||||
assert_eq!(std::fs::read(dir.join("model.bin")).unwrap(), payload());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_416_or_a_bad_complete_part_starts_over() {
|
||||
let srv = server();
|
||||
let dir = tmp("416");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
// the server refuses the range: the part is dropped and the file fetched whole (no loop)
|
||||
std::fs::write(part_path(&dir, "model.bin"), &payload()[..100]).unwrap();
|
||||
run(&[pinned()], &[format!("{}/range416", srv.base)], &dir, &AtomicBool::new(false)).0.unwrap();
|
||||
assert_eq!(std::fs::read(dir.join("model.bin")).unwrap(), payload());
|
||||
// as long as the file but wrong: hash fails → deleted → downloaded again from the same mirror
|
||||
std::fs::remove_file(dir.join("model.bin")).unwrap();
|
||||
std::fs::write(part_path(&dir, "model.bin"), vec![1u8; 300_000]).unwrap();
|
||||
run(&[pinned()], &[format!("{}/good", srv.base)], &dir, &AtomicBool::new(false)).0.unwrap();
|
||||
assert_eq!(std::fs::read(dir.join("model.bin")).unwrap(), payload());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunked_bodies_and_unpinned_files() {
|
||||
let srv = server();
|
||||
let dir = tmp("chunked");
|
||||
let f = FileSpec { name: "vocab.json", size: None, sha256: None, max: 1 << 20 };
|
||||
run(std::slice::from_ref(&f), &[format!("{}/chunked", srv.base)], &dir, &AtomicBool::new(false)).0.unwrap();
|
||||
assert_eq!(std::fs::read(dir.join("vocab.json")).unwrap(), payload());
|
||||
// more than the cap: refused, nothing kept
|
||||
let small = FileSpec { max: 1000, ..f };
|
||||
let dir2 = tmp("cap");
|
||||
let e = run(&[small], &[format!("{}/good", srv.base)], &dir2, &AtomicBool::new(false)).0.unwrap_err().to_string();
|
||||
assert!(e.contains("wrong size") || e.contains("more than"), "{e}");
|
||||
assert!(!dir2.join("vocab.json").exists() && !part_path(&dir2, "vocab.json").exists());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
let _ = std::fs::remove_dir_all(&dir2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_server_sending_too_much_is_cut_off() {
|
||||
let srv = server();
|
||||
let dir = tmp("toolong");
|
||||
let e = run(&[pinned()], &[format!("{}/toolong", srv.base)], &dir, &AtomicBool::new(false)).0.unwrap_err().to_string();
|
||||
assert!(e.contains("more than"), "{e}");
|
||||
assert!(!dir.join("model.bin").exists() && !part_path(&dir, "model.bin").exists());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stalled_server_times_out_and_the_next_mirror_finishes() {
|
||||
let srv = server();
|
||||
let dir = tmp("stall");
|
||||
let t = Instant::now();
|
||||
let (r, _) = run(&[pinned()], &[format!("{}/stall", srv.base), format!("{}/good", srv.base)], &dir, &AtomicBool::new(false));
|
||||
r.unwrap();
|
||||
assert!(t.elapsed() < Duration::from_secs(4), "{:?}", t.elapsed());
|
||||
assert_eq!(std::fs::read(dir.join("model.bin")).unwrap(), payload());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_stops_quickly_and_keeps_the_part_for_later() {
|
||||
let srv = server();
|
||||
let dir = tmp("cancel");
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let c = cancel.clone();
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(400));
|
||||
c.store(true, Ordering::SeqCst);
|
||||
});
|
||||
let t = Instant::now();
|
||||
let (r, _) = run(&[pinned()], &[format!("{}/slow", srv.base)], &dir, &cancel);
|
||||
assert_eq!(r, Err(DownloadError::Cancelled));
|
||||
assert!(t.elapsed() < Duration::from_secs(3), "{:?}", t.elapsed());
|
||||
assert!(!dir.join("model.bin").exists());
|
||||
let kept = std::fs::metadata(part_path(&dir, "model.bin")).map(|m| m.len()).unwrap_or(0);
|
||||
assert!(kept > 0 && kept < 300_000, "{kept}");
|
||||
// and the next run resumes it
|
||||
run(&[pinned()], &[format!("{}/good", srv.base)], &dir, &AtomicBool::new(false)).0.unwrap();
|
||||
assert_eq!(std::fs::read(dir.join("model.bin")).unwrap(), payload());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_mirrors_is_a_clear_error_and_mirror_lists_are_parsed() {
|
||||
let dir = tmp("nomirror");
|
||||
let e = run(&[pinned()], &[], &dir, &AtomicBool::new(false)).0.unwrap_err();
|
||||
assert_eq!(e, DownloadError::NoMirrors);
|
||||
assert!(e.to_string().contains(MIRRORS_ENV));
|
||||
let file = std::env::temp_dir().join(format!("lc-sam3-mirrors-{}.txt", std::process::id()));
|
||||
std::fs::write(&file, "# mine\nhttps://b.example/m/ # second\n\nnot a url\nhttps://a.example/x\n").unwrap();
|
||||
let m = mirrors(Some("https://a.example/x/, ftp://no"), Some(&file));
|
||||
assert_eq!(m[..2], ["https://a.example/x".to_string(), "https://b.example/m".to_string()]);
|
||||
assert_eq!(m.len(), 2 + DEFAULT_MIRRORS.len());
|
||||
assert_eq!(mirrors(None, Some(Path::new("/nonexistent/mirrors.txt"))).len(), DEFAULT_MIRRORS.len());
|
||||
let _ = std::fs::remove_file(&file);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_ranges() {
|
||||
assert_eq!(parse_content_range("bytes 100-299/300"), Some((100, Some(300))));
|
||||
assert_eq!(parse_content_range("bytes 0-0/*"), Some((0, None)));
|
||||
assert_eq!(parse_content_range("bytes 9-1/300"), None);
|
||||
assert_eq!(parse_content_range("items 1-2/3"), None);
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
//! SAM 3 (Segment Anything with Concepts, Meta, 2025) inference in pure Rust on candle.
|
||||
//!
|
||||
//! - **Point prompts** ([`Sam3::segment_clicks`]): positive and negative clicks select one
|
||||
//! object, via SAM 3's interactive (SAM 2 style) prompt encoder and mask decoder.
|
||||
//! - **Text prompts** ([`Sam3::segment_text`]): a noun phrase ("the red car", "sky") selects
|
||||
//! every instance of that concept, via the CLIP text encoder and the DETR detector.
|
||||
//!
|
||||
//! The expensive part, the 32-layer ViT over the 1008 × 1008 input, runs once per image
|
||||
//! ([`Sam3::encode`]); each click or prompt then reuses the [`Encoded`] features.
|
||||
//!
|
||||
//! On macOS the model runs on the GPU through Metal; elsewhere (for now) on the CPU. The
|
||||
//! weights are not part of LightCraft: they are the `facebook/sam3` checkpoint (SAM License),
|
||||
//! read from a directory the user downloads them to ([`Sam3::load`]).
|
||||
//!
|
||||
//! Ported from the Hugging Face `transformers` implementation (Apache-2.0); see `NOTICE`.
|
||||
//! This crate is therefore licensed under the Apache License 2.0 only (not the MIT option of
|
||||
//! the rest of LightCraft). No UI dependencies (L3).
|
||||
//!
|
||||
//! [`fetch`] downloads the model, only when the user asks for it (from configurable mirrors,
|
||||
//! verified before use). Nothing in LightCraft requires the model: without it, AI masks report
|
||||
//! that it isn't installed and everything else works.
|
||||
//!
|
||||
//! Modified work (Apache License 2.0, §4(b)): ported by the LightCraft contributors in 2026 from
|
||||
//! the Python/PyTorch SAM 3 code of Hugging Face Transformers (`src/transformers/models/sam3` and `sam3_tracker`), Copyright The HuggingFace
|
||||
//! Team and Meta Platforms, Inc.; translated to Rust on candle and restructured. See NOTICE.
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
mod clip;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
mod detector;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod fetch;
|
||||
pub mod mask;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
mod neck;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
mod nn;
|
||||
pub mod tokenizer;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
mod tracker;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
mod vit;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
mod weights;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use std::sync::Arc;
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use candle_core::{DType, Device, Tensor};
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use candle_nn::VarBuilder;
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub use tracker::Click;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error("SAM 3 model: {0}")]
|
||||
Model(String),
|
||||
#[error("SAM 3 model files not found in {0} (expected model.safetensors, vocab.json, merges.txt)")]
|
||||
Missing(PathBuf),
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[error(transparent)]
|
||||
Candle(#[from] candle_core::Error),
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
/// Side of the square model input.
|
||||
pub const INPUT: usize = 1008;
|
||||
/// Side of the low-resolution masks the model predicts.
|
||||
pub const MASK_SIDE: usize = 288;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
const _: () = assert!(MASK_SIDE == tracker::LOW_RES && INPUT == vit::IMAGE);
|
||||
|
||||
/// The files of a model directory.
|
||||
pub const WEIGHTS_FILE: &str = "model.safetensors";
|
||||
|
||||
/// Whether `dir` holds a usable checkpoint.
|
||||
pub fn is_model_dir(dir: &Path) -> bool {
|
||||
dir.join(WEIGHTS_FILE).is_file() && dir.join("vocab.json").is_file() && dir.join("merges.txt").is_file()
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
/// The best device here: Metal on macOS (when a GPU is available), else the CPU.
|
||||
pub fn best_device() -> Device {
|
||||
#[cfg(target_os = "macos")]
|
||||
if let Ok(d) = Device::new_metal(0) {
|
||||
return d;
|
||||
}
|
||||
Device::Cpu
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
/// A loaded model. The vision backbone loads with it; the point and text heads load on first use.
|
||||
pub struct Sam3 {
|
||||
dir: PathBuf,
|
||||
device: Device,
|
||||
weights: Arc<weights::Weights>,
|
||||
backbone: vit::Vit,
|
||||
tracker_neck: Option<neck::Neck>,
|
||||
tracker: Option<tracker::Tracker>,
|
||||
detector: Option<detector::Detector>,
|
||||
tokenizer: Option<tokenizer::Tokenizer>,
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
/// An image run through the backbone, with the head-specific features computed on demand.
|
||||
pub struct Encoded {
|
||||
backbone: Tensor,
|
||||
tracker: Option<tracker::Features>,
|
||||
detector: Option<detector::Features>,
|
||||
}
|
||||
|
||||
/// A predicted mask: `MASK_SIDE²` logits over the model's square input (which the image was
|
||||
/// stretched to), and a confidence 0..1.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Prediction {
|
||||
pub logits: Vec<f32>,
|
||||
pub score: f32,
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
impl Sam3 {
|
||||
/// Load from a directory holding `model.safetensors` (the `facebook/sam3` checkpoint in
|
||||
/// Hugging Face format), `vocab.json` and `merges.txt`.
|
||||
pub fn load(dir: &Path) -> Result<Self> {
|
||||
Self::load_on(dir, best_device())
|
||||
}
|
||||
|
||||
pub fn load_on(dir: &Path, device: Device) -> Result<Self> {
|
||||
if !is_model_dir(dir) {
|
||||
return Err(Error::Missing(dir.to_path_buf()));
|
||||
}
|
||||
let weights = weights::Weights::open(&dir.join(WEIGHTS_FILE))?;
|
||||
let backbone = vit::Vit::new(Self::vb(&weights, &device).pp("detector_model.vision_encoder.backbone"))?;
|
||||
Ok(Self { dir: dir.to_path_buf(), device, weights, backbone, tracker_neck: None, tracker: None, detector: None, tokenizer: None })
|
||||
}
|
||||
|
||||
fn vb<'a>(w: &Arc<weights::Weights>, device: &Device) -> VarBuilder<'a> {
|
||||
VarBuilder::from_backend(Box::new(weights::Backend(w.clone())), DType::F32, device.clone())
|
||||
}
|
||||
|
||||
pub fn device(&self) -> &Device {
|
||||
&self.device
|
||||
}
|
||||
|
||||
/// Run the backbone on an RGB image (8-bit, row-major, `w × h`), stretched to the square
|
||||
/// input like the reference preprocessing.
|
||||
pub fn encode(&self, rgb: &[u8], w: usize, h: usize) -> Result<Encoded> {
|
||||
let pixels = mask::preprocess(rgb, w, h, INPUT)?;
|
||||
let pixels = Tensor::from_vec(pixels, (1, 3, INPUT, INPUT), &self.device)?;
|
||||
self.encode_pixels(&pixels)
|
||||
}
|
||||
|
||||
/// Run the backbone on preprocessed pixels `[1, 3, 1008, 1008]` (−1..1).
|
||||
pub fn encode_pixels(&self, pixels: &Tensor) -> Result<Encoded> {
|
||||
Ok(Encoded { backbone: self.backbone.forward(&pixels.to_device(&self.device)?)?, tracker: None, detector: None })
|
||||
}
|
||||
|
||||
fn tracker_parts(&mut self) -> Result<(&neck::Neck, &tracker::Tracker)> {
|
||||
if self.tracker_neck.is_none() {
|
||||
self.tracker_neck = Some(neck::Neck::new(Self::vb(&self.weights, &self.device).pp("tracker_neck"))?);
|
||||
}
|
||||
if self.tracker.is_none() {
|
||||
self.tracker = Some(tracker::Tracker::new(Self::vb(&self.weights, &self.device).pp("tracker_model"))?);
|
||||
}
|
||||
match (&self.tracker_neck, &self.tracker) {
|
||||
(Some(n), Some(t)) => Ok((n, t)),
|
||||
_ => Err(Error::Model("tracker failed to load".into())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the click decoder's features for `enc` now (so the first click is fast).
|
||||
pub fn prepare_clicks(&mut self, enc: &mut Encoded) -> Result<()> {
|
||||
let (neck, tracker) = self.tracker_parts()?;
|
||||
if enc.tracker.is_none() {
|
||||
let levels = neck.forward(&enc.backbone, 3)?;
|
||||
enc.tracker = Some(tracker.features(&levels)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Segment one object from clicks (positions normalized to the image, 0..1).
|
||||
pub fn segment_clicks(&mut self, enc: &mut Encoded, clicks: &[Click]) -> Result<Prediction> {
|
||||
if clicks.is_empty() {
|
||||
return Err(Error::Model("no clicks".into()));
|
||||
}
|
||||
let (neck, tracker) = self.tracker_parts()?;
|
||||
if enc.tracker.is_none() {
|
||||
let levels = neck.forward(&enc.backbone, 3)?;
|
||||
enc.tracker = Some(tracker.features(&levels)?);
|
||||
}
|
||||
let f = enc.tracker.as_ref().ok_or_else(|| Error::Model("tracker features".into()))?;
|
||||
// one click: SAM's three alternatives, best by predicted IoU; more: the single mask
|
||||
let p = tracker.predict(f, clicks, clicks.len() == 1)?;
|
||||
Ok(Prediction { logits: p.logits, score: p.iou * sigmoid(p.object_score) })
|
||||
}
|
||||
|
||||
fn detector_parts(&mut self) -> Result<(&detector::Detector, &tokenizer::Tokenizer)> {
|
||||
if self.detector.is_none() {
|
||||
self.detector = Some(detector::Detector::new(Self::vb(&self.weights, &self.device).pp("detector_model"))?);
|
||||
}
|
||||
if self.tokenizer.is_none() {
|
||||
self.tokenizer = Some(tokenizer::Tokenizer::load(&self.dir)?);
|
||||
}
|
||||
match (&self.detector, &self.tokenizer) {
|
||||
(Some(d), Some(t)) => Ok((d, t)),
|
||||
_ => Err(Error::Model("detector failed to load".into())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Segment every instance of the concept `text` names. Instances scoring above
|
||||
/// `threshold` (0..1, 0.5 is the reference default) are merged into one mask; `None` when
|
||||
/// nothing matches.
|
||||
pub fn segment_text(&mut self, enc: &mut Encoded, text: &str, threshold: f32) -> Result<Option<Prediction>> {
|
||||
let (det, tok) = self.detector_parts()?;
|
||||
if enc.detector.is_none() {
|
||||
enc.detector = Some(det.features(&enc.backbone)?);
|
||||
}
|
||||
let f = enc.detector.as_ref().ok_or_else(|| Error::Model("detector features".into()))?;
|
||||
let (ids, valid) = tok.encode(text);
|
||||
let out = det.predict(f, &ids, &valid)?;
|
||||
Ok(detector::merge(&out, threshold))
|
||||
}
|
||||
|
||||
/// Raw detector outputs for `text` (tests and tools).
|
||||
pub fn detect(&mut self, enc: &mut Encoded, text: &str) -> Result<detector::Output> {
|
||||
let (det, tok) = self.detector_parts()?;
|
||||
if enc.detector.is_none() {
|
||||
enc.detector = Some(det.features(&enc.backbone)?);
|
||||
}
|
||||
let f = enc.detector.as_ref().ok_or_else(|| Error::Model("detector features".into()))?;
|
||||
let (ids, valid) = tok.encode(text);
|
||||
det.predict(f, &ids, &valid)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
impl Encoded {
|
||||
/// The backbone output `[1, 1024, 72, 72]` (tests).
|
||||
pub fn backbone(&self) -> &Tensor {
|
||||
&self.backbone
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn sigmoid(x: f32) -> f32 {
|
||||
1.0 / (1.0 + (-x).exp())
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub use detector::Output as DetectorOutput;
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
/// Wait for queued GPU work to finish (benchmarks).
|
||||
pub fn sync(model: &Sam3) -> Result<()> {
|
||||
Ok(model.device.synchronize()?)
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
//! Image preprocessing and mask post-processing (CPU, no model needed).
|
||||
//!
|
||||
//! Modified work (Apache License 2.0, §4(b)): ported by the LightCraft contributors in 2026 from
|
||||
//! the Python/PyTorch SAM 3 code of Hugging Face Transformers (`models/sam3/image_processing_sam3*.py`), Copyright The HuggingFace
|
||||
//! Team and Meta Platforms, Inc.; translated to Rust on candle and restructured. See NOTICE.
|
||||
|
||||
use crate::{Error, Result};
|
||||
|
||||
/// Separable resampling weights for `src → dst` samples with an antialiased triangle filter
|
||||
/// (Pillow's BILINEAR: the support widens with the downscale factor).
|
||||
fn weights(src: usize, dst: usize) -> Vec<(usize, Vec<f32>)> {
|
||||
let scale = src as f64 / dst as f64;
|
||||
let ss = scale.max(1.0);
|
||||
let support = ss;
|
||||
(0..dst)
|
||||
.map(|o| {
|
||||
let center = (o as f64 + 0.5) * scale;
|
||||
let lo = ((center - support + 0.5).floor().max(0.0)) as usize;
|
||||
let hi = ((center + support + 0.5).floor() as usize).min(src);
|
||||
let mut w: Vec<f32> = (lo..hi).map(|i| (1.0 - ((i as f64 - center + 0.5) / ss).abs()).max(0.0) as f32).collect();
|
||||
let sum: f32 = w.iter().sum();
|
||||
if sum > 0.0 {
|
||||
w.iter_mut().for_each(|v| *v /= sum);
|
||||
}
|
||||
(lo, w)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// An 8-bit RGB image (`w × h`, row-major) stretched to `side × side`, scaled to −1..1 and laid
|
||||
/// out channel-first (`[3, side, side]`), as SAM 3's processor does.
|
||||
pub fn preprocess(rgb: &[u8], w: usize, h: usize, side: usize) -> Result<Vec<f32>> {
|
||||
let n = w.checked_mul(h).and_then(|n| n.checked_mul(3)).ok_or_else(|| Error::Model("image too large".into()))?;
|
||||
if w == 0 || h == 0 || rgb.len() < n {
|
||||
return Err(Error::Model(format!("image buffer of {} bytes for {w} × {h}", rgb.len())));
|
||||
}
|
||||
let (wx, wy) = (weights(w, side), weights(h, side));
|
||||
// horizontal pass: h rows × side columns × 3
|
||||
let mut tmp = vec![0f32; h * side * 3];
|
||||
for y in 0..h {
|
||||
for (ox, (lo, ws)) in wx.iter().enumerate() {
|
||||
let mut acc = [0f32; 3];
|
||||
for (k, wt) in ws.iter().enumerate() {
|
||||
let i = (y * w + lo + k) * 3;
|
||||
for (c, a) in acc.iter_mut().enumerate() {
|
||||
*a += wt * f32::from(rgb.get(i + c).copied().unwrap_or(0));
|
||||
}
|
||||
}
|
||||
if let Some(dst) = tmp.get_mut((y * side + ox) * 3..(y * side + ox) * 3 + 3) {
|
||||
dst.copy_from_slice(&acc);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut out = vec![0f32; 3 * side * side];
|
||||
for (oy, (lo, ws)) in wy.iter().enumerate() {
|
||||
for ox in 0..side {
|
||||
let mut acc = [0f32; 3];
|
||||
for (k, wt) in ws.iter().enumerate() {
|
||||
let i = ((lo + k) * side + ox) * 3;
|
||||
for (c, a) in acc.iter_mut().enumerate() {
|
||||
*a += wt * tmp.get(i + c).copied().unwrap_or(0.0);
|
||||
}
|
||||
}
|
||||
for (c, a) in acc.iter().enumerate() {
|
||||
// Pillow rounds to 8 bits; then /255 and (x − 0.5) / 0.5
|
||||
let v = a.round().clamp(0.0, 255.0) / 255.0;
|
||||
if let Some(d) = out.get_mut((c * side + oy) * side + ox) {
|
||||
*d = (v - 0.5) / 0.5;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Mask `logits` (`side × side`) resampled bilinearly (half-pixel centres) to `w × h` and
|
||||
/// mapped to 0..1 coverage (sigmoid).
|
||||
pub fn alpha(logits: &[f32], side: usize, w: usize, h: usize) -> Vec<f32> {
|
||||
let mut out = vec![0f32; w * h];
|
||||
if side == 0 || logits.len() < side * side {
|
||||
return out;
|
||||
}
|
||||
let sample = |x: usize, y: usize| logits.get(y.min(side - 1) * side + x.min(side - 1)).copied().unwrap_or(-20.0);
|
||||
let (sx, sy) = (side as f32 / w.max(1) as f32, side as f32 / h.max(1) as f32);
|
||||
for y in 0..h {
|
||||
let fy = ((y as f32 + 0.5) * sy - 0.5).max(0.0);
|
||||
let (y0, ty) = (fy.floor() as usize, fy.fract());
|
||||
for x in 0..w {
|
||||
let fx = ((x as f32 + 0.5) * sx - 0.5).max(0.0);
|
||||
let (x0, tx) = (fx.floor() as usize, fx.fract());
|
||||
let top = sample(x0, y0) * (1.0 - tx) + sample(x0 + 1, y0) * tx;
|
||||
let bot = sample(x0, y0 + 1) * (1.0 - tx) + sample(x0 + 1, y0 + 1) * tx;
|
||||
let l = top * (1.0 - ty) + bot * ty;
|
||||
if let Some(o) = out.get_mut(y * w + x) {
|
||||
*o = 1.0 / (1.0 + (-l).exp());
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn preprocess_is_bounded_and_channel_first() {
|
||||
let rgb: Vec<u8> = (0..20 * 10 * 3).map(|i| (i % 256) as u8).collect();
|
||||
let p = preprocess(&rgb, 20, 10, 8).unwrap();
|
||||
assert_eq!(p.len(), 3 * 64);
|
||||
assert!(p.iter().all(|v| (-1.0..=1.0).contains(v)));
|
||||
// a flat image stays flat
|
||||
let flat = vec![255u8; 30 * 30 * 3];
|
||||
assert!(preprocess(&flat, 30, 30, 7).unwrap().iter().all(|v| (*v - 1.0).abs() < 1e-6));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_buffers_are_errors() {
|
||||
assert!(preprocess(&[0; 5], 2, 2, 4).is_err());
|
||||
assert!(preprocess(&[], 0, 0, 4).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alpha_follows_the_logits() {
|
||||
let logits: Vec<f32> = (0..16).map(|i| if i % 4 < 2 { 10.0 } else { -10.0 }).collect();
|
||||
let a = alpha(&logits, 4, 8, 2);
|
||||
assert!(a[0] > 0.99 && a[7] < 0.01);
|
||||
assert!(alpha(&[], 4, 3, 3).iter().all(|v| *v == 0.0));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
//! The feature pyramid (`Sam3VisionNeck`): the backbone's 72 × 72 map at four scales
|
||||
//! (288, 144, 72 and 36 per side), each projected to 256 channels.
|
||||
//!
|
||||
//! Modified work (Apache License 2.0, §4(b)): ported by the LightCraft contributors in 2026 from
|
||||
//! the Python/PyTorch SAM 3 code of Hugging Face Transformers (`models/sam3/modeling_sam3.py`), Copyright The HuggingFace
|
||||
//! Team and Meta Platforms, Inc.; translated to Rust on candle and restructured. See NOTICE.
|
||||
|
||||
use candle_core::{Module, Tensor};
|
||||
use candle_nn::{Conv2d, Conv2dConfig, ConvTranspose2d, ConvTranspose2dConfig, VarBuilder};
|
||||
|
||||
use crate::Result;
|
||||
use crate::vit::HIDDEN;
|
||||
|
||||
pub const FPN: usize = 256;
|
||||
|
||||
enum Scale {
|
||||
/// Two 2× transposed convolutions with GELU between (4×).
|
||||
Up4(ConvTranspose2d, ConvTranspose2d),
|
||||
Up2(ConvTranspose2d),
|
||||
Same,
|
||||
/// 2 × 2 max pooling (½×).
|
||||
Down2,
|
||||
}
|
||||
|
||||
struct Level {
|
||||
scale: Scale,
|
||||
proj1: Conv2d,
|
||||
proj2: Conv2d,
|
||||
}
|
||||
|
||||
pub struct Neck {
|
||||
levels: Vec<Level>,
|
||||
}
|
||||
|
||||
impl Neck {
|
||||
/// `vb` at `…neck` (detector) or `tracker_neck`.
|
||||
pub fn new(vb: VarBuilder) -> Result<Self> {
|
||||
let up = ConvTranspose2dConfig { stride: 2, ..Default::default() };
|
||||
let mut levels = Vec::new();
|
||||
for (i, factor) in [4.0f32, 2.0, 1.0, 0.5].into_iter().enumerate() {
|
||||
let l = vb.pp(format!("fpn_layers.{i}"));
|
||||
let s = l.pp("scale_layers");
|
||||
let (scale, inter) = if factor == 4.0 {
|
||||
let a = candle_nn::conv_transpose2d(HIDDEN, HIDDEN / 2, 2, up, s.pp("0"))?;
|
||||
let b = candle_nn::conv_transpose2d(HIDDEN / 2, HIDDEN / 4, 2, up, s.pp("2"))?;
|
||||
(Scale::Up4(a, b), HIDDEN / 4)
|
||||
} else if factor == 2.0 {
|
||||
(Scale::Up2(candle_nn::conv_transpose2d(HIDDEN, HIDDEN / 2, 2, up, s.pp("0"))?), HIDDEN / 2)
|
||||
} else if factor == 1.0 {
|
||||
(Scale::Same, HIDDEN)
|
||||
} else {
|
||||
(Scale::Down2, HIDDEN)
|
||||
};
|
||||
levels.push(Level {
|
||||
scale,
|
||||
proj1: candle_nn::conv2d(inter, FPN, 1, Conv2dConfig::default(), l.pp("proj1"))?,
|
||||
proj2: candle_nn::conv2d(FPN, FPN, 3, Conv2dConfig { padding: 1, ..Default::default() }, l.pp("proj2"))?,
|
||||
});
|
||||
}
|
||||
Ok(Self { levels })
|
||||
}
|
||||
|
||||
/// `x`: the backbone's `[1, 1024, 72, 72]`; `n`: how many levels (from the finest) to compute.
|
||||
pub fn forward(&self, x: &Tensor, n: usize) -> Result<Vec<Tensor>> {
|
||||
let mut out = Vec::with_capacity(n);
|
||||
for l in self.levels.iter().take(n) {
|
||||
let y = match &l.scale {
|
||||
Scale::Up4(a, b) => b.forward(&a.forward(x)?.gelu_erf()?)?,
|
||||
Scale::Up2(a) => a.forward(x)?,
|
||||
Scale::Same => x.clone(),
|
||||
Scale::Down2 => x.max_pool2d(2)?,
|
||||
};
|
||||
out.push(l.proj2.forward(&l.proj1.forward(&y)?)?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
//! Shared building blocks: attention, MLPs, layer norms, sine position encodings.
|
||||
//!
|
||||
//! Modified work (Apache License 2.0, §4(b)): ported by the LightCraft contributors in 2026 from
|
||||
//! the Python/PyTorch SAM 3 code of Hugging Face Transformers (`models/sam3/modeling_sam3.py`, `models/sam3_tracker`), Copyright The HuggingFace
|
||||
//! Team and Meta Platforms, Inc.; translated to Rust on candle and restructured. See NOTICE.
|
||||
|
||||
use candle_core::{D, DType, Device, Module, Tensor};
|
||||
use candle_nn::{LayerNorm, Linear, VarBuilder};
|
||||
|
||||
use crate::Result;
|
||||
|
||||
/// `nn.LayerNorm(d)` (PyTorch's default eps).
|
||||
pub const TORCH_LN_EPS: f64 = 1e-5;
|
||||
|
||||
pub fn ln(d: usize, eps: f64, vb: VarBuilder) -> Result<LayerNorm> {
|
||||
Ok(candle_nn::layer_norm(d, eps, vb)?)
|
||||
}
|
||||
|
||||
pub fn linear(i: usize, o: usize, vb: VarBuilder) -> Result<Linear> {
|
||||
Ok(candle_nn::linear(i, o, vb)?)
|
||||
}
|
||||
|
||||
/// Largest attention-score block computed at once (elements): the 5184-token global layers
|
||||
/// would otherwise need ~1.7 GB for all 16 heads.
|
||||
const SCORE_BLOCK: usize = 64 << 20;
|
||||
|
||||
/// Scaled dot-product attention on `[B, H, L, d]` tensors, with an optional additive bias
|
||||
/// broadcastable to `[B, H, Lq, Lk]`. Heads are processed in groups to bound memory.
|
||||
pub fn sdpa(q: &Tensor, k: &Tensor, v: &Tensor, bias: Option<&Tensor>, scale: f64) -> Result<Tensor> {
|
||||
let (b, h, lq, _) = q.dims4()?;
|
||||
let lk = k.dim(2)?;
|
||||
let per_head = b.saturating_mul(lq).saturating_mul(lk).max(1);
|
||||
let group = (SCORE_BLOCK / per_head).clamp(1, h);
|
||||
let bias_heads = match bias {
|
||||
Some(t) => t.dim(1)?,
|
||||
None => 1,
|
||||
};
|
||||
let mut outs = Vec::with_capacity(h.div_ceil(group));
|
||||
let mut start = 0;
|
||||
while start < h {
|
||||
let n = group.min(h - start);
|
||||
let qs = q.narrow(1, start, n)?;
|
||||
let ks = k.narrow(1, start, n)?;
|
||||
let vs = v.narrow(1, start, n)?;
|
||||
let mut att = (qs.matmul(&ks.t()?)? * scale)?;
|
||||
if let Some(bias) = bias {
|
||||
let bs = if bias_heads > 1 { bias.narrow(1, start, n)? } else { bias.clone() };
|
||||
att = att.broadcast_add(&bs)?;
|
||||
}
|
||||
let att = candle_nn::ops::softmax_last_dim(&att)?;
|
||||
outs.push(att.matmul(&vs)?);
|
||||
start += n;
|
||||
}
|
||||
Ok(Tensor::cat(&outs, 1)?)
|
||||
}
|
||||
|
||||
/// Multi-head attention with q/k/v/o projections (`Sam3Attention`, `Sam3TrackerAttention`
|
||||
/// with `internal` = hidden / downsample rate).
|
||||
pub struct Attention {
|
||||
q: Linear,
|
||||
k: Linear,
|
||||
v: Linear,
|
||||
o: Linear,
|
||||
heads: usize,
|
||||
head_dim: usize,
|
||||
}
|
||||
|
||||
impl Attention {
|
||||
pub fn new(hidden: usize, internal: usize, heads: usize, vb: VarBuilder) -> Result<Self> {
|
||||
Self::named(hidden, internal, heads, "o_proj", vb)
|
||||
}
|
||||
|
||||
/// With the output projection called `out` (CLIP's `out_proj`).
|
||||
pub fn named(hidden: usize, internal: usize, heads: usize, out: &str, vb: VarBuilder) -> Result<Self> {
|
||||
Ok(Self {
|
||||
q: linear(hidden, internal, vb.pp("q_proj"))?,
|
||||
k: linear(hidden, internal, vb.pp("k_proj"))?,
|
||||
v: linear(hidden, internal, vb.pp("v_proj"))?,
|
||||
o: linear(internal, hidden, vb.pp(out))?,
|
||||
heads,
|
||||
head_dim: internal / heads,
|
||||
})
|
||||
}
|
||||
|
||||
fn split(&self, x: &Tensor) -> Result<Tensor> {
|
||||
let (b, l, _) = x.dims3()?;
|
||||
Ok(x.reshape((b, l, self.heads, self.head_dim))?.transpose(1, 2)?.contiguous()?)
|
||||
}
|
||||
|
||||
/// The q, k and v projections of `x` (for attention variants that transform them).
|
||||
pub fn qkv(&self, x: &Tensor) -> Result<(Tensor, Tensor, Tensor)> {
|
||||
Ok((self.q.forward(x)?, self.k.forward(x)?, self.v.forward(x)?))
|
||||
}
|
||||
|
||||
/// The output projection.
|
||||
pub fn out(&self, x: &Tensor) -> Result<Tensor> {
|
||||
Ok(self.o.forward(x)?)
|
||||
}
|
||||
|
||||
/// `q`, `k`, `v`: `[B, L, hidden]`; `bias` broadcastable to `[B, heads, Lq, Lk]`.
|
||||
pub fn forward(&self, q: &Tensor, k: &Tensor, v: &Tensor, bias: Option<&Tensor>) -> Result<Tensor> {
|
||||
let (b, lq, _) = q.dims3()?;
|
||||
let q = self.split(&self.q.forward(q)?)?;
|
||||
let k = self.split(&self.k.forward(k)?)?;
|
||||
let v = self.split(&self.v.forward(v)?)?;
|
||||
let o = sdpa(&q, &k, &v, bias, (self.head_dim as f64).powf(-0.5))?;
|
||||
let o = o.transpose(1, 2)?.reshape((b, lq, self.heads * self.head_dim))?;
|
||||
Ok(self.o.forward(&o)?)
|
||||
}
|
||||
}
|
||||
|
||||
/// An additive key-padding bias `[1, 1, 1, L]` (0 for valid tokens, the f32 minimum for padding).
|
||||
pub fn padding_bias(valid: &[bool], device: &Device) -> Result<Tensor> {
|
||||
let v: Vec<f32> = valid.iter().map(|&ok| if ok { 0.0 } else { f32::MIN }).collect();
|
||||
Ok(Tensor::from_vec(v, (1, 1, 1, valid.len()), device)?)
|
||||
}
|
||||
|
||||
/// `Sam3MLP`: fc1 → activation → fc2.
|
||||
pub struct Mlp {
|
||||
fc1: Linear,
|
||||
fc2: Linear,
|
||||
gelu: bool,
|
||||
}
|
||||
|
||||
impl Mlp {
|
||||
pub fn new(hidden: usize, inter: usize, gelu: bool, vb: VarBuilder) -> Result<Self> {
|
||||
Ok(Self { fc1: linear(hidden, inter, vb.pp("fc1"))?, fc2: linear(inter, hidden, vb.pp("fc2"))?, gelu })
|
||||
}
|
||||
|
||||
pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
|
||||
let x = self.fc1.forward(x)?;
|
||||
let x = if self.gelu { x.gelu_erf()? } else { x.relu()? };
|
||||
Ok(self.fc2.forward(&x)?)
|
||||
}
|
||||
}
|
||||
|
||||
/// `Sam3DecoderMLP`: `layer1 … layerN` with ReLU between (2 or 3 layers).
|
||||
pub struct DecoderMlp {
|
||||
layers: Vec<Linear>,
|
||||
}
|
||||
|
||||
impl DecoderMlp {
|
||||
pub fn new(input: usize, hidden: usize, output: usize, n: usize, vb: VarBuilder) -> Result<Self> {
|
||||
let mut layers = Vec::with_capacity(n);
|
||||
for i in 0..n {
|
||||
let (a, b) = (if i == 0 { input } else { hidden }, if i + 1 == n { output } else { hidden });
|
||||
layers.push(linear(a, b, vb.pp(format!("layer{}", i + 1)))?);
|
||||
}
|
||||
Ok(Self { layers })
|
||||
}
|
||||
|
||||
pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
|
||||
let mut x = x.clone();
|
||||
for (i, l) in self.layers.iter().enumerate() {
|
||||
x = l.forward(&x)?;
|
||||
if i + 1 < self.layers.len() {
|
||||
x = x.relu()?;
|
||||
}
|
||||
}
|
||||
Ok(x)
|
||||
}
|
||||
}
|
||||
|
||||
/// `Sam3TrackerFeedForward`: proj_in → act → (layers.i → act)* → proj_out (→ sigmoid).
|
||||
pub struct FeedForward {
|
||||
proj_in: Linear,
|
||||
layers: Vec<Linear>,
|
||||
proj_out: Linear,
|
||||
sigmoid: bool,
|
||||
}
|
||||
|
||||
impl FeedForward {
|
||||
pub fn new(input: usize, hidden: usize, output: usize, n: usize, sigmoid: bool, vb: VarBuilder) -> Result<Self> {
|
||||
let mut layers = Vec::new();
|
||||
for i in 0..n.saturating_sub(2) {
|
||||
layers.push(linear(hidden, hidden, vb.pp(format!("layers.{i}")))?);
|
||||
}
|
||||
Ok(Self { proj_in: linear(input, hidden, vb.pp("proj_in"))?, layers, proj_out: linear(hidden, output, vb.pp("proj_out"))?, sigmoid })
|
||||
}
|
||||
|
||||
pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
|
||||
let mut x = self.proj_in.forward(x)?.relu()?;
|
||||
for l in &self.layers {
|
||||
x = l.forward(&x)?.relu()?;
|
||||
}
|
||||
let x = self.proj_out.forward(&x)?;
|
||||
Ok(if self.sigmoid { candle_nn::ops::sigmoid(&x)? } else { x })
|
||||
}
|
||||
}
|
||||
|
||||
/// Layer norm over the channel axis of an NCHW tensor (`Sam3TrackerLayerNorm`, channels_first).
|
||||
pub fn ln_channels(x: &Tensor, norm: &LayerNorm) -> Result<Tensor> {
|
||||
let x = x.permute((0, 2, 3, 1))?;
|
||||
Ok(norm.forward(&x)?.permute((0, 3, 1, 2))?.contiguous()?)
|
||||
}
|
||||
|
||||
/// The DETR sine embedding of an `h × w` grid, normalized (`Sam3SinePositionEmbedding` with
|
||||
/// `normalize=True`, scale 2π): `[1, 2·feats, h, w]`, y features first.
|
||||
pub fn sine_grid(h: usize, w: usize, feats: usize, device: &Device) -> Result<Tensor> {
|
||||
let scale = 2.0 * std::f64::consts::PI;
|
||||
let dim_t: Vec<f64> = (0..feats).map(|i| 10000f64.powf(2.0 * (i / 2) as f64 / feats as f64)).collect();
|
||||
let mut out = vec![0f32; 2 * feats * h * w];
|
||||
for y in 0..h {
|
||||
let ye = (y + 1) as f64 / (h as f64 + 1e-6) * scale;
|
||||
for x in 0..w {
|
||||
let xe = (x + 1) as f64 / (w as f64 + 1e-6) * scale;
|
||||
for (i, d) in dim_t.iter().enumerate() {
|
||||
let (py, px) = (ye / d, xe / d);
|
||||
let (vy, vx) = if i % 2 == 0 { (py.sin(), px.sin()) } else { (py.cos(), px.cos()) };
|
||||
if let Some(v) = out.get_mut((i * h + y) * w + x) {
|
||||
*v = vy as f32;
|
||||
}
|
||||
if let Some(v) = out.get_mut(((feats + i) * h + y) * w + x) {
|
||||
*v = vx as f32;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Tensor::from_vec(out, (1, 2 * feats, h, w), device)?)
|
||||
}
|
||||
|
||||
/// `inverse_sigmoid` with the reference's clamping (eps 1e-3).
|
||||
pub fn inverse_sigmoid(x: &Tensor) -> Result<Tensor> {
|
||||
let x = x.clamp(0f32, 1f32)?;
|
||||
let x1 = x.clamp(1e-3f32, f32::MAX)?;
|
||||
let x2 = (1.0 - &x)?.clamp(1e-3f32, f32::MAX)?;
|
||||
Ok((x1 / x2)?.log()?)
|
||||
}
|
||||
|
||||
/// `[..., 4]` boxes (cx, cy, w, h) → (x0, y0, x1, y1).
|
||||
pub fn cxcywh_to_xyxy(b: &Tensor) -> Result<Tensor> {
|
||||
let cx = b.narrow(D::Minus1, 0, 1)?;
|
||||
let cy = b.narrow(D::Minus1, 1, 1)?;
|
||||
let w = (b.narrow(D::Minus1, 2, 1)? * 0.5)?;
|
||||
let h = (b.narrow(D::Minus1, 3, 1)? * 0.5)?;
|
||||
Ok(Tensor::cat(&[(&cx - &w)?, (&cy - &h)?, (&cx + &w)?, (&cy + &h)?], D::Minus1)?)
|
||||
}
|
||||
|
||||
/// A tensor's values as `f32` on the CPU.
|
||||
pub fn to_vec(t: &Tensor) -> Result<Vec<f32>> {
|
||||
Ok(t.to_dtype(DType::F32)?.flatten_all()?.to_vec1::<f32>()?)
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
//! The CLIP byte-pair-encoding tokenizer (lower-cased, `</w>` word ends), padded to the
|
||||
//! 32-token context SAM 3's text encoder takes.
|
||||
//!
|
||||
//! Modified work (Apache License 2.0, §4(b)): ported by the LightCraft contributors in 2026 from
|
||||
//! the Python/PyTorch SAM 3 code of Hugging Face Transformers (the CLIP tokenizer used by `models/sam3`), Copyright The HuggingFace
|
||||
//! Team and Meta Platforms, Inc.; translated to Rust on candle and restructured. See NOTICE.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::{Error, Result};
|
||||
|
||||
pub const CONTEXT: usize = 32;
|
||||
const BOS: u32 = 49406;
|
||||
const EOS: u32 = 49407;
|
||||
|
||||
pub struct Tokenizer {
|
||||
vocab: HashMap<String, u32>,
|
||||
ranks: HashMap<(String, String), usize>,
|
||||
byte_char: [char; 256],
|
||||
}
|
||||
|
||||
/// GPT-2's reversible byte → printable character map.
|
||||
fn bytes_to_unicode() -> [char; 256] {
|
||||
let mut map = ['\0'; 256];
|
||||
let mut n = 0u32;
|
||||
for b in 0..=255u8 {
|
||||
let printable = (b'!'..=b'~').contains(&b) || (0xA1..=0xAC).contains(&b) || (0xAE..=0xFF).contains(&b);
|
||||
let c = if printable {
|
||||
u32::from(b)
|
||||
} else {
|
||||
n += 1;
|
||||
255 + n
|
||||
};
|
||||
if let Some(slot) = map.get_mut(usize::from(b)) {
|
||||
*slot = char::from_u32(c).unwrap_or('?');
|
||||
}
|
||||
}
|
||||
map
|
||||
}
|
||||
|
||||
impl Tokenizer {
|
||||
/// From `vocab.json` and `merges.txt` in `dir`.
|
||||
pub fn load(dir: &Path) -> Result<Self> {
|
||||
let read = |name: &str| std::fs::read_to_string(dir.join(name)).map_err(|e| Error::Model(format!("{name}: {e}")));
|
||||
let vocab: HashMap<String, u32> = serde_json::from_str(&read("vocab.json")?).map_err(|e| Error::Model(format!("vocab.json: {e}")))?;
|
||||
let merges = read("merges.txt")?;
|
||||
Ok(Self::new(vocab, &merges))
|
||||
}
|
||||
|
||||
pub fn new(vocab: HashMap<String, u32>, merges: &str) -> Self {
|
||||
let ranks = merges
|
||||
.lines()
|
||||
.filter(|l| !l.starts_with("#version") && !l.trim().is_empty())
|
||||
.filter_map(|l| l.split_once(' ').map(|(a, b)| (a.to_string(), b.to_string())))
|
||||
.enumerate()
|
||||
.map(|(i, p)| (p, i))
|
||||
.collect();
|
||||
Self { vocab, ranks, byte_char: bytes_to_unicode() }
|
||||
}
|
||||
|
||||
/// Token ids (start, words, end, padding) and which positions are real tokens.
|
||||
pub fn encode(&self, text: &str) -> (Vec<u32>, Vec<bool>) {
|
||||
let mut ids = vec![BOS];
|
||||
for word in pre_tokenize(&text.split_whitespace().collect::<Vec<_>>().join(" ").to_lowercase()) {
|
||||
let mapped: String = word.bytes().map(|b| self.byte_char.get(usize::from(b)).copied().unwrap_or('?')).collect();
|
||||
for piece in self.bpe(&mapped) {
|
||||
ids.push(self.vocab.get(&piece).copied().unwrap_or(EOS));
|
||||
}
|
||||
}
|
||||
ids.truncate(CONTEXT - 1);
|
||||
ids.push(EOS);
|
||||
let n = ids.len();
|
||||
ids.resize(CONTEXT, EOS);
|
||||
let valid = (0..CONTEXT).map(|i| i < n).collect();
|
||||
(ids, valid)
|
||||
}
|
||||
|
||||
fn bpe(&self, word: &str) -> Vec<String> {
|
||||
let mut syms: Vec<String> = word.chars().map(String::from).collect();
|
||||
if let Some(last) = syms.last_mut() {
|
||||
last.push_str("</w>");
|
||||
}
|
||||
loop {
|
||||
let best = syms
|
||||
.windows(2)
|
||||
.enumerate()
|
||||
.filter_map(|(i, w)| match w {
|
||||
[a, b] => self.ranks.get(&(a.clone(), b.clone())).map(|r| (*r, i)),
|
||||
_ => None,
|
||||
})
|
||||
.min();
|
||||
let Some((_, i)) = best else { break };
|
||||
let (Some(a), Some(b)) = (syms.get(i).cloned(), syms.get(i + 1).cloned()) else { break };
|
||||
let merged = a.clone() + &b;
|
||||
// merge every occurrence of the pair, left to right
|
||||
let mut out = Vec::with_capacity(syms.len());
|
||||
let mut j = 0;
|
||||
while j < syms.len() {
|
||||
if j + 1 < syms.len() && syms.get(j) == Some(&a) && syms.get(j + 1) == Some(&b) {
|
||||
out.push(merged.clone());
|
||||
j += 2;
|
||||
} else {
|
||||
out.extend(syms.get(j).cloned());
|
||||
j += 1;
|
||||
}
|
||||
}
|
||||
syms = out;
|
||||
if syms.len() == 1 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
syms
|
||||
}
|
||||
}
|
||||
|
||||
/// CLIP's pre-tokenizer: contractions, letter runs, single digits, and runs of other
|
||||
/// non-space characters.
|
||||
fn pre_tokenize(text: &str) -> Vec<String> {
|
||||
let chars: Vec<char> = text.chars().collect();
|
||||
let mut out = Vec::new();
|
||||
let mut i = 0;
|
||||
while let Some(&c) = chars.get(i) {
|
||||
if c.is_whitespace() {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if c == '\'' {
|
||||
let rest: String = chars.get(i + 1..(i + 3).min(chars.len())).map(|s| s.iter().collect()).unwrap_or_default();
|
||||
if let Some(k) = ["re", "ve", "ll"]
|
||||
.iter()
|
||||
.find(|k| rest.starts_with(**k))
|
||||
.map(|k| k.len())
|
||||
.or_else(|| rest.chars().next().filter(|c| matches!(c, 's' | 't' | 'm' | 'd')).map(|_| 1))
|
||||
{
|
||||
out.push(chars.get(i..i + 1 + k).map(|s| s.iter().collect()).unwrap_or_default());
|
||||
i += 1 + k;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let start = i;
|
||||
if c.is_alphabetic() {
|
||||
while chars.get(i).is_some_and(|c| c.is_alphabetic()) {
|
||||
i += 1;
|
||||
}
|
||||
} else if c.is_numeric() {
|
||||
i += 1;
|
||||
} else {
|
||||
while chars.get(i).is_some_and(|c| !c.is_whitespace() && !c.is_alphabetic() && !c.is_numeric()) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
out.push(chars.get(start..i).map(|s| s.iter().collect()).unwrap_or_default());
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn splits_like_clip() {
|
||||
assert_eq!(pre_tokenize("a dog's toy, 25 cats!"), ["a", "dog", "'s", "toy", ",", "2", "5", "cats", "!"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merges_by_rank() {
|
||||
let vocab: HashMap<String, u32> =
|
||||
[("c", 1), ("a", 2), ("t</w>", 3), ("ca", 4), ("cat</w>", 5)].into_iter().map(|(k, v)| (k.to_string(), v)).collect();
|
||||
let t = Tokenizer::new(vocab, "#version: 0.2\nc a\nca t</w>\n");
|
||||
let (ids, valid) = t.encode(" CAT ");
|
||||
assert_eq!(&ids[..3], &[BOS, 5, EOS]);
|
||||
assert_eq!(ids.len(), CONTEXT);
|
||||
assert_eq!(valid.iter().filter(|v| **v).count(), 3);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
//! Point prompts (SAM 3's interactive "tracker" head, SAM 2 style): a prompt encoder for
|
||||
//! positive/negative clicks and a two-way-transformer mask decoder over the image features.
|
||||
//!
|
||||
//! Modified work (Apache License 2.0, §4(b)): ported by the LightCraft contributors in 2026 from
|
||||
//! the Python/PyTorch SAM 3 code of Hugging Face Transformers (`models/sam3_tracker/modeling_sam3_tracker.py`), Copyright The HuggingFace
|
||||
//! Team and Meta Platforms, Inc.; translated to Rust on candle and restructured. See NOTICE.
|
||||
|
||||
use candle_core::{DType, Device, IndexOp, Module, Tensor};
|
||||
use candle_nn::{Conv2d, Conv2dConfig, ConvTranspose2d, ConvTranspose2dConfig, LayerNorm, VarBuilder};
|
||||
|
||||
use crate::nn::{Attention, FeedForward, TORCH_LN_EPS, ln, ln_channels};
|
||||
use crate::vit::{GRID, IMAGE};
|
||||
use crate::{Result, nn::to_vec};
|
||||
|
||||
const HIDDEN: usize = 256;
|
||||
const MASK_TOKENS: usize = 4;
|
||||
/// Low-resolution mask side (4 × the 72 grid).
|
||||
pub const LOW_RES: usize = 288;
|
||||
|
||||
/// The image features the decoder needs, computed once per image.
|
||||
pub struct Features {
|
||||
/// Level 2 (72 × 72, 256 channels).
|
||||
pub image: Tensor,
|
||||
/// `conv_s0`(level 0): `[1, 32, 288, 288]`.
|
||||
pub s0: Tensor,
|
||||
/// `conv_s1`(level 1): `[1, 64, 144, 144]`.
|
||||
pub s1: Tensor,
|
||||
}
|
||||
|
||||
/// One click: position normalized to the image (0..1), and whether it includes or excludes.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Click {
|
||||
pub x: f32,
|
||||
pub y: f32,
|
||||
pub positive: bool,
|
||||
}
|
||||
|
||||
/// A predicted mask: 288 × 288 logits over the (squared) model input, and its quality.
|
||||
pub struct Prediction {
|
||||
pub logits: Vec<f32>,
|
||||
pub iou: f32,
|
||||
pub object_score: f32,
|
||||
}
|
||||
|
||||
/// Random-Fourier positional encoding of points normalized to 0..1 (`Sam3TrackerPositionalEmbedding`).
|
||||
struct Fourier {
|
||||
pe: Vec<f32>, // [2, 128]
|
||||
}
|
||||
|
||||
impl Fourier {
|
||||
fn encode(&self, x: f32, y: f32) -> Vec<f32> {
|
||||
let half = self.pe.len() / 2;
|
||||
let (cx, cy) = (2.0 * x - 1.0, 2.0 * y - 1.0);
|
||||
let proj: Vec<f32> = (0..half)
|
||||
.map(|i| {
|
||||
let a = cx * self.pe.get(i).copied().unwrap_or(0.0) + cy * self.pe.get(half + i).copied().unwrap_or(0.0);
|
||||
2.0 * std::f32::consts::PI * a
|
||||
})
|
||||
.collect();
|
||||
proj.iter().map(|a| a.sin()).chain(proj.iter().map(|a| a.cos())).collect()
|
||||
}
|
||||
}
|
||||
|
||||
struct TwoWayBlock {
|
||||
self_attn: Attention,
|
||||
ln1: LayerNorm,
|
||||
t2i: Attention,
|
||||
ln2: LayerNorm,
|
||||
mlp: FeedForward,
|
||||
ln3: LayerNorm,
|
||||
ln4: LayerNorm,
|
||||
i2t: Attention,
|
||||
skip_first_pe: bool,
|
||||
}
|
||||
|
||||
/// `Sam3TrackerAttention` on `[1, L, C]` (the point batch of one is folded away).
|
||||
fn attn(a: &Attention, q: &Tensor, k: &Tensor, v: &Tensor) -> Result<Tensor> {
|
||||
a.forward(q, k, v, None)
|
||||
}
|
||||
|
||||
impl TwoWayBlock {
|
||||
fn forward(&self, queries: &Tensor, keys: &Tensor, qpe: &Tensor, kpe: &Tensor) -> Result<(Tensor, Tensor)> {
|
||||
let queries = if self.skip_first_pe {
|
||||
attn(&self.self_attn, queries, queries, queries)?
|
||||
} else {
|
||||
let q = (queries + qpe)?;
|
||||
(queries + attn(&self.self_attn, &q, &q, queries)?)?
|
||||
};
|
||||
let queries = self.ln1.forward(&queries)?;
|
||||
let (q, k) = ((&queries + qpe)?, (keys + kpe)?);
|
||||
let queries = self.ln2.forward(&(&queries + attn(&self.t2i, &q, &k, keys)?)?)?;
|
||||
let queries = self.ln3.forward(&(&queries + self.mlp.forward(&queries)?)?)?;
|
||||
let (q, k) = ((&queries + qpe)?, (keys + kpe)?);
|
||||
let keys = self.ln4.forward(&(keys + attn(&self.i2t, &k, &q, &queries)?)?)?;
|
||||
Ok((queries, keys))
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Tracker {
|
||||
device: Device,
|
||||
point_pe: Fourier,
|
||||
image_pe: Tensor,
|
||||
point_embed: Tensor,
|
||||
not_a_point: Tensor,
|
||||
no_mask: Tensor,
|
||||
conv_s0: Conv2d,
|
||||
conv_s1: Conv2d,
|
||||
output_tokens: Tensor,
|
||||
blocks: Vec<TwoWayBlock>,
|
||||
final_t2i: Attention,
|
||||
ln_final: LayerNorm,
|
||||
up1: ConvTranspose2d,
|
||||
up_ln: LayerNorm,
|
||||
up2: ConvTranspose2d,
|
||||
hyper: Vec<FeedForward>,
|
||||
iou_head: FeedForward,
|
||||
obj_head: FeedForward,
|
||||
}
|
||||
|
||||
fn read_vec(vb: &VarBuilder, shape: (usize, usize), name: &str) -> Result<Vec<f32>> {
|
||||
to_vec(&vb.get(shape, name)?)
|
||||
}
|
||||
|
||||
impl Tracker {
|
||||
/// `vb` at `tracker_model`.
|
||||
pub fn new(vb: VarBuilder) -> Result<Self> {
|
||||
let device = vb.device().clone();
|
||||
let pe = vb.pp("prompt_encoder");
|
||||
let md = vb.pp("mask_decoder");
|
||||
let image_fourier = Fourier { pe: read_vec(&vb, (2, HIDDEN / 2), "shared_image_embedding.positional_embedding")? };
|
||||
// image-wide positional embedding: pixel centres of the 72 × 72 grid
|
||||
let mut grid = Vec::with_capacity(GRID * GRID * HIDDEN);
|
||||
for r in 0..GRID {
|
||||
for c in 0..GRID {
|
||||
grid.extend(image_fourier.encode((c as f32 + 0.5) / GRID as f32, (r as f32 + 0.5) / GRID as f32));
|
||||
}
|
||||
}
|
||||
let image_pe = Tensor::from_vec(grid, (1, GRID * GRID, HIDDEN), &device)?;
|
||||
let t = md.pp("transformer");
|
||||
let mut blocks = Vec::new();
|
||||
for i in 0..2 {
|
||||
let b = t.pp(format!("layers.{i}"));
|
||||
blocks.push(TwoWayBlock {
|
||||
self_attn: Attention::new(HIDDEN, HIDDEN, 8, b.pp("self_attn"))?,
|
||||
ln1: ln(HIDDEN, TORCH_LN_EPS, b.pp("layer_norm1"))?,
|
||||
t2i: Attention::new(HIDDEN, HIDDEN / 2, 8, b.pp("cross_attn_token_to_image"))?,
|
||||
ln2: ln(HIDDEN, TORCH_LN_EPS, b.pp("layer_norm2"))?,
|
||||
mlp: FeedForward::new(HIDDEN, 2048, HIDDEN, 2, false, b.pp("mlp"))?,
|
||||
ln3: ln(HIDDEN, TORCH_LN_EPS, b.pp("layer_norm3"))?,
|
||||
ln4: ln(HIDDEN, TORCH_LN_EPS, b.pp("layer_norm4"))?,
|
||||
i2t: Attention::new(HIDDEN, HIDDEN / 2, 8, b.pp("cross_attn_image_to_token"))?,
|
||||
skip_first_pe: i == 0,
|
||||
});
|
||||
}
|
||||
let output_tokens = Tensor::cat(
|
||||
&[
|
||||
md.get((1, HIDDEN), "obj_score_token.weight")?,
|
||||
md.get((1, HIDDEN), "iou_token.weight")?,
|
||||
md.get((MASK_TOKENS, HIDDEN), "mask_tokens.weight")?,
|
||||
],
|
||||
0,
|
||||
)?;
|
||||
let up = ConvTranspose2dConfig { stride: 2, ..Default::default() };
|
||||
let one = Conv2dConfig::default();
|
||||
Ok(Self {
|
||||
point_pe: Fourier { pe: read_vec(&pe, (2, HIDDEN / 2), "shared_embedding.positional_embedding")? },
|
||||
image_pe,
|
||||
point_embed: pe.get((4, HIDDEN), "point_embed.weight")?.to_device(&Device::Cpu)?,
|
||||
not_a_point: pe.get((1, HIDDEN), "not_a_point_embed.weight")?.to_device(&Device::Cpu)?,
|
||||
no_mask: pe.get((1, HIDDEN), "no_mask_embed.weight")?.reshape((1, HIDDEN, 1, 1))?,
|
||||
conv_s0: candle_nn::conv2d(HIDDEN, HIDDEN / 8, 1, one, md.pp("conv_s0"))?,
|
||||
conv_s1: candle_nn::conv2d(HIDDEN, HIDDEN / 4, 1, one, md.pp("conv_s1"))?,
|
||||
output_tokens,
|
||||
blocks,
|
||||
final_t2i: Attention::new(HIDDEN, HIDDEN / 2, 8, t.pp("final_attn_token_to_image"))?,
|
||||
ln_final: ln(HIDDEN, TORCH_LN_EPS, t.pp("layer_norm_final_attn"))?,
|
||||
up1: candle_nn::conv_transpose2d(HIDDEN, HIDDEN / 4, 2, up, md.pp("upscale_conv1"))?,
|
||||
up_ln: ln(HIDDEN / 4, 1e-6, md.pp("upscale_layer_norm"))?,
|
||||
up2: candle_nn::conv_transpose2d(HIDDEN / 4, HIDDEN / 8, 2, up, md.pp("upscale_conv2"))?,
|
||||
hyper: (0..MASK_TOKENS)
|
||||
.map(|i| FeedForward::new(HIDDEN, HIDDEN, HIDDEN / 8, 3, false, md.pp(format!("output_hypernetworks_mlps.{i}"))))
|
||||
.collect::<Result<_>>()?,
|
||||
iou_head: FeedForward::new(HIDDEN, HIDDEN, MASK_TOKENS, 3, true, md.pp("iou_prediction_head"))?,
|
||||
obj_head: FeedForward::new(HIDDEN, HIDDEN, 1, 3, false, md.pp("pred_obj_score_head"))?,
|
||||
device,
|
||||
})
|
||||
}
|
||||
|
||||
/// The decoder's image features from the tracker neck's levels (288, 144, 72 per side).
|
||||
pub fn features(&self, levels: &[Tensor]) -> Result<Features> {
|
||||
let [l0, l1, l2, ..] = levels else { return Err(crate::Error::Model("tracker needs three feature levels".into())) };
|
||||
Ok(Features { image: l2.clone(), s0: self.conv_s0.forward(l0)?, s1: self.conv_s1.forward(l1)? })
|
||||
}
|
||||
|
||||
/// Sparse prompt embeddings `[1, n + 1, 256]` (a padding point closes the list).
|
||||
fn embed_clicks(&self, clicks: &[Click]) -> Result<Tensor> {
|
||||
let point_embed = to_vec(&self.point_embed)?;
|
||||
let not_a_point = to_vec(&self.not_a_point)?;
|
||||
let mut rows = Vec::with_capacity((clicks.len() + 1) * HIDDEN);
|
||||
for c in clicks {
|
||||
// the reference shifts pixel coordinates (in the 1008² input) by half a pixel
|
||||
let (x, y) = ((c.x * IMAGE as f32 + 0.5) / IMAGE as f32, (c.y * IMAGE as f32 + 0.5) / IMAGE as f32);
|
||||
let e = self.point_pe.encode(x, y);
|
||||
let label = usize::from(c.positive);
|
||||
rows.extend(e.iter().enumerate().map(|(i, v)| v + point_embed.get(label * HIDDEN + i).copied().unwrap_or(0.0)));
|
||||
}
|
||||
rows.extend_from_slice(¬_a_point);
|
||||
Ok(Tensor::from_vec(rows, (1, clicks.len() + 1, HIDDEN), &self.device)?)
|
||||
}
|
||||
|
||||
/// Predict masks for `clicks`. `multimask`: the three alternatives (best by predicted IoU is
|
||||
/// returned), else the single-mask output with SAM 2's stability fallback.
|
||||
pub fn predict(&self, f: &Features, clicks: &[Click], multimask: bool) -> Result<Prediction> {
|
||||
let sparse = self.embed_clicks(clicks)?;
|
||||
let tokens = Tensor::cat(&[self.output_tokens.unsqueeze(0)?, sparse], 1)?; // [1, T, 256]
|
||||
let image = f.image.broadcast_add(&self.no_mask)?; // dense prompt: "no mask"
|
||||
let (b, c, h, w) = image.dims4()?;
|
||||
let keys = image.flatten_from(2)?.transpose(1, 2)?.contiguous()?; // [1, HW, 256]
|
||||
let (mut q, mut k) = (tokens.clone(), keys);
|
||||
for blk in &self.blocks {
|
||||
(q, k) = blk.forward(&q, &k, &tokens, &self.image_pe)?;
|
||||
}
|
||||
let qf = (&q + &tokens)?;
|
||||
let kf = (&k + &self.image_pe)?;
|
||||
let q = self.ln_final.forward(&(&q + attn(&self.final_t2i, &qf, &kf, &k)?)?)?;
|
||||
let iou_token = q.i((.., 1, ..))?;
|
||||
let image = k.transpose(1, 2)?.reshape((b, c, h, w))?;
|
||||
let up = (self.up1.forward(&image)? + &f.s1)?;
|
||||
let up = ln_channels(&up, &self.up_ln)?.gelu_erf()?;
|
||||
let up = (self.up2.forward(&up)? + &f.s0)?.gelu_erf()?; // [1, 32, 288, 288]
|
||||
let (_, uc, uh, uw) = up.dims4()?;
|
||||
let hyper = (0..MASK_TOKENS)
|
||||
.map(|i| self.hyper.get(i).map_or(Err(crate::Error::Model("hypernetwork".into())), |m| m.forward(&q.i((.., 2 + i, ..))?)))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let hyper = Tensor::stack(&hyper, 1)?; // [1, 4, 32]
|
||||
let masks = hyper.matmul(&up.reshape((b, uc, uh * uw))?)?.reshape((b, MASK_TOKENS, uh, uw))?;
|
||||
let iou = to_vec(&self.iou_head.forward(&iou_token)?)?;
|
||||
let object_score = to_vec(&self.obj_head.forward(&q.i((.., 0, ..))?)?)?.first().copied().unwrap_or(0.0);
|
||||
let pick = |i: usize| -> Result<Prediction> {
|
||||
Ok(Prediction { logits: to_vec(&masks.i((0, i))?)?, iou: iou.get(i).copied().unwrap_or(0.0), object_score })
|
||||
};
|
||||
let best_multi = || (1..MASK_TOKENS).max_by(|a, b| iou.get(*a).partial_cmp(&iou.get(*b)).unwrap_or(std::cmp::Ordering::Equal)).unwrap_or(1);
|
||||
if multimask {
|
||||
return pick(best_multi());
|
||||
}
|
||||
// single-mask output unless it is unstable (area changes > 2 % when the threshold moves ±0.05)
|
||||
let single = masks.i((0, 0))?;
|
||||
let inter = single.gt(0.05)?.to_dtype(DType::F32)?.sum_all()?.to_scalar::<f32>()?;
|
||||
let union = single.gt(-0.05)?.to_dtype(DType::F32)?.sum_all()?.to_scalar::<f32>()?;
|
||||
let stability = if union > 0.0 { inter / union } else { 1.0 };
|
||||
if stability >= 0.98 { pick(0) } else { pick(best_multi()) }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
//! The SAM 3 vision backbone: a 32-layer ViT (Perception Encoder) with 2-D axial rotary
|
||||
//! position encoding, windowed attention (24 × 24) in most layers and global attention in four.
|
||||
//!
|
||||
//! Modified work (Apache License 2.0, §4(b)): ported by the LightCraft contributors in 2026 from
|
||||
//! the Python/PyTorch SAM 3 code of Hugging Face Transformers (`models/sam3/modeling_sam3.py`), Copyright The HuggingFace
|
||||
//! Team and Meta Platforms, Inc.; translated to Rust on candle and restructured. See NOTICE.
|
||||
|
||||
use candle_core::{D, Device, Module, Tensor};
|
||||
use candle_nn::{Conv2d, Conv2dConfig, LayerNorm, VarBuilder};
|
||||
|
||||
use crate::nn::{Attention, Mlp, ln, sdpa};
|
||||
use crate::{Error, Result};
|
||||
|
||||
pub const IMAGE: usize = 1008;
|
||||
pub const PATCH: usize = 14;
|
||||
/// Patches per side (72).
|
||||
pub const GRID: usize = IMAGE / PATCH;
|
||||
pub const HIDDEN: usize = 1024;
|
||||
const HEADS: usize = 16;
|
||||
const HEAD_DIM: usize = HIDDEN / HEADS;
|
||||
const LAYERS: usize = 32;
|
||||
const WINDOW: usize = 24;
|
||||
const GLOBAL: [usize; 4] = [7, 15, 23, 31];
|
||||
const PRETRAIN_GRID: usize = 336 / PATCH;
|
||||
const EPS: f64 = 1e-6;
|
||||
|
||||
/// Cosine and sine tables `[N, head_dim]` for a `side × side` grid whose coordinates are scaled
|
||||
/// by `scale` (`Sam3ViTRotaryEmbedding`: x frequencies then y, each duplicated pairwise).
|
||||
fn rope_tables(side: usize, scale: f64, device: &Device) -> Result<(Tensor, Tensor)> {
|
||||
let spatial = HEAD_DIM / 2;
|
||||
let inv: Vec<f64> = (0..spatial).step_by(2).map(|i| 1.0 / 10000f64.powf(i as f64 / spatial as f64)).collect();
|
||||
let n = side * side;
|
||||
let (mut cos, mut sin) = (Vec::with_capacity(n * HEAD_DIM), Vec::with_capacity(n * HEAD_DIM));
|
||||
for r in 0..side {
|
||||
for c in 0..side {
|
||||
let (x, y) = (c as f64 * scale, r as f64 * scale);
|
||||
for pos in [x, y] {
|
||||
for f in &inv {
|
||||
let a = (pos * f) as f32 as f64;
|
||||
let (s, co) = (a.sin() as f32, a.cos() as f32);
|
||||
cos.extend([co, co]);
|
||||
sin.extend([s, s]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok((Tensor::from_vec(cos, (n, HEAD_DIM), device)?, Tensor::from_vec(sin, (n, HEAD_DIM), device)?))
|
||||
}
|
||||
|
||||
/// `x·cos + rotate_pairwise(x)·sin`, where rotate_pairwise maps (a, b) pairs to (−b, a).
|
||||
fn apply_rope(x: &Tensor, cos: &Tensor, sin: &Tensor) -> Result<Tensor> {
|
||||
let dims = x.dims().to_vec();
|
||||
let last = *dims.last().ok_or_else(|| Error::Model("rope on a scalar".into()))?;
|
||||
let mut paired = dims.clone();
|
||||
paired.pop();
|
||||
paired.extend([last / 2, 2]);
|
||||
let p = x.reshape(paired.as_slice())?;
|
||||
let a = p.narrow(D::Minus1, 0, 1)?;
|
||||
let b = p.narrow(D::Minus1, 1, 1)?;
|
||||
let rot = Tensor::cat(&[b.neg()?, a], D::Minus1)?.reshape(dims.as_slice())?;
|
||||
Ok((x.broadcast_mul(cos)? + rot.broadcast_mul(sin)?)?)
|
||||
}
|
||||
|
||||
struct Layer {
|
||||
ln1: LayerNorm,
|
||||
attn: Attention,
|
||||
ln2: LayerNorm,
|
||||
mlp: Mlp,
|
||||
window: usize,
|
||||
}
|
||||
|
||||
impl Layer {
|
||||
/// `x`: `[B, H, W, C]`.
|
||||
fn forward(&self, x: &Tensor, rope: &(Tensor, Tensor)) -> Result<Tensor> {
|
||||
let (b, h, w, c) = x.dims4()?;
|
||||
let y = self.ln1.forward(x)?;
|
||||
let y = match (h.checked_div(self.window), w.checked_div(self.window)) {
|
||||
(Some(nh), Some(nw)) => {
|
||||
// non-overlapping windows (72 is a multiple of 24: no padding)
|
||||
let ws = self.window;
|
||||
let y = y.reshape((b, nh, ws, nw, ws, c))?.permute((0, 1, 3, 2, 4, 5))?.reshape((b * nh * nw, ws, ws, c))?;
|
||||
let y = self.attend(&y, rope)?;
|
||||
y.reshape((b, nh, nw, ws, ws, c))?.permute((0, 1, 3, 2, 4, 5))?.reshape((b, h, w, c))?
|
||||
}
|
||||
// global attention
|
||||
_ => self.attend(&y, rope)?,
|
||||
};
|
||||
let x = (x + y)?;
|
||||
let y = self.mlp.forward(&self.ln2.forward(&x)?)?;
|
||||
Ok((x + y)?)
|
||||
}
|
||||
|
||||
/// RoPE self-attention over `[B, H, W, C]`.
|
||||
fn attend(&self, x: &Tensor, rope: &(Tensor, Tensor)) -> Result<Tensor> {
|
||||
let (b, h, w, c) = x.dims4()?;
|
||||
let seq = x.reshape((b, h * w, c))?;
|
||||
Ok(self.attn_rope(&seq, rope)?.reshape((b, h, w, c))?)
|
||||
}
|
||||
|
||||
fn attn_rope(&self, x: &Tensor, rope: &(Tensor, Tensor)) -> Result<Tensor> {
|
||||
let (b, n, _) = x.dims3()?;
|
||||
let split = |t: Tensor| -> Result<Tensor> { Ok(t.reshape((b, n, HEADS, HEAD_DIM))?.transpose(1, 2)?.contiguous()?) };
|
||||
let (q, k, v) = self.attn.qkv(x)?;
|
||||
let q = apply_rope(&split(q)?, &rope.0, &rope.1)?;
|
||||
let k = apply_rope(&split(k)?, &rope.0, &rope.1)?;
|
||||
let v = split(v)?;
|
||||
let o = sdpa(&q, &k, &v, None, (HEAD_DIM as f64).powf(-0.5))?;
|
||||
let o = o.transpose(1, 2)?.reshape((b, n, HIDDEN))?;
|
||||
self.attn.out(&o)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Vit {
|
||||
patch: Conv2d,
|
||||
pos: Tensor,
|
||||
ln_pre: LayerNorm,
|
||||
layers: Vec<Layer>,
|
||||
rope_window: (Tensor, Tensor),
|
||||
rope_global: (Tensor, Tensor),
|
||||
}
|
||||
|
||||
impl Vit {
|
||||
/// `vb` at `detector_model.vision_encoder.backbone`.
|
||||
pub fn new(vb: VarBuilder) -> Result<Self> {
|
||||
let device = vb.device().clone();
|
||||
let cfg = Conv2dConfig { stride: PATCH, ..Default::default() };
|
||||
let patch = candle_nn::conv2d_no_bias(3, HIDDEN, PATCH, cfg, vb.pp("embeddings.patch_embeddings.projection"))?;
|
||||
let pos = vb.get((1, PRETRAIN_GRID * PRETRAIN_GRID, HIDDEN), "embeddings.position_embeddings")?;
|
||||
// tile (not interpolate) the 24 × 24 pretraining table over the 72 × 72 grid
|
||||
let reps = GRID.div_ceil(PRETRAIN_GRID);
|
||||
let pos = pos
|
||||
.reshape((1, PRETRAIN_GRID, PRETRAIN_GRID, HIDDEN))?
|
||||
.repeat((1, reps, reps, 1))?
|
||||
.narrow(1, 0, GRID)?
|
||||
.narrow(2, 0, GRID)?
|
||||
.reshape((1, GRID * GRID, HIDDEN))?;
|
||||
let mut layers = Vec::with_capacity(LAYERS);
|
||||
for i in 0..LAYERS {
|
||||
let l = vb.pp(format!("layers.{i}"));
|
||||
layers.push(Layer {
|
||||
ln1: ln(HIDDEN, EPS, l.pp("layer_norm1"))?,
|
||||
attn: Attention::new(HIDDEN, HIDDEN, HEADS, l.pp("attention"))?,
|
||||
ln2: ln(HIDDEN, EPS, l.pp("layer_norm2"))?,
|
||||
mlp: Mlp::new(HIDDEN, 4736, true, l.pp("mlp"))?,
|
||||
window: if GLOBAL.contains(&i) { 0 } else { WINDOW },
|
||||
});
|
||||
}
|
||||
Ok(Self {
|
||||
patch,
|
||||
pos,
|
||||
ln_pre: ln(HIDDEN, EPS, vb.pp("layer_norm"))?,
|
||||
layers,
|
||||
rope_window: rope_tables(WINDOW, 1.0, &device)?,
|
||||
rope_global: rope_tables(GRID, WINDOW as f64 / GRID as f64, &device)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// `pixels`: `[1, 3, 1008, 1008]` normalized to −1..1. Returns `[1, 1024, 72, 72]`.
|
||||
pub fn forward(&self, pixels: &Tensor) -> Result<Tensor> {
|
||||
let x = self.patch.forward(pixels)?; // [1, C, 72, 72]
|
||||
let b = x.dim(0)?;
|
||||
let x = x.flatten_from(2)?.transpose(1, 2)?.broadcast_add(&self.pos)?;
|
||||
let mut x = self.ln_pre.forward(&x.reshape((b, GRID, GRID, HIDDEN))?)?;
|
||||
for l in &self.layers {
|
||||
let rope = if l.window > 0 { &self.rope_window } else { &self.rope_global };
|
||||
x = l.forward(&x, rope)?;
|
||||
}
|
||||
Ok(x.permute((0, 3, 1, 2))?.contiguous()?)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
//! Safetensors weights read on demand.
|
||||
//!
|
||||
//! Tensors are read from the file when a module asks for them (positional reads, no memory
|
||||
//! map: `unsafe` stays in `lightcraft-sysmem`), so building only the point-prompt model never
|
||||
//! touches the text encoder's bytes.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use candle_core::{DType, Device, Shape, Tensor};
|
||||
use candle_nn::var_builder::SimpleBackend;
|
||||
|
||||
use crate::{Error, Result};
|
||||
|
||||
struct Entry {
|
||||
shape: Vec<usize>,
|
||||
start: u64,
|
||||
end: u64,
|
||||
}
|
||||
|
||||
/// An open safetensors file: its header, and the file to read tensor bytes from.
|
||||
pub struct Weights {
|
||||
file: Mutex<File>,
|
||||
entries: HashMap<String, Entry>,
|
||||
data_start: u64,
|
||||
}
|
||||
|
||||
impl Weights {
|
||||
pub fn open(path: &Path) -> Result<Arc<Self>> {
|
||||
let mut file = File::open(path).map_err(|e| Error::Model(format!("{}: {e}", path.display())))?;
|
||||
let file_len = file.metadata().map_err(|e| Error::Model(format!("{}: {e}", path.display())))?.len();
|
||||
let mut len = [0u8; 8];
|
||||
file.read_exact(&mut len).map_err(|e| Error::Model(format!("{}: {e}", path.display())))?;
|
||||
let n = u64::from_le_bytes(len);
|
||||
if n > 100 << 20 || n.saturating_add(8) > file_len {
|
||||
return Err(Error::Model(format!("{}: not a safetensors file (header of {n} bytes)", path.display())));
|
||||
}
|
||||
let mut header = vec![0u8; n as usize];
|
||||
file.read_exact(&mut header).map_err(|e| Error::Model(format!("{}: {e}", path.display())))?;
|
||||
let header: serde_json::Map<String, serde_json::Value> =
|
||||
serde_json::from_slice(&header).map_err(|e| Error::Model(format!("{}: bad header: {e}", path.display())))?;
|
||||
let mut entries = HashMap::new();
|
||||
for (name, v) in header {
|
||||
if name == "__metadata__" {
|
||||
continue;
|
||||
}
|
||||
let dtype = match v.get("dtype").and_then(|d| d.as_str()) {
|
||||
Some("F32") => DType::F32,
|
||||
other => return Err(Error::Model(format!("{name}: unsupported dtype {other:?} (the SAM 3 checkpoint is F32)"))),
|
||||
};
|
||||
let shape: Vec<usize> = v
|
||||
.get("shape")
|
||||
.and_then(|s| s.as_array())
|
||||
.map(|a| a.iter().filter_map(|x| x.as_u64().map(|x| x as usize)).collect())
|
||||
.unwrap_or_default();
|
||||
let offsets: Vec<u64> =
|
||||
v.get("data_offsets").and_then(|s| s.as_array()).map(|a| a.iter().filter_map(|x| x.as_u64()).collect()).unwrap_or_default();
|
||||
let [start, end] = offsets[..] else { return Err(Error::Model(format!("{name}: bad data_offsets"))) };
|
||||
let want = shape.iter().try_fold(dtype.size_in_bytes(), |a, d| a.checked_mul(*d));
|
||||
if end.checked_sub(start) != want.map(|w| w as u64) {
|
||||
return Err(Error::Model(format!("{name}: size does not match its shape")));
|
||||
}
|
||||
// the bytes must be in the file (a damaged or truncated download fails here, not later)
|
||||
if (8 + n).checked_add(end).is_none_or(|e| e > file_len) {
|
||||
return Err(Error::Model(format!("{}: tensor {name} lies outside the file (incomplete or damaged download?)", path.display())));
|
||||
}
|
||||
entries.insert(name, Entry { shape, start, end });
|
||||
}
|
||||
Ok(Arc::new(Self { file: Mutex::new(file), entries, data_start: 8 + n }))
|
||||
}
|
||||
|
||||
pub fn contains(&self, name: &str) -> bool {
|
||||
self.entries.contains_key(name)
|
||||
}
|
||||
|
||||
/// Tensor `name` as stored, on the CPU.
|
||||
pub fn read(&self, name: &str) -> Result<Tensor> {
|
||||
let e = self.entries.get(name).ok_or_else(|| Error::Model(format!("missing tensor `{name}`")))?;
|
||||
// checked against the file's length in `open`
|
||||
let len = usize::try_from(e.end.saturating_sub(e.start)).map_err(|_| Error::Model(format!("{name}: too large")))?;
|
||||
let mut bytes = vec![0u8; len];
|
||||
{
|
||||
let mut f = self.file.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
f.seek(SeekFrom::Start(self.data_start + e.start)).map_err(|err| Error::Model(format!("{name}: {err}")))?;
|
||||
f.read_exact(&mut bytes).map_err(|err| Error::Model(format!("{name}: {err}")))?;
|
||||
}
|
||||
let v: Vec<f32> = bytes.as_chunks::<4>().0.iter().map(|c| f32::from_le_bytes(*c)).collect();
|
||||
Ok(Tensor::from_vec(v, e.shape.as_slice(), &Device::Cpu)?)
|
||||
}
|
||||
}
|
||||
|
||||
/// [`Weights`] as a candle `VarBuilder` backend.
|
||||
pub struct Backend(pub Arc<Weights>);
|
||||
|
||||
impl SimpleBackend for Backend {
|
||||
fn get(&self, s: Shape, name: &str, _h: candle_nn::Init, dtype: DType, dev: &Device) -> candle_core::Result<Tensor> {
|
||||
let t = self.get_unchecked(name, dtype, dev)?;
|
||||
if t.shape() != &s {
|
||||
candle_core::bail!("shape mismatch for {name}: expected {s:?}, got {:?}", t.shape())
|
||||
}
|
||||
Ok(t)
|
||||
}
|
||||
|
||||
fn get_unchecked(&self, name: &str, dtype: DType, dev: &Device) -> candle_core::Result<Tensor> {
|
||||
let t = self.0.read(name).map_err(|e| candle_core::Error::Msg(e.to_string()))?;
|
||||
t.to_dtype(dtype)?.to_device(dev)
|
||||
}
|
||||
|
||||
fn contains_tensor(&self, name: &str) -> bool {
|
||||
self.0.contains(name)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn write(name: &str, header: &str, data_len: usize) -> std::path::PathBuf {
|
||||
let p = std::env::temp_dir().join(format!("lc-weights-{name}-{}.safetensors", std::process::id()));
|
||||
let mut b = (header.len() as u64).to_le_bytes().to_vec();
|
||||
b.extend_from_slice(header.as_bytes());
|
||||
b.extend(std::iter::repeat_n(0u8, data_len));
|
||||
std::fs::write(&p, b).unwrap();
|
||||
p
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tensors_must_lie_inside_the_file() {
|
||||
let ok = write("ok", r#"{"a":{"dtype":"F32","shape":[2,2],"data_offsets":[0,16]}}"#, 16);
|
||||
let w = Weights::open(&ok).unwrap();
|
||||
assert_eq!(w.read("a").unwrap().dims(), &[2, 2]);
|
||||
// a truncated download: the header promises bytes that aren't there
|
||||
let short = write("short", r#"{"a":{"dtype":"F32","shape":[2,2],"data_offsets":[0,16]}}"#, 8);
|
||||
assert!(Weights::open(&short).err().unwrap().to_string().contains("outside the file"));
|
||||
// hostile offsets: huge, or beyond u64
|
||||
let huge = write("huge", r#"{"a":{"dtype":"F32","shape":[1073741824,1073741824],"data_offsets":[0,4611686018427387904]}}"#, 16);
|
||||
assert!(Weights::open(&huge).is_err());
|
||||
let wrap = write("wrap", r#"{"a":{"dtype":"F32","shape":[1],"data_offsets":[18446744073709551612,18446744073709551616]}}"#, 16);
|
||||
assert!(Weights::open(&wrap).is_err());
|
||||
// a header longer than the file
|
||||
let p = std::env::temp_dir().join(format!("lc-weights-hdr-{}.safetensors", std::process::id()));
|
||||
std::fs::write(&p, 1000u64.to_le_bytes()).unwrap();
|
||||
assert!(Weights::open(&p).is_err());
|
||||
for f in [ok, short, huge, wrap, p] {
|
||||
let _ = std::fs::remove_file(f);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loaded 100 of 157 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user