Compare commits
1058
Commits
No files matched your search
@@ -26,9 +26,15 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 300
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
# Optional font build input, so the Japanese font tests run (they skip without it).
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: 8dcdacd5153e64560d109541a47d806f26f048c0
|
||||
path: craft-fonts
|
||||
- name: Build and test in a FreeBSD VM
|
||||
uses: vmactions/freebsd-vm@c46abacb49f09938ca4e1702d15d836285d694cc # v1
|
||||
uses: vmactions/freebsd-vm@c46abacb49f09938ca4e1702d15d836285d694cc # v1.5.9
|
||||
with:
|
||||
release: "14.3"
|
||||
usesh: true
|
||||
@@ -44,5 +50,6 @@ jobs:
|
||||
# The checkout is synced in under another owner; let tests that call git read it.
|
||||
git config --global --add safe.directory "*"
|
||||
rustc --version
|
||||
export CRAFT_FONTS_DIR="$PWD/craft-fonts" CRAFT_FONTS_REQUIRED=1
|
||||
CARGO_PROFILE_DEV_DEBUG=0 cargo build --workspace --all-targets --locked
|
||||
CARGO_PROFILE_DEV_DEBUG=0 cargo test --workspace --locked
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
env:
|
||||
ACTIONLINT_VERSION: 1.7.12
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- name: Install tools
|
||||
run: |
|
||||
sudo apt-get update
|
||||
@@ -47,7 +47,27 @@ jobs:
|
||||
- name: XML, desktop entry, AppStream, Flatpak manifest
|
||||
run: |
|
||||
xmllint --noout packaging/windows/lightcraft.wxs packaging/macos/*.plist* packaging/linux/*.xml*
|
||||
# Windows Installer needs Icon ids ending in .ico or .exe, or shortcuts show a blank icon.
|
||||
if grep -oE '<Icon Id="[^"]*"' packaging/windows/lightcraft.wxs | grep -vE '\.(ico|exe)"$'; then
|
||||
echo "::error::Icon Id must end in .ico or .exe"; exit 1
|
||||
fi
|
||||
# The interactive installer asks where to install and ends on a completion page (issues #18, #399).
|
||||
for want in '<UIRef Id="LightcraftUI" />' 'Value="InstallDirDlg"' '<Property Id="WIXUI_INSTALLDIR" Value="INSTALLFOLDER" />' \
|
||||
'Key="Software\LightCraft" Name="InstallFolder"' 'Value="LaunchLightcraft"'; do
|
||||
grep -qF "$want" packaging/windows/lightcraft.wxs || { echo "::error::lightcraft.wxs lost: $want"; exit 1; }
|
||||
done
|
||||
desktop-file-validate packaging/linux/ai.storyteller.lightcraft.desktop
|
||||
sed -e 's/@VERSION@/0.0.0/' -e 's/@DATE@/2026-01-01/' packaging/linux/ai.storyteller.lightcraft.metainfo.xml.in >"$RUNNER_TEMP/ai.storyteller.lightcraft.metainfo.xml"
|
||||
appstreamcli validate --no-net --explain "$RUNNER_TEMP/ai.storyteller.lightcraft.metainfo.xml"
|
||||
python3 -c "import yaml; m = yaml.safe_load(open('packaging/linux/flatpak/ai.storyteller.lightcraft.yml')); assert m['id'] == 'ai.storyteller.lightcraft' and m['command'] == 'lightcraft'"
|
||||
python3 - <<'EOF'
|
||||
import yaml
|
||||
src = yaml.safe_load(open("packaging/linux/flatpak/ai.storyteller.lightcraft.yml"))
|
||||
bundle = yaml.safe_load(open("packaging/linux/flatpak/ai.storyteller.lightcraft.bundle.yml"))
|
||||
for m in (src, bundle):
|
||||
assert m["id"] == "ai.storyteller.lightcraft" and m["command"] == "lightcraft", m["id"]
|
||||
for key in ("runtime", "runtime-version", "sdk", "finish-args"):
|
||||
assert src[key] == bundle[key], f"Flatpak manifests disagree on {key}: {src[key]} vs {bundle[key]}"
|
||||
print("Flatpak manifests ok")
|
||||
EOF
|
||||
- name: FreeBSD tarball layout (stub binaries)
|
||||
run: bash packaging/freebsd/package.sh --dry-run
|
||||
+162
-31
@@ -1,9 +1,12 @@
|
||||
# Release pipeline: every push to `release` builds signed installers for every platform and
|
||||
# creates or updates the draft GitHub Release "LightCraft v<version>". See docs/releasing.md.
|
||||
# Release pipeline: every push to `release` builds platform artifacts (signed where credentials
|
||||
# are configured) and creates or updates the draft GitHub Release "LightCraft v<version>".
|
||||
# See docs/releasing.md.
|
||||
#
|
||||
# All jobs run in the `release` environment (restricted to the `release` branch), which holds
|
||||
# the signing secrets. Every secret is optional: missing ones produce unsigned artifacts and a
|
||||
# warning, never a failed build.
|
||||
# The signing jobs (macOS, Windows) and the draft-release job run in the `release` environment
|
||||
# (restricted to the `release` branch), which holds the signing secrets. Every secret is optional:
|
||||
# missing ones produce unsigned artifacts and a warning, never a failed build. The jobs that sign
|
||||
# nothing (version, Linux, Flatpak, FreeBSD, web) run without it, so a dispatch on any branch is
|
||||
# a dry run of them; the release job needs the signed jobs, so a branch run never drafts a release.
|
||||
name: Release
|
||||
|
||||
on:
|
||||
@@ -35,13 +38,13 @@ jobs:
|
||||
version:
|
||||
name: version
|
||||
runs-on: ubuntu-latest
|
||||
environment: release
|
||||
# No `environment: release` here or on the other jobs that sign nothing: no release secrets.
|
||||
outputs:
|
||||
version: ${{ steps.v.outputs.version }}
|
||||
date: ${{ steps.v.outputs.date }}
|
||||
prerelease: ${{ steps.v.outputs.prerelease }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- id: v
|
||||
name: Resolve version
|
||||
run: |
|
||||
@@ -63,17 +66,23 @@ jobs:
|
||||
macos:
|
||||
name: macOS universal
|
||||
needs: version
|
||||
runs-on: macos-15
|
||||
runs-on: macos-15-xlarge # M2 Pro larger runner; use only in release.yml (actions_macos_xl budget caps spend)
|
||||
environment: release
|
||||
env:
|
||||
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
|
||||
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: 8dcdacd5153e64560d109541a47d806f26f048c0 # bump deliberately
|
||||
path: craft-fonts
|
||||
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
|
||||
with:
|
||||
targets: aarch64-apple-darwin, x86_64-apple-darwin
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
key: release-macos-universal
|
||||
- name: Apply version override
|
||||
@@ -87,11 +96,13 @@ jobs:
|
||||
run: packaging/macos/import-cert.sh
|
||||
- name: Build, sign, notarize
|
||||
env:
|
||||
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
|
||||
CRAFT_FONTS_REQUIRED: "1"
|
||||
APPLE_ID: ${{ secrets.APPLE_ID }}
|
||||
APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
|
||||
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
|
||||
run: packaging/macos/package.sh --arch universal
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-macos
|
||||
path: dist/release/*
|
||||
@@ -112,27 +123,39 @@ jobs:
|
||||
include:
|
||||
- { arch: x64, target: x86_64-pc-windows-msvc }
|
||||
- { arch: x86, target: i686-pc-windows-msvc }
|
||||
# Windows on ARM: cross-compiled on the x64 runner, so signing and WiX run as above.
|
||||
# .github/workflows/windows-arm64.yml installs and runs it on ARM64 hardware.
|
||||
- { arch: arm64, target: aarch64-pc-windows-msvc }
|
||||
env:
|
||||
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
|
||||
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: 8dcdacd5153e64560d109541a47d806f26f048c0 # bump deliberately
|
||||
path: craft-fonts
|
||||
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
key: release-windows-${{ matrix.arch }}
|
||||
- name: Apply version override
|
||||
if: inputs.version != ''
|
||||
shell: bash
|
||||
run: cargo xtask version set "$LIGHTCRAFT_VERSION"
|
||||
# package.ps1 adds the matching WixToolset.UI/Util.wixext (5.0.2) to the global extension cache.
|
||||
- name: Install WiX
|
||||
shell: pwsh
|
||||
run: dotnet tool install --global wix --version 5.0.2
|
||||
- name: Build, sign, package
|
||||
shell: pwsh
|
||||
env:
|
||||
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
|
||||
CRAFT_FONTS_REQUIRED: "1"
|
||||
WINDOWS_CERTIFICATE: ${{ secrets.WINDOWS_CERTIFICATE }}
|
||||
WINDOWS_CERTIFICATE_PASSWORD: ${{ secrets.WINDOWS_CERTIFICATE_PASSWORD }}
|
||||
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
|
||||
@@ -142,7 +165,7 @@ jobs:
|
||||
AZURE_SIGNING_ACCOUNT: ${{ secrets.AZURE_SIGNING_ACCOUNT }}
|
||||
AZURE_CERT_PROFILE: ${{ secrets.AZURE_CERT_PROFILE }}
|
||||
run: ./packaging/windows/package.ps1 -Arch ${{ matrix.arch }}
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-windows-${{ matrix.arch }}
|
||||
path: dist/release/*
|
||||
@@ -154,7 +177,6 @@ jobs:
|
||||
# The oldest GitHub-hosted image, so the binaries only need glibc >= 2.35 (Ubuntu 22.04+,
|
||||
# Debian 12+, Fedora 36+, RHEL 10). See docs/releasing.md.
|
||||
runs-on: ${{ matrix.runner }}
|
||||
environment: release
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -166,24 +188,33 @@ jobs:
|
||||
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
|
||||
NFPM_VERSION: 2.47.0
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: 8dcdacd5153e64560d109541a47d806f26f048c0 # bump deliberately
|
||||
path: craft-fonts
|
||||
- name: Install build and packaging tools
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
libxkbcommon-dev libwayland-dev libx11-dev libxrandr-dev libxi-dev libgl1-mesa-dev \
|
||||
desktop-file-utils appstream python3-yaml
|
||||
desktop-file-utils appstream python3-yaml zsync
|
||||
curl -fsSL -o "$RUNNER_TEMP/nfpm.deb" \
|
||||
"https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_${{ matrix.deb }}.deb"
|
||||
sudo dpkg -i "$RUNNER_TEMP/nfpm.deb"
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
|
||||
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
key: release-linux-${{ matrix.arch }}
|
||||
- name: Apply version override
|
||||
if: inputs.version != ''
|
||||
run: cargo xtask version set "$LIGHTCRAFT_VERSION"
|
||||
- name: Build and package (AppImage, deb, rpm, tar.gz)
|
||||
env:
|
||||
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
|
||||
CRAFT_FONTS_REQUIRED: "1"
|
||||
run: packaging/linux/package.sh
|
||||
- name: Check the packages
|
||||
run: |
|
||||
@@ -194,42 +225,142 @@ jobs:
|
||||
for appimage in dist/release/*.AppImage; do
|
||||
chmod +x "$appimage"
|
||||
APPIMAGE_EXTRACT_AND_RUN=1 "$appimage" --version
|
||||
# Update information is embedded, and the .zsync for delta updates sits beside it.
|
||||
"$appimage" --appimage-updateinformation | grep -F 'gh-releases-zsync|'
|
||||
test -s "$appimage.zsync"
|
||||
done
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-linux-${{ matrix.arch }}
|
||||
path: dist/release/*
|
||||
if-no-files-found: error
|
||||
|
||||
flatpak:
|
||||
name: Flatpak ${{ matrix.arch }}
|
||||
needs: [version, linux]
|
||||
# Repackages the Linux job's tarball (the same binaries as the AppImage/deb/rpm); no Rust
|
||||
# build here. 24.04 for a newer flatpak-builder (1.4) than 22.04 ships.
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { arch: x86_64, runner: ubuntu-24.04 }
|
||||
- { arch: aarch64, runner: ubuntu-24.04-arm }
|
||||
env:
|
||||
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
|
||||
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: release-linux-${{ matrix.arch }}
|
||||
path: dist/linux
|
||||
- name: Install flatpak-builder
|
||||
run: |
|
||||
sudo apt-get update
|
||||
# librsvg2-common: the SVG pixbuf loader. flatpak-builder runs the host's
|
||||
# `appstreamcli compose`, which renders the scalable icon and fails without it.
|
||||
sudo apt-get install -y --no-install-recommends flatpak flatpak-builder librsvg2-common
|
||||
# Ubuntu 24.04's AppArmor blocks the unprivileged user namespaces bubblewrap needs.
|
||||
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
||||
- name: Build the Flatpak bundle
|
||||
run: packaging/linux/flatpak-bundle.sh "dist/linux/lightcraft-$LIGHTCRAFT_VERSION-linux-${{ matrix.arch }}.tar.gz"
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-flatpak-${{ matrix.arch }}
|
||||
path: dist/release/*.flatpak
|
||||
if-no-files-found: error
|
||||
|
||||
freebsd:
|
||||
name: FreeBSD x86_64
|
||||
needs: version
|
||||
# GitHub has no FreeBSD runners: build in a FreeBSD 14.3 VM, the same image and packages as
|
||||
# .github/workflows/freebsd.yml.
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
env:
|
||||
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
|
||||
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: 8dcdacd5153e64560d109541a47d806f26f048c0 # bump deliberately
|
||||
path: craft-fonts
|
||||
- name: Build and package in a FreeBSD VM (tar.gz)
|
||||
# vmactions/freebsd-vm v1.5.9, pinned by commit (it runs our build).
|
||||
uses: vmactions/freebsd-vm@c46abacb49f09938ca4e1702d15d836285d694cc # v1.5.9
|
||||
with:
|
||||
release: "14.3"
|
||||
usesh: true
|
||||
cpu: 4
|
||||
mem: 12288
|
||||
# Passed into the VM by name; never templated into the script below.
|
||||
envs: "LIGHTCRAFT_VERSION LIGHTCRAFT_BUILD_DATE LIGHTCRAFT_BUILD_SHA INPUT_VERSION"
|
||||
# The FreeBSD CI packages, plus bash for the packaging script.
|
||||
prepare: |
|
||||
pkg install -y bash git curl pkgconf cmake python3 libxkbcommon wayland libX11 libXcursor libXrandr libXi libxcb mesa-libs vulkan-loader gtk3 fontconfig freetype2 alsa-lib
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
|
||||
run: |
|
||||
set -e
|
||||
. "$HOME/.cargo/env"
|
||||
git config --global --add safe.directory "$PWD"
|
||||
# Build outside the checkout so only dist/ is copied back out of the VM (copying the
|
||||
# target dir back fails on large builds). 4 jobs: more runs the VM out of memory.
|
||||
export CARGO_TARGET_DIR=/var/tmp/lightcraft-target CARGO_BUILD_JOBS=4
|
||||
if [ -n "$INPUT_VERSION" ]; then cargo xtask version set "$LIGHTCRAFT_VERSION"; fi
|
||||
# craft-fonts was checked out next to the sources and synced into the VM with them.
|
||||
export CRAFT_FONTS_DIR="$PWD/craft-fonts" CRAFT_FONTS_REQUIRED=1
|
||||
bash packaging/freebsd/package.sh
|
||||
tar -tzf dist/release/lightcraft-*-freebsd-x86_64.tar.gz \
|
||||
| grep -E '/bin/lightcraft$|/bin/lightcraft-cli$|/share/applications/ai\.storyteller\.lightcraft\.desktop$|/share/doc/lightcraft/LICENSE-MIT$'
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-freebsd-x86_64
|
||||
path: dist/release/*
|
||||
if-no-files-found: error
|
||||
|
||||
web:
|
||||
name: Web (wasm)
|
||||
needs: version
|
||||
runs-on: ubuntu-latest
|
||||
environment: release
|
||||
env:
|
||||
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
|
||||
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
|
||||
WASM_BINDGEN_VERSION: 0.2.129
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: 8dcdacd5153e64560d109541a47d806f26f048c0 # bump deliberately
|
||||
path: craft-fonts
|
||||
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
|
||||
with:
|
||||
targets: wasm32-unknown-unknown
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
key: release-web
|
||||
- name: Install wasm-bindgen CLI
|
||||
run: |
|
||||
# Must match the wasm-bindgen version in Cargo.lock exactly (cargo xtask web checks it).
|
||||
command -v wasm-bindgen >/dev/null && wasm-bindgen --version | grep -q "$WASM_BINDGEN_VERSION" \
|
||||
|| cargo install wasm-bindgen-cli --version "$WASM_BINDGEN_VERSION" --locked
|
||||
if ! { command -v wasm-bindgen >/dev/null && wasm-bindgen --version | grep -q "$WASM_BINDGEN_VERSION"; }; then
|
||||
cargo install wasm-bindgen-cli --version "$WASM_BINDGEN_VERSION" --locked
|
||||
fi
|
||||
wasm-bindgen --version
|
||||
- name: Apply version override
|
||||
if: inputs.version != ''
|
||||
run: cargo xtask version set "$LIGHTCRAFT_VERSION"
|
||||
- name: Build and zip
|
||||
env:
|
||||
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
|
||||
CRAFT_FONTS_REQUIRED: "1"
|
||||
run: packaging/web/package.sh
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-web
|
||||
path: dist/release/*
|
||||
@@ -237,7 +368,7 @@ jobs:
|
||||
|
||||
release:
|
||||
name: Draft GitHub Release
|
||||
needs: [version, macos, windows, linux, web]
|
||||
needs: [version, macos, windows, linux, flatpak, freebsd, web]
|
||||
runs-on: ubuntu-latest
|
||||
environment: release
|
||||
permissions:
|
||||
@@ -248,7 +379,7 @@ jobs:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
steps:
|
||||
- uses: actions/download-artifact@v4
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
pattern: release-*
|
||||
path: artifacts
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
# Windows on ARM: package the ARM64 release the way release.yml does (cross-compiled on an x64
|
||||
# runner, unsigned here), then install that MSI on a Windows 11 ARM64 runner, check the installed
|
||||
# binaries are ARM64, run the CLI natively, and uninstall. (A native ARM64 `cargo test` job is left
|
||||
# out for now: PdfCraft saw an intermittent access violation in egui UI tests there.)
|
||||
name: Windows ARM64
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
paths:
|
||||
- Cargo.toml
|
||||
- Cargo.lock
|
||||
- "**/Cargo.toml"
|
||||
- "**/build.rs"
|
||||
- .github/workflows/windows-arm64.yml
|
||||
- packaging/windows/**
|
||||
schedule:
|
||||
- cron: "41 4 * * 1" # weekly, so drift in the runner images shows up
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: windows-arm64-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
CARGO_INCREMENTAL: 0
|
||||
|
||||
jobs:
|
||||
package:
|
||||
name: package (cross-compiled on x64)
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
|
||||
with:
|
||||
targets: aarch64-pc-windows-msvc
|
||||
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
key: windows-arm64-package
|
||||
- name: Install WiX
|
||||
shell: pwsh
|
||||
run: dotnet tool install --global wix --version 5.0.2
|
||||
- name: Build and package (unsigned)
|
||||
shell: pwsh
|
||||
env:
|
||||
DIST: ${{ github.workspace }}\dist\arm64
|
||||
run: ./packaging/windows/package.ps1 -Arch arm64
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: lightcraft-windows-arm64
|
||||
path: dist/arm64/*
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
install:
|
||||
name: install and run on ARM64
|
||||
needs: package
|
||||
runs-on: windows-11-arm
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: lightcraft-windows-arm64
|
||||
path: dist
|
||||
- name: Install the MSI, check it, run it, uninstall it
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$msi = (Get-ChildItem dist/*.msi | Select-Object -First 1).FullName
|
||||
$p = Start-Process msiexec.exe -ArgumentList '/i', "`"$msi`"", '/qn', '/l*v', 'install.log' -Wait -PassThru
|
||||
if ($p.ExitCode -ne 0) { Get-Content install.log -Tail 60; throw "msiexec /i exited $($p.ExitCode)" }
|
||||
$dir = Join-Path $env:ProgramFiles 'LightCraft'
|
||||
foreach ($exe in 'lightcraft.exe', 'lightcraft-cli.exe') {
|
||||
$path = Join-Path $dir $exe
|
||||
if (-not (Test-Path $path)) { throw "$exe not installed in $dir" }
|
||||
$bytes = [System.IO.File]::ReadAllBytes($path)
|
||||
$machine = [BitConverter]::ToUInt16($bytes, [BitConverter]::ToInt32($bytes, 0x3C) + 4)
|
||||
if ($machine -ne 0xAA64) { throw "$exe is for machine 0x$('{0:X}' -f $machine), not ARM64" }
|
||||
Write-Output "ok $path (ARM64)"
|
||||
}
|
||||
# The installed CLI runs natively on ARM64.
|
||||
$cli = Join-Path $dir 'lightcraft-cli.exe'
|
||||
$out = & $cli --version 2>&1 | Out-String
|
||||
if ($LASTEXITCODE -ne 0) { Write-Output $out; throw "lightcraft-cli --version exited $LASTEXITCODE" }
|
||||
Write-Output "ok lightcraft-cli --version: $($out.Trim())"
|
||||
$p = Start-Process msiexec.exe -ArgumentList '/x', "`"$msi`"", '/qn' -Wait -PassThru
|
||||
if ($p.ExitCode -ne 0) { throw "msiexec /x exited $($p.ExitCode)" }
|
||||
if (Test-Path (Join-Path $dir 'lightcraft.exe')) { throw 'uninstall left lightcraft.exe behind' }
|
||||
Write-Output 'ok uninstalled'
|
||||
@@ -7,3 +7,10 @@ plan/
|
||||
/log
|
||||
.DS_Store
|
||||
.claude/
|
||||
/craft-fonts
|
||||
# nix build / nix develop
|
||||
/result
|
||||
/result-*
|
||||
# model weights are never committed (SAM 3: docs/ai-masks.md)
|
||||
*.safetensors
|
||||
*.safetensors.part
|
||||
@@ -1,6 +1,6 @@
|
||||
# LightCraft — instructions for agents
|
||||
|
||||
LightCraft is a clean-room, open-source, pure-Rust photo library + non-destructive raw developer targeting Adobe Lightroom parity (and beyond). Native on macOS, Windows, Linux; web via WASM. Sibling of `../printcraft` (Acrobat), `../photocraft` (Photoshop), `../drawcraft` (Illustrator) and `../filmcraft` (Premiere), with the same conventions.
|
||||
LightCraft is a clean-room, open-source, pure-Rust photo library + non-destructive raw developer targeting Adobe Lightroom parity (and beyond). Native on macOS, Windows, Linux; web via WASM. Sibling of `../pdfcraft` (Acrobat), `../photocraft` (Photoshop), `../vectorcraft` (Illustrator) and `../filmcraft` (Premiere), with the same conventions.
|
||||
|
||||
## Start every session here
|
||||
1. Read `plan/STATUS.md` (current milestone, next unchecked task, blockers).
|
||||
@@ -62,10 +62,25 @@ of it. Full standard: `../craftrules/standards/never-crash.md`
|
||||
## Assets: icons, images, fonts (ABSOLUTE RULE — never violate)
|
||||
- **Never use any iconography, image, artwork, font, sound or other asset from Adobe products** (no Lightroom/Creative Cloud icons, no screenshots, no presets/profiles/LUTs, no UI bitmaps — not even as a temporary placeholder or "reference copy"). Observing Adobe's UI to imitate *layout and behaviour* is allowed; copying or tracing its assets is not.
|
||||
- **This includes Adobe's open-licensed assets**: no Source Sans/Serif/Code or Source Han fonts, no Adobe Fonts, no
|
||||
Adobe-published icon sets, sample photos, colour profiles or LUTs — even when OFL/MIT. The UI font is Inter (OFL).
|
||||
Adobe-published icon sets, sample photos, colour profiles or LUTs — even when OFL/MIT. The UI font is Inter (OFL);
|
||||
Japanese and Chinese fonts come from craft-fonts (below). **One exception (maintainer decision, 2026-10-07): Noto
|
||||
Sans/Serif CJK** (Google-branded, OFL, co-developed with Adobe as Source Han) is allowed via craft-fonts for Chinese
|
||||
text, because nearly every OFL Chinese face derives from it. Use it unmodified under its OFL; this does not open the
|
||||
door to Source Han under Adobe's name or to any other Adobe asset.
|
||||
- Every asset in the repository must be one of: **our own original work** (e.g. icons drawn in code as vectors, procedurally generated demo photos), **public domain / CC0**, **Creative Commons** (CC-BY / CC-BY-SA with attribution honoured), **OFL** (fonts), or **permissive open-source** (MIT/Apache-2.0/BSD/ISC) — or contributed by a person who created the asset and licenses it openly.
|
||||
- **Exception: `docs/brand/`.** The ArtCraft name, wordmark and logos there are ArtCraft Team trademarks, not open source and not covered by LightCraft's MIT OR Apache-2.0 licence (`LICENSE-MIT`, `LICENSE-APACHE`, `NOTICE`); their terms are in `docs/brand/LICENSE-brand.txt`. Use them only unmodified and never redraw, recolour or derive from them.
|
||||
- **Every asset must have an entry in `assets/ATTRIBUTION.md`** (path, title, author/creator, source URL or "original work", licence, date added, modifications) and its licence text when required (e.g. `assets/fonts/OFL-*.txt`). Add the entry in the same commit as the asset. Assets without an attribution entry must not be committed.
|
||||
- **Fonts live in [storytold/craft-fonts](https://github.com/storytold/craft-fonts), never in this repo.** Don't commit
|
||||
font files here (Inter, already in `assets/fonts/`, is the one exception); add new fonts to craft-fonts. LightCraft
|
||||
uses it as the optional build input `CRAFT_FONTS_DIR`: `git clone https://github.com/storytold/craft-fonts ../craft-fonts`
|
||||
then `CRAFT_FONTS_DIR=../craft-fonts cargo run -p lightcraft` (or any cargo/xtask command). `crates/engine/build.rs`
|
||||
embeds the manifest's fonts as `lightcraft_engine::CRAFT_FONTS` (wasm32: BIZ UDPGothic Regular only); `cargo xtask web`
|
||||
ships Noto Sans CJK SC Regular as a separate asset loaded by the UI before startup. The UI
|
||||
(`theme::font_definitions`) and the export watermark renderer use CJK faces (picked by script) as fallbacks after Inter. Unset,
|
||||
`CRAFT_FONTS` is empty: everything builds, tests and runs, but Chinese and Japanese text lack glyphs. Releases always build
|
||||
with it (`release.yml`, `CRAFT_FONTS_REQUIRED=1`) and ship the fonts' OFL licences. Tests that need these fonts skip
|
||||
without it; the FreeBSD CI job runs them with it. Rules: craftrules
|
||||
[`standards/fonts.md`](https://github.com/storytold/craftrules/blob/main/standards/fonts.md).
|
||||
- Icons drawn in code (e.g. `crates/ui-egui/src/icons.rs`) are original work and are recorded in `assets/ATTRIBUTION.md` as such; do not trace them from Adobe icons.
|
||||
- Demo/test images: generated procedurally by `lightcraft-scenes`, or CC0 downloads kept in the gitignored `corpus/` with their source recorded. Screenshots of Adobe apps live only in the gitignored `plan/` and are never committed or published.
|
||||
- **Enforced:** `cargo xtask assets` (in `ci`) fails when an image/icon/font/sound/video/raw/ICC/XMP file is not matched
|
||||
@@ -108,4 +123,25 @@ of it. Full standard: `../craftrules/standards/never-crash.md`
|
||||
## Map of the code
|
||||
`geom`, `color`, `raster`, `tiff` (L0) → `raw`, `codecs`, `meta`, `develop` (L1) → `pipeline` → `catalog` → `engine`
|
||||
→ `ui-egui`, `mcp` (L5) → apps `lightcraft` (desktop), `lightcraft-cli` (render/commands/MCP). `scenes` generates demo
|
||||
photos. `xtask` = tooling (`ci`, `layers`, `assets`, `parity`, `wasm`, `corpus`, `stats`).
|
||||
photos. `xtask` = tooling (`ci`, `layers`, `assets`, `parity`, `wasm`, `corpus`, `stats`). `flake.nix` +
|
||||
`nix/package.nix` = the Nix package (`nix build` builds both binaries with the craft-fonts input, installs the
|
||||
desktop file/icons/AppStream metadata and runs `cargo test --workspace`; `nix develop` = dev shell). Community-maintained and not
|
||||
in CI: it may lag behind the workspace; see README → Quick start.
|
||||
|
||||
## Contributor credits (About window)
|
||||
|
||||
- About ▸ Contributors/Models are compiled into the binary from `contributors/contributors.json`
|
||||
(commit stats; generated, never hand-edit) and `contributors/people.toml` (names people chose for
|
||||
themselves). See `docs/contributors.md`.
|
||||
- **Agents working for a contributor:** when you prepare a PR, check whether your human's GitHub
|
||||
username has a `[people.<username>]` entry in `contributors/people.toml`. If not, ask them once
|
||||
whether they want to be credited by more than their username: a real name, a display name, and/or
|
||||
their public GitHub profile name (`sync_github_name = true`). If yes, add **only their own** entry
|
||||
(copy the template at the top of the file, or run
|
||||
`python3 ../../craftrules/scripts/contributors.py --add-me . --real-name "…" --sync-github-name`)
|
||||
and include it in their PR, committed as them. If no, change nothing: they are credited as
|
||||
`@username` anyway.
|
||||
- Never add, edit, guess or copy anyone else's entry or name (not from git config, commit authors or
|
||||
GitHub profiles). Never hand-edit `contributors.json`.
|
||||
- Maintainers refresh the stats with `python3 ../../craftrules/scripts/contributors.py .` (it also
|
||||
re-verifies who wrote each `people.toml` entry; `--check` only verifies).
|
||||
@@ -1,6 +1,6 @@
|
||||
# LightCraft — instructions for agents
|
||||
|
||||
LightCraft is a clean-room, open-source, pure-Rust photo library + non-destructive raw developer targeting Adobe Lightroom parity (and beyond). Native on macOS, Windows, Linux; web via WASM. Sibling of `../printcraft` (Acrobat), `../photocraft` (Photoshop), `../drawcraft` (Illustrator) and `../filmcraft` (Premiere), with the same conventions.
|
||||
LightCraft is a clean-room, open-source, pure-Rust photo library + non-destructive raw developer targeting Adobe Lightroom parity (and beyond). Native on macOS, Windows, Linux; web via WASM. Sibling of `../pdfcraft` (Acrobat), `../photocraft` (Photoshop), `../vectorcraft` (Illustrator) and `../filmcraft` (Premiere), with the same conventions.
|
||||
|
||||
## Start every session here
|
||||
1. Read `plan/STATUS.md` (current milestone, next unchecked task, blockers).
|
||||
@@ -62,10 +62,25 @@ of it. Full standard: `../craftrules/standards/never-crash.md`
|
||||
## Assets: icons, images, fonts (ABSOLUTE RULE — never violate)
|
||||
- **Never use any iconography, image, artwork, font, sound or other asset from Adobe products** (no Lightroom/Creative Cloud icons, no screenshots, no presets/profiles/LUTs, no UI bitmaps — not even as a temporary placeholder or "reference copy"). Observing Adobe's UI to imitate *layout and behaviour* is allowed; copying or tracing its assets is not.
|
||||
- **This includes Adobe's open-licensed assets**: no Source Sans/Serif/Code or Source Han fonts, no Adobe Fonts, no
|
||||
Adobe-published icon sets, sample photos, colour profiles or LUTs — even when OFL/MIT. The UI font is Inter (OFL).
|
||||
Adobe-published icon sets, sample photos, colour profiles or LUTs — even when OFL/MIT. The UI font is Inter (OFL);
|
||||
Japanese and Chinese fonts come from craft-fonts (below). **One exception (maintainer decision, 2026-10-07): Noto
|
||||
Sans/Serif CJK** (Google-branded, OFL, co-developed with Adobe as Source Han) is allowed via craft-fonts for Chinese
|
||||
text, because nearly every OFL Chinese face derives from it. Use it unmodified under its OFL; this does not open the
|
||||
door to Source Han under Adobe's name or to any other Adobe asset.
|
||||
- Every asset in the repository must be one of: **our own original work** (e.g. icons drawn in code as vectors, procedurally generated demo photos), **public domain / CC0**, **Creative Commons** (CC-BY / CC-BY-SA with attribution honoured), **OFL** (fonts), or **permissive open-source** (MIT/Apache-2.0/BSD/ISC) — or contributed by a person who created the asset and licenses it openly.
|
||||
- **Exception: `docs/brand/`.** The ArtCraft name, wordmark and logos there are ArtCraft Team trademarks, not open source and not covered by LightCraft's MIT OR Apache-2.0 licence (`LICENSE-MIT`, `LICENSE-APACHE`, `NOTICE`); their terms are in `docs/brand/LICENSE-brand.txt`. Use them only unmodified and never redraw, recolour or derive from them.
|
||||
- **Every asset must have an entry in `assets/ATTRIBUTION.md`** (path, title, author/creator, source URL or "original work", licence, date added, modifications) and its licence text when required (e.g. `assets/fonts/OFL-*.txt`). Add the entry in the same commit as the asset. Assets without an attribution entry must not be committed.
|
||||
- **Fonts live in [storytold/craft-fonts](https://github.com/storytold/craft-fonts), never in this repo.** Don't commit
|
||||
font files here (Inter, already in `assets/fonts/`, is the one exception); add new fonts to craft-fonts. LightCraft
|
||||
uses it as the optional build input `CRAFT_FONTS_DIR`: `git clone https://github.com/storytold/craft-fonts ../craft-fonts`
|
||||
then `CRAFT_FONTS_DIR=../craft-fonts cargo run -p lightcraft` (or any cargo/xtask command). `crates/engine/build.rs`
|
||||
embeds the manifest's fonts as `lightcraft_engine::CRAFT_FONTS` (wasm32: BIZ UDPGothic Regular only); `cargo xtask web`
|
||||
ships Noto Sans CJK SC Regular as a separate asset loaded by the UI before startup. The UI
|
||||
(`theme::font_definitions`) and the export watermark renderer use CJK faces (picked by script) as fallbacks after Inter. Unset,
|
||||
`CRAFT_FONTS` is empty: everything builds, tests and runs, but Chinese and Japanese text lack glyphs. Releases always build
|
||||
with it (`release.yml`, `CRAFT_FONTS_REQUIRED=1`) and ship the fonts' OFL licences. Tests that need these fonts skip
|
||||
without it; the FreeBSD CI job runs them with it. Rules: craftrules
|
||||
[`standards/fonts.md`](https://github.com/storytold/craftrules/blob/main/standards/fonts.md).
|
||||
- Icons drawn in code (e.g. `crates/ui-egui/src/icons.rs`) are original work and are recorded in `assets/ATTRIBUTION.md` as such; do not trace them from Adobe icons.
|
||||
- Demo/test images: generated procedurally by `lightcraft-scenes`, or CC0 downloads kept in the gitignored `corpus/` with their source recorded. Screenshots of Adobe apps live only in the gitignored `plan/` and are never committed or published.
|
||||
- **Enforced:** `cargo xtask assets` (in `ci`) fails when an image/icon/font/sound/video/raw/ICC/XMP file is not matched
|
||||
@@ -88,6 +103,9 @@ of it. Full standard: `../craftrules/standards/never-crash.md`
|
||||
the running app). See `docs/mcp.md`. Quick non-UI checks: `lightcraft-cli render in.jpg -o out.jpg --set light.exposure=1`.
|
||||
- Export goes through `lightcraft_engine::export` (one encoder for app, CLI, MCP and web); UI-only commands live in
|
||||
`crates/ui-egui/src/menus.rs`.
|
||||
- Text boxes use `crates/ui-egui/src/text_field.rs` (`TextField`), not `egui::TextEdit`: it gives every field the
|
||||
Cut / Copy / Paste / Select All menu, Esc restoring the text, and reports how an edit ended. Give each a widget id
|
||||
that is unique on screen. Older fields still on `TextEdit` move over as they are touched.
|
||||
- Shell gotcha: `mv`/`cp` are aliased interactive here — use `/bin/mv -f` / `/bin/cp -f`.
|
||||
- Parallel agents: separate git worktrees and `CARGO_TARGET_DIR=target/agent-<name>`; each agent uses its **own control port** (pick one in 18000–19999, never the default 7980) and its own scratch subfolder (`<scratch>/<agent-name>/`) — never `rm -rf` shared paths; delete your target dir when done (disk is shared); keep every `Cargo.toml` valid at all times (the `crates/*` glob means one broken manifest breaks everyone).
|
||||
- Test corpora: `cargo xtask corpus --download` into `corpus/` (gitignored, CC0 only). Never commit media.
|
||||
@@ -108,4 +126,7 @@ of it. Full standard: `../craftrules/standards/never-crash.md`
|
||||
## Map of the code
|
||||
`geom`, `color`, `raster`, `tiff` (L0) → `raw`, `codecs`, `meta`, `develop` (L1) → `pipeline` → `catalog` → `engine`
|
||||
→ `ui-egui`, `mcp` (L5) → apps `lightcraft` (desktop), `lightcraft-cli` (render/commands/MCP). `scenes` generates demo
|
||||
photos. `xtask` = tooling (`ci`, `layers`, `assets`, `parity`, `wasm`, `corpus`, `stats`).
|
||||
photos. `xtask` = tooling (`ci`, `layers`, `assets`, `parity`, `wasm`, `corpus`, `stats`). `flake.nix` +
|
||||
`nix/package.nix` = the Nix package (`nix build` builds both binaries with the craft-fonts input, installs the
|
||||
desktop file/icons/AppStream metadata and runs `cargo test --workspace`; `nix develop` = dev shell). Community-maintained and not
|
||||
in CI: it may lag behind the workspace; see README → Quick start.
|
||||
Generated
+1840
-32
File diff suppressed because it is too large.
Load diff
+16
-1
@@ -3,7 +3,7 @@ resolver = "3"
|
||||
members = ["crates/*", "apps/*", "xtask"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.2.0"
|
||||
version = "0.5.0"
|
||||
edition = "2024"
|
||||
license = "MIT OR Apache-2.0"
|
||||
rust-version = "1.90"
|
||||
@@ -11,7 +11,10 @@ repository = "https://github.com/storytold/lightcraft"
|
||||
|
||||
[workspace.dependencies]
|
||||
# internal
|
||||
lightcraft-denoise-core = { path = "crates/denoise-core" }
|
||||
lightcraft-denoise = { path = "crates/denoise", default-features = false }
|
||||
lightcraft-geom = { path = "crates/geom" }
|
||||
lightcraft-faces = { path = "crates/faces" }
|
||||
lightcraft-sysmem = { path = "crates/sysmem" }
|
||||
lightcraft-color = { path = "crates/color" }
|
||||
lightcraft-raster = { path = "crates/raster" }
|
||||
@@ -29,9 +32,13 @@ lightcraft-merge = { path = "crates/merge" }
|
||||
lightcraft-engine = { path = "crates/engine" }
|
||||
lightcraft-ui-egui = { path = "crates/ui-egui" }
|
||||
lightcraft-mcp = { path = "crates/mcp" }
|
||||
lightcraft-fetch = { path = "crates/fetch" }
|
||||
lightcraft-heif = { path = "crates/heif" }
|
||||
lightcraft-segment = { path = "crates/segment" }
|
||||
|
||||
# foundation
|
||||
thiserror = "2"
|
||||
sha2 = "0.10"
|
||||
serde = { version = "1", features = ["derive", "rc"] }
|
||||
serde_json = { version = "1", features = ["float_roundtrip"] }
|
||||
log = "0.4"
|
||||
@@ -59,9 +66,17 @@ tiff = "0.11"
|
||||
image-webp = "0.2"
|
||||
moxcms = { version = "0.9", default-features = false, features = ["lut"] }
|
||||
jxl-oxide = { version = "0.12", default-features = false }
|
||||
zune-jpegxl = { version = "0.5", default-features = false, features = ["std"] } # MIT/Apache-2.0/Zlib lossless JXL encoder (test fixtures only)
|
||||
ravif = { version = "0.13", default-features = false, features = ["threading"] } # no "asm": it needs nasm (non-Rust)
|
||||
# HDR AVIF (PQ): ravif fixes the sRGB transfer curve, so HDR files drive its rav1e + avif-serialize directly
|
||||
rav1e = { version = "0.8.1", default-features = false, features = ["threading"] }
|
||||
avif-serialize = "0.8.6"
|
||||
|
||||
# gpu (crates/gpu; native backends only, no GL: everything stays pure Rust)
|
||||
# SAM 3 inference (lightcraft-segment). 0.9.2: later releases pull `tokenizers` with the
|
||||
# Oniguruma C library into candle-core, and the product is pure Rust.
|
||||
candle-core = { version = "=0.9.2", default-features = false }
|
||||
candle-nn = { version = "=0.9.2", default-features = false }
|
||||
wgpu = { version = "30", default-features = false, features = ["std", "parking_lot", "metal", "vulkan", "dx12", "wgsl"] }
|
||||
pollster = "1"
|
||||
|
||||
|
||||
@@ -15,6 +15,29 @@ list (author, source, license) is in assets/ATTRIBUTION.md.
|
||||
Inter (assets/fonts/Inter-*.ttf) — SIL Open Font License 1.1. See assets/fonts/OFL-Inter.txt.
|
||||
Showcase photographs by Ansel Adams, Dorothea Lange and NASA (docs/images/) — public domain. See assets/ATTRIBUTION.md.
|
||||
|
||||
BIZ UDMincho: Copyright 2022 The BIZ UDMincho Project Authors
|
||||
(https://github.com/googlefonts/morisawa-biz-ud-mincho). SIL Open Font License 1.1;
|
||||
see assets/fonts/OFL-BIZUDMincho.txt.
|
||||
crates/segment (lightcraft-segment) ports the SAM 3 model code of Hugging Face Transformers
|
||||
(https://github.com/huggingface/transformers, src/transformers/models/sam3 and sam3_tracker),
|
||||
Copyright The HuggingFace Team and Meta Platforms, Inc., licensed under the Apache License 2.0.
|
||||
The port was modified by the LightCraft contributors (2026): translated from Python/PyTorch to
|
||||
Rust on candle and restructured; each ported file says so. Because of this, crates/segment is
|
||||
licensed under the Apache License 2.0 only (LICENSE-APACHE), not under the MIT option.
|
||||
|
||||
The SAM 3 weights are not part of LightCraft and are never distributed with it. A user downloads
|
||||
the facebook/sam3 checkpoint only when they choose to (in the app, or by hand), under Meta's SAM
|
||||
License (https://github.com/facebookresearch/sam3/blob/main/LICENSE), a separate, non-OSI
|
||||
licence. See docs/ai-masks.md.
|
||||
|
||||
AI denoise weights are not part of LightCraft and are never distributed with it.
|
||||
The default build offers user-supplied models only. The optional rawnind-model
|
||||
feature offers a separately downloaded RawNIND UtNet2 Bayer checkpoint under
|
||||
GPL-3.0 (https://www.gnu.org/licenses/gpl-3.0.html), after the user accepts its terms.
|
||||
Commercial use is subject to those terms, including applicable source, licence and
|
||||
notice obligations on redistribution. Enabling that offer in releases remains a
|
||||
maintainer decision. The ONNX reader and CPU/GPU executor are our Rust implementation;
|
||||
no darktable, RawSpeed, LibRaw or other GPL decoder/runtime code is included.
|
||||
|
||||
Builds made with craft-fonts (the optional CRAFT_FONTS_DIR build input; all official releases)
|
||||
also embed the fonts listed in https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md
|
||||
(the Japanese fonts BIZ UDPGothic, BIZ UDMincho and Shippori Mincho; the web build embeds only BIZ
|
||||
UDPGothic Regular), under the SIL Open Font License 1.1. Release packages include each font's
|
||||
licence as OFL-<family>.txt.
|
||||
@@ -59,6 +59,7 @@
|
||||
<a href="#feature-status">Status</a> ·
|
||||
<a href="#quick-start">Quick start</a> ·
|
||||
<a href="ROADMAP.md">Roadmap</a> ·
|
||||
<a href="#downloads">Downloads</a> ·
|
||||
<a href="#the-crafting-apps">Crafting Apps</a>
|
||||
</p>
|
||||
|
||||
@@ -200,7 +201,7 @@ A library that stays out of your way: **All Photos**, **Recently Added**, **Pick
|
||||
**Folders**, and **Recently Deleted**. Rate with <kbd>0</kbd>–<kbd>5</kbd>, flag with <kbd>P</kbd> / <kbd>X</kbd> /
|
||||
<kbd>U</kbd>, colour-label with <kbd>6</kbd>–<kbd>9</kbd>. Search understands fields:
|
||||
`rating:>3 flag:pick iso:>800 camera:x2 date:2026-04 keyword:mountains`. Every view sorts by capture date,
|
||||
import date, edit date, name, rating or size. The justified **Photo Grid** and **Square Grid** views are virtualized,
|
||||
import date, edit date, name, rating, size or at random (a stable shuffle; View → Sort → Reshuffle for a new one). The justified **Photo Grid** and **Square Grid** views are virtualized,
|
||||
so they stay smooth whether you have forty photos or forty thousand.
|
||||
|
||||
<table>
|
||||
@@ -233,11 +234,11 @@ lightcraft --control 7980 ~/Pictures/trip
|
||||
{"method": "ui.screenshot", "params": {"path": "after.png"}}
|
||||
```
|
||||
|
||||
- **91 engine commands** and **48 UI commands.** List them all with `engine.commands`; read every slider's range,
|
||||
default and current value with `develop.controls`.
|
||||
- **MCP server.** `lightcraft-cli mcp` gives Claude (or any MCP client) ~100 tools: import, query, develop, mask,
|
||||
render (returned as an image), export. It runs headless, or attached to the running app with screenshots, clicks
|
||||
and gestures. See [docs/mcp.md](docs/mcp.md).
|
||||
- **Command registry.** `engine.commands` lists the available commands; `develop.controls` lists every slider's
|
||||
range, default and current value.
|
||||
- **MCP server.** `lightcraft-cli mcp` exposes the command registry to Claude (or any MCP client), alongside
|
||||
helpers for import, query, develop, mask, render (returned as an image) and export. It runs headless, or attached
|
||||
to the running app with screenshots, clicks and gestures. See [docs/mcp.md](docs/mcp.md).
|
||||
|
||||
```sh
|
||||
cargo build --release -p lightcraft-cli
|
||||
@@ -248,6 +249,20 @@ lightcraft --control 7980 ~/Pictures/trip
|
||||
path=out.jpg longEdge=2048` runs any chain of commands (headless, on a saved library, or against the running app)
|
||||
and prints one JSON result per command; `lightcraft-cli render in.dng -o out.jpg --set light.exposure=0.7 --preset …`.
|
||||
- **Undo for everything**, including agent actions: a slider drag (or a scripted burst of updates) is one undo step.
|
||||
- **Contact sheets (native app / CLI):** select photos, then **File → Contact Sheet PDF…**. Choose A4 or Letter, landscape,
|
||||
rows/columns and filename captions; save a paginated PDF and print it from your PDF viewer. Exports use the
|
||||
current edits, fit each photo without further cropping, and render at 150 dpi in sRGB. The background export
|
||||
supports cancellation and protects originals and XMP sidecars. From the CLI:
|
||||
|
||||
```sh
|
||||
lightcraft-cli run --import ~/Pictures/shoot library.selectAll export.contactSheet path=Contact.pdf paper=a4 columns=3 rows=4 captions=true
|
||||
```
|
||||
|
||||
`export.contactSheet` also accepts explicit `ids`, `landscape=true`, and `paper=letter`. It replaces an existing
|
||||
output only after the entire PDF succeeds. Limits: 1,000 photos, 100 pages, 256 MiB; captions are rasterized
|
||||
with the same font coverage as watermarks (CJK requires craft-fonts), with long names shortened in the middle.
|
||||
This is a contact-sheet workflow,
|
||||
not a printer-driver or custom picture-package layout editor.
|
||||
- **Every widget is addressable** (`ui.widgets`) and clickable by name, so agents operate the real UI, not a
|
||||
side door.
|
||||
- The screenshots in this README were produced end to end by the [`docs/showcase/`](docs/showcase/) scripts.
|
||||
@@ -257,9 +272,10 @@ lightcraft --control 7980 ~/Pictures/trip
|
||||
|
||||
## Fast, native, private
|
||||
|
||||
- **Pure Rust, no C.** Our own RAW decoders (DNG, Canon CR2, Sony ARW, Nikon NEF, Fujifilm RAF incl. X-Trans,
|
||||
Panasonic RW2, Pentax PEF, Olympus ORF), our own colour science, our own pipeline. JPEG, PNG, TIFF, WebP, PSD
|
||||
composites and JPEG XL open today.
|
||||
- **Pure Rust, no C.** Our own RAW decoders (DNG, Canon CR2/CR3, Sony ARW, Nikon NEF, Fujifilm RAF incl. X-Trans,
|
||||
Panasonic RW2 / Leica RWL, Pentax PEF, Olympus ORF), our own colour science, our own pipeline. JPEG, PNG, TIFF, WebP,
|
||||
PSD composites and JPEG XL open today; HEIC/HEIF photos too with `--features heif` (HEVC is a build-time choice,
|
||||
as in PhotoCraft).
|
||||
- **Scene-referred & wide-gamut.** Linear Rec.2020 float internally, Bradford-adapted white balance, gamut mapping
|
||||
instead of clipping, a filmic shoulder for raw and pixel-exact pass-through for JPEGs you haven't touched.
|
||||
- **Resolution-independent edits.** Radii and brush sizes are relative to the image, so a 400 px preview, your
|
||||
@@ -267,7 +283,8 @@ lightcraft --control 7980 ~/Pictures/trip
|
||||
- **GPU-accelerated, CPU-exact.** The whole develop pipeline runs as wgpu compute kernels (Metal / Vulkan / DX12),
|
||||
checked against the CPU pipeline to within 1/255. On a 24 MP raw (Apple M4 Pro): a slider update re-renders in
|
||||
~4 ms, a cold 2.5 MP loupe in ~30 ms, and a full-size export in ~0.3 s including a parallel JPEG encode.
|
||||
Without a GPU the same pipeline runs on all CPU cores, redoing only the stages a slider affects.
|
||||
Without a GPU the same pipeline runs on all CPU cores, redoing only the stages a slider affects. Batch exports
|
||||
render several photos side by side, so one's decode and encode overlap another's render.
|
||||
- **Instant culling.** Opening a raw shows its embedded camera preview or cached render within ~0.1 s while the
|
||||
full render follows (~0.2–0.5 s for 24 MP). The next and previous photos are prepared in the background, so stepping
|
||||
through a shoot takes ~50 ms per photo.
|
||||
@@ -287,10 +304,10 @@ LightCraft is young and moving fast. **Where we honestly stand** (details in the
|
||||
- **As a day-to-day Lightroom replacement we're nearer 60–70%.** It's great for JPEG/DNG and most Nikon / Sony /
|
||||
older-Canon raws on one machine.
|
||||
- **The biggest gaps:**
|
||||
- **camera colour calibration:** raws other than DNG develop with a neutral colour matrix today, so colour is muted;
|
||||
- **CR3 and compressed Fujifilm / Olympus raws:** these open as embedded previews only;
|
||||
- **camera colour calibration:** Sony, Nikon, Panasonic, Fujifilm and Canon CR3 raws have guarded estimates from their camera JPEGs, with built-in ILCE-7CR, ILCE-7M4, X-H2S and X-T4 profiles; measured calibration is missing, and other raws or rejected fits retain a neutral matrix;
|
||||
- **compressed Olympus raws and unsupported CR3 variants:** these use embedded JPEG previews when present. Fujifilm lossless/lossy compressed RAF now decodes sensor data; [verification and existing-library reload instructions](docs/raf-compression.md);
|
||||
- **AI masks and denoise:** subject and sky selection are classical heuristics;
|
||||
- **HDR, video and the Classic Print / Book / Map modules.**
|
||||
- **on-screen HDR display, video and the Classic Print / Book / Map modules.**
|
||||
- **What's next:** see [where we're going](ROADMAP.md#where-were-going).
|
||||
|
||||
| Area | Status |
|
||||
@@ -301,20 +318,21 @@ LightCraft is young and moving fast. **Where we honestly stand** (details in the
|
||||
| Masking: brush, linear, radial, luminance/colour range, add/subtract/intersect | ✅ (AI subject/sky use classical heuristics for now) |
|
||||
| Crop, straighten tool + auto straighten, flip, rotate, aspect ratios, overlays | ✅ |
|
||||
| Profiles (Color, Neutral, Vivid, Landscape, Portrait, Monochrome: our own looks), presets, versions, history, copy/paste/sync settings | ✅ |
|
||||
| Camera colour: DNG files use their own matrices | ✅ · our own calibration for other raws ⬜ (top priority; neutral fallback today) |
|
||||
| Camera colour: DNG files use their own matrices | ✅ DNG · 🟡 own Sony/Fujifilm profiles; measured calibration database missing |
|
||||
| Native macOS menu bar (generated from the command registry), control channel + every widget addressable, headless UI snapshots | ✅ |
|
||||
| RAW: DNG, CR2, ARW, NEF (uncompressed + lossless/lossy compressed), Fujifilm RAF (uncompressed, Bayer + X-Trans), Panasonic RW2, Pentax PEF, Olympus ORF (uncompressed); embedded previews for every format incl. CR3 | ✅ · CR3, compressed RAF/ORF decode ⬜ |
|
||||
| RAW: DNG, CR2, CR3 (lossless CRX Bayer and version 0x100/0x200 C-RAW), ARW, NEF (uncompressed + lossless/lossy compressed), Fujifilm RAF (uncompressed + lossless/lossy compressed, Bayer + X-Trans), Panasonic RW2 / Leica RWL / Panasonic RAW (every raw format, DMC-LX1 to DC-S1RM2), Pentax PEF, Olympus ORF (uncompressed), Samsung SRW (uncompressed); embedded previews for every format incl. CR3 | 🟡 · CR3, compressed ORF and compressed SRW decode ⬜ |
|
||||
| Detail: sharpening, luminance + colour noise reduction | ✅ · AI Denoise, Super Resolution ⬜ |
|
||||
| Remove / Heal / Clone spots (auto source), Visualize Spots, Red Eye and Pet Eye (auto pupil detection, catchlight) | ✅ · content-aware fill, spot pin editing 🚧 |
|
||||
| Export: JPEG / PNG / TIFF / WebP / AVIF / DNG / original, sizing, file-size limit, output sharpening, naming templates, batch, metadata policy, text or image watermark | ✅ · HDR export ⬜ |
|
||||
| Export: JPEG / PNG / TIFF / WebP / AVIF / DNG / original, sizing, file-size limit, output sharpening, naming templates, batch, metadata policy, text or image watermark | ✅ |
|
||||
| HDR: HDR editing with a headroom limit, SDR rendition, Visualize HDR; export as ISO 21496-1 gain map JPEG, PQ AVIF or 32-bit float TIFF | ✅ · HDR display ⬜ |
|
||||
| Library persistence (crash-safe op log + snapshots, background compaction, failed saves reported), disk thumbnail cache | ✅ |
|
||||
| Import: Add in place / Copy / Move, rename and folder templates, devices, duplicate detection, watched folders; Local folder browsing | ✅ |
|
||||
| MCP server (headless or live app, persistent libraries), CLI, control channel | ✅ |
|
||||
| XMP sidecars (read/write, auto-write), reading `crs:` develop settings, preset files (`.lcpreset`, XMP presets) | ✅ |
|
||||
| Optics (distortion, vignetting, auto + manual CA, defringe, DNG-embedded lens corrections), Geometry (transforms, Constrain Crop), Upright (Auto/Level/Vertical/Full/Guided) | ✅ · camera lens profiles (our own) ⬜ |
|
||||
| Optics (distortion, vignetting, auto + manual CA, defringe, lens corrections embedded in DNG files and Panasonic / Leica RW2 / RWL distortion data), Geometry (transforms, Constrain Crop), Upright (Auto/Level/Vertical/Full/Guided) | ✅ · camera lens profiles (our own) ⬜ |
|
||||
| Photo Merge: HDR (auto-align, deghost), Panorama (spherical/cylindrical/perspective, boundary warp, auto crop), HDR Panorama → DNG | ✅ |
|
||||
| GPU pipeline (wgpu compute, CPU-exact within 1/255), CPU fallback on device limits / errors | ✅ · WebGPU in the browser 🚧 |
|
||||
| AI: segmentation masks, AI denoise, super resolution, faces; HDR editing; video | ⬜ (see [roadmap](ROADMAP.md#where-were-going)) |
|
||||
| AI: segmentation masks, AI denoise, super resolution, faces; video | ⬜ (see [roadmap](ROADMAP.md#where-were-going)) |
|
||||
| Web build (same UI in the browser via WASM): persistent library in OPFS/IndexedDB, Web Worker rendering, export downloads | ✅ · WebGPU, Safari/Firefox testing 🚧 |
|
||||
|
||||
<sub>✅ works today · 🚧 in progress · ⬜ not started</sub>
|
||||
@@ -334,9 +352,69 @@ cargo run --release -p lightcraft-cli -- render photo.jpg -o out.jpg --set light
|
||||
cargo xtask ci # fmt, clippy, tests, layering, wasm checks
|
||||
```
|
||||
|
||||
CI defaults to line-table debug information and one build job per 1.5 GB of RAM
|
||||
available when it starts (at most one per CPU; 4 if memory can't be read), so an
|
||||
8 GB machine with 6 GB free builds with 4, and a busy machine with little free
|
||||
builds with fewer. Test threads follow, capped at 4. Full-debuginfo linkers, one
|
||||
per CPU, otherwise exhaust RAM before any test runs. GPU coverage is unchanged. Explicit
|
||||
`CARGO_PROFILE_DEV_DEBUG`, `CARGO_BUILD_JOBS` and `RUST_TEST_THREADS` settings
|
||||
override these defaults. Ordinary development commands keep their own settings.
|
||||
|
||||
**Chinese and Japanese text** need the shared font repo, an optional build input (official releases always include it):
|
||||
|
||||
```sh
|
||||
git clone https://github.com/storytold/craft-fonts ../craft-fonts
|
||||
CRAFT_FONTS_DIR=../craft-fonts cargo run --release -p lightcraft
|
||||
```
|
||||
|
||||
Without it LightCraft builds and runs the same, but Chinese and Japanese text have no glyphs. Fonts are never committed to this
|
||||
repo; see [craftrules `standards/fonts.md`](https://github.com/storytold/craftrules/blob/main/standards/fonts.md).
|
||||
|
||||
**Interface language:** **Edit → Language** (English, 简体中文, 繁體中文(台灣), 日本語, Português (Brasil), Español, Deutsch, Русский, Français) or **Settings → General →
|
||||
Language**; the choice applies immediately and persists. See [docs/localization.md](docs/localization.md).
|
||||
|
||||
**Logs:** the desktop app writes its log to standard error and to `logs/lightcraft.log` in its settings folder
|
||||
(Linux `$XDG_CONFIG_HOME/lightcraft/logs/`, by default `~/.config/lightcraft/logs/`; macOS
|
||||
`~/Library/Application Support/LightCraft/logs/`; Windows `%APPDATA%\LightCraft\logs\`), never in the library. A
|
||||
launch from a desktop menu or the Dock has no terminal, so attach this file to a bug report (**Help ▸ Open Log
|
||||
Folder** shows it in the file manager). Each start moves the
|
||||
previous log to `lightcraft.1.log` and that one to `lightcraft.2.log`, so the log of a run that crashed survives the
|
||||
next start; the file stops growing at 16 MiB, `--version` and `--help` write none, and runs with
|
||||
`LIGHTCRAFT_NO_PREFS` log to standard error only. By default LightCraft's own crates log at `info` and everything else
|
||||
at `warn`. `LIGHTCRAFT_LOG=info` or `debug` works as before (that level for LightCraft's own crates, warnings and
|
||||
errors from the rest; any other value: warnings and errors only) and wins over `RUST_LOG`, which otherwise replaces
|
||||
the default with env_logger-style directives such as `RUST_LOG=debug` or `RUST_LOG=warn,lightcraft_pipeline=trace` (a
|
||||
directive ending in `*` covers every target starting with it, as in `lightcraft*=debug`). Panics are recorded there
|
||||
too, and still in `lightcraft-panics.log` in the temp folder. The logger is `apps/lightcraft/src/logging.rs`.
|
||||
|
||||
The web build needs the `wasm32-unknown-unknown` target and the matching `wasm-bindgen` CLI
|
||||
(`cargo xtask web` prints the exact install command); see [docs/web.md](docs/web.md).
|
||||
|
||||
**Nix** builds the desktop app and `lightcraft-cli` (the Nix build always includes the craft-fonts input, so Japanese
|
||||
text has glyphs):
|
||||
|
||||
```sh
|
||||
nix run github:storytold/lightcraft # the desktop app
|
||||
nix build github:storytold/lightcraft # → ./result/bin/{lightcraft,lightcraft-cli}
|
||||
nix develop github:storytold/lightcraft # rust toolchain + native deps + fonts
|
||||
```
|
||||
|
||||
In a flake configuration (NixOS, home-manager, nix-darwin):
|
||||
|
||||
```nix
|
||||
# flake.nix
|
||||
inputs.lightcraft.url = "github:storytold/lightcraft";
|
||||
# optional: build against your own nixpkgs instead of the one LightCraft pins
|
||||
# inputs.lightcraft.inputs.nixpkgs.follows = "nixpkgs";
|
||||
|
||||
# then, in a NixOS or home-manager module (where `inputs` is in scope):
|
||||
nixpkgs.overlays = [ inputs.lightcraft.overlays.default ]; # makes `pkgs.lightcraft` available
|
||||
environment.systemPackages = [ pkgs.lightcraft ]; # home-manager: home.packages = [ pkgs.lightcraft ];
|
||||
```
|
||||
|
||||
`nix build` installs the same desktop file, hicolor icons and AppStream metadata as the .deb/.rpm, and runs
|
||||
`cargo test --workspace` as its check phase (skip it with `pkgs.lightcraft.overrideAttrs { doCheck = false; }`).
|
||||
|
||||
**Keyboard:** <kbd>G</kbd> grid · <kbd>D</kbd> detail · <kbd>E</kbd> edit · <kbd>C</kbd> crop · <kbd>M</kbd> masking ·
|
||||
<kbd>Shift</kbd>+<kbd>P</kbd> presets · <kbd>\\</kbd> original · <kbd>Y</kbd> before/after · <kbd>Z</kbd> zoom ·
|
||||
<kbd>J</kbd> clipping · <kbd>⌘Z</kbd> undo · <kbd>⌘/</kbd> all shortcuts.
|
||||
@@ -356,7 +434,8 @@ Humans and agents follow the same rules, so read [AGENTS.md](AGENTS.md) first. T
|
||||
work from public specs and black-box observation.
|
||||
- **No Adobe assets, ever:** no icons, screenshots, presets, profiles, LUTs or fonts from Adobe products. Every
|
||||
image, icon and font in the repo is original, public domain, Creative Commons, OFL or permissively licensed, and has
|
||||
an entry in [assets/ATTRIBUTION.md](assets/ATTRIBUTION.md) added in the same commit.
|
||||
an entry in [assets/ATTRIBUTION.md](assets/ATTRIBUTION.md) added in the same commit. New fonts go to
|
||||
[storytold/craft-fonts](https://github.com/storytold/craft-fonts), not here.
|
||||
- **Pure Rust**, enforced crate layering, everything is a command, and `cargo xtask ci` green before every commit
|
||||
(one task id per commit).
|
||||
- **Never crash.** Non-test code returns errors instead of panicking: no `unwrap()`, `expect()`, `panic!` or
|
||||
@@ -367,6 +446,55 @@ Questions, ideas or a bug you'd like to talk through first? Bring them to [Disco
|
||||
|
||||
<br>
|
||||
|
||||
## Downloads
|
||||
|
||||
**New to LightCraft?** Download it from the [LightCraft page on getartcraft.com](https://getartcraft.com/apps/lightcraft). That's the easiest way to install it.
|
||||
|
||||
**Want a specific build or format?** On GitHub, the [latest release](https://github.com/storytold/lightcraft/releases/latest) has every build listed below, and [all releases](https://github.com/storytold/lightcraft/releases) has earlier versions and their notes. `<ver>` in the file names is the version number, and `SHA256SUMS.txt` lists a checksum for every file.
|
||||
|
||||
### Windows
|
||||
|
||||
| Build | Installer | Portable |
|
||||
|---|---|---|
|
||||
| x64 (64-bit Intel/AMD) | `lightcraft-<ver>-windows-x64.msi` | `lightcraft-<ver>-windows-x64-portable.zip` |
|
||||
| arm64 (Snapdragon and other ARM PCs) | `lightcraft-<ver>-windows-arm64.msi` | `lightcraft-<ver>-windows-arm64-portable.zip` |
|
||||
| x86 (32-bit) | `lightcraft-<ver>-windows-x86.msi` | `lightcraft-<ver>-windows-x86-portable.zip` |
|
||||
|
||||
Installers and executables are code-signed. The installer asks where to install (`C:\Program Files\LightCraft` by
|
||||
default; upgrades stay in the folder you chose) and ends on a page confirming LightCraft was installed, with an
|
||||
option to start it. For unattended installs: `msiexec /i lightcraft-<ver>-windows-x64.msi /qn INSTALLFOLDER="D:\Apps\LightCraft\"`.
|
||||
|
||||
### macOS
|
||||
|
||||
| Build | File | Notes |
|
||||
|---|---|---|
|
||||
| App, universal (Apple silicon + Intel) | `lightcraft-<ver>-macos-universal.dmg` | Signed and notarized |
|
||||
| Command-line tool, universal | `lightcraft-cli-<ver>-macos-universal.zip` | Signed and notarized |
|
||||
|
||||
### Linux
|
||||
|
||||
| Format | x86_64 | aarch64 (ARM64) | Notes |
|
||||
|---|---|---|---|
|
||||
| AppImage | `lightcraft-<ver>-linux-x86_64.AppImage` | `lightcraft-<ver>-linux-aarch64.AppImage` | Runs anywhere; updates itself with [AppImageUpdate](https://github.com/AppImageCommunity/AppImageUpdate) (`.zsync` files) |
|
||||
| Flatpak | `lightcraft-<ver>-linux-x86_64.flatpak` | `lightcraft-<ver>-linux-aarch64.flatpak` | Sandboxed; `flatpak install --user <file>` |
|
||||
| Debian/Ubuntu | `lightcraft-<ver>-linux-x86_64.deb` | `lightcraft-<ver>-linux-aarch64.deb` | |
|
||||
| Fedora/RHEL/openSUSE | `lightcraft-<ver>-linux-x86_64.rpm` | `lightcraft-<ver>-linux-aarch64.rpm` | |
|
||||
| Tarball | `lightcraft-<ver>-linux-x86_64.tar.gz` | `lightcraft-<ver>-linux-aarch64.tar.gz` | Unpack anywhere |
|
||||
|
||||
### FreeBSD
|
||||
|
||||
| Build | File |
|
||||
|---|---|
|
||||
| x86_64 | `lightcraft-<ver>-freebsd-x86_64.tar.gz` |
|
||||
|
||||
### Web (WebAssembly)
|
||||
|
||||
| Build | File | Notes |
|
||||
|---|---|---|
|
||||
| Static site | `lightcraft-web-<ver>.zip` | Runs in a modern browser; host it on any static server |
|
||||
|
||||
<br>
|
||||
|
||||
## The Crafting Apps
|
||||
|
||||
LightCraft is one of the **Crafting Apps**: free, open-source creative tools from the
|
||||
@@ -379,7 +507,7 @@ stand on its own.
|
||||
| <img src="https://raw.githubusercontent.com/storytold/vectorcraft/main/assets/app-icon/hicolor/64x64/apps/ai.storyteller.vectorcraft.png" alt="" width="32" height="32"> | **VectorCraft** | Vector illustration | [GitHub](https://github.com/storytold/vectorcraft) | [Website](https://getartcraft.com/apps/vectorcraft) |
|
||||
| <img src="https://raw.githubusercontent.com/storytold/filmcraft/main/assets/app-icon/hicolor/64x64/apps/ai.storyteller.filmcraft.png" alt="" width="32" height="32"> | **FilmCraft** | Video editing, color and sound | [GitHub](https://github.com/storytold/filmcraft) | [Website](https://getartcraft.com/apps/filmcraft) |
|
||||
| <img src="https://raw.githubusercontent.com/storytold/lightcraft/main/assets/app-icon/hicolor/64x64/apps/ai.storyteller.lightcraft.png" alt="" width="32" height="32"> | **LightCraft** | **Photo library and raw development · you are here** | [GitHub](https://github.com/storytold/lightcraft) | [Website](https://getartcraft.com/apps/lightcraft) |
|
||||
| <img src="https://raw.githubusercontent.com/storytold/printcraft/main/assets/app-icon/hicolor/64x64/apps/ai.storyteller.printcraft.png" alt="" width="32" height="32"> | **PrintCraft** | Reading, organizing and protecting PDFs | [GitHub](https://github.com/storytold/printcraft) | [Website](https://getartcraft.com/apps/printcraft) |
|
||||
| <img src="https://raw.githubusercontent.com/storytold/pdfcraft/main/assets/app-icon/hicolor/64x64/apps/ai.storyteller.pdfcraft.png" alt="" width="32" height="32"> | **PdfCraft** | Reading, organizing and protecting PDFs | [GitHub](https://github.com/storytold/pdfcraft) | [Website](https://getartcraft.com/apps/pdfcraft) |
|
||||
| <img src="https://raw.githubusercontent.com/storytold/effectcraft/main/assets/app-icon/hicolor/64x64/apps/ai.storyteller.effectcraft.png" alt="" width="32" height="32"> | **EffectCraft** | Motion graphics and visual effects | [GitHub](https://github.com/storytold/effectcraft) | [Website](https://getartcraft.com/apps/effectcraft) |
|
||||
| <img src="https://raw.githubusercontent.com/storytold/designcraft/main/assets/app-icon/hicolor/64x64/apps/ai.storyteller.designcraft.png" alt="" width="32" height="32"> | **DesignCraft** | Page layout and publishing | [GitHub](https://github.com/storytold/designcraft) | [Website](https://getartcraft.com/apps/designcraft) |
|
||||
|
||||
@@ -420,7 +548,9 @@ with its author, source and license in [assets/ATTRIBUTION.md](assets/ATTRIBUTIO
|
||||
Showcase photographs are public-domain works, used via Wikimedia Commons: Ansel Adams, *The Tetons and the Snake River*
|
||||
(1942, U.S. National Archives); Dorothea Lange, *Migrant Mother* (1936, Library of Congress); Bill Anders / NASA,
|
||||
*Earthrise* (1968); NASA, *The Blue Marble* (1972). The demo library is procedurally generated by LightCraft. UI font:
|
||||
Inter (SIL OFL 1.1). All icons are original.
|
||||
Inter (SIL OFL 1.1). Builds made with [craft-fonts](https://github.com/storytold/craft-fonts) (all official releases)
|
||||
also embed its Chinese and Japanese fonts (Noto Sans CJK SC, BIZ UDPGothic, BIZ UDMincho, Shippori Mincho; SIL OFL 1.1), listed in its
|
||||
[ATTRIBUTION.md](https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md). All icons are original.
|
||||
|
||||
The ArtCraft name, wordmark and logos in [`docs/brand/`](docs/brand/) are trademarks of the
|
||||
ArtCraft Team and are not covered by this license. They may be used only unmodified, and only as
|
||||
@@ -433,3 +563,7 @@ Forks and modified versions must remove them.
|
||||
<a href="https://getartcraft.com/"><img alt="ArtCraft" src="docs/brand/artcraft-mark.svg" width="28"></a><br>
|
||||
<sub>Made by the <a href="https://getartcraft.com/">ArtCraft</a> team and community.</sub>
|
||||
</p>
|
||||
|
||||
## Star history
|
||||
|
||||
[](https://www.star-history.com/?repos=storytold%2Flightcraft&type=date&legend=top-left)
|
||||
+84
-45
@@ -6,11 +6,11 @@ projects (DrawCraft reached its first four milestones in ≈ 4½ h) and are revi
|
||||
|
||||
## Where we stand
|
||||
|
||||
*Honest assessment, 2026-10-05. Agents: read this before picking work. The checklist in
|
||||
*Honest assessment, 2026-10-08. Agents: read this before picking work. The checklist in
|
||||
[`docs/parity.md`](docs/parity.md) counts features that **exist**; this section is about whether a photographer can
|
||||
**switch** from Lightroom. Update it when a gap below closes.*
|
||||
|
||||
**In one line:** the checklist says **79%** (P0 98.5%, P1 95.9%, P2 39%), but measured by whether a working
|
||||
**In one line:** the checklist says **79.7%** (P0 98.2%, P1 95.6%, P2 41.6%), but measured by whether a working
|
||||
photographer could replace Lightroom without noticing, we are at roughly **60–70%**. The remaining gap is mostly
|
||||
**quality of results and camera coverage**, not missing buttons.
|
||||
|
||||
@@ -22,32 +22,35 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78).
|
||||
|
||||
| Dimension | Estimate | What's true today | Biggest gaps |
|
||||
|---|---:|---|---|
|
||||
| **Feature checklist** | 79% | P0 core and P1 nearly complete: import (Add / Copy / Move, templates, devices), library, grid/loupe/compare/survey, every Edit slider, curves, colour grading, masking tools, crop/Upright, heal/clone, presets/profiles, versions/history, sync, export, menus, shortcuts | P1: lens-profile database, content-aware fill (patch synthesis), video playback/trim |
|
||||
| **RAW coverage** (formats people shoot) | ~50% | DNG (all kinds), CR2, ARW, NEF (uncompressed + Huffman lossless/lossy), uncompressed RAF/ORF, packed RW2, PEF; every container's embedded preview (incl. CR3) | **CR3** (every Canon since ~2018), compressed RAF/ORF, RW2 v4, Nikon lossy-after-split, Canon sRAW, HEIC/AVIF. Per-model verification is thin (~40 corpus files vs >1,000 models) |
|
||||
| **Colour & image quality** | ~55–65% | Pipeline is complete and fast; GPU path CPU-exact within 1/255 | **No camera colour calibration**: non-DNG raws use a neutral matrix, so colour is muted/inaccurate. No lens-profile database. No measured fidelity against Lightroom (tone, highlights, texture/clarity, NR, sharpening are tuned by eye) |
|
||||
| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics | Real segmentation masks (subject, sky, people, objects, landscape, depth), AI denoise, super resolution, lens blur, generative remove, faces/people, natural-language search. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) |
|
||||
| **Feature checklist** | 79.7% | P0 core and P1 nearly complete: import (Add / Copy / Move, templates, devices), library, grid/loupe/compare/survey, every Edit slider, curves, colour grading, masking tools, crop/Upright, heal/clone, presets/profiles, versions/history, sync, export, menus, shortcuts | P0: full-resolution zoom detail; P1: lens-profile database, content-aware fill (patch synthesis), video playback/trim |
|
||||
| **RAW coverage** (formats people shoot) | ~55% | DNG (all kinds), CR2, CR3 lossless Bayer and version 0x100/0x200 C-RAW (M50/R100/R8 pixel-exact corpus), ARW, NEF (uncompressed + Huffman lossless/lossy), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans), uncompressed ORF, uncompressed Samsung SRW (NX5 to NX1100, EX1, WB2000), RW2 / RWL / Panasonic RAW (every raw format, checked on 178 files from 118 Panasonic and Leica bodies), PEF; every container's embedded preview (incl. CR3) | **CR3** (every Canon since ~2018), compressed ORF, compressed Samsung SRW (NX1, NX30, NX300, NX500, NX2000, NX3000, NX3300, NX mini), Nikon lossy-after-split, Canon sRAW, HEIC/AVIF. NEFs labelled compressed but stored uncompressed (Z 6 packed 14-bit, D850 12-bit uncompressed). Nikon High Efficiency NEF (HE / HE★: Z 8, Z 9, Z 6III, Z f; JPEG XS-based, preview only, #193). Per-model verification is thin (~55 corpus files vs >1,000 models) |
|
||||
| **Colour & image quality** | ~55–65% | Pipeline is complete and fast; GPU path CPU-exact within 1/255 | **No measured camera calibration database**: ARW, NEF, RW2, RAF, CR3, CR2 and PEF have guarded camera-JPEG colour estimates, with bundled ILCE-7CR, ILCE-7M4, X-H2S and X-T4 profiles (docs/camera-preview-colour.md); other non-DNG raws and rejected estimates use a neutral matrix. Colour fidelity remains incomplete. No lens-profile database. No measured fidelity against Lightroom (tone, highlights, texture/clarity, NR, sharpening are tuned by eye) |
|
||||
| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics, or the photo's own segmentation mattes when it ships them (DNG semantic masks, e.g. iPhone ProRAW's sky matte); non-destructive Bayer RAW denoise with opt-in weights on pure-Rust CPU/GPU; **faces (MVP, [docs/faces.md](docs/faces.md))**: a pure-Rust detector (a 232 KB opt-in download), opt-in recognition models (one-click download or add a file; they run on the shared pure-Rust CPU interpreter; pinned SFace/AuraFace embeddings agree with an independent reference runtime on 40 public crops; one-thread medians 34–36 / 233–252 ms (AuraFace remains expensive)), a background scan that finds and embeds faces, name suggestions, a person's page with look-alike faces to confirm, an Unnamed faces section (look-alikes together, select a group and name it at once) | Real segmentation masks (subject, sky, people, objects, landscape, depth), denoise model policy / linear RGB / quality verification, super resolution, lens blur, generative remove, natural-language search; for faces: clusters with separators between groups, small faces in large photos (the detector looks at 640 px), thresholds calibrated on live faces, writing face regions back to XMP. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) |
|
||||
| **Workflow & library** | ~85% (single machine) | Robust catalog (journal + snapshots, background compaction, crash-tested), 85k-photo libraries stay responsive, Local browsing with automatic cleanup, XMP interop, keywords, smart albums, Move import | Opening an 85k library takes 1.7–4.7 s; no cloud sync (out of scope), no tablet companion (#74, roadmap), shared albums, publish services, tethering |
|
||||
| **Classic modules** | ~30% | Geotagging from GPX track logs, soft proofing (partial), slideshow (basic) | **Map view, Book, Print, Slideshow module, Web, publish services**: ~40 tracker rows ⬜ |
|
||||
| **HDR & video** | 0% | | HDR edit/display/export; video play/trim/edit/export |
|
||||
| **Platform & robustness** | ~70% | macOS native; Windows/Linux builds; web via WASM; no-panic lints workspace-wide, `unsafe` confined to `crates/sysmem`; failed saves are reported; GPU errors fall back to CPU | Windows installer UI unverified on Windows (PR #79); GPU path proven only on Apple + user reports; no localisation; accessibility partial; headless UI tests time out under machine load |
|
||||
| **Classic modules** | ~30% | Geotagging from GPX track logs, soft proofing (partial), slideshow (basic), paginated contact-sheet PDF export (A4/Letter, layout/captions, current edits, 150 dpi sRGB) | **Map view, Book, full Print module, Slideshow module, Web, publish services**: contact sheets work, custom print packages and printer colour management remain missing |
|
||||
| **HDR & video** | ~25% | HDR editing (headroom limit, SDR rendition, Visualize HDR); gain map JPEG, PQ AVIF and float TIFF export | On-screen HDR display, HLG, HDR settings from crs XMP; video play/trim/edit/export |
|
||||
| **Platform & robustness** | ~70% | macOS native; Windows/Linux builds; web via WASM; no-panic lints workspace-wide, `unsafe` confined to `crates/sysmem`; failed saves are reported; GPU errors fall back to CPU | Windows installer UI unverified on Windows (PR #79); GPU path proven only on Apple + user reports; English, Simplified/Traditional Chinese, Japanese, Brazilian Portuguese, Spanish, German, Russian and Ukrainian UI (see docs/localization.md); remaining technical errors; Traditional Chinese uses the Simplified Chinese font until craft-fonts has a TC face; accessibility partial; headless UI tests time out under machine load |
|
||||
|
||||
### By kind of user
|
||||
|
||||
| User | Readiness | What blocks them |
|
||||
|---|---:|---|
|
||||
| JPEG / DNG shooter, single machine | ~85% | Fidelity polish, AI masks |
|
||||
| Nikon / Sony / older-Canon raw shooter | ~65% | Camera colour (muted colour on every photo) |
|
||||
| Canon CR3 / Fujifilm / Olympus shooter | ~35% | Their raws open as embedded previews only |
|
||||
| Nikon / Sony / Panasonic / older-Canon raw shooter | ~65% | Camera colour fidelity and coverage (ARW, NEF and RW2 preview estimates are only a starting point); Panasonic compacts and kit zooms also miss their embedded distortion correction |
|
||||
| Fujifilm raw shooter | ~65% | Uncompressed and lossless/lossy compressed RAF decoded; camera colour calibration remains missing. Verified on 13 bodies, including 14/16-bit GFX; see `docs/raf-compression.md` |
|
||||
| Canon CR3 / Olympus compressed-raw shooter | ~45% | CR3 lossless Bayer and version 0x100/0x200 C-RAW develop from sensor data (M50, R100, R8 verified pixel for pixel); other CRX variants and compressed ORF open as embedded JPEG previews. Camera colour and broader model verification remain |
|
||||
| Lightroom Classic power user | ~45% | Print, Book, Map, publish, tethering |
|
||||
| Relies on AI (masks, denoise) | ~25% | No segmentation or AI denoise models |
|
||||
| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); Bayer AI denoise needs separately installed weights; X-Trans/linear RGB, model policy and quality parity remain open |
|
||||
|
||||
## Where we're going
|
||||
|
||||
Priorities, in order. Each points at tracker rows in [`docs/parity.md`](docs/parity.md) → *Top gaps*.
|
||||
|
||||
1. **Camera colour calibration of our own** (LR-PROF-CAMERACOLOR, P0): fit each camera to its own embedded JPEG, use
|
||||
matrices the files carry themselves, then chart shots. Raises colour quality for every non-DNG raw at once.
|
||||
2. **Raw formats, clean-room** (LR-IMP-FORMATS, P0): **CR3** first, then compressed RAF / ORF, RW2 v4, NEF
|
||||
matrices the files carry themselves, then chart shots. Sony ARW, Nikon NEF, Panasonic RW2, Fujifilm RAF, Canon CR3/CR2,
|
||||
Pentax PEF, Samsung SRW and Olympus ORF have the file-local fit (matrix + tone curve from their own JPEG; CR2 and PEF
|
||||
since issue #310, not yet checked on the corpus); then validate fidelity.
|
||||
2. **Raw formats, clean-room** (LR-IMP-FORMATS, P0): Remaining **unverified CR3 CRX** variants, compressed ORF, NEF
|
||||
lossy-after-split, sRAW. Decided 2026-10-05: write our own decoders from prose descriptions (never decoder source,
|
||||
no LGPL dependency); compressed NEF (#86) is the template.
|
||||
3. **Verified camera coverage** (LR-IMP-CAMERA-COVERAGE, P0): a CC0 sample per model in the corpus, each decoded and
|
||||
@@ -56,14 +59,25 @@ Priorities, in order. Each points at tracker rows in [`docs/parity.md`](docs/par
|
||||
(references stay in the local `plan/`), then tune against the numbers.
|
||||
5. **Lens profiles of our own** (LR-EDIT-OPTICS-PROFILE, P1).
|
||||
6. **AI model strategy** (maintainer decision): which permissively licensed models (or our own training) for
|
||||
segmentation masks and denoise; then pure-Rust inference. Unblocks M12 and Enhance.
|
||||
7. **Then:** HDR (Q), the Classic output modules (Print first, then Map view, Book, Slideshow), video (R), localisation
|
||||
segmentation masks and denoise. Bayer denoise inference is implemented in pure Rust with an opt-in GPL model; release policy and independently licensed weights remain open. Unblocks M12 and Enhance. Face detection and recognition run on our own checked Rust engine. Object / Describe masks now
|
||||
run Meta's SAM 3 in pure Rust (optional, user-consented download under the non-OSI SAM License; CDN mirrors still
|
||||
to be set up, docs/ai-masks.md); Subject / Sky / People, Raw Details and Super Resolution are still open.
|
||||
7. **Then:** HDR (Q), the Classic output modules (expand contact-sheet PDF export into full Print, then Map view, Book, Slideshow), video (R), localisation
|
||||
and accessibility.
|
||||
|
||||
Already closed in the week of 2026-10-03: compressed NEF (#10), Move import (#29), per-library smart-preview folder,
|
||||
Local roots / cleanup, grid and catalog performance at 85k photos (#35, #37), failed-save reporting, GPU export
|
||||
hardening (#78), copyright metadata (#51), GPX geotagging (#60), import tag help and folder templates (#31, #32).
|
||||
|
||||
Library folders now have records of their own in the catalog (`Catalog.folder_records`, keyed by folder path; colour
|
||||
labels first). Photos still store full paths, so moving a whole disk or folder relinks each photo. If that becomes a
|
||||
problem (an "Update Folder Location" for a renamed or replaced drive, journal growth on large moves), the next step is
|
||||
the Classic model: photos point to a folder id (root + relative path), and the records keyed by path become those rows.
|
||||
|
||||
Image navigation now supports native pinch zoom and two-finger pan in Detail, Compare and Reference views.
|
||||
The whole-image preview still has a configurable size cap (2,560 px default), so 100% on larger photos does not
|
||||
yet guarantee original-pixel detail; visible-region rendering remains a gap (LR-VIEW-ZOOM).
|
||||
|
||||
## Milestones
|
||||
|
||||
**Status legend:** ✅ done · 🚧 in progress · ⬜ not started
|
||||
@@ -77,28 +91,28 @@ hardening (#78), copyright metadata (#51), GPX geotagging (#60), import tag help
|
||||
| M4 | Crop, geometry, optics | crop tool + overlays, straighten, Upright (auto/level/vertical/full/guided), manual transforms, CA, defringe, manual lens corrections | 6–10 | ✅ |
|
||||
| M5 | Performance | source pyramids, wgpu compute pipeline (CPU oracle), draft/full renders, prefetch, budgets (16 ms slider updates on 24 MP) | 10–15 | 🚧 (stage cache, source pyramid, wgpu pipeline, prefetch, memory budget ✅; colour NR at half resolution, GPU histogram ⬜) |
|
||||
| M6 | Masking | brush, linear/radial gradients, colour/luminance/depth range, add/subtract/intersect/invert, all local adjustments, masks panel | 8–12 | 🚧 (brush/linear/radial/colour/luminance range, add/subtract/intersect, masks panel ✅; depth range, AI masks ⬜) |
|
||||
| M7 | Detail | sharpening + masking preview, luminance/colour NR, Denoise, Raw Details, Super Resolution | 6–10 | 🚧 (sharpening, luminance/colour NR ✅; AI Denoise, Raw Details, Super Resolution ⬜) |
|
||||
| M7 | Detail | sharpening + masking preview, luminance/colour NR, Denoise, Raw Details, Super Resolution | 6–10 | 🚧 (sharpening, luminance/colour NR ✅; AI Denoise 🟡 (Bayer, optional weights; policy/fidelity open); Raw Details, Super Resolution ⬜) |
|
||||
| M8 | Heal / Remove | content-aware remove (PatchMatch), heal, clone, brush spots, visualize spots, red/pet eye | 6–10 | 🚧 (heal, clone, auto source, visualize spots, red/pet eye ✅; PatchMatch remove ⬜) |
|
||||
| M9 | Presets, profiles, versions, sync | preset browser + amount, create/import presets, profile browser, versions, history, copy/paste/sync settings | 5–8 | ✅ |
|
||||
| M10 | Export & share | export dialog (JPEG/PNG/TIFF/DNG/AVIF/JXL/original), sizing, sharpening, metadata, watermark, naming, batch jobs, XMP sidecars, HDR export | 6–10 | 🚧 (all formats incl. DNG/original, sizing, presets, background jobs ✅; JXL encode, HDR export ⬜) |
|
||||
| M11 | RAW II | CR3, RAF (X-Trans), ORF, RW2, PEF, SRW, 3FR, IIQ + long tail; camera calibration DB; HEIC/AVIF/JXL import | 20–35 | 🚧 (RAF uncompressed, RW2 packed, PEF, ORF uncompressed ✅; **camera colour calibration** ⬜ (top priority), CR3, compressed ORF/RAF ⬜) |
|
||||
| M10 | Export & share | export dialog (JPEG/PNG/TIFF/DNG/AVIF/JXL/original), sizing, sharpening, metadata, watermark, naming, batch jobs, XMP sidecars, HDR export | 6–10 | 🚧 (all formats incl. DNG/original, sizing, presets, background jobs, HDR gain map JPEG / PQ AVIF ✅; JXL encode ⬜) |
|
||||
| M11 | RAW II | CR3, RAF (X-Trans), ORF, RW2, PEF, SRW, 3FR, IIQ + long tail; camera calibration DB; HEIC/AVIF/JXL import | 20–35 | 🚧 (CR3 lossless CRX and C-RAW, RAF uncompressed and lossless/lossy compressed, RW2 (every raw format), PEF, ORF uncompressed ✅; **camera colour calibration** 🚧 (guarded ARW, NEF, RW2 and RAF preview fitting; measured database still missing), CR3, compressed ORF ⬜) |
|
||||
| M12 | AI & smart features | subject/sky/background/people/object masks, semantic search, faces/People (permissively licensed models, pure-Rust inference) | 20–40 | ⬜ |
|
||||
| M13 | Merge | HDR merge (deghost), panorama (projections, boundary warp, fill edges), HDR panorama | 10–15 | ✅ |
|
||||
| M14 | Video | import/playback/trim via FilmCraft crates, global edits + presets on video, video export | 6–10 | ⬜ |
|
||||
| M15 | Classic modules | Map, Book, Slideshow, Print, Web; smart collections, stacks, virtual copies, publish services, tethering | 25–40 | 🚧 (smart albums, stacks, virtual copies, compare/survey ✅; Map/Book/Slideshow/Print/Web ⬜) |
|
||||
| M16 | 1.0 polish | preferences, shortcut editor, accessibility, localization, packaging (dmg/msi/AppImage/web), hardening | 10–20 | 🚧 (settings, keyboard shortcuts sheet, packaging basics ✅; accessibility, localisation ⬜) |
|
||||
| M16 | 1.0 polish | preferences, shortcut editor, accessibility, localization, packaging (dmg/msi/AppImage/web), hardening | 10–20 | 🚧 (settings, keyboard shortcuts sheet, packaging basics ✅; English/Chinese (Simplified, Traditional)/Japanese/Brazilian Portuguese/Spanish/German/Russian/Ukrainian localisation 🟡; accessibility and further locales ⬜) |
|
||||
|
||||
## Parity estimate (feature count updated 2026-10-05; effort estimate from 2026-10-02)
|
||||
## Parity estimate (feature count updated 2026-10-08; effort estimate from 2026-10-02)
|
||||
|
||||
**By feature count** — from `docs/parity.md` (one row per Lightroom feature, menu item and shortcut; `cargo xtask
|
||||
parity` prints this line on every run, so it stays current):
|
||||
|
||||
| Scope | Weighted completion | Rows | 2026-10-02 |
|
||||
|---|---:|---:|---:|
|
||||
| P0 (core) | **98.5%** | 200 | 98.7% |
|
||||
| P1 (important parity) | **95.9%** | 147 | 94.4% |
|
||||
| P2 (later / AI / niche, incl. Classic modules) | **39.2%** | 158 | 29.4% |
|
||||
| **All in-scope rows** | **79.2%** | 505 | 75.6% |
|
||||
| P0 (core) | **98.2%** | 200 | 98.7% |
|
||||
| P1 (important parity) | **95.6%** | 149 | 94.4% |
|
||||
| P2 (later / AI / niche, incl. Classic modules) | **41.6%** | 160 | 29.4% |
|
||||
| **All in-scope rows** | **79.7%** | 509 | 75.6% |
|
||||
|
||||
The 2026-10-05 count includes three new 🟡 rows that make quality gaps visible (camera colour calibration, verified
|
||||
camera coverage, render fidelity), which is why P0 dipped slightly.
|
||||
@@ -115,7 +129,7 @@ trip, slideshow, auto import…) in ≈ 12 h on 2026-10-02; four to six parallel
|
||||
| Work package | Tracker rows | Agent-hours | Risk |
|
||||
|---|---|---:|---|
|
||||
| Remaining P0/P1 UI and library features (folder rename/move, keyword painter, people view…) | ≈ 8 | 5–10 | low |
|
||||
| Raw codecs: CR3 (CRX), compressed ORF / RAF, NEF lossy-after-split, RW2 v4, HEIC/AVIF decode, JPEG XL DNG | LR-IMP-FORMATS | 40–80 | **high** — clean-room black-box analysis, no permissive specs |
|
||||
| Raw codecs: CR3 (CRX), compressed ORF, NEF lossy-after-split, HEIC/AVIF decode | LR-IMP-FORMATS | 40–80 | **high** — clean-room black-box analysis, no permissive specs |
|
||||
| Lens-profile database of our own (calibration targets, fitting, data) | LR-EDIT-OPTICS-PROFILE | 15–30 | data collection |
|
||||
| Video: playback, trim, edits, export (pure-Rust decode, ideally shared with FilmCraft) | R. Video | 20–40 | medium |
|
||||
| AI: subject / sky / background / people / object masks, object-aware remove, AI denoise, super resolution, lens blur, people & faces, natural-language search, culling | ≈ 30 | 80–150 | **high** — permissively licensed weights, pure-Rust inference, maybe training |
|
||||
@@ -155,46 +169,71 @@ The milestone estimates in the table above were made before work started and are
|
||||
published alongside GPL code); decoder source is never read. Still open: freedom-to-operate review for local
|
||||
Laplacian filters, PatchMatch and HEVC (HEIC).
|
||||
- **Look parity** with Adobe's default rendering is tuned by eye today; the planned fidelity suite (LR-BEHAV-RENDER-FIDELITY)
|
||||
turns it into measured comparisons against local-only Lightroom references.
|
||||
turns it into measured comparisons against local-only Lightroom references. Retuning changes existing edits, so it
|
||||
ships as a new process version (`docs/process-versions.md`): photos keep their look until they are updated.
|
||||
|
||||
## Raw format coverage and known gaps
|
||||
|
||||
Sony ILCE-7M4 downsized lossless ARW (subsampled YCbCr 4:2:0 / 4:2:2 tiles) now decodes to linear RGB;
|
||||
private 3:2 & 4:3 examples verified through native rendering & full-resolution 16-bit export.
|
||||
|
||||
CR3 lossless Bayer and version 0x100/0x200 C-RAW sensor decoding is verified pixel-exactly on Canon M50, R100 and R8; any other CR3, or one that fails to decode, opens from its embedded JPEG as before. See [`docs/cr3.md`](docs/cr3.md) for coding limits, reference provenance and colour limitations.
|
||||
|
||||
Decoded (CC0 corpus from raw.pixls.us, `cargo xtask corpus --download`, `crates/raw/tests/corpus.rs`): DNG (uncompressed,
|
||||
LJ92, lossy JPEG / Smart Previews, Deflate, float, linear), CR2, ARW (uncompressed, ARW2, LJ92), NEF/NRW uncompressed and Huffman-compressed (lossless, lossy type 1/2, 12/14-bit), RAF uncompressed (Bayer and
|
||||
X-Trans), RW2 packed 12/14-bit, PEF (uncompressed and Huffman), ORF uncompressed (16-bit and 12-bit packed). Every
|
||||
supported container also yields its embedded JPEG preview (CR3 too), and the engine shows that preview for raw variants
|
||||
it can't decode yet.
|
||||
LJ92, lossy JPEG / Smart Previews, Deflate, float, linear, DNG 1.7 JPEG XL tiles: lossless ones bit-exact on synthetic files,
|
||||
lossy ones unclipped in the declared linear primaries, checked on one real file), CR2, ARW (uncompressed, ARW2, LJ92; as-shot white balance and black level of pre-2017 bodies from the encrypted
|
||||
`SR2SubIFD`, decrypted with a keystream recovered by black-box analysis, `crates/raw/src/vendor/sr2.rs`, `crates/raw/src/vendor/arw.rs`), NEF/NRW uncompressed and Huffman-compressed (lossless, lossy type 1/2, 12/14-bit), RAF uncompressed and lossless/lossy compressed (Bayer and
|
||||
X-Trans, 14/16-bit real-file verification), RW2 / Leica RWL / Panasonic RAW in every raw format (compressed formats 4 and 6, the prefix-coded strips of format 8,
|
||||
packed formats 2, 5 and 7, the 16-bit words of the 2005–2007 bodies; all recovered by black-box analysis of 178 CC0 files from
|
||||
118 bodies, `crates/raw/src/vendor/rw2.rs`), PEF (uncompressed and Huffman), ORF uncompressed (16-bit and 12-bit packed). Supported containers yield their embedded JPEG preview when present (including CR3). The engine uses it for RAW variants it cannot decode yet; HEVC-only CR3 previews are not decoded.
|
||||
|
||||
Not decoded yet — preview only (no permissively licensed description; black-box analysis incomplete):
|
||||
- **Nikon "lossy after split" NEF** (non-zero split row in maker note `0x0096`, e.g. some D3400/D5000/D5200/D5500/D5600
|
||||
files): rows above the split decode with the regular lossy table; from the split row on a different code is used
|
||||
that our black-box analysis has not recovered yet (none of the four regular tables fits, also not with byte
|
||||
alignment or reset predictors at the split row). The other Nikon Huffman variants are decoded (`crates/raw/src/vendor/nefc.rs` documents the analysis).
|
||||
- **Panasonic RW2 raw format 4** (quantised): the block layout is known (0x4000-byte chunks rotated by 0x1ff8; 128-bit
|
||||
blocks of two 12-bit seeds plus four groups of a 2-bit scale and three 8-bit codes; scales 0/1 are ×1/×2 differences
|
||||
from the same-colour pixel two to the left), the reconstruction rule for scales 2/3 is not established.
|
||||
- **Olympus compressed ORF**, **Fujifilm compressed RAF**, **Canon CR3/CRX** (M11.1), **Canon sRAW/mRAW**, lossy DNG.
|
||||
- **Olympus compressed ORF**, **Canon CR3 unverified CRX variants** (M11.1), **Canon sRAW/mRAW**.
|
||||
|
||||
**Camera colour matrices:** non-DNG raws use the documented neutral fallback (camera RGB ≈ linear sRGB, flagged
|
||||
**Camera colour matrices:** ARW, NEF, RW2, RAF, CR3, CR2, PEF, Samsung SRW and Olympus ORF files can use guarded, separate chromaticity and tone estimates from their own embedded JPEG (see `docs/camera-preview-colour.md`); this is a per-file camera-look estimate with relative WB, not measured calibration or absolute-Kelvin WB. Other non-DNG raws and rejected fits use the documented neutral fallback (camera RGB ≈ linear sRGB, flagged
|
||||
`matrix_is_fallback`) with the file's as-shot white-balance multipliers. Clean sources to evaluate next: manufacturer
|
||||
matrices stored in the files themselves (Olympus ImageProcessing `ColorMatrix`, Pentax/Panasonic equivalents) and our
|
||||
matrices stored in the files themselves (Olympus ImageProcessing `ColorMatrix`, Pentax equivalents; Panasonic RW2 files carry none) and our
|
||||
own chart-based calibration (M11.4). Adobe matrices are never used.
|
||||
|
||||
## Lightroom Classic catalog migration
|
||||
|
||||
File → Import Lightroom Catalog… reads `.lrcat` plus committed WAL pages directly through a pure-Rust
|
||||
SQLite reader. Originals stay in place; ratings, flags, labels, XMP metadata, hierarchical keywords,
|
||||
collections/sets, virtual copies and supported develop settings migrate into LightCraft. Existing edits
|
||||
are preserved by default, reimport identities are persistent, and source settings/history/snapshots are
|
||||
archived before catalog mutation. Missing originals remain available for relinking.
|
||||
|
||||
Rendering is approximate: unsupported Adobe profiles/AI/process settings are reported and archived;
|
||||
history/snapshots remain source data, and smart collections import current membership. The Lightroom
|
||||
database is read-only; LightCraft owns subsequent edits. See `docs/lightroom-catalog-import.md`.
|
||||
|
||||
## Log
|
||||
- 2026-09-30: roadmap created; M0 in progress; research docs (Lightroom reference, Rust imaging ecosystem) complete.
|
||||
- 2026-09-30 (later): app running with the full Lightroom-style UI; pipeline v0; DNG/CR2/ARW; README showcase. ≈ 8 h elapsed.
|
||||
- 2026-10-02: parity estimate added (xtask parity prints weighted completion); milestone statuses refreshed.
|
||||
- 2026-10-08: Fujifilm camera colour: guarded JPEG matching and relative WB, plus bundled X-H2S/X-T4 profiles from 201/545 photos. On 150 withheld files the pooled profiles improve mean ΔE slightly over per-file fitting (X-H2S 1.70 → 1.62, X-T4 2.43 → 2.34); dark X-H2S samples 8.74 → 2.25. Measured calibration remains open.
|
||||
- 2026-10-08: Fujifilm lossless/lossy compressed RAF: pure-Rust striped decoder; full sensor arrays verified against a black-box reference on 17 CC0 files and 7 supplied X-H2S / X-T4 files (13 camera bodies, 14/16-bit Bayer and X-Trans). Camera colour calibration remains a separate gap.
|
||||
- 2026-10-01: RAW II formats: RAF (uncompressed Bayer + X-Trans), RW2 (packed), PEF (incl. Huffman), ORF (uncompressed); embedded previews for every container incl. CR3; raw corpus test with 37 CC0 samples.
|
||||
- 2026-10-03 – 10-05: community PRs (#42–#51, #60) and 25+ issues landed: compressed NEF, Move import, folder templates,
|
||||
Local roots and cleanup, 85k-photo grid and catalog performance, background compaction, failed-save errors, GPU
|
||||
export hardening, clippy 1.99. Added the honest *Where we stand* assessment and *Where we're going* priorities; added
|
||||
tracker rows for camera colour, camera coverage and render fidelity.
|
||||
- 2026-10-07: Panasonic RW2 / Leica RWL / Panasonic RAW decode in every raw format (compressed 4/6/8, packed 2/5/7,
|
||||
16-bit words), black levels, in-camera aspect crops and mapped-out defects established on 178 CC0 files from 118
|
||||
bodies; RW2 gets the file-local camera look (accepted on 140 of 174 files with a preview).
|
||||
- 2026-10-08: independent Rust CR3 lossless and version 0x100/0x200 C-RAW decoding, six exact full-sensor regressions on M50/R100/R8; any CR3 decode error falls back to the embedded JPEG; guarded CR3 camera-JPEG colour fitting with independent framing validation (`docs/cr3.md`). Unverified CRX variants and broader model verification remain.
|
||||
- 2026-10-09: Sony ILCE-7CR camera colour: bundled profile from 125 photos. On 26 withheld photos mean ΔE vs the camera JPEG 4.07 → 3.29; all 131 photos accepted, where 8 fell back to the neutral matrix without it. Measured calibration remains open.
|
||||
|
||||
## Japanese interface and text watermarks
|
||||
## Chinese and Japanese interfaces, and text watermarks
|
||||
|
||||
English/Japanese interface language is persisted in UI state. Core menus have Japanese
|
||||
translations; untranslated panels and dialogs retain English. BIZ UDMincho is bundled
|
||||
under OFL for Japanese glyph coverage without system fonts. Text watermarks now accept
|
||||
English, Simplified Chinese, Traditional Chinese (Taiwan) and Japanese interface languages are persisted in UI state, and the
|
||||
language table (`crates/ui-egui/src/i18n.rs`) drives the menus, settings and fonts, so another
|
||||
language is a table entry plus two catalogs (docs/localization.md). Core menus and panels are
|
||||
translated; untranslated text retains English. Ukrainian covers all existing catalog keys, command/control/rule labels
|
||||
and format messages, with count-neutral labels and numeric dates; technical errors from lower layers remain English.
|
||||
CJK glyphs (Chinese: Noto Sans CJK SC; Japanese UI:
|
||||
BIZ UDPGothic; watermarks: BIZ UDMincho) come from storytold/craft-fonts, embedded by builds made
|
||||
with the optional `CRAFT_FONTS_DIR` input (all releases), so no system fonts are needed. Text watermarks now accept
|
||||
`vertical: true` in export JSON/presets and expose an orientation selector. Japanese
|
||||
characters stay upright in top-to-bottom columns, with newlines starting columns to the
|
||||
left. This is basic lettering, without tate-chu-yoko, ruby, kinsoku, or general vertical
|
||||
|
||||
@@ -25,7 +25,14 @@ serde_json = { workspace = true }
|
||||
dhat = { workspace = true, optional = true }
|
||||
|
||||
[features]
|
||||
default = ["denoise"]
|
||||
denoise = ["lightcraft-engine/denoise"]
|
||||
rawnind-model = ["denoise", "lightcraft-engine/rawnind-model"]
|
||||
# Heap profiling: `cargo run --release -p lightcraft-cli --features dhat-heap -- snapshot …` counts
|
||||
# every allocation (live/peak bytes in `library.memory` / `ui.inspect`) and writes dhat-heap.json
|
||||
# (open it in DHAT's viewer: allocation sites by bytes at the peak). Slows allocation-heavy work.
|
||||
dhat-heap = ["dep:dhat"]
|
||||
|
||||
# Open HEIC/HEIF photos (the optional lightcraft-heif crate). Not default so distributors choose
|
||||
# it (HEVC patents); build with `--features heif` to include it.
|
||||
heif = ["lightcraft-codecs/heif"]
|
||||
+204
-45
@@ -9,6 +9,7 @@
|
||||
//! lightcraft-cli synth-merge hdr|panorama -o DIR
|
||||
//! lightcraft-cli commands [--json]
|
||||
//! lightcraft-cli controls [--json]
|
||||
//! lightcraft-cli calibrate [--max N] [--out DIR] FOLDERS/FILES…
|
||||
//! ```
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
@@ -70,6 +71,8 @@ USAGE:
|
||||
per line: {\"method\": \"ui.set\", \"params\": {\"view\": \"detail\"}}.
|
||||
Replies go to stdout. `ui.screenshot` without a path writes -o (then
|
||||
OUT-2.png, OUT-3.png…); `ui.settle {timeoutMs?}` waits for renders.
|
||||
A failed request (\"ok\": false) does not stop the script, but the exit
|
||||
status is non-zero when any request failed.
|
||||
-o, --output OUT PNG path (a final screenshot is written here if the script took none)
|
||||
--size WxH window size in points (default 1600x1000)
|
||||
--scale S pixels per point (default 1)
|
||||
@@ -84,6 +87,12 @@ USAGE:
|
||||
Write synthetic merge inputs (procedural scene; bracketed DNGs or overlapping PNG views).
|
||||
lightcraft-cli commands [--json] list every command id with its parameters
|
||||
lightcraft-cli controls [--json] list every develop control id with its range
|
||||
lightcraft-cli calibrate [--max N] [--out DIR] FOLDERS/FILES…
|
||||
Fit a colour profile per camera model from raw files and their embedded camera JPEGs
|
||||
(Sony ARW, Nikon NEF, Fujifilm RAF): up to N files spread over the folders (default 300; 0 = all), pooled per
|
||||
model, written as <model>.json to DIR (default: the profiles folder LightCraft reads,
|
||||
<config>/camera-profiles, or $LIGHTCRAFT_CAMERA_PROFILES). Raws of a profiled model then
|
||||
take their colour from the profile and only their tone from their own JPEG.
|
||||
lightcraft-cli --version | --help
|
||||
";
|
||||
|
||||
@@ -91,6 +100,25 @@ USAGE:
|
||||
#[global_allocator]
|
||||
static ALLOC: dhat::Alloc = dhat::Alloc;
|
||||
|
||||
/// Why `--library DIR` can't be opened; for a library open in another program, how to work with
|
||||
/// that one instead (issue #99).
|
||||
/// Print the open library's warnings (an unlockable library, damaged settings files) on stderr:
|
||||
/// stdout carries command results / MCP protocol messages only.
|
||||
fn library_warnings(session: &mut Session, who: &str) {
|
||||
for w in session.take_library_warnings() {
|
||||
eprintln!("{who}: warning: {w}");
|
||||
}
|
||||
}
|
||||
|
||||
fn library_error(dir: &str, e: lightcraft_engine::EngineError) -> String {
|
||||
match e {
|
||||
lightcraft_engine::EngineError::LibraryInUse(why) => format!(
|
||||
"{dir}: {why}\nTo work with the library while the app has it open, start the app with `--control PORT` and use `lightcraft-cli mcp --connect 127.0.0.1:PORT`."
|
||||
),
|
||||
e => format!("{dir}: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
// `--features dhat-heap`: count allocations; the profile is written when `_heap` drops
|
||||
// (LIGHTCRAFT_DHAT_FILE, default dhat-heap.json).
|
||||
@@ -104,6 +132,9 @@ fn main() -> ExitCode {
|
||||
dhat::Profiler::builder().file_name(file).build()
|
||||
};
|
||||
alloc_release::install();
|
||||
// Warnings (a GPU render redone on the CPU, an unknown backend name) on stderr; LIGHTCRAFT_LOG
|
||||
// or RUST_LOG picks another level (#168).
|
||||
lightcraft_engine::logging::install("lightcraft-cli");
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
let r = match args.first().map(String::as_str) {
|
||||
Some("run") => run(&args[1..]),
|
||||
@@ -114,6 +145,7 @@ fn main() -> ExitCode {
|
||||
Some("merge") => merge(&args[1..]),
|
||||
Some("synth-merge") => synth_merge(&args[1..]),
|
||||
Some("controls") => controls(&args[1..]),
|
||||
Some("calibrate") => calibrate(&args[1..]),
|
||||
Some("--version" | "-V" | "version") => {
|
||||
println!("lightcraft-cli {}", env!("CARGO_PKG_VERSION"));
|
||||
Ok(())
|
||||
@@ -133,6 +165,115 @@ fn main() -> ExitCode {
|
||||
}
|
||||
}
|
||||
|
||||
/// Raw files below `path` (or `path` itself), skipping hidden and NAS metadata folders.
|
||||
fn raw_files(path: &Path, out: &mut Vec<std::path::PathBuf>, depth: usize) {
|
||||
let is_raw =
|
||||
|p: &Path| p.extension().and_then(|e| e.to_str()).is_some_and(|e| ["arw", "nef", "nrw", "raf"].iter().any(|x| e.eq_ignore_ascii_case(x)));
|
||||
if path.is_file() {
|
||||
if is_raw(path) {
|
||||
out.push(path.to_path_buf());
|
||||
}
|
||||
return;
|
||||
}
|
||||
let Ok(entries) = std::fs::read_dir(path) else { return };
|
||||
if depth > 32 {
|
||||
return;
|
||||
}
|
||||
for entry in entries.flatten() {
|
||||
let name = entry.file_name();
|
||||
if name.to_string_lossy().starts_with(['.', '@']) {
|
||||
continue;
|
||||
}
|
||||
raw_files(&entry.path(), out, depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
fn calibrate(args: &[String]) -> Result<(), String> {
|
||||
let mut max = 300usize;
|
||||
let mut out: Option<std::path::PathBuf> = None;
|
||||
let mut inputs = Vec::new();
|
||||
let mut i = 0;
|
||||
while i < args.len() {
|
||||
match args[i].as_str() {
|
||||
"--max" => max = take_value(args, &mut i, "--max")?.parse().map_err(|_| "--max needs a number".to_string())?,
|
||||
"--out" => out = Some(take_value(args, &mut i, "--out")?.into()),
|
||||
a if a.starts_with("--") => return Err(format!("unknown option `{a}`")),
|
||||
f => inputs.push(f.to_string()),
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
if inputs.is_empty() {
|
||||
return Err("calibrate needs folders or raw files".into());
|
||||
}
|
||||
let mut files = Vec::new();
|
||||
for input in &inputs {
|
||||
raw_files(Path::new(input), &mut files, 0);
|
||||
}
|
||||
files.sort();
|
||||
files.dedup();
|
||||
let found = files.len();
|
||||
if max > 0 && files.len() > max {
|
||||
// spread over all folders (dates, scenes) rather than the first N
|
||||
files = (0..max).filter_map(|k| files.get(k * found / max).cloned()).collect();
|
||||
}
|
||||
eprintln!("calibrate: {} of {found} raw files", files.len());
|
||||
let dir = out.or_else(lightcraft_engine::camera_profiles::dir).ok_or("no profiles folder: pass --out DIR")?;
|
||||
let workers = std::thread::available_parallelism().map_or(4, |n| n.get()).clamp(1, 6);
|
||||
let next = std::sync::atomic::AtomicUsize::new(0);
|
||||
let done = std::sync::atomic::AtomicUsize::new(0);
|
||||
let pools: Vec<lightcraft_engine::camera_profiles::Pool> = std::thread::scope(|scope| {
|
||||
let handles: Vec<_> = (0..workers)
|
||||
.map(|_| {
|
||||
scope.spawn(|| {
|
||||
let mut pool = lightcraft_engine::camera_profiles::Pool::default();
|
||||
while let Some(path) = files.get(next.fetch_add(1, std::sync::atomic::Ordering::Relaxed)) {
|
||||
let result = std::fs::read(path).map_err(|e| e.to_string()).and_then(|bytes| pool.add(&bytes));
|
||||
let n = done.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
|
||||
match result {
|
||||
Ok(Some(model)) => eprintln!("[{n}/{}] {} ({model})", files.len(), path.display()),
|
||||
Ok(None) => eprintln!("[{n}/{}] {}: skipped (no usable camera JPEG or colour)", files.len(), path.display()),
|
||||
Err(e) => eprintln!("[{n}/{}] {}: {e}", files.len(), path.display()),
|
||||
}
|
||||
}
|
||||
pool
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
handles.into_iter().filter_map(|h| h.join().ok()).collect()
|
||||
});
|
||||
let mut pool = lightcraft_engine::camera_profiles::Pool::default();
|
||||
for p in pools {
|
||||
pool.merge(p);
|
||||
}
|
||||
let mut written = 0;
|
||||
for result in pool.fit(5) {
|
||||
match result {
|
||||
Ok(profile) => {
|
||||
let path = lightcraft_engine::camera_profiles::save(&profile, &dir)?;
|
||||
println!(
|
||||
"{}: {} photos, {} colour pairs, hue/saturation table {} → {}",
|
||||
profile.model,
|
||||
profile.files,
|
||||
profile.samples,
|
||||
profile.hue_sat.is_some(),
|
||||
path.display()
|
||||
);
|
||||
written += 1;
|
||||
}
|
||||
Err(e) => eprintln!("calibrate: {e}"),
|
||||
}
|
||||
}
|
||||
for (model, n) in pool.files() {
|
||||
if n < 5 {
|
||||
eprintln!("calibrate: {model}: only {n} usable photo(s), at least 5 needed");
|
||||
}
|
||||
}
|
||||
if written == 0 {
|
||||
return Err("no profile written".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn mcp(args: &[String]) -> Result<(), String> {
|
||||
let mut connect: Option<String> = None;
|
||||
let mut demo = false;
|
||||
@@ -182,8 +323,9 @@ fn mcp(args: &[String]) -> Result<(), String> {
|
||||
let mut h = match &library {
|
||||
Some(dir) => {
|
||||
let mut h = Headless::default();
|
||||
let r = h.session.open_library(dir, demo).map_err(|e| e.to_string())?;
|
||||
let r = h.session.open_library(dir, demo).map_err(|e| library_error(dir, e))?;
|
||||
eprintln!("lightcraft-cli mcp: opened library {dir} ({r:?})");
|
||||
library_warnings(&mut h.session, "lightcraft-cli mcp");
|
||||
h
|
||||
}
|
||||
None if demo => Headless::demo(),
|
||||
@@ -200,7 +342,7 @@ fn mcp(args: &[String]) -> Result<(), String> {
|
||||
eprintln!("lightcraft-cli mcp: serving MCP on stdio ({})", backend.describe());
|
||||
let mut server = Server::new(backend).with_command_tools(!compact);
|
||||
let stdin = std::io::stdin();
|
||||
server.serve(BufReader::new(stdin.lock()), std::io::stdout().lock()).map_err(|e| e.to_string())
|
||||
server.serve(BufReader::new(stdin), std::io::stdout()).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
fn merge(args: &[String]) -> Result<(), String> {
|
||||
@@ -249,7 +391,7 @@ fn merge(args: &[String]) -> Result<(), String> {
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
let mut s = Session::new().with_fs();
|
||||
let mut s = Session::new().with_fs().with_default_denoise_models().with_system_clock();
|
||||
let paths = expand_paths(&files);
|
||||
let r = s.execute("library.import", &json!({"paths": paths})).map_err(|e| e.to_string())?;
|
||||
let mut ids: Vec<u64> = r["imported"].as_array().map(|a| a.iter().filter_map(Value::as_u64).collect()).unwrap_or_default();
|
||||
@@ -405,39 +547,40 @@ fn run(args: &[String]) -> Result<(), String> {
|
||||
if steps.is_empty() {
|
||||
return Err("run: no command given (see `lightcraft-cli commands`)".into());
|
||||
}
|
||||
let mut backend: Box<dyn Backend> = match connect {
|
||||
Some(addr) => {
|
||||
if demo || library.is_some() || !imports.is_empty() {
|
||||
return Err("--demo, --library and --import apply to headless mode only".into());
|
||||
}
|
||||
Box::new(
|
||||
Remote::connect(&addr)
|
||||
.map_err(|e| format!("LightCraft is not reachable at {addr} ({e}); start it with `lightcraft --control 7980`"))?,
|
||||
)
|
||||
}
|
||||
None => {
|
||||
let mut h = match &library {
|
||||
Some(dir) => {
|
||||
let mut h = Headless::default();
|
||||
h.session.open_library(dir, demo).map_err(|e| e.to_string())?;
|
||||
h
|
||||
}
|
||||
None if demo => Headless::demo(),
|
||||
None => Headless::default(),
|
||||
};
|
||||
if !imports.is_empty() {
|
||||
let paths = expand_paths(&imports);
|
||||
let r = h.session.execute("library.import", &json!({"paths": paths})).map_err(|e| e.to_string())?;
|
||||
// what follows acts on the imported photos (already-known files are reported as duplicates)
|
||||
let mut ids: Vec<Value> = r["imported"].as_array().cloned().unwrap_or_default();
|
||||
ids.extend(r["duplicates"].as_array().into_iter().flatten().filter_map(|d| d.get("existing").filter(|v| v.is_u64()).cloned()));
|
||||
if let Some(first) = ids.first().cloned() {
|
||||
h.session.execute("library.select", &json!({"ids": ids, "active": first})).map_err(|e| e.to_string())?;
|
||||
let mut backend: Box<dyn Backend> =
|
||||
match connect {
|
||||
Some(addr) => {
|
||||
if demo || library.is_some() || !imports.is_empty() {
|
||||
return Err("--demo, --library and --import apply to headless mode only".into());
|
||||
}
|
||||
Box::new(Remote::connect(&addr).map_err(|e| {
|
||||
format!("LightCraft is not reachable at {addr} ({e}); start it with `lightcraft --control {}`", connect_port(&addr))
|
||||
})?)
|
||||
}
|
||||
Box::new(h)
|
||||
}
|
||||
};
|
||||
None => {
|
||||
let mut h = match &library {
|
||||
Some(dir) => {
|
||||
let mut h = Headless::default();
|
||||
h.session.open_library(dir, demo).map_err(|e| library_error(dir, e))?;
|
||||
library_warnings(&mut h.session, "lightcraft-cli");
|
||||
h
|
||||
}
|
||||
None if demo => Headless::demo(),
|
||||
None => Headless::default(),
|
||||
};
|
||||
if !imports.is_empty() {
|
||||
let paths = expand_paths(&imports);
|
||||
let r = h.session.execute("library.import", &json!({"paths": paths})).map_err(|e| e.to_string())?;
|
||||
// what follows acts on the imported photos (already-known files are reported as duplicates)
|
||||
let mut ids: Vec<Value> = r["imported"].as_array().cloned().unwrap_or_default();
|
||||
ids.extend(r["duplicates"].as_array().into_iter().flatten().filter_map(|d| d.get("existing").filter(|v| v.is_u64()).cloned()));
|
||||
if let Some(first) = ids.first().cloned() {
|
||||
h.session.execute("library.select", &json!({"ids": ids, "active": first})).map_err(|e| e.to_string())?;
|
||||
}
|
||||
}
|
||||
Box::new(h)
|
||||
}
|
||||
};
|
||||
let mut out = std::io::stdout().lock();
|
||||
let mut failed = 0;
|
||||
for s in &steps {
|
||||
@@ -463,6 +606,12 @@ fn take_value<'a>(args: &'a [String], i: &mut usize, flag: &str) -> Result<&'a s
|
||||
args.get(*i).map(String::as_str).ok_or_else(|| format!("{flag} needs a value"))
|
||||
}
|
||||
|
||||
/// The port an `--connect` address names, for the recovery hint. The whole address when it has
|
||||
/// no port, so a hint is never built from a guess.
|
||||
fn connect_port(addr: &str) -> &str {
|
||||
addr.rsplit_once(':').map_or(addr, |(_, port)| port)
|
||||
}
|
||||
|
||||
fn render(args: &[String]) -> Result<(), String> {
|
||||
let mut input = None;
|
||||
let mut output = None;
|
||||
@@ -503,7 +652,7 @@ fn render(args: &[String]) -> Result<(), String> {
|
||||
}
|
||||
let input = input.ok_or("render: missing input file")?;
|
||||
let output = output.ok_or("render: missing -o OUTPUT")?;
|
||||
let mut s = Session::new().with_fs();
|
||||
let mut s = Session::new().with_fs().with_default_denoise_models().with_system_clock();
|
||||
let abs = expand_paths(std::slice::from_ref(&input));
|
||||
let r = s.execute("library.import", &json!({"paths": abs})).map_err(|e| e.to_string())?;
|
||||
let id = r["imported"][0].as_u64().ok_or_else(|| format!("{input}: not a readable photo"))?;
|
||||
@@ -526,7 +675,7 @@ fn render(args: &[String]) -> Result<(), String> {
|
||||
for (k, v) in opts {
|
||||
p[k] = v;
|
||||
}
|
||||
let mut o = ExportOptions::from_json(&p);
|
||||
let mut o = ExportOptions::from_params(&p).map_err(|e| e.to_string())?;
|
||||
if p.get("format").is_none() {
|
||||
let ext = Path::new(&output).extension().map(|e| e.to_string_lossy().to_string()).unwrap_or_default();
|
||||
o.format = ExportFormat::parse(&ext)
|
||||
@@ -579,19 +728,19 @@ fn snapshot(args: &[String]) -> Result<(), String> {
|
||||
if script.is_none() && output.is_none() {
|
||||
return Err("snapshot: give -o OUT.png and/or --script FILE".into());
|
||||
}
|
||||
if !(scale > 0.0 && size[0] >= 1.0 && size[1] >= 1.0) {
|
||||
return Err("snapshot: bad --size/--scale".into());
|
||||
}
|
||||
lightcraft_ui_egui::headless::viewport_pixels(size, scale).map_err(|e| format!("snapshot: bad --size/--scale: {e}"))?;
|
||||
let t0 = Instant::now();
|
||||
let mut session = match &library {
|
||||
Some(dir) => {
|
||||
let mut s = Session::new().with_fs();
|
||||
s.open_library(dir, false).map_err(|e| format!("{dir}: {e}"))?;
|
||||
let mut s = Session::new().with_fs().with_default_denoise_models().with_system_clock().with_default_face_models();
|
||||
s.open_library(dir, false).map_err(|e| library_error(dir, e))?;
|
||||
s
|
||||
}
|
||||
None if files.is_empty() => Session::with_demo().with_fs(),
|
||||
None => Session::new().with_fs(),
|
||||
None if files.is_empty() => Session::with_demo().with_fs().with_default_denoise_models().with_system_clock().with_default_face_models(),
|
||||
None => Session::new().with_fs().with_default_denoise_models().with_system_clock().with_default_face_models(),
|
||||
};
|
||||
// Apply compute policy before imports or the first scripted query can discover an adapter.
|
||||
session.execute("app.gpu", &json!({"enabled": false})).map_err(|e| e.to_string())?;
|
||||
if !files.is_empty() {
|
||||
let ti = Instant::now();
|
||||
let r = session.execute("library.import", &json!({"paths": expand_paths(&files)})).map_err(|e| e.to_string())?;
|
||||
@@ -606,7 +755,11 @@ fn snapshot(args: &[String]) -> Result<(), String> {
|
||||
})),
|
||||
..Default::default()
|
||||
};
|
||||
let app = lightcraft_ui_egui::LightcraftApp::new(session, services);
|
||||
let mut app = lightcraft_ui_egui::LightcraftApp::new(session, services);
|
||||
// Snapshot sessions promise no GPU: disable photo compute as well as the compositor
|
||||
// before the first frame can start background adapter discovery.
|
||||
// This UI state is session-local: the CLI neither loads nor saves desktop ui.json.
|
||||
app.ui.settings.gpu = false;
|
||||
let mut h = Headless::new(app, size, scale);
|
||||
let timeout = Duration::from_secs(60);
|
||||
let mut shots = 0usize;
|
||||
@@ -622,6 +775,9 @@ fn snapshot(args: &[String]) -> Result<(), String> {
|
||||
Some(p.with_file_name(format!("{stem}-{shots}.{ext}")).to_string_lossy().to_string())
|
||||
};
|
||||
let mut wrote_output = false;
|
||||
// Issue #166: a scripted request that fails still gets its reply printed and the run carries
|
||||
// on (like `run --keep-going`), but the exit status reports it — CI judges by exit status.
|
||||
let mut failed = 0usize;
|
||||
if let Some(script) = &script {
|
||||
let text = std::fs::read_to_string(script).map_err(|e| format!("{script}: {e}"))?;
|
||||
let mut out = std::io::stdout().lock();
|
||||
@@ -654,6 +810,9 @@ fn snapshot(args: &[String]) -> Result<(), String> {
|
||||
}
|
||||
_ => h.request(&method, params, timeout),
|
||||
};
|
||||
if reply["ok"] != true {
|
||||
failed += 1;
|
||||
}
|
||||
if let Some(o) = reply.as_object_mut() {
|
||||
o.insert("id".into(), id);
|
||||
// wall time of the request (incl. the frames it ran), and since the start
|
||||
@@ -690,7 +849,7 @@ fn snapshot(args: &[String]) -> Result<(), String> {
|
||||
}
|
||||
}
|
||||
eprintln!("lightcraft-cli snapshot: done in {:.2} s ({} frames)", t0.elapsed().as_secs_f64(), h.frames());
|
||||
Ok(())
|
||||
if failed > 0 { Err(format!("{failed} scripted request(s) failed")) } else { Ok(()) }
|
||||
}
|
||||
|
||||
fn commands(args: &[String]) -> Result<(), String> {
|
||||
|
||||
@@ -7,6 +7,46 @@ use serde_json::{Value, json};
|
||||
|
||||
const BIN: &str = env!("CARGO_BIN_EXE_lightcraft-cli");
|
||||
|
||||
#[test]
|
||||
fn contact_sheet_exports_pdf_and_protects_originals() {
|
||||
let input = tmp("contact-sheet-input.png");
|
||||
gradient_png(&input);
|
||||
let out = tmp("contact-sheet.pdf");
|
||||
let (ok, lines, err) = run_cli(
|
||||
&[
|
||||
"--import",
|
||||
input.to_str().unwrap(),
|
||||
"export.contactSheet",
|
||||
&format!("path={}", out.display()),
|
||||
"paper=letter",
|
||||
"landscape=true",
|
||||
"columns=2",
|
||||
"rows=3",
|
||||
],
|
||||
None,
|
||||
);
|
||||
assert!(ok, "{err}: {lines:?}");
|
||||
assert_eq!(lines[0]["result"]["pages"], 1);
|
||||
assert_eq!(lines[0]["result"]["photos"], 1);
|
||||
let bytes = std::fs::read(&out).unwrap();
|
||||
assert!(bytes.starts_with(b"%PDF-1.4"));
|
||||
assert!(String::from_utf8_lossy(&bytes).contains("/MediaBox [0 0 792.00 612.00]"));
|
||||
let before = std::fs::read(&input).unwrap();
|
||||
for target in [input.clone(), input.with_extension("xmp")] {
|
||||
if target != input {
|
||||
std::fs::write(&target, b"existing sidecar").unwrap();
|
||||
}
|
||||
let (ok, _, _) = run_cli(&["--import", input.to_str().unwrap(), "export.contactSheet", &format!("path={}", target.display())], None);
|
||||
assert!(!ok, "overwrote protected file");
|
||||
}
|
||||
assert_eq!(std::fs::read(&input).unwrap(), before);
|
||||
assert_eq!(std::fs::read(input.with_extension("xmp")).unwrap(), b"existing sidecar");
|
||||
let prior = std::fs::read(&out).unwrap();
|
||||
let (ok, _, _) = run_cli(&["--import", input.to_str().unwrap(), "export.contactSheet", &format!("path={}", out.display()), "columns=0"], None);
|
||||
assert!(!ok);
|
||||
assert_eq!(std::fs::read(&out).unwrap(), prior, "failed export changed prior output");
|
||||
}
|
||||
|
||||
fn tmp(name: &str) -> std::path::PathBuf {
|
||||
let d = std::env::temp_dir().join(format!("lightcraft-cli-test-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&d).unwrap();
|
||||
@@ -19,6 +59,19 @@ fn gradient_png(path: &std::path::Path) {
|
||||
std::fs::write(path, png).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn calibrate_reads_raf_inputs_and_reports_bad_files() {
|
||||
let input = tmp("invalid.RAF");
|
||||
std::fs::write(&input, b"FUJIFILMCCD-RAW").unwrap();
|
||||
let out = tmp("raf-profiles");
|
||||
let result = Command::new(BIN).args(["calibrate", "--out"]).arg(&out).arg(&input).output().unwrap();
|
||||
assert!(!result.status.success());
|
||||
let stderr = String::from_utf8_lossy(&result.stderr);
|
||||
assert!(stderr.contains("1 of 1 raw files"), "RAF was silently ignored: {stderr}");
|
||||
assert!(stderr.contains("invalid.RAF") && stderr.contains("no profile written"), "{stderr}");
|
||||
assert!(!out.join("X-T4.json").exists(), "bad input must not produce a profile");
|
||||
}
|
||||
|
||||
fn mean(path: &std::path::Path) -> f64 {
|
||||
let d = lightcraft_codecs::decode(&std::fs::read(path).unwrap(), Default::default()).unwrap();
|
||||
let img = d.to_srgb8();
|
||||
@@ -154,10 +207,14 @@ fn snapshot_subcommand_renders_the_ui_headlessly() {
|
||||
std::fs::write(
|
||||
&script,
|
||||
format!(
|
||||
"# comment\n{}\n{}\n{}\n{}\n",
|
||||
"# comment\n{}\n{}\n{}\n{}\n{}\n{}\n",
|
||||
json!({"method": "ui.set", "params": {"view": "photoGrid"}}),
|
||||
// (settled: under load a grid shot taken before its thumbnails is darker than the
|
||||
// dimmed export dialog over them, and the comparison below fails)
|
||||
json!({"method": "ui.settle", "params": {"timeoutMs": 20000}}),
|
||||
json!({"method": "ui.screenshot"}),
|
||||
json!({"method": "engine.execute", "params": {"command": "dialog.export"}}),
|
||||
json!({"method": "ui.settle", "params": {"timeoutMs": 20000}}),
|
||||
json!({"method": "ui.screenshot", "params": {"path": b.to_str().unwrap()}}),
|
||||
),
|
||||
)
|
||||
@@ -168,7 +225,7 @@ fn snapshot_subcommand_renders_the_ui_headlessly() {
|
||||
.unwrap();
|
||||
assert!(o.status.success(), "{}", String::from_utf8_lossy(&o.stderr));
|
||||
let replies: Vec<Value> = String::from_utf8_lossy(&o.stdout).lines().map(|l| serde_json::from_str(l).unwrap()).collect();
|
||||
assert_eq!(replies.len(), 4);
|
||||
assert_eq!(replies.len(), 6);
|
||||
assert!(replies.iter().all(|r| r["ok"] == true), "{replies:?}");
|
||||
for (p, dimmed) in [(&a, false), (&b, true)] {
|
||||
let d = lightcraft_codecs::decode(&std::fs::read(p).unwrap(), Default::default()).unwrap();
|
||||
@@ -183,6 +240,124 @@ fn snapshot_subcommand_renders_the_ui_headlessly() {
|
||||
assert_eq!((d.width, d.height), (640, 480));
|
||||
}
|
||||
|
||||
/// Issue #166: a scripted request that fails makes `snapshot` exit non-zero. The failed reply is
|
||||
/// still printed, the later lines still run, and the final `-o` screenshot is still written.
|
||||
#[test]
|
||||
fn snapshot_script_failure_exits_non_zero() {
|
||||
let script = tmp("snap-fail.jsonl");
|
||||
let out = tmp("snap-fail.png");
|
||||
std::fs::write(
|
||||
&script,
|
||||
format!(
|
||||
"{}\n{}\n",
|
||||
json!({"method": "ui.set", "params": {"view": "grid"}}), // not a view: `photoGrid`, `detail`, …
|
||||
json!({"method": "ui.set", "params": {"view": "photoGrid"}}),
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
let _ = std::fs::remove_file(&out);
|
||||
let o = Command::new(BIN)
|
||||
.args(["snapshot", "--demo", "--script", script.to_str().unwrap(), "-o", out.to_str().unwrap(), "--size", "320x200"])
|
||||
.output()
|
||||
.unwrap();
|
||||
let stderr = String::from_utf8_lossy(&o.stderr);
|
||||
assert!(!o.status.success(), "a failed scripted request must fail the run\n{stderr}");
|
||||
assert!(stderr.contains("1 scripted request(s) failed"), "{stderr}");
|
||||
let replies: Vec<Value> = String::from_utf8_lossy(&o.stdout).lines().map(|l| serde_json::from_str(l).unwrap()).collect();
|
||||
assert_eq!(replies.len(), 2, "{replies:?}");
|
||||
assert_eq!(replies[0]["ok"], false, "{replies:?}");
|
||||
assert!(replies[0]["error"].as_str().unwrap_or("").contains("grid"), "{replies:?}");
|
||||
assert_eq!(replies[1]["ok"], true, "{replies:?}");
|
||||
// continue-and-report: the final screenshot is still written
|
||||
let d = lightcraft_codecs::decode(&std::fs::read(&out).unwrap(), Default::default()).unwrap();
|
||||
assert_eq!((d.width, d.height), (320, 200));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_ui_zoom_keeps_requested_pixel_dimensions() {
|
||||
let script = tmp("snap-ui-zoom.jsonl");
|
||||
std::fs::write(&script, format!("{}\n{}\n", json!({"method": "ui.zoomFactor", "params": {"factor": 1.1}}), json!({"method": "ui.inspect"})))
|
||||
.unwrap();
|
||||
for (size, scale, expected) in [("400x240", "1", (400, 240)), ("345x200", "0.9", (311, 180))] {
|
||||
let out = tmp(&format!("snap-ui-zoom-{scale}.png"));
|
||||
let o = Command::new(BIN)
|
||||
.args(["snapshot", "--script", script.to_str().unwrap(), "-o", out.to_str().unwrap(), "--size", size, "--scale", scale])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(o.status.success(), "{}", String::from_utf8_lossy(&o.stderr));
|
||||
let replies: Vec<Value> = String::from_utf8_lossy(&o.stdout).lines().map(|line| serde_json::from_str(line).unwrap()).collect();
|
||||
assert!(replies.iter().all(|reply| reply["ok"] == true), "{replies:?}");
|
||||
assert!(replies[1]["result"]["pixelsPerPoint"].as_f64().unwrap() > scale.parse::<f64>().unwrap());
|
||||
let decoded = lightcraft_codecs::decode(&std::fs::read(out).unwrap(), Default::default()).unwrap();
|
||||
assert_eq!((decoded.width, decoded.height), expected);
|
||||
}
|
||||
}
|
||||
|
||||
/// Headless snapshots default to CPU photo rendering as well as CPU UI rasterization.
|
||||
#[test]
|
||||
fn snapshot_defaults_to_cpu_without_gpu_environment_overrides() {
|
||||
let script = tmp("default-cpu.jsonl");
|
||||
std::fs::write(
|
||||
&script,
|
||||
format!("{}\n{}\n", json!({"method": "engine.execute", "params": {"command": "app.gpu"}}), json!({"method": "ui.screenshot"})),
|
||||
)
|
||||
.unwrap();
|
||||
let image = tmp("default-cpu.png");
|
||||
let output = Command::new(BIN)
|
||||
.env_remove("LIGHTCRAFT_GPU")
|
||||
.env_remove("LIGHTCRAFT_GPU_BACKEND")
|
||||
.env_remove("WGPU_BACKEND")
|
||||
.args(["snapshot", "--demo", "--script", script.to_str().unwrap(), "-o", image.to_str().unwrap(), "--size", "480x320"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(output.status.success(), "status {}: {}", output.status, String::from_utf8_lossy(&output.stderr));
|
||||
let replies: Vec<Value> = String::from_utf8_lossy(&output.stdout).lines().map(|line| serde_json::from_str(line).unwrap()).collect();
|
||||
assert!(replies.iter().all(|reply| reply["ok"] == true), "{replies:?}");
|
||||
let gpu = &replies[0]["result"];
|
||||
assert_eq!(gpu["enabled"], false, "{gpu}");
|
||||
assert_eq!(gpu["available"], false, "{gpu}");
|
||||
assert_eq!(gpu["adapter"], Value::Null, "no device is created: {gpu}");
|
||||
assert!(gpu["reason"].as_str().is_some_and(|reason| reason.contains("preference")), "{gpu}");
|
||||
let decoded = lightcraft_codecs::decode(&std::fs::read(image).unwrap(), Default::default()).unwrap();
|
||||
assert_eq!((decoded.width, decoded.height), (480, 320));
|
||||
}
|
||||
|
||||
/// Issue #136: with the GPU switched off from the environment the UI starts and renders on the CPU,
|
||||
/// without creating a GPU device (no driver is loaded), and says why.
|
||||
#[test]
|
||||
fn snapshot_starts_without_a_gpu() {
|
||||
let script = tmp("nogpu.jsonl");
|
||||
std::fs::write(
|
||||
&script,
|
||||
format!(
|
||||
"{}\n{}\n{}\n",
|
||||
json!({"method": "ui.set", "params": {"view": "photoGrid"}}),
|
||||
json!({"method": "engine.execute", "params": {"command": "app.gpu"}}),
|
||||
json!({"method": "ui.screenshot"}),
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
for (var, value, reason) in [("LIGHTCRAFT_GPU", "0", "LIGHTCRAFT_GPU=0"), ("LIGHTCRAFT_GPU_BACKEND", "off", "LIGHTCRAFT_GPU_BACKEND=off")] {
|
||||
let out = tmp(&format!("nogpu-{var}.png"));
|
||||
let o = Command::new(BIN)
|
||||
.env_remove("LIGHTCRAFT_GPU")
|
||||
.env_remove("LIGHTCRAFT_GPU_BACKEND")
|
||||
.env(var, value)
|
||||
.args(["snapshot", "--demo", "--script", script.to_str().unwrap(), "-o", out.to_str().unwrap(), "--size", "480x320"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(o.status.success(), "{var}: {}", String::from_utf8_lossy(&o.stderr));
|
||||
let replies: Vec<Value> = String::from_utf8_lossy(&o.stdout).lines().map(|l| serde_json::from_str(l).unwrap()).collect();
|
||||
assert!(replies.iter().all(|r| r["ok"] == true), "{replies:?}");
|
||||
let gpu = &replies[1]["result"];
|
||||
assert_eq!(gpu["available"], false, "{var}: {gpu}");
|
||||
assert_eq!(gpu["adapter"], Value::Null, "{var}: no device was created: {gpu}");
|
||||
assert!(gpu["reason"].as_str().is_some_and(|r| r.contains(reason)), "{var}: {gpu}");
|
||||
let d = lightcraft_codecs::decode(&std::fs::read(&out).unwrap(), Default::default()).unwrap();
|
||||
assert_eq!((d.width, d.height), (480, 320));
|
||||
}
|
||||
}
|
||||
|
||||
fn run_cli(args: &[&str], stdin: Option<&str>) -> (bool, Vec<Value>, String) {
|
||||
let mut child = Command::new(BIN).arg("run").args(args).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap();
|
||||
if let Some(text) = stdin {
|
||||
@@ -256,7 +431,149 @@ fn devices_are_listed_and_imported_from() {
|
||||
assert_eq!(dev["name"], "CARD", "{line}");
|
||||
// the import review of its DCIM folder finds the photo
|
||||
let dcim = dev["path"].as_str().unwrap().to_string();
|
||||
let o = Command::new(BIN).args(["run", "library.importPreview", &format!("paths=[\"{dcim}\"]")]).output().unwrap();
|
||||
// JSON-escaped: a Windows path has backslashes
|
||||
let paths = serde_json::to_string(&[&dcim]).unwrap();
|
||||
let o = Command::new(BIN).args(["run", "library.importPreview", &format!("paths={paths}")]).output().unwrap();
|
||||
let line: Value = serde_json::from_slice(o.stdout.split(|b| *b == b'\n').next().unwrap()).unwrap();
|
||||
assert_eq!(line["result"]["candidates"].as_array().map(Vec::len), Some(1), "{line}");
|
||||
}
|
||||
|
||||
/// Issue #99: a library open in one process (here `mcp --library`) is refused by a second one,
|
||||
/// with who has it and how to drive the running app instead; free again once the first exits.
|
||||
#[test]
|
||||
fn a_library_open_in_another_process_is_refused() {
|
||||
let lib = tmp("locked-lib");
|
||||
let _ = std::fs::remove_dir_all(&lib);
|
||||
let lib_s = lib.to_str().unwrap();
|
||||
let mut holder =
|
||||
Command::new(BIN).args(["mcp", "--library", lib_s]).stdin(Stdio::piped()).stdout(Stdio::null()).stderr(Stdio::piped()).spawn().unwrap();
|
||||
let mut err = BufReader::new(holder.stderr.take().unwrap());
|
||||
let mut line = String::new();
|
||||
while !line.contains("opened library") {
|
||||
line.clear();
|
||||
assert!(err.read_line(&mut line).unwrap() > 0, "mcp exited before opening the library");
|
||||
}
|
||||
let (ok, _, stderr) = run_cli(&["--library", lib_s, "library.info"], None);
|
||||
assert!(!ok);
|
||||
assert!(stderr.contains("already open in lightcraft-cli") && stderr.contains(&format!("process {}", holder.id())), "{stderr}");
|
||||
assert!(stderr.contains("mcp --connect"), "{stderr}");
|
||||
|
||||
drop(holder.stdin.take()); // EOF: the server exits and lets go of the library
|
||||
assert!(holder.wait().unwrap().success());
|
||||
let (ok, lines, stderr) = run_cli(&["--library", lib_s, "library.info"], None);
|
||||
assert!(ok, "{stderr}");
|
||||
assert_eq!(lines[0]["ok"], true);
|
||||
let _ = std::fs::remove_dir_all(&lib);
|
||||
}
|
||||
|
||||
/// Issue #167: when `--connect ADDR` cannot reach the app, the recovery hint must name the port
|
||||
/// ADDR actually names — following the old fixed "7980" left the retry connecting elsewhere.
|
||||
#[test]
|
||||
fn connect_failure_hint_names_the_attempted_port() {
|
||||
let (ok, _lines, err) = run_cli(&["--connect", "127.0.0.1:18437", "ui.inspect"], None);
|
||||
assert!(!ok);
|
||||
assert!(err.contains("--control 18437"), "the hint must name the port it tried: {err}");
|
||||
assert!(!err.contains("--control 7980"), "{err}");
|
||||
}
|
||||
|
||||
/// The CLI logs warnings on stderr (#168): with `RUST_LOG=warn` an unknown GPU backend name is
|
||||
/// reported, and `RUST_LOG=off` silences it. Before, no logger was installed and nothing appeared.
|
||||
#[test]
|
||||
fn warnings_are_logged_on_stderr() {
|
||||
let input = tmp("log-in.png");
|
||||
gradient_png(&input);
|
||||
let out = tmp("log-out.jpg");
|
||||
let run = |level: &str| {
|
||||
let o = Command::new(BIN)
|
||||
.args(["render", input.to_str().unwrap(), "-o", out.to_str().unwrap()])
|
||||
.env("LIGHTCRAFT_GPU_BACKEND", "bogus")
|
||||
.env("RUST_LOG", level)
|
||||
.env_remove("LIGHTCRAFT_LOG")
|
||||
.env_remove("LIGHTCRAFT_GPU")
|
||||
.output()
|
||||
.unwrap();
|
||||
let stderr = String::from_utf8_lossy(&o.stderr).into_owned();
|
||||
assert!(o.status.success(), "{stderr}");
|
||||
stderr
|
||||
};
|
||||
let warn = run("warn");
|
||||
assert!(warn.contains("LIGHTCRAFT_GPU_BACKEND=bogus names no known backend"), "{warn}");
|
||||
let off = run("off");
|
||||
assert!(!off.contains("names no known backend"), "{off}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_rejects_invalid_dimensions_before_creating_an_image() {
|
||||
for (index, (size, scale)) in
|
||||
[("NaNx100", "1"), ("100x100", "inf"), ("9000x9000", "1"), ("9000x100", "2"), ("100x100", "0.001")].into_iter().enumerate()
|
||||
{
|
||||
let image = tmp(&format!("invalid-viewport-{index}.png"));
|
||||
let output = Command::new(BIN).args(["snapshot", "--size", size, "--scale", scale, "-o"]).arg(&image).output().unwrap();
|
||||
assert!(!output.status.success());
|
||||
assert!(String::from_utf8_lossy(&output.stderr).contains("snapshot: bad --size/--scale"), "{}", String::from_utf8_lossy(&output.stderr));
|
||||
assert!(!image.exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_preserves_fractional_scale_rounding() {
|
||||
let image = tmp("fractional-scale-rounding.png");
|
||||
let output =
|
||||
Command::new(BIN).env("LIGHTCRAFT_GPU", "0").args(["snapshot", "--size", "345x200", "--scale", "0.9", "-o"]).arg(&image).output().unwrap();
|
||||
assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr));
|
||||
let decoded = lightcraft_codecs::decode(&std::fs::read(image).unwrap(), Default::default()).unwrap();
|
||||
assert_eq!((decoded.width, decoded.height), (311, 180));
|
||||
}
|
||||
|
||||
/// Snapshot's CPU setting must never replace the desktop preference, including with --library.
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn snapshot_leaves_desktop_gpu_preferences_untouched() {
|
||||
let dir = tmp("snapshot-desktop-preferences");
|
||||
let library = dir.join("library");
|
||||
let mut session = lightcraft_engine::Session::new().with_fs();
|
||||
session.open_library(&library, true).unwrap();
|
||||
drop(session);
|
||||
let config = dir.join("config");
|
||||
let app_config = config.join("lightcraft");
|
||||
std::fs::create_dir_all(&app_config).unwrap();
|
||||
let ui_path = app_config.join("ui.json");
|
||||
let mut ui = lightcraft_ui_egui::UiState::default();
|
||||
ui.settings.gpu = true;
|
||||
let saved = serde_json::to_vec_pretty(&ui).unwrap();
|
||||
std::fs::write(&ui_path, &saved).unwrap();
|
||||
let script = dir.join("preferences.jsonl");
|
||||
std::fs::write(
|
||||
&script,
|
||||
format!("{}\n{}\n", json!({"method": "engine.execute", "params": {"command": "app.gpu"}}), json!({"method": "ui.inspect"})),
|
||||
)
|
||||
.unwrap();
|
||||
for gpu_environment in [None, Some("1")] {
|
||||
let image = dir.join(format!("{}.png", gpu_environment.unwrap_or("default")));
|
||||
let mut cmd = Command::new(BIN);
|
||||
cmd.env("XDG_CONFIG_HOME", &config).env_remove("LIGHTCRAFT_GPU").env_remove("LIGHTCRAFT_GPU_BACKEND").env_remove("WGPU_BACKEND");
|
||||
if let Some(value) = gpu_environment {
|
||||
cmd.env("LIGHTCRAFT_GPU", value);
|
||||
}
|
||||
let output = cmd
|
||||
.args(["snapshot", "--library"])
|
||||
.arg(&library)
|
||||
.args(["--script"])
|
||||
.arg(&script)
|
||||
.args(["-o"])
|
||||
.arg(&image)
|
||||
.args(["--size", "480x320"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr));
|
||||
let replies: Vec<Value> = String::from_utf8_lossy(&output.stdout).lines().map(|line| serde_json::from_str(line).unwrap()).collect();
|
||||
assert!(replies.iter().all(|reply| reply["ok"] == true), "{replies:?}");
|
||||
assert_eq!(replies[0]["result"]["enabled"], false);
|
||||
assert_eq!(replies[0]["result"]["available"], false);
|
||||
assert_eq!(replies[0]["result"]["adapter"], Value::Null);
|
||||
assert_eq!(replies[1]["result"]["ui"]["settings"]["gpu"], false);
|
||||
assert_eq!(std::fs::read(&ui_path).unwrap(), saved, "snapshot changed the desktop preferences");
|
||||
assert!(!app_config.join("ui.json.tmp").exists());
|
||||
assert!(image.exists());
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ lightcraft-preview = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
log = { workspace = true }
|
||||
crc32fast = { workspace = true }
|
||||
|
||||
[target.'cfg(target_arch = "wasm32")'.dependencies]
|
||||
eframe = { workspace = true, features = ["default_fonts", "glow"] }
|
||||
@@ -70,3 +71,9 @@ web-sys = { workspace = true, features = [
|
||||
"WorkerType",
|
||||
"WritableStream",
|
||||
] }
|
||||
|
||||
[features]
|
||||
# Open HEIC/HEIF photos (the optional lightcraft-heif crate). Not default so distributors choose
|
||||
# it (HEVC patents); build with `--features heif` to include it. On wasm32 panics abort, so
|
||||
# heic-rs's `catch_unwind` guard does not apply there.
|
||||
heif = ["lightcraft-codecs/heif"]
|
||||
@@ -24,13 +24,21 @@
|
||||
<script type="module">
|
||||
import init, { start, command } from "./lightcraft_web.js";
|
||||
(async () => {
|
||||
// The Chinese font is a separate file: embedding it would exceed the host's WASM size limit.
|
||||
// Font-free development builds can still run, with the same missing-glyph warning as before.
|
||||
const chineseFont = fetch("./lightcraft_zh_hans.otf")
|
||||
.then(async response => response.ok ? new Uint8Array(await response.arrayBuffer()) : new Uint8Array())
|
||||
.catch(() => new Uint8Array());
|
||||
// Compile once; the render workers (worker.js) instantiate the same compiled module.
|
||||
const module = await WebAssembly.compileStreaming(fetch("./lightcraft_web_bg.wasm"));
|
||||
const [module, font] = await Promise.all([
|
||||
WebAssembly.compileStreaming(fetch("./lightcraft_web_bg.wasm")),
|
||||
chineseFont,
|
||||
]);
|
||||
globalThis.lightcraftModule = module;
|
||||
await init({ module_or_path: module });
|
||||
// automation: `await lightcraft.command("library.info", "{}")` → JSON text
|
||||
globalThis.lightcraft = { command };
|
||||
start();
|
||||
start(font);
|
||||
})().catch((e) => {
|
||||
console.error(e);
|
||||
const el = document.getElementById("lightcraft_loading");
|
||||
|
||||
@@ -232,6 +232,18 @@ impl Backend {
|
||||
}
|
||||
}
|
||||
|
||||
/// Is there a file at `path`?
|
||||
pub async fn exists(&self, path: &str) -> Result<bool, String> {
|
||||
match self {
|
||||
Backend::Opfs(root) => Ok(Self::opfs_file(root, path, false).await?.is_some()),
|
||||
Backend::Idb(db) => {
|
||||
let (_tx, store) = Self::idb_store(db, false)?;
|
||||
let n = idb_req(&store.count_with_key(&path.into()).map_err(err)?).await?;
|
||||
Ok(n.as_f64().is_some_and(|n| n > 0.0))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn remove(&self, path: &str) -> Result<(), String> {
|
||||
match self {
|
||||
Backend::Opfs(root) => {
|
||||
@@ -309,14 +321,25 @@ async fn dir_keys(dir: &FileSystemDirectoryHandle) -> Result<Vec<String>, String
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Ask the browser not to evict our storage under pressure (best effort; fire and forget).
|
||||
pub fn request_persistence() {
|
||||
if let Some(sm) = storage_manager()
|
||||
&& let Ok(p) = sm.persist()
|
||||
{
|
||||
wasm_bindgen_futures::spawn_local(async move {
|
||||
let granted = JsFuture::from(p).await.ok().is_some_and(|v| v.is_truthy());
|
||||
log::info!("lightcraft: persistent storage {}", if granted { "granted" } else { "not granted (best effort)" });
|
||||
});
|
||||
}
|
||||
/// Ask the browser not to evict our storage under pressure; `on_result(granted)` once known
|
||||
/// (`false` also when the browser has no storage manager).
|
||||
pub fn request_persistence(on_result: impl FnOnce(bool) + 'static) {
|
||||
let Some(sm) = storage_manager() else {
|
||||
on_result(false);
|
||||
return;
|
||||
};
|
||||
wasm_bindgen_futures::spawn_local(async move {
|
||||
// already persistent: nothing to ask
|
||||
let already = match sm.persisted() {
|
||||
Ok(p) => JsFuture::from(p).await.ok().is_some_and(|v| v.is_truthy()),
|
||||
Err(_) => false,
|
||||
};
|
||||
let granted = already
|
||||
|| match sm.persist() {
|
||||
Ok(p) => JsFuture::from(p).await.ok().is_some_and(|v| v.is_truthy()),
|
||||
Err(_) => false,
|
||||
};
|
||||
log::info!("lightcraft: persistent storage {}", if granted { "granted" } else { "not granted" });
|
||||
on_result(granted);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
//! Library backups for the browser build: a plain zip (entries stored, not compressed — photos
|
||||
//! don't shrink) of the library files and every stored original, built one entry at a time so
|
||||
//! only one file is in wasm memory at once, and read back the same way for a restore.
|
||||
//!
|
||||
//! Layout of a backup:
|
||||
//! - `library/catalog.snap`, `library/catalog.log`, `library/presets.json`, … (see
|
||||
//! [`crate::files::LIBRARY_FILES`]);
|
||||
//! - `originals/<content hash>/<file name>`: each imported photo's original bytes, under its own
|
||||
//! name so an unzipped backup is browsable;
|
||||
//! - `README.txt`.
|
||||
//!
|
||||
//! Restoring never deletes anything: the library files go to a new library folder in browser
|
||||
//! storage, originals that aren't stored yet are added (they're named by content, so existing
|
||||
//! ones are identical; a restored one must match its content hash, see [`verify_original`]),
|
||||
//! and only then does a small pointer file ([`ACTIVE_LIBRARY`]) switch to the restored library.
|
||||
//! The library that was there before stays in storage.
|
||||
//!
|
||||
//! No zip64: a backup holds at most [`MAX_BYTES`] and 65 535 entries.
|
||||
|
||||
/// Storage key of the file naming the library folder in use (absent: `library`).
|
||||
pub const ACTIVE_LIBRARY: &str = "active-library";
|
||||
|
||||
/// The default library folder.
|
||||
pub const DEFAULT_LIBRARY_DIR: &str = "library";
|
||||
|
||||
/// Largest backup (offsets are 32-bit without zip64).
|
||||
pub const MAX_BYTES: u64 = 0xFFFF_FFFF - (64 << 20);
|
||||
|
||||
const LOCAL_SIG: u32 = 0x0403_4b50;
|
||||
const CENTRAL_SIG: u32 = 0x0201_4b50;
|
||||
const END_SIG: u32 = 0x0605_4b50;
|
||||
/// General-purpose flag bit 11: names are UTF-8.
|
||||
const UTF8: u16 = 0x0800;
|
||||
/// 1980-01-01 00:00 in MS-DOS format (date, time).
|
||||
const DOS_DATE: u16 = (1 << 5) | 1;
|
||||
|
||||
/// The readme inside every backup.
|
||||
pub const README: &str = "LightCraft library backup (browser version)\n\n\
|
||||
library/ the catalog (catalog.snap + catalog.log), presets, view state and preferences\n\
|
||||
originals/ every imported photo, as originals/<content hash>/<file name>\n\n\
|
||||
Restore it in LightCraft for the web with File > Restore Library from Backup...\n\
|
||||
The photos in originals/ are the untouched files you imported; edits live in the catalog.\n";
|
||||
|
||||
/// Is `name` a safe library folder name (`library` or `library-<letters, digits, ->`)?
|
||||
pub fn valid_library_dir(name: &str) -> bool {
|
||||
name == DEFAULT_LIBRARY_DIR
|
||||
|| name.strip_prefix("library-").is_some_and(|s| !s.is_empty() && s.len() <= 64 && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '-'))
|
||||
}
|
||||
|
||||
/// Is `h` a plausible content hash (used as a storage key segment)?
|
||||
pub fn valid_hash(h: &str) -> bool {
|
||||
!h.is_empty() && h.len() <= 128 && h.chars().all(|c| c.is_ascii_alphanumeric())
|
||||
}
|
||||
|
||||
/// Zip entry name for an original.
|
||||
pub fn original_entry(hash: &str, file_name: &str) -> String {
|
||||
let name: String = file_name.chars().map(|c| if c == '/' || c == '\\' || c.is_control() { '_' } else { c }).collect();
|
||||
let name = if name.is_empty() || name == "." || name == ".." { "original".to_string() } else { name };
|
||||
format!("originals/{hash}/{name}")
|
||||
}
|
||||
|
||||
/// Where a backup entry goes when restoring into library folder `dir`: `Some(storage key)`, or
|
||||
/// `None` to skip it (README, thumbnails, anything unknown or unsafe).
|
||||
pub fn restore_key(entry: &str, dir: &str) -> Option<String> {
|
||||
if let Some(name) = entry.strip_prefix("library/") {
|
||||
return crate::files::LIBRARY_FILES.contains(&name).then(|| format!("{dir}/{name}"));
|
||||
}
|
||||
let rest = entry.strip_prefix("originals/")?;
|
||||
let (hash, _) = rest.split_once('/').unwrap_or((rest, ""));
|
||||
valid_hash(hash).then(|| format!("originals/{hash}"))
|
||||
}
|
||||
|
||||
fn u16le(v: &mut Vec<u8>, x: u16) {
|
||||
v.extend_from_slice(&x.to_le_bytes());
|
||||
}
|
||||
fn u32le(v: &mut Vec<u8>, x: u32) {
|
||||
v.extend_from_slice(&x.to_le_bytes());
|
||||
}
|
||||
|
||||
struct Central {
|
||||
name: String,
|
||||
crc: u32,
|
||||
size: u32,
|
||||
offset: u32,
|
||||
}
|
||||
|
||||
/// Writes a stored (uncompressed) zip piece by piece: [`ZipWriter::entry`] returns the local
|
||||
/// header to emit before the entry's bytes, [`ZipWriter::finish`] the central directory.
|
||||
#[derive(Default)]
|
||||
pub struct ZipWriter {
|
||||
entries: Vec<Central>,
|
||||
offset: u64,
|
||||
}
|
||||
|
||||
impl ZipWriter {
|
||||
/// The local header for `name` holding `data` (emit it, then `data`).
|
||||
pub fn entry(&mut self, name: &str, data: &[u8]) -> Result<Vec<u8>, String> {
|
||||
let size = u32::try_from(data.len()).map_err(|_| format!("{name}: larger than 4 GB"))?;
|
||||
let name_len = u16::try_from(name.len()).map_err(|_| format!("{name}: name too long"))?;
|
||||
if self.entries.len() >= usize::from(u16::MAX) {
|
||||
return Err("too many files for one backup".into());
|
||||
}
|
||||
let header_len = 30 + u64::from(name_len);
|
||||
if self.offset + header_len + u64::from(size) > MAX_BYTES {
|
||||
return Err("the library is too large for one backup file (4 GB)".into());
|
||||
}
|
||||
let offset = u32::try_from(self.offset).map_err(|_| "backup too large".to_string())?;
|
||||
let crc = crc32fast::hash(data);
|
||||
let mut h = Vec::with_capacity(header_len as usize);
|
||||
u32le(&mut h, LOCAL_SIG);
|
||||
u16le(&mut h, 20); // version needed
|
||||
u16le(&mut h, UTF8);
|
||||
u16le(&mut h, 0); // stored
|
||||
u16le(&mut h, 0); // time
|
||||
u16le(&mut h, DOS_DATE);
|
||||
u32le(&mut h, crc);
|
||||
u32le(&mut h, size);
|
||||
u32le(&mut h, size);
|
||||
u16le(&mut h, name_len);
|
||||
u16le(&mut h, 0); // extra
|
||||
h.extend_from_slice(name.as_bytes());
|
||||
self.offset += header_len + u64::from(size);
|
||||
self.entries.push(Central { name: name.to_string(), crc, size, offset });
|
||||
Ok(h)
|
||||
}
|
||||
|
||||
/// The central directory and end record (emit last).
|
||||
pub fn finish(self) -> Result<Vec<u8>, String> {
|
||||
let mut cd = Vec::new();
|
||||
for e in &self.entries {
|
||||
u32le(&mut cd, CENTRAL_SIG);
|
||||
u16le(&mut cd, 20); // made by
|
||||
u16le(&mut cd, 20); // needed
|
||||
u16le(&mut cd, UTF8);
|
||||
u16le(&mut cd, 0);
|
||||
u16le(&mut cd, 0);
|
||||
u16le(&mut cd, DOS_DATE);
|
||||
u32le(&mut cd, e.crc);
|
||||
u32le(&mut cd, e.size);
|
||||
u32le(&mut cd, e.size);
|
||||
u16le(&mut cd, e.name.len() as u16);
|
||||
u16le(&mut cd, 0); // extra
|
||||
u16le(&mut cd, 0); // comment
|
||||
u16le(&mut cd, 0); // disk
|
||||
u16le(&mut cd, 0); // internal attrs
|
||||
u32le(&mut cd, 0); // external attrs
|
||||
u32le(&mut cd, e.offset);
|
||||
cd.extend_from_slice(e.name.as_bytes());
|
||||
}
|
||||
let cd_offset = u32::try_from(self.offset).map_err(|_| "backup too large".to_string())?;
|
||||
let cd_size = u32::try_from(cd.len()).map_err(|_| "backup too large".to_string())?;
|
||||
let n = self.entries.len() as u16;
|
||||
let mut out = cd;
|
||||
u32le(&mut out, END_SIG);
|
||||
u16le(&mut out, 0);
|
||||
u16le(&mut out, 0);
|
||||
u16le(&mut out, n);
|
||||
u16le(&mut out, n);
|
||||
u32le(&mut out, cd_size);
|
||||
u32le(&mut out, cd_offset);
|
||||
u16le(&mut out, 0); // comment
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// One entry of a zip's central directory.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ZipEntry {
|
||||
pub name: String,
|
||||
pub crc: u32,
|
||||
pub size: u64,
|
||||
/// Offset of the entry's local header.
|
||||
pub header: u64,
|
||||
/// 0 = stored (the only method a restore reads).
|
||||
pub method: u16,
|
||||
}
|
||||
|
||||
fn rd16(b: &[u8], at: usize) -> Option<u16> {
|
||||
Some(u16::from_le_bytes(b.get(at..at + 2)?.try_into().ok()?))
|
||||
}
|
||||
fn rd32(b: &[u8], at: usize) -> Option<u32> {
|
||||
Some(u32::from_le_bytes(b.get(at..at + 4)?.try_into().ok()?))
|
||||
}
|
||||
|
||||
/// How many bytes from the end of a zip to read to find its end record.
|
||||
pub fn tail_len(file_len: u64) -> u64 {
|
||||
file_len.min(22 + 65_535)
|
||||
}
|
||||
|
||||
/// The central directory's (offset, size) from the last [`tail_len`] bytes of a zip.
|
||||
pub fn find_central(tail: &[u8]) -> Result<(u64, u64), String> {
|
||||
let at = (0..tail.len().saturating_sub(21)).rev().find(|&i| rd32(tail, i) == Some(END_SIG)).ok_or("not a zip file (no end record)")?;
|
||||
let size = rd32(tail, at + 12).ok_or("truncated zip")?;
|
||||
let offset = rd32(tail, at + 16).ok_or("truncated zip")?;
|
||||
if size == u32::MAX || offset == u32::MAX {
|
||||
return Err("zip64 backups are not supported".into());
|
||||
}
|
||||
Ok((u64::from(offset), u64::from(size)))
|
||||
}
|
||||
|
||||
/// Parse a central directory.
|
||||
pub fn parse_central(cd: &[u8]) -> Result<Vec<ZipEntry>, String> {
|
||||
let mut out = Vec::new();
|
||||
let mut i = 0;
|
||||
while i + 46 <= cd.len() {
|
||||
if rd32(cd, i) != Some(CENTRAL_SIG) {
|
||||
return Err("damaged zip directory".into());
|
||||
}
|
||||
let bad = || "damaged zip directory".to_string();
|
||||
let method = rd16(cd, i + 10).ok_or_else(bad)?;
|
||||
let crc = rd32(cd, i + 16).ok_or_else(bad)?;
|
||||
let size = rd32(cd, i + 24).ok_or_else(bad)?;
|
||||
let n = usize::from(rd16(cd, i + 28).ok_or_else(bad)?);
|
||||
let extra = usize::from(rd16(cd, i + 30).ok_or_else(bad)?);
|
||||
let comment = usize::from(rd16(cd, i + 32).ok_or_else(bad)?);
|
||||
let header = rd32(cd, i + 42).ok_or_else(bad)?;
|
||||
let name = cd.get(i + 46..i + 46 + n).ok_or_else(bad)?;
|
||||
out.push(ZipEntry { name: String::from_utf8_lossy(name).into_owned(), crc, size: u64::from(size), header: u64::from(header), method });
|
||||
i += 46 + n + extra + comment;
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Offset of an entry's data from its local header's first 30 bytes.
|
||||
pub fn data_offset(entry: &ZipEntry, local: &[u8]) -> Result<u64, String> {
|
||||
if rd32(local, 0) != Some(LOCAL_SIG) {
|
||||
return Err(format!("{}: damaged entry", entry.name));
|
||||
}
|
||||
let n = rd16(local, 26).ok_or("truncated entry")?;
|
||||
let extra = rd16(local, 28).ok_or("truncated entry")?;
|
||||
Ok(entry.header + 30 + u64::from(n) + u64::from(extra))
|
||||
}
|
||||
|
||||
/// Check an entry's bytes.
|
||||
pub fn verify(entry: &ZipEntry, data: &[u8]) -> Result<(), String> {
|
||||
if entry.method != 0 {
|
||||
return Err(format!("{}: compressed entries are not supported (not a LightCraft backup?)", entry.name));
|
||||
}
|
||||
if data.len() as u64 != entry.size || crc32fast::hash(data) != entry.crc {
|
||||
return Err(format!("{}: damaged (checksum mismatch)", entry.name));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check that an original's bytes are the photo its storage key names: originals are stored
|
||||
/// under their content hash, and a restore that kept bytes under another photo's hash would
|
||||
/// give that photo's catalog record a different image (the zip checksum can't tell).
|
||||
pub fn verify_original(entry: &ZipEntry, hash: &str, data: &[u8]) -> Result<(), String> {
|
||||
if crate::store::content_hash(data) != hash {
|
||||
return Err(format!("{}: the file doesn't match its content hash (damaged or altered backup)", entry.name));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A name for the restored library's folder.
|
||||
pub fn restored_dir_name(now_ms: f64) -> String {
|
||||
format!("library-restored-{}", now_ms.max(0.0) as u64)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn build(files: &[(&str, &[u8])]) -> Vec<u8> {
|
||||
let mut w = ZipWriter::default();
|
||||
let mut out = Vec::new();
|
||||
for (n, d) in files {
|
||||
out.extend(w.entry(n, d).unwrap());
|
||||
out.extend_from_slice(d);
|
||||
}
|
||||
out.extend(w.finish().unwrap());
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn written_backups_read_back() {
|
||||
let files: [(&str, &[u8]); 3] =
|
||||
[("README.txt", README.as_bytes()), ("library/catalog.log", b"{\"seq\":1}\n"), ("originals/abc123/IMG 1.jpg", &[0, 1, 2, 3, 255])];
|
||||
let zip = build(&files);
|
||||
let tail = &zip[zip.len() - tail_len(zip.len() as u64) as usize..];
|
||||
let (off, size) = find_central(tail).unwrap();
|
||||
let entries = parse_central(&zip[off as usize..(off + size) as usize]).unwrap();
|
||||
assert_eq!(entries.len(), 3);
|
||||
for (e, (name, data)) in entries.iter().zip(files) {
|
||||
assert_eq!(e.name, name);
|
||||
let start = data_offset(e, &zip[e.header as usize..e.header as usize + 30]).unwrap() as usize;
|
||||
let got = &zip[start..start + e.size as usize];
|
||||
assert_eq!(got, data);
|
||||
verify(e, got).unwrap();
|
||||
}
|
||||
// damage is caught
|
||||
let e = &entries[2];
|
||||
assert!(verify(e, &[0, 1, 2, 3, 254]).is_err());
|
||||
assert!(find_central(b"not a zip at all, just some text").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn originals_must_match_their_content_hash() {
|
||||
let red: &[u8] = b"red corner photo";
|
||||
let blue: &[u8] = b"blue corner photo";
|
||||
let hash = crate::store::content_hash(red);
|
||||
let name = original_entry(&hash, "IMG 1.png");
|
||||
let key = restore_key(&name, "d").unwrap();
|
||||
let key_hash = key.strip_prefix("originals/").unwrap();
|
||||
assert_eq!(key_hash, hash);
|
||||
// the entry's own bytes pass; other bytes under the same name (valid zip CRC) don't
|
||||
let zip = build(&[(name.as_str(), blue)]);
|
||||
let tail = &zip[zip.len() - tail_len(zip.len() as u64) as usize..];
|
||||
let (off, size) = find_central(tail).unwrap();
|
||||
let entries = parse_central(&zip[off as usize..(off + size) as usize]).unwrap();
|
||||
let entry = &entries[0];
|
||||
verify(entry, blue).unwrap();
|
||||
let err = verify_original(entry, key_hash, blue).unwrap_err();
|
||||
assert!(err.contains(&name), "{err}");
|
||||
verify_original(entry, key_hash, red).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restore_keys_are_safe() {
|
||||
assert_eq!(restore_key("library/catalog.snap", "library-restored-1").as_deref(), Some("library-restored-1/catalog.snap"));
|
||||
assert_eq!(restore_key("library/../../x", "library-r"), None);
|
||||
assert_eq!(restore_key("library/evil.bin", "library-r"), None);
|
||||
assert_eq!(restore_key("originals/abc123/IMG.jpg", "d").as_deref(), Some("originals/abc123"));
|
||||
assert_eq!(restore_key("originals/../library/catalog.log", "d"), None);
|
||||
assert_eq!(restore_key("README.txt", "d"), None);
|
||||
assert_eq!(restore_key("thumbs/x.jpg", "d"), None);
|
||||
assert_eq!(original_entry("abc", "a/b\\c.jpg"), "originals/abc/a_b_c.jpg");
|
||||
assert_eq!(original_entry("abc", ".."), "originals/abc/original");
|
||||
assert!(valid_library_dir("library") && valid_library_dir("library-restored-17"));
|
||||
assert!(!valid_library_dir("library-../x") && !valid_library_dir("originals") && !valid_library_dir("library-"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_limits_are_errors_not_panics() {
|
||||
let mut w = ZipWriter { offset: MAX_BYTES - 10, ..Default::default() };
|
||||
assert!(w.entry("x", &[0; 64]).is_err());
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,8 @@ struct Mirror {
|
||||
files: BTreeMap<String, Vec<u8>>,
|
||||
/// Dirty files, least recently modified first.
|
||||
dirty: Vec<(String, Dirty)>,
|
||||
/// Why the last flush to browser storage failed (quota, eviction…), until one succeeds.
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
impl Mirror {
|
||||
@@ -138,6 +140,18 @@ impl Files {
|
||||
}
|
||||
}
|
||||
|
||||
/// Record the outcome of a flush to browser storage: `Some(why)` when it failed, `None` once
|
||||
/// one succeeds. While a flush is failing, the catalog [`Store`] refuses new writes, so the
|
||||
/// engine keeps the changes queued and reports them unsaved (`library.info` → `unsavedOps`,
|
||||
/// the top bar's warning) instead of pretending they were saved.
|
||||
pub fn set_error(&self, e: Option<String>) {
|
||||
self.lock().error = e;
|
||||
}
|
||||
|
||||
pub fn error(&self) -> Option<String> {
|
||||
self.lock().error.clone()
|
||||
}
|
||||
|
||||
/// A catalog [`Store`] over these files.
|
||||
pub fn store(&self) -> FilesStore {
|
||||
FilesStore(self.clone())
|
||||
@@ -147,19 +161,32 @@ impl Files {
|
||||
/// [`Store`] implementation over [`Files`].
|
||||
pub struct FilesStore(Files);
|
||||
|
||||
impl FilesStore {
|
||||
/// Writes fail while browser storage does (see [`Files::set_error`]).
|
||||
fn writable(&self) -> io::Result<()> {
|
||||
match self.0.error() {
|
||||
Some(e) => Err(io::Error::other(format!("browser storage: {e}"))),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Store for FilesStore {
|
||||
fn read(&mut self, name: &str) -> io::Result<Option<Vec<u8>>> {
|
||||
Ok(self.0.get(name))
|
||||
}
|
||||
fn write_atomic(&mut self, name: &str, data: &[u8]) -> io::Result<()> {
|
||||
self.writable()?;
|
||||
self.0.write(name, data);
|
||||
Ok(())
|
||||
}
|
||||
fn append(&mut self, name: &str, data: &[u8]) -> io::Result<()> {
|
||||
self.writable()?;
|
||||
self.0.append(name, data);
|
||||
Ok(())
|
||||
}
|
||||
fn truncate(&mut self, name: &str, len: u64) -> io::Result<()> {
|
||||
self.writable()?;
|
||||
self.0.truncate(name, len as usize);
|
||||
Ok(())
|
||||
}
|
||||
@@ -224,6 +251,37 @@ mod tests {
|
||||
assert_eq!(names, ["catalog.snap", "catalog.log"]);
|
||||
}
|
||||
|
||||
/// A failing flush (quota exceeded, storage evicted) is not silent: commands report the change
|
||||
/// as unsaved and the engine keeps it queued; once storage works again it is written.
|
||||
#[test]
|
||||
fn flush_failures_show_as_unsaved_and_are_retried() {
|
||||
let mut disk = BTreeMap::new();
|
||||
let files = Files::default();
|
||||
let mut s = Session::new();
|
||||
let stores = LibraryStores { dir: "browser".into(), catalog: Box::new(files.store()), files: Box::new(files.store()), on_disk: false };
|
||||
s.open_library_in(stores, true).unwrap();
|
||||
flush(&files, &mut disk);
|
||||
files.set_error(Some("QuotaExceededError".into()));
|
||||
let r = s.execute("photo.rate", &json!({"rating": 5}));
|
||||
assert!(matches!(r, Err(lightcraft_engine::EngineError::NotSaved(_))), "{r:?}");
|
||||
let (ops, why) = s.unsaved().unwrap();
|
||||
assert!(ops > 0 && why.contains("QuotaExceeded"), "{why}");
|
||||
files.set_error(None);
|
||||
s.persist_if_dirty();
|
||||
s.persist().unwrap();
|
||||
assert!(s.unsaved().is_none(), "written once storage works again");
|
||||
flush(&files, &mut disk);
|
||||
let id = s.selection.active.unwrap();
|
||||
let files2 = Files::default();
|
||||
for (k, v) in &disk {
|
||||
files2.preload(k, v.clone());
|
||||
}
|
||||
let mut s2 = Session::new();
|
||||
let stores = LibraryStores { dir: "browser".into(), catalog: Box::new(files2.store()), files: Box::new(files2.store()), on_disk: false };
|
||||
s2.open_library_in(stores, true).unwrap();
|
||||
assert_eq!(s2.catalog.photo(id).unwrap().rating, 5, "the rating survived");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn library_survives_reload_through_flushes() {
|
||||
let mut disk = BTreeMap::new();
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
pub mod backup;
|
||||
pub mod bench;
|
||||
pub mod files;
|
||||
pub mod store;
|
||||
@@ -24,6 +25,8 @@ pub mod wire;
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
mod backend;
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
mod safety;
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
mod web;
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
mod workers;
|
||||
@@ -0,0 +1,275 @@
|
||||
//! Keeping a library in the browser safe: notices the user must see (a failed save, storage the
|
||||
//! browser may evict, a file that couldn't be stored), one tab at a time per library, a message
|
||||
//! instead of a dead page after a panic, and backup / restore of the whole library
|
||||
//! ([`crate::backup`]).
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use js_sys::{Reflect, Uint8Array};
|
||||
use wasm_bindgen::JsCast;
|
||||
use wasm_bindgen::prelude::*;
|
||||
use wasm_bindgen_futures::JsFuture;
|
||||
|
||||
use crate::backend::Backend;
|
||||
use crate::backup::{
|
||||
ACTIVE_LIBRARY, README, ZipWriter, data_offset, find_central, original_entry, parse_central, restore_key, restored_dir_name, tail_len,
|
||||
valid_hash, verify, verify_original,
|
||||
};
|
||||
use crate::files::{Files, LIBRARY_FILES};
|
||||
use crate::store::storage_key;
|
||||
|
||||
thread_local! {
|
||||
/// Messages for the user, shown by the app on its next frame.
|
||||
static NOTICES: RefCell<Vec<String>> = const { RefCell::new(Vec::new()) };
|
||||
}
|
||||
|
||||
/// Queue a message for the user (shown for several seconds; also logged).
|
||||
pub fn notice(text: impl Into<String>) {
|
||||
let text = text.into();
|
||||
log::warn!("{text}");
|
||||
NOTICES.with(|n| n.borrow_mut().push(text));
|
||||
crate::web::request_repaint();
|
||||
}
|
||||
|
||||
/// Messages queued since the last call.
|
||||
pub fn take_notices() -> Vec<String> {
|
||||
NOTICES.with(|n| std::mem::take(&mut *n.borrow_mut()))
|
||||
}
|
||||
|
||||
fn document() -> Option<web_sys::Document> {
|
||||
web_sys::window()?.document()
|
||||
}
|
||||
|
||||
/// Ask the user (the browser's own confirm box). `false` when it can't be shown.
|
||||
pub fn confirm(text: &str) -> bool {
|
||||
web_sys::window().and_then(|w| w.confirm_with_message(text).ok()).unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Cover the page with a message (the app can't continue, or must not start).
|
||||
pub fn show_blocking(text: &str) {
|
||||
let Some(doc) = document() else { return };
|
||||
let el = match doc.get_element_by_id("lightcraft_blocking") {
|
||||
Some(el) => el,
|
||||
None => {
|
||||
let Ok(el) = doc.create_element("div") else { return };
|
||||
el.set_id("lightcraft_blocking");
|
||||
let _ = el.set_attribute(
|
||||
"style",
|
||||
"position:fixed;inset:0;z-index:10;display:flex;align-items:center;justify-content:center;padding:32px;\
|
||||
background:#1c1c1c;color:#e0e0e0;font:15px/1.5 system-ui,-apple-system,'Segoe UI',sans-serif;text-align:center;white-space:pre-line",
|
||||
);
|
||||
if let Some(body) = doc.body() {
|
||||
let _ = body.append_child(&el);
|
||||
}
|
||||
el
|
||||
}
|
||||
};
|
||||
el.set_text_content(Some(text));
|
||||
if let Some(loading) = doc.get_element_by_id("lightcraft_loading") {
|
||||
loading.remove();
|
||||
}
|
||||
}
|
||||
|
||||
/// `panic = "abort"` on wasm: a panic ends the module. Say so on the page instead of leaving a
|
||||
/// frozen canvas, and say that the stored library is intact.
|
||||
pub fn install_panic_hook() {
|
||||
std::panic::set_hook(Box::new(|info| {
|
||||
let msg = match (info.payload().downcast_ref::<&str>(), info.payload().downcast_ref::<String>()) {
|
||||
(Some(s), _) => (*s).to_string(),
|
||||
(_, Some(s)) => s.clone(),
|
||||
_ => "unknown error".into(),
|
||||
};
|
||||
let at = info.location().map(|l| format!(" at {}:{}", l.file(), l.line())).unwrap_or_default();
|
||||
log::error!("LightCraft panicked{at}: {msg}");
|
||||
show_blocking(&format!(
|
||||
"LightCraft stopped after an internal error:\n{msg}\n\nThe library saved in this browser is kept. Reload the page to continue.\n\
|
||||
If it happens again right away, a photo may be the cause: reload with ?workers=0 or report the error."
|
||||
));
|
||||
}));
|
||||
}
|
||||
|
||||
/// The name of the lock that keeps a library to one tab.
|
||||
const TAB_LOCK: &str = "lightcraft-library";
|
||||
|
||||
/// Hold the library lock for as long as this page lives (Web Locks API). `Some(true)`: ours;
|
||||
/// `Some(false)`: another tab of this site has it; `None`: the browser can't tell (no Web Locks, or
|
||||
/// it refused the request).
|
||||
pub async fn acquire_tab_lock() -> Option<bool> {
|
||||
let nav = Reflect::get(&js_sys::global(), &"navigator".into()).ok()?;
|
||||
let locks = Reflect::get(&nav, &"locks".into()).ok().filter(|l| !l.is_undefined() && !l.is_null())?;
|
||||
let request: js_sys::Function = Reflect::get(&locks, &"request".into()).ok()?.dyn_into().ok()?;
|
||||
let opts = js_sys::Object::new();
|
||||
Reflect::set(&opts, &"ifAvailable".into(), &true.into()).ok()?;
|
||||
let granted = js_sys::Promise::new(&mut |resolve, reject| {
|
||||
let cb = Closure::once_into_js(move |lock: JsValue| -> JsValue {
|
||||
let ours = !lock.is_null() && !lock.is_undefined();
|
||||
let _ = resolve.call1(&JsValue::NULL, &ours.into());
|
||||
// keep the lock until the page goes away: a promise that never settles
|
||||
if ours { js_sys::Promise::new(&mut |_, _| {}).into() } else { JsValue::NULL }
|
||||
});
|
||||
// a refused request never calls `cb`: settle with its error instead of waiting forever
|
||||
match request.call3(&locks, &TAB_LOCK.into(), &opts, &cb) {
|
||||
Ok(pending) => {
|
||||
let catch = Reflect::get(&pending, &"catch".into()).ok().and_then(|c| c.dyn_into::<js_sys::Function>().ok());
|
||||
if let Some(catch) = catch {
|
||||
let _ = catch.call1(&pending, &reject);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = reject.call1(&JsValue::NULL, &e);
|
||||
}
|
||||
}
|
||||
});
|
||||
match JsFuture::from(granted).await {
|
||||
Ok(v) => Some(v.is_truthy()),
|
||||
Err(e) => {
|
||||
log::warn!("Web Locks request failed, starting without the one-tab guard: {e:?}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Offer a blob as a download named `name`.
|
||||
pub fn download_blob(name: &str, blob: &web_sys::Blob) -> Result<(), String> {
|
||||
let e = |e: JsValue| format!("download failed: {e:?}");
|
||||
let url = web_sys::Url::create_object_url_with_blob(blob).map_err(e)?;
|
||||
let doc = document().ok_or("no document")?;
|
||||
let a: web_sys::HtmlAnchorElement = doc.create_element("a").map_err(e)?.unchecked_into();
|
||||
a.set_href(&url);
|
||||
a.set_download(name);
|
||||
a.click();
|
||||
// revoke once the download has started
|
||||
let revoke = Closure::once_into_js(move || {
|
||||
let _ = web_sys::Url::revoke_object_url(&url);
|
||||
});
|
||||
if let Some(w) = web_sys::window() {
|
||||
let _ = w.set_timeout_with_callback_and_timeout_and_arguments_0(revoke.unchecked_ref(), 60_000);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// File ▸ Back Up Library…: a zip of the library files (as the app has them, saved or not) and
|
||||
/// every original in storage, built entry by entry into the browser's memory (outside the wasm
|
||||
/// heap) and downloaded. `names`: content hash → the photo's file name.
|
||||
pub async fn backup(files: Files, backend: Backend, names: HashMap<String, String>) -> Result<(usize, f64), String> {
|
||||
let parts = js_sys::Array::new();
|
||||
let mut zip = ZipWriter::default();
|
||||
let mut bytes = 0f64;
|
||||
let mut add = |zip: &mut ZipWriter, name: &str, data: &[u8]| -> Result<(), String> {
|
||||
let header = zip.entry(name, data)?;
|
||||
parts.push(&Uint8Array::from(&header[..]));
|
||||
parts.push(&Uint8Array::from(data));
|
||||
bytes += (header.len() + data.len()) as f64;
|
||||
Ok(())
|
||||
};
|
||||
add(&mut zip, "README.txt", README.as_bytes())?;
|
||||
for name in LIBRARY_FILES {
|
||||
if let Some(d) = files.get(name) {
|
||||
add(&mut zip, &format!("library/{name}"), &d)?;
|
||||
}
|
||||
}
|
||||
let mut originals = 0;
|
||||
for hash in backend.list("originals").await? {
|
||||
if !valid_hash(&hash) {
|
||||
continue;
|
||||
}
|
||||
match backend.read(&storage_key(&hash)).await {
|
||||
Ok(Some(data)) => {
|
||||
let name = names.get(&hash).map_or("original", String::as_str);
|
||||
add(&mut zip, &original_entry(&hash, name), &data)?;
|
||||
originals += 1;
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(e) => return Err(format!("reading an original from browser storage: {e}")),
|
||||
}
|
||||
}
|
||||
let tail = zip.finish()?;
|
||||
parts.push(&Uint8Array::from(&tail[..]));
|
||||
let opts = web_sys::BlobPropertyBag::new();
|
||||
opts.set_type("application/zip");
|
||||
let blob = web_sys::Blob::new_with_u8_array_sequence_and_options(&parts, &opts).map_err(|e| format!("{e:?}"))?;
|
||||
let day: String = String::from(js_sys::Date::new_0().to_iso_string()).chars().take(10).collect();
|
||||
download_blob(&format!("lightcraft-library-backup-{day}.zip"), &blob)?;
|
||||
Ok((originals, bytes + tail.len() as f64))
|
||||
}
|
||||
|
||||
/// Bytes `start..end` of a picked file.
|
||||
async fn read_range(file: &web_sys::File, start: u64, end: u64) -> Result<Vec<u8>, String> {
|
||||
let blob = file.slice_with_f64_and_f64(start as f64, end as f64).map_err(|e| format!("{e:?}"))?;
|
||||
let buf = JsFuture::from(blob.array_buffer()).await.map_err(|e| format!("{e:?}"))?;
|
||||
Ok(Uint8Array::new(&buf).to_vec())
|
||||
}
|
||||
|
||||
/// Restore a backup into a new library folder (nothing in storage is deleted or overwritten),
|
||||
/// then point [`ACTIVE_LIBRARY`] at it. Returns the folder's name; the caller reloads the page.
|
||||
pub async fn restore(file: web_sys::File, backend: Backend) -> Result<String, String> {
|
||||
let len = file.size() as u64;
|
||||
let tail = read_range(&file, len - tail_len(len), len).await?;
|
||||
let (cd_offset, cd_size) = find_central(&tail)?;
|
||||
let cd = read_range(&file, cd_offset, cd_offset + cd_size).await?;
|
||||
let entries = parse_central(&cd)?;
|
||||
if !entries.iter().any(|e| e.name == "library/catalog.snap" || e.name == "library/catalog.log") {
|
||||
return Err("this isn't a LightCraft library backup (no library/catalog files)".into());
|
||||
}
|
||||
let dir = restored_dir_name(js_sys::Date::now());
|
||||
let mut written = 0usize;
|
||||
for e in &entries {
|
||||
let Some(key) = restore_key(&e.name, &dir) else { continue };
|
||||
// originals are named by their content: one already stored is the same file
|
||||
if key.starts_with("originals/") && backend.exists(&key).await? {
|
||||
continue;
|
||||
}
|
||||
let local = read_range(&file, e.header, e.header + 30).await?;
|
||||
let start = data_offset(e, &local)?;
|
||||
let data = read_range(&file, start, start + e.size).await?;
|
||||
verify(e, &data)?;
|
||||
if let Some(hash) = key.strip_prefix("originals/") {
|
||||
verify_original(e, hash, &data)?;
|
||||
}
|
||||
backend.write(&key, &data).await.map_err(|err| format!("storing {}: {err}", e.name))?;
|
||||
written += 1;
|
||||
}
|
||||
backend.write(ACTIVE_LIBRARY, dir.as_bytes()).await?;
|
||||
log::info!("lightcraft: restored {written} files into {dir}");
|
||||
Ok(dir)
|
||||
}
|
||||
|
||||
/// File ▸ Restore Library from Backup…: confirm, pick the zip, restore, reload.
|
||||
pub fn pick_and_restore(backend: Backend, before_reload: impl FnOnce() + 'static) {
|
||||
let ok = confirm(
|
||||
"Restore a LightCraft library backup?\n\nThe backup becomes the library in this browser and the page reloads. \
|
||||
The current library is not deleted: it stays in browser storage.",
|
||||
);
|
||||
if !ok {
|
||||
return;
|
||||
}
|
||||
let Some(doc) = document() else { return };
|
||||
let Ok(input) = doc.create_element("input").map(|e| e.unchecked_into::<web_sys::HtmlInputElement>()) else { return };
|
||||
input.set_type("file");
|
||||
input.set_accept(".zip,application/zip");
|
||||
let inp = input.clone();
|
||||
let mut before_reload = Some(before_reload);
|
||||
let on_change = Closure::<dyn FnMut()>::new(move || {
|
||||
let Some(file) = inp.files().and_then(|f| f.get(0)) else { return };
|
||||
let backend = backend.clone();
|
||||
let before = before_reload.take();
|
||||
notice(format!("Restoring {}…", file.name()));
|
||||
wasm_bindgen_futures::spawn_local(async move {
|
||||
match restore(file, backend).await {
|
||||
Ok(_) => {
|
||||
if let Some(f) = before {
|
||||
f();
|
||||
}
|
||||
if let Some(w) = web_sys::window() {
|
||||
let _ = w.location().reload();
|
||||
}
|
||||
}
|
||||
Err(e) => notice(format!("Restore failed: {e}. Nothing was changed: the current library is still in use.")),
|
||||
}
|
||||
});
|
||||
});
|
||||
input.set_onchange(Some(on_change.as_ref().unchecked_ref()));
|
||||
on_change.forget();
|
||||
input.click();
|
||||
}
|
||||
+188
-29
@@ -15,16 +15,21 @@ use wasm_bindgen::JsCast;
|
||||
use wasm_bindgen::prelude::*;
|
||||
|
||||
use crate::backend::Backend;
|
||||
use crate::backup::{ACTIVE_LIBRARY, DEFAULT_LIBRARY_DIR, valid_library_dir};
|
||||
use crate::bench::Bench;
|
||||
use crate::files::{Files, FlushOp, LIBRARY_FILES};
|
||||
use crate::safety::{self, notice};
|
||||
use crate::store::{Originals, content_hash, download_name, storage_key};
|
||||
use crate::wire::ThumbIndex;
|
||||
use crate::workers::{THUMB_INDEX, Workers};
|
||||
|
||||
const ACCEPT: &str = ".jpg,.jpeg,.png,.tif,.tiff,.webp,.dng,.cr2,.nef,.arw,.psd,.jxl,.gif,.bmp";
|
||||
|
||||
/// Where the library files live in browser storage.
|
||||
const LIBRARY_DIR: &str = "library";
|
||||
/// After a failed save, wait this long before trying again (ms).
|
||||
const SAVE_RETRY_MS: f64 = 2000.0;
|
||||
|
||||
/// How long a notice stays on screen (s).
|
||||
const NOTICE_SECS: f64 = 8.0;
|
||||
|
||||
/// How often view state, UI prefs and the thumbnail index are saved (ms).
|
||||
const SAVE_EVERY_MS: f64 = 1000.0;
|
||||
@@ -45,8 +50,18 @@ fn set_status(text: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
fn library_key(name: &str) -> String {
|
||||
format!("{LIBRARY_DIR}/{name}")
|
||||
/// Storage key of library file `name` in library folder `dir`.
|
||||
fn library_key(dir: &str, name: &str) -> String {
|
||||
format!("{dir}/{name}")
|
||||
}
|
||||
|
||||
/// Repaint the app (notices from async tasks).
|
||||
pub(crate) fn request_repaint() {
|
||||
CTX.with(|c| {
|
||||
if let Some(c) = c.borrow().as_ref() {
|
||||
c.request_repaint();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// URL options (`?bench&store=idb&workers=2&reset`).
|
||||
@@ -86,8 +101,9 @@ async fn store_file(originals: Originals, backend: Option<Backend>, name: String
|
||||
if let Some(b) = &backend
|
||||
&& let Err(e) = b.write(&storage_key(&hash), &bytes).await
|
||||
{
|
||||
// still importable for this session; it won't survive a reload
|
||||
log::error!("storing {name} in browser storage failed: {e}");
|
||||
// a photo whose bytes aren't stored would be gone after a reload: don't add it
|
||||
notice(format!("{name} was not added: browser storage refused it ({e}). Free up space or back up and clear old photos."));
|
||||
return;
|
||||
}
|
||||
originals.added(&name, &hash, Arc::from(bytes));
|
||||
ctx.request_repaint();
|
||||
@@ -148,16 +164,58 @@ fn download(path: &str, bytes: &[u8]) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn services(originals: Originals, backend: Option<Backend>, ctx: egui::Context) -> Services {
|
||||
fn services(originals: Originals, backend: Option<Backend>, files: Files, frozen: Rc<Cell<bool>>, ctx: egui::Context) -> Services {
|
||||
let (backup_backend, backup_files) = (backend.clone(), files);
|
||||
let restore_backend = backend.clone();
|
||||
Services {
|
||||
picker: None,
|
||||
pick_display_profile: None,
|
||||
backup_library: Some(Box::new(move |session: &mut Session| {
|
||||
let Some(b) = backup_backend.clone() else { return Err("nothing is stored in this browser session (?store=memory)".into()) };
|
||||
// the photos' own names for the originals in the zip
|
||||
let names: std::collections::HashMap<String, String> = session
|
||||
.catalog
|
||||
.photos()
|
||||
.filter_map(|p| match &p.source {
|
||||
Source::File { path } => crate::store::hash_of_path(path).map(|h| (h.to_string(), p.file_name.clone())),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
let files = backup_files.clone();
|
||||
notice("Backing up the library… (the download starts when it's ready)");
|
||||
wasm_bindgen_futures::spawn_local(async move {
|
||||
match safety::backup(files, b, names).await {
|
||||
Ok((n, bytes)) => notice(format!(
|
||||
"Library backed up: {n} original{} ({:.1} MB). Keep the file somewhere safe.",
|
||||
if n == 1 { "" } else { "s" },
|
||||
bytes / 1e6
|
||||
)),
|
||||
Err(e) => notice(format!("Backup failed: {e}")),
|
||||
}
|
||||
});
|
||||
Ok(json!({"started": true}))
|
||||
})),
|
||||
restore_library: Some(Box::new(move |_session: &mut Session| {
|
||||
let Some(b) = restore_backend.clone() else { return Err("nothing is stored in this browser session (?store=memory)".into()) };
|
||||
let frozen = frozen.clone();
|
||||
// once the backup is in place, stop saving the old library (the page reloads)
|
||||
safety::pick_and_restore(b, move || frozen.set(true));
|
||||
Ok(json!({"started": true}))
|
||||
})),
|
||||
pick_files: Some(Box::new(move || {
|
||||
open_picker(originals.clone(), backend.clone(), ctx.clone());
|
||||
Vec::new() // files arrive asynchronously and are imported on a later frame
|
||||
})),
|
||||
pick_denoise_model: None,
|
||||
// Preset files: browser pickers are asynchronous; not wired on the web yet.
|
||||
pick_preset_files: None,
|
||||
pick_tracklog: None,
|
||||
pick_lightroom_catalog: None,
|
||||
// Face models need a folder to live in; the web has none.
|
||||
pick_model_file: None,
|
||||
save_preset_file: None,
|
||||
pick_keyword_list: None,
|
||||
save_keyword_list: None,
|
||||
pick_curve_preset_files: None,
|
||||
save_curve_preset_file: None,
|
||||
write: Some(Box::new(download)),
|
||||
@@ -167,6 +225,7 @@ fn services(originals: Originals, backend: Option<Backend>, ctx: egui::Context)
|
||||
lightcraft_codecs::encode_png(&lightcraft_codecs::EncodeImage::rgba8(img), &lightcraft_codecs::EncodeMeta::default()).unwrap_or_default()
|
||||
})),
|
||||
reveal: None,
|
||||
log_file: None,
|
||||
open_with: None,
|
||||
open_url: Some(Box::new(|url: &str| {
|
||||
let w = web_sys::window().ok_or("no window")?;
|
||||
@@ -176,21 +235,29 @@ fn services(originals: Originals, backend: Option<Backend>, ctx: egui::Context)
|
||||
}
|
||||
}
|
||||
|
||||
/// Write dirty library files to storage, in order, until none are left.
|
||||
async fn flush(files: Files, backend: Backend, flushing: Rc<Cell<bool>>) {
|
||||
/// Write dirty library files to storage, in order, until none are left. A failure (quota,
|
||||
/// storage cleared) is recorded in `files` — the catalog then reports changes as unsaved instead
|
||||
/// of accepting them silently — and retried after [`SAVE_RETRY_MS`].
|
||||
async fn flush(files: Files, backend: Backend, dir: String, flushing: Rc<Cell<bool>>, failed_at: Rc<Cell<f64>>) {
|
||||
'outer: loop {
|
||||
let ops = files.take_dirty();
|
||||
if ops.is_empty() {
|
||||
if files.error().is_some() {
|
||||
log::info!("lightcraft: saving works again");
|
||||
}
|
||||
files.set_error(None);
|
||||
break;
|
||||
}
|
||||
for (i, op) in ops.iter().enumerate() {
|
||||
let r = match op {
|
||||
FlushOp::Write { name, data } => backend.write(&library_key(name), data).await,
|
||||
FlushOp::Append { name, offset, data } => backend.write_at(&library_key(name), Some(*offset), data).await,
|
||||
FlushOp::Write { name, data } => backend.write(&library_key(&dir, name), data).await,
|
||||
FlushOp::Append { name, offset, data } => backend.write_at(&library_key(&dir, name), Some(*offset), data).await,
|
||||
};
|
||||
if let Err(e) = r {
|
||||
log::error!("saving {} failed: {e}", op.name());
|
||||
files.failed(&ops[i..]);
|
||||
files.set_error(Some(e));
|
||||
failed_at.set(perf_now());
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
@@ -204,6 +271,8 @@ struct Boot {
|
||||
backend: Option<Backend>,
|
||||
files: Files,
|
||||
index: ThumbIndex,
|
||||
/// The library folder in storage (`library`, or a restored backup's).
|
||||
lib_dir: String,
|
||||
}
|
||||
|
||||
async fn boot(opts: Options) -> Boot {
|
||||
@@ -221,15 +290,32 @@ async fn boot(opts: Options) -> Boot {
|
||||
};
|
||||
let files = Files::default();
|
||||
let mut index = ThumbIndex::default();
|
||||
let mut lib_dir = DEFAULT_LIBRARY_DIR.to_string();
|
||||
if let Some(b) = &backend {
|
||||
if opts.reset {
|
||||
match b.clear().await {
|
||||
Ok(()) => log::info!("lightcraft: storage cleared (?reset)"),
|
||||
Err(e) => log::error!("clearing storage: {e}"),
|
||||
// everything this site keeps in the browser, photos included: never without asking
|
||||
let ok = safety::confirm(
|
||||
"?reset deletes the LightCraft library stored in this browser, including every imported photo. \
|
||||
This can't be undone. Delete it?",
|
||||
);
|
||||
if ok {
|
||||
match b.clear().await {
|
||||
Ok(()) => log::info!("lightcraft: storage cleared (?reset)"),
|
||||
Err(e) => log::error!("clearing storage: {e}"),
|
||||
}
|
||||
} else {
|
||||
log::info!("lightcraft: ?reset declined; storage kept");
|
||||
}
|
||||
}
|
||||
// a restored backup lives in its own folder (see `backup`)
|
||||
if let Ok(Some(d)) = b.read(ACTIVE_LIBRARY).await {
|
||||
let d = String::from_utf8_lossy(&d).trim().to_string();
|
||||
if valid_library_dir(&d) {
|
||||
lib_dir = d;
|
||||
}
|
||||
}
|
||||
for name in LIBRARY_FILES {
|
||||
match b.read(&library_key(name)).await {
|
||||
match b.read(&library_key(&lib_dir, name)).await {
|
||||
Ok(Some(data)) => files.preload(name, data),
|
||||
Ok(None) => {}
|
||||
Err(e) => log::error!("reading {name}: {e}"),
|
||||
@@ -238,7 +324,14 @@ async fn boot(opts: Options) -> Boot {
|
||||
if let Ok(Some(j)) = b.read(THUMB_INDEX).await {
|
||||
index = ThumbIndex::from_json(&j);
|
||||
}
|
||||
crate::backend::request_persistence();
|
||||
crate::backend::request_persistence(|granted| {
|
||||
if !granted {
|
||||
notice(
|
||||
"This browser may delete LightCraft's library when it runs short of space (persistent storage wasn't granted). \
|
||||
Keep your original photos elsewhere and use File ▸ Back Up Library… now and then.",
|
||||
);
|
||||
}
|
||||
});
|
||||
// thumbnails stored without an index entry (index saved late, or lost): delete them
|
||||
let b2 = b.clone();
|
||||
let known: std::collections::HashSet<String> = b
|
||||
@@ -266,7 +359,7 @@ async fn boot(opts: Options) -> Boot {
|
||||
index.len(),
|
||||
index.total() as f64 / 1e6
|
||||
);
|
||||
Boot { opts, backend, files, index }
|
||||
Boot { opts, backend, files, index, lib_dir }
|
||||
}
|
||||
|
||||
struct WebApp {
|
||||
@@ -276,6 +369,11 @@ struct WebApp {
|
||||
backend: Option<Backend>,
|
||||
workers: Option<Workers>,
|
||||
flushing: Rc<Cell<bool>>,
|
||||
/// When the last save to storage failed (ms; 0 = never).
|
||||
flush_failed_at: Rc<Cell<f64>>,
|
||||
/// Saving stopped (a restored backup is about to replace this library: the page reloads).
|
||||
frozen: Rc<Cell<bool>>,
|
||||
lib_dir: String,
|
||||
bench: Option<Bench>,
|
||||
first_frame_logged: bool,
|
||||
last_save: f64,
|
||||
@@ -283,18 +381,19 @@ struct WebApp {
|
||||
}
|
||||
|
||||
impl WebApp {
|
||||
fn new(cc: &eframe::CreationContext<'_>, boot: Boot) -> Self {
|
||||
let Boot { opts, backend, files, index } = boot;
|
||||
fn new(cc: &eframe::CreationContext<'_>, boot: Boot, chinese_font: Vec<u8>) -> Self {
|
||||
let Boot { opts, backend, files, index, lib_dir } = boot;
|
||||
let originals = Originals::default();
|
||||
let t = perf_now();
|
||||
let mut session = Session::new();
|
||||
originals.install(&mut session);
|
||||
let stores = LibraryStores {
|
||||
dir: format!("browser:{}/{LIBRARY_DIR}", backend.as_ref().map_or("memory", |b| b.kind())).into(),
|
||||
dir: format!("browser:{}/{lib_dir}", backend.as_ref().map_or("memory", |b| b.kind())).into(),
|
||||
catalog: Box::new(files.store()),
|
||||
files: Box::new(files.store()),
|
||||
on_disk: false,
|
||||
};
|
||||
let mut problem = None;
|
||||
match session.open_library_in(stores, true).cloned() {
|
||||
Ok(r) => log::info!(
|
||||
"lightcraft: library {} in {:.0} ms ({} photos; snapshot seq {}, {} ops replayed)",
|
||||
@@ -305,25 +404,52 @@ impl WebApp {
|
||||
r.replayed
|
||||
),
|
||||
Err(e) => {
|
||||
log::error!("opening the library failed: {e}; starting a temporary one");
|
||||
session = Session::with_demo();
|
||||
// never a silent demo session (issue #100): an empty one, and the window says why
|
||||
log::error!("opening the library failed: {e}");
|
||||
session = Session::new();
|
||||
originals.install(&mut session);
|
||||
problem = Some(lightcraft_ui_egui::panels::library_problem::LibraryProblem {
|
||||
can_retry: false,
|
||||
..lightcraft_ui_egui::panels::library_problem::LibraryProblem::new(
|
||||
"the browser's storage for this page",
|
||||
format!("{e}. Reload the page to try again."),
|
||||
)
|
||||
});
|
||||
}
|
||||
}
|
||||
if backend.is_some() {
|
||||
notice(
|
||||
"LightCraft in the browser is experimental. Your library is kept in this browser's storage: keep your original photos \
|
||||
elsewhere and back up with File ▸ Back Up Library….",
|
||||
);
|
||||
}
|
||||
// previews are large (≤ 2560 px, f32): keep few in a 32-bit address space
|
||||
session.media.preview_capacity = 3;
|
||||
let mut app = LightcraftApp::new(session, services(originals.clone(), backend.clone(), cc.egui_ctx.clone()));
|
||||
let frozen = Rc::new(Cell::new(false));
|
||||
let mut app = LightcraftApp::new(session, services(originals.clone(), backend.clone(), files.clone(), frozen.clone(), cc.egui_ctx.clone()))
|
||||
.with_chinese_font(chinese_font);
|
||||
let ui_written = files.get("ui.json").unwrap_or_default();
|
||||
if let Ok(ui) = serde_json::from_slice::<UiState>(&ui_written) {
|
||||
app.ui = ui;
|
||||
}
|
||||
app.ui = app.ui.sanitized();
|
||||
app.library_problem = problem;
|
||||
let n = opts.workers.unwrap_or_else(|| {
|
||||
let cores = window().map_or(1, |w| w.navigator().hardware_concurrency() as usize);
|
||||
cores.saturating_sub(1).clamp(1, 4)
|
||||
});
|
||||
let workers =
|
||||
(n > 0).then(|| Workers::start(n, backend.as_ref().map_or("memory", |b| b.kind()), backend.clone(), index, cc.egui_ctx.clone()));
|
||||
let cache = app.session.media.rendered.clone();
|
||||
let workers = (n > 0).then(|| {
|
||||
Workers::start(
|
||||
n,
|
||||
backend.as_ref().map_or("memory", |b| b.kind()),
|
||||
backend.clone(),
|
||||
index,
|
||||
&cache,
|
||||
app.session.cache_bytes(),
|
||||
cc.egui_ctx.clone(),
|
||||
)
|
||||
});
|
||||
if let Some(w) = &workers {
|
||||
app.renderer.set_offload(Box::new(w.clone()));
|
||||
}
|
||||
@@ -337,6 +463,9 @@ impl WebApp {
|
||||
backend,
|
||||
workers,
|
||||
flushing: Rc::new(Cell::new(false)),
|
||||
flush_failed_at: Rc::new(Cell::new(0.0)),
|
||||
frozen,
|
||||
lib_dir,
|
||||
bench: opts.bench.then(|| Bench::new(origin)),
|
||||
first_frame_logged: false,
|
||||
last_save: 0.0,
|
||||
@@ -398,6 +527,9 @@ impl WebApp {
|
||||
|
||||
/// Save view state, UI prefs and the thumbnail index now and then; flush dirty files.
|
||||
fn save(&mut self) {
|
||||
if self.frozen.get() {
|
||||
return;
|
||||
}
|
||||
let now = perf_now();
|
||||
if now - self.last_save >= SAVE_EVERY_MS {
|
||||
self.last_save = now;
|
||||
@@ -419,12 +551,16 @@ impl WebApp {
|
||||
});
|
||||
}
|
||||
}
|
||||
// after a failure, retry every couple of seconds rather than every frame
|
||||
let backing_off = self.files.error().is_some() && now - self.flush_failed_at.get() < SAVE_RETRY_MS;
|
||||
if let Some(b) = &self.backend
|
||||
&& self.files.is_dirty()
|
||||
&& !self.flushing.get()
|
||||
&& !backing_off
|
||||
{
|
||||
self.flushing.set(true);
|
||||
wasm_bindgen_futures::spawn_local(flush(self.files.clone(), b.clone(), self.flushing.clone()));
|
||||
let (files, dir, flushing, failed_at) = (self.files.clone(), self.lib_dir.clone(), self.flushing.clone(), self.flush_failed_at.clone());
|
||||
wasm_bindgen_futures::spawn_local(flush(files, b.clone(), dir, flushing, failed_at));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -454,8 +590,10 @@ impl WebApp {
|
||||
let (orig_n, orig_bytes) = self.originals.usage();
|
||||
json!({
|
||||
"storage": self.backend.as_ref().map_or("memory", |b| b.kind()),
|
||||
"library": self.lib_dir,
|
||||
"dirty": self.files.is_dirty(),
|
||||
"flushing": self.flushing.get(),
|
||||
"saveError": self.files.error(),
|
||||
"workers": {"alive": alive, "ready": ready, "remoteJobs": remote, "inlineJobs": inline},
|
||||
"originalsInMemory": orig_n,
|
||||
"originalBytesInMemory": orig_bytes,
|
||||
@@ -480,9 +618,18 @@ impl WebApp {
|
||||
|
||||
impl eframe::App for WebApp {
|
||||
fn logic(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) {
|
||||
for text in safety::take_notices() {
|
||||
let now = ctx.input(|i| i.time);
|
||||
self.app.ui.toast = Some((text, now + NOTICE_SECS, None));
|
||||
}
|
||||
self.run_inbox();
|
||||
self.import_dropped(ctx);
|
||||
self.load_originals(ctx);
|
||||
if let Some(w) = &self.workers {
|
||||
// `library.preferences ▸ cacheMb` is read by the session; the worker index prunes on
|
||||
// its own thread, so it gets the budget every frame and applies it on the next store.
|
||||
w.set_budget(self.app.session.cache_bytes());
|
||||
}
|
||||
self.app.logic(ctx);
|
||||
self.save();
|
||||
if let Some(b) = self.bench.as_mut()
|
||||
@@ -513,13 +660,25 @@ impl eframe::App for WebApp {
|
||||
}
|
||||
}
|
||||
|
||||
/// Start the app (called by `index.html` after instantiating the module).
|
||||
/// Start the app with the separately fetched Simplified Chinese font (empty in font-free builds).
|
||||
#[wasm_bindgen]
|
||||
pub fn start() {
|
||||
pub fn start(chinese_font: Vec<u8>) {
|
||||
eframe::WebLogger::init(log::LevelFilter::Info).ok();
|
||||
safety::install_panic_hook();
|
||||
log::info!("lightcraft: wasm instantiated at {:.0} ms", perf_now());
|
||||
if chinese_font.is_empty() {
|
||||
log::warn!("lightcraft: no Simplified Chinese font supplied; Chinese-only glyphs will show as boxes");
|
||||
}
|
||||
let opts = Options::from_url();
|
||||
wasm_bindgen_futures::spawn_local(async move {
|
||||
// one tab per library: two would each keep their own copy and overwrite each other's saves
|
||||
if opts.store != "memory" && safety::acquire_tab_lock().await == Some(false) {
|
||||
safety::show_blocking(
|
||||
"LightCraft is already open in another tab or window of this browser.\n\n\
|
||||
Switch to that tab, or close it and reload this page. (Two tabs would overwrite each other's changes.)",
|
||||
);
|
||||
return;
|
||||
}
|
||||
let Some(canvas) = window()
|
||||
.and_then(|w| w.document())
|
||||
.and_then(|d| d.get_element_by_id("lightcraft_canvas"))
|
||||
@@ -530,7 +689,7 @@ pub fn start() {
|
||||
};
|
||||
let boot = boot(opts).await;
|
||||
let runner = eframe::WebRunner::new();
|
||||
let r = runner.start(canvas, eframe::WebOptions::default(), Box::new(move |cc| Ok(Box::new(WebApp::new(cc, boot))))).await;
|
||||
let r = runner.start(canvas, eframe::WebOptions::default(), Box::new(move |cc| Ok(Box::new(WebApp::new(cc, boot, chinese_font))))).await;
|
||||
match r {
|
||||
Ok(()) => {
|
||||
if let Some(el) = window().and_then(|w| w.document()).and_then(|d| d.get_element_by_id("lightcraft_loading")) {
|
||||
|
||||
+203
-14
@@ -11,14 +11,13 @@
|
||||
//! ([`ThumbIndex`]).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, Weak};
|
||||
|
||||
use lightcraft_engine::catalog::Source;
|
||||
use lightcraft_engine::develop::{DevelopSettings, EmbeddedLens};
|
||||
use lightcraft_engine::media::{MediaCache, RenderJob, SourceLevel, SourceRef};
|
||||
use lightcraft_engine::media::{DecodedSource, MediaCache, RenderJob, SourceLevel, SourceRef};
|
||||
use lightcraft_engine::pipeline::{Quality, RenderRequest, Rendered, SourceInfo, StageCache};
|
||||
use lightcraft_preview::Lru;
|
||||
use lightcraft_raster::Rgb32f;
|
||||
use lightcraft_preview::{Hash128, Lru, PreviewCache};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::store::hash_of_path;
|
||||
@@ -26,11 +25,17 @@ use crate::store::hash_of_path;
|
||||
/// One render request for a worker.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct WireJob {
|
||||
#[serde(default)]
|
||||
pub cache_generation: u64,
|
||||
#[serde(default)]
|
||||
pub source_identity: Option<String>,
|
||||
pub level: SourceLevel,
|
||||
pub origin: Source,
|
||||
/// Decode the source at most this long.
|
||||
pub max_edge: usize,
|
||||
pub raw: bool,
|
||||
#[serde(default)]
|
||||
pub relative_wb: bool,
|
||||
pub as_shot_temp: f64,
|
||||
pub as_shot_tint: f64,
|
||||
pub lens: Option<EmbeddedLens>,
|
||||
@@ -48,6 +53,9 @@ pub struct WireJob {
|
||||
/// The request's diagnostic overlay (`Overlay::to_parts`).
|
||||
#[serde(default)]
|
||||
pub overlay: (u8, f64),
|
||||
/// Render only this window `[x, y, w, h]` of the output `max_w × max_h` describes (a zoomed view).
|
||||
#[serde(default)]
|
||||
pub window: Option<[usize; 4]>,
|
||||
}
|
||||
|
||||
impl WireJob {
|
||||
@@ -55,14 +63,17 @@ impl WireJob {
|
||||
pub fn from_job(job: &RenderJob, thumb_cached: bool, view: &str) -> WireJob {
|
||||
let max_edge = match &job.source {
|
||||
SourceRef::Demo { max_edge, .. } | SourceRef::File { max_edge, .. } => *max_edge,
|
||||
SourceRef::Loaded(img) => img.width.max(img.height),
|
||||
SourceRef::Loaded(img) => img.image.width.max(img.image.height),
|
||||
SourceRef::Smart { .. } => 2560,
|
||||
};
|
||||
WireJob {
|
||||
cache_generation: job.cache_generation,
|
||||
source_identity: job.source_key.map(|key| key.to_string()),
|
||||
level: job.level,
|
||||
origin: job.origin.clone(),
|
||||
max_edge,
|
||||
raw: job.info.raw,
|
||||
relative_wb: job.info.relative_wb,
|
||||
as_shot_temp: job.info.as_shot_temp,
|
||||
as_shot_tint: job.info.as_shot_tint,
|
||||
lens: job.info.lens,
|
||||
@@ -75,11 +86,19 @@ impl WireJob {
|
||||
thumb_cached,
|
||||
stages: job.stages.is_some().then(|| view.to_string()),
|
||||
overlay: job.request.overlay.to_parts(),
|
||||
window: job.request.window.map(|w| [w.x, w.y, w.w, w.h]),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn info(&self) -> SourceInfo {
|
||||
SourceInfo { raw: self.raw, as_shot_temp: self.as_shot_temp, as_shot_tint: self.as_shot_tint, lens: self.lens }
|
||||
SourceInfo {
|
||||
raw: self.raw,
|
||||
as_shot_temp: self.as_shot_temp,
|
||||
as_shot_tint: self.as_shot_tint,
|
||||
lens: self.lens,
|
||||
relative_wb: self.relative_wb,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn request(&self) -> RenderRequest {
|
||||
@@ -93,14 +112,16 @@ impl WireJob {
|
||||
space: lightcraft_engine::pipeline::OutputSpace::Srgb,
|
||||
depth: lightcraft_engine::pipeline::OutputDepth::U8,
|
||||
proof: None,
|
||||
window: self.window.map(|[x, y, w, h]| lightcraft_engine::pipeline::PixelWindow { x, y, w, h }),
|
||||
display: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Identifies the decoded source this job needs.
|
||||
fn source_key(&self) -> String {
|
||||
match &self.origin {
|
||||
Source::File { path } => format!("{}:{path}", self.max_edge),
|
||||
Source::Demo { scene } => format!("{}:demo:{scene}", self.max_edge),
|
||||
Source::File { path } => format!("{}:{:?}:{}:{path}", self.cache_generation, self.source_identity, self.max_edge),
|
||||
Source::Demo { scene } => format!("{}:{:?}:{}:demo:{scene}", self.cache_generation, self.source_identity, self.max_edge),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,7 +140,7 @@ const WORKER_SOURCE_BYTES: usize = 192 << 20;
|
||||
|
||||
/// The synchronous part of a worker: decoded-source cache + rendering.
|
||||
pub struct WorkerCore {
|
||||
sources: Lru<String, Arc<Rgb32f>>,
|
||||
sources: Lru<String, DecodedSource>,
|
||||
media: MediaCache,
|
||||
stages: HashMap<String, Arc<StageCache>>,
|
||||
}
|
||||
@@ -150,21 +171,25 @@ impl WorkerCore {
|
||||
Some(s) => s,
|
||||
None => {
|
||||
let src = match (&job.origin, original) {
|
||||
(Source::File { .. }, Some(bytes)) => Arc::new(lightcraft_engine::files::load_bytes(bytes, job.max_edge)?.0),
|
||||
(Source::File { .. }, Some(bytes)) => {
|
||||
let (image, info) = lightcraft_engine::files::load_bytes(bytes, job.max_edge)?;
|
||||
DecodedSource::new(Arc::new(image), Some(info))
|
||||
}
|
||||
(Source::File { path }, None) => return Err(format!("{path}: original not found in browser storage")),
|
||||
(origin @ Source::Demo { .. }, _) => self.media.origin_ref(origin, job.max_edge).load()?,
|
||||
(origin @ Source::Demo { .. }, _) => self.media.origin_ref(origin, job.max_edge).load_source()?,
|
||||
};
|
||||
let cost = src.width * src.height * 12 + 64;
|
||||
let cost = src.image.width * src.image.height * 12 + 64;
|
||||
self.sources.insert(key, src.clone(), cost);
|
||||
src
|
||||
}
|
||||
};
|
||||
let info = src.info_or(job.info());
|
||||
Ok(match &job.stages {
|
||||
Some(view) => {
|
||||
let st = self.stages.entry(view.clone()).or_default().clone();
|
||||
lightcraft_engine::pipeline::render_cached(&src, &job.info(), &job.settings, &job.request(), &st)
|
||||
lightcraft_engine::pipeline::render_cached(&src.image, &info, &job.settings, &job.request(), &st)
|
||||
}
|
||||
None => lightcraft_engine::pipeline::render(&src, &job.info(), &job.settings, &job.request()),
|
||||
None => lightcraft_engine::pipeline::render(&src.image, &info, &job.settings, &job.request()),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -173,6 +198,8 @@ impl WorkerCore {
|
||||
/// pruned like the native disk cache (least recently used first, down to 80 % of the budget).
|
||||
#[derive(Default, Serialize, Deserialize)]
|
||||
pub struct ThumbIndex {
|
||||
#[serde(default)]
|
||||
namespace: u64,
|
||||
/// hex key → (bytes, last use).
|
||||
entries: HashMap<String, (u64, u64)>,
|
||||
clock: u64,
|
||||
@@ -183,6 +210,24 @@ pub struct ThumbIndex {
|
||||
}
|
||||
|
||||
impl ThumbIndex {
|
||||
pub fn namespace(&self) -> u64 {
|
||||
self.namespace
|
||||
}
|
||||
|
||||
/// A persistent new namespace prevents late old workers restoring an invalidated disk hit.
|
||||
pub fn cache_key(&self, key: Hash128) -> String {
|
||||
if self.namespace == 0 {
|
||||
return key.to_string();
|
||||
}
|
||||
lightcraft_preview::Hasher128::new().str(&key.to_string()).str("cleared").u64(self.namespace).finish().to_string()
|
||||
}
|
||||
|
||||
pub fn invalidate(&mut self) -> Vec<String> {
|
||||
self.namespace = self.namespace.wrapping_add(1);
|
||||
self.total = 0;
|
||||
self.dirty = true;
|
||||
self.entries.drain().map(|(key, _)| key).collect()
|
||||
}
|
||||
pub fn from_json(bytes: &[u8]) -> ThumbIndex {
|
||||
let mut i: ThumbIndex = serde_json::from_slice(bytes).unwrap_or_default();
|
||||
i.total = i.entries.values().map(|e| e.0).sum();
|
||||
@@ -260,11 +305,89 @@ impl ThumbIndex {
|
||||
}
|
||||
}
|
||||
|
||||
/// Which rendered-thumbnail cache, at which generation, the stored thumbnails ([`ThumbIndex`])
|
||||
/// belong to (main thread). Clearing the previews bumps the cache's generation, and opening
|
||||
/// another library replaces the cache: either makes every stored thumbnail obsolete.
|
||||
pub struct CacheWatch(Option<(Weak<PreviewCache>, u64)>);
|
||||
|
||||
impl CacheWatch {
|
||||
/// Watch `active`, the session's cache when the workers start: the persisted index belongs
|
||||
/// to it at its current generation. (Without this baseline, the first observation after a
|
||||
/// clear would only record the cleared generation and keep the obsolete thumbnails.)
|
||||
pub fn new(active: &Arc<PreviewCache>) -> CacheWatch {
|
||||
CacheWatch(Some((Arc::downgrade(active), active.generation())))
|
||||
}
|
||||
|
||||
/// The watched cache, while it's alive.
|
||||
pub fn cache(&self) -> Option<Arc<PreviewCache>> {
|
||||
self.0.as_ref().and_then(|(cache, _)| cache.upgrade())
|
||||
}
|
||||
|
||||
/// `cache` at `generation` is the active one now. If it was cleared or replaced since the
|
||||
/// last observation, invalidate `index` and return the keys whose files should be deleted.
|
||||
pub fn observe(&mut self, cache: &Arc<PreviewCache>, generation: u64, index: &mut ThumbIndex) -> Vec<String> {
|
||||
let changed = self.0.as_ref().is_some_and(|(old, g)| old.as_ptr() != Arc::as_ptr(cache) || *g != generation);
|
||||
self.0 = Some((Arc::downgrade(cache), generation));
|
||||
if changed { index.invalidate() } else { Vec::new() }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use lightcraft_engine::Session;
|
||||
|
||||
#[test]
|
||||
fn clearing_index_changes_disk_namespace_and_survives_restart() {
|
||||
let key = lightcraft_preview::hash_bytes(b"thumbnail");
|
||||
let mut index = ThumbIndex::default();
|
||||
let old = index.cache_key(key);
|
||||
assert_eq!(old, key.to_string());
|
||||
index.insert(&old, 100);
|
||||
assert_eq!(index.invalidate(), vec![old.clone()]);
|
||||
let new = index.cache_key(key);
|
||||
assert_ne!(old, new);
|
||||
index.insert(&new, 100);
|
||||
let reopened = ThumbIndex::from_json(&index.to_json());
|
||||
assert_eq!(reopened.cache_key(key), new);
|
||||
assert!(!reopened.contains(&old));
|
||||
assert!(reopened.contains(&new));
|
||||
}
|
||||
|
||||
// Issue #323: a worker given a zoomed view's window renders that window, not the whole frame
|
||||
#[test]
|
||||
fn a_window_job_crosses_the_wire_and_renders_the_same_window() {
|
||||
use lightcraft_engine::pipeline::PixelWindow;
|
||||
let mut s = Session::with_demo();
|
||||
let id = s.visible_cloned()[0];
|
||||
let win = PixelWindow { x: 300, y: 200, w: 160, h: 120 };
|
||||
let job = s.region_job(id, 1920, 1280, win, true).unwrap();
|
||||
let wire = WireJob::from_job(&job, false, "Region");
|
||||
let wire: WireJob = serde_json::from_str(&serde_json::to_string(&wire).unwrap()).unwrap();
|
||||
assert_eq!(wire.request().window, Some(win));
|
||||
let mut core = WorkerCore::default();
|
||||
let a = core.render(&wire, None).unwrap();
|
||||
let b = job.run().rendered.unwrap();
|
||||
assert_eq!((a.image.width, a.image.height), (160, 120));
|
||||
assert_eq!((b.image.width, b.image.height), (160, 120));
|
||||
// the worker renders on the CPU; the engine job uses a GPU when the machine has one, which
|
||||
// stays within docs/gpu-pipeline.md's bound (≤ 3 LSB per channel) rather than bit-exact
|
||||
let max = a.image.data.iter().zip(&b.image.data).flat_map(|(p, q)| (0..3).map(move |c| p[c].abs_diff(q[c]))).max();
|
||||
assert!(max.is_some_and(|m| m <= 3), "max channel difference {max:?}");
|
||||
}
|
||||
|
||||
// a job from before windows existed has none
|
||||
#[test]
|
||||
fn a_job_without_a_window_field_has_no_window() {
|
||||
let mut s = Session::with_demo();
|
||||
let id = s.visible_cloned()[0];
|
||||
let wire = WireJob::from_job(&s.thumb_job(id, 128).unwrap(), false, "Thumb");
|
||||
let mut v = serde_json::to_value(&wire).unwrap();
|
||||
v.as_object_mut().unwrap().remove("window");
|
||||
let old: WireJob = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(old.request().window, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wire_job_renders_like_the_engine() {
|
||||
let mut s = Session::with_demo();
|
||||
@@ -291,10 +414,13 @@ mod tests {
|
||||
let hash = crate::store::content_hash(&bytes);
|
||||
let origin = Source::File { path: crate::store::original_path(&hash, "a.png") };
|
||||
let job = WireJob {
|
||||
cache_generation: 0,
|
||||
source_identity: None,
|
||||
level: SourceLevel::Thumb,
|
||||
origin,
|
||||
max_edge: 512,
|
||||
raw: false,
|
||||
relative_wb: false,
|
||||
as_shot_temp: 6500.0,
|
||||
as_shot_tint: 0.0,
|
||||
lens: None,
|
||||
@@ -307,6 +433,7 @@ mod tests {
|
||||
thumb_cached: false,
|
||||
stages: Some("Main".into()),
|
||||
overlay: (0, 0.0),
|
||||
window: None,
|
||||
};
|
||||
let mut core = WorkerCore::default();
|
||||
assert_eq!(core.needs_original(&job).as_deref(), Some(hash.as_str()));
|
||||
@@ -314,6 +441,68 @@ mod tests {
|
||||
let r = core.render(&job, Some(&bytes)).unwrap();
|
||||
assert_eq!(r.image.width, 32);
|
||||
assert_eq!(core.needs_original(&job), None, "decoded source is kept");
|
||||
let mut reloaded = job.clone();
|
||||
reloaded.source_identity = Some("changed content".into());
|
||||
assert_eq!(core.needs_original(&reloaded).as_deref(), Some(hash.as_str()), "content reload must not reuse an old decoded source");
|
||||
let mut refreshed = job.clone();
|
||||
refreshed.cache_generation = 1;
|
||||
assert_eq!(core.needs_original(&refreshed).as_deref(), Some(hash.as_str()), "explicit refresh must not reuse the old decoded source");
|
||||
}
|
||||
|
||||
/// The persisted index as a restarted page loads it: one stored thumbnail.
|
||||
fn persisted_index(key: Hash128) -> (ThumbIndex, String) {
|
||||
let mut index = ThumbIndex::default();
|
||||
let hex = index.cache_key(key);
|
||||
index.insert(&hex, 100);
|
||||
(ThumbIndex::from_json(&index.to_json()), hex)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clearing_previews_before_the_first_request_invalidates_stored_thumbnails() {
|
||||
let key = lightcraft_preview::hash_bytes(b"thumbnail");
|
||||
let (mut index, old) = persisted_index(key);
|
||||
let cache = Arc::new(PreviewCache::memory(1 << 20));
|
||||
let mut watch = CacheWatch::new(&cache);
|
||||
let mut gone = Vec::new();
|
||||
// each frame (`Workers::finished`) observes the watched cache, even with no requests
|
||||
let mut frame = |watch: &mut CacheWatch, index: &mut ThumbIndex| {
|
||||
if let Some(c) = watch.cache() {
|
||||
gone.extend(watch.observe(&c, c.generation(), index));
|
||||
}
|
||||
};
|
||||
frame(&mut watch, &mut index); // an empty filtered view: no render request yet
|
||||
cache.clear(); // File ▸ Clear Preview Cache
|
||||
frame(&mut watch, &mut index);
|
||||
// the filter is removed: the first thumbnail request (`Workers::try_start`)
|
||||
gone.extend(watch.observe(&cache, cache.generation(), &mut index));
|
||||
let disk_key = index.cache_key(key);
|
||||
assert!(!index.touch(&disk_key), "the pre-clear thumbnail must not be read back");
|
||||
assert_eq!(gone, vec![old.clone()], "the stored file is deleted");
|
||||
assert_ne!(disk_key, old, "a fresh namespace: a late old write can't come back");
|
||||
assert!(index.dirty, "the invalidated index is saved");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replacing_the_cache_before_the_first_request_invalidates_stored_thumbnails() {
|
||||
let key = lightcraft_preview::hash_bytes(b"thumbnail");
|
||||
let (mut index, old) = persisted_index(key);
|
||||
let first = Arc::new(PreviewCache::memory(1 << 20));
|
||||
let mut watch = CacheWatch::new(&first);
|
||||
let other = Arc::new(PreviewCache::memory(1 << 20)); // another library, generation 0 again
|
||||
assert_eq!(watch.observe(&other, other.generation(), &mut index), vec![old]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_normal_start_keeps_stored_thumbnails() {
|
||||
let key = lightcraft_preview::hash_bytes(b"thumbnail");
|
||||
let (mut index, old) = persisted_index(key);
|
||||
let cache = Arc::new(PreviewCache::memory(1 << 20));
|
||||
let mut watch = CacheWatch::new(&cache);
|
||||
for _ in 0..3 {
|
||||
assert!(watch.observe(&cache, cache.generation(), &mut index).is_empty());
|
||||
}
|
||||
assert_eq!(index.cache_key(key), old);
|
||||
assert!(index.touch(&old), "persisted thumbnails survive a restart");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -18,7 +18,7 @@ use std::sync::Arc;
|
||||
|
||||
use js_sys::{Array, Object, Reflect, Uint8Array, Uint32Array};
|
||||
use lightcraft_engine::catalog::PhotoId;
|
||||
use lightcraft_engine::media::{DISK_CACHE_BYTES, RenderJob, RenderResult, SourceLevel};
|
||||
use lightcraft_engine::media::{RenderJob, RenderResult, SourceLevel};
|
||||
use lightcraft_engine::pipeline::Rendered;
|
||||
use lightcraft_preview::{Hash128, PreviewCache};
|
||||
use lightcraft_raster::{Histogram, Rgba8};
|
||||
@@ -27,7 +27,7 @@ use wasm_bindgen::JsCast;
|
||||
use wasm_bindgen::prelude::*;
|
||||
|
||||
use crate::backend::Backend;
|
||||
use crate::wire::{ThumbIndex, WireJob, WorkerCore, thumb_storage_key};
|
||||
use crate::wire::{CacheWatch, ThumbIndex, WireJob, WorkerCore, thumb_storage_key};
|
||||
|
||||
/// Storage key of the thumbnail index.
|
||||
pub const THUMB_INDEX: &str = "thumbs/index.json";
|
||||
@@ -155,6 +155,10 @@ async fn handle(scope: &web_sys::DedicatedWorkerGlobalScope, backend: Option<Bac
|
||||
// main side
|
||||
|
||||
struct Busy {
|
||||
disk_key: Option<String>,
|
||||
namespace: u64,
|
||||
request_id: u64,
|
||||
cache_generation: u64,
|
||||
id: u32,
|
||||
slot: Slot,
|
||||
photo: PhotoId,
|
||||
@@ -171,12 +175,17 @@ struct W {
|
||||
}
|
||||
|
||||
struct Inner {
|
||||
/// The rendered-thumbnail cache the stored thumbnails belong to.
|
||||
preview_generation: CacheWatch,
|
||||
workers: Vec<W>,
|
||||
done: Vec<(Slot, RenderResult, f64)>,
|
||||
next_id: u32,
|
||||
/// (photo, level) → worker that last decoded it.
|
||||
affinity: HashMap<(PhotoId, SourceLevel), usize>,
|
||||
index: ThumbIndex,
|
||||
/// The session's thumbnail cache budget in bytes. Kept here because the prune happens on a
|
||||
/// worker message, where the session is out of reach; the app refreshes it every frame.
|
||||
budget: u64,
|
||||
backend: Option<Backend>,
|
||||
ctx: egui::Context,
|
||||
/// Jobs finished by workers / inline (for diagnostics).
|
||||
@@ -185,19 +194,46 @@ struct Inner {
|
||||
inline_this_frame: bool,
|
||||
}
|
||||
|
||||
impl Inner {
|
||||
fn observe_cache(&mut self, cache: &Arc<PreviewCache>, generation: u64) {
|
||||
let gone = self.preview_generation.observe(cache, generation, &mut self.index);
|
||||
if !gone.is_empty()
|
||||
&& let Some(backend) = self.backend.clone()
|
||||
{
|
||||
wasm_bindgen_futures::spawn_local(async move {
|
||||
for key in gone {
|
||||
let _ = backend.remove(&thumb_storage_key(&key)).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The main thread's handle on the render workers.
|
||||
#[derive(Clone)]
|
||||
pub struct Workers(Rc<RefCell<Inner>>);
|
||||
|
||||
impl Workers {
|
||||
/// Start `n` workers (none: every job runs inline). `store` is the backend kind for workers.
|
||||
pub fn start(n: usize, store: &str, backend: Option<Backend>, index: ThumbIndex, ctx: egui::Context) -> Workers {
|
||||
/// `index` lists the stored thumbnails of `cache`, the session's rendered-thumbnail cache
|
||||
/// now: watching it from the start, a clear before the first render request still counts.
|
||||
pub fn start(
|
||||
n: usize,
|
||||
store: &str,
|
||||
backend: Option<Backend>,
|
||||
index: ThumbIndex,
|
||||
cache: &Arc<PreviewCache>,
|
||||
budget: u64,
|
||||
ctx: egui::Context,
|
||||
) -> Workers {
|
||||
let w = Workers(Rc::new(RefCell::new(Inner {
|
||||
preview_generation: CacheWatch::new(cache),
|
||||
workers: Vec::new(),
|
||||
done: Vec::new(),
|
||||
next_id: 0,
|
||||
affinity: HashMap::new(),
|
||||
index,
|
||||
budget,
|
||||
backend,
|
||||
ctx,
|
||||
remote_done: 0,
|
||||
@@ -248,7 +284,17 @@ impl Workers {
|
||||
w.dead = true;
|
||||
w.worker.terminate();
|
||||
if let Some(b) = w.busy.take() {
|
||||
let r = RenderResult { photo: b.photo, level: b.level, key: b.key, rendered: Err(msg), loaded: None, quick: None };
|
||||
let r = RenderResult {
|
||||
request_id: b.request_id,
|
||||
source_key: None,
|
||||
photo: b.photo,
|
||||
level: b.level,
|
||||
key: b.key,
|
||||
rendered: Err(msg),
|
||||
loaded: None,
|
||||
quick: None,
|
||||
display: None,
|
||||
};
|
||||
g.done.push((b.slot, r, 0.0));
|
||||
}
|
||||
}
|
||||
@@ -284,16 +330,21 @@ impl Workers {
|
||||
};
|
||||
if let Ok(r) = &rendered
|
||||
&& let Some((cache, key)) = &b.cache
|
||||
&& cache.generation() == b.cache_generation
|
||||
&& b.namespace == g.index.namespace()
|
||||
{
|
||||
cache.put(*key, Arc::new(r.image.clone()));
|
||||
let hex = key.to_string();
|
||||
cache.put_at(b.cache_generation, *key, Arc::new(r.image.clone()));
|
||||
let hex = b.disk_key.clone().unwrap_or_else(|| key.to_string());
|
||||
let stored = get(&data, "stored").as_f64().unwrap_or(0.0) as u64;
|
||||
if get(&data, "cacheMiss").is_truthy() {
|
||||
g.index.remove(&hex);
|
||||
}
|
||||
if stored > 0 {
|
||||
g.index.insert(&hex, stored);
|
||||
let gone = g.index.prune(DISK_CACHE_BYTES);
|
||||
// The session's own budget (`library.preferences ▸ cacheMb`), not the native
|
||||
// default: the browser accepted the setting, so it has to be the one enforced
|
||||
// (#245).
|
||||
let gone = g.index.prune(g.budget);
|
||||
if !gone.is_empty()
|
||||
&& let Some(be) = g.backend.clone()
|
||||
{
|
||||
@@ -307,8 +358,32 @@ impl Workers {
|
||||
}
|
||||
}
|
||||
g.affinity.insert((b.photo, b.level), i);
|
||||
// A cleared worker may finish writing after the initial deletion. Its old namespace
|
||||
// is never reused, so removing this file cannot delete a replacement's cache entry.
|
||||
if let Some((cache, _)) = &b.cache
|
||||
&& (cache.generation() != b.cache_generation || b.namespace != g.index.namespace())
|
||||
&& let (Some(backend), Some(key)) = (g.backend.clone(), b.disk_key.clone())
|
||||
{
|
||||
wasm_bindgen_futures::spawn_local(async move {
|
||||
let _ = backend.remove(&thumb_storage_key(&key)).await;
|
||||
});
|
||||
}
|
||||
g.remote_done += 1;
|
||||
g.done.push((b.slot, RenderResult { photo: b.photo, level: b.level, key: b.key, rendered, loaded: None, quick: None }, ms));
|
||||
g.done.push((
|
||||
b.slot,
|
||||
RenderResult {
|
||||
request_id: b.request_id,
|
||||
source_key: None,
|
||||
photo: b.photo,
|
||||
level: b.level,
|
||||
key: b.key,
|
||||
rendered,
|
||||
loaded: None,
|
||||
quick: None,
|
||||
display: None,
|
||||
},
|
||||
ms,
|
||||
));
|
||||
}
|
||||
|
||||
/// The thumbnail index as JSON if it changed since the last call (the host saves it).
|
||||
@@ -328,25 +403,37 @@ impl Workers {
|
||||
let ready = g.workers.iter().filter(|w| w.ready && !w.dead).count();
|
||||
(alive, ready, g.remote_done, g.inline_done)
|
||||
}
|
||||
|
||||
/// The budget the thumbnail index is pruned to. The app refreshes it from the session every
|
||||
/// frame, so `library.preferences ▸ cacheMb` applies to the next stored thumbnail.
|
||||
pub fn set_budget(&self, bytes: u64) {
|
||||
self.0.borrow_mut().budget = bytes;
|
||||
}
|
||||
}
|
||||
|
||||
impl RenderOffload for Workers {
|
||||
fn try_start(&mut self, slot: Slot, job: RenderJob) -> Option<RenderJob> {
|
||||
let mut g = self.0.borrow_mut();
|
||||
let g = &mut *g;
|
||||
if let Some((cache, _)) = job.cache.as_ref().or(job.view_cache.as_ref()) {
|
||||
g.observe_cache(cache, job.cache_generation);
|
||||
}
|
||||
// rendered thumbnail in memory: no worker needed
|
||||
if let Some((cache, key)) = &job.cache
|
||||
&& let Some(img) = cache.get(*key)
|
||||
&& let Some(img) = cache.get_at(job.cache_generation, *key)
|
||||
{
|
||||
let image = Arc::unwrap_or_clone(img);
|
||||
let histogram = Histogram::of_srgb8(&image);
|
||||
let r = RenderResult {
|
||||
request_id: job.request_id,
|
||||
source_key: job.source_key,
|
||||
photo: job.photo,
|
||||
level: job.level,
|
||||
key: job.key,
|
||||
rendered: Ok(Rendered { image, histogram, deep: None }),
|
||||
loaded: None,
|
||||
quick: None,
|
||||
display: None,
|
||||
};
|
||||
g.done.push((slot, r, 0.0));
|
||||
return None;
|
||||
@@ -374,25 +461,45 @@ impl RenderOffload for Workers {
|
||||
let Some(i) = pick else { return Some(job) };
|
||||
// decided here, so the next job for this photo follows even before this one finishes
|
||||
g.affinity.insert((job.photo, job.level), i);
|
||||
let thumb_cached = job.cache.as_ref().is_some_and(|(_, k)| g.index.touch(&k.to_string()));
|
||||
let wire = WireJob::from_job(&job, thumb_cached, &format!("{slot:?}"));
|
||||
let disk_key = job.cache.as_ref().map(|(_, key)| g.index.cache_key(*key));
|
||||
let thumb_cached = disk_key.as_ref().is_some_and(|key| g.index.touch(key));
|
||||
let mut wire = WireJob::from_job(&job, thumb_cached, &format!("{slot:?}"));
|
||||
// The namespace also changes when a new library's cache starts again at generation zero.
|
||||
wire.cache_generation = g.index.namespace();
|
||||
wire.thumb = disk_key.clone();
|
||||
let Ok(text) = serde_json::to_string(&wire) else { return Some(job) };
|
||||
g.next_id = g.next_id.wrapping_add(1);
|
||||
let id = g.next_id;
|
||||
let msg = Object::new();
|
||||
set(&msg, "id", &id.into());
|
||||
set(&msg, "job", &text.into());
|
||||
let namespace = g.index.namespace();
|
||||
let w = &mut g.workers[i];
|
||||
if let Err(e) = w.worker.post_message(&msg) {
|
||||
log::warn!("render worker {i}: {e:?}");
|
||||
return Some(job);
|
||||
}
|
||||
w.busy = Some(Busy { id, slot, photo: job.photo, level: job.level, key: job.key, cache: job.cache });
|
||||
w.busy = Some(Busy {
|
||||
disk_key,
|
||||
namespace,
|
||||
request_id: job.request_id,
|
||||
cache_generation: job.cache_generation,
|
||||
id,
|
||||
slot,
|
||||
photo: job.photo,
|
||||
level: job.level,
|
||||
key: job.key,
|
||||
cache: job.cache,
|
||||
});
|
||||
None
|
||||
}
|
||||
|
||||
fn finished(&mut self) -> Vec<(Slot, RenderResult, f64)> {
|
||||
let mut g = self.0.borrow_mut();
|
||||
// Persist invalidation even when the cleared library has no visible render requests.
|
||||
if let Some(cache) = g.preview_generation.cache() {
|
||||
g.observe_cache(&cache, cache.generation());
|
||||
}
|
||||
g.inline_this_frame = false;
|
||||
std::mem::take(&mut g.done)
|
||||
}
|
||||
|
||||
@@ -9,8 +9,16 @@ default-run = "lightcraft"
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
default = ["denoise"]
|
||||
denoise = ["lightcraft-engine/denoise"]
|
||||
rawnind-model = ["denoise", "lightcraft-engine/rawnind-model"]
|
||||
# Open HEIC/HEIF photos (the optional lightcraft-heif crate). Not default so distributors choose
|
||||
# it (HEVC patents); build with `--features heif` to include it.
|
||||
heif = ["lightcraft-codecs/heif"]
|
||||
|
||||
[dependencies]
|
||||
lightcraft-engine = { workspace = true }
|
||||
lightcraft-engine = { workspace = true, features = ["sam"] }
|
||||
lightcraft-sysmem = { workspace = true }
|
||||
lightcraft-ui-egui = { workspace = true }
|
||||
lightcraft-codecs = { workspace = true }
|
||||
|
||||
@@ -1,62 +1,267 @@
|
||||
//! Loopback JSON-lines control server: one request per line, one reply per line.
|
||||
//! This is the transport the MCP server (`lightcraft-cli mcp --connect`) wraps.
|
||||
//!
|
||||
//! The server only ever reads requests: a line that is not a JSON object with a string `method`
|
||||
//! (an HTTP request line from a browser's cross-origin `fetch`, a stray `nc`, binary junk) gets
|
||||
//! one error reply and the connection is closed, so nothing after it is executed. Lines are
|
||||
//! bounded ([`MAX_LINE`]) and so is the number of open connections ([`MAX_CONNECTIONS`]); junk
|
||||
//! never reaches the UI thread. See `docs/control-protocol.md`.
|
||||
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::io::{BufRead, BufReader, Read, Write};
|
||||
use std::net::{TcpListener, TcpStream};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::mpsc::{Receiver, Sender, channel};
|
||||
use std::time::Duration;
|
||||
|
||||
use lightcraft_ui_egui::ControlRequest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
/// Longest accepted request line (bytes, without the newline). Requests are small JSON objects;
|
||||
/// anything longer is refused and the connection closed.
|
||||
pub const MAX_LINE: usize = 4 * 1024 * 1024;
|
||||
/// Connections served at once; further ones get an error line and are closed.
|
||||
pub const MAX_CONNECTIONS: usize = 16;
|
||||
|
||||
pub fn start(port: u16, ctx: egui::Context) -> Receiver<ControlRequest> {
|
||||
let (tx, rx) = channel::<ControlRequest>();
|
||||
let listener = match TcpListener::bind(("127.0.0.1", port)) {
|
||||
Ok(l) => l,
|
||||
Err(e) => {
|
||||
eprintln!("lightcraft: control server failed to bind 127.0.0.1:{port}: {e}");
|
||||
log::error!("control server failed to bind 127.0.0.1:{port}: {e}");
|
||||
return rx;
|
||||
}
|
||||
};
|
||||
eprintln!("lightcraft: control server listening on 127.0.0.1:{port}");
|
||||
std::thread::spawn(move || {
|
||||
for stream in listener.incoming().flatten() {
|
||||
let tx = tx.clone();
|
||||
let ctx = ctx.clone();
|
||||
std::thread::spawn(move || serve(stream, tx, ctx));
|
||||
}
|
||||
});
|
||||
log::info!("control server listening on 127.0.0.1:{port}");
|
||||
std::thread::spawn(move || accept_loop(listener, tx, ctx));
|
||||
rx
|
||||
}
|
||||
|
||||
fn serve(stream: TcpStream, tx: Sender<ControlRequest>, ctx: egui::Context) {
|
||||
let Ok(read) = stream.try_clone() else { return };
|
||||
let mut out = stream;
|
||||
for line in BufReader::new(read).lines() {
|
||||
let Ok(line) = line else { break };
|
||||
if line.trim().is_empty() {
|
||||
fn accept_loop(listener: TcpListener, tx: Sender<ControlRequest>, ctx: egui::Context) {
|
||||
let open = Arc::new(AtomicUsize::new(0));
|
||||
for mut stream in listener.incoming().flatten() {
|
||||
if open.fetch_add(1, Ordering::SeqCst) >= MAX_CONNECTIONS {
|
||||
open.fetch_sub(1, Ordering::SeqCst);
|
||||
let _ = writeln!(stream, "{}", json!({"ok": false, "error": "too many control connections"}));
|
||||
continue;
|
||||
}
|
||||
let reply = match serde_json::from_str::<Value>(&line) {
|
||||
Ok(msg) => {
|
||||
let id = msg.get("id").cloned().unwrap_or(Value::Null);
|
||||
let method = msg.get("method").and_then(Value::as_str).unwrap_or("").to_string();
|
||||
let params = msg.get("params").cloned().unwrap_or(json!({}));
|
||||
let (req, rrx) = ControlRequest::new(method, params);
|
||||
if tx.send(req).is_err() {
|
||||
break;
|
||||
}
|
||||
ctx.request_repaint();
|
||||
let mut r = rrx.recv_timeout(Duration::from_secs(60)).unwrap_or_else(|_| json!({"ok": false, "error": "timeout"}));
|
||||
if let Some(o) = r.as_object_mut() {
|
||||
o.insert("id".into(), id);
|
||||
let tx = tx.clone();
|
||||
let ctx = ctx.clone();
|
||||
let open = Arc::clone(&open);
|
||||
std::thread::spawn(move || {
|
||||
serve(stream, &tx, &ctx);
|
||||
open.fetch_sub(1, Ordering::SeqCst);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn serve(stream: TcpStream, tx: &Sender<ControlRequest>, ctx: &egui::Context) {
|
||||
let Ok(read) = stream.try_clone() else { return };
|
||||
serve_lines(read, stream, |method, params| {
|
||||
let (req, rrx) = ControlRequest::new(method, params);
|
||||
tx.send(req).ok()?;
|
||||
ctx.request_repaint();
|
||||
Some(rrx.recv_timeout(Duration::from_secs(60)).unwrap_or_else(|_| json!({"ok": false, "error": "timeout"})))
|
||||
});
|
||||
}
|
||||
|
||||
/// One parsed line.
|
||||
enum Line {
|
||||
Request {
|
||||
id: Value,
|
||||
method: String,
|
||||
params: Value,
|
||||
},
|
||||
Blank,
|
||||
/// Not a request: reply with this error and close the connection.
|
||||
Reject(String),
|
||||
}
|
||||
|
||||
fn parse_line(bytes: &[u8]) -> Line {
|
||||
let Ok(text) = std::str::from_utf8(bytes) else {
|
||||
return Line::Reject("not a control request (invalid UTF-8); closing the connection".into());
|
||||
};
|
||||
let text = text.trim();
|
||||
if text.is_empty() {
|
||||
return Line::Blank;
|
||||
}
|
||||
let msg = match serde_json::from_str::<Value>(text) {
|
||||
Ok(v) => v,
|
||||
Err(e) => return Line::Reject(format!("bad JSON ({e}); expected one JSON request object per line; closing the connection")),
|
||||
};
|
||||
let Some(obj) = msg.as_object() else {
|
||||
return Line::Reject("not a control request (expected a JSON object); closing the connection".into());
|
||||
};
|
||||
let Some(method) = obj.get("method").and_then(Value::as_str) else {
|
||||
return Line::Reject("not a control request (missing string `method`); closing the connection".into());
|
||||
};
|
||||
Line::Request {
|
||||
id: obj.get("id").cloned().unwrap_or(Value::Null),
|
||||
method: method.to_string(),
|
||||
params: obj.get("params").cloned().unwrap_or(json!({})),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read one `\n`-terminated line of at most `max` bytes into `buf` (newline stripped).
|
||||
/// `Ok(false)` at end of stream; `Err` when the line is too long or the read fails.
|
||||
fn read_bounded_line(reader: &mut impl BufRead, buf: &mut Vec<u8>, max: usize) -> Result<bool, String> {
|
||||
buf.clear();
|
||||
// `take` bounds the bytes buffered for one line: a peer can't make us allocate more than `max`.
|
||||
let limit = u64::try_from(max).unwrap_or(u64::MAX).saturating_add(1);
|
||||
let n = reader.take(limit).read_until(b'\n', buf).map_err(|e| e.to_string())?;
|
||||
if n == 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
if buf.last() == Some(&b'\n') {
|
||||
buf.pop();
|
||||
if buf.last() == Some(&b'\r') {
|
||||
buf.pop();
|
||||
}
|
||||
return Ok(true);
|
||||
}
|
||||
if buf.len() > max {
|
||||
return Err(format!("request line longer than {max} bytes; closing the connection"));
|
||||
}
|
||||
// Last line without a newline, then end of stream.
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Serve requests from `read`, answering on `out`, until the peer closes, sends something that is
|
||||
/// not a request, or `handle` returns `None` (app gone). `handle` gets `(method, params)` and
|
||||
/// returns the reply object (the request's `id` is added here).
|
||||
fn serve_lines<R: Read, W: Write>(read: R, mut out: W, mut handle: impl FnMut(String, Value) -> Option<Value>) {
|
||||
let mut reader = BufReader::new(read);
|
||||
let mut buf = Vec::new();
|
||||
loop {
|
||||
let reject = match read_bounded_line(&mut reader, &mut buf, MAX_LINE) {
|
||||
Ok(false) => return,
|
||||
Err(e) => e,
|
||||
Ok(true) => match parse_line(&buf) {
|
||||
Line::Blank => continue,
|
||||
Line::Reject(e) => e,
|
||||
Line::Request { id, method, params } => {
|
||||
let Some(mut r) = handle(method, params) else { return };
|
||||
if let Some(o) = r.as_object_mut() {
|
||||
o.insert("id".into(), id);
|
||||
}
|
||||
if writeln!(out, "{r}").and_then(|()| out.flush()).is_err() {
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
r
|
||||
}
|
||||
Err(e) => json!({"ok": false, "error": format!("bad JSON: {e}")}),
|
||||
},
|
||||
};
|
||||
if writeln!(out, "{reply}").is_err() {
|
||||
break;
|
||||
let _ = writeln!(out, "{}", json!({"ok": false, "error": reject})).and_then(|()| out.flush());
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn run(input: &[u8]) -> (Vec<Value>, Vec<String>) {
|
||||
let mut out = Vec::new();
|
||||
let mut calls = Vec::new();
|
||||
serve_lines(input, &mut out, |m, _p| {
|
||||
calls.push(m);
|
||||
Some(json!({"ok": true, "result": null}))
|
||||
});
|
||||
let replies = String::from_utf8(out).unwrap().lines().map(|l| serde_json::from_str::<Value>(l).unwrap()).collect();
|
||||
(replies, calls)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn requests_are_answered_in_order_with_ids() {
|
||||
let (r, calls) = run(b"{\"id\":1,\"method\":\"a\"}\n\n{\"id\":2,\"method\":\"b\",\"params\":{}}\r\n{\"method\":\"c\"}");
|
||||
assert_eq!(calls, ["a", "b", "c"]);
|
||||
assert_eq!(r.len(), 3);
|
||||
assert_eq!(r[0]["id"], 1);
|
||||
assert_eq!(r[1]["id"], 2);
|
||||
assert_eq!(r[2]["id"], Value::Null);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_request_closes_before_the_body_runs() {
|
||||
// A browser's cross-origin `fetch` POST: the request line is rejected and the connection
|
||||
// closed, so the JSON body is never executed.
|
||||
let http = b"POST / HTTP/1.1\r\nHost: 127.0.0.1:7980\r\nContent-Type: text/plain\r\n\r\n\
|
||||
{\"id\":1,\"method\":\"engine.execute\",\"params\":{\"command\":\"folder.move\"}}\n";
|
||||
let (r, calls) = run(http);
|
||||
assert!(calls.is_empty(), "nothing may run: {calls:?}");
|
||||
assert_eq!(r.len(), 1);
|
||||
assert_eq!(r[0]["ok"], false);
|
||||
assert!(r[0]["error"].as_str().unwrap().contains("closing the connection"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_request_json_closes_the_connection() {
|
||||
for junk in [
|
||||
&b"[1,2]\n{\"method\":\"a\"}\n"[..],
|
||||
b"{\"id\":1}\n{\"method\":\"a\"}\n",
|
||||
b"{\"method\":5}\n{\"method\":\"a\"}\n",
|
||||
b"\xff\xfe\n{\"method\":\"a\"}\n",
|
||||
] {
|
||||
let (r, calls) = run(junk);
|
||||
assert!(calls.is_empty());
|
||||
assert_eq!(r.len(), 1);
|
||||
assert_eq!(r[0]["ok"], false);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_request_before_junk_still_runs() {
|
||||
let (r, calls) = run(b"{\"method\":\"a\"}\nGET / HTTP/1.1\n{\"method\":\"b\"}\n");
|
||||
assert_eq!(calls, ["a"]);
|
||||
assert_eq!(r.len(), 2);
|
||||
assert_eq!(r[1]["ok"], false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overlong_line_is_refused_without_buffering_it_all() {
|
||||
// An endless line without a newline: reading stops after MAX_LINE + 1 bytes.
|
||||
let endless = std::io::repeat(b'x');
|
||||
let mut out = Vec::new();
|
||||
let mut ran = false;
|
||||
serve_lines(endless, &mut out, |_, _| {
|
||||
ran = true;
|
||||
None
|
||||
});
|
||||
assert!(!ran);
|
||||
let reply: Value = serde_json::from_slice(out.strip_suffix(b"\n").unwrap()).unwrap();
|
||||
assert!(reply["error"].as_str().unwrap().contains("longer than"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_at_the_limit_is_accepted() {
|
||||
let mut buf = Vec::new();
|
||||
let line = vec![b'a'; 10];
|
||||
let mut input = line.clone();
|
||||
input.push(b'\n');
|
||||
assert_eq!(read_bounded_line(&mut &input[..], &mut buf, 10), Ok(true));
|
||||
assert_eq!(buf, line);
|
||||
assert!(read_bounded_line(&mut &[b'a'; 11][..], &mut buf, 10).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tcp_connection_is_closed_after_junk() {
|
||||
// End to end over a real loopback socket (the transport the app uses).
|
||||
let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = std::thread::spawn(move || {
|
||||
let (s, _) = listener.accept().unwrap();
|
||||
let read = s.try_clone().unwrap();
|
||||
serve_lines(read, s, |_, _| Some(json!({"ok": true})));
|
||||
});
|
||||
let mut c = TcpStream::connect(addr).unwrap();
|
||||
c.set_read_timeout(Some(Duration::from_secs(10))).unwrap();
|
||||
c.write_all(b"{\"id\":7,\"method\":\"x\"}\nhello\n{\"method\":\"y\"}\n").unwrap();
|
||||
let mut text = String::new();
|
||||
c.read_to_string(&mut text).unwrap(); // returns at EOF: the server closed the connection
|
||||
server.join().unwrap();
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
assert_eq!(lines.len(), 2, "{text}");
|
||||
assert!(lines[0].contains("\"id\":7"));
|
||||
assert!(lines[1].contains("closing the connection"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
//! File-dialog filters that match extensions whatever their letter case.
|
||||
//!
|
||||
//! On Linux, `rfd` turns each extension into a `*.ext` glob for the GTK dialog and for the
|
||||
//! desktop portal (`xdg-desktop-portal` FileChooser), and both match globs case-sensitively, so
|
||||
//! a filter of `nef` hid `DSC_1234.NEF` (issue #342). Windows and macOS ignore case. Every
|
||||
//! dialog filter of the app goes through [`FileDialogExt::add_filter_nocase`] (a test checks
|
||||
//! that nothing else calls `add_filter`), so no extension can hit this.
|
||||
|
||||
use rfd::FileDialog;
|
||||
|
||||
/// The patterns to hand to `rfd` for `exts`, which are plain extensions without the dot.
|
||||
///
|
||||
/// Windows and macOS take the extension literally and ignore case: lower case, de-duplicated.
|
||||
/// Linux (`linux`): `rfd` formats each entry as `*.{entry}`, which GTK 3 and the portal match
|
||||
/// as a glob, so an entry may carry glob syntax. Each extension gives the lower-case form, the
|
||||
/// UPPER-case form (kept in case a portal backend does not do `[...]` classes) and a bracket
|
||||
/// form such as `[nN][eE][fF]` that matches every mix (`.Nef`, `.nEf`, ...).
|
||||
pub fn dialog_extensions_for(exts: &[&str], linux: bool) -> Vec<String> {
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
let mut push = |s: String| {
|
||||
if !s.is_empty() && !out.contains(&s) {
|
||||
out.push(s);
|
||||
}
|
||||
};
|
||||
for ext in exts {
|
||||
let ext = ext.trim_start_matches('.');
|
||||
let lower = ext.to_lowercase();
|
||||
push(lower.clone());
|
||||
if !linux {
|
||||
continue;
|
||||
}
|
||||
push(ext.to_uppercase());
|
||||
// `[` `]` `*` `?` in an extension would already be glob syntax: leave those alone
|
||||
if lower.chars().any(|c| c.is_alphabetic()) && !lower.contains(['[', ']', '*', '?', '\\']) {
|
||||
push(
|
||||
lower
|
||||
.chars()
|
||||
.map(|c| {
|
||||
let up: String = c.to_uppercase().collect();
|
||||
if up.chars().count() == 1 && up != c.to_string() { format!("[{c}{up}]") } else { c.to_string() }
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// [`dialog_extensions_for`] for the platform this is built for.
|
||||
pub fn dialog_extensions(exts: &[&str]) -> Vec<String> {
|
||||
dialog_extensions_for(exts, cfg!(all(unix, not(target_os = "macos"))))
|
||||
}
|
||||
|
||||
/// `add_filter` that matches the extensions in any letter case.
|
||||
pub trait FileDialogExt {
|
||||
fn add_filter_nocase(self, name: impl Into<String>, exts: &[&str]) -> Self;
|
||||
}
|
||||
|
||||
impl FileDialogExt for FileDialog {
|
||||
fn add_filter_nocase(self, name: impl Into<String>, exts: &[&str]) -> Self {
|
||||
self.add_filter(name, &dialog_extensions(exts))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn other_platforms_get_lower_case_once() {
|
||||
assert_eq!(dialog_extensions_for(&["NEF", "nef", ".Jpg", "jpg", ""], false), ["nef", "jpg"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn linux_gets_lower_upper_and_any_mix() {
|
||||
assert_eq!(dialog_extensions_for(&["nef"], true), ["nef", "NEF", "[nN][eE][fF]"]);
|
||||
// digits have no case; a duplicate adds nothing
|
||||
assert_eq!(dialog_extensions_for(&["mp4", "MP4"], true), ["mp4", "MP4", "[mM][pP]4"]);
|
||||
assert_eq!(dialog_extensions_for(&["7z"], true), ["7z", "7Z", "7[zZ]"]);
|
||||
assert_eq!(dialog_extensions_for(&["123"], true), ["123"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn linux_glob_syntax_is_not_wrapped_again() {
|
||||
assert_eq!(dialog_extensions_for(&["[nN]ef"], true), ["[nn]ef", "[NN]EF"]);
|
||||
}
|
||||
|
||||
/// The Linux patterns match like GTK's glob (`*.{entry}`): checked against a tiny matcher of
|
||||
/// the subset used here (verified against real GTK 3.24 `GtkFileFilter` in the PR notes).
|
||||
fn glob_ext(entry: &str, ext: &str) -> bool {
|
||||
let mut pat = entry.chars().peekable();
|
||||
let mut s = ext.chars();
|
||||
while let Some(p) = pat.next() {
|
||||
let Some(c) = s.next() else { return false };
|
||||
if p == '[' {
|
||||
let mut hit = false;
|
||||
for q in pat.by_ref() {
|
||||
if q == ']' {
|
||||
break;
|
||||
}
|
||||
hit |= q == c;
|
||||
}
|
||||
if !hit {
|
||||
return false;
|
||||
}
|
||||
} else if p != c {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
s.next().is_none()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn linux_patterns_match_every_case_mix() {
|
||||
let pats = dialog_extensions_for(&["nef", "cr2", "xmp"], true);
|
||||
let matches = |name: &str| pats.iter().any(|p| glob_ext(p, name));
|
||||
for ok in ["nef", "NEF", "Nef", "nEF", "CR2", "Cr2", "XMP", "xMp"] {
|
||||
assert!(matches(ok), "{ok}");
|
||||
}
|
||||
for bad in ["nex", "ne", "nefs", "jpg", "cr3"] {
|
||||
assert!(!matches(bad), "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The app's own extension list (the one the folder scan uses) is covered whole.
|
||||
#[test]
|
||||
fn import_extensions_are_all_covered_in_both_cases() {
|
||||
let pats = dialog_extensions_for(lightcraft_engine::import::EXTENSIONS, true);
|
||||
for e in lightcraft_engine::import::EXTENSIONS {
|
||||
assert!(pats.iter().any(|p| glob_ext(p, &e.to_uppercase())), "{e}");
|
||||
assert!(pats.iter().any(|p| p == e), "{e}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Every dialog filter goes through the helper: `main.rs` must not call rfd's own
|
||||
/// `add_filter`.
|
||||
#[test]
|
||||
fn no_dialog_bypasses_the_helper() {
|
||||
for (file, src) in [("main.rs", include_str!("main.rs")), ("control_server.rs", include_str!("control_server.rs"))] {
|
||||
assert!(!src.contains(".add_filter("), "{file} calls rfd's add_filter; use add_filter_nocase");
|
||||
}
|
||||
assert!(include_str!("main.rs").contains(".add_filter_nocase("));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,689 @@
|
||||
//! The desktop app's logger: `log::` records go to standard error and to a log file.
|
||||
//!
|
||||
//! It replaces the stderr-only `StderrLog` that `main.rs` used to install. A launch from a
|
||||
//! desktop menu or the Dock has no terminal, so the file is what a bug report can attach:
|
||||
//! `<settings folder>/logs/lightcraft.log`, next to `ui.json` (`config_dir` in `main.rs`; Linux
|
||||
//! `$XDG_CONFIG_HOME/lightcraft/logs`, by default `~/.config/lightcraft/logs`). Never in the
|
||||
//! library folder. Each start moves the previous log to `lightcraft.1.log` (and that one to `.2`),
|
||||
//! so the log of a run that crashed survives the next launch. Runs with `LIGHTCRAFT_NO_PREFS`
|
||||
//! (tests, scripts) log to standard error only, so they don't rotate away the user's own logs.
|
||||
//!
|
||||
//! Levels ([`filter_spec`]): `info` for LightCraft's own crates, `warn` for everything else (wgpu
|
||||
//! and naga are chatty). `LIGHTCRAFT_LOG` keeps the meaning it had with the old logger: `info` or
|
||||
//! `debug` lowers LightCraft's own crates to that level, any other value means warnings and errors
|
||||
//! only. Without it, `RUST_LOG` replaces the default with env_logger-style directives: `debug`,
|
||||
//! `warn,lightcraft_pipeline=trace`, `wgpu_core=info`. A directive ending in `*` matches every
|
||||
//! target that starts with it (`lightcraft*=debug`).
|
||||
//!
|
||||
//! Records logged before the settings folder is known are kept (up to [`MAX_PENDING`]) and
|
||||
//! written once the file is attached. Writing never panics: a file that can't be created or
|
||||
//! written leaves standard error as the only sink. Panics that escape everything are added to the
|
||||
//! file by [`AppLogger::record_panic`] (the default hook already printed them on standard error).
|
||||
//!
|
||||
//! This is the desktop app's logger only; `lightcraft-cli` doesn't share a process with it.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Mutex, OnceLock, PoisonError, TryLockError};
|
||||
use std::time::SystemTime;
|
||||
|
||||
use log::LevelFilter;
|
||||
|
||||
/// The current log file's name inside the log folder.
|
||||
pub const LOG_FILE: &str = "lightcraft.log";
|
||||
/// How many previous logs are kept (`lightcraft.1.log` … `lightcraft.<KEEP>.log`).
|
||||
pub const KEEP: usize = 2;
|
||||
/// The log file stops growing past this size (a runaway warning can't fill the disk).
|
||||
pub const MAX_FILE_BYTES: u64 = 16 * 1024 * 1024;
|
||||
/// Records kept in memory until the log file is attached.
|
||||
pub const MAX_PENDING: usize = 512;
|
||||
/// The built-in filter when neither `LIGHTCRAFT_LOG` nor `RUST_LOG` says otherwise.
|
||||
pub const DEFAULT_FILTER: &str = "warn,lightcraft*=info";
|
||||
|
||||
/// The filter directives for this run from `LIGHTCRAFT_LOG` and `RUST_LOG` (blank = unset).
|
||||
///
|
||||
/// `LIGHTCRAFT_LOG` wins when set and means what it did for the old stderr logger: `info` or
|
||||
/// `debug` for LightCraft's own crates (warnings and errors from everything else), any other value
|
||||
/// warnings and errors only. Else `RUST_LOG` as env_logger directives, else [`DEFAULT_FILTER`].
|
||||
pub fn filter_spec(lightcraft_log: Option<&str>, rust_log: Option<&str>) -> String {
|
||||
let set = |v: Option<&str>| v.map(str::trim).filter(|v| !v.is_empty()).map(str::to_owned);
|
||||
if let Some(level) = set(lightcraft_log) {
|
||||
return match level.as_str() {
|
||||
"debug" => "warn,lightcraft*=debug".to_owned(),
|
||||
"info" => "warn,lightcraft*=info".to_owned(),
|
||||
_ => "warn".to_owned(),
|
||||
};
|
||||
}
|
||||
set(rust_log).unwrap_or_else(|| DEFAULT_FILTER.to_owned())
|
||||
}
|
||||
|
||||
/// Per-target level filter parsed from env_logger-style directives.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Filter {
|
||||
default: LevelFilter,
|
||||
directives: Vec<(String, LevelFilter)>,
|
||||
}
|
||||
|
||||
impl Filter {
|
||||
/// Parse `spec`; unknown levels and empty parts are skipped. Without a bare level, targets no
|
||||
/// directive names log errors only.
|
||||
pub fn parse(spec: &str) -> Filter {
|
||||
let mut f = Filter { default: LevelFilter::Error, directives: Vec::new() };
|
||||
for part in spec.split(',').map(str::trim).filter(|p| !p.is_empty()) {
|
||||
match part.split_once('=') {
|
||||
Some((name, level)) => {
|
||||
let name = name.trim();
|
||||
if let (false, Some(level)) = (name.is_empty(), parse_level(level)) {
|
||||
f.directives.push((name.to_owned(), level));
|
||||
}
|
||||
}
|
||||
None => match parse_level(part) {
|
||||
Some(level) => f.default = level,
|
||||
// A bare target name: everything from it (env_logger does the same).
|
||||
None => f.directives.push((part.to_owned(), LevelFilter::Trace)),
|
||||
},
|
||||
}
|
||||
}
|
||||
f
|
||||
}
|
||||
|
||||
/// The level that applies to `target` (the most specific matching directive wins).
|
||||
pub fn level_for(&self, target: &str) -> LevelFilter {
|
||||
self.directives.iter().filter(|(name, _)| matches(name, target)).max_by_key(|(name, _)| name.len()).map_or(self.default, |(_, level)| *level)
|
||||
}
|
||||
|
||||
/// The most verbose level any target can reach (for `log::set_max_level`).
|
||||
pub fn max(&self) -> LevelFilter {
|
||||
self.directives.iter().map(|(_, level)| *level).fold(self.default, Ord::max)
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_level(s: &str) -> Option<LevelFilter> {
|
||||
s.trim().parse().ok()
|
||||
}
|
||||
|
||||
/// `name` matches `target` itself and its submodules (`a` matches `a` and `a::b`, not `ab`);
|
||||
/// `name*` matches every target starting with `name`.
|
||||
fn matches(name: &str, target: &str) -> bool {
|
||||
match name.strip_suffix('*') {
|
||||
Some(prefix) => target.starts_with(prefix),
|
||||
None => target.strip_prefix(name).is_some_and(|rest| rest.is_empty() || rest.starts_with("::")),
|
||||
}
|
||||
}
|
||||
|
||||
/// `2026-10-08T07:59:17.728Z` (UTC, milliseconds). A clock before 1970 reads as the epoch.
|
||||
pub fn timestamp(t: SystemTime) -> String {
|
||||
let since = t.duration_since(std::time::UNIX_EPOCH).unwrap_or_default();
|
||||
let secs = since.as_secs();
|
||||
let (days, rem) = (secs / 86_400, secs % 86_400);
|
||||
let (y, m, d) = civil_from_days(days);
|
||||
format!("{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}.{:03}Z", rem / 3600, rem % 3600 / 60, rem % 60, since.subsec_millis())
|
||||
}
|
||||
|
||||
/// Days since 1970-01-01 to a proleptic Gregorian (year, month, day) (Howard Hinnant's
|
||||
/// `civil_from_days`, unsigned because `timestamp` clamps at the epoch). Saturating, so no clock
|
||||
/// value can overflow it.
|
||||
fn civil_from_days(days: u64) -> (u64, u64, u64) {
|
||||
let z = days.saturating_add(719_468);
|
||||
let era = z / 146_097;
|
||||
let doe = z % 146_097;
|
||||
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
|
||||
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
||||
let mp = (5 * doy + 2) / 153;
|
||||
let d = doy - (153 * mp + 2) / 5 + 1;
|
||||
let m = if mp < 10 { mp + 3 } else { mp - 9 };
|
||||
let y = era.saturating_mul(400).saturating_add(yoe).saturating_add(u64::from(m <= 2));
|
||||
(y, m, d)
|
||||
}
|
||||
|
||||
fn numbered(dir: &Path, n: usize) -> PathBuf {
|
||||
if n == 0 { dir.join(LOG_FILE) } else { dir.join(format!("lightcraft.{n}.log")) }
|
||||
}
|
||||
|
||||
/// Shift the previous logs up one (`.log` → `.1.log` → … → `.<KEEP>.log`, the oldest dropped)
|
||||
/// and create a fresh, empty log file in `dir` (created if missing).
|
||||
pub fn rotate(dir: &Path) -> std::io::Result<(PathBuf, File)> {
|
||||
std::fs::create_dir_all(dir)?;
|
||||
for n in (1..=KEEP).rev() {
|
||||
match std::fs::rename(numbered(dir, n - 1), numbered(dir, n)) {
|
||||
Err(e) if e.kind() != std::io::ErrorKind::NotFound => return Err(e),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let path = numbered(dir, 0);
|
||||
let file = File::create(&path)?;
|
||||
Ok((path, file))
|
||||
}
|
||||
|
||||
/// One formatted log line: `<timestamp> <LEVEL> [<thread>] <target>: <message>\n`.
|
||||
pub fn format_line(ts: &str, level: log::Level, thread: &str, target: &str, message: &str) -> String {
|
||||
let message = message.strip_suffix('\n').unwrap_or(message);
|
||||
format!("{ts} {level:<5} [{thread}] {target}: {message}\n")
|
||||
}
|
||||
|
||||
/// Where formatted lines go besides standard error: kept in memory until a file is attached,
|
||||
/// then appended to it until it reaches its size cap.
|
||||
pub struct Sink {
|
||||
file: Option<File>,
|
||||
pending: Vec<String>,
|
||||
dropped: usize,
|
||||
written: u64,
|
||||
max_bytes: u64,
|
||||
capped: bool,
|
||||
}
|
||||
|
||||
impl Sink {
|
||||
pub fn new(max_bytes: u64) -> Sink {
|
||||
Sink { file: None, pending: Vec::new(), dropped: 0, written: 0, max_bytes, capped: false }
|
||||
}
|
||||
|
||||
/// Append `line` to the file, or keep it until [`Sink::attach`].
|
||||
pub fn write(&mut self, line: &str) {
|
||||
if self.file.is_none() {
|
||||
if self.capped {
|
||||
// No file is coming ([`Sink::no_file`]): standard error had the line.
|
||||
} else if self.pending.len() < MAX_PENDING {
|
||||
self.pending.push(line.to_owned());
|
||||
} else {
|
||||
self.dropped = self.dropped.saturating_add(1);
|
||||
}
|
||||
return;
|
||||
}
|
||||
self.append(line);
|
||||
}
|
||||
|
||||
/// Start writing to `file` (a fresh one: the size cap starts over), first the lines kept so far.
|
||||
pub fn attach(&mut self, file: File) {
|
||||
self.file = Some(file);
|
||||
self.written = 0;
|
||||
self.capped = false;
|
||||
for line in std::mem::take(&mut self.pending) {
|
||||
self.append(&line);
|
||||
}
|
||||
if self.dropped > 0 {
|
||||
let note = format!("{} earlier log lines were dropped before the log file was opened\n", self.dropped);
|
||||
self.dropped = 0;
|
||||
self.append(¬e);
|
||||
}
|
||||
}
|
||||
|
||||
/// There will be no log file: forget the lines kept for it and keep no more.
|
||||
pub fn no_file(&mut self) {
|
||||
self.file = None;
|
||||
self.pending = Vec::new();
|
||||
self.dropped = 0;
|
||||
self.capped = true;
|
||||
}
|
||||
|
||||
fn append(&mut self, line: &str) {
|
||||
if self.capped {
|
||||
return;
|
||||
}
|
||||
let Some(file) = self.file.as_mut() else { return };
|
||||
let len = u64::try_from(line.len()).unwrap_or(u64::MAX);
|
||||
let text = if self.written.saturating_add(len) > self.max_bytes {
|
||||
self.capped = true;
|
||||
format!("log file reached {} bytes; later records go to standard error only\n", self.max_bytes)
|
||||
} else {
|
||||
self.written = self.written.saturating_add(len);
|
||||
line.to_owned()
|
||||
};
|
||||
// A full disk or a vanished file must not take the app down: drop the file sink.
|
||||
if file.write_all(text.as_bytes()).is_err() {
|
||||
self.file = None;
|
||||
self.capped = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The installed logger: a [`Filter`], standard error and a [`Sink`].
|
||||
pub struct AppLogger {
|
||||
filter: Filter,
|
||||
stderr: bool,
|
||||
sink: Mutex<Sink>,
|
||||
}
|
||||
|
||||
impl AppLogger {
|
||||
pub fn new(filter: Filter, stderr: bool) -> AppLogger {
|
||||
AppLogger { filter, stderr, sink: Mutex::new(Sink::new(MAX_FILE_BYTES)) }
|
||||
}
|
||||
|
||||
/// Rotate the logs in `dir` and send records to the fresh file; returns its path. On an error
|
||||
/// standard error stays the only sink.
|
||||
pub fn attach_dir(&self, dir: &Path) -> Result<PathBuf, String> {
|
||||
// A thread that panicked while logging leaves the sink usable: take it back.
|
||||
let mut sink = self.sink.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
match rotate(dir) {
|
||||
Ok((path, file)) => {
|
||||
sink.attach(file);
|
||||
Ok(path)
|
||||
}
|
||||
Err(e) => {
|
||||
sink.no_file();
|
||||
Err(format!("{}: {e}", dir.display()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// No log file for this run (`LIGHTCRAFT_NO_PREFS`): log to standard error only.
|
||||
pub fn no_file(&self) {
|
||||
self.sink.lock().unwrap_or_else(PoisonError::into_inner).no_file();
|
||||
}
|
||||
|
||||
/// Add a panic report to the log file only (the default panic hook has printed it on standard
|
||||
/// error), whatever the filter. Skipped when the sink is busy: the panic may have happened on
|
||||
/// this very thread while it held the lock, and waiting would hang the app.
|
||||
pub fn record_panic(&self, thread: &str, report: &str) {
|
||||
let line = format_line(×tamp(SystemTime::now()), log::Level::Error, thread, "lightcraft", report);
|
||||
let mut sink = match self.sink.try_lock() {
|
||||
Ok(sink) => sink,
|
||||
Err(TryLockError::Poisoned(p)) => p.into_inner(),
|
||||
Err(TryLockError::WouldBlock) => return,
|
||||
};
|
||||
sink.write(&line);
|
||||
}
|
||||
}
|
||||
|
||||
impl log::Log for AppLogger {
|
||||
fn enabled(&self, metadata: &log::Metadata<'_>) -> bool {
|
||||
metadata.level() <= self.filter.level_for(metadata.target())
|
||||
}
|
||||
|
||||
fn log(&self, record: &log::Record<'_>) {
|
||||
if !self.enabled(record.metadata()) {
|
||||
return;
|
||||
}
|
||||
// Formatted before the sink is locked, so a message that logs while it is formatted can't deadlock.
|
||||
let thread = std::thread::current();
|
||||
let line =
|
||||
format_line(×tamp(SystemTime::now()), record.level(), thread.name().unwrap_or("?"), record.target(), &record.args().to_string());
|
||||
if self.stderr {
|
||||
// No terminal (a Windows GUI build, a closed pipe) is not an error worth reporting.
|
||||
let _ = std::io::stderr().write_all(line.as_bytes());
|
||||
}
|
||||
self.sink.lock().unwrap_or_else(PoisonError::into_inner).write(&line);
|
||||
}
|
||||
|
||||
fn flush(&self) {}
|
||||
}
|
||||
|
||||
/// Install the logger (filter from [`filter_spec`]); `None` when another logger was installed
|
||||
/// first. Call [`AppLogger::attach_dir`] (or [`AppLogger::no_file`]) once the arguments are parsed.
|
||||
pub fn install() -> Option<&'static AppLogger> {
|
||||
static LOGGER: OnceLock<AppLogger> = OnceLock::new();
|
||||
let var = |name| std::env::var(name).ok();
|
||||
let filter = Filter::parse(&filter_spec(var("LIGHTCRAFT_LOG").as_deref(), var("RUST_LOG").as_deref()));
|
||||
let max = filter.max();
|
||||
let logger = LOGGER.get_or_init(|| AppLogger::new(filter, true));
|
||||
log::set_logger(logger).ok()?;
|
||||
log::set_max_level(max);
|
||||
Some(logger)
|
||||
}
|
||||
|
||||
/// Chain a panic hook after the current one (the engine guard's: standard error plus
|
||||
/// `lightcraft-panics.log` in the temp folder) that also puts the panic in the log file.
|
||||
pub fn record_panics(logger: &'static AppLogger) {
|
||||
let previous = std::panic::take_hook();
|
||||
std::panic::set_hook(Box::new(move |info| {
|
||||
previous(info);
|
||||
let thread = std::thread::current();
|
||||
let at = info.location().map(|l| format!(" at {}:{}", l.file(), l.line())).unwrap_or_default();
|
||||
let report = format!("panic{at}: {}", lightcraft_engine::guard::panic_message(info.payload()));
|
||||
logger.record_panic(thread.name().unwrap_or("?"), &report);
|
||||
}));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::{Duration, UNIX_EPOCH};
|
||||
|
||||
fn temp_dir(tag: &str) -> PathBuf {
|
||||
static N: AtomicUsize = AtomicUsize::new(0);
|
||||
let d = std::env::temp_dir().join(format!("lightcraft-logging-{tag}-{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed)));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
d
|
||||
}
|
||||
|
||||
fn read(path: &Path) -> String {
|
||||
std::fs::read_to_string(path).expect("read log")
|
||||
}
|
||||
|
||||
fn record(logger: &AppLogger, level: log::Level, target: &str, msg: &str) {
|
||||
use log::Log;
|
||||
logger.log(&log::Record::builder().level(level).target(target).args(format_args!("{msg}")).build());
|
||||
}
|
||||
|
||||
/// The logger this replaced (`StderrLog`): warnings and errors from every target, `info` and
|
||||
/// `debug` only from LightCraft's own crates, and only when `LIGHTCRAFT_LOG` asked for them.
|
||||
fn old_stderr_log_enabled(lightcraft_log: &str, level: log::Level, target: &str) -> bool {
|
||||
let max = match lightcraft_log {
|
||||
"debug" => LevelFilter::Debug,
|
||||
"info" => LevelFilter::Info,
|
||||
_ => LevelFilter::Warn,
|
||||
};
|
||||
level <= max && (level <= log::Level::Warn || target.starts_with("lightcraft"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_default_filter_shows_lightcraft_info_and_other_crates_warnings() {
|
||||
let f = Filter::parse(DEFAULT_FILTER);
|
||||
assert_eq!(f.level_for("lightcraft"), LevelFilter::Info);
|
||||
assert_eq!(f.level_for("lightcraft::control_server"), LevelFilter::Info);
|
||||
assert_eq!(f.level_for("lightcraft_engine::guard"), LevelFilter::Info);
|
||||
assert_eq!(f.level_for("lightcraft_ui_egui::render"), LevelFilter::Info);
|
||||
assert_eq!(f.level_for("wgpu_core::device"), LevelFilter::Warn);
|
||||
assert_eq!(f.level_for("naga"), LevelFilter::Warn);
|
||||
assert_eq!(f.max(), LevelFilter::Info);
|
||||
assert_eq!(filter_spec(None, None), DEFAULT_FILTER);
|
||||
// An empty variable is the same as an unset one.
|
||||
assert_eq!(filter_spec(Some(""), Some(" ")), DEFAULT_FILTER);
|
||||
}
|
||||
|
||||
/// `LIGHTCRAFT_LOG` keeps working exactly as it did with the old stderr-only logger.
|
||||
#[test]
|
||||
fn lightcraft_log_keeps_its_meaning() {
|
||||
let targets =
|
||||
["lightcraft", "lightcraft::control_server", "lightcraft_engine::segment", "lightcraft_gpu", "wgpu_core::device", "naga", "eframe"];
|
||||
let levels = [log::Level::Error, log::Level::Warn, log::Level::Info, log::Level::Debug, log::Level::Trace];
|
||||
for value in ["info", "debug", "warn", "error", "verbose", "INFO"] {
|
||||
let f = Filter::parse(&filter_spec(Some(value), None));
|
||||
for target in targets {
|
||||
for level in levels {
|
||||
assert_eq!(level <= f.level_for(target), old_stderr_log_enabled(value, level, target), "LIGHTCRAFT_LOG={value} {level} {target}");
|
||||
}
|
||||
}
|
||||
}
|
||||
assert_eq!(Filter::parse(&filter_spec(Some("debug"), None)).max(), LevelFilter::Debug);
|
||||
assert_eq!(Filter::parse(&filter_spec(Some("warn"), None)).max(), LevelFilter::Warn);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rust_log_replaces_the_default_and_lightcraft_log_wins_over_it() {
|
||||
assert_eq!(filter_spec(None, Some("warn,wgpu_core=info")), "warn,wgpu_core=info");
|
||||
let f = Filter::parse(&filter_spec(None, Some("debug")));
|
||||
assert_eq!(f.level_for("eframe"), LevelFilter::Debug);
|
||||
// The app's own variable is the more specific choice.
|
||||
let f = Filter::parse(&filter_spec(Some("info"), Some("trace")));
|
||||
assert_eq!(f.level_for("eframe"), LevelFilter::Warn);
|
||||
assert_eq!(f.level_for("lightcraft_engine"), LevelFilter::Info);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directives_follow_env_logger_and_the_most_specific_one_wins() {
|
||||
let f = Filter::parse("info,wgpu_core=error,lightcraft_pipeline=trace,lightcraft_pipeline::tiles=off");
|
||||
assert_eq!(f.level_for("eframe"), LevelFilter::Info);
|
||||
assert_eq!(f.level_for("wgpu_core::instance"), LevelFilter::Error);
|
||||
assert_eq!(f.level_for("lightcraft_pipeline::stage"), LevelFilter::Trace);
|
||||
assert_eq!(f.level_for("lightcraft_pipeline::tiles"), LevelFilter::Off);
|
||||
assert_eq!(f.max(), LevelFilter::Trace);
|
||||
// A module name is matched at `::` boundaries, not as a bare prefix.
|
||||
assert_eq!(f.level_for("wgpu_core_extra"), LevelFilter::Info);
|
||||
// A trailing `*` is a prefix.
|
||||
assert_eq!(Filter::parse("lightcraft*=debug").level_for("lightcraft_raw::cr2"), LevelFilter::Debug);
|
||||
assert_eq!(Filter::parse("lightcraft*=debug").level_for("eframe"), LevelFilter::Error);
|
||||
// A bare target name sets that target to the most verbose level, as env_logger does.
|
||||
assert_eq!(Filter::parse("naga").level_for("naga::front"), LevelFilter::Trace);
|
||||
assert_eq!(Filter::parse("naga").level_for("eframe"), LevelFilter::Error);
|
||||
// Levels are case-insensitive and surrounding blanks are ignored.
|
||||
assert_eq!(Filter::parse(" WARN , lightcraft = Debug ").level_for("lightcraft"), LevelFilter::Debug);
|
||||
assert_eq!(Filter::parse(" WARN , lightcraft = Debug ").level_for("eframe"), LevelFilter::Warn);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_specs_never_panic_and_fall_back_sensibly() {
|
||||
let long = "x".repeat(100_000);
|
||||
let many = "a=info,".repeat(10_000);
|
||||
for spec in [
|
||||
"",
|
||||
",,,",
|
||||
"=",
|
||||
"==",
|
||||
"=debug",
|
||||
"lightcraft=",
|
||||
"nonsense=loud",
|
||||
"🦀=info",
|
||||
"*",
|
||||
"*=",
|
||||
"=*",
|
||||
"a=b=c",
|
||||
"warn,,lightcraft=DEBUG",
|
||||
"\0",
|
||||
"é::ü=trace",
|
||||
&long,
|
||||
&many,
|
||||
] {
|
||||
let f = Filter::parse(spec);
|
||||
for target in ["", "lightcraft", "🦀", "a::b", "*"] {
|
||||
let _ = f.level_for(target);
|
||||
}
|
||||
let _ = f.max();
|
||||
let _ = filter_spec(Some(spec), Some(spec));
|
||||
}
|
||||
assert_eq!(Filter::parse("warn,,lightcraft=DEBUG").level_for("lightcraft"), LevelFilter::Debug);
|
||||
// An unknown level is skipped: that target keeps the default.
|
||||
assert_eq!(Filter::parse("nonsense=loud").level_for("nonsense"), LevelFilter::Error);
|
||||
assert_eq!(Filter::parse("lightcraft=").level_for("lightcraft"), LevelFilter::Error);
|
||||
assert_eq!(Filter::parse("=debug").max(), LevelFilter::Error);
|
||||
assert_eq!(Filter::parse("🦀=info").level_for("🦀::claw"), LevelFilter::Info);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timestamps_are_utc_with_milliseconds() {
|
||||
assert_eq!(timestamp(UNIX_EPOCH), "1970-01-01T00:00:00.000Z");
|
||||
assert_eq!(timestamp(UNIX_EPOCH + Duration::from_millis(1_791_446_357_728)), "2026-10-08T07:59:17.728Z");
|
||||
// Leap day, and the day after it.
|
||||
assert_eq!(timestamp(UNIX_EPOCH + Duration::from_secs(1_709_164_800)), "2024-02-29T00:00:00.000Z");
|
||||
assert_eq!(timestamp(UNIX_EPOCH + Duration::from_secs(1_709_251_199)), "2024-02-29T23:59:59.000Z");
|
||||
assert_eq!(timestamp(UNIX_EPOCH + Duration::from_secs(1_709_251_200)), "2024-03-01T00:00:00.000Z");
|
||||
// 2000 is a leap year (divisible by 400), 2100 is not.
|
||||
assert_eq!(timestamp(UNIX_EPOCH + Duration::from_secs(951_782_400)), "2000-02-29T00:00:00.000Z");
|
||||
assert_eq!(timestamp(UNIX_EPOCH + Duration::from_secs(4_107_542_400)), "2100-03-01T00:00:00.000Z");
|
||||
// Before the epoch (a clock set wrong) clamps instead of panicking.
|
||||
assert_eq!(timestamp(UNIX_EPOCH - Duration::from_secs(5)), "1970-01-01T00:00:00.000Z");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_line_carries_time_level_thread_target_and_message() {
|
||||
assert_eq!(
|
||||
format_line("2026-10-08T07:59:17.728Z", log::Level::Info, "main", "lightcraft", "library /photos: 12 photos"),
|
||||
"2026-10-08T07:59:17.728Z INFO [main] lightcraft: library /photos: 12 photos\n"
|
||||
);
|
||||
// A multi-line message keeps its lines; the record still ends in one newline.
|
||||
assert!(format_line("t", log::Level::Error, "w", "x", "a\nb\n").ends_with("x: a\nb\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rotation_keeps_the_previous_logs_and_starts_an_empty_file() {
|
||||
let dir = temp_dir("rotate");
|
||||
for run in 1..=4 {
|
||||
let (path, _file) = rotate(&dir).expect("rotate");
|
||||
assert_eq!(path, dir.join(LOG_FILE));
|
||||
assert_eq!(read(&path), "");
|
||||
std::fs::write(&path, format!("run {run}")).expect("write");
|
||||
}
|
||||
assert_eq!(read(&dir.join(LOG_FILE)), "run 4");
|
||||
assert_eq!(read(&dir.join("lightcraft.1.log")), "run 3");
|
||||
assert_eq!(read(&dir.join("lightcraft.2.log")), "run 2");
|
||||
assert!(!dir.join("lightcraft.3.log").exists(), "only {KEEP} old logs are kept");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rotation_into_an_unusable_directory_is_an_error_not_a_panic() {
|
||||
let dir = temp_dir("blocked");
|
||||
std::fs::create_dir_all(dir.parent().expect("parent")).expect("tmp");
|
||||
std::fs::write(&dir, "a file where the directory should be").expect("block");
|
||||
assert!(rotate(&dir).is_err());
|
||||
let _ = std::fs::remove_file(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lines_logged_before_the_file_exists_are_written_when_it_is_attached() {
|
||||
let dir = temp_dir("pending");
|
||||
let (path, file) = rotate(&dir).expect("rotate");
|
||||
let mut sink = Sink::new(MAX_FILE_BYTES);
|
||||
sink.write("early 1\n");
|
||||
sink.write("early 2\n");
|
||||
sink.attach(file);
|
||||
sink.write("late\n");
|
||||
assert_eq!(read(&path), "early 1\nearly 2\nlate\n");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_pending_buffer_is_bounded_and_says_how_much_it_dropped() {
|
||||
let dir = temp_dir("bounded");
|
||||
let (path, file) = rotate(&dir).expect("rotate");
|
||||
let mut sink = Sink::new(MAX_FILE_BYTES);
|
||||
for i in 0..MAX_PENDING + 5 {
|
||||
sink.write(&format!("line {i}\n"));
|
||||
}
|
||||
sink.attach(file);
|
||||
let text = read(&path);
|
||||
assert_eq!(text.lines().filter(|l| l.starts_with("line ")).count(), MAX_PENDING);
|
||||
assert!(text.contains("5 earlier log lines were dropped"), "{text}");
|
||||
assert!(text.contains("line 0\n") && !text.contains(&format!("line {MAX_PENDING}\n")));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exactly_a_full_pending_buffer_drops_nothing() {
|
||||
let dir = temp_dir("full");
|
||||
let (path, file) = rotate(&dir).expect("rotate");
|
||||
let mut sink = Sink::new(MAX_FILE_BYTES);
|
||||
for i in 0..MAX_PENDING {
|
||||
sink.write(&format!("line {i}\n"));
|
||||
}
|
||||
sink.attach(file);
|
||||
let text = read(&path);
|
||||
assert_eq!(text.lines().count(), MAX_PENDING);
|
||||
assert!(!text.contains("dropped"), "{text}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_a_file_nothing_is_kept_in_memory() {
|
||||
let mut sink = Sink::new(MAX_FILE_BYTES);
|
||||
sink.write("early\n");
|
||||
sink.no_file();
|
||||
sink.write("late\n");
|
||||
assert!(sink.pending.is_empty() && sink.dropped == 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_file_stops_at_its_size_cap_with_one_note() {
|
||||
let dir = temp_dir("cap");
|
||||
let (path, file) = rotate(&dir).expect("rotate");
|
||||
let mut sink = Sink::new(100);
|
||||
sink.attach(file);
|
||||
for i in 0..50 {
|
||||
sink.write(&format!("line {i:02} with some padding\n"));
|
||||
}
|
||||
let text = read(&path);
|
||||
assert_eq!(text.lines().filter(|l| l.starts_with("line ")).count(), 3, "{text}");
|
||||
assert_eq!(text.matches("log file reached").count(), 1, "{text}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_logger_filters_records_and_writes_them_to_the_attached_file() {
|
||||
use log::Log;
|
||||
let dir = temp_dir("logger");
|
||||
let logger = AppLogger::new(Filter::parse(DEFAULT_FILTER), false);
|
||||
record(&logger, log::Level::Info, "lightcraft", "before the settings directory");
|
||||
record(&logger, log::Level::Info, "wgpu_core::device", "too chatty");
|
||||
let path = logger.attach_dir(&dir).expect("attach");
|
||||
assert_eq!(path, dir.join(LOG_FILE));
|
||||
record(&logger, log::Level::Warn, "wgpu_hal::vulkan", "a real warning");
|
||||
record(&logger, log::Level::Error, "lightcraft_engine::guard", "`develop.reset` failed unexpectedly: boom");
|
||||
record(&logger, log::Level::Debug, "lightcraft", "below info");
|
||||
let text = read(&path);
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
assert_eq!(lines.len(), 3, "{text}");
|
||||
assert!(lines[0].contains(" INFO [") && lines[0].ends_with("lightcraft: before the settings directory"), "{text}");
|
||||
assert!(lines[1].ends_with("wgpu_hal::vulkan: a real warning"), "{text}");
|
||||
assert!(lines[2].contains(" ERROR [") && lines[2].ends_with("lightcraft_engine::guard: `develop.reset` failed unexpectedly: boom"), "{text}");
|
||||
assert!(logger.enabled(&log::Metadata::builder().level(log::Level::Info).target("lightcraft_ui_egui").build()));
|
||||
assert!(!logger.enabled(&log::Metadata::builder().level(log::Level::Info).target("naga").build()));
|
||||
// Attaching again rotates: the first log becomes `.1`.
|
||||
logger.attach_dir(&dir).expect("attach again");
|
||||
assert_eq!(read(&dir.join("lightcraft.1.log")), text);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn records_from_many_threads_are_whole_lines() {
|
||||
let dir = temp_dir("threads");
|
||||
let logger = std::sync::Arc::new(AppLogger::new(Filter::parse("info"), false));
|
||||
let path = logger.attach_dir(&dir).expect("attach");
|
||||
let workers: Vec<_> = (0..8)
|
||||
.map(|t| {
|
||||
let l = std::sync::Arc::clone(&logger);
|
||||
std::thread::spawn(move || {
|
||||
for i in 0..100 {
|
||||
record(&l, log::Level::Info, "lightcraft", &format!("thread {t} record {i}"));
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
for w in workers {
|
||||
w.join().expect("worker");
|
||||
}
|
||||
let text = read(&path);
|
||||
assert_eq!(text.lines().count(), 800);
|
||||
assert!(text.lines().all(|l| l.contains(" lightcraft: thread ")), "{text}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attaching_to_an_unusable_directory_reports_an_error_and_keeps_logging() {
|
||||
let dir = temp_dir("unusable");
|
||||
std::fs::create_dir_all(dir.parent().expect("parent")).expect("tmp");
|
||||
std::fs::write(&dir, "not a directory").expect("block");
|
||||
let logger = AppLogger::new(Filter::parse("info"), false);
|
||||
record(&logger, log::Level::Info, "lightcraft", "early");
|
||||
assert!(logger.attach_dir(&dir).is_err());
|
||||
record(&logger, log::Level::Error, "x", "still fine");
|
||||
// Nothing waits in memory for a file that won't come.
|
||||
assert!(logger.sink.lock().expect("sink").pending.is_empty());
|
||||
let _ = std::fs::remove_file(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_poisoned_sink_still_logs() {
|
||||
let dir = temp_dir("poison");
|
||||
let logger = std::sync::Arc::new(AppLogger::new(Filter::parse("info"), false));
|
||||
let l = std::sync::Arc::clone(&logger);
|
||||
let _ = std::thread::spawn(move || {
|
||||
let _guard = l.sink.lock().expect("lock");
|
||||
panic!("poison the sink");
|
||||
})
|
||||
.join();
|
||||
assert!(logger.sink.is_poisoned());
|
||||
let path = logger.attach_dir(&dir).expect("attach");
|
||||
record(&logger, log::Level::Warn, "lightcraft", "after the poison");
|
||||
assert!(read(&path).contains("lightcraft: after the poison"));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// The panic hook's report lands in the file, and never deadlocks when the panic happened
|
||||
/// while this thread held the sink.
|
||||
#[test]
|
||||
fn a_panic_report_goes_to_the_file_and_never_deadlocks() {
|
||||
let dir = temp_dir("panic");
|
||||
let logger = AppLogger::new(Filter::parse("off"), false);
|
||||
let path = logger.attach_dir(&dir).expect("attach");
|
||||
logger.record_panic("render", "panicked at crates/x/src/lib.rs:3:5:\nboom");
|
||||
{
|
||||
let _held = logger.sink.lock().expect("lock");
|
||||
logger.record_panic("main", "while the sink is held");
|
||||
}
|
||||
let text = read(&path);
|
||||
assert!(text.contains(" ERROR [render] lightcraft: panicked at crates/x/src/lib.rs:3:5:\nboom\n"), "{text}");
|
||||
assert!(!text.contains("while the sink is held"), "{text}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
+777
-76
File diff suppressed because it is too large.
Load diff
@@ -2,9 +2,12 @@
|
||||
//! the in-window menus ([`lightcraft_ui_egui::menubar`]): the registry's menu paths, shortcuts,
|
||||
//! enabled and checked state, and live labels ("Undo Exposure").
|
||||
//!
|
||||
//! Key handling: a native key equivalent consumes the key press before egui sees it, so
|
||||
//! - shortcuts installed natively are listed in `LightcraftApp::native_shortcuts` and skipped by
|
||||
//! the egui shortcut handler (nothing fires twice);
|
||||
//! Key handling: AppKit offers a key press to the menu bar only when it carries ⌘ or ⌃, or is a
|
||||
//! function key (F1…). Every other key (`E`, `1`, `⇧P`, `⌥Y`…) goes straight to the window, whose
|
||||
//! winit view always takes it, so a menu item never fires from it. Hence
|
||||
//! - shortcuts the menu bar really receives ([`menu_delivers`]) are listed in
|
||||
//! `LightcraftApp::native_shortcuts` and skipped by the egui shortcut handler (nothing fires
|
||||
//! twice); the others stay on their menu items for display and egui runs them;
|
||||
//! - while a text field has keyboard focus, accelerators without ⌘ (`G`, `1`, `Delete`…) and the
|
||||
//! text-editing ones (⌘A/⌘C/⌘V/⌘X/⌘Z) are removed, so typing and text editing work; they come
|
||||
//! back when the field loses focus.
|
||||
@@ -51,6 +54,9 @@ pub struct NativeMenu {
|
||||
/// Structure of the installed menu (ids, kinds, submenu names); a change rebuilds it.
|
||||
structure: String,
|
||||
text_focus: bool,
|
||||
/// The keyboard shortcuts editor is recording a key: every accelerator is off so the key
|
||||
/// press reaches it.
|
||||
capturing: bool,
|
||||
}
|
||||
|
||||
/// `Cmd+Shift+Z` → a muda accelerator (`None` for keys we leave to egui, e.g. Escape).
|
||||
@@ -146,9 +152,21 @@ fn yields_to_text(sc: &str) -> bool {
|
||||
!sc.contains("Cmd") && !sc.contains("Ctrl") || TEXT_EDIT.contains(&sc)
|
||||
}
|
||||
|
||||
/// Whether AppKit hands this shortcut to the menu bar: only key presses with ⌘ or ⌃, and function
|
||||
/// keys. A plain `C` or `⇧P` never reaches it (the window's view takes the key), so egui has to run
|
||||
/// those even though their menu item shows the key.
|
||||
fn menu_delivers(sc: &str) -> bool {
|
||||
sc.split('+').any(|part| matches!(part, "Cmd" | "Ctrl") || part.strip_prefix('F').is_some_and(|n| n.parse::<u8>().is_ok()))
|
||||
}
|
||||
|
||||
/// The shortcuts the menu bar runs itself, for `LightcraftApp::native_shortcuts`.
|
||||
fn owned_by_menu<'a>(installed: impl Iterator<Item = &'a str>, text_focus: bool) -> HashSet<String> {
|
||||
installed.filter(|sc| menu_delivers(sc) && !(text_focus && yields_to_text(sc))).map(str::to_string).collect()
|
||||
}
|
||||
|
||||
/// `&` marks a mnemonic in muda labels.
|
||||
fn label_text(s: &str) -> String {
|
||||
s.replace('&', "&&")
|
||||
lightcraft_ui_egui::i18n::tr(s).replace('&', "&&")
|
||||
}
|
||||
|
||||
fn structure_of(bar: &[(String, Vec<MenuNode>)]) -> String {
|
||||
@@ -171,7 +189,7 @@ fn structure_of(bar: &[(String, Vec<MenuNode>)]) -> String {
|
||||
out.push(';');
|
||||
}
|
||||
}
|
||||
let mut s = String::new();
|
||||
let mut s = format!("{};", lightcraft_ui_egui::i18n::language().code());
|
||||
for (t, items) in bar {
|
||||
s.push_str(t);
|
||||
walk(items, &mut s);
|
||||
@@ -196,7 +214,7 @@ impl NativeMenu {
|
||||
let _ = tx.send(e.id.0);
|
||||
repaint.request_repaint();
|
||||
}));
|
||||
let mut m = NativeMenu { menu: Menu::new(), items: HashMap::new(), rx, structure: String::new(), text_focus: false };
|
||||
let mut m = NativeMenu { menu: Menu::new(), items: HashMap::new(), rx, structure: String::new(), text_focus: false, capturing: false };
|
||||
m.rebuild(app);
|
||||
app.native_menu = true;
|
||||
m
|
||||
@@ -210,30 +228,30 @@ impl NativeMenu {
|
||||
|
||||
// the application menu
|
||||
let app_menu = Submenu::new("LightCraft", true);
|
||||
let about = MenuItem::with_id("app.about", "About LightCraft", true, None);
|
||||
let settings = MenuItem::with_id(SETTINGS, app.ui.language.tr("Settings…"), true, accelerator(SETTINGS_KEY));
|
||||
let quit = MenuItem::with_id(QUIT, "Quit LightCraft", true, accelerator("Cmd+Q"));
|
||||
let about = MenuItem::with_id("app.about", lightcraft_ui_egui::i18n::tr("About LightCraft"), true, None);
|
||||
let settings = MenuItem::with_id(SETTINGS, lightcraft_ui_egui::i18n::tr("Settings…"), true, accelerator(SETTINGS_KEY));
|
||||
let quit = MenuItem::with_id(QUIT, lightcraft_ui_egui::i18n::tr("Quit LightCraft"), true, accelerator("Cmd+Q"));
|
||||
let _ = app_menu.append_items(&[
|
||||
&about,
|
||||
&PredefinedMenuItem::separator(),
|
||||
&settings,
|
||||
&PredefinedMenuItem::separator(),
|
||||
&PredefinedMenuItem::services(None),
|
||||
&PredefinedMenuItem::services(Some(lightcraft_ui_egui::i18n::tr("Services"))),
|
||||
&PredefinedMenuItem::separator(),
|
||||
&PredefinedMenuItem::hide(Some("Hide LightCraft")),
|
||||
&PredefinedMenuItem::hide_others(None),
|
||||
&PredefinedMenuItem::show_all(None),
|
||||
&PredefinedMenuItem::hide(Some(lightcraft_ui_egui::i18n::tr("Hide LightCraft"))),
|
||||
&PredefinedMenuItem::hide_others(Some(lightcraft_ui_egui::i18n::tr("Hide Others"))),
|
||||
&PredefinedMenuItem::show_all(Some(lightcraft_ui_egui::i18n::tr("Show All"))),
|
||||
&PredefinedMenuItem::separator(),
|
||||
&quit,
|
||||
]);
|
||||
let _ = self.menu.append(&app_menu);
|
||||
|
||||
for (title, nodes) in &bar {
|
||||
let sub = Submenu::new(title, true);
|
||||
if title == app.ui.language.tr("Window") {
|
||||
let sub = Submenu::new(label_text(title), true);
|
||||
if title == "Window" {
|
||||
let _ = sub.append_items(&[
|
||||
&PredefinedMenuItem::minimize(None),
|
||||
&PredefinedMenuItem::maximize(Some("Zoom")),
|
||||
&PredefinedMenuItem::minimize(Some(lightcraft_ui_egui::i18n::tr("Minimize"))),
|
||||
&PredefinedMenuItem::maximize(Some(lightcraft_ui_egui::i18n::tr("Zoom"))),
|
||||
&PredefinedMenuItem::separator(),
|
||||
]);
|
||||
}
|
||||
@@ -245,21 +263,28 @@ impl NativeMenu {
|
||||
.collect();
|
||||
let nodes = tidy_separators(nodes);
|
||||
self.append_nodes(&sub, &nodes);
|
||||
if title == app.ui.language.tr("File") {
|
||||
if title == "File" {
|
||||
// ⌘W, the system's own item
|
||||
let _ = sub.append_items(&[&PredefinedMenuItem::separator(), &PredefinedMenuItem::close_window(None)]);
|
||||
let _ = sub.append_items(&[
|
||||
&PredefinedMenuItem::separator(),
|
||||
&PredefinedMenuItem::close_window(Some(lightcraft_ui_egui::i18n::tr("Close Window"))),
|
||||
]);
|
||||
}
|
||||
if title == app.ui.language.tr("Window") {
|
||||
let _ = sub.append_items(&[&PredefinedMenuItem::separator(), &PredefinedMenuItem::bring_all_to_front(None)]);
|
||||
if title == "Window" {
|
||||
let _ = sub.append_items(&[
|
||||
&PredefinedMenuItem::separator(),
|
||||
&PredefinedMenuItem::bring_all_to_front(Some(lightcraft_ui_egui::i18n::tr("Bring All to Front"))),
|
||||
]);
|
||||
sub.set_as_windows_menu_for_nsapp();
|
||||
}
|
||||
if title == app.ui.language.tr("Help") {
|
||||
if title == "Help" {
|
||||
sub.set_as_help_menu_for_nsapp();
|
||||
}
|
||||
let _ = self.menu.append(&sub);
|
||||
}
|
||||
self.menu.init_for_nsapp();
|
||||
self.text_focus = false;
|
||||
self.capturing = false;
|
||||
self.publish_shortcuts(app);
|
||||
}
|
||||
|
||||
@@ -282,12 +307,23 @@ impl NativeMenu {
|
||||
let accel = shortcut.as_deref().filter(|s| !CONTEXTUAL.contains(s)).and_then(accelerator);
|
||||
let handle = match checked {
|
||||
Some(c) => {
|
||||
let it = CheckMenuItem::with_id(key.clone(), label_text(label), *enabled, *c, accel);
|
||||
let it = CheckMenuItem::with_id(
|
||||
key.clone(),
|
||||
lightcraft_ui_egui::menubar::display_item_label(id, params, label).replace('&', "&&"),
|
||||
*enabled,
|
||||
*c,
|
||||
accel,
|
||||
);
|
||||
let _ = sub.append(&it);
|
||||
Handle::Check(it)
|
||||
}
|
||||
None => {
|
||||
let it = MenuItem::with_id(key.clone(), label_text(label), *enabled, accel);
|
||||
let it = MenuItem::with_id(
|
||||
key.clone(),
|
||||
lightcraft_ui_egui::menubar::display_item_label(id, params, label).replace('&', "&&"),
|
||||
*enabled,
|
||||
accel,
|
||||
);
|
||||
let _ = sub.append(&it);
|
||||
Handle::Plain(it)
|
||||
}
|
||||
@@ -312,22 +348,22 @@ impl NativeMenu {
|
||||
|
||||
/// Tell the egui shortcut handler which shortcuts the menu bar currently owns.
|
||||
fn publish_shortcuts(&self, app: &mut LightcraftApp) {
|
||||
app.native_shortcuts = self
|
||||
.items
|
||||
.values()
|
||||
.filter(|i| i.accel.is_some())
|
||||
.filter_map(|i| i.shortcut.clone())
|
||||
.filter(|sc| !(self.text_focus && yields_to_text(sc)))
|
||||
.collect::<HashSet<_>>();
|
||||
let installed = self.items.values().filter(|i| i.accel.is_some()).filter_map(|i| i.shortcut.as_deref());
|
||||
// (none while the keymap editor records a shortcut: every key goes to it)
|
||||
app.native_shortcuts = if self.capturing { HashSet::new() } else { owned_by_menu(installed, self.text_focus) };
|
||||
app.native_shortcuts.insert(SETTINGS_KEY.to_string());
|
||||
}
|
||||
|
||||
/// Per frame: run chosen items, then sync labels / enabled / checked and the text-focus
|
||||
/// accelerators with the app state.
|
||||
pub fn update(&mut self, app: &mut LightcraftApp, ctx: &egui::Context) {
|
||||
/// accelerators with the app state. Returns whether Quit was chosen: the caller saves
|
||||
/// everything and exits the process itself (closing the viewport while AppKit terminates the
|
||||
/// app can crash in winit's teardown, PR #444).
|
||||
pub fn update(&mut self, app: &mut LightcraftApp, ctx: &egui::Context) -> bool {
|
||||
lightcraft_ui_egui::i18n::set_language(app.ui.language);
|
||||
let mut quit = false;
|
||||
while let Ok(key) = self.rx.try_recv() {
|
||||
if key == QUIT {
|
||||
ctx.send_viewport_cmd(egui::ViewportCommand::Close);
|
||||
quit = true;
|
||||
continue;
|
||||
}
|
||||
if key == "app.about" || key == SETTINGS {
|
||||
@@ -342,7 +378,7 @@ impl NativeMenu {
|
||||
let bar = menu_bar(app);
|
||||
if structure_of(&bar) != self.structure {
|
||||
self.rebuild(app);
|
||||
return;
|
||||
return quit;
|
||||
}
|
||||
fn walk(nodes: &[MenuNode], items: &mut HashMap<String, Item>) {
|
||||
for n in nodes {
|
||||
@@ -352,8 +388,8 @@ impl NativeMenu {
|
||||
let Some(it) = items.get_mut(&MenuNode::key(id, params)) else { continue };
|
||||
if it.label != *label {
|
||||
match &it.handle {
|
||||
Handle::Plain(h) => h.set_text(label_text(label)),
|
||||
Handle::Check(h) => h.set_text(label_text(label)),
|
||||
Handle::Plain(h) => h.set_text(lightcraft_ui_egui::menubar::display_item_label(id, params, label).replace('&', "&&")),
|
||||
Handle::Check(h) => h.set_text(lightcraft_ui_egui::menubar::display_item_label(id, params, label).replace('&', "&&")),
|
||||
}
|
||||
it.label = label.clone();
|
||||
}
|
||||
@@ -379,14 +415,16 @@ impl NativeMenu {
|
||||
walk(&bar.iter().flat_map(|(_, v)| v.clone()).collect::<Vec<_>>(), &mut self.items);
|
||||
|
||||
let focus = ctx.egui_wants_keyboard_input();
|
||||
if focus != self.text_focus {
|
||||
let capturing = app.recording_shortcut.is_some();
|
||||
if focus != self.text_focus || capturing != self.capturing {
|
||||
self.text_focus = focus;
|
||||
self.capturing = capturing;
|
||||
for it in self.items.values() {
|
||||
let Some(sc) = it.shortcut.as_deref() else { continue };
|
||||
if it.accel.is_none() || !yields_to_text(sc) {
|
||||
if it.accel.is_none() {
|
||||
continue;
|
||||
}
|
||||
let accel = if focus { None } else { it.accel };
|
||||
let accel = if capturing || (focus && yields_to_text(sc)) { None } else { it.accel };
|
||||
let _ = match &it.handle {
|
||||
Handle::Plain(h) => h.set_accelerator(accel),
|
||||
Handle::Check(h) => h.set_accelerator(accel),
|
||||
@@ -394,6 +432,7 @@ impl NativeMenu {
|
||||
}
|
||||
self.publish_shortcuts(app);
|
||||
}
|
||||
quit
|
||||
}
|
||||
}
|
||||
|
||||
@@ -401,6 +440,27 @@ impl NativeMenu {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The menu model's titles stay English, so only the language can tell the native menu to
|
||||
/// rebuild its titles (and the macOS app menu): every language switch — also between two CJK
|
||||
/// languages — must change the structure key.
|
||||
#[test]
|
||||
fn switching_language_rebuilds_native_menu_structure() {
|
||||
use lightcraft_ui_egui::i18n::{Locale, set_language};
|
||||
let app = LightcraftApp::new(lightcraft_engine::Session::with_demo(), Default::default());
|
||||
set_language(Locale::En);
|
||||
let bar = menu_bar(&app);
|
||||
let mut seen = std::collections::BTreeSet::new();
|
||||
for language in Locale::ALL {
|
||||
set_language(*language);
|
||||
assert!(seen.insert(structure_of(&bar)), "{language:?} shares another language's menu structure key");
|
||||
}
|
||||
set_language(Locale::ZhHant);
|
||||
assert_eq!(label_text("File"), "檔案");
|
||||
assert_eq!(label_text("Settings…"), "設定…");
|
||||
assert_eq!(label_text("Quit LightCraft"), "結束 LightCraft");
|
||||
set_language(Locale::En);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shortcuts_map_to_accelerators() {
|
||||
assert_eq!(accelerator("Cmd+Shift+Z"), Some(Accelerator::new(Modifiers::META | Modifiers::SHIFT, Code::KeyZ)));
|
||||
@@ -428,4 +488,39 @@ mod tests {
|
||||
assert!(accelerator(sc).is_some(), "{sc}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Single keys shown in the menu bar (E, C, ⇧P, ratings…) never reach it on macOS, so egui must
|
||||
/// keep handling them; ⌘ / ⌃ combinations and function keys are the menu bar's own.
|
||||
#[test]
|
||||
fn single_key_shortcuts_stay_with_egui() {
|
||||
assert!(menu_delivers("Cmd+Shift+H") && menu_delivers("Ctrl+H") && menu_delivers("F2") && menu_delivers("Cmd+F11"));
|
||||
assert!(!menu_delivers("E") && !menu_delivers("Shift+P") && !menu_delivers("Alt+Y") && !menu_delivers("1") && !menu_delivers("F"));
|
||||
let app = LightcraftApp::new(lightcraft_engine::Session::with_demo(), Default::default());
|
||||
fn all(n: &[MenuNode], out: &mut Vec<String>) {
|
||||
for x in n {
|
||||
match x {
|
||||
MenuNode::Item { shortcut: Some(s), .. } if accelerator(s).is_some() => out.push(s.clone()),
|
||||
MenuNode::Submenu { children, .. } => all(children, out),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut installed = Vec::new();
|
||||
for (_, v) in menu_bar(&app) {
|
||||
all(&v, &mut installed);
|
||||
}
|
||||
for sc in ["E", "C", "Shift+P", "Cmd+Shift+H", "F2"] {
|
||||
assert!(installed.iter().any(|s| s == sc), "{sc} is in the menu bar");
|
||||
}
|
||||
let owned = owned_by_menu(installed.iter().map(String::as_str), false);
|
||||
for sc in ["E", "C", "Shift+P", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "P", "U"] {
|
||||
assert!(installed.iter().any(|s| s == sc), "{sc} is displayed in the menu");
|
||||
assert!(!owned.contains(sc), "{sc} must be left to egui");
|
||||
}
|
||||
for sc in ["Cmd+Shift+H", "F2"] {
|
||||
assert!(owned.contains(sc), "{sc} is run by the menu bar");
|
||||
}
|
||||
// while typing, the menu bar gives up F2 too (the text field has it)
|
||||
assert!(!owned_by_menu(installed.iter().map(String::as_str), true).contains("F2"));
|
||||
}
|
||||
}
|
||||
+11
-2
@@ -9,9 +9,12 @@ covers a whole directory), or if a licence file referenced in the Licence column
|
||||
| Path | Asset | Creator | Source | Licence | Added | Modifications |
|
||||
|---|---|---|---|---|---|---|
|
||||
| `assets/fonts/Inter-*.ttf` (Regular, Medium, SemiBold) | Inter | Rasmus Andersson | https://github.com/rsms/inter | SIL Open Font License 1.1 (`assets/fonts/OFL-Inter.txt`) | 2026-09-30 | none |
|
||||
| `assets/fonts/BIZUDMincho-Regular.ttf` | BIZ UDMincho Regular | The BIZ UDMincho Project Authors (Morisawa Inc., design TypeBank Co., Ltd.) | https://github.com/google/fonts/tree/63833b7d10bb1f86a8f0b91cba2d3ae1f68d1aa3/ofl/bizudmincho | OFL-1.1 (`assets/fonts/OFL-BIZUDMincho.txt`) | 2026-10-05 | none |
|
||||
| `assets/fonts/OFL-BIZUDMincho.txt` | BIZ UDMincho licence | The BIZ UDMincho Project Authors | https://github.com/google/fonts/blob/63833b7d10bb1f86a8f0b91cba2d3ae1f68d1aa3/ofl/bizudmincho/OFL.txt | OFL-1.1 | 2026-10-05 | none |
|
||||
| `assets/app-icon/` (`lightcraft.svg` and the PNG/ICNS/ICO/SVG files derived from it) | LightCraft app icon (lynx) | LightCraft project owner (drawn in ArtCraft) | original work (vectorised from the owner's drawing; see `assets/app-icon/README.md`) | MIT OR Apache-2.0 (`assets/app-icon/LICENSE.txt`) | 2026-10-02 | vectorised to SVG; rendered to PNG/ICNS/ICO by `packaging/icons.sh` |
|
||||
| `assets/camera-profiles/*.json` | Camera colour profiles (matrix + hue/saturation/value table), built into LightCraft | ILCE-7M4: Harald Hoyer | original work: fitted by `lightcraft-cli calibrate` to the contributor's own raw photos and their in-camera JPEGs (597 ILCE-7M4 photos, 2026); aggregate colour statistics only, no image content; no Adobe or third-party profile data | MIT OR Apache-2.0 | 2026-10-07 | none |
|
||||
| `packaging/macos/dmg/` (`background.svg`, the `background.tiff` rendered from it by `generate.py`, and the `dmg-layout.DS_Store` layout file) | LightCraft macOS DMG window background | @XusBadia | original work for this repository, derived from the LightCraft app icon (`assets/app-icon/lightcraft.svg`, referenced via `<image>`, not copied); text is outlined Inter / JetBrains Mono glyph paths | MIT OR Apache-2.0 | 2026-10-08 | none (original); used for the Finder window of the macOS DMG, `packaging/macos/dmg/generate.py` renders it to `background.tiff` |
|
||||
| `assets/camera-profiles/ILCE-7CR.json` | Sony ILCE-7CR camera colour profile (matrix + hue/saturation/value table), built into LightCraft | LightCraft contributors; source photographs supplied by @AlexChambers | original work: fitted by `lightcraft-cli calibrate` to 125 usable ILCE-7CR ARWs (of 131, September 2025 to September 2026, Standard creative style) and their in-camera JPEGs; aggregate colour statistics only, no image content; no Adobe or third-party profile data | MIT OR Apache-2.0 | 2026-10-09 | none |
|
||||
| `assets/camera-profiles/X-H2S.json`, `assets/camera-profiles/X-T4.json` | Fujifilm camera colour profiles (matrix + hue/saturation/value table), built into LightCraft | LightCraft contributors; source photographs supplied by Sebastian Pfluegelmeier | original work: fitted by `lightcraft-cli calibrate` to 201 X-H2S and 545 X-T4 RAFs and their in-camera JPEGs; aggregate colour statistics only, no image content; no Adobe or third-party profile data | MIT OR Apache-2.0 | 2026-10-08 | none |
|
||||
| `assets/camera-tone/sony-dro-off.json` | Sony global tone curve with the Dynamic Range Optimizer off (32 knots), built into LightCraft for ARWs shot with DRO on | LightCraft contributors; source photographs by raw.pixls.us contributors | original work: tone curve quantile-matched over the pooled pixel pairs of 12 CC0 raw.pixls.us ARWs (DRO off, Standard creative style; ILCE-3500, -5000, -6600, -7CR, -7M3 ×3, -7S, -9M2, ILCA-77M2, SLT-A57, NEX-F3) and their embedded camera JPEGs; the file lists each source URL and SHA-256 and how the curve was derived (see also `docs/camera-preview-colour.md`); aggregate tone statistics only, no image content; no Adobe or third-party profile data | MIT OR Apache-2.0 (sources CC0) | 2026-10-09 | none |
|
||||
| `crates/scenes/` (generated images) | Procedural demo photographs | LightCraft contributors | original work (generated at runtime, no source imagery) | MIT OR Apache-2.0 | 2026-09-30 | n/a |
|
||||
| `crates/ui-egui/src/icons.rs` | UI icons drawn as vector paths in code | LightCraft contributors | original work | MIT OR Apache-2.0 | 2026-09-30 | n/a |
|
||||
| `docs/images/*-tetons.jpg`, `docs/images/grid-pd.jpg` (screenshots containing it) | "The Tetons and the Snake River" | Ansel Adams (1942), U.S. National Archives | https://commons.wikimedia.org/wiki/File:Adams_The_Tetons_and_the_Snake_River.jpg | Public domain (U.S. federal government work) | 2026-09-30 | developed in LightCraft; shown inside app screenshots |
|
||||
@@ -20,7 +23,13 @@ covers a whole directory), or if a licence file referenced in the Licence column
|
||||
| `docs/images/*-blue-marble.jpg`, `docs/images/grid-pd.jpg` | "The Blue Marble" (AS17-148-22727) | NASA, Apollo 17 crew (1972) | https://commons.wikimedia.org/wiki/File:The_Earth_seen_from_Apollo_17.jpg | Public domain (NASA) | 2026-09-30 | developed in LightCraft; shown inside app screenshots |
|
||||
| `docs/images/*.jpg` (UI) | LightCraft application screenshots | LightCraft contributors | original work (captured with `docs/showcase/`) | MIT OR Apache-2.0 | 2026-09-30 | n/a |
|
||||
| `docs/brand/` (artcraft-logo and artcraft-mark, SVG and PNG) | ArtCraft wordmark and mark | ArtCraft Team | original work (https://getartcraft.com/) | ArtCraft trademark, see `docs/brand/LICENSE-brand.txt` (not open source) | 2026-10-01 | none |
|
||||
| craft-fonts: optional build input, not files in this repository | BIZ UDPGothic (Regular, Bold), BIZ UDMincho Regular, Shippori Mincho Regular, Noto Sans CJK SC Regular | Morisawa Inc. / The BIZ UDGothic and BIZ UDMincho Project Authors; The Shippori Mincho Project Authors; Google and Adobe (Noto Sans CJK, maintainer-approved exception; see AGENTS.md) (per craft-fonts' attribution) | https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md | SIL Open Font License 1.1 (each font's OFL.txt in craft-fonts; release packages ship it as OFL-<family>.txt) | 2026-10-06 | none; native builds made with `CRAFT_FONTS_DIR` embed the manifest fonts (all official releases); the web build embeds BIZ UDPGothic Regular and ships Noto Sans CJK SC Regular as a separate font file |
|
||||
| `crates/ui-egui/locales/uk.json`, `crates/ui-egui/locales/uk-formats.json` | Ukrainian interface catalogs | @dmatviichuk with Codex; PhotoCraft, FilmCraft and VectorCraft translation contributors for reused labels | original translations from English source; 337 initial English-key matches reviewed from [PhotoCraft 69c41f3](https://github.com/storytold/photocraft/blob/69c41f3523e147eb215da8c7b4c29b18b35d1c63/crates/ui-egui/src/i18n/uk.tsv), [FilmCraft 4854c27](https://github.com/storytold/filmcraft/blob/4854c277536fe8118257362304b6a7d82d8e704b/crates/ui-egui/src/i18n/uk.tsv), [VectorCraft 4b44265](https://github.com/storytold/vectorcraft/blob/4b44265965fdd6ffedfd6182a892c1a6c360d344/crates/ui-egui/src/i18n/uk.tsv) | MIT OR Apache-2.0 (`LICENSE-MIT`, `LICENSE-APACHE`, also the source repositories' licences) | 2026-10-09 | adapted to LightCraft photo-development context, completed catalogs, reviewed spelling and placeholders |
|
||||
|
||||
Notes
|
||||
- Fonts authored or published by Adobe (Source Sans/Serif/Code, Source Han, …) are not used, even though some are
|
||||
OFL-licensed: the asset rule excludes anything from Adobe. Source Sans 3 was removed on 2026-09-30 and replaced by Inter.
|
||||
- CJK fonts (BIZ UDPGothic, BIZ UDMincho, Shippori Mincho, Noto Sans CJK SC) are not in this repository. They live in
|
||||
[storytold/craft-fonts](https://github.com/storytold/craft-fonts), with their licences and attribution in its
|
||||
`ATTRIBUTION.md`, and are embedded only in builds made with the optional `CRAFT_FONTS_DIR` build input (all official
|
||||
releases). Never commit font files here; add new fonts to craft-fonts (craftrules `standards/fonts.md`).
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1 @@
|
||||
{"version":1,"model":"ILCE-7M4","files":597,"samples":2182876,"matrix":[[1.06975,-0.0709886,-0.000770444],[-0.0169822,1.08012,-0.0651276],[-0.114835,-0.601525,1.74804]],"hue_sat":{"hue_divisions":72,"sat_divisions":5,"val_divisions":5,"data":[[0.0,1.0,1.0],[-0.819008,1.14375,1.0],[-6.07446,1.10974,1.0],[-4.0478,1.04285,1.0],[-0.546764,1.00329,1.0],[0.0,1.0,1.0],[-1.0412,1.10597,1.0],[-7.22645,1.05878,1.0],[-7.30635,0.989387,1.0],[-5.18589,0.974522,1.0],[0.0,1.0,1.0],[-1.02364,1.06303,1.0],[-6.94669,1.01579,1.0],[-7.47614,0.955435,1.0],[-7.02579,0.940854,1.0],[0.0,1.0,1.0],[-0.29852,1.04349,1.0],[-6.37202,1.00021,1.0],[-7.52098,0.935457,1.0],[-7.45702,0.922239,1.0],[0.0,1.0,1.0],[0.367464,1.03266,1.0],[-4.96951,0.996822,1.0],[-7.41706,0.948709,1.0],[-8.04735,0.91672,1.0],[0.0,1.0,1.0],[0.0388419,1.02587,1.0],[-4.47266,1.01282,1.0],[-7.26834,0.987279,1.0],[-9.02713,0.921453,1.0],[0.0,1.0,1.0],[-0.297476,1.02333,1.0],[-4.6323,1.00993,1.0],[-8.57248,0.962973,1.0],[-12.2375,0.872088,1.0],[0.0,1.0,1.0],[-0.785867,1.02786,1.0],[-4.73124,1.00876,1.0],[-9.09856,0.955343,1.0],[-12.4324,0.875472,1.0],[0.0,1.0,1.0],[-0.444072,1.03863,1.0],[-3.94773,1.01784,1.0],[-7.97996,0.975996,1.0],[-10.673,0.917401,1.0],[0.0,1.0,1.0],[1.22651,1.0559,1.0],[-1.89459,1.02934,1.0],[-5.54545,0.993046,1.0],[-8.48877,0.941628,1.0],[0.0,1.0,1.0],[3.24516,1.08292,1.0],[1.20522,1.0577,1.0],[-0.826224,1.01668,1.0],[-3.71884,0.946733,1.0],[0.0,1.0,1.0],[4.68802,1.12039,1.0],[3.23147,1.08619,1.0],[1.84476,1.04,1.0],[0.0970375,0.962298,1.0],[0.0,1.0,1.0],[5.36041,1.16313,1.0],[4.5937,1.10933,1.0],[3.83314,1.05056,1.0],[3.122,0.959876,1.0],[0.0,1.0,1.0],[5.22222,1.17824,1.0],[5.09129,1.1142,1.0],[4.99294,1.04813,1.0],[5.01485,0.957528,1.0],[0.0,1.0,1.0],[4.34382,1.18446,1.0],[4.9457,1.11351,1.0],[5.56715,1.04462,1.0],[6.27579,0.961892,1.0],[0.0,1.0,1.0],[3.5428,1.18985,1.0],[4.9116,1.11394,1.0],[6.11507,1.04098,1.0],[7.03638,0.964331,1.0],[0.0,1.0,1.0],[2.92641,1.20663,1.0],[4.57132,1.1281,1.0],[5.96964,1.04895,1.0],[6.98579,0.96261,1.0],[0.0,1.0,1.0],[2.07123,1.23257,1.0],[3.51736,1.15454,1.0],[4.85051,1.06995,1.0],[5.97805,0.971597,1.0],[0.0,1.0,1.0],[0.869122,1.26762,1.0],[2.22009,1.19021,1.0],[3.54297,1.10536,1.0],[4.31783,1.00254,1.0],[0.0,1.0,1.0],[-0.409355,1.31042,1.0],[0.778039,1.23845,1.0],[1.87845,1.15455,1.0],[1.53759,1.02961,1.0],[0.0,1.0,1.0],[-1.82067,1.35265,1.0],[-0.910751,1.28551,1.0],[-0.100378,1.18585,1.0],[0.0425375,1.01463,1.0],[0.0,1.0,1.0],[-3.26659,1.391,1.0],[-2.50117,1.33263,1.0],[-1.55789,1.18102,1.0],[-0.0853484,1.00594,1.0],[0.0,1.0,1.0],[-4.92446,1.43632,1.0],[-3.92066,1.35798,1.0],[-1.66274,1.11605,1.0],[-0.0358972,1.00178,1.0],[0.0,1.0,1.0],[-5.82474,1.46137,1.0],[-4.22466,1.34716,1.0],[-0.64525,1.04813,1.0],[-0.00357121,1.00023,1.0],[0.0,1.0,1.0],[-7.0672,1.47242,1.0],[-5.38667,1.32663,1.0],[-0.553542,1.02668,1.0],[-0.00188478,1.00008,1.0],[0.0,1.0,1.0],[-8.37522,1.43136,1.0],[-6.13631,1.28174,1.0],[-0.515332,1.01808,1.0],[-0.0015762,1.00005,1.0],[0.0,1.0,1.0],[-10.5196,1.39935,1.0],[-7.49316,1.24442,1.0],[-0.517621,1.01282,1.0],[-0.001445,1.00003,1.0],[0.0,1.0,1.0],[-10.5952,1.3534,1.0],[-7.81548,1.20414,1.0],[-0.63473,1.0111,1.0],[-0.00208267,1.00003,1.0],[0.0,1.0,1.0],[-10.879,1.32051,1.0],[-8.3374,1.18517,1.0],[-0.91817,1.01139,1.0],[-0.00474972,1.00003,1.0],[0.0,1.0,1.0],[-11.8094,1.29525,1.0],[-9.18818,1.17522,1.0],[-0.968552,1.01157,1.0],[-0.00411555,1.00003,1.0],[0.0,1.0,1.0],[-12.2038,1.27777,1.0],[-8.97374,1.15856,1.0],[-0.751018,1.00872,1.0],[-0.00250012,1.00002,1.0],[0.0,1.0,1.0],[-12.0564,1.2709,1.0],[-8.38288,1.15926,1.0],[-0.643806,1.00878,1.0],[-0.00202184,1.00002,1.0],[0.0,1.0,1.0],[-12.8403,1.25618,1.0],[-8.85664,1.14517,1.0],[-0.646111,1.00756,1.0],[-0.00195373,1.00001,1.0],[0.0,1.0,1.0],[-13.2604,1.25094,1.0],[-8.74013,1.11989,1.0],[-0.556009,1.00457,1.0],[-0.00173404,1.00001,1.0],[0.0,1.0,1.0],[-13.3939,1.24388,1.0],[-8.23916,1.11659,1.0],[-0.376536,1.00355,1.0],[-0.000916476,1.00001,1.0],[0.0,1.0,1.0],[-13.6578,1.24212,1.0],[-7.97883,1.11035,1.0],[-0.409859,1.00287,1.0],[-0.0018203,1.0,1.0],[0.0,1.0,1.0],[-15.7225,1.22414,1.0],[-10.2345,1.10217,1.0],[-0.968177,1.00427,1.0],[-0.00650675,1.00001,1.0],[0.0,1.0,1.0],[-16.1557,1.16423,1.0],[-12.721,1.06603,1.0],[-2.02337,1.00334,1.0],[-0.0134173,1.00001,1.0],[0.0,1.0,1.0],[-17.253,1.09714,1.0],[-15.785,1.02184,1.0],[-4.02264,0.996649,1.0],[-0.0279292,0.999956,1.0],[0.0,1.0,1.0],[-17.6206,1.04331,1.0],[-16.2039,0.99401,1.0],[-4.09728,0.990246,1.0],[-0.0269801,0.999896,1.0],[0.0,1.0,1.0],[-17.2302,0.987569,1.0],[-15.1656,0.964114,1.0],[-3.21398,0.983725,1.0],[-0.0202065,0.999807,1.0],[0.0,1.0,1.0],[-16.2184,0.943678,1.0],[-13.4633,0.938141,1.0],[-3.40639,0.971571,1.0],[-0.00923182,0.999563,1.0],[0.0,1.0,1.0],[-14.5653,0.909699,1.0],[-11.6873,0.915733,1.0],[-5.4654,0.942337,1.0],[-0.124331,0.998194,1.0],[0.0,1.0,1.0],[-11.5122,0.906891,1.0],[-9.31547,0.923874,1.0],[-6.12222,0.939832,1.0],[-0.337538,0.995963,1.0],[0.0,1.0,1.0],[-7.77619,0.92294,1.0]Line truncated
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"description": "Sony's global tone curve with the Dynamic Range Optimizer (DRO) off, Standard creative style: knots of scene luminance (white-balanced camera RGB with baseline exposure, through the colour model) to display-linear luminance. The camera look of an ARW shot with DRO on is lowered to it wherever it is darker (crates/engine/src/camera_preview.rs, docs/camera-preview-colour.md, issue #244).",
|
||||
"derivation": "Each source raw was paired with its own embedded camera JPEG at the 96-pixel proxy of the per-photo fit (camera_preview::collect_pairs). The pairs of all 12 files were pooled as one model, as lightcraft-cli calibrate pools a camera profile: a colour matrix and hue/saturation table were fitted to the pool, and the tone curve was quantile-matched (camera_preview::fit_tone) between the pooled scene luminances through that colour model and the JPEG luminances. Knots rounded to 5 decimals. Aggregate statistics only, no image content.",
|
||||
"sources": [
|
||||
{ "body": "ILCA-77M2", "url": "https://raw.pixls.us/data/Sony/ILCA-77M2/_DSC0244.ARW", "sha256": "f41ceff680b675460cc1fe258a24d8c8c01c52a300933f3ab6401d9a20b60dcc", "licence": "CC0" },
|
||||
{ "body": "ILCE-3500", "url": "https://raw.pixls.us/data/Sony/ILCE-3500/DSC06923.ARW", "sha256": "04c4fe04425c3e6fff7c8af0f811923c35e750300373357bd54342150c5505d8", "licence": "CC0" },
|
||||
{ "body": "ILCE-5000", "url": "https://raw.pixls.us/data/Sony/ILCE-5000/_DSC0793.ARW", "sha256": "a919c8535eed68577f1745fb9674ccf333962628ccdec6d48a6f635208467f6c", "licence": "CC0" },
|
||||
{ "body": "ILCE-6600", "url": "https://raw.pixls.us/data/Sony/ILCE-6600/_DSC0268.ARW", "sha256": "041d71d70148ca77164f19d061e3582adb34399ea5a4baba32960f2956fb0362", "licence": "CC0" },
|
||||
{ "body": "ILCE-7CR", "url": "https://raw.pixls.us/data/Sony/ILCE-7CR/DSC00798.ARW", "sha256": "f5096e8fbccf0842c8a57763cabd2836608f019eb7fd54f7b22260c2f5bc4da5", "licence": "CC0" },
|
||||
{ "body": "ILCE-7M3", "url": "https://raw.pixls.us/data/Sony/ILCE-7M3/UNCOMPRESSED_14bit.ARW", "sha256": "ece80551abf64949dbe826985a80f1d9b265401ee4ea6b989e1829549616fdcd", "licence": "CC0" },
|
||||
{ "body": "ILCE-7M3", "url": "https://raw.pixls.us/data/Sony/ILCE-7M3/_DSC0009.ARW", "sha256": "250784580ea527442c09004417bb0eead484f2bf3ee8f9121a776ac65bb50d0f", "licence": "CC0" },
|
||||
{ "body": "ILCE-7M3", "url": "https://raw.pixls.us/data/Sony/ILCE-7M3/_DSC0010.ARW", "sha256": "3ae2ff75cfa7240d0b4458f662afd28665c5fd1e83d230ac73f5ad69d3df8592", "licence": "CC0" },
|
||||
{ "body": "ILCE-7S", "url": "https://raw.pixls.us/data/Sony/ILCE-7S/DSC04125.ARW", "sha256": "7cc338a0abc8fdad32d61006f1f8f412297b93e040c7efba572eb2bd8f8e8be2", "licence": "CC0" },
|
||||
{ "body": "ILCE-9M2", "url": "https://raw.pixls.us/data/Sony/ILCE-9M2/SONY_A9II_(ILCE-9M2)_-_compressed_(3:2)_14bit.ARW", "sha256": "161c2a9da2b5f1e50be6117a0b4da0ce249660b5d7de13568d301c4034716b97", "licence": "CC0" },
|
||||
{ "body": "NEX-F3", "url": "https://raw.pixls.us/data/Sony/NEX-F3/_DSC4017.ARW", "sha256": "cfdc60d118b5ce2dce10d459dcb98ad63c6db3e609999932ca8964d86daf836b", "licence": "CC0" },
|
||||
{ "body": "SLT-A57", "url": "https://raw.pixls.us/data/Sony/SLT-A57/a57-06125.arw", "sha256": "0e9064f369a3947f892c1d7b1971f75d3879f5673134932d5cc7a67bf4263ba4", "licence": "CC0" }
|
||||
],
|
||||
"knots": [
|
||||
[0.01655, 0.01799],
|
||||
[0.01970, 0.02376],
|
||||
[0.02212, 0.03008],
|
||||
[0.02469, 0.03634],
|
||||
[0.02737, 0.04320],
|
||||
[0.03049, 0.05044],
|
||||
[0.03425, 0.05812],
|
||||
[0.03824, 0.06667],
|
||||
[0.04232, 0.07817],
|
||||
[0.04658, 0.09059],
|
||||
[0.05095, 0.10268],
|
||||
[0.05550, 0.11731],
|
||||
[0.05962, 0.13173],
|
||||
[0.06409, 0.14529],
|
||||
[0.06843, 0.15939],
|
||||
[0.07314, 0.17363],
|
||||
[0.07800, 0.18897],
|
||||
[0.08298, 0.20502],
|
||||
[0.08909, 0.22159],
|
||||
[0.09693, 0.23964],
|
||||
[0.10540, 0.26082],
|
||||
[0.11579, 0.28882],
|
||||
[0.12763, 0.31986],
|
||||
[0.14079, 0.35198],
|
||||
[0.15786, 0.39293],
|
||||
[0.17949, 0.44101],
|
||||
[0.20330, 0.49407],
|
||||
[0.23466, 0.54244],
|
||||
[0.27207, 0.60078],
|
||||
[0.31673, 0.66533],
|
||||
[0.38682, 0.74143],
|
||||
[0.52622, 0.80307]
|
||||
]
|
||||
}
|
||||
Binary file not shown.
@@ -1,93 +0,0 @@
|
||||
Copyright 2022 The BIZ UDMincho Project Authors (https://github.com/googlefonts/morisawa-biz-ud-mincho)
|
||||
|
||||
This Font Software is licensed under the SIL Open Font License, Version 1.1.
|
||||
This license is copied below, and is also available with a FAQ at:
|
||||
https://scripts.sil.org/OFL
|
||||
|
||||
|
||||
-----------------------------------------------------------
|
||||
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
|
||||
-----------------------------------------------------------
|
||||
|
||||
PREAMBLE
|
||||
The goals of the Open Font License (OFL) are to stimulate worldwide
|
||||
development of collaborative font projects, to support the font creation
|
||||
efforts of academic and linguistic communities, and to provide a free and
|
||||
open framework in which fonts may be shared and improved in partnership
|
||||
with others.
|
||||
|
||||
The OFL allows the licensed fonts to be used, studied, modified and
|
||||
redistributed freely as long as they are not sold by themselves. The
|
||||
fonts, including any derivative works, can be bundled, embedded,
|
||||
redistributed and/or sold with any software provided that any reserved
|
||||
names are not used by derivative works. The fonts and derivatives,
|
||||
however, cannot be released under any other type of license. The
|
||||
requirement for fonts to remain under this license does not apply
|
||||
to any document created using the fonts or their derivatives.
|
||||
|
||||
DEFINITIONS
|
||||
"Font Software" refers to the set of files released by the Copyright
|
||||
Holder(s) under this license and clearly marked as such. This may
|
||||
include source files, build scripts and documentation.
|
||||
|
||||
"Reserved Font Name" refers to any names specified as such after the
|
||||
copyright statement(s).
|
||||
|
||||
"Original Version" refers to the collection of Font Software components as
|
||||
distributed by the Copyright Holder(s).
|
||||
|
||||
"Modified Version" refers to any derivative made by adding to, deleting,
|
||||
or substituting -- in part or in whole -- any of the components of the
|
||||
Original Version, by changing formats or by porting the Font Software to a
|
||||
new environment.
|
||||
|
||||
"Author" refers to any designer, engineer, programmer, technical
|
||||
writer or other person who contributed to the Font Software.
|
||||
|
||||
PERMISSION & CONDITIONS
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of the Font Software, to use, study, copy, merge, embed, modify,
|
||||
redistribute, and sell modified and unmodified copies of the Font
|
||||
Software, subject to the following conditions:
|
||||
|
||||
1) Neither the Font Software nor any of its individual components,
|
||||
in Original or Modified Versions, may be sold by itself.
|
||||
|
||||
2) Original or Modified Versions of the Font Software may be bundled,
|
||||
redistributed and/or sold with any software, provided that each copy
|
||||
contains the above copyright notice and this license. These can be
|
||||
included either as stand-alone text files, human-readable headers or
|
||||
in the appropriate machine-readable metadata fields within text or
|
||||
binary files as long as those fields can be easily viewed by the user.
|
||||
|
||||
3) No Modified Version of the Font Software may use the Reserved Font
|
||||
Name(s) unless explicit written permission is granted by the corresponding
|
||||
Copyright Holder. This restriction only applies to the primary font name as
|
||||
presented to the users.
|
||||
|
||||
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
|
||||
Software shall not be used to promote, endorse or advertise any
|
||||
Modified Version, except to acknowledge the contribution(s) of the
|
||||
Copyright Holder(s) and the Author(s) or with their explicit written
|
||||
permission.
|
||||
|
||||
5) The Font Software, modified or unmodified, in part or in whole,
|
||||
must be distributed entirely under this license, and must not be
|
||||
distributed under any other license. The requirement for fonts to
|
||||
remain under this license does not apply to any document created
|
||||
using the Font Software.
|
||||
|
||||
TERMINATION
|
||||
This license becomes null and void if any of the above conditions are
|
||||
not met.
|
||||
|
||||
DISCLAIMER
|
||||
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
|
||||
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
|
||||
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
|
||||
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
|
||||
OTHER DEALINGS IN THE FONT SOFTWARE.
|
||||
@@ -0,0 +1,883 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"repo": "storytold/lightcraft",
|
||||
"branch": "main",
|
||||
"head": "88ea1a804b35709e1fe436e3792adf0428b97daa",
|
||||
"head_date": "2026-10-09T13:05:04Z",
|
||||
"commits": 729,
|
||||
"unattributed_commits": 0,
|
||||
"contributors": [
|
||||
{
|
||||
"login": "echelon",
|
||||
"people_entry": true,
|
||||
"github_name": null,
|
||||
"prs": 83,
|
||||
"commits": 348,
|
||||
"lines_added": 127920,
|
||||
"lines_deleted": 8745,
|
||||
"binary_added": 38,
|
||||
"binary_deleted": 8,
|
||||
"first_commit": "2026-09-30T14:04:12Z",
|
||||
"last_commit": "2026-10-09T13:05:04Z"
|
||||
},
|
||||
{
|
||||
"login": "leipsfur",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 131,
|
||||
"lines_deleted": 5,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-02T15:51:58Z",
|
||||
"last_commit": "2026-10-02T15:51:58Z"
|
||||
},
|
||||
{
|
||||
"login": "GlemNL",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 9,
|
||||
"commits": 13,
|
||||
"lines_added": 1233,
|
||||
"lines_deleted": 96,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-02T21:16:29Z",
|
||||
"last_commit": "2026-10-04T09:42:39Z"
|
||||
},
|
||||
{
|
||||
"login": "storybored379",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 8,
|
||||
"commits": 9,
|
||||
"lines_added": 1925,
|
||||
"lines_deleted": 119,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-04T00:28:49Z",
|
||||
"last_commit": "2026-10-09T06:02:21Z"
|
||||
},
|
||||
{
|
||||
"login": "midasdf",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 5,
|
||||
"commits": 5,
|
||||
"lines_added": 755,
|
||||
"lines_deleted": 80,
|
||||
"binary_added": 1,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-05T16:07:26Z",
|
||||
"last_commit": "2026-10-09T06:04:40Z"
|
||||
},
|
||||
{
|
||||
"login": "voshart",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 8,
|
||||
"commits": 26,
|
||||
"lines_added": 30524,
|
||||
"lines_deleted": 4416,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-05T16:23:27Z",
|
||||
"last_commit": "2026-10-09T02:31:04Z"
|
||||
},
|
||||
{
|
||||
"login": "arata-1972",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 2,
|
||||
"commits": 4,
|
||||
"lines_added": 2892,
|
||||
"lines_deleted": 726,
|
||||
"binary_added": 2,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-05T20:14:35Z",
|
||||
"last_commit": "2026-10-05T22:16:15Z"
|
||||
},
|
||||
{
|
||||
"login": "benlukka",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 49,
|
||||
"lines_deleted": 4,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-06T11:18:03Z",
|
||||
"last_commit": "2026-10-06T11:18:03Z"
|
||||
},
|
||||
{
|
||||
"login": "kywk",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 72,
|
||||
"lines_deleted": 8,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-06T20:21:14Z",
|
||||
"last_commit": "2026-10-06T20:21:14Z"
|
||||
},
|
||||
{
|
||||
"login": "Exalted5373",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 5,
|
||||
"commits": 11,
|
||||
"lines_added": 1988,
|
||||
"lines_deleted": 132,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-06T21:01:23Z",
|
||||
"last_commit": "2026-10-09T01:33:43Z"
|
||||
},
|
||||
{
|
||||
"login": "treyreynolds",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 159,
|
||||
"lines_deleted": 27,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-06T22:21:46Z",
|
||||
"last_commit": "2026-10-06T22:21:46Z"
|
||||
},
|
||||
{
|
||||
"login": "kovalskiguinther",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 0,
|
||||
"commits": 1,
|
||||
"lines_added": 4328,
|
||||
"lines_deleted": 17,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-06T23:15:04Z",
|
||||
"last_commit": "2026-10-06T23:15:04Z"
|
||||
},
|
||||
{
|
||||
"login": "cruzer45",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 2,
|
||||
"commits": 3,
|
||||
"lines_added": 658,
|
||||
"lines_deleted": 26,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T01:30:23Z",
|
||||
"last_commit": "2026-10-07T03:53:09Z"
|
||||
},
|
||||
{
|
||||
"login": "coygeek",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 4,
|
||||
"lines_deleted": 0,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T02:27:15Z",
|
||||
"last_commit": "2026-10-07T02:27:15Z"
|
||||
},
|
||||
{
|
||||
"login": "lzjtugjc",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 1895,
|
||||
"lines_deleted": 294,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T03:34:33Z",
|
||||
"last_commit": "2026-10-07T03:34:33Z"
|
||||
},
|
||||
{
|
||||
"login": "itsmunzir",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 2,
|
||||
"commits": 1,
|
||||
"lines_added": 46,
|
||||
"lines_deleted": 30,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T07:40:55Z",
|
||||
"last_commit": "2026-10-07T07:40:55Z"
|
||||
},
|
||||
{
|
||||
"login": "haraldh",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 2,
|
||||
"commits": 5,
|
||||
"lines_added": 1163,
|
||||
"lines_deleted": 141,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T07:54:27Z",
|
||||
"last_commit": "2026-10-07T13:16:08Z"
|
||||
},
|
||||
{
|
||||
"login": "sakamoto824",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 2,
|
||||
"lines_added": 3649,
|
||||
"lines_deleted": 167,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T09:32:12Z",
|
||||
"last_commit": "2026-10-07T10:54:19Z"
|
||||
},
|
||||
{
|
||||
"login": "anarion80",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 402,
|
||||
"lines_deleted": 2,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T11:54:51Z",
|
||||
"last_commit": "2026-10-07T11:54:51Z"
|
||||
},
|
||||
{
|
||||
"login": "radialmonster",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 26,
|
||||
"commits": 53,
|
||||
"lines_added": 7227,
|
||||
"lines_deleted": 717,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T12:33:51Z",
|
||||
"last_commit": "2026-10-09T11:31:46Z"
|
||||
},
|
||||
{
|
||||
"login": "bogusyogi",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 2,
|
||||
"commits": 17,
|
||||
"lines_added": 5124,
|
||||
"lines_deleted": 603,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T17:06:00Z",
|
||||
"last_commit": "2026-10-08T19:26:28Z"
|
||||
},
|
||||
{
|
||||
"login": "nledford",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 10,
|
||||
"commits": 88,
|
||||
"lines_added": 9299,
|
||||
"lines_deleted": 1024,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T17:33:38Z",
|
||||
"last_commit": "2026-10-09T00:59:12Z"
|
||||
},
|
||||
{
|
||||
"login": "MusseJusse",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 34,
|
||||
"lines_deleted": 3,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T18:38:23Z",
|
||||
"last_commit": "2026-10-07T18:38:23Z"
|
||||
},
|
||||
{
|
||||
"login": "Ndsanes",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 2,
|
||||
"commits": 2,
|
||||
"lines_added": 332,
|
||||
"lines_deleted": 86,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T19:08:28Z",
|
||||
"last_commit": "2026-10-07T19:08:33Z"
|
||||
},
|
||||
{
|
||||
"login": "RicSchonfelder",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 1439,
|
||||
"lines_deleted": 6,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T20:03:20Z",
|
||||
"last_commit": "2026-10-07T20:03:20Z"
|
||||
},
|
||||
{
|
||||
"login": "Gremiger",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 19,
|
||||
"lines_deleted": 0,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T21:43:23Z",
|
||||
"last_commit": "2026-10-07T21:43:23Z"
|
||||
},
|
||||
{
|
||||
"login": "DWAA1660",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 2,
|
||||
"commits": 2,
|
||||
"lines_added": 206,
|
||||
"lines_deleted": 42,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T22:08:01Z",
|
||||
"last_commit": "2026-10-08T17:04:19Z"
|
||||
},
|
||||
{
|
||||
"login": "sebastianpfluegelmeier",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 2,
|
||||
"commits": 4,
|
||||
"lines_added": 23063,
|
||||
"lines_deleted": 17968,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T22:14:20Z",
|
||||
"last_commit": "2026-10-08T14:38:56Z"
|
||||
},
|
||||
{
|
||||
"login": "agorevski",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 68,
|
||||
"lines_deleted": 11,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-07T22:18:44Z",
|
||||
"last_commit": "2026-10-07T22:18:44Z"
|
||||
},
|
||||
{
|
||||
"login": "andrewyao",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 1,
|
||||
"lines_deleted": 1,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T00:53:19Z",
|
||||
"last_commit": "2026-10-08T00:53:19Z"
|
||||
},
|
||||
{
|
||||
"login": "Hanmo123",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 4,
|
||||
"commits": 6,
|
||||
"lines_added": 922,
|
||||
"lines_deleted": 84,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T02:16:44Z",
|
||||
"last_commit": "2026-10-09T05:35:02Z"
|
||||
},
|
||||
{
|
||||
"login": "wondarkus",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 8,
|
||||
"lines_added": 2776,
|
||||
"lines_deleted": 1387,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T02:56:08Z",
|
||||
"last_commit": "2026-10-08T04:23:30Z"
|
||||
},
|
||||
{
|
||||
"login": "xak47d",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 58,
|
||||
"lines_deleted": 10,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T04:10:21Z",
|
||||
"last_commit": "2026-10-08T04:10:21Z"
|
||||
},
|
||||
{
|
||||
"login": "marceldenzer",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 2864,
|
||||
"lines_deleted": 261,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T05:27:32Z",
|
||||
"last_commit": "2026-10-08T05:27:32Z"
|
||||
},
|
||||
{
|
||||
"login": "bflatastic",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 25,
|
||||
"commits": 2,
|
||||
"lines_added": 108,
|
||||
"lines_deleted": 24,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T06:30:19Z",
|
||||
"last_commit": "2026-10-08T14:00:00Z"
|
||||
},
|
||||
{
|
||||
"login": "wstock",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 8,
|
||||
"commits": 8,
|
||||
"lines_added": 834,
|
||||
"lines_deleted": 67,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T07:45:39Z",
|
||||
"last_commit": "2026-10-08T11:55:57Z"
|
||||
},
|
||||
{
|
||||
"login": "marek-hubar",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 212,
|
||||
"lines_deleted": 29,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T09:01:44Z",
|
||||
"last_commit": "2026-10-08T09:01:44Z"
|
||||
},
|
||||
{
|
||||
"login": "switch87",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 762,
|
||||
"lines_deleted": 34,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T11:00:29Z",
|
||||
"last_commit": "2026-10-08T11:00:29Z"
|
||||
},
|
||||
{
|
||||
"login": "h-a-n-a-s-h-i",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 0,
|
||||
"commits": 2,
|
||||
"lines_added": 50,
|
||||
"lines_deleted": 50,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T12:12:53Z",
|
||||
"last_commit": "2026-10-08T14:01:45Z"
|
||||
},
|
||||
{
|
||||
"login": "Issuenate",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 2,
|
||||
"lines_added": 107,
|
||||
"lines_deleted": 12,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T12:26:27Z",
|
||||
"last_commit": "2026-10-08T12:28:18Z"
|
||||
},
|
||||
{
|
||||
"login": "XusBadia",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 231,
|
||||
"lines_deleted": 1,
|
||||
"binary_added": 2,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T13:12:32Z",
|
||||
"last_commit": "2026-10-08T13:12:32Z"
|
||||
},
|
||||
{
|
||||
"login": "MarekSpodBiedry",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 5,
|
||||
"commits": 6,
|
||||
"lines_added": 336,
|
||||
"lines_deleted": 29,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T13:44:47Z",
|
||||
"last_commit": "2026-10-09T08:45:45Z"
|
||||
},
|
||||
{
|
||||
"login": "peiyan0",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 3,
|
||||
"commits": 3,
|
||||
"lines_added": 215,
|
||||
"lines_deleted": 80,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T15:36:56Z",
|
||||
"last_commit": "2026-10-09T08:09:23Z"
|
||||
},
|
||||
{
|
||||
"login": "hemna",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 459,
|
||||
"lines_deleted": 5,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T16:45:33Z",
|
||||
"last_commit": "2026-10-08T16:45:33Z"
|
||||
},
|
||||
{
|
||||
"login": "joshuaswarren",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 349,
|
||||
"lines_deleted": 18,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T16:47:59Z",
|
||||
"last_commit": "2026-10-08T16:47:59Z"
|
||||
},
|
||||
{
|
||||
"login": "wrgdbtry",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 2733,
|
||||
"lines_deleted": 63,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T19:22:13Z",
|
||||
"last_commit": "2026-10-08T19:22:13Z"
|
||||
},
|
||||
{
|
||||
"login": "parkerdotexe",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 0,
|
||||
"commits": 1,
|
||||
"lines_added": 712,
|
||||
"lines_deleted": 84,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T19:54:50Z",
|
||||
"last_commit": "2026-10-08T19:54:50Z"
|
||||
},
|
||||
{
|
||||
"login": "ragtux",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 3,
|
||||
"commits": 3,
|
||||
"lines_added": 805,
|
||||
"lines_deleted": 50,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-08T22:50:18Z",
|
||||
"last_commit": "2026-10-08T23:42:19Z"
|
||||
},
|
||||
{
|
||||
"login": "sanyamsingh001",
|
||||
"people_entry": true,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 431,
|
||||
"lines_deleted": 19,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-09T03:19:47Z",
|
||||
"last_commit": "2026-10-09T03:19:47Z"
|
||||
},
|
||||
{
|
||||
"login": "onemannoplan",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 1,
|
||||
"lines_deleted": 2,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-09T04:53:21Z",
|
||||
"last_commit": "2026-10-09T04:53:21Z"
|
||||
},
|
||||
{
|
||||
"login": "Yijie-Si",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 0,
|
||||
"commits": 1,
|
||||
"lines_added": 267,
|
||||
"lines_deleted": 43,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-09T06:48:00Z",
|
||||
"last_commit": "2026-10-09T06:48:00Z"
|
||||
},
|
||||
{
|
||||
"login": "HarrisXiu",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 69,
|
||||
"lines_deleted": 1,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-09T09:34:51Z",
|
||||
"last_commit": "2026-10-09T09:34:51Z"
|
||||
},
|
||||
{
|
||||
"login": "rock-meister",
|
||||
"people_entry": true,
|
||||
"github_name": "rock-meister",
|
||||
"prs": 3,
|
||||
"commits": 4,
|
||||
"lines_added": 263,
|
||||
"lines_deleted": 44,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-09T10:15:52Z",
|
||||
"last_commit": "2026-10-09T11:26:47Z"
|
||||
},
|
||||
{
|
||||
"login": "QuirkyTurtle94",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 1,
|
||||
"lines_added": 153,
|
||||
"lines_deleted": 39,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": "2026-10-09T11:08:11Z",
|
||||
"last_commit": "2026-10-09T11:08:11Z"
|
||||
},
|
||||
{
|
||||
"login": "BuonBreda",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 0,
|
||||
"lines_added": 0,
|
||||
"lines_deleted": 0,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": null,
|
||||
"last_commit": null
|
||||
},
|
||||
{
|
||||
"login": "cookingwithsimon",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 0,
|
||||
"lines_added": 0,
|
||||
"lines_deleted": 0,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": null,
|
||||
"last_commit": null
|
||||
},
|
||||
{
|
||||
"login": "GuintherKovalski",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 0,
|
||||
"lines_added": 0,
|
||||
"lines_deleted": 0,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": null,
|
||||
"last_commit": null
|
||||
},
|
||||
{
|
||||
"login": "henu043",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 0,
|
||||
"lines_added": 0,
|
||||
"lines_deleted": 0,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": null,
|
||||
"last_commit": null
|
||||
},
|
||||
{
|
||||
"login": "stano-bustor",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 0,
|
||||
"lines_added": 0,
|
||||
"lines_deleted": 0,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": null,
|
||||
"last_commit": null
|
||||
},
|
||||
{
|
||||
"login": "XRTce",
|
||||
"people_entry": false,
|
||||
"github_name": null,
|
||||
"prs": 1,
|
||||
"commits": 0,
|
||||
"lines_added": 0,
|
||||
"lines_deleted": 0,
|
||||
"binary_added": 0,
|
||||
"binary_deleted": 0,
|
||||
"first_commit": null,
|
||||
"last_commit": null
|
||||
}
|
||||
],
|
||||
"models": [
|
||||
{
|
||||
"id": "anthropic/claude-opus/5.5",
|
||||
"company": "Anthropic",
|
||||
"model": "Claude Opus",
|
||||
"version": "5.5",
|
||||
"commits": 475,
|
||||
"lines_added": 147839,
|
||||
"lines_deleted": 10702
|
||||
},
|
||||
{
|
||||
"id": "anthropic/claude-sonnet/5.5",
|
||||
"company": "Anthropic",
|
||||
"model": "Claude Sonnet",
|
||||
"version": "5.5",
|
||||
"commits": 116,
|
||||
"lines_added": 15071,
|
||||
"lines_deleted": 2624
|
||||
},
|
||||
{
|
||||
"id": "anthropic/claude",
|
||||
"company": "Anthropic",
|
||||
"model": "Claude",
|
||||
"version": "",
|
||||
"commits": 65,
|
||||
"lines_added": 8385,
|
||||
"lines_deleted": 952
|
||||
},
|
||||
{
|
||||
"id": "openai/codex",
|
||||
"company": "OpenAI",
|
||||
"model": "Codex",
|
||||
"version": "",
|
||||
"commits": 33,
|
||||
"lines_added": 14418,
|
||||
"lines_deleted": 1647
|
||||
},
|
||||
{
|
||||
"id": "anthropic/claude-fable/5.1",
|
||||
"company": "Anthropic",
|
||||
"model": "Claude Fable",
|
||||
"version": "5.1",
|
||||
"commits": 9,
|
||||
"lines_added": 1548,
|
||||
"lines_deleted": 112
|
||||
},
|
||||
{
|
||||
"id": "anthropic/claude-sonnet/5",
|
||||
"company": "Anthropic",
|
||||
"model": "Claude Sonnet",
|
||||
"version": "5",
|
||||
"commits": 2,
|
||||
"lines_added": 298,
|
||||
"lines_deleted": 2
|
||||
},
|
||||
{
|
||||
"id": "anthropic/claude-opus/4.8",
|
||||
"company": "Anthropic",
|
||||
"model": "Claude Opus",
|
||||
"version": "4.8",
|
||||
"commits": 1,
|
||||
"lines_added": 1633,
|
||||
"lines_deleted": 3
|
||||
},
|
||||
{
|
||||
"id": "deepseek/deepseek-flash/v4",
|
||||
"company": "DeepSeek",
|
||||
"model": "DeepSeek Flash",
|
||||
"version": "V4",
|
||||
"commits": 1,
|
||||
"lines_added": 1439,
|
||||
"lines_deleted": 6
|
||||
},
|
||||
{
|
||||
"id": "deepseek/deepseek/v3.2",
|
||||
"company": "DeepSeek",
|
||||
"model": "DeepSeek",
|
||||
"version": "V3.2",
|
||||
"commits": 1,
|
||||
"lines_added": 1895,
|
||||
"lines_deleted": 294
|
||||
},
|
||||
{
|
||||
"id": "openai/gpt/6",
|
||||
"company": "OpenAI",
|
||||
"model": "GPT",
|
||||
"version": "6",
|
||||
"commits": 1,
|
||||
"lines_added": 2864,
|
||||
"lines_deleted": 261
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
# Contributor names for the About window (craftrules standards/contributors.md).
|
||||
#
|
||||
# Everyone who contributes is credited by GitHub username automatically. Anything more is up to
|
||||
# you: add YOUR OWN entry below, in your own PR (commit it yourself). Never add someone else; a
|
||||
# maintainer may add a person only when that person asked, recording where in consent_source.
|
||||
#
|
||||
# Easiest: python3 ../../craftrules/scripts/contributors.py --add-me . \
|
||||
# [--real-name "Ada Lovelace"] [--display-name "Ada"] [--sync-github-name]
|
||||
#
|
||||
# Or copy this block, replacing your-github-username (every field is optional):
|
||||
#
|
||||
# [people.your-github-username]
|
||||
# real_name = "Your Name" # shown when the viewer picks "Real name"
|
||||
# display_name = "Nickname" # shown when the viewer picks "Display name"
|
||||
# sync_github_name = true # or: use the name on your public GitHub profile as display name
|
||||
#
|
||||
# To remove your names, delete your entry (or set fields empty) in a PR. The next
|
||||
# contributors.py run, and the next build, drop them.
|
||||
|
||||
[people.echelon]
|
||||
real_name = "Brandon Thomas"
|
||||
display_name = "Brandon Thomas"
|
||||
sync_github_name = false
|
||||
|
||||
[people.rock-meister]
|
||||
sync_github_name = true
|
||||
|
||||
[people.sanyamsingh001]
|
||||
display_name = "sanyamsingh001"
|
||||
@@ -10,6 +10,7 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
lightcraft-develop = { workspace = true }
|
||||
lightcraft-meta = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
@@ -18,4 +19,5 @@ log = { workspace = true }
|
||||
web-time = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
lightcraft-geom = { workspace = true }
|
||||
proptest = { workspace = true }
|
||||
@@ -40,11 +40,17 @@ pub fn weekday(iso: &str) -> Option<&'static str> {
|
||||
}
|
||||
|
||||
/// Shift an ISO time by `secs`, keeping anything after the seconds (fractions, zone) as it was.
|
||||
/// A date without a time gets one. `None` if it doesn't parse.
|
||||
/// A date without a time gets one. `None` if it doesn't parse, or if the result leaves years
|
||||
/// 0000–9999 (an ISO date here has four year digits, so it couldn't be read back); `secs` may be
|
||||
/// anything, it never overflows.
|
||||
pub fn shift_iso(iso: &str, secs: i64) -> Option<String> {
|
||||
let t = iso_seconds(iso)?;
|
||||
let t = iso_seconds(iso)?.checked_add(secs)?;
|
||||
let (first, last) = (iso_seconds("0000-01-01T00:00:00")?, iso_seconds("9999-12-31T23:59:59")?);
|
||||
if !(first..=last).contains(&t) {
|
||||
return None;
|
||||
}
|
||||
let tail = iso.trim().get(19..).unwrap_or("");
|
||||
Some(format!("{}{tail}", civil(t + secs)))
|
||||
Some(format!("{}{tail}", civil(t)))
|
||||
}
|
||||
|
||||
/// Normalize user input to `YYYY-MM-DDTHH:MM:SS` (accepts a space instead of `T`, missing
|
||||
@@ -155,7 +161,7 @@ impl Catalog {
|
||||
let date_of = |id: &PhotoId| -> String {
|
||||
let Some(p) = self.photo(*id) else { return String::new() };
|
||||
let d = match key {
|
||||
SortKey::CaptureDate => Some(p.date()),
|
||||
SortKey::CaptureDate => p.captured.as_deref(),
|
||||
SortKey::ImportDate => Some(p.imported.as_str()),
|
||||
SortKey::EditDate => p.edited.as_deref(),
|
||||
_ => None,
|
||||
@@ -204,6 +210,14 @@ mod tests {
|
||||
assert_eq!(shift_iso("2026-12-31T23:30:00", 3600).as_deref(), Some("2027-01-01T00:30:00"));
|
||||
assert_eq!(shift_iso("2026-03-01T00:00:00.25+02:00", -1).as_deref(), Some("2026-02-28T23:59:59.25+02:00"));
|
||||
assert_eq!(shift_iso("nope", 5), None);
|
||||
// hostile shifts never overflow, and a result outside years 0000–9999 (which no ISO date
|
||||
// here can read back) is refused rather than stored
|
||||
for secs in [i64::MAX, i64::MIN, i64::MAX / 2, -(i64::MAX / 2)] {
|
||||
assert_eq!(shift_iso("2026-01-01T00:00:00", secs), None, "{secs}");
|
||||
}
|
||||
assert_eq!(shift_iso("9999-12-31T23:59:59", 1), None);
|
||||
assert_eq!(shift_iso("0000-01-01T00:00:00", -1), None);
|
||||
assert_eq!(shift_iso("9999-12-31T23:59:58", 1).as_deref(), Some("9999-12-31T23:59:59"));
|
||||
assert_eq!(normalize_iso("2026-04-01 10:05").as_deref(), Some("2026-04-01T10:05:00"));
|
||||
assert_eq!(normalize_iso("2026-02-30"), None);
|
||||
}
|
||||
@@ -222,7 +236,7 @@ mod tests {
|
||||
}
|
||||
let shape = |r: &[DateRun]| r.iter().map(|r| (r.key.clone(), r.start, r.count)).collect::<Vec<_>>();
|
||||
let day = c.date_runs(&ids, SortKey::CaptureDate, GroupBy::Day);
|
||||
// the undated photo falls back to its import date
|
||||
// the undated photo is grouped under Unknown Date
|
||||
assert_eq!(
|
||||
shape(&day),
|
||||
vec![
|
||||
@@ -230,16 +244,20 @@ mod tests {
|
||||
("2026-09-29".into(), 2, 1),
|
||||
("2026-08-01".into(), 3, 1),
|
||||
("2025-12-31".into(), 4, 1),
|
||||
("2026-10-01".into(), 5, 1)
|
||||
(String::new(), 5, 1)
|
||||
]
|
||||
);
|
||||
assert_eq!(day[0].label, "Wednesday, 30 September 2026");
|
||||
assert_eq!(day[4].label, "Unknown Date");
|
||||
assert_eq!(c.date_runs(&ids, SortKey::CaptureDate, GroupBy::Auto), day);
|
||||
assert_eq!(
|
||||
shape(&c.date_runs(&ids[..5], SortKey::CaptureDate, GroupBy::Month)),
|
||||
vec![("2026-09".into(), 0, 3), ("2026-08".into(), 3, 1), ("2025-12".into(), 4, 1)]
|
||||
shape(&c.date_runs(&ids, SortKey::CaptureDate, GroupBy::Month)),
|
||||
vec![("2026-09".into(), 0, 3), ("2026-08".into(), 3, 1), ("2025-12".into(), 4, 1), (String::new(), 5, 1)]
|
||||
);
|
||||
assert_eq!(
|
||||
shape(&c.date_runs(&ids, SortKey::CaptureDate, GroupBy::Year)),
|
||||
vec![("2026".into(), 0, 4), ("2025".into(), 4, 1), (String::new(), 5, 1)]
|
||||
);
|
||||
assert_eq!(shape(&c.date_runs(&ids[..5], SortKey::CaptureDate, GroupBy::Year)), vec![("2026".into(), 0, 4), ("2025".into(), 4, 1)]);
|
||||
assert!(c.date_runs(&ids, SortKey::FileName, GroupBy::Day).is_empty());
|
||||
assert!(c.date_runs(&ids, SortKey::CaptureDate, GroupBy::None).is_empty());
|
||||
// edit date: unedited photos have no date
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
//! The library's folder tree: where on disk the imported photos live.
|
||||
//!
|
||||
//! A *folder* here is a folder holding at least one photo of the library (imported, not deleted,
|
||||
//! not merely browsed in Local; demo scenes have no folder). The tree lists those folders and the
|
||||
//! ones above them, never a folder that holds nothing of the library, with how many photos each
|
||||
//! holds — its subfolders included, like a year in By Date counts its months.
|
||||
//!
|
||||
//! Every folder is a row of its own, the ones above the imported photos included
|
||||
//! (`/Users` → `me` → `Pictures`), so a row is exactly the folder it names. A folder spelled two
|
||||
//! ways (`/` or `\`, repeated or trailing separators, `.`) is one folder (see
|
||||
//! [`crate::query::folder_key`]).
|
||||
//!
|
||||
//! Each disk is a *volume*, a top-level row of its own: a drive letter (`D:`), a Windows share
|
||||
//! (`//server/share`), a mounted disk or share (macOS `/Volumes/tokyo`, Linux `/media/me/usb`,
|
||||
//! `/mnt/disk`, `/run/user/1000/gvfs/smb-share:…`, `/net/host`) or, for everything else, the
|
||||
//! startup disk (`/`). Only the spelling of the stored paths tells them apart: no disk is
|
||||
//! touched, so a share that is asleep or gone lists and counts like one that is there, and the
|
||||
//! sidebar never waits on the network.
|
||||
//!
|
||||
//! Limits of reading only the stored spelling of a path: file names are compared as written
|
||||
//! (Windows folds case, other systems do not), so on a case-insensitive disk `/Volumes/tokyo` and
|
||||
//! `/Volumes/TOKYO` are two volumes here; and one place reached by two spellings
|
||||
//! (`/Volumes/Macintosh HD/Users/me` and `/Users/me`, `/media/me/usb` and `/run/media/me/usb`) is
|
||||
//! two rows. Each row is still consistent: choosing it shows exactly the photos it counts.
|
||||
//!
|
||||
//! A folder is otherwise only where photos are, but the library can also keep a
|
||||
//! [`FolderRecord`] about it (its colour label), stored under the folder's identity
|
||||
//! ([`crate::query::folder_key`]). A record follows its folder when the folder is renamed or moved
|
||||
//! here ([`Catalog::folder_records_follow`]); one about a folder that no longer holds a photo is
|
||||
//! kept (undo can bring the photos back) and shows again when the folder does.
|
||||
//!
|
||||
//! This is the library's own view of its photos; browsing any folder on disk without importing
|
||||
//! is Local's job (see [`crate::local`]).
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::query::{folder_key, folder_rest, folder_within, key_within, split_path};
|
||||
use crate::{Catalog, ColorLabel, Op, Source};
|
||||
|
||||
/// Folder levels kept per photo. No real path comes near it; a deeper one is counted at its
|
||||
/// ancestor this many levels down, which bounds the tree's depth (and so every walk of it)
|
||||
/// whatever a catalog holds.
|
||||
pub const MAX_DEPTH: usize = 64;
|
||||
|
||||
/// One row of the library's folder tree.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
pub struct FolderNode {
|
||||
/// What the row says: the folder's name; for a volume, its name (`nas`, `D:`,
|
||||
/// `//server/share`, `/` for the startup disk).
|
||||
pub name: String,
|
||||
/// The folder the row stands for, written out from the names of the photos' paths (forward
|
||||
/// slashes, no verbatim prefix); hand it back to [`crate::Filter::library_folder`] to see its
|
||||
/// photos.
|
||||
pub path: String,
|
||||
/// Photos in this folder or any folder inside it.
|
||||
pub count: usize,
|
||||
/// Photos directly in this folder.
|
||||
pub own: usize,
|
||||
/// A disk rather than a folder in one; its `path` is where it is mounted.
|
||||
pub volume: bool,
|
||||
/// Whether choosing the row (`Filter::library_folder`) shows exactly its `count` photos. Not
|
||||
/// so for the startup disk and for folders that hold other disks (`/Volumes`, `/mnt`…): their
|
||||
/// path covers those disks' photos too, so choosing them is not offered.
|
||||
pub selectable: bool,
|
||||
/// The folder's colour label (see [`FolderRecord`]).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub label: Option<ColorLabel>,
|
||||
pub children: Vec<FolderNode>,
|
||||
}
|
||||
|
||||
/// What the library keeps about one of its folders beyond the photos in it. A record with
|
||||
/// nothing in it is not kept.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct FolderRecord {
|
||||
/// A colour label, as a photo has one.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub label: Option<ColorLabel>,
|
||||
}
|
||||
|
||||
impl FolderRecord {
|
||||
/// Whether the record says nothing (and so is not kept).
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.label.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
/// The identity a folder's record is kept under: its [`folder_key`]; `None` for a path that
|
||||
/// names no folder (empty, relative, `.`).
|
||||
pub(crate) fn record_key(path: &str) -> Option<String> {
|
||||
let key = folder_key(path);
|
||||
(split_path(path, true).absolute && !key.is_empty()).then_some(key)
|
||||
}
|
||||
|
||||
/// A folder while the tree is being built (children by identity key).
|
||||
struct Level {
|
||||
name: String,
|
||||
path: String,
|
||||
own: usize,
|
||||
volume: bool,
|
||||
children: Vec<String>,
|
||||
}
|
||||
|
||||
/// Where a stored folder path lies: its volume and the folder names below it.
|
||||
struct Placed {
|
||||
mount: String,
|
||||
volume: String,
|
||||
names: Vec<String>,
|
||||
}
|
||||
|
||||
/// A gvfs mount's name made readable: `smb-share:server=tokyo,share=photos` → `photos on tokyo`;
|
||||
/// any other (`sftp:host=nas`) as it is.
|
||||
fn mount_name(n: &str) -> String {
|
||||
let field = |key: &str| {
|
||||
let rest = n.strip_prefix("smb-share:")?;
|
||||
rest.split(',').find_map(|kv| kv.strip_prefix(key)?.strip_prefix('=')).filter(|v| !v.is_empty())
|
||||
};
|
||||
match (field("share"), field("server")) {
|
||||
(Some(share), Some(server)) => format!("{share} on {server}"),
|
||||
_ => n.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The volume `dir` lies on and the folder names below it, read the way [`folder_key`] reads a
|
||||
/// path (so a row and the filter that chooses it never disagree). `None` for a path that is not
|
||||
/// absolute or names no share.
|
||||
fn place(dir: &str) -> Option<Placed> {
|
||||
let sp = split_path(dir, false);
|
||||
let names: Vec<&str> = sp.parts.iter().map(String::as_str).collect();
|
||||
let named = |mount: String, volume: &str, used: usize| (mount, volume.to_string(), used);
|
||||
let (mount, volume, used) = if sp.prefix.starts_with("//") {
|
||||
// `//server/share`: a server alone (no share) is no place to keep photos
|
||||
if sp.prefix.ends_with('/') {
|
||||
return None;
|
||||
}
|
||||
named(sp.prefix.clone(), &sp.prefix, 0)
|
||||
} else if !sp.prefix.is_empty() {
|
||||
named(sp.prefix.clone(), &sp.prefix, 0)
|
||||
} else if sp.absolute {
|
||||
match names.as_slice() {
|
||||
["Volumes", n, ..] => named(format!("/Volumes/{n}"), n, 2),
|
||||
["run", "media", u, n, ..] => named(format!("/run/media/{u}/{n}"), n, 4),
|
||||
["media", u, n, ..] => named(format!("/media/{u}/{n}"), n, 3),
|
||||
["mnt", n, ..] => named(format!("/mnt/{n}"), n, 2),
|
||||
// Linux desktops mount network places per user, naming server and share
|
||||
["run", "user", uid, "gvfs", n, ..] => named(format!("/run/user/{uid}/gvfs/{n}"), &mount_name(n), 5),
|
||||
// NFS automounts: Linux `/net/<host>`, macOS `/Network/Servers/<host>`
|
||||
["net", host, ..] => named(format!("/net/{host}"), host, 2),
|
||||
["Network", "Servers", host, ..] => named(format!("/Network/Servers/{host}"), host, 3),
|
||||
_ => named("/".to_string(), "/", 0),
|
||||
}
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
Some(Placed { mount, volume, names: names.iter().skip(used).take(MAX_DEPTH).map(|n| n.to_string()).collect() })
|
||||
}
|
||||
|
||||
/// The disk or share (`/Volumes/nas`, `D:`, `/` for the startup disk) a file-backed photo is on,
|
||||
/// from its stored path alone.
|
||||
pub fn volume_of(p: &crate::Photo) -> Option<String> {
|
||||
let Source::File { path } = &p.source else { return None };
|
||||
let dir = path.rfind(['/', '\\']).and_then(|i| path.get(..i.max(1)))?;
|
||||
place(dir).map(|pl| folder_key(&pl.mount))
|
||||
}
|
||||
|
||||
/// Whether `path` is a whole disk or share (`/`, `C:\`, `\\srv\share`, `/Volumes/nas`), not a
|
||||
/// folder in one.
|
||||
pub fn is_disk_root(path: &str) -> bool {
|
||||
place(path).is_some_and(|p| p.names.is_empty())
|
||||
}
|
||||
|
||||
/// Whether `path` is the startup disk's root (`/`), whose path also covers every other disk.
|
||||
pub fn is_startup_disk(path: &str) -> bool {
|
||||
place(path).is_some_and(|p| p.names.is_empty() && p.mount == "/")
|
||||
}
|
||||
|
||||
/// A folder named by its last two names (`photos/travel`), so two folders called `travel` are
|
||||
/// told apart; the path itself when it has no names (`/`).
|
||||
pub fn folder_label(path: &str) -> String {
|
||||
let names: Vec<&str> = path.split(['/', '\\']).filter(|n| !n.is_empty()).collect();
|
||||
if names.is_empty() { path.to_string() } else { names.iter().skip(names.len().saturating_sub(2)).copied().collect::<Vec<_>>().join("/") }
|
||||
}
|
||||
|
||||
impl Catalog {
|
||||
/// What the library keeps about folder `path`, however it is spelled.
|
||||
pub fn folder_record(&self, path: &str) -> Option<&FolderRecord> {
|
||||
record_key(path).and_then(|k| self.folder_records.get(&k))
|
||||
}
|
||||
|
||||
/// The colour label of folder `path`, however it is spelled.
|
||||
pub fn folder_color_label(&self, path: &str) -> Option<ColorLabel> {
|
||||
self.folder_record(path).and_then(|r| r.label)
|
||||
}
|
||||
|
||||
/// The op that gives folder `path` colour label `label` (`None`: takes it off), keeping the
|
||||
/// rest of its record. Applying it checks the path.
|
||||
pub fn folder_label_op(&self, path: &str, label: Option<ColorLabel>) -> Op {
|
||||
let mut record = self.folder_record(path).cloned().unwrap_or_default();
|
||||
record.label = label;
|
||||
Op::SetFolderRecord { folder: path.to_string(), record: Some(record) }
|
||||
}
|
||||
|
||||
/// The ops that carry the records of folder `from` and the folders inside it over to `to`
|
||||
/// (the same place below it), for a folder renamed or moved on disk: commit them with the
|
||||
/// photos' relinks so undo puts both back. Records already at or below `to` are left from a
|
||||
/// folder that is no longer there and are dropped (undo restores them). Empty when there is
|
||||
/// nothing to carry or drop.
|
||||
pub fn folder_records_follow(&self, from: &str, to: &str) -> Vec<Op> {
|
||||
let (Some(root), Some(dest)) = (record_key(from), record_key(to)) else { return Vec::new() };
|
||||
if root == dest {
|
||||
return Vec::new();
|
||||
}
|
||||
let moved: Vec<(String, FolderRecord)> = self
|
||||
.folder_records
|
||||
.iter()
|
||||
.filter(|(k, _)| key_within(k, &root))
|
||||
.filter_map(|(k, r)| {
|
||||
let rest = folder_rest(k, &root)?;
|
||||
let base = dest.trim_end_matches('/');
|
||||
let path = if rest.is_empty() { dest.clone() } else { format!("{base}/{}", rest.join("/")) };
|
||||
Some((folder_key(&path), r.clone()))
|
||||
})
|
||||
.collect();
|
||||
// what was known at the old place goes, and so does what is left at the new one (also
|
||||
// when the moved folder carries nothing: the folder now there is not the one it was about)
|
||||
let stale = self.folder_records.keys().filter(|k| key_within(k, &root) || key_within(k, &dest));
|
||||
let mut ops: Vec<Op> = stale.map(|k| Op::SetFolderRecord { folder: k.clone(), record: None }).collect();
|
||||
ops.extend(moved.into_iter().map(|(k, r)| Op::SetFolderRecord { folder: k, record: Some(r) }));
|
||||
ops
|
||||
}
|
||||
|
||||
/// The library's volumes with their folders and photo counts: volumes by name
|
||||
/// (case-insensitive), each folder with its subfolders the same way.
|
||||
pub fn folder_tree(&self) -> Vec<FolderNode> {
|
||||
// photos per containing folder, as spelled (sorted, so the spelling that names a folder
|
||||
// never depends on hash order)
|
||||
let mut dirs: BTreeMap<&str, usize> = BTreeMap::new();
|
||||
for p in self.photos().filter(|p| p.in_library()) {
|
||||
if let Source::File { path } = &p.source
|
||||
&& let Some(i) = path.rfind(['/', '\\'])
|
||||
// a photo straight in the root has an empty folder part: the root itself
|
||||
&& let Some(dir) = path.get(..i.max(1))
|
||||
{
|
||||
*dirs.entry(dir).or_default() += 1;
|
||||
}
|
||||
}
|
||||
let mut levels: HashMap<String, Level> = HashMap::new();
|
||||
let mut roots: Vec<String> = Vec::new();
|
||||
for (dir, n) in dirs {
|
||||
let Some(placed) = place(dir) else { continue };
|
||||
let vkey = folder_key(&placed.mount);
|
||||
if !levels.contains_key(&vkey) {
|
||||
levels.insert(
|
||||
vkey.clone(),
|
||||
Level { name: placed.volume.clone(), path: placed.mount.clone(), own: 0, volume: true, children: Vec::new() },
|
||||
);
|
||||
roots.push(vkey.clone());
|
||||
}
|
||||
let mut prefix = if placed.mount == "/" { String::new() } else { placed.mount.clone() };
|
||||
let mut parent = vkey;
|
||||
for name in &placed.names {
|
||||
prefix.push('/');
|
||||
prefix.push_str(name);
|
||||
let key = folder_key(&prefix);
|
||||
if !levels.contains_key(&key) {
|
||||
levels.insert(key.clone(), Level { name: name.clone(), path: prefix.clone(), own: 0, volume: false, children: Vec::new() });
|
||||
if let Some(l) = levels.get_mut(&parent) {
|
||||
l.children.push(key.clone());
|
||||
}
|
||||
}
|
||||
parent = key;
|
||||
}
|
||||
if let Some(l) = levels.get_mut(&parent) {
|
||||
l.own += n;
|
||||
}
|
||||
}
|
||||
let mut tree: Vec<FolderNode> = roots.iter().filter_map(|k| build(k, &levels, &self.folder_records)).collect();
|
||||
sort(&mut tree);
|
||||
// a folder that holds another disk's mount covers that disk's photos too
|
||||
let mounts: Vec<String> = tree.iter().filter(|v| v.path != "/").map(|v| v.path.clone()).collect();
|
||||
mark_selectable(&mut tree, &mounts);
|
||||
tree
|
||||
}
|
||||
}
|
||||
|
||||
/// The node for `key` with everything below it (at most [`MAX_DEPTH`] deep).
|
||||
fn build(key: &str, levels: &HashMap<String, Level>, records: &BTreeMap<String, FolderRecord>) -> Option<FolderNode> {
|
||||
let l = levels.get(key)?;
|
||||
let mut children: Vec<FolderNode> = l.children.iter().filter_map(|k| build(k, levels, records)).collect();
|
||||
sort(&mut children);
|
||||
let count = l.own + children.iter().map(|c| c.count).sum::<usize>();
|
||||
Some(FolderNode {
|
||||
name: l.name.clone(),
|
||||
path: l.path.clone(),
|
||||
count,
|
||||
own: l.own,
|
||||
volume: l.volume,
|
||||
selectable: true,
|
||||
label: records.get(key).and_then(|r| r.label),
|
||||
children,
|
||||
})
|
||||
}
|
||||
|
||||
fn mark_selectable(nodes: &mut [FolderNode], mounts: &[String]) {
|
||||
for n in nodes {
|
||||
n.selectable = !(n.volume && n.path == "/") && (n.volume || !mounts.iter().any(|m| folder_within(m, &n.path)));
|
||||
mark_selectable(&mut n.children, mounts);
|
||||
}
|
||||
}
|
||||
|
||||
fn sort(v: &mut [FolderNode]) {
|
||||
v.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase()).then_with(|| a.name.cmp(&b.name)));
|
||||
}
|
||||
+130
-21
@@ -1,8 +1,9 @@
|
||||
//! Crash-safe catalog persistence: an append-only op log plus periodic snapshots.
|
||||
//!
|
||||
//! Files (in a [`Store`]):
|
||||
//! - `catalog.snap` — `{"format":"lightcraft-catalog","version":1,"seq":N,"catalog":{…}}`, the
|
||||
//! state after op `N`; replaced atomically (temp + fsync + rename).
|
||||
//! - `catalog.snap` — `{"format":"lightcraft-catalog","version":V,"seq":N,"catalog":{…}}`, the
|
||||
//! state after op `N` in catalog format `V` ([`VERSION`]); replaced atomically (temp + fsync +
|
||||
//! rename).
|
||||
//! - `catalog.log` — JSON lines, one per op applied after the snapshot:
|
||||
//! `{"seq":N,"crc":C,"op":{…}}` where `C` is the CRC-32 of the op's JSON text. Appends are
|
||||
//! fsynced before [`Journal::append`] returns.
|
||||
@@ -33,6 +34,36 @@
|
||||
//!
|
||||
//! At most one snapshot is in flight; a synchronous [`Journal::snapshot`] and dropping the journal
|
||||
//! first wait for it, so an older snapshot can never replace a newer one.
|
||||
//!
|
||||
//! # Format versions
|
||||
//!
|
||||
//! The snapshot's `version` is the catalog format, and it covers the log beside it: the log has
|
||||
//! no header of its own, and is only ever appended to under a snapshot of this build's format.
|
||||
//!
|
||||
//! | Version | Written by | Adds |
|
||||
//! |---|---|---|
|
||||
//! | (none) | early builds: a log without a snapshot | |
|
||||
//! | 1 | up to v0.2.0 | |
|
||||
//! | 2 | after v0.2.0 | `Op::SetBrowsed`, `Catalog.browsed`, `Photo.local_baseline` |
|
||||
//! | 3 | albums by hand | `Op::SetAlbumOrder`, `Album.order` |
|
||||
//! | 4 | Reload reads lens data | `Op::SetEmbeddedLens` |
|
||||
//! | 5 | the keyword list | `Op::SetKeyword`, `Catalog.keyword_list` |
|
||||
//! | 6 | folder labels | `Op::SetFolderRecord`, `Catalog.folder_records` |
|
||||
//!
|
||||
//! Rules:
|
||||
//! - **Bump [`VERSION`]** (and add a row above) in the change that adds an [`Op`] variant or a
|
||||
//! serialized field to the catalog, a photo, an album, a stack or develop settings that an older
|
||||
//! build would misread or silently drop. New fields must deserialize from older data
|
||||
//! (`#[serde(default)]`), so newer builds keep reading every older format.
|
||||
//! - **Opening an older format upgrades it**: right after loading, [`Journal::open`] writes a
|
||||
//! snapshot in this build's format (also for a new library), before anything is appended — so a
|
||||
//! library is never a mix, and older builds refuse it from then on instead of reading part of it.
|
||||
//! [`LoadReport::upgraded_from`] says when that happened.
|
||||
//! - **A newer format is refused, untouched**: a snapshot whose `version` is above [`VERSION`],
|
||||
//! or a log record whose CRC matches (so it is exactly what was written) but whose op doesn't
|
||||
//! parse, fails [`Journal::open`] with [`CatalogError::Newer`] before any file is modified — it
|
||||
//! is never mistaken for a torn tail or damage. (Builds up to v0.2.0 refuse a v2 snapshot with
|
||||
//! "unsupported format".)
|
||||
|
||||
use crate::store::Store;
|
||||
use crate::{Catalog, CatalogError, Op, Result};
|
||||
@@ -40,7 +71,9 @@ use crate::{Catalog, CatalogError, Op, Result};
|
||||
pub const SNAPSHOT: &str = "catalog.snap";
|
||||
pub const LOG: &str = "catalog.log";
|
||||
const FORMAT: &str = "lightcraft-catalog";
|
||||
const VERSION: u32 = 1;
|
||||
/// The catalog format this build writes (and the newest it reads). See the module docs →
|
||||
/// *Format versions*; bump it whenever an [`Op`] variant or a serialized field is added.
|
||||
pub const VERSION: u32 = 6;
|
||||
|
||||
/// When [`Journal::wants_snapshot`] says it's time to compact the log.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
@@ -75,6 +108,9 @@ pub struct LoadReport {
|
||||
pub damaged: Option<String>,
|
||||
/// No catalog files existed (a new library).
|
||||
pub created: bool,
|
||||
/// The library was in an older catalog format (`0` = before format versions: a log without a
|
||||
/// snapshot) and was rewritten in this version's format ([`VERSION`]).
|
||||
pub upgraded_from: Option<u32>,
|
||||
}
|
||||
|
||||
/// Where persistence time goes (reported by `library.info` → `persistence`; printed to stderr
|
||||
@@ -180,18 +216,46 @@ pub fn encode_record(seq: u64, op: &Op) -> String {
|
||||
format!("{{\"seq\":{seq},\"crc\":{crc},\"op\":{body}}}")
|
||||
}
|
||||
|
||||
/// Parse one log line (without the newline). `None` if it's malformed or the CRC doesn't match.
|
||||
/// Parse one log line (without the newline). `None` if it's malformed, the CRC doesn't match, or
|
||||
/// the op is one this version doesn't know (see [`Journal::open`]).
|
||||
pub fn decode_record(line: &str) -> Option<(u64, Op)> {
|
||||
let rest = line.strip_prefix("{\"seq\":")?;
|
||||
let (seq, rest) = rest.split_once(",\"crc\":")?;
|
||||
let (crc, rest) = rest.split_once(",\"op\":")?;
|
||||
let body = rest.strip_suffix('}')?;
|
||||
let seq: u64 = seq.parse().ok()?;
|
||||
let crc: u32 = crc.parse().ok()?;
|
||||
if crc32fast::hash(body.as_bytes()) != crc {
|
||||
return None;
|
||||
match decode_line(line) {
|
||||
Line::Record(seq, op) => Some((seq, op)),
|
||||
Line::Newer(_) | Line::Bad => None,
|
||||
}
|
||||
serde_json::from_str(body).ok().map(|op| (seq, op))
|
||||
}
|
||||
|
||||
/// One log line, decoded.
|
||||
enum Line {
|
||||
Record(u64, Op),
|
||||
/// The CRC matches — the line is exactly what was written — but the op doesn't parse: it
|
||||
/// was written by a newer LightCraft (an op or field this version doesn't know), not damaged.
|
||||
Newer(u64),
|
||||
/// Malformed or a CRC mismatch: torn or damaged.
|
||||
Bad,
|
||||
}
|
||||
|
||||
fn decode_line(line: &str) -> Line {
|
||||
let parts = || {
|
||||
let rest = line.strip_prefix("{\"seq\":")?;
|
||||
let (seq, rest) = rest.split_once(",\"crc\":")?;
|
||||
let (crc, rest) = rest.split_once(",\"op\":")?;
|
||||
let body = rest.strip_suffix('}')?;
|
||||
Some((seq.parse::<u64>().ok()?, crc.parse::<u32>().ok()?, body))
|
||||
};
|
||||
let Some((seq, crc, body)) = parts() else { return Line::Bad };
|
||||
if crc32fast::hash(body.as_bytes()) != crc {
|
||||
return Line::Bad;
|
||||
}
|
||||
match serde_json::from_str(body) {
|
||||
Ok(op) => Line::Record(seq, op),
|
||||
Err(_) => Line::Newer(seq),
|
||||
}
|
||||
}
|
||||
|
||||
/// The error for a library written by a newer LightCraft.
|
||||
fn newer(what: String) -> CatalogError {
|
||||
CatalogError::Newer(format!("{what}; this version reads catalog format v{VERSION} and older"))
|
||||
}
|
||||
|
||||
/// A line that doesn't decode but ends with a whole record: a fragment of a failed append followed
|
||||
@@ -229,10 +293,16 @@ fn write_snapshot(store: &mut dyn Store, seq: u64, catalog: &Catalog) -> std::io
|
||||
Ok(SnapshotTiming { serialize_ms, write_sync_ms: (ms_since(t0) - serialize_ms).max(0.0), bytes, ..Default::default() })
|
||||
}
|
||||
|
||||
/// Just the identification of `catalog.snap` (read before the catalog itself).
|
||||
#[derive(serde::Deserialize)]
|
||||
struct SnapFile {
|
||||
struct SnapHeader {
|
||||
format: String,
|
||||
version: u32,
|
||||
}
|
||||
|
||||
/// The whole `catalog.snap` (after [`SnapHeader`] was checked; `format` and `version` are ignored).
|
||||
#[derive(serde::Deserialize)]
|
||||
struct SnapFile {
|
||||
seq: u64,
|
||||
catalog: Catalog,
|
||||
}
|
||||
@@ -245,15 +315,20 @@ impl Journal {
|
||||
let snap = store.read(SNAPSHOT).map_err(io)?;
|
||||
let log = store.read(LOG).map_err(io)?;
|
||||
report.created = snap.is_none() && log.is_none();
|
||||
let (mut catalog, snapshot_seq) = match snap {
|
||||
let (mut catalog, snapshot_seq, snapshot_version) = match snap {
|
||||
Some(bytes) => {
|
||||
let s: SnapFile = serde_json::from_slice(&bytes).map_err(|e| CatalogError::Corrupt(format!("{SNAPSHOT}: {e}")))?;
|
||||
if s.format != FORMAT || s.version > VERSION {
|
||||
return Err(CatalogError::Corrupt(format!("{SNAPSHOT}: unsupported format {} v{}", s.format, s.version)));
|
||||
// the header first: a newer snapshot may not parse as this version's catalog
|
||||
let h: SnapHeader = serde_json::from_slice(&bytes).map_err(|e| CatalogError::Corrupt(format!("{SNAPSHOT}: {e}")))?;
|
||||
if h.format != FORMAT {
|
||||
return Err(CatalogError::Corrupt(format!("{SNAPSHOT}: not a LightCraft catalog (format {:?})", h.format)));
|
||||
}
|
||||
(s.catalog, s.seq)
|
||||
if h.version > VERSION {
|
||||
return Err(newer(format!("{SNAPSHOT} is catalog format v{}", h.version)));
|
||||
}
|
||||
let s: SnapFile = serde_json::from_slice(&bytes).map_err(|e| CatalogError::Corrupt(format!("{SNAPSHOT}: {e}")))?;
|
||||
(s.catalog, s.seq, Some(h.version))
|
||||
}
|
||||
None => (Catalog::new(), 0),
|
||||
None => (Catalog::new(), 0, None),
|
||||
};
|
||||
report.snapshot_seq = snapshot_seq;
|
||||
let mut j = Journal {
|
||||
@@ -296,6 +371,11 @@ impl Journal {
|
||||
report.torn_fragments += 1;
|
||||
record = Some(r);
|
||||
}
|
||||
if record.is_none()
|
||||
&& let Some(Line::Newer(seq)) = line.map(decode_line)
|
||||
{
|
||||
return Err(newer(format!("{LOG} record {seq} holds a change this version doesn't know")));
|
||||
}
|
||||
match record {
|
||||
Some((seq, op)) if seq <= j.seq => {
|
||||
// already in the snapshot
|
||||
@@ -312,7 +392,15 @@ impl Journal {
|
||||
}
|
||||
_ => {
|
||||
// bad record (or a gap): torn tail if nothing good follows, else damage
|
||||
let rest_has_good = log[next..].split(|b| *b == b'\n').any(|l| std::str::from_utf8(l).ok().and_then(decode_record).is_some());
|
||||
let mut rest_has_good = false;
|
||||
for l in log[next..].split(|b| *b == b'\n') {
|
||||
match std::str::from_utf8(l).map(|l| decode_line(l.trim_end_matches('\r'))) {
|
||||
Ok(Line::Record(..)) => rest_has_good = true,
|
||||
// refused before anything is modified
|
||||
Ok(Line::Newer(seq)) => return Err(newer(format!("{LOG} record {seq} holds a change this version doesn't know"))),
|
||||
Ok(Line::Bad) | Err(_) => {}
|
||||
}
|
||||
}
|
||||
if rest_has_good {
|
||||
damaged_at = Some(pos);
|
||||
}
|
||||
@@ -344,6 +432,27 @@ impl Journal {
|
||||
j.log_bytes = good_end as u64;
|
||||
}
|
||||
}
|
||||
// Bring an older (or new, or snapshot-less) library to this version's format before
|
||||
// anything is appended: a snapshot in this format makes older builds refuse the library
|
||||
// instead of misreading ops or dropping fields they don't know.
|
||||
if report.created {
|
||||
// tiny: written directly, not counted as a compaction
|
||||
let empty = format!("{{\"format\":\"{FORMAT}\",\"version\":{VERSION},\"seq\":0,\"catalog\":{}}}\n", catalog.to_snapshot());
|
||||
if let Err(e) = j.store.write_atomic(SNAPSHOT, empty.as_bytes()) {
|
||||
log::warn!("catalog: can't write the new library's snapshot: {e}");
|
||||
}
|
||||
} else if report.damaged.is_none() && snapshot_version != Some(VERSION) {
|
||||
report.upgraded_from = Some(snapshot_version.unwrap_or(0));
|
||||
match j.snapshot(&catalog) {
|
||||
Ok(()) => {
|
||||
if let Some(v) = report.upgraded_from {
|
||||
log::info!("catalog: upgraded from format v{v} to v{VERSION}");
|
||||
}
|
||||
}
|
||||
// nothing lost: the old files are intact and still load; retried on next open
|
||||
Err(e) => log::warn!("catalog: can't write the format v{VERSION} snapshot: {e}"),
|
||||
}
|
||||
}
|
||||
catalog.revision = 0;
|
||||
Ok((j, catalog, report))
|
||||
}
|
||||
|
||||
+1184
-20
File diff suppressed because it is too large.
Load diff
+201
-4
@@ -5,13 +5,20 @@
|
||||
//! snapshot — crash-safe and diff-friendly (see [`journal`]);
|
||||
//! - **undo/redo**: the engine keeps inverse ops;
|
||||
//! - **determinism**: replaying the log reproduces the state exactly (property-tested).
|
||||
//!
|
||||
//! **Catalog format version** ([`journal::VERSION`], see [`journal`] → *Format versions*): adding
|
||||
//! an [`Op`] variant or a serialized field means bumping it. Newer builds read every older format
|
||||
//! (and upgrade it on open); older builds refuse a newer library with [`CatalogError::Newer`]
|
||||
//! instead of reading part of it.
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
pub mod dates;
|
||||
pub mod folders;
|
||||
pub mod journal;
|
||||
pub mod keywords;
|
||||
pub mod local;
|
||||
pub mod lock;
|
||||
pub mod model;
|
||||
pub mod query;
|
||||
pub mod rules;
|
||||
@@ -23,12 +30,14 @@ use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub use dates::{DateRun, GroupBy};
|
||||
pub use folders::{FolderNode, FolderRecord};
|
||||
pub use journal::{Journal, LoadReport, PersistStats, SnapshotPolicy, SnapshotTiming};
|
||||
pub use keywords::KeywordNode;
|
||||
use lightcraft_develop::DevelopSettings;
|
||||
pub use local::{DEFAULT_FORGET_DAYS, ForgetPlan, folder_of};
|
||||
pub use lock::{LibraryLock, LockError, LockOwner};
|
||||
pub use model::*;
|
||||
pub use query::{DateGroup, Filter, RatingOp, Sort, SortKey};
|
||||
pub use query::{DateGroup, Filter, Person, RatingOp, Sort, SortKey, mix64};
|
||||
pub use rules::{Match, Rule, RuleSet};
|
||||
use serde::{Deserialize, Serialize};
|
||||
pub use store::{FsStore, MemStore, Store};
|
||||
@@ -43,10 +52,17 @@ pub enum CatalogError {
|
||||
NoStack(StackId),
|
||||
#[error("invalid: {0}")]
|
||||
Invalid(String),
|
||||
/// A keyword of that name is there already: moving or renaming onto it would merge the two.
|
||||
#[error("there is a keyword “{0}” already")]
|
||||
KeywordExists(String),
|
||||
#[error("corrupt catalog data: {0}")]
|
||||
Corrupt(String),
|
||||
#[error("catalog storage: {0}")]
|
||||
Io(String),
|
||||
/// The library was written by a newer LightCraft (a newer catalog format, or a change this
|
||||
/// version doesn't know). Nothing was read into the session and nothing was modified.
|
||||
#[error("this library was written by a newer version of LightCraft ({0}); update LightCraft to open it. The library was left unchanged.")]
|
||||
Newer(String),
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, CatalogError>;
|
||||
@@ -123,6 +139,11 @@ pub enum Op {
|
||||
id: AlbumId,
|
||||
parent: Option<AlbumId>,
|
||||
},
|
||||
/// Where the album stands among its siblings (`None`: by name). Format version 3.
|
||||
SetAlbumOrder {
|
||||
id: AlbumId,
|
||||
order: Option<u32>,
|
||||
},
|
||||
SetAlbumPhotos {
|
||||
id: AlbumId,
|
||||
photos: Vec<PhotoId>,
|
||||
@@ -188,6 +209,11 @@ pub enum Op {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
preview_only: Option<String>,
|
||||
},
|
||||
/// The lens data a photo's file carries (what Reload finds when it was read after import).
|
||||
SetEmbeddedLens {
|
||||
id: PhotoId,
|
||||
lens: Option<Box<lightcraft_develop::EmbeddedLens>>,
|
||||
},
|
||||
/// The name shown for a colour label (`None` = its colour's name).
|
||||
SetLabelName {
|
||||
label: ColorLabel,
|
||||
@@ -199,6 +225,18 @@ pub enum Op {
|
||||
folder: String,
|
||||
at: Option<String>,
|
||||
},
|
||||
/// List a keyword in the library's keyword list with these attributes, or take it off the list
|
||||
/// (`None`; photos that carry it keep it). Format version 4.
|
||||
SetKeyword {
|
||||
path: String,
|
||||
info: Option<keywords::KeywordInfo>,
|
||||
},
|
||||
/// What the library keeps about one of its folders (see [`folders::FolderRecord`]), under
|
||||
/// its identity ([`query::folder_key`]); `None` or an empty record = keep nothing.
|
||||
SetFolderRecord {
|
||||
folder: String,
|
||||
record: Option<FolderRecord>,
|
||||
},
|
||||
/// Several ops as one step (undo applies the inverses in reverse).
|
||||
Batch {
|
||||
ops: Vec<Op>,
|
||||
@@ -221,6 +259,12 @@ pub struct Catalog {
|
||||
/// When each Local folder was last browsed (folder path → ISO 8601), see [`local`].
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
browsed: BTreeMap<String, String>,
|
||||
/// Keywords listed on their own or given attributes, by lower-case path (see [`keywords`]).
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
keyword_list: BTreeMap<String, keywords::ListedKeyword>,
|
||||
/// What the library keeps about its folders (folder identity → record), see [`folders`].
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
folder_records: BTreeMap<String, FolderRecord>,
|
||||
/// Increments on every applied op.
|
||||
#[serde(skip)]
|
||||
pub revision: u64,
|
||||
@@ -269,6 +313,28 @@ impl Catalog {
|
||||
pub fn albums(&self) -> impl Iterator<Item = &Album> {
|
||||
self.albums.values()
|
||||
}
|
||||
/// The albums and folders directly inside `parent` (`None`: the top level), in the order the
|
||||
/// sidebar lists them: folders first, then by their place if the user ordered them (those
|
||||
/// without one after, by name), else by name.
|
||||
pub fn album_children(&self, parent: Option<AlbumId>) -> Vec<&Album> {
|
||||
let mut kids: Vec<&Album> = self.albums.values().filter(|a| a.parent == parent).collect();
|
||||
kids.sort_by_cached_key(|a| album_order_key(a));
|
||||
kids
|
||||
}
|
||||
/// [`Self::album_children`] of every folder (and of the top level, under `None`) in one pass:
|
||||
/// what a tree view that draws all of them wants each frame.
|
||||
pub fn album_children_by_parent(&self) -> std::collections::HashMap<Option<AlbumId>, Vec<&Album>> {
|
||||
let mut map: std::collections::HashMap<Option<AlbumId>, Vec<&Album>> = std::collections::HashMap::new();
|
||||
for a in self.albums.values() {
|
||||
map.entry(a.parent).or_default().push(a);
|
||||
}
|
||||
map.values_mut().for_each(|kids| kids.sort_by_cached_key(|a| album_order_key(a)));
|
||||
map
|
||||
}
|
||||
/// Whether anything inside `parent` has a place of its own (the folder is ordered by hand).
|
||||
pub fn album_children_are_ordered(&self, parent: Option<AlbumId>) -> bool {
|
||||
self.albums.values().any(|a| a.parent == parent && a.order.is_some())
|
||||
}
|
||||
/// The Quick Collection, once something was added to it.
|
||||
pub fn quick_collection(&self) -> Option<AlbumId> {
|
||||
self.albums.values().find(|a| a.quick).map(|a| a.id)
|
||||
@@ -284,12 +350,57 @@ impl Catalog {
|
||||
/// no smart album).
|
||||
pub fn album_contains(&self, id: AlbumId, p: &Photo) -> bool {
|
||||
match self.albums.get(&id) {
|
||||
Some(Album { smart: Some(rules), .. }) => !p.deleted && rules.matches(p, self),
|
||||
// guarded: a smart album testing smart albums can't loop or recurse without end
|
||||
Some(Album { smart: Some(rules), .. }) => !p.deleted && rules::smart_album_holds(id, p.id, || rules.matches(p, self)),
|
||||
Some(a) => a.photos.contains(&p.id),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether smart album `from` tests `to`, directly or through the smart albums it tests (the
|
||||
/// rules of `from` would then change when those of `to` do). Loops in saved rules end the
|
||||
/// search, they don't repeat it.
|
||||
pub fn album_reaches(&self, from: AlbumId, to: AlbumId) -> bool {
|
||||
let mut seen: std::collections::HashSet<AlbumId> = std::collections::HashSet::new();
|
||||
let mut next = vec![from];
|
||||
while let Some(a) = next.pop() {
|
||||
if !seen.insert(a) {
|
||||
continue;
|
||||
}
|
||||
let tested = self.albums.get(&a).and_then(|al| al.smart.as_deref()).map(Filter::albums_tested).unwrap_or_default();
|
||||
if tested.contains(&to) {
|
||||
return true;
|
||||
}
|
||||
next.extend(tested);
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// What is wrong with a saved smart album's rules now (`RuleSet::check`, after
|
||||
/// `RuleSet::upgrade`): typically a rule testing an album that has since been deleted. Empty
|
||||
/// for a sound smart album, a plain album or no album.
|
||||
pub fn smart_album_problems(&self, id: AlbumId) -> Vec<rules::Problem> {
|
||||
let Some(filter) = self.albums.get(&id).and_then(|a| a.smart.as_deref()) else { return Vec::new() };
|
||||
let mut out: Vec<rules::Problem> = self.album_filter_problem(filter, Some(id)).into_iter().collect();
|
||||
if let Some(mut rules) = filter.rule_set.clone() {
|
||||
rules.upgrade();
|
||||
out.extend(rules.check_for(self, Some(id)));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A loop through `filter`'s own album field (not its rules): smart album `owner` filtered to
|
||||
/// itself, or to an album that leads back to it.
|
||||
pub fn album_filter_problem(&self, filter: &Filter, owner: Option<AlbumId>) -> Option<rules::Problem> {
|
||||
let (a, owner) = (filter.album?, owner?);
|
||||
(a == owner || self.album_reaches(a, owner)).then(|| rules::Problem {
|
||||
path: Vec::new(),
|
||||
field: Some("album".into()),
|
||||
issue: rules::Issue::AlbumLoop,
|
||||
message: format!("its album filter (album {}) would make this album include itself", a.0),
|
||||
})
|
||||
}
|
||||
|
||||
/// The photos of an album: the stored list, or a smart album's current matches (id order).
|
||||
pub fn album_photos(&self, id: AlbumId) -> Vec<PhotoId> {
|
||||
match self.albums.get(&id) {
|
||||
@@ -326,6 +437,18 @@ impl Catalog {
|
||||
pub fn label_name(&self, l: ColorLabel) -> String {
|
||||
self.label_names.get(&l).cloned().unwrap_or_else(|| format!("{l:?}"))
|
||||
}
|
||||
/// A listed keyword's attributes (any case), `None` when it isn't listed.
|
||||
pub fn keyword_info(&self, path: &str) -> Option<&keywords::KeywordInfo> {
|
||||
self.keyword_list.get(&keywords::clean(path).to_lowercase()).map(|k| &k.info)
|
||||
}
|
||||
/// How the keyword list spells a listed keyword (any case), `None` when it isn't listed.
|
||||
pub fn listed_path(&self, path: &str) -> Option<&str> {
|
||||
self.keyword_list.get(&keywords::clean(path).to_lowercase()).map(|k| k.path.as_str())
|
||||
}
|
||||
/// The keyword list: keywords listed on their own or given attributes, by path.
|
||||
pub fn listed_keywords(&self) -> impl Iterator<Item = &keywords::ListedKeyword> {
|
||||
self.keyword_list.values()
|
||||
}
|
||||
/// The custom name of a colour label, if any.
|
||||
pub fn custom_label_name(&self, l: ColorLabel) -> Option<&str> {
|
||||
self.label_names.get(&l).map(String::as_str)
|
||||
@@ -333,7 +456,9 @@ impl Catalog {
|
||||
/// The label a name stands for: a custom name first, then a colour's own name (any case).
|
||||
pub fn label_from_name(&self, name: &str) -> Option<ColorLabel> {
|
||||
let name = name.trim();
|
||||
self.label_names.iter().find(|(_, n)| n.trim().eq_ignore_ascii_case(name)).map(|(l, _)| *l).or_else(|| ColorLabel::parse(name))
|
||||
// any case, accents included (Été / été), as rules compare names
|
||||
let name_lower = name.to_lowercase();
|
||||
self.label_names.iter().find(|(_, n)| n.trim().to_lowercase() == name_lower).map(|(l, _)| *l).or_else(|| ColorLabel::parse(name))
|
||||
}
|
||||
|
||||
// ---- writes
|
||||
@@ -466,7 +591,20 @@ impl Catalog {
|
||||
}
|
||||
}
|
||||
let a = self.album_mut(id)?;
|
||||
Op::MoveAlbum { id, parent: std::mem::replace(&mut a.parent, parent) }
|
||||
if a.parent == parent {
|
||||
return Ok(Op::MoveAlbum { id, parent });
|
||||
}
|
||||
// its place belonged to the old neighbours: the undo gives it back with the folder
|
||||
let old_order = a.order.take();
|
||||
let old_parent = std::mem::replace(&mut a.parent, parent);
|
||||
match old_order {
|
||||
Some(_) => Op::Batch { ops: vec![Op::MoveAlbum { id, parent: old_parent }, Op::SetAlbumOrder { id, order: old_order }] },
|
||||
None => Op::MoveAlbum { id, parent: old_parent },
|
||||
}
|
||||
}
|
||||
Op::SetAlbumOrder { id, order } => {
|
||||
let a = self.album_mut(id)?;
|
||||
Op::SetAlbumOrder { id, order: std::mem::replace(&mut a.order, order) }
|
||||
}
|
||||
Op::SetAlbumPhotos { id, photos } => {
|
||||
let a = self.album_mut(id)?;
|
||||
@@ -543,6 +681,10 @@ impl Catalog {
|
||||
preview_only: std::mem::replace(&mut p.preview_only, preview_only),
|
||||
}
|
||||
}
|
||||
Op::SetEmbeddedLens { id, lens } => {
|
||||
let p = self.photo_mut(id)?;
|
||||
Op::SetEmbeddedLens { id, lens: std::mem::replace(&mut p.embedded_lens, lens.map(|l| *l)).map(Box::new) }
|
||||
}
|
||||
Op::SetFile { id, file_name, source } => {
|
||||
if file_name.trim().is_empty() {
|
||||
return Err(CatalogError::Invalid("empty file name".into()));
|
||||
@@ -559,6 +701,21 @@ impl Catalog {
|
||||
};
|
||||
Op::SetLabelName { label, name: old }
|
||||
}
|
||||
Op::SetKeyword { path, info } => {
|
||||
let path = keywords::clean(&path);
|
||||
if path.is_empty() {
|
||||
return Err(CatalogError::Invalid("keyword names can't be empty".into()));
|
||||
}
|
||||
let key = path.to_lowercase();
|
||||
let old = match info {
|
||||
Some(info) => self.keyword_list.insert(key, keywords::ListedKeyword { path: path.clone(), info }),
|
||||
None => self.keyword_list.remove(&key),
|
||||
};
|
||||
match old {
|
||||
Some(old) => Op::SetKeyword { path: old.path, info: Some(old.info) },
|
||||
None => Op::SetKeyword { path, info: None },
|
||||
}
|
||||
}
|
||||
Op::SetBrowsed { folder, at } => {
|
||||
let folder = crate::query::folder_key(&folder);
|
||||
let old = match at {
|
||||
@@ -567,6 +724,14 @@ impl Catalog {
|
||||
};
|
||||
Op::SetBrowsed { folder, at: old }
|
||||
}
|
||||
Op::SetFolderRecord { folder, record } => {
|
||||
let key = folders::record_key(&folder).ok_or_else(|| CatalogError::Invalid(format!("not a folder: {folder}")))?;
|
||||
let old = match record.filter(|r| !r.is_empty()) {
|
||||
Some(r) => self.folder_records.insert(key.clone(), r),
|
||||
None => self.folder_records.remove(&key),
|
||||
};
|
||||
Op::SetFolderRecord { folder: key, record: old }
|
||||
}
|
||||
Op::Batch { ops } => {
|
||||
let mut inverses = Vec::with_capacity(ops.len());
|
||||
for op in ops {
|
||||
@@ -601,6 +766,22 @@ impl Catalog {
|
||||
Op::Batch { ops }
|
||||
}
|
||||
|
||||
/// [`Self::delete_permanently_ops`] for several photos at once. One op list built against the
|
||||
/// catalog as it is now: albums lose all of them in one write each, and stacks are shortened
|
||||
/// (or dissolved) once, however many of their photos go.
|
||||
pub fn delete_photos_permanently_ops(&self, ids: &[PhotoId]) -> Op {
|
||||
let gone: std::collections::HashSet<PhotoId> = ids.iter().copied().collect();
|
||||
let mut ops: Vec<Op> = self
|
||||
.albums
|
||||
.values()
|
||||
.filter(|a| a.photos.iter().any(|p| gone.contains(p)))
|
||||
.map(|a| Op::SetAlbumPhotos { id: a.id, photos: a.photos.iter().copied().filter(|p| !gone.contains(p)).collect() })
|
||||
.collect();
|
||||
ops.extend(self.remove_from_stacks_ops(ids));
|
||||
ops.extend(gone.iter().map(|id| Op::RemovePhoto { id: *id }));
|
||||
Op::Batch { ops }
|
||||
}
|
||||
|
||||
// ---- persistence
|
||||
|
||||
/// Full snapshot as JSON.
|
||||
@@ -639,13 +820,29 @@ impl Catalog {
|
||||
}
|
||||
}
|
||||
|
||||
/// How siblings are listed: folders first, then the ones with a place of their own by it, then by
|
||||
/// name; the id settles any tie.
|
||||
fn album_order_key(a: &Album) -> (bool, bool, u32, String, AlbumId) {
|
||||
(!a.folder, a.order.is_none(), a.order.unwrap_or(0), a.name.to_lowercase(), a.id)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
#[cfg(test)]
|
||||
mod tests_album_order;
|
||||
#[cfg(test)]
|
||||
mod tests_background;
|
||||
#[cfg(test)]
|
||||
mod tests_folder_records;
|
||||
#[cfg(test)]
|
||||
mod tests_folders;
|
||||
#[cfg(test)]
|
||||
mod tests_format_version;
|
||||
#[cfg(test)]
|
||||
mod tests_journal;
|
||||
#[cfg(test)]
|
||||
mod tests_local;
|
||||
#[cfg(test)]
|
||||
mod tests_lock;
|
||||
#[cfg(test)]
|
||||
mod tests_torn_append;
|
||||
@@ -283,7 +283,8 @@ impl Op {
|
||||
| Op::SetAnalysis { id, .. }
|
||||
| Op::SetFile { id, .. }
|
||||
| Op::Relink { id, .. }
|
||||
| Op::SetContent { id, .. } => out(*id),
|
||||
| Op::SetContent { id, .. }
|
||||
| Op::SetEmbeddedLens { id, .. } => out(*id),
|
||||
Op::AddAlbum { album } => {
|
||||
album.photos.iter().for_each(|p| out(*p));
|
||||
album.cover.into_iter().for_each(&mut *out);
|
||||
@@ -295,10 +296,13 @@ impl Op {
|
||||
Op::RemoveAlbum { .. }
|
||||
| Op::RenameAlbum { .. }
|
||||
| Op::MoveAlbum { .. }
|
||||
| Op::SetAlbumOrder { .. }
|
||||
| Op::SetAlbumRules { .. }
|
||||
| Op::RemoveStack { .. }
|
||||
| Op::SetLabelName { .. }
|
||||
| Op::SetBrowsed { .. } => {}
|
||||
| Op::SetKeyword { .. }
|
||||
| Op::SetBrowsed { .. }
|
||||
| Op::SetFolderRecord { .. } => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
//! One process per library: an exclusive OS lock on `<library>/catalog.lock`, held for as long as
|
||||
//! the library is open (issue #99).
|
||||
//!
|
||||
//! Two processes on the same library (the app and `lightcraft-cli --library`, two app instances,
|
||||
//! two computers on a shared folder) would each keep their own catalog and `seq` counter, and
|
||||
//! whichever wrote the last snapshot would silently drop the other's edits. So opening a library
|
||||
//! takes the lock first, and a second opener gets [`LockError::InUse`] instead.
|
||||
//!
|
||||
//! - The lock is the OS's (`flock` on Unix, `LockFileEx` on Windows, via [`std::fs::File::try_lock`]):
|
||||
//! it is released when the [`LibraryLock`] is dropped and, after a crash or a kill, by the OS when
|
||||
//! the process ends. A `catalog.lock` file left behind is therefore never "stale": only a lock
|
||||
//! that's actually held refuses the open.
|
||||
//! - Who holds it (process id, computer, program, version) is written to `catalog.lock.owner`, for
|
||||
//! the error message. It's a separate file because a Windows lock also blocks reading the
|
||||
//! locked file. It's only a hint: it may be missing or left over from a crash.
|
||||
//! - File systems without lock support (some network shares): the library opens unlocked, with a
|
||||
//! warning — refusing it would lock the user out of their own library. Locks over SMB/NFS are
|
||||
//! advisory and depend on the server; the owner file is then the only hint.
|
||||
//! - Browser storage (OPFS) has no file locks: see the web host for its own guard.
|
||||
|
||||
use std::fs::{File, OpenOptions, TryLockError};
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// The lock file in the library directory.
|
||||
pub const LOCK: &str = "catalog.lock";
|
||||
/// Who holds the lock (JSON, see [`LockOwner`]).
|
||||
pub const OWNER: &str = "catalog.lock.owner";
|
||||
|
||||
/// Who has the library open (as written in [`OWNER`]).
|
||||
#[derive(Clone, Debug, Default, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct LockOwner {
|
||||
pub pid: u32,
|
||||
/// Computer name, if known.
|
||||
pub host: String,
|
||||
/// The program (`LightCraft`, `lightcraft-cli`…) and its version.
|
||||
pub program: String,
|
||||
pub version: String,
|
||||
/// When it opened the library (seconds since 1970).
|
||||
pub since: u64,
|
||||
}
|
||||
|
||||
impl LockOwner {
|
||||
/// This process, as `program`.
|
||||
pub fn this_process(program: &str) -> LockOwner {
|
||||
LockOwner {
|
||||
pid: std::process::id(),
|
||||
host: host_name(),
|
||||
program: program.to_string(),
|
||||
version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
since: web_time::SystemTime::now().duration_since(web_time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// "LightCraft 0.3.0 (process 123 on studio-mac)".
|
||||
pub fn describe(&self) -> String {
|
||||
let program = if self.program.is_empty() { "another LightCraft" } else { self.program.as_str() };
|
||||
let version = if self.version.is_empty() { String::new() } else { format!(" {}", self.version) };
|
||||
let host = if self.host.is_empty() {
|
||||
String::new()
|
||||
} else if self.host == host_name() {
|
||||
" on this computer".to_string()
|
||||
} else {
|
||||
format!(" on {}", self.host)
|
||||
};
|
||||
format!("{program}{version} (process {}{host})", self.pid)
|
||||
}
|
||||
}
|
||||
|
||||
/// The computer's name, best effort (empty if unknown).
|
||||
fn host_name() -> String {
|
||||
let env = ["COMPUTERNAME", "HOSTNAME"].iter().find_map(|k| std::env::var(k).ok().filter(|h| !h.trim().is_empty()));
|
||||
env.or_else(|| std::fs::read_to_string("/etc/hostname").ok()).map(|h| h.trim().to_string()).filter(|h| !h.is_empty()).unwrap_or_default()
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum LockError {
|
||||
/// Another process has the library open.
|
||||
#[error("This library is already open in {}. Close it there first.", .0.as_ref().map(LockOwner::describe).unwrap_or_else(|| "another LightCraft program".into()))]
|
||||
InUse(Option<LockOwner>),
|
||||
}
|
||||
|
||||
/// The exclusive lock on a library directory; released on drop (and by the OS if the process
|
||||
/// dies).
|
||||
#[derive(Debug)]
|
||||
pub struct LibraryLock {
|
||||
dir: PathBuf,
|
||||
/// `None` when the file system can't lock (the library is open unlocked).
|
||||
_file: Option<File>,
|
||||
held: bool,
|
||||
}
|
||||
|
||||
impl LibraryLock {
|
||||
/// Lock `dir` (which must exist) for this process. Fails only when another process holds it.
|
||||
pub fn acquire(dir: &Path, program: &str) -> Result<LibraryLock, LockError> {
|
||||
let unlocked = |why: io::Error| {
|
||||
log::warn!("library {}: can't lock {LOCK} ({why}); opening it without protection against a second program", dir.display());
|
||||
LibraryLock { dir: dir.to_path_buf(), _file: None, held: false }
|
||||
};
|
||||
let file = match OpenOptions::new().read(true).write(true).create(true).truncate(false).open(dir.join(LOCK)) {
|
||||
Ok(f) => f,
|
||||
Err(e) => return Ok(unlocked(e)),
|
||||
};
|
||||
match file.try_lock() {
|
||||
Ok(()) => {}
|
||||
Err(TryLockError::WouldBlock) => {
|
||||
let owner = std::fs::read(dir.join(OWNER)).ok().and_then(|b| serde_json::from_slice::<LockOwner>(&b).ok());
|
||||
return Err(LockError::InUse(owner));
|
||||
}
|
||||
Err(TryLockError::Error(e)) => return Ok(unlocked(e)),
|
||||
}
|
||||
let owner = LockOwner::this_process(program);
|
||||
if let Ok(json) = serde_json::to_vec_pretty(&owner)
|
||||
&& let Err(e) = std::fs::write(dir.join(OWNER), json)
|
||||
{
|
||||
log::warn!("library {}: can't write {OWNER}: {e}", dir.display());
|
||||
}
|
||||
Ok(LibraryLock { dir: dir.to_path_buf(), _file: Some(file), held: true })
|
||||
}
|
||||
|
||||
/// The OS lock is held (`false`: the file system can't lock; opened unprotected).
|
||||
pub fn held(&self) -> bool {
|
||||
self.held
|
||||
}
|
||||
|
||||
pub fn dir(&self) -> &Path {
|
||||
&self.dir
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for LibraryLock {
|
||||
/// Remove the owner note while still holding the lock (so it can't delete the next holder's),
|
||||
/// then the file handle closes and the OS releases the lock.
|
||||
fn drop(&mut self) {
|
||||
if self.held {
|
||||
let _ = std::fs::remove_file(self.dir.join(OWNER));
|
||||
}
|
||||
}
|
||||
}
|
||||
+154
-7
@@ -2,7 +2,7 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use lightcraft_develop::DevelopSettings;
|
||||
use lightcraft_develop::{DevelopSettings, WbMode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
@@ -152,6 +152,17 @@ impl CopyrightStatus {
|
||||
}
|
||||
}
|
||||
|
||||
/// A shutter speed as cameras write it (`1/250`, `0.5`, `2"`, `30s`) in seconds; `None` for
|
||||
/// anything that isn't a positive, finite time.
|
||||
pub fn parse_shutter_seconds(s: &str) -> Option<f64> {
|
||||
let s = s.trim().trim_end_matches(['s', '"']).trim();
|
||||
match s.split_once('/') {
|
||||
Some((n, d)) => Some(n.trim().parse::<f64>().ok()? / d.trim().parse::<f64>().ok()?),
|
||||
None => s.parse().ok(),
|
||||
}
|
||||
.filter(|v: &f64| v.is_finite() && *v > 0.0)
|
||||
}
|
||||
|
||||
/// Descriptive + capture metadata.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
@@ -183,6 +194,11 @@ pub struct Meta {
|
||||
pub copyright_url: String,
|
||||
pub creator: String,
|
||||
pub keywords: Vec<String>,
|
||||
/// Face/pet/focus regions read from XMP (MWG-RS), on the upright (EXIF-oriented) photo.
|
||||
/// Removing or resizing one edits the catalog only; LightCraft never writes regions to XMP.
|
||||
/// Left out of the catalog JSON when empty (most photos), so older catalogs read unchanged.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub regions: Vec<lightcraft_meta::Region>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
@@ -291,6 +307,14 @@ pub struct Analysis {
|
||||
pub best: bool,
|
||||
}
|
||||
|
||||
/// Raw formats whose readers have vendor white-balance multipliers but no measured camera
|
||||
/// illuminant: their white balance is developed relative to the as-shot look (6500 K / 0 means
|
||||
/// as shot), as for rendered photographs. `format` is the file's extension or Lightroom's
|
||||
/// `fileFormat` name, in any case. See [`Photo::relative_wb`].
|
||||
pub fn relative_wb_format(format: &str) -> bool {
|
||||
["ARW", "NEF", "NRW", "RW2", "RWL", "RAW", "RAF", "CR3", "CR2", "PEF", "SRW", "ORF"].iter().any(|f| format.eq_ignore_ascii_case(f))
|
||||
}
|
||||
|
||||
impl Photo {
|
||||
pub fn new(id: PhotoId, source: Source, file_name: &str, format: &str, width: u32, height: u32, imported: &str) -> Photo {
|
||||
Photo {
|
||||
@@ -331,33 +355,91 @@ impl Photo {
|
||||
pub fn develops_raw(&self) -> bool {
|
||||
self.kind == MediaKind::Raw && self.preview_only.is_none()
|
||||
}
|
||||
/// The current ARW, NEF, RW2, RAF, CR3, CR2, PEF, SRW and ORF readers have vendor WB multipliers but no measured camera
|
||||
/// illuminant. Use adjustments relative to the camera's as-shot look, as for rendered
|
||||
/// photographs (the engine's `camera_preview::file_local_look` covers the same formats; RWL and
|
||||
/// RAW are Leica's and the oldest Panasonic bodies' names for RW2 files).
|
||||
pub fn relative_wb(&self) -> bool {
|
||||
self.develops_raw() && relative_wb_format(&self.format)
|
||||
}
|
||||
/// The develop settings import gave this photo: [`Photo::camera_defaults`], or the user's
|
||||
/// default preset applied on top of them ([`Photo::import_look`]).
|
||||
/// default preset applied on top of them ([`Photo::import_look`]). Like the camera defaults,
|
||||
/// in the photo's own rendering process.
|
||||
pub fn import_defaults(&self) -> DevelopSettings {
|
||||
match &self.import_look {
|
||||
Some(l) => (**l).clone(),
|
||||
Some(l) => DevelopSettings { process: self.develop.process, ..(**l).clone() },
|
||||
None => self.camera_defaults(),
|
||||
}
|
||||
}
|
||||
/// The built-in defaults for this photo, before any user default preset: raws start from
|
||||
/// their as-shot white balance; embedded lens corrections on when the file has them.
|
||||
/// their as-shot white balance; embedded lens corrections on when the file has them. They
|
||||
/// carry the photo's own rendering process ([`lightcraft_develop::ProcessVersion`]): which
|
||||
/// process a photo renders with is not an edit, so comparisons with its defaults (edited?
|
||||
/// still as imported?) don't change when a newer process exists. A new photo has the latest.
|
||||
pub fn camera_defaults(&self) -> DevelopSettings {
|
||||
let mut d = match self.as_shot_wb {
|
||||
let wb = if self.relative_wb() { Some((6500.0, 0.0)) } else { self.as_shot_wb };
|
||||
let mut d = match wb {
|
||||
Some((t, tint)) => DevelopSettings::for_raw(t, tint),
|
||||
None => DevelopSettings::default(),
|
||||
};
|
||||
if self.embedded_lens.is_some() {
|
||||
d.optics.lens_profile = true;
|
||||
}
|
||||
d.process = self.develop.process;
|
||||
d
|
||||
}
|
||||
/// The file name's extension without the dot, as written (`CR2` for `IMG_0042.CR2`); empty
|
||||
/// when the name has none (`README`, a dot file such as `.hidden`).
|
||||
pub fn extension(&self) -> &str {
|
||||
self.file_name.rsplit_once('.').filter(|(stem, _)| !stem.is_empty()).map_or("", |(_, ext)| ext)
|
||||
}
|
||||
/// Width and height in pixels as the photo is shown: turned by its orientation and cut by its
|
||||
/// crop. Straightening and Constrain Crop shrink the crop without changing its shape, so the
|
||||
/// aspect is exact and the edges can read a little larger than an export.
|
||||
pub fn shown_size(&self) -> (f64, f64) {
|
||||
let (w, h) = (self.width as f64, self.height as f64);
|
||||
let (w, h) = if self.develop.orientation.swaps_axes() { (h, w) } else { (w, h) };
|
||||
let r = self.develop.crop.geometry.rect;
|
||||
// a crop read from a file or an agent is hostile: no NaN, nothing outside the frame
|
||||
let part = |len: f64| if len.is_finite() { len.clamp(0.0, 1.0) } else { 1.0 };
|
||||
(w * part(r.width()), h * part(r.height()))
|
||||
}
|
||||
/// Cropped or straightened.
|
||||
pub fn is_cropped(&self) -> bool {
|
||||
!self.develop.crop.geometry.is_identity()
|
||||
}
|
||||
/// How many keywords the photo has: a hierarchical keyword (`travel|italy|rome`) is one, and
|
||||
/// the same keyword in different case or with stray spaces counts once; blank ones don't count.
|
||||
pub fn keyword_count(&self) -> usize {
|
||||
let mut seen: Vec<String> = self.meta.keywords.iter().map(|k| k.trim().to_lowercase()).filter(|k| !k.is_empty()).collect();
|
||||
seen.sort_unstable();
|
||||
seen.dedup();
|
||||
seen.len()
|
||||
}
|
||||
/// The people in the photo: the names on its face regions, trimmed, each once (names that
|
||||
/// differ only in case are one person, as first seen). Pets and unnamed faces are not people.
|
||||
pub fn people(&self) -> Vec<&str> {
|
||||
let mut out: Vec<&str> = Vec::new();
|
||||
for r in self.meta.regions.iter().filter(|r| r.kind == lightcraft_meta::RegionKind::Face) {
|
||||
let Some(name) = r.name.as_deref().map(str::trim).filter(|n| !n.is_empty()) else { continue };
|
||||
if !out.iter().any(|o| o.to_lowercase() == name.to_lowercase()) {
|
||||
out.push(name);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
/// In the library: not deleted and not only browsed (Local).
|
||||
pub fn in_library(&self) -> bool {
|
||||
!self.deleted && !self.local
|
||||
}
|
||||
/// Edited by the user: settings differ from what import gave the photo.
|
||||
pub fn is_edited(&self) -> bool {
|
||||
*self.develop != self.import_defaults() && !self.develop.is_unedited()
|
||||
let mut defaults = self.import_defaults();
|
||||
// As Shot reads the source's current WB; stored numbers can predate a decoder update.
|
||||
if self.develop.wb.mode == WbMode::AsShot && defaults.wb.mode == WbMode::AsShot {
|
||||
defaults.wb = self.develop.wb;
|
||||
}
|
||||
*self.develop != defaults && !self.develop.is_unedited()
|
||||
}
|
||||
/// Capture time if known, else import time (sort key).
|
||||
pub fn date(&self) -> &str {
|
||||
@@ -386,12 +468,16 @@ pub struct Album {
|
||||
/// The Quick Collection: a regular album that B adds to (one per library).
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub quick: bool,
|
||||
/// Its place among the albums of the same kind in its folder, once the user ordered them by
|
||||
/// hand (see [`crate::Catalog::album_children`]); `None`: listed by name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub order: Option<u32>,
|
||||
}
|
||||
|
||||
impl Album {
|
||||
/// A regular (manual) album.
|
||||
pub fn new(id: AlbumId, name: impl Into<String>) -> Album {
|
||||
Album { id, name: name.into(), parent: None, folder: false, photos: Vec::new(), cover: None, smart: None, quick: false }
|
||||
Album { id, name: name.into(), parent: None, folder: false, photos: Vec::new(), cover: None, smart: None, quick: false, order: None }
|
||||
}
|
||||
pub fn is_smart(&self) -> bool {
|
||||
self.smart.is_some()
|
||||
@@ -415,4 +501,65 @@ mod edited_tests {
|
||||
p.develop = Arc::new(DevelopSettings::default());
|
||||
assert!(!p.is_edited(), "a full reset is unedited too");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_process_a_photo_renders_with_is_not_an_edit() {
|
||||
use lightcraft_develop::ProcessVersion;
|
||||
// a photo on another process than the latest (a V1 photo once a later process is the
|
||||
// latest, or a photo saved by a newer LightCraft)
|
||||
let other = ProcessVersion(ProcessVersion::LATEST.0 + 1);
|
||||
let mut p = Photo::new(PhotoId(1), Source::Demo { scene: 0 }, "a.dng", "DNG", 10, 10, "2026-10-01T00:00:00");
|
||||
p.kind = MediaKind::Raw;
|
||||
p.as_shot_wb = Some((5200.0, 4.0));
|
||||
assert_eq!(p.camera_defaults().process, ProcessVersion::LATEST, "a new photo gets the latest");
|
||||
p.develop = Arc::new(DevelopSettings { process: other, ..p.import_defaults() });
|
||||
assert_eq!((p.camera_defaults().process, p.import_defaults().process), (other, other));
|
||||
assert!(!p.is_edited(), "still as imported");
|
||||
// with a default preset's look as well
|
||||
let mut look = DevelopSettings::for_raw(5200.0, 4.0);
|
||||
look.light.exposure = 0.3;
|
||||
p.import_look = Some(Arc::new(look.clone()));
|
||||
p.develop = Arc::new(DevelopSettings { process: other, ..look });
|
||||
assert_eq!(p.import_defaults().process, other);
|
||||
assert!(!p.is_edited());
|
||||
let mut d = (*p.develop).clone();
|
||||
d.light.exposure = 1.0;
|
||||
p.develop = Arc::new(d);
|
||||
assert!(p.is_edited(), "an edit still is one");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn supported_raws_use_relative_white_balance() {
|
||||
for (name, format, relative) in [
|
||||
("a.arw", "ARW", true),
|
||||
("a.nef", "NEF", true),
|
||||
("a.nrw", "nrw", true),
|
||||
("a.rw2", "RW2", true),
|
||||
("a.rwl", "RWL", true),
|
||||
("a.raw", "RAW", true),
|
||||
("a.raf", "rAf", true),
|
||||
("a.cr3", "cr3", true),
|
||||
("a.srw", "SRW", true),
|
||||
("a.cr2", "CR2", true),
|
||||
("a.pef", "pef", true),
|
||||
("a.orf", "ORF", true),
|
||||
("a.dng", "DNG", false),
|
||||
] {
|
||||
let mut p = Photo::new(PhotoId(1), Source::Demo { scene: 0 }, name, format, 10, 10, "2026-10-01T00:00:00");
|
||||
p.kind = MediaKind::Raw;
|
||||
p.as_shot_wb = Some((5200.0, 4.0));
|
||||
assert_eq!(p.relative_wb(), relative, "{format}");
|
||||
let wb = p.camera_defaults().wb;
|
||||
assert_eq!((wb.temp, wb.tint), if relative { (6500.0, 0.0) } else { (5200.0, 4.0) }, "{format}");
|
||||
p.develop = Arc::new(DevelopSettings::for_raw(5200.0, 4.0));
|
||||
assert!(!p.is_edited(), "{format}: old As Shot values are not an edit");
|
||||
let mut custom = (*p.develop).clone();
|
||||
custom.wb.mode = WbMode::Custom;
|
||||
p.develop = Arc::new(custom);
|
||||
assert!(p.is_edited(), "{format}: Custom WB is still an edit");
|
||||
// shown from the embedded preview: a rendered image, not a relative-WB raw
|
||||
p.preview_only = Some("unsupported".into());
|
||||
assert!(!p.relative_wb());
|
||||
}
|
||||
}
|
||||
}
|
||||
+223
-29
@@ -37,8 +37,14 @@ pub struct Filter {
|
||||
pub deleted: bool,
|
||||
/// Capture date prefix (`2026`, `2026-04`, `2026-04-12`).
|
||||
pub date: Option<String>,
|
||||
/// A folder the library's photos were imported from: those photos and the ones imported from
|
||||
/// the folders inside it (see [`crate::folders`]). Photos only browsed in Local are never part
|
||||
/// of it, unlike [`Filter::folder`].
|
||||
pub library_folder: Option<String>,
|
||||
/// A keyword; hierarchical keywords match their children too (`travel` finds `travel|italy`).
|
||||
pub keyword: Option<String>,
|
||||
/// A person: photos with a named face region of this name (case-insensitive), as read from XMP.
|
||||
pub person: Option<String>,
|
||||
pub camera: Option<String>,
|
||||
/// Lens (case-insensitive substring).
|
||||
pub lens: Option<String>,
|
||||
@@ -70,6 +76,9 @@ pub enum SortKey {
|
||||
FileName,
|
||||
Rating,
|
||||
FileSize,
|
||||
/// A shuffle fixed by [`Sort::seed`]: the same seed always gives the same order, and photos
|
||||
/// added or edited later never reshuffle the others. Pick a new seed to reshuffle.
|
||||
Random,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -79,14 +88,30 @@ pub struct Sort {
|
||||
pub ascending: bool,
|
||||
/// Date headers in the grid (date sort keys only).
|
||||
pub group: crate::GroupBy,
|
||||
/// Which shuffle [`SortKey::Random`] gives (ignored by the other keys).
|
||||
pub seed: u64,
|
||||
}
|
||||
|
||||
impl Default for Sort {
|
||||
fn default() -> Self {
|
||||
Sort { key: SortKey::CaptureDate, ascending: false, group: crate::GroupBy::Auto }
|
||||
Sort { key: SortKey::CaptureDate, ascending: false, group: crate::GroupBy::Auto, seed: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
/// splitmix64 finaliser: a stateless, platform-independent mix (no RNG state, no `rand` version
|
||||
/// to drift), so a seed names the same shuffle on every machine.
|
||||
pub fn mix64(x: u64) -> u64 {
|
||||
let mut z = x.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
|
||||
/// A photo's place in the shuffle for `seed`.
|
||||
fn shuffle_rank(seed: u64, id: PhotoId) -> u64 {
|
||||
mix64(mix64(seed) ^ id.0)
|
||||
}
|
||||
|
||||
fn token_matches(p: &Photo, tok: &str) -> bool {
|
||||
let t = tok.to_lowercase();
|
||||
if let Some((field, val)) = t.split_once(':') {
|
||||
@@ -113,9 +138,10 @@ fn token_matches(p: &Photo, tok: &str) -> bool {
|
||||
"camera" => p.meta.camera.to_lowercase().contains(val),
|
||||
"lens" => p.meta.lens.to_lowercase().contains(val),
|
||||
"keyword" | "kw" => p.meta.keywords.iter().any(|k| crate::keywords::is_under(k, val)),
|
||||
"person" | "who" => has_person(p, val),
|
||||
"type" | "kind" => format!("{:?}", p.kind).eq_ignore_ascii_case(val),
|
||||
"edited" => (val == "true" || val == "yes") == p.is_edited(),
|
||||
"date" => p.date().starts_with(val),
|
||||
"date" => p.captured.as_deref().is_some_and(|c| c.starts_with(val)),
|
||||
"copy" | "virtual" => (val == "true" || val == "yes") == p.copy_of.is_some(),
|
||||
"name" | "file" => p.file_name.to_lowercase().contains(val),
|
||||
_ => false,
|
||||
@@ -125,9 +151,24 @@ fn token_matches(p: &Photo, tok: &str) -> bool {
|
||||
hay.iter().any(|h| h.to_lowercase().contains(&t)) || p.meta.keywords.iter().any(|k| k.to_lowercase().contains(&t))
|
||||
}
|
||||
|
||||
/// Whether `p` has a named face region called `name` (case-insensitive).
|
||||
fn has_person(p: &Photo, name: &str) -> bool {
|
||||
let name = name.trim().to_lowercase();
|
||||
p.people().iter().any(|n| n.to_lowercase() == name)
|
||||
}
|
||||
|
||||
impl Filter {
|
||||
/// Human-readable summary of the active rules (smart album tooltips, `album.list`).
|
||||
pub fn describe(&self) -> String {
|
||||
self.describe_in(None)
|
||||
}
|
||||
|
||||
/// [`Filter::describe`] naming the albums its rules test (`RuleSet::describe_with`).
|
||||
pub fn describe_with(&self, cat: &crate::Catalog) -> String {
|
||||
self.describe_in(Some(cat))
|
||||
}
|
||||
|
||||
fn describe_in(&self, cat: Option<&crate::Catalog>) -> String {
|
||||
let mut v: Vec<String> = Vec::new();
|
||||
if self.rating > 0 {
|
||||
let op = match self.rating_op {
|
||||
@@ -152,9 +193,15 @@ impl Filter {
|
||||
if let Some(e) = self.edited {
|
||||
v.push(if e { "edited".into() } else { "unedited".into() });
|
||||
}
|
||||
for (name, val) in
|
||||
[("keyword", &self.keyword), ("camera", &self.camera), ("lens", &self.lens), ("date", &self.date), ("imported", &self.imported)]
|
||||
{
|
||||
for (name, val) in [
|
||||
("keyword", &self.keyword),
|
||||
("person", &self.person),
|
||||
("camera", &self.camera),
|
||||
("lens", &self.lens),
|
||||
("date", &self.date),
|
||||
("folder", &self.library_folder),
|
||||
("imported", &self.imported),
|
||||
] {
|
||||
if let Some(x) = val {
|
||||
v.push(format!("{name} {x}"));
|
||||
}
|
||||
@@ -169,17 +216,37 @@ impl Filter {
|
||||
v.push(format!("“{}”", self.text.trim()));
|
||||
}
|
||||
if let Some(rs) = self.rule_set.as_ref().filter(|r| !r.rules.is_empty()) {
|
||||
v.push(rs.describe());
|
||||
v.push(cat.map_or_else(|| rs.describe(), |c| rs.describe_with(c)));
|
||||
}
|
||||
if v.is_empty() { "all photos".into() } else { v.join(", ") }
|
||||
}
|
||||
|
||||
/// The albums this filter tests: its album field and its rules' Album rules.
|
||||
pub fn albums_tested(&self) -> Vec<crate::AlbumId> {
|
||||
let mut out: Vec<crate::AlbumId> = self.album.into_iter().collect();
|
||||
out.extend(self.rule_set.as_ref().map(crate::RuleSet::albums_tested).unwrap_or_default());
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether matches depend on the clock (only "in the last…" rules do).
|
||||
pub fn depends_on_now(&self) -> bool {
|
||||
self.rule_set.as_ref().is_some_and(crate::RuleSet::depends_on_now)
|
||||
}
|
||||
|
||||
pub fn matches(&self, p: &Photo, cat: &Catalog) -> bool {
|
||||
self.matches_in(p, cat, self.library_root().as_deref())
|
||||
}
|
||||
|
||||
/// [`Filter::library_folder`] as a folder identity ([`folder_key`]); `None` without a choice.
|
||||
fn library_root(&self) -> Option<String> {
|
||||
// as given: a folder may be named with spaces at its edge, and trimming would make it
|
||||
// another folder
|
||||
self.library_folder.as_deref().filter(|d| !d.trim().is_empty()).map(folder_key)
|
||||
}
|
||||
|
||||
/// [`Filter::matches`] with the library folder already turned into its identity, so a query
|
||||
/// over a big library does that once, not once per photo.
|
||||
fn matches_in(&self, p: &Photo, cat: &Catalog, root: Option<&str>) -> bool {
|
||||
if p.deleted != self.deleted {
|
||||
return false;
|
||||
}
|
||||
@@ -233,12 +300,18 @@ impl Filter {
|
||||
return false;
|
||||
}
|
||||
if let Some(from) = &self.date_from
|
||||
&& p.date() < from.as_str()
|
||||
&& match p.captured.as_deref() {
|
||||
Some(c) => c < from.as_str(),
|
||||
None => true,
|
||||
}
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if let Some(to) = &self.date_to
|
||||
&& p.date().get(..to.len()).unwrap_or(p.date()) > to.as_str()
|
||||
&& match p.captured.as_deref() {
|
||||
Some(c) => c.get(..to.len()).unwrap_or(c) > to.as_str(),
|
||||
None => true,
|
||||
}
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -248,7 +321,12 @@ impl Filter {
|
||||
return false;
|
||||
}
|
||||
if let Some(d) = &self.date
|
||||
&& !p.date().starts_with(d.as_str())
|
||||
&& !p.captured.as_deref().is_some_and(|c| c.starts_with(d.as_str()))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if let Some(root) = root
|
||||
&& !photo_in_root(p, root)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -267,6 +345,11 @@ impl Filter {
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if let Some(n) = &self.person
|
||||
&& !has_person(p, n)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if let Some(c) = &self.camera
|
||||
&& !p.meta.camera.eq_ignore_ascii_case(c)
|
||||
{
|
||||
@@ -279,15 +362,17 @@ impl Filter {
|
||||
impl Catalog {
|
||||
/// Photos matching `filter`, in `sort` order (ties broken by id for stability).
|
||||
pub fn query(&self, filter: &Filter, sort: &Sort) -> Vec<PhotoId> {
|
||||
let mut v: Vec<&Photo> = self.photos().map(|p| p.as_ref()).filter(|p| filter.matches(p, self)).collect();
|
||||
let root = filter.library_root();
|
||||
let mut v: Vec<&Photo> = self.photos().map(|p| p.as_ref()).filter(|p| filter.matches_in(p, self, root.as_deref())).collect();
|
||||
v.sort_by(|a, b| {
|
||||
let o = match sort.key {
|
||||
SortKey::CaptureDate => a.date().cmp(b.date()),
|
||||
SortKey::CaptureDate => a.captured.cmp(&b.captured).then_with(|| a.imported.cmp(&b.imported)),
|
||||
SortKey::ImportDate => a.imported.cmp(&b.imported),
|
||||
SortKey::EditDate => a.edited.cmp(&b.edited),
|
||||
SortKey::FileName => a.file_name.to_lowercase().cmp(&b.file_name.to_lowercase()),
|
||||
SortKey::Rating => a.rating.cmp(&b.rating),
|
||||
SortKey::FileSize => a.file_size.cmp(&b.file_size),
|
||||
SortKey::Random => shuffle_rank(sort.seed, a.id).cmp(&shuffle_rank(sort.seed, b.id)),
|
||||
}
|
||||
.then(a.id.cmp(&b.id));
|
||||
if sort.ascending { o } else { o.reverse() }
|
||||
@@ -300,7 +385,7 @@ impl Catalog {
|
||||
use std::collections::BTreeMap;
|
||||
let mut years: BTreeMap<&str, (BTreeMap<&str, usize>, BTreeMap<&str, usize>)> = BTreeMap::new();
|
||||
for p in self.photos().filter(|p| p.in_library()) {
|
||||
let d = p.date();
|
||||
let Some(d) = p.captured.as_deref() else { continue };
|
||||
if d.len() >= 10 {
|
||||
let (months, days) = years.entry(&d[..4]).or_default();
|
||||
*months.entry(&d[..7]).or_default() += 1;
|
||||
@@ -330,6 +415,55 @@ impl Catalog {
|
||||
}
|
||||
m.into_iter().collect()
|
||||
}
|
||||
|
||||
/// The people named on faces in the library (MWG regions read from XMP): how many photos each
|
||||
/// appears in and the photo showing their largest face (for a card's picture). Most photos first,
|
||||
/// then by name. Names that differ only in case are one person, shown as first seen; a person
|
||||
/// twice in one photo counts once.
|
||||
pub fn people(&self) -> Vec<Person> {
|
||||
self.people_in(&Filter::default())
|
||||
}
|
||||
|
||||
/// [`Self::people`] among the photos `filter` lets through (its `person` is ignored): what the
|
||||
/// People view offers while other filters (a date, a rating, an album…) are active, so picking
|
||||
/// a person never ends in an empty grid.
|
||||
pub fn people_in(&self, filter: &Filter) -> Vec<Person> {
|
||||
let filter = Filter { person: None, ..filter.clone() };
|
||||
let mut m: std::collections::HashMap<String, (Person, f64)> = Default::default();
|
||||
let root = filter.library_root();
|
||||
for p in self.photos().filter(|p| filter.matches_in(p, self, root.as_deref())) {
|
||||
let mut seen: Vec<String> = Vec::new();
|
||||
for r in p.meta.regions.iter().filter(|r| r.kind == lightcraft_meta::RegionKind::Face) {
|
||||
let Some(name) = r.name.as_deref().map(str::trim).filter(|n| !n.is_empty()) else { continue };
|
||||
let key = name.to_lowercase();
|
||||
// the face's size in pixels of the photo
|
||||
let area = (r.rect.x1 - r.rect.x0) * (r.rect.y1 - r.rect.y0) * p.width as f64 * p.height as f64;
|
||||
let (person, best) =
|
||||
m.entry(key.clone()).or_insert_with(|| (Person { name: name.to_string(), count: 0, photo: p.id, face: r.rect }, area));
|
||||
if !seen.contains(&key) {
|
||||
person.count += 1;
|
||||
seen.push(key);
|
||||
}
|
||||
if area > *best || (area == *best && p.id < person.photo) {
|
||||
(person.photo, person.face, *best) = (p.id, r.rect, area);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut v: Vec<Person> = m.into_values().map(|(p, _)| p).collect();
|
||||
v.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.name.to_lowercase().cmp(&b.name.to_lowercase())));
|
||||
v
|
||||
}
|
||||
}
|
||||
|
||||
/// A person named on faces in the library ([`Catalog::people`]).
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Person {
|
||||
pub name: String,
|
||||
/// Photos they appear in.
|
||||
pub count: usize,
|
||||
/// The photo with their largest face, and that face (normalized, in the photo's upright frame).
|
||||
pub photo: PhotoId,
|
||||
pub face: lightcraft_meta::Rect,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
@@ -351,12 +485,18 @@ pub fn in_folder(path: &str, dir: &str, deep: bool) -> bool {
|
||||
!rest.is_empty() && (deep || !rest.contains(['/', '\\']))
|
||||
}
|
||||
|
||||
/// A folder path's identity, for telling whether two spellings name the same folder: `/` and
|
||||
/// `\\` are both separators, repeated separators, `.` and a trailing separator are dropped,
|
||||
/// `..` is resolved lexically, a Windows verbatim prefix (`\\?\`) is removed and the drive
|
||||
/// letter lower-cased; on Windows (case-insensitive file names) the whole path is lower-cased.
|
||||
/// No file-system access: the paths compared should already be absolute.
|
||||
pub fn folder_key(path: &str) -> String {
|
||||
/// A path read the way [`folder_key`] reads it: the part `..` can't climb above (`//server/share`,
|
||||
/// `c:`; empty for `/` and relative paths), the names below it, and whether it is absolute.
|
||||
pub(crate) struct SplitPath {
|
||||
pub prefix: String,
|
||||
pub parts: Vec<String>,
|
||||
pub absolute: bool,
|
||||
}
|
||||
|
||||
/// Both separators, repeated separators (also between a server and its share), `.` and a
|
||||
/// trailing separator dropped, `..` resolved lexically, a Windows verbatim prefix removed;
|
||||
/// `lower_drive` lower-cases a drive letter (the identity does, a display name wants it as is).
|
||||
pub(crate) fn split_path(path: &str, lower_drive: bool) -> SplitPath {
|
||||
let mut s = path.replace('\\', "/");
|
||||
if let Some(rest) = s.strip_prefix("//?/").or_else(|| s.strip_prefix("//./")) {
|
||||
s = match rest.strip_prefix("UNC/") {
|
||||
@@ -364,28 +504,39 @@ pub fn folder_key(path: &str) -> String {
|
||||
None => rest.to_string(),
|
||||
};
|
||||
}
|
||||
// the part `..` can't climb above: `//server/share`, `c:` or `/`
|
||||
let (prefix, rest) = if let Some(unc) = s.strip_prefix("//").filter(|u| !u.is_empty() && !u.starts_with('/')) {
|
||||
let mut it = unc.splitn(3, '/');
|
||||
let (server, share, rest) = (it.next().unwrap_or(""), it.next().unwrap_or(""), it.next().unwrap_or(""));
|
||||
(format!("//{server}/{share}"), rest.to_string())
|
||||
} else if s.len() >= 2 && s.as_bytes()[1] == b':' && s.as_bytes()[0].is_ascii_alphabetic() {
|
||||
(s[..2].to_ascii_lowercase(), s[2..].to_string())
|
||||
let mut segs = unc.split('/').filter(|x| !x.is_empty());
|
||||
let (server, share) = (segs.next().unwrap_or(""), segs.next().unwrap_or(""));
|
||||
(format!("//{server}/{share}"), segs.collect::<Vec<_>>().join("/"))
|
||||
} else if let (true, Some(drive), Some(rest)) = (matches!(s.as_bytes(), [a, b':', ..] if a.is_ascii_alphabetic()), s.get(..2), s.get(2..)) {
|
||||
(if lower_drive { drive.to_ascii_lowercase() } else { drive.to_string() }, rest.to_string())
|
||||
} else {
|
||||
(String::new(), s.clone())
|
||||
};
|
||||
let absolute = rest.starts_with('/') || !prefix.is_empty();
|
||||
let mut parts: Vec<&str> = Vec::new();
|
||||
let mut parts: Vec<String> = Vec::new();
|
||||
for c in rest.split('/') {
|
||||
match c {
|
||||
"" | "." => {}
|
||||
".." if parts.last().is_some_and(|p| *p != "..") => {
|
||||
".." if parts.last().is_some_and(|p| p != "..") => {
|
||||
parts.pop();
|
||||
}
|
||||
".." if absolute => {}
|
||||
c => parts.push(c),
|
||||
c => parts.push(c.to_string()),
|
||||
}
|
||||
}
|
||||
SplitPath { prefix, parts, absolute }
|
||||
}
|
||||
|
||||
/// A folder path's identity, for telling whether two spellings name the same folder: `/` and
|
||||
/// `\\` are both separators, repeated separators, `.` and a trailing separator are dropped,
|
||||
/// `..` is resolved lexically, a Windows verbatim prefix (`\\?\`) is removed and the drive
|
||||
/// letter lower-cased; on Windows (case-insensitive file names) the whole path is lower-cased.
|
||||
/// File names are otherwise compared as written, so on a case-insensitive disk elsewhere
|
||||
/// (macOS, a Samba share) two spellings that differ in case are two folders here.
|
||||
/// No file-system access: the paths compared should already be absolute.
|
||||
pub fn folder_key(path: &str) -> String {
|
||||
let SplitPath { prefix, parts, absolute } = split_path(path, true);
|
||||
let key = if absolute { format!("{prefix}/{}", parts.join("/")) } else { parts.join("/") };
|
||||
if cfg!(windows) { key.to_lowercase() } else { key }
|
||||
}
|
||||
@@ -393,8 +544,51 @@ pub fn folder_key(path: &str) -> String {
|
||||
/// Whether `path` is folder `root` or lies somewhere inside it, however either is spelled
|
||||
/// (see [`folder_key`]).
|
||||
pub fn folder_within(path: &str, root: &str) -> bool {
|
||||
let (p, r) = (folder_key(path), folder_key(root));
|
||||
p == r || (p.starts_with(&r) && (r.ends_with('/') || p[r.len()..].starts_with('/')))
|
||||
key_within(&folder_key(path), &folder_key(root))
|
||||
}
|
||||
|
||||
/// The names below folder `root` that lead to `path`, as the caller spelled them (`None` when
|
||||
/// `path` is not at or inside `root`; empty when it is `root`). Compared by [`folder_key`], but
|
||||
/// the names keep their case, which the key lowers on Windows.
|
||||
pub fn folder_rest(path: &str, root: &str) -> Option<Vec<String>> {
|
||||
if !key_within(&folder_key(path), &folder_key(root)) {
|
||||
return None;
|
||||
}
|
||||
let depth = split_path(root, false).parts.len();
|
||||
Some(split_path(path, false).parts.into_iter().skip(depth).collect())
|
||||
}
|
||||
|
||||
/// Whether a library photo (not one only browsed in Local) lies in the folder whose
|
||||
/// [`folder_key`] is `root`. A plain POSIX path (no `\`, `.`, `..`, repeated or trailing
|
||||
/// separators) is its own identity, so it is read without building a key for it: the check runs
|
||||
/// for every photo of the library on each refresh, and this keeps it cheap.
|
||||
pub(crate) fn photo_in_root(p: &Photo, root: &str) -> bool {
|
||||
if p.local {
|
||||
return false;
|
||||
}
|
||||
let crate::Source::File { path } = &p.source else { return false };
|
||||
let plain = path.starts_with('/')
|
||||
&& !path.starts_with("//")
|
||||
&& !path.ends_with('/')
|
||||
&& !path.contains(['\\'])
|
||||
&& !path.contains("//")
|
||||
&& !path.contains("/.");
|
||||
if !cfg!(windows) && plain && root.starts_with('/') && !root.starts_with("//") {
|
||||
let r = root.trim_end_matches('/');
|
||||
return path.strip_prefix(r).is_some_and(|rest| rest.starts_with('/'));
|
||||
}
|
||||
crate::local::folder_of(p).is_some_and(|f| key_within(&f, root))
|
||||
}
|
||||
|
||||
/// [`folder_within`] for two paths already turned into their [`folder_key`].
|
||||
pub(crate) fn key_within(p: &str, r: &str) -> bool {
|
||||
// `.` or `a/..` name no folder: not "everything"
|
||||
if r.is_empty() {
|
||||
return false;
|
||||
}
|
||||
// a disk's root key ends in `/` (`c:/`, `//srv/share/`), a folder's does not: same folder
|
||||
let (p, r) = (p.trim_end_matches('/'), r.trim_end_matches('/'));
|
||||
p == r || p.strip_prefix(r).is_some_and(|rest| rest.starts_with('/'))
|
||||
}
|
||||
|
||||
/// The kind of Photo Merge result a file is, from the name merges give it (`IMG_1-HDR.dng`,
|
||||
|
||||
+1241
-65
File diff suppressed because it is too large.
Load diff
@@ -1,11 +1,23 @@
|
||||
//! Crash-safe writes of user-visible files: exports, renders, DNG conversions, smart previews and
|
||||
//! XMP sidecars all go through here (the catalog's own files use [`crate::store::FsStore`]).
|
||||
//!
|
||||
//! **Write policy** (issue #134): what can't be recreated — originals, XMP sidecars, the catalog,
|
||||
//! settings, DNGs that replace raws, smart previews (the only stand-in while an original is
|
||||
//! offline), an edit copy that becomes a library photo — is written **durably**: synced to disk
|
||||
//! before it counts as written. What can always be made again from the originals — exports,
|
||||
//! renders, screenshots, merge previews — is written **atomically** only: a half-written file
|
||||
//! never replaces a good one, but there is no per-file sync (on a USB drive or a NAS that sync
|
||||
//! dominates a large export).
|
||||
//!
|
||||
//! - [`write_atomic`] replaces a file so that after a crash, a full disk or an unplugged drive
|
||||
//! either the old or the complete new content is there — never a truncated mix. The content
|
||||
//! goes to a new temp file in the same folder (unique name, `create_new`), is synced to disk and
|
||||
//! renamed over the target; then the folder is synced (POSIX). Any failure removes the temp
|
||||
//! file and leaves the target as it was.
|
||||
//! - [`write_atomic_nosync`] is the same temp file + rename without the two syncs, for
|
||||
//! recreatable outputs only: a failed write still removes the temp file and leaves the target as
|
||||
//! it was, but after a power cut the new file may be lost (on some file systems left empty) —
|
||||
//! it is then simply exported again.
|
||||
//! - [`write_new`] / [`write_new_unique`] write a file that must not exist yet: the same temp file
|
||||
//! and sync, plus a read-back check, and the file then appears under its final name **without
|
||||
//! replacing anything** (a hard link, which fails on a taken name; a check-then-rename where the
|
||||
@@ -21,6 +33,19 @@ use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
thread_local! {
|
||||
static FAIL_AFTER: Cell<Option<u64>> = const { Cell::new(None) };
|
||||
static SYNCS: Cell<u64> = const { Cell::new(0) };
|
||||
}
|
||||
|
||||
/// How many file and folder syncs this module made **on the calling thread** (for tests: a
|
||||
/// durable write path syncs, a recreatable output doesn't).
|
||||
#[doc(hidden)]
|
||||
pub fn syncs_on_this_thread() -> u64 {
|
||||
SYNCS.with(Cell::get)
|
||||
}
|
||||
|
||||
fn sync_file(f: &File) -> io::Result<()> {
|
||||
SYNCS.with(|c| c.set(c.get() + 1));
|
||||
f.sync_all()
|
||||
}
|
||||
|
||||
/// Fault injection for tests (in any crate): while the returned guard lives, every write made by
|
||||
@@ -78,7 +103,7 @@ fn parent_of(path: &Path) -> &Path {
|
||||
/// where NTFS journals the rename itself, so errors are ignored.
|
||||
pub fn sync_dir(dir: &Path) {
|
||||
if let Ok(d) = File::open(dir) {
|
||||
let _ = d.sync_all();
|
||||
let _ = sync_file(&d);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,9 +125,9 @@ fn create_temp(path: &Path) -> io::Result<(PathBuf, File)> {
|
||||
Err(io::Error::new(io::ErrorKind::AlreadyExists, format!("no free temp file name in {}", dir.display())))
|
||||
}
|
||||
|
||||
/// Write a temp file next to `path` with what `fill` writes, synced to disk. On failure the temp
|
||||
/// file is removed. Returns its path and length.
|
||||
fn write_temp(path: &Path, fill: &mut dyn FnMut(&mut dyn Write) -> io::Result<()>) -> io::Result<(PathBuf, u64)> {
|
||||
/// Write a temp file next to `path` with what `fill` writes, synced to disk when `sync`. On
|
||||
/// failure the temp file is removed. Returns its path and length.
|
||||
fn write_temp(path: &Path, sync: bool, fill: &mut dyn FnMut(&mut dyn Write) -> io::Result<()>) -> io::Result<(PathBuf, u64)> {
|
||||
let (tmp, file) = create_temp(path)?;
|
||||
let r = (|| {
|
||||
let limit = FAIL_AFTER.with(Cell::get);
|
||||
@@ -111,7 +136,9 @@ fn write_temp(path: &Path, fill: &mut dyn FnMut(&mut dyn Write) -> io::Result<()
|
||||
w.flush()?;
|
||||
let n = w.n;
|
||||
let f = w.inner.into_inner().map_err(|e| e.into_error())?;
|
||||
f.sync_all()?;
|
||||
if sync {
|
||||
sync_file(&f)?;
|
||||
}
|
||||
Ok(n)
|
||||
})();
|
||||
match r {
|
||||
@@ -132,12 +159,26 @@ pub fn write_atomic(path: &Path, data: &[u8]) -> io::Result<()> {
|
||||
/// [`write_atomic`] with streamed content: `fill` writes it (an error aborts and leaves the old
|
||||
/// file). Returns the length written.
|
||||
pub fn write_atomic_with(path: &Path, fill: &mut dyn FnMut(&mut dyn Write) -> io::Result<()>) -> io::Result<u64> {
|
||||
let (tmp, n) = write_temp(path, fill)?;
|
||||
replace_with(path, true, fill)
|
||||
}
|
||||
|
||||
/// Replace (or create) `path` with `data` atomically but **without syncing** it to disk — only for
|
||||
/// outputs that can always be recreated (exports, renders, screenshots); see the module docs.
|
||||
/// Anything that can't be made again goes through [`write_atomic`].
|
||||
pub fn write_atomic_nosync(path: &Path, data: &[u8]) -> io::Result<()> {
|
||||
replace_with(path, false, &mut |w| w.write_all(data)).map(|_| ())
|
||||
}
|
||||
|
||||
/// Temp file (synced when `sync`) renamed over `path`, then the folder synced when `sync`.
|
||||
fn replace_with(path: &Path, sync: bool, fill: &mut dyn FnMut(&mut dyn Write) -> io::Result<()>) -> io::Result<u64> {
|
||||
let (tmp, n) = write_temp(path, sync, fill)?;
|
||||
if let Err(e) = fs::rename(&tmp, path) {
|
||||
let _ = fs::remove_file(&tmp);
|
||||
return Err(e);
|
||||
}
|
||||
sync_dir(parent_of(path));
|
||||
if sync {
|
||||
sync_dir(parent_of(path));
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
@@ -155,8 +196,11 @@ pub fn write_new_unique(candidates: &mut dyn Iterator<Item = PathBuf>, data: &[u
|
||||
let Some(first) = candidates.peek().cloned() else {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "no file name to write"));
|
||||
};
|
||||
let (tmp, _) = write_temp(&first, &mut |w| w.write_all(data))?;
|
||||
let (tmp, _) = write_temp(&first, true, &mut |w| w.write_all(data))?;
|
||||
let r = (|| {
|
||||
// Kept (issue #134 point 4): these files replace raws (Convert / Copy as DNG) or become
|
||||
// library photos (merges). On a local disk the read usually comes from the cache, but on a
|
||||
// network volume it can go back to the server, and next to encoding a DNG it costs little.
|
||||
if !same_content(&tmp, data)? {
|
||||
return Err(io::Error::other("the written file reads back different (failing drive or connection?)"));
|
||||
}
|
||||
@@ -287,6 +331,33 @@ mod tests {
|
||||
let _ = fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recreatable_outputs_are_atomic_without_a_sync() {
|
||||
let d = temp("nosync");
|
||||
let p = d.join("export.jpg");
|
||||
let before = syncs_on_this_thread();
|
||||
write_atomic_nosync(&p, b"one").unwrap();
|
||||
write_atomic_nosync(&p, b"two two").unwrap();
|
||||
assert_eq!(syncs_on_this_thread(), before, "no file or folder sync");
|
||||
assert_eq!(fs::read(&p).unwrap(), b"two two");
|
||||
assert_eq!(names(&d), vec!["export.jpg"], "no temp file left");
|
||||
// still atomic: a failure part-way leaves the previous file and no temp file
|
||||
{
|
||||
let _f = fail_writes_after(3);
|
||||
assert!(write_atomic_nosync(&p, &[9u8; 10_000]).is_err());
|
||||
}
|
||||
assert_eq!(fs::read(&p).unwrap(), b"two two");
|
||||
assert_eq!(names(&d), vec!["export.jpg"]);
|
||||
// the durable writers do sync (the file; the folder too where it can be opened)
|
||||
let before = syncs_on_this_thread();
|
||||
write_atomic(&p, b"three").unwrap();
|
||||
let mid = syncs_on_this_thread();
|
||||
assert!(mid > before);
|
||||
write_new(&d.join("new.dng"), b"dng").unwrap();
|
||||
assert!(syncs_on_this_thread() > mid);
|
||||
let _ = fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_files_never_replace_anything() {
|
||||
let d = temp("new");
|
||||
|
||||
+378
-2
@@ -58,12 +58,22 @@ fn albums_and_folders() {
|
||||
let a = photo(&mut c, "a.jpg", "2026-04-01");
|
||||
let f = c.alloc_album_id();
|
||||
c.apply(Op::AddAlbum {
|
||||
album: Album { id: f, name: "Trips".into(), parent: None, folder: true, photos: vec![], cover: None, smart: None, quick: false },
|
||||
album: Album { id: f, name: "Trips".into(), parent: None, folder: true, photos: vec![], cover: None, smart: None, quick: false, order: None },
|
||||
})
|
||||
.unwrap();
|
||||
let al = c.alloc_album_id();
|
||||
c.apply(Op::AddAlbum {
|
||||
album: Album { id: al, name: "Alps".into(), parent: Some(f), folder: false, photos: vec![a], cover: None, smart: None, quick: false },
|
||||
album: Album {
|
||||
id: al,
|
||||
name: "Alps".into(),
|
||||
parent: Some(f),
|
||||
folder: false,
|
||||
photos: vec![a],
|
||||
cover: None,
|
||||
smart: None,
|
||||
quick: false,
|
||||
order: None,
|
||||
},
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(c.albums_of(a), vec![al]);
|
||||
@@ -297,3 +307,369 @@ fn folder_identity_ignores_spelling() {
|
||||
assert!(folder_within("/a/b", "/"));
|
||||
assert!(folder_within("C:\\x", "c:\\"));
|
||||
}
|
||||
|
||||
/// People come from named *face* regions: counted once per photo, case-insensitive, most photos
|
||||
/// first; pets and unnamed faces are not people; the `person` filter and `person:` token match.
|
||||
#[test]
|
||||
fn people_from_named_face_regions() {
|
||||
use lightcraft_meta::{Rect, Region, RegionKind};
|
||||
let region = |name: Option<&str>, kind: RegionKind| Region {
|
||||
rect: Rect { x0: 0.4, y0: 0.4, x1: 0.6, y1: 0.6 },
|
||||
kind,
|
||||
name: name.map(str::to_string),
|
||||
description: None,
|
||||
};
|
||||
let mut c = Catalog::new();
|
||||
let mut add = |name: &str, regions: Vec<Region>| {
|
||||
let id = c.alloc_photo_id();
|
||||
let mut p = Photo::new(id, Source::Demo { scene: 1 }, name, "JPEG", 6000, 4000, "2026-09-30T10:00:00");
|
||||
p.meta.regions = regions;
|
||||
c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
|
||||
id
|
||||
};
|
||||
let a = add(
|
||||
"a.jpg",
|
||||
vec![region(Some("Jane Doe"), RegionKind::Face), region(Some("jane doe"), RegionKind::Face), region(Some("Rex"), RegionKind::Pet)],
|
||||
);
|
||||
let b =
|
||||
add("b.jpg", vec![region(Some("JANE DOE"), RegionKind::Face), region(Some("John Roe"), RegionKind::Face), region(None, RegionKind::Face)]);
|
||||
let big = Rect { x0: 0.2, y0: 0.2, x1: 0.8, y1: 0.8 };
|
||||
let d = add("d.jpg", vec![Region { rect: big, ..region(Some("John Roe"), RegionKind::Face) }]);
|
||||
let e = add("e.jpg", vec![region(Some("Sam"), RegionKind::Face)]);
|
||||
add("f.jpg", vec![]);
|
||||
let people = c.people();
|
||||
let summary: Vec<(&str, usize)> = people.iter().map(|p| (p.name.as_str(), p.count)).collect();
|
||||
assert_eq!(summary, vec![("Jane Doe", 2), ("John Roe", 2), ("Sam", 1)], "once per photo, pets and unnamed faces left out");
|
||||
// the picture is the person's largest face; ties go to the lower photo id
|
||||
assert_eq!((people[0].photo, people[0].face), (a, Rect { x0: 0.4, y0: 0.4, x1: 0.6, y1: 0.6 }));
|
||||
assert_eq!((people[1].photo, people[1].face), (d, big), "the larger face wins over an earlier, smaller one");
|
||||
|
||||
let q = |f: Filter| c.query(&f, &Sort::default());
|
||||
assert_eq!(q(Filter { person: Some("jane doe".into()), ..Default::default() }).len(), 2);
|
||||
let mut got = q(Filter { person: Some("John Roe".into()), ..Default::default() });
|
||||
got.sort();
|
||||
assert_eq!(got, vec![b, d]);
|
||||
assert!(q(Filter { person: Some("Rex".into()), ..Default::default() }).is_empty(), "a pet is not a person");
|
||||
assert_eq!(q(Filter { text: "person:SAM".into(), ..Default::default() }), vec![e], "the search token finds a person, any case");
|
||||
assert!(q(Filter { person: Some("Jane Doe".into()), ..Default::default() }).contains(&a));
|
||||
|
||||
// other filters narrow who is offered (so picking a person never ends in an empty grid); the
|
||||
// `person` filter itself does not
|
||||
c.apply(Op::SetRating { id: b, rating: 3 }).unwrap();
|
||||
let names = |f: Filter| c.people_in(&f).into_iter().map(|p| (p.name, p.count)).collect::<Vec<_>>();
|
||||
let rated = Filter { rating: 1, ..Default::default() };
|
||||
assert_eq!(
|
||||
names(rated.clone()),
|
||||
vec![("JANE DOE".to_string(), 1), ("John Roe".to_string(), 1)],
|
||||
"only the rated photo's people, counted within it, spelled as first seen there"
|
||||
);
|
||||
assert_eq!(
|
||||
names(Filter { person: Some("Sam".into()), ..rated }),
|
||||
names(Filter { rating: 1, ..Default::default() }),
|
||||
"the person filter is ignored"
|
||||
);
|
||||
assert_eq!(names(Filter { rating: 5, ..Default::default() }), vec![], "nobody in the filtered photos");
|
||||
}
|
||||
|
||||
/// Most photos have no regions: the field is left out of the catalog JSON then, and a catalog written
|
||||
/// before regions existed (no `regions` key) reads with none.
|
||||
#[test]
|
||||
fn empty_regions_are_not_serialized_and_default_when_missing() {
|
||||
let m = Meta { keywords: vec!["k".into()], ..Default::default() };
|
||||
let v = serde_json::to_value(&m).unwrap();
|
||||
assert!(v.get("regions").is_none(), "{v}");
|
||||
let back: Meta = serde_json::from_value(v).unwrap();
|
||||
assert!(back.regions.is_empty());
|
||||
let mut with = m.clone();
|
||||
with.regions.push(lightcraft_meta::Region {
|
||||
rect: lightcraft_meta::Rect { x0: 0.1, y0: 0.1, x1: 0.2, y1: 0.2 },
|
||||
kind: lightcraft_meta::RegionKind::Face,
|
||||
name: Some("A".into()),
|
||||
description: None,
|
||||
});
|
||||
let back: Meta = serde_json::from_str(&serde_json::to_string(&with).unwrap()).unwrap();
|
||||
assert_eq!(back, with);
|
||||
}
|
||||
|
||||
// ---- random sort: Given a seed, the order is a stable, reproducible shuffle of the matching photos
|
||||
|
||||
fn many(c: &mut Catalog, n: usize) -> Vec<PhotoId> {
|
||||
(0..n).map(|i| photo(c, &format!("p{i:03}.jpg"), &format!("2026-04-{:02}T10:00:00", i % 28 + 1))).collect()
|
||||
}
|
||||
|
||||
fn random(seed: u64) -> Sort {
|
||||
Sort { key: SortKey::Random, seed, ..Default::default() }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_sort_is_a_reproducible_permutation() {
|
||||
let mut c = Catalog::new();
|
||||
let ids = many(&mut c, 50);
|
||||
let a = c.query(&Filter::default(), &random(7));
|
||||
assert_eq!(a, c.query(&Filter::default(), &random(7)), "same seed, same order");
|
||||
let mut sorted = a.clone();
|
||||
sorted.sort();
|
||||
let mut all = ids.clone();
|
||||
all.sort();
|
||||
assert_eq!(sorted, all, "every photo exactly once");
|
||||
assert_ne!(a, c.query(&Filter::default(), &random(8)), "another seed, another order");
|
||||
assert_ne!(a, c.query(&Filter::default(), &Sort::default()), "not just the date order");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_sort_keeps_the_relative_order_when_the_set_changes() {
|
||||
let mut c = Catalog::new();
|
||||
let ids = many(&mut c, 30);
|
||||
let before = c.query(&Filter::default(), &random(3));
|
||||
let added = photo(&mut c, "new.jpg", "2026-05-01T10:00:00");
|
||||
c.apply(Op::SetRating { id: ids[0], rating: 5 }).unwrap();
|
||||
let after: Vec<PhotoId> = c.query(&Filter::default(), &random(3)).into_iter().filter(|id| *id != added).collect();
|
||||
assert_eq!(after, before, "adding a photo or editing metadata must not reshuffle the others");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_sort_respects_the_filter_and_handles_tiny_sets() {
|
||||
let mut c = Catalog::new();
|
||||
assert!(c.query(&Filter::default(), &random(1)).is_empty());
|
||||
let ids = many(&mut c, 10);
|
||||
c.apply(Op::SetRating { id: ids[4], rating: 5 }).unwrap();
|
||||
assert_eq!(c.query(&Filter { rating: 5, ..Default::default() }, &random(1)), vec![ids[4]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_sort_direction_reverses_and_serde_defaults_hold() {
|
||||
let mut c = Catalog::new();
|
||||
many(&mut c, 20);
|
||||
let desc = c.query(&Filter::default(), &Sort { ascending: false, ..random(5) });
|
||||
let mut asc = c.query(&Filter::default(), &Sort { ascending: true, ..random(5) });
|
||||
asc.reverse();
|
||||
assert_eq!(desc, asc);
|
||||
// saved sorts from before the seed existed still load
|
||||
let old: Sort = serde_json::from_str(r#"{"key":"fileName","ascending":true}"#).unwrap();
|
||||
assert_eq!(old.seed, 0);
|
||||
assert_eq!(serde_json::from_str::<Sort>(r#"{"key":"random","seed":9}"#).unwrap(), Sort { key: SortKey::Random, seed: 9, ..Default::default() });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_sort_has_no_date_headers() {
|
||||
let mut c = Catalog::new();
|
||||
let ids = many(&mut c, 5);
|
||||
assert!(c.date_runs(&ids, SortKey::Random, GroupBy::Day).is_empty());
|
||||
}
|
||||
|
||||
/// Reload stores the lens data a file carries; the inverse (through the journal's JSON) takes it back.
|
||||
#[test]
|
||||
fn set_embedded_lens_is_undoable_and_journaled() {
|
||||
let mut c = Catalog::default();
|
||||
let a = c.alloc_photo_id();
|
||||
c.apply(Op::AddPhoto {
|
||||
photo: Box::new(Photo::new(a, Source::File { path: "/x.rw2".into() }, "x.rw2", "RW2", 4000, 3000, "2026-10-06T00:00:00")),
|
||||
})
|
||||
.unwrap();
|
||||
let lens = lightcraft_develop::EmbeddedLens { warp: Some(Default::default()), vignette: None };
|
||||
let op = Op::SetEmbeddedLens { id: a, lens: Some(Box::new(lens)) };
|
||||
let op: Op = serde_json::from_str(&serde_json::to_string(&op).unwrap()).unwrap();
|
||||
let inv = c.apply(op).unwrap();
|
||||
assert_eq!(c.photo(a).unwrap().embedded_lens, Some(lens));
|
||||
let inv: Op = serde_json::from_str(&serde_json::to_string(&inv).unwrap()).unwrap();
|
||||
c.apply(inv).unwrap();
|
||||
assert_eq!(c.photo(a).unwrap().embedded_lens, None);
|
||||
}
|
||||
|
||||
/// A saved smart album whose rules no longer check (the album a rule tests was deleted) is found,
|
||||
/// with its problems; a good one has none; an old album operator is read as it was meant.
|
||||
#[test]
|
||||
fn smart_album_problems_flag_stale_rules() {
|
||||
let mut c = Catalog::new();
|
||||
c.apply(Op::AddAlbum { album: Album::new(AlbumId(1), "Trip") }).unwrap();
|
||||
let smart = |id: u64, rules: serde_json::Value| {
|
||||
let f: Filter = serde_json::from_value(serde_json::json!({"ruleSet": {"rules": rules}})).unwrap();
|
||||
Album { smart: Some(Box::new(f)), ..Album::new(AlbumId(id), "Smart") }
|
||||
};
|
||||
c.apply(Op::AddAlbum { album: smart(2, serde_json::json!([{"field": "album", "op": "is", "value": 1}])) }).unwrap();
|
||||
c.apply(Op::AddAlbum { album: smart(3, serde_json::json!([{"field": "album", "op": "gte", "value": 1}])) }).unwrap();
|
||||
assert!(c.smart_album_problems(AlbumId(2)).is_empty());
|
||||
assert!(c.smart_album_problems(AlbumId(3)).is_empty(), "an old operator is upgraded, not a problem");
|
||||
assert!(c.smart_album_problems(AlbumId(1)).is_empty(), "a plain album has no rules");
|
||||
c.apply(Op::RemoveAlbum { id: AlbumId(1) }).unwrap();
|
||||
let p = c.smart_album_problems(AlbumId(2));
|
||||
assert_eq!(p.iter().map(ToString::to_string).collect::<Vec<_>>(), vec!["rule 1: no album 1".to_string()]);
|
||||
}
|
||||
|
||||
/// A smart album can test another smart album: "Keywords contain travel" and "Album isn't Excluded
|
||||
/// Photos" (red or rejected) leaves out the excluded photos, and follows that album's rules as they
|
||||
/// change.
|
||||
#[test]
|
||||
fn a_smart_album_can_exclude_another() {
|
||||
let mut c = Catalog::new();
|
||||
let smart = |id: u64, name: &str, rules: serde_json::Value| {
|
||||
let f: Filter = serde_json::from_value(serde_json::json!({"ruleSet": rules})).unwrap();
|
||||
Album { smart: Some(Box::new(f)), ..Album::new(AlbumId(id), name) }
|
||||
};
|
||||
c.apply(Op::AddAlbum {
|
||||
album: smart(
|
||||
1,
|
||||
"Excluded Photos",
|
||||
serde_json::json!({"match": "any", "rules": [
|
||||
{"field": "label", "op": "is", "value": "red"}, {"field": "flag", "op": "is", "value": "reject"}]}),
|
||||
),
|
||||
})
|
||||
.unwrap();
|
||||
let travel = |c: &mut Catalog, name: &str, label: Option<ColorLabel>, flag: Flag| {
|
||||
let id = c.alloc_photo_id();
|
||||
let mut p = Photo::new(id, Source::Demo { scene: 1 }, name, "JPEG", 6000, 4000, "2026-09-30T10:00:00");
|
||||
p.meta.keywords = vec!["travel".into()];
|
||||
p.label = label;
|
||||
p.flag = flag;
|
||||
c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
|
||||
id
|
||||
};
|
||||
let keep = travel(&mut c, "keep.jpg", None, Flag::Pick);
|
||||
let red = travel(&mut c, "red.jpg", Some(ColorLabel::Red), Flag::None);
|
||||
let rejected = travel(&mut c, "rejected.jpg", None, Flag::Reject);
|
||||
let rules: RuleSet = serde_json::from_value(serde_json::json!({"rules": [
|
||||
{"field": "keywords", "op": "contains", "value": "travel"}, {"field": "album", "op": "isNot", "value": 1}]}))
|
||||
.unwrap();
|
||||
assert!(rules.check(&c).is_empty(), "a smart album can be tested: {:?}", rules.check(&c));
|
||||
let matched = |c: &Catalog| c.photos().filter(|p| rules.matches(p, c)).map(|p| p.id).collect::<Vec<_>>();
|
||||
assert_eq!(matched(&c), vec![keep]);
|
||||
// "is" works the other way round
|
||||
let inside: RuleSet = serde_json::from_value(serde_json::json!({"rules": [{"field": "album", "op": "is", "value": 1}]})).unwrap();
|
||||
assert_eq!(c.photos().filter(|p| inside.matches(p, &c)).count(), 2);
|
||||
let _ = (red, rejected);
|
||||
}
|
||||
|
||||
/// An album can't include itself, directly or through other smart albums: the check refuses the
|
||||
/// loop for the album being edited, and a loop saved anyway (an older version, a hand-edited file)
|
||||
/// neither recurses forever nor crashes: inside it, the album being evaluated counts as holding no
|
||||
/// photos, and every album in it is reported.
|
||||
#[test]
|
||||
fn smart_album_loops_are_refused_and_survived() {
|
||||
let mut c = Catalog::new();
|
||||
let refers = |to: u64| -> Filter {
|
||||
serde_json::from_value(serde_json::json!({"ruleSet": {"rules": [{"field": "album", "op": "isNot", "value": to}]}})).unwrap()
|
||||
};
|
||||
let smart = |id: u64, f: Filter| Album { smart: Some(Box::new(f)), ..Album::new(AlbumId(id), "S") };
|
||||
// A → B → C
|
||||
c.apply(Op::AddAlbum { album: smart(3, Filter::default()) }).unwrap();
|
||||
c.apply(Op::AddAlbum { album: smart(2, refers(3)) }).unwrap();
|
||||
c.apply(Op::AddAlbum { album: smart(1, refers(2)) }).unwrap();
|
||||
let rules = |to: u64| refers(to).rule_set.unwrap();
|
||||
assert!(rules(1).check_for(&c, Some(AlbumId(3))).iter().any(|p| p.issue == crate::rules::Issue::AlbumLoop), "C → A closes A → B → C → A");
|
||||
assert!(rules(3).check_for(&c, Some(AlbumId(3))).iter().any(|p| p.issue == crate::rules::Issue::AlbumLoop), "an album testing itself");
|
||||
assert!(rules(1).check_for(&c, None).is_empty(), "a new album can't be in a loop yet");
|
||||
assert!(rules(3).check_for(&c, Some(AlbumId(1))).is_empty(), "A testing C again is no loop");
|
||||
assert!(!c.album_reaches(AlbumId(3), AlbumId(1)) && c.album_reaches(AlbumId(1), AlbumId(3)));
|
||||
// a loop saved anyway: C → A
|
||||
c.apply(Op::SetAlbumRules { id: AlbumId(3), rules: Box::new(refers(1)) }).unwrap();
|
||||
let id = photo(&mut c, "a.jpg", "2026-09-01T10:00:00");
|
||||
let p = c.photo(id).unwrap().clone();
|
||||
for a in [1, 2, 3] {
|
||||
let _ = c.album_contains(AlbumId(a), &p); // returns, no stack overflow
|
||||
assert!(c.smart_album_problems(AlbumId(a)).iter().any(|p| p.issue == crate::rules::Issue::AlbumLoop), "album {a} is reported");
|
||||
}
|
||||
}
|
||||
|
||||
/// Summaries name the albums Album rules test ("album isn't “Excluded Photos”"), not their ids;
|
||||
/// one that is gone shows as its number.
|
||||
#[test]
|
||||
fn summaries_name_albums() {
|
||||
let mut c = Catalog::new();
|
||||
c.apply(Op::AddAlbum { album: Album::new(AlbumId(4), "Excluded Photos") }).unwrap();
|
||||
let rules: RuleSet = serde_json::from_value(serde_json::json!({"rules": [
|
||||
{"field": "keywords", "op": "contains", "value": "travel"},
|
||||
{"group": {"match": "any", "rules": [{"field": "album", "op": "isNot", "value": 4}, {"field": "album", "op": "is", "value": 9}]}}]}))
|
||||
.unwrap();
|
||||
assert_eq!(rules.describe_with(&c), "keywords contains travel and (album isn't “Excluded Photos” or album is #9)");
|
||||
let f = Filter { rule_set: Some(rules), ..Default::default() };
|
||||
assert!(f.describe_with(&c).contains("album isn't “Excluded Photos”"));
|
||||
}
|
||||
|
||||
/// A chain of smart albums each testing the one before twice ("all of: album is X(k-1), album is
|
||||
/// X(k-1)") costs a pass per album, not 2^depth: each album's answer for a photo is worked out once
|
||||
/// per question. 25 albums deep stays instant.
|
||||
#[test]
|
||||
fn chains_of_smart_albums_stay_linear() {
|
||||
let mut c = Catalog::new();
|
||||
let smart = |id: u64, f: Filter| Album { smart: Some(Box::new(f)), ..Album::new(AlbumId(id), "X") };
|
||||
c.apply(Op::AddAlbum { album: smart(1, Filter::default()) }).unwrap();
|
||||
for k in 2..=26u64 {
|
||||
let f: Filter = serde_json::from_value(serde_json::json!({"ruleSet": {"rules": [
|
||||
{"field": "album", "op": "is", "value": k - 1}, {"field": "album", "op": "is", "value": k - 1}]}}))
|
||||
.unwrap();
|
||||
c.apply(Op::AddAlbum { album: smart(k, f) }).unwrap();
|
||||
}
|
||||
for i in 0..20 {
|
||||
photo(&mut c, &format!("p{i}.jpg"), "2026-09-01T10:00:00");
|
||||
}
|
||||
let start = std::time::Instant::now();
|
||||
assert_eq!(c.album_count(AlbumId(26)), 20);
|
||||
assert!(start.elapsed() < std::time::Duration::from_secs(2), "took {:?}", start.elapsed());
|
||||
}
|
||||
|
||||
/// A smart album's own album filter (not only its rules) counts in loops: A filtered to "in A" is
|
||||
/// reported, and so is A filtered to B while B's rules test A.
|
||||
#[test]
|
||||
fn an_album_filter_counts_in_loops() {
|
||||
let mut c = Catalog::new();
|
||||
let smart = |id: u64, f: Filter| Album { smart: Some(Box::new(f)), ..Album::new(AlbumId(id), "S") };
|
||||
c.apply(Op::AddAlbum { album: smart(1, Filter { album: Some(AlbumId(1)), ..Default::default() }) }).unwrap();
|
||||
let loops = |c: &Catalog, id: u64| c.smart_album_problems(AlbumId(id)).iter().any(|p| p.issue == crate::rules::Issue::AlbumLoop);
|
||||
assert!(loops(&c, 1), "filtered to itself");
|
||||
let tests_3: Filter = serde_json::from_value(serde_json::json!({"ruleSet": {"rules": [{"field": "album", "op": "is", "value": 3}]}})).unwrap();
|
||||
c.apply(Op::AddAlbum { album: smart(2, Filter { album: Some(AlbumId(3)), ..Default::default() }) }).unwrap();
|
||||
c.apply(Op::AddAlbum {
|
||||
album: smart(3, serde_json::from_value(serde_json::json!({"ruleSet": {"rules": [{"field": "album", "op": "is", "value": 2}]}})).unwrap()),
|
||||
})
|
||||
.unwrap();
|
||||
assert!(c.album_reaches(AlbumId(3), AlbumId(2)) && c.album_reaches(AlbumId(2), AlbumId(3)));
|
||||
assert!(loops(&c, 2) && loops(&c, 3));
|
||||
let _ = tests_3;
|
||||
let shown: Vec<String> = c.smart_album_problems(AlbumId(1)).iter().map(ToString::to_string).collect();
|
||||
assert!(shown.iter().all(|s| !s.starts_with("rule :")), "{shown:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn undated_photos_group_under_unknown_date_and_sort_together() {
|
||||
let mut c = Catalog::new();
|
||||
let add_photo = |c: &mut Catalog, name: &str, captured: Option<&str>, imported: &str| {
|
||||
let id = c.alloc_photo_id();
|
||||
let mut p = Photo::new(id, Source::Demo { scene: 0 }, name, "JPEG", 100, 100, imported);
|
||||
p.captured = captured.map(str::to_string);
|
||||
c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
|
||||
id
|
||||
};
|
||||
let p_dated1 = add_photo(&mut c, "dated1.jpg", Some("2026-05-10T12:00:00"), "2026-10-01T00:00:00");
|
||||
let p_dated2 = add_photo(&mut c, "dated2.jpg", Some("2026-05-20T12:00:00"), "2026-10-01T00:00:00");
|
||||
let p_undated1 = add_photo(&mut c, "undated1.jpg", None, "2026-10-05T00:00:00");
|
||||
let p_undated2 = add_photo(&mut c, "undated2.jpg", None, "2026-10-02T00:00:00");
|
||||
|
||||
// Descending sort (default, newest first): dated photos first (newest to oldest), undated photos at the end
|
||||
let desc = c.query(&Filter::default(), &Sort { key: SortKey::CaptureDate, ascending: false, ..Default::default() });
|
||||
assert_eq!(desc, vec![p_dated2, p_dated1, p_undated1, p_undated2]);
|
||||
|
||||
// Ascending sort (oldest first): undated photos at the beginning, dated photos from oldest to newest
|
||||
let asc = c.query(&Filter::default(), &Sort { key: SortKey::CaptureDate, ascending: true, ..Default::default() });
|
||||
assert_eq!(asc, vec![p_undated2, p_undated1, p_dated1, p_dated2]);
|
||||
|
||||
// date_runs groups undated photos under Unknown Date
|
||||
let runs = c.date_runs(&desc, SortKey::CaptureDate, GroupBy::Day);
|
||||
assert_eq!(runs.len(), 3);
|
||||
assert_eq!(runs[0].key, "2026-05-20");
|
||||
assert_eq!(runs[1].key, "2026-05-10");
|
||||
assert_eq!(runs[2].key, "");
|
||||
assert_eq!(runs[2].label, "Unknown Date");
|
||||
assert_eq!(runs[2].count, 2);
|
||||
|
||||
// date_groups sidebar tree only counts photos with capture dates
|
||||
let groups = c.date_groups();
|
||||
assert_eq!(groups.len(), 1);
|
||||
assert_eq!(groups[0].year, "2026");
|
||||
assert_eq!(groups[0].count, 2);
|
||||
|
||||
// Filter by capture date prefix does not match undated photos
|
||||
let filter_day = Filter { date: Some("2026-10-05".into()), ..Default::default() };
|
||||
assert_eq!(c.query(&filter_day, &Sort::default()), vec![]);
|
||||
let filter_year = Filter { date: Some("2026".into()), ..Default::default() };
|
||||
assert_eq!(c.query(&filter_year, &Sort { key: SortKey::CaptureDate, ascending: false, ..Default::default() }), vec![p_dated2, p_dated1]);
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
//! Album order: the children of a folder (or of the top level) are listed folders first, then
|
||||
//! alphabetically, until some of them are given a place of their own (`Album.order`); moving an
|
||||
//! album to another folder drops its place, and undoing the move brings it back.
|
||||
|
||||
use crate::*;
|
||||
|
||||
fn add(c: &mut Catalog, name: &str, parent: Option<AlbumId>, folder: bool) -> AlbumId {
|
||||
let id = c.alloc_album_id();
|
||||
c.apply(Op::AddAlbum { album: Album { parent, folder, ..Album::new(id, name) } }).unwrap();
|
||||
id
|
||||
}
|
||||
|
||||
fn names(c: &Catalog, parent: Option<AlbumId>) -> Vec<String> {
|
||||
c.album_children(parent).iter().map(|a| a.name.clone()).collect()
|
||||
}
|
||||
|
||||
fn place(c: &mut Catalog, id: AlbumId, order: Option<u32>) {
|
||||
c.apply(Op::SetAlbumOrder { id, order }).unwrap();
|
||||
}
|
||||
|
||||
/// Given albums that were never ordered, when they are listed, then folders come first and each
|
||||
/// group is alphabetical regardless of case.
|
||||
#[test]
|
||||
fn children_are_folders_first_then_alphabetical() {
|
||||
let mut c = Catalog::new();
|
||||
add(&mut c, "beta", None, false);
|
||||
add(&mut c, "Alpha", None, false);
|
||||
add(&mut c, "zoo", None, true);
|
||||
add(&mut c, "Archive", None, true);
|
||||
assert_eq!(names(&c, None), ["Archive", "zoo", "Alpha", "beta"]);
|
||||
assert!(!c.album_children_are_ordered(None));
|
||||
}
|
||||
|
||||
/// Given some children with a place of their own, then they follow it; the others come after, by
|
||||
/// name; folders stay ahead of albums.
|
||||
#[test]
|
||||
fn ordered_children_follow_their_order() {
|
||||
let mut c = Catalog::new();
|
||||
let a = add(&mut c, "A", None, false);
|
||||
let b = add(&mut c, "B", None, false);
|
||||
let cc = add(&mut c, "C", None, false);
|
||||
add(&mut c, "D", None, false);
|
||||
let f = add(&mut c, "F", None, true);
|
||||
place(&mut c, cc, Some(0));
|
||||
place(&mut c, a, Some(1));
|
||||
assert_eq!(names(&c, None), ["F", "C", "A", "B", "D"]);
|
||||
assert!(c.album_children_are_ordered(None));
|
||||
// equal places fall back to the name, so a damaged catalog still lists deterministically
|
||||
place(&mut c, b, Some(1));
|
||||
assert_eq!(names(&c, None), ["F", "C", "A", "B", "D"]);
|
||||
// another container is unaffected
|
||||
let inner = add(&mut c, "Z", Some(f), false);
|
||||
add(&mut c, "Y", Some(f), false);
|
||||
assert_eq!(names(&c, Some(f)), ["Y", "Z"]);
|
||||
assert!(!c.album_children_are_ordered(Some(f)));
|
||||
place(&mut c, inner, Some(0));
|
||||
assert_eq!(names(&c, Some(f)), ["Z", "Y"]);
|
||||
}
|
||||
|
||||
/// The one-pass map lists every folder's children exactly as `album_children` does.
|
||||
#[test]
|
||||
fn the_children_map_matches_the_children_of_each_folder() {
|
||||
let mut c = Catalog::new();
|
||||
let f = add(&mut c, "F", None, true);
|
||||
let g = add(&mut c, "G", Some(f), true);
|
||||
for (name, parent) in [("b", None), ("a", None), ("y", Some(f)), ("x", Some(f)), ("q", Some(g))] {
|
||||
add(&mut c, name, parent, false);
|
||||
}
|
||||
let x = c.albums().find(|a| a.name == "x").unwrap().id;
|
||||
place(&mut c, x, Some(7));
|
||||
let map = c.album_children_by_parent();
|
||||
for parent in [None, Some(f), Some(g)] {
|
||||
let from_map: Vec<AlbumId> = map.get(&parent).map(|v| v.iter().map(|a| a.id).collect()).unwrap_or_default();
|
||||
let direct: Vec<AlbumId> = c.album_children(parent).iter().map(|a| a.id).collect();
|
||||
assert_eq!(from_map, direct, "{parent:?}");
|
||||
}
|
||||
assert!(!map.contains_key(&Some(AlbumId(999))));
|
||||
}
|
||||
|
||||
/// A place is one undo step, and `None` puts the album back among the alphabetical ones.
|
||||
#[test]
|
||||
fn a_place_can_be_set_cleared_and_undone() {
|
||||
let mut c = Catalog::new();
|
||||
let a = add(&mut c, "A", None, false);
|
||||
add(&mut c, "B", None, false);
|
||||
let inv = c.apply(Op::SetAlbumOrder { id: a, order: Some(5) }).unwrap();
|
||||
assert_eq!(c.album(a).unwrap().order, Some(5));
|
||||
c.apply(inv).unwrap();
|
||||
assert_eq!(c.album(a).unwrap().order, None);
|
||||
assert!(c.apply(Op::SetAlbumOrder { id: AlbumId(999), order: Some(1) }).is_err(), "no such album");
|
||||
}
|
||||
|
||||
/// Moving to another folder drops the album's place there (it belonged to the old neighbours),
|
||||
/// and undoing the move gives it back.
|
||||
#[test]
|
||||
fn moving_drops_the_place_and_undo_restores_it() {
|
||||
let mut c = Catalog::new();
|
||||
let from = add(&mut c, "From", None, true);
|
||||
let to = add(&mut c, "To", None, true);
|
||||
let a = add(&mut c, "A", Some(from), false);
|
||||
place(&mut c, a, Some(3));
|
||||
let inv = c.apply(Op::MoveAlbum { id: a, parent: Some(to) }).unwrap();
|
||||
assert_eq!((c.album(a).unwrap().parent, c.album(a).unwrap().order), (Some(to), None));
|
||||
c.apply(inv).unwrap();
|
||||
assert_eq!((c.album(a).unwrap().parent, c.album(a).unwrap().order), (Some(from), Some(3)));
|
||||
// "moving" to where it already is changes nothing
|
||||
c.apply(Op::MoveAlbum { id: a, parent: Some(from) }).unwrap();
|
||||
assert_eq!(c.album(a).unwrap().order, Some(3));
|
||||
}
|
||||
|
||||
/// Catalogs written before albums had an order load as they were; an unset order is not written.
|
||||
#[test]
|
||||
fn albums_without_an_order_load_and_save_unchanged() {
|
||||
let old = r#"{"id":4,"name":"Old","parent":null,"folder":false,"photos":[],"cover":null}"#;
|
||||
let a: Album = serde_json::from_str(old).unwrap();
|
||||
assert_eq!(a.order, None);
|
||||
assert!(!serde_json::to_string(&a).unwrap().contains("order"));
|
||||
let mut a = a;
|
||||
a.order = Some(2);
|
||||
let back: Album = serde_json::from_str(&serde_json::to_string(&a).unwrap()).unwrap();
|
||||
assert_eq!(back.order, Some(2));
|
||||
}
|
||||
@@ -180,7 +180,8 @@ fn failed_background_snapshot_loses_nothing_and_retries_later() {
|
||||
wait_written(&j);
|
||||
assert!(j.poll().is_err());
|
||||
assert_eq!((j.snapshot_seq(), j.stats().failed_snapshots), (0, 1));
|
||||
assert!(g.files.get(SNAPSHOT).is_none());
|
||||
// still the (empty) snapshot written when the library was created
|
||||
assert_eq!(g.snapshot_seq_on_disk(), Some(0));
|
||||
assert_recovers(&g, &live);
|
||||
// no retry storm: the next attempt waits for another `max_records` records
|
||||
assert!(!j.wants_snapshot());
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
//! What the library keeps about its folders: a colour label, like a photo's (see
|
||||
//! [`crate::folders::FolderRecord`]).
|
||||
//!
|
||||
//! Scenarios, in the words of someone organising their library by folder:
|
||||
//!
|
||||
//! * Given a folder of the library, when I give it a colour label, its row in the folder tree
|
||||
//! shows that label; the folders around it show none.
|
||||
//! * However the folder is spelled, it is one folder with one label.
|
||||
//! * When I take the label off, the library keeps nothing about the folder.
|
||||
//! * Every change is one undo step: undo puts back the label the folder had before.
|
||||
//! * A path that names no folder is refused and nothing changes.
|
||||
//! * When the folder is renamed or moved, its label and those of the folders inside it go along;
|
||||
//! a folder beside it whose name merely starts the same way keeps its own. Undo brings them
|
||||
//! back, also a label that was left at the destination.
|
||||
//! * Labels survive closing and reopening the library.
|
||||
|
||||
use crate::*;
|
||||
|
||||
fn add(c: &mut Catalog, path: &str) {
|
||||
let id = c.alloc_photo_id();
|
||||
let name = path.rsplit('/').next().unwrap_or(path);
|
||||
let p = Photo::new(id, Source::File { path: path.into() }, name, "JPEG", 60, 40, "2026-01-01T10:00:00");
|
||||
c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
|
||||
}
|
||||
|
||||
fn library() -> Catalog {
|
||||
let mut c = Catalog::new();
|
||||
for p in ["/pics/trip/a.jpg", "/pics/trip/day1/b.jpg", "/pics/trip2/c.jpg", "/pics/home/d.jpg"] {
|
||||
add(&mut c, p);
|
||||
}
|
||||
c
|
||||
}
|
||||
|
||||
fn label(c: &mut Catalog, path: &str, l: Option<ColorLabel>) -> Op {
|
||||
let op = c.folder_label_op(path, l);
|
||||
c.apply(op).unwrap()
|
||||
}
|
||||
|
||||
/// The label of each row of the tree, by path, depth first (rows without one left out).
|
||||
fn tree_labels(c: &Catalog) -> Vec<(String, ColorLabel)> {
|
||||
fn walk(n: &FolderNode, out: &mut Vec<(String, ColorLabel)>) {
|
||||
if let Some(l) = n.label {
|
||||
out.push((n.path.clone(), l));
|
||||
}
|
||||
n.children.iter().for_each(|c| walk(c, out));
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
c.folder_tree().iter().for_each(|n| walk(n, &mut out));
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_new_library_knows_nothing_about_its_folders() {
|
||||
let c = library();
|
||||
assert_eq!(c.folder_record("/pics/trip"), None);
|
||||
assert_eq!(c.folder_color_label("/pics/trip"), None);
|
||||
assert!(tree_labels(&c).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_labelled_folder_shows_its_label_in_the_tree_and_only_there() {
|
||||
let mut c = library();
|
||||
label(&mut c, "/pics/trip", Some(ColorLabel::Red));
|
||||
assert_eq!(c.folder_color_label("/pics/trip"), Some(ColorLabel::Red));
|
||||
assert_eq!(tree_labels(&c), vec![("/pics/trip".to_string(), ColorLabel::Red)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn however_a_folder_is_spelled_it_has_one_label() {
|
||||
let mut c = library();
|
||||
label(&mut c, "/pics//trip/", Some(ColorLabel::Green));
|
||||
assert_eq!(c.folder_color_label("/pics/./trip"), Some(ColorLabel::Green));
|
||||
label(&mut c, "/pics/home/../trip", Some(ColorLabel::Blue));
|
||||
assert_eq!(c.folder_color_label("/pics/trip"), Some(ColorLabel::Blue));
|
||||
assert_eq!(tree_labels(&c).len(), 1, "one folder, one record");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn taking_the_label_off_leaves_no_record() {
|
||||
let mut c = library();
|
||||
label(&mut c, "/pics/trip", Some(ColorLabel::Purple));
|
||||
label(&mut c, "/pics/trip", None);
|
||||
assert_eq!(c.folder_record("/pics/trip"), None);
|
||||
assert!(!c.to_snapshot().contains("folder_records"), "nothing empty is saved");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_record_is_no_record() {
|
||||
let mut c = library();
|
||||
c.apply(Op::SetFolderRecord { folder: "/pics/trip".into(), record: Some(FolderRecord::default()) }).unwrap();
|
||||
assert_eq!(c.folder_record("/pics/trip"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn undo_puts_back_the_label_the_folder_had() {
|
||||
let mut c = library();
|
||||
let undo_first = label(&mut c, "/pics/trip", Some(ColorLabel::Yellow));
|
||||
let undo_second = label(&mut c, "/pics/trip", Some(ColorLabel::Red));
|
||||
c.apply(undo_second).unwrap();
|
||||
assert_eq!(c.folder_color_label("/pics/trip"), Some(ColorLabel::Yellow));
|
||||
c.apply(undo_first).unwrap();
|
||||
assert_eq!(c.folder_record("/pics/trip"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_path_that_names_no_folder_is_refused() {
|
||||
let mut c = library();
|
||||
let before = c.to_snapshot();
|
||||
for bad in ["", "relative/trip", ".", "a/.."] {
|
||||
let op = Op::SetFolderRecord { folder: bad.into(), record: Some(FolderRecord { label: Some(ColorLabel::Red) }) };
|
||||
assert!(c.apply(op).is_err(), "{bad:?} names no folder");
|
||||
}
|
||||
assert_eq!(c.to_snapshot(), before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_moved_folder_takes_its_labels_and_its_subfolders_labels_along() {
|
||||
let mut c = library();
|
||||
label(&mut c, "/pics/trip", Some(ColorLabel::Red));
|
||||
label(&mut c, "/pics/trip/day1", Some(ColorLabel::Green));
|
||||
label(&mut c, "/pics/trip2", Some(ColorLabel::Blue));
|
||||
let ops = c.folder_records_follow("/pics/trip", "/archive/2026/trip");
|
||||
c.apply(Op::Batch { ops }).unwrap();
|
||||
assert_eq!(c.folder_record("/pics/trip"), None);
|
||||
assert_eq!(c.folder_color_label("/archive/2026/trip"), Some(ColorLabel::Red));
|
||||
assert_eq!(c.folder_color_label("/archive/2026/trip/day1"), Some(ColorLabel::Green));
|
||||
assert_eq!(c.folder_color_label("/pics/trip2"), Some(ColorLabel::Blue), "a neighbour that only starts the same stays");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn undoing_a_move_brings_back_every_label_also_one_left_at_the_destination() {
|
||||
let mut c = library();
|
||||
label(&mut c, "/pics/trip", Some(ColorLabel::Red));
|
||||
// a folder that once was at the destination left its labels behind
|
||||
label(&mut c, "/pics/holiday", Some(ColorLabel::Purple));
|
||||
label(&mut c, "/pics/holiday/day9", Some(ColorLabel::Yellow));
|
||||
let before = c.to_snapshot();
|
||||
let ops = c.folder_records_follow("/pics/trip", "/pics/holiday");
|
||||
let undo = c.apply(Op::Batch { ops }).unwrap();
|
||||
assert_eq!(c.folder_color_label("/pics/holiday"), Some(ColorLabel::Red), "the moved folder's label wins");
|
||||
assert_eq!(c.folder_record("/pics/holiday/day9"), None, "what was known about the old folder goes");
|
||||
c.apply(undo).unwrap();
|
||||
assert_eq!(c.to_snapshot(), before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folder_without_labels_moves_without_ops() {
|
||||
let c = library();
|
||||
assert!(c.folder_records_follow("/pics/trip", "/elsewhere/trip").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unlabelled_folder_moved_onto_a_stale_label_does_not_inherit_it() {
|
||||
let mut c = library();
|
||||
// a folder that was at the destination once (its photos are gone) left its label behind
|
||||
label(&mut c, "/pics/holiday", Some(ColorLabel::Purple));
|
||||
let ops = c.folder_records_follow("/pics/trip", "/pics/holiday");
|
||||
let undo = c.apply(Op::Batch { ops }).unwrap();
|
||||
assert_eq!(c.folder_record("/pics/holiday"), None, "the folder now there is another one");
|
||||
c.apply(undo).unwrap();
|
||||
assert_eq!(c.folder_color_label("/pics/holiday"), Some(ColorLabel::Purple));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn labels_survive_the_journal_without_a_snapshot() {
|
||||
let m = MemStore::new();
|
||||
let (mut j, mut c, _) = Journal::open(Box::new(m.clone())).unwrap();
|
||||
{
|
||||
let p = "/pics/trip/a.jpg";
|
||||
let id = c.alloc_photo_id();
|
||||
let op = Op::AddPhoto { photo: Box::new(Photo::new(id, Source::File { path: p.into() }, "a.jpg", "JPEG", 6, 4, "2026-01-01")) };
|
||||
c.apply(op.clone()).unwrap();
|
||||
j.append(std::slice::from_ref(&op)).unwrap();
|
||||
}
|
||||
let op = c.folder_label_op("/pics/trip", Some(ColorLabel::Yellow));
|
||||
c.apply(op.clone()).unwrap();
|
||||
j.append(std::slice::from_ref(&op)).unwrap();
|
||||
drop(j);
|
||||
let (_, reopened, r) = Journal::open(Box::new(m)).unwrap();
|
||||
assert!(r.replayed >= 2, "{r:?}");
|
||||
assert_eq!(reopened.folder_color_label("/pics/trip"), Some(ColorLabel::Yellow));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn labels_survive_closing_and_reopening_the_library() {
|
||||
let mut c = library();
|
||||
label(&mut c, "/pics/home", Some(ColorLabel::Green));
|
||||
let reopened = Catalog::from_snapshot(&c.to_snapshot()).unwrap();
|
||||
assert_eq!(reopened.folder_color_label("/pics/home"), Some(ColorLabel::Green));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_library_saved_before_folder_records_opens_without_any() {
|
||||
let c = library();
|
||||
let old = c.to_snapshot();
|
||||
assert!(!old.contains("folder_records"));
|
||||
let reopened = Catalog::from_snapshot(&old).unwrap();
|
||||
assert_eq!(reopened.folder_record("/pics/trip"), None);
|
||||
}
|
||||
@@ -0,0 +1,404 @@
|
||||
//! The library's folder tree: where the imported photos live on disk (see [`crate::folders`]).
|
||||
//!
|
||||
//! Scenarios, in the words of someone browsing their library:
|
||||
//!
|
||||
//! * Given photos imported from several folders, the tree lists those folders and no others,
|
||||
//! each with how many photos it holds, under the disk (volume) they are on.
|
||||
//! * A folder's count includes the folders inside it.
|
||||
//! * Every folder is a row of its own, the ones above the imported photos included.
|
||||
//! * Photos only browsed in Local, deleted photos and demo scenes are not "imported from" anywhere.
|
||||
//! * However a folder is spelled, it is one folder.
|
||||
//! * Choosing a folder shows what was imported from it and from the folders inside it — exactly
|
||||
//! as many photos as the row says.
|
||||
|
||||
use crate::*;
|
||||
|
||||
fn add(c: &mut Catalog, path: &str) -> PhotoId {
|
||||
add_with(c, path, |_| {})
|
||||
}
|
||||
|
||||
fn add_with(c: &mut Catalog, path: &str, tweak: impl FnOnce(&mut Photo)) -> PhotoId {
|
||||
let id = c.alloc_photo_id();
|
||||
let name = path.rsplit(['/', '\\']).next().unwrap_or(path);
|
||||
let mut p = Photo::new(id, Source::File { path: path.into() }, name, "JPEG", 60, 40, "2026-01-01T10:00:00");
|
||||
tweak(&mut p);
|
||||
c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
|
||||
id
|
||||
}
|
||||
|
||||
fn library(paths: &[&str]) -> Catalog {
|
||||
let mut c = Catalog::new();
|
||||
paths.iter().for_each(|p| {
|
||||
add(&mut c, p);
|
||||
});
|
||||
c
|
||||
}
|
||||
|
||||
/// `(name, count, own)` of each node, depth first.
|
||||
fn outline(nodes: &[FolderNode]) -> Vec<(String, usize, usize)> {
|
||||
fn walk(n: &FolderNode, out: &mut Vec<(String, usize, usize)>) {
|
||||
out.push((n.name.clone(), n.count, n.own));
|
||||
n.children.iter().for_each(|c| walk(c, out));
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
nodes.iter().for_each(|n| walk(n, &mut out));
|
||||
out
|
||||
}
|
||||
|
||||
fn row(name: &str, count: usize, own: usize) -> (String, usize, usize) {
|
||||
(name.to_string(), count, own)
|
||||
}
|
||||
|
||||
fn rows(paths: &[&str]) -> Vec<(String, usize, usize)> {
|
||||
outline(&library(paths).folder_tree())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_library_has_no_folders() {
|
||||
assert!(Catalog::new().folder_tree().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_folders_with_imported_photos_are_listed_with_their_counts() {
|
||||
let c = library(&["/pics/trip/a.jpg", "/pics/trip/b.jpg", "/pics/home/c.jpg"]);
|
||||
let tree = c.folder_tree();
|
||||
assert_eq!(outline(&tree), vec![row("/", 3, 0), row("pics", 3, 0), row("home", 1, 1), row("trip", 2, 2)]);
|
||||
assert!(tree[0].volume && !tree[0].children[0].volume);
|
||||
assert_eq!(tree[0].children[0].children[1].path, "/pics/trip", "a row knows its folder, for choosing it");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folders_count_includes_the_folders_inside_it() {
|
||||
let r = rows(&["/pics/trip/a.jpg", "/pics/trip/day1/b.jpg", "/pics/trip/day1/c.jpg", "/pics/trip/day2/d.jpg"]);
|
||||
assert_eq!(r, vec![row("/", 4, 0), row("pics", 4, 0), row("trip", 4, 1), row("day1", 2, 2), row("day2", 1, 1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_folder_is_a_row_of_its_own_even_when_it_holds_nothing_itself() {
|
||||
let c = library(&["/Users/me/Pictures/2024/a.jpg", "/Users/me/Pictures/2025/b.jpg", "/Volumes/nas/photos/c.jpg"]);
|
||||
let tree = c.folder_tree();
|
||||
assert_eq!(
|
||||
outline(&tree),
|
||||
vec![
|
||||
row("/", 2, 0),
|
||||
row("Users", 2, 0),
|
||||
row("me", 2, 0),
|
||||
row("Pictures", 2, 0),
|
||||
row("2024", 1, 1),
|
||||
row("2025", 1, 1),
|
||||
row("nas", 1, 0),
|
||||
row("photos", 1, 1)
|
||||
]
|
||||
);
|
||||
assert_eq!(tree[0].children[0].children[0].children[0].path, "/Users/me/Pictures", "a row is exactly the folder it names");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folder_with_photos_of_its_own_is_not_folded_into_its_child() {
|
||||
assert_eq!(rows(&["/pics/a.jpg", "/pics/trip/b.jpg"]), vec![row("/", 2, 0), row("pics", 2, 1), row("trip", 1, 1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn browsed_deleted_and_demo_photos_are_not_imported_from_anywhere() {
|
||||
let mut c = Catalog::new();
|
||||
add(&mut c, "/pics/kept/a.jpg");
|
||||
add_with(&mut c, "/pics/browsed/b.jpg", |p| p.local = true);
|
||||
add_with(&mut c, "/pics/gone/c.jpg", |p| p.deleted = true);
|
||||
let id = c.alloc_photo_id();
|
||||
let demo = Photo::new(id, Source::Demo { scene: 1 }, "demo", "JPEG", 60, 40, "2026-01-01T10:00:00");
|
||||
c.apply(Op::AddPhoto { photo: Box::new(demo) }).unwrap();
|
||||
assert_eq!(outline(&c.folder_tree()), vec![row("/", 1, 0), row("pics", 1, 0), row("kept", 1, 1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn however_a_folder_is_spelled_it_is_one_folder() {
|
||||
let r = rows(&["/pics/trip/a.jpg", "/pics//trip/./b.jpg", "/pics\\trip\\c.jpg", "/pics/x/../trip/d.jpg"]);
|
||||
assert_eq!(r, vec![row("/", 4, 0), row("pics", 4, 0), row("trip", 4, 4)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_disk_is_a_volume_of_its_own() {
|
||||
// macOS, Linux and Windows habits for where disks appear
|
||||
let r = rows(&["/Volumes/nas/p/1.jpg", "/media/me/usb/2.jpg", "/run/media/me/card/3.jpg", "/mnt/disk/4.jpg"]);
|
||||
assert_eq!(r, vec![row("card", 1, 1), row("disk", 1, 1), row("nas", 1, 0), row("p", 1, 1), row("usb", 1, 1)]);
|
||||
let r = rows(&["D:\\Photos\\2024\\a.jpg", "D:\\Photos\\2025\\b.jpg", "C:\\x\\c.jpg"]);
|
||||
assert_eq!(r, vec![row("C:", 1, 0), row("x", 1, 1), row("D:", 2, 0), row("Photos", 2, 0), row("2024", 1, 1), row("2025", 1, 1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_network_share_is_a_volume_and_two_spellings_are_one() {
|
||||
let r = rows(&["\\\\srv\\s1\\a\\1.jpg", "\\\\srv\\s2\\b\\2.jpg"]);
|
||||
assert_eq!(r, vec![row("//srv/s1", 1, 0), row("a", 1, 1), row("//srv/s2", 1, 0), row("b", 1, 1)]);
|
||||
// the verbatim form Windows APIs give is the same share
|
||||
let r = rows(&["\\\\?\\UNC\\srv\\s1\\a\\1.jpg", "\\\\srv\\s1\\a\\2.jpg"]);
|
||||
assert_eq!(r, vec![row("//srv/s1", 2, 0), row("a", 2, 2)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_samba_share_mounted_on_a_mac_is_a_volume_named_after_the_share() {
|
||||
// Finder mounts every share under /Volumes, and a second mount of the same share gets a suffix
|
||||
let r = rows(&["/Volumes/tokyo/Photos/2024/a.jpg", "/Volumes/tokyo/Photos/2025/b.jpg", "/Volumes/tokyo-1/old/c.jpg", "/Volumes/東京 NAS/d.jpg"]);
|
||||
assert_eq!(
|
||||
r,
|
||||
vec![
|
||||
row("tokyo", 2, 0),
|
||||
row("Photos", 2, 0),
|
||||
row("2024", 1, 1),
|
||||
row("2025", 1, 1),
|
||||
row("tokyo-1", 1, 0),
|
||||
row("old", 1, 1),
|
||||
row("東京 NAS", 1, 1)
|
||||
]
|
||||
);
|
||||
let tree = library(&["/Volumes/tokyo/Photos/a.jpg"]).folder_tree();
|
||||
assert_eq!((tree[0].path.as_str(), tree[0].volume), ("/Volumes/tokyo", true), "choosing the share shows everything on it");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shares_the_other_systems_mount_are_volumes_too() {
|
||||
// Linux desktops: gvfs mounts per user, with the server and share in the mount's name
|
||||
let gvfs = "/run/user/1000/gvfs/smb-share:server=tokyo,share=photos";
|
||||
let r = rows(&[&format!("{gvfs}/2024/a.jpg"), "/run/user/1000/gvfs/sftp:host=nas/b.jpg"]);
|
||||
assert_eq!(r, vec![row("photos on tokyo", 1, 0), row("2024", 1, 1), row("sftp:host=nas", 1, 1)]);
|
||||
// NFS automounts: Linux /net/<host>, macOS /Network/Servers/<host>
|
||||
let r = rows(&["/net/tokyo/export/a.jpg", "/Network/Servers/osaka/vol/b.jpg"]);
|
||||
assert_eq!(r, vec![row("osaka", 1, 0), row("vol", 1, 1), row("tokyo", 1, 0), row("export", 1, 1)]);
|
||||
// Windows: a share by name or by address, the WSL file system, a mapped drive
|
||||
let r = rows(&[r"\\tokyo.local\photos\a.jpg", r"\\192.168.1.5\scans\b.jpg", r"\\wsl$\Ubuntu\home\me\c.jpg", r"Z:\d.jpg"]);
|
||||
assert_eq!(
|
||||
r,
|
||||
vec![
|
||||
row("//192.168.1.5/scans", 1, 1),
|
||||
row("//tokyo.local/photos", 1, 1),
|
||||
row("//wsl$/Ubuntu", 1, 0),
|
||||
row("home", 1, 0),
|
||||
row("me", 1, 1),
|
||||
row("Z:", 1, 1)
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_share_that_is_not_mounted_costs_nothing() {
|
||||
// the tree is made from the catalog's own records: it never looks at a disk, so a share that
|
||||
// is asleep or gone lists (and counts) exactly like one that is there
|
||||
let c = library(&["/Volumes/definitely-not-mounted/a/b.jpg", r"\\unreachable.invalid\share\c.jpg"]);
|
||||
let tree = c.folder_tree();
|
||||
assert_eq!(tree.iter().map(|n| n.count).sum::<usize>(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_photo_in_the_root_of_a_disk_counts_for_the_disk() {
|
||||
assert_eq!(rows(&["/x.jpg", "/d/y.jpg"]), vec![row("/", 2, 1), row("d", 1, 1)]);
|
||||
assert_eq!(rows(&["C:\\x.jpg"]), vec![row("C:", 1, 1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn photos_with_no_folder_are_left_out() {
|
||||
assert!(rows(&["x.jpg", ""]).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn folders_are_listed_by_name_ignoring_case() {
|
||||
let names: Vec<String> = rows(&["/p/b/1.jpg", "/p/A/2.jpg", "/p/c/3.jpg"]).into_iter().skip(2).map(|r| r.0).collect();
|
||||
assert_eq!(names, ["A", "b", "c"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absurdly_deep_paths_neither_crash_nor_lose_photos() {
|
||||
let deep = format!("/{}/a.jpg", vec!["d"; 5000].join("/"));
|
||||
let tree = library(&[&deep]).folder_tree();
|
||||
assert_eq!(tree.iter().map(|n| n.count).sum::<usize>(), 1);
|
||||
assert!(outline(&tree).len() <= crate::folders::MAX_DEPTH + 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn choosing_a_folder_shows_what_was_imported_from_it_and_below() {
|
||||
let mut c = Catalog::new();
|
||||
let trip = add(&mut c, "/pics/trip/a.jpg");
|
||||
let day = add(&mut c, "/pics/trip/day1/b.jpg");
|
||||
let sibling = add(&mut c, "/pics/trip2/c.jpg"); // shares the name's start, not the folder
|
||||
let browsed = add_with(&mut c, "/pics/trip/d.jpg", |p| p.local = true);
|
||||
let f = Filter { library_folder: Some("/pics//trip/".into()), ..Default::default() };
|
||||
let shown = |id: PhotoId| f.matches(c.photo(id).unwrap(), &c);
|
||||
assert!(shown(trip) && shown(day));
|
||||
assert!(!shown(sibling), "/pics/trip2 is another folder");
|
||||
assert!(!shown(browsed), "browsed photos are not part of the library's folders");
|
||||
assert_eq!(f.describe(), "folder /pics//trip/");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recently_deleted_shows_the_deleted_photos_of_the_chosen_folder() {
|
||||
let mut c = Catalog::new();
|
||||
let gone = add_with(&mut c, "/pics/trip/a.jpg", |p| p.deleted = true);
|
||||
let kept = add(&mut c, "/pics/trip/b.jpg");
|
||||
let f = Filter { library_folder: Some("/pics/trip".into()), deleted: true, ..Default::default() };
|
||||
assert!(f.matches(c.photo(gone).unwrap(), &c));
|
||||
assert!(!f.matches(c.photo(kept).unwrap(), &c));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_folder_choice_is_no_choice() {
|
||||
let c = library(&["/pics/a.jpg"]);
|
||||
let f = Filter { library_folder: Some(" ".into()), ..Default::default() };
|
||||
assert_eq!(c.query(&f, &Sort::default()).len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_row_shows_exactly_as_many_photos_as_it_says() {
|
||||
let c = library(&[
|
||||
"/Users/me/Pictures/2024/a.jpg",
|
||||
"/Users/me/Pictures/2024/b.jpg",
|
||||
"/Users/me/Pictures/2025/c.jpg",
|
||||
"/Users/me/Pictures/d.jpg",
|
||||
"/Volumes/nas/photos/e.jpg",
|
||||
"/Volumes/nas/f.jpg",
|
||||
"/x.jpg",
|
||||
"/a/../b/g.jpg",
|
||||
"D:\\Photos\\2024\\h.jpg",
|
||||
"d:/Photos/i.jpg",
|
||||
"\\\\srv\\s1\\a\\j.jpg",
|
||||
"\\\\?\\UNC\\srv\\s2\\k.jpg",
|
||||
"/pics/trip/l.jpg",
|
||||
"/pics/trip2/m.jpg",
|
||||
"/Volumes/tokyo/Photos/2024/n.jpg",
|
||||
"/Volumes/tokyo/o.jpg",
|
||||
"/run/user/1000/gvfs/smb-share:server=tokyo,share=photos/p.jpg",
|
||||
"/net/osaka/export/q.jpg",
|
||||
r"\\wsl$\Ubuntu\home\me\r.jpg",
|
||||
// odd spellings the tree and the filter must read the same way
|
||||
"//server/share/../other/1.jpg",
|
||||
"//server/share/ok/2.jpg",
|
||||
"//a b//A/f.jpg",
|
||||
r"\\wsl$\Ubuntu\..\a\1.jpg",
|
||||
// a folder that holds other disks as well as photos of its own
|
||||
"/Volumes/1.jpg",
|
||||
"/mnt/2.jpg",
|
||||
"/mnt/disk/3.jpg",
|
||||
]);
|
||||
fn check(c: &Catalog, nodes: &[FolderNode], seen: &mut usize) {
|
||||
for n in nodes {
|
||||
// the startup volume's row only opens: its path "/" would cover every other disk
|
||||
if n.selectable {
|
||||
let f = Filter { library_folder: Some(n.path.clone()), ..Default::default() };
|
||||
assert_eq!(c.query(&f, &Sort::default()).len(), n.count, "{} ({})", n.name, n.path);
|
||||
*seen += 1;
|
||||
}
|
||||
check(c, &n.children, seen);
|
||||
}
|
||||
}
|
||||
let mut seen = 0;
|
||||
check(&c, &c.folder_tree(), &mut seen);
|
||||
assert!(seen > 10, "{seen} rows checked");
|
||||
}
|
||||
|
||||
fn find<'a>(nodes: &'a [FolderNode], path: &str) -> Option<&'a FolderNode> {
|
||||
nodes.iter().find_map(|n| if n.path == path { Some(n) } else { find(&n.children, path) })
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_row_that_would_cover_other_disks_is_not_offered_for_choosing() {
|
||||
let tree = library(&["/Users/me/a.jpg", "/Volumes/1.jpg", "/Volumes/tokyo/x.jpg", "/mnt/2.jpg"]).folder_tree();
|
||||
let sel = |p: &str| find(&tree, p).map(|n| n.selectable);
|
||||
assert_eq!(sel("/"), Some(false), "the startup disk's path covers every disk");
|
||||
assert_eq!(sel("/Volumes"), Some(false), "it holds the tokyo disk");
|
||||
assert_eq!(sel("/Volumes/tokyo"), Some(true));
|
||||
assert_eq!(sel("/Users/me"), Some(true));
|
||||
assert_eq!(sel("/mnt"), Some(true), "no disk is mounted below it in this library");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_tree_and_the_filter_read_odd_spellings_the_same_way() {
|
||||
// `..` cannot climb above a share, and repeated separators between server and share are one
|
||||
let tree = library(&["//server/share/../other/1.jpg", "//a b//A/f.jpg"]).folder_tree();
|
||||
assert_eq!(outline(&tree), vec![row("//a b/A", 1, 1), row("//server/share", 1, 0), row("other", 1, 1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_path_that_names_no_folder_chooses_nothing() {
|
||||
let c = library(&["/pics/a.jpg", "C:\\b\\c.jpg"]);
|
||||
for dot in [".", "./", "a/..", "../.."] {
|
||||
let f = Filter { library_folder: Some(dot.into()), ..Default::default() };
|
||||
assert_eq!(c.query(&f, &Sort::default()).len(), 0, "{dot:?} is no folder, not everything");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folder_is_labelled_by_its_last_two_names() {
|
||||
use crate::folders::folder_label;
|
||||
assert_eq!(folder_label("/Volumes/tokyo/photos/travel"), "photos/travel");
|
||||
assert_eq!(folder_label("/Volumes/tokyo/"), "Volumes/tokyo");
|
||||
assert_eq!(folder_label(r"C:\Photos\2024"), "Photos/2024");
|
||||
assert_eq!(folder_label("/"), "/");
|
||||
assert_eq!(folder_label(""), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_roots_are_told_from_folders() {
|
||||
use crate::folders::{is_disk_root, is_startup_disk};
|
||||
for p in ["/", "C:", "C:\\", r"\\?\C:\", r"\\srv\share", "/Volumes/nas", "/mnt/disk", "/net/host"] {
|
||||
assert!(is_disk_root(p), "{p}");
|
||||
}
|
||||
for p in ["/Volumes", "/Users", "C:\\Photos", r"\\srv\share\x", "/Volumes/nas/p", "."] {
|
||||
assert!(!is_disk_root(p), "{p}");
|
||||
}
|
||||
assert!(is_startup_disk("/") && is_startup_disk("//") && !is_startup_disk("/Volumes/nas") && !is_startup_disk("C:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folder_name_with_spaces_at_its_edge_is_its_own_folder() {
|
||||
// legal on macOS and Linux: `/p/shoot ` is not `/p/shoot`
|
||||
let c = library(&["/p/shoot /a.jpg", "/p/shoot/b.jpg", "/p/other/c.jpg"]);
|
||||
let tree = c.folder_tree();
|
||||
let shoot_space = find(&tree, "/p/shoot ").expect("its own row");
|
||||
let shoot = find(&tree, "/p/shoot").expect("and the other");
|
||||
for (node, want) in [(shoot_space, "a.jpg"), (shoot, "b.jpg")] {
|
||||
let f = Filter { library_folder: Some(node.path.clone()), ..Default::default() };
|
||||
let ids = c.query(&f, &Sort::default());
|
||||
let names: Vec<&str> = ids.iter().filter_map(|id| c.photo(*id)).map(|p| p.file_name.as_str()).collect();
|
||||
assert_eq!(names, [want], "{:?} shows its own photo", node.path);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_local_photo_never_counts_as_imported_even_when_a_disk_folder_is_also_shown() {
|
||||
let mut c = Catalog::new();
|
||||
let browsed = add_with(&mut c, "/pics/trip/b.jpg", |p| p.local = true);
|
||||
let f = Filter { folder: Some("/pics/trip".into()), library_folder: Some("/pics/trip".into()), ..Default::default() };
|
||||
assert!(!f.matches(c.photo(browsed).unwrap(), &c));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_quick_check_agrees_with_the_careful_one() {
|
||||
use crate::query::{folder_key, key_within, photo_in_root};
|
||||
let paths = [
|
||||
"/a/b/c.jpg",
|
||||
"/a/b/d/e.jpg",
|
||||
"/a/bc/x.jpg",
|
||||
"/a/./b/x.jpg",
|
||||
"/a/../a/b/x.jpg",
|
||||
"/a//b/x.jpg",
|
||||
"/a\\b\\x.jpg",
|
||||
"/a/b ",
|
||||
"/a/b /x.jpg",
|
||||
"/.hidden/x.jpg",
|
||||
"/a/.b/x.jpg",
|
||||
"/x.jpg",
|
||||
"//srv/share/x.jpg",
|
||||
"C:\\a\\b\\x.jpg",
|
||||
"relative/a/b/x.jpg",
|
||||
"/Volumes/tokyo/a/x.jpg",
|
||||
"/日本/写真/x.jpg",
|
||||
];
|
||||
let roots = ["/", "/a", "/a/b", "/a/b/", "/a/bc", "/a/b ", "/Volumes/tokyo", "//srv/share", "C:\\a", "/日本", "relative", "/a/../a/b"];
|
||||
for p in paths {
|
||||
let mut c = Catalog::new();
|
||||
let id = add(&mut c, p);
|
||||
let photo = c.photo(id).unwrap();
|
||||
for r in roots {
|
||||
let root = folder_key(r);
|
||||
let slow = crate::local::folder_of(photo).is_some_and(|f| key_within(&f, &root));
|
||||
assert_eq!(photo_in_root(photo, &root), slow, "{p:?} in {r:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
//! Issue #102: catalog format versions. Older formats load and are upgraded; a newer format is
|
||||
//! refused with a clear error and left untouched (never read in part, truncated or set aside).
|
||||
|
||||
use crate::journal::{LOG, SNAPSHOT, VERSION, encode_record};
|
||||
use crate::*;
|
||||
|
||||
fn photo(c: &mut Catalog) -> Op {
|
||||
let id = c.alloc_photo_id();
|
||||
Op::AddPhoto { photo: Box::new(Photo::new(id, Source::File { path: format!("/p/{}.jpg", id.0) }, "p.jpg", "JPEG", 4, 3, "2026-01-01")) }
|
||||
}
|
||||
|
||||
/// A catalog with two photos and the log line for a third op on top of it.
|
||||
fn legacy_parts() -> (Catalog, String, Catalog) {
|
||||
let mut c = Catalog::new();
|
||||
for _ in 0..2 {
|
||||
let op = photo(&mut c);
|
||||
c.apply(op).unwrap();
|
||||
}
|
||||
let base = c.clone();
|
||||
let id = c.photos().next().unwrap().id;
|
||||
let op = Op::SetRating { id, rating: 4 };
|
||||
c.apply(op.clone()).unwrap();
|
||||
(base, format!("{}\n", encode_record(3, &op)), c)
|
||||
}
|
||||
|
||||
fn snapshot_version(m: &MemStore) -> u64 {
|
||||
let v: serde_json::Value = serde_json::from_slice(&m.get(SNAPSHOT).unwrap()).unwrap();
|
||||
v["version"].as_u64().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v1_library_loads_and_is_upgraded() {
|
||||
let (base, log, full) = legacy_parts();
|
||||
let m = MemStore::new();
|
||||
m.set(SNAPSHOT, format!("{{\"format\":\"lightcraft-catalog\",\"version\":1,\"seq\":2,\"catalog\":{}}}\n", base.to_snapshot()).into_bytes());
|
||||
m.set(LOG, log.into_bytes());
|
||||
let (mut j, c, r) = Journal::open(Box::new(m.clone())).unwrap();
|
||||
assert_eq!(c.to_snapshot(), full.to_snapshot());
|
||||
assert_eq!((r.replayed, r.upgraded_from), (1, Some(1)));
|
||||
// rewritten in this format before anything is appended
|
||||
assert_eq!(snapshot_version(&m), u64::from(VERSION));
|
||||
assert!(m.get(LOG).unwrap().is_empty());
|
||||
assert_eq!(j.seq(), 3);
|
||||
j.append(&[Op::SetLabelName { label: ColorLabel::ALL[0], name: Some("x".into()) }]).unwrap();
|
||||
drop(j);
|
||||
let (_, _, r) = Journal::open(Box::new(m)).unwrap();
|
||||
assert_eq!((r.replayed, r.upgraded_from), (1, None));
|
||||
}
|
||||
|
||||
/// A library saved before albums could be ordered by hand opens as it was and is rewritten in the current format.
|
||||
#[test]
|
||||
fn v2_library_loads_and_is_upgraded() {
|
||||
let (base, log, full) = legacy_parts();
|
||||
let m = MemStore::new();
|
||||
m.set(SNAPSHOT, format!("{{\"format\":\"lightcraft-catalog\",\"version\":2,\"seq\":2,\"catalog\":{}}}\n", base.to_snapshot()).into_bytes());
|
||||
m.set(LOG, log.into_bytes());
|
||||
let (_, c, r) = Journal::open(Box::new(m.clone())).unwrap();
|
||||
assert_eq!(c.to_snapshot(), full.to_snapshot());
|
||||
assert_eq!((r.replayed, r.upgraded_from), (1, Some(2)));
|
||||
assert_eq!(snapshot_version(&m), u64::from(VERSION));
|
||||
}
|
||||
|
||||
/// A library saved before keywords could be listed on their own opens as it was and is rewritten
|
||||
/// in the current format.
|
||||
#[test]
|
||||
fn v3_library_loads_and_is_upgraded() {
|
||||
let (base, log, full) = legacy_parts();
|
||||
let m = MemStore::new();
|
||||
m.set(SNAPSHOT, format!("{{\"format\":\"lightcraft-catalog\",\"version\":3,\"seq\":2,\"catalog\":{}}}\n", base.to_snapshot()).into_bytes());
|
||||
m.set(LOG, log.into_bytes());
|
||||
let (_, c, r) = Journal::open(Box::new(m.clone())).unwrap();
|
||||
assert_eq!(c.to_snapshot(), full.to_snapshot());
|
||||
assert_eq!((r.replayed, r.upgraded_from), (1, Some(3)));
|
||||
assert_eq!(snapshot_version(&m), u64::from(VERSION));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn versionless_log_only_library_is_upgraded() {
|
||||
let mut c = Catalog::new();
|
||||
let op = photo(&mut c);
|
||||
c.apply(op.clone()).unwrap();
|
||||
let m = MemStore::new();
|
||||
m.set(LOG, format!("{}\n", encode_record(1, &op)).into_bytes());
|
||||
let (_, loaded, r) = Journal::open(Box::new(m.clone())).unwrap();
|
||||
assert_eq!(loaded.to_snapshot(), c.to_snapshot());
|
||||
assert_eq!(r.upgraded_from, Some(0));
|
||||
assert_eq!(snapshot_version(&m), u64::from(VERSION));
|
||||
}
|
||||
|
||||
/// A new library gets a snapshot in this format at once, so an older build never sees a bare log
|
||||
/// it would read in part.
|
||||
#[test]
|
||||
fn new_library_is_written_in_this_format() {
|
||||
let m = MemStore::new();
|
||||
let (_, _, r) = Journal::open(Box::new(m.clone())).unwrap();
|
||||
assert!(r.created);
|
||||
assert_eq!(r.upgraded_from, None);
|
||||
assert_eq!(snapshot_version(&m), u64::from(VERSION));
|
||||
}
|
||||
|
||||
fn assert_refused_untouched(m: &MemStore) {
|
||||
let before = m.files.lock().unwrap().clone();
|
||||
let e = Journal::open(Box::new(m.clone())).err().expect("refused");
|
||||
assert!(matches!(e, CatalogError::Newer(_)), "{e:?}");
|
||||
let msg = e.to_string();
|
||||
assert!(msg.contains("newer version of LightCraft") && msg.contains("left unchanged"), "{msg}");
|
||||
assert_eq!(*m.files.lock().unwrap(), before, "nothing modified, nothing renamed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn newer_snapshot_is_refused_untouched() {
|
||||
let (base, log, _) = legacy_parts();
|
||||
let m = MemStore::new();
|
||||
m.set(
|
||||
SNAPSHOT,
|
||||
format!("{{\"format\":\"lightcraft-catalog\",\"version\":{},\"seq\":2,\"catalog\":{}}}\n", VERSION + 1, base.to_snapshot()).into_bytes(),
|
||||
);
|
||||
m.set(LOG, log.into_bytes());
|
||||
assert_refused_untouched(&m);
|
||||
// even when its catalog doesn't parse as this version's
|
||||
let m = MemStore::new();
|
||||
m.set(
|
||||
SNAPSHOT,
|
||||
format!("{{\"format\":\"lightcraft-catalog\",\"version\":{},\"seq\":2,\"catalog\":{{\"photos\":7}}}}\n", VERSION + 1).into_bytes(),
|
||||
);
|
||||
assert_refused_untouched(&m);
|
||||
let e = Journal::open(Box::new(m)).err().unwrap().to_string();
|
||||
assert!(e.contains(&format!("v{}", VERSION + 1)), "{e}");
|
||||
}
|
||||
|
||||
/// A log record with a valid CRC whose op this version doesn't know: refused, not taken for a
|
||||
/// torn tail (truncated) or damage (renamed) — wherever it is in the log.
|
||||
#[test]
|
||||
fn unknown_op_in_the_log_is_refused_untouched() {
|
||||
let (base, log, _) = legacy_parts();
|
||||
let future = |seq: u64| {
|
||||
let body = r#"{"op":"fromTheFuture","id":1,"what":"something new"}"#;
|
||||
format!("{{\"seq\":{seq},\"crc\":{},\"op\":{body}}}\n", crc32fast::hash(body.as_bytes()))
|
||||
};
|
||||
let snap = format!("{{\"format\":\"lightcraft-catalog\",\"version\":{VERSION},\"seq\":2,\"catalog\":{}}}\n", base.to_snapshot());
|
||||
let id = base.photos().next().unwrap().id;
|
||||
let after = format!("{}\n", encode_record(5, &Op::SetRating { id, rating: 1 }));
|
||||
for log in [format!("{log}{}", future(4)), format!("{log}{}{after}", future(4)), format!("{}{log}", future(3))] {
|
||||
let m = MemStore::new();
|
||||
m.set(SNAPSHOT, snap.clone().into_bytes());
|
||||
m.set(LOG, log.into_bytes());
|
||||
assert_refused_untouched(&m);
|
||||
}
|
||||
// a known op with a field of an unknown shape is "newer" too
|
||||
let body = r#"{"op":"setRating","id":1,"rating":"five stars"}"#;
|
||||
let m = MemStore::new();
|
||||
m.set(SNAPSHOT, snap.into_bytes());
|
||||
m.set(LOG, format!("{log}{{\"seq\":4,\"crc\":{},\"op\":{body}}}\n", crc32fast::hash(body.as_bytes())).into_bytes());
|
||||
assert_refused_untouched(&m);
|
||||
}
|
||||
|
||||
/// A CRC mismatch is still damage / a torn tail (not "newer").
|
||||
#[test]
|
||||
fn bad_crc_is_still_a_torn_tail() {
|
||||
let (base, log, _) = legacy_parts();
|
||||
let m = MemStore::new();
|
||||
m.set(
|
||||
SNAPSHOT,
|
||||
format!("{{\"format\":\"lightcraft-catalog\",\"version\":{VERSION},\"seq\":2,\"catalog\":{}}}\n", base.to_snapshot()).into_bytes(),
|
||||
);
|
||||
let body = r#"{"op":"fromTheFuture"}"#;
|
||||
m.set(LOG, format!("{log}{{\"seq\":4,\"crc\":1,\"op\":{body}}}\n").into_bytes());
|
||||
let (_, _, r) = Journal::open(Box::new(m)).unwrap();
|
||||
assert!(r.torn_bytes > 0 && r.damaged.is_none(), "{r:?}");
|
||||
}
|
||||
|
||||
/// Guard for the version rule (see `journal` → *Format versions*): adding an [`Op`] variant breaks
|
||||
/// this match. When it does, bump `journal::VERSION`, add a row to the format table, and add the
|
||||
/// variant here under the new version.
|
||||
#[test]
|
||||
fn op_variants_are_versioned() {
|
||||
fn since(op: &Op) -> u32 {
|
||||
match op {
|
||||
Op::AddPhoto { .. }
|
||||
| Op::RemovePhoto { .. }
|
||||
| Op::SetRating { .. }
|
||||
| Op::SetFlag { .. }
|
||||
| Op::SetLabel { .. }
|
||||
| Op::SetDevelop { .. }
|
||||
| Op::SetMeta { .. }
|
||||
| Op::SetDeleted { .. }
|
||||
| Op::SetLocal { .. }
|
||||
| Op::SetVersions { .. }
|
||||
| Op::SetHistory { .. }
|
||||
| Op::PushHistory { .. }
|
||||
| Op::AddAlbum { .. }
|
||||
| Op::RemoveAlbum { .. }
|
||||
| Op::RenameAlbum { .. }
|
||||
| Op::MoveAlbum { .. }
|
||||
| Op::SetAlbumPhotos { .. }
|
||||
| Op::SetAlbumCover { .. }
|
||||
| Op::SetAlbumRules { .. }
|
||||
| Op::AddStack { .. }
|
||||
| Op::RemoveStack { .. }
|
||||
| Op::SetStack { .. }
|
||||
| Op::SetCaptured { .. }
|
||||
| Op::SetAnalysis { .. }
|
||||
| Op::SetFile { .. }
|
||||
| Op::Relink { .. }
|
||||
| Op::SetContent { .. }
|
||||
| Op::SetLabelName { .. }
|
||||
| Op::Batch { .. } => 1,
|
||||
Op::SetBrowsed { .. } => 2,
|
||||
Op::SetAlbumOrder { .. } => 3,
|
||||
Op::SetEmbeddedLens { .. } => 4,
|
||||
Op::SetKeyword { .. } => 5,
|
||||
Op::SetFolderRecord { .. } => 6,
|
||||
}
|
||||
}
|
||||
let newest = since(&Op::SetFolderRecord { folder: "/".into(), record: None });
|
||||
assert_eq!(newest, VERSION, "the newest op's version must be the current format version");
|
||||
}
|
||||
@@ -59,6 +59,7 @@ fn op_for(c: &mut Catalog, kind: u8, a: u8, b: u8) -> Op {
|
||||
cover: None,
|
||||
smart: None,
|
||||
quick: false,
|
||||
order: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -77,6 +78,16 @@ fn op_for(c: &mut Catalog, kind: u8, a: u8, b: u8) -> Op {
|
||||
13 => Op::Batch { ops: c.remove_from_stacks_ops(&[pid]) },
|
||||
14 => c.set_top_ops(pid).unwrap_or(Op::Batch { ops: vec![] }),
|
||||
9 if b.is_multiple_of(5) => Op::SetLabelName { label: ColorLabel::ALL[a as usize % 5], name: (a % 3 != 1).then(|| format!("Label {b}")) },
|
||||
9 if b % 7 == 5 => Op::SetKeyword {
|
||||
path: format!("travel|place {}", b % 4),
|
||||
info: (!b.is_multiple_of(3)).then(|| crate::keywords::KeywordInfo {
|
||||
person: a.is_multiple_of(2),
|
||||
synonyms: vec![format!("trip {a}")],
|
||||
..Default::default()
|
||||
}),
|
||||
},
|
||||
9 if b % 11 == 4 => c.folder_label_op(&format!("/p/f{}", a % 4), ColorLabel::ALL.get(usize::from(b % 6)).copied()),
|
||||
9 if b % 7 == 3 => Op::SetAlbumOrder { id: aid, order: (!b.is_multiple_of(3)).then_some(u32::from(b % 5)) },
|
||||
9 if b.is_multiple_of(2) => {
|
||||
Op::SetFile { id: pid, file_name: format!("Renamed_{b}.jpg"), source: Source::File { path: format!("/p/Renamed_{b}.jpg") } }
|
||||
}
|
||||
@@ -283,7 +294,12 @@ fn streamed_snapshot_keeps_the_on_disk_format() {
|
||||
let (mut j, c, _) = open(&m);
|
||||
assert_eq!(c.to_snapshot(), full.to_snapshot());
|
||||
j.snapshot(&c).unwrap();
|
||||
let legacy = format!("{{\"format\":\"lightcraft-catalog\",\"version\":1,\"seq\":{},\"catalog\":{}}}\n", j.seq(), c.to_snapshot());
|
||||
let legacy = format!(
|
||||
"{{\"format\":\"lightcraft-catalog\",\"version\":{},\"seq\":{},\"catalog\":{}}}\n",
|
||||
crate::journal::VERSION,
|
||||
j.seq(),
|
||||
c.to_snapshot()
|
||||
);
|
||||
assert_eq!(String::from_utf8(m.get(SNAPSHOT).unwrap()).unwrap(), legacy);
|
||||
assert_eq!(j.stats().last_snapshot.bytes, legacy.len() as u64);
|
||||
|
||||
@@ -319,3 +335,44 @@ fn failed_streamed_write_keeps_the_old_file() {
|
||||
assert_eq!(std::fs::read(dir.join(SNAPSHOT)).unwrap(), b"new");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// A damaged rendering process in a saved photo (snapshot or log) reads as V1 and costs nothing
|
||||
/// else: the photo and its edits load (`docs/process-versions.md`).
|
||||
#[test]
|
||||
fn a_damaged_process_value_loads_as_v1_with_the_edits() {
|
||||
use lightcraft_develop::ProcessVersion;
|
||||
let mut c = Catalog::new();
|
||||
let id = c.alloc_photo_id();
|
||||
let mut p = Photo::new(id, Source::Demo { scene: 1 }, "a.jpg", "JPEG", 10, 10, "2026-10-09T00:00:00");
|
||||
p.develop = Arc::new(DevelopSettings {
|
||||
process: ProcessVersion(7),
|
||||
light: lightcraft_develop::Light { exposure: 0.5, ..Default::default() },
|
||||
..Default::default()
|
||||
});
|
||||
c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
|
||||
let snapshot = c.to_snapshot();
|
||||
assert!(snapshot.contains(r#""process":7"#));
|
||||
let snapshot = format!(
|
||||
"{{\"format\":\"lightcraft-catalog\",\"version\":{},\"seq\":0,\"catalog\":{}}}\n",
|
||||
crate::journal::VERSION,
|
||||
snapshot.replace(r#""process":7"#, r#""process":-1"#)
|
||||
);
|
||||
let m = MemStore::new();
|
||||
m.set(SNAPSHOT, snapshot.into_bytes());
|
||||
let (_, loaded, _) = open(&m);
|
||||
let d = &loaded.photo(id).unwrap().develop;
|
||||
assert_eq!((d.process, d.light.exposure), (ProcessVersion::V1, 0.5));
|
||||
// and in a (CRC-valid) log record
|
||||
let edit = DevelopSettings {
|
||||
process: ProcessVersion(7),
|
||||
light: lightcraft_develop::Light { exposure: 0.8, ..Default::default() },
|
||||
..Default::default()
|
||||
};
|
||||
let op = Op::SetDevelop { id, settings: Arc::new(edit), label: "Edit".into(), edited: None };
|
||||
let body = serde_json::to_string(&op).unwrap().replace(r#""process":7"#, r#""process":"seven""#);
|
||||
assert!(body.contains("seven"));
|
||||
m.set(LOG, format!("{{\"seq\":1,\"crc\":{},\"op\":{body}}}\n", crc32fast::hash(body.as_bytes())).into_bytes());
|
||||
let (_, loaded, _) = open(&m);
|
||||
let d = &loaded.photo(id).unwrap().develop;
|
||||
assert_eq!((d.process, d.light.exposure), (ProcessVersion::V1, 0.8));
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
//! Issue #99: one process per library.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use crate::lock::{LOCK, OWNER};
|
||||
use crate::*;
|
||||
|
||||
fn temp_dir(tag: &str) -> PathBuf {
|
||||
let d = std::env::temp_dir().join(format!("lc-lock-{tag}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
std::fs::create_dir_all(&d).unwrap();
|
||||
d
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn second_opener_is_refused_until_the_first_lets_go() {
|
||||
let dir = temp_dir("twice");
|
||||
let first = LibraryLock::acquire(&dir, "LightCraft").unwrap();
|
||||
assert!(first.held());
|
||||
let owner: LockOwner = serde_json::from_slice(&std::fs::read(dir.join(OWNER)).unwrap()).unwrap();
|
||||
assert_eq!((owner.pid, owner.program.as_str()), (std::process::id(), "LightCraft"));
|
||||
|
||||
let e = LibraryLock::acquire(&dir, "lightcraft-cli").unwrap_err();
|
||||
let LockError::InUse(Some(who)) = &e else { panic!("{e:?}") };
|
||||
assert_eq!(who.pid, std::process::id());
|
||||
let msg = e.to_string();
|
||||
assert!(msg.contains("already open in LightCraft") && msg.contains(&format!("process {}", std::process::id())), "{msg}");
|
||||
|
||||
drop(first);
|
||||
assert!(!dir.join(OWNER).exists(), "the owner note goes with the lock");
|
||||
assert!(dir.join(LOCK).exists(), "the lock file itself may stay: only a held lock refuses");
|
||||
let again = LibraryLock::acquire(&dir, "lightcraft-cli").unwrap();
|
||||
assert!(again.held());
|
||||
drop(again);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// Files left behind by a crash (lock file + owner note) don't lock anyone out.
|
||||
#[test]
|
||||
fn leftover_lock_files_are_not_a_lock() {
|
||||
let dir = temp_dir("stale");
|
||||
std::fs::write(dir.join(LOCK), b"").unwrap();
|
||||
std::fs::write(dir.join(OWNER), br#"{"pid":999999,"host":"elsewhere","program":"LightCraft","version":"0.2.0","since":1}"#).unwrap();
|
||||
let l = LibraryLock::acquire(&dir, "LightCraft").unwrap();
|
||||
assert!(l.held());
|
||||
let owner: LockOwner = serde_json::from_slice(&std::fs::read(dir.join(OWNER)).unwrap()).unwrap();
|
||||
assert_eq!(owner.pid, std::process::id(), "the note now names this process");
|
||||
drop(l);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// Another process holds the lock; when it is killed (a crash), the OS releases it.
|
||||
#[test]
|
||||
fn lock_held_by_another_process_is_released_when_it_dies() {
|
||||
let dir = temp_dir("child");
|
||||
let exe = std::env::current_exe().unwrap();
|
||||
let mut child = std::process::Command::new(exe)
|
||||
.args(["--exact", "tests_lock::lock_holder_child", "--ignored", "--test-threads=1"])
|
||||
.env("LC_LOCK_CHILD_DIR", &dir)
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
|
||||
while !dir.join("child-locked").exists() {
|
||||
assert!(child.try_wait().unwrap().is_none(), "child exited before locking");
|
||||
assert!(std::time::Instant::now() < deadline, "child never locked");
|
||||
std::thread::sleep(std::time::Duration::from_millis(20));
|
||||
}
|
||||
let e = LibraryLock::acquire(&dir, "LightCraft").unwrap_err();
|
||||
let LockError::InUse(Some(who)) = &e else { panic!("{e:?}") };
|
||||
assert_eq!(who.pid, child.id());
|
||||
assert!(e.to_string().contains("on this computer") || who.host.is_empty(), "{e}");
|
||||
|
||||
child.kill().unwrap(); // SIGKILL: no clean-up runs, like a crash
|
||||
child.wait().unwrap();
|
||||
assert!(dir.join(OWNER).exists(), "the crashed holder's note is left behind");
|
||||
let l = LibraryLock::acquire(&dir, "LightCraft").expect("the OS released the dead process's lock");
|
||||
assert!(l.held());
|
||||
drop(l);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// Run by [`lock_held_by_another_process_is_released_when_it_dies`] in a child process: take the
|
||||
/// lock, say so, and wait to be killed.
|
||||
#[test]
|
||||
#[ignore = "helper process for lock_held_by_another_process_is_released_when_it_dies"]
|
||||
fn lock_holder_child() {
|
||||
let Some(dir) = std::env::var_os("LC_LOCK_CHILD_DIR") else { return };
|
||||
let _l = LibraryLock::acquire(std::path::Path::new(&dir), "lightcraft-cli").unwrap();
|
||||
std::fs::write(std::path::Path::new(&dir).join("child-locked"), b"").unwrap();
|
||||
std::thread::sleep(std::time::Duration::from_secs(120));
|
||||
}
|
||||
@@ -12,6 +12,7 @@ workspace = true
|
||||
lightcraft-geom = { workspace = true }
|
||||
lightcraft-color = { workspace = true }
|
||||
lightcraft-raster = { workspace = true }
|
||||
lightcraft-heif = { workspace = true, optional = true }
|
||||
serde = { workspace = true }
|
||||
bytemuck = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
@@ -29,13 +30,21 @@ jxl-oxide = { workspace = true, optional = true }
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
||||
ravif = { workspace = true, optional = true }
|
||||
rav1e = { workspace = true, optional = true }
|
||||
avif-serialize = { workspace = true, optional = true }
|
||||
|
||||
[features]
|
||||
default = ["parallel", "jxl", "avif"]
|
||||
parallel = ["dep:rayon", "lightcraft-raster/parallel", "jpeg-decoder/rayon"]
|
||||
jxl = ["dep:jxl-oxide"]
|
||||
# HEIC/HEIF decode via the optional lightcraft-heif crate (heic-rs, pure Rust). Off by default so
|
||||
# distributors choose it at build time (HEVC patents are a distributor's call, same policy as
|
||||
# PhotoCraft); `cargo xtask ci` clippies and tests it (`--features heif`), releases don't ship it yet.
|
||||
heif = ["dep:lightcraft-heif"]
|
||||
## AVIF encoding via ravif/rav1e (pure Rust, BSD). Native targets only; a no-op on wasm32.
|
||||
avif = ["dep:ravif"]
|
||||
avif = ["dep:ravif", "dep:rav1e", "dep:avif-serialize"]
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = { workspace = true }
|
||||
# Lossless synthetic HEIC files for the `heif` feature's tests.
|
||||
lightcraft-heif = { workspace = true, features = ["testdata"] }
|
||||
+25
-5
@@ -14,6 +14,7 @@ let d: Decoded = decode(&bytes, DecodeOptions::default())?;
|
||||
let working: Rgb32f = to_working(&d); // linear Rec.2020 D65 (Bradford)
|
||||
let fast = decode(&bytes, DecodeOptions::fit(2048, 2048))?; // JPEG: DCT-domain 1/2..1/8 scaling
|
||||
let t: Thumbnail = decode_thumbnail(&bytes, 256)?; // embedded EXIF/MPF preview or scaled decode
|
||||
let h: Header = read_header(&bytes)?; // stored width/height + orientation, no pixels decoded
|
||||
|
||||
let icc = icc::write_named(NamedSpace::DisplayP3); // or icc::write_matrix_trc(&space, &Trc::…)
|
||||
let meta = EncodeMeta { icc: Some(&icc), exif: Some(&exif), xmp: Some(&xmp) };
|
||||
@@ -22,8 +23,20 @@ encode_png(&EncodeImage::new(w, h, 4, Samples::U16(&px)), &meta)?;
|
||||
encode_tiff(&EncodeImage::new(w, h, 3, Samples::F32(&px)), TiffCompression::Deflate, &meta)?;
|
||||
encode_webp_lossless(&img, &meta)?;
|
||||
encode_avif(&img, 70, 6, &meta)?; // native + feature `avif`
|
||||
|
||||
// HDR gain map JPEG (ISO 21496-1): SDR + HDR renditions → base + gain map, and back
|
||||
let (map, gm) = gainmap::compute(&sdr_linear, &hdr_linear, w, h, luma, &GainMapOptions::default())?;
|
||||
gainmap::encode_jpeg(&EncodeImage::new(w, h, 3, Samples::U8(&base8)), &map, &gm, 90, ChromaSubsampling::S444, &meta)?;
|
||||
let found = gainmap::read_jpeg(&bytes); // ours, cameras', Lightroom's
|
||||
gainmap::apply(&mut base_linear, w, h, &found.gain, &found.meta, 1.0)?;
|
||||
```
|
||||
|
||||
Thumbnail fallback decoding limits the source to 64 million pixels by default.
|
||||
Non-JPEG decoders allocate at source resolution before resizing, even for a tiny
|
||||
thumbnail. Use `decode_thumbnail_with` and `ThumbnailOptions::max_pixels` to set
|
||||
a different source budget. Oversized sources return `Error::TooLarge`; embedded
|
||||
JPEG previews can still be used without decoding the full source image.
|
||||
|
||||
`Format::RawTiffLike` (DNG, CR2, NEF, ARW, PEF, ORF, RW2, SRW, …) and `Format::RawOther` (CR3, RAF,
|
||||
CRW, MRW, X3F) are detected so the engine can route them to `lightcraft-raw`; `decode` returns
|
||||
`Error::Unsupported` for them.
|
||||
@@ -39,8 +52,9 @@ CRW, MRW, X3F) are detected so the engine can route them to `lightcraft-raw`; `d
|
||||
| GIF / BMP | yes (first GIF frame) | — | via `image` |
|
||||
| PSD / PSB | merged composite: 8/16/32-bit gray, RGB, CMYK, indexed, duotone (as gray); raw/RLE | — | ICC (1039), EXIF (1058), XMP (1060); ZIP-compressed composite, Lab and 1-bit unsupported |
|
||||
| JPEG XL | yes (jxl-oxide, feature `jxl`, default on) | — | enum colour → rendered straight to linear Rec.2020; ICC → our ICC path; orientation applied by the decoder (reported as 1) |
|
||||
| AVIF | **no** | yes (ravif/rav1e, native only, feature `avif`) | 8-bit sRGB, EXIF; no ICC in the muxer |
|
||||
| HEIC/HEIF | **no** (sniff only) | — | see gaps |
|
||||
| AVIF | **no** | yes (ravif/rav1e, native only, feature `avif`) | 8/10-bit sRGB, EXIF; no ICC in the muxer. HDR: `encode_avif_pq` writes 10-bit BT.2020 PQ (SDR white at 203 cd/m², CICP `nclx` + `clli`) |
|
||||
| HEIC/HEIF | yes with feature `heif` (lightcraft-heif: heic-rs + libheif-matching colour; 4:2:0/mono, 8–10-bit, grids, irot/imir/clap, ICC/nclx/VUI, EXIF/XMP, thumbnail item) | — | 4:2:2/4:4:4 (Canon/Sony HIF) unsupported |
|
||||
| Gain map JPEG (HDR) | gain map found and decoded (`gainmap::read_jpeg`): ISO 21496-1 metadata, else Adobe `hdrgm` XMP | yes (`gainmap::encode_jpeg`) | CIPA DC-007 MPF index (gain map typed `0x050000`), ISO 21496-1 APP2 on both images, `hdrgm` + Container XMP for Android/Chrome, Apple `HDRGainMap`/`HDRToneMap` XMP; gain maps are never used as thumbnails |
|
||||
|
||||
## Colour
|
||||
|
||||
@@ -50,6 +64,10 @@ or CMYK profile converted by the CMS straight to linear Rec.2020), `Container` (
|
||||
EXIF `R03` Adobe RGB hint), `Untagged` (assumed sRGB; untagged float data assumed linear),
|
||||
`IccUnsupported` (profile present but unusable → **sRGB fallback**), `Naive` (CMYK without profile).
|
||||
|
||||
`decode_jpeg_with_fallback(bytes, opts, space)` lets an enclosing RAW container supply the colour
|
||||
space of a JPEG with no ICC or EXIF metadata. It applies the transfer curve before linear-light
|
||||
resizing; a JPEG carrying its own metadata retains its existing interpretation.
|
||||
|
||||
ICC profiles are parsed with `moxcms` (v2/v4, `curv`/`para` TRCs, `chad`); sRGB, Display P3,
|
||||
Adobe RGB (1998), ProPhoto (ROMM) and Rec.2020 are recognised by colorants. `icc::write_matrix_trc`
|
||||
emits v4 display profiles (D50 PCS, Bradford `chad`) for export embedding.
|
||||
@@ -89,9 +107,11 @@ Run `cargo test --release -p lightcraft-codecs --test bench -- --ignored --nocap
|
||||
|
||||
## Gaps / limitations
|
||||
|
||||
- **AVIF and HEIC decode**: no permissively licensed, pure-Rust AV1/HEVC decoder exists today
|
||||
(dav1d/libheif are C; rav1d-safe and imazen's heic are AGPL). Files are sniffed and rejected with
|
||||
`Error::Unsupported`; a raw-style embedded-preview path could be added for HEIC thumbnails.
|
||||
- **AVIF decode**: no permissively licensed, pure-Rust AV1 decoder exists today (dav1d is C;
|
||||
rav1d-safe is AGPL). Files are sniffed and rejected with `Error::Unsupported`.
|
||||
- **HEIC** needs the `heif` feature (off by default: HEVC patents are the distributor's call);
|
||||
without it, `Error::Unsupported` says the build has no HEIC support. 4:2:2 and 4:4:4 HEIF
|
||||
(Canon/Sony HIF) is refused: heic-rs 0.1.1 loses sync on those bitstreams.
|
||||
- JPEG: arithmetic coding unsupported (as in both decoders); extended XMP (> 64 KiB) neither read
|
||||
nor written; EXIF > 64 KiB not written. The encoder uses standard Huffman tables (interleaved
|
||||
baseline) for maximum compatibility.
|
||||
|
||||
@@ -0,0 +1,442 @@
|
||||
//! Monitor (display) ICC profiles: showing photos correctly on wide-gamut and calibrated displays.
|
||||
//!
|
||||
//! The loupe renders straight into the display's own primaries: the develop pipeline targets the
|
||||
//! display's linear RGB ([`DisplayProfile::to_rec2020`]) and encodes it with the sRGB curve, as it
|
||||
//! does for every 8-bit output. That "proxy" encoding then goes to the display's real device
|
||||
//! values with [`DisplayProfile::correct`]: its tone curves for a matrix/TRC profile (a per-channel
|
||||
//! table), or its LUTs for a LUT-based profile (a CMS transform). Images made for sRGB (thumbnails,
|
||||
//! cached previews, embedded camera JPEGs) go through [`DisplayProfile::from_srgb`] instead.
|
||||
//!
|
||||
//! All conversions are relative colorimetric: the display's white is the image's white.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::icc;
|
||||
use crate::space::Trc;
|
||||
use lightcraft_color::{D50, D65, Mat3, REC2020, bradford};
|
||||
use lightcraft_raster::Rgba8;
|
||||
use moxcms::{ColorProfile, DataColorSpace, Layout, RenderingIntent, Transform8BitExecutor, TransformOptions, Xyzd};
|
||||
|
||||
/// Largest profile file accepted (monitor profiles are a few KB; LUT profiles up to a few MB).
|
||||
pub const MAX_PROFILE_BYTES: usize = 32 << 20;
|
||||
|
||||
/// How a display profile describes the display.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum DisplayKind {
|
||||
/// Primaries + tone curves (exact).
|
||||
MatrixTrc,
|
||||
/// LUT-based (A2B/B2A tables): applied through the CMS; the primaries the pipeline targets are
|
||||
/// the profile's matrix tags when it has them, else measured through its tables.
|
||||
Lut,
|
||||
}
|
||||
|
||||
impl DisplayKind {
|
||||
pub fn label(self) -> &'static str {
|
||||
match self {
|
||||
DisplayKind::MatrixTrc => "matrix",
|
||||
DisplayKind::Lut => "lut",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The proxy → device step of a display.
|
||||
enum Correction {
|
||||
/// The proxy encoding is the display's (sRGB-curve matrix profile): nothing to do.
|
||||
None,
|
||||
/// Per-channel tables (matrix/TRC profile: same primaries, other curves).
|
||||
Curves(Box<[[u8; 256]; 3]>),
|
||||
/// A full CMS transform (LUT-based profile).
|
||||
Cms(Arc<Transform8BitExecutor>),
|
||||
}
|
||||
|
||||
/// A parsed display profile with its transforms (built once, applied to every preview).
|
||||
pub struct DisplayProfile {
|
||||
/// The profile's own description (or "Display profile").
|
||||
pub description: String,
|
||||
/// Identifies the profile (hash of its bytes): part of render keys.
|
||||
pub id: u64,
|
||||
pub kind: DisplayKind,
|
||||
/// Linear display RGB → linear Rec.2020 (D65; display white → (1, 1, 1)).
|
||||
pub to_rec2020: Mat3,
|
||||
correction: Correction,
|
||||
from_srgb: Arc<Transform8BitExecutor>,
|
||||
to_srgb: Arc<Transform8BitExecutor>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for DisplayProfile {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("DisplayProfile")
|
||||
.field("description", &self.description)
|
||||
.field("id", &self.id)
|
||||
.field("kind", &self.kind)
|
||||
.field("to_rec2020", &self.to_rec2020)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
fn opts() -> TransformOptions {
|
||||
TransformOptions { rendering_intent: RenderingIntent::RelativeColorimetric, ..TransformOptions::default() }
|
||||
}
|
||||
|
||||
fn xyzd(v: [f64; 3]) -> Xyzd {
|
||||
Xyzd { x: v[0], y: v[1], z: v[2] }
|
||||
}
|
||||
|
||||
/// Linear Rec.2020 → XYZ relative to D50 (the ICC PCS), Bradford-adapted.
|
||||
fn rec2020_to_xyz_d50() -> Mat3 {
|
||||
bradford(D65, D50).mul(&REC2020.to_xyz())
|
||||
}
|
||||
|
||||
/// Scale `m`'s columns so that (1, 1, 1) maps to (1, 1, 1): the display's white is the image's.
|
||||
fn white_balanced(m: Mat3) -> Option<Mat3> {
|
||||
let s = m.inverse()?.apply([1.0; 3]);
|
||||
s.iter().all(|v| v.is_finite() && *v > 0.0).then(|| m.mul(&Mat3::diag(s[0], s[1], s[2])))
|
||||
}
|
||||
|
||||
fn hash(bytes: &[u8]) -> u64 {
|
||||
use std::hash::{Hash, Hasher};
|
||||
let mut h = std::collections::hash_map::DefaultHasher::new();
|
||||
bytes.hash(&mut h);
|
||||
h.finish() | 1
|
||||
}
|
||||
|
||||
/// Runs a moxcms call that may panic on a hostile profile.
|
||||
fn guarded<T>(f: impl FnOnce() -> Option<T>) -> Option<T> {
|
||||
std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).ok().flatten()
|
||||
}
|
||||
|
||||
impl DisplayProfile {
|
||||
/// Parse a display profile (an RGB ICC profile, matrix/TRC or LUT-based) and build its
|
||||
/// transforms. Errors say what is wrong with the file; nothing here panics.
|
||||
pub fn from_icc(bytes: &[u8]) -> Result<DisplayProfile, String> {
|
||||
if bytes.len() > MAX_PROFILE_BYTES {
|
||||
return Err(format!("the profile is too large ({} MB)", bytes.len() >> 20));
|
||||
}
|
||||
let p = guarded(|| ColorProfile::new_from_slice(bytes).ok()).ok_or("not a valid ICC profile")?;
|
||||
if p.color_space != DataColorSpace::Rgb {
|
||||
return Err("not an RGB profile (a display profile describes an RGB device)".into());
|
||||
}
|
||||
if !matches!(p.pcs, DataColorSpace::Xyz | DataColorSpace::Lab) {
|
||||
return Err("unsupported profile connection space".into());
|
||||
}
|
||||
let info = icc::parse(bytes).ok_or("not a valid ICC profile")?;
|
||||
let description = info
|
||||
.description
|
||||
.as_deref()
|
||||
.map(|d| d.trim_start_matches('\u{feff}').trim())
|
||||
.filter(|d| !d.is_empty())
|
||||
.unwrap_or("Display profile")
|
||||
.to_string();
|
||||
let has_lut = p.lut_a_to_b_colorimetric.is_some()
|
||||
|| p.lut_a_to_b_perceptual.is_some()
|
||||
|| p.lut_b_to_a_colorimetric.is_some()
|
||||
|| p.lut_b_to_a_perceptual.is_some();
|
||||
let srgb = ColorProfile::new_srgb();
|
||||
|
||||
// the primaries the pipeline renders into
|
||||
let (kind, to_xyz_d50, curves) = match &info.kind {
|
||||
icc::IccKind::MatrixTrc { to_xyz_d50, trc } if !has_lut => (DisplayKind::MatrixTrc, *to_xyz_d50, Some(trc.clone())),
|
||||
icc::IccKind::MatrixTrc { to_xyz_d50, .. } => (DisplayKind::Lut, *to_xyz_d50, None),
|
||||
_ => (DisplayKind::Lut, measured_to_xyz_d50(&p)?, None),
|
||||
};
|
||||
let to_rec2020 =
|
||||
rec2020_to_xyz_d50().inverse().map(|m| m.mul(&to_xyz_d50)).and_then(white_balanced).ok_or("the profile's primaries are degenerate")?;
|
||||
if to_rec2020.0.iter().flatten().any(|v| !v.is_finite()) {
|
||||
return Err("the profile's primaries are degenerate".into());
|
||||
}
|
||||
|
||||
// the proxy: the display's primaries (as the pipeline targets them), sRGB curve
|
||||
let proxy_xyz = rec2020_to_xyz_d50().mul(&to_rec2020).0;
|
||||
let mut proxy = ColorProfile::new_srgb();
|
||||
proxy.cicp = None;
|
||||
proxy.red_colorant = xyzd([proxy_xyz[0][0], proxy_xyz[1][0], proxy_xyz[2][0]]);
|
||||
proxy.green_colorant = xyzd([proxy_xyz[0][1], proxy_xyz[1][1], proxy_xyz[2][1]]);
|
||||
proxy.blue_colorant = xyzd([proxy_xyz[0][2], proxy_xyz[1][2], proxy_xyz[2][2]]);
|
||||
|
||||
let transform = |src: &ColorProfile, dst: &ColorProfile| guarded(|| src.create_transform_8bit(Layout::Rgba, dst, Layout::Rgba, opts()).ok());
|
||||
let correction = match curves {
|
||||
Some(trc) if trc.iter().all(|t| t.approx_eq(&Trc::Srgb, 2e-4)) => Correction::None,
|
||||
Some(trc) => Correction::Curves(Box::new(curve_tables(&trc))),
|
||||
None => Correction::Cms(transform(&proxy, &p).ok_or("the profile's tables can't be applied")?),
|
||||
};
|
||||
let from_srgb = transform(&srgb, &p).ok_or("can't convert sRGB to this profile")?;
|
||||
let to_srgb = transform(&proxy, &srgb).ok_or("can't convert this profile to sRGB")?;
|
||||
Ok(DisplayProfile { description, id: hash(bytes), kind, to_rec2020, correction, from_srgb, to_srgb })
|
||||
}
|
||||
|
||||
/// Does [`correct`](Self::correct) change anything (the display's curves aren't sRGB's)?
|
||||
pub fn needs_correction(&self) -> bool {
|
||||
!matches!(self.correction, Correction::None)
|
||||
}
|
||||
|
||||
/// A render in the display's primaries with the sRGB curve (the pipeline's display target) →
|
||||
/// the display's device values, in place.
|
||||
pub fn correct(&self, img: &mut Rgba8) -> Result<(), String> {
|
||||
match &self.correction {
|
||||
Correction::None => Ok(()),
|
||||
Correction::Curves(t) => {
|
||||
let map = |p: &mut [u8; 4]| *p = [t[0][p[0] as usize], t[1][p[1] as usize], t[2][p[2] as usize], p[3]];
|
||||
#[cfg(feature = "parallel")]
|
||||
{
|
||||
use rayon::prelude::*;
|
||||
img.data.par_chunks_mut(1 << 14).for_each(|c| c.iter_mut().for_each(map));
|
||||
}
|
||||
#[cfg(not(feature = "parallel"))]
|
||||
img.data.iter_mut().for_each(map);
|
||||
Ok(())
|
||||
}
|
||||
Correction::Cms(x) => apply(x, img),
|
||||
}
|
||||
}
|
||||
|
||||
/// An sRGB image → the display's device values, in place.
|
||||
pub fn from_srgb(&self, img: &mut Rgba8) -> Result<(), String> {
|
||||
apply(&self.from_srgb, img)
|
||||
}
|
||||
|
||||
/// A render in the display's primaries with the sRGB curve → sRGB (colours outside sRGB are
|
||||
/// clipped): for histograms and cached previews, which stay sRGB.
|
||||
pub fn to_srgb(&self, img: &Rgba8) -> Result<Rgba8, String> {
|
||||
let mut out = img.clone();
|
||||
apply(&self.to_srgb, &mut out)?;
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// Device RGB → PCS (XYZ D50) of the pure primaries, measured through the profile's tables
|
||||
/// (LUT-only profiles have no colorant tags).
|
||||
fn measured_to_xyz_d50(p: &ColorProfile) -> Result<Mat3, String> {
|
||||
let lin = icc::linear_rec2020_profile();
|
||||
let x = guarded(|| p.create_transform_f32(Layout::Rgb, &lin, Layout::Rgb, opts()).ok()).ok_or("the profile's tables can't be applied")?;
|
||||
let src = [1.0f32, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0];
|
||||
let mut dst = [0f32; 9];
|
||||
guarded(|| x.transform(&src, &mut dst).ok()).ok_or("the profile's tables can't be applied")?;
|
||||
let c = |i: usize| [dst[i] as f64, dst[i + 3] as f64, dst[i + 6] as f64];
|
||||
// columns: each primary in linear Rec.2020
|
||||
let m = Mat3([c(0), c(1), c(2)]);
|
||||
if m.0.iter().flatten().any(|v| !v.is_finite()) || m.determinant().abs() < 1e-6 {
|
||||
return Err("the profile's primaries are degenerate".into());
|
||||
}
|
||||
Ok(rec2020_to_xyz_d50().mul(&m))
|
||||
}
|
||||
|
||||
/// sRGB-encoded 8-bit value → the display curve's code value, per channel.
|
||||
fn curve_tables(trc: &[Trc; 3]) -> [[u8; 256]; 3] {
|
||||
let mut t = [[0u8; 256]; 3];
|
||||
for (k, tab) in t.iter_mut().enumerate() {
|
||||
for (i, v) in tab.iter_mut().enumerate() {
|
||||
let lin = lightcraft_color::transfer::srgb_to_linear(i as f32 / 255.0);
|
||||
*v = (from_linear(&trc[k], lin) * 255.0 + 0.5).clamp(0.0, 255.0) as u8;
|
||||
}
|
||||
}
|
||||
t
|
||||
}
|
||||
|
||||
/// Inverse of [`Trc::to_linear`] on 0..1 (bisection: works for tables and every parametric form).
|
||||
fn from_linear(t: &Trc, y: f32) -> f32 {
|
||||
let y = y.clamp(0.0, 1.0);
|
||||
let (mut lo, mut hi) = (0.0f32, 1.0f32);
|
||||
for _ in 0..24 {
|
||||
let mid = 0.5 * (lo + hi);
|
||||
if t.to_linear(mid) < y {
|
||||
lo = mid;
|
||||
} else {
|
||||
hi = mid;
|
||||
}
|
||||
}
|
||||
0.5 * (lo + hi)
|
||||
}
|
||||
|
||||
fn apply(x: &Arc<Transform8BitExecutor>, img: &mut Rgba8) -> Result<(), String> {
|
||||
const CHUNK: usize = 1 << 14; // pixels
|
||||
let src = img.data.clone();
|
||||
let run = |s: &[[u8; 4]], d: &mut [[u8; 4]]| guarded(|| x.transform(s.as_flattened(), d.as_flattened_mut()).ok()).is_some();
|
||||
#[cfg(feature = "parallel")]
|
||||
let ok = {
|
||||
use rayon::prelude::*;
|
||||
src.par_chunks(CHUNK).zip(img.data.par_chunks_mut(CHUNK)).all(|(s, d)| run(s, d))
|
||||
};
|
||||
#[cfg(not(feature = "parallel"))]
|
||||
let ok = src.chunks(CHUNK).zip(img.data.chunks_mut(CHUNK)).all(|(s, d)| run(s, d));
|
||||
if ok { Ok(()) } else { Err("display profile transform failed".into()) }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::space::NamedSpace;
|
||||
use lightcraft_color::{DISPLAY_P3, SRGB};
|
||||
|
||||
fn img(px: &[[u8; 4]]) -> Rgba8 {
|
||||
Rgba8 { width: px.len(), height: 1, data: px.to_vec() }
|
||||
}
|
||||
|
||||
fn close(a: [u8; 4], b: [u8; 4], tol: i32) -> bool {
|
||||
(0..4).all(|i| (a[i] as i32 - b[i] as i32).abs() <= tol)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn srgb_display_changes_nothing() {
|
||||
let d = DisplayProfile::from_icc(&icc::write_named(NamedSpace::Srgb)).unwrap();
|
||||
assert_eq!(d.kind, DisplayKind::MatrixTrc);
|
||||
assert!(!d.needs_correction());
|
||||
for (i, row) in d.to_rec2020.0.iter().enumerate() {
|
||||
let want = SRGB.to_space(&REC2020).0[i];
|
||||
for k in 0..3 {
|
||||
assert!((row[k] - want[k]).abs() < 2e-3, "{:?}", d.to_rec2020);
|
||||
}
|
||||
}
|
||||
let px = [[0, 0, 0, 255], [255, 255, 255, 255], [200, 30, 90, 128], [17, 140, 251, 7]];
|
||||
let mut a = img(&px);
|
||||
d.correct(&mut a).unwrap();
|
||||
assert_eq!(a.data, px);
|
||||
let mut b = img(&px);
|
||||
d.from_srgb(&mut b).unwrap();
|
||||
for (x, y) in b.data.iter().zip(&px) {
|
||||
assert!(close(*x, *y, 1), "{x:?} {y:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// On a Display P3 monitor, sRGB red is shown with less red drive than full; P3 green (out of
|
||||
/// sRGB) is reachable in the display's own primaries.
|
||||
#[test]
|
||||
fn p3_display() {
|
||||
let d = DisplayProfile::from_icc(&icc::write_named(NamedSpace::DisplayP3)).unwrap();
|
||||
assert!(!d.needs_correction(), "P3 uses the sRGB curve");
|
||||
let want = DISPLAY_P3.to_space(&REC2020).0;
|
||||
for i in 0..3 {
|
||||
for k in 0..3 {
|
||||
assert!((d.to_rec2020.0[i][k] - want[i][k]).abs() < 2e-3, "{:?}", d.to_rec2020);
|
||||
}
|
||||
}
|
||||
let mut red = img(&[[255, 0, 0, 255], [128, 128, 128, 255]]);
|
||||
d.from_srgb(&mut red).unwrap();
|
||||
assert!(red.data[0][0] < 245 && red.data[0][1] > 40, "{:?}", red.data[0]);
|
||||
assert!(close(red.data[1], [128, 128, 128, 255], 1), "grey stays grey: {:?}", red.data[1]);
|
||||
// the display's own pure green is outside sRGB: it clips there
|
||||
let srgb = d.to_srgb(&img(&[[0, 255, 0, 255]])).unwrap();
|
||||
assert!(srgb.data[0][0] < 10 && srgb.data[0][1] == 255, "{:?}", srgb.data[0]);
|
||||
}
|
||||
|
||||
/// A gamma 2.2 display (Adobe RGB-like): the proxy's sRGB curve goes to the display's curve.
|
||||
#[test]
|
||||
fn gamma_display_uses_its_curve() {
|
||||
let bytes = icc::write_matrix_trc(&lightcraft_color::ADOBE_RGB, &Trc::Gamma(2.2));
|
||||
let d = DisplayProfile::from_icc(&bytes).unwrap();
|
||||
assert!(d.needs_correction());
|
||||
let mut a = img(&[[0, 0, 0, 255], [255, 255, 255, 255], [10, 128, 240, 9]]);
|
||||
d.correct(&mut a).unwrap();
|
||||
assert_eq!(a.data[0], [0, 0, 0, 255]);
|
||||
assert_eq!(a.data[1], [255, 255, 255, 255]);
|
||||
for (k, v) in [10u8, 128, 240].into_iter().enumerate() {
|
||||
let lin = lightcraft_color::transfer::srgb_to_linear(v as f32 / 255.0);
|
||||
let want = (lin.powf(1.0 / 2.2) * 255.0).round() as i32;
|
||||
assert!((a.data[2][k] as i32 - want).abs() <= 1, "{k}: {} vs {want}", a.data[2][k]);
|
||||
}
|
||||
assert_eq!(a.data[2][3], 9, "alpha kept");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_bad_profiles() {
|
||||
assert!(DisplayProfile::from_icc(&[]).is_err());
|
||||
assert!(DisplayProfile::from_icc(&[0u8; 300]).is_err());
|
||||
let mut gray = ColorProfile::new_gray_with_gamma(2.2);
|
||||
gray.cicp = None;
|
||||
assert!(DisplayProfile::from_icc(&gray.encode().unwrap()).is_err());
|
||||
// truncated / corrupted real profiles never panic
|
||||
let good = icc::write_named(NamedSpace::DisplayP3);
|
||||
for n in [10, 128, 140, 200, good.len() - 1] {
|
||||
let _ = DisplayProfile::from_icc(&good[..n]);
|
||||
}
|
||||
for i in (0..good.len()).step_by(7) {
|
||||
let mut b = good.clone();
|
||||
b[i] ^= 0xa5;
|
||||
let _ = DisplayProfile::from_icc(&b);
|
||||
}
|
||||
}
|
||||
|
||||
/// A LUT-based display profile with no colorant or TRC tags (its A2B/B2A tables describe a
|
||||
/// Display P3 panel with a gamma 2.2 response): the primaries are measured through the tables
|
||||
/// and every preview goes through the CMS.
|
||||
fn lut_only_p3_gamma22() -> Vec<u8> {
|
||||
use moxcms::{LutMultidimensionalType, LutWarehouse, Matrix3d, ProfileClass, ToneReprCurve, Vector3d};
|
||||
let m = crate::space::rgb_to_xyz_d50(&DISPLAY_P3).0;
|
||||
let inv = crate::space::rgb_to_xyz_d50(&DISPLAY_P3).inverse().unwrap().0;
|
||||
let curves = |v: Vec<f32>| vec![ToneReprCurve::Parametric(v.clone()), ToneReprCurve::Parametric(v.clone()), ToneReprCurve::Parametric(v)];
|
||||
let ident = || curves(vec![1.0]);
|
||||
// mAB: A → CLUT → M → matrix → B (no CLUT: M curves linearize, the matrix goes to XYZ)
|
||||
let a2b = LutWarehouse::Multidimensional(LutMultidimensionalType {
|
||||
num_input_channels: 3,
|
||||
num_output_channels: 3,
|
||||
grid_points: [0; 16],
|
||||
clut: None,
|
||||
a_curves: vec![],
|
||||
m_curves: curves(vec![2.2]),
|
||||
matrix: Matrix3d { v: m.map(|r| r.map(|v| v * 32768.0 / 65535.0)) },
|
||||
b_curves: ident(),
|
||||
bias: Vector3d::default(),
|
||||
});
|
||||
let b2a = LutWarehouse::Multidimensional(LutMultidimensionalType {
|
||||
num_input_channels: 3,
|
||||
num_output_channels: 3,
|
||||
grid_points: [0; 16],
|
||||
clut: None,
|
||||
a_curves: vec![],
|
||||
m_curves: curves(vec![1.0 / 2.2]),
|
||||
matrix: Matrix3d { v: inv.map(|r| r.map(|v| v * 65535.0 / 32768.0)) },
|
||||
b_curves: ident(),
|
||||
bias: Vector3d::default(),
|
||||
});
|
||||
let mut p = ColorProfile::new_srgb();
|
||||
p.cicp = None;
|
||||
p.profile_class = ProfileClass::DisplayDevice;
|
||||
p.red_colorant = Xyzd::default();
|
||||
p.green_colorant = Xyzd::default();
|
||||
p.blue_colorant = Xyzd::default();
|
||||
p.red_trc = None;
|
||||
p.green_trc = None;
|
||||
p.blue_trc = None;
|
||||
p.lut_a_to_b_colorimetric = Some(a2b.clone());
|
||||
p.lut_a_to_b_perceptual = Some(a2b);
|
||||
p.lut_b_to_a_colorimetric = Some(b2a.clone());
|
||||
p.lut_b_to_a_perceptual = Some(b2a);
|
||||
p.encode().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lut_profile_goes_through_the_cms() {
|
||||
let bytes = lut_only_p3_gamma22();
|
||||
let parsed = ColorProfile::new_from_slice(&bytes).unwrap();
|
||||
assert!(!parsed.is_matrix_shaper() && parsed.lut_b_to_a_colorimetric.is_some(), "a LUT-only profile");
|
||||
let d = DisplayProfile::from_icc(&bytes).unwrap();
|
||||
assert_eq!(d.kind, DisplayKind::Lut);
|
||||
assert!(d.needs_correction());
|
||||
// the measured primaries are P3's
|
||||
let want = DISPLAY_P3.to_space(&REC2020).0;
|
||||
for i in 0..3 {
|
||||
for k in 0..3 {
|
||||
assert!((d.to_rec2020.0[i][k] - want[i][k]).abs() < 0.02, "{:?} vs {want:?}", d.to_rec2020);
|
||||
}
|
||||
}
|
||||
// the correction is the gamma 2.2 curve; grey stays grey
|
||||
let mut a = img(&[[0, 0, 0, 255], [255, 255, 255, 255], [128, 128, 128, 200]]);
|
||||
d.correct(&mut a).unwrap();
|
||||
assert!(close(a.data[0], [0, 0, 0, 255], 1) && close(a.data[1], [255, 255, 255, 255], 1), "{:?}", a.data);
|
||||
let lin = lightcraft_color::transfer::srgb_to_linear(128.0 / 255.0);
|
||||
let g = (lin.powf(1.0 / 2.2) * 255.0).round() as u8;
|
||||
assert!(close(a.data[2], [g, g, g, 200], 2), "{:?} vs {g}", a.data[2]);
|
||||
// sRGB red → less than full red drive on a P3 panel
|
||||
let mut red = img(&[[255, 0, 0, 255]]);
|
||||
d.from_srgb(&mut red).unwrap();
|
||||
assert!(red.data[0][0] < 245 && red.data[0][1] > 30, "{:?}", red.data[0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ids_identify_profiles() {
|
||||
let a = DisplayProfile::from_icc(&icc::write_named(NamedSpace::DisplayP3)).unwrap();
|
||||
let b = DisplayProfile::from_icc(&icc::write_named(NamedSpace::AdobeRgb)).unwrap();
|
||||
let a2 = DisplayProfile::from_icc(&icc::write_named(NamedSpace::DisplayP3)).unwrap();
|
||||
assert_ne!(a.id, b.id);
|
||||
assert_eq!(a.id, a2.id);
|
||||
}
|
||||
}
|
||||
+176
-29
@@ -79,6 +79,46 @@ pub enum ChromaSubsampling {
|
||||
|
||||
const XMP_NS: &[u8] = b"http://ns.adobe.com/xap/1.0/\0";
|
||||
|
||||
/// The APP segments [`encode_jpeg`] writes for `meta` (marker byte, payload): JFIF density, EXIF,
|
||||
/// XMP and the ICC profile in chunks, in that order.
|
||||
pub(crate) fn jpeg_app_segments(meta: &EncodeMeta) -> Result<Vec<(u8, Vec<u8>)>> {
|
||||
let mut segs: Vec<(u8, Vec<u8>)> = Vec::new();
|
||||
let ppi = meta.ppi.filter(|p| *p > 0).map_or([0u8, 0, 1, 0, 1], |p| {
|
||||
let [hi, lo] = p.to_be_bytes();
|
||||
[1, hi, lo, hi, lo]
|
||||
});
|
||||
segs.push((0xE0, [&b"JFIF\0\x01\x02"[..], &ppi, &[0, 0]].concat()));
|
||||
if let Some(exif) = meta.exif {
|
||||
if exif.len() > 65_527 {
|
||||
return Err(Error::Encode("EXIF larger than one APP1 segment (64 KiB)".into()));
|
||||
}
|
||||
segs.push((0xE1, [&b"Exif\0\0"[..], exif].concat()));
|
||||
}
|
||||
if let Some(xmp) = meta.xmp {
|
||||
segs.push((0xE1, xmp_segment(xmp)?));
|
||||
}
|
||||
if let Some(icc) = meta.icc {
|
||||
// ICC.1 Annex B.4: "ICC_PROFILE\0", 1-based chunk number, chunk count
|
||||
let chunks: Vec<&[u8]> = icc.chunks(65_519).collect();
|
||||
if chunks.len() > 255 {
|
||||
return Err(Error::Encode("ICC profile too large for JPEG".into()));
|
||||
}
|
||||
for (i, c) in chunks.iter().enumerate() {
|
||||
segs.push((0xE2, [&b"ICC_PROFILE\0"[..], &[i as u8 + 1, chunks.len() as u8], c].concat()));
|
||||
}
|
||||
}
|
||||
Ok(segs)
|
||||
}
|
||||
|
||||
/// An APP1 XMP payload (namespace + packet), refusing packets larger than one segment.
|
||||
pub(crate) fn xmp_segment(xmp: &str) -> Result<Vec<u8>> {
|
||||
let seg = [XMP_NS, xmp.as_bytes()].concat();
|
||||
if seg.len() > 65_533 {
|
||||
return Err(Error::Encode("XMP larger than one APP1 segment (extended XMP not supported)".into()));
|
||||
}
|
||||
Ok(seg)
|
||||
}
|
||||
|
||||
/// Encode baseline JPEG (8-bit; alpha is dropped). `quality` 1..=100.
|
||||
pub fn encode_jpeg(img: &EncodeImage, quality: u8, subsampling: ChromaSubsampling, meta: &EncodeMeta) -> Result<Vec<u8>> {
|
||||
img.validate()?;
|
||||
@@ -97,35 +137,7 @@ pub fn encode_jpeg(img: &EncodeImage, quality: u8, subsampling: ChromaSubsamplin
|
||||
};
|
||||
if subsampling != ChromaSubsampling::S422 {
|
||||
// Parallel encoder (restart-interval bands on all cores).
|
||||
let mut segs: Vec<(u8, Vec<u8>)> = Vec::new();
|
||||
let ppi = meta.ppi.filter(|p| *p > 0).map_or([0u8, 0, 1, 0, 1], |p| {
|
||||
let [hi, lo] = p.to_be_bytes();
|
||||
[1, hi, lo, hi, lo]
|
||||
});
|
||||
segs.push((0xE0, [&b"JFIF\0\x01\x02"[..], &ppi, &[0, 0]].concat()));
|
||||
if let Some(exif) = meta.exif {
|
||||
if exif.len() > 65_527 {
|
||||
return Err(Error::Encode("EXIF larger than one APP1 segment (64 KiB)".into()));
|
||||
}
|
||||
segs.push((0xE1, [&b"Exif\0\0"[..], exif].concat()));
|
||||
}
|
||||
if let Some(xmp) = meta.xmp {
|
||||
let seg = [XMP_NS, xmp.as_bytes()].concat();
|
||||
if seg.len() > 65_533 {
|
||||
return Err(Error::Encode("XMP larger than one APP1 segment (extended XMP not supported)".into()));
|
||||
}
|
||||
segs.push((0xE1, seg));
|
||||
}
|
||||
if let Some(icc) = meta.icc {
|
||||
// ICC.1 Annex B.4: "ICC_PROFILE\0", 1-based chunk number, chunk count
|
||||
let chunks: Vec<&[u8]> = icc.chunks(65_519).collect();
|
||||
if chunks.len() > 255 {
|
||||
return Err(Error::Encode("ICC profile too large for JPEG".into()));
|
||||
}
|
||||
for (i, c) in chunks.iter().enumerate() {
|
||||
segs.push((0xE2, [&b"ICC_PROFILE\0"[..], &[i as u8 + 1, chunks.len() as u8], c].concat()));
|
||||
}
|
||||
}
|
||||
let segs = jpeg_app_segments(meta)?;
|
||||
let ch = if img.channels == 2 { 1 } else { img.channels as usize };
|
||||
// `data` is already gray for 2-channel input (see above)
|
||||
return Ok(crate::jpeg_par::encode(&data, img.width as usize, img.height as usize, ch, quality, subsampling, &segs));
|
||||
@@ -351,6 +363,141 @@ pub fn encode_avif(img: &EncodeImage, quality: u8, speed: u8, meta: &EncodeMeta)
|
||||
}
|
||||
}
|
||||
|
||||
/// Absolute luminance of SDR white in an HDR file (ITU-R BT.2408 reference white), cd/m².
|
||||
pub const HDR_REFERENCE_WHITE_NITS: f32 = 203.0;
|
||||
|
||||
/// SMPTE ST 2084 (PQ) inverse EOTF: luminance in cd/m² → signal 0..1.
|
||||
pub fn pq_encode(nits: f32) -> f32 {
|
||||
const M1: f32 = 2610.0 / 16384.0;
|
||||
const M2: f32 = 2523.0 / 4096.0 * 128.0;
|
||||
const C1: f32 = 3424.0 / 4096.0;
|
||||
const C2: f32 = 2413.0 / 4096.0 * 32.0;
|
||||
const C3: f32 = 2392.0 / 4096.0 * 32.0;
|
||||
let y = (nits / 10_000.0).clamp(0.0, 1.0).powf(M1);
|
||||
((C1 + C2 * y) / (1.0 + C3 * y)).powf(M2)
|
||||
}
|
||||
|
||||
/// SMPTE ST 2084 (PQ) EOTF: signal 0..1 → luminance in cd/m².
|
||||
pub fn pq_decode(e: f32) -> f32 {
|
||||
const M1: f32 = 2610.0 / 16384.0;
|
||||
const M2: f32 = 2523.0 / 4096.0 * 128.0;
|
||||
const C1: f32 = 3424.0 / 4096.0;
|
||||
const C2: f32 = 2413.0 / 4096.0 * 32.0;
|
||||
const C3: f32 = 2392.0 / 4096.0 * 32.0;
|
||||
let p = e.clamp(0.0, 1.0).powf(1.0 / M2);
|
||||
((p - C1).max(0.0) / (C2 - C3 * p)).powf(1.0 / M1) * 10_000.0
|
||||
}
|
||||
|
||||
/// Encode an HDR AVIF: `rgb` is linear light in **Rec. 2020** primaries with SDR white at 1.0
|
||||
/// (`width × height × 3`). Written as 10-bit 4:4:4 BT.2020 non-constant-luminance YCbCr with the
|
||||
/// PQ transfer curve, SDR white at [`HDR_REFERENCE_WHITE_NITS`], and a content light level box.
|
||||
/// `quality` 1..=100, `speed` 1..=10. Native targets with the `avif` feature only.
|
||||
pub fn encode_avif_pq(width: u32, height: u32, rgb: &[f32], quality: u8, speed: u8, meta: &EncodeMeta) -> Result<Vec<u8>> {
|
||||
let (w, h) = (width as usize, height as usize);
|
||||
let n = w.checked_mul(h).ok_or_else(|| Error::Encode("image too large".into()))?;
|
||||
if n == 0 || n.checked_mul(3).is_none_or(|need| rgb.len() < need) {
|
||||
return Err(Error::Encode("HDR AVIF: sample buffer too short".into()));
|
||||
}
|
||||
#[cfg(all(feature = "avif", not(target_arch = "wasm32")))]
|
||||
{
|
||||
use rav1e::prelude::*;
|
||||
const K: [f32; 3] = [0.2627, 0.6780, 0.0593]; // BT.2020 NCL
|
||||
let mut max_nits = 0f32;
|
||||
let mut sum_nits = 0f64;
|
||||
let planes: Vec<[u16; 3]> = rgb
|
||||
.as_chunks::<3>()
|
||||
.0
|
||||
.iter()
|
||||
.take(n)
|
||||
.map(|c| {
|
||||
let nits = c.map(|v| (if v.is_finite() { v.max(0.0) } else { 0.0 }) * HDR_REFERENCE_WHITE_NITS);
|
||||
let y_nits = K[0] * nits[0] + K[1] * nits[1] + K[2] * nits[2];
|
||||
max_nits = max_nits.max(nits[0].max(nits[1]).max(nits[2]));
|
||||
sum_nits += y_nits as f64;
|
||||
let [r, g, b] = nits.map(pq_encode);
|
||||
let y = K[0] * r + K[1] * g + K[2] * b;
|
||||
let cb = (b - y) / (2.0 * (1.0 - K[2])) + 0.5;
|
||||
let cr = (r - y) / (2.0 * (1.0 - K[0])) + 0.5;
|
||||
[y, cb, cr].map(|v| (v * 1023.0).round().clamp(0.0, 1023.0) as u16)
|
||||
})
|
||||
.collect();
|
||||
let q = quality.clamp(1, 100) as f32 / 100.0;
|
||||
// ravif's quality → quantizer curve, so qualities match the SDR AVIF path
|
||||
let x = if q >= 0.82 {
|
||||
(1.0 - q) * 2.6
|
||||
} else if q > 0.25 {
|
||||
1.0 - 0.125 - q * 0.5
|
||||
} else {
|
||||
1.0 - q
|
||||
};
|
||||
let quantizer = (x * 255.0).round() as usize;
|
||||
let mut ec = EncoderConfig::with_speed_preset(speed.clamp(1, 10));
|
||||
ec.width = w;
|
||||
ec.height = h;
|
||||
ec.time_base = Rational::new(1, 1);
|
||||
ec.bit_depth = 10;
|
||||
ec.chroma_sampling = ChromaSampling::Cs444;
|
||||
ec.pixel_range = PixelRange::Full;
|
||||
ec.color_description = Some(ColorDescription {
|
||||
color_primaries: ColorPrimaries::BT2020,
|
||||
transfer_characteristics: TransferCharacteristics::SMPTE2084,
|
||||
matrix_coefficients: MatrixCoefficients::BT2020NCL,
|
||||
});
|
||||
let cll = (max_nits.ceil().clamp(1.0, 10_000.0) as u16, ((sum_nits / n as f64).ceil().clamp(1.0, 10_000.0)) as u16);
|
||||
ec.content_light = Some(ContentLight { max_content_light_level: cll.0, max_frame_average_light_level: cll.1 });
|
||||
ec.still_picture = true;
|
||||
ec.quantizer = quantizer;
|
||||
ec.min_quantizer = quantizer.min(255) as u8;
|
||||
ec.speed_settings.multiref = false;
|
||||
ec.speed_settings.rdo_lookahead_frames = 1;
|
||||
ec.speed_settings.scene_detection_mode = SceneDetectionSpeed::None;
|
||||
let e = |e: &dyn std::fmt::Display| Error::Encode(format!("AV1: {e}"));
|
||||
let mut ctx: Context<u16> = Config::new().with_encoder_config(ec).new_context().map_err(|x| e(&x))?;
|
||||
let mut frame = ctx.new_frame();
|
||||
{
|
||||
let mut it = planes.iter();
|
||||
let mut fp = frame.planes.iter_mut();
|
||||
let (Some(py), Some(pu), Some(pv)) = (fp.next(), fp.next(), fp.next()) else {
|
||||
return Err(Error::Encode("AV1: frame has no planes".into()));
|
||||
};
|
||||
let (mut sy, mut su, mut sv) = (py.mut_slice(Default::default()), pu.mut_slice(Default::default()), pv.mut_slice(Default::default()));
|
||||
for ((ry, ru), rv) in sy.rows_iter_mut().zip(su.rows_iter_mut()).zip(sv.rows_iter_mut()).take(h) {
|
||||
for ((y, u), v) in ry.iter_mut().zip(ru.iter_mut()).zip(rv.iter_mut()).take(w) {
|
||||
let Some(p) = it.next() else { break };
|
||||
(*y, *u, *v) = (p[0], p[1], p[2]);
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx.send_frame(frame).map_err(|x| e(&x))?;
|
||||
ctx.flush();
|
||||
let mut av1 = Vec::new();
|
||||
loop {
|
||||
match ctx.receive_packet() {
|
||||
Ok(mut p) if p.frame_type == FrameType::KEY => av1.append(&mut p.data),
|
||||
Ok(_) => {}
|
||||
Err(EncoderStatus::Encoded | EncoderStatus::LimitReached) => break,
|
||||
Err(x) => return Err(e(&x)),
|
||||
}
|
||||
}
|
||||
use avif_serialize::constants as c;
|
||||
let mut a = avif_serialize::Aviffy::new();
|
||||
a.set_matrix_coefficients(c::MatrixCoefficients::Bt2020Ncl)
|
||||
.set_transfer_characteristics(c::TransferCharacteristics::Smpte2084)
|
||||
.set_color_primaries(c::ColorPrimaries::Bt2020)
|
||||
.set_full_color_range(true)
|
||||
.set_content_light_level(cll.0, cll.1);
|
||||
if let Some(exif) = meta.exif {
|
||||
a.set_exif(exif.to_vec());
|
||||
}
|
||||
Ok(a.to_vec(&av1, None, width, height, 10))
|
||||
}
|
||||
#[cfg(not(all(feature = "avif", not(target_arch = "wasm32"))))]
|
||||
{
|
||||
let _ = (quality, speed, meta);
|
||||
Err(Error::Encode("AVIF encoding is not available in this build".into()))
|
||||
}
|
||||
}
|
||||
|
||||
/// 16-bit RGB(A) → 10-bit BT.601 full-range YCbCr planes (the matrix `ravif` uses for 8-bit input).
|
||||
#[cfg(all(feature = "avif", not(target_arch = "wasm32")))]
|
||||
fn avif_10bit(enc: &ravif::Encoder, img: &EncodeImage, s: &[u16]) -> Result<Vec<u8>> {
|
||||
|
||||
@@ -0,0 +1,834 @@
|
||||
//! Gain map HDR images (ISO 21496-1): an SDR base image plus a gain map that a decoder uses to
|
||||
//! rebuild the HDR rendition, so the file looks right everywhere and lights up on HDR displays.
|
||||
//!
|
||||
//! - [`compute`] measures a gain map from an SDR and an HDR rendition of the same picture (linear
|
||||
//! light, same primaries). Measure it against the base *as written* (decoded from its 8-bit
|
||||
//! codes), so quantisation in the base is handed back by the gain map.
|
||||
//! - [`apply`] rebuilds the HDR rendition from a base and its gain map.
|
||||
//! - [`encode_jpeg`] writes a gain map JPEG: the base with an MPF index (CIPA DC-007), the ISO
|
||||
//! 21496-1 marker and Adobe/Google container XMP, followed by the gain map JPEG carrying the ISO
|
||||
//! 21496-1 metadata, the Adobe `hdrgm` XMP (Android, Chrome) and Apple's description of the same
|
||||
//! map (what survives an iMessage send).
|
||||
//! - [`read_jpeg`] finds and decodes the gain map of a gain map JPEG (ours, cameras, Lightroom),
|
||||
//! from its ISO 21496-1 metadata or, failing that, its `hdrgm` XMP.
|
||||
//!
|
||||
//! All gains are log2. A decoder recovers `log2(hdr + offset_hdr) − log2(sdr + offset_sdr)` as
|
||||
//! `min + (max − min) · (code / 255)^(1 / gamma)`, scaled by a weight in 0..1 that depends on how
|
||||
//! much headroom the display has between `base_headroom` and `alternate_headroom`.
|
||||
|
||||
use crate::encode::{EncodeImage, EncodeMeta, Samples, jpeg_app_segments, xmp_segment};
|
||||
use crate::{ChromaSubsampling, Error, Format, Result, jpeg, jpeg_par};
|
||||
|
||||
/// The ISO 21496-1 namespace that opens its APP2 payloads (NUL terminated).
|
||||
pub const ISO_URN: &[u8] = b"urn:iso:std:iso:ts:21496:-1\0";
|
||||
|
||||
/// Fixed denominator for the metadata's rationals: one value per setting, byte-identical output.
|
||||
const DENOMINATOR: u32 = 1_000_000;
|
||||
|
||||
/// Gain map parameters (ISO 21496-1 / Adobe `hdrgm`). Per-channel arrays repeat the first value
|
||||
/// when the map is single-channel.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct GainMapMeta {
|
||||
/// Three gain channels (R, G, B) instead of one luminance gain.
|
||||
pub multichannel: bool,
|
||||
/// The gain applies in the base image's colour space.
|
||||
pub use_base_colour_space: bool,
|
||||
/// log2 headroom at which the base is shown as is (0: an SDR base).
|
||||
pub base_headroom: f32,
|
||||
/// log2 headroom at which the gain map applies in full.
|
||||
pub alternate_headroom: f32,
|
||||
pub min: [f32; 3],
|
||||
pub max: [f32; 3],
|
||||
pub gamma: [f32; 3],
|
||||
pub base_offset: [f32; 3],
|
||||
pub alternate_offset: [f32; 3],
|
||||
}
|
||||
|
||||
impl Default for GainMapMeta {
|
||||
fn default() -> Self {
|
||||
GainMapMeta {
|
||||
multichannel: false,
|
||||
use_base_colour_space: true,
|
||||
base_headroom: 0.0,
|
||||
alternate_headroom: 1.0,
|
||||
min: [0.0; 3],
|
||||
max: [1.0; 3],
|
||||
gamma: [1.0; 3],
|
||||
base_offset: [1.0 / 64.0; 3],
|
||||
alternate_offset: [1.0 / 64.0; 3],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl GainMapMeta {
|
||||
fn channels(&self) -> usize {
|
||||
if self.multichannel { 3 } else { 1 }
|
||||
}
|
||||
|
||||
/// Usable by [`apply`]: finite, positive gammas, max ≥ min, headrooms in order.
|
||||
pub fn is_valid(&self) -> bool {
|
||||
let fin = |a: &[f32; 3]| a.iter().all(|v| v.is_finite());
|
||||
fin(&self.min)
|
||||
&& fin(&self.max)
|
||||
&& fin(&self.gamma)
|
||||
&& fin(&self.base_offset)
|
||||
&& fin(&self.alternate_offset)
|
||||
&& self.gamma.iter().all(|g| *g > 0.0)
|
||||
&& self.min.iter().zip(&self.max).all(|(a, b)| b >= a)
|
||||
&& self.base_headroom.is_finite()
|
||||
&& self.alternate_headroom.is_finite()
|
||||
&& self.alternate_headroom > self.base_headroom
|
||||
}
|
||||
|
||||
/// The ISO 21496-1 APP2 payload of the gain map image, URN included.
|
||||
pub fn to_iso(&self) -> Vec<u8> {
|
||||
let mut p = ISO_URN.to_vec();
|
||||
p.extend_from_slice(&0u16.to_be_bytes()); // minimum_version: readers gate on this
|
||||
p.extend_from_slice(&0u16.to_be_bytes()); // writer_version
|
||||
let mut flags = 0u8;
|
||||
if self.multichannel {
|
||||
flags |= FLAG_MULTICHANNEL;
|
||||
}
|
||||
if self.use_base_colour_space {
|
||||
flags |= FLAG_BASE_COLOUR_SPACE;
|
||||
}
|
||||
p.push(flags);
|
||||
let unsigned = |p: &mut Vec<u8>, v: f32| {
|
||||
let n = (v as f64 * DENOMINATOR as f64).round().clamp(0.0, u32::MAX as f64) as u32;
|
||||
p.extend_from_slice(&n.to_be_bytes());
|
||||
p.extend_from_slice(&DENOMINATOR.to_be_bytes());
|
||||
};
|
||||
let signed = |p: &mut Vec<u8>, v: f32| {
|
||||
let n = (v as f64 * DENOMINATOR as f64).round().clamp(i32::MIN as f64, i32::MAX as f64) as i32;
|
||||
p.extend_from_slice(&n.to_be_bytes());
|
||||
p.extend_from_slice(&DENOMINATOR.to_be_bytes());
|
||||
};
|
||||
unsigned(&mut p, self.base_headroom.max(0.0));
|
||||
unsigned(&mut p, self.alternate_headroom.max(0.0));
|
||||
for c in 0..self.channels() {
|
||||
signed(&mut p, self.min[c]);
|
||||
signed(&mut p, self.max[c]);
|
||||
unsigned(&mut p, self.gamma[c]);
|
||||
signed(&mut p, self.base_offset[c]);
|
||||
signed(&mut p, self.alternate_offset[c]);
|
||||
}
|
||||
p
|
||||
}
|
||||
|
||||
/// Parse an ISO 21496-1 payload (the bytes after [`ISO_URN`]). `None` for the versions-only
|
||||
/// marker a primary image carries, unknown versions, a base that is the HDR rendition, or
|
||||
/// values no decoder could use.
|
||||
pub fn from_iso(p: &[u8]) -> Option<GainMapMeta> {
|
||||
let mut r = Reader { b: p, at: 0 };
|
||||
let min_version = r.u16()?;
|
||||
let _writer_version = r.u16()?;
|
||||
if min_version != 0 {
|
||||
return None;
|
||||
}
|
||||
let flags = r.u8()?;
|
||||
if flags & FLAG_BACKWARD != 0 {
|
||||
return None;
|
||||
}
|
||||
let multichannel = flags & FLAG_MULTICHANNEL != 0;
|
||||
let common = flags & FLAG_COMMON_DENOMINATOR != 0;
|
||||
let common_d = if common { r.u32()? } else { 0 };
|
||||
let unsigned = |r: &mut Reader| -> Option<f32> {
|
||||
let n = r.u32()?;
|
||||
let d = if common { common_d } else { r.u32()? };
|
||||
(d != 0).then(|| (n as f64 / d as f64) as f32)
|
||||
};
|
||||
let signed = |r: &mut Reader| -> Option<f32> {
|
||||
let n = r.u32()? as i32;
|
||||
let d = if common { common_d } else { r.u32()? };
|
||||
(d != 0).then(|| (n as f64 / d as f64) as f32)
|
||||
};
|
||||
let mut m = GainMapMeta {
|
||||
multichannel,
|
||||
use_base_colour_space: flags & FLAG_BASE_COLOUR_SPACE != 0,
|
||||
base_headroom: unsigned(&mut r)?,
|
||||
alternate_headroom: unsigned(&mut r)?,
|
||||
..GainMapMeta::default()
|
||||
};
|
||||
for c in 0..m.channels() {
|
||||
m.min[c] = signed(&mut r)?;
|
||||
m.max[c] = signed(&mut r)?;
|
||||
m.gamma[c] = unsigned(&mut r)?;
|
||||
m.base_offset[c] = signed(&mut r)?;
|
||||
m.alternate_offset[c] = signed(&mut r)?;
|
||||
}
|
||||
if !multichannel {
|
||||
m.fill_from_first();
|
||||
}
|
||||
m.is_valid().then_some(m)
|
||||
}
|
||||
|
||||
fn fill_from_first(&mut self) {
|
||||
for a in [&mut self.min, &mut self.max, &mut self.gamma, &mut self.base_offset, &mut self.alternate_offset] {
|
||||
a[1] = a[0];
|
||||
a[2] = a[0];
|
||||
}
|
||||
}
|
||||
|
||||
/// The gain parameters from Adobe `hdrgm` XMP (the gain map image's packet), for files
|
||||
/// without ISO 21496-1 metadata (the first Ultra HDR files). Attributes or `rdf:Seq` lists.
|
||||
pub fn from_hdrgm_xmp(xmp: &str) -> Option<GainMapMeta> {
|
||||
if !xmp.contains("hdrgm:") {
|
||||
return None;
|
||||
}
|
||||
let base_rendition_hdr = xmp_values(xmp, "BaseRenditionIsHDR").is_some_and(|v| v.first().is_some_and(|s| s.eq_ignore_ascii_case("true")));
|
||||
if base_rendition_hdr {
|
||||
return None;
|
||||
}
|
||||
let get = |name: &str, default: f32| -> Option<([f32; 3], bool)> {
|
||||
let Some(v) = xmp_values(xmp, name) else { return Some(([default; 3], false)) };
|
||||
let f: Vec<f32> = v.iter().map(|s| s.trim().parse::<f32>().ok()).collect::<Option<_>>()?;
|
||||
match f.as_slice() {
|
||||
[a] => Some(([*a; 3], false)),
|
||||
[a, b, c] => Some(([*a, *b, *c], true)),
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
let (max, m1) = get("GainMapMax", f32::NAN)?;
|
||||
if max.iter().any(|v| v.is_nan()) {
|
||||
return None; // the one required parameter
|
||||
}
|
||||
let (min, m2) = get("GainMapMin", 0.0)?;
|
||||
let (gamma, m3) = get("Gamma", 1.0)?;
|
||||
let (base_offset, m4) = get("OffsetSDR", 1.0 / 64.0)?;
|
||||
let (alternate_offset, m5) = get("OffsetHDR", 1.0 / 64.0)?;
|
||||
let (cap_min, _) = get("HDRCapacityMin", 0.0)?;
|
||||
let max_gain = max.iter().copied().fold(f32::MIN, f32::max);
|
||||
let (cap_max, _) = get("HDRCapacityMax", max_gain)?;
|
||||
let m = GainMapMeta {
|
||||
multichannel: m1 || m2 || m3 || m4 || m5,
|
||||
use_base_colour_space: true,
|
||||
base_headroom: cap_min[0].max(0.0),
|
||||
alternate_headroom: cap_max[0],
|
||||
min,
|
||||
max,
|
||||
gamma,
|
||||
base_offset,
|
||||
alternate_offset,
|
||||
};
|
||||
m.is_valid().then_some(m)
|
||||
}
|
||||
|
||||
/// The gain map image's XMP: the Adobe `hdrgm` parameters (read by Android and Chrome, which
|
||||
/// ignore the ISO payload) and Apple's description of the same map (what iMessage keeps).
|
||||
pub fn to_gain_map_xmp(&self) -> String {
|
||||
let names = ["GainMapMin", "GainMapMax", "Gamma", "OffsetSDR", "OffsetHDR"];
|
||||
let values = [&self.min, &self.max, &self.gamma, &self.base_offset, &self.alternate_offset];
|
||||
let uniform = |v: &[f32; 3]| !self.multichannel || (v[0] == v[1] && v[1] == v[2]);
|
||||
let mut s = String::from(XMP_OPEN);
|
||||
s.push_str(r#"<rdf:Description rdf:about="" xmlns:hdrgm="http://ns.adobe.com/hdr-gain-map/1.0/" hdrgm:Version="1.0""#);
|
||||
s.push_str(&format!(r#" hdrgm:HDRCapacityMin="{}""#, num(self.base_headroom)));
|
||||
s.push_str(&format!(r#" hdrgm:HDRCapacityMax="{}""#, num(self.alternate_headroom)));
|
||||
s.push_str(r#" hdrgm:BaseRenditionIsHDR="False""#);
|
||||
for (n, v) in names.iter().zip(values) {
|
||||
if uniform(v) {
|
||||
s.push_str(&format!(r#" hdrgm:{n}="{}""#, num(v[0])));
|
||||
}
|
||||
}
|
||||
s.push('>');
|
||||
// per-channel values are an rdf:Seq of three, not a comma list: readers parse each item
|
||||
for (n, v) in names.iter().zip(values) {
|
||||
if !uniform(v) {
|
||||
s.push_str(&format!("<hdrgm:{n}><rdf:Seq>"));
|
||||
for x in v {
|
||||
s.push_str(&format!("<rdf:li>{}</rdf:li>", num(*x)));
|
||||
}
|
||||
s.push_str(&format!("</rdf:Seq></hdrgm:{n}>"));
|
||||
}
|
||||
}
|
||||
s.push_str("</rdf:Description>");
|
||||
s.push_str(&self.apple_description());
|
||||
s.push_str(XMP_CLOSE);
|
||||
s
|
||||
}
|
||||
|
||||
/// Apple's own description of the map (`apdi`, `HDRGainMap`, `HDRToneMap` namespaces).
|
||||
fn apple_description(&self) -> String {
|
||||
// 'L008' (8-bit luminance) or '444f'; the stored map must really have that many channels
|
||||
let format = if self.multichannel { 0x3434_3466u32 } else { 0x4C30_3038 };
|
||||
let mut s = String::from(
|
||||
r#"<rdf:Description rdf:about="" xmlns:apdi="http://ns.apple.com/pixeldatainfo/1.0/" xmlns:HDRGainMap="http://ns.apple.com/HDRGainMap/1.0/" xmlns:HDRToneMap="http://ns.apple.com/HDRToneMap/1.0/">"#,
|
||||
);
|
||||
s.push_str("<apdi:AuxiliaryImageType>urn:com:apple:photo:2020:aux:hdrgainmap</apdi:AuxiliaryImageType>");
|
||||
s.push_str(&format!("<apdi:NativeFormat>{format}</apdi:NativeFormat><apdi:StoredFormat>{format}</apdi:StoredFormat>"));
|
||||
s.push_str("<HDRGainMap:HDRGainMapVersion>131072</HDRGainMap:HDRGainMapVersion>");
|
||||
// linear, not log2: the headroom multiplier itself
|
||||
s.push_str(&format!("<HDRGainMap:HDRGainMapHeadroom>{}</HDRGainMap:HDRGainMapHeadroom>", num(self.alternate_headroom.exp2())));
|
||||
s.push_str(&format!("<HDRToneMap:AlternateHeadroom>{}</HDRToneMap:AlternateHeadroom>", num(self.alternate_headroom)));
|
||||
s.push_str("<HDRToneMap:ChannelMetadata><rdf:Seq>");
|
||||
for c in 0..self.channels() {
|
||||
s.push_str(&format!(
|
||||
r#"<rdf:li rdf:parseType="Resource"><HDRToneMap:GainMapMin>{}</HDRToneMap:GainMapMin><HDRToneMap:GainMapMax>{}</HDRToneMap:GainMapMax><HDRToneMap:Gamma>{}</HDRToneMap:Gamma><HDRToneMap:BaseOffset>{}</HDRToneMap:BaseOffset><HDRToneMap:AlternateOffset>{}</HDRToneMap:AlternateOffset></rdf:li>"#,
|
||||
num(self.min[c]),
|
||||
num(self.max[c]),
|
||||
num(self.gamma[c]),
|
||||
num(self.base_offset[c]),
|
||||
num(self.alternate_offset[c])
|
||||
));
|
||||
}
|
||||
s.push_str("</rdf:Seq></HDRToneMap:ChannelMetadata>");
|
||||
s.push_str(&format!("<HDRToneMap:BaseHeadroom>{}</HDRToneMap:BaseHeadroom>", num(self.base_headroom)));
|
||||
s.push_str("<HDRToneMap:BaseColorIsWorkingColor>True</HDRToneMap:BaseColorIsWorkingColor><HDRToneMap:Version>1</HDRToneMap:Version>");
|
||||
s.push_str("</rdf:Description>");
|
||||
s
|
||||
}
|
||||
}
|
||||
|
||||
const FLAG_MULTICHANNEL: u8 = 0x80;
|
||||
const FLAG_BASE_COLOUR_SPACE: u8 = 0x40;
|
||||
const FLAG_COMMON_DENOMINATOR: u8 = 0x08;
|
||||
const FLAG_BACKWARD: u8 = 0x04;
|
||||
|
||||
const XMP_OPEN: &str = "<?xpacket begin=\"\u{feff}\" id=\"W5M0MpCehiHzreSzNTczkc9d\"?><x:xmpmeta xmlns:x=\"adobe:ns:meta/\"><rdf:RDF xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">";
|
||||
const XMP_CLOSE: &str = "</rdf:RDF></x:xmpmeta><?xpacket end=\"w\"?>";
|
||||
|
||||
/// A short decimal (up to 6 significant digits, no exponent for ordinary values).
|
||||
fn num(v: f32) -> String {
|
||||
if !v.is_finite() {
|
||||
return "0".into();
|
||||
}
|
||||
let s = format!("{:.6}", v);
|
||||
let s = s.trim_end_matches('0').trim_end_matches('.');
|
||||
if s == "-0" || s.is_empty() { "0".into() } else { s.to_string() }
|
||||
}
|
||||
|
||||
/// The values of `hdrgm:<name>`: an attribute (`name="v"`) or an element holding an `rdf:Seq`.
|
||||
fn xmp_values(xmp: &str, name: &str) -> Option<Vec<String>> {
|
||||
let attr = format!("hdrgm:{name}=");
|
||||
if let Some(i) = xmp.find(&attr) {
|
||||
let rest = xmp.get(i + attr.len()..)?;
|
||||
let q = rest.chars().next()?;
|
||||
if q == '"' || q == '\'' {
|
||||
let body = rest.get(1..)?;
|
||||
let end = body.find(q)?;
|
||||
return Some(vec![body.get(..end)?.to_string()]);
|
||||
}
|
||||
}
|
||||
let open = format!("<hdrgm:{name}>");
|
||||
let close = format!("</hdrgm:{name}>");
|
||||
let i = xmp.find(&open)?;
|
||||
let body = xmp.get(i + open.len()..)?;
|
||||
let body = body.get(..body.find(&close)?)?;
|
||||
let mut out = Vec::new();
|
||||
let mut rest = body;
|
||||
while let Some(a) = rest.find("<rdf:li>") {
|
||||
let after = rest.get(a + 8..)?;
|
||||
let b = after.find("</rdf:li>")?;
|
||||
out.push(after.get(..b)?.to_string());
|
||||
rest = after.get(b..)?;
|
||||
if out.len() > 3 {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
if out.is_empty() {
|
||||
// a plain element value
|
||||
out.push(body.trim().to_string());
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
struct Reader<'a> {
|
||||
b: &'a [u8],
|
||||
at: usize,
|
||||
}
|
||||
|
||||
impl Reader<'_> {
|
||||
fn take<const N: usize>(&mut self) -> Option<[u8; N]> {
|
||||
let s = self.b.get(self.at..self.at.checked_add(N)?)?;
|
||||
self.at += N;
|
||||
s.try_into().ok()
|
||||
}
|
||||
fn u8(&mut self) -> Option<u8> {
|
||||
self.take::<1>().map(|a| a[0])
|
||||
}
|
||||
fn u16(&mut self) -> Option<u16> {
|
||||
self.take().map(u16::from_be_bytes)
|
||||
}
|
||||
fn u32(&mut self) -> Option<u32> {
|
||||
self.take().map(u32::from_be_bytes)
|
||||
}
|
||||
}
|
||||
|
||||
/// A gain map image: `channels` (1 or 3) interleaved 8-bit codes.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct GainImage {
|
||||
pub width: usize,
|
||||
pub height: usize,
|
||||
pub channels: usize,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
impl GainImage {
|
||||
/// Bilinear code of channel `c` at gain map coordinates (`x`, `y`) (pixel centres at +0.5).
|
||||
fn sample(&self, x: f32, y: f32, c: usize) -> f32 {
|
||||
let (w, h) = (self.width, self.height);
|
||||
let fx = (x - 0.5).clamp(0.0, (w - 1) as f32);
|
||||
let fy = (y - 0.5).clamp(0.0, (h - 1) as f32);
|
||||
let (x0, y0) = (fx as usize, fy as usize);
|
||||
let (x1, y1) = ((x0 + 1).min(w - 1), (y0 + 1).min(h - 1));
|
||||
let (tx, ty) = (fx - x0 as f32, fy - y0 as f32);
|
||||
let at = |x: usize, y: usize| self.data.get((y * w + x) * self.channels + c).copied().unwrap_or(0) as f32;
|
||||
let a = at(x0, y0) + (at(x1, y0) - at(x0, y0)) * tx;
|
||||
let b = at(x0, y1) + (at(x1, y1) - at(x0, y1)) * tx;
|
||||
a + (b - a) * ty
|
||||
}
|
||||
}
|
||||
|
||||
/// How [`compute`] builds the map.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct GainMapOptions {
|
||||
/// Downscale factor of the map (1, 2 or 4).
|
||||
pub scale: usize,
|
||||
/// Three gains (R, G, B) instead of one luminance gain. Apple's readers want a single
|
||||
/// channel, so this is off by default.
|
||||
pub multichannel: bool,
|
||||
pub gamma: f32,
|
||||
/// Offsets added to SDR and HDR before the ratio (keep blacks finite and noise quiet).
|
||||
pub offset: f32,
|
||||
}
|
||||
|
||||
impl Default for GainMapOptions {
|
||||
fn default() -> Self {
|
||||
GainMapOptions { scale: 2, multichannel: false, gamma: 1.0, offset: 1.0 / 64.0 }
|
||||
}
|
||||
}
|
||||
|
||||
/// Measure the gain map that turns `sdr` into `hdr` (both `width × height` linear RGB in the same
|
||||
/// primaries; `luma` = that space's luminance weights).
|
||||
pub fn compute(
|
||||
sdr: &[[f32; 3]],
|
||||
hdr: &[[f32; 3]],
|
||||
width: usize,
|
||||
height: usize,
|
||||
luma: [f32; 3],
|
||||
o: &GainMapOptions,
|
||||
) -> Result<(GainImage, GainMapMeta)> {
|
||||
let n = width.checked_mul(height).ok_or_else(|| Error::Encode("gain map: image too large".into()))?;
|
||||
if n == 0 || sdr.len() < n || hdr.len() < n {
|
||||
return Err(Error::Encode("gain map: SDR and HDR renditions must match the image size".into()));
|
||||
}
|
||||
if !matches!(o.scale, 1 | 2 | 4) || !(o.gamma > 0.0 && o.gamma.is_finite()) || !(o.offset > 0.0 && o.offset.is_finite()) {
|
||||
return Err(Error::Encode("gain map: scale must be 1, 2 or 4, gamma and offset positive".into()));
|
||||
}
|
||||
let s = o.scale;
|
||||
let (gw, gh) = (width.div_ceil(s), height.div_ceil(s));
|
||||
let ch = if o.multichannel { 3 } else { 1 };
|
||||
let y = |p: &[f32; 3]| luma[0] * p[0] + luma[1] * p[1] + luma[2] * p[2];
|
||||
let ratio = |a: f32, b: f32| ((b.max(0.0) + o.offset) / (a.max(0.0) + o.offset)).log2();
|
||||
// block means of the log gains
|
||||
let mut gains = vec![0f32; gw * gh * ch];
|
||||
let mut lo = [f32::MAX; 3];
|
||||
let mut hi = [f32::MIN; 3];
|
||||
for gy in 0..gh {
|
||||
for gx in 0..gw {
|
||||
let mut acc = [0f32; 3];
|
||||
let mut count = 0f32;
|
||||
for yy in gy * s..((gy + 1) * s).min(height) {
|
||||
for xx in gx * s..((gx + 1) * s).min(width) {
|
||||
let i = yy * width + xx;
|
||||
let (Some(a), Some(b)) = (sdr.get(i), hdr.get(i)) else { continue };
|
||||
if o.multichannel {
|
||||
for c in 0..3 {
|
||||
acc[c] += ratio(a[c], b[c]);
|
||||
}
|
||||
} else {
|
||||
acc[0] += ratio(y(a), y(b));
|
||||
}
|
||||
count += 1.0;
|
||||
}
|
||||
}
|
||||
for c in 0..ch {
|
||||
let g = if count > 0.0 { acc[c] / count } else { 0.0 };
|
||||
let g = if g.is_finite() { g } else { 0.0 };
|
||||
gains[(gy * gw + gx) * ch + c] = g;
|
||||
lo[c] = lo[c].min(g);
|
||||
hi[c] = hi[c].max(g);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut m = GainMapMeta {
|
||||
multichannel: o.multichannel,
|
||||
gamma: [o.gamma; 3],
|
||||
base_offset: [o.offset; 3],
|
||||
alternate_offset: [o.offset; 3],
|
||||
..GainMapMeta::default()
|
||||
};
|
||||
for c in 0..ch {
|
||||
// the range always includes 0 (no change) so an SDR-only area encodes exactly
|
||||
m.min[c] = lo[c].min(0.0);
|
||||
m.max[c] = hi[c].max(m.min[c] + 1e-4);
|
||||
}
|
||||
if !o.multichannel {
|
||||
m.fill_from_first();
|
||||
}
|
||||
m.base_headroom = 0.0;
|
||||
m.alternate_headroom = m.max.iter().copied().fold(0.0f32, f32::max).max(1e-3);
|
||||
let data = gains
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, g)| {
|
||||
let c = i % ch;
|
||||
let norm = ((g - m.min[c]) / (m.max[c] - m.min[c])).clamp(0.0, 1.0);
|
||||
(norm.powf(o.gamma) * 255.0).round() as u8
|
||||
})
|
||||
.collect();
|
||||
Ok((GainImage { width: gw, height: gh, channels: ch, data }, m))
|
||||
}
|
||||
|
||||
/// Rebuild the HDR rendition in place: `base` (`width × height` linear RGB) is replaced by the
|
||||
/// rendition at `weight` (0 = the base, 1 = the full alternate rendition; a display with headroom
|
||||
/// `H` uses `(H − base_headroom) / (alternate_headroom − base_headroom)`, clamped).
|
||||
pub fn apply(base: &mut [[f32; 3]], width: usize, height: usize, gain: &GainImage, m: &GainMapMeta, weight: f32) -> Result<()> {
|
||||
let n = width.checked_mul(height).ok_or_else(|| Error::Encode("gain map: image too large".into()))?;
|
||||
if base.len() < n
|
||||
|| gain.width == 0
|
||||
|| gain.height == 0
|
||||
|| !matches!(gain.channels, 1 | 3)
|
||||
|| gain.data.len() < gain.width * gain.height * gain.channels
|
||||
{
|
||||
return Err(Error::Encode("gain map: base and gain map sizes don't match".into()));
|
||||
}
|
||||
if !m.is_valid() {
|
||||
return Err(Error::Encode("gain map: unusable metadata".into()));
|
||||
}
|
||||
let weight = if weight.is_finite() { weight.clamp(0.0, 1.0) } else { 1.0 };
|
||||
let (sx, sy) = (gain.width as f32 / width as f32, gain.height as f32 / height as f32);
|
||||
let inv_gamma = m.gamma.map(|g| 1.0 / g);
|
||||
for (i, p) in base.iter_mut().enumerate().take(n) {
|
||||
let (x, y) = ((i % width) as f32 + 0.5, (i / width) as f32 + 0.5);
|
||||
let (gx, gy) = (x * sx, y * sy);
|
||||
let mut g = [0f32; 3];
|
||||
for (c, gc) in g.iter_mut().enumerate() {
|
||||
let src = if gain.channels == 3 { c } else { 0 };
|
||||
let norm = (gain.sample(gx, gy, src) / 255.0).clamp(0.0, 1.0).powf(inv_gamma[c]);
|
||||
*gc = m.min[c] + (m.max[c] - m.min[c]) * norm;
|
||||
}
|
||||
for c in 0..3 {
|
||||
let v = (p[c] + m.base_offset[c]) * (g[c] * weight).exp2() - m.alternate_offset[c];
|
||||
p[c] = v.max(0.0);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The primary image's container XMP: `hdrgm:Version` (marks the file as a gain map image for
|
||||
/// Adobe/Google readers) and the Container directory naming the gain map and its length.
|
||||
fn primary_description(gain_len: usize) -> String {
|
||||
format!(
|
||||
concat!(
|
||||
r#"<rdf:Description rdf:about="" xmlns:Container="http://ns.google.com/photos/1.0/container/" xmlns:Item="http://ns.google.com/photos/1.0/container/item/" xmlns:hdrgm="http://ns.adobe.com/hdr-gain-map/1.0/" hdrgm:Version="1.0">"#,
|
||||
r#"<Container:Directory><rdf:Seq>"#,
|
||||
r#"<rdf:li rdf:parseType="Resource"><Container:Item Item:Semantic="Primary" Item:Mime="image/jpeg"/></rdf:li>"#,
|
||||
r#"<rdf:li rdf:parseType="Resource"><Container:Item Item:Semantic="GainMap" Item:Mime="image/jpeg" Item:Length="{}"/></rdf:li>"#,
|
||||
r#"</rdf:Seq></Container:Directory></rdf:Description>"#
|
||||
),
|
||||
gain_len
|
||||
)
|
||||
}
|
||||
|
||||
/// `xmp` with `description` added as another `rdf:Description` (a fresh packet when there is
|
||||
/// none, or when it has no `rdf:RDF` to add to).
|
||||
fn merge_xmp(xmp: Option<&str>, description: &str) -> String {
|
||||
if let Some(x) = xmp
|
||||
&& let Some(i) = x.rfind("</rdf:RDF>")
|
||||
&& let (Some(a), Some(b)) = (x.get(..i), x.get(i..))
|
||||
{
|
||||
return format!("{a}{description}{b}");
|
||||
}
|
||||
format!("{XMP_OPEN}{description}{XMP_CLOSE}")
|
||||
}
|
||||
|
||||
/// Offsets within the MPF payload (from "MPF\0"): the endian field, then the index IFD (3 tags).
|
||||
const MPF_ENTRIES_FROM_ENDIAN: usize = 8 + 2 + 3 * 12 + 4;
|
||||
|
||||
/// The primary image's MPF payload, entry sizes and offset to be patched once the file exists.
|
||||
fn mpf_placeholder() -> Vec<u8> {
|
||||
let mut p = b"MPF\0MM".to_vec();
|
||||
p.extend_from_slice(&42u16.to_be_bytes());
|
||||
p.extend_from_slice(&8u32.to_be_bytes()); // first IFD
|
||||
p.extend_from_slice(&3u16.to_be_bytes());
|
||||
let tag = |p: &mut Vec<u8>, tag: u16, ty: u16, count: u32, value: [u8; 4]| {
|
||||
p.extend_from_slice(&tag.to_be_bytes());
|
||||
p.extend_from_slice(&ty.to_be_bytes());
|
||||
p.extend_from_slice(&count.to_be_bytes());
|
||||
p.extend_from_slice(&value);
|
||||
};
|
||||
tag(&mut p, 0xB000, 7, 4, *b"0100"); // MPFVersion
|
||||
tag(&mut p, 0xB001, 4, 1, 2u32.to_be_bytes()); // NumberOfImages
|
||||
tag(&mut p, 0xB002, 7, 32, (MPF_ENTRIES_FROM_ENDIAN as u32).to_be_bytes()); // MPEntry
|
||||
p.extend_from_slice(&0u32.to_be_bytes()); // no next IFD
|
||||
// entry 0: baseline primary (0x030000) flagged as the representative image (bit 29)
|
||||
for attr in [0x2003_0000u32, jpeg::MPF_GAIN_MAP] {
|
||||
p.extend_from_slice(&attr.to_be_bytes());
|
||||
p.extend_from_slice(&[0; 12]); // size, offset, two dependent-image entries
|
||||
}
|
||||
p
|
||||
}
|
||||
|
||||
/// Encode a gain map JPEG: `base` (8-bit SDR, 3 or 4 channels, colour-managed by `meta.icc`) and
|
||||
/// its gain map, both at `quality`. `meta.xmp` is kept, with the gain map container added.
|
||||
pub fn encode_jpeg(base: &EncodeImage, gain: &GainImage, m: &GainMapMeta, quality: u8, sub: ChromaSubsampling, meta: &EncodeMeta) -> Result<Vec<u8>> {
|
||||
let Samples::U8(data) = base.samples else {
|
||||
return Err(Error::Encode("gain map JPEG needs an 8-bit base image".into()));
|
||||
};
|
||||
let (w, h) = (base.width as usize, base.height as usize);
|
||||
if !matches!(base.channels, 3 | 4) || data.len() < w * h * base.channels as usize {
|
||||
return Err(Error::Encode("gain map JPEG needs an RGB base image".into()));
|
||||
}
|
||||
if w == 0 || h == 0 || w > u16::MAX as usize || h > u16::MAX as usize {
|
||||
return Err(Error::Encode("JPEG dimensions are limited to 65535".into()));
|
||||
}
|
||||
if !matches!(gain.channels, 1 | 3) || gain.width == 0 || gain.height == 0 || gain.data.len() < gain.width * gain.height * gain.channels {
|
||||
return Err(Error::Encode("gain map image is malformed".into()));
|
||||
}
|
||||
if !m.is_valid() || m.multichannel != (gain.channels == 3) {
|
||||
return Err(Error::Encode("gain map metadata doesn't match the map".into()));
|
||||
}
|
||||
// jpeg_par writes 4:4:4 or 4:2:0
|
||||
let sub = if sub == ChromaSubsampling::S422 { ChromaSubsampling::S420 } else { sub };
|
||||
|
||||
let gain_segs = vec![(0xE1, xmp_segment(&m.to_gain_map_xmp())?), (0xE2, m.to_iso())];
|
||||
let gain_jpeg = jpeg_par::encode(&gain.data, gain.width, gain.height, gain.channels, quality, ChromaSubsampling::S444, &gain_segs);
|
||||
|
||||
let xmp = merge_xmp(meta.xmp, &primary_description(gain_jpeg.len()));
|
||||
let mut segs = jpeg_app_segments(&EncodeMeta { xmp: Some(&xmp), ..*meta })?;
|
||||
let mut iso_marker = ISO_URN.to_vec();
|
||||
iso_marker.extend_from_slice(&[0, 0, 0, 0]); // versions only: "this image has a gain map"
|
||||
segs.push((0xE2, iso_marker));
|
||||
segs.push((0xE2, mpf_placeholder()));
|
||||
let mut primary = jpeg_par::encode(data, w, h, base.channels as usize, quality, sub, &segs);
|
||||
|
||||
// patch the MPF index: sizes, and the gain map's offset from the MPF endian field
|
||||
let markers = jpeg::parse_markers(&primary).ok_or_else(|| Error::Encode("gain map JPEG: primary image unreadable".into()))?;
|
||||
let endian = markers.mpf.as_ref().map(|m| m.0).ok_or_else(|| Error::Encode("gain map JPEG: MPF segment missing".into()))?;
|
||||
let entries = endian + MPF_ENTRIES_FROM_ENDIAN;
|
||||
let (plen, glen) = (primary.len(), gain_jpeg.len());
|
||||
let fields = [(entries + 4, plen), (entries + 8, 0), (entries + 20, glen), (entries + 24, plen - endian)];
|
||||
for (at, v) in fields {
|
||||
let v = u32::try_from(v).map_err(|_| Error::Encode("gain map JPEG larger than 4 GiB".into()))?;
|
||||
let slot = primary.get_mut(at..at + 4).ok_or_else(|| Error::Encode("gain map JPEG: truncated MPF segment".into()))?;
|
||||
slot.copy_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
primary.extend_from_slice(&gain_jpeg);
|
||||
Ok(primary)
|
||||
}
|
||||
|
||||
/// The gain map of a gain map JPEG.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct JpegGainMap {
|
||||
pub gain: GainImage,
|
||||
pub meta: GainMapMeta,
|
||||
}
|
||||
|
||||
/// Find and decode the gain map of a JPEG (`None` when it has none or it's unusable). The
|
||||
/// primary image is decoded as usual ([`crate::decode`]); combine the two with [`apply`].
|
||||
pub fn read_jpeg(bytes: &[u8]) -> Option<JpegGainMap> {
|
||||
let m = jpeg::parse_markers(bytes)?;
|
||||
let marked = m.iso_gainmap.is_some() || m.xmp.as_deref().is_some_and(|x| x.contains("hdrgm:Version"));
|
||||
if !marked {
|
||||
return None;
|
||||
}
|
||||
for e in jpeg::mpf_images(&m) {
|
||||
let Some(data) = bytes.get(e.offset..e.offset.saturating_add(e.len)) else { continue };
|
||||
let Some(gm) = jpeg::parse_markers(data) else { continue };
|
||||
let meta = gm.iso_gainmap.as_deref().and_then(GainMapMeta::from_iso).or_else(|| gm.xmp.as_deref().and_then(GainMapMeta::from_hdrgm_xmp));
|
||||
let Some(meta) = meta else { continue };
|
||||
let Ok((w, h, ch, codes)) = jpeg::decode_codes8(data) else { continue };
|
||||
if ch == 3 && !meta.multichannel {
|
||||
// a single-channel map stored as RGB: use its first channel
|
||||
let g: Vec<u8> = codes.as_chunks::<3>().0.iter().map(|c| c[0]).collect();
|
||||
return Some(JpegGainMap { gain: GainImage { width: w, height: h, channels: 1, data: g }, meta });
|
||||
}
|
||||
let mut meta = meta;
|
||||
if ch == 1 && meta.multichannel {
|
||||
meta.multichannel = false;
|
||||
meta.fill_from_first();
|
||||
}
|
||||
return Some(JpegGainMap { gain: GainImage { width: w, height: h, channels: ch, data: codes }, meta });
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Whether `bytes` is a gain map JPEG with a usable gain map (cheap: no pixel decode).
|
||||
pub fn is_gain_map_jpeg(bytes: &[u8]) -> bool {
|
||||
crate::sniff(bytes) == Some(Format::Jpeg)
|
||||
&& jpeg::parse_markers(bytes).is_some_and(|m| {
|
||||
(m.iso_gainmap.is_some() || m.xmp.as_deref().is_some_and(|x| x.contains("hdrgm:Version")))
|
||||
&& jpeg::mpf_images(&m)
|
||||
.iter()
|
||||
.any(|e| bytes.get(e.offset..e.offset.saturating_add(e.len)).and_then(jpeg::parse_markers).is_some_and(|g| g.is_gain_map()))
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn meta3() -> GainMapMeta {
|
||||
GainMapMeta {
|
||||
multichannel: true,
|
||||
use_base_colour_space: true,
|
||||
base_headroom: 0.0,
|
||||
alternate_headroom: 2.5,
|
||||
min: [-0.25, -0.5, 0.0],
|
||||
max: [2.5, 2.25, 2.0],
|
||||
gamma: [1.0, 1.25, 0.75],
|
||||
base_offset: [0.015625; 3],
|
||||
alternate_offset: [0.015625, 0.03125, 0.015625],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn iso_metadata_round_trips() {
|
||||
for m in [GainMapMeta { alternate_headroom: 3.0, max: [3.0; 3], ..GainMapMeta::default() }, meta3()] {
|
||||
let p = m.to_iso();
|
||||
assert!(p.starts_with(ISO_URN));
|
||||
let back = GainMapMeta::from_iso(&p[ISO_URN.len()..]).unwrap();
|
||||
for (a, b) in [(back.min, m.min), (back.max, m.max), (back.gamma, m.gamma), (back.alternate_offset, m.alternate_offset)] {
|
||||
for c in 0..3 {
|
||||
assert!((a[c] - b[c]).abs() < 1e-6, "{a:?} {b:?}");
|
||||
}
|
||||
}
|
||||
assert_eq!(back.multichannel, m.multichannel);
|
||||
assert!((back.alternate_headroom - m.alternate_headroom).abs() < 1e-6);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn iso_common_denominator_and_rejections() {
|
||||
// common-denominator form, single channel: d = 4, headrooms 0 and 2, min 0, max 2, gamma 1, offsets 0
|
||||
let mut p = vec![0, 0, 0, 0, FLAG_COMMON_DENOMINATOR];
|
||||
for v in [4u32, 0, 8, 0, 8, 4, 0, 0] {
|
||||
p.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
let m = GainMapMeta::from_iso(&p).unwrap();
|
||||
assert_eq!((m.alternate_headroom, m.max[2], m.gamma[1]), (2.0, 2.0, 1.0));
|
||||
// versions only (a primary image's marker), unknown version, truncated, backward
|
||||
assert!(GainMapMeta::from_iso(&[0, 0, 0, 0]).is_none());
|
||||
assert!(GainMapMeta::from_iso(&[0, 1, 0, 0, 0]).is_none());
|
||||
assert!(GainMapMeta::from_iso(&p[..p.len() - 1]).is_none());
|
||||
let mut back = p.clone();
|
||||
back[4] |= FLAG_BACKWARD;
|
||||
assert!(GainMapMeta::from_iso(&back).is_none());
|
||||
// a zero denominator is rejected, not divided by
|
||||
let mut z = p.clone();
|
||||
z[5..9].copy_from_slice(&0u32.to_be_bytes());
|
||||
assert!(GainMapMeta::from_iso(&z).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hdrgm_xmp_round_trips_including_per_channel_sequences() {
|
||||
for m in [GainMapMeta { alternate_headroom: 2.0, max: [2.0; 3], ..GainMapMeta::default() }, meta3()] {
|
||||
let x = m.to_gain_map_xmp();
|
||||
assert!(x.contains("apdi:AuxiliaryImageType"));
|
||||
if m.multichannel {
|
||||
assert!(x.contains("<hdrgm:GainMapMax><rdf:Seq><rdf:li>2.5</rdf:li>"), "{x}");
|
||||
}
|
||||
let back = GainMapMeta::from_hdrgm_xmp(&x).unwrap();
|
||||
assert_eq!(back.multichannel, m.multichannel);
|
||||
for c in 0..3 {
|
||||
assert!((back.max[c] - m.max[c]).abs() < 1e-5 && (back.gamma[c] - m.gamma[c]).abs() < 1e-5);
|
||||
}
|
||||
assert!((back.alternate_headroom - m.alternate_headroom).abs() < 1e-5);
|
||||
}
|
||||
assert!(GainMapMeta::from_hdrgm_xmp("<x hdrgm:Version=\"1.0\"/>").is_none(), "no GainMapMax");
|
||||
assert!(GainMapMeta::from_hdrgm_xmp("hdrgm:GainMapMax=\"nope\"").is_none());
|
||||
}
|
||||
|
||||
fn renditions(w: usize, h: usize) -> (Vec<[f32; 3]>, Vec<[f32; 3]>) {
|
||||
// a ramp: SDR rolls off at 1, HDR keeps going to 6
|
||||
let hdr: Vec<[f32; 3]> = (0..w * h)
|
||||
.map(|i| {
|
||||
let t = (i % w) as f32 / (w - 1) as f32;
|
||||
let v = 0.01 + 6.0 * t * t;
|
||||
[v, v * 0.8, v * 0.6]
|
||||
})
|
||||
.collect();
|
||||
// a luminance tone map (one scale per pixel, as LightCraft's renders do), so a single
|
||||
// luminance gain can rebuild it exactly
|
||||
let sdr = hdr
|
||||
.iter()
|
||||
.map(|p| {
|
||||
let y = 0.2126 * p[0] + 0.7152 * p[1] + 0.0722 * p[2];
|
||||
let k = 1.0 / (1.0 + y * 0.5);
|
||||
p.map(|v| v * k)
|
||||
})
|
||||
.collect();
|
||||
(sdr, hdr)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_then_apply_recovers_the_hdr_rendition() {
|
||||
let (w, h) = (64, 8);
|
||||
let (sdr, hdr) = renditions(w, h);
|
||||
for multichannel in [false, true] {
|
||||
let o = GainMapOptions { scale: 1, multichannel, ..GainMapOptions::default() };
|
||||
let (g, m) = compute(&sdr, &hdr, w, h, [0.2126, 0.7152, 0.0722], &o).unwrap();
|
||||
assert!(m.is_valid() && m.alternate_headroom > 1.0);
|
||||
let mut out = sdr.clone();
|
||||
apply(&mut out, w, h, &g, &m, 1.0).unwrap();
|
||||
for (a, b) in out.iter().zip(&hdr) {
|
||||
for c in 0..3 {
|
||||
// 8-bit codes over the gain range: within ~2 % in linear light
|
||||
assert!((a[c] - b[c]).abs() <= 0.03 * b[c] + 0.01, "{multichannel}: {a:?} vs {b:?}");
|
||||
}
|
||||
}
|
||||
// weight 0 is the base
|
||||
let mut base = sdr.clone();
|
||||
apply(&mut base, w, h, &g, &m, 0.0).unwrap();
|
||||
for (a, b) in base.iter().zip(&sdr) {
|
||||
assert!((a[0] - b[0]).abs() < 1e-5);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jpeg_round_trip_and_container_structure() {
|
||||
let (w, h) = (96, 64);
|
||||
let (sdr, hdr) = renditions(w, h);
|
||||
let (g, m) = compute(&sdr, &hdr, w, h, [0.2126, 0.7152, 0.0722], &GainMapOptions::default()).unwrap();
|
||||
let base: Vec<u8> =
|
||||
sdr.iter().flat_map(|p| p.map(|v| (lightcraft_color::transfer::linear_to_srgb(v.clamp(0.0, 1.0)) * 255.0).round() as u8)).collect();
|
||||
let user_xmp = r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><rdf:Description rdf:about="" xmlns:dc="http://purl.org/dc/elements/1.1/"/></rdf:RDF></x:xmpmeta>"#;
|
||||
let meta = EncodeMeta { xmp: Some(user_xmp), ..EncodeMeta::default() };
|
||||
let file = encode_jpeg(&EncodeImage::new(w as u32, h as u32, 3, Samples::U8(&base)), &g, &m, 90, ChromaSubsampling::S444, &meta).unwrap();
|
||||
assert!(is_gain_map_jpeg(&file));
|
||||
// the primary decodes as an ordinary JPEG of the right size, and keeps the user's XMP
|
||||
let d = crate::decode(&file, crate::DecodeOptions::default()).unwrap();
|
||||
assert_eq!((d.width, d.height), (w as u32, h as u32));
|
||||
let x = d.xmp.unwrap();
|
||||
assert!(x.contains("xmlns:dc=") && x.contains("hdrgm:Version") && x.contains("Item:Semantic=\"GainMap\""));
|
||||
// the MPF index points at the gain map, typed as one
|
||||
let pm = jpeg::parse_markers(&file).unwrap();
|
||||
let e = jpeg::mpf_images(&pm);
|
||||
assert_eq!(e.len(), 1);
|
||||
assert_eq!(e[0].kind, jpeg::MPF_GAIN_MAP);
|
||||
assert_eq!(e[0].offset + e[0].len, file.len());
|
||||
// and the gain map reads back
|
||||
let r = read_jpeg(&file).unwrap();
|
||||
assert_eq!((r.gain.width, r.gain.height, r.gain.channels), (48, 32, 1));
|
||||
assert!((r.meta.max[0] - m.max[0]).abs() < 1e-5);
|
||||
// the gain map is never a thumbnail
|
||||
if let Some(t) = jpeg::embedded_thumbnail(&file, 32, 16) {
|
||||
assert_ne!(t.source, crate::ThumbnailSource::MpfPreview);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn garbage_never_panics() {
|
||||
assert!(read_jpeg(b"").is_none());
|
||||
assert!(read_jpeg(&[0xFF, 0xD8, 0xFF, 0xE2, 0x00, 0x04, 0, 0]).is_none());
|
||||
assert!(!is_gain_map_jpeg(&[0xFF, 0xD8]));
|
||||
let mut b = vec![0xFF, 0xD8, 0xFF, 0xE2, 0, 40];
|
||||
b.extend_from_slice(ISO_URN);
|
||||
b.extend_from_slice(&[0, 0, 0, 0, 0xFF, 1, 2, 3, 4]);
|
||||
assert!(read_jpeg(&b).is_none());
|
||||
let g = GainImage { width: 2, height: 2, channels: 1, data: vec![0; 3] };
|
||||
assert!(apply(&mut [[0.0; 3]; 4], 2, 2, &g, &GainMapMeta::default(), 1.0).is_err());
|
||||
assert!(compute(&[], &[], 0, 0, [0.3, 0.6, 0.1], &GainMapOptions::default()).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
//! HEIF / HEIC (the iPhone and Mac photo format), read-only.
|
||||
//!
|
||||
//! Decoding lives in the optional `lightcraft-heif` crate (heic-rs, a pure-Rust HEVC
|
||||
//! still-picture decoder, plus a libheif-matching colour conversion), enabled by this crate's
|
||||
//! `heif` feature — off by default because HEVC is patent-encumbered and whether a build carries
|
||||
//! an HEVC decoder is the distributor's call (the same policy as the sibling PhotoCraft). Without
|
||||
//! the feature, HEIF files are still detected and opening one is an [`Error::Unsupported`] error
|
||||
//! saying so, never a panic.
|
||||
//!
|
||||
//! HEIF records orientation in the container (`irot`/`imir`, plus a `clap` crop), not in EXIF,
|
||||
//! so the decoder applies them and `orientation` is reported as 1: the EXIF Orientation tag only
|
||||
//! mirrors what the container says, and applying it as well would turn the photo twice.
|
||||
//! The samples come back as 16-bit RGB in the container's own primaries, with its ICC profile
|
||||
//! (iPhones: Display P3) or, without one, its `nclx` / VUI colour description; like every decoder
|
||||
//! here, the colour interpretation happens in [`crate::convert::finish`].
|
||||
|
||||
#[cfg(feature = "heif")]
|
||||
use crate::convert::{Buf, Meta, Model, Raw, finish};
|
||||
use crate::{DecodeOptions, Decoded, Error, Format, Result};
|
||||
|
||||
const F: Format = Format::Heif;
|
||||
|
||||
/// The reason given when this build has no HEIF decoder.
|
||||
#[cfg(not(feature = "heif"))]
|
||||
pub(crate) const NOT_IN_BUILD: &str = "HEIC/HEIF support isn't included in this build of LightCraft";
|
||||
|
||||
#[cfg(not(feature = "heif"))]
|
||||
pub(crate) fn decode(_bytes: &[u8], _opts: &DecodeOptions) -> Result<Decoded> {
|
||||
Err(Error::Unsupported(F, NOT_IN_BUILD))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "heif"))]
|
||||
pub(crate) fn header(_bytes: &[u8]) -> Result<(u32, u32, u16)> {
|
||||
Err(Error::Unsupported(F, NOT_IN_BUILD))
|
||||
}
|
||||
|
||||
#[cfg(feature = "heif")]
|
||||
fn err(e: lightcraft_heif::Error) -> Error {
|
||||
match e {
|
||||
lightcraft_heif::Error::Unsupported(why) => Error::Unsupported(F, why),
|
||||
lightcraft_heif::Error::Limit(m) | lightcraft_heif::Error::Malformed(m) => Error::Malformed(F, m),
|
||||
}
|
||||
}
|
||||
|
||||
/// The displayed size (container transforms applied, so orientation 1), from the container
|
||||
/// alone: no picture is decoded.
|
||||
#[cfg(feature = "heif")]
|
||||
pub(crate) fn header(bytes: &[u8]) -> Result<(u32, u32, u16)> {
|
||||
let info = lightcraft_heif::probe(bytes).map_err(err)?;
|
||||
Ok((info.width, info.height, 1))
|
||||
}
|
||||
|
||||
#[cfg(feature = "heif")]
|
||||
pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
// The container alone: the declared size is checked before any pixel is decoded.
|
||||
let info = lightcraft_heif::probe(bytes).map_err(err)?;
|
||||
if (info.width as u64).saturating_mul(info.height as u64) > opts.max_pixels {
|
||||
return Err(Error::TooLarge(info.width as u64, info.height as u64));
|
||||
}
|
||||
let options = lightcraft_heif::Options { max_pixels: opts.max_pixels, apply_transforms: true };
|
||||
// A small decode (a thumbnail or a cataloguing probe) uses the file's own thumbnail image when
|
||||
// it is at least as large as asked: iPhones store one of 320 × 240, and decoding it is a
|
||||
// hundredth of the work of the 12 MP photo.
|
||||
let thumb = match opts.max_size {
|
||||
Some((mw, mh)) if mw > 0 && mh > 0 && mw.max(mh) < info.width.max(info.height) => {
|
||||
lightcraft_heif::decode_thumbnail(bytes, mw.max(mh), &options).ok().flatten().filter(|t| {
|
||||
// Only a thumbnail of the same picture: same orientation and aspect within 1 %.
|
||||
let aspect = |w: u32, h: u32| w as f64 / h.max(1) as f64;
|
||||
(aspect(t.width, t.height) / aspect(info.width, info.height) - 1.0).abs() < 0.01
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let mut decoded = match thumb {
|
||||
Some(t) => t,
|
||||
None => lightcraft_heif::decode(bytes, &options).map_err(err)?,
|
||||
};
|
||||
let raw = Raw {
|
||||
width: decoded.width as usize,
|
||||
height: decoded.height as usize,
|
||||
model: Model::Rgb,
|
||||
alpha: decoded.has_alpha,
|
||||
premultiplied: false,
|
||||
buf: Buf::U16(std::mem::take(&mut decoded.data)),
|
||||
bit_depth: decoded.bit_depth,
|
||||
};
|
||||
let c = decoded.colour;
|
||||
let meta = Meta {
|
||||
icc: decoded.icc,
|
||||
exif: decoded.exif,
|
||||
xmp: decoded.xmp,
|
||||
orientation: Some(1),
|
||||
// Used when there is no (usable) ICC profile.
|
||||
hint: crate::space::SourceSpace::from_cicp(c.primaries, c.transfer),
|
||||
..Default::default()
|
||||
};
|
||||
finish(F, raw, meta, (info.width, info.height), opts)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::{Format, sniff};
|
||||
|
||||
/// The container header both builds recognise, whatever the feature says.
|
||||
const HEIC_HEADER: &[u8] = b"\0\0\0\x18ftypheic\0\0\0\0mif1heic";
|
||||
|
||||
#[test]
|
||||
fn heif_is_always_recognised() {
|
||||
assert_eq!(sniff(HEIC_HEADER), Some(Format::Heif));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn can_decode_follows_the_feature() {
|
||||
assert_eq!(Format::Heif.can_decode(), cfg!(feature = "heif"));
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "heif"))]
|
||||
#[test]
|
||||
fn without_the_feature_heif_is_a_clear_unsupported_error() {
|
||||
let r = crate::decode(HEIC_HEADER, crate::DecodeOptions::default());
|
||||
assert!(matches!(&r, Err(crate::Error::Unsupported(Format::Heif, why)) if why.contains("isn't included in this build")), "{r:?}");
|
||||
let h = crate::read_header(HEIC_HEADER);
|
||||
assert!(matches!(&h, Err(crate::Error::Unsupported(Format::Heif, why)) if why.contains("isn't included in this build")), "{h:?}");
|
||||
}
|
||||
|
||||
/// Found by PhotoCraft's `decode_heif` fuzz target: a malformed box makes heic-rs 0.1.1 slice
|
||||
/// out of range (`boxes.rs:130`). It must be an error, not a crash.
|
||||
#[cfg(feature = "heif")]
|
||||
const HEIC_RS_BOX_PANIC: [u8; 72] = [
|
||||
0x00, 0x00, 0x00, 0x24, 0x66, 0x74, 0x79, 0x70, 0x68, 0x65, 0x69, 0x63, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x69, 0x66, 0x31, 0x4d, 0x69, 0x50,
|
||||
0x72, 0x6d, 0x69, 0x61, 0x66, 0x4d, 0x69, 0x48, 0x42, 0x72, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x65, 0x74, 0x61, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x75, 0x75, 0x69, 0x64, 0x00, 0x00, 0x00, 0x00, 0x03, 0x08, 0x08, 0x08, 0x00, 0x03, 0x01, 0x03, 0x70,
|
||||
0x00, 0xa8, 0x00,
|
||||
];
|
||||
|
||||
#[cfg(feature = "heif")]
|
||||
#[test]
|
||||
fn a_heic_rs_panic_is_a_malformed_error() {
|
||||
let r = crate::decode(&HEIC_RS_BOX_PANIC, crate::DecodeOptions::default());
|
||||
assert!(matches!(&r, Err(crate::Error::Malformed(Format::Heif, _))), "{r:?}");
|
||||
assert!(crate::read_header(&HEIC_RS_BOX_PANIC).is_err());
|
||||
}
|
||||
|
||||
#[cfg(feature = "heif")]
|
||||
#[test]
|
||||
fn an_image_sequence_is_unsupported() {
|
||||
// An ftyp with no meta box: a video track, not a still.
|
||||
let r = crate::decode(b"\0\0\0\x18ftypmsf1\0\0\0\0msf1hevc", crate::DecodeOptions::default());
|
||||
assert!(matches!(&r, Err(crate::Error::Unsupported(Format::Heif, why)) if why.contains("sequence")), "{r:?}");
|
||||
}
|
||||
}
|
||||
@@ -146,7 +146,19 @@ pub fn write_matrix_trc(space: &RgbSpace, trc: &Trc) -> Vec<u8> {
|
||||
};
|
||||
p.description = Some(ProfileText::Localizable(vec![LocalizableString::new("en".into(), "US".into(), desc)]));
|
||||
p.copyright = Some(ProfileText::Localizable(vec![LocalizableString::new("en".into(), "US".into(), "No copyright, use freely".into())]));
|
||||
align_tags(&p.encode().unwrap_or_default())
|
||||
align_tags(&fixed_date(p.encode().unwrap_or_default()))
|
||||
}
|
||||
|
||||
/// Zero the header's creation date/time (ICC.1 §7.2.8, bytes 24..36) so a profile's bytes depend
|
||||
/// only on its colour space and curve. `moxcms` stamps the wall clock there on every `encode`
|
||||
/// (ignoring `ColorProfile::creation_date_time`), which made two exports of one photo differ in
|
||||
/// the embedded profile and nowhere else, so output hashes could not be compared (#180). Readers
|
||||
/// do not interpret the field; all zeros is the conventional "unspecified" value.
|
||||
fn fixed_date(mut b: Vec<u8>) -> Vec<u8> {
|
||||
if let Some(date) = b.get_mut(24..36) {
|
||||
date.fill(0);
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
/// Re-lay a profile's tag data on 4-byte boundaries (ICC.1 §7.3.1). The encoder packs tags back to
|
||||
@@ -193,7 +205,7 @@ pub fn write_named(n: NamedSpace) -> Vec<u8> {
|
||||
write_matrix_trc(&n.rgb_space(), &n.trc())
|
||||
}
|
||||
|
||||
fn linear_rec2020_profile() -> ColorProfile {
|
||||
pub(crate) fn linear_rec2020_profile() -> ColorProfile {
|
||||
let mut p = ColorProfile::new_bt2020();
|
||||
p.cicp = None;
|
||||
let lin = ToneReprCurve::Parametric(vec![1.0]);
|
||||
@@ -300,6 +312,23 @@ mod tests {
|
||||
let _ = SRGB;
|
||||
}
|
||||
|
||||
/// Two profiles for one space are byte-identical, and the header date is fixed: exports of the
|
||||
/// same photo with the same settings hash the same (#180).
|
||||
#[test]
|
||||
fn written_profiles_are_deterministic() {
|
||||
for n in NamedSpace::ALL {
|
||||
let a = write_named(n);
|
||||
let b = write_named(n);
|
||||
assert_eq!(a, b, "{n:?}");
|
||||
assert_eq!(&a[24..36], &[0u8; 12], "{n:?} header date/time");
|
||||
// the profile ID is "not computed", not a hash of a timestamp
|
||||
assert_eq!(&a[84..100], &[0u8; 16], "{n:?} profile ID");
|
||||
}
|
||||
let lin = write_matrix_trc(&PROPHOTO, &Trc::Linear);
|
||||
assert_eq!(lin, write_matrix_trc(&PROPHOTO, &Trc::Linear));
|
||||
assert_eq!(&lin[24..36], &[0u8; 12]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn garbage_is_none() {
|
||||
assert!(parse(&[]).is_none());
|
||||
|
||||
+199
-8
@@ -25,6 +25,17 @@ pub(crate) struct Markers {
|
||||
pub adobe_transform: Option<u8>,
|
||||
/// Offset of the MPF TIFF header within the file and its bytes.
|
||||
pub mpf: Option<(usize, Vec<u8>)>,
|
||||
/// The ISO 21496-1 APP2 payload after its URN: 4 bytes (versions only) on a primary image
|
||||
/// that has a gain map, the full metadata on the gain map image itself.
|
||||
pub iso_gainmap: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl Markers {
|
||||
/// This image is a gain map (ISO 21496-1 metadata or Adobe `hdrgm` gain parameters), not a
|
||||
/// picture: never a preview, never the image to show.
|
||||
pub fn is_gain_map(&self) -> bool {
|
||||
self.iso_gainmap.as_ref().is_some_and(|p| p.len() > 4) || self.xmp.as_deref().is_some_and(|x| x.contains("hdrgm:GainMapMax"))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_markers(b: &[u8]) -> Option<Markers> {
|
||||
@@ -88,6 +99,10 @@ pub(crate) fn parse_markers(b: &[u8]) -> Option<Markers> {
|
||||
icc_chunks.push((seg[12], &seg[14..]));
|
||||
} else if seg.starts_with(b"MPF\0") && m.mpf.is_none() {
|
||||
m.mpf = Some((start + 4, seg[4..].to_vec()));
|
||||
} else if let Some(p) = seg.strip_prefix(crate::gainmap::ISO_URN)
|
||||
&& m.iso_gainmap.is_none()
|
||||
{
|
||||
m.iso_gainmap = Some(p.to_vec());
|
||||
}
|
||||
}
|
||||
0xEE if seg.starts_with(b"Adobe") && seg.len() >= 12 => {
|
||||
@@ -105,12 +120,67 @@ pub(crate) fn parse_markers(b: &[u8]) -> Option<Markers> {
|
||||
}
|
||||
|
||||
pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
decode_with_fallback(bytes, opts, None)
|
||||
}
|
||||
|
||||
/// Stored dimensions and EXIF orientation from the markers, without decoding the scan data. The
|
||||
/// file is refused, as a decode would refuse it, when it has no frame header, an unsupported frame
|
||||
/// type or component count, or when its markers don't run to an end-of-image marker after the
|
||||
/// image data (a truncated file). Damage *inside* the entropy-coded data can't be seen this way.
|
||||
pub(crate) fn header(bytes: &[u8]) -> Result<(u32, u32, u16)> {
|
||||
let m = parse_markers(bytes).ok_or_else(|| Error::Malformed(F, "missing SOI".into()))?;
|
||||
if m.components == 0 {
|
||||
return Err(Error::Malformed(F, "no frame header".into()));
|
||||
}
|
||||
check_size(F, m.width as u64, m.height as u64, &DecodeOptions::default())?;
|
||||
if !matches!(m.sof, 0xC0..=0xC3) {
|
||||
return Err(Error::Unsupported(F, "arithmetic-coded or hierarchical JPEG"));
|
||||
}
|
||||
if !matches!(m.components, 1 | 3 | 4) {
|
||||
return Err(Error::Malformed(F, format!("{} colour components", m.components)));
|
||||
}
|
||||
if m.first_scan_components == 0 || !reaches_eoi(bytes) {
|
||||
return Err(Error::Malformed(F, "truncated: the image data doesn't end with an end-of-image marker".into()));
|
||||
}
|
||||
let orientation = m.exif.as_deref().map(exif::summarize).unwrap_or_default().orientation.unwrap_or(1);
|
||||
Ok((m.width, m.height, orientation))
|
||||
}
|
||||
|
||||
/// Whether the markers after SOI, followed through every segment and scan, reach an EOI marker
|
||||
/// (the first one: the primary image's, ahead of any MPF images appended to the file). Segments
|
||||
/// are skipped by their length; in scan data `FF 00` (stuffing), `FF FF` (fill) and RSTn are not
|
||||
/// markers that end it.
|
||||
fn reaches_eoi(b: &[u8]) -> bool {
|
||||
let mut p = 2usize;
|
||||
loop {
|
||||
// entropy-coded data (or garbage between segments, as `parse_markers` tolerates it)
|
||||
let Some(skip) = b.get(p..).and_then(find_ff) else { return false };
|
||||
p = p.saturating_add(skip);
|
||||
let Some(&marker) = b.get(p.saturating_add(1)) else { return false };
|
||||
p = match marker {
|
||||
0xD9 => return true,
|
||||
0xFF => p.saturating_add(1),
|
||||
0x00 | 0x01 | 0xD0..=0xD8 => p.saturating_add(2),
|
||||
_ => {
|
||||
let Some(len) = b.get(p.saturating_add(2)..p.saturating_add(4)).map(|l| u16::from_be_bytes([l[0], l[1]]) as usize) else {
|
||||
return false;
|
||||
};
|
||||
if len < 2 {
|
||||
return false;
|
||||
}
|
||||
p.saturating_add(2).saturating_add(len)
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn decode_with_fallback(bytes: &[u8], opts: &DecodeOptions, fallback: Option<crate::NamedSpace>) -> Result<Decoded> {
|
||||
let m = parse_markers(bytes).ok_or_else(|| Error::Malformed(F, "missing SOI".into()))?;
|
||||
if m.components == 0 {
|
||||
return Err(Error::Malformed(F, "no frame header".into()));
|
||||
}
|
||||
check_size(F, m.width as u64, m.height as u64, opts)?;
|
||||
let scale_to = opts.max_size.filter(|&(mw, mh)| mw > 0 && mh > 0 && (m.width >= 2 * mw || m.height >= 2 * mh));
|
||||
let scale_to = opts.max_size.filter(|&(mw, mh)| mw > 0 && mh > 0 && (m.width >= mw.saturating_mul(2) || m.height >= mh.saturating_mul(2)));
|
||||
// zune-jpeg handles the common cases fastest; jpeg-decoder covers DCT scaling, CMYK/YCCK,
|
||||
// 12-bit and non-interleaved sequential scans (which zune-jpeg 0.5 mis-decodes with subsampling).
|
||||
let non_interleaved = matches!(m.sof, 0xC0 | 0xC1) && m.first_scan_components < m.components;
|
||||
@@ -119,7 +189,10 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
} else {
|
||||
decode_zune(bytes, &m)?
|
||||
};
|
||||
let meta = Meta { icc: m.icc, exif: m.exif, xmp: m.xmp, ..Default::default() };
|
||||
// A raw container may carry the colour declaration of an otherwise untagged preview.
|
||||
// Apply it before linearization/resizing; existing JPEG metadata always wins.
|
||||
let hint = fallback.filter(|_| m.icc.is_none() && m.exif.is_none()).map(|space| crate::SourceSpace::named(space, crate::SpaceOrigin::Container));
|
||||
let meta = Meta { icc: m.icc, exif: m.exif, xmp: m.xmp, hint, ..Default::default() };
|
||||
finish(F, raw, meta, (m.width, m.height), opts)
|
||||
}
|
||||
|
||||
@@ -168,6 +241,21 @@ fn decode_jpeg_decoder(bytes: &[u8], m: &Markers, scale_to: Option<(u32, u32)>)
|
||||
Ok(Raw { width: w, height: h, model, alpha: false, premultiplied: false, buf, bit_depth: depth })
|
||||
}
|
||||
|
||||
/// The first `0xFF` byte, eight bytes at a time (scan data has one every few hundred bytes: this
|
||||
/// is most of [`header`]'s time).
|
||||
fn find_ff(b: &[u8]) -> Option<usize> {
|
||||
const ONES: u64 = 0x0101_0101_0101_0101;
|
||||
let (words, tail) = b.as_chunks::<8>();
|
||||
for (i, w) in words.iter().enumerate() {
|
||||
// a zero byte in !w is an 0xFF byte in w
|
||||
let x = !u64::from_ne_bytes(*w);
|
||||
if x.wrapping_sub(ONES) & !x & (ONES << 7) != 0 {
|
||||
return w.iter().position(|&v| v == 0xFF).map(|j| i * 8 + j);
|
||||
}
|
||||
}
|
||||
tail.iter().position(|&v| v == 0xFF).map(|j| words.len() * 8 + j)
|
||||
}
|
||||
|
||||
/// The box to request from `jpeg-decoder::scale` so the decoded image still covers `mw × mh` once
|
||||
/// fitted with the source aspect ratio.
|
||||
fn scaled_request(w: u32, h: u32, mw: u32, mh: u32) -> (u16, u16) {
|
||||
@@ -187,8 +275,11 @@ pub(crate) fn embedded_thumbnail(bytes: &[u8], max_edge: u32, min_edge: u32) ->
|
||||
if let (Some(ex), Some((o, l))) = (m.exif.as_deref(), summary.thumbnail) {
|
||||
candidates.push((&ex[o..o + l], ThumbnailSource::ExifThumbnail));
|
||||
}
|
||||
for (o, l) in mpf_images(&m) {
|
||||
if let Some(s) = bytes.get(o..o.saturating_add(l)) {
|
||||
for e in mpf_images(&m) {
|
||||
if e.kind == MPF_GAIN_MAP {
|
||||
continue;
|
||||
}
|
||||
if let Some(s) = bytes.get(e.offset..e.offset.saturating_add(e.len)) {
|
||||
candidates.push((s, ThumbnailSource::MpfPreview));
|
||||
}
|
||||
}
|
||||
@@ -196,6 +287,10 @@ pub(crate) fn embedded_thumbnail(bytes: &[u8], max_edge: u32, min_edge: u32) ->
|
||||
let mut best: Option<(u32, &[u8], ThumbnailSource)> = None;
|
||||
for (data, src) in candidates {
|
||||
let Some(pm) = parse_markers(data) else { continue };
|
||||
// a gain map (also when its MPF entry doesn't say so) is greyscale gain data, not a preview
|
||||
if pm.is_gain_map() {
|
||||
continue;
|
||||
}
|
||||
let long = pm.width.max(pm.height);
|
||||
// Previews must have the main image's aspect ratio (within 2%), else they are letterboxed/cropped.
|
||||
let aspect_ok = m.width > 0
|
||||
@@ -217,8 +312,21 @@ pub(crate) fn embedded_thumbnail(bytes: &[u8], max_edge: u32, min_edge: u32) ->
|
||||
Some(Thumbnail { image: d.to_srgb8(), orientation, source, source_width: m.width, source_height: m.height })
|
||||
}
|
||||
|
||||
/// (file offset, length) of MPF-listed images other than the primary.
|
||||
fn mpf_images(m: &Markers) -> Vec<(usize, usize)> {
|
||||
/// MP type code of a gain map image (CIPA DC-007 entry attribute, low 24 bits).
|
||||
pub(crate) const MPF_GAIN_MAP: u32 = 0x05_0000;
|
||||
|
||||
/// An MPF-listed image other than the primary.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct MpfEntry {
|
||||
/// File offset and length of the image.
|
||||
pub offset: usize,
|
||||
pub len: usize,
|
||||
/// MP type code (attribute low 24 bits).
|
||||
pub kind: u32,
|
||||
}
|
||||
|
||||
/// The MPF-listed images other than the primary.
|
||||
pub(crate) fn mpf_images(m: &Markers) -> Vec<MpfEntry> {
|
||||
let Some((base, data)) = &m.mpf else { return vec![] };
|
||||
let Some(t) = exif::Tiff::new(data) else { return vec![] };
|
||||
let Some(ifd) = t.first_ifd() else { return vec![] };
|
||||
@@ -234,24 +342,107 @@ fn mpf_images(m: &Markers) -> Vec<(usize, usize)> {
|
||||
let a = [rec[o], rec[o + 1], rec[o + 2], rec[o + 3]];
|
||||
if data.starts_with(b"II") { u32::from_le_bytes(a) } else { u32::from_be_bytes(a) }
|
||||
};
|
||||
let (size, off) = (rd(4) as usize, rd(8) as usize);
|
||||
let (attr, size, off) = (rd(0), rd(4) as usize, rd(8) as usize);
|
||||
if size > 0 && off > 0 {
|
||||
out.push((base + off, size));
|
||||
out.push(MpfEntry { offset: base.saturating_add(off), len: size, kind: attr & 0x00FF_FFFF });
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Decode to the file's 8-bit samples as stored (no colour management): 1 channel for greyscale
|
||||
/// files, else 3 (RGB). For data images such as gain maps, whose values are codes, not colours.
|
||||
pub(crate) fn decode_codes8(bytes: &[u8]) -> Result<(usize, usize, usize, Vec<u8>)> {
|
||||
let m = parse_markers(bytes).ok_or_else(|| Error::Malformed(F, "missing SOI".into()))?;
|
||||
if m.components == 0 {
|
||||
return Err(Error::Malformed(F, "no frame header".into()));
|
||||
}
|
||||
check_size(F, m.width as u64, m.height as u64, &DecodeOptions::default())?;
|
||||
if !matches!(m.components, 1 | 3) || m.precision != 8 {
|
||||
return Err(Error::Malformed(F, "expected an 8-bit greyscale or RGB image".into()));
|
||||
}
|
||||
let non_interleaved = matches!(m.sof, 0xC0 | 0xC1) && m.first_scan_components < m.components;
|
||||
let raw = if non_interleaved { decode_jpeg_decoder(bytes, &m, None)? } else { decode_zune(bytes, &m)? };
|
||||
let ch = raw.model.channels();
|
||||
match raw.buf {
|
||||
Buf::U8(v) if v.len() >= raw.width * raw.height * ch => Ok((raw.width, raw.height, ch, v)),
|
||||
_ => Err(Error::Malformed(F, "unexpected sample format".into())),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn container_fallback_matches_tagged_jpeg_before_resizing_and_keeps_own_metadata() {
|
||||
use crate::{ChromaSubsampling, EncodeImage, EncodeMeta, NamedSpace, Samples, SpaceOrigin, encode_jpeg};
|
||||
let pixels: Vec<u8> = (0..16 * 16).flat_map(|i| if i % 2 == 0 { [120, 180, 90] } else { [30, 100, 200] }).collect();
|
||||
let encode = |icc, exif| {
|
||||
encode_jpeg(
|
||||
&EncodeImage::new(16, 16, 3, Samples::U8(&pixels)),
|
||||
100,
|
||||
ChromaSubsampling::S444,
|
||||
&EncodeMeta { icc, exif, ..Default::default() },
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
let untagged = encode(None, None);
|
||||
let adobe = crate::icc::write_named(NamedSpace::AdobeRgb);
|
||||
let tagged = encode(Some(&adobe), None);
|
||||
for opts in [DecodeOptions::default(), DecodeOptions::fit(3, 3)] {
|
||||
let with = crate::decode_jpeg_with_fallback(&untagged, opts, NamedSpace::AdobeRgb).unwrap();
|
||||
let expected = crate::decode(&tagged, opts).unwrap();
|
||||
assert_eq!(with.space.origin, SpaceOrigin::Container);
|
||||
let (a, b) = (with.to_working(), expected.to_working());
|
||||
for (p, q) in a.data.iter().zip(&b.data) {
|
||||
assert!(p.iter().zip(q).all(|(a, b)| (a - b).abs() < 0.0001), "{p:?} != {q:?}");
|
||||
}
|
||||
}
|
||||
let srgb = crate::icc::write_named(NamedSpace::Srgb);
|
||||
let exif = crate::exif::minimal_exif(6);
|
||||
for bytes in [encode(Some(&srgb), None), encode(None, Some(&exif)), encode(Some(b"bad ICC"), None)] {
|
||||
let opts = DecodeOptions::fit(5, 5);
|
||||
let with = crate::decode_jpeg_with_fallback(&bytes, opts, NamedSpace::AdobeRgb).unwrap();
|
||||
let expected = crate::decode(&bytes, opts).unwrap();
|
||||
assert_eq!(with.space, expected.space);
|
||||
assert_eq!(with.image, expected.image);
|
||||
assert_eq!(with.orientation, expected.orientation);
|
||||
}
|
||||
assert!(crate::decode_jpeg_with_fallback(b"bad JPEG", DecodeOptions::default(), NamedSpace::AdobeRgb).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scaled_request_covers() {
|
||||
assert_eq!(scaled_request(6000, 4000, 256, 256), (256, 171));
|
||||
assert_eq!(scaled_request(100, 100, 256, 256), (100, 100));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_ff_matches_a_plain_search() {
|
||||
let mut b: Vec<u8> = (0..300u32).map(|i| (i * 7 % 255) as u8).collect();
|
||||
for at in [None, Some(0), Some(7), Some(8), Some(15), Some(203), Some(296), Some(299)] {
|
||||
if let Some(i) = at {
|
||||
b[i] = 0xFF;
|
||||
}
|
||||
for start in [0, 1, 5, 9, 290, 299, 300] {
|
||||
let s = &b[start..];
|
||||
assert_eq!(find_ff(s), s.iter().position(|&v| v == 0xFF), "{at:?} from {start}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unbounded_fit_box_decodes_at_full_size() {
|
||||
// a full-size load asks for the largest box (`usize::MAX` saturated to `u32::MAX`): no DCT scaling,
|
||||
// and no overflow working out whether to scale
|
||||
use crate::{ChromaSubsampling, EncodeImage, EncodeMeta, Samples, encode_jpeg};
|
||||
let pixels = vec![128u8; 32 * 24 * 3];
|
||||
let jpeg = encode_jpeg(&EncodeImage::new(32, 24, 3, Samples::U8(&pixels)), 90, ChromaSubsampling::S444, &EncodeMeta::default()).unwrap();
|
||||
let d = crate::decode(&jpeg, DecodeOptions::fit(u32::MAX, u32::MAX)).unwrap();
|
||||
assert_eq!((d.width, d.height), (32, 24));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn markers_on_garbage() {
|
||||
assert!(parse_markers(b"").is_none());
|
||||
|
||||
@@ -6,6 +6,8 @@
|
||||
//! profiles (matrix/TRC exactly, LUT/CMYK via the `moxcms` CMS), 16-bit and float precision are
|
||||
//! preserved. EXIF/XMP/ICC blobs are passed through untouched; orientation is reported, never applied.
|
||||
//! - [`to_working`] converts a decode result to linear Rec.2020 D65 (the pipeline working space).
|
||||
//! - [`read_header`] reads a file's stored dimensions and orientation from its headers, without
|
||||
//! decoding pixels (import probes).
|
||||
//! - [`decode_thumbnail`] is the fast path for grid thumbnails (EXIF thumbnail or DCT-scaled decode).
|
||||
//! - [`encode`] writes JPEG, PNG, TIFF, lossless WebP and (native, feature `avif`) AVIF, embedding
|
||||
//! ICC/EXIF/XMP. [`icc::write_matrix_trc`] builds profiles for export.
|
||||
@@ -15,8 +17,11 @@
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
mod convert;
|
||||
pub mod display;
|
||||
pub mod encode;
|
||||
pub mod exif;
|
||||
pub mod gainmap;
|
||||
mod heif;
|
||||
pub mod icc;
|
||||
mod jpeg;
|
||||
pub mod jpeg_par;
|
||||
@@ -31,7 +36,8 @@ mod tiff_codec;
|
||||
mod webp;
|
||||
|
||||
pub use encode::{
|
||||
ChromaSubsampling, EncodeImage, EncodeMeta, Samples, TiffCompression, encode_avif, encode_jpeg, encode_png, encode_tiff, encode_webp_lossless,
|
||||
ChromaSubsampling, EncodeImage, EncodeMeta, HDR_REFERENCE_WHITE_NITS, Samples, TiffCompression, encode_avif, encode_avif_pq, encode_jpeg,
|
||||
encode_png, encode_tiff, encode_webp_lossless,
|
||||
};
|
||||
pub use sniff::{Format, sniff};
|
||||
pub use space::{NamedSpace, SourceSpace, SpaceOrigin, Trc};
|
||||
@@ -156,10 +162,67 @@ pub fn decode(bytes: &[u8], opts: DecodeOptions) -> Result<Decoded> {
|
||||
std::panic::catch_unwind(|| decode_unguarded(bytes, format, &opts)).unwrap_or_else(|_| Err(Error::Malformed(format, "decoder panicked".into())))
|
||||
}
|
||||
|
||||
/// What a file's headers say about its image: enough to catalogue it without decoding pixels.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Header {
|
||||
pub format: Format,
|
||||
/// Full-resolution stored dimensions, orientation **not** applied ([`Decoded::source_width`]).
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
/// EXIF/TIFF orientation 1..=8, 1 when absent ([`Decoded::orientation`]).
|
||||
pub orientation: u16,
|
||||
}
|
||||
|
||||
/// Read the stored dimensions and orientation of any format [`decode`] supports, agreeing with
|
||||
/// what a decode reports, without decoding pixels: JPEG, PNG, TIFF, WebP, PSD and HEIF are read
|
||||
/// from their headers (markers, chunks, IFD, resources, boxes). GIF, BMP and JPEG XL fall back to
|
||||
/// a small decode.
|
||||
///
|
||||
/// The header readers also refuse what a decode would refuse before reaching the pixels (an
|
||||
/// unsupported layout, compression or frame type) and files whose image data is cut short
|
||||
/// (truncated: the data runs past the end of the file, or a JPEG's markers don't reach its
|
||||
/// end-of-image marker). Damage *inside* compressed data that is all present is only found by a
|
||||
/// decode. Malformed input yields an error, never a panic (as for [`decode`], a panic inside a
|
||||
/// third-party parser is caught on unwinding targets).
|
||||
pub fn read_header(bytes: &[u8]) -> Result<Header> {
|
||||
let format = sniff(bytes).ok_or(Error::UnknownFormat)?;
|
||||
std::panic::catch_unwind(|| read_header_unguarded(bytes, format))
|
||||
.unwrap_or_else(|_| Err(Error::Malformed(format, "header reader panicked".into())))
|
||||
}
|
||||
|
||||
/// [`read_header`] without the panic guard (for fuzzing our own code paths).
|
||||
#[doc(hidden)]
|
||||
pub fn read_header_unguarded(bytes: &[u8], format: Format) -> Result<Header> {
|
||||
let (width, height, orientation) = match format {
|
||||
Format::Jpeg => jpeg::header(bytes)?,
|
||||
Format::Png => png_codec::header(bytes)?,
|
||||
Format::Tiff => tiff_codec::header(bytes)?,
|
||||
Format::WebP => webp::header(bytes)?,
|
||||
Format::Psd => psd::header(bytes)?,
|
||||
Format::Heif => heif::header(bytes)?,
|
||||
// rare here (or, for JPEG XL, without a header reader yet): a small decode, as before
|
||||
_ => {
|
||||
let d = decode_unguarded(bytes, format, &DecodeOptions::fit(64, 64))?;
|
||||
(d.source_width, d.source_height, d.orientation)
|
||||
}
|
||||
};
|
||||
Ok(Header { format, width, height, orientation })
|
||||
}
|
||||
|
||||
/// Decode a JPEG preview with its enclosing container's colour space as a fallback.
|
||||
/// A JPEG carrying ICC or EXIF metadata keeps its own interpretation. The fallback is applied
|
||||
/// before transfer decoding and linear-light resizing, never to already-resampled pixels.
|
||||
pub fn decode_jpeg_with_fallback(bytes: &[u8], opts: DecodeOptions, fallback: NamedSpace) -> Result<Decoded> {
|
||||
std::panic::catch_unwind(|| jpeg::decode_with_fallback(bytes, &opts, Some(fallback)))
|
||||
.unwrap_or_else(|_| Err(Error::Malformed(Format::Jpeg, "decoder panicked".into())))
|
||||
}
|
||||
|
||||
/// [`decode`] without the panic guard (for fuzzing our own code paths).
|
||||
#[doc(hidden)]
|
||||
pub fn decode_unguarded(bytes: &[u8], format: Format, opts: &DecodeOptions) -> Result<Decoded> {
|
||||
let opts = *opts;
|
||||
// The reason a format can't be decoded lives in one place, `Format::not_decodable`.
|
||||
let unsupported = || Err(Error::Unsupported(format, format.not_decodable().unwrap_or("not decodable in this build")));
|
||||
match format {
|
||||
Format::Jpeg => jpeg::decode(bytes, &opts),
|
||||
Format::Png => png_codec::decode(bytes, &opts),
|
||||
@@ -170,10 +233,9 @@ pub fn decode_unguarded(bytes: &[u8], format: Format, opts: &DecodeOptions) -> R
|
||||
#[cfg(feature = "jxl")]
|
||||
Format::Jxl => jxl::decode(bytes, &opts),
|
||||
#[cfg(not(feature = "jxl"))]
|
||||
Format::Jxl => Err(Error::Unsupported(format, "built without the `jxl` feature")),
|
||||
Format::Avif => Err(Error::Unsupported(format, "no pure-Rust, permissively licensed AV1 decoder yet")),
|
||||
Format::Heif => Err(Error::Unsupported(format, "no pure-Rust, permissively licensed HEVC decoder yet")),
|
||||
Format::RawTiffLike | Format::RawOther => Err(Error::Unsupported(format, "camera raw: decode with lightcraft-raw")),
|
||||
Format::Jxl => unsupported(),
|
||||
Format::Heif => heif::decode(bytes, &opts),
|
||||
Format::Avif | Format::RawTiffLike | Format::RawOther => unsupported(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,11 +279,19 @@ pub struct ThumbnailOptions {
|
||||
/// smaller size). Defaults to `max_edge`, i.e. only previews that need no upscaling. Lower it
|
||||
/// (e.g. 160 for typical EXIF thumbnails) for an instant placeholder.
|
||||
pub min_embedded_edge: u32,
|
||||
/// Maximum source pixels for fallback decoding (default 64 MP). Non-JPEG formats
|
||||
/// decode at source resolution before resizing, so output size does not bound memory.
|
||||
#[serde(default = "thumbnail_max_pixels")]
|
||||
pub max_pixels: u64,
|
||||
}
|
||||
|
||||
fn thumbnail_max_pixels() -> u64 {
|
||||
64_000_000
|
||||
}
|
||||
|
||||
impl ThumbnailOptions {
|
||||
pub fn new(max_edge: u32) -> Self {
|
||||
ThumbnailOptions { max_edge, min_embedded_edge: max_edge }
|
||||
ThumbnailOptions { max_edge, min_embedded_edge: max_edge, max_pixels: thumbnail_max_pixels() }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -240,7 +310,7 @@ pub fn decode_thumbnail_with(bytes: &[u8], opts: &ThumbnailOptions) -> Result<Th
|
||||
{
|
||||
return Ok(t);
|
||||
}
|
||||
let d = decode(bytes, DecodeOptions::fit(max_edge, max_edge))?;
|
||||
let d = decode(bytes, DecodeOptions { max_size: Some((max_edge, max_edge)), max_pixels: opts.max_pixels })?;
|
||||
Ok(Thumbnail {
|
||||
image: d.to_srgb8(),
|
||||
orientation: d.orientation,
|
||||
|
||||
@@ -57,26 +57,80 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
finish(F, raw, meta, (w, h), opts)
|
||||
}
|
||||
|
||||
fn container_hint(info: &png::Info) -> Option<SourceSpace> {
|
||||
if let Some(c) = &info.coding_independent_code_points {
|
||||
let named = match c.color_primaries {
|
||||
1 => Some(NamedSpace::Srgb),
|
||||
9 => Some(NamedSpace::Rec2020),
|
||||
12 => Some(NamedSpace::DisplayP3),
|
||||
_ => None,
|
||||
};
|
||||
let trc = match c.transfer_function {
|
||||
13 => Some(Trc::Srgb),
|
||||
1 | 6 | 14 | 15 => Some(Trc::Rec709),
|
||||
8 => Some(Trc::Linear),
|
||||
4 => Some(Trc::Gamma(2.2)),
|
||||
_ => None,
|
||||
};
|
||||
if let (Some(n), Some(t)) = (named, trc) {
|
||||
let mut s = SourceSpace::named(n, SpaceOrigin::Container);
|
||||
s.trc = Some([t.clone(), t.clone(), t]);
|
||||
return Some(s);
|
||||
/// Stored dimensions and EXIF orientation (`eXIf`, wherever it is) from the chunk headers, without
|
||||
/// inflating the image data. Refused, as a decode would refuse it: an invalid IHDR, an indexed
|
||||
/// image without a palette, no IDAT, or a file that ends before the chunk after the last IDAT
|
||||
/// begins (truncated image data). Chunk CRCs and the deflate stream are not checked.
|
||||
pub(crate) fn header(b: &[u8]) -> Result<(u32, u32, u16)> {
|
||||
if !b.starts_with(b"\x89PNG\r\n\x1a\n") {
|
||||
return Err(err("bad signature"));
|
||||
}
|
||||
let be32 = |p: usize| b.get(p..p.checked_add(4)?).map(|v| u32::from_be_bytes([v[0], v[1], v[2], v[3]]));
|
||||
if be32(8) != Some(13) || b.get(12..16) != Some(b"IHDR") {
|
||||
return Err(err("missing IHDR"));
|
||||
}
|
||||
let ihdr = b.get(16..29).ok_or_else(|| err("truncated IHDR"))?;
|
||||
let (w, h) = (u32::from_be_bytes([ihdr[0], ihdr[1], ihdr[2], ihdr[3]]), u32::from_be_bytes([ihdr[4], ihdr[5], ihdr[6], ihdr[7]]));
|
||||
let (depth, color, compression, filter, interlace) = (ihdr[8], ihdr[9], ihdr[10], ihdr[11], ihdr[12]);
|
||||
if w > i32::MAX as u32 || h > i32::MAX as u32 {
|
||||
return Err(err("dimensions out of range"));
|
||||
}
|
||||
check_size(F, w as u64, h as u64, &DecodeOptions::default())?;
|
||||
let depth_ok = match color {
|
||||
0 => matches!(depth, 1 | 2 | 4 | 8 | 16),
|
||||
3 => matches!(depth, 1 | 2 | 4 | 8),
|
||||
2 | 4 | 6 => matches!(depth, 8 | 16),
|
||||
_ => false,
|
||||
};
|
||||
if !depth_ok || compression != 0 || filter != 0 || interlace > 1 {
|
||||
return Err(err("invalid IHDR"));
|
||||
}
|
||||
let (mut p, mut palette, mut idat, mut after_idat, mut exif) = (8usize, false, false, false, None);
|
||||
while let Some(len) = be32(p) {
|
||||
let Some(kind) = p.checked_add(4).and_then(|k| b.get(k..k.checked_add(4)?)) else { break };
|
||||
if idat && kind != b"IDAT" {
|
||||
// a chunk follows the image data: the IDAT run is complete
|
||||
after_idat = true;
|
||||
}
|
||||
if kind == b"IEND" {
|
||||
break;
|
||||
}
|
||||
let start = p.saturating_add(8);
|
||||
let data = (len <= i32::MAX as u32).then(|| start.checked_add(len as usize).and_then(|end| b.get(start..end))).flatten();
|
||||
let Some(data) = data else {
|
||||
if after_idat {
|
||||
break; // a damaged chunk after the image data: a decode ignores it too
|
||||
}
|
||||
return Err(err("truncated"));
|
||||
};
|
||||
match kind {
|
||||
b"PLTE" => palette = true,
|
||||
b"IDAT" if after_idat => {} // a stray IDAT after other chunks: the first run is the image
|
||||
b"IDAT" => {
|
||||
if color == 3 && !palette {
|
||||
return Err(err("indexed image without a palette"));
|
||||
}
|
||||
idat = true;
|
||||
}
|
||||
b"eXIf" if exif.is_none() => exif = Some(data.strip_prefix(b"Exif\0\0").unwrap_or(data)),
|
||||
_ => {}
|
||||
}
|
||||
p = start.saturating_add(data.len()).saturating_add(4);
|
||||
}
|
||||
if !idat {
|
||||
return Err(err("no image data"));
|
||||
}
|
||||
if !after_idat {
|
||||
return Err(err("truncated image data"));
|
||||
}
|
||||
Ok((w, h, exif.map(crate::exif::summarize).unwrap_or_default().orientation.unwrap_or(1)))
|
||||
}
|
||||
|
||||
fn container_hint(info: &png::Info) -> Option<SourceSpace> {
|
||||
if let Some(s) =
|
||||
info.coding_independent_code_points.as_ref().and_then(|c| SourceSpace::from_cicp(c.color_primaries.into(), c.transfer_function.into()))
|
||||
{
|
||||
return Some(s);
|
||||
}
|
||||
if info.srgb.is_some() {
|
||||
return Some(SourceSpace::named(NamedSpace::Srgb, SpaceOrigin::Container));
|
||||
|
||||
+265
-72
@@ -69,8 +69,8 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
let (icc, exif, xmp) = resources(res);
|
||||
|
||||
// Layer and mask information: only the layer count sign matters (merged transparency).
|
||||
let lm_len = if psb { r.u64()? as usize } else { r.u32()? as usize };
|
||||
let lm = r.take(lm_len)?;
|
||||
let lm_len = if psb { r.u64()? } else { u64::from(r.u32()?) };
|
||||
let lm = r.take(usize::try_from(lm_len).map_err(|_| err("truncated"))?)?;
|
||||
let merged_transparency = if lm.len() >= if psb { 10 } else { 6 } {
|
||||
let off = if psb { 8 } else { 4 };
|
||||
let count = i16::from_be_bytes([lm[off], lm[off + 1]]);
|
||||
@@ -88,33 +88,6 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
1 => return Err(Error::Unsupported(F, "1-bit bitmap PSD")),
|
||||
_ => return Err(err("bad depth")),
|
||||
};
|
||||
let plane = width * height * bpc;
|
||||
let row_bytes = width * bpc;
|
||||
let mut planes: Vec<Vec<u8>> = Vec::with_capacity(channels);
|
||||
match compression {
|
||||
0 => {
|
||||
for _ in 0..channels {
|
||||
planes.push(r.take(plane)?.to_vec());
|
||||
}
|
||||
}
|
||||
1 => {
|
||||
let counts_len = channels * height;
|
||||
let mut counts = Vec::with_capacity(counts_len);
|
||||
for _ in 0..counts_len {
|
||||
counts.push(if psb { r.u32()? as usize } else { r.u16()? as usize });
|
||||
}
|
||||
for c in 0..channels {
|
||||
let mut out = Vec::with_capacity(plane);
|
||||
for y in 0..height {
|
||||
let src = r.take(counts[c * height + y])?;
|
||||
unpackbits(src, row_bytes, &mut out);
|
||||
}
|
||||
planes.push(out);
|
||||
}
|
||||
}
|
||||
_ => return Err(Error::Unsupported(F, "ZIP-compressed merged image data")),
|
||||
}
|
||||
|
||||
let (model, n_color) = match mode {
|
||||
1 | 8 => (Model::Gray, 1),
|
||||
2 => (Model::Rgb, 1),
|
||||
@@ -128,50 +101,106 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
return Err(err("too few channels"));
|
||||
}
|
||||
let alpha = merged_transparency && channels > n_color && mode != 2;
|
||||
// Only the colour (+ alpha) planes are decoded; extra channels (spot colours, masks) are
|
||||
// stored after them and never touched.
|
||||
let used = n_color + alpha as usize;
|
||||
let n = width * height;
|
||||
let too_large = || Error::TooLarge(width as u64, height as u64);
|
||||
let n = width.checked_mul(height).ok_or_else(too_large)?;
|
||||
let row_bytes = width.checked_mul(bpc).ok_or_else(too_large)?;
|
||||
let plane = n.checked_mul(bpc).ok_or_else(too_large)?;
|
||||
let mut planes: Vec<Vec<u8>> = Vec::with_capacity(used);
|
||||
match compression {
|
||||
0 => {
|
||||
// Raw planes are borrowed from the input, so their size is bounded by the file.
|
||||
for _ in 0..used {
|
||||
planes.push(r.take(plane)?.to_vec());
|
||||
}
|
||||
}
|
||||
1 => {
|
||||
// Row byte-count table: one entry per row per channel (all channels, since the data
|
||||
// follows it). Taken from the input before anything is allocated from it.
|
||||
let entry = if psb { 4 } else { 2 };
|
||||
let table_len = channels.checked_mul(height).and_then(|c| c.checked_mul(entry)).ok_or_else(|| err("truncated"))?;
|
||||
let table = r.take(table_len)?;
|
||||
let count = |i: usize| -> usize {
|
||||
let at = i.saturating_mul(entry);
|
||||
match table.get(at..at.saturating_add(entry)) {
|
||||
Some(&[a, b]) => usize::from(u16::from_be_bytes([a, b])),
|
||||
Some(&[a, b, c, d]) => u32::from_be_bytes([a, b, c, d]) as usize,
|
||||
_ => 0,
|
||||
}
|
||||
};
|
||||
// PackBits expands at most 64× (a 2-byte repeat run yields 128 bytes), so a row needs
|
||||
// at least this many encoded bytes. Requiring it (and the data being present) before
|
||||
// allocating bounds the decoded planes by the file size: a tiny file can no longer
|
||||
// declare gigabytes of zero-padded or repeat-run rows.
|
||||
let min_row = row_bytes.div_ceil(128).saturating_mul(2);
|
||||
let mut data_len = 0usize;
|
||||
for i in 0..used.saturating_mul(height) {
|
||||
let c = count(i);
|
||||
if c < min_row {
|
||||
return Err(err("RLE row too short for its width"));
|
||||
}
|
||||
data_len = data_len.saturating_add(c);
|
||||
}
|
||||
if data_len > bytes.len().saturating_sub(r.p) {
|
||||
return Err(err("truncated"));
|
||||
}
|
||||
for c in 0..used {
|
||||
let mut out = zeroed::<u8>(plane, too_large)?;
|
||||
for (y, row) in out.chunks_exact_mut(row_bytes).enumerate() {
|
||||
let src = r.take(count(c * height + y))?;
|
||||
unpackbits(src, row)?;
|
||||
}
|
||||
planes.push(out);
|
||||
}
|
||||
}
|
||||
_ => return Err(Error::Unsupported(F, "ZIP-compressed merged image data")),
|
||||
}
|
||||
let samples = n.checked_mul(used).ok_or_else(too_large)?;
|
||||
|
||||
// Interleave to device samples.
|
||||
let buf = if mode == 2 {
|
||||
// Indexed → 8-bit RGB via the 768-byte palette.
|
||||
if color_mode_data.len() < 768 || depth != 8 {
|
||||
let (Some(pal), true) = (color_mode_data.get(..768), depth == 8) else {
|
||||
return Err(err("bad indexed palette"));
|
||||
}
|
||||
let mut v = Vec::with_capacity(n * 3);
|
||||
for &i in &planes[0] {
|
||||
};
|
||||
let idx = planes.first().ok_or_else(|| err("missing plane"))?;
|
||||
let mut v = zeroed::<u8>(n.checked_mul(3).ok_or_else(too_large)?, too_large)?;
|
||||
for (o, &i) in v.as_chunks_mut::<3>().0.iter_mut().zip(idx) {
|
||||
let i = i as usize;
|
||||
v.extend_from_slice(&[color_mode_data[i], color_mode_data[256 + i], color_mode_data[512 + i]]);
|
||||
*o = [pal[i], pal[256 + i], pal[512 + i]];
|
||||
}
|
||||
Buf::U8(v)
|
||||
} else {
|
||||
let cmyk = model == Model::Cmyk;
|
||||
match bpc {
|
||||
1 => {
|
||||
let mut v = vec![0u8; n * used];
|
||||
for (c, p) in planes.iter().take(used).enumerate() {
|
||||
for i in 0..n {
|
||||
let x = p[i];
|
||||
let mut v = zeroed::<u8>(samples, too_large)?;
|
||||
for (c, p) in planes.iter().enumerate() {
|
||||
for (o, &x) in v.iter_mut().skip(c).step_by(used).zip(p) {
|
||||
// PSD CMYK stores 255 = no ink; our CMYK convention is 0 = no ink.
|
||||
v[i * used + c] = if model == Model::Cmyk && c < 4 { 255 - x } else { x };
|
||||
*o = if cmyk && c < 4 { 255 - x } else { x };
|
||||
}
|
||||
}
|
||||
Buf::U8(v)
|
||||
}
|
||||
2 => {
|
||||
let mut v = vec![0u16; n * used];
|
||||
for (c, p) in planes.iter().take(used).enumerate() {
|
||||
for i in 0..n {
|
||||
let x = u16::from_be_bytes([p[2 * i], p[2 * i + 1]]);
|
||||
v[i * used + c] = if model == Model::Cmyk && c < 4 { 65535 - x } else { x };
|
||||
let mut v = zeroed::<u16>(samples, too_large)?;
|
||||
for (c, p) in planes.iter().enumerate() {
|
||||
for (o, s) in v.iter_mut().skip(c).step_by(used).zip(p.as_chunks::<2>().0) {
|
||||
let x = u16::from_be_bytes(*s);
|
||||
*o = if cmyk && c < 4 { 65535 - x } else { x };
|
||||
}
|
||||
}
|
||||
Buf::U16(v)
|
||||
}
|
||||
_ => {
|
||||
let mut v = vec![0f32; n * used];
|
||||
for (c, p) in planes.iter().take(used).enumerate() {
|
||||
for i in 0..n {
|
||||
let x = f32::from_be_bytes([p[4 * i], p[4 * i + 1], p[4 * i + 2], p[4 * i + 3]]);
|
||||
v[i * used + c] = if model == Model::Cmyk && c < 4 { 1.0 - x } else { x };
|
||||
let mut v = zeroed::<f32>(samples, too_large)?;
|
||||
for (c, p) in planes.iter().enumerate() {
|
||||
for (o, s) in v.iter_mut().skip(c).step_by(used).zip(p.as_chunks::<4>().0) {
|
||||
let x = f32::from_be_bytes(*s);
|
||||
*o = if cmyk && c < 4 { 1.0 - x } else { x };
|
||||
}
|
||||
}
|
||||
Buf::F32(v)
|
||||
@@ -182,25 +211,111 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
finish(F, raw, Meta { icc, exif, xmp, ..Default::default() }, (width as u32, height as u32), opts)
|
||||
}
|
||||
|
||||
/// PackBits: append exactly `row_bytes` bytes (zero-padded if the run data is short).
|
||||
fn unpackbits(mut src: &[u8], row_bytes: usize, out: &mut Vec<u8>) {
|
||||
let target = out.len() + row_bytes;
|
||||
while out.len() < target && !src.is_empty() {
|
||||
let h = src[0] as i8;
|
||||
src = &src[1..];
|
||||
if h >= 0 {
|
||||
let n = (h as usize + 1).min(src.len()).min(target - out.len());
|
||||
out.extend_from_slice(&src[..n]);
|
||||
src = &src[(h as usize + 1).min(src.len())..];
|
||||
} else if h != -128 {
|
||||
let n = ((1 - h as isize) as usize).min(target - out.len());
|
||||
if let Some(&b) = src.first() {
|
||||
out.extend(std::iter::repeat_n(b, n));
|
||||
src = &src[1..];
|
||||
/// Stored dimensions and EXIF orientation (resource 1058) from the header and image resources,
|
||||
/// without reading the merged image data. Refused, as a decode would refuse it: an unsupported
|
||||
/// depth, colour mode or compression, or merged image data that runs past the end of the file
|
||||
/// (truncated). The run-length data itself is not unpacked.
|
||||
pub(crate) fn header(bytes: &[u8]) -> Result<(u32, u32, u16)> {
|
||||
let mut r = Rd { b: bytes, p: 0 };
|
||||
if r.take(4)? != b"8BPS" {
|
||||
return Err(err("bad signature"));
|
||||
}
|
||||
let psb = match r.u16()? {
|
||||
1 => false,
|
||||
2 => true,
|
||||
_ => return Err(err("bad version")),
|
||||
};
|
||||
r.take(6)?;
|
||||
let channels = r.u16()? as usize;
|
||||
let height = r.u32()? as usize;
|
||||
let width = r.u32()? as usize;
|
||||
let depth = r.u16()?;
|
||||
let mode = r.u16()?;
|
||||
check_size(F, width as u64, height as u64, &DecodeOptions::default())?;
|
||||
if channels == 0 || channels > 56 {
|
||||
return Err(err("bad channel count"));
|
||||
}
|
||||
let cm_len = r.u32()? as usize;
|
||||
let color_mode_data = r.take(cm_len)?;
|
||||
let res_len = r.u32()? as usize;
|
||||
let (_, exif, _) = resources(r.take(res_len)?);
|
||||
let lm_len = if psb { r.u64()? as usize } else { r.u32()? as usize };
|
||||
r.take(lm_len)?;
|
||||
let compression = r.u16()?;
|
||||
let bpc = match depth {
|
||||
8 => 1usize,
|
||||
16 => 2,
|
||||
32 => 4,
|
||||
1 => return Err(Error::Unsupported(F, "1-bit bitmap PSD")),
|
||||
_ => return Err(err("bad depth")),
|
||||
};
|
||||
let n_color = match mode {
|
||||
1 | 8 | 7 | 2 => 1,
|
||||
3 => 3,
|
||||
4 => 4,
|
||||
9 => return Err(Error::Unsupported(F, "Lab PSD")),
|
||||
_ => return Err(err("unknown colour mode")),
|
||||
};
|
||||
if channels < n_color {
|
||||
return Err(err("too few channels"));
|
||||
}
|
||||
if mode == 2 && (color_mode_data.len() < 768 || depth != 8) {
|
||||
return Err(err("bad indexed palette"));
|
||||
}
|
||||
let rest = bytes.len().saturating_sub(r.p);
|
||||
let need = match compression {
|
||||
0 => width.checked_mul(height).and_then(|n| n.checked_mul(bpc)).and_then(|n| n.checked_mul(channels)),
|
||||
1 => {
|
||||
// the row byte counts, then the rows they count
|
||||
let rows = channels.checked_mul(height).ok_or_else(|| err("truncated"))?;
|
||||
let table = rows.checked_mul(if psb { 4 } else { 2 }).filter(|&t| t <= rest).ok_or_else(|| err("truncated"))?;
|
||||
let mut total = table;
|
||||
for _ in 0..rows {
|
||||
total = total.saturating_add(if psb { r.u32()? as usize } else { r.u16()? as usize });
|
||||
}
|
||||
Some(total)
|
||||
}
|
||||
_ => return Err(Error::Unsupported(F, "ZIP-compressed merged image data")),
|
||||
};
|
||||
if need.is_none_or(|n| n > rest) {
|
||||
return Err(err("truncated"));
|
||||
}
|
||||
let orientation = exif.as_deref().map(crate::exif::summarize).unwrap_or_default().orientation.unwrap_or(1);
|
||||
Ok((width as u32, height as u32, orientation))
|
||||
}
|
||||
|
||||
/// A zero-filled buffer whose allocation failure is an error, not an abort.
|
||||
fn zeroed<T: Clone + Default>(len: usize, too_large: impl Fn() -> Error) -> Result<Vec<T>> {
|
||||
let mut v = Vec::new();
|
||||
v.try_reserve_exact(len).map_err(|_| too_large())?;
|
||||
v.resize(len, T::default());
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// PackBits: decode one row into `row` (pre-zeroed, so short run data leaves zero padding).
|
||||
/// A run that would write past the end of the row is an error; trailing bytes are ignored.
|
||||
fn unpackbits(mut src: &[u8], row: &mut [u8]) -> Result<()> {
|
||||
let mut at = 0;
|
||||
while at < row.len() {
|
||||
let Some((&h, rest)) = src.split_first() else { break };
|
||||
src = rest;
|
||||
let h = h as i8;
|
||||
if h >= 0 {
|
||||
let n = h as usize + 1;
|
||||
let lit = src.get(..n).unwrap_or(src);
|
||||
let dst = row.get_mut(at..at + lit.len()).ok_or_else(|| err("RLE row overruns its width"))?;
|
||||
dst.copy_from_slice(lit);
|
||||
at += lit.len();
|
||||
src = src.get(n..).unwrap_or(&[]);
|
||||
} else if h != -128 {
|
||||
let n = (1 - h as isize) as usize;
|
||||
let Some((&b, rest)) = src.split_first() else { break };
|
||||
src = rest;
|
||||
row.get_mut(at..at + n).ok_or_else(|| err("RLE row overruns its width"))?.fill(b);
|
||||
at += n;
|
||||
}
|
||||
}
|
||||
out.resize(target, 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resources(mut b: &[u8]) -> (Option<Vec<u8>>, Option<Vec<u8>>, Option<String>) {
|
||||
@@ -234,11 +349,89 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn packbits() {
|
||||
let mut out = Vec::new();
|
||||
unpackbits(&[0xFE, 0xAA, 0x02, 0x80, 0x00, 0x2A], 6, &mut out);
|
||||
assert_eq!(out, vec![0xAA, 0xAA, 0xAA, 0x80, 0x00, 0x2A]);
|
||||
let mut out = Vec::new();
|
||||
unpackbits(&[0x05, 1], 4, &mut out);
|
||||
assert_eq!(out, vec![1, 0, 0, 0]);
|
||||
let mut out = [0u8; 6];
|
||||
unpackbits(&[0xFE, 0xAA, 0x02, 0x80, 0x00, 0x2A], &mut out).unwrap();
|
||||
assert_eq!(out, [0xAA, 0xAA, 0xAA, 0x80, 0x00, 0x2A]);
|
||||
// Short run data is zero-padded.
|
||||
let mut out = [0u8; 4];
|
||||
unpackbits(&[0x01, 1], &mut out).unwrap();
|
||||
assert_eq!(out, [1, 0, 0, 0]);
|
||||
// Truncated literal run: copies what is there.
|
||||
let mut out = [0u8; 4];
|
||||
unpackbits(&[0x03, 7, 8], &mut out).unwrap();
|
||||
assert_eq!(out, [7, 8, 0, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn packbits_overrun_is_an_error() {
|
||||
// Repeat run of 128 into a 4-byte row.
|
||||
let mut out = [0u8; 4];
|
||||
assert!(unpackbits(&[0x81, 0xAA], &mut out).is_err());
|
||||
// Literal run of 6 into a 4-byte row.
|
||||
assert!(unpackbits(&[0x05, 1, 2, 3, 4, 5, 6], &mut out).is_err());
|
||||
}
|
||||
|
||||
/// PSD header + empty colour-mode/resources/layer sections + compression id.
|
||||
fn header(channels: u16, w: u32, h: u32, depth: u16, mode: u16, compression: u16) -> Vec<u8> {
|
||||
let mut b = b"8BPS".to_vec();
|
||||
b.extend_from_slice(&1u16.to_be_bytes());
|
||||
b.extend_from_slice(&[0; 6]);
|
||||
b.extend_from_slice(&channels.to_be_bytes());
|
||||
b.extend_from_slice(&h.to_be_bytes());
|
||||
b.extend_from_slice(&w.to_be_bytes());
|
||||
b.extend_from_slice(&depth.to_be_bytes());
|
||||
b.extend_from_slice(&mode.to_be_bytes());
|
||||
b.extend_from_slice(&[0; 12]); // colour mode data, resources, layer & mask: all empty
|
||||
b.extend_from_slice(&compression.to_be_bytes());
|
||||
b
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn huge_rle_with_truncated_count_table_is_rejected_before_allocating() {
|
||||
// 1 × 2^30 px, 56 channels: the row table alone would be 56 Gi entries; the file has 4 bytes.
|
||||
let mut b = header(56, 1, 1 << 30, 32, 3, 1);
|
||||
b.extend_from_slice(&[0, 0, 0, 0]);
|
||||
let e = decode(&b, &DecodeOptions::default()).unwrap_err();
|
||||
assert!(matches!(e, Error::Malformed(_, ref m) if m == "truncated"), "{e:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn huge_rle_of_empty_rows_is_rejected_before_allocating() {
|
||||
// 2^30 × 1 px RGB at 32 bits: every row count is 0, which used to zero-pad 3 × 4 GiB.
|
||||
let mut b = header(3, 1 << 30, 1, 32, 3, 1);
|
||||
b.extend_from_slice(&[0; 6]);
|
||||
let e = decode(&b, &DecodeOptions::default()).unwrap_err();
|
||||
assert!(matches!(e, Error::Malformed(_, ref m) if m.contains("too short")), "{e:?}");
|
||||
// 1 Gpx RGB 8-bit whose (plausible) row counts claim more data than the file holds.
|
||||
let mut b = header(3, 1 << 20, 1 << 10, 8, 3, 1);
|
||||
b.extend_from_slice(&[0xFF; 3 * 1024 * 2]);
|
||||
let e = decode(&b, &DecodeOptions::default()).unwrap_err();
|
||||
assert!(matches!(e, Error::Malformed(_, ref m) if m == "truncated"), "{e:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rle_repeat_run_past_row_width_is_an_error() {
|
||||
// 4 × 1 gray 8-bit; the single row is a 128-byte repeat run.
|
||||
let mut b = header(1, 4, 1, 8, 1, 1);
|
||||
b.extend_from_slice(&2u16.to_be_bytes());
|
||||
b.extend_from_slice(&[0x81, 0x55]);
|
||||
let e = decode(&b, &DecodeOptions::default()).unwrap_err();
|
||||
assert!(matches!(e, Error::Malformed(_, ref m) if m.contains("overruns")), "{e:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rle_gray16_and_extra_channels_decode() {
|
||||
// 2 × 2 gray 16-bit with an extra (unused) channel whose data is absent: only the used
|
||||
// plane is decoded. Row: repeat run of 4 bytes 0x80 -> samples 0x8080.
|
||||
let mut b = header(2, 2, 2, 16, 1, 1);
|
||||
for _ in 0..4 {
|
||||
b.extend_from_slice(&2u16.to_be_bytes());
|
||||
}
|
||||
b.extend_from_slice(&[0xFD, 0x80, 0xFD, 0x80]);
|
||||
let d = decode(&b, &DecodeOptions::default()).unwrap();
|
||||
assert_eq!((d.width, d.height), (2, 2));
|
||||
let p = d.image.get(1, 1);
|
||||
let q = d.image.get(0, 0);
|
||||
assert!(p[0] > 0.0 && (p[0] - q[0]).abs() < 1e-6, "{p:?}");
|
||||
}
|
||||
}
|
||||
@@ -64,10 +64,20 @@ impl Format {
|
||||
|
||||
/// Whether [`crate::decode`] can decode this format in this build.
|
||||
pub fn can_decode(self) -> bool {
|
||||
self.not_decodable().is_none()
|
||||
}
|
||||
|
||||
/// Why [`crate::decode`] can't decode this format in this build (`None`: it can). The
|
||||
/// [`crate::Error::Unsupported`] a decode returns carries the same reason.
|
||||
pub fn not_decodable(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Format::Jxl => cfg!(feature = "jxl"),
|
||||
Format::Avif | Format::Heif | Format::RawTiffLike | Format::RawOther => false,
|
||||
_ => true,
|
||||
#[cfg(not(feature = "jxl"))]
|
||||
Format::Jxl => Some("built without the `jxl` feature"),
|
||||
#[cfg(not(feature = "heif"))]
|
||||
Format::Heif => Some(crate::heif::NOT_IN_BUILD),
|
||||
Format::Avif => Some("no pure-Rust, permissively licensed AV1 decoder yet"),
|
||||
Format::RawTiffLike | Format::RawOther => Some("camera raw: decode with lightcraft-raw"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,6 +173,9 @@ fn tiff_is_raw(b: &[u8]) -> bool {
|
||||
}
|
||||
let Some(t) = Tiff::new(b) else { return false };
|
||||
let Some(ifd0) = t.first_ifd() else { return false };
|
||||
let sony = t.ifd(ifd0).is_some_and(|(entries, _)| {
|
||||
entries.iter().any(|e| e.tag == 0x010f && t.bytes(e).and_then(|v| v.get(..4)).is_some_and(|v| v.eq_ignore_ascii_case(b"SONY")))
|
||||
});
|
||||
let mut stack = vec![ifd0];
|
||||
let mut seen = 0;
|
||||
while let Some(pos) = stack.pop() {
|
||||
@@ -171,6 +184,12 @@ fn tiff_is_raw(b: &[u8]) -> bool {
|
||||
break;
|
||||
}
|
||||
let Some((entries, _)) = t.ifd(pos) else { continue };
|
||||
// Sony M/S lossless ARWs retain a dummy CFA pattern, but store linear YCbCr.
|
||||
// Requiring the raw pattern plus this layout keeps ordinary Sony TIFFs as TIFFs.
|
||||
let uint = |tag| entries.iter().find(|e| e.tag == tag).and_then(|e| t.uint(e));
|
||||
if sony && entries.iter().any(|e| e.tag == 0x828e) && uint(0x0103) == Some(7) && uint(0x0106) == Some(6) && uint(0x0115) == Some(3) {
|
||||
return true;
|
||||
}
|
||||
for e in &entries {
|
||||
match e.tag {
|
||||
0xC612 => return true, // DNGVersion
|
||||
@@ -233,4 +252,38 @@ mod tests {
|
||||
b.extend_from_slice(&0u32.to_le_bytes());
|
||||
assert_eq!(sniff(&b), Some(Format::RawTiffLike));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sony_linear_ycbcr_subifd_is_raw_but_ordinary_tiff_is_not() {
|
||||
fn fixture(compression: u32, pattern: bool) -> Vec<u8> {
|
||||
let mut b = b"II*\0\x08\0\0\0".to_vec();
|
||||
let entry = |b: &mut Vec<u8>, tag: u16, typ: u16, count: u32, value: u32| {
|
||||
b.extend_from_slice(&tag.to_le_bytes());
|
||||
b.extend_from_slice(&typ.to_le_bytes());
|
||||
b.extend_from_slice(&count.to_le_bytes());
|
||||
b.extend_from_slice(&value.to_le_bytes());
|
||||
};
|
||||
b.extend_from_slice(&2u16.to_le_bytes());
|
||||
entry(&mut b, 0x010f, 2, 5, 38);
|
||||
entry(&mut b, 0x014a, 4, 1, 44);
|
||||
b.extend_from_slice(&0u32.to_le_bytes());
|
||||
b.extend_from_slice(b"SONY\0\0");
|
||||
b.extend_from_slice(&(if pattern { 4u16 } else { 3 }).to_le_bytes());
|
||||
entry(&mut b, 0x0103, 3, 1, compression);
|
||||
entry(&mut b, 0x0106, 3, 1, 6);
|
||||
entry(&mut b, 0x0115, 3, 1, 3);
|
||||
if pattern {
|
||||
entry(&mut b, 0x828e, 1, 4, u32::MAX);
|
||||
}
|
||||
b.extend_from_slice(&0u32.to_le_bytes());
|
||||
b
|
||||
}
|
||||
let mut raw = fixture(7, true);
|
||||
assert_eq!(sniff(&raw), Some(Format::RawTiffLike));
|
||||
assert_eq!(sniff(&fixture(7, false)), Some(Format::Tiff));
|
||||
assert_eq!(sniff(&fixture(1, true)), Some(Format::Tiff));
|
||||
assert_eq!(sniff(&raw[..38]), Some(Format::Tiff));
|
||||
raw[38..42].copy_from_slice(b"TEST");
|
||||
assert_eq!(sniff(&raw), Some(Format::Tiff));
|
||||
}
|
||||
}
|
||||
@@ -255,6 +255,27 @@ impl SourceSpace {
|
||||
SourceSpace { named: Some(n), to_xyz_d50: n.to_xyz_d50(), trc: Some([t.clone(), t.clone(), t]), origin }
|
||||
}
|
||||
|
||||
/// The space named by ITU-T H.273 code points (PNG `cICP`, HEIF `nclx`, HEVC VUI), when
|
||||
/// both the primaries and the transfer are ones this crate models.
|
||||
pub(crate) fn from_cicp(primaries: u16, transfer: u16) -> Option<SourceSpace> {
|
||||
let named = match primaries {
|
||||
1 => NamedSpace::Srgb,
|
||||
9 => NamedSpace::Rec2020,
|
||||
12 => NamedSpace::DisplayP3,
|
||||
_ => return None,
|
||||
};
|
||||
let trc = match transfer {
|
||||
13 => Trc::Srgb,
|
||||
1 | 6 | 14 | 15 => Trc::Rec709,
|
||||
8 => Trc::Linear,
|
||||
4 => Trc::Gamma(2.2),
|
||||
_ => return None,
|
||||
};
|
||||
let mut s = SourceSpace::named(named, SpaceOrigin::Container);
|
||||
s.trc = Some([trc.clone(), trc.clone(), trc]);
|
||||
Some(s)
|
||||
}
|
||||
|
||||
/// Linear source RGB → linear `dst` RGB (Bradford from D50 to `dst`'s white).
|
||||
pub fn to_space(&self, dst: &RgbSpace) -> Mat3 {
|
||||
if let Some(n) = self.named {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
//! TIFF decode via the `tiff` crate: 8/16/32-bit integer, 16/32/64-bit float, gray/RGB/CMYK/palette,
|
||||
//! alpha (associated or not), ICC (34675), XMP (700), orientation (274). First image only.
|
||||
//! TIFF decode via the `tiff` crate: 1/2/4/8/16/32/64-bit integer, 16/32/64-bit float, gray/RGB/CMYK,
|
||||
//! alpha (associated or not), ICC (34675), XMP (700), orientation (274). First image only. Palette
|
||||
//! (indexed-colour) images are refused by the `tiff` crate itself, on every read path.
|
||||
|
||||
use crate::convert::{Buf, Meta, Model, Raw, check_size, finish};
|
||||
use crate::{DecodeOptions, Decoded, Error, Format, Result};
|
||||
@@ -14,10 +15,7 @@ fn err(e: impl std::fmt::Display) -> Error {
|
||||
}
|
||||
|
||||
pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
let mut limits = Limits::default();
|
||||
limits.decoding_buffer_size = opts.max_pixels.saturating_mul(4 * 8).min(usize::MAX as u64) as usize;
|
||||
limits.ifd_value_size = 32 << 20;
|
||||
let mut d = Decoder::new(std::io::Cursor::new(bytes)).map_err(err)?.with_limits(limits);
|
||||
let mut d = Decoder::new(std::io::Cursor::new(bytes)).map_err(err)?.with_limits(limits(opts));
|
||||
let (w, h) = d.dimensions().map_err(err)?;
|
||||
check_size(F, w as u64, h as u64, opts)?;
|
||||
let ct = d.colortype().map_err(err)?;
|
||||
@@ -25,26 +23,12 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
let icc = tag_bytes(&mut d, Tag::IccProfile);
|
||||
let xmp = tag_bytes(&mut d, Tag::from_u16_exhaustive(700)).map(|v| String::from_utf8_lossy(&v).trim_end_matches('\0').to_string());
|
||||
let orientation = d.get_tag_u32(Tag::Orientation).ok().map(|v| v as u16);
|
||||
let photometric = d.get_tag_u32(Tag::PhotometricInterpretation).ok();
|
||||
let planar = d.get_tag_u32(Tag::PlanarConfiguration).ok() == Some(2);
|
||||
let extra = d.get_tag_u16_vec(Tag::ExtraSamples).ok().unwrap_or_default();
|
||||
let colormap = d.get_tag_u16_vec(Tag::ColorMap).ok();
|
||||
let compression = d.get_tag_u32(Tag::Compression).ok();
|
||||
|
||||
let (model, n_color, n_samples, bits) = match ct {
|
||||
ColorType::Gray(b) => (Model::Gray, 1, 1, b),
|
||||
ColorType::GrayA(b) => (Model::Gray, 1, 2, b),
|
||||
ColorType::RGB(b) => (Model::Rgb, 3, 3, b),
|
||||
ColorType::RGBA(b) => (Model::Rgb, 3, 4, b),
|
||||
ColorType::CMYK(b) => (Model::Cmyk, 4, 4, b),
|
||||
ColorType::CMYKA(b) => (Model::Cmyk, 4, 5, b),
|
||||
ColorType::Palette(b) => (Model::Rgb, 1, 1, b),
|
||||
// JPEG-compressed YCbCr is converted to RGB by the JPEG decoder.
|
||||
ColorType::YCbCr(b) if compression == Some(7) => (Model::Rgb, 3, 3, b),
|
||||
ColorType::Multiband { bit_depth, num_samples } if num_samples >= 3 => (Model::Rgb, 3, num_samples as usize, bit_depth),
|
||||
ColorType::Multiband { bit_depth, num_samples } if num_samples >= 1 => (Model::Gray, 1, num_samples as usize, bit_depth),
|
||||
other => return Err(Error::Unsupported(F, unsupported_name(other))),
|
||||
};
|
||||
let (model, n_color, n_samples, bits) = layout(ct, compression, planar)?;
|
||||
let has_alpha = n_samples > n_color && !matches!(ct, ColorType::Palette(_));
|
||||
let premultiplied = has_alpha && extra.first() == Some(&1);
|
||||
|
||||
@@ -52,6 +36,7 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
let (wu, hu) = (w as usize, h as usize);
|
||||
let n = wu * hu;
|
||||
let mut buf = match res {
|
||||
DecodingResult::U8(v) if bits < 8 => Buf::U8(unpack(&v, wu, hu, n_samples, bits)?),
|
||||
DecodingResult::U8(v) => Buf::U8(v),
|
||||
DecodingResult::U16(v) => Buf::U16(v),
|
||||
// Rare integer widths: keep 16 significant bits (plenty for display-referred data).
|
||||
@@ -94,14 +79,8 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
return finish(F, raw, meta, (w, h), opts);
|
||||
}
|
||||
|
||||
// WhiteIsZero → invert.
|
||||
if photometric == Some(0) {
|
||||
match &mut buf {
|
||||
Buf::U8(v) => v.iter_mut().for_each(|x| *x = 255 - *x),
|
||||
Buf::U16(v) => v.iter_mut().for_each(|x| *x = 65535 - *x),
|
||||
Buf::F32(v) => v.iter_mut().for_each(|x| *x = 1.0 - *x),
|
||||
}
|
||||
}
|
||||
// WhiteIsZero needs nothing here: the `tiff` crate inverts those samples as it reads them (and
|
||||
// refuses WhiteIsZero layouts it can't invert), so inverting again would show a negative.
|
||||
|
||||
// Drop extra samples beyond colour + one alpha.
|
||||
let keep = n_color + has_alpha as usize;
|
||||
@@ -113,6 +92,67 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
finish(F, raw, meta, (w, h), opts)
|
||||
}
|
||||
|
||||
fn limits(opts: &DecodeOptions) -> Limits {
|
||||
let mut limits = Limits::default();
|
||||
limits.decoding_buffer_size = opts.max_pixels.saturating_mul(4 * 8).min(usize::MAX as u64) as usize;
|
||||
limits.ifd_value_size = 32 << 20;
|
||||
limits
|
||||
}
|
||||
|
||||
/// The sample layout of a colour type we can decode: (model, colour samples, samples per pixel,
|
||||
/// bits per sample).
|
||||
fn layout(ct: ColorType, compression: Option<u32>, planar: bool) -> Result<(Model, usize, usize, u8)> {
|
||||
let l = match ct {
|
||||
ColorType::Gray(b) => (Model::Gray, 1, 1, b),
|
||||
ColorType::GrayA(b) => (Model::Gray, 1, 2, b),
|
||||
ColorType::RGB(b) => (Model::Rgb, 3, 3, b),
|
||||
ColorType::RGBA(b) => (Model::Rgb, 3, 4, b),
|
||||
ColorType::CMYK(b) => (Model::Cmyk, 4, 4, b),
|
||||
ColorType::CMYKA(b) => (Model::Cmyk, 4, 5, b),
|
||||
ColorType::Palette(b) => (Model::Rgb, 1, 1, b),
|
||||
// JPEG-compressed YCbCr is converted to RGB by the JPEG decoder.
|
||||
ColorType::YCbCr(b) if compression == Some(7) => (Model::Rgb, 3, 3, b),
|
||||
ColorType::Multiband { bit_depth, num_samples } if num_samples >= 3 => (Model::Rgb, 3, num_samples as usize, bit_depth),
|
||||
ColorType::Multiband { bit_depth, num_samples } if num_samples >= 1 => (Model::Gray, 1, num_samples as usize, bit_depth),
|
||||
other => return Err(Error::Unsupported(F, unsupported_name(other))),
|
||||
};
|
||||
if l.3 < 8 && planar && l.2 > 1 {
|
||||
return Err(Error::Unsupported(F, "planar TIFF with samples narrower than a byte"));
|
||||
}
|
||||
Ok(l)
|
||||
}
|
||||
|
||||
/// Stored dimensions and orientation from the first IFD, without reading the image data. Refused,
|
||||
/// as a decode would refuse it: a layout or compression we can't decode, or strips/tiles that
|
||||
/// start past the end of the file (truncated). The compressed data itself is not checked.
|
||||
pub(crate) fn header(bytes: &[u8]) -> Result<(u32, u32, u16)> {
|
||||
let mut d = Decoder::new(std::io::Cursor::new(bytes)).map_err(err)?.with_limits(limits(&DecodeOptions::default()));
|
||||
let (w, h) = d.dimensions().map_err(err)?;
|
||||
check_size(F, w as u64, h as u64, &DecodeOptions::default())?;
|
||||
let ct = d.colortype().map_err(err)?;
|
||||
let compression = d.get_tag_u32(Tag::Compression).ok();
|
||||
let planar = d.get_tag_u32(Tag::PlanarConfiguration).ok() == Some(2);
|
||||
layout(ct, compression, planar)?;
|
||||
// what this build of the `tiff` crate decompresses: none, CCITT G4, LZW, JPEG, Deflate, PackBits
|
||||
if !matches!(compression.unwrap_or(1), 1 | 4 | 5 | 7 | 8 | 32773 | 32946) {
|
||||
return Err(Error::Unsupported(F, "compression method"));
|
||||
}
|
||||
let chunks = |offsets: Tag, counts: Tag, d: &mut Decoder<std::io::Cursor<&[u8]>>| {
|
||||
Some((d.get_tag_u64_vec(offsets).ok()?, d.get_tag_u64_vec(counts).ok()?))
|
||||
};
|
||||
let (offsets, counts) = chunks(Tag::StripOffsets, Tag::StripByteCounts, &mut d)
|
||||
.or_else(|| chunks(Tag::TileOffsets, Tag::TileByteCounts, &mut d))
|
||||
.ok_or_else(|| err("no image data"))?;
|
||||
// Only where each chunk starts: some writers overstate StripByteCounts, and the `tiff` crate
|
||||
// decodes such files (it stops once it has the pixels), so a chunk's end proves nothing.
|
||||
let len = bytes.len() as u64;
|
||||
if offsets.is_empty() || offsets.iter().zip(&counts).any(|(&o, &c)| c > 0 && o >= len) {
|
||||
return Err(err("truncated: image data past the end of the file"));
|
||||
}
|
||||
let orientation = d.get_tag_u32(Tag::Orientation).ok().map(|v| v as u16).filter(|o| (1..=8).contains(o)).unwrap_or(1);
|
||||
Ok((w, h, orientation))
|
||||
}
|
||||
|
||||
/// A BYTE/UNDEFINED tag as bytes (the decoder may surface unknown byte tags as unsigned values).
|
||||
fn tag_bytes<R: std::io::Read + std::io::Seek>(d: &mut Decoder<R>, tag: Tag) -> Option<Vec<u8>> {
|
||||
use tiff::decoder::ifd::Value;
|
||||
@@ -140,6 +180,29 @@ fn unsupported_name(ct: ColorType) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
/// Samples narrower than a byte (1-bit bilevel and fax scans, 2- and 4-bit gray) arrive packed, most
|
||||
/// significant bits first, each row padded to a whole byte. Spread them out one per byte, scaled to
|
||||
/// 0–255 like any 8-bit sample.
|
||||
fn unpack(packed: &[u8], width: usize, height: usize, samples: usize, bits: u8) -> Result<Vec<u8>> {
|
||||
if !matches!(bits, 1 | 2 | 4) {
|
||||
return Err(Error::Unsupported(F, "TIFF samples of 3, 5, 6 or 7 bits"));
|
||||
}
|
||||
let bits = bits as usize;
|
||||
let per_row = width.checked_mul(samples).ok_or_else(|| err("row too wide"))?;
|
||||
let row_bytes = per_row.checked_mul(bits).ok_or_else(|| err("row too wide"))?.div_ceil(8);
|
||||
if row_bytes == 0 || packed.len() / row_bytes < height {
|
||||
return Err(err("sample buffer too short"));
|
||||
}
|
||||
let max = (1u8 << bits) - 1;
|
||||
let scale = 255 / max; // 255, 85, 17: exact
|
||||
let mut out = Vec::with_capacity(per_row.saturating_mul(height));
|
||||
for row in packed.chunks_exact(row_bytes).take(height) {
|
||||
let values = row.iter().flat_map(|&b| (1..=8 / bits).map(move |i| (b >> (8 - bits * i)) & max));
|
||||
out.extend(values.take(per_row).map(|v| v * scale));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn interleave(buf: Buf, n: usize, s: usize) -> Buf {
|
||||
fn go<T: Copy + Default>(v: Vec<T>, n: usize, s: usize) -> Vec<T> {
|
||||
if v.len() < n * s {
|
||||
|
||||
@@ -12,6 +12,9 @@ fn err(e: impl std::fmt::Display) -> Error {
|
||||
|
||||
pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
let mut d = image_webp::WebPDecoder::new(std::io::Cursor::new(bytes)).map_err(err)?;
|
||||
// RIFF chunk lengths are untrusted. Metadata cannot exceed the input;
|
||||
// otherwise a tiny truncated file can request gigabytes before read_exact fails.
|
||||
d.set_memory_limit(bytes.len());
|
||||
let (w, h) = d.dimensions();
|
||||
check_size(F, w as u64, h as u64, opts)?;
|
||||
let alpha = d.has_alpha();
|
||||
@@ -24,3 +27,30 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
let raw = Raw { width: w as usize, height: h as usize, model: Model::Rgb, alpha, premultiplied: false, buf: Buf::U8(buf), bit_depth: 8 };
|
||||
finish(F, raw, Meta { icc, exif, xmp, ..Default::default() }, (w, h), opts)
|
||||
}
|
||||
|
||||
/// Stored dimensions and EXIF orientation from the RIFF chunks and the bitstream header, without
|
||||
/// decoding the image. Refused, as a decode would refuse it: no image chunk (`VP8 `, `VP8L` or an
|
||||
/// animation frame), or one that runs past the end of the file (truncated). The compressed data
|
||||
/// itself is not checked.
|
||||
pub(crate) fn header(bytes: &[u8]) -> Result<(u32, u32, u16)> {
|
||||
let mut p = 12usize;
|
||||
loop {
|
||||
let Some(h) = bytes.get(p..p.saturating_add(8)) else { return Err(err("no image data")) };
|
||||
let size = u32::from_le_bytes([h[4], h[5], h[6], h[7]]) as usize;
|
||||
let end = p.saturating_add(8).saturating_add(size);
|
||||
if matches!(&h[0..4], b"VP8 " | b"VP8L" | b"ANMF") {
|
||||
if end > bytes.len() {
|
||||
return Err(err("truncated image data"));
|
||||
}
|
||||
break;
|
||||
}
|
||||
p = end.saturating_add(size & 1);
|
||||
}
|
||||
let mut d = image_webp::WebPDecoder::new(std::io::Cursor::new(bytes)).map_err(err)?;
|
||||
d.set_memory_limit(bytes.len());
|
||||
let (w, h) = d.dimensions();
|
||||
check_size(F, w as u64, h as u64, &DecodeOptions::default())?;
|
||||
let exif = d.exif_metadata().ok().flatten();
|
||||
let exif = exif.as_deref().map(|e| e.strip_prefix(b"Exif\0\0").unwrap_or(e));
|
||||
Ok((w, h, exif.map(crate::exif::summarize).unwrap_or_default().orientation.unwrap_or(1)))
|
||||
}
|
||||
@@ -30,10 +30,59 @@ fn try_all(bytes: &[u8]) {
|
||||
if let Some(f) = sniff(bytes) {
|
||||
let _ = decode_unguarded(bytes, f, &small_opts());
|
||||
let _ = decode_unguarded(bytes, f, &DecodeOptions { max_size: Some((8, 8)), max_pixels: 1 << 22 });
|
||||
let _ = read_header_unguarded(bytes, f);
|
||||
}
|
||||
let _ = decode(bytes, small_opts());
|
||||
let _ = decode_thumbnail(bytes, 16);
|
||||
let _ = read_header(bytes);
|
||||
let _ = decode_thumbnail_with(bytes, &ThumbnailOptions { max_pixels: small_opts().max_pixels, ..ThumbnailOptions::new(16) });
|
||||
let _ = lightcraft_codecs::icc::parse(bytes);
|
||||
let _ = lightcraft_codecs::gainmap::read_jpeg(bytes);
|
||||
let _ = lightcraft_codecs::gainmap::is_gain_map_jpeg(bytes);
|
||||
}
|
||||
|
||||
fn gain_map_jpeg(meta: &EncodeMeta) -> Vec<u8> {
|
||||
use lightcraft_codecs::gainmap::{GainMapOptions, compute, encode_jpeg};
|
||||
let hdr: Vec<[f32; 3]> = (0..24 * 16).map(|i| [i as f32 * 0.02; 3]).collect();
|
||||
let sdr: Vec<[f32; 3]> = hdr.iter().map(|p| p.map(|v| v / (1.0 + v))).collect();
|
||||
let (g, m) = compute(&sdr, &hdr, 24, 16, [0.2126, 0.7152, 0.0722], &GainMapOptions::default()).unwrap();
|
||||
let base: Vec<u8> = sdr.iter().flat_map(|p| p.map(|v| (v * 255.0) as u8)).collect();
|
||||
encode_jpeg(&EncodeImage::new(24, 16, 3, Samples::U8(&base)), &g, &m, 80, ChromaSubsampling::S420, meta).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn webp_truncated_metadata_cannot_allocate_declared_chunk_size() {
|
||||
let mut bytes = seeds()[5].clone();
|
||||
let tag = bytes.windows(4).rposition(|w| w == b"XMP ").unwrap();
|
||||
bytes[tag + 4..tag + 8].copy_from_slice(&1_946_157_060u32.to_le_bytes());
|
||||
let decoded = decode(&bytes, small_opts()).unwrap();
|
||||
assert_eq!((decoded.image.width, decoded.image.height), (24, 16));
|
||||
assert!(decoded.xmp.is_none(), "invalid optional metadata is ignored");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thumbnail_rejects_huge_gif_before_allocating_source_pixels() {
|
||||
// A logical screen header without a global palette; previously this reached a
|
||||
// multi-gigabyte output allocation before discovering that no frame follows.
|
||||
let bytes = [
|
||||
0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0x8f, 0xa4, 0xb0, 0x3f, 0x57, 0x14, 0xe8, 0x8a, 0xf8, 0xe0, 0xc7, 0x41, 0x9d, 0x0c, 0x10, 0xde, 0x54,
|
||||
0x24, 0xcb, 0xed, 0xff, 0x4a, 0xe7, 0xa0, 0x82, 0x10, 0xa0, 0xbc, 0xac, 0xe5, 0x33, 0x3f, 0xc9, 0x8d, 0x69, 0xf4, 0x24, 0xce, 0x99, 0x4f,
|
||||
0x0d, 0x47, 0xa0, 0xb0, 0xf4, 0xa8, 0x59, 0xad, 0x29, 0xbd, 0x09, 0x48, 0x63, 0xea, 0x59, 0xb7, 0xb7, 0xf2, 0xa1, 0x90, 0xeb, 0x92, 0xc5,
|
||||
0x9c, 0xde, 0xcc, 0x96, 0x67, 0xc4, 0xe5, 0x8c, 0x03, 0x50, 0xb2, 0x30, 0xf8, 0xf4, 0x69, 0x1d, 0xeb, 0xb9, 0xbd, 0x4e, 0xfb, 0xbd, 0xb6,
|
||||
0x02,
|
||||
];
|
||||
let result = decode_thumbnail(&bytes, 16);
|
||||
assert!(matches!(result, Err(Error::TooLarge(42_127, 16_304))), "{result:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thumbnail_respects_configured_source_pixel_limit() {
|
||||
let image = Rgba8::from_fn(3, 2, |_, _| [40, 80, 120, 255]);
|
||||
let bytes = encode_png(&EncodeImage::rgba8(&image), &EncodeMeta::default()).unwrap();
|
||||
let mut opts = ThumbnailOptions::new(1);
|
||||
opts.max_pixels = 5;
|
||||
assert!(matches!(decode_thumbnail_with(&bytes, &opts), Err(Error::TooLarge(3, 2))));
|
||||
opts.max_pixels = 6;
|
||||
assert_eq!(decode_thumbnail_with(&bytes, &opts).unwrap().image.width, 1);
|
||||
}
|
||||
|
||||
fn seeds() -> &'static Vec<Vec<u8>> {
|
||||
@@ -53,6 +102,7 @@ fn seeds() -> &'static Vec<Vec<u8>> {
|
||||
encode_tiff(&e, TiffCompression::Lzw, &meta).unwrap(),
|
||||
encode_tiff(&EncodeImage::new(24, 16, 3, Samples::F32(&f32s)), TiffCompression::Deflate, &meta).unwrap(),
|
||||
encode_webp_lossless(&e, &meta).unwrap(),
|
||||
gain_map_jpeg(&meta),
|
||||
]
|
||||
})
|
||||
}
|
||||
@@ -68,7 +118,7 @@ proptest! {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mutated_valid_files(which in 0usize..6, flips in proptest::collection::vec((any::<prop::sample::Index>(), any::<u8>()), 1..12), cut in any::<prop::sample::Index>()) {
|
||||
fn mutated_valid_files(which in 0usize..7, flips in proptest::collection::vec((any::<prop::sample::Index>(), any::<u8>()), 1..12), cut in any::<prop::sample::Index>()) {
|
||||
let mut b = seeds()[which].clone();
|
||||
for (i, v) in flips {
|
||||
let i = i.index(b.len());
|
||||
|
||||
@@ -0,0 +1,366 @@
|
||||
//! `read_header` (import probes) agrees with a decode on dimensions and orientation, refuses what a
|
||||
//! decode refuses (truncated files included), and returns errors — never panics — on hostile input.
|
||||
|
||||
use lightcraft_codecs::exif::minimal_exif;
|
||||
use lightcraft_codecs::*;
|
||||
use lightcraft_raster::Rgba8;
|
||||
|
||||
fn pixels(w: usize, h: usize) -> Rgba8 {
|
||||
Rgba8::from_fn(w, h, |x, y| [(x * 37 % 256) as u8, (y * 53 % 256) as u8, ((x ^ y) * 9 % 256) as u8, (128 + x % 100) as u8])
|
||||
}
|
||||
|
||||
fn meta(exif: &Option<Vec<u8>>) -> EncodeMeta<'_> {
|
||||
EncodeMeta { exif: exif.as_deref(), xmp: Some("<x/>"), ..Default::default() }
|
||||
}
|
||||
|
||||
/// A TIFF with an Orientation tag (`encode_tiff` writes none), via the `tiff` crate's encoder.
|
||||
fn tiff_oriented(img: &Rgba8, orientation: u16) -> Vec<u8> {
|
||||
use tiff::encoder::{TiffEncoder, colortype};
|
||||
let rgb: Vec<u8> = img.data.iter().flat_map(|p| [p[0], p[1], p[2]]).collect();
|
||||
let mut cur = std::io::Cursor::new(Vec::new());
|
||||
let mut enc = TiffEncoder::new(&mut cur).unwrap();
|
||||
let mut im = enc.new_image::<colortype::RGB8>(img.width as u32, img.height as u32).unwrap();
|
||||
im.encoder().write_tag(tiff::tags::Tag::Orientation, orientation).unwrap();
|
||||
im.write_data(&rgb).unwrap();
|
||||
cur.into_inner()
|
||||
}
|
||||
|
||||
/// A minimal 8-bit RGB PSD (raw or PackBits-literal merged data) with an EXIF resource.
|
||||
fn psd(w: usize, h: usize, orientation: Option<u16>, rle: bool) -> Vec<u8> {
|
||||
let mut b = Vec::new();
|
||||
b.extend_from_slice(b"8BPS");
|
||||
b.extend_from_slice(&1u16.to_be_bytes());
|
||||
b.extend_from_slice(&[0; 6]);
|
||||
b.extend_from_slice(&3u16.to_be_bytes());
|
||||
b.extend_from_slice(&(h as u32).to_be_bytes());
|
||||
b.extend_from_slice(&(w as u32).to_be_bytes());
|
||||
b.extend_from_slice(&8u16.to_be_bytes());
|
||||
b.extend_from_slice(&3u16.to_be_bytes());
|
||||
b.extend_from_slice(&0u32.to_be_bytes()); // colour mode data
|
||||
let mut res = Vec::new();
|
||||
if let Some(o) = orientation {
|
||||
let exif = minimal_exif(o);
|
||||
res.extend_from_slice(b"8BIM");
|
||||
res.extend_from_slice(&1058u16.to_be_bytes());
|
||||
res.extend_from_slice(&[0, 0]); // empty pascal name, padded
|
||||
res.extend_from_slice(&(exif.len() as u32).to_be_bytes());
|
||||
res.extend_from_slice(&exif);
|
||||
if exif.len() % 2 == 1 {
|
||||
res.push(0);
|
||||
}
|
||||
}
|
||||
b.extend_from_slice(&(res.len() as u32).to_be_bytes());
|
||||
b.extend_from_slice(&res);
|
||||
b.extend_from_slice(&0u32.to_be_bytes()); // layer and mask info
|
||||
b.extend_from_slice(&(rle as u16).to_be_bytes());
|
||||
let plane = |c: usize| (0..w * h).map(move |i| ((i * (c + 3) * 11) % 256) as u8);
|
||||
if rle {
|
||||
// every row as literal runs of at most 128 bytes (PackBits' longest)
|
||||
let row_len = w + w.div_ceil(128);
|
||||
for _ in 0..3 * h {
|
||||
b.extend_from_slice(&(row_len as u16).to_be_bytes());
|
||||
}
|
||||
for c in 0..3 {
|
||||
let p: Vec<u8> = plane(c).collect();
|
||||
for row in p.chunks(w) {
|
||||
for run in row.chunks(128) {
|
||||
b.push(run.len() as u8 - 1);
|
||||
b.extend_from_slice(run);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for c in 0..3 {
|
||||
b.extend(plane(c));
|
||||
}
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
/// Generated files of every format with a header reader, plus a fallback one (BMP via `image`).
|
||||
fn fixtures(w: usize, h: usize) -> Vec<(String, Vec<u8>)> {
|
||||
let img = pixels(w, h);
|
||||
let e = EncodeImage::rgba8(&img);
|
||||
let rgb: Vec<u8> = img.data.iter().flat_map(|p| [p[0], p[1], p[2]]).collect();
|
||||
let rgb_e = EncodeImage::new(w as u32, h as u32, 3, Samples::U8(&rgb));
|
||||
let u16s: Vec<u16> = (0..w * h * 3).map(|i| (i * 331) as u16).collect();
|
||||
let mut out = Vec::new();
|
||||
for o in [None, Some(1), Some(6), Some(8)] {
|
||||
let exif = o.map(minimal_exif);
|
||||
let tag = format!("o{}", o.unwrap_or(0));
|
||||
out.push((format!("jpeg 420 {tag}"), encode_jpeg(&rgb_e, 85, ChromaSubsampling::S420, &meta(&exif)).unwrap()));
|
||||
out.push((format!("jpeg 444 {tag}"), encode_jpeg(&rgb_e, 85, ChromaSubsampling::S444, &meta(&exif)).unwrap()));
|
||||
out.push((format!("png rgba {tag}"), encode_png(&e, &meta(&exif)).unwrap()));
|
||||
out.push((format!("png 16-bit {tag}"), encode_png(&EncodeImage::new(w as u32, h as u32, 3, Samples::U16(&u16s)), &meta(&exif)).unwrap()));
|
||||
out.push((format!("webp {tag}"), encode_webp_lossless(&e, &meta(&exif)).unwrap()));
|
||||
out.push((format!("psd {tag}"), psd(w, h, o, false)));
|
||||
out.push((format!("psd rle {tag}"), psd(w, h, o, true)));
|
||||
out.push((format!("tiff {tag}"), tiff_oriented(&img, o.unwrap_or(1))));
|
||||
}
|
||||
out.push(("tiff lzw".into(), encode_tiff(&e, TiffCompression::Lzw, &EncodeMeta::default()).unwrap()));
|
||||
out.push(("tiff packbits".into(), encode_tiff(&e, TiffCompression::PackBits, &EncodeMeta::default()).unwrap()));
|
||||
let mut bmp = Vec::new();
|
||||
image::codecs::bmp::BmpEncoder::new(&mut bmp).encode(&rgb, w as u32, h as u32, image::ExtendedColorType::Rgb8).unwrap();
|
||||
out.push(("bmp".into(), bmp));
|
||||
out
|
||||
}
|
||||
|
||||
/// What the import probe learnt before it read headers: a small decode.
|
||||
fn decoded(bytes: &[u8]) -> Result<Header> {
|
||||
let d = decode(bytes, DecodeOptions::fit(64, 64))?;
|
||||
Ok(Header { format: d.format, width: d.source_width, height: d.source_height, orientation: d.orientation })
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_matches_decode() {
|
||||
for (w, h) in [(160, 96), (96, 160), (7, 5)] {
|
||||
for (name, bytes) in fixtures(w, h) {
|
||||
let header = read_header(&bytes).unwrap_or_else(|e| panic!("{name} {w}×{h}: {e}"));
|
||||
assert_eq!(header, decoded(&bytes).unwrap(), "{name} {w}×{h}");
|
||||
assert_eq!((header.width, header.height), (w as u32, h as u32), "{name}");
|
||||
}
|
||||
}
|
||||
let o = |name: &str| fixtures(16, 8).into_iter().find(|f| f.0 == name).map(|f| read_header(&f.1).unwrap().orientation).unwrap();
|
||||
assert_eq!((o("jpeg 420 o6"), o("png rgba o8"), o("webp o6"), o("psd o8"), o("tiff o6"), o("jpeg 444 o0")), (6, 8, 6, 8, 6, 1));
|
||||
}
|
||||
|
||||
/// Whenever the decode the probe used to run accepts a file (here: every cut of every fixture),
|
||||
/// the header agrees with it: reading headers never refuses a file that decodes. The reverse
|
||||
/// is checked for truncation, which the header readers detect.
|
||||
#[test]
|
||||
fn truncated_files_agree_with_decode() {
|
||||
// 160 px wide: large JPEGs take the DCT-scaled (strict) decode path the probe took for photos
|
||||
for (name, bytes) in fixtures(160, 96) {
|
||||
let n = bytes.len();
|
||||
let mut cuts: Vec<usize> = (0..100).map(|i| n * i / 100).collect();
|
||||
cuts.extend(n.saturating_sub(64)..n);
|
||||
for cut in cuts {
|
||||
let part = &bytes[..cut];
|
||||
if let Ok(d) = decoded(part) {
|
||||
assert_eq!(read_header(part).ok(), Some(d), "{name} cut at {cut}/{n}");
|
||||
}
|
||||
}
|
||||
for cut in [n / 4, n / 2, n * 9 / 10] {
|
||||
assert!(read_header(&bytes[..cut]).is_err(), "{name}: cut at {cut}/{n} accepted");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncation_details() {
|
||||
let f = fixtures(160, 96);
|
||||
let get = |name: &str| f.iter().find(|x| x.0 == name).map(|x| x.1.clone()).unwrap();
|
||||
let jpeg = get("jpeg 420 o6");
|
||||
assert!(read_header(&jpeg[..jpeg.len() - 2]).is_err(), "a JPEG without its EOI is truncated");
|
||||
let mut trailing = jpeg.clone();
|
||||
trailing.extend_from_slice(&[0; 100]);
|
||||
assert_eq!(read_header(&trailing).unwrap().orientation, 6, "bytes after the EOI are fine");
|
||||
let png = get("png rgba o6");
|
||||
assert!(read_header(&png[..png.len() - 2]).is_ok(), "a damaged IEND is ignored, as by a decode");
|
||||
assert!(read_header(&png[..png.len() - 12]).is_err(), "nothing after the image data: truncated");
|
||||
}
|
||||
|
||||
/// eXIf after the image data counts, as it does for a decode (which reads the whole file).
|
||||
#[test]
|
||||
fn png_exif_after_idat() {
|
||||
let img = pixels(9, 4);
|
||||
let plain = encode_png(&EncodeImage::rgba8(&img), &EncodeMeta::default()).unwrap();
|
||||
let exif = minimal_exif(6);
|
||||
let mut chunk = (exif.len() as u32).to_be_bytes().to_vec();
|
||||
chunk.extend_from_slice(b"eXIf");
|
||||
chunk.extend_from_slice(&exif);
|
||||
chunk.extend_from_slice(&[0; 4]); // CRC: not checked by either reader
|
||||
let iend = plain.len() - 12;
|
||||
let mut b = plain[..iend].to_vec();
|
||||
b.extend_from_slice(&chunk);
|
||||
b.extend_from_slice(&plain[iend..]);
|
||||
assert_eq!(read_header(&b).unwrap().orientation, 6);
|
||||
}
|
||||
|
||||
fn jpeg_segments(sof: u8, w: u16, h: u16, comps: u8) -> Vec<u8> {
|
||||
let mut b = vec![0xFF, 0xD8, 0xFF, sof, 0, 8 + 3 * comps as u16 as u8, 8];
|
||||
b.extend_from_slice(&h.to_be_bytes());
|
||||
b.extend_from_slice(&w.to_be_bytes());
|
||||
b.push(comps);
|
||||
for c in 0..comps {
|
||||
b.extend_from_slice(&[c + 1, 0x11, 0]);
|
||||
}
|
||||
b.extend_from_slice(&[0xFF, 0xDA, 0, 8, 1, 1, 0, 0, 63, 0]);
|
||||
b.extend_from_slice(&[0x12, 0xFF, 0x00, 0x34, 0xFF, 0xD0, 0x56]);
|
||||
b.extend_from_slice(&[0xFF, 0xD9]);
|
||||
b
|
||||
}
|
||||
|
||||
fn png_ihdr(w: u32, h: u32, depth: u8, color: u8) -> Vec<u8> {
|
||||
let mut b = b"\x89PNG\r\n\x1a\n\0\0\0\x0dIHDR".to_vec();
|
||||
b.extend_from_slice(&w.to_be_bytes());
|
||||
b.extend_from_slice(&h.to_be_bytes());
|
||||
b.extend_from_slice(&[depth, color, 0, 0, 0, 0, 0, 0, 0]);
|
||||
b
|
||||
}
|
||||
|
||||
fn png_chunk(b: &mut Vec<u8>, kind: &[u8; 4], len: u32, data: &[u8]) {
|
||||
b.extend_from_slice(&len.to_be_bytes());
|
||||
b.extend_from_slice(kind);
|
||||
b.extend_from_slice(data);
|
||||
b.extend_from_slice(&[0; 4]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_headers_are_errors() {
|
||||
// a well-formed synthetic JPEG header is read
|
||||
assert_eq!(read_header(&jpeg_segments(0xC0, 33, 21, 3)).unwrap(), Header { format: Format::Jpeg, width: 33, height: 21, orientation: 1 });
|
||||
let jpeg_bad = [
|
||||
jpeg_segments(0xC0, 0, 21, 3), // zero width
|
||||
jpeg_segments(0xC0, 33, 0, 3), // zero height (DNL)
|
||||
jpeg_segments(0xC0, 65535, 65535, 3), // 4.3 Gpx
|
||||
jpeg_segments(0xC9, 33, 21, 3), // arithmetic coding
|
||||
jpeg_segments(0xC5, 33, 21, 3), // hierarchical
|
||||
jpeg_segments(0xC0, 33, 21, 2), // two components
|
||||
jpeg_segments(0xC0, 33, 21, 3)[..30].to_vec(),
|
||||
vec![0xFF, 0xD8, 0xFF, 0xE1, 0, 0], // segment length 0
|
||||
vec![0xFF, 0xD8, 0xFF, 0xE1, 0xFF, 0xFF], // segment past the end
|
||||
vec![0xFF, 0xD8, 0xFF, 0xD9],
|
||||
];
|
||||
for (i, b) in jpeg_bad.iter().enumerate() {
|
||||
assert!(read_header(b).is_err(), "JPEG case {i}");
|
||||
}
|
||||
let mut no_sos = jpeg_segments(0xC0, 33, 21, 3);
|
||||
no_sos.truncate(2 + 2 + 17);
|
||||
no_sos.extend_from_slice(&[0xFF, 0xD9]);
|
||||
assert!(read_header(&no_sos).is_err(), "JPEG without a scan");
|
||||
|
||||
let mut ok = png_ihdr(5, 3, 8, 2);
|
||||
png_chunk(&mut ok, b"IDAT", 1, &[0]);
|
||||
png_chunk(&mut ok, b"IEND", 0, &[]);
|
||||
assert_eq!(read_header(&ok).unwrap(), Header { format: Format::Png, width: 5, height: 3, orientation: 1 });
|
||||
let png_case = |ihdr: Vec<u8>, chunks: &[(&[u8; 4], u32, &[u8])]| {
|
||||
let mut b = ihdr;
|
||||
for (k, l, d) in chunks {
|
||||
png_chunk(&mut b, k, *l, d);
|
||||
}
|
||||
b
|
||||
};
|
||||
let idat: (&[u8; 4], u32, &[u8]) = (b"IDAT", 1, &[0]);
|
||||
let iend: (&[u8; 4], u32, &[u8]) = (b"IEND", 0, &[]);
|
||||
let png_bad = [
|
||||
png_case(png_ihdr(0, 3, 8, 2), &[idat, iend]),
|
||||
png_case(png_ihdr(5, 0, 8, 2), &[idat, iend]),
|
||||
png_case(png_ihdr(0x8000_0000, 1, 8, 2), &[idat, iend]),
|
||||
png_case(png_ihdr(65536, 65536, 8, 2), &[idat, iend]),
|
||||
png_case(png_ihdr(5, 3, 4, 2), &[idat, iend]), // RGB at 4 bits
|
||||
png_case(png_ihdr(5, 3, 8, 5), &[idat, iend]), // no colour type 5
|
||||
png_case(png_ihdr(5, 3, 8, 3), &[idat, iend]), // indexed without PLTE
|
||||
png_case(png_ihdr(5, 3, 8, 2), &[iend]), // no IDAT
|
||||
png_case(png_ihdr(5, 3, 8, 2), &[idat]), // nothing after the image data
|
||||
png_case(png_ihdr(5, 3, 8, 2), &[(b"IDAT", 0xFFFF_FFFF, &[0]), iend]),
|
||||
png_case(png_ihdr(5, 3, 8, 2), &[(b"tEXt", 0x7FFF_FFFF, &[0]), idat, iend]),
|
||||
png_ihdr(5, 3, 8, 2)[..20].to_vec(),
|
||||
b"\x89PNG\r\n\x1a\n\0\0\0\x0dIHDX\0\0\0\x05\0\0\0\x03\x08\x02\0\0\0".to_vec(),
|
||||
];
|
||||
for (i, b) in png_bad.iter().enumerate() {
|
||||
assert!(read_header(b).is_err(), "PNG case {i}");
|
||||
}
|
||||
|
||||
let tiff = encode_tiff(&EncodeImage::rgba8(&pixels(6, 4)), TiffCompression::None, &EncodeMeta::default()).unwrap();
|
||||
let ifd = u32::from_le_bytes([tiff[4], tiff[5], tiff[6], tiff[7]]) as usize;
|
||||
let mut tiff_bad = vec![b"II*\0\xff\xff\xff\x7f".to_vec(), b"II*\0\x08\0\0\0".to_vec(), b"II*\0\x04\0\0\0".to_vec()];
|
||||
// cut short after the IFD
|
||||
assert!(ifd < tiff.len() / 2, "the IFD comes before the image data");
|
||||
tiff_bad.push(tiff[..tiff.len() - 10].to_vec());
|
||||
for (i, b) in tiff_bad.iter().enumerate() {
|
||||
assert!(read_header(b).is_err(), "TIFF case {i}");
|
||||
}
|
||||
// a strip offset pointing past the end of the file
|
||||
let t = |b: &[u8], p: usize| u16::from_le_bytes([b[p], b[p + 1]]);
|
||||
let n = t(&tiff, ifd) as usize;
|
||||
let mut far = tiff.clone();
|
||||
for e in 0..n {
|
||||
let p = ifd + 2 + e * 12;
|
||||
if t(&far, p) == 273 && t(&far, p + 2) == 4 && u32::from_le_bytes([far[p + 4], far[p + 5], far[p + 6], far[p + 7]]) == 1 {
|
||||
far[p + 8..p + 12].copy_from_slice(&0x7FFF_0000u32.to_le_bytes());
|
||||
}
|
||||
}
|
||||
assert_ne!(far, tiff, "the strip offset was found");
|
||||
assert!(read_header(&far).is_err(), "strip past the end");
|
||||
|
||||
let webp_bad = [
|
||||
b"RIFF\0\0\0\0WEBP".to_vec(),
|
||||
b"RIFF\0\0\0\0WEBPVP8L\xff\xff\xff\xff\x2f".to_vec(),
|
||||
b"RIFF\0\0\0\0WEBPXXXX\xff\xff\xff\xff".to_vec(),
|
||||
b"RIFF\0\0\0\0WEBPVP8 \x04\0\0\0\0\0\0\0".to_vec(),
|
||||
];
|
||||
for (i, b) in webp_bad.iter().enumerate() {
|
||||
assert!(read_header(b).is_err(), "WebP case {i}");
|
||||
}
|
||||
|
||||
let good = psd(4, 3, Some(6), true);
|
||||
assert_eq!(read_header(&good).unwrap().orientation, 6);
|
||||
let set = |pos: usize, v: &[u8]| {
|
||||
let mut b = good.clone();
|
||||
b[pos..pos + v.len()].copy_from_slice(v);
|
||||
b
|
||||
};
|
||||
let res_len = u32::from_be_bytes([good[30], good[31], good[32], good[33]]) as usize;
|
||||
let compression = 34 + res_len + 4;
|
||||
let mut psd_bad = vec![
|
||||
set(12, &0u16.to_be_bytes()), // no channels
|
||||
set(12, &57u16.to_be_bytes()), // too many
|
||||
set(12, &2u16.to_be_bytes()), // too few for RGB
|
||||
set(14, &0x7FFF_FFFFu32.to_be_bytes()), // absurd height
|
||||
set(18, &0u32.to_be_bytes()), // zero width
|
||||
set(22, &1u16.to_be_bytes()), // 1-bit
|
||||
set(22, &7u16.to_be_bytes()), // bad depth
|
||||
set(24, &9u16.to_be_bytes()), // Lab
|
||||
set(24, &77u16.to_be_bytes()), // unknown mode
|
||||
set(24, &2u16.to_be_bytes()), // indexed without a palette
|
||||
set(compression, &2u16.to_be_bytes()), // ZIP
|
||||
set(26, &0xFFFF_FFFFu32.to_be_bytes()), // colour mode data past the end
|
||||
set(30, &0xFFFF_FFF0u32.to_be_bytes()), // resources past the end
|
||||
];
|
||||
// a 1 × 2^30 RLE image declared in a tiny file: refused without reading 2^30 row counts
|
||||
let mut tall = good.clone();
|
||||
tall[14..18].copy_from_slice(&(1u32 << 30).to_be_bytes());
|
||||
tall[18..22].copy_from_slice(&1u32.to_be_bytes());
|
||||
psd_bad.push(tall);
|
||||
psd_bad.push(good[..good.len() - 1].to_vec());
|
||||
for (i, b) in psd_bad.iter().enumerate() {
|
||||
assert!(read_header(b).is_err(), "PSD case {i}");
|
||||
}
|
||||
|
||||
for b in [&b""[..], b"\xFF\xD8\xFF", b"\x89PNG\r\n\x1a\n", b"II*\0", b"MM\0*", b"8BPS\0\x01", b"GIF89a", b"not an image at all"] {
|
||||
assert!(read_header(b).is_err(), "{b:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The same with a few random bytes changed: whenever the decode accepts the damaged file, the
|
||||
/// header agrees with it (deterministic seed).
|
||||
#[test]
|
||||
fn mutated_files_agree_with_decode() {
|
||||
let mut s: u64 = 0x1234_5678_9abc_def1;
|
||||
let mut rnd = || {
|
||||
s ^= s << 13;
|
||||
s ^= s >> 7;
|
||||
s ^= s << 17;
|
||||
s
|
||||
};
|
||||
let (mut decodable, mut disagreements) = (0, Vec::new());
|
||||
for (name, bytes) in fixtures(160, 96) {
|
||||
for _ in 0..300 {
|
||||
let mut b = bytes.clone();
|
||||
for _ in 0..1 + rnd() % 3 {
|
||||
let i = rnd() as usize % b.len();
|
||||
b[i] = rnd() as u8;
|
||||
}
|
||||
if let Ok(d) = decoded(&b) {
|
||||
decodable += 1;
|
||||
let h = read_header(&b);
|
||||
if h.as_ref().ok() != Some(&d) {
|
||||
disagreements.push(format!("{name}: decode {d:?}, header {h:?}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(decodable > 1000, "{decodable}");
|
||||
assert!(disagreements.is_empty(), "{disagreements:#?}");
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
//! HEIC/HEIF through the codec API (feature `heif`), on lossless synthetic files from
|
||||
//! `lightcraft_heif::testdata`: an iPhone-like grid with a Display P3 profile, rotation, EXIF, a
|
||||
//! thumbnail and an HDR gain map; `nclx`-only colour; 10-bit; and hostile input.
|
||||
#![cfg(feature = "heif")]
|
||||
|
||||
use lightcraft_codecs::*;
|
||||
use lightcraft_heif::testdata::{Prop, Spec, build};
|
||||
|
||||
fn grey(x: u32, y: u32) -> [u16; 3] {
|
||||
[((x * 7 + y * 13) % 220 + 20) as u16, 128, 128]
|
||||
}
|
||||
|
||||
/// EXIF whose Orientation (6) only mirrors the container's `irot`: it must not turn the photo again.
|
||||
fn exif_orientation_6() -> Vec<u8> {
|
||||
let mut t = b"MM\0\x2a\0\0\0\x08\0\x01".to_vec();
|
||||
t.extend_from_slice(&[0x01, 0x12, 0, 3, 0, 0, 0, 1, 0, 6, 0, 0]);
|
||||
t.extend_from_slice(&[0, 0, 0, 0]);
|
||||
t
|
||||
}
|
||||
|
||||
/// The thumbnail is flat mid grey so a test can tell it from the photo.
|
||||
fn iphone_like() -> Vec<u8> {
|
||||
build(&Spec {
|
||||
props: vec![Prop::Icc(icc::write_named(NamedSpace::DisplayP3)), Prop::Irot(3)],
|
||||
exif: Some(exif_orientation_6()),
|
||||
thumbnail: Some(&|_, _| [126, 128, 128]),
|
||||
ignored_aux: vec!["urn:com:apple:photo:2020:aux:hdrgainmap"],
|
||||
..Spec::new(96, 64, &grey)
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_iphone_like_heic_decodes_upright_in_display_p3() {
|
||||
let bytes = iphone_like();
|
||||
assert_eq!(sniff(&bytes), Some(Format::Heif));
|
||||
let d = decode(&bytes, DecodeOptions::default()).unwrap();
|
||||
// irot 3 = 90° clockwise, applied by the decoder; the EXIF Orientation is not applied again.
|
||||
assert_eq!((d.width, d.height, d.source_width, d.source_height, d.orientation), (64, 96, 64, 96, 1));
|
||||
assert_eq!(d.space.named, Some(NamedSpace::DisplayP3));
|
||||
assert_eq!(d.space.origin, SpaceOrigin::IccMatrixTrc);
|
||||
assert_eq!((d.bit_depth, d.has_alpha, d.float), (8, false, false));
|
||||
assert!(d.exif.is_some() && d.icc.is_some());
|
||||
// No nclx and no VUI: limited range, BT.601; neutral chroma stays neutral.
|
||||
let srgb = d.to_srgb8();
|
||||
let px = srgb.data[0];
|
||||
assert!(px[0].abs_diff(px[1]) <= 1 && px[1].abs_diff(px[2]) <= 1, "{px:?}");
|
||||
|
||||
let h = read_header(&bytes).unwrap();
|
||||
assert_eq!((h.format, h.width, h.height, h.orientation), (Format::Heif, 64, 96, 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_small_decode_uses_the_embedded_thumbnail() {
|
||||
let bytes = iphone_like();
|
||||
// The 48 × 32 thumbnail, turned like the photo, covers a 32-pixel request.
|
||||
let d = decode(&bytes, DecodeOptions::fit(32, 32)).unwrap();
|
||||
assert!(d.width.max(d.height) <= 32 && d.height > d.width, "{}x{}", d.width, d.height);
|
||||
assert_eq!((d.source_width, d.source_height), (64, 96));
|
||||
assert_eq!(d.space.named, Some(NamedSpace::DisplayP3));
|
||||
let flat = d.to_srgb8();
|
||||
assert!(flat.data.iter().all(|p| *p == flat.data[0]), "the flat thumbnail, not the photo");
|
||||
// A thumbnail too small for the request: the photo is decoded and resized.
|
||||
let d = decode(&bytes, DecodeOptions::fit(60, 60)).unwrap();
|
||||
assert_eq!((d.width.max(d.height), d.source_width), (60, 64));
|
||||
let photo = d.to_srgb8();
|
||||
assert!(photo.data.iter().any(|p| *p != photo.data[0]));
|
||||
let t = decode_thumbnail(&bytes, 32).unwrap();
|
||||
assert_eq!((t.source_width, t.source_height), (64, 96));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nclx_colour_and_ten_bit() {
|
||||
// Display P3 primaries, sRGB transfer, BT.709 matrix, full range: the nclx alone names the space.
|
||||
let bytes = build(&Spec { bit_depth: 10, props: vec![Prop::Nclx(12, 13, 1, true)], ..Spec::new(64, 32, &|x, _| [(x * 16) as u16, 512, 512]) });
|
||||
let d = decode(&bytes, DecodeOptions::default()).unwrap();
|
||||
assert_eq!(d.space.named, Some(NamedSpace::DisplayP3));
|
||||
assert_eq!(d.space.origin, SpaceOrigin::Container);
|
||||
assert_eq!(d.bit_depth, 10);
|
||||
// Full-range 10-bit neutral steps of 16 code values: linear values rise monotonically.
|
||||
let row: Vec<f32> = (0..64).map(|x| d.image.row(0)[x][1]).collect();
|
||||
assert!(row.windows(2).all(|w| w[1] > w[0]), "{row:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_heic_is_an_error_never_a_panic() {
|
||||
let bytes = iphone_like();
|
||||
let full = decode(&bytes, DecodeOptions::default()).unwrap();
|
||||
for cut in (12..bytes.len()).step_by(11) {
|
||||
let b = &bytes[..cut];
|
||||
// Cut inside the photo: an error. Cut in a trailing item (gain map, thumbnail): the photo.
|
||||
if let (Ok(_), Ok(d)) = (read_header(b), decode(b, DecodeOptions::default())) {
|
||||
assert_eq!(d.image.data, full.image.data, "cut at {cut}");
|
||||
}
|
||||
let _ = decode_thumbnail(b, 16);
|
||||
}
|
||||
let r = decode(&bytes, DecodeOptions { max_pixels: 64 * 96 - 1, ..Default::default() });
|
||||
assert!(matches!(r, Err(Error::TooLarge(64, 96))), "{r:?}");
|
||||
}
|
||||
@@ -245,6 +245,80 @@ fn tiff_cmyk_and_orientation() {
|
||||
assert!(red[0] > 0.99 && red[1] < 0.01 && red[2] < 0.01, "{red:?}");
|
||||
}
|
||||
|
||||
/// A one-strip, uncompressed, single-sample TIFF (little-endian) whose samples are `bits` wide and
|
||||
/// already packed into `strip`, one padded row after another.
|
||||
fn packed_tiff(width: u16, height: u16, bits: u16, photometric: u16, strip: &[u8]) -> Vec<u8> {
|
||||
const STRIP_AT: u32 = 8 + 2 + 9 * 12 + 4;
|
||||
let short = |tag: u16, v: u16| [&tag.to_le_bytes()[..], &3u16.to_le_bytes(), &1u32.to_le_bytes(), &v.to_le_bytes(), &[0, 0]].concat();
|
||||
let long = |tag: u16, v: u32| [&tag.to_le_bytes()[..], &4u16.to_le_bytes(), &1u32.to_le_bytes(), &v.to_le_bytes()].concat();
|
||||
let mut b = b"II*\0".to_vec();
|
||||
b.extend(8u32.to_le_bytes());
|
||||
b.extend(9u16.to_le_bytes());
|
||||
b.extend(short(256, width));
|
||||
b.extend(short(257, height));
|
||||
b.extend(short(258, bits));
|
||||
b.extend(short(259, 1));
|
||||
b.extend(short(262, photometric));
|
||||
b.extend(long(273, STRIP_AT));
|
||||
b.extend(short(277, 1));
|
||||
b.extend(short(278, height));
|
||||
b.extend(long(279, strip.len() as u32));
|
||||
b.extend(0u32.to_le_bytes());
|
||||
assert_eq!(b.len(), STRIP_AT as usize);
|
||||
b.extend_from_slice(strip);
|
||||
b
|
||||
}
|
||||
|
||||
/// WhiteIsZero (photometric 0) gray: the lowest sample is white. The `tiff` crate already inverts these
|
||||
/// samples, and a second inversion here used to show every WhiteIsZero scan as a negative.
|
||||
#[test]
|
||||
fn tiff_white_is_zero_is_inverted_once() {
|
||||
// Two pixels: the lowest sample, then the highest, at 8 and 16 bits.
|
||||
for (bits, strip) in [(8, vec![0u8, 255]), (16, [0u16.to_le_bytes(), 65535u16.to_le_bytes()].concat())] {
|
||||
for (photometric, first) in [(0, 1.0), (1, 0.0)] {
|
||||
let d = decode(&packed_tiff(2, 1, bits, photometric, &strip), DecodeOptions::default()).unwrap();
|
||||
let (a, b) = (d.image.get(0, 0)[1], d.image.get(1, 0)[1]);
|
||||
assert!((a - first).abs() < 1e-6 && (b - (1.0 - first)).abs() < 1e-6, "{bits}-bit, photometric {photometric}: {a} {b}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 1-, 2- and 4-bit samples (bilevel scans, fax images, low-depth gray) arrive packed eight, four or
|
||||
/// two to a byte with each row padded to a byte; they used to be rejected as "sample buffer too short".
|
||||
#[test]
|
||||
fn tiff_samples_narrower_than_a_byte() {
|
||||
let gray = |d: &Decoded, x: usize, y: usize| d.image.get(x, y)[1];
|
||||
|
||||
// 10 × 2 bilevel: rows are 2 bytes, the last 6 bits of each are padding (set, to prove they are ignored).
|
||||
// Row 0 alternates set/clear bits; row 1 is clear but for its last pixel.
|
||||
let rows = [0b1010_1010, 0b1011_1111, 0b0000_0000, 0b0111_1111];
|
||||
// What a set bit decodes to: white for BlackIsZero (1), black for WhiteIsZero (0).
|
||||
for (photometric, set_bit) in [(1, 1.0), (0, 0.0)] {
|
||||
let d = decode(&packed_tiff(10, 2, 1, photometric, &rows), DecodeOptions::default()).unwrap();
|
||||
assert_eq!((d.image.width, d.image.height, d.bit_depth), (10, 2, 1));
|
||||
assert!(d.grayscale);
|
||||
let clear_bit = 1.0 - set_bit;
|
||||
for x in 0..10 {
|
||||
let want = (if x % 2 == 0 { set_bit } else { clear_bit }, if x == 9 { set_bit } else { clear_bit });
|
||||
assert!((gray(&d, x, 0) - want.0).abs() < 1e-6, "row 0, x {x}, photometric {photometric}");
|
||||
assert!((gray(&d, x, 1) - want.1).abs() < 1e-6, "row 1, x {x}, photometric {photometric}");
|
||||
}
|
||||
}
|
||||
|
||||
// 2-bit: 0, 1, 2, 3 → 0, 85, 170, 255. 4-bit: 0, 15, 8 → 0, 255, 136 (the row's last nibble is padding).
|
||||
let d = decode(&packed_tiff(4, 1, 2, 1, &[0b00_01_10_11]), DecodeOptions::default()).unwrap();
|
||||
for (x, v) in [0, 85, 170, 255].into_iter().enumerate() {
|
||||
assert!((gray(&d, x, 0) - srgb_to_linear(v as f32 / 255.0)).abs() < 1e-6, "2-bit x {x}");
|
||||
}
|
||||
let d = decode(&packed_tiff(3, 1, 4, 1, &[0x0F, 0x8F]), DecodeOptions::default()).unwrap();
|
||||
for (x, v) in [0, 255, 136].into_iter().enumerate() {
|
||||
assert!((gray(&d, x, 0) - srgb_to_linear(v as f32 / 255.0)).abs() < 1e-6, "4-bit x {x}");
|
||||
}
|
||||
|
||||
// A strip shorter than its rows is an error, never a panic.
|
||||
assert!(decode(&packed_tiff(10, 2, 1, 1, &rows[..3]), DecodeOptions::default()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn webp_lossless_exact() {
|
||||
let img = gradient(31, 19);
|
||||
@@ -461,7 +535,7 @@ fn thumbnail_accepts_smaller_preview_when_asked() {
|
||||
encode_jpeg(&EncodeImage::rgba8(&main), 80, ChromaSubsampling::S420, &EncodeMeta { exif: Some(&exif), ..Default::default() }).unwrap();
|
||||
let t = decode_thumbnail(&bytes, 256).unwrap();
|
||||
assert_eq!(t.source, ThumbnailSource::Scaled);
|
||||
let t = decode_thumbnail_with(&bytes, &ThumbnailOptions { max_edge: 256, min_embedded_edge: 160 }).unwrap();
|
||||
let t = decode_thumbnail_with(&bytes, &ThumbnailOptions { min_embedded_edge: 160, ..ThumbnailOptions::new(256) }).unwrap();
|
||||
assert_eq!(t.source, ThumbnailSource::ExifThumbnail);
|
||||
assert_eq!((t.image.width, t.image.height), (160, 120));
|
||||
}
|
||||
|
||||
+17
-6
@@ -2,7 +2,9 @@
|
||||
//!
|
||||
//! The Planckian locus uses the Kim et al. (2002) cubic approximation (1667–25000 K); outside that
|
||||
//! range it is extended linearly in mired space. Tint is a signed offset perpendicular to the locus in
|
||||
//! CIE 1960 (u, v): **positive tint = magenta** (below the locus), `Duv = −tint / TINT_SCALE`.
|
||||
//! CIE 1960 (u, v). Tint describes the **correction**: positive adds magenta, negative adds green.
|
||||
//! A positive correction neutralises a green illuminant above the locus, so its source white uses
|
||||
//! `Duv = tint / TINT_SCALE`. Bradford adaptation removes that source cast.
|
||||
|
||||
use crate::{Mat3, RgbSpace, Xy, bradford};
|
||||
|
||||
@@ -65,11 +67,12 @@ fn locus_normal(t: f64) -> (f64, f64) {
|
||||
if ny < 0.0 { (-nx, -ny) } else { (nx, ny) }
|
||||
}
|
||||
|
||||
/// Chromaticity of the white point for temperature `t` (K) and `tint`.
|
||||
/// Source illuminant chromaticity for temperature `t` (K) and correction `tint`.
|
||||
/// Positive tint means a green source white, whose correction adds magenta to the image.
|
||||
pub fn temp_tint_to_xy(t: f64, tint: f64) -> Xy {
|
||||
let (u, v) = locus_uv(t);
|
||||
let (nu, nv) = locus_normal(t);
|
||||
let duv = -tint / TINT_SCALE;
|
||||
let duv = tint / TINT_SCALE;
|
||||
uv_to_xy(u + nu * duv, v + nv * duv)
|
||||
}
|
||||
|
||||
@@ -104,7 +107,7 @@ pub fn xy_to_temp_tint(p: Xy) -> (f64, f64) {
|
||||
let (lu, lv) = locus_uv(t);
|
||||
let (nu, nv) = locus_normal(t);
|
||||
let duv = (u - lu) * nu + (v - lv) * nv;
|
||||
(t, -duv * TINT_SCALE)
|
||||
(t, duv * TINT_SCALE)
|
||||
}
|
||||
|
||||
/// Matrix (linear RGB in `space` → same space) that white-balances a scene lit by `src` so that it
|
||||
@@ -141,10 +144,18 @@ mod tests {
|
||||
let warm = temp_tint_to_xy(3000.0, 0.0);
|
||||
let cool = temp_tint_to_xy(9000.0, 0.0);
|
||||
assert!(warm.x > cool.x);
|
||||
// positive tint is magenta: lower v
|
||||
// A positive magenta correction specifies a greener source illuminant: higher v.
|
||||
let (_, v0) = xy_to_uv(temp_tint_to_xy(5000.0, 0.0));
|
||||
let (_, v1) = xy_to_uv(temp_tint_to_xy(5000.0, 50.0));
|
||||
assert!(v1 < v0);
|
||||
assert!(v1 > v0);
|
||||
}
|
||||
|
||||
/// Daylight lies above the Planckian locus, so D65 reads as a small positive tint, as
|
||||
/// Lightroom's Daylight preset (5500 K, +10) has it (#188).
|
||||
#[test]
|
||||
fn daylight_has_a_positive_tint() {
|
||||
let (_, tint) = xy_to_temp_tint(crate::D65);
|
||||
assert!((5.0..15.0).contains(&tint), "{tint}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
[package]
|
||||
name = "lightcraft-denoise-core"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
description = "LightCraft denoise model metadata, Bayer tiling and cache formats (L1; no inference backend)"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
rawnind-model = []
|
||||
|
||||
[dependencies]
|
||||
lightcraft-raster = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
miniz_oxide = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
web-time = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = { workspace = true }
|
||||
@@ -0,0 +1,7 @@
|
||||
# lightcraft-denoise-core
|
||||
|
||||
L1 model metadata, bounded archive/cache formats and Bayer packing/tiling helpers.
|
||||
No weights or inference backend. See [the user workflow](../../docs/denoise.md).
|
||||
`lightcraft-denoise` re-exports these types for CPU/GPU users; the engine uses this
|
||||
crate directly when inference is disabled. The default download catalog is empty.
|
||||
Run `cargo test -p lightcraft-denoise-core`; optional catalog checks use `--features rawnind-model`.
|
||||
@@ -0,0 +1,279 @@
|
||||
//! Taking one file out of a `.zip` (a `.dtmodel` is one), the little of the format a model download needs: stored and
|
||||
//! deflated entries, found by name, checked against their CRC-32 and a size limit, written next to where they are
|
||||
//! going and moved into place. It is a stranger's file, so every length and offset in it is checked, a name is only
|
||||
//! ever compared (never used as a path), and zip64, encrypted and multi-disk archives are refused.
|
||||
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
/// How far from the end the end-of-central-directory record can start (a 22-byte record and a 64 KiB comment).
|
||||
const EOCD_SEARCH: u64 = 22 + 65535;
|
||||
/// Largest central directory we read.
|
||||
const MAX_DIRECTORY: u64 = 1 << 20;
|
||||
|
||||
#[derive(Debug, PartialEq, thiserror::Error)]
|
||||
pub enum ArchiveError {
|
||||
#[error("the archive could not be read: {0}")]
|
||||
Io(String),
|
||||
#[error("not a usable archive: {0}")]
|
||||
Format(String),
|
||||
#[error("the archive has no file called `{0}`")]
|
||||
Missing(String),
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for ArchiveError {
|
||||
fn from(e: std::io::Error) -> Self {
|
||||
ArchiveError::Io(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn format<T>(why: &str) -> Result<T, ArchiveError> {
|
||||
Err(ArchiveError::Format(why.into()))
|
||||
}
|
||||
|
||||
fn u16_at(b: &[u8], at: usize) -> Option<u16> {
|
||||
Some(u16::from_le_bytes(b.get(at..at.checked_add(2)?)?.try_into().ok()?))
|
||||
}
|
||||
|
||||
fn u32_at(b: &[u8], at: usize) -> Option<u32> {
|
||||
Some(u32::from_le_bytes(b.get(at..at.checked_add(4)?)?.try_into().ok()?))
|
||||
}
|
||||
|
||||
/// CRC-32 (IEEE) of `bytes`.
|
||||
fn crc32(bytes: &[u8]) -> u32 {
|
||||
let mut table = [0u32; 256];
|
||||
for (i, t) in table.iter_mut().enumerate() {
|
||||
let mut c = i as u32;
|
||||
for _ in 0..8 {
|
||||
c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 };
|
||||
}
|
||||
*t = c;
|
||||
}
|
||||
!bytes.iter().fold(!0u32, |c, &b| table[((c ^ u32::from(b)) & 0xff) as usize] ^ (c >> 8))
|
||||
}
|
||||
|
||||
struct Entry {
|
||||
method: u16,
|
||||
crc: u32,
|
||||
compressed: u64,
|
||||
size: u64,
|
||||
local_header: u64,
|
||||
}
|
||||
|
||||
/// Find the entry called `name` (exactly, or as the last part of a longer name: `folder/name`) in the central directory.
|
||||
fn find(f: &mut (impl Read + Seek), name: &str) -> Result<Entry, ArchiveError> {
|
||||
let len = f.seek(SeekFrom::End(0))?;
|
||||
let tail_len = len.min(EOCD_SEARCH);
|
||||
f.seek(SeekFrom::Start(len - tail_len))?;
|
||||
let mut tail = vec![0u8; tail_len as usize];
|
||||
f.read_exact(&mut tail)?;
|
||||
let eocd = (0..tail.len().saturating_sub(21)).rev().find(|&i| u32_at(&tail, i) == Some(0x0605_4b50));
|
||||
let Some(e) = eocd else { return format("this is not a zip file") };
|
||||
let (entries, size, offset) = (u16_at(&tail, e + 10), u32_at(&tail, e + 12), u32_at(&tail, e + 16));
|
||||
let (Some(entries), Some(size), Some(offset)) = (entries, size, offset) else { return format("the end of the archive is cut short") };
|
||||
if entries == 0xffff || size == u32::MAX || offset == u32::MAX {
|
||||
return format("zip64 archives are not supported");
|
||||
}
|
||||
if u64::from(size) > MAX_DIRECTORY || u64::from(offset) + u64::from(size) > len {
|
||||
return format("the list of files is not where it says");
|
||||
}
|
||||
f.seek(SeekFrom::Start(u64::from(offset)))?;
|
||||
let mut dir = vec![0u8; size as usize];
|
||||
f.read_exact(&mut dir)?;
|
||||
let mut at = 0usize;
|
||||
for _ in 0..entries {
|
||||
if u32_at(&dir, at) != Some(0x0201_4b50) {
|
||||
return format("the list of files is damaged");
|
||||
}
|
||||
let get = |o: usize| at.checked_add(o);
|
||||
let (flags, method, crc) = (u16_at(&dir, at + 8), u16_at(&dir, at + 10), u32_at(&dir, at + 16));
|
||||
let (compressed, uncompressed) = (u32_at(&dir, at + 20), u32_at(&dir, at + 24));
|
||||
let (n, x, c) = (u16_at(&dir, at + 28), u16_at(&dir, at + 30), u16_at(&dir, at + 32));
|
||||
let local = u32_at(&dir, at + 42);
|
||||
let (Some(flags), Some(method), Some(crc), Some(compressed), Some(uncompressed), Some(n), Some(x), Some(c), Some(local), Some(start)) =
|
||||
(flags, method, crc, compressed, uncompressed, n, x, c, local, get(46))
|
||||
else {
|
||||
return format("the list of files is cut short");
|
||||
};
|
||||
let Some(entry_name) = dir.get(start..start + usize::from(n)) else { return format("a file name is cut short") };
|
||||
let matches = entry_name == name.as_bytes() || entry_name.strip_suffix(name.as_bytes()).is_some_and(|p| p.ends_with(b"/"));
|
||||
if matches {
|
||||
if flags & 1 != 0 {
|
||||
return format("encrypted files are not supported");
|
||||
}
|
||||
if compressed == u32::MAX || uncompressed == u32::MAX {
|
||||
return format("zip64 files are not supported");
|
||||
}
|
||||
return Ok(Entry { method, crc, compressed: u64::from(compressed), size: u64::from(uncompressed), local_header: u64::from(local) });
|
||||
}
|
||||
at = start + usize::from(n) + usize::from(x) + usize::from(c);
|
||||
}
|
||||
Err(ArchiveError::Missing(name.into()))
|
||||
}
|
||||
|
||||
/// Take the file called `name` out of the archive at `archive` and write it to `dest` (through a temporary file next
|
||||
/// to it); `max_bytes` bounds its size. Returns the number of bytes written.
|
||||
pub fn extract(archive: &Path, name: &str, dest: &Path, max_bytes: u64) -> Result<u64, ArchiveError> {
|
||||
let mut f = std::io::BufReader::new(std::fs::File::open(archive)?);
|
||||
let e = find(&mut f, name)?;
|
||||
if e.size > max_bytes || e.compressed > max_bytes {
|
||||
return format("the file in the archive is larger than allowed");
|
||||
}
|
||||
f.seek(SeekFrom::Start(e.local_header))?;
|
||||
let mut head = [0u8; 30];
|
||||
f.read_exact(&mut head)?;
|
||||
if u32_at(&head, 0) != Some(0x0403_4b50) {
|
||||
return format("the file's header is not where it says");
|
||||
}
|
||||
let skip = u64::from(u16_at(&head, 26).unwrap_or(0)) + u64::from(u16_at(&head, 28).unwrap_or(0));
|
||||
f.seek_relative(i64::try_from(skip).map_err(|_| ArchiveError::Format("a header is too long".into()))?)?;
|
||||
let len = usize::try_from(e.compressed)
|
||||
.ok()
|
||||
.filter(|&n| n <= 256 * 1024 * 1024)
|
||||
.ok_or_else(|| ArchiveError::Format("the compressed model is too large".into()))?;
|
||||
if e.size > 256 * 1024 * 1024 {
|
||||
return format("the extracted model is too large");
|
||||
}
|
||||
let mut packed = Vec::new();
|
||||
packed.try_reserve_exact(len).map_err(|_| ArchiveError::Format("not enough memory for the model archive".into()))?;
|
||||
packed.resize(len, 0u8);
|
||||
f.read_exact(&mut packed)?;
|
||||
let data = match e.method {
|
||||
0 => packed,
|
||||
8 => miniz_oxide::inflate::decompress_to_vec_with_limit(&packed, max_bytes.min(256 * 1024 * 1024) as usize)
|
||||
.map_err(|err| ArchiveError::Format(format!("{err:?}")))?,
|
||||
m => return format(&format!("compression method {m} is not supported")),
|
||||
};
|
||||
if data.len() as u64 != e.size || crc32(&data) != e.crc {
|
||||
return format("the file in the archive is damaged");
|
||||
}
|
||||
let tmp = dest.with_extension("part");
|
||||
let written = (|| -> std::io::Result<()> {
|
||||
let mut out = std::fs::File::create(&tmp)?;
|
||||
out.write_all(&data)?;
|
||||
out.flush()?;
|
||||
drop(out);
|
||||
std::fs::rename(&tmp, dest)
|
||||
})();
|
||||
if let Err(err) = written {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
return Err(err.into());
|
||||
}
|
||||
Ok(data.len() as u64)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A zip with the given (name, bytes, deflate) entries, written by hand.
|
||||
pub(crate) fn make_zip(files: &[(&str, &[u8], bool)]) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
let mut dir = Vec::new();
|
||||
for (name, data, deflate) in files {
|
||||
let packed = if *deflate { miniz_oxide::deflate::compress_to_vec(data, 6) } else { data.to_vec() };
|
||||
let offset = out.len() as u32;
|
||||
let method: u16 = if *deflate { 8 } else { 0 };
|
||||
out.extend_from_slice(&0x0403_4b50u32.to_le_bytes());
|
||||
out.extend_from_slice(&[20, 0, 0, 0]);
|
||||
out.extend_from_slice(&method.to_le_bytes());
|
||||
out.extend_from_slice(&[0, 0, 0, 0]);
|
||||
out.extend_from_slice(&crc32(data).to_le_bytes());
|
||||
out.extend_from_slice(&(packed.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(&(data.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&[0, 0]);
|
||||
out.extend_from_slice(name.as_bytes());
|
||||
out.extend_from_slice(&packed);
|
||||
dir.extend_from_slice(&0x0201_4b50u32.to_le_bytes());
|
||||
dir.extend_from_slice(&[20, 0, 20, 0, 0, 0]);
|
||||
dir.extend_from_slice(&method.to_le_bytes());
|
||||
dir.extend_from_slice(&[0, 0, 0, 0]);
|
||||
dir.extend_from_slice(&crc32(data).to_le_bytes());
|
||||
dir.extend_from_slice(&(packed.len() as u32).to_le_bytes());
|
||||
dir.extend_from_slice(&(data.len() as u32).to_le_bytes());
|
||||
dir.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
dir.extend_from_slice(&[0; 12]);
|
||||
dir.extend_from_slice(&offset.to_le_bytes());
|
||||
dir.extend_from_slice(name.as_bytes());
|
||||
}
|
||||
let cd_offset = out.len() as u32;
|
||||
out.extend_from_slice(&dir);
|
||||
out.extend_from_slice(&0x0605_4b50u32.to_le_bytes());
|
||||
out.extend_from_slice(&[0, 0, 0, 0]);
|
||||
out.extend_from_slice(&(files.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(files.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(dir.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(&cd_offset.to_le_bytes());
|
||||
out.extend_from_slice(&[0, 0]);
|
||||
out
|
||||
}
|
||||
|
||||
fn temp(name: &str) -> std::path::PathBuf {
|
||||
std::env::temp_dir().join(format!("lc-denoise-zip-{name}-{}", std::process::id()))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stored_and_a_deflated_file_come_out_whole() {
|
||||
let big: Vec<u8> = (0..200_000u32).map(|i| (i * 7 % 251) as u8).collect();
|
||||
let zip = make_zip(&[("m/config.json", b"{}", false), ("m/model.onnx", &big, true), ("m/other.onnx", b"zz", true)]);
|
||||
let (a, out) = (temp("a.zip"), temp("a.onnx"));
|
||||
std::fs::write(&a, &zip).unwrap();
|
||||
assert_eq!(extract(&a, "model.onnx", &out, 1 << 20).unwrap(), big.len() as u64);
|
||||
assert_eq!(std::fs::read(&out).unwrap(), big);
|
||||
assert_eq!(extract(&a, "m/config.json", &out, 1 << 20).unwrap(), 2);
|
||||
assert_eq!(std::fs::read(&out).unwrap(), b"{}");
|
||||
// a name must match whole path parts: `del.onnx` is not `model.onnx`
|
||||
assert!(matches!(extract(&a, "del.onnx", &out, 1 << 20), Err(ArchiveError::Missing(_))));
|
||||
assert!(matches!(extract(&a, "nothing", &out, 1 << 20), Err(ArchiveError::Missing(_))));
|
||||
// the size limit applies
|
||||
assert!(matches!(extract(&a, "model.onnx", &out, 1000), Err(ArchiveError::Format(_))));
|
||||
for p in [a, out] {
|
||||
let _ = std::fs::remove_file(p);
|
||||
}
|
||||
}
|
||||
|
||||
/// Opt-in: `LC_DENOISE_DTMODEL=<rawdenoise-nind.dtmodel> cargo test -p lightcraft-denoise -- --ignored archive`
|
||||
#[test]
|
||||
#[ignore = "needs the real download: set LC_DENOISE_DTMODEL"]
|
||||
fn the_real_download_gives_the_model_it_should() {
|
||||
let Some(path) = std::env::var_os("LC_DENOISE_DTMODEL") else { return };
|
||||
let known = crate::known::find("rawnind-bayer").unwrap();
|
||||
let (d, entry) = (known.download.unwrap(), known.entry.unwrap());
|
||||
assert_eq!(crate::hash::sha256_file(Path::new(&path)).unwrap(), d.sha256);
|
||||
let out = temp("real.onnx");
|
||||
let n = extract(Path::new(&path), entry, &out, crate::manifest::MAX_MODEL_BYTES).unwrap();
|
||||
assert_eq!(Some(n), known.manifest.size_bytes);
|
||||
assert_eq!(Some(crate::hash::sha256_file(&out).unwrap()), known.manifest.sha256);
|
||||
let _ = std::fs::remove_file(out);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn damaged_and_hostile_archives_are_errors_not_panics() {
|
||||
let zip = make_zip(&[("m/model.onnx", &vec![9u8; 5000], true)]);
|
||||
let (a, out) = (temp("b.zip"), temp("b.onnx"));
|
||||
// cut anywhere
|
||||
for n in (0..zip.len()).step_by(3) {
|
||||
std::fs::write(&a, &zip[..n]).unwrap();
|
||||
assert!(extract(&a, "model.onnx", &out, 1 << 20).is_err(), "cut at {n}");
|
||||
}
|
||||
// one byte changed anywhere: an error, or (when it falls in an unused field) the right file, never a panic
|
||||
for at in 0..zip.len() {
|
||||
let mut bad = zip.clone();
|
||||
bad[at] ^= 0x5a;
|
||||
std::fs::write(&a, &bad).unwrap();
|
||||
if extract(&a, "model.onnx", &out, 1 << 20).is_ok() {
|
||||
assert_eq!(std::fs::read(&out).unwrap(), vec![9u8; 5000], "a changed byte at {at} gave another file");
|
||||
}
|
||||
}
|
||||
// not a zip at all, and an empty file
|
||||
for junk in [Vec::new(), b"PK".to_vec(), vec![0u8; 100_000]] {
|
||||
std::fs::write(&a, junk).unwrap();
|
||||
assert!(extract(&a, "model.onnx", &out, 1 << 20).is_err());
|
||||
}
|
||||
assert!(extract(&temp("missing.zip"), "model.onnx", &out, 1 << 20).is_err());
|
||||
for p in [a, out] {
|
||||
let _ = std::fs::remove_file(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
//! Bayer geometry: which way a mosaic is laid out, and how a patch of it is packed into the four planes
|
||||
//! (red, green on the red row, green on the blue row, blue) a denoise model takes.
|
||||
//!
|
||||
//! Every Bayer layout is the same 2 × 2 cell shifted by one pixel at most, so a layout is just where its
|
||||
//! first red sample is. The cell grid starts one pixel before the picture when that red sample is in
|
||||
//! column or row 1, and the picture is read with its edge mirrored (without repeating the edge sample, so
|
||||
//! the colour of a mirrored sample is the colour that belongs at that place).
|
||||
|
||||
/// Where the first red sample of a Bayer mosaic sits, `(dx, dy)` in `{0, 1}²`: RGGB is `(0, 0)`, GRBG `(1, 0)`,
|
||||
/// GBRG `(0, 1)`, BGGR `(1, 1)`.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Layout {
|
||||
pub dx: usize,
|
||||
pub dy: usize,
|
||||
}
|
||||
|
||||
impl Layout {
|
||||
/// The layout of a 2 × 2 colour pattern, row-major, colours 0 = red, 1 = green, 2 = blue; `None` when
|
||||
/// it is not a Bayer cell (one red, one blue, the greens on the other diagonal).
|
||||
pub fn from_cell(cell: [u8; 4]) -> Option<Layout> {
|
||||
let at = |c: u8| cell.iter().position(|&v| v == c);
|
||||
let red = at(0)?;
|
||||
let blue = at(2)?;
|
||||
let greens = cell.iter().filter(|&&v| v == 1).count();
|
||||
// red and blue sit on one diagonal: indices 0+3 or 1+2
|
||||
if greens != 2 || red + blue != 3 {
|
||||
return None;
|
||||
}
|
||||
Some(Layout { dx: red % 2, dy: red / 2 })
|
||||
}
|
||||
|
||||
/// The cell grid's origin: `-1` along an axis whose first red sample is at 1, so that cell 0 starts on a red column or row.
|
||||
fn origin(self) -> (isize, isize) {
|
||||
(isize::try_from(self.dx).unwrap_or(isize::MAX).saturating_neg(), isize::try_from(self.dy).unwrap_or(isize::MAX).saturating_neg())
|
||||
}
|
||||
|
||||
/// Cells across and down covering a `width × height` mosaic.
|
||||
pub fn cells(self, width: usize, height: usize) -> (usize, usize) {
|
||||
let cells = |n: usize, phase: usize| n / 2 + (n % 2 + phase.min(1)).div_ceil(2);
|
||||
(cells(width, self.dx), cells(height, self.dy))
|
||||
}
|
||||
|
||||
/// The mosaic position of cell `(cx, cy)`'s top-left sample (may lie outside the picture).
|
||||
pub fn position(self, cx: isize, cy: isize) -> (isize, isize) {
|
||||
let (ox, oy) = self.origin();
|
||||
(ox.saturating_add(cx.saturating_mul(2)), oy.saturating_add(cy.saturating_mul(2)))
|
||||
}
|
||||
}
|
||||
|
||||
/// `i` mirrored into `0..n` (ping-pong, edge not repeated), so `-1` is `1` and `n` is `n - 2`. `n == 0` gives 0.
|
||||
pub fn reflect(i: isize, n: usize) -> usize {
|
||||
if n <= 1 {
|
||||
return 0;
|
||||
}
|
||||
if let Some(period) = isize::try_from(n).ok().and_then(|n| (n - 1).checked_mul(2)) {
|
||||
let m = i.rem_euclid(period);
|
||||
return (if m < n as isize { m } else { period - m }) as usize;
|
||||
}
|
||||
let period = 2 * (n as i128 - 1);
|
||||
let m = (i as i128).rem_euclid(period);
|
||||
// The reflected value is in 0..n, so this conversion always fits usize.
|
||||
(if m < n as i128 { m } else { period - m }) as usize
|
||||
}
|
||||
|
||||
/// Pack the `tile × tile` cells whose top-left cell is `(cx0, cy0)` into `out` as four planes `[R, G1, G2, B]`
|
||||
/// (`4 · tile²` values, NCHW). G1 is the green beside the red sample, G2 the green under it. Samples outside
|
||||
/// the picture are mirrored. Returns `false` (and writes nothing) when the sizes do not fit.
|
||||
pub fn pack_tile(mosaic: &[f32], width: usize, height: usize, layout: Layout, cx0: isize, cy0: isize, tile: usize, out: &mut [f32]) -> bool {
|
||||
let Some(plane) = tile.checked_mul(tile) else { return false };
|
||||
let Some(values) = plane.checked_mul(4) else { return false };
|
||||
if width == 0
|
||||
|| height == 0
|
||||
|| tile == 0
|
||||
|| layout.dx > 1
|
||||
|| layout.dy > 1
|
||||
|| width.checked_mul(height) != Some(mosaic.len())
|
||||
|| out.len() != values
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let Ok(last) = isize::try_from(tile - 1) else { return false };
|
||||
let (ox, oy) = layout.origin();
|
||||
for (start, origin) in [(cx0, ox), (cy0, oy)] {
|
||||
if start.checked_mul(2).and_then(|v| v.checked_add(origin)).is_none()
|
||||
|| start.checked_add(last).and_then(|v| v.checked_mul(2)).and_then(|v| v.checked_add(origin)).and_then(|v| v.checked_add(1)).is_none()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// the four samples of a cell, as (dx, dy) from its top-left sample
|
||||
const SUB: [(isize, isize); 4] = [(0, 0), (1, 0), (0, 1), (1, 1)];
|
||||
for (p, &(sx, sy)) in SUB.iter().enumerate() {
|
||||
let Some(dst) = out.get_mut(p * plane..(p + 1) * plane) else { return false };
|
||||
for (j, row) in dst.chunks_exact_mut(tile).enumerate() {
|
||||
let (_, y) = layout.position(cx0, cy0 + j as isize);
|
||||
let ry = reflect(y + sy, height);
|
||||
let Some(src) = mosaic.get(ry * width..(ry + 1) * width) else { return false };
|
||||
for (i, v) in row.iter_mut().enumerate() {
|
||||
let (x, _) = layout.position(cx0 + i as isize, cy0);
|
||||
*v = src.get(reflect(x + sx, width)).copied().unwrap_or(0.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn hostile_sizes_and_positions_never_panic_or_write_output() {
|
||||
let layout = Layout::from_cell([0, 1, 1, 2]).unwrap();
|
||||
let mut out = [7.0; 4];
|
||||
for (w, h, tile, x) in [(usize::MAX, 2, 1, 0), (1, 1, usize::MAX, 0), (1, 1, 0, 0), (1, 1, 1, isize::MAX), (1, 1, 1, isize::MIN)] {
|
||||
assert!(!pack_tile(&[0.5], w, h, layout, x, 0, tile, &mut out));
|
||||
assert_eq!(out, [7.0; 4]);
|
||||
}
|
||||
for n in [1, 2, isize::MAX as usize, usize::MAX] {
|
||||
for i in [isize::MIN, -1, 0, isize::MAX] {
|
||||
assert!(reflect(i, n) < n);
|
||||
}
|
||||
}
|
||||
assert_eq!(layout.cells(usize::MAX, usize::MAX), (usize::MAX / 2 + 1, usize::MAX / 2 + 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn layouts_from_cells() {
|
||||
assert_eq!(Layout::from_cell([0, 1, 1, 2]), Some(Layout { dx: 0, dy: 0 }));
|
||||
assert_eq!(Layout::from_cell([1, 0, 2, 1]), Some(Layout { dx: 1, dy: 0 }));
|
||||
assert_eq!(Layout::from_cell([1, 2, 0, 1]), Some(Layout { dx: 0, dy: 1 }));
|
||||
assert_eq!(Layout::from_cell([2, 1, 1, 0]), Some(Layout { dx: 1, dy: 1 }));
|
||||
// not Bayer: two reds, a missing blue, greens on the wrong diagonal
|
||||
assert_eq!(Layout::from_cell([0, 0, 1, 2]), None);
|
||||
assert_eq!(Layout::from_cell([0, 1, 2, 1]), None);
|
||||
assert_eq!(Layout::from_cell([1, 1, 1, 1]), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reflection_keeps_the_colour_of_a_site() {
|
||||
assert_eq!(reflect(-1, 10), 1);
|
||||
assert_eq!(reflect(-2, 10), 2);
|
||||
assert_eq!(reflect(10, 10), 8);
|
||||
assert_eq!(reflect(11, 10), 7);
|
||||
assert_eq!(reflect(25, 10), 7);
|
||||
assert_eq!(reflect(5, 0), 0);
|
||||
assert_eq!(reflect(5, 1), 0);
|
||||
for i in -40..60 {
|
||||
assert_eq!(reflect(i, 11) % 2, (i.rem_euclid(2)) as usize, "parity at {i}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn packing_puts_each_colour_in_its_plane_for_every_layout() {
|
||||
// a mosaic whose sample value says which colour it is: R = 10, G = 20, B = 30
|
||||
for cell in [[0u8, 1, 1, 2], [1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]] {
|
||||
let layout = Layout::from_cell(cell).unwrap();
|
||||
let (w, h) = (10usize, 8usize);
|
||||
let mosaic: Vec<f32> = (0..w * h).map(|i| [10.0, 20.0, 30.0][cell[(i / w % 2) * 2 + i % w % 2] as usize]).collect();
|
||||
let (nx, ny) = layout.cells(w, h);
|
||||
let tile = nx.max(ny);
|
||||
let mut out = vec![0.0; 4 * tile * tile];
|
||||
assert!(pack_tile(&mosaic, w, h, layout, 0, 0, tile, &mut out));
|
||||
let plane = tile * tile;
|
||||
for (p, want) in [10.0, 20.0, 20.0, 30.0].into_iter().enumerate() {
|
||||
assert!(out[p * plane..(p + 1) * plane].iter().all(|&v| v == want), "layout {cell:?} plane {p}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn g1_is_beside_the_red_sample_and_g2_under_it() {
|
||||
// RGGB with a distinct value at every site
|
||||
let (w, h) = (4, 4);
|
||||
let mosaic: Vec<f32> = (0..16).map(|i| i as f32).collect();
|
||||
let mut out = vec![0.0; 4 * 4];
|
||||
assert!(pack_tile(&mosaic, w, h, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out));
|
||||
// cell (0,0): R = site (0,0)=0, G1 = site (1,0)=1, G2 = site (0,1)=4, B = site (1,1)=5
|
||||
assert_eq!([out[0], out[4], out[8], out[12]], [0.0, 1.0, 4.0, 5.0]);
|
||||
// cell (1,0): sites (2,0)=2, (3,0)=3, (2,1)=6, (3,1)=7
|
||||
assert_eq!([out[1], out[5], out[9], out[13]], [2.0, 3.0, 6.0, 7.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn packing_rejects_wrong_sizes() {
|
||||
let mut out = vec![0.0; 16];
|
||||
assert!(!pack_tile(&[0.0; 15], 4, 4, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out));
|
||||
assert!(!pack_tile(&[0.0; 16], 4, 4, Layout { dx: 0, dy: 0 }, 0, 0, 3, &mut out));
|
||||
assert!(!pack_tile(&[], 0, 0, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
//! SHA-256 of a model file, streamed (a model can be hundreds of megabytes).
|
||||
|
||||
use std::io::{self, Read};
|
||||
use std::path::Path;
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// Lowercase hex SHA-256 of everything `r` yields.
|
||||
pub fn sha256_reader<R: Read>(mut r: R) -> io::Result<String> {
|
||||
let mut h = Sha256::new();
|
||||
let mut buf = vec![0u8; 1 << 20];
|
||||
loop {
|
||||
let n = r.read(&mut buf)?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
h.update(buf.get(..n).unwrap_or(&[]));
|
||||
}
|
||||
Ok(hex(&h.finalize()))
|
||||
}
|
||||
|
||||
/// Lowercase hex SHA-256 of a file.
|
||||
pub fn sha256_file(path: &Path) -> io::Result<String> {
|
||||
sha256_reader(io::BufReader::new(std::fs::File::open(path)?))
|
||||
}
|
||||
|
||||
/// Lowercase hex SHA-256 of `bytes`.
|
||||
pub fn sha256_hex(bytes: &[u8]) -> String {
|
||||
hex(&Sha256::digest(bytes))
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
bytes.iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn known_digests() {
|
||||
assert_eq!(sha256_hex(b""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
|
||||
assert_eq!(sha256_hex(b"abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
|
||||
assert_eq!(sha256_reader(&b"abc"[..]).unwrap(), sha256_hex(b"abc"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
//! The denoise models LightCraft knows about: what each one is, who may use it, and where to fetch it.
|
||||
//!
|
||||
//! None is part of LightCraft. A model with an address here can be downloaded when the user asks and has accepted
|
||||
//! its terms; anything else is brought by the user as an `.onnx` file with a `denoise-model.json` beside it.
|
||||
|
||||
#[cfg(any(feature = "rawnind-model", test))]
|
||||
use crate::licence::{Commercial, Licence};
|
||||
|
||||
use crate::manifest::DenoiserManifest;
|
||||
#[cfg(any(feature = "rawnind-model", test))]
|
||||
use crate::manifest::{Domain, Gain};
|
||||
|
||||
/// A known model, and how to get it.
|
||||
pub struct Known {
|
||||
pub manifest: DenoiserManifest,
|
||||
/// What to fetch (an archive or the `.onnx` itself); `None` when the user must bring the file.
|
||||
pub download: Option<Download>,
|
||||
/// The `.onnx` inside the downloaded archive (`None`: the download is the `.onnx`).
|
||||
pub entry: Option<&'static str>,
|
||||
}
|
||||
|
||||
/// RawNIND's UtNet2 for Bayer sensors, as published for darktable (`rawdenoise-nind.dtmodel`, a zip with a Bayer and a
|
||||
/// linear model). Its weights are GPL-3.0; the training pictures are CC BY 4.0 and CC0.
|
||||
#[cfg(any(feature = "rawnind-model", test))]
|
||||
fn rawnind_bayer() -> Known {
|
||||
let manifest = DenoiserManifest {
|
||||
id: "rawnind-bayer".into(),
|
||||
name: "RawNIND UtNet2 (Bayer)".into(),
|
||||
version: "1.0".into(),
|
||||
licence: Licence {
|
||||
name: "GPL-3.0".into(),
|
||||
commercial: Commercial::Yes,
|
||||
url: Some("https://www.gnu.org/licenses/gpl-3.0.html".into()),
|
||||
notice: "The weights are published under the GNU General Public License v3. LightCraft does not include them: they \
|
||||
are downloaded to your computer at your request and used by it only. If you pass the model file on, the \
|
||||
GPL applies to that copy. Commercial use is subject to the GPL terms, including applicable source, licence and notice obligations on redistribution."
|
||||
.into(),
|
||||
},
|
||||
source: Some("https://github.com/darktable-org/darktable-ai/tree/master/models/rawdenoise-nind".into()),
|
||||
sha256: Some("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901".into()),
|
||||
size_bytes: Some(31_056_425),
|
||||
provenance: "Trained on RawNIND, real photographs of the same scenes taken noisy and clean (images CC BY 4.0 and CC0, \
|
||||
published on Wikimedia Commons and by the University of Louvain). Paper: arXiv 2501.08924."
|
||||
.into(),
|
||||
domain: Domain::BayerToRgb,
|
||||
tile: 512,
|
||||
overlap: 64,
|
||||
gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },
|
||||
};
|
||||
let download = Download {
|
||||
id: manifest.id.clone(),
|
||||
url: "https://github.com/darktable-org/darktable-ai/releases/download/release-5.6.0/rawdenoise-nind.dtmodel".into(),
|
||||
file_name: "rawdenoise-nind.dtmodel".into(),
|
||||
size_bytes: 57_700_134,
|
||||
sha256: "d71b5f1e727c85a359e6f74dca9e2016c9d8fc3e2f7ac3e9b347d80ceca969af".into(),
|
||||
};
|
||||
Known { manifest, download: Some(download), entry: Some("model_bayer.onnx") }
|
||||
}
|
||||
|
||||
/// Every model LightCraft knows.
|
||||
pub fn all() -> Vec<Known> {
|
||||
#[cfg(feature = "rawnind-model")]
|
||||
{
|
||||
vec![rawnind_bayer()]
|
||||
}
|
||||
#[cfg(not(feature = "rawnind-model"))]
|
||||
{
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// The known model with this id.
|
||||
pub fn find(id: &str) -> Option<Known> {
|
||||
all().into_iter().find(|k| k.manifest.id == id)
|
||||
}
|
||||
|
||||
/// The known model whose `.onnx` has this SHA-256.
|
||||
pub fn by_sha256(sha256: &str) -> Option<DenoiserManifest> {
|
||||
all().into_iter().map(|k| k.manifest).find(|m| m.sha256.as_deref() == Some(sha256))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::manifest::validate;
|
||||
|
||||
#[test]
|
||||
fn known_models_are_valid_unique_and_downloaded_over_https() {
|
||||
let all = all();
|
||||
assert_eq!(all.is_empty(), !cfg!(feature = "rawnind-model"));
|
||||
validate(&rawnind_bayer().manifest).unwrap();
|
||||
for (i, k) in all.iter().enumerate() {
|
||||
validate(&k.manifest).unwrap();
|
||||
assert!(all.iter().skip(i + 1).all(|o| o.manifest.id != k.manifest.id), "unique ids");
|
||||
assert!(!k.manifest.licence.name.is_empty() && !k.manifest.licence.notice.is_empty(), "terms are shown before use");
|
||||
if let Some(d) = &k.download {
|
||||
assert_eq!(d.id, k.manifest.id);
|
||||
assert!(d.url.starts_with("https://") && d.sha256.len() == 64 && d.size_bytes > 0);
|
||||
// a name that cannot leave the staging folder
|
||||
assert!(!d.file_name.contains(['/', '\\']) && !d.file_name.starts_with('.'));
|
||||
}
|
||||
// a model that may not be used commercially is never offered for download
|
||||
if k.manifest.licence.commercial == Commercial::No {
|
||||
assert!(k.download.is_none(), "{}", k.manifest.id);
|
||||
}
|
||||
}
|
||||
assert_eq!(find("rawnind-bayer").is_some(), cfg!(feature = "rawnind-model"));
|
||||
assert!(find("nothing").is_none());
|
||||
assert_eq!(
|
||||
by_sha256("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901").map(|m| m.id),
|
||||
if cfg!(feature = "rawnind-model") { Some("rawnind-bayer".into()) } else { None }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Download {
|
||||
pub id: String,
|
||||
pub url: String,
|
||||
/// The name the file is kept under while it waits for the user to accept its terms.
|
||||
pub file_name: String,
|
||||
pub size_bytes: u64,
|
||||
pub sha256: String,
|
||||
}
|
||||
|
||||
pub fn host(url: &str) -> &str {
|
||||
url.strip_prefix("https://").and_then(|r| r.split('/').next()).unwrap_or("")
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
//! Model metadata, bounded archive/cache formats and Bayer tiling, independent of inference.
|
||||
//! Used by featureless engine and web builds; no neural-network or matrix backend is linked.
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
pub mod archive;
|
||||
pub mod bayer;
|
||||
pub mod hash;
|
||||
pub mod known;
|
||||
pub mod licence;
|
||||
pub mod manifest;
|
||||
pub mod product;
|
||||
pub mod run;
|
||||
pub mod tiles;
|
||||
|
||||
pub use manifest::{DenoiserManifest, Domain, Gain, ManifestError};
|
||||
pub use run::{Control, Error, Params, TileRunner, denoise_bayer};
|
||||
@@ -0,0 +1,34 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Commercial {
|
||||
Yes,
|
||||
/// Research or personal use only. Never bundled or redistributed by LightCraft.
|
||||
No,
|
||||
/// The licence or the training data is unclear: the user is told so before installing.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct Licence {
|
||||
/// SPDX id or a short name ("MIT", "Apache-2.0", "InsightFace non-commercial research").
|
||||
pub name: String,
|
||||
pub commercial: Commercial,
|
||||
pub url: Option<String>,
|
||||
/// Plain-language terms shown before the model is enabled.
|
||||
pub notice: String,
|
||||
}
|
||||
|
||||
impl Default for Licence {
|
||||
fn default() -> Self {
|
||||
Self { name: String::new(), commercial: Commercial::Unknown, url: None, notice: String::new() }
|
||||
}
|
||||
}
|
||||
|
||||
pub fn valid_id(id: &str) -> bool {
|
||||
let mut bytes = id.bytes();
|
||||
let first_ok = bytes.next().is_some_and(|b| b.is_ascii_lowercase() || b.is_ascii_digit());
|
||||
first_ok && id.len() <= 64 && bytes.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || matches!(b, b'.' | b'_' | b'-'))
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
//! A denoise model's manifest: what the model takes and gives, how big its tiles are and who may use it, as
|
||||
//! plain data.
|
||||
//!
|
||||
//! It comes from a `denoise-model.json` next to the `.onnx` file or from the user's `catalog.json`. Both are
|
||||
//! untrusted: [`validate`] must pass before a manifest is used, and [`parse`] applies it after reading JSON of
|
||||
//! bounded size. The licence, id and address rules are the face models' (`crate::licence`).
|
||||
|
||||
use crate::licence::{Licence, valid_id};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Largest `denoise-model.json` we read.
|
||||
pub const MAX_MANIFEST_BYTES: usize = 64 * 1024;
|
||||
/// Largest model file we accept (bytes).
|
||||
pub const MAX_MODEL_BYTES: u64 = 2 * 1024 * 1024 * 1024;
|
||||
|
||||
/// What the model takes and what it gives.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Domain {
|
||||
/// The sensor's Bayer mosaic (black subtracted, white at 1, not white balanced) packed as four planes
|
||||
/// `[R, G1, G2, B]` of its 2 × 2 cells, forced to RGGB; gives camera RGB at the mosaic's resolution, so
|
||||
/// the picture is denoised and demosaiced in one step and must not be demosaiced again.
|
||||
BayerToRgb,
|
||||
}
|
||||
|
||||
/// How the model's output scale is brought back to the input's.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "camelCase")]
|
||||
pub enum Gain {
|
||||
/// The output already has the input's scale.
|
||||
None,
|
||||
/// The output has a scale of its own, `nominal` times the input's (a million, for one model) and a little
|
||||
/// different from tile to tile: each tile is scaled so that its channel means equal the input's. A tile too
|
||||
/// dark to measure, or whose measured scale is further than `maxDeviation` (a fraction, 0.05 = 5 %) from
|
||||
/// `nominal`, is scaled as if it were within that limit.
|
||||
MatchMean { nominal: f32, max_deviation: f32 },
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DenoiserManifest {
|
||||
/// Lowercase letters, digits, `.` `_` `-`.
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub version: String,
|
||||
#[serde(default)]
|
||||
pub licence: Licence,
|
||||
/// Where the weights come from (an `https://` page or file), for the user's information.
|
||||
#[serde(default)]
|
||||
pub source: Option<String>,
|
||||
/// Lowercase hex SHA-256 of the `.onnx` file, when known.
|
||||
#[serde(default)]
|
||||
pub sha256: Option<String>,
|
||||
#[serde(default)]
|
||||
pub size_bytes: Option<u64>,
|
||||
/// What the model was trained on, in plain words; says "undisclosed" when it is.
|
||||
#[serde(default)]
|
||||
pub provenance: String,
|
||||
pub domain: Domain,
|
||||
/// Side of the square the model runs on, in packed cells (the file's fixed input size).
|
||||
pub tile: u32,
|
||||
/// Cells neighbouring tiles share; the picture is blended across them.
|
||||
pub overlap: u32,
|
||||
pub gain: Gain,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, thiserror::Error)]
|
||||
pub enum ManifestError {
|
||||
#[error("`{0}` is not valid: {1}")]
|
||||
Field(&'static str, String),
|
||||
#[error("not a denoise-model manifest: {0}")]
|
||||
Parse(String),
|
||||
#[error("a manifest may be at most {MAX_MANIFEST_BYTES} bytes")]
|
||||
TooLarge,
|
||||
}
|
||||
|
||||
fn bad<T>(field: &'static str, why: impl Into<String>) -> Result<T, ManifestError> {
|
||||
Err(ManifestError::Field(field, why.into()))
|
||||
}
|
||||
|
||||
fn text(field: &'static str, s: &str, min: usize, max: usize) -> Result<(), ManifestError> {
|
||||
let n = s.chars().count();
|
||||
if n < min || n > max {
|
||||
return bad(field, format!("must be {min} to {max} characters"));
|
||||
}
|
||||
if s.chars().any(|c| c.is_control() && c != '\n') {
|
||||
return bad(field, "must not contain control characters");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn url(field: &'static str, u: &Option<String>) -> Result<(), ManifestError> {
|
||||
match u {
|
||||
None => Ok(()),
|
||||
Some(u) if u.len() <= 500 && u.starts_with("https://") && !u.chars().any(|c| c.is_control() || c.is_whitespace()) => Ok(()),
|
||||
Some(_) => bad(field, "must be an https:// address of at most 500 characters"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check a manifest from outside before trusting it.
|
||||
pub fn validate(m: &DenoiserManifest) -> Result<(), ManifestError> {
|
||||
if !valid_id(&m.id) {
|
||||
return bad("id", "1 to 64 lowercase letters, digits, `.`, `_` or `-`, starting with a letter or digit");
|
||||
}
|
||||
text("name", &m.name, 1, 80)?;
|
||||
text("version", &m.version, 1, 40)?;
|
||||
text("provenance", &m.provenance, 0, 1000)?;
|
||||
text("licence.name", &m.licence.name, 0, 80)?;
|
||||
text("licence.notice", &m.licence.notice, 0, 2000)?;
|
||||
url("licence.url", &m.licence.url)?;
|
||||
url("source", &m.source)?;
|
||||
if let Some(h) = &m.sha256
|
||||
&& !(h.len() == 64 && h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))
|
||||
{
|
||||
return bad("sha256", "must be 64 lowercase hex digits");
|
||||
}
|
||||
if let Some(n) = m.size_bytes
|
||||
&& (n == 0 || n > MAX_MODEL_BYTES)
|
||||
{
|
||||
return bad("sizeBytes", format!("must be between 1 and {MAX_MODEL_BYTES}"));
|
||||
}
|
||||
// a U-Net with four poolings wants a multiple of 16; nothing smaller than 64 or larger than 2048 is a tile
|
||||
if !(64..=2048).contains(&m.tile) || !m.tile.is_multiple_of(16) {
|
||||
return bad("tile", "must be a multiple of 16 from 64 to 2048");
|
||||
}
|
||||
// tiles must overlap by less than half, or one tile's blend ramps would run into its other neighbour's
|
||||
if !m.overlap.is_multiple_of(2) || m.overlap < 16 || m.overlap.saturating_mul(2) >= m.tile {
|
||||
return bad("overlap", "must be even, at least 16 and less than half the tile");
|
||||
}
|
||||
if let Gain::MatchMean { nominal, max_deviation } = m.gain {
|
||||
if !(nominal.is_finite() && (1e-6..=1e12).contains(&nominal)) {
|
||||
return bad("gain.nominal", "must be between 0.000001 and 1000000000000");
|
||||
}
|
||||
if !(max_deviation.is_finite() && (0.0..=0.5).contains(&max_deviation)) {
|
||||
return bad("gain.maxDeviation", "must be between 0 and 0.5");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read and validate a `denoise-model.json`.
|
||||
pub fn parse(bytes: &[u8]) -> Result<DenoiserManifest, ManifestError> {
|
||||
if bytes.len() > MAX_MANIFEST_BYTES {
|
||||
return Err(ManifestError::TooLarge);
|
||||
}
|
||||
let bytes = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]).unwrap_or(bytes);
|
||||
let m: DenoiserManifest = serde_json::from_slice(bytes).map_err(|e| ManifestError::Parse(e.to_string()))?;
|
||||
validate(&m)?;
|
||||
Ok(m)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
pub(crate) fn sample() -> DenoiserManifest {
|
||||
DenoiserManifest {
|
||||
id: "nind-bayer".into(),
|
||||
name: "Bayer denoiser".into(),
|
||||
version: "1".into(),
|
||||
licence: Licence::default(),
|
||||
source: Some("https://example.org/model.onnx".into()),
|
||||
sha256: None,
|
||||
size_bytes: Some(31_000_000),
|
||||
provenance: "trained on real raw noise pairs".into(),
|
||||
domain: Domain::BayerToRgb,
|
||||
tile: 512,
|
||||
overlap: 64,
|
||||
gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_good_manifest_round_trips() {
|
||||
let m = sample();
|
||||
let json = serde_json::to_vec(&m).unwrap();
|
||||
assert_eq!(parse(&json).unwrap(), m);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_fields_are_named() {
|
||||
let cases: Vec<(&str, Box<dyn Fn(&mut DenoiserManifest)>)> = vec![
|
||||
("id", Box::new(|m| m.id = "../evil".into())),
|
||||
("id", Box::new(|m| m.id = String::new())),
|
||||
("name", Box::new(|m| m.name = "x\u{7}".into())),
|
||||
("source", Box::new(|m| m.source = Some("http://insecure".into()))),
|
||||
("sha256", Box::new(|m| m.sha256 = Some("XYZ".into()))),
|
||||
("sizeBytes", Box::new(|m| m.size_bytes = Some(0))),
|
||||
("tile", Box::new(|m| m.tile = 500)),
|
||||
("tile", Box::new(|m| m.tile = 4096)),
|
||||
("overlap", Box::new(|m| m.overlap = 300)),
|
||||
("overlap", Box::new(|m| m.overlap = 63)),
|
||||
("gain.maxDeviation", Box::new(|m| m.gain = Gain::MatchMean { nominal: 1.0, max_deviation: f32::NAN })),
|
||||
("gain.maxDeviation", Box::new(|m| m.gain = Gain::MatchMean { nominal: 1.0, max_deviation: 2.0 })),
|
||||
("gain.nominal", Box::new(|m| m.gain = Gain::MatchMean { nominal: 0.0, max_deviation: 0.1 })),
|
||||
("gain.nominal", Box::new(|m| m.gain = Gain::MatchMean { nominal: f32::INFINITY, max_deviation: 0.1 })),
|
||||
];
|
||||
for (field, edit) in cases {
|
||||
let mut m = sample();
|
||||
edit(&mut m);
|
||||
match validate(&m) {
|
||||
Err(ManifestError::Field(f, _)) => assert_eq!(f, field),
|
||||
other => panic!("{field}: {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_is_bounded_and_tolerant() {
|
||||
assert_eq!(parse(&vec![b' '; MAX_MANIFEST_BYTES + 1]), Err(ManifestError::TooLarge));
|
||||
assert!(matches!(parse(b"{ nope"), Err(ManifestError::Parse(_))));
|
||||
assert!(matches!(parse(b""), Err(ManifestError::Parse(_))));
|
||||
let mut with_bom = vec![0xEF, 0xBB, 0xBF];
|
||||
with_bom.extend(serde_json::to_vec(&sample()).unwrap());
|
||||
assert!(parse(&with_bom).is_ok());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,616 @@
|
||||
//! The denoised picture as cached data, and the blend the Amount slider controls.
|
||||
//!
|
||||
//! A product is camera RGB at the sensor's resolution (what the model gave, before the raw's own opcodes and
|
||||
//! crop), kept as half floats in strips of rows, each strip byte-shuffled and deflated with its own checksum.
|
||||
//! It is derived data: any problem reading it (missing, truncated, corrupt, made for another photo or model) is
|
||||
//! just a cache miss, never a panic, and the picture can be made again.
|
||||
//!
|
||||
//! File: `LCDN`, version, header length, header JSON (`width`, `height`, `strip`, `key`), then for each strip its
|
||||
//! compressed length and CRC-32, then the strips' bytes. `key` says what the product was made from (the photo's
|
||||
//! content, the model, this code's version, the settings it depends on); a file with another key is stale.
|
||||
|
||||
use std::io::{Read, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use lightcraft_raster::Rgb32f;
|
||||
use rayon::prelude::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
const MAGIC: &[u8; 4] = b"LCDN";
|
||||
const VERSION: u8 = 1;
|
||||
const MAX_HEADER: usize = 4096;
|
||||
const MAX_SIDE: usize = 1 << 17;
|
||||
/// Most pixels a product may have (the denoiser's own limit, [`crate::run::MAX_PIXELS`]).
|
||||
const MAX_PIXELS: usize = crate::run::MAX_PIXELS;
|
||||
/// Rows per strip: about a megapixel of three planes at 24 MP widths.
|
||||
const STRIP_ROWS: usize = 128;
|
||||
/// Bumped when the meaning of a product changes (included in keys by the caller).
|
||||
pub const ALGORITHM: u32 = 1;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ProductError {
|
||||
#[error("the denoised picture could not be read or written: {0}")]
|
||||
Io(String),
|
||||
#[error("not a usable denoised picture: {0}")]
|
||||
Format(String),
|
||||
/// A product made from something else (another photo state, model or version).
|
||||
#[error("the denoised picture is out of date")]
|
||||
Stale,
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for ProductError {
|
||||
fn from(e: std::io::Error) -> Self {
|
||||
ProductError::Io(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Header {
|
||||
pub width: usize,
|
||||
pub height: usize,
|
||||
pub strip: usize,
|
||||
pub key: String,
|
||||
}
|
||||
|
||||
fn allocate<T: Clone>(n: usize, value: T) -> Result<Vec<T>, ProductError> {
|
||||
let mut v = Vec::new();
|
||||
v.try_reserve_exact(n).map_err(|_| ProductError::Format("not enough memory for the denoised picture".into()))?;
|
||||
v.resize(n, value);
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// `f32` as IEEE half-float bits, rounded to nearest even; beyond the range it saturates, NaN becomes 0.
|
||||
pub fn to_f16(v: f32) -> u16 {
|
||||
if !v.is_finite() {
|
||||
return if v.is_nan() {
|
||||
0
|
||||
} else if v > 0.0 {
|
||||
0x7bff
|
||||
} else {
|
||||
0xfbff
|
||||
};
|
||||
}
|
||||
let b = v.to_bits();
|
||||
let sign = ((b >> 16) & 0x8000) as u16;
|
||||
let exp = ((b >> 23) & 0xff) as i32 - 127 + 15;
|
||||
let man = b & 0x7f_ffff;
|
||||
if exp >= 31 {
|
||||
return sign | 0x7bff;
|
||||
}
|
||||
if exp <= 0 {
|
||||
if exp < -10 {
|
||||
return sign;
|
||||
}
|
||||
let man = man | 0x80_0000;
|
||||
let shift = (14 - exp) as u32;
|
||||
let (half, rem, halfway) = (man >> shift, man & ((1u32 << shift) - 1), 1u32 << (shift - 1));
|
||||
let mut h = half as u16;
|
||||
if rem > halfway || (rem == halfway && h & 1 == 1) {
|
||||
h += 1;
|
||||
}
|
||||
return sign | h;
|
||||
}
|
||||
let mut h = (((exp as u32) << 10) | (man >> 13)) as u16;
|
||||
let rem = man & 0x1fff;
|
||||
if rem > 0x1000 || (rem == 0x1000 && h & 1 == 1) {
|
||||
h += 1;
|
||||
}
|
||||
if h >= 0x7c00 {
|
||||
h = 0x7bff;
|
||||
}
|
||||
sign | h
|
||||
}
|
||||
|
||||
/// Half-float bits as `f32`.
|
||||
pub fn from_f16(h: u16) -> f32 {
|
||||
let sign = if h & 0x8000 != 0 { -1.0f32 } else { 1.0 };
|
||||
let exp = i32::from((h >> 10) & 0x1f);
|
||||
let man = f32::from(h & 0x3ff);
|
||||
match exp {
|
||||
0 => sign * man * 2f32.powi(-24),
|
||||
31 => sign * 65504.0,
|
||||
_ => sign * (1.0 + man / 1024.0) * 2f32.powi(exp - 15),
|
||||
}
|
||||
}
|
||||
|
||||
fn crc(bytes: &[u8]) -> u32 {
|
||||
// CRC-32 (IEEE), bitwise: strips are small and this runs once per strip
|
||||
let mut table = [0u32; 256];
|
||||
for (i, t) in table.iter_mut().enumerate() {
|
||||
let mut c = i as u32;
|
||||
for _ in 0..8 {
|
||||
c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 };
|
||||
}
|
||||
*t = c;
|
||||
}
|
||||
!bytes.iter().fold(!0u32, |c, &b| table[((c ^ u32::from(b)) & 0xff) as usize] ^ (c >> 8))
|
||||
}
|
||||
|
||||
/// One strip as bytes: for each plane (R, G, B) the low bytes of its half floats, then the high bytes.
|
||||
fn encode_strip(img: &Rgb32f, y0: usize, y1: usize) -> Result<Vec<u8>, ProductError> {
|
||||
let n = (y1 - y0) * img.width;
|
||||
let mut raw = allocate(6 * n, 0u8)?;
|
||||
let rows = img.data.get(y0 * img.width..y1 * img.width).unwrap_or(&[]);
|
||||
for (i, px) in rows.iter().enumerate() {
|
||||
for (c, &v) in px.iter().enumerate() {
|
||||
let h = to_f16(v);
|
||||
if let Some(lo) = raw.get_mut(2 * c * n + i) {
|
||||
*lo = h as u8;
|
||||
}
|
||||
if let Some(hi) = raw.get_mut((2 * c + 1) * n + i) {
|
||||
*hi = (h >> 8) as u8;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(miniz_oxide::deflate::compress_to_vec(&raw, 3))
|
||||
}
|
||||
|
||||
/// Write `img` to `path` (through a temporary file next to it, so a reader never sees half a product).
|
||||
pub fn write(path: &Path, img: &Rgb32f, key: &str) -> Result<(), ProductError> {
|
||||
let pixels = img.width.checked_mul(img.height).filter(|&n| n <= MAX_PIXELS);
|
||||
if img.width == 0 || img.height == 0 || img.width > MAX_SIDE || img.height > MAX_SIDE || pixels != Some(img.data.len()) {
|
||||
return Err(ProductError::Format("the picture has no usable size".into()));
|
||||
}
|
||||
let strips: Vec<(usize, usize)> = (0..img.height).step_by(STRIP_ROWS).map(|y| (y, (y + STRIP_ROWS).min(img.height))).collect();
|
||||
let packed: Vec<Vec<u8>> = strips.par_iter().map(|&(a, b)| encode_strip(img, a, b)).collect::<Result<_, _>>()?;
|
||||
let header = serde_json::to_vec(&Header { width: img.width, height: img.height, strip: STRIP_ROWS, key: key.to_string() })
|
||||
.map_err(|e| ProductError::Format(e.to_string()))?;
|
||||
let tmp = path.with_extension("tmp");
|
||||
let result = (|| -> Result<(), ProductError> {
|
||||
if let Some(dir) = path.parent() {
|
||||
std::fs::create_dir_all(dir)?;
|
||||
}
|
||||
let mut f = std::io::BufWriter::new(std::fs::File::create(&tmp)?);
|
||||
f.write_all(MAGIC)?;
|
||||
f.write_all(&[VERSION, 0, 0, 0])?;
|
||||
f.write_all(&(header.len() as u32).to_le_bytes())?;
|
||||
f.write_all(&header)?;
|
||||
for p in &packed {
|
||||
f.write_all(&(p.len() as u32).to_le_bytes())?;
|
||||
f.write_all(&crc(p).to_le_bytes())?;
|
||||
}
|
||||
for p in &packed {
|
||||
f.write_all(p)?;
|
||||
}
|
||||
f.flush()?;
|
||||
drop(f);
|
||||
std::fs::rename(&tmp, path)?;
|
||||
Ok(())
|
||||
})();
|
||||
if result.is_err() {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// The parsed start of a product file: header, each strip's (compressed length, CRC-32), and the file length the
|
||||
/// start says the whole file has.
|
||||
struct Start {
|
||||
header: Header,
|
||||
table: Vec<(usize, u32)>,
|
||||
expected_len: u64,
|
||||
}
|
||||
|
||||
fn read_start(r: &mut impl Read) -> Result<Start, ProductError> {
|
||||
let mut fixed = [0u8; 12];
|
||||
r.read_exact(&mut fixed)?;
|
||||
if fixed.get(0..4) != Some(&MAGIC[..]) || fixed.get(4) != Some(&VERSION) {
|
||||
return Err(ProductError::Format("not a denoised picture of this version".into()));
|
||||
}
|
||||
let len = fixed.get(8..12).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(u32::from_le_bytes).unwrap_or(u32::MAX) as usize;
|
||||
if len > MAX_HEADER {
|
||||
return Err(ProductError::Format("the header is too large".into()));
|
||||
}
|
||||
let mut hb = allocate(len, 0u8)?;
|
||||
r.read_exact(&mut hb)?;
|
||||
let h: Header = serde_json::from_slice(&hb).map_err(|e| ProductError::Format(e.to_string()))?;
|
||||
let pixels = h.width.saturating_mul(h.height);
|
||||
if h.width == 0 || h.height == 0 || h.width > MAX_SIDE || h.height > MAX_SIDE || pixels > MAX_PIXELS || h.strip != STRIP_ROWS {
|
||||
return Err(ProductError::Format("the size is not usable".into()));
|
||||
}
|
||||
let n = h.height.div_ceil(h.strip);
|
||||
let mut table = Vec::with_capacity(n);
|
||||
let mut total = 12 + len as u64 + 8 * n as u64;
|
||||
for _ in 0..n {
|
||||
let mut e = [0u8; 8];
|
||||
r.read_exact(&mut e)?;
|
||||
let l = u32::from_le_bytes([e[0], e[1], e[2], e[3]]) as usize;
|
||||
let c = u32::from_le_bytes([e[4], e[5], e[6], e[7]]);
|
||||
// a strip never compresses to much more than its raw size
|
||||
if l > 12 * h.strip * h.width + 4096 {
|
||||
return Err(ProductError::Format("a strip is larger than it can be".into()));
|
||||
}
|
||||
total += l as u64;
|
||||
table.push((l, c));
|
||||
}
|
||||
Ok(Start { header: h, table, expected_len: total })
|
||||
}
|
||||
|
||||
/// The header of the product at `path`, when it is one.
|
||||
pub fn read_header_at(path: &Path) -> Result<Header, ProductError> {
|
||||
let mut f = std::io::BufReader::new(std::fs::File::open(path)?);
|
||||
Ok(read_start(&mut f)?.header)
|
||||
}
|
||||
|
||||
/// Whether `path` holds a complete-looking product made from `key` (the header and the file's length are checked;
|
||||
/// strips are checked when read).
|
||||
pub fn is_current(path: &Path, key: &str) -> bool {
|
||||
let check = || -> Result<bool, ProductError> {
|
||||
let mut f = std::io::BufReader::new(std::fs::File::open(path)?);
|
||||
let s = read_start(&mut f)?;
|
||||
Ok(s.header.key == key && std::fs::metadata(path)?.len() == s.expected_len)
|
||||
};
|
||||
check().unwrap_or(false)
|
||||
}
|
||||
|
||||
/// A rectangle of a product, in the product's pixels.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Window {
|
||||
pub x: usize,
|
||||
pub y: usize,
|
||||
pub width: usize,
|
||||
pub height: usize,
|
||||
}
|
||||
|
||||
/// Half floats as `f32`, for all 65536 bit patterns (a table is much faster than computing each).
|
||||
fn f16_table() -> &'static [f32] {
|
||||
static TABLE: std::sync::OnceLock<Vec<f32>> = std::sync::OnceLock::new();
|
||||
TABLE.get_or_init(|| (0..=u16::MAX).map(from_f16).collect())
|
||||
}
|
||||
|
||||
/// Strips decoded together (they are inflated in parallel, then folded in order).
|
||||
const GROUP: usize = 8;
|
||||
|
||||
/// One strip, checked and inflated: for each plane the low bytes of its half floats, then the high bytes.
|
||||
fn decode_strip(index: usize, packed: &[u8], expect_crc: u32, n: usize) -> Result<Vec<u8>, ProductError> {
|
||||
if crc(packed) != expect_crc {
|
||||
return Err(ProductError::Format(format!("strip {index} is damaged")));
|
||||
}
|
||||
let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(packed, 6 * n).map_err(|e| ProductError::Format(format!("strip {index}: {e:?}")))?;
|
||||
if raw.len() != 6 * n {
|
||||
return Err(ProductError::Format(format!("strip {index} has the wrong size")));
|
||||
}
|
||||
Ok(raw)
|
||||
}
|
||||
|
||||
/// Read the product at `path`, made from `key`, with every `factor × factor` block of pixels averaged into one
|
||||
/// (`factor` 1 = full size). The result is camera RGB.
|
||||
///
|
||||
/// With a `window`, only that rectangle is read and the blocks start at its corner; only whole blocks are made, so
|
||||
/// the result is `window.width / factor × window.height / factor` (the same blocks a binned development of the same
|
||||
/// crop makes). Without one the whole product is read and blocks at the right and bottom edges may be smaller.
|
||||
///
|
||||
/// With `clip` (white is 1), a block in which any sample of a colour is at or above it takes that colour's
|
||||
/// maximum instead of the mean, so a clipped highlight stays clipped for highlight reconstruction.
|
||||
pub fn read_window(path: &Path, key: &str, window: Option<Window>, factor: usize, clip: Option<f32>) -> Result<Rgb32f, ProductError> {
|
||||
let factor = factor.clamp(1, 1 << 12);
|
||||
let mut f = std::io::BufReader::new(std::fs::File::open(path)?);
|
||||
let Start { header: h, table, expected_len } = read_start(&mut f)?;
|
||||
if h.key != key {
|
||||
return Err(ProductError::Stale);
|
||||
}
|
||||
if std::fs::metadata(path)?.len() != expected_len {
|
||||
return Err(ProductError::Format("the file is not the length its header says".into()));
|
||||
}
|
||||
let w = window.unwrap_or(Window { x: 0, y: 0, width: h.width, height: h.height });
|
||||
let fits = |a: usize, b: usize, limit: usize| a.checked_add(b).is_some_and(|e| e <= limit);
|
||||
if w.width == 0 || w.height == 0 || !fits(w.x, w.width, h.width) || !fits(w.y, w.height, h.height) {
|
||||
return Err(ProductError::Format("the window does not lie inside the picture".into()));
|
||||
}
|
||||
// whole blocks for a window; partial ones at the edges when the whole picture is read
|
||||
let whole = window.is_none();
|
||||
let (ow, oh) = if whole { (w.width.div_ceil(factor), w.height.div_ceil(factor)) } else { (w.width / factor, w.height / factor) };
|
||||
if ow == 0 || oh == 0 {
|
||||
return Err(ProductError::Format("the window is smaller than one block".into()));
|
||||
}
|
||||
// the picture rows and columns the blocks cover
|
||||
let (x_end, y_end) = if whole { (w.x + w.width, w.y + w.height) } else { (w.x + ow * factor, w.y + oh * factor) };
|
||||
let mut out = Rgb32f { width: ow, height: oh, data: allocate(ow * oh, [0.0; 3])? };
|
||||
let lut = f16_table();
|
||||
let value = |raw: &[u8], c: usize, n: usize, i: usize| -> f32 {
|
||||
let (lo, hi) = (raw.get(2 * c * n + i).copied().unwrap_or(0), raw.get((2 * c + 1) * n + i).copied().unwrap_or(0));
|
||||
lut.get(usize::from(lo) | usize::from(hi) << 8).copied().unwrap_or(0.0)
|
||||
};
|
||||
// which strips are needed, and where each starts in the file (after the header and the strip table)
|
||||
let wanted = |s: usize| s * h.strip < y_end && (s + 1) * h.strip > w.y;
|
||||
let (mut sum, mut peak) = (allocate(ow, [0f32; 3])?, allocate(ow, [f32::MIN; 3])?);
|
||||
let (mut rows_in, mut oy) = (0usize, 0usize);
|
||||
let strips: Vec<usize> = (0..table.len()).filter(|&s| wanted(s)).collect();
|
||||
// skip the strips before the first one wanted
|
||||
let skipped: usize = table.iter().take(strips.first().copied().unwrap_or(0)).map(|e| e.0).sum();
|
||||
f.seek_relative(i64::try_from(skipped).map_err(|_| ProductError::Format("the file is too large".into()))?)?;
|
||||
for group in strips.chunks(GROUP) {
|
||||
let mut packed = Vec::with_capacity(group.len());
|
||||
for &s in group {
|
||||
let Some(&(len, _)) = table.get(s) else { return Err(ProductError::Format("a strip is missing".into())) };
|
||||
let mut p = allocate(len, 0u8)?;
|
||||
f.read_exact(&mut p)?;
|
||||
packed.push(p);
|
||||
}
|
||||
let decoded: Vec<Result<Vec<u8>, ProductError>> = group
|
||||
.par_iter()
|
||||
.zip(packed.par_iter())
|
||||
.map(|(&s, p)| {
|
||||
let rows = h.height.saturating_sub(s * h.strip).min(h.strip);
|
||||
decode_strip(s, p, table.get(s).map_or(0, |e| e.1), rows * h.width)
|
||||
})
|
||||
.collect();
|
||||
for (&s, raw) in group.iter().zip(decoded) {
|
||||
let raw = raw?;
|
||||
let y0 = s * h.strip;
|
||||
let rows = h.height.saturating_sub(y0).min(h.strip);
|
||||
let n = rows * h.width;
|
||||
for r in 0..rows {
|
||||
let y = y0 + r;
|
||||
if y < w.y || y >= y_end {
|
||||
continue;
|
||||
}
|
||||
let base = r * h.width;
|
||||
for (ox, (sm, pk)) in sum.iter_mut().zip(peak.iter_mut()).enumerate() {
|
||||
let x0 = w.x + ox * factor;
|
||||
let x1 = (x0 + factor).min(x_end);
|
||||
for x in x0..x1 {
|
||||
for c in 0..3 {
|
||||
let v = value(&raw, c, n, base + x);
|
||||
if let (Some(s_), Some(p_)) = (sm.get_mut(c), pk.get_mut(c)) {
|
||||
*s_ += v;
|
||||
*p_ = p_.max(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rows_in += 1;
|
||||
if rows_in == factor || y + 1 == y_end {
|
||||
for (ox, (sm, pk)) in sum.iter_mut().zip(peak.iter_mut()).enumerate() {
|
||||
// the block's size at the right and bottom edges may be smaller
|
||||
let bw = (x_end - (w.x + ox * factor)).min(factor);
|
||||
let k = 1.0 / (rows_in * bw).max(1) as f32;
|
||||
if let Some(o) = out.data.get_mut(oy * ow + ox) {
|
||||
for c in 0..3 {
|
||||
let (s_, p_) = (sm.get(c).copied().unwrap_or(0.0), pk.get(c).copied().unwrap_or(0.0));
|
||||
if let Some(slot) = o.get_mut(c) {
|
||||
*slot = if factor > 1 && clip.is_some_and(|t| p_ >= t) { p_ } else { s_ * k };
|
||||
}
|
||||
}
|
||||
}
|
||||
*sm = [0.0; 3];
|
||||
*pk = [f32::MIN; 3];
|
||||
}
|
||||
rows_in = 0;
|
||||
oy += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// [`read_window`] of the whole product: every `factor × factor` block averaged into one.
|
||||
pub fn read(path: &Path, key: &str, factor: usize) -> Result<Rgb32f, ProductError> {
|
||||
read_window(path, key, None, factor, None)
|
||||
}
|
||||
|
||||
/// `base + (denoised − base) · amount` for every pixel (`amount` 0 to 1), in place in `base`. The two pictures must
|
||||
/// be the same size; if they are not, `base` is left as it was and `false` is returned.
|
||||
pub fn blend(base: &mut Rgb32f, denoised: &Rgb32f, amount: f32) -> bool {
|
||||
if base.width != denoised.width || base.height != denoised.height {
|
||||
return false;
|
||||
}
|
||||
let a = if amount.is_finite() { amount.clamp(0.0, 1.0) } else { 0.0 };
|
||||
base.data.par_iter_mut().zip(denoised.data.par_iter()).for_each(|(b, d)| {
|
||||
for c in 0..3 {
|
||||
b[c] += (d[c] - b[c]) * a;
|
||||
}
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn oversized_product_header_is_rejected_before_allocating_pixels() {
|
||||
let p = temp("oversized-header");
|
||||
let header = serde_json::to_vec(&Header { width: 10001, height: 10000, strip: STRIP_ROWS, key: "k".into() }).unwrap();
|
||||
let mut bytes = Vec::from(MAGIC.as_slice());
|
||||
bytes.extend_from_slice(&[VERSION, 0, 0, 0]);
|
||||
bytes.extend_from_slice(&(header.len() as u32).to_le_bytes());
|
||||
bytes.extend_from_slice(&header);
|
||||
std::fs::write(&p, bytes).unwrap();
|
||||
assert!(read_header_at(&p).is_err());
|
||||
let _ = std::fs::remove_file(p);
|
||||
}
|
||||
|
||||
fn temp(name: &str) -> std::path::PathBuf {
|
||||
std::env::temp_dir().join(format!("lc-denoise-{name}-{}.lcdn", std::process::id()))
|
||||
}
|
||||
|
||||
fn picture(w: usize, h: usize) -> Rgb32f {
|
||||
Rgb32f {
|
||||
width: w,
|
||||
height: h,
|
||||
data: (0..w * h).map(|i| [(i % w) as f32 / w as f32, (i / w) as f32 / h as f32, 0.001 + 0.5 * ((i % 7) as f32 / 7.0)]).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn half_floats_round_trip_closely() {
|
||||
for v in [0.0f32, 1.0, -1.0, 0.5, 0.333_333, 1e-4, 6.1e-5, 3e-6, 123.456, 65000.0, -0.002] {
|
||||
let back = from_f16(to_f16(v));
|
||||
assert!((back - v).abs() <= v.abs() * 0.001 + 1e-7, "{v} -> {back}");
|
||||
}
|
||||
assert_eq!(from_f16(to_f16(1e10)), 65504.0);
|
||||
assert_eq!(to_f16(f32::NAN), 0);
|
||||
assert_eq!(from_f16(to_f16(f32::NEG_INFINITY)), -65504.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_product_survives_a_round_trip_at_full_and_reduced_size() {
|
||||
let (w, h) = (301, 517);
|
||||
let img = picture(w, h);
|
||||
let p = temp("round");
|
||||
write(&p, &img, "k1").unwrap();
|
||||
assert!(is_current(&p, "k1") && !is_current(&p, "k2"));
|
||||
assert_eq!(read_header_at(&p).unwrap().width, w);
|
||||
let full = read(&p, "k1", 1).unwrap();
|
||||
assert_eq!((full.width, full.height), (w, h));
|
||||
for (a, b) in full.data.iter().zip(&img.data) {
|
||||
for c in 0..3 {
|
||||
assert!((a[c] - b[c]).abs() < 0.002, "{a:?} vs {b:?}");
|
||||
}
|
||||
}
|
||||
// averaged into blocks of 4: the sizes round up, and the mean of the picture is kept
|
||||
let small = read(&p, "k1", 4).unwrap();
|
||||
assert_eq!((small.width, small.height), (w.div_ceil(4), h.div_ceil(4)));
|
||||
let mean = |i: &Rgb32f| i.data.iter().map(|p| f64::from(p[1])).sum::<f64>() / i.data.len() as f64;
|
||||
assert!((mean(&small) - mean(&img)).abs() < 0.01);
|
||||
// a factor that does not divide the strip height is the same average
|
||||
let odd = read(&p, "k1", 5).unwrap();
|
||||
assert_eq!((odd.width, odd.height), (w.div_ceil(5), h.div_ceil(5)));
|
||||
assert!((mean(&odd) - mean(&img)).abs() < 0.01);
|
||||
assert!(matches!(read(&p, "other", 1), Err(ProductError::Stale)));
|
||||
std::fs::remove_file(&p).unwrap();
|
||||
}
|
||||
|
||||
/// What binning a window of `img` by `k` means, written the slow obvious way.
|
||||
fn blocks(img: &Rgb32f, win: Window, k: usize, clip: Option<f32>) -> Rgb32f {
|
||||
let (ow, oh) = (win.width / k, win.height / k);
|
||||
let mut out = Rgb32f { width: ow, height: oh, data: vec![[0.0; 3]; ow * oh] };
|
||||
for by in 0..oh {
|
||||
for bx in 0..ow {
|
||||
for c in 0..3 {
|
||||
let vals: Vec<f32> = (0..k * k).map(|i| img.data[(win.y + by * k + i / k) * img.width + win.x + bx * k + i % k][c]).collect();
|
||||
let max = vals.iter().copied().fold(f32::MIN, f32::max);
|
||||
out.data[by * ow + bx][c] =
|
||||
if k > 1 && clip.is_some_and(|t| max >= t) { max } else { vals.iter().sum::<f32>() / vals.len() as f32 };
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_window_is_read_as_the_blocks_of_that_crop() {
|
||||
let (w, h) = (301, 517);
|
||||
let mut img = picture(w, h);
|
||||
// a clipped sample in the middle of a block
|
||||
img.data[(130 + 2) * w + 61 + 1] = [1.0, 0.4, 0.4];
|
||||
let p = temp("window");
|
||||
write(&p, &img, "k1").unwrap();
|
||||
let near = |a: &Rgb32f, b: &Rgb32f| {
|
||||
assert_eq!((a.width, a.height), (b.width, b.height));
|
||||
for (x, y) in a.data.iter().zip(&b.data) {
|
||||
for c in 0..3 {
|
||||
assert!((x[c] - y[c]).abs() < 0.003, "{x:?} vs {y:?}");
|
||||
}
|
||||
}
|
||||
};
|
||||
// windows at odd places, across strip boundaries (128 rows), with factors that do and do not divide them
|
||||
for (win, k) in [
|
||||
(Window { x: 3, y: 5, width: 200, height: 300 }, 1),
|
||||
(Window { x: 3, y: 5, width: 200, height: 300 }, 4),
|
||||
(Window { x: 61, y: 130, width: 121, height: 250 }, 5),
|
||||
(Window { x: 0, y: 0, width: w, height: h }, 2),
|
||||
(Window { x: 1, y: 380, width: 90, height: 137 }, 3),
|
||||
] {
|
||||
let got = read_window(&p, "k1", Some(win), k, None).unwrap();
|
||||
near(&got, &blocks(&img, win, k, None));
|
||||
}
|
||||
// with a clip level the block with the clipped sample keeps its highest value, the others stay means
|
||||
let win = Window { x: 61, y: 130, width: 40, height: 20 };
|
||||
let clipped = read_window(&p, "k1", Some(win), 4, Some(0.99)).unwrap();
|
||||
near(&clipped, &blocks(&img, win, 4, Some(0.99)));
|
||||
assert!(clipped.data[0][0] > 0.99, "the clipped block stays clipped: {:?}", clipped.data[0]);
|
||||
let plain = read_window(&p, "k1", Some(win), 4, None).unwrap();
|
||||
assert!(plain.data[0][0] < 0.99, "an average falls below the clip level");
|
||||
// windows that do not fit, or are smaller than a block, are errors
|
||||
for bad in [
|
||||
Window { x: 0, y: 0, width: 0, height: 5 },
|
||||
Window { x: 290, y: 0, width: 20, height: 5 },
|
||||
Window { x: 0, y: 510, width: 5, height: 10 },
|
||||
Window { x: usize::MAX, y: 0, width: 2, height: 2 },
|
||||
Window { x: 0, y: 0, width: 3, height: 3 },
|
||||
] {
|
||||
assert!(read_window(&p, "k1", Some(bad), 4, None).is_err(), "{bad:?}");
|
||||
}
|
||||
std::fs::remove_file(&p).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_smooth_picture_compresses_well() {
|
||||
let img = Rgb32f {
|
||||
width: 512,
|
||||
height: 512,
|
||||
data: (0..512 * 512)
|
||||
.map(|i| {
|
||||
let v = 0.2 + 0.0001 * (i % 512) as f32;
|
||||
[v, v * 0.8, v * 0.6]
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
let p = temp("size");
|
||||
write(&p, &img, "k").unwrap();
|
||||
let bytes = std::fs::metadata(&p).unwrap().len() as usize;
|
||||
assert!(bytes < 512 * 512 * 6 / 2, "{bytes} bytes for a smooth picture");
|
||||
std::fs::remove_file(&p).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn damaged_files_are_errors_not_panics() {
|
||||
let img = picture(100, 300);
|
||||
let p = temp("damage");
|
||||
write(&p, &img, "k").unwrap();
|
||||
let good = std::fs::read(&p).unwrap();
|
||||
// truncated at every length
|
||||
for cut in (0..good.len()).step_by(37) {
|
||||
std::fs::write(&p, &good[..cut]).unwrap();
|
||||
assert!(read(&p, "k", 1).is_err(), "cut at {cut}");
|
||||
assert!(!is_current(&p, "k"));
|
||||
}
|
||||
// a flipped byte anywhere is caught by a checksum, the header parse, or the key
|
||||
for at in (0..good.len()).step_by(53) {
|
||||
let mut bad = good.clone();
|
||||
bad[at] ^= 0xff;
|
||||
std::fs::write(&p, &bad).unwrap();
|
||||
let _ = read(&p, "k", 1); // must not panic; most are errors
|
||||
}
|
||||
// a header that claims an absurd size
|
||||
let mut huge = b"LCDN\x01\0\0\0".to_vec();
|
||||
let hdr = br#"{"width":4000000000,"height":4000000000,"strip":128,"key":"k"}"#;
|
||||
huge.extend((hdr.len() as u32).to_le_bytes());
|
||||
huge.extend(hdr);
|
||||
std::fs::write(&p, &huge).unwrap();
|
||||
assert!(matches!(read(&p, "k", 1), Err(ProductError::Format(_))));
|
||||
std::fs::write(&p, b"").unwrap();
|
||||
assert!(read(&p, "k", 1).is_err());
|
||||
assert!(read(&temp("missing"), "k", 1).is_err());
|
||||
std::fs::remove_file(&p).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writing_rejects_unusable_pictures() {
|
||||
let p = temp("bad");
|
||||
assert!(write(&p, &Rgb32f { width: 0, height: 0, data: vec![] }, "k").is_err());
|
||||
assert!(write(&p, &Rgb32f { width: 3, height: 3, data: vec![[0.0; 3]; 8] }, "k").is_err());
|
||||
assert!(!p.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blend_mixes_by_amount_and_rejects_other_sizes() {
|
||||
let mut base = Rgb32f { width: 2, height: 1, data: vec![[0.0; 3], [1.0; 3]] };
|
||||
let dn = Rgb32f { width: 2, height: 1, data: vec![[1.0; 3], [0.0; 3]] };
|
||||
assert!(blend(&mut base, &dn, 0.25));
|
||||
assert_eq!(base.data, vec![[0.25; 3], [0.75; 3]]);
|
||||
assert!(blend(&mut base, &dn, f32::NAN));
|
||||
assert_eq!(base.data, vec![[0.25; 3], [0.75; 3]]);
|
||||
assert!(!blend(&mut base, &Rgb32f { width: 1, height: 1, data: vec![[0.0; 3]] }, 1.0));
|
||||
assert!(blend(&mut base, &dn, 1.0));
|
||||
assert_eq!(base.data, dn.data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crc_matches_the_standard_check_value() {
|
||||
assert_eq!(crc(b"123456789"), 0xcbf4_3926);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,589 @@
|
||||
//! Denoising a whole Bayer mosaic with a tile model: cut it into overlapping tiles, run the model on each, bring
|
||||
//! each tile's scale back to the input's, keep clipped highlights as they were, and blend the tiles together.
|
||||
//!
|
||||
//! The model itself is a [`TileRunner`], so everything around it (packing, tiling, scale matching, highlight
|
||||
//! protection, blending, cancelling, progress) is tested without one.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use lightcraft_raster::Rgb32f;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::bayer::{Layout, pack_tile};
|
||||
use crate::manifest::Gain;
|
||||
use crate::tiles::{starts, weight};
|
||||
|
||||
/// Most pixels one picture may have (guards the allocations that follow from its size).
|
||||
pub const MAX_PIXELS: usize = 100_000_000;
|
||||
/// A tile whose mean signal is below this (white = 1) is too dark to measure its scale from.
|
||||
const MIN_SIGNAL: f32 = 0.003;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error("{0}")]
|
||||
Input(String),
|
||||
#[error("the model gave {0}")]
|
||||
Model(String),
|
||||
#[error("the model could not run: {0}")]
|
||||
Runtime(String),
|
||||
#[error("the model could not be loaded: {0}")]
|
||||
Load(String),
|
||||
#[error("cancelled")]
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
fn allocate<T: Clone>(n: usize, value: T) -> Result<Vec<T>, Error> {
|
||||
let mut v = Vec::new();
|
||||
v.try_reserve_exact(n).map_err(|_| Error::Input("not enough memory for denoise".into()))?;
|
||||
v.resize(n, value);
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// Something that runs the model on one tile.
|
||||
pub trait TileRunner: Sync {
|
||||
/// `input` is the tile as four planes `[R, G1, G2, B]` of `tile × tile` cells (NCHW, batch 1); the answer is
|
||||
/// camera RGB, three planes of `2·tile × 2·tile` pixels (CHW), in the model's own scale.
|
||||
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error>;
|
||||
}
|
||||
|
||||
pub struct Params {
|
||||
/// Side of the model's square, in cells.
|
||||
pub tile: usize,
|
||||
/// Cells neighbouring tiles share.
|
||||
pub overlap: usize,
|
||||
pub gain: Gain,
|
||||
/// White is 1; samples at or above this (minus a little) are clipped and keep their value.
|
||||
pub clip: Option<f32>,
|
||||
/// Tiles run at once (each model call may itself be single-threaded).
|
||||
pub parallel: usize,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Control<'a> {
|
||||
pub cancel: Option<&'a AtomicBool>,
|
||||
/// `(tiles done, tiles in all)`.
|
||||
pub progress: Option<&'a (dyn Fn(usize, usize) + Sync)>,
|
||||
}
|
||||
|
||||
/// The tile grid of a picture: `(starts across, starts down)`.
|
||||
pub fn grid(width: usize, height: usize, layout: Layout, tile: usize, overlap: usize) -> (Vec<usize>, Vec<usize>) {
|
||||
let (nx, ny) = layout.cells(width, height);
|
||||
(starts(nx, tile, overlap), starts(ny, tile, overlap))
|
||||
}
|
||||
|
||||
/// How many model calls [`denoise_bayer`] makes for a picture.
|
||||
pub fn tile_count(width: usize, height: usize, layout: Layout, tile: usize, overlap: usize) -> usize {
|
||||
let (sx, sy) = grid(width, height, layout, tile, overlap);
|
||||
sx.len().saturating_mul(sy.len())
|
||||
}
|
||||
|
||||
/// Denoise (and demosaic) the normalised mosaic `mosaic` of `width × height` samples, laid out as `layout`.
|
||||
/// Gives camera RGB at the mosaic's resolution.
|
||||
pub fn denoise_bayer(
|
||||
mosaic: &[f32],
|
||||
width: usize,
|
||||
height: usize,
|
||||
layout: Layout,
|
||||
runner: &dyn TileRunner,
|
||||
p: &Params,
|
||||
ctl: &Control<'_>,
|
||||
) -> Result<Rgb32f, Error> {
|
||||
let pixels = width
|
||||
.checked_mul(height)
|
||||
.filter(|&n| n > 0 && n <= MAX_PIXELS)
|
||||
.ok_or_else(|| Error::Input("the picture has no size, or is too large".into()))?;
|
||||
if mosaic.len() != pixels {
|
||||
return Err(Error::Input("the mosaic does not have width × height samples".into()));
|
||||
}
|
||||
if !(16..=2048).contains(&p.tile) || p.overlap >= p.tile.div_ceil(2) || layout.dx > 1 || layout.dy > 1 {
|
||||
return Err(Error::Input("the tile size and overlap do not fit".into()));
|
||||
}
|
||||
if let Gain::MatchMean { nominal, max_deviation } = p.gain
|
||||
&& !(nominal.is_finite() && (1e-6..=1e12).contains(&nominal) && max_deviation.is_finite() && (0.0..=0.5).contains(&max_deviation))
|
||||
{
|
||||
return Err(Error::Input("the model gain is not finite or within its supported range".into()));
|
||||
}
|
||||
if p.clip.is_some_and(|v| !v.is_finite() || v <= 0.0) {
|
||||
return Err(Error::Input("the highlight clip must be finite and positive".into()));
|
||||
}
|
||||
let (nx, ny) = layout.cells(width, height);
|
||||
let (sx, sy) = (starts(nx, p.tile, p.overlap), starts(ny, p.tile, p.overlap));
|
||||
let (cols, rows) = (sx.len(), sy.len());
|
||||
if cols == 0 || rows == 0 {
|
||||
return Err(Error::Input("the picture needs too many tiles".into()));
|
||||
}
|
||||
let tiles: Vec<(usize, usize, bool, bool, bool, bool)> = sy
|
||||
.iter()
|
||||
.enumerate()
|
||||
.flat_map(|(j, &y0)| sx.iter().enumerate().map(move |(i, &x0)| (x0, y0, i > 0, i + 1 < cols, j > 0, j + 1 < rows)))
|
||||
.collect();
|
||||
let total = tiles.len();
|
||||
|
||||
let mut acc = allocate(pixels, [0f32; 3])?;
|
||||
let mut wsum = allocate(nx * ny, 0f32)?;
|
||||
let group = p.parallel.clamp(1, 4);
|
||||
let mut done = 0;
|
||||
// where the time goes, printed under `LIGHTCRAFT_PROFILE`
|
||||
let (mut t_wait, mut t_total) = (std::time::Duration::ZERO, std::time::Duration::ZERO);
|
||||
let began = web_time::Instant::now();
|
||||
|
||||
// One group of tiles at a time: pack, run the model, check and bring each to the input's scale, in parallel.
|
||||
// Blending (sequential: the tiles add into one picture, in tile order so the result does not depend on timing) of
|
||||
// the previous group runs while the next group is on the model, so neither the card nor the cores wait for it.
|
||||
type Tile = (usize, usize, bool, bool, bool, bool);
|
||||
let compute = |chunk: &[Tile]| -> Vec<Result<Vec<f32>, Error>> {
|
||||
chunk
|
||||
.par_iter()
|
||||
.map(|&(x0, y0, bx, ax, by, ay)| {
|
||||
let mut input = allocate(4 * p.tile * p.tile, 0f32)?;
|
||||
if !pack_tile(mosaic, width, height, layout, x0 as isize, y0 as isize, p.tile, &mut input) {
|
||||
return Err(Error::Input("a tile could not be packed".into()));
|
||||
}
|
||||
let mut out = runner.run(&input)?;
|
||||
let side = 2 * p.tile;
|
||||
if out.len() != 3 * side * side {
|
||||
return Err(Error::Model(format!("{} numbers for a tile, not the {} expected", out.len(), 3 * side * side)));
|
||||
}
|
||||
if out.iter().any(|v| !v.is_finite()) {
|
||||
return Err(Error::Model("numbers that are not finite".into()));
|
||||
}
|
||||
finish_tile(&input, &mut out, p, (bx, ax, by, ay));
|
||||
Ok(out)
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
let mut blend = |chunk: &[Tile], outs: Vec<Result<Vec<f32>, Error>>| -> Result<(), Error> {
|
||||
for (&(x0, y0, bx, ax, by, ay), r) in chunk.iter().zip(outs) {
|
||||
blend_tile(&r?, &mut acc, &mut wsum, (width, height), (nx, ny), layout, (x0, y0), p, (bx, ax, by, ay));
|
||||
done += 1;
|
||||
if let Some(f) = ctl.progress {
|
||||
f(done, total);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let mut previous: Option<(&[Tile], Vec<Result<Vec<f32>, Error>>)> = None;
|
||||
for chunk in tiles.chunks(group) {
|
||||
if ctl.cancel.is_some_and(|c| c.load(Ordering::Relaxed)) {
|
||||
return Err(Error::Cancelled);
|
||||
}
|
||||
let (outs, blended) = rayon::join(
|
||||
|| compute(chunk),
|
||||
|| match previous.take() {
|
||||
Some((c, o)) => {
|
||||
let started = web_time::Instant::now();
|
||||
let r = blend(c, o);
|
||||
(r, started.elapsed())
|
||||
}
|
||||
None => (Ok(()), std::time::Duration::ZERO),
|
||||
},
|
||||
);
|
||||
blended.0?;
|
||||
t_wait += blended.1;
|
||||
previous = Some((chunk, outs));
|
||||
}
|
||||
if let Some((c, o)) = previous.take() {
|
||||
let started = web_time::Instant::now();
|
||||
blend(c, o)?;
|
||||
t_wait += started.elapsed();
|
||||
}
|
||||
t_total += began.elapsed();
|
||||
let started = web_time::Instant::now();
|
||||
// normalise by the weights
|
||||
let (ox, oy) = (layout.position(0, 0).0, layout.position(0, 0).1);
|
||||
acc.par_chunks_mut(width).enumerate().for_each(|(y, row)| {
|
||||
let cy = ((y as isize - oy) / 2) as usize;
|
||||
for (x, px) in row.iter_mut().enumerate() {
|
||||
let cx = ((x as isize - ox) / 2) as usize;
|
||||
let w = wsum.get(cy * nx + cx).copied().unwrap_or(0.0);
|
||||
if w > 0.0 {
|
||||
*px = px.map(|v| v / w);
|
||||
}
|
||||
}
|
||||
});
|
||||
if std::env::var_os("LIGHTCRAFT_PROFILE").is_some() {
|
||||
eprintln!(
|
||||
"[denoise] {total} tiles, {} at once: tiles {:.0} ms (blending, {:.0} ms of it, overlaps the model), normalise {:.0} ms",
|
||||
group,
|
||||
t_total.as_secs_f64() * 1e3,
|
||||
t_wait.as_secs_f64() * 1e3,
|
||||
started.elapsed().as_secs_f64() * 1e3
|
||||
);
|
||||
}
|
||||
Ok(Rgb32f { width, height, data: acc })
|
||||
}
|
||||
|
||||
/// Bring a tile's output to the input's scale and keep clipped highlights as they were (in place).
|
||||
fn finish_tile(input: &[f32], out: &mut [f32], p: &Params, edges: (bool, bool, bool, bool)) {
|
||||
let t = p.tile;
|
||||
let side = 2 * t;
|
||||
if let Gain::MatchMean { nominal, max_deviation } = p.gain {
|
||||
// the part of the tile whose scale is measured: away from the edges that blend into a neighbour
|
||||
let m = p.overlap / 2;
|
||||
let (i0, i1) = (if edges.0 { m } else { 0 }, t - if edges.1 { m } else { 0 });
|
||||
let (j0, j1) = (if edges.2 { m } else { 0 }, t - if edges.3 { m } else { 0 });
|
||||
let cells = ((i1 - i0) * (j1 - j0)) as f64;
|
||||
let plane_sum = |pl: usize| -> f64 {
|
||||
(j0..j1)
|
||||
.map(|j| input.get(pl * t * t + j * t + i0..pl * t * t + j * t + i1).map_or(0.0, |r| r.iter().map(|&v| f64::from(v)).sum::<f64>()))
|
||||
.sum()
|
||||
};
|
||||
let in_mean = [plane_sum(0) / cells, (plane_sum(1) + plane_sum(2)) / (2.0 * cells), plane_sum(3) / cells];
|
||||
for (c, &mean_in) in in_mean.iter().enumerate() {
|
||||
let sum_out: f64 = (2 * j0..2 * j1)
|
||||
.map(|y| {
|
||||
out.get(c * side * side + y * side + 2 * i0..c * side * side + y * side + 2 * i1)
|
||||
.map_or(0.0, |r| r.iter().map(|&v| f64::from(v)).sum::<f64>())
|
||||
})
|
||||
.sum();
|
||||
let mean_out = sum_out / (4.0 * cells);
|
||||
let measured = if mean_in as f32 >= MIN_SIGNAL && mean_out > 0.0 { (mean_out / mean_in) as f32 } else { nominal };
|
||||
let g = measured.clamp(nominal * (1.0 - max_deviation), nominal * (1.0 + max_deviation));
|
||||
if g.is_finite() && g > 0.0 {
|
||||
let inv = 1.0 / g;
|
||||
if let Some(plane) = out.get_mut(c * side * side..(c + 1) * side * side) {
|
||||
plane.iter_mut().for_each(|v| *v *= inv);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(clip) = p.clip {
|
||||
protect_clipped(input, out, t, clip);
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the mosaic is at (or within a cell of) white, the output becomes the plain cell values again, so a clipped
|
||||
/// area stays exactly as clipped as the camera recorded it (highlight recovery later depends on that).
|
||||
fn protect_clipped(input: &[f32], out: &mut [f32], t: usize, clip: f32) {
|
||||
let side = 2 * t;
|
||||
let plane = t * t;
|
||||
let (Some(r), Some(g1), Some(g2), Some(b)) =
|
||||
(input.get(0..plane), input.get(plane..2 * plane), input.get(2 * plane..3 * plane), input.get(3 * plane..4 * plane))
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let mut m: Vec<f32> = (0..plane).map(|i| r[i].max(g1[i]).max(g2[i]).max(b[i])).collect();
|
||||
if !m.iter().any(|&v| v >= clip) {
|
||||
return;
|
||||
}
|
||||
// 3×3 maximum over cells (separable)
|
||||
let dilate = |src: &[f32], horizontal: bool| -> Vec<f32> {
|
||||
(0..plane)
|
||||
.map(|i| {
|
||||
let (x, y) = (i % t, i / t);
|
||||
let mut best = src[i];
|
||||
for d in [-1isize, 1] {
|
||||
let (nx, ny) = if horizontal { (x as isize + d, y as isize) } else { (x as isize, y as isize + d) };
|
||||
if nx >= 0 && ny >= 0 && (nx as usize) < t && (ny as usize) < t {
|
||||
best = best.max(src[ny as usize * t + nx as usize]);
|
||||
}
|
||||
}
|
||||
best
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
m = dilate(&dilate(&m, true), false);
|
||||
let lo = clip - 0.03;
|
||||
for j in 0..t {
|
||||
for i in 0..t {
|
||||
let k = j * t + i;
|
||||
let s = ((m[k] - lo) / (clip - lo)).clamp(0.0, 1.0);
|
||||
if s <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
let s = s * s * (3.0 - 2.0 * s);
|
||||
let plain = [r[k], 0.5 * (g1[k] + g2[k]), b[k]];
|
||||
for (c, &pv) in plain.iter().enumerate() {
|
||||
for (sy, sx) in [(0, 0), (0, 1), (1, 0), (1, 1)] {
|
||||
if let Some(v) = out.get_mut(c * side * side + (2 * j + sy) * side + 2 * i + sx) {
|
||||
*v += (pv - *v) * s;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a finished tile to the picture with its blend weights.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn blend_tile(
|
||||
out: &[f32],
|
||||
acc: &mut [[f32; 3]],
|
||||
wsum: &mut [f32],
|
||||
size: (usize, usize),
|
||||
cells: (usize, usize),
|
||||
layout: Layout,
|
||||
at: (usize, usize),
|
||||
p: &Params,
|
||||
edges: (bool, bool, bool, bool),
|
||||
) {
|
||||
let (width, height) = size;
|
||||
let (nx, ny) = cells;
|
||||
let (t, side) = (p.tile, 2 * p.tile);
|
||||
let fade = p.overlap;
|
||||
let wx: Vec<f32> = (0..t).map(|i| weight(i, t, fade, edges.0, edges.1)).collect();
|
||||
let wy: Vec<f32> = (0..t).map(|j| weight(j, t, fade, edges.2, edges.3)).collect();
|
||||
for j in 0..t {
|
||||
let cy = at.1 + j;
|
||||
for i in 0..t {
|
||||
let cx = at.0 + i;
|
||||
// cells past the picture's edge (a picture smaller than one tile is padded) belong to no pixel
|
||||
if cx >= nx || cy >= ny {
|
||||
continue;
|
||||
}
|
||||
let (Some(&a), Some(&b)) = (wx.get(i), wy.get(j)) else { continue };
|
||||
let w = a * b;
|
||||
if let Some(s) = wsum.get_mut(cy * nx + cx) {
|
||||
*s += w;
|
||||
}
|
||||
let (px0, py0) = layout.position(cx as isize, cy as isize);
|
||||
for (sy, sx) in [(0isize, 0isize), (0, 1), (1, 0), (1, 1)] {
|
||||
let (x, y) = (px0 + sx, py0 + sy);
|
||||
if x < 0 || y < 0 || x as usize >= width || y as usize >= height {
|
||||
continue;
|
||||
}
|
||||
let o = (2 * j + sy as usize) * side + 2 * i + sx as usize;
|
||||
let (Some(&r), Some(&g), Some(&bl)) = (out.get(o), out.get(side * side + o), out.get(2 * side * side + o)) else { continue };
|
||||
if let Some(px) = acc.get_mut(y as usize * width + x as usize) {
|
||||
px[0] += w * r;
|
||||
px[1] += w * g;
|
||||
px[2] += w * bl;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A stand-in model: the cell's samples as an RGB picture at twice the resolution (a plain 2 × 2 "demosaic"),
|
||||
/// times `scale`, so the output has a scale of its own like the real model's.
|
||||
struct Mock {
|
||||
tile: usize,
|
||||
scale: f32,
|
||||
}
|
||||
impl TileRunner for Mock {
|
||||
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
|
||||
let (t, side) = (self.tile, 2 * self.tile);
|
||||
let plane = t * t;
|
||||
let mut out = vec![0.0; 3 * side * side];
|
||||
for j in 0..t {
|
||||
for i in 0..t {
|
||||
let k = j * t + i;
|
||||
let rgb = [input[k], 0.5 * (input[plane + k] + input[2 * plane + k]), input[3 * plane + k]];
|
||||
for (c, v) in rgb.iter().enumerate() {
|
||||
for (sy, sx) in [(0, 0), (0, 1), (1, 0), (1, 1)] {
|
||||
out[c * side * side + (2 * j + sy) * side + 2 * i + sx] = v * self.scale;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
fn params(tile: usize, overlap: usize, gain: Gain) -> Params {
|
||||
Params { tile, overlap, gain, clip: None, parallel: 2 }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overflowing_tiles_and_nan_gain_are_errors_before_the_model_runs() {
|
||||
let layout = Layout::from_cell([0, 1, 1, 2]).unwrap();
|
||||
let model = Mock { tile: 16, scale: 1.0 };
|
||||
for (tile, overlap, gain) in [
|
||||
(usize::MAX, 0, Gain::None),
|
||||
(16, usize::MAX, Gain::None),
|
||||
(16, 0, Gain::MatchMean { nominal: 1.0, max_deviation: f32::NAN }),
|
||||
(16, 0, Gain::MatchMean { nominal: f32::INFINITY, max_deviation: 0.1 }),
|
||||
] {
|
||||
assert!(matches!(denoise_bayer(&[0.5], 1, 1, layout, &model, ¶ms(tile, overlap, gain), &Control::default()), Err(Error::Input(_))));
|
||||
}
|
||||
}
|
||||
|
||||
/// A smooth picture whose colour at a sample is `base[colour] · (1 + shading)`.
|
||||
fn mosaic(w: usize, h: usize, cell: [u8; 4]) -> Vec<f32> {
|
||||
(0..w * h)
|
||||
.map(|i| {
|
||||
let (x, y) = (i % w, i / w);
|
||||
let c = cell[(y % 2) * 2 + x % 2] as usize;
|
||||
[0.5, 0.3, 0.2][c] * (0.6 + 0.4 * ((x as f32 / 23.0).sin() * (y as f32 / 31.0).cos()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn reference(m: &[f32], w: usize, h: usize, layout: Layout) -> Rgb32f {
|
||||
// what the mock gives for the whole picture in one tile
|
||||
let (nx, ny) = layout.cells(w, h);
|
||||
let t = nx.max(ny).div_ceil(16) * 16;
|
||||
denoise_bayer(m, w, h, layout, &Mock { tile: t, scale: 1.0 }, ¶ms(t, 16, Gain::None), &Control::default()).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiles_blend_to_the_same_picture_as_one_tile_for_every_layout() {
|
||||
for cell in [[0u8, 1, 1, 2], [1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]] {
|
||||
let layout = Layout::from_cell(cell).unwrap();
|
||||
let (w, h) = (150usize, 110usize);
|
||||
let m = mosaic(w, h, cell);
|
||||
let want = reference(&m, w, h, layout);
|
||||
let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, ¶ms(32, 8, Gain::None), &Control::default()).unwrap();
|
||||
assert_eq!((got.width, got.height), (w, h));
|
||||
for (a, b) in got.data.iter().zip(&want.data) {
|
||||
for c in 0..3 {
|
||||
assert!((a[c] - b[c]).abs() < 1e-5, "layout {cell:?}: {a:?} vs {b:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_picture_does_not_depend_on_how_many_tiles_run_at_once_or_how_long_each_takes() {
|
||||
// tiles that finish in a different order every time: blending still adds them in tile order
|
||||
struct Jittery(Mock);
|
||||
impl TileRunner for Jittery {
|
||||
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
|
||||
let wait = (input.iter().take(64).sum::<f32>() * 1000.0) as u64 % 7;
|
||||
std::thread::sleep(std::time::Duration::from_millis(wait));
|
||||
self.0.run(input)
|
||||
}
|
||||
}
|
||||
let layout = Layout { dx: 0, dy: 1 };
|
||||
let (w, h) = (150usize, 110usize);
|
||||
let m = mosaic(w, h, [1, 0, 2, 1]);
|
||||
let run = |parallel: usize| {
|
||||
let mut p = params(32, 8, Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 });
|
||||
p.parallel = parallel;
|
||||
denoise_bayer(&m, w, h, layout, &Jittery(Mock { tile: 32, scale: 1.01e6 }), &p, &Control::default()).unwrap()
|
||||
};
|
||||
let one = run(1);
|
||||
for parallel in [2, 3, 8] {
|
||||
assert_eq!(run(parallel).data, one.data, "{parallel} at once");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failing_tile_or_a_cancel_ends_the_picture_without_a_panic() {
|
||||
struct Fails(Mock, std::sync::atomic::AtomicUsize);
|
||||
impl TileRunner for Fails {
|
||||
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
|
||||
if self.1.fetch_add(1, Ordering::Relaxed) == 4 {
|
||||
return Err(Error::Runtime("boom".into()));
|
||||
}
|
||||
self.0.run(input)
|
||||
}
|
||||
}
|
||||
let layout = Layout { dx: 0, dy: 0 };
|
||||
let (w, h) = (150usize, 110usize);
|
||||
let m = mosaic(w, h, [0, 1, 1, 2]);
|
||||
let r = denoise_bayer(
|
||||
&m,
|
||||
w,
|
||||
h,
|
||||
layout,
|
||||
&Fails(Mock { tile: 32, scale: 1.0 }, Default::default()),
|
||||
¶ms(32, 8, Gain::None),
|
||||
&Control::default(),
|
||||
);
|
||||
assert!(matches!(r, Err(Error::Runtime(_))), "{r:?}");
|
||||
let stop = AtomicBool::new(true);
|
||||
let ctl = Control { cancel: Some(&stop), progress: None };
|
||||
let r = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, ¶ms(32, 8, Gain::None), &ctl);
|
||||
assert!(matches!(r, Err(Error::Cancelled)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_scale_of_its_own_is_taken_out_tile_by_tile() {
|
||||
let layout = Layout { dx: 0, dy: 0 };
|
||||
let (w, h) = (150usize, 110usize);
|
||||
let m = mosaic(w, h, [0, 1, 1, 2]);
|
||||
let want = reference(&m, w, h, layout);
|
||||
let gain = Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 };
|
||||
// the scale differs a little from nominal; each tile's own scale is measured and removed
|
||||
let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.02e6 }, ¶ms(32, 8, gain), &Control::default()).unwrap();
|
||||
let mean = |img: &Rgb32f, c: usize| img.data.iter().map(|p| f64::from(p[c])).sum::<f64>() / img.data.len() as f64;
|
||||
for c in 0..3 {
|
||||
let (a, b) = (mean(&got, c), mean(&want, c));
|
||||
assert!((a - b).abs() / b < 0.01, "channel {c}: {a} vs {b}");
|
||||
}
|
||||
// a scale far from nominal is clamped rather than trusted
|
||||
let wild = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 3.0e6 }, ¶ms(32, 8, gain), &Control::default()).unwrap();
|
||||
assert!(mean(&wild, 1) > 2.0 * mean(&want, 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_picture_smaller_than_a_tile_is_padded_by_mirroring() {
|
||||
let layout = Layout { dx: 1, dy: 1 };
|
||||
let (w, h) = (20usize, 14usize);
|
||||
let m = mosaic(w, h, [2, 1, 1, 0]);
|
||||
let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 64, scale: 1.0 }, ¶ms(64, 16, Gain::None), &Control::default()).unwrap();
|
||||
assert_eq!((got.width, got.height), (w, h));
|
||||
assert!(got.data.iter().all(|p| p.iter().all(|v| v.is_finite() && *v > 0.0)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clipped_highlights_stay_clipped() {
|
||||
// a model that darkens everything by 10 % would pull a clipped plateau below white
|
||||
struct Dim(Mock);
|
||||
impl TileRunner for Dim {
|
||||
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
|
||||
Ok(self.0.run(input)?.into_iter().map(|v| v * 0.9).collect())
|
||||
}
|
||||
}
|
||||
let layout = Layout { dx: 0, dy: 0 };
|
||||
let (w, h) = (96usize, 96usize);
|
||||
let m: Vec<f32> = (0..w * h).map(|i| if i % w > 40 && i % w < 70 { 1.0 } else { 0.4 }).collect();
|
||||
let mut p = params(32, 8, Gain::None);
|
||||
p.clip = Some(0.99);
|
||||
let got = denoise_bayer(&m, w, h, layout, &Dim(Mock { tile: 32, scale: 1.0 }), &p, &Control::default()).unwrap();
|
||||
let mid = got.data[48 * w + 55];
|
||||
assert!(mid.iter().all(|&v| v > 0.995), "plateau {mid:?}");
|
||||
let dark = got.data[48 * w + 10];
|
||||
assert!(dark.iter().all(|&v| (v - 0.36).abs() < 0.01), "unclipped area still goes through the model: {dark:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelling_and_progress_work() {
|
||||
let layout = Layout { dx: 0, dy: 0 };
|
||||
let (w, h) = (150usize, 110usize);
|
||||
let m = mosaic(w, h, [0, 1, 1, 2]);
|
||||
let cancel = AtomicBool::new(true);
|
||||
let r = denoise_bayer(
|
||||
&m,
|
||||
w,
|
||||
h,
|
||||
layout,
|
||||
&Mock { tile: 32, scale: 1.0 },
|
||||
¶ms(32, 8, Gain::None),
|
||||
&Control { cancel: Some(&cancel), progress: None },
|
||||
);
|
||||
assert!(matches!(r, Err(Error::Cancelled)));
|
||||
let seen = std::sync::Mutex::new(Vec::new());
|
||||
let f = |d: usize, n: usize| seen.lock().unwrap().push((d, n));
|
||||
denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, ¶ms(32, 8, Gain::None), &Control { cancel: None, progress: Some(&f) })
|
||||
.unwrap();
|
||||
let seen = seen.into_inner().unwrap();
|
||||
let total = tile_count(w, h, layout, 32, 8);
|
||||
assert!(total > 1);
|
||||
assert_eq!(seen.last(), Some(&(total, total)));
|
||||
assert!(seen.windows(2).all(|p| p[1].0 == p[0].0 + 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_inputs_are_errors() {
|
||||
let layout = Layout { dx: 0, dy: 0 };
|
||||
let mock = Mock { tile: 32, scale: 1.0 };
|
||||
let p = params(32, 8, Gain::None);
|
||||
assert!(matches!(denoise_bayer(&[], 0, 0, layout, &mock, &p, &Control::default()), Err(Error::Input(_))));
|
||||
assert!(matches!(denoise_bayer(&[0.0; 10], 4, 4, layout, &mock, &p, &Control::default()), Err(Error::Input(_))));
|
||||
assert!(matches!(denoise_bayer(&[0.0; 16], 4, 4, layout, &mock, ¶ms(32, 16, Gain::None), &Control::default()), Err(Error::Input(_))));
|
||||
struct Bad(Vec<f32>);
|
||||
impl TileRunner for Bad {
|
||||
fn run(&self, _: &[f32]) -> Result<Vec<f32>, Error> {
|
||||
Ok(self.0.clone())
|
||||
}
|
||||
}
|
||||
assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![0.0; 5]), &p, &Control::default()), Err(Error::Model(_))));
|
||||
assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![f32::NAN; 3 * 64 * 64]), &p, &Control::default()), Err(Error::Model(_))));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
//! Cutting a picture into overlapping tiles and putting the results back together.
|
||||
//!
|
||||
//! A model runs on a fixed square, so a picture is covered by tiles spaced evenly along each axis, each sharing
|
||||
//! at least `overlap` cells with its neighbours. Each tile's influence fades towards its own edges (except at the
|
||||
//! picture's edge), and the result is the sum of tile × weight divided by the sum of weights, so any set of
|
||||
//! overlapping tiles blends consistently and a seam between two tiles of slightly different scale is a smooth ramp.
|
||||
|
||||
/// Where the tiles start along one axis of `n` cells: `[0]` when the axis fits one tile (the tile is then padded by
|
||||
/// mirroring), else evenly spaced starts, the first at 0 and the last at `n − tile`, each tile sharing at least
|
||||
/// `overlap` cells with the next.
|
||||
pub fn starts(n: usize, tile: usize, overlap: usize) -> Vec<usize> {
|
||||
if n <= tile || tile == 0 {
|
||||
return vec![0];
|
||||
}
|
||||
let step = tile.saturating_sub(overlap).max(1);
|
||||
// tiles needed so that start k·spacing with spacing ≤ step reaches n − tile
|
||||
let Some(k) = (n - tile).div_ceil(step).checked_add(1).filter(|&k| k <= 1_000_000) else { return Vec::new() };
|
||||
let last = n - tile;
|
||||
(0..k).map(|i| if k == 1 { 0 } else { ((i as u128 * last as u128 + (k - 1) as u128 / 2) / (k - 1) as u128) as usize }).collect()
|
||||
}
|
||||
|
||||
/// Weight of the cell at offset `p` (0-based, inside a tile of `tile` cells) along one axis: 1 in the middle, easing
|
||||
/// down to a sliver over `fade` cells at an edge that has a neighbour, 1 right up to an edge that is the picture's.
|
||||
pub fn weight(p: usize, tile: usize, fade: usize, has_before: bool, has_after: bool) -> f32 {
|
||||
let fade = fade.max(1) as f32;
|
||||
let ramp = |d: usize| {
|
||||
let t = ((d as f32 + 0.5) / fade).min(1.0);
|
||||
// smoothstep, never exactly 0 so that a cell is always covered
|
||||
(t * t * (3.0 - 2.0 * t)).max(1e-3)
|
||||
};
|
||||
let before = if has_before { ramp(p) } else { 1.0 };
|
||||
let after = if has_after { ramp(tile.saturating_sub(1).saturating_sub(p)) } else { 1.0 };
|
||||
before.min(after)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn hostile_axes_have_bounded_tile_counts_and_checked_spacing() {
|
||||
assert!(starts(usize::MAX, 1, 0).is_empty());
|
||||
assert!(starts(usize::MAX, 2, usize::MAX).is_empty());
|
||||
let s = starts(usize::MAX, usize::MAX / 2, 0);
|
||||
assert_eq!(s.first(), Some(&0));
|
||||
assert_eq!(s.last(), Some(&(usize::MAX - usize::MAX / 2)));
|
||||
assert!(s.windows(2).all(|s| s[0] < s[1]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_tile_when_it_fits() {
|
||||
assert_eq!(starts(100, 512, 64), vec![0]);
|
||||
assert_eq!(starts(512, 512, 64), vec![0]);
|
||||
assert_eq!(starts(0, 512, 64), vec![0]);
|
||||
assert_eq!(starts(10, 0, 0), vec![0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiles_cover_the_axis_and_overlap_enough() {
|
||||
for n in [513usize, 600, 900, 1000, 1024, 1500, 3024, 3600, 5000, 12345] {
|
||||
let s = starts(n, 512, 64);
|
||||
assert_eq!(s[0], 0, "n={n}");
|
||||
assert_eq!(*s.last().unwrap(), n - 512, "n={n}");
|
||||
for w in s.windows(2) {
|
||||
assert!(w[1] > w[0], "starts increase, n={n}");
|
||||
assert!(w[0] + 512 >= w[1] + 64, "at least 64 shared cells between {} and {}, n={n}", w[0], w[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn weights_fade_only_towards_neighbours() {
|
||||
let (t, f) = (512, 64);
|
||||
// a lone tile: full weight everywhere
|
||||
assert!((0..t).all(|p| weight(p, t, f, false, false) == 1.0));
|
||||
// the first tile fades on its far side only
|
||||
assert_eq!(weight(0, t, f, false, true), 1.0);
|
||||
assert!(weight(t - 1, t, f, false, true) < 0.01);
|
||||
// the last tile fades on its near side only
|
||||
assert!(weight(0, t, f, true, false) < 0.01);
|
||||
assert_eq!(weight(t - 1, t, f, true, false), 1.0);
|
||||
// a middle tile is 1 in the middle and everywhere positive
|
||||
assert_eq!(weight(t / 2, t, f, true, true), 1.0);
|
||||
assert!((0..t).all(|p| weight(p, t, f, true, true) > 0.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_overlapping_tiles_blend_to_a_partition_of_unity() {
|
||||
// at every cell of the shared region the normalised weights are in [0, 1] and sum to 1
|
||||
let (t, f) = (512usize, 64usize);
|
||||
let s = starts(900, t, 64);
|
||||
assert_eq!(s.len(), 2);
|
||||
for cell in s[1]..s[0] + t {
|
||||
let a = weight(cell - s[0], t, f, false, true);
|
||||
let b = weight(cell - s[1], t, f, true, false);
|
||||
let (na, nb) = (a / (a + b), b / (a + b));
|
||||
assert!((na + nb - 1.0).abs() < 1e-6 && na >= 0.0 && nb >= 0.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
[package]
|
||||
name = "lightcraft-denoise"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
description = "LightCraft AI denoise (L1): denoise-model manifests, Bayer packing, tiling and blending, the cache file, and pure-Rust CPU inference"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
runtime = ["dep:faer"]
|
||||
# Product-policy switch: absent by default; users may install their own model.
|
||||
rawnind-model = ["lightcraft-denoise-core/rawnind-model"]
|
||||
|
||||
[dependencies]
|
||||
lightcraft-denoise-core = { workspace = true }
|
||||
# Safe matrix views over the pure-Rust GEMM backend; no BLAS, cc/build scripts
|
||||
# or inner thread pool. Its dependencies include architecture-specific assembly.
|
||||
faer = { package = "faer-core", version = "=0.17.1", default-features = false, features = ["std"], optional = true }
|
||||
lightcraft-raster = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
miniz_oxide = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
web-time = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = { workspace = true }
|
||||
@@ -0,0 +1,18 @@
|
||||
# lightcraft-denoise
|
||||
|
||||
L1 pure-Rust AI RAW denoise primitives and CPU inference. No weights are bundled.
|
||||
See [user workflow and limitations](../../docs/denoise.md).
|
||||
|
||||
`lightcraft-denoise-core` provides metadata, Bayer helpers and cache formats,
|
||||
re-exported here. `manifest` and `known` validate opt-in models; `onnx` reads a bounded operator subset
|
||||
into `net`; `cpu` executes it and `reference` supplies scalar checks. `bayer`, `tiles`
|
||||
and `run` pack sensor data and blend overlapping tiles while preserving clipped
|
||||
highlights. `product` reads/writes the disposable `.lcdn` cache. `runtime` supplies
|
||||
the installation self-test. Synthetic networks are generated in Rust for tests.
|
||||
|
||||
Run `cargo test -p lightcraft-denoise --features runtime`. The CPU backend is optional;
|
||||
featureless and web builds keep the model metadata and cache format without faer/GEMM.
|
||||
The `rawnind-model` feature enables the otherwise empty download catalog and requires
|
||||
an explicit release-policy decision. Real-model tests are ignored by default and
|
||||
require an explicitly supplied local model. The GPU twin is `lightcraft-gpu::nn`;
|
||||
model management, background jobs and export integration live in `lightcraft-engine`.
|
||||
@@ -0,0 +1,386 @@
|
||||
//! Single-thread CPU execution of the checked denoise network.
|
||||
//!
|
||||
//! Convolutions lower at most 256 spatial positions at a time, then use faer's safe views over pure-Rust GEMM.
|
||||
//! No inner thread pool: `run::denoise_bayer` already runs tiles on the application's Rayon pool. Tensor lifetimes
|
||||
//! are planned once; buffers and im2col storage are recycled between calls, independently for concurrent callers.
|
||||
|
||||
use crate::net::{Net, NetError, Op};
|
||||
use crate::run::{Error, TileRunner};
|
||||
use faer::{Parallelism, mat};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
const BLOCK: usize = 256;
|
||||
/// Cap all tensor buffers and convolution scratch for one tile together (512 MiB).
|
||||
const MAX_VALUES: usize = 128 * 1024 * 1024;
|
||||
const MAX_TILE: usize = 2048;
|
||||
const MAX_IDLE: usize = 1;
|
||||
|
||||
type Result<T> = std::result::Result<T, NetError>;
|
||||
fn bad(why: &str) -> NetError {
|
||||
NetError::Invalid(why.into())
|
||||
}
|
||||
fn count(c: usize, side: usize) -> Result<usize> {
|
||||
c.checked_mul(side)
|
||||
.and_then(|n| n.checked_mul(side))
|
||||
.filter(|&n| n > 0 && n <= MAX_VALUES)
|
||||
.ok_or_else(|| bad("a tensor has no size or is too large"))
|
||||
}
|
||||
fn zeros(n: usize) -> Result<Vec<f32>> {
|
||||
let mut v = Vec::new();
|
||||
v.try_reserve_exact(n).map_err(|_| bad("not enough memory for a denoise tile"))?;
|
||||
v.resize(n, 0.0);
|
||||
Ok(v)
|
||||
}
|
||||
fn sources(op: &Op) -> (usize, Option<usize>) {
|
||||
match op {
|
||||
Op::Conv(c) => (c.src, c.src2),
|
||||
Op::ConvT2(c) => (c.src, None),
|
||||
Op::MaxPool2 { src } | Op::DepthToSpace2 { src } => (*src, None),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Copy)]
|
||||
struct Tensor {
|
||||
slot: usize,
|
||||
side: usize,
|
||||
channels: usize,
|
||||
}
|
||||
struct Plan {
|
||||
net: Net,
|
||||
tensors: Vec<Tensor>,
|
||||
sizes: Vec<usize>,
|
||||
lower: usize,
|
||||
product: usize,
|
||||
up_weights: Vec<Option<Vec<f32>>>,
|
||||
}
|
||||
struct Workspace {
|
||||
buffers: Vec<Vec<f32>>,
|
||||
lower: Vec<f32>,
|
||||
product: Vec<f32>,
|
||||
}
|
||||
|
||||
/// A checked Net on square CHW tiles. Clones share immutable weights and the idle workspace cache.
|
||||
#[derive(Clone)]
|
||||
pub struct NetRunner {
|
||||
plan: Arc<Plan>,
|
||||
idle: Arc<Mutex<Vec<Workspace>>>,
|
||||
}
|
||||
|
||||
impl NetRunner {
|
||||
pub fn new(net: &Net, tile: usize) -> Result<Self> {
|
||||
if tile == 0
|
||||
|| tile > MAX_TILE
|
||||
|| !net.fits_tile(tile)
|
||||
|| net.in_channels != 4
|
||||
|| net.out_channels() != 3
|
||||
|| net.shape(0).is_none_or(|s| s.channels != net.in_channels)
|
||||
{
|
||||
return Err(bad("the tile or Bayer-to-RGB contract does not fit the network"));
|
||||
}
|
||||
let mut last = vec![0; net.ops().len() + 1];
|
||||
for (i, op) in net.ops().iter().enumerate() {
|
||||
let (a, b) = sources(op);
|
||||
for id in std::iter::once(a).chain(b) {
|
||||
*last.get_mut(id).ok_or_else(|| bad("missing tensor"))? = i + 1;
|
||||
}
|
||||
}
|
||||
let mut tensors = vec![Tensor { slot: 0, side: tile, channels: net.in_channels }];
|
||||
let mut sizes = vec![count(net.in_channels, tile)?];
|
||||
let mut busy = vec![true];
|
||||
let (mut lower, mut product) = (0, 0);
|
||||
let mut up_weights = Vec::new();
|
||||
for (i, op) in net.ops().iter().enumerate() {
|
||||
let (src, src2) = sources(op);
|
||||
let a = *tensors.get(src).ok_or_else(|| bad("missing tensor"))?;
|
||||
let (channels, side) = match op {
|
||||
Op::Conv(c) => {
|
||||
if let Some(s) = src2
|
||||
&& tensors.get(s).is_none_or(|b| b.side != a.side)
|
||||
{
|
||||
return Err(bad("a join has different sizes"));
|
||||
}
|
||||
(c.cout, a.side)
|
||||
}
|
||||
Op::ConvT2(c) => (c.cout, a.side.checked_mul(2).ok_or_else(|| bad("size overflow"))?),
|
||||
Op::MaxPool2 { .. } => {
|
||||
if a.side < 2 || !a.side.is_multiple_of(2) {
|
||||
return Err(bad("pooling does not divide the tile"));
|
||||
}
|
||||
(a.channels, a.side / 2)
|
||||
}
|
||||
Op::DepthToSpace2 { .. } => (a.channels / 4, a.side.checked_mul(2).ok_or_else(|| bad("size overflow"))?),
|
||||
};
|
||||
let n = count(channels, side)?;
|
||||
let slot = busy
|
||||
.iter()
|
||||
.zip(&sizes)
|
||||
.enumerate()
|
||||
.filter(|(_, (used, _))| !**used)
|
||||
.min_by_key(|(_, (_, size))| (n.saturating_sub(**size), **size))
|
||||
.map_or(busy.len(), |(i, _)| i);
|
||||
if slot == sizes.len() {
|
||||
sizes.push(n);
|
||||
busy.push(true);
|
||||
} else {
|
||||
let sz = sizes.get_mut(slot).ok_or_else(|| bad("missing buffer"))?;
|
||||
*sz = (*sz).max(n);
|
||||
*busy.get_mut(slot).ok_or_else(|| bad("missing buffer"))? = true;
|
||||
}
|
||||
tensors.push(Tensor { slot, side, channels });
|
||||
// Release sources only after the output has its own buffer (also handles reading the same tensor twice).
|
||||
for id in std::iter::once(src).chain(src2) {
|
||||
if last.get(id) == Some(&(i + 1)) {
|
||||
let t = tensors.get(id).ok_or_else(|| bad("missing tensor"))?;
|
||||
*busy.get_mut(t.slot).ok_or_else(|| bad("missing buffer"))? = false;
|
||||
}
|
||||
}
|
||||
let (k, m, positions) = match op {
|
||||
Op::Conv(c) => (c.cin * c.k * c.k, c.cout, a.side * a.side),
|
||||
Op::ConvT2(c) => (c.cin, c.cout * 4, a.side * a.side),
|
||||
_ => (0, 0, 0),
|
||||
};
|
||||
lower = lower.max(k.saturating_mul(BLOCK.min(positions)));
|
||||
product = product.max(m.saturating_mul(BLOCK.min(positions)));
|
||||
let packed = if let Op::ConvT2(c) = op {
|
||||
let mut w = zeros(c.weight.len())?;
|
||||
// [cin][cout][dy][dx] -> [cout*4][cin], so an up-convolution is one GEMM and a shuffle.
|
||||
for (ci, row) in c.weight.chunks_exact(c.cout * 4).enumerate() {
|
||||
for (co, &v) in row.iter().enumerate() {
|
||||
*w.get_mut(co * c.cin + ci).ok_or_else(|| bad("invalid transposed weights"))? = v;
|
||||
}
|
||||
}
|
||||
Some(w)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
up_weights.push(packed);
|
||||
}
|
||||
let total = sizes
|
||||
.iter()
|
||||
.try_fold(lower.saturating_add(product), |n, &s| n.checked_add(s))
|
||||
.filter(|&n| n <= MAX_VALUES)
|
||||
.ok_or_else(|| bad("the tile needs too much working memory"))?;
|
||||
let _ = total;
|
||||
Ok(Self { plan: Arc::new(Plan { net: net.clone(), tensors, sizes, lower, product, up_weights }), idle: Arc::new(Mutex::new(Vec::new())) })
|
||||
}
|
||||
/// Immutable data for the GPU runner, without re-reading the model.
|
||||
pub fn net(&self) -> &Net {
|
||||
&self.plan.net
|
||||
}
|
||||
pub fn bytes_per_tile(&self) -> usize {
|
||||
(self.plan.sizes.iter().sum::<usize>() + self.plan.lower + self.plan.product) * 4
|
||||
}
|
||||
fn workspace(&self) -> Result<Workspace> {
|
||||
if let Some(w) = self.idle.lock().unwrap_or_else(std::sync::PoisonError::into_inner).pop() {
|
||||
return Ok(w);
|
||||
}
|
||||
Ok(Workspace {
|
||||
buffers: self.plan.sizes.iter().map(|&n| zeros(n)).collect::<Result<_>>()?,
|
||||
lower: zeros(self.plan.lower)?,
|
||||
product: zeros(self.plan.product)?,
|
||||
})
|
||||
}
|
||||
fn execute(&self, input: &[f32], w: &mut Workspace) -> Result<Vec<f32>> {
|
||||
let first = self.plan.tensors.first().ok_or_else(|| bad("no input"))?;
|
||||
let want = count(first.channels, first.side)?;
|
||||
w.buffers.first_mut().and_then(|b| b.get_mut(..want)).ok_or_else(|| bad("missing input buffer"))?.copy_from_slice(input);
|
||||
let profile = std::env::var_os("LIGHTCRAFT_PROFILE").is_some();
|
||||
for (i, op) in self.plan.net.ops().iter().enumerate() {
|
||||
let started = web_time::Instant::now();
|
||||
let out = *self.plan.tensors.get(i + 1).ok_or_else(|| bad("missing output tensor"))?;
|
||||
let (before, rest) = w.buffers.split_at_mut_checked(out.slot).ok_or_else(|| bad("missing output buffer"))?;
|
||||
let (dest, after) = rest.split_first_mut().ok_or_else(|| bad("missing output buffer"))?;
|
||||
let get = |id: usize| -> Result<(&[f32], Tensor)> {
|
||||
let t = *self.plan.tensors.get(id).ok_or_else(|| bad("missing source tensor"))?;
|
||||
let data = if t.slot < out.slot {
|
||||
before.get(t.slot)
|
||||
} else {
|
||||
after.get(t.slot.checked_sub(out.slot + 1).ok_or_else(|| bad("aliased buffers"))?)
|
||||
}
|
||||
.and_then(|b| b.get(..count(t.channels, t.side).ok()?))
|
||||
.ok_or_else(|| bad("missing source buffer"))?;
|
||||
Ok((data, t))
|
||||
};
|
||||
let (src, src2) = sources(op);
|
||||
let (a, shape) = get(src)?;
|
||||
let b = src2.map(get).transpose()?.map(|v| v.0);
|
||||
let dest = dest.get_mut(..count(out.channels, out.side)?).ok_or_else(|| bad("short output buffer"))?;
|
||||
match op {
|
||||
Op::Conv(c) => conv(c, a, b, shape.side, dest, &mut w.lower, &mut w.product)?,
|
||||
Op::ConvT2(c) => up(
|
||||
c,
|
||||
a,
|
||||
shape.side,
|
||||
dest,
|
||||
self.plan.up_weights.get(i).and_then(Option::as_deref).ok_or_else(|| bad("missing up weights"))?,
|
||||
&mut w.lower,
|
||||
&mut w.product,
|
||||
)?,
|
||||
Op::MaxPool2 { .. } => pool(a, shape.side, dest)?,
|
||||
Op::DepthToSpace2 { .. } => shuffle(a, shape.side, dest)?,
|
||||
}
|
||||
if dest.iter().any(|v| !v.is_finite()) {
|
||||
return Err(bad("the model produced non-finite values"));
|
||||
}
|
||||
if profile {
|
||||
eprintln!("[denoise-cpu] layer {i}: {}x{}x{} {:.2} ms", out.channels, out.side, out.side, started.elapsed().as_secs_f64() * 1e3);
|
||||
}
|
||||
}
|
||||
let t = self.plan.tensors.last().ok_or_else(|| bad("no output"))?;
|
||||
let data = w.buffers.get(t.slot).and_then(|b| b.get(..count(t.channels, t.side).ok()?)).ok_or_else(|| bad("missing result"))?;
|
||||
let mut out = zeros(data.len())?;
|
||||
out.copy_from_slice(data);
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
impl TileRunner for NetRunner {
|
||||
fn run(&self, input: &[f32]) -> std::result::Result<Vec<f32>, Error> {
|
||||
let first = self.plan.tensors.first().ok_or_else(|| Error::Runtime("missing input shape".into()))?;
|
||||
let want = count(first.channels, first.side).map_err(|e| Error::Runtime(e.to_string()))?;
|
||||
if input.len() != want || input.iter().any(|v| !v.is_finite()) {
|
||||
return Err(Error::Input("the tile has the wrong size or non-finite values".into()));
|
||||
}
|
||||
let mut w = self.workspace().map_err(|e| Error::Runtime(e.to_string()))?;
|
||||
let result = self.execute(input, &mut w).map_err(|e| Error::Runtime(e.to_string()));
|
||||
let mut idle = self.idle.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if idle.len() < MAX_IDLE {
|
||||
idle.push(w);
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
fn activate(x: f32, bias: f32, alpha: Option<f32>) -> f32 {
|
||||
let x = x + bias;
|
||||
if x < 0.0 { x * alpha.unwrap_or(1.0) } else { x }
|
||||
}
|
||||
|
||||
/// All views are over exact, checked slices. The matrix library's assertions therefore never see file-derived sizes.
|
||||
fn multiply(w: &[f32], x: &[f32], y: &mut [f32], m: usize, k: usize, n: usize) -> Result<()> {
|
||||
if m.checked_mul(k) != Some(w.len()) || k.checked_mul(n) != Some(x.len()) || m.checked_mul(n) != Some(y.len()) || m == 0 || k == 0 || n == 0 {
|
||||
return Err(bad("invalid matrix dimensions"));
|
||||
}
|
||||
faer::mul::matmul(
|
||||
mat::from_row_major_slice_mut::<f32>(y, m, n),
|
||||
mat::from_row_major_slice::<f32>(w, m, k),
|
||||
mat::from_row_major_slice::<f32>(x, k, n),
|
||||
None,
|
||||
1.0,
|
||||
Parallelism::None,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn lower(a: &[f32], b: Option<&[f32]>, side: usize, k: usize, start: usize, n: usize, dest: &mut [f32]) -> Result<()> {
|
||||
let pad = k / 2;
|
||||
let pixels = side * side;
|
||||
// Whole row segments are copied, so bounds checks and edge branches stay out of the multiply loop.
|
||||
for (ci, plane) in a.chunks_exact(pixels).chain(b.into_iter().flat_map(|b| b.chunks_exact(pixels))).enumerate() {
|
||||
for ky in 0..k {
|
||||
for kx in 0..k {
|
||||
let row = (ci * k * k + ky * k + kx) * n;
|
||||
let out = dest.get_mut(row..row + n).ok_or_else(|| bad("short im2col buffer"))?;
|
||||
let mut at = 0;
|
||||
while at < n {
|
||||
let p = start + at;
|
||||
let (y, x) = (p / side, p % side);
|
||||
let len = (side - x).min(n - at);
|
||||
if let Some(yy) = y.checked_add(ky).and_then(|v| v.checked_sub(pad)).filter(|&v| v < side) {
|
||||
let lo = x.max(pad.saturating_sub(kx));
|
||||
let hi = (x + len).min((side + pad).saturating_sub(kx));
|
||||
if lo < hi {
|
||||
let from = yy * side + lo + kx - pad;
|
||||
let to = at + lo - x;
|
||||
out.get_mut(at..to).ok_or_else(|| bad("short im2col padding"))?.fill(0.0);
|
||||
out.get_mut(to..to + hi - lo)
|
||||
.ok_or_else(|| bad("short im2col row"))?
|
||||
.copy_from_slice(plane.get(from..from + hi - lo).ok_or_else(|| bad("short source row"))?);
|
||||
out.get_mut(to + hi - lo..at + len).ok_or_else(|| bad("short im2col padding"))?.fill(0.0);
|
||||
} else {
|
||||
out.get_mut(at..at + len).ok_or_else(|| bad("short im2col row"))?.fill(0.0);
|
||||
}
|
||||
} else {
|
||||
out.get_mut(at..at + len).ok_or_else(|| bad("short im2col row"))?.fill(0.0);
|
||||
}
|
||||
at += len;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn conv(c: &crate::net::Conv, a: &[f32], b: Option<&[f32]>, side: usize, dest: &mut [f32], scratch: &mut [f32], product: &mut [f32]) -> Result<()> {
|
||||
let pixels = side * side;
|
||||
let k = c.cin * c.k * c.k;
|
||||
for start in (0..pixels).step_by(BLOCK) {
|
||||
let n = BLOCK.min(pixels - start);
|
||||
let x = scratch.get_mut(..k * n).ok_or_else(|| bad("short im2col storage"))?;
|
||||
lower(a, b, side, c.k, start, n, x)?;
|
||||
let y = product.get_mut(..c.cout * n).ok_or_else(|| bad("short product storage"))?;
|
||||
multiply(&c.weight, x, y, c.cout, k, n)?;
|
||||
for ((row, &bias), out) in y.chunks_exact(n).zip(&c.bias).zip(dest.chunks_exact_mut(pixels)) {
|
||||
let to = out.get_mut(start..start + n).ok_or_else(|| bad("short convolution output"))?;
|
||||
for (d, &v) in to.iter_mut().zip(row) {
|
||||
*d = activate(v, bias, c.leaky);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn up(c: &crate::net::ConvT2, a: &[f32], side: usize, dest: &mut [f32], weights: &[f32], scratch: &mut [f32], product: &mut [f32]) -> Result<()> {
|
||||
let pixels = side * side;
|
||||
let out_side = 2 * side;
|
||||
for start in (0..pixels).step_by(BLOCK) {
|
||||
let n = BLOCK.min(pixels - start);
|
||||
let x = scratch.get_mut(..c.cin * n).ok_or_else(|| bad("short up input storage"))?;
|
||||
for (to, from) in x.chunks_exact_mut(n).zip(a.chunks_exact(pixels)) {
|
||||
to.copy_from_slice(from.get(start..start + n).ok_or_else(|| bad("short up input"))?);
|
||||
}
|
||||
let y = product.get_mut(..c.cout * 4 * n).ok_or_else(|| bad("short up product storage"))?;
|
||||
multiply(weights, x, y, c.cout * 4, c.cin, n)?;
|
||||
for (co, (&bias, out)) in c.bias.iter().zip(dest.chunks_exact_mut(4 * pixels)).enumerate() {
|
||||
for d in 0..4 {
|
||||
let row = y.get((4 * co + d) * n..(4 * co + d + 1) * n).ok_or_else(|| bad("short up product"))?;
|
||||
for (j, &v) in row.iter().enumerate() {
|
||||
let p = start + j;
|
||||
let at = (2 * (p / side) + d / 2) * out_side + 2 * (p % side) + d % 2;
|
||||
*out.get_mut(at).ok_or_else(|| bad("short up output"))? = activate(v, bias, c.leaky);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn pool(a: &[f32], side: usize, dest: &mut [f32]) -> Result<()> {
|
||||
let s = side / 2;
|
||||
for (input, out) in a.chunks_exact(side * side).zip(dest.chunks_exact_mut(s * s)) {
|
||||
for (y, row) in out.chunks_exact_mut(s).enumerate() {
|
||||
let lo = input.get(2 * y * side..(2 * y + 1) * side).ok_or_else(|| bad("short pool input"))?;
|
||||
let hi = input.get((2 * y + 1) * side..(2 * y + 2) * side).ok_or_else(|| bad("short pool input"))?;
|
||||
for ((d, lo), hi) in row.iter_mut().zip(lo.as_chunks::<2>().0).zip(hi.as_chunks::<2>().0) {
|
||||
let [a, b] = *lo;
|
||||
let [c, e] = *hi;
|
||||
*d = a.max(b).max(c).max(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn shuffle(a: &[f32], side: usize, dest: &mut [f32]) -> Result<()> {
|
||||
let pixels = side * side;
|
||||
for (input, out) in a.chunks_exact(4 * pixels).zip(dest.chunks_exact_mut(4 * pixels)) {
|
||||
for d in 0..4 {
|
||||
let plane = input.get(d * pixels..(d + 1) * pixels).ok_or_else(|| bad("short shuffle input"))?;
|
||||
for (y, row) in plane.chunks_exact(side).enumerate() {
|
||||
let at = (2 * y + d / 2) * 2 * side;
|
||||
let to = out.get_mut(at..at + 2 * side).ok_or_else(|| bad("short shuffle output"))?;
|
||||
for (to, &v) in to.as_chunks_mut::<2>().0.iter_mut().zip(row) {
|
||||
if let Some(t) = to.get_mut(d % 2) {
|
||||
*t = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,151 @@
|
||||
use super::*;
|
||||
use crate::{
|
||||
net::{Conv, ConvT2},
|
||||
reference, synthetic,
|
||||
};
|
||||
|
||||
fn random(n: usize, seed: u32) -> Vec<f32> {
|
||||
let mut s = seed | 1;
|
||||
(0..n)
|
||||
.map(|_| {
|
||||
s ^= s << 13;
|
||||
s ^= s >> 17;
|
||||
s ^= s << 5;
|
||||
(s >> 8) as f32 / (1u32 << 23) as f32 - 1.0
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
fn net(tile: usize, ch: u64, depth: usize, seed: u64) -> Net {
|
||||
let p = std::env::temp_dir().join(format!("lc-purecpu-{}-{tile}-{ch}-{depth}-{seed}.onnx", std::process::id()));
|
||||
std::fs::write(&p, synthetic::unet_onnx(tile as u64, ch, depth, seed)).unwrap();
|
||||
let n = crate::onnx::read(&p).unwrap();
|
||||
std::fs::remove_file(p).unwrap();
|
||||
n
|
||||
}
|
||||
fn agree(a: &[f32], b: &[f32]) {
|
||||
assert_eq!(a.len(), b.len());
|
||||
assert!(a.iter().chain(b).all(|v| v.is_finite()));
|
||||
let scale = b.iter().fold(1e-6f32, |m, v| m.max(v.abs()));
|
||||
let error = a.iter().zip(b).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale;
|
||||
assert!(error <= 1e-3, "relative maximum error {error:e}");
|
||||
}
|
||||
#[test]
|
||||
fn all_operators_match_scalar_on_random_nets_and_odd_and_tiny_shapes() {
|
||||
for (t, ch, d) in [(1, 3, 0), (3, 5, 0), (5, 7, 0), (6, 3, 1), (20, 5, 2), (36, 9, 2), (64, 8, 3)] {
|
||||
for seed in [1, 73] {
|
||||
let n = net(t, ch, d, seed);
|
||||
let r = NetRunner::new(&n, t).unwrap();
|
||||
let x = random(4 * t * t, seed as u32);
|
||||
let want = reference::run(&n, t, &x).unwrap();
|
||||
agree(&r.run(&x).unwrap(), &want);
|
||||
assert_eq!(r.run(&x).unwrap(), r.run(&x).unwrap());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_values_fail_before_execution_and_workspace_sizes_are_bounded() {
|
||||
let mut n = net(4, 3, 1, 11);
|
||||
for t in [0, 1, 3, usize::MAX, 4096] {
|
||||
assert!(NetRunner::new(&n, t).is_err());
|
||||
}
|
||||
for ch in [0, 3, usize::MAX] {
|
||||
n.in_channels = ch;
|
||||
assert!(NetRunner::new(&n, 4).is_err());
|
||||
}
|
||||
let n = Net::new(
|
||||
4,
|
||||
vec![
|
||||
Op::Conv(Conv { src: 0, src2: None, cin: 4, cout: 4096, k: 1, weight: vec![0.1; 4 * 4096], bias: vec![0.0; 4096], leaky: None }),
|
||||
Op::Conv(Conv { src: 1, src2: None, cin: 4096, cout: 12, k: 1, weight: vec![0.1; 4096 * 12], bias: vec![0.0; 12], leaky: None }),
|
||||
Op::DepthToSpace2 { src: 2 },
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
assert!(NetRunner::new(&n, 2048).is_err());
|
||||
let ops = (0..100).map(|i| Op::MaxPool2 { src: i }).collect();
|
||||
let n = Net::new(4, ops).unwrap();
|
||||
assert!(NetRunner::new(&n, 16).is_err());
|
||||
let c = Conv { src: 0, src2: None, cin: 4, cout: 12, k: 1, weight: vec![0.25; 48], bias: vec![0.0; 12], leaky: None };
|
||||
for mode in 0..6 {
|
||||
let mut bad = c.clone();
|
||||
match mode {
|
||||
0 => bad.src = usize::MAX,
|
||||
1 => bad.weight.pop().map(|_| ()).unwrap(),
|
||||
2 => bad.bias[0] = f32::NAN,
|
||||
3 => bad.leaky = Some(f32::INFINITY),
|
||||
4 => bad.cout = usize::MAX,
|
||||
_ => bad.k = usize::MAX,
|
||||
}
|
||||
assert!(Net::new(4, vec![Op::Conv(bad), Op::DepthToSpace2 { src: 1 }]).is_err());
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn bad_tiles_and_arithmetic_overflow_are_errors_and_do_not_poison_reuse() {
|
||||
let c = Conv { src: 0, src2: None, cin: 4, cout: 12, k: 1, weight: vec![f32::MAX; 48], bias: vec![0.0; 12], leaky: None };
|
||||
let n = Net::new(4, vec![Op::Conv(c), Op::DepthToSpace2 { src: 1 }]).unwrap();
|
||||
let r = NetRunner::new(&n, 1).unwrap();
|
||||
assert!(r.run(&[]).is_err());
|
||||
assert!(r.run(&[f32::NAN; 4]).is_err());
|
||||
assert!(r.run(&[f32::INFINITY; 4]).is_err());
|
||||
assert_eq!(r.idle.lock().unwrap().len(), 0, "invalid input never allocates a workspace");
|
||||
assert!(r.run(&[2.0; 4]).is_err());
|
||||
assert_eq!(r.run(&[0.0; 4]).unwrap(), vec![0.0; 12]);
|
||||
// Matrix shape checks are the boundary in front of the dependency's asserting constructors.
|
||||
assert!(multiply(&[], &[], &mut [], usize::MAX, 2, 2).is_err());
|
||||
}
|
||||
#[test]
|
||||
fn shared_runner_serves_concurrent_tiles_without_cross_contamination() {
|
||||
let n = net(12, 5, 1, 97);
|
||||
let r = NetRunner::new(&n, 12).unwrap();
|
||||
let inputs: Vec<_> = (0..8).map(|s| random(4 * 12 * 12, s)).collect();
|
||||
let answers: Vec<_> = inputs.iter().map(|x| reference::run(&n, 12, x).unwrap()).collect();
|
||||
std::thread::scope(|scope| {
|
||||
let mut handles = Vec::new();
|
||||
for (x, want) in inputs.iter().zip(&answers) {
|
||||
let r = &r;
|
||||
handles.push(scope.spawn(move || {
|
||||
for _ in 0..3 {
|
||||
agree(&r.run(x).unwrap(), want);
|
||||
}
|
||||
}));
|
||||
}
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
});
|
||||
}
|
||||
#[test]
|
||||
fn join_may_read_the_same_tensor_twice_and_leaky_up_convolution_is_supported() {
|
||||
let c = ConvT2 { src: 0, cin: 4, cout: 3, weight: random(48, 2), bias: vec![0.1, -0.1, 0.0], leaky: Some(0.13) };
|
||||
let n = Net::new(
|
||||
4,
|
||||
vec![
|
||||
Op::ConvT2(c),
|
||||
Op::Conv(Conv { src: 1, src2: Some(1), cin: 6, cout: 3, k: 3, weight: random(162, 3), bias: vec![0.0; 3], leaky: Some(0.0) }),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
let x = random(4 * 3 * 3, 7);
|
||||
agree(&NetRunner::new(&n, 3).unwrap().run(&x).unwrap(), &reference::run(&n, 3, &x).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reused_im2col_clears_all_edge_padding_between_images() {
|
||||
for t in [1, 3, 5, 36] {
|
||||
let n = net(t, 5, 0, 121);
|
||||
let r = NetRunner::new(&n, t).unwrap();
|
||||
for x in [vec![1.0; 4 * t * t], vec![0.0; 4 * t * t], random(4 * t * t, 9)] {
|
||||
agree(&r.run(&x).unwrap(), &reference::run(&n, t, &x).unwrap());
|
||||
}
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
#[ignore = "needs LC_DENOISE_MODEL; one tile with LIGHTCRAFT_PROFILE=1"]
|
||||
fn profile_one_real_tile() {
|
||||
let model = std::env::var_os("LC_DENOISE_MODEL").expect("set LC_DENOISE_MODEL");
|
||||
let m = crate::known::find("rawnind-bayer").unwrap().manifest;
|
||||
let r = crate::runtime::CpuRunner::load(std::path::Path::new(&model), &m).unwrap();
|
||||
println!("workspace {:.1} MB", r.bytes_per_tile() as f64 / 1e6);
|
||||
r.run(&crate::runtime::check_tile(512)).unwrap();
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
//! AI denoise for LightCraft, as a step between the sensor and the demosaic that the rest of the develop
|
||||
//! pipeline never sees: the raw file stays untouched, the denoised picture is cached data that can always be
|
||||
//! made again, and the amount is an ordinary develop setting.
|
||||
//!
|
||||
//! - [`manifest`]: what a denoise model takes and gives, and who may use it (untrusted JSON, validated).
|
||||
//! - [`known`] and [`archive`]: the models LightCraft can fetch, and taking the model file out of the `.zip` it comes in.
|
||||
//! - [`bayer`]: Bayer layouts and the packing of a mosaic into the four planes a model takes.
|
||||
//! - [`tiles`] and [`run`]: cutting a picture into overlapping tiles for a fixed-size model, bringing each tile's
|
||||
//! output scale back to the input's, keeping clipped highlights as they were, and blending the tiles together.
|
||||
//! The model itself is a [`run::TileRunner`], so all of this is tested without one.
|
||||
//! - [`product`]: the cache file the denoised picture is kept in (half floats, compressed in strips) and the
|
||||
//! blend with the plain demosaic that the Amount slider controls.
|
||||
//! - [`cpu`] and [`runtime`]: a single-thread pure-Rust CPU runner and its installation self-test.
|
||||
//! - [`net`] and [`onnx`]: checked plain network data and bounded ONNX decoding, shared by CPU and GPU.
|
||||
//!
|
||||
//! Everything here treats model files, manifests and cache files as hostile input: sizes are capped, numbers must
|
||||
//! be finite, and a malformed file is an error, never a panic.
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
#[cfg(feature = "runtime")]
|
||||
pub mod cpu;
|
||||
|
||||
pub mod net;
|
||||
pub mod onnx;
|
||||
mod onnx_proto;
|
||||
|
||||
pub mod reference;
|
||||
|
||||
#[cfg(feature = "runtime")]
|
||||
pub mod runtime;
|
||||
#[doc(hidden)]
|
||||
pub mod synthetic;
|
||||
|
||||
pub use lightcraft_denoise_core::*;
|
||||
|
||||
pub mod synthetic_proto;
|
||||
@@ -0,0 +1,345 @@
|
||||
//! A denoise network as plain data: the handful of layer kinds a U-Net denoiser is made of, with their weights.
|
||||
//!
|
||||
//! [`onnx`](crate::onnx) reads the file into this description, shared by the pure-Rust CPU and GPU runners.
|
||||
//! Unsupported operators, groups, dilation or activations are reported as errors when loading the model.
|
||||
//!
|
||||
//! The description is checked ([`Net::new`]) because it comes from a stranger's file: channel counts must chain,
|
||||
//! weights must have the sizes the shapes need and be finite, and the depth must stay small.
|
||||
|
||||
/// Most layers a network may have.
|
||||
pub const MAX_OPS: usize = 512;
|
||||
/// Most channels a layer may have.
|
||||
pub const MAX_CHANNELS: usize = 4096;
|
||||
/// Most weights (numbers) a network may have in all.
|
||||
pub const MAX_WEIGHTS: usize = 128 * 1024 * 1024;
|
||||
|
||||
#[derive(Debug, PartialEq, thiserror::Error)]
|
||||
pub enum NetError {
|
||||
/// The model uses something this denoise network description cannot express.
|
||||
#[error("not supported by the denoise runners: {0}")]
|
||||
Unsupported(String),
|
||||
/// The model is not a well-formed network.
|
||||
#[error("not a valid network: {0}")]
|
||||
Invalid(String),
|
||||
}
|
||||
|
||||
fn invalid<T>(why: impl Into<String>) -> Result<T, NetError> {
|
||||
Err(NetError::Invalid(why.into()))
|
||||
}
|
||||
|
||||
/// A tensor is named by its number: 0 is the network's input, and layer `i` makes tensor `i + 1`.
|
||||
pub type TensorId = usize;
|
||||
|
||||
/// A convolution with stride 1 and "same" padding (`k` is 1 or 3), over one tensor or two joined along the channels
|
||||
/// (`src` first), then a bias and optionally a leaky ReLU.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Conv {
|
||||
pub src: TensorId,
|
||||
/// A second source, whose channels follow `src`'s (a skip connection's concatenation).
|
||||
pub src2: Option<TensorId>,
|
||||
pub cin: usize,
|
||||
pub cout: usize,
|
||||
pub k: usize,
|
||||
/// `[cout][cin][k][k]`, as ONNX has it.
|
||||
pub weight: Vec<f32>,
|
||||
pub bias: Vec<f32>,
|
||||
/// `Some(alpha)`: a leaky ReLU with this slope follows.
|
||||
pub leaky: Option<f32>,
|
||||
}
|
||||
|
||||
/// A transposed convolution with a 2 × 2 kernel and stride 2: every input pixel becomes a 2 × 2 block.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ConvT2 {
|
||||
pub src: TensorId,
|
||||
pub cin: usize,
|
||||
pub cout: usize,
|
||||
/// `[cin][cout][2][2]`, as ONNX has it.
|
||||
pub weight: Vec<f32>,
|
||||
pub bias: Vec<f32>,
|
||||
pub leaky: Option<f32>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum Op {
|
||||
Conv(Conv),
|
||||
ConvT2(ConvT2),
|
||||
/// 2 × 2 maximum, stride 2.
|
||||
MaxPool2 {
|
||||
src: TensorId,
|
||||
},
|
||||
/// Depth-to-space with block size 2 in "CRD" order: output channel `c` at `(2y + dy, 2x + dx)` is input channel
|
||||
/// `4c + 2dy + dx` at `(y, x)`.
|
||||
DepthToSpace2 {
|
||||
src: TensorId,
|
||||
},
|
||||
}
|
||||
|
||||
/// What a tensor looks like: its channels, and its resolution as a power of two below the input's (a negative
|
||||
/// level is above it).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Shape {
|
||||
pub channels: usize,
|
||||
pub level: i32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Net {
|
||||
pub in_channels: usize,
|
||||
ops: Vec<Op>,
|
||||
shapes: Vec<Shape>,
|
||||
}
|
||||
|
||||
impl Net {
|
||||
/// Check `ops` (layer `i` makes tensor `i + 1`) over an input of `in_channels` and describe every tensor.
|
||||
pub fn new(in_channels: usize, ops: Vec<Op>) -> Result<Net, NetError> {
|
||||
if in_channels == 0 || in_channels > MAX_CHANNELS {
|
||||
return invalid("the input has no channels or too many");
|
||||
}
|
||||
if ops.is_empty() || ops.len() > MAX_OPS {
|
||||
return invalid("the network has no layers or too many");
|
||||
}
|
||||
let mut shapes = vec![Shape { channels: in_channels, level: 0 }];
|
||||
let mut weights = 0usize;
|
||||
for (i, op) in ops.iter().enumerate() {
|
||||
let shape_of = |t: TensorId| {
|
||||
shapes
|
||||
.get(t)
|
||||
.copied()
|
||||
.filter(|_| t <= i)
|
||||
.ok_or_else(|| NetError::Invalid(format!("layer {i} reads tensor {t}, which does not exist yet")))
|
||||
};
|
||||
let out = match op {
|
||||
Op::Conv(c) => {
|
||||
let a = shape_of(c.src)?;
|
||||
let (cin, level) = match c.src2 {
|
||||
Some(s2) => {
|
||||
let b = shape_of(s2)?;
|
||||
if b.level != a.level {
|
||||
return invalid(format!("layer {i} joins tensors of different resolutions"));
|
||||
}
|
||||
(a.channels + b.channels, a.level)
|
||||
}
|
||||
None => (a.channels, a.level),
|
||||
};
|
||||
if c.cin != cin || c.cout == 0 || c.cout > MAX_CHANNELS || !(c.k == 1 || c.k == 3) {
|
||||
return invalid(format!(
|
||||
"layer {i}: a convolution of {} to {} channels with a {}-wide kernel does not fit",
|
||||
c.cin, c.cout, c.k
|
||||
));
|
||||
}
|
||||
check_weights(
|
||||
i,
|
||||
&c.weight,
|
||||
c.cout.checked_mul(cin).and_then(|n| n.checked_mul(c.k * c.k)),
|
||||
&c.bias,
|
||||
c.cout,
|
||||
c.leaky,
|
||||
&mut weights,
|
||||
)?;
|
||||
Shape { channels: c.cout, level }
|
||||
}
|
||||
Op::ConvT2(c) => {
|
||||
let a = shape_of(c.src)?;
|
||||
if c.cin != a.channels || c.cout == 0 || c.cout > MAX_CHANNELS {
|
||||
return invalid(format!("layer {i}: a transposed convolution of {} to {} channels does not fit", c.cin, c.cout));
|
||||
}
|
||||
check_weights(i, &c.weight, c.cin.checked_mul(c.cout).and_then(|n| n.checked_mul(4)), &c.bias, c.cout, c.leaky, &mut weights)?;
|
||||
Shape { channels: c.cout, level: a.level - 1 }
|
||||
}
|
||||
Op::MaxPool2 { src } => {
|
||||
let a = shape_of(*src)?;
|
||||
Shape { channels: a.channels, level: a.level + 1 }
|
||||
}
|
||||
Op::DepthToSpace2 { src } => {
|
||||
let a = shape_of(*src)?;
|
||||
if !a.channels.is_multiple_of(4) {
|
||||
return invalid(format!("layer {i}: depth-to-space needs a multiple of 4 channels"));
|
||||
}
|
||||
Shape { channels: a.channels / 4, level: a.level - 1 }
|
||||
}
|
||||
};
|
||||
shapes.push(out);
|
||||
}
|
||||
Ok(Net { in_channels, ops, shapes })
|
||||
}
|
||||
|
||||
/// The layers: layer `i` makes tensor `i + 1`.
|
||||
pub fn ops(&self) -> &[Op] {
|
||||
&self.ops
|
||||
}
|
||||
|
||||
/// The shape of tensor `t` (0 is the input).
|
||||
pub fn shape(&self, t: TensorId) -> Option<Shape> {
|
||||
self.shapes.get(t).copied()
|
||||
}
|
||||
|
||||
/// The tensor the network gives (the last layer's).
|
||||
pub fn output(&self) -> TensorId {
|
||||
self.ops.len()
|
||||
}
|
||||
|
||||
pub fn out_channels(&self) -> usize {
|
||||
self.shapes.last().map_or(0, |s| s.channels)
|
||||
}
|
||||
|
||||
/// The deepest resolution level: a tile must be a multiple of `2^this` cells wide.
|
||||
pub fn depth(&self) -> u32 {
|
||||
self.shapes.iter().map(|s| s.level.max(0) as u32).max().unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Can the network run on a `tile × tile` input and give `[out_channels, 2·tile, 2·tile]`? (Pooling must divide
|
||||
/// the tile, and the output must be one level above the input.)
|
||||
pub fn fits_tile(&self, tile: usize) -> bool {
|
||||
let Some(multiple) = 1usize.checked_shl(self.depth()) else { return false };
|
||||
tile > 0 && tile.is_multiple_of(multiple) && self.shapes.last().is_some_and(|s| s.level == -1)
|
||||
}
|
||||
|
||||
/// Multiply-adds for one tile, for estimating time.
|
||||
pub fn macs(&self, tile: usize) -> u64 {
|
||||
let pixels = |level: i32| -> u64 {
|
||||
let side = if level >= 0 {
|
||||
tile.checked_shr(level as u32).unwrap_or(0)
|
||||
} else {
|
||||
tile.saturating_mul(1usize.checked_shl(level.unsigned_abs()).unwrap_or(usize::MAX))
|
||||
};
|
||||
(side as u64).saturating_mul(side as u64)
|
||||
};
|
||||
self.ops
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, op)| {
|
||||
let out = self.shapes.get(i + 1).map_or(0, |s| pixels(s.level));
|
||||
match op {
|
||||
Op::Conv(c) => out.saturating_mul((c.cin * c.cout * c.k * c.k) as u64),
|
||||
// every input pixel makes a 2 × 2 block: as many multiply-adds as the 2 × 2-wide kernel has weights
|
||||
Op::ConvT2(c) => pixels(self.shapes.get(c.src).map_or(0, |s| s.level)).saturating_mul((c.cin * c.cout * 4) as u64),
|
||||
_ => 0,
|
||||
}
|
||||
})
|
||||
.fold(0u64, u64::saturating_add)
|
||||
}
|
||||
}
|
||||
|
||||
fn check_weights(
|
||||
layer: usize,
|
||||
weight: &[f32],
|
||||
want: Option<usize>,
|
||||
bias: &[f32],
|
||||
cout: usize,
|
||||
leaky: Option<f32>,
|
||||
total: &mut usize,
|
||||
) -> Result<(), NetError> {
|
||||
if want != Some(weight.len()) || bias.len() != cout {
|
||||
return invalid(format!("layer {layer}: the weights are not the size its shape needs"));
|
||||
}
|
||||
*total = total.saturating_add(weight.len()).saturating_add(bias.len());
|
||||
if *total > MAX_WEIGHTS {
|
||||
return invalid("the network has too many weights");
|
||||
}
|
||||
if weight.iter().chain(bias).any(|v| !v.is_finite()) || leaky.is_some_and(|a| !a.is_finite()) {
|
||||
return invalid(format!("layer {layer}: a weight is not a finite number"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn conv(src: TensorId, src2: Option<TensorId>, cin: usize, cout: usize, k: usize) -> Op {
|
||||
Op::Conv(Conv { src, src2, cin, cout, k, weight: vec![0.1; cout * cin * k * k], bias: vec![0.0; cout], leaky: Some(0.2) })
|
||||
}
|
||||
|
||||
fn convt(src: TensorId, cin: usize, cout: usize) -> Op {
|
||||
Op::ConvT2(ConvT2 { src, cin, cout, weight: vec![0.1; cin * cout * 4], bias: vec![0.0; cout], leaky: None })
|
||||
}
|
||||
|
||||
/// A two-level network of the shape the denoisers have: 4 → 8 channels, pool, 16 channels, up, join, 12, and the
|
||||
/// 1 × 1 convolution to the colours.
|
||||
fn unet() -> Vec<Op> {
|
||||
vec![
|
||||
conv(0, None, 4, 8, 3), // 1: 8 ch, level 0
|
||||
Op::MaxPool2 { src: 1 }, // 2: level 1
|
||||
conv(2, None, 8, 16, 3), // 3: 16 ch, level 1
|
||||
convt(3, 16, 8), // 4: 8 ch, level 0
|
||||
conv(4, Some(1), 16, 8, 3), // 5: join [up, skip]
|
||||
conv(5, None, 8, 12, 1), // 6: 12 ch
|
||||
Op::DepthToSpace2 { src: 6 }, // 7: 3 ch, level -1
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deep_networks_and_huge_cost_queries_never_overflow_or_truncate_depth() {
|
||||
let n = Net::new(4, unet()).unwrap();
|
||||
assert_eq!(n.macs(usize::MAX), u64::MAX);
|
||||
assert_eq!(n.macs(0), 0);
|
||||
let mut ops = Vec::new();
|
||||
for i in 0..21 {
|
||||
ops.push(Op::MaxPool2 { src: i });
|
||||
}
|
||||
for i in 21..42 {
|
||||
ops.push(convt(i, 4, 4));
|
||||
}
|
||||
ops.push(conv(42, None, 4, 12, 1));
|
||||
ops.push(Op::DepthToSpace2 { src: 43 });
|
||||
let n = Net::new(4, ops).unwrap();
|
||||
assert!(!n.fits_tile(1 << 20));
|
||||
assert!(n.fits_tile(1 << 21));
|
||||
let _ = n.macs(16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_u_net_is_described_with_its_shapes_and_costs() {
|
||||
let net = Net::new(4, unet()).unwrap();
|
||||
assert_eq!(net.out_channels(), 3);
|
||||
assert_eq!(net.shape(3), Some(Shape { channels: 16, level: 1 }));
|
||||
assert_eq!(net.shape(7), Some(Shape { channels: 3, level: -1 }));
|
||||
assert_eq!(net.depth(), 1);
|
||||
assert!(net.fits_tile(64) && !net.fits_tile(63) && !net.fits_tile(0));
|
||||
// 64 × 64 cells: layer 1 = 4096·4·8·9, and so on; just check it is the sum of what the layers cost
|
||||
let t = 64u64;
|
||||
let want = t * t * 4 * 8 * 9 + (t / 2) * (t / 2) * 8 * 16 * 9 + (t / 2) * (t / 2) * 16 * 8 * 4 + t * t * 16 * 8 * 9 + t * t * 8 * 12;
|
||||
assert_eq!(net.macs(64), want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_network_that_does_not_chain_is_refused() {
|
||||
// wrong input channel count for the first convolution
|
||||
assert!(Net::new(3, unet()).is_err());
|
||||
// a layer reading a tensor that does not exist yet
|
||||
let mut ops = unet();
|
||||
ops[2] = conv(9, None, 8, 16, 3);
|
||||
assert!(Net::new(4, ops).is_err());
|
||||
// joining tensors of different resolutions
|
||||
let mut ops = unet();
|
||||
ops[4] = conv(3, Some(1), 24, 8, 3);
|
||||
assert!(Net::new(4, ops).is_err());
|
||||
// weights of the wrong size, and not finite
|
||||
let mut ops = unet();
|
||||
if let Op::Conv(c) = &mut ops[0] {
|
||||
c.weight.pop();
|
||||
}
|
||||
assert!(Net::new(4, ops).is_err());
|
||||
let mut ops = unet();
|
||||
if let Op::Conv(c) = &mut ops[0] {
|
||||
c.bias[0] = f32::NAN;
|
||||
}
|
||||
assert!(Net::new(4, ops).is_err());
|
||||
// a kernel that is neither 1 nor 3 wide, no layers, depth-to-space on a channel count it cannot split
|
||||
let mut ops = unet();
|
||||
if let Op::Conv(c) = &mut ops[0] {
|
||||
c.k = 5;
|
||||
}
|
||||
assert!(Net::new(4, ops).is_err());
|
||||
assert!(Net::new(4, vec![]).is_err());
|
||||
assert!(Net::new(4, vec![Op::DepthToSpace2 { src: 0 }]).is_ok());
|
||||
assert!(Net::new(4, vec![Op::DepthToSpace2 { src: 0 }, Op::DepthToSpace2 { src: 1 }]).is_err());
|
||||
assert!(Net::new(2, vec![Op::DepthToSpace2 { src: 0 }]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_network_that_does_not_end_one_level_up_does_not_fit_a_tile() {
|
||||
let net = Net::new(4, unet()[..6].to_vec()).unwrap();
|
||||
assert!(!net.fits_tile(64), "no depth-to-space: the output is not at twice the resolution");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,369 @@
|
||||
//! Reading an `.onnx` file into a [`Net`] with bounded, pure-Rust protobuf decoding.
|
||||
//!
|
||||
//! Only what the denoise U-Nets are made of is understood: `Conv` (3 × 3 "same" or 1 × 1, stride 1, no groups, no
|
||||
//! dilation) optionally followed by `LeakyRelu`, `ConvTranspose` (2 × 2, stride 2) likewise, `MaxPool` (2 × 2, stride 2),
|
||||
//! a two-input `Concat` on the channels that feeds a convolution, and a final `DepthToSpace` (2, `CRD`). Any other
|
||||
//! operator or attribute is [`NetError::Unsupported`], which is reported to the user by both runners.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::onnx_proto::{self, AttributeProto, NodeProto, TensorProto, ValueInfoProto};
|
||||
use std::io::Read;
|
||||
|
||||
use crate::net::{Conv, ConvT2, MAX_WEIGHTS, Net, NetError, Op, TensorId};
|
||||
|
||||
fn unsupported<T>(why: impl Into<String>) -> Result<T, NetError> {
|
||||
Err(NetError::Unsupported(why.into()))
|
||||
}
|
||||
|
||||
fn invalid<T>(why: impl Into<String>) -> Result<T, NetError> {
|
||||
Err(NetError::Invalid(why.into()))
|
||||
}
|
||||
|
||||
/// Read the model at `path`.
|
||||
pub fn read(path: &Path) -> Result<Net, NetError> {
|
||||
let file = std::fs::File::open(path).map_err(|e| NetError::Invalid(e.to_string()))?;
|
||||
let size = usize::try_from(file.metadata().map_err(|e| NetError::Invalid(e.to_string()))?.len())
|
||||
.ok()
|
||||
.filter(|&n| n <= onnx_proto::MAX_BYTES)
|
||||
.ok_or_else(|| NetError::Invalid("model file too large".into()))?;
|
||||
let mut bytes = Vec::new();
|
||||
bytes.try_reserve_exact(size).map_err(|_| NetError::Invalid("not enough memory to load the model".into()))?;
|
||||
file.take(onnx_proto::MAX_BYTES as u64 + 1).read_to_end(&mut bytes).map_err(|e| NetError::Invalid(e.to_string()))?;
|
||||
read_bytes(&bytes)
|
||||
}
|
||||
|
||||
fn read_bytes(bytes: &[u8]) -> Result<Net, NetError> {
|
||||
let graph = onnx_proto::decode(bytes)?;
|
||||
|
||||
let initialisers: HashMap<&str, &TensorProto> = graph.initializer.iter().map(|t| (t.name.as_str(), t)).collect();
|
||||
if initialisers.len() != graph.initializer.len() {
|
||||
return invalid("duplicate initializer names");
|
||||
}
|
||||
let inputs: Vec<_> = graph.input.iter().filter(|i| !initialisers.contains_key(i.name.as_str())).collect();
|
||||
let ([input], [output]) = (inputs.as_slice(), graph.output.as_slice()) else { return unsupported("not one input and one output") };
|
||||
let in_channels = match channel_dim(input) {
|
||||
Some(c) => c,
|
||||
None => return unsupported("the input's channel count is not written in the file"),
|
||||
};
|
||||
|
||||
// how many nodes read each name (a graph output counts as a reader)
|
||||
let mut readers: HashMap<&str, usize> = HashMap::new();
|
||||
for n in &graph.node {
|
||||
for i in &n.input {
|
||||
*readers.entry(i.as_str()).or_default() += 1;
|
||||
}
|
||||
}
|
||||
*readers.entry(output.name.as_str()).or_default() += 1;
|
||||
|
||||
let mut ids: HashMap<String, TensorId> = HashMap::new();
|
||||
ids.insert(input.name.clone(), 0);
|
||||
// a Concat is not a layer of its own: the convolution that reads it takes both sources
|
||||
let mut joins: HashMap<String, (TensorId, TensorId)> = HashMap::new();
|
||||
let mut ops: Vec<Op> = Vec::new();
|
||||
let mut weights_total = 0usize;
|
||||
|
||||
let nodes = &graph.node;
|
||||
let mut i = 0;
|
||||
while i < nodes.len() {
|
||||
let Some(node) = nodes.get(i) else { break };
|
||||
i += 1;
|
||||
if node.output.len() != 1 || node.output.first().is_none_or(|n| n.is_empty() || ids.contains_key(n) || joins.contains_key(n)) {
|
||||
return invalid("a layer has missing, repeated or multiple outputs");
|
||||
}
|
||||
let allowed: &[&str] = match node.op_type.as_str() {
|
||||
"Conv" => &["kernel_shape", "pads", "strides", "dilations", "group", "auto_pad"],
|
||||
"ConvTranspose" => &["kernel_shape", "pads", "strides", "dilations", "group", "output_padding", "auto_pad"],
|
||||
"MaxPool" => &["kernel_shape", "pads", "strides", "dilations", "ceil_mode", "storage_order", "auto_pad"],
|
||||
"Concat" => &["axis"],
|
||||
"DepthToSpace" => &["blocksize", "mode"],
|
||||
_ => &[],
|
||||
};
|
||||
if node.attribute.iter().any(|a| !allowed.contains(&a.name.as_str())) {
|
||||
return unsupported("an unknown operator attribute");
|
||||
}
|
||||
let this = ops.len() + 1;
|
||||
let lookup = |name: &str| -> Result<TensorId, NetError> {
|
||||
ids.get(name).copied().ok_or_else(|| NetError::Invalid(format!("`{}` reads `{name}`, which nothing makes", node.op_type)))
|
||||
};
|
||||
// a LeakyRelu straight after this layer, which nothing else reads, is part of it
|
||||
let mut fuse_leaky = |out: &str| -> Result<(Option<f32>, String), NetError> {
|
||||
if let Some(next) = nodes.get(i)
|
||||
&& next.op_type == "LeakyRelu"
|
||||
&& next.input.first().map(String::as_str) == Some(out)
|
||||
&& readers.get(out).copied() == Some(1)
|
||||
{
|
||||
if next.input.len() != 1 || next.output.len() != 1 || next.attribute.iter().any(|a| a.name != "alpha") {
|
||||
return invalid("malformed LeakyRelu");
|
||||
}
|
||||
let alpha = attr(next, "alpha").map_or(0.01, |a| a.f);
|
||||
i += 1;
|
||||
let name = next.output.first().cloned().ok_or_else(|| NetError::Invalid("LeakyRelu without an output".into()))?;
|
||||
return Ok((Some(alpha), name));
|
||||
}
|
||||
Ok((None, out.to_string()))
|
||||
};
|
||||
match node.op_type.as_str() {
|
||||
"Concat" => {
|
||||
if attr(node, "axis").map(|a| a.i) != Some(1) {
|
||||
return unsupported("Concat on another axis than the channels");
|
||||
}
|
||||
let [a, b] = node.input.as_slice() else { return unsupported("Concat of other than two tensors") };
|
||||
let out = node.output.first().ok_or_else(|| NetError::Invalid("Concat without an output".into()))?;
|
||||
if readers.get(out.as_str()).copied() != Some(1) {
|
||||
return unsupported("a Concat that is read by more than one layer");
|
||||
}
|
||||
joins.insert(out.clone(), (lookup(a)?, lookup(b)?));
|
||||
}
|
||||
"Conv" => {
|
||||
let (w, b) = weights_of(node, &initialisers, &mut weights_total)?;
|
||||
let (cout, cin, kh, kw) = four(&w.0)?;
|
||||
let b = if b.is_empty() { vec![0.0; cout] } else { b };
|
||||
if kh != kw || !(kh == 1 || kh == 3) {
|
||||
return unsupported(format!("a {kh} × {kw} convolution"));
|
||||
}
|
||||
if ints_or(node, "strides", &[1, 1]) != [1, 1]
|
||||
|| ints_or(node, "dilations", &[1, 1]) != [1, 1]
|
||||
|| attr(node, "group").is_some_and(|a| a.i != 1)
|
||||
{
|
||||
return unsupported("a convolution with strides, dilation or groups");
|
||||
}
|
||||
if attr(node, "kernel_shape").is_some_and(|a| a.ints != [kh as i64, kw as i64]) {
|
||||
return invalid("kernel shape does not match the weights");
|
||||
}
|
||||
let pad = (kh as i64 - 1) / 2;
|
||||
if ints_or(node, "pads", &[0; 4]) != [pad; 4] || attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") {
|
||||
return unsupported("a convolution that is not padded to keep its size");
|
||||
}
|
||||
let src_name = node.input.first().ok_or_else(|| NetError::Invalid("Conv without an input".into()))?;
|
||||
let (src, src2) = match joins.remove(src_name) {
|
||||
Some((a, b)) => (a, Some(b)),
|
||||
None => (lookup(src_name)?, None),
|
||||
};
|
||||
let out = node.output.first().ok_or_else(|| NetError::Invalid("Conv without an output".into()))?;
|
||||
let (leaky, out_name) = fuse_leaky(out)?;
|
||||
ids.insert(out_name, this);
|
||||
ops.push(Op::Conv(Conv { src, src2, cin, cout, k: kh, weight: w.1, bias: b, leaky }));
|
||||
}
|
||||
"ConvTranspose" => {
|
||||
let (w, b) = weights_of(node, &initialisers, &mut weights_total)?;
|
||||
let (cin, cout, kh, kw) = four(&w.0)?;
|
||||
let b = if b.is_empty() { vec![0.0; cout] } else { b };
|
||||
if (kh, kw) != (2, 2)
|
||||
|| ints_or(node, "strides", &[1, 1]) != [2, 2]
|
||||
|| ints_or(node, "pads", &[0; 4]) != [0; 4]
|
||||
|| ints_or(node, "dilations", &[1, 1]) != [1, 1]
|
||||
|| attr(node, "group").is_some_and(|a| a.i != 1)
|
||||
|| attr(node, "output_padding").is_some_and(|a| a.ints.iter().any(|&v| v != 0))
|
||||
{
|
||||
return unsupported("a transposed convolution other than 2 × 2 with stride 2");
|
||||
}
|
||||
if attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") {
|
||||
return unsupported("transposed convolution auto padding");
|
||||
}
|
||||
if attr(node, "kernel_shape").is_some_and(|a| a.ints != [2, 2]) {
|
||||
return invalid("transposed kernel shape does not match the weights");
|
||||
}
|
||||
let src = lookup(node.input.first().map_or("", String::as_str))?;
|
||||
let out = node.output.first().ok_or_else(|| NetError::Invalid("ConvTranspose without an output".into()))?;
|
||||
let (leaky, out_name) = fuse_leaky(out)?;
|
||||
ids.insert(out_name, this);
|
||||
ops.push(Op::ConvT2(ConvT2 { src, cin, cout, weight: w.1, bias: b, leaky }));
|
||||
}
|
||||
"MaxPool" => {
|
||||
if ints_or(node, "kernel_shape", &[]) != [2, 2]
|
||||
|| ints_or(node, "strides", &[1, 1]) != [2, 2]
|
||||
|| ints_or(node, "pads", &[0; 4]) != [0; 4]
|
||||
|| ints_or(node, "dilations", &[1, 1]) != [1, 1]
|
||||
|| attr(node, "ceil_mode").is_some_and(|a| a.i != 0)
|
||||
|| attr(node, "storage_order").is_some_and(|a| a.i != 0)
|
||||
|| attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET")
|
||||
{
|
||||
return unsupported("a max-pool other than 2 × 2 with stride 2");
|
||||
}
|
||||
let src = lookup(node.input.first().map_or("", String::as_str))?;
|
||||
let out = node.output.first().ok_or_else(|| NetError::Invalid("MaxPool without an output".into()))?;
|
||||
ids.insert(out.clone(), this);
|
||||
ops.push(Op::MaxPool2 { src });
|
||||
}
|
||||
"DepthToSpace" => {
|
||||
if attr(node, "blocksize").map(|a| a.i) != Some(2) || attr(node, "mode").map(|a| a.s.as_slice()) != Some(&b"CRD"[..]) {
|
||||
return unsupported("depth-to-space other than block size 2 in CRD order");
|
||||
}
|
||||
let src = lookup(node.input.first().map_or("", String::as_str))?;
|
||||
let out = node.output.first().ok_or_else(|| NetError::Invalid("DepthToSpace without an output".into()))?;
|
||||
ids.insert(out.clone(), this);
|
||||
ops.push(Op::DepthToSpace2 { src });
|
||||
}
|
||||
other => return unsupported(format!("the `{other}` operator")),
|
||||
}
|
||||
}
|
||||
if !joins.is_empty() {
|
||||
return unsupported("a Concat whose result no convolution reads");
|
||||
}
|
||||
if ids.get(output.name.as_str()) != Some(&ops.len()) {
|
||||
return unsupported("the output is not the last layer");
|
||||
}
|
||||
Net::new(in_channels, ops)
|
||||
}
|
||||
|
||||
fn attr<'a>(n: &'a NodeProto, name: &str) -> Option<&'a AttributeProto> {
|
||||
n.attribute.iter().find(|a| a.name == name)
|
||||
}
|
||||
|
||||
/// The integer list attribute `name`, or `default` when the node does not have it.
|
||||
fn ints_or(n: &NodeProto, name: &str, default: &[i64]) -> Vec<i64> {
|
||||
attr(n, name).map_or_else(|| default.to_vec(), |a| a.ints.clone())
|
||||
}
|
||||
|
||||
fn four(dims: &[i64]) -> Result<(usize, usize, usize, usize), NetError> {
|
||||
match dims {
|
||||
[a, b, c, d] if dims.iter().all(|&v| v > 0 && v <= 1 << 20) => Ok((*a as usize, *b as usize, *c as usize, *d as usize)),
|
||||
_ => invalid("a weight tensor that is not four-dimensional"),
|
||||
}
|
||||
}
|
||||
|
||||
type Weights = ((Vec<i64>, Vec<f32>), Vec<f32>);
|
||||
|
||||
/// The weight and bias initialisers of a convolution node.
|
||||
fn weights_of(node: &NodeProto, initialisers: &HashMap<&str, &TensorProto>, total: &mut usize) -> Result<Weights, NetError> {
|
||||
let mut get = |idx: usize| -> Result<(Vec<i64>, Vec<f32>), NetError> {
|
||||
let name = node.input.get(idx).ok_or_else(|| NetError::Invalid(format!("`{}` without its weights", node.op_type)))?;
|
||||
let t = initialisers.get(name.as_str()).ok_or_else(|| NetError::Unsupported(format!("weights that are computed (`{name}`)")))?;
|
||||
let values = floats(t)?;
|
||||
*total = total.saturating_add(values.len());
|
||||
if *total > MAX_WEIGHTS {
|
||||
return invalid("the network has too many weights");
|
||||
}
|
||||
Ok((t.dims.clone(), values))
|
||||
};
|
||||
let w = get(1)?;
|
||||
let b = if node.input.len() > 2 {
|
||||
let (dims, values) = get(2)?;
|
||||
if dims.len() != 1 || values.is_empty() {
|
||||
return invalid("a bias tensor must be a nonempty vector");
|
||||
}
|
||||
values
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
Ok((w, b))
|
||||
}
|
||||
|
||||
fn floats(t: &TensorProto) -> Result<Vec<f32>, NetError> {
|
||||
if t.data_type != 1 {
|
||||
return unsupported("weights that are not 32-bit floats");
|
||||
}
|
||||
let want: usize = t
|
||||
.dims
|
||||
.iter()
|
||||
.try_fold(1usize, |n, &d| usize::try_from(d).ok().and_then(|d| n.checked_mul(d)))
|
||||
.ok_or_else(|| NetError::Invalid("a tensor with a negative or huge size".into()))?;
|
||||
if want > MAX_WEIGHTS {
|
||||
return invalid("a tensor with too many numbers");
|
||||
}
|
||||
if !t.raw_data.is_empty() && (!t.float_data.is_empty() || !t.raw_data.len().is_multiple_of(4)) {
|
||||
return invalid("ambiguous or truncated tensor data");
|
||||
}
|
||||
let actual = if t.raw_data.is_empty() { t.float_data.len() } else { t.raw_data.len() / 4 };
|
||||
if actual != want {
|
||||
return invalid("tensor data does not match its dimensions");
|
||||
}
|
||||
let mut values = Vec::new();
|
||||
values.try_reserve_exact(want).map_err(|_| NetError::Invalid("not enough memory for weights".into()))?;
|
||||
if !t.raw_data.is_empty() {
|
||||
values.extend(t.raw_data.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)));
|
||||
} else {
|
||||
values.extend_from_slice(&t.float_data);
|
||||
}
|
||||
if values.len() != want {
|
||||
return invalid(format!("tensor `{}` has {} numbers, not the {want} its shape says", t.name, values.len()));
|
||||
}
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
/// The channel count in an input's declared shape `[batch, channels, …]`.
|
||||
fn channel_dim(v: &ValueInfoProto) -> Option<usize> {
|
||||
v.channels
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::synthetic::unet_onnx;
|
||||
|
||||
fn temp_model(name: &str, bytes: &[u8]) -> std::path::PathBuf {
|
||||
let p = std::env::temp_dir().join(format!("lc-denoise-onnx-{name}-{}.onnx", std::process::id()));
|
||||
std::fs::write(&p, bytes).unwrap();
|
||||
p
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_u_net_is_read_with_its_layers_fused() {
|
||||
let (tile, depth) = (32usize, 2usize);
|
||||
let p = temp_model("unet", &unet_onnx(tile as u64, 8, depth, 7));
|
||||
let net = read(&p).unwrap();
|
||||
// per level two convolutions; a pool per level but the last; going up a transposed convolution and two convolutions
|
||||
// per level (the join is part of the first); then the 1 × 1 convolution and depth-to-space
|
||||
assert_eq!(net.ops().len(), 2 * (depth + 1) + depth + depth * 3 + 2);
|
||||
assert!(matches!(net.ops().last(), Some(crate::net::Op::DepthToSpace2 { .. })));
|
||||
assert!(net.ops().iter().any(|o| matches!(o, crate::net::Op::Conv(c) if c.src2.is_some())), "the joins became two-source convolutions");
|
||||
assert!(
|
||||
net.ops().iter().all(|o| !matches!(o, crate::net::Op::Conv(c) if c.leaky.is_none() && c.k == 3)),
|
||||
"every 3 × 3 convolution has its activation"
|
||||
);
|
||||
assert_eq!((net.in_channels, net.out_channels(), net.depth()), (4, 3, 2));
|
||||
assert!(net.fits_tile(tile) && !net.fits_tile(30));
|
||||
|
||||
let _ = std::fs::remove_file(p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_explicit_empty_or_nonvector_bias_is_not_treated_as_missing() {
|
||||
let w = TensorProto { name: "w".into(), dims: vec![12, 4, 1, 1], data_type: 1, float_data: vec![0.1; 48], ..Default::default() };
|
||||
let n = NodeProto { input: vec!["data".into(), "w".into(), "b".into()], ..Default::default() };
|
||||
for dims in [vec![0], vec![3, 4]] {
|
||||
let b = TensorProto {
|
||||
name: "b".into(),
|
||||
float_data: vec![0.0; dims.iter().product::<i64>() as usize],
|
||||
dims,
|
||||
data_type: 1,
|
||||
..Default::default()
|
||||
};
|
||||
let inits = HashMap::from([("w", &w), ("b", &b)]);
|
||||
assert!(weights_of(&n, &inits, &mut 0).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_and_mutated_protobuf_is_bounded_and_never_panics() {
|
||||
let good = unet_onnx(4, 3, 1, 7);
|
||||
for n in (0..good.len()).step_by(31) {
|
||||
let _ = read_bytes(&good[..n]);
|
||||
}
|
||||
for at in (0..good.len()).step_by(37) {
|
||||
let mut bad = good.clone();
|
||||
bad[at] ^= 0xff;
|
||||
let _ = read_bytes(&bad);
|
||||
}
|
||||
assert!(read_bytes(&[0x3a, 0xff, 0xff, 0xff, 0xff, 0x7f]).is_err());
|
||||
assert!(read_bytes(&[0x3a, 0, 0x3a, 0]).is_err());
|
||||
}
|
||||
proptest::proptest! {
|
||||
#[test]
|
||||
fn arbitrary_model_bytes_never_panic(bytes in proptest::collection::vec(proptest::prelude::any::<u8>(), 0..4096)) {
|
||||
let _ = read_bytes(&bytes);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_the_description_cannot_say_is_unsupported_and_garbage_is_an_error() {
|
||||
// not a model
|
||||
let p = temp_model("junk", b"this is not an onnx file");
|
||||
assert!(matches!(read(&p), Err(NetError::Invalid(_)) | Err(NetError::Unsupported(_))));
|
||||
let _ = std::fs::remove_file(p);
|
||||
// a missing file
|
||||
assert!(read(Path::new("/does/not/exist.onnx")).is_err());
|
||||
}
|
||||
}
|
||||
Loaded 100 of 504 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user