Compare commits

...
Author SHA1 Message Date
Brandon Thomas 53711eb1ef Merge PR #273
# Conflicts:
#	docs/parity.md
2026-10-09 12:04:03 -04:00
Brandon Thomas 1727477382 Merge PR #464 2026-10-09 12:03:42 -04:00
Brandon Thomas 03b3d606ce Merge pull request #546 from storytold/claude/integrate-batch-19
Integrate #474, #448, #447, #499, #459
2026-10-09 12:02:17 -04:00
Brandon Thomas de087c0c6f Merge remote-tracking branch 'origin/main' into claude/integrate-batch-19 2026-10-09 12:02:10 -04:00
Brandon Thomas da6cbcf968 Merge pull request #545 from storytold/claude/integrate-batch-18
Integrate #478, #368, #460, #449, #451, #468
2026-10-09 11:56:38 -04:00
Brandon ThomasandClaude Opus 5.5 95d8aed986 Merge origin/main into profile-gain-table-map (RENDER_CACHE_VERSION: main's 21 + 1 = 22)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 11:52:23 -04:00
Brandon Thomas 38431cb4db Merge PR #459
# Conflicts:
#	crates/engine/src/import.rs
#	crates/mcp/src/headless.rs
#	crates/raw/src/lib.rs
#	docs/parity.md
#	xtask/src/main.rs
2026-10-09 11:38:44 -04:00
Brandon Thomas abe4c5fe1b Merge PR #499 2026-10-09 11:37:31 -04:00
Brandon Thomas c6745bc6f7 Merge PR #447
# Conflicts:
#	docs/parity.md
2026-10-09 11:37:19 -04:00
Brandon Thomas 093c948f8b Merge PR #448 2026-10-09 11:36:52 -04:00
Brandon Thomas aa1daf5dc9 Merge PR #474 2026-10-09 11:36:51 -04:00
Brandon Thomas 38b8ebdfa5 Merge PR #468 2026-10-09 11:36:32 -04:00
Brandon Thomas 5328100722 Merge PR #451 2026-10-09 11:36:31 -04:00
Brandon ThomasandClaude Opus 5.5 6b671c9912 Merge origin/main into #499 (render cache version 22, as main is at 21)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 09:45:47 -04:00
Brandon ThomasandClaude Opus 5.5 b7e2f4eff4 Merge origin/main into #447 (render cache version 22, as main is at 21; parity row keeps main's CR3 AspectInfo note)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 09:39:22 -04:00
Brandon ThomasandClaude Opus 5.5 eae3df481d Merge origin/main into #459 (render cache version 22, as main is at 21; parity row keeps main's CR3 AspectInfo note)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 09:35:21 -04:00
Brandon ThomasandClaude Opus 5.5 a276febabe Merge origin/main into #448 (render cache version 22, as main is at 21; parity row keeps main's text plus the 3FR note)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 09:31:04 -04:00
Brandon Thomas 967b858129 Merge PR #449 2026-10-09 09:26:00 -04:00
Brandon Thomas d31f805945 Merge PR #460 2026-10-09 09:25:59 -04:00
Brandon Thomas 12a2a99f2a Merge PR #368 2026-10-09 09:25:58 -04:00
Brandon Thomas 5e6d33065a Merge PR #478 2026-10-09 09:25:57 -04:00
Brandon Thomas 65ad290e89 Merge pull request #512 from storytold/claude/integrate-batch-17
Integrate #489, #493, #495, #494, #444, #484, #477
2026-10-09 09:25:11 -04:00
Brandon Thomas e111734532 Merge remote-tracking branch 'origin/main' into claude/integrate-batch-17 2026-10-09 09:24:54 -04:00
Brandon Thomas 8f924a9e60 Merge pull request #511 from storytold/claude/integrate-batch-16
Integrate #453, #470, #482, #492, #433, #465
2026-10-09 09:08:47 -04:00
Brandon ThomasandClaude Opus 5.5 7a42f99cf0 MCP: the export tool forwards every option it advertises
preset, background, resize, subfolder, conflict, tiffCompression and
dngCompression were in the export tool's schema (and so passed the new
argument check) but were dropped before reaching app.export.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 09:05:04 -04:00
Brandon Thomas 081a619c51 Merge PR #477 2026-10-09 09:04:07 -04:00
Brandon Thomas 7e350eccb8 Merge PR #484 2026-10-09 09:04:06 -04:00
Brandon Thomas 8abcd2fc54 Merge PR #444 2026-10-09 09:04:05 -04:00
Brandon Thomas b7331d6c13 Merge PR #494 2026-10-09 09:04:04 -04:00
Brandon Thomas 5bd4302037 Merge PR #495 2026-10-09 09:04:03 -04:00
Brandon Thomas 1b4e646df5 Merge PR #493 2026-10-09 09:04:02 -04:00
Brandon Thomas 9fef5ac702 Merge PR #489 2026-10-09 09:04:01 -04:00
Brandon Thomas 0b6cfd89ec Merge remote-tracking branch 'origin/main' into fix-449 2026-10-09 08:57:20 -04:00
Brandon ThomasandClaude Opus 5.5 56b06426db docs/parity.md: drop the radial-drag note from Top gaps (the LR-MASK-RADIAL row carries it)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 08:57:20 -04:00
Brandon ThomasandClaude Opus 5.5 d418292fb0 Show in Explorer: drop quotes from the path so the raw command line stays one argument
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 08:56:53 -04:00
Brandon Thomas 11e739cd02 Merge PR #465 2026-10-09 08:34:42 -04:00
Brandon Thomas 7317e64d74 Merge PR #433
# Conflicts:
#	crates/engine/src/media.rs
2026-10-09 08:34:41 -04:00
Brandon Thomas ceb35048bb Merge PR #492
# Conflicts:
#	crates/engine/src/media.rs
2026-10-09 08:34:40 -04:00
Brandon Thomas f731922e16 Merge PR #482
# Conflicts:
#	crates/engine/src/media.rs
2026-10-09 08:34:38 -04:00
Brandon Thomas a18a6e6a5e Merge PR #470 2026-10-09 08:34:37 -04:00
Brandon Thomas ffd91332e9 Merge PR #453 2026-10-09 08:34:36 -04:00
Brandon ThomasandClaude Opus 5.5 8d96dab1af LR-IMP-FORMATS: checked sum of the sRAW slice widths (an overflowing or zero sum is an error, not a panic)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 08:24:34 -04:00
Brandon ThomasandClaude Opus 5.5 761631c56c Quit from the macOS menu: save settings and the library (and honour the unsaved-changes prompt) before exiting
std::process::exit skipped eframe's on_exit (ui.json, close_library) and the
may_close check, so changes still in memory could be lost on Cmd+Q.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 08:19:34 -04:00
Brandon ThomasandClaude Opus 5.5 0a4dc3f3ec Merge main into #433 (parity row: keep main's text plus the AspectInfo note; render cache version 17, as main is at 16 already)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 08:16:13 -04:00
Brandon Thomas 9eb8151984 Merge origin/main 2026-10-09 08:05:59 -04:00
radialmonster cdc2e57776 LR-PROF-CAMERACOLOR: docs: archive-wide numbers for the tone fit and its fallback
The earlier text quoted the 13 investigation files (held-out RMS within 0.005); the whole CC0 archive shows a worst increase of 0.013 and 7 files that need the conditional-median fallback.
2026-10-09 07:31:53 -04:00
rock-meisterandClaude Opus 5.5 369c56a6ed LRC-LIB-CATALOG-IMPORT: import straightened catalog crops as Lightroom shows them
With a CropAngle, Lightroom's CropLeft/CropTop and CropRight/CropBottom
are the top-left and bottom-right corners of the tilted crop on the
stored image, and its angle turns the other way to LightCraft's. The
import treated the edges as an axis-aligned rectangle in the rotated
frame and kept the angle's sign, so straightened crops came out the
wrong shape and size.

Carry the two corners through the orientation and then LightCraft's
straighten rotation (angle = -CropAngle, mirrored orientations flip it
back), using the stored image's aspect from fileWidth/fileHeight. On a
real catalog every one of 223 straightened crops now matches the frame
of Lightroom's export, and the content difference falls to that of
unstraightened crops.

Refs #483

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 06:26:47 -05:00
Elliot GrahamandCodex b9f951998f #491: Keep Local import headers within the centre pane
Co-Authored-By: Codex <noreply@openai.com>
2026-10-09 12:08:11 +01:00
radialmonster c691bcfa02 LR-PROF-CAMERACOLOR: fall back to the conditional-median tone fit when the quantile look misses the gates
Quantile matching fits the held-out pixels slightly worse than the conditional-median fit it replaced. On 7 of 1023 archive files (compacts, bridge cameras, early Micro Four Thirds) the look passed the acceptance gates only with the old curve, and the quantile curve lost it. The search now runs with the quantile fit first (all pixels, away from edges, with and without the camera profile) and, only when no look passes, runs again with the earlier conditional-median plus isotonic fit, restored as fit_tone_conditional. The gates are unchanged, so no file loses its look relative to the previous fit, and the quantile curve is used wherever it passes. Tests: the flat-step test stays; a synthetic photo whose JPEG carries per-pixel noise misses the gates with the quantile fit and passes with the fallback, and the order of the two is checked.
2026-10-09 07:08:01 -04:00
rock-meisterandClaude Opus 5.5 2ec2b5bf4a LRC-LIB-CATALOG-IMPORT: carry catalog crops through the photo orientation
crs crop edges are normalized to the stored (unrotated) image, while
LightCraft's crop rectangle lives in the oriented frame. Catalog import
applied them unchanged, so crops on portrait photos (EXIF 6 or 8) landed
along the wrong axes.

Read Lightroom's orientation code (Adobe_images.orientation: AB, BC, CD,
DA and the mirrored BA, CB, DC, AD) and map the rectangle into the
oriented frame with Orientation::map_norm_rect; a mirrored orientation
also mirrors the straighten angle. The code comes from the catalog, so
this works while the originals are still missing at import time.

Refs #483

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 05:49:59 -05:00
radialmonster e5ff5461e3 LR-IMP-FORMATS: render cache version 21 for the Canon sRAW/mRAW decode
Open PRs already use 17 to 20, so this one takes the next free number.
2026-10-09 06:33:11 -04:00
MarekSpodBiedry 44735cd5e7 Issue #272: Show in Explorer selects photos whose path has a space
Explorer only selects a file when the path after /select, is quoted. Rust's argument quoting wraps the whole "/select,C:\My Photos\a.jpg" switch in quotes as soon as the path has a space, so Explorer opened Documents. Build the switch as /select,"<path>" (backslashes, verbatim \\?\ and \\?\UNC\ prefixes stripped) and pass it with raw_arg on Windows. The library folder tree's context menu now shows the platform's reveal label too, and the Russian catalog gains "Show in Explorer" and "Show in File Manager".
2026-10-09 12:19:27 +02:00
rock-meisterandClaude Opus 5.5 c5449926b9 Contributors: add rock-meister (sync GitHub name)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 05:15:52 -05:00
rock-meisterandClaude Opus 5.5 fcef4c0fc9 LRC-LIB-CATALOG-IMPORT: import catalog crops that have no HasCrop
Lightroom Classic catalogs store the crop edges and straighten angle in
their develop settings but, unlike XMP, never a HasCrop flag. The shared
crs mapper only applies a crop when crs:HasCrop is True, so every crop in
an imported catalog was dropped and reported as unmapped.

Infer HasCrop in the catalog path when any crop field is present. The XMP
path is unchanged: there HasCrop="False" with crop fields means the crop
is switched off.

Refs #483

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 05:15:52 -05:00
radialmonster 0778874d13 LR-PROF-CAMERACOLOR: bump the render cache version for the new tone fit
Renders change for files with a fitted camera look. Open PRs already claim versions 17, 18 and 19.
2026-10-09 05:45:01 -04:00
radialmonster 24427912cb LR-PROF-CAMERACOLOR: fit the camera tone curve by quantile matching (no flat steps)
The per-photo look fitted its tone curve as the camera JPEG's median per
scene-luminance bin followed by isotonic pooling. On lens-corrected, sharpened
JPEGs of busy scenes neighbouring bins disagree by more than the gap between
them, so pooling produced flat steps: 13 of 31 segments on a CC0 Z 6II sample,
flat segments in 9 of 13 fitted Nikon Z / ILCE-7 samples. Each step maps a
range of scene tones to one output tone, which renders as posterised patches
with hard edges (issue #475, also #211, #454), and the conditional medians
flatten both ends of the curve.

Quantile matching (sorted scene quantiles against sorted JPEG quantiles) is
increasing by construction and insensitive to misregistration. On the same 13
files no fit has a flat segment, all still pass the gates, held-out RMS within
0.005 of before. Regression test: a misregistered synthetic JPEG.
2026-10-09 05:41:24 -04:00
hxwandClaude Opus 5.5 da62ea8c49 LR-BEHAV-GPU: compile DX12 shaders with FXC, never a stray dxcompiler.dll (issue #471)
wgpu's default DX12 compiler (Auto) loads whichever dxcompiler.dll the DLL
search path finds first. LightCraft ships none, so it picked up another
program's copy; one without dxil.dll beside it (e.g. DXC 1.7) warns that
the DXIL is unsigned, wgpu treats that as a compile error, its
indirect-validation pipelines fail, the device is lost and the window
never opens ("Parent device is lost", exit code 1).

Every wgpu instance (window and both compute devices) now uses
lightcraft_gpu::backend::backend_options(): wgpu's env options with the
DX12 compiler pinned to FXC (d3dcompiler_47.dll, part of Windows), unless
WGPU_DX12_COMPILER asks for another.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 04:34:51 -05:00
radialmonster cfca299945 LR-IMP-FORMATS: report Sony A200/A230/A350 packed ARW as unsupported (preview fallback)
The one-strip Sony ARW layout was accepted when the raw strip was 1.00-1.25x width x height bytes. The DSLR-A200, A230 and A350 sample files carry strips of 1.06x, 1.23x and 1.04x, passed that test and decoded as colour noise. Every file that decodes correctly (108 in the test archive) has a strip of exactly width x height bytes, so the layout is now accepted only then; the rest report "Sony ARW version 1 / packed compressed variant" and open from the embedded preview like the A290/A380. Part of #461 (not a decoder). RENDER_CACHE_VERSION 16 -> 19.
2026-10-09 05:29:23 -04:00
BuonBredaandClaude Opus 5.5 fa1565d855 LR-PROF-CAMERACOLOR: camera local tone mapping (ProfileGainTableMap) as a per-photo opt-in
Renders a raw's DNG ProfileGainTableMap (Apple ProRAW's local tone mapping)
when the photo's Profile option "Camera local tone mapping" is on
(`profile.camera_local_tone`, off by default: Lightroom Classic renders
ProRAW without it). Built on #427's parser; supersedes the decode-time
applier of the first version of #273.

- raw/gaintable: GainTableMap::gain / evaluator (GainEval) per DNG 1.7.1:
  bilinear between tables with edges replicated, input clamp(weights .
  (R,G,B,min,max))^gamma in linear ProPhoto after the baseline exposure,
  index input x MapPointsN clamped to the last point. Saturating cell
  math; maps whose sizes don't match their gains aren't evaluated.
  SourcePlacement: where a developed source sits in the active area.
- pipeline/local_tone: the gain per output pixel in the scene-linear
  stage, before the white balance, mapped back through the frame (crop,
  user orientation, warp) and the EXIF orientation. lin_needs_cpu routes
  the stage to the CPU when it's on (CPU and GPU match); lin_key includes
  it. SourceInfo::local_tone carries the map from the decoder.
- develop: Profile::camera_local_tone (omitted from JSON when off, so
  existing settings and their hashes are unchanged); Profile row checkbox
  when the source has a map; translations.
- RENDER_CACHE_VERSION 17. Docs: parity row and What's New as an opt-in.

Off renders pixel-identical to main on two iPhone 15 Pro ProRAWs (12 and
48 MP). On adds ~60-75 ms to the scene-linear stage at 1536x2048 on a
2-core VM, the same for both files (cost follows output pixels).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VFqow2P1EtDxt6nRjjQoTA
2026-10-09 11:11:52 +02:00
MarekSpodBiedry bbcd2f47e8 Web restore: reject originals that don't match their content hash (#227) 2026-10-09 10:46:29 +02:00
MarekSpodBiedry 5fc85ee034 Fix #226: a refused Web Locks request no longer leaves the web app loading 2026-10-09 10:45:45 +02:00
radialmonster cec18712d6 LR-IMP-CAMERA-COVERAGE: xtask corpus verifies the sha256 of every downloaded corpus file
Each RAW_SAMPLES entry now carries its SHA-256 (computed from the file as downloaded; the ProRAW value was only a comment). cargo xtask corpus --download hashes every file, new or already present, deletes any mismatch so the next run fetches it again, and fails listing the actual and expected hashes. A unit test keeps names unique and hashes well-formed.
2026-10-09 04:43:19 -04:00
radialmonster 3af7a0c0ae LR-IMP-FORMATS: drop the 40D luma floor from Canon sRAW decoding
The 512 floor and its ColorDataVersion 3 keying rested on one file.
Versions without a known black-level position now fall back to black 0
for the white level, so the 40D sRAW renders with slightly raised blacks.
2026-10-09 04:18:46 -04:00
peiyan0 e7da5d34a2 catalog: group photos without capture date under Unknown Date (#437) 2026-10-09 16:09:23 +08:00
radialmonster 1399c445ef LR-IMP-FORMATS: read the PEF colour-filter layout from the Exif CFAPattern for Huffman-compressed files too
The Huffman-compressed Pentax files (compression 65535) assumed BGGR from
the array origin. The K10D, K200D, K2000 and every body from the K-3 on are
RGGB, so red and blue were exchanged before demosaicing. They now use the
Exif CFAPattern from the first sample of the stored array, like the other
compressions; BGGR stays the fallback for files without the tag. Extends the
CFA test to compression 65535. RENDER_CACHE_VERSION 17 to 18.
2026-10-09 04:02:20 -04:00
Claude a45c2b661b M11.1: decode EOS R7 C-RAW (odd-height CRX QP map)
An odd-height adaptive QP map (EOS R7: 4732 sensor rows, 1183 QP rows)
was rejected as unsupported, so R7 C-RAW files opened from their embedded
JPEG. The map decodes to the exact end of its entropy stream; the last
row of the second-level high-frequency bands has no partner row and its
QP is used alone. Even-height maps never reach this rule, so the six
verified M50/R100/R8 fixtures are unchanged.

Checked on paired R7 RAW and C-RAW captures (ISO 100 and 6400): all
six C-RAW files decode, developed images match their RAW counterparts
visually, and ISO 6400 sensor means agree within 0.1%.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Gs6roPby7THvteijZwptQt
2026-10-09 07:38:24 +00:00
radialmonster c7719ddcbf LR-IMP-FORMATS: read the PEF Exif CFAPattern from the array origin, not the image-area origin
The pattern was shifted by the image-area origin for compression 1 and
32773. The *ist D (area origin 19, 11, tag RGGB) fits RGGB from the array
origin in a mosaic-level fit against its camera JPEG, so the shifted reading
exchanged red and blue. The six 32773 files have an even origin and keep
their layout. Adds a hand-made-bytes test with an odd area origin for both
compressions.
2026-10-09 03:28:35 -04:00
supper_yijie bf35a3c6c5 fix #425: Restore Simplified Chinese font rendering
1.Added Simplified Chinese font support for web and desktop builds.
2.Kept Inter as the default font, with Chinese used only as a fallback.
3.Verified Chinese glyph tests and a macOS universal build.

Signed-off-by: supper_yijie <syj794844854@163.com>
2026-10-09 15:12:33 +08:00
radialmonster d9fd2f8ae0 LR-IMP-FORMATS: decode Samsung SRW files with uncompressed raw data
Samsung SRW raws whose raw IFD (private compression 32769 or 32770) stores the samples uncompressed are decoded from the sensor data: 12-bit samples packed two in three bytes (bit order told apart from the samples; NX5, NX10, NX11 and NX20, NX200, NX210, NX1000, NX1100) and 16-bit little-endian words (EX1, WB2000). CFA layout from the raw IFD's CFAPattern; framing, white balance, black and saturation levels from the maker note (value offsets relative to the note; the vectors are stored offset by its EncryptionKey); a one-direction Exif picture size crops to the centre of the framing; black from dark rows or columns when the file states none; saturation plateau below the stated level wins. Compressed codings (32770 with fewer bytes, 32772, 32773) stay preview only with that reason. srw is imported from folders; SRW takes the file-local camera look. RENDER_CACHE_VERSION 17.
2026-10-09 02:52:18 -04:00
radialmonster 1206d3fd52 LR-IMP-FORMATS: decode Canon sRAW and mRAW CR2 from their YCbCr data
sRAW and mRAW store subsampled YCbCr in a lossless-JPEG frame cut into
slices of whole MCUs. Reuse the subsampled lossless decoder added for
Sony M/S ARW with Canon's per-component sequential prediction, re-chop
the stream into image planes, and convert to linear white-balanced RGB
with chroma gains measured per file against the camera's own IFD2 RGB.
Render cache version 17.
2026-10-09 02:47:10 -04:00
midasdf 35718bc92c M1: keep deletion selection near the current photo 2026-10-09 15:04:40 +09:00
storybored379andOpenAI Codex dedc049692 MCP: add export progress and cancellation between photos
Answer ping at photo boundaries, preserve queued requests and EOF completion, and suppress cancelled responses. Keep completed exports and the existing atomic file writer; cover cancellation, output preservation and transport poison recovery.

Co-Authored-By: OpenAI Codex <noreply@openai.com>
2026-10-08 23:02:21 -07:00
storybored379andOpenAI Codex 5c16320f6b MCP: add shared core tools, hints and resources
Keep documented helpers listed, reject unknown helper keys, and recover escaped backend panics. Cover core tools, resource snapshots and modern result metadata with regression tests.

Co-Authored-By: OpenAI Codex <noreply@openai.com>
2026-10-08 23:02:13 -07:00
Brandon Thomas c435d143de Merge pull request #445 from storytold/claude/fix-freebsd-ci
CI: restore green FreeBSD builds
2026-10-09 01:38:58 -04:00
Brandon Thomas 82fc317922 Merge remote-tracking branch 'origin/main' into fix-445
# Conflicts:
#	crates/engine/src/media.rs
2026-10-09 01:36:13 -04:00
Hanmo 40652fc5f9 M6: move radial masks by dragging inside their ellipses 2026-10-09 13:35:02 +08:00
radialmonster 989422398c LR-IMP-FORMATS: read plain packed 12-bit PEF strips tagged as PackBits (older Pentax, Samsung GX-1L)
Compression 32773 is accepted in the PEF reader only when every strip has exactly the packed size of its rows at the declared bit depth, and is then read as uncompressed MSB-first samples. Any other 32773 size stays unsupported. Uncompressed PEFs take their colour filter from the Exif CFAPattern as seen from the image area origin. A Samsung-branded TIFF whose maker note has the Pentax layout is probed as PEF. RENDER_CACHE_VERSION 16 to 17.
2026-10-09 01:24:32 -04:00
Brandon Thomas 68e45585d0 Merge pull request #384 from voshart/pr/denoise-standalone
AI RAW denoise with opt-in models and pure-Rust CPU/GPU inference
2026-10-09 01:08:46 -04:00
Brandon ThomasandClaude Opus 5.5 7dcfbed05c LR-EDIT-DETAIL-DENOISE: list settings tabs from TABS; fix the SAM 3 worker's wasm check
The unknown-tab error now names every Settings tab (it missed AI Denoise after the
faces merge). lightcraft-sam3-worker (#359) only builds its server off wasm32, so
`cargo xtask wasm` passes again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 01:07:43 -04:00
Brandon ThomasandClaude Opus 5.5 d9457e6b23 CI: generate each procedural demo scene once per process (FreeBSD UI tests timing out)
Every demo-photo source load evaluated the procedural scene from scratch. A 24 MP demo
original costs ~200 CPU-seconds (AlpineLake, the photo the Detail tests open), and #323
added ~30 loupe tests that each load it at 1:1, Before and After separately. On the 4-vCPU
FreeBSD VM the ui-egui lib tests went from 1198 s to 3046 s, and the tests' 120 s/300 s
waits expired (loupe windows, fixture thumbnails starved of the shared rayon pool, the demo
detail snapshot).

Demo scene pixels are now generated once per (scene, size) and shared: concurrent loads
wait for one generation (Before/After at 1:1 no longer generate the original twice) and
the most recent results are kept up to half the memory budget. Locally, pinned to 4 CPUs,
the ui-egui lib suite now finishes in 155 s; without the change it had done 231 of 338
tests after 25 minutes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 01:03:40 -04:00
onemannoplan 5c1dcd4f97 fix(macos): prevent SIGILL crash on exit by replacing viewport close with clean process exit
### Summary
When selecting "Quit LightCraft" from the native macOS menu bar, the application occasionally crashes with an `EXC_BAD_INSTRUCTION` (SIGILL) error report during shutdown.

### Root Cause
1. The native menu action triggered `ctx.send_viewport_cmd(egui::ViewportCommand::Close)`.
2. Forcing an explicit viewport close via the command queue while macOS and AppKit are initiating their own termination sequence creates a race condition with `winit`.
3. This conflict during native window, view, and observer teardown leads to an illegal instruction fault.

### Solution
- Replaced the manual `ViewportCommand::Close` trigger on `QUIT` with a clean `std::process::exit(0)`.
- This bypasses the problematic `winit`/AppKit teardown race condition on macOS, allowing the application to shut down instantly and cleanly without generating crash reports.
2026-10-08 23:53:21 -05:00
Brandon ThomasandClaude Opus 5.5 1e1eb5fe16 Merge origin/main (batches 14 and 15) into AI RAW denoise (#384)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 00:30:17 -04:00
Brandon ThomasandClaude Opus 5.5 c65065de2f Merge the faces stack (#172/#173/#174, integrate-batch-14) into AI RAW denoise (#384)
One config-dir helper (engine::config::config_dir; camera_profiles re-exports it) with both
the face-models and denoise-models defaults; hosts set up both model folders; Settings has
Faces and AI Denoise tabs; both command sets registered; both model dialogs; locale entries
from both; one [features] table per app manifest; Cargo.lock regenerated by cargo; the
DecodedSource cache cost counts both segmentation mattes and the denoised twin.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 00:29:59 -04:00
Brandon Thomas 3e76713c1d Merge pull request #440 from storytold/claude/integrate-batch-15
Integrate #423, #359
2026-10-09 00:24:40 -04:00
radialmonster 79b19fd886 LR-IMP-FORMATS: decode Hasselblad 3FR files with uncompressed raw data
TIFF raws that describe a full-resolution CFA IFD with DNG's tags (levels, crop, AsShotNeutral) but carry no DNG version tag are read by the DNG reader (RawFormat::CfaTiff, dng::decode_as). RGGB when the file states no CFAPattern. Lossless JPEG scans with a predictor selection value above 7 are rejected with that reason and stay preview-only. .3fr and .fff are imported from folders. RENDER_CACHE_VERSION 17.
2026-10-09 00:24:39 -04:00
radialmonster 82f904495f LR-IMP-FORMATS: decode lossy JPEG XL DNG tiles completely and without clipping
Decode every JPEG XL tile with wide (32-bit) modular buffers: the
modular_16bit_buffers header flag does not hold on real lossy files (68
of 108 tiles failed with narrow buffers).

Lossy (XYB) tiles are taken from the decoder as linear sRGB primaries
and restored to the declared primaries with a plain 3x3 matrix, instead
of the decoder's gamut-mapped, [0, 1]-clipped conversion. Lossy tiles
whose declared encoding cannot be restored faithfully (ICC profile,
non-linear transfer function, non-D65 white point, intensity target
above 255 nits, single plane) are unsupported. Lossless output is
unchanged.

A JPEG XL tile that fails to decode is now an error for the image
(the file falls back to its embedded preview) instead of a zero tile.

Render cache version 17.
2026-10-09 00:21:39 -04:00
Brandon Thomas ae2f06a035 Merge PR #359 (fmt) 2026-10-09 00:20:30 -04:00
Brandon ThomasandClaude Opus 5.5 b341101ab6 Remote SAM3 worker: cargo fmt
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 00:20:28 -04:00
Brandon Thomas 7365d06c13 Merge PR #359 2026-10-09 00:17:59 -04:00
Brandon Thomas ab73d3d2e0 Merge PR #423 2026-10-09 00:17:58 -04:00
Brandon Thomas c4c96ecabb Merge pull request #439 from storytold/claude/integrate-batch-14
Integrate #429, #426, #422, #395, #172, #173, #174, #335, #427
2026-10-09 00:16:54 -04:00
Brandon Thomas cae7bea08e Merge remote-tracking branch 'origin/main' into fix-359
# Conflicts:
#	crates/engine/src/segment/worker.rs
2026-10-08 23:57:36 -04:00
Brandon Thomas 24615cc2d7 Merge PR #335 (clippy fix) 2026-10-08 23:57:23 -04:00
Brandon ThomasandClaude Opus 5.5 d0749ed7a9 HEIC: as_chunks for 16-bit samples (clippy 1.99, caught by the new heif CI step)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 23:57:22 -04:00
Brandon ThomasandClaude Opus 5.5 bb7c46ae3c Remote SAM3 worker: crate lint attributes; document the trust boundary
The worker binary gets the never-crash deny list and forbid(unsafe_code) like
every production crate root. docs/ai-masks.md spells out who can reach the
unauthenticated loopback listener.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 23:55:26 -04:00
Brandon ThomasandClaude Opus 5.5 7db3b8a1f9 Batch 14: one [features] table in apps/lightcraft (faces + heif), parity summary
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 23:53:54 -04:00
Brandon Thomas a628d5bc38 Merge PR #427
# Conflicts:
#	ROADMAP.md
#	docs/parity.md
2026-10-08 23:46:58 -04:00
Brandon Thomas 42f966eec2 Merge PR #335 2026-10-08 23:46:17 -04:00
Brandon Thomas ca97329ae1 Merge PR #174 2026-10-08 23:45:49 -04:00
Brandon Thomas 3368eb49ee Merge PR #173 2026-10-08 23:45:49 -04:00
Brandon Thomas cbf9935b6e Merge PR #172
# Conflicts:
#	docs/parity.md
2026-10-08 23:45:49 -04:00
Brandon ThomasandClaude Opus 5.5 4977c648f6 Merge main into formats/heif-heif-decode; CI gates the heif feature
Resolve the docs/parity.md conflict, keep the HEIC rows. `cargo xtask ci` gets
a heif step (clippy + tests for lightcraft-codecs with --features heif), since
the workspace build leaves the feature off. The Cargo.toml comments no longer
say official builds enable it; the web one notes that wasm32 aborts on panic.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 23:24:28 -04:00
Brandon Thomas 91d317900f Merge PR #395 2026-10-08 23:22:56 -04:00
sanyamsingh001andCodex 56754e5703 LR-BEHAV-HEADLESS: keep CPU snapshots isolated from saved preferences
Co-authored-by: Codex <codex@openai.com>
2026-10-09 13:19:47 +10:00
radialmonster 04e8aea577 LR-IMP-FORMATS: CR3 photos render in the aspect ratio set in the camera
The Canon maker note's AspectInfo (tag 0x009a: aspect code, cropped width,
height, left, top) holds the rectangle of the shot, measured from the
recommended image crop. Use it as the CR3 default crop, so 3:2, 4:3, 16:9 and
1:1 shots develop in the shape the photographer chose. The tag is ignored when
it is missing, does not fit, or the recommended crop itself is inconsistent.

RENDER_CACHE_VERSION 15 -> 16. The M50 C-RAW corpus sample (raw.pixls.us 2663)
was shot at 1:1 and now develops at 4000 x 4000.
2026-10-08 23:06:57 -04:00
Brandon Thomas 1a2d2ab975 Merge PR #422 2026-10-08 22:43:10 -04:00
Brandon Thomas 1f30856774 Merge PR #426 2026-10-08 22:43:10 -04:00
Brandon Thomas 05769798f6 Merge PR #429 2026-10-08 22:43:10 -04:00
Daniel Voshart 59ab9b6e9b LR-EDIT-DETAIL-DENOISE: refresh the current parity summary
All seven gates pass with the hosted fonts and the earlier shared window-test correction.

Generated with Codex.
2026-10-08 22:31:04 -04:00
Daniel Voshart 111cc893ec LR-LIB-PEOPLE: preserve later person navigation on current main
Port the original Escape return and last-person shortcut behavior, keeping current translations and a no-wrap chip. The existing headless regressions and all seven quality gates pass with the hosted fonts on the refreshed recognition dependency chain.

Generated with Codex.
2026-10-08 22:29:31 -04:00
Daniel Voshart 24f87a91a0 LR-PREF-PEOPLE: update pure-Rust recognition and preserve valid model replacements
Keep current upstream UI behavior and the later detector contract checks. Validate candidate models before replacing working weights and records, with a synthetic regression that failed before the fix. Remove the unrelated GPU equivalence-test relaxation. All seven quality gates pass with the pinned hosted fonts.

Generated with Codex.
2026-10-08 22:29:31 -04:00
Daniel VoshartandClaude Opus 5.5 20a801d6d2 Faces: YuNet face detection and one-click model downloads, with nothing bundled
Photo > Detect Faces runs the YuNet detector (a small pure-Rust interpreter over the ONNX graph, no
runtime dependency) and adds the faces it finds as unnamed face regions in one undo step. The detector
and the recognition models are opt-in downloads: Settings > Faces shows each model's licence and
training-data note, Download stays disabled until the terms are accepted, and the file is fetched over
https with lightcraft-fetch from the model's own repository at a commit-pinned address, checked against
its recorded size and SHA-256 and then installed. Detect Faces offers the detector's download the first
time it is used. No model file is part of the repository (the real-model tests read LC_YUNET_MODEL; two
ignored tests download from the real hosts).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 22:29:30 -04:00
Daniel Voshart 03b665e7a3 LR-PREF-PEOPLE: refresh the generated parity summary
Generated with Codex.
2026-10-08 22:29:29 -04:00
Daniel Voshart fd4019300a LR-EDIT-DETAIL-DENOISE: address installation and model-policy review
Generated with Codex.
2026-10-08 22:07:23 -04:00
7935999d94 LR-EDIT-DETAIL-DENOISE: non-destructive AI RAW denoise with pure-Rust CPU/GPU
Recover the unpublished denoise work against current upstream. Preserve later opt-in model setup, shared Rust HTTPS downloads, pending-photo resume and an Amount-independent switch. Keep original RAWs unchanged and share the current color finish between ordinary and denoised sources.

Bound public tiling inputs, refuse corrupt-cache exports, and retain synthetic/real-model coverage. Include two Windows path spelling fixes in existing tests. No model weights, face code, media or Tract runtime are included.

Validation: cargo xtask ci (all seven gates); installed RawNIND CPU/GPU agreement 1.38e-6; native CPU preview/export on a public 24 MP Sony RAW, unchanged original SHA-256.

Co-authored-by: Codex <noreply@openai.com>

Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>

Generated with Codex.
2026-10-08 22:07:20 -04:00
Daniel Voshart 6e8db91673 LR-PREF-PEOPLE: refresh model manager for current platform helpers
Keep upstream tests and the fixed SAM worker, refresh the workspace version and parity summary, and route the model picker through the case-insensitive native filter helper. Carry the shared translated font-symbol correction; validation includes the pinned hosted fonts and all seven CI gates.

Generated with Codex.
2026-10-08 21:59:02 -04:00
Daniel VoshartandClaude Sonnet 5.5 7845d515e8 Faces: nothing is bundled, so YuNet is not reported as installed
The model list marked the YuNet detector as bundled with LightCraft and installed, but no model ships
inside LightCraft (the weights are opt-in downloads or files the user adds). Drop the notion: a
model is installed when its folder is, it can be removed like any other, and the licence notice no
longer says it is bundled.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:59:02 -04:00
Daniel VoshartandClaude Sonnet 5.5 cb34bb4842 Faces: Settings > Faces tab and the add-a-model licence dialog
An off-by-default "Recognise faces" toggle and the face model list in Settings,
with a one-drop install: pick a .onnx (Add a model file…) or drop one on the
window. The engine inspects it (recognised by hash, or described from its
shape), a dialog shows its licence, whether commercial use is allowed, what it
was trained on and what LightCraft had to assume, and Install stays disabled
until "I accept" is ticked. "Get…" opens a model's own page in the browser:
LightCraft never fetches a model itself. A file it cannot use says why.

The models folder is shared by the app, the CLI and the MCP server
(`engine::config`, `$LIGHTCRAFT_FACE_MODELS` overrides `<config>/models`), so a
model installed from one is there for the others. A session has none until a host
asks (`with_default_face_models`), which keeps tests hermetic.

Nothing runs a model yet, and the tab says so. Headless tests cover the whole
flow (the dialog, refusing until accepted, installing, a file it cannot use).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:59:01 -04:00
Daniel VoshartandClaude Sonnet 5.5 7ca7113b32 Faces: model commands - list, inspect, install behind the licence, select, remove, on/off
`faces.models.list / inspect / install / remove / select` and `faces.enable`, all
reachable from the UI, CLI, control channel and MCP. Models live in a folder the
host provides (`Session::face_models_dir`): one subfolder each with the model, its
manifest and a record of what the user accepted, plus settings.json.

- install needs `acknowledged: true` (the user has been shown the licence), copies
  the file under a .part name, checks the copy's SHA-256 against the original, then
  renames; a failure leaves nothing behind.
- ids are validated before they become folder names (and a manifest that disagrees
  with its folder is ignored); remove only ever deletes a folder that has a model
  manifest. A manifest id can no longer be `.`, `..` or start with a dot.
- inspect describes any .onnx without writing anything; unusable files are an
  answer ("unsupported" with the reason), not a crash.
- the list says plainly that running a model is not built in yet.

`faces::synthetic` makes a minimal valid ONNX for tests (and the later self-check).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:59:01 -04:00
Daniel VoshartandClaude Sonnet 5.5 39de0e6a1c Faces: model manifests, known-model table and an ONNX probe (no runtime yet)
New L1 crate `lightcraft-faces`, the base of a one-drop model install:
- `ModelManifest` (role, licence with a commercial yes / no / unknown flag and a
  plain-language notice, provenance, input preprocessing, output, thresholds)
  with strict validation: sizes capped, numbers finite and sane, https-only
  addresses, bounded text. A manifest from any source is untrusted.
- `known`: models recognised by SHA-256 (YuNet, SFace, AuraFace v1), with what is
  and is not known about their licences and training data.
- `onnx`: a streaming, hostile-input-safe protobuf probe of a model's graph
  inputs and outputs. It reads a 248 MB model in about 4 ms and never loads the
  weights; truncation at every length and random corruption never panic.
- `suggest`: known hash -> its manifest; an unknown model shaped like a face
  embedder -> a draft manifest with the assumptions spelled out; anything else
  -> a plain reason it is not supported yet.
- `hash`: streamed SHA-256.

Checked against the real model files with an opt-in test (LC_FACE_MODELS). It
found an old exporter listing every weight as an input, now handled. Running a
model (tract) is not part of this crate yet.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:59:00 -04:00
Exalted5373andClaude Opus 5.5 3c3b527b7f Web: the window wire test allows the CPU/GPU difference
a_window_job_crosses_the_wire_and_renders_the_same_window compared the worker's
CPU render with the engine job's render bit for bit. On a machine with a GPU the
engine job renders there, so the test failed by 1 LSB (on main too); without a
GPU (CI) both run on the CPU. It now checks both sizes and a maximum channel
difference of 3, the bound docs/gpu-pipeline.md documents.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 18:33:43 -07:00
radialmonster 3d9f8492da LR-IMP-FORMATS: Coolpix NRW black level from the NRW data block
In NRW data version 0104 the u32 at byte 0x20 is the black level in sample units: 200 where the darkest samples of the B700, P950, P1000 and P1100 sit, 0 for the A1000 whose samples reach 0. Used when the maker note has no 0x003d.
2026-10-08 21:26:26 -04:00
radialmonster a12f1697a1 LR-IMP-FORMATS: decode 12-bit raws packed in 32-bit little-endian words (Fujifilm RAF flag 0xf006, Coolpix NRW data block)
Raw-IFD flag 0xf006 = 1 marks RAF samples packed MSB-first into 32-bit little-endian words at any depth; 12-bit files with it (FinePix S1, X-A10, SL1000) were read as an LSB-first stream. Nikon uncompressed 12-bit strips of exactly 1.5 bytes per pixel in a file whose maker note has an NRW data block (COOLPIX B700, P950, P1000, P1100) use the same words in file byte order; without the block (E5700, E8400, E8800) they stay an MSB-first byte stream. Shared unpack_words32_msb and Packing::Words32Msb. RENDER_CACHE_VERSION 15 to 16.
2026-10-08 21:26:26 -04:00
Exalted5373andClaude Opus 5.5 0ac5cf4430 LR-EDIT-COLOR-TEMP: camera-space white balance for raws with a colour matrix
A custom white balance on a raw with a colour matrix (DNGs, and other raws
without a file-local look) re-develops the pixels for the chosen white
through the file's own colour model: lightcraft_raw::color::rebalance
re-evaluates the camera transform at the new white and maps the as-shot
development onto it, so a neutral lit by that white renders neutral and
saturated colours move as the camera's matrices say (DNG spec ch. 6, as
Lightroom does). The decoder passes the colour tags and the white the
pixels were developed for in SourceInfo::camera_color; other photos, and
relative white balances, keep the Bradford adaptation. The eyedropper and
Auto invert the same model (color::neutral_white), so a picked grey card
renders neutral.

The Tint sign is main's (#349): unchanged here. Render cache version 15.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 18:18:54 -07:00
Exalted5373andClaude Opus 5.5 ce7b8f511e LR-IMP-FORMATS: read DNG BaselineSharpness and keep it in DNG exports
ColorData::baseline_sharpness holds the file's BaselineSharpness (raw IFD
or IFD 0; zero, negative, non-finite or above 16 is ignored) and the DNG
writer writes it back, so a conversion or smart preview keeps it. Merge
outputs leave it unset. Rendering doesn't use it yet.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 18:18:54 -07:00
Exalted5373andClaude Opus 5.5 ed59d78e83 LR-MASK-SKY: AI masks use DNG semantic mattes (iPhone ProRAW sky matte)
Read DNG 1.6 semantic masks (SemanticName, SemanticInstanceID,
MaskSubArea; any coding the tile/strip reader decodes, e.g. ProRAW's
lossy-JPEG matte) with lightcraft_raw::semantic_masks, map them onto the
developed image, and carry Apple's sky / portrait / skin / hair / teeth /
glasses mattes in SourceInfo::mattes (SourceInfo is no longer Copy).
Sky, Subject, Background and People components sample them through
orientation, warp (Warp::frame, as geometry.rs) and crop like the image,
so preview = export, instead of the heuristics; files without mattes
render as before. Malformed mask IFDs are skipped, and only the first
MAX_MASKS mask IFDs are decoded, so a file full of them costs bounded
work. The render cache version is bumped once, in the white-balance commit.

The corpus test reads corpus/raw/dng-apple-iphone12pro-proraw.dng, the
name `cargo xtask corpus` downloads it under.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 18:18:54 -07:00
Exalted5373andClaude Opus 5.5 a28dc4dbdf LR-PROF-CAMERACOLOR: read and keep DNG ProfileGainTableMap (not rendered)
Parse ProfileGainTableMap (DNG 1.6) and ProfileGainTableMap2 (DNG 1.7):
both versions, DNG 1.7 precedence (version 2 in IFD 0, then the raw IFD;
version 1 in the raw IFD, then IFD 0), u8/u16/f16/f32 gains, gamma.
Malformed tags (wrong size, zero or huge grids, non-finite or negative
gains, bad spacing or gamma) are ignored, never allocated for.

Byte order: the specification gives this tag no byte order of its own,
unlike opcode lists, so the payload is read in the file's byte order with
a big-endian fallback; a wrong-order read can't pass the size checks.
The module docs quote the DNG 1.6 "Byte Order" rule.

The map is kept in ColorData, written back by the DNG writer (version 1,
or version 2 when its gamma isn't 1) and dropped from merge outputs, whose
geometry no longer matches the capture's active area. It is not applied
to the render: Lightroom Classic renders Apple ProRAW without it.
Supersedes the parser of #273.

The CC0 iPhone 12 Pro ProRAW from raw.pixls.us joins the corpus.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 18:18:16 -07:00
Exalted5373andClaude Opus 5.5 1ed80c65bf LR-IMP-FORMATS: decode JPEG XL compressed DNG tiles (DNG 1.7)
Compression 52546 tiles and strips decode through jxl-oxide (new
default-on `jxl` feature of lightcraft-raw): bare codestreams and
ISO-BMFF containers, 1 or 3 planes, 8-16-bit integers scaled back
exactly from the codestream's own bit depth, 16-bit floats. Edge tiles
may be coded at their nominal or in-image size. Headers are checked
against the IFD in header-only probing too. A single-chunk JPEG XL
preview IFD (RGB or grey) is offered as the embedded preview.

The decoder's memory is bounded by the tile: 16 bytes per sample (three
times that for a lossy grey tile, which the decoder expands to three
channels) plus 64 KiB; jxl-oxide 0.12 was measured at <= 10.3 bytes per
sample for lossless and <= 13.5 per colour sample for lossy tiles. A
codestream needing more (e.g. extra channels) is an error.

Lossless tiles are checked bit-exact on synthetic CFA and LinearRaw
files (zune-jpegxl as a test-only encoder); lossy tiles are unverified
on a real file.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 18:18:16 -07:00
BFlat b1883231bd Merge pull request #424 from storytold/claude/integrate-batch-13
Integrate #403, #411, #412, #414, #415, #416
2026-10-09 08:15:07 +07:00
Nathaniel LedfordandClaude Sonnet 5.5 733846b326 Docs: sidebar auto-scroll while dragging
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:07 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 8ff45f5237 Sidebar: dragging albums or photos near its top or bottom edge scrolls it
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:07 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 5fefe91640 Docs: top-level drop is on the Albums header
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:07 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 24b80b4ee3 Albums: the Albums header is the way back to the top level
It replaces the strip under the tree, which could be scrolled out of view
and shifted the rows when it appeared; the header is always visible and
nothing moves.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:07 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 922ea25d88 Docs: album tree, drag and hand order (parity, what's new)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:07 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 085c25b10b Albums: drop beside a row to place it by hand; Sort A–Z goes back to by name
Dropping on the edge of a row places the album there (album.reorder) with
a line showing where; a folder's middle still means inside. A closed folder
opens only when the drop would go inside it. Sort Contents A–Z and Sort
Albums A–Z (album.sort) appear once a folder is ordered by hand.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:07 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 8036f98aca Catalog: tidy a test assertion
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:07 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 34a4b61f7e Albums: one-pass children map, before-itself is a no-op, more order tests
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:07 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 2c5cb708bb Engine: album.reorder and album.sort
Place an album or folder before a sibling of its kind (also in another
folder), or last of its kind, in one undo step; album.sort goes back to
by name for one folder.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:07 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 39f5f114de Catalog: albums can be ordered by hand (format version 3)
Album.order and Op::SetAlbumOrder; album_children lists folders first, then
by place, then by name. Moving an album drops its place and undo restores it.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:07 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 67aed0b4f4 Albums: a drag ends with the main button, Esc cancels it, hidden rows are no targets
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:07 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 63335e2b0a Parity: albums and folders move by dragging onto a folder
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:07 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 aaae39fae0 Albums: drag an album or folder onto a folder to move it
A folder under the pointer is outlined when the drop is allowed and opens
after resting there; a drop zone under the tree takes an album back to the
top level. The move is album.move, so it is one undo step.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:07 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 5443ae28f9 Albums: address review of folder creation
Show a dialog's error and keep it open instead of closing it, open the
folder menu from the triangle too, draw a triangle only on folders that
hold something, reveal even when the section is folded, reuse the
translated Create … labels, and fix a misplaced doc comment.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:06 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 4abf70e254 Parity: album folders fold like the other trees and create inside themselves
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:06 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 79403a5680 Albums: create albums, smart albums and folders inside a folder from its context menu
The dialog.new* commands take the folder to create in, the dialogs
remember it, and the Albums tree opens the folders down to the new row.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:06 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 96a9c6e04d Albums: folders get the same disclosure triangle as the other sidebar trees
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:06 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 e8b18ae636 Sidebar: one disclosure-triangle helper for the Local, By Date, Folders and Keywords trees
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:06 -04:00
Nathaniel LedfordandClaude Sonnet 5.5 a46c0071a0 Albums: pin that albums are created only inside folders
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-08 21:02:06 -04:00
Claude 12188161f9 Merge remote-tracking branch 'origin/main' into integ-b13 2026-10-09 00:46:05 +00:00
BFlat de79310ae0 Merge pull request #419 from storytold/claude/integrate-batch-12
Integrate #351 (sharp zoomed Detail view, #323) with follow-ups
2026-10-09 07:45:11 +07:00
radialmonster caa102aabd MENU-BAR: pointing at another menu title switches the open menu in the in-window bar 2026-10-08 20:40:46 -04:00
Claude 373efc280e Merge PR #416 2026-10-09 00:20:27 +00:00
Claude c75389dda1 Merge PR #415 2026-10-09 00:20:27 +00:00
Claude 8ee6d63d41 Merge PR #414 2026-10-09 00:20:27 +00:00
Claude f578b27273 Merge PR #412 2026-10-09 00:20:26 +00:00
Claude 43419dcfb8 Merge PR #411 2026-10-09 00:20:26 +00:00
Claude 83ea5e14df Merge PR #403 2026-10-09 00:20:26 +00:00
ragtuxandClaude Fable 5.1 0cf3ceb438 Part of #260: Help ▸ Open Log Folder shows the log file in the file manager
The desktop app keeps logs/lightcraft.log in its settings folder (#286)
and a failed start names it (#370), but a user still had to find the
folder by hand. The host now hands the UI its log file (Services
.log_file) and Help ▸ Open Log Folder (app.openLogFolder) reveals it
through the existing reveal service: Finder and Explorer select the
file, Linux opens the folder. Off where no log is kept (the web,
--memory, LIGHTCRAFT_NO_PREFS). Translated in de, es, ja, pt-br, ru,
zh-hans and zh-hant; parity row MENU-HELP-LOGFOLDER; What's New and
README note it.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-08 19:13:35 -05:00
midasdf 93b900f8d7 M16: keep grapheme clusters intact in vertical watermarks 2026-10-09 09:02:24 +09:00
midasdf 0cb78071de M16: use paintable separators in localized UI text 2026-10-09 08:55:21 +09:00
Exalted5373andClaude Opus 5.5 7173242ab5 LR-IMP-FORMATS: no overflow deciding whether to scale a full-size JPEG decode
A full-size load asks the decoder for the largest box (`SourceLevel::Full` is
`usize::MAX`, cast to `u32::MAX`). `jpeg::decode` then computed `2 * u32::MAX` to
decide on DCT scaling: an overflow panic in debug builds (caught, so every
full-size JPEG load failed with "decoder panicked"), and a wrap that happened to
give the right answer in release. The comparison saturates now, and the engine
converts its `usize` edge with `u32::try_from` instead of a truncating `as`.

Found decoding camera JPEGs (no JFIF/EXIF header) at full size.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 16:26:29 -07:00
Exalted5373andClaude Opus 5.5 31aa638383 LR-IMP-PROFILES: red / green / blue curves only with the master curve; neutral test values
Lightroom Classic reads `ToneCurvePV2012Red/Green/Blue` only together with
`ToneCurvePV2012`: a packet with a channel curve and no master curve leaves the
photo unchanged there (rendered on a chart: the channel curve alone has no
effect, with an identity master curve it applies). Imports now do the same.

The preset-packet test helper uses made-up as-shot values.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 16:25:25 -07:00
Exalted5373andClaude Opus 5.5 09d5284e5a LR-EDIT-OPTICS-PROFILE: a preset's LensProfileEnable=0 keeps embedded lens corrections
XMP presets usually say LensProfileEnable=0, which turned optics.lens_profile
off and with it the DNG-embedded WarpRectilinear / FixVignetteRadial
corrections (iPhone ProRAW and other files that carry them). Lightroom
applies a file's built-in corrections whatever that switch says (our
inference), and LightCraft's profile corrections are only those embedded
ones, so crs import now carries over only an enabling switch. The Enable
Lens Corrections checkbox and the amount sliders keep working as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 16:25:25 -07:00
Exalted5373andClaude Opus 5.5 7e0250e11e LR-IMP-PROFILES: don't report preset bookkeeping fields as unmapped
XMP presets carry fields that describe the preset or the photo rather than
an adjustment (Cluster, SupportsAmount2, RequiresRGBTables, SortName,
Description, CropConstrainToWarp, OverrideLookVignette, AsShotTemperature /
AsShotTint). preset.import listed them as settings that couldn't be carried
over; so did an HDREditMode of 0 and PointColors slots that are all -1.
They are now treated as bookkeeping; an HDR edit, a used Point Color slot and
unknown adjustments are still reported.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 16:25:25 -07:00
Exalted5373andClaude Opus 5.5 9087d3ad7f PSD: as_chunks for the palette and 16/32-bit plane loops (clippy 1.98)
clippy 1.98's manual_chunks lint fails `cargo xtask ci` on main at psd.rs:170/191/201
(chunks_exact with a constant size). The fixed-size chunks also drop the per-sample
indexing. No behaviour change; the codecs tests pass.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 16:25:17 -07:00
Joshua Warren 0613f925de Support remote SAM3 inference through a private Metal worker 2026-10-08 11:47:59 -05:00
Walter Boringandopencode 853ada7754 LR-IMP-FORMATS: HEIC/HEIF decode behind the optional heif feature
New crate `lightcraft-heif` (a port of PhotoCraft's photocraft-heif): a thin,
panic-guarded wrapper around heic-rs (pure Rust, pinned =0.1.1) - 8/10-bit,
alpha, grid tiles, container irot/imir/clap, ICC/EXIF/XMP. `lightcraft-codecs`
decodes HEIF behind its non-default `heif` feature (HEVC patents are the
distributor's call, the same policy as PhotoCraft); the apps forward it. HEIC
files stay recognised without the feature: opening one is a clear
"isn't included in this build" error. Fuzz-found heic-rs panics arrive as
regression fixtures (inline bytes, no media committed).

Verified end-to-end on real iPhone HEICs (lightcraft-cli render, rotation and
metadata correct). Note for reviewers: measured here, the engine currently
ignores the decoded image's source-space matrix on export for ANY format (a
true Display-P3 PNG round-trips unchanged); the ICC and space are carried for
all codecs, so this inherits - and does not worsen - existing behaviour.

Co-Authored-By: opencode <noreply@opencode.ai>
2026-10-08 12:45:33 -04:00
248 changed files with 35299 additions and 912 deletions

No files matched your search

+4 -3
View File
@@ -74,9 +74,10 @@ of it. Full standard: `../craftrules/standards/never-crash.md`
font files here (Inter, already in `assets/fonts/`, is the one exception); add new fonts to craft-fonts. LightCraft
uses it as the optional build input `CRAFT_FONTS_DIR`: `git clone https://github.com/storytold/craft-fonts ../craft-fonts`
then `CRAFT_FONTS_DIR=../craft-fonts cargo run -p lightcraft` (or any cargo/xtask command). `crates/engine/build.rs`
embeds the manifest's fonts as `lightcraft_engine::CRAFT_FONTS` (wasm32: BIZ UDPGothic Regular only); the UI
(`theme::font_definitions`) and the export watermark renderer use its CJK faces (picked by script) as fallbacks after Inter. Unset,
`CRAFT_FONTS` is empty: everything builds, tests and runs, but Japanese text has no glyphs. Releases always build
embeds the manifest's fonts as `lightcraft_engine::CRAFT_FONTS` (wasm32: BIZ UDPGothic Regular only); `cargo xtask web`
ships Noto Sans CJK SC Regular as a separate asset loaded by the UI before startup. The UI
(`theme::font_definitions`) and the export watermark renderer use CJK faces (picked by script) as fallbacks after Inter. Unset,
`CRAFT_FONTS` is empty: everything builds, tests and runs, but Chinese and Japanese text lack glyphs. Releases always build
with it (`release.yml`, `CRAFT_FONTS_REQUIRED=1`) and ship the fonts' OFL licences. Tests that need these fonts skip
without it; the FreeBSD CI job runs them with it. Rules: craftrules
[`standards/fonts.md`](https://github.com/storytold/craftrules/blob/main/standards/fonts.md).
+4 -3
View File
@@ -74,9 +74,10 @@ of it. Full standard: `../craftrules/standards/never-crash.md`
font files here (Inter, already in `assets/fonts/`, is the one exception); add new fonts to craft-fonts. LightCraft
uses it as the optional build input `CRAFT_FONTS_DIR`: `git clone https://github.com/storytold/craft-fonts ../craft-fonts`
then `CRAFT_FONTS_DIR=../craft-fonts cargo run -p lightcraft` (or any cargo/xtask command). `crates/engine/build.rs`
embeds the manifest's fonts as `lightcraft_engine::CRAFT_FONTS` (wasm32: BIZ UDPGothic Regular only); the UI
(`theme::font_definitions`) and the export watermark renderer use its CJK faces (picked by script) as fallbacks after Inter. Unset,
`CRAFT_FONTS` is empty: everything builds, tests and runs, but Japanese text has no glyphs. Releases always build
embeds the manifest's fonts as `lightcraft_engine::CRAFT_FONTS` (wasm32: BIZ UDPGothic Regular only); `cargo xtask web`
ships Noto Sans CJK SC Regular as a separate asset loaded by the UI before startup. The UI
(`theme::font_definitions`) and the export watermark renderer use CJK faces (picked by script) as fallbacks after Inter. Unset,
`CRAFT_FONTS` is empty: everything builds, tests and runs, but Chinese and Japanese text lack glyphs. Releases always build
with it (`release.yml`, `CRAFT_FONTS_REQUIRED=1`) and ship the fonts' OFL licences. Tests that need these fonts skip
without it; the FreeBSD CI job runs them with it. Rules: craftrules
[`standards/fonts.md`](https://github.com/storytold/craftrules/blob/main/standards/fonts.md).
Generated
+377 -33
View File
@@ -209,7 +209,7 @@ version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b"
dependencies = [
"equator",
"equator 0.4.2",
]
[[package]]
@@ -967,6 +967,12 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "coe-rs"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e8f1e641542c07631228b1e0dc04b69ae3c1d58ef65d5691a439711d805c698"
[[package]]
name = "color"
version = "0.3.3"
@@ -1256,6 +1262,16 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76"
[[package]]
name = "dyn-stack"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b"
dependencies = [
"bytemuck",
"reborrow",
]
[[package]]
name = "dyn-stack"
version = "0.13.2"
@@ -1537,13 +1553,33 @@ version = "0.36.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773fa9c96dd0dbef887e39d0ed6177f141cce4d68e6041df77570aa3702dfa13"
[[package]]
name = "equator"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3b0a88aa91d0ad2b9684e4479aed31a17d3f9051bdbbc634bd2c01bc5a5eee8"
dependencies = [
"equator-macro 0.1.9",
]
[[package]]
name = "equator"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc"
dependencies = [
"equator-macro",
"equator-macro 0.4.2",
]
[[package]]
name = "equator-macro"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60d08acb9849f7fb4401564f251be5a526829183a3645a90197dea8e786cf3ae"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
@@ -1608,6 +1644,42 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "faer-core"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32da0f6262445261106407bdfb548df4f8f1c89bcc3cff1bb6d08e7f525b575a"
dependencies = [
"bytemuck",
"coe-rs",
"dyn-stack 0.10.0",
"equator 0.1.10",
"faer-entity",
"gemm 0.17.1",
"matrixcompare-core",
"num-complex",
"num-traits",
"paste",
"pulp 0.18.22",
"reborrow",
"seq-macro",
]
[[package]]
name = "faer-entity"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fb12f36da3162b3fefc0c7343d905d5bd2c2bf444a7f63ceeaa2bc003311d3"
dependencies = [
"bytemuck",
"coe-rs",
"libm",
"num-complex",
"num-traits",
"pulp 0.18.22",
"reborrow",
]
[[package]]
name = "fastrand"
version = "2.5.0"
@@ -1783,13 +1855,33 @@ dependencies = [
"slab",
]
[[package]]
name = "gemm"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32"
dependencies = [
"dyn-stack 0.10.0",
"gemm-c32 0.17.1",
"gemm-c64 0.17.1",
"gemm-common 0.17.1",
"gemm-f16 0.17.1",
"gemm-f32 0.17.1",
"gemm-f64 0.17.1",
"num-complex",
"num-traits",
"paste",
"raw-cpuid 10.7.0",
"seq-macro",
]
[[package]]
name = "gemm"
version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab96b703d31950f1aeddded248bc95543c9efc7ac9c4a21fda8703a83ee35451"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-c32 0.18.2",
"gemm-c64 0.18.2",
"gemm-common 0.18.2",
@@ -1799,7 +1891,7 @@ dependencies = [
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
@@ -1809,7 +1901,7 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa0673db364b12263d103b68337a68fbecc541d6f6b61ba72fe438654709eacb"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-c32 0.19.0",
"gemm-c64 0.19.0",
"gemm-common 0.19.0",
@@ -1819,7 +1911,22 @@ dependencies = [
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
[[package]]
name = "gemm-c32"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0"
dependencies = [
"dyn-stack 0.10.0",
"gemm-common 0.17.1",
"num-complex",
"num-traits",
"paste",
"raw-cpuid 10.7.0",
"seq-macro",
]
@@ -1829,12 +1936,12 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6db9fd9f40421d00eea9dd0770045a5603b8d684654816637732463f4073847"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.18.2",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
@@ -1844,12 +1951,27 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "086936dbdcb99e37aad81d320f98f670e53c1e55a98bee70573e83f95beb128c"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.19.0",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
[[package]]
name = "gemm-c64"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a"
dependencies = [
"dyn-stack 0.10.0",
"gemm-common 0.17.1",
"num-complex",
"num-traits",
"paste",
"raw-cpuid 10.7.0",
"seq-macro",
]
@@ -1859,12 +1981,12 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfcad8a3d35a43758330b635d02edad980c1e143dc2f21e6fd25f9e4eada8edf"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.18.2",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
@@ -1874,15 +1996,34 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20c8aeeeec425959bda4d9827664029ba1501a90a0d1e6228e48bef741db3a3f"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.19.0",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
[[package]]
name = "gemm-common"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8"
dependencies = [
"bytemuck",
"dyn-stack 0.10.0",
"half",
"num-complex",
"num-traits",
"once_cell",
"paste",
"pulp 0.18.22",
"raw-cpuid 10.7.0",
"seq-macro",
"sysctl 0.5.5",
]
[[package]]
name = "gemm-common"
version = "0.18.2"
@@ -1890,7 +2031,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a352d4a69cbe938b9e2a9cb7a3a63b7e72f9349174a2752a558a8a563510d0f3"
dependencies = [
"bytemuck",
"dyn-stack",
"dyn-stack 0.13.2",
"half",
"libm",
"num-complex",
@@ -1898,10 +2039,10 @@ dependencies = [
"once_cell",
"paste",
"pulp 0.21.5",
"raw-cpuid",
"raw-cpuid 11.6.0",
"rayon",
"seq-macro",
"sysctl",
"sysctl 0.6.0",
]
[[package]]
@@ -1911,7 +2052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88027625910cc9b1085aaaa1c4bc46bb3a36aad323452b33c25b5e4e7c8e2a3e"
dependencies = [
"bytemuck",
"dyn-stack",
"dyn-stack 0.13.2",
"half",
"libm",
"num-complex",
@@ -1919,10 +2060,27 @@ dependencies = [
"once_cell",
"paste",
"pulp 0.22.3",
"raw-cpuid",
"raw-cpuid 11.6.0",
"rayon",
"seq-macro",
"sysctl",
"sysctl 0.6.0",
]
[[package]]
name = "gemm-f16"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4"
dependencies = [
"dyn-stack 0.10.0",
"gemm-common 0.17.1",
"gemm-f32 0.17.1",
"half",
"num-complex",
"num-traits",
"paste",
"raw-cpuid 10.7.0",
"seq-macro",
]
[[package]]
@@ -1931,14 +2089,14 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cff95ae3259432f3c3410eaa919033cd03791d81cebd18018393dc147952e109"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.18.2",
"gemm-f32 0.18.2",
"half",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"rayon",
"seq-macro",
]
@@ -1949,30 +2107,45 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3df7a55202e6cd6739d82ae3399c8e0c7e1402859b30e4cb780e61525d9486e"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.19.0",
"gemm-f32 0.19.0",
"half",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"rayon",
"seq-macro",
]
[[package]]
name = "gemm-f32"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113"
dependencies = [
"dyn-stack 0.10.0",
"gemm-common 0.17.1",
"num-complex",
"num-traits",
"paste",
"raw-cpuid 10.7.0",
"seq-macro",
]
[[package]]
name = "gemm-f32"
version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc8d3d4385393304f407392f754cd2dc4b315d05063f62cf09f47b58de276864"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.18.2",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
@@ -1982,12 +2155,27 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02e0b8c9da1fbec6e3e3ab2ce6bc259ef18eb5f6f0d3e4edf54b75f9fd41a81c"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.19.0",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
[[package]]
name = "gemm-f64"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0"
dependencies = [
"dyn-stack 0.10.0",
"gemm-common 0.17.1",
"num-complex",
"num-traits",
"paste",
"raw-cpuid 10.7.0",
"seq-macro",
]
@@ -1997,12 +2185,12 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35b2a4f76ce4b8b16eadc11ccf2e083252d8237c1b589558a49b0183545015bd"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.18.2",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
@@ -2012,12 +2200,12 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "056131e8f2a521bfab322f804ccd652520c79700d81209e9d9275bbdecaadc6a"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.19.0",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
@@ -2296,6 +2484,15 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "heic-rs"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adf9dde634bb6d5b718bacd01bef80192f2334306b2f785519a0136cca7b9618"
dependencies = [
"rayon",
]
[[package]]
name = "hermit-abi"
version = "0.5.3"
@@ -2837,6 +3034,7 @@ dependencies = [
"jxl-oxide",
"lightcraft-color",
"lightcraft-geom",
"lightcraft-heif",
"lightcraft-raster",
"moxcms 0.9.1",
"png",
@@ -2859,6 +3057,38 @@ dependencies = [
"serde",
]
[[package]]
name = "lightcraft-denoise"
version = "0.4.0"
dependencies = [
"faer-core",
"lightcraft-denoise-core",
"lightcraft-raster",
"miniz_oxide 0.8.9",
"proptest",
"rayon",
"serde",
"serde_json",
"sha2",
"thiserror 2.0.21",
"web-time",
]
[[package]]
name = "lightcraft-denoise-core"
version = "0.4.0"
dependencies = [
"lightcraft-raster",
"miniz_oxide 0.8.9",
"proptest",
"rayon",
"serde",
"serde_json",
"sha2",
"thiserror 2.0.21",
"web-time",
]
[[package]]
name = "lightcraft-develop"
version = "0.4.0"
@@ -2881,7 +3111,11 @@ dependencies = [
"lightcraft-catalog",
"lightcraft-codecs",
"lightcraft-color",
"lightcraft-denoise",
"lightcraft-denoise-core",
"lightcraft-develop",
"lightcraft-faces",
"lightcraft-fetch",
"lightcraft-geom",
"lightcraft-gpu",
"lightcraft-merge",
@@ -2899,6 +3133,21 @@ dependencies = [
"serde",
"serde_json",
"thiserror 2.0.21",
"unicode-segmentation",
"web-time",
]
[[package]]
name = "lightcraft-faces"
version = "0.4.0"
dependencies = [
"matrixmultiply",
"ndarray",
"proptest",
"serde",
"serde_json",
"sha2",
"thiserror 2.0.21",
"web-time",
]
@@ -2927,6 +3176,7 @@ version = "0.4.0"
dependencies = [
"bytemuck",
"lightcraft-color",
"lightcraft-denoise",
"lightcraft-develop",
"lightcraft-geom",
"lightcraft-pipeline",
@@ -2937,6 +3187,13 @@ dependencies = [
"wgpu",
]
[[package]]
name = "lightcraft-heif"
version = "0.4.0"
dependencies = [
"heic-rs",
]
[[package]]
name = "lightcraft-mcp"
version = "0.4.0"
@@ -2985,6 +3242,7 @@ dependencies = [
"lightcraft-develop",
"lightcraft-geom",
"lightcraft-raster",
"lightcraft-raw",
"lightcraft-scenes",
"proptest",
"rayon",
@@ -3021,6 +3279,7 @@ name = "lightcraft-raw"
version = "0.4.0"
dependencies = [
"jpeg-encoder",
"jxl-oxide",
"lightcraft-color",
"lightcraft-geom",
"lightcraft-meta",
@@ -3034,6 +3293,7 @@ dependencies = [
"thiserror 2.0.21",
"zune-core",
"zune-jpeg",
"zune-jpegxl",
]
[[package]]
@@ -3081,8 +3341,10 @@ dependencies = [
"lightcraft-catalog",
"lightcraft-codecs",
"lightcraft-color",
"lightcraft-denoise",
"lightcraft-develop",
"lightcraft-engine",
"lightcraft-faces",
"lightcraft-geom",
"lightcraft-meta",
"lightcraft-pipeline",
@@ -3173,6 +3435,22 @@ dependencies = [
"libc",
]
[[package]]
name = "matrixcompare-core"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0bdabb30db18805d5290b3da7ceaccbddba795620b86c02145d688e04900a73"
[[package]]
name = "matrixmultiply"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7"
dependencies = [
"autocfg",
"rawpointer",
]
[[package]]
name = "maybe-rayon"
version = "0.1.1"
@@ -3326,6 +3604,21 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "ndarray"
version = "0.17.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d"
dependencies = [
"matrixmultiply",
"num-complex",
"num-integer",
"num-traits",
"portable-atomic",
"portable-atomic-util",
"rawpointer",
]
[[package]]
name = "ndk"
version = "0.9.0"
@@ -4290,6 +4583,18 @@ dependencies = [
"unarray",
]
[[package]]
name = "pulp"
version = "0.18.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6"
dependencies = [
"bytemuck",
"libm",
"num-complex",
"reborrow",
]
[[package]]
name = "pulp"
version = "0.21.5"
@@ -4316,7 +4621,7 @@ dependencies = [
"num-complex",
"paste",
"pulp-wasm-simd-flag",
"raw-cpuid",
"raw-cpuid 11.6.0",
"reborrow",
"version_check",
]
@@ -4517,6 +4822,15 @@ dependencies = [
"rgb",
]
[[package]]
name = "raw-cpuid"
version = "10.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "raw-cpuid"
version = "11.6.0"
@@ -4544,6 +4858,12 @@ dependencies = [
"objc2-quartz-core 0.3.2",
]
[[package]]
name = "rawpointer"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
[[package]]
name = "rayon"
version = "1.12.0"
@@ -5244,6 +5564,20 @@ dependencies = [
"syn 3.0.6",
]
[[package]]
name = "sysctl"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea"
dependencies = [
"bitflags 2.13.2",
"byteorder",
"enum-as-inner",
"libc",
"thiserror 1.0.69",
"walkdir",
]
[[package]]
name = "sysctl"
version = "0.6.0"
@@ -6490,6 +6824,7 @@ dependencies = [
"brotli",
"flate2",
"serde_json",
"sha2",
]
[[package]]
@@ -6751,6 +7086,15 @@ dependencies = [
"zune-core",
]
[[package]]
name = "zune-jpegxl"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cba11ecd07cc23351500e840ae03841b7e4997923ec8e4832ee3ae199ea0b6b4"
dependencies = [
"zune-core",
]
[[package]]
name = "zvariant"
version = "5.15.0"
+6
View File
@@ -11,7 +11,10 @@ repository = "https://github.com/storytold/lightcraft"
[workspace.dependencies]
# internal
lightcraft-denoise-core = { path = "crates/denoise-core" }
lightcraft-denoise = { path = "crates/denoise", default-features = false }
lightcraft-geom = { path = "crates/geom" }
lightcraft-faces = { path = "crates/faces" }
lightcraft-sysmem = { path = "crates/sysmem" }
lightcraft-color = { path = "crates/color" }
lightcraft-raster = { path = "crates/raster" }
@@ -30,10 +33,12 @@ lightcraft-engine = { path = "crates/engine" }
lightcraft-ui-egui = { path = "crates/ui-egui" }
lightcraft-mcp = { path = "crates/mcp" }
lightcraft-fetch = { path = "crates/fetch" }
lightcraft-heif = { path = "crates/heif" }
lightcraft-segment = { path = "crates/segment" }
# foundation
thiserror = "2"
sha2 = "0.10"
serde = { version = "1", features = ["derive", "rc"] }
serde_json = { version = "1", features = ["float_roundtrip"] }
log = "0.4"
@@ -61,6 +66,7 @@ tiff = "0.11"
image-webp = "0.2"
moxcms = { version = "0.9", default-features = false, features = ["lut"] }
jxl-oxide = { version = "0.12", default-features = false }
zune-jpegxl = { version = "0.5", default-features = false, features = ["std"] } # MIT/Apache-2.0/Zlib lossless JXL encoder (test fixtures only)
ravif = { version = "0.13", default-features = false, features = ["threading"] } # no "asm": it needs nasm (non-Rust)
# gpu (crates/gpu; native backends only, no GL: everything stays pure Rust)
+9
View File
@@ -27,6 +27,15 @@ the facebook/sam3 checkpoint only when they choose to (in the app, or by hand),
License (https://github.com/facebookresearch/sam3/blob/main/LICENSE), a separate, non-OSI
licence. See docs/ai-masks.md.
AI denoise weights are not part of LightCraft and are never distributed with it.
The default build offers user-supplied models only. The optional rawnind-model
feature offers a separately downloaded RawNIND UtNet2 Bayer checkpoint under
GPL-3.0 (https://www.gnu.org/licenses/gpl-3.0.html), after the user accepts its terms.
Commercial use is subject to those terms, including applicable source, licence and
notice obligations on redistribution. Enabling that offer in releases remains a
maintainer decision. The ONNX reader and CPU/GPU executor are our Rust implementation;
no darktable, RawSpeed, LibRaw or other GPL decoder/runtime code is included.
Builds made with craft-fonts (the optional CRAFT_FONTS_DIR build input; all official releases)
also embed the fonts listed in https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md
(the Japanese fonts BIZ UDPGothic, BIZ UDMincho and Shippori Mincho; the web build embeds only BIZ
+5 -3
View File
@@ -260,7 +260,8 @@ lightcraft --control 7980 ~/Pictures/trip
- **Pure Rust, no C.** Our own RAW decoders (DNG, Canon CR2/CR3, Sony ARW, Nikon NEF, Fujifilm RAF incl. X-Trans,
Panasonic RW2 / Leica RWL, Pentax PEF, Olympus ORF), our own colour science, our own pipeline. JPEG, PNG, TIFF, WebP,
PSD composites and JPEG XL open today.
PSD composites and JPEG XL open today; HEIC/HEIF photos too with `--features heif` (HEVC is a build-time choice,
as in PhotoCraft).
- **Scene-referred & wide-gamut.** Linear Rec.2020 float internally, Bradford-adapted white balance, gamut mapping
instead of clipping, a filmic shoulder for raw and pixel-exact pass-through for JPEGs you haven't touched.
- **Resolution-independent edits.** Radii and brush sizes are relative to the image, so a 400 px preview, your
@@ -304,7 +305,7 @@ LightCraft is young and moving fast. **Where we honestly stand** (details in the
| Profiles (Color, Neutral, Vivid, Landscape, Portrait, Monochrome: our own looks), presets, versions, history, copy/paste/sync settings | ✅ |
| Camera colour: DNG files use their own matrices | ✅ DNG · 🟡 own Sony/Fujifilm profiles; measured calibration database missing |
| Native macOS menu bar (generated from the command registry), control channel + every widget addressable, headless UI snapshots | ✅ |
| RAW: DNG, CR2, CR3 (lossless CRX Bayer and version 0x100/0x200 C-RAW), ARW, NEF (uncompressed + lossless/lossy compressed), Fujifilm RAF (uncompressed + lossless/lossy compressed, Bayer + X-Trans), Panasonic RW2 / Leica RWL / Panasonic RAW (every raw format, DMC-LX1 to DC-S1RM2), Pentax PEF, Olympus ORF (uncompressed); embedded previews for every format incl. CR3 | 🟡 · CR3, compressed ORF decode ⬜ |
| RAW: DNG, CR2, CR3 (lossless CRX Bayer and version 0x100/0x200 C-RAW), ARW, NEF (uncompressed + lossless/lossy compressed), Fujifilm RAF (uncompressed + lossless/lossy compressed, Bayer + X-Trans), Panasonic RW2 / Leica RWL / Panasonic RAW (every raw format, DMC-LX1 to DC-S1RM2), Pentax PEF, Olympus ORF (uncompressed), Samsung SRW (uncompressed); embedded previews for every format incl. CR3 | 🟡 · CR3, compressed ORF and compressed SRW decode ⬜ |
| Detail: sharpening, luminance + colour noise reduction | ✅ · AI Denoise, Super Resolution ⬜ |
| Remove / Heal / Clone spots (auto source), Visualize Spots, Red Eye and Pet Eye (auto pupil detection, catchlight) | ✅ · content-aware fill, spot pin editing 🚧 |
| Export: JPEG / PNG / TIFF / WebP / AVIF / DNG / original, sizing, file-size limit, output sharpening, naming templates, batch, metadata policy, text or image watermark | ✅ · HDR export ⬜ |
@@ -359,7 +360,8 @@ Language**; the choice applies immediately and persists. See [docs/localization.
**Logs:** the desktop app writes its log to standard error and to `logs/lightcraft.log` in its settings folder
(Linux `$XDG_CONFIG_HOME/lightcraft/logs/`, by default `~/.config/lightcraft/logs/`; macOS
`~/Library/Application Support/LightCraft/logs/`; Windows `%APPDATA%\LightCraft\logs\`), never in the library. A
launch from a desktop menu or the Dock has no terminal, so attach this file to a bug report. Each start moves the
launch from a desktop menu or the Dock has no terminal, so attach this file to a bug report (**Help ▸ Open Log
Folder** shows it in the file manager). Each start moves the
previous log to `lightcraft.1.log` and that one to `lightcraft.2.log`, so the log of a run that crashed survives the
next start; the file stops growing at 16 MiB, `--version` and `--help` write none, and runs with
`LIGHTCRAFT_NO_PREFS` log to standard error only. By default LightCraft's own crates log at `info` and everything else
+13 -12
View File
@@ -23,9 +23,9 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78).
| Dimension | Estimate | What's true today | Biggest gaps |
|---|---:|---|---|
| **Feature checklist** | 79.7% | P0 core and P1 nearly complete: import (Add / Copy / Move, templates, devices), library, grid/loupe/compare/survey, every Edit slider, curves, colour grading, masking tools, crop/Upright, heal/clone, presets/profiles, versions/history, sync, export, menus, shortcuts | P0: full-resolution zoom detail; P1: lens-profile database, content-aware fill (patch synthesis), video playback/trim |
| **RAW coverage** (formats people shoot) | ~55% | DNG (all kinds), CR2, CR3 lossless Bayer and version 0x100/0x200 C-RAW (M50/R100/R8 pixel-exact corpus), ARW, NEF (uncompressed + Huffman lossless/lossy), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans), uncompressed ORF, RW2 / RWL / Panasonic RAW (every raw format, checked on 178 files from 118 Panasonic and Leica bodies), PEF; every container's embedded preview (incl. CR3) | **CR3** (every Canon since ~2018), compressed ORF, Nikon lossy-after-split, Canon sRAW, HEIC/AVIF. NEFs labelled compressed but stored uncompressed (Z 6 packed 14-bit, D850 12-bit uncompressed). Per-model verification is thin (~55 corpus files vs >1,000 models) |
| **RAW coverage** (formats people shoot) | ~55% | DNG (all kinds), CR2, CR3 lossless Bayer and version 0x100/0x200 C-RAW (M50/R100/R8 pixel-exact corpus), ARW, NEF (uncompressed + Huffman lossless/lossy), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans), uncompressed ORF, uncompressed Samsung SRW (NX5 to NX1100, EX1, WB2000), RW2 / RWL / Panasonic RAW (every raw format, checked on 178 files from 118 Panasonic and Leica bodies), PEF; every container's embedded preview (incl. CR3) | **CR3** (every Canon since ~2018), compressed ORF, compressed Samsung SRW (NX1, NX30, NX300, NX500, NX2000, NX3000, NX3300, NX mini), Nikon lossy-after-split, Canon sRAW, HEIC/AVIF. NEFs labelled compressed but stored uncompressed (Z 6 packed 14-bit, D850 12-bit uncompressed). Per-model verification is thin (~55 corpus files vs >1,000 models) |
| **Colour & image quality** | ~55–65% | Pipeline is complete and fast; GPU path CPU-exact within 1/255 | **No measured camera calibration database**: ARW, NEF, RW2, RAF, CR3, CR2 and PEF have guarded camera-JPEG colour estimates, with bundled ILCE-7M4, X-H2S and X-T4 profiles (docs/camera-preview-colour.md); other non-DNG raws and rejected estimates use a neutral matrix. Colour fidelity remains incomplete. No lens-profile database. No measured fidelity against Lightroom (tone, highlights, texture/clarity, NR, sharpening are tuned by eye) |
| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics | Real segmentation masks (subject, sky, people, objects, landscape, depth), AI denoise, super resolution, lens blur, generative remove, faces/people, natural-language search. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) |
| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics, or the photo's own segmentation mattes when it ships them (DNG semantic masks, e.g. iPhone ProRAW's sky matte); non-destructive Bayer RAW denoise with opt-in weights on pure-Rust CPU/GPU; **faces (MVP, [docs/faces.md](docs/faces.md))**: a pure-Rust detector (a 232 KB opt-in download), opt-in recognition models (one-click download or add a file; they run on the shared pure-Rust CPU interpreter; pinned SFace/AuraFace embeddings agree with an independent reference runtime on 40 public crops; one-thread medians 34–36 / 233–252 ms (AuraFace remains expensive)), a background scan that finds and embeds faces, name suggestions, a person's page with look-alike faces to confirm, an Unnamed faces section (look-alikes together, select a group and name it at once) | Real segmentation masks (subject, sky, people, objects, landscape, depth), denoise model policy / linear RGB / quality verification, super resolution, lens blur, generative remove, natural-language search; for faces: clusters with separators between groups, small faces in large photos (the detector looks at 640 px), thresholds calibrated on live faces, writing face regions back to XMP. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) |
| **Workflow & library** | ~85% (single machine) | Robust catalog (journal + snapshots, background compaction, crash-tested), 85k-photo libraries stay responsive, Local browsing with automatic cleanup, XMP interop, keywords, smart albums, Move import | Opening an 85k library takes 1.7–4.7 s; no cloud sync (out of scope), no tablet companion (#74, roadmap), shared albums, publish services, tethering |
| **Classic modules** | ~30% | Geotagging from GPX track logs, soft proofing (partial), slideshow (basic) | **Map view, Book, Print, Slideshow module, Web, publish services**: ~40 tracker rows ⬜ |
| **HDR & video** | 0% | | HDR edit/display/export; video play/trim/edit/export |
@@ -40,15 +40,15 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78).
| Fujifilm raw shooter | ~65% | Uncompressed and lossless/lossy compressed RAF decoded; camera colour calibration remains missing. Verified on 13 bodies, including 14/16-bit GFX; see `docs/raf-compression.md` |
| Canon CR3 / Olympus compressed-raw shooter | ~45% | CR3 lossless Bayer and version 0x100/0x200 C-RAW develop from sensor data (M50, R100, R8 verified pixel for pixel); other CRX variants and compressed ORF open as embedded JPEG previews. Camera colour and broader model verification remain |
| Lightroom Classic power user | ~45% | Print, Book, Map, publish, tethering |
| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); no AI denoise |
| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); Bayer AI denoise needs separately installed weights; X-Trans/linear RGB, model policy and quality parity remain open |
## Where we're going
Priorities, in order. Each points at tracker rows in [`docs/parity.md`](docs/parity.md) → *Top gaps*.
1. **Camera colour calibration of our own** (LR-PROF-CAMERACOLOR, P0): fit each camera to its own embedded JPEG, use
matrices the files carry themselves, then chart shots. Sony ARW, Nikon NEF, Panasonic RW2, Fujifilm RAF, Canon CR3/CR2
and Pentax PEF have the file-local fit (matrix + tone curve from their own JPEG; CR2 and PEF since issue #310, not yet
matrices the files carry themselves, then chart shots. Sony ARW, Nikon NEF, Panasonic RW2, Fujifilm RAF, Canon CR3/CR2,
Pentax PEF and Samsung SRW have the file-local fit (matrix + tone curve from their own JPEG; CR2 and PEF since issue #310, not yet
checked on the corpus); then ORF, then validate fidelity.
2. **Raw formats, clean-room** (LR-IMP-FORMATS, P0): Remaining **unverified CR3 CRX** variants, compressed ORF, NEF
lossy-after-split, sRAW. Decided 2026-10-05: write our own decoders from prose descriptions (never decoder source,
@@ -59,9 +59,9 @@ Priorities, in order. Each points at tracker rows in [`docs/parity.md`](docs/par
(references stay in the local `plan/`), then tune against the numbers.
5. **Lens profiles of our own** (LR-EDIT-OPTICS-PROFILE, P1).
6. **AI model strategy** (maintainer decision): which permissively licensed models (or our own training) for
segmentation masks and denoise; then pure-Rust inference. Unblocks M12 and Enhance. Object / Describe masks now
segmentation masks and denoise. Bayer denoise inference is implemented in pure Rust with an opt-in GPL model; release policy and independently licensed weights remain open. Unblocks M12 and Enhance. Face detection and recognition run on our own checked Rust engine. Object / Describe masks now
run Meta's SAM 3 in pure Rust (optional, user-consented download under the non-OSI SAM License; CDN mirrors still
to be set up, docs/ai-masks.md); Subject / Sky / People and Enhance are still open.
to be set up, docs/ai-masks.md); Subject / Sky / People, Raw Details and Super Resolution are still open.
7. **Then:** HDR (Q), the Classic output modules (Print first, then Map view, Book, Slideshow), video (R), localisation
and accessibility.
@@ -86,7 +86,7 @@ yet guarantee original-pixel detail; visible-region rendering remains a gap (LR-
| M4 | Crop, geometry, optics | crop tool + overlays, straighten, Upright (auto/level/vertical/full/guided), manual transforms, CA, defringe, manual lens corrections | 6–10 | ✅ |
| M5 | Performance | source pyramids, wgpu compute pipeline (CPU oracle), draft/full renders, prefetch, budgets (16 ms slider updates on 24 MP) | 10–15 | 🚧 (stage cache, source pyramid, wgpu pipeline, prefetch, memory budget ✅; colour NR at half resolution, GPU histogram ⬜) |
| M6 | Masking | brush, linear/radial gradients, colour/luminance/depth range, add/subtract/intersect/invert, all local adjustments, masks panel | 8–12 | 🚧 (brush/linear/radial/colour/luminance range, add/subtract/intersect, masks panel ✅; depth range, AI masks ⬜) |
| M7 | Detail | sharpening + masking preview, luminance/colour NR, Denoise, Raw Details, Super Resolution | 6–10 | 🚧 (sharpening, luminance/colour NR ✅; AI Denoise, Raw Details, Super Resolution ⬜) |
| M7 | Detail | sharpening + masking preview, luminance/colour NR, Denoise, Raw Details, Super Resolution | 6–10 | 🚧 (sharpening, luminance/colour NR ✅; AI Denoise 🟡 (Bayer, optional weights; policy/fidelity open); Raw Details, Super Resolution ⬜) |
| M8 | Heal / Remove | content-aware remove (PatchMatch), heal, clone, brush spots, visualize spots, red/pet eye | 6–10 | 🚧 (heal, clone, auto source, visualize spots, red/pet eye ✅; PatchMatch remove ⬜) |
| M9 | Presets, profiles, versions, sync | preset browser + amount, create/import presets, profile browser, versions, history, copy/paste/sync settings | 5–8 | ✅ |
| M10 | Export & share | export dialog (JPEG/PNG/TIFF/DNG/AVIF/JXL/original), sizing, sharpening, metadata, watermark, naming, batch jobs, XMP sidecars, HDR export | 6–10 | 🚧 (all formats incl. DNG/original, sizing, presets, background jobs ✅; JXL encode, HDR export ⬜) |
@@ -124,7 +124,7 @@ trip, slideshow, auto import…) in ≈ 12 h on 2026-10-02; four to six parallel
| Work package | Tracker rows | Agent-hours | Risk |
|---|---|---:|---|
| Remaining P0/P1 UI and library features (folder rename/move, keyword painter, people view…) | ≈ 8 | 5–10 | low |
| Raw codecs: CR3 (CRX), compressed ORF, NEF lossy-after-split, HEIC/AVIF decode, JPEG XL DNG | LR-IMP-FORMATS | 40–80 | **high** — clean-room black-box analysis, no permissive specs |
| Raw codecs: CR3 (CRX), compressed ORF, NEF lossy-after-split, HEIC/AVIF decode | LR-IMP-FORMATS | 40–80 | **high** — clean-room black-box analysis, no permissive specs |
| Lens-profile database of our own (calibration targets, fitting, data) | LR-EDIT-OPTICS-PROFILE | 15–30 | data collection |
| Video: playback, trim, edits, export (pure-Rust decode, ideally shared with FilmCraft) | R. Video | 20–40 | medium |
| AI: subject / sky / background / people / object masks, object-aware remove, AI denoise, super resolution, lens blur, people & faces, natural-language search, culling | ≈ 30 | 80–150 | **high** — permissively licensed weights, pure-Rust inference, maybe training |
@@ -174,7 +174,8 @@ private 3:2 & 4:3 examples verified through native rendering & full-resolution 1
CR3 lossless Bayer and version 0x100/0x200 C-RAW sensor decoding is verified pixel-exactly on Canon M50, R100 and R8; any other CR3, or one that fails to decode, opens from its embedded JPEG as before. See [`docs/cr3.md`](docs/cr3.md) for coding limits, reference provenance and colour limitations.
Decoded (CC0 corpus from raw.pixls.us, `cargo xtask corpus --download`, `crates/raw/tests/corpus.rs`): DNG (uncompressed,
LJ92, lossy JPEG / Smart Previews, Deflate, float, linear), CR2, ARW (uncompressed, ARW2, LJ92; as-shot white balance and black level of pre-2017 bodies from the enciphered
LJ92, lossy JPEG / Smart Previews, Deflate, float, linear, DNG 1.7 JPEG XL tiles: lossless ones bit-exact on synthetic files,
lossy ones unclipped in the declared linear primaries, checked on one real file), CR2, ARW (uncompressed, ARW2, LJ92; as-shot white balance and black level of pre-2017 bodies from the enciphered
maker-note `Tag2010` and the encrypted `SR2SubIFD`, both recovered by black-box analysis, `crates/raw/src/vendor/arw.rs`), NEF/NRW uncompressed and Huffman-compressed (lossless, lossy type 1/2, 12/14-bit), RAF uncompressed and lossless/lossy compressed (Bayer and
X-Trans, 14/16-bit real-file verification), RW2 / Leica RWL / Panasonic RAW in every raw format (compressed formats 4 and 6, the prefix-coded strips of format 8,
packed formats 2, 5 and 7, the 16-bit words of the 2005–2007 bodies; all recovered by black-box analysis of 178 CC0 files from
@@ -185,9 +186,9 @@ Not decoded yet — preview only (no permissively licensed description; black-bo
files): rows above the split decode with the regular lossy table; from the split row on a different code is used
that our black-box analysis has not recovered yet (none of the four regular tables fits, also not with byte
alignment or reset predictors at the split row). The other Nikon Huffman variants are decoded (`crates/raw/src/vendor/nefc.rs` documents the analysis).
- **Olympus compressed ORF**, **Canon CR3 unverified CRX variants** (M11.1), **Canon sRAW/mRAW**, lossy DNG.
- **Olympus compressed ORF**, **Canon CR3 unverified CRX variants** (M11.1), **Canon sRAW/mRAW**.
**Camera colour matrices:** ARW, NEF, RW2, RAF, CR3, CR2 and PEF files can use guarded, separate chromaticity and tone estimates from their own embedded JPEG (see `docs/camera-preview-colour.md`); this is a per-file camera-look estimate with relative WB, not measured calibration or absolute-Kelvin WB. Other non-DNG raws and rejected fits use the documented neutral fallback (camera RGB ≈ linear sRGB, flagged
**Camera colour matrices:** ARW, NEF, RW2, RAF, CR3, CR2, PEF and Samsung SRW files can use guarded, separate chromaticity and tone estimates from their own embedded JPEG (see `docs/camera-preview-colour.md`); this is a per-file camera-look estimate with relative WB, not measured calibration or absolute-Kelvin WB. Other non-DNG raws and rejected fits use the documented neutral fallback (camera RGB ≈ linear sRGB, flagged
`matrix_is_fallback`) with the file's as-shot white-balance multipliers. Clean sources to evaluate next: manufacturer
matrices stored in the files themselves (Olympus ImageProcessing `ColorMatrix`, Pentax equivalents; Panasonic RW2 files carry none) and our
own chart-based calibration (M11.4). Adobe matrices are never used.
+7
View File
@@ -25,7 +25,14 @@ serde_json = { workspace = true }
dhat = { workspace = true, optional = true }
[features]
default = ["denoise"]
denoise = ["lightcraft-engine/denoise"]
rawnind-model = ["denoise", "lightcraft-engine/rawnind-model"]
# Heap profiling: `cargo run --release -p lightcraft-cli --features dhat-heap -- snapshot …` counts
# every allocation (live/peak bytes in `library.memory` / `ui.inspect`) and writes dhat-heap.json
# (open it in DHAT's viewer: allocation sites by bytes at the peak). Slows allocation-heavy work.
dhat-heap = ["dep:dhat"]
# Open HEIC/HEIF photos (the optional lightcraft-heif crate). Not default so distributors choose
# it (HEVC patents); build with `--features heif` to include it.
heif = ["lightcraft-codecs/heif"]
+14 -10
View File
@@ -342,7 +342,7 @@ fn mcp(args: &[String]) -> Result<(), String> {
eprintln!("lightcraft-cli mcp: serving MCP on stdio ({})", backend.describe());
let mut server = Server::new(backend).with_command_tools(!compact);
let stdin = std::io::stdin();
server.serve(BufReader::new(stdin.lock()), std::io::stdout().lock()).map_err(|e| e.to_string())
server.serve(BufReader::new(stdin), std::io::stdout()).map_err(|e| e.to_string())
}
fn merge(args: &[String]) -> Result<(), String> {
@@ -391,7 +391,7 @@ fn merge(args: &[String]) -> Result<(), String> {
}
i += 1;
}
let mut s = Session::new().with_fs();
let mut s = Session::new().with_fs().with_default_denoise_models();
let paths = expand_paths(&files);
let r = s.execute("library.import", &json!({"paths": paths})).map_err(|e| e.to_string())?;
let mut ids: Vec<u64> = r["imported"].as_array().map(|a| a.iter().filter_map(Value::as_u64).collect()).unwrap_or_default();
@@ -652,7 +652,7 @@ fn render(args: &[String]) -> Result<(), String> {
}
let input = input.ok_or("render: missing input file")?;
let output = output.ok_or("render: missing -o OUTPUT")?;
let mut s = Session::new().with_fs();
let mut s = Session::new().with_fs().with_default_denoise_models();
let abs = expand_paths(std::slice::from_ref(&input));
let r = s.execute("library.import", &json!({"paths": abs})).map_err(|e| e.to_string())?;
let id = r["imported"][0].as_u64().ok_or_else(|| format!("{input}: not a readable photo"))?;
@@ -728,19 +728,19 @@ fn snapshot(args: &[String]) -> Result<(), String> {
if script.is_none() && output.is_none() {
return Err("snapshot: give -o OUT.png and/or --script FILE".into());
}
if !(scale > 0.0 && size[0] >= 1.0 && size[1] >= 1.0) {
return Err("snapshot: bad --size/--scale".into());
}
lightcraft_ui_egui::headless::viewport_pixels(size, scale).map_err(|e| format!("snapshot: bad --size/--scale: {e}"))?;
let t0 = Instant::now();
let mut session = match &library {
Some(dir) => {
let mut s = Session::new().with_fs();
let mut s = Session::new().with_fs().with_default_denoise_models().with_default_face_models();
s.open_library(dir, false).map_err(|e| library_error(dir, e))?;
s
}
None if files.is_empty() => Session::with_demo().with_fs(),
None => Session::new().with_fs(),
None if files.is_empty() => Session::with_demo().with_fs().with_default_denoise_models().with_default_face_models(),
None => Session::new().with_fs().with_default_denoise_models().with_default_face_models(),
};
// Apply compute policy before imports or the first scripted query can discover an adapter.
session.execute("app.gpu", &json!({"enabled": false})).map_err(|e| e.to_string())?;
if !files.is_empty() {
let ti = Instant::now();
let r = session.execute("library.import", &json!({"paths": expand_paths(&files)})).map_err(|e| e.to_string())?;
@@ -755,7 +755,11 @@ fn snapshot(args: &[String]) -> Result<(), String> {
})),
..Default::default()
};
let app = lightcraft_ui_egui::LightcraftApp::new(session, services);
let mut app = lightcraft_ui_egui::LightcraftApp::new(session, services);
// Snapshot sessions promise no GPU: disable photo compute as well as the compositor
// before the first frame can start background adapter discovery.
// This UI state is session-local: the CLI neither loads nor saves desktop ui.json.
app.ui.settings.gpu = false;
let mut h = Headless::new(app, size, scale);
let timeout = Duration::from_secs(60);
let mut shots = 0usize;
+128
View File
@@ -233,6 +233,58 @@ fn snapshot_script_failure_exits_non_zero() {
assert_eq!((d.width, d.height), (320, 200));
}
#[test]
fn snapshot_ui_zoom_keeps_requested_pixel_dimensions() {
let script = tmp("snap-ui-zoom.jsonl");
std::fs::write(
&script,
format!("{}\n{}\n", json!({"method": "ui.key", "params": {"key": "Plus", "cmd": true}}), json!({"method": "ui.inspect"})),
)
.unwrap();
for (size, scale, expected) in [("400x240", "1", (400, 240)), ("345x200", "0.9", (311, 180))] {
let out = tmp(&format!("snap-ui-zoom-{scale}.png"));
let o = Command::new(BIN)
.args(["snapshot", "--script", script.to_str().unwrap(), "-o", out.to_str().unwrap(), "--size", size, "--scale", scale])
.output()
.unwrap();
assert!(o.status.success(), "{}", String::from_utf8_lossy(&o.stderr));
let replies: Vec<Value> = String::from_utf8_lossy(&o.stdout).lines().map(|line| serde_json::from_str(line).unwrap()).collect();
assert!(replies.iter().all(|reply| reply["ok"] == true), "{replies:?}");
assert!(replies[1]["result"]["pixelsPerPoint"].as_f64().unwrap() > scale.parse::<f64>().unwrap());
let decoded = lightcraft_codecs::decode(&std::fs::read(out).unwrap(), Default::default()).unwrap();
assert_eq!((decoded.width, decoded.height), expected);
}
}
/// Headless snapshots default to CPU photo rendering as well as CPU UI rasterization.
#[test]
fn snapshot_defaults_to_cpu_without_gpu_environment_overrides() {
let script = tmp("default-cpu.jsonl");
std::fs::write(
&script,
format!("{}\n{}\n", json!({"method": "engine.execute", "params": {"command": "app.gpu"}}), json!({"method": "ui.screenshot"})),
)
.unwrap();
let image = tmp("default-cpu.png");
let output = Command::new(BIN)
.env_remove("LIGHTCRAFT_GPU")
.env_remove("LIGHTCRAFT_GPU_BACKEND")
.env_remove("WGPU_BACKEND")
.args(["snapshot", "--demo", "--script", script.to_str().unwrap(), "-o", image.to_str().unwrap(), "--size", "480x320"])
.output()
.unwrap();
assert!(output.status.success(), "status {}: {}", output.status, String::from_utf8_lossy(&output.stderr));
let replies: Vec<Value> = String::from_utf8_lossy(&output.stdout).lines().map(|line| serde_json::from_str(line).unwrap()).collect();
assert!(replies.iter().all(|reply| reply["ok"] == true), "{replies:?}");
let gpu = &replies[0]["result"];
assert_eq!(gpu["enabled"], false, "{gpu}");
assert_eq!(gpu["available"], false, "{gpu}");
assert_eq!(gpu["adapter"], Value::Null, "no device is created: {gpu}");
assert!(gpu["reason"].as_str().is_some_and(|reason| reason.contains("preference")), "{gpu}");
let decoded = lightcraft_codecs::decode(&std::fs::read(image).unwrap(), Default::default()).unwrap();
assert_eq!((decoded.width, decoded.height), (480, 320));
}
/// Issue #136: with the GPU switched off from the environment the UI starts and renders on the CPU,
/// without creating a GPU device (no driver is loaded), and says why.
#[test]
@@ -412,3 +464,79 @@ fn warnings_are_logged_on_stderr() {
let off = run("off");
assert!(!off.contains("names no known backend"), "{off}");
}
#[test]
fn snapshot_rejects_invalid_dimensions_before_creating_an_image() {
for (index, (size, scale)) in
[("NaNx100", "1"), ("100x100", "inf"), ("9000x9000", "1"), ("9000x100", "2"), ("100x100", "0.001")].into_iter().enumerate()
{
let image = tmp(&format!("invalid-viewport-{index}.png"));
let output = Command::new(BIN).args(["snapshot", "--size", size, "--scale", scale, "-o"]).arg(&image).output().unwrap();
assert!(!output.status.success());
assert!(String::from_utf8_lossy(&output.stderr).contains("snapshot: bad --size/--scale"), "{}", String::from_utf8_lossy(&output.stderr));
assert!(!image.exists());
}
}
#[test]
fn snapshot_preserves_fractional_scale_rounding() {
let image = tmp("fractional-scale-rounding.png");
let output =
Command::new(BIN).env("LIGHTCRAFT_GPU", "0").args(["snapshot", "--size", "345x200", "--scale", "0.9", "-o"]).arg(&image).output().unwrap();
assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr));
let decoded = lightcraft_codecs::decode(&std::fs::read(image).unwrap(), Default::default()).unwrap();
assert_eq!((decoded.width, decoded.height), (311, 180));
}
/// Snapshot's CPU setting must never replace the desktop preference, including with --library.
#[cfg(target_os = "linux")]
#[test]
fn snapshot_leaves_desktop_gpu_preferences_untouched() {
let dir = tmp("snapshot-desktop-preferences");
let library = dir.join("library");
let mut session = lightcraft_engine::Session::new().with_fs();
session.open_library(&library, true).unwrap();
drop(session);
let config = dir.join("config");
let app_config = config.join("lightcraft");
std::fs::create_dir_all(&app_config).unwrap();
let ui_path = app_config.join("ui.json");
let mut ui = lightcraft_ui_egui::UiState::default();
ui.settings.gpu = true;
let saved = serde_json::to_vec_pretty(&ui).unwrap();
std::fs::write(&ui_path, &saved).unwrap();
let script = dir.join("preferences.jsonl");
std::fs::write(
&script,
format!("{}\n{}\n", json!({"method": "engine.execute", "params": {"command": "app.gpu"}}), json!({"method": "ui.inspect"})),
)
.unwrap();
for gpu_environment in [None, Some("1")] {
let image = dir.join(format!("{}.png", gpu_environment.unwrap_or("default")));
let mut cmd = Command::new(BIN);
cmd.env("XDG_CONFIG_HOME", &config).env_remove("LIGHTCRAFT_GPU").env_remove("LIGHTCRAFT_GPU_BACKEND").env_remove("WGPU_BACKEND");
if let Some(value) = gpu_environment {
cmd.env("LIGHTCRAFT_GPU", value);
}
let output = cmd
.args(["snapshot", "--library"])
.arg(&library)
.args(["--script"])
.arg(&script)
.args(["-o"])
.arg(&image)
.args(["--size", "480x320"])
.output()
.unwrap();
assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr));
let replies: Vec<Value> = String::from_utf8_lossy(&output.stdout).lines().map(|line| serde_json::from_str(line).unwrap()).collect();
assert!(replies.iter().all(|reply| reply["ok"] == true), "{replies:?}");
assert_eq!(replies[0]["result"]["enabled"], false);
assert_eq!(replies[0]["result"]["available"], false);
assert_eq!(replies[0]["result"]["adapter"], Value::Null);
assert_eq!(replies[1]["result"]["ui"]["settings"]["gpu"], false);
assert_eq!(std::fs::read(&ui_path).unwrap(), saved, "snapshot changed the desktop preferences");
assert!(!app_config.join("ui.json.tmp").exists());
assert!(image.exists());
}
}
+6
View File
@@ -71,3 +71,9 @@ web-sys = { workspace = true, features = [
"WorkerType",
"WritableStream",
] }
[features]
# Open HEIC/HEIF photos (the optional lightcraft-heif crate). Not default so distributors choose
# it (HEVC patents); build with `--features heif` to include it. On wasm32 panics abort, so
# heic-rs's `catch_unwind` guard does not apply there.
heif = ["lightcraft-codecs/heif"]
+10 -2
View File
@@ -24,13 +24,21 @@
<script type="module">
import init, { start, command } from "./lightcraft_web.js";
(async () => {
// The Chinese font is a separate file: embedding it would exceed the host's WASM size limit.
// Font-free development builds can still run, with the same missing-glyph warning as before.
const chineseFont = fetch("./lightcraft_zh_hans.otf")
.then(async response => response.ok ? new Uint8Array(await response.arrayBuffer()) : new Uint8Array())
.catch(() => new Uint8Array());
// Compile once; the render workers (worker.js) instantiate the same compiled module.
const module = await WebAssembly.compileStreaming(fetch("./lightcraft_web_bg.wasm"));
const [module, font] = await Promise.all([
WebAssembly.compileStreaming(fetch("./lightcraft_web_bg.wasm")),
chineseFont,
]);
globalThis.lightcraftModule = module;
await init({ module_or_path: module });
// automation: `await lightcraft.command("library.info", "{}")` → JSON text
globalThis.lightcraft = { command };
start();
start(font);
})().catch((e) => {
console.error(e);
const el = document.getElementById("lightcraft_loading");
+34 -2
View File
@@ -11,8 +11,9 @@
//!
//! Restoring never deletes anything: the library files go to a new library folder in browser
//! storage, originals that aren't stored yet are added (they're named by content, so existing
//! ones are identical), and only then does a small pointer file ([`ACTIVE_LIBRARY`]) switch to
//! the restored library. The library that was there before stays in storage.
//! ones are identical; a restored one must match its content hash, see [`verify_original`]),
//! and only then does a small pointer file ([`ACTIVE_LIBRARY`]) switch to the restored library.
//! The library that was there before stays in storage.
//!
//! No zip64: a backup holds at most [`MAX_BYTES`] and 65 535 entries.
@@ -241,6 +242,16 @@ pub fn verify(entry: &ZipEntry, data: &[u8]) -> Result<(), String> {
Ok(())
}
/// Check that an original's bytes are the photo its storage key names: originals are stored
/// under their content hash, and a restore that kept bytes under another photo's hash would
/// give that photo's catalog record a different image (the zip checksum can't tell).
pub fn verify_original(entry: &ZipEntry, hash: &str, data: &[u8]) -> Result<(), String> {
if crate::store::content_hash(data) != hash {
return Err(format!("{}: the file doesn't match its content hash (damaged or altered backup)", entry.name));
}
Ok(())
}
/// A name for the restored library's folder.
pub fn restored_dir_name(now_ms: f64) -> String {
format!("library-restored-{}", now_ms.max(0.0) as u64)
@@ -283,6 +294,27 @@ mod tests {
assert!(find_central(b"not a zip at all, just some text").is_err());
}
#[test]
fn originals_must_match_their_content_hash() {
let red: &[u8] = b"red corner photo";
let blue: &[u8] = b"blue corner photo";
let hash = crate::store::content_hash(red);
let name = original_entry(&hash, "IMG 1.png");
let key = restore_key(&name, "d").unwrap();
let key_hash = key.strip_prefix("originals/").unwrap();
assert_eq!(key_hash, hash);
// the entry's own bytes pass; other bytes under the same name (valid zip CRC) don't
let zip = build(&[(name.as_str(), blue)]);
let tail = &zip[zip.len() - tail_len(zip.len() as u64) as usize..];
let (off, size) = find_central(tail).unwrap();
let entries = parse_central(&zip[off as usize..(off + size) as usize]).unwrap();
let entry = &entries[0];
verify(entry, blue).unwrap();
let err = verify_original(entry, key_hash, blue).unwrap_err();
assert!(err.contains(&name), "{err}");
verify_original(entry, key_hash, red).unwrap();
}
#[test]
fn restore_keys_are_safe() {
assert_eq!(restore_key("library/catalog.snap", "library-restored-1").as_deref(), Some("library-restored-1/catalog.snap"));
+25 -6
View File
@@ -14,7 +14,7 @@ use wasm_bindgen_futures::JsFuture;
use crate::backend::Backend;
use crate::backup::{
ACTIVE_LIBRARY, README, ZipWriter, data_offset, find_central, original_entry, parse_central, restore_key, restored_dir_name, tail_len,
valid_hash, verify,
valid_hash, verify, verify_original,
};
use crate::files::{Files, LIBRARY_FILES};
use crate::store::storage_key;
@@ -93,25 +93,41 @@ pub fn install_panic_hook() {
const TAB_LOCK: &str = "lightcraft-library";
/// Hold the library lock for as long as this page lives (Web Locks API). `Some(true)`: ours;
/// `Some(false)`: another tab of this site has it; `None`: the browser can't tell (no Web Locks).
/// `Some(false)`: another tab of this site has it; `None`: the browser can't tell (no Web Locks, or
/// it refused the request).
pub async fn acquire_tab_lock() -> Option<bool> {
let nav = Reflect::get(&js_sys::global(), &"navigator".into()).ok()?;
let locks = Reflect::get(&nav, &"locks".into()).ok().filter(|l| !l.is_undefined() && !l.is_null())?;
let request: js_sys::Function = Reflect::get(&locks, &"request".into()).ok()?.dyn_into().ok()?;
let opts = js_sys::Object::new();
Reflect::set(&opts, &"ifAvailable".into(), &true.into()).ok()?;
let granted = js_sys::Promise::new(&mut |resolve, _reject| {
let granted = js_sys::Promise::new(&mut |resolve, reject| {
let cb = Closure::once_into_js(move |lock: JsValue| -> JsValue {
let ours = !lock.is_null() && !lock.is_undefined();
let _ = resolve.call1(&JsValue::NULL, &ours.into());
// keep the lock until the page goes away: a promise that never settles
if ours { js_sys::Promise::new(&mut |_, _| {}).into() } else { JsValue::NULL }
});
if let Err(e) = request.call3(&locks, &TAB_LOCK.into(), &opts, &cb) {
log::warn!("Web Locks request failed: {e:?}");
// a refused request never calls `cb`: settle with its error instead of waiting forever
match request.call3(&locks, &TAB_LOCK.into(), &opts, &cb) {
Ok(pending) => {
let catch = Reflect::get(&pending, &"catch".into()).ok().and_then(|c| c.dyn_into::<js_sys::Function>().ok());
if let Some(catch) = catch {
let _ = catch.call1(&pending, &reject);
}
}
Err(e) => {
let _ = reject.call1(&JsValue::NULL, &e);
}
}
});
JsFuture::from(granted).await.ok().map(|v| v.is_truthy())
match JsFuture::from(granted).await {
Ok(v) => Some(v.is_truthy()),
Err(e) => {
log::warn!("Web Locks request failed, starting without the one-tab guard: {e:?}");
None
}
}
}
/// Offer a blob as a download named `name`.
@@ -208,6 +224,9 @@ pub async fn restore(file: web_sys::File, backend: Backend) -> Result<String, St
let start = data_offset(e, &local)?;
let data = read_range(&file, start, start + e.size).await?;
verify(e, &data)?;
if let Some(hash) = key.strip_prefix("originals/") {
verify_original(e, hash, &data)?;
}
backend.write(&key, &data).await.map_err(|err| format!("storing {}: {err}", e.name))?;
written += 1;
}
+13 -5
View File
@@ -205,10 +205,13 @@ fn services(originals: Originals, backend: Option<Backend>, files: Files, frozen
open_picker(originals.clone(), backend.clone(), ctx.clone());
Vec::new() // files arrive asynchronously and are imported on a later frame
})),
pick_denoise_model: None,
// Preset files: browser pickers are asynchronous; not wired on the web yet.
pick_preset_files: None,
pick_tracklog: None,
pick_lightroom_catalog: None,
// Face models need a folder to live in; the web has none.
pick_model_file: None,
save_preset_file: None,
pick_curve_preset_files: None,
save_curve_preset_file: None,
@@ -219,6 +222,7 @@ fn services(originals: Originals, backend: Option<Backend>, files: Files, frozen
lightcraft_codecs::encode_png(&lightcraft_codecs::EncodeImage::rgba8(img), &lightcraft_codecs::EncodeMeta::default()).unwrap_or_default()
})),
reveal: None,
log_file: None,
open_with: None,
open_url: Some(Box::new(|url: &str| {
let w = web_sys::window().ok_or("no window")?;
@@ -374,7 +378,7 @@ struct WebApp {
}
impl WebApp {
fn new(cc: &eframe::CreationContext<'_>, boot: Boot) -> Self {
fn new(cc: &eframe::CreationContext<'_>, boot: Boot, chinese_font: Vec<u8>) -> Self {
let Boot { opts, backend, files, index, lib_dir } = boot;
let originals = Originals::default();
let t = perf_now();
@@ -419,7 +423,8 @@ impl WebApp {
// previews are large (≤ 2560 px, f32): keep few in a 32-bit address space
session.media.preview_capacity = 3;
let frozen = Rc::new(Cell::new(false));
let mut app = LightcraftApp::new(session, services(originals.clone(), backend.clone(), files.clone(), frozen.clone(), cc.egui_ctx.clone()));
let mut app = LightcraftApp::new(session, services(originals.clone(), backend.clone(), files.clone(), frozen.clone(), cc.egui_ctx.clone()))
.with_chinese_font(chinese_font);
let ui_written = files.get("ui.json").unwrap_or_default();
if let Ok(ui) = serde_json::from_slice::<UiState>(&ui_written) {
app.ui = ui;
@@ -652,12 +657,15 @@ impl eframe::App for WebApp {
}
}
/// Start the app (called by `index.html` after instantiating the module).
/// Start the app with the separately fetched Simplified Chinese font (empty in font-free builds).
#[wasm_bindgen]
pub fn start() {
pub fn start(chinese_font: Vec<u8>) {
eframe::WebLogger::init(log::LevelFilter::Info).ok();
safety::install_panic_hook();
log::info!("lightcraft: wasm instantiated at {:.0} ms", perf_now());
if chinese_font.is_empty() {
log::warn!("lightcraft: no Simplified Chinese font supplied; Chinese-only glyphs will show as boxes");
}
let opts = Options::from_url();
wasm_bindgen_futures::spawn_local(async move {
// one tab per library: two would each keep their own copy and overwrite each other's saves
@@ -678,7 +686,7 @@ pub fn start() {
};
let boot = boot(opts).await;
let runner = eframe::WebRunner::new();
let r = runner.start(canvas, eframe::WebOptions::default(), Box::new(move |cc| Ok(Box::new(WebApp::new(cc, boot))))).await;
let r = runner.start(canvas, eframe::WebOptions::default(), Box::new(move |cc| Ok(Box::new(WebApp::new(cc, boot, chinese_font))))).await;
match r {
Ok(()) => {
if let Some(el) = window().and_then(|w| w.document()).and_then(|d| d.get_element_by_id("lightcraft_loading")) {
+5 -1
View File
@@ -368,7 +368,11 @@ mod tests {
let a = core.render(&wire, None).unwrap();
let b = job.run().rendered.unwrap();
assert_eq!((a.image.width, a.image.height), (160, 120));
assert_eq!(a.image.data, b.image.data);
assert_eq!((b.image.width, b.image.height), (160, 120));
// the worker renders on the CPU; the engine job uses a GPU when the machine has one, which
// stays within docs/gpu-pipeline.md's bound (≤ 3 LSB per channel) rather than bit-exact
let max = a.image.data.iter().zip(&b.image.data).flat_map(|(p, q)| (0..3).map(move |c| p[c].abs_diff(q[c]))).max();
assert!(max.is_some_and(|m| m <= 3), "max channel difference {max:?}");
}
// a job from before windows existed has none
+8
View File
@@ -9,6 +9,14 @@ default-run = "lightcraft"
[lints]
workspace = true
[features]
default = ["denoise"]
denoise = ["lightcraft-engine/denoise"]
rawnind-model = ["denoise", "lightcraft-engine/rawnind-model"]
# Open HEIC/HEIF photos (the optional lightcraft-heif crate). Not default so distributors choose
# it (HEVC patents); build with `--features heif` to include it.
heif = ["lightcraft-codecs/heif"]
[dependencies]
lightcraft-engine = { workspace = true, features = ["sam"] }
lightcraft-sysmem = { workspace = true }
+107 -13
View File
@@ -52,8 +52,15 @@ struct App(LightcraftApp, PrefsWriter, #[cfg(target_os = "macos")] Option<native
impl eframe::App for App {
fn logic(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) {
#[cfg(target_os = "macos")]
if let Some(m) = self.2.as_mut() {
m.update(&mut self.0, ctx);
if let Some(m) = self.2.as_mut()
&& m.update(&mut self.0, ctx)
&& lightcraft_ui_egui::panels::notices::may_close(&mut self.0)
{
// Quit from the menu bar: save exactly what closing the window saves (settings and
// library, after the unsaved-changes check; otherwise its prompt shows), then end the
// process instead of closing the viewport while AppKit is terminating (PR #444).
eframe::App::on_exit(self);
std::process::exit(0);
}
self.0.logic(ctx);
self.1.tick(&mut self.0, ctx);
@@ -76,11 +83,13 @@ impl eframe::App for App {
/// The window's renderer (egui-wgpu): eframe's defaults, with LightCraft's backend choice — DX12
/// alone on Windows unless `LIGHTCRAFT_GPU_BACKEND` / `WGPU_BACKEND` say otherwise (issue #136:
/// with Vulkan in the set, wgpu loads the Vulkan driver even when it then picks DX12).
/// with Vulkan in the set, wgpu loads the Vulkan driver even when it then picks DX12) — and DX12
/// shaders compiled with FXC, never a stray `dxcompiler.dll` (issue #471).
fn window_wgpu_options() -> eframe::egui_wgpu::WgpuConfiguration {
let mut c = eframe::egui_wgpu::WgpuConfiguration::default();
if let eframe::egui_wgpu::WgpuSetup::CreateNew(n) = &mut c.wgpu_setup {
n.instance_descriptor.backends = lightcraft_engine::gpu::backend::window_backends();
n.instance_descriptor.backend_options = lightcraft_engine::gpu::backend::backend_options();
}
c
}
@@ -125,7 +134,7 @@ fn gpu_crash_notice(what: &str) -> String {
}
fn config_dir() -> Option<std::path::PathBuf> {
lightcraft_engine::camera_profiles::config_dir()
lightcraft_engine::config::config_dir()
}
/// `<config>/ui.json`, the saved UI state and app settings — `None` when nothing is read or
@@ -281,8 +290,42 @@ fn windows_open_url_command(url: &str) -> std::process::Command {
c
}
/// The platform services; `ctx` repaints when a file dialog closes.
fn services(ctx: egui::Context) -> Services {
/// Explorer's `/select,<path>` switch (issue #272). Explorer parses its own command line and only
/// selects the file when the path after the comma is quoted (`/select,"C:\My Photos\a.jpg"`),
/// spelled with backslashes and free of the verbatim prefix (`\\?\`, `\\?\UNC\`) that
/// `canonicalize` adds; given anything else it opens Documents. Windows file names can't contain
/// `"`; a path that does anyway (a hostile catalog) has them dropped, so the raw command line
/// always holds exactly this one argument.
fn explorer_select_arg(path: &str) -> String {
let path = path.replace('/', "\\").replace('"', "");
let path = match path.strip_prefix(r"\\?\UNC\") {
Some(unc) => format!(r"\\{unc}"),
None => path.strip_prefix(r"\\?\").unwrap_or(path.as_str()).to_string(),
};
format!("/select,\"{path}\"")
}
/// `explorer /select,"<path>"`. Rust's own argument quoting wraps the whole switch in quotes
/// (`"/select,C:\My Photos\a.jpg"`) whenever the path has a space, which Explorer doesn't
/// recognise, so on Windows the switch goes on the command line exactly as built.
fn explorer_select_command(path: &str) -> std::process::Command {
let mut c = std::process::Command::new("explorer");
let arg = explorer_select_arg(path);
#[cfg(windows)]
{
use std::os::windows::process::CommandExt;
c.raw_arg(arg);
}
#[cfg(not(windows))]
{
c.arg(arg);
}
c
}
/// The platform services; `ctx` repaints when a file dialog closes, and `log_file` is
/// for Help ▸ Open Log Folder.
fn services(ctx: egui::Context, log_file: Option<&std::path::Path>) -> Services {
Services {
// Commands' file dialogs run on their own thread (#191): a dialog on the UI thread
// stopped the window answering the compositor, which then reported the app as hung.
@@ -317,6 +360,7 @@ fn services(ctx: egui::Context) -> Services {
.map_err(|e| format!("could not start the file dialog: {e}"))?;
Ok(rx)
})),
log_file: log_file.map(|p| p.display().to_string()),
pick_lightroom_catalog: Some(Box::new(|| {
rfd::FileDialog::new()
.set_title(lightcraft_ui_egui::i18n::tr("Import Lightroom Catalog"))
@@ -366,8 +410,7 @@ fn services(ctx: egui::Context) -> Services {
let status = if cfg!(target_os = "macos") {
std::process::Command::new("open").args(["-R", path]).status()
} else if cfg!(target_os = "windows") {
let win_path = path.replace('/', "\\");
std::process::Command::new("explorer").arg(format!("/select,{win_path}")).status()
explorer_select_command(path).status()
} else {
let dir = std::path::Path::new(path).parent().map(|d| d.to_string_lossy().to_string()).unwrap_or_else(|| ".".into());
std::process::Command::new("xdg-open").arg(dir).status()
@@ -385,6 +428,13 @@ fn services(ctx: egui::Context) -> Services {
.map(|p| p.to_string_lossy().to_string())
.collect()
})),
pick_denoise_model: Some(Box::new(|| {
rfd::FileDialog::new()
.add_filter_nocase(lightcraft_ui_egui::i18n::tr("ONNX model"), &["onnx"])
.pick_file()
.map(|p| vec![p.to_string_lossy().into_owned()])
.unwrap_or_default()
})),
pick_preset_files: Some(Box::new(|| {
rfd::FileDialog::new()
.set_title(lightcraft_ui_egui::i18n::tr("Import Presets"))
@@ -398,6 +448,14 @@ fn services(ctx: egui::Context) -> Services {
.map(|p| p.to_string_lossy().to_string())
.collect()
})),
pick_model_file: Some(Box::new(|| {
rfd::FileDialog::new()
.set_title(lightcraft_ui_egui::i18n::tr("Add a Face Recognition Model"))
.add_filter_nocase(lightcraft_ui_egui::i18n::tr("Face models (ONNX)"), &["onnx"])
.pick_file()
.map(|p| vec![p.to_string_lossy().to_string()])
.unwrap_or_default()
})),
pick_tracklog: Some(Box::new(|| {
rfd::FileDialog::new()
.set_title(lightcraft_ui_egui::i18n::tr("Auto-Tag from Tracklog"))
@@ -444,22 +502,29 @@ fn services(ctx: egui::Context) -> Services {
}
}
/// The library session with the folder face models are kept in (`<config>/models`).
fn open_session(in_memory: bool, dir: Option<std::path::PathBuf>, seed_demo: bool) -> (Session, Option<LibraryProblem>) {
let (mut s, problem) = open_library_session(in_memory, dir, seed_demo);
s.face_models_dir = lightcraft_engine::config::default_face_models_dir();
(s, problem)
}
/// The persistent library session, or with `--memory` an in-memory one (demo photos).
///
/// If the library can't be opened the session is empty and in memory — never seeded with the
/// demo photos, never written anywhere — and the problem is returned: the window then says so
/// and offers Try Again / Choose Another Library… / Continue Without Saving / Quit (issue #100).
fn open_session(in_memory: bool, dir: Option<std::path::PathBuf>, seed_demo: bool) -> (Session, Option<LibraryProblem>) {
fn open_library_session(in_memory: bool, dir: Option<std::path::PathBuf>, seed_demo: bool) -> (Session, Option<LibraryProblem>) {
if in_memory {
return (if seed_demo { Session::with_demo() } else { Session::new() }.with_fs().with_system_clock(), None);
return (if seed_demo { Session::with_demo() } else { Session::new() }.with_fs().with_default_denoise_models().with_system_clock(), None);
}
let unopened = || Session::new().with_fs().with_system_clock();
let unopened = || Session::new().with_fs().with_default_denoise_models().with_system_clock();
let Some(dir) = dir else {
log::warn!("no library location (set --library or LIGHTCRAFT_LIBRARY)");
return (unopened(), Some(LibraryProblem::new("", "There is no home folder to keep the library in. Choose a folder for it.")));
};
let t0 = std::time::Instant::now();
let mut s = Session::new().with_fs().with_system_clock();
let mut s = Session::new().with_fs().with_default_denoise_models().with_system_clock();
match s.open_library(&dir, seed_demo) {
Ok(r) => {
let (replayed, torn) = (r.replayed, r.torn_bytes);
@@ -637,6 +702,8 @@ fn main() -> eframe::Result {
wgpu_options: window_wgpu_options(),
..Default::default()
};
// the app's copy: `log_file` is still named in the message of a start that fails
let app_log_file = log_file.clone();
let started = eframe::run_native(
"LightCraft",
options,
@@ -648,7 +715,7 @@ fn main() -> eframe::Result {
std::env::var_os("LIGHTCRAFT_SAM3_DIR").map(std::path::PathBuf::from).or_else(|| config_dir().map(|d| d.join("models").join("sam3")));
// the user's own download locations, one base URL per line (LIGHTCRAFT_SAM3_MIRRORS too)
session.segmenter.mirrors_file = config_dir().map(|d| d.join("models").join("sam3-mirrors.txt"));
let mut app = LightcraftApp::new(session, services(cc.egui_ctx.clone()));
let mut app = LightcraftApp::new(session, services(cc.egui_ctx.clone(), app_log_file.as_deref()));
if let Some(ui) = prefs {
app.ui = ui;
}
@@ -872,6 +939,16 @@ mod tests {
assert!(!b.is_empty());
}
/// Issue #471: the window's DX12 shaders compile with FXC — wgpu's default loaded any
/// `dxcompiler.dll` on the search path, and one without `dxil.dll` kept the window from opening.
#[test]
fn window_compiles_dx12_shaders_with_fxc() {
let eframe::egui_wgpu::WgpuSetup::CreateNew(n) = window_wgpu_options().wgpu_setup else { panic!("expected CreateNew") };
if std::env::var_os("WGPU_DX12_COMPILER").is_none() {
assert!(matches!(n.instance_descriptor.backend_options.dx12.shader_compiler, eframe::wgpu::Dx12Compiler::Fxc));
}
}
/// Issue #234: Windows opens links with the URL protocol handler (the default browser), not
/// File Explorer, and passes the whole map link, `?`, `&` and `#` included, as one argument.
#[test]
@@ -883,6 +960,23 @@ mod tests {
assert_eq!(args, ["url.dll,FileProtocolHandler", url]);
}
/// Issue #272: Explorer gets the file to select quoted after `/select,`, with backslashes and
/// without the verbatim prefix, even when the path has spaces (else it opens Documents).
#[test]
fn explorer_select_quotes_the_path() {
assert_eq!(explorer_select_arg(r"C:\Users\Me\My Photos\IMG 1.jpg"), r#"/select,"C:\Users\Me\My Photos\IMG 1.jpg""#);
assert_eq!(explorer_select_arg("D:/Example/Photos/IMG_0001.CR3"), r#"/select,"D:\Example\Photos\IMG_0001.CR3""#);
assert_eq!(explorer_select_arg(r"\\server\share\My Photos\a.jpg"), r#"/select,"\\server\share\My Photos\a.jpg""#);
assert_eq!(explorer_select_arg(r"\\?\D:\My Photos\a.jpg"), r#"/select,"D:\My Photos\a.jpg""#);
assert_eq!(explorer_select_arg(r"\\?\UNC\server\share\a.jpg"), r#"/select,"\\server\share\a.jpg""#);
// a `"` can't end the quoted path and add arguments to the raw command line
assert_eq!(explorer_select_arg(r#"C:\a" "C:\b.jpg"#), r#"/select,"C:\a C:\b.jpg""#);
let c = explorer_select_command(r"C:\My Photos\a.jpg");
assert_eq!(c.get_program(), "explorer");
let args: Vec<&std::ffi::OsStr> = c.get_args().collect();
assert_eq!(args, [r#"/select,"C:\My Photos\a.jpg""#]);
}
#[test]
fn a_control_port_that_is_not_a_number_is_ignored() {
assert_eq!(control_port_from("--control", Some("7980".into())), Some(7980));
+8 -4
View File
@@ -355,12 +355,15 @@ impl NativeMenu {
}
/// Per frame: run chosen items, then sync labels / enabled / checked and the text-focus
/// accelerators with the app state.
pub fn update(&mut self, app: &mut LightcraftApp, ctx: &egui::Context) {
/// accelerators with the app state. Returns whether Quit was chosen: the caller saves
/// everything and exits the process itself (closing the viewport while AppKit terminates the
/// app can crash in winit's teardown, PR #444).
pub fn update(&mut self, app: &mut LightcraftApp, ctx: &egui::Context) -> bool {
lightcraft_ui_egui::i18n::set_language(app.ui.language);
let mut quit = false;
while let Ok(key) = self.rx.try_recv() {
if key == QUIT {
ctx.send_viewport_cmd(egui::ViewportCommand::Close);
quit = true;
continue;
}
if key == "app.about" || key == SETTINGS {
@@ -375,7 +378,7 @@ impl NativeMenu {
let bar = menu_bar(app);
if structure_of(&bar) != self.structure {
self.rebuild(app);
return;
return quit;
}
fn walk(nodes: &[MenuNode], items: &mut HashMap<String, Item>) {
for n in nodes {
@@ -429,6 +432,7 @@ impl NativeMenu {
}
self.publish_shortcuts(app);
}
quit
}
}
+2 -2
View File
@@ -21,12 +21,12 @@ covers a whole directory), or if a licence file referenced in the Licence column
| `docs/images/*-blue-marble.jpg`, `docs/images/grid-pd.jpg` | "The Blue Marble" (AS17-148-22727) | NASA, Apollo 17 crew (1972) | https://commons.wikimedia.org/wiki/File:The_Earth_seen_from_Apollo_17.jpg | Public domain (NASA) | 2026-09-30 | developed in LightCraft; shown inside app screenshots |
| `docs/images/*.jpg` (UI) | LightCraft application screenshots | LightCraft contributors | original work (captured with `docs/showcase/`) | MIT OR Apache-2.0 | 2026-09-30 | n/a |
| `docs/brand/` (artcraft-logo and artcraft-mark, SVG and PNG) | ArtCraft wordmark and mark | ArtCraft Team | original work (https://getartcraft.com/) | ArtCraft trademark, see `docs/brand/LICENSE-brand.txt` (not open source) | 2026-10-01 | none |
| craft-fonts: optional build input, not files in this repository | BIZ UDPGothic (Regular, Bold), BIZ UDMincho Regular, Shippori Mincho Regular | Morisawa Inc. / The BIZ UDGothic and BIZ UDMincho Project Authors; The Shippori Mincho Project Authors (per craft-fonts' attribution) | https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md | SIL Open Font License 1.1 (each font's OFL.txt in craft-fonts; release packages ship it as OFL-<family>.txt) | 2026-10-06 | none; embedded only in builds made with `CRAFT_FONTS_DIR` (all official releases); the web (wasm32) build embeds BIZ UDPGothic Regular only |
| craft-fonts: optional build input, not files in this repository | BIZ UDPGothic (Regular, Bold), BIZ UDMincho Regular, Shippori Mincho Regular, Noto Sans CJK SC Regular | Morisawa Inc. / The BIZ UDGothic and BIZ UDMincho Project Authors; The Shippori Mincho Project Authors; Google and Adobe (Noto Sans CJK, maintainer-approved exception; see AGENTS.md) (per craft-fonts' attribution) | https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md | SIL Open Font License 1.1 (each font's OFL.txt in craft-fonts; release packages ship it as OFL-<family>.txt) | 2026-10-06 | none; native builds made with `CRAFT_FONTS_DIR` embed the manifest fonts (all official releases); the web build embeds BIZ UDPGothic Regular and ships Noto Sans CJK SC Regular as a separate font file |
Notes
- Fonts authored or published by Adobe (Source Sans/Serif/Code, Source Han, …) are not used, even though some are
OFL-licensed: the asset rule excludes anything from Adobe. Source Sans 3 was removed on 2026-09-30 and replaced by Inter.
- Japanese fonts (BIZ UDPGothic, BIZ UDMincho, Shippori Mincho) are not in this repository. They live in
- CJK fonts (BIZ UDPGothic, BIZ UDMincho, Shippori Mincho, Noto Sans CJK SC) are not in this repository. They live in
[storytold/craft-fonts](https://github.com/storytold/craft-fonts), with their licences and attribution in its
`ATTRIBUTION.md`, and are embedded only in builds made with the optional `CRAFT_FONTS_DIR` build input (all official
releases). Never commit font files here; add new fonts to craft-fonts (craftrules `standards/fonts.md`).
+6
View File
@@ -21,3 +21,9 @@
real_name = "Brandon Thomas"
display_name = "Brandon Thomas"
sync_github_name = false
[people.rock-meister]
sync_github_name = true
[people.sanyamsingh001]
display_name = "sanyamsingh001"
+10 -6
View File
@@ -155,7 +155,7 @@ impl Catalog {
let date_of = |id: &PhotoId| -> String {
let Some(p) = self.photo(*id) else { return String::new() };
let d = match key {
SortKey::CaptureDate => Some(p.date()),
SortKey::CaptureDate => p.captured.as_deref(),
SortKey::ImportDate => Some(p.imported.as_str()),
SortKey::EditDate => p.edited.as_deref(),
_ => None,
@@ -222,7 +222,7 @@ mod tests {
}
let shape = |r: &[DateRun]| r.iter().map(|r| (r.key.clone(), r.start, r.count)).collect::<Vec<_>>();
let day = c.date_runs(&ids, SortKey::CaptureDate, GroupBy::Day);
// the undated photo falls back to its import date
// the undated photo is grouped under Unknown Date
assert_eq!(
shape(&day),
vec![
@@ -230,16 +230,20 @@ mod tests {
("2026-09-29".into(), 2, 1),
("2026-08-01".into(), 3, 1),
("2025-12-31".into(), 4, 1),
("2026-10-01".into(), 5, 1)
(String::new(), 5, 1)
]
);
assert_eq!(day[0].label, "Wednesday, 30 September 2026");
assert_eq!(day[4].label, "Unknown Date");
assert_eq!(c.date_runs(&ids, SortKey::CaptureDate, GroupBy::Auto), day);
assert_eq!(
shape(&c.date_runs(&ids[..5], SortKey::CaptureDate, GroupBy::Month)),
vec![("2026-09".into(), 0, 3), ("2026-08".into(), 3, 1), ("2025-12".into(), 4, 1)]
shape(&c.date_runs(&ids, SortKey::CaptureDate, GroupBy::Month)),
vec![("2026-09".into(), 0, 3), ("2026-08".into(), 3, 1), ("2025-12".into(), 4, 1), (String::new(), 5, 1)]
);
assert_eq!(
shape(&c.date_runs(&ids, SortKey::CaptureDate, GroupBy::Year)),
vec![("2026".into(), 0, 4), ("2025".into(), 4, 1), (String::new(), 5, 1)]
);
assert_eq!(shape(&c.date_runs(&ids[..5], SortKey::CaptureDate, GroupBy::Year)), vec![("2026".into(), 0, 4), ("2025".into(), 4, 1)]);
assert!(c.date_runs(&ids, SortKey::FileName, GroupBy::Day).is_empty());
assert!(c.date_runs(&ids, SortKey::CaptureDate, GroupBy::None).is_empty());
// edit date: unedited photos have no date
+2 -1
View File
@@ -45,6 +45,7 @@
//! | (none) | early builds: a log without a snapshot | |
//! | 1 | up to v0.2.0 | |
//! | 2 | after v0.2.0 | `Op::SetBrowsed`, `Catalog.browsed`, `Photo.local_baseline` |
//! | 3 | albums by hand | `Op::SetAlbumOrder`, `Album.order` |
//!
//! Rules:
//! - **Bump [`VERSION`]** (and add a row above) in the change that adds an [`Op`] variant or a
@@ -69,7 +70,7 @@ pub const LOG: &str = "catalog.log";
const FORMAT: &str = "lightcraft-catalog";
/// The catalog format this build writes (and the newest it reads). See the module docs →
/// *Format versions*; bump it whenever an [`Op`] variant or a serialized field is added.
pub const VERSION: u32 = 2;
pub const VERSION: u32 = 3;
/// When [`Journal::wants_snapshot`] says it's time to compact the log.
#[derive(Clone, Copy, Debug)]
+49 -1
View File
@@ -136,6 +136,11 @@ pub enum Op {
id: AlbumId,
parent: Option<AlbumId>,
},
/// Where the album stands among its siblings (`None`: by name). Format version 3.
SetAlbumOrder {
id: AlbumId,
order: Option<u32>,
},
SetAlbumPhotos {
id: AlbumId,
photos: Vec<PhotoId>,
@@ -282,6 +287,28 @@ impl Catalog {
pub fn albums(&self) -> impl Iterator<Item = &Album> {
self.albums.values()
}
/// The albums and folders directly inside `parent` (`None`: the top level), in the order the
/// sidebar lists them: folders first, then by their place if the user ordered them (those
/// without one after, by name), else by name.
pub fn album_children(&self, parent: Option<AlbumId>) -> Vec<&Album> {
let mut kids: Vec<&Album> = self.albums.values().filter(|a| a.parent == parent).collect();
kids.sort_by_cached_key(|a| album_order_key(a));
kids
}
/// [`Self::album_children`] of every folder (and of the top level, under `None`) in one pass:
/// what a tree view that draws all of them wants each frame.
pub fn album_children_by_parent(&self) -> std::collections::HashMap<Option<AlbumId>, Vec<&Album>> {
let mut map: std::collections::HashMap<Option<AlbumId>, Vec<&Album>> = std::collections::HashMap::new();
for a in self.albums.values() {
map.entry(a.parent).or_default().push(a);
}
map.values_mut().for_each(|kids| kids.sort_by_cached_key(|a| album_order_key(a)));
map
}
/// Whether anything inside `parent` has a place of its own (the folder is ordered by hand).
pub fn album_children_are_ordered(&self, parent: Option<AlbumId>) -> bool {
self.albums.values().any(|a| a.parent == parent && a.order.is_some())
}
/// The Quick Collection, once something was added to it.
pub fn quick_collection(&self) -> Option<AlbumId> {
self.albums.values().find(|a| a.quick).map(|a| a.id)
@@ -479,7 +506,20 @@ impl Catalog {
}
}
let a = self.album_mut(id)?;
Op::MoveAlbum { id, parent: std::mem::replace(&mut a.parent, parent) }
if a.parent == parent {
return Ok(Op::MoveAlbum { id, parent });
}
// its place belonged to the old neighbours: the undo gives it back with the folder
let old_order = a.order.take();
let old_parent = std::mem::replace(&mut a.parent, parent);
match old_order {
Some(_) => Op::Batch { ops: vec![Op::MoveAlbum { id, parent: old_parent }, Op::SetAlbumOrder { id, order: old_order }] },
None => Op::MoveAlbum { id, parent: old_parent },
}
}
Op::SetAlbumOrder { id, order } => {
let a = self.album_mut(id)?;
Op::SetAlbumOrder { id, order: std::mem::replace(&mut a.order, order) }
}
Op::SetAlbumPhotos { id, photos } => {
let a = self.album_mut(id)?;
@@ -668,9 +708,17 @@ impl Catalog {
}
}
/// How siblings are listed: folders first, then the ones with a place of their own by it, then by
/// name; the id settles any tie.
fn album_order_key(a: &Album) -> (bool, bool, u32, String, AlbumId) {
(!a.folder, a.order.is_none(), a.order.unwrap_or(0), a.name.to_lowercase(), a.id)
}
#[cfg(test)]
mod tests;
#[cfg(test)]
mod tests_album_order;
#[cfg(test)]
mod tests_background;
#[cfg(test)]
mod tests_folders;
+1
View File
@@ -295,6 +295,7 @@ impl Op {
Op::RemoveAlbum { .. }
| Op::RenameAlbum { .. }
| Op::MoveAlbum { .. }
| Op::SetAlbumOrder { .. }
| Op::SetAlbumRules { .. }
| Op::RemoveStack { .. }
| Op::SetLabelName { .. }
+8 -3
View File
@@ -336,13 +336,13 @@ impl Photo {
pub fn develops_raw(&self) -> bool {
self.kind == MediaKind::Raw && self.preview_only.is_none()
}
/// The current ARW, NEF, RW2, RAF, CR3, CR2 and PEF readers have vendor WB multipliers but no measured camera
/// The current ARW, NEF, RW2, RAF, CR3, CR2, PEF and SRW readers have vendor WB multipliers but no measured camera
/// illuminant. Use adjustments relative to the camera's as-shot look, as for rendered
/// photographs (the engine's `camera_preview::file_local_look` covers the same formats; RWL and
/// RAW are Leica's and the oldest Panasonic bodies' names for RW2 files).
pub fn relative_wb(&self) -> bool {
self.develops_raw()
&& ["ARW", "NEF", "NRW", "RW2", "RWL", "RAW", "RAF", "CR3", "CR2", "PEF"].iter().any(|f| self.format.eq_ignore_ascii_case(f))
&& ["ARW", "NEF", "NRW", "RW2", "RWL", "RAW", "RAF", "CR3", "CR2", "PEF", "SRW"].iter().any(|f| self.format.eq_ignore_ascii_case(f))
}
/// The develop settings import gave this photo: [`Photo::camera_defaults`], or the user's
/// default preset applied on top of them ([`Photo::import_look`]).
@@ -405,12 +405,16 @@ pub struct Album {
/// The Quick Collection: a regular album that B adds to (one per library).
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub quick: bool,
/// Its place among the albums of the same kind in its folder, once the user ordered them by
/// hand (see [`crate::Catalog::album_children`]); `None`: listed by name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub order: Option<u32>,
}
impl Album {
/// A regular (manual) album.
pub fn new(id: AlbumId, name: impl Into<String>) -> Album {
Album { id, name: name.into(), parent: None, folder: false, photos: Vec::new(), cover: None, smart: None, quick: false }
Album { id, name: name.into(), parent: None, folder: false, photos: Vec::new(), cover: None, smart: None, quick: false, order: None }
}
pub fn is_smart(&self) -> bool {
self.smart.is_some()
@@ -446,6 +450,7 @@ mod edited_tests {
("a.raw", "RAW", true),
("a.raf", "rAf", true),
("a.cr3", "cr3", true),
("a.srw", "SRW", true),
("a.dng", "DNG", false),
] {
let mut p = Photo::new(PhotoId(1), Source::Demo { scene: 0 }, name, format, 10, 10, "2026-10-01T00:00:00");
+12 -6
View File
@@ -141,7 +141,7 @@ fn token_matches(p: &Photo, tok: &str) -> bool {
"person" | "who" => has_person(p, val),
"type" | "kind" => format!("{:?}", p.kind).eq_ignore_ascii_case(val),
"edited" => (val == "true" || val == "yes") == p.is_edited(),
"date" => p.date().starts_with(val),
"date" => p.captured.as_deref().is_some_and(|c| c.starts_with(val)),
"copy" | "virtual" => (val == "true" || val == "yes") == p.copy_of.is_some(),
"name" | "file" => p.file_name.to_lowercase().contains(val),
_ => false,
@@ -284,12 +284,18 @@ impl Filter {
return false;
}
if let Some(from) = &self.date_from
&& p.date() < from.as_str()
&& match p.captured.as_deref() {
Some(c) => c < from.as_str(),
None => true,
}
{
return false;
}
if let Some(to) = &self.date_to
&& p.date().get(..to.len()).unwrap_or(p.date()) > to.as_str()
&& match p.captured.as_deref() {
Some(c) => c.get(..to.len()).unwrap_or(c) > to.as_str(),
None => true,
}
{
return false;
}
@@ -299,7 +305,7 @@ impl Filter {
return false;
}
if let Some(d) = &self.date
&& !p.date().starts_with(d.as_str())
&& !p.captured.as_deref().is_some_and(|c| c.starts_with(d.as_str()))
{
return false;
}
@@ -344,7 +350,7 @@ impl Catalog {
let mut v: Vec<&Photo> = self.photos().map(|p| p.as_ref()).filter(|p| filter.matches_in(p, self, root.as_deref())).collect();
v.sort_by(|a, b| {
let o = match sort.key {
SortKey::CaptureDate => a.date().cmp(b.date()),
SortKey::CaptureDate => a.captured.cmp(&b.captured).then_with(|| a.imported.cmp(&b.imported)),
SortKey::ImportDate => a.imported.cmp(&b.imported),
SortKey::EditDate => a.edited.cmp(&b.edited),
SortKey::FileName => a.file_name.to_lowercase().cmp(&b.file_name.to_lowercase()),
@@ -363,7 +369,7 @@ impl Catalog {
use std::collections::BTreeMap;
let mut years: BTreeMap<&str, (BTreeMap<&str, usize>, BTreeMap<&str, usize>)> = BTreeMap::new();
for p in self.photos().filter(|p| p.in_library()) {
let d = p.date();
let Some(d) = p.captured.as_deref() else { continue };
if d.len() >= 10 {
let (months, days) = years.entry(&d[..4]).or_default();
*months.entry(&d[..7]).or_default() += 1;
+57 -2
View File
@@ -58,12 +58,22 @@ fn albums_and_folders() {
let a = photo(&mut c, "a.jpg", "2026-04-01");
let f = c.alloc_album_id();
c.apply(Op::AddAlbum {
album: Album { id: f, name: "Trips".into(), parent: None, folder: true, photos: vec![], cover: None, smart: None, quick: false },
album: Album { id: f, name: "Trips".into(), parent: None, folder: true, photos: vec![], cover: None, smart: None, quick: false, order: None },
})
.unwrap();
let al = c.alloc_album_id();
c.apply(Op::AddAlbum {
album: Album { id: al, name: "Alps".into(), parent: Some(f), folder: false, photos: vec![a], cover: None, smart: None, quick: false },
album: Album {
id: al,
name: "Alps".into(),
parent: Some(f),
folder: false,
photos: vec![a],
cover: None,
smart: None,
quick: false,
order: None,
},
})
.unwrap();
assert_eq!(c.albums_of(a), vec![al]);
@@ -446,3 +456,48 @@ fn random_sort_has_no_date_headers() {
let ids = many(&mut c, 5);
assert!(c.date_runs(&ids, SortKey::Random, GroupBy::Day).is_empty());
}
#[test]
fn undated_photos_group_under_unknown_date_and_sort_together() {
let mut c = Catalog::new();
let add_photo = |c: &mut Catalog, name: &str, captured: Option<&str>, imported: &str| {
let id = c.alloc_photo_id();
let mut p = Photo::new(id, Source::Demo { scene: 0 }, name, "JPEG", 100, 100, imported);
p.captured = captured.map(str::to_string);
c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
id
};
let p_dated1 = add_photo(&mut c, "dated1.jpg", Some("2026-05-10T12:00:00"), "2026-10-01T00:00:00");
let p_dated2 = add_photo(&mut c, "dated2.jpg", Some("2026-05-20T12:00:00"), "2026-10-01T00:00:00");
let p_undated1 = add_photo(&mut c, "undated1.jpg", None, "2026-10-05T00:00:00");
let p_undated2 = add_photo(&mut c, "undated2.jpg", None, "2026-10-02T00:00:00");
// Descending sort (default, newest first): dated photos first (newest to oldest), undated photos at the end
let desc = c.query(&Filter::default(), &Sort { key: SortKey::CaptureDate, ascending: false, ..Default::default() });
assert_eq!(desc, vec![p_dated2, p_dated1, p_undated1, p_undated2]);
// Ascending sort (oldest first): undated photos at the beginning, dated photos from oldest to newest
let asc = c.query(&Filter::default(), &Sort { key: SortKey::CaptureDate, ascending: true, ..Default::default() });
assert_eq!(asc, vec![p_undated2, p_undated1, p_dated1, p_dated2]);
// date_runs groups undated photos under Unknown Date
let runs = c.date_runs(&desc, SortKey::CaptureDate, GroupBy::Day);
assert_eq!(runs.len(), 3);
assert_eq!(runs[0].key, "2026-05-20");
assert_eq!(runs[1].key, "2026-05-10");
assert_eq!(runs[2].key, "");
assert_eq!(runs[2].label, "Unknown Date");
assert_eq!(runs[2].count, 2);
// date_groups sidebar tree only counts photos with capture dates
let groups = c.date_groups();
assert_eq!(groups.len(), 1);
assert_eq!(groups[0].year, "2026");
assert_eq!(groups[0].count, 2);
// Filter by capture date prefix does not match undated photos
let filter_day = Filter { date: Some("2026-10-05".into()), ..Default::default() };
assert_eq!(c.query(&filter_day, &Sort::default()), vec![]);
let filter_year = Filter { date: Some("2026".into()), ..Default::default() };
assert_eq!(c.query(&filter_year, &Sort { key: SortKey::CaptureDate, ascending: false, ..Default::default() }), vec![p_dated2, p_dated1]);
}
+122
View File
@@ -0,0 +1,122 @@
//! Album order: the children of a folder (or of the top level) are listed folders first, then
//! alphabetically, until some of them are given a place of their own (`Album.order`); moving an
//! album to another folder drops its place, and undoing the move brings it back.
use crate::*;
fn add(c: &mut Catalog, name: &str, parent: Option<AlbumId>, folder: bool) -> AlbumId {
let id = c.alloc_album_id();
c.apply(Op::AddAlbum { album: Album { parent, folder, ..Album::new(id, name) } }).unwrap();
id
}
fn names(c: &Catalog, parent: Option<AlbumId>) -> Vec<String> {
c.album_children(parent).iter().map(|a| a.name.clone()).collect()
}
fn place(c: &mut Catalog, id: AlbumId, order: Option<u32>) {
c.apply(Op::SetAlbumOrder { id, order }).unwrap();
}
/// Given albums that were never ordered, when they are listed, then folders come first and each
/// group is alphabetical regardless of case.
#[test]
fn children_are_folders_first_then_alphabetical() {
let mut c = Catalog::new();
add(&mut c, "beta", None, false);
add(&mut c, "Alpha", None, false);
add(&mut c, "zoo", None, true);
add(&mut c, "Archive", None, true);
assert_eq!(names(&c, None), ["Archive", "zoo", "Alpha", "beta"]);
assert!(!c.album_children_are_ordered(None));
}
/// Given some children with a place of their own, then they follow it; the others come after, by
/// name; folders stay ahead of albums.
#[test]
fn ordered_children_follow_their_order() {
let mut c = Catalog::new();
let a = add(&mut c, "A", None, false);
let b = add(&mut c, "B", None, false);
let cc = add(&mut c, "C", None, false);
add(&mut c, "D", None, false);
let f = add(&mut c, "F", None, true);
place(&mut c, cc, Some(0));
place(&mut c, a, Some(1));
assert_eq!(names(&c, None), ["F", "C", "A", "B", "D"]);
assert!(c.album_children_are_ordered(None));
// equal places fall back to the name, so a damaged catalog still lists deterministically
place(&mut c, b, Some(1));
assert_eq!(names(&c, None), ["F", "C", "A", "B", "D"]);
// another container is unaffected
let inner = add(&mut c, "Z", Some(f), false);
add(&mut c, "Y", Some(f), false);
assert_eq!(names(&c, Some(f)), ["Y", "Z"]);
assert!(!c.album_children_are_ordered(Some(f)));
place(&mut c, inner, Some(0));
assert_eq!(names(&c, Some(f)), ["Z", "Y"]);
}
/// The one-pass map lists every folder's children exactly as `album_children` does.
#[test]
fn the_children_map_matches_the_children_of_each_folder() {
let mut c = Catalog::new();
let f = add(&mut c, "F", None, true);
let g = add(&mut c, "G", Some(f), true);
for (name, parent) in [("b", None), ("a", None), ("y", Some(f)), ("x", Some(f)), ("q", Some(g))] {
add(&mut c, name, parent, false);
}
let x = c.albums().find(|a| a.name == "x").unwrap().id;
place(&mut c, x, Some(7));
let map = c.album_children_by_parent();
for parent in [None, Some(f), Some(g)] {
let from_map: Vec<AlbumId> = map.get(&parent).map(|v| v.iter().map(|a| a.id).collect()).unwrap_or_default();
let direct: Vec<AlbumId> = c.album_children(parent).iter().map(|a| a.id).collect();
assert_eq!(from_map, direct, "{parent:?}");
}
assert!(!map.contains_key(&Some(AlbumId(999))));
}
/// A place is one undo step, and `None` puts the album back among the alphabetical ones.
#[test]
fn a_place_can_be_set_cleared_and_undone() {
let mut c = Catalog::new();
let a = add(&mut c, "A", None, false);
add(&mut c, "B", None, false);
let inv = c.apply(Op::SetAlbumOrder { id: a, order: Some(5) }).unwrap();
assert_eq!(c.album(a).unwrap().order, Some(5));
c.apply(inv).unwrap();
assert_eq!(c.album(a).unwrap().order, None);
assert!(c.apply(Op::SetAlbumOrder { id: AlbumId(999), order: Some(1) }).is_err(), "no such album");
}
/// Moving to another folder drops the album's place there (it belonged to the old neighbours),
/// and undoing the move gives it back.
#[test]
fn moving_drops_the_place_and_undo_restores_it() {
let mut c = Catalog::new();
let from = add(&mut c, "From", None, true);
let to = add(&mut c, "To", None, true);
let a = add(&mut c, "A", Some(from), false);
place(&mut c, a, Some(3));
let inv = c.apply(Op::MoveAlbum { id: a, parent: Some(to) }).unwrap();
assert_eq!((c.album(a).unwrap().parent, c.album(a).unwrap().order), (Some(to), None));
c.apply(inv).unwrap();
assert_eq!((c.album(a).unwrap().parent, c.album(a).unwrap().order), (Some(from), Some(3)));
// "moving" to where it already is changes nothing
c.apply(Op::MoveAlbum { id: a, parent: Some(from) }).unwrap();
assert_eq!(c.album(a).unwrap().order, Some(3));
}
/// Catalogs written before albums had an order load as they were; an unset order is not written.
#[test]
fn albums_without_an_order_load_and_save_unchanged() {
let old = r#"{"id":4,"name":"Old","parent":null,"folder":false,"photos":[],"cover":null}"#;
let a: Album = serde_json::from_str(old).unwrap();
assert_eq!(a.order, None);
assert!(!serde_json::to_string(&a).unwrap().contains("order"));
let mut a = a;
a.order = Some(2);
let back: Album = serde_json::from_str(&serde_json::to_string(&a).unwrap()).unwrap();
assert_eq!(back.order, Some(2));
}
+15 -1
View File
@@ -47,6 +47,19 @@ fn v1_library_loads_and_is_upgraded() {
assert_eq!((r.replayed, r.upgraded_from), (1, None));
}
/// A library saved before albums could be ordered by hand opens as it was and is rewritten as v3.
#[test]
fn v2_library_loads_and_is_upgraded() {
let (base, log, full) = legacy_parts();
let m = MemStore::new();
m.set(SNAPSHOT, format!("{{\"format\":\"lightcraft-catalog\",\"version\":2,\"seq\":2,\"catalog\":{}}}\n", base.to_snapshot()).into_bytes());
m.set(LOG, log.into_bytes());
let (_, c, r) = Journal::open(Box::new(m.clone())).unwrap();
assert_eq!(c.to_snapshot(), full.to_snapshot());
assert_eq!((r.replayed, r.upgraded_from), (1, Some(2)));
assert_eq!(snapshot_version(&m), 3);
}
#[test]
fn versionless_log_only_library_is_upgraded() {
let mut c = Catalog::new();
@@ -179,8 +192,9 @@ fn op_variants_are_versioned() {
| Op::SetLabelName { .. }
| Op::Batch { .. } => 1,
Op::SetBrowsed { .. } => 2,
Op::SetAlbumOrder { .. } => 3,
}
}
let newest = since(&Op::SetBrowsed { folder: String::new(), at: None });
let newest = since(&Op::SetAlbumOrder { id: AlbumId(0), order: None });
assert_eq!(newest, VERSION, "the newest op's version must be the current format version");
}
+2
View File
@@ -59,6 +59,7 @@ fn op_for(c: &mut Catalog, kind: u8, a: u8, b: u8) -> Op {
cover: None,
smart: None,
quick: false,
order: None,
},
}
}
@@ -77,6 +78,7 @@ fn op_for(c: &mut Catalog, kind: u8, a: u8, b: u8) -> Op {
13 => Op::Batch { ops: c.remove_from_stacks_ops(&[pid]) },
14 => c.set_top_ops(pid).unwrap_or(Op::Batch { ops: vec![] }),
9 if b.is_multiple_of(5) => Op::SetLabelName { label: ColorLabel::ALL[a as usize % 5], name: (a % 3 != 1).then(|| format!("Label {b}")) },
9 if b % 7 == 3 => Op::SetAlbumOrder { id: aid, order: (!b.is_multiple_of(3)).then_some(u32::from(b % 5)) },
9 if b.is_multiple_of(2) => {
Op::SetFile { id: pid, file_name: format!("Renamed_{b}.jpg"), source: Source::File { path: format!("/p/Renamed_{b}.jpg") } }
}
+5
View File
@@ -12,6 +12,7 @@ workspace = true
lightcraft-geom = { workspace = true }
lightcraft-color = { workspace = true }
lightcraft-raster = { workspace = true }
lightcraft-heif = { workspace = true, optional = true }
serde = { workspace = true }
bytemuck = { workspace = true }
thiserror = { workspace = true }
@@ -34,6 +35,10 @@ ravif = { workspace = true, optional = true }
default = ["parallel", "jxl", "avif"]
parallel = ["dep:rayon", "lightcraft-raster/parallel", "jpeg-decoder/rayon"]
jxl = ["dep:jxl-oxide"]
# HEIC/HEIF decode via the optional lightcraft-heif crate (heic-rs, pure Rust). Off by default so
# distributors choose it at build time (HEVC patents are a distributor's call, same policy as
# PhotoCraft); `cargo xtask ci` clippies and tests it (`--features heif`), releases don't ship it yet.
heif = ["dep:lightcraft-heif"]
## AVIF encoding via ravif/rav1e (pure Rust, BSD). Native targets only; a no-op on wasm32.
avif = ["dep:ravif"]
+125
View File
@@ -0,0 +1,125 @@
//! HEIF / HEIC (the iPhone and Mac photo format), read-only.
//!
//! Decoding lives in the optional `lightcraft-heif` crate (heic-rs, a pure-Rust HEVC
//! still-picture decoder), enabled by this crate's `heif` feature — off by default because HEVC
//! is patent-encumbered and whether a build carries an HEVC decoder is the distributor's call
//! (the same policy as the sibling PhotoCraft). Without the feature, HEIF files are still
//! detected and opening one is an [`Error::Unsupported`] error, never a panic.
//!
//! HEIF records orientation in the container (`irot`/`imir`, plus a `clap` crop), not in EXIF,
//! so the decoder applies them and `orientation` is reported as 1: the EXIF Orientation tag only
//! mirrors what the container says, and applying it as well would turn the photo twice.
//! The samples come back in the container's own primaries with its ICC profile (if any); like
//! every decoder here, the colour interpretation happens in [`crate::convert::finish`].
#[cfg(feature = "heif")]
use crate::convert::{Buf, Meta, Model, Raw, finish};
use crate::{DecodeOptions, Decoded, Error, Format, Result};
const F: Format = Format::Heif;
/// The reason given when this build has no HEIF decoder.
#[cfg(not(feature = "heif"))]
pub(crate) const NOT_IN_BUILD: &str = "HEIC/HEIF support isn't included in this build of LightCraft";
/// The reason a HEIF file that no build can open is refused (this crate decodes HEVC stills
/// only; the wrapper names the exact construct in its own errors).
#[cfg(feature = "heif")]
const NOT_DECODED: &str = "the image is not a decodable HEVC still (image sequence, overlay, alpha-less grid or AVIF-coded item)";
#[cfg(not(feature = "heif"))]
pub(crate) fn decode(_bytes: &[u8], _opts: &DecodeOptions) -> Result<Decoded> {
Err(Error::Unsupported(F, NOT_IN_BUILD))
}
#[cfg(feature = "heif")]
fn err(e: lightcraft_heif::Error) -> Error {
match e {
// This crate's Unsupported takes a fixed reason; the dynamic wording lives in the
// lightcraft-heif error itself.
lightcraft_heif::Error::Unsupported(_) => Error::Unsupported(F, NOT_DECODED),
lightcraft_heif::Error::Limit(m) | lightcraft_heif::Error::Malformed(m) => Error::Malformed(F, m),
}
}
#[cfg(feature = "heif")]
pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
// The container alone: the declared size is checked before any pixel is decoded.
let info = lightcraft_heif::probe(bytes).map_err(err)?;
if (info.width as u64).saturating_mul(info.height as u64) > opts.max_pixels {
return Err(Error::TooLarge(info.width as u64, info.height as u64));
}
let options = lightcraft_heif::Options { max_pixels: opts.max_pixels, apply_transforms: true };
let decoded = lightcraft_heif::decode(bytes, &options).map_err(err)?;
// The decoder re-reads the container; trust its output's shape, not the probe's.
let buf = if decoded.sixteen_bit {
let mut v = Vec::with_capacity(decoded.data.len() / 2);
for pair in decoded.data.as_chunks::<2>().0 {
v.push(u16::from_ne_bytes(*pair));
}
Buf::U16(v)
} else {
Buf::U8(decoded.data)
};
let raw = Raw {
width: decoded.width as usize,
height: decoded.height as usize,
model: Model::Rgb,
alpha: decoded.has_alpha,
premultiplied: false,
buf,
bit_depth: if decoded.sixteen_bit { 16 } else { 8 },
};
let meta = Meta { icc: decoded.icc, exif: decoded.exif, xmp: decoded.xmp, orientation: Some(1), ..Default::default() };
finish(F, raw, meta, (decoded.width, decoded.height), opts)
}
#[cfg(test)]
mod tests {
use crate::{Format, sniff};
/// The container header both builds recognise, whatever the feature says.
const HEIC_HEADER: &[u8] = b"\0\0\0\x18ftypheic\0\0\0\0mif1heic";
#[test]
fn heif_is_always_recognised() {
assert_eq!(sniff(HEIC_HEADER), Some(Format::Heif));
}
#[test]
fn can_decode_follows_the_feature() {
assert_eq!(Format::Heif.can_decode(), cfg!(feature = "heif"));
}
#[cfg(not(feature = "heif"))]
#[test]
fn without_the_feature_heif_is_a_clear_unsupported_error() {
let r = crate::decode(HEIC_HEADER, crate::DecodeOptions::default());
assert!(matches!(&r, Err(crate::Error::Unsupported(Format::Heif, why)) if why.contains("isn't included in this build")), "{r:?}");
}
/// Found by PhotoCraft's `decode_heif` fuzz target: a malformed box makes heic-rs 0.1.1 slice
/// out of range (`boxes.rs:130`). It must be an error, not a crash.
#[cfg(feature = "heif")]
const HEIC_RS_BOX_PANIC: [u8; 72] = [
0x00, 0x00, 0x00, 0x24, 0x66, 0x74, 0x79, 0x70, 0x68, 0x65, 0x69, 0x63, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x69, 0x66, 0x31, 0x4d, 0x69, 0x50,
0x72, 0x6d, 0x69, 0x61, 0x66, 0x4d, 0x69, 0x48, 0x42, 0x72, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x65, 0x74, 0x61, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x75, 0x75, 0x69, 0x64, 0x00, 0x00, 0x00, 0x00, 0x03, 0x08, 0x08, 0x08, 0x00, 0x03, 0x01, 0x03, 0x70,
0x00, 0xa8, 0x00,
];
#[cfg(feature = "heif")]
#[test]
fn a_heic_rs_panic_is_a_malformed_error() {
let r = crate::decode(&HEIC_RS_BOX_PANIC, crate::DecodeOptions::default());
assert!(matches!(&r, Err(crate::Error::Malformed(Format::Heif, _))), "{r:?}");
}
#[cfg(feature = "heif")]
#[test]
fn an_image_sequence_is_unsupported() {
// An ftyp with no meta box: a video track, not a still.
let r = crate::decode(b"\0\0\0\x18ftypmsf1\0\0\0\0msf1hevc", crate::DecodeOptions::default());
assert!(matches!(&r, Err(crate::Error::Unsupported(Format::Heif, _))), "{r:?}");
}
}
+12 -1
View File
@@ -165,7 +165,7 @@ pub(crate) fn decode_with_fallback(bytes: &[u8], opts: &DecodeOptions, fallback:
return Err(Error::Malformed(F, "no frame header".into()));
}
check_size(F, m.width as u64, m.height as u64, opts)?;
let scale_to = opts.max_size.filter(|&(mw, mh)| mw > 0 && mh > 0 && (m.width >= 2 * mw || m.height >= 2 * mh));
let scale_to = opts.max_size.filter(|&(mw, mh)| mw > 0 && mh > 0 && (m.width >= mw.saturating_mul(2) || m.height >= mh.saturating_mul(2)));
// zune-jpeg handles the common cases fastest; jpeg-decoder covers DCT scaling, CMYK/YCCK,
// 12-bit and non-interleaved sequential scans (which zune-jpeg 0.5 mis-decodes with subsampling).
let non_interleaved = matches!(m.sof, 0xC0 | 0xC1) && m.first_scan_components < m.components;
@@ -377,6 +377,17 @@ mod tests {
}
}
#[test]
fn unbounded_fit_box_decodes_at_full_size() {
// a full-size load asks for the largest box (`usize::MAX` saturated to `u32::MAX`): no DCT scaling,
// and no overflow working out whether to scale
use crate::{ChromaSubsampling, EncodeImage, EncodeMeta, Samples, encode_jpeg};
let pixels = vec![128u8; 32 * 24 * 3];
let jpeg = encode_jpeg(&EncodeImage::new(32, 24, 3, Samples::U8(&pixels)), 90, ChromaSubsampling::S444, &EncodeMeta::default()).unwrap();
let d = crate::decode(&jpeg, DecodeOptions::fit(u32::MAX, u32::MAX)).unwrap();
assert_eq!((d.width, d.height), (32, 24));
}
#[test]
fn markers_on_garbage() {
assert!(parse_markers(b"").is_none());
+2 -1
View File
@@ -19,6 +19,7 @@
mod convert;
pub mod encode;
pub mod exif;
mod heif;
pub mod icc;
mod jpeg;
pub mod jpeg_par;
@@ -227,8 +228,8 @@ pub fn decode_unguarded(bytes: &[u8], format: Format, opts: &DecodeOptions) -> R
Format::Jxl => jxl::decode(bytes, &opts),
#[cfg(not(feature = "jxl"))]
Format::Jxl => Err(Error::Unsupported(format, "built without the `jxl` feature")),
Format::Heif => heif::decode(bytes, &opts),
Format::Avif => Err(Error::Unsupported(format, "no pure-Rust, permissively licensed AV1 decoder yet")),
Format::Heif => Err(Error::Unsupported(format, "no pure-Rust, permissively licensed HEVC decoder yet")),
Format::RawTiffLike | Format::RawOther => Err(Error::Unsupported(format, "camera raw: decode with lightcraft-raw")),
}
}
+6 -6
View File
@@ -167,9 +167,9 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
};
let idx = planes.first().ok_or_else(|| err("missing plane"))?;
let mut v = zeroed::<u8>(n.checked_mul(3).ok_or_else(too_large)?, too_large)?;
for (o, &i) in v.chunks_exact_mut(3).zip(idx) {
for (o, &i) in v.as_chunks_mut::<3>().0.iter_mut().zip(idx) {
let i = i as usize;
o.copy_from_slice(&[pal[i], pal[256 + i], pal[512 + i]]);
*o = [pal[i], pal[256 + i], pal[512 + i]];
}
Buf::U8(v)
} else {
@@ -188,8 +188,8 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
2 => {
let mut v = zeroed::<u16>(samples, too_large)?;
for (c, p) in planes.iter().enumerate() {
for (o, s) in v.iter_mut().skip(c).step_by(used).zip(p.chunks_exact(2)) {
let x = u16::from_be_bytes([s[0], s[1]]);
for (o, s) in v.iter_mut().skip(c).step_by(used).zip(p.as_chunks::<2>().0) {
let x = u16::from_be_bytes(*s);
*o = if cmyk && c < 4 { 65535 - x } else { x };
}
}
@@ -198,8 +198,8 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
_ => {
let mut v = zeroed::<f32>(samples, too_large)?;
for (c, p) in planes.iter().enumerate() {
for (o, s) in v.iter_mut().skip(c).step_by(used).zip(p.chunks_exact(4)) {
let x = f32::from_be_bytes([s[0], s[1], s[2], s[3]]);
for (o, s) in v.iter_mut().skip(c).step_by(used).zip(p.as_chunks::<4>().0) {
let x = f32::from_be_bytes(*s);
*o = if cmyk && c < 4 { 1.0 - x } else { x };
}
}
+2 -1
View File
@@ -66,7 +66,8 @@ impl Format {
pub fn can_decode(self) -> bool {
match self {
Format::Jxl => cfg!(feature = "jxl"),
Format::Avif | Format::Heif | Format::RawTiffLike | Format::RawOther => false,
Format::Heif => cfg!(feature = "heif"),
Format::Avif | Format::RawTiffLike | Format::RawOther => false,
_ => true,
}
}
+26
View File
@@ -0,0 +1,26 @@
[package]
name = "lightcraft-denoise-core"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "LightCraft denoise model metadata, Bayer tiling and cache formats (L1; no inference backend)"
[lints]
workspace = true
[features]
default = []
rawnind-model = []
[dependencies]
lightcraft-raster = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
sha2 = { workspace = true }
thiserror = { workspace = true }
miniz_oxide = { workspace = true }
rayon = { workspace = true }
web-time = { workspace = true }
[dev-dependencies]
proptest = { workspace = true }
+7
View File
@@ -0,0 +1,7 @@
# lightcraft-denoise-core
L1 model metadata, bounded archive/cache formats and Bayer packing/tiling helpers.
No weights or inference backend. See [the user workflow](../../docs/denoise.md).
`lightcraft-denoise` re-exports these types for CPU/GPU users; the engine uses this
crate directly when inference is disabled. The default download catalog is empty.
Run `cargo test -p lightcraft-denoise-core`; optional catalog checks use `--features rawnind-model`.
+279
View File
@@ -0,0 +1,279 @@
//! Taking one file out of a `.zip` (a `.dtmodel` is one), the little of the format a model download needs: stored and
//! deflated entries, found by name, checked against their CRC-32 and a size limit, written next to where they are
//! going and moved into place. It is a stranger's file, so every length and offset in it is checked, a name is only
//! ever compared (never used as a path), and zip64, encrypted and multi-disk archives are refused.
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::Path;
/// How far from the end the end-of-central-directory record can start (a 22-byte record and a 64 KiB comment).
const EOCD_SEARCH: u64 = 22 + 65535;
/// Largest central directory we read.
const MAX_DIRECTORY: u64 = 1 << 20;
#[derive(Debug, PartialEq, thiserror::Error)]
pub enum ArchiveError {
#[error("the archive could not be read: {0}")]
Io(String),
#[error("not a usable archive: {0}")]
Format(String),
#[error("the archive has no file called `{0}`")]
Missing(String),
}
impl From<std::io::Error> for ArchiveError {
fn from(e: std::io::Error) -> Self {
ArchiveError::Io(e.to_string())
}
}
fn format<T>(why: &str) -> Result<T, ArchiveError> {
Err(ArchiveError::Format(why.into()))
}
fn u16_at(b: &[u8], at: usize) -> Option<u16> {
Some(u16::from_le_bytes(b.get(at..at.checked_add(2)?)?.try_into().ok()?))
}
fn u32_at(b: &[u8], at: usize) -> Option<u32> {
Some(u32::from_le_bytes(b.get(at..at.checked_add(4)?)?.try_into().ok()?))
}
/// CRC-32 (IEEE) of `bytes`.
fn crc32(bytes: &[u8]) -> u32 {
let mut table = [0u32; 256];
for (i, t) in table.iter_mut().enumerate() {
let mut c = i as u32;
for _ in 0..8 {
c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 };
}
*t = c;
}
!bytes.iter().fold(!0u32, |c, &b| table[((c ^ u32::from(b)) & 0xff) as usize] ^ (c >> 8))
}
struct Entry {
method: u16,
crc: u32,
compressed: u64,
size: u64,
local_header: u64,
}
/// Find the entry called `name` (exactly, or as the last part of a longer name: `folder/name`) in the central directory.
fn find(f: &mut (impl Read + Seek), name: &str) -> Result<Entry, ArchiveError> {
let len = f.seek(SeekFrom::End(0))?;
let tail_len = len.min(EOCD_SEARCH);
f.seek(SeekFrom::Start(len - tail_len))?;
let mut tail = vec![0u8; tail_len as usize];
f.read_exact(&mut tail)?;
let eocd = (0..tail.len().saturating_sub(21)).rev().find(|&i| u32_at(&tail, i) == Some(0x0605_4b50));
let Some(e) = eocd else { return format("this is not a zip file") };
let (entries, size, offset) = (u16_at(&tail, e + 10), u32_at(&tail, e + 12), u32_at(&tail, e + 16));
let (Some(entries), Some(size), Some(offset)) = (entries, size, offset) else { return format("the end of the archive is cut short") };
if entries == 0xffff || size == u32::MAX || offset == u32::MAX {
return format("zip64 archives are not supported");
}
if u64::from(size) > MAX_DIRECTORY || u64::from(offset) + u64::from(size) > len {
return format("the list of files is not where it says");
}
f.seek(SeekFrom::Start(u64::from(offset)))?;
let mut dir = vec![0u8; size as usize];
f.read_exact(&mut dir)?;
let mut at = 0usize;
for _ in 0..entries {
if u32_at(&dir, at) != Some(0x0201_4b50) {
return format("the list of files is damaged");
}
let get = |o: usize| at.checked_add(o);
let (flags, method, crc) = (u16_at(&dir, at + 8), u16_at(&dir, at + 10), u32_at(&dir, at + 16));
let (compressed, uncompressed) = (u32_at(&dir, at + 20), u32_at(&dir, at + 24));
let (n, x, c) = (u16_at(&dir, at + 28), u16_at(&dir, at + 30), u16_at(&dir, at + 32));
let local = u32_at(&dir, at + 42);
let (Some(flags), Some(method), Some(crc), Some(compressed), Some(uncompressed), Some(n), Some(x), Some(c), Some(local), Some(start)) =
(flags, method, crc, compressed, uncompressed, n, x, c, local, get(46))
else {
return format("the list of files is cut short");
};
let Some(entry_name) = dir.get(start..start + usize::from(n)) else { return format("a file name is cut short") };
let matches = entry_name == name.as_bytes() || entry_name.strip_suffix(name.as_bytes()).is_some_and(|p| p.ends_with(b"/"));
if matches {
if flags & 1 != 0 {
return format("encrypted files are not supported");
}
if compressed == u32::MAX || uncompressed == u32::MAX {
return format("zip64 files are not supported");
}
return Ok(Entry { method, crc, compressed: u64::from(compressed), size: u64::from(uncompressed), local_header: u64::from(local) });
}
at = start + usize::from(n) + usize::from(x) + usize::from(c);
}
Err(ArchiveError::Missing(name.into()))
}
/// Take the file called `name` out of the archive at `archive` and write it to `dest` (through a temporary file next
/// to it); `max_bytes` bounds its size. Returns the number of bytes written.
pub fn extract(archive: &Path, name: &str, dest: &Path, max_bytes: u64) -> Result<u64, ArchiveError> {
let mut f = std::io::BufReader::new(std::fs::File::open(archive)?);
let e = find(&mut f, name)?;
if e.size > max_bytes || e.compressed > max_bytes {
return format("the file in the archive is larger than allowed");
}
f.seek(SeekFrom::Start(e.local_header))?;
let mut head = [0u8; 30];
f.read_exact(&mut head)?;
if u32_at(&head, 0) != Some(0x0403_4b50) {
return format("the file's header is not where it says");
}
let skip = u64::from(u16_at(&head, 26).unwrap_or(0)) + u64::from(u16_at(&head, 28).unwrap_or(0));
f.seek_relative(i64::try_from(skip).map_err(|_| ArchiveError::Format("a header is too long".into()))?)?;
let len = usize::try_from(e.compressed)
.ok()
.filter(|&n| n <= 256 * 1024 * 1024)
.ok_or_else(|| ArchiveError::Format("the compressed model is too large".into()))?;
if e.size > 256 * 1024 * 1024 {
return format("the extracted model is too large");
}
let mut packed = Vec::new();
packed.try_reserve_exact(len).map_err(|_| ArchiveError::Format("not enough memory for the model archive".into()))?;
packed.resize(len, 0u8);
f.read_exact(&mut packed)?;
let data = match e.method {
0 => packed,
8 => miniz_oxide::inflate::decompress_to_vec_with_limit(&packed, max_bytes.min(256 * 1024 * 1024) as usize)
.map_err(|err| ArchiveError::Format(format!("{err:?}")))?,
m => return format(&format!("compression method {m} is not supported")),
};
if data.len() as u64 != e.size || crc32(&data) != e.crc {
return format("the file in the archive is damaged");
}
let tmp = dest.with_extension("part");
let written = (|| -> std::io::Result<()> {
let mut out = std::fs::File::create(&tmp)?;
out.write_all(&data)?;
out.flush()?;
drop(out);
std::fs::rename(&tmp, dest)
})();
if let Err(err) = written {
let _ = std::fs::remove_file(&tmp);
return Err(err.into());
}
Ok(data.len() as u64)
}
#[cfg(test)]
mod tests {
use super::*;
/// A zip with the given (name, bytes, deflate) entries, written by hand.
pub(crate) fn make_zip(files: &[(&str, &[u8], bool)]) -> Vec<u8> {
let mut out = Vec::new();
let mut dir = Vec::new();
for (name, data, deflate) in files {
let packed = if *deflate { miniz_oxide::deflate::compress_to_vec(data, 6) } else { data.to_vec() };
let offset = out.len() as u32;
let method: u16 = if *deflate { 8 } else { 0 };
out.extend_from_slice(&0x0403_4b50u32.to_le_bytes());
out.extend_from_slice(&[20, 0, 0, 0]);
out.extend_from_slice(&method.to_le_bytes());
out.extend_from_slice(&[0, 0, 0, 0]);
out.extend_from_slice(&crc32(data).to_le_bytes());
out.extend_from_slice(&(packed.len() as u32).to_le_bytes());
out.extend_from_slice(&(data.len() as u32).to_le_bytes());
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
out.extend_from_slice(&[0, 0]);
out.extend_from_slice(name.as_bytes());
out.extend_from_slice(&packed);
dir.extend_from_slice(&0x0201_4b50u32.to_le_bytes());
dir.extend_from_slice(&[20, 0, 20, 0, 0, 0]);
dir.extend_from_slice(&method.to_le_bytes());
dir.extend_from_slice(&[0, 0, 0, 0]);
dir.extend_from_slice(&crc32(data).to_le_bytes());
dir.extend_from_slice(&(packed.len() as u32).to_le_bytes());
dir.extend_from_slice(&(data.len() as u32).to_le_bytes());
dir.extend_from_slice(&(name.len() as u16).to_le_bytes());
dir.extend_from_slice(&[0; 12]);
dir.extend_from_slice(&offset.to_le_bytes());
dir.extend_from_slice(name.as_bytes());
}
let cd_offset = out.len() as u32;
out.extend_from_slice(&dir);
out.extend_from_slice(&0x0605_4b50u32.to_le_bytes());
out.extend_from_slice(&[0, 0, 0, 0]);
out.extend_from_slice(&(files.len() as u16).to_le_bytes());
out.extend_from_slice(&(files.len() as u16).to_le_bytes());
out.extend_from_slice(&(dir.len() as u32).to_le_bytes());
out.extend_from_slice(&cd_offset.to_le_bytes());
out.extend_from_slice(&[0, 0]);
out
}
fn temp(name: &str) -> std::path::PathBuf {
std::env::temp_dir().join(format!("lc-denoise-zip-{name}-{}", std::process::id()))
}
#[test]
fn a_stored_and_a_deflated_file_come_out_whole() {
let big: Vec<u8> = (0..200_000u32).map(|i| (i * 7 % 251) as u8).collect();
let zip = make_zip(&[("m/config.json", b"{}", false), ("m/model.onnx", &big, true), ("m/other.onnx", b"zz", true)]);
let (a, out) = (temp("a.zip"), temp("a.onnx"));
std::fs::write(&a, &zip).unwrap();
assert_eq!(extract(&a, "model.onnx", &out, 1 << 20).unwrap(), big.len() as u64);
assert_eq!(std::fs::read(&out).unwrap(), big);
assert_eq!(extract(&a, "m/config.json", &out, 1 << 20).unwrap(), 2);
assert_eq!(std::fs::read(&out).unwrap(), b"{}");
// a name must match whole path parts: `del.onnx` is not `model.onnx`
assert!(matches!(extract(&a, "del.onnx", &out, 1 << 20), Err(ArchiveError::Missing(_))));
assert!(matches!(extract(&a, "nothing", &out, 1 << 20), Err(ArchiveError::Missing(_))));
// the size limit applies
assert!(matches!(extract(&a, "model.onnx", &out, 1000), Err(ArchiveError::Format(_))));
for p in [a, out] {
let _ = std::fs::remove_file(p);
}
}
/// Opt-in: `LC_DENOISE_DTMODEL=<rawdenoise-nind.dtmodel> cargo test -p lightcraft-denoise -- --ignored archive`
#[test]
#[ignore = "needs the real download: set LC_DENOISE_DTMODEL"]
fn the_real_download_gives_the_model_it_should() {
let Some(path) = std::env::var_os("LC_DENOISE_DTMODEL") else { return };
let known = crate::known::find("rawnind-bayer").unwrap();
let (d, entry) = (known.download.unwrap(), known.entry.unwrap());
assert_eq!(crate::hash::sha256_file(Path::new(&path)).unwrap(), d.sha256);
let out = temp("real.onnx");
let n = extract(Path::new(&path), entry, &out, crate::manifest::MAX_MODEL_BYTES).unwrap();
assert_eq!(Some(n), known.manifest.size_bytes);
assert_eq!(Some(crate::hash::sha256_file(&out).unwrap()), known.manifest.sha256);
let _ = std::fs::remove_file(out);
}
#[test]
fn damaged_and_hostile_archives_are_errors_not_panics() {
let zip = make_zip(&[("m/model.onnx", &vec![9u8; 5000], true)]);
let (a, out) = (temp("b.zip"), temp("b.onnx"));
// cut anywhere
for n in (0..zip.len()).step_by(3) {
std::fs::write(&a, &zip[..n]).unwrap();
assert!(extract(&a, "model.onnx", &out, 1 << 20).is_err(), "cut at {n}");
}
// one byte changed anywhere: an error, or (when it falls in an unused field) the right file, never a panic
for at in 0..zip.len() {
let mut bad = zip.clone();
bad[at] ^= 0x5a;
std::fs::write(&a, &bad).unwrap();
if extract(&a, "model.onnx", &out, 1 << 20).is_ok() {
assert_eq!(std::fs::read(&out).unwrap(), vec![9u8; 5000], "a changed byte at {at} gave another file");
}
}
// not a zip at all, and an empty file
for junk in [Vec::new(), b"PK".to_vec(), vec![0u8; 100_000]] {
std::fs::write(&a, junk).unwrap();
assert!(extract(&a, "model.onnx", &out, 1 << 20).is_err());
}
assert!(extract(&temp("missing.zip"), "model.onnx", &out, 1 << 20).is_err());
for p in [a, out] {
let _ = std::fs::remove_file(p);
}
}
}
+191
View File
@@ -0,0 +1,191 @@
//! Bayer geometry: which way a mosaic is laid out, and how a patch of it is packed into the four planes
//! (red, green on the red row, green on the blue row, blue) a denoise model takes.
//!
//! Every Bayer layout is the same 2 × 2 cell shifted by one pixel at most, so a layout is just where its
//! first red sample is. The cell grid starts one pixel before the picture when that red sample is in
//! column or row 1, and the picture is read with its edge mirrored (without repeating the edge sample, so
//! the colour of a mirrored sample is the colour that belongs at that place).
/// Where the first red sample of a Bayer mosaic sits, `(dx, dy)` in `{0, 1}²`: RGGB is `(0, 0)`, GRBG `(1, 0)`,
/// GBRG `(0, 1)`, BGGR `(1, 1)`.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Layout {
pub dx: usize,
pub dy: usize,
}
impl Layout {
/// The layout of a 2 × 2 colour pattern, row-major, colours 0 = red, 1 = green, 2 = blue; `None` when
/// it is not a Bayer cell (one red, one blue, the greens on the other diagonal).
pub fn from_cell(cell: [u8; 4]) -> Option<Layout> {
let at = |c: u8| cell.iter().position(|&v| v == c);
let red = at(0)?;
let blue = at(2)?;
let greens = cell.iter().filter(|&&v| v == 1).count();
// red and blue sit on one diagonal: indices 0+3 or 1+2
if greens != 2 || red + blue != 3 {
return None;
}
Some(Layout { dx: red % 2, dy: red / 2 })
}
/// The cell grid's origin: `-1` along an axis whose first red sample is at 1, so that cell 0 starts on a red column or row.
fn origin(self) -> (isize, isize) {
(isize::try_from(self.dx).unwrap_or(isize::MAX).saturating_neg(), isize::try_from(self.dy).unwrap_or(isize::MAX).saturating_neg())
}
/// Cells across and down covering a `width × height` mosaic.
pub fn cells(self, width: usize, height: usize) -> (usize, usize) {
let cells = |n: usize, phase: usize| n / 2 + (n % 2 + phase.min(1)).div_ceil(2);
(cells(width, self.dx), cells(height, self.dy))
}
/// The mosaic position of cell `(cx, cy)`'s top-left sample (may lie outside the picture).
pub fn position(self, cx: isize, cy: isize) -> (isize, isize) {
let (ox, oy) = self.origin();
(ox.saturating_add(cx.saturating_mul(2)), oy.saturating_add(cy.saturating_mul(2)))
}
}
/// `i` mirrored into `0..n` (ping-pong, edge not repeated), so `-1` is `1` and `n` is `n - 2`. `n == 0` gives 0.
pub fn reflect(i: isize, n: usize) -> usize {
if n <= 1 {
return 0;
}
if let Some(period) = isize::try_from(n).ok().and_then(|n| (n - 1).checked_mul(2)) {
let m = i.rem_euclid(period);
return (if m < n as isize { m } else { period - m }) as usize;
}
let period = 2 * (n as i128 - 1);
let m = (i as i128).rem_euclid(period);
// The reflected value is in 0..n, so this conversion always fits usize.
(if m < n as i128 { m } else { period - m }) as usize
}
/// Pack the `tile × tile` cells whose top-left cell is `(cx0, cy0)` into `out` as four planes `[R, G1, G2, B]`
/// (`4 · tile²` values, NCHW). G1 is the green beside the red sample, G2 the green under it. Samples outside
/// the picture are mirrored. Returns `false` (and writes nothing) when the sizes do not fit.
pub fn pack_tile(mosaic: &[f32], width: usize, height: usize, layout: Layout, cx0: isize, cy0: isize, tile: usize, out: &mut [f32]) -> bool {
let Some(plane) = tile.checked_mul(tile) else { return false };
let Some(values) = plane.checked_mul(4) else { return false };
if width == 0
|| height == 0
|| tile == 0
|| layout.dx > 1
|| layout.dy > 1
|| width.checked_mul(height) != Some(mosaic.len())
|| out.len() != values
{
return false;
}
let Ok(last) = isize::try_from(tile - 1) else { return false };
let (ox, oy) = layout.origin();
for (start, origin) in [(cx0, ox), (cy0, oy)] {
if start.checked_mul(2).and_then(|v| v.checked_add(origin)).is_none()
|| start.checked_add(last).and_then(|v| v.checked_mul(2)).and_then(|v| v.checked_add(origin)).and_then(|v| v.checked_add(1)).is_none()
{
return false;
}
}
// the four samples of a cell, as (dx, dy) from its top-left sample
const SUB: [(isize, isize); 4] = [(0, 0), (1, 0), (0, 1), (1, 1)];
for (p, &(sx, sy)) in SUB.iter().enumerate() {
let Some(dst) = out.get_mut(p * plane..(p + 1) * plane) else { return false };
for (j, row) in dst.chunks_exact_mut(tile).enumerate() {
let (_, y) = layout.position(cx0, cy0 + j as isize);
let ry = reflect(y + sy, height);
let Some(src) = mosaic.get(ry * width..(ry + 1) * width) else { return false };
for (i, v) in row.iter_mut().enumerate() {
let (x, _) = layout.position(cx0 + i as isize, cy0);
*v = src.get(reflect(x + sx, width)).copied().unwrap_or(0.0);
}
}
}
true
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn hostile_sizes_and_positions_never_panic_or_write_output() {
let layout = Layout::from_cell([0, 1, 1, 2]).unwrap();
let mut out = [7.0; 4];
for (w, h, tile, x) in [(usize::MAX, 2, 1, 0), (1, 1, usize::MAX, 0), (1, 1, 0, 0), (1, 1, 1, isize::MAX), (1, 1, 1, isize::MIN)] {
assert!(!pack_tile(&[0.5], w, h, layout, x, 0, tile, &mut out));
assert_eq!(out, [7.0; 4]);
}
for n in [1, 2, isize::MAX as usize, usize::MAX] {
for i in [isize::MIN, -1, 0, isize::MAX] {
assert!(reflect(i, n) < n);
}
}
assert_eq!(layout.cells(usize::MAX, usize::MAX), (usize::MAX / 2 + 1, usize::MAX / 2 + 1));
}
#[test]
fn layouts_from_cells() {
assert_eq!(Layout::from_cell([0, 1, 1, 2]), Some(Layout { dx: 0, dy: 0 }));
assert_eq!(Layout::from_cell([1, 0, 2, 1]), Some(Layout { dx: 1, dy: 0 }));
assert_eq!(Layout::from_cell([1, 2, 0, 1]), Some(Layout { dx: 0, dy: 1 }));
assert_eq!(Layout::from_cell([2, 1, 1, 0]), Some(Layout { dx: 1, dy: 1 }));
// not Bayer: two reds, a missing blue, greens on the wrong diagonal
assert_eq!(Layout::from_cell([0, 0, 1, 2]), None);
assert_eq!(Layout::from_cell([0, 1, 2, 1]), None);
assert_eq!(Layout::from_cell([1, 1, 1, 1]), None);
}
#[test]
fn reflection_keeps_the_colour_of_a_site() {
assert_eq!(reflect(-1, 10), 1);
assert_eq!(reflect(-2, 10), 2);
assert_eq!(reflect(10, 10), 8);
assert_eq!(reflect(11, 10), 7);
assert_eq!(reflect(25, 10), 7);
assert_eq!(reflect(5, 0), 0);
assert_eq!(reflect(5, 1), 0);
for i in -40..60 {
assert_eq!(reflect(i, 11) % 2, (i.rem_euclid(2)) as usize, "parity at {i}");
}
}
#[test]
fn packing_puts_each_colour_in_its_plane_for_every_layout() {
// a mosaic whose sample value says which colour it is: R = 10, G = 20, B = 30
for cell in [[0u8, 1, 1, 2], [1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]] {
let layout = Layout::from_cell(cell).unwrap();
let (w, h) = (10usize, 8usize);
let mosaic: Vec<f32> = (0..w * h).map(|i| [10.0, 20.0, 30.0][cell[(i / w % 2) * 2 + i % w % 2] as usize]).collect();
let (nx, ny) = layout.cells(w, h);
let tile = nx.max(ny);
let mut out = vec![0.0; 4 * tile * tile];
assert!(pack_tile(&mosaic, w, h, layout, 0, 0, tile, &mut out));
let plane = tile * tile;
for (p, want) in [10.0, 20.0, 20.0, 30.0].into_iter().enumerate() {
assert!(out[p * plane..(p + 1) * plane].iter().all(|&v| v == want), "layout {cell:?} plane {p}");
}
}
}
#[test]
fn g1_is_beside_the_red_sample_and_g2_under_it() {
// RGGB with a distinct value at every site
let (w, h) = (4, 4);
let mosaic: Vec<f32> = (0..16).map(|i| i as f32).collect();
let mut out = vec![0.0; 4 * 4];
assert!(pack_tile(&mosaic, w, h, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out));
// cell (0,0): R = site (0,0)=0, G1 = site (1,0)=1, G2 = site (0,1)=4, B = site (1,1)=5
assert_eq!([out[0], out[4], out[8], out[12]], [0.0, 1.0, 4.0, 5.0]);
// cell (1,0): sites (2,0)=2, (3,0)=3, (2,1)=6, (3,1)=7
assert_eq!([out[1], out[5], out[9], out[13]], [2.0, 3.0, 6.0, 7.0]);
}
#[test]
fn packing_rejects_wrong_sizes() {
let mut out = vec![0.0; 16];
assert!(!pack_tile(&[0.0; 15], 4, 4, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out));
assert!(!pack_tile(&[0.0; 16], 4, 4, Layout { dx: 0, dy: 0 }, 0, 0, 3, &mut out));
assert!(!pack_tile(&[], 0, 0, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out));
}
}
+46
View File
@@ -0,0 +1,46 @@
//! SHA-256 of a model file, streamed (a model can be hundreds of megabytes).
use std::io::{self, Read};
use std::path::Path;
use sha2::{Digest, Sha256};
/// Lowercase hex SHA-256 of everything `r` yields.
pub fn sha256_reader<R: Read>(mut r: R) -> io::Result<String> {
let mut h = Sha256::new();
let mut buf = vec![0u8; 1 << 20];
loop {
let n = r.read(&mut buf)?;
if n == 0 {
break;
}
h.update(buf.get(..n).unwrap_or(&[]));
}
Ok(hex(&h.finalize()))
}
/// Lowercase hex SHA-256 of a file.
pub fn sha256_file(path: &Path) -> io::Result<String> {
sha256_reader(io::BufReader::new(std::fs::File::open(path)?))
}
/// Lowercase hex SHA-256 of `bytes`.
pub fn sha256_hex(bytes: &[u8]) -> String {
hex(&Sha256::digest(bytes))
}
fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn known_digests() {
assert_eq!(sha256_hex(b""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
assert_eq!(sha256_hex(b"abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
assert_eq!(sha256_reader(&b"abc"[..]).unwrap(), sha256_hex(b"abc"));
}
}
+128
View File
@@ -0,0 +1,128 @@
//! The denoise models LightCraft knows about: what each one is, who may use it, and where to fetch it.
//!
//! None is part of LightCraft. A model with an address here can be downloaded when the user asks and has accepted
//! its terms; anything else is brought by the user as an `.onnx` file with a `denoise-model.json` beside it.
#[cfg(any(feature = "rawnind-model", test))]
use crate::licence::{Commercial, Licence};
use crate::manifest::DenoiserManifest;
#[cfg(any(feature = "rawnind-model", test))]
use crate::manifest::{Domain, Gain};
/// A known model, and how to get it.
pub struct Known {
pub manifest: DenoiserManifest,
/// What to fetch (an archive or the `.onnx` itself); `None` when the user must bring the file.
pub download: Option<Download>,
/// The `.onnx` inside the downloaded archive (`None`: the download is the `.onnx`).
pub entry: Option<&'static str>,
}
/// RawNIND's UtNet2 for Bayer sensors, as published for darktable (`rawdenoise-nind.dtmodel`, a zip with a Bayer and a
/// linear model). Its weights are GPL-3.0; the training pictures are CC BY 4.0 and CC0.
#[cfg(any(feature = "rawnind-model", test))]
fn rawnind_bayer() -> Known {
let manifest = DenoiserManifest {
id: "rawnind-bayer".into(),
name: "RawNIND UtNet2 (Bayer)".into(),
version: "1.0".into(),
licence: Licence {
name: "GPL-3.0".into(),
commercial: Commercial::Yes,
url: Some("https://www.gnu.org/licenses/gpl-3.0.html".into()),
notice: "The weights are published under the GNU General Public License v3. LightCraft does not include them: they \
are downloaded to your computer at your request and used by it only. If you pass the model file on, the \
GPL applies to that copy. Commercial use is subject to the GPL terms, including applicable source, licence and notice obligations on redistribution."
.into(),
},
source: Some("https://github.com/darktable-org/darktable-ai/tree/master/models/rawdenoise-nind".into()),
sha256: Some("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901".into()),
size_bytes: Some(31_056_425),
provenance: "Trained on RawNIND, real photographs of the same scenes taken noisy and clean (images CC BY 4.0 and CC0, \
published on Wikimedia Commons and by the University of Louvain). Paper: arXiv 2501.08924."
.into(),
domain: Domain::BayerToRgb,
tile: 512,
overlap: 64,
gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },
};
let download = Download {
id: manifest.id.clone(),
url: "https://github.com/darktable-org/darktable-ai/releases/download/release-5.6.0/rawdenoise-nind.dtmodel".into(),
file_name: "rawdenoise-nind.dtmodel".into(),
size_bytes: 57_700_134,
sha256: "d71b5f1e727c85a359e6f74dca9e2016c9d8fc3e2f7ac3e9b347d80ceca969af".into(),
};
Known { manifest, download: Some(download), entry: Some("model_bayer.onnx") }
}
/// Every model LightCraft knows.
pub fn all() -> Vec<Known> {
#[cfg(feature = "rawnind-model")]
{
vec![rawnind_bayer()]
}
#[cfg(not(feature = "rawnind-model"))]
{
Vec::new()
}
}
/// The known model with this id.
pub fn find(id: &str) -> Option<Known> {
all().into_iter().find(|k| k.manifest.id == id)
}
/// The known model whose `.onnx` has this SHA-256.
pub fn by_sha256(sha256: &str) -> Option<DenoiserManifest> {
all().into_iter().map(|k| k.manifest).find(|m| m.sha256.as_deref() == Some(sha256))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::manifest::validate;
#[test]
fn known_models_are_valid_unique_and_downloaded_over_https() {
let all = all();
assert_eq!(all.is_empty(), !cfg!(feature = "rawnind-model"));
validate(&rawnind_bayer().manifest).unwrap();
for (i, k) in all.iter().enumerate() {
validate(&k.manifest).unwrap();
assert!(all.iter().skip(i + 1).all(|o| o.manifest.id != k.manifest.id), "unique ids");
assert!(!k.manifest.licence.name.is_empty() && !k.manifest.licence.notice.is_empty(), "terms are shown before use");
if let Some(d) = &k.download {
assert_eq!(d.id, k.manifest.id);
assert!(d.url.starts_with("https://") && d.sha256.len() == 64 && d.size_bytes > 0);
// a name that cannot leave the staging folder
assert!(!d.file_name.contains(['/', '\\']) && !d.file_name.starts_with('.'));
}
// a model that may not be used commercially is never offered for download
if k.manifest.licence.commercial == Commercial::No {
assert!(k.download.is_none(), "{}", k.manifest.id);
}
}
assert_eq!(find("rawnind-bayer").is_some(), cfg!(feature = "rawnind-model"));
assert!(find("nothing").is_none());
assert_eq!(
by_sha256("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901").map(|m| m.id),
if cfg!(feature = "rawnind-model") { Some("rawnind-bayer".into()) } else { None }
);
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct Download {
pub id: String,
pub url: String,
/// The name the file is kept under while it waits for the user to accept its terms.
pub file_name: String,
pub size_bytes: u64,
pub sha256: String,
}
pub fn host(url: &str) -> &str {
url.strip_prefix("https://").and_then(|r| r.split('/').next()).unwrap_or("")
}
+16
View File
@@ -0,0 +1,16 @@
//! Model metadata, bounded archive/cache formats and Bayer tiling, independent of inference.
//! Used by featureless engine and web builds; no neural-network or matrix backend is linked.
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
pub mod archive;
pub mod bayer;
pub mod hash;
pub mod known;
pub mod licence;
pub mod manifest;
pub mod product;
pub mod run;
pub mod tiles;
pub use manifest::{DenoiserManifest, Domain, Gain, ManifestError};
pub use run::{Control, Error, Params, TileRunner, denoise_bayer};
+34
View File
@@ -0,0 +1,34 @@
use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum Commercial {
Yes,
/// Research or personal use only. Never bundled or redistributed by LightCraft.
No,
/// The licence or the training data is unclear: the user is told so before installing.
Unknown,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(default, rename_all = "camelCase")]
pub struct Licence {
/// SPDX id or a short name ("MIT", "Apache-2.0", "InsightFace non-commercial research").
pub name: String,
pub commercial: Commercial,
pub url: Option<String>,
/// Plain-language terms shown before the model is enabled.
pub notice: String,
}
impl Default for Licence {
fn default() -> Self {
Self { name: String::new(), commercial: Commercial::Unknown, url: None, notice: String::new() }
}
}
pub fn valid_id(id: &str) -> bool {
let mut bytes = id.bytes();
let first_ok = bytes.next().is_some_and(|b| b.is_ascii_lowercase() || b.is_ascii_digit());
first_ok && id.len() <= 64 && bytes.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || matches!(b, b'.' | b'_' | b'-'))
}
+217
View File
@@ -0,0 +1,217 @@
//! A denoise model's manifest: what the model takes and gives, how big its tiles are and who may use it, as
//! plain data.
//!
//! It comes from a `denoise-model.json` next to the `.onnx` file or from the user's `catalog.json`. Both are
//! untrusted: [`validate`] must pass before a manifest is used, and [`parse`] applies it after reading JSON of
//! bounded size. The licence, id and address rules are the face models' (`crate::licence`).
use crate::licence::{Licence, valid_id};
use serde::{Deserialize, Serialize};
/// Largest `denoise-model.json` we read.
pub const MAX_MANIFEST_BYTES: usize = 64 * 1024;
/// Largest model file we accept (bytes).
pub const MAX_MODEL_BYTES: u64 = 2 * 1024 * 1024 * 1024;
/// What the model takes and what it gives.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum Domain {
/// The sensor's Bayer mosaic (black subtracted, white at 1, not white balanced) packed as four planes
/// `[R, G1, G2, B]` of its 2 × 2 cells, forced to RGGB; gives camera RGB at the mosaic's resolution, so
/// the picture is denoised and demosaiced in one step and must not be demosaiced again.
BayerToRgb,
}
/// How the model's output scale is brought back to the input's.
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "camelCase")]
pub enum Gain {
/// The output already has the input's scale.
None,
/// The output has a scale of its own, `nominal` times the input's (a million, for one model) and a little
/// different from tile to tile: each tile is scaled so that its channel means equal the input's. A tile too
/// dark to measure, or whose measured scale is further than `maxDeviation` (a fraction, 0.05 = 5 %) from
/// `nominal`, is scaled as if it were within that limit.
MatchMean { nominal: f32, max_deviation: f32 },
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DenoiserManifest {
/// Lowercase letters, digits, `.` `_` `-`.
pub id: String,
pub name: String,
pub version: String,
#[serde(default)]
pub licence: Licence,
/// Where the weights come from (an `https://` page or file), for the user's information.
#[serde(default)]
pub source: Option<String>,
/// Lowercase hex SHA-256 of the `.onnx` file, when known.
#[serde(default)]
pub sha256: Option<String>,
#[serde(default)]
pub size_bytes: Option<u64>,
/// What the model was trained on, in plain words; says "undisclosed" when it is.
#[serde(default)]
pub provenance: String,
pub domain: Domain,
/// Side of the square the model runs on, in packed cells (the file's fixed input size).
pub tile: u32,
/// Cells neighbouring tiles share; the picture is blended across them.
pub overlap: u32,
pub gain: Gain,
}
#[derive(Debug, PartialEq, thiserror::Error)]
pub enum ManifestError {
#[error("`{0}` is not valid: {1}")]
Field(&'static str, String),
#[error("not a denoise-model manifest: {0}")]
Parse(String),
#[error("a manifest may be at most {MAX_MANIFEST_BYTES} bytes")]
TooLarge,
}
fn bad<T>(field: &'static str, why: impl Into<String>) -> Result<T, ManifestError> {
Err(ManifestError::Field(field, why.into()))
}
fn text(field: &'static str, s: &str, min: usize, max: usize) -> Result<(), ManifestError> {
let n = s.chars().count();
if n < min || n > max {
return bad(field, format!("must be {min} to {max} characters"));
}
if s.chars().any(|c| c.is_control() && c != '\n') {
return bad(field, "must not contain control characters");
}
Ok(())
}
fn url(field: &'static str, u: &Option<String>) -> Result<(), ManifestError> {
match u {
None => Ok(()),
Some(u) if u.len() <= 500 && u.starts_with("https://") && !u.chars().any(|c| c.is_control() || c.is_whitespace()) => Ok(()),
Some(_) => bad(field, "must be an https:// address of at most 500 characters"),
}
}
/// Check a manifest from outside before trusting it.
pub fn validate(m: &DenoiserManifest) -> Result<(), ManifestError> {
if !valid_id(&m.id) {
return bad("id", "1 to 64 lowercase letters, digits, `.`, `_` or `-`, starting with a letter or digit");
}
text("name", &m.name, 1, 80)?;
text("version", &m.version, 1, 40)?;
text("provenance", &m.provenance, 0, 1000)?;
text("licence.name", &m.licence.name, 0, 80)?;
text("licence.notice", &m.licence.notice, 0, 2000)?;
url("licence.url", &m.licence.url)?;
url("source", &m.source)?;
if let Some(h) = &m.sha256
&& !(h.len() == 64 && h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))
{
return bad("sha256", "must be 64 lowercase hex digits");
}
if let Some(n) = m.size_bytes
&& (n == 0 || n > MAX_MODEL_BYTES)
{
return bad("sizeBytes", format!("must be between 1 and {MAX_MODEL_BYTES}"));
}
// a U-Net with four poolings wants a multiple of 16; nothing smaller than 64 or larger than 2048 is a tile
if !(64..=2048).contains(&m.tile) || !m.tile.is_multiple_of(16) {
return bad("tile", "must be a multiple of 16 from 64 to 2048");
}
// tiles must overlap by less than half, or one tile's blend ramps would run into its other neighbour's
if !m.overlap.is_multiple_of(2) || m.overlap < 16 || m.overlap.saturating_mul(2) >= m.tile {
return bad("overlap", "must be even, at least 16 and less than half the tile");
}
if let Gain::MatchMean { nominal, max_deviation } = m.gain {
if !(nominal.is_finite() && (1e-6..=1e12).contains(&nominal)) {
return bad("gain.nominal", "must be between 0.000001 and 1000000000000");
}
if !(max_deviation.is_finite() && (0.0..=0.5).contains(&max_deviation)) {
return bad("gain.maxDeviation", "must be between 0 and 0.5");
}
}
Ok(())
}
/// Read and validate a `denoise-model.json`.
pub fn parse(bytes: &[u8]) -> Result<DenoiserManifest, ManifestError> {
if bytes.len() > MAX_MANIFEST_BYTES {
return Err(ManifestError::TooLarge);
}
let bytes = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]).unwrap_or(bytes);
let m: DenoiserManifest = serde_json::from_slice(bytes).map_err(|e| ManifestError::Parse(e.to_string()))?;
validate(&m)?;
Ok(m)
}
#[cfg(test)]
mod tests {
use super::*;
pub(crate) fn sample() -> DenoiserManifest {
DenoiserManifest {
id: "nind-bayer".into(),
name: "Bayer denoiser".into(),
version: "1".into(),
licence: Licence::default(),
source: Some("https://example.org/model.onnx".into()),
sha256: None,
size_bytes: Some(31_000_000),
provenance: "trained on real raw noise pairs".into(),
domain: Domain::BayerToRgb,
tile: 512,
overlap: 64,
gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },
}
}
#[test]
fn a_good_manifest_round_trips() {
let m = sample();
let json = serde_json::to_vec(&m).unwrap();
assert_eq!(parse(&json).unwrap(), m);
}
#[test]
fn bad_fields_are_named() {
let cases: Vec<(&str, Box<dyn Fn(&mut DenoiserManifest)>)> = vec![
("id", Box::new(|m| m.id = "../evil".into())),
("id", Box::new(|m| m.id = String::new())),
("name", Box::new(|m| m.name = "x\u{7}".into())),
("source", Box::new(|m| m.source = Some("http://insecure".into()))),
("sha256", Box::new(|m| m.sha256 = Some("XYZ".into()))),
("sizeBytes", Box::new(|m| m.size_bytes = Some(0))),
("tile", Box::new(|m| m.tile = 500)),
("tile", Box::new(|m| m.tile = 4096)),
("overlap", Box::new(|m| m.overlap = 300)),
("overlap", Box::new(|m| m.overlap = 63)),
("gain.maxDeviation", Box::new(|m| m.gain = Gain::MatchMean { nominal: 1.0, max_deviation: f32::NAN })),
("gain.maxDeviation", Box::new(|m| m.gain = Gain::MatchMean { nominal: 1.0, max_deviation: 2.0 })),
("gain.nominal", Box::new(|m| m.gain = Gain::MatchMean { nominal: 0.0, max_deviation: 0.1 })),
("gain.nominal", Box::new(|m| m.gain = Gain::MatchMean { nominal: f32::INFINITY, max_deviation: 0.1 })),
];
for (field, edit) in cases {
let mut m = sample();
edit(&mut m);
match validate(&m) {
Err(ManifestError::Field(f, _)) => assert_eq!(f, field),
other => panic!("{field}: {other:?}"),
}
}
}
#[test]
fn parse_is_bounded_and_tolerant() {
assert_eq!(parse(&vec![b' '; MAX_MANIFEST_BYTES + 1]), Err(ManifestError::TooLarge));
assert!(matches!(parse(b"{ nope"), Err(ManifestError::Parse(_))));
assert!(matches!(parse(b""), Err(ManifestError::Parse(_))));
let mut with_bom = vec![0xEF, 0xBB, 0xBF];
with_bom.extend(serde_json::to_vec(&sample()).unwrap());
assert!(parse(&with_bom).is_ok());
}
}
+616
View File
@@ -0,0 +1,616 @@
//! The denoised picture as cached data, and the blend the Amount slider controls.
//!
//! A product is camera RGB at the sensor's resolution (what the model gave, before the raw's own opcodes and
//! crop), kept as half floats in strips of rows, each strip byte-shuffled and deflated with its own checksum.
//! It is derived data: any problem reading it (missing, truncated, corrupt, made for another photo or model) is
//! just a cache miss, never a panic, and the picture can be made again.
//!
//! File: `LCDN`, version, header length, header JSON (`width`, `height`, `strip`, `key`), then for each strip its
//! compressed length and CRC-32, then the strips' bytes. `key` says what the product was made from (the photo's
//! content, the model, this code's version, the settings it depends on); a file with another key is stale.
use std::io::{Read, Write};
use std::path::Path;
use lightcraft_raster::Rgb32f;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
const MAGIC: &[u8; 4] = b"LCDN";
const VERSION: u8 = 1;
const MAX_HEADER: usize = 4096;
const MAX_SIDE: usize = 1 << 17;
/// Most pixels a product may have (the denoiser's own limit, [`crate::run::MAX_PIXELS`]).
const MAX_PIXELS: usize = crate::run::MAX_PIXELS;
/// Rows per strip: about a megapixel of three planes at 24 MP widths.
const STRIP_ROWS: usize = 128;
/// Bumped when the meaning of a product changes (included in keys by the caller).
pub const ALGORITHM: u32 = 1;
#[derive(Debug, thiserror::Error)]
pub enum ProductError {
#[error("the denoised picture could not be read or written: {0}")]
Io(String),
#[error("not a usable denoised picture: {0}")]
Format(String),
/// A product made from something else (another photo state, model or version).
#[error("the denoised picture is out of date")]
Stale,
}
impl From<std::io::Error> for ProductError {
fn from(e: std::io::Error) -> Self {
ProductError::Io(e.to_string())
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Header {
pub width: usize,
pub height: usize,
pub strip: usize,
pub key: String,
}
fn allocate<T: Clone>(n: usize, value: T) -> Result<Vec<T>, ProductError> {
let mut v = Vec::new();
v.try_reserve_exact(n).map_err(|_| ProductError::Format("not enough memory for the denoised picture".into()))?;
v.resize(n, value);
Ok(v)
}
/// `f32` as IEEE half-float bits, rounded to nearest even; beyond the range it saturates, NaN becomes 0.
pub fn to_f16(v: f32) -> u16 {
if !v.is_finite() {
return if v.is_nan() {
0
} else if v > 0.0 {
0x7bff
} else {
0xfbff
};
}
let b = v.to_bits();
let sign = ((b >> 16) & 0x8000) as u16;
let exp = ((b >> 23) & 0xff) as i32 - 127 + 15;
let man = b & 0x7f_ffff;
if exp >= 31 {
return sign | 0x7bff;
}
if exp <= 0 {
if exp < -10 {
return sign;
}
let man = man | 0x80_0000;
let shift = (14 - exp) as u32;
let (half, rem, halfway) = (man >> shift, man & ((1u32 << shift) - 1), 1u32 << (shift - 1));
let mut h = half as u16;
if rem > halfway || (rem == halfway && h & 1 == 1) {
h += 1;
}
return sign | h;
}
let mut h = (((exp as u32) << 10) | (man >> 13)) as u16;
let rem = man & 0x1fff;
if rem > 0x1000 || (rem == 0x1000 && h & 1 == 1) {
h += 1;
}
if h >= 0x7c00 {
h = 0x7bff;
}
sign | h
}
/// Half-float bits as `f32`.
pub fn from_f16(h: u16) -> f32 {
let sign = if h & 0x8000 != 0 { -1.0f32 } else { 1.0 };
let exp = i32::from((h >> 10) & 0x1f);
let man = f32::from(h & 0x3ff);
match exp {
0 => sign * man * 2f32.powi(-24),
31 => sign * 65504.0,
_ => sign * (1.0 + man / 1024.0) * 2f32.powi(exp - 15),
}
}
fn crc(bytes: &[u8]) -> u32 {
// CRC-32 (IEEE), bitwise: strips are small and this runs once per strip
let mut table = [0u32; 256];
for (i, t) in table.iter_mut().enumerate() {
let mut c = i as u32;
for _ in 0..8 {
c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 };
}
*t = c;
}
!bytes.iter().fold(!0u32, |c, &b| table[((c ^ u32::from(b)) & 0xff) as usize] ^ (c >> 8))
}
/// One strip as bytes: for each plane (R, G, B) the low bytes of its half floats, then the high bytes.
fn encode_strip(img: &Rgb32f, y0: usize, y1: usize) -> Result<Vec<u8>, ProductError> {
let n = (y1 - y0) * img.width;
let mut raw = allocate(6 * n, 0u8)?;
let rows = img.data.get(y0 * img.width..y1 * img.width).unwrap_or(&[]);
for (i, px) in rows.iter().enumerate() {
for (c, &v) in px.iter().enumerate() {
let h = to_f16(v);
if let Some(lo) = raw.get_mut(2 * c * n + i) {
*lo = h as u8;
}
if let Some(hi) = raw.get_mut((2 * c + 1) * n + i) {
*hi = (h >> 8) as u8;
}
}
}
Ok(miniz_oxide::deflate::compress_to_vec(&raw, 3))
}
/// Write `img` to `path` (through a temporary file next to it, so a reader never sees half a product).
pub fn write(path: &Path, img: &Rgb32f, key: &str) -> Result<(), ProductError> {
let pixels = img.width.checked_mul(img.height).filter(|&n| n <= MAX_PIXELS);
if img.width == 0 || img.height == 0 || img.width > MAX_SIDE || img.height > MAX_SIDE || pixels != Some(img.data.len()) {
return Err(ProductError::Format("the picture has no usable size".into()));
}
let strips: Vec<(usize, usize)> = (0..img.height).step_by(STRIP_ROWS).map(|y| (y, (y + STRIP_ROWS).min(img.height))).collect();
let packed: Vec<Vec<u8>> = strips.par_iter().map(|&(a, b)| encode_strip(img, a, b)).collect::<Result<_, _>>()?;
let header = serde_json::to_vec(&Header { width: img.width, height: img.height, strip: STRIP_ROWS, key: key.to_string() })
.map_err(|e| ProductError::Format(e.to_string()))?;
let tmp = path.with_extension("tmp");
let result = (|| -> Result<(), ProductError> {
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir)?;
}
let mut f = std::io::BufWriter::new(std::fs::File::create(&tmp)?);
f.write_all(MAGIC)?;
f.write_all(&[VERSION, 0, 0, 0])?;
f.write_all(&(header.len() as u32).to_le_bytes())?;
f.write_all(&header)?;
for p in &packed {
f.write_all(&(p.len() as u32).to_le_bytes())?;
f.write_all(&crc(p).to_le_bytes())?;
}
for p in &packed {
f.write_all(p)?;
}
f.flush()?;
drop(f);
std::fs::rename(&tmp, path)?;
Ok(())
})();
if result.is_err() {
let _ = std::fs::remove_file(&tmp);
}
result
}
/// The parsed start of a product file: header, each strip's (compressed length, CRC-32), and the file length the
/// start says the whole file has.
struct Start {
header: Header,
table: Vec<(usize, u32)>,
expected_len: u64,
}
fn read_start(r: &mut impl Read) -> Result<Start, ProductError> {
let mut fixed = [0u8; 12];
r.read_exact(&mut fixed)?;
if fixed.get(0..4) != Some(&MAGIC[..]) || fixed.get(4) != Some(&VERSION) {
return Err(ProductError::Format("not a denoised picture of this version".into()));
}
let len = fixed.get(8..12).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(u32::from_le_bytes).unwrap_or(u32::MAX) as usize;
if len > MAX_HEADER {
return Err(ProductError::Format("the header is too large".into()));
}
let mut hb = allocate(len, 0u8)?;
r.read_exact(&mut hb)?;
let h: Header = serde_json::from_slice(&hb).map_err(|e| ProductError::Format(e.to_string()))?;
let pixels = h.width.saturating_mul(h.height);
if h.width == 0 || h.height == 0 || h.width > MAX_SIDE || h.height > MAX_SIDE || pixels > MAX_PIXELS || h.strip != STRIP_ROWS {
return Err(ProductError::Format("the size is not usable".into()));
}
let n = h.height.div_ceil(h.strip);
let mut table = Vec::with_capacity(n);
let mut total = 12 + len as u64 + 8 * n as u64;
for _ in 0..n {
let mut e = [0u8; 8];
r.read_exact(&mut e)?;
let l = u32::from_le_bytes([e[0], e[1], e[2], e[3]]) as usize;
let c = u32::from_le_bytes([e[4], e[5], e[6], e[7]]);
// a strip never compresses to much more than its raw size
if l > 12 * h.strip * h.width + 4096 {
return Err(ProductError::Format("a strip is larger than it can be".into()));
}
total += l as u64;
table.push((l, c));
}
Ok(Start { header: h, table, expected_len: total })
}
/// The header of the product at `path`, when it is one.
pub fn read_header_at(path: &Path) -> Result<Header, ProductError> {
let mut f = std::io::BufReader::new(std::fs::File::open(path)?);
Ok(read_start(&mut f)?.header)
}
/// Whether `path` holds a complete-looking product made from `key` (the header and the file's length are checked;
/// strips are checked when read).
pub fn is_current(path: &Path, key: &str) -> bool {
let check = || -> Result<bool, ProductError> {
let mut f = std::io::BufReader::new(std::fs::File::open(path)?);
let s = read_start(&mut f)?;
Ok(s.header.key == key && std::fs::metadata(path)?.len() == s.expected_len)
};
check().unwrap_or(false)
}
/// A rectangle of a product, in the product's pixels.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Window {
pub x: usize,
pub y: usize,
pub width: usize,
pub height: usize,
}
/// Half floats as `f32`, for all 65536 bit patterns (a table is much faster than computing each).
fn f16_table() -> &'static [f32] {
static TABLE: std::sync::OnceLock<Vec<f32>> = std::sync::OnceLock::new();
TABLE.get_or_init(|| (0..=u16::MAX).map(from_f16).collect())
}
/// Strips decoded together (they are inflated in parallel, then folded in order).
const GROUP: usize = 8;
/// One strip, checked and inflated: for each plane the low bytes of its half floats, then the high bytes.
fn decode_strip(index: usize, packed: &[u8], expect_crc: u32, n: usize) -> Result<Vec<u8>, ProductError> {
if crc(packed) != expect_crc {
return Err(ProductError::Format(format!("strip {index} is damaged")));
}
let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(packed, 6 * n).map_err(|e| ProductError::Format(format!("strip {index}: {e:?}")))?;
if raw.len() != 6 * n {
return Err(ProductError::Format(format!("strip {index} has the wrong size")));
}
Ok(raw)
}
/// Read the product at `path`, made from `key`, with every `factor × factor` block of pixels averaged into one
/// (`factor` 1 = full size). The result is camera RGB.
///
/// With a `window`, only that rectangle is read and the blocks start at its corner; only whole blocks are made, so
/// the result is `window.width / factor × window.height / factor` (the same blocks a binned development of the same
/// crop makes). Without one the whole product is read and blocks at the right and bottom edges may be smaller.
///
/// With `clip` (white is 1), a block in which any sample of a colour is at or above it takes that colour's
/// maximum instead of the mean, so a clipped highlight stays clipped for highlight reconstruction.
pub fn read_window(path: &Path, key: &str, window: Option<Window>, factor: usize, clip: Option<f32>) -> Result<Rgb32f, ProductError> {
let factor = factor.clamp(1, 1 << 12);
let mut f = std::io::BufReader::new(std::fs::File::open(path)?);
let Start { header: h, table, expected_len } = read_start(&mut f)?;
if h.key != key {
return Err(ProductError::Stale);
}
if std::fs::metadata(path)?.len() != expected_len {
return Err(ProductError::Format("the file is not the length its header says".into()));
}
let w = window.unwrap_or(Window { x: 0, y: 0, width: h.width, height: h.height });
let fits = |a: usize, b: usize, limit: usize| a.checked_add(b).is_some_and(|e| e <= limit);
if w.width == 0 || w.height == 0 || !fits(w.x, w.width, h.width) || !fits(w.y, w.height, h.height) {
return Err(ProductError::Format("the window does not lie inside the picture".into()));
}
// whole blocks for a window; partial ones at the edges when the whole picture is read
let whole = window.is_none();
let (ow, oh) = if whole { (w.width.div_ceil(factor), w.height.div_ceil(factor)) } else { (w.width / factor, w.height / factor) };
if ow == 0 || oh == 0 {
return Err(ProductError::Format("the window is smaller than one block".into()));
}
// the picture rows and columns the blocks cover
let (x_end, y_end) = if whole { (w.x + w.width, w.y + w.height) } else { (w.x + ow * factor, w.y + oh * factor) };
let mut out = Rgb32f { width: ow, height: oh, data: allocate(ow * oh, [0.0; 3])? };
let lut = f16_table();
let value = |raw: &[u8], c: usize, n: usize, i: usize| -> f32 {
let (lo, hi) = (raw.get(2 * c * n + i).copied().unwrap_or(0), raw.get((2 * c + 1) * n + i).copied().unwrap_or(0));
lut.get(usize::from(lo) | usize::from(hi) << 8).copied().unwrap_or(0.0)
};
// which strips are needed, and where each starts in the file (after the header and the strip table)
let wanted = |s: usize| s * h.strip < y_end && (s + 1) * h.strip > w.y;
let (mut sum, mut peak) = (allocate(ow, [0f32; 3])?, allocate(ow, [f32::MIN; 3])?);
let (mut rows_in, mut oy) = (0usize, 0usize);
let strips: Vec<usize> = (0..table.len()).filter(|&s| wanted(s)).collect();
// skip the strips before the first one wanted
let skipped: usize = table.iter().take(strips.first().copied().unwrap_or(0)).map(|e| e.0).sum();
f.seek_relative(i64::try_from(skipped).map_err(|_| ProductError::Format("the file is too large".into()))?)?;
for group in strips.chunks(GROUP) {
let mut packed = Vec::with_capacity(group.len());
for &s in group {
let Some(&(len, _)) = table.get(s) else { return Err(ProductError::Format("a strip is missing".into())) };
let mut p = allocate(len, 0u8)?;
f.read_exact(&mut p)?;
packed.push(p);
}
let decoded: Vec<Result<Vec<u8>, ProductError>> = group
.par_iter()
.zip(packed.par_iter())
.map(|(&s, p)| {
let rows = h.height.saturating_sub(s * h.strip).min(h.strip);
decode_strip(s, p, table.get(s).map_or(0, |e| e.1), rows * h.width)
})
.collect();
for (&s, raw) in group.iter().zip(decoded) {
let raw = raw?;
let y0 = s * h.strip;
let rows = h.height.saturating_sub(y0).min(h.strip);
let n = rows * h.width;
for r in 0..rows {
let y = y0 + r;
if y < w.y || y >= y_end {
continue;
}
let base = r * h.width;
for (ox, (sm, pk)) in sum.iter_mut().zip(peak.iter_mut()).enumerate() {
let x0 = w.x + ox * factor;
let x1 = (x0 + factor).min(x_end);
for x in x0..x1 {
for c in 0..3 {
let v = value(&raw, c, n, base + x);
if let (Some(s_), Some(p_)) = (sm.get_mut(c), pk.get_mut(c)) {
*s_ += v;
*p_ = p_.max(v);
}
}
}
}
rows_in += 1;
if rows_in == factor || y + 1 == y_end {
for (ox, (sm, pk)) in sum.iter_mut().zip(peak.iter_mut()).enumerate() {
// the block's size at the right and bottom edges may be smaller
let bw = (x_end - (w.x + ox * factor)).min(factor);
let k = 1.0 / (rows_in * bw).max(1) as f32;
if let Some(o) = out.data.get_mut(oy * ow + ox) {
for c in 0..3 {
let (s_, p_) = (sm.get(c).copied().unwrap_or(0.0), pk.get(c).copied().unwrap_or(0.0));
if let Some(slot) = o.get_mut(c) {
*slot = if factor > 1 && clip.is_some_and(|t| p_ >= t) { p_ } else { s_ * k };
}
}
}
*sm = [0.0; 3];
*pk = [f32::MIN; 3];
}
rows_in = 0;
oy += 1;
}
}
}
}
Ok(out)
}
/// [`read_window`] of the whole product: every `factor × factor` block averaged into one.
pub fn read(path: &Path, key: &str, factor: usize) -> Result<Rgb32f, ProductError> {
read_window(path, key, None, factor, None)
}
/// `base + (denoised − base) · amount` for every pixel (`amount` 0 to 1), in place in `base`. The two pictures must
/// be the same size; if they are not, `base` is left as it was and `false` is returned.
pub fn blend(base: &mut Rgb32f, denoised: &Rgb32f, amount: f32) -> bool {
if base.width != denoised.width || base.height != denoised.height {
return false;
}
let a = if amount.is_finite() { amount.clamp(0.0, 1.0) } else { 0.0 };
base.data.par_iter_mut().zip(denoised.data.par_iter()).for_each(|(b, d)| {
for c in 0..3 {
b[c] += (d[c] - b[c]) * a;
}
});
true
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn oversized_product_header_is_rejected_before_allocating_pixels() {
let p = temp("oversized-header");
let header = serde_json::to_vec(&Header { width: 10001, height: 10000, strip: STRIP_ROWS, key: "k".into() }).unwrap();
let mut bytes = Vec::from(MAGIC.as_slice());
bytes.extend_from_slice(&[VERSION, 0, 0, 0]);
bytes.extend_from_slice(&(header.len() as u32).to_le_bytes());
bytes.extend_from_slice(&header);
std::fs::write(&p, bytes).unwrap();
assert!(read_header_at(&p).is_err());
let _ = std::fs::remove_file(p);
}
fn temp(name: &str) -> std::path::PathBuf {
std::env::temp_dir().join(format!("lc-denoise-{name}-{}.lcdn", std::process::id()))
}
fn picture(w: usize, h: usize) -> Rgb32f {
Rgb32f {
width: w,
height: h,
data: (0..w * h).map(|i| [(i % w) as f32 / w as f32, (i / w) as f32 / h as f32, 0.001 + 0.5 * ((i % 7) as f32 / 7.0)]).collect(),
}
}
#[test]
fn half_floats_round_trip_closely() {
for v in [0.0f32, 1.0, -1.0, 0.5, 0.333_333, 1e-4, 6.1e-5, 3e-6, 123.456, 65000.0, -0.002] {
let back = from_f16(to_f16(v));
assert!((back - v).abs() <= v.abs() * 0.001 + 1e-7, "{v} -> {back}");
}
assert_eq!(from_f16(to_f16(1e10)), 65504.0);
assert_eq!(to_f16(f32::NAN), 0);
assert_eq!(from_f16(to_f16(f32::NEG_INFINITY)), -65504.0);
}
#[test]
fn a_product_survives_a_round_trip_at_full_and_reduced_size() {
let (w, h) = (301, 517);
let img = picture(w, h);
let p = temp("round");
write(&p, &img, "k1").unwrap();
assert!(is_current(&p, "k1") && !is_current(&p, "k2"));
assert_eq!(read_header_at(&p).unwrap().width, w);
let full = read(&p, "k1", 1).unwrap();
assert_eq!((full.width, full.height), (w, h));
for (a, b) in full.data.iter().zip(&img.data) {
for c in 0..3 {
assert!((a[c] - b[c]).abs() < 0.002, "{a:?} vs {b:?}");
}
}
// averaged into blocks of 4: the sizes round up, and the mean of the picture is kept
let small = read(&p, "k1", 4).unwrap();
assert_eq!((small.width, small.height), (w.div_ceil(4), h.div_ceil(4)));
let mean = |i: &Rgb32f| i.data.iter().map(|p| f64::from(p[1])).sum::<f64>() / i.data.len() as f64;
assert!((mean(&small) - mean(&img)).abs() < 0.01);
// a factor that does not divide the strip height is the same average
let odd = read(&p, "k1", 5).unwrap();
assert_eq!((odd.width, odd.height), (w.div_ceil(5), h.div_ceil(5)));
assert!((mean(&odd) - mean(&img)).abs() < 0.01);
assert!(matches!(read(&p, "other", 1), Err(ProductError::Stale)));
std::fs::remove_file(&p).unwrap();
}
/// What binning a window of `img` by `k` means, written the slow obvious way.
fn blocks(img: &Rgb32f, win: Window, k: usize, clip: Option<f32>) -> Rgb32f {
let (ow, oh) = (win.width / k, win.height / k);
let mut out = Rgb32f { width: ow, height: oh, data: vec![[0.0; 3]; ow * oh] };
for by in 0..oh {
for bx in 0..ow {
for c in 0..3 {
let vals: Vec<f32> = (0..k * k).map(|i| img.data[(win.y + by * k + i / k) * img.width + win.x + bx * k + i % k][c]).collect();
let max = vals.iter().copied().fold(f32::MIN, f32::max);
out.data[by * ow + bx][c] =
if k > 1 && clip.is_some_and(|t| max >= t) { max } else { vals.iter().sum::<f32>() / vals.len() as f32 };
}
}
}
out
}
#[test]
fn a_window_is_read_as_the_blocks_of_that_crop() {
let (w, h) = (301, 517);
let mut img = picture(w, h);
// a clipped sample in the middle of a block
img.data[(130 + 2) * w + 61 + 1] = [1.0, 0.4, 0.4];
let p = temp("window");
write(&p, &img, "k1").unwrap();
let near = |a: &Rgb32f, b: &Rgb32f| {
assert_eq!((a.width, a.height), (b.width, b.height));
for (x, y) in a.data.iter().zip(&b.data) {
for c in 0..3 {
assert!((x[c] - y[c]).abs() < 0.003, "{x:?} vs {y:?}");
}
}
};
// windows at odd places, across strip boundaries (128 rows), with factors that do and do not divide them
for (win, k) in [
(Window { x: 3, y: 5, width: 200, height: 300 }, 1),
(Window { x: 3, y: 5, width: 200, height: 300 }, 4),
(Window { x: 61, y: 130, width: 121, height: 250 }, 5),
(Window { x: 0, y: 0, width: w, height: h }, 2),
(Window { x: 1, y: 380, width: 90, height: 137 }, 3),
] {
let got = read_window(&p, "k1", Some(win), k, None).unwrap();
near(&got, &blocks(&img, win, k, None));
}
// with a clip level the block with the clipped sample keeps its highest value, the others stay means
let win = Window { x: 61, y: 130, width: 40, height: 20 };
let clipped = read_window(&p, "k1", Some(win), 4, Some(0.99)).unwrap();
near(&clipped, &blocks(&img, win, 4, Some(0.99)));
assert!(clipped.data[0][0] > 0.99, "the clipped block stays clipped: {:?}", clipped.data[0]);
let plain = read_window(&p, "k1", Some(win), 4, None).unwrap();
assert!(plain.data[0][0] < 0.99, "an average falls below the clip level");
// windows that do not fit, or are smaller than a block, are errors
for bad in [
Window { x: 0, y: 0, width: 0, height: 5 },
Window { x: 290, y: 0, width: 20, height: 5 },
Window { x: 0, y: 510, width: 5, height: 10 },
Window { x: usize::MAX, y: 0, width: 2, height: 2 },
Window { x: 0, y: 0, width: 3, height: 3 },
] {
assert!(read_window(&p, "k1", Some(bad), 4, None).is_err(), "{bad:?}");
}
std::fs::remove_file(&p).unwrap();
}
#[test]
fn a_smooth_picture_compresses_well() {
let img = Rgb32f {
width: 512,
height: 512,
data: (0..512 * 512)
.map(|i| {
let v = 0.2 + 0.0001 * (i % 512) as f32;
[v, v * 0.8, v * 0.6]
})
.collect(),
};
let p = temp("size");
write(&p, &img, "k").unwrap();
let bytes = std::fs::metadata(&p).unwrap().len() as usize;
assert!(bytes < 512 * 512 * 6 / 2, "{bytes} bytes for a smooth picture");
std::fs::remove_file(&p).unwrap();
}
#[test]
fn damaged_files_are_errors_not_panics() {
let img = picture(100, 300);
let p = temp("damage");
write(&p, &img, "k").unwrap();
let good = std::fs::read(&p).unwrap();
// truncated at every length
for cut in (0..good.len()).step_by(37) {
std::fs::write(&p, &good[..cut]).unwrap();
assert!(read(&p, "k", 1).is_err(), "cut at {cut}");
assert!(!is_current(&p, "k"));
}
// a flipped byte anywhere is caught by a checksum, the header parse, or the key
for at in (0..good.len()).step_by(53) {
let mut bad = good.clone();
bad[at] ^= 0xff;
std::fs::write(&p, &bad).unwrap();
let _ = read(&p, "k", 1); // must not panic; most are errors
}
// a header that claims an absurd size
let mut huge = b"LCDN\x01\0\0\0".to_vec();
let hdr = br#"{"width":4000000000,"height":4000000000,"strip":128,"key":"k"}"#;
huge.extend((hdr.len() as u32).to_le_bytes());
huge.extend(hdr);
std::fs::write(&p, &huge).unwrap();
assert!(matches!(read(&p, "k", 1), Err(ProductError::Format(_))));
std::fs::write(&p, b"").unwrap();
assert!(read(&p, "k", 1).is_err());
assert!(read(&temp("missing"), "k", 1).is_err());
std::fs::remove_file(&p).unwrap();
}
#[test]
fn writing_rejects_unusable_pictures() {
let p = temp("bad");
assert!(write(&p, &Rgb32f { width: 0, height: 0, data: vec![] }, "k").is_err());
assert!(write(&p, &Rgb32f { width: 3, height: 3, data: vec![[0.0; 3]; 8] }, "k").is_err());
assert!(!p.exists());
}
#[test]
fn blend_mixes_by_amount_and_rejects_other_sizes() {
let mut base = Rgb32f { width: 2, height: 1, data: vec![[0.0; 3], [1.0; 3]] };
let dn = Rgb32f { width: 2, height: 1, data: vec![[1.0; 3], [0.0; 3]] };
assert!(blend(&mut base, &dn, 0.25));
assert_eq!(base.data, vec![[0.25; 3], [0.75; 3]]);
assert!(blend(&mut base, &dn, f32::NAN));
assert_eq!(base.data, vec![[0.25; 3], [0.75; 3]]);
assert!(!blend(&mut base, &Rgb32f { width: 1, height: 1, data: vec![[0.0; 3]] }, 1.0));
assert!(blend(&mut base, &dn, 1.0));
assert_eq!(base.data, dn.data);
}
#[test]
fn crc_matches_the_standard_check_value() {
assert_eq!(crc(b"123456789"), 0xcbf4_3926);
}
}
+589
View File
@@ -0,0 +1,589 @@
//! Denoising a whole Bayer mosaic with a tile model: cut it into overlapping tiles, run the model on each, bring
//! each tile's scale back to the input's, keep clipped highlights as they were, and blend the tiles together.
//!
//! The model itself is a [`TileRunner`], so everything around it (packing, tiling, scale matching, highlight
//! protection, blending, cancelling, progress) is tested without one.
use std::sync::atomic::{AtomicBool, Ordering};
use lightcraft_raster::Rgb32f;
use rayon::prelude::*;
use crate::bayer::{Layout, pack_tile};
use crate::manifest::Gain;
use crate::tiles::{starts, weight};
/// Most pixels one picture may have (guards the allocations that follow from its size).
pub const MAX_PIXELS: usize = 100_000_000;
/// A tile whose mean signal is below this (white = 1) is too dark to measure its scale from.
const MIN_SIGNAL: f32 = 0.003;
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("{0}")]
Input(String),
#[error("the model gave {0}")]
Model(String),
#[error("the model could not run: {0}")]
Runtime(String),
#[error("the model could not be loaded: {0}")]
Load(String),
#[error("cancelled")]
Cancelled,
}
fn allocate<T: Clone>(n: usize, value: T) -> Result<Vec<T>, Error> {
let mut v = Vec::new();
v.try_reserve_exact(n).map_err(|_| Error::Input("not enough memory for denoise".into()))?;
v.resize(n, value);
Ok(v)
}
/// Something that runs the model on one tile.
pub trait TileRunner: Sync {
/// `input` is the tile as four planes `[R, G1, G2, B]` of `tile × tile` cells (NCHW, batch 1); the answer is
/// camera RGB, three planes of `2·tile × 2·tile` pixels (CHW), in the model's own scale.
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error>;
}
pub struct Params {
/// Side of the model's square, in cells.
pub tile: usize,
/// Cells neighbouring tiles share.
pub overlap: usize,
pub gain: Gain,
/// White is 1; samples at or above this (minus a little) are clipped and keep their value.
pub clip: Option<f32>,
/// Tiles run at once (each model call may itself be single-threaded).
pub parallel: usize,
}
#[derive(Default)]
pub struct Control<'a> {
pub cancel: Option<&'a AtomicBool>,
/// `(tiles done, tiles in all)`.
pub progress: Option<&'a (dyn Fn(usize, usize) + Sync)>,
}
/// The tile grid of a picture: `(starts across, starts down)`.
pub fn grid(width: usize, height: usize, layout: Layout, tile: usize, overlap: usize) -> (Vec<usize>, Vec<usize>) {
let (nx, ny) = layout.cells(width, height);
(starts(nx, tile, overlap), starts(ny, tile, overlap))
}
/// How many model calls [`denoise_bayer`] makes for a picture.
pub fn tile_count(width: usize, height: usize, layout: Layout, tile: usize, overlap: usize) -> usize {
let (sx, sy) = grid(width, height, layout, tile, overlap);
sx.len().saturating_mul(sy.len())
}
/// Denoise (and demosaic) the normalised mosaic `mosaic` of `width × height` samples, laid out as `layout`.
/// Gives camera RGB at the mosaic's resolution.
pub fn denoise_bayer(
mosaic: &[f32],
width: usize,
height: usize,
layout: Layout,
runner: &dyn TileRunner,
p: &Params,
ctl: &Control<'_>,
) -> Result<Rgb32f, Error> {
let pixels = width
.checked_mul(height)
.filter(|&n| n > 0 && n <= MAX_PIXELS)
.ok_or_else(|| Error::Input("the picture has no size, or is too large".into()))?;
if mosaic.len() != pixels {
return Err(Error::Input("the mosaic does not have width × height samples".into()));
}
if !(16..=2048).contains(&p.tile) || p.overlap >= p.tile.div_ceil(2) || layout.dx > 1 || layout.dy > 1 {
return Err(Error::Input("the tile size and overlap do not fit".into()));
}
if let Gain::MatchMean { nominal, max_deviation } = p.gain
&& !(nominal.is_finite() && (1e-6..=1e12).contains(&nominal) && max_deviation.is_finite() && (0.0..=0.5).contains(&max_deviation))
{
return Err(Error::Input("the model gain is not finite or within its supported range".into()));
}
if p.clip.is_some_and(|v| !v.is_finite() || v <= 0.0) {
return Err(Error::Input("the highlight clip must be finite and positive".into()));
}
let (nx, ny) = layout.cells(width, height);
let (sx, sy) = (starts(nx, p.tile, p.overlap), starts(ny, p.tile, p.overlap));
let (cols, rows) = (sx.len(), sy.len());
if cols == 0 || rows == 0 {
return Err(Error::Input("the picture needs too many tiles".into()));
}
let tiles: Vec<(usize, usize, bool, bool, bool, bool)> = sy
.iter()
.enumerate()
.flat_map(|(j, &y0)| sx.iter().enumerate().map(move |(i, &x0)| (x0, y0, i > 0, i + 1 < cols, j > 0, j + 1 < rows)))
.collect();
let total = tiles.len();
let mut acc = allocate(pixels, [0f32; 3])?;
let mut wsum = allocate(nx * ny, 0f32)?;
let group = p.parallel.clamp(1, 4);
let mut done = 0;
// where the time goes, printed under `LIGHTCRAFT_PROFILE`
let (mut t_wait, mut t_total) = (std::time::Duration::ZERO, std::time::Duration::ZERO);
let began = web_time::Instant::now();
// One group of tiles at a time: pack, run the model, check and bring each to the input's scale, in parallel.
// Blending (sequential: the tiles add into one picture, in tile order so the result does not depend on timing) of
// the previous group runs while the next group is on the model, so neither the card nor the cores wait for it.
type Tile = (usize, usize, bool, bool, bool, bool);
let compute = |chunk: &[Tile]| -> Vec<Result<Vec<f32>, Error>> {
chunk
.par_iter()
.map(|&(x0, y0, bx, ax, by, ay)| {
let mut input = allocate(4 * p.tile * p.tile, 0f32)?;
if !pack_tile(mosaic, width, height, layout, x0 as isize, y0 as isize, p.tile, &mut input) {
return Err(Error::Input("a tile could not be packed".into()));
}
let mut out = runner.run(&input)?;
let side = 2 * p.tile;
if out.len() != 3 * side * side {
return Err(Error::Model(format!("{} numbers for a tile, not the {} expected", out.len(), 3 * side * side)));
}
if out.iter().any(|v| !v.is_finite()) {
return Err(Error::Model("numbers that are not finite".into()));
}
finish_tile(&input, &mut out, p, (bx, ax, by, ay));
Ok(out)
})
.collect()
};
let mut blend = |chunk: &[Tile], outs: Vec<Result<Vec<f32>, Error>>| -> Result<(), Error> {
for (&(x0, y0, bx, ax, by, ay), r) in chunk.iter().zip(outs) {
blend_tile(&r?, &mut acc, &mut wsum, (width, height), (nx, ny), layout, (x0, y0), p, (bx, ax, by, ay));
done += 1;
if let Some(f) = ctl.progress {
f(done, total);
}
}
Ok(())
};
let mut previous: Option<(&[Tile], Vec<Result<Vec<f32>, Error>>)> = None;
for chunk in tiles.chunks(group) {
if ctl.cancel.is_some_and(|c| c.load(Ordering::Relaxed)) {
return Err(Error::Cancelled);
}
let (outs, blended) = rayon::join(
|| compute(chunk),
|| match previous.take() {
Some((c, o)) => {
let started = web_time::Instant::now();
let r = blend(c, o);
(r, started.elapsed())
}
None => (Ok(()), std::time::Duration::ZERO),
},
);
blended.0?;
t_wait += blended.1;
previous = Some((chunk, outs));
}
if let Some((c, o)) = previous.take() {
let started = web_time::Instant::now();
blend(c, o)?;
t_wait += started.elapsed();
}
t_total += began.elapsed();
let started = web_time::Instant::now();
// normalise by the weights
let (ox, oy) = (layout.position(0, 0).0, layout.position(0, 0).1);
acc.par_chunks_mut(width).enumerate().for_each(|(y, row)| {
let cy = ((y as isize - oy) / 2) as usize;
for (x, px) in row.iter_mut().enumerate() {
let cx = ((x as isize - ox) / 2) as usize;
let w = wsum.get(cy * nx + cx).copied().unwrap_or(0.0);
if w > 0.0 {
*px = px.map(|v| v / w);
}
}
});
if std::env::var_os("LIGHTCRAFT_PROFILE").is_some() {
eprintln!(
"[denoise] {total} tiles, {} at once: tiles {:.0} ms (blending, {:.0} ms of it, overlaps the model), normalise {:.0} ms",
group,
t_total.as_secs_f64() * 1e3,
t_wait.as_secs_f64() * 1e3,
started.elapsed().as_secs_f64() * 1e3
);
}
Ok(Rgb32f { width, height, data: acc })
}
/// Bring a tile's output to the input's scale and keep clipped highlights as they were (in place).
fn finish_tile(input: &[f32], out: &mut [f32], p: &Params, edges: (bool, bool, bool, bool)) {
let t = p.tile;
let side = 2 * t;
if let Gain::MatchMean { nominal, max_deviation } = p.gain {
// the part of the tile whose scale is measured: away from the edges that blend into a neighbour
let m = p.overlap / 2;
let (i0, i1) = (if edges.0 { m } else { 0 }, t - if edges.1 { m } else { 0 });
let (j0, j1) = (if edges.2 { m } else { 0 }, t - if edges.3 { m } else { 0 });
let cells = ((i1 - i0) * (j1 - j0)) as f64;
let plane_sum = |pl: usize| -> f64 {
(j0..j1)
.map(|j| input.get(pl * t * t + j * t + i0..pl * t * t + j * t + i1).map_or(0.0, |r| r.iter().map(|&v| f64::from(v)).sum::<f64>()))
.sum()
};
let in_mean = [plane_sum(0) / cells, (plane_sum(1) + plane_sum(2)) / (2.0 * cells), plane_sum(3) / cells];
for (c, &mean_in) in in_mean.iter().enumerate() {
let sum_out: f64 = (2 * j0..2 * j1)
.map(|y| {
out.get(c * side * side + y * side + 2 * i0..c * side * side + y * side + 2 * i1)
.map_or(0.0, |r| r.iter().map(|&v| f64::from(v)).sum::<f64>())
})
.sum();
let mean_out = sum_out / (4.0 * cells);
let measured = if mean_in as f32 >= MIN_SIGNAL && mean_out > 0.0 { (mean_out / mean_in) as f32 } else { nominal };
let g = measured.clamp(nominal * (1.0 - max_deviation), nominal * (1.0 + max_deviation));
if g.is_finite() && g > 0.0 {
let inv = 1.0 / g;
if let Some(plane) = out.get_mut(c * side * side..(c + 1) * side * side) {
plane.iter_mut().for_each(|v| *v *= inv);
}
}
}
}
if let Some(clip) = p.clip {
protect_clipped(input, out, t, clip);
}
}
/// Where the mosaic is at (or within a cell of) white, the output becomes the plain cell values again, so a clipped
/// area stays exactly as clipped as the camera recorded it (highlight recovery later depends on that).
fn protect_clipped(input: &[f32], out: &mut [f32], t: usize, clip: f32) {
let side = 2 * t;
let plane = t * t;
let (Some(r), Some(g1), Some(g2), Some(b)) =
(input.get(0..plane), input.get(plane..2 * plane), input.get(2 * plane..3 * plane), input.get(3 * plane..4 * plane))
else {
return;
};
let mut m: Vec<f32> = (0..plane).map(|i| r[i].max(g1[i]).max(g2[i]).max(b[i])).collect();
if !m.iter().any(|&v| v >= clip) {
return;
}
// 3×3 maximum over cells (separable)
let dilate = |src: &[f32], horizontal: bool| -> Vec<f32> {
(0..plane)
.map(|i| {
let (x, y) = (i % t, i / t);
let mut best = src[i];
for d in [-1isize, 1] {
let (nx, ny) = if horizontal { (x as isize + d, y as isize) } else { (x as isize, y as isize + d) };
if nx >= 0 && ny >= 0 && (nx as usize) < t && (ny as usize) < t {
best = best.max(src[ny as usize * t + nx as usize]);
}
}
best
})
.collect()
};
m = dilate(&dilate(&m, true), false);
let lo = clip - 0.03;
for j in 0..t {
for i in 0..t {
let k = j * t + i;
let s = ((m[k] - lo) / (clip - lo)).clamp(0.0, 1.0);
if s <= 0.0 {
continue;
}
let s = s * s * (3.0 - 2.0 * s);
let plain = [r[k], 0.5 * (g1[k] + g2[k]), b[k]];
for (c, &pv) in plain.iter().enumerate() {
for (sy, sx) in [(0, 0), (0, 1), (1, 0), (1, 1)] {
if let Some(v) = out.get_mut(c * side * side + (2 * j + sy) * side + 2 * i + sx) {
*v += (pv - *v) * s;
}
}
}
}
}
}
/// Add a finished tile to the picture with its blend weights.
#[allow(clippy::too_many_arguments)]
fn blend_tile(
out: &[f32],
acc: &mut [[f32; 3]],
wsum: &mut [f32],
size: (usize, usize),
cells: (usize, usize),
layout: Layout,
at: (usize, usize),
p: &Params,
edges: (bool, bool, bool, bool),
) {
let (width, height) = size;
let (nx, ny) = cells;
let (t, side) = (p.tile, 2 * p.tile);
let fade = p.overlap;
let wx: Vec<f32> = (0..t).map(|i| weight(i, t, fade, edges.0, edges.1)).collect();
let wy: Vec<f32> = (0..t).map(|j| weight(j, t, fade, edges.2, edges.3)).collect();
for j in 0..t {
let cy = at.1 + j;
for i in 0..t {
let cx = at.0 + i;
// cells past the picture's edge (a picture smaller than one tile is padded) belong to no pixel
if cx >= nx || cy >= ny {
continue;
}
let (Some(&a), Some(&b)) = (wx.get(i), wy.get(j)) else { continue };
let w = a * b;
if let Some(s) = wsum.get_mut(cy * nx + cx) {
*s += w;
}
let (px0, py0) = layout.position(cx as isize, cy as isize);
for (sy, sx) in [(0isize, 0isize), (0, 1), (1, 0), (1, 1)] {
let (x, y) = (px0 + sx, py0 + sy);
if x < 0 || y < 0 || x as usize >= width || y as usize >= height {
continue;
}
let o = (2 * j + sy as usize) * side + 2 * i + sx as usize;
let (Some(&r), Some(&g), Some(&bl)) = (out.get(o), out.get(side * side + o), out.get(2 * side * side + o)) else { continue };
if let Some(px) = acc.get_mut(y as usize * width + x as usize) {
px[0] += w * r;
px[1] += w * g;
px[2] += w * bl;
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// A stand-in model: the cell's samples as an RGB picture at twice the resolution (a plain 2 × 2 "demosaic"),
/// times `scale`, so the output has a scale of its own like the real model's.
struct Mock {
tile: usize,
scale: f32,
}
impl TileRunner for Mock {
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
let (t, side) = (self.tile, 2 * self.tile);
let plane = t * t;
let mut out = vec![0.0; 3 * side * side];
for j in 0..t {
for i in 0..t {
let k = j * t + i;
let rgb = [input[k], 0.5 * (input[plane + k] + input[2 * plane + k]), input[3 * plane + k]];
for (c, v) in rgb.iter().enumerate() {
for (sy, sx) in [(0, 0), (0, 1), (1, 0), (1, 1)] {
out[c * side * side + (2 * j + sy) * side + 2 * i + sx] = v * self.scale;
}
}
}
}
Ok(out)
}
}
fn params(tile: usize, overlap: usize, gain: Gain) -> Params {
Params { tile, overlap, gain, clip: None, parallel: 2 }
}
#[test]
fn overflowing_tiles_and_nan_gain_are_errors_before_the_model_runs() {
let layout = Layout::from_cell([0, 1, 1, 2]).unwrap();
let model = Mock { tile: 16, scale: 1.0 };
for (tile, overlap, gain) in [
(usize::MAX, 0, Gain::None),
(16, usize::MAX, Gain::None),
(16, 0, Gain::MatchMean { nominal: 1.0, max_deviation: f32::NAN }),
(16, 0, Gain::MatchMean { nominal: f32::INFINITY, max_deviation: 0.1 }),
] {
assert!(matches!(denoise_bayer(&[0.5], 1, 1, layout, &model, &params(tile, overlap, gain), &Control::default()), Err(Error::Input(_))));
}
}
/// A smooth picture whose colour at a sample is `base[colour] · (1 + shading)`.
fn mosaic(w: usize, h: usize, cell: [u8; 4]) -> Vec<f32> {
(0..w * h)
.map(|i| {
let (x, y) = (i % w, i / w);
let c = cell[(y % 2) * 2 + x % 2] as usize;
[0.5, 0.3, 0.2][c] * (0.6 + 0.4 * ((x as f32 / 23.0).sin() * (y as f32 / 31.0).cos()))
})
.collect()
}
fn reference(m: &[f32], w: usize, h: usize, layout: Layout) -> Rgb32f {
// what the mock gives for the whole picture in one tile
let (nx, ny) = layout.cells(w, h);
let t = nx.max(ny).div_ceil(16) * 16;
denoise_bayer(m, w, h, layout, &Mock { tile: t, scale: 1.0 }, &params(t, 16, Gain::None), &Control::default()).unwrap()
}
#[test]
fn tiles_blend_to_the_same_picture_as_one_tile_for_every_layout() {
for cell in [[0u8, 1, 1, 2], [1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]] {
let layout = Layout::from_cell(cell).unwrap();
let (w, h) = (150usize, 110usize);
let m = mosaic(w, h, cell);
let want = reference(&m, w, h, layout);
let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, &params(32, 8, Gain::None), &Control::default()).unwrap();
assert_eq!((got.width, got.height), (w, h));
for (a, b) in got.data.iter().zip(&want.data) {
for c in 0..3 {
assert!((a[c] - b[c]).abs() < 1e-5, "layout {cell:?}: {a:?} vs {b:?}");
}
}
}
}
#[test]
fn the_picture_does_not_depend_on_how_many_tiles_run_at_once_or_how_long_each_takes() {
// tiles that finish in a different order every time: blending still adds them in tile order
struct Jittery(Mock);
impl TileRunner for Jittery {
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
let wait = (input.iter().take(64).sum::<f32>() * 1000.0) as u64 % 7;
std::thread::sleep(std::time::Duration::from_millis(wait));
self.0.run(input)
}
}
let layout = Layout { dx: 0, dy: 1 };
let (w, h) = (150usize, 110usize);
let m = mosaic(w, h, [1, 0, 2, 1]);
let run = |parallel: usize| {
let mut p = params(32, 8, Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 });
p.parallel = parallel;
denoise_bayer(&m, w, h, layout, &Jittery(Mock { tile: 32, scale: 1.01e6 }), &p, &Control::default()).unwrap()
};
let one = run(1);
for parallel in [2, 3, 8] {
assert_eq!(run(parallel).data, one.data, "{parallel} at once");
}
}
#[test]
fn a_failing_tile_or_a_cancel_ends_the_picture_without_a_panic() {
struct Fails(Mock, std::sync::atomic::AtomicUsize);
impl TileRunner for Fails {
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
if self.1.fetch_add(1, Ordering::Relaxed) == 4 {
return Err(Error::Runtime("boom".into()));
}
self.0.run(input)
}
}
let layout = Layout { dx: 0, dy: 0 };
let (w, h) = (150usize, 110usize);
let m = mosaic(w, h, [0, 1, 1, 2]);
let r = denoise_bayer(
&m,
w,
h,
layout,
&Fails(Mock { tile: 32, scale: 1.0 }, Default::default()),
&params(32, 8, Gain::None),
&Control::default(),
);
assert!(matches!(r, Err(Error::Runtime(_))), "{r:?}");
let stop = AtomicBool::new(true);
let ctl = Control { cancel: Some(&stop), progress: None };
let r = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, &params(32, 8, Gain::None), &ctl);
assert!(matches!(r, Err(Error::Cancelled)));
}
#[test]
fn a_scale_of_its_own_is_taken_out_tile_by_tile() {
let layout = Layout { dx: 0, dy: 0 };
let (w, h) = (150usize, 110usize);
let m = mosaic(w, h, [0, 1, 1, 2]);
let want = reference(&m, w, h, layout);
let gain = Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 };
// the scale differs a little from nominal; each tile's own scale is measured and removed
let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.02e6 }, &params(32, 8, gain), &Control::default()).unwrap();
let mean = |img: &Rgb32f, c: usize| img.data.iter().map(|p| f64::from(p[c])).sum::<f64>() / img.data.len() as f64;
for c in 0..3 {
let (a, b) = (mean(&got, c), mean(&want, c));
assert!((a - b).abs() / b < 0.01, "channel {c}: {a} vs {b}");
}
// a scale far from nominal is clamped rather than trusted
let wild = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 3.0e6 }, &params(32, 8, gain), &Control::default()).unwrap();
assert!(mean(&wild, 1) > 2.0 * mean(&want, 1));
}
#[test]
fn a_picture_smaller_than_a_tile_is_padded_by_mirroring() {
let layout = Layout { dx: 1, dy: 1 };
let (w, h) = (20usize, 14usize);
let m = mosaic(w, h, [2, 1, 1, 0]);
let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 64, scale: 1.0 }, &params(64, 16, Gain::None), &Control::default()).unwrap();
assert_eq!((got.width, got.height), (w, h));
assert!(got.data.iter().all(|p| p.iter().all(|v| v.is_finite() && *v > 0.0)));
}
#[test]
fn clipped_highlights_stay_clipped() {
// a model that darkens everything by 10 % would pull a clipped plateau below white
struct Dim(Mock);
impl TileRunner for Dim {
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
Ok(self.0.run(input)?.into_iter().map(|v| v * 0.9).collect())
}
}
let layout = Layout { dx: 0, dy: 0 };
let (w, h) = (96usize, 96usize);
let m: Vec<f32> = (0..w * h).map(|i| if i % w > 40 && i % w < 70 { 1.0 } else { 0.4 }).collect();
let mut p = params(32, 8, Gain::None);
p.clip = Some(0.99);
let got = denoise_bayer(&m, w, h, layout, &Dim(Mock { tile: 32, scale: 1.0 }), &p, &Control::default()).unwrap();
let mid = got.data[48 * w + 55];
assert!(mid.iter().all(|&v| v > 0.995), "plateau {mid:?}");
let dark = got.data[48 * w + 10];
assert!(dark.iter().all(|&v| (v - 0.36).abs() < 0.01), "unclipped area still goes through the model: {dark:?}");
}
#[test]
fn cancelling_and_progress_work() {
let layout = Layout { dx: 0, dy: 0 };
let (w, h) = (150usize, 110usize);
let m = mosaic(w, h, [0, 1, 1, 2]);
let cancel = AtomicBool::new(true);
let r = denoise_bayer(
&m,
w,
h,
layout,
&Mock { tile: 32, scale: 1.0 },
&params(32, 8, Gain::None),
&Control { cancel: Some(&cancel), progress: None },
);
assert!(matches!(r, Err(Error::Cancelled)));
let seen = std::sync::Mutex::new(Vec::new());
let f = |d: usize, n: usize| seen.lock().unwrap().push((d, n));
denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, &params(32, 8, Gain::None), &Control { cancel: None, progress: Some(&f) })
.unwrap();
let seen = seen.into_inner().unwrap();
let total = tile_count(w, h, layout, 32, 8);
assert!(total > 1);
assert_eq!(seen.last(), Some(&(total, total)));
assert!(seen.windows(2).all(|p| p[1].0 == p[0].0 + 1));
}
#[test]
fn hostile_inputs_are_errors() {
let layout = Layout { dx: 0, dy: 0 };
let mock = Mock { tile: 32, scale: 1.0 };
let p = params(32, 8, Gain::None);
assert!(matches!(denoise_bayer(&[], 0, 0, layout, &mock, &p, &Control::default()), Err(Error::Input(_))));
assert!(matches!(denoise_bayer(&[0.0; 10], 4, 4, layout, &mock, &p, &Control::default()), Err(Error::Input(_))));
assert!(matches!(denoise_bayer(&[0.0; 16], 4, 4, layout, &mock, &params(32, 16, Gain::None), &Control::default()), Err(Error::Input(_))));
struct Bad(Vec<f32>);
impl TileRunner for Bad {
fn run(&self, _: &[f32]) -> Result<Vec<f32>, Error> {
Ok(self.0.clone())
}
}
assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![0.0; 5]), &p, &Control::default()), Err(Error::Model(_))));
assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![f32::NAN; 3 * 64 * 64]), &p, &Control::default()), Err(Error::Model(_))));
}
}
+100
View File
@@ -0,0 +1,100 @@
//! Cutting a picture into overlapping tiles and putting the results back together.
//!
//! A model runs on a fixed square, so a picture is covered by tiles spaced evenly along each axis, each sharing
//! at least `overlap` cells with its neighbours. Each tile's influence fades towards its own edges (except at the
//! picture's edge), and the result is the sum of tile × weight divided by the sum of weights, so any set of
//! overlapping tiles blends consistently and a seam between two tiles of slightly different scale is a smooth ramp.
/// Where the tiles start along one axis of `n` cells: `[0]` when the axis fits one tile (the tile is then padded by
/// mirroring), else evenly spaced starts, the first at 0 and the last at `n − tile`, each tile sharing at least
/// `overlap` cells with the next.
pub fn starts(n: usize, tile: usize, overlap: usize) -> Vec<usize> {
if n <= tile || tile == 0 {
return vec![0];
}
let step = tile.saturating_sub(overlap).max(1);
// tiles needed so that start k·spacing with spacing ≤ step reaches n − tile
let Some(k) = (n - tile).div_ceil(step).checked_add(1).filter(|&k| k <= 1_000_000) else { return Vec::new() };
let last = n - tile;
(0..k).map(|i| if k == 1 { 0 } else { ((i as u128 * last as u128 + (k - 1) as u128 / 2) / (k - 1) as u128) as usize }).collect()
}
/// Weight of the cell at offset `p` (0-based, inside a tile of `tile` cells) along one axis: 1 in the middle, easing
/// down to a sliver over `fade` cells at an edge that has a neighbour, 1 right up to an edge that is the picture's.
pub fn weight(p: usize, tile: usize, fade: usize, has_before: bool, has_after: bool) -> f32 {
let fade = fade.max(1) as f32;
let ramp = |d: usize| {
let t = ((d as f32 + 0.5) / fade).min(1.0);
// smoothstep, never exactly 0 so that a cell is always covered
(t * t * (3.0 - 2.0 * t)).max(1e-3)
};
let before = if has_before { ramp(p) } else { 1.0 };
let after = if has_after { ramp(tile.saturating_sub(1).saturating_sub(p)) } else { 1.0 };
before.min(after)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn hostile_axes_have_bounded_tile_counts_and_checked_spacing() {
assert!(starts(usize::MAX, 1, 0).is_empty());
assert!(starts(usize::MAX, 2, usize::MAX).is_empty());
let s = starts(usize::MAX, usize::MAX / 2, 0);
assert_eq!(s.first(), Some(&0));
assert_eq!(s.last(), Some(&(usize::MAX - usize::MAX / 2)));
assert!(s.windows(2).all(|s| s[0] < s[1]));
}
#[test]
fn one_tile_when_it_fits() {
assert_eq!(starts(100, 512, 64), vec![0]);
assert_eq!(starts(512, 512, 64), vec![0]);
assert_eq!(starts(0, 512, 64), vec![0]);
assert_eq!(starts(10, 0, 0), vec![0]);
}
#[test]
fn tiles_cover_the_axis_and_overlap_enough() {
for n in [513usize, 600, 900, 1000, 1024, 1500, 3024, 3600, 5000, 12345] {
let s = starts(n, 512, 64);
assert_eq!(s[0], 0, "n={n}");
assert_eq!(*s.last().unwrap(), n - 512, "n={n}");
for w in s.windows(2) {
assert!(w[1] > w[0], "starts increase, n={n}");
assert!(w[0] + 512 >= w[1] + 64, "at least 64 shared cells between {} and {}, n={n}", w[0], w[1]);
}
}
}
#[test]
fn weights_fade_only_towards_neighbours() {
let (t, f) = (512, 64);
// a lone tile: full weight everywhere
assert!((0..t).all(|p| weight(p, t, f, false, false) == 1.0));
// the first tile fades on its far side only
assert_eq!(weight(0, t, f, false, true), 1.0);
assert!(weight(t - 1, t, f, false, true) < 0.01);
// the last tile fades on its near side only
assert!(weight(0, t, f, true, false) < 0.01);
assert_eq!(weight(t - 1, t, f, true, false), 1.0);
// a middle tile is 1 in the middle and everywhere positive
assert_eq!(weight(t / 2, t, f, true, true), 1.0);
assert!((0..t).all(|p| weight(p, t, f, true, true) > 0.0));
}
#[test]
fn two_overlapping_tiles_blend_to_a_partition_of_unity() {
// at every cell of the shared region the normalised weights are in [0, 1] and sum to 1
let (t, f) = (512usize, 64usize);
let s = starts(900, t, 64);
assert_eq!(s.len(), 2);
for cell in s[1]..s[0] + t {
let a = weight(cell - s[0], t, f, false, true);
let b = weight(cell - s[1], t, f, true, false);
let (na, nb) = (a / (a + b), b / (a + b));
assert!((na + nb - 1.0).abs() < 1e-6 && na >= 0.0 && nb >= 0.0);
}
}
}
+32
View File
@@ -0,0 +1,32 @@
[package]
name = "lightcraft-denoise"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "LightCraft AI denoise (L1): denoise-model manifests, Bayer packing, tiling and blending, the cache file, and pure-Rust CPU inference"
[lints]
workspace = true
[features]
default = []
runtime = ["dep:faer"]
# Product-policy switch: absent by default; users may install their own model.
rawnind-model = ["lightcraft-denoise-core/rawnind-model"]
[dependencies]
lightcraft-denoise-core = { workspace = true }
# Safe matrix views over the pure-Rust GEMM backend; no BLAS, cc/build scripts
# or inner thread pool. Its dependencies include architecture-specific assembly.
faer = { package = "faer-core", version = "=0.17.1", default-features = false, features = ["std"], optional = true }
lightcraft-raster = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
sha2 = { workspace = true }
thiserror = { workspace = true }
miniz_oxide = { workspace = true }
rayon = { workspace = true }
web-time = { workspace = true }
[dev-dependencies]
proptest = { workspace = true }
+18
View File
@@ -0,0 +1,18 @@
# lightcraft-denoise
L1 pure-Rust AI RAW denoise primitives and CPU inference. No weights are bundled.
See [user workflow and limitations](../../docs/denoise.md).
`lightcraft-denoise-core` provides metadata, Bayer helpers and cache formats,
re-exported here. `manifest` and `known` validate opt-in models; `onnx` reads a bounded operator subset
into `net`; `cpu` executes it and `reference` supplies scalar checks. `bayer`, `tiles`
and `run` pack sensor data and blend overlapping tiles while preserving clipped
highlights. `product` reads/writes the disposable `.lcdn` cache. `runtime` supplies
the installation self-test. Synthetic networks are generated in Rust for tests.
Run `cargo test -p lightcraft-denoise --features runtime`. The CPU backend is optional;
featureless and web builds keep the model metadata and cache format without faer/GEMM.
The `rawnind-model` feature enables the otherwise empty download catalog and requires
an explicit release-policy decision. Real-model tests are ignored by default and
require an explicitly supplied local model. The GPU twin is `lightcraft-gpu::nn`;
model management, background jobs and export integration live in `lightcraft-engine`.
+386
View File
@@ -0,0 +1,386 @@
//! Single-thread CPU execution of the checked denoise network.
//!
//! Convolutions lower at most 256 spatial positions at a time, then use faer's safe views over pure-Rust GEMM.
//! No inner thread pool: `run::denoise_bayer` already runs tiles on the application's Rayon pool. Tensor lifetimes
//! are planned once; buffers and im2col storage are recycled between calls, independently for concurrent callers.
use crate::net::{Net, NetError, Op};
use crate::run::{Error, TileRunner};
use faer::{Parallelism, mat};
use std::sync::{Arc, Mutex};
const BLOCK: usize = 256;
/// Cap all tensor buffers and convolution scratch for one tile together (512 MiB).
const MAX_VALUES: usize = 128 * 1024 * 1024;
const MAX_TILE: usize = 2048;
const MAX_IDLE: usize = 1;
type Result<T> = std::result::Result<T, NetError>;
fn bad(why: &str) -> NetError {
NetError::Invalid(why.into())
}
fn count(c: usize, side: usize) -> Result<usize> {
c.checked_mul(side)
.and_then(|n| n.checked_mul(side))
.filter(|&n| n > 0 && n <= MAX_VALUES)
.ok_or_else(|| bad("a tensor has no size or is too large"))
}
fn zeros(n: usize) -> Result<Vec<f32>> {
let mut v = Vec::new();
v.try_reserve_exact(n).map_err(|_| bad("not enough memory for a denoise tile"))?;
v.resize(n, 0.0);
Ok(v)
}
fn sources(op: &Op) -> (usize, Option<usize>) {
match op {
Op::Conv(c) => (c.src, c.src2),
Op::ConvT2(c) => (c.src, None),
Op::MaxPool2 { src } | Op::DepthToSpace2 { src } => (*src, None),
}
}
#[derive(Clone, Copy)]
struct Tensor {
slot: usize,
side: usize,
channels: usize,
}
struct Plan {
net: Net,
tensors: Vec<Tensor>,
sizes: Vec<usize>,
lower: usize,
product: usize,
up_weights: Vec<Option<Vec<f32>>>,
}
struct Workspace {
buffers: Vec<Vec<f32>>,
lower: Vec<f32>,
product: Vec<f32>,
}
/// A checked Net on square CHW tiles. Clones share immutable weights and the idle workspace cache.
#[derive(Clone)]
pub struct NetRunner {
plan: Arc<Plan>,
idle: Arc<Mutex<Vec<Workspace>>>,
}
impl NetRunner {
pub fn new(net: &Net, tile: usize) -> Result<Self> {
if tile == 0
|| tile > MAX_TILE
|| !net.fits_tile(tile)
|| net.in_channels != 4
|| net.out_channels() != 3
|| net.shape(0).is_none_or(|s| s.channels != net.in_channels)
{
return Err(bad("the tile or Bayer-to-RGB contract does not fit the network"));
}
let mut last = vec![0; net.ops().len() + 1];
for (i, op) in net.ops().iter().enumerate() {
let (a, b) = sources(op);
for id in std::iter::once(a).chain(b) {
*last.get_mut(id).ok_or_else(|| bad("missing tensor"))? = i + 1;
}
}
let mut tensors = vec![Tensor { slot: 0, side: tile, channels: net.in_channels }];
let mut sizes = vec![count(net.in_channels, tile)?];
let mut busy = vec![true];
let (mut lower, mut product) = (0, 0);
let mut up_weights = Vec::new();
for (i, op) in net.ops().iter().enumerate() {
let (src, src2) = sources(op);
let a = *tensors.get(src).ok_or_else(|| bad("missing tensor"))?;
let (channels, side) = match op {
Op::Conv(c) => {
if let Some(s) = src2
&& tensors.get(s).is_none_or(|b| b.side != a.side)
{
return Err(bad("a join has different sizes"));
}
(c.cout, a.side)
}
Op::ConvT2(c) => (c.cout, a.side.checked_mul(2).ok_or_else(|| bad("size overflow"))?),
Op::MaxPool2 { .. } => {
if a.side < 2 || !a.side.is_multiple_of(2) {
return Err(bad("pooling does not divide the tile"));
}
(a.channels, a.side / 2)
}
Op::DepthToSpace2 { .. } => (a.channels / 4, a.side.checked_mul(2).ok_or_else(|| bad("size overflow"))?),
};
let n = count(channels, side)?;
let slot = busy
.iter()
.zip(&sizes)
.enumerate()
.filter(|(_, (used, _))| !**used)
.min_by_key(|(_, (_, size))| (n.saturating_sub(**size), **size))
.map_or(busy.len(), |(i, _)| i);
if slot == sizes.len() {
sizes.push(n);
busy.push(true);
} else {
let sz = sizes.get_mut(slot).ok_or_else(|| bad("missing buffer"))?;
*sz = (*sz).max(n);
*busy.get_mut(slot).ok_or_else(|| bad("missing buffer"))? = true;
}
tensors.push(Tensor { slot, side, channels });
// Release sources only after the output has its own buffer (also handles reading the same tensor twice).
for id in std::iter::once(src).chain(src2) {
if last.get(id) == Some(&(i + 1)) {
let t = tensors.get(id).ok_or_else(|| bad("missing tensor"))?;
*busy.get_mut(t.slot).ok_or_else(|| bad("missing buffer"))? = false;
}
}
let (k, m, positions) = match op {
Op::Conv(c) => (c.cin * c.k * c.k, c.cout, a.side * a.side),
Op::ConvT2(c) => (c.cin, c.cout * 4, a.side * a.side),
_ => (0, 0, 0),
};
lower = lower.max(k.saturating_mul(BLOCK.min(positions)));
product = product.max(m.saturating_mul(BLOCK.min(positions)));
let packed = if let Op::ConvT2(c) = op {
let mut w = zeros(c.weight.len())?;
// [cin][cout][dy][dx] -> [cout*4][cin], so an up-convolution is one GEMM and a shuffle.
for (ci, row) in c.weight.chunks_exact(c.cout * 4).enumerate() {
for (co, &v) in row.iter().enumerate() {
*w.get_mut(co * c.cin + ci).ok_or_else(|| bad("invalid transposed weights"))? = v;
}
}
Some(w)
} else {
None
};
up_weights.push(packed);
}
let total = sizes
.iter()
.try_fold(lower.saturating_add(product), |n, &s| n.checked_add(s))
.filter(|&n| n <= MAX_VALUES)
.ok_or_else(|| bad("the tile needs too much working memory"))?;
let _ = total;
Ok(Self { plan: Arc::new(Plan { net: net.clone(), tensors, sizes, lower, product, up_weights }), idle: Arc::new(Mutex::new(Vec::new())) })
}
/// Immutable data for the GPU runner, without re-reading the model.
pub fn net(&self) -> &Net {
&self.plan.net
}
pub fn bytes_per_tile(&self) -> usize {
(self.plan.sizes.iter().sum::<usize>() + self.plan.lower + self.plan.product) * 4
}
fn workspace(&self) -> Result<Workspace> {
if let Some(w) = self.idle.lock().unwrap_or_else(std::sync::PoisonError::into_inner).pop() {
return Ok(w);
}
Ok(Workspace {
buffers: self.plan.sizes.iter().map(|&n| zeros(n)).collect::<Result<_>>()?,
lower: zeros(self.plan.lower)?,
product: zeros(self.plan.product)?,
})
}
fn execute(&self, input: &[f32], w: &mut Workspace) -> Result<Vec<f32>> {
let first = self.plan.tensors.first().ok_or_else(|| bad("no input"))?;
let want = count(first.channels, first.side)?;
w.buffers.first_mut().and_then(|b| b.get_mut(..want)).ok_or_else(|| bad("missing input buffer"))?.copy_from_slice(input);
let profile = std::env::var_os("LIGHTCRAFT_PROFILE").is_some();
for (i, op) in self.plan.net.ops().iter().enumerate() {
let started = web_time::Instant::now();
let out = *self.plan.tensors.get(i + 1).ok_or_else(|| bad("missing output tensor"))?;
let (before, rest) = w.buffers.split_at_mut_checked(out.slot).ok_or_else(|| bad("missing output buffer"))?;
let (dest, after) = rest.split_first_mut().ok_or_else(|| bad("missing output buffer"))?;
let get = |id: usize| -> Result<(&[f32], Tensor)> {
let t = *self.plan.tensors.get(id).ok_or_else(|| bad("missing source tensor"))?;
let data = if t.slot < out.slot {
before.get(t.slot)
} else {
after.get(t.slot.checked_sub(out.slot + 1).ok_or_else(|| bad("aliased buffers"))?)
}
.and_then(|b| b.get(..count(t.channels, t.side).ok()?))
.ok_or_else(|| bad("missing source buffer"))?;
Ok((data, t))
};
let (src, src2) = sources(op);
let (a, shape) = get(src)?;
let b = src2.map(get).transpose()?.map(|v| v.0);
let dest = dest.get_mut(..count(out.channels, out.side)?).ok_or_else(|| bad("short output buffer"))?;
match op {
Op::Conv(c) => conv(c, a, b, shape.side, dest, &mut w.lower, &mut w.product)?,
Op::ConvT2(c) => up(
c,
a,
shape.side,
dest,
self.plan.up_weights.get(i).and_then(Option::as_deref).ok_or_else(|| bad("missing up weights"))?,
&mut w.lower,
&mut w.product,
)?,
Op::MaxPool2 { .. } => pool(a, shape.side, dest)?,
Op::DepthToSpace2 { .. } => shuffle(a, shape.side, dest)?,
}
if dest.iter().any(|v| !v.is_finite()) {
return Err(bad("the model produced non-finite values"));
}
if profile {
eprintln!("[denoise-cpu] layer {i}: {}x{}x{} {:.2} ms", out.channels, out.side, out.side, started.elapsed().as_secs_f64() * 1e3);
}
}
let t = self.plan.tensors.last().ok_or_else(|| bad("no output"))?;
let data = w.buffers.get(t.slot).and_then(|b| b.get(..count(t.channels, t.side).ok()?)).ok_or_else(|| bad("missing result"))?;
let mut out = zeros(data.len())?;
out.copy_from_slice(data);
Ok(out)
}
}
impl TileRunner for NetRunner {
fn run(&self, input: &[f32]) -> std::result::Result<Vec<f32>, Error> {
let first = self.plan.tensors.first().ok_or_else(|| Error::Runtime("missing input shape".into()))?;
let want = count(first.channels, first.side).map_err(|e| Error::Runtime(e.to_string()))?;
if input.len() != want || input.iter().any(|v| !v.is_finite()) {
return Err(Error::Input("the tile has the wrong size or non-finite values".into()));
}
let mut w = self.workspace().map_err(|e| Error::Runtime(e.to_string()))?;
let result = self.execute(input, &mut w).map_err(|e| Error::Runtime(e.to_string()));
let mut idle = self.idle.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if idle.len() < MAX_IDLE {
idle.push(w);
}
result
}
}
fn activate(x: f32, bias: f32, alpha: Option<f32>) -> f32 {
let x = x + bias;
if x < 0.0 { x * alpha.unwrap_or(1.0) } else { x }
}
/// All views are over exact, checked slices. The matrix library's assertions therefore never see file-derived sizes.
fn multiply(w: &[f32], x: &[f32], y: &mut [f32], m: usize, k: usize, n: usize) -> Result<()> {
if m.checked_mul(k) != Some(w.len()) || k.checked_mul(n) != Some(x.len()) || m.checked_mul(n) != Some(y.len()) || m == 0 || k == 0 || n == 0 {
return Err(bad("invalid matrix dimensions"));
}
faer::mul::matmul(
mat::from_row_major_slice_mut::<f32>(y, m, n),
mat::from_row_major_slice::<f32>(w, m, k),
mat::from_row_major_slice::<f32>(x, k, n),
None,
1.0,
Parallelism::None,
);
Ok(())
}
fn lower(a: &[f32], b: Option<&[f32]>, side: usize, k: usize, start: usize, n: usize, dest: &mut [f32]) -> Result<()> {
let pad = k / 2;
let pixels = side * side;
// Whole row segments are copied, so bounds checks and edge branches stay out of the multiply loop.
for (ci, plane) in a.chunks_exact(pixels).chain(b.into_iter().flat_map(|b| b.chunks_exact(pixels))).enumerate() {
for ky in 0..k {
for kx in 0..k {
let row = (ci * k * k + ky * k + kx) * n;
let out = dest.get_mut(row..row + n).ok_or_else(|| bad("short im2col buffer"))?;
let mut at = 0;
while at < n {
let p = start + at;
let (y, x) = (p / side, p % side);
let len = (side - x).min(n - at);
if let Some(yy) = y.checked_add(ky).and_then(|v| v.checked_sub(pad)).filter(|&v| v < side) {
let lo = x.max(pad.saturating_sub(kx));
let hi = (x + len).min((side + pad).saturating_sub(kx));
if lo < hi {
let from = yy * side + lo + kx - pad;
let to = at + lo - x;
out.get_mut(at..to).ok_or_else(|| bad("short im2col padding"))?.fill(0.0);
out.get_mut(to..to + hi - lo)
.ok_or_else(|| bad("short im2col row"))?
.copy_from_slice(plane.get(from..from + hi - lo).ok_or_else(|| bad("short source row"))?);
out.get_mut(to + hi - lo..at + len).ok_or_else(|| bad("short im2col padding"))?.fill(0.0);
} else {
out.get_mut(at..at + len).ok_or_else(|| bad("short im2col row"))?.fill(0.0);
}
} else {
out.get_mut(at..at + len).ok_or_else(|| bad("short im2col row"))?.fill(0.0);
}
at += len;
}
}
}
}
Ok(())
}
fn conv(c: &crate::net::Conv, a: &[f32], b: Option<&[f32]>, side: usize, dest: &mut [f32], scratch: &mut [f32], product: &mut [f32]) -> Result<()> {
let pixels = side * side;
let k = c.cin * c.k * c.k;
for start in (0..pixels).step_by(BLOCK) {
let n = BLOCK.min(pixels - start);
let x = scratch.get_mut(..k * n).ok_or_else(|| bad("short im2col storage"))?;
lower(a, b, side, c.k, start, n, x)?;
let y = product.get_mut(..c.cout * n).ok_or_else(|| bad("short product storage"))?;
multiply(&c.weight, x, y, c.cout, k, n)?;
for ((row, &bias), out) in y.chunks_exact(n).zip(&c.bias).zip(dest.chunks_exact_mut(pixels)) {
let to = out.get_mut(start..start + n).ok_or_else(|| bad("short convolution output"))?;
for (d, &v) in to.iter_mut().zip(row) {
*d = activate(v, bias, c.leaky);
}
}
}
Ok(())
}
fn up(c: &crate::net::ConvT2, a: &[f32], side: usize, dest: &mut [f32], weights: &[f32], scratch: &mut [f32], product: &mut [f32]) -> Result<()> {
let pixels = side * side;
let out_side = 2 * side;
for start in (0..pixels).step_by(BLOCK) {
let n = BLOCK.min(pixels - start);
let x = scratch.get_mut(..c.cin * n).ok_or_else(|| bad("short up input storage"))?;
for (to, from) in x.chunks_exact_mut(n).zip(a.chunks_exact(pixels)) {
to.copy_from_slice(from.get(start..start + n).ok_or_else(|| bad("short up input"))?);
}
let y = product.get_mut(..c.cout * 4 * n).ok_or_else(|| bad("short up product storage"))?;
multiply(weights, x, y, c.cout * 4, c.cin, n)?;
for (co, (&bias, out)) in c.bias.iter().zip(dest.chunks_exact_mut(4 * pixels)).enumerate() {
for d in 0..4 {
let row = y.get((4 * co + d) * n..(4 * co + d + 1) * n).ok_or_else(|| bad("short up product"))?;
for (j, &v) in row.iter().enumerate() {
let p = start + j;
let at = (2 * (p / side) + d / 2) * out_side + 2 * (p % side) + d % 2;
*out.get_mut(at).ok_or_else(|| bad("short up output"))? = activate(v, bias, c.leaky);
}
}
}
}
Ok(())
}
fn pool(a: &[f32], side: usize, dest: &mut [f32]) -> Result<()> {
let s = side / 2;
for (input, out) in a.chunks_exact(side * side).zip(dest.chunks_exact_mut(s * s)) {
for (y, row) in out.chunks_exact_mut(s).enumerate() {
let lo = input.get(2 * y * side..(2 * y + 1) * side).ok_or_else(|| bad("short pool input"))?;
let hi = input.get((2 * y + 1) * side..(2 * y + 2) * side).ok_or_else(|| bad("short pool input"))?;
for ((d, lo), hi) in row.iter_mut().zip(lo.as_chunks::<2>().0).zip(hi.as_chunks::<2>().0) {
let [a, b] = *lo;
let [c, e] = *hi;
*d = a.max(b).max(c).max(e);
}
}
}
Ok(())
}
fn shuffle(a: &[f32], side: usize, dest: &mut [f32]) -> Result<()> {
let pixels = side * side;
for (input, out) in a.chunks_exact(4 * pixels).zip(dest.chunks_exact_mut(4 * pixels)) {
for d in 0..4 {
let plane = input.get(d * pixels..(d + 1) * pixels).ok_or_else(|| bad("short shuffle input"))?;
for (y, row) in plane.chunks_exact(side).enumerate() {
let at = (2 * y + d / 2) * 2 * side;
let to = out.get_mut(at..at + 2 * side).ok_or_else(|| bad("short shuffle output"))?;
for (to, &v) in to.as_chunks_mut::<2>().0.iter_mut().zip(row) {
if let Some(t) = to.get_mut(d % 2) {
*t = v;
}
}
}
}
}
Ok(())
}
#[cfg(test)]
mod tests;
+151
View File
@@ -0,0 +1,151 @@
use super::*;
use crate::{
net::{Conv, ConvT2},
reference, synthetic,
};
fn random(n: usize, seed: u32) -> Vec<f32> {
let mut s = seed | 1;
(0..n)
.map(|_| {
s ^= s << 13;
s ^= s >> 17;
s ^= s << 5;
(s >> 8) as f32 / (1u32 << 23) as f32 - 1.0
})
.collect()
}
fn net(tile: usize, ch: u64, depth: usize, seed: u64) -> Net {
let p = std::env::temp_dir().join(format!("lc-purecpu-{}-{tile}-{ch}-{depth}-{seed}.onnx", std::process::id()));
std::fs::write(&p, synthetic::unet_onnx(tile as u64, ch, depth, seed)).unwrap();
let n = crate::onnx::read(&p).unwrap();
std::fs::remove_file(p).unwrap();
n
}
fn agree(a: &[f32], b: &[f32]) {
assert_eq!(a.len(), b.len());
assert!(a.iter().chain(b).all(|v| v.is_finite()));
let scale = b.iter().fold(1e-6f32, |m, v| m.max(v.abs()));
let error = a.iter().zip(b).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale;
assert!(error <= 1e-3, "relative maximum error {error:e}");
}
#[test]
fn all_operators_match_scalar_on_random_nets_and_odd_and_tiny_shapes() {
for (t, ch, d) in [(1, 3, 0), (3, 5, 0), (5, 7, 0), (6, 3, 1), (20, 5, 2), (36, 9, 2), (64, 8, 3)] {
for seed in [1, 73] {
let n = net(t, ch, d, seed);
let r = NetRunner::new(&n, t).unwrap();
let x = random(4 * t * t, seed as u32);
let want = reference::run(&n, t, &x).unwrap();
agree(&r.run(&x).unwrap(), &want);
assert_eq!(r.run(&x).unwrap(), r.run(&x).unwrap());
}
}
}
#[test]
fn malformed_values_fail_before_execution_and_workspace_sizes_are_bounded() {
let mut n = net(4, 3, 1, 11);
for t in [0, 1, 3, usize::MAX, 4096] {
assert!(NetRunner::new(&n, t).is_err());
}
for ch in [0, 3, usize::MAX] {
n.in_channels = ch;
assert!(NetRunner::new(&n, 4).is_err());
}
let n = Net::new(
4,
vec![
Op::Conv(Conv { src: 0, src2: None, cin: 4, cout: 4096, k: 1, weight: vec![0.1; 4 * 4096], bias: vec![0.0; 4096], leaky: None }),
Op::Conv(Conv { src: 1, src2: None, cin: 4096, cout: 12, k: 1, weight: vec![0.1; 4096 * 12], bias: vec![0.0; 12], leaky: None }),
Op::DepthToSpace2 { src: 2 },
],
)
.unwrap();
assert!(NetRunner::new(&n, 2048).is_err());
let ops = (0..100).map(|i| Op::MaxPool2 { src: i }).collect();
let n = Net::new(4, ops).unwrap();
assert!(NetRunner::new(&n, 16).is_err());
let c = Conv { src: 0, src2: None, cin: 4, cout: 12, k: 1, weight: vec![0.25; 48], bias: vec![0.0; 12], leaky: None };
for mode in 0..6 {
let mut bad = c.clone();
match mode {
0 => bad.src = usize::MAX,
1 => bad.weight.pop().map(|_| ()).unwrap(),
2 => bad.bias[0] = f32::NAN,
3 => bad.leaky = Some(f32::INFINITY),
4 => bad.cout = usize::MAX,
_ => bad.k = usize::MAX,
}
assert!(Net::new(4, vec![Op::Conv(bad), Op::DepthToSpace2 { src: 1 }]).is_err());
}
}
#[test]
fn bad_tiles_and_arithmetic_overflow_are_errors_and_do_not_poison_reuse() {
let c = Conv { src: 0, src2: None, cin: 4, cout: 12, k: 1, weight: vec![f32::MAX; 48], bias: vec![0.0; 12], leaky: None };
let n = Net::new(4, vec![Op::Conv(c), Op::DepthToSpace2 { src: 1 }]).unwrap();
let r = NetRunner::new(&n, 1).unwrap();
assert!(r.run(&[]).is_err());
assert!(r.run(&[f32::NAN; 4]).is_err());
assert!(r.run(&[f32::INFINITY; 4]).is_err());
assert_eq!(r.idle.lock().unwrap().len(), 0, "invalid input never allocates a workspace");
assert!(r.run(&[2.0; 4]).is_err());
assert_eq!(r.run(&[0.0; 4]).unwrap(), vec![0.0; 12]);
// Matrix shape checks are the boundary in front of the dependency's asserting constructors.
assert!(multiply(&[], &[], &mut [], usize::MAX, 2, 2).is_err());
}
#[test]
fn shared_runner_serves_concurrent_tiles_without_cross_contamination() {
let n = net(12, 5, 1, 97);
let r = NetRunner::new(&n, 12).unwrap();
let inputs: Vec<_> = (0..8).map(|s| random(4 * 12 * 12, s)).collect();
let answers: Vec<_> = inputs.iter().map(|x| reference::run(&n, 12, x).unwrap()).collect();
std::thread::scope(|scope| {
let mut handles = Vec::new();
for (x, want) in inputs.iter().zip(&answers) {
let r = &r;
handles.push(scope.spawn(move || {
for _ in 0..3 {
agree(&r.run(x).unwrap(), want);
}
}));
}
for h in handles {
h.join().unwrap();
}
});
}
#[test]
fn join_may_read_the_same_tensor_twice_and_leaky_up_convolution_is_supported() {
let c = ConvT2 { src: 0, cin: 4, cout: 3, weight: random(48, 2), bias: vec![0.1, -0.1, 0.0], leaky: Some(0.13) };
let n = Net::new(
4,
vec![
Op::ConvT2(c),
Op::Conv(Conv { src: 1, src2: Some(1), cin: 6, cout: 3, k: 3, weight: random(162, 3), bias: vec![0.0; 3], leaky: Some(0.0) }),
],
)
.unwrap();
let x = random(4 * 3 * 3, 7);
agree(&NetRunner::new(&n, 3).unwrap().run(&x).unwrap(), &reference::run(&n, 3, &x).unwrap());
}
#[test]
fn reused_im2col_clears_all_edge_padding_between_images() {
for t in [1, 3, 5, 36] {
let n = net(t, 5, 0, 121);
let r = NetRunner::new(&n, t).unwrap();
for x in [vec![1.0; 4 * t * t], vec![0.0; 4 * t * t], random(4 * t * t, 9)] {
agree(&r.run(&x).unwrap(), &reference::run(&n, t, &x).unwrap());
}
}
}
#[test]
#[ignore = "needs LC_DENOISE_MODEL; one tile with LIGHTCRAFT_PROFILE=1"]
fn profile_one_real_tile() {
let model = std::env::var_os("LC_DENOISE_MODEL").expect("set LC_DENOISE_MODEL");
let m = crate::known::find("rawnind-bayer").unwrap().manifest;
let r = crate::runtime::CpuRunner::load(std::path::Path::new(&model), &m).unwrap();
println!("workspace {:.1} MB", r.bytes_per_tile() as f64 / 1e6);
r.run(&crate::runtime::check_tile(512)).unwrap();
}
+37
View File
@@ -0,0 +1,37 @@
//! AI denoise for LightCraft, as a step between the sensor and the demosaic that the rest of the develop
//! pipeline never sees: the raw file stays untouched, the denoised picture is cached data that can always be
//! made again, and the amount is an ordinary develop setting.
//!
//! - [`manifest`]: what a denoise model takes and gives, and who may use it (untrusted JSON, validated).
//! - [`known`] and [`archive`]: the models LightCraft can fetch, and taking the model file out of the `.zip` it comes in.
//! - [`bayer`]: Bayer layouts and the packing of a mosaic into the four planes a model takes.
//! - [`tiles`] and [`run`]: cutting a picture into overlapping tiles for a fixed-size model, bringing each tile's
//! output scale back to the input's, keeping clipped highlights as they were, and blending the tiles together.
//! The model itself is a [`run::TileRunner`], so all of this is tested without one.
//! - [`product`]: the cache file the denoised picture is kept in (half floats, compressed in strips) and the
//! blend with the plain demosaic that the Amount slider controls.
//! - [`cpu`] and [`runtime`]: a single-thread pure-Rust CPU runner and its installation self-test.
//! - [`net`] and [`onnx`]: checked plain network data and bounded ONNX decoding, shared by CPU and GPU.
//!
//! Everything here treats model files, manifests and cache files as hostile input: sizes are capped, numbers must
//! be finite, and a malformed file is an error, never a panic.
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#[cfg(feature = "runtime")]
pub mod cpu;
pub mod net;
pub mod onnx;
mod onnx_proto;
pub mod reference;
#[cfg(feature = "runtime")]
pub mod runtime;
#[doc(hidden)]
pub mod synthetic;
pub use lightcraft_denoise_core::*;
pub mod synthetic_proto;
+345
View File
@@ -0,0 +1,345 @@
//! A denoise network as plain data: the handful of layer kinds a U-Net denoiser is made of, with their weights.
//!
//! [`onnx`](crate::onnx) reads the file into this description, shared by the pure-Rust CPU and GPU runners.
//! Unsupported operators, groups, dilation or activations are reported as errors when loading the model.
//!
//! The description is checked ([`Net::new`]) because it comes from a stranger's file: channel counts must chain,
//! weights must have the sizes the shapes need and be finite, and the depth must stay small.
/// Most layers a network may have.
pub const MAX_OPS: usize = 512;
/// Most channels a layer may have.
pub const MAX_CHANNELS: usize = 4096;
/// Most weights (numbers) a network may have in all.
pub const MAX_WEIGHTS: usize = 128 * 1024 * 1024;
#[derive(Debug, PartialEq, thiserror::Error)]
pub enum NetError {
/// The model uses something this denoise network description cannot express.
#[error("not supported by the denoise runners: {0}")]
Unsupported(String),
/// The model is not a well-formed network.
#[error("not a valid network: {0}")]
Invalid(String),
}
fn invalid<T>(why: impl Into<String>) -> Result<T, NetError> {
Err(NetError::Invalid(why.into()))
}
/// A tensor is named by its number: 0 is the network's input, and layer `i` makes tensor `i + 1`.
pub type TensorId = usize;
/// A convolution with stride 1 and "same" padding (`k` is 1 or 3), over one tensor or two joined along the channels
/// (`src` first), then a bias and optionally a leaky ReLU.
#[derive(Clone, Debug, PartialEq)]
pub struct Conv {
pub src: TensorId,
/// A second source, whose channels follow `src`'s (a skip connection's concatenation).
pub src2: Option<TensorId>,
pub cin: usize,
pub cout: usize,
pub k: usize,
/// `[cout][cin][k][k]`, as ONNX has it.
pub weight: Vec<f32>,
pub bias: Vec<f32>,
/// `Some(alpha)`: a leaky ReLU with this slope follows.
pub leaky: Option<f32>,
}
/// A transposed convolution with a 2 × 2 kernel and stride 2: every input pixel becomes a 2 × 2 block.
#[derive(Clone, Debug, PartialEq)]
pub struct ConvT2 {
pub src: TensorId,
pub cin: usize,
pub cout: usize,
/// `[cin][cout][2][2]`, as ONNX has it.
pub weight: Vec<f32>,
pub bias: Vec<f32>,
pub leaky: Option<f32>,
}
#[derive(Clone, Debug, PartialEq)]
pub enum Op {
Conv(Conv),
ConvT2(ConvT2),
/// 2 × 2 maximum, stride 2.
MaxPool2 {
src: TensorId,
},
/// Depth-to-space with block size 2 in "CRD" order: output channel `c` at `(2y + dy, 2x + dx)` is input channel
/// `4c + 2dy + dx` at `(y, x)`.
DepthToSpace2 {
src: TensorId,
},
}
/// What a tensor looks like: its channels, and its resolution as a power of two below the input's (a negative
/// level is above it).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Shape {
pub channels: usize,
pub level: i32,
}
#[derive(Clone, Debug, PartialEq)]
pub struct Net {
pub in_channels: usize,
ops: Vec<Op>,
shapes: Vec<Shape>,
}
impl Net {
/// Check `ops` (layer `i` makes tensor `i + 1`) over an input of `in_channels` and describe every tensor.
pub fn new(in_channels: usize, ops: Vec<Op>) -> Result<Net, NetError> {
if in_channels == 0 || in_channels > MAX_CHANNELS {
return invalid("the input has no channels or too many");
}
if ops.is_empty() || ops.len() > MAX_OPS {
return invalid("the network has no layers or too many");
}
let mut shapes = vec![Shape { channels: in_channels, level: 0 }];
let mut weights = 0usize;
for (i, op) in ops.iter().enumerate() {
let shape_of = |t: TensorId| {
shapes
.get(t)
.copied()
.filter(|_| t <= i)
.ok_or_else(|| NetError::Invalid(format!("layer {i} reads tensor {t}, which does not exist yet")))
};
let out = match op {
Op::Conv(c) => {
let a = shape_of(c.src)?;
let (cin, level) = match c.src2 {
Some(s2) => {
let b = shape_of(s2)?;
if b.level != a.level {
return invalid(format!("layer {i} joins tensors of different resolutions"));
}
(a.channels + b.channels, a.level)
}
None => (a.channels, a.level),
};
if c.cin != cin || c.cout == 0 || c.cout > MAX_CHANNELS || !(c.k == 1 || c.k == 3) {
return invalid(format!(
"layer {i}: a convolution of {} to {} channels with a {}-wide kernel does not fit",
c.cin, c.cout, c.k
));
}
check_weights(
i,
&c.weight,
c.cout.checked_mul(cin).and_then(|n| n.checked_mul(c.k * c.k)),
&c.bias,
c.cout,
c.leaky,
&mut weights,
)?;
Shape { channels: c.cout, level }
}
Op::ConvT2(c) => {
let a = shape_of(c.src)?;
if c.cin != a.channels || c.cout == 0 || c.cout > MAX_CHANNELS {
return invalid(format!("layer {i}: a transposed convolution of {} to {} channels does not fit", c.cin, c.cout));
}
check_weights(i, &c.weight, c.cin.checked_mul(c.cout).and_then(|n| n.checked_mul(4)), &c.bias, c.cout, c.leaky, &mut weights)?;
Shape { channels: c.cout, level: a.level - 1 }
}
Op::MaxPool2 { src } => {
let a = shape_of(*src)?;
Shape { channels: a.channels, level: a.level + 1 }
}
Op::DepthToSpace2 { src } => {
let a = shape_of(*src)?;
if !a.channels.is_multiple_of(4) {
return invalid(format!("layer {i}: depth-to-space needs a multiple of 4 channels"));
}
Shape { channels: a.channels / 4, level: a.level - 1 }
}
};
shapes.push(out);
}
Ok(Net { in_channels, ops, shapes })
}
/// The layers: layer `i` makes tensor `i + 1`.
pub fn ops(&self) -> &[Op] {
&self.ops
}
/// The shape of tensor `t` (0 is the input).
pub fn shape(&self, t: TensorId) -> Option<Shape> {
self.shapes.get(t).copied()
}
/// The tensor the network gives (the last layer's).
pub fn output(&self) -> TensorId {
self.ops.len()
}
pub fn out_channels(&self) -> usize {
self.shapes.last().map_or(0, |s| s.channels)
}
/// The deepest resolution level: a tile must be a multiple of `2^this` cells wide.
pub fn depth(&self) -> u32 {
self.shapes.iter().map(|s| s.level.max(0) as u32).max().unwrap_or(0)
}
/// Can the network run on a `tile × tile` input and give `[out_channels, 2·tile, 2·tile]`? (Pooling must divide
/// the tile, and the output must be one level above the input.)
pub fn fits_tile(&self, tile: usize) -> bool {
let Some(multiple) = 1usize.checked_shl(self.depth()) else { return false };
tile > 0 && tile.is_multiple_of(multiple) && self.shapes.last().is_some_and(|s| s.level == -1)
}
/// Multiply-adds for one tile, for estimating time.
pub fn macs(&self, tile: usize) -> u64 {
let pixels = |level: i32| -> u64 {
let side = if level >= 0 {
tile.checked_shr(level as u32).unwrap_or(0)
} else {
tile.saturating_mul(1usize.checked_shl(level.unsigned_abs()).unwrap_or(usize::MAX))
};
(side as u64).saturating_mul(side as u64)
};
self.ops
.iter()
.enumerate()
.map(|(i, op)| {
let out = self.shapes.get(i + 1).map_or(0, |s| pixels(s.level));
match op {
Op::Conv(c) => out.saturating_mul((c.cin * c.cout * c.k * c.k) as u64),
// every input pixel makes a 2 × 2 block: as many multiply-adds as the 2 × 2-wide kernel has weights
Op::ConvT2(c) => pixels(self.shapes.get(c.src).map_or(0, |s| s.level)).saturating_mul((c.cin * c.cout * 4) as u64),
_ => 0,
}
})
.fold(0u64, u64::saturating_add)
}
}
fn check_weights(
layer: usize,
weight: &[f32],
want: Option<usize>,
bias: &[f32],
cout: usize,
leaky: Option<f32>,
total: &mut usize,
) -> Result<(), NetError> {
if want != Some(weight.len()) || bias.len() != cout {
return invalid(format!("layer {layer}: the weights are not the size its shape needs"));
}
*total = total.saturating_add(weight.len()).saturating_add(bias.len());
if *total > MAX_WEIGHTS {
return invalid("the network has too many weights");
}
if weight.iter().chain(bias).any(|v| !v.is_finite()) || leaky.is_some_and(|a| !a.is_finite()) {
return invalid(format!("layer {layer}: a weight is not a finite number"));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn conv(src: TensorId, src2: Option<TensorId>, cin: usize, cout: usize, k: usize) -> Op {
Op::Conv(Conv { src, src2, cin, cout, k, weight: vec![0.1; cout * cin * k * k], bias: vec![0.0; cout], leaky: Some(0.2) })
}
fn convt(src: TensorId, cin: usize, cout: usize) -> Op {
Op::ConvT2(ConvT2 { src, cin, cout, weight: vec![0.1; cin * cout * 4], bias: vec![0.0; cout], leaky: None })
}
/// A two-level network of the shape the denoisers have: 4 → 8 channels, pool, 16 channels, up, join, 12, and the
/// 1 × 1 convolution to the colours.
fn unet() -> Vec<Op> {
vec![
conv(0, None, 4, 8, 3), // 1: 8 ch, level 0
Op::MaxPool2 { src: 1 }, // 2: level 1
conv(2, None, 8, 16, 3), // 3: 16 ch, level 1
convt(3, 16, 8), // 4: 8 ch, level 0
conv(4, Some(1), 16, 8, 3), // 5: join [up, skip]
conv(5, None, 8, 12, 1), // 6: 12 ch
Op::DepthToSpace2 { src: 6 }, // 7: 3 ch, level -1
]
}
#[test]
fn deep_networks_and_huge_cost_queries_never_overflow_or_truncate_depth() {
let n = Net::new(4, unet()).unwrap();
assert_eq!(n.macs(usize::MAX), u64::MAX);
assert_eq!(n.macs(0), 0);
let mut ops = Vec::new();
for i in 0..21 {
ops.push(Op::MaxPool2 { src: i });
}
for i in 21..42 {
ops.push(convt(i, 4, 4));
}
ops.push(conv(42, None, 4, 12, 1));
ops.push(Op::DepthToSpace2 { src: 43 });
let n = Net::new(4, ops).unwrap();
assert!(!n.fits_tile(1 << 20));
assert!(n.fits_tile(1 << 21));
let _ = n.macs(16);
}
#[test]
fn a_u_net_is_described_with_its_shapes_and_costs() {
let net = Net::new(4, unet()).unwrap();
assert_eq!(net.out_channels(), 3);
assert_eq!(net.shape(3), Some(Shape { channels: 16, level: 1 }));
assert_eq!(net.shape(7), Some(Shape { channels: 3, level: -1 }));
assert_eq!(net.depth(), 1);
assert!(net.fits_tile(64) && !net.fits_tile(63) && !net.fits_tile(0));
// 64 × 64 cells: layer 1 = 4096·4·8·9, and so on; just check it is the sum of what the layers cost
let t = 64u64;
let want = t * t * 4 * 8 * 9 + (t / 2) * (t / 2) * 8 * 16 * 9 + (t / 2) * (t / 2) * 16 * 8 * 4 + t * t * 16 * 8 * 9 + t * t * 8 * 12;
assert_eq!(net.macs(64), want);
}
#[test]
fn a_network_that_does_not_chain_is_refused() {
// wrong input channel count for the first convolution
assert!(Net::new(3, unet()).is_err());
// a layer reading a tensor that does not exist yet
let mut ops = unet();
ops[2] = conv(9, None, 8, 16, 3);
assert!(Net::new(4, ops).is_err());
// joining tensors of different resolutions
let mut ops = unet();
ops[4] = conv(3, Some(1), 24, 8, 3);
assert!(Net::new(4, ops).is_err());
// weights of the wrong size, and not finite
let mut ops = unet();
if let Op::Conv(c) = &mut ops[0] {
c.weight.pop();
}
assert!(Net::new(4, ops).is_err());
let mut ops = unet();
if let Op::Conv(c) = &mut ops[0] {
c.bias[0] = f32::NAN;
}
assert!(Net::new(4, ops).is_err());
// a kernel that is neither 1 nor 3 wide, no layers, depth-to-space on a channel count it cannot split
let mut ops = unet();
if let Op::Conv(c) = &mut ops[0] {
c.k = 5;
}
assert!(Net::new(4, ops).is_err());
assert!(Net::new(4, vec![]).is_err());
assert!(Net::new(4, vec![Op::DepthToSpace2 { src: 0 }]).is_ok());
assert!(Net::new(4, vec![Op::DepthToSpace2 { src: 0 }, Op::DepthToSpace2 { src: 1 }]).is_err());
assert!(Net::new(2, vec![Op::DepthToSpace2 { src: 0 }]).is_err());
}
#[test]
fn a_network_that_does_not_end_one_level_up_does_not_fit_a_tile() {
let net = Net::new(4, unet()[..6].to_vec()).unwrap();
assert!(!net.fits_tile(64), "no depth-to-space: the output is not at twice the resolution");
}
}
+369
View File
@@ -0,0 +1,369 @@
//! Reading an `.onnx` file into a [`Net`] with bounded, pure-Rust protobuf decoding.
//!
//! Only what the denoise U-Nets are made of is understood: `Conv` (3 × 3 "same" or 1 × 1, stride 1, no groups, no
//! dilation) optionally followed by `LeakyRelu`, `ConvTranspose` (2 × 2, stride 2) likewise, `MaxPool` (2 × 2, stride 2),
//! a two-input `Concat` on the channels that feeds a convolution, and a final `DepthToSpace` (2, `CRD`). Any other
//! operator or attribute is [`NetError::Unsupported`], which is reported to the user by both runners.
use std::collections::HashMap;
use std::path::Path;
use crate::onnx_proto::{self, AttributeProto, NodeProto, TensorProto, ValueInfoProto};
use std::io::Read;
use crate::net::{Conv, ConvT2, MAX_WEIGHTS, Net, NetError, Op, TensorId};
fn unsupported<T>(why: impl Into<String>) -> Result<T, NetError> {
Err(NetError::Unsupported(why.into()))
}
fn invalid<T>(why: impl Into<String>) -> Result<T, NetError> {
Err(NetError::Invalid(why.into()))
}
/// Read the model at `path`.
pub fn read(path: &Path) -> Result<Net, NetError> {
let file = std::fs::File::open(path).map_err(|e| NetError::Invalid(e.to_string()))?;
let size = usize::try_from(file.metadata().map_err(|e| NetError::Invalid(e.to_string()))?.len())
.ok()
.filter(|&n| n <= onnx_proto::MAX_BYTES)
.ok_or_else(|| NetError::Invalid("model file too large".into()))?;
let mut bytes = Vec::new();
bytes.try_reserve_exact(size).map_err(|_| NetError::Invalid("not enough memory to load the model".into()))?;
file.take(onnx_proto::MAX_BYTES as u64 + 1).read_to_end(&mut bytes).map_err(|e| NetError::Invalid(e.to_string()))?;
read_bytes(&bytes)
}
fn read_bytes(bytes: &[u8]) -> Result<Net, NetError> {
let graph = onnx_proto::decode(bytes)?;
let initialisers: HashMap<&str, &TensorProto> = graph.initializer.iter().map(|t| (t.name.as_str(), t)).collect();
if initialisers.len() != graph.initializer.len() {
return invalid("duplicate initializer names");
}
let inputs: Vec<_> = graph.input.iter().filter(|i| !initialisers.contains_key(i.name.as_str())).collect();
let ([input], [output]) = (inputs.as_slice(), graph.output.as_slice()) else { return unsupported("not one input and one output") };
let in_channels = match channel_dim(input) {
Some(c) => c,
None => return unsupported("the input's channel count is not written in the file"),
};
// how many nodes read each name (a graph output counts as a reader)
let mut readers: HashMap<&str, usize> = HashMap::new();
for n in &graph.node {
for i in &n.input {
*readers.entry(i.as_str()).or_default() += 1;
}
}
*readers.entry(output.name.as_str()).or_default() += 1;
let mut ids: HashMap<String, TensorId> = HashMap::new();
ids.insert(input.name.clone(), 0);
// a Concat is not a layer of its own: the convolution that reads it takes both sources
let mut joins: HashMap<String, (TensorId, TensorId)> = HashMap::new();
let mut ops: Vec<Op> = Vec::new();
let mut weights_total = 0usize;
let nodes = &graph.node;
let mut i = 0;
while i < nodes.len() {
let Some(node) = nodes.get(i) else { break };
i += 1;
if node.output.len() != 1 || node.output.first().is_none_or(|n| n.is_empty() || ids.contains_key(n) || joins.contains_key(n)) {
return invalid("a layer has missing, repeated or multiple outputs");
}
let allowed: &[&str] = match node.op_type.as_str() {
"Conv" => &["kernel_shape", "pads", "strides", "dilations", "group", "auto_pad"],
"ConvTranspose" => &["kernel_shape", "pads", "strides", "dilations", "group", "output_padding", "auto_pad"],
"MaxPool" => &["kernel_shape", "pads", "strides", "dilations", "ceil_mode", "storage_order", "auto_pad"],
"Concat" => &["axis"],
"DepthToSpace" => &["blocksize", "mode"],
_ => &[],
};
if node.attribute.iter().any(|a| !allowed.contains(&a.name.as_str())) {
return unsupported("an unknown operator attribute");
}
let this = ops.len() + 1;
let lookup = |name: &str| -> Result<TensorId, NetError> {
ids.get(name).copied().ok_or_else(|| NetError::Invalid(format!("`{}` reads `{name}`, which nothing makes", node.op_type)))
};
// a LeakyRelu straight after this layer, which nothing else reads, is part of it
let mut fuse_leaky = |out: &str| -> Result<(Option<f32>, String), NetError> {
if let Some(next) = nodes.get(i)
&& next.op_type == "LeakyRelu"
&& next.input.first().map(String::as_str) == Some(out)
&& readers.get(out).copied() == Some(1)
{
if next.input.len() != 1 || next.output.len() != 1 || next.attribute.iter().any(|a| a.name != "alpha") {
return invalid("malformed LeakyRelu");
}
let alpha = attr(next, "alpha").map_or(0.01, |a| a.f);
i += 1;
let name = next.output.first().cloned().ok_or_else(|| NetError::Invalid("LeakyRelu without an output".into()))?;
return Ok((Some(alpha), name));
}
Ok((None, out.to_string()))
};
match node.op_type.as_str() {
"Concat" => {
if attr(node, "axis").map(|a| a.i) != Some(1) {
return unsupported("Concat on another axis than the channels");
}
let [a, b] = node.input.as_slice() else { return unsupported("Concat of other than two tensors") };
let out = node.output.first().ok_or_else(|| NetError::Invalid("Concat without an output".into()))?;
if readers.get(out.as_str()).copied() != Some(1) {
return unsupported("a Concat that is read by more than one layer");
}
joins.insert(out.clone(), (lookup(a)?, lookup(b)?));
}
"Conv" => {
let (w, b) = weights_of(node, &initialisers, &mut weights_total)?;
let (cout, cin, kh, kw) = four(&w.0)?;
let b = if b.is_empty() { vec![0.0; cout] } else { b };
if kh != kw || !(kh == 1 || kh == 3) {
return unsupported(format!("a {kh} × {kw} convolution"));
}
if ints_or(node, "strides", &[1, 1]) != [1, 1]
|| ints_or(node, "dilations", &[1, 1]) != [1, 1]
|| attr(node, "group").is_some_and(|a| a.i != 1)
{
return unsupported("a convolution with strides, dilation or groups");
}
if attr(node, "kernel_shape").is_some_and(|a| a.ints != [kh as i64, kw as i64]) {
return invalid("kernel shape does not match the weights");
}
let pad = (kh as i64 - 1) / 2;
if ints_or(node, "pads", &[0; 4]) != [pad; 4] || attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") {
return unsupported("a convolution that is not padded to keep its size");
}
let src_name = node.input.first().ok_or_else(|| NetError::Invalid("Conv without an input".into()))?;
let (src, src2) = match joins.remove(src_name) {
Some((a, b)) => (a, Some(b)),
None => (lookup(src_name)?, None),
};
let out = node.output.first().ok_or_else(|| NetError::Invalid("Conv without an output".into()))?;
let (leaky, out_name) = fuse_leaky(out)?;
ids.insert(out_name, this);
ops.push(Op::Conv(Conv { src, src2, cin, cout, k: kh, weight: w.1, bias: b, leaky }));
}
"ConvTranspose" => {
let (w, b) = weights_of(node, &initialisers, &mut weights_total)?;
let (cin, cout, kh, kw) = four(&w.0)?;
let b = if b.is_empty() { vec![0.0; cout] } else { b };
if (kh, kw) != (2, 2)
|| ints_or(node, "strides", &[1, 1]) != [2, 2]
|| ints_or(node, "pads", &[0; 4]) != [0; 4]
|| ints_or(node, "dilations", &[1, 1]) != [1, 1]
|| attr(node, "group").is_some_and(|a| a.i != 1)
|| attr(node, "output_padding").is_some_and(|a| a.ints.iter().any(|&v| v != 0))
{
return unsupported("a transposed convolution other than 2 × 2 with stride 2");
}
if attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") {
return unsupported("transposed convolution auto padding");
}
if attr(node, "kernel_shape").is_some_and(|a| a.ints != [2, 2]) {
return invalid("transposed kernel shape does not match the weights");
}
let src = lookup(node.input.first().map_or("", String::as_str))?;
let out = node.output.first().ok_or_else(|| NetError::Invalid("ConvTranspose without an output".into()))?;
let (leaky, out_name) = fuse_leaky(out)?;
ids.insert(out_name, this);
ops.push(Op::ConvT2(ConvT2 { src, cin, cout, weight: w.1, bias: b, leaky }));
}
"MaxPool" => {
if ints_or(node, "kernel_shape", &[]) != [2, 2]
|| ints_or(node, "strides", &[1, 1]) != [2, 2]
|| ints_or(node, "pads", &[0; 4]) != [0; 4]
|| ints_or(node, "dilations", &[1, 1]) != [1, 1]
|| attr(node, "ceil_mode").is_some_and(|a| a.i != 0)
|| attr(node, "storage_order").is_some_and(|a| a.i != 0)
|| attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET")
{
return unsupported("a max-pool other than 2 × 2 with stride 2");
}
let src = lookup(node.input.first().map_or("", String::as_str))?;
let out = node.output.first().ok_or_else(|| NetError::Invalid("MaxPool without an output".into()))?;
ids.insert(out.clone(), this);
ops.push(Op::MaxPool2 { src });
}
"DepthToSpace" => {
if attr(node, "blocksize").map(|a| a.i) != Some(2) || attr(node, "mode").map(|a| a.s.as_slice()) != Some(&b"CRD"[..]) {
return unsupported("depth-to-space other than block size 2 in CRD order");
}
let src = lookup(node.input.first().map_or("", String::as_str))?;
let out = node.output.first().ok_or_else(|| NetError::Invalid("DepthToSpace without an output".into()))?;
ids.insert(out.clone(), this);
ops.push(Op::DepthToSpace2 { src });
}
other => return unsupported(format!("the `{other}` operator")),
}
}
if !joins.is_empty() {
return unsupported("a Concat whose result no convolution reads");
}
if ids.get(output.name.as_str()) != Some(&ops.len()) {
return unsupported("the output is not the last layer");
}
Net::new(in_channels, ops)
}
fn attr<'a>(n: &'a NodeProto, name: &str) -> Option<&'a AttributeProto> {
n.attribute.iter().find(|a| a.name == name)
}
/// The integer list attribute `name`, or `default` when the node does not have it.
fn ints_or(n: &NodeProto, name: &str, default: &[i64]) -> Vec<i64> {
attr(n, name).map_or_else(|| default.to_vec(), |a| a.ints.clone())
}
fn four(dims: &[i64]) -> Result<(usize, usize, usize, usize), NetError> {
match dims {
[a, b, c, d] if dims.iter().all(|&v| v > 0 && v <= 1 << 20) => Ok((*a as usize, *b as usize, *c as usize, *d as usize)),
_ => invalid("a weight tensor that is not four-dimensional"),
}
}
type Weights = ((Vec<i64>, Vec<f32>), Vec<f32>);
/// The weight and bias initialisers of a convolution node.
fn weights_of(node: &NodeProto, initialisers: &HashMap<&str, &TensorProto>, total: &mut usize) -> Result<Weights, NetError> {
let mut get = |idx: usize| -> Result<(Vec<i64>, Vec<f32>), NetError> {
let name = node.input.get(idx).ok_or_else(|| NetError::Invalid(format!("`{}` without its weights", node.op_type)))?;
let t = initialisers.get(name.as_str()).ok_or_else(|| NetError::Unsupported(format!("weights that are computed (`{name}`)")))?;
let values = floats(t)?;
*total = total.saturating_add(values.len());
if *total > MAX_WEIGHTS {
return invalid("the network has too many weights");
}
Ok((t.dims.clone(), values))
};
let w = get(1)?;
let b = if node.input.len() > 2 {
let (dims, values) = get(2)?;
if dims.len() != 1 || values.is_empty() {
return invalid("a bias tensor must be a nonempty vector");
}
values
} else {
Vec::new()
};
Ok((w, b))
}
fn floats(t: &TensorProto) -> Result<Vec<f32>, NetError> {
if t.data_type != 1 {
return unsupported("weights that are not 32-bit floats");
}
let want: usize = t
.dims
.iter()
.try_fold(1usize, |n, &d| usize::try_from(d).ok().and_then(|d| n.checked_mul(d)))
.ok_or_else(|| NetError::Invalid("a tensor with a negative or huge size".into()))?;
if want > MAX_WEIGHTS {
return invalid("a tensor with too many numbers");
}
if !t.raw_data.is_empty() && (!t.float_data.is_empty() || !t.raw_data.len().is_multiple_of(4)) {
return invalid("ambiguous or truncated tensor data");
}
let actual = if t.raw_data.is_empty() { t.float_data.len() } else { t.raw_data.len() / 4 };
if actual != want {
return invalid("tensor data does not match its dimensions");
}
let mut values = Vec::new();
values.try_reserve_exact(want).map_err(|_| NetError::Invalid("not enough memory for weights".into()))?;
if !t.raw_data.is_empty() {
values.extend(t.raw_data.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)));
} else {
values.extend_from_slice(&t.float_data);
}
if values.len() != want {
return invalid(format!("tensor `{}` has {} numbers, not the {want} its shape says", t.name, values.len()));
}
Ok(values)
}
/// The channel count in an input's declared shape `[batch, channels, …]`.
fn channel_dim(v: &ValueInfoProto) -> Option<usize> {
v.channels
}
#[cfg(test)]
mod tests {
use super::*;
use crate::synthetic::unet_onnx;
fn temp_model(name: &str, bytes: &[u8]) -> std::path::PathBuf {
let p = std::env::temp_dir().join(format!("lc-denoise-onnx-{name}-{}.onnx", std::process::id()));
std::fs::write(&p, bytes).unwrap();
p
}
#[test]
fn a_u_net_is_read_with_its_layers_fused() {
let (tile, depth) = (32usize, 2usize);
let p = temp_model("unet", &unet_onnx(tile as u64, 8, depth, 7));
let net = read(&p).unwrap();
// per level two convolutions; a pool per level but the last; going up a transposed convolution and two convolutions
// per level (the join is part of the first); then the 1 × 1 convolution and depth-to-space
assert_eq!(net.ops().len(), 2 * (depth + 1) + depth + depth * 3 + 2);
assert!(matches!(net.ops().last(), Some(crate::net::Op::DepthToSpace2 { .. })));
assert!(net.ops().iter().any(|o| matches!(o, crate::net::Op::Conv(c) if c.src2.is_some())), "the joins became two-source convolutions");
assert!(
net.ops().iter().all(|o| !matches!(o, crate::net::Op::Conv(c) if c.leaky.is_none() && c.k == 3)),
"every 3 × 3 convolution has its activation"
);
assert_eq!((net.in_channels, net.out_channels(), net.depth()), (4, 3, 2));
assert!(net.fits_tile(tile) && !net.fits_tile(30));
let _ = std::fs::remove_file(p);
}
#[test]
fn an_explicit_empty_or_nonvector_bias_is_not_treated_as_missing() {
let w = TensorProto { name: "w".into(), dims: vec![12, 4, 1, 1], data_type: 1, float_data: vec![0.1; 48], ..Default::default() };
let n = NodeProto { input: vec!["data".into(), "w".into(), "b".into()], ..Default::default() };
for dims in [vec![0], vec![3, 4]] {
let b = TensorProto {
name: "b".into(),
float_data: vec![0.0; dims.iter().product::<i64>() as usize],
dims,
data_type: 1,
..Default::default()
};
let inits = HashMap::from([("w", &w), ("b", &b)]);
assert!(weights_of(&n, &inits, &mut 0).is_err());
}
}
#[test]
fn truncated_and_mutated_protobuf_is_bounded_and_never_panics() {
let good = unet_onnx(4, 3, 1, 7);
for n in (0..good.len()).step_by(31) {
let _ = read_bytes(&good[..n]);
}
for at in (0..good.len()).step_by(37) {
let mut bad = good.clone();
bad[at] ^= 0xff;
let _ = read_bytes(&bad);
}
assert!(read_bytes(&[0x3a, 0xff, 0xff, 0xff, 0xff, 0x7f]).is_err());
assert!(read_bytes(&[0x3a, 0, 0x3a, 0]).is_err());
}
proptest::proptest! {
#[test]
fn arbitrary_model_bytes_never_panic(bytes in proptest::collection::vec(proptest::prelude::any::<u8>(), 0..4096)) {
let _ = read_bytes(&bytes);
}
}
#[test]
fn what_the_description_cannot_say_is_unsupported_and_garbage_is_an_error() {
// not a model
let p = temp_model("junk", b"this is not an onnx file");
assert!(matches!(read(&p), Err(NetError::Invalid(_)) | Err(NetError::Unsupported(_))));
let _ = std::fs::remove_file(p);
// a missing file
assert!(read(Path::new("/does/not/exist.onnx")).is_err());
}
}
+295
View File
@@ -0,0 +1,295 @@
//! Bounded, borrowed decoding of the ONNX fields used by denoise, from the public Apache-2.0 schema:
//! https://github.com/onnx/onnx/blob/main/onnx/onnx.proto3
//! No generated code, protoc, recursive graph decoding, or external tensor files.
use crate::net::{MAX_OPS, MAX_WEIGHTS, NetError};
const MAX_ITEMS: usize = MAX_OPS * 4;
const MAX_STRING: usize = 4096;
pub(super) const MAX_BYTES: usize = MAX_WEIGHTS * 4 + 16 * 1024 * 1024;
type Result<T> = std::result::Result<T, NetError>;
fn bad() -> NetError {
NetError::Invalid("malformed or oversized ONNX protobuf".into())
}
struct Fields<'a> {
rest: &'a [u8],
count: usize,
}
enum Wire<'a> {
Int(u64),
Bytes(&'a [u8]),
F32(f32),
Other,
}
fn varint(rest: &mut &[u8]) -> Result<u64> {
let mut value = 0u64;
for shift in (0..70).step_by(7) {
let (b, tail) = rest.split_first().ok_or_else(bad)?;
*rest = tail;
if shift == 63 && *b > 1 {
return Err(bad());
}
value |= u64::from(*b & 127) << shift;
if b & 128 == 0 {
return Ok(value);
}
}
Err(bad())
}
impl<'a> Fields<'a> {
fn new(rest: &'a [u8]) -> Self {
Self { rest, count: 0 }
}
fn take(&mut self, n: usize) -> Result<&'a [u8]> {
let out = self.rest.get(..n).ok_or_else(bad)?;
self.rest = self.rest.get(n..).ok_or_else(bad)?;
Ok(out)
}
fn next(&mut self) -> Result<Option<(u32, Wire<'a>)>> {
if self.rest.is_empty() {
return Ok(None);
}
self.count += 1;
if self.count > MAX_WEIGHTS {
return Err(bad());
}
let tag = varint(&mut self.rest)?;
let number = u32::try_from(tag >> 3).map_err(|_| bad())?;
if number == 0 || number >= 1 << 29 {
return Err(bad());
}
let wire = match tag & 7 {
0 => Wire::Int(varint(&mut self.rest)?),
1 => {
self.take(8)?;
Wire::Other
}
2 => {
let n = usize::try_from(varint(&mut self.rest)?).map_err(|_| bad())?;
Wire::Bytes(self.take(n)?)
}
5 => {
let b: [u8; 4] = self.take(4)?.try_into().map_err(|_| bad())?;
Wire::F32(f32::from_le_bytes(b))
}
_ => return Err(bad()),
};
Ok(Some((number, wire)))
}
}
fn string(b: &[u8]) -> Result<String> {
if b.len() > MAX_STRING {
return Err(bad());
}
std::str::from_utf8(b).map(str::to_owned).map_err(|_| bad())
}
fn push<T>(v: &mut Vec<T>, value: T, cap: usize) -> Result<()> {
if v.len() >= cap {
return Err(bad());
}
v.try_reserve(1).map_err(|_| bad())?;
v.push(value);
Ok(())
}
fn ints(w: Wire<'_>, out: &mut Vec<i64>, cap: usize) -> Result<()> {
match w {
Wire::Int(v) => push(out, v as i64, cap),
Wire::Bytes(mut b) => {
while !b.is_empty() {
let v = varint(&mut b)?;
push(out, v as i64, cap)?;
}
Ok(())
}
_ => Err(bad()),
}
}
#[derive(Default)]
pub(super) struct AttributeProto {
pub name: String,
pub f: f32,
pub i: i64,
pub s: Vec<u8>,
pub ints: Vec<i64>,
}
#[derive(Default)]
pub(super) struct NodeProto {
pub input: Vec<String>,
pub output: Vec<String>,
pub op_type: String,
pub attribute: Vec<AttributeProto>,
}
#[derive(Default)]
pub(super) struct TensorProto<'a> {
pub name: String,
pub dims: Vec<i64>,
pub data_type: i32,
pub raw_data: &'a [u8],
pub float_data: Vec<f32>,
}
pub(super) struct ValueInfoProto {
pub name: String,
pub channels: Option<usize>,
}
#[derive(Default)]
pub(super) struct Graph<'a> {
pub node: Vec<NodeProto>,
pub initializer: Vec<TensorProto<'a>>,
pub input: Vec<ValueInfoProto>,
pub output: Vec<ValueInfoProto>,
}
fn attribute(b: &[u8]) -> Result<AttributeProto> {
let mut a = AttributeProto::default();
let mut fields = Fields::new(b);
while let Some((n, w)) = fields.next()? {
match (n, w) {
(1, Wire::Bytes(b)) => a.name = string(b)?,
(2, Wire::F32(v)) => a.f = v,
(3, Wire::Int(v)) => a.i = v as i64,
(4, Wire::Bytes(b)) => {
if b.len() > MAX_STRING {
return Err(bad());
}
a.s = b.to_vec();
}
(8, w) => ints(w, &mut a.ints, 16)?,
(20, Wire::Int(1 | 2 | 3 | 7)) => {}
_ => return Err(NetError::Unsupported("unknown attribute value".into())),
}
}
Ok(a)
}
fn node(b: &[u8]) -> Result<NodeProto> {
let mut a = NodeProto::default();
let mut fields = Fields::new(b);
while let Some((n, w)) = fields.next()? {
match (n, w) {
(1, Wire::Bytes(b)) => push(&mut a.input, string(b)?, 4)?,
(2, Wire::Bytes(b)) => push(&mut a.output, string(b)?, 2)?,
(4, Wire::Bytes(b)) => a.op_type = string(b)?,
(5, Wire::Bytes(b)) => push(&mut a.attribute, attribute(b)?, 16)?,
(7, Wire::Bytes(b)) if !b.is_empty() => return Err(NetError::Unsupported("a custom operator domain".into())),
_ => {}
}
}
Ok(a)
}
fn tensor(b: &[u8]) -> Result<TensorProto<'_>> {
let mut a = TensorProto::default();
let mut fields = Fields::new(b);
while let Some((n, w)) = fields.next()? {
match (n, w) {
(1, w) => ints(w, &mut a.dims, 8)?,
(2, Wire::Int(v)) => a.data_type = i32::try_from(v).map_err(|_| bad())?,
(4, Wire::F32(v)) => push(&mut a.float_data, v, MAX_WEIGHTS)?,
(4, Wire::Bytes(b)) => {
if !b.len().is_multiple_of(4) {
return Err(bad());
}
for chunk in b.as_chunks::<4>().0 {
push(&mut a.float_data, f32::from_le_bytes(*chunk), MAX_WEIGHTS)?;
}
}
(8, Wire::Bytes(b)) => a.name = string(b)?,
(9, Wire::Bytes(b)) => {
if b.len() > MAX_WEIGHTS * 4 {
return Err(bad());
}
a.raw_data = b;
}
(13, _) | (14, Wire::Int(1..)) => return Err(NetError::Unsupported("external tensor data".into())),
_ => {}
}
}
Ok(a)
}
fn channel_shape(b: &[u8]) -> Result<Option<usize>> {
let mut fields = Fields::new(b);
let mut dim = 0;
let mut channels = None;
while let Some((n, w)) = fields.next()? {
if let (1, Wire::Bytes(b)) = (n, w) {
if dim >= 8 {
return Err(bad());
}
let mut df = Fields::new(b);
while let Some((n, w)) = df.next()? {
if let (1, Wire::Int(v)) = (n, w)
&& dim == 1
{
channels = usize::try_from(v).ok().filter(|&v| v > 0);
}
}
dim += 1;
}
}
Ok(channels)
}
fn channel_type(b: &[u8]) -> Result<Option<usize>> {
let mut fields = Fields::new(b);
while let Some((n, w)) = fields.next()? {
if let (1, Wire::Bytes(b)) = (n, w) {
let mut tf = Fields::new(b);
let (mut elem, mut channels) = (0, None);
while let Some((n, w)) = tf.next()? {
match (n, w) {
(1, Wire::Int(v)) => elem = v,
(2, Wire::Bytes(b)) => channels = channel_shape(b)?,
_ => {}
}
}
return Ok(if elem == 1 { channels } else { None });
}
}
Ok(None)
}
fn value_info(b: &[u8]) -> Result<ValueInfoProto> {
let mut out = ValueInfoProto { name: String::new(), channels: None };
let mut fields = Fields::new(b);
while let Some((n, w)) = fields.next()? {
match (n, w) {
(1, Wire::Bytes(b)) => out.name = string(b)?,
(2, Wire::Bytes(b)) => out.channels = channel_type(b)?,
_ => {}
}
}
Ok(out)
}
fn graph(b: &[u8]) -> Result<Graph<'_>> {
let mut out = Graph::default();
let mut fields = Fields::new(b);
while let Some((n, w)) = fields.next()? {
match (n, w) {
(1, Wire::Bytes(b)) => push(&mut out.node, node(b)?, MAX_ITEMS)?,
(5, Wire::Bytes(b)) => push(&mut out.initializer, tensor(b)?, MAX_ITEMS)?,
(11, Wire::Bytes(b)) => push(&mut out.input, value_info(b)?, MAX_ITEMS)?,
(12, Wire::Bytes(b)) => push(&mut out.output, value_info(b)?, 1)?,
(15, _) => return Err(NetError::Unsupported("sparse initializers".into())),
_ => {}
}
}
Ok(out)
}
pub(super) fn decode(b: &[u8]) -> Result<Graph<'_>> {
if b.len() > MAX_BYTES {
return Err(bad());
}
let mut fields = Fields::new(b);
let mut out = None;
while let Some((n, w)) = fields.next()? {
if let (7, Wire::Bytes(b)) = (n, w) {
if out.is_some() {
return Err(bad());
}
out = Some(graph(b)?);
}
}
out.ok_or_else(bad)
}
+133
View File
@@ -0,0 +1,133 @@
//! A plain-loop interpreter of a [`Net`], slow and obviously right: the reference the parser and the GPU runner are
//! tested against (tract's answer on the real model is the other). It is not used on photos.
use crate::net::{Net, NetError, Op, TensorId};
use crate::run::{Error, TileRunner};
/// A tensor in `[channels][side][side]` order.
struct T {
channels: usize,
side: usize,
data: Vec<f32>,
}
/// Run `net` on `input` (`in_channels` planes of `tile × tile`).
pub fn run(net: &Net, tile: usize, input: &[f32]) -> Result<Vec<f32>, NetError> {
if !net.fits_tile(tile) || input.len() != net.in_channels * tile * tile {
return Err(NetError::Invalid("the tile does not fit the network".into()));
}
let mut tensors: Vec<T> = vec![T { channels: net.in_channels, side: tile, data: input.to_vec() }];
let get = |tensors: &[T], id: TensorId| -> Result<usize, NetError> {
if id < tensors.len() { Ok(id) } else { Err(NetError::Invalid("a layer reads a tensor that does not exist yet".into())) }
};
for op in net.ops() {
let out = match op {
Op::Conv(c) => {
let a = get(&tensors, c.src)?;
let b = c.src2.map(|s| get(&tensors, s)).transpose()?;
let side = tensors[a].side;
let pad = (c.k - 1) / 2;
let mut data = vec![0f32; c.cout * side * side];
// the input channels in order: the first source's, then the second's
let planes: Vec<&[f32]> =
std::iter::once(&tensors[a]).chain(b.map(|b| &tensors[b])).flat_map(|t| t.data.chunks(side * side)).collect();
for co in 0..c.cout {
for y in 0..side {
for x in 0..side {
let mut sum = c.bias[co];
for (ci, plane) in planes.iter().enumerate() {
for ky in 0..c.k {
for kx in 0..c.k {
let (yy, xx) = ((y + ky) as isize - pad as isize, (x + kx) as isize - pad as isize);
if yy < 0 || xx < 0 || yy >= side as isize || xx >= side as isize {
continue;
}
sum += plane[yy as usize * side + xx as usize] * c.weight[((co * c.cin + ci) * c.k + ky) * c.k + kx];
}
}
}
data[(co * side + y) * side + x] = leaky(sum, c.leaky);
}
}
}
T { channels: c.cout, side, data }
}
Op::ConvT2(c) => {
let a = get(&tensors, c.src)?;
let side = tensors[a].side;
let out_side = 2 * side;
let mut data = vec![0f32; c.cout * out_side * out_side];
for co in 0..c.cout {
for y in 0..side {
for x in 0..side {
for dy in 0..2 {
for dx in 0..2 {
let mut sum = c.bias[co];
for ci in 0..c.cin {
sum += tensors[a].data[(ci * side + y) * side + x] * c.weight[((ci * c.cout + co) * 2 + dy) * 2 + dx];
}
data[(co * out_side + 2 * y + dy) * out_side + 2 * x + dx] = leaky(sum, c.leaky);
}
}
}
}
}
T { channels: c.cout, side: out_side, data }
}
Op::MaxPool2 { src } => {
let a = get(&tensors, *src)?;
let (channels, side) = (tensors[a].channels, tensors[a].side / 2);
let mut data = vec![0f32; channels * side * side];
for ch in 0..channels {
for y in 0..side {
for x in 0..side {
let at = |dy: usize, dx: usize| tensors[a].data[(ch * 2 * side + 2 * y + dy) * 2 * side + 2 * x + dx];
data[(ch * side + y) * side + x] = at(0, 0).max(at(0, 1)).max(at(1, 0)).max(at(1, 1));
}
}
}
T { channels, side, data }
}
Op::DepthToSpace2 { src } => {
let a = get(&tensors, *src)?;
let (channels, side) = (tensors[a].channels / 4, tensors[a].side);
let out_side = 2 * side;
let mut data = vec![0f32; channels * out_side * out_side];
for ch in 0..channels {
for y in 0..side {
for x in 0..side {
for dy in 0..2 {
for dx in 0..2 {
data[(ch * out_side + 2 * y + dy) * out_side + 2 * x + dx] =
tensors[a].data[((4 * ch + 2 * dy + dx) * side + y) * side + x];
}
}
}
}
}
T { channels, side: out_side, data }
}
};
tensors.push(out);
}
tensors.pop().map(|t| t.data).ok_or_else(|| NetError::Invalid("no layers".into()))
}
fn leaky(v: f32, alpha: Option<f32>) -> f32 {
match alpha {
Some(a) if v < 0.0 => v * a,
_ => v,
}
}
/// The reference as a [`TileRunner`] (for tests of the code around the model).
pub struct ReferenceRunner {
pub net: Net,
pub tile: usize,
}
impl TileRunner for ReferenceRunner {
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
run(&self.net, self.tile, input).map_err(|e| Error::Runtime(e.to_string()))
}
}
+433
View File
@@ -0,0 +1,433 @@
//! Model loading, a shared installation self-test, and deterministic test tiles.
//! `CpuRunner` is the pure-Rust executor; scalar operator tests live in `reference`.
use std::path::Path;
use web_time::Instant;
use serde::Serialize;
use crate::manifest::{DenoiserManifest, Gain};
use crate::run::{Error, TileRunner};
/// What `self_test` found.
#[derive(Clone, Debug, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct SelfTest {
pub ok: bool,
/// Milliseconds to load and optimise the model.
pub load_ms: f64,
/// Median milliseconds to run one tile.
pub tile_ms: f64,
/// Megapixels of picture one tile covers, for estimating a whole photo's time.
pub tile_megapixels: f64,
/// The model's output scale relative to its input on the test picture (what `Gain::MatchMean` takes out).
pub scale: f64,
/// Each check: what was checked and whether it held.
pub checks: Vec<(String, bool)>,
}
/// The default pure-Rust CPU model, with the same manifest and installation self-test as the reference.
#[derive(Clone)]
pub struct CpuRunner {
runner: crate::cpu::NetRunner,
manifest: DenoiserManifest,
load_ms: f64,
}
impl CpuRunner {
pub fn load(path: &Path, manifest: &DenoiserManifest) -> Result<Self, Error> {
crate::manifest::validate(manifest).map_err(|e| Error::Load(e.to_string()))?;
let started = Instant::now();
let net = crate::onnx::read(path).map_err(|e| Error::Load(e.to_string()))?;
let runner = crate::cpu::NetRunner::new(&net, manifest.tile as usize).map_err(|e| Error::Load(e.to_string()))?;
Ok(Self { runner, manifest: manifest.clone(), load_ms: started.elapsed().as_secs_f64() * 1e3 })
}
pub fn manifest(&self) -> &DenoiserManifest {
&self.manifest
}
pub fn net(&self) -> &crate::net::Net {
self.runner.net()
}
pub fn bytes_per_tile(&self) -> usize {
self.runner.bytes_per_tile()
}
pub fn self_test(&self) -> SelfTest {
self_test(self, &self.manifest, self.load_ms)
}
}
impl TileRunner for CpuRunner {
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
self.runner.run(input)
}
}
fn self_test(runner: &dyn TileRunner, manifest: &DenoiserManifest, load_ms: f64) -> SelfTest {
let t = manifest.tile as usize;
let mut checks: Vec<(String, bool)> = Vec::new();
let (input, base) = test_tile(t);
let (mut times, mut scale) = (Vec::new(), 0.0f64);
let mut run = || {
let started = Instant::now();
let r = runner.run(&input);
times.push(started.elapsed().as_secs_f64() * 1000.0);
r
};
let (a, b) = (run(), run());
match (&a, &b) {
(Ok(a), Ok(b)) => {
checks.push((format!("gives {} finite numbers per tile", a.len()), true));
checks.push((
"gives the same answer for the same tile".into(),
a.iter().zip(b).all(|(x, y)| (x - y).abs() <= 1e-4 * x.abs().max(y.abs()).max(1e-9)),
));
let side = 2 * t;
let mean_in = base.iter().map(|&v| f64::from(v)).sum::<f64>() / base.len().max(1) as f64;
let mean_out = a.iter().map(|&v| f64::from(v)).sum::<f64>() / a.len().max(1) as f64;
scale = if mean_in > 0.0 { mean_out / mean_in } else { 0.0 };
if let Gain::MatchMean { nominal, max_deviation } = manifest.gain {
let ratio = scale / f64::from(nominal);
let tolerance = f64::from(max_deviation).max(0.05) * 4.0;
checks.push((format!("its scale is the {nominal:.3e} the description says (found {scale:.3e})"), (ratio - 1.0).abs() <= tolerance));
}
// noise left after bringing the answer back to the input's scale, against the noise that went in
let g = if scale > 0.0 { 1.0 / scale } else { 1.0 };
let (mut noise_in, mut noise_out, mut n_in, mut n_out) = (0.0f64, 0.0f64, 0usize, 0usize);
for (i, (&v, &s)) in input.iter().zip(base.iter().cycle()).enumerate() {
// centre of each plane only: edges are the model's guess
let (x, y) = ((i % (t * t)) % t, (i % (t * t)) / t);
if x >= t / 4 && x < 3 * t / 4 && y >= t / 4 && y < 3 * t / 4 {
noise_in += f64::from(v - s).powi(2);
n_in += 1;
}
}
for c in 0..3 {
for y in side / 4..3 * side / 4 {
for x in side / 4..3 * side / 4 {
if let (Some(&o), Some(&s)) = (a.get(c * side * side + y * side + x), base.get((y / 2) * t + x / 2)) {
noise_out += (f64::from(o) * g - f64::from(s)).powi(2);
n_out += 1;
}
}
}
}
let (rms_in, rms_out) = ((noise_in / n_in.max(1) as f64).sqrt(), (noise_out / n_out.max(1) as f64).sqrt());
checks.push((format!("takes noise down (from {rms_in:.4} to {rms_out:.4})"), rms_out < 0.8 * rms_in));
}
(Err(e), _) | (_, Err(e)) => checks.push((format!("runs: {e}"), false)),
}
times.sort_by(|x, y| x.total_cmp(y));
SelfTest {
ok: !checks.is_empty() && checks.iter().all(|(_, ok)| *ok),
load_ms,
tile_ms: times.get(times.len() / 2).copied().unwrap_or(0.0),
tile_megapixels: (2 * t * 2 * t) as f64 / 1e6,
scale,
checks,
}
}
/// A deterministic test tile: four planes of a smooth gradient with noise on top, and the gradient alone for one
/// plane (`tile × tile`).
pub fn test_tile(t: usize) -> (Vec<f32>, Vec<f32>) {
if t == 0 || t > 2048 {
return (Vec::new(), Vec::new());
}
let mut seed = 0x2545_f491u32;
let mut next = move || {
seed ^= seed << 13;
seed ^= seed >> 17;
seed ^= seed << 5;
(seed >> 8) as f32 / (1u32 << 24) as f32
};
let base: Vec<f32> = (0..t * t).map(|i| 0.12 + 0.3 * (((i % t) + (i / t)) as f32 / (2 * t) as f32)).collect();
let input = (0..4 * t * t)
.map(|i| {
// sum of four uniforms: close to Gaussian, deviation 0.0577 × 2 before scaling
let n = next() + next() + next() + next() - 2.0;
base.get(i % (t * t)).copied().unwrap_or(0.0) + 0.05 * n
})
.collect();
(input, base)
}
/// A tile for checking that two runners of one model agree: the whole range a raw photo has rather than the test tile's
/// mid-greys — a noisy ramp from black to white across the tile, a flat black corner and a clipped white one (four
/// planes of `tile × tile`).
pub fn check_tile(t: usize) -> Vec<f32> {
if t == 0 || t > 2048 {
return Vec::new();
}
let mut seed = 0x9e37_79b9u32;
let mut next = move || {
seed ^= seed << 13;
seed ^= seed >> 17;
seed ^= seed << 5;
(seed >> 8) as f32 / (1u32 << 24) as f32
};
let (edge, span) = ((t / 8).max(1), (2 * t).saturating_sub(2).max(1) as f32);
(0..4 * t * t)
.map(|i| {
let (x, y) = ((i % (t * t)) % t, (i % (t * t)) / t);
let n = next() + next() + next() + next() - 2.0;
if x < edge && y < edge {
0.0
} else if x + edge >= t && y + edge >= t {
1.0
} else {
((x + y) as f32 / span + 0.05 * n).clamp(0.0, 1.0)
}
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::licence::Licence;
use crate::manifest::Domain;
use crate::synthetic_proto::{len_field, value_info_bytes};
fn varint(mut v: u64, out: &mut Vec<u8>) {
loop {
let b = (v & 0x7f) as u8;
v >>= 7;
if v == 0 {
out.push(b);
return;
}
out.push(b | 0x80);
}
}
fn varint_field(n: u64, v: u64, out: &mut Vec<u8>) {
varint(n << 3, out);
varint(v, out);
}
fn node(op: &str, inputs: &[&str], output: &str, ints: &[(&str, i64)]) -> Vec<u8> {
let mut n = Vec::new();
for i in inputs {
len_field(1, i.as_bytes(), &mut n);
}
len_field(2, output.as_bytes(), &mut n);
len_field(4, op.as_bytes(), &mut n);
for (name, value) in ints {
let mut a = Vec::new();
len_field(1, name.as_bytes(), &mut a);
varint_field(3, *value as u64, &mut a);
varint_field(20, 2, &mut a);
len_field(5, &a, &mut n);
}
n
}
fn tensor(name: &str, dims: &[u64], values: &[f32]) -> Vec<u8> {
let mut t = Vec::new();
for d in dims {
varint_field(1, *d, &mut t);
}
varint_field(2, 1, &mut t);
len_field(8, name.as_bytes(), &mut t);
let raw: Vec<u8> = values.iter().flat_map(|v| v.to_le_bytes()).collect();
len_field(9, &raw, &mut t);
t
}
/// A tiny working denoiser-shaped network: `[1, 4, t, t]` → a 1 × 1 convolution to 12 channels (each output
/// channel is `scale` times one input plane, so the scale is known) → depth-to-space → `[1, 3, 2t, 2t]`.
fn tiny_model(t: u64, scale: f32) -> Vec<u8> {
tiny_model_with(t, scale, 12)
}
/// The same with another number of convolution channels; only 12 can be arranged into the three colours.
fn tiny_model_with(t: u64, scale: f32, channels: u64) -> Vec<u8> {
let mut graph = Vec::new();
len_field(1, &node("Conv", &["data", "w"], if channels == 12 { "c" } else { "rgb" }, &[("group", 1)]), &mut graph);
if channels == 12 {
let mut depth_to_space = node("DepthToSpace", &["c"], "rgb", &[("blocksize", 2)]);
let mut mode = Vec::new();
len_field(1, b"mode", &mut mode);
len_field(4, b"CRD", &mut mode);
varint_field(20, 3, &mut mode);
len_field(5, &mode, &mut depth_to_space);
len_field(1, &depth_to_space, &mut graph);
}
// CRD output channel colour·4 + dy·2 + dx reads input plane [R, G1, B] of the tile.
let mut w = vec![0f32; channels as usize * 4];
for oc in 0..channels as usize {
let colour = (oc / 4).min(2);
let plane = [0, 1, 3][colour];
w[oc * 4 + plane] = scale;
}
len_field(5, &tensor("w", &[channels, 4, 1, 1], &w), &mut graph);
len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(t), Ok(t)]), &mut graph);
len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(2 * t), Ok(2 * t)]), &mut graph);
let mut ops = Vec::new();
len_field(1, b"", &mut ops);
varint_field(2, 13, &mut ops);
let mut model = Vec::new();
varint_field(1, 8, &mut model);
len_field(7, &graph, &mut model);
len_field(8, &ops, &mut model);
model
}
fn manifest(tile: u32, nominal: f32) -> DenoiserManifest {
DenoiserManifest {
id: "tiny".into(),
name: "Tiny".into(),
version: "1".into(),
licence: Licence::default(),
source: None,
sha256: None,
size_bytes: None,
provenance: String::new(),
domain: Domain::BayerToRgb,
tile,
overlap: 16,
gain: Gain::MatchMean { nominal, max_deviation: 0.05 },
}
}
fn temp_model(name: &str, bytes: &[u8]) -> std::path::PathBuf {
let p = std::env::temp_dir().join(format!("lc-denoise-runtime-{name}-{}.onnx", std::process::id()));
std::fs::write(&p, bytes).unwrap();
p
}
#[test]
fn a_working_model_loads_runs_and_has_the_scale_it_says() {
let p = temp_model("ok", &tiny_model(64, 1000.0));
let r = CpuRunner::load(&p, &manifest(64, 1000.0)).unwrap();
let input: Vec<f32> = (0..4 * 64 * 64).map(|i| (i % 97) as f32 / 97.0).collect();
let out = r.run(&input).unwrap();
assert_eq!(out.len(), 3 * 128 * 128);
// output (colour c, pixel (2y+dy, 2x+dx)) = scale × input plane [R, G1, B][c] at (y, x)
for (c, plane) in [0usize, 1, 3].into_iter().enumerate() {
let (y, x) = (5usize, 9usize);
assert!((out[c * 128 * 128 + (2 * y + 1) * 128 + 2 * x] - 1000.0 * input[plane * 64 * 64 + y * 64 + x]).abs() < 1e-2);
}
let _ = std::fs::remove_file(p);
}
#[test]
fn the_self_test_checks_scale_and_noise() {
// a model that only multiplies cannot take noise down: it must fail that check but pass the others
let p = temp_model("selftest", &tiny_model(64, 1000.0));
let t = CpuRunner::load(&p, &manifest(64, 1000.0)).unwrap().self_test();
assert!(!t.ok, "{t:?}");
assert!(t.checks.iter().any(|(c, ok)| c.contains("same answer") && *ok), "{t:?}");
assert!(t.checks.iter().any(|(c, ok)| c.contains("scale") && *ok), "{t:?}");
assert!(t.checks.iter().any(|(c, ok)| c.contains("noise") && !*ok), "{t:?}");
// and one that claims the wrong scale fails that check
let q = CpuRunner::load(&p, &manifest(64, 5.0)).unwrap().self_test();
assert!(q.checks.iter().any(|(c, ok)| c.contains("scale") && !*ok), "{q:?}");
let _ = std::fs::remove_file(p);
}
#[test]
fn wrong_sizes_and_broken_files_are_errors() {
let p = temp_model("sizes", &tiny_model(64, 1.0));
let r = CpuRunner::load(&p, &manifest(64, 1.0)).unwrap();
assert!(matches!(r.run(&[0.0; 10]), Err(Error::Input(_))));
// a fully convolutional model works at whatever tile size the manifest asks for
let wide = CpuRunner::load(&p, &manifest(128, 1.0)).unwrap();
assert_eq!(wide.run(&vec![0.5; 4 * 128 * 128]).unwrap().len(), 3 * 256 * 256);
// a model that gives another number of values than the manifest promises is an error
let q = temp_model("channels", &tiny_model_with(64, 1.0, 8));
assert!(CpuRunner::load(&q, &manifest(64, 1.0)).is_err(), "shape is refused before execution");
let _ = std::fs::remove_file(q);
for (name, bytes) in [("junk", b"nonsense".to_vec()), ("empty", Vec::new())] {
let q = temp_model(name, &bytes);
assert!(CpuRunner::load(&q, &manifest(64, 1.0)).is_err(), "{name}");
let _ = std::fs::remove_file(q);
}
assert!(CpuRunner::load(&std::env::temp_dir().join("does-not-exist.onnx"), &manifest(64, 1.0)).is_err());
let _ = std::fs::remove_file(p);
}
#[test]
fn the_check_tile_covers_black_to_white() {
for t in [1, 2, 7, 64] {
let c = check_tile(t);
assert_eq!(c.len(), 4 * t * t);
assert!(c.iter().all(|v| (0.0..=1.0).contains(v)), "{t}");
}
let c = check_tile(64);
let near = |lo: f32, hi: f32| c.iter().filter(|v| (lo..=hi).contains(*v)).count();
assert!(near(0.0, 0.0) >= 4 * 8 * 8 && near(1.0, 1.0) >= 4 * 8 * 8, "a flat black and a clipped white corner");
for band in 0..10 {
let lo = band as f32 / 10.0;
assert!(near(lo, lo + 0.1) > 100, "values around {lo}");
}
assert_eq!(check_tile(64), c, "the same tile every time");
}
#[test]
fn truncated_and_corrupted_models_never_panic() {
let good = tiny_model(64, 1.0);
for n in (0..good.len()).step_by(5) {
let q = temp_model("trunc", &good[..n]);
let _ = CpuRunner::load(&q, &manifest(64, 1.0));
let _ = std::fs::remove_file(q);
}
for at in (0..good.len()).step_by(7) {
let mut bad = good.clone();
if let Some(b) = bad.get_mut(at) {
*b = b.wrapping_add(0x55);
}
let q = temp_model("flip", &bad);
if let Ok(r) = CpuRunner::load(&q, &manifest(64, 1.0)) {
let _ = r.run(&vec![0.5; 4 * 64 * 64]);
}
let _ = std::fs::remove_file(q);
}
}
/// Opt-in: `LC_DENOISE_MODEL=<model_bayer.onnx> cargo test -p lightcraft-denoise --release
/// -- --ignored --nocapture` self-tests a real model (RawNIND's Bayer model: tile 512, scale about 1e6).
#[test]
#[ignore = "needs a real model file: set LC_DENOISE_MODEL"]
fn a_real_model_passes_the_self_test() {
let Some(path) = std::env::var_os("LC_DENOISE_MODEL") else { return };
let m = manifest(512, 1.0e6);
let r = CpuRunner::load(Path::new(&path), &m).unwrap();
let t = r.self_test();
println!("load {:.0} ms | {:.0} ms per tile | scale {:.4e} | {:?}", t.load_ms, t.tile_ms, t.scale, t.checks);
assert!(t.ok);
}
}
#[cfg(test)]
mod cpu_tests {
use super::*;
#[test]
fn manifest_and_test_tile_sizes_are_checked_without_tract() {
for t in [0, 2049, usize::MAX] {
assert!(test_tile(t).0.is_empty());
assert!(check_tile(t).is_empty());
}
let path = std::env::temp_dir().join(format!("lc-cpu-load-{}.onnx", std::process::id()));
std::fs::write(&path, crate::synthetic::unet_onnx(64, 5, 1, 7)).unwrap();
let mut m = DenoiserManifest {
id: "synthetic".into(),
name: "Synthetic".into(),
version: "1".into(),
licence: Default::default(),
source: None,
sha256: None,
size_bytes: None,
provenance: "synthetic test".into(),
domain: crate::manifest::Domain::BayerToRgb,
tile: 64,
overlap: 16,
gain: Gain::None,
};
let r = CpuRunner::load(&path, &m).unwrap();
let x = check_tile(64);
assert_eq!(r.run(&x).unwrap(), r.run(&x).unwrap());
assert!(r.self_test().checks.iter().any(|(text, ok)| text.contains("same answer") && *ok));
m.tile = u32::MAX;
assert!(CpuRunner::load(&path, &m).is_err());
std::fs::remove_file(path).unwrap();
}
}
+234
View File
@@ -0,0 +1,234 @@
//! A small ONNX U-Net built in memory, for tests of the layers that read and run denoise networks (here, in the GPU
//! runner and in the engine): the same layer kinds and wiring as the real denoisers, with deterministic random weights.
//! Nothing here is a model anyone would use.
use crate::synthetic_proto::{len_field, value_info_bytes};
fn varint(mut v: u64, out: &mut Vec<u8>) {
loop {
let b = (v & 0x7f) as u8;
v >>= 7;
if v == 0 {
out.push(b);
return;
}
out.push(b | 0x80);
}
}
fn varint_field(n: u64, v: u64, out: &mut Vec<u8>) {
varint(n << 3, out);
varint(v, out);
}
enum A<'a> {
Int(i64),
Ints(&'a [i64]),
Float(f32),
Str(&'a str),
}
fn node(op: &str, inputs: &[&str], output: &str, attrs: &[(&str, A)]) -> Vec<u8> {
let mut n = Vec::new();
for i in inputs {
len_field(1, i.as_bytes(), &mut n);
}
len_field(2, output.as_bytes(), &mut n);
len_field(4, op.as_bytes(), &mut n);
for (name, value) in attrs {
let mut a = Vec::new();
len_field(1, name.as_bytes(), &mut a);
match value {
A::Int(v) => {
varint_field(3, *v as u64, &mut a);
varint_field(20, 2, &mut a);
}
A::Ints(vs) => {
for v in *vs {
varint_field(8, *v as u64, &mut a);
}
varint_field(20, 7, &mut a);
}
A::Float(v) => {
varint((2 << 3) | 5, &mut a);
a.extend_from_slice(&v.to_le_bytes());
varint_field(20, 1, &mut a);
}
A::Str(s) => {
len_field(4, s.as_bytes(), &mut a);
varint_field(20, 3, &mut a);
}
}
len_field(5, &a, &mut n);
}
n
}
fn tensor(name: &str, dims: &[u64], values: &[f32]) -> Vec<u8> {
let mut t = Vec::new();
for d in dims {
varint_field(1, *d, &mut t);
}
varint_field(2, 1, &mut t);
len_field(8, name.as_bytes(), &mut t);
let raw: Vec<u8> = values.iter().flat_map(|v| v.to_le_bytes()).collect();
len_field(9, &raw, &mut t);
t
}
struct Rng(u64);
impl Rng {
fn next(&mut self) -> f32 {
self.0 ^= self.0 >> 12;
self.0 ^= self.0 << 25;
self.0 ^= self.0 >> 27;
((self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0
}
}
struct Builder {
nodes: Vec<Vec<u8>>,
inits: Vec<Vec<u8>>,
rng: Rng,
n: usize,
}
impl Builder {
fn name(&mut self, kind: &str) -> String {
self.n += 1;
format!("{kind}{}", self.n)
}
/// Random weights scaled so activations keep a sane size through the layers.
fn weights(&mut self, name: &str, dims: &[u64], fan_in: usize) -> String {
let count: u64 = dims.iter().product();
let scale = (3.0 / fan_in as f32).sqrt();
let values: Vec<f32> = (0..count).map(|_| self.rng.next() * scale).collect();
self.inits.push(tensor(name, dims, &values));
name.to_string()
}
fn bias(&mut self, name: &str, n: u64) -> String {
let values: Vec<f32> = (0..n).map(|_| self.rng.next() * 0.1).collect();
self.inits.push(tensor(name, &[n], &values));
name.to_string()
}
/// `Conv` (+ `LeakyRelu` when `leaky`); gives the name of the tensor that results.
fn conv(&mut self, src: &str, cin: u64, cout: u64, k: u64, leaky: bool) -> String {
let (w, b, out) = (self.name("w"), self.name("b"), self.name("conv"));
let w = self.weights(&w, &[cout, cin, k, k], (cin * k * k) as usize);
let b = self.bias(&b, cout);
let pad = (k as i64 - 1) / 2;
self.nodes.push(node(
"Conv",
&[src, &w, &b],
&out,
&[("kernel_shape", A::Ints(&[k as i64, k as i64])), ("pads", A::Ints(&[pad; 4])), ("strides", A::Ints(&[1, 1])), ("group", A::Int(1))],
));
if !leaky {
return out;
}
let act = self.name("act");
self.nodes.push(node("LeakyRelu", &[&out], &act, &[("alpha", A::Float(0.2))]));
act
}
fn pool(&mut self, src: &str) -> String {
let out = self.name("pool");
self.nodes.push(node(
"MaxPool",
&[src],
&out,
&[("kernel_shape", A::Ints(&[2, 2])), ("strides", A::Ints(&[2, 2])), ("pads", A::Ints(&[0; 4]))],
));
out
}
fn up(&mut self, src: &str, cin: u64, cout: u64) -> String {
let (w, b, out) = (self.name("uw"), self.name("ub"), self.name("up"));
let w = self.weights(&w, &[cin, cout, 2, 2], (cin * 4) as usize);
let b = self.bias(&b, cout);
self.nodes.push(node(
"ConvTranspose",
&[src, &w, &b],
&out,
&[("kernel_shape", A::Ints(&[2, 2])), ("strides", A::Ints(&[2, 2])), ("pads", A::Ints(&[0; 4]))],
));
out
}
fn concat(&mut self, a: &str, b: &str) -> String {
let out = self.name("cat");
self.nodes.push(node("Concat", &[a, b], &out, &[("axis", A::Int(1))]));
out
}
}
/// A denoiser-shaped U-Net as ONNX bytes: `[1, 4, tile, tile]` → `[1, 3, 2·tile, 2·tile]`, with `depth` poolings
/// (so `tile` must be a multiple of `2^depth`) and `ch` channels on the first level, doubling on each (every count a
/// multiple of 4). Each level is two 3 × 3 convolutions with a leaky ReLU, as in the real networks; the output is a
/// 1 × 1 convolution to 12 channels and depth-to-space.
pub fn unet_onnx(tile: u64, ch: u64, depth: usize, seed: u64) -> Vec<u8> {
let mut b = Builder { nodes: Vec::new(), inits: Vec::new(), rng: Rng(seed | 1), n: 0 };
let mut skips: Vec<(String, u64)> = Vec::new();
let (mut cur, mut c_in, mut c) = ("data".to_string(), 4u64, ch);
for level in 0..=depth {
let x = b.conv(&cur, c_in, c, 3, true);
let x = b.conv(&x, c, c, 3, true);
if level < depth {
skips.push((x.clone(), c));
cur = b.pool(&x);
c_in = c;
c *= 2;
} else {
cur = x;
}
}
for (skip, sc) in skips.into_iter().rev() {
let up = b.up(&cur, c, sc);
let joined = b.concat(&up, &skip);
let x = b.conv(&joined, 2 * sc, sc, 3, true);
cur = b.conv(&x, sc, sc, 3, true);
c = sc;
}
let rgb = b.conv(&cur, c, 12, 1, false);
b.nodes.push(node("DepthToSpace", &[&rgb], "rgb", &[("blocksize", A::Int(2)), ("mode", A::Str("CRD"))]));
let mut graph = Vec::new();
for n in &b.nodes {
len_field(1, n, &mut graph);
}
for t in &b.inits {
len_field(5, t, &mut graph);
}
len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(tile), Ok(tile)]), &mut graph);
len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(2 * tile), Ok(2 * tile)]), &mut graph);
let mut ops = Vec::new();
len_field(1, b"", &mut ops);
varint_field(2, 13, &mut ops);
let mut model = Vec::new();
varint_field(1, 8, &mut model);
len_field(7, &graph, &mut model);
len_field(8, &ops, &mut model);
model
}
/// A tiny smoothing network for installing a real CPU model in setup-flow tests.
/// It averages a 3 × 3 neighborhood of the four Bayer planes and returns three identical channels.
pub fn smoothing_onnx() -> Vec<u8> {
let mut graph = Vec::new();
len_field(1, &node("Conv", &["data", "weights"], "samples", &[("kernel_shape", A::Ints(&[3, 3])), ("pads", A::Ints(&[1, 1, 1, 1]))]), &mut graph);
len_field(1, &node("DepthToSpace", &["samples"], "rgb", &[("blocksize", A::Int(2)), ("mode", A::Str("CRD"))]), &mut graph);
len_field(5, &tensor("weights", &[12, 4, 3, 3], &[1.0 / 36.0; 12 * 4 * 3 * 3]), &mut graph);
len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(64), Ok(64)]), &mut graph);
len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(128), Ok(128)]), &mut graph);
let mut model = Vec::new();
varint_field(1, 8, &mut model);
len_field(7, &graph, &mut model);
let mut ops = Vec::new();
varint_field(2, 13, &mut ops);
len_field(8, &ops, &mut model);
model
}
+44
View File
@@ -0,0 +1,44 @@
pub(crate) fn varint(mut v: u64, out: &mut Vec<u8>) {
loop {
let b = (v & 0x7f) as u8;
v >>= 7;
if v == 0 {
out.push(b);
return;
}
out.push(b | 0x80);
}
}
pub fn len_field(n: u64, body: &[u8], out: &mut Vec<u8>) {
varint(n << 3 | 2, out);
varint(body.len() as u64, out);
out.extend_from_slice(body);
}
pub(crate) fn varint_field(n: u64, v: u64, out: &mut Vec<u8>) {
varint(n << 3, out);
varint(v, out);
}
/// A graph input or output: `Ok(n)` is a fixed dimension, `Err(name)` a named (dynamic) one.
pub fn value_info_bytes(name: &str, elem: u64, dims: &[std::result::Result<u64, &str>]) -> Vec<u8> {
let mut shape = Vec::new();
for d in dims {
let mut dim = Vec::new();
match d {
Ok(v) => varint_field(1, *v, &mut dim),
Err(p) => len_field(2, p.as_bytes(), &mut dim),
}
len_field(1, &dim, &mut shape);
}
let mut tensor = Vec::new();
varint_field(1, elem, &mut tensor);
len_field(2, &shape, &mut tensor);
let mut ty = Vec::new();
len_field(1, &tensor, &mut ty);
let mut vi = Vec::new();
len_field(1, name.as_bytes(), &mut vi);
len_field(2, &ty, &mut vi);
vi
}
+250
View File
@@ -0,0 +1,250 @@
//! Opt-in end-to-end check of the real RawNIND Bayer model on real mosaics with noise added:
//!
//! ```text
//! LC_DENOISE_MODEL=<model_bayer.onnx> LC_DENOISE_MOSAICS=<folder> LC_DENOISE_OUT=<folder> \
//! cargo test --release -p lightcraft-denoise --features runtime --test real_model -- --ignored --nocapture
//! ```
//!
//! `LC_DENOISE_MOSAICS` holds `<name>.mosaic.f32` (little-endian f32, normalised: black 0, white 1, not white
//! balanced), `<name>.rgb.f32` (a clean demosaic of it, camera RGB, interleaved) and `<name>.txt` (`width`,
//! `height`, `cfa`, `wb` lines). Noise is Poisson-Gaussian, made here. Optional: `LC_DENOISE_PARALLEL` (tiles at
//! once, default 8), `LC_DENOISE_ONLY` (one name), `LC_DENOISE_ISO` (noise strength, default 1.0; 0 adds none),
//! `LC_DENOISE_FULL` (the whole frame instead of a 2048 × 1536 crop).
#![cfg(feature = "runtime")]
use std::path::{Path, PathBuf};
use std::time::Instant;
use lightcraft_denoise::bayer::Layout;
use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain};
use lightcraft_denoise::run::{Control, Params, denoise_bayer};
use lightcraft_denoise::runtime::CpuRunner;
struct Rng(u64);
impl Rng {
fn next(&mut self) -> f64 {
self.0 ^= self.0 >> 12;
self.0 ^= self.0 << 25;
self.0 ^= self.0 >> 27;
((self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 11) as f64 + 0.5) / (1u64 << 53) as f64
}
fn gauss(&mut self) -> f64 {
(-2.0 * self.next().ln()).sqrt() * (std::f64::consts::TAU * self.next()).cos()
}
}
fn read_f32(path: &Path) -> Vec<f32> {
std::fs::read(path).unwrap().as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect()
}
struct Dump {
width: usize,
height: usize,
layout: Layout,
wb: [f32; 3],
mosaic: Vec<f32>,
rgb: Vec<f32>,
}
fn load(dir: &Path, name: &str) -> Dump {
let text = std::fs::read_to_string(dir.join(format!("{name}.txt"))).unwrap();
let field = |k: &str| text.lines().find_map(|l| l.strip_prefix(k).map(|v| v.trim().to_string())).unwrap();
let cell = match field("cfa").as_str() {
"RGGB" => [0, 1, 1, 2],
"BGGR" => [2, 1, 1, 0],
"GRBG" => [1, 0, 2, 1],
"GBRG" => [1, 2, 0, 1],
other => panic!("cfa {other}"),
};
let wb: Vec<f32> = field("wb").split_whitespace().map(|v| v.parse().unwrap()).collect();
Dump {
width: field("width").parse().unwrap(),
height: field("height").parse().unwrap(),
layout: Layout::from_cell(cell).unwrap(),
wb: [wb[0], wb[1], wb[2]],
mosaic: read_f32(&dir.join(format!("{name}.mosaic.f32"))),
rgb: read_f32(&dir.join(format!("{name}.rgb.f32"))),
}
}
/// Crop with an even origin so the Bayer phase is kept.
fn crop(d: &Dump, x0: usize, y0: usize, w: usize, h: usize) -> (Vec<f32>, Vec<f32>) {
let mut m = Vec::with_capacity(w * h);
let mut c = Vec::with_capacity(w * h * 3);
for y in y0..y0 + h {
m.extend_from_slice(&d.mosaic[y * d.width + x0..y * d.width + x0 + w]);
c.extend_from_slice(&d.rgb[(y * d.width + x0) * 3..(y * d.width + x0 + w) * 3]);
}
(m, c)
}
/// 0 = red, 1 = green, 2 = blue at a mosaic position.
fn site(layout: Layout, x: usize, y: usize) -> usize {
match ((x + layout.dx) % 2, (y + layout.dy) % 2) {
(0, 0) => 0,
(1, 1) => 2,
_ => 1,
}
}
/// The simplest demosaic (each missing colour is the mean of the same-coloured samples in the 3 × 3 around),
/// as the baseline a user would get from the noisy picture without AI.
fn bilinear(m: &[f32], w: usize, h: usize, layout: Layout) -> Vec<f32> {
let mut out = vec![0f32; w * h * 3];
for y in 0..h {
for x in 0..w {
let mut sum = [0f32; 3];
let mut n = [0f32; 3];
for yy in y.saturating_sub(1)..(y + 2).min(h) {
for xx in x.saturating_sub(1)..(x + 2).min(w) {
let c = site(layout, xx, yy);
sum[c] += m[yy * w + xx];
n[c] += 1.0;
}
}
for c in 0..3 {
out[(y * w + x) * 3 + c] = if n[c] > 0.0 { sum[c] / n[c] } else { 0.0 };
}
}
}
out
}
/// Signal-to-error ratio in dB of `got` against `want` (both interleaved RGB), ignoring a border.
fn snr(got: &[f32], want: &[f32], w: usize, h: usize, border: usize) -> f64 {
let (mut sig, mut err) = (0f64, 0f64);
for y in border..h - border {
for x in border..w - border {
for c in 0..3 {
let (g, t) = (f64::from(got[(y * w + x) * 3 + c]), f64::from(want[(y * w + x) * 3 + c]));
sig += t * t;
err += (g - t).powi(2);
}
}
}
10.0 * (sig / err.max(1e-30)).log10()
}
fn crc32(data: &[u8]) -> u32 {
let mut c = !0u32;
for &b in data {
c ^= u32::from(b);
for _ in 0..8 {
c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 };
}
}
!c
}
/// A viewable PNG of interleaved camera-RGB panels side by side: white balanced, one exposure for all, gamma.
fn write_png(path: &Path, panels: &[&[f32]], w: usize, h: usize, wb: [f32; 3], exposure: f32) {
let total_w = w * panels.len();
let mut raw = Vec::with_capacity((total_w * 3 + 1) * h);
for y in 0..h {
raw.push(0u8);
for p in panels {
for x in 0..w {
for c in 0..3 {
let v = (p[(y * w + x) * 3 + c] * wb[c] * exposure).clamp(0.0, 1.0);
raw.push((v.powf(1.0 / 2.2) * 255.0 + 0.5) as u8);
}
}
}
}
let z = miniz_oxide::deflate::compress_to_vec_zlib(&raw, 6);
let mut png = b"\x89PNG\r\n\x1a\n".to_vec();
let mut chunk = |kind: &[u8; 4], body: &[u8]| {
png.extend_from_slice(&(body.len() as u32).to_be_bytes());
let mut k = kind.to_vec();
k.extend_from_slice(body);
png.extend_from_slice(&k);
png.extend_from_slice(&crc32(&k).to_be_bytes());
};
let mut ihdr = Vec::new();
ihdr.extend_from_slice(&(total_w as u32).to_be_bytes());
ihdr.extend_from_slice(&(h as u32).to_be_bytes());
ihdr.extend_from_slice(&[8, 2, 0, 0, 0]);
chunk(b"IHDR", &ihdr);
chunk(b"IDAT", &z);
chunk(b"IEND", &[]);
std::fs::write(path, png).unwrap();
}
#[test]
#[ignore = "needs a real model and mosaics: see the module docs"]
fn the_real_model_denoises_real_mosaics() {
let (Some(model), Some(dir)) = (std::env::var_os("LC_DENOISE_MODEL"), std::env::var_os("LC_DENOISE_MOSAICS")) else { return };
let dir = PathBuf::from(dir);
let out_dir = std::env::var_os("LC_DENOISE_OUT").map(PathBuf::from);
let parallel: usize = std::env::var("LC_DENOISE_PARALLEL").ok().and_then(|v| v.parse().ok()).unwrap_or(8);
let iso: f64 = std::env::var("LC_DENOISE_ISO").ok().and_then(|v| v.parse().ok()).unwrap_or(1.0);
let names: Vec<String> = match std::env::var("LC_DENOISE_ONLY") {
Ok(n) => vec![n],
Err(_) => ["arw-sony-a7m3-compressed", "cr2-canon-6d", "nef-nikon-d5100-lossless", "dng-google-pixel2xl", "pef-pentax-k3"]
.map(String::from)
.to_vec(),
};
let manifest = DenoiserManifest {
id: "rawnind-bayer".into(),
name: "RawNIND Bayer".into(),
version: "1".into(),
licence: Default::default(),
source: None,
sha256: None,
size_bytes: None,
provenance: String::new(),
domain: Domain::BayerToRgb,
tile: 512,
overlap: 64,
gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },
};
let runner = CpuRunner::load(Path::new(&model), &manifest).unwrap();
let params = Params { tile: 512, overlap: 64, gain: manifest.gain, clip: Some(1.0), parallel };
for name in names {
let d = load(&dir, &name);
// a crop of about 2 × 1.5 tiles, away from the edges, even origin
let full = std::env::var_os("LC_DENOISE_FULL").is_some();
let (w, h) = if full { (d.width & !1, d.height & !1) } else { (2048usize, 1536usize) };
let (x0, y0) = (((d.width - w) / 3) & !1, ((d.height - h) / 2) & !1);
let (clean, reference) = crop(&d, x0, y0, w, h);
// Poisson-Gaussian noise: variance a·signal + b, as a sensor at high ISO
let (a, b) = (1.5e-3 * iso, (4.0e-3 * iso.sqrt()).powi(2));
let mut rng = Rng(0x9e37_79b9_7f4a_7c15 ^ name.len() as u64);
let noisy: Vec<f32> = clean
.iter()
.map(|&v| {
let v = f64::from(v);
let sigma = (a * v.max(0.0) + b).sqrt();
((v + sigma * rng.gauss()) as f32).min(1.0)
})
.collect();
let started = Instant::now();
let done = std::sync::atomic::AtomicUsize::new(0);
let progress = |n: usize, total: usize| done.store(n * 1000 / total, std::sync::atomic::Ordering::Relaxed);
let denoised = denoise_bayer(&noisy, w, h, d.layout, &runner, &params, &Control { cancel: None, progress: Some(&progress) }).unwrap();
let secs = started.elapsed().as_secs_f64();
let baseline = bilinear(&noisy, w, h, d.layout);
let clean_bilinear = bilinear(&clean, w, h, d.layout);
let den: Vec<f32> = denoised.data.iter().flat_map(|p| p.iter().copied()).collect();
let (s_noisy, s_den) = (snr(&baseline, &reference, w, h, 64), snr(&den, &reference, w, h, 64));
let s_floor = snr(&clean_bilinear, &reference, w, h, 64);
println!(
"{name}: {w}×{h} crop, {} tiles, parallel {parallel}, {secs:.1} s | SNR vs clean demosaic: noisy bilinear {s_noisy:.2} dB, denoised {s_den:.2} dB (clean bilinear {s_floor:.2} dB)",
lightcraft_denoise::run::tile_count(w, h, d.layout, 512, 64)
);
if let Some(out) = &out_dir {
std::fs::create_dir_all(out).unwrap();
let mean = reference.iter().map(|&v| f64::from(v)).sum::<f64>() / reference.len() as f64;
let exposure = (0.18 / mean.max(1e-6)) as f32;
// three panels, a 512 × 384 window each, at 1:1
let (pw, ph, px, py) = (512usize, 384usize, 700usize, 500usize);
let window =
|buf: &[f32]| -> Vec<f32> { (0..ph).flat_map(|y| buf[((py + y) * w + px) * 3..((py + y) * w + px + pw) * 3].to_vec()).collect() };
write_png(&out.join(format!("{name}.png")), &[&window(&baseline), &window(&den), &window(&reference)], pw, ph, d.wb, exposure);
}
if iso >= 4.0 {
assert!(s_den > s_noisy + 2.0, "{name}: denoised {s_den:.2} dB is not clearly better than noisy {s_noisy:.2} dB");
}
}
}
+1
View File
@@ -207,6 +207,7 @@ controls! {
"grain.amount" => grain.amount, "Grain", Grain, 0, 100, 0, 1, 0, Plain;
"grain.size" => grain.size, "Size", Grain, 0, 100, 25, 1, 0, Plain;
"grain.roughness" => grain.roughness, "Roughness", Grain, 0, 100, 50, 1, 0, Plain;
"enhance.denoise" => enhance.denoise, "Amount", Detail, 0, 100, 0, 1, 0, Plain;
"detail.sharpenAmount" => detail.sharpen_amount, "Sharpening", Detail, 0, 150, 0, 1, 0, Plain;
"detail.sharpenRadius" => detail.sharpen_radius, "Radius", Detail, 0.5, 3, 1, 0.1, 1, Plain;
"detail.sharpenDetail" => detail.sharpen_detail, "Detail", Detail, 0, 100, 25, 1, 0, Plain;
+44
View File
@@ -94,6 +94,13 @@ impl DevelopSettings {
std::borrow::Cow::Owned(d)
}
/// How much of the AI-denoised picture to mix in, 0 (none) to 1 (all of it): the Denoise amount, unless the Detail
/// section is switched off or the value is not a number.
pub fn denoise_amount(&self) -> f32 {
let a = self.enhance.denoise;
if a.is_finite() && self.enhance.denoise_enabled() && self.section_enabled("detail") { (a / 100.0).clamp(0.0, 1.0) as f32 } else { 0.0 }
}
pub fn set_section_enabled(&mut self, section: &str, on: bool) {
self.disabled_sections.retain(|s| s != section);
if !on {
@@ -112,6 +119,7 @@ impl DevelopSettings {
self.curve = d;
}
Section::Color => self.wb.mode = WbMode::AsShot,
Section::Detail => self.enhance.denoise_on = None,
_ => {}
}
}
@@ -193,4 +201,40 @@ mod tests {
s.set_section_enabled("effects", true);
assert!(s.section_enabled("effects"));
}
#[test]
fn the_denoise_amount_is_a_slider_and_follows_the_detail_section() {
let mut s = DevelopSettings::default();
assert_eq!(s.denoise_amount(), 0.0);
assert!(controls::set(&mut s, "enhance.denoise", 60.0));
assert_eq!(controls::get(&s, "enhance.denoise"), Some(60.0));
assert!((s.denoise_amount() - 0.6).abs() < 1e-6);
// the slider's range is 0 to 100; values outside it are brought in
controls::set(&mut s, "enhance.denoise", 500.0);
assert_eq!(s.denoise_amount(), 1.0);
controls::set(&mut s, "enhance.denoise", -5.0);
assert_eq!(s.denoise_amount(), 0.0);
// switching the Detail section off switches the denoise off with it; a settings file with a bad number does nothing
controls::set(&mut s, "enhance.denoise", 80.0);
s.set_section_enabled("detail", false);
assert_eq!(s.denoise_amount(), 0.0);
s.set_section_enabled("detail", true);
s.enhance.denoise = f64::NAN;
assert_eq!(s.denoise_amount(), 0.0);
// it is part of Detail: a reset of the section clears it
controls::set(&mut s, "enhance.denoise", 40.0);
s.reset_section(Section::Detail);
assert_eq!(s.enhance.denoise, 0.0);
// The later independent switch preserves Amount when off, and can be on at zero.
controls::set(&mut s, "enhance.denoise", 75.0);
s.enhance.denoise_on = Some(false);
assert_eq!(s.denoise_amount(), 0.0);
assert_eq!(s.enhance.denoise, 75.0);
s.enhance.denoise_on = Some(true);
assert_eq!(s.denoise_amount(), 0.75);
controls::set(&mut s, "enhance.denoise", 0.0);
assert!(s.enhance.denoise_enabled());
s.reset_section(Section::Detail);
assert_eq!(s.enhance.denoise_on, None);
}
}
+14 -1
View File
@@ -88,11 +88,16 @@ pub struct Profile {
pub id: String,
/// Creative profile amount 0..200 (%).
pub amount: f64,
/// Render the raw's own local tone mapping (DNG `ProfileGainTableMap`, Apple ProRAW), as the
/// camera does. Off: rendered as Lightroom Classic does, without it. Omitted when off, so
/// settings without it keep their JSON (and hash).
#[serde(skip_serializing_if = "std::ops::Not::not")]
pub camera_local_tone: bool,
}
impl Default for Profile {
fn default() -> Self {
Self { id: "lc.color".into(), amount: 100.0 }
Self { id: "lc.color".into(), amount: 100.0, camera_local_tone: false }
}
}
@@ -915,6 +920,14 @@ pub struct LensBlur {
#[serde(default)]
pub struct Enhance {
pub denoise: f64,
#[serde(skip_serializing_if = "Option::is_none")]
pub denoise_on: Option<bool>,
pub raw_details: bool,
pub super_resolution: bool,
}
impl Enhance {
pub fn denoise_enabled(&self) -> bool {
self.denoise_on.unwrap_or(self.denoise > 0.0)
}
}
+10
View File
@@ -21,7 +21,10 @@ web-time = { workspace = true }
lightcraft-preview = { workspace = true }
lightcraft-codecs = { workspace = true }
lightcraft-raw = { workspace = true }
lightcraft-denoise-core = { workspace = true }
lightcraft-denoise = { workspace = true, optional = true }
lightcraft-meta = { workspace = true }
lightcraft-faces = { workspace = true }
miniz_oxide = { workspace = true }
lightcraft-merge = { workspace = true }
rayon = { workspace = true }
@@ -32,11 +35,18 @@ log = { workspace = true }
ab_glyph = { workspace = true }
# Apply the font’s vertical substitutions and origins to Japanese watermarks.
harfrust = { version = "0.12", default-features = false, features = ["std"] }
unicode-segmentation = "1.13"
lightcraft-segment = { workspace = true, optional = true }
# model downloads run natively only (the web build fetches no models)
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
lightcraft-fetch = { workspace = true }
[features]
# AI masks with SAM 3 (Object / Describe). Off for the web and CLI builds.
sam = ["dep:lightcraft-segment"]
denoise = ["dep:lightcraft-denoise", "lightcraft-denoise/runtime", "lightcraft-gpu/denoise"]
rawnind-model = ["denoise", "lightcraft-denoise/rawnind-model", "lightcraft-denoise-core/rawnind-model"]
[dev-dependencies]
libc = { workspace = true }
+16 -3
View File
@@ -13,19 +13,22 @@ fn main() {
println!("cargo::rerun-if-changed=build.rs");
println!("cargo::rerun-if-env-changed=CRAFT_FONTS_DIR");
println!("cargo::rerun-if-env-changed=CRAFT_FONTS_REQUIRED");
let required = std::env::var_os("CRAFT_FONTS_REQUIRED").is_some();
let mut src = String::from("pub static CRAFT_FONTS: &[CraftFont] = &[\n");
let mut embedded = 0;
if let Some(dir) = std::env::var_os("CRAFT_FONTS_DIR").map(PathBuf::from).map(resolve) {
match craft_fonts(&dir) {
match craft_fonts(&dir, required) {
Ok(entries) => {
embedded = entries.lines().count();
src.push_str(&entries);
}
Err(e) if std::env::var_os("CRAFT_FONTS_REQUIRED").is_some() => {
Err(e) if required => {
println!("cargo::error=CRAFT_FONTS_DIR={}: {e}", dir.display());
}
Err(e) => println!("cargo::warning=building without craft-fonts: CRAFT_FONTS_DIR={}: {e}", dir.display()),
}
} else if required {
println!("cargo::error=CRAFT_FONTS_REQUIRED is set but CRAFT_FONTS_DIR is missing");
}
src.push_str("];\n");
// Without them the app still runs and every language still selectable, so Chinese and Japanese
@@ -58,17 +61,21 @@ fn resolve(dir: PathBuf) -> PathBuf {
}
/// One `CraftFont { .. }` initialiser per manifest line.
fn craft_fonts(dir: &Path) -> Result<String, String> {
fn craft_fonts(dir: &Path, required: bool) -> Result<String, String> {
let manifest = dir.join("fonts/manifest.txt");
println!("cargo::rerun-if-changed={}", manifest.display());
let text = std::fs::read_to_string(&manifest).map_err(|e| format!("{}: {e}", manifest.display()))?;
let wasm = std::env::var("CARGO_CFG_TARGET_ARCH").is_ok_and(|a| a == "wasm32");
let mut out = String::new();
let mut has_japanese_ui = false;
let mut has_chinese_ui = false;
for line in text.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')) {
let f: Vec<&str> = line.split(" | ").map(str::trim).collect();
let [family, style, file, scripts, ..] = f.as_slice() else {
return Err(format!("malformed manifest line: {line}"));
};
has_japanese_ui |= (*family, *style) == ("BIZ UDPGothic", "Regular");
has_chinese_ui |= (*family, *style) == ("Noto Sans CJK SC", "Regular") && scripts.split(',').any(|script| script.trim() == "Hans");
if wasm && (*family, *style) != ("BIZ UDPGothic", "Regular") {
continue;
}
@@ -82,5 +89,11 @@ fn craft_fonts(dir: &Path) -> Result<String, String> {
path.display().to_string(),
);
}
if required && !has_japanese_ui {
return Err("required Japanese UI face BIZ UDPGothic Regular is missing from fonts/manifest.txt".into());
}
if required && !wasm && !has_chinese_ui {
return Err("required Simplified Chinese UI face Noto Sans CJK SC Regular (Hans) is missing from fonts/manifest.txt".into());
}
Ok(out)
}
+179 -12
View File
@@ -33,7 +33,15 @@ pub(crate) const PROFILE_PROXY: usize = 192;
pub(crate) fn file_local_look(format: RawFormat) -> bool {
matches!(
format,
RawFormat::Arw | RawFormat::Nef | RawFormat::Nrw | RawFormat::Rw2 | RawFormat::Raf | RawFormat::Cr3 | RawFormat::Cr2 | RawFormat::Pef
RawFormat::Arw
| RawFormat::Nef
| RawFormat::Nrw
| RawFormat::Rw2
| RawFormat::Raf
| RawFormat::Cr3
| RawFormat::Cr2
| RawFormat::Pef
| RawFormat::Srw
)
}
@@ -402,19 +410,38 @@ const EDGE_CONTRAST: f32 = 3.0;
/// it is tried once more on the pixels away from edges (same gates): there colour pairs stay
/// valid when the camera JPEG's geometry differs slightly from the raw's.
fn fit_pairs_with(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>) -> Option<CameraLook> {
fit_pairs_ordered(sensor, reference, colour, &[ToneFit::Quantile, ToneFit::ConditionalMedian])
}
/// [`fit_pairs_with`] with the tone fits to try, in order: the first whose look passes the
/// acceptance gates is used. Quantile matching comes first because it cannot produce flat steps;
/// the conditional-median fit follows because on a few photos (compacts, early Micro Four Thirds)
/// its slightly lower held-out error is what clears the gates, and a photo must not lose its look
/// to the change of fit. The whole search (all pixels, then away from edges, with and without the
/// camera profile's colour) runs with one tone fit before the next is tried, so the fallback
/// accepts exactly what the previous releases accepted.
fn fit_pairs_ordered(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>, order: &[ToneFit]) -> Option<CameraLook> {
let has_profile = colour.is_some();
// A different picture style can make a camera profile fail the gates. Preserve the
// photo's own colour fit before resorting to the neutral fallback.
fit_candidate(sensor, reference, colour).or_else(|| has_profile.then(|| fit_candidate(sensor, reference, None)).flatten())
order.iter().find_map(|&tone| {
// A different picture style can make a camera profile fail the gates. Preserve the
// photo's own colour fit before resorting to the neutral fallback.
fit_candidate(sensor, reference, colour.clone(), tone).or_else(|| has_profile.then(|| fit_candidate(sensor, reference, None, tone)).flatten())
})
}
/// One candidate colour model, fitted on all pixels and, when that fails the gates, once more on
/// the pixels away from edges (issue #232).
fn fit_candidate(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>) -> Option<CameraLook> {
fit_pairs_on(sensor, reference, colour.clone(), None).or_else(|| fit_pairs_on(sensor, reference, colour, Some(EDGE_CONTRAST)))
fn fit_candidate(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>, tone: ToneFit) -> Option<CameraLook> {
fit_pairs_on(sensor, reference, colour.clone(), None, tone).or_else(|| fit_pairs_on(sensor, reference, colour, Some(EDGE_CONTRAST), tone))
}
fn fit_pairs_on(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>, edge_limit: Option<f32>) -> Option<CameraLook> {
fn fit_pairs_on(
sensor: &Rgb32f,
reference: &Rgb32f,
colour: Option<(Mat3, Option<HsvTable>)>,
edge_limit: Option<f32>,
tone_fit: ToneFit,
) -> Option<CameraLook> {
// A known colour model needs enough signal for tone fitting, not a scene rich enough to
// learn a new colour matrix. Still reject monochrome references.
let (pairs, bright) = collect_pairs(sensor, reference, if colour.is_some() { 0.005 } else { 0.05 }, edge_limit)?;
@@ -437,7 +464,7 @@ fn fit_pairs_on(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Optio
};
let tone_pairs: Vec<_> =
pairs.iter().enumerate().filter(|(i, _)| i % 3 != 0).map(|(_, (x, y))| (luma(colour(*x).map(|v| v.max(0.0))), luma(*y))).collect();
let Some(curve) = fit_tone(tone_pairs) else { continue };
let Some(curve) = tone_fit.fit(tone_pairs) else { continue };
let mut look = CameraLook { matrix, tone: curve, hue_sat: hue_sat.clone() };
if let Some(tone) = fit_chroma(&bright, &look) {
look.tone = tone;
@@ -468,10 +495,11 @@ fn fit_pairs_on(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Optio
.sum();
if lightcraft_pipeline::profiling() {
eprintln!(
"[profile] camera look holdout RMS {:.5} -> {:.5} ({samples} channels{})",
"[profile] camera look holdout RMS {:.5} -> {:.5} ({samples} channels{}{})",
(before / samples as f64).sqrt(),
(after / samples as f64).sqrt(),
if edge_limit.is_some() { ", away from edges" } else { "" }
if edge_limit.is_some() { ", away from edges" } else { "" },
if tone_fit == ToneFit::ConditionalMedian { ", conditional-median tone" } else { "" }
);
}
if samples == 0 || after >= before * MIN_IMPROVEMENT || after / samples as f64 > MAX_HOLDOUT_RMS.powi(2) {
@@ -681,12 +709,68 @@ fn fit_chroma(pairs: &[([f64; 3], [f64; 3])], look: &CameraLook) -> Option<Camer
(after.is_finite() && after < before).then_some(fitted)
}
/// How the per-photo tone curve is fitted ([`fit_pairs_ordered`] tries them in order).
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
enum ToneFit {
/// [`fit_tone`]: no flat steps, no pixel-exact registration needed.
Quantile,
/// [`fit_tone_conditional`]: the earlier fit, kept as the fallback.
ConditionalMedian,
}
impl ToneFit {
fn fit(self, pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
match self {
ToneFit::Quantile => fit_tone(pairs),
ToneFit::ConditionalMedian => fit_tone_conditional(pairs),
}
}
}
/// The camera's tone curve from scene/JPEG luminance pairs, by quantile matching: knot `i` pairs
/// the median of the `i`-th 1/32 of the sorted scene luminances with the median of the `i`-th
/// 1/32 of the sorted JPEG luminances. A tone curve is monotone, so it maps each quantile of the
/// scene to the same quantile of the JPEG; matching the two distributions needs no pixel-exact
/// registration and is increasing by construction.
///
/// Binning the pairs by scene luminance and taking the JPEG's median per bin instead (followed by
/// isotonic pooling) fails on real files: camera JPEGs are lens-corrected and sharpened, so on a
/// 96 px proxy of a busy scene neighbouring pairs disagree by more than the gap between bins.
/// The conditional medians then zig-zag, pooling turns them into flat steps (13 of 31 segments
/// on a Z 6II forest, issue #475) that render as posterised patches, and regression toward the
/// mean flattens both ends of the curve (lifted blacks, dull highlights).
fn fit_tone(pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
if pairs.len() < 128 || !pairs.iter().all(|(x, y)| x.is_finite() && *x > 0.0 && y.is_finite()) {
return None;
}
let mut xs: Vec<f64> = pairs.iter().map(|p| p.0).collect();
let mut ys: Vec<f64> = pairs.iter().map(|p| p.1).collect();
xs.sort_by(f64::total_cmp);
ys.sort_by(f64::total_cmp);
let n = xs.len();
let mut knots = [[0.0; 2]; 32];
for (i, knot) in knots.iter_mut().enumerate() {
let (a, b) = (i * n / 32, (i + 1) * n / 32);
let mid = a + (b - a) / 2;
*knot = [*xs.get(mid)? as f32, *ys.get(mid)? as f32];
}
if knots[31][0] < knots[0][0] * 1.5 {
return None;
}
CameraTone::new(knots)
}
fn median(values: &mut [f64]) -> Option<f64> {
values.sort_by(f64::total_cmp);
values.get(values.len() / 2).copied()
}
fn fit_tone(mut pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
/// The earlier tone fit, kept as the fallback of [`ToneFit`]: the JPEG's median per equal-population
/// bin of scene luminance, then pooled-adjacent-violators isotonic regression. It follows the
/// per-pixel pairs closely when the two images are registered, which is why it can clear the
/// acceptance gates where [`fit_tone`] just misses them, but on lens-corrected JPEGs of busy scenes
/// the pooling leaves flat steps (issue #475).
fn fit_tone_conditional(mut pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
if pairs.len() < 128 || !pairs.iter().all(|(x, y)| x.is_finite() && *x > 0.0 && y.is_finite()) {
return None;
}
@@ -1021,7 +1105,17 @@ mod tests {
#[test]
fn supported_raws_get_a_file_local_look() {
// every decoded raw without a colour matrix of its own (issue #310: CR2 rendered flat with the fallback)
let local = [RawFormat::Arw, RawFormat::Nef, RawFormat::Nrw, RawFormat::Rw2, RawFormat::Raf, RawFormat::Cr3, RawFormat::Cr2, RawFormat::Pef];
let local = [
RawFormat::Arw,
RawFormat::Nef,
RawFormat::Nrw,
RawFormat::Rw2,
RawFormat::Raf,
RawFormat::Cr3,
RawFormat::Cr2,
RawFormat::Pef,
RawFormat::Srw,
];
assert!(local.into_iter().all(file_local_look));
// DNG carries its own colour model
assert!(!file_local_look(RawFormat::Dng));
@@ -1228,4 +1322,77 @@ mod tests {
reference.data.truncate(8);
assert!(fit_pairs(&sensor, &reference).is_none());
}
/// A busy scene whose JPEG is the camera curve of the scene two proxy pixels to the right (a
/// lens-corrected preview): the fitted curve follows the camera's and has no flat steps.
#[test]
fn tone_fit_has_no_steps_on_a_misregistered_jpeg() {
let camera = |x: f64| 0.6 * x.powf(0.55) / (1.0 + 0.2 * x);
let (w, h) = (96usize, 64usize);
let mut seed = 0x2545_f491_u32;
let mut scene = Vec::with_capacity(w * h);
for _ in 0..w * h {
seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
scene.push(0.01 * (5.0 * f64::from(seed >> 8) / f64::from(1u32 << 24)).exp());
}
let pairs: Vec<(f64, f64)> =
(0..h).flat_map(|y| (0..w - 2).map(move |x| (y * w + x, y * w + x + 2))).map(|(i, j)| (scene[i], camera(scene[j]))).collect();
let curve = fit_tone(pairs).unwrap();
let mut sorted = scene.clone();
sorted.sort_by(f64::total_cmp);
let (lo, hi) = (sorted[sorted.len() / 50], sorted[sorted.len() * 49 / 50]);
let mut x = lo;
while x < hi {
let (a, b) = (curve.apply(x as f32), curve.apply((x * 1.02) as f32));
assert!(b > a, "flat step at {x}: {a} -> {b}");
assert!((f64::from(a) / camera(x) - 1.0).abs() < 0.05, "at {x}: {a} vs {}", camera(x));
x *= 1.02;
}
}
/// A synthetic photo whose JPEG is the camera look of the scene scaled per pixel by noise of
/// `spread` (a pseudo-random, registration-free stand-in for local tone mapping): the per-bin
/// median recovers the curve, while the JPEG's broadened distribution stretches the quantile fit.
fn noisy_look(spread: f32) -> (Rgb32f, Rgb32f) {
let known = Mat3([[1.8, -0.4, -0.1], [-0.2, 1.5, -0.1], [-0.05, -0.3, 1.7]]);
let (w, h) = (96usize, 64usize);
let mut sensor = Rgb32f::new(w, h);
let mut reference = sensor.clone();
let mut seed = 0x9e37_79b9_u32;
let mut next = move || {
seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
f32::from((seed >> 16) as u16) / 32768.0 - 1.0
};
for (src, dst) in sensor.data.iter_mut().zip(&mut reference.data) {
let ev = 0.05 + (next() + 1.0) * 0.2;
*src = [ev * (0.8 + (next() + 1.0) * 0.1), ev, ev * (0.8 + (next() + 1.0) * 0.1)];
let p = known.apply_f32(*src);
let y = luminance_2020(p);
let scale = (1.0 - (-2.5 * y).exp()) / y * (1.0 + spread * next());
*dst = p.map(|v| v * scale);
}
(sensor, reference)
}
/// Quantile matching is tried first; the conditional-median fit is the fallback and is used
/// only where the quantile look misses the acceptance gates, so no photo loses its look.
#[test]
fn tone_fit_order_quantile_first_conditional_median_as_fallback() {
// Clean data: both fits pass, the default order uses the quantile curve (increasing).
let (sensor, reference) = noisy_look(0.0);
let default = fit_pairs(&sensor, &reference).expect("clean look");
let quantile = fit_pairs_ordered(&sensor, &reference, None, &[ToneFit::Quantile]).expect("quantile passes");
assert_eq!(default.tone, quantile.tone);
// Noisy data: the quantile fit misses the gates, the conditional median still passes.
let (sensor, reference) = noisy_look(0.33);
assert!(fit_pairs_ordered(&sensor, &reference, None, &[ToneFit::Quantile]).is_none(), "quantile alone misses the gates");
let conditional = fit_pairs_ordered(&sensor, &reference, None, &[ToneFit::ConditionalMedian]).expect("conditional median passes");
let default = fit_pairs(&sensor, &reference).expect("the fallback keeps the look");
assert_eq!(default.tone, conditional.tone);
// The order matters only through the first pass; with the fits swapped the conditional one wins even on clean data.
let (sensor, reference) = noisy_look(0.0);
let swapped = fit_pairs_ordered(&sensor, &reference, None, &[ToneFit::ConditionalMedian, ToneFit::Quantile]).unwrap();
let conditional = fit_pairs_ordered(&sensor, &reference, None, &[ToneFit::ConditionalMedian]).unwrap();
assert_eq!(swapped.tone, conditional.tone);
}
}
+2 -13
View File
@@ -65,19 +65,8 @@ impl CameraProfile {
}
}
/// LightCraft's configuration folder (settings, GPU marker, camera profiles).
pub fn config_dir() -> Option<PathBuf> {
if cfg!(target_os = "macos") {
std::env::var_os("HOME").map(|h| PathBuf::from(h).join("Library/Application Support/LightCraft"))
} else if cfg!(windows) {
std::env::var_os("APPDATA").map(|a| PathBuf::from(a).join("LightCraft"))
} else {
std::env::var_os("XDG_CONFIG_HOME")
.map(PathBuf::from)
.or_else(|| std::env::var_os("HOME").map(|h| PathBuf::from(h).join(".config")))
.map(|c| c.join("lightcraft"))
}
}
/// LightCraft's configuration folder (settings, GPU marker, camera profiles); one helper for the app.
pub use crate::config::config_dir;
/// Where camera profiles are read from and written to.
pub fn dir() -> Option<PathBuf> {
+659
View File
@@ -0,0 +1,659 @@
//! Denoise commands (`denoise.*`): the models (list, install, download, test, remove, choose), the settings, and the
//! work (queue, cancel, status, pump, clear). The machinery they drive is [`crate::denoise`].
//!
//! Models live in the folder the host provides (`Session::denoise.models_dir`): one subfolder per model with
//! `model.onnx`, `denoise-model.json` (its manifest) and `installed.json` (what the user accepted and the file's hash),
//! plus `settings.json` beside them. Model files, archives and manifests are hostile input: sizes are capped, ids are
//! validated before they become folder names, and nothing outside a model's own folder is ever touched.
use std::path::{Path, PathBuf};
use std::sync::{Arc, PoisonError, mpsc};
use lightcraft_catalog::PhotoId;
use lightcraft_denoise::archive;
use lightcraft_denoise::hash::sha256_file;
use lightcraft_denoise::known::{self, Known};
use lightcraft_denoise::manifest::{self, DenoiserManifest, MAX_MANIFEST_BYTES, MAX_MODEL_BYTES};
use serde_json::{Value, json};
use super::{CommandSpec, always, bad, bool_or, cmd, str_param};
use crate::denoise::{Pace, RunOn, Settings, forget_gpu_set_ups, installed_models, read_capped, read_settings, write_atomic, write_settings};
use crate::model_download::State;
use crate::{EngineError, Result, Session};
fn fail(what: &str, e: impl std::fmt::Display) -> EngineError {
EngineError::Other(format!("{what}: {e}"))
}
fn models_dir(s: &Session, cmd: &str) -> Result<PathBuf> {
s.denoise.models_dir.clone().ok_or_else(|| bad(cmd, "this build has nowhere to keep denoise models (the desktop app does)"))
}
/// What a command says about a model.
fn row(m: &DenoiserManifest, download_host: Option<String>, installed: bool, selected: bool, accepted: &Value) -> Value {
json!({
"id": m.id,
"name": m.name,
"version": m.version,
"licence": m.licence,
"provenance": m.provenance,
"source": m.source,
"sizeBytes": m.size_bytes,
"sha256": m.sha256,
"tile": m.tile,
"known": known::find(&m.id).is_some(),
// a pinned address LightCraft can fetch it from (the user presses Download), and which site that is
"downloadHost": download_host,
"installed": installed,
"selected": selected,
"accepted": accepted,
})
}
fn host_of(k: &Known) -> Option<String> {
k.download.as_ref().map(|d| lightcraft_denoise::known::host(&d.url).to_string())
}
fn list(s: &mut Session, _: &Value) -> Result<Value> {
s.denoise_refresh_active(true);
let dir = s.denoise.models_dir.clone();
let st = dir.as_deref().map(read_settings).unwrap_or_default();
let installed = dir.as_deref().map(installed_models).unwrap_or_default();
let on_disk = |id: &str| installed.iter().find(|i| i.manifest.id == id);
let chosen = |id: &str| st.model.as_deref() == Some(id);
let mut models: Vec<Value> = Vec::new();
for k in known::all() {
let found = on_disk(&k.manifest.id);
models.push(row(&k.manifest, host_of(&k), found.is_some(), chosen(&k.manifest.id), found.map_or(&Value::Null, |f| &f.accepted)));
}
for i in installed.iter().filter(|i| known::find(&i.manifest.id).is_none()) {
models.push(row(&i.manifest, None, true, chosen(&i.manifest.id), &i.accepted));
}
Ok(json!({
"dir": dir.as_ref().map(|d| d.display().to_string()),
"productsDir": s.denoise_products_dir().map(|d| d.display().to_string()),
"model": st.model,
// whether this build can run denoise models (the `denoise` feature: our pure-Rust CPU/GPU runners)
"runtime": cfg!(feature = "denoise"),
"auto": st.auto(),
"cacheGb": st.cache_bytes() >> 30,
"threads": st.threads,
"runOn": st.run_on().name(),
"models": models,
}))
}
/// What a file the user pointed at is, as far as installing it goes.
struct Source {
manifest: DenoiserManifest,
/// The `.onnx` inside the archive; `None` when the file is the `.onnx`.
entry: Option<&'static str>,
file_name: String,
}
fn identify(path: &Path, c: &str, sha_known: Option<&str>) -> Result<Source> {
let meta = std::fs::metadata(path).map_err(|e| bad(c, format!("cannot read `{}`: {e}", path.display())))?;
if !meta.is_file() {
return Err(bad(c, format!("`{}` is not a file", path.display())));
}
if meta.len() == 0 || meta.len() > MAX_MODEL_BYTES {
return Err(bad(c, "a model file must be between 1 byte and 2 GiB"));
}
let sha256 = match sha_known {
Some(h) => h.to_string(),
None => sha256_file(path).map_err(|e| fail(&format!("could not read {}", path.display()), e))?,
};
let file_name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default();
// an archive LightCraft knows (the file darktable publishes the model in)
if let Some(k) = known::all().into_iter().find(|k| k.download.as_ref().is_some_and(|d| d.sha256 == sha256)) {
return Ok(Source { manifest: k.manifest, entry: k.entry, file_name });
}
// the `.onnx` of a known model
if let Some(m) = known::by_sha256(&sha256) {
return Ok(Source { manifest: m, entry: None, file_name });
}
// an `.onnx` with its manifest beside it
let sibling = path.with_file_name("denoise-model.json");
if path.extension().is_some_and(|e| e.eq_ignore_ascii_case("onnx"))
&& let Some(bytes) = read_capped(&sibling, MAX_MANIFEST_BYTES)
{
let m = manifest::parse(&bytes).map_err(|e| bad(c, format!("{}: {e}", sibling.display())))?;
return Ok(Source { manifest: m, entry: None, file_name });
}
Err(bad(
c,
"LightCraft does not know this file. A denoise model is an .onnx file with a denoise-model.json beside it that says how to use it \
(see docs/denoise.md), or an archive LightCraft can fetch itself (`denoise.models.download`)",
))
}
/// Install the model in `src`. A model that passes its self-test becomes the one in use (`activate`, the default): installing
/// a model is how someone says they want it. `verified` is for a download whose hash was already checked.
fn install_file(
dir: &Path,
loader: &crate::denoise::Loader,
run_on: RunOn,
accepted_at: String,
c: &str,
src: &Path,
acknowledged: bool,
verified: bool,
) -> Result<(Value, Arc<dyn crate::denoise::Model>)> {
let ident = identify(src, c, None)?;
let m = ident.manifest.clone();
manifest::validate(&m).map_err(|e| bad(c, e.to_string()))?;
if !acknowledged {
return Err(bad(
c,
"the licence has not been accepted: show the user its terms (see `denoise.models.list`) and pass `acknowledged: true` once they agree",
));
}
let home = dir.join(&m.id);
if home.join("model.onnx").exists() {
return Err(bad(c, "that model is already installed; remove it before installing a replacement"));
}
std::fs::create_dir_all(&home).map_err(|e| fail("could not create the model's folder", e))?;
let mut cleanup = InstallCleanup { home: home.clone(), armed: true };
let (part, final_path) = (home.join("model.onnx.part"), home.join("model.onnx"));
let _ = std::fs::remove_file(&part);
let placed: Result<()> = match ident.entry {
Some(entry) => {
archive::extract(src, entry, &part, MAX_MODEL_BYTES).map(|_| ()).map_err(|e| fail("could not take the model out of the archive", e))
}
None if verified && std::fs::rename(src, &part).is_ok() => Ok(()),
None => std::fs::copy(src, &part).map(|_| ()).map_err(|e| fail("could not copy the model (is the disk full?)", e)),
};
let sha256 = placed.and_then(|_| sha256_file(&part).map_err(|e| fail("could not check the model", e))).and_then(|h| match &m.sha256 {
Some(want) if *want != h => Err(EngineError::Other("the model file is not the one its description names (its SHA-256 differs)".into())),
_ => Ok(h),
});
let sha256 = match sha256 {
Ok(h) => h,
Err(e) => {
let _ = std::fs::remove_file(&part);
return Err(e);
}
};
// Test the staged file first. A failed replacement preserves the installed model.
let model = loader(&part, &m).map_err(|e| bad(c, e))?;
let test = match model.self_test(run_on) {
Ok(t) => t,
Err(e) => {
let _ = std::fs::remove_file(&part);
return Err(bad(c, e));
}
};
// An installed id is immutable: installing again must never destroy a usable model.
if final_path.exists() {
let _ = std::fs::remove_file(&part);
return Err(bad(c, "that model is already installed; remove it before installing a replacement"));
}
std::fs::rename(&part, &final_path).map_err(|e| fail("could not finish the copy", e))?;
write_atomic(&home.join("denoise-model.json"), &serde_json::to_vec_pretty(&m).map_err(|e| fail("manifest", e))?).map_err(EngineError::Other)?;
let accepted = json!({
"acceptedAt": accepted_at,
"licence": m.licence.name,
"commercial": m.licence.commercial,
"fileName": ident.file_name,
"sha256": sha256,
"selfTest": test,
});
write_atomic(&home.join("installed.json"), &serde_json::to_vec_pretty(&accepted).map_err(|e| fail("record", e))?).map_err(EngineError::Other)?;
cleanup.armed = false;
Ok((json!({"installed": row(&m, known::find(&m.id).and_then(|k| host_of(&k)), true, false, &accepted), "model": m.id}), model))
}
/// Removes only files created for this new installation, including on a worker panic.
struct InstallCleanup {
home: PathBuf,
armed: bool,
}
impl Drop for InstallCleanup {
fn drop(&mut self) {
if self.armed {
for name in ["model.onnx.part", "model.onnx", "denoise-model.json", "installed.json"] {
let _ = std::fs::remove_file(self.home.join(name));
}
let _ = std::fs::remove_dir(&self.home);
}
}
}
pub(crate) struct InstallJob {
id: String,
activate: bool,
result: mpsc::Receiver<std::result::Result<(Value, Arc<dyn crate::denoise::Model>), String>>,
}
fn activate_install(s: &mut Session, result: &Value, model: Arc<dyn crate::denoise::Model>, activate: bool) -> Result<Value> {
let dir = models_dir(s, "denoise.models.install")?;
let id = result["installed"]["id"].as_str().ok_or_else(|| bad("denoise.models.install", "missing installed id"))?;
let mut st = read_settings(&dir);
if activate {
st.model = Some(id.to_string());
write_settings(&dir, &st).map_err(EngineError::Other)?;
}
s.denoise.touch();
s.denoise_refresh_active(true);
if let Some(a) = &s.denoise.active
&& a.id == id
{
*a.model.lock().unwrap_or_else(PoisonError::into_inner) = Some(model);
}
let mut v = result.clone();
v["installed"]["selected"] = json!(st.model.as_deref() == Some(id));
v["model"] = json!(st.model);
Ok(v)
}
fn start_install(s: &mut Session, src: PathBuf, id: String, activate: bool, verified: bool) -> Result<Value> {
const C: &str = "denoise.models.install";
if s.denoise.install.is_some() {
return Err(bad(C, "another model is being installed"));
}
let dir = models_dir(s, C)?;
let loader = s.denoise.loader.clone();
let run_on = s.denoise.settings.run_on();
let at = (s.clock)();
let (tx, rx) = mpsc::channel();
std::thread::Builder::new()
.name("denoise-model-install".into())
.stack_size(16 * 1024 * 1024)
.spawn(move || {
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _held = crate::memory::work_gate().acquire(512 * 1024 * 1024);
install_file(&dir, &loader, run_on, at, C, &src, true, verified).map_err(|e| plain(&e))
}))
.unwrap_or_else(|_| Err("the model installation stopped unexpectedly".into()));
if verified {
let _ = std::fs::remove_file(&src);
}
let _ = tx.send(outcome);
})
.map_err(|e| bad(C, format!("could not start installation: {e}")))?;
s.denoise.install = Some(InstallJob { id: id.clone(), activate, result: rx });
s.denoise.downloads.set_outcome(&id, State::Installing);
Ok(json!({"started": id}))
}
fn install(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.install";
let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?;
let acknowledged = bool_or(p, "acknowledged", false);
let activate = bool_or(p, "activate", true);
if !acknowledged {
return Err(bad(C, "the licence has not been accepted: show its terms first"));
}
if bool_or(p, "background", false) {
return start_install(s, PathBuf::from(path), "local-install".into(), activate, false);
}
if s.denoise.install.is_some() {
return Err(bad(C, "another model is being installed"));
}
let (result, model) =
install_file(&models_dir(s, C)?, &s.denoise.loader, s.denoise.settings.run_on(), (s.clock)(), C, Path::new(path), acknowledged, false)?;
activate_install(s, &result, model, activate)
}
/// Read only the small adjacent manifest; hashing and execution happen in the install worker.
fn inspect(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.inspect";
let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?;
let src = Path::new(path);
if !src.extension().is_some_and(|e| e.eq_ignore_ascii_case("onnx")) {
return Err(bad(C, "choose an .onnx file with denoise-model.json beside it"));
}
let bytes = read_capped(&src.with_file_name("denoise-model.json"), MAX_MANIFEST_BYTES)
.ok_or_else(|| bad(C, "denoise-model.json is missing or too large"))?;
let m = manifest::parse(&bytes).map_err(|e| bad(C, e.to_string()))?;
let dir = models_dir(s, C)?;
Ok(json!({"kind": "model", "domain": "denoise", "path": path, "fileName": src.file_name().map(|n| n.to_string_lossy()), "model": m,
"sizeBytes": std::fs::metadata(src).ok().map(|m| m.len()), "alreadyInstalled": dir.join(&m.id).join("model.onnx").is_file()}))
}
/// What an error says, without the command it came from (for a line shown to the user).
fn plain(e: &EngineError) -> String {
match e {
EngineError::BadParams { msg, .. } => msg.clone(),
other => other.to_string(),
}
}
/// Install every download that has arrived: each was accepted when it was started, so it is installed and chosen without
/// another question. Called by `denoise.models.downloads` and by every frame's `denoise.pump`.
pub(crate) fn finish_downloads(s: &mut Session) {
if let Some(job) = &s.denoise.install {
let outcome = match job.result.try_recv() {
Ok(v) => Some(v),
Err(mpsc::TryRecvError::Empty) => None,
Err(mpsc::TryRecvError::Disconnected) => Some(Err("the installation worker stopped".into())),
};
if let Some(outcome) = outcome
&& let Some(job) = s.denoise.install.take()
{
let state = match outcome {
Ok((v, model)) => match activate_install(s, &v, model, job.activate) {
Ok(_) => State::Installed,
Err(e) => State::Failed(plain(&e)),
},
Err(e) => State::Failed(e),
};
s.denoise.downloads.set_outcome(&job.id, state);
}
}
if s.denoise.install.is_none()
&& let Some((id, path, _sha)) = s.denoise.downloads.finished().into_iter().next()
&& let Err(e) = start_install(s, path, id.clone(), true, true)
{
s.denoise.downloads.set_outcome(&id, State::Failed(plain(&e)));
}
}
fn download(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.download";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
let dir = models_dir(s, C)?;
let k = known::find(id).ok_or_else(|| bad(C, "LightCraft has no download for that model: get the file from its page, then add it"))?;
let spec = k.download.clone().ok_or_else(|| bad(C, "LightCraft has no download for that model: get the file from its page, then add it"))?;
if !cfg!(feature = "denoise") {
return Err(bad(C, "this build cannot run denoise models, so there is nothing to download for it"));
}
if !bool_or(p, "acknowledged", false) {
return Err(bad(
C,
"the licence has not been accepted: show the user the model's terms (see `denoise.models.list`) and pass `acknowledged: true` once they agree",
));
}
if installed_models(&dir).iter().any(|i| i.manifest.id == spec.id) {
return Err(bad(C, "that model is already installed"));
}
let host = lightcraft_denoise::known::host(&spec.url).to_string();
s.denoise.downloads.start(spec, &dir).map_err(|e| bad(C, e))?;
Ok(json!({"started": id, "from": host}))
}
fn download_row(id: &str, state: &State) -> Value {
let from = known::find(id).and_then(|k| host_of(&k));
match state {
State::Running { bytes, total } => json!({"id": id, "state": "running", "bytes": bytes, "total": total, "from": from}),
State::Done { path, .. } => json!({"id": id, "state": "done", "path": path.display().to_string(), "from": from}),
State::Installing => json!({"id": id, "state": "installing", "from": from}),
State::Installed => json!({"id": id, "state": "installed", "from": from}),
State::Failed(why) => json!({"id": id, "state": "failed", "error": why, "from": from}),
State::Cancelled => json!({"id": id, "state": "cancelled", "from": from}),
}
}
fn downloads(s: &mut Session, _: &Value) -> Result<Value> {
finish_downloads(s);
let all: Vec<Value> = s.denoise.downloads.snapshot().iter().map(|(id, st)| download_row(id, st)).collect();
Ok(json!({"running": s.denoise.downloads.running(), "downloads": all}))
}
fn download_cancel(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.downloadCancel";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
Ok(json!({"discarded": s.denoise.downloads.discard(id)}))
}
fn test(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.test";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
let dir = models_dir(s, C)?;
let found = installed_models(&dir).into_iter().find(|i| i.manifest.id == id).ok_or_else(|| bad(C, "that model is not installed"))?;
let run_on = s.denoise.settings.run_on();
let result = (s.denoise.loader)(&found.onnx, &found.manifest).and_then(|m| m.self_test(run_on));
let (ok, detail) = match &result {
Ok(v) => (true, v.clone()),
Err(e) => (false, json!(e)),
};
// keep the latest outcome with the model's record
let mut record = found.accepted;
if let Some(o) = record.as_object_mut() {
o.insert("selfTest".into(), if ok { detail.clone() } else { json!({"ok": false, "error": detail}) });
write_atomic(&found.onnx.with_file_name("installed.json"), &serde_json::to_vec_pretty(&record).map_err(|e| fail("record", e))?)
.map_err(EngineError::Other)?;
}
Ok(json!({"id": id, "ok": ok, "result": detail}))
}
fn remove(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.remove";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
let dir = models_dir(s, C)?;
if !lightcraft_denoise::licence::valid_id(id) {
return Err(bad(C, "not a model id"));
}
let home = dir.join(id);
// only a folder that is a model (it has a manifest) is ever deleted
if !home.join("denoise-model.json").is_file() {
return Err(bad(C, "that model is not installed"));
}
std::fs::remove_dir_all(&home).map_err(|e| fail("could not remove the model", e))?;
let mut st = read_settings(&dir);
if st.model.as_deref() == Some(id) {
st.model = installed_models(&dir).into_iter().map(|i| i.manifest.id).find(|m| m != id);
write_settings(&dir, &st).map_err(EngineError::Other)?;
}
s.denoise.touch();
s.denoise_refresh_active(true);
Ok(json!({"removed": id, "model": st.model}))
}
fn select(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.select";
let dir = models_dir(s, C)?;
let id = match p.get("id") {
None | Some(Value::Null) => None,
Some(v) => Some(v.as_str().ok_or_else(|| bad(C, "`id` must be a model id or null"))?.to_string()),
};
if let Some(id) = &id
&& !installed_models(&dir).iter().any(|i| &i.manifest.id == id)
{
return Err(bad(C, "that denoise model is not installed"));
}
let mut st = read_settings(&dir);
st.model = id;
write_settings(&dir, &st).map_err(EngineError::Other)?;
s.denoise.touch();
s.denoise_refresh_active(true);
Ok(json!({"model": st.model}))
}
fn settings(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.settings";
let dir = models_dir(s, C)?;
let mut st: Settings = read_settings(&dir);
if let Some(v) = p.get("auto") {
st.auto = Some(v.as_bool().ok_or_else(|| bad(C, "`auto` must be true or false"))?);
}
if let Some(v) = p.get("cacheGb") {
let n =
v.as_u64().filter(|n| (1..=10_000).contains(n)).ok_or_else(|| bad(C, "`cacheGb` must be a whole number of gigabytes from 1 to 10000"))?;
st.cache_gb = Some(n as u32);
}
if let Some(v) = p.get("threads") {
st.threads = match v {
Value::Null => None,
v => Some(v.as_u64().filter(|n| (1..=64).contains(n)).ok_or_else(|| bad(C, "`threads` must be from 1 to 64, or null"))? as u32),
};
}
let run_on = match p.get("runOn") {
None => None,
Some(v) => Some(v.as_str().and_then(RunOn::parse).ok_or_else(|| bad(C, "`runOn` must be auto, gpu or cpu"))?),
};
if let Some(r) = run_on {
st.run_on = Some(r.name().to_string());
st.gpu = None;
}
write_settings(&dir, &st).map_err(EngineError::Other)?;
if run_on.is_some() {
// a choice of where denoise runs is also a go-ahead to try the card again: a set-up that crashed or gave up
// before is forgotten, and the loaded model sets the card up afresh for the next photo
forget_gpu_set_ups(&dir);
if let Some(a) = &s.denoise.active {
*a.model.lock().unwrap_or_else(PoisonError::into_inner) = None;
}
}
s.denoise.touch();
s.denoise_refresh_active(true);
// a smaller limit takes effect now
s.denoise_evict(None);
Ok(json!({"auto": st.auto(), "cacheGb": st.cache_bytes() >> 30, "threads": st.threads, "runOn": st.run_on().name()}))
}
fn ids_param(p: &Value) -> Option<Vec<PhotoId>> {
p.get("ids").and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_u64).map(PhotoId).collect())
}
fn status_json(s: &Session, photo: Option<PhotoId>) -> Value {
let (files, bytes) = s.denoise_cache_usage();
let st = &s.denoise.settings;
let model = s.denoise.active.as_ref().map(|a| json!({"id": a.id, "name": a.manifest.name}));
let mut v = json!({
"enabled": s.denoise.active.is_some(),
"model": model,
"runtime": cfg!(feature = "denoise"),
"productsDir": s.denoise_products_dir().map(|d| d.display().to_string()),
"queued": s.denoise.queued(),
"running": s.denoise_running_json(),
"ready": s.media.denoise.len(),
"made": s.denoise_made(),
"auto": st.auto(),
"runOn": st.run_on().name(),
"device": s.denoise_device(),
"cache": {"files": files, "bytes": bytes, "limitBytes": st.cache_bytes()},
"generation": s.denoise.generation,
});
if let (Some(id), Some(o)) = (photo, v.as_object_mut()) {
o.insert("photo".into(), serde_json::to_value(s.denoise_photo_state(id)).unwrap_or(Value::Null));
}
v
}
/// A per-photo switch, independent of the selection and Auto Sync. Turning on starts at 50%.
fn toggle(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.toggle";
let id = match p.get("id") {
Some(v) => PhotoId(v.as_u64().ok_or_else(|| bad(C, "`id` must be a photo id"))?),
None => s.active().ok_or_else(|| bad(C, "select a raw photo"))?,
};
let photo = s.catalog.photo(id).ok_or_else(|| bad(C, "no such photo"))?;
if !crate::denoise::eligible(photo) {
return Err(bad(C, "AI Denoise applies to raw photos developed from their sensor data"));
}
let mut d = (*s.develop_of(id).ok_or_else(|| bad(C, "no such photo"))?).clone();
let on = match p.get("enabled") {
Some(v) => v.as_bool().ok_or_else(|| bad(C, "`enabled` must be true or false"))?,
None => !d.enhance.denoise_enabled(),
};
if on {
if !d.enhance.denoise.is_finite() {
return Err(bad(C, "the photo has an invalid Denoise amount"));
}
s.denoise_refresh_active(true);
if s.denoise.active.is_none() {
return Err(bad(C, "AI Denoise needs a model: install one in Settings > AI Denoise"));
}
}
// The switch is its own setting: the Amount stays where it is (0 included) when it is flipped, so flipping compares
// with and without. A photo that never had an Amount starts at 50 the first time.
let amount = if on && d.enhance.denoise_on.is_none() && d.enhance.denoise <= 0.0 { 50.0 } else { d.enhance.denoise };
if d.enhance.denoise != amount || d.enhance.denoise_on != Some(on) {
s.end_interaction()?;
d.enhance.denoise = amount;
d.enhance.denoise_on = Some(on);
let op = s.develop_op(id, d, "AI Denoise").ok_or_else(|| bad(C, "no such photo"))?;
s.commit("AI Denoise", op)?;
}
if on {
s.denoise_enqueue(&[id], true, false);
} else {
s.denoise_cancel(Some(id), false);
}
Ok(json!({"id": id.0, "enabled": on, "amount": amount}))
}
fn queue(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.queue";
s.denoise_refresh_active(true);
if s.denoise.active.is_none() {
return Err(bad(C, "no denoise model is chosen: install one first (`denoise.models.download` or `denoise.models.install`)"));
}
if s.denoise_products_dir().is_none() {
return Err(bad(C, "denoise keeps its pictures in the library's folder: open a library on disk first"));
}
let ids = match ids_param(p) {
Some(ids) => ids,
None => match str_param(p, "scope").unwrap_or("selected") {
"selected" => s.selection.active.iter().chain(s.selection.ids.iter()).copied().collect(),
"visible" => s.visible_cloned().to_vec(),
"withAmount" => s.catalog.photos().filter(|ph| ph.develop.denoise_amount() > 0.0).map(|ph| ph.id).collect(),
other => return Err(bad(C, format!("unknown scope `{other}`: use selected, visible or withAmount"))),
},
};
let queued = s.denoise_enqueue(&ids, false, bool_or(p, "retry", false));
let mut v = status_json(s, None);
if let Some(o) = v.as_object_mut() {
o.insert("added".into(), json!(queued));
}
Ok(v)
}
fn cancel(s: &mut Session, p: &Value) -> Result<Value> {
let photo = p.get("id").and_then(Value::as_u64).map(PhotoId);
let n = s.denoise_cancel(photo, bool_or(p, "all", photo.is_none()));
Ok(json!({"cancelled": n}))
}
fn status(s: &mut Session, p: &Value) -> Result<Value> {
s.denoise_refresh_active(false);
Ok(status_json(s, p.get("id").and_then(Value::as_u64).map(PhotoId)))
}
/// `denoise.pump {pace?}`: cheap to call every frame. Takes in what the background job has finished and keeps one going.
fn pump(s: &mut Session, p: &Value) -> Result<Value> {
s.denoise_pump(Pace::parse(p.get("pace").and_then(Value::as_str)));
Ok(json!({
"active": s.denoise.active.is_some(),
"running": s.denoise_running_json(),
"queued": s.denoise.queued(),
"ready": s.media.denoise.len(),
"generation": s.denoise.generation,
"downloading": s.denoise.downloads.running(),
}))
}
fn clear(s: &mut Session, _: &Value) -> Result<Value> {
let (files, bytes) = s.denoise_clear();
Ok(json!({"deleted": files, "bytes": bytes}))
}
pub fn specs() -> Vec<CommandSpec> {
vec![
cmd!(query "denoise.models.list", "Denoise Models", [], None, "{} → {dir, productsDir, model, runtime, auto, cacheGb, threads, runOn, models: [{id, name, version, licence{name, commercial, url, notice}, provenance, source, sizeBytes, sha256, tile, known, downloadHost, installed, selected, accepted}]}", always, list),
cmd!(query "denoise.models.inspect", "Inspect Denoise Model", [], None, "{path} → model terms from an adjacent manifest; never executes the file", always, inspect),
cmd!(query "denoise.models.install", "Install Denoise Model", [], None, "{path, acknowledged: true, activate?: true, background?: false} → {installed, model} or {started} — install a denoise model from a file: an .onnx with a denoise-model.json beside it, or an archive LightCraft knows (the darktable `.dtmodel`). `acknowledged` must be true: the user has been shown the model's licence and accepted it. Unless `activate` is false it becomes the model in use", always, install),
cmd!(query "denoise.models.download", "Download Denoise Model", [], None, "{id, acknowledged: true} → {started, from} — fetch a model LightCraft has a pinned address for (see `downloadHost` in the list) in the background over HTTPS with lightcraft-fetch. `acknowledged` must be true: the user has been shown the model's terms and accepted them. It is checked against its size and SHA-256, installed and chosen by itself; `denoise.models.downloads` shows how far it is", always, download),
cmd!(query "denoise.models.downloads", "Denoise Model Downloads", [], None, "{} → {running, downloads: [{id, state: running | done | installed | failed | cancelled, bytes, total, error, from}]} — also installs any download that has arrived", always, downloads),
cmd!(query "denoise.models.downloadCancel", "Cancel Denoise Model Download", [], None, "{id} → {discarded} — stop a download, or delete a finished one that was not installed", always, download_cancel),
cmd!(query "denoise.models.test", "Test Denoise Model", [], None, "{id} → {ok, result} — load an installed model and check it gives sensible pictures; needs the denoise runtime", always, test),
cmd!(query "denoise.models.remove", "Remove Denoise Model", [], None, "{id} → {removed, model} — delete an installed model (its cached pictures stay until they are evicted)", always, remove),
cmd!(query "denoise.models.select", "Choose Denoise Model", [], None, "{id: installed model | null} → {model} — null switches denoise off", always, select),
cmd!(query "denoise.settings", "Denoise Settings", [], None, "{auto?: bool, cacheGb?: 1..10000, threads?: 1..64 | null, runOn?: auto | gpu | cpu} → the settings — `auto`: make the picture of the photos being looked at that have a Denoise amount; `cacheGb`: how much the cached pictures may take; `threads`: most tiles run at once; `runOn`: where the model runs — `auto` (the default) the graphics card where it is faster than the processor (both timed on this computer when the card is set up), `gpu` the card whenever it can run the model, `cpu` the processor. Setting `runOn` also lets the card be tried again after a set-up that failed or closed LightCraft", always, settings),
cmd!(
"denoise.toggle",
"AI Denoise",
[],
None,
"{id?: photoId, enabled?: bool} → {id, enabled, amount} — turn AI Denoise on (50% initially) or off for one raw photo, independent of selection and Auto Sync; needs an installed model to turn on",
always,
toggle
),
cmd!(query "denoise.queue", "Denoise Photos", [], None, "{ids?: [photoId], scope?: selected | visible | withAmount, retry?: false} → {added, …status} — make the denoised pictures of raw photos that have none, in the background. `retry` tries photos that failed before", always, queue),
cmd!(query "denoise.cancel", "Cancel Denoise", [], None, "{id?, all?} → {cancelled} — stop the photo being made and/or forget the ones waiting (all of them when `id` is omitted)", always, cancel),
cmd!(query "denoise.status", "Denoise Status", [], None, "{id?} → {enabled, model, runtime, productsDir, queued, running: {photo, done, total, seconds} | null, ready, made, auto, runOn, device: {kind: gpu | cpu | pending | none, adapter?, reason?, cardMs?, cpuMs?, fellBack?}, cache: {files, bytes, limitBytes}, generation, photo?: {state: notApplicable | noModel | ready | queued | running | failed | idle, …}}", always, status),
cmd!(query "denoise.pump", "Denoise Pump", [], None, "{pace?: pause | light | normal | full} → {active, running, queued, ready, generation, downloading} — called every frame by the app: takes in finished work and keeps one photo going", always, pump),
cmd!(query "denoise.clear", "Clear Denoise Cache", [], None, "{} → {deleted, bytes} — delete every cached denoised picture of this library (they are made again when wanted)", always, clear),
]
}
+184
View File
@@ -0,0 +1,184 @@
//! `faces.detect`: find faces with the YuNet detector the user has downloaded (Settings > Faces).
//!
//! The photo is rendered upright and uncropped with default settings (so edits, crops and rotations you have
//! made do not matter), the detector runs on it, and the faces come back as fractions of the photo's width
//! and height, the same frame face regions from XMP use. Unless `apply` is false, they are added to the photo as
//! unnamed face regions in one undoable step; earlier detections are replaced, and regions that came from XMP (or
//! that you drew or named) are never touched. Like every region edit it is catalog-only: no sidecar is written.
use std::path::PathBuf;
use std::sync::{Arc, Mutex, PoisonError};
use std::time::Instant;
use lightcraft_catalog::Op;
use lightcraft_develop::DevelopSettings;
use lightcraft_faces::known::YUNET_ID;
use lightcraft_faces::yunet::{Detector, Face, Options};
use lightcraft_meta::{Region, RegionKind};
use serde_json::{Value, json};
use super::{CommandSpec, bad, cmd, has_selection};
use crate::{Result, Session};
const C: &str = "faces.detect";
/// Regions made by this command say so in their description; only those are replaced on a new run.
pub(crate) const MARK: &str = "Detected by ";
pub(crate) const LABEL: &str = "YuNet 2023mar";
/// Long edge of the picture the detector looks at (it shrinks it to its own 640 anyway).
const LOOK_EDGE: usize = 1280;
/// YuNet is 232 KB; a file this much larger in its folder is not it.
const MAX_MODEL_BYTES: u64 = 8 << 20;
/// How much a detection must overlap a region the photo already has to count as the same face.
const EXISTING_IOU: f64 = 0.3;
/// What to tell someone who has not downloaded the detector yet.
const NOT_INSTALLED: &str = "finding faces needs the YuNet face detector (232 KB, MIT): download it in Settings > Faces";
/// The installed YuNet, loaded once and kept (a new one is loaded when the model folder changes).
/// (also used by the background scan, which aligns faces by the detector's landmarks and finds faces in photos that have none)
pub(crate) fn detector(s: &Session) -> std::result::Result<Arc<Detector>, String> {
static LOADED: Mutex<Option<(PathBuf, Arc<Detector>)>> = Mutex::new(None);
let home = s.face_models_dir.as_deref().ok_or("this build has nowhere to keep face models (the desktop app does)")?.join(YUNET_ID);
let path = home.join("model.onnx");
// installing writes the manifest last, so a folder without one is not (yet) a model
if !path.is_file() || !home.join("face-model.json").is_file() {
return Err(NOT_INSTALLED.into());
}
let mut loaded = LOADED.lock().unwrap_or_else(PoisonError::into_inner);
if let Some((p, d)) = loaded.as_ref()
&& *p == path
{
return Ok(d.clone());
}
let len = std::fs::metadata(&path).map_err(|e| format!("could not read the detector: {e}"))?.len();
if len > MAX_MODEL_BYTES {
return Err("the detector's file is far bigger than YuNet: remove it in Settings > Faces and download it again".into());
}
let bytes = std::fs::read(&path).map_err(|e| format!("could not read the detector: {e}"))?;
let d = Arc::new(Detector::new(&bytes).map_err(|e| format!("{e}: remove the detector in Settings > Faces and download it again"))?);
*loaded = Some((path, d.clone()));
Ok(d)
}
fn face_json(f: &Face) -> Value {
json!({
"rect": {"x0": f.x0, "y0": f.y0, "x1": f.x1, "y1": f.y1},
"score": f.score,
"landmarks": f.landmarks.iter().map(|(x, y)| json!([x, y])).collect::<Vec<_>>(),
})
}
/// A detected face that lies on a region the photo already has (from XMP, drawn or named) is that region, not a new face.
fn overlaps_existing(f: &Face, regions: &[Region]) -> bool {
regions.iter().any(|r| {
let (w, h) = (
(r.rect.x1.min(f64::from(f.x1)) - r.rect.x0.max(f64::from(f.x0))).max(0.0),
(r.rect.y1.min(f64::from(f.y1)) - r.rect.y0.max(f64::from(f.y0))).max(0.0),
);
let inter = w * h;
let union = (r.rect.x1 - r.rect.x0) * (r.rect.y1 - r.rect.y0) + f64::from((f.x1 - f.x0) * (f.y1 - f.y0)) - inter;
union > 0.0 && inter / union >= EXISTING_IOU
})
}
pub(crate) fn is_detected(r: &Region) -> bool {
r.description.as_deref().is_some_and(|d| d.starts_with(MARK))
}
/// The photo as the faces see it: upright, uncropped, default settings, as 8-bit RGB with its size, its long edge at
/// most `edge`.
pub(crate) fn render_rgb(s: &mut Session, id: lightcraft_catalog::PhotoId, edge: usize) -> std::result::Result<(Vec<u8>, usize, usize), String> {
let settings = DevelopSettings::default();
let job = s.preview_job(id, edge, edge, false, &settings).ok_or("no such photo")?;
let image = job.run().rendered?.image;
let rgb: Vec<u8> = image.data.iter().flat_map(|px| [px[0], px[1], px[2]]).collect();
Ok((rgb, image.width, image.height))
}
fn detect(s: &mut Session, p: &Value) -> Result<Value> {
let apply = p.get("apply").and_then(Value::as_bool).unwrap_or(true);
let defaults = Options::default();
let opts = Options {
score: p.get("score").and_then(Value::as_f64).map_or(defaults.score, |v| v as f32),
nms_iou: p.get("nmsIou").and_then(Value::as_f64).map_or(defaults.nms_iou, |v| v as f32),
max_faces: p.get("maxFaces").and_then(Value::as_u64).map_or(defaults.max_faces, |v| usize::try_from(v).unwrap_or(defaults.max_faces)),
};
let detector = detector(s).map_err(|e| bad(C, e))?;
let started = Instant::now();
let (mut results, mut ops) = (Vec::new(), Vec::new());
for id in s.targets(p) {
let (rgb, width, height) = match render_rgb(s, id, LOOK_EDGE) {
Ok(r) => r,
Err(e) => {
results.push(json!({"id": id.0, "error": e}));
continue;
}
};
let faces = match detector.detect(&rgb, width, height, &opts) {
Ok(f) => f,
Err(e) => {
results.push(json!({"id": id.0, "error": e.to_string()}));
continue;
}
};
if apply && let Some(photo) = s.catalog.photo(id) {
let mut meta = photo.meta.clone();
meta.regions.retain(|r| !is_detected(r));
let fresh: Vec<&Face> = faces.iter().filter(|f| !overlaps_existing(f, &meta.regions)).collect();
for f in fresh {
meta.regions.push(Region {
rect: lightcraft_geom::Rect { x0: f64::from(f.x0), y0: f64::from(f.y0), x1: f64::from(f.x1), y1: f64::from(f.y1) },
kind: RegionKind::Face,
name: None,
description: Some(format!("{MARK}{LABEL}")),
});
}
ops.push(Op::SetMeta { id, meta: Box::new(meta) });
}
results.push(json!({"id": id.0, "faces": faces.iter().map(face_json).collect::<Vec<_>>()}));
}
let applied = !ops.is_empty();
if applied {
s.commit("Detect Faces", Op::Batch { ops })?;
// regions are not written to XMP, so a sidecar must not be rewritten for this
s.skip_auto_write = true;
}
Ok(json!({"detector": LABEL, "applied": applied, "photos": results, "ms": started.elapsed().as_secs_f64() * 1000.0}))
}
pub fn specs() -> Vec<CommandSpec> {
vec![cmd!(
"faces.detect",
"Detect Faces",
["Photo"],
None,
"{ids?, apply?: bool (default true), score?: 0.6, nmsIou?: 0.3, maxFaces?} → {photos: [{id, faces: [{rect, score, landmarks}]}]}. Finds faces with the YuNet detector (download it first: `faces.models.download {id: \"yunet-2023mar\"}`) and, unless `apply` is false, adds them as unnamed face regions (one undo step), replacing earlier detections but never regions from XMP or ones you named; the sidecar is not touched",
has_selection,
detect
)]
}
#[cfg(test)]
mod tests {
use super::*;
fn face(x0: f32, y0: f32, x1: f32, y1: f32) -> Face {
Face { x0, y0, x1, y1, score: 0.9, landmarks: [(0.0, 0.0); 5] }
}
fn region(x0: f64, y0: f64, x1: f64, y1: f64) -> Region {
Region { rect: lightcraft_geom::Rect { x0, y0, x1, y1 }, kind: RegionKind::Face, name: Some("Ann".into()), description: None }
}
#[test]
fn a_detection_on_an_existing_region_is_not_a_new_face() {
let existing = [region(0.40, 0.20, 0.60, 0.50), region(0.10, 0.10, 0.20, 0.20)];
// the same face, a bit larger or shifted (a Lightroom box is not YuNet's box)
assert!(overlaps_existing(&face(0.38, 0.18, 0.64, 0.54), &existing));
assert!(overlaps_existing(&face(0.42, 0.22, 0.58, 0.48), &existing));
// another face, a sliver of overlap, and no regions at all
assert!(!overlaps_existing(&face(0.70, 0.20, 0.90, 0.50), &existing));
assert!(!overlaps_existing(&face(0.59, 0.49, 0.80, 0.80), &existing));
assert!(!overlaps_existing(&face(0.0, 0.0, 0.0, 0.0), &existing), "an empty box overlaps nothing");
assert!(!overlaps_existing(&face(0.4, 0.2, 0.6, 0.5), &[]));
}
}
+511
View File
@@ -0,0 +1,511 @@
//! Face model commands (`faces.*`): which models exist, inspecting a file the user dropped in, installing
//! it once its licence is accepted, removing it, and the faces on/off setting.
//!
//! Models live in the folder the host provides (`Session::face_models_dir`): one subfolder per model with
//! `model.onnx`, `face-model.json` (its manifest) and `installed.json` (what the user accepted), plus
//! `settings.json` beside them. Model files and manifests are hostile input: sizes are capped, ids are
//! validated before they become folder names, and nothing outside a model's own folder is ever touched.
use std::io::Read;
use std::path::{Path, PathBuf};
use lightcraft_faces::catalog::{self, Catalog};
use lightcraft_faces::hash::sha256_file;
use lightcraft_faces::manifest::{self, MAX_MANIFEST_BYTES, MAX_MODEL_BYTES, ModelManifest, Role};
use lightcraft_faces::suggest::{Suggestion, suggest_with};
use lightcraft_faces::{known, onnx};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use super::{CommandSpec, always, bad, cmd, str_param};
use crate::face_download::{self, State};
use crate::{EngineError, Result, Session};
#[derive(Default, Serialize, Deserialize)]
#[serde(default, rename_all = "camelCase")]
pub(super) struct FaceSettings {
pub(super) enabled: bool,
/// The installed recogniser in use.
pub(super) embedder: Option<String>,
}
fn fail(what: &str, e: impl std::fmt::Display) -> EngineError {
EngineError::Other(format!("{what}: {e}"))
}
fn models_dir(s: &Session, cmd: &str) -> Result<PathBuf> {
s.face_models_dir.clone().ok_or_else(|| bad(cmd, "this build has nowhere to keep face models (the desktop app does)"))
}
/// Read at most `max` bytes of a small file.
fn read_capped(path: &Path, max: usize) -> Option<Vec<u8>> {
let mut buf = Vec::new();
std::fs::File::open(path).ok()?.take(max as u64 + 1).read_to_end(&mut buf).ok()?;
(buf.len() <= max).then_some(buf)
}
pub(super) fn read_settings(dir: &Path) -> FaceSettings {
read_capped(&dir.join("settings.json"), 4096).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default()
}
fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> {
let part = path.with_extension("part");
std::fs::write(&part, bytes).map_err(|e| fail(&format!("could not write {}", path.display()), e))?;
std::fs::rename(&part, path).map_err(|e| {
let _ = std::fs::remove_file(&part);
fail(&format!("could not write {}", path.display()), e)
})
}
fn write_settings(dir: &Path, st: &FaceSettings) -> Result<()> {
std::fs::create_dir_all(dir).map_err(|e| fail("could not create the face models folder", e))?;
write_atomic(&dir.join("settings.json"), &serde_json::to_vec_pretty(st).map_err(|e| fail("settings", e))?)
}
pub(super) struct Installed {
pub(super) manifest: ModelManifest,
pub(super) accepted: Value,
}
/// The models in the folder. A folder that is not a valid model (wrong name, no model file, a manifest that
/// does not validate or does not match its folder) is ignored, never trusted.
pub(super) fn installed_models(dir: &Path) -> Vec<Installed> {
let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() };
let mut out = Vec::new();
for e in entries.flatten() {
let p = e.path();
let Some(name) = p.file_name().and_then(|n| n.to_str()).map(str::to_string) else { continue };
if !manifest::valid_id(&name) || !p.is_dir() || !p.join("model.onnx").is_file() {
continue;
}
let Some(m) = read_capped(&p.join("face-model.json"), MAX_MANIFEST_BYTES).and_then(|b| manifest::parse(&b).ok()) else { continue };
if m.id != name {
continue;
}
let accepted = read_capped(&p.join("installed.json"), 4096).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or(Value::Null);
out.push(Installed { manifest: m, accepted });
}
out.sort_by(|a, b| a.manifest.id.cmp(&b.manifest.id));
out
}
/// Read the user's catalog (`catalog.json` in the models folder) anew: a few kilobytes, so it is always current.
fn refresh_catalog(s: &mut Session) {
let Some(dir) = s.face_models_dir.clone() else { return };
let file = dir.join("catalog.json");
s.face_catalog = match read_capped(&file, catalog::MAX_CATALOG_BYTES) {
Some(bytes) => catalog::parse(&bytes),
None if file.is_file() => catalog::parse(&vec![b' '; catalog::MAX_CATALOG_BYTES + 1]),
None => Catalog::default(),
};
}
/// The manifests of the catalog's models (what a file's hash is matched against, besides the built-in ones).
fn catalog_manifests(s: &Session) -> Vec<ModelManifest> {
s.face_catalog.entries.iter().map(|e| e.manifest.clone()).collect()
}
/// How to fetch a model: the built-in address, or the one in the user's catalog.
fn download_spec(s: &Session, id: &str) -> Option<known::Download> {
known::download(id).or_else(|| s.face_catalog.entries.iter().find(|e| e.manifest.id == id).and_then(catalog::Entry::download))
}
fn row(s: &Session, m: &ModelManifest, installed: bool, selected: bool, accepted: &Value) -> Value {
let from_catalog = s.face_catalog.entries.iter().any(|e| e.manifest.id == m.id);
json!({
"id": m.id,
"name": m.name,
"role": m.role,
"licence": m.licence,
"provenance": m.provenance,
"source": m.source,
"sizeBytes": m.size_bytes,
"sha256": m.sha256,
// how fast it is as a multiple of a ResNet-100 model (computers differ, ratios between models hardly do), and
// that in words; null when nobody has measured it
"speed": m.speed,
"speedText": m.speed.map(manifest::describe_speed).filter(|t| !t.is_empty()),
"known": from_catalog || known::all().iter().any(|k| k.id == m.id),
// from the user's own catalog file
"fromCatalog": from_catalog,
// a pinned address LightCraft can fetch it from (the user presses Download), and which site that is
"downloadHost": download_spec(s, &m.id).map(|d| known::host(&d.url).to_string()),
"installed": installed,
"selected": selected,
"accepted": accepted,
})
}
struct Inspection {
sha256: String,
size: u64,
file_name: String,
suggestion: Suggestion,
}
/// Look at a model file. `known_sha` is its SHA-256 when something already checked it (a finished download), so a
/// file of hundreds of megabytes is not read again just to be hashed.
fn inspect_file(path: &str, cmd: &str, known_sha: Option<&str>, extra: &[ModelManifest]) -> Result<Inspection> {
let p = Path::new(path);
let meta = std::fs::metadata(p).map_err(|e| bad(cmd, format!("cannot read `{path}`: {e}")))?;
if !meta.is_file() {
return Err(bad(cmd, format!("`{path}` is not a file")));
}
if meta.len() == 0 || meta.len() > MAX_MODEL_BYTES {
return Err(bad(cmd, "a model file must be between 1 byte and 512 MiB"));
}
let sha256 = match known_sha {
Some(h) => h.to_string(),
None => sha256_file(p).map_err(|e| fail(&format!("could not read {path}"), e))?,
};
let file_name = p.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default();
let suggestion = match onnx::probe_path(p) {
Ok(info) => suggest_with(&info, &sha256, meta.len(), &file_name, extra),
Err(e) => Suggestion::Unsupported(e.to_string()),
};
Ok(Inspection { sha256, size: meta.len(), file_name, suggestion })
}
fn list(s: &mut Session, _: &Value) -> Result<Value> {
refresh_catalog(s);
let dir = s.face_models_dir.clone();
let (settings, installed) = match &dir {
Some(d) => (read_settings(d), installed_models(d)),
None => (FaceSettings::default(), Vec::new()),
};
let on_disk = |id: &str| installed.iter().find(|i| i.manifest.id == id);
let mut models: Vec<Value> = Vec::new();
for k in known::all() {
let found = on_disk(&k.id);
models.push(row(s, &k, found.is_some(), settings.embedder.as_deref() == Some(k.id.as_str()), found.map_or(&Value::Null, |f| &f.accepted)));
}
// the user's own catalog
for e in &s.face_catalog.entries {
let found = on_disk(&e.manifest.id);
let chosen = settings.embedder.as_deref() == Some(e.manifest.id.as_str());
models.push(row(s, &e.manifest, found.is_some(), chosen, found.map_or(&Value::Null, |f| &f.accepted)));
}
for i in &installed {
let listed = known::all().iter().any(|k| k.id == i.manifest.id) || s.face_catalog.entries.iter().any(|e| e.manifest.id == i.manifest.id);
if !listed {
models.push(row(s, &i.manifest, true, settings.embedder.as_deref() == Some(i.manifest.id.as_str()), &i.accepted));
}
}
Ok(json!({
"dir": dir.as_ref().map(|d| d.display().to_string()),
"enabled": settings.enabled,
"embedder": settings.embedder,
// Native builds always have the pure-Rust runner; model files/background scanning remain native-only.
"runtime": cfg!(not(target_arch = "wasm32")),
"models": models,
// the user's own list of models to download: where it goes, how many it holds, and what was wrong with the rest
"catalog": {
"path": dir.as_ref().map(|d| d.join("catalog.json").display().to_string()),
"models": s.face_catalog.entries.len(),
"errors": s.face_catalog.errors,
},
}))
}
fn inspect(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.models.inspect";
let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?;
refresh_catalog(s);
let ins = inspect_file(path, C, None, &catalog_manifests(s))?;
let installed = s.face_models_dir.as_deref().map(installed_models).unwrap_or_default();
let (kind, model, assumptions, reason) = match &ins.suggestion {
Suggestion::Known(m) => ("known", Some(m), Vec::new(), None),
Suggestion::Draft { manifest, assumptions } => ("draft", Some(manifest), assumptions.clone(), None),
Suggestion::Unsupported(why) => ("unsupported", None, Vec::new(), Some(why.clone())),
};
let already = model.is_some_and(|m| installed.iter().any(|i| i.manifest.sha256 == m.sha256 && i.manifest.id == m.id));
Ok(json!({
"path": path,
"fileName": ins.file_name,
"sizeBytes": ins.size,
"sha256": ins.sha256,
"kind": kind,
"model": model.map(|m| row(s, m, already, false, &Value::Null)),
"assumptions": assumptions,
"reason": reason,
"alreadyInstalled": already,
}))
}
fn install(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.models.install";
let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?;
let acknowledged = p.get("acknowledged").and_then(Value::as_bool).unwrap_or(false);
let activate = p.get("activate").and_then(Value::as_bool).unwrap_or(true);
refresh_catalog(s);
let ins = inspect_file(path, C, None, &catalog_manifests(s))?;
install_file(s, C, Path::new(path), ins, acknowledged, activate, false)
}
/// Install the inspected model file `src`. A recognition model that passes its self-test becomes the one in use and
/// recognition is switched on (`activate`, the default): installing a model is how someone says they want it.
/// `verified` is for a download that was already checked against its recorded hash: it is moved into place instead
/// of being copied and read again.
fn install_file(s: &mut Session, c: &str, src: &Path, ins: Inspection, acknowledged: bool, activate: bool, verified: bool) -> Result<Value> {
let dir = models_dir(s, c)?;
let m = match ins.suggestion {
Suggestion::Known(m) | Suggestion::Draft { manifest: m, .. } => m,
Suggestion::Unsupported(why) => return Err(bad(c, format!("this model cannot be used yet: {why}"))),
};
if !matches!(m.role, Role::Embedder | Role::Detector) {
return Err(bad(c, "only face detection and recognition models can be installed"));
}
manifest::validate(&m).map_err(|e| bad(c, e.to_string()))?;
if !acknowledged {
return Err(bad(c, "the licence has not been accepted: show the user its terms and pass `acknowledged: true` once they agree"));
}
let home = dir.join(&m.id);
std::fs::create_dir_all(&home).map_err(|e| fail("could not create the model's folder", e))?;
let (part, final_path) = (home.join("model.onnx.part"), home.join("model.onnx"));
let moved = verified && std::fs::rename(src, &part).is_ok();
let placed = if moved {
Ok(())
} else {
std::fs::copy(src, &part).map_err(|e| fail("could not copy the model (is the disk full?)", e)).and_then(|_| match sha256_file(&part) {
Ok(h) if h == ins.sha256 => Ok(()),
Ok(_) => Err(EngineError::Other("the copy does not match the original (the file changed while copying?)".into())),
Err(e) => Err(fail("could not check the copy", e)),
})
};
if let Err(e) = placed {
let _ = std::fs::remove_file(&part);
return Err(e);
}
// Validate the candidate before replacing a working installation.
let test = match self_test(&part, &m) {
Ok(t) => t,
Err(e) => {
let _ = std::fs::remove_file(&part);
let _ = std::fs::remove_dir(&home);
return Err(bad(c, e));
}
};
if let Err(e) = std::fs::rename(&part, &final_path) {
let _ = std::fs::remove_file(&part);
return Err(fail("could not finish the copy", e));
}
write_atomic(&home.join("face-model.json"), &serde_json::to_vec_pretty(&m).map_err(|e| fail("manifest", e))?)?;
let accepted = json!({"acceptedAt": (s.clock)(), "licence": m.licence.name, "commercial": m.licence.commercial, "fileName": ins.file_name, "selfTest": test});
write_atomic(&home.join("installed.json"), &serde_json::to_vec_pretty(&accepted).map_err(|e| fail("record", e))?)?;
// a file that was downloaded and waited in the staging folder has served: the installed copy is the model now
if src.parent() == Some(dir.join(face_download::STAGING).as_path()) {
let _ = std::fs::remove_file(src);
s.face_downloads.staged_file_used(src);
}
// the newest model is the one in use (the earlier one stays installed, and `faces.models.select` switches back)
if activate && m.role == Role::Embedder {
let mut st = read_settings(&dir);
st.embedder = Some(m.id.clone());
st.enabled = true;
write_settings(&dir, &st)?;
}
let now = read_settings(&dir);
let selected = now.embedder.as_deref() == Some(m.id.as_str());
Ok(json!({"installed": row(s, &m, true, selected, &accepted), "active": {"embedder": now.embedder, "enabled": now.enabled}}))
}
/// What an error says, without the command it came from (for a line shown to the user).
fn plain(e: &EngineError) -> String {
match e {
EngineError::BadParams { msg, .. } => msg.clone(),
other => other.to_string(),
}
}
/// Install every download that has arrived: each was accepted when it was started, so it is installed, chosen and
/// switched on without another question. Called by `faces.models.downloads` and by every frame's `faces.pump`.
pub(super) fn finish_downloads(s: &mut Session) {
const C: &str = "faces.models.download";
let extra = catalog_manifests(s);
for (id, path, sha) in s.face_downloads.finished() {
let outcome = inspect_file(&path.to_string_lossy(), C, Some(&sha), &extra).and_then(|ins| install_file(s, C, &path, ins, true, true, true));
match outcome {
Ok(_) => s.face_downloads.set_outcome(&id, State::Installed),
Err(e) => {
// a file that cannot be installed is of no use to anyone: throw it away, say why
let _ = std::fs::remove_file(&path);
s.face_downloads.set_outcome(&id, State::Failed(plain(&e)));
}
}
}
}
/// `faces.models.download {id, acknowledged: true}`: fetch a model LightCraft knows an address for, in the background;
/// when it has arrived and checked out it is installed.
fn download(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.models.download";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
let dir = models_dir(s, C)?;
refresh_catalog(s);
let spec = download_spec(s, id).ok_or_else(|| bad(C, "LightCraft has no download for that model: get the file from its page, then add it"))?;
if cfg!(target_arch = "wasm32") {
return Err(bad(C, "the web build downloads no models"));
}
if p.get("acknowledged").and_then(Value::as_bool) != Some(true) {
return Err(bad(
C,
"the licence has not been accepted: show the user the model's terms (see `faces.models.list`) and pass `acknowledged: true` once they agree",
));
}
if installed_models(&dir).iter().any(|i| i.manifest.id == spec.id) {
return Err(bad(C, "that model is already installed"));
}
let host = spec.host().to_string();
s.face_downloads.start(spec, &dir).map_err(|e| bad(C, e))?;
Ok(json!({"started": id, "from": host}))
}
fn download_row(s: &Session, id: &str, state: &State) -> Value {
let from = download_spec(s, id).map(|d| d.host().to_string());
match state {
State::Running { bytes, total } => json!({"id": id, "state": "running", "bytes": bytes, "total": total, "from": from}),
State::Done { path, .. } => json!({"id": id, "state": "done", "path": path.display().to_string(), "from": from}),
State::Installed => json!({"id": id, "state": "installed", "from": from}),
State::Failed(why) => json!({"id": id, "state": "failed", "error": why, "from": from}),
State::Cancelled => json!({"id": id, "state": "cancelled", "from": from}),
}
}
/// `faces.models.downloads`: where each download stands (a finished one is installed first).
fn downloads(s: &mut Session, _: &Value) -> Result<Value> {
finish_downloads(s);
let all: Vec<Value> = s.face_downloads.snapshot().iter().map(|(id, st)| download_row(s, id, st)).collect();
Ok(json!({"running": s.face_downloads.running(), "downloads": all}))
}
/// `faces.models.downloadCancel {id}`: stop a download, or clear a finished one (a staged file is deleted).
fn download_cancel(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.models.downloadCancel";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
Ok(json!({"discarded": s.face_downloads.discard(id)}))
}
/// Load the model and run its self-test: `Ok(null)` in a build without the recognition runtime (nothing can
/// be checked), `Ok(result)` when it passes, `Err(reason)` when it does not load or fails a check.
#[cfg(not(target_arch = "wasm32"))]
fn self_test(path: &Path, m: &ModelManifest) -> std::result::Result<Value, String> {
if m.role == Role::Detector {
let bytes = read_capped(path, 8 << 20).ok_or("the detector cannot be read within its 8 MiB limit")?;
let detector = lightcraft_faces::yunet::Detector::new(&bytes).map_err(|e| e.to_string())?;
if Some(detector.input_side()) != usize::try_from(m.input.width).ok()
|| m.input.width != m.input.height
|| !matches!(&m.output, manifest::OutputSpec::Detector { decoder } if decoder == "yunet-v2")
{
return Err("unsupported detector input or output decoder".into());
}
detector.detect(&[0u8; 12], 2, 2, &lightcraft_faces::yunet::Options::default()).map_err(|e| e.to_string())?;
return Ok(json!({"ok": true}));
}
let embedder = lightcraft_faces::runtime::Embedder::load(path, m).map_err(|e| e.to_string())?;
let t = embedder.self_test();
if !t.ok {
let failed: Vec<&str> = t.checks.iter().filter(|(_, ok)| !*ok).map(|(c, _)| c.as_str()).collect();
return Err(format!("the model failed its self-test: it does not {}", failed.join(", and does not ")));
}
serde_json::to_value(&t).map_err(|e| e.to_string())
}
#[cfg(target_arch = "wasm32")]
fn self_test(_: &Path, _: &ModelManifest) -> std::result::Result<Value, String> {
Ok(Value::Null)
}
/// `faces.models.test {id}`: run an installed model's self-test again.
fn test(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.models.test";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
let dir = models_dir(s, C)?;
if !cfg!(not(target_arch = "wasm32")) {
return Err(bad(C, "this build cannot run face recognition models"));
}
let found = installed_models(&dir).into_iter().find(|i| i.manifest.id == id).ok_or_else(|| bad(C, "that model is not installed"))?;
if found.manifest.role != Role::Embedder {
return Err(bad(C, "only recognition models have a self-test"));
}
let result = self_test(&dir.join(id).join("model.onnx"), &found.manifest);
let (ok, detail) = match &result {
Ok(v) => (true, v.clone()),
Err(e) => (false, json!(e)),
};
// keep the latest outcome with the model's record
let mut record = found.accepted;
if let Some(o) = record.as_object_mut() {
o.insert("selfTest".into(), if ok { detail.clone() } else { json!({"ok": false, "error": detail}) });
write_atomic(&dir.join(id).join("installed.json"), &serde_json::to_vec_pretty(&record).map_err(|e| fail("record", e))?)?;
}
Ok(json!({"id": id, "ok": ok, "result": detail}))
}
fn remove(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.models.remove";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
let dir = models_dir(s, C)?;
if !manifest::valid_id(id) {
return Err(bad(C, "not a model id"));
}
let home = dir.join(id);
// only a folder that is a model (it has a manifest) is ever deleted
if !home.join("face-model.json").is_file() {
return Err(bad(C, "that model is not installed"));
}
std::fs::remove_dir_all(&home).map_err(|e| fail("could not remove the model", e))?;
let mut st = read_settings(&dir);
if st.embedder.as_deref() == Some(id) {
// another installed recogniser takes over, so recognition keeps working; with none left nothing is chosen
st.embedder = installed_models(&dir).into_iter().find(|i| i.manifest.id != id && i.manifest.role == Role::Embedder).map(|i| i.manifest.id);
write_settings(&dir, &st)?;
}
Ok(json!({"removed": id, "embedder": st.embedder}))
}
fn select(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.models.select";
let dir = models_dir(s, C)?;
let id = match p.get("id") {
None | Some(Value::Null) => None,
Some(v) => Some(v.as_str().ok_or_else(|| bad(C, "`id` must be a model id or null"))?.to_string()),
};
if let Some(id) = &id
&& !installed_models(&dir).iter().any(|i| &i.manifest.id == id && i.manifest.role == Role::Embedder)
{
return Err(bad(C, "that recognition model is not installed"));
}
let mut st = read_settings(&dir);
st.embedder = id;
write_settings(&dir, &st)?;
Ok(json!({"embedder": st.embedder}))
}
fn enable(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.enable";
let dir = models_dir(s, C)?;
let mut st = read_settings(&dir);
st.enabled = p.get("enabled").and_then(Value::as_bool).unwrap_or(!st.enabled);
write_settings(&dir, &st)?;
// the background scan notices at once, not within the second it trusts its last look
#[cfg(not(target_arch = "wasm32"))]
{
s.faces.enabled_seen = None;
}
Ok(json!({"enabled": st.enabled}))
}
pub fn specs() -> Vec<CommandSpec> {
vec![
cmd!(query "faces.models.list", "Face Models", [], None, "{} → {dir, enabled, embedder, runtime, models: [{id, name, role, licence{name, commercial, url, notice}, provenance, source, sizeBytes, known, installed, selected}]}", always, list),
cmd!(query "faces.models.inspect", "Inspect Face Model File", [], None, "{path} → what a .onnx file is: {kind: known | draft | unsupported, model, assumptions, reason, alreadyInstalled}; installs nothing", always, inspect),
cmd!(query "faces.models.install", "Install Face Model", [], None, "{path, acknowledged: true, activate?: true} → {installed, active: {embedder, enabled}} — copy a .onnx face recognition model into the models folder. `acknowledged` must be true: the user has been shown its licence (see inspect) and accepted it. Unless `activate` is false the model becomes the one in use and recognition is switched on; an earlier model stays installed", always, install),
cmd!(query "faces.models.download", "Download Face Model", [], None, "{id, acknowledged: true} → {started, from} — fetch a recognition model LightCraft has a pinned address for (see `downloadHost` in the list), in the background over pure-Rust https (lightcraft-fetch). `acknowledged` must be true: the user has been shown the model's terms and accepted them. It is checked against its size and SHA-256, then installed, chosen and switched on by itself; `faces.models.downloads` shows how far it is", always, download),
cmd!(query "faces.models.downloads", "Face Model Downloads", [], None, "{} → {running, downloads: [{id, state: running | done | installed | failed | cancelled, bytes, total, error, from}]} — also installs any download that has arrived; `installed` stays listed until `faces.models.downloadCancel` clears it", always, downloads),
cmd!(query "faces.models.downloadCancel", "Cancel Face Model Download", [], None, "{id} → {discarded} — stop a download, or delete a finished one that was not installed", always, download_cancel),
cmd!(query "faces.models.test", "Test Face Model", [], None, "{id} → {ok, result: {loadMs, embedMs, dimension, checks}} — load an installed recognition model and check it gives sensible faces; needs the recognition runtime", always, test),
cmd!(query "faces.models.remove", "Remove Face Model", [], None, "{id} — delete an installed model", always, remove),
cmd!(query "faces.models.select", "Choose Face Recognition Model", [], None, "{id: installed recogniser | null}", always, select),
cmd!(query "faces.enable", "Face Recognition On/Off", [], None, "{enabled?: bool} (toggles when omitted)", always, enable),
]
}
File diff suppressed because it is too large. Load diff
+95 -3
View File
@@ -119,6 +119,14 @@ fn album_param(p: &Value, key: &str, c: &str) -> Result<AlbumId> {
p.get(key).and_then(Value::as_u64).map(AlbumId).ok_or_else(|| bad(c, format!("missing album `{key}`")))
}
/// An optional album id: absent or `null` is `None`, anything but a number is an error.
fn opt_album_param(p: &Value, key: &str, c: &str) -> Result<Option<AlbumId>> {
match p.get(key) {
None | Some(Value::Null) => Ok(None),
Some(v) => v.as_u64().map(|n| Some(AlbumId(n))).ok_or_else(|| bad(c, format!("`{key}` must be an album id or null"))),
}
}
fn strs(p: &Value, key: &str) -> Vec<String> {
p.get(key).and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_str).map(str::to_string).collect()).unwrap_or_default()
}
@@ -653,9 +661,19 @@ pub fn specs() -> Vec<CommandSpec> {
),
// ---- delete / restore
cmd!("photo.delete", "Delete Photo", ["Photo"], Some("Delete"), "{ids?} — moves to Recently Deleted", has_selection, |s, p| {
let before = s.visible_cloned();
let at = before.iter().position(|id| Some(*id) == s.selection.active);
let v = for_targets(s, p, "Delete", |id| Some(Op::SetDeleted { id, deleted: true }))?;
let vis = s.visible_cloned();
s.selection = vis.first().map(|f| Selection::single(*f)).unwrap_or_default();
let remaining: std::collections::HashSet<_> = s.visible_cloned().into_iter().collect();
s.selection.ids.retain(|id| remaining.contains(id));
s.selection.active = s.selection.active.filter(|id| remaining.contains(id)).or_else(|| s.selection.ids.first().copied());
if s.selection.active.is_none()
&& let Some(at) = at
{
// Keep culling near the deleted photo: next visible survivor, or the previous one at the end.
let neighbour = before.iter().skip(at.saturating_add(1)).chain(before.iter().take(at).rev()).find(|id| remaining.contains(id));
s.selection = neighbour.map(|id| Selection::single(*id)).unwrap_or_default();
}
Ok(v)
}),
cmd!("photo.restore", "Restore", ["Photo"], None, "{ids?}", has_selection, |s, p| for_targets(s, p, "Restore", |id| Some(Op::SetDeleted {
@@ -784,7 +802,7 @@ pub fn specs() -> Vec<CommandSpec> {
let cover = photos.first().copied();
s.commit(
if folder { "New Folder" } else { "New Album" },
Op::AddAlbum { album: Album { id, name, parent, folder, photos, cover, smart: None, quick: false } },
Op::AddAlbum { album: Album { id, name, parent, folder, photos, cover, smart: None, quick: false, order: None } },
)?;
Ok(json!({"id": id.0}))
}),
@@ -880,6 +898,80 @@ pub fn specs() -> Vec<CommandSpec> {
s.commit("Move Album", Op::MoveAlbum { id, parent })?;
ok()
}),
cmd!(
"album.reorder",
"Reorder Album",
[],
None,
"{id, parent?: folderId|null, before?: albumId|null} — places the album (or folder) in `parent` (default: where it is; null: the top level) before the sibling `before` of the same kind, or last of its kind; the folder is then ordered by hand. One undo step. → {changed}: how many edits it took (0: already there)",
always,
|s, p| {
let id = album_param(p, "id", "album.reorder")?;
let me = s.catalog.album(id).ok_or_else(|| bad("album.reorder", "no such album"))?;
let parent = if p.get("parent").is_some() { opt_album_param(p, "parent", "album.reorder")? } else { me.parent };
let before = opt_album_param(p, "before", "album.reorder")?;
if before == Some(id) {
// before itself: where it is
return Ok(json!({"changed": 0}));
}
let mut order: Vec<AlbumId> = s.catalog.album_children(parent).into_iter().filter(|a| a.id != id).map(|a| a.id).collect();
let at = match before {
Some(b) => {
let at = order.iter().position(|x| *x == b).ok_or_else(|| bad("album.reorder", "`before` is not inside that folder"))?;
if s.catalog.album(b).is_none_or(|x| x.folder != me.folder) {
return Err(bad("album.reorder", "folders are placed among folders and albums among albums"));
}
at
}
None if me.folder => order.iter().take_while(|x| s.catalog.album(**x).is_some_and(|a| a.folder)).count(),
None => order.len(),
};
order.insert(at.min(order.len()), id);
let moved = me.parent != parent;
let mut ops = Vec::new();
if moved {
ops.push(Op::MoveAlbum { id, parent });
}
for (i, aid) in order.iter().enumerate() {
let want = u32::try_from(i).ok();
// a moved album starts without a place in its new folder
let have = if moved && *aid == id { None } else { s.catalog.album(*aid).and_then(|a| a.order) };
if have != want {
ops.push(Op::SetAlbumOrder { id: *aid, order: want });
}
}
if ops.is_empty() {
return Ok(json!({"changed": 0}));
}
let changed = ops.len();
s.commit("Reorder Album", Op::Batch { ops })?;
Ok(json!({"changed": changed}))
}
),
cmd!(
"album.sort",
"Sort Albums by Name",
[],
None,
"{parent?: folderId|null} — drops the hand order of the albums inside the folder (default: the top level), so they are listed by name → {changed}: how many albums lost their place",
always,
|s, p| {
let parent = opt_album_param(p, "parent", "album.sort")?;
let ops: Vec<Op> = s
.catalog
.album_children(parent)
.into_iter()
.filter(|a| a.order.is_some())
.map(|a| Op::SetAlbumOrder { id: a.id, order: None })
.collect();
if ops.is_empty() {
return Ok(json!({"changed": 0}));
}
let changed = ops.len();
s.commit("Sort Albums by Name", Op::Batch { ops })?;
Ok(json!({"changed": changed}))
}
),
cmd!("album.addPhotos", "Add to Album", ["Photo"], None, "{id: albumId, ids?: [photoIds]} (default: selection)", has_selection, |s, p| {
let id = album_param(p, "id", "album.addPhotos")?;
let targets = ids_param(p).unwrap_or_else(|| s.targets(&Value::Null));
+1
View File
@@ -432,6 +432,7 @@ pub fn specs() -> Vec<CommandSpec> {
"available": crate::segment::Segmenter::AVAILABLE,
"installed": g.installed(),
"dir": g.dir.as_ref().map(|d| d.display().to_string()),
"remote": g.remote_endpoint(),
"loaded": g.loaded(),
"busy": g.busy(),
"analyzing": g.analyzing(),
+8
View File
@@ -10,9 +10,13 @@ mod color;
pub(crate) mod convert;
mod cull;
pub mod curves;
pub(crate) mod denoise;
mod develop;
mod edit;
mod export;
pub(crate) mod face_detect;
mod face_models;
mod face_recognize;
pub mod filters;
pub mod keywords;
pub mod library;
@@ -140,6 +144,10 @@ pub fn command_specs() -> &'static [CommandSpec] {
v.extend(missing::specs());
v.extend(metadata::specs());
v.extend(filters::specs());
v.extend(denoise::specs());
v.extend(face_models::specs());
v.extend(face_detect::specs());
v.extend(face_recognize::specs());
v
})
}
+58
View File
@@ -0,0 +1,58 @@
//! Where LightCraft keeps per-user data on this machine: the config folder (app settings, camera
//! profiles, face and denoise models).
//!
//! Hosts that have a file system (the desktop app, the CLI, the MCP server) share these defaults so a model
//! installed from one is there for the others. Nothing here is applied automatically: a [`Session`] has no
//! face- or denoise-models folder until a host asks for one ([`Session::with_default_face_models`],
//! [`Session::with_default_denoise_models`]), so tests stay hermetic.
use std::path::PathBuf;
use crate::Session;
/// LightCraft's per-user config folder (`…/LightCraft`), if the platform tells us where.
pub fn config_dir() -> Option<PathBuf> {
if cfg!(target_os = "macos") {
std::env::var_os("HOME").map(|h| PathBuf::from(h).join("Library/Application Support/LightCraft"))
} else if cfg!(windows) {
std::env::var_os("APPDATA").map(|a| PathBuf::from(a).join("LightCraft"))
} else {
std::env::var_os("XDG_CONFIG_HOME")
.map(PathBuf::from)
.or_else(|| std::env::var_os("HOME").map(|h| PathBuf::from(h).join(".config")))
.map(|c| c.join("lightcraft"))
}
}
/// Where face models are kept: `$LIGHTCRAFT_FACE_MODELS` if set, else `<config>/models`.
pub fn default_face_models_dir() -> Option<PathBuf> {
std::env::var_os("LIGHTCRAFT_FACE_MODELS").filter(|v| !v.is_empty()).map(PathBuf::from).or_else(|| config_dir().map(|d| d.join("models")))
}
/// Where opt-in denoise models are kept: `$LIGHTCRAFT_DENOISE_MODELS` if set, else `<config>/denoise-models`.
pub fn default_denoise_models_dir() -> Option<PathBuf> {
std::env::var_os("LIGHTCRAFT_DENOISE_MODELS")
.filter(|v| !v.is_empty())
.map(PathBuf::from)
.or_else(|| config_dir().map(|d| d.join("denoise-models")))
}
impl Session {
/// Keep face models in the shared default folder ([`default_face_models_dir`]).
pub fn with_default_face_models(mut self) -> Self {
self.face_models_dir = default_face_models_dir();
self
}
/// Keep denoise models in the shared default folder ([`default_denoise_models_dir`]).
pub fn with_default_denoise_models(mut self) -> Self {
self.set_denoise_models_dir(default_denoise_models_dir());
self
}
/// Point AI denoise at `dir` (or at no folder), dropping what was learned about the old one.
pub fn set_denoise_models_dir(&mut self, dir: Option<PathBuf>) {
self.denoise.models_dir = dir;
self.denoise.touch();
}
}
+97 -10
View File
@@ -44,6 +44,17 @@ const NON_ADJUSTMENT: &[&str] = &[
"crs:AlreadyApplied",
"crs:RawFileName",
"crs:HasCrop",
// preset/library bookkeeping and tool options that don't change the rendering
"crs:Cluster",
"crs:SupportsAmount2",
"crs:RequiresRGBTables",
"crs:SortName",
"crs:Description",
"crs:CropConstrainToWarp",
"crs:OverrideLookVignette",
// the file's as-shot white, recorded next to the edit (ours comes from the file itself)
"crs:AsShotTemperature",
"crs:AsShotTint",
];
/// True if the packet carries any `crs:` adjustment (not just bookkeeping fields).
@@ -233,11 +244,14 @@ pub fn to_partial(props: &Props, raw: Option<bool>) -> Value {
put(o, "curve.master", c);
}
}
for (crs, ch) in
[("ToneCurvePV2012", "master"), ("ToneCurvePV2012Red", "red"), ("ToneCurvePV2012Green", "green"), ("ToneCurvePV2012Blue", "blue")]
{
if let Some(c) = curve(props, &format!("crs:{crs}")) {
put(o, &format!("curve.{ch}"), c);
// Lightroom Classic reads the red / green / blue curves only together with the master 2012
// curve: a channel curve without `ToneCurvePV2012` leaves the photo unchanged
if let Some(c) = curve(props, "crs:ToneCurvePV2012") {
put(o, "curve.master", c);
for (crs, ch) in [("ToneCurvePV2012Red", "red"), ("ToneCurvePV2012Green", "green"), ("ToneCurvePV2012Blue", "blue")] {
if let Some(c) = curve(props, &format!("crs:{crs}")) {
put(o, &format!("curve.{ch}"), c);
}
}
}
@@ -287,9 +301,12 @@ pub fn to_partial(props: &Props, raw: Option<bool>) -> Value {
n(o, "GrainSize", "grain.size");
n(o, "GrainFrequency", "grain.roughness");
// ---- Optics (manual corrections; lens profiles are ours, only the switch carries over)
if let Some(b) = boolean(props, "crs:LensProfileEnable") {
put(o, "optics.lens_profile", json!(b));
// ---- Optics (manual corrections; lens profiles are ours, only the switch carries over).
// Our profile corrections are the file's own (DNG-embedded) ones, which Lightroom applies
// whatever its "Enable Profile Corrections" box says [inferred]: so only an enabling switch
// carries over, and `LensProfileEnable=0` (the default in most presets) leaves ours as it is.
if boolean(props, "crs:LensProfileEnable") == Some(true) {
put(o, "optics.lens_profile", json!(true));
}
if let Some(b) = boolean(props, "crs:AutoLateralCA") {
put(o, "optics.remove_ca", json!(b));
@@ -373,8 +390,19 @@ pub fn to_partial_report(props: &Props, values: Option<&crate::crs_masks::Values
mask_skips = skipped.into_iter().map(|k| format!("Mask: {k}")).collect();
}
}
// fields that only switch a panel on/off or name things: not adjustments by themselves
let quiet = |k: &str| k.starts_with("Enable") || k.starts_with("ToneCurveName") || k == "AutoTone" || k == "AutoGrayscaleMix";
// fields that only switch a panel on/off or name things: not adjustments by themselves; an
// HDR edit mode that is off, and Point Color slots that are all empty (-1)
let value_is = |k: &str, f: &dyn Fn(&[String]) -> bool| props.get(&format!("crs:{k}")).is_some_and(|v| f(v));
let off = |v: &[String]| v.iter().all(|s| s.trim() == "0");
let empty_points = |v: &[String]| v.iter().all(|s| s.split(',').all(|n| n.trim().parse::<f64>().is_ok_and(|x| x == -1.0) || n.trim().is_empty()));
let quiet = |k: &str| {
k.starts_with("Enable")
|| k.starts_with("ToneCurveName")
|| k == "AutoTone"
|| k == "AutoGrayscaleMix"
|| (k == "HDREditMode" && value_is(k, &off))
|| (k == "PointColors" && value_is(k, &empty_points))
};
let mut unmapped: Vec<String> = props
.keys()
.filter(|k| k.starts_with("crs:"))
@@ -539,4 +567,63 @@ mod tests {
assert!(!p.builtin);
assert!(preset_from_xmp("<x/>", "f").is_none());
}
/// A preset packet (written for this test) with the bookkeeping fields Lightroom presets carry.
fn preset_packet(extra_attrs: &str, extra_elems: &str) -> String {
format!(
r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<rdf:Description rdf:about="" xmlns:crs="http://ns.adobe.com/camera-raw-settings/1.0/"
crs:PresetType="Normal" crs:Cluster="" crs:UUID="0123ABCD" crs:SupportsAmount2="True" crs:SupportsAmount="True"
crs:RequiresRGBTables="False" crs:Contrast2012="+25" crs:LensProfileEnable="0" crs:OverrideLookVignette="False"
crs:CropConstrainToWarp="0" crs:AsShotTemperature="5500" crs:AsShotTint="10" {extra_attrs}>
<crs:SortName><rdf:Alt><rdf:li xml:lang="x-default"/></rdf:Alt></crs:SortName>
<crs:Description><rdf:Alt><rdf:li xml:lang="x-default"/></rdf:Alt></crs:Description>
{extra_elems}
</rdf:Description></rdf:RDF></x:xmpmeta>"#
)
}
#[test]
fn channel_curves_need_the_master_curve() {
// Lightroom ignores a red / green / blue curve without `ToneCurvePV2012`
let packet = |master: &str| {
format!(
r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<rdf:Description rdf:about="" xmlns:crs="http://ns.adobe.com/camera-raw-settings/1.0/" crs:ToneCurveName2012="Custom">{master}
<crs:ToneCurvePV2012Green><rdf:Seq><rdf:li>0, 0</rdf:li><rdf:li>64, 40</rdf:li><rdf:li>255, 255</rdf:li></rdf:Seq></crs:ToneCurvePV2012Green>
</rdf:Description></rdf:RDF></x:xmpmeta>"#
)
};
let alone = to_partial(&props(&packet("")), Some(true));
assert!(alone.pointer("/curve/green").is_none(), "{alone}");
let master = "<crs:ToneCurvePV2012><rdf:Seq><rdf:li>0, 0</rdf:li><rdf:li>255, 255</rdf:li></rdf:Seq></crs:ToneCurvePV2012>";
let with = to_partial(&props(&packet(master)), Some(true));
assert!(with.pointer("/curve/green").is_some() && with.pointer("/curve/master").is_some(), "{with}");
}
fn unmapped(x: &str) -> Vec<String> {
to_partial_report(&props(x), None, None, 1.5).1
}
#[test]
fn preset_bookkeeping_is_not_reported_but_real_gaps_are() {
let empty_points = "<crs:PointColors><rdf:Seq><rdf:li>-1.000000, -1.000000, -1.000000, -1.000000</rdf:li></rdf:Seq></crs:PointColors>";
assert_eq!(unmapped(&preset_packet(r#"crs:HDREditMode="0""#, empty_points)), Vec::<String>::new());
// an HDR edit, a used Point Color slot and an unknown adjustment still are
let used_points = "<crs:PointColors><rdf:Seq><rdf:li>0.5, 0.2, 0.1, 10, 0, 0, 0, 0</rdf:li></rdf:Seq></crs:PointColors>";
let got = unmapped(&preset_packet(r#"crs:HDREditMode="1" crs:FutureSlider="12""#, used_points));
assert_eq!(got, ["FutureSlider", "HDREditMode", "PointColors"]);
}
#[test]
fn disabled_lens_profile_switch_keeps_embedded_corrections() {
// Lightroom applies a file's built-in lens corrections regardless of its profile switch,
// and ours are only those: a preset's `LensProfileEnable=0` must not turn them off
let p = props(&preset_packet("", ""));
let mut d = DevelopSettings::default();
d.optics.lens_profile = true;
assert!(apply_partial(&d, &to_partial(&p, None), 1.0).optics.lens_profile);
let on = props(&preset_packet("", "").replace(r#"crs:LensProfileEnable="0""#, r#"crs:LensProfileEnable="1""#));
assert!(apply_partial(&DevelopSettings::default(), &to_partial(&on, None), 1.0).optics.lens_profile);
}
}
+154 -2
View File
@@ -62,7 +62,17 @@ pub fn load(s: &mut Session) {
}
let folder = s.catalog.alloc_album_id();
ops.push(Op::AddAlbum {
album: Album { id: folder, name: "Travel 2026".into(), parent: None, folder: true, photos: vec![], cover: None, smart: None, quick: false },
album: Album {
id: folder,
name: "Travel 2026".into(),
parent: None,
folder: true,
photos: vec![],
cover: None,
smart: None,
quick: false,
order: None,
},
});
let by_kw = |kw: &str| -> Vec<_> { scenes.iter().zip(&ids).filter(|(sc, _)| sc.meta.keywords.contains(&kw)).map(|(_, id)| *id).collect() };
for (name, parent, photos) in [
@@ -75,7 +85,9 @@ pub fn load(s: &mut Session) {
] {
let id = s.catalog.alloc_album_id();
let cover = photos.first().copied();
ops.push(Op::AddAlbum { album: Album { id, name: name.into(), parent, folder: false, photos, cover, smart: None, quick: false } });
ops.push(Op::AddAlbum {
album: Album { id, name: name.into(), parent, folder: false, photos, cover, smart: None, quick: false, order: None },
});
}
for op in ops {
let _ = s.catalog.apply(op);
@@ -84,3 +96,143 @@ pub fn load(s: &mut Session) {
s.selection = crate::Selection::single(*first);
}
}
/// Procedural scenes are deterministic but costly to evaluate: every pixel runs the scene's noise
/// functions, so a 24 MP demo original takes minutes of CPU time on a small machine (a 4-core CI
/// VM). Each scene at each size is therefore generated once per process and shared:
///
/// - concurrent requests wait for one generation (Before and After at 1:1 both load the
/// original; sessions side by side, e.g. parallel tests, open the same photo);
/// - the most recently used results are kept, up to [`scene_cache_limit`] bytes. They are the same
/// allocations the sessions' source caches hold, so they cost extra memory only once no session
/// holds them any more.
///
/// Only demo photos use this; real files are decoded by their loader as before.
pub(crate) fn scene_pixels(scene: &lightcraft_scenes::Scene, max_edge: usize) -> Arc<lightcraft_raster::Rgb32f> {
let key = SceneKey { kind: scene.kind, seed: scene.seed, width: scene.width, height: scene.height, max_edge };
let slot = {
let mut c = scene_cache();
c.tick += 1;
let tick = c.tick;
match c.entries.iter_mut().find(|e| e.key == key) {
Some(e) => {
e.used = tick;
e.slot.clone()
}
None => {
let slot = Arc::new(std::sync::OnceLock::new());
c.entries.push(SceneEntry { key, slot: slot.clone(), used: tick });
slot
}
}
};
// (no lock held while generating: other scenes and sizes are served meanwhile)
let pixels = slot.get_or_init(|| Arc::new(scene.render_fit(max_edge))).clone();
trim_scene_cache(&mut scene_cache(), scene_cache_limit());
pixels
}
/// Bytes of generated scenes kept for reuse: half the memory budget (768 MB by default; enough
/// for one 24 MP original and the previews around it).
fn scene_cache_limit() -> usize {
crate::memory::budget() / 2
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
struct SceneKey {
kind: lightcraft_scenes::Kind,
seed: u32,
width: u32,
height: u32,
max_edge: usize,
}
type SceneSlot = Arc<std::sync::OnceLock<Arc<lightcraft_raster::Rgb32f>>>;
struct SceneEntry {
key: SceneKey,
slot: SceneSlot,
used: u64,
}
#[derive(Default)]
struct SceneCache {
entries: Vec<SceneEntry>,
tick: u64,
}
fn scene_cache() -> std::sync::MutexGuard<'static, SceneCache> {
static CACHE: std::sync::OnceLock<std::sync::Mutex<SceneCache>> = std::sync::OnceLock::new();
CACHE.get_or_init(Default::default).lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn scene_bytes(slot: &SceneSlot) -> usize {
slot.get().map_or(0, |img| img.data.len().saturating_mul(std::mem::size_of::<[f32; 3]>()))
}
/// Drop the least recently used generated scenes until the rest fit in `limit` bytes. Scenes still
/// being generated are kept (their waiters hold the slot anyway).
fn trim_scene_cache(c: &mut SceneCache, limit: usize) {
loop {
let total: usize = c.entries.iter().map(|e| scene_bytes(&e.slot)).fold(0, usize::saturating_add);
if total <= limit {
return;
}
let Some(i) = c.entries.iter().enumerate().filter(|(_, e)| e.slot.get().is_some()).min_by_key(|(_, e)| e.used).map(|(i, _)| i) else {
return;
};
c.entries.swap_remove(i);
}
}
#[cfg(test)]
mod tests {
use super::*;
fn small_scene() -> lightcraft_scenes::Scene {
lightcraft_scenes::demo_library().swap_remove(0)
}
#[test]
fn a_scene_is_generated_once_per_size_and_shared() {
let s = small_scene();
let a = scene_pixels(&s, 48);
let b = scene_pixels(&s, 48);
assert!(Arc::ptr_eq(&a, &b), "the second request reuses the first generation");
assert_eq!(a.width.max(a.height), 48);
assert_eq!(a.data, s.render_fit(48).data, "the same pixels as generating it directly");
let c = scene_pixels(&s, 40);
assert_eq!(c.width.max(c.height), 40, "another size is its own generation");
}
#[test]
fn concurrent_requests_wait_for_one_generation() {
let s = small_scene();
let edge = 37; // a size no other test asks for
let got: Vec<_> = std::thread::scope(|sc| {
let hs: Vec<_> = (0..4).map(|_| sc.spawn(|| scene_pixels(&s, edge))).collect();
hs.into_iter().map(|h| h.join().unwrap()).collect()
});
assert!(got.windows(2).all(|w| Arc::ptr_eq(&w[0], &w[1])), "every request got the same pixels");
}
#[test]
fn the_cache_keeps_the_most_recent_scenes_within_its_limit() {
let key = |max_edge| SceneKey { kind: lightcraft_scenes::Kind::Dunes, seed: 1, width: 3, height: 2, max_edge };
let entry = |max_edge, used, px: Option<usize>| {
let slot: SceneSlot = Arc::new(std::sync::OnceLock::new());
if let Some(n) = px {
let _ = slot.set(Arc::new(lightcraft_raster::Rgb32f::new(n, 1)));
}
SceneEntry { key: key(max_edge), slot, used }
};
let px = std::mem::size_of::<[f32; 3]>();
let mut c = SceneCache { entries: vec![entry(1, 3, Some(10)), entry(2, 1, Some(10)), entry(3, 0, None), entry(4, 2, Some(10))], tick: 3 };
trim_scene_cache(&mut c, 25 * px);
let mut kept: Vec<usize> = c.entries.iter().map(|e| e.key.max_edge).collect();
kept.sort_unstable();
assert_eq!(kept, vec![1, 3, 4], "the least recently used finished scene went; the one in progress stays");
trim_scene_cache(&mut c, 0);
assert_eq!(c.entries.iter().map(|e| e.key.max_edge).collect::<Vec<_>>(), vec![3], "only the scene in progress is left");
}
}
File diff suppressed because it is too large. Load diff
+91 -25
View File
@@ -258,33 +258,41 @@ fn watermark_fonts(craft: &'static [crate::fonts::CraftFont]) -> Vec<ab_glyph::F
.collect()
}
/// The font's vertical form, positioned from its vertical origin within the watermark cell.
/// Shape a whole grapheme in one cell. CJK uses vertical origins and forms; Latin stays upright.
/// Unlike Unicode presentation-form characters, GSUB works with BIZ UD fonts too.
fn vertical_watermark_glyph(
fn watermark_cell_glyphs(
font: &ab_glyph::FontRef<'_>,
data: &harfrust::ShaperData,
ch: char,
text: &str,
vertical: bool,
px: f32,
left: f32,
top: f32,
) -> Option<ab_glyph::Glyph> {
) -> Option<Vec<ab_glyph::Glyph>> {
use ab_glyph::{Font, ScaleFont};
let face = harfrust::FontRef::new(font.font_data()).ok()?;
let mut buf = harfrust::UnicodeBuffer::new();
buf.add(ch, 0);
buf.set_direction(harfrust::Direction::TopToBottom);
buf.push_str(text);
buf.set_direction(if vertical { harfrust::Direction::TopToBottom } else { harfrust::Direction::LeftToRight });
buf.guess_segment_properties();
let shaped = data.shaper(&face).build().shape(buf, harfrust::ShapeOptions::new());
if shaped.glyph_infos().len() != 1 {
if shaped.glyph_infos().is_empty() {
return None;
}
let gid = u16::try_from(shaped.glyph_infos().first()?.glyph_id).ok().filter(|g| *g != 0)?;
let pos = shaped.glyph_positions().first()?;
let sf = font.as_scaled(px);
let (sx, sy) = (sf.h_scale_factor(), sf.v_scale_factor());
let height = -(pos.y_advance as f32) * sy;
let pen = ab_glyph::point(left + px / 2.0 + pos.x_offset as f32 * sx, top + (px - height) / 2.0 - pos.y_offset as f32 * sy);
Some(ab_glyph::GlyphId(gid).with_scale_and_position(px, pen))
let advance_x = shaped.glyph_positions().iter().map(|p| p.x_advance as f32 * sx).sum::<f32>();
let advance_y = shaped.glyph_positions().iter().map(|p| -(p.y_advance as f32) * sy).sum::<f32>();
let (mut x, mut y) = if vertical { (left + px / 2.0, top + (px - advance_y) / 2.0) } else { (left + (px - advance_x) / 2.0, top + sf.ascent()) };
let mut glyphs = Vec::new();
for (info, pos) in shaped.glyph_infos().iter().zip(shaped.glyph_positions()) {
let gid = u16::try_from(info.glyph_id).ok().filter(|g| *g != 0)?;
let pen = ab_glyph::point(x + pos.x_offset as f32 * sx, y - pos.y_offset as f32 * sy);
glyphs.push(ab_glyph::GlyphId(gid).with_scale_and_position(px, pen));
x += pos.x_advance as f32 * sx;
y -= pos.y_advance as f32 * sy;
}
Some(glyphs)
}
/// Draw `wm` onto `img` (straight alpha blending of the encoded values).
@@ -355,8 +363,13 @@ fn watermark_coverage(
let mut prev = None;
let columns = text.split('\n').count();
let mut column = 0usize;
for original in text.chars() {
if original == '\n' {
use unicode_segmentation::UnicodeSegmentation;
// Horizontal watermarks retain scalar layout and kerning; only vertical cells use graphemes.
let cells: Box<dyn Iterator<Item = &str>> =
if wm.vertical { Box::new(text.graphemes(true)) } else { Box::new(text.char_indices().filter_map(|(i, c)| text.get(i..i + c.len_utf8()))) };
for cell in cells {
let Some(original) = cell.chars().next() else { continue };
if cell == "\n" || (wm.vertical && cell == "\r\n") {
if wm.vertical {
column = column.saturating_add(1);
tw = tw.max(y);
@@ -369,17 +382,20 @@ fn watermark_coverage(
prev = None;
continue;
}
if wm.vertical
&& matches!(original as u32, 0x3000..=0x30FF | 0x3400..=0x9FFF | 0xFF01..=0xFF60)
&& let Some(face) = fonts.iter().position(|font| font.glyph_id(original).0 != 0)
&& let (Some(font), Some(Some(data))) = (fonts.get(face), vertical_data.get(face))
{
let cjk = matches!(original as u32, 0x3000..=0x30FF | 0x3400..=0x9FFF | 0xFF01..=0xFF60);
if wm.vertical && (cjk || cell.chars().count() > 1) {
let left = columns.saturating_sub(column.saturating_add(1)) as f32 * px;
if let Some(glyph) = vertical_watermark_glyph(font, data, original, px, left, y)
&& (glyph.id != font.glyph_id(original)
|| !matches!(original, '、' | '。' | 'ー' | '(' | ')' | '「' | '」' | '『' | '』' | '【' | '】'))
{
glyphs.push((face, glyph, None));
let shaped = fonts.iter().zip(&vertical_data).enumerate().find_map(|(face, (font, data))| {
let data = data.as_ref()?;
let cluster = watermark_cell_glyphs(font, data, cell, cjk, px, left, y)?;
let substituted = cluster.len() != 1 || cluster.first().is_some_and(|g| g.id != font.glyph_id(original));
if cjk && !substituted && matches!(original, '、' | '。' | 'ー' | '(' | ')' | '「' | '」' | '『' | '』' | '【' | '】') {
return None;
}
Some((face, cluster))
});
if let Some((face, cluster)) = shaped {
glyphs.extend(cluster.into_iter().map(|g| (face, g, None)));
y += px;
continue;
}
@@ -1706,7 +1722,7 @@ mod tests {
assert_eq!(font.glyph_id('︒').0, 0);
let data = harfrust::ShaperData::new(&harfrust::FontRef::new(entry.bytes).unwrap());
for ch in ['、', '。'] {
let glyph = vertical_watermark_glyph(&font, &data, ch, 100.0, 0.0, 0.0).unwrap();
let glyph = watermark_cell_glyphs(&font, &data, &ch.to_string(), true, 100.0, 0.0, 0.0).unwrap().remove(0);
assert_ne!(glyph.id, font.glyph_id(ch), "the font's vertical alternate for {ch}");
let rect = font.outline_glyph(glyph).unwrap().px_bounds();
assert!(rect.min.x > 50.0 && rect.max.y < 50.0, "{ch} in the upper right: {rect:?}");
@@ -1736,6 +1752,56 @@ mod tests {
}
}
#[test]
fn vertical_watermarks_keep_combining_marks_in_one_cell() {
let coverage = |text: &str, craft| {
let wm = Watermark {
text: text.into(),
vertical: true,
size: 0.1,
anchor: Anchor::Center,
inset: 0.0,
shadow: false,
opacity: 1.0,
..Default::default()
};
let mut pixels = vec![0.0f32; 400 * 300];
watermark_coverage(400, 300, &wm, craft, |x, y, k, _| pixels[y * 400 + x] = k);
assert!(pixels.iter().any(|&k| k > 0.0));
pixels
};
let compare = |a: &str, b: &str, craft| {
let (a_pixels, b_pixels) = (coverage(a, craft), coverage(b, craft));
let difference = a_pixels.iter().zip(&b_pixels).position(|(a, b)| a != b);
assert!(difference.is_none(), "{a:?} vs {b:?}: first differing pixel {difference:?}");
};
for (composed, decomposed) in [("éA", "e\u{301}A"), ("Å\nA", "A\u{30a}\nA"), ("A\nB", "A\r\nB"), ("日A", "日\u{e0100}A")] {
compare(composed, decomposed, &[]);
}
if crate::fonts::CRAFT_FONTS.iter().any(|f| f.family == "BIZ UDMincho") {
for (composed, decomposed) in [("が日", "か\u{3099}日"), ("ぱ\n日", "は\u{309a}\n日")] {
compare(composed, decomposed, crate::fonts::CRAFT_FONTS);
}
} else {
eprintln!("skipped Japanese comparisons: built without BIZ UDMincho from craft-fonts");
}
}
#[test]
fn upright_watermark_cell_retains_multiple_glyphs() {
use ab_glyph::{Font, ScaleFont};
let font = ab_glyph::FontRef::try_from_slice(WATERMARK_FONT).unwrap();
let data = harfrust::ShaperData::new(&harfrust::FontRef::new(WATERMARK_FONT).unwrap());
// There is no precomposed A with a combining long solidus overlay.
let glyphs = watermark_cell_glyphs(&font, &data, "A\u{338}", false, 30.0, 0.0, 0.0).unwrap();
assert_eq!(glyphs.len(), 2, "retain the base and the separately positioned combining glyph");
let baseline = font.as_scaled(30.0).ascent();
for glyph in glyphs {
assert!((glyph.position.y - baseline).abs() < 1.0, "no extra vertical cell for the mark");
assert!(font.outline_glyph(glyph).is_some(), "both glyphs have ink");
}
}
#[test]
fn japanese_watermarks_support_vertical_columns() {
use ab_glyph::Font;
+372
View File
@@ -0,0 +1,372 @@
//! Downloading a face model the user asked for (Settings ▸ Faces ▸ Download).
//!
//! The transfer is [`lightcraft_fetch`] (pure-Rust HTTPS with resume, size and SHA-256 checks), on a background thread,
//! so the UI never waits for the network. Only a model listed in [`lightcraft_faces::known::download`] can be fetched:
//! the address is a pinned constant, never something typed or passed in, and no token is ever sent with it. What arrives
//! matches the model's recorded size and SHA-256 or it is thrown away; it then waits in a staging folder until the model
//! is installed (`faces.models.install`, or by itself once the user has accepted the model's terms in the download).
//!
//! Failure is a message, never a panic: no network, a stalled or truncated transfer, a full disk, a file that is not the
//! one expected, and a panic inside the thread itself all end as [`State::Failed`].
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use std::time::Duration;
use lightcraft_faces::known::Download;
/// The folder inside the models folder where a downloaded file waits to be installed. Its name starts with a dot, so it
/// can never be a model's own folder (ids start with a letter or digit).
pub const STAGING: &str = ".downloads";
/// Where a download stands.
#[derive(Clone, Debug, PartialEq)]
pub enum State {
/// Fetching (`bytes` of `total` so far); the file is checked as the last step.
Running {
bytes: u64,
total: u64,
},
/// Fetched and verified (its SHA-256 is `sha256`), waiting in the staging folder at `path` to be installed.
Done {
path: PathBuf,
sha256: String,
},
/// Installed and in use: nothing more to do but tell the user.
Installed,
Failed(String),
Cancelled,
}
struct Shared {
state: Mutex<State>,
cancel: AtomicBool,
}
impl Shared {
fn new(total: u64) -> Self {
Shared { state: Mutex::new(State::Running { bytes: 0, total }), cancel: AtomicBool::new(false) }
}
fn set(&self, s: State) {
*self.state.lock().unwrap_or_else(PoisonError::into_inner) = s;
}
fn get(&self) -> State {
self.state.lock().unwrap_or_else(PoisonError::into_inner).clone()
}
}
/// The downloads of one session, by model id.
#[derive(Default)]
pub struct Downloads {
jobs: HashMap<String, Arc<Shared>>,
}
impl Downloads {
/// Start fetching `spec` into `<models dir>/.downloads/`. Different models can download side by side; the same
/// one is started once.
pub fn start(&mut self, spec: Download, models_dir: &Path) -> Result<(), String> {
if self.jobs.get(spec.id.as_str()).is_some_and(|j| matches!(j.get(), State::Running { .. })) {
return Err("that model is already downloading".into());
}
let staging = models_dir.join(STAGING);
std::fs::create_dir_all(&staging).map_err(|e| format!("could not create the download folder: {e}"))?;
let shared = Arc::new(Shared::new(spec.size_bytes));
let worker = shared.clone();
let id = spec.id.clone();
std::thread::Builder::new()
.name("face-model-download".into())
.spawn(move || {
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run(&spec, spec.base(), &staging, &worker)));
if outcome.is_err() {
worker.set(State::Failed("the download stopped unexpectedly".into()));
}
})
.map_err(|e| format!("could not start the download: {e}"))?;
self.jobs.insert(id, shared);
Ok(())
}
/// A download that has arrived and been checked, as if the transfer had just ended (tests of installing).
#[cfg(test)]
pub(crate) fn arrived(&mut self, id: &str, path: PathBuf, sha256: &str) {
let j = Shared::new(0);
j.set(State::Done { path, sha256: sha256.to_string() });
self.jobs.insert(id.to_string(), Arc::new(j));
}
/// Every download this session has started and not discarded, by model id.
pub fn snapshot(&self) -> Vec<(String, State)> {
let mut all: Vec<_> = self.jobs.iter().map(|(id, j)| (id.clone(), j.get())).filter(|(_, s)| *s != State::Cancelled).collect();
all.sort_by(|a, b| a.0.cmp(&b.0));
all
}
pub fn running(&self) -> bool {
self.jobs.values().any(|j| matches!(j.get(), State::Running { .. }))
}
/// Stop a running download, or discard a finished one (its staged file is deleted). `true` if there was one.
pub fn discard(&mut self, id: &str) -> bool {
let Some(job) = self.jobs.get(id) else { return false };
job.cancel.store(true, Ordering::Relaxed);
match job.get() {
// the thread notices, deletes the partial file and marks itself cancelled
State::Running { .. } => {}
State::Done { path, .. } => {
let _ = std::fs::remove_file(path);
self.jobs.remove(id);
}
State::Installed | State::Failed(_) | State::Cancelled => {
self.jobs.remove(id);
}
}
true
}
/// The staged file at `path` has been installed (its copy is the model now) or is no longer wanted.
pub fn staged_file_used(&mut self, path: &Path) {
self.jobs.retain(|_, j| !matches!(j.get(), State::Done { path: p, .. } if p == path));
}
/// Downloads that have arrived and been checked, ready to install: (model id, staged file, its SHA-256).
pub fn finished(&self) -> Vec<(String, PathBuf, String)> {
let mut out: Vec<_> = self
.jobs
.iter()
.filter_map(|(id, j)| match j.get() {
State::Done { path, sha256 } => Some((id.clone(), path, sha256)),
_ => None,
})
.collect();
out.sort_by(|a, b| a.0.cmp(&b.0));
out
}
/// Record how installing a finished download went (`Installed` or `Failed`).
pub fn set_outcome(&mut self, id: &str, state: State) {
match self.jobs.get(id) {
Some(j) => j.set(state),
None => {
let j = Shared::new(0);
j.set(state);
self.jobs.insert(id.to_string(), Arc::new(j));
}
}
}
}
impl Drop for Downloads {
/// Quitting while a download runs stops it instead of leaving it behind.
fn drop(&mut self) {
for job in self.jobs.values() {
job.cancel.store(true, Ordering::Relaxed);
}
let started = std::time::Instant::now();
while self.running() && started.elapsed() < Duration::from_secs(1) {
std::thread::sleep(Duration::from_millis(10));
}
}
}
/// What went wrong, in words the user can act on.
#[cfg(not(target_arch = "wasm32"))]
fn explain(e: &lightcraft_fetch::DownloadError) -> String {
use lightcraft_fetch::DownloadError;
match e {
DownloadError::AllMirrorsFailed { errors, .. } => {
let detail: String = errors.join("; ").chars().take(300).collect();
format!("could not download the model ({detail}): check the internet connection, or get the file from its page and add it yourself")
}
other => other.to_string(),
}
}
/// Fetch `spec` from `mirror` (the address without the file name) into `staging`, check it and leave it there.
#[cfg(not(target_arch = "wasm32"))]
fn fetch(spec: &Download, mirror: &str, staging: &Path, shared: &Shared) -> Result<(), lightcraft_fetch::DownloadError> {
let file = lightcraft_fetch::FileSpec { name: &spec.file_name, size: Some(spec.size_bytes), sha256: Some(&spec.sha256), max: spec.size_bytes };
lightcraft_fetch::download(&[file], &[mirror.to_string()], staging, &lightcraft_fetch::Options::default(), &shared.cancel, &mut |p| {
shared.set(State::Running { bytes: p.done.min(spec.size_bytes), total: spec.size_bytes });
})
}
#[cfg(target_arch = "wasm32")]
fn fetch(_: &Download, _: &str, _: &Path, _: &Shared) -> Result<(), String> {
Err("the web build downloads no models".into())
}
/// The whole job: reuse a verified earlier copy, else fetch, check and stage. Always ends in a final [`State`].
fn run(spec: &Download, mirror: &str, staging: &Path, shared: &Shared) {
let target = staging.join(&spec.file_name);
let part = staging.join(format!("{}.part", spec.file_name));
match fetch(spec, mirror, staging, shared) {
Ok(()) => shared.set(State::Done { path: target, sha256: spec.sha256.to_string() }),
#[cfg(not(target_arch = "wasm32"))]
Err(lightcraft_fetch::DownloadError::Cancelled) => {
let _ = std::fs::remove_file(&part);
shared.set(State::Cancelled);
}
#[cfg(not(target_arch = "wasm32"))]
Err(e) => {
let _ = std::fs::remove_file(&part);
shared.set(State::Failed(explain(&e)));
}
#[cfg(target_arch = "wasm32")]
Err(e) => {
let _ = std::fs::remove_file(&part);
shared.set(State::Failed(e));
}
}
}
#[cfg(all(test, not(target_arch = "wasm32")))]
mod tests {
use super::*;
use lightcraft_faces::hash::sha256_hex;
use std::io::{BufRead, BufReader, Write};
use std::net::TcpListener;
fn scratch(name: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("lc-face-dl-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
fn spec_for(bytes: &[u8]) -> Download {
Download {
id: "test-model".into(),
url: "https://example.invalid/m/test-model.onnx".into(),
file_name: "test-model.onnx".into(),
size_bytes: bytes.len() as u64,
sha256: sha256_hex(bytes),
}
}
/// A local server that answers every request with `status` and `body` (the whole file, no ranges).
fn serve(status: &'static str, body: Vec<u8>) -> String {
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let base = format!("http://{}/m", l.local_addr().unwrap());
std::thread::spawn(move || {
for mut s in l.incoming().flatten() {
let mut r = BufReader::new(s.try_clone().unwrap());
loop {
let mut line = String::new();
if r.read_line(&mut line).unwrap_or(0) == 0 || line.trim().is_empty() {
break;
}
}
let _ = s.write_all(format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).as_bytes());
let _ = s.write_all(&body);
}
});
base
}
#[test]
fn fetches_checks_and_stages_a_file() {
let dir = scratch("ok");
let bytes: Vec<u8> = (0..200_000u32).map(|i| (i % 251) as u8).collect();
let (staging, spec) = (dir.join(STAGING), spec_for(&bytes));
std::fs::create_dir_all(&staging).unwrap();
let mirror = serve("200 OK", bytes.clone());
let shared = Shared::new(spec.size_bytes);
run(&spec, &mirror, &staging, &shared);
let State::Done { path, sha256 } = shared.get() else { panic!("{:?}", shared.get()) };
assert_eq!((std::fs::read(&path).unwrap(), sha256), (bytes, spec.sha256.to_string()));
assert!(!staging.join("test-model.onnx.part").exists());
// asked again with the copy still staged: no fetch needed (nothing listens there)
let again = Shared::new(spec.size_bytes);
run(&spec, "http://127.0.0.1:1/m", &staging, &again);
assert!(matches!(again.get(), State::Done { .. }), "{:?}", again.get());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_file_that_does_not_match_is_thrown_away() {
let dir = scratch("bad");
let staging = dir.join(STAGING);
std::fs::create_dir_all(&staging).unwrap();
// the right size and the wrong content
let spec = spec_for(&[7u8; 5000]);
let shared = Shared::new(spec.size_bytes);
run(&spec, &serve("200 OK", vec![8u8; 5000]), &staging, &shared);
assert!(matches!(shared.get(), State::Failed(_)), "{:?}", shared.get());
// nothing is left behind: not the file, not a partial one
assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn an_unreachable_or_refusing_server_is_a_message() {
let dir = scratch("down");
let staging = dir.join(STAGING);
std::fs::create_dir_all(&staging).unwrap();
let spec = spec_for(b"abc");
for mirror in ["http://127.0.0.1:1/m".to_string(), serve("404 Not Found", b"not found".to_vec())] {
let shared = Shared::new(3);
run(&spec, &mirror, &staging, &shared);
let State::Failed(why) = shared.get() else { panic!("{:?}", shared.get()) };
assert!(why.contains("internet connection"), "{why}");
}
assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_cancelled_download_leaves_nothing() {
let dir = scratch("cancel");
let staging = dir.join(STAGING);
std::fs::create_dir_all(&staging).unwrap();
let spec = spec_for(b"abc");
let shared = Shared::new(3);
shared.cancel.store(true, Ordering::Relaxed);
run(&spec, &serve("200 OK", b"abc".to_vec()), &staging, &shared);
assert_eq!(shared.get(), State::Cancelled);
assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn dropping_the_session_stops_a_running_download() {
let shared = Arc::new(Shared::new(10));
// a stand-in for the download thread: notices the cancel flag the way the downloader does
let worker = shared.clone();
let handle = std::thread::spawn(move || {
while !worker.cancel.load(Ordering::Relaxed) {
std::thread::sleep(Duration::from_millis(5));
}
worker.set(State::Cancelled);
});
let mut d = Downloads::default();
d.jobs.insert("m".into(), shared.clone());
assert!(d.running());
let started = std::time::Instant::now();
drop(d);
assert!(started.elapsed() < Duration::from_millis(900), "quitting must not wait for a transfer to finish");
assert_eq!(shared.get(), State::Cancelled);
handle.join().unwrap();
}
#[test]
fn discarding_a_finished_download_deletes_its_file() {
let dir = scratch("discard");
let file = dir.join("staged.onnx");
std::fs::write(&file, b"x").unwrap();
let mut d = Downloads::default();
let shared = Arc::new(Shared::new(1));
shared.set(State::Done { path: file.clone(), sha256: String::new() });
d.jobs.insert("m".into(), shared);
assert_eq!(d.snapshot().len(), 1);
assert!(!d.running());
assert!(d.discard("m"));
assert!(!file.exists());
assert!(d.snapshot().is_empty());
assert!(!d.discard("m"), "nothing left to discard");
let _ = std::fs::remove_dir_all(&dir);
}
}
+455
View File
@@ -0,0 +1,455 @@
//! The face index: one embedding per face region, kept in memory and cached beside the library.
//!
//! Embedding a face costs a model run (tens to hundreds of milliseconds), so each region is embedded once and
//! remembered, keyed by its photo and its box. The cache file (`face-embeddings-<model id>.bin` in the library folder,
//! one per model, so switching back to an earlier model does not start over) belongs to one model: its header names the
//! model and its file hash, and a file made by another model, an older format, or a damaged one is simply ignored and
//! rebuilt. Embeddings are kept out of the catalog on purpose: they are large, rebuildable, and not comparable between
//! models.
//!
//! Each face also keeps the box to cut its picture from (its "view"): the detector's box when the detector found the same
//! face, so every face is shown equally close however loosely or tightly its own region was drawn (a Lightroom box, an
//! older tool's box), else the region's own box.
//!
//! Layout (little-endian): `"LCFE"`, version `2`, tag length `u16`, the tag (`<model id>@<sha256>`), dimension `u32`,
//! then records of `photo u64`, four `i32` (the region's box in ten-thousandths), four `i32` (the view box likewise), and
//! `dim` `f32`.
use std::collections::HashMap;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use lightcraft_geom::Rect;
const MAGIC: &[u8; 4] = b"LCFE";
const VERSION: u8 = 2;
/// Largest cache file we read.
const MAX_FILE: u64 = 1 << 30;
pub(crate) type Key = (u64, [i32; 4]);
/// One face's record: its embedding and its view box (quantized like a key).
type Entry = (Arc<[f32]>, [i32; 4]);
/// A quantized box as a rectangle again.
fn rect_of(k: [i32; 4]) -> Rect {
let f = |v: i32| f64::from(v) / 10_000.0;
Rect { x0: f(k[0]), y0: f(k[1]), x1: f(k[2]), y1: f(k[3]) }
}
/// A region's box as the integer key it is cached under.
pub(crate) fn region_key(r: &Rect) -> [i32; 4] {
let q = |v: f64| if v.is_finite() { (v.clamp(-1e3, 1e3) * 10_000.0).round() as i32 } else { 0 };
[q(r.x0), q(r.y0), q(r.x1), q(r.y1)]
}
#[derive(Default)]
pub(crate) struct Index {
/// `<model id>@<sha256>` of the model the embeddings belong to.
pub tag: String,
pub dim: usize,
entries: HashMap<Key, Entry>,
unsaved: Vec<(Key, Entry)>,
/// Faces that could not be embedded this session (so they are not retried on every call).
skipped: std::collections::HashSet<Key>,
/// The cache file on disk belongs to this model, so new records can be appended to it.
file_matches: bool,
pub path: Option<PathBuf>,
}
impl Index {
pub fn len(&self) -> usize {
self.entries.len()
}
pub fn get(&self, photo: u64, rect: &Rect) -> Option<&Arc<[f32]>> {
self.entries.get(&(photo, region_key(rect))).map(|e| &e.0)
}
/// The box to cut this face's picture from, once the scan has looked at it.
pub fn view(&self, photo: u64, rect: &Rect) -> Option<Rect> {
self.entries.get(&(photo, region_key(rect))).map(|e| rect_of(e.1))
}
#[cfg(test)]
pub fn contains(&self, photo: u64, rect: &Rect) -> bool {
self.entries.contains_key(&(photo, region_key(rect)))
}
/// Whether this face still has to be embedded.
pub fn needs(&self, photo: u64, rect: &Rect) -> bool {
let key = (photo, region_key(rect));
!self.entries.contains_key(&key) && !self.skipped.contains(&key)
}
pub fn skip(&mut self, photo: u64, rect: &Rect) {
self.skipped.insert((photo, region_key(rect)));
}
#[cfg(test)]
pub fn insert(&mut self, photo: u64, rect: &Rect, embedding: Vec<f32>) {
self.insert_with_view(photo, rect, embedding, rect);
}
/// Remember a face's embedding and the box to show it by.
pub fn insert_with_view(&mut self, photo: u64, rect: &Rect, embedding: Vec<f32>, view: &Rect) {
let entry: Entry = (embedding.into(), region_key(view));
let key = (photo, region_key(rect));
self.entries.insert(key, entry.clone());
self.unsaved.push((key, entry));
}
/// Whether there are embeddings not yet written to the cache file.
pub fn has_unsaved(&self) -> bool {
!self.unsaved.is_empty()
}
/// Start over for `tag` (`dim` numbers per face), loading what the cache file at `path` holds for it. What the
/// previous model's index had not written yet is written first, so switching models loses nothing.
pub fn reset(&mut self, tag: &str, dim: usize, path: Option<PathBuf>) {
let _ = self.save();
self.tag = tag.to_string();
self.dim = dim;
self.path = path;
self.entries.clear();
self.unsaved.clear();
self.skipped.clear();
self.file_matches = false;
if let Some(p) = self.path.clone() {
self.load(&p);
}
}
fn load(&mut self, path: &Path) {
let Ok(meta) = std::fs::metadata(path) else { return };
if meta.len() > MAX_FILE {
return;
}
let Ok(bytes) = std::fs::read(path) else { return };
let Some((tag_len, rest)) = bytes
.strip_prefix(MAGIC)
.and_then(|r| r.split_first())
.filter(|(v, _)| **v == VERSION)
.and_then(|(_, r)| r.split_at_checked(2))
.map(|(l, r)| (usize::from(u16::from_le_bytes([l[0], l[1]])), r))
else {
return;
};
let Some((tag, rest)) = rest.split_at_checked(tag_len) else { return };
let Some((dim, mut rest)) = rest.split_at_checked(4).map(|(d, r)| (u32::from_le_bytes([d[0], d[1], d[2], d[3]]) as usize, r)) else { return };
if tag != self.tag.as_bytes() || dim != self.dim || dim == 0 {
return;
}
self.file_matches = true;
let record = 8 + 16 + 16 + dim * 4;
while let Some((rec, tail)) = rest.split_at_checked(record) {
rest = tail;
let photo = u64::from_le_bytes([rec[0], rec[1], rec[2], rec[3], rec[4], rec[5], rec[6], rec[7]]);
let int = |at: usize| i32::from_le_bytes([rec[at], rec[at + 1], rec[at + 2], rec[at + 3]]);
let key = [int(8), int(12), int(16), int(20)];
let view = [int(24), int(28), int(32), int(36)];
let emb: Vec<f32> = rec[40..].as_chunks::<4>().0.iter().map(|c| f32::from_le_bytes(*c)).collect();
if emb.iter().all(|v| v.is_finite()) {
self.entries.insert((photo, key), (emb.into(), view));
}
}
}
fn header(&self) -> Vec<u8> {
let tag = self.tag.as_bytes();
let mut h = Vec::with_capacity(11 + tag.len());
h.extend_from_slice(MAGIC);
h.push(VERSION);
h.extend_from_slice(&u16::try_from(tag.len()).unwrap_or(0).to_le_bytes());
h.extend_from_slice(tag);
h.extend_from_slice(&(self.dim as u32).to_le_bytes());
h
}
fn record(&self, key: &Key, entry: &Entry) -> Vec<u8> {
let mut r = Vec::with_capacity(40 + entry.0.len() * 4);
r.extend_from_slice(&key.0.to_le_bytes());
key.1.iter().chain(entry.1.iter()).for_each(|k| r.extend_from_slice(&k.to_le_bytes()));
entry.0.iter().for_each(|v| r.extend_from_slice(&v.to_le_bytes()));
r
}
/// Write what was added since the last save: appended when the file already belongs to this model, else the
/// whole index is written fresh (atomically). A failure is returned, the records stay pending.
pub fn save(&mut self) -> std::io::Result<()> {
let Some(path) = self.path.clone() else {
self.unsaved.clear();
return Ok(());
};
if self.unsaved.is_empty() {
return Ok(());
}
if self.file_matches {
let mut f = std::fs::OpenOptions::new().append(true).open(&path)?;
let mut buf = Vec::new();
for (k, e) in &self.unsaved {
buf.extend(self.record(k, e));
}
f.write_all(&buf)?;
} else {
let mut buf = self.header();
for (k, e) in &self.entries {
buf.extend(self.record(k, e));
}
let part = path.with_extension("part");
std::fs::write(&part, &buf)?;
std::fs::rename(&part, &path)?;
self.file_matches = true;
}
self.unsaved.clear();
Ok(())
}
}
impl Drop for Index {
fn drop(&mut self) {
let _ = self.save();
}
}
const SCAN_MAGIC: &[u8; 4] = b"LCFS";
const SCAN_VERSION: u8 = 1;
/// The detector a search was made with: a record made by another one is ignored (a better detector may find more).
pub(crate) const SCAN_TAG: &str = "yunet-2023mar";
/// Largest record file we read.
const SCAN_MAX_FILE: u64 = 64 << 20;
/// The photos the detector has already searched for faces, so each is searched once, however many times LightCraft
/// starts. Kept beside the library (`face-scanned.bin`: a header naming the detector, then photo ids, appended to).
#[derive(Default)]
pub(crate) struct Scanned {
set: std::collections::HashSet<u64>,
unsaved: Vec<u64>,
path: Option<PathBuf>,
file_matches: bool,
}
impl Scanned {
pub fn path(&self) -> Option<&Path> {
self.path.as_deref()
}
pub fn contains(&self, photo: u64) -> bool {
self.set.contains(&photo)
}
pub fn len(&self) -> usize {
self.set.len()
}
pub fn insert(&mut self, photo: u64) {
if self.set.insert(photo) {
self.unsaved.push(photo);
}
}
pub fn has_unsaved(&self) -> bool {
!self.unsaved.is_empty()
}
/// Start over with the record at `path` (written first: what was not yet saved is not lost).
pub fn reset(&mut self, path: Option<PathBuf>) {
let _ = self.save();
self.set.clear();
self.unsaved.clear();
self.file_matches = false;
self.path = path;
if let Some(p) = self.path.clone() {
self.load(&p);
}
}
fn header() -> Vec<u8> {
let mut h = Vec::new();
h.extend_from_slice(SCAN_MAGIC);
h.push(SCAN_VERSION);
h.extend_from_slice(&(SCAN_TAG.len() as u16).to_le_bytes());
h.extend_from_slice(SCAN_TAG.as_bytes());
h
}
fn load(&mut self, path: &Path) {
let Ok(meta) = std::fs::metadata(path) else { return };
if meta.len() > SCAN_MAX_FILE {
return;
}
let Ok(bytes) = std::fs::read(path) else { return };
let head = Self::header();
let Some(rest) = bytes.strip_prefix(head.as_slice()) else { return };
self.file_matches = true;
// a half-written last id (a crash while appending) is ignored
self.set.extend(rest.as_chunks::<8>().0.iter().map(|c| u64::from_le_bytes(*c)));
}
/// Write what was added since the last save: appended when the file is this detector's, else the whole record
/// is written fresh (atomically). A failure is returned and the ids stay pending.
pub fn save(&mut self) -> std::io::Result<()> {
let Some(path) = self.path.clone() else {
self.unsaved.clear();
return Ok(());
};
if self.unsaved.is_empty() {
return Ok(());
}
if self.file_matches {
let mut f = std::fs::OpenOptions::new().append(true).open(&path)?;
let buf: Vec<u8> = self.unsaved.iter().flat_map(|id| id.to_le_bytes()).collect();
f.write_all(&buf)?;
} else {
let mut buf = Self::header();
buf.extend(self.set.iter().flat_map(|id| id.to_le_bytes()));
let part = path.with_extension("part");
std::fs::write(&part, &buf)?;
std::fs::rename(&part, &path)?;
self.file_matches = true;
}
self.unsaved.clear();
Ok(())
}
}
impl Drop for Scanned {
fn drop(&mut self) {
let _ = self.save();
}
}
#[cfg(test)]
mod tests {
use super::*;
fn rect(x: f64) -> Rect {
Rect { x0: x, y0: 0.1, x1: x + 0.2, y1: 0.4 }
}
fn temp(name: &str) -> PathBuf {
std::env::temp_dir().join(format!("lc-faceindex-{name}-{}.bin", std::process::id()))
}
#[test]
fn embeddings_survive_a_save_and_load_and_append() {
let path = temp("roundtrip");
let _ = std::fs::remove_file(&path);
let mut a = Index::default();
a.reset("m@abc", 3, Some(path.clone()));
a.insert(7, &rect(0.1), vec![1.0, 0.0, 0.5]);
a.insert(8, &rect(0.3), vec![0.0, 1.0, 0.25]);
a.save().unwrap();
a.insert(9, &rect(0.5), vec![0.5, 0.5, 0.5]);
a.save().unwrap(); // appended
let mut b = Index::default();
b.reset("m@abc", 3, Some(path.clone()));
assert_eq!(b.len(), 3);
assert_eq!(b.get(8, &rect(0.3)).map(|e| e.to_vec()), Some(vec![0.0, 1.0, 0.25]));
assert!(b.contains(9, &rect(0.5)) && !b.contains(9, &rect(0.6)) && !b.contains(1, &rect(0.5)));
let _ = std::fs::remove_file(path);
}
#[test]
fn another_model_a_damaged_file_or_a_torn_tail_are_handled() {
let path = temp("other");
let _ = std::fs::remove_file(&path);
let mut a = Index::default();
a.reset("m@abc", 3, Some(path.clone()));
a.insert(1, &rect(0.1), vec![1.0, 0.0, 0.0]);
a.insert(2, &rect(0.2), vec![0.0, 1.0, 0.0]);
a.save().unwrap();
// another model: nothing is loaded, and saving replaces the file for the new model
let mut other = Index::default();
other.reset("n@def", 4, Some(path.clone()));
assert_eq!(other.len(), 0);
other.insert(5, &rect(0.1), vec![1.0, 0.0, 0.0, 0.0]);
other.save().unwrap();
let mut again = Index::default();
again.reset("n@def", 4, Some(path.clone()));
assert_eq!(again.len(), 1);
let mut old = Index::default();
old.reset("m@abc", 3, Some(path.clone()));
assert_eq!(old.len(), 0, "the old model's records are gone");
// a torn last record loses only that record
let bytes = std::fs::read(&path).unwrap();
std::fs::write(&path, &bytes[..bytes.len() - 5]).unwrap();
let mut torn = Index::default();
torn.reset("n@def", 4, Some(path.clone()));
assert_eq!(torn.len(), 0);
// garbage of every kind is ignored
for junk in [Vec::new(), b"LCFE".to_vec(), b"LCFE\x01\xff\xff".to_vec(), vec![0xff; 200], b"LCFE\x02\x00\x00".to_vec()] {
std::fs::write(&path, &junk).unwrap();
let mut j = Index::default();
j.reset("n@def", 4, Some(path.clone()));
assert_eq!(j.len(), 0);
}
// and a missing file is fine
let _ = std::fs::remove_file(&path);
let mut m = Index::default();
m.reset("n@def", 4, Some(path.clone()));
assert_eq!(m.len(), 0);
}
#[test]
fn non_finite_boxes_and_vectors_do_not_break_the_keys() {
assert_eq!(region_key(&Rect { x0: f64::NAN, y0: f64::INFINITY, x1: 1e300, y1: -1e300 }), [0, 0, 10_000_000, -10_000_000]);
let path = temp("nan");
let _ = std::fs::remove_file(&path);
let mut a = Index::default();
a.reset("m@abc", 2, Some(path.clone()));
a.insert(1, &rect(0.1), vec![f32::NAN, 1.0]);
a.save().unwrap();
let mut b = Index::default();
b.reset("m@abc", 2, Some(path.clone()));
assert_eq!(b.len(), 0, "a non-finite vector is not loaded back");
let _ = std::fs::remove_file(path);
}
}
/// What the recognition commands keep between calls: the loaded model and the embeddings made with it.
#[derive(Default)]
pub(crate) struct FacesState {
/// `<model id>@<sha256>` and the loaded model.
pub embedder: Option<(String, Arc<lightcraft_faces::runtime::Embedder>)>,
pub index: Index,
/// When the models folder was last looked at (the per-frame pump trusts a look for a second).
pub checked: Option<std::time::Instant>,
/// Whether recognition was switched on when last looked up, and when.
pub enabled_seen: Option<(std::time::Instant, bool)>,
pub worker: Option<crate::faces_worker::Worker>,
/// After a failure to load the chosen model, the background pump leaves it alone until then.
pub retry_at: Option<std::time::Instant>,
/// Photos the worker is busy with.
pub in_flight: std::collections::HashSet<lightcraft_catalog::PhotoId>,
/// Photos still to hand to the worker, and the catalog revision this list was made at.
pub queue: Vec<lightcraft_catalog::PhotoId>,
pub queue_stamp: Option<u64>,
/// When the index was last written to its cache file by the background pump.
pub saved_at: Option<std::time::Instant>,
/// Bumped when another library is opened: what a worker was making for the old one is dropped.
pub epoch: u64,
/// Photos already searched for faces.
pub scanned: Scanned,
/// Photos that could not be searched this session (unreadable files): left alone until the next launch.
pub scan_failed: std::collections::HashSet<u64>,
}
impl FacesState {
/// Another library is being opened: what was learned about the old one is saved, and nothing of it is kept (photo
/// ids are only unique within a library). The loaded model stays; the index is reloaded for the new library.
pub fn library_changed(&mut self) {
self.index.reset("", 0, None);
self.scanned.reset(None);
self.scan_failed.clear();
self.embedder = None;
self.checked = None;
self.queue.clear();
self.queue_stamp = None;
self.in_flight.clear();
self.retry_at = None;
self.saved_at = None;
self.epoch = self.epoch.wrapping_add(1);
}
}
+483
View File
@@ -0,0 +1,483 @@
//! The heavy part of scanning faces, run off the session thread, on a few threads at once.
//!
//! For one photo: render it (upright, uncropped, default settings), find its faces with the YuNet detector (when installed)
//! so each region can be aligned by landmarks, cut each region out and run the recognition model. A photo that has
//! no face regions at all and has not been searched yet also gets its detections back, with their embeddings, so the
//! scan decodes each photo once. The session thread only prepares the job (which needs the catalog) and later ingests
//! the result, so the window never waits for a model.
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering};
use std::sync::mpsc::{Receiver, Sender, channel};
use std::sync::{Arc, Condvar, Mutex, PoisonError};
use std::time::Duration;
use lightcraft_catalog::PhotoId;
use lightcraft_faces::align::{Rgb, align_to_template, crop_box};
use lightcraft_faces::runtime::Embedder;
use lightcraft_faces::yunet::{Detector, Face, Options};
use lightcraft_geom::Rect;
use crate::media::{PreviewLoader, RenderJob};
/// Long edge of the picture faces are cut from.
pub(crate) const EDGE: usize = 2048;
/// A box from an XMP sidecar is cut with this much room around it (there are no landmarks to align by).
pub(crate) const BOX_PAD: f32 = 1.5;
/// How much a detected face must overlap a region to lend it its landmarks.
pub(crate) const MATCH_IOU: f32 = 0.3;
/// A detection must be this sure to become a face region (the bar `faces.detect` uses).
pub(crate) const REGION_SCORE: f32 = 0.6;
/// A raw's embedded preview is used for the scan only when its long edge is at least this (a thumbnail is too small to
/// find faces in; the raw itself is decoded then).
const MIN_PREVIEW_EDGE: usize = 1000;
/// Most photos ever worked on at once.
const MAX_WORKERS: usize = 24;
/// What one photo in progress holds: the file, its decoded picture and the copies the detector and recogniser read.
/// Measured: peak memory grew by about 180 MB for each extra worker.
const WORKER_BYTES: usize = 180 << 20;
/// Photos queued behind the ones running, so a worker that finishes never waits for the next frame to be fed.
pub(crate) const PREFETCH: usize = 2;
/// How hard the background scan may work right now, as the app judges from what the user is doing.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Pace {
/// Start nothing new: the user is dragging, typing or scrolling.
Pause = 0,
/// One photo at a time: the user is around, or the window is minimized.
Light = 1,
/// Half the machine: the user is idle.
Normal = 2,
/// Most of the machine: the user is watching the progress.
Full = 3,
}
impl Pace {
fn from_u8(n: u8) -> Pace {
match n {
0 => Pace::Pause,
1 => Pace::Light,
3 => Pace::Full,
_ => Pace::Normal,
}
}
/// `pause`, `light`, `normal` or `full`; anything else (or nothing) is `normal`.
pub fn parse(name: Option<&str>) -> Pace {
match name {
Some("pause") => Pace::Pause,
Some("light") => Pace::Light,
Some("full") => Pace::Full,
_ => Pace::Normal,
}
}
}
/// How many photos are worked on at once at `pace` on a machine with `cores` threads (as the system reports them, which
/// respects container limits and affinity). `cap` is a limit the user set (`LIGHTCRAFT_FACE_THREADS`). Memory is not
/// decided here: decodes wait at the process-wide memory gate ([`crate::memory::work_gate`]), sized from the RAM.
pub(crate) fn workers_for(cores: usize, pace: Pace, cap: Option<usize>) -> usize {
let n = match pace {
Pace::Pause => return 0,
Pace::Light => 1,
Pace::Normal => cores / 2,
Pace::Full => cores * 4 / 5,
}
.clamp(1, MAX_WORKERS);
cap.map_or(n, |c| n.min(c.clamp(1, MAX_WORKERS)))
}
fn cores() -> usize {
std::thread::available_parallelism().map_or(1, |n| n.get())
}
fn cap_from_env() -> Option<usize> {
std::env::var("LIGHTCRAFT_FACE_THREADS").ok().and_then(|v| v.trim().parse::<usize>().ok())
}
/// The most photos the memory budget can hold in progress at once: half of it (the rest is the window's, the caches',
/// the renders'), a worker at a time. Raising the budget (`LIGHTCRAFT_MEMORY_MB`) raises this.
fn memory_workers() -> usize {
(crate::memory::budget() / 2 / WORKER_BYTES).max(1)
}
/// Photos to work on at once at `pace` on this machine: by the processor's threads and the pace, and never more than
/// the memory allows. A limit the user sets (`LIGHTCRAFT_FACE_THREADS`) replaces the memory one.
pub(crate) fn target_workers(pace: Pace) -> usize {
workers_for(cores(), pace, Some(cap_from_env().unwrap_or_else(memory_workers)))
}
/// Threads the worker pool starts with: enough for the fullest pace (the idle ones cost nothing).
pub(crate) fn max_workers() -> usize {
target_workers(Pace::Full)
}
/// Threads for the scan's own parallel work (decoding a picture, developing it) at `pace` on a machine with `cores` threads:
/// two when light, half the machine when normal, four fifths when watched. A photo's parallel parts are most of its cost,
/// so this, more than the number of photos at once, is how much of the processor the scan takes. Measured on a 32-thread
/// desktop (4 photos at once): 8 threads scanned 2.2 photos a second, 16 threads 5.8, 25 threads 6.4.
fn pool_threads_for(cores: usize, pace: Pace, cap: Option<usize>) -> usize {
let n = match pace {
Pace::Pause | Pace::Light => 2,
Pace::Normal => cores / 2,
Pace::Full => cores * 4 / 5,
}
.clamp(2, 32);
cap.map_or(n, |c| n.min(c.max(1)))
}
/// One photo's faces to embed (and, if `detect`, to find first).
pub(crate) struct Prepared {
/// `<model id>@<sha256>` the embeddings are for.
pub tag: String,
/// The session's face epoch when this was prepared: another library has been opened since if it differs.
pub epoch: u64,
pub id: PhotoId,
pub job: RenderJob,
/// A raw's embedded preview, tried before the render job (much faster than decoding the raw).
pub preview: Option<(String, PreviewLoader)>,
pub todo: Vec<Rect>,
/// The photo has no face regions and has not been searched: look for faces too.
pub detect: bool,
/// The YuNet detector, when the user has installed it: faces are aligned by its landmarks, and it finds the faces of
/// photos that have none. Without it a scan only embeds the faces the photos already have, cut by their boxes.
pub detector: Option<Arc<Detector>>,
}
/// The embeddings made for a [`Prepared`]; `None` for a face that could not be embedded.
pub(crate) struct Done {
pub tag: String,
pub epoch: u64,
pub id: PhotoId,
/// Each face's region, its embedding (`None` when it could not be made) and the box to show it by.
pub results: Vec<(Rect, Option<Vec<f32>>, Rect)>,
/// Faces were to be looked for in this photo.
pub detect_wanted: bool,
/// What the detector found (their embeddings are in `results`); `None` when the photo could not be searched.
pub found: Option<Vec<Rect>>,
}
fn overlap(a: &Rect, f: &Face) -> f32 {
let (w, h) = ((a.x1.min(f64::from(f.x1)) - a.x0.max(f64::from(f.x0))).max(0.0), (a.y1.min(f64::from(f.y1)) - a.y0.max(f64::from(f.y0))).max(0.0));
let inter = w * h;
let union = (a.x1 - a.x0) * (a.y1 - a.y0) + f64::from((f.x1 - f.x0) * (f.y1 - f.y0)) - inter;
if union > 0.0 { (inter / union) as f32 } else { 0.0 }
}
/// A detection as a region box: inside the photo and not a sliver; `None` for anything else (it is only ever data).
fn face_rect(f: &Face) -> Option<Rect> {
if ![f.x0, f.y0, f.x1, f.y1].iter().all(|v| v.is_finite()) {
return None;
}
let c = |v: f32| f64::from(v.clamp(0.0, 1.0));
let r = Rect { x0: c(f.x0), y0: c(f.y0), x1: c(f.x1), y1: c(f.y1) };
(r.x1 - r.x0 > 0.005 && r.y1 - r.y0 > 0.005).then_some(r)
}
/// Do the work for one photo. Never panics: whatever goes wrong, its faces come back as `None`.
pub(crate) fn process(p: Prepared, embedder: &Embedder) -> Done {
let Prepared { tag, epoch, id, job, preview, todo, detect, detector } = p;
let fallback = |todo: Vec<Rect>| todo.into_iter().map(|r| (r, None, r)).collect::<Vec<_>>();
let rects = todo.clone();
let run = catch_unwind(AssertUnwindSafe(|| {
let from_preview = preview.and_then(|(path, load)| load(&path, EDGE)).filter(|img| img.width.max(img.height) >= MIN_PREVIEW_EDGE);
let image = match from_preview {
Some(img) => img,
None => match job.run().rendered {
Ok(rendered) => rendered.image,
Err(_) => return (fallback(todo), None),
},
};
let (w, h) = (image.width, image.height);
let rgb: Vec<u8> = image.data.iter().flat_map(|px| [px[0], px[1], px[2]]).collect();
// the detector's landmarks, where it finds the same face: faces are aligned by them, not cut by a box
let detected = detector.as_deref().and_then(|d| d.detect(&rgb, w, h, &Options { score: 0.5, ..Options::default() }).ok());
let found = detected.clone().unwrap_or_default();
let img = Rgb { data: &rgb, width: w, height: h };
let (iw, ih) = embedder.input_size();
let scaled = |f: &Face| f.landmarks.map(|(x, y)| (x * w as f32, y * h as f32));
let mut results: Vec<(Rect, Option<Vec<f32>>, Rect)> = todo
.into_iter()
.map(|rect| {
let best = found.iter().map(|f| (overlap(&rect, f), f)).filter(|(o, _)| *o >= MATCH_IOU).max_by(|a, b| a.0.total_cmp(&b.0));
let crop = match best {
Some((_, f)) => align_to_template(&img, &scaled(f), iw, ih),
None => crop_box(
&img,
rect.x0 as f32 * w as f32,
rect.y0 as f32 * h as f32,
rect.x1 as f32 * w as f32,
rect.y1 as f32 * h as f32,
BOX_PAD,
iw,
ih,
),
};
let v = crop.and_then(|c| embedder.embed(&c).ok());
// shown by the detector's box where it found this face: every face then has the same tightness
let view = best.and_then(|(_, f)| face_rect(f)).unwrap_or(rect);
(rect, v, view)
})
.collect();
// a photo with no face regions: what the detector is sure of becomes its faces, embedded in the same pass
let new_faces = if detect {
detected.map(|faces| {
let mut rects = Vec::new();
for f in faces.iter().filter(|f| f.score >= REGION_SCORE) {
let Some(rect) = face_rect(f) else { continue };
let v = align_to_template(&img, &scaled(f), iw, ih).and_then(|c| embedder.embed(&c).ok());
results.push((rect, v, rect));
rects.push(rect);
}
rects
})
} else {
None
};
(results, new_faces)
}));
let (results, found) = run.unwrap_or_else(|_| (fallback(rects), None));
Done { tag, epoch, id, results, detect_wanted: detect, found }
}
type Job = (Arc<Embedder>, Prepared);
/// How many photos may be worked on at once, changed at any moment by the pace: a thread that has a job waits here until
/// fewer than that are running. A pace of zero holds every job where it is, with nothing decoded.
struct Limit {
allowed: AtomicUsize,
/// The pace the limit was last set for (a [`Pace`] as a number): the pool a photo's parallel work uses.
pace: AtomicU8,
running: Mutex<usize>,
wake: Condvar,
}
impl Limit {
fn new() -> Limit {
Limit { allowed: AtomicUsize::new(0), pace: AtomicU8::new(Pace::Normal as u8), running: Mutex::new(0), wake: Condvar::new() }
}
fn set(&self, n: usize, pace: Pace) {
self.pace.store(pace as u8, Ordering::Relaxed);
self.allowed.store(n, Ordering::Relaxed);
self.wake.notify_all();
}
fn pace(&self) -> Pace {
Pace::from_u8(self.pace.load(Ordering::Relaxed))
}
/// Wait for a place; the limit is looked at again whenever it changes and at least every 100 ms.
fn enter(&self) {
let mut running = self.running.lock().unwrap_or_else(PoisonError::into_inner);
while *running >= self.allowed.load(Ordering::Relaxed) {
running = self.wake.wait_timeout(running, Duration::from_millis(100)).unwrap_or_else(PoisonError::into_inner).0;
}
*running += 1;
}
fn leave(&self) {
let mut running = self.running.lock().unwrap_or_else(PoisonError::into_inner);
*running = running.saturating_sub(1);
drop(running);
self.wake.notify_one();
}
}
/// The threads a photo's parallel work (decoding, developing a picture) runs on at `pace`: one pool for each of three sizes
/// ([`pool_threads_for`]), made when first needed, whose idle threads cost nothing. rayon's global pool serves the
/// interactive work (the loupe, exports) and has no priorities, so scan work queued there would make a slider drag wait
/// behind it; here a light scan can use only two threads however large the machine is.
fn scan_pool(pace: Pace) -> Option<&'static rayon::ThreadPool> {
static LIGHT: std::sync::OnceLock<Option<rayon::ThreadPool>> = std::sync::OnceLock::new();
static NORMAL: std::sync::OnceLock<Option<rayon::ThreadPool>> = std::sync::OnceLock::new();
static FULL: std::sync::OnceLock<Option<rayon::ThreadPool>> = std::sync::OnceLock::new();
let (cell, pace) = match pace {
Pace::Pause | Pace::Light => (&LIGHT, Pace::Light),
Pace::Normal => (&NORMAL, Pace::Normal),
Pace::Full => (&FULL, Pace::Full),
};
cell.get_or_init(|| {
rayon::ThreadPoolBuilder::new()
.num_threads(pool_threads_for(cores(), pace, cap_from_env()))
.thread_name(move |i| format!("lightcraft-faces-{}-{i}", pace as u8))
.build()
.ok()
})
.as_ref()
}
/// Run one photo on the pool for the pace, so its parallel parts stay there.
///
/// It must NOT wait at the memory gate ([`crate::memory::work_gate`], which background decodes do): a job running on a
/// pool thread that waits inside a nested rayon call can pick up another job to run on top of itself, and if that one waits
/// for memory the first holds, neither can finish (seen as a few photos that never come back). The memory a scan may use is
/// instead limited by how many photos run at once ([`memory_workers`]); its decodes count as interactive ones at the gate,
/// which never wait.
fn process_in_background(prepared: Prepared, embedder: &Embedder, pace: Pace) -> Done {
let work = || process(prepared, embedder);
match scan_pool(pace) {
Some(pool) => pool.install(work),
None => work(),
}
}
/// Threads that run [`process`] for the jobs they are given, each on one photo at a time.
pub(crate) struct Worker {
jobs: Sender<Job>,
done: Receiver<Done>,
limit: Arc<Limit>,
}
impl Worker {
pub fn start(threads: usize) -> Option<Worker> {
let (jobs, job_rx) = channel::<Job>();
let (done_tx, done) = channel::<Done>();
let job_rx = Arc::new(Mutex::new(job_rx));
let limit = Arc::new(Limit::new());
let mut started = 0;
for i in 0..threads.clamp(1, MAX_WORKERS) {
let (rx, tx, limit) = (job_rx.clone(), done_tx.clone(), limit.clone());
let spawned = std::thread::Builder::new().name(format!("lightcraft-faces-{i}")).spawn(move || {
loop {
// the lock is held only while waiting for a job, never while working on one; the threads end when
// the session (the only sender) is dropped
let next = rx.lock().unwrap_or_else(PoisonError::into_inner).recv();
let Ok((embedder, prepared)) = next else { break };
limit.enter();
let done = process_in_background(prepared, &embedder, limit.pace());
limit.leave();
if tx.send(done).is_err() {
break;
}
}
});
if spawned.is_ok() {
started += 1;
}
}
(started > 0).then_some(Worker { jobs, done, limit })
}
/// How many photos may be worked on at once from now on (what is running is not interrupted).
pub fn allow(&self, n: usize, pace: Pace) {
self.limit.set(n, pace);
}
pub fn submit(&self, embedder: Arc<Embedder>, prepared: Prepared) -> bool {
self.jobs.send((embedder, prepared)).is_ok()
}
/// Everything finished since the last call.
pub fn finished(&self) -> Vec<Done> {
self.done.try_iter().collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_pace_sets_how_much_of_the_machine_the_scan_may_use() {
use Pace::*;
// a 32-thread desktop: nothing while the user works, one photo around them, half when idle, most when watched
assert_eq!([Pause, Light, Normal, Full].map(|p| workers_for(32, p, None)), [0, 1, 16, 24]);
assert_eq!([Pause, Light, Normal, Full].map(|p| workers_for(8, p, None)), [0, 1, 4, 6]);
assert_eq!([Pause, Light, Normal, Full].map(|p| workers_for(4, p, None)), [0, 1, 2, 3]);
// small machines still get one worker (except while paused)
assert_eq!([Pause, Light, Normal, Full].map(|p| workers_for(1, p, None)), [0, 1, 1, 1]);
assert_eq!(workers_for(0, Full, None), 1);
assert_eq!(workers_for(10_000, Full, None), MAX_WORKERS);
// a limit set by the user caps every pace, and is itself kept sane
assert_eq!([Light, Normal, Full].map(|p| workers_for(32, p, Some(3))), [1, 3, 3]);
assert_eq!((workers_for(32, Full, Some(0)), workers_for(32, Pause, Some(5))), (1, 0));
assert_eq!(workers_for(32, Full, Some(usize::MAX)), 24);
}
#[test]
fn the_limit_holds_threads_to_the_pace_and_can_change_at_any_moment() {
let limit = Arc::new(Limit::new());
let (now, peak) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
let started = Arc::new(AtomicUsize::new(0));
let threads: Vec<_> = (0..6)
.map(|_| {
let (limit, now, peak, started) = (limit.clone(), now.clone(), peak.clone(), started.clone());
std::thread::spawn(move || {
limit.enter();
started.fetch_add(1, Ordering::SeqCst);
let n = now.fetch_add(1, Ordering::SeqCst) + 1;
peak.fetch_max(n, Ordering::SeqCst);
std::thread::sleep(Duration::from_millis(40));
now.fetch_sub(1, Ordering::SeqCst);
limit.leave();
})
})
.collect();
// paused: nothing starts
std::thread::sleep(Duration::from_millis(150));
assert_eq!(started.load(Ordering::SeqCst), 0);
// two at a time, never more
limit.set(2, Pace::Normal);
std::thread::sleep(Duration::from_millis(100));
assert!(started.load(Ordering::SeqCst) >= 2);
// raised, the rest go; every thread finishes
limit.set(6, Pace::Full);
for t in threads {
t.join().unwrap();
}
assert_eq!(started.load(Ordering::SeqCst), 6);
assert!(peak.load(Ordering::SeqCst) >= 2, "it did run in parallel");
assert_eq!(*limit.running.lock().unwrap(), 0, "every place was given back");
}
#[test]
fn memory_limits_the_workers_and_a_users_limit_replaces_it() {
// the default budget (a quarter of the RAM, at most 1.5 GiB) holds a few photos in progress, not dozens
assert!((1..=8).contains(&memory_workers()), "{}", memory_workers());
assert!(target_workers(Pace::Full) >= 1);
assert_eq!(target_workers(Pace::Pause), 0);
}
#[test]
fn pace_names_are_forgiving() {
assert_eq!(["pause", "light", "normal", "full"].map(|n| Pace::parse(Some(n))), [Pace::Pause, Pace::Light, Pace::Normal, Pace::Full]);
assert_eq!(
(Pace::parse(None), Pace::parse(Some("")), Pace::parse(Some("FULL")), Pace::parse(Some("turbo"))),
(Pace::Normal, Pace::Normal, Pace::Normal, Pace::Normal)
);
}
#[test]
fn the_scans_parallel_work_gets_a_pool_the_size_of_the_pace() {
use Pace::*;
assert_eq!([Pause, Light, Normal, Full].map(|p| pool_threads_for(32, p, None)), [2, 2, 16, 25]);
assert_eq!([Light, Normal, Full].map(|p| pool_threads_for(8, p, None)), [2, 4, 6]);
assert_eq!([Light, Normal, Full].map(|p| pool_threads_for(2, p, None)), [2, 2, 2]);
assert_eq!(pool_threads_for(10_000, Full, None), 32);
assert_eq!([Light, Normal, Full].map(|p| pool_threads_for(32, p, Some(3))), [2, 3, 3]);
// the pools exist and are the sizes asked for
for pace in [Light, Normal, Full] {
let pool = scan_pool(pace).expect("a pool can be built");
assert_eq!(pool.install(rayon::current_num_threads), pool_threads_for(cores(), pace, cap_from_env()));
}
// a paused scan has the light pool; the pace is remembered as a number and read back
assert_eq!(scan_pool(Pause).map(|p| p as *const _), scan_pool(Light).map(|p| p as *const _));
assert_eq!([0, 1, 2, 3, 9].map(Pace::from_u8), [Pause, Light, Normal, Full, Normal]);
}
#[test]
fn detections_become_boxes_only_when_they_make_sense() {
let f = |x0, y0, x1, y1| Face { x0, y0, x1, y1, score: 0.9, landmarks: [(0.0, 0.0); 5] };
assert!(face_rect(&f(0.2, 0.2, 0.4, 0.5)).is_some());
// a box that spills over the edge is brought back inside
let r = face_rect(&f(-0.1, 0.1, 0.3, 1.2)).unwrap();
assert_eq!((r.x0, r.y1), (0.0, 1.0));
// slivers, inverted boxes and non-numbers are dropped, never turned into regions
for bad in [f(0.2, 0.2, 0.2, 0.5), f(0.4, 0.2, 0.2, 0.5), f(f32::NAN, 0.2, 0.4, 0.5), f(0.2, 0.2, f32::INFINITY, 0.5), f(1.5, 0.2, 2.0, 0.5)]
{
assert!(face_rect(&bad).is_none(), "{bad:?}");
}
}
}
+388 -42
View File
@@ -11,7 +11,7 @@ use lightcraft_pipeline::SourceInfo;
use lightcraft_raster::Rgb32f;
use lightcraft_raster::resample::{Filter, fit};
use crate::media::{FileLoader, FileProbe, PreviewLoader, ProbeInfo};
use crate::media::{DenoiseSpec, FileLoader, FileProbe, PairLoader, PreviewLoader, ProbeInfo};
fn meta_of(m: &lightcraft_meta::Metadata) -> (Meta, Option<String>) {
let shutter = m.exposure_time.map(|t| if t >= 1.0 { format!("{t:.0}") } else { format!("1/{:.0}", 1.0 / t) }).unwrap_or_default();
@@ -220,22 +220,56 @@ pub fn bin_factor(raw: &lightcraft_raw::RawImage, max_edge: usize) -> Option<usi
/// instead of each one waking the pool from outside and waiting for it (which costs more than the
/// loops themselves when the machine is busy).
pub fn load_bytes(bytes: &[u8], max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> {
rayon::scope(|_| load_bytes_now(std::borrow::Cow::Borrowed(bytes), max_edge))
rayon::scope(|_| load_bytes_now(std::borrow::Cow::Borrowed(bytes), max_edge, None).map(|(img, _, info)| (img, info)))
}
/// [`load_bytes`] taking the file's bytes: a raw file's bytes are freed as soon as it is decoded
/// (less memory held while it is developed).
pub fn load_vec(bytes: Vec<u8>, max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> {
rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge))
rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge, None).map(|(img, _, info)| (img, info)))
}
fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> {
/// [`load_vec`] that also develops the photo's cached denoised picture (`spec`) from the same decode: the plain
/// picture, the denoised one (`None` when it cannot be read or does not fit this raw: the plain one is then all
/// there is) and the source facts. Both pictures are the same size.
pub fn load_vec_pair(bytes: Vec<u8>, max_edge: usize, spec: &DenoiseSpec) -> Result<(Rgb32f, Option<Rgb32f>, SourceInfo), String> {
rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge, Some(spec)))
}
/// The denoised camera RGB of `raw` from its cached product, as the plain development would have it: the default
/// crop, and `factor` × `factor` blocks averaged into one pixel when the plain one was binned. `None` when the product
/// is missing, damaged, made from something else or not the size of this raw's active area.
fn denoised_camera_rgb(raw: &lightcraft_raw::RawImage, spec: &DenoiseSpec, factor: usize) -> Option<Rgb32f> {
use lightcraft_denoise::product::{self, Window};
let a = raw.active_area;
let header = product::read_header_at(&spec.product).ok()?;
if header.key != spec.key || (header.width, header.height) != (a.width, a.height) {
return None;
}
if raw.opcodes.list3.is_empty() {
let c = raw.develop_crop(a.width, a.height);
let window = Window { x: c.x, y: c.y, width: c.width, height: c.height };
product::read_window(&spec.product, &spec.key, Some(window), factor, Some(HIGHLIGHT_CLIP)).ok()
} else {
// (the plain development is not binned either when the file has opcodes of this kind: `factor` is 1)
let whole = product::read(&spec.product, &spec.key, 1).ok()?;
Some(raw.finish_demosaiced(whole))
}
}
fn load_bytes_now(
bytes: std::borrow::Cow<'_, [u8]>,
max_edge: usize,
denoise: Option<&DenoiseSpec>,
) -> Result<(Rgb32f, Option<Rgb32f>, SourceInfo), String> {
if lightcraft_raw::probe(&bytes).is_some() {
let mut raw = match lightcraft_raw::decode(&bytes).map_err(|e| preview_reason(&bytes, e)) {
Ok(r) => r,
Err(Ok(why)) => {
// show the camera's embedded JPEG (rendered, not raw) until the variant is supported
return load_embedded_preview(&bytes, max_edge).ok_or(format!("unsupported raw ({why}) without an embedded preview"));
return load_embedded_preview(&bytes, max_edge)
.map(|(img, info)| (img, None, info))
.ok_or(format!("unsupported raw ({why}) without an embedded preview"));
}
Err(Err(e)) => return Err(e),
};
@@ -245,53 +279,63 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<
raw.opcodes.list3.retain(|op| !op.is_lens_correction());
let xy = lightcraft_raw::color::as_shot_white_xy(&raw);
let t = lightcraft_raw::color::camera_transform(&raw, xy);
let camera_look = crate::camera_preview::fit_preview(&raw, &bytes, &t);
// the source's segmentation mattes (DNG semantic masks), read while the preview is fitted
let (camera_look, mattes) = rayon::join(|| crate::camera_preview::fit_preview(&raw, &bytes, &t), || dng_mattes(&bytes, &raw));
drop(bytes);
// Previews and thumbnails bin the mosaic straight to (about) the size they need; only
// larger levels (exports, 1:1) demosaic the whole sensor.
let t0 = web_time::Instant::now();
let binned = match bin_factor(&raw, max_edge) {
let bin = bin_factor(&raw, max_edge);
let binned = match bin {
Some(k) => raw.develop_binned(k, HIGHLIGHT_CLIP).map_err(|e| e.to_string())?,
None => None,
};
let mut img = match binned {
let factor = if binned.is_some() { bin.unwrap_or(1) } else { 1 };
let img = match binned {
Some(img) => img,
None => {
let method = if max_edge <= 600 { lightcraft_raw::Method::Bilinear } else { lightcraft_raw::Method::Ahd };
raw.develop(method).map_err(|e| e.to_string())?
}
};
// the same development from the denoised mosaic, when the photo has been denoised
let twin = denoise.and_then(|spec| denoised_camera_rgb(&raw, spec, factor)).filter(|d| d.width == img.width && d.height == img.height);
// the samples aren't needed any more (the colour model below reads only the tags)
raw.data = lightcraft_raw::RawData::U16(Vec::new());
let mut stages = vec![("develop", t0.elapsed())];
stages.push(("transform", t0.elapsed()));
lightcraft_raw::highlight::reconstruct(&mut img, t.wb, HIGHLIGHT_CLIP);
stages.push(("highlights", t0.elapsed()));
let wb = t.wb;
let m = camera_look.as_ref().map(|p| p.matrix.mul(&t.matrix)).unwrap_or(t.matrix).to_f32();
let hue_sat = camera_look.as_ref().and_then(|p| p.hue_sat.as_ref()).and_then(crate::camera_preview::HueSat::new);
let gain = 2f32.powf(t.baseline_exposure as f32);
let wb = t.wb;
// A DNG's own profile look (hue/saturation map, look table), DNG spec chapter 6.
let tables = lightcraft_raw::profile::ProfileTables::new(&raw.color.profile, lightcraft_raw::color::illuminant_weight(&raw.color, xy));
img.map_in_place(|p| {
let c = [p[0] * wb[0], p[1] * wb[1], p[2] * wb[2]];
let rgb = [
m[0][0] * c[0] + m[0][1] * c[1] + m[0][2] * c[2],
m[1][0] * c[0] + m[1][1] * c[1] + m[1][2] * c[2],
m[2][0] * c[0] + m[2][1] * c[1] + m[2][2] * c[2],
];
let rgb = match &tables {
Some(tables) => tables.apply(rgb, gain),
None => rgb.map(|v| v * gain),
};
// after the baseline exposure, as when it was fitted
hue_sat.as_ref().map_or(rgb, |h| h.apply(rgb)).map(|v| v.max(0.0))
});
stages.push(("colour", t0.elapsed()));
let img = fit(&img, max_edge, max_edge, Filter::Box);
stages.push(("fit", t0.elapsed()));
let img = img.into_oriented(raw.orientation);
stages.push(("orient", t0.elapsed()));
// camera RGB → what the pipeline takes: clipped highlights rebuilt, white balance and the colour model applied,
// fitted to `max_edge` and upright (the same for the plain and the denoised picture)
let finish = |mut img: Rgb32f| {
let mut stages = vec![("develop", t0.elapsed())];
stages.push(("transform", t0.elapsed()));
lightcraft_raw::highlight::reconstruct(&mut img, wb, HIGHLIGHT_CLIP);
stages.push(("highlights", t0.elapsed()));
img.map_in_place(|p| {
let c = [p[0] * wb[0], p[1] * wb[1], p[2] * wb[2]];
let rgb = [
m[0][0] * c[0] + m[0][1] * c[1] + m[0][2] * c[2],
m[1][0] * c[0] + m[1][1] * c[1] + m[1][2] * c[2],
m[2][0] * c[0] + m[2][1] * c[1] + m[2][2] * c[2],
];
let rgb = match &tables {
Some(tables) => tables.apply(rgb, gain),
None => rgb.map(|v| v * gain),
};
hue_sat.as_ref().map_or(rgb, |h| h.apply(rgb)).map(|v| v.max(0.0))
});
stages.push(("colour", t0.elapsed()));
let img = fit(&img, max_edge, max_edge, Filter::Box);
stages.push(("fit", t0.elapsed()));
let img = img.into_oriented(raw.orientation);
stages.push(("orient", t0.elapsed()));
(img, stages)
};
let (img, stages) = finish(img);
if lightcraft_pipeline::profiling() {
let mut prev = std::time::Duration::ZERO;
let parts: Vec<String> = stages
@@ -304,17 +348,87 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<
.collect();
eprintln!("[profile] raw source {}×{} (max {max_edge}, ms after decode): {}", img.width, img.height, parts.join(", "));
}
let twin = twin.map(|picture| finish(picture).0);
let (temp, tint) = xy_to_temp_tint(xy);
let relative = crate::camera_preview::file_local_look(raw.format) && t.matrix_is_fallback;
// White balance re-evaluates the file's own colour model (when it has one and no
// file-local look matrix sits on top of it)
let camera_color = (!t.matrix_is_fallback && camera_look.is_none()).then(|| {
let tags = lightcraft_raw::ColorData { profile: Default::default(), ..raw.color.clone() };
Arc::new(lightcraft_pipeline::CameraColor { tags, developed_for: xy })
});
let camera_tone = camera_look.as_ref().map(|p| p.tone).or_else(|| raw.color.profile.tone_curve.as_ref().and_then(dng_tone_curve));
let local_tone = local_tone(&raw);
let (temp, tint) = if relative { (6500.0, 0.0) } else { (temp.round(), tint.round()) };
return Ok((img, SourceInfo { raw: true, as_shot_temp: temp, as_shot_tint: tint, lens, relative_wb: relative, camera_tone }));
return Ok((
img,
twin,
SourceInfo {
raw: true,
as_shot_temp: temp,
as_shot_tint: tint,
lens,
relative_wb: relative,
camera_color,
camera_tone,
mattes,
local_tone,
},
));
}
let d = lightcraft_codecs::decode(&bytes, lightcraft_codecs::DecodeOptions::fit(max_edge as u32, max_edge as u32)).map_err(|e| e.to_string())?;
let d = lightcraft_codecs::decode(&bytes, fit_box(max_edge)).map_err(|e| e.to_string())?;
drop(bytes);
let img = d.to_working();
let img = if img.width.max(img.height) > max_edge { fit(&img, max_edge, max_edge, Filter::Mitchell) } else { img };
Ok((img.into_oriented(Orientation::from_exif(d.orientation)), SourceInfo::default()))
Ok((img.into_oriented(Orientation::from_exif(d.orientation)), None, SourceInfo::default()))
}
/// The raw's gain table map with where its developed picture (the default crop, oriented) sits in
/// the active area: rendered when the photo's "Camera local tone mapping" option is on.
fn local_tone(raw: &lightcraft_raw::RawImage) -> Option<Arc<lightcraft_pipeline::local_tone::LocalTone>> {
let map = raw.color.profile.gain_table_map.clone()?;
let a = raw.active_area;
let (aw, ah) = (a.width.max(1) as f64, a.height.max(1) as f64);
let c = raw.develop_crop(a.width, a.height);
let rect = [c.x as f64 / aw, c.y as f64 / ah, c.width as f64 / aw, c.height as f64 / ah];
let placement = lightcraft_raw::gaintable::SourcePlacement { rect, orientation: raw.orientation };
Some(Arc::new(lightcraft_pipeline::local_tone::LocalTone { map, placement }))
}
/// The semantic masks of a DNG that AI masks understand, over the developed image (default crop,
/// oriented like it). Person mattes without a confidently selected pixel are left out (the
/// Subject mask then falls back to its heuristic); a sky matte counts even when empty (no sky).
fn dng_mattes(bytes: &[u8], raw: &lightcraft_raw::RawImage) -> Option<Arc<lightcraft_pipeline::masks::Mattes>> {
use lightcraft_pipeline::masks::{MatteKind, Mattes};
if raw.format != lightcraft_raw::RawFormat::Dng {
return None;
}
let mut mattes = Mattes::default();
for m in lightcraft_raw::semantic_masks(bytes) {
let Some(kind) = matte_kind(&m.name) else { continue };
let Some(img) = m.developed(raw.active_area, raw.crop) else { continue };
if kind != MatteKind::Sky && !img.data.iter().any(|&v| v >= 128) {
continue;
}
mattes.push(kind, img.into_oriented(raw.orientation));
}
(!mattes.is_empty()).then(|| Arc::new(mattes))
}
/// What a semantic mask selects, by its `SemanticName`: Apple's (iPhone ProRAW) are named after
/// their auxiliary image types, `urn:com:apple:photo:<year>:aux:<type>`. Other names: `None`.
fn matte_kind(name: &str) -> Option<lightcraft_pipeline::masks::MatteKind> {
use lightcraft_pipeline::masks::MatteKind::*;
let kind = name.strip_prefix("urn:com:apple:photo:")?.rsplit(':').next()?;
Some(match kind {
"semanticskymatte" => Sky,
"semanticskinmatte" => Skin,
"semantichairmatte" => Hair,
"semanticteethmatte" => Teeth,
"semanticglassesmatte" => Glasses,
"portraiteffectsmatte" => Person,
_ => return None,
})
}
/// A DNG `ProfileToneCurve` (linear in, linear out, 1.0 = white after exposure compensation) as the
@@ -378,7 +492,7 @@ fn embedded_preview_size(bytes: &[u8]) -> Option<(u32, u32)> {
/// The embedded preview of a raw file as a working-space image no larger than `max_edge`, oriented.
pub fn load_embedded_preview(bytes: &[u8], max_edge: usize) -> Option<(Rgb32f, SourceInfo)> {
let d = decode_raw_preview(bytes, lightcraft_codecs::DecodeOptions::fit(max_edge as u32, max_edge as u32))?;
let d = decode_raw_preview(bytes, fit_box(max_edge))?;
let img = d.to_working();
let img = if img.width.max(img.height) > max_edge { fit(&img, max_edge, max_edge, Filter::Mitchell) } else { img };
Some((img.oriented(preview_orientation(bytes, d.orientation)), SourceInfo::default()))
@@ -387,7 +501,7 @@ pub fn load_embedded_preview(bytes: &[u8], max_edge: usize) -> Option<(Rgb32f, S
/// The embedded preview of a raw file for display (sRGB, oriented, no larger than `max_edge`): the
/// loupe and grid show it until the raw itself has been developed ([`crate::media::QuickJob`]).
pub fn embedded_preview_srgb(bytes: &[u8], max_edge: usize) -> Option<lightcraft_raster::Rgba8> {
let mut d = decode_raw_preview(bytes, lightcraft_codecs::DecodeOptions::fit(max_edge as u32, max_edge as u32))?;
let mut d = decode_raw_preview(bytes, fit_box(max_edge))?;
if d.image.width.max(d.image.height) > max_edge {
d.image = fit(&d.image, max_edge, max_edge, Filter::Box);
d.alpha = None;
@@ -396,9 +510,39 @@ pub fn embedded_preview_srgb(bytes: &[u8], max_edge: usize) -> Option<lightcraft
Some(d.to_srgb8().oriented(o))
}
/// Decode options fitting into a `max_edge` square; `usize::MAX` (a full-size load) or any edge past
/// `u32::MAX` asks for the largest box rather than wrapping to a small one.
fn fit_box(max_edge: usize) -> lightcraft_codecs::DecodeOptions {
let e = u32::try_from(max_edge).unwrap_or(u32::MAX);
lightcraft_codecs::DecodeOptions::fit(e, e)
}
/// Filesystem-backed embedded-preview hook (native).
///
/// Reads the whole file to find the preview, so it holds the memory gate for twice the file's size (the file and the
/// decoded preview): background work (the face scan) waits for room, interactive work is counted and never waits.
pub fn fs_preview_loader() -> PreviewLoader {
Arc::new(|path: &str, max_edge: usize| embedded_preview_srgb(&std::fs::read(path).ok()?, max_edge))
Arc::new(|path: &str, max_edge: usize| {
let len = std::fs::metadata(path).map(|m| usize::try_from(m.len()).unwrap_or(usize::MAX)).unwrap_or(0);
let gate = crate::memory::work_gate();
let _permit = if crate::memory::is_background() { gate.acquire(len.saturating_mul(2)) } else { gate.acquire_urgent(len.saturating_mul(2)) };
embedded_preview_srgb(&std::fs::read(path).ok()?, max_edge)
})
}
/// The filesystem-backed [`PairLoader`]: the file is read and decoded once, and developed as usual and from the photo's
/// denoised picture. It holds two pictures at once, so it counts for more at the memory gate than a plain load.
pub fn fs_pair_loader() -> PairLoader {
Arc::new(|path: &str, max_edge: usize, spec: &DenoiseSpec| {
let len = std::fs::metadata(path).map(|m| m.len() as usize).unwrap_or(0);
let weight = len * if max_edge <= crate::media::SourceLevel::Thumb.max_edge() { 4 } else { 10 };
let gate = crate::memory::work_gate();
let _permit = if crate::memory::is_background() { gate.acquire(weight) } else { gate.acquire_urgent(weight) };
let bytes = std::fs::read(path).map_err(|e| format!("{path}: {e}"))?;
let r = load_vec_pair(bytes, max_edge, spec);
crate::memory::release();
r
})
}
/// Filesystem-backed hooks (native). On the web the host installs bytes-based hooks instead.
@@ -410,7 +554,7 @@ pub fn fs_preview_loader() -> PreviewLoader {
/// yields to them.
pub fn fs_hooks() -> (FileLoader, FileProbe) {
let loader: FileLoader = Arc::new(|path: &str, max_edge: usize| {
let len = std::fs::metadata(path).map(|m| m.len() as usize).unwrap_or(0);
let len = std::fs::metadata(path).map(|m| usize::try_from(m.len()).unwrap_or(usize::MAX)).unwrap_or(0);
let weight = len * if max_edge <= crate::media::SourceLevel::Thumb.max_edge() { 3 } else { 6 };
let gate = crate::memory::work_gate();
let _permit = if crate::memory::is_background() { gate.acquire(weight) } else { gate.acquire_urgent(weight) };
@@ -421,7 +565,7 @@ pub fn fs_hooks() -> (FileLoader, FileProbe) {
r
});
let probe: FileProbe = Arc::new(|path: &str| {
let len = std::fs::metadata(path).map(|m| m.len() as usize).unwrap_or(0);
let len = std::fs::metadata(path).map(|m| usize::try_from(m.len()).unwrap_or(usize::MAX)).unwrap_or(0);
let _permit = crate::memory::work_gate().acquire(len);
let bytes = std::fs::read(path).map_err(|e| format!("{path}: {e}"))?;
probe_bytes(path, &bytes)
@@ -436,6 +580,7 @@ impl crate::Session {
self.media.file_loader = Some(l);
self.media.file_probe = Some(p);
self.media.preview_loader = Some(fs_preview_loader());
self.media.denoise.loader = Some(fs_pair_loader());
self
}
}
@@ -476,6 +621,37 @@ mod tests {
assert!(probe_bytes("x.jpg", b"not an image").is_err());
}
#[test]
fn cached_denoise_uses_the_same_white_balance_profile_and_orientation_as_plain_raw() {
use lightcraft_denoise::product;
use lightcraft_raw::{DngWriteOptions, Method, Orientation, profile::HsvTable, write_dng};
let dir = crate::tests_xmp::temp_dir("denoise-color-finish");
let fixture = crate::tests_xmp::synthetic_dng_with(None, Default::default());
let mut raw = lightcraft_raw::decode(&fixture).unwrap();
raw.orientation = Orientation::Rotate90;
raw.color.baseline_exposure = 0.5;
raw.color.profile.look_table =
Some(HsvTable { hue_divisions: 1, sat_divisions: 2, val_divisions: 1, data: vec![[23.0, 0.65, 0.8]; 2], srgb_value: false });
let bytes = write_dng(&raw, &DngWriteOptions::default()).unwrap();
let decoded = lightcraft_raw::decode(&bytes).unwrap();
assert!(decoded.color.profile.look_table.is_some());
// A cache containing the ordinary demosaic must undergo the identical downstream finish.
let camera_rgb = decoded.develop(Method::Ahd).unwrap();
let key = "synthetic-color-finish";
let path = dir.join("same.lcdn");
product::write(&path, &camera_rgb, key).unwrap();
let spec = DenoiseSpec { product: path, key: key.into(), make: None };
let (ordinary, _) = load_bytes(&bytes, 2048).unwrap();
let (plain, cached, _) = load_vec_pair(bytes, 2048, &spec).unwrap();
assert_eq!(plain.data, ordinary.data, "requesting a twin must not change the ordinary path");
let cached = cached.unwrap();
assert_eq!((plain.width, plain.height), (24, 32));
assert_eq!((cached.width, cached.height), (plain.width, plain.height));
let worst = plain.data.iter().zip(&cached.data).flat_map(|(a, b)| a.iter().zip(b).map(|(a, b)| (a - b).abs())).fold(0.0f32, f32::max);
assert!(worst < 0.002, "color finish differs beyond cache half-float rounding: {worst}");
let _ = std::fs::remove_dir_all(dir);
}
/// Luminance (from linear RGB) minus its 9 × 9 local mean: tone differences between a render and a camera JPEG
/// mostly cancel, edges remain.
fn detail(w: usize, h: usize, lum: impl Fn(usize, usize) -> f32) -> Vec<f32> {
@@ -669,8 +845,12 @@ mod tests {
assert!(info.camera_tone.is_none());
// a map that removes all saturation, and a tone curve
let grey = HsvTable { hue_divisions: 4, sat_divisions: 2, val_divisions: 1, data: vec![[0.0, 0.0, 1.0]; 8], srgb_value: false };
raw.color.profile =
ProfileLook { hue_sat_map: [Some(grey), None], look_table: None, tone_curve: ToneCurve::from_tag(&[0.0, 0.0, 0.18, 0.3, 1.0, 1.0]) };
raw.color.profile = ProfileLook {
hue_sat_map: [Some(grey), None],
look_table: None,
tone_curve: ToneCurve::from_tag(&[0.0, 0.0, 0.18, 0.3, 1.0, 1.0]),
gain_table_map: None,
};
let with = lightcraft_raw::write_dng(&raw, &Default::default()).unwrap();
let (after, info) = load_bytes(&with, 64).unwrap();
assert!(sat(&after) < 1e-3, "saturation {} → {}", sat(&before), sat(&after));
@@ -688,6 +868,172 @@ mod tests {
assert!((mean(&dim) / mean(&before) - 0.5).abs() < 0.02, "{} vs {}", mean(&dim), mean(&before));
}
/// iPhone ProRAW-style semantic masks drive the Sky mask: Apple's sky matte (here the right
/// half of the sensor, i.e. the bottom of the photo once turned 90° clockwise — where the sky
/// heuristic would never look) is used; a matte with an unknown name is ignored.
#[test]
fn dng_sky_matte_drives_the_sky_mask() {
use lightcraft_develop::{DevelopSettings, LocalAdjustments, Mask, MaskComponent, MaskOp, MaskShape};
use lightcraft_pipeline::{RenderRequest, render};
use lightcraft_tiff::tags::{self as t, photometric};
use lightcraft_tiff::{ByteOrder, IfdBuilder, ImageData, TiffWriter, Value};
let (w, h) = (16u32, 8u32);
let image = |ifd: &mut IfdBuilder, w: u32, h: u32, cpp: u16, bits: u16, data: Vec<u8>| {
ifd.set(t::IMAGE_WIDTH, Value::Long(vec![w]));
ifd.set(t::IMAGE_LENGTH, Value::Long(vec![h]));
ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![bits; cpp as usize]));
ifd.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![cpp]));
ifd.set(t::COMPRESSION, Value::Short(vec![1]));
ifd.set_image(ImageData::Strips { rows_per_strip: h, strips: vec![data] });
};
let mut raw = IfdBuilder::new();
raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0]));
raw.set(t::PHOTOMETRIC, Value::Short(vec![photometric::LINEAR_RAW]));
image(&mut raw, w, h, 3, 16, 12000u16.to_le_bytes().repeat((w * h * 3) as usize));
let matte = |name: &str, right: bool| {
let mut m = IfdBuilder::new();
m.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![t::SUBFILE_SEMANTIC_MASK]));
m.set(t::PHOTOMETRIC, Value::Short(vec![photometric::MASK]));
m.set(t::SEMANTIC_NAME, Value::Ascii(name.into()));
image(&mut m, w / 2, h / 2, 1, 8, (0..w / 2 * h / 2).map(|i| if (i % (w / 2) >= w / 4) == right { 255 } else { 0 }).collect());
m
};
let mut ifd0 = IfdBuilder::new();
ifd0.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![1]));
ifd0.set(t::DNG_VERSION, Value::Byte(vec![1, 6, 0, 0]));
ifd0.set(t::MAKE, Value::Ascii("Apple".into()));
ifd0.set(t::ORIENTATION, Value::Short(vec![6]));
ifd0.set(t::COLOR_MATRIX_1, Value::SRational(vec![(1, 1), (0, 1), (0, 1), (0, 1), (1, 1), (0, 1), (0, 1), (0, 1), (1, 1)]));
ifd0.set(t::CALIBRATION_ILLUMINANT_1, Value::Short(vec![21]));
ifd0.set(t::PHOTOMETRIC, Value::Short(vec![photometric::RGB]));
image(&mut ifd0, 4, 2, 3, 8, vec![128; 24]);
ifd0.add_sub_ifd(raw);
ifd0.add_sub_ifd(matte("urn:com:apple:photo:2020:aux:semanticskymatte", true));
ifd0.add_sub_ifd(matte("urn:com:apple:photo:2020:aux:semanticsomethingelse", false));
let bytes = TiffWriter::new(ByteOrder::Little, false).write(&[ifd0]).unwrap();
let (img, info) = load_bytes(&bytes, 64).unwrap();
assert_eq!((img.width, img.height), (8, 16));
assert!(info.mattes.is_some());
let sky = Mask {
components: vec![MaskComponent { name: None, op: MaskOp::Add, invert: false, shape: MaskShape::Sky }],
adjust: LocalAdjustments { exposure: -3.0, ..Default::default() },
..Default::default()
};
let s = DevelopSettings { masks: vec![sky], ..Default::default() };
let req = RenderRequest::fit(8, 16);
let green = |s: &DevelopSettings, info: &SourceInfo, y: usize| render(&img, info, s, &req).image.get(4, y)[1] as i32;
let plain = DevelopSettings::default();
assert_eq!(green(&s, &info, 2), green(&plain, &info, 2), "no sky at the top");
assert!(green(&s, &info, 13) + 40 < green(&plain, &info, 13), "the sky at the bottom is darkened");
// without the matte the heuristic looks at the top of the frame instead
let heuristic = SourceInfo { mattes: None, ..info.clone() };
assert!(green(&s, &heuristic, 2) < green(&plain, &heuristic, 2));
assert_eq!(green(&s, &heuristic, 13), green(&plain, &heuristic, 13));
}
/// A DNG's white balance is re-evaluated through its own colour model: the decoder hands the
/// pipeline the file's colour tags (without the profile look, applied at load) and the white
/// the pixels were developed for, and a custom white balance renders a neutral differently
/// from the Bradford adaptation it replaces.
#[test]
fn dng_white_balance_uses_the_camera_colour_model() {
use lightcraft_pipeline::{RenderRequest, render};
let dng = crate::tests_xmp::synthetic_dng_with(None, Default::default());
let raw = lightcraft_raw::decode(&dng).unwrap();
let (img, info) = load_bytes(&dng, 64).unwrap();
let cc = info.camera_color.as_ref().expect("a DNG with a colour matrix");
assert_eq!(cc.developed_for, lightcraft_raw::color::as_shot_white_xy(&raw));
assert!(cc.tags.profile.is_empty());
assert_eq!(cc.tags.color_matrix, raw.color.color_matrix);
let mut s = lightcraft_develop::DevelopSettings::default();
(s.wb.mode, s.wb.temp, s.wb.tint) = (lightcraft_develop::WbMode::Custom, 3000.0, 20.0);
let req = RenderRequest::fit(32, 32);
let camera = render(&img, &info, &s, &req).image;
let adapted = render(&img, &SourceInfo { camera_color: None, ..info.clone() }, &s, &req).image;
assert_ne!(camera.data, adapted.data);
}
/// Apple ProRAW: like Lightroom Classic, the default render ignores the file's
/// `ProfileGainTableMap` (Apple's local tone mapping); a map that would double every pixel
/// changes nothing.
#[test]
fn dng_gain_table_map_is_not_rendered() {
use lightcraft_raw::gaintable::GainTableMap;
let plain = crate::tests_xmp::synthetic_dng_with(None, Default::default());
let mut raw = lightcraft_raw::decode(&plain).unwrap();
let (before, _) = load_bytes(&lightcraft_raw::write_dng(&raw, &Default::default()).unwrap(), 64).unwrap();
raw.color.profile.gain_table_map = Some(GainTableMap {
points_v: 1,
points_h: 1,
points_n: 1,
spacing_v: 1.0,
spacing_h: 1.0,
origin_v: 0.0,
origin_h: 0.0,
weights: [0.2, 0.2, 0.2, 0.2, 0.2],
gamma: 1.0,
gains: vec![2.0],
});
let with_map = lightcraft_raw::write_dng(&raw, &Default::default()).unwrap();
assert!(lightcraft_raw::decode(&with_map).unwrap().color.profile.gain_table_map.is_some(), "the map is kept");
let (after, _) = load_bytes(&with_map, 64).unwrap();
assert_eq!(before.data, after.data);
}
/// The Profile option "Camera local tone mapping" renders the map by position (in the raw's
/// active area, whatever the photo's orientation), and only when it's on.
#[test]
fn dng_gain_table_map_renders_when_asked() {
use lightcraft_pipeline::{RenderRequest, render};
use lightcraft_raw::gaintable::GainTableMap;
let plain = crate::tests_xmp::synthetic_dng_with(None, Default::default());
let mut raw = lightcraft_raw::decode(&plain).unwrap();
// ×1 at the active area's left edge rising to ×4 at its right edge, whatever the colour
raw.color.profile.gain_table_map = Some(GainTableMap {
points_v: 1,
points_h: 2,
points_n: 1,
spacing_v: 1.0,
spacing_h: 1.0,
origin_v: 0.0,
origin_h: 0.0,
weights: [1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0, 0.0, 0.0],
gamma: 1.0,
gains: vec![1.0, 4.0],
});
let with_map = lightcraft_raw::write_dng(&raw, &Default::default()).unwrap();
let (img, info) = load_bytes(&with_map, 64).unwrap();
let (_, plain_info) = load_bytes(&plain, 64).unwrap();
assert!(info.local_tone.is_some() && plain_info.local_tone.is_none());
let req = RenderRequest::fit(64, 64);
let mut s = lightcraft_develop::DevelopSettings::default();
// (darker, so that ×4 never reaches white)
s.light.exposure = -3.0;
let off = render(&img, &info, &s, &req).image;
assert_eq!(off.data, render(&img, &plain_info, &s, &req).image.data, "off: the map changes nothing");
s.profile.camera_local_tone = true;
assert!(s.to_json().to_string().contains("camera_local_tone"), "the option is part of the settings (and their hash)");
let on = render(&img, &info, &s, &req).image;
// brightness gained, per column: none at the left edge, most at the right
let column = |im: &lightcraft_raster::Rgba8, x: usize| (0..im.height).map(|y| im.data[y * im.width + x][1] as f32).sum::<f32>();
let gain = |x: usize| column(&on, x) / column(&off, x).max(1.0);
let last = on.width - 1;
assert!((gain(0) - 1.0).abs() < 0.08, "left ×{}", gain(0));
assert!(gain(last) > 1.4 && gain(last) > gain(last / 2) && gain(last / 2) > gain(0), "{} {} {}", gain(0), gain(last / 2), gain(last));
// the photo turned upside down: the map follows the raw, so the brightened side swaps
s.orientation = lightcraft_geom::Orientation::Rotate180;
s.profile.camera_local_tone = false;
let off = render(&img, &info, &s, &req).image;
s.profile.camera_local_tone = true;
let on = render(&img, &info, &s, &req).image;
let gain = |x: usize| column(&on, x) / column(&off, x).max(1.0);
assert!(gain(0) > 1.4 && (gain(last) - 1.0).abs() < 0.08, "rotated: {} … {}", gain(0), gain(last));
// a rendered (non-raw) source never gets it
let jpeg = SourceInfo { raw: false, ..info.clone() };
assert!(!lightcraft_pipeline::local_tone::enabled(&jpeg, &s));
}
#[test]
fn unsupported_raw_falls_back_to_embedded_preview() {
let b = cr3_with_preview(48, 32);
+3 -3
View File
@@ -32,9 +32,9 @@ use crate::media::ProbeInfo;
/// File extensions LightCraft imports (lower case).
pub const EXTENSIONS: &[&str] = &[
"jpg", "jpeg", "png", "tif", "tiff", "webp", "dng", "cr2", "cr3", "nef", "nrw", "arw", "raf", "orf", "rw2", "rwl", "raw", "pef", "psd", "jxl",
"gif", "bmp", "heic", "avif", // containers LightCraft cannot decode but imports as preview only (their embedded JPEG)
"iiq", "crw", "mrw", "x3f", "kdc", "mos", "erf",
"jpg", "jpeg", "png", "tif", "tiff", "webp", "dng", "cr2", "cr3", "nef", "nrw", "arw", "raf", "orf", "rw2", "rwl", "raw", "pef", "srw", "psd",
"jxl", "gif", "bmp", "heic", "avif", // containers LightCraft cannot decode but imports as preview only (their embedded JPEG)
"iiq", "crw", "mrw", "x3f", "kdc", "mos", "erf", "3fr", "fff",
];
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+66
View File
@@ -10,15 +10,26 @@
//! run them off the UI thread.
#![forbid(unsafe_code)]
// Model/cache types remain available without linking the optional inference crate.
#[cfg(not(feature = "denoise"))]
extern crate lightcraft_denoise_core as lightcraft_denoise;
pub mod availability;
mod camera_preview;
pub mod camera_profiles;
pub mod cmd;
pub mod config;
pub mod crs;
pub mod crs_masks;
pub mod demo;
pub mod denoise;
pub mod devices;
pub mod export;
pub mod face_download;
#[cfg(not(target_arch = "wasm32"))]
mod faces_index;
#[cfg(not(target_arch = "wasm32"))]
mod faces_worker;
pub mod files;
pub mod fonts;
pub mod guard;
@@ -33,6 +44,7 @@ pub mod logging;
pub mod media;
pub mod memory;
pub mod merge;
mod model_download;
pub mod originals;
pub mod preset_import;
pub mod preset_luminar;
@@ -132,8 +144,10 @@ pub struct Interaction {
/// Source of [`Session::visible_shared`] generations (process-wide, so two sessions never share one).
static VISIBLE_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
static LIBRARY_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
pub struct Session {
library_generation: u64,
/// Auto Sync: edits to the active photo also change the other selected photos (the settings
/// that changed, nothing else).
pub auto_sync: bool,
@@ -157,6 +171,17 @@ pub struct Session {
/// Set by a command whose change must not rewrite the photo's XMP sidecar even with auto-write on
/// (a catalog-only edit of data the sidecar writer does not emit); consumed when the command ends.
pub(crate) skip_auto_write: bool,
/// AI denoise: the model in use, the photos that have their picture and the work in progress.
pub(crate) denoise: denoise::State,
/// Where the host keeps face models (one folder each); `None` where there is no file system (the web).
pub face_models_dir: Option<std::path::PathBuf>,
/// Face model downloads started this session (the staged files wait in `<face_models_dir>/.downloads`).
pub face_downloads: face_download::Downloads,
/// The user's own list of models to download (`catalog.json` in the models folder), as last read.
pub(crate) face_catalog: lightcraft_faces::catalog::Catalog,
/// The loaded recognition model and the face embeddings made with it.
#[cfg(not(target_arch = "wasm32"))]
pub(crate) faces: faces_index::FacesState,
/// Copied develop settings (partial JSON) for Paste.
pub clipboard: Option<Value>,
/// The folder on disk the [`LibrarySource::Folder`] view browses.
@@ -247,8 +272,13 @@ impl Default for Session {
}
impl Session {
pub fn library_generation(&self) -> u64 {
self.library_generation
}
pub fn new() -> Session {
Session {
library_generation: LIBRARY_GEN.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
auto_sync: false,
catalog: Catalog::new(),
source: LibrarySource::All,
@@ -263,6 +293,12 @@ impl Session {
redo: Vec::new(),
interaction: None,
skip_auto_write: false,
denoise: Default::default(),
face_models_dir: None,
face_downloads: Default::default(),
face_catalog: Default::default(),
#[cfg(not(target_arch = "wasm32"))]
faces: Default::default(),
clipboard: None,
meta_clipboard: None,
browse: None,
@@ -403,6 +439,27 @@ impl Session {
Ok(())
}
/// The box to cut a face's picture from: the detector's, when the scan has looked at the face (every face is then shown
/// equally close, however loosely or tightly its own region was drawn), else the region's own box.
pub fn face_view(&self, id: PhotoId, rect: lightcraft_geom::Rect) -> lightcraft_geom::Rect {
#[cfg(not(target_arch = "wasm32"))]
if let Some(v) = self.faces.index.view(id.0, &rect) {
return v;
}
let _ = id;
rect
}
/// Apply an op that is LightCraft's own bookkeeping rather than something the user did (faces found by the background
/// scan): journaled like any op, but not an undo step, and it leaves the redo stack alone.
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn apply_system(&mut self, op: Op) -> Result<()> {
let fwd = op.clone();
self.catalog.apply(op)?;
self.pending_log.push(fwd);
Ok(())
}
/// Leaving photo `id` after editing it: keep its settings as an automatic version (when they
/// differ from its latest version; at most [`AUTO_VERSIONS`] auto versions, oldest dropped).
/// Saved with the library but not an undo step.
@@ -781,10 +838,19 @@ pub fn json_delta(old: &Value, new: &Value) -> Option<Value> {
#[cfg(test)]
mod tests;
#[cfg(test)]
mod tests_album_order;
#[cfg(test)]
mod tests_color;
#[cfg(test)]
mod tests_denoise;
#[cfg(test)]
mod tests_export;
#[cfg(test)]
mod tests_face_models;
#[cfg(test)]
#[cfg(not(target_arch = "wasm32"))]
mod tests_face_recognize;
#[cfg(test)]
mod tests_folders;
#[cfg(test)]
mod tests_forget_local;
+7
View File
@@ -297,6 +297,12 @@ impl Session {
{
log::error!("library: {e}");
}
// Photo ids belong to one library; denoise work must not carry into the next.
self.denoise.library_changed();
self.media.denoise.clear();
// what was learned about the old library's faces is saved, and none of it carries over (photo ids are per library)
#[cfg(not(target_arch = "wasm32"))]
self.faces.library_changed();
let (mut journal, mut catalog, report) = Journal::open(catalog)?;
// A loaded catalog counts revisions from 0, like every other one. Caches (sidebar counts,
// keyword tree, the grid's list…) are keyed on the revision, so give each library loaded
@@ -305,6 +311,7 @@ impl Session {
static LOADS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
catalog.revision = LOADS.fetch_add(1, std::sync::atomic::Ordering::Relaxed).wrapping_add(1) << 32;
self.catalog = catalog;
self.library_generation = crate::LIBRARY_GEN.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
self.undo.clear();
self.redo.clear();
self.interaction = None;
+223 -7
View File
@@ -10,6 +10,7 @@ use std::sync::{
use crate::lightroom_sqlite::{Database, LiveTable, Value as SqlValue};
use lightcraft_catalog::{Album, Flag, Op, Photo, PhotoId, Source};
use lightcraft_geom::{Affine, Orientation, Point, Rect};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
@@ -477,17 +478,77 @@ fn ensure_read_active(cancel: &AtomicBool) -> Result<(), String> {
if cancel.load(Ordering::Relaxed) { Err("Lightroom import cancelled".into()) } else { Ok(()) }
}
fn mapped_settings(text: &str, raw: bool, aspect: f64) -> Result<(Value, Vec<String>), String> {
fn mapped_settings(text: &str, raw: bool, aspect: f64, orientation: Orientation, stored_aspect: f64) -> Result<(Value, Vec<String>), String> {
let mut root = crate::preset_import::parse_lua(text)?;
let deferred = strip_lua_sentinels(&mut root);
let (props, values) = crate::preset_import::lua_settings_props(&root)?;
let (partial, mut unknown) = crate::crs::to_partial_report(&props, Some(&values), Some(raw), aspect);
let (mut props, values) = crate::preset_import::lua_settings_props(&root)?;
// A catalog's develop settings carry the crop edges and angle but no `HasCrop` (XMP has it):
// any crop field means the photo is cropped.
const CROP: [&str; 5] = ["crs:CropLeft", "crs:CropTop", "crs:CropRight", "crs:CropBottom", "crs:CropAngle"];
if !props.contains_key("crs:HasCrop") && CROP.iter().any(|k| props.contains_key(*k)) {
props.insert("crs:HasCrop".into(), vec!["True".into()]);
}
let (mut partial, mut unknown) = crate::crs::to_partial_report(&props, Some(&values), Some(raw), aspect);
catalog_crop(&mut partial, orientation, stored_aspect);
if deferred > 0 {
unknown.push("Deferred Lightroom adjustments (-999999)".into());
}
Ok((partial, unknown))
}
/// Lightroom's `Adobe_images.orientation`: where the stored image's top-left and top-right corners
/// land in the displayed photo, naming the displayed corners A, B, C, D clockwise from the top left
/// (`AB` upright, `BC` a quarter turn clockwise, `DA` a quarter turn counter-clockwise, `BA` mirrored).
fn lightroom_orientation(code: &str) -> Option<Orientation> {
const ALL: [Orientation; 8] = [
Orientation::Normal,
Orientation::FlipH,
Orientation::Rotate180,
Orientation::FlipV,
Orientation::Transpose,
Orientation::Rotate90,
Orientation::Transverse,
Orientation::Rotate270,
];
let corner = |o: Orientation, x: f64, y: f64| match o.map(x, y, 1.0, 1.0) {
(u, v) if u < 0.5 && v < 0.5 => 'A',
(_, v) if v < 0.5 => 'B',
(u, _) if u >= 0.5 => 'C',
_ => 'D',
};
ALL.into_iter().find(|&o| code.chars().eq([corner(o, 0.0, 0.0), corner(o, 1.0, 0.0)]))
}
/// Lightroom's crop in LightCraft's frame. `crs` gives the crop's top-left (`CropLeft`, `CropTop`)
/// and bottom-right (`CropRight`, `CropBottom`) corners on the stored image (before orientation and
/// straightening), normalized to its size, plus a straighten angle that turns the other way to
/// LightCraft's. LightCraft's crop is a rectangle in the oriented image rotated by `angle` about its
/// centre, normalized to the oriented image's size, so the corners go through the orientation and
/// then the rotation; `stored_aspect` (width / height of the stored image) matters once the
/// rectangle turns. On a real catalog this reproduced the frame aspect of all 223 straightened crops
/// in Lightroom's exports within 1%.
fn catalog_crop(partial: &mut Value, orientation: Orientation, stored_aspect: f64) {
let Some(geometry) = partial.pointer_mut("/crop/geometry").and_then(Value::as_object_mut) else { return };
let edge = |k: &str| geometry.get("rect").and_then(|r| r.get(k)).and_then(Value::as_f64);
let (Some(x0), Some(y0), Some(x1), Some(y1)) = (edge("x0"), edge("y0"), edge("x1"), edge("y1")) else { return };
let lightroom_angle = geometry.get("angle").and_then(Value::as_f64).unwrap_or(0.0);
if orientation == Orientation::Normal && lightroom_angle == 0.0 {
return;
}
let w = if stored_aspect.is_finite() && stored_aspect > 0.0 { stored_aspect } else { 1.0 };
let (ow, oh) = if orientation.swaps_axes() { (1.0, w) } else { (w, 1.0) };
// a mirror turns the straighten the other way too
let angle = if orientation.to_parts().0 { lightroom_angle } else { -lightroom_angle };
let to_rotated = Affine::rotate_about(angle.to_radians(), Point::new(ow / 2.0, oh / 2.0));
let corner = |x: f64, y: f64| {
let (u, v) = orientation.map(x * w, y, w, 1.0);
to_rotated.apply(Point::new(u, v))
};
let r = Rect::from_points(corner(x0, y0), corner(x1, y1));
geometry.insert("rect".into(), json!({"x0": r.x0 / ow, "y0": r.y0 / oh, "x1": r.x1 / ow, "y1": r.y1 / oh}));
geometry.insert("angle".into(), json!(angle));
}
fn strip_lua_sentinels(value: &mut crate::preset_import::Lua) -> usize {
use crate::preset_import::Lua;
match value {
@@ -680,7 +741,15 @@ pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, contex
p.meta.keywords = src.keywords.clone();
}
if !src.settings.is_empty() && !keep_develop {
match mapped_settings(&src.settings, p.kind == lightcraft_catalog::MediaKind::Raw, p.width.max(1) as f64 / p.height.max(1) as f64) {
let orientation = lightroom_orientation(text(&src.image, "orientation")).unwrap_or_default();
let stored_aspect = number(&src.image, "fileWidth") as f64 / number(&src.image, "fileHeight") as f64;
match mapped_settings(
&src.settings,
p.kind == lightcraft_catalog::MediaKind::Raw,
p.width.max(1) as f64 / p.height.max(1) as f64,
orientation,
stored_aspect,
) {
Ok((partial, unknown)) => {
p.develop = Arc::new(lightcraft_develop::apply_partial(&p.develop, &partial, 1.0));
p.edited = Some(now.clone());
@@ -788,6 +857,7 @@ pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, contex
#[cfg(test)]
mod tests {
use super::*;
use lightcraft_geom::CropGeometry;
fn row(v: Value) -> Row {
serde_json::from_value(v).unwrap()
}
@@ -866,7 +936,7 @@ mod tests {
cyclic.collections[0].insert("parent".into(), json!(10));
assert!(apply(&mut s, cyclic, false).is_err());
assert_eq!(s.catalog.to_snapshot(), before);
assert!(mapped_settings(&format!("{}0{}", "{".repeat(1000), "}".repeat(1000)), true, 1.0).is_err());
assert!(mapped_settings(&format!("{}0{}", "{".repeat(1000), "}".repeat(1000)), true, 1.0, Orientation::Normal, 1.5).is_err());
}
#[test]
fn compressed_xmp_is_bounded_and_settings_are_data_only() {
@@ -876,10 +946,156 @@ mod tests {
assert_eq!(xmp(Some(&json!(encoded))).unwrap(), String::from_utf8_lossy(packet));
encoded[0] = 0xff;
assert!(xmp(Some(&json!(encoded))).is_err());
let (partial, _) = mapped_settings("s = { Exposure2012 = 0.75, ToneCurvePV2012 = {0,0,128,150,255,255} }", true, 1.5).unwrap();
let (partial, _) =
mapped_settings("s = { Exposure2012 = 0.75, ToneCurvePV2012 = {0,0,128,150,255,255} }", true, 1.5, Orientation::Normal, 1.5).unwrap();
assert_eq!(partial["light"]["exposure"], 0.75);
}
#[test]
fn catalog_crops_without_has_crop_are_imported() {
// Lightroom Classic catalogs store crop edges and angle but never `HasCrop`
let (partial, unknown) = mapped_settings(
"s = { CropConstrainAspectRatio = true, CropLeft = 0.042702, CropRight = 0.957298, CropTop = 0.1 }",
true,
1.5,
Orientation::Normal,
1.5,
)
.unwrap();
let crop = &partial["crop"]["geometry"];
assert_eq!(crop["rect"]["x0"], 0.042702);
assert_eq!(crop["rect"]["y0"], 0.1);
assert_eq!(crop["rect"]["x1"], 0.957298);
assert_eq!(crop["rect"]["y1"], 1.0);
assert_eq!(crop["angle"], 0.0);
assert!(!unknown.iter().any(|k| k.starts_with("CropLeft")), "{unknown:?}");
// an angle alone still means a crop (Lightroom's angle turns the other way to LightCraft's)
let (partial, unknown) = mapped_settings("s = { CropAngle = -1.5 }", true, 1.5, Orientation::Normal, 1.5).unwrap();
assert_eq!(partial["crop"]["geometry"]["angle"], 1.5);
assert!(!unknown.iter().any(|k| k.starts_with("CropAngle")), "{unknown:?}");
// no crop fields: the crop stays untouched
let (partial, _) = mapped_settings("s = { Exposure2012 = 0.5 }", true, 1.5, Orientation::Normal, 1.5).unwrap();
assert!(partial.get("crop").is_none());
}
#[test]
fn lightroom_orientation_codes() {
// the three seen in a real catalog, checked against each file's EXIF orientation (1, 6, 8)
assert_eq!(lightroom_orientation("AB"), Some(Orientation::Normal));
assert_eq!(lightroom_orientation("BC"), Some(Orientation::Rotate90));
assert_eq!(lightroom_orientation("DA"), Some(Orientation::Rotate270));
assert_eq!(lightroom_orientation("CD"), Some(Orientation::Rotate180));
assert_eq!(lightroom_orientation("BA"), Some(Orientation::FlipH));
assert_eq!(lightroom_orientation("DC"), Some(Orientation::FlipV));
assert_eq!(lightroom_orientation("AD"), Some(Orientation::Transpose));
assert_eq!(lightroom_orientation("CB"), Some(Orientation::Transverse));
for code in ["", "A", "ABC", "AC", "ab", "XY"] {
assert_eq!(lightroom_orientation(code), None, "{code:?}");
}
}
#[test]
fn catalog_crops_follow_the_photo_orientation() {
let crop = |o: Orientation| {
let settings = "s = { CropLeft = 0.1, CropRight = 0.6, CropTop = 0.2, CropBottom = 0.9 }";
let (partial, _) = mapped_settings(settings, true, 1.5, o, 1.5).unwrap();
let g = &partial["crop"]["geometry"];
let r = &g["rect"];
[r["x0"].as_f64(), r["y0"].as_f64(), r["x1"].as_f64(), r["y1"].as_f64(), g["angle"].as_f64()].map(|v| (v.unwrap() * 1e9).round() / 1e9)
};
// (x, y) of the stored image → (1 − y, x) a quarter turn clockwise, (y, 1 − x) counter-clockwise
assert_eq!(crop(Orientation::Normal), [0.1, 0.2, 0.6, 0.9, 0.0]);
assert_eq!(crop(Orientation::Rotate90), [0.1, 0.1, 0.8, 0.6, 0.0]);
assert_eq!(crop(Orientation::Rotate270), [0.2, 0.4, 0.9, 0.9, 0.0]);
assert_eq!(crop(Orientation::Rotate180), [0.4, 0.1, 0.9, 0.8, 0.0]);
assert_eq!(crop(Orientation::FlipH), [0.4, 0.2, 0.9, 0.9, 0.0]);
// a mirror turns the straighten the other way
let angle = |o: Orientation| {
let (partial, _) = mapped_settings("s = { CropAngle = 2 }", true, 1.5, o, 1.5).unwrap();
partial["crop"]["geometry"]["angle"].as_f64().unwrap()
};
assert_eq!(angle(Orientation::Rotate90), -2.0);
assert_eq!(angle(Orientation::FlipH), 2.0);
// no crop: nothing to orient
let (partial, _) = mapped_settings("s = { Exposure2012 = 0.5 }", true, 1.5, Orientation::Rotate90, 1.5).unwrap();
assert!(partial.get("crop").is_none());
}
/// The `crs` corners and angle Lightroom would store for LightCraft crop `g` on a stored image
/// of aspect `w` (height 1) shown with orientation `o`.
fn lightroom_crop(g: CropGeometry, o: Orientation, w: f64) -> String {
let (ow, oh) = if o.swaps_axes() { (1.0, w) } else { (w, 1.0) };
let to_source = Affine::rotate_about(-g.angle.to_radians(), Point::new(ow / 2.0, oh / 2.0));
let stored = |x: f64, y: f64| {
let p = to_source.apply(Point::new(x * ow, y * oh));
o.inverse().map(p.x, p.y, ow, oh)
};
// the rectangle's corners that land top-left and bottom-right on the stored image
let corners = [stored(g.rect.x0, g.rect.y0), stored(g.rect.x1, g.rect.y0), stored(g.rect.x1, g.rect.y1), stored(g.rect.x0, g.rect.y1)];
let (tl, br) = (0..4).map(|i| (corners[i], corners[(i + 2) % 4])).find(|(a, b)| a.0 < b.0 && a.1 < b.1).unwrap();
let angle = if o.to_parts().0 { g.angle } else { -g.angle };
format!("s = {{ CropLeft = {}, CropTop = {}, CropRight = {}, CropBottom = {}, CropAngle = {angle} }}", tl.0 / w, tl.1, br.0 / w, br.1)
}
#[test]
fn straightened_catalog_crops_round_trip() {
let g = CropGeometry { rect: Rect::new(0.2, 0.15, 0.7, 0.8), angle: 6.5 };
for o in [Orientation::Normal, Orientation::Rotate90, Orientation::Rotate270, Orientation::Rotate180, Orientation::FlipH] {
for angle in [6.5, -6.5] {
let g = CropGeometry { angle, ..g };
let (partial, _) = mapped_settings(&lightroom_crop(g, o, 1.5), true, 1.5, o, 1.5).unwrap();
let got: CropGeometry = serde_json::from_value(partial["crop"]["geometry"].clone()).unwrap();
let close = |a: f64, b: f64| (a - b).abs() < 1e-9;
assert!(
close(got.angle, g.angle)
&& [(got.rect.x0, g.rect.x0), (got.rect.y0, g.rect.y0), (got.rect.x1, g.rect.x1), (got.rect.y1, g.rect.y1)]
.iter()
.all(|&(a, b)| close(a, b)),
"{o:?} {angle}: {got:?}"
);
}
}
}
#[test]
fn straightened_catalog_crops_match_lightroom_exports() {
// Settings and pixel sizes from a real Lightroom Classic catalog (9504 × 6336 stored) and the
// JPEGs Lightroom exported from it at full size.
for (settings, o, export) in [
(
"s = { CropAngle = -9.34444, CropBottom = 0.802034, CropLeft = 0.05499, CropRight = 0.94501, CropTop = 0.197966 }",
Orientation::Normal,
(7725.0, 5150.0),
),
(
"s = { CropAngle = -1.48964, CropBottom = 0.962457, CropLeft = 0.010429, CropRight = 0.989571, CropTop = 0.037543 }",
Orientation::Rotate90,
(6100.0, 9150.0),
),
] {
let (partial, _) = mapped_settings(settings, true, 1.5, o, 9504.0 / 6336.0).unwrap();
let r: Rect = serde_json::from_value(partial["crop"]["geometry"]["rect"].clone()).unwrap();
let (ow, oh) = if o.swaps_axes() { (6336.0, 9504.0) } else { (9504.0, 6336.0) };
let (w, h) = (r.width() * ow, r.height() * oh);
assert!((w - export.0).abs() <= 3.0 && (h - export.1).abs() <= 3.0, "{o:?}: {w:.1} × {h:.1}, Lightroom exported {export:?}");
}
}
#[test]
fn imported_crops_use_the_catalog_orientation() {
let mut data = sample();
data.photos.truncate(1);
data.photos[0].image.remove("masterImage");
data.photos[0].image.insert("orientation".into(), json!("BC"));
data.photos[0].settings = "s = { CropLeft = 0.1, CropRight = 0.6, CropTop = 0.2, CropBottom = 0.9 }".into();
data.photos[0].xmp = String::new();
let mut s = crate::Session::new();
apply(&mut s, data, false).unwrap();
let g = s.catalog.photos().next().unwrap().develop.crop.geometry;
let r = [g.rect.x0, g.rect.y0, g.rect.x1, g.rect.y1].map(|v| (v * 1e9).round() / 1e9);
assert_eq!(r, [0.1, 0.1, 0.8, 0.6]);
}
#[test]
fn deferred_auto_tone_sentinels_never_become_real_slider_values() {
let mut data = sample();
@@ -896,7 +1112,7 @@ mod tests {
assert_eq!(p.develop.light.shadows, 65.0);
assert_eq!(p.develop.light.blacks, -22.0);
assert!(report["unmapped"]["2"].as_array().unwrap().iter().any(|v| v.as_str().is_some_and(|v| v.contains("Deferred"))));
let (partial, _) = mapped_settings("s = { Exposure2012 = -5, Contrast2012 = -100 }", true, 1.0).unwrap();
let (partial, _) = mapped_settings("s = { Exposure2012 = -5, Contrast2012 = -100 }", true, 1.0, Orientation::Normal, 1.5).unwrap();
assert_eq!(partial["light"]["exposure"], -5.0);
assert_eq!(partial["light"]["contrast"], -100.0);
}
+250 -31
View File
@@ -62,7 +62,7 @@ impl SettingsHashes {
}
/// Bump when the pipeline's output changes, to invalidate cached thumbnails.
pub const RENDER_CACHE_VERSION: u64 = 15;
pub const RENDER_CACHE_VERSION: u64 = 22;
/// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache).
pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512];
@@ -125,6 +125,43 @@ pub type FileLoader = Arc<dyn Fn(&str, usize) -> Result<(Rgb32f, SourceInfo), St
/// `max_edge` (set by the app). `None`: no usable preview.
pub type PreviewLoader = Arc<dyn Fn(&str, usize) -> Option<Rgba8> + Send + Sync>;
/// Makes a photo's denoised picture when it is not cached yet (an export does not wait for the background queue):
/// `Ok(true)` once it is there, `Ok(false)` when denoise does not apply to this photo (render it plain), `Err` when it
/// should have worked and did not.
pub type MakeProduct = Arc<dyn Fn() -> Result<bool, String> + Send + Sync>;
/// A raw photo's cached AI-denoised picture (see [`crate::denoise`]): where the file is and the key it must have been
/// made with. A file that is missing, damaged or made for something else is a cache miss, never an error.
#[derive(Clone)]
pub struct DenoiseSpec {
pub product: std::path::PathBuf,
pub key: String,
/// Run before the picture is read, by a job that must not render without it (an export).
pub make: Option<MakeProduct>,
}
/// Decodes a raw file once and develops it twice: as usual, and from the denoised picture in `spec` when that can be
/// read (`None` when it cannot). Both are at most `max_edge` long and the same size.
pub type PairLoader = Arc<dyn Fn(&str, usize, &DenoiseSpec) -> Result<(Rgb32f, Option<Rgb32f>, SourceInfo), String> + Send + Sync>;
/// A photo's denoised picture next to its plain one, and the last mix of the two that was asked for.
pub struct Twin {
pub image: Arc<Rgb32f>,
/// The Amount (in thousandths) and the picture mixed for it, so dragging other sliders keeps one source picture.
mix: std::sync::Mutex<Option<(u32, Arc<Rgb32f>)>>,
}
impl Twin {
pub fn new(image: Arc<Rgb32f>) -> Twin {
Twin { image, mix: std::sync::Mutex::new(None) }
}
/// Bytes reserved for the twin and its lazy mix, before either cache cost can grow.
fn bytes(&self) -> usize {
self.image.data.len().saturating_mul(24)
}
}
/// Pixels and the source interpretation learned while decoding; they must be cached together.
#[derive(Clone)]
pub struct DecodedSource {
@@ -133,17 +170,53 @@ pub struct DecodedSource {
/// A smart preview's stored camera tone curve: the one decoder fact its pixels need that the
/// catalog's header facts lack (used when `info` is `None`).
pub camera_tone: Option<lightcraft_pipeline::tone::CameraTone>,
/// The same picture developed from the AI-denoised mosaic, when the photo has one cached.
pub denoised: Option<Arc<Twin>>,
}
impl DecodedSource {
pub fn new(image: Arc<Rgb32f>, info: Option<SourceInfo>) -> Self {
DecodedSource { image, info, camera_tone: None }
DecodedSource { image, info, camera_tone: None, denoised: None }
}
/// The picture to develop for a Denoise `amount` (0 to 1, see [`DevelopSettings::denoise_amount`]): the plain one,
/// the denoised one, or a mix of the two. A mix is made once per amount and kept, so the stages after it are
/// reused until the amount changes.
pub fn image_for(&self, amount: f32) -> Arc<Rgb32f> {
let Some(twin) = &self.denoised else { return self.image.clone() };
if amount.is_nan() || amount <= 0.0 || twin.image.width != self.image.width || twin.image.height != self.image.height {
return self.image.clone();
}
if amount >= 1.0 {
return twin.image.clone();
}
let thousandths = (amount * 1000.0).round().clamp(1.0, 999.0) as u32;
let mut memo = twin.mix.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some((t, img)) = memo.as_ref()
&& *t == thousandths
{
return img.clone();
}
let mut mixed = (*self.image).clone();
if !lightcraft_denoise::product::blend(&mut mixed, &twin.image, thousandths as f32 / 1000.0) {
return self.image.clone();
}
let mixed = Arc::new(mixed);
*memo = Some((thousandths, mixed.clone()));
mixed
}
/// Cache cost: the picture, its segmentation mattes, its denoised twin and room for the lazy mix.
pub fn bytes(&self) -> usize {
let mattes = self.info.as_ref().and_then(|i| i.mattes.as_ref()).map_or(0, |m| m.bytes());
let mattes = mattes + self.info.as_ref().and_then(|i| i.local_tone.as_ref()).map_or(0, |t| t.map.gains.len() * 4);
self.image.data.len() * 12 + 64 + std::mem::size_of::<SourceInfo>() + mattes + self.denoised.as_ref().map_or(0, |t| t.bytes())
}
/// What to render these pixels against: the decoder's facts, else `header` (the catalog's)
/// with any stored camera tone curve.
pub fn info_or(&self, header: SourceInfo) -> SourceInfo {
self.info.unwrap_or(SourceInfo { camera_tone: self.camera_tone.or(header.camera_tone), ..header })
self.info.clone().unwrap_or_else(|| SourceInfo { camera_tone: self.camera_tone.or(header.camera_tone), ..header })
}
}
@@ -161,6 +234,8 @@ pub enum SourceRef {
/// The photo's smart preview, used when the original can't be read (offline drive). The
/// UI thread never checks whether the original is there; the render worker finds out.
fallback: Option<std::path::PathBuf>,
/// The photo has a cached denoised picture: load it with the plain one (see [`PairLoader`]).
denoise: Option<(DenoiseSpec, PairLoader)>,
},
/// A smart preview standing in for a missing original.
Smart {
@@ -176,16 +251,46 @@ impl SourceRef {
pub fn load_source(&self) -> Result<DecodedSource, String> {
let image = match self {
SourceRef::Loaded(a) => return Ok((**a).clone()),
SourceRef::Demo { scene, max_edge } => Arc::new(scene.render_fit(*max_edge)),
SourceRef::File { path, max_edge, loader, fallback } => {
let r = match loader {
SourceRef::Demo { scene, max_edge } => crate::demo::scene_pixels(scene, *max_edge),
SourceRef::File { path, max_edge, loader, fallback, denoise } => {
let plain = || match loader {
Some(l) => l(path, *max_edge).map(|(image, info)| DecodedSource::new(Arc::new(image), Some(info))),
None => Err(format!("no decoder available for {path}")),
};
// the denoised picture comes with the plain one from a single decode; when it cannot be read the
// pair loader gives the plain picture alone (the cached file is never a reason to fail a render)
let r = match denoise {
Some((spec, pair)) => {
// a job that needs the denoised picture has it made first; if that fails the job fails (no smart
// preview stands in: an export must not quietly come out without the denoise it was asked for)
let made = match &spec.make {
Some(make) => make()?,
None => true,
};
if made {
pair(path, *max_edge, spec).and_then(|(image, twin, info)| {
if spec.make.is_some() && twin.is_none() {
return Err("the cached AI Denoise picture could not be read; clear the denoise cache and retry".into());
}
Ok(DecodedSource {
image: Arc::new(image),
info: Some(info),
camera_tone: None,
denoised: twin.map(|t| Arc::new(Twin::new(Arc::new(t)))),
})
})
} else {
plain()
}
}
None => plain(),
};
// An offline original renders from its smart preview (no decoder facts: header ones).
return match (r, fallback) {
#[cfg(not(target_arch = "wasm32"))]
(Err(e), Some(sp)) if crate::smart::is_valid(sp) => crate::smart::load(sp).map_err(|e2| format!("{e}; smart preview: {e2}")),
(Err(e), Some(sp)) if denoise.as_ref().is_none_or(|(spec, _)| spec.make.is_none()) && crate::smart::is_valid(sp) => {
crate::smart::load(sp).map_err(|e2| format!("{e}; smart preview: {e2}"))
}
(r, _) => r,
};
}
@@ -224,6 +329,54 @@ pub struct MediaCache {
scenes: Vec<lightcraft_scenes::Scene>,
/// Rendered thumbnails (memory, plus disk once a library is attached).
pub rendered: Arc<PreviewCache>,
/// Which photos have a cached denoised picture, and how to read it.
pub denoise: DenoiseIndex,
}
/// The photos that have a cached denoised picture for the denoise model in use. Only memory: nothing here touches the
/// file system, so it can be asked on the UI thread.
#[derive(Default)]
pub struct DenoiseIndex {
ready: std::collections::HashMap<PhotoId, DenoiseSpec>,
/// Reads the pictures (`None`: this build cannot, and nothing is ever ready).
pub loader: Option<PairLoader>,
}
impl DenoiseIndex {
/// What to load for `id` besides its plain picture, and how.
pub fn get(&self, id: PhotoId) -> Option<(&DenoiseSpec, &PairLoader)> {
self.ready.get(&id).zip(self.loader.as_ref())
}
pub fn spec(&self, id: PhotoId) -> Option<&DenoiseSpec> {
self.ready.get(&id)
}
pub fn set(&mut self, id: PhotoId, spec: DenoiseSpec) {
self.ready.insert(id, spec);
}
pub fn remove(&mut self, id: PhotoId) -> bool {
self.ready.remove(&id).is_some()
}
pub fn clear(&mut self) {
self.ready.clear();
}
pub fn len(&self) -> usize {
self.ready.len()
}
pub fn is_empty(&self) -> bool {
self.ready.is_empty()
}
pub fn ids(&self) -> Vec<PhotoId> {
let mut v: Vec<PhotoId> = self.ready.keys().copied().collect();
v.sort();
v
}
}
impl Default for MediaCache {
@@ -244,6 +397,7 @@ impl Default for MediaCache {
availability: Default::default(),
scenes: Vec::new(),
rendered: Arc::new(PreviewCache::memory(rendered_budget(budget))),
denoise: DenoiseIndex::default(),
}
}
}
@@ -253,10 +407,6 @@ fn rendered_budget(share: usize) -> usize {
RENDERED_MEM_BYTES.min(share / 4)
}
fn source_bytes(img: &Rgb32f) -> usize {
img.data.len() * 12 + 64 + std::mem::size_of::<SourceInfo>()
}
impl MediaCache {
/// Keep rendered thumbnails on disk in `dir` as well. The cache this replaces is retired
/// (its files stay): render jobs still holding it can't write through it any more, even
@@ -319,8 +469,8 @@ impl MediaCache {
let Some((tick, which)) = candidates.into_iter().flatten().min() else { break };
let freed = match which {
0 => self.thumbs.pop_oldest(),
1 => self.previews.iter().position(|e| e.2 == tick).map(|i| source_bytes(&self.previews.remove(i).1.image)),
2 => self.full.take().map(|e| source_bytes(&e.1.image)),
1 => self.previews.iter().position(|e| e.2 == tick).map(|i| self.previews.remove(i).1.bytes()),
2 => self.full.take().map(|e| e.1.bytes()),
_ => self.rendered.evict_oldest(),
};
match freed {
@@ -372,7 +522,7 @@ impl MediaCache {
let tick = lightcraft_preview::next_tick();
match level {
SourceLevel::Thumb => {
let cost = source_bytes(&img.image);
let cost = img.bytes();
self.thumbs.insert(id, img, cost);
}
SourceLevel::Preview => {
@@ -415,13 +565,42 @@ impl MediaCache {
/// Decoded sources held: (thumbnail level, preview level, full size).
pub fn usage(&self) -> (crate::memory::Usage, crate::memory::Usage, crate::memory::Usage) {
use crate::memory::Usage;
let previews = Usage::new(self.previews.len(), self.previews.iter().map(|e| source_bytes(&e.1.image)).sum());
let full = self.full.as_ref().map(|e| Usage::new(1, source_bytes(&e.1.image))).unwrap_or_default();
let previews = Usage::new(self.previews.len(), self.previews.iter().map(|e| e.1.bytes()).sum());
let full = self.full.as_ref().map(|e| Usage::new(1, e.1.bytes())).unwrap_or_default();
(Usage::new(self.thumbs.len(), self.thumbs.cost()), previews, full)
}
/// What a render of photo `id` with settings `s` adds to its identity for the denoised picture it will use: 0 unless
/// it uses one (the Denoise amount is above 0 and the photo has a cached denoised picture).
pub fn denoise_salt(&self, id: PhotoId, s: &DevelopSettings) -> u64 {
match self.denoise.spec(id) {
Some(spec) if s.denoise_amount() > 0.0 => Hasher128::new().str(&spec.key).finish().0 as u64 | 1,
_ => 0,
}
}
/// How to load photo `p` at `level`: from the cache when its source is there, else from its file (with its denoised
/// picture beside it when it has one), its smart preview or the demo scene.
pub fn source_ref(&mut self, p: &Photo, level: SourceLevel) -> SourceRef {
if let Some(a) = self.get_source(p.id, level) {
self.source_ref_with(p, level, true)
}
/// [`Self::source_ref`] that never takes a decoded source from memory: for a job that needs something the cached
/// one lacks (an export, whose denoised picture is made when it runs).
pub fn source_ref_uncached(&mut self, p: &Photo, level: SourceLevel) -> SourceRef {
self.source_ref_with(p, level, false)
}
fn source_ref_with(&mut self, p: &Photo, level: SourceLevel, cached: bool) -> SourceRef {
let mut r = self.source_ref_plain(p, level, cached);
if let (SourceRef::File { denoise, .. }, Some((spec, loader))) = (&mut r, self.denoise.get(p.id)) {
*denoise = Some((spec.clone(), loader.clone()));
}
r
}
fn source_ref_plain(&mut self, p: &Photo, level: SourceLevel, cached: bool) -> SourceRef {
if cached && let Some(a) = self.get_source(p.id, level) {
return SourceRef::Loaded(Box::new(a));
}
// Procedural scenes have a nominal size: "full" is that size, not unbounded.
@@ -459,10 +638,12 @@ impl MediaCache {
}
match self.scenes.iter().find(|s| s.id == *scene) {
Some(s) => SourceRef::Demo { scene: Box::new(s.clone()), max_edge },
None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None, fallback: None },
None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None, fallback: None, denoise: None },
}
}
Source::File { path } => SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone(), fallback: None },
Source::File { path } => {
SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone(), fallback: None, denoise: None }
}
}
}
}
@@ -573,7 +754,8 @@ impl RenderJob {
let was_loaded = matches!(self.source, SourceRef::Loaded(_));
match self.source.load_source() {
Ok(source) => {
let src = &source.image;
// the plain picture, or (Denoise amount above 0 and a denoised picture cached) the mix the amount asks for
let src = &source.image_for(self.settings.denoise_amount());
let info = source.info_or(self.info);
// Thumbnails (many small jobs side by side) stay on the CPU; views and exports use
// the GPU when there is one.
@@ -757,22 +939,23 @@ impl crate::Session {
let content = source_key.0 as u64;
// Temporary "before" settings must not replace the photo's cached develop hash.
let settings_hash = if before { settings.hash64() } else { self.media.settings_hashes.get(id, &settings) };
let salt = self.media.denoise_salt(id, &settings);
let key = settings_hash
^ ((max_w as u64) << 40)
^ ((max_h as u64) << 20)
^ (apply_crop as u64)
^ (level as u64) << 60
^ id.0.wrapping_mul(0x9e37_79b9_7f4a_7c15)
^ content.rotate_left(17);
^ content.rotate_left(17)
^ salt;
let cache = thumb_bucket.map(|b| {
let k = Hasher128::new()
.str(&content_key)
.u64(settings_hash)
.u64(b as u64)
.u64(RENDER_CACHE_VERSION)
.u64(crate::camera_profiles::cache_key())
.finish();
(self.media.rendered.clone(), k)
let mut k = Hasher128::new();
k.str(&content_key).u64(settings_hash).u64(b as u64).u64(RENDER_CACHE_VERSION).u64(crate::camera_profiles::cache_key());
// (only a render that uses a denoised picture has it in its key, so every other thumbnail stays valid)
if salt != 0 {
k.u64(salt);
}
(self.media.rendered.clone(), k.finish())
});
Some(RenderJob {
request_id: 0,
@@ -979,6 +1162,17 @@ impl crate::Session {
}
}
/// The camera's embedded preview of a raw file, for the face scan. Faces are read from the unedited picture, so the
/// photo's edits do not matter, and the camera's own JPEG (oriented, display-ready) costs a fraction of decoding the
/// raw. `None` for anything else, and where the host installed no loader.
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn scan_preview_of(&self, p: &Photo) -> Option<(String, PreviewLoader)> {
match (&p.source, &self.media.preview_loader) {
(Source::File { path }, Some(l)) if p.kind == MediaKind::Raw => Some((path.clone(), l.clone())),
_ => None,
}
}
/// Something to show in the loupe right away for `id` (see [`QuickJob`]): its cached view
/// render, else the embedded preview of an unedited raw, else a cached or fresh thumbnail.
pub fn quick_view_job(&mut self, id: PhotoId, max_edge: usize, apply_crop: bool) -> Option<QuickJob> {
@@ -1052,6 +1246,15 @@ impl crate::Session {
depth: lightcraft_pipeline::OutputDepth,
) -> Result<RenderJob, String> {
let mut job = self.render_job(id, max_w, max_h, false, true).ok_or("no such photo")?;
// An export is made from the denoised picture its Denoise amount asks for, made when the job runs if the
// background queue has not got to it. (A source kept in memory may lack the picture, so the job reads the file.)
if let Some(wanted) = self.denoise_for_export(id, &job.settings)? {
let p = self.catalog.photo(id).ok_or("no such photo")?.clone();
job.source = self.media.source_ref_uncached(&p, job.level);
if let SourceRef::File { denoise, .. } = &mut job.source {
*denoise = Some(wanted);
}
}
job.request.space = space;
job.request.depth = depth;
job.key ^= (space as u64 + 1).wrapping_mul(0xa076_1d64_78bd_642f) ^ (depth as u64 + 1).wrapping_mul(0xe703_7ed1_a0b4_28db);
@@ -1199,6 +1402,20 @@ mod thumbnail_hash_tests {
mod tests {
use super::*;
#[test]
fn denoise_mix_is_reserved_before_the_source_enters_the_cache() {
let mut plain = Rgb32f::new(8, 8);
plain.data.fill([1.0; 3]);
let mut source = DecodedSource::new(Arc::new(plain), None);
source.denoised = Some(Arc::new(Twin::new(Arc::new(Rgb32f::new(8, 8)))));
let reserved = source.bytes();
assert!(reserved >= 3 * 8 * 8 * 12);
assert_eq!(source.image_for(0.5).data[0], [0.5; 3]);
assert_eq!(source.bytes(), reserved, "creating a mix cannot evade the LRU's recorded cost");
let _ = source.image_for(0.25);
assert_eq!(source.bytes(), reserved, "changing Amount replaces the reserved mix");
}
#[test]
fn decoder_info_survives_render_jobs_cache_and_eviction() {
let tone = lightcraft_pipeline::tone::CameraTone::new(std::array::from_fn(|i| {
@@ -1210,20 +1427,22 @@ mod tests {
let mut s = crate::Session::with_demo();
let id = s.active().unwrap();
let mut job = s.render_job(id, 64, 64, false, true).unwrap();
let loaded_info = info.clone();
job.source = SourceRef::File {
path: "synthetic.arw".into(),
max_edge: 64,
loader: Some(Arc::new(move |_, _| {
let mut image = Rgb32f::new(64, 64);
image.data.fill([0.1; 3]);
Ok((image, info))
Ok((image, loaded_info.clone()))
})),
fallback: None,
denoise: None,
};
job.info = SourceInfo::default(); // Header facts cannot override decoder facts.
job.settings = Arc::new(DevelopSettings::default());
let r = job.clone().run();
assert_eq!(r.loaded.as_ref().unwrap().info, Some(info));
assert_eq!(r.loaded.as_ref().unwrap().info.as_ref(), Some(&info));
let expected = lightcraft_pipeline::render(&r.loaded.as_ref().unwrap().image, &info, &job.settings, &job.request);
assert_eq!(r.rendered.as_ref().unwrap().image.data, expected.image.data);
s.accept(&r);
+359
View File
@@ -0,0 +1,359 @@
//! Downloading a denoise model explicitly requested after showing its terms.
//!
//! The transfer is [`lightcraft_fetch`] (pure-Rust HTTPS with resume, size and SHA-256 checks), on a background thread,
//! so the UI never waits for the network. Only a model listed in [`lightcraft_denoise::known::Download`] can be fetched:
//! the address and hash come from our pinned catalog or the user's model catalog, and no token is sent with it. What arrives
//! matches the model's recorded size and SHA-256 or it is thrown away; it then waits in a staging folder until the model
//! is installed (`denoise.models.install`, or by itself once the user has accepted the model's terms in the download).
//!
//! Failure is a message, never a panic: no network, a stalled or truncated transfer, a full disk, a file that is not the
//! one expected, and a panic inside the thread itself all end as [`State::Failed`].
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use std::time::Duration;
use lightcraft_denoise::known::Download;
/// The folder inside the models folder where a downloaded file waits to be installed. Its name starts with a dot, so it
/// can never be a model's own folder (ids start with a letter or digit).
pub const STAGING: &str = ".downloads";
/// Where a download stands.
#[derive(Clone, Debug, PartialEq)]
pub enum State {
/// Fetching (`bytes` of `total` so far); the file is checked as the last step.
Running {
bytes: u64,
total: u64,
},
/// Fetched and verified (its SHA-256 is `sha256`), waiting in the staging folder at `path` to be installed.
Done {
path: PathBuf,
sha256: String,
},
/// Installed and in use: nothing more to do but tell the user.
Installing,
Installed,
Failed(String),
Cancelled,
}
struct Shared {
state: Mutex<State>,
cancel: AtomicBool,
}
impl Shared {
fn new(total: u64) -> Self {
Shared { state: Mutex::new(State::Running { bytes: 0, total }), cancel: AtomicBool::new(false) }
}
fn set(&self, s: State) {
*self.state.lock().unwrap_or_else(PoisonError::into_inner) = s;
}
fn get(&self) -> State {
self.state.lock().unwrap_or_else(PoisonError::into_inner).clone()
}
}
/// The downloads of one session, by model id.
#[derive(Default)]
pub struct Downloads {
jobs: HashMap<String, Arc<Shared>>,
}
impl Downloads {
/// Start fetching `spec` into `<models dir>/.downloads/`. Different models can download side by side; the same
/// one is started once.
pub fn start(&mut self, spec: Download, models_dir: &Path) -> Result<(), String> {
if self.jobs.get(&spec.id).is_some_and(|j| matches!(j.get(), State::Running { .. })) {
return Err("that model is already downloading".into());
}
let staging = models_dir.join(STAGING);
std::fs::create_dir_all(&staging).map_err(|e| format!("could not create the download folder: {e}"))?;
let shared = Arc::new(Shared::new(spec.size_bytes));
let worker = shared.clone();
let id = spec.id.clone();
std::thread::Builder::new()
.name("denoise-model-download".into())
.spawn(move || {
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run(&spec, &spec.url, &staging, &worker)));
if outcome.is_err() {
worker.set(State::Failed("the download stopped unexpectedly".into()));
}
})
.map_err(|e| format!("could not start the download: {e}"))?;
self.jobs.insert(id, shared);
Ok(())
}
/// Every download this session has started and not discarded, by model id.
pub fn snapshot(&self) -> Vec<(String, State)> {
let mut all: Vec<_> = self.jobs.iter().map(|(id, j)| (id.clone(), j.get())).filter(|(_, s)| *s != State::Cancelled).collect();
all.sort_by(|a, b| a.0.cmp(&b.0));
all
}
pub fn running(&self) -> bool {
self.jobs.values().any(|j| matches!(j.get(), State::Running { .. }))
}
/// Stop a running download, or discard a finished one (its staged file is deleted). `true` if there was one.
pub fn discard(&mut self, id: &str) -> bool {
let Some(job) = self.jobs.get(id) else { return false };
job.cancel.store(true, Ordering::Relaxed);
match job.get() {
// the thread notices, deletes the partial file and marks itself cancelled
State::Running { .. } | State::Installing => {}
State::Done { path, .. } => {
let _ = std::fs::remove_file(path);
self.jobs.remove(id);
}
State::Installed | State::Failed(_) | State::Cancelled => {
self.jobs.remove(id);
}
}
true
}
/// Downloads that have arrived and been checked, ready to install: (model id, staged file, its SHA-256).
pub fn finished(&self) -> Vec<(String, PathBuf, String)> {
let mut out: Vec<_> = self
.jobs
.iter()
.filter_map(|(id, j)| match j.get() {
State::Done { path, sha256 } => Some((id.clone(), path, sha256)),
_ => None,
})
.collect();
out.sort_by(|a, b| a.0.cmp(&b.0));
out
}
/// Record how installing a finished download went (`Installed` or `Failed`).
pub fn set_outcome(&mut self, id: &str, state: State) {
match self.jobs.get(id) {
Some(j) => j.set(state),
None => {
let j = Shared::new(0);
j.set(state);
self.jobs.insert(id.to_string(), Arc::new(j));
}
}
}
}
impl Drop for Downloads {
/// Quitting while a download runs stops it instead of leaving it behind.
fn drop(&mut self) {
for job in self.jobs.values() {
job.cancel.store(true, Ordering::Relaxed);
}
let started = web_time::Instant::now();
while self.running() && started.elapsed() < Duration::from_secs(1) {
std::thread::sleep(Duration::from_millis(10));
}
}
}
/// What went wrong, in words the user can act on.
#[cfg(not(target_arch = "wasm32"))]
fn explain(e: &lightcraft_fetch::DownloadError) -> String {
use lightcraft_fetch::DownloadError;
match e {
DownloadError::AllMirrorsFailed { errors, .. } => {
let detail: String = errors.join("; ").chars().take(300).collect();
format!("could not download the model ({detail}): check the internet connection, or get the file from its page and add it yourself")
}
other => other.to_string(),
}
}
/// Fetch `spec` from its exact URL into `staging`, check it and leave it there.
#[cfg(not(target_arch = "wasm32"))]
fn fetch(spec: &Download, mirror: &str, staging: &Path, shared: &Shared) -> Result<(), lightcraft_fetch::DownloadError> {
let file = lightcraft_fetch::FileSpec { name: &spec.file_name, size: Some(spec.size_bytes), sha256: Some(&spec.sha256), max: spec.size_bytes };
lightcraft_fetch::download_url(&file, mirror, staging, &lightcraft_fetch::Options::default(), &shared.cancel, &mut |p| {
shared.set(State::Running { bytes: p.done.min(spec.size_bytes), total: spec.size_bytes });
})
}
#[cfg(target_arch = "wasm32")]
fn fetch(_: &Download, _: &str, _: &Path, _: &Shared) -> Result<(), String> {
Err("the web build downloads no models".into())
}
/// The whole job: reuse a verified earlier copy, else fetch, check and stage. Always ends in a final [`State`].
fn run(spec: &Download, mirror: &str, staging: &Path, shared: &Shared) {
let target = staging.join(&spec.file_name);
let part = staging.join(format!("{}.part", spec.file_name));
match fetch(spec, mirror, staging, shared) {
Ok(()) => shared.set(State::Done { path: target, sha256: spec.sha256.to_string() }),
#[cfg(not(target_arch = "wasm32"))]
Err(lightcraft_fetch::DownloadError::Cancelled) => {
let _ = std::fs::remove_file(&part);
shared.set(State::Cancelled);
}
#[cfg(not(target_arch = "wasm32"))]
Err(e) => {
shared.set(State::Failed(explain(&e)));
}
#[cfg(target_arch = "wasm32")]
Err(e) => {
let _ = std::fs::remove_file(&part);
shared.set(State::Failed(e));
}
}
}
#[cfg(all(test, not(target_arch = "wasm32")))]
mod tests {
use super::*;
use lightcraft_denoise::hash::sha256_hex;
use std::io::{BufRead, BufReader, Write};
use std::net::TcpListener;
fn scratch(name: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("lc-face-dl-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
fn spec_for(bytes: &[u8]) -> Download {
Download {
id: "test-model".into(),
url: "https://example.invalid/m/test-model.onnx".into(),
file_name: "test-model.onnx".into(),
size_bytes: bytes.len() as u64,
sha256: sha256_hex(bytes),
}
}
/// A local server that answers every request with `status` and `body` (the whole file, no ranges).
fn serve(status: &'static str, body: Vec<u8>) -> String {
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let base = format!("http://{}/m", l.local_addr().unwrap());
std::thread::spawn(move || {
for mut s in l.incoming().flatten() {
let mut r = BufReader::new(s.try_clone().unwrap());
loop {
let mut line = String::new();
if r.read_line(&mut line).unwrap_or(0) == 0 || line.trim().is_empty() {
break;
}
}
let _ = s.write_all(format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).as_bytes());
let _ = s.write_all(&body);
}
});
base
}
#[test]
fn fetches_checks_and_stages_a_file() {
let dir = scratch("ok");
let bytes: Vec<u8> = (0..200_000u32).map(|i| (i % 251) as u8).collect();
let (staging, spec) = (dir.join(STAGING), spec_for(&bytes));
std::fs::create_dir_all(&staging).unwrap();
let mirror = serve("200 OK", bytes.clone());
let shared = Shared::new(spec.size_bytes);
run(&spec, &mirror, &staging, &shared);
let State::Done { path, sha256 } = shared.get() else { panic!("{:?}", shared.get()) };
assert_eq!((std::fs::read(&path).unwrap(), sha256), (bytes, spec.sha256.to_string()));
assert!(!staging.join("test-model.onnx.part").exists());
// asked again with the copy still staged: no fetch needed (nothing listens there)
let again = Shared::new(spec.size_bytes);
run(&spec, "http://127.0.0.1:1/m", &staging, &again);
assert!(matches!(again.get(), State::Done { .. }), "{:?}", again.get());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_file_that_does_not_match_is_thrown_away() {
let dir = scratch("bad");
let staging = dir.join(STAGING);
std::fs::create_dir_all(&staging).unwrap();
// the right size and the wrong content
let spec = spec_for(&[7u8; 5000]);
let shared = Shared::new(spec.size_bytes);
run(&spec, &serve("200 OK", vec![8u8; 5000]), &staging, &shared);
assert!(matches!(shared.get(), State::Failed(_)), "{:?}", shared.get());
// nothing is left behind: not the file, not a partial one
assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn an_unreachable_or_refusing_server_is_a_message() {
let dir = scratch("down");
let staging = dir.join(STAGING);
std::fs::create_dir_all(&staging).unwrap();
let spec = spec_for(b"abc");
for mirror in ["http://127.0.0.1:1/m".to_string(), serve("404 Not Found", b"not found".to_vec())] {
let shared = Shared::new(3);
run(&spec, &mirror, &staging, &shared);
let State::Failed(why) = shared.get() else { panic!("{:?}", shared.get()) };
assert!(why.contains("internet connection"), "{why}");
}
assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_cancelled_download_leaves_nothing() {
let dir = scratch("cancel");
let staging = dir.join(STAGING);
std::fs::create_dir_all(&staging).unwrap();
let spec = spec_for(b"abc");
let shared = Shared::new(3);
shared.cancel.store(true, Ordering::Relaxed);
run(&spec, &serve("200 OK", b"abc".to_vec()), &staging, &shared);
assert_eq!(shared.get(), State::Cancelled);
assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn dropping_the_session_stops_a_running_download() {
let shared = Arc::new(Shared::new(10));
// a stand-in for the download thread: notices the cancel flag the way the downloader does
let worker = shared.clone();
let handle = std::thread::spawn(move || {
while !worker.cancel.load(Ordering::Relaxed) {
std::thread::sleep(Duration::from_millis(5));
}
worker.set(State::Cancelled);
});
let mut d = Downloads::default();
d.jobs.insert("m".into(), shared.clone());
assert!(d.running());
let started = web_time::Instant::now();
drop(d);
assert!(started.elapsed() < Duration::from_millis(900), "quitting must not wait for a transfer to finish");
assert_eq!(shared.get(), State::Cancelled);
handle.join().unwrap();
}
#[test]
fn discarding_a_finished_download_deletes_its_file() {
let dir = scratch("discard");
let file = dir.join("staged.onnx");
std::fs::write(&file, b"x").unwrap();
let mut d = Downloads::default();
let shared = Arc::new(Shared::new(1));
shared.set(State::Done { path: file.clone(), sha256: String::new() });
d.jobs.insert("m".into(), shared);
assert_eq!(d.snapshot().len(), 1);
assert!(!d.running());
assert!(d.discard("m"));
assert!(!file.exists());
assert!(d.snapshot().is_empty());
assert!(!d.discard("m"), "nothing left to discard");
let _ = std::fs::remove_dir_all(&dir);
}
}
+12 -1
View File
@@ -157,7 +157,14 @@ impl Segmenter {
/// Whether this build can compute AI masks at all.
pub const AVAILABLE: bool = cfg!(feature = "sam");
/// Whether the model's files are in place.
pub fn remote_endpoint(&self) -> Option<String> {
#[cfg(feature = "sam")]
return lightcraft_segment::remote::configured();
#[cfg(not(feature = "sam"))]
None
}
/// Whether the local model files are in place.
pub fn installed(&self) -> bool {
#[cfg(feature = "sam")]
if let Some(dir) = &self.dir {
@@ -230,6 +237,10 @@ impl Segmenter {
if !Self::AVAILABLE {
return Err("AI masks are not available in this build".into());
}
#[cfg(feature = "sam")]
if lightcraft_segment::remote::configured().is_some() {
return Ok(PathBuf::new());
}
let dir = self.dir.clone().ok_or("no folder is set for the SAM 3 model")?;
if self.installed() {
return Ok(dir);
+59 -16
View File
@@ -17,7 +17,10 @@ use std::sync::mpsc::{self, RecvTimeoutError};
use std::time::Duration;
use lightcraft_develop::SegMask;
use lightcraft_segment::{Encoded, Sam3};
use lightcraft_segment::{
Encoded, Sam3,
remote::{self, Operation, Request, text_logits},
};
use super::{Input, Prompt, Tag};
use crate::media::RenderJob;
@@ -175,6 +178,7 @@ impl Worker {
struct State {
model: Option<(PathBuf, Sam3)>,
cache: Option<(u64, Encoded)>,
remote: Option<remote::Client>,
}
/// Whether `later` makes `job` pointless (it asks for the same selection again).
@@ -195,7 +199,7 @@ fn run(rx: &mpsc::Receiver<Job>, shared: &Shared, idle: Duration) {
match rx.recv_timeout(idle) {
Ok(j) => queue.push_back(j),
Err(RecvTimeoutError::Timeout) => {
if state.model.is_some() {
if state.model.is_some() || state.remote.is_some() {
state = State::default();
shared.loaded.store(false, Ordering::SeqCst);
log::info!("SAM 3 unloaded after {} minutes without use", idle.as_secs() / 60);
@@ -231,7 +235,7 @@ fn run(rx: &mpsc::Receiver<Job>, shared: &Shared, idle: Duration) {
if result.is_err() && !is_detail {
state.cache = None;
}
shared.loaded.store(state.model.is_some(), Ordering::SeqCst);
shared.loaded.store(state.model.is_some() || state.remote.is_some(), Ordering::SeqCst);
drop(done);
let _ = job.reply.send(Outcome { tag: job.tag, result, superseded: false });
}
@@ -284,29 +288,68 @@ fn encoded<'a>(state: &'a mut State, shared: &Shared, job: &mut Job) -> Result<(
log::info!("SAM 3 encoded {}×{} in {:?}", input.w, input.h, t.elapsed());
state.cache = Some((job.key, enc));
}
let State { model, cache } = state;
let State { model, cache, .. } = state;
match (model.as_mut(), cache.as_mut()) {
(Some((_, m)), Some((_, e))) => Ok((m, e)),
_ => Err("the model is not ready".into()),
}
}
/// Every phrase of `text` ("car, road"), merged (the per-pixel maximum); `None` when nothing
/// matches.
fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> Result<Option<Vec<f32>>, String> {
let mut merged: Option<Vec<f32>> = None;
for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) {
if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? {
merged = Some(match merged {
None => p.logits,
Some(m) => m.iter().zip(&p.logits).map(|(a, b)| a.max(*b)).collect(),
});
}
fn remote_job(state: &mut State, shared: &Shared, job: &mut Job, endpoint: &str) -> Result<Option<SegMask>, String> {
let _busy = Flag::raise(&shared.encoding);
if state.remote.is_none() {
state.remote = Some(remote::Client::connect(endpoint)?);
}
Ok(merged)
let client = state.remote.as_mut().ok_or("remote SAM unavailable")?;
let mut region = None;
let operation = match std::mem::replace(&mut job.kind, Kind::Prepare) {
Kind::Prepare => Operation::Prepare,
Kind::Text(text) => Operation::Text(text),
Kind::Clicks(clicks) => Operation::Clicks(clicks.iter().map(|c| [c.x, c.y, if c.positive { 1.0 } else { 0.0 }]).collect()),
Kind::Detail { prompt, region: r } => {
region = Some(r);
let (rw, rh) = (r[2] - r[0], r[3] - r[1]);
if !(rw > 0.0 && rh > 0.0) {
return Ok(None);
}
match prompt {
Prompt::Text(text) => Operation::Text(text),
Prompt::Clicks(clicks) => {
let inside: Vec<_> = clicks
.iter()
.map(|c| [((c.at.x - r[0]) / rw) as f32, ((c.at.y - r[1]) / rh) as f32, if c.include { 1.0 } else { 0.0 }])
.filter(|p| (0.0..=1.0).contains(&p[0]) && (0.0..=1.0).contains(&p[1]))
.collect();
if !inside.iter().any(|p| p[2] == 1.0) {
return Ok(None);
}
Operation::Clicks(inside)
}
}
}
#[cfg(test)]
Kind::Panic => return Err("test panic request".into()),
};
let input = if region.is_some() || client.key != Some(job.key) { render(job)? } else { Input { rgb: Vec::new(), w: 0, h: 0 } };
let request = Request { width: input.w, height: input.h, temporary: region.is_some(), operation };
let logits = client.request(&request, &input.rgb)?;
if region.is_none() {
client.key = Some(job.key);
}
Ok(logits.map(|l| match region {
Some(r) => SegMask::from_logits_in(lightcraft_segment::MASK_SIDE, &l, r),
None => SegMask::from_logits(lightcraft_segment::MASK_SIDE, &l),
}))
}
fn run_job(state: &mut State, shared: &Shared, job: &mut Job) -> Result<Option<SegMask>, String> {
if let Some(endpoint) = remote::configured() {
let result = remote_job(state, shared, job, &endpoint);
if result.is_err() {
state.remote = None;
}
return result.map_err(|e| format!("Remote SAM 3 failed: {e}. Check the worker and SSH tunnel, then retry."));
}
let side = lightcraft_segment::MASK_SIDE;
match std::mem::replace(&mut job.kind, Kind::Prepare) {
Kind::Prepare => {
+1 -1
View File
@@ -190,7 +190,7 @@ pub fn is_valid(path: &Path) -> bool {
/// Load the proxy at `path`.
pub fn load(path: &Path) -> Result<crate::media::DecodedSource, String> {
let b = std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?;
decode(&b).map(|(image, camera_tone)| crate::media::DecodedSource { image: Arc::new(image), info: None, camera_tone })
decode(&b).map(|(image, camera_tone)| crate::media::DecodedSource { image: Arc::new(image), info: None, camera_tone, denoised: None })
}
#[cfg(test)]
+65 -1
View File
@@ -1,6 +1,6 @@
use serde_json::{Value, json};
use crate::{Session, command_specs};
use crate::{Selection, Session, command_specs};
fn demo() -> Session {
Session::with_demo()
@@ -118,6 +118,29 @@ fn albums_crud() {
s.execute("album.delete", &json!({"id": f})).unwrap();
}
/// Given an album and a folder, when something is created inside each, then only the folder
/// accepts it: the sidebar only lists children of folders, so a child of a plain album would
/// be invisible.
#[test]
fn albums_and_smart_albums_are_created_only_inside_folders() {
let mut s = demo();
let folder = s.execute("album.create", &json!({"name": "Trips", "folder": true})).unwrap()["id"].as_u64().unwrap();
let album = s.execute("album.create", &json!({"name": "Best", "parent": folder})).unwrap()["id"].as_u64().unwrap();
for (cmd, params) in [
("album.create", json!({"name": "Nested", "parent": album})),
("album.create", json!({"name": "Nested", "folder": true, "parent": album})),
("album.createSmart", json!({"name": "Nested", "parent": album})),
("album.create", json!({"name": "Nested", "parent": 987_654})),
] {
assert!(s.execute(cmd, &params).is_err(), "{cmd} {params} must be refused");
}
let n = s.catalog.albums().count();
let sub = s.execute("album.create", &json!({"name": "Sub", "folder": true, "parent": folder})).unwrap()["id"].as_u64().unwrap();
let smart = s.execute("album.createSmart", &json!({"name": "Rated", "parent": sub})).unwrap()["id"].as_u64().unwrap();
assert_eq!(s.catalog.albums().count(), n + 2);
assert_eq!(s.catalog.album(lightcraft_catalog::AlbumId(smart)).unwrap().parent, Some(lightcraft_catalog::AlbumId(sub)));
}
#[test]
fn filters_and_delete_restore() {
let mut s = demo();
@@ -134,6 +157,47 @@ fn filters_and_delete_restore() {
assert_eq!(s.visible().len(), 24);
}
#[test]
fn deleting_photos_selects_a_neighbour_in_the_current_view() {
for params in [json!({}), json!({"key": "fileName", "ascending": false}), json!({"key": "random", "seed": 123})] {
for at_end in [false, true] {
let mut s = demo();
s.execute("library.sort", &params).unwrap();
s.execute("library.filter", &json!({"rating": 2})).unwrap();
let visible = s.visible_cloned();
assert!(visible.len() > 5);
let at = if at_end { visible.len() - 2 } else { visible.len() / 2 };
let deleted = &visible[at..at + 2];
s.execute("library.select", &json!({"ids": deleted, "active": deleted[0]})).unwrap();
s.execute("photo.delete", &json!({})).unwrap();
let expected = if at_end { visible[at - 1] } else { visible[at + 2] };
assert_eq!(s.selection, Selection::single(expected), "sort {params}, end {at_end}");
assert!(deleted.iter().all(|id| s.catalog.photo(*id).unwrap().deleted));
s.execute("edit.undo", &json!({})).unwrap();
assert!(deleted.iter().all(|id| !s.catalog.photo(*id).unwrap().deleted));
s.execute("edit.redo", &json!({})).unwrap();
assert!(deleted.iter().all(|id| s.catalog.photo(*id).unwrap().deleted));
}
}
let mut s = demo();
s.execute("library.selectAll", &json!({})).unwrap();
s.execute("photo.delete", &json!({})).unwrap();
assert!(s.visible().is_empty());
assert_eq!(s.selection, Selection::default());
}
#[test]
fn deleting_explicit_targets_keeps_the_surviving_selection() {
let mut s = demo();
let visible = s.visible_cloned();
s.execute("library.select", &json!({"ids": [visible[8], visible[9]], "active": visible[9]})).unwrap();
let before = s.selection.clone();
s.execute("photo.delete", &json!({"ids": [visible[3]]})).unwrap();
assert_eq!(s.selection, before, "deleting an unselected photo doesn't change selection");
s.execute("photo.delete", &json!({"ids": [visible[9]]})).unwrap();
assert_eq!(s.selection, Selection::single(visible[8]), "partial deletion retains a selected survivor");
}
#[test]
fn masks_crop_and_render() {
let mut s = demo();
+159
View File
@@ -0,0 +1,159 @@
//! Ordering albums by hand: `album.reorder` puts an album (or folder) before a sibling, or last of
//! its kind, in one undo step, also when it changes folder; `album.sort` goes back to by name.
use lightcraft_catalog::AlbumId;
use serde_json::{Value, json};
use crate::Session;
fn make(s: &mut Session, name: &str, parent: Option<u64>, folder: bool) -> u64 {
let mut p = json!({"name": name, "folder": folder});
if let Some(parent) = parent {
p["parent"] = json!(parent);
}
s.execute("album.create", &p).unwrap()["id"].as_u64().unwrap()
}
fn names(s: &Session, parent: Option<u64>) -> Vec<String> {
s.catalog.album_children(parent.map(AlbumId)).iter().map(|a| a.name.clone()).collect()
}
fn reorder(s: &mut Session, id: u64, params: Value) -> Result<Value, String> {
let mut p = params;
p["id"] = json!(id);
s.execute("album.reorder", &p).map_err(|e| e.to_string())
}
/// Given three albums listed by name, when one is placed before another, then the list follows;
/// "no sibling" puts it last (among albums).
#[test]
fn an_album_is_placed_before_a_sibling_or_last() {
let mut s = Session::new();
let (a, b, c) = (make(&mut s, "A", None, false), make(&mut s, "B", None, false), make(&mut s, "C", None, false));
reorder(&mut s, c, json!({"before": a})).unwrap();
assert_eq!(names(&s, None), ["C", "A", "B"]);
assert!(s.catalog.album_children_are_ordered(None));
reorder(&mut s, c, json!({"before": null})).unwrap();
assert_eq!(names(&s, None), ["A", "B", "C"]);
reorder(&mut s, a, json!({})).unwrap();
assert_eq!(names(&s, None), ["B", "C", "A"], "no `before`: last");
assert_eq!(reorder(&mut s, b, json!({"before": b})).unwrap()["changed"], 0, "before itself is where it is");
assert_eq!(names(&s, None), ["B", "C", "A"]);
}
/// Folders stay ahead of albums: a folder is placed among folders, an album among albums.
#[test]
fn folders_and_albums_keep_to_their_own_group() {
let mut s = Session::new();
let (f1, f2) = (make(&mut s, "F1", None, true), make(&mut s, "F2", None, true));
let (a1, a2) = (make(&mut s, "A1", None, false), make(&mut s, "A2", None, false));
reorder(&mut s, f2, json!({"before": f1})).unwrap();
reorder(&mut s, a2, json!({"before": a1})).unwrap();
assert_eq!(names(&s, None), ["F2", "F1", "A2", "A1"]);
reorder(&mut s, f1, json!({"before": a1})).expect_err("a folder is not placed among albums");
reorder(&mut s, a1, json!({"before": f1})).expect_err("an album is not placed among folders");
reorder(&mut s, f2, json!({"before": null})).unwrap();
assert_eq!(names(&s, None), ["F1", "F2", "A2", "A1"], "last of the folders");
}
/// Placing an album in another folder moves it there, and one undo undoes both.
#[test]
fn placing_across_folders_moves_it_and_undoes_in_one_step() {
let mut s = Session::new();
let folder = make(&mut s, "Trips", None, true);
let (x, y) = (make(&mut s, "X", Some(folder), false), make(&mut s, "Y", Some(folder), false));
let loose = make(&mut s, "Loose", None, false);
reorder(&mut s, y, json!({"before": x})).unwrap();
assert_eq!(names(&s, Some(folder)), ["Y", "X"]);
reorder(&mut s, loose, json!({"parent": folder, "before": x})).unwrap();
assert_eq!(names(&s, Some(folder)), ["Y", "Loose", "X"]);
assert_eq!(s.catalog.album(AlbumId(loose)).unwrap().parent, Some(AlbumId(folder)));
s.execute("edit.undo", &json!({})).unwrap();
assert_eq!(names(&s, Some(folder)), ["Y", "X"]);
assert_eq!(s.catalog.album(AlbumId(loose)).unwrap().parent, None);
assert_eq!(s.catalog.album(AlbumId(loose)).unwrap().order, None);
}
/// Undo and redo of placing across folders, of sorting, and of moving a folder that has children.
#[test]
fn placing_and_sorting_undo_and_redo() {
let mut s = Session::new();
let top = make(&mut s, "Top", None, true);
let child = make(&mut s, "Child", Some(top), false);
let (a, b) = (make(&mut s, "A", None, false), make(&mut s, "B", None, false));
reorder(&mut s, b, json!({"before": a})).unwrap();
// a folder with children, placed inside another folder
let other = make(&mut s, "Other", None, true);
reorder(&mut s, top, json!({"parent": other})).unwrap();
assert_eq!(names(&s, Some(other)), ["Top"]);
assert_eq!(names(&s, Some(top)), ["Child"], "its children came along");
s.execute("edit.undo", &json!({})).unwrap();
assert_eq!(s.catalog.album(AlbumId(top)).unwrap().parent, None);
s.execute("edit.redo", &json!({})).unwrap();
assert_eq!(s.catalog.album(AlbumId(top)).unwrap().parent, Some(AlbumId(other)));
// sort, undo, redo
s.execute("album.sort", &json!({})).unwrap();
assert_eq!(names(&s, None), ["Other", "A", "B"]);
s.execute("edit.undo", &json!({})).unwrap();
assert_eq!(names(&s, None), ["Other", "B", "A"], "the hand order is back");
s.execute("edit.redo", &json!({})).unwrap();
assert_eq!(names(&s, None), ["Other", "A", "B"]);
assert_eq!(s.catalog.album(AlbumId(child)).unwrap().parent, Some(AlbumId(top)));
}
/// Bad requests change nothing: a `before` that is not in the target folder, a target that is not
/// a folder or lies inside the album, an unknown album.
#[test]
fn bad_placements_are_refused_and_change_nothing() {
let mut s = Session::new();
let f = make(&mut s, "F", None, true);
let inner = make(&mut s, "Inner", Some(f), true);
let (a, b) = (make(&mut s, "A", None, false), make(&mut s, "B", Some(f), false));
let before = s.catalog.to_snapshot();
reorder(&mut s, a, json!({"before": b})).expect_err("B is in F, not at the top level");
reorder(&mut s, a, json!({"parent": b})).expect_err("a plain album is no folder");
reorder(&mut s, f, json!({"parent": inner})).expect_err("a folder into itself");
reorder(&mut s, 987_654, json!({})).expect_err("no such album");
reorder(&mut s, a, json!({"before": 987_654})).expect_err("no such sibling");
assert_eq!(s.catalog.to_snapshot(), before, "nothing changed");
}
/// `album.sort` drops the hand order of one folder (and only that one): by name again.
#[test]
fn sorting_goes_back_to_by_name_for_one_folder() {
let mut s = Session::new();
let f = make(&mut s, "F", None, true);
let (x, y) = (make(&mut s, "X", Some(f), false), make(&mut s, "Y", Some(f), false));
let (a, b) = (make(&mut s, "A", None, false), make(&mut s, "B", None, false));
reorder(&mut s, y, json!({"before": x})).unwrap();
reorder(&mut s, b, json!({"before": a})).unwrap();
assert_eq!(names(&s, Some(f)), ["Y", "X"]);
assert_eq!(names(&s, None), ["F", "B", "A"]);
let r = s.execute("album.sort", &json!({"parent": f})).unwrap();
assert_eq!(r["changed"], 2);
assert_eq!(names(&s, Some(f)), ["X", "Y"]);
assert_eq!(names(&s, None), ["F", "B", "A"], "the top level keeps its order");
assert!(!s.catalog.album_children_are_ordered(Some(AlbumId(f))));
// nothing to do: no undo step is made
let r = s.execute("album.sort", &json!({"parent": f})).unwrap();
assert_eq!(r["changed"], 0);
s.execute("edit.undo", &json!({})).unwrap();
assert_eq!(names(&s, Some(f)), ["Y", "X"], "the one undo step was the sort");
let r = s.execute("album.sort", &json!({})).unwrap();
assert_eq!(r["changed"], 3, "every child of the top level had a place: F, B and A");
assert_eq!(names(&s, None), ["F", "A", "B"]);
}
/// Moving an album with `album.move` out of an ordered folder leaves the old neighbours' order.
#[test]
fn moving_out_of_an_ordered_folder_keeps_the_others_in_place() {
let mut s = Session::new();
let f = make(&mut s, "F", None, true);
let (x, y, z) = (make(&mut s, "X", Some(f), false), make(&mut s, "Y", Some(f), false), make(&mut s, "Z", Some(f), false));
reorder(&mut s, z, json!({"before": x})).unwrap();
assert_eq!(names(&s, Some(f)), ["Z", "X", "Y"]);
s.execute("album.move", &json!({"id": x, "parent": null})).unwrap();
assert_eq!(names(&s, Some(f)), ["Z", "Y"]);
assert_eq!(names(&s, None), ["F", "X"]);
let _ = y;
}
+759
View File
@@ -0,0 +1,759 @@
//! AI denoise through the engine: installing a model, making a photo's denoised picture in the background and for an
//! export, the Amount slider mixing it in, and the cache. A tiny stand-in model (it halves every sample) takes the
//! place of a real one, which is enough to see what the machinery does with its answer; how well real models denoise is
//! measured separately (`crates/denoise/tests/real_model.rs`).
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain};
use lightcraft_denoise::run::TileRunner;
use serde_json::{Value, json};
use crate::Session;
use crate::denoise::{Loader, Model, PhotoState, RunOn};
use crate::export::{ExportOptions, export_photo};
use crate::tests_xmp::{synthetic_dng_of, temp_dir};
/// A model that makes every output pixel half of its cell's colour: a picture that is plainly not the plain one.
struct Dim {
runs: Arc<AtomicUsize>,
}
impl TileRunner for Dim {
fn run(&self, input: &[f32]) -> Result<Vec<f32>, lightcraft_denoise::Error> {
self.runs.fetch_add(1, Ordering::Relaxed);
let t = ((input.len() / 4) as f64).sqrt() as usize;
let side = 2 * t;
let mut out = vec![0f32; 3 * side * side];
for y in 0..side {
for x in 0..side {
let i = (y / 2) * t + x / 2;
let (r, g, b) = (input[i], (input[t * t + i] + input[2 * t * t + i]) / 2.0, input[3 * t * t + i]);
out[y * side + x] = r * 0.5;
out[side * side + y * side + x] = g * 0.5;
out[2 * side * side + y * side + x] = b * 0.5;
}
}
Ok(out)
}
}
impl Model for Dim {
fn runner(&self, _: RunOn, threads: usize) -> (&dyn TileRunner, usize) {
(self, threads)
}
fn self_test(&self, _: RunOn) -> Result<Value, String> {
Ok(json!({"ok": true}))
}
}
fn dim_loader(runs: &Arc<AtomicUsize>) -> Loader {
let runs = runs.clone();
Arc::new(move |_, _| Ok(Arc::new(Dim { runs: runs.clone() }) as Arc<dyn Model>))
}
fn manifest(id: &str) -> DenoiserManifest {
DenoiserManifest {
id: id.into(),
name: "Stand-in".into(),
version: "1".into(),
licence: Default::default(),
source: None,
sha256: None,
size_bytes: None,
provenance: "made up for a test".into(),
domain: Domain::BayerToRgb,
tile: 64,
overlap: 16,
gain: Gain::None,
}
}
/// The model's `.onnx` (not a real one: the stand-in loader never reads it) with its manifest beside it.
fn model_files(dir: &Path, id: &str) -> PathBuf {
let src = dir.join(format!("src-{id}"));
std::fs::create_dir_all(&src).unwrap();
std::fs::write(src.join("denoise-model.json"), serde_json::to_vec(&manifest(id)).unwrap()).unwrap();
std::fs::write(src.join("model.onnx"), format!("not an onnx file, only a stand-in for {id}")).unwrap();
src.join("model.onnx")
}
#[test]
fn model_installation_keeps_the_frame_pump_responsive_and_reuses_the_loaded_model() {
use std::sync::{Mutex, mpsc};
use std::time::Duration;
let mut x = setup("background-install", false);
let path = model_files(&x.dir, "stand-in");
let (entered_tx, entered_rx) = mpsc::channel();
let (release_tx, release_rx) = mpsc::channel();
let release_rx = Mutex::new(release_rx);
let runs = x.runs.clone();
let loads = Arc::new(AtomicUsize::new(0));
let worker_loads = loads.clone();
x.s.denoise.loader = Arc::new(move |_, _| {
worker_loads.fetch_add(1, Ordering::Relaxed);
entered_tx.send(()).map_err(|e| e.to_string())?;
release_rx.lock().unwrap().recv_timeout(Duration::from_secs(3)).map_err(|e| e.to_string())?;
Ok(Arc::new(Dim { runs: runs.clone() }))
});
let info = x.s.execute("denoise.models.inspect", &json!({"path": path})).unwrap();
assert_eq!(info["model"]["id"], "stand-in");
assert_eq!(loads.load(Ordering::Relaxed), 0, "showing terms never executes the model");
let r = x.s.execute("denoise.models.install", &json!({"path": path, "acknowledged": true, "background": true})).unwrap();
assert_eq!(r["started"], "local-install");
entered_rx.recv_timeout(Duration::from_secs(3)).unwrap();
// The loader is blocked: the frame pump must still return, reporting installation in progress.
let state = x.s.execute("denoise.models.downloads", &json!({})).unwrap();
assert_eq!(state["downloads"][0]["state"], "installing");
assert_eq!(x.s.execute("denoise.pump", &json!({"pace": "pause"})).unwrap()["active"], false);
assert!(x.s.execute("denoise.models.install", &json!({"path": path, "acknowledged": true, "background": true})).is_err());
release_tx.send(()).unwrap();
for _ in 0..600 {
let v = x.s.execute("denoise.models.downloads", &json!({})).unwrap();
if v["downloads"][0]["state"] == "installed" {
break;
}
std::thread::sleep(Duration::from_millis(5));
}
assert_eq!(x.s.execute("denoise.models.downloads", &json!({})).unwrap()["downloads"][0]["state"], "installed");
let a = x.s.denoise.active.as_ref().unwrap();
assert!(a.model.lock().unwrap().is_some(), "the worker's tested model is reused");
assert_eq!(loads.load(Ordering::Relaxed), 1);
drop(x.s);
let _ = std::fs::remove_dir_all(x.dir);
}
#[test]
fn failed_model_replacement_preserves_the_installed_model() {
let mut x = setup("replacement", true);
let installed = x.dir.join("denoise-models/stand-in/model.onnx");
let original = std::fs::read(&installed).unwrap();
let source = model_files(&x.dir, "stand-in");
std::fs::write(&source, "a different broken model").unwrap();
assert!(x.s.execute("denoise.models.install", &json!({"path": source, "acknowledged": true})).is_err());
assert_eq!(std::fs::read(installed).unwrap(), original);
assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["model"], "stand-in");
drop(x.s);
let _ = std::fs::remove_dir_all(x.dir);
}
/// A Bayer DNG that is a smooth ramp, big enough for a 64-cell tile and a second one.
fn ramp_dng(w: usize, h: usize, cfa: &str) -> Vec<u8> {
let data: Vec<u16> = (0..w * h).map(|i| 600 + ((i % w) * 30 + (i / w) * 35) as u16).collect();
synthetic_dng_of(w, h, cfa, data, None, Default::default())
}
struct Setup {
dir: PathBuf,
s: Session,
runs: Arc<AtomicUsize>,
}
/// A session with a library on disk holding one raw photo, and the stand-in model installed and chosen.
fn setup(tag: &str, with_model: bool) -> Setup {
let dir = temp_dir(tag);
let originals = dir.join("originals");
std::fs::create_dir_all(&originals).unwrap();
std::fs::write(originals.join("ramp.dng"), ramp_dng(192, 160, "RGGB")).unwrap();
let runs = Arc::new(AtomicUsize::new(0));
let mut s = Session::new().with_fs();
s.open_library(dir.join("library"), false).unwrap();
s.execute("library.import", &json!({"paths": [originals.join("ramp.dng").to_string_lossy()]})).unwrap();
let id = s.catalog.photos().next().unwrap().id;
s.execute("library.select", &json!({"ids": [id.0]})).unwrap();
s.set_denoise_models_dir(Some(dir.join("denoise-models")));
s.denoise.loader = dim_loader(&runs);
if with_model {
let onnx = model_files(&dir, "stand-in");
s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap();
}
Setup { dir, s, runs }
}
fn photo(s: &Session) -> lightcraft_catalog::PhotoId {
s.catalog.photos().next().unwrap().id
}
/// Pump until `n` photos are ready (the job runs on its own thread).
fn pump_until_ready(s: &mut Session, n: u64) -> Value {
for _ in 0..600 {
let r = s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
if r["ready"].as_u64() >= Some(n) {
return r;
}
std::thread::sleep(std::time::Duration::from_millis(50));
}
panic!("not ready: {}", s.execute("denoise.status", &json!({})).unwrap());
}
fn mean(img: &lightcraft_raster::Rgba8) -> f64 {
let sum: u64 = img.data.iter().map(|p| u64::from(p[0]) + u64::from(p[1]) + u64::from(p[2])).sum();
sum as f64 / (img.data.len() * 3) as f64
}
fn rendered_mean(s: &mut Session, id: lightcraft_catalog::PhotoId) -> f64 {
mean(&s.render_now(id, 160, 160).unwrap().image)
}
fn set_amount(s: &mut Session, amount: f64) {
s.execute("develop.set", &json!({"control": "enhance.denoise", "value": amount})).unwrap();
}
#[test]
fn a_model_is_installed_chosen_and_removed_with_the_terms_accepted() {
let mut x = setup("model", false);
let onnx = model_files(&x.dir, "stand-in");
// the terms come first
let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy()})).unwrap_err().to_string();
assert!(e.contains("licence has not been accepted"), "{e}");
let r = x.s.execute("denoise.models.list", &json!({})).unwrap();
assert_eq!(r["model"], Value::Null);
assert_eq!(r["runtime"], cfg!(feature = "denoise"));
// a file with no manifest is not guessed at
let stray = x.dir.join("stray.onnx");
std::fs::write(&stray, b"who knows").unwrap();
let e = x.s.execute("denoise.models.install", &json!({"path": stray.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();
assert!(e.contains("does not know this file"), "{e}");
// installed, it is the model in use
let r = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap();
assert_eq!((r["installed"]["id"].clone(), r["model"].clone()), (json!("stand-in"), json!("stand-in")), "{r}");
assert!(r["installed"]["accepted"]["sha256"].as_str().is_some_and(|h| h.len() == 64), "the file's hash is recorded: {r}");
assert!(x.s.denoise.active.is_some());
assert_eq!(x.s.execute("denoise.status", &json!({})).unwrap()["model"]["id"], "stand-in");
// it can be checked again, switched off and on, and removed
assert_eq!(x.s.execute("denoise.models.test", &json!({"id": "stand-in"})).unwrap()["ok"], true);
assert_eq!(x.s.execute("denoise.models.select", &json!({"id": null})).unwrap()["model"], Value::Null);
assert!(x.s.denoise.active.is_none());
assert!(x.s.execute("denoise.models.select", &json!({"id": "nothing"})).is_err());
x.s.execute("denoise.models.select", &json!({"id": "stand-in"})).unwrap();
assert!(x.s.denoise.active.is_some());
let r = x.s.execute("denoise.models.remove", &json!({"id": "stand-in"})).unwrap();
assert_eq!(r["model"], Value::Null);
assert!(x.s.denoise.active.is_none());
assert!(!x.dir.join("denoise-models").join("stand-in").exists());
// ids that are not folder names, and folders that are not models, are never touched
for bad in ["../library", "", "a/b", "settings.json"] {
assert!(x.s.execute("denoise.models.remove", &json!({"id": bad})).is_err(), "{bad}");
}
assert!(x.dir.join("library").is_dir());
// a model that fails its self-test is not left installed
x.s.denoise.loader = Arc::new(|_, _| Err("it does not load".into()));
let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();
assert!(e.contains("it does not load"), "{e}");
assert!(!x.dir.join("denoise-models").join("stand-in").exists());
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn a_manifest_that_does_not_validate_or_name_the_file_is_refused() {
let mut x = setup("manifest", false);
let onnx = model_files(&x.dir, "stand-in");
let json_path = onnx.with_file_name("denoise-model.json");
// a tile the model cannot run on
let mut m = manifest("stand-in");
m.tile = 50;
std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap();
let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();
assert!(e.contains("tile"), "{e}");
// a hash that is not the file's
let mut m = manifest("stand-in");
m.sha256 = Some("0".repeat(64));
std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap();
let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();
assert!(e.contains("SHA-256"), "{e}");
assert!(!x.dir.join("denoise-models").join("stand-in").join("model.onnx").exists(), "nothing half-installed is left");
// an id that would leave the folder
let mut m = manifest("../escape");
m.id = "../escape".into();
std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap();
assert!(x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).is_err());
assert!(!x.dir.join("escape").exists());
// garbage instead of a manifest
std::fs::write(&json_path, b"{ not json").unwrap();
assert!(x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).is_err());
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn a_photo_is_denoised_in_the_background_and_the_amount_mixes_it_in() {
let mut x = setup("background", true);
let id = photo(&x.s);
// nothing is made until a photo has an amount
let r = x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
assert_eq!((r["queued"].clone(), r["running"].clone(), r["ready"].clone()), (json!(0), Value::Null, json!(0)), "{r}");
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);
let plain = rendered_mean(&mut x.s, id);
let key_before = x.s.render_job(id, 160, 160, false, true).unwrap().key;
set_amount(&mut x.s, 100.0);
// the photo being edited is picked up by itself, and made on another thread
let r = pump_until_ready(&mut x.s, 1);
assert_eq!(r["queued"], 0, "{r}");
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);
assert!(x.runs.load(Ordering::Relaxed) >= 4, "a 192 × 160 mosaic is four 64-cell tiles: {}", x.runs.load(Ordering::Relaxed));
let products: Vec<_> = std::fs::read_dir(x.dir.join("library").join("denoise")).unwrap().flatten().collect();
assert_eq!(products.len(), 1, "one cached picture, no DNG or anything else in the library");
assert!(products[0].path().extension().is_some_and(|e| e == "lcdn"));
let originals: Vec<_> = std::fs::read_dir(x.dir.join("originals")).unwrap().flatten().collect();
assert_eq!(originals.len(), 1, "the original is untouched and alone");
// the render now comes from the denoised picture: darker at 100, in between at 50, the plain one at 0
let full = rendered_mean(&mut x.s, id);
assert!(full < plain - 3.0, "denoised {full} vs plain {plain}");
set_amount(&mut x.s, 50.0);
let half = rendered_mean(&mut x.s, id);
assert!(half < plain - 1.0 && half > full + 1.0, "{full} < {half} < {plain}");
set_amount(&mut x.s, 0.0);
assert!((rendered_mean(&mut x.s, id) - plain).abs() < 0.01, "amount 0 is the plain picture again");
// …and a render that uses it has another identity than one that does not
assert_ne!(key_before, {
set_amount(&mut x.s, 100.0);
x.s.render_job(id, 160, 160, false, true).unwrap().key
});
// switching the Detail section off switches the denoise off with it
set_amount(&mut x.s, 100.0);
x.s.execute("develop.sectionEnabled", &json!({"section": "detail", "enabled": false})).unwrap();
assert!((rendered_mean(&mut x.s, id) - plain).abs() < 0.01);
assert_eq!(x.s.media.denoise_salt(id, &x.s.catalog.photo(id).unwrap().develop), 0);
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn what_was_made_is_found_again_by_a_new_session_and_a_new_model_starts_over() {
let mut x = setup("again", true);
let id = photo(&x.s);
set_amount(&mut x.s, 100.0);
pump_until_ready(&mut x.s, 1);
let made = x.runs.load(Ordering::Relaxed);
x.s.close_library().unwrap();
drop(x.s);
// reopened: ready without running the model again
let mut s2 = Session::new().with_fs();
s2.set_denoise_models_dir(Some(x.dir.join("denoise-models")));
s2.denoise.loader = dim_loader(&x.runs);
s2.open_library(x.dir.join("library"), false).unwrap();
s2.denoise_pump(crate::denoise::Pace::Full);
assert_eq!(s2.denoise_photo_state(id), PhotoState::Ready);
assert_eq!(x.runs.load(Ordering::Relaxed), made, "nothing was made again");
// another model's pictures are different pictures
let onnx = model_files(&x.dir, "other");
s2.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap();
s2.denoise_pump(crate::denoise::Pace::Pause);
assert!(
matches!(s2.denoise_photo_state(id), PhotoState::Queued { .. }),
"the first model's picture is not this model's: it is to be made: {:?}",
s2.denoise_photo_state(id)
);
// and choosing the first again finds its picture where it was
s2.execute("denoise.models.select", &json!({"id": "stand-in"})).unwrap();
s2.denoise_pump(crate::denoise::Pace::Pause);
assert_eq!(s2.denoise_photo_state(id), PhotoState::Ready);
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn an_export_makes_its_picture_when_the_queue_has_not_and_never_comes_out_without_it() {
let mut x = setup("export", true);
let id = photo(&x.s);
let o = ExportOptions::from_json(&json!({"format": "png", "width": 160, "height": 160}));
let decode = |bytes: &[u8]| {
let d = lightcraft_codecs::decode(bytes, lightcraft_codecs::DecodeOptions::fit(160, 160)).unwrap();
let w = d.to_working();
w.data.iter().map(|p| f64::from(p[0] + p[1] + p[2])).sum::<f64>() / (w.data.len() * 3) as f64
};
let plain = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes);
assert_eq!(x.runs.load(Ordering::Relaxed), 0, "no amount, no denoise");
set_amount(&mut x.s, 100.0);
// no pump: the export makes the picture itself
let denoised = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes);
assert!(x.runs.load(Ordering::Relaxed) >= 4);
assert!(denoised < plain * 0.9, "denoised {denoised} vs plain {plain}");
let products = std::fs::read_dir(x.dir.join("library").join("denoise")).unwrap().flatten().count();
assert_eq!(products, 1);
// a second export finds it
let runs = x.runs.load(Ordering::Relaxed);
let again = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes);
assert_eq!(x.runs.load(Ordering::Relaxed), runs);
assert!((again - denoised).abs() < 1e-6);
// the pump takes in what the export made
x.s.denoise_pump(crate::denoise::Pace::Pause);
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);
// no model: the export says so instead of leaving the denoise out
x.s.execute("denoise.models.select", &json!({"id": null})).unwrap();
let e = export_photo(&mut x.s, id, &o, 1).err().unwrap();
assert!(e.contains("no denoise model is chosen") && e.contains("ramp.dng"), "{e}");
set_amount(&mut x.s, 0.0);
assert!(export_photo(&mut x.s, id, &o, 1).is_ok(), "without an amount nothing is asked of the model");
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn a_model_that_fails_fails_the_export_and_is_not_retried_by_the_queue() {
let mut x = setup("fails", true);
let id = photo(&x.s);
x.s.denoise.loader = Arc::new(|_, _| Err("the model went away".into()));
// Simulate loss of the in-memory model; successful installation now retains its tested instance.
*x.s.denoise.active.as_ref().unwrap().model.lock().unwrap() = None;
set_amount(&mut x.s, 80.0);
let o = ExportOptions::from_json(&json!({"format": "png", "width": 100, "height": 100}));
let e = export_photo(&mut x.s, id, &o, 1).err().unwrap();
assert!(e.contains("ramp.dng") && e.contains("the model went away"), "{e}");
// the background job fails too, says why once and does not try again every frame
for _ in 0..200 {
x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
if matches!(x.s.denoise_photo_state(id), PhotoState::Failed { .. }) {
break;
}
std::thread::sleep(std::time::Duration::from_millis(20));
}
match x.s.denoise_photo_state(id) {
PhotoState::Failed { why, unsupported } => assert!(why.contains("the model went away") && !unsupported, "{why}"),
other => panic!("{other:?}"),
}
let r = x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
assert_eq!((r["queued"].clone(), r["running"].clone()), (json!(0), Value::Null), "{r}");
// asking again by hand does retry
x.s.denoise.loader = dim_loader(&x.runs);
let r = x.s.execute("denoise.queue", &json!({"retry": true})).unwrap();
assert_eq!(r["added"], 1, "{r}");
pump_until_ready(&mut x.s, 1);
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn a_corrupt_cache_with_a_current_header_never_silently_exports_the_plain_photo() {
let mut x = setup("corrupt-export", true);
let id = photo(&x.s);
set_amount(&mut x.s, 100.0);
let options = ExportOptions::from_json(&json!({"format": "png", "width": 160, "height": 160}));
export_photo(&mut x.s, id, &options, 1).unwrap();
x.s.denoise_pump(crate::denoise::Pace::Pause);
let spec = x.s.media.denoise.spec(id).unwrap().clone();
let mut bytes = std::fs::read(&spec.product).unwrap();
for byte in bytes.iter_mut().rev().take(64) {
*byte ^= 0xff;
}
std::fs::write(&spec.product, bytes).unwrap();
assert!(lightcraft_denoise::product::is_current(&spec.product, &spec.key), "header/length still look current");
x.s.media.forget(id);
let error = export_photo(&mut x.s, id, &options, 1).err().unwrap();
assert!(error.contains("AI Denoise picture could not be read"), "{error}");
let _ = std::fs::remove_dir_all(x.dir);
}
#[test]
fn only_raw_photos_on_disk_are_denoised_and_nothing_is_queued_without_a_model() {
let dir = temp_dir("eligible");
let mut demo = Session::with_demo();
demo.set_denoise_models_dir(Some(dir.join("models")));
let id = demo.active().unwrap();
assert_eq!(demo.denoise_photo_state(id), PhotoState::NotApplicable, "a generated photo has no sensor data");
let e = demo.execute("denoise.queue", &json!({})).unwrap_err().to_string();
assert!(e.contains("no denoise model is chosen"), "{e}");
// no models folder at all (the web build): the commands say so, nothing panics
let mut web = Session::with_demo();
for cmd in ["denoise.models.install", "denoise.models.select", "denoise.models.remove", "denoise.settings"] {
assert!(web.execute(cmd, &json!({"path": "x", "acknowledged": true, "id": "x"})).is_err(), "{cmd}");
}
assert_eq!(web.execute("denoise.pump", &json!({})).unwrap()["active"], false);
assert_eq!(web.execute("denoise.status", &json!({})).unwrap()["enabled"], false);
// no library open: nowhere to keep pictures, and the queue says so
let x = setup("eligible2", true);
let mut bare = Session::new().with_fs();
bare.set_denoise_models_dir(Some(x.dir.join("denoise-models")));
assert!(bare.denoise_products_dir().is_none());
let e = bare.execute("denoise.queue", &json!({})).unwrap_err().to_string();
assert!(e.contains("open a library on disk first"), "{e}");
assert_eq!(bare.execute("denoise.pump", &json!({})).unwrap()["queued"], 0);
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn files_that_cannot_be_denoised_fail_with_a_reason_and_never_a_panic() {
let dir = temp_dir("makefails");
let spec = |path: &Path| crate::denoise::JobSpec {
path: path.to_string_lossy().into_owned(),
read: None,
product: dir.join("p.lcdn"),
key: "k".into(),
manifest: manifest("stand-in"),
onnx: dir.join("model.onnx"),
model: Default::default(),
loader: dim_loader(&Arc::new(AtomicUsize::new(0))),
parallel: 2,
run_on: RunOn::Auto,
};
use crate::denoise::{MakeError, make_product};
// not a file
assert!(matches!(make_product(&spec(&dir.join("missing.dng")), None), Err(MakeError::Failed(_))));
// not a raw
let junk = dir.join("junk.dng");
std::fs::write(&junk, b"this is not a raw file at all").unwrap();
assert!(matches!(make_product(&spec(&junk), None), Err(MakeError::Unsupported(_))));
// a raw cut short
let whole = ramp_dng(192, 160, "RGGB");
let cut = dir.join("cut.dng");
std::fs::write(&cut, &whole[..whole.len() / 3]).unwrap();
assert!(make_product(&spec(&cut), None).is_err());
// a model that gives the wrong number of values is an error from the model, not a crash
struct Wrong;
impl TileRunner for Wrong {
fn run(&self, _: &[f32]) -> Result<Vec<f32>, lightcraft_denoise::Error> {
Ok(vec![0.0; 7])
}
}
impl Model for Wrong {
fn runner(&self, _: RunOn, threads: usize) -> (&dyn TileRunner, usize) {
(self, threads)
}
fn self_test(&self, _: RunOn) -> Result<Value, String> {
Err("no".into())
}
}
let good = dir.join("good.dng");
std::fs::write(&good, &whole).unwrap();
let mut s = spec(&good);
s.loader = Arc::new(|_, _| Ok(Arc::new(Wrong) as Arc<dyn Model>));
match make_product(&s, None) {
Err(MakeError::Failed(why)) => assert!(why.contains("7 numbers"), "{why}"),
other => panic!("{other:?}"),
}
assert!(!dir.join("p.lcdn").exists(), "a failed job leaves no product");
// a job told to stop stops before doing anything
let progress = crate::denoise::Progress::default();
progress.cancel_now();
assert_eq!(make_product(&spec(&good), Some(&progress)), Err(MakeError::Cancelled));
// a Bayer layout of the other kinds is denoised too, and a product that exists is not made again
for cfa in ["BGGR", "GRBG", "GBRG"] {
let f = dir.join(format!("{cfa}.dng"));
std::fs::write(&f, ramp_dng(192, 160, cfa)).unwrap();
let mut s = spec(&f);
s.product = dir.join(format!("{cfa}.lcdn"));
assert_eq!(make_product(&s, None), Ok(true), "{cfa}");
assert_eq!(make_product(&s, None), Ok(false), "{cfa} is already there");
}
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn two_threads_making_the_same_picture_make_it_once() {
let dir = temp_dir("twice");
let f = dir.join("a.dng");
std::fs::write(&f, ramp_dng(192, 160, "RGGB")).unwrap();
let runs = Arc::new(AtomicUsize::new(0));
let spec = crate::denoise::JobSpec {
path: f.to_string_lossy().into_owned(),
read: None,
product: dir.join("p.lcdn"),
key: "k".into(),
manifest: manifest("stand-in"),
onnx: dir.join("model.onnx"),
model: Default::default(),
loader: dim_loader(&runs),
parallel: 2,
run_on: RunOn::Auto,
};
let made: Vec<bool> = std::thread::scope(|sc| {
let hs: Vec<_> = (0..4).map(|_| sc.spawn(|| crate::denoise::make_product(&spec, None).unwrap())).collect();
hs.into_iter().map(|h| h.join().unwrap()).collect()
});
assert_eq!(made.iter().filter(|m| **m).count(), 1, "{made:?}");
assert_eq!(runs.load(Ordering::Relaxed), 4, "one photo's four tiles, once");
assert!(lightcraft_denoise::product::is_current(&spec.product, "k"));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn the_cache_is_kept_within_its_limit_and_what_photos_use_goes_last() {
let mut x = setup("evict", true);
let id = photo(&x.s);
set_amount(&mut x.s, 100.0);
pump_until_ready(&mut x.s, 1);
let used = x.dir.join("library").join("denoise");
// two older pictures nobody uses
for (i, name) in ["aaaa", "bbbb"].iter().enumerate() {
let f = used.join(format!("{name}.lcdn"));
std::fs::write(&f, vec![7u8; 1000]).unwrap();
let t = std::time::SystemTime::now() - std::time::Duration::from_secs(1000 - i as u64);
std::fs::File::options().write(true).open(&f).unwrap().set_modified(t).unwrap();
}
let size_of = |p: &Path| std::fs::metadata(p).unwrap().len();
let mine: PathBuf = std::fs::read_dir(&used).unwrap().flatten().map(|e| e.path()).find(|p| p.file_stem().is_some_and(|s| s.len() == 32)).unwrap();
// room for the photo's own picture and one stray: the oldest stray goes, then the other
x.s.denoise_evict_to(size_of(&mine) + 1500, None);
assert!(!used.join("aaaa.lcdn").exists() && used.join("bbbb.lcdn").exists() && mine.exists());
x.s.denoise_evict_to(size_of(&mine) + 10, None);
assert!(!used.join("bbbb.lcdn").exists() && mine.exists());
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);
// too small for even that: the photo's own picture is the last thing to go, and the photo knows
x.s.denoise_evict_to(10, None);
assert!(!mine.exists());
assert_ne!(x.s.denoise_photo_state(id), PhotoState::Ready);
// …and it is made again, because the photo still has its amount
pump_until_ready(&mut x.s, 1);
assert!(mine.exists());
// clearing deletes it all
let r = x.s.execute("denoise.clear", &json!({})).unwrap();
assert_eq!(r["deleted"], 1, "{r}");
assert_eq!(std::fs::read_dir(&used).unwrap().flatten().count(), 0);
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn settings_are_checked_saved_and_a_paused_pump_starts_nothing() {
let mut x = setup("settings", true);
let r = x.s.execute("denoise.settings", &json!({"auto": false, "cacheGb": 5, "threads": 3})).unwrap();
assert_eq!((r["auto"].clone(), r["cacheGb"].clone(), r["threads"].clone()), (json!(false), json!(5), json!(3)), "{r}");
for bad in [
json!({"cacheGb": 0}),
json!({"cacheGb": "big"}),
json!({"threads": 0}),
json!({"threads": 1000}),
json!({"auto": "yes"}),
json!({"runOn": "fastest"}),
json!({"runOn": true}),
] {
assert!(x.s.execute("denoise.settings", &bad).is_err(), "{bad}");
}
assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["cacheGb"], 5, "a refused change changes nothing");
// the faster of the card and the processor unless the user says otherwise, and the interface can read the choice back
assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "auto");
assert_eq!(x.s.execute("denoise.settings", &json!({"runOn": "cpu"})).unwrap()["runOn"], "cpu");
assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "cpu");
let status = x.s.execute("denoise.status", &json!({})).unwrap();
assert_eq!((status["runOn"].clone(), status["device"]["kind"].clone()), (json!("cpu"), json!("none")), "no model is loaded yet: {status}");
// the setting before `runOn` still reads as what it meant
let models = x.s.denoise.models_dir.clone().unwrap();
let mut st = crate::denoise::read_settings(&models);
(st.run_on, st.gpu) = (None, Some(false));
crate::denoise::write_settings(&models, &st).unwrap();
assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "cpu");
// choosing where it runs lets the card be tried again after a set-up that closed LightCraft
let onnx = crate::denoise::installed_models(&models).first().unwrap().onnx.clone();
let marker = onnx.with_file_name(crate::denoise::GPU_SETUP_MARKER);
std::fs::write(&marker, "setting up").unwrap();
assert_eq!(x.s.execute("denoise.settings", &json!({"runOn": "auto"})).unwrap()["runOn"], "auto");
assert!(!marker.exists(), "the crash is forgotten");
assert_eq!(crate::denoise::read_settings(&models).gpu, None, "the old setting is not kept beside the new one");
// with auto off nothing is started by itself, and the photo is made when asked
let id = photo(&x.s);
set_amount(&mut x.s, 100.0);
for _ in 0..3 {
x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
}
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);
x.s.execute("denoise.settings", &json!({"auto": true})).unwrap();
// paused: it is queued but nothing starts
for _ in 0..3 {
let r = x.s.execute("denoise.pump", &json!({"pace": "pause"})).unwrap();
assert_eq!(r["running"], Value::Null);
}
assert!(matches!(x.s.denoise_photo_state(id), PhotoState::Queued { .. }), "{:?}", x.s.denoise_photo_state(id));
// …and cancelling forgets it
assert_eq!(x.s.execute("denoise.cancel", &json!({})).unwrap()["cancelled"], 1);
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);
// when the pace allows, it is made
x.s.execute("denoise.queue", &json!({"scope": "withAmount"})).unwrap();
pump_until_ready(&mut x.s, 1);
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn a_photo_whose_file_changed_has_no_picture_until_it_is_made_again() {
let mut x = setup("changed", true);
let id = photo(&x.s);
set_amount(&mut x.s, 100.0);
pump_until_ready(&mut x.s, 1);
// the photo's content is something else now (a replaced file): its old picture is not its picture
let p = x.s.catalog.photo(id).unwrap().clone();
x.s.commit(
"test",
lightcraft_catalog::Op::SetContent {
id,
width: p.width,
height: p.height,
file_size: p.file_size + 1,
content_hash: Some("ffffffffffffffffffffffffffffffff".into()),
preview_only: None,
},
)
.unwrap();
x.s.denoise_pump(crate::denoise::Pace::Pause);
assert_ne!(x.s.denoise_photo_state(id), PhotoState::Ready);
// the old picture is not used by a render either
assert_eq!(x.s.media.denoise_salt(id, &x.s.catalog.photo(id).unwrap().develop), 0);
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn a_photo_waits_for_its_turn_unless_pictures_are_not_made_on_their_own_or_have_nowhere_to_go() {
let mut x = setup("waits", true);
let id = photo(&x.s);
set_amount(&mut x.s, 80.0);
// automatic off: the open photo is left alone until it is asked for
x.s.execute("denoise.settings", &json!({"auto": false})).unwrap();
assert!(!x.s.denoise_auto());
x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);
assert!(!x.s.denoise_busy());
x.s.execute("denoise.queue", &json!({"ids": [id.0]})).unwrap();
assert!(x.s.denoise_busy(), "queued with a model: the headless runs wait for it");
pump_until_ready(&mut x.s, 1);
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);
assert!(!x.s.denoise_busy());
// a session whose library is not on disk has nowhere to keep it: the photo says so instead of waiting for ever
x.s.denoise.settings.auto = Some(true);
if let Some(l) = x.s.library.as_mut() {
l.on_disk = false;
}
match x.s.denoise_photo_state(id) {
PhotoState::Failed { why, unsupported } => assert!(unsupported && why.contains("library folder"), "{why}"),
other => panic!("{other:?}"),
}
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn the_denoise_switch_validates_arguments_and_accepts_explicit_targets_without_selection() {
let mut x = setup("toggle", false);
let id = photo(&x.s);
for params in [json!({"id": "bad"}), json!({"id": u64::MAX}), json!({"id": id.0, "enabled": "yes"})] {
assert!(x.s.execute("denoise.toggle", &params).is_err());
}
assert!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).is_err());
assert_eq!(x.s.develop_of(id).unwrap().enhance.denoise, 0.0);
let model = model_files(&x.dir, "toggle-model");
x.s.execute("denoise.models.install", &json!({"path": model.to_string_lossy(), "acknowledged": true})).unwrap();
x.s.selection = Default::default();
let undo = x.s.undo.len();
let r = x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap();
assert_eq!(r["amount"], 50.0);
assert_eq!(x.s.undo.len(), undo + 1);
x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).unwrap();
assert_eq!(x.s.develop_of(id).unwrap().denoise_amount(), 0.0);
// the switch is its own setting: flipping it keeps the Amount (so it compares with and without), and an Amount of 0
// does not switch it off
x.s.execute("library.select", &json!({"ids": [id.0]})).unwrap();
x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap();
x.s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 80.0})).unwrap();
let off = x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).unwrap();
assert_eq!(off["amount"], 80.0);
let d = x.s.develop_of(id).unwrap();
assert!(!d.enhance.denoise_enabled() && d.denoise_amount() == 0.0);
assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap()["amount"], 80.0);
assert!((x.s.develop_of(id).unwrap().denoise_amount() - 0.8).abs() < 1e-6);
x.s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 0.0})).unwrap();
assert!(x.s.develop_of(id).unwrap().enhance.denoise_enabled(), "sliding to 0 leaves it on");
assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap()["amount"], 0.0);
// with no `enabled`, the toggle flips what the switch says
assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0})).unwrap()["enabled"], false);
drop(x.s);
let _ = std::fs::remove_dir_all(x.dir);
}
+586
View File
@@ -0,0 +1,586 @@
//! The `faces.*` model commands: list, inspect, install behind the licence, select, remove, the on/off
//! setting, and the refusals (no folder, licence not accepted, unsupported or hostile files and ids).
use std::path::PathBuf;
use serde_json::{Value, json};
use crate::Session;
fn temp(name: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("lc-facemodels-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
fn session(dir: &std::path::Path) -> Session {
let mut s = Session::new();
s.face_models_dir = Some(dir.join("models"));
s
}
fn model_file(dir: &std::path::Path, name: &str, dim: u64) -> String {
let p = dir.join(name);
std::fs::write(&p, lightcraft_faces::synthetic::tiny_embedder_model(dim)).unwrap();
p.to_string_lossy().into_owned()
}
fn find<'a>(list: &'a Value, id: &str) -> &'a Value {
list["models"].as_array().unwrap().iter().find(|m| m["id"] == id).unwrap_or(&Value::Null)
}
#[test]
fn the_list_knows_the_known_models_and_is_honest_about_the_runtime() {
let d = temp("list");
let mut s = session(&d);
let l = s.execute("faces.models.list", &json!({})).unwrap();
assert_eq!(l["enabled"], false);
assert_eq!(l["runtime"], cfg!(not(target_arch = "wasm32")));
assert_eq!(find(&l, "yunet-2023mar")["installed"], false, "nothing ships with LightCraft");
assert_eq!(find(&l, "auraface-v1")["installed"], false);
assert_eq!(find(&l, "auraface-v1")["licence"]["commercial"], "yes");
assert_eq!(find(&l, "sface-2021dec")["licence"]["commercial"], "unknown");
// speed is a ratio to a ResNet-100 model, in words, for models that are not installed yet too; never milliseconds
assert_eq!(find(&l, "sface-2021dec")["speedText"], "6.9× faster than a ResNet-100 model");
assert_eq!(find(&l, "auraface-v1")["speedText"], "Same speed as a ResNet-100 model");
assert_eq!(find(&l, "yunet-2023mar")["speedText"], Value::Null, "nobody timed the detector against it");
// a build with no folder lists too, and cannot install
let mut web = Session::new();
assert!(web.execute("faces.models.list", &json!({})).is_ok());
let f = model_file(&d, "m.onnx", 512);
assert!(web.execute("faces.models.install", &json!({"path": f, "acknowledged": true})).is_err());
assert!(web.execute("faces.enable", &json!({"enabled": true})).is_err());
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn inspect_describes_a_file_and_installs_nothing() {
let d = temp("inspect");
let mut s = session(&d);
let f = model_file(&d, "My Model.onnx", 512);
let r = s.execute("faces.models.inspect", &json!({"path": f})).unwrap();
assert_eq!(r["kind"], "draft");
assert_eq!(r["model"]["licence"]["commercial"], "unknown");
assert_eq!(r["model"]["known"], false);
assert!(r["assumptions"].as_array().unwrap().iter().any(|a| a.as_str().unwrap().contains("127.5")));
assert_eq!(r["alreadyInstalled"], false);
assert!(!d.join("models").exists(), "inspecting writes nothing");
// not a model: a plain answer, not an error
let junk = d.join("junk.onnx");
std::fs::write(&junk, b"this is not a model").unwrap();
let r = s.execute("faces.models.inspect", &json!({"path": junk.to_string_lossy()})).unwrap();
assert_eq!(r["kind"], "unsupported");
assert!(r["reason"].as_str().unwrap().len() > 5);
// missing and empty files and folders are errors
assert!(s.execute("faces.models.inspect", &json!({"path": d.join("nope.onnx").to_string_lossy()})).is_err());
assert!(s.execute("faces.models.inspect", &json!({"path": d.to_string_lossy()})).is_err());
let empty = d.join("empty.onnx");
std::fs::write(&empty, b"").unwrap();
assert!(s.execute("faces.models.inspect", &json!({"path": empty.to_string_lossy()})).is_err());
assert!(s.execute("faces.models.inspect", &json!({})).is_err());
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn install_needs_the_licence_accepted_then_select_enable_and_remove_work() {
let d = temp("flow");
let mut s = session(&d);
let f = model_file(&d, "Mine.onnx", 512);
assert!(s.execute("faces.models.install", &json!({"path": f})).is_err(), "no acknowledgement");
assert!(s.execute("faces.models.install", &json!({"path": f, "acknowledged": false})).is_err());
assert!(s.execute("faces.models.install", &json!({"path": f, "acknowledged": "yes"})).is_err(), "must be the boolean true");
assert!(!d.join("models").exists(), "refused installs leave nothing behind");
let r = s.execute("faces.models.install", &json!({"path": f, "acknowledged": true})).unwrap();
let id = r["installed"]["id"].as_str().unwrap().to_string();
assert!(id.starts_with("custom-mine-"));
let home = d.join("models").join(&id);
assert!(home.join("model.onnx").is_file() && home.join("face-model.json").is_file() && home.join("installed.json").is_file());
assert!(!home.join("model.onnx.part").exists());
assert_eq!(std::fs::read(home.join("model.onnx")).unwrap(), std::fs::read(&f).unwrap());
let rec: Value = serde_json::from_slice(&std::fs::read(home.join("installed.json")).unwrap()).unwrap();
assert_eq!(rec["commercial"], "unknown");
assert_eq!(rec["fileName"], "Mine.onnx");
// listed as installed and custom; installing again is harmless
let l = s.execute("faces.models.list", &json!({})).unwrap();
assert_eq!(find(&l, &id)["installed"], true);
assert_eq!(find(&l, &id)["known"], false);
s.execute("faces.models.install", &json!({"path": f, "acknowledged": true})).unwrap();
assert_eq!(s.execute("faces.models.inspect", &json!({"path": f})).unwrap()["alreadyInstalled"], true);
// choose it; the choice and the on/off setting survive a new session on the same folder
assert!(s.execute("faces.models.select", &json!({"id": "auraface-v1"})).is_err(), "not installed");
assert!(s.execute("faces.models.select", &json!({"id": "yunet-2023mar"})).is_err(), "a detector is not a recogniser");
s.execute("faces.models.select", &json!({"id": id})).unwrap();
assert_eq!(s.execute("faces.enable", &json!({"enabled": true})).unwrap()["enabled"], true);
let mut again = session(&d);
let l = again.execute("faces.models.list", &json!({})).unwrap();
assert_eq!((l["enabled"].clone(), l["embedder"].clone()), (json!(true), json!(id)));
assert_eq!(again.execute("faces.enable", &json!({})).unwrap()["enabled"], false, "toggles when omitted");
// removing it clears the choice
assert_eq!(s.execute("faces.models.remove", &json!({"id": id})).unwrap()["removed"], id);
assert!(!home.exists());
assert_eq!(s.execute("faces.models.list", &json!({})).unwrap()["embedder"], Value::Null);
assert!(s.execute("faces.models.remove", &json!({"id": id})).is_err(), "already gone");
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn hostile_ids_unsupported_models_and_odd_folders_are_handled() {
let d = temp("hostile");
let mut s = session(&d);
let models = d.join("models");
std::fs::create_dir_all(&models).unwrap();
// something that is not a model sits in the folder: it is never listed and never deleted
let precious = models.join("precious");
std::fs::create_dir_all(&precious).unwrap();
std::fs::write(precious.join("keep.txt"), b"mine").unwrap();
for id in ["precious", "..", ".", "", "../x", "a/b", r"a\b", ".hidden", "yunet-2023mar", "UPPER"] {
assert!(s.execute("faces.models.remove", &json!({"id": id})).is_err(), "{id:?}");
}
assert!(precious.join("keep.txt").is_file());
assert!(d.join("models").is_dir());
// a folder whose manifest lies about its id is ignored
let liar = models.join("liar");
std::fs::create_dir_all(&liar).unwrap();
std::fs::write(liar.join("model.onnx"), lightcraft_faces::synthetic::embedder_model(64)).unwrap();
let mut m = lightcraft_faces::known::auraface();
m.id = "someone-else".into();
std::fs::write(liar.join("face-model.json"), serde_json::to_vec(&m).unwrap()).unwrap();
std::fs::write(models.join("settings.json"), b"{ not json").unwrap();
let l = s.execute("faces.models.list", &json!({})).unwrap();
assert!(find(&l, "someone-else").is_null() && find(&l, "liar").is_null());
assert_eq!(l["enabled"], false, "a corrupt settings file is ignored");
// not-a-model and wrong-shaped files cannot be installed
let junk = d.join("junk.onnx");
std::fs::write(&junk, b"nonsense").unwrap();
let e = s.execute("faces.models.install", &json!({"path": junk.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();
assert!(e.contains("cannot be used yet"), "{e}");
let tokens = d.join("tokens.onnx");
{
use lightcraft_faces::synthetic::{len_field, value_info_bytes};
let mut graph = Vec::new();
len_field(11, &value_info_bytes("pixels", 1, &[Err("N"), Ok(3), Ok(224), Ok(224)]), &mut graph);
len_field(12, &value_info_bytes("tokens", 1, &[Err("N"), Ok(257), Ok(384)]), &mut graph);
let mut model = Vec::new();
len_field(7, &graph, &mut model);
std::fs::write(&tokens, model).unwrap();
}
assert!(s.execute("faces.models.install", &json!({"path": tokens.to_string_lossy(), "acknowledged": true})).is_err());
assert!(s.execute("faces.models.select", &json!({"id": 5})).is_err());
assert!(s.execute("faces.models.select", &json!({"id": ""})).is_err());
let _ = std::fs::remove_dir_all(&d);
}
/// A demo session with the real YuNet installed, for the tests that need it: `LC_YUNET_MODEL=<the .onnx file>` (what
/// Settings > Faces downloads; nothing is committed). Without it those tests say so and pass.
fn demo_with_yunet() -> Option<Session> {
let Some(file) = std::env::var_os("LC_YUNET_MODEL") else {
eprintln!("LC_YUNET_MODEL is not set: skipping");
return None;
};
let dir = temp("yunet");
let mut s = Session::with_demo();
s.face_models_dir = Some(dir.join("models"));
let r = s.execute("faces.models.install", &json!({"path": file.to_string_lossy(), "acknowledged": true}));
assert!(r.is_ok(), "{r:?}");
Some(s)
}
#[test]
fn installing_the_detector_does_not_choose_it_as_the_recogniser_or_switch_recognition_on() {
let Some(mut s) = demo_with_yunet() else { return };
let l = s.execute("faces.models.list", &json!({})).unwrap();
assert_eq!(find(&l, "yunet-2023mar")["installed"], true);
assert_eq!(l["enabled"], false, "{l}");
assert!(l["embedder"].is_null(), "{l}");
assert!(s.execute("faces.models.test", &json!({"id": "yunet-2023mar"})).is_err(), "a detector has no recognition self-test");
}
#[test]
fn detect_without_the_model_says_where_to_get_it() {
let d = temp("nodetector");
let mut s = Session::with_demo();
// no folder for models at all, then a folder that has none yet
assert!(s.execute("faces.detect", &json!({})).is_err());
s.face_models_dir = Some(d.join("models"));
let e = s.execute("faces.detect", &json!({})).unwrap_err().to_string();
assert!(e.contains("Settings > Faces"), "{e}");
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn the_detector_and_the_recognisers_can_be_downloaded_and_the_download_is_refused_otherwise() {
let d = temp("dl-refusals");
let mut s = session(&d);
let l = s.execute("faces.models.list", &json!({})).unwrap();
for (id, host) in [("yunet-2023mar", "github.com"), ("sface-2021dec", "github.com"), ("auraface-v1", "huggingface.co")] {
assert_eq!(find(&l, id)["downloadHost"], host, "{id}");
}
// without the user's acceptance, for an id nobody knows, and without a folder: nothing starts
assert!(s.execute("faces.models.download", &json!({"id": "yunet-2023mar"})).is_err());
assert!(s.execute("faces.models.download", &json!({"id": "yunet-2023mar", "acknowledged": false})).is_err());
for id in ["nope", "", "../yunet-2023mar"] {
assert!(s.execute("faces.models.download", &json!({"id": id, "acknowledged": true})).is_err(), "{id}");
}
assert!(Session::new().execute("faces.models.download", &json!({"id": "yunet-2023mar", "acknowledged": true})).is_err());
assert_eq!(s.execute("faces.models.downloads", &json!({})).unwrap()["downloads"], json!([]));
assert!(!d.join("models").join(".downloads").exists(), "a refused download touches nothing");
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn a_download_that_has_arrived_is_installed_by_itself_and_its_staged_file_goes() {
let d = temp("dl-arrived");
let mut s = session(&d);
let staging = d.join("models").join(".downloads");
std::fs::create_dir_all(&staging).unwrap();
let staged = staging.join("model.onnx");
std::fs::write(&staged, lightcraft_faces::synthetic::tiny_embedder_model(512)).unwrap();
let sha = lightcraft_faces::hash::sha256_file(&staged).unwrap();
s.face_downloads.arrived("my-model", staged.clone(), &sha);
let r = s.execute("faces.models.downloads", &json!({})).unwrap();
let row = &r["downloads"][0];
assert_eq!((row["id"].as_str(), row["state"].as_str()), (Some("my-model"), Some("installed")), "{r}");
assert!(!staged.exists(), "the staged file was moved into the model's folder");
let l = s.execute("faces.models.list", &json!({})).unwrap();
assert_eq!(l["models"].as_array().unwrap().iter().filter(|m| m["installed"] == true).count(), 1);
// announced once, cleared on request
assert_eq!(s.execute("faces.models.downloadCancel", &json!({"id": "my-model"})).unwrap()["discarded"], true);
assert_eq!(s.execute("faces.models.downloads", &json!({})).unwrap()["downloads"], json!([]));
assert_eq!(s.execute("faces.models.downloadCancel", &json!({"id": "my-model"})).unwrap()["discarded"], false);
// a file that cannot be installed is thrown away with a reason
std::fs::write(&staged, b"this is not a model").unwrap();
let sha = lightcraft_faces::hash::sha256_file(&staged).unwrap();
s.face_downloads.arrived("bad", staged.clone(), &sha);
let r = s.execute("faces.models.downloads", &json!({})).unwrap();
assert_eq!(r["downloads"][0]["state"], "failed", "{r}");
assert!(!staged.exists());
let _ = std::fs::remove_dir_all(&d);
}
/// `faces.detect` on photos without faces: it reports nothing, and a new run replaces earlier detections but
/// never regions that came from XMP or were named; one undo step restores everything.
#[test]
fn detect_replaces_only_earlier_detections_and_is_one_undo_step() {
use lightcraft_catalog::Op;
use lightcraft_meta::{Region, RegionKind};
let region = |name: Option<&str>, description: Option<&str>, x: f64| Region {
rect: lightcraft_geom::Rect { x0: x, y0: 0.2, x1: x + 0.2, y1: 0.5 },
kind: RegionKind::Face,
name: name.map(str::to_string),
description: description.map(str::to_string),
};
let Some(mut s) = demo_with_yunet() else { return };
let id = s.active().unwrap();
let mut meta = s.catalog.photo(id).unwrap().meta.clone();
meta.regions =
vec![region(Some("Jane Doe"), None, 0.1), region(None, Some("Detected by YuNet 2023mar"), 0.5), region(None, Some("Drawn by hand"), 0.7)];
s.commit("setup", Op::SetMeta { id, meta: Box::new(meta) }).unwrap();
let undo_before = s.undo.len();
// a dry run reports and changes nothing
let r = s.execute("faces.detect", &json!({"apply": false})).unwrap();
assert_eq!(r["applied"], false);
assert_eq!(s.catalog.photo(id).unwrap().meta.regions.len(), 3);
assert_eq!(s.undo.len(), undo_before);
// a real run (the default) drops the earlier detection (the demo has no faces) and keeps the others
let r = s.execute("faces.detect", &json!({})).unwrap();
assert_eq!(r["applied"], true);
assert_eq!(r["detector"], "YuNet 2023mar");
let photo = r["photos"].as_array().unwrap().iter().find(|p| p["id"] == id.0).unwrap();
assert_eq!(photo["faces"].as_array().unwrap().len(), 0, "the procedural demo photos have no faces");
let names: Vec<_> = s.catalog.photo(id).unwrap().meta.regions.iter().map(|r| (r.name.clone(), r.description.clone())).collect();
assert_eq!(names, vec![(Some("Jane Doe".to_string()), None), (None, Some("Drawn by hand".to_string()))]);
assert_eq!(s.undo.len(), undo_before + 1, "one undo step for the whole run");
s.execute("edit.undo", &json!({})).unwrap();
assert_eq!(s.catalog.photo(id).unwrap().meta.regions.len(), 3, "undo brings the detection back");
// odd parameters are tolerated, and nothing is detected without a selection
for p in [json!({"score": 5}), json!({"score": -1}), json!({"nmsIou": 99}), json!({"maxFaces": 0}), json!({"score": "x"})] {
assert!(s.execute("faces.detect", &p).is_ok(), "{p}");
}
let mut empty = Session::new();
assert!(empty.execute("faces.detect", &json!({})).is_err());
}
/// Opt-in, needs the internet: `cargo test -p lightcraft-engine real_download -- --ignored --nocapture`. Downloads YuNet
/// from its pinned address the way the app does, waits for it to be installed, then finds faces with it.
#[test]
#[ignore = "downloads from github.com"]
fn real_download_of_yunet_installs_it_and_detects() {
let d = temp("real-download");
let mut s = Session::with_demo();
s.face_models_dir = Some(d.join("models"));
let started = s.execute("faces.models.download", &json!({"id": "yunet-2023mar", "acknowledged": true})).unwrap();
assert_eq!(started["from"], "github.com");
let t0 = std::time::Instant::now();
loop {
let r = s.execute("faces.models.downloads", &json!({})).unwrap();
let row = r["downloads"][0].clone();
match row["state"].as_str() {
Some("installed") => break,
Some("running") | Some("done") => {}
_ => panic!("{r}"),
}
assert!(t0.elapsed() < std::time::Duration::from_secs(120), "{r}");
std::thread::sleep(std::time::Duration::from_millis(100));
}
println!("downloaded and installed in {:?}", t0.elapsed());
let l = s.execute("faces.models.list", &json!({})).unwrap();
assert_eq!(find(&l, "yunet-2023mar")["installed"], true);
let r = s.execute("faces.detect", &json!({"apply": false})).unwrap();
assert_eq!(r["detector"], "YuNet 2023mar");
// asking again once installed is refused
assert!(s.execute("faces.models.download", &json!({"id": "yunet-2023mar", "acknowledged": true})).is_err());
let _ = std::fs::remove_dir_all(&d);
}
/// Opt-in, needs the internet: the recognisers are big, so this only checks that bytes start to arrive (through the
/// hosts' redirects) and that cancelling leaves nothing behind.
#[test]
#[ignore = "downloads from github.com and huggingface.co"]
fn real_downloads_of_the_recognisers_start_and_can_be_cancelled() {
let d = temp("real-start");
let mut s = Session::with_demo();
s.face_models_dir = Some(d.join("models"));
for id in ["sface-2021dec", "auraface-v1"] {
s.execute("faces.models.download", &json!({"id": id, "acknowledged": true})).unwrap();
let t0 = std::time::Instant::now();
loop {
let r = s.execute("faces.models.downloads", &json!({})).unwrap();
let row = r["downloads"].as_array().unwrap().iter().find(|x| x["id"] == id).unwrap().clone();
if row["state"] == "running" && row["bytes"].as_u64().unwrap_or(0) > 100_000 {
println!("{id}: {} of {} bytes after {:?}", row["bytes"], row["total"], t0.elapsed());
break;
}
assert!(row["state"] == "running", "{r}");
assert!(t0.elapsed() < std::time::Duration::from_secs(60), "{r}");
std::thread::sleep(std::time::Duration::from_millis(50));
}
s.execute("faces.models.downloadCancel", &json!({"id": id})).unwrap();
}
drop(s);
let staging = d.join("models").join(".downloads");
let left: Vec<_> = std::fs::read_dir(&staging).map(|r| r.flatten().map(|e| e.file_name()).collect()).unwrap_or_default();
println!("left in staging: {left:?}");
let _ = std::fs::remove_dir_all(&d);
}
/// With the recognition runtime: installing runs the model's self-test, a model that does not work is refused
/// and leaves nothing behind, and `faces.models.test` runs it again.
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn installing_runs_the_self_test_and_refuses_models_that_do_not_work() {
let d = temp("selftest");
let mut s = session(&d);
// a working model passes; its record keeps the result
let f = model_file(&d, "Works.onnx", 64);
let r = s.execute("faces.models.install", &json!({"path": f, "acknowledged": true})).unwrap();
let id = r["installed"]["id"].as_str().unwrap().to_string();
assert_eq!(r["installed"]["accepted"]["selfTest"]["ok"], true, "{r}");
assert_eq!(r["installed"]["accepted"]["selfTest"]["dimension"], 64);
let t = s.execute("faces.models.test", &json!({"id": id})).unwrap();
assert_eq!(t["ok"], true, "{t}");
assert!(t["result"]["embedMs"].as_f64().unwrap() >= 0.0);
assert!(s.execute("faces.models.test", &json!({"id": "nope"})).is_err());
assert!(s.execute("faces.models.test", &json!({})).is_err());
// a graph with no layers passes the shape check but cannot run: refused, and nothing is left on disk
let broken = d.join("Broken.onnx");
std::fs::write(&broken, lightcraft_faces::synthetic::embedder_model(64)).unwrap();
let e = s.execute("faces.models.install", &json!({"path": broken.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();
assert!(e.contains("could not be loaded") || e.contains("self-test"), "{e}");
let leftovers: Vec<_> = std::fs::read_dir(d.join("models"))
.unwrap()
.flatten()
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|n| n != "settings.json")
.collect();
assert!(leftovers.len() == 1 && leftovers[0].starts_with("custom-works-"), "only the working model remains: {leftovers:?}");
let _ = std::fs::remove_dir_all(&d);
}
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn failed_replacement_keeps_the_installed_model_and_records() {
let d = temp("replacement");
let models = d.join("models");
std::fs::create_dir_all(&models).unwrap();
let good = model_file(&d, "Good.onnx", 64);
let write_catalog = |path: &std::path::Path| {
let catalog = json!({"models": [{
"id": "replacement", "name": "Replacement", "version": "1", "role": "embedder",
"url": "https://models.example.org/weights.onnx",
"sha256": lightcraft_faces::hash::sha256_file(path).unwrap(),
"sizeBytes": std::fs::metadata(path).unwrap().len(),
"licence": {"name": "MIT", "commercial": "yes"},
"output": {"kind": "embedding", "dim": 64}
}]});
std::fs::write(models.join("catalog.json"), catalog.to_string()).unwrap();
};
write_catalog(std::path::Path::new(&good));
let mut s = session(&d);
s.execute("faces.models.install", &json!({"path": good, "acknowledged": true})).unwrap();
let home = models.join("replacement");
let files = ["model.onnx", "face-model.json", "installed.json"];
let before: Vec<_> = files.iter().map(|name| std::fs::read(home.join(name)).unwrap()).collect();
let settings = std::fs::read(models.join("settings.json")).unwrap();
let bad = d.join("Bad.onnx");
std::fs::write(&bad, lightcraft_faces::synthetic::embedder_model(64)).unwrap();
write_catalog(&bad);
let mut s = session(&d);
assert!(s.execute("faces.models.install", &json!({"path": bad, "acknowledged": true})).is_err());
for (name, bytes) in files.iter().zip(before) {
assert_eq!(std::fs::read(home.join(name)).unwrap(), bytes, "{name} was replaced");
}
assert_eq!(std::fs::read(models.join("settings.json")).unwrap(), settings);
assert!(!home.join("model.onnx.part").exists());
let _ = std::fs::remove_dir_all(d);
}
fn settings_of(d: &std::path::Path) -> Value {
serde_json::from_slice(&std::fs::read(d.join("models").join("settings.json")).unwrap()).unwrap()
}
#[test]
fn installing_a_model_chooses_it_and_switches_recognition_on() {
let d = temp("activate");
let mut s = session(&d);
let a = s.execute("faces.models.install", &json!({"path": model_file(&d, "First.onnx", 64), "acknowledged": true})).unwrap();
let a_id = a["installed"]["id"].as_str().unwrap().to_string();
assert_eq!(a["active"], json!({"embedder": a_id, "enabled": true}));
assert_eq!(a["installed"]["selected"], true);
assert_eq!(settings_of(&d), json!({"enabled": true, "embedder": a_id}));
// a second model takes over (the newest is the one the user just asked for); the first stays installed
let b = s.execute("faces.models.install", &json!({"path": model_file(&d, "Second.onnx", 96), "acknowledged": true})).unwrap();
let b_id = b["installed"]["id"].as_str().unwrap().to_string();
assert_ne!(a_id, b_id);
assert_eq!(b["active"]["embedder"], json!(b_id));
let l = s.execute("faces.models.list", &json!({})).unwrap();
assert_eq!((find(&l, &a_id)["installed"].clone(), find(&l, &a_id)["selected"].clone()), (json!(true), json!(false)));
assert_eq!(find(&l, &b_id)["selected"], true);
// switching back is a choice, and a model added with `activate: false` leaves the choice alone
s.execute("faces.models.select", &json!({"id": a_id})).unwrap();
let c = s.execute("faces.models.install", &json!({"path": model_file(&d, "Third.onnx", 128), "acknowledged": true, "activate": false})).unwrap();
let c_id = c["installed"]["id"].as_str().unwrap().to_string();
assert_eq!(c["active"]["embedder"], json!(a_id));
assert_eq!(c["installed"]["selected"], false);
// a switched-off recognition is switched back on by the next install, not by a refused one
s.execute("faces.enable", &json!({"enabled": false})).unwrap();
assert!(s.execute("faces.models.install", &json!({"path": model_file(&d, "Nope.onnx", 64)})).is_err());
assert_eq!(settings_of(&d)["enabled"], false);
s.execute("faces.models.install", &json!({"path": model_file(&d, "Fourth.onnx", 64), "acknowledged": true})).unwrap();
assert_eq!(settings_of(&d)["enabled"], true);
// removing the model in use hands over to another installed one instead of leaving nothing chosen
let in_use = settings_of(&d)["embedder"].as_str().unwrap().to_string();
let r = s.execute("faces.models.remove", &json!({"id": in_use})).unwrap();
let next = r["embedder"].as_str().unwrap().to_string();
assert_ne!(next, in_use);
assert!([&a_id, &b_id, &c_id].contains(&&next) || next.starts_with("custom-"), "{next}");
assert_eq!(settings_of(&d)["embedder"], json!(next));
// removing a model that is not in use changes nothing
let other = [&a_id, &b_id, &c_id].into_iter().find(|i| **i != next && **i != in_use).unwrap().clone();
s.execute("faces.models.remove", &json!({"id": other})).unwrap();
assert_eq!(settings_of(&d)["embedder"], json!(next));
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn a_download_that_arrives_is_installed_and_switched_on_by_itself() {
use crate::face_download::{STAGING, State};
let d = temp("finish");
let mut s = session(&d);
let staging = d.join("models").join(STAGING);
std::fs::create_dir_all(&staging).unwrap();
// what the download thread leaves when it succeeds: a verified file in the staging folder and a finished job
let staged = model_file(&staging, "arrived.onnx", 64);
let sha = lightcraft_faces::hash::sha256_file(std::path::Path::new(&staged)).unwrap();
s.face_downloads.set_outcome("fake-model", State::Done { path: staged.clone().into(), sha256: sha });
let r = s.execute("faces.models.downloads", &json!({})).unwrap();
assert_eq!(r["downloads"][0]["id"], "fake-model");
assert_eq!(r["downloads"][0]["state"], "installed", "{r}");
let st = settings_of(&d);
assert_eq!(st["enabled"], true);
assert!(st["embedder"].as_str().unwrap().starts_with("custom-arrived-"), "{st}");
assert!(!std::path::Path::new(&staged).exists(), "the staged file was moved into place");
// the record stays until it is cleared, and is cleared once
assert_eq!(s.execute("faces.models.downloads", &json!({})).unwrap()["downloads"][0]["state"], "installed");
assert_eq!(s.execute("faces.models.downloadCancel", &json!({"id": "fake-model"})).unwrap()["discarded"], true);
assert_eq!(s.execute("faces.models.downloads", &json!({})).unwrap()["downloads"], json!([]));
// a download that turns out not to be a usable model is thrown away, and the reason is shown
let junk = staging.join("junk.onnx");
std::fs::write(&junk, b"definitely not a model").unwrap();
let sha = lightcraft_faces::hash::sha256_file(&junk).unwrap();
s.face_downloads.set_outcome("fake-junk", State::Done { path: junk.clone(), sha256: sha });
let r = s.execute("faces.models.downloads", &json!({})).unwrap();
assert_eq!(r["downloads"][0]["state"], "failed", "{r}");
assert!(r["downloads"][0]["error"].as_str().unwrap().contains("cannot be used"), "{r}");
assert!(!junk.exists());
assert_eq!(settings_of(&d)["embedder"], st["embedder"], "a failed download leaves the choice alone");
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn the_users_own_catalog_adds_models_with_the_same_download_and_install() {
use crate::face_download::State;
let d = temp("catalog");
let mut s = session(&d);
let models = d.join("models");
std::fs::create_dir_all(&models).unwrap();
// a model file, and a catalog that lists it by its hash (plus an entry that cannot be used)
let file = model_file(&d, "weights.onnx", 64);
let (sha, size) = (lightcraft_faces::hash::sha256_file(std::path::Path::new(&file)).unwrap(), std::fs::metadata(&file).unwrap().len());
let catalog = json!({"models": [
{"id": "my-research-model", "name": "My research model", "version": "1", "role": "embedder", "url": "https://models.example.org/w/weights.onnx",
"sha256": sha, "sizeBytes": size, "licence": {"name": "Research only", "commercial": "no", "notice": "Not for commercial use."},
"provenance": "Some dataset", "output": {"kind": "embedding", "dim": 64}, "thresholds": {"matchCosine": 0.42}},
{"id": "broken", "name": "No hash", "version": "1", "role": "embedder", "url": "https://models.example.org/x.onnx", "output": {"kind": "embedding", "dim": 64}},
]});
std::fs::write(models.join("catalog.json"), catalog.to_string()).unwrap();
// it is listed like a built-in model, with its own terms, a download address, and the problem with the other entry
let l = s.execute("faces.models.list", &json!({})).unwrap();
let m = find(&l, "my-research-model");
assert_eq!(
(m["fromCatalog"].clone(), m["downloadHost"].clone(), m["licence"]["commercial"].clone()),
(json!(true), json!("models.example.org"), json!("no"))
);
assert_eq!(m["installed"], false);
assert_eq!(find(&l, "broken"), &Value::Null);
assert_eq!(l["catalog"]["models"], 1);
assert!(l["catalog"]["errors"][0].as_str().unwrap().contains("broken"), "{}", l["catalog"]);
// nothing is fetched without the terms being accepted
assert!(s.execute("faces.models.download", &json!({"id": "my-research-model"})).is_err());
// a file that matches is recognised as that model (not as a draft), so its terms and settings are used
let seen = s.execute("faces.models.inspect", &json!({"path": file})).unwrap();
assert_eq!((seen["kind"].clone(), seen["model"]["id"].clone()), (json!("known"), json!("my-research-model")));
// a finished download of it installs under the catalog's id, with the catalog's licence record
let staging = models.join(crate::face_download::STAGING);
std::fs::create_dir_all(&staging).unwrap();
let staged = staging.join("weights.onnx");
std::fs::copy(&file, &staged).unwrap();
s.face_downloads.set_outcome("my-research-model", State::Done { path: staged, sha256: sha });
let r = s.execute("faces.models.downloads", &json!({})).unwrap();
assert_eq!(r["downloads"][0]["state"], "installed", "{r}");
let st = settings_of(&d);
assert_eq!((st["embedder"].clone(), st["enabled"].clone()), (json!("my-research-model"), json!(true)));
let l = s.execute("faces.models.list", &json!({})).unwrap();
assert_eq!(
(find(&l, "my-research-model")["installed"].clone(), find(&l, "my-research-model")["accepted"]["commercial"].clone()),
(json!(true), json!("no"))
);
// a broken or hostile catalog never stops the list
for bytes in [&b"{"[..], b"[]", &vec![b'x'; 400_000]] {
std::fs::write(models.join("catalog.json"), bytes).unwrap();
let l = s.execute("faces.models.list", &json!({})).unwrap();
assert!(l["catalog"]["errors"].as_array().is_some_and(|e| !e.is_empty()), "{}", l["catalog"]);
assert!(find(&l, "sface-2021dec")["id"] == "sface-2021dec");
}
let _ = std::fs::remove_dir_all(&d);
}
+489
View File
@@ -0,0 +1,489 @@
//! Recognition: indexing faces, suggesting names, naming, with a tiny working network standing in for a real
//! model (its embedding depends on the picture's colours, which is enough to test the machinery; how well real
//! models recognise people is measured separately, on real photos).
use std::path::PathBuf;
use lightcraft_catalog::{Op, PhotoId};
use lightcraft_geom::Rect;
use lightcraft_meta::{Region, RegionKind};
use serde_json::{Value, json};
use crate::Session;
pub(crate) fn temp(name: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("lc-facerec-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
/// A demo session with a tiny model installed and chosen.
pub(crate) fn setup(d: &std::path::Path, dim: u64) -> (Session, String) {
let mut s = Session::with_demo();
s.face_models_dir = Some(d.join("models"));
let model = d.join(format!("tiny{dim}.onnx"));
std::fs::write(&model, lightcraft_faces::synthetic::tiny_embedder_model(dim)).unwrap();
let r = s.execute("faces.models.install", &json!({"path": model.to_string_lossy(), "acknowledged": true})).unwrap();
let id = r["installed"]["id"].as_str().unwrap().to_string();
s.execute("faces.models.select", &json!({"id": id})).unwrap();
(s, id)
}
/// Installs the real YuNet from `LC_YUNET_MODEL=<the .onnx file>` (what Settings > Faces downloads; nothing is
/// committed) and says whether it did: the parts of a test that need to find faces run only then.
pub(crate) fn install_yunet(s: &mut Session) -> bool {
let Some(file) = std::env::var_os("LC_YUNET_MODEL") else {
eprintln!("LC_YUNET_MODEL is not set: skipping the face-finding part");
return false;
};
let r = s.execute("faces.models.install", &json!({"path": file.to_string_lossy(), "acknowledged": true}));
assert!(r.is_ok(), "{r:?}");
true
}
pub(crate) fn region(x: f64, name: Option<&str>) -> Region {
Region { rect: Rect { x0: x, y0: 0.2, x1: x + 0.25, y1: 0.6 }, kind: RegionKind::Face, name: name.map(str::to_string), description: None }
}
pub(crate) fn set_regions(s: &mut Session, id: PhotoId, regions: Vec<Region>) {
let mut meta = s.catalog.photo(id).unwrap().meta.clone();
meta.regions = regions;
s.commit("setup", Op::SetMeta { id, meta: Box::new(meta) }).unwrap();
}
pub(crate) fn two_photos(s: &Session) -> (PhotoId, PhotoId) {
let mut ids = s.catalog.photos().map(|p| p.id).collect::<Vec<_>>();
ids.sort();
(ids[0], ids[1])
}
#[test]
fn indexing_embeds_each_face_once_and_reports_progress() {
let d = temp("index");
let (mut s, _) = setup(&d, 64);
let (a, b) = two_photos(&s);
set_regions(&mut s, a, vec![region(0.1, Some("Ann")), region(0.5, None)]);
set_regions(&mut s, b, vec![region(0.3, None)]);
// only a report
let r = s.execute("faces.index", &json!({"budgetMs": 0})).unwrap();
assert_eq!((r["embedded"].clone(), r["photosDone"].clone(), r["pendingPhotos"].clone()), (json!(0), json!(0), json!(2)), "{r}");
// a generous budget does it all, the named photo first
let r = s.execute("faces.index", &json!({"budgetMs": 60_000})).unwrap();
assert_eq!((r["embedded"].clone(), r["pendingPhotos"].clone(), r["indexedFaces"].clone()), (json!(3), json!(0), json!(3)), "{r}");
// nothing is embedded twice
let r = s.execute("faces.index", &json!({"budgetMs": 60_000})).unwrap();
assert_eq!((r["embedded"].clone(), r["photosDone"].clone()), (json!(0), json!(0)));
// a small budget still makes progress (at least one photo per call), and `ids` go first
let c = s.catalog.photos().map(|p| p.id).find(|id| *id != a && *id != b).unwrap();
set_regions(&mut s, c, vec![region(0.2, None)]);
set_regions(&mut s, b, vec![region(0.3, None), region(0.6, None)]);
let r = s.execute("faces.index", &json!({"budgetMs": 1, "ids": [c.0]})).unwrap();
assert_eq!(r["photosDone"], 1, "{r}");
assert_eq!(r["pendingPhotos"], 1, "{r}");
assert!(s.faces.index.contains(c.0, &region(0.2, None).rect), "the photo asked for first was done first");
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn suggestions_come_from_the_named_faces_and_are_never_applied() {
let d = temp("suggest");
let (mut s, _) = setup(&d, 64);
let (a, b) = two_photos(&s);
set_regions(&mut s, a, vec![region(0.1, Some("Ann"))]);
set_regions(&mut s, b, vec![region(0.1, None), region(0.5, Some("Pet?")), region(0.7, None)]);
// thresholds that accept anything: Ann is the only person with a named face except "Pet?"
let r = s.execute("faces.suggest", &json!({"ids": [b.0], "threshold": -1.0, "margin": -2.0, "budgetMs": 60_000})).unwrap();
assert_eq!(r["galleryFaces"], 2, "{r}");
let faces = r["photos"][0]["faces"].as_array().unwrap();
assert_eq!(faces.len(), 2, "only the unnamed faces get suggestions: {r}");
assert!(faces.iter().all(|f| f["suggestion"]["name"].is_string()), "{r}");
assert_eq!(faces[0]["index"], 0);
assert_eq!(faces[1]["index"], 2);
assert!(faces[0]["candidates"].as_array().unwrap().len() == 2);
// an impossible threshold suggests nobody but still lists the candidates
let r = s.execute("faces.suggest", &json!({"ids": [b.0], "threshold": 2.0, "budgetMs": 60_000})).unwrap();
assert!(r["photos"][0]["faces"][0]["suggestion"].is_null());
assert!(!r["photos"][0]["faces"][0]["candidates"].as_array().unwrap().is_empty());
// nothing was named by suggesting
assert!(s.catalog.photo(b).unwrap().meta.regions.iter().filter(|r| r.name.is_none()).count() == 2);
// pets and other kinds are not people
let mut meta = s.catalog.photo(b).unwrap().meta.clone();
meta.regions[1].kind = RegionKind::Pet;
s.commit("pet", Op::SetMeta { id: b, meta: Box::new(meta) }).unwrap();
let r = s.execute("faces.suggest", &json!({"ids": [b.0], "threshold": -1.0, "margin": -2.0, "budgetMs": 60_000})).unwrap();
assert_eq!(r["galleryFaces"], 1, "{r}");
// odd parameters are tolerated
for p in [json!({"threshold": "x"}), json!({"margin": null}), json!({"budgetMs": 0}), json!({"ids": []}), json!({"ids": [999999]})] {
assert!(s.execute("faces.suggest", &p).is_ok(), "{p}");
}
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn naming_a_face_is_undoable_and_makes_it_yours() {
let d = temp("name");
let (mut s, _) = setup(&d, 64);
let (a, _) = two_photos(&s);
let mut detected = region(0.1, None);
detected.description = Some("Detected by YuNet 2023mar".into());
set_regions(&mut s, a, vec![detected, region(0.5, None)]);
let undo = s.undo.len();
let r = s.execute("faces.setName", &json!({"id": a.0, "index": 0, "name": " Ann "})).unwrap();
assert_eq!(r["name"], "Ann");
let regions = s.catalog.photo(a).unwrap().meta.regions.clone();
assert_eq!((regions[0].name.as_deref(), regions[0].description.as_deref()), (Some("Ann"), None), "a named detection loses its detected marker");
assert_eq!(s.undo.len(), undo + 1);
// a new detection run now leaves it alone
if install_yunet(&mut s) {
s.execute("faces.detect", &json!({"id": a.0})).unwrap();
assert_eq!(s.catalog.photo(a).unwrap().meta.regions.iter().filter(|r| r.name.as_deref() == Some("Ann")).count(), 1);
}
// clearing, undo
s.execute("faces.setName", &json!({"id": a.0, "index": 0, "name": null})).unwrap();
assert_eq!(s.catalog.photo(a).unwrap().meta.regions[0].name, None);
s.execute("edit.undo", &json!({})).unwrap();
assert_eq!(s.catalog.photo(a).unwrap().meta.regions[0].name.as_deref(), Some("Ann"));
// bad input
for p in [
json!({"id": a.0}),
json!({"id": a.0, "index": 9, "name": "x"}),
json!({"id": a.0, "index": 0, "name": 5}),
json!({"id": a.0, "index": 0, "name": "a\u{7}b"}),
json!({"id": a.0, "index": 0, "name": "x".repeat(201)}),
json!({"id": 999999, "index": 0, "name": "x"}),
] {
assert!(s.execute("faces.setName", &p).is_err(), "{p}");
}
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn embeddings_are_cached_beside_the_library_and_reset_for_another_model() {
let d = temp("persist");
let lib = d.join("lib");
let models = d.join("models");
let install = |s: &mut Session, dim: u64| -> String {
let model = d.join(format!("tiny{dim}.onnx"));
std::fs::write(&model, lightcraft_faces::synthetic::tiny_embedder_model(dim)).unwrap();
let r = s.execute("faces.models.install", &json!({"path": model.to_string_lossy(), "acknowledged": true})).unwrap();
let id = r["installed"]["id"].as_str().unwrap().to_string();
s.execute("faces.models.select", &json!({"id": id})).unwrap();
id
};
let mut s = Session::new().with_fs();
s.face_models_dir = Some(models.clone());
s.open_library(&lib, true).unwrap();
let (a, _) = two_photos(&s);
set_regions(&mut s, a, vec![region(0.1, Some("Ann")), region(0.5, None)]);
install(&mut s, 64);
s.execute("faces.index", &json!({"budgetMs": 60_000})).unwrap();
assert!(lib.join(format!("face-embeddings-{}.bin", s.faces.index.tag.split('@').next().unwrap())).is_file());
s.close_library().unwrap();
drop(s);
// a new session finds the embeddings on disk and has nothing left to do
let mut s2 = Session::new().with_fs();
s2.face_models_dir = Some(models);
s2.open_library(&lib, true).unwrap();
let r = s2.execute("faces.index", &json!({"budgetMs": 60_000})).unwrap();
assert_eq!((r["embedded"].clone(), r["indexedFaces"].clone(), r["pendingPhotos"].clone()), (json!(0), json!(2), json!(0)), "{r}");
// choosing another model starts over: its faces are different numbers, the old ones are not used
install(&mut s2, 32);
let r = s2.execute("faces.index", &json!({"budgetMs": 0})).unwrap();
assert_eq!((r["indexedFaces"].clone(), r["pendingPhotos"].clone()), (json!(0), json!(1)), "{r}");
let r = s2.execute("faces.index", &json!({"budgetMs": 60_000})).unwrap();
assert_eq!(r["embedded"], 2);
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn without_a_chosen_model_recognition_says_so() {
let d = temp("nomodel");
let mut s = Session::with_demo();
assert!(s.execute("faces.index", &json!({})).is_err(), "no models folder");
s.face_models_dir = Some(d.join("models"));
for cmd in ["faces.index", "faces.suggest"] {
let e = s.execute(cmd, &json!({})).unwrap_err().to_string();
assert!(e.contains("no recognition model is chosen"), "{cmd}: {e}");
}
let _: Value = json!(null);
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn the_background_pump_indexes_by_itself_once_recognition_is_on() {
let d = temp("pump");
let (mut s, _) = setup(&d, 64);
let (a, b) = two_photos(&s);
set_regions(&mut s, a, vec![region(0.1, Some("Ann")), region(0.5, None)]);
set_regions(&mut s, b, vec![region(0.3, None)]);
// installing the model switched recognition on; switched off, the pump does nothing, and says so
assert_eq!(s.execute("faces.enable", &json!({"enabled": false})).unwrap()["enabled"], false);
assert_eq!(s.execute("faces.pump", &json!({})).unwrap()["active"], false);
assert_eq!(s.faces.index.len(), 0);
s.execute("faces.enable", &json!({"enabled": true})).unwrap();
// `enabled` is looked up at most once a second, so the first pump may still see "off"
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
let mut last = json!(null);
while std::time::Instant::now() < deadline {
last = s.execute("faces.pump", &json!({})).unwrap();
if last["active"] == true && last["pendingPhotos"] == 0 && last["indexedFaces"] == 3 {
break;
}
std::thread::sleep(std::time::Duration::from_millis(50));
}
assert_eq!((last["active"].clone(), last["pendingPhotos"].clone(), last["indexedFaces"].clone()), (json!(true), json!(0), json!(3)), "{last}");
// a new face is picked up (the catalog changed), and suggestions use what the pump embedded
set_regions(&mut s, b, vec![region(0.3, None), region(0.6, None)]);
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
while std::time::Instant::now() < deadline && s.faces.index.len() < 4 {
s.execute("faces.pump", &json!({})).unwrap();
std::thread::sleep(std::time::Duration::from_millis(50));
}
assert_eq!(s.faces.index.len(), 4);
let r = s.execute("faces.suggest", &json!({"ids": [b.0], "threshold": -1.0, "margin": -2.0, "budgetMs": 0})).unwrap();
assert_eq!(r["photos"][0]["faces"].as_array().unwrap().len(), 2);
assert!(r["photos"][0]["faces"][0]["suggestion"]["name"].is_string(), "{r}");
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn the_background_pump_saves_what_it_embeds_for_the_next_launch() {
let d = temp("pumpsave");
let (lib, models) = (d.join("lib"), d.join("models"));
let mut s = Session::new().with_fs();
s.face_models_dir = Some(models);
s.open_library(&lib, true).unwrap();
let model = d.join("tiny64.onnx");
std::fs::write(&model, lightcraft_faces::synthetic::tiny_embedder_model(64)).unwrap();
// installing switches recognition on: nothing else is asked for
s.execute("faces.models.install", &json!({"path": model.to_string_lossy(), "acknowledged": true})).unwrap();
// without the detector (not bundled) photos without faces are left unsearched
let searching = install_yunet(&mut s);
let (a, b) = two_photos(&s);
set_regions(&mut s, a, vec![region(0.1, Some("Ann")), region(0.5, None)]);
set_regions(&mut s, b, vec![region(0.3, None)]);
let photos = s.catalog.photos().count();
let searched_photos = if searching { photos - 2 } else { 0 };
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(120);
let mut last = json!(null);
while std::time::Instant::now() < deadline {
last = s.execute("faces.pump", &json!({})).unwrap();
if last["pendingPhotos"] == 0 && s.faces.index.len() == 3 && s.faces.scanned.len() == searched_photos {
break;
}
std::thread::sleep(std::time::Duration::from_millis(50));
}
assert_eq!((s.faces.index.len(), s.faces.scanned.len()), (3, searched_photos), "{last}");
let (tag, path) = (s.faces.index.tag.clone(), s.faces.index.path.clone().unwrap());
assert!(path.file_name().unwrap().to_string_lossy().starts_with("face-embeddings-custom-"), "one cache file per model: {}", path.display());
// the first batch is written at once; whatever is left is written when the session ends, so a launch never redoes
// work: neither the embeddings nor the list of photos already searched for faces
s.close_library().unwrap();
drop(s);
let mut again = crate::faces_index::Index::default();
again.reset(&tag, 64, Some(path));
assert_eq!(again.len(), 3);
let mut searched = crate::faces_index::Scanned::default();
searched.reset(Some(lib.join("face-scanned.bin")));
assert_eq!(searched.len(), searched_photos, "every photo without faces is remembered as searched");
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn faces_person_lists_a_persons_faces_and_looks_for_more_only_while_recognition_runs() {
let d = temp("person");
let (mut s, _) = setup(&d, 64);
s.execute("faces.index", &json!({"budgetMs": 0})).unwrap();
let (a, b) = two_photos(&s);
set_regions(&mut s, a, vec![region(0.1, Some("Ann")), region(0.5, Some("Bob"))]);
set_regions(&mut s, b, vec![region(0.1, None), region(0.5, Some("ann"))]);
let unit = |parts: &[(usize, f32)]| {
let mut v = vec![0.0f32; 64];
for (i, x) in parts {
v[*i] = *x;
}
let n = v.iter().map(|x| x * x).sum::<f32>().sqrt();
v.iter().map(|x| x / n).collect::<Vec<f32>>()
};
for (id, x, v) in [(a, 0.1, unit(&[(0, 1.0)])), (a, 0.5, unit(&[(1, 1.0)])), (b, 0.1, unit(&[(0, 0.9), (1, 0.1)])), (b, 0.5, unit(&[(0, 1.0)]))] {
s.faces.index.insert(id.0, &region(x, None).rect, v);
}
// the name is matched without regard to case or surrounding spaces; every face is listed, wherever it is
let r = s.execute("faces.person", &json!({"name": " ANN "})).unwrap();
assert_eq!((r["name"].clone(), r["total"].clone()), (json!("Ann"), json!(2)));
let mut at: Vec<(u64, u64)> =
r["confirmed"].as_array().unwrap().iter().map(|f| (f["photo"].as_u64().unwrap(), f["index"].as_u64().unwrap())).collect();
at.sort();
let mut want = vec![(a.0, 0), (b.0, 1)];
want.sort();
assert_eq!(at, want);
assert!(r["confirmed"].as_array().unwrap().iter().any(|f| f["rect"]["x0"] == 0.1 && f["rect"]["y1"] == 0.6));
// recognition is on (installing the model switched it on): the unnamed face that looks like Ann is offered
assert_eq!(r["ready"], true);
let more = r["more"].as_array().unwrap();
assert_eq!((more.len(), more[0]["photo"].clone(), more[0]["index"].clone()), (1, json!(b.0), json!(0)), "{r}");
assert!(more[0]["score"].as_f64().unwrap() > 0.9);
// confirming it moves it from one list to the other (and is one undo step)
s.execute("faces.setName", &json!({"id": b.0, "index": 0, "name": "Ann"})).unwrap();
let r = s.execute("faces.person", &json!({"name": "Ann"})).unwrap();
assert_eq!((r["total"].clone(), r["more"].as_array().unwrap().len()), (json!(3), 0));
// switched off, the faces are still listed but nothing is searched for
s.execute("faces.enable", &json!({"enabled": false})).unwrap();
let r = s.execute("faces.person", &json!({"name": "Ann"})).unwrap();
assert_eq!((r["total"].clone(), r["ready"].clone(), r["more"].as_array().unwrap().len()), (json!(3), json!(false), 0));
// nobody by that name, and bad input, are plain answers and errors
let r = s.execute("faces.person", &json!({"name": "Nobody"})).unwrap();
assert_eq!((r["total"].clone(), r["confirmed"].as_array().unwrap().len(), r["ready"].clone()), (json!(0), 0, json!(false)));
for bad in [json!({}), json!({"name": ""}), json!({"name": " "}), json!({"name": 7})] {
assert!(s.execute("faces.person", &bad).is_err(), "{bad}");
}
assert!(s.execute("faces.person", &json!({"name": "Ann", "more": u64::MAX})).is_ok());
let _ = std::fs::remove_dir_all(&d);
}
fn unit64(parts: &[(usize, f32)]) -> Vec<f32> {
let mut v = vec![0.0f32; 64];
for (i, x) in parts {
v[*i] = *x;
}
let n = v.iter().map(|x| x * x).sum::<f32>().sqrt();
v.iter().map(|x| x / n).collect()
}
#[test]
fn many_faces_are_named_in_one_undo_step() {
let d = temp("namemany");
let (mut s, _) = setup(&d, 64);
let (a, b) = two_photos(&s);
let detected = |x: f64| Region { description: Some("Detected by YuNet 2023mar".into()), ..region(x, None) };
set_regions(&mut s, a, vec![detected(0.1), region(0.5, None)]);
set_regions(&mut s, b, vec![detected(0.3)]);
let undo = s.undo.len();
let r = s
.execute(
"faces.nameFaces",
&json!({"faces": [{"photo": a.0, "index": 0}, {"photo": a.0, "index": 1}, {"photo": b.0, "index": 0}], "name": " Jane Doe "}),
)
.unwrap();
assert_eq!((r["named"].clone(), r["photos"].clone()), (json!(3), json!(2)));
assert_eq!(s.undo.len(), undo + 1, "one step for the whole group");
for (id, i) in [(a, 0), (a, 1), (b, 0)] {
let reg = &s.catalog.photo(id).unwrap().meta.regions[i];
assert_eq!(reg.name.as_deref(), Some("Jane Doe"));
assert!(reg.description.is_none() || !reg.description.as_deref().unwrap().starts_with("Detected by"), "a named face is the user's now");
}
// undo restores every one of them together
s.execute("edit.undo", &json!({})).unwrap();
assert!(s.catalog.photo(a).unwrap().meta.regions.iter().chain(s.catalog.photo(b).unwrap().meta.regions.iter()).all(|r| r.name.is_none()));
// an empty name clears; faces that do not exist are skipped, and with none left it is an error
s.execute("faces.nameFaces", &json!({"faces": [{"photo": a.0, "index": 0}], "name": "Ann"})).unwrap();
let r = s
.execute(
"faces.nameFaces",
&json!({"faces": [{"photo": a.0, "index": 0}, {"photo": a.0, "index": 99}, {"photo": 987_654, "index": 0}], "name": null}),
)
.unwrap();
assert_eq!(r["named"], 1);
assert!(s.catalog.photo(a).unwrap().meta.regions[0].name.is_none());
for bad in [
json!({"name": "x"}),
json!({"faces": "x", "name": "x"}),
json!({"faces": [{"photo": a.0}], "name": "x"}),
json!({"faces": [{"photo": 987_654, "index": 0}], "name": "x"}),
json!({"faces": [{"photo": a.0, "index": 0}], "name": 7}),
json!({"faces": [{"photo": a.0, "index": 0}], "name": "bad\u{7}name"}),
json!({"faces": vec![json!({"photo": a.0, "index": 0}); 5001], "name": "x"}),
] {
assert!(s.execute("faces.nameFaces", &bad).is_err(), "{bad}");
}
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn unnamed_faces_come_with_look_alikes_together_and_the_name_the_named_ones_suggest() {
let d = temp("unnamed");
let (mut s, _) = setup(&d, 64);
s.execute("faces.index", &json!({"budgetMs": 0})).unwrap();
let (a, b) = two_photos(&s);
let c = s.catalog.photos().map(|p| p.id).find(|id| *id != a && *id != b).unwrap();
set_regions(&mut s, a, vec![region(0.1, Some("Ann")), region(0.5, None), region(0.7, None)]);
set_regions(&mut s, b, vec![region(0.1, None), region(0.5, None)]);
set_regions(&mut s, c, vec![region(0.3, None)]);
// three faces like Ann, two like Bob (unnamed), one unembedded
let put = |s: &mut Session, id: lightcraft_catalog::PhotoId, x: f64, v: Vec<f32>| s.faces.index.insert(id.0, &region(x, None).rect, v);
put(&mut s, a, 0.1, unit64(&[(0, 1.0)])); // Ann, named
put(&mut s, a, 0.5, unit64(&[(1, 1.0)])); // like Bob
put(&mut s, b, 0.1, unit64(&[(0, 0.97), (2, 0.2)])); // like Ann
put(&mut s, b, 0.5, unit64(&[(1, 0.98), (2, 0.2)])); // like Bob
put(&mut s, c, 0.3, unit64(&[(0, 0.95), (3, 0.3)])); // like Ann
// (a's last face, at 0.7, is not embedded yet)
let r = s.execute("faces.unnamed", &json!({})).unwrap();
assert_eq!((r["total"].clone(), r["ordered"].clone(), r["ready"].clone()), (json!(5), json!(true), json!(true)), "{r}");
let faces = r["faces"].as_array().unwrap();
let at = |f: &Value| (f["photo"].as_u64().unwrap(), f["index"].as_u64().unwrap());
let order: Vec<(u64, u64)> = faces.iter().map(at).collect();
// look-alikes are neighbours: the two like Bob are adjacent, and the three like Ann are adjacent; the unembedded one is last
let pos = |x: (u64, u64)| order.iter().position(|o| *o == x).unwrap();
assert_eq!(pos((a.0, 1)).abs_diff(pos((b.0, 1))), 1, "{order:?}");
assert_eq!(order.last(), Some(&(a.0, 2)), "{order:?}");
// a face the named ones recognise carries the name they suggest
let like_ann = faces.iter().find(|f| at(f) == (b.0, 0)).unwrap();
assert_eq!(like_ann["suggestion"]["name"], "Ann", "{like_ann}");
assert!(faces.iter().find(|f| at(f) == (b.0, 1)).unwrap()["suggestion"].is_null(), "Bob is nobody the named faces know");
// each face says which box to show it by; with no scan view it is its own box
for k in ["x0", "y0", "x1", "y1"] {
assert!((like_ann["view"][k].as_f64().unwrap() - like_ann["rect"][k].as_f64().unwrap()).abs() < 1e-3, "{like_ann}");
}
// a limit cuts the list, not the count
let r = s.execute("faces.unnamed", &json!({"limit": 2})).unwrap();
assert_eq!((r["total"].clone(), r["faces"].as_array().unwrap().len()), (json!(5), 2));
// switched off, the faces are still listed, in photo order, without suggestions
s.execute("faces.enable", &json!({"enabled": false})).unwrap();
let r = s.execute("faces.unnamed", &json!({})).unwrap();
assert_eq!((r["total"].clone(), r["ordered"].clone(), r["ready"].clone()), (json!(5), json!(false), json!(false)));
assert!(r["faces"].as_array().unwrap().iter().all(|f| f["suggestion"].is_null()));
for odd in [json!({"limit": 0}), json!({"limit": "x"}), json!({"limit": u64::MAX})] {
assert!(s.execute("faces.unnamed", &odd).is_ok(), "{odd}");
}
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn every_scanned_face_is_shown_by_the_same_kind_of_box() {
let d = temp("view");
let (mut s, _) = setup(&d, 64);
s.execute("faces.index", &json!({"budgetMs": 0})).unwrap();
let (a, _) = two_photos(&s);
// a loosely drawn box (from some other tool) and the detector's tighter box of the same face
let loose = lightcraft_geom::Rect { x0: 0.2, y0: 0.1, x1: 0.8, y1: 0.9 };
let tight = lightcraft_geom::Rect { x0: 0.35, y0: 0.25, x1: 0.65, y1: 0.65 };
let mut meta = s.catalog.photo(a).unwrap().meta.clone();
meta.regions = vec![Region { rect: loose, kind: RegionKind::Face, name: Some("Ann".into()), description: None }];
s.commit("setup", Op::SetMeta { id: a, meta: Box::new(meta) }).unwrap();
// before the scan has looked at it, the face is shown by its own box
assert_eq!(s.face_view(a, loose), loose);
s.faces.index.insert_with_view(a.0, &loose, unit64(&[(0, 1.0)]), &tight);
let shown = s.face_view(a, loose);
assert!((shown.x0 - tight.x0).abs() < 1e-3 && (shown.y1 - tight.y1).abs() < 1e-3, "shown by the detector's box once known: {shown:?}");
// a face nobody has looked at, or in another photo, is shown by its own box
let other = lightcraft_geom::Rect { x0: 0.1, y0: 0.1, x1: 0.2, y1: 0.2 };
assert_eq!(s.face_view(a, other), other);
// the page of the person uses it too
let r = s.execute("faces.person", &json!({"name": "Ann"})).unwrap();
let view = &r["confirmed"][0]["view"];
assert!((view["x0"].as_f64().unwrap() - tight.x0).abs() < 1e-3, "{r}");
// and it survives a restart: the index file keeps it beside the embedding
let (tag, path) = (s.faces.index.tag.clone(), d.join("view-cache.bin"));
let mut written = crate::faces_index::Index::default();
written.reset(&tag, 64, Some(path.clone()));
written.insert_with_view(a.0, &loose, unit64(&[(0, 1.0)]), &tight);
written.save().unwrap();
drop(written);
let mut again = crate::faces_index::Index::default();
again.reset(&tag, 64, Some(path));
let v = again.view(a.0, &loose).expect("the face is remembered");
assert!((v.x0 - tight.x0).abs() < 1e-3 && (v.y0 - tight.y0).abs() < 1e-3, "{v:?}");
let _ = std::fs::remove_dir_all(&d);
}
+7 -2
View File
@@ -335,10 +335,15 @@ fn synthetic_dng(xmp: &str) -> Vec<u8> {
/// A tiny Bayer DNG written by our own DNG writer (optional embedded XMP, camera metadata).
pub(crate) fn synthetic_dng_with(xmp: Option<&str>, metadata: lightcraft_meta::Metadata) -> Vec<u8> {
use lightcraft_raw::*;
let (w, h) = (32usize, 24usize);
let cfa = Cfa::bayer("RGGB").unwrap();
let data: Vec<u16> = (0..w * h).map(|i| 256 + ((i % w) * 300 + (i / w) * 200) as u16).collect();
synthetic_dng_of(w, h, "RGGB", data, xmp, metadata)
}
/// A synthetic Bayer DNG for denoise and cache tests.
pub(crate) fn synthetic_dng_of(w: usize, h: usize, cfa: &str, data: Vec<u16>, xmp: Option<&str>, metadata: lightcraft_meta::Metadata) -> Vec<u8> {
use lightcraft_raw::*;
let cfa = Cfa::bayer(cfa).unwrap();
let raw = RawImage {
format: RawFormat::Dng,
width: w,
Loaded 100 of 248 files, more files were not shown because too many files have changed in this diff. Show more