diff --git a/crates/engine/src/analysis/models.rs b/crates/engine/src/analysis/models.rs index a64e880d1..5d5cf436d 100644 --- a/crates/engine/src/analysis/models.rs +++ b/crates/engine/src/analysis/models.rs @@ -635,6 +635,9 @@ fn download(s: &mut Session, p: &Value, task: Task) -> Result { let k = kind(task); let cmd = format!("{}.model.download", k.prefix); let m = model_p(p, &cmd, task)?; + if m.url.is_empty() { + return Err(bad(&cmd, format!("{} has no app download available: use Install from File with the SHA-256-verified weights", m.name))); + } let dir = s.models_dir().ok_or_else(|| bad(&cmd, "no settings folder to install into"))?; if s.models.status(task).busy.is_some() { return Err(bad(&cmd, "a model is already downloading or loading")); @@ -748,6 +751,19 @@ pub fn specs() -> Vec { mod availability_tests { use super::*; + /// SAM 2 cannot start a network operation; file installation still verifies the published hash. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn sam2_is_install_from_file_only() { + let mut session = Session::default(); + for model in [seg::SAM2_TINY, seg::SAM2_SMALL, seg::SAM2_BASE_PLUS, seg::SAM2_LARGE] { + let error = session.execute("roto.model.download", json!({"id": model.id})).unwrap_err(); + assert!(error.to_string().contains("Install from File")); + assert!(session.models.status(Task::Mask).busy.is_none()); + assert!(install_bytes(Path::new("unused"), &model, b"invalid weights").unwrap_err().contains("SHA-256 differs")); + } + } + #[cfg(not(target_arch = "wasm32"))] #[test] fn stale_model_worker_completions_do_not_poison_the_current_choice() { diff --git a/crates/engine/src/tests/models.rs b/crates/engine/src/tests/models.rs index e0347ffe1..c34d02ed7 100644 --- a/crates/engine/src/tests/models.rs +++ b/crates/engine/src/tests/models.rs @@ -249,19 +249,23 @@ fn reloading_for_a_stroke_does_not_block_or_retry_a_failed_load() { let _ = std::fs::remove_dir_all(d); } -/// With `EFFECTCRAFT_DOWNLOAD_SIZE=sam2.1-hiera-small` (or another registry id): the in-app download (Settings ▸ Roto -/// Brush ▸ Download) fetches the pinned file over the network, verifies it, installs it with its notice and loads it. -/// Skipped without the variable: it needs the network and hundreds of megabytes. +/// With `EFFECTCRAFT_DOWNLOAD_SIZE=mobilesam` (or another downloadable registry id): the in-app +/// download verifies and loads the file. SAM 2 must refuse until a storytold mirror is available. +/// Skipped without the variable: downloadable entries need the network and large files. #[cfg(not(target_arch = "wasm32"))] #[test] fn downloading_a_size_installs_and_loads_it() { let Ok(id) = std::env::var("EFFECTCRAFT_DOWNLOAD_SIZE") else { return }; crate::tests_roto::hold_roto_cache_test_lock(); let (mut s, d) = session("download"); + let info = effectcraft_segment::info(&id).unwrap(); + if info.url.is_empty() { + assert!(s.execute_checked("roto.model.download", json!({"id": id, "wait": true})).unwrap_err().to_string().contains("Install from File")); + return; + } let r = s.execute_checked("roto.model.download", json!({"id": id, "wait": true})).unwrap(); assert_eq!(r["id"], id.as_str(), "{r}"); let dir = s.models_dir().unwrap(); - let info = effectcraft_segment::info(&id).unwrap(); let file = dir.join(info.file_name); assert_eq!(std::fs::metadata(&file).unwrap().len(), info.size, "the whole file arrived"); assert!(dir.join(format!("{}.NOTICE.txt", info.file_name)).exists()); diff --git a/crates/segment/README.md b/crates/segment/README.md index e6e4c6c98..06ffce7ad 100644 --- a/crates/segment/README.md +++ b/crates/segment/README.md @@ -12,7 +12,8 @@ Trained models for Roto Brush and face tracking, each kind behind one swappable depends on one model's layout. The classical face tracker stays built in as the fallback. - **`MODELS`**: the registry. Every entry must be open source under a licence compatible with EffectCraft's MIT OR Apache-2.0 (`ALLOWED_LICENCES`), with its task, authors, homepage, licence - URL, the official weights URL, size and SHA-256. `load` refuses anything else. + URL, an app download URL when available, size and SHA-256. `load` refuses anything else. + SAM 2 is Install from File only until a storytold-controlled mirror is available. - **Weights are never bundled.** They are downloaded on demand (or installed from a file) by the engine (`roto.model.*` / `face.model.*`, Settings ▸ Roto Brush / Face Tracking), verified against the registry's SHA-256 (`sha256`, dependency-free) and kept with a notice naming their diff --git a/crates/segment/src/lib.rs b/crates/segment/src/lib.rs index 04894cb2b..0d9cc3134 100644 --- a/crates/segment/src/lib.rs +++ b/crates/segment/src/lib.rs @@ -141,7 +141,7 @@ pub struct ModelInfo { pub licence_url: &'static str, /// The project's page. pub homepage: &'static str, - /// The official weights file. + /// App download URL; empty when only Install from File is available. pub url: &'static str, pub file_name: &'static str, pub size: u64, @@ -168,8 +168,8 @@ pub const MOBILE_SAM: ModelInfo = ModelInfo { }; /// SAM 2.1 Tiny (Meta AI, Apache-2.0): a video object tracker with streaming memory. The weights are -/// the Hugging Face `transformers` checkpoint, pinned to a commit so the SHA-256 stays valid; they -/// are loaded by `effectcraft-sam2` (native only), not by [`load`]. +/// the Hugging Face `transformers` checkpoint, identified by SHA-256. Until a storytold mirror +/// is available, install it from a file. `effectcraft-sam2` (native only) loads it, not [`load`]. pub const SAM2_TINY: ModelInfo = ModelInfo { id: "sam2.1-hiera-tiny", task: Task::Mask, @@ -179,7 +179,7 @@ pub const SAM2_TINY: ModelInfo = ModelInfo { licence: "Apache-2.0", licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE", homepage: "https://github.com/facebookresearch/sam2", - url: "https://huggingface.co/facebook/sam2.1-hiera-tiny/resolve/de431c4043854a71d8101e17995dfe596bf101a5/model.safetensors", + url: "", file_name: "sam2.1-hiera-tiny.safetensors", size: 155_908_064, sha256: "48c14467e5cf9e51870511feb72c89688e82dd74523142c0538b663e193ac2a7", @@ -195,7 +195,7 @@ pub const SAM2_SMALL: ModelInfo = ModelInfo { licence: "Apache-2.0", licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE", homepage: "https://github.com/facebookresearch/sam2", - url: "https://huggingface.co/facebook/sam2.1-hiera-small/resolve/ee5bba1d82bb8749febdf90f45e84b687142ba03/model.safetensors", + url: "", file_name: "sam2.1-hiera-small.safetensors", size: 184_305_280, sha256: "0a4067b11ce1e23d5229203f11c718a823060d15a4b23fa2372a7d4b77cbbc60", @@ -211,7 +211,7 @@ pub const SAM2_BASE_PLUS: ModelInfo = ModelInfo { licence: "Apache-2.0", licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE", homepage: "https://github.com/facebookresearch/sam2", - url: "https://huggingface.co/facebook/sam2.1-hiera-base-plus/resolve/b7320756a13354e7530a63935656d35b2f91a290/model.safetensors", + url: "", file_name: "sam2.1-hiera-base-plus.safetensors", size: 323_476_296, sha256: "2012733a0de5d03efd1bba550a2847c4551be9ef2e0d497c83074df66189f780", @@ -227,7 +227,7 @@ pub const SAM2_LARGE: ModelInfo = ModelInfo { licence: "Apache-2.0", licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE", homepage: "https://github.com/facebookresearch/sam2", - url: "https://huggingface.co/facebook/sam2.1-hiera-large/resolve/665f8e2ad61cf5f53d65644ff27c8ee525124610/model.safetensors", + url: "", file_name: "sam2.1-hiera-large.safetensors", size: 897_897_416, sha256: "dc407dce21301fd94abb395c5099b4f2c455fdc8a8f261ac3d0ea6d4cd197230", @@ -314,9 +314,10 @@ fn loader(id: &str) -> Option { /// The notice kept next to installed weights: what they are, who made them, their licence. pub fn notice(m: &ModelInfo) -> String { + let source = if m.url.is_empty() { m.homepage } else { m.url }; format!( - "{}\n\nAuthors: {}\nLicence: {} (full text: https://spdx.org/licenses/{}.html; as stated by the authors: {})\nSource: {}\nProject: {}\n\nDownloaded unmodified from the source above. EffectCraft runs it with its own implementation\nand is not affiliated with or endorsed by its authors.\n", - m.name, m.authors, m.licence, m.licence, m.licence_url, m.url, m.homepage + "{}\n\nAuthors: {}\nLicence: {} (full text: https://spdx.org/licenses/{}.html; as stated by the authors: {})\nSource: {}\nProject: {}\n\nInstalled unmodified. EffectCraft runs it with its own implementation\nand is not affiliated with or endorsed by its authors.\n", + m.name, m.authors, m.licence, m.licence, m.licence_url, source, m.homepage ) } @@ -418,14 +419,19 @@ mod tests { for m in MODELS { assert!(ALLOWED_LICENCES.contains(&m.licence), "{}", m.id); assert_eq!(m.sha256.len(), 64); - assert!(m.url.starts_with("https://") && m.licence_url.starts_with("https://") && m.homepage.starts_with("https://")); + assert!(m.url.is_empty() || m.url.starts_with("https://")); + assert!(m.licence_url.starts_with("https://") && m.homepage.starts_with("https://")); assert!(!m.authors.is_empty() && notice(m).contains(m.authors)); } assert_eq!( models(Task::Mask).map(|m| m.id).collect::>(), ["mobilesam", "sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"] ); - assert!(SAM2_TINY.url.contains("/resolve/") && SAM2_TINY.file_name.ends_with(".safetensors"), "pinned to a commit, not a branch"); + for model in [SAM2_TINY, SAM2_SMALL, SAM2_BASE_PLUS, SAM2_LARGE] { + assert!(model.url.is_empty(), "SAM 2 has no app download until a storytold mirror is available"); + assert!(model.file_name.ends_with(".safetensors")); + assert!(notice(&model).contains(&format!("Source: {}", model.homepage))); + } assert_eq!(models(Task::Face).map(|m| m.id).collect::>(), ["mediapipe-face"]); assert!(load("mediapipe-face", b"nope").err().is_some_and(|e| e.contains("does not match"))); // A file that isn't the published one is refused before anything is parsed. diff --git a/crates/ui-egui/src/panels/settings.rs b/crates/ui-egui/src/panels/settings.rs index 8b4133fbc..5fdbf1c50 100644 --- a/crates/ui-egui/src/panels/settings.rs +++ b/crates/ui-egui/src/panels/settings.rs @@ -513,7 +513,7 @@ fn models_ui(app: &mut EffectcraftApp, ui: &mut egui::Ui, t: &Tokens, acts: &mut return; } ui.horizontal(|ui| { - if !installed && !web { + if !installed && !web && m["url"].as_str().is_some_and(|url| !url.is_empty()) { let r = ui.add_enabled(busy.is_none(), egui::Button::new(tr("Download"))); reg(app, &format!("settings.{prefix}.download.{id}"), &r, "Download"); if r.clicked() { @@ -924,3 +924,27 @@ pub fn recovery(app: &mut EffectcraftApp, ctx: &egui::Context, t: &Tokens) { app.ui.status = e; } } + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use super::*; + + /// Install-only models must not offer a download that the command will reject. + #[test] + fn sam2_settings_offer_file_installation_without_downloads() { + let mut app = EffectcraftApp::new(effectcraft_engine::Session::default()); + let ctx = egui::Context::default(); + let tokens = app.tokens; + let mut actions = vec![]; + let mut output = ctx + .run_ui(egui::RawInput { screen_rect: Some(egui::Rect::from_min_size(egui::Pos2::ZERO, vec2(900.0, 2200.0))), ..Default::default() }, |ui| { + models_ui(&mut app, ui, &tokens, &mut actions, Task::Mask) + }); + output.textures_delta.clear(); + for id in ["sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"] { + assert!(app.auto.find(&format!("settings.roto.download.{id}")).is_none()); + assert!(app.auto.find(&format!("settings.roto.install.{id}")).is_some()); + } + assert!(actions.is_empty()); + } +} diff --git a/docs/agents.md b/docs/agents.md index ef88acccb..1a5035f8f 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -504,8 +504,9 @@ reference. Matte settings are ordinary properties (`effects/#1/rotoBrushMatte/se Trained model (Roto Brush 2.0 / 3.0): - `roto.models` lists the built-in engine and every registered model, with its authors, licence, size, URL, SHA-256 and whether it is installed, selected and active. -- `roto.model.download {id}` fetches and verifies the official weights in the background; - `roto.model.install {path, id?}` installs a file you already have. +- `roto.model.download {id}` fetches and verifies weights when the registry provides an app download URL; + `roto.model.install {path, id?}` installs a file you already have. SAM 2 is Install from File only until + a storytold-controlled mirror is available; its size and SHA-256 are verified before installation. - `roto.model.select {id: classical|mobilesam|sam2.1-hiera-tiny|sam2.1-hiera-small|sam2.1-hiera-base-plus|sam2.1-hiera-large}` chooses which one Roto Brush uses (SAM 2 also tracks the object through the clip, see [roto-brush-sam2.md](roto-brush-sam2.md); `roto.status` then reports `source` and per-frame `tracker.frames` with `confidence` and `occluded`). Loading runs in the background; `"wait": true` (also on download and install) returns once it is in use. - `roto.propagate` never falls back to the classic engine silently: a chosen model that is not installed, or failed diff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.md index f2a2cb20d..5b8038552 100644 --- a/docs/roto-brush-sam2.md +++ b/docs/roto-brush-sam2.md @@ -4,10 +4,10 @@ Roto Brush can follow an object through a clip with Meta's **SAM 2.1 Tiny** (Hie Apache-2.0). It is optional: nothing changes until you install the model and choose it. ## Using it -1. Settings ▸ Roto Brush ▸ SAM 2.1 Tiny ▸ **Download** (downloads `model.safetensors`, 155.9 MB, from the pinned - Hugging Face commit; the SHA-256 is checked and the licence notice is kept next to the file). Or install from a - file you downloaded. Agents: `roto.model.download {id: "sam2.1-hiera-tiny", wait: true}`, or - `roto.model.install {id: "sam2.1-hiera-tiny", path: "model.safetensors"}` for a file you already have. +1. Settings ▸ Roto Brush ▸ SAM 2.1 Tiny ▸ **Install from File…** with the published `model.safetensors` + (155.9 MB). The file's size and SHA-256 are checked and the licence notice is kept next to it. + Agents: `roto.model.install {id: "sam2.1-hiera-tiny", path: "model.safetensors"}`. + SAM 2 has no in-app download until the weights are mirrored under storytold control. 2. Choose it (`roto.model.select {id: "sam2.1-hiera-tiny"}`). 3. Paint strokes on the base frame as usual, then **Propagate**. One tracking pass follows the object through the whole span in both directions, using SAM 2's streaming memory, so it keeps tracking through occlusion and when the @@ -59,7 +59,7 @@ instances can see different pixels. Objects inside one instance always see the s Four checkpoints are in the registry (Settings ▸ Roto Brush); the size is detected from the file, and each is pinned to a Hugging Face commit with its SHA-256. All run the same tracker; a bigger backbone costs time and memory. -| Model | Download | GPU encode (RX 7900 GRE) | Agreement with PyTorch | +| Model | File size | GPU encode (RX 7900 GRE) | Agreement with PyTorch | |---|---|---|---| | SAM 2.1 Tiny (`sam2.1-hiera-tiny`) | 155.9 MB | 0.24 s | encoder 6.8e-6 | | SAM 2.1 Small (`sam2.1-hiera-small`) | 184.3 MB | 0.20 s | encoder 1.0e-5 |