mirror of
https://github.com/storytold/effectcraft.git
synced 2026-10-31 07:38:24 +00:00
#1539: keep SAM2 weights install-only until mirrored
This commit is contained in:
1 parent
0fba3429d9
commit
33f545cf64
7 files changed
+76
-24
No files matched your search
@@ -635,6 +635,9 @@ fn download(s: &mut Session, p: &Value, task: Task) -> Result<Value> {
|
||||
let k = kind(task);
|
||||
let cmd = format!("{}.model.download", k.prefix);
|
||||
let m = model_p(p, &cmd, task)?;
|
||||
if m.url.is_empty() {
|
||||
return Err(bad(&cmd, format!("{} has no app download available: use Install from File with the SHA-256-verified weights", m.name)));
|
||||
}
|
||||
let dir = s.models_dir().ok_or_else(|| bad(&cmd, "no settings folder to install into"))?;
|
||||
if s.models.status(task).busy.is_some() {
|
||||
return Err(bad(&cmd, "a model is already downloading or loading"));
|
||||
@@ -748,6 +751,19 @@ pub fn specs() -> Vec<CommandSpec> {
|
||||
mod availability_tests {
|
||||
use super::*;
|
||||
|
||||
/// SAM 2 cannot start a network operation; file installation still verifies the published hash.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[test]
|
||||
fn sam2_is_install_from_file_only() {
|
||||
let mut session = Session::default();
|
||||
for model in [seg::SAM2_TINY, seg::SAM2_SMALL, seg::SAM2_BASE_PLUS, seg::SAM2_LARGE] {
|
||||
let error = session.execute("roto.model.download", json!({"id": model.id})).unwrap_err();
|
||||
assert!(error.to_string().contains("Install from File"));
|
||||
assert!(session.models.status(Task::Mask).busy.is_none());
|
||||
assert!(install_bytes(Path::new("unused"), &model, b"invalid weights").unwrap_err().contains("SHA-256 differs"));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[test]
|
||||
fn stale_model_worker_completions_do_not_poison_the_current_choice() {
|
||||
|
||||
@@ -249,19 +249,23 @@ fn reloading_for_a_stroke_does_not_block_or_retry_a_failed_load() {
|
||||
let _ = std::fs::remove_dir_all(d);
|
||||
}
|
||||
|
||||
/// With `EFFECTCRAFT_DOWNLOAD_SIZE=sam2.1-hiera-small` (or another registry id): the in-app download (Settings ▸ Roto
|
||||
/// Brush ▸ Download) fetches the pinned file over the network, verifies it, installs it with its notice and loads it.
|
||||
/// Skipped without the variable: it needs the network and hundreds of megabytes.
|
||||
/// With `EFFECTCRAFT_DOWNLOAD_SIZE=mobilesam` (or another downloadable registry id): the in-app
|
||||
/// download verifies and loads the file. SAM 2 must refuse until a storytold mirror is available.
|
||||
/// Skipped without the variable: downloadable entries need the network and large files.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[test]
|
||||
fn downloading_a_size_installs_and_loads_it() {
|
||||
let Ok(id) = std::env::var("EFFECTCRAFT_DOWNLOAD_SIZE") else { return };
|
||||
crate::tests_roto::hold_roto_cache_test_lock();
|
||||
let (mut s, d) = session("download");
|
||||
let info = effectcraft_segment::info(&id).unwrap();
|
||||
if info.url.is_empty() {
|
||||
assert!(s.execute_checked("roto.model.download", json!({"id": id, "wait": true})).unwrap_err().to_string().contains("Install from File"));
|
||||
return;
|
||||
}
|
||||
let r = s.execute_checked("roto.model.download", json!({"id": id, "wait": true})).unwrap();
|
||||
assert_eq!(r["id"], id.as_str(), "{r}");
|
||||
let dir = s.models_dir().unwrap();
|
||||
let info = effectcraft_segment::info(&id).unwrap();
|
||||
let file = dir.join(info.file_name);
|
||||
assert_eq!(std::fs::metadata(&file).unwrap().len(), info.size, "the whole file arrived");
|
||||
assert!(dir.join(format!("{}.NOTICE.txt", info.file_name)).exists());
|
||||
|
||||
@@ -12,7 +12,8 @@ Trained models for Roto Brush and face tracking, each kind behind one swappable
|
||||
depends on one model's layout. The classical face tracker stays built in as the fallback.
|
||||
- **`MODELS`**: the registry. Every entry must be open source under a licence compatible with
|
||||
EffectCraft's MIT OR Apache-2.0 (`ALLOWED_LICENCES`), with its task, authors, homepage, licence
|
||||
URL, the official weights URL, size and SHA-256. `load` refuses anything else.
|
||||
URL, an app download URL when available, size and SHA-256. `load` refuses anything else.
|
||||
SAM 2 is Install from File only until a storytold-controlled mirror is available.
|
||||
- **Weights are never bundled.** They are downloaded on demand (or installed from a file) by the
|
||||
engine (`roto.model.*` / `face.model.*`, Settings ▸ Roto Brush / Face Tracking), verified
|
||||
against the registry's SHA-256 (`sha256`, dependency-free) and kept with a notice naming their
|
||||
|
||||
+17
-11
@@ -141,7 +141,7 @@ pub struct ModelInfo {
|
||||
pub licence_url: &'static str,
|
||||
/// The project's page.
|
||||
pub homepage: &'static str,
|
||||
/// The official weights file.
|
||||
/// App download URL; empty when only Install from File is available.
|
||||
pub url: &'static str,
|
||||
pub file_name: &'static str,
|
||||
pub size: u64,
|
||||
@@ -168,8 +168,8 @@ pub const MOBILE_SAM: ModelInfo = ModelInfo {
|
||||
};
|
||||
|
||||
/// SAM 2.1 Tiny (Meta AI, Apache-2.0): a video object tracker with streaming memory. The weights are
|
||||
/// the Hugging Face `transformers` checkpoint, pinned to a commit so the SHA-256 stays valid; they
|
||||
/// are loaded by `effectcraft-sam2` (native only), not by [`load`].
|
||||
/// the Hugging Face `transformers` checkpoint, identified by SHA-256. Until a storytold mirror
|
||||
/// is available, install it from a file. `effectcraft-sam2` (native only) loads it, not [`load`].
|
||||
pub const SAM2_TINY: ModelInfo = ModelInfo {
|
||||
id: "sam2.1-hiera-tiny",
|
||||
task: Task::Mask,
|
||||
@@ -179,7 +179,7 @@ pub const SAM2_TINY: ModelInfo = ModelInfo {
|
||||
licence: "Apache-2.0",
|
||||
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
|
||||
homepage: "https://github.com/facebookresearch/sam2",
|
||||
url: "https://huggingface.co/facebook/sam2.1-hiera-tiny/resolve/de431c4043854a71d8101e17995dfe596bf101a5/model.safetensors",
|
||||
url: "",
|
||||
file_name: "sam2.1-hiera-tiny.safetensors",
|
||||
size: 155_908_064,
|
||||
sha256: "48c14467e5cf9e51870511feb72c89688e82dd74523142c0538b663e193ac2a7",
|
||||
@@ -195,7 +195,7 @@ pub const SAM2_SMALL: ModelInfo = ModelInfo {
|
||||
licence: "Apache-2.0",
|
||||
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
|
||||
homepage: "https://github.com/facebookresearch/sam2",
|
||||
url: "https://huggingface.co/facebook/sam2.1-hiera-small/resolve/ee5bba1d82bb8749febdf90f45e84b687142ba03/model.safetensors",
|
||||
url: "",
|
||||
file_name: "sam2.1-hiera-small.safetensors",
|
||||
size: 184_305_280,
|
||||
sha256: "0a4067b11ce1e23d5229203f11c718a823060d15a4b23fa2372a7d4b77cbbc60",
|
||||
@@ -211,7 +211,7 @@ pub const SAM2_BASE_PLUS: ModelInfo = ModelInfo {
|
||||
licence: "Apache-2.0",
|
||||
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
|
||||
homepage: "https://github.com/facebookresearch/sam2",
|
||||
url: "https://huggingface.co/facebook/sam2.1-hiera-base-plus/resolve/b7320756a13354e7530a63935656d35b2f91a290/model.safetensors",
|
||||
url: "",
|
||||
file_name: "sam2.1-hiera-base-plus.safetensors",
|
||||
size: 323_476_296,
|
||||
sha256: "2012733a0de5d03efd1bba550a2847c4551be9ef2e0d497c83074df66189f780",
|
||||
@@ -227,7 +227,7 @@ pub const SAM2_LARGE: ModelInfo = ModelInfo {
|
||||
licence: "Apache-2.0",
|
||||
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
|
||||
homepage: "https://github.com/facebookresearch/sam2",
|
||||
url: "https://huggingface.co/facebook/sam2.1-hiera-large/resolve/665f8e2ad61cf5f53d65644ff27c8ee525124610/model.safetensors",
|
||||
url: "",
|
||||
file_name: "sam2.1-hiera-large.safetensors",
|
||||
size: 897_897_416,
|
||||
sha256: "dc407dce21301fd94abb395c5099b4f2c455fdc8a8f261ac3d0ea6d4cd197230",
|
||||
@@ -314,9 +314,10 @@ fn loader(id: &str) -> Option<Loader> {
|
||||
|
||||
/// The notice kept next to installed weights: what they are, who made them, their licence.
|
||||
pub fn notice(m: &ModelInfo) -> String {
|
||||
let source = if m.url.is_empty() { m.homepage } else { m.url };
|
||||
format!(
|
||||
"{}\n\nAuthors: {}\nLicence: {} (full text: https://spdx.org/licenses/{}.html; as stated by the authors: {})\nSource: {}\nProject: {}\n\nDownloaded unmodified from the source above. EffectCraft runs it with its own implementation\nand is not affiliated with or endorsed by its authors.\n",
|
||||
m.name, m.authors, m.licence, m.licence, m.licence_url, m.url, m.homepage
|
||||
"{}\n\nAuthors: {}\nLicence: {} (full text: https://spdx.org/licenses/{}.html; as stated by the authors: {})\nSource: {}\nProject: {}\n\nInstalled unmodified. EffectCraft runs it with its own implementation\nand is not affiliated with or endorsed by its authors.\n",
|
||||
m.name, m.authors, m.licence, m.licence, m.licence_url, source, m.homepage
|
||||
)
|
||||
}
|
||||
|
||||
@@ -418,14 +419,19 @@ mod tests {
|
||||
for m in MODELS {
|
||||
assert!(ALLOWED_LICENCES.contains(&m.licence), "{}", m.id);
|
||||
assert_eq!(m.sha256.len(), 64);
|
||||
assert!(m.url.starts_with("https://") && m.licence_url.starts_with("https://") && m.homepage.starts_with("https://"));
|
||||
assert!(m.url.is_empty() || m.url.starts_with("https://"));
|
||||
assert!(m.licence_url.starts_with("https://") && m.homepage.starts_with("https://"));
|
||||
assert!(!m.authors.is_empty() && notice(m).contains(m.authors));
|
||||
}
|
||||
assert_eq!(
|
||||
models(Task::Mask).map(|m| m.id).collect::<Vec<_>>(),
|
||||
["mobilesam", "sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"]
|
||||
);
|
||||
assert!(SAM2_TINY.url.contains("/resolve/") && SAM2_TINY.file_name.ends_with(".safetensors"), "pinned to a commit, not a branch");
|
||||
for model in [SAM2_TINY, SAM2_SMALL, SAM2_BASE_PLUS, SAM2_LARGE] {
|
||||
assert!(model.url.is_empty(), "SAM 2 has no app download until a storytold mirror is available");
|
||||
assert!(model.file_name.ends_with(".safetensors"));
|
||||
assert!(notice(&model).contains(&format!("Source: {}", model.homepage)));
|
||||
}
|
||||
assert_eq!(models(Task::Face).map(|m| m.id).collect::<Vec<_>>(), ["mediapipe-face"]);
|
||||
assert!(load("mediapipe-face", b"nope").err().is_some_and(|e| e.contains("does not match")));
|
||||
// A file that isn't the published one is refused before anything is parsed.
|
||||
|
||||
@@ -513,7 +513,7 @@ fn models_ui(app: &mut EffectcraftApp, ui: &mut egui::Ui, t: &Tokens, acts: &mut
|
||||
return;
|
||||
}
|
||||
ui.horizontal(|ui| {
|
||||
if !installed && !web {
|
||||
if !installed && !web && m["url"].as_str().is_some_and(|url| !url.is_empty()) {
|
||||
let r = ui.add_enabled(busy.is_none(), egui::Button::new(tr("Download")));
|
||||
reg(app, &format!("settings.{prefix}.download.{id}"), &r, "Download");
|
||||
if r.clicked() {
|
||||
@@ -924,3 +924,27 @@ pub fn recovery(app: &mut EffectcraftApp, ctx: &egui::Context, t: &Tokens) {
|
||||
app.ui.status = e;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, not(target_arch = "wasm32")))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Install-only models must not offer a download that the command will reject.
|
||||
#[test]
|
||||
fn sam2_settings_offer_file_installation_without_downloads() {
|
||||
let mut app = EffectcraftApp::new(effectcraft_engine::Session::default());
|
||||
let ctx = egui::Context::default();
|
||||
let tokens = app.tokens;
|
||||
let mut actions = vec![];
|
||||
let mut output = ctx
|
||||
.run_ui(egui::RawInput { screen_rect: Some(egui::Rect::from_min_size(egui::Pos2::ZERO, vec2(900.0, 2200.0))), ..Default::default() }, |ui| {
|
||||
models_ui(&mut app, ui, &tokens, &mut actions, Task::Mask)
|
||||
});
|
||||
output.textures_delta.clear();
|
||||
for id in ["sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"] {
|
||||
assert!(app.auto.find(&format!("settings.roto.download.{id}")).is_none());
|
||||
assert!(app.auto.find(&format!("settings.roto.install.{id}")).is_some());
|
||||
}
|
||||
assert!(actions.is_empty());
|
||||
}
|
||||
}
|
||||
+3
-2
@@ -504,8 +504,9 @@ reference. Matte settings are ordinary properties (`effects/#1/rotoBrushMatte/se
|
||||
Trained model (Roto Brush 2.0 / 3.0):
|
||||
- `roto.models` lists the built-in engine and every registered model, with its authors,
|
||||
licence, size, URL, SHA-256 and whether it is installed, selected and active.
|
||||
- `roto.model.download {id}` fetches and verifies the official weights in the background;
|
||||
`roto.model.install {path, id?}` installs a file you already have.
|
||||
- `roto.model.download {id}` fetches and verifies weights when the registry provides an app download URL;
|
||||
`roto.model.install {path, id?}` installs a file you already have. SAM 2 is Install from File only until
|
||||
a storytold-controlled mirror is available; its size and SHA-256 are verified before installation.
|
||||
- `roto.model.select {id: classical|mobilesam|sam2.1-hiera-tiny|sam2.1-hiera-small|sam2.1-hiera-base-plus|sam2.1-hiera-large}` chooses which one Roto Brush uses (SAM 2 also tracks the object through the clip, see [roto-brush-sam2.md](roto-brush-sam2.md); `roto.status` then reports `source` and per-frame `tracker.frames` with `confidence` and `occluded`). Loading runs
|
||||
in the background; `"wait": true` (also on download and install) returns once it is in use.
|
||||
- `roto.propagate` never falls back to the classic engine silently: a chosen model that is not installed, or failed
|
||||
|
||||
@@ -4,10 +4,10 @@ Roto Brush can follow an object through a clip with Meta's **SAM 2.1 Tiny** (Hie
|
||||
Apache-2.0). It is optional: nothing changes until you install the model and choose it.
|
||||
|
||||
## Using it
|
||||
1. Settings ▸ Roto Brush ▸ SAM 2.1 Tiny ▸ **Download** (downloads `model.safetensors`, 155.9 MB, from the pinned
|
||||
Hugging Face commit; the SHA-256 is checked and the licence notice is kept next to the file). Or install from a
|
||||
file you downloaded. Agents: `roto.model.download {id: "sam2.1-hiera-tiny", wait: true}`, or
|
||||
`roto.model.install {id: "sam2.1-hiera-tiny", path: "model.safetensors"}` for a file you already have.
|
||||
1. Settings ▸ Roto Brush ▸ SAM 2.1 Tiny ▸ **Install from File…** with the published `model.safetensors`
|
||||
(155.9 MB). The file's size and SHA-256 are checked and the licence notice is kept next to it.
|
||||
Agents: `roto.model.install {id: "sam2.1-hiera-tiny", path: "model.safetensors"}`.
|
||||
SAM 2 has no in-app download until the weights are mirrored under storytold control.
|
||||
2. Choose it (`roto.model.select {id: "sam2.1-hiera-tiny"}`).
|
||||
3. Paint strokes on the base frame as usual, then **Propagate**. One tracking pass follows the object through the
|
||||
whole span in both directions, using SAM 2's streaming memory, so it keeps tracking through occlusion and when the
|
||||
@@ -59,7 +59,7 @@ instances can see different pixels. Objects inside one instance always see the s
|
||||
Four checkpoints are in the registry (Settings ▸ Roto Brush); the size is detected from the file, and each is
|
||||
pinned to a Hugging Face commit with its SHA-256. All run the same tracker; a bigger backbone costs time and memory.
|
||||
|
||||
| Model | Download | GPU encode (RX 7900 GRE) | Agreement with PyTorch |
|
||||
| Model | File size | GPU encode (RX 7900 GRE) | Agreement with PyTorch |
|
||||
|---|---|---|---|
|
||||
| SAM 2.1 Tiny (`sam2.1-hiera-tiny`) | 155.9 MB | 0.24 s | encoder 6.8e-6 |
|
||||
| SAM 2.1 Small (`sam2.1-hiera-small`) | 184.3 MB | 0.20 s | encoder 1.0e-5 |
|
||||
|
||||
Reference in new issue
Block a user