From c9ccb7680f84dff3c6e9d7459da1a74d3b514b87 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 13 May 2025 04:31:35 -0400 Subject: [PATCH] remove background with fal --- Cargo.lock | 116 +++++++++++++++++- Cargo.toml | 2 + crates/lib/api_clients/fal_client/Cargo.toml | 48 ++++++++ .../fal_client/src/creds/fal_api_key.rs | 13 ++ .../api_clients/fal_client/src/creds/mod.rs | 1 + .../fal_client/src/fal_error_plus.rs | 34 +++++ crates/lib/api_clients/fal_client/src/lib.rs | 7 ++ .../fal_client/src/requests/mod.rs | 1 + .../src/requests/remove_background_rembg.rs | 73 +++++++++++ .../src/utils/image_file_to_base64_url.rs | 0 .../lib/files/file_to_base64_url/Cargo.toml | 48 ++++++++ .../src/file_to_base64_url.rs | 26 ++++ .../lib/files/file_to_base64_url/src/lib.rs | 1 + 13 files changed, 369 insertions(+), 1 deletion(-) create mode 100644 crates/lib/api_clients/fal_client/Cargo.toml create mode 100644 crates/lib/api_clients/fal_client/src/creds/fal_api_key.rs create mode 100644 crates/lib/api_clients/fal_client/src/creds/mod.rs create mode 100644 crates/lib/api_clients/fal_client/src/fal_error_plus.rs create mode 100644 crates/lib/api_clients/fal_client/src/lib.rs create mode 100644 crates/lib/api_clients/fal_client/src/requests/mod.rs create mode 100644 crates/lib/api_clients/fal_client/src/requests/remove_background_rembg.rs create mode 100644 crates/lib/api_clients/fal_client/src/utils/image_file_to_base64_url.rs create mode 100644 crates/lib/files/file_to_base64_url/Cargo.toml create mode 100644 crates/lib/files/file_to_base64_url/src/file_to_base64_url.rs create mode 100644 crates/lib/files/file_to_base64_url/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index ee7a8d8a5c..4c99d3265a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1063,7 +1063,7 @@ dependencies = [ "serde_path_to_error", "serde_qs 0.10.1", "sha2 0.10.8", - "smart-default", + "smart-default 0.6.0", "smol_str", "thiserror 1.0.63", "tokio", @@ -4180,6 +4180,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "eventsource-stream" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab" +dependencies = [ + "futures-core", + "nom", + "pin-project-lite", +] + [[package]] name = "exr" version = "1.73.0" @@ -4201,6 +4212,67 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" +[[package]] +name = "fal" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd7dd8565317310f05569296166e2cbdd2bc870173b2b24b77ed681166268d0" +dependencies = [ + "base64 0.22.1", + "eventsource-stream", + "fal-derive", + "futures 0.3.29", + "image", + "reqwest 0.12.15", + "serde", + "serde_json", + "smart-default 0.7.1", + "thiserror 1.0.63", +] + +[[package]] +name = "fal-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f377f2f18a6bf107f907aa687df6bda2874bd38b25b1531f2a6fb38a9ca643" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "fal_client" +version = "0.0.1" +dependencies = [ + "anyhow", + "base64 0.22.1", + "chrono", + "cookie 0.18.1", + "crockford", + "errors", + "fal", + "file_to_base64_url", + "filesys", + "futures 0.3.29", + "hex-literal", + "idempotency", + "log", + "once_cell", + "rand 0.9.0", + "reqwest 0.12.15", + "serde", + "serde_derive", + "serde_json", + "sha3", + "subprocess_common", + "testing", + "thiserror 2.0.12", + "tokio", + "toml 0.8.20", + "workspace-build-acceleration", +] + [[package]] name = "fancy-regex" version = "0.13.0" @@ -4281,6 +4353,37 @@ dependencies = [ "rustc_version 0.4.0", ] +[[package]] +name = "file_to_base64_url" +version = "0.0.1" +dependencies = [ + "anyhow", + "base64 0.22.1", + "chrono", + "cookie 0.18.1", + "crockford", + "errors", + "fal", + "filesys", + "hex-literal", + "idempotency", + "log", + "mimetypes", + "once_cell", + "rand 0.9.0", + "reqwest 0.12.15", + "serde", + "serde_derive", + "serde_json", + "sha3", + "subprocess_common", + "testing", + "thiserror 2.0.12", + "tokio", + "toml 0.8.20", + "workspace-build-acceleration", +] + [[package]] name = "filesys" version = "0.0.1" @@ -11243,6 +11346,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "smart-default" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "smol_str" version = "0.1.24" diff --git a/Cargo.toml b/Cargo.toml index 2db8ee9fc0..2b4f547be7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ members = [ "crates/lib/actix_cors_configs", "crates/lib/actix_helpers", "crates/lib/api_clients/cloud_storage", + "crates/lib/api_clients/fal_client", "crates/lib/api_clients/google_drive_common", "crates/lib/api_clients/google_sign_in", "crates/lib/api_clients/openai_sora_client", @@ -54,6 +55,7 @@ members = [ "crates/lib/email_sender", "crates/lib/errors", "crates/lib/files/archives", + "crates/lib/files/file_to_base64_url", "crates/lib/files/filesys", "crates/lib/files/hashing", "crates/lib/files/images", diff --git a/crates/lib/api_clients/fal_client/Cargo.toml b/crates/lib/api_clients/fal_client/Cargo.toml new file mode 100644 index 0000000000..864f41c39f --- /dev/null +++ b/crates/lib/api_clients/fal_client/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "fal_client" +edition = "2021" +version = "0.0.1" +publish = false + +[package.metadata.cargo-udeps.ignore] +normal = ["workspace-build-acceleration"] + +[lib] +name = "fal_client" +path = "src/lib.rs" + +[dependencies] + +# Workspace hack (faster compile times) +workspace-build-acceleration = { workspace = true } + +# Internal +crockford = { path = "../../data/crockford" } +file_to_base64_url = { path = "../../files/file_to_base64_url" } +errors = { path = "../../errors" } +idempotency = { path = "../../idempotency" } +subprocess_common = { path = "../../subprocess_common" } + +# External +anyhow = { workspace = true } +base64 = { workspace = true } +chrono = { workspace = true} +cookie = { workspace = true } +fal = { version = "0.3.6", features = ["endpoints"] } +hex-literal = { workspace = true } +log = { workspace = true } +once_cell= { workspace = true } +rand = { workspace = true } +reqwest = { workspace = true, features = ["multipart", "gzip"] } +serde = { workspace = true } +serde_derive = { workspace = true } +serde_json = { workspace = true } +sha3 = { workspace = true } +thiserror = { workspace = true } +tokio = { version = "1.38.0", features = ["macros"] } +toml = { workspace = true } +futures = "0.3.29" + +[dev-dependencies] +testing = { path = "../../testing" } +filesys = { path = "../../files/filesys" } diff --git a/crates/lib/api_clients/fal_client/src/creds/fal_api_key.rs b/crates/lib/api_clients/fal_client/src/creds/fal_api_key.rs new file mode 100644 index 0000000000..5e3b0beb45 --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/creds/fal_api_key.rs @@ -0,0 +1,13 @@ + +#[derive(Clone)] +pub struct FalApiKey(pub String); + +impl FalApiKey { + pub fn new(api_key: String) -> Self { + Self(api_key) + } + + pub fn from_str(api_key: &str) -> Self { + Self(api_key.to_string()) + } +} diff --git a/crates/lib/api_clients/fal_client/src/creds/mod.rs b/crates/lib/api_clients/fal_client/src/creds/mod.rs new file mode 100644 index 0000000000..1da716b9aa --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/creds/mod.rs @@ -0,0 +1 @@ +pub mod fal_api_key; \ No newline at end of file diff --git a/crates/lib/api_clients/fal_client/src/fal_error_plus.rs b/crates/lib/api_clients/fal_client/src/fal_error_plus.rs new file mode 100644 index 0000000000..b0de4d4002 --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/fal_error_plus.rs @@ -0,0 +1,34 @@ +use std::error::Error; +use std::fmt::{Display, Formatter}; + +/// Additional errors that aren't included in `fal::FalError`. +#[derive(Debug)] +pub enum FalErrorPlus { + /// An error arising in the `fal` crate. + FalError(fal::FalError), + /// Another error we didn't handle. + AnyhowError(anyhow::Error), +} + +impl Display for FalErrorPlus { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::FalError(err) => write!(f, "FalErrorPlus::FalError: {:?}", err), + Self::AnyhowError(err) => write!(f, "FalErrorPlus::AnyhowError: {:?}", err), + } + } +} + +impl Error for FalErrorPlus {} + +impl From for FalErrorPlus { + fn from(err: fal::FalError) -> Self { + FalErrorPlus::FalError(err) + } +} + +impl From for FalErrorPlus { + fn from(err: anyhow::Error) -> Self { + FalErrorPlus::AnyhowError(err) + } +} diff --git a/crates/lib/api_clients/fal_client/src/lib.rs b/crates/lib/api_clients/fal_client/src/lib.rs new file mode 100644 index 0000000000..82d82db2bd --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/lib.rs @@ -0,0 +1,7 @@ +//! A wrapper for the "fal" crate that bakes in a few extra recipes and utilities. + +extern crate fal; + +pub mod creds; +pub mod fal_error_plus; +pub mod requests; diff --git a/crates/lib/api_clients/fal_client/src/requests/mod.rs b/crates/lib/api_clients/fal_client/src/requests/mod.rs new file mode 100644 index 0000000000..6a40215b31 --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/requests/mod.rs @@ -0,0 +1 @@ +pub mod remove_background_rembg; \ No newline at end of file diff --git a/crates/lib/api_clients/fal_client/src/requests/remove_background_rembg.rs b/crates/lib/api_clients/fal_client/src/requests/remove_background_rembg.rs new file mode 100644 index 0000000000..27dd7d8ce4 --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/requests/remove_background_rembg.rs @@ -0,0 +1,73 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::fal_error_plus::FalErrorPlus; +use fal::endpoints::fal_ai::imageutils::rembg::RemoveBackgroundInput; +use fal::prelude::fal_ai::imageutils::rembg::rembg; +use fal::prelude::*; +use fal_client::file_to_base64_url::file_to_base64_url; +use futures::StreamExt; +use std::path::Path; + +pub struct RemBgArgs<'a, P: AsRef> { + pub image_path: P, + pub api_key: &'a FalApiKey, +} + +pub struct RemBgResponse { + pub image_url: String, +} + +/// remove background using `fal_ai/imageutils/rembg` +pub async fn remove_background_rembg>(args: RemBgArgs<'_, P>) -> Result { + let image_url = file_to_base64_url(&args.image_path)?; + + let request = RemoveBackgroundInput { + image_url, + crop_to_bbox: None, + sync_mode: None + }; + + let result = rembg(request) + .with_api_key(&args.api_key.0) + .queue() + .await?; + + let mut stream = result.stream(true).await?; + + while let Some(status) = stream.next().await { + let status = status?; + if status.status == Status::Completed { + break; + } + } + + let output = result.response().await?; + + Ok(RemBgResponse { + image_url: output.image.url, + }) +} + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::remove_background_rembg::{remove_background_rembg, RemBgArgs}; + use errors::AnyhowResult; + use testing::test_file_path::test_file_path; + + #[tokio::test] + #[ignore] + async fn test_remove_background_rembg() -> AnyhowResult<()> { + let image = test_file_path("test_data/image/juno.jpg")?; + + // TODO: DO NOT COMMIT SECRET ! + const KEY : &str = "DO NOT COMMIT"; + let api_key = FalApiKey::from_str(KEY); + + let result = remove_background_rembg(RemBgArgs { + image_path: image, + api_key: &api_key, + }).await?; + + Ok(()) + } +} diff --git a/crates/lib/api_clients/fal_client/src/utils/image_file_to_base64_url.rs b/crates/lib/api_clients/fal_client/src/utils/image_file_to_base64_url.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/lib/files/file_to_base64_url/Cargo.toml b/crates/lib/files/file_to_base64_url/Cargo.toml new file mode 100644 index 0000000000..bb8ffbf716 --- /dev/null +++ b/crates/lib/files/file_to_base64_url/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "file_to_base64_url" +edition = "2021" +version = "0.0.1" +publish = false + +[package.metadata.cargo-udeps.ignore] +normal = ["workspace-build-acceleration"] + +[lib] +name = "fal_client" +path = "src/lib.rs" + +[dependencies] + +# Workspace hack (faster compile times) +workspace-build-acceleration = { workspace = true } + +# Internal +crockford = { path = "../../data/crockford" } +errors = { path = "../../errors" } +idempotency = { path = "../../idempotency" } +subprocess_common = { path = "../../subprocess_common" } + +# External +anyhow = { workspace = true } +base64 = { workspace = true } +chrono = { workspace = true} +cookie = { workspace = true } +fal = { version = "0.3.6", features = ["endpoints"] } +hex-literal = { workspace = true } +log = { workspace = true } +filesys = { workspace = true } +mimetypes = { workspace = true } +once_cell= { workspace = true } +rand = { workspace = true } +reqwest = { workspace = true, features = ["multipart", "gzip"] } +serde = { workspace = true } +serde_derive = { workspace = true } +serde_json = { workspace = true } +sha3 = { workspace = true } +thiserror = { workspace = true } +tokio = { version = "1.38.0", features = ["macros"] } +toml = { workspace = true } + +[dev-dependencies] +testing = { path = "../../testing" } +filesys = { path = "../../files/filesys" } diff --git a/crates/lib/files/file_to_base64_url/src/file_to_base64_url.rs b/crates/lib/files/file_to_base64_url/src/file_to_base64_url.rs new file mode 100644 index 0000000000..6e6097b8be --- /dev/null +++ b/crates/lib/files/file_to_base64_url/src/file_to_base64_url.rs @@ -0,0 +1,26 @@ +use anyhow::anyhow; +use base64::prelude::{BASE64_STANDARD, BASE64_URL_SAFE, BASE64_URL_SAFE_NO_PAD}; +use base64::Engine; +use errors::AnyhowResult; +use mimetypes::mimetype_info::mimetype_info::MimetypeInfo; +use std::path::Path; + +/// Read a file into a base64 URL-safe string. +pub fn file_to_base64_url>(path: P) -> AnyhowResult { + let bytes = std::fs::read(path.as_ref())?; + + let mime = match MimetypeInfo::get_for_bytes(&bytes) { + Some(mime) => mime, + None => { + return Err(anyhow!("could not determine mimetype")); + }, + }; + + let base64_bytes = BASE64_STANDARD.encode(&bytes); + + match mime.mime_type() { + "image/png" => Ok(format!("data:image/png;base64,{}", base64_bytes)), + "image/jpeg" => Ok(format!("data:image/jpeg;base64,{}", base64_bytes)), + _ => Err(anyhow!("unknown or unsupported mime type: {}", mime.mime_type())), + } +} diff --git a/crates/lib/files/file_to_base64_url/src/lib.rs b/crates/lib/files/file_to_base64_url/src/lib.rs new file mode 100644 index 0000000000..ad02798042 --- /dev/null +++ b/crates/lib/files/file_to_base64_url/src/lib.rs @@ -0,0 +1 @@ +pub mod file_to_base64_url; \ No newline at end of file