Version of tauri app for 4o

Cargo.lock

@@ -653,47 +653,6 @@ dependencies = [
  "backtrace",
 ]
 
-[[package]]
-name = "app"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "assertor",
- "base64 0.22.1",
- "bytes 1.10.0",
- "chrono",
- "const_format",
- "crockford",
- "cudarc",
- "directories 6.0.0",
- "dirs 6.0.0",
- "filesys",
- "futures 0.3.29",
- "half",
- "image",
- "libloading 0.8.6",
- "log",
- "memory_caching",
- "ml_models",
- "ml_weights_registry",
- "once_cell",
- "ort",
- "primitives",
- "rand 0.8.5",
- "reqwest 0.12.15",
- "serde",
- "serde_json",
- "serde_yaml",
- "serial_test",
- "tauri",
- "tauri-build",
- "tauri-plugin-log",
- "tempdir",
- "tempfile",
- "tokio",
- "tokio-util",
-]
-
 [[package]]
 name = "approx"
 version = "0.5.1"
@@ -12092,6 +12051,47 @@ dependencies = [
  "windows 0.58.0",
 ]
 
+[[package]]
+name = "tauri-4o"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "assertor",
+ "base64 0.22.1",
+ "bytes 1.10.0",
+ "chrono",
+ "const_format",
+ "crockford",
+ "cudarc",
+ "directories 6.0.0",
+ "dirs 6.0.0",
+ "filesys",
+ "futures 0.3.29",
+ "half",
+ "image",
+ "libloading 0.8.6",
+ "log",
+ "memory_caching",
+ "ml_models",
+ "ml_weights_registry",
+ "once_cell",
+ "ort",
+ "primitives",
+ "rand 0.8.5",
+ "reqwest 0.12.15",
+ "serde",
+ "serde_json",
+ "serde_yaml",
+ "serial_test",
+ "tauri",
+ "tauri-build",
+ "tauri-plugin-log",
+ "tempdir",
+ "tempfile",
+ "tokio",
+ "tokio-util",
+]
+
 [[package]]
 name = "tauri-build"
 version = "2.0.5"
@@ -12194,6 +12194,47 @@ dependencies = [
  "time",
 ]
 
+[[package]]
+name = "tauri-realtime"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "assertor",
+ "base64 0.22.1",
+ "bytes 1.10.0",
+ "chrono",
+ "const_format",
+ "crockford",
+ "cudarc",
+ "directories 6.0.0",
+ "dirs 6.0.0",
+ "filesys",
+ "futures 0.3.29",
+ "half",
+ "image",
+ "libloading 0.8.6",
+ "log",
+ "memory_caching",
+ "ml_models",
+ "ml_weights_registry",
+ "once_cell",
+ "ort",
+ "primitives",
+ "rand 0.8.5",
+ "reqwest 0.12.15",
+ "serde",
+ "serde_json",
+ "serde_yaml",
+ "serial_test",
+ "tauri",
+ "tauri-build",
+ "tauri-plugin-log",
+ "tempdir",
+ "tempfile",
+ "tokio",
+ "tokio-util",
+]
+
 [[package]]
 name = "tauri-runtime"
 version = "2.3.0"

Cargo.toml

@@ -22,9 +22,10 @@ members = [
   "crates/cli/production/migration-tool",
 
   # Desktop apps
-  "crates/desktop/src-tauri",
   "crates/desktop/ml_models",
   "crates/desktop/ml_weights_registry",
+  "crates/desktop/tauri-4o",
+  "crates/desktop/tauri-realtime",
 
   # Shared libraries
   "crates/lib/actix_cors_configs",

crates/desktop/tauri-4o/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "app"
+name = "tauri-4o"
 version = "0.1.0"
 description = "A Tauri App"
 authors = ["you"]

crates/desktop/tauri-realtime/.gitignore

@@ -0,0 +1,4 @@
+# Generated by Cargo
+# will have compiled files and executables
+/target/
+/gen/schemas

crates/desktop/tauri-realtime/.taurignore

@@ -0,0 +1,4 @@
+# Don't restart the dev server on Rust changes
+crates/
+Cargo.toml
+Cargo.lock

crates/desktop/tauri-realtime/Cargo.toml

@@ -0,0 +1,86 @@
+[package]
+name = "tauri-realtime"
+version = "0.1.0"
+description = "A Tauri App"
+authors = ["you"]
+license = ""
+repository = ""
+edition = "2021"
+rust-version = "1.77.2"
+publish = false
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[lib]
+name = "app_lib"
+crate-type = ["staticlib", "cdylib", "rlib"]
+
+[build-dependencies]
+tauri-build = { version = "2.0.4", features = [] }
+
+[dependencies]
+
+# Internal
+crockford = { path = "../../lib/data/crockford" }
+filesys = { path = "../../lib/files/filesys" }
+memory_caching = { path = "../../lib/caching/memory_caching" }
+ml_weights_registry = { path = "../ml_weights_registry" }
+primitives = { path = "../../lib/data/primitives" }
+
+# Internal ML
+# TODO: Only on Windows and Linux
+#ml_models = { path = "../ml_models", optional = true }
+# Bypass cudarc issues on Windows https://github.com/coreylowman/cudarc/issues/332
+cudarc = { version = "0.13.5", features = ["std", "cublas", "cublaslt", "curand", "driver", "nvrtc", "f16", "cuda-version-from-build-system", "dynamic-linking"], default-features=false }
+
+# External
+#openssl = { version = "0.10", features = ["vendored"] }
+# NB(bt,2025-03-10): default-features = false on Windows while I debug openssl / rustls
+anyhow = "1.0"
+base64 = "0.22.1"
+bytes = "1.10"
+chrono = "0.4.40"
+const_format = "0.2.34"
+directories = "6.0.0"
+dirs = "6.0.0"
+futures = "0.3"
+half = { version = "2.3.1", features = ["num-traits", "use-intrinsics", "rand_distr"] }
+image = "0.25.5"
+log = "0.4"
+once_cell = "1.20.3"
+ort = { version = "1.16.2", features = ["cuda", "load-dynamic"] }
+rand = "0.8.5"
+reqwest = { version = "0.12.12", features = ["stream"], default-features = false }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+serde_yaml = "0.9"
+tauri = { version = "2.2.4", features = [] }
+tauri-plugin-log = "2"
+tempdir = "0.3.7"
+tempfile = "3.18.0"
+tokio = { version = "1.42", features = ["rt", "rt-multi-thread", "fs"] }
+tokio-util = { version = "0.7", features = ["codec"] }
+
+# tracing-subscriber = { version = "0.3", features = [ "env-filter", "fmt" ] }
+
+[target.'cfg(not(target_os = "macos"))'.dependencies]
+ml_models = { path = "../ml_models" }
+
+[dev-dependencies]
+libloading = "0.8.6"
+serial_test = "3.2.0"
+assertor = "0.0.3"
+
+#[target.'cfg(target_os = "macos")'.features]
+#default  = []
+#ml_models = ["dep:ml_models"]
+
+[profile.dev]
+incremental = true # Compile your binary in smaller steps.
+
+[profile.release]
+codegen-units = 1 # Allows LLVM to perform better optimization.
+lto = true # Enables link-time-optimizations.
+opt-level = 3 # `s` prioritizes small binary size. Use `3` if you prefer speed.
+panic = "abort" # Higher performance by disabling panic handlers.
+strip = true # Ensures debug symbols are removed.

crates/desktop/tauri-realtime/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+  tauri_build::build()
+}

crates/desktop/tauri-realtime/capabilities/default.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "../gen/schemas/desktop-schema.json",
+  "identifier": "default",
+  "description": "enables the default permissions",
+  "windows": [
+    "main"
+  ],
+  "permissions": [
+    "core:default",
+    "log:default"
+  ]
+}

crates/desktop/tauri-realtime/src/endpoints/download_models.rs

@@ -0,0 +1,12 @@
+use crate::state::app_dir::AppDataRoot;
+use tauri::State;
+
+#[tauri::command]
+pub async fn download_models(
+  _app_data_root: State<'_, AppDataRoot>,
+) -> Result<String, String> {
+
+  // NB: This endpoint has been removed for now as we now download models in the app lifecycle.
+
+  Ok(("downloaded".to_string()))
+}

crates/desktop/tauri-realtime/src/endpoints/flip_image.rs

@@ -0,0 +1,36 @@
+use base64::prelude::BASE64_STANDARD;
+use base64::Engine;
+use image::codecs::png::{CompressionType, FilterType, PngEncoder};
+use image::ImageReader;
+use std::io::Cursor;
+
+#[tauri::command]
+pub fn flip_image(image: &str) -> Result<String, String> {
+  println!("infer_image called; processing image...");
+
+  let bytes = BASE64_STANDARD.decode(image)
+    .map_err(|err| format!("Base64 decode error: {}", err))?;
+
+  let image = ImageReader::new(Cursor::new(bytes))
+    .with_guessed_format()
+    .map_err(|err| format!("Image format error: {}", err))?
+    .decode()
+    .map_err(|err| format!("Image decode error: {}", err))?;
+
+  let rotated_image = image.rotate180();
+
+  let mut buffer = Vec::new(); // TODO: Preallocate, better data structure
+  let mut writer = Cursor::new(&mut buffer);
+
+  let encoder = PngEncoder::new_with_quality
+    (&mut writer, CompressionType::Fast, FilterType::Adaptive);
+  
+  rotated_image.write_with_encoder(encoder)
+    .map_err(|err| format!("Image encoding error: {}", err))?;
+
+  let encoded = BASE64_STANDARD.encode(&buffer);
+
+  println!("Encoded: {:?}", encoded.split_at(10).0);
+
+  Ok(encoded)
+}

crates/desktop/tauri-realtime/src/endpoints/inpaint_image_endpoint.rs

@@ -0,0 +1,13 @@
+use log::info;
+
+
+#[tauri::command]
+pub async fn inpaint_image(
+  image: &str,
+  mask: &str,
+  prompt: String,
+) -> Result<String, String> {
+  info!("inpaint_endpoint endpoint called.");
+
+  Ok(mask.to_string())
+}

crates/desktop/tauri-realtime/src/endpoints/mod.rs

@@ -0,0 +1,7 @@
+pub mod download_models;
+pub mod flip_image;
+pub mod inpaint_image_endpoint;
+pub mod realtime_image_endpoint;
+pub mod remove_background_endpoint;
+pub mod save_image_endpoint;
+pub mod text_to_image_endpoint;

crates/desktop/tauri-realtime/src/endpoints/realtime_image_endpoint.rs

@@ -0,0 +1,148 @@
+#[cfg(not(target_os = "macos"))]
+use ml_models::ml::stable_diffusion::lcm_pipeline::{lcm_pipeline, Args};
+
+use crate::state::app_config::AppConfig;
+use crate::state::app_dir::AppDataRoot;
+use crate::stubs::model_cache::ModelCache;
+use crate::stubs::prompt_cache::PromptCache;
+use crate::utils::image::decode_base64_image::decode_base64_image;
+use crate::utils::image::encode_dynamic_image_base64_png::encode_dynamic_image_base64_png;
+use crate::utils::image::encode_rgb_image_base64_png::encode_rgb_image_base64_png;
+use image::imageops::FilterType;
+use image::{DynamicImage, RgbImage};
+use log::{error, info};
+use ml_weights_registry::weights_registry::weights::{CLIP_JSON, LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16, LYKON_DEAMSHAPER_7_VAE, SDXL_TURBO_CLIP_TEXT_ENCODER, SIMIANLUO_LCM_DREAMSHAPER_V7_UNET};
+use std::path::PathBuf;
+use tauri::{AppHandle, State};
+
+
+const PROMPT_FILENAME : &str = "prompt.txt";
+const PROMPT: &str = "shiba inu, cute dog, detailed, walking in a wooded forest, photorealistic, 8k";
+
+const NEGATIVE_PROMPT: &str = "bad quality, bad faces, poor quality, blurry faces, watermark";
+
+const RANDOM_SEED: u32 = 42;
+
+
+/// This handler takes an image (as a base64 encoded string) and a prompt and returns
+/// an image (as a base64-encoded string).
+#[tauri::command]
+pub async fn infer_image(
+  image: &str,
+  prompt: Option<String>,
+  strength: Option<f64>,
+  seed: Option<u64>,
+  model_config: State<'_, AppConfig>,
+  model_cache: State<'_, ModelCache>,
+  prompt_cache: State<'_, PromptCache>,
+  app_data_root: State<'_, AppDataRoot>,
+  app: AppHandle,
+) -> Result<String, String> {
+  info!("infer_image endpoint called.");
+
+  let prompt = get_prompt_or_fallback(prompt);
+
+  info!("Strength: {:?}; Prompt: {}", strength, prompt);
+
+  let image = decode_base64_image(image)
+    .map_err(|err| format!("Couldn't hydrate image from base64: {}", err))?;
+  
+  // TODO(bt,2025-02-17): Running out of vram with full image buffer size
+  let mut image = image.resize(512, 512, FilterType::CatmullRom);
+
+  let seed = seed.or(model_config.seed);
+
+  #[cfg(not(target_os = "macos"))]
+  {
+    let result = infer_image_impl(
+      &prompt,
+      image,
+      strength,
+      seed,
+      &model_config,
+      &model_cache,
+      prompt_cache,
+      &app_data_root
+    ).await;
+
+    image = match result {
+      Ok(img) => DynamicImage::ImageRgb8(img),
+      Err(err) => {
+        error!("There was an error generating the image: {:?}", err);
+        return Err(format!("There was an error generating the image: {}", err));
+      }
+    };
+  }
+
+  let bytes = encode_dynamic_image_base64_png(image)
+    .map_err(|err| format!("failure to encode image: {:?}", err))?;
+
+  info!("Inference successful; image converted to base64, serving back to browser...");
+
+  Ok(bytes)
+}
+
+#[cfg(not(target_os = "macos"))]
+async fn infer_image_impl(
+  prompt: &str,
+  image: DynamicImage,
+  strength: Option<f64>,
+  seed: Option<u64>,
+  config: &AppConfig,
+  model_cache: &ModelCache,
+  prompt_cache: State<'_, PromptCache>,
+  app_data_root: &AppDataRoot,
+) -> Result<RgbImage, String> {
+  
+  let weights_dir = app_data_root.weights_dir();
+  let vae_path = weights_dir.weight_path(&LYKON_DEAMSHAPER_7_VAE);
+  let unet_path = weights_dir.weight_path(&SIMIANLUO_LCM_DREAMSHAPER_V7_UNET);
+  let clip_json_path = weights_dir.weight_path(&CLIP_JSON);
+  //let clip_weights_path= weights_dir.weight_path(&LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16);
+  let clip_weights_path= weights_dir.weight_path(&SDXL_TURBO_CLIP_TEXT_ENCODER);
+
+  let args = Args {
+    image: &image,
+    prompt: prompt.to_string(),
+    uncond_prompt: NEGATIVE_PROMPT.to_string(),
+    model_cache,
+    prompt_cache: &prompt_cache,
+    img2img_strength: strength,
+    cfg_scale: config.cfg_scale,
+    use_flash_attn: true,
+    model_config: &config.model_config,
+    maybe_seed: seed,
+    scheduler_steps: config.scheduler_steps,
+    vae_path: &vae_path,
+    unet_path: &unet_path,
+    clip_json_path: &clip_json_path,
+    clip_weights_path: &clip_weights_path,
+  };
+
+  let image = lcm_pipeline(args)
+    .map_err(|err| format!("failure to encode image: {:?}", err))?;
+
+  Ok(image)
+}
+
+fn get_prompt_or_fallback(user_prompt: Option<String>) -> String {
+  let user_prompt = user_prompt.map(|prompt| prompt.trim().to_string())
+    .filter(|prompt| !prompt.is_empty());
+
+  if let Some(prompt) = user_prompt {
+    return prompt;
+  }
+
+  let prompt_file = PathBuf::from(PROMPT_FILENAME)
+    .canonicalize()
+    .unwrap_or_else(|_| PathBuf::from(PROMPT_FILENAME));
+
+  std::fs::read_to_string(&prompt_file)
+    .map_err(|err| format!("Failed to read prompt file: {}", err))
+    .unwrap_or_else(|_| {
+      error!("Failed to read prompt file: {:?}", prompt_file);
+      PROMPT.to_string()
+    })
+    .trim()
+    .to_string()
+}

crates/desktop/tauri-realtime/src/endpoints/remove_background_endpoint.rs

@@ -0,0 +1,45 @@
+//! Adapted from https://github.com/dnanhkhoa/rust-background-removal
+
+#[cfg(not(target_os = "macos"))]
+use ml_models::ml::background_removal::remove_image_background::remove_image_background;
+
+use crate::state::app_dir::AppDataRoot;
+use crate::utils::image::decode_base64_image::decode_base64_image;
+use crate::utils::image::encode_dynamic_image_base64_png::encode_dynamic_image_base64_png;
+use image::{imageops, DynamicImage};
+use log::info;
+use ml_weights_registry::weights_registry::weights::DIS_MEDIUM_ONNX;
+use tauri::State;
+
+/// This handler removes the background from an image.
+#[tauri::command]
+pub async fn remove_background(
+  image: &str,
+  app_data_root: State<'_, AppDataRoot>,
+) -> Result<String, String> {
+  info!("remove_background endpoint called.");
+
+  let mut image = decode_base64_image(image)
+      .map_err(|err| format!("Couldn't hydrate image from base64: {}", err))?;
+
+  #[cfg(not(target_os = "macos"))]
+  {
+    image = remove_background_impl(&app_data_root, image).await?;
+  }
+
+  let image = encode_dynamic_image_base64_png(image)
+    .map_err(|err| format!("failure to encode image: {:?}", err))?;
+
+  Ok(image)
+}
+
+#[cfg(not(target_os = "macos"))]
+async fn remove_background_impl(app_data_root: &AppDataRoot, image: DynamicImage) -> Result<DynamicImage, String> {
+  let model_path = app_data_root.weights_dir().weight_path(&DIS_MEDIUM_ONNX);
+
+  let image = remove_image_background(model_path, image)
+      .await
+      .map_err(|err| format!("Couldn't remove image from image: {}", err))?;
+
+  Ok(image)
+}

crates/desktop/tauri-realtime/src/endpoints/save_image_endpoint.rs

@@ -0,0 +1,38 @@
+use crate::state::app_config::AppConfig;
+use crate::state::app_dir::AppDataRoot;
+use crate::stubs::model_cache::ModelCache;
+use crate::stubs::prompt_cache::PromptCache;
+use crate::utils::image::decode_base64_image::decode_base64_image;
+use chrono::Local;
+use crockford::crockford_entropy_lower;
+use image::{ImageFormat, RgbImage};
+use log::info;
+use tauri::{AppHandle, State};
+
+const FILENAME_LENGTH : usize = 10;
+
+/// This handler takes an image (as a base64 encoded string) and saves it.
+#[tauri::command]
+pub async fn save_image(
+  image: &str,
+  app_data_root: State<'_, AppDataRoot>,
+) -> Result<String, String> {
+  info!("save_image endpoint called.");
+
+  let image = decode_base64_image(image)
+    .map_err(|err| format!("Couldn't hydrate image from base64: {}", err))?;
+
+  let assets_dir = app_data_root.assets_dir();
+  
+  let date_save_dir = assets_dir.make_or_get_current_date_dir()
+    .map_err(|err| format!("Couldn't create assets save directory: {}", err))?;
+
+  let entropy = crockford_entropy_lower(FILENAME_LENGTH); // TODO: These are ugly. A monotonic counter would be better.
+  let filename = format!("{}.png", entropy);
+  let filename = date_save_dir.join(filename);
+  
+  image.save_with_format(filename,  ImageFormat::Png)
+    .map_err(|err| format!("Couldn't save image: {}", err))?;
+
+  Ok("".to_string())
+}

crates/desktop/tauri-realtime/src/endpoints/text_to_image_endpoint.rs

@@ -0,0 +1,114 @@
+#[cfg(not(target_os = "macos"))]
+use ml_models::ml::flux::flux_pipeline::{flux_pipeline, FluxModel, FluxPipelineArgs};
+
+use crate::state::app_config::AppConfig;
+use crate::state::app_dir::{AppDataRoot, AppWeightsDir};
+use crate::stubs::model_cache::ModelCache;
+use crate::stubs::prompt_cache::PromptCache;
+use crate::utils::image::encode_dynamic_image_base64_png::encode_dynamic_image_base64_png;
+use image::imageops::FilterType;
+use image::{DynamicImage, ImageReader, RgbImage};
+use log::{error, info};
+
+use ml_weights_registry::weights_registry::flux_weights::{FLUX_DEV, FLUX_SCHNELL, FLUX_SCHNELL_AUTOENCODER, GOOGLE_T5_V1_1_XXL_CONFIG, GOOGLE_T5_V1_1_XXL_MODEL, LMZ_CANDLE_FLUX_SCHNELL_QUANTIZED_GGUF, LMZ_T5_TOKENIZER_JSON, OPENAI_CLIP_VIT_P14_MODEL, OPENAI_CLIP_VIT_P14_TOKENIZER_JSON};
+use once_cell::sync::Lazy;
+use std::io::Cursor;
+use tauri::{AppHandle, State};
+
+const STRENGTH: f64 = 100.0;
+const PNG_BYTES : &[u8] = include_bytes!("../../binary_includes/1024.png");
+
+// NB: We're just hacking a temporary "text to image" test endpoint leveraging the existing image to image modality.
+static IMAGE : Lazy<DynamicImage> = Lazy::new(||{
+  ImageReader::new(Cursor::new(PNG_BYTES))
+    .with_guessed_format()
+    .expect("failed to detect png format")
+    .decode()
+    .expect("failed to decode image")
+});
+
+#[tauri::command]
+pub async fn text_to_image(
+  prompt: String,
+  model_config: State<'_, AppConfig>,
+  model_cache: State<'_, ModelCache>,
+  prompt_cache: State<'_, PromptCache>,
+  app: AppHandle,
+  app_data_root: State<'_, AppDataRoot>,
+) -> Result<String, String> {
+  info!("text_to_image endpoint called.");
+
+  let mut image = IMAGE.resize(512, 512, FilterType::CatmullRom);
+
+  #[cfg(not(target_os = "macos"))]
+  {
+    let result = text_to_image_impl(
+      &prompt,
+      image,
+      Some(STRENGTH),
+      &model_config,
+      &model_cache,
+      prompt_cache,
+      app_data_root.weights_dir(),
+      app,
+    ).await;
+
+    image = match result {
+      Ok(img) => DynamicImage::ImageRgb8(img),
+      Err(err) => {
+        error!("There was an error generating the image: {:?}", err);
+        return Err(format!("There was an error generating the image: {}", err));
+      }
+    };
+  }
+
+  let bytes = encode_dynamic_image_base64_png(image)
+    .map_err(|err| format!("failure to encode image: {:?}", err))?;
+
+  info!("Inference successful; image converted to base64, serving back to browser...");
+
+  Ok(bytes)
+}
+
+#[cfg(not(target_os = "macos"))]
+async fn text_to_image_impl(
+  prompt: &str,
+  image: DynamicImage,
+  strength: Option<f64>,
+  config: &AppConfig,
+  model_cache: &ModelCache,
+  prompt_cache: State<'_, PromptCache>,
+  weights_dir: &AppWeightsDir,
+  app: AppHandle,
+) -> Result<RgbImage, String> {
+  
+  let args = FluxPipelineArgs {
+    image: &image,
+    prompt: prompt.to_string(),
+    model_cache,
+    prompt_cache: &prompt_cache,
+    app: &app,
+    flux_model: FluxModel::Schnell,
+    active_blocks: None,
+    use_cuda_stream: Some(true),
+    prefetch_next_batch: Some(false),
+    use_full_gpu: Some(true),
+    img2img_strength: strength,
+    seed: None,
+    use_quantized_model: Some(true),
+    model_config: &config.model_config,
+    flux_dev_or_schnell_path: &weights_dir.weight_path(&FLUX_SCHNELL),
+    flux_schnell_quantized_gguf: &weights_dir.weight_path(&LMZ_CANDLE_FLUX_SCHNELL_QUANTIZED_GGUF),
+    flux_autoencoder_path: &weights_dir.weight_path(&FLUX_SCHNELL_AUTOENCODER),
+    google_t5_model_path: &weights_dir.weight_path(&GOOGLE_T5_V1_1_XXL_MODEL),
+    google_t5_config_path: &weights_dir.weight_path(&GOOGLE_T5_V1_1_XXL_CONFIG),
+    lmz_t5_tokenizer_path: &weights_dir.weight_path(&LMZ_T5_TOKENIZER_JSON),
+    openai_clip_model_path: &weights_dir.weight_path(&OPENAI_CLIP_VIT_P14_MODEL),
+    openai_clip_tokenizer_json_path: &weights_dir.weight_path(&OPENAI_CLIP_VIT_P14_TOKENIZER_JSON),
+  };
+
+  let image = flux_pipeline(args)
+    .map_err(|err| format!("failure to encode image: {:?}", err))?;
+
+  Ok(image)
+}

crates/desktop/tauri-realtime/src/events/mod.rs

@@ -0,0 +1 @@
+pub mod notification_event;

crates/desktop/tauri-realtime/src/events/notification_event.rs

@@ -0,0 +1,45 @@
+use serde::Serialize;
+
+#[derive(Clone, Copy, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum NotificationModelType {
+  ImageSegmentation,
+  TextEncoder,
+  TextTokenizer,
+  Unet,
+  Vae,
+  Json,
+  ClipEncoder,
+  FluxModel,
+}
+
+#[derive(Clone, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum NotificationEvent<'a> {
+  ModelDownloadStarted {
+    model_name: &'a str, 
+    model_type: NotificationModelType,
+  },
+  ModelDownloadProgress {
+    model_name: &'a str,
+    model_type: NotificationModelType,
+    currently_downloaded_bytes: usize,
+    total_file_bytes: usize,
+  },
+  ModelDownloadComplete {
+    model_name: &'a str,
+    model_type: NotificationModelType,
+  },
+  ModelLoading {
+    model_name: String,
+    model_type: NotificationModelType,
+  },
+  ModelInference {
+    model_name: String,
+    model_type: NotificationModelType,
+  },
+  ModelInferenceComplete {
+    model_name: String,
+    model_type: NotificationModelType,
+  },
+}

crates/desktop/tauri-realtime/src/lib.rs

@@ -0,0 +1,86 @@
+pub mod endpoints;
+pub mod events;
+pub mod state;
+pub mod stubs;
+pub mod threads;
+pub mod transfer;
+pub mod utils;
+
+use crate::endpoints::download_models::download_models;
+use crate::endpoints::flip_image::flip_image;
+use crate::endpoints::inpaint_image_endpoint::inpaint_image;
+use crate::endpoints::realtime_image_endpoint::infer_image;
+use crate::endpoints::remove_background_endpoint::remove_background;
+use crate::endpoints::save_image_endpoint::save_image;
+use crate::endpoints::text_to_image_endpoint::text_to_image;
+use crate::state::app_config::AppConfig;
+use crate::stubs::model_cache::ModelCache;
+use crate::stubs::prompt_cache::PromptCache;
+use crate::threads::downloader_thread::downloader_thread;
+
+use tauri_plugin_log::Target;
+use tauri_plugin_log::TargetKind;
+
+#[cfg_attr(mobile, tauri::mobile_entry_point)]
+pub fn run() {
+  // NB: Tauri wants to install the logger itself, so we can't rely on the logger crate 
+  // until the tauri runtime begins.
+  println!("Loading model config...");
+
+  let config = AppConfig::init()
+    .expect("config should load");
+
+  let prompt_cache = PromptCache::with_capacity(8)
+    .expect("prompt cache should load");
+
+  println!("Creating model cache...");
+  
+  let model_cache = ModelCache::new();
+  
+  let app_data_root = config.app_data_root.clone();
+  let app_data_root2 = config.app_data_root.clone();
+
+  println!("Initializing backend runtime...");
+
+  tauri::Builder::default()
+    .plugin(tauri_plugin_log::Builder::new()
+      .level(log::LevelFilter::Info)
+      .targets(vec![
+        Target::new(TargetKind::Stdout),
+        Target::new(TargetKind::LogDir { file_name: Some(app_data_root.log_file_name_str().to_string()) }),
+      ])
+      .build())
+    .setup(|app| {
+      // TODO(bt): This is broken on windows
+      // log_environment_details();
+
+      //if cfg!(debug_assertions) {
+      //  app.handle().plugin(
+      //    tauri_plugin_log::Builder::default()
+      //      .level(log::LevelFilter::Info)
+      //      .build(),
+      //  )?;
+      //}
+      let app = app.handle().clone();
+
+      tauri::async_runtime::spawn(downloader_thread(app_data_root2, app));
+
+      Ok(())
+    })
+    .manage(config)
+    .manage(prompt_cache)
+    .manage(model_cache)
+    .manage(app_data_root)
+    .invoke_handler(tauri::generate_handler![
+      download_models,
+      flip_image,
+      infer_image,
+      inpaint_image,
+      remove_background,
+      save_image,
+      text_to_image,
+      text_to_image,
+    ])
+    .run(tauri::generate_context!())
+    .expect("error while running tauri application");
+}

crates/desktop/tauri-realtime/src/main.rs

@@ -0,0 +1,15 @@
+// Prevents additional console window on Windows in release, DO NOT REMOVE!!
+#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
+
+use std::fs::File;
+
+fn main() {
+  //// NB: This is a test to see the behavior of the application with dynamic loading modules.
+  //// We should be able to execute code before loading CUDA and other dependencies.
+  //let r = File::create("program_started.txt");
+  //match r {
+  //  Err(err) => println!("Failed to create program_started.txt: {}", err),
+  //  Ok(_) => {}
+  //}
+  app_lib::run();
+}

crates/desktop/tauri-realtime/src/state/app_config.rs

@@ -0,0 +1,62 @@
+use crate::state::app_dir::AppDataRoot;
+use crate::state::os_platform::OsPlatform;
+use crate::state::yaml_config::YamlConfig;
+use crate::stubs::model_config::ModelConfig;
+
+const DEFAULT_SD_IMAGE_WIDTH: usize = 1024;
+const DEFAULT_SD_IMAGE_HEIGHT: usize = 1024;
+
+
+/// A central place to configure app-wide details.
+pub struct AppConfig {
+  pub image_height: usize,
+  pub image_width: usize,
+  pub scheduler_steps: usize,
+  pub scheduler_samples: usize,
+  
+  pub seed: Option<u64>,
+
+  pub cfg_scale: Option<f64>,
+  
+  /// Root location for application data.
+  pub app_data_root: AppDataRoot,
+  
+  pub model_config: ModelConfig,
+}
+
+impl AppConfig {
+  
+  pub fn init() -> anyhow::Result<Self> {
+    let yaml_configs = YamlConfig::load_from_config_file_recursive();
+    
+    println!("Configs: {:?}", yaml_configs);
+    
+    let os_platform = OsPlatform::get();
+
+    let default_data_root = match os_platform {
+      OsPlatform::Linux => yaml_configs.linux_default_data_path.as_deref(),
+      OsPlatform::MacOs => yaml_configs.mac_default_data_path.as_deref(),
+      OsPlatform::Windows => yaml_configs.windows_default_data_path.as_deref(),
+    };
+    
+    println!("Possible user-defined data root: {:?}", default_data_root);
+
+    let app_data_root = match default_data_root {
+      None => AppDataRoot::create_default()?,
+      Some(path) => AppDataRoot::create_existing(path)?,
+    };
+
+    println!("App data root: {:?}", app_data_root.path());
+
+    Ok(Self {
+      image_height: yaml_configs.image_height.unwrap_or(DEFAULT_SD_IMAGE_HEIGHT),
+      image_width: yaml_configs.image_width.unwrap_or(DEFAULT_SD_IMAGE_WIDTH),
+      scheduler_steps: yaml_configs.scheduler_steps.unwrap_or(4),
+      scheduler_samples: yaml_configs.scheduler_samples.unwrap_or(15),
+      seed: yaml_configs.seed,
+      cfg_scale: yaml_configs.cfg_scale,
+      app_data_root,
+      model_config: ModelConfig::init()?,
+    })
+  }
+}

crates/desktop/tauri-realtime/src/state/app_dir.rs

@@ -0,0 +1,220 @@
+use crate::state::expanduser::expanduser;
+use crate::state::os_platform::OsPlatform;
+use anyhow::anyhow;
+use directories::UserDirs;
+use ml_weights_registry::weights_registry::weight_descriptor::WeightDescriptor;
+use std::path::{Path, PathBuf};
+use chrono::{DateTime, Local};
+use tempdir::TempDir;
+use tempfile::{Builder, NamedTempFile};
+
+const DEFAULT_DATA_DIR : &str = "artcraft";
+const ASSETS_SUBDIRECTORY : &str = "assets";
+const WEIGHTS_SUBDIRECTORY : &str = "weights";
+
+const TEMPORARY_SUBDIRECTORY : &str = "temp";
+
+/// Note: Tauri appends ".log" to the end of the filename.
+const LOG_FILE_NAME : &str = "application_debug";
+
+/// The path to the application data directory, which includes "asset" and "weights" data.
+#[derive(Clone)]
+pub struct AppDataRoot {
+  path: PathBuf,
+  assets_dir: AppAssetsDir,
+  weights_dir: AppWeightsDir,
+  temp_dir: TemporaryDir,
+  log_file_name: PathBuf,
+  log_file_name_string: String,
+}
+
+#[derive(Clone)]
+pub struct AppAssetsDir {
+  path: PathBuf,
+}
+
+#[derive(Clone)]
+pub struct AppWeightsDir {
+  path: PathBuf,
+}
+
+#[derive(Clone)]
+pub struct TemporaryDir {
+  path: PathBuf,
+}
+
+impl AppDataRoot {
+  pub fn create_default() -> anyhow::Result<Self> {
+    let directory = get_default_data_dir()?;
+    Self::create_existing(directory)
+  }
+
+  pub fn create_existing<P: AsRef<Path>>(dir: P) -> anyhow::Result<Self> {
+    let mut dir = dir.as_ref().to_path_buf();
+    
+    match OsPlatform::get() {
+      OsPlatform::Linux | OsPlatform::MacOs => {
+        if let Some(d) = dir.as_os_str().to_str() {
+          dir = expanduser(d)?;
+        }
+      },
+      OsPlatform::Windows => {}
+    }
+    
+    if !dir.is_dir() {
+      println!("Creating directory {:?}", dir);
+      std::fs::create_dir_all(&dir)?;
+    }
+
+    match dir.canonicalize() {
+      Ok(d) => dir = d,
+      Err(err) => {
+        println!("Error canonicalizing {:?}: {}", dir, err);
+      }
+    }
+    
+    let assets_dir = AppAssetsDir::create_existing(dir.join(ASSETS_SUBDIRECTORY))?;
+    let weights_dir = AppWeightsDir::create_existing(dir.join(WEIGHTS_SUBDIRECTORY))?;
+    let temp_dir = TemporaryDir::create_existing(dir.join(TEMPORARY_SUBDIRECTORY))?;
+    let log_file_name = dir.join(LOG_FILE_NAME);
+    let log_file_name_string = log_file_name
+        .to_str()
+        .ok_or(anyhow!("couldn't convert log path to str"))?
+        .to_string();
+
+    Ok(Self {
+      path: dir,
+      assets_dir,
+      weights_dir,
+      temp_dir,
+      log_file_name,
+      log_file_name_string,
+    })
+  }
+  
+  pub fn assets_dir(&self) -> &AppAssetsDir {
+    &self.assets_dir
+  }
+  
+  pub fn weights_dir(&self) -> &AppWeightsDir {
+    &self.weights_dir
+  }
+
+  pub fn temp_dir(&self) -> &TemporaryDir {
+    &self.temp_dir
+  }
+
+  pub fn path(&self) -> &Path {
+    &self.path
+  }
+
+  pub fn log_file_name(&self) -> &Path {
+    &self.log_file_name
+  }
+
+  pub fn log_file_name_str(&self) -> &str {
+    &self.log_file_name_string
+  }
+}
+
+impl AppAssetsDir {
+  pub fn create_existing<P: AsRef<Path>>(dir: P) -> anyhow::Result<Self> {
+    let mut dir = dir.as_ref().to_path_buf();
+    match dir.canonicalize() {
+      Ok(d) => dir = d,
+      Err(err) => {
+        println!("Error canonicalizing {:?}: {}", dir, err);
+      }
+    }
+    if !dir.exists() {
+      println!("Creating directory {:?}", dir);
+      std::fs::create_dir(&dir)?;
+    }
+    Ok(Self {
+      path: dir,
+    })
+  }
+
+  pub fn path(&self) -> &Path {
+    &self.path
+  }
+  
+  /// Return the current date asset directory
+  /// If it doesn't exist, create it.
+  pub fn make_or_get_current_date_dir(&self) -> anyhow::Result<PathBuf> {
+    let format: DateTime<Local> = Local::now();
+    let date_directory_name = format.format("%Y-%d-%m").to_string();
+    let full_path = self.path.join(date_directory_name);
+    if !full_path.exists() {
+      std::fs::create_dir(&full_path)?;
+    }
+    Ok(full_path)
+  }
+}
+
+impl AppWeightsDir {
+  pub fn create_existing<P: AsRef<Path>>(dir: P) -> anyhow::Result<Self> {
+    let mut dir = dir.as_ref().to_path_buf();
+    match dir.canonicalize() {
+      Ok(d) => dir = d,
+      Err(err) => {
+        println!("Error canonicalizing {:?}: {}", dir, err);
+      }
+    }
+    if !dir.exists() {
+      println!("Creating directory {:?}", dir);
+      std::fs::create_dir(&dir)?;
+    }
+    Ok(Self {
+      path: dir,
+    })
+  }
+
+  pub fn path(&self) -> &Path {
+    &self.path
+  }
+
+  pub fn weight_path(&self, descriptor: &WeightDescriptor) -> PathBuf {
+    self.path.join(descriptor.filename)
+  }
+}
+
+impl TemporaryDir {
+  pub fn create_existing<P: AsRef<Path>>(dir: P) -> anyhow::Result<Self> {
+    let mut dir = dir.as_ref().to_path_buf();
+    match dir.canonicalize() {
+      Ok(d) => dir = d,
+      Err(err) => {
+        println!("Error canonicalizing {:?}: {}", dir, err);
+      }
+    }
+    if !dir.exists() {
+      println!("Creating directory {:?}", dir);
+      std::fs::create_dir(&dir)?;
+    }
+    Ok(Self {
+      path: dir,
+    })
+  }
+
+  pub fn with_prefix(&self, prefix: &str) -> anyhow::Result<TempDir> {
+    let tempdir = TempDir::new_in(&self.path, prefix)?;
+    Ok(tempdir)
+  }
+
+  pub fn new_named_temp_file(&self) -> anyhow::Result<NamedTempFile> {
+    let tempfile = Builder::new()
+      .prefix("temp_")
+      .suffix(".bin")
+      .tempfile_in(&self.path)?;
+    Ok(tempfile)
+  }
+}
+
+// eg. /home/bob/artcraft, /Users/bob/artcraft, or C:\Users\Alice\artcraft
+fn get_default_data_dir() -> anyhow::Result<PathBuf> {
+  Ok(UserDirs::new()
+      .ok_or_else(|| anyhow!("could not determine user home directory"))?
+      .home_dir()
+      .join(DEFAULT_DATA_DIR))
+}

crates/desktop/tauri-realtime/src/state/expanduser.rs

@@ -0,0 +1,73 @@
+use once_cell::sync::Lazy;
+use std::io;
+use std::path::{PathBuf, MAIN_SEPARATOR};
+
+static PREFIX: Lazy<String> = Lazy::new(|| format!("~{}", MAIN_SEPARATOR));
+
+/// The "expanduser" crate doesn't compile on Windows, so we replace its functionality slightly
+pub fn expanduser<P: AsRef<str>>(path: P) -> io::Result<PathBuf> {
+
+   Ok(match path.as_ref() {
+       // matches an exact "~"
+       s if s == "~" => {
+           home_dir()?
+       },
+       // matches paths that start with `~/`
+       s if s.starts_with(&*PREFIX) => {
+           let home = home_dir()?;
+           home.join(&s[2..])
+       },
+       // // matches paths that start with `~` but not `~/`, might be a `~username/` path
+       // s if s.starts_with("~") => {
+       //     let mut parts = s[1..].splitn(2, MAIN_SEPARATOR);
+       //     let user = parts.next()
+       //         .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "malformed path"))?;
+       //     let user = Passwd::from_name(&user)
+       //         .map_err(|_| io::Error::new(io::ErrorKind::Other, "error searching for user"))?
+       //         .ok_or_else(|| io::Error::new(io::ErrorKind::Other, format!("user '{}', does not exist", &user)))?;
+       //     if let Some(ref path) = parts.next() {
+       //         PathBuf::from(user.dir).join(&path)
+       //     } else {
+       //         PathBuf::from(user.dir)
+       //     }
+       // },
+       // nothing to expand, just make a PathBuf
+       s => PathBuf::from(s)
+   })
+}
+
+pub fn home_dir() -> io::Result<PathBuf> {
+    dirs::home_dir()
+        .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "no home directory is set"))
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::state::expanduser::expanduser;
+    use std::env;
+    use std::path::PathBuf;
+
+    #[test]
+    #[serial_test::serial]
+    fn test_success() {
+        let old_home = env::var("HOME").expect("no home dir set");
+        let new_home = "/home/foo";
+        env::set_var("HOME", new_home);
+        let path = expanduser("~/path/to/directory");
+        env::set_var("HOME", old_home);
+        assert_eq!(path.expect("io error"), PathBuf::from("/home/foo/path/to/directory"));
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn test_only_tilde() {
+        let old_home = env::var("HOME").expect("no home dir set");
+        let new_home = "/home/foo";
+        env::set_var("HOME", new_home);
+        let pathstr = "~";
+        let path = expanduser(pathstr);
+        env::set_var("HOME", old_home);
+        assert_eq!(path.expect("io error"), PathBuf::from("/home/foo"));
+    }
+
+}

crates/desktop/tauri-realtime/src/state/mod.rs

@@ -0,0 +1,5 @@
+pub mod app_config;
+pub mod app_dir;
+pub mod os_platform;
+pub mod yaml_config;
+pub mod expanduser;

crates/desktop/tauri-realtime/src/state/os_platform.rs

@@ -0,0 +1,20 @@
+
+#[derive(Copy,Clone,Eq,PartialEq,Debug)]
+pub enum OsPlatform {
+  Linux,
+  MacOs,
+  Windows,
+}
+
+impl OsPlatform {
+  pub fn get() -> OsPlatform {
+    // Totally scientific.
+    // https://doc.rust-lang.org/std/env/consts/constant.OS.html
+    match std::env::consts::OS {
+      "linux" => OsPlatform::Linux,
+      "windows" => OsPlatform::Windows,
+      "macos" | "apple" => OsPlatform::MacOs,
+      _ => OsPlatform::Linux,
+    }
+  }
+}