mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-19 16:15:04 +00:00
some progress
This commit is contained in:
1 parent
8cd5ffd604
commit
d52fa10c6e
2 files changed
+406
-365
No files matched your search
@@ -8,44 +8,51 @@ const PROMPT: &str = "A beautiful landscape with mountains and a lake";
|
||||
|
||||
#[tauri::command]
|
||||
pub fn infer_image(image: &str) -> Result<String, String> {
|
||||
println!("infer_image called; generating image with SDXL Turbo...");
|
||||
|
||||
// Check CUDA availability
|
||||
let use_cpu = match Device::new_cuda(0) {
|
||||
Ok(_) => {
|
||||
println!("CUDA device available");
|
||||
false
|
||||
}
|
||||
Err(e) => {
|
||||
println!("CUDA not available ({}), falling back to CPU", e);
|
||||
true
|
||||
}
|
||||
};
|
||||
|
||||
let args = Args {
|
||||
prompt: PROMPT.to_string(),
|
||||
uncond_prompt: "".to_string(),
|
||||
cpu: use_cpu,
|
||||
height: Some(512),
|
||||
width: Some(512),
|
||||
n_steps: Some(1),
|
||||
num_samples: 1,
|
||||
seed: None,
|
||||
sd_version: StableDiffusionVersion::Turbo,
|
||||
guidance_scale: Some(0.0),
|
||||
};
|
||||
println!("infer_image called; generating image with SDXL Turbo...");
|
||||
|
||||
match sd::run(args) {
|
||||
Ok(_) => {
|
||||
let img_data = std::fs::read("temp.png")
|
||||
.map_err(|e| format!("Failed to read generated image: {}", e))?;
|
||||
let encoded = BASE64_STANDARD.encode(&img_data);
|
||||
println!("Generated image encoded successfully");
|
||||
|
||||
let _ = std::fs::remove_file("temp.png");
|
||||
|
||||
Ok(encoded)
|
||||
}
|
||||
Err(e) => Err(format!("Failed to generate image: {}", e))
|
||||
}
|
||||
// Check CUDA availability
|
||||
let use_cpu = match Device::new_cuda(0) {
|
||||
Ok(_) => {
|
||||
println!("CUDA device available");
|
||||
false
|
||||
},
|
||||
Err(e) => {
|
||||
println!("CUDA not available ({}), falling back to CPU", e);
|
||||
true
|
||||
},
|
||||
};
|
||||
|
||||
let decoded = BASE64_STANDARD.decode(&image).unwrap();
|
||||
std::fs::write("/tmp/mask.png", decoded).expect("Unable to write file");
|
||||
let args = Args {
|
||||
prompt: PROMPT.to_string(),
|
||||
uncond_prompt: "".to_string(),
|
||||
cpu: use_cpu,
|
||||
height: Some(512),
|
||||
width: Some(512),
|
||||
n_steps: Some(15),
|
||||
num_samples: 1,
|
||||
seed: None,
|
||||
sd_version: StableDiffusionVersion::Turbo,
|
||||
guidance_scale: None,
|
||||
// img2img: None,
|
||||
// img2img_strength: 0.0,
|
||||
// use_f16: false,
|
||||
img2img: Some("/tmp/mask.png".to_string()),
|
||||
img2img_strength: 0.75,
|
||||
use_f16: true,
|
||||
};
|
||||
|
||||
match sd::run(args) {
|
||||
Ok(_) => {
|
||||
let img_data = std::fs::read("temp.png").map_err(|e| format!("Failed to read generated image: {}", e))?;
|
||||
let encoded = BASE64_STANDARD.encode(&img_data);
|
||||
println!("Generated image encoded successfully");
|
||||
|
||||
let _ = std::fs::remove_file("temp.png");
|
||||
|
||||
Ok(encoded)
|
||||
},
|
||||
Err(e) => Err(format!("Failed to generate image: {}", e)),
|
||||
}
|
||||
}
|
||||
+360
-326
@@ -16,367 +16,401 @@ use tokenizers::Tokenizer;
|
||||
|
||||
// Helper function to save image
|
||||
fn save_image<P: AsRef<std::path::Path>>(img: &Tensor, p: P) -> Result<()> {
|
||||
println!("Saving image to {:?}", p.as_ref());
|
||||
let p = p.as_ref();
|
||||
let (channel, height, width) = img.dims3()?;
|
||||
println!("Image dimensions: {}x{} with {} channels", width, height, channel);
|
||||
|
||||
if channel != 3 {
|
||||
anyhow::bail!("save_image expects an input of shape (3, height, width)")
|
||||
}
|
||||
let img = img.permute((1, 2, 0))?.flatten_all()?;
|
||||
let pixels = img.to_vec1::<u8>()?;
|
||||
println!("Converting tensor to image buffer...");
|
||||
|
||||
let image: ImageBuffer<Rgb<u8>, Vec<u8>> =
|
||||
match ImageBuffer::from_raw(width as u32, height as u32, pixels) {
|
||||
Some(image) => image,
|
||||
None => anyhow::bail!("error saving image {p:?}"),
|
||||
};
|
||||
println!("Successfully created image buffer");
|
||||
|
||||
image.save(p).map_err(E::from)?;
|
||||
println!("Successfully saved image to disk");
|
||||
Ok(())
|
||||
println!("Saving image to {:?}", p.as_ref());
|
||||
let p = p.as_ref();
|
||||
let (channel, height, width) = img.dims3()?;
|
||||
println!("Image dimensions: {}x{} with {} channels", width, height, channel);
|
||||
|
||||
if channel != 3 {
|
||||
anyhow::bail!("save_image expects an input of shape (3, height, width)")
|
||||
}
|
||||
let img = img.permute((1, 2, 0))?.flatten_all()?;
|
||||
let pixels = img.to_vec1::<u8>()?;
|
||||
println!("Converting tensor to image buffer...");
|
||||
|
||||
let image: ImageBuffer<Rgb<u8>, Vec<u8>> = match ImageBuffer::from_raw(width as u32, height as u32, pixels) {
|
||||
Some(image) => image,
|
||||
None => anyhow::bail!("error saving image {p:?}"),
|
||||
};
|
||||
println!("Successfully created image buffer");
|
||||
|
||||
image.save(p).map_err(E::from)?;
|
||||
println!("Successfully saved image to disk");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum StableDiffusionVersion {
|
||||
V1_5,
|
||||
V1_5Inpaint,
|
||||
V2_1,
|
||||
V2Inpaint,
|
||||
Xl,
|
||||
XlInpaint,
|
||||
Turbo,
|
||||
V1_5,
|
||||
V1_5Inpaint,
|
||||
V2_1,
|
||||
V2Inpaint,
|
||||
Xl,
|
||||
XlInpaint,
|
||||
Turbo,
|
||||
}
|
||||
|
||||
pub struct Args {
|
||||
pub prompt: String,
|
||||
pub uncond_prompt: String,
|
||||
pub cpu: bool,
|
||||
pub height: Option<usize>,
|
||||
pub width: Option<usize>,
|
||||
pub n_steps: Option<usize>,
|
||||
pub num_samples: usize,
|
||||
pub seed: Option<u64>,
|
||||
pub sd_version: StableDiffusionVersion,
|
||||
pub guidance_scale: Option<f64>,
|
||||
pub prompt: String,
|
||||
pub uncond_prompt: String,
|
||||
pub cpu: bool,
|
||||
pub height: Option<usize>,
|
||||
pub width: Option<usize>,
|
||||
pub n_steps: Option<usize>,
|
||||
pub num_samples: usize,
|
||||
pub seed: Option<u64>,
|
||||
pub sd_version: StableDiffusionVersion,
|
||||
pub guidance_scale: Option<f64>,
|
||||
pub img2img: Option<String>,
|
||||
pub img2img_strength: f64,
|
||||
pub use_f16: bool,
|
||||
}
|
||||
|
||||
impl StableDiffusionVersion {
|
||||
fn repo(&self) -> &'static str {
|
||||
match self {
|
||||
Self::XlInpaint => "diffusers/stable-diffusion-xl-1.0-inpainting-0.1",
|
||||
Self::Xl => "stabilityai/stable-diffusion-xl-base-1.0",
|
||||
Self::V2Inpaint => "stabilityai/stable-diffusion-2-inpainting",
|
||||
Self::V2_1 => "stabilityai/stable-diffusion-2-1",
|
||||
Self::V1_5 => "runwayml/stable-diffusion-v1-5",
|
||||
Self::V1_5Inpaint => "stable-diffusion-v1-5/stable-diffusion-inpainting",
|
||||
Self::Turbo => "stabilityai/sdxl-turbo",
|
||||
}
|
||||
fn repo(&self) -> &'static str {
|
||||
match self {
|
||||
Self::XlInpaint => "diffusers/stable-diffusion-xl-1.0-inpainting-0.1",
|
||||
Self::Xl => "stabilityai/stable-diffusion-xl-base-1.0",
|
||||
Self::V2Inpaint => "stabilityai/stable-diffusion-2-inpainting",
|
||||
Self::V2_1 => "stabilityai/stable-diffusion-2-1",
|
||||
Self::V1_5 => "runwayml/stable-diffusion-v1-5",
|
||||
Self::V1_5Inpaint => "stable-diffusion-v1-5/stable-diffusion-inpainting",
|
||||
Self::Turbo => "stabilityai/sdxl-turbo",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ModelFile {
|
||||
Tokenizer,
|
||||
Tokenizer2,
|
||||
Clip,
|
||||
Clip2,
|
||||
Unet,
|
||||
Vae,
|
||||
Tokenizer,
|
||||
Tokenizer2,
|
||||
Clip,
|
||||
Clip2,
|
||||
Unet,
|
||||
Vae,
|
||||
}
|
||||
|
||||
impl ModelFile {
|
||||
fn get(
|
||||
&self,
|
||||
filename: Option<String>,
|
||||
version: StableDiffusionVersion,
|
||||
use_f16: bool,
|
||||
) -> Result<std::path::PathBuf> {
|
||||
use hf_hub::api::sync::Api;
|
||||
match filename {
|
||||
Some(filename) => Ok(std::path::PathBuf::from(filename)),
|
||||
None => {
|
||||
let (repo, path) = match self {
|
||||
Self::Tokenizer => {
|
||||
let tokenizer_repo = match version {
|
||||
StableDiffusionVersion::V1_5
|
||||
| StableDiffusionVersion::V2_1
|
||||
| StableDiffusionVersion::V1_5Inpaint
|
||||
| StableDiffusionVersion::V2Inpaint => "openai/clip-vit-base-patch32",
|
||||
StableDiffusionVersion::Xl
|
||||
| StableDiffusionVersion::XlInpaint
|
||||
| StableDiffusionVersion::Turbo => {
|
||||
"openai/clip-vit-large-patch14"
|
||||
}
|
||||
};
|
||||
(tokenizer_repo, "tokenizer.json")
|
||||
}
|
||||
Self::Tokenizer2 => {
|
||||
("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", "tokenizer.json")
|
||||
}
|
||||
Self::Clip => (version.repo(), "text_encoder/model.safetensors"),
|
||||
Self::Clip2 => (version.repo(), "text_encoder_2/model.safetensors"),
|
||||
Self::Unet => (version.repo(), "unet/diffusion_pytorch_model.safetensors"),
|
||||
Self::Vae => (version.repo(), "vae/diffusion_pytorch_model.safetensors"),
|
||||
};
|
||||
let filename = Api::new()?.model(repo.to_string()).get(path)?;
|
||||
Ok(filename)
|
||||
fn get(&self, filename: Option<String>, version: StableDiffusionVersion, use_f16: bool) -> Result<std::path::PathBuf> {
|
||||
use hf_hub::api::sync::Api;
|
||||
match filename {
|
||||
Some(filename) => Ok(std::path::PathBuf::from(filename)),
|
||||
None => {
|
||||
let (repo, path) = match self {
|
||||
Self::Tokenizer => {
|
||||
let tokenizer_repo = match version {
|
||||
StableDiffusionVersion::V1_5 | StableDiffusionVersion::V2_1 | StableDiffusionVersion::V1_5Inpaint | StableDiffusionVersion::V2Inpaint => "openai/clip-vit-base-patch32",
|
||||
StableDiffusionVersion::Xl | StableDiffusionVersion::XlInpaint | StableDiffusionVersion::Turbo => "openai/clip-vit-large-patch14",
|
||||
};
|
||||
(tokenizer_repo, "tokenizer.json")
|
||||
},
|
||||
Self::Tokenizer2 => ("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", "tokenizer.json"),
|
||||
Self::Clip => (version.repo(), if use_f16 { "text_encoder/model.fp16.safetensors" } else { "text_encoder/model.safetensors" }),
|
||||
Self::Clip2 => (version.repo(), if use_f16 { "text_encoder_2/model.fp16.safetensors" } else { "text_encoder_2/model.safetensors" }),
|
||||
Self::Unet => (version.repo(), if use_f16 { "unet/diffusion_pytorch_model.fp16.safetensors" } else { "unet/diffusion_pytorch_model.safetensors" }),
|
||||
Self::Vae => {
|
||||
if matches!(version, StableDiffusionVersion::Xl | StableDiffusionVersion::Turbo) && use_f16 {
|
||||
("madebyollin/sdxl-vae-fp16-fix", "diffusion_pytorch_model.safetensors")
|
||||
} else {
|
||||
(version.repo(), if use_f16 { "vae/diffusion_pytorch_model.fp16.safetensors" } else { "vae/diffusion_pytorch_model.safetensors" })
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
let filename = Api::new()?.model(repo.to_string()).get(path)?;
|
||||
Ok(filename)
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run(args: Args) -> Result<()> {
|
||||
println!("Starting image generation with the following configuration:");
|
||||
println!(" Model: {:?}", args.sd_version);
|
||||
println!(" Prompt: {}", args.prompt);
|
||||
println!(" Steps: {}", args.n_steps.unwrap_or(1));
|
||||
println!(" Using CPU: {}", args.cpu);
|
||||
|
||||
let device = if args.cpu {
|
||||
println!("Using CPU for computation");
|
||||
Device::Cpu
|
||||
} else {
|
||||
println!("Attempting to use CUDA device");
|
||||
match Device::new_cuda(0) {
|
||||
Ok(cuda_device) => {
|
||||
println!("Successfully initialized CUDA device");
|
||||
cuda_device
|
||||
}
|
||||
Err(e) => {
|
||||
println!("Failed to initialize CUDA device: {}. Falling back to CPU", e);
|
||||
Device::Cpu
|
||||
}
|
||||
}
|
||||
};
|
||||
let dtype = DType::F32;
|
||||
println!("Using {:?} data type", dtype);
|
||||
println!("Starting image generation with the following configuration:");
|
||||
println!(" Model: {:?}", args.sd_version);
|
||||
println!(" Prompt: {}", args.prompt);
|
||||
println!(" Steps: {}", args.n_steps.unwrap_or(1));
|
||||
println!(" Using CPU: {}", args.cpu);
|
||||
|
||||
let sd_config = match args.sd_version {
|
||||
StableDiffusionVersion::Turbo => {
|
||||
println!("Configuring SDXL Turbo");
|
||||
stable_diffusion::StableDiffusionConfig::sdxl_turbo(None, args.height, args.width)
|
||||
}
|
||||
_ => {
|
||||
println!("Configuring SD v2.1");
|
||||
stable_diffusion::StableDiffusionConfig::v2_1(None, args.height, args.width)
|
||||
}
|
||||
};
|
||||
println!("Model dimensions: {}x{}", sd_config.width, sd_config.height);
|
||||
|
||||
println!("Building scheduler...");
|
||||
let mut scheduler = sd_config.build_scheduler(args.n_steps.unwrap_or(1))?;
|
||||
let seed = args.seed.unwrap_or_else(|| rand::thread_rng().gen());
|
||||
println!("Using seed: {}", seed);
|
||||
device.set_seed(seed)?;
|
||||
|
||||
println!("Initializing Hugging Face API...");
|
||||
let api = hf_hub::api::sync::Api::new()?;
|
||||
let repo = args.sd_version.repo();
|
||||
println!("Downloading model files from: {}", repo);
|
||||
|
||||
println!("Downloading VAE and UNet...");
|
||||
let vae_file = api.model(repo.to_string()).get("vae/diffusion_pytorch_model.safetensors")?;
|
||||
let unet_file = api.model(repo.to_string()).get("unet/diffusion_pytorch_model.safetensors")?;
|
||||
|
||||
println!("Downloading text encoders...");
|
||||
let clip_file = api.model(repo.to_string()).get("text_encoder/model.safetensors")?;
|
||||
let clip2_file = api.model(repo.to_string()).get("text_encoder_2/model.safetensors")?;
|
||||
let tokenizer = api.model("openai/clip-vit-large-patch14".to_string()).get("tokenizer.json")?;
|
||||
let tokenizer2 = api.model("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k".to_string()).get("tokenizer.json")?;
|
||||
|
||||
println!("Building models...");
|
||||
let vae = sd_config.build_vae(vae_file, &device, dtype)?;
|
||||
let unet = sd_config.build_unet(unet_file, &device, 4, false, dtype)?;
|
||||
|
||||
// Build text encoders
|
||||
println!("Building text encoders...");
|
||||
let text_model = stable_diffusion::build_clip_transformer(&sd_config.clip, clip_file, &device, dtype)?;
|
||||
let text_model2 = stable_diffusion::build_clip_transformer(sd_config.clip2.as_ref().unwrap(), clip2_file, &device, dtype)?;
|
||||
|
||||
// Tokenize and encode text
|
||||
println!("Processing text prompt...");
|
||||
let tokenizer = tokenizers::Tokenizer::from_file(tokenizer).map_err(E::msg)?;
|
||||
let tokenizer2 = tokenizers::Tokenizer::from_file(tokenizer2).map_err(E::msg)?;
|
||||
|
||||
println!("Preparing text embeddings...");
|
||||
let which = match args.sd_version {
|
||||
StableDiffusionVersion::Xl
|
||||
| StableDiffusionVersion::XlInpaint
|
||||
| StableDiffusionVersion::Turbo => vec![true, false],
|
||||
_ => vec![true],
|
||||
};
|
||||
|
||||
let text_embeddings = which
|
||||
.iter()
|
||||
.map(|first| {
|
||||
text_embeddings(
|
||||
&args.prompt,
|
||||
&args.uncond_prompt,
|
||||
None, // tokenizer
|
||||
None, // clip_weights
|
||||
None, // clip2_weights
|
||||
args.sd_version,
|
||||
&sd_config,
|
||||
false, // use_f16
|
||||
&device,
|
||||
dtype,
|
||||
false, // use_guide_scale
|
||||
*first,
|
||||
)
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
|
||||
let text_embeddings = Tensor::cat(&text_embeddings, D::Minus1)?;
|
||||
println!("Text embeddings shape: {:?}", text_embeddings.shape());
|
||||
|
||||
println!("Generating initial noise...");
|
||||
let timesteps = scheduler.timesteps().to_vec();
|
||||
let latents = Tensor::randn(
|
||||
0f32,
|
||||
1f32,
|
||||
(1, 4, sd_config.height / 8, sd_config.width / 8),
|
||||
&device,
|
||||
)?;
|
||||
let latents = (latents * scheduler.init_noise_sigma())?;
|
||||
println!("Initial noise shape: {:?}", latents.shape());
|
||||
|
||||
println!("Starting diffusion process...");
|
||||
let mut latents = latents;
|
||||
for (timestep_index, ×tep) in timesteps.iter().enumerate() {
|
||||
println!("Processing step {}/{}", timestep_index + 1, timesteps.len());
|
||||
let latent_model_input = latents.clone();
|
||||
println!("Latent input shape: {:?}", latent_model_input.shape());
|
||||
|
||||
println!("Scaling model input...");
|
||||
let latent_model_input = scheduler.scale_model_input(latent_model_input, timestep)?;
|
||||
println!("Scaled input shape: {:?}", latent_model_input.shape());
|
||||
|
||||
println!("Running UNet inference with timestep {}...", timestep);
|
||||
let noise_pred = match unet.forward(&latent_model_input, timestep as f64, &text_embeddings) {
|
||||
Ok(pred) => {
|
||||
println!("UNet inference successful");
|
||||
pred
|
||||
}
|
||||
Err(e) => {
|
||||
println!("UNet inference failed with error: {}", e);
|
||||
return Err(anyhow::anyhow!("UNet inference failed: {}", e));
|
||||
}
|
||||
};
|
||||
println!("Noise prediction shape: {:?}", noise_pred.shape());
|
||||
|
||||
println!("Applying scheduler step...");
|
||||
latents = scheduler.step(&noise_pred, timestep, &latents)?;
|
||||
println!("Step {}/{} completed", timestep_index + 1, timesteps.len());
|
||||
let device = if args.cpu {
|
||||
println!("Using CPU for computation");
|
||||
Device::Cpu
|
||||
} else {
|
||||
println!("Attempting to use CUDA device");
|
||||
match Device::new_cuda(0) {
|
||||
Ok(cuda_device) => {
|
||||
println!("Successfully initialized CUDA device");
|
||||
cuda_device
|
||||
},
|
||||
Err(e) => {
|
||||
println!("Failed to initialize CUDA device: {}. Falling back to CPU", e);
|
||||
Device::Cpu
|
||||
},
|
||||
}
|
||||
};
|
||||
let dtype = if args.use_f16 {
|
||||
println!("Using FP16 for reduced memory usage");
|
||||
DType::F16
|
||||
} else {
|
||||
println!("Using FP32");
|
||||
DType::F32
|
||||
};
|
||||
println!("Using {:?} data type", dtype);
|
||||
|
||||
println!("Diffusion process completed, decoding image...");
|
||||
let image = vae.decode(&(latents / 0.13025)?)?;
|
||||
println!("VAE decode completed");
|
||||
let image = ((image / 2.)? + 0.5)?.to_device(&Device::Cpu)?;
|
||||
println!("Normalized image values");
|
||||
let image = (image.clamp(0f32, 1.)? * 255.)?.to_dtype(DType::U8)?;
|
||||
println!("Converted to 8-bit format");
|
||||
|
||||
save_image(&image.i(0)?, "temp.png")?;
|
||||
println!("Image generation completed successfully");
|
||||
let sd_config = match args.sd_version {
|
||||
StableDiffusionVersion::Turbo => {
|
||||
println!("Configuring SDXL Turbo");
|
||||
stable_diffusion::StableDiffusionConfig::sdxl_turbo(None, args.height, args.width)
|
||||
},
|
||||
_ => {
|
||||
println!("Configuring SD v2.1");
|
||||
stable_diffusion::StableDiffusionConfig::v2_1(None, args.height, args.width)
|
||||
},
|
||||
};
|
||||
println!("Model dimensions: {}x{}", sd_config.width, sd_config.height);
|
||||
|
||||
Ok(())
|
||||
println!("Building scheduler...");
|
||||
let mut scheduler = sd_config.build_scheduler(args.n_steps.unwrap_or(1))?;
|
||||
let seed = args.seed.unwrap_or_else(|| rand::thread_rng().gen());
|
||||
println!("Using seed: {}", seed);
|
||||
device.set_seed(seed)?;
|
||||
|
||||
println!("Initializing Hugging Face API...");
|
||||
let api = hf_hub::api::sync::Api::new()?;
|
||||
let repo = args.sd_version.repo();
|
||||
println!("Downloading model files from: {}", repo);
|
||||
|
||||
println!("Downloading model files...");
|
||||
let vae_file = ModelFile::Vae.get(None, args.sd_version, args.use_f16)?;
|
||||
let unet_file = ModelFile::Unet.get(None, args.sd_version, args.use_f16)?;
|
||||
let clip_file = ModelFile::Clip.get(None, args.sd_version, args.use_f16)?;
|
||||
let clip2_file = ModelFile::Clip2.get(None, args.sd_version, args.use_f16)?;
|
||||
let tokenizer = ModelFile::Tokenizer.get(None, args.sd_version, args.use_f16)?;
|
||||
let tokenizer2 = ModelFile::Tokenizer2.get(None, args.sd_version, args.use_f16)?;
|
||||
|
||||
println!("Building models...");
|
||||
let vae = sd_config.build_vae(vae_file, &device, dtype)?;
|
||||
let unet = sd_config.build_unet(unet_file, &device, 4, false, dtype)?;
|
||||
|
||||
// Build text encoders
|
||||
println!("Building text encoders...");
|
||||
let text_model = stable_diffusion::build_clip_transformer(&sd_config.clip, clip_file.clone(), &device, dtype)?;
|
||||
let text_model2 = stable_diffusion::build_clip_transformer(sd_config.clip2.as_ref().unwrap(), clip2_file.clone(), &device, dtype)?;
|
||||
|
||||
// Tokenize and encode text
|
||||
println!("Processing text prompt...");
|
||||
let tokenizer = tokenizers::Tokenizer::from_file(tokenizer).map_err(E::msg)?;
|
||||
let tokenizer2 = tokenizers::Tokenizer::from_file(tokenizer2).map_err(E::msg)?;
|
||||
|
||||
println!("Preparing text embeddings...");
|
||||
let which = match args.sd_version {
|
||||
StableDiffusionVersion::Xl | StableDiffusionVersion::XlInpaint | StableDiffusionVersion::Turbo => vec![true, false],
|
||||
_ => vec![true],
|
||||
};
|
||||
|
||||
let guidance_scale = match args.guidance_scale {
|
||||
Some(guidance_scale) => guidance_scale,
|
||||
None => match args.sd_version {
|
||||
StableDiffusionVersion::V1_5 | StableDiffusionVersion::V1_5Inpaint | StableDiffusionVersion::V2_1 | StableDiffusionVersion::V2Inpaint | StableDiffusionVersion::XlInpaint | StableDiffusionVersion::Xl => 7.5,
|
||||
StableDiffusionVersion::Turbo => 0.,
|
||||
},
|
||||
};
|
||||
let use_guide_scale = guidance_scale > 1.0;
|
||||
|
||||
// Clone the PathBufs before converting to strings
|
||||
let clip_file_str = clip_file.clone().to_string_lossy().to_string();
|
||||
let clip2_file_str = clip2_file.clone().to_string_lossy().to_string();
|
||||
|
||||
let text_embeddings = which.iter().map(|first| text_embeddings(&args.prompt, &args.uncond_prompt, None, Some(clip_file_str.clone()), Some(clip2_file_str.clone()), args.sd_version, &sd_config, args.use_f16, &device, dtype, use_guide_scale, *first)).collect::<Result<Vec<_>>>()?;
|
||||
|
||||
let text_embeddings = Tensor::cat(&text_embeddings, D::Minus1)?;
|
||||
println!("Text embeddings shape: {:?}", text_embeddings.shape());
|
||||
|
||||
// Add this before latent initialization
|
||||
let vae_scale = match args.sd_version {
|
||||
StableDiffusionVersion::V1_5 | StableDiffusionVersion::V1_5Inpaint | StableDiffusionVersion::V2_1 | StableDiffusionVersion::V2Inpaint | StableDiffusionVersion::XlInpaint | StableDiffusionVersion::Xl => 0.18215,
|
||||
StableDiffusionVersion::Turbo => 0.13025,
|
||||
};
|
||||
println!("Using VAE scale factor: {}", vae_scale);
|
||||
|
||||
// Add this section after building the VAE
|
||||
println!("Starting img2img processing...");
|
||||
let (image, init_latent_dist) = match &args.img2img {
|
||||
None => {
|
||||
println!("No input image provided, will generate from scratch");
|
||||
(None, None)
|
||||
},
|
||||
Some(image_path) => {
|
||||
println!("Processing input image from path: {}", image_path);
|
||||
let image = match image_preprocess(image_path) {
|
||||
Ok(img) => {
|
||||
println!("Successfully preprocessed input image");
|
||||
img
|
||||
},
|
||||
Err(e) => {
|
||||
println!("Failed to preprocess input image: {}", e);
|
||||
return Err(e);
|
||||
},
|
||||
};
|
||||
|
||||
println!("Moving image to device and converting dtype...");
|
||||
let image = image.to_device(&device)?.to_dtype(dtype)?;
|
||||
println!("Image shape after preprocessing: {:?}", image.shape());
|
||||
|
||||
println!("Encoding image through VAE...");
|
||||
let encoded = match vae.encode(&image) {
|
||||
Ok(encoded) => {
|
||||
println!("Successfully encoded image through VAE");
|
||||
encoded
|
||||
},
|
||||
Err(e) => {
|
||||
println!("Failed to encode image through VAE: {}", e);
|
||||
return Err(e.into());
|
||||
},
|
||||
};
|
||||
(Some(image.clone()), Some(encoded))
|
||||
},
|
||||
};
|
||||
|
||||
println!("Calculating start step for diffusion process...");
|
||||
let t_start = if args.img2img.is_some() {
|
||||
let start = args.n_steps.unwrap_or(1) - (args.n_steps.unwrap_or(1) as f64 * args.img2img_strength) as usize;
|
||||
println!("Starting from step {} of {} (strength: {})", start, args.n_steps.unwrap_or(1), args.img2img_strength);
|
||||
start
|
||||
} else {
|
||||
println!("Starting from beginning (step 0)");
|
||||
0
|
||||
};
|
||||
|
||||
println!("Initializing latents...");
|
||||
let latents = match &init_latent_dist {
|
||||
Some(init_latent_dist) => {
|
||||
println!("Generating latents from input image...");
|
||||
let latents = (init_latent_dist.sample()? * vae_scale)?.to_device(&device)?;
|
||||
println!("Initial latents shape: {:?}", latents.shape());
|
||||
|
||||
if t_start < scheduler.timesteps().len() {
|
||||
println!("Adding noise to latents...");
|
||||
let noise = latents.randn_like(0f64, 1f64)?;
|
||||
scheduler.add_noise(&latents, noise, scheduler.timesteps()[t_start])?
|
||||
} else {
|
||||
println!("Using latents directly (no noise addition needed)");
|
||||
latents
|
||||
}
|
||||
},
|
||||
None => {
|
||||
println!("Generating random latents...");
|
||||
let latents = Tensor::randn(0f32, 1f32, (1, 4, sd_config.height / 8, sd_config.width / 8), &device)?;
|
||||
println!("Random latents shape: {:?}", latents.shape());
|
||||
// scale the initial noise by the standard deviation required by the scheduler
|
||||
(latents * scheduler.init_noise_sigma())?
|
||||
},
|
||||
};
|
||||
println!("Latents initialized successfully");
|
||||
|
||||
// Get timesteps before the loop
|
||||
let timesteps: Vec<_> = scheduler.timesteps().iter().copied().collect();
|
||||
println!("Starting diffusion process...");
|
||||
let mut latents = latents;
|
||||
for (timestep_index, ×tep) in timesteps.iter().enumerate() {
|
||||
if timestep_index < t_start {
|
||||
continue;
|
||||
}
|
||||
println!("Processing step {}/{}", timestep_index + 1, timesteps.len());
|
||||
let latent_model_input = latents.clone();
|
||||
println!("Latent input shape: {:?}", latent_model_input.shape());
|
||||
|
||||
println!("Scaling model input...");
|
||||
let latent_model_input = scheduler.scale_model_input(latent_model_input, timestep)?;
|
||||
println!("Scaled input shape: {:?}", latent_model_input.shape());
|
||||
|
||||
println!("Running UNet inference with timestep {}...", timestep);
|
||||
let noise_pred = match unet.forward(&latent_model_input, timestep as f64, &text_embeddings) {
|
||||
Ok(pred) => {
|
||||
println!("UNet inference successful");
|
||||
pred
|
||||
},
|
||||
Err(e) => {
|
||||
println!("UNet inference failed with error: {}", e);
|
||||
return Err(anyhow::anyhow!("UNet inference failed: {}", e));
|
||||
},
|
||||
};
|
||||
println!("Noise prediction shape: {:?}", noise_pred.shape());
|
||||
|
||||
println!("Applying scheduler step...");
|
||||
latents = scheduler.step(&noise_pred, timestep, &latents)?;
|
||||
println!("Step {}/{} completed", timestep_index + 1, timesteps.len());
|
||||
}
|
||||
|
||||
println!("Diffusion process completed, decoding image...");
|
||||
let image = vae.decode(&(latents.div(vae_scale)?))?;
|
||||
println!("VAE decode completed");
|
||||
let image = ((image.div(2.))? + 0.5)?.to_device(&Device::Cpu)?;
|
||||
println!("Normalized image values");
|
||||
let image = (image.clamp(0f32, 1.)? * 255.)?.to_dtype(DType::U8)?;
|
||||
println!("Converted to 8-bit format");
|
||||
|
||||
save_image(&image.i(0)?, "temp.png")?;
|
||||
println!("Image generation completed successfully");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn text_embeddings(
|
||||
prompt: &str,
|
||||
uncond_prompt: &str,
|
||||
tokenizer: Option<String>,
|
||||
clip_weights: Option<String>,
|
||||
clip2_weights: Option<String>,
|
||||
sd_version: StableDiffusionVersion,
|
||||
sd_config: &stable_diffusion::StableDiffusionConfig,
|
||||
use_f16: bool,
|
||||
device: &Device,
|
||||
dtype: DType,
|
||||
use_guide_scale: bool,
|
||||
first: bool,
|
||||
) -> Result<Tensor> {
|
||||
let tokenizer_file = if first {
|
||||
ModelFile::Tokenizer
|
||||
} else {
|
||||
ModelFile::Tokenizer2
|
||||
};
|
||||
let tokenizer = tokenizer_file.get(tokenizer, sd_version, use_f16)?;
|
||||
let tokenizer = Tokenizer::from_file(tokenizer).map_err(E::msg)?;
|
||||
let pad_id = match &sd_config.clip.pad_with {
|
||||
Some(padding) => *tokenizer.get_vocab(true).get(padding.as_str()).unwrap(),
|
||||
None => *tokenizer.get_vocab(true).get("<|endoftext|>").unwrap(),
|
||||
};
|
||||
|
||||
println!("Running with prompt \"{prompt}\".");
|
||||
let mut tokens = tokenizer
|
||||
.encode(prompt, true)
|
||||
.map_err(E::msg)?
|
||||
.get_ids()
|
||||
.to_vec();
|
||||
if tokens.len() > sd_config.clip.max_position_embeddings {
|
||||
anyhow::bail!(
|
||||
"the prompt is too long, {} > max-tokens ({})",
|
||||
tokens.len(),
|
||||
sd_config.clip.max_position_embeddings
|
||||
)
|
||||
fn text_embeddings(prompt: &str, uncond_prompt: &str, tokenizer: Option<String>, clip_weights: Option<String>, clip2_weights: Option<String>, sd_version: StableDiffusionVersion, sd_config: &stable_diffusion::StableDiffusionConfig, use_f16: bool, device: &Device, dtype: DType, use_guide_scale: bool, first: bool) -> Result<Tensor> {
|
||||
let tokenizer_file = if first { ModelFile::Tokenizer } else { ModelFile::Tokenizer2 };
|
||||
let tokenizer = tokenizer_file.get(tokenizer, sd_version, use_f16)?;
|
||||
let tokenizer = Tokenizer::from_file(tokenizer).map_err(E::msg)?;
|
||||
let pad_id = match &sd_config.clip.pad_with {
|
||||
Some(padding) => *tokenizer.get_vocab(true).get(padding.as_str()).unwrap(),
|
||||
None => *tokenizer.get_vocab(true).get("<|endoftext|>").unwrap(),
|
||||
};
|
||||
|
||||
println!("Running with prompt \"{prompt}\".");
|
||||
let mut tokens = tokenizer.encode(prompt, true).map_err(E::msg)?.get_ids().to_vec();
|
||||
if tokens.len() > sd_config.clip.max_position_embeddings {
|
||||
anyhow::bail!("the prompt is too long, {} > max-tokens ({})", tokens.len(), sd_config.clip.max_position_embeddings)
|
||||
}
|
||||
while tokens.len() < sd_config.clip.max_position_embeddings {
|
||||
tokens.push(pad_id)
|
||||
}
|
||||
let tokens = Tensor::new(tokens.as_slice(), device)?.unsqueeze(0)?;
|
||||
|
||||
println!("Building the Clip transformer.");
|
||||
let clip_weights_file = if first { ModelFile::Clip } else { ModelFile::Clip2 };
|
||||
let clip_weights = if first { clip_weights_file.get(clip_weights, sd_version, use_f16)? } else { clip_weights_file.get(clip2_weights, sd_version, use_f16)? };
|
||||
let clip_config = if first { &sd_config.clip } else { sd_config.clip2.as_ref().unwrap() };
|
||||
let text_model = stable_diffusion::build_clip_transformer(clip_config, clip_weights, device, DType::F32)?;
|
||||
|
||||
let text_embeddings = text_model.forward(&tokens)?;
|
||||
|
||||
let text_embeddings = if use_guide_scale {
|
||||
let mut uncond_tokens = tokenizer.encode(uncond_prompt, true).map_err(E::msg)?.get_ids().to_vec();
|
||||
if uncond_tokens.len() > sd_config.clip.max_position_embeddings {
|
||||
anyhow::bail!("the negative prompt is too long, {} > max-tokens ({})", uncond_tokens.len(), sd_config.clip.max_position_embeddings)
|
||||
}
|
||||
while tokens.len() < sd_config.clip.max_position_embeddings {
|
||||
tokens.push(pad_id)
|
||||
while uncond_tokens.len() < sd_config.clip.max_position_embeddings {
|
||||
uncond_tokens.push(pad_id)
|
||||
}
|
||||
let tokens = Tensor::new(tokens.as_slice(), device)?.unsqueeze(0)?;
|
||||
|
||||
println!("Building the Clip transformer.");
|
||||
let clip_weights_file = if first {
|
||||
ModelFile::Clip
|
||||
} else {
|
||||
ModelFile::Clip2
|
||||
};
|
||||
let clip_weights = if first {
|
||||
clip_weights_file.get(clip_weights, sd_version, use_f16)?
|
||||
} else {
|
||||
clip_weights_file.get(clip2_weights, sd_version, use_f16)?
|
||||
};
|
||||
let clip_config = if first {
|
||||
&sd_config.clip
|
||||
} else {
|
||||
sd_config.clip2.as_ref().unwrap()
|
||||
};
|
||||
let text_model = stable_diffusion::build_clip_transformer(clip_config, clip_weights, device, DType::F32)?;
|
||||
|
||||
let text_embeddings = text_model.forward(&tokens)?;
|
||||
let uncond_tokens = Tensor::new(uncond_tokens.as_slice(), device)?.unsqueeze(0)?;
|
||||
let uncond_embeddings = text_model.forward(&uncond_tokens)?;
|
||||
|
||||
let text_embeddings = if use_guide_scale {
|
||||
let mut uncond_tokens = tokenizer
|
||||
.encode(uncond_prompt, true)
|
||||
.map_err(E::msg)?
|
||||
.get_ids()
|
||||
.to_vec();
|
||||
if uncond_tokens.len() > sd_config.clip.max_position_embeddings {
|
||||
anyhow::bail!(
|
||||
"the negative prompt is too long, {} > max-tokens ({})",
|
||||
uncond_tokens.len(),
|
||||
sd_config.clip.max_position_embeddings
|
||||
)
|
||||
}
|
||||
while uncond_tokens.len() < sd_config.clip.max_position_embeddings {
|
||||
uncond_tokens.push(pad_id)
|
||||
}
|
||||
Tensor::cat(&[uncond_embeddings, text_embeddings], 0)?.to_dtype(dtype)?
|
||||
} else {
|
||||
text_embeddings.to_dtype(dtype)?
|
||||
};
|
||||
Ok(text_embeddings)
|
||||
}
|
||||
|
||||
let uncond_tokens = Tensor::new(uncond_tokens.as_slice(), device)?.unsqueeze(0)?;
|
||||
let uncond_embeddings = text_model.forward(&uncond_tokens)?;
|
||||
|
||||
Tensor::cat(&[uncond_embeddings, text_embeddings], 0)?.to_dtype(dtype)?
|
||||
} else {
|
||||
text_embeddings.to_dtype(dtype)?
|
||||
};
|
||||
Ok(text_embeddings)
|
||||
fn image_preprocess<T: AsRef<std::path::Path>>(path: T) -> Result<Tensor> {
|
||||
let img = image::ImageReader::open(path)?.decode()?;
|
||||
let (height, width) = (img.height() as usize, img.width() as usize);
|
||||
let height = height - height % 32;
|
||||
let width = width - width % 32;
|
||||
let img = img.resize_to_fill(width as u32, height as u32, image::imageops::FilterType::CatmullRom);
|
||||
let img = img.to_rgb8();
|
||||
let img = img.into_raw();
|
||||
let img = Tensor::from_vec(img, (height, width, 3), &Device::Cpu)?.permute((2, 0, 1))?.to_dtype(DType::F32)?.affine(2. / 255., -1.)?.unsqueeze(0)?;
|
||||
Ok(img)
|
||||
}
|
||||
|
||||
// fn main() -> Result<()> {
|
||||
|
||||
Reference in new issue
Block a user