diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs index d081b1ff56..572564e086 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs @@ -32,6 +32,7 @@ use log::{error, info, warn}; use serde::{Deserialize, Serialize}; use sqlite_tasks::queries::create_task::{create_task, CreateTaskArgs}; use tauri::{AppHandle, State}; +use tokens::tokens::media_files::MediaFileToken; /// This is used in the Tauri command bridge. /// Don't change the serializations without coordinating with the frontend. @@ -73,6 +74,13 @@ pub struct EnqueueTextToImageRequest { /// The number of images to generate. pub number_images: Option, + /// OPTIONAL. + /// Reference images (without semantics) + /// The purpose varies on a model-by-model basis, but they + /// are not semantically treated as "style reference", + /// "character/object reference", etc. + pub image_media_tokens: Option>, + /// OPTIONAL. /// Name of the frontend caller. /// We'll use this to selectively trigger events. diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/gemini_25_flash/handle_gemini_25_flash_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/gemini_25_flash/handle_gemini_25_flash_artcraft.rs index 4018891f34..839c7ca873 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/gemini_25_flash/handle_gemini_25_flash_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/gemini_25_flash/handle_gemini_25_flash_artcraft.rs @@ -46,13 +46,6 @@ pub async fn handle_gemini_25_flash_artcraft( let uuid_idempotency_token = generate_random_uuid(); - // let image_media_token = match &request.image_media_token { - // Some(token) => token.clone(), - // None => { - // return Err(InternalImageInpaintError::NoSourceImageSpecified); - // }, - // }; - let num_images = match request.number_images{ None => None, Some(1) => Some(Gemini25FlashEditImageNumImages::One), @@ -65,7 +58,7 @@ pub async fn handle_gemini_25_flash_artcraft( let request = Gemini25FlashEditImageRequest { uuid_idempotency_token, prompt: request.prompt.clone(), - image_media_tokens: Some(vec![]), // TODO: Needs image media tokens. + image_media_tokens: request.image_media_tokens.clone(), num_images, image_quality: None, }; diff --git a/frontend/apps/editor3d/app/src/pages/PageImage/TextToImage.tsx b/frontend/apps/editor3d/app/src/pages/PageImage/TextToImage.tsx index c947975b0c..bac29a3e90 100644 --- a/frontend/apps/editor3d/app/src/pages/PageImage/TextToImage.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageImage/TextToImage.tsx @@ -1,6 +1,7 @@ import { useEffect, useMemo, useRef, useState } from "react"; import { JobContextType } from "@storyteller/common"; import { PromptBoxImage } from "@storyteller/ui-promptbox"; +import { uploadImage } from "../../components/reusable/UploadModalMedia/uploadImage"; import BackgroundGallery from "./BackgroundGallery"; import { TEXT_TO_IMAGE_PAGE_MODEL_LIST, @@ -21,6 +22,7 @@ import { galleryModalLightboxVisible, } from "@storyteller/ui-gallery-modal"; import { Badge } from "@storyteller/ui-badge"; +import { twMerge } from "tailwind-merge"; const PAGE_ID: ModelPage = ModelPage.TextToImage; @@ -30,8 +32,12 @@ const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => { const startBatch = useTextToImageStore((s) => s.startBatch); const completeBatch = useTextToImageStore((s) => s.completeBatch); const resetBatches = useTextToImageStore((s) => s.reset); + const [imageRowVisible, setImageRowVisible] = useState(false); + const promptContentRef = useRef(null); + const [promptHeight, setPromptHeight] = useState(138); - const selectedImageModel : ImageModel | undefined = getSelectedImageModel(PAGE_ID); + const selectedImageModel: ImageModel | undefined = + getSelectedImageModel(PAGE_ID); const jobContext: JobContextType = { jobTokens: [], @@ -59,7 +65,18 @@ const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => { return () => window.removeEventListener("resize", onResize); }, []); - const bottomOffsetPx = 138; + useEffect(() => { + const el = promptContentRef.current; + if (!el || typeof ResizeObserver === "undefined") return; + const update = () => setPromptHeight(el.offsetHeight); + update(); + const ro = new ResizeObserver(() => update()); + ro.observe(el); + return () => ro.disconnect(); + }, []); + + const bottomMarginPx = 24; + const bottomOffsetPx = promptHeight + bottomMarginPx; const targetTop = showPromptAtBottom ? Math.max(0, vh - bottomOffsetPx) : Math.floor(vh / 2); @@ -80,7 +97,12 @@ const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => {
{!showPromptAtBottom && ( -
+
Generate Image Add a prompt, then generate @@ -89,7 +111,10 @@ const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => { )} {hasAnyBatches && ( -
+
{inverseBatch.map((batch) => (
@@ -154,7 +179,9 @@ const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => { style={promptAnim} > {showPromptAtBottom && batches.length > 0 && ( -
+
)} - { - return jobContext; - }} - selectedModel={selectedImageModel} - imageMediaId={imageMediaId} - url={imageUrl ?? undefined} - onEnqueuePressed={async (prompt, count, subscriberId) => { - const modelLabel = selectedImageModel?.fullName ?? ""; - startBatch(prompt, count, modelLabel, subscriberId); - }} - /> +
+ { + return jobContext; + }} + uploadImage={uploadImage} + selectedModel={selectedImageModel} + imageMediaId={imageMediaId} + url={imageUrl ?? undefined} + onImageRowVisibilityChange={setImageRowVisible} + onEnqueuePressed={async (prompt, count, subscriberId) => { + const modelLabel = selectedImageModel?.fullName ?? ""; + startBatch(prompt, count, modelLabel, subscriberId); + }} + /> +
{!showPromptAtBottom && } diff --git a/frontend/libs/components/button/src/lib/button.tsx b/frontend/libs/components/button/src/lib/button.tsx index 28e10d9321..b67b0ef068 100644 --- a/frontend/libs/components/button/src/lib/button.tsx +++ b/frontend/libs/components/button/src/lib/button.tsx @@ -59,7 +59,7 @@ export const Button = ({ ); const className = twMerge( - "w-fit text-sm font-medium rounded-lg px-3 py-2 border border-transparent shadow-sm focus-visible:outline focus-visible:outline-0 focus-visible:outline-offset-0 transition-all duration-150 flex gap-2 items-center justify-center", + "w-fit text-sm font-medium rounded-lg px-3 py-2 border border-transparent shadow-sm focus-visible:outline focus-visible:outline-0 focus-visible:outline-offset-0 transition-all duration-150 flex gap-2 items-center justify-center active:scale-95 transform", getVariantClassNames(propsVariant), propsClassName, disabledClass diff --git a/frontend/libs/components/popover/src/lib/popover.tsx b/frontend/libs/components/popover/src/lib/popover.tsx index 9630fe88bb..4db86a8067 100644 --- a/frontend/libs/components/popover/src/lib/popover.tsx +++ b/frontend/libs/components/popover/src/lib/popover.tsx @@ -89,6 +89,7 @@ export const PopoverMenu = ({ "flex gap-2 items-center justify-center outline-none", "transition-all duration-150", "bg-[#5F5F68]/60 px-3 text-white hover:bg-[#5F5F68]/90", + "active:scale-95 transform", buttonClassName ); diff --git a/frontend/libs/components/promptbox/src/lib/PromptBoxImage.tsx b/frontend/libs/components/promptbox/src/lib/PromptBoxImage.tsx index f18dd8cb96..517873ab7d 100644 --- a/frontend/libs/components/promptbox/src/lib/PromptBoxImage.tsx +++ b/frontend/libs/components/promptbox/src/lib/PromptBoxImage.tsx @@ -1,13 +1,20 @@ -import { useState, useRef, useEffect } from "react"; +import { useState, useRef, useEffect, ReactNode } from "react"; +import { unstable_batchedUpdates } from "react-dom"; import { useSignals } from "@preact/signals-react/runtime"; -import { JobContextType } from "@storyteller/common"; +import { + JobContextType, + UploaderState, + UploaderStates, +} from "@storyteller/common"; import { downloadFileFromUrl } from "@storyteller/api"; +import { toast } from "@storyteller/ui-toaster"; import { PopoverMenu, PopoverItem } from "@storyteller/ui-popover"; import { Tooltip } from "@storyteller/ui-tooltip"; import { Button, ToggleButton } from "@storyteller/ui-button"; import { Modal } from "@storyteller/ui-modal"; import { EnqueueTextToImage, + EnqueueTextToImageRequest, EnqueueTextToImageSize, } from "@storyteller/tauri-api"; import { @@ -15,25 +22,37 @@ import { faMessageCheck, faSparkles, faSpinnerThird, - faTimes, - faPlus, - faImages, faCopy, + faTrashAlt, + faPlus, + faXmark, + faImages, } from "@fortawesome/pro-solid-svg-icons"; import { faRectangle, faSquare, faRectangleVertical, + faImage, } from "@fortawesome/pro-regular-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { IconProp } from "@fortawesome/fontawesome-svg-core"; import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal"; -import { ImageModel, ModelInfo, getCapabilitiesForModel } from "@storyteller/model-list"; +import { ImageModel, getCapabilitiesForModel } from "@storyteller/model-list"; import { usePromptImageStore, RefImage } from "./promptStore"; import { gtagEvent } from "@storyteller/google-analytics"; +import { twMerge } from "tailwind-merge"; interface PromptBoxImageProps { useJobContext: () => JobContextType; + uploadImage?: ({ + title, + assetFile, + progressCallback, + }: { + title: string; + assetFile: File; + progressCallback: (newState: UploaderState) => void; + }) => Promise; onEnqueuePressed?: ( prompt: string, count: number, @@ -42,14 +61,17 @@ interface PromptBoxImageProps { selectedModel?: ImageModel; imageMediaId?: string; url?: string; + onImageRowVisibilityChange?: (visible: boolean) => void; } export const PromptBoxImage = ({ useJobContext, + uploadImage, onEnqueuePressed, selectedModel, imageMediaId, url, + onImageRowVisibilityChange, }: PromptBoxImageProps) => { useSignals(); @@ -67,7 +89,7 @@ export const PromptBoxImage = ({ }, [imageMediaId, url]); const [isModalOpen, setIsModalOpen] = useState(false); - const [content, setContent] = useState(""); + const [content, setContent] = useState(null); const prompt = usePromptImageStore((s) => s.prompt); const setPrompt = usePromptImageStore((s) => s.setPrompt); @@ -86,6 +108,15 @@ export const PromptBoxImage = ({ const [uploadingImages, _setUploadingImages] = useState< { id: string; file: File }[] >([]); + const [showImagePrompts, setShowImagePrompts] = useState(false); + const isImageRowVisible = + showImagePrompts || + referenceImages.length > 0 || + uploadingImages.length > 0; + + useEffect(() => { + onImageRowVisibilityChange?.(isImageRowVisible); + }, [isImageRowVisible, onImageRowVisibilityChange]); const [aspectRatioList, setAspectRatioList] = useState([ { label: "3:2", @@ -198,51 +229,150 @@ export const PromptBoxImage = ({ }; const [isGalleryModalOpen, setIsGalleryModalOpen] = useState(false); + const maxImagePrompts = Math.max(1, selectedModel?.maxImagePromptCount ?? 1); + const availableSlotsRender = Math.max( + 0, + maxImagePrompts - referenceImages.length - uploadingImages.length + ); + const usedSlotsRender = Math.min( + maxImagePrompts, + referenceImages.length + uploadingImages.length + ); + const handleFileUpload = (event: React.ChangeEvent) => { + const files = Array.from(event.target.files || []); + if (files.length === 0) return; - const handleGallerySelect = () => setIsGalleryModalOpen(true); + const maxCount = Math.max(1, selectedModel?.maxImagePromptCount ?? 1); + const currentCount = referenceImages.length + uploadingImages.length; + const availableSlots = Math.max(0, maxCount - currentCount); + if (availableSlots <= 0) { + toast.error("Image limit reached for this model."); + if (fileInputRef.current) fileInputRef.current.value = ""; + return; + } + + const filesToProcess = files.slice(0, availableSlots); + if (filesToProcess.length < files.length) { + toast.error( + `Max ${availableSlots} image${ + availableSlots === 1 ? "" : "s" + } allowed. Added the first ${filesToProcess.length}.` + ); + } + + filesToProcess.forEach((file) => { + const uploadId = Math.random().toString(36).substring(7); + _setUploadingImages((prev) => [...prev, { id: uploadId, file }]); + + const reader = new FileReader(); + reader.onloadend = async () => { + if (uploadImage) { + await uploadImage({ + title: `reference-image-${Math.random() + .toString(36) + .substring(2, 15)}`, + assetFile: file, + progressCallback: (newState: UploaderState) => { + if (newState.status === UploaderStates.success && newState.data) { + const referenceImage: RefImage = { + id: Math.random().toString(36).substring(7), + url: reader.result as string, + file, + mediaToken: newState.data, + }; + unstable_batchedUpdates(() => { + _setUploadingImages((prev) => + prev.filter((img) => img.id !== uploadId) + ); + const latestRefs = + usePromptImageStore.getState().referenceImages; + setReferenceImages([...latestRefs, referenceImage]); + }); + } else if ( + newState.status === UploaderStates.assetError || + newState.status === UploaderStates.imageCreateError + ) { + unstable_batchedUpdates(() => { + _setUploadingImages((prev) => + prev.filter((img) => img.id !== uploadId) + ); + }); + } + }, + }); + } else { + const referenceImage: RefImage = { + id: Math.random().toString(36).substring(7), + url: reader.result as string, + file, + mediaToken: "", + }; + unstable_batchedUpdates(() => { + _setUploadingImages((prev) => + prev.filter((img) => img.id !== uploadId) + ); + const latestRefs = usePromptImageStore.getState().referenceImages; + setReferenceImages([...latestRefs, referenceImage]); + }); + } + + if (fileInputRef.current) fileInputRef.current.value = ""; + }; + reader.readAsDataURL(file); + }); + }; + const handleUploadClick = () => fileInputRef.current?.click(); + const toggleImageOptions = () => { + const anyImages = referenceImages.length > 0 || uploadingImages.length > 0; + setShowImagePrompts((prev) => { + if (prev) { + return anyImages; + } + return true; + }); + }; const handleGalleryClose = () => { setIsGalleryModalOpen(false); setSelectedGalleryImages([]); }; const handleImageSelect = (id: string) => { + const maxSelect = Math.max(1, selectedModel?.maxImagePromptCount ?? 1); setSelectedGalleryImages((prev) => { if (prev.includes(id)) { - return []; + return prev.filter((imageId) => imageId !== id); } - return [id]; + if (prev.length >= maxSelect) { + return prev; + } + return [...prev, id]; }); }; const handleGalleryImages = (selectedItems: GalleryItem[]) => { - // Clear existing reference images first - setReferenceImages([]); + const maxCount = Math.max(1, selectedModel?.maxImagePromptCount ?? 1); + const availableSlots = Math.max(0, maxCount - referenceImages.length); + if (availableSlots <= 0) { + setIsGalleryModalOpen(false); + setSelectedGalleryImages([]); + return; + } - // Only take the first selected item - const item = selectedItems[0]; - if (!item || !item.fullImage) return; - - const referenceImage: RefImage = { - id: Math.random().toString(36).substring(7), - url: item.fullImage, - file: new File([], "library-image"), - mediaToken: item.id, - }; - setReferenceImages([referenceImage]); + const newRefs = [...referenceImages]; + selectedItems.slice(0, availableSlots).forEach((item) => { + if (!item.fullImage) return; + newRefs.push({ + id: Math.random().toString(36).substring(7), + url: item.fullImage, + file: new File([], "library-image"), + mediaToken: item.id, + }); + }); + setReferenceImages(newRefs); setIsGalleryModalOpen(false); setSelectedGalleryImages([]); }; - const handleAction = (action: string) => { - switch (action) { - case "gallery": - handleGallerySelect(); - break; - default: - console.log("Unknown action:", action); - } - }; - const handlePaste = (e: React.ClipboardEvent) => { e.preventDefault(); const pastedText = e.clipboardData.getData("text"); @@ -280,15 +410,21 @@ export const PromptBoxImage = ({ }, 10000); const aspectRatio = getCurrentAspectRatio(); - - const generateResponse = await EnqueueTextToImage({ + + const request : EnqueueTextToImageRequest = { prompt: prompt, model: selectedModel, aspect_ratio: aspectRatio, number_images: generationCount, frontend_caller: "text_to_image", frontend_subscriber_id: subscriberId, - }); + }; + + if (selectedModel?.canUseImagePrompt && !!referenceImages && referenceImages.length > 0) { + request.image_media_tokens = referenceImages.map((image) => image.mediaToken); + } + + const generateResponse = await EnqueueTextToImage(request); console.log("PromptBoxImage - generateResponse", generateResponse); @@ -328,82 +464,217 @@ export const PromptBoxImage = ({ isOpen={isModalOpen} onClose={() => { setIsModalOpen(false); - setContent(""); + setContent(null); }} + className="max-w-4xl max-h-[80vh]" > {content} -
- {(referenceImages.length > 0 || uploadingImages.length > 0) && ( -
- {referenceImages.map((image) => ( -
- Reference - -
- ))} - {uploadingImages.map(({ id, file }) => { - const previewUrl = URL.createObjectURL(file); - return ( -
-
- Uploading preview -
-
- + +
+ 1} + /> + {isImageRowVisible && selectedModel?.canUseImagePrompt && ( +
+
+
+
+
+ + + Image Prompts + + ({usedSlotsRender}/{selectedModel?.maxImagePromptCount}) + +
+ + Use the elements of an image +
- ); - })} + +
+ {referenceImages + .slice( + 0, + Math.max(0, selectedModel?.maxImagePromptCount ?? 1) + ) + .map((image) => ( +
{ + setContent( + Reference preview + ); + setIsModalOpen(true); + }} + > + Reference + +
+ ))} + {uploadingImages + .slice( + 0, + Math.max( + 0, + (selectedModel?.maxImagePromptCount ?? 1) - + referenceImages.length + ) + ) + .map(({ id, file }) => { + const previewUrl = URL.createObjectURL(file); + return ( +
+
+ Uploading preview +
+
+ +
+
+ ); + })} + {referenceImages.length + uploadingImages.length < + (selectedModel?.maxImagePromptCount ?? 1) && ( + + + +
+ } + > + + + )} +
+
+
+
+
+
+
+
)} -
+ +
- , - action: "gallery", - }, - ]} - onPanelAction={handleAction} - showIconsInList - buttonClassName="bg-transparent hover:bg-transparent py-1.5 px-0 pr-1 m-0 hover:opacity-50 transition-opacity duration-100 ring-0 border-none focus:ring-0 outline-none" - triggerIcon={ - - } - /> + {selectedModel?.canUseImagePrompt && ( + + + + )}