diff --git a/frontend/apps/editor3d/app/src/components/signaled/TopBar/TopBar.tsx b/frontend/apps/editor3d/app/src/components/signaled/TopBar/TopBar.tsx index 173b0793f8..f655eaed87 100644 --- a/frontend/apps/editor3d/app/src/components/signaled/TopBar/TopBar.tsx +++ b/frontend/apps/editor3d/app/src/components/signaled/TopBar/TopBar.tsx @@ -19,7 +19,6 @@ import { Signal } from "@preact/signals-react"; import { useSignals } from "@preact/signals-react/runtime"; import { setLogoutStates } from "~/signals/authentication/utilities"; import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext"; -import { useAppUiContext } from "~/pages/Page2d/contextSignals/appUi"; function isEditorPath(path: string) { if (path === "/") return true; @@ -37,7 +36,7 @@ interface Props { const appTabs: TabItem[] = [ { id: "2D", label: "2D" }, { id: "3D", label: "3D" }, - // { id: "VIDEO", label: "Video" }, + { id: "VIDEO", label: "Video" }, ]; export const TopBar = ({ diff --git a/frontend/apps/editor3d/app/src/pages/Page2d/components/features/DialogAddImage/DialogAddImage.tsx b/frontend/apps/editor3d/app/src/pages/Page2d/components/features/DialogAddImage/DialogAddImage.tsx index 3c60b693cc..b6953d6968 100644 --- a/frontend/apps/editor3d/app/src/pages/Page2d/components/features/DialogAddImage/DialogAddImage.tsx +++ b/frontend/apps/editor3d/app/src/pages/Page2d/components/features/DialogAddImage/DialogAddImage.tsx @@ -13,11 +13,13 @@ export const DialogAddImage = ({ isOpen, closeCallback, onAddImage, + isProcessing = false, }: { stagedImage?: File | null; isOpen: boolean; closeCallback: () => void; onAddImage: (file: File) => void; + isProcessing?: boolean; }) => { const [assetFile, setAssetFile] = useState(null); const previouslyStagedImageRef = useRef(null); @@ -37,10 +39,13 @@ export const DialogAddImage = ({ closeCallback(); } + // Show dialog if no image is set (currFile is null) and not processing + const shouldShowDialog = isOpen && (!currFile || !isProcessing); + return ( - +
- Upload Image +

Upload Image

{ const containerRef = useRef(null); + const [isDialogOpen, setIsDialogOpen] = useState(false); + const [isImageBeingProcessed, setIsImageBeingProcessed] = useState(false); + const [selectedImage, setSelectedImage] = useState(null); + const [imagePreviewUrl, setImagePreviewUrl] = useState(""); - const uploadImage = ({ - title, - assetFile, - progressCallback, - }: { - title: string; - assetFile: File; - progressCallback: (newState: UploaderState) => void; - }): Promise => { - throw new Error("Function not implemented."); + const handleClose = () => { + setIsDialogOpen(false); }; - const getCanvasRenderBitmap = (): MaybeCanvasRenderBitmapType => { - throw new Error("Function not implemented."); + const handleAddImage = (image: File) => { + setSelectedImage(image); + setIsImageBeingProcessed(true); + setImagePreviewUrl(URL.createObjectURL(image)); + + // TODO: Implement actual image processing here + // For now, we'll just simulate processing + setTimeout(() => { + setIsImageBeingProcessed(false); + }, 3000); }; - const EncodeImageBitmapToBase64 = ( - imageBitmap: ImageBitmap, - ): Promise => { - throw new Error("Function not implemented."); - }; - - const useJobContext = (): JobContextType => { - throw new Error("Function not implemented."); + const jobContext: JobContextType = { + jobTokens: [], + addJobToken: () => {}, + removeJobToken: () => {}, + clearJobTokens: () => {}, }; return (
-
-
-
Image Display
-
- Preview { - const target = e.target as HTMLImageElement; - target.style.display = "none"; - }} - /> -
+
+
+ { + // TODO: Implement file upload logic + console.log("Uploading file:", file); + return { url: "", mediaToken: "" }; + }} + getCanvasRenderBitmap={() => {}} + EncodeImageBitmapToBase64={async (bitmap: ImageBitmap) => { + // TODO: Implement bitmap encoding + console.log("Encoding bitmap to base64"); + return ""; + }} + useJobContext={() => { + // TODO: Implement job context logic + console.log("Using job context"); + return jobContext; + }} + onEnqueuePressed={async () => { + // TODO: Implement enqueue logic + console.log("Enqueue pressed"); + }} + />
diff --git a/frontend/apps/editor3d/app/src/signals/appTab.ts b/frontend/apps/editor3d/app/src/signals/appTab.ts index 3729c7e16f..acdd4d9cdb 100644 --- a/frontend/apps/editor3d/app/src/signals/appTab.ts +++ b/frontend/apps/editor3d/app/src/signals/appTab.ts @@ -1,6 +1,6 @@ import { signal } from "@preact/signals-react"; -export const appTabId = signal("3D"); +export const appTabId = signal("VIDEO"); export const setAppTabId = (newId: string) => { if (newId != "2D" && newId != "3D" && newId != "VIDEO") { diff --git a/frontend/apps/editor3d/package.json b/frontend/apps/editor3d/package.json index 761c2f581f..d0e1e0e586 100644 --- a/frontend/apps/editor3d/package.json +++ b/frontend/apps/editor3d/package.json @@ -58,9 +58,11 @@ "detect-gpu": "^5.0.38", "http-proxy-middleware": "^3.0.0", "isbot": "^4.1.0", + "konva": "^9.3.20", "lodash.debounce": "^4.0.8", "posthog-js": "^1.141.0", "react": "^18.2.0", + "react-colorful": "^5.6.1", "react-copy-to-clipboard": "^5.1.0", "react-dom": "^18.2.0", "react-drag-drop-files": "^2.3.10", diff --git a/frontend/libs/components/promptbox/src/index.ts b/frontend/libs/components/promptbox/src/index.ts index 0f0c83bdca..819236eb81 100644 --- a/frontend/libs/components/promptbox/src/index.ts +++ b/frontend/libs/components/promptbox/src/index.ts @@ -1,2 +1,3 @@ export * from "./lib/PromptBox2D"; export * from "./lib/PromptBox3D"; +export * from "./lib/PromptBoxVideo"; diff --git a/frontend/libs/components/promptbox/src/lib/PromptBox2D.tsx b/frontend/libs/components/promptbox/src/lib/PromptBox2D.tsx index 3bdf3f0f56..be52f39152 100644 --- a/frontend/libs/components/promptbox/src/lib/PromptBox2D.tsx +++ b/frontend/libs/components/promptbox/src/lib/PromptBox2D.tsx @@ -32,7 +32,12 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { IsDesktopApp } from "@storyteller/tauri-utils"; import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal"; import { PromptsApi } from "@storyteller/api"; -import { CommandSuccessStatus, GetAppPreferences, SoraImageRemix, SoraImageRemixErrorType } from "@storyteller/tauri-api"; +import { + CommandSuccessStatus, + GetAppPreferences, + SoraImageRemix, + SoraImageRemixErrorType, +} from "@storyteller/tauri-api"; import { SoundRegistry } from "@storyteller/soundboard"; interface ReferenceImage { @@ -291,9 +296,7 @@ export const PromptBox2D = ({ snapshot_media_token: snapshotMediaToken.data, disable_system_prompt: !useSystemPrompt, prompt: prompt, - maybe_additional_images: referenceImages.map( - (image) => image.mediaToken, - ), + maybe_additional_images: referenceImages.map((image) => image.mediaToken), maybe_number_of_samples: 1, }); @@ -307,7 +310,6 @@ export const PromptBox2D = ({ registry.playSound(soundName); } toast.success("Image added to queue"); - } else { const prefs = await GetAppPreferences(); const soundName = prefs.preferences?.enqueue_failure_sound; @@ -321,16 +323,19 @@ export const PromptBox2D = ({ if ("error_type" in generateResponse) { switch (generateResponse.error_type) { case SoraImageRemixErrorType.SoraLoginRequired: - errorMessage = "You need to log into Sora to continue. See the settings menu."; + errorMessage = + "You need to log into Sora to continue. See the settings menu."; break; case SoraImageRemixErrorType.ServerError: errorMessage = "Server error. Failed to enqueue image."; break; case SoraImageRemixErrorType.TooManyConcurrentTasks: - errorMessage = "You have too many Sora image generation tasks running. Please wait a moment."; + errorMessage = + "You have too many Sora image generation tasks running. Please wait a moment."; break; case SoraImageRemixErrorType.SoraUsernameNotYetCreated: - errorMessage = "Your Sora username is not yet created. Please visit the Sora.com website to create it first."; + errorMessage = + "Your Sora username is not yet created. Please visit the Sora.com website to create it first."; break; case SoraImageRemixErrorType.SoraIsHavingProblems: errorMessage = "Sora is having problems. Please try again later."; @@ -339,7 +344,6 @@ export const PromptBox2D = ({ } toast.error(errorMessage); } - }; const handleWebEnqueue = async () => { diff --git a/frontend/libs/components/promptbox/src/lib/PromptBoxVideo.tsx b/frontend/libs/components/promptbox/src/lib/PromptBoxVideo.tsx new file mode 100644 index 0000000000..c2662b0bfc --- /dev/null +++ b/frontend/libs/components/promptbox/src/lib/PromptBoxVideo.tsx @@ -0,0 +1,629 @@ +import { useState, useRef, useEffect } from "react"; +import { useSignals } from "@preact/signals-react/runtime"; +import { toast } from "@storyteller/ui-toaster"; +import { + JobContextType, + MaybeCanvasRenderBitmapType, + UploaderState, + UploaderStates, +} from "@storyteller/common"; +import { downloadFileFromUrl } from "@storyteller/api"; +import { PopoverMenu, PopoverItem } from "@storyteller/ui-popover"; +import { Tooltip } from "@storyteller/ui-tooltip"; +import { Button, ToggleButton } from "@storyteller/ui-button"; +import { Modal } from "@storyteller/ui-modal"; +import { + faMessageXmark, + faMessageCheck, + faSparkles, + faSpinnerThird, + faTimes, + faPlus, + faUpload, + faImages, +} from "@fortawesome/pro-solid-svg-icons"; +import { + faRectangleVertical, + faSquare, + faRectangle, +} from "@fortawesome/pro-regular-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { IsDesktopApp } from "@storyteller/tauri-utils"; +import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal"; +import { PromptsApi } from "@storyteller/api"; +import { + CommandSuccessStatus, + GetAppPreferences, + SoraImageRemix, + SoraImageRemixErrorType, +} from "@storyteller/tauri-api"; +import { SoundRegistry } from "@storyteller/soundboard"; + +interface ReferenceImage { + id: string; + url: string; + file: File; + mediaToken: string; +} + +interface PromptBoxVideoProps { + uploadImage: ({ + title, + assetFile, + progressCallback, + }: { + title: string; + assetFile: File; + progressCallback: (newState: UploaderState) => void; + }) => Promise; + getCanvasRenderBitmap: () => MaybeCanvasRenderBitmapType; + EncodeImageBitmapToBase64: (imageBitmap: ImageBitmap) => Promise; + useJobContext: () => JobContextType; + onEnqueuePressed?: () => void | Promise; +} + +export const PromptBoxVideo = ({ + uploadImage, + getCanvasRenderBitmap, + EncodeImageBitmapToBase64, + useJobContext, + onEnqueuePressed, +}: PromptBoxVideoProps) => { + useSignals(); + + console.log("Is this a desktop app?", IsDesktopApp()); + const [isModalOpen, setIsModalOpen] = useState(false); + const [content, setContent] = useState(""); + const { jobTokens, addJobToken, removeJobToken, clearJobTokens } = + useJobContext(); + + //const { lastRenderedBitmap } = useCanvasSignal(); + const [prompt, setPrompt] = useState(""); + const [isEnqueueing, setIsEnqueueing] = useState(false); + const [useSystemPrompt, setUseSystemPrompt] = useState(true); + const [isGalleryModalOpen, setIsGalleryModalOpen] = useState(false); + const [selectedGalleryImages, setSelectedGalleryImages] = useState( + [] + ); + const [activeGalleryTab, setActiveGalleryTab] = useState("my-media"); + const [referenceImages, setReferenceImages] = useState([]); + const [uploadingImages, setUploadingImages] = useState< + { id: string; file: File }[] + >([]); + const [aspectRatioList, setAspectRatioList] = useState([ + { + label: "3:2", + selected: true, + icon: , + }, + { + label: "2:3", + selected: false, + icon: , + }, + { + label: "1:1", + selected: false, + icon: , + }, + ]); + + const fileInputRef = useRef(null); + const textareaRef = useRef(null); + + useEffect(() => { + if (textareaRef.current) { + textareaRef.current.style.height = "auto"; + textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`; + } + }); + + const handleAspectRatioSelect = (selectedItem: PopoverItem) => { + setAspectRatioList((prev) => + prev.map((item) => ({ + ...item, + selected: item.label === selectedItem.label, + })) + ); + }; + + const handleFileUpload = (event: React.ChangeEvent) => { + const files = event.target.files; + if (files) { + Array.from(files).forEach((file) => { + const uploadId = Math.random().toString(36).substring(7); + setUploadingImages((prev) => [...prev, { id: uploadId, file }]); + + const reader = new FileReader(); + reader.onloadend = () => { + uploadImage({ + title: `reference-image-${Math.random() + .toString(36) + .substring(2, 15)}`, + assetFile: file, + progressCallback: (newState) => { + console.debug("Upload progress:", newState.data); + if (newState.status === UploaderStates.success && newState.data) { + const referenceImage: ReferenceImage = { + id: Math.random().toString(36).substring(7), + url: reader.result as string, + file, + mediaToken: newState.data || "", + }; + setReferenceImages((prev) => [...prev, referenceImage]); + setUploadingImages((prev) => + prev.filter((img) => img.id !== uploadId) + ); + toast.success("Reference image added."); + } else if ( + newState.status === UploaderStates.assetError || + newState.status === UploaderStates.imageCreateError + ) { + setUploadingImages((prev) => + prev.filter((img) => img.id !== uploadId) + ); + + toast.error("Upload failed. Please try again."); + } + }, + }); + }; + + reader.readAsDataURL(file); + }); + } + }; + + const handleRemoveReference = (id: string) => { + setReferenceImages((prev) => prev.filter((img) => img.id !== id)); + if (fileInputRef.current) { + fileInputRef.current.value = ""; + } + }; + + const handleUploadClick = () => { + fileInputRef.current?.click(); + }; + + const handleGallerySelect = () => { + setIsGalleryModalOpen(true); + }; + + const handleGalleryClose = () => { + setIsGalleryModalOpen(false); + setSelectedGalleryImages([]); + }; + + const handleImageSelect = (id: string) => { + setSelectedGalleryImages((prev) => { + if (prev.includes(id)) { + return prev.filter((imageId) => imageId !== id); + } + if (prev.length >= 4) { + return prev; + } + return [...prev, id]; + }); + }; + + const handleGalleryImages = (selectedItems: GalleryItem[]) => { + selectedItems.forEach((item) => { + if (!item.fullImage) return; + const referenceImage: ReferenceImage = { + id: Math.random().toString(36).substring(7), + url: item.fullImage, + file: new File([], "library-image"), + mediaToken: item.id, + }; + setReferenceImages((prev) => [...prev, referenceImage]); + }); + setIsGalleryModalOpen(false); + setSelectedGalleryImages([]); + }; + + const handleAction = (action: string) => { + switch (action) { + case "upload": + handleUploadClick(); + break; + case "gallery": + handleGallerySelect(); + break; + default: + console.log("Unknown action:", action); + } + }; + + const handlePaste = (e: React.ClipboardEvent) => { + e.preventDefault(); + const pastedText = e.clipboardData.getData("text").trim(); + setPrompt(pastedText); + }; + + const handleChange = (e: React.ChangeEvent) => { + setPrompt(e.target.value); + }; + + const handleTauriEnqueue = async () => { + const api = new PromptsApi(); + let image = getCanvasRenderBitmap(); + if (image === undefined) { + console.log("image is undefined"); + return; + } + const base64Bitmap = await EncodeImageBitmapToBase64(image); + + const byteString = atob(base64Bitmap); + const mimeString = "image/png"; + + const ab = new ArrayBuffer(byteString.length); + const ia = new Uint8Array(ab); + + for (let i = 0; i < byteString.length; i++) { + ia[i] = byteString.charCodeAt(i); + } + + const uuid = crypto.randomUUID(); // Generate a new UUID + const file = new File([ab], `${uuid}.png`, { type: mimeString }); + + const snapshotMediaToken = await api.uploadSceneSnapshot({ + screenshot: file, + }); + + if (snapshotMediaToken.data === undefined) { + toast.error("Error: Unable to upload scene snapshot Please try again."); + return; + } + console.log("useSystemPrompt", useSystemPrompt); + console.log("Snapshot media token:", snapshotMediaToken.data); + + + const generateResponse = await SoraImageRemix({ + snapshot_media_token: snapshotMediaToken.data, + disable_system_prompt: !useSystemPrompt, + prompt: prompt, + maybe_additional_images: referenceImages.map((image) => image.mediaToken), + maybe_number_of_samples: 1, + }); + + console.log("generateResponse", generateResponse); + + if (generateResponse.status === CommandSuccessStatus.Success) { + const prefs = await GetAppPreferences(); + const soundName = prefs.preferences?.enqueue_success_sound; + if (soundName !== undefined) { + const registry = SoundRegistry.getInstance(); + registry.playSound(soundName); + } + toast.success("Image added to queue"); + } else { + const prefs = await GetAppPreferences(); + const soundName = prefs.preferences?.enqueue_failure_sound; + if (soundName !== undefined) { + const registry = SoundRegistry.getInstance(); + registry.playSound(soundName); + } + + let errorMessage = "Failed to enqueue image"; + + if ("error_type" in generateResponse) { + switch (generateResponse.error_type) { + case SoraImageRemixErrorType.SoraLoginRequired: + errorMessage = + "You need to log into Sora to continue. See the settings menu."; + break; + case SoraImageRemixErrorType.ServerError: + errorMessage = "Server error. Failed to enqueue image."; + break; + case SoraImageRemixErrorType.TooManyConcurrentTasks: + errorMessage = + "You have too many Sora image generation tasks running. Please wait a moment."; + break; + case SoraImageRemixErrorType.SoraUsernameNotYetCreated: + errorMessage = + "Your Sora username is not yet created. Please visit the Sora.com website to create it first."; + break; + case SoraImageRemixErrorType.SoraIsHavingProblems: + errorMessage = "Sora is having problems. Please try again later."; + break; + } + } + toast.error(errorMessage); + } + }; + + const handleWebEnqueue = async () => { + const api = new PromptsApi(); + // to take the snapshot of the canvas + // Call onEnqueuePressed if it exists + + let image = getCanvasRenderBitmap(); + if (image === undefined) { + toast.error( + "Error: Unable to generate image. Please check the input and try again." + ); + return; + } + const base64Bitmap = await EncodeImageBitmapToBase64(image); + + const byteString = atob(base64Bitmap); + const mimeString = "image/png"; + + const ab = new ArrayBuffer(byteString.length); + const ia = new Uint8Array(ab); + + for (let i = 0; i < byteString.length; i++) { + ia[i] = byteString.charCodeAt(i); + } + + const uuid = crypto.randomUUID(); // Generate a new UUID + const file = new File([ab], `${uuid}.png`, { type: mimeString }); + + const snapshotMediaToken = await api.uploadSceneSnapshot({ + screenshot: file, + }); + + if (snapshotMediaToken.data === undefined) { + toast.error("Error: Unable to upload scene snapshot Please try again."); + return; + } + console.log("useSystemPrompt", useSystemPrompt); + console.log("Snapshot media token:", snapshotMediaToken.data); + + const response = await api.enqueueImageGeneration({ + disableSystemPrompt: !useSystemPrompt, + prompt: prompt, + snapshotMediaToken: snapshotMediaToken.data, + additionalImages: referenceImages.map((image) => image.mediaToken), + }); + + if (response.success === true) { + toast.success("Please wait while we process your image."); + if (response.data) { + addJobToken(response.data); + } + return; + } else { + toast.error("Failed to enqueue image generation. Please try again."); + } + }; + const handleEnqueue = async () => { + console.log("Pressing Enqueue"); + if (onEnqueuePressed) { + try { + await onEnqueuePressed(); + } catch (error) { + console.error("Error in onEnqueuePressed callback:", error); + return; + } + } + + if (isEnqueueing) return; + setIsEnqueueing(true); + if (!prompt.trim()) return; + try { + console.log( + "(4) Enqueuing with prompt:", + prompt, + "and reference images:", + referenceImages + ); + + const isDesktop = IsDesktopApp(); + console.log("Is this a desktop app?", isDesktop); + + if (isDesktop) { + await handleTauriEnqueue(); + } else { + await handleWebEnqueue(); + } + } catch (error) { + console.error("Error during image generation:", error); + toast.error( + "An error occurred while generating the image. Please try again." + ); + } finally { + await new Promise((resolve) => setTimeout(resolve, 100)); + setIsEnqueueing(false); + } + }; + + const getCurrentAspectRatioIcon = () => { + const selected = aspectRatioList.find((item) => item.selected); + if (!selected || !selected.icon) return faRectangle; + const iconElement = selected.icon as React.ReactElement; + return iconElement.props.icon; + }; + + const handleKeyDown = (e: React.KeyboardEvent) => { + if (e.key === "Enter" && !e.shiftKey) { + e.preventDefault(); + + handleEnqueue(); + } + }; + + return ( + <> + { + setIsModalOpen(false); + setContent(""); + }} + > + {content} + +
+ {(referenceImages.length > 0 || uploadingImages.length > 0) && ( +
+ {referenceImages.map((image) => ( +
+ Reference + +
+ ))} + {uploadingImages.map(({ id, file }) => { + const previewUrl = URL.createObjectURL(file); + return ( +
+
+ Uploading preview +
+
+ +
+
+ ); + })} +
+ )} +
+ +
+ , + action: "upload", + }, + { + label: "Choose from library", + selected: false, + icon: , + action: "gallery", + }, + ]} + onPanelAction={handleAction} + showIconsInList + buttonClassName="bg-transparent hover:bg-transparent py-1.5 px-0 pr-1 m-0 hover:opacity-50 transition-opacity duration-100 ring-0 border-none focus:ring-0 outline-none" + triggerIcon={ + + } + /> + +