Trying to do add to scene for the image.

frontend/apps/editor3d/app/src/components/signaled/TopBar/TopBar.tsx

@@ -36,7 +36,7 @@ interface Props {
 const appTabs: TabItem[] = [
   { id: "2D", label: "2D" },
   { id: "3D", label: "3D" },
-  // { id: "VIDEO", label: "Video" },
+  { id: "VIDEO", label: "Video" },
 ];
 
 export const TopBar = ({
@@ -55,6 +55,9 @@ export const TopBar = ({
   const [activeLibraryTab, setActiveLibraryTab] = useState("my-media");
   const [isSettingsModalOpen, setIsSettingsModalOpen] = useState(false);
 
+  const [url, setUrl] = useState<string>("");
+  const [mediaId, setMediaId] = useState<string>("");
+
   const engine3D = useContext(EngineContext);
   const handleTabChange = (tabId: string) => {
     if (appTabIdSignal.peek() == "3D") {
@@ -75,35 +78,30 @@ export const TopBar = ({
     url: string,
     media_id: string | undefined,
   ) => {
-    console.log("add to scene", url);
+    console.log("Items to add to scene", currentAppTabId);
+    console.log("url", url);
+    console.log("media_id", media_id);
+
+    setUrl(url);
+    setMediaId(media_id ?? "");
+
     if (currentAppTabId === "2D") {
       console.log("Adding to 2D scene");
-
       // from the uploaded image url.
     } else if (currentAppTabId === "3D") {
       console.log("Adding to 3D scene");
-
-      // from the uploaded image url.
-      // engine3D?.activeScene.loadObjectFromUrl(
-      //   "https://cdn-2.fakeyou.com/media/r/h/0/5/3/rh053gbx8yvjwk1mdvag4gkr5c0h9wbh/image_rh053gbx8yvjwk1mdvag4gkr5c0h9wbh.png",
-      // );
-
-      // from the gallery modal.
-      // engine3D?.activeScene.loadObjectFromUrl(
-      //   "https://cdn-2.fakeyou.com/media/1/t/b/0/e/1tb0e53691mdgqmtd4fmzy58wp96vwxz/image_1tb0e53691mdgqmtd4fmzy58wp96vwxz.png",
-      // );
-
-      // // working media id m_ehq8051fx4hddhja54a7zmz897cgrk
-      // engine3D?.activeScene.loadObject(
-      //   "m_ehq8051fx4hddhja54a7zmz897cgrk",
-      //   "image",
-      //   true,
-      // );
-
       // media id from the image selected from gallery.
-      engine3D?.activeScene.loadObject(media_id, "image", true);
+      if (media_id) {
+        engine3D?.activeScene.loadObject(media_id, "image", true);
+      } else {
+        console.warn("No media id provided");
+      }
     } else if (currentAppTabId === "VIDEO") {
       console.log("Adding to Video scene");
+      if (media_id) {
+      } else {
+        console.warn("No media id provided");
+      }
     } else {
       console.warn(`Unknown tab type: ${currentAppTabId}`);
     }

frontend/apps/editor3d/app/src/pages/PageVideo/ImageToVideo.tsx

@@ -5,25 +5,24 @@ import BackgroundGallery from "./BackgroundGallery";
 import { PopoverMenu, type PopoverItem } from "@storyteller/ui-popover";
 import { faFilm, faClock } from "@fortawesome/pro-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { FalKlingImageToVideo } from "@storyteller/tauri-api";
 
-const ImageToVideo = () => {
+interface ImageToVideoProps {
+  imageMediaId?: string;
+  imageUrl?: string;
+}
+
+const ImageToVideo = ({ imageMediaId, imageUrl }: ImageToVideoProps) => {
   const containerRef = useRef<HTMLDivElement>(null);
   const [selectedModel, setSelectedModel] = useState<string>("Kling 2.0");
-  const [useSystemPrompt, setUseSystemPrompt] = useState(false);
 
+  const [useSystemPrompt, setUseSystemPrompt] = useState(false);
   const [isImageBeingProcessed, setIsImageBeingProcessed] = useState(false);
+
   const [selectedImage, setSelectedImage] = useState<File | null>(null);
   const [imagePreviewUrl, setImagePreviewUrl] = useState<string>("");
   const [isDialogOpen, setIsDialogOpen] = useState(false);
 
-  const handleClose = () => setIsDialogOpen(false);
-  const handleAddImage = (image: File) => {
-    setSelectedImage(image);
-    setIsImageBeingProcessed(true);
-    setImagePreviewUrl(URL.createObjectURL(image));
-    setTimeout(() => setIsImageBeingProcessed(false), 3000);
-  };
-
   const jobContext: JobContextType = {
     jobTokens: [],
     addJobToken: () => {},
@@ -72,8 +71,10 @@ const ImageToVideo = () => {
               console.log("Using job context");
               return jobContext;
             }}
+            model={selectedModel}
+            imageMediaId={imageMediaId}
+            url={imageUrl ?? imagePreviewUrl ?? undefined}
             onEnqueuePressed={async () => {
-              // TODO: Implement enqueue logic
               console.log("Enqueue pressed");
             }}
           />

frontend/apps/editor3d/app/src/signals/appTab.ts

@@ -1,7 +1,7 @@
 import { signal } from "@preact/signals-react";
 
 // change starting tab
-export const appTabId = signal("3D");
+export const appTabId = signal("VIDEO");
 
 export const setAppTabId = (newId: string) => {
   if (newId != "2D" && newId != "3D" && newId != "VIDEO") {

frontend/libs/components/lightbox-modal/src/lib/lightbox-modal.tsx

@@ -4,7 +4,10 @@ import { Transition, TransitionChild } from "@headlessui/react";
 import { createPortal } from "react-dom";
 import dayjs from "dayjs";
 import { faDownToLine } from "@fortawesome/pro-solid-svg-icons";
-import { FalKlingImageToVideo, FalHunyuanImageTo3d } from "@storyteller/tauri-api"
+import {
+  FalKlingImageToVideo,
+  FalHunyuanImageTo3d,
+} from "@storyteller/tauri-api";
 
 interface LightboxModalProps {
   isOpen: boolean;
@@ -117,13 +120,15 @@ export function LightboxModal({
                   {/* buttons with spacing */}
                   {(onAddToSceneClicked && downloadUrl) || downloadUrl ? (
                     <div className="mt-15 mb-15 flex justify-end gap-2">
-
                       <Button
                         onClick={async (e) => {
-                          let _result = await FalKlingImageToVideo({
-                            image_media_token: mediaId,
-                            //base64_image: downloadUrl,
-                          });
+                          if (onAddToSceneClicked) {
+                            onAddToSceneClicked(downloadUrl, mediaId);
+                          }
+                          // let _result = await FalKlingImageToVideo({
+                          //   image_media_token: mediaId,
+                          //   //base64_image: downloadUrl,
+                          // });
                           //e.stopPropagation();
                           //await onAddToSceneClicked(downloadUrl, mediaId);
                           //onClose(); // close the lightbox

frontend/libs/components/promptbox/src/lib/PromptBoxVideo.tsx

@@ -1,17 +1,13 @@
 import { useState, useRef, useEffect } from "react";
 import { useSignals } from "@preact/signals-react/runtime";
 import { toast } from "@storyteller/ui-toaster";
-import {
-  JobContextType,
-  MaybeCanvasRenderBitmapType,
-  UploaderState,
-  UploaderStates,
-} from "@storyteller/common";
+import { JobContextType } from "@storyteller/common";
 import { downloadFileFromUrl } from "@storyteller/api";
 import { PopoverMenu, PopoverItem } from "@storyteller/ui-popover";
 import { Tooltip } from "@storyteller/ui-tooltip";
 import { Button, ToggleButton } from "@storyteller/ui-button";
 import { Modal } from "@storyteller/ui-modal";
+import { FalKlingImageToVideo } from "@storyteller/tauri-api";
 import {
   faMessageXmark,
   faMessageCheck,
@@ -30,12 +26,9 @@ import {
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { IsDesktopApp } from "@storyteller/tauri-utils";
 import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal";
-import { PromptsApi } from "@storyteller/api";
 import {
   CommandSuccessStatus,
   GetAppPreferences,
-  SoraImageRemix,
-  SoraImageRemixErrorType,
 } from "@storyteller/tauri-api";
 import { SoundRegistry } from "@storyteller/soundboard";
 
@@ -47,30 +40,35 @@ interface ReferenceImage {
 }
 
 interface PromptBoxVideoProps {
-  uploadImage: ({
-    title,
-    assetFile,
-    progressCallback,
-  }: {
-    title: string;
-    assetFile: File;
-    progressCallback: (newState: UploaderState) => void;
-  }) => Promise<void>;
-  getCanvasRenderBitmap: () => MaybeCanvasRenderBitmapType;
-  EncodeImageBitmapToBase64: (imageBitmap: ImageBitmap) => Promise<string>;
   useJobContext: () => JobContextType;
   onEnqueuePressed?: () => void | Promise<void>;
+  model: string;
+  imageMediaId?: string;
+  url?: string;
 }
 
 export const PromptBoxVideo = ({
-  uploadImage,
-  getCanvasRenderBitmap,
-  EncodeImageBitmapToBase64,
   useJobContext,
   onEnqueuePressed,
+  model,
+  imageMediaId,
+  url,
 }: PromptBoxVideoProps) => {
   useSignals();
 
+  // for the image media id and url, we need to set the reference image gallery panel.
+  useEffect(() => {
+    if (imageMediaId && url) {
+      const referenceImage: ReferenceImage = {
+        id: Math.random().toString(36).substring(7),
+        url: url,
+        file: new File([], "library-image"),
+        mediaToken: imageMediaId,
+      };
+      setReferenceImages([referenceImage]);
+    }
+  }, [imageMediaId, url]);
+
   console.log("Is this a desktop app?", IsDesktopApp());
   const [isModalOpen, setIsModalOpen] = useState(false);
   const [content, setContent] = useState("");
@@ -112,6 +110,18 @@ export const PromptBoxVideo = ({
     }
   });
 
+  useEffect(() => {
+    if (imageMediaId && url) {
+      const referenceImage: ReferenceImage = {
+        id: Math.random().toString(36).substring(7),
+        url: url,
+        file: new File([], "library-image"),
+        mediaToken: imageMediaId,
+      };
+      setReferenceImages([referenceImage]);
+    }
+  }, [imageMediaId, url]);
+
   const handleResolutionSelect = (selectedItem: PopoverItem) => {
     setResolutionList((prev) =>
       prev.map((item) => ({
@@ -121,53 +131,6 @@ export const PromptBoxVideo = ({
     );
   };
 
-  const handleFileUpload = (event: React.ChangeEvent<HTMLInputElement>) => {
-    const files = event.target.files;
-    if (files) {
-      Array.from(files).forEach((file) => {
-        const uploadId = Math.random().toString(36).substring(7);
-        setUploadingImages((prev) => [...prev, { id: uploadId, file }]);
-
-        const reader = new FileReader();
-        reader.onloadend = () => {
-          uploadImage({
-            title: `reference-image-${Math.random()
-              .toString(36)
-              .substring(2, 15)}`,
-            assetFile: file,
-            progressCallback: (newState) => {
-              console.debug("Upload progress:", newState.data);
-              if (newState.status === UploaderStates.success && newState.data) {
-                const referenceImage: ReferenceImage = {
-                  id: Math.random().toString(36).substring(7),
-                  url: reader.result as string,
-                  file,
-                  mediaToken: newState.data || "",
-                };
-                setReferenceImages((prev) => [...prev, referenceImage]);
-                setUploadingImages((prev) =>
-                  prev.filter((img) => img.id !== uploadId)
-                );
-                toast.success("Reference image added.");
-              } else if (
-                newState.status === UploaderStates.assetError ||
-                newState.status === UploaderStates.imageCreateError
-              ) {
-                setUploadingImages((prev) =>
-                  prev.filter((img) => img.id !== uploadId)
-                );
-
-                toast.error("Upload failed. Please try again.");
-              }
-            },
-          });
-        };
-
-        reader.readAsDataURL(file);
-      });
-    }
-  };
-
   const handleRemoveReference = (id: string) => {
     setReferenceImages((prev) => prev.filter((img) => img.id !== id));
     if (fileInputRef.current) {
@@ -175,10 +138,6 @@ export const PromptBoxVideo = ({
     }
   };
 
-  const handleUploadClick = () => {
-    fileInputRef.current?.click();
-  };
-
   const handleGallerySelect = () => {
     setIsGalleryModalOpen(true);
   };
@@ -236,49 +195,11 @@ export const PromptBoxVideo = ({
     setPrompt(e.target.value);
   };
 
-  const handleTauriEnqueue = async () => {
-    const api = new PromptsApi();
-    let image = getCanvasRenderBitmap();
-    if (image === undefined) {
-      console.log("image is undefined");
-      return;
-    }
-    const base64Bitmap = await EncodeImageBitmapToBase64(image);
-
-    const byteString = atob(base64Bitmap);
-    const mimeString = "image/png";
-
-    const ab = new ArrayBuffer(byteString.length);
-    const ia = new Uint8Array(ab);
-
-    for (let i = 0; i < byteString.length; i++) {
-      ia[i] = byteString.charCodeAt(i);
-    }
-
-    const uuid = crypto.randomUUID(); // Generate a new UUID
-    const file = new File([ab], `${uuid}.png`, { type: mimeString });
-
-    const snapshotMediaToken = await api.uploadSceneSnapshot({
-      screenshot: file,
+  const handleEnqueue = async () => {
+    const generateResponse = await FalKlingImageToVideo({
+      image_media_token: referenceImages[0].mediaToken,
     });
-
-    if (snapshotMediaToken.data === undefined) {
-      toast.error("Error: Unable to upload scene snapshot Please try again.");
-      return;
-    }
-    console.log("useSystemPrompt", useSystemPrompt);
-    console.log("Snapshot media token:", snapshotMediaToken.data);
-
-    const generateResponse = await SoraImageRemix({
-      snapshot_media_token: snapshotMediaToken.data,
-      disable_system_prompt: !useSystemPrompt,
-      prompt: prompt,
-      maybe_additional_images: referenceImages.map((image) => image.mediaToken),
-      maybe_number_of_samples: 1,
-    });
-
-    console.log("generateResponse", generateResponse);
-
+    onEnqueuePressed?.();
     if (generateResponse.status === CommandSuccessStatus.Success) {
       const prefs = await GetAppPreferences();
       const soundName = prefs.preferences?.enqueue_success_sound;
@@ -294,129 +215,33 @@ export const PromptBoxVideo = ({
         const registry = SoundRegistry.getInstance();
         registry.playSound(soundName);
       }
-
-      let errorMessage = "Failed to enqueue image";
-
-      if ("error_type" in generateResponse) {
-        switch (generateResponse.error_type) {
-          case SoraImageRemixErrorType.SoraLoginRequired:
-            errorMessage =
-              "You need to log into Sora to continue. See the settings menu.";
-            break;
-          case SoraImageRemixErrorType.ServerError:
-            errorMessage = "Server error. Failed to enqueue image.";
-            break;
-          case SoraImageRemixErrorType.TooManyConcurrentTasks:
-            errorMessage =
-              "You have too many Sora image generation tasks running. Please wait a moment.";
-            break;
-          case SoraImageRemixErrorType.SoraUsernameNotYetCreated:
-            errorMessage =
-              "Your Sora username is not yet created. Please visit the Sora.com website to create it first.";
-            break;
-          case SoraImageRemixErrorType.SoraIsHavingProblems:
-            errorMessage = "Sora is having problems. Please try again later.";
-            break;
-        }
-      }
-      toast.error(errorMessage);
-    }
-  };
-
-  const handleWebEnqueue = async () => {
-    const api = new PromptsApi();
-    // to take the snapshot of the canvas
-    // Call onEnqueuePressed if it exists
-
-    let image = getCanvasRenderBitmap();
-    if (image === undefined) {
-      toast.error(
-        "Error: Unable to generate image. Please check the input and try again."
-      );
-      return;
-    }
-    const base64Bitmap = await EncodeImageBitmapToBase64(image);
-
-    const byteString = atob(base64Bitmap);
-    const mimeString = "image/png";
-
-    const ab = new ArrayBuffer(byteString.length);
-    const ia = new Uint8Array(ab);
-
-    for (let i = 0; i < byteString.length; i++) {
-      ia[i] = byteString.charCodeAt(i);
     }
 
-    const uuid = crypto.randomUUID(); // Generate a new UUID
-    const file = new File([ab], `${uuid}.png`, { type: mimeString });
-
-    const snapshotMediaToken = await api.uploadSceneSnapshot({
-      screenshot: file,
-    });
-
-    if (snapshotMediaToken.data === undefined) {
-      toast.error("Error: Unable to upload scene snapshot Please try again.");
-      return;
-    }
-    console.log("useSystemPrompt", useSystemPrompt);
-    console.log("Snapshot media token:", snapshotMediaToken.data);
-
-    const response = await api.enqueueImageGeneration({
-      disableSystemPrompt: !useSystemPrompt,
-      prompt: prompt,
-      snapshotMediaToken: snapshotMediaToken.data,
-      additionalImages: referenceImages.map((image) => image.mediaToken),
-    });
-
-    if (response.success === true) {
-      toast.success("Please wait while we process your image.");
-      if (response.data) {
-        addJobToken(response.data);
-      }
-      return;
-    } else {
-      toast.error("Failed to enqueue image generation. Please try again.");
-    }
-  };
-  const handleEnqueue = async () => {
-    console.log("Pressing Enqueue");
-    if (onEnqueuePressed) {
-      try {
-        await onEnqueuePressed();
-      } catch (error) {
-        console.error("Error in onEnqueuePressed callback:", error);
-        return;
-      }
-    }
-
-    if (isEnqueueing) return;
-    setIsEnqueueing(true);
-    if (!prompt.trim()) return;
-    try {
-      console.log(
-        "(4) Enqueuing with prompt:",
-        prompt,
-        "and reference images:",
-        referenceImages
-      );
-
-      const isDesktop = IsDesktopApp();
-      console.log("Is this a desktop app?", isDesktop);
-
-      if (isDesktop) {
-        await handleTauriEnqueue();
-      } else {
-        await handleWebEnqueue();
-      }
-    } catch (error) {
-      console.error("Error during image generation:", error);
-      toast.error(
-        "An error occurred while generating the image. Please try again."
-      );
-    } finally {
-      await new Promise((resolve) => setTimeout(resolve, 100));
-      setIsEnqueueing(false);
-    }
+    //   let errorMessage = "Failed to enqueue image";
+    //   if ("error_type" in generateResponse) {
+    //     switch (generateResponse.error_type) {
+    //       case SoraImageRemixErrorType.SoraLoginRequired:
+    //         errorMessage =
+    //           "You need to log into Sora to continue. See the settings menu.";
+    //         break;
+    //       case SoraImageRemixErrorType.ServerError:
+    //         errorMessage = "Server error. Failed to enqueue image.";
+    //         break;
+    //       case SoraImageRemixErrorType.TooManyConcurrentTasks:
+    //         errorMessage =
+    //           "You have too many Sora image generation tasks running. Please wait a moment.";
+    //         break;
+    //       case SoraImageRemixErrorType.SoraUsernameNotYetCreated:
+    //         errorMessage =
+    //           "Your Sora username is not yet created. Please visit the Sora.com website to create it first.";
+    //         break;
+    //       case SoraImageRemixErrorType.SoraIsHavingProblems:
+    //         errorMessage = "Sora is having problems. Please try again later.";
+    //         break;
+    //     }
+    //   }
+    //   toast.error(errorMessage);
+    // }
   };
 
   const getCurrentResolutionIcon = () => {
@@ -429,7 +254,6 @@ export const PromptBoxVideo = ({
   const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
     if (e.key === "Enter" && !e.shiftKey) {
       e.preventDefault();
-
       handleEnqueue();
     }
   };
@@ -492,14 +316,6 @@ export const PromptBoxVideo = ({
           </div>
         )}
         <div className="glass w-[730px] rounded-xl p-4">
-          <input
-            type="file"
-            ref={fileInputRef}
-            className="hidden"
-            accept="image/*"
-            onChange={handleFileUpload}
-            multiple
-          />
           <div className="flex justify-center gap-2">
             <PopoverMenu
               mode="button"