Added reference images to the generation

enqueueImageGeneration now accepts additionalImages
enqueueImageGeneration needs testing.
Polling for jobs doesn't work,
most api calls don't work and need integration testing.

frontend/editor3d/app/Classes/ApiManager/EngineApi.ts

@@ -47,130 +47,157 @@ export class EngineApi extends ApiManager {
     const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/scene_snapshot`;
 
     const formData = new FormData();
-    formData.append("screenshot", screenshot);
+    formData.append("snapshot", screenshot); // Changed from "screenshot" to "snapshot" to match API spec
     if (sceneMediaToken) {
       formData.append("scene_media_token", sceneMediaToken);
     }
 
-    return this.post<
-      FormData,
-      { success?: boolean; snapshot_media_token?: string; BadInput?: string }
-    >({
-      endpoint,
-      body: formData,
+    const response = await fetch(endpoint, {
+      method: "POST",
       headers: {
-        "Content-Type": "multipart/form-data",
+        Accept: "application/json",
       },
-    })
-      .then((response) => {
-        return {
-          success: response.success ?? false,
-          data: response.snapshot_media_token,
-          errorMessage: response.BadInput,
-        };
-      })
-      .catch((err) => {
-        return {
-          success: false,
-          errorMessage: err.message,
-        };
-      });
+      credentials: "include",
+      body: formData,
+    });
+
+    const postResponse = await response.json();
+
+    console.log(postResponse);
+
+    let result: { success: boolean; data?: string; errorMessage?: string };
+
+    if (postResponse.success) {
+      result = {
+        success: true,
+        data: postResponse.snapshot_media_token,
+        errorMessage: undefined,
+      };
+    } else {
+      result = {
+        success: false,
+        errorMessage: postResponse.BadInput,
+      };
+    }
+
+    return result;
   }
 
   public async enqueueImageGeneration({
     prompt,
-    sceneMediaToken,
+    snapshotMediaToken,
     additionalImages,
   }: {
     prompt: string;
-    sceneMediaToken: string;
+    snapshotMediaToken: string;
     additionalImages?: string[];
   }): Promise<ApiResponse<string>> {
     const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/prompt`;
 
     const body = {
       prompt,
-      scene_media_token: sceneMediaToken,
+      snapshot_media_token: snapshotMediaToken, // Changed from scene_media_token to snapshot_media_token
       additional_images: additionalImages,
     };
 
-    return this.post<
+    const postResponse = await this.post<
       {
         prompt: string;
-        scene_media_token: string;
+        snapshot_media_token: string;
         additional_images?: string[];
       },
       { success?: boolean; job_token?: string; BadInput?: string }
     >({
       endpoint,
       body,
-    })
-      .then((response) => {
-        return {
-          success: response.success ?? false,
-          data: response.job_token,
-          errorMessage: response.BadInput,
-        };
-      })
-      .catch((err) => {
-        return {
-          success: false,
-          errorMessage: err.message,
-        };
-      });
+    });
+
+    // Check if the response is successful
+    const isSuccess = postResponse.success ?? false;
+
+    // Prepare the result object
+    const result = {
+      success: isSuccess,
+      data: isSuccess ? postResponse.job_token : undefined,
+      errorMessage: isSuccess ? undefined : postResponse.BadInput,
+    };
+
+    return result;
   }
 
-  public async pollJobSession(jobToken: string): Promise<ApiResponse<any>> {
+  public async pollJobSession(jobToken: string): Promise<
+    ApiResponse<{
+      status: string;
+      result: {
+        generated_images?: string[];
+        error?: string;
+      };
+    }>
+  > {
     const endpoint = `${this.ApiTargets.BaseApi}/v1/jobs/session/${jobToken}`;
 
-    return this.get<{
+    const response = await this.get<{
       success?: boolean;
       status?: string;
-      result?: any;
+      result?: {
+        generated_images?: string[];
+        error?: string;
+      };
       BadInput?: string;
     }>({
       endpoint,
-    })
-      .then((response) => {
-        return {
-          success: response.success ?? false,
-          data: response.result,
-          errorMessage: response.BadInput,
-        };
-      })
-      .catch((err) => {
-        return {
-          success: false,
-          errorMessage: err.message,
-        };
-      });
+    });
+
+    const success = response.success ?? false;
+    const status = response.status ?? "";
+    const result = response.result ?? {
+      generated_images: [],
+      error: undefined,
+    };
+    const errorMessage = response.BadInput;
+
+    return {
+      success,
+      data: { status, result },
+      errorMessage,
+    };
   }
 
-  public async pollStudioSessionJobs(
-    jobToken: string,
-  ): Promise<ApiResponse<any>> {
+  public async pollStudioSessionJobs(jobToken: string): Promise<
+    ApiResponse<{
+      status: string;
+      result: {
+        generated_images?: string[];
+        error?: string;
+      };
+    }>
+  > {
     const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/session_jobs/${jobToken}`;
 
-    return this.get<{
+    const response = await this.get<{
       success?: boolean;
       status?: string;
-      result?: any;
+      result?: {
+        generated_images?: string[];
+        error?: string;
+      };
       BadInput?: string;
     }>({
       endpoint,
-    })
-      .then((response) => {
-        return {
-          success: response.success ?? false,
-          data: response.result,
-          errorMessage: response.BadInput,
-        };
-      })
-      .catch((err) => {
-        return {
-          success: false,
-          errorMessage: err.message,
-        };
-      });
+    });
+
+    const success = response.success ?? false;
+    const status = response.status ?? "";
+    const result = response.result ?? {
+      generated_images: [],
+      error: undefined,
+    };
+    const errorMessage = response.BadInput;
+
+    return {
+      success,
+      data: { status, result },
+      errorMessage,
+    };
   }
 }

frontend/editor3d/app/components/reusable/UploadModalMedia/uploadImage.ts

@@ -15,6 +15,8 @@ export const uploadImage = async ({
 }) => {
   const mediaUploadApi = new MediaUploadApi();
 
+  console.log("Uploading image:", assetFile);
+
   progressCallback({ status: UploaderStates.uploadingImage });
 
   const imageResponse = await mediaUploadApi.UploadImage({
@@ -24,7 +26,10 @@ export const uploadImage = async ({
     maybe_title: "char_frame_" + title,
   });
 
+  console.log("Image response:", imageResponse);
+
   if (imageResponse == undefined) {
+    console.log("Error: Could not upload image!");
     progressCallback({
       status: UploaderStates.imageCreateError,
       errorMessage: "Could not upload image!",
@@ -33,6 +38,7 @@ export const uploadImage = async ({
   }
 
   if (!imageResponse.success || !imageResponse.data) {
+    console.log("Error:", imageResponse.errorMessage);
     progressCallback({
       status: UploaderStates.imageCreateError,
       errorMessage: imageResponse.errorMessage,
@@ -40,5 +46,9 @@ export const uploadImage = async ({
     return;
   }
 
-  progressCallback({ status: UploaderStates.success, data: imageResponse.data });
+  console.log("Upload successful:", imageResponse.data);
+  progressCallback({
+    status: UploaderStates.success,
+    data: imageResponse.data,
+  });
 };

frontend/editor3d/app/pages/PageEnigma/Editor/editor.ts

@@ -657,14 +657,21 @@ class Editor {
 
   public snapShotOfCurrentFrame(shouldDownload: boolean = true) {
     const snapshot = this.renderer?.domElement.toDataURL("image/png");
+    if (!snapshot) {
+      console.error("Error: Snapshot could not be created.");
+      return null;
+    }
 
-    if (snapshot) {
+    const base64Snapshot = snapshot.split(",")[1];
+
+    if (shouldDownload) {
       const link = document.createElement("a");
       link.download = "scene-snapshot.png";
       link.href = snapshot;
       link.click();
     }
-    return snapshot;
+
+    return base64Snapshot;
   }
 
   public async newScene(sceneTitleInput: string) {

frontend/editor3d/app/pages/PageEnigma/comps/PromptBox/PromptBox.tsx

@@ -2,7 +2,7 @@ import { useState, useRef, useEffect } from "react";
 import { useSignals } from "@preact/signals-react/runtime";
 import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
 import { useContext } from "react";
-
+import { uploadImage } from "~/components/reusable/UploadModalMedia/uploadImage";
 import {
   faPlus,
   faCamera,
@@ -34,9 +34,11 @@ interface ReferenceImage {
   id: string;
   url: string;
   file: File;
+  mediaToken: string;
 }
 import { EngineApi } from "~/Classes/ApiManager/EngineApi";
 import { scene } from "~/signals/scene";
+import { UploaderStates } from "~/enums/UploaderStates";
 
 export const PromptBox = () => {
   useSignals();
@@ -169,14 +171,25 @@ export const PromptBox = () => {
       Array.from(files).forEach((file) => {
         const reader = new FileReader();
         reader.onloadend = () => {
-          setReferenceImages((prev) => [
-            ...prev,
-            {
-              id: Math.random().toString(36).substring(7),
-              url: reader.result as string,
-              file,
+          uploadImage({
+            title: `reference-image-${Math.random().toString(36).substring(2, 15)}`,
+            assetFile: file,
+            progressCallback: (newState) => {
+              console.debug("Upload progress:", newState.data);
+              if (newState.status === UploaderStates.success && newState.data) {
+                const referenceImage: ReferenceImage = {
+                  id: Math.random().toString(36).substring(7),
+                  url: reader.result as string,
+                  file,
+                  mediaToken: newState.data || "",
+                };
+                setReferenceImages((prev) => [...prev, referenceImage]);
+                console.log("Reference image added:", referenceImage);
+              } else {
+                console.error("No data from uploadImage");
+              }
             },
-          ]);
+          });
         };
         reader.readAsDataURL(file);
       });
@@ -226,28 +239,34 @@ export const PromptBox = () => {
 
     if (editorEngine) {
       editorEngine.positive_prompt = prompt;
-    }
 
-    const engineApi = new EngineApi();
+      const engineApi = new EngineApi();
+      const snapshot = editorEngine.snapShotOfCurrentFrame(false);
 
-    engineApi.enqueueImageGeneration({
-      prompt: prompt,
-      sceneMediaToken: scene.value.token || "",
-      additionalImages: [],
-    });
+      const snapshotResult = await engineApi.uploadSceneSnapshot({
+        screenshot: snapshot || "",
+        sceneMediaToken: "",
+      });
 
-    setisEnqueueing(true);
-    try {
-      // Here we would pass both the prompt and reference images to the generation
-      console.log(
-        "Enqueuing with prompt:",
-        prompt,
-        "and reference images:",
-        referenceImages,
-      );
-      await new Promise((resolve) => setTimeout(resolve, 2000));
-    } finally {
-      setisEnqueueing(false);
+      await engineApi.enqueueImageGeneration({
+        prompt: prompt,
+        snapshotMediaToken: snapshotResult.data || "",
+        additionalImages: referenceImages.map((image) => image.mediaToken),
+      });
+
+      setisEnqueueing(true);
+      try {
+        // Here we would pass both the prompt and reference images to the generation
+        console.log(
+          "Enqueuing with prompt:",
+          prompt,
+          "and reference images:",
+          referenceImages,
+        );
+        await new Promise((resolve) => setTimeout(resolve, 2000));
+      } finally {
+        setisEnqueueing(false);
+      }
     }
   };
 
@@ -257,37 +276,37 @@ export const PromptBox = () => {
       handleEnqueue();
     }
   };
-  
-// Get the current aspect ratio icon based on the cameraAspectRatio signal
-const getCurrentAspectRatioIcon = () => {
-  switch (cameraAspectRatio.value) {
-    case CameraAspectRatio.HORIZONTAL_16_9:
-      return faRectangleWide;
-    case CameraAspectRatio.HORIZONTAL_3_2:
-      return faRectangle;
-    case CameraAspectRatio.VERTICAL_2_3:
-      return faRectangleVertical;
-    case CameraAspectRatio.VERTICAL_9_16:
-      return faRectangleVertical;
-    case CameraAspectRatio.SQUARE_1_1:
-      return faSquare;
-    default:
-      return faRectangleWide;
+
+  // Get the current aspect ratio icon based on the cameraAspectRatio signal
+  const getCurrentAspectRatioIcon = () => {
+    switch (cameraAspectRatio.value) {
+      case CameraAspectRatio.HORIZONTAL_16_9:
+        return faRectangleWide;
+      case CameraAspectRatio.HORIZONTAL_3_2:
+        return faRectangle;
+      case CameraAspectRatio.VERTICAL_2_3:
+        return faRectangleVertical;
+      case CameraAspectRatio.VERTICAL_9_16:
+        return faRectangleVertical;
+      case CameraAspectRatio.SQUARE_1_1:
+        return faSquare;
+      default:
+        return faRectangleWide;
     }
   };
 
-  const handleSaveFrame = () => {
+  const handleSaveFrame = async () => {
     if (editorEngine) {
       const snapshot = editorEngine.snapShotOfCurrentFrame(false);
-
       const engineApi = new EngineApi();
-
-      engineApi.uploadSceneSnapshot({
+      const result = await engineApi.uploadSceneSnapshot({
         screenshot: snapshot || "",
-        sceneMediaToken: scene.value.token || "",
+        sceneMediaToken: "",
       });
+      console.log(result);
     }
   };
+
   const handleDownloadFrame = () => {
     if (editorEngine) {
       editorEngine.snapShotOfCurrentFrame(true);
@@ -396,6 +415,7 @@ const getCurrentAspectRatioIcon = () => {
               className="flex items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
               variant="secondary"
               icon={faDownload}
+              onClick={handleDownloadFrame}
             >
               Download frame
             </Button>
@@ -403,6 +423,7 @@ const getCurrentAspectRatioIcon = () => {
               className="flex items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
               variant="secondary"
               icon={faSave}
+              onClick={handleSaveFrame}
             >
               Save frame
             </Button>