Feature/ip adapter (#522)

app/Classes/ApiManager/ApiManager.ts

@@ -160,8 +160,6 @@ export class ApiManager {
       credentials: "include",
       body: formData,
     });
-
-    return response.json();
   }
 
   protected camelToSnakeCase(str: string) {

app/pages/PageEnigma/Editor/api_manager.ts

@@ -373,6 +373,7 @@ export class APIManager {
     use_strength: number = 1.0,
     use_lipsync: boolean = false,
     use_cinematic: boolean = false,
+    use_global_ipa_media_token: string | null = null,
   ) {
     const uuid = uuidv4();
 
@@ -390,6 +391,7 @@ export class APIManager {
       use_upscaler: use_upscaler,
       use_strength: use_strength,
       use_lipsync: use_lipsync,
+      global_ipa_media_token: use_global_ipa_media_token,
       use_cinematic: use_cinematic,
     };
 

app/pages/PageEnigma/Editor/editor.ts

@@ -145,6 +145,9 @@ class Editor {
 
   generation_options: GenerationOptions;
 
+  // just a passive the image to be uploaded. we store the token and use that in snapshots.
+  globalIpAdapterImage: File | undefined;
+
   media_upload: MediaUploadApi;
 
   sceneManager: SceneManager | undefined;
@@ -244,9 +247,12 @@ class Editor {
       styleStrength: 1.0,
       lipSync: false,
       cinematic: false,
+      globalIpAdapterImageMediaToken: "",
     };
 
     this.media_upload = new MediaUploadApi();
+    this.globalIpAdapterImage = undefined; // used to display when loading in the app. and to serialize to an image token
+    // TODO REMOVE
     this.outliner_feature_flag = true;
   }
   getRenderDimensions() {
@@ -487,20 +493,45 @@ class Editor {
     this.cam_obj = this.activeScene.get_object_by_name(this.camera_name);
 
     if (this.outliner_feature_flag) {
-      let result = this.sceneManager?.render_outliner(this.timeline.characters);
+      const result = this.sceneManager?.render_outliner(
+        this.timeline.characters,
+      );
       if (result) outlinerState.items.value = result.items;
     }
 
-    this.mouse_controls = new MouseControls(this.camera, this.camera_person_mode, this.lockControls,
-      this.camera_last_pos, this.orbitControls,
-      this.selectedCanvas, this.switchPreviewToggle, this.rendering, this.togglePlayback.bind(this),
-      this.deleteObject.bind(this), this.canvReference, this.mouse, this.timeline.mouse,
-      this.raycaster, this.control, this.outlinePass, this.activeScene.scene,
-      this.publishSelect.bind(this), this.updateSelectedUI.bind(this),
-      this.transform_interaction, this.last_selected, this.getAssetType.bind(this), this.setSelected.bind(this));
+    this.mouse_controls = new MouseControls(
+      this.camera,
+      this.camera_person_mode,
+      this.lockControls,
+      this.camera_last_pos,
+      this.orbitControls,
+      this.selectedCanvas,
+      this.switchPreviewToggle,
+      this.rendering,
+      this.togglePlayback.bind(this),
+      this.deleteObject.bind(this),
+      this.canvReference,
+      this.mouse,
+      this.timeline.mouse,
+      this.raycaster,
+      this.control,
+      this.outlinePass,
+      this.activeScene.scene,
+      this.publishSelect.bind(this),
+      this.updateSelectedUI.bind(this),
+      this.transform_interaction,
+      this.last_selected,
+      this.getAssetType.bind(this),
+      this.setSelected.bind(this),
+    );
 
     if (this.outliner_feature_flag) {
-      this.sceneManager = new SceneManager(this.version, this.mouse_controls, this.activeScene, true); // Enabled dev mode.
+      this.sceneManager = new SceneManager(
+        this.version,
+        this.mouse_controls,
+        this.activeScene,
+        true,
+      ); // Enabled dev mode.
       this.mouse_controls.sceneManager = this.sceneManager;
     }
 
@@ -588,15 +619,20 @@ class Editor {
     });
 
     if (this.outliner_feature_flag) {
-      let result = this.sceneManager?.render_outliner(this.timeline.characters);
+      const result = this.sceneManager?.render_outliner(
+        this.timeline.characters,
+      );
       if (result) outlinerState.items.value = result.items;
     }
   }
 
   public async loadScene(scene_media_token: string) {
     await this.save_manager.loadScene(scene_media_token);
+
     if (this.outliner_feature_flag) {
-      let result = this.sceneManager?.render_outliner(this.timeline.characters);
+      const result = this.sceneManager?.render_outliner(
+        this.timeline.characters,
+      );
       if (result) outlinerState.items.value = result.items;
     }
     // publish to the UI the values for the prompts and artistic style and settings?
@@ -622,8 +658,8 @@ class Editor {
   }
 
   lockUnlockObject(object_uuid: string): boolean {
-    let res = this.utils.lockUnlockObject(object_uuid);
-    if(this.outliner_feature_flag) {
+    const res = this.utils.lockUnlockObject(object_uuid);
+    if (this.outliner_feature_flag) {
       this.updateSelectedUI();
     }
     return res;
@@ -734,7 +770,9 @@ class Editor {
   deleteObject(uuid: string) {
     this.utils.deleteObject(uuid);
     if (this.outliner_feature_flag) {
-      let result = this.sceneManager?.render_outliner(this.timeline.characters);
+      const result = this.sceneManager?.render_outliner(
+        this.timeline.characters,
+      );
       if (result) outlinerState.items.value = result.items;
     }
   }
@@ -763,14 +801,14 @@ class Editor {
           this.render_camera_aspect_ratio === CameraAspectRatio.HORIZONTAL_16_9
             ? 1024
             : this.render_camera_aspect_ratio ===
-              CameraAspectRatio.VERTICAL_9_16
+                CameraAspectRatio.VERTICAL_9_16
               ? 576
               : 1000;
         const height =
           this.render_camera_aspect_ratio === CameraAspectRatio.HORIZONTAL_16_9
             ? 576
             : this.render_camera_aspect_ratio ===
-              CameraAspectRatio.VERTICAL_9_16
+                CameraAspectRatio.VERTICAL_9_16
               ? 1024
               : 1000;
 
@@ -818,7 +856,9 @@ class Editor {
       this.cam_obj = this.activeScene.get_object_by_name(this.camera_name);
 
       if (this.outliner_feature_flag) {
-        let result = this.sceneManager?.render_outliner(this.timeline.characters);
+        const result = this.sceneManager?.render_outliner(
+          this.timeline.characters,
+        );
         if (result) outlinerState.items.value = result.items;
       }
     }
@@ -929,6 +969,7 @@ class Editor {
   // This initializes the generation of a video render scene is where the core work happens
   generateVideo() {
     console.log("Generating video...", this.frame_buffer);
+
     this.timeline.is_playing = false;
     this.timeline.scrubber_frame_position = 0;
     this.timeline.current_time = 0;
@@ -987,7 +1028,10 @@ class Editor {
   updateSelectedUI() {
     let mainSelected;
     if (this.outliner_feature_flag) {
-      if (this.sceneManager?.selected_objects === undefined || this.timeline.is_playing) {
+      if (
+        this.sceneManager?.selected_objects === undefined ||
+        this.timeline.is_playing
+      ) {
         return;
       }
       if (this.sceneManager?.selected_objects.length <= 0) {
@@ -995,8 +1039,7 @@ class Editor {
       }
 
       mainSelected = this.sceneManager?.selected_objects[0];
-    }
-    else {
+    } else {
       if (this.timeline.is_playing) {
         return;
       }
@@ -1099,8 +1142,11 @@ class Editor {
   }
 
   publishSelect() {
-    if (this, this.outliner_feature_flag) {
-      if (this.sceneManager?.selected_objects && this.sceneManager?.selected_objects?.length > 0) {
+    if ((this, this.outliner_feature_flag)) {
+      if (
+        this.sceneManager?.selected_objects &&
+        this.sceneManager?.selected_objects?.length > 0
+      ) {
         Queue.publish({
           queueName: QueueNames.FROM_ENGINE,
           action: fromEngineActions.SELECT_OBJECT,
@@ -1113,16 +1159,14 @@ class Editor {
           } as MediaItem,
         });
         return;
-      }
-      else {
+      } else {
         Queue.publish({
           queueName: QueueNames.FROM_ENGINE,
           action: fromEngineActions.DESELECT_OBJECT,
           data: null,
         });
       }
-    }
-    else {
+    } else {
       if (this.selected) {
         Queue.publish({
           queueName: QueueNames.FROM_ENGINE,
@@ -1136,8 +1180,7 @@ class Editor {
           } as MediaItem,
         });
         return;
-      }
-      else {
+      } else {
         Queue.publish({
           queueName: QueueNames.FROM_ENGINE,
           action: fromEngineActions.DESELECT_OBJECT,

app/pages/PageEnigma/Editor/save_manager.ts

@@ -1,3 +1,4 @@
+import { scene } from "~/signals";
 import { SceneGenereationMetaData } from "../models/sceneGenerationMetadata";
 import { StoryTellerProxyScene } from "../proxy/storyteller_proxy_scene";
 import { StoryTellerProxyTimeline } from "../proxy/storyteller_proxy_timeline";
@@ -52,13 +53,6 @@ export class SaveManager {
       timeline: timeline_json,
     };
 
-    // const result = await this.editor.api_manager.saveSceneState({
-    //   saveJson: JSON.stringify(save_data),
-    //   sceneTitle,
-    //   sceneToken,
-    //   sceneThumbnail,
-    // });
-
     return JSON.stringify(save_data);
   }
 
@@ -124,6 +118,7 @@ export class SaveManager {
     return result;
   }
 
+  // TODO Refactor remove editor.
   public async loadScene(scene_media_token: string) {
     showEditorLoader();
 
@@ -150,6 +145,10 @@ export class SaveManager {
     // this calls the signal function to propagate the data to the UI
     restoreSceneGenerationMetadata(scene_json);
     // these propogate the values into the editor
+    if (scene_json.globalIpAdapterImageMediaToken) {
+      // this should be populated right after
+      this.editor.generation_options.globalIpAdapterImageMediaToken;
+    }
     if (scene_json.positivePrompt) {
       this.editor.positive_prompt = scene_json.positivePrompt;
     }
@@ -170,6 +169,7 @@ export class SaveManager {
     this.editor.cam_obj = this.editor.activeScene.get_object_by_name(
       this.editor.camera_name,
     );
+
     this.editor.cam_obj?.layers.set(1);
     this.editor.cam_obj?.children.forEach((child) => {
       child.layers.set(1);
@@ -189,6 +189,7 @@ export class SaveManager {
     this.editor.timeline.checkEditorCanPlay();
 
     hideEditorLoader();
+    // TODO figure out if this is a bug.
     this.editor.timeline.scrub({ data: { currentTime: 0 } });
   }
 }

app/pages/PageEnigma/Editor/timeline.ts

@@ -99,7 +99,6 @@ export class TimeLine {
     );
 
     this.current_time = 0;
-
     this.camera_name = camera_name;
   }
 
@@ -198,6 +197,7 @@ export class TimeLine {
         break;
       case toEngineActions.GENERATE_VIDEO: {
         const options = data.data; // super overloaded talk to the devs about this. TODO... refactor
+        // pass this in ... rather than doing it implicitly ...
         this.editorEngine.generation_options = options as GenerationOptions;
         this.editorEngine.generateVideo();
         break;

app/pages/PageEnigma/Editor/video_generation.ts

@@ -12,14 +12,18 @@ import * as THREE from "three";
 import { getSceneSignals } from "~/signals";
 import { v4 as uuidv4 } from "uuid";
 import { SceneGenereationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata";
+import { MediaUploadApi } from "~/Classes/ApiManager";
+import { adapterImage } from "../signals";
 
 // TODO THIS CLASS MAKES NO SENSE Refactor so we generate all the frames first. then pass it through this pipeline as a data structure process it. through this class.
 
 export class VideoGeneration {
   editor: Editor;
+  mediaUploadAPI: MediaUploadApi;
 
   constructor(editor: Editor) {
     this.editor = editor;
+    this.mediaUploadAPI = new MediaUploadApi();
   }
 
   async generateFrame() {
@@ -258,17 +262,40 @@ export class VideoGeneration {
     const style_name = this.editor.art_style.toString();
     const media_token = this.editor.current_scene_media_token || undefined;
 
+    // convert the ip adapter image and upload as a media token
+    const image_uuid = uuidv4();
+    let ipa_image_token = undefined;
+    if (this.editor.globalIpAdapterImage != undefined) {
+      const response = await this.mediaUploadAPI.UploadImage({
+        fileName: `${image_uuid}.ipa`,
+        blob: this.editor.globalIpAdapterImage,
+        uuid: image_uuid,
+      });
+      if (response.success) {
+        if (response.data) {
+          ipa_image_token = response.data;
+        }
+      }
+    }
+
+    let globalIPAMediaToken = "";
+
+    if (ipa_image_token) {
+      globalIPAMediaToken = ipa_image_token;
+    }
+
+    // TODO Remove so many of these around wtf. SceneGenereationMetaData should only be one place
     const metaData: SceneGenereationMetaData = {
       artisticStyle: this.editor.art_style,
       positivePrompt: this.editor.positive_prompt,
       negativePrompt: this.editor.negative_prompt,
       cameraAspectRatio: this.editor.render_camera_aspect_ratio,
-      adapterImageToken: "",
       upscale: this.editor.generation_options.upscale,
       faceDetail: this.editor.generation_options.faceDetail,
       styleStrength: this.editor.generation_options.styleStrength,
       lipSync: this.editor.generation_options.lipSync,
       cinematic: this.editor.generation_options.cinematic,
+      globalIPAMediaToken: globalIPAMediaToken,
     };
 
     // This is to save the snapshot of the scene for remixing...
@@ -335,10 +362,13 @@ export class VideoGeneration {
         this.editor.generation_options.styleStrength,
         this.editor.generation_options.lipSync,
         this.editor.generation_options.cinematic,
+        globalIPAMediaToken,
       )
       .catch((error) => {
         // TODO handle stylize error.
         console.log(error);
+        this.editor.updateLoad(100, "Failed To Render");
+        this.editor.endLoading();
       });
     // Not sure if this is needed will double check TODO: MC
     //this.editor.generating_preview = false;
@@ -362,6 +392,7 @@ export class VideoGeneration {
       canvas: this.editor.canvasRenderCamReference || undefined,
       preserveDrawingBuffer: true,
     });
+
     this.editor.activeScene.renderMode(false);
 
     this.editor.switchEdit();

app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx

@@ -26,7 +26,7 @@ import { LoadUserScenes } from "./LoadUserScenes";
 import { NewSceneFromTemplate } from "./NewSceneFromTemplate";
 
 import { getCurrentLocationWithoutParams } from "~/utilities";
-import { SceneGenereationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata";
+import { SceneGenereationMetaData as SceneGenerationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata";
 import {
   cameraAspectRatio,
   cinematic,
@@ -70,22 +70,26 @@ export const ControlsTopButtons = () => {
     editorEngine?.newScene(sceneTitleInput);
   };
 
-  const getSceneGenereationMetaData =
-    useCallback((): SceneGenereationMetaData => {
-      // when this is called, editor engine is guarunteed by it's caller
+  const getSceneGenereationMetaData = useCallback(():
+    | SceneGenerationMetaData
+    | undefined => {
+    // when this is called, editor engine is guarunteed by it's caller
+    if (editorEngine) {
       return {
-        positivePrompt: editorEngine!.positive_prompt,
-        negativePrompt: editorEngine!.negative_prompt,
-        artisticStyle: getArtStyle(editorEngine!.art_style.toString()),
+        positivePrompt: editorEngine.positive_prompt,
+        negativePrompt: editorEngine.negative_prompt,
+        artisticStyle: getArtStyle(editorEngine.art_style.toString()),
         cameraAspectRatio: cameraAspectRatio.value,
-        adapterImageToken: "",
+        globalIPAMediaToken:
+          editorEngine.generation_options.globalIpAdapterImageMediaToken,
         upscale: upscale.value,
         faceDetail: faceDetail.value,
         styleStrength: styleStrength.value,
         lipSync: lipSync.value,
         cinematic: cinematic.value,
       };
-    }, [editorEngine]);
+    }
+  }, [editorEngine]);
 
   const handleButtonSave = async () => {
     if (!editorEngine) {
@@ -173,9 +177,9 @@ export const ControlsTopButtons = () => {
     }
   });
 
-   const handleShowOutliner = () => {
-     outlinerIsShowing.value = !outlinerIsShowing.value;
-   };
+  const handleShowOutliner = () => {
+    outlinerIsShowing.value = !outlinerIsShowing.value;
+  };
 
   return (
     <div className="flex flex-col gap-2 pl-2 pt-2">
@@ -309,7 +313,6 @@ export const ControlsTopButtons = () => {
           ]}
         />
 
-        
         <Button
           icon={outlinerIsShowing.value ? faCheckSquare : faSquare}
           className="shadow-xl"
@@ -323,7 +326,7 @@ export const ControlsTopButtons = () => {
           onClick={handleShowOutliner}
         >
           Outliner
-        </Button> 
+        </Button>
 
         <ButtonDialogue
           buttonProps={{

app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/StylizeTab/IPAdapter.tsx

@@ -1,4 +1,10 @@
-import React, { useState, useCallback, useRef } from "react";
+import React, {
+  useState,
+  useCallback,
+  useRef,
+  useContext,
+  useEffect,
+} from "react";
 import { FileUploader } from "react-drag-drop-files";
 import {
   ReactCrop,
@@ -20,8 +26,10 @@ import { Button, Label, P, TransitionDialogue } from "~/components";
 import { useSignals } from "@preact/signals-react/runtime";
 import { adapterImage } from "~/pages/PageEnigma/signals";
 
+import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
 export const IPAdapter: React.FC = () => {
   useSignals();
+  const editorEngine = useContext(EngineContext);
   const [imageSrc, setImageSrc] = useState<string | null>(null);
   const [crop, setCrop] = useState<Crop>();
   const [completedCrop, setCompletedCrop] = useState<PixelCrop | null>(null);
@@ -30,10 +38,21 @@ export const IPAdapter: React.FC = () => {
   const imgRef = useRef<HTMLImageElement | null>(null);
   const FILE_TYPES = ["JPG", "PNG", "JPEG"];
 
+  useEffect(() => {
+    if (editorEngine) {
+      // load the image token if availible
+      // Bombay load the image from the image token here.
+      editorEngine.generation_options.globalIpAdapterImageMediaToken;
+    }
+  }, []);
+
   const onFileChange = (file: File) => {
     const reader = new FileReader();
     reader.onloadend = () => {
       setImageSrc(reader.result as string);
+      if (editorEngine) {
+        editorEngine.globalIpAdapterImage = file;
+      }
       setIsDialogOpen(true);
     };
     reader.readAsDataURL(file);

app/pages/PageEnigma/models/generationOptions.ts

@@ -4,6 +4,7 @@ export interface GenerationOptions {
   styleStrength: number;
   lipSync: boolean;
   cinematic: boolean;
+  globalIpAdapterImageMediaToken: string; // base 64 image
 }
 
 export class GenerationOptions {
@@ -12,18 +13,20 @@ export class GenerationOptions {
   public styleStrength: number;
   public lipSync: boolean;
   public cinematic: boolean;
-
+  public globalIpAdapterImageMediaToken: string;
   constructor(
     upscale: boolean,
     faceDetail: boolean,
     styleStrength: number,
     lipSync: boolean,
     cinematic: boolean,
+    globalIpAdapterImageMediaToken: string,
   ) {
     this.upscale = upscale;
     this.faceDetail = faceDetail;
     this.styleStrength = styleStrength;
     this.lipSync = lipSync;
     this.cinematic = cinematic;
+    this.globalIpAdapterImageMediaToken = globalIpAdapterImageMediaToken;
   }
 }

app/pages/PageEnigma/models/sceneGenerationMetadata.ts

@@ -7,7 +7,7 @@ export interface SceneGenereationMetaData {
   negativePrompt: string;
 
   cameraAspectRatio: CameraAspectRatio;
-  adapterImageToken: string;
+  globalIPAMediaToken: string;
 
   upscale: boolean;
   faceDetail: boolean;