Feature/Depth options (#609)

Co-authored-by: Quicksticks-oss <chongobongotoday@gmail.com>
Co-authored-by: bflatastic <bombayapirat@gmail.com>

app/api/index.tsx

@@ -1,34 +1,5 @@
 export const apiHost = "https://api.storyteller.ai";
 
-// Authentications
-// const createAccount = `${apiHost}}/create_account`;
-// export const login = `${apiHost}/v1/login`;
-// export const getSession = `${apiHost}/v1/session`;
-// export const logout = `${apiHost}/v1/logout`;
-
-// Inference Jobs
-// export const getRecentJobs = `${apiHost}/v1/jobs/session`;
-// export const getActiveJobs = `${apiHost}/v1/jobs/session?exclude_states=complete_success,complete_failure,dead,cancelled_by_user,cancelled_by_system`;
-// export const getJobStatus = (jobToken: string) =>
-// `${apiHost}/v1/model_inference/job_status/${jobToken}`;
-
-// TTS
-// export const inferTts = `${apiHost}/tts/inference`;
-// export const listTts = `${apiHost}/tts/list`;
-
-//V2V
-// export const listV2V = `${apiHost}/v1/voice_conversion/model_list`;
-// export const inferV2V = `${apiHost}/v1/voice_conversion/inference`;
-// export const uploadAudioV2V = `${apiHost}/v1/media_uploads/upload_audio`;
-
-// User Media
-// export const listMediaByUser = (username: string) =>
-//   `${apiHost}/v1/media_files/list/user/${username}`;
-
-// Get Any Media
-// export const getMediaFileByToken = (fileToken: string) =>
-//   `${apiHost}/v1/media_files/file/${fileToken}`;
-
 // Upload File
 export const uploadMedia = `${apiHost}/v1/media_files/upload`;
 
@@ -37,5 +8,3 @@ export const uploadNewScene = `${apiHost}/v1/media_files/upload/new_scene`;
 export const uploadThumbnail = `${apiHost}/v1/media_files/cover_image/`;
 export const updateExistingScene = (sceneToken: string) =>
   `${apiHost}/v1/media_files/upload/saved_scene/${sceneToken}`;
-// export const renameScene = (sceneToken: string) =>
-//   `${apiHost}/v1/media_files/rename/${sceneToken}`;

app/pages/PageEnigma/Editor/CustomOutlinePass.js

@@ -0,0 +1,278 @@
+import * as THREE from "three";
+import { Pass } from "three/examples/jsm/postprocessing/Pass.js";
+import { FullScreenQuad } from "three/examples/jsm/postprocessing/Pass.js";
+import {
+  getSurfaceIdMaterial,
+  getDebugSurfaceIdMaterial,
+} from "./FindSurfaces.js";
+
+// Follows the structure of
+// 		https://github.com/mrdoob/three.js/blob/master/examples/jsm/postprocessing/OutlinePass.js
+class CustomOutlinePass extends Pass {
+  constructor(resolution, scene, camera) {
+    super();
+
+    this.renderScene = scene;
+    this.renderCamera = camera;
+    this.resolution = new THREE.Vector2(resolution.x, resolution.y);
+
+    this.fsQuad = new FullScreenQuad(null);
+    this.fsQuad.material = this.createOutlinePostProcessMaterial();
+
+    // Create a buffer to store the normals of the scene onto
+    // or store the "surface IDs"
+    const surfaceBuffer = new THREE.WebGLRenderTarget(
+      this.resolution.x,
+      this.resolution.y,
+    );
+    surfaceBuffer.texture.format = THREE.RGBAFormat;
+    surfaceBuffer.texture.type = THREE.HalfFloatType;
+    surfaceBuffer.texture.minFilter = THREE.NearestFilter;
+    surfaceBuffer.texture.magFilter = THREE.NearestFilter;
+    surfaceBuffer.texture.generateMipmaps = false;
+    surfaceBuffer.stencilBuffer = false;
+    this.surfaceBuffer = surfaceBuffer;
+
+    this.normalOverrideMaterial = new THREE.MeshNormalMaterial();
+    this.surfaceIdOverrideMaterial = getSurfaceIdMaterial();
+    this.surfaceIdDebugOverrideMaterial = getDebugSurfaceIdMaterial();
+    this.save_ = false;
+  }
+
+  dispose() {
+    this.surfaceBuffer.dispose();
+    this.fsQuad.dispose();
+  }
+
+  updateMaxSurfaceId(maxSurfaceId) {
+    this.surfaceIdOverrideMaterial.uniforms.maxSurfaceId.value = maxSurfaceId;
+  }
+
+  setSize(width, height) {
+    this.surfaceBuffer.setSize(width, height);
+    this.resolution.set(width, height);
+
+    this.fsQuad.material.uniforms.screenSize.value.set(
+      this.resolution.x,
+      this.resolution.y,
+      1 / this.resolution.x,
+      1 / this.resolution.y,
+    );
+  }
+
+  getDebugVisualizeValue() {
+    return this.fsQuad.material.uniforms.debugVisualize.value;
+  }
+
+  isUsingSurfaceIds() {
+    const debugVisualize = this.getDebugVisualizeValue();
+
+    return (
+      debugVisualize == 0 || // Main outlines v2 mode
+      debugVisualize == 5 || // Render just surfaceID debug buffer
+      debugVisualize == 6
+    ); // Render just outlines with surfaceId
+  }
+
+  save() {
+    this.save_ = true;
+  }
+
+  render(renderer, writeBuffer, readBuffer) {
+    const depthBufferValue = writeBuffer.depthBuffer;
+    writeBuffer.depthBuffer = false;
+
+    renderer.setRenderTarget(this.surfaceBuffer);
+    const overrideMaterialValue = this.renderScene.overrideMaterial;
+    this.renderScene.overrideMaterial = this.normalOverrideMaterial;
+
+    renderer.render(this.renderScene, this.renderCamera);
+    this.renderScene.overrideMaterial = overrideMaterialValue;
+
+    this.fsQuad.material.uniforms["depthBuffer"].value =
+      readBuffer.depthTexture;
+    this.fsQuad.material.uniforms["surfaceBuffer"].value =
+      this.surfaceBuffer.texture;
+    this.fsQuad.material.uniforms["sceneColorBuffer"].value =
+      readBuffer.texture;
+
+    renderer.setRenderTarget(writeBuffer);
+    this.fsQuad.render(renderer);
+
+    // Reset the depthBuffer value so we continue writing to it in the next render.
+    writeBuffer.depthBuffer = depthBufferValue;
+  }
+
+  get vertexShader() {
+    return `
+			varying vec2 vUv;
+			void main() {
+				vUv = uv;
+				gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
+			}
+			`;
+  }
+  get fragmentShader() {
+    return `
+			#include <packing>
+			// The above include imports "perspectiveDepthToViewZ"
+			// and other GLSL functions from ThreeJS we need for reading depth.
+			uniform sampler2D sceneColorBuffer;
+			uniform sampler2D depthBuffer;
+			uniform sampler2D surfaceBuffer;
+			uniform float cameraNear;
+			uniform float cameraFar;
+			uniform vec4 screenSize;
+			uniform vec3 outlineColor;
+			uniform vec4 multiplierParameters;
+			uniform int debugVisualize;
+
+			varying vec2 vUv;
+
+			// Helper functions for reading from depth buffer.
+			float readDepth (sampler2D depthSampler, vec2 coord) {
+				float fragCoordZ = texture2D(depthSampler, coord).x;
+				float viewZ = perspectiveDepthToViewZ( fragCoordZ, cameraNear, cameraFar );
+				return viewZToOrthographicDepth( viewZ, cameraNear, cameraFar );
+			}
+			float getLinearDepth(vec3 pos) {
+				return -(viewMatrix * vec4(pos, 1.0)).z;
+			}
+
+			float getLinearScreenDepth(sampler2D map) {
+					vec2 uv = gl_FragCoord.xy * screenSize.zw;
+					return readDepth(map,uv);
+			}
+
+			// Helper functions for reading normals and depth of neighboring pixels.
+			float getPixelDepth(int x, int y) {
+				// screenSize.zw is pixel size 
+				// vUv is current position
+				return readDepth(depthBuffer, vUv + screenSize.zw * vec2(x, y));
+			}
+			// "surface value" is either the normal or the "surfaceID"
+			vec3 getSurfaceValue(int x, int y) {
+				vec3 val = texture2D(surfaceBuffer, vUv + screenSize.zw * vec2(x, y)).rgb;
+				return val;
+			}
+
+			float saturateValue(float num) {
+				return clamp(num, 0.0, 1.0);
+			}
+
+			float getSufaceIdDiff(vec3 surfaceValue) {
+				float surfaceIdDiff = 0.0;
+				surfaceIdDiff += distance(surfaceValue, getSurfaceValue(1, 0));
+				surfaceIdDiff += distance(surfaceValue, getSurfaceValue(0, 1));
+				surfaceIdDiff += distance(surfaceValue, getSurfaceValue(0, 1));
+				surfaceIdDiff += distance(surfaceValue, getSurfaceValue(0, -1));
+
+				surfaceIdDiff += distance(surfaceValue, getSurfaceValue(1, 1));
+				surfaceIdDiff += distance(surfaceValue, getSurfaceValue(1, -1));
+				surfaceIdDiff += distance(surfaceValue, getSurfaceValue(-1, 1));
+				surfaceIdDiff += distance(surfaceValue, getSurfaceValue(-1, -1));
+				return surfaceIdDiff;
+			}
+
+			void main() {
+				vec4 sceneColor = texture2D(sceneColorBuffer, vUv);
+				float depth = getPixelDepth(0, 0);
+
+				// "surfaceValue" is either the normal or the surfaceId
+				vec3 surfaceValue = getSurfaceValue(0, 0);
+
+				// Get the difference between depth of neighboring pixels and current.
+				float depthDiff = 0.0;
+				depthDiff += abs(depth - getPixelDepth(1, 0));
+				depthDiff += abs(depth - getPixelDepth(-1, 0));
+				depthDiff += abs(depth - getPixelDepth(0, 1));
+				depthDiff += abs(depth - getPixelDepth(0, -1));
+
+				// Get the difference between surface values of neighboring pixels
+				// and current
+				float surfaceValueDiff = getSufaceIdDiff(surfaceValue);
+				
+				// Apply multiplier & bias to each 
+				float depthBias = multiplierParameters.x;
+				float depthMultiplier = multiplierParameters.y;
+				float normalBias = multiplierParameters.z;
+				float normalMultiplier = multiplierParameters.w;
+
+				depthDiff = depthDiff * depthMultiplier;
+				depthDiff = saturateValue(depthDiff);
+				depthDiff = pow(depthDiff, depthBias);
+
+				if (debugVisualize != 0 && debugVisualize != 6) {
+					// Apply these params when using
+					// normals instead of surfaceIds
+					surfaceValueDiff = surfaceValueDiff * normalMultiplier;
+					surfaceValueDiff = saturateValue(surfaceValueDiff);
+					surfaceValueDiff = pow(surfaceValueDiff, normalBias);
+				} else {
+					if (surfaceValueDiff != 0.0) surfaceValueDiff = 1.0;
+				}
+
+				float outline = saturateValue(surfaceValueDiff + depthDiff);
+			
+				// Combine outline with scene color.
+				vec4 outlineColor = vec4(outlineColor, 1.0);
+				gl_FragColor = vec4(mix(sceneColor, outlineColor, outline));
+
+				//// For debug visualization of the different inputs to this shader.
+				if (debugVisualize == 2) {
+					gl_FragColor = sceneColor;
+				}
+				if (debugVisualize == 3) {
+					gl_FragColor = vec4(vec3(depth), 1.0);
+          gl_FragColor.rgb = 1.0 - gl_FragColor.rgb;
+				}
+				if (debugVisualize == 4 || debugVisualize == 5) {
+					// 4 visualizes the normal buffer
+					// 5 visualizes the surfaceID buffer 
+					// Either way they are the same buffer, we change 
+					// what we render into it
+					gl_FragColor = vec4(surfaceValue, 1.0);
+				}
+				if (debugVisualize == 6 || debugVisualize == 7) {
+          // Apply threshold
+          if (outline > 0.75) {
+              outline = 1.0;
+          } else {
+              outline = 0.0;
+          }
+					gl_FragColor = vec4(vec3(outline * outlineColor), 1.0);
+				}				
+			}
+			`;
+  }
+
+  createOutlinePostProcessMaterial() {
+    return new THREE.ShaderMaterial({
+      uniforms: {
+        debugVisualize: { value: 0 },
+        sceneColorBuffer: {},
+        depthBuffer: {},
+        surfaceBuffer: {},
+        outlineColor: { value: new THREE.Color(0xffffff) },
+        //4 scalar values packed in one uniform: depth multiplier, depth bias, and same for normals.
+        multiplierParameters: {
+          value: new THREE.Vector4(0.9, 20, 1, 1),
+        },
+        cameraNear: { value: 0.1 },
+        cameraFar: { value: 3 },
+        screenSize: {
+          value: new THREE.Vector4(
+            this.resolution.x,
+            this.resolution.y,
+            1 / this.resolution.x,
+            1 / this.resolution.y,
+          ),
+        },
+      },
+      vertexShader: this.vertexShader,
+      fragmentShader: this.fragmentShader,
+    });
+  }
+}
+
+export { CustomOutlinePass };

app/pages/PageEnigma/Editor/FindSurfaces.js

@@ -0,0 +1,193 @@
+import * as THREE from "three";
+
+/*
+  This class computes "surface IDs" for a given mesh.
+
+  A "surface" is defined as a set of triangles that share vertices.
+  
+  Inspired by Ian MacLarty, see:
+    https://twitter.com/ianmaclarty/status/1499494878908403712
+*/
+class FindSurfaces {
+  constructor() {
+    // This identifier, must be globally unique for each surface
+    // across all geometry rendered on screen
+    this.surfaceId = 0;
+  }
+
+  /*
+   * Returns the surface Ids as a Float32Array that can be inserted as a vertex attribute
+   */
+  getSurfaceIdAttribute(mesh) {
+    const bufferGeometry = mesh.geometry;
+    const numVertices = bufferGeometry.attributes.position.count;
+    const vertexIdToSurfaceId = this._generateSurfaceIds(mesh);
+
+    const colors = [];
+    for (let i = 0; i < numVertices; i++) {
+      const vertexId = i;
+      let surfaceId = vertexIdToSurfaceId[vertexId];
+
+      colors.push(surfaceId, 0, 0, 1);
+    }
+
+    const colorsTypedArray = new Float32Array(colors);
+    return colorsTypedArray;
+  }
+
+  /*
+   * Returns a `vertexIdToSurfaceId` map
+   * given a vertex, returns the surfaceId
+   */
+  _generateSurfaceIds(mesh) {
+    const bufferGeometry = mesh.geometry;
+    //const numVertices = bufferGeometry.attributes.position.count;
+    const numIndices = bufferGeometry.index.count;
+    const indexBuffer = bufferGeometry.index.array;
+    //const vertexBuffer = bufferGeometry.attributes.position.array;
+    // For each vertex, search all its neighbors
+    const vertexMap = {};
+    for (let i = 0; i < numIndices; i += 3) {
+      const i1 = indexBuffer[i + 0];
+      const i2 = indexBuffer[i + 1];
+      const i3 = indexBuffer[i + 2];
+
+      add(i1, i2);
+      add(i1, i3);
+      add(i2, i3);
+    }
+    function add(a, b) {
+      if (vertexMap[a] == undefined) vertexMap[a] = [];
+      if (vertexMap[b] == undefined) vertexMap[b] = [];
+
+      if (vertexMap[a].indexOf(b) == -1) vertexMap[a].push(b);
+      if (vertexMap[b].indexOf(a) == -1) vertexMap[b].push(a);
+    }
+
+    // Find cycles
+    const frontierNodes = Object.keys(vertexMap).map((v) => Number(v));
+    const exploredNodes = {};
+    const vertexIdToSurfaceId = {};
+
+    while (frontierNodes.length > 0) {
+      const node = frontierNodes.pop();
+      if (exploredNodes[node]) continue;
+
+      // Get all neighbors recursively
+      const surfaceVertices = getNeighborsNonRecursive(node);
+      // Mark them as explored
+      for (let v of surfaceVertices) {
+        exploredNodes[v] = true;
+        vertexIdToSurfaceId[v] = this.surfaceId;
+      }
+
+      this.surfaceId += 1;
+    }
+
+    // function getNeighbors(node, explored) {
+    //   const neighbors = vertexMap[node];
+    //   let result = [node];
+    //   explored[node] = true;
+
+    //   for (let n of neighbors) {
+    //     if (explored[n]) continue;
+    //     explored[n] = true;
+    //     const newNeighbors = getNeighbors(n, explored);
+    //     result = result.concat(newNeighbors);
+    //   }
+
+    //   return result;
+    // }
+
+    function getNeighborsNonRecursive(node) {
+      const frontier = [node];
+      const explored = {};
+      const result = [];
+
+      while (frontier.length > 0) {
+        const currentNode = frontier.pop();
+        if (explored[currentNode]) continue;
+        const neighbors = vertexMap[currentNode];
+        result.push(currentNode);
+
+        explored[currentNode] = true;
+
+        for (let n of neighbors) {
+          if (!explored[n]) {
+            frontier.push(n);
+          }
+        }
+      }
+
+      return result;
+    }
+
+    return vertexIdToSurfaceId;
+  }
+}
+
+export default FindSurfaces;
+
+export function getSurfaceIdMaterial() {
+  return new THREE.ShaderMaterial({
+    uniforms: {
+      maxSurfaceId: { value: 1 },
+    },
+    vertexShader: getVertexShader(),
+    fragmentShader: getFragmentShader(),
+    vertexColors: true,
+  });
+}
+
+function getVertexShader() {
+  return `
+  varying vec2 v_uv;
+  varying vec4 vColor;
+
+  void main() {
+     v_uv = uv;
+     vColor = color;
+
+     gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
+  }
+  `;
+}
+
+function getFragmentShader() {
+  return `
+  varying vec2 v_uv;
+  varying vec4 vColor;
+  uniform float maxSurfaceId;
+
+  void main() {
+    // Normalize the surfaceId when writing to texture
+    // Surface ID needs rounding as precision can be lost in perspective correct interpolation 
+    // - see https://github.com/OmarShehata/webgl-outlines/issues/9 for other solutions eg. flat interpolation.
+    float surfaceId = round(vColor.r) / maxSurfaceId;
+    gl_FragColor = vec4(surfaceId, 0.0, 0.0, 1.0);
+  }
+  `;
+}
+
+// For debug rendering, assign a random color
+// to each surfaceId
+export function getDebugSurfaceIdMaterial() {
+  return new THREE.ShaderMaterial({
+    uniforms: {},
+    vertexShader: getVertexShader(),
+    fragmentShader: `
+  varying vec2 v_uv;
+  varying vec4 vColor;
+
+  void main() {      
+      int surfaceId = int(round(vColor.r) * 100.0);
+      float R = float(surfaceId % 255) / 255.0;
+      float G = float((surfaceId + 50) % 255) / 255.0;
+      float B = float((surfaceId * 20) % 255) / 255.0;
+
+      gl_FragColor = vec4(R, G, B, 1.0);
+  }
+  `,
+    vertexColors: true,
+  });
+}

app/pages/PageEnigma/Editor/api_manager.ts

@@ -5,6 +5,24 @@ import { signalScene, startPollingActiveJobs } from "~/signals";
 import { updateExistingScene, uploadNewScene } from "./api_fetchers";
 import { uploadThumbnail } from "~/api";
 import environmentVariables from "~/Classes/EnvironmentVariables";
+
+interface StylizeVideoParams {
+  media_token: string;
+  style: ArtStyle;
+  positive_prompt: string;
+  negative_prompt: string;
+  visibility: Visibility;
+  use_face_detailer?: boolean;
+  use_upscaler?: boolean;
+  use_strength?: number;
+  use_lipsync?: boolean;
+  use_cinematic?: boolean;
+  use_global_ipa_media_token?: string | null;
+  input_depth_file?: string | null;
+  input_normal_file?: string | null;
+  input_outline_file?: string | null;
+}
+
 /**
  * Storyteller Studio API Manager
  * The source of truth of all these media items is the database in the cloud
@@ -358,19 +376,22 @@ export class APIManager {
     return result;
   }
 
-  public async stylizeVideo(
-    media_token: string,
-    style: ArtStyle,
-    positive_prompt: string,
-    negative_prompt: string,
-    visibility: Visibility,
-    use_face_detailer: boolean = false,
-    use_upscaler: boolean = false,
-    use_strength: number = 1.0,
-    use_lipsync: boolean = false,
-    use_cinematic: boolean = false,
-    use_global_ipa_media_token: string | null = null,
-  ) {
+  public async stylizeVideo({
+    media_token,
+    style,
+    positive_prompt,
+    negative_prompt,
+    visibility,
+    use_face_detailer = false,
+    use_upscaler = false,
+    use_strength = 1.0,
+    use_lipsync = false,
+    use_cinematic = false,
+    use_global_ipa_media_token = null,
+    input_depth_file = null,
+    input_normal_file = null,
+    input_outline_file = null,
+  }: StylizeVideoParams) {
     const uuid = uuidv4();
 
     const data = {
@@ -389,6 +410,9 @@ export class APIManager {
       use_lipsync: use_lipsync,
       global_ipa_media_token: use_global_ipa_media_token,
       use_cinematic: use_cinematic,
+      input_depth_file: input_depth_file,
+      input_normal_file: input_normal_file,
+      input_outline_file: input_outline_file,
     };
 
     const json_data = JSON.stringify(data);

app/pages/PageEnigma/Editor/editor.ts

@@ -41,11 +41,13 @@ import {
   signalScene,
 } from "~/signals";
 import { outlinerState, updateObjectPanel } from "../signals";
-import { GenerationOptions } from "../models/generationOptions";
+import { IGenerationOptions } from "../models/generationOptions";
 import { toEngineActions } from "../Queue/toEngineActions";
 import { SceneGenereationMetaData } from "../models/sceneGenerationMetadata";
 import { MediaUploadApi } from "~/Classes/ApiManager";
 import { SceneManager } from "./scene_manager_api";
+import { CustomOutlinePass } from "./CustomOutlinePass.js";
+import FindSurfaces from "./FindSurfaces.js";
 
 export type EditorInitializeConfig = {
   sceneToken: string;
@@ -68,11 +70,13 @@ class Editor {
   canvasRenderCamReference: HTMLCanvasElement | null = null;
 
   composer: EffectComposer | undefined;
+  render_composer: EffectComposer | undefined;
   outlinePass: OutlinePass | undefined;
   last_cam_pos: THREE.Vector3;
   last_cam_rot: THREE.Euler;
   saoPass: SAOPass | undefined;
   outputPass: OutputPass | undefined;
+  renderOutputPass: OutputPass | undefined;
   bloomPass: UnrealBloomPass | undefined;
   smaaPass: SMAAPass | undefined;
   control: TransformControls | undefined;
@@ -87,7 +91,7 @@ class Editor {
   cameraViewControls: FreeCam | undefined;
   orbitControls: OrbitControls | undefined;
   locked: boolean;
-  frame_buffer: string[] = [];
+  frame_buffer: any[] = [];
   render_timer: number;
   fps_number: number;
   cap_fps: number;
@@ -124,6 +128,7 @@ class Editor {
   positive_prompt: string;
   negative_prompt: string;
   art_style: ArtStyle;
+  rawRenderPass: RenderPass | undefined;
 
   last_scrub: number;
   recorder: MediaRecorder | undefined;
@@ -143,7 +148,7 @@ class Editor {
   mouse_controls: MouseControls | undefined;
   save_manager: SaveManager;
 
-  generation_options: GenerationOptions;
+  generation_options: IGenerationOptions;
 
   // just a passive the image to be uploaded. we store the token and use that in snapshots.
   globalIpAdapterImage: File | undefined;
@@ -152,9 +157,20 @@ class Editor {
 
   sceneManager: SceneManager | undefined;
 
-  outliner_feature_flag: boolean;
+  ///////////////////////////////////////////////
+  ///////////////////////////////////////////////
+
+  public outliner_feature_flag: boolean;
+  public engine_preprocessing: boolean = false; // this is to do preprocessing also called render fast.
+
+  ///////////////////////////////////////////////
+  ///////////////////////////////////////////////
+
   focused: boolean = false;
 
+  customOutlinerPass: CustomOutlinePass | undefined;
+  surfaceFinder: FindSurfaces | undefined;
+
   constructor() {
     console.log(
       "If you see this message twice! then it rendered twice, if you see it once it's all good.",
@@ -175,7 +191,11 @@ class Editor {
     // global names
     this.camera_name = "::CAM::";
 
-    this.activeScene = new Scene("" + this.version, this.camera_name);
+    this.activeScene = new Scene(
+      "" + this.version,
+      this.camera_name,
+      this.updateSurfaceIdAttributeToMesh.bind(this),
+    );
     this.activeScene.initialize();
     this.generating_preview = false;
     this.last_cam_pos = new THREE.Vector3(0, 0, 0);
@@ -554,6 +574,8 @@ class Editor {
       }
     });
 
+    this._configurePostProcessingRaw();
+
     loadingBarData.value = {
       ...loadingBarData.value,
       progress: 100,
@@ -706,6 +728,77 @@ class Editor {
     loadingBarIsShowing.value = false;
   }
 
+  _configurePostProcessingRaw() {
+    const width = this.canvasRenderCamReference?.width ?? 0;
+    const height = this.canvasRenderCamReference?.height ?? 0;
+    if (
+      this.rawRenderer == undefined ||
+      this.render_camera == undefined ||
+      this.renderer == undefined
+    ) {
+      return;
+    }
+
+    const depthTexture = new THREE.DepthTexture(width, height);
+    depthTexture.type = THREE.UnsignedShortType;
+
+    const renderTarget = new THREE.WebGLRenderTarget(
+      window.innerWidth,
+      window.innerHeight,
+      {
+        depthTexture: depthTexture,
+        depthBuffer: true,
+      },
+    );
+
+    this.customOutlinerPass = new CustomOutlinePass(
+      new THREE.Vector2(width, height),
+      this.activeScene.scene,
+      this.render_camera,
+    );
+    this.render_composer = new EffectComposer(this.rawRenderer, renderTarget);
+    this.surfaceFinder = new FindSurfaces();
+    this.rawRenderPass = new RenderPass(
+      this.activeScene.scene,
+      this.render_camera,
+    );
+    this.render_composer.addPass(this.rawRenderPass);
+    this.render_composer.addPass(this.customOutlinerPass);
+
+    this.renderOutputPass = new OutputPass();
+    this.render_composer.addPass(this.renderOutputPass);
+
+    this.setColorMap();
+  }
+
+  setRenderDepth() {
+    this.updateSurfaceIdAttributeToMesh(this.activeScene.scene);
+    if (this.render_camera && this.customOutlinerPass) {
+      this.customOutlinerPass.fsQuad.material.uniforms.debugVisualize.value = 3; // Depth
+    }
+  }
+
+  setNormalMap() {
+    this.updateSurfaceIdAttributeToMesh(this.activeScene.scene);
+    if (this.render_camera && this.customOutlinerPass) {
+      this.customOutlinerPass.fsQuad.material.uniforms.debugVisualize.value = 4; // Normal Map
+    }
+  }
+
+  setColorMap() {
+    this.updateSurfaceIdAttributeToMesh(this.activeScene.scene);
+    if (this.render_camera && this.customOutlinerPass) {
+      this.customOutlinerPass.fsQuad.material.uniforms.debugVisualize.value = 2; // Renderd Color
+    }
+  }
+
+  setOutlineRender() {
+    this.updateSurfaceIdAttributeToMesh(this.activeScene.scene);
+    if (this.render_camera && this.customOutlinerPass) {
+      this.customOutlinerPass.fsQuad.material.uniforms.debugVisualize.value = 7; // Outlines Only
+    }
+  }
+
   // Configure post processing.
   _configurePostProcessing() {
     const width = this.canvReference?.width ?? 0;
@@ -717,6 +810,7 @@ class Editor {
 
     this.composer = new EffectComposer(this.renderer);
     this.renderPass = new RenderPass(this.activeScene.scene, this.camera);
+
     this.composer.addPass(this.renderPass);
 
     this.outlinePass = new OutlinePass(
@@ -777,18 +871,32 @@ class Editor {
     return this.activeScene.instantiate(name, pos);
   }
 
+  updateSurfaceIdAttributeToMesh(scene: THREE.Scene) {
+    if (this.surfaceFinder === undefined) {
+      return;
+    }
+    this.surfaceFinder.surfaceId = 0;
+    this.customOutlinerPass?.updateMaxSurfaceId(
+      this.surfaceFinder.surfaceId + 1,
+    );
+  }
+
   // Render the scene to the camera, this is called in the update.
   async renderScene() {
-    if (this.composer != null && !this.rendering && this.rawRenderer) {
+    if (
+      this.composer != null &&
+      !this.rendering &&
+      this.rawRenderer &&
+      this.render_composer
+    ) {
       this.composer.render();
-      this.rawRenderer.render(this.activeScene.scene, this.render_camera!);
+      //this.rawRenderer.render(this.activeScene.scene, this.render_camera!);
+      this.render_composer.render();
     } else if (this.renderer && this.render_camera && !this.rendering) {
       this.renderer.setSize(this.render_width, this.render_height);
       this.renderer.render(this.activeScene.scene, this.render_camera);
-    } else if (this.rendering && this.rawRenderer) {
-      this.rawRenderer.render(this.activeScene.scene, this.render_camera!);
-    } else {
-      console.error("Could not render to canvas no render or composer!");
+    } else if (this.rendering && this.renderer){
+      this.renderer.setSize(this.render_width, this.render_height);
     }
 
     if (this.rendering && this.rawRenderer && this.clock && this.renderer) {
@@ -815,9 +923,47 @@ class Editor {
       this.render_timer += this.clock.getDelta();
       this.frames += 1;
       this.playback_location++;
-      const imgData = this.rawRenderer.domElement.toDataURL("image/png", 1.0); // High quality png.
-      this.frame_buffer.push(imgData);
-      this.render_timer += this.clock.getDelta();
+
+      this.utils.removeTransformControls(true);
+
+      if (this.timeline.is_playing) {
+        this.setColorMap();
+        this.render_composer?.render();
+        const imgData = this.rawRenderer.domElement.toDataURL("image/png", 1.0); // High quality png.
+
+        if (this.engine_preprocessing) {
+          this.setNormalMap();
+          this.render_composer?.render();
+          const normalImgData = this.rawRenderer.domElement.toDataURL(
+            "image/png",
+            1.0,
+          ); // High quality png.
+
+          this.setRenderDepth();
+          this.render_composer?.render();
+          const depthImgData = this.rawRenderer.domElement.toDataURL(
+            "image/png",
+            1.0,
+          ); // High quality png.
+
+          this.setOutlineRender();
+          this.render_composer?.render();
+          const outlineImgData = this.rawRenderer.domElement.toDataURL(
+            "image/png",
+            1.0,
+          ); // High quality png.
+
+          this.frame_buffer.push([
+            imgData,
+            normalImgData,
+            depthImgData,
+            outlineImgData,
+          ]);
+        } else {
+          this.frame_buffer.push([imgData]);
+        }
+        this.render_timer += this.clock.getDelta();
+      }
       if (!this.timeline.is_playing) {
         //this.recorder.stop();
         this.playback_location = 0;
@@ -936,7 +1082,11 @@ class Editor {
   }
 
   switchEdit() {
-    if (this.switchPreviewToggle && this.canvasRenderCamReference) {
+    if (
+      this.switchPreviewToggle &&
+      this.canvasRenderCamReference &&
+      this.rawRenderPass
+    ) {
       this.switchPreviewToggle = false;
       editorState.value = EditorStates.EDIT;
       setTimeout(() => {
@@ -950,6 +1100,8 @@ class Editor {
           canvas: this.canvasRenderCamReference || undefined,
           preserveDrawingBuffer: true,
         });
+        this._configurePostProcessingRaw();
+
         if (this.camera_person_mode) {
           this.switchCameraView();
         }
@@ -1112,10 +1264,13 @@ class Editor {
       this.composer.setSize(width, height);
     }
 
+    this.render_composer?.setSize(width, height);
+
     if (this.render_camera == undefined) {
       return;
     }
 
+    this.customOutlinerPass?.setSize(width, height);
     this.render_camera.aspect = this.getRenderDimensions().aspectRatio;
     this.render_camera.updateProjectionMatrix();
   }

app/pages/PageEnigma/Editor/scene.ts

@@ -27,8 +27,13 @@ class Scene {
   // loading indicator manager
   placeholder_manager: LoadingPlaceHolderManager | undefined;
 
+  updateSurfaceIdAttributeToMesh: Function;
 
-  constructor(name: string, camera_name: string) {
+  constructor(
+    name: string,
+    camera_name: string,
+    updateSurfaceIdAttributeToMesh: Function,
+  ) {
     this.name = name;
     this.gridHelper;
     this.scene = new THREE.Scene();
@@ -41,6 +46,7 @@ class Scene {
     // global names
     this.camera_name = camera_name;
     this.placeholder_manager = undefined;
+    this.updateSurfaceIdAttributeToMesh = updateSurfaceIdAttributeToMesh;
   }
 
   initialize() {
@@ -62,10 +68,7 @@ class Scene {
     this._create_camera_obj();
   }
 
-  instantiate(
-    name: string,
-    pos: THREE.Vector3 = new THREE.Vector3(0, 0, 0),
-  ) {
+  instantiate(name: string, pos: THREE.Vector3 = new THREE.Vector3(0, 0, 0)) {
     const material = new THREE.MeshPhongMaterial({ color: 0xdacbce });
     material.shininess = 0.0;
     let geometry;
@@ -111,8 +114,7 @@ class Scene {
       obj.userData["base"] = new THREE.Color(0x00d8ff).getHex();
       this.shader_objects.push(obj);
       obj.userData["media_id"] = "Parim";
-    } 
-    else if (name.includes("Image::")) {
+    } else if (name.includes("Image::")) {
       const image_token = name.replace("Image::", "");
       const loader = new THREE.TextureLoader();
       const texture = loader.load(image_token);
@@ -120,7 +122,7 @@ class Scene {
       const image_material = new THREE.MeshBasicMaterial({
         color: 0xffffff,
         map: texture,
-        transparent: true
+        transparent: true,
       });
       obj = new THREE.Mesh(geometry, image_material);
       obj.userData["media_id"] = name;
@@ -130,8 +132,7 @@ class Scene {
       });
       obj = new THREE.Mesh(geometry, light_material);
       obj.userData["media_id"] = "Parim";
-    } 
-    else {
+    } else {
       obj = new THREE.Mesh(geometry, material);
       obj.userData["media_id"] = "Parim";
     }
@@ -149,7 +150,7 @@ class Scene {
     this.scene.add(obj);
 
     if (name == "PointLight") {
-      const light = new THREE.PointLight( 0xffffff, 1, 100 );
+      const light = new THREE.PointLight(0xffffff, 1, 100);
       this.scene.add(light);
       light.position.copy(obj.position);
       obj.attach(light);
@@ -157,6 +158,7 @@ class Scene {
       obj.userData["light"] = light.uuid;
     }
 
+    this.updateSurfaceIdAttributeToMesh(this.scene);
     return obj;
   }
 
@@ -364,7 +366,7 @@ class Scene {
               c.material.color.set(new THREE.Color(hex_color));
             }
 
-            if(c.name == "PointLight") {
+            if (c.name == "PointLight") {
               const light = this.get_object_by_uuid(c.userData["light"]);
               if (light) {
                 (light as THREE.PointLight).color = new THREE.Color(hex_color);
@@ -432,6 +434,7 @@ class Scene {
     mmd.layers.enable(0);
     mmd.layers.enable(1);
     this.scene.add(mmd);
+    this.updateSurfaceIdAttributeToMesh(this.scene);
     return mmd;
   }
 
@@ -450,14 +453,13 @@ class Scene {
         auto_add,
         position,
       );
-    }
-    else if (
+    } else if (
       url.includes(".png") ||
       url.includes(".jpg") ||
       url.includes(".jpeg") ||
       url.includes(".gif")
     ) {
-      return this.instantiate("Image::"+url);
+      return this.instantiate("Image::" + url);
     }
     return await this.loadGlbWithPlaceholder(
       media_id,
@@ -587,7 +589,7 @@ class Scene {
     if (auto_add) {
       this.scene.add(child_result);
     }
-
+    this.updateSurfaceIdAttributeToMesh(this.scene);
     return child_result;
   }
 

app/pages/PageEnigma/Editor/timeline.ts

@@ -24,7 +24,7 @@ import {
 import { Keyframe, MediaItem, UpdateTime } from "~/pages/PageEnigma/models";
 import Editor from "~/pages/PageEnigma/Editor/editor";
 import EmotionEngine from "./emotion_engine";
-import { GenerationOptions } from "~/pages/PageEnigma/models/generationOptions";
+import { IGenerationOptions } from "~/pages/PageEnigma/models/generationOptions";
 import { Vector3 } from "three";
 
 import { outlinerState } from "../signals";
@@ -156,7 +156,11 @@ export class TimeLine {
       case toEngineActions.ADD_CHARACTER: {
         const newObject = await this.addCharacter(data.data as MediaItem);
         if (newObject)
-          this.queueNewObjectMessage(newObject, data.data as MediaItem, AssetType.CHARACTER);
+          this.queueNewObjectMessage(
+            newObject,
+            data.data as MediaItem,
+            AssetType.CHARACTER,
+          );
         const result = this.editorEngine.sceneManager?.render_outliner(
           this.characters,
         );
@@ -166,7 +170,11 @@ export class TimeLine {
       case toEngineActions.ADD_OBJECT: {
         const newObject = await this.addObject(data.data as MediaItem);
         if (newObject)
-          this.queueNewObjectMessage(newObject, data.data as MediaItem, AssetType.OBJECT);
+          this.queueNewObjectMessage(
+            newObject,
+            data.data as MediaItem,
+            AssetType.OBJECT,
+          );
         const result = this.editorEngine.sceneManager?.render_outliner(
           this.characters,
         );
@@ -176,7 +184,11 @@ export class TimeLine {
       }
       case toEngineActions.ADD_SHAPE: {
         const newShape = await this.addShape(data.data as MediaItem);
-        this.queueNewObjectMessage(newShape, data.data as MediaItem, AssetType.SHAPE);
+        this.queueNewObjectMessage(
+          newShape,
+          data.data as MediaItem,
+          AssetType.SHAPE,
+        );
         const result = this.editorEngine.sceneManager?.render_outliner(
           this.characters,
         );
@@ -200,7 +212,7 @@ export class TimeLine {
       case toEngineActions.GENERATE_VIDEO: {
         const options = data.data; // super overloaded talk to the devs about this. TODO... refactor
         // pass this in ... rather than doing it implicitly ...
-        this.editorEngine.generation_options = options as GenerationOptions;
+        this.editorEngine.generation_options = options as IGenerationOptions;
         this.editorEngine.generateVideo();
         break;
       }
@@ -266,7 +278,7 @@ export class TimeLine {
   queueNewObjectMessage(
     item: THREE.Object3D<THREE.Object3DEventMap>,
     data: MediaItem,
-    asset_type: AssetType.OBJECT | AssetType.CHARACTER | AssetType.SHAPE
+    asset_type: AssetType.OBJECT | AssetType.CHARACTER | AssetType.SHAPE,
   ) {
     Queue.publish({
       queueName: QueueNames.FROM_ENGINE,

app/pages/PageEnigma/Editor/video_generation.ts

@@ -26,6 +26,10 @@ export class VideoGeneration {
     this.mediaUploadAPI = new MediaUploadApi();
   }
 
+  sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
   async generateFrame() {
     if (!this.editor.generating_preview && this.editor.rawRenderer) {
       this.editor.generating_preview = true;
@@ -189,190 +193,237 @@ export class VideoGeneration {
     //const videoURL = URL.createObjectURL(videoBlob);
 
     this.editor.generating_preview = true;
-    const ffmpeg = createFFmpeg({ log: false });
-    await ffmpeg.load();
 
-    this.editor.updateLoad(50, "Processing...");
+    this.editor.updateLoad({
+      progress: 50,
+      message: "Processing...",
+    });
 
-    // Write the Uint8Array to the FFmpeg file system
-    //ffmpeg.FS("writeFile", "input.webm", await fetchFile(videoURL));
-
-    for (let index = 0; index < this.editor.frame_buffer.length; index++) {
-      const element = this.editor.frame_buffer[index];
-      await ffmpeg.FS(
-        "writeFile",
-        `image${index}.png`,
-        await fetchFile(element),
-      );
-    }
-
-    await ffmpeg.run(
-      "-framerate",
-      "" + this.editor.cap_fps,
-      "-i",
-      "image%d.png",
-      "input.mp4",
-    );
-
-    await ffmpeg.run(
-      "-i",
-      "input.mp4",
-      "-f",
-      "lavfi",
-      "-i",
-      "anullsrc", // This adds a silent audio track
-      "-max_muxing_queue_size",
-      "999999",
-      "-vf", // WE REMOVED THE BLACK FRAME IN FFMEPG INSTEAD OF THE ENGINE
-      "select=gte(n\\,1)", // WE REMOVED THE BLACK FRAME IN FFMEPG INSTEAD OF THE ENGINE
-      // "select=gte(n\\,1),scale=1024:576",
-      "-c:v",
-      "libx264", // Specify video codec (optional, but recommended for MP4)
-      "-c:a",
-      "aac", // Specify audio codec (optional, but recommended for MP4)
-      "-shortest", // Ensure output duration matches the shortest stream (video or audio)
-      "-pix_fmt",
-      "yuv420p",
-      "-f",
-      "mp4",
-      "0tmp.mp4",
-    );
-
-    let itteration = 0;
-
-    if (compile_audio) {
-      for (const clip of this.editor.timeline.timeline_items) {
-        if (clip.type == ClipType.AUDIO) {
-          await this.convertAudioClip(itteration, ffmpeg, clip);
-          itteration += 1;
-        }
-      }
-    }
-
-    const output = ffmpeg.FS("readFile", itteration + "tmp.mp4");
-
-    ffmpeg.exit();
-    this.editor.generating_preview = false;
-
-    // Create a Blob from the output file for downloading
-    const blob = new Blob([output.buffer], { type: "video/mp4" });
-
-    const title = getSceneSignals().title || "Untitled";
-
-    const style_name = this.editor.art_style.toString();
-    const media_token = this.editor.current_scene_media_token || undefined;
-
-    // convert the ip adapter image and upload as a media token
-    const image_uuid = uuidv4();
-    let ipa_image_token = undefined;
-
-    if (this.editor.globalIpAdapterImage != undefined) {
-      const response = await this.mediaUploadAPI.UploadImage({
-        fileName: `${image_uuid}.ipa`,
-        blob: this.editor.globalIpAdapterImage,
-        uuid: image_uuid,
-      });
-      if (response.success) {
-        if (response.data) {
-          ipa_image_token = response.data;
-        }
-      }
-    }
-    if (ipa_image_token) {
-      globalIPAMediaToken.value = ipa_image_token;
-    }
-
-    // TODO Remove so many of these around wtf. SceneGenereationMetaData should only be one place
-    const metaData: SceneGenereationMetaData = {
-      artisticStyle: this.editor.art_style,
-      positivePrompt: this.editor.positive_prompt,
-      negativePrompt: this.editor.negative_prompt,
-      cameraAspectRatio: this.editor.render_camera_aspect_ratio,
-      upscale: this.editor.generation_options.upscale,
-      faceDetail: this.editor.generation_options.faceDetail,
-      styleStrength: this.editor.generation_options.styleStrength,
-      lipSync: this.editor.generation_options.lipSync,
-      cinematic: this.editor.generation_options.cinematic,
-      globalIPAMediaToken: globalIPAMediaToken.value,
+    const upload_tokens = {
+      color: "",
+      normal: "",
+      depth: "",
+      outline: "",
     };
 
-    // This is to save the snapshot of the scene for remixing...
-    const uuid_snapshot = uuidv4();
+    // TODO fix and ensure that render fast, would actually render fast.
+    for (
+      let image_index = 0;
+      image_index < this.editor.frame_buffer[0].length;
+      image_index++
+    ) {
+      // Write the Uint8Array to the FFmpeg file system
+      //ffmpeg.FS("writeFile", "input.webm", await fetchFile(videoURL));
 
-    const saveData = await this.editor.save_manager.saveData({
-      sceneTitle: title,
-      sceneToken: media_token,
-      sceneGenerationMetadata: metaData,
-    });
+      const ffmpeg = createFFmpeg({ log: false });
+      await ffmpeg.load();
 
-    console.log("SAVE DATA:");
-    console.log(saveData);
-    const file = new File([saveData], `${title}.glb`, {
-      type: "application/json",
-    });
+      for (let index = 0; index < this.editor.frame_buffer.length; index++) {
+        const element = this.editor.frame_buffer[index][image_index];
+        await ffmpeg.FS(
+          "writeFile",
+          `image${index}.png`,
+          await fetchFile(element),
+        );
+      }
 
-    const response =
-      await this.editor.media_upload.UploadSceneSnapshotMediaFileForm({
-        maybe_title: title,
-        maybe_scene_source_media_file_token: media_token, // can be undefined or null
-        uuid: uuid_snapshot,
-        blob: file,
+      await ffmpeg.run(
+        "-framerate",
+        "" + this.editor.cap_fps,
+        "-i",
+        "image%d.png",
+        "input.mp4",
+      );
+
+      await ffmpeg.run(
+        "-i",
+        "input.mp4",
+        "-f",
+        "lavfi",
+        "-i",
+        "anullsrc", // This adds a silent audio track
+        "-max_muxing_queue_size",
+        "999999",
+        "-vf", // WE REMOVED THE BLACK FRAME IN FFMEPG INSTEAD OF THE ENGINE
+        "select=gte(n\\,1)", // WE REMOVED THE BLACK FRAME IN FFMEPG INSTEAD OF THE ENGINE
+        // "select=gte(n\\,1),scale=1024:576",
+        "-c:v",
+        "libx264", // Specify video codec (optional, but recommended for MP4)
+        "-c:a",
+        "aac", // Specify audio codec (optional, but recommended for MP4)
+        "-shortest", // Ensure output duration matches the shortest stream (video or audio)
+        "-pix_fmt",
+        "yuv420p",
+        "-f",
+        "mp4",
+        "0tmp.mp4",
+      );
+
+      let itteration = 0;
+
+      if (compile_audio) {
+        for (const clip of this.editor.timeline.timeline_items) {
+          if (clip.type == ClipType.AUDIO) {
+            await this.convertAudioClip(itteration, ffmpeg, clip);
+            itteration += 1;
+          }
+        }
+      }
+
+      const output = ffmpeg.FS("readFile", itteration + "tmp.mp4");
+
+      ffmpeg.exit();
+      this.editor.generating_preview = false;
+
+      // Create a Blob from the output file for downloading
+      const blob = new Blob([output.buffer], { type: "video/mp4" });
+
+      const title = getSceneSignals().title || "Untitled";
+
+      const style_name = this.editor.art_style.toString();
+      const media_token = this.editor.current_scene_media_token || undefined;
+
+      // convert the ip adapter image and upload as a media token
+      const image_uuid = uuidv4();
+      let ipa_image_token = undefined;
+
+      if (this.editor.globalIpAdapterImage != undefined) {
+        const response = await this.mediaUploadAPI.UploadImage({
+          fileName: `${image_uuid}.ipa`,
+          blob: this.editor.globalIpAdapterImage,
+          uuid: image_uuid,
+        });
+        if (response.success) {
+          if (response.data) {
+            ipa_image_token = response.data;
+          }
+        }
+      }
+      if (ipa_image_token) {
+        globalIPAMediaToken.value = ipa_image_token;
+      }
+
+      // TODO Remove so many of these around wtf. SceneGenereationMetaData should only be one place
+      const metaData: SceneGenereationMetaData = {
+        artisticStyle: this.editor.art_style,
+        positivePrompt: this.editor.positive_prompt,
+        negativePrompt: this.editor.negative_prompt,
+        cameraAspectRatio: this.editor.render_camera_aspect_ratio,
+        upscale: this.editor.generation_options.upscale,
+        faceDetail: this.editor.generation_options.faceDetail,
+        styleStrength: this.editor.generation_options.styleStrength,
+        lipSync: this.editor.generation_options.lipSync,
+        cinematic: this.editor.generation_options.cinematic,
+        globalIPAMediaToken: globalIPAMediaToken.value,
+      };
+
+      // This is to save the snapshot of the scene for remixing...
+      const uuid_snapshot = uuidv4();
+
+      const saveData = await this.editor.save_manager.saveData({
+        sceneTitle: title,
+        sceneToken: media_token,
+        sceneGenerationMetadata: metaData,
       });
 
-    let immutable_media_token = undefined;
-    if (response.success) {
-      if (response.data) {
-        immutable_media_token = response.data;
+      const file = new File([saveData], `${title}.glb`, {
+        type: "application/json",
+      });
+
+      const response =
+        await this.editor.media_upload.UploadSceneSnapshotMediaFileForm({
+          maybe_title: title,
+          maybe_scene_source_media_file_token: media_token, // can be undefined or null
+          uuid: uuid_snapshot,
+          blob: file,
+        });
+
+      let immutable_media_token = undefined;
+      if (response.success) {
+        if (response.data) {
+          immutable_media_token = response.data;
+        }
+      } else {
+        console.log("ERROR:");
+        console.log(response.errorMessage);
       }
-    } else {
-      console.log("ERROR:");
-      console.log(response.errorMessage);
+
+      console.log("Immutable Token:");
+      console.log(immutable_media_token);
+
+      /// TODO refactor this whole thing extract this out.
+      const data: any = await this.editor.api_manager.uploadMedia({
+        blob,
+        fileName: `${title}.mp4`,
+        title,
+        styleName: style_name,
+        maybe_scene_source_media_file_token: immutable_media_token,
+      });
+
+      if (data == null) {
+        return;
+      }
+      const upload_token = data["media_file_token"];
+
+      console.log(upload_token);
+
+      if (image_index === 0) {
+        upload_tokens.color = upload_token;
+      } else if (image_index === 1) {
+        upload_tokens.normal = upload_token;
+      } else if (image_index === 2) {
+        upload_tokens.depth = upload_token;
+      } else if (image_index === 3) {
+        upload_tokens.outline = upload_token;
+      }
+
+      await this.sleep(2000); // TODO: REMOVE THIS WHEN TOO MANY REQUESTS ERROR IS SOLVED.
+      console.log("Waiting for server to catch up...");
     }
 
-    console.log("Immutable Token:");
-    console.log(immutable_media_token);
+    this.editor.onWindowResize();
 
-    /// TODO refactor this whole thing extract this out.
-    const data: any = await this.editor.api_manager.uploadMedia({
-      blob,
-      fileName: `${title}.mp4`,
-      title,
-      styleName: style_name,
-      maybe_scene_source_media_file_token: immutable_media_token,
-    });
+    console.log(upload_tokens);
+    this.editor.setColorMap();
 
-    if (data == null) {
-      return;
-    }
-    const upload_token = data["media_file_token"];
+    console.log("https://storyteller.ai/media/" + upload_tokens.color);
+    console.log("https://storyteller.ai/media/" + upload_tokens.normal);
+    console.log("https://storyteller.ai/media/" + upload_tokens.depth);
+    console.log("https://storyteller.ai/media/" + upload_tokens.outline);
 
-    // create the scene token from here then snapshot it associate with the video generation.
-    // send immutable token with the video ... TODO:
     await this.editor.api_manager
-      .stylizeVideo(
-        upload_token,
-        this.editor.art_style,
-        this.editor.positive_prompt,
-        this.editor.negative_prompt,
-        Visibility.Public,
-        this.editor.generation_options.faceDetail,
-        this.editor.generation_options.upscale,
-        this.editor.generation_options.styleStrength,
-        this.editor.generation_options.lipSync,
-        this.editor.generation_options.cinematic,
-        globalIPAMediaToken.value,
-      )
+      .stylizeVideo({
+        media_token: upload_tokens.color,
+        style: this.editor.art_style,
+        positive_prompt: this.editor.positive_prompt,
+        negative_prompt: this.editor.negative_prompt,
+        visibility: Visibility.Public,
+        use_face_detailer: this.editor.generation_options.faceDetail,
+        use_upscaler: this.editor.generation_options.upscale,
+        use_strength: this.editor.generation_options.styleStrength,
+        use_lipsync: this.editor.generation_options.lipSync,
+        use_cinematic: this.editor.generation_options.cinematic,
+        use_global_ipa_media_token: globalIPAMediaToken.value,
+        input_depth_file: upload_tokens.depth,
+        input_normal_file: upload_tokens.normal,
+        input_outline_file: upload_tokens.outline,
+      })
       .catch((error) => {
-        // TODO handle stylize error.
         console.log(error);
-        this.editor.updateLoad(100, "Failed To Render");
+        this.editor.updateLoad({
+          progress: 100,
+          message: "Failed To Render",
+          label: "Error",
+        });
         this.editor.endLoading();
       });
-    // Not sure if this is needed will double check TODO: MC
-    //this.editor.generating_preview = false;
 
-    // {"success":true,"inference_job_token":"jinf_j3nbqbd15wqxb0xcks13qh3f3bz"}
-    this.editor.updateLoad(100, "Done Check Your Media Tab On Profile.");
+    this.editor.updateLoad({
+      progress: 100,
+      message: "Done Check Your Media Tab On Profile.",
+      label: "Success",
+    });
+
     this.editor.endLoading();
 
     this.editor.recorder = undefined;
@@ -390,6 +441,8 @@ export class VideoGeneration {
       canvas: this.editor.canvasRenderCamReference || undefined,
       preserveDrawingBuffer: true,
     });
+    this.editor._configurePostProcessingRaw();
+
 
     this.editor.activeScene.renderMode(false);
 

app/pages/PageEnigma/Queue/Queue.ts

@@ -11,7 +11,7 @@ import {
 import { toTimelineActions } from "./toTimelineActions";
 
 import { ClipUI } from "../datastructures/clips/clip_ui";
-import { GenerationOptions } from "../models/generationOptions";
+import { IGenerationOptions } from "../models/generationOptions";
 import { ToastDataType } from "~/components";
 
 export type UnionedActionTypes =
@@ -26,7 +26,7 @@ export type UnionedDataTypes =
   | ClipUI[]
   | MediaItem
   | ToastDataType
-  | GenerationOptions
+  | IGenerationOptions
   | CameraAspectRatio
   | null;
 

app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/GenerateMovieButton.tsx

@@ -12,6 +12,7 @@ import {
   lipSync,
   cinematic,
   globalIPAMediaToken,
+  enginePreProcessing,
 } from "~/pages/PageEnigma/signals/stylizeTab";
 import { useEffect, useRef } from "react";
 
@@ -59,6 +60,7 @@ export function GenerateMovieButton({
       lipSync.value,
       cinematic.value,
       globalIPAMediaToken.value,
+      enginePreProcessing.value,
     );
     Queue.publish({
       queueName: QueueNames.TO_ENGINE,

app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/StyleButtons.tsx

@@ -10,8 +10,6 @@ import Queue from "~/pages/PageEnigma/Queue/Queue";
 import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
 import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
 import { Switch } from "@headlessui/react";
-// import Slider from "rc-slider";
-// import "rc-slider/assets/index.css";
 
 import {
   faceDetail,
@@ -19,11 +17,15 @@ import {
   upscale,
   lipSync,
   cinematic,
+  enginePreProcessing,
 } from "~/pages/PageEnigma/signals/stylizeTab";
 import { twMerge } from "tailwind-merge";
 
+import { useContext } from "react";
+import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
 export function StyleButtons() {
   useSignals();
+  const editorEngine = useContext(EngineContext);
 
   const sliderChanged = (value: number | number[]) => {
     styleStrength.value = (value as number) / 100;
@@ -71,6 +73,13 @@ export function StyleButtons() {
     }
   };
 
+  const enginePreProcessingChange = () => {
+    enginePreProcessing.value = !enginePreProcessing.value;
+    if (editorEngine) {
+      editorEngine.engine_preprocessing = enginePreProcessing.value;
+    }
+  };
+
   const handleUpscaleChange = () => {
     upscale.value = !upscale.value;
     if (upscale.value) {
@@ -249,6 +258,36 @@ export function StyleButtons() {
               </div>
               <hr className="opacity-[5%]" />
             </div>
+            <hr className="opacity-[5%]" />
+            <div className="flex w-full items-center">
+              <Switch.Group>
+                <Switch.Label
+                  className={twMerge(
+                    "mr-3 grow text-sm font-medium transition-opacity",
+                  )}
+                >
+                  Engine PreProcessing
+                </Switch.Label>
+                <Switch
+                  checked={enginePreProcessing.value}
+                  onChange={enginePreProcessingChange}
+                  className={twMerge(
+                    enginePreProcessing.value
+                      ? "bg-brand-primary hover:bg-brand-primary-400"
+                      : "bg-gray-500 hover:bg-gray-400",
+                    "focus:ring-indigo-500 relative inline-flex h-6 w-11 items-center rounded-full transition-colors focus:outline-none focus:ring-0 focus:ring-offset-0",
+                  )}
+                >
+                  <span
+                    className={`${
+                      enginePreProcessing.value
+                        ? "translate-x-6"
+                        : "translate-x-1"
+                    } inline-block h-4 w-4 transform rounded-full bg-white transition-transform`}
+                  />
+                </Switch>
+              </Switch.Group>
+            </div>
           </PremiumLock>
         </div>
 

app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/StylizeTab.tsx

@@ -21,6 +21,7 @@ import {
 
 export function StylizeTab() {
   useSignals();
+
   const [view, setView] = useState(StylizeTabPages.MAIN);
   const [generateSectionHeight, setGenerateSectionHeight] = useState(114);
 

app/pages/PageEnigma/models/generationOptions.ts

@@ -1,19 +1,21 @@
-export interface GenerationOptions {
+export interface IGenerationOptions {
   upscale: boolean;
   faceDetail: boolean;
   styleStrength: number;
   lipSync: boolean;
   cinematic: boolean;
   globalIpAdapterImageMediaToken: string | null;
+  enginePreProcessing: boolean;
 }
 
-export class GenerationOptions {
+export class GenerationOptions implements IGenerationOptions {
   public upscale: boolean;
   public faceDetail: boolean;
   public styleStrength: number;
   public lipSync: boolean;
   public cinematic: boolean;
   public globalIpAdapterImageMediaToken: string | null;
+  public enginePreProcessing: boolean;
   constructor(
     upscale: boolean,
     faceDetail: boolean,
@@ -21,6 +23,7 @@ export class GenerationOptions {
     lipSync: boolean,
     cinematic: boolean,
     globalIpAdapterImageMediaToken: string | null,
+    enginePreprocessing: boolean,
   ) {
     this.upscale = upscale;
     this.faceDetail = faceDetail;
@@ -28,5 +31,6 @@ export class GenerationOptions {
     this.lipSync = lipSync;
     this.cinematic = cinematic;
     this.globalIpAdapterImageMediaToken = globalIpAdapterImageMediaToken;
+    this.enginePreProcessing = enginePreprocessing;
   }
 }

app/pages/PageEnigma/models/sceneGenerationMetadata.ts

@@ -14,4 +14,6 @@ export interface SceneGenereationMetaData {
   styleStrength: number;
   lipSync: boolean;
   cinematic: boolean;
+
+  enginePreProcessing: boolean;
 }

app/pages/PageEnigma/signals/stylizeTab.ts

@@ -19,6 +19,7 @@ export const faceDetail = signal(false);
 export const styleStrength = signal(1.0);
 export const lipSync = signal(false);
 export const cinematic = signal(true);
+export const enginePreProcessing = signal(false);
 
 export const setArtStyleSelection = (newStyle: ArtStyle) => {
   if (selectedArtStyle.value !== newStyle) {
@@ -37,6 +38,7 @@ export const resetSceneGenerationMetadata = () => {
   styleStrength.value = 1.0;
   lipSync.value = false;
   cinematic.value = false;
+  enginePreProcessing.value = false;
 };
 
 export const restoreSceneGenerationMetadata = (
@@ -72,4 +74,7 @@ export const restoreSceneGenerationMetadata = (
   if (newData.cinematic) {
     cinematic.value = newData.cinematic;
   }
+  if (newData.enginePreProcessing) {
+    enginePreProcessing.value = newData.enginePreProcessing;
+  }
 };

proxy/src/upload-video/uploadMediaFile.ts

@@ -13,7 +13,7 @@ declare const process: {
     BASE_API: string;
   };
 };
-
+// TODO REPLACE THIS
 export async function uploadMediaFile({
   blob,
   fileName,

proxy/src/upload-video/uploadVideo.ts

@@ -136,7 +136,7 @@ async function stopPlaybackAndUploadVideo(
   const canvas = new Canvas(1024, 576);
   const context = canvas.getContext("2d");
   const folder = uuid.v4();
-
+  // use then...
   await new Promise<void>((resolve) => {
     fs.mkdir(`./${folder}`, () => {
       fs.mkdir(`./${folder}/images`, () => {
@@ -256,6 +256,7 @@ async function stopPlaybackAndUploadVideo(
       const media_token = video.current_scene_media_token || undefined;
 
       console.log("Upload file");
+      // TODO REPLACE
       const uploadData: any = await uploadMediaFile({
         blob,
         fileName: `${title}.mp4`,