From 0838cedd29cd40bc90e10defbeb07a745a9b0acd Mon Sep 17 00:00:00 2001 From: ArEnSc <6252325+ArEnSc@users.noreply.github.com> Date: Fri, 12 Jul 2024 12:11:33 -0400 Subject: [PATCH] Feature/Depth options (#609) Co-authored-by: Quicksticks-oss Co-authored-by: bflatastic --- app/api/index.tsx | 31 -- .../PageEnigma/Editor/CustomOutlinePass.js | 278 +++++++++++++ app/pages/PageEnigma/Editor/FindSurfaces.js | 193 +++++++++ app/pages/PageEnigma/Editor/api_manager.ts | 50 ++- app/pages/PageEnigma/Editor/editor.ts | 185 ++++++++- app/pages/PageEnigma/Editor/scene.ts | 34 +- app/pages/PageEnigma/Editor/timeline.ts | 24 +- .../PageEnigma/Editor/video_generation.ts | 379 ++++++++++-------- app/pages/PageEnigma/Queue/Queue.ts | 4 +- .../StylizeTab/GenerateMovieButton.tsx | 2 + .../tabComps/StylizeTab/StyleButtons.tsx | 43 +- .../tabComps/StylizeTab/StylizeTab.tsx | 1 + .../PageEnigma/models/generationOptions.ts | 8 +- .../models/sceneGenerationMetadata.ts | 2 + app/pages/PageEnigma/signals/stylizeTab.ts | 5 + proxy/src/upload-video/uploadMediaFile.ts | 2 +- proxy/src/upload-video/uploadVideo.ts | 3 +- 17 files changed, 992 insertions(+), 252 deletions(-) create mode 100644 app/pages/PageEnigma/Editor/CustomOutlinePass.js create mode 100644 app/pages/PageEnigma/Editor/FindSurfaces.js diff --git a/app/api/index.tsx b/app/api/index.tsx index 7c7f522ec1..9dd8e927a1 100644 --- a/app/api/index.tsx +++ b/app/api/index.tsx @@ -1,34 +1,5 @@ export const apiHost = "https://api.storyteller.ai"; -// Authentications -// const createAccount = `${apiHost}}/create_account`; -// export const login = `${apiHost}/v1/login`; -// export const getSession = `${apiHost}/v1/session`; -// export const logout = `${apiHost}/v1/logout`; - -// Inference Jobs -// export const getRecentJobs = `${apiHost}/v1/jobs/session`; -// export const getActiveJobs = `${apiHost}/v1/jobs/session?exclude_states=complete_success,complete_failure,dead,cancelled_by_user,cancelled_by_system`; -// export const getJobStatus = (jobToken: string) => -// `${apiHost}/v1/model_inference/job_status/${jobToken}`; - -// TTS -// export const inferTts = `${apiHost}/tts/inference`; -// export const listTts = `${apiHost}/tts/list`; - -//V2V -// export const listV2V = `${apiHost}/v1/voice_conversion/model_list`; -// export const inferV2V = `${apiHost}/v1/voice_conversion/inference`; -// export const uploadAudioV2V = `${apiHost}/v1/media_uploads/upload_audio`; - -// User Media -// export const listMediaByUser = (username: string) => -// `${apiHost}/v1/media_files/list/user/${username}`; - -// Get Any Media -// export const getMediaFileByToken = (fileToken: string) => -// `${apiHost}/v1/media_files/file/${fileToken}`; - // Upload File export const uploadMedia = `${apiHost}/v1/media_files/upload`; @@ -37,5 +8,3 @@ export const uploadNewScene = `${apiHost}/v1/media_files/upload/new_scene`; export const uploadThumbnail = `${apiHost}/v1/media_files/cover_image/`; export const updateExistingScene = (sceneToken: string) => `${apiHost}/v1/media_files/upload/saved_scene/${sceneToken}`; -// export const renameScene = (sceneToken: string) => -// `${apiHost}/v1/media_files/rename/${sceneToken}`; diff --git a/app/pages/PageEnigma/Editor/CustomOutlinePass.js b/app/pages/PageEnigma/Editor/CustomOutlinePass.js new file mode 100644 index 0000000000..ee7b559f03 --- /dev/null +++ b/app/pages/PageEnigma/Editor/CustomOutlinePass.js @@ -0,0 +1,278 @@ +import * as THREE from "three"; +import { Pass } from "three/examples/jsm/postprocessing/Pass.js"; +import { FullScreenQuad } from "three/examples/jsm/postprocessing/Pass.js"; +import { + getSurfaceIdMaterial, + getDebugSurfaceIdMaterial, +} from "./FindSurfaces.js"; + +// Follows the structure of +// https://github.com/mrdoob/three.js/blob/master/examples/jsm/postprocessing/OutlinePass.js +class CustomOutlinePass extends Pass { + constructor(resolution, scene, camera) { + super(); + + this.renderScene = scene; + this.renderCamera = camera; + this.resolution = new THREE.Vector2(resolution.x, resolution.y); + + this.fsQuad = new FullScreenQuad(null); + this.fsQuad.material = this.createOutlinePostProcessMaterial(); + + // Create a buffer to store the normals of the scene onto + // or store the "surface IDs" + const surfaceBuffer = new THREE.WebGLRenderTarget( + this.resolution.x, + this.resolution.y, + ); + surfaceBuffer.texture.format = THREE.RGBAFormat; + surfaceBuffer.texture.type = THREE.HalfFloatType; + surfaceBuffer.texture.minFilter = THREE.NearestFilter; + surfaceBuffer.texture.magFilter = THREE.NearestFilter; + surfaceBuffer.texture.generateMipmaps = false; + surfaceBuffer.stencilBuffer = false; + this.surfaceBuffer = surfaceBuffer; + + this.normalOverrideMaterial = new THREE.MeshNormalMaterial(); + this.surfaceIdOverrideMaterial = getSurfaceIdMaterial(); + this.surfaceIdDebugOverrideMaterial = getDebugSurfaceIdMaterial(); + this.save_ = false; + } + + dispose() { + this.surfaceBuffer.dispose(); + this.fsQuad.dispose(); + } + + updateMaxSurfaceId(maxSurfaceId) { + this.surfaceIdOverrideMaterial.uniforms.maxSurfaceId.value = maxSurfaceId; + } + + setSize(width, height) { + this.surfaceBuffer.setSize(width, height); + this.resolution.set(width, height); + + this.fsQuad.material.uniforms.screenSize.value.set( + this.resolution.x, + this.resolution.y, + 1 / this.resolution.x, + 1 / this.resolution.y, + ); + } + + getDebugVisualizeValue() { + return this.fsQuad.material.uniforms.debugVisualize.value; + } + + isUsingSurfaceIds() { + const debugVisualize = this.getDebugVisualizeValue(); + + return ( + debugVisualize == 0 || // Main outlines v2 mode + debugVisualize == 5 || // Render just surfaceID debug buffer + debugVisualize == 6 + ); // Render just outlines with surfaceId + } + + save() { + this.save_ = true; + } + + render(renderer, writeBuffer, readBuffer) { + const depthBufferValue = writeBuffer.depthBuffer; + writeBuffer.depthBuffer = false; + + renderer.setRenderTarget(this.surfaceBuffer); + const overrideMaterialValue = this.renderScene.overrideMaterial; + this.renderScene.overrideMaterial = this.normalOverrideMaterial; + + renderer.render(this.renderScene, this.renderCamera); + this.renderScene.overrideMaterial = overrideMaterialValue; + + this.fsQuad.material.uniforms["depthBuffer"].value = + readBuffer.depthTexture; + this.fsQuad.material.uniforms["surfaceBuffer"].value = + this.surfaceBuffer.texture; + this.fsQuad.material.uniforms["sceneColorBuffer"].value = + readBuffer.texture; + + renderer.setRenderTarget(writeBuffer); + this.fsQuad.render(renderer); + + // Reset the depthBuffer value so we continue writing to it in the next render. + writeBuffer.depthBuffer = depthBufferValue; + } + + get vertexShader() { + return ` + varying vec2 vUv; + void main() { + vUv = uv; + gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); + } + `; + } + get fragmentShader() { + return ` + #include + // The above include imports "perspectiveDepthToViewZ" + // and other GLSL functions from ThreeJS we need for reading depth. + uniform sampler2D sceneColorBuffer; + uniform sampler2D depthBuffer; + uniform sampler2D surfaceBuffer; + uniform float cameraNear; + uniform float cameraFar; + uniform vec4 screenSize; + uniform vec3 outlineColor; + uniform vec4 multiplierParameters; + uniform int debugVisualize; + + varying vec2 vUv; + + // Helper functions for reading from depth buffer. + float readDepth (sampler2D depthSampler, vec2 coord) { + float fragCoordZ = texture2D(depthSampler, coord).x; + float viewZ = perspectiveDepthToViewZ( fragCoordZ, cameraNear, cameraFar ); + return viewZToOrthographicDepth( viewZ, cameraNear, cameraFar ); + } + float getLinearDepth(vec3 pos) { + return -(viewMatrix * vec4(pos, 1.0)).z; + } + + float getLinearScreenDepth(sampler2D map) { + vec2 uv = gl_FragCoord.xy * screenSize.zw; + return readDepth(map,uv); + } + + // Helper functions for reading normals and depth of neighboring pixels. + float getPixelDepth(int x, int y) { + // screenSize.zw is pixel size + // vUv is current position + return readDepth(depthBuffer, vUv + screenSize.zw * vec2(x, y)); + } + // "surface value" is either the normal or the "surfaceID" + vec3 getSurfaceValue(int x, int y) { + vec3 val = texture2D(surfaceBuffer, vUv + screenSize.zw * vec2(x, y)).rgb; + return val; + } + + float saturateValue(float num) { + return clamp(num, 0.0, 1.0); + } + + float getSufaceIdDiff(vec3 surfaceValue) { + float surfaceIdDiff = 0.0; + surfaceIdDiff += distance(surfaceValue, getSurfaceValue(1, 0)); + surfaceIdDiff += distance(surfaceValue, getSurfaceValue(0, 1)); + surfaceIdDiff += distance(surfaceValue, getSurfaceValue(0, 1)); + surfaceIdDiff += distance(surfaceValue, getSurfaceValue(0, -1)); + + surfaceIdDiff += distance(surfaceValue, getSurfaceValue(1, 1)); + surfaceIdDiff += distance(surfaceValue, getSurfaceValue(1, -1)); + surfaceIdDiff += distance(surfaceValue, getSurfaceValue(-1, 1)); + surfaceIdDiff += distance(surfaceValue, getSurfaceValue(-1, -1)); + return surfaceIdDiff; + } + + void main() { + vec4 sceneColor = texture2D(sceneColorBuffer, vUv); + float depth = getPixelDepth(0, 0); + + // "surfaceValue" is either the normal or the surfaceId + vec3 surfaceValue = getSurfaceValue(0, 0); + + // Get the difference between depth of neighboring pixels and current. + float depthDiff = 0.0; + depthDiff += abs(depth - getPixelDepth(1, 0)); + depthDiff += abs(depth - getPixelDepth(-1, 0)); + depthDiff += abs(depth - getPixelDepth(0, 1)); + depthDiff += abs(depth - getPixelDepth(0, -1)); + + // Get the difference between surface values of neighboring pixels + // and current + float surfaceValueDiff = getSufaceIdDiff(surfaceValue); + + // Apply multiplier & bias to each + float depthBias = multiplierParameters.x; + float depthMultiplier = multiplierParameters.y; + float normalBias = multiplierParameters.z; + float normalMultiplier = multiplierParameters.w; + + depthDiff = depthDiff * depthMultiplier; + depthDiff = saturateValue(depthDiff); + depthDiff = pow(depthDiff, depthBias); + + if (debugVisualize != 0 && debugVisualize != 6) { + // Apply these params when using + // normals instead of surfaceIds + surfaceValueDiff = surfaceValueDiff * normalMultiplier; + surfaceValueDiff = saturateValue(surfaceValueDiff); + surfaceValueDiff = pow(surfaceValueDiff, normalBias); + } else { + if (surfaceValueDiff != 0.0) surfaceValueDiff = 1.0; + } + + float outline = saturateValue(surfaceValueDiff + depthDiff); + + // Combine outline with scene color. + vec4 outlineColor = vec4(outlineColor, 1.0); + gl_FragColor = vec4(mix(sceneColor, outlineColor, outline)); + + //// For debug visualization of the different inputs to this shader. + if (debugVisualize == 2) { + gl_FragColor = sceneColor; + } + if (debugVisualize == 3) { + gl_FragColor = vec4(vec3(depth), 1.0); + gl_FragColor.rgb = 1.0 - gl_FragColor.rgb; + } + if (debugVisualize == 4 || debugVisualize == 5) { + // 4 visualizes the normal buffer + // 5 visualizes the surfaceID buffer + // Either way they are the same buffer, we change + // what we render into it + gl_FragColor = vec4(surfaceValue, 1.0); + } + if (debugVisualize == 6 || debugVisualize == 7) { + // Apply threshold + if (outline > 0.75) { + outline = 1.0; + } else { + outline = 0.0; + } + gl_FragColor = vec4(vec3(outline * outlineColor), 1.0); + } + } + `; + } + + createOutlinePostProcessMaterial() { + return new THREE.ShaderMaterial({ + uniforms: { + debugVisualize: { value: 0 }, + sceneColorBuffer: {}, + depthBuffer: {}, + surfaceBuffer: {}, + outlineColor: { value: new THREE.Color(0xffffff) }, + //4 scalar values packed in one uniform: depth multiplier, depth bias, and same for normals. + multiplierParameters: { + value: new THREE.Vector4(0.9, 20, 1, 1), + }, + cameraNear: { value: 0.1 }, + cameraFar: { value: 3 }, + screenSize: { + value: new THREE.Vector4( + this.resolution.x, + this.resolution.y, + 1 / this.resolution.x, + 1 / this.resolution.y, + ), + }, + }, + vertexShader: this.vertexShader, + fragmentShader: this.fragmentShader, + }); + } +} + +export { CustomOutlinePass }; diff --git a/app/pages/PageEnigma/Editor/FindSurfaces.js b/app/pages/PageEnigma/Editor/FindSurfaces.js new file mode 100644 index 0000000000..c56426ae7a --- /dev/null +++ b/app/pages/PageEnigma/Editor/FindSurfaces.js @@ -0,0 +1,193 @@ +import * as THREE from "three"; + +/* + This class computes "surface IDs" for a given mesh. + + A "surface" is defined as a set of triangles that share vertices. + + Inspired by Ian MacLarty, see: + https://twitter.com/ianmaclarty/status/1499494878908403712 +*/ +class FindSurfaces { + constructor() { + // This identifier, must be globally unique for each surface + // across all geometry rendered on screen + this.surfaceId = 0; + } + + /* + * Returns the surface Ids as a Float32Array that can be inserted as a vertex attribute + */ + getSurfaceIdAttribute(mesh) { + const bufferGeometry = mesh.geometry; + const numVertices = bufferGeometry.attributes.position.count; + const vertexIdToSurfaceId = this._generateSurfaceIds(mesh); + + const colors = []; + for (let i = 0; i < numVertices; i++) { + const vertexId = i; + let surfaceId = vertexIdToSurfaceId[vertexId]; + + colors.push(surfaceId, 0, 0, 1); + } + + const colorsTypedArray = new Float32Array(colors); + return colorsTypedArray; + } + + /* + * Returns a `vertexIdToSurfaceId` map + * given a vertex, returns the surfaceId + */ + _generateSurfaceIds(mesh) { + const bufferGeometry = mesh.geometry; + //const numVertices = bufferGeometry.attributes.position.count; + const numIndices = bufferGeometry.index.count; + const indexBuffer = bufferGeometry.index.array; + //const vertexBuffer = bufferGeometry.attributes.position.array; + // For each vertex, search all its neighbors + const vertexMap = {}; + for (let i = 0; i < numIndices; i += 3) { + const i1 = indexBuffer[i + 0]; + const i2 = indexBuffer[i + 1]; + const i3 = indexBuffer[i + 2]; + + add(i1, i2); + add(i1, i3); + add(i2, i3); + } + function add(a, b) { + if (vertexMap[a] == undefined) vertexMap[a] = []; + if (vertexMap[b] == undefined) vertexMap[b] = []; + + if (vertexMap[a].indexOf(b) == -1) vertexMap[a].push(b); + if (vertexMap[b].indexOf(a) == -1) vertexMap[b].push(a); + } + + // Find cycles + const frontierNodes = Object.keys(vertexMap).map((v) => Number(v)); + const exploredNodes = {}; + const vertexIdToSurfaceId = {}; + + while (frontierNodes.length > 0) { + const node = frontierNodes.pop(); + if (exploredNodes[node]) continue; + + // Get all neighbors recursively + const surfaceVertices = getNeighborsNonRecursive(node); + // Mark them as explored + for (let v of surfaceVertices) { + exploredNodes[v] = true; + vertexIdToSurfaceId[v] = this.surfaceId; + } + + this.surfaceId += 1; + } + + // function getNeighbors(node, explored) { + // const neighbors = vertexMap[node]; + // let result = [node]; + // explored[node] = true; + + // for (let n of neighbors) { + // if (explored[n]) continue; + // explored[n] = true; + // const newNeighbors = getNeighbors(n, explored); + // result = result.concat(newNeighbors); + // } + + // return result; + // } + + function getNeighborsNonRecursive(node) { + const frontier = [node]; + const explored = {}; + const result = []; + + while (frontier.length > 0) { + const currentNode = frontier.pop(); + if (explored[currentNode]) continue; + const neighbors = vertexMap[currentNode]; + result.push(currentNode); + + explored[currentNode] = true; + + for (let n of neighbors) { + if (!explored[n]) { + frontier.push(n); + } + } + } + + return result; + } + + return vertexIdToSurfaceId; + } +} + +export default FindSurfaces; + +export function getSurfaceIdMaterial() { + return new THREE.ShaderMaterial({ + uniforms: { + maxSurfaceId: { value: 1 }, + }, + vertexShader: getVertexShader(), + fragmentShader: getFragmentShader(), + vertexColors: true, + }); +} + +function getVertexShader() { + return ` + varying vec2 v_uv; + varying vec4 vColor; + + void main() { + v_uv = uv; + vColor = color; + + gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); + } + `; +} + +function getFragmentShader() { + return ` + varying vec2 v_uv; + varying vec4 vColor; + uniform float maxSurfaceId; + + void main() { + // Normalize the surfaceId when writing to texture + // Surface ID needs rounding as precision can be lost in perspective correct interpolation + // - see https://github.com/OmarShehata/webgl-outlines/issues/9 for other solutions eg. flat interpolation. + float surfaceId = round(vColor.r) / maxSurfaceId; + gl_FragColor = vec4(surfaceId, 0.0, 0.0, 1.0); + } + `; +} + +// For debug rendering, assign a random color +// to each surfaceId +export function getDebugSurfaceIdMaterial() { + return new THREE.ShaderMaterial({ + uniforms: {}, + vertexShader: getVertexShader(), + fragmentShader: ` + varying vec2 v_uv; + varying vec4 vColor; + + void main() { + int surfaceId = int(round(vColor.r) * 100.0); + float R = float(surfaceId % 255) / 255.0; + float G = float((surfaceId + 50) % 255) / 255.0; + float B = float((surfaceId * 20) % 255) / 255.0; + + gl_FragColor = vec4(R, G, B, 1.0); + } + `, + vertexColors: true, + }); +} diff --git a/app/pages/PageEnigma/Editor/api_manager.ts b/app/pages/PageEnigma/Editor/api_manager.ts index 5b01219947..b07448f794 100644 --- a/app/pages/PageEnigma/Editor/api_manager.ts +++ b/app/pages/PageEnigma/Editor/api_manager.ts @@ -5,6 +5,24 @@ import { signalScene, startPollingActiveJobs } from "~/signals"; import { updateExistingScene, uploadNewScene } from "./api_fetchers"; import { uploadThumbnail } from "~/api"; import environmentVariables from "~/Classes/EnvironmentVariables"; + +interface StylizeVideoParams { + media_token: string; + style: ArtStyle; + positive_prompt: string; + negative_prompt: string; + visibility: Visibility; + use_face_detailer?: boolean; + use_upscaler?: boolean; + use_strength?: number; + use_lipsync?: boolean; + use_cinematic?: boolean; + use_global_ipa_media_token?: string | null; + input_depth_file?: string | null; + input_normal_file?: string | null; + input_outline_file?: string | null; +} + /** * Storyteller Studio API Manager * The source of truth of all these media items is the database in the cloud @@ -358,19 +376,22 @@ export class APIManager { return result; } - public async stylizeVideo( - media_token: string, - style: ArtStyle, - positive_prompt: string, - negative_prompt: string, - visibility: Visibility, - use_face_detailer: boolean = false, - use_upscaler: boolean = false, - use_strength: number = 1.0, - use_lipsync: boolean = false, - use_cinematic: boolean = false, - use_global_ipa_media_token: string | null = null, - ) { + public async stylizeVideo({ + media_token, + style, + positive_prompt, + negative_prompt, + visibility, + use_face_detailer = false, + use_upscaler = false, + use_strength = 1.0, + use_lipsync = false, + use_cinematic = false, + use_global_ipa_media_token = null, + input_depth_file = null, + input_normal_file = null, + input_outline_file = null, + }: StylizeVideoParams) { const uuid = uuidv4(); const data = { @@ -389,6 +410,9 @@ export class APIManager { use_lipsync: use_lipsync, global_ipa_media_token: use_global_ipa_media_token, use_cinematic: use_cinematic, + input_depth_file: input_depth_file, + input_normal_file: input_normal_file, + input_outline_file: input_outline_file, }; const json_data = JSON.stringify(data); diff --git a/app/pages/PageEnigma/Editor/editor.ts b/app/pages/PageEnigma/Editor/editor.ts index e66aae2feb..4fe874dddf 100644 --- a/app/pages/PageEnigma/Editor/editor.ts +++ b/app/pages/PageEnigma/Editor/editor.ts @@ -41,11 +41,13 @@ import { signalScene, } from "~/signals"; import { outlinerState, updateObjectPanel } from "../signals"; -import { GenerationOptions } from "../models/generationOptions"; +import { IGenerationOptions } from "../models/generationOptions"; import { toEngineActions } from "../Queue/toEngineActions"; import { SceneGenereationMetaData } from "../models/sceneGenerationMetadata"; import { MediaUploadApi } from "~/Classes/ApiManager"; import { SceneManager } from "./scene_manager_api"; +import { CustomOutlinePass } from "./CustomOutlinePass.js"; +import FindSurfaces from "./FindSurfaces.js"; export type EditorInitializeConfig = { sceneToken: string; @@ -68,11 +70,13 @@ class Editor { canvasRenderCamReference: HTMLCanvasElement | null = null; composer: EffectComposer | undefined; + render_composer: EffectComposer | undefined; outlinePass: OutlinePass | undefined; last_cam_pos: THREE.Vector3; last_cam_rot: THREE.Euler; saoPass: SAOPass | undefined; outputPass: OutputPass | undefined; + renderOutputPass: OutputPass | undefined; bloomPass: UnrealBloomPass | undefined; smaaPass: SMAAPass | undefined; control: TransformControls | undefined; @@ -87,7 +91,7 @@ class Editor { cameraViewControls: FreeCam | undefined; orbitControls: OrbitControls | undefined; locked: boolean; - frame_buffer: string[] = []; + frame_buffer: any[] = []; render_timer: number; fps_number: number; cap_fps: number; @@ -124,6 +128,7 @@ class Editor { positive_prompt: string; negative_prompt: string; art_style: ArtStyle; + rawRenderPass: RenderPass | undefined; last_scrub: number; recorder: MediaRecorder | undefined; @@ -143,7 +148,7 @@ class Editor { mouse_controls: MouseControls | undefined; save_manager: SaveManager; - generation_options: GenerationOptions; + generation_options: IGenerationOptions; // just a passive the image to be uploaded. we store the token and use that in snapshots. globalIpAdapterImage: File | undefined; @@ -152,9 +157,20 @@ class Editor { sceneManager: SceneManager | undefined; - outliner_feature_flag: boolean; + /////////////////////////////////////////////// + /////////////////////////////////////////////// + + public outliner_feature_flag: boolean; + public engine_preprocessing: boolean = false; // this is to do preprocessing also called render fast. + + /////////////////////////////////////////////// + /////////////////////////////////////////////// + focused: boolean = false; + customOutlinerPass: CustomOutlinePass | undefined; + surfaceFinder: FindSurfaces | undefined; + constructor() { console.log( "If you see this message twice! then it rendered twice, if you see it once it's all good.", @@ -175,7 +191,11 @@ class Editor { // global names this.camera_name = "::CAM::"; - this.activeScene = new Scene("" + this.version, this.camera_name); + this.activeScene = new Scene( + "" + this.version, + this.camera_name, + this.updateSurfaceIdAttributeToMesh.bind(this), + ); this.activeScene.initialize(); this.generating_preview = false; this.last_cam_pos = new THREE.Vector3(0, 0, 0); @@ -554,6 +574,8 @@ class Editor { } }); + this._configurePostProcessingRaw(); + loadingBarData.value = { ...loadingBarData.value, progress: 100, @@ -706,6 +728,77 @@ class Editor { loadingBarIsShowing.value = false; } + _configurePostProcessingRaw() { + const width = this.canvasRenderCamReference?.width ?? 0; + const height = this.canvasRenderCamReference?.height ?? 0; + if ( + this.rawRenderer == undefined || + this.render_camera == undefined || + this.renderer == undefined + ) { + return; + } + + const depthTexture = new THREE.DepthTexture(width, height); + depthTexture.type = THREE.UnsignedShortType; + + const renderTarget = new THREE.WebGLRenderTarget( + window.innerWidth, + window.innerHeight, + { + depthTexture: depthTexture, + depthBuffer: true, + }, + ); + + this.customOutlinerPass = new CustomOutlinePass( + new THREE.Vector2(width, height), + this.activeScene.scene, + this.render_camera, + ); + this.render_composer = new EffectComposer(this.rawRenderer, renderTarget); + this.surfaceFinder = new FindSurfaces(); + this.rawRenderPass = new RenderPass( + this.activeScene.scene, + this.render_camera, + ); + this.render_composer.addPass(this.rawRenderPass); + this.render_composer.addPass(this.customOutlinerPass); + + this.renderOutputPass = new OutputPass(); + this.render_composer.addPass(this.renderOutputPass); + + this.setColorMap(); + } + + setRenderDepth() { + this.updateSurfaceIdAttributeToMesh(this.activeScene.scene); + if (this.render_camera && this.customOutlinerPass) { + this.customOutlinerPass.fsQuad.material.uniforms.debugVisualize.value = 3; // Depth + } + } + + setNormalMap() { + this.updateSurfaceIdAttributeToMesh(this.activeScene.scene); + if (this.render_camera && this.customOutlinerPass) { + this.customOutlinerPass.fsQuad.material.uniforms.debugVisualize.value = 4; // Normal Map + } + } + + setColorMap() { + this.updateSurfaceIdAttributeToMesh(this.activeScene.scene); + if (this.render_camera && this.customOutlinerPass) { + this.customOutlinerPass.fsQuad.material.uniforms.debugVisualize.value = 2; // Renderd Color + } + } + + setOutlineRender() { + this.updateSurfaceIdAttributeToMesh(this.activeScene.scene); + if (this.render_camera && this.customOutlinerPass) { + this.customOutlinerPass.fsQuad.material.uniforms.debugVisualize.value = 7; // Outlines Only + } + } + // Configure post processing. _configurePostProcessing() { const width = this.canvReference?.width ?? 0; @@ -717,6 +810,7 @@ class Editor { this.composer = new EffectComposer(this.renderer); this.renderPass = new RenderPass(this.activeScene.scene, this.camera); + this.composer.addPass(this.renderPass); this.outlinePass = new OutlinePass( @@ -777,18 +871,32 @@ class Editor { return this.activeScene.instantiate(name, pos); } + updateSurfaceIdAttributeToMesh(scene: THREE.Scene) { + if (this.surfaceFinder === undefined) { + return; + } + this.surfaceFinder.surfaceId = 0; + this.customOutlinerPass?.updateMaxSurfaceId( + this.surfaceFinder.surfaceId + 1, + ); + } + // Render the scene to the camera, this is called in the update. async renderScene() { - if (this.composer != null && !this.rendering && this.rawRenderer) { + if ( + this.composer != null && + !this.rendering && + this.rawRenderer && + this.render_composer + ) { this.composer.render(); - this.rawRenderer.render(this.activeScene.scene, this.render_camera!); + //this.rawRenderer.render(this.activeScene.scene, this.render_camera!); + this.render_composer.render(); } else if (this.renderer && this.render_camera && !this.rendering) { this.renderer.setSize(this.render_width, this.render_height); this.renderer.render(this.activeScene.scene, this.render_camera); - } else if (this.rendering && this.rawRenderer) { - this.rawRenderer.render(this.activeScene.scene, this.render_camera!); - } else { - console.error("Could not render to canvas no render or composer!"); + } else if (this.rendering && this.renderer){ + this.renderer.setSize(this.render_width, this.render_height); } if (this.rendering && this.rawRenderer && this.clock && this.renderer) { @@ -815,9 +923,47 @@ class Editor { this.render_timer += this.clock.getDelta(); this.frames += 1; this.playback_location++; - const imgData = this.rawRenderer.domElement.toDataURL("image/png", 1.0); // High quality png. - this.frame_buffer.push(imgData); - this.render_timer += this.clock.getDelta(); + + this.utils.removeTransformControls(true); + + if (this.timeline.is_playing) { + this.setColorMap(); + this.render_composer?.render(); + const imgData = this.rawRenderer.domElement.toDataURL("image/png", 1.0); // High quality png. + + if (this.engine_preprocessing) { + this.setNormalMap(); + this.render_composer?.render(); + const normalImgData = this.rawRenderer.domElement.toDataURL( + "image/png", + 1.0, + ); // High quality png. + + this.setRenderDepth(); + this.render_composer?.render(); + const depthImgData = this.rawRenderer.domElement.toDataURL( + "image/png", + 1.0, + ); // High quality png. + + this.setOutlineRender(); + this.render_composer?.render(); + const outlineImgData = this.rawRenderer.domElement.toDataURL( + "image/png", + 1.0, + ); // High quality png. + + this.frame_buffer.push([ + imgData, + normalImgData, + depthImgData, + outlineImgData, + ]); + } else { + this.frame_buffer.push([imgData]); + } + this.render_timer += this.clock.getDelta(); + } if (!this.timeline.is_playing) { //this.recorder.stop(); this.playback_location = 0; @@ -936,7 +1082,11 @@ class Editor { } switchEdit() { - if (this.switchPreviewToggle && this.canvasRenderCamReference) { + if ( + this.switchPreviewToggle && + this.canvasRenderCamReference && + this.rawRenderPass + ) { this.switchPreviewToggle = false; editorState.value = EditorStates.EDIT; setTimeout(() => { @@ -950,6 +1100,8 @@ class Editor { canvas: this.canvasRenderCamReference || undefined, preserveDrawingBuffer: true, }); + this._configurePostProcessingRaw(); + if (this.camera_person_mode) { this.switchCameraView(); } @@ -1112,10 +1264,13 @@ class Editor { this.composer.setSize(width, height); } + this.render_composer?.setSize(width, height); + if (this.render_camera == undefined) { return; } + this.customOutlinerPass?.setSize(width, height); this.render_camera.aspect = this.getRenderDimensions().aspectRatio; this.render_camera.updateProjectionMatrix(); } diff --git a/app/pages/PageEnigma/Editor/scene.ts b/app/pages/PageEnigma/Editor/scene.ts index 435b12463e..42e83f8a35 100644 --- a/app/pages/PageEnigma/Editor/scene.ts +++ b/app/pages/PageEnigma/Editor/scene.ts @@ -27,8 +27,13 @@ class Scene { // loading indicator manager placeholder_manager: LoadingPlaceHolderManager | undefined; + updateSurfaceIdAttributeToMesh: Function; - constructor(name: string, camera_name: string) { + constructor( + name: string, + camera_name: string, + updateSurfaceIdAttributeToMesh: Function, + ) { this.name = name; this.gridHelper; this.scene = new THREE.Scene(); @@ -41,6 +46,7 @@ class Scene { // global names this.camera_name = camera_name; this.placeholder_manager = undefined; + this.updateSurfaceIdAttributeToMesh = updateSurfaceIdAttributeToMesh; } initialize() { @@ -62,10 +68,7 @@ class Scene { this._create_camera_obj(); } - instantiate( - name: string, - pos: THREE.Vector3 = new THREE.Vector3(0, 0, 0), - ) { + instantiate(name: string, pos: THREE.Vector3 = new THREE.Vector3(0, 0, 0)) { const material = new THREE.MeshPhongMaterial({ color: 0xdacbce }); material.shininess = 0.0; let geometry; @@ -111,8 +114,7 @@ class Scene { obj.userData["base"] = new THREE.Color(0x00d8ff).getHex(); this.shader_objects.push(obj); obj.userData["media_id"] = "Parim"; - } - else if (name.includes("Image::")) { + } else if (name.includes("Image::")) { const image_token = name.replace("Image::", ""); const loader = new THREE.TextureLoader(); const texture = loader.load(image_token); @@ -120,7 +122,7 @@ class Scene { const image_material = new THREE.MeshBasicMaterial({ color: 0xffffff, map: texture, - transparent: true + transparent: true, }); obj = new THREE.Mesh(geometry, image_material); obj.userData["media_id"] = name; @@ -130,8 +132,7 @@ class Scene { }); obj = new THREE.Mesh(geometry, light_material); obj.userData["media_id"] = "Parim"; - } - else { + } else { obj = new THREE.Mesh(geometry, material); obj.userData["media_id"] = "Parim"; } @@ -149,7 +150,7 @@ class Scene { this.scene.add(obj); if (name == "PointLight") { - const light = new THREE.PointLight( 0xffffff, 1, 100 ); + const light = new THREE.PointLight(0xffffff, 1, 100); this.scene.add(light); light.position.copy(obj.position); obj.attach(light); @@ -157,6 +158,7 @@ class Scene { obj.userData["light"] = light.uuid; } + this.updateSurfaceIdAttributeToMesh(this.scene); return obj; } @@ -364,7 +366,7 @@ class Scene { c.material.color.set(new THREE.Color(hex_color)); } - if(c.name == "PointLight") { + if (c.name == "PointLight") { const light = this.get_object_by_uuid(c.userData["light"]); if (light) { (light as THREE.PointLight).color = new THREE.Color(hex_color); @@ -432,6 +434,7 @@ class Scene { mmd.layers.enable(0); mmd.layers.enable(1); this.scene.add(mmd); + this.updateSurfaceIdAttributeToMesh(this.scene); return mmd; } @@ -450,14 +453,13 @@ class Scene { auto_add, position, ); - } - else if ( + } else if ( url.includes(".png") || url.includes(".jpg") || url.includes(".jpeg") || url.includes(".gif") ) { - return this.instantiate("Image::"+url); + return this.instantiate("Image::" + url); } return await this.loadGlbWithPlaceholder( media_id, @@ -587,7 +589,7 @@ class Scene { if (auto_add) { this.scene.add(child_result); } - + this.updateSurfaceIdAttributeToMesh(this.scene); return child_result; } diff --git a/app/pages/PageEnigma/Editor/timeline.ts b/app/pages/PageEnigma/Editor/timeline.ts index b5a1ae644d..508a44e820 100644 --- a/app/pages/PageEnigma/Editor/timeline.ts +++ b/app/pages/PageEnigma/Editor/timeline.ts @@ -24,7 +24,7 @@ import { import { Keyframe, MediaItem, UpdateTime } from "~/pages/PageEnigma/models"; import Editor from "~/pages/PageEnigma/Editor/editor"; import EmotionEngine from "./emotion_engine"; -import { GenerationOptions } from "~/pages/PageEnigma/models/generationOptions"; +import { IGenerationOptions } from "~/pages/PageEnigma/models/generationOptions"; import { Vector3 } from "three"; import { outlinerState } from "../signals"; @@ -156,7 +156,11 @@ export class TimeLine { case toEngineActions.ADD_CHARACTER: { const newObject = await this.addCharacter(data.data as MediaItem); if (newObject) - this.queueNewObjectMessage(newObject, data.data as MediaItem, AssetType.CHARACTER); + this.queueNewObjectMessage( + newObject, + data.data as MediaItem, + AssetType.CHARACTER, + ); const result = this.editorEngine.sceneManager?.render_outliner( this.characters, ); @@ -166,7 +170,11 @@ export class TimeLine { case toEngineActions.ADD_OBJECT: { const newObject = await this.addObject(data.data as MediaItem); if (newObject) - this.queueNewObjectMessage(newObject, data.data as MediaItem, AssetType.OBJECT); + this.queueNewObjectMessage( + newObject, + data.data as MediaItem, + AssetType.OBJECT, + ); const result = this.editorEngine.sceneManager?.render_outliner( this.characters, ); @@ -176,7 +184,11 @@ export class TimeLine { } case toEngineActions.ADD_SHAPE: { const newShape = await this.addShape(data.data as MediaItem); - this.queueNewObjectMessage(newShape, data.data as MediaItem, AssetType.SHAPE); + this.queueNewObjectMessage( + newShape, + data.data as MediaItem, + AssetType.SHAPE, + ); const result = this.editorEngine.sceneManager?.render_outliner( this.characters, ); @@ -200,7 +212,7 @@ export class TimeLine { case toEngineActions.GENERATE_VIDEO: { const options = data.data; // super overloaded talk to the devs about this. TODO... refactor // pass this in ... rather than doing it implicitly ... - this.editorEngine.generation_options = options as GenerationOptions; + this.editorEngine.generation_options = options as IGenerationOptions; this.editorEngine.generateVideo(); break; } @@ -266,7 +278,7 @@ export class TimeLine { queueNewObjectMessage( item: THREE.Object3D, data: MediaItem, - asset_type: AssetType.OBJECT | AssetType.CHARACTER | AssetType.SHAPE + asset_type: AssetType.OBJECT | AssetType.CHARACTER | AssetType.SHAPE, ) { Queue.publish({ queueName: QueueNames.FROM_ENGINE, diff --git a/app/pages/PageEnigma/Editor/video_generation.ts b/app/pages/PageEnigma/Editor/video_generation.ts index 754ff415b8..36b3d54cdf 100644 --- a/app/pages/PageEnigma/Editor/video_generation.ts +++ b/app/pages/PageEnigma/Editor/video_generation.ts @@ -26,6 +26,10 @@ export class VideoGeneration { this.mediaUploadAPI = new MediaUploadApi(); } + sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + async generateFrame() { if (!this.editor.generating_preview && this.editor.rawRenderer) { this.editor.generating_preview = true; @@ -189,190 +193,237 @@ export class VideoGeneration { //const videoURL = URL.createObjectURL(videoBlob); this.editor.generating_preview = true; - const ffmpeg = createFFmpeg({ log: false }); - await ffmpeg.load(); - this.editor.updateLoad(50, "Processing..."); + this.editor.updateLoad({ + progress: 50, + message: "Processing...", + }); - // Write the Uint8Array to the FFmpeg file system - //ffmpeg.FS("writeFile", "input.webm", await fetchFile(videoURL)); - - for (let index = 0; index < this.editor.frame_buffer.length; index++) { - const element = this.editor.frame_buffer[index]; - await ffmpeg.FS( - "writeFile", - `image${index}.png`, - await fetchFile(element), - ); - } - - await ffmpeg.run( - "-framerate", - "" + this.editor.cap_fps, - "-i", - "image%d.png", - "input.mp4", - ); - - await ffmpeg.run( - "-i", - "input.mp4", - "-f", - "lavfi", - "-i", - "anullsrc", // This adds a silent audio track - "-max_muxing_queue_size", - "999999", - "-vf", // WE REMOVED THE BLACK FRAME IN FFMEPG INSTEAD OF THE ENGINE - "select=gte(n\\,1)", // WE REMOVED THE BLACK FRAME IN FFMEPG INSTEAD OF THE ENGINE - // "select=gte(n\\,1),scale=1024:576", - "-c:v", - "libx264", // Specify video codec (optional, but recommended for MP4) - "-c:a", - "aac", // Specify audio codec (optional, but recommended for MP4) - "-shortest", // Ensure output duration matches the shortest stream (video or audio) - "-pix_fmt", - "yuv420p", - "-f", - "mp4", - "0tmp.mp4", - ); - - let itteration = 0; - - if (compile_audio) { - for (const clip of this.editor.timeline.timeline_items) { - if (clip.type == ClipType.AUDIO) { - await this.convertAudioClip(itteration, ffmpeg, clip); - itteration += 1; - } - } - } - - const output = ffmpeg.FS("readFile", itteration + "tmp.mp4"); - - ffmpeg.exit(); - this.editor.generating_preview = false; - - // Create a Blob from the output file for downloading - const blob = new Blob([output.buffer], { type: "video/mp4" }); - - const title = getSceneSignals().title || "Untitled"; - - const style_name = this.editor.art_style.toString(); - const media_token = this.editor.current_scene_media_token || undefined; - - // convert the ip adapter image and upload as a media token - const image_uuid = uuidv4(); - let ipa_image_token = undefined; - - if (this.editor.globalIpAdapterImage != undefined) { - const response = await this.mediaUploadAPI.UploadImage({ - fileName: `${image_uuid}.ipa`, - blob: this.editor.globalIpAdapterImage, - uuid: image_uuid, - }); - if (response.success) { - if (response.data) { - ipa_image_token = response.data; - } - } - } - if (ipa_image_token) { - globalIPAMediaToken.value = ipa_image_token; - } - - // TODO Remove so many of these around wtf. SceneGenereationMetaData should only be one place - const metaData: SceneGenereationMetaData = { - artisticStyle: this.editor.art_style, - positivePrompt: this.editor.positive_prompt, - negativePrompt: this.editor.negative_prompt, - cameraAspectRatio: this.editor.render_camera_aspect_ratio, - upscale: this.editor.generation_options.upscale, - faceDetail: this.editor.generation_options.faceDetail, - styleStrength: this.editor.generation_options.styleStrength, - lipSync: this.editor.generation_options.lipSync, - cinematic: this.editor.generation_options.cinematic, - globalIPAMediaToken: globalIPAMediaToken.value, + const upload_tokens = { + color: "", + normal: "", + depth: "", + outline: "", }; - // This is to save the snapshot of the scene for remixing... - const uuid_snapshot = uuidv4(); + // TODO fix and ensure that render fast, would actually render fast. + for ( + let image_index = 0; + image_index < this.editor.frame_buffer[0].length; + image_index++ + ) { + // Write the Uint8Array to the FFmpeg file system + //ffmpeg.FS("writeFile", "input.webm", await fetchFile(videoURL)); - const saveData = await this.editor.save_manager.saveData({ - sceneTitle: title, - sceneToken: media_token, - sceneGenerationMetadata: metaData, - }); + const ffmpeg = createFFmpeg({ log: false }); + await ffmpeg.load(); - console.log("SAVE DATA:"); - console.log(saveData); - const file = new File([saveData], `${title}.glb`, { - type: "application/json", - }); + for (let index = 0; index < this.editor.frame_buffer.length; index++) { + const element = this.editor.frame_buffer[index][image_index]; + await ffmpeg.FS( + "writeFile", + `image${index}.png`, + await fetchFile(element), + ); + } - const response = - await this.editor.media_upload.UploadSceneSnapshotMediaFileForm({ - maybe_title: title, - maybe_scene_source_media_file_token: media_token, // can be undefined or null - uuid: uuid_snapshot, - blob: file, + await ffmpeg.run( + "-framerate", + "" + this.editor.cap_fps, + "-i", + "image%d.png", + "input.mp4", + ); + + await ffmpeg.run( + "-i", + "input.mp4", + "-f", + "lavfi", + "-i", + "anullsrc", // This adds a silent audio track + "-max_muxing_queue_size", + "999999", + "-vf", // WE REMOVED THE BLACK FRAME IN FFMEPG INSTEAD OF THE ENGINE + "select=gte(n\\,1)", // WE REMOVED THE BLACK FRAME IN FFMEPG INSTEAD OF THE ENGINE + // "select=gte(n\\,1),scale=1024:576", + "-c:v", + "libx264", // Specify video codec (optional, but recommended for MP4) + "-c:a", + "aac", // Specify audio codec (optional, but recommended for MP4) + "-shortest", // Ensure output duration matches the shortest stream (video or audio) + "-pix_fmt", + "yuv420p", + "-f", + "mp4", + "0tmp.mp4", + ); + + let itteration = 0; + + if (compile_audio) { + for (const clip of this.editor.timeline.timeline_items) { + if (clip.type == ClipType.AUDIO) { + await this.convertAudioClip(itteration, ffmpeg, clip); + itteration += 1; + } + } + } + + const output = ffmpeg.FS("readFile", itteration + "tmp.mp4"); + + ffmpeg.exit(); + this.editor.generating_preview = false; + + // Create a Blob from the output file for downloading + const blob = new Blob([output.buffer], { type: "video/mp4" }); + + const title = getSceneSignals().title || "Untitled"; + + const style_name = this.editor.art_style.toString(); + const media_token = this.editor.current_scene_media_token || undefined; + + // convert the ip adapter image and upload as a media token + const image_uuid = uuidv4(); + let ipa_image_token = undefined; + + if (this.editor.globalIpAdapterImage != undefined) { + const response = await this.mediaUploadAPI.UploadImage({ + fileName: `${image_uuid}.ipa`, + blob: this.editor.globalIpAdapterImage, + uuid: image_uuid, + }); + if (response.success) { + if (response.data) { + ipa_image_token = response.data; + } + } + } + if (ipa_image_token) { + globalIPAMediaToken.value = ipa_image_token; + } + + // TODO Remove so many of these around wtf. SceneGenereationMetaData should only be one place + const metaData: SceneGenereationMetaData = { + artisticStyle: this.editor.art_style, + positivePrompt: this.editor.positive_prompt, + negativePrompt: this.editor.negative_prompt, + cameraAspectRatio: this.editor.render_camera_aspect_ratio, + upscale: this.editor.generation_options.upscale, + faceDetail: this.editor.generation_options.faceDetail, + styleStrength: this.editor.generation_options.styleStrength, + lipSync: this.editor.generation_options.lipSync, + cinematic: this.editor.generation_options.cinematic, + globalIPAMediaToken: globalIPAMediaToken.value, + }; + + // This is to save the snapshot of the scene for remixing... + const uuid_snapshot = uuidv4(); + + const saveData = await this.editor.save_manager.saveData({ + sceneTitle: title, + sceneToken: media_token, + sceneGenerationMetadata: metaData, }); - let immutable_media_token = undefined; - if (response.success) { - if (response.data) { - immutable_media_token = response.data; + const file = new File([saveData], `${title}.glb`, { + type: "application/json", + }); + + const response = + await this.editor.media_upload.UploadSceneSnapshotMediaFileForm({ + maybe_title: title, + maybe_scene_source_media_file_token: media_token, // can be undefined or null + uuid: uuid_snapshot, + blob: file, + }); + + let immutable_media_token = undefined; + if (response.success) { + if (response.data) { + immutable_media_token = response.data; + } + } else { + console.log("ERROR:"); + console.log(response.errorMessage); } - } else { - console.log("ERROR:"); - console.log(response.errorMessage); + + console.log("Immutable Token:"); + console.log(immutable_media_token); + + /// TODO refactor this whole thing extract this out. + const data: any = await this.editor.api_manager.uploadMedia({ + blob, + fileName: `${title}.mp4`, + title, + styleName: style_name, + maybe_scene_source_media_file_token: immutable_media_token, + }); + + if (data == null) { + return; + } + const upload_token = data["media_file_token"]; + + console.log(upload_token); + + if (image_index === 0) { + upload_tokens.color = upload_token; + } else if (image_index === 1) { + upload_tokens.normal = upload_token; + } else if (image_index === 2) { + upload_tokens.depth = upload_token; + } else if (image_index === 3) { + upload_tokens.outline = upload_token; + } + + await this.sleep(2000); // TODO: REMOVE THIS WHEN TOO MANY REQUESTS ERROR IS SOLVED. + console.log("Waiting for server to catch up..."); } - console.log("Immutable Token:"); - console.log(immutable_media_token); + this.editor.onWindowResize(); - /// TODO refactor this whole thing extract this out. - const data: any = await this.editor.api_manager.uploadMedia({ - blob, - fileName: `${title}.mp4`, - title, - styleName: style_name, - maybe_scene_source_media_file_token: immutable_media_token, - }); + console.log(upload_tokens); + this.editor.setColorMap(); - if (data == null) { - return; - } - const upload_token = data["media_file_token"]; + console.log("https://storyteller.ai/media/" + upload_tokens.color); + console.log("https://storyteller.ai/media/" + upload_tokens.normal); + console.log("https://storyteller.ai/media/" + upload_tokens.depth); + console.log("https://storyteller.ai/media/" + upload_tokens.outline); - // create the scene token from here then snapshot it associate with the video generation. - // send immutable token with the video ... TODO: await this.editor.api_manager - .stylizeVideo( - upload_token, - this.editor.art_style, - this.editor.positive_prompt, - this.editor.negative_prompt, - Visibility.Public, - this.editor.generation_options.faceDetail, - this.editor.generation_options.upscale, - this.editor.generation_options.styleStrength, - this.editor.generation_options.lipSync, - this.editor.generation_options.cinematic, - globalIPAMediaToken.value, - ) + .stylizeVideo({ + media_token: upload_tokens.color, + style: this.editor.art_style, + positive_prompt: this.editor.positive_prompt, + negative_prompt: this.editor.negative_prompt, + visibility: Visibility.Public, + use_face_detailer: this.editor.generation_options.faceDetail, + use_upscaler: this.editor.generation_options.upscale, + use_strength: this.editor.generation_options.styleStrength, + use_lipsync: this.editor.generation_options.lipSync, + use_cinematic: this.editor.generation_options.cinematic, + use_global_ipa_media_token: globalIPAMediaToken.value, + input_depth_file: upload_tokens.depth, + input_normal_file: upload_tokens.normal, + input_outline_file: upload_tokens.outline, + }) .catch((error) => { - // TODO handle stylize error. console.log(error); - this.editor.updateLoad(100, "Failed To Render"); + this.editor.updateLoad({ + progress: 100, + message: "Failed To Render", + label: "Error", + }); this.editor.endLoading(); }); - // Not sure if this is needed will double check TODO: MC - //this.editor.generating_preview = false; - // {"success":true,"inference_job_token":"jinf_j3nbqbd15wqxb0xcks13qh3f3bz"} - this.editor.updateLoad(100, "Done Check Your Media Tab On Profile."); + this.editor.updateLoad({ + progress: 100, + message: "Done Check Your Media Tab On Profile.", + label: "Success", + }); + this.editor.endLoading(); this.editor.recorder = undefined; @@ -390,6 +441,8 @@ export class VideoGeneration { canvas: this.editor.canvasRenderCamReference || undefined, preserveDrawingBuffer: true, }); + this.editor._configurePostProcessingRaw(); + this.editor.activeScene.renderMode(false); diff --git a/app/pages/PageEnigma/Queue/Queue.ts b/app/pages/PageEnigma/Queue/Queue.ts index 5b8e6c35da..0f4c7ae3fa 100644 --- a/app/pages/PageEnigma/Queue/Queue.ts +++ b/app/pages/PageEnigma/Queue/Queue.ts @@ -11,7 +11,7 @@ import { import { toTimelineActions } from "./toTimelineActions"; import { ClipUI } from "../datastructures/clips/clip_ui"; -import { GenerationOptions } from "../models/generationOptions"; +import { IGenerationOptions } from "../models/generationOptions"; import { ToastDataType } from "~/components"; export type UnionedActionTypes = @@ -26,7 +26,7 @@ export type UnionedDataTypes = | ClipUI[] | MediaItem | ToastDataType - | GenerationOptions + | IGenerationOptions | CameraAspectRatio | null; diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/GenerateMovieButton.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/GenerateMovieButton.tsx index f29f7fead5..9e7b37d823 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/GenerateMovieButton.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/GenerateMovieButton.tsx @@ -12,6 +12,7 @@ import { lipSync, cinematic, globalIPAMediaToken, + enginePreProcessing, } from "~/pages/PageEnigma/signals/stylizeTab"; import { useEffect, useRef } from "react"; @@ -59,6 +60,7 @@ export function GenerateMovieButton({ lipSync.value, cinematic.value, globalIPAMediaToken.value, + enginePreProcessing.value, ); Queue.publish({ queueName: QueueNames.TO_ENGINE, diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/StyleButtons.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/StyleButtons.tsx index 22c772b31b..4bbe3efa83 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/StyleButtons.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/StyleButtons.tsx @@ -10,8 +10,6 @@ import Queue from "~/pages/PageEnigma/Queue/Queue"; import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; import { Switch } from "@headlessui/react"; -// import Slider from "rc-slider"; -// import "rc-slider/assets/index.css"; import { faceDetail, @@ -19,11 +17,15 @@ import { upscale, lipSync, cinematic, + enginePreProcessing, } from "~/pages/PageEnigma/signals/stylizeTab"; import { twMerge } from "tailwind-merge"; +import { useContext } from "react"; +import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext"; export function StyleButtons() { useSignals(); + const editorEngine = useContext(EngineContext); const sliderChanged = (value: number | number[]) => { styleStrength.value = (value as number) / 100; @@ -71,6 +73,13 @@ export function StyleButtons() { } }; + const enginePreProcessingChange = () => { + enginePreProcessing.value = !enginePreProcessing.value; + if (editorEngine) { + editorEngine.engine_preprocessing = enginePreProcessing.value; + } + }; + const handleUpscaleChange = () => { upscale.value = !upscale.value; if (upscale.value) { @@ -249,6 +258,36 @@ export function StyleButtons() {
+
+
+ + + Engine PreProcessing + + + + + +
diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/StylizeTab.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/StylizeTab.tsx index b8ef2e532c..bd74bd8a44 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/StylizeTab.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/StylizeTab.tsx @@ -21,6 +21,7 @@ import { export function StylizeTab() { useSignals(); + const [view, setView] = useState(StylizeTabPages.MAIN); const [generateSectionHeight, setGenerateSectionHeight] = useState(114); diff --git a/app/pages/PageEnigma/models/generationOptions.ts b/app/pages/PageEnigma/models/generationOptions.ts index 8c716bd126..fa187dc044 100644 --- a/app/pages/PageEnigma/models/generationOptions.ts +++ b/app/pages/PageEnigma/models/generationOptions.ts @@ -1,19 +1,21 @@ -export interface GenerationOptions { +export interface IGenerationOptions { upscale: boolean; faceDetail: boolean; styleStrength: number; lipSync: boolean; cinematic: boolean; globalIpAdapterImageMediaToken: string | null; + enginePreProcessing: boolean; } -export class GenerationOptions { +export class GenerationOptions implements IGenerationOptions { public upscale: boolean; public faceDetail: boolean; public styleStrength: number; public lipSync: boolean; public cinematic: boolean; public globalIpAdapterImageMediaToken: string | null; + public enginePreProcessing: boolean; constructor( upscale: boolean, faceDetail: boolean, @@ -21,6 +23,7 @@ export class GenerationOptions { lipSync: boolean, cinematic: boolean, globalIpAdapterImageMediaToken: string | null, + enginePreprocessing: boolean, ) { this.upscale = upscale; this.faceDetail = faceDetail; @@ -28,5 +31,6 @@ export class GenerationOptions { this.lipSync = lipSync; this.cinematic = cinematic; this.globalIpAdapterImageMediaToken = globalIpAdapterImageMediaToken; + this.enginePreProcessing = enginePreprocessing; } } diff --git a/app/pages/PageEnigma/models/sceneGenerationMetadata.ts b/app/pages/PageEnigma/models/sceneGenerationMetadata.ts index bda3c06ffb..34f8359471 100644 --- a/app/pages/PageEnigma/models/sceneGenerationMetadata.ts +++ b/app/pages/PageEnigma/models/sceneGenerationMetadata.ts @@ -14,4 +14,6 @@ export interface SceneGenereationMetaData { styleStrength: number; lipSync: boolean; cinematic: boolean; + + enginePreProcessing: boolean; } diff --git a/app/pages/PageEnigma/signals/stylizeTab.ts b/app/pages/PageEnigma/signals/stylizeTab.ts index 058601cd10..b7076ea125 100644 --- a/app/pages/PageEnigma/signals/stylizeTab.ts +++ b/app/pages/PageEnigma/signals/stylizeTab.ts @@ -19,6 +19,7 @@ export const faceDetail = signal(false); export const styleStrength = signal(1.0); export const lipSync = signal(false); export const cinematic = signal(true); +export const enginePreProcessing = signal(false); export const setArtStyleSelection = (newStyle: ArtStyle) => { if (selectedArtStyle.value !== newStyle) { @@ -37,6 +38,7 @@ export const resetSceneGenerationMetadata = () => { styleStrength.value = 1.0; lipSync.value = false; cinematic.value = false; + enginePreProcessing.value = false; }; export const restoreSceneGenerationMetadata = ( @@ -72,4 +74,7 @@ export const restoreSceneGenerationMetadata = ( if (newData.cinematic) { cinematic.value = newData.cinematic; } + if (newData.enginePreProcessing) { + enginePreProcessing.value = newData.enginePreProcessing; + } }; diff --git a/proxy/src/upload-video/uploadMediaFile.ts b/proxy/src/upload-video/uploadMediaFile.ts index 8f4984a7d5..db5aa3c90a 100644 --- a/proxy/src/upload-video/uploadMediaFile.ts +++ b/proxy/src/upload-video/uploadMediaFile.ts @@ -13,7 +13,7 @@ declare const process: { BASE_API: string; }; }; - +// TODO REPLACE THIS export async function uploadMediaFile({ blob, fileName, diff --git a/proxy/src/upload-video/uploadVideo.ts b/proxy/src/upload-video/uploadVideo.ts index 43632aa9cd..4d09a83a2e 100644 --- a/proxy/src/upload-video/uploadVideo.ts +++ b/proxy/src/upload-video/uploadVideo.ts @@ -136,7 +136,7 @@ async function stopPlaybackAndUploadVideo( const canvas = new Canvas(1024, 576); const context = canvas.getContext("2d"); const folder = uuid.v4(); - + // use then... await new Promise((resolve) => { fs.mkdir(`./${folder}`, () => { fs.mkdir(`./${folder}/images`, () => { @@ -256,6 +256,7 @@ async function stopPlaybackAndUploadVideo( const media_token = video.current_scene_media_token || undefined; console.log("Upload file"); + // TODO REPLACE const uploadData: any = await uploadMediaFile({ blob, fileName: `${title}.mp4`,