diff --git a/src/KonvaApp/Engine.ts b/src/KonvaApp/Engine.ts index 3c7d7775b6..5181dcd586 100644 --- a/src/KonvaApp/Engine.ts +++ b/src/KonvaApp/Engine.ts @@ -15,6 +15,18 @@ import { ResponseType } from "./WorkerPrimitives/SharedWorkerBase"; import * as ort from "onnxruntime-web"; +export interface RenderingOptions { + artstyle: string; + positivePrompt: string; + negativePrompt: string; + cinematic: boolean; + enginePreProcessing: boolean; + faceDetail: boolean; + lipSync: boolean; + upscale: boolean; + styleStrength: number; +} + export class Engine { private canvasReference: HTMLDivElement; private stage: Konva.Stage; @@ -24,6 +36,9 @@ export class Engine { private selectionManager: SelectionManager; + // WIL need to fix this ui issue. + public ranOnce: Boolean = false; + // signal reference constructor(canvasReference: HTMLDivElement) { console.log("Engine Created!"); @@ -124,6 +139,14 @@ export class Engine { title: "Generation Error", message: response.data, }); + + if (!this.renderEngine.videoLoadingCanvas) { + console.log("Did not setup video loading canvas."); + return; + } + + this.renderEngine.videoLoadingCanvas.kNode.hide(); + uiAccess.toolbarMain.loadingBar.hide(); } } @@ -168,15 +191,26 @@ export class Engine { this.addVideo(video.url); }); - uiEvents.aiStylize.onRequest((data) => { + uiEvents.aiStylize.onRequest(async (data) => { console.log("Engine heard AI Stylize request: ", data); - console.log( - "you can also get initial values via uiEvents.aiStylize.getInitialValues", - ); - console.log( - "if you missed the event, the current values can be accessed via uiEvents.aiStylize.getCurrentValues", - ); + //console.log(data); + // Hack to ensure this doesn't break because UI BUG + + if (this.ranOnce === true) { + try { + await this.renderEngine.startProcessing(data); + } catch (error) { + // throw error to retry + uiAccess.dialogueError.show({ + title: "Generation Error", + message: error.toString(), + }); + } + } + + this.ranOnce = true; }); + // TODO: You may listen to all the image toolbar events here uiEvents.toolbarImage.MOVE.onClick(() => { console.log("move"); @@ -191,22 +225,6 @@ export class Engine { //this.onRenderingSystemReceived(undefined); }); - uiEvents.aiStylize.onRequest(async () => { - // uiAccess.toolbarMain.changeButtonState( - // ToolbarMainButtonNames.AI_STYLIZE, - // { disabled: true }, - // ); - - // TODO pull the values for stlyization - - await this.renderEngine.startProcessing(); - - // uiAccess.toolbarMain.changeButtonState( - // ToolbarMainButtonNames.AI_STYLIZE, - // { disabled: false }, - // ); - }); - // WIL please default hide this. TODO Remove uiAccess.toolbarMain.loadingBar.hide(); uiAccess.toolbarMain.loadingBar.updateStatus(LoadingBarStatus.IDLE); @@ -292,7 +310,6 @@ export class Engine { public initializeStage(sceneToken: string) { // load canvas that was originaly saved TODO Save manager for resharing. - uiAccess.toolbarImage.hide(); uiAccess.loadingBar.hide(); this.setupStage(); @@ -331,27 +348,10 @@ export class Engine { // "https://storage.googleapis.com/vocodes-public/media/r/q/p/r/e/rqpret6mkh18dqwjqwghhdqf15x720s1/storyteller_rqpret6mkh18dqwjqwghhdqf15x720s1.mp4", // this.selectionManager, // ); - - // const videoNode2 = new VideoNode( - // "", - // this.offScreenCanvas, - // this.videoLayer, - // 1560, - // 1000, - // "https://storage.googleapis.com/vocodes-public/media/r/q/p/r/e/rqpret6mkh18dqwjqwghhdqf15x720s1/storyteller_rqpret6mkh18dqwjqwghhdqf15x720s1.mp4", - // this.selectionManager, - // ); - // CODE TO TEST RENDER ENGINE // Testing render engine // this.renderEngine.addNodes(videoNode); - // this.renderEngine.addNodes(videoNode2); - // await this.renderEngine.startProcessing(); - - // Call this when test video nodes - // await this.renderEngine.startProcessing(); - //videoNode.simulatedLoading(); // TODO support Text nodes @@ -367,6 +367,7 @@ export class Engine { imageFile, this.selectionManager, ); + this.renderEngine.addNodes(imageNode); } public addVideo(url: string) { // Adding nodes here diff --git a/src/KonvaApp/RenderingPrimitives/RenderEngine.ts b/src/KonvaApp/RenderingPrimitives/RenderEngine.ts index 9d17d218e0..7bb27ebf83 100644 --- a/src/KonvaApp/RenderingPrimitives/RenderEngine.ts +++ b/src/KonvaApp/RenderingPrimitives/RenderEngine.ts @@ -23,11 +23,14 @@ import { DiffusionSharedWorker, DiffusionSharedWorkerItemData, } from "../SharedWorkers/Diffusion/DiffusionSharedWorker"; +import { RenderingOptions } from "../Engine"; +import { ImageNode } from "../Nodes/ImageNode"; // https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio export class RenderEngine { private videoNodes: VideoNode[]; + private imageNodes: ImageNode[]; private offScreenCanvas: OffscreenCanvas; private context: OffscreenCanvasRenderingContext2D | null; @@ -143,6 +146,8 @@ export class RenderEngine { ) { this.videoLoadingCanvas = undefined; this.videoNodes = []; + this.imageNodes = []; + this.isProcessing = false; this.onRenderingSystemMessageRecieved = onRenderingSystemMessageRecieved; // TODO: Make this dynamic and update this on change of canvas. @@ -254,8 +259,12 @@ export class RenderEngine { return maxLength; } - public addNodes(node: VideoNode) { - this.videoNodes.push(node); + public addNodes(node: VideoNode | ImageNode) { + if (node instanceof VideoNode) { + this.videoNodes.push(node); + } else if (node instanceof ImageNode) { + this.imageNodes.push(node); + } } public removeNodes(node: VideoNode) { @@ -267,14 +276,17 @@ export class RenderEngine { } // Do a bunch of precondition checks and error out early on. - public async startProcessing() { + public async startProcessing(renderingOptions: RenderingOptions) { // Start processing and lock everything + this.isProcessing = true; + try { // or not loaded - + if (this.videoNodes.length + this.imageNodes.length < 1) { + throw Error("Must have atleast Media item on the board."); + } // error out if nodes are not all loaded. - // todo remove when we have error handling + and ui var failed = false; for (let i = 0; i < this.videoNodes.length; i++) { @@ -294,7 +306,7 @@ export class RenderEngine { // todo remove if (failed) { // throw error - throw Error("Wait For Items to Finish Processing"); + throw Error("Wait For Items to Finish Processing."); } this.videoNodes.forEach((item: VideoNode) => {}); @@ -304,7 +316,7 @@ export class RenderEngine { numberOfFrames = Math.min(numberOfFrames, this.upperMaxFrames); console.log(`Number Of Frames: ${numberOfFrames}`); - await this.render(numberOfFrames); + await this.render(numberOfFrames, renderingOptions); } catch (error) { console.log(error); throw error; @@ -333,7 +345,10 @@ export class RenderEngine { then seek through each node 1 step. stop ignore stepping if the duration is less. **/ - private async render(largestNumberOfFrames: number) { + private async render( + largestNumberOfFrames: number, + renderingOptions: RenderingOptions, + ) { if (!this.isProcessing) return; // Stop all nodes first @@ -429,6 +444,7 @@ export class RenderEngine { imageBitmap: this.offScreenCanvas.transferToImageBitmap(), frame: j, totalFrames: largestNumberOfFrames, + prompt: renderingOptions, }; let isDoneStreaming = false; diff --git a/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorker.ts b/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorker.ts index b6e223547e..62c7225226 100644 --- a/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorker.ts +++ b/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorker.ts @@ -18,6 +18,7 @@ import { ResponseType, } from "~/KonvaApp/WorkerPrimitives/SharedWorkerBase"; import { JobStatus } from "~/Classes/ApiManager/enums/Job"; +import { RenderingOptions } from "~/KonvaApp/Engine"; export interface DiffusionSharedWorkerProgressData { url: string; @@ -31,7 +32,9 @@ export interface DiffusionSharedWorkerItemData { frame: number; height: number; width: number; + prompt: RenderingOptions; } + export interface DiffusionSharedWorkerErrorData { error: string; } @@ -154,23 +157,24 @@ export class DiffusionSharedWorker extends SharedWorkerBase< throw Error("Media Token Not Availible"); } + console.log(item.prompt); const studioResponse = await this.videoAPI.EnqueueStudio({ enqueueVideo: { disable_lcm: false, - enable_lipsync: false, - input_file: mediaToken, - negative_prompt: "", - prompt: "High quality anime style", + enable_lipsync: item.prompt.lipSync, + input_file: mediaToken, // Replace with actual media token + negative_prompt: item.prompt.negativePrompt, + prompt: item.prompt.positivePrompt, remove_watermark: false, - style: ArtStyleNames.Anime2_5D, + style: item.prompt.artstyle, // Map to the appropriate art style frame_skip: 2, travel_prompt: "", trim_end_millis: 7000, trim_start_millis: 0, - use_cinematic: true, - use_face_detailer: false, - use_strength: 1.0, - use_upscaler: false, + use_cinematic: item.prompt.cinematic, + use_face_detailer: item.prompt.faceDetail, + use_strength: item.prompt.styleStrength, + use_upscaler: item.prompt.upscale, uuid_idempotency_token: uuidv4(), global_ipa_media_token: "", input_depth_file: "", @@ -214,14 +218,11 @@ export class DiffusionSharedWorker extends SharedWorkerBase< }; renderProgressData.status = status; + switch (status) { case JobStatus.PENDING: reportProgress(renderProgressData); // once finished gives you up to 50% break; - case JobStatus.DEAD: - renderProgressData.progress = 0; - reportProgress(renderProgressData); - break; case JobStatus.STARTED: reportProgress(renderProgressData); break; @@ -230,7 +231,6 @@ export class DiffusionSharedWorker extends SharedWorkerBase< break; case JobStatus.COMPLETE_SUCCESS: renderProgressData.progress = 100; - jobIsProcessing = false; if (!job.data.maybe_result.maybe_public_bucket_media_path) { throw Error("Server Failed To Return Result"); @@ -246,6 +246,7 @@ export class DiffusionSharedWorker extends SharedWorkerBase< case JobStatus.DEAD: jobIsProcessing = false; reportProgress(renderProgressData); + throw Error("Server Failed to Process Please Try Again."); break; case JobStatus.CANCCELLED_BY_SYSTEM: jobIsProcessing = false;