diff --git a/public/light.mp4 b/public/light.mp4 new file mode 100644 index 0000000000..063f7becec Binary files /dev/null and b/public/light.mp4 differ diff --git a/src/KonvaApp/Engine.ts b/src/KonvaApp/Engine.ts index b75e95075d..ebe0357cfe 100644 --- a/src/KonvaApp/Engine.ts +++ b/src/KonvaApp/Engine.ts @@ -13,7 +13,6 @@ import { ImageNode } from "./Nodes/ImageNode"; import { LoadingBarStatus } from "~/components/ui"; import { ResponseType } from "./WorkerPrimitives/SharedWorkerBase"; import { UndoStackManager } from "./UndoRedo/UndoRedoManager"; -import * as ort from "onnxruntime-web"; import { CreateCommand } from "./UndoRedo/CreateCommand"; import { DeleteCommand } from "./UndoRedo/DeleteCommand"; @@ -21,6 +20,8 @@ import { RotateCommand } from "./UndoRedo/RotateCommand"; import { ScaleCommand } from "./UndoRedo/ScaleCommand"; import { TranslateCommand } from "./UndoRedo/TranslateCommand"; +import { FileUtilities } from "./FileUtilities/FileUtilities"; + export interface RenderingOptions { artstyle: string; positivePrompt: string; @@ -117,17 +118,13 @@ export class Engine { const media_url = `${media_api_base_url}vocodes-public${response.data.videoUrl}`; const videoNode = new VideoNode( - uuidv4(), this.videoLayer, this.renderEngine.captureCanvas.position().x, this.renderEngine.captureCanvas.position().y, media_url, this.selectionManager, - undefined, - undefined, ); this.renderEngine.addNodes(videoNode); - // hide the loader //this.renderEngine.videoLoadingCanvas.kNode.hide(); uiAccess.toolbarMain.loadingBar.hide(); @@ -138,19 +135,17 @@ export class Engine { uiAccess.toolbarMain.loadingBar.updateProgress( response.data.progress * 100, ); + console.log(response); + + // if (response.data.zipBlob) { + // FileUtilities.downloadBlobZip(response.data.zipBlob); + // } } else { // throw error to retry uiAccess.dialogueError.show({ title: "Generation Error", message: response.data, }); - - // if (!this.renderEngine.videoLoadingCanvas) { - // console.log("Did not setup video loading canvas."); - // return; - // } - //this.renderEngine.videoLoadingCanvas.kNode.hide(); - uiAccess.toolbarMain.loadingBar.hide(); } } @@ -241,64 +236,9 @@ export class Engine { this.stage.width(this.canvasReference.offsetWidth); this.stage.height(this.canvasReference.offsetHeight); - this.stage.draw(); // Redraw the canvas } - public async onnx() { - try { - //DO NOT REMOVE, NECESSARY TO LOAD WASM FILES - ort.env.wasm.wasmPaths = "wasm/"; - - // Load the model and create InferenceSession - const modelPathE = "/models/image_encoder_hiera_t.onnx"; - const modelPath = "/models/mask_decoder_hiera_t.onnx"; - const modelPath1 = "/models/memory_attention_hiera_t.onnx"; - const modelPath2 = "/models/memory_encoder_hiera_t.onnx"; - const modelPath3 = "/models/mlp_hiera_t.onnx"; - const modelPath4 = "/models/prompt_encoder_hiera_t.onnx"; - - const mask_decoder_hiera_t = await ort.InferenceSession.create( - modelPath, - { - executionProviders: ["wasm"], - }, - ); - console.log(mask_decoder_hiera_t); - const memory_attention_hiera_t = await ort.InferenceSession.create( - modelPath1, - { - executionProviders: ["wasm"], - }, - ); - console.log(memory_attention_hiera_t); - const memory_encoder_hiera_t = await ort.InferenceSession.create( - modelPath2, - { - executionProviders: ["wasm"], - }, - ); - console.log(memory_encoder_hiera_t); - const mlp_hiera_t = await ort.InferenceSession.create(modelPath3, { - executionProviders: ["wasm"], - }); - console.log(mlp_hiera_t); - const prompt_encoder_hiera_t = await ort.InferenceSession.create( - modelPath4, - { - executionProviders: ["wasm"], - }, - ); - console.log(prompt_encoder_hiera_t); - - // // Run inference - // const outputs = await session.run({ input: inputTensor }); - // console.log(outputs); - } catch (err) { - console.error("error caught: ", err); - } - } - // Sandbox is quickly a way to test your idea. public async sandbox() {} @@ -311,11 +251,56 @@ export class Engine { // load canvas that was originaly saved TODO Save manager for resharing. uiAccess.toolbarImage.hide(); uiAccess.loadingBar.hide(); + this.setupStage(); } public isInitialized() { return this.stage !== null; } + + public async populateWithDebugItems() { + const imageFile = await FileUtilities.createImageFileFromUrl( + "https://static.miraheze.org/pgrwiki/0/0d/Dialogue-2B-Icon.png", + ); + const imageNode = new ImageNode( + this.videoLayer, + this.renderEngine.captureCanvas.position().x, + this.renderEngine.captureCanvas.position().y, + imageFile, + this.selectionManager, + ); + + const imageNode2 = new ImageNode( + this.videoLayer, + this.renderEngine.captureCanvas.position().x, + this.renderEngine.captureCanvas.position().y, + imageFile, + this.selectionManager, + ); + + // Adding nodes here + const videoNode = new VideoNode( + this.videoLayer, + this.renderEngine.captureCanvas.position().x, + this.renderEngine.captureCanvas.position().y, + "https://storage.googleapis.com/vocodes-public/media/r/q/p/r/e/rqpret6mkh18dqwjqwghhdqf15x720s1/storyteller_rqpret6mkh18dqwjqwghhdqf15x720s1.mp4", + this.selectionManager, + ); + + // CODE TO TEST RENDER ENGINE + // Testing render engine + // this.renderEngine.addNodes(videoNode); + // await this.renderEngine.startProcessing(); + //videoNode.simulatedLoading(); + // TODO support Text nodes + + this.renderEngine.addNodes(videoNode); + this.renderEngine.addNodes(imageNode2); + + // TODO if only image take image and just takes snapshots Edge case + this.renderEngine.addNodes(imageNode); + } + public async setupStage() { var textNode = new Konva.Text({ x: 0, @@ -339,23 +324,6 @@ export class Engine { }, this.videoLayer); anim.start(); - // Adding nodes here - // const videoNode = new VideoNode( - // "", - // this.offScreenCanvas, - // this.videoLayer, - // 1560, - // 400, - // "https://storage.googleapis.com/vocodes-public/media/r/q/p/r/e/rqpret6mkh18dqwjqwghhdqf15x720s1/storyteller_rqpret6mkh18dqwjqwghhdqf15x720s1.mp4", - // this.selectionManager, - // ); - // CODE TO TEST RENDER ENGINE - // Testing render engine - // this.renderEngine.addNodes(videoNode); - // await this.renderEngine.startProcessing(); - //videoNode.simulatedLoading(); - // TODO support Text nodes - this.videoLayer.add(textNode); this.addKeyboardShortcuts(); @@ -363,39 +331,39 @@ export class Engine { public addImage(imageFile: File) { const imageNode = new ImageNode( - uuidv4(), this.videoLayer, - 50, - 50, + this.renderEngine.captureCanvas.position().x, + this.renderEngine.captureCanvas.position().y, imageFile, this.selectionManager, ); this.renderEngine.addNodes(imageNode); } + public addVideo(url: string) { // Adding nodes here const videoNode = new VideoNode( - uuidv4(), this.videoLayer, - 1560, - 400, + this.renderEngine.captureCanvas.position().x, + this.renderEngine.captureCanvas.position().y, url, this.selectionManager, - undefined, - undefined, ); this.renderEngine.addNodes(videoNode); } + // Events for Undo and Redo private addKeyboardShortcuts() { window.addEventListener("keydown", (event) => { if (event.ctrlKey && event.key === "z") { - this.undo(); + console.log("Undo"); + this.undoStackManager.undo(); } else if ( (event.ctrlKey && event.key === "y") || (event.ctrlKey && event.shiftKey && event.key === "Z") ) { - this.redo(); + console.log("Redo"); + this.undoStackManager.redo(); } }); } @@ -410,26 +378,18 @@ export class Engine { this.undoStackManager.executeCommand(command); } + scaleNodes(nodes: Konva.Node[], newScaleX: number, newScaleY: number) { + const command = new ScaleCommand(nodes, newScaleX, newScaleY); + this.undoStackManager.executeCommand(command); + } + deleteNodes(nodes: Konva.Node[]) { const command = new DeleteCommand(nodes); this.undoStackManager.executeCommand(command); } createNodes(nodes: Konva.Node[]) { - const command = new CreateCommand(nodes, this.videoLayer); + const command = new CreateCommand(nodes); this.undoStackManager.executeCommand(command); } - - scaleNodes(nodes: Konva.Node[], newScaleX: number, newScaleY: number) { - const command = new ScaleCommand(nodes, newScaleX, newScaleY); - this.undoStackManager.executeCommand(command); - } - - undo() { - this.undoStackManager.undo(); - } - - redo() { - this.undoStackManager.redo(); - } } diff --git a/src/KonvaApp/FileUtilities/BlobDebugger.ts b/src/KonvaApp/FileUtilities/FileUtilities.ts similarity index 79% rename from src/KonvaApp/FileUtilities/BlobDebugger.ts rename to src/KonvaApp/FileUtilities/FileUtilities.ts index 75aadfcc6b..bf62cb14ac 100644 --- a/src/KonvaApp/FileUtilities/BlobDebugger.ts +++ b/src/KonvaApp/FileUtilities/FileUtilities.ts @@ -35,4 +35,14 @@ export class FileUtilities { console.log(error); } } + + static async createImageFileFromUrl( + url: string, + fileName: string = "image.jpg", + mimeType: string = "image/jpeg", + ): Promise { + const response = await fetch(url); + const blob = await response.blob(); + return new File([blob], fileName, { type: mimeType }); + } } diff --git a/src/KonvaApp/Nodes/ImageNode.ts b/src/KonvaApp/Nodes/ImageNode.ts index 01181b1576..0a8d47bdde 100644 --- a/src/KonvaApp/Nodes/ImageNode.ts +++ b/src/KonvaApp/Nodes/ImageNode.ts @@ -1,185 +1,72 @@ import Konva from "konva"; import { Layer } from "konva/lib/Layer"; import { NetworkedNodeContext } from "./NetworkedNodeContext"; -import { v4 as uuidv4 } from "uuid"; - -import { uiAccess } from "~/signals"; import { SelectionManager } from "../SelectionManager"; - -const toolbarVideo = uiAccess.toolbarImage; -const loadingBar = uiAccess.loadingBar; - export class ImageNode extends NetworkedNodeContext { - public selectionManagerRef: SelectionManager; - public imageURL: string; - public videoComponent: HTMLVideoElement; - public kNode: Konva.Image; + private imageObject: HTMLImageElement; - // do not modify internal - public didFinishLoading: boolean; - private videoLayer: Layer; - - public uuid: string; - - // This locks interaction when the render engine is rendering - private isProcessing: boolean = false; - - async setProcessing() { - this.isProcessing = true; - } - - public delete() { - // Do any other clean up and delete the konva node. - if (this.kNode) { - this.kNode.destroy(); - } - } - - public highlight() { - this.kNode.stroke("salmon"); - this.kNode.strokeWidth(12); - this.kNode.draw(); - } - public unHighLight() { - this.kNode.stroke(null); - this.kNode.strokeWidth(0); - this.kNode.draw(); - } - - private imageObject: Image; constructor( - uuid: string = uuidv4(), - videoLayer: Layer, + mediaLayer: Layer, x: number, y: number, imageFile: File, selectionManagerRef: SelectionManager, ) { - super(); + super(selectionManagerRef, mediaLayer); this.selectionManagerRef = selectionManagerRef; - this.uuid = uuid; - this.videoLayer = videoLayer; + this.mediaLayer = mediaLayer; this.didFinishLoading = false; - const imageObj = new Image(); - this.imageObject = imageObj; + const imageComponent = new Image(); + + this.imageObject = imageComponent; this.imageObject.crossOrigin = "anonymous"; - imageObj.onload = () => { - // TODO load this proper. - this.kNode = new Konva.Image({ - x: x, - y: y, - image: imageObj, - width: imageObj.width, - height: imageObj.height, - draggable: true, - }); - // TODO use loading image + this.kNode = new Konva.Image({ + x: x, + y: y, + image: undefined, // to do replace with placeholder + width: 100, + height: 100, + fill: "gray", + draggable: true, + }); + + this.mediaLayer.add(this.kNode); + + imageComponent.onload = () => { + if (!this.kNode) { + return; + } + + this.kNode.image(imageComponent); + this.kNode.width(imageComponent.width); + this.kNode.height(imageComponent.height); + + this.listenToBaseKNode(); + this.kNode.fill(null); + this.mediaLayer.draw(); + this.didFinishLoading = true; - - // add the shape to the layer - - this.kNode.on("dragstart", (e) => { - this.updateContextMenuPosition(); - // Multiselect - const isMultiSelect = e.evt.shiftKey; - this.selectionManagerRef.startDrag(this); - if (this.didFinishLoading == false) { - this.updateLoadingBarPosition(); - } - }); - - this.kNode.on("dragmove", (e) => { - // shouldn't be able to move if processing. - - this.updateContextMenuPosition(); - // Multiselect - const isMultiSelect = e.evt.shiftKey; - this.selectionManagerRef.dragging(this); - if (this.isProcessing) { - return; - } - - if (this.didFinishLoading == false) { - this.updateLoadingBarPosition(); - } - }); - - this.kNode.on("dragend", (e) => { - this.selectionManagerRef.draggingStopped(this); - }); - - this.kNode.on("dragend", () => { - this.updateContextMenuPosition(); - if (this.didFinishLoading == false) { - this.updateLoadingBarPosition(); - } - }); - - this.kNode.on("mousedown", (e) => { - toolbarVideo.show(); - - // selection on click doesn't do a good job. - const isMultiSelect = e.evt.shiftKey; - this.selectionManagerRef.selectNode(this, isMultiSelect); - - this.updateContextMenuPosition(); - - if (this.didFinishLoading == false) { - this.updateLoadingBarPosition(); - } else { - loadingBar.hide(); - } - }); - - this.kNode.on("mouseup", () => { - if (this.didFinishLoading == false) { - return; - } - }); - - videoLayer.add(this.kNode); - videoLayer.draw(); }; - imageObj.src = URL.createObjectURL(imageFile); + imageComponent.src = URL.createObjectURL(imageFile); } async updateImage(newImageSrc: string) { const newImage = new Image(); newImage.src = newImageSrc; newImage.onload = () => { + if (!this.kNode) { + return; + } this.kNode.image(newImage); + this.kNode.width(newImage.width); + this.kNode.height(newImage.height); + this.kNode.fill(null); this.kNode.draw(); }; } - - updateContextMenuPosition() { - toolbarVideo.setPosition({ - x: this.kNode.getPosition().x + this.kNode.getSize().width / 4, - y: this.kNode.getPosition().y - 150, - }); - } - - updateLoadingBarPosition() { - loadingBar.updatePosition({ - x: this.kNode.getPosition().x + this.kNode.getSize().width / 4, - y: this.kNode.getPosition().y - 60, - }); - } - - public bringToFront() { - this.kNode.moveUp(); - } - - public sendBack() { - // prevent canvas being in front. - if (this.kNode.zIndex() === 1) { - return; - } - this.kNode.moveDown(); - } } diff --git a/src/KonvaApp/Nodes/NetworkedNodeContext.ts b/src/KonvaApp/Nodes/NetworkedNodeContext.ts index 600206b56a..247b298e88 100644 --- a/src/KonvaApp/Nodes/NetworkedNodeContext.ts +++ b/src/KonvaApp/Nodes/NetworkedNodeContext.ts @@ -1,27 +1,198 @@ -// This class takes a url and then loads it async not blocking -// This also handles the UI +import { uiAccess } from "~/signals"; +import { SelectionManager } from "../SelectionManager"; +import Konva from "konva"; +import { Layer } from "konva/lib/Layer"; +import { v4 as uuidv4 } from "uuid"; + +const toolbarVideo = uiAccess.toolbarImage; +const loadingBar = uiAccess.loadingBar; export class NetworkedNodeContext { - constructor() {} + public selectionManagerRef: SelectionManager; + public kNode: Konva.Image | undefined; + protected mediaLayer: Layer; + public uuid: string; - async startLoading(): Promise {} - async endLoading(): Promise {} - async progressiveLoading(currentAmount: number): Promise {} - async updateCanvasElement() {} + // Internal State members + // do not modify internal + public didFinishLoading: boolean = false; + // This locks interaction when the render engine is rendering + protected isProcessing: boolean = false; - // Debug outputs - async blobToFile(blob: Blob) { - try { - const link = document.createElement("a"); - link.href = URL.createObjectURL(blob); - link.download = "canvas-output.jpg"; - // Trigger the download - link.click(); - // Clean up the URL object - URL.revokeObjectURL(link.href); - console.log("Downloaded Sample"); - } catch (error) { - console.log(error); + async setProcessing() { + this.isProcessing = true; + } + + constructor(selectionManagerRef: SelectionManager, mediaLayer: Layer) { + this.uuid = uuidv4(); + this.mediaLayer = mediaLayer; + this.selectionManagerRef = selectionManagerRef; + this.kNode = undefined; + this.didFinishLoading = false; + } + + public delete() { + // Do any other clean up and delete the konva node. + if (this.kNode) { + this.kNode.destroy(); } } + + public highlight() { + if (!this.kNode) { + return; + } + this.kNode.stroke("salmon"); + this.kNode.strokeWidth(10); + this.kNode.draw(); + } + + public unHighLight() { + if (!this.kNode) { + return; + } + + this.kNode.stroke(null); + this.kNode.strokeWidth(0); + this.kNode.draw(); + } + + updateContextMenuPosition() { + if (!this.kNode) { + return; + } + toolbarVideo.setPosition({ + x: this.kNode.getPosition().x + this.kNode.getSize().width / 4, + y: this.kNode.getPosition().y - 150, + }); + } + + updateLoadingBarPosition() { + if (!this.kNode) { + return; + } + loadingBar.updatePosition({ + x: this.kNode.getPosition().x + this.kNode.getSize().width / 4, + y: this.kNode.getPosition().y - 60, + }); + } + + public bringToFront() { + if (!this.kNode) { + return; + } + this.kNode.moveUp(); + } + + public sendBack() { + if (!this.kNode) { + return; + } + // prevent canvas being in front. + if (this.kNode.zIndex() === 1) { + return; + } + this.kNode.moveDown(); + } + + public async updateContextMenu() { + this.updateContextMenuPosition(); + this.updateLoadingBarPosition(); + } + + public listenToBaseKNode() { + if (!this.kNode) { + return; + } + + this.kNode.on("dragstart", (e) => { + // console.log("Drag start"); + + this.updateContextMenuPosition(); + + // Multiselect + const isMultiSelect = e.evt.shiftKey; + if (isMultiSelect) { + this.selectionManagerRef.startDrag(this); + } + if (this.didFinishLoading == false) { + this.updateLoadingBarPosition(); + } + }); + + this.kNode.on("dragmove", (e) => { + // shouldn't be able to move if processing. + // console.log("Drag Move"); + this.updateContextMenuPosition(); + + // Multiselect + const isMultiSelect = e.evt.shiftKey; + if (isMultiSelect) { + this.selectionManagerRef.dragging(this); + } + + if (this.isProcessing) { + return; + } + + if (this.didFinishLoading == false) { + this.updateLoadingBarPosition(); + } + }); + + this.kNode.on("dragend", (e) => { + // console.log("Drag End"); + + const isMultiSelect = e.evt.shiftKey; + if (isMultiSelect) { + this.selectionManagerRef.draggingStopped(this); + } + this.updateContextMenuPosition(); + if (this.didFinishLoading == false) { + this.updateLoadingBarPosition(); + } + }); + + this.kNode.on("click", (e) => { + console.log("click"); + const isMultiSelect = e.evt.shiftKey; + if (isMultiSelect) { + if (this.selectionManagerRef.isNodeSelected(this)) { + this.selectionManagerRef.deselectNode(this); + } else { + this.selectionManagerRef.selectNode(this); + } + } + }); + + this.kNode.on("mousedown", (e) => { + toolbarVideo.show(); + console.log("Mouse Down"); + + const isMultiSelect = e.evt.shiftKey; + + if (isMultiSelect) { + console.log("Shift"); + } else { + console.log("No Shift"); + this.selectionManagerRef.clearSelection(); + this.selectionManagerRef.selectNode(this); + } + + // selection on click doesn't do a good job. + this.updateContextMenuPosition(); + + if (this.didFinishLoading == false) { + this.updateLoadingBarPosition(); + } else { + loadingBar.hide(); + } + }); + + this.kNode.on("mouseup", (e) => { + if (this.didFinishLoading == false) { + return; + } + }); + } } diff --git a/src/KonvaApp/Nodes/VideoNode.ts b/src/KonvaApp/Nodes/VideoNode.ts index a1771ea0c0..4819ab47c4 100644 --- a/src/KonvaApp/Nodes/VideoNode.ts +++ b/src/KonvaApp/Nodes/VideoNode.ts @@ -1,66 +1,29 @@ import Konva from "konva"; import { Layer } from "konva/lib/Layer"; import { NetworkedNodeContext } from "./NetworkedNodeContext"; -import { v4 as uuidv4 } from "uuid"; - import { uiAccess } from "~/signals"; import { SelectionManager } from "../SelectionManager"; + const toolbarVideo = uiAccess.toolbarImage; const loadingBar = uiAccess.loadingBar; export class VideoNode extends NetworkedNodeContext { - public selectionManagerRef: SelectionManager | undefined; - public videoURL: string; public videoComponent: HTMLVideoElement; - public kNode: Konva.Image; - - // do not modify internal - public didFinishLoading: boolean; - private videoLayer: Layer; - - public uuid: string; - - private shouldPlay: boolean = true; - // Use Context Menu Item public duration: number = 0; private imageIndex: number = 0; - public fps: number = 24; - // This locks interaction when the render engine is rendering - private isProcessing: boolean = false; - async setProcessing() { this.isProcessing = true; } - private finishedLoadingOnStart: Promise; - - public delete() { - // Do any other clean up and delete the konva node. - if (this.kNode) { - this.kNode.destroy(); - } - } - getNumberFrames(): number { return this.fps * this.duration; } - public highlight() { - this.kNode.stroke("salmon"); - this.kNode.strokeWidth(12); - this.kNode.draw(); - } - public unHighLight() { - this.kNode.stroke(null); - this.kNode.strokeWidth(0); - this.kNode.draw(); - } - private imageSources: string[] = [ "https://images-ng.pixai.art/images/orig/7ef23baa-2fc8-4e2f-8299-4f9241920090", "https://images-ng.pixai.art/images/orig/98196e9f-b968-4fe1-97ec-083ffd77c263", @@ -71,34 +34,27 @@ export class VideoNode extends NetworkedNodeContext { "https://images-ng.pixai.art/images/orig/56dcbb5f-7a31-4328-b4ea-1312df6e77a0", "https://videos.pixai.art/f7df019d-79a2-4ed2-bb99-775c941f7ec6", ]; + private frameDidFinishSeeking: Promise; + private finishedLoadingOnStart: Promise; + constructor( - uuid: string = uuidv4(), videoLayer: Layer, x: number, y: number, videoURL: string, - selectionManagerRef: SelectionManager | undefined, - width: number | undefined, - height: number | undefined, + selectionManagerRef: SelectionManager, ) { - super(); - this.selectionManagerRef = selectionManagerRef; - this.shouldPlay = true; // start - this.uuid = uuid; - this.videoLayer = videoLayer; - // state manage the node + super(selectionManagerRef, videoLayer); + // state manage the node // use web codecs to get the frame rate 89% support // assume 60fps for now. + this.fps = 24; // need to query this from the media - - this.duration = -1; - - this.didFinishLoading = false; + this.duration = -1; // video duration this.videoURL = videoURL; - this.videoComponent = document.createElement("video"); this.videoComponent.crossOrigin = "anonymous"; @@ -107,33 +63,49 @@ export class VideoNode extends NetworkedNodeContext { this.finishedLoadingOnStart = new Promise(() => {}); this.kNode = new Konva.Image({ - image: this.videoComponent, + image: undefined, x: x, y: y, + width: 200, // to do fix this with placeholder + height: 200, draggable: true, + fill: "grey", }); + this.mediaLayer.add(this.kNode); - if (this.selectionManagerRef === undefined) { - this.kNode.listening(false); - } - + this.videoComponent.onloadstart = (event: Event) => { + loadingBar.show(); + loadingBar.updateProgress(0); + }; this.videoComponent.onloadedmetadata = (event: Event) => { + loadingBar.updateProgress(25); console.log("Loaded Metadata"); - if (height) { - this.kNode.height(height); - } else { - this.kNode.height(this.videoComponent.videoHeight); - } - if (width) { - this.kNode.width(width); - } else { - this.kNode.width(this.videoComponent.videoWidth); + if (!this.kNode) { + console.log("KNode Not Initialized Video Component"); + return; } + this.kNode.height(this.videoComponent.videoHeight); + this.kNode.width(this.videoComponent.videoWidth); + this.kNode.image(this.videoComponent); this.videoComponent.currentTime = 0; // ensure it shows up on screen // it might have length here which we will need to trim down to 7 seconds. console.log(`Video Duration: ${this.videoComponent.duration}`); this.duration = this.videoComponent.duration; + this.kNode.fill(null); + loadingBar.updateProgress(50); + }; + + this.videoComponent.onerror = ( + event: Event | string, + source?: string, + lineno?: number, + colno?: number, + error?: Error, + ) => { + loadingBar.hide(); + loadingBar.updateProgress(0); + console.error("Error loading video:", event); }; this.finishedLoadingOnStart = new Promise((resolve, reject) => { @@ -141,6 +113,8 @@ export class VideoNode extends NetworkedNodeContext { this.didFinishLoading = true; // Might have to auto click? on first load this doesn't work in general how about after ? this.videoComponent.play(); //sometimes race condition with the + loadingBar.updateProgress(100); + loadingBar.hide(); resolve(); }; }); @@ -148,129 +122,53 @@ export class VideoNode extends NetworkedNodeContext { // assign video to start process. this.videoComponent.src = this.videoURL; - this.videoLayer.add(this.kNode); - - this.kNode.on("dragstart", (e) => { - this.updateContextMenuPosition(); - // Multiselect - const isMultiSelect = e.evt.shiftKey; - if (this.selectionManagerRef) { - this.selectionManagerRef.startDrag(this); - } - if (this.didFinishLoading == false) { - this.updateLoadingBarPosition(); - } - }); - - this.kNode.on("dragmove", (e) => { - // shouldn't be able to move if processing. - - this.updateContextMenuPosition(); - // Multiselect - const isMultiSelect = e.evt.shiftKey; - if (this.selectionManagerRef) { - this.selectionManagerRef.dragging(this); - } - if (this.isProcessing) { - return; - } - - if (this.didFinishLoading == false) { - this.updateLoadingBarPosition(); - } - }); - - this.kNode.on("dragend", (e) => { - if (this.selectionManagerRef) { - this.selectionManagerRef.draggingStopped(this); - } - }); - - this.kNode.on("mouseover", () => { - //this.highlight(); - }); - - this.kNode.on("mouseout", () => { - // this.unHighLight(); - }); - - this.kNode.on("dragend", () => { - this.updateContextMenuPosition(); - if (this.didFinishLoading == false) { - this.updateLoadingBarPosition(); - } - }); - - this.kNode.on("mousedown", (e) => { - toolbarVideo.show(); - - // selection on click doesn't do a good job. - const isMultiSelect = e.evt.shiftKey; - if (this.selectionManagerRef) { - this.selectionManagerRef.selectNode(this, isMultiSelect); - } - - this.updateContextMenuPosition(); - - if (this.didFinishLoading == false) { - this.updateLoadingBarPosition(); - } else { - loadingBar.hide(); - } - }); - + this.listenToBaseKNode(); + // only for video this.kNode.on("mouseup", () => { + console.log("Mouse Up"); + // Shouldn't play if anything things are true if (this.didFinishLoading == false) { return; } - this.play(); + if (this.isProcessing == true) { + return; + } + + if (this.videoComponent.paused) { + console.log("Playing"); + this.videoComponent.play(); + } else { + console.log("Pause"); + this.videoComponent.pause(); + } }); } - stop() { - if (this.shouldPlay == false) { - this.shouldPlay = true; - this.videoComponent.pause(); - } + async reset() { + this.videoComponent.pause(); + await this.seek(0); } async updateImage(newImageSrc: string) { const newImage = new Image(); newImage.src = newImageSrc; newImage.onload = () => { + if (!this.kNode) { + console.log("selectedNode KNode is initialized"); + return; + } this.kNode.image(newImage); this.kNode.draw(); }; } - updateContextMenuPosition() { - toolbarVideo.setPosition({ - x: this.kNode.getPosition().x + this.kNode.getSize().width / 4, - y: this.kNode.getPosition().y - 150, - }); - } - - updateLoadingBarPosition() { - loadingBar.updatePosition({ - x: this.kNode.getPosition().x + this.kNode.getSize().width / 4, - y: this.kNode.getPosition().y - 60, - }); - } - - public bringToFront() { - this.kNode.moveUp(); - } - - public sendBack() { - // prevent canvas being in front. - if (this.kNode.zIndex() === 1) { - return; - } - this.kNode.moveDown(); - } - public async createVideoElement(newURL: string) { // try catch here with a retry button. + if (!this.kNode) { + console.log("selectedNode KNode is initialized"); + return; + } + this.kNode.fill("grey"); // todo fix with loader const videoComponent = document.createElement("video"); videoComponent.crossOrigin = "anonymous"; @@ -278,6 +176,7 @@ export class VideoNode extends NetworkedNodeContext { videoComponent.src = newURL; console.log("Video Data"); console.log(newURL); + videoComponent.onloadstart = (event: Event) => { this.didFinishLoading = false; console.log("OnLoadStart"); @@ -285,64 +184,21 @@ export class VideoNode extends NetworkedNodeContext { videoComponent.onloadedmetadata = (event: Event) => { console.log("Loaded Metadata"); + if (!this.kNode) { + console.log("selectedNode KNode is initialized"); + return; + } this.kNode.width(videoComponent.videoWidth); this.kNode.height(videoComponent.videoHeight); - + this.kNode.image(videoComponent); // it might have length here which we will need to trim down to 7 seconds. console.log(`Video Duration: ${videoComponent.duration}`); videoComponent.currentTime = 0; // ensure it shows up on screen - this.duration = videoComponent.duration; - this.videoComponent = videoComponent; }; - - // videoComponent.onloadeddata = (event: Event) => { - // try { - // console.log("LoadedData"); - - // this.kNode.image(videoComponent); - // this.kNode.draw(); - // this.videoComponent.loop = true; - // this.videoComponent.currentTime = 0; // ensure it shows up on screen - - // this.shouldPlay = true; // means its play state - // } catch (error) { - // console.log(error); - // } - // }; - - // videoComponent.oncanplaythrough = (event: Event) => { - // // remove loading ui - // loadingBar.updateProgress(100); - // loadingBar.hide(); - // this.didFinishLoading = true; - // // Might have to auto click? on first load this doesn't work in general how about after ? - // //this.play(); //sometimes race condition with the - // }; } - private play() { - try { - if (this.didFinishLoading === false || this.isProcessing) { - return; - } - - //console.log(`${this.didFinishLoading} ${this.shouldPlay}`); - if (this.didFinishLoading && this.shouldPlay === true) { - //console.log("Playing"); - this.shouldPlay = false; - this.videoComponent.play(); - } else { - this.shouldPlay = true; - this.videoComponent.pause(); - - //console.log("Pausing"); - } - } catch (error) { - //console.log(error); - } - } async simulatedLoading() { // need to block playing while loading this.didFinishLoading = false; @@ -403,7 +259,6 @@ export class VideoNode extends NetworkedNodeContext { resolve(); }; }); - // await await this.frameDidFinishSeeking; } else { console.log("Video Not Seekable"); diff --git a/src/KonvaApp/Onnx/onnx.ts b/src/KonvaApp/Onnx/onnx.ts new file mode 100644 index 0000000000..c03e880fbb --- /dev/null +++ b/src/KonvaApp/Onnx/onnx.ts @@ -0,0 +1,53 @@ +// public async onnx() { +// try { +// //DO NOT REMOVE, NECESSARY TO LOAD WASM FILES +// ort.env.wasm.wasmPaths = "wasm/"; + +// // Load the model and create InferenceSession +// const modelPathE = "/models/image_encoder_hiera_t.onnx"; +// const modelPath = "/models/mask_decoder_hiera_t.onnx"; +// const modelPath1 = "/models/memory_attention_hiera_t.onnx"; +// const modelPath2 = "/models/memory_encoder_hiera_t.onnx"; +// const modelPath3 = "/models/mlp_hiera_t.onnx"; +// const modelPath4 = "/models/prompt_encoder_hiera_t.onnx"; + +// const mask_decoder_hiera_t = await ort.InferenceSession.create( +// modelPath, +// { +// executionProviders: ["wasm"], +// }, +// ); +// console.log(mask_decoder_hiera_t); +// const memory_attention_hiera_t = await ort.InferenceSession.create( +// modelPath1, +// { +// executionProviders: ["wasm"], +// }, +// ); +// console.log(memory_attention_hiera_t); +// const memory_encoder_hiera_t = await ort.InferenceSession.create( +// modelPath2, +// { +// executionProviders: ["wasm"], +// }, +// ); +// console.log(memory_encoder_hiera_t); +// const mlp_hiera_t = await ort.InferenceSession.create(modelPath3, { +// executionProviders: ["wasm"], +// }); +// console.log(mlp_hiera_t); +// const prompt_encoder_hiera_t = await ort.InferenceSession.create( +// modelPath4, +// { +// executionProviders: ["wasm"], +// }, +// ); +// console.log(prompt_encoder_hiera_t); + +// // // Run inference +// // const outputs = await session.run({ input: inputTensor }); +// // console.log(outputs); +// } catch (err) { +// console.error("error caught: ", err); +// } +// } REFERENCE CODE. diff --git a/src/KonvaApp/RenderingPrimitives/OffScreenSceneCanvas.ts b/src/KonvaApp/RenderingPrimitives/OffScreenSceneCanvas.ts new file mode 100644 index 0000000000..c104666e2a --- /dev/null +++ b/src/KonvaApp/RenderingPrimitives/OffScreenSceneCanvas.ts @@ -0,0 +1,54 @@ +import { ICanvasConfig } from "konva/lib/Canvas"; // have to export and modify the code TODO:/ +import { Canvas } from "konva/lib/Canvas"; +import { OffScreenSceneContext } from "./OffScreenSceneContext"; + +export class OffScreenSceneCanvas extends Canvas { + constructor( + config: ICanvasConfig = { width: 0, height: 0, willReadFrequently: false }, + ) { + super(config); // creates canvas then you have to replace it + // replace here also kills the css but thats ok + this._canvas = new OffscreenCanvas(config.width, config.height); + + this.context = new OffScreenSceneContext(this, { + willReadFrequently: config.willReadFrequently, + }); + this.setSize(config.width, config.height); + } + + setPixelRatio(pixelRatio: number) { + console.log("overriden setPixelRatio"); + var previousRatio = this.pixelRatio; + this.pixelRatio = pixelRatio; + this.setSize( + this.getWidth() / previousRatio, + this.getHeight() / previousRatio, + ); + } + public override setWidth(width: number) { + // take into account pixel ratio + console.log("overriden setWidth"); + this.width = this._canvas.width = width * this.pixelRatio; + var pixelRatio = this.pixelRatio, + _context = this.getContext()._context; + _context.scale(pixelRatio, pixelRatio); + } + public override setHeight(height: number) { + // take into account pixel ratio + console.log("overriden setHeight"); + this.height = this._canvas.height = height * this.pixelRatio; + var pixelRatio = this.pixelRatio, + _context = this.getContext()._context; + _context.scale(pixelRatio, pixelRatio); + } + getWidth() { + return this.width; + } + getHeight() { + return this.height; + } + setSize(width: number | undefined, height: number | undefined) { + this.setWidth(width || 0); + this.setHeight(height || 0); + } +} diff --git a/src/KonvaApp/RenderingPrimitives/OffScreenSceneContext.ts b/src/KonvaApp/RenderingPrimitives/OffScreenSceneContext.ts new file mode 100644 index 0000000000..b01064dacd --- /dev/null +++ b/src/KonvaApp/RenderingPrimitives/OffScreenSceneContext.ts @@ -0,0 +1,11 @@ +import { Context } from "konva/lib/Context"; +import { Canvas } from "konva/lib/Canvas"; + +export class OffScreenSceneContext extends Context { + constructor(canvas: Canvas, { willReadFrequently = false } = {}) { + super(canvas); + this._context = canvas._canvas.getContext("2d", { + willReadFrequently, + }) as CanvasRenderingContext2D; + } +} diff --git a/src/KonvaApp/RenderingPrimitives/RenderEngine.ts b/src/KonvaApp/RenderingPrimitives/RenderEngine.ts index 4262679c33..6b51528a83 100644 --- a/src/KonvaApp/RenderingPrimitives/RenderEngine.ts +++ b/src/KonvaApp/RenderingPrimitives/RenderEngine.ts @@ -1,7 +1,9 @@ import { VideoNode } from "../Nodes/VideoNode"; import Konva from "konva"; import { RenderTask } from "./RenderTask"; - +import { Container } from "konva/lib/Container"; +import { Shape } from "konva/lib/Shape"; +import { Group } from "konva/lib/Group"; // Hide these two to start. WIL // uiAccess.toolbarMain.loadingBar.hide(); // uiEvents.toolbarMain.loadingBarRetry.onClick((e) => { @@ -25,12 +27,15 @@ import { } from "../SharedWorkers/Diffusion/DiffusionSharedWorker"; import { RenderingOptions } from "../Engine"; import { ImageNode } from "../Nodes/ImageNode"; +import { FileUtilities } from "../FileUtilities/FileUtilities"; +import { OffScreenSceneCanvas } from "./OffScreenSceneCanvas"; // https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio export class RenderEngine { private videoNodes: VideoNode[]; private imageNodes: ImageNode[]; + private offScreenCanvas: OffscreenCanvas; private context: OffscreenCanvasRenderingContext2D | null; @@ -83,6 +88,7 @@ export class RenderEngine { if (height) { this.height = height; } + // Ensures that all the nodes stag in the same place should // there be a window resize. // recompute the position @@ -132,6 +138,12 @@ export class RenderEngine { y: pos.y + deltaY, }); } + + // update the context menu + this.videoNodes.forEach((node) => { + // find selected node... TODO and update that position + node.updateContextMenu(); + }); this.videoLayer.batchDraw(); } constructor( @@ -217,8 +229,8 @@ export class RenderEngine { if (typeof SharedWorker !== "undefined") { console.log("Shared Workers are supported in this browser."); // Debug chrome://inspect/#workers + // "src\\KonvaApp\\SharedWorkers\\Diffusion\\DiffusionSharedWorker.ts", this.diffusionWorker = new DiffusionSharedWorkerClient( - "src\\KonvaApp\\SharedWorkers\\Diffusion\\DiffusionSharedWorker.ts", this.onRenderingSystemMessageRecieved, ); @@ -230,6 +242,26 @@ export class RenderEngine { } } + public placeDebugRect( + x: number, + y: number, + width: number, + height: number, + layer: Konva.Layer, + ): void { + const rect = new Konva.Rect({ + x: x, + y: y, + width: width, // Default width + height: height, // Default height + fill: "green", + draggable: false, + }); + + layer.add(rect); + layer.draw(); + } + // This function uses a portion of the video layer to capture just the capture canvas. // capture everything after seeking each video node. renderPortionOfLayer( @@ -291,7 +323,11 @@ export class RenderEngine { var failed = false; for (let i = 0; i < this.videoNodes.length; i++) { const item = this.videoNodes[i]; + if (!item.kNode) { + return; + } item.kNode.listening(false); + item.unHighLight(); if (item.didFinishLoading == false) { // error out and show error message //this.startProcessing(); @@ -309,7 +345,7 @@ export class RenderEngine { throw Error("Wait For Items to Finish Processing."); } - this.videoNodes.forEach((item: VideoNode) => {}); + //this.videoNodes.forEach((item: VideoNode) => {}); // find the longest video node let numberOfFrames = this.findLongestVideoLength(); @@ -326,6 +362,9 @@ export class RenderEngine { // enable all nodes again. for (let i = 0; i < this.videoNodes.length; i++) { const item = this.videoNodes[i]; + if (!item.kNode) { + return; + } item.kNode.listening(true); } } @@ -340,6 +379,79 @@ export class RenderEngine { return frameNumber / frameRate; } + /** + * + * @param config + */ + private async renderFrame(config: { + layerOfInterest: Konva.Layer; // layer where the element that you want to clip lives. + // XY and height of a captureCanvas ( region of interest ) + x?: number; // x position of the region of interest + y?: number; // y position of the region of interest + width?: number; // size of the region of interest + height?: number; // size of the region of interest + pixelRatio?: number; // higher means higher quality + mimeType?: string; // image/jpeg or image/png + quality?: number; // 1.0 is the best. + test: boolean; // true == blob else Image Bitmap + }): Promise { + try { + const box = config.layerOfInterest.getClientRect(); + const stage = config.layerOfInterest.getStage(); + + const x = config.x !== undefined ? config.x : Math.floor(box.x); + const y = config.y !== undefined ? config.y : Math.floor(box.y); + const pixelRatio = config.pixelRatio || 1; + + const container = config.layerOfInterest as Container; + + const offScreenSceneCanvas = new OffScreenSceneCanvas({ + width: + config.width || Math.ceil(box.width) || (stage ? stage.width() : 0), + height: + config.height || + Math.ceil(box.height) || + (stage ? stage.height() : 0), + pixelRatio: pixelRatio, + }); + + const context = offScreenSceneCanvas.getContext(); + + const buffer = new OffScreenSceneCanvas({ + width: + offScreenSceneCanvas.width / offScreenSceneCanvas.pixelRatio + + Math.abs(x), + height: + offScreenSceneCanvas.height / offScreenSceneCanvas.pixelRatio + + Math.abs(y), + pixelRatio: offScreenSceneCanvas.pixelRatio, + }); + + context.save(); + + if (x || y) { + context.translate(-1 * x, -1 * y); + } + container.drawScene(offScreenSceneCanvas, undefined, buffer); + // Not a type mistake ... DO NOT FIX + const offscreenCanvas = offScreenSceneCanvas._canvas as OffscreenCanvas; + let result = undefined; + if (config.test) { + const blob = await offscreenCanvas.convertToBlob({ + quality: config.quality ?? 1.0, + type: "image/jpeg", + }); + await FileUtilities.blobToFileJpeg(blob, "1"); + result = blob; + } else { + result = offscreenCanvas.transferToImageBitmap(); + } + context.restore(); + return result; + } catch (error) { + throw error; + } + } /** find longest video then seek through each node 1 step. @@ -353,13 +465,12 @@ export class RenderEngine { // Stop all nodes first console.log(`LargestNumberOfFrames:${largestNumberOfFrames}`); - for (let k = 0; k < this.videoNodes.length; k++) { const videoNode = this.videoNodes[k]; if (videoNode.didFinishLoading === false) { throw Error("Videos Did Not Finish Loading Please Try Again."); } - videoNode.stop(); + await videoNode.reset(); } // only pick nodes that intersect with the canvas on screen bounds to freeze. @@ -401,14 +512,14 @@ export class RenderEngine { this.width, this.height, ); - // TODO write the non webworker version + //TODO write the non webworker version + const blob = await this.offScreenCanvas.convertToBlob({ + quality: 1.0, + type: "image/jpeg", + }); + await FileUtilities.blobToFileJpeg(blob, "1"); - // const blob = await this.offScreenCanvas.convertToBlob({ - // quality: 1.0, - // type: "image/jpeg", - // }); - - //await this.blobToFile(blob, `${j}`); + break; } // end of for each frame } else { // decode on the shared webworker. @@ -422,44 +533,32 @@ export class RenderEngine { return; } - this.context.drawImage( - this.videoLayer.canvas._canvas, - this.positionX, - this.positionY, - this.width, - this.height, - 0, - 0, - this.width, - this.height, + console.log( + `context: x:${this.positionX} y:${this.positionY} ${this.width} x ${this.height}`, ); + const bitmap = await this.renderFrame({ + layerOfInterest: this.videoLayer, + x: this.captureCanvas.x(), + y: this.captureCanvas.y(), + width: this.width, + height: this.height, + mimeType: "image/jpeg", + pixelRatio: 1, + quality: 1.0, + test: false, + }); + const data: DiffusionSharedWorkerItemData = { height: this.height, width: this.width, - imageBitmap: this.offScreenCanvas.transferToImageBitmap(), + imageBitmap: bitmap as ImageBitmap, frame: j, totalFrames: largestNumberOfFrames, prompt: renderingOptions, }; - - // let isDoneStreaming = false; - console.log(`Processing Frame:${j} out of ${largestNumberOfFrames}`); - - // if (j == Math.floor(largestNumberOfFrames)) { - // isDoneStreaming = true; - // } - - //console.log(isDoneStreaming); - this.diffusionWorker.sendData(1, data, false); - - // // To ensure there is no lingering frames. - // if (isDoneStreaming) { - // console.log("Done streaming"); - // break; - // } } } // end of largest number of frames loop @@ -468,8 +567,10 @@ export class RenderEngine { console.log("Didn't Initialize Diffusion"); return; } + + const jobID = 1; this.diffusionWorker.sendData( - 1, + jobID, { height: this.height, width: this.width, diff --git a/src/KonvaApp/SelectionManager.ts b/src/KonvaApp/SelectionManager.ts index 3c08677869..fb0c0bf689 100644 --- a/src/KonvaApp/SelectionManager.ts +++ b/src/KonvaApp/SelectionManager.ts @@ -1,7 +1,8 @@ import Konva from "konva"; import { VideoNode } from "./Nodes/VideoNode"; import { ImageNode } from "./Nodes/ImageNode"; -type MediaNode = VideoNode | ImageNode; +import { NetworkedNodeContext } from "./Nodes/NetworkedNodeContext"; +type MediaNode = NetworkedNodeContext | VideoNode | ImageNode; export class SelectionManager { private selectedNodes: Set; private initialPositions: Map; @@ -11,18 +12,18 @@ export class SelectionManager { this.initialPositions = new Map(); } - public selectNode(node: MediaNode, isMultiSelect: boolean): void { - if (!isMultiSelect) { - this.clearSelection(); - } - + public selectNode(node: MediaNode): void { if (this.selectedNodes.has(node)) { - this.deselectNode(node); + return; } else { this.selectedNodes.add(node); node.highlight(); } + if (!node.kNode) { + console.log("KNode is initialized"); + return; + } node.kNode.getLayer()?.batchDraw(); console.log("Selected Nodes:"); @@ -31,6 +32,11 @@ export class SelectionManager { public deselectNode(node: MediaNode): void { this.selectedNodes.delete(node); + node.unHighLight(); + if (!node.kNode) { + console.log("KNode is initialized"); + return; + } node.kNode.getLayer()?.batchDraw(); } @@ -44,19 +50,27 @@ export class SelectionManager { } // Start State - public startDrag(node: MediaNode | ImageNode): void { + public startDrag(node: MediaNode): void { console.log("Starting Drag"); - - const position = node.kNode.position(); + if (!node.kNode) { + console.log("KNode is initialized"); + return; + } this.selectedNodes.forEach((selectedNode) => { + if (!selectedNode.kNode) { + console.log("selectedNode KNode is initialized"); + return; + } this.initialPositions.set(selectedNode, { x: selectedNode.kNode.x(), y: selectedNode.kNode.y(), }); }); } - + public isNodeSelected(node: MediaNode): boolean { + return this.selectedNodes.has(node); + } // Drag State public dragging(node: MediaNode): void { const initialPosition = this.initialPositions.get(node); @@ -64,7 +78,10 @@ export class SelectionManager { console.log("Initial Position Undefined"); return; } - + if (!node.kNode) { + console.log("selectedNode KNode is initialized"); + return; + } const dx = node.kNode.x() - initialPosition.x; const dy = node.kNode.y() - initialPosition.y; @@ -77,6 +94,11 @@ export class SelectionManager { console.log("Initial Position is Null"); return; } + + if (!selectedNode.kNode) { + console.log("selectedNode KNode is initialized"); + return; + } selectedNode.kNode.x(initialPosition.x + dx); selectedNode.kNode.y(initialPosition.y + dy); }); diff --git a/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorker.ts b/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorker.ts index d48beef67a..119197ccd8 100644 --- a/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorker.ts +++ b/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorker.ts @@ -24,6 +24,7 @@ export interface DiffusionSharedWorkerProgressData { url: string; status: JobStatus; progress: number; + mediaToken: string; } export interface DiffusionSharedWorkerItemData { @@ -137,7 +138,9 @@ export class DiffusionSharedWorker extends SharedWorkerBase< url: "", status: JobStatus.PENDING, progress: (item.frame / aproxSteps / 2 / totalPercent) * 100, + mediaToken: "", }; + console.log("Lets report progress."); reportProgress(progressData); // once finished gives you up to 50% @@ -150,6 +153,17 @@ export class DiffusionSharedWorker extends SharedWorkerBase< console.log(zipBlob); console.log("Zipped"); + + const progressMedia: DiffusionSharedWorkerProgressData = { + url: "", + status: JobStatus.PENDING, + progress: (item.frame / aproxSteps / 2 / totalPercent) * 100, + mediaToken: "", + }; + + console.log("Lets report progress."); + reportProgress(progressMedia); // once finished gives you up to 50% + const response = await this.mediaAPI.UploadStudioShot({ maybe_title: "", uuid_idempotency_token: uuidv4(), @@ -222,6 +236,7 @@ export class DiffusionSharedWorker extends SharedWorkerBase< url: "", status: JobStatus.DEAD, progress: computedProgress / totalPercent, + mediaToken: mediaToken, }; renderProgressData.status = status; @@ -254,7 +269,6 @@ export class DiffusionSharedWorker extends SharedWorkerBase< jobIsProcessing = false; reportProgress(renderProgressData); throw Error("Server Failed to Process Please Try Again."); - break; case JobStatus.CANCCELLED_BY_SYSTEM: jobIsProcessing = false; reportProgress(renderProgressData); diff --git a/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorkerClient.ts b/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorkerClient.ts index f2d1eccb02..65ddbd3c90 100644 --- a/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorkerClient.ts +++ b/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorkerClient.ts @@ -19,7 +19,6 @@ export class DiffusionSharedWorkerClient< >, ) => void; constructor( - workerPath: string, messageReceived: ( response: SharedWorkerResponse< DiffusionSharedWorkerResponseData, @@ -28,43 +27,25 @@ export class DiffusionSharedWorkerClient< ) => void, ) { this.messageReceived = messageReceived; - // if (import.meta.env.DEV) { - // console.log("This is running a worker in development"); - // this.sharedWorker = new SharedWorker(workerPath, { - // type: "module", - // }); - // } else { try { console.log("This is running a worker in production"); this.sharedWorker = new DiffusionSharedWorker(); - // Uncomment this to test your CORS - // this.sharedWorker = new SharedWorker("worker.js", { - // type: "module", - // }); - this.sharedWorker.addEventListener("error", (value) => { console.log("Shared worker ERROR:"); console.log(value); }); + this.port = this.sharedWorker.port; + this.port.onmessage = this.onMessage.bind(this); + this.port.start(); + console.log("launched shared worker (?)"); } catch (error) { console.log("ERROR with shared worker!"); console.log(error); + throw Error("Could Not Start Worker"); } - //} - // // in production this is a work around .. https://github.com/vitejs/vite/issues/13680 - // const js = `import ${JSON.stringify(new URL(diffusionWorkerURL, import.meta.url))}`; - // const blob = new Blob([js], { type: "application/javascript" }); - // const objURL = URL.createObjectURL(blob); - // this.sharedWorker = new SharedWorker(new URL(objURL, import.meta.url), { - // type: "module", - // }); - //} - this.port = this.sharedWorker.port; - this.port.onmessage = this.onMessage.bind(this); - this.port.start(); } async onMessage(event: MessageEvent) { @@ -91,7 +72,7 @@ export class DiffusionSharedWorkerClient< async sendData( jobID: number, - data: DiffusionSharedWorkerItemData | undefined, + data: DiffusionSharedWorkerItemData, isDoneStreaming: boolean, ) { const payload: SharedWorkerRequest = { diff --git a/src/KonvaApp/UndoRedo/MoveLayerDown.ts b/src/KonvaApp/UndoRedo/MoveLayerDown.ts new file mode 100644 index 0000000000..bc61d532e8 --- /dev/null +++ b/src/KonvaApp/UndoRedo/MoveLayerDown.ts @@ -0,0 +1,22 @@ +import { ICommand } from "./ICommand"; +import Konva from "konva"; + +export class MoveLayerDown implements ICommand { + private nodes: Konva.Node[]; + private layer: Konva.Layer; + + constructor(nodes: Konva.Node[]) { + this.nodes = nodes; + this.layer = nodes[0].getLayer()!; + } + + execute() { + // this.nodes.forEach((node) => this.layer.add(node)); + this.layer.draw(); + } + + undo() { + // this.nodes.forEach((node) => node.remove()); + this.layer.draw(); + } +} diff --git a/src/KonvaApp/UndoRedo/MoveLayerUp.ts b/src/KonvaApp/UndoRedo/MoveLayerUp.ts new file mode 100644 index 0000000000..580361bf31 --- /dev/null +++ b/src/KonvaApp/UndoRedo/MoveLayerUp.ts @@ -0,0 +1,22 @@ +import { ICommand } from "./ICommand"; +import Konva from "konva"; + +export class MoveLayerUp implements ICommand { + private nodes: Konva.Node[]; + private layer: Konva.Layer; + + constructor(nodes: Konva.Node[]) { + this.nodes = nodes; + this.layer = nodes[0].getLayer()!; + } + + execute() { + // this.nodes.forEach((node) => this.layer.add(node)); + this.layer.draw(); + } + + undo() { + // this.nodes.forEach((node) => node.remove()); + this.layer.draw(); + } +}