From 2f04b5d90adfea6b9f97d1a98db3c42463fabdbf Mon Sep 17 00:00:00 2001 From: wil Date: Wed, 23 Oct 2024 14:10:07 -0400 Subject: [PATCH] hotfix for chroma going blank --- src/KonvaApp/Nodes/ChromaWorker.ts | 85 +++++++++++++---------- src/KonvaApp/Nodes/VideoNode/VideoNode.ts | 40 ++++++++--- 2 files changed, 78 insertions(+), 47 deletions(-) diff --git a/src/KonvaApp/Nodes/ChromaWorker.ts b/src/KonvaApp/Nodes/ChromaWorker.ts index dc1453735d..b9b1dd2f38 100644 --- a/src/KonvaApp/Nodes/ChromaWorker.ts +++ b/src/KonvaApp/Nodes/ChromaWorker.ts @@ -1,51 +1,62 @@ - - // Handle incoming connections +// @ts-ignore self.onconnect = (event: MessageEvent) => { - const port = event.ports[0]; - // Listen for messages from connected ports - port.onmessage = (event: MessageEvent) => { - const jsonObject = event.data; + const port = event.ports[0]; + // Listen for messages from connected ports + port.onmessage = (event: MessageEvent) => { + console.log("recevied message"); + const jsonObject = event.data; - const chromaColor = jsonObject["color"] + const chromaColor = jsonObject["color"]; - const dataTransfer = jsonObject["dataTransfer"] - const offscreenCanvas = new OffscreenCanvas(dataTransfer.width, dataTransfer.height); + const dataTransfer = jsonObject["dataTransfer"]; + const offscreenCanvas = new OffscreenCanvas( + dataTransfer.width, + dataTransfer.height, + ); - const ctx = offscreenCanvas.getContext('2d'); + const ctx = offscreenCanvas.getContext("2d"); - if(ctx === null) { - console.error("Context is null in worker!") - return; - } + if (ctx === null) { + console.error("Context is null in worker!"); + return; + } - // Draw the ImageBitmap onto the offscreen canvas - ctx.drawImage(dataTransfer, 0, 0); + // Draw the ImageBitmap onto the offscreen canvas + ctx.drawImage(dataTransfer, 0, 0); - // Extract the pixel data from the canvas - const imageData = ctx.getImageData(0, 0, dataTransfer.width, dataTransfer.height); - const data = imageData.data; + // Extract the pixel data from the canvas + const imageData = ctx.getImageData( + 0, + 0, + dataTransfer.width, + dataTransfer.height, + ); + const data = imageData.data; - // Process the pixel data - for (let i = 0; i < data.length; i += 4) { - let red = data[i]; - let green = data[i + 1]; - let blue = data[i + 2]; + // Process the pixel data + for (let i = 0; i < data.length; i += 4) { + let red = data[i]; + let green = data[i + 1]; + let blue = data[i + 2]; - if (green > chromaColor["green"] && red < chromaColor["red"] && blue < chromaColor["blue"]) { - data[i + 3] = 0; // Set alpha to 0, effectively making it transparent - } - } + if ( + green > chromaColor["green"] && + red < chromaColor["red"] && + blue < chromaColor["blue"] + ) { + data[i + 3] = 0; // Set alpha to 0, effectively making it transparent + } + } - // Update the canvas with the modified pixel data - ctx.putImageData(imageData, 0, 0); + // Update the canvas with the modified pixel data + // console.log(URL.createObjectURL()); + ctx.putImageData(imageData, 0, 0); - // (Optional) Convert the canvas back to an ImageBitmap if needed - // const modifiedImageBitmap = offscreenCanvas.transferToImageBitmap(); - port.postMessage({ imageData: imageData }); + // (Optional) Convert the canvas back to an ImageBitmap if needed + // const modifiedImageBitmap = offscreenCanvas.transferToImageBitmap(); + port.postMessage({ imageData: imageData }); + }; - }; - - port.start(); // Start the port for communication + port.start(); // Start the port for communication }; - diff --git a/src/KonvaApp/Nodes/VideoNode/VideoNode.ts b/src/KonvaApp/Nodes/VideoNode/VideoNode.ts index 45d4848a26..c1e11c5a2b 100644 --- a/src/KonvaApp/Nodes/VideoNode/VideoNode.ts +++ b/src/KonvaApp/Nodes/VideoNode/VideoNode.ts @@ -137,7 +137,7 @@ export class VideoNode extends NetworkedNode { blue: 150, green: 120, }; - + this.createChromaWorker(); this.loadVideoFromUrl({ videoUrl: this.extractionUrl ?? this.mediaFileUrl, // videoUrl: "", // for debug loading video @@ -147,8 +147,6 @@ export class VideoNode extends NetworkedNode { }); this.listenToBaseKNode(); - - this.createChromaWorker(); } private listenToVideoPlayPause() { // TODO: for controling video playpause, can be improved with bette ui @@ -223,7 +221,7 @@ export class VideoNode extends NetworkedNode { this.videoComponent.onloadstart = () => { this.setProgress(25, { newStatus: UploadStatus.LOADING }); }; - this.videoComponent.onloadedmetadata = () => { + this.videoComponent.onloadedmetadata = async () => { this.setProgress(50, { newStatus: UploadStatus.LOADING }); console.log("Loaded Metadata"); this.mediaFileSize = { @@ -250,9 +248,16 @@ export class VideoNode extends NetworkedNode { "Replace Loading video with mediafile", this.videoComponent.src, ); - this.kNode.image(this.videoComponent); + this.duration = this.videoComponent.duration; this.videoComponent.currentTime = 0; // ensure it shows up on screen + await setTimeout(() => { + this.setChroma(this.isChroma); + // set chroma will do + // this.kNode.image(this.videoComponent); + // or + // this.kNode.image(this.videoCanvas); + }, 100); this.listenToVideoPlayPause(); this.kNode.fill(transparent); this.setProgress(75, { newStatus: UploadStatus.LOADING }); @@ -260,10 +265,10 @@ export class VideoNode extends NetworkedNode { } //can play through this.finishedLoadingOnStart = new Promise((resolve, reject) => { - this.videoComponent.oncanplaythrough = () => { + this.videoComponent.oncanplaythrough = async () => { + console.log("Can play through the Mediafile:", this.videoComponent.src); this.didFinishLoading = true; this.setProgress(100, { newStatus: UploadStatus.SUCCESS }); - this.setChroma(this.isChroma); resolve(); }; @@ -306,9 +311,10 @@ export class VideoNode extends NetworkedNode { this.isChroma = isChroma; if (this.isChroma === false) { - this.kNode?.image(this.videoComponent); + this.kNode.image(this.videoComponent); } else { - this.kNode?.image(this.videoCanvas); + console.log("set video to use its videocanvas counterpart"); + this.kNode.image(this.videoCanvas); if (this.videoComponent.paused || this.videoComponent.ended) { this.chromaKeyRender(0, false, false, true); } else { @@ -363,7 +369,6 @@ export class VideoNode extends NetworkedNode { ); const dataTransfer = this.drawingCanvas.transferToImageBitmap(); - await this.waitForWorkerResponse(dataTransfer, blockSeeking); } else { console.error("Context does not exist!"); @@ -375,6 +380,21 @@ export class VideoNode extends NetworkedNode { if (doLoop) requestAnimationFrame(this.chromaKeyRender.bind(this)); } + downloadOffscreenCanvas(canvas: OffscreenCanvas) { + canvas.convertToBlob().then((blob) => { + if (blob) { + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = "offscreen-canvas-image.png"; // Specify the file name + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); // Clean up + } + }); + } + // Method to post the message and wait for the response async waitForWorkerResponse( dataTransfer: ImageBitmap,