From 4deea53182e49159a45ed38f673cbceb3b20683b Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Thu, 13 Feb 2025 23:08:09 -0500 Subject: [PATCH] Blocking the updates so that it doesn't spam the server. --- frontend/realtime/package-lock.json | 221 ++++++++++++++++++ frontend/realtime/package.json | 1 + .../RenderingPrimitives/RealTimeDrawEngine.ts | 88 ++++++- frontend/realtime/yarn.lock | 26 +++ 4 files changed, 324 insertions(+), 12 deletions(-) diff --git a/frontend/realtime/package-lock.json b/frontend/realtime/package-lock.json index 6e412b3357..fcfb94d827 100644 --- a/frontend/realtime/package-lock.json +++ b/frontend/realtime/package-lock.json @@ -11,6 +11,7 @@ "@fortawesome/free-brands-svg-icons": "^6.6.0", "@headlessui/react": "^2.1.3", "@preact/signals-react": "^2.2.0", + "@tauri-apps/api": "^2.2.0", "@techstark/opencv-js": "^4.10.0-release.1", "@zip.js/zip.js": "^2.7.52", "konva": "^9.3.14", @@ -32,6 +33,7 @@ "@fortawesome/pro-solid-svg-icons": "^6.6.0", "@fortawesome/react-fontawesome": "^0.2.2", "@preact/signals-react-transform": "^0.4.0", + "@tauri-apps/cli": "^2.2.7", "@types/deep-equal": "^1.0.4", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", @@ -1229,6 +1231,215 @@ "url": "https://github.com/sponsors/tannerlinsley" } }, + "node_modules/@tauri-apps/api": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@tauri-apps/api/-/api-2.2.0.tgz", + "integrity": "sha512-R8epOeZl1eJEl603aUMIGb4RXlhPjpgxbGVEaqY+0G5JG9vzV/clNlzTeqc+NLYXVqXcn8mb4c5b9pJIUDEyAg==", + "license": "Apache-2.0 OR MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/tauri" + } + }, + "node_modules/@tauri-apps/cli": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/@tauri-apps/cli/-/cli-2.2.7.tgz", + "integrity": "sha512-ZnsS2B4BplwXP37celanNANiIy8TCYhvg5RT09n72uR/o+navFZtGpFSqljV8fy1Y4ixIPds8FrGSXJCN2BerA==", + "dev": true, + "license": "Apache-2.0 OR MIT", + "bin": { + "tauri": "tauri.js" + }, + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/tauri" + }, + "optionalDependencies": { + "@tauri-apps/cli-darwin-arm64": "2.2.7", + "@tauri-apps/cli-darwin-x64": "2.2.7", + "@tauri-apps/cli-linux-arm-gnueabihf": "2.2.7", + "@tauri-apps/cli-linux-arm64-gnu": "2.2.7", + "@tauri-apps/cli-linux-arm64-musl": "2.2.7", + "@tauri-apps/cli-linux-x64-gnu": "2.2.7", + "@tauri-apps/cli-linux-x64-musl": "2.2.7", + "@tauri-apps/cli-win32-arm64-msvc": "2.2.7", + "@tauri-apps/cli-win32-ia32-msvc": "2.2.7", + "@tauri-apps/cli-win32-x64-msvc": "2.2.7" + } + }, + "node_modules/@tauri-apps/cli-darwin-arm64": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/@tauri-apps/cli-darwin-arm64/-/cli-darwin-arm64-2.2.7.tgz", + "integrity": "sha512-54kcpxZ3X1Rq+pPTzk3iIcjEVY4yv493uRx/80rLoAA95vAC0c//31Whz75UVddDjJfZvXlXZ3uSZ+bnCOnt0A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 OR MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tauri-apps/cli-darwin-x64": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/@tauri-apps/cli-darwin-x64/-/cli-darwin-x64-2.2.7.tgz", + "integrity": "sha512-Vgu2XtBWemLnarB+6LqQeLanDlRj7CeFN//H8bVVdjbNzxcSxsvbLYMBP8+3boa7eBnjDrqMImRySSgL6IrwTw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 OR MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tauri-apps/cli-linux-arm-gnueabihf": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/@tauri-apps/cli-linux-arm-gnueabihf/-/cli-linux-arm-gnueabihf-2.2.7.tgz", + "integrity": "sha512-+Clha2iQAiK9zoY/KKW0KLHkR0k36O78YLx5Sl98tWkwI3OBZFg5H5WT1plH/4sbZIS2aLFN6dw58/JlY9Bu/g==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0 OR MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tauri-apps/cli-linux-arm64-gnu": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/@tauri-apps/cli-linux-arm64-gnu/-/cli-linux-arm64-gnu-2.2.7.tgz", + "integrity": "sha512-Z/Lp4SQe6BUEOays9BQAEum2pvZF4w9igyXijP+WbkOejZx4cDvarFJ5qXrqSLmBh7vxrdZcLwoLk9U//+yQrg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 OR MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tauri-apps/cli-linux-arm64-musl": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/@tauri-apps/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.2.7.tgz", + "integrity": "sha512-+8HZ+txff/Y3YjAh80XcLXcX8kpGXVdr1P8AfjLHxHdS6QD4Md+acSxGTTNbplmHuBaSHJvuTvZf9tU1eDCTDg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 OR MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tauri-apps/cli-linux-x64-gnu": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/@tauri-apps/cli-linux-x64-gnu/-/cli-linux-x64-gnu-2.2.7.tgz", + "integrity": "sha512-ahlSnuCnUntblp9dG7/w5ZWZOdzRFi3zl0oScgt7GF4KNAOEa7duADsxPA4/FT2hLRa0SvpqtD4IYFvCxoVv3Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 OR MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tauri-apps/cli-linux-x64-musl": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/@tauri-apps/cli-linux-x64-musl/-/cli-linux-x64-musl-2.2.7.tgz", + "integrity": "sha512-+qKAWnJRSX+pjjRbKAQgTdFY8ecdcu8UdJ69i7wn3ZcRn2nMMzOO2LOMOTQV42B7/Q64D1pIpmZj9yblTMvadA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 OR MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tauri-apps/cli-win32-arm64-msvc": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/@tauri-apps/cli-win32-arm64-msvc/-/cli-win32-arm64-msvc-2.2.7.tgz", + "integrity": "sha512-aa86nRnrwT04u9D9fhf5JVssuAZlUCCc8AjqQjqODQjMd4BMA2+d4K9qBMpEG/1kVh95vZaNsLogjEaqSTTw4A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 OR MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tauri-apps/cli-win32-ia32-msvc": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/@tauri-apps/cli-win32-ia32-msvc/-/cli-win32-ia32-msvc-2.2.7.tgz", + "integrity": "sha512-EiJ5/25tLSQOSGvv+t6o3ZBfOTKB5S3vb+hHQuKbfmKdRF0XQu2YPdIi1CQw1DU97ZAE0Dq4frvnyYEKWgMzVQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "Apache-2.0 OR MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tauri-apps/cli-win32-x64-msvc": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/@tauri-apps/cli-win32-x64-msvc/-/cli-win32-x64-msvc-2.2.7.tgz", + "integrity": "sha512-ZB8Kw90j8Ld+9tCWyD2fWCYfIrzbQohJ4DJSidNwbnehlZzP7wAz6Z3xjsvUdKtQ3ibtfoeTqVInzCCEpI+pWg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 OR MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@techstark/opencv-js": { "version": "4.10.0-release.1", "resolved": "https://registry.npmjs.org/@techstark/opencv-js/-/opencv-js-4.10.0-release.1.tgz", @@ -4314,6 +4525,8 @@ }, "node_modules/npm/node_modules/@isaacs/cliui/node_modules/string-width": { "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", "dev": true, "inBundle": true, "license": "MIT", @@ -5054,6 +5267,8 @@ }, "node_modules/npm/node_modules/cross-spawn/node_modules/which": { "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, "inBundle": true, "license": "ISC", @@ -5102,6 +5317,8 @@ }, "node_modules/npm/node_modules/debug/node_modules/ms": { "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true, "inBundle": true, "license": "MIT" @@ -6928,6 +7145,8 @@ }, "node_modules/npm/node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "dev": true, "inBundle": true, "license": "MIT", @@ -6965,6 +7184,8 @@ }, "node_modules/npm/node_modules/wrap-ansi/node_modules/string-width": { "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", "dev": true, "inBundle": true, "license": "MIT", diff --git a/frontend/realtime/package.json b/frontend/realtime/package.json index 22eb4c40b5..4ae7619369 100644 --- a/frontend/realtime/package.json +++ b/frontend/realtime/package.json @@ -36,6 +36,7 @@ "@fortawesome/pro-solid-svg-icons": "^6.6.0", "@fortawesome/react-fontawesome": "^0.2.2", "@preact/signals-react-transform": "^0.4.0", + "@tauri-apps/cli": "^2.2.7", "@types/deep-equal": "^1.0.4", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", diff --git a/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts b/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts index 1fa768956f..3b4dc062bd 100644 --- a/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts +++ b/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts @@ -3,14 +3,7 @@ import { Container } from "konva/lib/Container"; import { Shape } from "konva/lib/Shape"; import { Group } from "konva/lib/Group"; -import { - // SharedWorkerRequest, - SharedWorkerResponse, -} from "../WorkerPrimitives/SharedWorkerBase"; -import { - DiffusionSharedWorkerProgressData, - DiffusionSharedWorkerResponseData, -} from "../SharedWorkers/Diffusion/DiffusionSharedWorker"; +import { invoke } from "@tauri-apps/api/core"; import { FileUtilities } from "../FileUtilities/FileUtilities"; import { ImageNode, VideoNode, TextNode } from "../Nodes"; @@ -273,11 +266,15 @@ export class RealTimeDrawEngine { }); return canvas; } - - private handleNodeDragEnd = () => { + public isProcessing = false; + private handleNodeDragEnd = async () => { // Clean up any existing state + if (this.isProcessing) { + return; + } console.log("Node drag ended"); - this.render(); + this.isProcessing = true; + await this.render(); }; public addNodes(node: MediaNode) { if (node instanceof VideoNode) { @@ -379,6 +376,57 @@ export class RealTimeDrawEngine { } } + private async imageBitmapToBase64(imageBitmap: ImageBitmap): Promise { + // Create a temporary canvas + const canvas = document.createElement("canvas"); + canvas.width = imageBitmap.width; + canvas.height = imageBitmap.height; + + // Draw the ImageBitmap onto the canvas + const ctx = canvas.getContext("2d"); + if (!ctx) throw new Error("Failed to get 2D context"); + + ctx.drawImage(imageBitmap, 0, 0); + + // Convert to base64 + const base64String = canvas.toDataURL("image/png"); + + // Clean up + canvas.remove(); + + // Remove the data:image/png;base64, prefix if you want just the base64 string + return base64String.split(",")[1]; + } + + private async base64ToImageBitmap( + base64String: string, + ): Promise { + // Create an image element + const img = new Image(); + + // Convert base64 to data URL if it doesn't include the prefix + const dataUrl = base64String.startsWith("data:") + ? base64String + : `data:image/png;base64,${base64String}`; + + // Create a promise to handle the image loading + return new Promise((resolve, reject) => { + img.onload = async () => { + try { + const bitmap = await createImageBitmap(img); + resolve(bitmap); + } catch (error) { + reject(error); + } + }; + + img.onerror = () => reject(new Error("Failed to load image")); + + // Set the source to trigger loading + img.src = dataUrl; + }); + } + public async render() { // only pick nodes that intersect with the canvas on screen bounds to freeze. this.mediaLayerRef.draw(); @@ -399,6 +447,22 @@ export class RealTimeDrawEngine { test: false, })) as ImageBitmap; - this.previewCanvas.image(bitmap); + try { + const base64Bitmap = await this.imageBitmapToBase64(bitmap); + console.log("Sending to server"); + const base64BitmapResponse = await invoke("infer_image", { + image: base64Bitmap, + }); + console.log("RESPONSE IS"); + console.log(base64BitmapResponse); + const decoded = await this.base64ToImageBitmap( + base64BitmapResponse as string, + ); + this.previewCanvas.image(decoded); + } catch (error) { + console.error("Error during image processing:", error); + } finally { + this.isProcessing = false; + } } } diff --git a/frontend/realtime/yarn.lock b/frontend/realtime/yarn.lock index 6a4f57f16a..dcbace8a8d 100644 --- a/frontend/realtime/yarn.lock +++ b/frontend/realtime/yarn.lock @@ -811,6 +811,32 @@ resolved "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.10.5.tgz" integrity sha512-WlJp8CipNzBz8Q4g159uMfswYHZ8p1+xJwLBdnbBTcSSu0zVoY6i27Suw5J0Y88YBnJ9jHR8jJMH/qSuZWWfNw== +"@tauri-apps/api@^2.2.0": + version "2.2.0" + resolved "https://registry.npmjs.org/@tauri-apps/api/-/api-2.2.0.tgz" + integrity sha512-R8epOeZl1eJEl603aUMIGb4RXlhPjpgxbGVEaqY+0G5JG9vzV/clNlzTeqc+NLYXVqXcn8mb4c5b9pJIUDEyAg== + +"@tauri-apps/cli-win32-x64-msvc@2.2.7": + version "2.2.7" + resolved "https://registry.npmjs.org/@tauri-apps/cli-win32-x64-msvc/-/cli-win32-x64-msvc-2.2.7.tgz" + integrity sha512-ZB8Kw90j8Ld+9tCWyD2fWCYfIrzbQohJ4DJSidNwbnehlZzP7wAz6Z3xjsvUdKtQ3ibtfoeTqVInzCCEpI+pWg== + +"@tauri-apps/cli@^2.2.7": + version "2.2.7" + resolved "https://registry.npmjs.org/@tauri-apps/cli/-/cli-2.2.7.tgz" + integrity sha512-ZnsS2B4BplwXP37celanNANiIy8TCYhvg5RT09n72uR/o+navFZtGpFSqljV8fy1Y4ixIPds8FrGSXJCN2BerA== + optionalDependencies: + "@tauri-apps/cli-darwin-arm64" "2.2.7" + "@tauri-apps/cli-darwin-x64" "2.2.7" + "@tauri-apps/cli-linux-arm-gnueabihf" "2.2.7" + "@tauri-apps/cli-linux-arm64-gnu" "2.2.7" + "@tauri-apps/cli-linux-arm64-musl" "2.2.7" + "@tauri-apps/cli-linux-x64-gnu" "2.2.7" + "@tauri-apps/cli-linux-x64-musl" "2.2.7" + "@tauri-apps/cli-win32-arm64-msvc" "2.2.7" + "@tauri-apps/cli-win32-ia32-msvc" "2.2.7" + "@tauri-apps/cli-win32-x64-msvc" "2.2.7" + "@techstark/opencv-js@^4.10.0-release.1": version "4.10.0-release.1" resolved "https://registry.npmjs.org/@techstark/opencv-js/-/opencv-js-4.10.0-release.1.tgz"