From bef495fc36629b8c6bc23d1554bb1ecc930d0a7c Mon Sep 17 00:00:00 2001 From: wil Date: Tue, 26 Nov 2024 21:13:59 -0500 Subject: [PATCH] feature / Video Extraction UI v2 (#68) * refactor of existing features should work perfect * redo undo of adding points works * multi session fix * undo redo implemented correctly, include exclusion button * exclusion point functional * point stack object * clear extraction data from video node as well --- src/KonvaApp/Engine.ts | 6 +- .../VideoExtractionHandler.ts | 166 ------- .../AddPointsCommand.ts | 131 ++++++ .../VideoExtractionHandler/PointsStack.ts | 35 ++ .../VideoExtractionCommandsManager.ts | 58 +++ .../VideoExtractionHandler.ts | 430 ++++++++++++++++++ .../VideoExtractionHandler/type.ts | 4 + src/KonvaApp/Nodes/NetworkedNode.ts | 2 +- src/KonvaApp/Nodes/VideoNode/VideoNode.ts | 270 +---------- src/KonvaApp/UndoRedo/ICommand.ts | 2 +- src/KonvaApp/UndoRedo/UndoRedoManager.ts | 9 +- .../SignaledToolbarVideoExtraction.tsx | 2 + .../ToolbarVideoExtraction.tsx | 121 ++++- .../features/ToolbarVideoExtraction/enums.ts | 5 + .../uiAccess/toolbarVideoExtraction.ts | 16 + yarn.lock | 336 ++++++++++---- 16 files changed, 1056 insertions(+), 537 deletions(-) delete mode 100644 src/KonvaApp/EngineUtitlities/VideoExtractionHandler.ts create mode 100644 src/KonvaApp/EngineUtitlities/VideoExtractionHandler/AddPointsCommand.ts create mode 100644 src/KonvaApp/EngineUtitlities/VideoExtractionHandler/PointsStack.ts create mode 100644 src/KonvaApp/EngineUtitlities/VideoExtractionHandler/VideoExtractionCommandsManager.ts create mode 100644 src/KonvaApp/EngineUtitlities/VideoExtractionHandler/VideoExtractionHandler.ts create mode 100644 src/KonvaApp/EngineUtitlities/VideoExtractionHandler/type.ts diff --git a/src/KonvaApp/Engine.ts b/src/KonvaApp/Engine.ts index a12241c366..9410d63eff 100644 --- a/src/KonvaApp/Engine.ts +++ b/src/KonvaApp/Engine.ts @@ -33,7 +33,7 @@ import { ToolbarNodeButtonNames } from "~/components/features/ToolbarNode/enums" import { NavigateFunction } from "react-router-dom"; import { LoadingVideosProvider } from "./EngineUtitlities/LoadingVideosProvider"; import { MediaFile } from "~/Classes/ApiManager/models/MediaFile"; -import { VideoExtractionHandler } from "./EngineUtitlities/VideoExtractionHandler"; +import { VideoExtractionHandler } from "./EngineUtitlities/VideoExtractionHandler/VideoExtractionHandler"; // for testing loading files from system // import { FileUtilities } from "./FileUtilities/FileUtilities"; @@ -460,9 +460,7 @@ export class Engine { }); // Listen to other toolbars // VideoExtraction Toolbar - uiEvents.toolbarVideoExtraction.DONE.onClick(() => { - this.videoExtractionHandler.endVideoExtraction(); - }); + this.videoExtractionHandler.listenToToolbarEvents(); // Listen to other requests coming from the UI uiEvents.onGetStagedImage((image) => { diff --git a/src/KonvaApp/EngineUtitlities/VideoExtractionHandler.ts b/src/KonvaApp/EngineUtitlities/VideoExtractionHandler.ts deleted file mode 100644 index baf62046f4..0000000000 --- a/src/KonvaApp/EngineUtitlities/VideoExtractionHandler.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { uiAccess } from "~/signals"; -import { VideoNode } from "../Nodes"; -import { - NodeIsolator, - SelectionManager, - SelectorSquare, -} from "../NodesManagers"; -import { RGBColor } from "../types"; -import { UndoStackManager } from "../UndoRedo"; -import { CommandManager } from "./CommandManager"; -import { - VideoExtractionEventDetails, - VideoExtractionEvents, -} from "../types/events"; - -export class VideoExtractionHandler { - private node?: VideoNode; - private prevIsChroma?: boolean; - private prevChromaColor?: RGBColor; - // engine subclass references - private commandManagerRef: CommandManager; - private nodeIsolatorRef: NodeIsolator; - private selectionManagerRef: SelectionManager; - private selectorSquareRef: SelectorSquare; - private undoStackManagerRef: UndoStackManager; - - constructor({ - commandManagerRef, - nodeIsolatorRef, - selectionManagerRef, - selectorSquareRef, - undoStackManagerRef, - }: { - commandManagerRef: CommandManager; - nodeIsolatorRef: NodeIsolator; - selectionManagerRef: SelectionManager; - selectorSquareRef: SelectorSquare; - undoStackManagerRef: UndoStackManager; - }) { - this.commandManagerRef = commandManagerRef; - this.nodeIsolatorRef = nodeIsolatorRef; - this.selectionManagerRef = selectionManagerRef; - this.selectorSquareRef = selectorSquareRef; - this.undoStackManagerRef = undoStackManagerRef; - } - - private updateLoadingBar({ - detail, - }: CustomEvent) { - const { progress, status, message } = detail; - const disabled = progress !== 100; - const shouldShow = status !== VideoExtractionEvents.SESSION_CLOSED; - uiAccess.toolbarVideoExtraction.update({ - isShowing: shouldShow, - disabled: disabled, - loadingBarState: { - progress: progress, - status: status, - message: message, - }, - }); - } - public async startVideoExtraction(node: VideoNode) { - console.log("VideoExtraction on node", node); - if (this.node || this.prevIsChroma || this.prevChromaColor) { - if (import.meta.env.DEV) { - console.error("Video Extraction: Handler already has node"); - } - return; - } - this.node = node; - this.prevIsChroma = node.isChroma; - this.prevChromaColor = node.chromaColor; - this.node.progressEvent.addEventListener( - "videoextraction", - this.updateLoadingBar as EventListener, - ); - // when the button is pressed to enter extraction mode - console.log("ENGEINE prepare Extraction Session.", node); - // disable most of the UI before we get a session - document.body.style.cursor = "wait"; - this.selectionManagerRef.disable(); - this.selectorSquareRef.disable(); - node.lock(); - this.undoStackManagerRef.setDisabled(true); - // if the video has chroma, disable it - if (this.prevIsChroma) { - node.setChroma(false); - } - // if the video is already using extraction - // bring the original video back - if (node.extractionUrl === node.videoComponent.src) { - await node.loadVideoFromUrl({ - videoUrl: node.mediaFileUrl, - hasExistingTransform: true, - }); - } - // actually start and wait for session - await node.startSegmentation(); - this.nodeIsolatorRef.enterIsolation(node); - node.videoSegmentationMode(true); - this.selectionManagerRef.updateContextComponents(); - uiAccess.toolbarVideoExtraction.update({ - isShowing: true, - disabled: false, - loadingBarState: { - progress: 0, - status: VideoExtractionEvents.SESSION_IDLE, - message: "Select Points for Extraction", - }, - }); - document.body.style.cursor = "default"; - } - - public async endVideoExtraction() { - // when the button is pressed to exit extraction mode - console.log("ENGINE Attemping to close Extraction Session."); - document.body.style.cursor = "wait"; - if ( - !this.node || - this.prevIsChroma === undefined || - !this.prevChromaColor - ) { - if (import.meta.env.DEV) { - console.error("Video Extraction: node or chroma info unavailable"); - } - return; - } - uiAccess.toolbarVideoExtraction.disable(); - const endSessionResult = await this.node.endSession(); - if (typeof endSessionResult === "string") { - this.node.videoSegmentationMode(false); - this.node.progressEvent.removeEventListener( - "videoextraction", - this.updateLoadingBar as EventListener, - ); - this.commandManagerRef.useVideoExtraction({ - videoNode: this.node, - extractionUrl: endSessionResult, - prevIsChroma: this.prevIsChroma, - prevChromaColor: this.prevChromaColor, - }); - this.nodeIsolatorRef.exitIsolation(); - - // unlock the ui - this.undoStackManagerRef.setDisabled(false); - - this.selectorSquareRef.enable(); - this.node.unlock(); - this.selectionManagerRef.updateContextComponents(); - this.selectionManagerRef.enable(); - uiAccess.toolbarVideoExtraction.update({ - isShowing: false, - disabled: false, - }); - this.node = undefined; - this.prevIsChroma = undefined; - this.prevChromaColor = undefined; - // to close off the session. - } else { - uiAccess.toolbarVideoExtraction.enable(); - console.log("Busy Processing Video."); - } - document.body.style.cursor = "default"; - } -} diff --git a/src/KonvaApp/EngineUtitlities/VideoExtractionHandler/AddPointsCommand.ts b/src/KonvaApp/EngineUtitlities/VideoExtractionHandler/AddPointsCommand.ts new file mode 100644 index 0000000000..de707854c5 --- /dev/null +++ b/src/KonvaApp/EngineUtitlities/VideoExtractionHandler/AddPointsCommand.ts @@ -0,0 +1,131 @@ +import { Coordinates } from "~/Classes/ApiManager/SegmentationApi"; +import { ICommand } from "~/KonvaApp/UndoRedo/ICommand"; +import { uiAccess } from "~/signals"; +import { VideoExtractionEvents } from "~/KonvaApp/types/events"; +import { NodeUtilities } from "~/KonvaApp/Nodes/NodeUtilities"; +import { VideoNode } from "~/KonvaApp/Nodes"; +import { SegmentationApi } from "~/Classes/ApiManager/SegmentationApi"; +import { PointsStack } from "./PointsStack"; + +export class AddPointsCommand implements ICommand { + private prevPreviewUrl: string | undefined; + private selectedPointsRef: PointsStack; + private nodeRef: VideoNode; + private newPoint: Coordinates; + private newPreviewUrl: string | undefined; + private api: SegmentationApi; + private sessionId: string; + + constructor({ + selectedPointsRef, + nodeRef, + newPoint, + api, + sessionId, + }: { + selectedPointsRef: PointsStack; + nodeRef: VideoNode; + newPoint: Coordinates; + api: SegmentationApi; + sessionId: string; + }) { + this.selectedPointsRef = selectedPointsRef; + this.newPoint = newPoint; + this.nodeRef = nodeRef; + this.prevPreviewUrl = this.nodeRef.extractionPreviewUrl; + this.api = api; + this.sessionId = sessionId; + } + async execute() { + this.selectedPointsRef.addPoint(this.newPoint); + + if (this.newPreviewUrl) { + // case of redo, no need to do the server calls + await this.nodeRef.setVideoExtractionPreview(this.newPreviewUrl); + return true; + } + + // request for a new point. + uiAccess.toolbarVideoExtraction.update({ + loadingBarState: { + progress: 25, + status: VideoExtractionEvents.EXTRACTION_POINT_REQUEST, + message: "Start Processing Extraction...", + }, + }); + + try { + console.log("Requesting"); + const response = await this.api.addPointsToSession( + this.sessionId, + 24, + [ + { + timestamp: 0, + objects: [ + { + style: { + color: [0, 0, 1], + }, + object_id: 0, + points: this.selectedPointsRef.get(), + }, + ], + }, + ], + false, + ); + uiAccess.toolbarVideoExtraction.update({ + loadingBarState: { + progress: 50, + status: VideoExtractionEvents.EXTRACTION_POINT_REQUEST, + message: "Start Processing Extraction...", + }, + }); + const previewImageUrl = response.frames[0].preview_image_url; + + // TODO: we assumed success of the loading of the AssetUrl + await NodeUtilities.isAssetUrlAvailable({ url: previewImageUrl }); + uiAccess.toolbarVideoExtraction.update({ + loadingBarState: { + progress: 75, + status: VideoExtractionEvents.EXTRACTION_POINT_REQUEST, + message: "Processing Extraction Points...", + }, + }); + this.newPreviewUrl = previewImageUrl; + await this.nodeRef.setVideoExtractionPreview(previewImageUrl); + uiAccess.toolbarVideoExtraction.update({ + loadingBarState: { + progress: 100, + status: VideoExtractionEvents.EXTRACTION_POINT_REQUEST, + message: "Extraction of Region Done", + }, + }); + return true; + } catch (error) { + console.error(error); + ("Extraction of Region Done"); + uiAccess.toolbarVideoExtraction.update({ + loadingBarState: { + progress: 0, + status: VideoExtractionEvents.EXTRACTION_POINT_REQUEST, + message: `Error:${error} Please try picking extraction points again`, + }, + }); + return false; + } + } + async undo() { + this.selectedPointsRef.pop(); + + if (this.prevPreviewUrl) { + await this.nodeRef.setVideoExtractionPreview(this.prevPreviewUrl); + } else { + await this.nodeRef.loadVideoFromUrl({ + videoUrl: this.nodeRef.mediaFileUrl, + hasExistingTransform: true, + }); + } + } +} diff --git a/src/KonvaApp/EngineUtitlities/VideoExtractionHandler/PointsStack.ts b/src/KonvaApp/EngineUtitlities/VideoExtractionHandler/PointsStack.ts new file mode 100644 index 0000000000..1623b4578e --- /dev/null +++ b/src/KonvaApp/EngineUtitlities/VideoExtractionHandler/PointsStack.ts @@ -0,0 +1,35 @@ +import { Coordinates } from "~/Classes/ApiManager/SegmentationApi"; +import { uiAccess } from "~/signals"; +export class PointsStack { + private points: Coordinates[]; + constructor() { + this.points = []; + } + public get() { + return this.points; + } + public addPoint(point: Coordinates) { + this.points.push(point); + uiAccess.toolbarVideoExtraction.setReady(true); + } + public pop() { + this.points.pop(); + if (this.points.length === 0) { + uiAccess.toolbarVideoExtraction.setReady(false); + } + } + public removePoint(point: Coordinates) { + this.points = this.points.filter( + (curr) => + curr.coordinates[0] !== point.coordinates[0] && + curr.coordinates[1] != point.coordinates[1], + ); + if (this.points.length === 0) { + uiAccess.toolbarVideoExtraction.setReady(false); + } + } + public clear() { + this.points = []; + uiAccess.toolbarVideoExtraction.setReady(false); + } +} diff --git a/src/KonvaApp/EngineUtitlities/VideoExtractionHandler/VideoExtractionCommandsManager.ts b/src/KonvaApp/EngineUtitlities/VideoExtractionHandler/VideoExtractionCommandsManager.ts new file mode 100644 index 0000000000..8b5fa0148d --- /dev/null +++ b/src/KonvaApp/EngineUtitlities/VideoExtractionHandler/VideoExtractionCommandsManager.ts @@ -0,0 +1,58 @@ +import { UndoStackManager } from "~/KonvaApp/UndoRedo"; +import { VideoNode } from "~/KonvaApp/Nodes"; + +import { + SegmentationApi, + Coordinates, +} from "~/Classes/ApiManager/SegmentationApi"; + +import { AddPointsCommand } from "./AddPointsCommand"; +import { PointsStack } from "./PointsStack"; + +export class VideoExtractionCommandsManager { + private undoStack: UndoStackManager; + private selectedPointsRef: PointsStack; + private api: SegmentationApi; + private nodeRef: VideoNode; + private sessionId: string; + + constructor({ + selectedPointsRef, + api, + nodeRef, + sessionId, + }: { + selectedPointsRef: PointsStack; + api: SegmentationApi; + nodeRef: VideoNode; + sessionId: string; + }) { + this.selectedPointsRef = selectedPointsRef; + this.undoStack = new UndoStackManager(); + this.api = api; + this.nodeRef = nodeRef; + this.sessionId = sessionId; + } + public async addPoint(newPoint: Coordinates) { + const command = new AddPointsCommand({ + selectedPointsRef: this.selectedPointsRef, + nodeRef: this.nodeRef, + newPoint: newPoint, + api: this.api, + sessionId: this.sessionId, + }); + await this.undoStack.executeCommand(command); + } + // private removeInclusionPoint() {} + public clear() { + this.undoStack.clear(); + } + public undo() { + console.log("ve should undo"); + this.undoStack.undo(); + } + public redo() { + console.log("ve should redo"); + this.undoStack.redo(); + } +} diff --git a/src/KonvaApp/EngineUtitlities/VideoExtractionHandler/VideoExtractionHandler.ts b/src/KonvaApp/EngineUtitlities/VideoExtractionHandler/VideoExtractionHandler.ts new file mode 100644 index 0000000000..0e6c275af7 --- /dev/null +++ b/src/KonvaApp/EngineUtitlities/VideoExtractionHandler/VideoExtractionHandler.ts @@ -0,0 +1,430 @@ +import { uiAccess, uiEvents } from "~/signals"; +import { VideoNode } from "../../Nodes"; +import { + NodeIsolator, + SelectionManager, + SelectorSquare, +} from "../../NodesManagers"; +import { RGBColor } from "../../types"; +import { UndoStackManager } from "../../UndoRedo"; +import { CommandManager } from "../CommandManager"; +import { + VideoExtractionEventDetails, + VideoExtractionEvents, +} from "../../types/events"; +import { NodeUtilities } from "../../Nodes/NodeUtilities"; +import { SegmentationApi } from "~/Classes/ApiManager/SegmentationApi"; +import { VideoExtractionCommandsManager } from "./VideoExtractionCommandsManager"; +import { VideoExtractionMode } from "./type"; +import { PointsStack } from "./PointsStack"; + +export class VideoExtractionHandler { + private videoExtractionAPI = new SegmentationApi(); + private videoExtractionSession: { session_id: string } | undefined = + undefined; + private isProcessing: boolean = false; + private selectedPoints: PointsStack; + private mode: VideoExtractionMode; + private videoExtractionCommandsManager?: VideoExtractionCommandsManager; + + //node references + private node?: VideoNode; + private prevIsChroma?: boolean; + private prevChromaColor?: RGBColor; + + // engine subclass references + private commandManagerRef: CommandManager; + private nodeIsolatorRef: NodeIsolator; + private selectionManagerRef: SelectionManager; + private selectorSquareRef: SelectorSquare; + private undoStackManagerRef: UndoStackManager; + + constructor({ + commandManagerRef, + nodeIsolatorRef, + selectionManagerRef, + selectorSquareRef, + undoStackManagerRef, + }: { + commandManagerRef: CommandManager; + nodeIsolatorRef: NodeIsolator; + selectionManagerRef: SelectionManager; + selectorSquareRef: SelectorSquare; + undoStackManagerRef: UndoStackManager; + }) { + this.commandManagerRef = commandManagerRef; + this.nodeIsolatorRef = nodeIsolatorRef; + this.selectionManagerRef = selectionManagerRef; + this.selectorSquareRef = selectorSquareRef; + this.undoStackManagerRef = undoStackManagerRef; + this.mode = VideoExtractionMode.INCLUSION; + this.selectedPoints = new PointsStack(); + uiAccess.toolbarVideoExtraction; + } + private setMode(mode: VideoExtractionMode) { + this.mode = mode; + uiAccess.toolbarVideoExtraction.setMode(mode); + } + private updateLoadingBar({ + detail, + }: CustomEvent) { + const { progress, status, message } = detail; + const disabled = progress !== 100; + const shouldShow = status !== VideoExtractionEvents.SESSION_CLOSED; + uiAccess.toolbarVideoExtraction.update({ + isShowing: shouldShow, + disabled: disabled, + loadingBarState: { + progress: progress, + status: status, + message: message, + }, + }); + } + + public async startVideoExtraction(node: VideoNode) { + console.log("VideoExtraction on node", node); + if (this.node || this.prevIsChroma || this.prevChromaColor) { + if (import.meta.env.DEV) { + console.error("Video Extraction: Handler already has node"); + } + return; + } + if (this.videoExtractionSession) { + if (import.meta.env.DEV) { + console.error("Video Extraction: Session already exists"); + } + return; + } + this.node = node; + this.prevIsChroma = node.isChroma; + this.prevChromaColor = node.chromaColor; + this.node.progressEvent.addEventListener( + "videoextraction", + this.updateLoadingBar as EventListener, + ); + // when the button is pressed to enter extraction mode + console.log("ENGEINE prepare Extraction Session.", node); + // disable most of the UI before we get a session + document.body.style.cursor = "wait"; + this.selectionManagerRef.disable(); + this.selectorSquareRef.disable(); + node.lock(); + this.undoStackManagerRef.setDisabled(true); + // if the video has chroma, disable it + if (this.prevIsChroma) { + node.setChroma(false); + } + // if the video is already using extraction + // bring the original video back + if (node.extractionUrl === node.videoComponent.src) { + await node.loadVideoFromUrl({ + videoUrl: node.mediaFileUrl, + hasExistingTransform: true, + }); + } + node.videoComponent.pause(); + node.videoComponent.currentTime = 0; + + // actually start and wait for session + // await node.startSegmentation(); + + uiAccess.toolbarVideoExtraction.update({ + isShowing: true, + disabled: true, + loadingBarState: { + progress: 25, + status: VideoExtractionEvents.SESSION_CREATING, + message: "Loading Video Extractor...", + }, + }); + const blob = await NodeUtilities.urlToBlob(node.mediaFileUrl); + uiAccess.toolbarVideoExtraction.update({ + loadingBarState: { + progress: 75, + status: VideoExtractionEvents.SESSION_CREATING, + message: "Loading Video Extractor...", + }, + }); + this.videoExtractionSession = + await this.videoExtractionAPI.createSession(blob); + uiAccess.toolbarVideoExtraction.update({ + loadingBarState: { + progress: 100, + status: VideoExtractionEvents.SESSION_CREATING, + message: "Loading Video Extractor...", + }, + }); + + this.nodeIsolatorRef.enterIsolation(node); + node.enableExtractionMode(this.handleExtraction.bind(this)); + this.selectionManagerRef.updateContextComponents(); + uiAccess.toolbarVideoExtraction.update({ + disabled: false, + loadingBarState: { + progress: 0, + status: VideoExtractionEvents.SESSION_IDLE, + message: "Select Points for Extraction", + }, + }); + this.videoExtractionCommandsManager = new VideoExtractionCommandsManager({ + selectedPointsRef: this.selectedPoints, + api: this.videoExtractionAPI, + nodeRef: this.node, + sessionId: this.videoExtractionSession.session_id, + }); + document.body.style.cursor = "default"; + } + + public async endVideoExtraction() { + // when the button is pressed to exit extraction mode + console.log("ENGINE Attemping to close Extraction Session."); + document.body.style.cursor = "wait"; + if ( + !this.node || + this.prevIsChroma === undefined || + !this.prevChromaColor + ) { + if (import.meta.env.DEV) { + console.error("Video Extraction: node or chroma info unavailable"); + } + return; + } + if (this.isProcessing) { + console.warn( + "Video Extraction: Cannot close while still processing frame, please wait", + ); + return false; + } + + this.isProcessing = true; + uiAccess.toolbarVideoExtraction.update({ + disabled: true, + loadingBarState: { + progress: 100, + status: VideoExtractionEvents.SESSION_CLOSING, + message: "Processing Video...", + }, + }); + + const endSessionResult = await this.endSession(); + + if (typeof endSessionResult === "string") { + this.commandManagerRef.useVideoExtraction({ + videoNode: this.node, + extractionUrl: endSessionResult, + prevIsChroma: this.prevIsChroma, + prevChromaColor: this.prevChromaColor, + }); + this.unsetUi(); + } else { + uiAccess.toolbarVideoExtraction.enable(); + console.log("Busy Processing Video."); + } + document.body.style.cursor = "default"; + } + + private async endSession(): Promise { + if (!this.videoExtractionSession || !this.node) { + if (import.meta.env.DEV) { + console.error("Video Extraction: Session does not exist"); + } + return false; + } + try { + console.log("Requesting a close"); + const response = await this.videoExtractionAPI.addPointsToSession( + this.videoExtractionSession.session_id, + 24, + [ + { + timestamp: 0, + objects: [ + { + style: { + color: [0, 0, 1], + }, + object_id: 0, + points: this.selectedPoints.get(), + }, + ], + }, + ], + true, // propagation + // propagation = true = this requests the entire video to be processed + ); + uiAccess.toolbarVideoExtraction.update({ + loadingBarState: { + progress: 50, + status: VideoExtractionEvents.SESSION_CLOSING, + message: "Processing Video...", + }, + }); + // replace the video component and reregister all the other elements. + console.log( + "Extracted Video URL", + response["masked_video_cdn_url"], + response, + ); + // TODO: we assume the URL to be checked will eventually return true + const extractionUrl = response["masked_video_cdn_url"]; + await NodeUtilities.isAssetUrlAvailable({ + url: extractionUrl, + sleepDurationMs: 2000, + }); + uiAccess.toolbarVideoExtraction.update({ + loadingBarState: { + progress: 75, + status: VideoExtractionEvents.SESSION_CLOSING, + message: "Processing Video...", + }, + }); + this.node.extractionUrl = extractionUrl; + // set chroma automatically. + this.node.isChroma = true; + await this.node.loadVideoFromUrl({ + videoUrl: extractionUrl, + hasExistingTransform: true, + }); + uiAccess.toolbarVideoExtraction.update({ + loadingBarState: { + progress: 100, + status: VideoExtractionEvents.SESSION_CLOSING, + message: "Done Processing Video", + }, + }); + this.isProcessing = false; + + return extractionUrl; + } catch (error) { + console.error(error); + this.isProcessing = false; + return false; + } + } + + public async cancelVideoExtraction() { + // when the button is pressed to exit extraction mode + console.log("ENGINE cancel Extraction Session."); + + if (!this.node) { + // this case should not happen, without a node nothing could be done + if (import.meta.env.DEV) { + console.error("Video Extraction: node or chroma info unavailable"); + } + return; + } + if (this.prevIsChroma === undefined || !this.prevChromaColor) { + // this case should not happen, but could proceed + if (import.meta.env.DEV) { + console.error("Video Extraction: chroma info unavailable"); + } + } + + await this.node.loadVideoFromUrl({ + videoUrl: this.node.mediaFileUrl, + hasExistingTransform: true, + }); + this.unsetUi(); + this.isProcessing = false; + document.body.style.cursor = "default"; + } + private async handleExtraction() { + console.log("Handle Extraction"); + if (!this.node) { + console.log("Video Extraction does not have VideoNode"); + return; + } + if (!this.videoExtractionSession) { + console.log("Video Extraction Session Not Ready"); + return; + } + if (this.isProcessing) { + console.log("Still Processing"); + return; + } + + this.isProcessing = true; + document.body.style.cursor = "wait"; + + // Get the local coordinates of the click relative to the rectangle + const kNode = this.node.kNode; + const localPos = kNode.getRelativePointerPosition(); + const fileSize = this.node.mediaFileSize; + console.log("Local coordinates:mediaFileSize", localPos); + console.log("mediaFileSize:", this.node.mediaFileSize); + if (!localPos || !fileSize) { + // TODO: error handling + return; + } + const adjustedLocalPos = { + x: (localPos.x / kNode.width()) * kNode.scaleX() * fileSize.width, + y: (localPos.y / kNode.height()) * kNode.scaleY() * fileSize.height, + }; + + if (this.videoExtractionCommandsManager) { + await this.videoExtractionCommandsManager.addPoint({ + coordinates: [adjustedLocalPos.x, adjustedLocalPos.y], + include: this.mode === "inclusion", + }); + } + + document.body.style.cursor = "default"; + this.isProcessing = false; + } + public undo() { + this.videoExtractionCommandsManager?.undo(); + } + public redo() { + this.videoExtractionCommandsManager?.redo(); + } + + private unsetUi() { + // unlock the ui + this.nodeIsolatorRef.exitIsolation(); + this.undoStackManagerRef.setDisabled(false); + this.selectorSquareRef.enable(); + if (this.node) { + this.node.disableExtractionMode(); + this.node.progressEvent.removeEventListener( + "videoextraction", + this.updateLoadingBar as EventListener, + ); + this.node.unlock(); + } + this.selectionManagerRef.updateContextComponents(); + this.selectionManagerRef.enable(); + uiAccess.toolbarVideoExtraction.update({ + isShowing: false, + disabled: false, + }); + + // reset to close off the session. + this.videoExtractionCommandsManager?.clear(); + this.selectedPoints.clear(); + this.node = undefined; + this.prevIsChroma = undefined; + this.prevChromaColor = undefined; + this.videoExtractionSession = undefined; + } + public listenToToolbarEvents() { + uiEvents.toolbarVideoExtraction.INCLUSION_MODE.onClick(() => { + this.setMode(VideoExtractionMode.INCLUSION); + }); + uiEvents.toolbarVideoExtraction.EXCLUSION_MODE.onClick(() => { + this.setMode(VideoExtractionMode.EXCLUSION); + }); + uiEvents.toolbarVideoExtraction.UNDO.onClick(() => { + this.undo(); + }); + uiEvents.toolbarVideoExtraction.REDO.onClick(() => { + this.redo(); + }); + uiEvents.toolbarVideoExtraction.DONE.onClick(() => { + this.endVideoExtraction(); + }); + uiEvents.toolbarVideoExtraction.CANCEL.onClick(() => { + this.cancelVideoExtraction(); + }); + } +} diff --git a/src/KonvaApp/EngineUtitlities/VideoExtractionHandler/type.ts b/src/KonvaApp/EngineUtitlities/VideoExtractionHandler/type.ts new file mode 100644 index 0000000000..d2b6c363c9 --- /dev/null +++ b/src/KonvaApp/EngineUtitlities/VideoExtractionHandler/type.ts @@ -0,0 +1,4 @@ +export enum VideoExtractionMode { + INCLUSION = "inclusion", + EXCLUSION = "exclusion", +} diff --git a/src/KonvaApp/Nodes/NetworkedNode.ts b/src/KonvaApp/Nodes/NetworkedNode.ts index aadf5f36c8..dd531c4ecb 100644 --- a/src/KonvaApp/Nodes/NetworkedNode.ts +++ b/src/KonvaApp/Nodes/NetworkedNode.ts @@ -33,7 +33,7 @@ export abstract class NetworkedNode extends BaseNode { protected localFile?: File; protected mediaFileToken?: string; protected mediaFileUrl?: string; - protected mediaFileSize?: Size; + public mediaFileSize?: Size; // loading screen protected loadingVideosProviderRef?: LoadingVideosProvider; diff --git a/src/KonvaApp/Nodes/VideoNode/VideoNode.ts b/src/KonvaApp/Nodes/VideoNode/VideoNode.ts index d30f80c000..d612afed75 100644 --- a/src/KonvaApp/Nodes/VideoNode/VideoNode.ts +++ b/src/KonvaApp/Nodes/VideoNode/VideoNode.ts @@ -15,13 +15,8 @@ import { } from "../../types"; import { NodeUtilities } from "../NodeUtilities"; -import { - Coordinates, - SegmentationApi, -} from "~/Classes/ApiManager/SegmentationApi"; - import { LoadingVideosProvider } from "~/KonvaApp/EngineUtitlities/LoadingVideosProvider"; -import { VideoExtractionEvents } from "~/KonvaApp/types/events"; + interface VideoNodeContructor { mediaLayerRef: Konva.Layer; selectionManagerRef: SelectionManager; @@ -38,6 +33,7 @@ export class VideoNode extends NetworkedNode { public kNode: Konva.Image; public mediaFileUrl: string; public extractionUrl: string | undefined; + public extractionPreviewUrl: string | undefined; public currentUrl: string; public videoComponent: HTMLVideoElement; protected _isVideoEventListening: boolean = false; @@ -58,15 +54,8 @@ export class VideoNode extends NetworkedNode { isChroma: boolean; chromaColor: RGBColor; - // todo move to seg manager - selectedPointsForSegmentation: Coordinates[] = []; - - private videoSegmentationAPI = new SegmentationApi(); - private frameDidFinishSeeking: Promise; - - public isSegmentationMode: boolean = false; - private segmentationSession: { session_id: string } | undefined = undefined; + public videoExtractionClickCallback?: () => void; constructor({ mediaLayerRef, @@ -285,7 +274,8 @@ export class VideoNode extends NetworkedNode { if (this.isProcessing == true) { return; } - if (this.isSegmentationMode) { + if (this.videoExtractionClickCallback) { + // if (this.isSegmentationMode) { // console.log("Not Playing because Segmenting"); return; } @@ -458,246 +448,22 @@ export class VideoNode extends NetworkedNode { } } - public videoSegmentationMode(on: boolean) { - if (on) { - this.removeListenToBaseKNode(); - this.kNode.on("click", this.handleSegmentation.bind(this)); - this.isSegmentationMode = true; - } else { - this.isSegmentationMode = false; - this.kNode.off("click", this.handleSegmentation.bind(this)); - this.listenToBaseKNode(); + public enableExtractionMode(callback: () => void) { + this.removeListenToBaseKNode(); + this.kNode.on("click", callback); + this.videoExtractionClickCallback = callback; + } + public disableExtractionMode() { + if (this.videoExtractionClickCallback) { + this.kNode.off("click", this.videoExtractionClickCallback); } + this.extractionPreviewUrl = undefined; + this.videoExtractionClickCallback = undefined; + this.listenToBaseKNode(); } - public async startSegmentation() { - if (this.segmentationSession === undefined) { - this.didFinishLoading = false; - this.videoComponent.pause(); - this.videoComponent.currentTime = 0; - this.setProgress({ - name: "videoextraction", - progress: 25, - status: VideoExtractionEvents.SESSION_CREATING, - message: "Loading Video Extractor...", - }); - - const blob = await NodeUtilities.urlToBlob(this.mediaFileUrl); - this.setProgress({ - name: "videoextraction", - progress: 75, - status: VideoExtractionEvents.SESSION_CREATING, - message: "Loading Video Extractor...", - }); - - this.segmentationSession = - await this.videoSegmentationAPI.createSession(blob); - this.setProgress({ - name: "videoextraction", - progress: 100, - status: VideoExtractionEvents.SESSION_CREATING, - message: "Loaded Video Extractor", - }); - console.log("Sessions", this.segmentationSession); - } - } - - private isStillProcessingSegmentationEvent: Boolean = false; - - public async handleSegmentation() { - if (!this.isSegmentationMode) { - console.log("Segmentation Mode Not On"); - return; - } - if (!this.segmentationSession) { - console.log("Segmentation Session Not Ready"); - return; - } - if (this.isStillProcessingSegmentationEvent) { - console.log("Still Processing"); - return; - } - - this.isStillProcessingSegmentationEvent = true; - - // Get the local coordinates of the click relative to the rectangle - const localPos = this.kNode.getRelativePointerPosition(); - console.log("Local coordinates:", localPos); - console.log("mediaFileSize:", this.mediaFileSize); - if (!localPos || !this.mediaFileSize) { - // TODO: error handling - return; - } - document.body.style.cursor = "wait"; - - const adjustedLocalPos = { - x: - (localPos.x / this.kNode.width()) * - this.kNode.scaleX() * - this.mediaFileSize.width, - y: - (localPos.y / this.kNode.height()) * - this.kNode.scaleY() * - this.mediaFileSize.height, - }; - // handle undo as well. - this.selectedPointsForSegmentation.push({ - coordinates: [adjustedLocalPos.x, adjustedLocalPos.y], - include: true, - }); - this.setProgress({ - name: "videoextraction", - progress: 25, - status: VideoExtractionEvents.EXTRACTION_POINT_REQUEST, - message: "Start Processing Extraction...", - }); - - try { - console.log("Requesting"); - const response = await this.videoSegmentationAPI.addPointsToSession( - this.segmentationSession.session_id, - 24, - [ - { - timestamp: 0, - objects: [ - { - style: { - color: [0, 0, 1], - }, - object_id: 0, - points: this.selectedPointsForSegmentation, - }, - ], - }, - ], - false, - ); - this.setProgress({ - name: "videoextraction", - progress: 50, - status: VideoExtractionEvents.EXTRACTION_POINT_REQUEST, - message: "Processing Extraction Point...", - }); - const previewImageUrl = response.frames[0].preview_image_url; - - // TODO: we assumed success of the loading of the AssetUrl - await NodeUtilities.isAssetUrlAvailable({ url: previewImageUrl }); - this.setProgress({ - name: "videoextraction", - progress: 75, - status: VideoExtractionEvents.EXTRACTION_POINT_REQUEST, - message: "Processing Extraction Point...", - }); - await this.setSegementationPreview(previewImageUrl); - this.setProgress({ - name: "videoextraction", - progress: 100, - status: VideoExtractionEvents.EXTRACTION_POINT_REQUEST, - message: "Extraction of Region Done", - }); - } catch (error) { - console.error(error); - this.setProgress({ - name: "videoextraction", - progress: 0, - status: VideoExtractionEvents.EXTRACTION_POINT_REQUEST, - message: `Error:${error} Please try picking extraction points again`, - }); - } - document.body.style.cursor = "default"; - this.isStillProcessingSegmentationEvent = false; - } - - async endSession(): Promise { - // wait for url to come through and fake load - // edge case - if (this.isStillProcessingSegmentationEvent) { - console.log("Still Processing Frame Cannot Clip Please Wait"); - return false; - } - // prevent requests to add points while doing this. - this.isStillProcessingSegmentationEvent = true; - this.setProgress({ - name: "videoextraction", - progress: 25, - status: VideoExtractionEvents.SESSION_CLOSING, - message: "Processing Video...", - }); - if (!this.segmentationSession) { - console.log("Segmentation Session Lost?"); - return false; - } - - try { - console.log("Requesting a close"); - const response = await this.videoSegmentationAPI.addPointsToSession( - this.segmentationSession.session_id, - 24, - [ - { - timestamp: 0, - objects: [ - { - style: { - color: [0, 0, 1], - }, - object_id: 0, - points: this.selectedPointsForSegmentation, - }, - ], - }, - ], - true, // propagation = true, this requests the entire video to be processed - ); - this.setProgress({ - name: "videoextraction", - progress: 50, - status: VideoExtractionEvents.SESSION_CLOSING, - message: "Processing Video...", - }); - // replace the video component and reregister all the other elements. - console.log( - "Extracted Video URL", - response["masked_video_cdn_url"], - response, - ); - // TODO: we assume the URL to be checked will eventually return true - const extractionUrl = response["masked_video_cdn_url"]; - await NodeUtilities.isAssetUrlAvailable({ - url: extractionUrl, - sleepDurationMs: 2000, - }); - this.setProgress({ - name: "videoextraction", - progress: 75, - status: VideoExtractionEvents.SESSION_CLOSING, - message: "Processing Video...", - }); - this.extractionUrl = extractionUrl; - // set chroma automatically. - this.isChroma = true; - await this.loadVideoFromUrl({ - videoUrl: extractionUrl, - hasExistingTransform: true, - }); - this.setProgress({ - name: "videoextraction", - progress: 100, - status: VideoExtractionEvents.SESSION_CLOSING, - message: "Done Processing Video", - }); - this.isStillProcessingSegmentationEvent = false; - - return extractionUrl; - } catch (error) { - console.error(error); - this.isStillProcessingSegmentationEvent = false; - return false; - } - } - - private async setSegementationPreview(previewImageUrl: string) { + public async setVideoExtractionPreview(previewImageUrl: string) { + this.extractionPreviewUrl = previewImageUrl; const imageObj = new Image(); const imageLoadPromise = new Promise((resolve, reject) => { diff --git a/src/KonvaApp/UndoRedo/ICommand.ts b/src/KonvaApp/UndoRedo/ICommand.ts index 60ae66ea69..35ccc223c0 100644 --- a/src/KonvaApp/UndoRedo/ICommand.ts +++ b/src/KonvaApp/UndoRedo/ICommand.ts @@ -1,4 +1,4 @@ export interface ICommand { - execute(): void | boolean; + execute(): void | boolean | Promise; undo(): void; } diff --git a/src/KonvaApp/UndoRedo/UndoRedoManager.ts b/src/KonvaApp/UndoRedo/UndoRedoManager.ts index 2d2603f761..7a1cf29d8a 100644 --- a/src/KonvaApp/UndoRedo/UndoRedoManager.ts +++ b/src/KonvaApp/UndoRedo/UndoRedoManager.ts @@ -12,8 +12,8 @@ export class UndoStackManager { isDisabled() { return this.disabled; } - executeCommand(command: ICommand) { - const result = command.execute(); + async executeCommand(command: ICommand) { + const result = await command.execute(); if (result !== undefined && result === false) { //command is not executed if it deliberatly returned false; //in that case, no need to do anything @@ -31,7 +31,10 @@ export class UndoStackManager { } console.log("command stack:", this.undoStack); } - + clear() { + this.undoStack = []; + this.redoStack = []; + } undo() { if (this.disabled) { return; diff --git a/src/KonvaRootComponent/SignaledToolbarVideoExtraction.tsx b/src/KonvaRootComponent/SignaledToolbarVideoExtraction.tsx index b01ef2996a..72b57ff2fc 100644 --- a/src/KonvaRootComponent/SignaledToolbarVideoExtraction.tsx +++ b/src/KonvaRootComponent/SignaledToolbarVideoExtraction.tsx @@ -48,6 +48,8 @@ export const SignaledToolbarVideoExtraction = () => { show={toolbarVideoExtraction.signal.value.isShowing} > ; } export const ToolbarVideoExtraction = ({ + extractionMode = "inclusion", + readyToSubmit = false, disabled, buttonsProps, loadingBarProps, }: ToolbarVideoExtractionProps) => { + // const [frame, setFrame] = useState(0); const loadingBar = { colReverse: true, progress: 0, @@ -33,19 +50,103 @@ export const ToolbarVideoExtraction = ({ className={twMerge( paperWrapperStyles, disabled && "pointer-events-none cursor-default bg-ui-border shadow-md", - "flex items-center justify-center gap-2 pr-4 transition", + "flex flex-col items-center justify-center gap-2 pr-4 transition", )} > -
+
+ {/* { + console.log(val); + }} + step={1} + value={frame} + /> */} +
+
+ +
+ +
+ + +
+ +
+ + + + +
- -
); }; +const CancelButton = (buttonProps: ToolbarButtonProps) => { + const { + className: customButtonClassNames, + disabled, + active, + hidden, + onClick, + ...restButtonProps + } = buttonProps; -const DoneButton = (buttonProps: ToolbarButtonProps) => { + return ( + + ); +}; +type DoneButtonProps = { ready: boolean } & ToolbarButtonProps; +const DoneButton = ({ ready, ...buttonProps }: DoneButtonProps) => { const { className: customButtonClassNames, disabled, @@ -58,9 +159,9 @@ const DoneButton = (buttonProps: ToolbarButtonProps) => { return (