From 082c1f23cdb85cc25b37d562817ec1bc178b197f Mon Sep 17 00:00:00 2001 From: ArEnSc <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 17 Jul 2024 17:57:57 -0400 Subject: [PATCH] Feature/make movie generate great again (#640) --- app/Classes/ApiManager/VideoApi.ts | 15 +- app/pages/PageEnigma/Editor/editor.ts | 50 +- .../PageEnigma/Editor/keybinds_controls.ts | 1 + app/pages/PageEnigma/Editor/save_manager.ts | 60 ++- app/pages/PageEnigma/Editor/scene.ts | 6 + app/pages/PageEnigma/Editor/timeline.ts | 21 +- .../PageEnigma/Editor/video_generation.ts | 451 ++++++++++++------ .../StylizeTab/GenerateMovieButton.tsx | 3 +- test/app/Classes/ApiManager/VideoApi.test.ts | 8 +- 9 files changed, 440 insertions(+), 175 deletions(-) diff --git a/app/Classes/ApiManager/VideoApi.ts b/app/Classes/ApiManager/VideoApi.ts index c482722e22..aad617250c 100644 --- a/app/Classes/ApiManager/VideoApi.ts +++ b/app/Classes/ApiManager/VideoApi.ts @@ -2,27 +2,31 @@ import { ApiManager, ApiResponse } from "./ApiManager"; import { Visibility } from "~/enums"; interface GenerateVideoStyleTransferRequest { - creator_set_visibility?: Visibility; + creator_set_visibility: string; disable_lcm: boolean; enable_lipsync: boolean; + frame_skip: number; + global_ipa_media_token: string; + input_depth_file: string; input_file: string; - ipa_media_token: string; + input_normal_file: string; + input_outline_file: string; negative_prompt: string; prompt: string; remove_watermark: boolean; style: string; + travel_prompt: string; trim_end_millis: number; trim_start_millis: number; use_cinematic: boolean; use_face_detailer: boolean; - use_lipsync: boolean; use_strength: number; use_upscaler: boolean; uuid_idempotency_token: string; } export class VideoApi extends ApiManager { - public async GenerateVideoStyleTransfer({ + public async EnqueueStudio({ enqueueVideo, }: { enqueueVideo: GenerateVideoStyleTransferRequest; @@ -32,7 +36,7 @@ export class VideoApi extends ApiManager { inference_job_token_type?: string; }> > { - const endpoint = `${this.ApiTargets.BaseApi}/v1/video/enqueue_vst`; + const endpoint = `${this.ApiTargets.BaseApi}/v1/workflows/enqueue_studio`; const body = { ...enqueueVideo, @@ -59,6 +63,7 @@ export class VideoApi extends ApiManager { errorMessage: response.BadInput, })) .catch((err) => { + console.log(err.message); return { success: false, error_reason: err.message }; }); } diff --git a/app/pages/PageEnigma/Editor/editor.ts b/app/pages/PageEnigma/Editor/editor.ts index d6a3f55f23..35cf09dcdc 100644 --- a/app/pages/PageEnigma/Editor/editor.ts +++ b/app/pages/PageEnigma/Editor/editor.ts @@ -1010,7 +1010,6 @@ class Editor { this.render_timer += this.clock.getDelta(); } if (!this.timeline.is_playing) { - //this.recorder.stop(); this.playback_location = 0; for (const key in this.timeline.characters) { if (this.timeline.characters.hasOwnProperty(key)) { // Check if the key is the object's own property @@ -1026,6 +1025,20 @@ class Editor { } } + async useCachedMediaTokens(): Promise { + // if the preprecessing switch is not the same then we need to rerender + if ( + this.videoGeneration.last_position_of_preprocessing != + this.engine_preprocessing + ) { + return false; + } + // this is slower to do so do this last. + const checksum = await this.save_manager.computeSceneChecksum(); + const decision = this.videoGeneration.shouldRenderScenesAgain(checksum); + return decision; + } + // Basicly Unity 3D's update loop. async updateLoop() { setTimeout( @@ -1169,7 +1182,16 @@ class Editor { } // This initializes the generation of a video render scene is where the core work happens - generateVideo() { + async generateVideo() { + // cannot run this function reliably without ensuring state below doesn't blow everything up. + if (await this.checkAndUseCache()) { + console.log("Checking Cache"); + await this.videoGeneration.handleCachedEnqueue(); + return; + } + + // some state changes below + console.log("Generating Video", this.frame_buffer); this.timeline.is_playing = false; @@ -1182,7 +1204,12 @@ class Editor { this.showLoading(); - this.rendering = true; // has to go first to debounce + // This for debouncing and also trigging the toggle playback... + // has to be here or will break play back ... + this.rendering = true; + + console.log("Running without Cache"); + this.togglePlayback(); this.frame_buffer = []; this.render_timer = 0; @@ -1195,12 +1222,27 @@ class Editor { } } + // In first case where if it cached data this is for people to reprompt without leaving the app + // Skip performing any of this because we have not changed the scene, then exit the scene. + // This avoids any unknown state issues from the resulting code below and escapes all the random state changes. + // This sadly doesn't cover camera framing style changes, + // Reasoning is that there is no defined interface to get changes like that. + async checkAndUseCache(): Promise { + if (await this.useCachedMediaTokens()) { + console.log("Using Cache"); + return true; + } else { + console.log("Not Using Cache"); + return false; + } + } + togglePlayback() { this.updateLoad({ progress: 25, label: "Starting Processing", message: - "Please stay on this screen while your video is being processed.", + "Please stay on this screen and do not switch tabs! while your video is being processed.", }); if (this.rawRenderer) { this.startRenderWidth = this.rawRenderer.domElement.width; diff --git a/app/pages/PageEnigma/Editor/keybinds_controls.ts b/app/pages/PageEnigma/Editor/keybinds_controls.ts index 4278613efa..484da20351 100644 --- a/app/pages/PageEnigma/Editor/keybinds_controls.ts +++ b/app/pages/PageEnigma/Editor/keybinds_controls.ts @@ -253,6 +253,7 @@ export class MouseControls { this.timeline_mouse = this.mouse; if (this.isMouseClicked) { + // this causes an issue https://discourse.threejs.org/t/unable-to-use-pointer-lock-api/11092 this.lockControls?.lock(); } else { this.lockControls?.unlock(); diff --git a/app/pages/PageEnigma/Editor/save_manager.ts b/app/pages/PageEnigma/Editor/save_manager.ts index 71a696340f..7ec0b5715f 100644 --- a/app/pages/PageEnigma/Editor/save_manager.ts +++ b/app/pages/PageEnigma/Editor/save_manager.ts @@ -22,7 +22,58 @@ export class SaveManager { this.editor = editor; } - // JSON structure + async computeHashForBrowser(data: string): Promise { + // Encode the string data to a Uint8Array + const encoder = new TextEncoder(); + const encodedData = encoder.encode(data); + + // Compute the hash using the SubtleCrypto.digest method + const hashBuffer = await crypto.subtle.digest("SHA-256", encodedData); + + // Convert the ArrayBuffer to a hexadecimal string + const hashArray = Array.from(new Uint8Array(hashBuffer)); + const hashHex = hashArray + .map((b) => b.toString(16).padStart(2, "0")) + .join(""); + + return hashHex; + } + + public async computeSceneChecksum(): Promise { + const jsonString = await this.checkSumData(); + return this.computeHashForBrowser(jsonString); + } + + private async checkSumData(): Promise { + const proxyScene = new StoryTellerProxyScene( + this.editor.version, + this.editor.activeScene, + ); + const scene_json = await proxyScene.saveToScene(this.editor.version); + + const proxyTimeline = new StoryTellerProxyTimeline( + this.editor.version, + this.editor.timeline, + this.editor.transform_engine, + this.editor.animation_engine, + this.editor.audio_engine, + this.editor.lipsync_engine, + this.editor.emotion_engine, + ); + const timeline_json = await proxyTimeline.saveToJson(); + const check_sum_data = { + scene: scene_json, + timeline: timeline_json, + camera_data: { + position: this.editor.camera?.position, + rotation: this.editor.camera?.rotation, + }, + }; + const jsonString = JSON.stringify(check_sum_data); + return jsonString; + } + + // JSON structure should and can return snapshot public async saveData({ sceneGenerationMetadata, }: { @@ -58,8 +109,9 @@ export class SaveManager { rotation: this.editor.camera?.rotation, }, }; - - return JSON.stringify(save_data); + // take json scene and figure out checksum + const jsonString = JSON.stringify(save_data); + return jsonString; } // TODO Move this function into scene manager. @@ -214,6 +266,6 @@ export class SaveManager { hideEditorLoader(); // TODO figure out if this is a bug. - this.editor.timeline.scrub({ data: { currentTime: 0 } }); + this.editor.timeline.scrub({ currentTime: 0 }); } } diff --git a/app/pages/PageEnigma/Editor/scene.ts b/app/pages/PageEnigma/Editor/scene.ts index 6bc1704282..db9948233a 100644 --- a/app/pages/PageEnigma/Editor/scene.ts +++ b/app/pages/PageEnigma/Editor/scene.ts @@ -31,6 +31,10 @@ class Scene { updateSurfaceIdAttributeToMesh: Function; + // This is used to ensure we do not rerender or process the video if we already have done so. + // This allows us to reprompt things quickly. This is only written when a snap shot is taken. + // Which guarentees that the scene is availible. + current_scene_checksum: string; constructor( name: string, camera_name: string, @@ -50,6 +54,8 @@ class Scene { this.camera_name = camera_name; this.placeholder_manager = undefined; this.updateSurfaceIdAttributeToMesh = updateSurfaceIdAttributeToMesh; + + this.current_scene_checksum = ""; } initialize() { diff --git a/app/pages/PageEnigma/Editor/timeline.ts b/app/pages/PageEnigma/Editor/timeline.ts index 14d3838354..b782b96d20 100644 --- a/app/pages/PageEnigma/Editor/timeline.ts +++ b/app/pages/PageEnigma/Editor/timeline.ts @@ -60,6 +60,9 @@ export class TimeLine { camera_name: string; // ensure that the elements are loaded first. + + private debounce_generate_video: boolean; + constructor( editor: Editor, audio_engine: AudioEngine, @@ -91,7 +94,7 @@ export class TimeLine { this.emotion_engine = emotion_engine; this.scene = scene; - + this.debounce_generate_video = false; Queue.subscribe( QueueNames.TO_ENGINE, "engine", @@ -226,10 +229,18 @@ export class TimeLine { } break; case toEngineActions.GENERATE_VIDEO: { - const options = data.data; // super overloaded talk to the devs about this. TODO... refactor - // pass this in ... rather than doing it implicitly ... - this.editorEngine.generation_options = options as IGenerationOptions; - this.editorEngine.generateVideo(); + console.log("Calling Generate Video"); + // debounce generate video ... + if (this.debounce_generate_video == false) { + this.debounce_generate_video = true; + const options = data.data; // super overloaded talk to the devs about this. TODO... refactor + // pass this in ... rather than doing it implicitly ... + this.editorEngine.generation_options = options as IGenerationOptions; + + await this.editorEngine.generateVideo(); + this.debounce_generate_video = false; + } + console.log("Finished! Generate Video"); break; } case toEngineActions.CHANGE_CAMERA_ASPECT_RATIO: { diff --git a/app/pages/PageEnigma/Editor/video_generation.ts b/app/pages/PageEnigma/Editor/video_generation.ts index b586a68833..272953faf2 100644 --- a/app/pages/PageEnigma/Editor/video_generation.ts +++ b/app/pages/PageEnigma/Editor/video_generation.ts @@ -11,19 +11,40 @@ import { Visibility } from "./api_manager.js"; import * as THREE from "three"; import { getSceneSignals } from "~/signals"; import { v4 as uuidv4 } from "uuid"; -import { SceneGenereationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata"; -import { MediaUploadApi } from "~/Classes/ApiManager"; +import { SceneGenereationMetaData as SceneGenerationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata"; +import { MediaUploadApi, VideoApi } from "~/Classes/ApiManager"; import { globalIPAMediaToken } from "../signals"; +import { addToast } from "~/signals"; // TODO THIS CLASS MAKES NO SENSE Refactor so we generate all the frames first. then pass it through this pipeline as a data structure process it. through this class. +import { startPollingActiveJobs } from "~/signals"; + +interface MediaTokens { + color: string; + normal: string; + depth: string; + outline: string; +} export class VideoGeneration { editor: Editor; mediaUploadAPI: MediaUploadApi; + videoAPI: VideoApi; + + // For cached style Re-Generation + private last_scene_check_sum: string; + // Last Media token IDs + private last_media_tokens: MediaTokens; + // Last toggle position for Render + public last_position_of_preprocessing: boolean; constructor(editor: Editor) { this.editor = editor; this.mediaUploadAPI = new MediaUploadApi(); + this.videoAPI = new VideoApi(); + this.last_scene_check_sum = ""; + this.last_media_tokens = { color: "", normal: "", depth: "", outline: "" }; + this.last_position_of_preprocessing = false; } sleep(ms: number): Promise { @@ -186,35 +207,165 @@ export class VideoGeneration { ); } - async stopPlaybackAndUploadVideo(compile_audio: boolean = true) { - this.editor.rendering = false; + // to determine if we should capture frames again or not. + async shouldRenderScenesAgain(checkSumData: string): Promise { + // Should skip to rendering the scene if already processed the frames inputs for running a new style. + if (this.last_scene_check_sum === "") { + // no token means input was not processed so lets go do the video re-render process + this.last_scene_check_sum = checkSumData; + return false; + } else { + if (this.last_scene_check_sum === checkSumData) { + return true; + } else { + this.last_scene_check_sum = checkSumData; + return false; + } + } + } - //const videoBlob = new Blob(this.frame_buffer, { type: "video/webm" }); - //const videoURL = URL.createObjectURL(videoBlob); + async stopPlaybackAndUploadVideo(compile_audio: boolean = true) { + this.editor.rendering = false; // not sure about this variable ... this has so many state issues. + + // precondition checks, if we have no frames then we shouldn't try to do generations or snapshots + // This means no frames so error out + + if (this.editor.frame_buffer.length <= 0) { + await this.handleError( + "Failed to Render Nothing to Animate in the Scene.", + 5000, + ); + return; + } + + if (this.editor.frame_buffer[0].length <= 0) { + await this.handleError( + "Failed to Render Nothing to Animate in the Scene.", + 5000, + ); + return; + } this.editor.generating_preview = true; - this.editor.updateLoad({ + await this.editor.updateLoad({ progress: 50, message: - "Please stay on this screen while your video is being processed.", + "Please stay on this screen and do not switch tabs! while your video is being processed.", }); - const upload_tokens = { + const media_tokens: MediaTokens = { color: "", normal: "", depth: "", outline: "", }; - // TODO fix and ensure that render fast, would actually render fast. + // properties used by the external scope to the for loop. + const title = getSceneSignals().title || "Untitled"; + const style_name = this.editor.art_style.toString(); + const media_token = this.editor.current_scene_media_token || undefined; + + // TAKE A snap shot of the scene then use this token across all videos + // convert the ip adapter image and upload as a media token + const image_uuid = uuidv4(); + let ipa_image_token = undefined; + + if (this.editor.globalIpAdapterImage != undefined) { + const response = await this.mediaUploadAPI.UploadImage({ + fileName: `${image_uuid}.ipa`, + blob: this.editor.globalIpAdapterImage, + uuid: image_uuid, + }); + + if (response.success) { + if (response.data) { + ipa_image_token = response.data; + } + } else { + // ip adapter upload failed + await this.handleError( + "Reference Image Upload Failed Try Generating Movie Again.", + 5000, + ); + await this.handleSuccess( + "Done Check Your Movies Tab On Profile.", + 5000, + ); + return; + } + } + + // update the signal with this information + if (ipa_image_token) { + globalIPAMediaToken.value = ipa_image_token; + } + + // TODO Remove so many of these around wtf. SceneGenereationMetaData should only be one place + const metaData: SceneGenerationMetaData = { + artisticStyle: this.editor.art_style, + positivePrompt: this.editor.positive_prompt, + negativePrompt: this.editor.negative_prompt, + cameraAspectRatio: this.editor.render_camera_aspect_ratio, + upscale: this.editor.generation_options.upscale, + faceDetail: this.editor.generation_options.faceDetail, + styleStrength: this.editor.generation_options.styleStrength, + lipSync: this.editor.generation_options.lipSync, + cinematic: this.editor.generation_options.cinematic, + globalIPAMediaToken: ipa_image_token, + enginePreProcessing: this.editor.engine_preprocessing, // the only thing that will invalidate the cache + }; + + // for the one case where the engine preprecessing is turned we need to cache this. + this.last_position_of_preprocessing = this.editor.engine_preprocessing; + + // This is to save the snapshot of the scene for remixing + const uuid_snapshot = uuidv4(); + + // Save the scene + const saveData = await this.editor.save_manager.saveData({ + sceneTitle: title, + sceneToken: media_token, + sceneGenerationMetadata: metaData, + }); + + const file = new File([saveData], `${title}.glb`, { + type: "application/json", + }); + + const response = + await this.editor.media_upload.UploadSceneSnapshotMediaFileForm({ + maybe_title: title, + maybe_scene_source_media_file_token: media_token, // can be undefined or null + uuid: uuid_snapshot, + blob: file, + }); + + let immutable_media_token = undefined; + if (response.success) { + if (response.data) { + immutable_media_token = response.data; + } + } else { + await this.handleError( + "Scene Snapshot Failed Try Generating Movie Again.", + 5000, + ); + return; + } + + console.log(`Immutable Snapshot Token: ${immutable_media_token}`); + for ( let image_index = 0; image_index < this.editor.frame_buffer[0].length; image_index++ ) { - // Write the Uint8Array to the FFmpeg file system - //ffmpeg.FS("writeFile", "input.webm", await fetchFile(videoURL)); + await this.editor.updateLoad({ + progress: 50 + image_index * 5, + message: + "Please stay on this screen and do not switch tabs! while your video is being processed.", + }); const ffmpeg = createFFmpeg({ log: false }); await ffmpeg.load(); @@ -260,176 +411,172 @@ export class VideoGeneration { "0tmp.mp4", ); - let itteration = 0; + let iteration = 0; if (compile_audio) { for (const clip of this.editor.timeline.timeline_items) { if (clip.type == ClipType.AUDIO) { - await this.convertAudioClip(itteration, ffmpeg, clip); - itteration += 1; + await this.convertAudioClip(iteration, ffmpeg, clip); + iteration += 1; } } } - const output = ffmpeg.FS("readFile", itteration + "tmp.mp4"); - + // Upload individual videos or single color video + const output = ffmpeg.FS("readFile", iteration + "tmp.mp4"); ffmpeg.exit(); - this.editor.generating_preview = false; + this.editor.generating_preview = false; // Create a Blob from the output file for downloading const blob = new Blob([output.buffer], { type: "video/mp4" }); - const title = getSceneSignals().title || "Untitled"; - - const style_name = this.editor.art_style.toString(); - const media_token = this.editor.current_scene_media_token || undefined; - - // convert the ip adapter image and upload as a media token - const image_uuid = uuidv4(); - let ipa_image_token = undefined; - - if (this.editor.globalIpAdapterImage != undefined) { - const response = await this.mediaUploadAPI.UploadImage({ - fileName: `${image_uuid}.ipa`, - blob: this.editor.globalIpAdapterImage, - uuid: image_uuid, - }); - if (response.success) { - if (response.data) { - ipa_image_token = response.data; - } - } - } - if (ipa_image_token) { - globalIPAMediaToken.value = ipa_image_token; - } - - // TODO Remove so many of these around wtf. SceneGenereationMetaData should only be one place - const metaData: SceneGenereationMetaData = { - artisticStyle: this.editor.art_style, - positivePrompt: this.editor.positive_prompt, - negativePrompt: this.editor.negative_prompt, - cameraAspectRatio: this.editor.render_camera_aspect_ratio, - upscale: this.editor.generation_options.upscale, - faceDetail: this.editor.generation_options.faceDetail, - styleStrength: this.editor.generation_options.styleStrength, - lipSync: this.editor.generation_options.lipSync, - cinematic: this.editor.generation_options.cinematic, - globalIPAMediaToken: globalIPAMediaToken.value, - }; - - // This is to save the snapshot of the scene for remixing... - const uuid_snapshot = uuidv4(); - - const saveData = await this.editor.save_manager.saveData({ - sceneTitle: title, - sceneToken: media_token, - sceneGenerationMetadata: metaData, - }); - - const file = new File([saveData], `${title}.glb`, { - type: "application/json", - }); - - const response = - await this.editor.media_upload.UploadSceneSnapshotMediaFileForm({ - maybe_title: title, - maybe_scene_source_media_file_token: media_token, // can be undefined or null - uuid: uuid_snapshot, - blob: file, - }); - - let immutable_media_token = undefined; - if (response.success) { - if (response.data) { - immutable_media_token = response.data; - } - } else { - //console.log("ERROR:"); - console.log(response.errorMessage); - } - - console.log("Immutable Token:"); - console.log(immutable_media_token); - - /// TODO refactor this whole thing extract this out. - const data: any = await this.editor.api_manager.uploadMedia({ - blob, + const video_upload_response = await this.mediaUploadAPI.UploadNewVideo({ + blob: blob, fileName: `${title}.mp4`, - title, - styleName: style_name, + uuid: uuidv4(), + maybe_title: title, + maybe_visibility: Visibility.Public, + maybe_style_name: style_name, maybe_scene_source_media_file_token: immutable_media_token, }); - if (data == null) { + if (video_upload_response.success) { + if (video_upload_response.data) { + const upload_token = video_upload_response.data; + console.log(upload_token); + // saves these tokens for a rerun. + if (image_index === 0) { + media_tokens.color = upload_token; + } else if (image_index === 1) { + media_tokens.normal = upload_token; + } else if (image_index === 2) { + media_tokens.depth = upload_token; + } else if (image_index === 3) { + media_tokens.outline = upload_token; + } + } + } else { + await this.handleError( + "Failed To Preprocess: Generate Please Try Again", + 5000, + ); return; } - const upload_token = data["media_file_token"]; - - console.log(upload_token); - - if (image_index === 0) { - upload_tokens.color = upload_token; - } else if (image_index === 1 && this.editor.engineRenderSeparately == false) { - upload_tokens.normal = upload_token; - } else if (image_index === 2) { - upload_tokens.depth = upload_token; - } else if (image_index === 3) { - upload_tokens.outline = upload_token; - } else if (this.editor.engineRenderSeparately ) { - console.log("https://storyteller.ai/media/" + upload_tokens.color); - console.log("https://storyteller.ai/media/" + upload_token); - } - - await this.sleep(2000); // TODO: REMOVE THIS WHEN TOO MANY REQUESTS ERROR IS SOLVED. - //console.log("Waiting for server to catch up..."); - } + } // end generation for color or multiple channels of data. this.editor.onWindowResize(); - console.log(upload_tokens); + console.log(media_tokens); + this.editor.setColorMap(); - // console.log("https://storyteller.ai/media/" + upload_tokens.color); - // console.log("https://storyteller.ai/media/" + upload_tokens.normal); - // console.log("https://storyteller.ai/media/" + upload_tokens.depth); - // console.log("https://storyteller.ai/media/" + upload_tokens.outline); + console.log(`https://storyteller.ai/media/${media_tokens.color}`); + console.log(`https://storyteller.ai/media/${media_tokens.normal}`); + console.log(`https://storyteller.ai/media/${media_tokens.depth}`); + console.log(`https://storyteller.ai/media/${media_tokens.outline}`); - await this.editor.api_manager - .stylizeVideo({ - media_token: upload_tokens.color, - style: this.editor.art_style, - positive_prompt: this.editor.positive_prompt, - negative_prompt: this.editor.negative_prompt, - visibility: Visibility.Public, - use_face_detailer: this.editor.generation_options.faceDetail, - use_upscaler: this.editor.generation_options.upscale, - use_strength: this.editor.generation_options.styleStrength, - use_lipsync: this.editor.generation_options.lipSync, - use_cinematic: this.editor.generation_options.cinematic, - use_global_ipa_media_token: globalIPAMediaToken.value, - input_depth_file: upload_tokens.depth, - input_normal_file: upload_tokens.normal, - input_outline_file: upload_tokens.outline, - }) - .catch((error) => { - console.log(error); - this.editor.updateLoad({ - progress: 100, - message: "Failed To Render", - label: "Error", - }); - this.editor.endLoading(); - }); - - this.editor.updateLoad({ + await this.editor.updateLoad({ progress: 100, - message: "Done Check Your Media Tab On Profile.", + message: "Done Check Your Movies Tab On Profile.", label: "Success", }); - this.editor.endLoading(); + // this is so that its a check point just encase enqueue fails, if it does we can still restylize + this.last_media_tokens = media_tokens; + await this.handleEnqueue(media_tokens, ipa_image_token ?? ""); + + await this.EndLoadingState(); + } + + async handleCachedEnqueue() { + const enqueue_studio_response = await this.videoAPI.EnqueueStudio({ + enqueueVideo: { + disable_lcm: false, + enable_lipsync: this.editor.generation_options.lipSync, + input_file: this.last_media_tokens.color, + negative_prompt: this.editor.negative_prompt, + prompt: this.editor.positive_prompt, + remove_watermark: false, + style: this.editor.art_style.toString(), + frame_skip: 2, + travel_prompt: "", + trim_end_millis: 7000, + trim_start_millis: 0, + use_cinematic: this.editor.generation_options.cinematic, + use_face_detailer: this.editor.generation_options.faceDetail, + use_strength: this.editor.generation_options.styleStrength, + use_upscaler: this.editor.generation_options.upscale, + uuid_idempotency_token: uuidv4(), + global_ipa_media_token: globalIPAMediaToken.value ?? "", + input_depth_file: this.last_media_tokens.depth, + input_normal_file: this.last_media_tokens.normal, + input_outline_file: this.last_media_tokens.outline, + creator_set_visibility: Visibility.Public, + }, + }); + + if (enqueue_studio_response.success) { + console.log("Start Polling Active Jobs"); + startPollingActiveJobs(); + addToast( + ToastTypes.SUCCESS, + "Done Check Your Movies Tab On Profile.", + 5000, + ); + } else { + addToast(ToastTypes.ERROR, "Failed To Process Movie Try Again", 5000); + } + } + + async handleEnqueue(upload_tokens: MediaTokens, ipa_image_token: string) { + const enqueue_studio_response = await this.videoAPI.EnqueueStudio({ + enqueueVideo: { + disable_lcm: false, + enable_lipsync: this.editor.generation_options.lipSync, + input_file: upload_tokens.color, + negative_prompt: this.editor.negative_prompt, + prompt: this.editor.positive_prompt, + remove_watermark: false, + style: this.editor.art_style.toString(), + frame_skip: 2, + travel_prompt: "", + trim_end_millis: 7000, + trim_start_millis: 0, + use_cinematic: this.editor.generation_options.cinematic, + use_face_detailer: this.editor.generation_options.faceDetail, + use_strength: this.editor.generation_options.styleStrength, + use_upscaler: this.editor.generation_options.upscale, + uuid_idempotency_token: uuidv4(), + global_ipa_media_token: ipa_image_token, + input_depth_file: upload_tokens.depth, + input_normal_file: upload_tokens.normal, + input_outline_file: upload_tokens.outline, + creator_set_visibility: Visibility.Public, + }, + }); + + if (enqueue_studio_response.success) { + startPollingActiveJobs(); + } else { + await this.handleError("Failed To Process Movie Try Again", 5000); + return; + } + + this.handleSuccess("Done Check Your Movies Tab On Profile.", 5000); + } + + async handleSuccess(message: string, timeout: number) { + addToast(ToastTypes.SUCCESS, message, timeout); + } + async handleError(message: string, timeout: number) { + addToast(ToastTypes.ERROR, message, timeout); + await this.EndLoadingState(); + } + async EndLoadingState() { + this.editor.generating_preview = false; + this.editor.endLoading(); + this.editor.onWindowResize(); this.editor.recorder = undefined; if (this.editor.rawRenderer) { this.editor.rawRenderer.setSize( diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/GenerateMovieButton.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/GenerateMovieButton.tsx index 9e7b37d823..02c2a5afe0 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/GenerateMovieButton.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/GenerateMovieButton.tsx @@ -52,7 +52,8 @@ export function GenerateMovieButton({ // eslint-disable-next-line react-hooks/exhaustive-deps }, []); - const generateMovie = async () => { + const generateMovie = async (event: React.MouseEvent) => { + event.preventDefault(); const options = new GenerationOptions( upscale.value, faceDetail.value, diff --git a/test/app/Classes/ApiManager/VideoApi.test.ts b/test/app/Classes/ApiManager/VideoApi.test.ts index 1e3d3a6c61..9c707e72d3 100644 --- a/test/app/Classes/ApiManager/VideoApi.test.ts +++ b/test/app/Classes/ApiManager/VideoApi.test.ts @@ -39,7 +39,7 @@ describe("VideoApi", () => { inference_job_token_type: "ijtt1", success: true, }); - const response = await videoApi.GenerateVideoStyleTransfer({ + const response = await videoApi.EnqueueStudio({ enqueueVideo: mockRequest, }); expect(videoApi.fetch).toHaveBeenCalledWith( @@ -69,7 +69,7 @@ describe("VideoApi", () => { inference_job_token_type: "ijtt1", success: true, }); - const response = await videoApi.GenerateVideoStyleTransfer({ + const response = await videoApi.EnqueueStudio({ enqueueVideo: { ...mockRequest, creator_set_visibility: Visibility.Private, @@ -100,7 +100,7 @@ describe("VideoApi", () => { jest.spyOn(videoApi, "fetch").mockResolvedValue({ BadInput: "error", }); - const response = await videoApi.GenerateVideoStyleTransfer({ + const response = await videoApi.EnqueueStudio({ enqueueVideo: mockRequest, }); expect(videoApi.fetch).toHaveBeenCalledWith( @@ -129,7 +129,7 @@ describe("VideoApi", () => { jest.spyOn(videoApi, "fetch").mockResolvedValue({ BadInput: "error", }); - const response = await videoApi.GenerateVideoStyleTransfer({ + const response = await videoApi.EnqueueStudio({ enqueueVideo: mockRequest, }); expect(videoApi.fetch).toHaveBeenCalledWith(