From 50889ed3584bd04fd4fec412ee4682426ea8f468 Mon Sep 17 00:00:00 2001 From: ArEnSc <6252325+ArEnSc@users.noreply.github.com> Date: Mon, 20 May 2024 13:36:13 -0700 Subject: [PATCH] Editor Refactoring (#402) Co-authored-by: Christopher Oliphant Co-authored-by: Quicksticks-oss --- .eslintrc.cjs | 2 + app/api.tsx | 9 +- .../reusable/FileWrapper/FileWrapper.scss | 48 + .../reusable/FileWrapper/LoadingIcon.tsx | 12 + .../Editor/APIManagerResponseError.ts | 12 + .../Editor/APIManagerResponseSuccess.ts | 14 + .../{js => Editor}/TransformControls.js | 0 .../{js => Editor}/animation_engine.ts | 2 - .../PageEnigma/{js => Editor}/api_fetchers.ts | 0 .../PageEnigma/{js => Editor}/api_manager.ts | 0 .../PageEnigma/{js => Editor}/audio_engine.ts | 0 app/pages/PageEnigma/{js => Editor}/editor.ts | 924 ++---------------- .../{js => Editor}/emotion_engine.ts | 0 .../PageEnigma/{js => Editor}/free_cam.js | 0 app/pages/PageEnigma/Editor/free_cam.ts | 218 +++++ app/pages/PageEnigma/Editor/helper.ts | 218 +++++ .../PageEnigma/Editor/keybinds_controls.ts | 140 +++ .../{js => Editor}/lip_sync_engine.ts | 2 - .../PageEnigma/{js => Editor}/lipsync.js | 0 .../{js => Editor}/placeholder_manager.ts | 0 .../PageEnigma/{js => Editor}/retargeting.ts | 0 app/pages/PageEnigma/Editor/save_manager.ts | 109 +++ app/pages/PageEnigma/{js => Editor}/scene.ts | 0 .../PageEnigma/{js => Editor}/timeline.ts | 249 +++-- .../{js => Editor}/transform_engine.ts | 0 .../PageEnigma/Editor/video_generation.ts | 301 ++++++ app/pages/PageEnigma/Queue/Queue.ts | 4 +- .../ControlPanelSceneObject.tsx | 11 +- .../itemTabs/StylizeTab/ItemPicker.tsx | 8 +- .../itemTabs/StylizeTab/Prompts.tsx | 2 +- .../itemTabs/StylizeTab/StyleSelection.tsx | 2 +- .../itemTabs/StylizeTab/StylizeTab.tsx | 2 +- .../comps/StyleSelection/ItemPicker.tsx | 2 +- .../comps/StyleSelection/StyleSelection.tsx | 2 +- .../comps/Timeline/utils/useQueueHandler.ts | 1 + app/pages/PageEnigma/constants/RandomTexts.ts | 2 +- .../contexts/EngineContext/EngineContext.tsx | 4 +- .../contexts/EngineContext/EngineProvider.tsx | 2 +- .../datastructures/clips/animation_clip.ts | 2 +- .../datastructures/clips/lipsync_clip.ts | 2 +- .../PageEnigma/enums/AssetFilterOption.ts | 5 + app/pages/PageEnigma/enums/AssetType.ts | 10 + app/pages/PageEnigma/models/movies.ts | 3 +- .../proxy/storyteller_proxy_scene.ts | 11 +- .../proxy/storyteller_proxy_timeline.ts | 309 +++--- app/pages/PageEnigma/styleList.ts | 2 +- 46 files changed, 1516 insertions(+), 1130 deletions(-) create mode 100644 app/components/reusable/FileWrapper/FileWrapper.scss create mode 100644 app/components/reusable/FileWrapper/LoadingIcon.tsx create mode 100644 app/pages/PageEnigma/Editor/APIManagerResponseError.ts create mode 100644 app/pages/PageEnigma/Editor/APIManagerResponseSuccess.ts rename app/pages/PageEnigma/{js => Editor}/TransformControls.js (100%) rename app/pages/PageEnigma/{js => Editor}/animation_engine.ts (94%) rename app/pages/PageEnigma/{js => Editor}/api_fetchers.ts (100%) rename app/pages/PageEnigma/{js => Editor}/api_manager.ts (100%) rename app/pages/PageEnigma/{js => Editor}/audio_engine.ts (100%) rename app/pages/PageEnigma/{js => Editor}/editor.ts (51%) rename app/pages/PageEnigma/{js => Editor}/emotion_engine.ts (100%) rename app/pages/PageEnigma/{js => Editor}/free_cam.js (100%) create mode 100644 app/pages/PageEnigma/Editor/free_cam.ts create mode 100644 app/pages/PageEnigma/Editor/helper.ts create mode 100644 app/pages/PageEnigma/Editor/keybinds_controls.ts rename app/pages/PageEnigma/{js => Editor}/lip_sync_engine.ts (94%) rename app/pages/PageEnigma/{js => Editor}/lipsync.js (100%) rename app/pages/PageEnigma/{js => Editor}/placeholder_manager.ts (100%) rename app/pages/PageEnigma/{js => Editor}/retargeting.ts (100%) create mode 100644 app/pages/PageEnigma/Editor/save_manager.ts rename app/pages/PageEnigma/{js => Editor}/scene.ts (100%) rename app/pages/PageEnigma/{js => Editor}/timeline.ts (76%) rename app/pages/PageEnigma/{js => Editor}/transform_engine.ts (100%) create mode 100644 app/pages/PageEnigma/Editor/video_generation.ts create mode 100644 app/pages/PageEnigma/enums/AssetFilterOption.ts create mode 100644 app/pages/PageEnigma/enums/AssetType.ts diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 3a0ebe55fa..f37d6469fd 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -23,6 +23,8 @@ module.exports = { // Base config extends: ["eslint:recommended", "plugin:storybook/recommended"], + ignorePatterns: ["storybook-static/*"], + overrides: [ // React { diff --git a/app/api.tsx b/app/api.tsx index c09019a66f..984c857963 100644 --- a/app/api.tsx +++ b/app/api.tsx @@ -1,4 +1,4 @@ -export const apiHost = "https://api.fakeyou.com"; +export const apiHost = "https://api.storyteller.ai"; // Authentications // const createAccount = `${apiHost}}/create_account`; @@ -19,7 +19,6 @@ export const uploadAudioV2V = `${apiHost}/v1/media_uploads/upload_audio`; export const listMediaByUser = (username: string) => `${apiHost}/v1/media_files/list/user/${username}`; - // Get Any Media export const getMediaFileByToken = (fileToken: string) => `${apiHost}/v1/media_files/file/${fileToken}`; @@ -30,5 +29,7 @@ export const uploadMedia = `${apiHost}/v1/media_files/upload`; // Scenes export const uploadNewScene = `${apiHost}/v1/media_files/upload/new_scene`; export const uploadThumbnail = `${apiHost}/v1/media_files/cover_image/`; -export const updateExistingScene = (sceneToken:string)=>`${apiHost}/v1/media_files/upload/saved_scene/${sceneToken}` -export const renameScene = (sceneToken:string)=>`${apiHost}/v1/media_files/rename/${sceneToken}` \ No newline at end of file +export const updateExistingScene = (sceneToken: string) => + `${apiHost}/v1/media_files/upload/saved_scene/${sceneToken}`; +export const renameScene = (sceneToken: string) => + `${apiHost}/v1/media_files/rename/${sceneToken}`; diff --git a/app/components/reusable/FileWrapper/FileWrapper.scss b/app/components/reusable/FileWrapper/FileWrapper.scss new file mode 100644 index 0000000000..02cf3c1fcc --- /dev/null +++ b/app/components/reusable/FileWrapper/FileWrapper.scss @@ -0,0 +1,48 @@ +.fy-file-wrapper { + // flex-grow: 1; + // height: calc(100% - 4px); + + .file-picker-button { + cursor: pointer; + } + + .fy-file-wrapper-input { + visibility: hidden; + height: 0; + width: 0; + // overflow: hidden; + position: absolute; + + + label { + cursor: pointer; + } + } + .fy-uploader-layout { + display: flex; + align-items: center; + width: 100%; + + .fy-uploader-layout-icon { + font-size: 2.625rem; + margin-right: 1rem; + } + } +} + +.fy-file-dragging { + background: rgba(#676781, 0.25); + border-color: rgba(#fff, 0.33); +} + +.fy-uploader { + [type="file"] { + + label { + border: dashed 2px rgba(255, 255, 255, 0.12) !important; + border-radius: 0.5rem; + transition: 0.1s all; + &:hover { + // background-color: lighten($panel, 6%); + } + } + } +} diff --git a/app/components/reusable/FileWrapper/LoadingIcon.tsx b/app/components/reusable/FileWrapper/LoadingIcon.tsx new file mode 100644 index 0000000000..dce6894198 --- /dev/null +++ b/app/components/reusable/FileWrapper/LoadingIcon.tsx @@ -0,0 +1,12 @@ +import React from "react"; + +export default function LoadingIcon() { + return <> + + Loading... + ; +}; \ No newline at end of file diff --git a/app/pages/PageEnigma/Editor/APIManagerResponseError.ts b/app/pages/PageEnigma/Editor/APIManagerResponseError.ts new file mode 100644 index 0000000000..5915e4c77f --- /dev/null +++ b/app/pages/PageEnigma/Editor/APIManagerResponseError.ts @@ -0,0 +1,12 @@ +/** + * This is designed to surface user customer facing messages as errors. + * Errors shouldn't be 404 or something confusing should be + */ +class APIManagerResponseError extends Error { + constructor(message?: string) { + super(message); + this.name = "APIManagerResponseError"; + } +} + +export { APIManagerResponseError }; diff --git a/app/pages/PageEnigma/Editor/APIManagerResponseSuccess.ts b/app/pages/PageEnigma/Editor/APIManagerResponseSuccess.ts new file mode 100644 index 0000000000..2e619709e5 --- /dev/null +++ b/app/pages/PageEnigma/Editor/APIManagerResponseSuccess.ts @@ -0,0 +1,14 @@ +/** + * This is designed to surface user customer facing messages as errors. + */ +type Data = { [key: string]: any }; +class APIManagerResponseSuccess { + public user_message: string; + public data: Data | null; + constructor(user_message: string = "", data: Data | null = null) { + this.data = data; + this.user_message = user_message; + } +} + +export { APIManagerResponseSuccess }; diff --git a/app/pages/PageEnigma/js/TransformControls.js b/app/pages/PageEnigma/Editor/TransformControls.js similarity index 100% rename from app/pages/PageEnigma/js/TransformControls.js rename to app/pages/PageEnigma/Editor/TransformControls.js diff --git a/app/pages/PageEnigma/js/animation_engine.ts b/app/pages/PageEnigma/Editor/animation_engine.ts similarity index 94% rename from app/pages/PageEnigma/js/animation_engine.ts rename to app/pages/PageEnigma/Editor/animation_engine.ts index 4abf0c1fcc..fe22058186 100644 --- a/app/pages/PageEnigma/js/animation_engine.ts +++ b/app/pages/PageEnigma/Editor/animation_engine.ts @@ -22,5 +22,3 @@ export class AnimationEngine { //this.clips[object_uuid + media_id]._load_animation(); } } - -export default AnimationEngine; diff --git a/app/pages/PageEnigma/js/api_fetchers.ts b/app/pages/PageEnigma/Editor/api_fetchers.ts similarity index 100% rename from app/pages/PageEnigma/js/api_fetchers.ts rename to app/pages/PageEnigma/Editor/api_fetchers.ts diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/Editor/api_manager.ts similarity index 100% rename from app/pages/PageEnigma/js/api_manager.ts rename to app/pages/PageEnigma/Editor/api_manager.ts diff --git a/app/pages/PageEnigma/js/audio_engine.ts b/app/pages/PageEnigma/Editor/audio_engine.ts similarity index 100% rename from app/pages/PageEnigma/js/audio_engine.ts rename to app/pages/PageEnigma/Editor/audio_engine.ts diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/Editor/editor.ts similarity index 51% rename from app/pages/PageEnigma/js/editor.ts rename to app/pages/PageEnigma/Editor/editor.ts index c86ed8c0d3..49ed54751e 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/Editor/editor.ts @@ -3,7 +3,7 @@ import { OrbitControls } from "three/addons/controls/OrbitControls.js"; import { FreeCam } from "./free_cam"; import { TransformControls } from "./TransformControls.js"; import Scene from "./scene.js"; -import { APIManager, ArtStyle, Visibility } from "./api_manager.js"; +import { APIManager, ArtStyle } from "./api_manager.js"; import { EffectComposer } from "three/addons/postprocessing/EffectComposer.js"; import { RenderPass } from "three/addons/postprocessing/RenderPass.js"; import { OutlinePass } from "three/addons/postprocessing/OutlinePass.js"; @@ -11,57 +11,33 @@ import { OutputPass } from "three/addons/postprocessing/OutputPass.js"; import { SMAAPass } from "three/addons/postprocessing/SMAAPass.js"; import { SAOPass } from "three/addons/postprocessing/SAOPass.js"; import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js"; -import { BokehPass } from "three/addons/postprocessing/BokehPass.js"; -import { createFFmpeg, fetchFile, FFmpeg } from "@ffmpeg/ffmpeg"; - import AudioEngine from "./audio_engine.js"; import TransformEngine from "./transform_engine.js"; import EmotionEngine from "./emotion_engine"; - import { TimeLine } from "./timeline.js"; -import { ClipUI } from "../datastructures/clips/clip_ui.js"; import { LipSyncEngine } from "./lip_sync_engine.js"; import { AnimationEngine } from "./animation_engine.js"; - import { PointerLockControls } from "three/addons/controls/PointerLockControls.js"; -// import { APPUI_ACTION_TYPES, AppUiAction } from "~/contexts/reducers"; -import { ClipGroup, ClipType, EditorStates } from "~/pages/PageEnigma/enums"; - +import { ClipGroup, EditorStates } from "~/pages/PageEnigma/enums"; +import { AssetType } from "~/enums"; import { XYZ } from "../datastructures/common"; -import { StoryTellerProxyScene } from "../proxy/storyteller_proxy_scene"; -import { StoryTellerProxyTimeline } from "../proxy/storyteller_proxy_timeline"; import Queue from "~/pages/PageEnigma/Queue/Queue"; import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions"; import { MediaItem } from "~/pages/PageEnigma/models"; -import { editorState, previewSrc } from "~/pages/PageEnigma/signals/engine"; - -import { GenerationOptions } from "../models/generationOptions"; -import { - hotkeysStatus, - showEditorLoader, - hideEditorLoader, - showObjectPanel, - hideObjectPanel, - updateObjectPanel, -} from "~/pages/PageEnigma/signals"; +import { editorState } from "../signals/engine"; +import { Utils } from "./helper"; +import { VideoGeneration } from "./video_generation"; +import { MouseControls } from "./keybinds_controls"; +import { SaveManager } from "./save_manager"; import { authentication, loadingBarData, loadingBarIsShowing, - getSceneSignals, signalScene, } from "~/signals"; -import { ToastTypes, AssetType } from "~/enums"; - -// Main editor class that will call everything else all you need to call is " initialize() ". - -export type EditorConstructorConfig = { - // dispatchAppUiState: React.Dispatch; - // signalScene: (data: any) => void; - // getSceneSignals: () => SceneSignal; - // userToken: string; -}; +import { updateObjectPanel } from "../signals"; +import { GenerationOptions } from "../models/generationOptions"; export type EditorInitializeConfig = { sceneToken: string; @@ -70,15 +46,14 @@ export type EditorInitializeConfig = { class Editor { version: number; activeScene: Scene; - camera: any; - render_camera: any; + camera: THREE.PerspectiveCamera | null = null; + render_camera: THREE.PerspectiveCamera | null = null; renderer: THREE.WebGLRenderer | undefined; rawRenderer: THREE.WebGLRenderer | undefined; clock: THREE.Clock | undefined; - canvReference: any; - canvasRenderCamReference: any; + canvReference: HTMLCanvasElement | undefined = undefined; + canvasRenderCamReference: HTMLElement | undefined = undefined; composer: EffectComposer | undefined; - effectFXAA: EffectComposer | undefined; outlinePass: OutlinePass | undefined; last_cam_pos: THREE.Vector3; last_cam_rot: THREE.Euler; @@ -86,29 +61,24 @@ class Editor { outputPass: OutputPass | undefined; bloomPass: UnrealBloomPass | undefined; smaaPass: SMAAPass | undefined; - bokehPass: BokehPass | undefined; control: TransformControls | undefined; raycaster: THREE.Raycaster | undefined; mouse: THREE.Vector2 | undefined; selected: THREE.Object3D | undefined; last_selected: THREE.Object3D | undefined; last_selected_sum: number | undefined; - transform_interaction: any; + transform_interaction = false; rendering: boolean; api_manager: APIManager; - stats: any; cameraViewControls: FreeCam | undefined; orbitControls: OrbitControls | undefined; locked: boolean; - capturer: any; - frame_buffer: any; + frame_buffer: string[] = []; render_timer: number; fps_number: number; cap_fps: number; can_playback: boolean; - playback: boolean; playback_location: number; - max_length: number; audio_engine: AudioEngine; transform_engine: TransformEngine; emotion_engine: EmotionEngine; @@ -142,9 +112,8 @@ class Editor { art_style: ArtStyle; last_scrub: number; - record_stream: any | undefined; recorder: MediaRecorder | undefined; - container: any | undefined; + container: HTMLElement | null = null; selectedCanvas: boolean; startRenderHeight: number; @@ -155,8 +124,12 @@ class Editor { // global names of scene entities camera_name: string; - generation_options: GenerationOptions; + utils: Utils; + videoGeneration: VideoGeneration; + mouse_controls: MouseControls; + save_manager: SaveManager; + generation_options: GenerationOptions; constructor() { console.log( "If you see this message twice! then it rendered twice, if you see it once it's all good.", @@ -180,56 +153,26 @@ class Editor { this.activeScene = new Scene("" + this.version, this.camera_name); this.activeScene.initialize(); this.generating_preview = false; - this.camera; - this.render_camera; - this.renderer; - this.rawRenderer; - this.clock; - this.canvReference = null; - this.cam_obj; - this.composer; - this.effectFXAA; - this.outlinePass; this.last_cam_pos = new THREE.Vector3(0, 0, 0); this.last_cam_rot = new THREE.Euler(0, 0, 0); this.camera_last_pos = new THREE.Vector3(0, 0, 0); this.startRenderWidth = 0; this.startRenderHeight = 0; - this.lockControls; - this.saoPass; - this.outputPass; - this.bloomPass; - this.smaaPass; - this.bokehPass; - // Transform control and selection. - this.control; - this.raycaster; - this.mouse; - this.selected; - this.last_selected; - this.transform_interaction; this.rendering = false; this.lastCanvasSize = 0; this.switchPreviewToggle = false; // API. this.api_manager = new APIManager(); // Debug & Movement. - this.stats = null; - this.cameraViewControls; - this.orbitControls; this.camera_person_mode = false; this.locked = false; // Recording params. - this.capturer = null; - this.frame_buffer = []; this.render_timer = 0; this.fps_number = 60; this.cap_fps = 60; // Timeline settings. this.can_playback = false; - this.playback = false; this.playback_location = 0; - this.max_length = 10; this.last_scrub = 0; this.frames = 0; this.last_selected_sum = 0; @@ -239,8 +182,6 @@ class Editor { this.render_width = 1280; this.render_height = 720; - this.canvasRenderCamReference; - this.audio_engine = new AudioEngine(); this.emotion_engine = new EmotionEngine(this.version); this.transform_engine = new TransformEngine(this.version); @@ -260,13 +201,12 @@ class Editor { this.camera_name, ); - this.current_frame = 0; + this.utils = new Utils(this, this.activeScene); + this.videoGeneration = new VideoGeneration(this); + this.mouse_controls = new MouseControls(this); + this.save_manager = new SaveManager(this); - //setup reactland Callbacks - // this.dispatchAppUiState = dispatchAppUiState; - // this.signalScene = signalScene; - // this.getSceneSignals = getSceneSignals; - // this.userToken = userToken; + this.current_frame = 0; // Scene State this.current_scene_media_token = null; @@ -285,22 +225,22 @@ class Editor { }; } - isEmpty(value: string) { - return ( - value == null || (typeof value === "string" && value.trim().length === 0) - ); + isEmpty(value: string | null) { + return value === null || value.trim().length === 0; } initialize({ sceneToken }: EditorInitializeConfig) { - if (this.can_initialize == false) { + if (!this.can_initialize) { console.log("Editor Already Initialized"); return; } this.can_initialize = false; // Gets the canvas. - this.canvReference = document.getElementById("video-scene"); - this.canvasRenderCamReference = document.getElementById("camera-view"); + this.canvReference = (document.getElementById("video-scene") ?? + undefined) as HTMLCanvasElement | undefined; + this.canvasRenderCamReference = + document.getElementById("camera-view") ?? undefined; // Find the container element const container = document.getElementById("video-scene-container"); @@ -400,9 +340,21 @@ class Editor { console.log("Control Sensitivity:", this.control.sensitivity); // OnClick and MouseMove events. - window.addEventListener("mousemove", this.onMouseMove.bind(this), false); - window.addEventListener("click", this.onMouseClick.bind(this), false); - window.addEventListener("keydown", this.onkeydown.bind(this), false); + window.addEventListener( + "mousemove", + this.mouse_controls.onMouseMove.bind(this.mouse_controls), + false, + ); + window.addEventListener( + "click", + this.mouse_controls.onMouseClick.bind(this.mouse_controls), + false, + ); + window.addEventListener( + "keydown", + this.mouse_controls.onkeydown.bind(this.mouse_controls), + false, + ); // Base control and debug stuff remove debug in prod. if (this.control == undefined) { return; @@ -432,16 +384,14 @@ class Editor { this.timeline.scene = this.activeScene; - //this._test_demo(); - this.renderer.domElement.addEventListener( "mousedown", - this.onMouseDown.bind(this), + this.mouse_controls.onMouseDown.bind(this.mouse_controls), false, ); this.renderer.domElement.addEventListener( "mouseup", - this.onMouseUp.bind(this), + this.mouse_controls.onMouseUp.bind(this.mouse_controls), false, ); @@ -456,14 +406,14 @@ class Editor { this.updateLoop(); - if (this.isEmpty(sceneToken) == false) { + if (!this.utils.isEmpty(sceneToken)) { this.loadScene(sceneToken); } else { signalScene({ - isModified: false, title: "Untitled New Scene", token: undefined, ownerToken: authentication.userInfo.value?.user_token, + isModified: false, }); } @@ -520,10 +470,10 @@ class Editor { ? sceneTitleInput : "Untitled New Scene"; signalScene({ - isModified: false, title: sceneTitle, token: undefined, ownerToken: authentication.userInfo.value?.user_token, + isModified: false, }); Queue.publish({ queueName: QueueNames.FROM_ENGINE, @@ -532,105 +482,20 @@ class Editor { }); } - // Token comes in from the front end to load the scene from the site. - public async testBatchRequest() { - const result = await this.api_manager.getMediaBatch([ - "m_8fmp9hrvsqcryzka1fra597kg42s50", - "m_z4jzbst3xfh64h0qn4bqh4afenfps9", - ]); - } - - public async testTestTimelineEvents() {} - public async loadScene(scene_media_token: string) { - //TODO: DECOUPLE and use QUEUE in next REFACTOR STAGE - showEditorLoader(); - - this.current_scene_media_token = scene_media_token; - - const scene_json = await this.api_manager - .loadSceneState(this.current_scene_media_token) - .catch((err) => { - //TODO: DECOUPLE and use QUEUE in next REFACTOR STAGE - hideEditorLoader(); - throw err; - }); - const proxyScene = new StoryTellerProxyScene( - this.version, - this.activeScene, - ); - await proxyScene.loadFromSceneJson(scene_json["scene"]); - this.cam_obj = this.activeScene.get_object_by_name(this.camera_name); - this.cam_obj?.layers.set(1); - - const proxyTimeline = new StoryTellerProxyTimeline( - this.version, - this.timeline, - this.transform_engine, - this.animation_engine, - this.audio_engine, - this.lipsync_engine, - this.emotion_engine, - ); - await proxyTimeline.loadFromJson(scene_json["timeline"]); - - this.timeline.checkEditorCanPlay(); - - //TODO: DECOUPLE and use QUEUE in next REFACTOR STAGE - hideEditorLoader(); - this.timeline.scrub({ data: { currentTime: 0 } }); + this.save_manager.loadScene(scene_media_token); } isObjectLipsync(object_uuid: string) { - const object = this.activeScene.get_object_by_uuid(object_uuid); - let hasLipsync = false; - if (object) { - object.traverse((c: THREE.Object3D) => { - if (c instanceof THREE.Mesh) { - if (c.morphTargetInfluences && c.morphTargetDictionary) { - const blendShapeIndexE = c.morphTargetDictionary["E"]; - // console.log(c.morphTargetDictionary, blendShapeIndexE) - if (blendShapeIndexE !== null) { - hasLipsync = true; - } - } - } - }); - } - return hasLipsync; + return this.utils.isObjectLipsync(object_uuid); } - isObjectLocked(object_uuid: string) { - const object = this.activeScene.get_object_by_uuid(object_uuid); - if (object) { - if (object.userData["locked"] == undefined) { - object.userData["locked"] = false; - } - return object.userData["locked"]; - } - console.log("No object found."); - return false; + isObjectLocked(object_uuid: string): boolean { + return this.utils.isObjectLocked(object_uuid); } - lockUnlockObject(object_uuid: string) { - const object = this.activeScene.get_object_by_uuid(object_uuid); - if (object) { - if (object.userData["locked"] == undefined) { - object.userData["locked"] = false; - } - object.userData["locked"] = !object.userData["locked"]; - - if (object.userData["locked"]) { - this.removeTransformControls(false); - } else if (this.control) { - this.activeScene.scene.add(this.control); - this.control.attach(this.selected); - } - - return object.userData["locked"]; - } - console.log("No object found."); - return false; + lockUnlockObject(object_uuid: string): boolean { + return this.utils.lockUnlockObject(object_uuid); } setColor(object_uuid: string, hex_color: string) { @@ -639,19 +504,7 @@ class Editor { // TO UPDATE selected objects in the scene might want to add to the scene ... async setSelectedObject(position: XYZ, rotation: XYZ, scale: XYZ) { - if (this.selected != undefined || this.selected != null) { - this.selected.position.x = position.x; - this.selected.position.y = position.y; - this.selected.position.z = position.z; - - this.selected.rotation.x = THREE.MathUtils.degToRad(rotation.x); - this.selected.rotation.y = THREE.MathUtils.degToRad(rotation.y); - this.selected.rotation.z = THREE.MathUtils.degToRad(rotation.z); - - this.selected.scale.x = scale.x; - this.selected.scale.y = scale.y; - this.selected.scale.z = scale.z; - } + this.utils.setSelectedObject(position, rotation, scale); } public async saveScene({ @@ -661,57 +514,7 @@ class Editor { sceneTitle: string; sceneToken?: string; }): Promise { - this.generating_preview = true; // FIX THIS LATER WITH VICCCCCCCCCCCCCCCTORRRRRRRR - // remove controls when saving scene. - this.removeTransformControls(); - //TODO: DECOUPLE and use QUEUE in next REFACTOR STAGE - showEditorLoader(); - - const proxyScene = new StoryTellerProxyScene( - this.version, - this.activeScene, - ); - const scene_json = await proxyScene.saveToScene(); - - const proxyTimeline = new StoryTellerProxyTimeline( - this.version, - this.timeline, - this.transform_engine, - this.animation_engine, - this.audio_engine, - this.lipsync_engine, - this.emotion_engine, - ); - const timeline_json = await proxyTimeline.saveToJson(); - - const save_data = { - version: this.version, - scene: scene_json, - timeline: timeline_json, - }; - - // TODO turn scene information into and object ... - let sceneThumbnail = undefined; - - if (this.renderer) { - const imgData = this.renderer.domElement.toDataURL(); - const response = await fetch(imgData); // Fetch the data URL - sceneThumbnail = await response.blob(); // Convert to Blob - } - - const result = await this.api_manager.saveSceneState({ - saveJson: JSON.stringify(save_data), - sceneTitle, - sceneToken, - sceneThumbnail, - }); - - //TODO: DECOUPLE and use QUEUE in next REFACTOR STAGE - hideEditorLoader(); - - this.generating_preview = false; // FIX THIS LATER WITH VICCCCCCCCCCCCCCCTORRRRRRRR - - return result; + return await this.save_manager.saveScene({ sceneTitle, sceneToken }); } /** @@ -720,7 +523,7 @@ class Editor { * Doesn't retain those controls. * @returns */ - private removeTransformControls(remove_outline: boolean = true) { + removeTransformControls(remove_outline: boolean = true) { if (this.control == undefined) { return; } @@ -738,68 +541,7 @@ class Editor { } switchCameraView() { - this.camera_person_mode = !this.camera_person_mode; - this.cameraViewControls?.reset(); - if (this.cam_obj) { - if (this.camera_person_mode) { - this.last_cam_pos.copy(this.camera.position); - this.last_cam_rot.copy(this.camera.rotation); - - this.camera.position.copy(this.cam_obj.position); - this.camera.rotation.copy(this.cam_obj.rotation); - - if (this.orbitControls) { - this.orbitControls.enabled = false; - } - if (this.lockControls) { - this.activeScene.scene.add(this.lockControls.getObject()); - } - if (this.cameraViewControls) { - this.cameraViewControls.enabled = true; - } - this.cam_obj.scale.set(0, 0, 0); - - this.removeTransformControls(); - this.selected = this.cam_obj; - this.publishSelect(); - - // this.dispatchAppUiState({ - // type: APPUI_ACTION_TYPES.SHOW_CONTROLPANELS_SCENEOBJECT, - // }); - // showObjectPanel(); - this.updateSelectedUI(); - editorState.value = EditorStates.CAMERA_VIEW; - if (this.activeScene.hot_items) { - this.activeScene.hot_items.forEach((element) => { - element.visible = false; - }); - } - } else { - this.camera.position.copy(this.last_cam_pos); - this.camera.rotation.copy(this.last_cam_rot); - if (this.orbitControls) { - this.orbitControls.enabled = true; - } - if (this.lockControls) { - this.activeScene.scene.remove(this.lockControls.getObject()); - } - if (this.cameraViewControls) { - this.cameraViewControls.enabled = false; - } - this.cam_obj.scale.set(1, 1, 1); - if (this.activeScene.hot_items) { - this.activeScene.hot_items.forEach((element) => { - element.visible = true; - }); - } - - // this.dispatchAppUiState({ - // type: APPUI_ACTION_TYPES.HIDE_CONTROLPANELS_SCENEOBJECT, - // }); - hideObjectPanel(); - editorState.value = EditorStates.EDIT; - } - } + this.utils.switchCameraView(); } async showLoading() { @@ -815,14 +557,13 @@ class Editor { } async endLoading() { - console.log("stop loading"); loadingBarIsShowing.value = false; } // Configure post processing. _configurePostProcessing() { - const width = this.canvReference.width; - const height = this.canvReference.height; + const width = this.canvReference?.width ?? 0; + const height = this.canvReference?.height ?? 0; if (this.renderer == undefined || this.camera == undefined) { return; @@ -877,64 +618,29 @@ class Editor { } deleteObject(uuid: string) { - const obj = this.activeScene.get_object_by_uuid(uuid); - this.removeTransformControls(); - if (obj?.name === this.camera_name) { - return; - } - if (obj) { - this.activeScene.scene.remove(obj); - } - this.timeline.deleteObject(uuid); - Queue.publish({ - queueName: QueueNames.FROM_ENGINE, - action: fromEngineActions.DELETE_OBJECT, - data: { - version: 1, - type: AssetType.OBJECT, - media_id: "", - object_uuid: uuid, - name: "", - } as MediaItem, - }); - this.selected = undefined; - this.publishSelect(); - // this.dispatchAppUiState({ - // type: APPUI_ACTION_TYPES.HIDE_CONTROLPANELS_SCENEOBJECT, - // }); - hideObjectPanel(); - this.timeline.deleteObject(uuid); + this.utils.deleteObject(uuid); } create_parim(name: string, pos: THREE.Vector3) { return this.activeScene.instantiate(name, pos); } - renderMode() { - this.rendering = !this.rendering; - this.activeScene.renderMode(this.rendering); - } - - stepFrame(frames: number) { - this.timeline.stepFrame(frames); - } - // Render the scene to the camera, this is called in the update. async renderScene() { if (this.composer != null && !this.rendering && this.rawRenderer) { this.composer.render(); - this.rawRenderer.render(this.activeScene.scene, this.render_camera); + this.rawRenderer.render(this.activeScene.scene, this.render_camera!); } else if (this.renderer && this.render_camera && !this.rendering) { this.renderer.setSize(this.render_width, this.render_height); this.renderer.render(this.activeScene.scene, this.render_camera); } else if (this.rendering && this.rawRenderer) { - this.rawRenderer.render(this.activeScene.scene, this.render_camera); + this.rawRenderer.render(this.activeScene.scene, this.render_camera!); } else { console.error("Could not render to canvas no render or composer!"); } if (this.rendering && this.rawRenderer && this.clock && this.renderer) { - if (this.recorder == undefined) { + if (this.recorder === undefined && this.render_camera) { this.rawRenderer.setSize(1024, 576); this.render_camera.aspect = 1024 / 576; } @@ -942,10 +648,10 @@ class Editor { this.render_timer += this.clock.getDelta(); this.frames += 1; this.playback_location++; - const imgData = this.rawRenderer.domElement.toDataURL("image/png", 1.0); // Medium quality png for speed & size. + const imgData = this.rawRenderer.domElement.toDataURL("image/png", 1.0); // High quality png. this.frame_buffer.push(imgData); this.render_timer += this.clock.getDelta(); - if (this.timeline.is_playing == false) { + if (!this.timeline.is_playing) { //this.recorder.stop(); this.playback_location = 0; this.stopPlaybackAndUploadVideo(); @@ -953,23 +659,6 @@ class Editor { } } - getselectedSum() { - if (this.selected === undefined) { - return 0; - } - const posCombo = - this.selected.position.x + - this.selected.position.y + - this.selected.position.z; - const rotCombo = - this.selected.rotation.x + - this.selected.rotation.y + - this.selected.rotation.z; - const sclCombo = - this.selected.scale.x + this.selected.scale.y + this.selected.scale.z; - return posCombo + rotCombo + sclCombo; - } - // Basicly Unity 3D's update loop. async updateLoop() { setTimeout( @@ -982,7 +671,7 @@ class Editor { if (this.container === undefined) { this.container = document.getElementById("video-scene-container"); } - if (!this.rendering && this.container !== undefined) { + if (!this.rendering && this.container) { if ( this.container.clientWidth + this.container.clientHeight !== this.lastCanvasSize @@ -997,11 +686,6 @@ class Editor { this.cam_obj = this.activeScene.get_object_by_name(this.camera_name); } - // Updates debug stats. - if (this.stats != null) { - this.stats.update(); - } - if (this.clock == undefined || this.renderer == undefined) { return; } @@ -1010,11 +694,11 @@ class Editor { if (this.cameraViewControls && this.camera_person_mode) { this.cameraViewControls.update(5 * delta_time); - if (this.cam_obj) { + if (this.cam_obj && this.camera) { if (this.last_scrub != this.timeline.scrubber_frame_position) { this.camera.position.copy(this.cam_obj.position); this.camera.rotation.copy(this.cam_obj.rotation); - } else if (this.timeline.is_playing == false) { + } else if (!this.timeline.is_playing) { this.cam_obj.position.copy(this.camera.position); this.cam_obj.rotation.copy(this.camera.rotation); } else { @@ -1045,17 +729,17 @@ class Editor { } } else if ( this.last_scrub === this.timeline.scrubber_frame_position && - this.getselectedSum() !== this.last_selected_sum + this.utils.getselectedSum() !== this.last_selected_sum ) { this.updateSelectedUI(); } - this.last_selected_sum = this.getselectedSum(); + this.last_selected_sum = this.utils.getselectedSum(); await this.renderScene(); this.last_scrub = this.timeline.scrubber_frame_position; } - change_mode(type: any) { + change_mode(type: "translate" | "rotate" | "scale") { if (this.control == undefined) { return; } @@ -1063,221 +747,8 @@ class Editor { this.transform_interaction = true; } - async convertAudioClip(itteration: number, ffmpeg: FFmpeg, clip: ClipUI) { - const video_og = itteration + "tmp.mp4"; - const wav_name = itteration + "tmp.wav"; - const new_video = itteration + 1 + "tmp.mp4"; - let startFrame = clip.offset; - let endFrame = clip.length; - - if (endFrame > this.timeline.timeline_limit) { - endFrame = this.timeline.timeline_limit; - } - if (startFrame > this.timeline.timeline_limit) { - startFrame = this.timeline.timeline_limit - 1; - } - - const startTime = startFrame / this.cap_fps; - const endTime = endFrame / this.cap_fps; - const end = endTime - startTime; - - const audioSegment = "as_" + wav_name; - await ffmpeg.FS( - "writeFile", - wav_name, - await fetchFile(await this.api_manager.getMediaFile(clip.media_id)), - ); - await ffmpeg.run( - "-i", - wav_name, - "-ss", - "0", - "-to", - "" + end, - "-max_muxing_queue_size", - "999999", - audioSegment, - ); - - await ffmpeg.run( - "-i", - video_og, - "-max_muxing_queue_size", - "999999", - `${itteration}empty_tmp.wav`, - ); - - await ffmpeg.run( - "-i", - `${itteration}empty_tmp.wav`, - "-i", - audioSegment, - "-filter_complex", - "[1:a]adelay=" + - startTime * 1000 + - "|" + - startTime * 1000 + - "[a1];[0:a][a1]amix=inputs=2[a]", - "-map", - "[a]", - `${itteration}final_tmp.wav`, - ); - - await ffmpeg.run( - "-i", - video_og, - "-i", - `${itteration}final_tmp.wav`, - "-c:v", - "copy", - "-c:a", - "aac", - "-map", - "0:v:0", - "-map", - "1:a:0", - "-strict", - "experimental", - new_video, - ); - } - - async _debugDownloadVideo(videoURL: string) { - // DEBUG ONLY to download the video - - const a = document.createElement("a"); - a.href = videoURL; - a.download = "video.mp4"; // Name of the downloaded file - document.body.appendChild(a); - a.click(); // Trigger the download - } - async stopPlaybackAndUploadVideo(compile_audio: boolean = true) { - this.playback = false; - this.rendering = false; - - //const videoBlob = new Blob(this.frame_buffer, { type: "video/webm" }); - //const videoURL = URL.createObjectURL(videoBlob); - - this.generating_preview = true; - const ffmpeg = createFFmpeg({ log: false }); - await ffmpeg.load(); - - this.updateLoad(50, "Processing ..."); - - // Write the Uint8Array to the FFmpeg file system - //ffmpeg.FS("writeFile", "input.webm", await fetchFile(videoURL)); - - for (let index = 0; index < this.frame_buffer.length; index++) { - const element = this.frame_buffer[index]; - await ffmpeg.FS( - "writeFile", - `image${index}.png`, - await fetchFile(element), - ); - } - - await ffmpeg.run( - "-framerate", - "" + this.cap_fps, - "-i", - "image%d.png", - "input.mp4", - ); - - await ffmpeg.run( - "-i", - "input.mp4", - "-f", - "lavfi", - "-i", - "anullsrc", // This adds a silent audio track - "-max_muxing_queue_size", - "999999", - "-vf", - "select=gte(n\\,1),scale=1024:576", // scale=1024:576 - "-c:v", - "libx264", // Specify video codec (optional, but recommended for MP4) - "-c:a", - "aac", // Specify audio codec (optional, but recommended for MP4) - "-shortest", // Ensure output duration matches the shortest stream (video or audio) - "-pix_fmt", - "yuv420p", - "-f", - "mp4", - "0tmp.mp4", - ); - - let itteration = 0; - - if (compile_audio) { - for (const clip of this.timeline.timeline_items) { - if (clip.type == ClipType.AUDIO) { - await this.convertAudioClip(itteration, ffmpeg, clip); - itteration += 1; - } - } - } - - const output = ffmpeg.FS("readFile", itteration + "tmp.mp4"); - - ffmpeg.exit(); - this.generating_preview = false; - - // Create a Blob from the output file for downloading - const blob = new Blob([output.buffer], { type: "video/mp4" }); - - const title = getSceneSignals().title || "Untitled"; - - const style_name = this.art_style.toString(); - const media_token = this.current_scene_media_token || undefined; - - const data: any = await this.api_manager.uploadMedia({ - blob, - fileName: `${title}.mp4`, - title, - styleName: style_name, - maybe_scene_source_media_file_token: media_token, - }); - - if (data == null) { - return; - } - const upload_token = data["media_file_token"]; - - const result = await this.api_manager - .stylizeVideo( - upload_token, - this.art_style, - this.positive_prompt, - this.negative_prompt, - Visibility.Public, - this.generation_options.faceDetail, - this.generation_options.upscale, - this.generation_options.styleStrength, - ) - .catch((error) => { - console.log(error); - }); - - // {"success":true,"inference_job_token":"jinf_j3nbqbd15wqxb0xcks13qh3f3bz"} - this.updateLoad(100, "Done Check Your Media Tab On Profile."); - this.endLoading(); - - this.recorder = undefined; - if (this.rawRenderer) { - this.rawRenderer.setSize(this.startRenderWidth, this.startRenderHeight); - } - - this.canvasRenderCamReference = document.getElementById("camera-view"); - this.rawRenderer = new THREE.WebGLRenderer({ - antialias: false, - canvas: this.canvasRenderCamReference, - preserveDrawingBuffer: true, - }); - this.activeScene.renderMode(false); - - this.switchEdit(); + this.videoGeneration.stopPlaybackAndUploadVideo(compile_audio); } async switchPreview() { @@ -1292,11 +763,12 @@ class Editor { } switchEdit() { - if (this.switchPreviewToggle) { + if (this.switchPreviewToggle && this.canvasRenderCamReference) { this.switchPreviewToggle = false; editorState.value = EditorStates.EDIT; setTimeout(() => { - this.canvasRenderCamReference = document.getElementById("camera-view"); + this.canvasRenderCamReference = + document.getElementById("camera-view") ?? undefined; this.rawRenderer = new THREE.WebGLRenderer({ antialias: false, canvas: this.canvasRenderCamReference, @@ -1311,61 +783,7 @@ class Editor { } async generateFrame() { - if (!this.generating_preview && this.rawRenderer) { - this.generating_preview = true; - this.removeTransformControls(); - this.activeScene.renderMode(true); - if (this.activeScene.hot_items) { - this.activeScene.hot_items.forEach((element) => { - element.visible = false; - }); - } - - if (this.render_camera && this.cam_obj) { - this.render_camera.position.copy(this.cam_obj.position); - this.render_camera.rotation.copy(this.cam_obj.rotation); - } - - previewSrc.value = ""; - - this.rawRenderer.setSize(this.render_width, this.render_height); - this.render_camera.aspect = this.render_width / this.render_height; - this.render_camera.updateProjectionMatrix(); - this.rawRenderer.render(this.activeScene.scene, this.render_camera); - const imgData = this.rawRenderer.domElement.toDataURL(); - const response = await fetch(imgData); // Fetch the data URL - const blob = await response.blob(); // Convert to Blob - - if (!this.camera_person_mode) { - this.switchCameraView(); - editorState.value = EditorStates.PREVIEW; - } - - this.generating_preview = false; - - try { - const url = await this.api_manager.uploadMediaFrameGeneration( - blob, - "render.png", - this.art_style, - this.positive_prompt, - this.negative_prompt, - ); - - previewSrc.value = url; - return Promise.resolve(url); - } catch (err: any) { - Queue.publish({ - queueName: QueueNames.FROM_ENGINE, - action: fromEngineActions.POP_A_TOAST, - data: { - type: ToastTypes.ERROR, - message: err.message, - }, - }); - // return Promise.resolve(""); - } - } + this.videoGeneration.generateFrame(); } // This initializes the generation of a video render scene is where the core work happens @@ -1435,35 +853,6 @@ class Editor { const scale = this.selected.scale; // TODO this is a bug we need to only show when clicked on and use UPDATE when updating. - // this.dispatchAppUiState({ - // type: APPUI_ACTION_TYPES.UPDATE_CONTROLPANELS_SCENEOBJECT, - // payload: { - // group: - // this.selected.name === this.camera_name - // ? ClipGroup.CAMERA - // : ClipGroup.OBJECT, // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips - // object_uuid: this.selected.uuid, - // object_name: this.selected.name, - // version: String(this.version), - // objectVectors: { - // position: { - // x: parseFloat(pos.x.toFixed(2)), - // y: parseFloat(pos.y.toFixed(2)), - // z: parseFloat(pos.z.toFixed(2)), - // }, - // rotation: { - // x: parseFloat(THREE.MathUtils.radToDeg(rot.x).toFixed(2)), - // y: parseFloat(THREE.MathUtils.radToDeg(rot.y).toFixed(2)), - // z: parseFloat(THREE.MathUtils.radToDeg(rot.z).toFixed(2)), - // }, - // scale: { - // x: parseFloat(scale.x.toFixed(6)), - // y: parseFloat(scale.y.toFixed(6)), - // z: parseFloat(scale.z.toFixed(6)), - // }, - // }, - // }, - // }); //end dispatch updateObjectPanel({ group: this.selected.name === this.camera_name @@ -1524,13 +913,17 @@ class Editor { setupResizeObserver() { const container = document.getElementById("video-scene-container"); - if (!container) return; + if (!container) { + return; + } const resizeObserver = new ResizeObserver((entries) => { for (const entry of entries) { const { width, height } = entry.contentRect; - this.camera.aspect = width / height; - this.camera.updateProjectionMatrix(); + if (this.camera) { + this.camera.aspect = width / height; + this.camera.updateProjectionMatrix(); + } this.renderer?.setSize(width, height); this.renderer?.setPixelRatio(window.devicePixelRatio); } @@ -1539,143 +932,6 @@ class Editor { resizeObserver.observe(container); } - onMouseDown(event: any) { - if (event.button === 1 && this.camera_person_mode) { - this.lockControls?.lock(); - } - } - - onMouseUp(event: any) { - if (event.button === 1) { - this.lockControls?.unlock(); - } - - if (event.button !== 0) { - const camera_pos = new THREE.Vector3( - parseFloat(this.camera.position.x.toFixed(2)), - parseFloat(this.camera.position.y.toFixed(2)), - parseFloat(this.camera.position.z.toFixed(2)), - ); - this.camera_last_pos.copy(camera_pos); - } - } - - onkeydown(event: KeyboardEvent) { - if (hotkeysStatus.value.disabled) { - return; - } - console.log("down"); - if (event.key === "f" && this.selected && this.orbitControls) { - this.orbitControls.target.copy(this.selected.position); - this.orbitControls.maxDistance = 4; - this.orbitControls.update(); - this.orbitControls.maxDistance = 999; - return; - } - if (event.key === " ") { - if (!this.rendering && !this.switchPreviewToggle && this.selectedCanvas) { - this.togglePlayback(); - } - return; - } - if (event.key === "Backspace" || event.key === "Delete") { - if (this.selected) { - this.deleteObject(this.selected.uuid); - } - return; - } - } - - // Sets new mouse location usually used in raycasts. - onMouseMove(event: any) { - const rect = this.canvReference.getBoundingClientRect(); - if (this.mouse == undefined) { - return; - } - this.mouse.x = ((event.clientX - rect.left) / rect.width) * 2 - 1; - this.mouse.y = -((event.clientY - rect.top) / rect.height) * 2 + 1; - this.timeline.mouse = this.mouse; - } - - // When the mouse clicks the screen. - onMouseClick() { - const camera_pos = new THREE.Vector3( - parseFloat(this.camera.position.x.toFixed(2)), - parseFloat(this.camera.position.y.toFixed(2)), - parseFloat(this.camera.position.z.toFixed(2)), - ); - if (this.camera_last_pos.equals(new THREE.Vector3(0, 0, 0))) { - this.camera_last_pos.copy(camera_pos); - } - - if ( - this.raycaster == undefined || - this.mouse == undefined || - this.control == undefined || - this.outlinePass == undefined || - this.camera_person_mode || - !this.camera_last_pos.equals(camera_pos) - ) { - this.camera_last_pos.copy(camera_pos); - return; - } - this.camera_last_pos.copy(camera_pos); - - this.raycaster.setFromCamera(this.mouse, this.camera); - const interactable: any[] = []; - this.activeScene.scene.children.forEach((child: THREE.Object3D) => { - // console.log(child); - if (child.name != "") { - if ( - child.type == "Mesh" || - child.type == "Object3D" || - child.type == "Group" - ) { - interactable.push(child); - } - } - }); - const intersects = this.raycaster.intersectObjects(interactable, true); - - if (intersects.length > 0) { - if (intersects[0].object.type != "GridHelper") { - let currentObject = intersects[0].object; - while (currentObject.parent && currentObject.parent.type !== "Scene") { - currentObject = currentObject.parent; - } - this.selected = currentObject; - // Show panel here - - if (this.selected.type == "Scene") { - this.selected = intersects[0].object; - } - this.activeScene.selected = this.selected; - this.publishSelect(); - - // this.update_properties() - if (this.selected.userData["locked"] !== true) { - this.activeScene.scene.add(this.control); - this.control.attach(this.selected); - } - - this.outlinePass.selectedObjects = [this.selected]; - this.transform_interaction = true; - // Contact react land - // this.dispatchAppUiState({ - // type: APPUI_ACTION_TYPES.SHOW_CONTROLPANELS_SCENEOBJECT, - // }); - showObjectPanel(); - this.updateSelectedUI(); - } - } else { - this.removeTransformControls(); - // this.dispatchAppUiState({ - // type: APPUI_ACTION_TYPES.HIDE_CONTROLPANELS_SCENEOBJECT, - // }); - hideObjectPanel(); - } - } - getAssetType(selected: THREE.Object3D): AssetType { if (selected.type === "Mesh") { return selected.name === "::CAM::" ? AssetType.CAMERA : AssetType.OBJECT; diff --git a/app/pages/PageEnigma/js/emotion_engine.ts b/app/pages/PageEnigma/Editor/emotion_engine.ts similarity index 100% rename from app/pages/PageEnigma/js/emotion_engine.ts rename to app/pages/PageEnigma/Editor/emotion_engine.ts diff --git a/app/pages/PageEnigma/js/free_cam.js b/app/pages/PageEnigma/Editor/free_cam.js similarity index 100% rename from app/pages/PageEnigma/js/free_cam.js rename to app/pages/PageEnigma/Editor/free_cam.js diff --git a/app/pages/PageEnigma/Editor/free_cam.ts b/app/pages/PageEnigma/Editor/free_cam.ts new file mode 100644 index 0000000000..742fded2b4 --- /dev/null +++ b/app/pages/PageEnigma/Editor/free_cam.ts @@ -0,0 +1,218 @@ +import { EventDispatcher, Quaternion, Vector3, Vector2 } from "three"; +import * as THREE from "three"; + +class FreeCam extends EventDispatcher { + object: THREE.PerspectiveCamera; + domElement: HTMLElement | Document; + enabled = true; + movementSpeed = 1.0; + rollSpeed = 0.005; + dragToLook = false; + autoForward = false; + + lastMousePosition = new Vector2(); + mouseVelocity = new Vector2(); + tmpQuaternion = new Quaternion(); + + status = 0; + + moveState = { + up: 0, + down: 0, + left: 0, + right: 0, + forward: 0, + back: 0, + pitchUp: 0, + pitchDown: 0, + yawLeft: 0, + yawRight: 0, + rollLeft: 0, + rollRight: 0, + }; + moveVector = new Vector3(0, 0, 0); + rotationVector = new Vector3(0, 0, 0); + + constructor( + object: THREE.PerspectiveCamera, + domElement: HTMLElement | Document, + ) { + super(); + + this.object = object; + this.domElement = domElement; + + this.keydown = this.keydown.bind(this); + this.keyup = this.keyup.bind(this); + this.reset = this.reset.bind(this); + this.update = this.update.bind(this); + this.updateMovementVector = this.updateMovementVector.bind(this); + this.getContainerDimensions = this.getContainerDimensions.bind(this); + this.dispose = this.dispose.bind(this); + + window.addEventListener("keydown", this.keydown); + window.addEventListener("keyup", this.keyup); + + this.updateMovementVector(); + } + + keydown(event: KeyboardEvent) { + if (event.altKey || !this.enabled) { + return; + } + + switch (event.code) { + case "KeyW": + this.moveState.forward = 1; + break; + case "KeyS": + this.moveState.back = 1; + break; + + case "KeyA": + this.moveState.left = 1; + break; + case "KeyD": + this.moveState.right = 1; + break; + + case "KeyQ": + this.moveState.down = 1; + break; + case "KeyE": + this.moveState.up = 1; + break; + + case "ArrowUp": + this.moveState.pitchUp = 1; + break; + case "ArrowDown": + this.moveState.pitchDown = 1; + break; + + case "ArrowLeft": + this.moveState.yawLeft = 1; + break; + case "ArrowRight": + this.moveState.yawRight = 1; + break; + } + + this.updateMovementVector(); + } + + keyup(event: KeyboardEvent) { + if (!this.enabled) { + return; + } + + switch (event.code) { + case "KeyW": + this.moveState.forward = 0; + break; + case "KeyS": + this.moveState.back = 0; + break; + + case "KeyA": + this.moveState.left = 0; + break; + case "KeyD": + this.moveState.right = 0; + break; + + case "ArrowUp": + this.moveState.pitchUp = 0; + break; + case "ArrowDown": + this.moveState.pitchDown = 0; + break; + + case "ArrowLeft": + this.moveState.yawLeft = 0; + break; + case "ArrowRight": + this.moveState.yawRight = 0; + break; + + case "KeyQ": + this.moveState.down = 0; + break; + case "KeyE": + this.moveState.up = 0; + break; + } + + this.updateMovementVector(); + } + + reset() { + this.moveState.forward = 0; + this.moveState.back = 0; + this.moveState.left = 0; + this.moveState.right = 0; + this.moveState.pitchUp = 0; + this.moveState.pitchDown = 0; + this.moveState.yawLeft = 0; + this.moveState.yawRight = 0; + this.moveState.down = 0; + this.moveState.up = 0; + this.updateMovementVector(); + } + + update(delta: number) { + if (!this.enabled) { + return; + } + + const moveMulti = delta * this.movementSpeed; + + this.object.translateX(this.moveVector.x * moveMulti); + this.object.translateY(this.moveVector.y * moveMulti); + this.object.translateZ(this.moveVector.z * moveMulti); + } + + updateMovementVector() { + const forward = + this.moveState.forward || (this.autoForward && !this.moveState.back) + ? 1 + : 0; + + this.moveVector.x = -this.moveState.left + this.moveState.right; + this.moveVector.y = -this.moveState.down + this.moveState.up; + this.moveVector.z = -forward + this.moveState.back; + } + + getContainerDimensions() { + if (this.domElement !== document) { + return { + size: [ + (this.domElement as HTMLElement).offsetWidth, + (this.domElement as HTMLElement).offsetHeight, + ], + offset: [ + (this.domElement as HTMLElement).offsetLeft, + (this.domElement as HTMLElement).offsetTop, + ], + }; + } else { + return { + size: [window.innerWidth, window.innerHeight], + offset: [0, 0], + }; + } + } + + dispose() { + // this.domElement.removeEventListener("contextmenu", this.contextmenu); + // this.domElement.removeEventListener("pointerdown", this.pointerdown); + // this.domElement.removeEventListener("pointermove", this.pointermove); + // this.domElement.removeEventListener("pointerup", this.pointerup); + // this.domElement.removeEventListener("pointercancel", this.pointercancel); + // + window.removeEventListener("keydown", this.keydown); + window.removeEventListener("keyup", this.keyup); + } +} + +export { FreeCam }; diff --git a/app/pages/PageEnigma/Editor/helper.ts b/app/pages/PageEnigma/Editor/helper.ts new file mode 100644 index 0000000000..6f91476aa1 --- /dev/null +++ b/app/pages/PageEnigma/Editor/helper.ts @@ -0,0 +1,218 @@ +import Scene from "./scene"; +import Editor from "./editor"; +import * as THREE from "three"; +import { XYZ } from "../datastructures/common"; +import { editorState } from "../signals/engine"; +import { EditorStates } from "../enums"; +import Queue from "~/pages/PageEnigma/Queue/Queue"; +import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; +import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions"; +import { MediaItem } from "~/pages/PageEnigma/models"; +import { AssetType } from "~/enums"; +import { hideObjectPanel } from "../signals"; + +export class Utils { + scene: Scene; + editor: Editor; + constructor(editor: Editor, scene: Scene) { + this.scene = scene; + this.editor = editor; + } + + // If string is empty. + isEmpty(value: string): boolean { + return ( + value == null || (typeof value === "string" && value.trim().length === 0) + ); + } + + // For react to see if an object has lipsync capability or not. + isObjectLipsync(object_uuid: string) { + const object = this.scene.get_object_by_uuid(object_uuid); + let hasLipsync = false; + if (object) { + object.traverse((c: THREE.Object3D) => { + if (c instanceof THREE.Mesh) { + if (c.morphTargetInfluences && c.morphTargetDictionary) { + const blendShapeIndexE = c.morphTargetDictionary["E"]; + // console.log(c.morphTargetDictionary, blendShapeIndexE) + if (blendShapeIndexE !== null) { + hasLipsync = true; + } + } + } + }); + } + return hasLipsync; + } + + // Returns if the object is locked or unlocked. + isObjectLocked(object_uuid: string): boolean { + const object = this.scene.get_object_by_uuid(object_uuid); + if (object) { + if (object.userData["locked"] == undefined) { + object.userData["locked"] = false; + } + return object.userData["locked"]; + } + console.log("No object found."); + return false; + } + + // Locks or unlocks and object and returns its new state, + lockUnlockObject(object_uuid: string): boolean { + const object = this.scene.get_object_by_uuid(object_uuid); + if (object) { + if (object.userData["locked"] == undefined) { + object.userData["locked"] = false; + } + object.userData["locked"] = !object.userData["locked"]; + + if (object.userData["locked"]) { + this.removeTransformControls(false); + } else if (this.editor.control) { + this.scene.scene.add(this.editor.control); + this.editor.control.attach(this.editor.selected); + } + + return object.userData["locked"]; + } + console.log("No object found."); + return false; + } + + // Removes transform controls and publishes selected. + removeTransformControls(remove_outline: boolean = true) { + if (this.editor.control == undefined) { + return; + } + if (this.editor.outlinePass == undefined) { + return; + } + if (remove_outline) { + this.editor.last_selected = this.editor.selected; + this.editor.selected = undefined; + this.editor.publishSelect(); + } + this.editor.control.detach(); + this.editor.activeScene.scene.remove(this.editor.control); + if (remove_outline) this.editor.outlinePass.selectedObjects = []; + } + + // TO UPDATE selected objects in the scene might want to add to the scene ... + async setSelectedObject(position: XYZ, rotation: XYZ, scale: XYZ) { + if (this.editor.selected != undefined || this.editor.selected != null) { + this.editor.selected.position.x = position.x; + this.editor.selected.position.y = position.y; + this.editor.selected.position.z = position.z; + this.editor.selected.rotation.x = THREE.MathUtils.degToRad(rotation.x); + this.editor.selected.rotation.y = THREE.MathUtils.degToRad(rotation.y); + this.editor.selected.rotation.z = THREE.MathUtils.degToRad(rotation.z); + this.editor.selected.scale.x = scale.x; + this.editor.selected.scale.y = scale.y; + this.editor.selected.scale.z = scale.z; + } + } + + switchCameraView() { + this.editor.camera_person_mode = !this.editor.camera_person_mode; + this.editor.cameraViewControls?.reset(); + if (this.editor.cam_obj) { + if (this.editor.camera_person_mode) { + this.editor.last_cam_pos.copy(this.editor.camera.position); + this.editor.last_cam_rot.copy(this.editor.camera.rotation); + + this.editor.camera.position.copy(this.editor.cam_obj.position); + this.editor.camera.rotation.copy(this.editor.cam_obj.rotation); + + if (this.editor.orbitControls) { + this.editor.orbitControls.enabled = false; + } + if (this.editor.lockControls) { + this.editor.activeScene.scene.add(this.editor.lockControls.getObject()); + } + if (this.editor.cameraViewControls) { + this.editor.cameraViewControls.enabled = true; + } + this.editor.cam_obj.scale.set(0, 0, 0); + + this.editor.removeTransformControls(); + this.editor.selected = this.editor.cam_obj; + this.editor.publishSelect(); + this.editor.updateSelectedUI(); + editorState.value = EditorStates.CAMERA_VIEW; + if (this.editor.activeScene.hot_items) { + this.editor.activeScene.hot_items.forEach((element) => { + element.visible = false; + }); + } + } else { + this.editor.camera.position.copy(this.editor.last_cam_pos); + this.editor.camera.rotation.copy(this.editor.last_cam_rot); + if (this.editor.orbitControls) { + this.editor.orbitControls.enabled = true; + } + if (this.editor.lockControls) { + this.editor.activeScene.scene.remove(this.editor.lockControls.getObject()); + } + if (this.editor.cameraViewControls) { + this.editor.cameraViewControls.enabled = false; + } + this.editor.cam_obj.scale.set(1, 1, 1); + if (this.editor.activeScene.hot_items) { + this.editor.activeScene.hot_items.forEach((element) => { + element.visible = true; + }); + } + + hideObjectPanel(); + editorState.value = EditorStates.EDIT; + } + } + } + + // Returns the "check sum" of the editors selected object. + getselectedSum(): number { + if (this.editor.selected === undefined) { + return 0; + } + const posCombo = + this.editor.selected.position.x + + this.editor.selected.position.y + + this.editor.selected.position.z; + const rotCombo = + this.editor.selected.rotation.x + + this.editor.selected.rotation.y + + this.editor.selected.rotation.z; + const sclCombo = + this.editor.selected.scale.x + this.editor.selected.scale.y + this.editor.selected.scale.z; + return posCombo + rotCombo + sclCombo; + } + + deleteObject(uuid: string) { + const obj = this.scene.get_object_by_uuid(uuid); + this.removeTransformControls(); + if (obj?.name === this.editor.camera_name) { + return; + } + if (obj) { + this.scene.scene.remove(obj); + } + this.editor.timeline.deleteObject(uuid); + Queue.publish({ + queueName: QueueNames.FROM_ENGINE, + action: fromEngineActions.DELETE_OBJECT, + data: { + version: 1, + type: AssetType.OBJECT, + media_id: "", + object_uuid: uuid, + name: "", + } as MediaItem, + }); + this.editor.selected = undefined; + this.editor.publishSelect(); + hideObjectPanel(); + this.editor.timeline.deleteObject(uuid); + } +} diff --git a/app/pages/PageEnigma/Editor/keybinds_controls.ts b/app/pages/PageEnigma/Editor/keybinds_controls.ts new file mode 100644 index 0000000000..ac692efc01 --- /dev/null +++ b/app/pages/PageEnigma/Editor/keybinds_controls.ts @@ -0,0 +1,140 @@ +import Editor from "./editor"; +import * as THREE from "three"; +import { hideObjectPanel, hotkeysStatus, showObjectPanel } from "../signals"; + +export class MouseControls { + editor: Editor; + + constructor(editor: Editor) { + this.editor = editor; + } + + onMouseDown(event: any) { + if (event.button === 1 && this.editor.camera_person_mode) { + this.editor.lockControls?.lock(); + } + } + + onMouseUp(event: any) { + if (event.button === 1) { + this.editor.lockControls?.unlock(); + } + + if (event.button !== 0) { + const camera_pos = new THREE.Vector3( + parseFloat(this.editor.camera.position.x.toFixed(2)), + parseFloat(this.editor.camera.position.y.toFixed(2)), + parseFloat(this.editor.camera.position.z.toFixed(2)), + ); + this.editor.camera_last_pos.copy(camera_pos); + } + } + + onkeydown(event: KeyboardEvent) { + if (hotkeysStatus.value.disabled) { + return; + } + if (event.key === "f" && this.editor.selected && this.editor.orbitControls) { + this.editor.orbitControls.target.copy(this.editor.selected.position); + this.editor.orbitControls.maxDistance = 4; + this.editor.orbitControls.update(); + this.editor.orbitControls.maxDistance = 999; + return; + } + if (event.key === " ") { + if (!this.editor.rendering && !this.editor.switchPreviewToggle && this.editor.selectedCanvas) { + this.editor.togglePlayback(); + } + return; + } + if (event.key === "Backspace" || event.key === "Delete") { + if (this.editor.selected) { + this.editor.deleteObject(this.editor.selected.uuid); + } + return; + } + } + + // Sets new mouse location usually used in raycasts. + onMouseMove(event: any) { + const rect = this.editor.canvReference.getBoundingClientRect(); + if (this.editor.mouse == undefined) { + return; + } + this.editor.mouse.x = ((event.clientX - rect.left) / rect.width) * 2 - 1; + this.editor.mouse.y = -((event.clientY - rect.top) / rect.height) * 2 + 1; + this.editor.timeline.mouse = this.editor.mouse; + } + + // When the mouse clicks the screen. + onMouseClick() { + const camera_pos = new THREE.Vector3( + parseFloat(this.editor.camera.position.x.toFixed(2)), + parseFloat(this.editor.camera.position.y.toFixed(2)), + parseFloat(this.editor.camera.position.z.toFixed(2)), + ); + if (this.editor.camera_last_pos.equals(new THREE.Vector3(0, 0, 0))) { + this.editor.camera_last_pos.copy(camera_pos); + } + + if ( + this.editor.raycaster == undefined || + this.editor.mouse == undefined || + this.editor.control == undefined || + this.editor.outlinePass == undefined || + this.editor.camera_person_mode || + !this.editor.camera_last_pos.equals(camera_pos) + ) { + this.editor.camera_last_pos.copy(camera_pos); + return; + } + this.editor.camera_last_pos.copy(camera_pos); + + this.editor.raycaster.setFromCamera(this.editor.mouse, this.editor.camera); + const interactable: any[] = []; + this.editor.activeScene.scene.children.forEach((child: THREE.Object3D) => { + if (child.name != "") { + if ( + child.type == "Mesh" || + child.type == "Object3D" || + child.type == "Group" + ) { + interactable.push(child); + } + } + }); + const intersects = this.editor.raycaster.intersectObjects(interactable, true); + + if (intersects.length > 0) { + if (intersects[0].object.type != "GridHelper") { + let currentObject = intersects[0].object; + while (currentObject.parent && currentObject.parent.type !== "Scene") { + currentObject = currentObject.parent; + } + this.editor.selected = currentObject; + // Show panel here + + if (this.editor.selected.type == "Scene") { + this.editor.selected = intersects[0].object; + } + this.editor.activeScene.selected = this.editor.selected; + this.editor.publishSelect(); + + // this.editor.update_properties() + if (this.editor.selected.userData["locked"] !== true) { + this.editor.activeScene.scene.add(this.editor.control); + this.editor.control.attach(this.editor.selected); + } + + this.editor.outlinePass.selectedObjects = [this.editor.selected]; + this.editor.transform_interaction = true; + // Contact react land + showObjectPanel(); + this.editor.updateSelectedUI(); + } + } else { + this.editor.removeTransformControls(); + hideObjectPanel(); + } + } +} diff --git a/app/pages/PageEnigma/js/lip_sync_engine.ts b/app/pages/PageEnigma/Editor/lip_sync_engine.ts similarity index 94% rename from app/pages/PageEnigma/js/lip_sync_engine.ts rename to app/pages/PageEnigma/Editor/lip_sync_engine.ts index 4602ab0f98..50bc60937b 100644 --- a/app/pages/PageEnigma/js/lip_sync_engine.ts +++ b/app/pages/PageEnigma/Editor/lip_sync_engine.ts @@ -20,5 +20,3 @@ export class LipSyncEngine { ); } } - -export default LipSyncEngine; diff --git a/app/pages/PageEnigma/js/lipsync.js b/app/pages/PageEnigma/Editor/lipsync.js similarity index 100% rename from app/pages/PageEnigma/js/lipsync.js rename to app/pages/PageEnigma/Editor/lipsync.js diff --git a/app/pages/PageEnigma/js/placeholder_manager.ts b/app/pages/PageEnigma/Editor/placeholder_manager.ts similarity index 100% rename from app/pages/PageEnigma/js/placeholder_manager.ts rename to app/pages/PageEnigma/Editor/placeholder_manager.ts diff --git a/app/pages/PageEnigma/js/retargeting.ts b/app/pages/PageEnigma/Editor/retargeting.ts similarity index 100% rename from app/pages/PageEnigma/js/retargeting.ts rename to app/pages/PageEnigma/Editor/retargeting.ts diff --git a/app/pages/PageEnigma/Editor/save_manager.ts b/app/pages/PageEnigma/Editor/save_manager.ts new file mode 100644 index 0000000000..fab5db09b7 --- /dev/null +++ b/app/pages/PageEnigma/Editor/save_manager.ts @@ -0,0 +1,109 @@ +import { StoryTellerProxyScene } from "../proxy/storyteller_proxy_scene"; +import { StoryTellerProxyTimeline } from "../proxy/storyteller_proxy_timeline"; +import { hideEditorLoader, showEditorLoader } from "../signals"; +import Editor from "./editor"; + +export type EditorInitializeConfig = { + sceneToken: string; +}; + +export class SaveManager { + editor: Editor + constructor(editor: Editor) { + this.editor = editor; + } + + public async saveScene({ + sceneTitle, + sceneToken, + }: { + sceneTitle: string; + sceneToken?: string; + }): Promise { + this.editor.generating_preview = true; // Set this to true to stop control panel from flipping out. + // remove controls when saving scene. + this.editor.removeTransformControls(); + showEditorLoader(); + + const proxyScene = new StoryTellerProxyScene( + this.editor.version, + this.editor.activeScene, + ); + const scene_json = await proxyScene.saveToScene(); + + const proxyTimeline = new StoryTellerProxyTimeline( + this.editor.version, + this.editor.timeline, + this.editor.transform_engine, + this.editor.animation_engine, + this.editor.audio_engine, + this.editor.lipsync_engine, + this.editor.emotion_engine, + ); + const timeline_json = await proxyTimeline.saveToJson(); + + const save_data = { + version: this.editor.version, + scene: scene_json, + timeline: timeline_json, + }; + + // TODO turn scene information into and object ... + let sceneThumbnail = undefined; + + if (this.editor.renderer) { + const imgData = this.editor.renderer.domElement.toDataURL(); + const response = await fetch(imgData); // Fetch the data URL + sceneThumbnail = await response.blob(); // Convert to Blob + } + + const result = await this.editor.api_manager.saveSceneState({ + saveJson: JSON.stringify(save_data), + sceneTitle, + sceneToken, + sceneThumbnail, + }); + + hideEditorLoader(); + + this.editor.generating_preview = false; // FIX THIS LATER WITH VICCCCCCCCCCCCCCCTORRRRRRRR + + return result; + } + + public async loadScene(scene_media_token: string) { + showEditorLoader(); + + this.editor.current_scene_media_token = scene_media_token; + + const scene_json = await this.editor.api_manager + .loadSceneState(this.editor.current_scene_media_token) + .catch((err) => { + hideEditorLoader(); + throw err; + }); + const proxyScene = new StoryTellerProxyScene( + this.editor.version, + this.editor.activeScene, + ); + await proxyScene.loadFromSceneJson(scene_json["scene"]); + this.editor.cam_obj = this.editor.activeScene.get_object_by_name(this.editor.camera_name); + this.editor.cam_obj?.layers.set(1); + + const proxyTimeline = new StoryTellerProxyTimeline( + this.editor.version, + this.editor.timeline, + this.editor.transform_engine, + this.editor.animation_engine, + this.editor.audio_engine, + this.editor.lipsync_engine, + this.editor.emotion_engine, + ); + await proxyTimeline.loadFromJson(scene_json["timeline"]); + + this.editor.timeline.checkEditorCanPlay(); + + hideEditorLoader(); + this.editor.timeline.scrub({ data: { currentTime: 0 } }) + } +} diff --git a/app/pages/PageEnigma/js/scene.ts b/app/pages/PageEnigma/Editor/scene.ts similarity index 100% rename from app/pages/PageEnigma/js/scene.ts rename to app/pages/PageEnigma/Editor/scene.ts diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/Editor/timeline.ts similarity index 76% rename from app/pages/PageEnigma/js/timeline.ts rename to app/pages/PageEnigma/Editor/timeline.ts index a3f693e4fe..ee34c937de 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/Editor/timeline.ts @@ -5,32 +5,23 @@ import { ClipUI } from "../datastructures/clips/clip_ui"; import Scene from "./scene.js"; import AudioEngine from "./audio_engine"; import TransformEngine from "./transform_engine"; -import LipSyncEngine from "./lip_sync_engine"; -import AnimationEngine from "./animation_engine"; +import { LipSyncEngine } from "./lip_sync_engine"; +import { AnimationEngine } from "./animation_engine"; -import Queue from "~/pages/PageEnigma/Queue/Queue"; +import Queue, { + UnionedActionTypes, + UnionedDataTypes, +} from "~/pages/PageEnigma/Queue/Queue"; import { QueueNames } from "../Queue/QueueNames"; import { toEngineActions } from "../Queue/toEngineActions"; import { fromEngineActions } from "../Queue/fromEngineActions"; import { AssetType } from "~/enums"; import { ClipGroup, ClipType } from "~/pages/PageEnigma/enums"; -import { MediaItem } from "~/pages/PageEnigma/models"; -import Editor from "~/pages/PageEnigma/js/editor"; +import { Keyframe, MediaItem, UpdateTime } from "~/pages/PageEnigma/models"; +import Editor from "~/pages/PageEnigma/Editor/editor"; import EmotionEngine from "./emotion_engine"; -import { GenerationOptions } from '../models/generationOptions'; - -// Every object uuid / entity has a track. -export class TimelineDataState { - timeline_items: ClipUI[]; - scrubber_frame_position: number; - constructor( - timeline_items: ClipUI[] = [], - scrubber_frame_position: number = 0, - ) { - this.timeline_items = timeline_items; - this.scrubber_frame_position = scrubber_frame_position; - } -} +import { GenerationOptions } from "~/pages/PageEnigma/models/generationOptions"; +import { Vector3 } from "three"; export class TimeLine { editorEngine: Editor; @@ -56,7 +47,7 @@ export class TimeLine { characters: { [key: string]: ClipGroup }; scene: Scene; - camera: THREE.Camera; + camera: THREE.Camera | null; mouse: THREE.Vector2 | undefined; current_time: number; @@ -71,7 +62,7 @@ export class TimeLine { animation_engine: AnimationEngine, emotion_engine: EmotionEngine, scene: Scene, - camera: THREE.Camera, + camera: THREE.Camera | null, mouse: THREE.Vector2 | undefined, camera_name: string, ) { @@ -103,7 +94,6 @@ export class TimeLine { this.current_time = 0; this.camera_name = camera_name; - } public async updateUI() { @@ -114,7 +104,7 @@ export class TimeLine { }); } - public async pushEvent(action: fromEngineActions, data: any) { + public async pushEvent(action: fromEngineActions, data: UnionedDataTypes) { //this.current_time += 0.75; Queue.publish({ queueName: QueueNames.FROM_ENGINE, @@ -123,47 +113,50 @@ export class TimeLine { }); } - public async handleTimelineActions(data: any) { - const action = data["action"]; + public async handleTimelineActions(data: { + action: UnionedActionTypes; + data: UnionedDataTypes; + }) { + const action = data.action; switch (action) { case toEngineActions.ADD_KEYFRAME: - await this.addKeyFrame(data); + await this.addKeyFrame(data.data as Keyframe); break; case toEngineActions.UPDATE_KEYFRAME: - await this.updateKeyFrame(data); + await this.updateKeyFrame(data.data as Keyframe); break; case toEngineActions.DELETE_KEYFRAME: - await this.deleteKeyFrame(data); + await this.deleteKeyFrame(data.data as Keyframe); break; case toEngineActions.ADD_CLIP: - await this.addClip(data); + await this.addClip(data.data as ClipUI); break; case toEngineActions.DELETE_CLIP: - await this.deleteClip(data); + await this.deleteClip(data.data as ClipUI); break; case toEngineActions.UPDATE_CLIP: - await this.updateClip(data); + await this.updateClip(data.data as ClipUI); break; case toEngineActions.UPDATE_TIME: - await this.scrub(data); + await this.scrub(data.data as UpdateTime); break; case toEngineActions.MUTE: - await this.mute(data, false); + await this.mute(data.data as ClipUI, false); break; case toEngineActions.UNMUTE: - await this.mute(data, true); + await this.mute(data.data as ClipUI, true); break; case toEngineActions.ADD_CHARACTER: - this.addCharacter(data); + this.addCharacter(data.data as MediaItem); break; case toEngineActions.ADD_OBJECT: { - const newObject = await this.addObject(data); - this.queueNewObjectMessage(newObject, data); + const newObject = await this.addObject(data.data as MediaItem); + this.queueNewObjectMessage(newObject, data.data as MediaItem); break; } case toEngineActions.ADD_SHAPE: { - const newShape = await this.addShape(data); - this.queueNewObjectMessage(newShape, data); + const newShape = await this.addShape(data.data as MediaItem); + this.queueNewObjectMessage(newShape, data.data as MediaItem); break; } case toEngineActions.ENTER_PREVIEW_STATE: @@ -175,21 +168,22 @@ export class TimeLine { case toEngineActions.TOGGLE_CAMERA_STATE: this.editorEngine.switchCameraView(); break; - case toEngineActions.GENERATE_VIDEO: - const options = data["data"]; // super overloaded talk to the devs about this. TODO... refactor - this.editorEngine.generation_options = options; + case toEngineActions.GENERATE_VIDEO: { + const options = data.data; // super overloaded talk to the devs about this. TODO... refactor + this.editorEngine.generation_options = options as GenerationOptions; this.editorEngine.generateVideo(); break; + } default: console.log("Action Not Wired", action); } } - public async addCharacter(data: { data: MediaItem }) { - const media_id = data.data.media_id; - const name = data.data.name; + public async addCharacter(data: MediaItem) { + const media_id = data.media_id; + const name = data.name; const pos = this.getPos(); - const new_data = { ...data.data }; + const new_data = { ...data }; const obj = await this.scene.loadGlbWithPlaceholder( media_id, @@ -214,7 +208,7 @@ export class TimeLine { this.addPlayableClip( new ClipUI( - data.data["version"], + data.version, ClipType.FAKE, ClipGroup.CHARACTER, "Default", @@ -297,10 +291,10 @@ export class TimeLine { return new THREE.Vector3(0, 0, 0); } - public async addObject(data: { data: MediaItem }) { + public async addObject(data: MediaItem) { const pos = this.getPos(); - const media_id = data.data.media_id; - const name = data.data.name; + const media_id = data.media_id; + const name = data.name; const obj = await this.scene.loadGlbWithPlaceholder( media_id, name, @@ -313,12 +307,12 @@ export class TimeLine { return obj; } - public async addShape({ data }: { data: MediaItem }) { + public async addShape(data: MediaItem) { const pos = this.getPos(); return this.editorEngine.create_parim(data.media_id, pos); } - public async addKeyFrame(data: any) { + public async addKeyFrame(data: Keyframe) { // KeyFrame Object // version: number; // clip_uuid: string; @@ -329,9 +323,9 @@ export class TimeLine { // rotation: XYZ; // scale: XYZ; // selected?: boolean; - const data_json = data["data"]; - const uuid = data_json["object_uuid"]; - const keyframe_uuid = data_json["keyframe_uuid"]; + const data_json = data; + const uuid = data_json.object_uuid; + const keyframe_uuid = data_json.keyframe_uuid; let object_name = this.scene.get_object_by_uuid(uuid)?.name; if (object_name === undefined) { @@ -340,12 +334,12 @@ export class TimeLine { this.transform_engine.addFrame( uuid, - data_json["offset"], - data_json["position"], - data_json["rotation"], - data_json["scale"], - data_json["offset"], - data_json["keyframe_uuid"], + data_json.offset, + data_json.position as Vector3, + data_json.rotation as Vector3, + data_json.scale as Vector3, + data_json.offset, + data_json.keyframe_uuid, ); await this.addPlayableClip( @@ -365,10 +359,10 @@ export class TimeLine { ); const point = this.scene.createPoint( - data_json["position"], - data_json["rotation"], - data_json["scale"], - data_json["keyframe_uuid"], + data_json.position as Vector3, + data_json.rotation as Vector3, + data_json.scale as Vector3, + data_json.keyframe_uuid, ); if (this.editorEngine.camera_person_mode) { point.visible = false; @@ -377,11 +371,7 @@ export class TimeLine { } public checkEditorCanPlay() { - if (this.getEndPoint() <= 1) { - this.editorEngine.can_playback = false; - } else { - this.editorEngine.can_playback = true; - } + this.editorEngine.can_playback = this.getEndPoint() > 1; this.editorEngine.updateSelectedUI(); } @@ -405,28 +395,25 @@ export class TimeLine { // Update react land here. } - public async addClip(data: any) { - const object_uuid = data["data"]["object_uuid"]; - const media_id = data["data"]["media_id"]; - const name = data["data"]["name"]; - const group = data["data"]["group"]; - const version = data["data"]["group"]; - const type = data["data"]["type"]; - const offset = data["data"]["offset"]; - const end_offset = data["data"]["length"] + offset; - let object_name = this.scene.get_object_by_uuid(object_uuid)?.name; - const clip_uuid = data["data"]["clip_uuid"]; - - if (object_name === undefined) { - object_name = "Undefined."; - } + public async addClip(data: ClipUI) { + const object_uuid = data.object_uuid; + const media_id = data.media_id; + const name = data.name; + const group = data.group; + const version = 1; + const type = data.type; + const offset = data.offset; + const end_offset = data.length + offset; + const object_name = + this.scene.get_object_by_uuid(object_uuid)?.name ?? "undefined"; + const clip_uuid = data.clip_uuid; switch (type) { case "animation": this.animation_engine.load_object(object_uuid, media_id, name); break; case "transform": - this.transform_engine.loadObject(object_uuid, data["data"]["length"]); + this.transform_engine.loadObject(object_uuid, data.length); break; case "expression": this.emotion_engine.loadClip(object_uuid, media_id); @@ -475,9 +462,9 @@ export class TimeLine { this.checkEditorCanPlay(); } - public async deleteKeyFrame(data: any) { - const keyframe_uuid = data["data"]["keyframe_uuid"]; - const object_uuid = data["data"]["object_uuid"]; + public async deleteKeyFrame(data: Keyframe) { + const keyframe_uuid = data.keyframe_uuid; + const object_uuid = data.object_uuid; this.transform_engine.clips[object_uuid].removeKeyframe(keyframe_uuid); this.scene.deletePoint(keyframe_uuid); for (const element of this.timeline_items) { @@ -498,14 +485,14 @@ export class TimeLine { this.checkEditorCanPlay(); } - public async updateKeyFrame(data: any) { - const keyframe_uuid = data["data"]["keyframe_uuid"]; - const keyframe_offset = data["data"]["offset"]; - const object_uuid = data["data"]["object_uuid"]; + public async updateKeyFrame(data: Keyframe) { + const keyframe_uuid = data.keyframe_uuid; + const keyframe_offset = data.offset; + const object_uuid = data.object_uuid; - const keyframe_pos = data["data"]["position"]; - const keyframe_rot = data["data"]["rotation"]; - const keyframe_scl = data["data"]["scale"]; + const keyframe_pos = data.position; + const keyframe_rot = data.rotation; + const keyframe_scl = data.scale; this.transform_engine.clips[object_uuid].setOffset( keyframe_uuid, @@ -513,26 +500,26 @@ export class TimeLine { ); this.transform_engine.clips[object_uuid].setTransform( keyframe_uuid, - keyframe_pos, - keyframe_rot, - keyframe_scl, + keyframe_pos as Vector3, + keyframe_rot as Vector3, + keyframe_scl as Vector3, ); this.scene.updatePoint( keyframe_uuid, - keyframe_pos, - keyframe_rot, - keyframe_scl, + keyframe_pos as Vector3, + keyframe_rot as Vector3, + keyframe_scl as Vector3, ); this.checkEditorCanPlay(); } - public async updateClip(data: any) { + public async updateClip(data: ClipUI) { // only length and offset changes here. - const object_uuid = data["data"]["object_uuid"]; - const media_id = data["data"]["media_id"]; - const offset = data["data"]["offset"]; - const length = data["data"]["length"] + offset; - const clip_uuid = data["data"]["clip_uuid"]; + const object_uuid = data.object_uuid; + const media_id = data.media_id; + const offset = data.offset; + const length = data.length + offset; + const clip_uuid = data.clip_uuid; for (const element of this.timeline_items) { if ( @@ -547,12 +534,10 @@ export class TimeLine { this.checkEditorCanPlay(); } - public async deleteClip(data: any) { - //const json_data = data["data"]; - const object_uuid = data["data"]["object_uuid"]; - const media_id = data["data"]["media_id"]; - //const type = data["type"]; - const clip_uuid = data["data"]["clip_uuid"]; + public async deleteClip(data: ClipUI) { + const object_uuid = data.object_uuid; + const media_id = data.media_id; + const clip_uuid = data.clip_uuid; for (let i = 0; i < this.timeline_items.length; i++) { const element = this.timeline_items[i]; @@ -569,13 +554,9 @@ export class TimeLine { this.checkEditorCanPlay(); } - public async scrubberUpdate(data: any) { - console.log(data); - } - - public async mute(data: any, isMute: boolean) { + public async mute(data: ClipUI, isMute: boolean) { this.timeline_items.forEach((element) => { - if (element.group === data.data["group"]) { + if (element.group === data.group) { element.should_play = isMute; } }); @@ -585,28 +566,20 @@ export class TimeLine { this.timeline_items.push(clip); } - public async scrub(data: any): Promise { + public async scrub(data: UpdateTime): Promise { if (this.is_playing) { return; } - const value = Math.floor(data["data"]["currentTime"]); - this.setScrubberPosition(value); + const value = Math.floor(data.currentTime); + await this.setScrubberPosition(value); this.current_time = value; - this.update(); + await this.update(); if (this.editorEngine.switchPreviewToggle) { - this.editorEngine.generateFrame(); + await this.editorEngine.generateFrame(); } } - public async stepFrame(frames: number) { - this.update(); - this.scrubber_frame_position += frames; - this.pushEvent(fromEngineActions.UPDATE_TIME, { - currentTime: this.scrubber_frame_position, - }); - } - // public streaming events into the timeline from public async setScrubberPosition(offset: number) { this.scrubber_frame_position = offset; // in ms @@ -696,7 +669,7 @@ export class TimeLine { if ( element.offset <= this.scrubber_frame_position && this.scrubber_frame_position <= element.length && - element.should_play === true + element.should_play ) { // run async // element.play() @@ -750,8 +723,12 @@ export class TimeLine { object.uuid + element.media_id ].play(object); const fps = 60; - await this.animation_engine.clips[object.uuid + element.media_id].step( - (this.scrubber_frame_position-element.offset) / fps, this.is_playing, this.scrubber_frame_position// Double FPS for best result. + await this.animation_engine.clips[ + object.uuid + element.media_id + ].step( + (this.scrubber_frame_position - element.offset) / fps, + this.is_playing, + this.scrubber_frame_position, // Double FPS for best result. ); //this.animation_engine.clips[object.uuid + element.media_id].update_bones(); } diff --git a/app/pages/PageEnigma/js/transform_engine.ts b/app/pages/PageEnigma/Editor/transform_engine.ts similarity index 100% rename from app/pages/PageEnigma/js/transform_engine.ts rename to app/pages/PageEnigma/Editor/transform_engine.ts diff --git a/app/pages/PageEnigma/Editor/video_generation.ts b/app/pages/PageEnigma/Editor/video_generation.ts new file mode 100644 index 0000000000..ea1fa33f54 --- /dev/null +++ b/app/pages/PageEnigma/Editor/video_generation.ts @@ -0,0 +1,301 @@ +import Editor from "./editor"; +import { EditorStates } from "~/pages/PageEnigma/enums"; +import { editorState, previewSrc } from "../signals/engine"; +import Queue from "~/pages/PageEnigma/Queue/Queue"; +import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; +import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions"; +import { ToastTypes } from "~/enums"; +import { createFFmpeg, fetchFile, FFmpeg } from "@ffmpeg/ffmpeg"; +import { ClipUI } from "../datastructures/clips/clip_ui.js"; +import { ClipType } from "~/pages/PageEnigma/enums"; +import { Visibility } from "./api_manager.js"; +import * as THREE from "three"; +import { getSceneSignals } from "~/signals"; + +export class VideoGeneration { + editor: Editor; + + constructor(editor: Editor) { + this.editor = editor; + } + + async generateFrame() { + if (!this.editor.generating_preview && this.editor.rawRenderer) { + this.editor.generating_preview = true; + this.editor.removeTransformControls(); + this.editor.activeScene.renderMode(true); + if (this.editor.activeScene.hot_items) { + this.editor.activeScene.hot_items.forEach((element) => { + element.visible = false; + }); + } + + // if (this.editor.render_camera && this.editor.cam_obj) { + // this.editor.render_camera.position.copy(this.editor.cam_obj.position); + // this.editor.render_camera.rotation.copy(this.editor.cam_obj.rotation); + // } + + previewSrc.value = ""; + + this.editor.rawRenderer.setSize( + this.editor.render_width, + this.editor.render_height, + ); + this.editor.render_camera.aspect = + this.editor.render_width / this.editor.render_height; + this.editor.render_camera.updateProjectionMatrix(); + this.editor.rawRenderer.render( + this.editor.activeScene.scene, + this.editor.render_camera, + ); + const imgData = this.editor.rawRenderer.domElement.toDataURL(); + const response = await fetch(imgData); // Fetch the data URL + const blob = await response.blob(); // Convert to Blob + + if (!this.editor.camera_person_mode) { + this.editor.switchCameraView(); + editorState.value = EditorStates.PREVIEW; + } + + this.editor.generating_preview = false; + + try { + const url = await this.editor.api_manager.uploadMediaFrameGeneration( + blob, + "render.png", + this.editor.art_style, + this.editor.positive_prompt, + this.editor.negative_prompt, + ); + + previewSrc.value = url; + return Promise.resolve(url); + } catch (err: any) { + Queue.publish({ + queueName: QueueNames.FROM_ENGINE, + action: fromEngineActions.POP_A_TOAST, + data: { + type: ToastTypes.ERROR, + message: err.message, + }, + }); + } + } + } + + async convertAudioClip(itteration: number, ffmpeg: FFmpeg, clip: ClipUI) { + // Base audio clip names. + const video_og: string = itteration + "tmp.mp4"; + const wav_name: string = itteration + "tmp.wav"; + const new_video: string = itteration + 1 + "tmp.mp4"; + let startFrame: number = clip.offset; + let endFrame: number = clip.length; + + if (endFrame > this.editor.timeline.timeline_limit) { + endFrame = this.editor.timeline.timeline_limit; + } + if (startFrame > this.editor.timeline.timeline_limit) { + startFrame = this.editor.timeline.timeline_limit - 1; + } + + const startTime: number = startFrame / this.editor.cap_fps; + const endTime: number = endFrame / this.editor.cap_fps; + const end: number = endTime - startTime; + + // Use ffmpeg to convert audio to video. + const audioSegment: string = "as_" + wav_name; + await ffmpeg.FS( + "writeFile", + wav_name, + await fetchFile( + await this.editor.api_manager.getMediaFile(clip.media_id), + ), + ); + await ffmpeg.run( + "-i", + wav_name, + "-ss", + "0", + "-to", + "" + end, + "-max_muxing_queue_size", + "999999", + audioSegment, + ); + + await ffmpeg.run( + "-i", + video_og, + "-max_muxing_queue_size", + "999999", + `${itteration}empty_tmp.wav`, + ); + + await ffmpeg.run( + "-i", + `${itteration}empty_tmp.wav`, + "-i", + audioSegment, + "-filter_complex", + "[1:a]adelay=" + + startTime * 1000 + + "|" + + startTime * 1000 + + "[a1];[0:a][a1]amix=inputs=2[a]", + "-map", + "[a]", + `${itteration}final_tmp.wav`, + ); + + // Combine final audio with video. + await ffmpeg.run( + "-i", + video_og, + "-i", + `${itteration}final_tmp.wav`, + "-c:v", + "copy", + "-c:a", + "aac", + "-map", + "0:v:0", + "-map", + "1:a:0", + "-strict", + "experimental", + new_video, + ); + } + + async stopPlaybackAndUploadVideo(compile_audio: boolean = true) { + this.editor.rendering = false; + + //const videoBlob = new Blob(this.frame_buffer, { type: "video/webm" }); + //const videoURL = URL.createObjectURL(videoBlob); + + this.editor.generating_preview = true; + const ffmpeg = createFFmpeg({ log: false }); + await ffmpeg.load(); + + this.editor.updateLoad(50, "Processing ..."); + + // Write the Uint8Array to the FFmpeg file system + //ffmpeg.FS("writeFile", "input.webm", await fetchFile(videoURL)); + + for (let index = 0; index < this.editor.frame_buffer.length; index++) { + const element = this.editor.frame_buffer[index]; + await ffmpeg.FS( + "writeFile", + `image${index}.png`, + await fetchFile(element), + ); + } + + await ffmpeg.run( + "-framerate", + "" + this.editor.cap_fps, + "-i", + "image%d.png", + "input.mp4", + ); + + await ffmpeg.run( + "-i", + "input.mp4", + "-f", + "lavfi", + "-i", + "anullsrc", // This adds a silent audio track + "-max_muxing_queue_size", + "999999", + "-vf", + "select=gte(n\\,1),scale=1024:576", // scale=1024:576 + "-c:v", + "libx264", // Specify video codec (optional, but recommended for MP4) + "-c:a", + "aac", // Specify audio codec (optional, but recommended for MP4) + "-shortest", // Ensure output duration matches the shortest stream (video or audio) + "-pix_fmt", + "yuv420p", + "-f", + "mp4", + "0tmp.mp4", + ); + + let itteration = 0; + + if (compile_audio) { + for (const clip of this.editor.timeline.timeline_items) { + if (clip.type == ClipType.AUDIO) { + await this.convertAudioClip(itteration, ffmpeg, clip); + itteration += 1; + } + } + } + + const output = ffmpeg.FS("readFile", itteration + "tmp.mp4"); + + ffmpeg.exit(); + this.editor.generating_preview = false; + + // Create a Blob from the output file for downloading + const blob = new Blob([output.buffer], { type: "video/mp4" }); + + const title = getSceneSignals().title || "Untitled"; + + const style_name = this.editor.art_style.toString(); + const media_token = this.editor.current_scene_media_token || undefined; + + const data: any = await this.editor.api_manager.uploadMedia({ + blob, + fileName: `${title}.mp4`, + title, + styleName: style_name, + maybe_scene_source_media_file_token: media_token, + }); + + if (data == null) { + return; + } + const upload_token = data["media_file_token"]; + + const result = await this.editor.api_manager + .stylizeVideo( + upload_token, + this.editor.art_style, + this.editor.positive_prompt, + this.editor.negative_prompt, + Visibility.Public, + this.editor.generation_options.faceDetail, + this.editor.generation_options.upscale, + this.editor.generation_options.styleStrength, + ) + .catch((error) => { + // TODO handle stylize error. + console.log(error); + }); + + // {"success":true,"inference_job_token":"jinf_j3nbqbd15wqxb0xcks13qh3f3bz"} + this.editor.updateLoad(100, "Done Check Your Media Tab On Profile."); + this.editor.endLoading(); + + this.editor.recorder = undefined; + if (this.editor.rawRenderer) { + this.editor.rawRenderer.setSize( + this.editor.startRenderWidth, + this.editor.startRenderHeight, + ); + } + + this.editor.canvasRenderCamReference = + document.getElementById("camera-view"); + this.editor.rawRenderer = new THREE.WebGLRenderer({ + antialias: false, + canvas: this.editor.canvasRenderCamReference, + preserveDrawingBuffer: true, + }); + this.editor.activeScene.renderMode(false); + + this.editor.switchEdit(); + } +} diff --git a/app/pages/PageEnigma/Queue/Queue.ts b/app/pages/PageEnigma/Queue/Queue.ts index 405b2223b4..e792643510 100644 --- a/app/pages/PageEnigma/Queue/Queue.ts +++ b/app/pages/PageEnigma/Queue/Queue.ts @@ -18,12 +18,12 @@ export type ToastDataType = { message: string; }; -type UnionedActionTypes = +export type UnionedActionTypes = | fromEngineActions | toEngineActions | toTimelineActions; -type UnionedDataTypes = +export type UnionedDataTypes = | QueueClip | UpdateTime | QueueKeyframe diff --git a/app/pages/PageEnigma/comps/ControlPanelSceneObject/ControlPanelSceneObject.tsx b/app/pages/PageEnigma/comps/ControlPanelSceneObject/ControlPanelSceneObject.tsx index e90ca38987..9c4b1e6cbd 100644 --- a/app/pages/PageEnigma/comps/ControlPanelSceneObject/ControlPanelSceneObject.tsx +++ b/app/pages/PageEnigma/comps/ControlPanelSceneObject/ControlPanelSceneObject.tsx @@ -65,8 +65,6 @@ export const ControlPanelSceneObject = () => { const [color, setColor] = useState("#ffffff"); - console.log("color", color); - const colorInputId = useId(); const toggleCollapse = () => { @@ -111,8 +109,6 @@ export const ControlPanelSceneObject = () => { return; } - console.log("44"); - const vectors = currentSceneObject.objectVectors; // local state relies on strings @@ -179,7 +175,12 @@ export const ControlPanelSceneObject = () => { const handleOnAddKeyFrame = () => { if (currentSceneObject) { - if (position == null || rotation == null || scale == null) { + if ( + position == null || + rotation == null || + scale == null || + currentSceneObject == null + ) { return; } diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/StylizeTab/ItemPicker.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/StylizeTab/ItemPicker.tsx index bce8651310..faa54f5ea9 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/StylizeTab/ItemPicker.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/StylizeTab/ItemPicker.tsx @@ -1,5 +1,5 @@ import { twMerge } from "tailwind-merge"; -import { ArtStyle } from "~/pages/PageEnigma/js/api_manager"; +import { ArtStyle } from "~/pages/PageEnigma/Editor/api_manager"; import { H4 } from "~/components"; import { ImgHTMLAttributes } from "react"; @@ -37,7 +37,8 @@ export const ItemPicker = ({ maxWidth: (width as number) + 4, maxHeight: (height as number) + 4, }} - onClick={handleSelected}> + onClick={handleSelected} + > + }`} + > { label: string; diff --git a/app/pages/PageEnigma/comps/StyleSelection/StyleSelection.tsx b/app/pages/PageEnigma/comps/StyleSelection/StyleSelection.tsx index 8a5cbdcd4f..6c0762f665 100644 --- a/app/pages/PageEnigma/comps/StyleSelection/StyleSelection.tsx +++ b/app/pages/PageEnigma/comps/StyleSelection/StyleSelection.tsx @@ -4,7 +4,7 @@ import { faAngleLeft, faAngleRight } from "@fortawesome/pro-solid-svg-icons"; import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext"; import { styleList } from "~/pages/PageEnigma/styleList"; -import { ArtStyle } from "~/pages/PageEnigma/js/api_manager"; +import { ArtStyle } from "~/pages/PageEnigma/Editor/api_manager"; import { ButtonIcon, Label, Textarea } from "~/components"; import { ItemPicker } from "./ItemPicker"; diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts b/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts index df53b2234f..7c8bf3f8b0 100644 --- a/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts +++ b/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts @@ -20,6 +20,7 @@ import { QueueKeyframe, UpdateTime, } from "~/pages/PageEnigma/models"; +import { ClipGroup } from "~/pages/PageEnigma/enums"; import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext"; import { addToast } from "~/signals"; import { ToastTypes } from "~/enums"; diff --git a/app/pages/PageEnigma/constants/RandomTexts.ts b/app/pages/PageEnigma/constants/RandomTexts.ts index 7359a49230..807ca6dd25 100644 --- a/app/pages/PageEnigma/constants/RandomTexts.ts +++ b/app/pages/PageEnigma/constants/RandomTexts.ts @@ -1,4 +1,4 @@ -import { ArtStyle } from "~/pages/PageEnigma/js/api_manager"; +import { ArtStyle } from "~/pages/PageEnigma/Editor/api_manager"; export const RandomTextsPositive: Record = { [ArtStyle.AnimeStandard]: [ diff --git a/app/pages/PageEnigma/contexts/EngineContext/EngineContext.tsx b/app/pages/PageEnigma/contexts/EngineContext/EngineContext.tsx index 9da0df1808..e15c567e1c 100644 --- a/app/pages/PageEnigma/contexts/EngineContext/EngineContext.tsx +++ b/app/pages/PageEnigma/contexts/EngineContext/EngineContext.tsx @@ -1,4 +1,4 @@ import { createContext } from "react"; -import Editor from "~/pages/PageEnigma/js/editor"; +import Editor from "~/pages/PageEnigma/Editor/editor"; -export const EngineContext = createContext(null); \ No newline at end of file +export const EngineContext = createContext(null); diff --git a/app/pages/PageEnigma/contexts/EngineContext/EngineProvider.tsx b/app/pages/PageEnigma/contexts/EngineContext/EngineProvider.tsx index c9b59269c7..55ce6a5f14 100644 --- a/app/pages/PageEnigma/contexts/EngineContext/EngineProvider.tsx +++ b/app/pages/PageEnigma/contexts/EngineContext/EngineProvider.tsx @@ -1,7 +1,7 @@ import { ReactNode, useEffect, useState } from "react"; import { EngineContext } from "./EngineContext"; -import Editor from "~/pages/PageEnigma/js/editor"; +import Editor from "~/pages/PageEnigma/Editor/editor"; interface Props { sceneToken?: string; diff --git a/app/pages/PageEnigma/datastructures/clips/animation_clip.ts b/app/pages/PageEnigma/datastructures/clips/animation_clip.ts index 030b2c8cfb..48719eb7c8 100644 --- a/app/pages/PageEnigma/datastructures/clips/animation_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/animation_clip.ts @@ -2,7 +2,7 @@ import * as THREE from "three"; import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; import { FBXLoader } from "three/examples/jsm/Addons.js"; -import { MoveAIResult, Retarget } from "../../js/retargeting"; +import { MoveAIResult, Retarget } from "../../Editor/retargeting"; import { environmentVariables } from "~/signals"; export class AnimationClip { diff --git a/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts index 39b80c19c6..7988f635f4 100644 --- a/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts @@ -1,4 +1,4 @@ -import { LipSync } from "../../js/lipsync"; +import { LipSync } from "../../Editor/lipsync"; import * as THREE from "three"; import { environmentVariables } from "~/signals"; interface AudioData { diff --git a/app/pages/PageEnigma/enums/AssetFilterOption.ts b/app/pages/PageEnigma/enums/AssetFilterOption.ts new file mode 100644 index 0000000000..8b3ac7988b --- /dev/null +++ b/app/pages/PageEnigma/enums/AssetFilterOption.ts @@ -0,0 +1,5 @@ +export enum AssetFilterOption { + ALL, + MINE, + BOOKMARKED, +} diff --git a/app/pages/PageEnigma/enums/AssetType.ts b/app/pages/PageEnigma/enums/AssetType.ts new file mode 100644 index 0000000000..c3fbea58b1 --- /dev/null +++ b/app/pages/PageEnigma/enums/AssetType.ts @@ -0,0 +1,10 @@ +export enum AssetType { + ANIMATION = "animation", + AUDIO = "audio", + CAMERA = "camera", + CHARACTER = "character", + EXPRESSION = "expression", + OBJECT = "object", + SHAPE = "shape", + STYLE = "style", // TODO Remove +} diff --git a/app/pages/PageEnigma/models/movies.ts b/app/pages/PageEnigma/models/movies.ts index 3760fb8972..3db8b6e74a 100644 --- a/app/pages/PageEnigma/models/movies.ts +++ b/app/pages/PageEnigma/models/movies.ts @@ -1,4 +1,5 @@ -import { UserDetailsLight } from "./voice"; +import { UserDetailsLight } from "./types"; + import { MediaFileClass, MediaFileSubtype, diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts index d27b01e33d..a70fd96845 100644 --- a/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts @@ -4,7 +4,7 @@ import { StoryTellerProxy3DObject, ObjectJSON, } from "./storyteller_proxy_3d_object"; -import Scene from "../js/scene"; +import Scene from "../Editor/scene"; interface LookUpDictionary { [key: string]: StoryTellerProxy3DObject; @@ -70,7 +70,9 @@ export class StoryTellerProxyScene { let obj; switch (token) { case "Parim": - let prim_uuid = this.scene.instantiate(json_object.object_name).uuid; + let prim_uuid = this.scene.instantiate( + json_object.object_name, + ).uuid; obj = this.scene.get_object_by_uuid(prim_uuid); break; case "DirectionalLight": @@ -78,7 +80,10 @@ export class StoryTellerProxyScene { break; default: if (token.includes("m_")) { - obj = await this.scene.loadGlbWithPlaceholder(token, json_object.object_name); + obj = await this.scene.loadGlbWithPlaceholder( + token, + json_object.object_name, + ); } else if (token.includes("Point::")) { let keyframe_uuid = token.replace("Point::", ""); obj = this.scene.createPoint( diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts index a3e69a6259..3b3b1db5c9 100644 --- a/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts @@ -4,151 +4,208 @@ import { ClipUI } from "../datastructures/clips/clip_ui"; import { EmotionClip } from "../datastructures/clips/emotion_clip"; import { LipSyncClip } from "../datastructures/clips/lipsync_clip"; import { TransformClip } from "../datastructures/clips/transform_clip"; -import AnimationEngine from "../js/animation_engine"; -import AudioEngine from "../js/audio_engine"; -import EmotionEngine from "../js/emotion_engine"; -import LipSyncEngine from "../js/lip_sync_engine"; -import { TimeLine } from "../js/timeline"; -import TransformEngine from "../js/transform_engine"; - +import AnimationEngine from "../Editor/animation_engine"; +import AudioEngine from "../Editor/audio_engine"; +import EmotionEngine from "../Editor/emotion_engine"; +import LipSyncEngine from "../Editor/lip_sync_engine"; +import { TimeLine } from "../Editor/timeline"; +import TransformEngine from "../Editor/transform_engine"; export class StoryTellerProxyTimeline { - timeline: TimeLine - transform_engine: TransformEngine - animation_engine: AnimationEngine - audio_engine: AudioEngine - lipsync_engine: LipSyncEngine - emotion_engine: EmotionEngine + timeline: TimeLine; + transform_engine: TransformEngine; + animation_engine: AnimationEngine; + audio_engine: AudioEngine; + lipsync_engine: LipSyncEngine; + emotion_engine: EmotionEngine; - constructor(version: number, timeline: TimeLine, transform_engine: TransformEngine, animation_engine: AnimationEngine, audio_engine: AudioEngine, lipsync_engine: LipSyncEngine, emotion_engine: EmotionEngine) { - this.timeline = timeline; - this.transform_engine = transform_engine; - this.animation_engine = animation_engine; - this.audio_engine = audio_engine; - this.lipsync_engine = lipsync_engine; - this.emotion_engine = emotion_engine; + constructor( + version: number, + timeline: TimeLine, + transform_engine: TransformEngine, + animation_engine: AnimationEngine, + audio_engine: AudioEngine, + lipsync_engine: LipSyncEngine, + emotion_engine: EmotionEngine, + ) { + this.timeline = timeline; + this.transform_engine = transform_engine; + this.animation_engine = animation_engine; + this.audio_engine = audio_engine; + this.lipsync_engine = lipsync_engine; + this.emotion_engine = emotion_engine; + } + + private async getItems(items: any[]): Promise { + let timeline_items_data = []; + for (let index = 0; index < items.length; index++) { + const element = items[index]; + timeline_items_data.push(element.toJSON()); } + return timeline_items_data; + } - private async getItems(items: any[]): Promise { - let timeline_items_data = []; - for (let index = 0; index < items.length; index++) { - const element = items[index]; - timeline_items_data.push(element.toJSON()); - }; - return timeline_items_data; + private async getItemsDict(items: { + [key: string]: any; + }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = element.toJSON(); + } } + return timeline_items_data; + } - private async getItemsDict(items: { [key: string]: any }): Promise<{ [key: string]: any }> { - let timeline_items_data: { [key: string]: any } = {}; - for (const key in items) { - if (items.hasOwnProperty(key)) { - const element = items[key]; - timeline_items_data[key] = element.toJSON(); - } - } - return timeline_items_data; + public async saveToJson(): Promise { + let timeline_json = { + timeline: await this.getItems(this.timeline.timeline_items), + transform: await this.getItemsDict(this.transform_engine.clips), + animation: await this.getItemsDict(this.animation_engine.clips), + audio: await this.getItemsDict(this.audio_engine.clips), + lipsync: await this.getItemsDict(this.lipsync_engine.clips), + emotion: await this.getItemsDict(this.emotion_engine.clips), + }; + console.log(timeline_json); + + return timeline_json; + } + + private async loadTimelineClips(timeline_clips: any[]): Promise { + let new_clips = []; + if (timeline_clips) { + for (let index = 0; index < timeline_clips.length; index++) { + const element = timeline_clips[index]; + let clip = new ClipUI( + element.version, + element.type, + element.group, + element.name, + element.media_id, + element.clip_uuid, + element.object_uuid, + element.object_name, + element.start_offset, + element.ending_offset, + element.keyframe_offset, + ); + new_clips.push(clip); + } } + return new_clips; + } - public async saveToJson(): Promise { - let timeline_json = { - timeline: await this.getItems(this.timeline.timeline_items), - transform: await this.getItemsDict(this.transform_engine.clips), - animation: await this.getItemsDict(this.animation_engine.clips), - audio: await this.getItemsDict(this.audio_engine.clips), - lipsync: await this.getItemsDict(this.lipsync_engine.clips), - emotion: await this.getItemsDict(this.emotion_engine.clips), - } - console.log(timeline_json) - - return timeline_json; + private async loadTransformClips(items: { + [key: string]: any; + }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = new TransformClip( + element.version, + element.object_uuid, + element.length, + element.media_id, + ); + timeline_items_data[key].keyframes = element.keyframes; + } } + return timeline_items_data; + } - private async loadTimelineClips(timeline_clips: any[]): Promise { - let new_clips = []; - if (timeline_clips) { - for (let index = 0; index < timeline_clips.length; index++) { - const element = timeline_clips[index]; - let clip = new ClipUI( - element.version, element.type, - element.group, element.name, - element.media_id, element.clip_uuid, - element.object_uuid, - element.object_name, - element.start_offset, element.ending_offset, - element.keyframe_offset); - new_clips.push(clip); - } - } - return new_clips; + private async loadAudioClips(items: { + [key: string]: any; + }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = new AudioClip( + element.version, + element.media_id, + element.volume, + ); + } } + return timeline_items_data; + } - private async loadTransformClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> { - let timeline_items_data: { [key: string]: any } = {}; - for (const key in items) { - if (items.hasOwnProperty(key)) { - const element = items[key]; - timeline_items_data[key] = new TransformClip(element.version, element.object_uuid, element.length, element.media_id); - timeline_items_data[key].keyframes = element.keyframes; - } - } - return timeline_items_data; + private async loadAnimationClips(items: { + [key: string]: any; + }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = new AnimationClip( + element.version, + element.media_id, + "remote", + element.object_uuid, + element.speed, + element.length, + element.clip_name, + ); + } } + return timeline_items_data; + } - private async loadAudioClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> { - let timeline_items_data: { [key: string]: any } = {}; - for (const key in items) { - if (items.hasOwnProperty(key)) { - const element = items[key]; - timeline_items_data[key] = new AudioClip(element.version, element.media_id, element.volume); - } - } - return timeline_items_data; + private async loadLipsyncClips(items: { + [key: string]: any; + }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = new LipSyncClip( + element.version, + element.media_id, + element.volume, + ); + } } + return timeline_items_data; + } - private async loadAnimationClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> { - let timeline_items_data: { [key: string]: any } = {}; - for (const key in items) { - if (items.hasOwnProperty(key)) { - const element = items[key]; - timeline_items_data[key] = new AnimationClip(element.version, element.media_id, "remote", element.object_uuid, element.speed, element.length, element.clip_name); - } - } - return timeline_items_data; + private async loadEmotionClips(items: { + [key: string]: any; + }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = new EmotionClip( + element.version, + element.media_id, + ); + } } + return timeline_items_data; + } - private async loadLipsyncClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> { - let timeline_items_data: { [key: string]: any } = {}; - for (const key in items) { - if (items.hasOwnProperty(key)) { - const element = items[key]; - timeline_items_data[key] = new LipSyncClip(element.version, element.media_id, element.volume); - } - } - return timeline_items_data; - } + public async loadFromJson(timeline: any) { + this.timeline.timeline_items = await this.loadTimelineClips( + timeline["timeline"], + ); + this.transform_engine.clips = await this.loadTransformClips( + timeline["transform"], + ); + this.audio_engine.clips = await this.loadAudioClips(timeline["audio"]); + this.lipsync_engine.clips = await this.loadLipsyncClips( + timeline["lipsync"], + ); + this.animation_engine.clips = await this.loadAnimationClips( + timeline["animation"], + ); + this.emotion_engine.clips = await this.loadEmotionClips( + timeline["emotion"], + ); - private async loadEmotionClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> { - let timeline_items_data: { [key: string]: any } = {}; - for (const key in items) { - if (items.hasOwnProperty(key)) { - const element = items[key]; - timeline_items_data[key] = new EmotionClip(element.version, element.media_id); - } - } - return timeline_items_data; - } - - public async loadFromJson(timeline: any) { - this.timeline.timeline_items = await this.loadTimelineClips(timeline['timeline']); - this.transform_engine.clips = await this.loadTransformClips(timeline['transform']); - this.audio_engine.clips = await this.loadAudioClips(timeline['audio']); - this.lipsync_engine.clips = await this.loadLipsyncClips(timeline['lipsync']); - this.animation_engine.clips = await this.loadAnimationClips(timeline['animation']); - this.emotion_engine.clips = await this.loadEmotionClips(timeline['emotion']); - - console.log(this.timeline.timeline_items) - - this.timeline.updateUI() - } + console.log(this.timeline.timeline_items); + this.timeline.updateUI(); + } } diff --git a/app/pages/PageEnigma/styleList.ts b/app/pages/PageEnigma/styleList.ts index d50878949a..b800386e1a 100644 --- a/app/pages/PageEnigma/styleList.ts +++ b/app/pages/PageEnigma/styleList.ts @@ -1,4 +1,4 @@ -import { ArtStyle } from "~/pages/PageEnigma/js/api_manager"; +import { ArtStyle } from "~/pages/PageEnigma/Editor/api_manager"; export const styleList = [ {