From f258f89a903e4963e83878542861a57df17b7054 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Tue, 26 Mar 2024 17:51:11 -0400 Subject: [PATCH 01/23] Saving the scene and loading it need to load the clips and serialize that. --- .../ControlsTopButtons/ControlsTopButtons.tsx | 2 +- app/pages/PageEnigma/js/api_manager.ts | 101 ++++++++++++++---- app/pages/PageEnigma/js/editor.js | 12 ++- 3 files changed, 90 insertions(+), 25 deletions(-) diff --git a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx index 75a5ecd93c..3a9195e78d 100644 --- a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx +++ b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx @@ -24,7 +24,7 @@ export const ControlsTopButtons = () => { }; const handleButtonRender = () => { console.log("Saving GLB to server for reference"); - editorEngine._upload_for_testing(); + editorEngine?._upload_for_testing(); //editorEngine?.togglePlayback(); }; const handleButtonPlay = () => {}; diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index f1e484af16..0d27a8fbf2 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -18,46 +18,97 @@ class APIManager { // it will always return a boolean success and the media_file_token // they take an optional media_file_token as a parameter for the file to overwrite, which must belong to the user. //If it's an anonymous user, we check the anonymous visitor token/cookie + async saveSceneState( scene: THREE.Scene, timeline: TimelineCurrentReactState, media_file_token: string | null = null, ): Promise { - if (media_file_token == null) { // create new scene. let file = await this.gltfExport(scene); let response = await this.uploadGLB(file); - let success = response["success"]; if (success) { let media_file_token = response["media_file_token"]; - await this._saveSceneAndTimelineToJSONSpec(media_file_token, timeline); - } else { - throw Error("Saving the Scene Resulted in an Error Success False"); - } - } else { - // update scene or overwrite glb scene. - } - return ""; - } + + console.log(`media file token: ${media_file_token}`) - // which will create or update/replace scene descriptor JSON files + const result = await this._saveSceneAndTimelineToJSONSpec(media_file_token, timeline); + return result + } + + throw Error("Saving the Scene Resulted in an Error Success False"); + } + + // For JSON files. + // Form-multipart POST with parameters: + // uuid_idempotency_token + // file + // media_file_token (optional; the file to replace if present) async _saveSceneAndTimelineToJSONSpec( media_file_token: string, timeline: TimelineCurrentReactState, ): Promise { const url = `${this.baseUrl}/v1/media_files/write/scene_file`; let uuid = uuidv4(); - return ""; + + // turn json into a blob + const file_test = {"scene_media_file_id":uuid} + const json = JSON.stringify(file_test); + const blob = new Blob([json], {type: 'application/json'}); + const fileName = `${uuid}.json`; + + const formData = new FormData(); + formData.append("uuid_idempotency_token", uuid); + formData.append("file", blob, fileName); + formData.append("source", "file"); + formData.append("type", "scene_json"); + formData.append("source", "file"); + const response = await fetch(url, { + method: "POST", + credentials: "include", + headers: { + Accept: "application/json", + }, + body: formData, + }); + + if (!response.ok) { + throw new Error("Failed to Send Data"); + } + + console.log(`RESPONSE ${response}`) + + return response.json(); // or handle the response as appropriate } // media_file_token that describes the scene async loadScene(media_file_token: string): Promise { - // grab the json file from media files - let json_response = `{ - - }` - return "" + let api_base_url = "https://api.fakeyou.com"; + let url = `${api_base_url}/v1/media_files/file/${media_file_token}` + let response = await fetch(url); + let json = await JSON.parse(await response.text()); + let bucketPath = json["media_file"]["public_bucket_path"]; + let media_base_url = "https://storage.googleapis.com/vocodes-public" + let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path + + let fileResponse = await fetch(media_url); + + if (!fileResponse.ok) { + throw new Error("Failed to download file"); + } + + // Convert the response to a Blob + let blob = await fileResponse.blob(); + + const json_value:string = await new Promise((resolve, reject) => { + let reader = new FileReader(); + reader.onloadend = () => resolve(JSON.parse(reader.result as string)); + reader.onerror = reject; + reader.readAsText(blob); + }); + //console.log(json_value) + return json_value } // Provides bucket path to the actual file to download from the media file @@ -114,10 +165,20 @@ class APIManager { return response.json(); // or handle the response as appropriate } - // to save to a file then upload the scene. + // To save to a file then upload the scene. + // /v1/media_files/write/engine_asset + // An upsert for files. + // If the file exists, it'll check ownership prior to overwriting. + // For GLB, GLTF, BVH, FBX, etc. game engine-type files. + // Form-multipart POST with parameters: + // uuid_idempotency_token + // file + // media_file_token (optional; the file to replace if present) + // media_file_subtype (optional; mixamo, mocap_net, scene_import, animation_only) + // media_file_class (audio, image, video, animation, character, prop, scene, unknown) async uploadGLB(file: File): Promise { // /v1/media_files/write/engine_asset REPLACE with this api endpoint - const url = `${this.baseUrl}/v1/media_files/upload/engine_asset`; + const url = `${this.baseUrl}/v1/media_files/write/engine_asset`; let uuid = uuidv4(); const formData = new FormData(); formData.append("uuid_idempotency_token", uuid); diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index c8b2b04515..18cdf2f5ae 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -173,11 +173,15 @@ class Editor { // This is the default scene. // "m_189p8hj0eyypbg74kkhcpehwpjhnkz" scene with the fox. + //let result = await this.api_manager.saveSceneState(this.activeScene.scene) - let result = await this.api_manager.getMediaFile( - "m_189p8hj0eyypbg74kkhcpehwpjhnkz", - ); - console.log(result); + + // let result = await this.api_manager.getMediaFile( + // "m_189p8hj0eyypbg74kkhcpehwpjhnkz", + // ); + + //const result = this.api_manager.saveSceneState(this.activeScene.scene) + this.api_manager.loadScene("m_n3k7nc0r5scr5zf92febh52h39g721") } async _serialize_timeline() { From 5bf90952010d6f9c56866b1b533e8a6c0489a102 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Tue, 26 Mar 2024 21:00:44 -0400 Subject: [PATCH 02/23] added 3d controls. --- .../PageEnigma/comps/Controls3D/Controls3D.tsx | 2 +- .../comps/ControlsTopButtons/ControlsTopButtons.tsx | 4 +++- app/pages/PageEnigma/js/editor.js | 13 ++++++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/app/pages/PageEnigma/comps/Controls3D/Controls3D.tsx b/app/pages/PageEnigma/comps/Controls3D/Controls3D.tsx index c36187a072..1c072efdae 100644 --- a/app/pages/PageEnigma/comps/Controls3D/Controls3D.tsx +++ b/app/pages/PageEnigma/comps/Controls3D/Controls3D.tsx @@ -48,7 +48,7 @@ export const Controls3D = () => { if (!editorEngine) { return; } - editorEngine.change_mode("transform"); + editorEngine.change_mode("translate"); }; const handleRotateArrows = () => { if (!editorEngine) { diff --git a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx index 3a9195e78d..73196bc6c5 100644 --- a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx +++ b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx @@ -20,7 +20,9 @@ export const ControlsTopButtons = () => { editorEngine?.start_playback(); }; const handleButtonLoad = () => { - document.getElementById("load-upload")?.click(); + //document.getElementById("load-upload")?.click(); + if(editorEngine == null) {return;} + editorEngine._load_for_testing(); }; const handleButtonRender = () => { console.log("Saving GLB to server for reference"); diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index 18cdf2f5ae..12f3a74d17 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -164,7 +164,7 @@ class Editor { this.timeline.scene = this.activeScene; - this._test_demo() + //this._test_demo() } // uploading some objects for testing to get their media ids from my account. @@ -179,9 +179,16 @@ class Editor { // let result = await this.api_manager.getMediaFile( // "m_189p8hj0eyypbg74kkhcpehwpjhnkz", // ); + console.log("Saveing"); + const result = await this.api_manager.saveSceneState(this.activeScene.scene); + console.log("Saved!"); + console.log(result); + //this.api_manager.loadScene("m_n3k7nc0r5scr5zf92febh52h39g721") + } - //const result = this.api_manager.saveSceneState(this.activeScene.scene) - this.api_manager.loadScene("m_n3k7nc0r5scr5zf92febh52h39g721") + async _load_for_testing() { + const result = await this.api_manager.loadScene("m_6zgwx3aybyq086mwqhxw3me84rdgpt") + console.log(result); } async _serialize_timeline() { From ce1296ec2e185183c996a55805c95e2441d8c5c5 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Tue, 26 Mar 2024 21:01:41 -0400 Subject: [PATCH 03/23] Made changes to structure need miles to sync Removed unneeded things in timeline. --- app/pages/PageEnigma/js/api_manager.ts | 37 +++++++++++++------------- app/pages/PageEnigma/js/editor.js | 2 +- app/pages/PageEnigma/js/timeline.ts | 5 +--- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index 0d27a8fbf2..d4788ed8b2 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -53,17 +53,19 @@ class APIManager { let uuid = uuidv4(); // turn json into a blob - const file_test = {"scene_media_file_id":uuid} + const file_test = {"glb_media_file_id":uuid,"entities":[], "timeline":""} const json = JSON.stringify(file_test); const blob = new Blob([json], {type: 'application/json'}); const fileName = `${uuid}.json`; const formData = new FormData(); + formData.append("uuid_idempotency_token", uuid); formData.append("file", blob, fileName); formData.append("source", "file"); formData.append("type", "scene_json"); formData.append("source", "file"); + const response = await fetch(url, { method: "POST", credentials: "include", @@ -77,7 +79,7 @@ class APIManager { throw new Error("Failed to Send Data"); } - console.log(`RESPONSE ${response}`) + console.log(`saveSceneAndTimelineToJSONSpecResponse: ${response}`) return response.json(); // or handle the response as appropriate } @@ -107,7 +109,7 @@ class APIManager { reader.onerror = reject; reader.readAsText(blob); }); - //console.log(json_value) + console.log(json_value) return json_value } @@ -125,7 +127,6 @@ class APIManager { } async batchGetMedia(media_tokens: []): Promise { - return ""; } @@ -161,23 +162,23 @@ class APIManager { if (!response.ok) { throw new Error("Failed to Send Data"); } - - return response.json(); // or handle the response as appropriate + return response.json(); } - // To save to a file then upload the scene. - // /v1/media_files/write/engine_asset - // An upsert for files. - // If the file exists, it'll check ownership prior to overwriting. - // For GLB, GLTF, BVH, FBX, etc. game engine-type files. - // Form-multipart POST with parameters: - // uuid_idempotency_token - // file - // media_file_token (optional; the file to replace if present) - // media_file_subtype (optional; mixamo, mocap_net, scene_import, animation_only) - // media_file_class (audio, image, video, animation, character, prop, scene, unknown) + /** + To save the scene as a file get the id back to serialize + /v1/media_files/write/engine_asset + An upsert for files. + If the file exists, it'll check ownership prior to overwriting. + For GLB, GLTF, BVH, FBX, etc. game engine-type files. + Form-multipart POST with parameters: + uuid_idempotency_token + file + media_file_token (optional; the file to replace if present) + media_file_subtype (optional; mixamo, mocap_net, scene_import, animation_only) + media_file_class (audio, image, video, animation, character, prop, scene, unknown) + **/ async uploadGLB(file: File): Promise { - // /v1/media_files/write/engine_asset REPLACE with this api endpoint const url = `${this.baseUrl}/v1/media_files/write/engine_asset`; let uuid = uuidv4(); const formData = new FormData(); diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index 18cdf2f5ae..a12dedad73 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -179,8 +179,8 @@ class Editor { // let result = await this.api_manager.getMediaFile( // "m_189p8hj0eyypbg74kkhcpehwpjhnkz", // ); + const result = this.api_manager.saveSceneState(this.activeScene.scene) - //const result = this.api_manager.saveSceneState(this.activeScene.scene) this.api_manager.loadScene("m_n3k7nc0r5scr5zf92febh52h39g721") } diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 4803e30349..84fa27d49c 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -20,11 +20,8 @@ isEditable: boolean export class TimeLine { - timelineItems: ClipUI[] - runningClips: ClipUI[] - //timerID: NodeJS.Timeout | null timeLineLimit: number scrubberPosition: number isPlaying: boolean @@ -50,7 +47,7 @@ export class TimeLine { this.timelineItems = [] this.timeLineLimit = 60 * 10 // 10 seconds - this.runningClips = [] + this.isPlaying = false this.scrubberPosition = 0 // in frames into the tl this.timelineState = new TimelineCurrentReactState() From 5ee0719c051e4c8e86270609592657439adaa425 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Tue, 26 Mar 2024 21:09:24 -0400 Subject: [PATCH 04/23] fixed mapping issue. --- app/pages/PageEnigma/js/api_manager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index d4788ed8b2..59f69d92d5 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -53,7 +53,7 @@ class APIManager { let uuid = uuidv4(); // turn json into a blob - const file_test = {"glb_media_file_id":uuid,"entities":[], "timeline":""} + const file_test = {"glb_media_file_id":media_file_token,"entities":[], "timeline":""} const json = JSON.stringify(file_test); const blob = new Blob([json], {type: 'application/json'}); const fileName = `${uuid}.json`; From d67c37e5c1ea5a23ec62e9141fb11d5c42fbc6c2 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Tue, 26 Mar 2024 21:09:59 -0400 Subject: [PATCH 05/23] editor media id --- app/pages/PageEnigma/js/editor.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index 12f3a74d17..8308d00e53 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -187,8 +187,10 @@ class Editor { } async _load_for_testing() { - const result = await this.api_manager.loadScene("m_6zgwx3aybyq086mwqhxw3me84rdgpt") + const result = await this.api_manager.loadScene("m_3nwr2kejyrrd4mmvbpmn032gea60p0") console.log(result); + let bucket_path = await this.api_manager.getMediaFile("m_3nwr2kejyrrd4mmvbpmn032gea60p0"); + console.log(bucket_path); } async _serialize_timeline() { From 4359c719393aa5cd49ebc37249f9b8dd2448ea2e Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Tue, 26 Mar 2024 21:41:08 -0400 Subject: [PATCH 06/23] added loading. --- app/pages/PageEnigma/js/api_manager.ts | 2 -- app/pages/PageEnigma/js/editor.js | 30 ++++++++++++++++++++------ app/pages/PageEnigma/js/scene.ts | 2 ++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index 59f69d92d5..ec240ae0ff 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -99,10 +99,8 @@ class APIManager { if (!fileResponse.ok) { throw new Error("Failed to download file"); } - // Convert the response to a Blob let blob = await fileResponse.blob(); - const json_value:string = await new Promise((resolve, reject) => { let reader = new FileReader(); reader.onloadend = () => resolve(JSON.parse(reader.result as string)); diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index 8308d00e53..c8db8478d9 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -12,6 +12,7 @@ import { SAOPass } from "three/addons/postprocessing/SAOPass.js"; import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js"; import { BokehPass } from "three/addons/postprocessing/BokehPass.js"; import { createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg"; +import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; import AudioEngine from "./audio_engine.ts"; import TransformEngine from "./transform_engine.ts"; @@ -146,8 +147,9 @@ class Editor { window.addEventListener("resize", this.onWindowResize.bind(this)); // Current scene for saving and loading. this.activeScene = new Scene(); - this._configure_post_pro(); this.activeScene.initialize(); + + this._configure_post_pro(); // Controls and movement. this.orbit = new OrbitControls(this.camera, this.renderer.domElement); this.control = new TransformControls(this.camera, this.renderer.domElement); @@ -179,7 +181,7 @@ class Editor { // let result = await this.api_manager.getMediaFile( // "m_189p8hj0eyypbg74kkhcpehwpjhnkz", // ); - console.log("Saveing"); + console.log("Saving..."); const result = await this.api_manager.saveSceneState(this.activeScene.scene); console.log("Saved!"); console.log(result); @@ -187,10 +189,26 @@ class Editor { } async _load_for_testing() { - const result = await this.api_manager.loadScene("m_3nwr2kejyrrd4mmvbpmn032gea60p0") - console.log(result); - let bucket_path = await this.api_manager.getMediaFile("m_3nwr2kejyrrd4mmvbpmn032gea60p0"); - console.log(bucket_path); + const result = await this.api_manager.loadScene("m_gc7n4z3p82keydyaxtb0s5qt4yys54") + let bucket_path = await this.api_manager.getMediaFile(result["glb_media_file_id"]); + let glbLoader = new GLTFLoader(); + glbLoader.load(bucket_path, (glb) => { + this.activeScene.scene.children = glb.scene.children; + this.activeScene.scene.children.forEach(child => { + child.parent = this.activeScene.scene; + if(child.type == "DirectionalLight") { + let pos = child.position; + let rot = child.rotation; + + let light = this.activeScene._create_base_lighting(); + light.position.set(pos.x, pos.y, pos.z); + light.rotation.set(rot.x, rot.y, rot.z); + + this.activeScene.scene.remove(child); + } + }); + }); + } async _serialize_timeline() { diff --git a/app/pages/PageEnigma/js/scene.ts b/app/pages/PageEnigma/js/scene.ts index 125a2663d4..1f48136e42 100644 --- a/app/pages/PageEnigma/js/scene.ts +++ b/app/pages/PageEnigma/js/scene.ts @@ -13,6 +13,7 @@ class Scene { } initialize() { + this.scene = new THREE.Scene(); this._createGrid(); this._create_base_lighting(); this._create_skybox(); @@ -128,6 +129,7 @@ class Scene { this.scene.add(directional_light); this.scene.add(directional_light.target); + return directional_light; } _createGrid() { From e4ac2e3d495ac4b71d783cc9e0083801721436a6 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Tue, 26 Mar 2024 21:44:14 -0400 Subject: [PATCH 07/23] scene with haishi. --- app/pages/PageEnigma/js/editor.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index c8db8478d9..bd0bd5b5d1 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -166,7 +166,7 @@ class Editor { this.timeline.scene = this.activeScene; - //this._test_demo() + this._test_demo() } // uploading some objects for testing to get their media ids from my account. @@ -184,12 +184,11 @@ class Editor { console.log("Saving..."); const result = await this.api_manager.saveSceneState(this.activeScene.scene); console.log("Saved!"); - console.log(result); //this.api_manager.loadScene("m_n3k7nc0r5scr5zf92febh52h39g721") } async _load_for_testing() { - const result = await this.api_manager.loadScene("m_gc7n4z3p82keydyaxtb0s5qt4yys54") + const result = await this.api_manager.loadScene("m_7f3k4qysab4aja96vpc7b31tmq8y3k"); let bucket_path = await this.api_manager.getMediaFile(result["glb_media_file_id"]); let glbLoader = new GLTFLoader(); glbLoader.load(bucket_path, (glb) => { From 3c2514ea674a8f00b407d8463367aa2a2c0fef7a Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Tue, 26 Mar 2024 22:12:10 -0400 Subject: [PATCH 08/23] added test scene media id --- app/pages/PageEnigma/js/editor.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index bd0bd5b5d1..70c6153866 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -115,6 +115,8 @@ class Editor { this.test_box_uuid = null; this.current_frame = 0; this.test_playback = false; + + this.test_scene_load_media_id = ""; // Test media id for saving and loading. } // Initializes the main scene and ThreeJS essentials. @@ -184,11 +186,12 @@ class Editor { console.log("Saving..."); const result = await this.api_manager.saveSceneState(this.activeScene.scene); console.log("Saved!"); - //this.api_manager.loadScene("m_n3k7nc0r5scr5zf92febh52h39g721") + console.log("Media ID is:", result); + this.test_scene_load_media_id = result["media_file_token"]; } async _load_for_testing() { - const result = await this.api_manager.loadScene("m_7f3k4qysab4aja96vpc7b31tmq8y3k"); + const result = await this.api_manager.loadScene(this.test_scene_load_media_id); let bucket_path = await this.api_manager.getMediaFile(result["glb_media_file_id"]); let glbLoader = new GLTFLoader(); glbLoader.load(bucket_path, (glb) => { @@ -198,11 +201,9 @@ class Editor { if(child.type == "DirectionalLight") { let pos = child.position; let rot = child.rotation; - let light = this.activeScene._create_base_lighting(); light.position.set(pos.x, pos.y, pos.z); light.rotation.set(rot.x, rot.y, rot.z); - this.activeScene.scene.remove(child); } }); From 366f9b563e3f01f9bf645e6325a1dc8e0de76b59 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Tue, 26 Mar 2024 22:21:45 -0400 Subject: [PATCH 09/23] Just notes --- app/pages/PageEnigma/js/editor.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index 70c6153866..c918f68052 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -268,12 +268,20 @@ class Editor { "./resources/models/fox/fox.glb", ); + // Load timeline creates the the clips from the datastructure and loads them in here. // load object into the engine for lip syncing this.lipsync_engine.load_object( object.uuid, "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", ); - + this.animation_engine.load_object( + object.uuid, + "/resources/models/fox/fox_idle.glb", + "clip3", + ); + // then it creates clip ui to load the playable clips + // then refreshes the timeline. + // create the clip with the same id for a reference to the media this.timeline.addPlayableClip( new ClipUI( @@ -304,11 +312,7 @@ class Editor { 400, ), ); - this.animation_engine.load_object( - object.uuid, - "/resources/models/fox/fox_idle.glb", - "clip3", - ); + } // Configure post processing. From c7647f4e6dca3133ced0ccc7f7c00461aa35dd72 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 27 Mar 2024 11:06:55 -0400 Subject: [PATCH 10/23] Working on saving and loading. Modal doesn't do saving and or show the loading indicator. --- .../ControlsTopButtons/ControlsTopButtons.tsx | 1 + app/pages/PageEnigma/js/api_manager.ts | 33 ++++++++++++------- app/pages/PageEnigma/js/editor.js | 14 +++++++- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx index 73196bc6c5..1aaf710623 100644 --- a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx +++ b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx @@ -10,6 +10,7 @@ export const ControlsTopButtons = () => { const handleButtonSave = () => { console.log(`SceneName is ${sceneName}`); // editorEngine?.save(); + }; const handleButtonCameraView = () => { diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index ec240ae0ff..8167bcd0e4 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -22,20 +22,21 @@ class APIManager { async saveSceneState( scene: THREE.Scene, timeline: TimelineCurrentReactState, - media_file_token: string | null = null, + scene_name: string, + scene_media_file_token: string | null = null, ): Promise { - // create new scene. + // create new scene + let file = await this.gltfExport(scene); let response = await this.uploadGLB(file); let success = response["success"]; if (success) { - let media_file_token = response["media_file_token"]; - - console.log(`media file token: ${media_file_token}`) - - const result = await this._saveSceneAndTimelineToJSONSpec(media_file_token, timeline); - return result - } + let media_file_token = response["media_file_token"]; + console.log(`media file token: ${media_file_token}`) + const result = await this._saveSceneAndTimelineToJSONSpec(media_file_token,scene_name, timeline); + return result + } + throw Error("Saving the Scene Resulted in an Error Success False"); } @@ -47,16 +48,18 @@ class APIManager { // media_file_token (optional; the file to replace if present) async _saveSceneAndTimelineToJSONSpec( media_file_token: string, + scene_file_name: string, timeline: TimelineCurrentReactState, ): Promise { const url = `${this.baseUrl}/v1/media_files/write/scene_file`; let uuid = uuidv4(); // turn json into a blob - const file_test = {"glb_media_file_id":media_file_token,"entities":[], "timeline":""} + const file_test = {"glb_media_file_id":media_file_token,"name":scene_file_name,"entities":[], "timeline":""} + const json = JSON.stringify(file_test); const blob = new Blob([json], {type: 'application/json'}); - const fileName = `${uuid}.json`; + const fileName = `${uuid}-${scene_file_name}.json`; const formData = new FormData(); @@ -176,11 +179,17 @@ class APIManager { media_file_subtype (optional; mixamo, mocap_net, scene_import, animation_only) media_file_class (audio, image, video, animation, character, prop, scene, unknown) **/ - async uploadGLB(file: File): Promise { + async uploadGLB(file: File, scene_media_file_token:string | null): Promise { const url = `${this.baseUrl}/v1/media_files/write/engine_asset`; let uuid = uuidv4(); const formData = new FormData(); formData.append("uuid_idempotency_token", uuid); + + // update existing scene otherwise create new scene and use it's media_file_id + if (scene_media_file_token != null) { + formData.append("media_file_token", scene_media_file_token); + } + formData.append("file", file); formData.append("source", "file"); formData.append("media_file_subtype", "scene_import"); diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index c918f68052..8cc07f9c87 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -169,6 +169,18 @@ class Editor { this.timeline.scene = this.activeScene; this._test_demo() + + + // saving state of the scene + this.scene_file_token = null + } + + + async saveScene(name) { + // check if I already have the scene token. + console.log("Saving..."); + const result = await this.api_manager.saveSceneState(this.activeScene.scene,this.scene_file_token); + console.log("Saved!"); } // uploading some objects for testing to get their media ids from my account. @@ -281,7 +293,7 @@ class Editor { ); // then it creates clip ui to load the playable clips // then refreshes the timeline. - + // create the clip with the same id for a reference to the media this.timeline.addPlayableClip( new ClipUI( From f3ca24183fffe799b6274a9dc780dc6b607fa0eb Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 27 Mar 2024 16:55:46 -0400 Subject: [PATCH 11/23] 1.Made changes to OOP for the timeline to be explicit to what the functions are intended for. 2.Internal state vs external events added public and private modifiers. 3.Renamed the clips to from track clip to clip to reduce the confusion and solidify the communication. 4.Modified Scene Saving and Loading and refactored to make it more consistent and understandable. --- app/pages/PageEnigma/PageEnigmaComponent.tsx | 2 +- .../ControlsTopButtons/ControlsTopButtons.tsx | 7 +- ...mation_track_clip.ts => animation_clip.ts} | 14 +- .../{audio_track_clip.ts => audio_clip.ts} | 9 +- ...{lipsync_track_clip.ts => lipsync_clip.ts} | 11 +- ...nsform_track_clip.ts => transform_clip.ts} | 13 +- .../datastructures/scene/scene_object.ts | 121 ----- app/pages/PageEnigma/js/animation_engine.ts | 6 +- app/pages/PageEnigma/js/api_manager.ts | 499 +++++++++++++----- app/pages/PageEnigma/js/audio_engine.ts | 6 +- app/pages/PageEnigma/js/editor.js | 42 +- app/pages/PageEnigma/js/lip_sync_engine.ts | 10 +- app/pages/PageEnigma/js/timeline.ts | 65 ++- app/pages/PageEnigma/js/transform_engine.ts | 8 +- 14 files changed, 467 insertions(+), 346 deletions(-) rename app/pages/PageEnigma/datastructures/clips/{animation_track_clip.ts => animation_clip.ts} (88%) rename app/pages/PageEnigma/datastructures/clips/{audio_track_clip.ts => audio_clip.ts} (91%) rename app/pages/PageEnigma/datastructures/clips/{lipsync_track_clip.ts => lipsync_clip.ts} (94%) rename app/pages/PageEnigma/datastructures/clips/{transform_track_clip.ts => transform_clip.ts} (89%) delete mode 100644 app/pages/PageEnigma/datastructures/scene/scene_object.ts diff --git a/app/pages/PageEnigma/PageEnigmaComponent.tsx b/app/pages/PageEnigma/PageEnigmaComponent.tsx index 0f8db07a61..df6bb91257 100644 --- a/app/pages/PageEnigma/PageEnigmaComponent.tsx +++ b/app/pages/PageEnigma/PageEnigmaComponent.tsx @@ -25,7 +25,7 @@ export const PageEnigmaComponent = () => { useEffect(()=>{ setTimeout(()=>dispatchAppUiState({ type: ACTION_TYPES.HIDE_EDITOR_LOADER - }), 3000); + }), 500); },[]); return (
diff --git a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx index bbf4dcd81e..1c5e7fe47b 100644 --- a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx +++ b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx @@ -9,8 +9,7 @@ export const ControlsTopButtons = () => { const handleButtonSave = () => { console.log(`SceneName is ${sceneName}`); - // editorEngine?.save(); - + editorEngine?.saveScene(); }; const handleButtonCameraView = () => { @@ -23,11 +22,11 @@ export const ControlsTopButtons = () => { const handleButtonLoad = () => { //document.getElementById("load-upload")?.click(); if(editorEngine == null) {return;} - editorEngine._load_for_testing(); + editorEngine.loadScene(); }; const handleButtonRender = () => { console.log("Saving GLB to server for reference"); - editorEngine?._upload_for_testing(); + editorEngine?.saveScene(); //editorEngine?.togglePlayback(); }; const handleButtonPlay = () => {}; diff --git a/app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/animation_clip.ts similarity index 88% rename from app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts rename to app/pages/PageEnigma/datastructures/clips/animation_clip.ts index ea38c59885..50110653d4 100644 --- a/app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/animation_clip.ts @@ -1,19 +1,7 @@ -import { faL } from '@fortawesome/pro-solid-svg-icons'; import * as THREE from 'three'; import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'; -export interface AnimationTrackClip { - version: number - media_id: string - object_uuid: string - type: "animation" - location: "glb" | "remote" - speed: number - length: number - clip_name: string -} - -export class AnimationTrackClip implements AnimationTrackClip { +export class AnimationClip { version: number; media_id: string; // comes from the server object_uuid: string; diff --git a/app/pages/PageEnigma/datastructures/clips/audio_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/audio_clip.ts similarity index 91% rename from app/pages/PageEnigma/datastructures/clips/audio_track_clip.ts rename to app/pages/PageEnigma/datastructures/clips/audio_clip.ts index 805741b550..cc5c19079c 100644 --- a/app/pages/PageEnigma/datastructures/clips/audio_track_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/audio_clip.ts @@ -14,14 +14,7 @@ class AudioData implements AudioData{ } } -export interface AudioTrackClip { - version: number; - media_id: string; - type: "audio"; - volume: number; -} - -export class AudioTrackClip implements AudioTrackClip { +export class AudioClip { version: number; media_id: string; type: "audio" = "audio"; diff --git a/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts similarity index 94% rename from app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts rename to app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts index 412b8b0bc3..a06808e72c 100644 --- a/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts @@ -16,13 +16,6 @@ class AudioData implements AudioData { } } -export interface LipSyncTrackClip { - version: number; - media_id: string; - type: "lipsync"; - volume: number; -} - class BlendShapeHelper { ah: number; ee: number; @@ -34,7 +27,7 @@ class BlendShapeHelper { } } -export class LipSyncTrackClip implements LipSyncTrackClip { +export class LipSyncClip { version: number; media_id: string; type: "lipsync" = "lipsync"; @@ -98,7 +91,7 @@ export class LipSyncTrackClip implements LipSyncTrackClip { } - async play(object: THREE.Object3D) { +async play(object: THREE.Object3D) { if (this.audio_data?.audioBuffer == null) { await this.download_audio(); } if (this.lipsync.face == null) { this.lipsync = new LipSync(await this._detect_face(object)); diff --git a/app/pages/PageEnigma/datastructures/clips/transform_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/transform_clip.ts similarity index 89% rename from app/pages/PageEnigma/datastructures/clips/transform_track_clip.ts rename to app/pages/PageEnigma/datastructures/clips/transform_clip.ts index 729f36621f..f167843319 100644 --- a/app/pages/PageEnigma/datastructures/clips/transform_track_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/transform_clip.ts @@ -1,17 +1,6 @@ -import { faL } from '@fortawesome/pro-solid-svg-icons'; import * as THREE from 'three'; -export interface TransformTrackClip { - version: number; - media_id: string; - type: "transform"; - positions: THREE.Vector3[]; - rotations: THREE.Vector3[]; - scales: THREE.Vector3[]; - length: number; -} - -export class TransformTrackClip implements TransformTrackClip { +export class TransformClip { version: number; media_id: string; object_uuid: string; diff --git a/app/pages/PageEnigma/datastructures/scene/scene_object.ts b/app/pages/PageEnigma/datastructures/scene/scene_object.ts deleted file mode 100644 index 1242295f67..0000000000 --- a/app/pages/PageEnigma/datastructures/scene/scene_object.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { AudioTrackClip } from "../clips/audio_track_clip"; -import { AnimationTrackClip } from "../clips/animation_track_clip"; -import { TransformTrackClip } from "../clips/transform_track_clip"; -import { ClipOffset } from "../clips/clip_offset"; - -export interface Scene { - version: number; - glb_media_id: number; // Assuming a numerical ID, adjust as needed - skybox_type: "cube" | "six_sided_cube" | "sphere"; - skybox_media_ids: { [key: string]: number }; - timeline: Timeline; -} - -export interface Timeline { - version: number; - last_scrubber_position: number; - audio_track_clips: AudioTrackClip[]; // Assuming defined elsewhere - transform_track_clips: TransformTrackClip[]; // Placeholder, define as needed - animation_track_clips: AnimationTrackClip[]; // Placeholder, define as needed - entities: Entity[]; - globalAudio: GlobalAudio; - camera: Camera; -} - -export interface Entity { - object_uuid: number; - clip_offsets: ClipOffset[]; -} - -export interface GlobalAudio { - clip_offsets: ClipOffset[]; -} - -export interface Camera { - clip_offsets: ClipOffset[]; -} - -export class Scene implements Scene { - version: number; - glb_media_id: number; - skybox_type: "cube" | "six_sided_cube" | "sphere"; - skybox_media_ids: { [key: string]: number }; - timeline: Timeline; - - constructor( - version: number, - glb_media_id: number, - skybox_type: "cube" | "six_sided_cube" | "sphere", - skybox_image_ids: { [key: string]: number }, - timeline: Timeline, - ) - { - this.version = version; - this.glb_media_id = glb_media_id; - this.skybox_type = skybox_type; - this.skybox_media_ids = skybox_image_ids; - this.timeline = timeline; - } - - toJSON(): string { - return JSON.stringify(this); - } -} - -export class Timeline implements Timeline { - version: number; - last_scrubber_position: number; - audio_track_clips: AudioTrackClip[]; - transform_track_clips: TransformTrackClip[]; - animation_track_clips: AnimationTrackClip[]; - entities: Entity[]; - globalAudio: GlobalAudio; - camera: Camera; - - constructor( - version: number, - last_scrubber_position: number, - audio_track_clips: AudioTrackClip[], - transform_track_clips: TransformTrackClip[], - animation_track_clips: AnimationTrackClip[], - entities: Entity[], - globalAudio: GlobalAudio, - camera: Camera, - ) - { - this.version = version; - this.last_scrubber_position = last_scrubber_position; - this.audio_track_clips = audio_track_clips; - this.transform_track_clips = transform_track_clips; - this.animation_track_clips = animation_track_clips; - this.entities = entities; - this.globalAudio = globalAudio; - this.camera = camera; - } - - toJSON(): string { - return JSON.stringify(this); - } - // For testing purposes - static generateMockJson(): string { - const mock = { - version: 1, - last_scrubber_position: 5000, - audio_track_clips: [ - { version: 1, uuid: 0, media_id: 1, type: "audio", volume: 100 } - ], - transform_track_clips: [ - { version: 1, uuid: 1, media_id: 2, type: "transform", position: [{ x: 0, y: 0, z: 0 }], rotation: [{ x: 0, y: 0, z: 0 }], scale: [{ x: 1, y: 1, z: 1 }] } - ], - animation_track_clips: [ - { version: 1, uuid: 2, media_id: 3, type: "animation", speed: 1.0, length: 100, starting_offset: 0, clip_name: "some clip name" } - ], - entities: [ - { object_uuid: 0, clip_offsets: [] } // Assuming more details would be filled as necessary - ], - globalAudio: { clip_offsets: [] }, // Assuming more details would be filled as necessary - camera: { clip_offsets: [] } // Assuming more details would be filled as necessary - }; - return JSON.stringify(mock); - } -} diff --git a/app/pages/PageEnigma/js/animation_engine.ts b/app/pages/PageEnigma/js/animation_engine.ts index fb155ea7d2..a2fde07189 100644 --- a/app/pages/PageEnigma/js/animation_engine.ts +++ b/app/pages/PageEnigma/js/animation_engine.ts @@ -1,8 +1,8 @@ -import { AnimationTrackClip } from "../datastructures/clips/animation_track_clip"; +import { AnimationClip } from "../datastructures/clips/animation_clip"; import * as THREE from 'three'; export class AnimationEngine { - clips: { [key: string]: AnimationTrackClip } = {}; + clips: { [key: string]: AnimationClip } = {}; version: number; length: number; @@ -13,7 +13,7 @@ export class AnimationEngine { } load_object(object_uuid: string, media_id:string, clip_name: string) { - this.clips[object_uuid] = new AnimationTrackClip( + this.clips[object_uuid] = new AnimationClip( this.version, media_id, "glb", diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index 8167bcd0e4..a081738d41 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -3,9 +3,37 @@ import { Scene } from "../datastructures/scene/scene_object"; import * as THREE from "three"; import { GLTFExporter } from "three/addons/exporters/GLTFExporter.js"; import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; -import { TimelineCurrentReactState } from "./timeline"; +import { TimelineDataState } from "./timeline"; + +/** + * Storyteller Studio API Manager + * The source of truth of all these media items is the database in the cloud + */ + +/** + * This is designed to surface user customer facing messages as errors. + */ +type Tokens = { [key: string]: string }; +class APIManagerResponseSuccess { + public user_message:String + public data:Tokens | null + constructor(user_message:String = "", data:Tokens | null = null) { + this.data = data + this.user_message = user_message + } +} + +/** + * This is designed to surface user customer facing messages as errors. + * Errors shouldn't be 404 or something confusing should be + */ +class APIManagerResponseError extends Error { + constructor(message?: string) { + super(message); + this.name = "APIManagerResponseError"; + } +} -// Basically the data will exist on the database class APIManager { baseUrl: String; @@ -14,60 +42,116 @@ class APIManager { //this.baseUrl = "http://localhost:12345" } - // Function to save or update the scene - // it will always return a boolean success and the media_file_token - // they take an optional media_file_token as a parameter for the file to overwrite, which must belong to the user. - //If it's an anonymous user, we check the anonymous visitor token/cookie + /** + * @param scene The 3JS Scene we want to save. + * @param scene_name The Scene name we want to display + * @param scene_media_file_token If null we will we will create a new save or copy the scene, if provided we will overwrite the scene. + * @returns APIManagerResponseMessage + */ + public async saveSceneState(scene: THREE.Scene, + scene_name:string, + scene_media_file_token:string | null = null, + timeline_state:TimelineDataState):Promise { + + const file = await this.gltfExport(scene) + console.log(`FILE: ${file}`) + // will overwrite the scene on db if token exists + const upload_response = await this.uploadGLB(file,scene_media_file_token) + await this.saveSceneAndTimeLineState(scene_media_file_token,scene_name,timeline_state) + return new APIManagerResponseSuccess("Scene Saved") + } + + public async loadSceneState(scene_media_file_token: string): Promise { + const api_base_url = "https://api.fakeyou.com" + const url = `${api_base_url}/v1/media_files/file/${scene_media_file_token}` + const response = await fetch(url) + const json = await JSON.parse(await response.text()) + const bucket_path = json["media_file"]["public_bucket_path"] + const media_base_url = "https://storage.googleapis.com/vocodes-public" + const media_url = `${media_base_url}${bucket_path}` // gets you a bucket path - async saveSceneState( - scene: THREE.Scene, - timeline: TimelineCurrentReactState, - scene_name: string, - scene_media_file_token: string | null = null, - ): Promise { - // create new scene - - let file = await this.gltfExport(scene); - let response = await this.uploadGLB(file); - let success = response["success"]; - if (success) { - let media_file_token = response["media_file_token"]; - console.log(`media file token: ${media_file_token}`) - const result = await this._saveSceneAndTimelineToJSONSpec(media_file_token,scene_name, timeline); - return result - } - - - throw Error("Saving the Scene Resulted in an Error Success False"); + const file_response = await fetch(media_url) + + if (!file_response.ok) { + throw new APIManagerResponseError("Failed to download file") + } + // Convert the response to a Blob + let blob = await file_response.blob() + const json_result:string = await new Promise((resolve, reject) => { + let reader = new FileReader() + reader.onloadend = () => resolve(JSON.parse(reader.result as string)) + reader.onerror = reject + reader.readAsText(blob) + }) + + console.log(`JSON Result: ${json_result}`) + + const scene_glb_media_file_id:string = json_result["glb_media_file_id"] + const media_bucket_path = await this.getMediaFile(scene_glb_media_file_id) + let glbLoader = new GLTFLoader() + + // promisify this + const loadGlb = (bucket_path: string): Promise => { + return new Promise((resolve, reject) => { + glbLoader.load(bucket_path, (glb) => { + if (glb) { + const scene:THREE.Scene = glb.scene + resolve(scene) + } else { + reject(new APIManagerResponseError('Failed to Load GLB Scene')) + } + }) + }) + } + loadGlb(media_bucket_path) + } + + /** + * This recieves the bucket path of a media file + * @param media_file_token + * @returns + */ + private async getMediaFile(media_file_token: string): Promise { + let api_base_url = "https://api.fakeyou.com"; + let url = `${api_base_url}/v1/media_files/file/${media_file_token}` + let response = await fetch(url); + let json = await JSON.parse(await response.text()); + let bucketPath = json["media_file"]["public_bucket_path"]; + let media_base_url = "https://storage.googleapis.com/vocodes-public" + let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path + return media_url + } + + /** + This will save the scene to keep ids positions. + It will also give the file a name which will be a uuidv4() + @param scene The 3JS Scene we want to make a file to be uploaded as multipart form. + */ + private async gltfExport(scene: THREE.Scene):Promise { + const gltfExporter = new GLTFExporter(); + const uuid = uuidv4(); + const result = await gltfExporter.parseAsync(scene); + const file = new File([JSON.stringify(result)], `${uuid}.glb`, { + type: "application/json", + }); + return file; } - // For JSON files. - // Form-multipart POST with parameters: - // uuid_idempotency_token - // file - // media_file_token (optional; the file to replace if present) - async _saveSceneAndTimelineToJSONSpec( - media_file_token: string, - scene_file_name: string, - timeline: TimelineCurrentReactState, - ): Promise { - const url = `${this.baseUrl}/v1/media_files/write/scene_file`; + private async uploadGLB(file: File, scene_media_file_token:string | null): Promise { + const url = `${this.baseUrl}/v1/media_files/write/engine_asset`; let uuid = uuidv4(); + const form_data = new FormData(); + form_data.append("uuid_idempotency_token", uuid); - // turn json into a blob - const file_test = {"glb_media_file_id":media_file_token,"name":scene_file_name,"entities":[], "timeline":""} - - const json = JSON.stringify(file_test); - const blob = new Blob([json], {type: 'application/json'}); - const fileName = `${uuid}-${scene_file_name}.json`; - - const formData = new FormData(); - - formData.append("uuid_idempotency_token", uuid); - formData.append("file", blob, fileName); - formData.append("source", "file"); - formData.append("type", "scene_json"); - formData.append("source", "file"); + // update existing scene otherwise create new scene and use it's media_file_id + if (scene_media_file_token != null) { + form_data.append("media_file_token", scene_media_file_token); + } + + form_data.append("file", file); + form_data.append("source", "file"); + form_data.append("media_file_subtype", "scene_import"); + form_data.append("media_file_class", "scene"); const response = await fetch(url, { method: "POST", @@ -75,82 +159,71 @@ class APIManager { headers: { Accept: "application/json", }, - body: formData, + body: form_data, }); if (!response.ok) { throw new Error("Failed to Send Data"); } - - console.log(`saveSceneAndTimelineToJSONSpecResponse: ${response}`) - - return response.json(); // or handle the response as appropriate + const json_data = await response.json() + // const json_string = JSON.stringify(json_data) + // console.log(`uploadGLB: ${json_string}`) + return json_data; // or handle the response as appropriate } - // media_file_token that describes the scene - async loadScene(media_file_token: string): Promise { - let api_base_url = "https://api.fakeyou.com"; - let url = `${api_base_url}/v1/media_files/file/${media_file_token}` - let response = await fetch(url); - let json = await JSON.parse(await response.text()); - let bucketPath = json["media_file"]["public_bucket_path"]; - let media_base_url = "https://storage.googleapis.com/vocodes-public" - let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path - - let fileResponse = await fetch(media_url); + private async saveSceneAndTimeLineState( + scene_media_file_token: string | null, + scene_file_name: string, + timeline_state: TimelineDataState | null, // only for now. + ): Promise { + const url = `${this.baseUrl}/v1/media_files/write/scene_file` + let uuid = uuidv4() - if (!fileResponse.ok) { - throw new Error("Failed to download file"); + // turn json into a blob + const scene_schema = {"glb_media_file_id":scene_media_file_token,"scene_name":scene_file_name,"timeline": { "objects":[] } } + const json = JSON.stringify(scene_schema) + const blob = new Blob([json], {type: 'application/json'}) + const file_name = `${uuid}.json` + + const form_data = new FormData() + + form_data.append("uuid_idempotency_token", uuid) + form_data.append("file", blob, file_name) + form_data.append("source", "file") + form_data.append("type", "scene_json") + form_data.append("source", "file") + + const response = await fetch(url, { + method: "POST", + credentials: "include", + headers: { + Accept: "application/json", + }, + body: form_data, + }) + + if (!response.ok) { + throw new Error("Failed to Send Data") } - // Convert the response to a Blob - let blob = await fileResponse.blob(); - const json_value:string = await new Promise((resolve, reject) => { - let reader = new FileReader(); - reader.onloadend = () => resolve(JSON.parse(reader.result as string)); - reader.onerror = reject; - reader.readAsText(blob); - }); - console.log(json_value) - return json_value - } + const json_data = await response.json() - // Provides bucket path to the actual file to download from the media file - async getMediaFile(media_file_token: string): Promise { - let api_base_url = "https://api.fakeyou.com"; - let url = `${api_base_url}/v1/media_files/file/${media_file_token}` - let response = await fetch(url); - let json = await JSON.parse(await response.text()); - let bucketPath = json["media_file"]["public_bucket_path"]; - let media_base_url = "https://storage.googleapis.com/vocodes-public" - let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path - // TODO get it as binary. - return media_url - } - - async batchGetMedia(media_tokens: []): Promise { - return ""; + // example response {"success":true,"media_file_token":"m_r1ztnn501g2rn0vv2np08nd6zy2fvt"} + console.log(`saveSceneAndTimeLineState: ${JSON.stringify(json_data)}`) + // should return the same token if it is same file else new token. + return json_data // or handle the response as appropriate } - async gltfExport(scene: THREE.Scene) { - let gltfExporter = new GLTFExporter(); - let uuid = uuidv4(); - // save the scene to keep ids positions etc - let result = await gltfExporter.parseAsync(scene); - const file = new File([JSON.stringify(result)], `${uuid}.glb`, { - type: "application/json", - }); - return file; - } - // two core functions to upload things to the server and get id's - async uploadMedia(blob: any, fileName: string): Promise { + private async uploadMedia(blob:any,fileName:string):Promise { const url = `${this.baseUrl}/v1/media_uploads/upload`; let uuid = uuidv4(); + const formData = new FormData(); formData.append("uuid_idempotency_token", uuid); formData.append("file", blob, fileName); formData.append("source", "file"); formData.append("type", "video"); formData.append("source", "file"); + const response = await fetch(url, { method: "POST", credentials: "include", @@ -161,11 +234,167 @@ class APIManager { }); if (!response.ok) { - throw new Error("Failed to Send Data"); + throw new APIManagerResponseError("Upload Media Failed to send file") + } + else { + const json_data = await response.json() + return new APIManagerResponseSuccess("Uploaded Media Success") } - return response.json(); } + public async batchGetMedia(media_tokens: []): Promise { + return ""; + } + + // Function to save or update the scene + // it will always return a boolean success and the media_file_token + // they take an optional media_file_token as a parameter for the file to overwrite, which must belong to the user. + // If it's an anonymous user, we check the anonymous visitor token/cookie + + // async saveSceneState2( + // scene: THREE.Scene, + // timeline: TimelineDataState, + // scene_name: string, + // scene_media_file_token: string | null = null, + // ): Promise { + // // create new scene + + // let file = await this.gltfExport(scene); + // let response = await this.uploadGLB(file); + // let success = response["success"]; + // if (success) { + // let media_file_token = response["media_file_token"]; + // console.log(`media file token: ${media_file_token}`) + // const result = await this._saveSceneAndTimelineToJSONSpec(media_file_token,scene_name, timeline); + // return result + // } + + + // throw Error("Saving the Scene Resulted in an Error Success False"); + // } + + // // For JSON files. + // // Form-multipart POST with parameters: + // // uuid_idempotency_token + // // file + // // media_file_token (optional; the file to replace if present) + // async _saveSceneAndTimelineToJSONSpec( + // media_file_token: string, + // scene_file_name: string, + // timeline: TimelineDataState, + // ): Promise { + // const url = `${this.baseUrl}/v1/media_files/write/scene_file`; + // let uuid = uuidv4(); + + // // turn json into a blob + // const file_test = {"glb_media_file_id":media_file_token,"name":scene_file_name,"entities":[], "timeline":""} + + // const json = JSON.stringify(file_test); + // const blob = new Blob([json], {type: 'application/json'}); + // const fileName = `${uuid}-${scene_file_name}.json`; + + // const formData = new FormData(); + + // formData.append("uuid_idempotency_token", uuid); + // formData.append("file", blob, fileName); + // formData.append("source", "file"); + // formData.append("type", "scene_json"); + // formData.append("source", "file"); + + // const response = await fetch(url, { + // method: "POST", + // credentials: "include", + // headers: { + // Accept: "application/json", + // }, + // body: formData, + // }); + + // if (!response.ok) { + // throw new Error("Failed to Send Data"); + // } + + // console.log(`saveSceneAndTimelineToJSONSpecResponse: ${response}`) + + // return response.json(); // or handle the response as appropriate + // } + + // // media_file_token that describes the scene + // async loadScene(media_file_token: string): Promise { + // let api_base_url = "https://api.fakeyou.com"; + // let url = `${api_base_url}/v1/media_files/file/${media_file_token}` + // let response = await fetch(url); + // let json = await JSON.parse(await response.text()); + // let bucketPath = json["media_file"]["public_bucket_path"]; + // let media_base_url = "https://storage.googleapis.com/vocodes-public" + // let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path + + // let fileResponse = await fetch(media_url); + + // if (!fileResponse.ok) { + // throw new Error("Failed to download file"); + // } + // // Convert the response to a Blob + // let blob = await fileResponse.blob(); + // const json_value:string = await new Promise((resolve, reject) => { + // let reader = new FileReader(); + // reader.onloadend = () => resolve(JSON.parse(reader.result as string)); + // reader.onerror = reject; + // reader.readAsText(blob); + // }); + // console.log(json_value) + // return json_value + // } + + // // Provides bucket path to the actual file to download from the media file + // async getMediaFile(media_file_token: string): Promise { + // let api_base_url = "https://api.fakeyou.com"; + // let url = `${api_base_url}/v1/media_files/file/${media_file_token}` + // let response = await fetch(url); + // let json = await JSON.parse(await response.text()); + // let bucketPath = json["media_file"]["public_bucket_path"]; + // let media_base_url = "https://storage.googleapis.com/vocodes-public" + // let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path + // // TODO get it as binary. + // return media_url + // } + + + // async gltfExport2(scene: THREE.Scene) { + // let gltfExporter = new GLTFExporter(); + // let uuid = uuidv4(); + // // save the scene to keep ids positions etc + // let result = await gltfExporter.parseAsync(scene); + // const file = new File([JSON.stringify(result)], `${uuid}.glb`, { + // type: "application/json", + // }); + // return file; + // } + // // two core functions to upload things to the server and get id's + // async uploadMedia2(blob: any, fileName: string): Promise { + // const url = `${this.baseUrl}/v1/media_uploads/upload`; + // let uuid = uuidv4(); + // const formData = new FormData(); + // formData.append("uuid_idempotency_token", uuid); + // formData.append("file", blob, fileName); + // formData.append("source", "file"); + // formData.append("type", "video"); + // formData.append("source", "file"); + // const response = await fetch(url, { + // method: "POST", + // credentials: "include", + // headers: { + // Accept: "application/json", + // }, + // body: formData, + // }); + + // if (!response.ok) { + // throw new Error("Failed to Send Data"); + // } + // return response.json(); + // } + /** To save the scene as a file get the id back to serialize /v1/media_files/write/engine_asset @@ -179,35 +408,35 @@ class APIManager { media_file_subtype (optional; mixamo, mocap_net, scene_import, animation_only) media_file_class (audio, image, video, animation, character, prop, scene, unknown) **/ - async uploadGLB(file: File, scene_media_file_token:string | null): Promise { - const url = `${this.baseUrl}/v1/media_files/write/engine_asset`; - let uuid = uuidv4(); - const formData = new FormData(); - formData.append("uuid_idempotency_token", uuid); + // async uploadGLB2(file: File, scene_media_file_token:string | null): Promise { + // const url = `${this.baseUrl}/v1/media_files/write/engine_asset`; + // let uuid = uuidv4(); + // const formData = new FormData(); + // formData.append("uuid_idempotency_token", uuid); - // update existing scene otherwise create new scene and use it's media_file_id - if (scene_media_file_token != null) { - formData.append("media_file_token", scene_media_file_token); - } + // // update existing scene otherwise create new scene and use it's media_file_id + // if (scene_media_file_token != null) { + // formData.append("media_file_token", scene_media_file_token); + // } - formData.append("file", file); - formData.append("source", "file"); - formData.append("media_file_subtype", "scene_import"); - formData.append("media_file_class", "scene"); - const response = await fetch(url, { - method: "POST", - credentials: "include", - headers: { - Accept: "application/json", - }, - body: formData, - }); + // formData.append("file", file); + // formData.append("source", "file"); + // formData.append("media_file_subtype", "scene_import"); + // formData.append("media_file_class", "scene"); + // const response = await fetch(url, { + // method: "POST", + // credentials: "include", + // headers: { + // Accept: "application/json", + // }, + // body: formData, + // }); - if (!response.ok) { - throw new Error("Failed to Send Data"); - } - return response.json(); // or handle the response as appropriate - } + // if (!response.ok) { + // throw new Error("Failed to Send Data"); + // } + // return response.json(); // or handle the response as appropriate + // } } export default APIManager; diff --git a/app/pages/PageEnigma/js/audio_engine.ts b/app/pages/PageEnigma/js/audio_engine.ts index 17e36ac368..e6576636da 100644 --- a/app/pages/PageEnigma/js/audio_engine.ts +++ b/app/pages/PageEnigma/js/audio_engine.ts @@ -1,8 +1,8 @@ import { faL } from "@fortawesome/pro-solid-svg-icons"; -import { AudioTrackClip } from "../datastructures/clips/audio_track_clip"; +import { AudioClip } from "../datastructures/clips/audio_clip"; class AudioEngine { - clips: { [key: string]: AudioTrackClip } = {}; + clips: { [key: string]: AudioClip } = {}; audio_sources: { [key: string]: AudioBufferSourceNode } = {}; version: number; playing: string[]; @@ -17,7 +17,7 @@ class AudioEngine { // loads clips into the engine to cache loadClip(audio_media_id: string) { if(this.clips[audio_media_id] != null) { return; } - this.clips[audio_media_id] = new AudioTrackClip(this.version, audio_media_id, 1.0); + this.clips[audio_media_id] = new AudioClip(this.version, audio_media_id, 1.0); } // plays from the timeline. diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index 8cc07f9c87..565f38678f 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -16,7 +16,7 @@ import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; import AudioEngine from "./audio_engine.ts"; import TransformEngine from "./transform_engine.ts"; -import { TimeLine } from "./timeline.ts"; +import { TimeLine, TimelineDataState } from "./timeline.ts"; import { ClipUI } from "../datastructures/clips/clip_offset.ts"; import { LipSync } from "./lipsync.js"; @@ -36,6 +36,13 @@ if (typeof window !== "undefined") { } class EditorState { + + // { + // action: "ShowLoadingIndicator" + // source: "Editor" + // data: { "message" : "saving scene" } + // } + constructor() { this.selected_object = null; this.is_loading = false; @@ -104,6 +111,7 @@ class Editor { this.transform_engine = new TransformEngine(); this.lipsync_engine = new LipSyncEngine(); this.animation_engine = new AnimationEngine(); + this.timeline = new TimeLine( this.audio_engine, this.transform_engine, @@ -168,19 +176,39 @@ class Editor { this.timeline.scene = this.activeScene; - this._test_demo() - + //this._test_demo() // saving state of the scene this.scene_file_token = null } + // Token comes in from the front end to load the scene from the site. + async loadScene() { + + const scene = await this.api_manager.loadSceneState(this.test_scene_load_media_id); + this.activeScene.scene.children = scene.children; + + this.activeScene.scene.children.forEach(child => { + + child.parent = this.activeScene.scene; + + if(child.type == "DirectionalLight") { + let pos = child.position; + let rot = child.rotation; + let light = this.activeScene._create_base_lighting(); + light.position.set(pos.x, pos.y, pos.z); + light.rotation.set(rot.x, rot.y, rot.z); + this.activeScene.scene.remove(child); + } + }) + + } async saveScene(name) { - // check if I already have the scene token. - console.log("Saving..."); - const result = await this.api_manager.saveSceneState(this.activeScene.scene,this.scene_file_token); - console.log("Saved!"); + const result = await this.api_manager.saveSceneState(this.activeScene.scene,name, + this.scene_file_token, + new TimelineDataState()); + // dispatch call wil's engine. } // uploading some objects for testing to get their media ids from my account. diff --git a/app/pages/PageEnigma/js/lip_sync_engine.ts b/app/pages/PageEnigma/js/lip_sync_engine.ts index 15d808e1e7..1bfb590da2 100644 --- a/app/pages/PageEnigma/js/lip_sync_engine.ts +++ b/app/pages/PageEnigma/js/lip_sync_engine.ts @@ -1,24 +1,22 @@ -import { LipSyncTrackClip } from "../datastructures/clips/lipsync_track_clip"; -import { LipSync } from "./lipsync"; -import * as THREE from 'three'; +import { LipSyncClip } from "../datastructures/clips/lipsync_clip"; + // needs to be apart of the editor window. export class LipSyncEngine { - clips: { [key: string]: LipSyncTrackClip } = {} + clips: { [key: string]: LipSyncClip } = {} audio_sources: { [key: string]: AudioBufferSourceNode } = {} version: number constructor() { - this.clips = {} this.audio_sources = {} this.version = 1.0 } load_object(object_uuid: string, audio_media_id: string) { - this.clips[object_uuid] = new LipSyncTrackClip(this.version, audio_media_id, 1.0); + this.clips[object_uuid] = new LipSyncClip(this.version, audio_media_id, 1.0); } } diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 84fa27d49c..208a7e20e9 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -6,18 +6,46 @@ import AudioEngine from "./audio_engine"; import TransformEngine from "./transform_engine"; import LipSyncEngine from "./lip_sync_engine"; import AnimationEngine from "./animation_engine"; +import { AudioClip } from "../models/track"; +import { AnimationClip } from "three"; +import { LipSyncClip } from "../datastructures/clips/lipsync_clip"; +import { TransformClip } from "../datastructures/clips/transform_clip"; // Every object uuid / entity has a track. -export class TimelineCurrentReactState { -isEditable: boolean +export class TimelineCurrentState { + isEditable: boolean selectedObjectID: number - constructor() { this.isEditable = true // can add clips to it this.selectedObjectID = 0 } } +export class TimelineDataState { + timelineItems: ClipUI[] + scrubberPosition: number + + // Data to serialize with the scene, used to load into each engine. + audioClips:AudioClip[] + animationClips:AnimationClip[] + lipSyncClips:LipSyncClip[] + transformClips:TransformClip[] + + constructor(timelineItems: ClipUI[] = [], + audioClips: AudioClip[] = [], + animationClips: AnimationClip[] = [], + transformClips: TransformClip[] = [], + lipSyncClips: LipSyncClip[] = [], + scrubberPosition:number = 0) { + this.timelineItems = timelineItems + this.scrubberPosition = scrubberPosition + this.audioClips = audioClips + this.animationClips = animationClips + this.lipSyncClips = lipSyncClips + this.transformClips = transformClips + } +} + export class TimeLine { timelineItems: ClipUI[] @@ -26,8 +54,6 @@ export class TimeLine { scrubberPosition: number isPlaying: boolean - timelineState: TimelineCurrentReactState - // plays audio audioEngine: AudioEngine // key framing @@ -50,7 +76,6 @@ export class TimeLine { this.isPlaying = false this.scrubberPosition = 0 // in frames into the tl - this.timelineState = new TimelineCurrentReactState() // this will be used to play the audio clips this.audioEngine = audioEngine @@ -61,60 +86,60 @@ export class TimeLine { this.scene = scene; } - async addPlayableClip(clip: ClipUI): Promise { + public async addPlayableClip(clip: ClipUI): Promise { this.timelineItems.push(clip) } // when given a media id item it will create the clip. // Then the clip will be loaded by the engines, if they come from outside of the loaded scene. - async createClipOffset(media_id: string, object_uuid:string, type: string): Promise { + public async createClipOffset(media_id: string, object_uuid:string, type: string): Promise { // use engine to load based off media id and type animation | transform | } // this will update the state of the clips based off uuid easing? - async updateClip(clip_uuid: string, updates: AnyJson): Promise { + public async updateClip(clip_uuid: string, updates: AnyJson): Promise { } - async deleteClip(clip_uuid: string): Promise { + public async deleteClip(clip_uuid: string): Promise { } // Events that will trigger from react - async clipDidEnterDropZone() { + public async clipDidEnterDropZone() { } - async clipDidExitDropZone() { + public async clipDidExitDropZone() { } // timeline controls this. - async scrubberDidStart(offset_frame: number) { + public async scrubberDidStart(offset_frame: number) { } - async scrub(offset_frame: number): Promise { + public async scrub(offset_frame: number): Promise { // only stream through to the position and rotation keyframes // debounce not really } - async scrubberDidStop(offset_frame: number) { + public async scrubberDidStop(offset_frame: number) { } // public streaming events into the timeline from - async setScrubberPosition(offset: number) { + public async setScrubberPosition(offset: number) { this.scrubberPosition = offset // in ms } // should play from the clip that is closest to the to scrubber - async play(): Promise { + public async play(): Promise { console.log(`Starting Timeline`) this.isPlaying = true } - async reset_scene() { + private async reset_scene() { for (const element of this.timelineItems) { if (element.type == "transform") { let object = this.scene.get_object_by_uuid(element.object_uuid); @@ -136,7 +161,7 @@ export class TimeLine { } // called by the editor update loop on each frame - async update(deltatime:number) { + public async update(deltatime:number) { if (this.isPlaying == false) return; // start and stop if (this.scrubberPosition <= 0) { @@ -191,7 +216,7 @@ export class TimeLine { } } - async stop(): Promise { + private async stop(): Promise { this.isPlaying = false console.log(`Stopping Timeline`) } diff --git a/app/pages/PageEnigma/js/transform_engine.ts b/app/pages/PageEnigma/js/transform_engine.ts index c5eeb170b0..12c089f913 100644 --- a/app/pages/PageEnigma/js/transform_engine.ts +++ b/app/pages/PageEnigma/js/transform_engine.ts @@ -1,8 +1,8 @@ -import { TransformTrackClip } from "../datastructures/clips/transform_track_clip"; +import { TransformClip } from "../datastructures/clips/transform_clip"; import * as THREE from 'three'; class TransformEngine { - clips: { [key: string]: TransformTrackClip } = {}; + clips: { [key: string]: TransformClip } = {}; version: number; length: number; @@ -13,13 +13,13 @@ class TransformEngine { } loadObject(object_uuid: string, clip_length:number = 2) { - this.clips[object_uuid] = new TransformTrackClip(this.version, object_uuid, clip_length, object_uuid); // replace the last object_uuid with the media ID when its ready. + this.clips[object_uuid] = new TransformClip(this.version, object_uuid, clip_length, object_uuid); // replace the last object_uuid with the media ID when its ready. return this.clips[object_uuid] } addFrame(object: THREE.Object3D, clip_length:number = 2) { if(this.clips[object.uuid] == null) { - this.clips[object.uuid] = new TransformTrackClip(this.version, object.uuid, clip_length); + this.clips[object.uuid] = new TransformClip(this.version, object.uuid, clip_length); } this.clips[object.uuid].add_position(object.position); this.clips[object.uuid].add_rotation(new THREE.Vector3(object.rotation.x, object.rotation.y, object.rotation.z)); From 72155d8a9b3ee369ab27b4f3738ab72476bc75be Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Wed, 27 Mar 2024 17:33:41 -0400 Subject: [PATCH 12/23] converted to ts --- app/pages/PageEnigma/js/animation_engine.ts | 4 +- app/pages/PageEnigma/js/api_manager.ts | 4 +- .../PageEnigma/js/{editor.js => editor.ts} | 261 ++++++++++-------- app/pages/PageEnigma/js/scene.ts | 1 + app/pages/PageEnigma/js/transform_engine.ts | 4 +- 5 files changed, 152 insertions(+), 122 deletions(-) rename app/pages/PageEnigma/js/{editor.js => editor.ts} (76%) diff --git a/app/pages/PageEnigma/js/animation_engine.ts b/app/pages/PageEnigma/js/animation_engine.ts index a2fde07189..1ec3dc1bf7 100644 --- a/app/pages/PageEnigma/js/animation_engine.ts +++ b/app/pages/PageEnigma/js/animation_engine.ts @@ -4,12 +4,10 @@ import * as THREE from 'three'; export class AnimationEngine { clips: { [key: string]: AnimationClip } = {}; version: number; - length: number; - constructor(version: number, length: number) { + constructor(version: number) { this.clips = {}; this.version = version; - this.length = length; } load_object(object_uuid: string, media_id:string, clip_name: string) { diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index a081738d41..707dc9c19b 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -51,7 +51,7 @@ class APIManager { public async saveSceneState(scene: THREE.Scene, scene_name:string, scene_media_file_token:string | null = null, - timeline_state:TimelineDataState):Promise { + timeline_state:TimelineDataState | null):Promise { const file = await this.gltfExport(scene) console.log(`FILE: ${file}`) @@ -213,7 +213,7 @@ class APIManager { return json_data // or handle the response as appropriate } - private async uploadMedia(blob:any,fileName:string):Promise { + public async uploadMedia(blob:any,fileName:string):Promise { const url = `${this.baseUrl}/v1/media_uploads/upload`; let uuid = uuidv4(); diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.ts similarity index 76% rename from app/pages/PageEnigma/js/editor.js rename to app/pages/PageEnigma/js/editor.ts index 565f38678f..299e391d23 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.ts @@ -1,8 +1,8 @@ import * as THREE from "three"; import { OrbitControls } from "three/addons/controls/OrbitControls.js"; import { TransformControls } from "three/addons/controls/TransformControls.js"; -import Scene from "./scene.ts"; -import APIManager from "./api_manager.ts"; +import Scene from "./scene.js"; +import APIManager from "./api_manager.js"; import { EffectComposer } from "three/addons/postprocessing/EffectComposer.js"; import { RenderPass } from "three/addons/postprocessing/RenderPass.js"; import { OutlinePass } from "three/addons/postprocessing/OutlinePass.js"; @@ -14,27 +14,16 @@ import { BokehPass } from "three/addons/postprocessing/BokehPass.js"; import { createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg"; import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; -import AudioEngine from "./audio_engine.ts"; -import TransformEngine from "./transform_engine.ts"; -import { TimeLine, TimelineDataState } from "./timeline.ts"; -import { ClipUI } from "../datastructures/clips/clip_offset.ts"; +import AudioEngine from "./audio_engine.js"; +import TransformEngine from "./transform_engine.js"; +import { TimeLine, TimelineDataState } from "./timeline.js"; +import { ClipUI } from "../datastructures/clips/clip_offset.js"; import { LipSync } from "./lipsync.js"; -import { LipSyncEngine } from "./lip_sync_engine.ts"; -import { AnimationEngine } from "./animation_engine.ts"; +import { LipSyncEngine } from "./lip_sync_engine.js"; +import { AnimationEngine } from "./animation_engine.js"; import { faL } from "@fortawesome/pro-solid-svg-icons"; -if (typeof window !== "undefined") { - import("ccapture.js") - .then((module) => { - const CCapture = module.CCapture; - // You can use CCapture here - }) - .catch((error) => { - console.error("Failed to load CCapture:", error); - }); -} - class EditorState { // { @@ -43,34 +32,75 @@ class EditorState { // data: { "message" : "saving scene" } // } + selected_object: THREE.Object3D | undefined; + is_loading: boolean; + constructor() { - this.selected_object = null; + this.selected_object; this.is_loading = false; } } // Main editor class that will call everything else all you need to call is " initialize(); ". class Editor { + version: number; + activeScene: Scene; + camera: any; + renderer: THREE.WebGLRenderer | undefined; + clock: THREE.Clock | undefined; + canvReference: any; + composer: EffectComposer | undefined; + effectFXAA: EffectComposer | undefined; + outlinePass: OutlinePass | undefined; + last_cam_pos: THREE.Vector3 | undefined; + saoPass: SAOPass | undefined; + outputPass: OutlinePass | undefined; + bloomPass: UnrealBloomPass | undefined; + smaaPass: SMAAPass | undefined; + bokehPass: BokehPass | undefined; + control: TransformControls | undefined; + raycaster: THREE.Raycaster | undefined; + mouse: THREE.Vector2 | undefined; + selected: THREE.Object3D | undefined; + last_selected: THREE.Object3D | undefined; + transform_interaction: any; + rendering: boolean; + api_manager: APIManager; + stats: any; + orbit: OrbitControls | undefined; + locked: boolean; + capturer: any; + frame_buffer: any; + render_timer: number; + fps_number: number; + cap_fps: number; + playback: boolean; + playback_location: number; + max_length: number; + audio_engine: AudioEngine; + transform_engine: TransformEngine; + lipsync_engine: LipSyncEngine; + animation_engine: AnimationEngine; + timeline: TimeLine; + current_frame: number; + test_scene_load_media_id: string; + scene_file_token: any; + renderPass: RenderPass | undefined; + + // Default params. constructor() { console.log( "If you see this message twice! then it rendered twice, if you see it once it's all good.", ); - // For making sure the editor only gets created onece. - this.can_initailize = false; - let one_element = document.getElementById("created-one-element"); - //if (one_element != null) { return; } - this.can_initailize = true; - let newElement = document.createElement("div"); - newElement.id = "created-one-element"; - document.body.appendChild(newElement); // Version and name. - this.version = "v0.1"; + this.version = 0.1; // Clock, scene and camera essentials. - this.activeScene = null; - this.camera = null; - this.renderer = null; - this.clock = null; + this.activeScene = new Scene("" + this.version); + this.activeScene.initialize(); + this.camera; + this.renderer; + this.clock; this.canvReference = null; this.composer; this.effectFXAA; @@ -82,18 +112,18 @@ class Editor { this.smaaPass; this.bokehPass; // Transform control and selection. - this.control = null; - this.raycaster = null; - this.mouse = null; - this.selected = null; - this.last_selected = null; + this.control; + this.raycaster; + this.mouse; + this.selected; + this.last_selected; this.transform_interaction; this.rendering = false; // API. this.api_manager = new APIManager(); // Debug & Movement. this.stats = null; - this.orbit = null; + this.orbit; this.locked = false; // Recording params. this.capturer = null; @@ -108,9 +138,9 @@ class Editor { // Audio Engine Test. this.audio_engine = new AudioEngine(); - this.transform_engine = new TransformEngine(); + this.transform_engine = new TransformEngine(this.version); this.lipsync_engine = new LipSyncEngine(); - this.animation_engine = new AnimationEngine(); + this.animation_engine = new AnimationEngine(this.version); this.timeline = new TimeLine( this.audio_engine, @@ -120,18 +150,13 @@ class Editor { this.activeScene, ); - this.test_box_uuid = null; this.current_frame = 0; - this.test_playback = false; this.test_scene_load_media_id = ""; // Test media id for saving and loading. } // Initializes the main scene and ThreeJS essentials. initialize() { - if (this.can_initailize == false) { - return; - } // Gets the canvas. this.canvReference = document.getElementById("video-scene"); // Base width and height. @@ -149,16 +174,11 @@ class Editor { preserveDrawingBuffer: true, }); this.renderer.shadowMap.enabled = true; - this.renderer.shadowMapSoft = true; this.clock = new THREE.Clock(); // Resizes the renderer. this.renderer.setSize(width, height); //document.body.appendChild(this.renderer.domElement); window.addEventListener("resize", this.onWindowResize.bind(this)); - // Current scene for saving and loading. - this.activeScene = new Scene(); - this.activeScene.initialize(); - this._configure_post_pro(); // Controls and movement. this.orbit = new OrbitControls(this.camera, this.renderer.domElement); @@ -172,7 +192,6 @@ class Editor { this.onWindowResize(); // Creates the main update loop. this.renderer.setAnimationLoop(this.update_loop.bind(this)); - this.test_playback = false; this.timeline.scene = this.activeScene; @@ -188,26 +207,26 @@ class Editor { const scene = await this.api_manager.loadSceneState(this.test_scene_load_media_id); this.activeScene.scene.children = scene.children; - this.activeScene.scene.children.forEach(child => { + this.activeScene.scene.children.forEach((child: THREE.Object3D) => { - child.parent = this.activeScene.scene; + child.parent = this.activeScene.scene; - if(child.type == "DirectionalLight") { - let pos = child.position; - let rot = child.rotation; - let light = this.activeScene._create_base_lighting(); - light.position.set(pos.x, pos.y, pos.z); - light.rotation.set(rot.x, rot.y, rot.z); - this.activeScene.scene.remove(child); + if (child.type == "DirectionalLight") { + let pos = child.position; + let rot = child.rotation; + let light = this.activeScene._create_base_lighting(); + light.position.set(pos.x, pos.y, pos.z); + light.rotation.set(rot.x, rot.y, rot.z); + this.activeScene.scene.remove(child); } }) - + } - async saveScene(name) { - const result = await this.api_manager.saveSceneState(this.activeScene.scene,name, - this.scene_file_token, - new TimelineDataState()); + async saveScene(name: string) { + const result = await this.api_manager.saveSceneState(this.activeScene.scene, name, + this.scene_file_token, + new TimelineDataState()); // dispatch call wil's engine. } @@ -217,44 +236,45 @@ class Editor { // This is the default scene. // "m_189p8hj0eyypbg74kkhcpehwpjhnkz" scene with the fox. - + //let result = await this.api_manager.saveSceneState(this.activeScene.scene) - + // let result = await this.api_manager.getMediaFile( // "m_189p8hj0eyypbg74kkhcpehwpjhnkz", // ); console.log("Saving..."); - const result = await this.api_manager.saveSceneState(this.activeScene.scene); + const result = await this.api_manager.saveSceneState(this.activeScene.scene, "", null, null); console.log("Saved!"); console.log("Media ID is:", result); - this.test_scene_load_media_id = result["media_file_token"]; + if (result.data) { + this.test_scene_load_media_id = result.data["media_file_token"]; + } } - async _load_for_testing() { - const result = await this.api_manager.loadScene(this.test_scene_load_media_id); - let bucket_path = await this.api_manager.getMediaFile(result["glb_media_file_id"]); - let glbLoader = new GLTFLoader(); - glbLoader.load(bucket_path, (glb) => { - this.activeScene.scene.children = glb.scene.children; - this.activeScene.scene.children.forEach(child => { - child.parent = this.activeScene.scene; - if(child.type == "DirectionalLight") { - let pos = child.position; - let rot = child.rotation; - let light = this.activeScene._create_base_lighting(); - light.position.set(pos.x, pos.y, pos.z); - light.rotation.set(rot.x, rot.y, rot.z); - this.activeScene.scene.remove(child); - } - }); - }); - - } + //async _load_for_testing() { + // const result = await this.api_manager.loadScene(this.test_scene_load_media_id); + // let bucket_path = await this.api_manager.getMediaFile(result["glb_media_file_id"]); + // let glbLoader = new GLTFLoader(); + // glbLoader.load(bucket_path, (glb) => { + // this.activeScene.scene.children = glb.scene.children; + // this.activeScene.scene.children.forEach((child: THREE.Object3D) => { + // child.parent = this.activeScene.scene; + // if (child.type == "DirectionalLight") { + // let pos = child.position; + // let rot = child.rotation; + // let light = this.activeScene._create_base_lighting(); + // light.position.set(pos.x, pos.y, pos.z); + // light.rotation.set(rot.x, rot.y, rot.z); + // this.activeScene.scene.remove(child); + // } + // }); + // }); + //} async _serialize_timeline() { // note the database from the server is the source of truth for all the data. // Test code here - let object = await this.activeScene.load_glb( + let object: any = await this.activeScene.load_glb( "./resources/models/fox/fox.glb", ); @@ -304,7 +324,7 @@ class Editor { async _test_demo() { // note the database from the server is the source of truth for all the data. // Test code here - let object = await this.activeScene.load_glb( + let object: any = await this.activeScene.load_glb( "./resources/models/fox/fox.glb", ); @@ -319,8 +339,8 @@ class Editor { "/resources/models/fox/fox_idle.glb", "clip3", ); - // then it creates clip ui to load the playable clips - // then refreshes the timeline. + // then it creates clip ui to load the playable clips + // then refreshes the timeline. // create the clip with the same id for a reference to the media this.timeline.addPlayableClip( @@ -352,7 +372,7 @@ class Editor { 400, ), ); - + } // Configure post processing. @@ -360,6 +380,10 @@ class Editor { let width = this.canvReference.width; let height = this.canvReference.height; + if(this.renderer == undefined || this.camera == undefined) { + return; + } + this.composer = new EffectComposer(this.renderer); this.renderPass = new RenderPass(this.activeScene.scene, this.camera); this.composer.addPass(this.renderPass); @@ -374,7 +398,6 @@ class Editor { this.outlinePass.edgeGlow = 0.2; this.outlinePass.edgeThickness = 1.0; this.outlinePass.pulsePeriod = 3; - this.outlinePass.rotate = false; this.outlinePass.usePatternTexture = false; this.outlinePass.visibleEdgeColor.set(0xe66462); @@ -412,11 +435,11 @@ class Editor { this.composer.addPass(this.smaaPass); this.composer.addPass(this.bokehPass); - this.outputPass = new OutputPass(); - this.composer.addPass(this.outputPass); + //this.outputPass = new OutputPass(); + //this.composer.addPass(this.outputPass); } - create_parim(name) { + create_parim(name: string) { let uuid = this.activeScene.instantiate(name); } @@ -464,17 +487,18 @@ class Editor { //this.transform_engine.clips[this.test_box_uuid].step(obj); //console.log(this.transform_engine.clips[this.test_box_uuid].current_pos) //this.current_frame += 1; - this.test_playback = !this.test_playback; //this.transform_engine.clips[this.test_box_uuid].reset(this.activeScene.get_object_by_uuid(this.test_box_uuid)); } // Basicly Unity 3D's update loop. - update_loop(time) { + update_loop(time: number) { // Updates debug stats. if (this.stats != null) { this.stats.update(); } + if(this.clock == undefined || this.renderer == undefined) {return;} + let delta_time = this.clock.getDelta(); // All calls that are not super important like timeline go here. @@ -495,13 +519,14 @@ class Editor { this.timeline.isPlaying = true; } - change_mode(type) { + change_mode(type: any) { + if(this.control == undefined) {return;} this.control.mode = type; this.transform_interaction = true; } // Sets the fps to a specific number - set_fps(fps_number) { + set_fps(fps_number: number) { this.cap_fps = fps_number; } @@ -517,7 +542,7 @@ class Editor { } } - async loadWavAsBlob(url) { + async loadWavAsBlob(url: string) { const response = await fetch(url); const blob = await response.blob(); return blob; @@ -565,13 +590,15 @@ class Editor { // Initializes transform x y z changes. _initialize_control() { + if(this.control == undefined) {return;} this.control.addEventListener("change", this.render_scene.bind(this)); this.control.addEventListener( "dragging-changed", - function (event) { + (event: any) => { + if(this.orbit == undefined) { return; } this.orbit.enabled = !event.value; // this.update_properties(); - }.bind(this), + } ); this.control.setSize(0.5); // Good default value for visuals. this.raycaster = new THREE.Raycaster(); @@ -587,13 +614,16 @@ class Editor { // Render the scene to the camera. render_scene() { + if (this.composer != null) { this.composer.render(); - } else { + } else if (this.renderer && this.camera) { this.renderer.render(this.activeScene.scene, this.camera); + } else { + console.error("Could not render to canvas no render or composer!"); } - if (this.rendering) { + if (this.rendering && this.renderer && this.clock) { this.playback_location++; let imgData = this.renderer.domElement.toDataURL(); this.frame_buffer.push(imgData); @@ -612,6 +642,7 @@ class Editor { let width = window.innerWidth; // / aspect_adjust; let height = window.innerHeight; // / aspectRatio; + if(this.camera == undefined || this.renderer == undefined) { return; } // Set the camera aspect to the desired aspect ratio this.camera.aspect = width / height; this.camera.updateProjectionMatrix(); @@ -624,17 +655,19 @@ class Editor { } // Sets new mouse location usually used in raycasts. - onMouseMove(event) { + onMouseMove(event: any) { const rect = this.canvReference.getBoundingClientRect(); + if(this.mouse == undefined) { return; } this.mouse.x = ((event.clientX - rect.left) / rect.width) * 2 - 1; this.mouse.y = -((event.clientY - rect.top) / rect.height) * 2 + 1; } // When the mouse clicks the screen. onMouseClick() { + if(this.raycaster == undefined || this.mouse == undefined || this.control == undefined || this.outlinePass == undefined) { return; } this.raycaster.setFromCamera(this.mouse, this.camera); - let interactable = []; - this.activeScene.scene.children.forEach((child) => { + let interactable: any[] = []; + this.activeScene.scene.children.forEach((child: THREE.Object3D) => { if (child.name != "") { if (child.type == "Mesh" || child.type == "Object3D") { interactable.push(child); @@ -646,9 +679,9 @@ class Editor { if (intersects.length > 0) { if (intersects[0].object.type != "GridHelper") { let currentObject = intersects[0].object; - while (currentObject.parent.type !== "Scene") { + while (currentObject.parent && currentObject.parent.type !== "Scene") { currentObject = currentObject.parent; - } + } this.selected = currentObject; if (this.selected.type == "Scene") { this.selected = intersects[0].object; @@ -661,7 +694,7 @@ class Editor { } } else if (this.transform_interaction == false) { this.last_selected = this.selected; - this.control.detach(this.selected); + this.control.detach(); this.activeScene.scene.remove(this.control); this.outlinePass.selectedObjects = []; } else { diff --git a/app/pages/PageEnigma/js/scene.ts b/app/pages/PageEnigma/js/scene.ts index 1f48136e42..52693599cf 100644 --- a/app/pages/PageEnigma/js/scene.ts +++ b/app/pages/PageEnigma/js/scene.ts @@ -99,6 +99,7 @@ class Scene { '/resources/skybox/night/Night_Moon_Burst_Cam_1_Back-Z.png', ]); this.scene.background = texture; + console.log("Backround creation..") } // deafult image skybox. diff --git a/app/pages/PageEnigma/js/transform_engine.ts b/app/pages/PageEnigma/js/transform_engine.ts index 12c089f913..e1e29ccf6e 100644 --- a/app/pages/PageEnigma/js/transform_engine.ts +++ b/app/pages/PageEnigma/js/transform_engine.ts @@ -4,11 +4,9 @@ import * as THREE from 'three'; class TransformEngine { clips: { [key: string]: TransformClip } = {}; version: number; - length: number; - constructor(version: number, length: number) { + constructor(version: number) { this.version = version; - this.length = length; this.clips = {}; } From 2ed5978fd02fdd138ce760e9e20204a4e08dccb3 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 27 Mar 2024 17:42:54 -0400 Subject: [PATCH 13/23] Committing before pulling changes from ts editor changes. --- app/pages/PageEnigma/js/editor.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index 0c58279759..4713bdace9 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -193,6 +193,7 @@ class Editor { // saving state of the scene this.scene_file_token = null + } // Token comes in from the front end to load the scene from the site. @@ -218,9 +219,18 @@ class Editor { } async saveScene(name) { - const result = await this.api_manager.saveSceneState(this.activeScene.scene,name, - this.scene_file_token, - new TimelineDataState()); + this.dispatchAppUiState({ + type: ACTION_TYPES.SHOW_EDITOR_LOADER + }); + setInterval(()=> { + this.dispatchAppUiState({ + type: ACTION_TYPES.HIDE_EDITOR_LOADER + }); + },4000) + // const result = await this.api_manager.saveSceneState(this.activeScene.scene,name, + // this.scene_file_token, + // new TimelineDataState()); + // dispatch call wil's engine. } From 98749177c7390951a9af987be0fae5454c4301f8 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 27 Mar 2024 18:04:43 -0400 Subject: [PATCH 14/23] Editor buttons reassigned and removed ; --- .../ControlsTopButtons/ControlsTopButtons.tsx | 13 +- app/pages/PageEnigma/js/editor.ts | 671 +++++++++--------- 2 files changed, 343 insertions(+), 341 deletions(-) diff --git a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx index 72864b5cfb..3ed0d03d8d 100644 --- a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx +++ b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx @@ -10,12 +10,12 @@ export const ControlsTopButtons = () => { const handleButtonSave = () => { console.log(`SceneName is ${sceneName}`); - editorEngine?.saveScene(); + editorEngine?.saveScene(sceneName); }; const handleButtonLoadScene = () => { console.log(`Scene Token is ${sceneToken}`); - // editorEngine?.dosomething(); + editorEngine?.loadScene(sceneToken); }; // const handleButtonCameraView = () => { @@ -26,14 +26,11 @@ export const ControlsTopButtons = () => { editorEngine?.start_playback(); }; const handleButtonLoad = () => { - //document.getElementById("load-upload")?.click(); - if(editorEngine == null) {return;} - editorEngine.loadScene(); + + }; const handleButtonRender = () => { - console.log("Saving GLB to server for reference"); - editorEngine?.saveScene(); - //editorEngine?.togglePlayback(); + }; const handleButtonPlay = () => {}; diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 299e391d23..32365a8360 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -1,28 +1,28 @@ -import * as THREE from "three"; -import { OrbitControls } from "three/addons/controls/OrbitControls.js"; -import { TransformControls } from "three/addons/controls/TransformControls.js"; -import Scene from "./scene.js"; -import APIManager from "./api_manager.js"; -import { EffectComposer } from "three/addons/postprocessing/EffectComposer.js"; -import { RenderPass } from "three/addons/postprocessing/RenderPass.js"; -import { OutlinePass } from "three/addons/postprocessing/OutlinePass.js"; -import { OutputPass } from "three/addons/postprocessing/OutputPass.js"; -import { SMAAPass } from "three/addons/postprocessing/SMAAPass.js"; -import { SAOPass } from "three/addons/postprocessing/SAOPass.js"; -import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js"; -import { BokehPass } from "three/addons/postprocessing/BokehPass.js"; -import { createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg"; -import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; +import * as THREE from "three" +import { OrbitControls } from "three/addons/controls/OrbitControls.js" +import { TransformControls } from "three/addons/controls/TransformControls.js" +import Scene from "./scene.js" +import APIManager from "./api_manager.js" +import { EffectComposer } from "three/addons/postprocessing/EffectComposer.js" +import { RenderPass } from "three/addons/postprocessing/RenderPass.js" +import { OutlinePass } from "three/addons/postprocessing/OutlinePass.js" +import { OutputPass } from "three/addons/postprocessing/OutputPass.js" +import { SMAAPass } from "three/addons/postprocessing/SMAAPass.js" +import { SAOPass } from "three/addons/postprocessing/SAOPass.js" +import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js" +import { BokehPass } from "three/addons/postprocessing/BokehPass.js" +import { createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg" +import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js" -import AudioEngine from "./audio_engine.js"; -import TransformEngine from "./transform_engine.js"; -import { TimeLine, TimelineDataState } from "./timeline.js"; -import { ClipUI } from "../datastructures/clips/clip_offset.js"; +import AudioEngine from "./audio_engine.js" +import TransformEngine from "./transform_engine.js" +import { TimeLine, TimelineDataState } from "./timeline.js" +import { ClipUI } from "../datastructures/clips/clip_offset.js" -import { LipSync } from "./lipsync.js"; -import { LipSyncEngine } from "./lip_sync_engine.js"; -import { AnimationEngine } from "./animation_engine.js"; -import { faL } from "@fortawesome/pro-solid-svg-icons"; +import { LipSync } from "./lipsync.js" +import { LipSyncEngine } from "./lip_sync_engine.js" +import { AnimationEngine } from "./animation_engine.js" +import { faL } from "@fortawesome/pro-solid-svg-icons" class EditorState { @@ -32,115 +32,116 @@ class EditorState { // data: { "message" : "saving scene" } // } - selected_object: THREE.Object3D | undefined; - is_loading: boolean; + selected_object: THREE.Object3D | undefined + is_loading: boolean constructor() { - this.selected_object; - this.is_loading = false; + this.selected_object + this.is_loading = false } } -// Main editor class that will call everything else all you need to call is " initialize(); ". +// Main editor class that will call everything else all you need to call is " initialize() ". class Editor { - version: number; - activeScene: Scene; - camera: any; - renderer: THREE.WebGLRenderer | undefined; - clock: THREE.Clock | undefined; - canvReference: any; - composer: EffectComposer | undefined; - effectFXAA: EffectComposer | undefined; - outlinePass: OutlinePass | undefined; - last_cam_pos: THREE.Vector3 | undefined; - saoPass: SAOPass | undefined; - outputPass: OutlinePass | undefined; - bloomPass: UnrealBloomPass | undefined; - smaaPass: SMAAPass | undefined; - bokehPass: BokehPass | undefined; - control: TransformControls | undefined; - raycaster: THREE.Raycaster | undefined; - mouse: THREE.Vector2 | undefined; - selected: THREE.Object3D | undefined; - last_selected: THREE.Object3D | undefined; - transform_interaction: any; - rendering: boolean; - api_manager: APIManager; - stats: any; - orbit: OrbitControls | undefined; - locked: boolean; - capturer: any; - frame_buffer: any; - render_timer: number; - fps_number: number; - cap_fps: number; - playback: boolean; - playback_location: number; - max_length: number; - audio_engine: AudioEngine; - transform_engine: TransformEngine; - lipsync_engine: LipSyncEngine; - animation_engine: AnimationEngine; - timeline: TimeLine; - current_frame: number; - test_scene_load_media_id: string; - scene_file_token: any; - renderPass: RenderPass | undefined; + version: number + activeScene: Scene + camera: any + renderer: THREE.WebGLRenderer | undefined + clock: THREE.Clock | undefined + canvReference: any + composer: EffectComposer | undefined + effectFXAA: EffectComposer | undefined + outlinePass: OutlinePass | undefined + last_cam_pos: THREE.Vector3 | undefined + saoPass: SAOPass | undefined + outputPass: OutlinePass | undefined + bloomPass: UnrealBloomPass | undefined + smaaPass: SMAAPass | undefined + bokehPass: BokehPass | undefined + control: TransformControls | undefined + raycaster: THREE.Raycaster | undefined + mouse: THREE.Vector2 | undefined + selected: THREE.Object3D | undefined + last_selected: THREE.Object3D | undefined + transform_interaction: any + rendering: boolean + api_manager: APIManager + stats: any + orbit: OrbitControls | undefined + locked: boolean + capturer: any + frame_buffer: any + render_timer: number + fps_number: number + cap_fps: number + playback: boolean + playback_location: number + max_length: number + audio_engine: AudioEngine + transform_engine: TransformEngine + lipsync_engine: LipSyncEngine + animation_engine: AnimationEngine + timeline: TimeLine + current_frame: number + test_scene_load_media_id: string + scene_file_token: any + renderPass: RenderPass | undefined + current_scene_token: string | null // Default params. constructor() { console.log( "If you see this message twice! then it rendered twice, if you see it once it's all good.", - ); + ) // Version and name. - this.version = 0.1; + this.version = 0.1 // Clock, scene and camera essentials. - this.activeScene = new Scene("" + this.version); - this.activeScene.initialize(); - this.camera; - this.renderer; - this.clock; - this.canvReference = null; - this.composer; - this.effectFXAA; - this.outlinePass; - this.last_cam_pos; - this.saoPass; - this.outputPass; - this.bloomPass; - this.smaaPass; - this.bokehPass; + this.activeScene = new Scene("" + this.version) + this.activeScene.initialize() + this.camera + this.renderer + this.clock + this.canvReference = null + this.composer + this.effectFXAA + this.outlinePass + this.last_cam_pos + this.saoPass + this.outputPass + this.bloomPass + this.smaaPass + this.bokehPass // Transform control and selection. - this.control; - this.raycaster; - this.mouse; - this.selected; - this.last_selected; - this.transform_interaction; - this.rendering = false; + this.control + this.raycaster + this.mouse + this.selected + this.last_selected + this.transform_interaction + this.rendering = false // API. - this.api_manager = new APIManager(); + this.api_manager = new APIManager() // Debug & Movement. - this.stats = null; - this.orbit; - this.locked = false; + this.stats = null + this.orbit + this.locked = false // Recording params. - this.capturer = null; - this.frame_buffer = []; - this.render_timer = 0; - this.fps_number = 60; - this.cap_fps = 60; + this.capturer = null + this.frame_buffer = [] + this.render_timer = 0 + this.fps_number = 60 + this.cap_fps = 60 // Timeline settings. - this.playback = false; - this.playback_location = 0; - this.max_length = 10; + this.playback = false + this.playback_location = 0 + this.max_length = 10 // Audio Engine Test. - this.audio_engine = new AudioEngine(); - this.transform_engine = new TransformEngine(this.version); - this.lipsync_engine = new LipSyncEngine(); - this.animation_engine = new AnimationEngine(this.version); + this.audio_engine = new AudioEngine() + this.transform_engine = new TransformEngine(this.version) + this.lipsync_engine = new LipSyncEngine() + this.animation_engine = new AnimationEngine(this.version) this.timeline = new TimeLine( this.audio_engine, @@ -148,85 +149,89 @@ class Editor { this.lipsync_engine, this.animation_engine, this.activeScene, - ); + ) - this.current_frame = 0; + this.current_frame = 0 - this.test_scene_load_media_id = ""; // Test media id for saving and loading. + this.test_scene_load_media_id = "" // Test media id for saving and loading. + + this.current_scene_token = null } // Initializes the main scene and ThreeJS essentials. initialize() { // Gets the canvas. - this.canvReference = document.getElementById("video-scene"); + this.canvReference = document.getElementById("video-scene") // Base width and height. - let width = this.canvReference.width; - let height = this.canvReference.height; + let width = this.canvReference.width + let height = this.canvReference.height // Sets up camera and base position. - this.camera = new THREE.PerspectiveCamera(70, width / height, 0.1, 50); - this.camera.position.z = 3; - this.camera.position.y = 3; - this.camera.position.x = -3; + this.camera = new THREE.PerspectiveCamera(70, width / height, 0.1, 50) + this.camera.position.z = 3 + this.camera.position.y = 3 + this.camera.position.x = -3 // Base WebGL render and clock for delta time. this.renderer = new THREE.WebGLRenderer({ antialias: true, canvas: this.canvReference, preserveDrawingBuffer: true, - }); - this.renderer.shadowMap.enabled = true; - this.clock = new THREE.Clock(); + }) + this.renderer.shadowMap.enabled = true + this.clock = new THREE.Clock() // Resizes the renderer. - this.renderer.setSize(width, height); - //document.body.appendChild(this.renderer.domElement); - window.addEventListener("resize", this.onWindowResize.bind(this)); - this._configure_post_pro(); + this.renderer.setSize(width, height) + //document.body.appendChild(this.renderer.domElement) + window.addEventListener("resize", this.onWindowResize.bind(this)) + this._configure_post_pro() // Controls and movement. - this.orbit = new OrbitControls(this.camera, this.renderer.domElement); - this.control = new TransformControls(this.camera, this.renderer.domElement); + this.orbit = new OrbitControls(this.camera, this.renderer.domElement) + this.control = new TransformControls(this.camera, this.renderer.domElement) // OnClick and MouseMove events. - window.addEventListener("mousemove", this.onMouseMove.bind(this), false); - window.addEventListener("click", this.onMouseClick.bind(this), false); + window.addEventListener("mousemove", this.onMouseMove.bind(this), false) + window.addEventListener("click", this.onMouseClick.bind(this), false) // Base control and debug stuff remove debug in prod. - this._initialize_control(); + this._initialize_control() // Resets canvas size. - this.onWindowResize(); + this.onWindowResize() // Creates the main update loop. - this.renderer.setAnimationLoop(this.update_loop.bind(this)); + this.renderer.setAnimationLoop(this.update_loop.bind(this)) - this.timeline.scene = this.activeScene; + this.timeline.scene = this.activeScene - //this._test_demo() + this._test_demo() - // saving state of the scene - this.scene_file_token = null + // saving state of the scene + this.current_scene_token = null } // Token comes in from the front end to load the scene from the site. - async loadScene() { - - const scene = await this.api_manager.loadSceneState(this.test_scene_load_media_id); - this.activeScene.scene.children = scene.children; + public async loadScene(scene_token:string) { + if (scene_token != null) { + this.current_scene_token = scene_token + } + const scene = await this.api_manager.loadSceneState(this.current_scene_token) + this.activeScene.scene.children = scene.children this.activeScene.scene.children.forEach((child: THREE.Object3D) => { - child.parent = this.activeScene.scene; + child.parent = this.activeScene.scene if (child.type == "DirectionalLight") { - let pos = child.position; - let rot = child.rotation; - let light = this.activeScene._create_base_lighting(); - light.position.set(pos.x, pos.y, pos.z); - light.rotation.set(rot.x, rot.y, rot.z); - this.activeScene.scene.remove(child); + let pos = child.position + let rot = child.rotation + let light = this.activeScene._create_base_lighting() + light.position.set(pos.x, pos.y, pos.z) + light.rotation.set(rot.x, rot.y, rot.z) + this.activeScene.scene.remove(child) } }) } - async saveScene(name: string) { + public async saveScene(name: string) { const result = await this.api_manager.saveSceneState(this.activeScene.scene, name, this.scene_file_token, - new TimelineDataState()); + new TimelineDataState()) // dispatch call wil's engine. } @@ -241,34 +246,34 @@ class Editor { // let result = await this.api_manager.getMediaFile( // "m_189p8hj0eyypbg74kkhcpehwpjhnkz", - // ); - console.log("Saving..."); - const result = await this.api_manager.saveSceneState(this.activeScene.scene, "", null, null); - console.log("Saved!"); - console.log("Media ID is:", result); + // ) + console.log("Saving...") + const result = await this.api_manager.saveSceneState(this.activeScene.scene, "", null, null) + console.log("Saved!") + console.log("Media ID is:", result) if (result.data) { - this.test_scene_load_media_id = result.data["media_file_token"]; + this.test_scene_load_media_id = result.data["media_file_token"] } } //async _load_for_testing() { - // const result = await this.api_manager.loadScene(this.test_scene_load_media_id); - // let bucket_path = await this.api_manager.getMediaFile(result["glb_media_file_id"]); - // let glbLoader = new GLTFLoader(); + // const result = await this.api_manager.loadScene(this.test_scene_load_media_id) + // let bucket_path = await this.api_manager.getMediaFile(result["glb_media_file_id"]) + // let glbLoader = new GLTFLoader() // glbLoader.load(bucket_path, (glb) => { - // this.activeScene.scene.children = glb.scene.children; + // this.activeScene.scene.children = glb.scene.children // this.activeScene.scene.children.forEach((child: THREE.Object3D) => { - // child.parent = this.activeScene.scene; + // child.parent = this.activeScene.scene // if (child.type == "DirectionalLight") { - // let pos = child.position; - // let rot = child.rotation; - // let light = this.activeScene._create_base_lighting(); - // light.position.set(pos.x, pos.y, pos.z); - // light.rotation.set(rot.x, rot.y, rot.z); - // this.activeScene.scene.remove(child); + // let pos = child.position + // let rot = child.rotation + // let light = this.activeScene._create_base_lighting() + // light.position.set(pos.x, pos.y, pos.z) + // light.rotation.set(rot.x, rot.y, rot.z) + // this.activeScene.scene.remove(child) // } - // }); - // }); + // }) + // }) //} async _serialize_timeline() { @@ -276,13 +281,13 @@ class Editor { // Test code here let object: any = await this.activeScene.load_glb( "./resources/models/fox/fox.glb", - ); + ) // load object into the engine for lip syncing this.lipsync_engine.load_object( object.uuid, "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", - ); + ) // create the clip with the same id for a reference to the media this.timeline.addPlayableClip( @@ -295,12 +300,12 @@ class Editor { 150, 400, ), - ); + ) // media id for this is up in the air but when a path is created you should be able to store and delete it this.timeline.addPlayableClip( new ClipUI(1.0, "transform", "clip2", object.uuid, object.uuid, 0, 150), - ); + ) // media id for this as well it can be downloaded this.timeline.addPlayableClip( @@ -313,12 +318,12 @@ class Editor { 0, 400, ), - ); + ) this.animation_engine.load_object( object.uuid, "/resources/models/fox/fox_idle.glb", "clip3", - ); + ) } async _test_demo() { @@ -326,19 +331,19 @@ class Editor { // Test code here let object: any = await this.activeScene.load_glb( "./resources/models/fox/fox.glb", - ); + ) // Load timeline creates the the clips from the datastructure and loads them in here. // load object into the engine for lip syncing this.lipsync_engine.load_object( object.uuid, "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", - ); + ) this.animation_engine.load_object( object.uuid, "/resources/models/fox/fox_idle.glb", "clip3", - ); + ) // then it creates clip ui to load the playable clips // then refreshes the timeline. @@ -353,12 +358,12 @@ class Editor { 150, 400, ), - ); + ) // media id for this is up in the air but when a path is created you should be able to store and delete it this.timeline.addPlayableClip( new ClipUI(1.0, "transform", "clip2", object.uuid, object.uuid, 0, 150), - ); + ) // media id for this as well it can be downloaded this.timeline.addPlayableClip( @@ -371,194 +376,194 @@ class Editor { 0, 400, ), - ); + ) } // Configure post processing. _configure_post_pro() { - let width = this.canvReference.width; - let height = this.canvReference.height; + let width = this.canvReference.width + let height = this.canvReference.height if(this.renderer == undefined || this.camera == undefined) { - return; + return } - this.composer = new EffectComposer(this.renderer); - this.renderPass = new RenderPass(this.activeScene.scene, this.camera); - this.composer.addPass(this.renderPass); + this.composer = new EffectComposer(this.renderer) + this.renderPass = new RenderPass(this.activeScene.scene, this.camera) + this.composer.addPass(this.renderPass) this.outlinePass = new OutlinePass( new THREE.Vector2(width, height), this.activeScene.scene, this.camera, - ); + ) - this.outlinePass.edgeStrength = 6.0; - this.outlinePass.edgeGlow = 0.2; - this.outlinePass.edgeThickness = 1.0; - this.outlinePass.pulsePeriod = 3; - this.outlinePass.usePatternTexture = false; - this.outlinePass.visibleEdgeColor.set(0xe66462); + this.outlinePass.edgeStrength = 6.0 + this.outlinePass.edgeGlow = 0.2 + this.outlinePass.edgeThickness = 1.0 + this.outlinePass.pulsePeriod = 3 + this.outlinePass.usePatternTexture = false + this.outlinePass.visibleEdgeColor.set(0xe66462) - this.composer.addPass(this.outlinePass); + this.composer.addPass(this.outlinePass) - this.saoPass = new SAOPass(this.activeScene.scene, this.camera); + this.saoPass = new SAOPass(this.activeScene.scene, this.camera) - this.saoPass.params.saoBias = 3.1; - this.saoPass.params.saoIntensity = 1.0; - this.saoPass.params.saoScale = 6.0; - this.saoPass.params.saoKernelRadius = 5.0; - this.saoPass.params.saoMinResolution = 0.0; + this.saoPass.params.saoBias = 3.1 + this.saoPass.params.saoIntensity = 1.0 + this.saoPass.params.saoScale = 6.0 + this.saoPass.params.saoKernelRadius = 5.0 + this.saoPass.params.saoMinResolution = 0.0 this.bloomPass = new UnrealBloomPass( new THREE.Vector2(width, height), 1.5, 0.4, 0.85, - ); - this.bloomPass.strength = 0.25; + ) + this.bloomPass.strength = 0.25 this.smaaPass = new SMAAPass( width * this.renderer.getPixelRatio(), height * this.renderer.getPixelRatio(), - ); + ) this.bokehPass = new BokehPass(this.activeScene.scene, this.camera, { focus: 3.0, aperture: 0.00001, maxblur: 0.01, - }); + }) - this.composer.addPass(this.saoPass); - this.composer.addPass(this.bloomPass); - this.composer.addPass(this.smaaPass); - this.composer.addPass(this.bokehPass); + this.composer.addPass(this.saoPass) + this.composer.addPass(this.bloomPass) + this.composer.addPass(this.smaaPass) + this.composer.addPass(this.bokehPass) - //this.outputPass = new OutputPass(); - //this.composer.addPass(this.outputPass); + //this.outputPass = new OutputPass() + //this.composer.addPass(this.outputPass) } create_parim(name: string) { - let uuid = this.activeScene.instantiate(name); + let uuid = this.activeScene.instantiate(name) } render_mode() { - this.rendering = !this.rendering; - this.activeScene.render_mode(this.rendering); + this.rendering = !this.rendering + this.activeScene.render_mode(this.rendering) //if (this.rendering) { - // this._remove_post_processing(); + // this._remove_post_processing() //} else { - // this._add_post_processing(); + // this._add_post_processing() //} } togglePlay() { - this.playback = !this.playback; - this.playback_location = 0; + this.playback = !this.playback + this.playback_location = 0 if (this.playback == false) { - this.stopPlayback(); + this.stopPlayback() } else { // REMOVE THIS NEXT LINE IN PROD ONLY FOR TEST!!!!! - //this.activeScene.play_anim_demo(this.activeScene.activeCharacter); + //this.activeScene.play_anim_demo(this.activeScene.activeCharacter) } } save() { - //console.log(this.control); - //if(this.selected != null){ this.control.detach(this.selected); } - //this.activeScene.scene.remove(this.control); - //this.activeScene.scene.remove(this.activeScene.gridHelper); - //this.save_manager.save(this.activeScene.scene, this._save_to_cloud.bind(this), this.audio_manager, this.timeline, this.activeScene.animations); - //this.activeScene._createGrid(); - //this.audio_engine.playClip("m_f7jnwt3d1ddchatdk5vaqt0n4mb1hg"); - //console.log(this.selected); + //console.log(this.control) + //if(this.selected != null){ this.control.detach(this.selected) } + //this.activeScene.scene.remove(this.control) + //this.activeScene.scene.remove(this.activeScene.gridHelper) + //this.save_manager.save(this.activeScene.scene, this._save_to_cloud.bind(this), this.audio_manager, this.timeline, this.activeScene.animations) + //this.activeScene._createGrid() + //this.audio_engine.playClip("m_f7jnwt3d1ddchatdk5vaqt0n4mb1hg") + //console.log(this.selected) if (this.selected == null) { - return; + return } - this.transform_engine.addFrame(this.selected); - console.log("Frame taken."); + this.transform_engine.addFrame(this.selected) + console.log("Frame taken.") } change_camera_view() { - //let obj = this.activeScene.get_object_by_uuid(this.test_box_uuid); - //this.transform_engine.clips[this.test_box_uuid].step(obj); + //let obj = this.activeScene.get_object_by_uuid(this.test_box_uuid) + //this.transform_engine.clips[this.test_box_uuid].step(obj) //console.log(this.transform_engine.clips[this.test_box_uuid].current_pos) - //this.current_frame += 1; - //this.transform_engine.clips[this.test_box_uuid].reset(this.activeScene.get_object_by_uuid(this.test_box_uuid)); + //this.current_frame += 1 + //this.transform_engine.clips[this.test_box_uuid].reset(this.activeScene.get_object_by_uuid(this.test_box_uuid)) } // Basicly Unity 3D's update loop. update_loop(time: number) { // Updates debug stats. if (this.stats != null) { - this.stats.update(); + this.stats.update() } - if(this.clock == undefined || this.renderer == undefined) {return;} + if(this.clock == undefined || this.renderer == undefined) {return} - let delta_time = this.clock.getDelta(); + let delta_time = this.clock.getDelta() // All calls that are not super important like timeline go here. - //this.activeScene.update(delta_time); - //this.orbit.update(0.1); + //this.activeScene.update(delta_time) + //this.orbit.update(0.1) - //console.log(this.transform_engine.clips[this.test_box_uuid]); + //console.log(this.transform_engine.clips[this.test_box_uuid]) - this.timeline.update(delta_time); + this.timeline.update(delta_time) - this.render_scene(); + this.render_scene() if (this.capturer != null) { - this.capturer.capture(this.renderer.domElement); + this.capturer.capture(this.renderer.domElement) } // Record scene. } start_playback() { - this.timeline.isPlaying = true; + this.timeline.isPlaying = true } change_mode(type: any) { - if(this.control == undefined) {return;} - this.control.mode = type; - this.transform_interaction = true; + if(this.control == undefined) {return} + this.control.mode = type + this.transform_interaction = true } // Sets the fps to a specific number set_fps(fps_number: number) { - this.cap_fps = fps_number; + this.cap_fps = fps_number } // Toggles playback and recording. togglePlayback() { - this.togglePlay(); + this.togglePlay() if (this.playback == false) { - //this.stopPlayback(); - this.render_mode(); + //this.stopPlayback() + this.render_mode() } else { - //this.startPlayback(); - this.render_mode(); + //this.startPlayback() + this.render_mode() } } async loadWavAsBlob(url: string) { - const response = await fetch(url); - const blob = await response.blob(); - return blob; + const response = await fetch(url) + const blob = await response.blob() + return blob } async stopPlayback() { - this.render_mode(); - let ffmpeg = createFFmpeg({ log: true }); - await ffmpeg.load(); - for (let index = 0; index < this.frame_buffer.length; index++) { - const element = this.frame_buffer[index]; + this.render_mode() + let ffmpeg = createFFmpeg({ log: true }) + await ffmpeg.load() + for (let index = 0 index < this.frame_buffer.length index++) { + const element = this.frame_buffer[index] await ffmpeg.FS( "writeFile", `image${index}.png`, await fetchFile(element), - ); + ) } await ffmpeg.run( "-framerate", @@ -566,72 +571,72 @@ class Editor { "-i", "image%d.png", "output.mp4", - ); - let output = await ffmpeg.FS("readFile", "output.mp4"); + ) + let output = await ffmpeg.FS("readFile", "output.mp4") // Create a Blob from the output file for downloading - const blob = new Blob([output.buffer], { type: "video/mp4" }); - const url = URL.createObjectURL(blob); - await this.api_manager.uploadMedia(blob, "output.mp4"); + const blob = new Blob([output.buffer], { type: "video/mp4" }) + const url = URL.createObjectURL(blob) + await this.api_manager.uploadMedia(blob, "output.mp4") // Create a link to download the file - const downloadLink = document.createElement("a"); - downloadLink.href = url; - downloadLink.download = "output.mp4"; - document.body.appendChild(downloadLink); - downloadLink.click(); + const downloadLink = document.createElement("a") + downloadLink.href = url + downloadLink.download = "output.mp4" + document.body.appendChild(downloadLink) + downloadLink.click() // Clean up - URL.revokeObjectURL(url); - document.body.removeChild(downloadLink); + URL.revokeObjectURL(url) + document.body.removeChild(downloadLink) } startPlayback() { - this.playback_location = 0; - this._initialize_recording(); + this.playback_location = 0 + this._initialize_recording() } // Initializes transform x y z changes. _initialize_control() { - if(this.control == undefined) {return;} - this.control.addEventListener("change", this.render_scene.bind(this)); + if(this.control == undefined) {return} + this.control.addEventListener("change", this.render_scene.bind(this)) this.control.addEventListener( "dragging-changed", (event: any) => { - if(this.orbit == undefined) { return; } - this.orbit.enabled = !event.value; - // this.update_properties(); + if(this.orbit == undefined) { return } + this.orbit.enabled = !event.value + // this.update_properties() } - ); - this.control.setSize(0.5); // Good default value for visuals. - this.raycaster = new THREE.Raycaster(); - this.mouse = new THREE.Vector2(); - this.activeScene.scene.add(this.control); + ) + this.control.setSize(0.5) // Good default value for visuals. + this.raycaster = new THREE.Raycaster() + this.mouse = new THREE.Vector2() + this.activeScene.scene.add(this.control) } // Initializes CCapture for capturing the scene to send over to backend. _initialize_recording() { - this.frame_buffer = []; - this.render_timer = 0; + this.frame_buffer = [] + this.render_timer = 0 } // Render the scene to the camera. render_scene() { if (this.composer != null) { - this.composer.render(); + this.composer.render() } else if (this.renderer && this.camera) { - this.renderer.render(this.activeScene.scene, this.camera); + this.renderer.render(this.activeScene.scene, this.camera) } else { - console.error("Could not render to canvas no render or composer!"); + console.error("Could not render to canvas no render or composer!") } if (this.rendering && this.renderer && this.clock) { - this.playback_location++; - let imgData = this.renderer.domElement.toDataURL(); - this.frame_buffer.push(imgData); - this.render_timer += this.clock.getDelta(); + this.playback_location++ + let imgData = this.renderer.domElement.toDataURL() + this.frame_buffer.push(imgData) + this.render_timer += this.clock.getDelta() if (this.playback_location >= this.fps_number * 3) { - this.stopPlayback(); - this.playback_location = 0; - this.rendering = false; + this.stopPlayback() + this.playback_location = 0 + this.rendering = false } } } @@ -639,68 +644,68 @@ class Editor { // Automaticly resize scene. onWindowResize() { // Calculate the maximum possible dimensions while maintaining the aspect ratio - let width = window.innerWidth; // / aspect_adjust; - let height = window.innerHeight; // / aspectRatio; + let width = window.innerWidth // / aspect_adjust + let height = window.innerHeight // / aspectRatio - if(this.camera == undefined || this.renderer == undefined) { return; } + if(this.camera == undefined || this.renderer == undefined) { return } // Set the camera aspect to the desired aspect ratio - this.camera.aspect = width / height; - this.camera.updateProjectionMatrix(); + this.camera.aspect = width / height + this.camera.updateProjectionMatrix() // Set the renderer size to the calculated dimensions - this.renderer.setSize(width, height); + this.renderer.setSize(width, height) if (this.composer != null) { - this.composer.setSize(width, height); + this.composer.setSize(width, height) } } // Sets new mouse location usually used in raycasts. onMouseMove(event: any) { - const rect = this.canvReference.getBoundingClientRect(); - if(this.mouse == undefined) { return; } - this.mouse.x = ((event.clientX - rect.left) / rect.width) * 2 - 1; - this.mouse.y = -((event.clientY - rect.top) / rect.height) * 2 + 1; + const rect = this.canvReference.getBoundingClientRect() + if(this.mouse == undefined) { return } + this.mouse.x = ((event.clientX - rect.left) / rect.width) * 2 - 1 + this.mouse.y = -((event.clientY - rect.top) / rect.height) * 2 + 1 } // When the mouse clicks the screen. onMouseClick() { - if(this.raycaster == undefined || this.mouse == undefined || this.control == undefined || this.outlinePass == undefined) { return; } - this.raycaster.setFromCamera(this.mouse, this.camera); - let interactable: any[] = []; + if(this.raycaster == undefined || this.mouse == undefined || this.control == undefined || this.outlinePass == undefined) { return } + this.raycaster.setFromCamera(this.mouse, this.camera) + let interactable: any[] = [] this.activeScene.scene.children.forEach((child: THREE.Object3D) => { if (child.name != "") { if (child.type == "Mesh" || child.type == "Object3D") { - interactable.push(child); + interactable.push(child) } } - }); - let intersects = this.raycaster.intersectObjects(interactable, true); + }) + let intersects = this.raycaster.intersectObjects(interactable, true) if (intersects.length > 0) { if (intersects[0].object.type != "GridHelper") { - let currentObject = intersects[0].object; + let currentObject = intersects[0].object while (currentObject.parent && currentObject.parent.type !== "Scene") { - currentObject = currentObject.parent; + currentObject = currentObject.parent } - this.selected = currentObject; + this.selected = currentObject if (this.selected.type == "Scene") { - this.selected = intersects[0].object; + this.selected = intersects[0].object } - // this.update_properties(); - this.activeScene.scene.add(this.control); - this.control.attach(this.selected); - this.outlinePass.selectedObjects = [this.selected]; - this.transform_interaction = true; + // this.update_properties() + this.activeScene.scene.add(this.control) + this.control.attach(this.selected) + this.outlinePass.selectedObjects = [this.selected] + this.transform_interaction = true } } else if (this.transform_interaction == false) { - this.last_selected = this.selected; - this.control.detach(); - this.activeScene.scene.remove(this.control); - this.outlinePass.selectedObjects = []; + this.last_selected = this.selected + this.control.detach() + this.activeScene.scene.remove(this.control) + this.outlinePass.selectedObjects = [] } else { - this.transform_interaction = false; + this.transform_interaction = false } } } -export default Editor; +export default Editor From 6eef4e4c65caeed3a9eac52d3db91a8bc101f8e0 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 27 Mar 2024 18:06:04 -0400 Subject: [PATCH 15/23] Fixed forloop --- app/pages/PageEnigma/js/editor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 32365a8360..3588fa90ac 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -557,7 +557,7 @@ class Editor { this.render_mode() let ffmpeg = createFFmpeg({ log: true }) await ffmpeg.load() - for (let index = 0 index < this.frame_buffer.length index++) { + for (let index = 0 ; index < this.frame_buffer.length; index++) { const element = this.frame_buffer[index] await ffmpeg.FS( "writeFile", From 103c484cd41d699ad07082e6661196c415560abe Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 27 Mar 2024 18:21:07 -0400 Subject: [PATCH 16/23] Dispatch needs to be fixed going through old code. --- app/pages/PageEnigma/contexts/EngineProvider.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/pages/PageEnigma/contexts/EngineProvider.tsx b/app/pages/PageEnigma/contexts/EngineProvider.tsx index c10b04eb6e..899de9ff72 100644 --- a/app/pages/PageEnigma/contexts/EngineProvider.tsx +++ b/app/pages/PageEnigma/contexts/EngineProvider.tsx @@ -33,7 +33,7 @@ export const EngineProvider = ({ children }: Props) => { }, []); useEffect(()=>{ - if (editor && editor.can_initailize && dispatchAppUiState!==null){ + if (editor && editor.can_initialize && dispatchAppUiState!==null){ console.log("initializing Editor"); editor.initialize({ dispatchAppUiState From e7df8148c3985a917a075db2d1cd02aedcae9761 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 27 Mar 2024 18:23:12 -0400 Subject: [PATCH 17/23] Editor shows up again no lightning though. --- app/pages/PageEnigma/js/editor.ts | 664 ++++++++++++++++-------------- 1 file changed, 352 insertions(+), 312 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 3588fa90ac..509c63df2e 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -1,147 +1,156 @@ -import * as THREE from "three" -import { OrbitControls } from "three/addons/controls/OrbitControls.js" -import { TransformControls } from "three/addons/controls/TransformControls.js" -import Scene from "./scene.js" -import APIManager from "./api_manager.js" -import { EffectComposer } from "three/addons/postprocessing/EffectComposer.js" -import { RenderPass } from "three/addons/postprocessing/RenderPass.js" -import { OutlinePass } from "three/addons/postprocessing/OutlinePass.js" -import { OutputPass } from "three/addons/postprocessing/OutputPass.js" -import { SMAAPass } from "three/addons/postprocessing/SMAAPass.js" -import { SAOPass } from "three/addons/postprocessing/SAOPass.js" -import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js" -import { BokehPass } from "three/addons/postprocessing/BokehPass.js" -import { createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg" -import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js" +import * as THREE from "three"; +import { OrbitControls } from "three/addons/controls/OrbitControls.js"; +import { TransformControls } from "three/addons/controls/TransformControls.js"; +import Scene from "./scene.js"; +import APIManager from "./api_manager.js"; +import { EffectComposer } from "three/addons/postprocessing/EffectComposer.js"; +import { RenderPass } from "three/addons/postprocessing/RenderPass.js"; +import { OutlinePass } from "three/addons/postprocessing/OutlinePass.js"; +import { OutputPass } from "three/addons/postprocessing/OutputPass.js"; +import { SMAAPass } from "three/addons/postprocessing/SMAAPass.js"; +import { SAOPass } from "three/addons/postprocessing/SAOPass.js"; +import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js"; +import { BokehPass } from "three/addons/postprocessing/BokehPass.js"; +import { createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg"; +import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; -import AudioEngine from "./audio_engine.js" -import TransformEngine from "./transform_engine.js" -import { TimeLine, TimelineDataState } from "./timeline.js" -import { ClipUI } from "../datastructures/clips/clip_offset.js" +import AudioEngine from "./audio_engine.js"; +import TransformEngine from "./transform_engine.js"; +import { TimeLine, TimelineDataState } from "./timeline.js"; +import { ClipUI } from "../datastructures/clips/clip_offset.js"; -import { LipSync } from "./lipsync.js" -import { LipSyncEngine } from "./lip_sync_engine.js" -import { AnimationEngine } from "./animation_engine.js" -import { faL } from "@fortawesome/pro-solid-svg-icons" +import { LipSync } from "./lipsync.js"; +import { LipSyncEngine } from "./lip_sync_engine.js"; +import { AnimationEngine } from "./animation_engine.js"; +import { faL } from "@fortawesome/pro-solid-svg-icons"; class EditorState { - // { // action: "ShowLoadingIndicator" - // source: "Editor" + // source: "Editor" // data: { "message" : "saving scene" } // } - selected_object: THREE.Object3D | undefined - is_loading: boolean + selected_object: THREE.Object3D | undefined; + is_loading: boolean; constructor() { - this.selected_object - this.is_loading = false + this.selected_object; + this.is_loading = false; } } // Main editor class that will call everything else all you need to call is " initialize() ". class Editor { - version: number - activeScene: Scene - camera: any - renderer: THREE.WebGLRenderer | undefined - clock: THREE.Clock | undefined - canvReference: any - composer: EffectComposer | undefined - effectFXAA: EffectComposer | undefined - outlinePass: OutlinePass | undefined - last_cam_pos: THREE.Vector3 | undefined - saoPass: SAOPass | undefined - outputPass: OutlinePass | undefined - bloomPass: UnrealBloomPass | undefined - smaaPass: SMAAPass | undefined - bokehPass: BokehPass | undefined - control: TransformControls | undefined - raycaster: THREE.Raycaster | undefined - mouse: THREE.Vector2 | undefined - selected: THREE.Object3D | undefined - last_selected: THREE.Object3D | undefined - transform_interaction: any - rendering: boolean - api_manager: APIManager - stats: any - orbit: OrbitControls | undefined - locked: boolean - capturer: any - frame_buffer: any - render_timer: number - fps_number: number - cap_fps: number - playback: boolean - playback_location: number - max_length: number - audio_engine: AudioEngine - transform_engine: TransformEngine - lipsync_engine: LipSyncEngine - animation_engine: AnimationEngine - timeline: TimeLine - current_frame: number - test_scene_load_media_id: string - scene_file_token: any - renderPass: RenderPass | undefined - - current_scene_token: string | null + version: number; + activeScene: Scene; + camera: any; + renderer: THREE.WebGLRenderer | undefined; + clock: THREE.Clock | undefined; + canvReference: any; + composer: EffectComposer | undefined; + effectFXAA: EffectComposer | undefined; + outlinePass: OutlinePass | undefined; + last_cam_pos: THREE.Vector3 | undefined; + saoPass: SAOPass | undefined; + outputPass: OutlinePass | undefined; + bloomPass: UnrealBloomPass | undefined; + smaaPass: SMAAPass | undefined; + bokehPass: BokehPass | undefined; + control: TransformControls | undefined; + raycaster: THREE.Raycaster | undefined; + mouse: THREE.Vector2 | undefined; + selected: THREE.Object3D | undefined; + last_selected: THREE.Object3D | undefined; + transform_interaction: any; + rendering: boolean; + api_manager: APIManager; + stats: any; + orbit: OrbitControls | undefined; + locked: boolean; + capturer: any; + frame_buffer: any; + render_timer: number; + fps_number: number; + cap_fps: number; + playback: boolean; + playback_location: number; + max_length: number; + audio_engine: AudioEngine; + transform_engine: TransformEngine; + lipsync_engine: LipSyncEngine; + animation_engine: AnimationEngine; + timeline: TimeLine; + current_frame: number; + test_scene_load_media_id: string; + scene_file_token: any; + renderPass: RenderPass | undefined; + current_scene_token: string | null; + can_initialize: boolean; // Default params. constructor() { console.log( "If you see this message twice! then it rendered twice, if you see it once it's all good.", - ) + ); + // Special fix + // For making sure the editor only gets created onece. + this.can_initialize = false; + let one_element = document.getElementById("created-one-element"); + //if (one_element != null) { return; } + this.can_initialize = true; + let newElement = document.createElement("div"); + newElement.id = "created-one-element"; + document.body.appendChild(newElement); + // life cycle fix // Version and name. - this.version = 0.1 + this.version = 0.1; // Clock, scene and camera essentials. - this.activeScene = new Scene("" + this.version) - this.activeScene.initialize() - this.camera - this.renderer - this.clock - this.canvReference = null - this.composer - this.effectFXAA - this.outlinePass - this.last_cam_pos - this.saoPass - this.outputPass - this.bloomPass - this.smaaPass - this.bokehPass + this.activeScene = new Scene("" + this.version); + this.activeScene.initialize(); + this.camera; + this.renderer; + this.clock; + this.canvReference = null; + this.composer; + this.effectFXAA; + this.outlinePass; + this.last_cam_pos; + this.saoPass; + this.outputPass; + this.bloomPass; + this.smaaPass; + this.bokehPass; // Transform control and selection. - this.control - this.raycaster - this.mouse - this.selected - this.last_selected - this.transform_interaction - this.rendering = false + this.control; + this.raycaster; + this.mouse; + this.selected; + this.last_selected; + this.transform_interaction; + this.rendering = false; // API. - this.api_manager = new APIManager() + this.api_manager = new APIManager(); // Debug & Movement. - this.stats = null - this.orbit - this.locked = false + this.stats = null; + this.orbit; + this.locked = false; // Recording params. - this.capturer = null - this.frame_buffer = [] - this.render_timer = 0 - this.fps_number = 60 - this.cap_fps = 60 + this.capturer = null; + this.frame_buffer = []; + this.render_timer = 0; + this.fps_number = 60; + this.cap_fps = 60; // Timeline settings. - this.playback = false - this.playback_location = 0 - this.max_length = 10 + this.playback = false; + this.playback_location = 0; + this.max_length = 10; // Audio Engine Test. - this.audio_engine = new AudioEngine() - this.transform_engine = new TransformEngine(this.version) - this.lipsync_engine = new LipSyncEngine() - this.animation_engine = new AnimationEngine(this.version) + this.audio_engine = new AudioEngine(); + this.transform_engine = new TransformEngine(this.version); + this.lipsync_engine = new LipSyncEngine(); + this.animation_engine = new AnimationEngine(this.version); this.timeline = new TimeLine( this.audio_engine, @@ -149,89 +158,101 @@ class Editor { this.lipsync_engine, this.animation_engine, this.activeScene, - ) + ); - this.current_frame = 0 + this.current_frame = 0; - this.test_scene_load_media_id = "" // Test media id for saving and loading. + this.test_scene_load_media_id = ""; // Test media id for saving and loading. - this.current_scene_token = null + this.current_scene_token = null; } // Initializes the main scene and ThreeJS essentials. - initialize() { + + initialize(config:any) { + console.log( + "Calling Initialize", + ); + if (this.can_initialize == false) { + return; + } + this.can_initialize = false; + // Gets the canvas. - this.canvReference = document.getElementById("video-scene") + this.canvReference = document.getElementById("video-scene"); // Base width and height. - let width = this.canvReference.width - let height = this.canvReference.height + let width = this.canvReference.width; + let height = this.canvReference.height; // Sets up camera and base position. - this.camera = new THREE.PerspectiveCamera(70, width / height, 0.1, 50) - this.camera.position.z = 3 - this.camera.position.y = 3 - this.camera.position.x = -3 + this.camera = new THREE.PerspectiveCamera(70, width / height, 0.1, 50); + this.camera.position.z = 3; + this.camera.position.y = 3; + this.camera.position.x = -3; // Base WebGL render and clock for delta time. this.renderer = new THREE.WebGLRenderer({ antialias: true, canvas: this.canvReference, preserveDrawingBuffer: true, - }) - this.renderer.shadowMap.enabled = true - this.clock = new THREE.Clock() + }); + this.renderer.shadowMap.enabled = true; + this.clock = new THREE.Clock(); // Resizes the renderer. - this.renderer.setSize(width, height) + this.renderer.setSize(width, height); //document.body.appendChild(this.renderer.domElement) - window.addEventListener("resize", this.onWindowResize.bind(this)) - this._configure_post_pro() + window.addEventListener("resize", this.onWindowResize.bind(this)); + this._configure_post_pro(); // Controls and movement. - this.orbit = new OrbitControls(this.camera, this.renderer.domElement) - this.control = new TransformControls(this.camera, this.renderer.domElement) + this.orbit = new OrbitControls(this.camera, this.renderer.domElement); + this.control = new TransformControls(this.camera, this.renderer.domElement); // OnClick and MouseMove events. - window.addEventListener("mousemove", this.onMouseMove.bind(this), false) - window.addEventListener("click", this.onMouseClick.bind(this), false) + window.addEventListener("mousemove", this.onMouseMove.bind(this), false); + window.addEventListener("click", this.onMouseClick.bind(this), false); // Base control and debug stuff remove debug in prod. - this._initialize_control() + this._initialize_control(); // Resets canvas size. - this.onWindowResize() + this.onWindowResize(); // Creates the main update loop. - this.renderer.setAnimationLoop(this.update_loop.bind(this)) + this.renderer.setAnimationLoop(this.update_loop.bind(this)); - this.timeline.scene = this.activeScene + this.timeline.scene = this.activeScene; - this._test_demo() + this._test_demo(); - // saving state of the scene - this.current_scene_token = null + // saving state of the scene + this.current_scene_token = null; } // Token comes in from the front end to load the scene from the site. - public async loadScene(scene_token:string) { + public async loadScene(scene_token: string) { if (scene_token != null) { - this.current_scene_token = scene_token - } - const scene = await this.api_manager.loadSceneState(this.current_scene_token) - this.activeScene.scene.children = scene.children + this.current_scene_token = scene_token; + } + const scene = await this.api_manager.loadSceneState( + this.current_scene_token, + ); + this.activeScene.scene.children = scene.children; this.activeScene.scene.children.forEach((child: THREE.Object3D) => { - - child.parent = this.activeScene.scene + child.parent = this.activeScene.scene; if (child.type == "DirectionalLight") { - let pos = child.position - let rot = child.rotation - let light = this.activeScene._create_base_lighting() - light.position.set(pos.x, pos.y, pos.z) - light.rotation.set(rot.x, rot.y, rot.z) - this.activeScene.scene.remove(child) + let pos = child.position; + let rot = child.rotation; + let light = this.activeScene._create_base_lighting(); + light.position.set(pos.x, pos.y, pos.z); + light.rotation.set(rot.x, rot.y, rot.z); + this.activeScene.scene.remove(child); } - }) - + }); } public async saveScene(name: string) { - const result = await this.api_manager.saveSceneState(this.activeScene.scene, name, + const result = await this.api_manager.saveSceneState( + this.activeScene.scene, + name, this.scene_file_token, - new TimelineDataState()) + new TimelineDataState(), + ); // dispatch call wil's engine. } @@ -247,12 +268,17 @@ class Editor { // let result = await this.api_manager.getMediaFile( // "m_189p8hj0eyypbg74kkhcpehwpjhnkz", // ) - console.log("Saving...") - const result = await this.api_manager.saveSceneState(this.activeScene.scene, "", null, null) - console.log("Saved!") - console.log("Media ID is:", result) + console.log("Saving..."); + const result = await this.api_manager.saveSceneState( + this.activeScene.scene, + "", + null, + null, + ); + console.log("Saved!"); + console.log("Media ID is:", result); if (result.data) { - this.test_scene_load_media_id = result.data["media_file_token"] + this.test_scene_load_media_id = result.data["media_file_token"]; } } @@ -281,13 +307,13 @@ class Editor { // Test code here let object: any = await this.activeScene.load_glb( "./resources/models/fox/fox.glb", - ) + ); // load object into the engine for lip syncing this.lipsync_engine.load_object( object.uuid, "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", - ) + ); // create the clip with the same id for a reference to the media this.timeline.addPlayableClip( @@ -300,12 +326,12 @@ class Editor { 150, 400, ), - ) + ); // media id for this is up in the air but when a path is created you should be able to store and delete it this.timeline.addPlayableClip( new ClipUI(1.0, "transform", "clip2", object.uuid, object.uuid, 0, 150), - ) + ); // media id for this as well it can be downloaded this.timeline.addPlayableClip( @@ -318,12 +344,12 @@ class Editor { 0, 400, ), - ) + ); this.animation_engine.load_object( object.uuid, "/resources/models/fox/fox_idle.glb", "clip3", - ) + ); } async _test_demo() { @@ -331,20 +357,20 @@ class Editor { // Test code here let object: any = await this.activeScene.load_glb( "./resources/models/fox/fox.glb", - ) + ); // Load timeline creates the the clips from the datastructure and loads them in here. // load object into the engine for lip syncing this.lipsync_engine.load_object( object.uuid, "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", - ) + ); this.animation_engine.load_object( object.uuid, "/resources/models/fox/fox_idle.glb", "clip3", - ) - // then it creates clip ui to load the playable clips + ); + // then it creates clip ui to load the playable clips // then refreshes the timeline. // create the clip with the same id for a reference to the media @@ -358,12 +384,12 @@ class Editor { 150, 400, ), - ) + ); // media id for this is up in the air but when a path is created you should be able to store and delete it this.timeline.addPlayableClip( new ClipUI(1.0, "transform", "clip2", object.uuid, object.uuid, 0, 150), - ) + ); // media id for this as well it can be downloaded this.timeline.addPlayableClip( @@ -376,81 +402,80 @@ class Editor { 0, 400, ), - ) - + ); } // Configure post processing. _configure_post_pro() { - let width = this.canvReference.width - let height = this.canvReference.height + let width = this.canvReference.width; + let height = this.canvReference.height; - if(this.renderer == undefined || this.camera == undefined) { - return + if (this.renderer == undefined || this.camera == undefined) { + return; } - this.composer = new EffectComposer(this.renderer) - this.renderPass = new RenderPass(this.activeScene.scene, this.camera) - this.composer.addPass(this.renderPass) + this.composer = new EffectComposer(this.renderer); + this.renderPass = new RenderPass(this.activeScene.scene, this.camera); + this.composer.addPass(this.renderPass); this.outlinePass = new OutlinePass( new THREE.Vector2(width, height), this.activeScene.scene, this.camera, - ) + ); - this.outlinePass.edgeStrength = 6.0 - this.outlinePass.edgeGlow = 0.2 - this.outlinePass.edgeThickness = 1.0 - this.outlinePass.pulsePeriod = 3 - this.outlinePass.usePatternTexture = false - this.outlinePass.visibleEdgeColor.set(0xe66462) + this.outlinePass.edgeStrength = 6.0; + this.outlinePass.edgeGlow = 0.2; + this.outlinePass.edgeThickness = 1.0; + this.outlinePass.pulsePeriod = 3; + this.outlinePass.usePatternTexture = false; + this.outlinePass.visibleEdgeColor.set(0xe66462); - this.composer.addPass(this.outlinePass) + this.composer.addPass(this.outlinePass); - this.saoPass = new SAOPass(this.activeScene.scene, this.camera) + this.saoPass = new SAOPass(this.activeScene.scene, this.camera); - this.saoPass.params.saoBias = 3.1 - this.saoPass.params.saoIntensity = 1.0 - this.saoPass.params.saoScale = 6.0 - this.saoPass.params.saoKernelRadius = 5.0 - this.saoPass.params.saoMinResolution = 0.0 + this.saoPass.params.saoBias = 3.1; + this.saoPass.params.saoIntensity = 1.0; + this.saoPass.params.saoScale = 6.0; + this.saoPass.params.saoKernelRadius = 5.0; + this.saoPass.params.saoMinResolution = 0.0; this.bloomPass = new UnrealBloomPass( new THREE.Vector2(width, height), 1.5, 0.4, 0.85, - ) - this.bloomPass.strength = 0.25 + ); + this.bloomPass.strength = 0.25; this.smaaPass = new SMAAPass( width * this.renderer.getPixelRatio(), height * this.renderer.getPixelRatio(), - ) + ); this.bokehPass = new BokehPass(this.activeScene.scene, this.camera, { focus: 3.0, aperture: 0.00001, maxblur: 0.01, - }) + }); - this.composer.addPass(this.saoPass) - this.composer.addPass(this.bloomPass) - this.composer.addPass(this.smaaPass) - this.composer.addPass(this.bokehPass) + this.composer.addPass(this.saoPass); + this.composer.addPass(this.bloomPass); + this.composer.addPass(this.smaaPass); + this.composer.addPass(this.bokehPass); //this.outputPass = new OutputPass() //this.composer.addPass(this.outputPass) } create_parim(name: string) { - let uuid = this.activeScene.instantiate(name) + let uuid = this.activeScene.instantiate(name); } render_mode() { - this.rendering = !this.rendering - this.activeScene.render_mode(this.rendering) + this.rendering = !this.rendering; + this.activeScene.render_mode(this.rendering); //if (this.rendering) { // this._remove_post_processing() @@ -460,10 +485,10 @@ class Editor { } togglePlay() { - this.playback = !this.playback - this.playback_location = 0 + this.playback = !this.playback; + this.playback_location = 0; if (this.playback == false) { - this.stopPlayback() + this.stopPlayback(); } else { // REMOVE THIS NEXT LINE IN PROD ONLY FOR TEST!!!!! //this.activeScene.play_anim_demo(this.activeScene.activeCharacter) @@ -481,10 +506,10 @@ class Editor { //console.log(this.selected) if (this.selected == null) { - return + return; } - this.transform_engine.addFrame(this.selected) - console.log("Frame taken.") + this.transform_engine.addFrame(this.selected); + console.log("Frame taken."); } change_camera_view() { @@ -499,12 +524,14 @@ class Editor { update_loop(time: number) { // Updates debug stats. if (this.stats != null) { - this.stats.update() + this.stats.update(); } - if(this.clock == undefined || this.renderer == undefined) {return} + if (this.clock == undefined || this.renderer == undefined) { + return; + } - let delta_time = this.clock.getDelta() + let delta_time = this.clock.getDelta(); // All calls that are not super important like timeline go here. //this.activeScene.update(delta_time) @@ -512,58 +539,60 @@ class Editor { //console.log(this.transform_engine.clips[this.test_box_uuid]) - this.timeline.update(delta_time) + this.timeline.update(delta_time); - this.render_scene() + this.render_scene(); if (this.capturer != null) { - this.capturer.capture(this.renderer.domElement) + this.capturer.capture(this.renderer.domElement); } // Record scene. } start_playback() { - this.timeline.isPlaying = true + this.timeline.isPlaying = true; } change_mode(type: any) { - if(this.control == undefined) {return} - this.control.mode = type - this.transform_interaction = true + if (this.control == undefined) { + return; + } + this.control.mode = type; + this.transform_interaction = true; } // Sets the fps to a specific number set_fps(fps_number: number) { - this.cap_fps = fps_number + this.cap_fps = fps_number; } // Toggles playback and recording. togglePlayback() { - this.togglePlay() + this.togglePlay(); if (this.playback == false) { //this.stopPlayback() - this.render_mode() + this.render_mode(); } else { //this.startPlayback() - this.render_mode() + this.render_mode(); } } async loadWavAsBlob(url: string) { - const response = await fetch(url) - const blob = await response.blob() - return blob + const response = await fetch(url); + const blob = await response.blob(); + return blob; } async stopPlayback() { - this.render_mode() - let ffmpeg = createFFmpeg({ log: true }) - await ffmpeg.load() - for (let index = 0 ; index < this.frame_buffer.length; index++) { - const element = this.frame_buffer[index] + this.render_mode(); + let ffmpeg = createFFmpeg({ log: true }); + await ffmpeg.load(); + for (let index = 0; index < this.frame_buffer.length; index++) { + const element = this.frame_buffer[index]; await ffmpeg.FS( "writeFile", `image${index}.png`, await fetchFile(element), - ) + ); } await ffmpeg.run( "-framerate", @@ -571,72 +600,72 @@ class Editor { "-i", "image%d.png", "output.mp4", - ) - let output = await ffmpeg.FS("readFile", "output.mp4") + ); + let output = await ffmpeg.FS("readFile", "output.mp4"); // Create a Blob from the output file for downloading - const blob = new Blob([output.buffer], { type: "video/mp4" }) - const url = URL.createObjectURL(blob) - await this.api_manager.uploadMedia(blob, "output.mp4") + const blob = new Blob([output.buffer], { type: "video/mp4" }); + const url = URL.createObjectURL(blob); + await this.api_manager.uploadMedia(blob, "output.mp4"); // Create a link to download the file - const downloadLink = document.createElement("a") - downloadLink.href = url - downloadLink.download = "output.mp4" - document.body.appendChild(downloadLink) - downloadLink.click() + const downloadLink = document.createElement("a"); + downloadLink.href = url; + downloadLink.download = "output.mp4"; + document.body.appendChild(downloadLink); + downloadLink.click(); // Clean up - URL.revokeObjectURL(url) - document.body.removeChild(downloadLink) + URL.revokeObjectURL(url); + document.body.removeChild(downloadLink); } startPlayback() { - this.playback_location = 0 - this._initialize_recording() + this.playback_location = 0; + this._initialize_recording(); } // Initializes transform x y z changes. _initialize_control() { - if(this.control == undefined) {return} - this.control.addEventListener("change", this.render_scene.bind(this)) - this.control.addEventListener( - "dragging-changed", - (event: any) => { - if(this.orbit == undefined) { return } - this.orbit.enabled = !event.value - // this.update_properties() + if (this.control == undefined) { + return; + } + this.control.addEventListener("change", this.render_scene.bind(this)); + this.control.addEventListener("dragging-changed", (event: any) => { + if (this.orbit == undefined) { + return; } - ) - this.control.setSize(0.5) // Good default value for visuals. - this.raycaster = new THREE.Raycaster() - this.mouse = new THREE.Vector2() - this.activeScene.scene.add(this.control) + this.orbit.enabled = !event.value; + // this.update_properties() + }); + this.control.setSize(0.5); // Good default value for visuals. + this.raycaster = new THREE.Raycaster(); + this.mouse = new THREE.Vector2(); + this.activeScene.scene.add(this.control); } // Initializes CCapture for capturing the scene to send over to backend. _initialize_recording() { - this.frame_buffer = [] - this.render_timer = 0 + this.frame_buffer = []; + this.render_timer = 0; } // Render the scene to the camera. render_scene() { - if (this.composer != null) { - this.composer.render() + this.composer.render(); } else if (this.renderer && this.camera) { - this.renderer.render(this.activeScene.scene, this.camera) + this.renderer.render(this.activeScene.scene, this.camera); } else { - console.error("Could not render to canvas no render or composer!") + console.error("Could not render to canvas no render or composer!"); } if (this.rendering && this.renderer && this.clock) { - this.playback_location++ - let imgData = this.renderer.domElement.toDataURL() - this.frame_buffer.push(imgData) - this.render_timer += this.clock.getDelta() + this.playback_location++; + let imgData = this.renderer.domElement.toDataURL(); + this.frame_buffer.push(imgData); + this.render_timer += this.clock.getDelta(); if (this.playback_location >= this.fps_number * 3) { - this.stopPlayback() - this.playback_location = 0 - this.rendering = false + this.stopPlayback(); + this.playback_location = 0; + this.rendering = false; } } } @@ -644,68 +673,79 @@ class Editor { // Automaticly resize scene. onWindowResize() { // Calculate the maximum possible dimensions while maintaining the aspect ratio - let width = window.innerWidth // / aspect_adjust - let height = window.innerHeight // / aspectRatio + let width = window.innerWidth; // / aspect_adjust + let height = window.innerHeight; // / aspectRatio - if(this.camera == undefined || this.renderer == undefined) { return } + if (this.camera == undefined || this.renderer == undefined) { + return; + } // Set the camera aspect to the desired aspect ratio - this.camera.aspect = width / height - this.camera.updateProjectionMatrix() + this.camera.aspect = width / height; + this.camera.updateProjectionMatrix(); // Set the renderer size to the calculated dimensions - this.renderer.setSize(width, height) + this.renderer.setSize(width, height); if (this.composer != null) { - this.composer.setSize(width, height) + this.composer.setSize(width, height); } } // Sets new mouse location usually used in raycasts. onMouseMove(event: any) { - const rect = this.canvReference.getBoundingClientRect() - if(this.mouse == undefined) { return } - this.mouse.x = ((event.clientX - rect.left) / rect.width) * 2 - 1 - this.mouse.y = -((event.clientY - rect.top) / rect.height) * 2 + 1 + const rect = this.canvReference.getBoundingClientRect(); + if (this.mouse == undefined) { + return; + } + this.mouse.x = ((event.clientX - rect.left) / rect.width) * 2 - 1; + this.mouse.y = -((event.clientY - rect.top) / rect.height) * 2 + 1; } // When the mouse clicks the screen. onMouseClick() { - if(this.raycaster == undefined || this.mouse == undefined || this.control == undefined || this.outlinePass == undefined) { return } - this.raycaster.setFromCamera(this.mouse, this.camera) - let interactable: any[] = [] + if ( + this.raycaster == undefined || + this.mouse == undefined || + this.control == undefined || + this.outlinePass == undefined + ) { + return; + } + this.raycaster.setFromCamera(this.mouse, this.camera); + let interactable: any[] = []; this.activeScene.scene.children.forEach((child: THREE.Object3D) => { if (child.name != "") { if (child.type == "Mesh" || child.type == "Object3D") { - interactable.push(child) + interactable.push(child); } } - }) - let intersects = this.raycaster.intersectObjects(interactable, true) + }); + let intersects = this.raycaster.intersectObjects(interactable, true); if (intersects.length > 0) { if (intersects[0].object.type != "GridHelper") { - let currentObject = intersects[0].object + let currentObject = intersects[0].object; while (currentObject.parent && currentObject.parent.type !== "Scene") { - currentObject = currentObject.parent - } - this.selected = currentObject + currentObject = currentObject.parent; + } + this.selected = currentObject; if (this.selected.type == "Scene") { - this.selected = intersects[0].object + this.selected = intersects[0].object; } // this.update_properties() - this.activeScene.scene.add(this.control) - this.control.attach(this.selected) - this.outlinePass.selectedObjects = [this.selected] - this.transform_interaction = true + this.activeScene.scene.add(this.control); + this.control.attach(this.selected); + this.outlinePass.selectedObjects = [this.selected]; + this.transform_interaction = true; } } else if (this.transform_interaction == false) { - this.last_selected = this.selected - this.control.detach() - this.activeScene.scene.remove(this.control) - this.outlinePass.selectedObjects = [] + this.last_selected = this.selected; + this.control.detach(); + this.activeScene.scene.remove(this.control); + this.outlinePass.selectedObjects = []; } else { - this.transform_interaction = false + this.transform_interaction = false; } } } -export default Editor +export default Editor; From 9d78b550b809fa216f017d7175bf4b9a4a4f03a8 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 27 Mar 2024 18:35:54 -0400 Subject: [PATCH 18/23] Moved to type script for the main editor --- app/pages/PageEnigma/js/api_manager.ts | 2 +- app/pages/PageEnigma/js/editor.ts | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index 707dc9c19b..4fd15b2452 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -61,7 +61,7 @@ class APIManager { return new APIManagerResponseSuccess("Scene Saved") } - public async loadSceneState(scene_media_file_token: string): Promise { + public async loadSceneState(scene_media_file_token: string | null): Promise { const api_base_url = "https://api.fakeyou.com" const url = `${api_base_url}/v1/media_files/file/${scene_media_file_token}` const response = await fetch(url) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 509c63df2e..59673ba081 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -52,7 +52,7 @@ class Editor { outlinePass: OutlinePass | undefined; last_cam_pos: THREE.Vector3 | undefined; saoPass: SAOPass | undefined; - outputPass: OutlinePass | undefined; + outputPass: OutputPass | undefined; bloomPass: UnrealBloomPass | undefined; smaaPass: SMAAPass | undefined; bokehPass: BokehPass | undefined; @@ -170,10 +170,10 @@ class Editor { // Initializes the main scene and ThreeJS essentials. initialize(config:any) { - console.log( - "Calling Initialize", - ); if (this.can_initialize == false) { + console.log( + "Editor Already Initialized", + ); return; } this.can_initialize = false; @@ -194,6 +194,7 @@ class Editor { canvas: this.canvReference, preserveDrawingBuffer: true, }); + this.renderer.shadowMap.enabled = true; this.clock = new THREE.Clock(); // Resizes the renderer. @@ -465,8 +466,8 @@ class Editor { this.composer.addPass(this.smaaPass); this.composer.addPass(this.bokehPass); - //this.outputPass = new OutputPass() - //this.composer.addPass(this.outputPass) + this.outputPass = new OutputPass() + this.composer.addPass(this.outputPass) } create_parim(name: string) { From ff1581c9d1e0e7ce661c28a53d01725f1caee430 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 27 Mar 2024 19:05:30 -0400 Subject: [PATCH 19/23] Wiring in dispatch state --- app/pages/PageEnigma/js/editor.ts | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 59673ba081..749cf8bec7 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -12,17 +12,16 @@ import { SAOPass } from "three/addons/postprocessing/SAOPass.js"; import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js"; import { BokehPass } from "three/addons/postprocessing/BokehPass.js"; import { createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg"; -import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; import AudioEngine from "./audio_engine.js"; import TransformEngine from "./transform_engine.js"; import { TimeLine, TimelineDataState } from "./timeline.js"; import { ClipUI } from "../datastructures/clips/clip_offset.js"; -import { LipSync } from "./lipsync.js"; import { LipSyncEngine } from "./lip_sync_engine.js"; import { AnimationEngine } from "./animation_engine.js"; -import { faL } from "@fortawesome/pro-solid-svg-icons"; +import { Context } from "react"; + class EditorState { // { @@ -87,6 +86,8 @@ class Editor { current_scene_token: string | null; can_initialize: boolean; + + dispatchAppUiState: any; // todo figure out the type // Default params. constructor() { console.log( @@ -165,6 +166,9 @@ class Editor { this.test_scene_load_media_id = ""; // Test media id for saving and loading. this.current_scene_token = null; + + + this.dispatchAppUiState = null; } // Initializes the main scene and ThreeJS essentials. @@ -221,6 +225,9 @@ class Editor { // saving state of the scene this.current_scene_token = null; + + //setup reactland Callbacks + this.dispatchAppUiState = config.dispatchAppUiState } // Token comes in from the front end to load the scene from the site. @@ -254,6 +261,16 @@ class Editor { this.scene_file_token, new TimelineDataState(), ); + + this.dispatchAppUiState({ + type: ACTION_TYPES.SHOW_EDITOR_LOADER + }); + setInterval(()=> { + this.dispatchAppUiState({ + type: ACTION_TYPES.HIDE_EDITOR_LOADER + }); + },4000) + // dispatch call wil's engine. } From e62c0fdee36573d2fc59267e8dd7d0713c8cf348 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 27 Mar 2024 19:30:14 -0400 Subject: [PATCH 20/23] Wiring in the loading / saving screens now. --- app/pages/PageEnigma/js/api_manager.ts | 7 ++- app/pages/PageEnigma/js/editor.ts | 64 +++++++++++--------------- 2 files changed, 31 insertions(+), 40 deletions(-) diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index 4fd15b2452..e999caf033 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -1,5 +1,4 @@ import { v4 as uuidv4 } from "uuid"; -import { Scene } from "../datastructures/scene/scene_object"; import * as THREE from "three"; import { GLTFExporter } from "three/addons/exporters/GLTFExporter.js"; import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; @@ -13,11 +12,11 @@ import { TimelineDataState } from "./timeline"; /** * This is designed to surface user customer facing messages as errors. */ -type Tokens = { [key: string]: string }; +type Data = { [key: string]: string }; class APIManagerResponseSuccess { public user_message:String - public data:Tokens | null - constructor(user_message:String = "", data:Tokens | null = null) { + public data:Data | null + constructor(user_message:String = "", data:Data | null = null) { this.data = data this.user_message = user_message } diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 749cf8bec7..29d5239367 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -12,16 +12,13 @@ import { SAOPass } from "three/addons/postprocessing/SAOPass.js"; import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js"; import { BokehPass } from "three/addons/postprocessing/BokehPass.js"; import { createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg"; - import AudioEngine from "./audio_engine.js"; import TransformEngine from "./transform_engine.js"; import { TimeLine, TimelineDataState } from "./timeline.js"; import { ClipUI } from "../datastructures/clips/clip_offset.js"; - import { LipSyncEngine } from "./lip_sync_engine.js"; import { AnimationEngine } from "./animation_engine.js"; -import { Context } from "react"; - +import { ACTION_TYPES } from "../reducer"; class EditorState { // { @@ -80,24 +77,23 @@ class Editor { animation_engine: AnimationEngine; timeline: TimeLine; current_frame: number; - test_scene_load_media_id: string; - scene_file_token: any; + + renderPass: RenderPass | undefined; - current_scene_token: string | null; + current_scene_media_token_id: string | null; can_initialize: boolean; - dispatchAppUiState: any; // todo figure out the type // Default params. constructor() { console.log( "If you see this message twice! then it rendered twice, if you see it once it's all good.", ); - // Special fix + + // Special async react lifecycle fix // For making sure the editor only gets created onece. this.can_initialize = false; let one_element = document.getElementById("created-one-element"); - //if (one_element != null) { return; } this.can_initialize = true; let newElement = document.createElement("div"); newElement.id = "created-one-element"; @@ -105,7 +101,7 @@ class Editor { // life cycle fix // Version and name. - this.version = 0.1; + this.version = 1.0; // Clock, scene and camera essentials. this.activeScene = new Scene("" + this.version); this.activeScene.initialize(); @@ -162,17 +158,10 @@ class Editor { ); this.current_frame = 0; - - this.test_scene_load_media_id = ""; // Test media id for saving and loading. - - this.current_scene_token = null; - - + this.current_scene_media_token_id = null; this.dispatchAppUiState = null; } - // Initializes the main scene and ThreeJS essentials. - initialize(config:any) { if (this.can_initialize == false) { console.log( @@ -224,20 +213,21 @@ class Editor { this._test_demo(); // saving state of the scene - this.current_scene_token = null; - + this.current_scene_media_token_id = null; //setup reactland Callbacks this.dispatchAppUiState = config.dispatchAppUiState } // Token comes in from the front end to load the scene from the site. public async loadScene(scene_token: string) { + if (scene_token != null) { - this.current_scene_token = scene_token; + this.current_scene_media_token_id = scene_token; } const scene = await this.api_manager.loadSceneState( - this.current_scene_token, + this.current_scene_media_token_id, ); + this.activeScene.scene.children = scene.children; this.activeScene.scene.children.forEach((child: THREE.Object3D) => { @@ -255,23 +245,25 @@ class Editor { } public async saveScene(name: string) { - const result = await this.api_manager.saveSceneState( - this.activeScene.scene, - name, - this.scene_file_token, - new TimelineDataState(), - ); - this.dispatchAppUiState({ type: ACTION_TYPES.SHOW_EDITOR_LOADER }); - setInterval(()=> { - this.dispatchAppUiState({ - type: ACTION_TYPES.HIDE_EDITOR_LOADER - }); - },4000) - // dispatch call wil's engine. + const result = await this.api_manager.saveSceneState( + this.activeScene.scene, + name, + this.current_scene_media_token_id, + new TimelineDataState(), + ) + + const scene_media_token_id = result.data["scene_media_token_id"] + if (scene_media_token_id != null) { + this.current_scene_media_token_id = scene_media_token_id + } + + this.dispatchAppUiState({ + type: ACTION_TYPES.HIDE_EDITOR_LOADER + }); } // uploading some objects for testing to get their media ids from my account. From 58b023147f88145a5be7d4c31e4395776d637983 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 27 Mar 2024 20:29:59 -0400 Subject: [PATCH 21/23] Scene loading --- app/pages/PageEnigma/js/api_manager.ts | 247 ++++++++++++++----------- app/pages/PageEnigma/js/editor.ts | 22 ++- 2 files changed, 153 insertions(+), 116 deletions(-) diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index e999caf033..3ae7d150fb 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -12,24 +12,24 @@ import { TimelineDataState } from "./timeline"; /** * This is designed to surface user customer facing messages as errors. */ -type Data = { [key: string]: string }; +type Data = { [key: string]: any }; class APIManagerResponseSuccess { - public user_message:String - public data:Data | null - constructor(user_message:String = "", data:Data | null = null) { - this.data = data - this.user_message = user_message + public user_message: String; + public data: Data | null; + constructor(user_message: String = "", data: Data | null = null) { + this.data = data; + this.user_message = user_message; } } /** * This is designed to surface user customer facing messages as errors. - * Errors shouldn't be 404 or something confusing should be + * Errors shouldn't be 404 or something confusing should be */ class APIManagerResponseError extends Error { constructor(message?: string) { - super(message); - this.name = "APIManagerResponseError"; + super(message); + this.name = "APIManagerResponseError"; } } @@ -47,78 +47,97 @@ class APIManager { * @param scene_media_file_token If null we will we will create a new save or copy the scene, if provided we will overwrite the scene. * @returns APIManagerResponseMessage */ - public async saveSceneState(scene: THREE.Scene, - scene_name:string, - scene_media_file_token:string | null = null, - timeline_state:TimelineDataState | null):Promise { + public async saveSceneState( + scene: THREE.Scene, + scene_name: string, + scene_media_file_token: string | null = null, + timeline_state: TimelineDataState | null, + ): Promise { + const file = await this.gltfExport(scene); - const file = await this.gltfExport(scene) - console.log(`FILE: ${file}`) // will overwrite the scene on db if token exists - const upload_response = await this.uploadGLB(file,scene_media_file_token) - await this.saveSceneAndTimeLineState(scene_media_file_token,scene_name,timeline_state) - return new APIManagerResponseSuccess("Scene Saved") + const upload_glb_response = await this.uploadGLB( + file, + scene_media_file_token, + ); + const glb_media_file_token = upload_glb_response["media_file_token"]; + + // now write the scene + const save_scene_timeline_response = await this.saveSceneAndTimeLineState( + glb_media_file_token, + scene_name, + timeline_state, + ); + const result_scene_media_file_token = + save_scene_timeline_response["media_file_token"]; + + const data = { media_file_token: result_scene_media_file_token }; + return new APIManagerResponseSuccess("Scene Saved", data); } - public async loadSceneState(scene_media_file_token: string | null): Promise { - const api_base_url = "https://api.fakeyou.com" - const url = `${api_base_url}/v1/media_files/file/${scene_media_file_token}` - const response = await fetch(url) - const json = await JSON.parse(await response.text()) - const bucket_path = json["media_file"]["public_bucket_path"] - const media_base_url = "https://storage.googleapis.com/vocodes-public" - const media_url = `${media_base_url}${bucket_path}` // gets you a bucket path - - const file_response = await fetch(media_url) + public async loadSceneState( + scene_media_file_token: string | null, + ): Promise { + const api_base_url = "https://api.fakeyou.com"; + const url = `${api_base_url}/v1/media_files/file/${scene_media_file_token}`; + const response = await fetch(url); + const json = await JSON.parse(await response.text()); + const bucket_path = json["media_file"]["public_bucket_path"]; + const media_base_url = "https://storage.googleapis.com/vocodes-public"; + const media_url = `${media_base_url}${bucket_path}`; // gets you a bucket path + + const file_response = await fetch(media_url); if (!file_response.ok) { - throw new APIManagerResponseError("Failed to download file") + throw new APIManagerResponseError("Failed to download file"); } - // Convert the response to a Blob - let blob = await file_response.blob() - const json_result:string = await new Promise((resolve, reject) => { - let reader = new FileReader() - reader.onloadend = () => resolve(JSON.parse(reader.result as string)) - reader.onerror = reject - reader.readAsText(blob) - }) + // Convert the response from a blob to json text + let blob = await file_response.blob(); + const json_result: string = await new Promise((resolve, reject) => { + let reader = new FileReader(); + reader.onloadend = () => resolve(JSON.parse(reader.result as string)); + reader.onerror = reject; + reader.readAsText(blob); + }); - console.log(`JSON Result: ${json_result}`) + console.log(`loadSceneState : ${json_result}`); - const scene_glb_media_file_id:string = json_result["glb_media_file_id"] - const media_bucket_path = await this.getMediaFile(scene_glb_media_file_id) - let glbLoader = new GLTFLoader() + const scene_glb_media_file_id: string = json_result["glb_media_file_id"]; + const media_bucket_path = await this.getMediaFile(scene_glb_media_file_id); + let glbLoader = new GLTFLoader(); // promisify this const loadGlb = (bucket_path: string): Promise => { return new Promise((resolve, reject) => { - glbLoader.load(bucket_path, (glb) => { - if (glb) { - const scene:THREE.Scene = glb.scene - resolve(scene) - } else { - reject(new APIManagerResponseError('Failed to Load GLB Scene')) - } - }) - }) - } - loadGlb(media_bucket_path) - } + glbLoader.load(bucket_path, (glb) => { + if (glb) { + const scene: THREE.Scene = glb.scene; + const data = { "media_file_token": scene_glb_media_file_id , scene: scene }; + resolve(new APIManagerResponseSuccess("Success Loaded", data)); + } else { + reject(new APIManagerResponseError("Failed to Load GLB Scene")); + } + }); + }); + }; + + loadGlb(media_bucket_path); + } /** - * This recieves the bucket path of a media file - * @param media_file_token - * @returns + * This recieves the bucket path of a media file + * @param media_file_token + * @returns */ - private async getMediaFile(media_file_token: string): Promise { + private async getMediaFile(media_file_token: string): Promise { let api_base_url = "https://api.fakeyou.com"; - let url = `${api_base_url}/v1/media_files/file/${media_file_token}` + let url = `${api_base_url}/v1/media_files/file/${media_file_token}`; let response = await fetch(url); let json = await JSON.parse(await response.text()); let bucketPath = json["media_file"]["public_bucket_path"]; - let media_base_url = "https://storage.googleapis.com/vocodes-public" - let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path - return media_url + let media_base_url = "https://storage.googleapis.com/vocodes-public"; + let media_url = `${media_base_url}${bucketPath}`; // gets you a bucket path + return media_url; } /** @@ -126,7 +145,7 @@ class APIManager { It will also give the file a name which will be a uuidv4() @param scene The 3JS Scene we want to make a file to be uploaded as multipart form. */ - private async gltfExport(scene: THREE.Scene):Promise { + private async gltfExport(scene: THREE.Scene): Promise { const gltfExporter = new GLTFExporter(); const uuid = uuidv4(); const result = await gltfExporter.parseAsync(scene); @@ -135,8 +154,11 @@ class APIManager { }); return file; } - - private async uploadGLB(file: File, scene_media_file_token:string | null): Promise { + + private async uploadGLB( + file: File, + scene_media_file_token: string | null, + ): Promise { const url = `${this.baseUrl}/v1/media_files/write/engine_asset`; let uuid = uuidv4(); const form_data = new FormData(); @@ -146,7 +168,7 @@ class APIManager { if (scene_media_file_token != null) { form_data.append("media_file_token", scene_media_file_token); } - + form_data.append("file", file); form_data.append("source", "file"); form_data.append("media_file_subtype", "scene_import"); @@ -163,11 +185,11 @@ class APIManager { if (!response.ok) { throw new Error("Failed to Send Data"); + } else { + const json_data = await response.json(); + console.log(`uploadGLB: ${JSON.stringify(json_data)}`); + return json_data; // or handle the response as appropriate } - const json_data = await response.json() - // const json_string = JSON.stringify(json_data) - // console.log(`uploadGLB: ${json_string}`) - return json_data; // or handle the response as appropriate } private async saveSceneAndTimeLineState( @@ -175,22 +197,26 @@ class APIManager { scene_file_name: string, timeline_state: TimelineDataState | null, // only for now. ): Promise { - const url = `${this.baseUrl}/v1/media_files/write/scene_file` - let uuid = uuidv4() + const url = `${this.baseUrl}/v1/media_files/write/scene_file`; + let uuid = uuidv4(); // turn json into a blob - const scene_schema = {"glb_media_file_id":scene_media_file_token,"scene_name":scene_file_name,"timeline": { "objects":[] } } - const json = JSON.stringify(scene_schema) - const blob = new Blob([json], {type: 'application/json'}) - const file_name = `${uuid}.json` + const scene_schema = { + glb_media_file_id: scene_media_file_token, + scene_name: scene_file_name, + timeline: { objects: [] }, + }; + const json = JSON.stringify(scene_schema); + const blob = new Blob([json], { type: "application/json" }); + const file_name = `${uuid}.json`; - const form_data = new FormData() + const form_data = new FormData(); - form_data.append("uuid_idempotency_token", uuid) - form_data.append("file", blob, file_name) - form_data.append("source", "file") - form_data.append("type", "scene_json") - form_data.append("source", "file") + form_data.append("uuid_idempotency_token", uuid); + form_data.append("file", blob, file_name); + form_data.append("source", "file"); + form_data.append("type", "scene_json"); + form_data.append("source", "file"); const response = await fetch(url, { method: "POST", @@ -199,23 +225,26 @@ class APIManager { Accept: "application/json", }, body: form_data, - }) + }); if (!response.ok) { - throw new Error("Failed to Send Data") + throw new APIManagerResponseError("Failed to Send GLB Data"); + } else { + const json_data = await response.json(); + // example response {"success":true,"media_file_token":"m_r1ztnn501g2rn0vv2np08nd6zy2fvt"} + console.log(`saveSceneAndTimeLineState: ${JSON.stringify(json_data)}`); + // should return the same token if it is same file else new token. + return json_data; // or handle the response as appropriate } - const json_data = await response.json() - - // example response {"success":true,"media_file_token":"m_r1ztnn501g2rn0vv2np08nd6zy2fvt"} - console.log(`saveSceneAndTimeLineState: ${JSON.stringify(json_data)}`) - // should return the same token if it is same file else new token. - return json_data // or handle the response as appropriate } - public async uploadMedia(blob:any,fileName:string):Promise { + public async uploadMedia( + blob: any, + fileName: string, + ): Promise { const url = `${this.baseUrl}/v1/media_uploads/upload`; let uuid = uuidv4(); - + const formData = new FormData(); formData.append("uuid_idempotency_token", uuid); formData.append("file", blob, fileName); @@ -233,11 +262,11 @@ class APIManager { }); if (!response.ok) { - throw new APIManagerResponseError("Upload Media Failed to send file") - } - else { - const json_data = await response.json() - return new APIManagerResponseSuccess("Uploaded Media Success") + throw new APIManagerResponseError("Upload Media Failed to send file"); + } else { + const json_data = await response.json(); + console.log(`uploadMedia: ${JSON.stringify(json_data)}`); + return json_data; } } @@ -249,7 +278,7 @@ class APIManager { // it will always return a boolean success and the media_file_token // they take an optional media_file_token as a parameter for the file to overwrite, which must belong to the user. // If it's an anonymous user, we check the anonymous visitor token/cookie - + // async saveSceneState2( // scene: THREE.Scene, // timeline: TimelineDataState, @@ -257,7 +286,7 @@ class APIManager { // scene_media_file_token: string | null = null, // ): Promise { // // create new scene - + // let file = await this.gltfExport(scene); // let response = await this.uploadGLB(file); // let success = response["success"]; @@ -266,12 +295,11 @@ class APIManager { // console.log(`media file token: ${media_file_token}`) // const result = await this._saveSceneAndTimelineToJSONSpec(media_file_token,scene_name, timeline); // return result - // } - - + // } + // throw Error("Saving the Scene Resulted in an Error Success False"); // } - + // // For JSON files. // // Form-multipart POST with parameters: // // uuid_idempotency_token @@ -319,15 +347,15 @@ class APIManager { // } // // media_file_token that describes the scene - // async loadScene(media_file_token: string): Promise { + // async loadScene(media_file_token: string): Promise { // let api_base_url = "https://api.fakeyou.com"; // let url = `${api_base_url}/v1/media_files/file/${media_file_token}` // let response = await fetch(url); // let json = await JSON.parse(await response.text()); // let bucketPath = json["media_file"]["public_bucket_path"]; // let media_base_url = "https://storage.googleapis.com/vocodes-public" - // let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path - + // let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path + // let fileResponse = await fetch(media_url); // if (!fileResponse.ok) { @@ -343,22 +371,21 @@ class APIManager { // }); // console.log(json_value) // return json_value - // } + // } // // Provides bucket path to the actual file to download from the media file - // async getMediaFile(media_file_token: string): Promise { + // async getMediaFile(media_file_token: string): Promise { // let api_base_url = "https://api.fakeyou.com"; // let url = `${api_base_url}/v1/media_files/file/${media_file_token}` // let response = await fetch(url); // let json = await JSON.parse(await response.text()); // let bucketPath = json["media_file"]["public_bucket_path"]; // let media_base_url = "https://storage.googleapis.com/vocodes-public" - // let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path + // let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path // // TODO get it as binary. // return media_url // } - // async gltfExport2(scene: THREE.Scene) { // let gltfExporter = new GLTFExporter(); // let uuid = uuidv4(); @@ -391,7 +418,7 @@ class APIManager { // if (!response.ok) { // throw new Error("Failed to Send Data"); // } - // return response.json(); + // return response.json(); // } /** diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 29d5239367..bb8b4925e9 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -220,15 +220,21 @@ class Editor { // Token comes in from the front end to load the scene from the site. public async loadScene(scene_token: string) { + this.dispatchAppUiState({ + type: ACTION_TYPES.SHOW_EDITOR_LOADER + }); if (scene_token != null) { this.current_scene_media_token_id = scene_token; } - const scene = await this.api_manager.loadSceneState( + + const load_scene_state_response = await this.api_manager.loadSceneState( this.current_scene_media_token_id, ); - - this.activeScene.scene.children = scene.children; + + const loaded_scene = load_scene_state_response.data["scene"] + this.current_scene_media_token_id = load_scene_state_response.data["media_file_token"] + this.activeScene.scene.children = loaded_scene.children; this.activeScene.scene.children.forEach((child: THREE.Object3D) => { child.parent = this.activeScene.scene; @@ -242,6 +248,10 @@ class Editor { this.activeScene.scene.remove(child); } }); + + this.dispatchAppUiState({ + type: ACTION_TYPES.HIDE_EDITOR_LOADER + }); } public async saveScene(name: string) { @@ -287,9 +297,9 @@ class Editor { ); console.log("Saved!"); console.log("Media ID is:", result); - if (result.data) { - this.test_scene_load_media_id = result.data["media_file_token"]; - } + // if (result.data) { + // this.test_scene_load_media_id = result.data["media_file_token"]; + // } } //async _load_for_testing() { From b735b8c5e55a28e5716785e5fa13855f6b16335c Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 27 Mar 2024 21:21:52 -0400 Subject: [PATCH 22/23] Fixed saving and loading and removed controls. --- app/pages/PageEnigma/js/api_manager.ts | 17 +++++++++-------- app/pages/PageEnigma/js/editor.ts | 22 +++++++++++++++++----- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index 3ae7d150fb..7a5cd656d5 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -75,9 +75,8 @@ class APIManager { return new APIManagerResponseSuccess("Scene Saved", data); } - public async loadSceneState( - scene_media_file_token: string | null, - ): Promise { + + public async loadSceneState(scene_media_file_token: string | null): Promise { const api_base_url = "https://api.fakeyou.com"; const url = `${api_base_url}/v1/media_files/file/${scene_media_file_token}`; const response = await fetch(url); @@ -100,28 +99,30 @@ class APIManager { reader.readAsText(blob); }); - console.log(`loadSceneState : ${json_result}`); + console.log(`loadSceneState: ${JSON.stringify(json_result)}`); const scene_glb_media_file_id: string = json_result["glb_media_file_id"]; const media_bucket_path = await this.getMediaFile(scene_glb_media_file_id); + console.log(`GLB ${media_bucket_path}`) let glbLoader = new GLTFLoader(); // promisify this - const loadGlb = (bucket_path: string): Promise => { + const loadGlb = (bucket_path: string): Promise => { return new Promise((resolve, reject) => { glbLoader.load(bucket_path, (glb) => { if (glb) { const scene: THREE.Scene = glb.scene; - const data = { "media_file_token": scene_glb_media_file_id , scene: scene }; + const data = { "media_file_token": scene_glb_media_file_id , "scene": scene }; + console.log(`Data: ${data}`) resolve(new APIManagerResponseSuccess("Success Loaded", data)); } else { - reject(new APIManagerResponseError("Failed to Load GLB Scene")); + throw new APIManagerResponseError("Failed to Load GLB Scene"); } }); }); }; - loadGlb(media_bucket_path); + return await loadGlb(media_bucket_path); } /** diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index bb8b4925e9..b9a21e65ec 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -231,7 +231,9 @@ class Editor { const load_scene_state_response = await this.api_manager.loadSceneState( this.current_scene_media_token_id, ); - + + console.log(load_scene_state_response) + const loaded_scene = load_scene_state_response.data["scene"] this.current_scene_media_token_id = load_scene_state_response.data["media_file_token"] this.activeScene.scene.children = loaded_scene.children; @@ -254,7 +256,20 @@ class Editor { }); } + public removeTransformControls() { + if (this.control == undefined) { return }; + if (this.outlinePass == undefined) { return }; + + this.last_selected = this.selected; + this.control.detach(); + this.activeScene.scene.remove(this.control); + this.outlinePass.selectedObjects = []; + } + public async saveScene(name: string) { + // remove controls when saving scene. + this.removeTransformControls(); + this.dispatchAppUiState({ type: ACTION_TYPES.SHOW_EDITOR_LOADER }); @@ -758,10 +773,7 @@ class Editor { this.transform_interaction = true; } } else if (this.transform_interaction == false) { - this.last_selected = this.selected; - this.control.detach(); - this.activeScene.scene.remove(this.control); - this.outlinePass.selectedObjects = []; + this.removeTransformControls(); } else { this.transform_interaction = false; } From 6a20e886dd4c45cab15b2272e48435a703674268 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 27 Mar 2024 21:26:14 -0400 Subject: [PATCH 23/23] Editor Save and Load Done. --- app/pages/PageEnigma/js/api_manager.ts | 189 ------------------------- app/pages/PageEnigma/js/editor.ts | 45 ------ 2 files changed, 234 deletions(-) diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index 7a5cd656d5..f1ffe4b311 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -275,195 +275,6 @@ class APIManager { return ""; } - // Function to save or update the scene - // it will always return a boolean success and the media_file_token - // they take an optional media_file_token as a parameter for the file to overwrite, which must belong to the user. - // If it's an anonymous user, we check the anonymous visitor token/cookie - - // async saveSceneState2( - // scene: THREE.Scene, - // timeline: TimelineDataState, - // scene_name: string, - // scene_media_file_token: string | null = null, - // ): Promise { - // // create new scene - - // let file = await this.gltfExport(scene); - // let response = await this.uploadGLB(file); - // let success = response["success"]; - // if (success) { - // let media_file_token = response["media_file_token"]; - // console.log(`media file token: ${media_file_token}`) - // const result = await this._saveSceneAndTimelineToJSONSpec(media_file_token,scene_name, timeline); - // return result - // } - - // throw Error("Saving the Scene Resulted in an Error Success False"); - // } - - // // For JSON files. - // // Form-multipart POST with parameters: - // // uuid_idempotency_token - // // file - // // media_file_token (optional; the file to replace if present) - // async _saveSceneAndTimelineToJSONSpec( - // media_file_token: string, - // scene_file_name: string, - // timeline: TimelineDataState, - // ): Promise { - // const url = `${this.baseUrl}/v1/media_files/write/scene_file`; - // let uuid = uuidv4(); - - // // turn json into a blob - // const file_test = {"glb_media_file_id":media_file_token,"name":scene_file_name,"entities":[], "timeline":""} - - // const json = JSON.stringify(file_test); - // const blob = new Blob([json], {type: 'application/json'}); - // const fileName = `${uuid}-${scene_file_name}.json`; - - // const formData = new FormData(); - - // formData.append("uuid_idempotency_token", uuid); - // formData.append("file", blob, fileName); - // formData.append("source", "file"); - // formData.append("type", "scene_json"); - // formData.append("source", "file"); - - // const response = await fetch(url, { - // method: "POST", - // credentials: "include", - // headers: { - // Accept: "application/json", - // }, - // body: formData, - // }); - - // if (!response.ok) { - // throw new Error("Failed to Send Data"); - // } - - // console.log(`saveSceneAndTimelineToJSONSpecResponse: ${response}`) - - // return response.json(); // or handle the response as appropriate - // } - - // // media_file_token that describes the scene - // async loadScene(media_file_token: string): Promise { - // let api_base_url = "https://api.fakeyou.com"; - // let url = `${api_base_url}/v1/media_files/file/${media_file_token}` - // let response = await fetch(url); - // let json = await JSON.parse(await response.text()); - // let bucketPath = json["media_file"]["public_bucket_path"]; - // let media_base_url = "https://storage.googleapis.com/vocodes-public" - // let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path - - // let fileResponse = await fetch(media_url); - - // if (!fileResponse.ok) { - // throw new Error("Failed to download file"); - // } - // // Convert the response to a Blob - // let blob = await fileResponse.blob(); - // const json_value:string = await new Promise((resolve, reject) => { - // let reader = new FileReader(); - // reader.onloadend = () => resolve(JSON.parse(reader.result as string)); - // reader.onerror = reject; - // reader.readAsText(blob); - // }); - // console.log(json_value) - // return json_value - // } - - // // Provides bucket path to the actual file to download from the media file - // async getMediaFile(media_file_token: string): Promise { - // let api_base_url = "https://api.fakeyou.com"; - // let url = `${api_base_url}/v1/media_files/file/${media_file_token}` - // let response = await fetch(url); - // let json = await JSON.parse(await response.text()); - // let bucketPath = json["media_file"]["public_bucket_path"]; - // let media_base_url = "https://storage.googleapis.com/vocodes-public" - // let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path - // // TODO get it as binary. - // return media_url - // } - - // async gltfExport2(scene: THREE.Scene) { - // let gltfExporter = new GLTFExporter(); - // let uuid = uuidv4(); - // // save the scene to keep ids positions etc - // let result = await gltfExporter.parseAsync(scene); - // const file = new File([JSON.stringify(result)], `${uuid}.glb`, { - // type: "application/json", - // }); - // return file; - // } - // // two core functions to upload things to the server and get id's - // async uploadMedia2(blob: any, fileName: string): Promise { - // const url = `${this.baseUrl}/v1/media_uploads/upload`; - // let uuid = uuidv4(); - // const formData = new FormData(); - // formData.append("uuid_idempotency_token", uuid); - // formData.append("file", blob, fileName); - // formData.append("source", "file"); - // formData.append("type", "video"); - // formData.append("source", "file"); - // const response = await fetch(url, { - // method: "POST", - // credentials: "include", - // headers: { - // Accept: "application/json", - // }, - // body: formData, - // }); - - // if (!response.ok) { - // throw new Error("Failed to Send Data"); - // } - // return response.json(); - // } - - /** - To save the scene as a file get the id back to serialize - /v1/media_files/write/engine_asset - An upsert for files. - If the file exists, it'll check ownership prior to overwriting. - For GLB, GLTF, BVH, FBX, etc. game engine-type files. - Form-multipart POST with parameters: - uuid_idempotency_token - file - media_file_token (optional; the file to replace if present) - media_file_subtype (optional; mixamo, mocap_net, scene_import, animation_only) - media_file_class (audio, image, video, animation, character, prop, scene, unknown) - **/ - // async uploadGLB2(file: File, scene_media_file_token:string | null): Promise { - // const url = `${this.baseUrl}/v1/media_files/write/engine_asset`; - // let uuid = uuidv4(); - // const formData = new FormData(); - // formData.append("uuid_idempotency_token", uuid); - - // // update existing scene otherwise create new scene and use it's media_file_id - // if (scene_media_file_token != null) { - // formData.append("media_file_token", scene_media_file_token); - // } - - // formData.append("file", file); - // formData.append("source", "file"); - // formData.append("media_file_subtype", "scene_import"); - // formData.append("media_file_class", "scene"); - // const response = await fetch(url, { - // method: "POST", - // credentials: "include", - // headers: { - // Accept: "application/json", - // }, - // body: formData, - // }); - - // if (!response.ok) { - // throw new Error("Failed to Send Data"); - // } - // return response.json(); // or handle the response as appropriate - // } } export default APIManager; diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index b9a21e65ec..e10f37f004 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -291,51 +291,6 @@ class Editor { }); } - // uploading some objects for testing to get their media ids from my account. - async _upload_for_testing() { - // have to upload as a file first - - // This is the default scene. - // "m_189p8hj0eyypbg74kkhcpehwpjhnkz" scene with the fox. - - //let result = await this.api_manager.saveSceneState(this.activeScene.scene) - - // let result = await this.api_manager.getMediaFile( - // "m_189p8hj0eyypbg74kkhcpehwpjhnkz", - // ) - console.log("Saving..."); - const result = await this.api_manager.saveSceneState( - this.activeScene.scene, - "", - null, - null, - ); - console.log("Saved!"); - console.log("Media ID is:", result); - // if (result.data) { - // this.test_scene_load_media_id = result.data["media_file_token"]; - // } - } - - //async _load_for_testing() { - // const result = await this.api_manager.loadScene(this.test_scene_load_media_id) - // let bucket_path = await this.api_manager.getMediaFile(result["glb_media_file_id"]) - // let glbLoader = new GLTFLoader() - // glbLoader.load(bucket_path, (glb) => { - // this.activeScene.scene.children = glb.scene.children - // this.activeScene.scene.children.forEach((child: THREE.Object3D) => { - // child.parent = this.activeScene.scene - // if (child.type == "DirectionalLight") { - // let pos = child.position - // let rot = child.rotation - // let light = this.activeScene._create_base_lighting() - // light.position.set(pos.x, pos.y, pos.z) - // light.rotation.set(rot.x, rot.y, rot.z) - // this.activeScene.scene.remove(child) - // } - // }) - // }) - //} async _serialize_timeline() { // note the database from the server is the source of truth for all the data.