From 8d0ff33176fd4dd06bc60385ac879d5498b617de Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Thu, 4 Apr 2024 19:04:34 -0400 Subject: [PATCH 01/14] Adding proxy --- .../proxy/storyteller_proxy_3d_object.ts | 67 ++++++++ .../proxy/storyteller_proxy_scene.ts | 155 ++++++++++++++++++ 2 files changed, 222 insertions(+) create mode 100644 app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts create mode 100644 app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts new file mode 100644 index 0000000000..4df3aa79ec --- /dev/null +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts @@ -0,0 +1,67 @@ +import * as THREE from "three"; + +export class StoryTellerProxy3DObject { + version: number; + + position: THREE.Vector3; + rotation: THREE.Euler; + scale: THREE.Vector3; + + object_uuid: string; + object_name: string; + object_user_data_name: string; // changable name + media_file_token: string; + + constructor( + version: number, + media_file_token: string, + ) { + this.version = version; + this.media_file_token = media_file_token; + + this.position = new THREE.Vector3(0.0, 0.0, 0.0); + this.rotation = new THREE.Euler(0.0, 0.0, 0.0); + this.scale = new THREE.Vector3(1.0, 1.0, 1.0); + + this.object_name = ""; + this.object_user_data_name = ""; + this.object_uuid = ""; + } + + public async initialize(object: THREE.Object3D) { + this.position = object.position; + this.rotation = object.rotation; + this.scale = object.scale; + + this.object_name = object.name; + this.object_user_data_name = object.userData.name; + this.object_uuid = object.uuid; + } + + public async toJSON(): Promise { + const json = { + version: this.version, + position: { + x: this.position.x, + y: this.position.y, + z: this.position.z, + }, + rotation: { + x: this.rotation.x, + y: this.rotation.y, + z: this.rotation.z, + }, + scale: { + x: this.scale.x, + y: this.scale.y, + z: this.scale.z, + }, + object_name: this.object_name, + object_uuid: this.object_uuid, + object_user_data_name: this.object_user_data_name, + media_file_token: this.media_file_token, + }; + const json_string = JSON.stringify(json); + return json_string; + } +} \ No newline at end of file diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts new file mode 100644 index 0000000000..a8aeedca5b --- /dev/null +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts @@ -0,0 +1,155 @@ +import * as THREE from "three"; +import { GLTFLoader, GLTF } from "three/addons/loaders/GLTFLoader.js"; +import { StoryTellerProxy3DObject } from "./storyteller_proxy_3d_object" + +interface LookUpDictionary { + [key: string]: StoryTellerProxy3DObject; + } + +export class StoryTellerProxyScene { + sceneItemProxy: StoryTellerProxy3DObject[]; + scene: THREE.Scene; + glbLoader: GLTFLoader; + + lookUpDictionary: LookUpDictionary; + version: number; + + constructor(version:number,scene: THREE.Scene) { + this.version = version + this.scene = scene; + this.glbLoader = new GLTFLoader(); + this.lookUpDictionary = {} + this.sceneItemProxy = [] + } + + private async get_media_url(media_file_token: string): Promise { + //This is for prod when we have the proper info on the url. + let api_base_url = "https://api.fakeyou.com"; + let url = `${api_base_url}/v1/media_files/file/${media_file_token}`; + let responce = await fetch(url); + let json = await JSON.parse(await responce.text()); + let bucketPath = json["media_file"]["public_bucket_path"]; + let media_base_url = "https://storage.googleapis.com/vocodes-public"; + let media_url = `${media_base_url}${bucketPath}`; + return media_url; + } + + // TODO can you throw custom error from here? + private async loadGLB(bucket_path: string): Promise { + return new Promise((resolve, reject) => { + this.glbLoader.load(bucket_path, (glb) => { + if (glb != null) { + resolve(glb); + } else { + reject(new Error("No GLB Data was Loaded.")); + } + }); + }); + } + + /** + * This is to load a scene from the json format from the server + * @param proxyObjects3D + */ + public async load(proxyObjects3D: StoryTellerProxy3DObject[]) { + // So first scene loading from scratch and everytime | create empty 3js scene + proxyObjects3D.forEach(async (element) => { + // construct the bucket path media_file_token + try { + const bucket_path = await this.get_media_url(element.media_file_token); + const glb = await this.loadGLB(bucket_path); + // extract children from scene any save glb and add it into the scene. + glb.scene.children.forEach((child) => { + // needs to be modified before adding to the scene. + child.traverse((c: THREE.Object3D) => { + if (c instanceof THREE.Mesh) { + c.material.metalness = 0.0; + c.material.specular = 0.5; + c.castShadow = true; + c.receiveShadow = true; + c.material.transparent = false; + } + }); + child.parent = this.scene; + this.scene.add(child); + this.sceneItemProxy.push(element) + // for easy lookup later + this.lookUpDictionary[element.object_uuid] = element + }); + } catch (error) { + console.log(error) + throw error + } + }); + } + + public async saveToScene():Promise { + // retrieve all elements in 3js scene update the proxy objects 3D and turn into json. + let result:string[] = [] + if (this.scene != null) { + this.scene.children.forEach(async (child) => { + + const proxyObject3D:StoryTellerProxy3DObject = this.lookUpDictionary[child.uuid] + proxyObject3D.position = child.position + proxyObject3D.rotation = child.rotation + proxyObject3D.scale = child.scale + proxyObject3D.object_user_data_name = child.userData.name; + proxyObject3D.object_name = child.name + proxyObject3D.object_uuid = child.uuid + + const json_data = await proxyObject3D.toJSON() + + result.push(json_data) + }); + } else { + console.log("Scene doesn't exist needs to be assigned") + } + const json_string = JSON.stringify(result) + return json_string + } + + /** + * This will add to and load into scene StoryTellerProxy3DObject + * @param media_file_token + */ + public async loadFromMediaFileToken(media_file_token: string) { + // load it from the db + // Everything you load is a scene | objects are a scene as well. + try { + const bucket_path = await this.get_media_url(media_file_token); + const glb = await this.loadGLB(bucket_path); + // extract children from scene any save glb and add it into the scene. + glb.scene.children.forEach((child) => { + // needs to be modified before adding to the scene. + child.traverse((c: THREE.Object3D) => { + if (c instanceof THREE.Mesh) { + c.material.metalness = 0.0; + c.material.specular = 0.5; + c.castShadow = true; + c.receiveShadow = true; + c.material.transparent = false; + } + }); + + child.parent = this.scene; + if (this.scene != null) { + this.scene.add(child); + // now add the record + const result = new StoryTellerProxy3DObject(this.version,media_file_token) + result.initialize(child) + this.sceneItemProxy.push(result) + this.lookUpDictionary[child.uuid] = result + } else { + console.log("Scene doesn't exist needs to be assigned") + } + }); + } catch (error) { + console.log(error) + } + } + + public async deleteWithUUID(uuid:String) { + + } + }; + \ No newline at end of file From d32a53199c1d84cc8ad713596e078f9a35c17982 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Thu, 4 Apr 2024 22:03:26 -0400 Subject: [PATCH 02/14] Need to back merge --- app/pages/PageEnigma/js/editor.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index cdb223aebe..4b1b14fd5e 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -24,7 +24,7 @@ import { PointerLockControls } from "three/addons/controls/PointerLockControls.j import { APPUI_ACTION_TYPES } from "../reducers"; import { ClipGroup } from "~/pages/PageEnigma/models/track"; -import { XYZ } from "../../datastructures/common"; +import { XYZ } from "../datastructures/common"; class EditorState { // { // action: "ShowLoadingIndicator" @@ -150,7 +150,6 @@ class Editor { this.raycaster; this.mouse; this.selected; - this.selectedState = false this.last_selected; this.transform_interaction; this.rendering = false; From ff961a326922a6af1c219d916d0bdedecabb2c49 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Thu, 4 Apr 2024 23:02:40 -0400 Subject: [PATCH 03/14] trying to test the proxy delete written --- app/pages/PageEnigma/js/editor.ts | 24 +++++++++---------- .../proxy/storyteller_proxy_scene.ts | 24 +++++++++++++++---- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 2692f241b0..b981d60363 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -25,6 +25,8 @@ import { APPUI_ACTION_TYPES } from "../reducers"; import { ClipGroup } from "~/pages/PageEnigma/models/track"; import { XYZ } from "../datastructures/common"; +import { StoryTellerProxyScene } from "../proxy/storyteller_proxy_scene"; +import { StoryTellerProxy3DObject } from "../proxy/storyteller_proxy_3d_object"; class EditorState { // { // action: "ShowLoadingIndicator" @@ -107,6 +109,10 @@ class Editor { last_scrub: number; // Default params. + + // scene proxy for serialization + storyteller_proxy_scene:StoryTellerProxyScene; + constructor() { console.log( "If you see this message twice! then it rendered twice, if you see it once it's all good.", @@ -206,6 +212,9 @@ class Editor { "((masterpiece, best quality, 8K, detailed)), colorful, epic, fantasy, (fox, red fox:1.2), no humans, 1other, ((koi pond)), outdoors, pond, rocks, stones, koi fish, ((watercolor))), lilypad, fish swimming around."; this.negative_prompt = ""; this.art_style = ArtStyle.Anime2DFlat; + + + this.storyteller_proxy_scene = new StoryTellerProxyScene(this.version,this.activeScene.scene) } initialize(config: any) { @@ -372,19 +381,8 @@ class Editor { public async testTestTimelineEvents() {} - public async testStylizeRequest() { - const result = await this.api_manager - .stylizeVideo( - "mu_6wy1570a0c3c0tpkkncf4tsvb5234", - this.art_style, - this.positive_prompt, - this.negative_prompt, - Visibility.Public, - ) - .catch((error) => { - console.log(error); - }); - console.log(result); + public async testLoadTokenAssets() { + await this.storyteller_proxy_scene.loadFromMediaFileToken("m_z4jzbst3xfh64h0qn4bqh4afenfps9") } public async loadScene(scene_media_token: string) { diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts index a8aeedca5b..8e6b60fbf7 100644 --- a/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts @@ -4,7 +4,7 @@ import { StoryTellerProxy3DObject } from "./storyteller_proxy_3d_object" interface LookUpDictionary { [key: string]: StoryTellerProxy3DObject; - } +} export class StoryTellerProxyScene { sceneItemProxy: StoryTellerProxy3DObject[]; @@ -21,7 +21,8 @@ export class StoryTellerProxyScene { this.lookUpDictionary = {} this.sceneItemProxy = [] } - + + // TODO: refactor this and dependency inject api manager. private async get_media_url(media_file_token: string): Promise { //This is for prod when we have the proper info on the url. let api_base_url = "https://api.fakeyou.com"; @@ -149,7 +150,22 @@ export class StoryTellerProxyScene { } public async deleteWithUUID(uuid:String) { - - } + if (this.scene != null) { + const object = this.scene.getObjectByProperty('uuid', uuid); + if (object) { + this.scene.remove(object); + // Remove from sceneItemProxy and lookUpDictionary + const index = this.sceneItemProxy.findIndex(item => uuid === uuid); + if (index !== -1) { + this.sceneItemProxy.splice(index, 1); + } + delete this.lookUpDictionary[uuid]; + } else { + console.log(`Object with UUID ${uuid} not found in the scene`); + } + } else { + console.log("Scene doesn't exist needs to be assigned"); + } + } }; \ No newline at end of file From 5d0cfddc90de7316b95813c5a5bda3f2b78aea07 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Thu, 4 Apr 2024 23:27:12 -0400 Subject: [PATCH 04/14] =?UTF-8?q?added=20Cartoon,=20J=C3=A9ja=20-=20On=20&?= =?UTF-8?q?=20On=20(feat.=20Daniel=20Levi)=20|=20Electronic=20Pop=20|=20NC?= =?UTF-8?q?S=20-=20Copyright=20Free=20Music?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../PageEnigma/contexts/TrackContext/TrackProvider.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx index 3c2a7efda1..1dc4c59bb0 100644 --- a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx +++ b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx @@ -129,6 +129,13 @@ export const TrackProvider = ({ children }: Props) => { type: ClipType.AUDIO, length: 25, name: "Talk", + }, + { + version: 1, + media_id: "m_w5nn3kjh1fbkmjrdac5b2qaba0pmyt", + type: ClipType.AUDIO, + length: 25, + name: "NCS Song", } ]); setScale(1); From f06f8cae12a600714acf7b4a9d56e4d947757420 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Fri, 5 Apr 2024 00:11:48 -0400 Subject: [PATCH 05/14] fixed rendering and video uploads. --- app/pages/PageEnigma/js/api_manager.ts | 2 +- app/pages/PageEnigma/js/editor.ts | 134 +++++++++++++------------ app/pages/PageEnigma/js/timeline.ts | 6 +- 3 files changed, 75 insertions(+), 67 deletions(-) diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index fde45ddd37..1a9ad9c151 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -310,7 +310,7 @@ export class APIManager { blob: any, fileName: string, ): Promise { - const url = `${this.baseUrl}/v1/media_uploads/upload`; + const url = `${this.baseUrl}/v1/media_files/upload`; const uuid = uuidv4(); const formData = new FormData(); diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index bf42059add..3e8c1e7736 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -91,6 +91,7 @@ class Editor { cam_obj: THREE.Object3D | undefined; renderPass: RenderPass | undefined; generating_preview: boolean; + frames: number; camera_person_mode: boolean; current_scene_media_token: string | null; @@ -172,6 +173,7 @@ class Editor { this.playback_location = 0; this.max_length = 10; this.last_scrub = 0; + this.frames = 0; // Audio Engine Test. this.render_width = 1280; @@ -361,7 +363,7 @@ class Editor { console.log(result); } - public async testTestTimelineEvents() {} + public async testTestTimelineEvents() { } public async testStylizeRequest() { const result = await this.api_manager @@ -433,8 +435,8 @@ class Editor { } // TO UPDATE selected objects in the scene might want to add to the scene ... - async setSelectedObject(position:XYZ,rotation:XYZ,scale:XYZ) { - if (this.selected !=undefined || this.selected !=null) { + async setSelectedObject(position: XYZ, rotation: XYZ, scale: XYZ) { + if (this.selected != undefined || this.selected != null) { //console.log(`triggering setSelectedObject`) this.selected.position.x = position.x this.selected.position.y = position.y @@ -449,7 +451,7 @@ class Editor { this.selected.scale.z = scale.z } - } + } public async saveScene(name: string) { @@ -813,7 +815,7 @@ class Editor { } // Basicly Unity 3D's update loop. - updateLoop(time: number) { + async updateLoop(time: number) { if (this.cam_obj == undefined) { this.cam_obj = this.activeScene.get_object_by_name("::CAM::"); } @@ -831,11 +833,11 @@ class Editor { if (this.cameraViewControls && this.camera_person_mode) { this.cameraViewControls.update(5 * delta_time); - if(this.cam_obj){ + if (this.cam_obj) { if (this.last_scrub != this.timeline.scrubber_frame_position) { this.camera.position.copy(this.cam_obj.position); this.camera.rotation.copy(this.cam_obj.rotation); - } else if(this.timeline.is_playing == false) { + } else if (this.timeline.is_playing == false) { this.cam_obj.position.copy(this.camera.position); this.cam_obj.rotation.copy(this.camera.rotation); } else { @@ -846,25 +848,24 @@ class Editor { this.cam_obj.visible = false; } } - else if (this.cam_obj){ + else if (this.cam_obj) { this.cam_obj.visible = true; } if (this.render_camera && this.cam_obj) { this.render_camera.position.copy(this.cam_obj.position); this.render_camera.rotation.copy(this.cam_obj.rotation); - this.cam_obj.scale.copy(new THREE.Vector3(1,1,1)); + this.cam_obj.scale.copy(new THREE.Vector3(1, 1, 1)); } - - if(this.timeline.is_playing){ - this.timeline.update(); - } - else if (this.last_scrub == this.timeline.scrubber_frame_position){ + + if (this.timeline.is_playing) { + await this.timeline.update(this.rendering); + } + else if (this.last_scrub == this.timeline.scrubber_frame_position) { this.updateSelectedUI(); } this.last_scrub = this.timeline.scrubber_frame_position; - this.renderScene(); } @@ -933,10 +934,10 @@ class Editor { audioSegment, "-filter_complex", "[1:a]adelay=" + - startTime * 1000 + - "|" + - startTime * 1000 + - "[a1];[0:a][a1]amix=inputs=2[a]", + startTime * 1000 + + "|" + + startTime * 1000 + + "[a1];[0:a][a1]amix=inputs=2[a]", "-map", "[a]", `${itteration}final_tmp.wav`, @@ -962,6 +963,8 @@ class Editor { } async stopPlayback(compile_audio: boolean = true) { + console.log(this.frames, this.frame_buffer.length); + this.renderMode(); const ffmpeg = createFFmpeg({ log: true }); await ffmpeg.load(); @@ -975,7 +978,7 @@ class Editor { } await ffmpeg.run( "-framerate", - "" + this.cap_fps, + "" + this.cap_fps/2, "-i", "image%d.png", "-f", @@ -1020,22 +1023,24 @@ class Editor { console.log("data", data) if (data == null) { return; } - let upload_token = data['upload_token']; + let upload_token = data['media_file_token']; console.log(upload_token); // Create a link to download the file stylize video using api .. //{"success":true,"upload_token":"mu_x9kr5cfafn512pjbygdszvbdpktrr"} payload const result = await this.api_manager - .stylizeVideo( - upload_token, - this.art_style, - this.positive_prompt, - this.negative_prompt, - Visibility.Public, - ) - .catch((error) => { - console.log(error); - }); + .stylizeVideo( + upload_token, + this.art_style, + this.positive_prompt, + this.negative_prompt, + Visibility.Public, + ) + .catch((error) => { + console.log(error); + }); + + // {"success":true,"inference_job_token":"jinf_j3nbqbd15wqxb0xcks13qh3f3bz"} console.log(result); @@ -1167,6 +1172,7 @@ class Editor { } if (this.rendering && this.renderer && this.clock) { + this.frames += 1; this.playback_location++; const imgData = this.renderer.domElement.toDataURL(); this.frame_buffer.push(imgData); @@ -1186,41 +1192,41 @@ class Editor { if (this.selected == undefined) { return; } - const pos = this.selected.position; - const rot = this.selected.rotation; - const scale = this.selected.scale; - - // TODO this is a bug we need to only show when clicked on and use UPDATE when updating. - this.dispatchAppUiState({ - type: APPUI_ACTION_TYPES.UPDATE_CONTROLPANELS_SCENEOBJECT, - payload: { - group: - this.selected.name === "::CAM::" - ? ClipGroup.CAMERA - : ClipGroup.OBJECT, // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips - object_uuid: this.selected.uuid, - object_name: this.selected.name, - version: this.version, - objectVectors: { - position: { - x: parseFloat(pos.x.toFixed(2)), - y: parseFloat(pos.y.toFixed(2)), - z: parseFloat(pos.z.toFixed(2)), - }, - rotation: { - x: parseFloat(rot.x.toFixed(2)), - y: parseFloat(rot.y.toFixed(2)), - z: parseFloat(rot.z.toFixed(2)), - }, - scale: { - x: parseFloat(scale.x.toFixed(2)), - y: parseFloat(scale.y.toFixed(2)), - z: parseFloat(scale.z.toFixed(2)), - }, + const pos = this.selected.position; + const rot = this.selected.rotation; + const scale = this.selected.scale; + + // TODO this is a bug we need to only show when clicked on and use UPDATE when updating. + this.dispatchAppUiState({ + type: APPUI_ACTION_TYPES.UPDATE_CONTROLPANELS_SCENEOBJECT, + payload: { + group: + this.selected.name === "::CAM::" + ? ClipGroup.CAMERA + : ClipGroup.OBJECT, // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips + object_uuid: this.selected.uuid, + object_name: this.selected.name, + version: this.version, + objectVectors: { + position: { + x: parseFloat(pos.x.toFixed(2)), + y: parseFloat(pos.y.toFixed(2)), + z: parseFloat(pos.z.toFixed(2)), + }, + rotation: { + x: parseFloat(rot.x.toFixed(2)), + y: parseFloat(rot.y.toFixed(2)), + z: parseFloat(rot.z.toFixed(2)), + }, + scale: { + x: parseFloat(scale.x.toFixed(2)), + y: parseFloat(scale.y.toFixed(2)), + z: parseFloat(scale.z.toFixed(2)), }, }, - }); - + }, + }); + } // Automaticly resize scene. diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 093c6f9da0..29b801af67 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -347,7 +347,7 @@ export class TimeLine { } // called by the editor update loop on each frame - public async update() { + public async update(isRendering=false) { //if (this.is_playing == false) return; // start and stop this.timeline_limit = this.getEndPoint(); if (this.is_playing) { @@ -399,8 +399,10 @@ export class TimeLine { } else if (element.type === ClipType.ANIMATION) { if (object) { await this.animation_engine.clips[object.uuid+element.media_id].play(object); + let fps = 120; + if(isRendering) { fps = 60; } this.animation_engine.clips[object.uuid+element.media_id].step( - this.scrubber_frame_position / 120, // Double FPS for best result. + this.scrubber_frame_position / fps, // Double FPS for best result. ); } } else { From e35ef9181703855e4c0098a2c0d7c7fbe011a549 Mon Sep 17 00:00:00 2001 From: FromTheLOLAshes Date: Fri, 5 Apr 2024 11:39:52 +0200 Subject: [PATCH 06/14] Fix scrubber + scrolling --- app/modules/LowerPanel/LowerPanel.tsx | 13 +++++++++++-- .../comps/Timeline/utils/useMouseEventsAnimation.ts | 13 ++++++++++--- .../comps/Timeline/utils/useMouseEventsClip.ts | 2 +- app/pages/PageEnigma/store/timing.ts | 2 ++ 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/app/modules/LowerPanel/LowerPanel.tsx b/app/modules/LowerPanel/LowerPanel.tsx index a95d7c9e8e..14657651f5 100644 --- a/app/modules/LowerPanel/LowerPanel.tsx +++ b/app/modules/LowerPanel/LowerPanel.tsx @@ -1,6 +1,7 @@ -import React, { useCallback } from "react"; +import React, { UIEvent, UIEventHandler, useCallback } from "react"; import { useMouseEventsHeight } from "~/pages/PageEnigma/comps/Timeline/utils/useMouseEventsHeight"; import { + currentScroll, currentTime, overTimeline, scale, @@ -22,7 +23,10 @@ export const LowerPanel = ({ children }: LowerPanelPropsI) => { const onTimelineClick = useCallback( (event: React.PointerEvent) => { if (event.button === 0) { - const newTime = Math.round((event.clientX - 92) / 4 / scale.value); + console.log("??", event.clientX, event.pageX, currentScroll.value); + const newTime = Math.round( + (event.clientX + currentScroll.value - 92) / 4 / scale.value, + ); if (newTime < 0) { return; } @@ -37,6 +41,10 @@ export const LowerPanel = ({ children }: LowerPanelPropsI) => { [], ); + const onScroll = useCallback((event: UIEvent) => { + currentScroll.value = event.currentTarget.scrollLeft; + }, []); + return ( <>
{ }} onPointerLeave={() => (overTimeline.value = false)} onPointerDown={onTimelineClick} + onScroll={onScroll} > {children}
diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsAnimation.ts b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsAnimation.ts index f3da2a8edb..8f4d302b36 100644 --- a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsAnimation.ts +++ b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsAnimation.ts @@ -1,5 +1,10 @@ import React, { useCallback, useEffect, useState } from "react"; -import { currentTime, filmLength, scale } from "~/pages/PageEnigma/store"; +import { + currentScroll, + currentTime, + filmLength, + scale, +} from "~/pages/PageEnigma/store"; import Queue from "~/pages/PageEnigma/Queue/Queue"; import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; @@ -21,11 +26,13 @@ export const useMouseEventsAnimation = () => { Queue.publish({ queueName: QueueNames.TO_ENGINE, action: toEngineActions.UPDATE_TIME, - data: { currentTime: Math.round(time) }, + data: { currentTime: Math.round(time) + currentScroll.value }, }); } }; + console.log("scroll", currentScroll.value); + const onMouseMove = (event: MouseEvent) => { const delta = Math.round( (event.clientX - clientX) / 4 / scale.value + currentTime.value, @@ -41,7 +48,7 @@ export const useMouseEventsAnimation = () => { Queue.publish({ queueName: QueueNames.TO_ENGINE, action: toEngineActions.UPDATE_TIME, - data: { currentTime: delta }, + data: { currentTime: delta + currentScroll.value }, }); } return delta; diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts index 4746501ee8..410298cf37 100644 --- a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts +++ b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts @@ -38,7 +38,7 @@ export const useMouseEventsClip = ( if (deltaOffset < min) { currOffset.current = min; } else if (deltaOffset + currLength.current > max) { - currOffset.current = max; + currOffset.current = max - currLength.current; } else { currOffset.current = deltaOffset; } diff --git a/app/pages/PageEnigma/store/timing.ts b/app/pages/PageEnigma/store/timing.ts index d64bb7525a..482fa09c8b 100644 --- a/app/pages/PageEnigma/store/timing.ts +++ b/app/pages/PageEnigma/store/timing.ts @@ -1,3 +1,5 @@ import { signal } from "@preact/signals-core"; export const currentTime = signal(0); + +export const currentScroll = signal(0); From bcb9c0f8dc4f7bb0ea46f245e49027e883d2e878 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Fri, 5 Apr 2024 17:52:46 -0400 Subject: [PATCH 07/14] Fixed generation crash --- app/pages/PageEnigma/js/editor.ts | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 3e8c1e7736..1ac70615ca 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -966,6 +966,9 @@ class Editor { console.log(this.frames, this.frame_buffer.length); this.renderMode(); + + if(this.generating_preview){return;} + this.generating_preview = true; const ffmpeg = createFFmpeg({ log: true }); await ffmpeg.load(); for (let index = 0; index < this.frame_buffer.length; index++) { @@ -1006,18 +1009,13 @@ class Editor { } } - const output = await ffmpeg.FS("readFile", itteration + "tmp.mp4"); + const output = ffmpeg.FS("readFile", itteration + "tmp.mp4"); + + ffmpeg.exit(); + this.generating_preview = false; + // Create a Blob from the output file for downloading const blob = new Blob([output.buffer], { type: "video/mp4" }); - const url = URL.createObjectURL(blob); - const downloadLink = document.createElement("a"); - downloadLink.href = url; - downloadLink.download = "render.mp4"; - document.body.appendChild(downloadLink); - downloadLink.click(); - // Clean up - URL.revokeObjectURL(url); - document.body.removeChild(downloadLink); const data: any = await this.api_manager.uploadMedia(blob, "render.mp4"); console.log("data", data) @@ -1116,7 +1114,7 @@ class Editor { // This initializes the generation of a video render scene is where the core work happens generateVideo() { console.log("Generating video..."); - if (this.rendering) { + if (this.rendering || this.generating_preview) { return; } this.rendering = true; // has to go first to debounce From 821b1ad3f9aa1a16026ddf54daa0eefa7b2d4cb8 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Fri, 5 Apr 2024 22:00:13 -0400 Subject: [PATCH 08/14] Added serialization and loading. --- app/pages/PageEnigma/js/api_manager.ts | 116 ++++----- app/pages/PageEnigma/js/editor.ts | 84 +++---- app/pages/PageEnigma/js/scene.ts | 3 + .../proxy/storyteller_proxy_3d_object.ts | 30 ++- .../proxy/storyteller_proxy_scene.ts | 229 ++++++------------ 5 files changed, 191 insertions(+), 271 deletions(-) diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index 1a9ad9c151..713caa74d8 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -88,43 +88,44 @@ export class APIManager { * @returns APIManagerResponseMessage */ public async saveSceneState( - scene: THREE.Scene, + scene_json: string, scene_name: string, scene_glb_media_file_token: string | null = null, scene_media_file_token: string | null = null, timeline_state: TimelineDataState | null = null, - ): Promise { - const file = await this.gltfExport(scene); + ): Promise { + const file = new File([scene_json], `${scene_name}.glb`, { + type: "application/json", + }); // will overwrite the scene on db if token exists - const upload_glb_response = await this.uploadGLB( + const upload_glb_response = await this.uploadEngineAsset( file, scene_glb_media_file_token, ); - const result_scene_glb_media_file_token = - upload_glb_response["media_file_token"]; - // now write the scene - const save_scene_timeline_response = await this.saveSceneAndTimeLineState( - result_scene_glb_media_file_token, - scene_media_file_token, - scene_name, - timeline_state, - ); + //const result_scene_glb_media_file_token = + // upload_glb_response["media_file_token"]; + //// now write the scene + //const save_scene_timeline_response = await this.saveSceneAndTimeLineState( + // result_scene_glb_media_file_token, + // scene_media_file_token, + // scene_name, + // timeline_state, + //); + //const result_scene_media_file_token = + // save_scene_timeline_response["media_file_token"]; + //const data = { + // scene_glb_media_file_token: result_scene_glb_media_file_token, + // scene_media_file_token: result_scene_media_file_token, + //}; - const result_scene_media_file_token = - save_scene_timeline_response["media_file_token"]; - - const data = { - scene_glb_media_file_token: result_scene_glb_media_file_token, - scene_media_file_token: result_scene_media_file_token, - }; - return new APIManagerResponseSuccess("Scene Saved", data); + return upload_glb_response['media_file_token']; } public async loadSceneState( scene_media_file_token: string | null, - ): Promise { + ): Promise { const api_base_url = "https://api.fakeyou.com"; const url = `${api_base_url}/v1/media_files/file/${scene_media_file_token}`; const response = await fetch(url); @@ -148,38 +149,38 @@ export class APIManager { }); console.log(`loadSceneState: ${JSON.stringify(json_result)}`); + return json_result; - const scene_glb_media_file_token: string = - json_result["scene_glb_media_file_token"]; + // const scene_glb_media_file_token: string = + // json_result["scene_glb_media_file_token"]; + // const media_bucket_path = await this.getMediaFile( + // scene_glb_media_file_token, + // ); + // console.log(`GLB ${media_bucket_path}`); + // const glbLoader = new GLTFLoader(); + // // promisify this + // const loadGlb = ( + // bucket_path: string, + // ): Promise => { + // return new Promise((resolve, reject) => { + // glbLoader.load(bucket_path, (glb) => { + // if (glb) { + // const scene: THREE.Scene = glb.scene; + // const data = { + // scene_glb_media_file_token: scene_glb_media_file_token, + // scene_media_file_token: scene_media_file_token, + // scene: scene, + // }; + // console.log(`Data: ${data}`); + // resolve(new APIManagerResponseSuccess("Success Loaded", data)); + // } else { + // throw new APIManagerResponseError("Failed to Load GLB Scene"); + // } + // }); + // }); + // }; - const media_bucket_path = await this.getMediaFile( - scene_glb_media_file_token, - ); - console.log(`GLB ${media_bucket_path}`); - const glbLoader = new GLTFLoader(); - // promisify this - const loadGlb = ( - bucket_path: string, - ): Promise => { - return new Promise((resolve, reject) => { - glbLoader.load(bucket_path, (glb) => { - if (glb) { - const scene: THREE.Scene = glb.scene; - const data = { - scene_glb_media_file_token: scene_glb_media_file_token, - scene_media_file_token: scene_media_file_token, - scene: scene, - }; - console.log(`Data: ${data}`); - resolve(new APIManagerResponseSuccess("Success Loaded", data)); - } else { - throw new APIManagerResponseError("Failed to Load GLB Scene"); - } - }); - }); - }; - - return await loadGlb(media_bucket_path); + //return await loadGlb(media_bucket_path); } /** @@ -213,18 +214,18 @@ export class APIManager { return file; } - private async uploadGLB( + private async uploadEngineAsset( file: File, - scene_glb_media_file_token: string | null, - ): Promise { + media_file_token: string | null, + ): Promise { const url = `${this.baseUrl}/v1/media_files/write/engine_asset`; const uuid = uuidv4(); const form_data = new FormData(); form_data.append("uuid_idempotency_token", uuid); // update existing scene otherwise create new glb scene and use it's media_file_id - if (scene_glb_media_file_token != null) { - form_data.append("media_file_token", scene_glb_media_file_token); + if (media_file_token != null) { + form_data.append("media_file_token", media_file_token); } form_data.append("file", file); @@ -245,7 +246,6 @@ export class APIManager { throw new Error("Failed to Send Data"); } else { const json_data = await response.json(); - console.log(`uploadGLB: ${JSON.stringify(json_data)}`); return json_data; // or handle the response as appropriate } } diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index d60482f6bb..d5ffc5fe13 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -383,49 +383,27 @@ class Editor { type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER, }); - if (scene_media_token != null) { - this.current_scene_media_token = scene_media_token; - } + this.current_scene_media_token = scene_media_token; - const load_scene_state_response = await this.api_manager.loadSceneState( + const scene_json = await this.api_manager.loadSceneState( this.current_scene_media_token, ); - console.log(load_scene_state_response); - if (load_scene_state_response.data == null) { - console.log("load_scene_state_response Missing Data"); - return; - } + let proxyScene = new StoryTellerProxyScene(this.version, this.activeScene); + await proxyScene.loadFromSceneJson(scene_json); + this.cam_obj = this.activeScene.get_object_by_name("::CAM::"); - const loaded_scene = load_scene_state_response.data["scene"]; - - if (load_scene_state_response.data == null) { - return; - } - // Load these so you can rewrite the scene glb using it's token. - this.current_scene_media_token = - load_scene_state_response.data["scene_media_file_token"]; - this.current_scene_glb_media_token = - load_scene_state_response.data["scene_glb_media_file_token"]; - - console.log( - `loadScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`, - ); - - this.activeScene.scene.children = loaded_scene.children; - - this.activeScene.scene.children.forEach((child: THREE.Object3D) => { - child.parent = this.activeScene.scene; - - if (child.type == "DirectionalLight") { - const pos = child.position; - const rot = child.rotation; - const light = this.activeScene._create_base_lighting(); - light.position.set(pos.x, pos.y, pos.z); - light.rotation.set(rot.x, rot.y, rot.z); - this.activeScene.scene.remove(child); - } - }); + // this.activeScene.scene.children.forEach((child: THREE.Object3D) => { + // child.parent = this.activeScene.scene; + // if (child.type == "DirectionalLight") { + // const pos = child.position; + // const rot = child.rotation; + // const light = this.activeScene._create_base_lighting(); + // light.position.set(pos.x, pos.y, pos.z); + // light.rotation.set(rot.x, rot.y, rot.z); + // this.activeScene.scene.remove(child); + // } + // }); this.dispatchAppUiState({ type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER, @@ -458,31 +436,31 @@ class Editor { this.dispatchAppUiState({ type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER, }); - console.log( - `saveScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`, - ); + let proxyScene = new StoryTellerProxyScene(this.version, this.activeScene); + let scene_json = await proxyScene.saveToScene(); // TODO turn scene information into and object ... const result = await this.api_manager.saveSceneState( - this.activeScene.scene, + scene_json, name, this.current_scene_glb_media_token, this.current_scene_media_token, new TimelineDataState(), ); - if (result.data == null) { - return; - } + console.log(result) - const scene_media_token = result.data["scene_media_file_token"]; - if (scene_media_token != null) { - this.current_scene_media_token = scene_media_token; - } - const scene_glb_media_token = result.data["scene_glb_media_file_token"]; - if (scene_glb_media_token != null) { - this.current_scene_glb_media_token = scene_glb_media_token; - } + //if (result.data == null) { + // return; + //} + //const scene_media_token = result.data["scene_media_file_token"]; + //if (scene_media_token != null) { + // this.current_scene_media_token = scene_media_token; + //} + //const scene_glb_media_token = result.data["scene_glb_media_file_token"]; + //if (scene_glb_media_token != null) { + // this.current_scene_glb_media_token = scene_glb_media_token; + //} this.dispatchAppUiState({ type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER, diff --git a/app/pages/PageEnigma/js/scene.ts b/app/pages/PageEnigma/js/scene.ts index 0228012696..66e3c44b54 100644 --- a/app/pages/PageEnigma/js/scene.ts +++ b/app/pages/PageEnigma/js/scene.ts @@ -46,6 +46,7 @@ class Scene { let obj = new THREE.Mesh(geometry, material); obj.receiveShadow = true; obj.castShadow = true; + obj.userData["media_id"] = "Parim"; //obj.type = "Object3D"; obj.name = name; this.scene.add(obj); @@ -128,6 +129,7 @@ class Scene { resolve(child.children[0]); return; } + child.userData["media_id"] = media_id; if (auto_add) { this.scene.add(child); } resolve(child); }); @@ -201,6 +203,7 @@ class Scene { directional_light.shadow.map = null; directional_light.castShadow = true; directional_light.shadow.bias = 0.00004; + directional_light.userData["media_id"] = "DirectionalLight"; this.scene.add(directional_light); this.scene.add(directional_light.target); diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts index 4df3aa79ec..d61734c79a 100644 --- a/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts @@ -1,5 +1,28 @@ import * as THREE from "three"; +export interface ObjectJSON { + version: number; + position: { + x: number; + y: number; + z: number; + }; + rotation: { + x: number; + y: number; + z: number; + }; + scale: { + x: number; + y: number; + z: number; + }; + object_name: string; + object_uuid: string; + object_user_data_name: string; + media_file_token: string; +} + export class StoryTellerProxy3DObject { version: number; @@ -38,8 +61,8 @@ export class StoryTellerProxy3DObject { this.object_uuid = object.uuid; } - public async toJSON(): Promise { - const json = { + public async toJSON(): Promise { + const json:ObjectJSON = { version: this.version, position: { x: this.position.x, @@ -61,7 +84,6 @@ export class StoryTellerProxy3DObject { object_user_data_name: this.object_user_data_name, media_file_token: this.media_file_token, }; - const json_string = JSON.stringify(json); - return json_string; + return json; } } \ No newline at end of file diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts index 8e6b60fbf7..24d82092e6 100644 --- a/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts @@ -1,171 +1,88 @@ import * as THREE from "three"; import { GLTFLoader, GLTF } from "three/addons/loaders/GLTFLoader.js"; -import { StoryTellerProxy3DObject } from "./storyteller_proxy_3d_object" +import { StoryTellerProxy3DObject, ObjectJSON } from "./storyteller_proxy_3d_object" +import Scene from "../js/scene"; interface LookUpDictionary { - [key: string]: StoryTellerProxy3DObject; + [key: string]: StoryTellerProxy3DObject; } - + export class StoryTellerProxyScene { - sceneItemProxy: StoryTellerProxy3DObject[]; - scene: THREE.Scene; - glbLoader: GLTFLoader; - - lookUpDictionary: LookUpDictionary; - version: number; + sceneItemProxy: StoryTellerProxy3DObject[]; + scene: Scene; + glbLoader: GLTFLoader; - constructor(version:number,scene: THREE.Scene) { - this.version = version - this.scene = scene; - this.glbLoader = new GLTFLoader(); - this.lookUpDictionary = {} - this.sceneItemProxy = [] - } - - // TODO: refactor this and dependency inject api manager. - private async get_media_url(media_file_token: string): Promise { - //This is for prod when we have the proper info on the url. - let api_base_url = "https://api.fakeyou.com"; - let url = `${api_base_url}/v1/media_files/file/${media_file_token}`; - let responce = await fetch(url); - let json = await JSON.parse(await responce.text()); - let bucketPath = json["media_file"]["public_bucket_path"]; - let media_base_url = "https://storage.googleapis.com/vocodes-public"; - let media_url = `${media_base_url}${bucketPath}`; - return media_url; - } - - // TODO can you throw custom error from here? - private async loadGLB(bucket_path: string): Promise { - return new Promise((resolve, reject) => { - this.glbLoader.load(bucket_path, (glb) => { - if (glb != null) { - resolve(glb); - } else { - reject(new Error("No GLB Data was Loaded.")); + lookUpDictionary: LookUpDictionary; + version: number; + + constructor(version: number, scene: Scene) { + this.version = version + this.scene = scene; + this.glbLoader = new GLTFLoader(); + this.lookUpDictionary = {} + this.sceneItemProxy = [] + } + + public async saveToScene(): Promise { + let results: ObjectJSON[] = []; + if (this.scene.scene != null) { + for (const child of this.scene.scene.children) { + if (child.userData['media_id'] != undefined) { + if (this.lookUpDictionary[child.uuid] == null) { + this.lookUpDictionary[child.uuid] = new StoryTellerProxy3DObject(this.version, child.userData['media_id']); } - }); - }); - } - - /** - * This is to load a scene from the json format from the server - * @param proxyObjects3D - */ - public async load(proxyObjects3D: StoryTellerProxy3DObject[]) { - // So first scene loading from scratch and everytime | create empty 3js scene - proxyObjects3D.forEach(async (element) => { - // construct the bucket path media_file_token - try { - const bucket_path = await this.get_media_url(element.media_file_token); - const glb = await this.loadGLB(bucket_path); - // extract children from scene any save glb and add it into the scene. - glb.scene.children.forEach((child) => { - // needs to be modified before adding to the scene. - child.traverse((c: THREE.Object3D) => { - if (c instanceof THREE.Mesh) { - c.material.metalness = 0.0; - c.material.specular = 0.5; - c.castShadow = true; - c.receiveShadow = true; - c.material.transparent = false; - } - }); - child.parent = this.scene; - this.scene.add(child); - this.sceneItemProxy.push(element) - // for easy lookup later - this.lookUpDictionary[element.object_uuid] = element - }); - } catch (error) { - console.log(error) - throw error - } - }); - } - - public async saveToScene():Promise { - // retrieve all elements in 3js scene update the proxy objects 3D and turn into json. - let result:string[] = [] - if (this.scene != null) { - this.scene.children.forEach(async (child) => { - - const proxyObject3D:StoryTellerProxy3DObject = this.lookUpDictionary[child.uuid] - proxyObject3D.position = child.position - proxyObject3D.rotation = child.rotation - proxyObject3D.scale = child.scale + let proxyObject3D: StoryTellerProxy3DObject = this.lookUpDictionary[child.uuid]; + proxyObject3D.position.copy(child.position); + proxyObject3D.rotation.copy(child.rotation); + proxyObject3D.scale.copy(child.scale); proxyObject3D.object_user_data_name = child.userData.name; - proxyObject3D.object_name = child.name - proxyObject3D.object_uuid = child.uuid - - const json_data = await proxyObject3D.toJSON() - - result.push(json_data) - }); - } else { - console.log("Scene doesn't exist needs to be assigned") - } - const json_string = JSON.stringify(result) - return json_string - } - - /** - * This will add to and load into scene StoryTellerProxy3DObject - * @param media_file_token - */ - public async loadFromMediaFileToken(media_file_token: string) { - // load it from the db - // Everything you load is a scene | objects are a scene as well. - try { - const bucket_path = await this.get_media_url(media_file_token); - const glb = await this.loadGLB(bucket_path); - // extract children from scene any save glb and add it into the scene. - glb.scene.children.forEach((child) => { - // needs to be modified before adding to the scene. - child.traverse((c: THREE.Object3D) => { - if (c instanceof THREE.Mesh) { - c.material.metalness = 0.0; - c.material.specular = 0.5; - c.castShadow = true; - c.receiveShadow = true; - c.material.transparent = false; - } - }); - - child.parent = this.scene; - if (this.scene != null) { - this.scene.add(child); - // now add the record - const result = new StoryTellerProxy3DObject(this.version,media_file_token) - result.initialize(child) - this.sceneItemProxy.push(result) - this.lookUpDictionary[child.uuid] = result - } else { - console.log("Scene doesn't exist needs to be assigned") - } - }); - } catch (error) { - console.log(error) + proxyObject3D.object_name = child.name; + proxyObject3D.object_uuid = child.uuid; + const json_data = await proxyObject3D.toJSON(); + results.push(json_data); } + } + } else { + console.log("Scene doesn't exist needs to be assigned"); } + const json_string = JSON.stringify(results); + return json_string; + } - public async deleteWithUUID(uuid:String) { - if (this.scene != null) { - const object = this.scene.getObjectByProperty('uuid', uuid); - if (object) { - this.scene.remove(object); - // Remove from sceneItemProxy and lookUpDictionary - const index = this.sceneItemProxy.findIndex(item => uuid === uuid); - if (index !== -1) { - this.sceneItemProxy.splice(index, 1); + public async loadFromSceneJson(scene_json: ObjectJSON[]) { + if (scene_json != null && this.scene != null) { + while (this.scene.scene.children.length > 0) { + this.scene.scene.remove(this.scene.scene.children[0]); + } + for (const json_object of scene_json) { + let token: string = json_object.media_file_token; + let obj; + switch (token) { + case 'Parim': + let prim_uuid = this.scene.instantiate(json_object.object_name); + obj = this.scene.get_object_by_uuid(prim_uuid); + break; + case 'DirectionalLight': + obj = this.scene._create_base_lighting(); + break; + default: + if(token.includes('m_')){ + obj = await this.scene.load_glb(token); } - delete this.lookUpDictionary[uuid]; - } else { - console.log(`Object with UUID ${uuid} not found in the scene`); - } - } else { - console.log("Scene doesn't exist needs to be assigned"); + break; + } + if (obj) { + obj.position.copy(json_object.position); + obj.rotation.copy(new THREE.Euler(json_object.rotation.x, json_object.rotation.y, json_object.rotation.z)); + obj.scale.copy(json_object.scale); + obj.name = json_object.object_name; + obj.userData.name = json_object.object_user_data_name; + obj.uuid = json_object.object_uuid; + obj.userData['media_id'] = json_object.media_file_token; } } - }; - \ No newline at end of file + this.scene._createGrid(); + } + } + +}; From a04f287fbedb698252a24b933ff1326566cbec91 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Fri, 5 Apr 2024 22:27:49 -0400 Subject: [PATCH 09/14] added modal to load from media token --- .../ControlsTopButtons/ControlsTopButtons.tsx | 29 ++++++++++++++++++- app/pages/PageEnigma/js/editor.ts | 3 ++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx index 527bd5226b..791a3b6484 100644 --- a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx +++ b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx @@ -12,18 +12,25 @@ export const ControlsTopButtons = () => { const [sceneName, setSceneName] = useState(""); const [sceneToken, setSceneToken] = useState(""); + // for testing + const [mediaToken, setMediaToken] = useState(""); + const handleButtonSave = () => { console.log(`SceneName is ${sceneName}`); editorEngine?.saveScene(sceneName); }; + const handleMediaToken = async () => { + await editorEngine?.test_loadMediaToken(mediaToken) + }; + const handleButtonLoadScene = () => { console.log(`Scene Token is ${sceneToken}`); editorEngine?.loadScene(sceneToken); }; const handleButtonTest = () => { - editorEngine?.testStylizeRequest(); + //editorEngine?.testStylizeRequest(); }; const handleTestButton2 = () => { @@ -86,6 +93,26 @@ export const ControlsTopButtons = () => { /> + + { + setMediaToken(e.target.value); + }} + /> + + Date: Fri, 5 Apr 2024 22:47:16 -0400 Subject: [PATCH 10/14] allow saves. --- .../PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx index 791a3b6484..4cdead6641 100644 --- a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx +++ b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx @@ -100,7 +100,7 @@ export const ControlsTopButtons = () => { }} confirmButtonProps={{ label: "Save", - disabled: sceneName === "", + disabled: mediaToken === "", onClick: handleMediaToken, }} title="Add Scene Object via Media Token" From 5ceacd03a071447f5773b92f042ce7440e20ce35 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Fri, 5 Apr 2024 23:22:51 -0400 Subject: [PATCH 11/14] added ability to load media tokens. --- app/pages/PageEnigma/js/editor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 8236dbcdea..f8504f42ac 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -430,7 +430,7 @@ class Editor { } public async test_loadMediaToken(media_file_token: string) { - // + this.activeScene.load_glb(media_file_token); } public async saveScene(name: string) { @@ -624,7 +624,7 @@ class Editor { // note the database from the server is the source of truth for all the data. // Test code here const object: THREE.Object3D = await this.activeScene.load_glb( - "m_77z28zfaxc3sdtt5cc68vpz2n40qed", + "m_fmxy8wjnep1hdaz7qdg4n7y15d2bsp", ); object.uuid = "CH1"; From 111892b270b973930733309ab358e5a672181262 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Sat, 6 Apr 2024 00:54:51 -0400 Subject: [PATCH 12/14] Added saving and loading of timeline. --- .../datastructures/clips/animation_clip.ts | 6 ++-- .../datastructures/clips/audio_clip.ts | 6 ++-- .../datastructures/clips/clip_ui.ts | 6 ++-- .../datastructures/clips/lipsync_clip.ts | 6 ++-- .../datastructures/clips/transform_clip.ts | 6 ++-- app/pages/PageEnigma/js/api_manager.ts | 5 ++- app/pages/PageEnigma/js/editor.ts | 31 ++++++++----------- .../proxy/storyteller_proxy_scene.ts | 5 ++- 8 files changed, 32 insertions(+), 39 deletions(-) diff --git a/app/pages/PageEnigma/datastructures/clips/animation_clip.ts b/app/pages/PageEnigma/datastructures/clips/animation_clip.ts index 3242e7b664..f2c9e8cd63 100644 --- a/app/pages/PageEnigma/datastructures/clips/animation_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/animation_clip.ts @@ -101,8 +101,8 @@ export class AnimationClip { this.mixer?.setTime(0); } - toJSON(): string { - return JSON.stringify({ + toJSON(): any { + return { version: this.version, media_id: this.media_id, object_uuid: this.object_uuid, @@ -110,6 +110,6 @@ export class AnimationClip { speed: this.speed, length: this.length, clip_name: this.clip_name, - }) + }; } } diff --git a/app/pages/PageEnigma/datastructures/clips/audio_clip.ts b/app/pages/PageEnigma/datastructures/clips/audio_clip.ts index cc5c19079c..8a4ffcde26 100644 --- a/app/pages/PageEnigma/datastructures/clips/audio_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/audio_clip.ts @@ -52,12 +52,12 @@ export class AudioClip { return new AudioData(audioContext, audioBuffer); } - toJSON(): string { - return JSON.stringify({ + toJSON(): any { + return { version: this.version, media_id: this.media_id, type: this.type, volume: this.volume, - }); + }; } } diff --git a/app/pages/PageEnigma/datastructures/clips/clip_ui.ts b/app/pages/PageEnigma/datastructures/clips/clip_ui.ts index 04dce4428c..45d10af865 100644 --- a/app/pages/PageEnigma/datastructures/clips/clip_ui.ts +++ b/app/pages/PageEnigma/datastructures/clips/clip_ui.ts @@ -34,8 +34,8 @@ export class ClipUI { this.length = length; } - toJSON(): string { - return JSON.stringify({ + toJSON(): any { + return { version: this.version, group: this.group, name: this.name, @@ -44,6 +44,6 @@ export class ClipUI { media_id: this.media_id, start_offset: this.offset, ending_offset: this.length, - }); + }; } } diff --git a/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts index 77fdd5f6f6..d484fda885 100644 --- a/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts @@ -129,12 +129,12 @@ export class LipSyncClip { this.lipsync = new LipSync(); } - toJSON(): string { - return JSON.stringify({ + toJSON(): any { + return { version: this.version, media_id: this.media_id, type: this.type, volume: this.volume, - }); + }; } } diff --git a/app/pages/PageEnigma/datastructures/clips/transform_clip.ts b/app/pages/PageEnigma/datastructures/clips/transform_clip.ts index df7fe39603..21c0d05bc2 100644 --- a/app/pages/PageEnigma/datastructures/clips/transform_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/transform_clip.ts @@ -140,14 +140,14 @@ export class TransformClip { keyframe_uuid)); } - toJSON(): string { - return JSON.stringify({ + toJSON(): any { + return { version: this.version, media_id: this.media_id, object_uuid: this.object_uuid, length: this.length, type: this.type, keyframes: this.keyframes, - }); + }; } } diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index 713caa74d8..8e5f91a902 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -88,13 +88,12 @@ export class APIManager { * @returns APIManagerResponseMessage */ public async saveSceneState( - scene_json: string, + save_json: string, scene_name: string, scene_glb_media_file_token: string | null = null, scene_media_file_token: string | null = null, - timeline_state: TimelineDataState | null = null, ): Promise { - const file = new File([scene_json], `${scene_name}.glb`, { + const file = new File([save_json], `${scene_name}.glb`, { type: "application/json", }); diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index f8504f42ac..d62c57c9ab 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -26,7 +26,8 @@ import { ClipGroup } from "~/pages/PageEnigma/models/track"; import { XYZ } from "../datastructures/common"; import { StoryTellerProxyScene } from "../proxy/storyteller_proxy_scene"; -import { StoryTellerProxy3DObject } from "../proxy/storyteller_proxy_3d_object"; +import { StoryTellerProxyTimeline } from "../proxy/storyteller_proxy_timeline"; + class EditorState { // { // action: "ShowLoadingIndicator" @@ -390,9 +391,12 @@ class Editor { ); let proxyScene = new StoryTellerProxyScene(this.version, this.activeScene); - await proxyScene.loadFromSceneJson(scene_json); + await proxyScene.loadFromSceneJson(scene_json['scene']); this.cam_obj = this.activeScene.get_object_by_name("::CAM::"); + let proxyTimeline = new StoryTellerProxyTimeline(this.version, this.timeline, this.transform_engine, this.animation_engine, this.audio_engine, this.lipsync_engine); + await proxyTimeline.loadFromJson(scene_json['timeline']); + // this.activeScene.scene.children.forEach((child: THREE.Object3D) => { // child.parent = this.activeScene.scene; // if (child.type == "DirectionalLight") { @@ -442,29 +446,20 @@ class Editor { let proxyScene = new StoryTellerProxyScene(this.version, this.activeScene); let scene_json = await proxyScene.saveToScene(); + + let proxyTimeline = new StoryTellerProxyTimeline(this.version, this.timeline, this.transform_engine, this.animation_engine, this.audio_engine, this.lipsync_engine); + let timeline_json = await proxyTimeline.saveToJson(); + + let save_data = {scene: scene_json, timeline: timeline_json}; + // TODO turn scene information into and object ... const result = await this.api_manager.saveSceneState( - scene_json, + JSON.stringify(save_data), name, this.current_scene_glb_media_token, this.current_scene_media_token, - new TimelineDataState(), ); - console.log(result) - - //if (result.data == null) { - // return; - //} - //const scene_media_token = result.data["scene_media_file_token"]; - //if (scene_media_token != null) { - // this.current_scene_media_token = scene_media_token; - //} - //const scene_glb_media_token = result.data["scene_glb_media_file_token"]; - //if (scene_glb_media_token != null) { - // this.current_scene_glb_media_token = scene_glb_media_token; - //} - this.dispatchAppUiState({ type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER, }); diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts index 24d82092e6..eec868b2e2 100644 --- a/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts @@ -23,7 +23,7 @@ export class StoryTellerProxyScene { this.sceneItemProxy = [] } - public async saveToScene(): Promise { + public async saveToScene(): Promise { let results: ObjectJSON[] = []; if (this.scene.scene != null) { for (const child of this.scene.scene.children) { @@ -45,8 +45,7 @@ export class StoryTellerProxyScene { } else { console.log("Scene doesn't exist needs to be assigned"); } - const json_string = JSON.stringify(results); - return json_string; + return results; } public async loadFromSceneJson(scene_json: ObjectJSON[]) { From d9c1fc44229c197c709e20c0398a7a083a1ef3e3 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Sat, 6 Apr 2024 01:21:16 -0400 Subject: [PATCH 13/14] Fix firefox compatibility issue on video export --- app/pages/PageEnigma/js/editor.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 1ac70615ca..a8f0839986 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -995,6 +995,10 @@ class Editor { "-c:a", "aac", // Specify audio codec (optional, but recommended for MP4) "-shortest", // Ensure output duration matches the shortest stream (video or audio) + "-pix_fmt" , + "yuv420p", + "-f", + "mp4", "0tmp.mp4", ); From 17196cf62f5a7099fb767124788b64451e7239ca Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Sat, 6 Apr 2024 02:06:04 -0400 Subject: [PATCH 14/14] added timeline proxy --- .../proxy/storyteller_proxy_timeline.ts | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts new file mode 100644 index 0000000000..ad2e7a473b --- /dev/null +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts @@ -0,0 +1,127 @@ +import { AnimationClip } from "../datastructures/clips/animation_clip"; +import { AudioClip } from "../datastructures/clips/audio_clip"; +import { ClipUI } from "../datastructures/clips/clip_ui"; +import { LipSyncClip } from "../datastructures/clips/lipsync_clip"; +import { TransformClip } from "../datastructures/clips/transform_clip"; +import AnimationEngine from "../js/animation_engine"; +import AudioEngine from "../js/audio_engine"; +import LipSyncEngine from "../js/lip_sync_engine"; +import { TimeLine } from "../js/timeline"; +import TransformEngine from "../js/transform_engine"; + + +export class StoryTellerProxyTimeline { + timeline: TimeLine + transform_engine: TransformEngine + animation_engine: AnimationEngine + audio_engine: AudioEngine + lipsync_engine: LipSyncEngine + + constructor(version: number, timeline: TimeLine, transform_engine: TransformEngine, animation_engine: AnimationEngine, audio_engine: AudioEngine, lipsync_engine: LipSyncEngine) { + this.timeline = timeline; + this.transform_engine = transform_engine; + this.animation_engine = animation_engine; + this.audio_engine = audio_engine; + this.lipsync_engine = lipsync_engine; + } + + private async getItems(items: any[]): Promise { + let timeline_items_data = []; + for (let index = 0; index < items.length; index++) { + const element = items[index]; + timeline_items_data.push(element.toJSON()); + }; + return timeline_items_data; + } + + private async getItemsDict(items: { [key: string]: any }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = element.toJSON(); + } + } + return timeline_items_data; + } + + public async saveToJson(): Promise { + let timeline_json = { + timeline: await this.getItems(this.timeline.timeline_items), + transform: await this.getItemsDict(this.transform_engine.clips), + animation: await this.getItemsDict(this.animation_engine.clips), + audio: await this.getItemsDict(this.audio_engine.clips), + lipsync: await this.getItemsDict(this.lipsync_engine.clips), + } + return timeline_json; + } + + private async loadTimelineClips(timeline_clips: any[]): Promise { + let new_clips = []; + if (timeline_clips) { + for (let index = 0; index < timeline_clips.length; index++) { + const element = timeline_clips[index]; + let clip = new ClipUI( + element.version, element.type, + element.group, element.name, + element.media_id, element.object_uuid, + element.start_offset, element.ending_offset); + new_clips.push(clip); + } + } + return new_clips; + } + + private async loadTransformClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = new TransformClip(element.version, element.object_uuid, element.length, element.media_id); + timeline_items_data[key].keyframes = element.keyframes; + } + } + return timeline_items_data; + } + + private async loadAudioClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = new AudioClip(element.version, element.media_id, element.volume); + } + } + return timeline_items_data; + } + + private async loadAnimationClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = new AnimationClip(element.version, element.media_id, "remote", element.object_uuid, element.speed, element.length, element.clip_name); + } + } + return timeline_items_data; + } + + private async loadLipsyncClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = new LipSyncClip(element.version, element.media_id, element.volume); + } + } + return timeline_items_data; + } + + public async loadFromJson(timeline: any) { + this.timeline.timeline_items = await this.loadTimelineClips(timeline['timeline']); + this.transform_engine.clips = await this.loadTransformClips(timeline['transform']); + this.audio_engine.clips = await this.loadAudioClips(timeline['audio']); + this.lipsync_engine.clips = await this.loadLipsyncClips(timeline['lipsync']); + this.animation_engine.clips = await this.loadAnimationClips(timeline['animation']); + } +}