diff --git a/app/modules/LowerPanel/LowerPanel.tsx b/app/modules/LowerPanel/LowerPanel.tsx index a95d7c9e8e..14657651f5 100644 --- a/app/modules/LowerPanel/LowerPanel.tsx +++ b/app/modules/LowerPanel/LowerPanel.tsx @@ -1,6 +1,7 @@ -import React, { useCallback } from "react"; +import React, { UIEvent, UIEventHandler, useCallback } from "react"; import { useMouseEventsHeight } from "~/pages/PageEnigma/comps/Timeline/utils/useMouseEventsHeight"; import { + currentScroll, currentTime, overTimeline, scale, @@ -22,7 +23,10 @@ export const LowerPanel = ({ children }: LowerPanelPropsI) => { const onTimelineClick = useCallback( (event: React.PointerEvent) => { if (event.button === 0) { - const newTime = Math.round((event.clientX - 92) / 4 / scale.value); + console.log("??", event.clientX, event.pageX, currentScroll.value); + const newTime = Math.round( + (event.clientX + currentScroll.value - 92) / 4 / scale.value, + ); if (newTime < 0) { return; } @@ -37,6 +41,10 @@ export const LowerPanel = ({ children }: LowerPanelPropsI) => { [], ); + const onScroll = useCallback((event: UIEvent) => { + currentScroll.value = event.currentTarget.scrollLeft; + }, []); + return ( <>
{ }} onPointerLeave={() => (overTimeline.value = false)} onPointerDown={onTimelineClick} + onScroll={onScroll} > {children}
diff --git a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx index 77d1148bf4..e64f6ebbd5 100644 --- a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx +++ b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx @@ -11,10 +11,17 @@ export const ControlsTopButtons = () => { const [sceneName, setSceneName] = useState(""); const [sceneToken, setSceneToken] = useState(""); + // for testing + const [mediaToken, setMediaToken] = useState(""); + const handleButtonSave = () => { editorEngine?.saveScene(sceneName); }; + const handleMediaToken = async () => { + await editorEngine?.test_loadMediaToken(mediaToken) + }; + const handleButtonLoadScene = () => { editorEngine?.loadScene(sceneToken); }; @@ -62,6 +69,26 @@ export const ControlsTopButtons = () => { /> + + { + setMediaToken(e.target.value); + }} + /> + + { Queue.publish({ queueName: QueueNames.TO_ENGINE, action: toEngineActions.UPDATE_TIME, - data: { currentTime: Math.round(time) }, + data: { currentTime: Math.round(time) + currentScroll.value }, }); } }; + console.log("scroll", currentScroll.value); + const onMouseMove = (event: MouseEvent) => { const delta = Math.round( (event.clientX - clientX) / 4 / scale.value + currentTime.value, @@ -41,7 +48,7 @@ export const useMouseEventsAnimation = () => { Queue.publish({ queueName: QueueNames.TO_ENGINE, action: toEngineActions.UPDATE_TIME, - data: { currentTime: delta }, + data: { currentTime: delta + currentScroll.value }, }); } return delta; diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts index 4746501ee8..410298cf37 100644 --- a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts +++ b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts @@ -38,7 +38,7 @@ export const useMouseEventsClip = ( if (deltaOffset < min) { currOffset.current = min; } else if (deltaOffset + currLength.current > max) { - currOffset.current = max; + currOffset.current = max - currLength.current; } else { currOffset.current = deltaOffset; } diff --git a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx index 3c2a7efda1..1dc4c59bb0 100644 --- a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx +++ b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx @@ -129,6 +129,13 @@ export const TrackProvider = ({ children }: Props) => { type: ClipType.AUDIO, length: 25, name: "Talk", + }, + { + version: 1, + media_id: "m_w5nn3kjh1fbkmjrdac5b2qaba0pmyt", + type: ClipType.AUDIO, + length: 25, + name: "NCS Song", } ]); setScale(1); diff --git a/app/pages/PageEnigma/datastructures/clips/animation_clip.ts b/app/pages/PageEnigma/datastructures/clips/animation_clip.ts index 3242e7b664..f2c9e8cd63 100644 --- a/app/pages/PageEnigma/datastructures/clips/animation_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/animation_clip.ts @@ -101,8 +101,8 @@ export class AnimationClip { this.mixer?.setTime(0); } - toJSON(): string { - return JSON.stringify({ + toJSON(): any { + return { version: this.version, media_id: this.media_id, object_uuid: this.object_uuid, @@ -110,6 +110,6 @@ export class AnimationClip { speed: this.speed, length: this.length, clip_name: this.clip_name, - }) + }; } } diff --git a/app/pages/PageEnigma/datastructures/clips/audio_clip.ts b/app/pages/PageEnigma/datastructures/clips/audio_clip.ts index cc5c19079c..8a4ffcde26 100644 --- a/app/pages/PageEnigma/datastructures/clips/audio_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/audio_clip.ts @@ -52,12 +52,12 @@ export class AudioClip { return new AudioData(audioContext, audioBuffer); } - toJSON(): string { - return JSON.stringify({ + toJSON(): any { + return { version: this.version, media_id: this.media_id, type: this.type, volume: this.volume, - }); + }; } } diff --git a/app/pages/PageEnigma/datastructures/clips/clip_ui.ts b/app/pages/PageEnigma/datastructures/clips/clip_ui.ts index 04dce4428c..45d10af865 100644 --- a/app/pages/PageEnigma/datastructures/clips/clip_ui.ts +++ b/app/pages/PageEnigma/datastructures/clips/clip_ui.ts @@ -34,8 +34,8 @@ export class ClipUI { this.length = length; } - toJSON(): string { - return JSON.stringify({ + toJSON(): any { + return { version: this.version, group: this.group, name: this.name, @@ -44,6 +44,6 @@ export class ClipUI { media_id: this.media_id, start_offset: this.offset, ending_offset: this.length, - }); + }; } } diff --git a/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts index 77fdd5f6f6..d484fda885 100644 --- a/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts @@ -129,12 +129,12 @@ export class LipSyncClip { this.lipsync = new LipSync(); } - toJSON(): string { - return JSON.stringify({ + toJSON(): any { + return { version: this.version, media_id: this.media_id, type: this.type, volume: this.volume, - }); + }; } } diff --git a/app/pages/PageEnigma/datastructures/clips/transform_clip.ts b/app/pages/PageEnigma/datastructures/clips/transform_clip.ts index df7fe39603..21c0d05bc2 100644 --- a/app/pages/PageEnigma/datastructures/clips/transform_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/transform_clip.ts @@ -140,14 +140,14 @@ export class TransformClip { keyframe_uuid)); } - toJSON(): string { - return JSON.stringify({ + toJSON(): any { + return { version: this.version, media_id: this.media_id, object_uuid: this.object_uuid, length: this.length, type: this.type, keyframes: this.keyframes, - }); + }; } } diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index fde45ddd37..8e5f91a902 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -88,43 +88,43 @@ export class APIManager { * @returns APIManagerResponseMessage */ public async saveSceneState( - scene: THREE.Scene, + save_json: string, scene_name: string, scene_glb_media_file_token: string | null = null, scene_media_file_token: string | null = null, - timeline_state: TimelineDataState | null = null, - ): Promise { - const file = await this.gltfExport(scene); + ): Promise { + const file = new File([save_json], `${scene_name}.glb`, { + type: "application/json", + }); // will overwrite the scene on db if token exists - const upload_glb_response = await this.uploadGLB( + const upload_glb_response = await this.uploadEngineAsset( file, scene_glb_media_file_token, ); - const result_scene_glb_media_file_token = - upload_glb_response["media_file_token"]; - // now write the scene - const save_scene_timeline_response = await this.saveSceneAndTimeLineState( - result_scene_glb_media_file_token, - scene_media_file_token, - scene_name, - timeline_state, - ); + //const result_scene_glb_media_file_token = + // upload_glb_response["media_file_token"]; + //// now write the scene + //const save_scene_timeline_response = await this.saveSceneAndTimeLineState( + // result_scene_glb_media_file_token, + // scene_media_file_token, + // scene_name, + // timeline_state, + //); + //const result_scene_media_file_token = + // save_scene_timeline_response["media_file_token"]; + //const data = { + // scene_glb_media_file_token: result_scene_glb_media_file_token, + // scene_media_file_token: result_scene_media_file_token, + //}; - const result_scene_media_file_token = - save_scene_timeline_response["media_file_token"]; - - const data = { - scene_glb_media_file_token: result_scene_glb_media_file_token, - scene_media_file_token: result_scene_media_file_token, - }; - return new APIManagerResponseSuccess("Scene Saved", data); + return upload_glb_response['media_file_token']; } public async loadSceneState( scene_media_file_token: string | null, - ): Promise { + ): Promise { const api_base_url = "https://api.fakeyou.com"; const url = `${api_base_url}/v1/media_files/file/${scene_media_file_token}`; const response = await fetch(url); @@ -148,38 +148,38 @@ export class APIManager { }); console.log(`loadSceneState: ${JSON.stringify(json_result)}`); + return json_result; - const scene_glb_media_file_token: string = - json_result["scene_glb_media_file_token"]; + // const scene_glb_media_file_token: string = + // json_result["scene_glb_media_file_token"]; + // const media_bucket_path = await this.getMediaFile( + // scene_glb_media_file_token, + // ); + // console.log(`GLB ${media_bucket_path}`); + // const glbLoader = new GLTFLoader(); + // // promisify this + // const loadGlb = ( + // bucket_path: string, + // ): Promise => { + // return new Promise((resolve, reject) => { + // glbLoader.load(bucket_path, (glb) => { + // if (glb) { + // const scene: THREE.Scene = glb.scene; + // const data = { + // scene_glb_media_file_token: scene_glb_media_file_token, + // scene_media_file_token: scene_media_file_token, + // scene: scene, + // }; + // console.log(`Data: ${data}`); + // resolve(new APIManagerResponseSuccess("Success Loaded", data)); + // } else { + // throw new APIManagerResponseError("Failed to Load GLB Scene"); + // } + // }); + // }); + // }; - const media_bucket_path = await this.getMediaFile( - scene_glb_media_file_token, - ); - console.log(`GLB ${media_bucket_path}`); - const glbLoader = new GLTFLoader(); - // promisify this - const loadGlb = ( - bucket_path: string, - ): Promise => { - return new Promise((resolve, reject) => { - glbLoader.load(bucket_path, (glb) => { - if (glb) { - const scene: THREE.Scene = glb.scene; - const data = { - scene_glb_media_file_token: scene_glb_media_file_token, - scene_media_file_token: scene_media_file_token, - scene: scene, - }; - console.log(`Data: ${data}`); - resolve(new APIManagerResponseSuccess("Success Loaded", data)); - } else { - throw new APIManagerResponseError("Failed to Load GLB Scene"); - } - }); - }); - }; - - return await loadGlb(media_bucket_path); + //return await loadGlb(media_bucket_path); } /** @@ -213,18 +213,18 @@ export class APIManager { return file; } - private async uploadGLB( + private async uploadEngineAsset( file: File, - scene_glb_media_file_token: string | null, - ): Promise { + media_file_token: string | null, + ): Promise { const url = `${this.baseUrl}/v1/media_files/write/engine_asset`; const uuid = uuidv4(); const form_data = new FormData(); form_data.append("uuid_idempotency_token", uuid); // update existing scene otherwise create new glb scene and use it's media_file_id - if (scene_glb_media_file_token != null) { - form_data.append("media_file_token", scene_glb_media_file_token); + if (media_file_token != null) { + form_data.append("media_file_token", media_file_token); } form_data.append("file", file); @@ -245,7 +245,6 @@ export class APIManager { throw new Error("Failed to Send Data"); } else { const json_data = await response.json(); - console.log(`uploadGLB: ${JSON.stringify(json_data)}`); return json_data; // or handle the response as appropriate } } @@ -310,7 +309,7 @@ export class APIManager { blob: any, fileName: string, ): Promise { - const url = `${this.baseUrl}/v1/media_uploads/upload`; + const url = `${this.baseUrl}/v1/media_files/upload`; const uuid = uuidv4(); const formData = new FormData(); diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index bf42059add..fa12ce6879 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -25,6 +25,9 @@ import { APPUI_ACTION_TYPES } from "../reducers"; import { ClipGroup } from "~/pages/PageEnigma/models/track"; import { XYZ } from "../datastructures/common"; +import { StoryTellerProxyScene } from "../proxy/storyteller_proxy_scene"; +import { StoryTellerProxyTimeline } from "../proxy/storyteller_proxy_timeline"; + class EditorState { // { // action: "ShowLoadingIndicator" @@ -91,6 +94,7 @@ class Editor { cam_obj: THREE.Object3D | undefined; renderPass: RenderPass | undefined; generating_preview: boolean; + frames: number; camera_person_mode: boolean; current_scene_media_token: string | null; @@ -107,6 +111,10 @@ class Editor { last_scrub: number; // Default params. + + // scene proxy for serialization + storyteller_proxy_scene:StoryTellerProxyScene; + constructor() { console.log( "If you see this message twice! then it rendered twice, if you see it once it's all good.", @@ -172,6 +180,7 @@ class Editor { this.playback_location = 0; this.max_length = 10; this.last_scrub = 0; + this.frames = 0; // Audio Engine Test. this.render_width = 1280; @@ -206,6 +215,9 @@ class Editor { "((masterpiece, best quality, 8K, detailed)), colorful, epic, fantasy, (fox, red fox:1.2), no humans, 1other, ((koi pond)), outdoors, pond, rocks, stones, koi fish, ((watercolor))), lilypad, fish swimming around."; this.negative_prompt = ""; this.art_style = ArtStyle.Anime2DFlat; + + + this.storyteller_proxy_scene = new StoryTellerProxyScene(this.version,this.activeScene.scene) } initialize(config: any) { @@ -361,21 +373,10 @@ class Editor { console.log(result); } - public async testTestTimelineEvents() {} + public async testTestTimelineEvents() { } - public async testStylizeRequest() { - const result = await this.api_manager - .stylizeVideo( - "mu_6wy1570a0c3c0tpkkncf4tsvb5234", - this.art_style, - this.positive_prompt, - this.negative_prompt, - Visibility.Public, - ) - .catch((error) => { - console.log(error); - }); - console.log(result); + public async testLoadTokenAssets() { + await this.storyteller_proxy_scene.loadFromMediaFileToken("m_z4jzbst3xfh64h0qn4bqh4afenfps9") } public async loadScene(scene_media_token: string) { @@ -383,49 +384,30 @@ class Editor { type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER, }); - if (scene_media_token != null) { - this.current_scene_media_token = scene_media_token; - } + this.current_scene_media_token = scene_media_token; - const load_scene_state_response = await this.api_manager.loadSceneState( + const scene_json = await this.api_manager.loadSceneState( this.current_scene_media_token, ); - console.log(load_scene_state_response); - if (load_scene_state_response.data == null) { - console.log("load_scene_state_response Missing Data"); - return; - } + let proxyScene = new StoryTellerProxyScene(this.version, this.activeScene); + await proxyScene.loadFromSceneJson(scene_json['scene']); + this.cam_obj = this.activeScene.get_object_by_name("::CAM::"); - const loaded_scene = load_scene_state_response.data["scene"]; + let proxyTimeline = new StoryTellerProxyTimeline(this.version, this.timeline, this.transform_engine, this.animation_engine, this.audio_engine, this.lipsync_engine); + await proxyTimeline.loadFromJson(scene_json['timeline']); - if (load_scene_state_response.data == null) { - return; - } - // Load these so you can rewrite the scene glb using it's token. - this.current_scene_media_token = - load_scene_state_response.data["scene_media_file_token"]; - this.current_scene_glb_media_token = - load_scene_state_response.data["scene_glb_media_file_token"]; - - console.log( - `loadScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`, - ); - - this.activeScene.scene.children = loaded_scene.children; - - this.activeScene.scene.children.forEach((child: THREE.Object3D) => { - child.parent = this.activeScene.scene; - - if (child.type == "DirectionalLight") { - const pos = child.position; - const rot = child.rotation; - const light = this.activeScene._create_base_lighting(); - light.position.set(pos.x, pos.y, pos.z); - light.rotation.set(rot.x, rot.y, rot.z); - this.activeScene.scene.remove(child); - } - }); + // this.activeScene.scene.children.forEach((child: THREE.Object3D) => { + // child.parent = this.activeScene.scene; + // if (child.type == "DirectionalLight") { + // const pos = child.position; + // const rot = child.rotation; + // const light = this.activeScene._create_base_lighting(); + // light.position.set(pos.x, pos.y, pos.z); + // light.rotation.set(rot.x, rot.y, rot.z); + // this.activeScene.scene.remove(child); + // } + // }); this.dispatchAppUiState({ type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER, @@ -433,8 +415,8 @@ class Editor { } // TO UPDATE selected objects in the scene might want to add to the scene ... - async setSelectedObject(position:XYZ,rotation:XYZ,scale:XYZ) { - if (this.selected !=undefined || this.selected !=null) { + async setSelectedObject(position: XYZ, rotation: XYZ, scale: XYZ) { + if (this.selected != undefined || this.selected != null) { //console.log(`triggering setSelectedObject`) this.selected.position.x = position.x this.selected.position.y = position.y @@ -449,8 +431,11 @@ class Editor { this.selected.scale.z = scale.z } - } + } + public async test_loadMediaToken(media_file_token: string) { + this.activeScene.load_glb(media_file_token); + } public async saveScene(name: string) { // remove controls when saving scene. @@ -458,32 +443,23 @@ class Editor { this.dispatchAppUiState({ type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER, }); - console.log( - `saveScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`, - ); + + let proxyScene = new StoryTellerProxyScene(this.version, this.activeScene); + let scene_json = await proxyScene.saveToScene(); + + let proxyTimeline = new StoryTellerProxyTimeline(this.version, this.timeline, this.transform_engine, this.animation_engine, this.audio_engine, this.lipsync_engine); + let timeline_json = await proxyTimeline.saveToJson(); + + let save_data = {scene: scene_json, timeline: timeline_json}; // TODO turn scene information into and object ... const result = await this.api_manager.saveSceneState( - this.activeScene.scene, + JSON.stringify(save_data), name, this.current_scene_glb_media_token, this.current_scene_media_token, - new TimelineDataState(), ); - if (result.data == null) { - return; - } - - const scene_media_token = result.data["scene_media_file_token"]; - if (scene_media_token != null) { - this.current_scene_media_token = scene_media_token; - } - const scene_glb_media_token = result.data["scene_glb_media_file_token"]; - if (scene_glb_media_token != null) { - this.current_scene_glb_media_token = scene_glb_media_token; - } - this.dispatchAppUiState({ type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER, }); @@ -643,7 +619,7 @@ class Editor { // note the database from the server is the source of truth for all the data. // Test code here const object: THREE.Object3D = await this.activeScene.load_glb( - "m_77z28zfaxc3sdtt5cc68vpz2n40qed", + "m_fmxy8wjnep1hdaz7qdg4n7y15d2bsp", ); object.uuid = "CH1"; @@ -813,7 +789,7 @@ class Editor { } // Basicly Unity 3D's update loop. - updateLoop(time: number) { + async updateLoop(time: number) { if (this.cam_obj == undefined) { this.cam_obj = this.activeScene.get_object_by_name("::CAM::"); } @@ -831,11 +807,11 @@ class Editor { if (this.cameraViewControls && this.camera_person_mode) { this.cameraViewControls.update(5 * delta_time); - if(this.cam_obj){ + if (this.cam_obj) { if (this.last_scrub != this.timeline.scrubber_frame_position) { this.camera.position.copy(this.cam_obj.position); this.camera.rotation.copy(this.cam_obj.rotation); - } else if(this.timeline.is_playing == false) { + } else if (this.timeline.is_playing == false) { this.cam_obj.position.copy(this.camera.position); this.cam_obj.rotation.copy(this.camera.rotation); } else { @@ -846,25 +822,24 @@ class Editor { this.cam_obj.visible = false; } } - else if (this.cam_obj){ + else if (this.cam_obj) { this.cam_obj.visible = true; } if (this.render_camera && this.cam_obj) { this.render_camera.position.copy(this.cam_obj.position); this.render_camera.rotation.copy(this.cam_obj.rotation); - this.cam_obj.scale.copy(new THREE.Vector3(1,1,1)); + this.cam_obj.scale.copy(new THREE.Vector3(1, 1, 1)); } - - if(this.timeline.is_playing){ - this.timeline.update(); - } - else if (this.last_scrub == this.timeline.scrubber_frame_position){ + + if (this.timeline.is_playing) { + await this.timeline.update(this.rendering); + } + else if (this.last_scrub == this.timeline.scrubber_frame_position) { this.updateSelectedUI(); } this.last_scrub = this.timeline.scrubber_frame_position; - this.renderScene(); } @@ -933,10 +908,10 @@ class Editor { audioSegment, "-filter_complex", "[1:a]adelay=" + - startTime * 1000 + - "|" + - startTime * 1000 + - "[a1];[0:a][a1]amix=inputs=2[a]", + startTime * 1000 + + "|" + + startTime * 1000 + + "[a1];[0:a][a1]amix=inputs=2[a]", "-map", "[a]", `${itteration}final_tmp.wav`, @@ -962,7 +937,12 @@ class Editor { } async stopPlayback(compile_audio: boolean = true) { + console.log(this.frames, this.frame_buffer.length); + this.renderMode(); + + if(this.generating_preview){return;} + this.generating_preview = true; const ffmpeg = createFFmpeg({ log: true }); await ffmpeg.load(); for (let index = 0; index < this.frame_buffer.length; index++) { @@ -975,7 +955,7 @@ class Editor { } await ffmpeg.run( "-framerate", - "" + this.cap_fps, + "" + this.cap_fps/2, "-i", "image%d.png", "-f", @@ -989,6 +969,10 @@ class Editor { "-c:a", "aac", // Specify audio codec (optional, but recommended for MP4) "-shortest", // Ensure output duration matches the shortest stream (video or audio) + "-pix_fmt" , + "yuv420p", + "-f", + "mp4", "0tmp.mp4", ); @@ -1003,39 +987,36 @@ class Editor { } } - const output = await ffmpeg.FS("readFile", itteration + "tmp.mp4"); + const output = ffmpeg.FS("readFile", itteration + "tmp.mp4"); + + ffmpeg.exit(); + this.generating_preview = false; + // Create a Blob from the output file for downloading const blob = new Blob([output.buffer], { type: "video/mp4" }); - const url = URL.createObjectURL(blob); - const downloadLink = document.createElement("a"); - downloadLink.href = url; - downloadLink.download = "render.mp4"; - document.body.appendChild(downloadLink); - downloadLink.click(); - // Clean up - URL.revokeObjectURL(url); - document.body.removeChild(downloadLink); const data: any = await this.api_manager.uploadMedia(blob, "render.mp4"); console.log("data", data) if (data == null) { return; } - let upload_token = data['upload_token']; + let upload_token = data['media_file_token']; console.log(upload_token); // Create a link to download the file stylize video using api .. //{"success":true,"upload_token":"mu_x9kr5cfafn512pjbygdszvbdpktrr"} payload const result = await this.api_manager - .stylizeVideo( - upload_token, - this.art_style, - this.positive_prompt, - this.negative_prompt, - Visibility.Public, - ) - .catch((error) => { - console.log(error); - }); + .stylizeVideo( + upload_token, + this.art_style, + this.positive_prompt, + this.negative_prompt, + Visibility.Public, + ) + .catch((error) => { + console.log(error); + }); + + // {"success":true,"inference_job_token":"jinf_j3nbqbd15wqxb0xcks13qh3f3bz"} console.log(result); @@ -1111,7 +1092,7 @@ class Editor { // This initializes the generation of a video render scene is where the core work happens generateVideo() { console.log("Generating video..."); - if (this.rendering) { + if (this.rendering || this.generating_preview) { return; } this.rendering = true; // has to go first to debounce @@ -1167,6 +1148,7 @@ class Editor { } if (this.rendering && this.renderer && this.clock) { + this.frames += 1; this.playback_location++; const imgData = this.renderer.domElement.toDataURL(); this.frame_buffer.push(imgData); @@ -1186,41 +1168,41 @@ class Editor { if (this.selected == undefined) { return; } - const pos = this.selected.position; - const rot = this.selected.rotation; - const scale = this.selected.scale; - - // TODO this is a bug we need to only show when clicked on and use UPDATE when updating. - this.dispatchAppUiState({ - type: APPUI_ACTION_TYPES.UPDATE_CONTROLPANELS_SCENEOBJECT, - payload: { - group: - this.selected.name === "::CAM::" - ? ClipGroup.CAMERA - : ClipGroup.OBJECT, // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips - object_uuid: this.selected.uuid, - object_name: this.selected.name, - version: this.version, - objectVectors: { - position: { - x: parseFloat(pos.x.toFixed(2)), - y: parseFloat(pos.y.toFixed(2)), - z: parseFloat(pos.z.toFixed(2)), - }, - rotation: { - x: parseFloat(rot.x.toFixed(2)), - y: parseFloat(rot.y.toFixed(2)), - z: parseFloat(rot.z.toFixed(2)), - }, - scale: { - x: parseFloat(scale.x.toFixed(2)), - y: parseFloat(scale.y.toFixed(2)), - z: parseFloat(scale.z.toFixed(2)), - }, + const pos = this.selected.position; + const rot = this.selected.rotation; + const scale = this.selected.scale; + + // TODO this is a bug we need to only show when clicked on and use UPDATE when updating. + this.dispatchAppUiState({ + type: APPUI_ACTION_TYPES.UPDATE_CONTROLPANELS_SCENEOBJECT, + payload: { + group: + this.selected.name === "::CAM::" + ? ClipGroup.CAMERA + : ClipGroup.OBJECT, // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips + object_uuid: this.selected.uuid, + object_name: this.selected.name, + version: this.version, + objectVectors: { + position: { + x: parseFloat(pos.x.toFixed(2)), + y: parseFloat(pos.y.toFixed(2)), + z: parseFloat(pos.z.toFixed(2)), + }, + rotation: { + x: parseFloat(rot.x.toFixed(2)), + y: parseFloat(rot.y.toFixed(2)), + z: parseFloat(rot.z.toFixed(2)), + }, + scale: { + x: parseFloat(scale.x.toFixed(2)), + y: parseFloat(scale.y.toFixed(2)), + z: parseFloat(scale.z.toFixed(2)), }, }, - }); - + }, + }); + } // Automaticly resize scene. diff --git a/app/pages/PageEnigma/js/scene.ts b/app/pages/PageEnigma/js/scene.ts index 0228012696..66e3c44b54 100644 --- a/app/pages/PageEnigma/js/scene.ts +++ b/app/pages/PageEnigma/js/scene.ts @@ -46,6 +46,7 @@ class Scene { let obj = new THREE.Mesh(geometry, material); obj.receiveShadow = true; obj.castShadow = true; + obj.userData["media_id"] = "Parim"; //obj.type = "Object3D"; obj.name = name; this.scene.add(obj); @@ -128,6 +129,7 @@ class Scene { resolve(child.children[0]); return; } + child.userData["media_id"] = media_id; if (auto_add) { this.scene.add(child); } resolve(child); }); @@ -201,6 +203,7 @@ class Scene { directional_light.shadow.map = null; directional_light.castShadow = true; directional_light.shadow.bias = 0.00004; + directional_light.userData["media_id"] = "DirectionalLight"; this.scene.add(directional_light); this.scene.add(directional_light.target); diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 093c6f9da0..29b801af67 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -347,7 +347,7 @@ export class TimeLine { } // called by the editor update loop on each frame - public async update() { + public async update(isRendering=false) { //if (this.is_playing == false) return; // start and stop this.timeline_limit = this.getEndPoint(); if (this.is_playing) { @@ -399,8 +399,10 @@ export class TimeLine { } else if (element.type === ClipType.ANIMATION) { if (object) { await this.animation_engine.clips[object.uuid+element.media_id].play(object); + let fps = 120; + if(isRendering) { fps = 60; } this.animation_engine.clips[object.uuid+element.media_id].step( - this.scrubber_frame_position / 120, // Double FPS for best result. + this.scrubber_frame_position / fps, // Double FPS for best result. ); } } else { diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts new file mode 100644 index 0000000000..d61734c79a --- /dev/null +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts @@ -0,0 +1,89 @@ +import * as THREE from "three"; + +export interface ObjectJSON { + version: number; + position: { + x: number; + y: number; + z: number; + }; + rotation: { + x: number; + y: number; + z: number; + }; + scale: { + x: number; + y: number; + z: number; + }; + object_name: string; + object_uuid: string; + object_user_data_name: string; + media_file_token: string; +} + +export class StoryTellerProxy3DObject { + version: number; + + position: THREE.Vector3; + rotation: THREE.Euler; + scale: THREE.Vector3; + + object_uuid: string; + object_name: string; + object_user_data_name: string; // changable name + media_file_token: string; + + constructor( + version: number, + media_file_token: string, + ) { + this.version = version; + this.media_file_token = media_file_token; + + this.position = new THREE.Vector3(0.0, 0.0, 0.0); + this.rotation = new THREE.Euler(0.0, 0.0, 0.0); + this.scale = new THREE.Vector3(1.0, 1.0, 1.0); + + this.object_name = ""; + this.object_user_data_name = ""; + this.object_uuid = ""; + } + + public async initialize(object: THREE.Object3D) { + this.position = object.position; + this.rotation = object.rotation; + this.scale = object.scale; + + this.object_name = object.name; + this.object_user_data_name = object.userData.name; + this.object_uuid = object.uuid; + } + + public async toJSON(): Promise { + const json:ObjectJSON = { + version: this.version, + position: { + x: this.position.x, + y: this.position.y, + z: this.position.z, + }, + rotation: { + x: this.rotation.x, + y: this.rotation.y, + z: this.rotation.z, + }, + scale: { + x: this.scale.x, + y: this.scale.y, + z: this.scale.z, + }, + object_name: this.object_name, + object_uuid: this.object_uuid, + object_user_data_name: this.object_user_data_name, + media_file_token: this.media_file_token, + }; + return json; + } +} \ No newline at end of file diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts new file mode 100644 index 0000000000..eec868b2e2 --- /dev/null +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts @@ -0,0 +1,87 @@ +import * as THREE from "three"; +import { GLTFLoader, GLTF } from "three/addons/loaders/GLTFLoader.js"; +import { StoryTellerProxy3DObject, ObjectJSON } from "./storyteller_proxy_3d_object" +import Scene from "../js/scene"; + +interface LookUpDictionary { + [key: string]: StoryTellerProxy3DObject; +} + +export class StoryTellerProxyScene { + sceneItemProxy: StoryTellerProxy3DObject[]; + scene: Scene; + glbLoader: GLTFLoader; + + lookUpDictionary: LookUpDictionary; + version: number; + + constructor(version: number, scene: Scene) { + this.version = version + this.scene = scene; + this.glbLoader = new GLTFLoader(); + this.lookUpDictionary = {} + this.sceneItemProxy = [] + } + + public async saveToScene(): Promise { + let results: ObjectJSON[] = []; + if (this.scene.scene != null) { + for (const child of this.scene.scene.children) { + if (child.userData['media_id'] != undefined) { + if (this.lookUpDictionary[child.uuid] == null) { + this.lookUpDictionary[child.uuid] = new StoryTellerProxy3DObject(this.version, child.userData['media_id']); + } + let proxyObject3D: StoryTellerProxy3DObject = this.lookUpDictionary[child.uuid]; + proxyObject3D.position.copy(child.position); + proxyObject3D.rotation.copy(child.rotation); + proxyObject3D.scale.copy(child.scale); + proxyObject3D.object_user_data_name = child.userData.name; + proxyObject3D.object_name = child.name; + proxyObject3D.object_uuid = child.uuid; + const json_data = await proxyObject3D.toJSON(); + results.push(json_data); + } + } + } else { + console.log("Scene doesn't exist needs to be assigned"); + } + return results; + } + + public async loadFromSceneJson(scene_json: ObjectJSON[]) { + if (scene_json != null && this.scene != null) { + while (this.scene.scene.children.length > 0) { + this.scene.scene.remove(this.scene.scene.children[0]); + } + for (const json_object of scene_json) { + let token: string = json_object.media_file_token; + let obj; + switch (token) { + case 'Parim': + let prim_uuid = this.scene.instantiate(json_object.object_name); + obj = this.scene.get_object_by_uuid(prim_uuid); + break; + case 'DirectionalLight': + obj = this.scene._create_base_lighting(); + break; + default: + if(token.includes('m_')){ + obj = await this.scene.load_glb(token); + } + break; + } + if (obj) { + obj.position.copy(json_object.position); + obj.rotation.copy(new THREE.Euler(json_object.rotation.x, json_object.rotation.y, json_object.rotation.z)); + obj.scale.copy(json_object.scale); + obj.name = json_object.object_name; + obj.userData.name = json_object.object_user_data_name; + obj.uuid = json_object.object_uuid; + obj.userData['media_id'] = json_object.media_file_token; + } + } + this.scene._createGrid(); + } + } + +}; diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts new file mode 100644 index 0000000000..ad2e7a473b --- /dev/null +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts @@ -0,0 +1,127 @@ +import { AnimationClip } from "../datastructures/clips/animation_clip"; +import { AudioClip } from "../datastructures/clips/audio_clip"; +import { ClipUI } from "../datastructures/clips/clip_ui"; +import { LipSyncClip } from "../datastructures/clips/lipsync_clip"; +import { TransformClip } from "../datastructures/clips/transform_clip"; +import AnimationEngine from "../js/animation_engine"; +import AudioEngine from "../js/audio_engine"; +import LipSyncEngine from "../js/lip_sync_engine"; +import { TimeLine } from "../js/timeline"; +import TransformEngine from "../js/transform_engine"; + + +export class StoryTellerProxyTimeline { + timeline: TimeLine + transform_engine: TransformEngine + animation_engine: AnimationEngine + audio_engine: AudioEngine + lipsync_engine: LipSyncEngine + + constructor(version: number, timeline: TimeLine, transform_engine: TransformEngine, animation_engine: AnimationEngine, audio_engine: AudioEngine, lipsync_engine: LipSyncEngine) { + this.timeline = timeline; + this.transform_engine = transform_engine; + this.animation_engine = animation_engine; + this.audio_engine = audio_engine; + this.lipsync_engine = lipsync_engine; + } + + private async getItems(items: any[]): Promise { + let timeline_items_data = []; + for (let index = 0; index < items.length; index++) { + const element = items[index]; + timeline_items_data.push(element.toJSON()); + }; + return timeline_items_data; + } + + private async getItemsDict(items: { [key: string]: any }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = element.toJSON(); + } + } + return timeline_items_data; + } + + public async saveToJson(): Promise { + let timeline_json = { + timeline: await this.getItems(this.timeline.timeline_items), + transform: await this.getItemsDict(this.transform_engine.clips), + animation: await this.getItemsDict(this.animation_engine.clips), + audio: await this.getItemsDict(this.audio_engine.clips), + lipsync: await this.getItemsDict(this.lipsync_engine.clips), + } + return timeline_json; + } + + private async loadTimelineClips(timeline_clips: any[]): Promise { + let new_clips = []; + if (timeline_clips) { + for (let index = 0; index < timeline_clips.length; index++) { + const element = timeline_clips[index]; + let clip = new ClipUI( + element.version, element.type, + element.group, element.name, + element.media_id, element.object_uuid, + element.start_offset, element.ending_offset); + new_clips.push(clip); + } + } + return new_clips; + } + + private async loadTransformClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = new TransformClip(element.version, element.object_uuid, element.length, element.media_id); + timeline_items_data[key].keyframes = element.keyframes; + } + } + return timeline_items_data; + } + + private async loadAudioClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = new AudioClip(element.version, element.media_id, element.volume); + } + } + return timeline_items_data; + } + + private async loadAnimationClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = new AnimationClip(element.version, element.media_id, "remote", element.object_uuid, element.speed, element.length, element.clip_name); + } + } + return timeline_items_data; + } + + private async loadLipsyncClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = new LipSyncClip(element.version, element.media_id, element.volume); + } + } + return timeline_items_data; + } + + public async loadFromJson(timeline: any) { + this.timeline.timeline_items = await this.loadTimelineClips(timeline['timeline']); + this.transform_engine.clips = await this.loadTransformClips(timeline['transform']); + this.audio_engine.clips = await this.loadAudioClips(timeline['audio']); + this.lipsync_engine.clips = await this.loadLipsyncClips(timeline['lipsync']); + this.animation_engine.clips = await this.loadAnimationClips(timeline['animation']); + } +} diff --git a/app/pages/PageEnigma/store/timing.ts b/app/pages/PageEnigma/store/timing.ts index d64bb7525a..482fa09c8b 100644 --- a/app/pages/PageEnigma/store/timing.ts +++ b/app/pages/PageEnigma/store/timing.ts @@ -1,3 +1,5 @@ import { signal } from "@preact/signals-core"; export const currentTime = signal(0); + +export const currentScroll = signal(0);