From 87676d65e9f89d3df88487bc2da7e7c36c613f31 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Sat, 30 Mar 2024 20:43:09 -0400 Subject: [PATCH] added basic animation clips. --- .../contexts/TrackContext/TrackProvider.tsx | 2 +- .../datastructures/clips/animation_clip.ts | 50 +++++---- app/pages/PageEnigma/js/editor.ts | 103 +++++++++--------- app/pages/PageEnigma/js/scene.ts | 2 + app/pages/PageEnigma/js/timeline.ts | 87 ++++++++++----- 5 files changed, 140 insertions(+), 104 deletions(-) diff --git a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx index bc528311bd..ee8b41971a 100644 --- a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx +++ b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx @@ -76,7 +76,7 @@ export const TrackProvider = ({ children }: Props) => { setAnimationClips([ { version: 1, - media_id: uuid.v4(), + media_id: "m_5q9s6esz8ymjqz0bheh8nf4crtj2kx", type: ClipType.ANIMATION, length: 25, name: "Sit", diff --git a/app/pages/PageEnigma/datastructures/clips/animation_clip.ts b/app/pages/PageEnigma/datastructures/clips/animation_clip.ts index 50110653d4..d43b840122 100644 --- a/app/pages/PageEnigma/datastructures/clips/animation_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/animation_clip.ts @@ -36,22 +36,37 @@ export class AnimationClip { this.clip_action; } - // Takes a glb animation loads from the server - _load_animation(url: string): Promise { - return new Promise((resolve) => { - const glbLoader = new GLTFLoader(); - - glbLoader.load( - url, - (gltf) => { - // Assuming the animation is the first one in the animations array - const animationClip = gltf.animations[0]; - resolve(animationClip); - }, - ); - }); + async get_media_url() { + //This is for prod when we have the proper info on the url. + let api_base_url = "https://api.fakeyou.com"; + let url = `${api_base_url}/v1/media_files/file/${this.media_id}` + let responce = await fetch(url); + let json = await JSON.parse(await responce.text()); + let bucketPath = json["media_file"]["public_bucket_path"]; + let media_base_url = "https://storage.googleapis.com/vocodes-public" + let media_url = `${media_base_url}${bucketPath}` + return media_url; } + _load_animation(): Promise { + // Return the promise chain starting from `this.get_media_url()` + return this.get_media_url().then(url => { + // Return a new Promise that resolves with the animation clip + return new Promise((resolve) => { + const glbLoader = new GLTFLoader(); + + glbLoader.load( + url, + (gltf) => { + // Assuming the animation is the first one in the animations array + const animationClip = gltf.animations[0]; + resolve(animationClip); + }, + ); + }); + }); + } + _create_mixer(object: THREE.Object3D) { this.mixer = new THREE.AnimationMixer(object); return this.mixer; @@ -59,7 +74,7 @@ export class AnimationClip { async _get_clip() { if (this.animation_clip == null) { - this.animation_clip = await this._load_animation(this.media_id); + this.animation_clip = await this._load_animation(); } return this.animation_clip; } @@ -74,19 +89,16 @@ export class AnimationClip { this.clip_action.play(); } } - console.log("Play") } } step(deltatime: number) { if (this.mixer == null) { return; } - this.mixer?.update(deltatime); - console.log("Update") + this.mixer?.setTime(deltatime); } stop() { this.mixer?.stopAllAction(); - console.log("stop.") } toJSON(): string { diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index e90c3c9959..84decda680 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -597,63 +597,57 @@ class Editor { // Load timeline creates the the clips from the datastructure and loads them in here. // load object into the engine for lip syncing - this.lipsync_engine.load_object( - object.uuid, - "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", - ); - this.animation_engine.load_object( - object.uuid, - "/resources/models/fox/fox_idle.glb", - "clip3", - ); + //this.lipsync_engine.load_object( + // object.uuid, + // "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", + //); + //this.animation_engine.load_object( + // object.uuid, + // "/resources/models/fox/fox_idle.glb", + // "clip3", + //); // then it creates clip ui to load the playable clips // then refreshes the timeline. - // create the clip with the same id for a reference to the media - this.timeline.addPlayableClip( - new ClipUI( - 1.0, - "lipsync", - "character", - "clip1", - "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", - object.uuid, - 150, - 400, - ), - ); - - // media id for this is up in the air but when a path is created you should be able to store and delete it - this.timeline.addPlayableClip( - new ClipUI(1.0, "transform", "character", "clip2", object.uuid, object.uuid, 0, 150), - ); - + //this.timeline.addPlayableClip( + // new ClipUI( + // 1.0, + // "lipsync", + // "character", + // "clip1", + // "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", + // object.uuid, + // 150, + // 400, + // ), + //); + //// media id for this is up in the air but when a path is created you should be able to store and delete it + //this.timeline.addPlayableClip( + // new ClipUI(1.0, "transform", "character", "clip2", object.uuid, object.uuid, 0, 150), + //); // media id for this as well it can be downloaded - this.timeline.addPlayableClip( - new ClipUI( - 1.0, - "animation", - "character", - "clip3", - "/resources/models/fox/fox_idle.glb", - object.uuid, - 0, - 400, - ), - ); - - - this.audio_engine.loadClip("m_h33vytxs5eqqqf07nsy14qzrf9ww4v"); - - await this.timeline.addPlayableClip(new ClipUI( - 1.0, - "audio", - "global_audio", - "AudioClip", - "m_h33vytxs5eqqqf07nsy14qzrf9ww4v", - "", - 0, - 50)); + //this.timeline.addPlayableClip( + // new ClipUI( + // 1.0, + // "animation", + // "character", + // "clip3", + // "/resources/models/fox/fox_idle.glb", + // object.uuid, + // 0, + // 400, + // ), + //); + //this.audio_engine.loadClip("m_h33vytxs5eqqqf07nsy14qzrf9ww4v"); + //await this.timeline.addPlayableClip(new ClipUI( + // 1.0, + // "audio", + // "global_audio", + // "AudioClip", + // "m_h33vytxs5eqqqf07nsy14qzrf9ww4v", + // "", + // 0, + // 50)); } // Configure post processing. @@ -800,7 +794,7 @@ class Editor { this.render_camera.rotation.copy(this.cam_obj.rotation); } - this.timeline.update(delta_time); + this.timeline.update(); this.renderScene(); } @@ -1171,6 +1165,7 @@ class Editor { if (this.selected.type == "Scene") { this.selected = intersects[0].object; } + this.activeScene.selected = this.selected; this.updateSelectedUI(); // this.update_properties() diff --git a/app/pages/PageEnigma/js/scene.ts b/app/pages/PageEnigma/js/scene.ts index 8d7d51fe9a..81367ce85f 100644 --- a/app/pages/PageEnigma/js/scene.ts +++ b/app/pages/PageEnigma/js/scene.ts @@ -6,12 +6,14 @@ class Scene { gridHelper: THREE.GridHelper | undefined; scene: THREE.Scene; hot_items: THREE.Object3D[] | undefined; + selected: THREE.Object3D | undefined; constructor(name: string) { this.name = name; this.gridHelper; this.scene = new THREE.Scene(); this.hot_items = []; + this.selected = undefined; } initialize() { diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 90cf552201..a4368a45de 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -27,7 +27,7 @@ export class TimelineDataState { timeline_items: ClipUI[] scrubber_frame_position: number constructor(timeline_items: ClipUI[] = [], - scrubber_frame_position: number = 0) { + scrubber_frame_position: number = 0) { this.timeline_items = timeline_items this.scrubber_frame_position = scrubber_frame_position } @@ -52,17 +52,17 @@ export class TimeLine { scene: Scene - current_time:number + current_time: number // ensure that the elements are loaded first. constructor(audio_engine: AudioEngine, transform_engine: TransformEngine, lipsync_engine: LipSyncEngine, animation_engine: AnimationEngine, scene: Scene) { - + this.timeline_items = [] this.timeline_limit = 60 * 5 // 5 seconds - + this.is_playing = false this.scrubber_frame_position = 0 // in frames into the tl @@ -74,9 +74,8 @@ export class TimeLine { this.scene = scene; + Queue.subscribe(QueueNames.TO_ENGINE, this.handleTimelineActions.bind(this)); - Queue.subscribe(QueueNames.TO_ENGINE, this.handleTimelineActions); - this.current_time = 0 // TODO: How to move the timeline should put in update. // setInterval(()=> { @@ -85,7 +84,7 @@ export class TimeLine { // },50) } - public async pushEvent(action:fromEngineActions, data:any) { + public async pushEvent(action: fromEngineActions, data: any) { this.current_time += 1 Queue.publish({ queueName: QueueNames.FROM_ENGINE, @@ -94,7 +93,7 @@ export class TimeLine { }); } - public async handleTimelineActions(data:any) { + public async handleTimelineActions(data: any) { const action = data["action"] switch (action) { case toEngineActions.ADD_CLIP: @@ -107,7 +106,7 @@ export class TimeLine { await this.updateClip(data) break case toEngineActions.UPDATE_TIME: - await this.scrubberUpdate(data) + await this.scrub(data) break case toEngineActions.MUTE: await this.mute(data) @@ -115,26 +114,55 @@ export class TimeLine { default: console.log("Action Not Wired") } - + } - - public async addClip(data:any) { + + public async addClip(data: any) { // map to clip ui - console.log(data) + console.log("Add Clip:", data); + + let object_uuid = data['data']['object_uuid']; + + if (object_uuid.length < 6){ + object_uuid = this.scene.selected?.uuid;//data['data']['object_uuid']; + if(object_uuid == undefined) { return; } + } + + let media_id = data['data']['media_id']; + let name = data['data']['name']; + let group = data['data']['group']; + let version = data['data']['group']; + let type = data['data']['type']; + let offset = data['data']['offset']; + let end_offset = data['data']['length']+offset; + + + // media id for this as well it can be downloaded + this.addPlayableClip( + new ClipUI(version, type, + group, name, + media_id, object_uuid, + offset, end_offset)); + + switch(type) { + case "animation": + this.animation_engine.load_object(object_uuid, media_id, name); + break + } } - public async updateClip(data:any) { + public async updateClip(data: any) { // only length and offset changes here. console.log(data) } - public async deleteClip(data:any) { + public async deleteClip(data: any) { console.log(data) } - public async scrubberUpdate(data:any) { + public async scrubberUpdate(data: any) { console.log(data) } - public async mute(data:any) { + public async mute(data: any) { console.log(data) } @@ -144,7 +172,7 @@ export class TimeLine { // when given a media id item it will create the clip. // Then the clip will be loaded by the engines, if they come from outside of the loaded scene. - public async createClipOffset(media_id: string, object_uuid:string, type: string): Promise { + public async createClipOffset(media_id: string, object_uuid: string, type: string): Promise { // use engine to load based off media id and type animation | transform | } @@ -154,13 +182,12 @@ export class TimeLine { } public async deletePlayableClip(clip_uuid: string): Promise { - + } - public async scrub(offset_frame: number): Promise { + public async scrub(data: any): Promise { if (this.is_playing) { return } - // only stream through to the position and rotation keyframes - // debounce not really + this.setScrubberPosition(data['data']['currentTime']); } public async scrubberDidStop(offset_frame: number) { @@ -200,14 +227,14 @@ export class TimeLine { } // called by the editor update loop on each frame - public async update(deltatime:number) { - if (this.is_playing == false) return; // start and stop + public async update(deltatime: number) { + //if (this.is_playing == false) return; // start and stop if (this.scrubber_frame_position <= 0) { await this.resetScene(); } - this.scrubber_frame_position += 1; + //this.scrubber_frame_position += 1; //2. allow stopping. //3. smallest unit is a frame and it is set by the scene and is in fps, our videos will be 60fps but we can reprocess them using the pipeline. for (const element of this.timeline_items) { @@ -223,7 +250,7 @@ export class TimeLine { } } else if (element.type == "audio") { - if(this.scrubber_frame_position+1 >= element.length){ + if (this.scrubber_frame_position + 1 >= element.length) { this.audio_engine.stopClip(element.media_id); } else { @@ -231,7 +258,7 @@ export class TimeLine { } } else if (element.type == "lipsync") { - if(this.scrubber_frame_position+1 >= element.length){ + if (this.scrubber_frame_position + 1 >= element.length) { this.lipSync_engine.clips[element.object_uuid].stop(); } else if (object) { @@ -240,9 +267,9 @@ export class TimeLine { } } else if (element.type == "animation") { - if (object) { - await this.animation_engine.clips[object.uuid].play(object); - this.animation_engine.clips[object.uuid].step(deltatime); + if (object) { + await this.animation_engine.clips[object.uuid].play(object); + this.animation_engine.clips[object.uuid].step(this.scrubber_frame_position/60); } } else {