From d2b509fd51654cda24f364096341e7774714a3d3 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Sun, 24 Mar 2024 21:10:50 -0400 Subject: [PATCH 01/15] Names added to clip offset --- app/pages/PageEnigma/datastructures/clips/clip_offset.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts index 9225276b74..5508f3976a 100644 --- a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts +++ b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts @@ -1,5 +1,6 @@ export interface ClipUI { version: number + name: string type: "transform" | "audio" | "animation" | "lipsync" media_id: string object_uuid: string @@ -12,6 +13,7 @@ export interface ClipUI { export class ClipUI implements ClipUI { version: number type: "transform" | "audio" | "animation" | "lipsync" + name: string media_id: string object_uuid: string start_offset: number // in frames @@ -19,6 +21,7 @@ export class ClipUI implements ClipUI { constructor( version: number, + name: string, type: "transform" | "audio" | "animation" | "lipsync", media_id: string, object_uuid: string, @@ -26,6 +29,7 @@ export class ClipUI implements ClipUI { ending_offset: number, ) { this.version = version + this.name = name this.type = type this.object_uuid = object_uuid this.media_id = media_id @@ -36,6 +40,7 @@ export class ClipUI implements ClipUI { toJSON(): string { return JSON.stringify({ version: this.version, + name: this.name, type: this.type, object_uuid: this.object_uuid, media_id: this.media_id, From 0763fe9cd335e6c953218cfca0368b130461cdb9 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Sun, 24 Mar 2024 22:10:47 -0400 Subject: [PATCH 02/15] completed lipsync. --- .../datastructures/clips/clip_offset.ts | 14 +- .../clips/lipsync_track_clip.ts | 217 ++++++++++-------- app/pages/PageEnigma/js/editor.js | 9 +- app/pages/PageEnigma/js/lipsync.js | 18 +- app/pages/PageEnigma/js/timeline.ts | 15 +- 5 files changed, 147 insertions(+), 126 deletions(-) diff --git a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts index 5508f3976a..154c630561 100644 --- a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts +++ b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts @@ -1,16 +1,6 @@ -export interface ClipUI { - version: number - name: string - type: "transform" | "audio" | "animation" | "lipsync" - media_id: string - object_uuid: string - start_offset: number // in frames so 60 frames per second on the timeline. - ending_offset: number // ending offset -} - // Clip offsets represent the state of the clip on the timeline as well as what type of clip it is. // it is created from a media id. -export class ClipUI implements ClipUI { +export class ClipUI { version: number type: "transform" | "audio" | "animation" | "lipsync" name: string @@ -21,8 +11,8 @@ export class ClipUI implements ClipUI { constructor( version: number, - name: string, type: "transform" | "audio" | "animation" | "lipsync", + name: string, media_id: string, object_uuid: string, start_offset: number, diff --git a/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts index 8030039a11..412b8b0bc3 100644 --- a/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts @@ -1,99 +1,134 @@ import { LipSync } from "../../js/lipsync"; import * as THREE from 'three'; interface AudioData { - audioContext: AudioContext; - audioBuffer: AudioBuffer; + audioContext: AudioContext; + audioBuffer: AudioBuffer; +} + +class AudioData implements AudioData { + audioContext: AudioContext; + audioBuffer: AudioBuffer; + source: AudioBufferSourceNode | undefined + + constructor(audioContext: AudioContext, audioBuffer: AudioBuffer) { + this.audioContext = audioContext; + this.audioBuffer = audioBuffer; } - - class AudioData implements AudioData{ - audioContext: AudioContext; - audioBuffer: AudioBuffer; - source: AudioBufferSourceNode | undefined - - constructor (audioContext: AudioContext, audioBuffer: AudioBuffer) { - this.audioContext = audioContext; - this.audioBuffer = audioBuffer; - } +} + +export interface LipSyncTrackClip { + version: number; + media_id: string; + type: "lipsync"; + volume: number; +} + +class BlendShapeHelper { + ah: number; + ee: number; + oh: number; + constructor(ah: number, ee: number, oh: number) { + this.ah = ah; + this.ee = ee; + this.oh = oh; } - - export interface LipSyncTrackClip { - version: number; - media_id: string; - type: "lipsync"; - volume: number; +} + +export class LipSyncTrackClip implements LipSyncTrackClip { + version: number; + media_id: string; + type: "lipsync" = "lipsync"; + volume: number; + audio_data: AudioData | undefined; + lipsync: LipSync; + blendshape_helper: BlendShapeHelper; + + constructor(version: number, media_id: string, volume: number) { + this.version = version; + this.media_id = media_id; + this.type = "lipsync"; + this.volume = volume; + this.download_audio().then(data => { + this.audio_data = data; + }); + this.blendshape_helper = new BlendShapeHelper(0, 0, 0); + // we might need 3 of these one for each character ... + this.lipsync = new LipSync(); } - - export class LipSyncTrackClip implements LipSyncTrackClip { - version: number; - media_id: string; - type: "lipsync" = "lipsync"; - volume: number; - audio_data: AudioData | undefined; - lipsync: LipSync; - - constructor(version: number, media_id: string, volume: number) { - this.version = version; - this.media_id = media_id; - this.type = "lipsync"; - this.volume = volume; - this.download_audio().then(data => { - this.audio_data = data; + + // lip sync will be generated through TTS + async get_media_url() { + //This is for prod when we have the proper info on the url. + let api_base_url = "https://api.fakeyou.com"; + let url = `${api_base_url}/v1/media_files/file/${this.media_id}` + let responce = await fetch(url); + let json = await JSON.parse(await responce.text()); + let bucketPath = json["media_file"]["public_bucket_path"]; + let media_base_url = "https://storage.googleapis.com/vocodes-public" + let media_url = `${media_base_url}${bucketPath}` + return media_url; + } + + async download_audio() { + let url = await this.get_media_url(); + const audioContext = new AudioContext(); + const response = await fetch(url); + const arrayBuffer = await response.arrayBuffer(); + const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); + return new AudioData(audioContext, audioBuffer); + } + + async _detect_face(object: THREE.Object3D): Promise { + return new Promise((resolve) => { + object.traverse((c: THREE.Object3D) => { + if (c instanceof THREE.Mesh) { + if (c.morphTargetInfluences && c.morphTargetDictionary) { + const blendShapeIndexE = c.morphTargetDictionary["E"]; + const blendShapeIndexO = c.morphTargetDictionary["O"]; + const blendShapeIndexA = c.morphTargetDictionary["aa"]; + if (blendShapeIndexE != null) { + this.blendshape_helper = new BlendShapeHelper(blendShapeIndexA, blendShapeIndexE, blendShapeIndexO); + resolve(c); + return; + } + } + } }); - - // we might need 3 of these one for each character ... - this.lipsync = new LipSync() - } - - // lip sync will be generated through TTS - async get_media_url() { - //This is for prod when we have the proper info on the url. - let api_base_url = "https://api.fakeyou.com"; - let url = `${api_base_url}/v1/media_files/file/${this.media_id}` - let responce = await fetch(url); - let json = await JSON.parse(await responce.text()); - let bucketPath = json["media_file"]["public_bucket_path"]; - let media_base_url = "https://storage.googleapis.com/vocodes-public" - let media_url = `${media_base_url}${bucketPath}` - return media_url; - } - - async download_audio() { - let url = await this.get_media_url(); - const audioContext = new AudioContext(); - const response = await fetch(url); - const arrayBuffer = await response.arrayBuffer(); - const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); - return new AudioData(audioContext, audioBuffer); - } - - play(face: THREE.Object3D) { - if (this.audio_data?.audioBuffer == null) { this.download_audio(); } - this.lipsync = new LipSync() - this.lipsync.face = face; - this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer); - } - - stop() { - if(this.lipsync == null) { return; } - this.lipsync.destroy(); - } - - step() { - if(this.lipsync == null) { return; } - this.lipsync.update(); - } - - reset() { - this.lipsync = new LipSync() - } - - toJSON(): string { - return JSON.stringify({ - version: this.version, - media_id: this.media_id, - type: this.type, - volume: this.volume, - }); - } + }); } - \ No newline at end of file + + + async play(object: THREE.Object3D) { + if (this.audio_data?.audioBuffer == null) { await this.download_audio(); } + if (this.lipsync.face == null) { + this.lipsync = new LipSync(await this._detect_face(object)); + this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer); + } +} + +stop() { + if (this.lipsync == null) { return; } + this.lipsync.destroy(); +} + +step() { + if (this.lipsync == null) { return; } + const positions = this.lipsync.update(); + this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ee] = positions["ee"]; + this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ah] = positions["ah"]; + this.lipsync.face.morphTargetInfluences[this.blendshape_helper.oh] = positions["oh"]; +} + +reset() { + this.lipsync = new LipSync(); +} + +toJSON(): string { + return JSON.stringify({ + version: this.version, + media_id: this.media_id, + type: this.type, + volume: this.volume, + }); +} +} diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index c3e44ac9a1..2fce1d9e1b 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -155,8 +155,6 @@ class Editor { this.timeline.scene = this.activeScene; - - this._test_demo() } @@ -169,7 +167,7 @@ class Editor { let object = await this.activeScene.load_glb("./resources/models/fox/fox.glb") this.lipsync_engine.load_object(object.uuid, "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72"); - this.timeline.addPlayableClip(new ClipUI(1.0, "lipsync", object.object_uuid, object.media_id, 0, 100)); + this.timeline.addPlayableClip(new ClipUI(1.0, "lipsync", "clip1", "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", object.uuid, 10, 200)); } // Configure post processing. @@ -278,11 +276,12 @@ class Editor { //console.log(this.transform_engine.clips[this.test_box_uuid]); - this.timeline.update(this.clock.getDelta()); + this.timeline.update(this.clock.getDelta()).catch(data => { + console.log(data); + }); this.render_scene(); if (this.capturer != null) { this.capturer.capture(this.renderer.domElement); } // Record scene. - } start_playback() { diff --git a/app/pages/PageEnigma/js/lipsync.js b/app/pages/PageEnigma/js/lipsync.js index 5db5dc24e3..a70ab40cf5 100644 --- a/app/pages/PageEnigma/js/lipsync.js +++ b/app/pages/PageEnigma/js/lipsync.js @@ -18,7 +18,7 @@ for (let m = 0; m < BoundingFrequencyFem.length; m++) { } export class LipSync { - constructor(face=null) { + constructor(face) { this.face = face; @@ -90,19 +90,15 @@ export class LipSync { update() { if (this.meter) { const { volume } = this.meter; + let { ah, oh, ee } = this.update2(); if (volume < 0.01) { - // get the shape keys from the face avatar - - this.face.morphTargetInfluences[this.ee] = 0; - this.face.morphTargetInfluences[this.ah] = 0; - this.face.morphTargetInfluences[this.oh] = 0; - } else { - const { ah, oh, ee } = this.update2(); - this.face.morphTargetInfluences[this.ee] = ee; - this.face.morphTargetInfluences[this.ah] = ah; - this.face.morphTargetInfluences[this.oh] = oh; + ah = 0; + oh = 0; + ee = 0; } + return { "ah": ah, "oh": oh, "ee": ee }; } + return { "ah": 0, "oh": 0, "ee": 0 }; } update2() { diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 5e64964371..42aa46b75e 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -123,6 +123,9 @@ export class TimeLine { this.audioEngine.loadClip(element.media_id); } else if (element.type == "animation") { + } + else if (element.type == "lipsync") { + } else { this.stop() @@ -140,10 +143,8 @@ export class TimeLine { } this.scrubberPosition += 1; - //2. allow stopping. //3. smallest unit is a frame and it is set by the scene and is in fps, our videos will be 60fps but we can reprocess them using the pipeline. - for (const element of this.timelineItems) { if (element.start_offset <= this.scrubberPosition && this.scrubberPosition <= element.ending_offset) { // run async @@ -161,15 +162,15 @@ export class TimeLine { this.audioEngine.playClip(element.media_id) } else if (element.type == "lipsync") { - let face_object = this.scene.get_object_by_uuid(element.object_uuid); - if (face_object) { - this.lipSyncEngine.clips[element.object_uuid].play(face_object); + if(this.scrubberPosition+1 >= element.ending_offset){ + this.lipSyncEngine.clips[element.object_uuid].stop(); + } + else if (object) { + await this.lipSyncEngine.clips[element.object_uuid].play(object); this.lipSyncEngine.clips[element.object_uuid].step(); } } - else if (element.type == "animation") { - let object = this.scene.get_object_by_uuid(element.object_uuid); if (object) { await this.animationEngine.clips[object.uuid].play(object); this.animationEngine.clips[object.uuid].step(deltatime); From 1feba3bc6456a446482cf73a8540d84d60f1ff29 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Sun, 24 Mar 2024 22:23:49 -0400 Subject: [PATCH 03/15] completed timeline with lipsync --- app/pages/PageEnigma/js/editor.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index 2fce1d9e1b..19920ccbdd 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -162,12 +162,12 @@ class Editor { // Test code here // this.test_box_uuid = this.activeScene.instantiate("Box"); // let object = this.transform_engine.loadObject(this.test_box_uuid); - // this.timeline.addPlayableClip(new ClipUI(1.0, "transform", object.object_uuid, object.media_id, 50, 300)); // this.timeline.addPlayableClip(new ClipUI(1.0, "audio", "m_f7jnwt3d1ddchatdk5vaqt0n4mb1hg", null, 50, 50)); let object = await this.activeScene.load_glb("./resources/models/fox/fox.glb") this.lipsync_engine.load_object(object.uuid, "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72"); - this.timeline.addPlayableClip(new ClipUI(1.0, "lipsync", "clip1", "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", object.uuid, 10, 200)); + this.timeline.addPlayableClip(new ClipUI(1.0, "lipsync", "clip1", "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", object.uuid, 150, 400)); + this.timeline.addPlayableClip(new ClipUI(1.0, "transform", "clip2", object.uuid, object.uuid, 0, 150)); } // Configure post processing. @@ -253,6 +253,7 @@ class Editor { if(this.selected == null) {return;} this.transform_engine.addFrame(this.selected) + console.log("Frame taken.") } change_camera_view() { From 1e1dc74a7cc8c64c5eec71fd11f4c0ba42602e8f Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Sun, 24 Mar 2024 22:53:21 -0400 Subject: [PATCH 04/15] Completed animation engine. --- .../clips/animation_track_clip.ts | 22 +++++++++++-------- .../clips/transform_track_clip.ts | 1 - app/pages/PageEnigma/js/editor.js | 14 +++++------- app/pages/PageEnigma/js/timeline.ts | 4 ++-- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts index 85fa881056..ea38c59885 100644 --- a/app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts @@ -66,35 +66,39 @@ export class AnimationTrackClip implements AnimationTrackClip { _create_mixer(object: THREE.Object3D) { this.mixer = new THREE.AnimationMixer(object); + return this.mixer; } async _get_clip() { if (this.animation_clip == null) { - this.animation_clip = await this._load_animation(this.clip_name); + this.animation_clip = await this._load_animation(this.media_id); } return this.animation_clip; } async play(object: THREE.Object3D) { - if (this.mixer == null) { this._create_mixer(object) } - - let anim_clip = await this._get_clip(); - this.clip_action = this.mixer?.clipAction(anim_clip); - if (this.clip_action) { - if (this.clip_action?.isRunning() == false) { - this.clip_action.play(); + if (this.mixer == null) { + this.mixer = this._create_mixer(object); + let anim_clip = await this._get_clip(); + this.clip_action = this.mixer?.clipAction(anim_clip); + if (this.clip_action) { + if (this.clip_action?.isRunning() == false) { + this.clip_action.play(); + } } + console.log("Play") } - } step(deltatime: number) { if (this.mixer == null) { return; } this.mixer?.update(deltatime); + console.log("Update") } stop() { this.mixer?.stopAllAction(); + console.log("stop.") } toJSON(): string { diff --git a/app/pages/PageEnigma/datastructures/clips/transform_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/transform_track_clip.ts index 14847f46cd..729f36621f 100644 --- a/app/pages/PageEnigma/datastructures/clips/transform_track_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/transform_track_clip.ts @@ -68,7 +68,6 @@ export class TransformTrackClip implements TransformTrackClip { object.position.copy(first_pos); object.rotation.set(first_rot.x, first_rot.y, first_rot.z); object.scale.copy(first_scl); - this.step_frame = 0; } } diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index 19920ccbdd..563367f0b4 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -160,14 +160,12 @@ class Editor { async _test_demo() { // Test code here - // this.test_box_uuid = this.activeScene.instantiate("Box"); - // let object = this.transform_engine.loadObject(this.test_box_uuid); - // this.timeline.addPlayableClip(new ClipUI(1.0, "audio", "m_f7jnwt3d1ddchatdk5vaqt0n4mb1hg", null, 50, 50)); - let object = await this.activeScene.load_glb("./resources/models/fox/fox.glb") this.lipsync_engine.load_object(object.uuid, "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72"); this.timeline.addPlayableClip(new ClipUI(1.0, "lipsync", "clip1", "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", object.uuid, 150, 400)); this.timeline.addPlayableClip(new ClipUI(1.0, "transform", "clip2", object.uuid, object.uuid, 0, 150)); + this.animation_engine.load_object(object.uuid, "/resources/models/fox/fox_idle.glb", "clip3"); + this.timeline.addPlayableClip(new ClipUI(1.0, "animation", "clip3", "/resources/models/fox/fox_idle.glb", object.uuid, 0, 400)); } // Configure post processing. @@ -271,15 +269,15 @@ class Editor { // Updates debug stats. if (this.stats != null) { this.stats.update(); } + let delta_time = this.clock.getDelta() + // All calls that are not super important like timeline go here. - this.activeScene.update(this.clock.getDelta()); + this.activeScene.update(delta_time); //this.orbit.update(0.1); //console.log(this.transform_engine.clips[this.test_box_uuid]); - this.timeline.update(this.clock.getDelta()).catch(data => { - console.log(data); - }); + this.timeline.update(delta_time); this.render_scene(); if (this.capturer != null) { this.capturer.capture(this.renderer.domElement); } // Record scene. diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 42aa46b75e..fbfa90caa0 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -117,7 +117,7 @@ export class TimeLine { for (const element of this.timelineItems) { if (element.type == "transform") { let object = this.scene.get_object_by_uuid(element.object_uuid); - if (object) { this.transformEngine.clips[element.object_uuid].reset(object); } + if (object && this.transformEngine.clips[element.object_uuid]) { this.transformEngine.clips[element.object_uuid].reset(object); } } else if (element.type == "audio") { this.audioEngine.loadClip(element.media_id); @@ -152,7 +152,7 @@ export class TimeLine { // remove the element from the list let object = this.scene.get_object_by_uuid(element.object_uuid) if (element.type == "transform") { - if (object) { + if (object && this.transformEngine.clips[element.object_uuid]) { this.transformEngine.clips[element.object_uuid].length = (element.ending_offset - element.start_offset) this.transformEngine.clips[element.object_uuid].step(object) } From ec304d8546972381114ece6a16d23cf2b1953662 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Sun, 24 Mar 2024 22:56:44 -0400 Subject: [PATCH 05/15] Adding note on endpoint --- .../notes/note_batch_media_endpoint.json | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 app/pages/PageEnigma/notes/note_batch_media_endpoint.json diff --git a/app/pages/PageEnigma/notes/note_batch_media_endpoint.json b/app/pages/PageEnigma/notes/note_batch_media_endpoint.json new file mode 100644 index 0000000000..39f2b788f5 --- /dev/null +++ b/app/pages/PageEnigma/notes/note_batch_media_endpoint.json @@ -0,0 +1,113 @@ +https://api.fakeyou.com/v1/media_files/batch?tokens=m_jh74zz1nfj2jxrkw7n00yamt32zrhb&tokens=m_0rqwx4kbnn2gz5s37apzftkw25vkzb&tokens=m_je04hbk7bqjdq9cnegp1j10mf4dzdb +{ + "success": true, + "media_files": [ + { + "token": "m_0rqwx4kbnn2gz5s37apzftkw25vkzb", + "media_type": "video", + "media_class": "unknown", + "maybe_media_subtype": null, + "maybe_engine_extension": null, + "maybe_batch_token": null, + "public_bucket_path": "/media/c/0/n/g/p/c0ngp9c570w66q3vhs0ygmwpasf2sfk1/videoc0ngp9c570w66q3vhs0ygmwpasf2sfk1mp4", + "maybe_model_weight_info": null, + "maybe_creator_user": { + "user_token": "U:937C1VN0S5G8H", + "username": "echelon", + "display_name": "echelon", + "gravatar_hash": "37e7e18f19d8f654a1ace405c846a9de", + "default_avatar": { + "image_index": 10, + "color_index": 3 + } + }, + "creator_set_visibility": "private", + "maybe_text_transcript": null, + "maybe_prompt_token": "prompt_3sf5p1n7g3sna6pc44nqgpvyd", + "maybe_original_filename": null, + "is_emulated_media_file": false, + "stats": { + "positive_rating_count": 0, + "bookmark_count": 0 + }, + "created_at": "2024-03-25T02:25:18Z", + "updated_at": "2024-03-25T02:25:18Z" + }, + { + "token": "m_je04hbk7bqjdq9cnegp1j10mf4dzdb", + "media_type": "video", + "media_class": "unknown", + "maybe_media_subtype": null, + "maybe_engine_extension": null, + "maybe_batch_token": null, + "public_bucket_path": "/media/d/a/3/b/6/da3b640rh3zjj1easv96vc4d7v4jq8c1/da3b640rh3zjj1easv96vc4d7v4jq8c1.mp4", + "maybe_model_weight_info": null, + "maybe_creator_user": { + "user_token": "U:937C1VN0S5G8H", + "username": "echelon", + "display_name": "echelon", + "gravatar_hash": "37e7e18f19d8f654a1ace405c846a9de", + "default_avatar": { + "image_index": 10, + "color_index": 3 + } + }, + "creator_set_visibility": "public", + "maybe_text_transcript": null, + "maybe_prompt_token": null, + "maybe_original_filename": null, + "is_emulated_media_file": false, + "stats": { + "positive_rating_count": 0, + "bookmark_count": 0 + }, + "created_at": "2024-03-25T02:04:22Z", + "updated_at": "2024-03-25T02:04:22Z" + }, + { + "token": "m_jh74zz1nfj2jxrkw7n00yamt32zrhb", + "media_type": "video", + "media_class": "unknown", + "maybe_media_subtype": null, + "maybe_engine_extension": null, + "maybe_batch_token": null, + "public_bucket_path": "/media/a/c/s/z/e/acszeddd464edypg9hcp5cga0482sjfv/videoacszeddd464edypg9hcp5cga0482sjfvmp4", + "maybe_model_weight_info": null, + "maybe_creator_user": { + "user_token": "U:937C1VN0S5G8H", + "username": "echelon", + "display_name": "echelon", + "gravatar_hash": "37e7e18f19d8f654a1ace405c846a9de", + "default_avatar": { + "image_index": 10, + "color_index": 3 + } + }, + "creator_set_visibility": "private", + "maybe_text_transcript": null, + "maybe_prompt_token": "prompt_dc8x82w2gqsvbpmp5ksxypssj", + "maybe_original_filename": null, + "is_emulated_media_file": false, + "stats": { + "positive_rating_count": 0, + "bookmark_count": 0 + }, + "created_at": "2024-03-25T02:12:46Z", + "updated_at": "2024-03-25T02:12:46Z" + } + ] +} + +// lip sync will be generated through TTS +async get_media_url() { + //This is for prod when we have the proper info on the url. + let api_base_url = "https://api.fakeyou.com/"; + let url = ${api_base_url}/v1/media_files/file/${this.media_id} + let responce = await fetch(url); + let json = await JSON.parse(await responce.text()); + let bucketPath = json["media_file"]["public_bucket_path"]; + let media_base_url = "https://storage.googleapis.com/vocodes-public" + let media_url = ${media_base_url}${bucketPath} + return media_url; + } +https://storage.googleapis.com/vocodes-public \ No newline at end of file From b657abe90d513b21d6d277c035f1618157242233 Mon Sep 17 00:00:00 2001 From: wil Date: Mon, 25 Mar 2024 10:40:08 -0400 Subject: [PATCH 06/15] editor engine is now in wrapped in a context and provided throughout the app --- app/contexts/EngineContext.tsx | 4 + app/contexts/EngineProvider.tsx | 30 +++++ app/pages/PageEnigma/PageEnigma.tsx | 121 ++++++------------ .../ControlsTopButtons/ControlsTopButtons.tsx | 43 +++++++ .../comps/ControlsTopButtons/index.tsx | 2 + .../PageEnigma/comps/Timeline/Timeline.tsx | 12 +- 6 files changed, 127 insertions(+), 85 deletions(-) create mode 100644 app/contexts/EngineContext.tsx create mode 100644 app/contexts/EngineProvider.tsx create mode 100644 app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx create mode 100644 app/pages/PageEnigma/comps/ControlsTopButtons/index.tsx diff --git a/app/contexts/EngineContext.tsx b/app/contexts/EngineContext.tsx new file mode 100644 index 0000000000..9da0df1808 --- /dev/null +++ b/app/contexts/EngineContext.tsx @@ -0,0 +1,4 @@ +import { createContext } from "react"; +import Editor from "~/pages/PageEnigma/js/editor"; + +export const EngineContext = createContext(null); \ No newline at end of file diff --git a/app/contexts/EngineProvider.tsx b/app/contexts/EngineProvider.tsx new file mode 100644 index 0000000000..d5124cb01d --- /dev/null +++ b/app/contexts/EngineProvider.tsx @@ -0,0 +1,30 @@ +import { + ReactNode, + useLayoutEffect, + useRef, +} from "react"; +import { EngineContext } from "./EngineContext"; +import Editor from "~/pages/PageEnigma/js/editor"; + + +interface Props{ + children:ReactNode; +} + +export const EngineProvider = ({ children }: Props)=>{ + const editorRef = useRef(null); + + useLayoutEffect(() => { + //componentDidMount + if (editorRef.current == null) { + editorRef.current = new Editor(); + editorRef.current.initialize(); + } + }, []); + + return( + + {children} + + ); +} \ No newline at end of file diff --git a/app/pages/PageEnigma/PageEnigma.tsx b/app/pages/PageEnigma/PageEnigma.tsx index 0b72e150d1..30bda49cb5 100644 --- a/app/pages/PageEnigma/PageEnigma.tsx +++ b/app/pages/PageEnigma/PageEnigma.tsx @@ -1,6 +1,6 @@ import { useCallback, useEffect, useRef, useState } from "react"; -import { faWandSparkles } from "@fortawesome/pro-solid-svg-icons"; +import { faSparkles } from "@fortawesome/pro-solid-svg-icons"; import { Button, ButtonLink } from "~/components"; import { ButtonDialogue } from "~/modules/ButtonDialogue"; @@ -8,45 +8,20 @@ import { TopBarHelmet } from "~/modules/TopBarHelmet/TopBarHelmet"; import { SidePanel } from "~/modules/SidePanel"; import { Tabs } from "~/modules/Tabs"; import { Controls3D } from "./comps/Controls3D"; +import { ControlsTopButtons } from "./comps/ControlsTopButtons"; import { ControlsVideo } from "./comps/ControlsVideo"; import { Timeline } from "./comps/Timeline"; - -import Editor from "./js/editor"; -import { TrackContext } from "~/contexts/TrackContext"; import { TrackProvider } from "~/contexts/TrackProvider"; +import { EngineProvider } from "~/contexts/EngineProvider"; export const PageEnigma = () => { - // const { setTopBarInner } = useContext(TopBarInnerContext) || {}; - const canvasRef = useRef(null); - const editorRef = useRef(null); + const [timelineHeight, setTimelineHeight] = useState(0); const timelineRef = useRef(null); - const editorCallback = useCallback(() => { - // handle editorCallback here - }, []); - useEffect(() => { - //componentDidMount - if (editorRef.current == null) { - editorRef.current = new Editor(); - editorRef.current.initialize(); - } - }, []); - - const handleButtonSave = () => { - editorRef.current?.save(); - }; - - const handleButtonCameraView = () => { - editorRef.current?.change_camera_view(); - }; - - const handleButtonPlayBack = () => { - editorRef.current?.start_playback(); - }; const updateTimelineHeight = useCallback(() => { if (timelineRef.current) { @@ -72,68 +47,52 @@ export const PageEnigma = () => { return (
- + + +
+ {/* Engine section/side panel */} +
+
+ -
- {/* Engine section/side panel */} -
-
- - - {/* Top controls */} -
-
-
- - - -

Do you need help?

-

Ask Michael about this project

-

Ask Miles about ThreeJS

-

Ask Wil about React

- + {/* Top controls */} +
+
+ +
+
- + {/* Bottom controls */} +
+
- {/* Bottom controls */} -
- -
+ {/* Side panel */} +
- {/* Side panel */} - + {/* Timeline */} +
+ + + +
- - {/* Timeline */} -
- - - -
-
+
); }; diff --git a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx new file mode 100644 index 0000000000..f42211022b --- /dev/null +++ b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx @@ -0,0 +1,43 @@ +import { useContext } from "react"; +import { Button } from "~/components"; +import { ButtonDialogue } from "~/modules/ButtonDialogue"; +import { EngineContext } from "~/contexts/EngineContext"; + + +export const ControlsTopButtons = () => { + const editorEngine = useContext(EngineContext); + + const handleButtonSave = () => { + editorEngine?.save(); + }; + + const handleButtonCameraView = () => { + editorEngine?.change_camera_view(); + }; + + const handleButtonPlayBack = () => { + editorEngine?.start_playback(); + }; + return( +
+ + + +

Do you need help?

+

Ask Michael about this project

+

Ask Miles about ThreeJS

+

Ask Wil about React

+ +
+ ); +}; \ No newline at end of file diff --git a/app/pages/PageEnigma/comps/ControlsTopButtons/index.tsx b/app/pages/PageEnigma/comps/ControlsTopButtons/index.tsx new file mode 100644 index 0000000000..6e31c86712 --- /dev/null +++ b/app/pages/PageEnigma/comps/ControlsTopButtons/index.tsx @@ -0,0 +1,2 @@ +export { ControlsTopButtons } from './ControlsTopButtons'; + diff --git a/app/pages/PageEnigma/comps/Timeline/Timeline.tsx b/app/pages/PageEnigma/comps/Timeline/Timeline.tsx index 73ee72e905..3b969b474e 100644 --- a/app/pages/PageEnigma/comps/Timeline/Timeline.tsx +++ b/app/pages/PageEnigma/comps/Timeline/Timeline.tsx @@ -1,24 +1,28 @@ import { LowerPanel } from "~/modules/LowerPanel"; -import Editor from "../../js/editor"; +// import Editor from "../../js/editor"; import { Character } from "./Character"; import { useCallback, useContext } from "react"; +import { EngineContext } from "~/contexts/EngineContext"; import { TrackContext } from "~/contexts/TrackContext"; import { Button } from "~/components"; export const Timeline = ({ - editorCurrent, + // editorCurrent, time, }: { - editorCurrent: Editor | null; + // editorCurrent: Editor | null; time: number; }) => { + const editorEngine = useContext(EngineContext); + // editorEngine replaced the engineRef.current passed as a prop + const { characters, updateCharacters } = useContext(TrackContext); const handleButtonLoad = () => { document.getElementById("load-upload")?.click(); }; const handleButtonRender = () => { - editorCurrent?.togglePlayback(); + editorEngine?.togglePlayback(); }; const handleButtonPlay = () => {}; From 06a089c44b155e616232017490d3323da956103e Mon Sep 17 00:00:00 2001 From: wil Date: Mon, 25 Mar 2024 10:53:59 -0400 Subject: [PATCH 07/15] use useEffect --- app/contexts/EngineProvider.tsx | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/app/contexts/EngineProvider.tsx b/app/contexts/EngineProvider.tsx index d5124cb01d..7c708816c4 100644 --- a/app/contexts/EngineProvider.tsx +++ b/app/contexts/EngineProvider.tsx @@ -1,6 +1,6 @@ import { ReactNode, - useLayoutEffect, + useEffect, useRef, } from "react"; import { EngineContext } from "./EngineContext"; @@ -14,17 +14,23 @@ interface Props{ export const EngineProvider = ({ children }: Props)=>{ const editorRef = useRef(null); - useLayoutEffect(() => { + + useEffect(() => { //componentDidMount - if (editorRef.current == null) { + if (editorRef.current !== null ) { + console.warn('Editor Engine already exist'); + }else if (document.getElementById("video-scene") === null){ + console.error('Editor Engine need a target cavas with the id "video-scene"'); + }else{ editorRef.current = new Editor(); editorRef.current.initialize(); } }, []); - + return( {children} ); -} \ No newline at end of file +} + From 8188b5085e9280d382fc986b935b76863e0045cf Mon Sep 17 00:00:00 2001 From: wil Date: Mon, 25 Mar 2024 13:36:12 -0400 Subject: [PATCH 08/15] styling panel --- app/components/H4.tsx | 19 +++++++ app/components/ItemPicker.tsx | 17 ++++++ app/components/Textarea.tsx | 17 ++++++ app/components/index.tsx | 7 ++- app/modules/SidePanel/SidePanel.tsx | 30 +++------- app/pages/PageEnigma/PageEnigma.tsx | 9 ++- .../comps/SidePanelTabs/TabStyling.tsx | 53 ++++++++++++++++++ .../PageEnigma/comps/SidePanelTabs/index.tsx | 28 +++++++++ public/resources/avatars/0.webp | Bin 0 -> 75364 bytes public/resources/avatars/1.webp | Bin 0 -> 72420 bytes public/resources/avatars/10.webp | Bin 0 -> 69366 bytes public/resources/avatars/11.webp | Bin 0 -> 71656 bytes public/resources/avatars/12.webp | Bin 0 -> 78702 bytes public/resources/avatars/13.webp | Bin 0 -> 75914 bytes public/resources/avatars/14.webp | Bin 0 -> 72844 bytes public/resources/avatars/15.webp | Bin 0 -> 75094 bytes public/resources/avatars/16.webp | Bin 0 -> 68960 bytes public/resources/avatars/17.webp | Bin 0 -> 72890 bytes public/resources/avatars/18.webp | Bin 0 -> 70348 bytes public/resources/avatars/19.webp | Bin 0 -> 70496 bytes public/resources/avatars/2.webp | Bin 0 -> 74462 bytes public/resources/avatars/20.webp | Bin 0 -> 76262 bytes public/resources/avatars/21.webp | Bin 0 -> 73002 bytes public/resources/avatars/22.webp | Bin 0 -> 77762 bytes public/resources/avatars/23.webp | Bin 0 -> 77922 bytes public/resources/avatars/24.webp | Bin 0 -> 83286 bytes public/resources/avatars/3.webp | Bin 0 -> 61554 bytes public/resources/avatars/4.webp | Bin 0 -> 65758 bytes public/resources/avatars/5.webp | Bin 0 -> 68836 bytes public/resources/avatars/6.webp | Bin 0 -> 78026 bytes public/resources/avatars/7.webp | Bin 0 -> 70572 bytes public/resources/avatars/8.webp | Bin 0 -> 78934 bytes public/resources/avatars/9.webp | Bin 0 -> 65826 bytes 33 files changed, 155 insertions(+), 25 deletions(-) create mode 100644 app/components/H4.tsx create mode 100644 app/components/ItemPicker.tsx create mode 100644 app/components/Textarea.tsx create mode 100644 app/pages/PageEnigma/comps/SidePanelTabs/TabStyling.tsx create mode 100644 app/pages/PageEnigma/comps/SidePanelTabs/index.tsx create mode 100644 public/resources/avatars/0.webp create mode 100644 public/resources/avatars/1.webp create mode 100644 public/resources/avatars/10.webp create mode 100644 public/resources/avatars/11.webp create mode 100644 public/resources/avatars/12.webp create mode 100644 public/resources/avatars/13.webp create mode 100644 public/resources/avatars/14.webp create mode 100644 public/resources/avatars/15.webp create mode 100644 public/resources/avatars/16.webp create mode 100644 public/resources/avatars/17.webp create mode 100644 public/resources/avatars/18.webp create mode 100644 public/resources/avatars/19.webp create mode 100644 public/resources/avatars/2.webp create mode 100644 public/resources/avatars/20.webp create mode 100644 public/resources/avatars/21.webp create mode 100644 public/resources/avatars/22.webp create mode 100644 public/resources/avatars/23.webp create mode 100644 public/resources/avatars/24.webp create mode 100644 public/resources/avatars/3.webp create mode 100644 public/resources/avatars/4.webp create mode 100644 public/resources/avatars/5.webp create mode 100644 public/resources/avatars/6.webp create mode 100644 public/resources/avatars/7.webp create mode 100644 public/resources/avatars/8.webp create mode 100644 public/resources/avatars/9.webp diff --git a/app/components/H4.tsx b/app/components/H4.tsx new file mode 100644 index 0000000000..bfbf79d630 --- /dev/null +++ b/app/components/H4.tsx @@ -0,0 +1,19 @@ +import { twMerge } from "tailwind-merge" + +export const H4 = ({ + className, + children, +}:{ + className?: string, + children: React.ReactNode, +})=>{ + return( +

+ {children} +

+ ) +} \ No newline at end of file diff --git a/app/components/ItemPicker.tsx b/app/components/ItemPicker.tsx new file mode 100644 index 0000000000..90eae68411 --- /dev/null +++ b/app/components/ItemPicker.tsx @@ -0,0 +1,17 @@ +import { H4 } from "."; + +interface ItemPickerProps extends React.ImgHTMLAttributes { + label: string +} + +export const ItemPicker = ({ + label, + ...imgProps +}:ItemPickerProps) => { + return ( +
+ +

{label}

+
+ ) +} \ No newline at end of file diff --git a/app/components/Textarea.tsx b/app/components/Textarea.tsx new file mode 100644 index 0000000000..f9ca528ab9 --- /dev/null +++ b/app/components/Textarea.tsx @@ -0,0 +1,17 @@ +import { TextareaHTMLAttributes } from "react" +import { twMerge } from "tailwind-merge"; + +export const Textarea = ({ + className, + ...rest +}:TextareaHTMLAttributes) => { + return( +