diff --git a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx index e426334ad3..e2008b8c70 100644 --- a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx +++ b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx @@ -103,7 +103,7 @@ export const TrackProvider = ({ children }: Props) => { }, { version: 1, - media_id: uuid.v4(), + media_id: "m_w5t517krrf63f3tj2288vsffmj87zw", type: ClipType.ANIMATION, length: 25, name: "Walk", diff --git a/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts index 3b526f21ca..77fdd5f6f6 100644 --- a/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts @@ -35,6 +35,7 @@ export class LipSyncClip { audio_data: AudioData | undefined; lipsync: LipSync; blendshape_helper: BlendShapeHelper; + faces: THREE.Mesh[] constructor(version: number, media_id: string, volume: number) { this.version = version; @@ -47,9 +48,7 @@ export class LipSyncClip { this.blendshape_helper = new BlendShapeHelper(0, 0, 0); // we might need 3 of these one for each character ... this.lipsync = new LipSync(); - - - console.log("Lip!") + this.faces = []; } // lip sync will be generated through TTS @@ -84,8 +83,8 @@ export class LipSyncClip { const blendShapeIndexA = c.morphTargetDictionary["aa"]; if (blendShapeIndexE != null) { this.blendshape_helper = new BlendShapeHelper(blendShapeIndexA, blendShapeIndexE, blendShapeIndexO); + this.faces.push(c); resolve(c); - return; } } } @@ -94,44 +93,48 @@ export class LipSyncClip { } -async play(object: THREE.Object3D) { - if (this.audio_data?.audioBuffer == null) { await this.download_audio(); } - if (this.lipsync.face == null) { - this.lipsync = new LipSync(await this._detect_face(object)); - console.log("Face", this.lipsync.face); - this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer); + async play(object: THREE.Object3D) { + if (this.audio_data?.audioBuffer == null) { await this.download_audio(); } + if (this.lipsync.face == null) { + this.lipsync = new LipSync(await this._detect_face(object)); + this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer); + } + } + + stop() { + if (this.lipsync == null) { return; } + this.lipsync.destroy(); + } + + setBlends(ah: number, ee: number, oh: number) { + this.faces.forEach((element: THREE.Mesh) => { + if (element.morphTargetInfluences) { + element.morphTargetInfluences[this.blendshape_helper.ee] = ee; + element.morphTargetInfluences[this.blendshape_helper.ah] = ah; + element.morphTargetInfluences[this.blendshape_helper.oh] = oh; + } + }); + } + + step() { + if (this.lipsync == null) { return; } + const positions = this.lipsync.update(); + this.setBlends(positions["ee"], positions["ah"], positions["oh"]); + } + + reset() { + if (this.lipsync.face != undefined) { + this.setBlends(0,0,0); + } + this.lipsync = new LipSync(); + } + + toJSON(): string { + return JSON.stringify({ + version: this.version, + media_id: this.media_id, + type: this.type, + volume: this.volume, + }); } } - -stop() { - if (this.lipsync == null) { return; } - this.lipsync.destroy(); -} - -step() { - if (this.lipsync == null) { return; } - const positions = this.lipsync.update(); - console.log("Face", this.lipsync.face); - this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ee] = positions["ee"]; - this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ah] = positions["ah"]; - this.lipsync.face.morphTargetInfluences[this.blendshape_helper.oh] = positions["oh"]; -} - -reset() { - if (this.lipsync.face != undefined) { - this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ee] = 0; - this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ah] = 0 - this.lipsync.face.morphTargetInfluences[this.blendshape_helper.oh] = 0; - } - this.lipsync = new LipSync(); -} - -toJSON(): string { - return JSON.stringify({ - version: this.version, - media_id: this.media_id, - type: this.type, - volume: this.volume, - }); -} -} diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 55e4911521..367670be48 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -652,7 +652,7 @@ class Editor { // note the database from the server is the source of truth for all the data. // Test code here const object: THREE.Object3D = await this.activeScene.load_glb( - "./resources/models/fox/fox.glb", + "m_9f3d3z94kk6m25zywyz6an3p43fjtw", ); object.uuid = "CH1"; @@ -851,6 +851,9 @@ class Editor { } else if(this.timeline.is_playing == false) { this.cam_obj.position.copy(this.camera.position); this.cam_obj.rotation.copy(this.camera.rotation); + } else { + this.camera.position.copy(this.cam_obj.position); + this.camera.rotation.copy(this.cam_obj.rotation); } } } @@ -1019,9 +1022,9 @@ class Editor { URL.revokeObjectURL(url); document.body.removeChild(downloadLink); - const data = await this.api_manager.uploadMedia(blob, "tmp.wav"); - console.log(data); + const data = await this.api_manager.uploadMedia(blob, "render.mp4"); // Create a link to download the file + } async generateFrame() { diff --git a/app/pages/PageEnigma/js/scene.ts b/app/pages/PageEnigma/js/scene.ts index 81367ce85f..0228012696 100644 --- a/app/pages/PageEnigma/js/scene.ts +++ b/app/pages/PageEnigma/js/scene.ts @@ -77,7 +77,7 @@ class Scene { } _create_camera_obj() { - this.load_glb("/resources/models/camera/camera.glb", false).then((cam_obj) => { + this.load_glb("m_cxh4asqhapdz10j880755dg4yevshb", false).then((cam_obj) => { cam_obj.userData["name"] = "::CAM::"; cam_obj.name = "::CAM::"; cam_obj.position.set(0, 0.6, 1.5); @@ -97,7 +97,45 @@ class Scene { } } - async load_glb(filepath: string, auto_add: boolean = true): Promise { //: Promise { + async getMediaURL(media_id: string) { + //This is for prod when we have the proper info on the url. + let api_base_url = "https://api.fakeyou.com"; + let url = `${api_base_url}/v1/media_files/file/${media_id}` + let responce = await fetch(url); + let json = await JSON.parse(await responce.text()); + let bucketPath = json["media_file"]["public_bucket_path"]; + let media_base_url = "https://storage.googleapis.com/vocodes-public" + let media_url = `${media_base_url}${bucketPath}` + return media_url; + } + + async load_glb(media_id: string, auto_add: boolean = true): Promise { //: Promise { + return new Promise(async (resolve) => { + let glbLoader = new GLTFLoader(); + glbLoader.load(await this.getMediaURL(media_id), (glb) => { + glb.scene.children.forEach(child => { + child.traverse((c: THREE.Object3D) => { + if (c instanceof THREE.Mesh) { + c.material.metalness = 0.0; + c.material.specular = 0.5; + c.castShadow = true; + c.receiveShadow = true; + c.material.transparent = false; + } + }); + if (child.type == "Group") { + if (auto_add) { this.scene.add(child.children[0]); } + resolve(child.children[0]); + return; + } + if (auto_add) { this.scene.add(child); } + resolve(child); + }); + }); + }); + } + + async load_glb_absolute(filepath: string, auto_add: boolean = true): Promise { //: Promise { return new Promise((resolve) => { let glbLoader = new GLTFLoader(); glbLoader.load(filepath, (glb) => { diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 5fe9e27621..73ab488526 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -392,7 +392,7 @@ export class TimeLine { if (object) { await this.animation_engine.clips[object.uuid].play(object); this.animation_engine.clips[object.uuid].step( - this.scrubber_frame_position / 60, + this.scrubber_frame_position / 120, // Double FPS for best result. ); } } else { diff --git a/public/resources/models/pose/blend_pose.glb b/public/resources/models/pose/blend_pose.glb new file mode 100644 index 0000000000..3cce1fa632 Binary files /dev/null and b/public/resources/models/pose/blend_pose.glb differ