diff --git a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts index 5508f3976a..154c630561 100644 --- a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts +++ b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts @@ -1,16 +1,6 @@ -export interface ClipUI { - version: number - name: string - type: "transform" | "audio" | "animation" | "lipsync" - media_id: string - object_uuid: string - start_offset: number // in frames so 60 frames per second on the timeline. - ending_offset: number // ending offset -} - // Clip offsets represent the state of the clip on the timeline as well as what type of clip it is. // it is created from a media id. -export class ClipUI implements ClipUI { +export class ClipUI { version: number type: "transform" | "audio" | "animation" | "lipsync" name: string @@ -21,8 +11,8 @@ export class ClipUI implements ClipUI { constructor( version: number, - name: string, type: "transform" | "audio" | "animation" | "lipsync", + name: string, media_id: string, object_uuid: string, start_offset: number, diff --git a/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts index 8030039a11..412b8b0bc3 100644 --- a/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts @@ -1,99 +1,134 @@ import { LipSync } from "../../js/lipsync"; import * as THREE from 'three'; interface AudioData { - audioContext: AudioContext; - audioBuffer: AudioBuffer; + audioContext: AudioContext; + audioBuffer: AudioBuffer; +} + +class AudioData implements AudioData { + audioContext: AudioContext; + audioBuffer: AudioBuffer; + source: AudioBufferSourceNode | undefined + + constructor(audioContext: AudioContext, audioBuffer: AudioBuffer) { + this.audioContext = audioContext; + this.audioBuffer = audioBuffer; } - - class AudioData implements AudioData{ - audioContext: AudioContext; - audioBuffer: AudioBuffer; - source: AudioBufferSourceNode | undefined - - constructor (audioContext: AudioContext, audioBuffer: AudioBuffer) { - this.audioContext = audioContext; - this.audioBuffer = audioBuffer; - } +} + +export interface LipSyncTrackClip { + version: number; + media_id: string; + type: "lipsync"; + volume: number; +} + +class BlendShapeHelper { + ah: number; + ee: number; + oh: number; + constructor(ah: number, ee: number, oh: number) { + this.ah = ah; + this.ee = ee; + this.oh = oh; } - - export interface LipSyncTrackClip { - version: number; - media_id: string; - type: "lipsync"; - volume: number; +} + +export class LipSyncTrackClip implements LipSyncTrackClip { + version: number; + media_id: string; + type: "lipsync" = "lipsync"; + volume: number; + audio_data: AudioData | undefined; + lipsync: LipSync; + blendshape_helper: BlendShapeHelper; + + constructor(version: number, media_id: string, volume: number) { + this.version = version; + this.media_id = media_id; + this.type = "lipsync"; + this.volume = volume; + this.download_audio().then(data => { + this.audio_data = data; + }); + this.blendshape_helper = new BlendShapeHelper(0, 0, 0); + // we might need 3 of these one for each character ... + this.lipsync = new LipSync(); } - - export class LipSyncTrackClip implements LipSyncTrackClip { - version: number; - media_id: string; - type: "lipsync" = "lipsync"; - volume: number; - audio_data: AudioData | undefined; - lipsync: LipSync; - - constructor(version: number, media_id: string, volume: number) { - this.version = version; - this.media_id = media_id; - this.type = "lipsync"; - this.volume = volume; - this.download_audio().then(data => { - this.audio_data = data; + + // lip sync will be generated through TTS + async get_media_url() { + //This is for prod when we have the proper info on the url. + let api_base_url = "https://api.fakeyou.com"; + let url = `${api_base_url}/v1/media_files/file/${this.media_id}` + let responce = await fetch(url); + let json = await JSON.parse(await responce.text()); + let bucketPath = json["media_file"]["public_bucket_path"]; + let media_base_url = "https://storage.googleapis.com/vocodes-public" + let media_url = `${media_base_url}${bucketPath}` + return media_url; + } + + async download_audio() { + let url = await this.get_media_url(); + const audioContext = new AudioContext(); + const response = await fetch(url); + const arrayBuffer = await response.arrayBuffer(); + const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); + return new AudioData(audioContext, audioBuffer); + } + + async _detect_face(object: THREE.Object3D): Promise { + return new Promise((resolve) => { + object.traverse((c: THREE.Object3D) => { + if (c instanceof THREE.Mesh) { + if (c.morphTargetInfluences && c.morphTargetDictionary) { + const blendShapeIndexE = c.morphTargetDictionary["E"]; + const blendShapeIndexO = c.morphTargetDictionary["O"]; + const blendShapeIndexA = c.morphTargetDictionary["aa"]; + if (blendShapeIndexE != null) { + this.blendshape_helper = new BlendShapeHelper(blendShapeIndexA, blendShapeIndexE, blendShapeIndexO); + resolve(c); + return; + } + } + } }); - - // we might need 3 of these one for each character ... - this.lipsync = new LipSync() - } - - // lip sync will be generated through TTS - async get_media_url() { - //This is for prod when we have the proper info on the url. - let api_base_url = "https://api.fakeyou.com"; - let url = `${api_base_url}/v1/media_files/file/${this.media_id}` - let responce = await fetch(url); - let json = await JSON.parse(await responce.text()); - let bucketPath = json["media_file"]["public_bucket_path"]; - let media_base_url = "https://storage.googleapis.com/vocodes-public" - let media_url = `${media_base_url}${bucketPath}` - return media_url; - } - - async download_audio() { - let url = await this.get_media_url(); - const audioContext = new AudioContext(); - const response = await fetch(url); - const arrayBuffer = await response.arrayBuffer(); - const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); - return new AudioData(audioContext, audioBuffer); - } - - play(face: THREE.Object3D) { - if (this.audio_data?.audioBuffer == null) { this.download_audio(); } - this.lipsync = new LipSync() - this.lipsync.face = face; - this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer); - } - - stop() { - if(this.lipsync == null) { return; } - this.lipsync.destroy(); - } - - step() { - if(this.lipsync == null) { return; } - this.lipsync.update(); - } - - reset() { - this.lipsync = new LipSync() - } - - toJSON(): string { - return JSON.stringify({ - version: this.version, - media_id: this.media_id, - type: this.type, - volume: this.volume, - }); - } + }); } - \ No newline at end of file + + + async play(object: THREE.Object3D) { + if (this.audio_data?.audioBuffer == null) { await this.download_audio(); } + if (this.lipsync.face == null) { + this.lipsync = new LipSync(await this._detect_face(object)); + this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer); + } +} + +stop() { + if (this.lipsync == null) { return; } + this.lipsync.destroy(); +} + +step() { + if (this.lipsync == null) { return; } + const positions = this.lipsync.update(); + this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ee] = positions["ee"]; + this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ah] = positions["ah"]; + this.lipsync.face.morphTargetInfluences[this.blendshape_helper.oh] = positions["oh"]; +} + +reset() { + this.lipsync = new LipSync(); +} + +toJSON(): string { + return JSON.stringify({ + version: this.version, + media_id: this.media_id, + type: this.type, + volume: this.volume, + }); +} +} diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index c3e44ac9a1..2fce1d9e1b 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -155,8 +155,6 @@ class Editor { this.timeline.scene = this.activeScene; - - this._test_demo() } @@ -169,7 +167,7 @@ class Editor { let object = await this.activeScene.load_glb("./resources/models/fox/fox.glb") this.lipsync_engine.load_object(object.uuid, "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72"); - this.timeline.addPlayableClip(new ClipUI(1.0, "lipsync", object.object_uuid, object.media_id, 0, 100)); + this.timeline.addPlayableClip(new ClipUI(1.0, "lipsync", "clip1", "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", object.uuid, 10, 200)); } // Configure post processing. @@ -278,11 +276,12 @@ class Editor { //console.log(this.transform_engine.clips[this.test_box_uuid]); - this.timeline.update(this.clock.getDelta()); + this.timeline.update(this.clock.getDelta()).catch(data => { + console.log(data); + }); this.render_scene(); if (this.capturer != null) { this.capturer.capture(this.renderer.domElement); } // Record scene. - } start_playback() { diff --git a/app/pages/PageEnigma/js/lipsync.js b/app/pages/PageEnigma/js/lipsync.js index 5db5dc24e3..a70ab40cf5 100644 --- a/app/pages/PageEnigma/js/lipsync.js +++ b/app/pages/PageEnigma/js/lipsync.js @@ -18,7 +18,7 @@ for (let m = 0; m < BoundingFrequencyFem.length; m++) { } export class LipSync { - constructor(face=null) { + constructor(face) { this.face = face; @@ -90,19 +90,15 @@ export class LipSync { update() { if (this.meter) { const { volume } = this.meter; + let { ah, oh, ee } = this.update2(); if (volume < 0.01) { - // get the shape keys from the face avatar - - this.face.morphTargetInfluences[this.ee] = 0; - this.face.morphTargetInfluences[this.ah] = 0; - this.face.morphTargetInfluences[this.oh] = 0; - } else { - const { ah, oh, ee } = this.update2(); - this.face.morphTargetInfluences[this.ee] = ee; - this.face.morphTargetInfluences[this.ah] = ah; - this.face.morphTargetInfluences[this.oh] = oh; + ah = 0; + oh = 0; + ee = 0; } + return { "ah": ah, "oh": oh, "ee": ee }; } + return { "ah": 0, "oh": 0, "ee": 0 }; } update2() { diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 5e64964371..42aa46b75e 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -123,6 +123,9 @@ export class TimeLine { this.audioEngine.loadClip(element.media_id); } else if (element.type == "animation") { + } + else if (element.type == "lipsync") { + } else { this.stop() @@ -140,10 +143,8 @@ export class TimeLine { } this.scrubberPosition += 1; - //2. allow stopping. //3. smallest unit is a frame and it is set by the scene and is in fps, our videos will be 60fps but we can reprocess them using the pipeline. - for (const element of this.timelineItems) { if (element.start_offset <= this.scrubberPosition && this.scrubberPosition <= element.ending_offset) { // run async @@ -161,15 +162,15 @@ export class TimeLine { this.audioEngine.playClip(element.media_id) } else if (element.type == "lipsync") { - let face_object = this.scene.get_object_by_uuid(element.object_uuid); - if (face_object) { - this.lipSyncEngine.clips[element.object_uuid].play(face_object); + if(this.scrubberPosition+1 >= element.ending_offset){ + this.lipSyncEngine.clips[element.object_uuid].stop(); + } + else if (object) { + await this.lipSyncEngine.clips[element.object_uuid].play(object); this.lipSyncEngine.clips[element.object_uuid].step(); } } - else if (element.type == "animation") { - let object = this.scene.get_object_by_uuid(element.object_uuid); if (object) { await this.animationEngine.clips[object.uuid].play(object); this.animationEngine.clips[object.uuid].step(deltatime);