diff --git a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts index b03e0e7d8f..9225276b74 100644 --- a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts +++ b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts @@ -1,8 +1,8 @@ export interface ClipUI { version: number - type: "transform" | "audio" | "animation" - media_id: number - object_uuid: number + type: "transform" | "audio" | "animation" | "lipsync" + media_id: string + object_uuid: string start_offset: number // in frames so 60 frames per second on the timeline. ending_offset: number // ending offset } @@ -11,17 +11,17 @@ export interface ClipUI { // it is created from a media id. export class ClipUI implements ClipUI { version: number - type: "transform" | "audio" | "animation" - media_id: number - object_uuid: number + type: "transform" | "audio" | "animation" | "lipsync" + media_id: string + object_uuid: string start_offset: number // in frames ending_offset: number // in frames constructor( version: number, - type: "transform" | "audio" | "animation", - media_id: number, - object_uuid: number, + type: "transform" | "audio" | "animation" | "lipsync", + media_id: string, + object_uuid: string, start_offset: number, ending_offset: number, ) { @@ -36,8 +36,8 @@ export class ClipUI implements ClipUI { toJSON(): string { return JSON.stringify({ version: this.version, - object_uuid: this.object_uuid, type: this.type, + object_uuid: this.object_uuid, media_id: this.media_id, start_offset: this.start_offset, ending_offset: this.ending_offset diff --git a/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts new file mode 100644 index 0000000000..94d8a49b86 --- /dev/null +++ b/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts @@ -0,0 +1,72 @@ +interface AudioData { + audioContext: AudioContext; + audioBuffer: AudioBuffer; + } + + class AudioData implements AudioData{ + audioContext: AudioContext; + audioBuffer: AudioBuffer; + source: AudioBufferSourceNode | undefined + + constructor (audioContext: AudioContext, audioBuffer: AudioBuffer) { + this.audioContext = audioContext; + this.audioBuffer = audioBuffer; + } + } + + export interface LipSyncTrackClip { + version: number; + media_id: string; + type: "lipsync"; + volume: number; + } + + export class LipSyncTrackClip implements LipSyncTrackClip { + version: number; + media_id: string; + type: "lipsync" = "lipsync"; + volume: number; + audio_data: AudioData | undefined; + + constructor(version: number, media_id: string, volume: number) { + this.version = version; + this.media_id = media_id; + this.type = "lipsync"; + this.volume = volume; + this.download_audio().then(data => { + this.audio_data = data; + }); + } + + // lip sync will be generated through TTS + async get_media_url() { + //This is for prod when we have the proper info on the url. + let api_base_url = "https://api.fakeyou.com"; + let url = `${api_base_url}/v1/media_files/file/${this.media_id}` + let responce = await fetch(url); + let json = await JSON.parse(await responce.text()); + let bucketPath = json["media_file"]["public_bucket_path"]; + let media_base_url = "https://storage.googleapis.com/vocodes-public" + let media_url = `${media_base_url}${bucketPath}` + return media_url; + } + + async download_audio() { + let url = await this.get_media_url(); + const audioContext = new AudioContext(); + const response = await fetch(url); + const arrayBuffer = await response.arrayBuffer(); + const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); + return new AudioData(audioContext, audioBuffer); + } + + toJSON(): string { + return JSON.stringify({ + version: this.version, + media_id: this.media_id, + type: this.type, + volume: this.volume, + }); + } + } + \ No newline at end of file diff --git a/app/pages/PageEnigma/js/animation_engine.ts b/app/pages/PageEnigma/js/animation_engine.ts new file mode 100644 index 0000000000..c8eba78ba9 --- /dev/null +++ b/app/pages/PageEnigma/js/animation_engine.ts @@ -0,0 +1,7 @@ + + +export class AnimationEngine { + + + +} \ No newline at end of file diff --git a/app/pages/PageEnigma/js/audio_engine.ts b/app/pages/PageEnigma/js/audio_engine.ts index 1dff3ee3c6..b7c3b5ee66 100644 --- a/app/pages/PageEnigma/js/audio_engine.ts +++ b/app/pages/PageEnigma/js/audio_engine.ts @@ -1,6 +1,5 @@ import { AudioTrackClip } from "../datastructures/clips/audio_track_clip"; - class AudioEngine { clips: { [key: string]: AudioTrackClip } = {}; audio_sources: { [key: string]: AudioBufferSourceNode } = {}; @@ -12,10 +11,12 @@ class AudioEngine { this.version = 1.0; } + // loads clips into the engine to cache loadClip(audio_media_id: string) { this.clips[audio_media_id] = new AudioTrackClip(this.version, audio_media_id, 1.0); } + // plays from the timeline. playClip(audio_media_id: string) { const clip = this.clips[audio_media_id]; if (clip.audio_data?.audioContext) { @@ -28,6 +29,7 @@ class AudioEngine { } } + // stops the clips stopClip(audio_media_id: string) { const clip = this.clips[audio_media_id]; if (clip.audio_data?.source) { diff --git a/app/pages/PageEnigma/js/lip_sync_engine.ts b/app/pages/PageEnigma/js/lip_sync_engine.ts new file mode 100644 index 0000000000..d6e32c4795 --- /dev/null +++ b/app/pages/PageEnigma/js/lip_sync_engine.ts @@ -0,0 +1,50 @@ + +import { LipSyncTrackClip } from "../datastructures/clips/lipsync_track_clip"; +import { LipSync } from "./lipsync"; +import * as THREE from 'three'; + +// needs to be apart of the editor window. +export class LipSyncEngine { + + clips: { [key: string]: LipSyncTrackClip } = {} + audio_sources: { [key: string]: AudioBufferSourceNode } = {} + version: number + lipsync: LipSync + + constructor() { + + this.clips = {} + this.audio_sources = {} + this.version = 1.0 + // we might need 3 of these one for each character ... + this.lipsync = new LipSync() + } + + loadClip(audio_media_id: string) { + this.clips[audio_media_id] = new LipSyncTrackClip(this.version, audio_media_id, 1.0); + } + + // needs to be called by the engine. + update(delta_time: number) { + this.lipsync.update(delta_time) + } + + playClip(face: THREE.Object3D,audio_media_id: string) { + // face movement here + this.lipsync.face = face + const clip = this.clips[audio_media_id]; + // this is a buffer right? + this.lipsync.startLipSyncFromAudioBuffer(clip) + } + + stopClip(audio_media_id: string) { + const clip = this.clips[audio_media_id]; + if (clip.audio_data?.source) { + clip.audio_data.source.stop(); + } else { + console.warn(`AudioManager: AudioClip with id "${audio_media_id}" not found.`); + } + } +} + +export default LipSyncEngine; diff --git a/app/pages/PageEnigma/js/lipsync.js b/app/pages/PageEnigma/js/lipsync.js index 2233bba786..c72356fd9a 100644 --- a/app/pages/PageEnigma/js/lipsync.js +++ b/app/pages/PageEnigma/js/lipsync.js @@ -7,32 +7,26 @@ const samplingFrequency = 44100; for (let m = 0; m < BoundingFrequencyMasc.length; m++) { IndicesFrequencyMale[m] = Math.round( - ((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyMasc[m] + ((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyMasc[m], ); } for (let m = 0; m < BoundingFrequencyFem.length; m++) { IndicesFrequencyFemale[m] = Math.round( - ((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyFem[m] + ((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyFem[m], ); } export class LipSync { - constructor(vrm) { - this.vrm = vrm; - - this.ee = this.vrm.morphTargetDictionary["E"]; - this.oh = this.vrm.morphTargetDictionary["O"]; - this.ah = this.vrm.morphTargetDictionary["aa"]; - - const update = (deltaTime, elapsedTime) => { - requestAnimationFrame(update); - this.update(deltaTime, elapsedTime); - }; - - update(); + constructor(face) { + this.face = face; + // we have to modify this for our + this.ee = this.face.morphTargetDictionary["E"]; + this.oh = this.face.morphTargetDictionary["O"]; + this.ah = this.face.morphTargetDictionary["aa"]; } + // for streaming sound? start(stream) { this.audioContext = new AudioContext(); this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream); @@ -40,6 +34,7 @@ export class LipSync { this.mediaStreamSource.connect(this.meter); } + // testing only startFromAudioFile(file) { if (!this.audioContext) this.audioContext = new AudioContext(); @@ -63,6 +58,25 @@ export class LipSync { }); } + startLipSyncFromAudioBuffer(buffer) { + if (!this.audioContext) this.audioContext = new AudioContext(); + if (!this.userSpeechAnalyzer) + this.userSpeechAnalyzer = this.audioContext.createAnalyser(); + this.userSpeechAnalyzer.smoothingTimeConstant = 0.5; + this.userSpeechAnalyzer.fftSize = FFT_SIZE; + + if (this.mediaStreamSource) this.mediaStreamSource.stop(); + + this.mediaStreamSource = this.audioContext.createBufferSource(); + this.mediaStreamSource.buffer = buffer; + this.meter = LipSync.createAudioMeter(this.audioContext); + this.mediaStreamSource.connect(this.meter); + this.mediaStreamSource.connect(this.audioContext.destination); + this.mediaStreamSource.start(); + // connect the output of mediaStreamSource to the input of userSpeechAnalyzer + this.mediaStreamSource.connect(this.userSpeechAnalyzer); + } + destroy() { this.meter?.shutdown(); this.meter = null; @@ -70,23 +84,20 @@ export class LipSync { return this.audioContext?.close().catch(() => {}) || Promise.resolve(); } - update(deltaTime) { + update() { if (this.meter) { const { volume } = this.meter; if (volume < 0.01) { - // get the shape keys from the VRM avatar + // get the shape keys from the face avatar - this.vrm.morphTargetInfluences[this.ee] = 0; - this.vrm.morphTargetInfluences[this.ah] = 0; - this.vrm.morphTargetInfluences[this.oh] = 0; + this.face.morphTargetInfluences[this.ee] = 0; + this.face.morphTargetInfluences[this.ah] = 0; + this.face.morphTargetInfluences[this.oh] = 0; } else { const { ah, oh, ee } = this.update2(); - this.vrm.morphTargetInfluences[this.ee] = ee; - this.vrm.morphTargetInfluences[this.ah] = ah; - this.vrm.morphTargetInfluences[this.oh] = oh; - - // update the shape keys - //this.vrm.expressionManager.update(deltaTime); + this.face.morphTargetInfluences[this.ee] = ee; + this.face.morphTargetInfluences[this.ah] = ah; + this.face.morphTargetInfluences[this.oh] = oh; } } } @@ -102,7 +113,7 @@ export class LipSync { } const spectrum = new Float32Array( - this.userSpeechAnalyzer.frequencyBinCount + this.userSpeechAnalyzer.frequencyBinCount, ); // Populate frequency data for computing frequency intensities this.userSpeechAnalyzer.getFloatFrequencyData(spectrum); // getByteTimeDomainData gets volumes over the sample time @@ -203,4 +214,4 @@ export class LipSync { return processor; } -} \ No newline at end of file +} diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 7e2acce6d0..1833bd9f86 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -1,9 +1,11 @@ import { AnyJson } from "three/examples/jsm/nodes/core/constants.js"; import { ClipUI } from "../datastructures/clips/clip_offset"; + +import Scene from "./scene.js"; import AudioEngine from "./audio_engine"; import TransformEngine from "./transform_engine"; -import Scene from "./scene.js"; -import * as THREE from 'three'; +import { LipSyncEngine } from "./lip_sync_engine"; +import { AnimationEngine } from "./animation_engine"; // Every object uuid / entity has a track. export class TimelineCurrentReactState { @@ -27,14 +29,22 @@ export class TimeLine { timelineState: TimelineCurrentReactState + // plays audio audioEngine: AudioEngine + // key framing transformEngine: TransformEngine // animation engine + animationEngine: AnimationEngine // lip sync engine + lipSyncEngine: LipSyncEngine scene: Scene - - constructor(audioEngine:AudioEngine, transformEngine:TransformEngine, scene:Scene) { + // ensure that the elements are loaded first. + constructor(audioEngine:AudioEngine, + transformEngine:TransformEngine, + lipsyncEngine:LipSyncEngine, + animationEngine:AnimationEngine, + scene:Scene) { this.timelineItems = [] this.timeLineLimit = 60 * 10 // 10 seconds this.runningClips = [] @@ -45,6 +55,9 @@ export class TimeLine { // this will be used to play the audio clips this.audioEngine = audioEngine this.transformEngine = transformEngine + this.lipSyncEngine = lipsyncEngine + this.animationEngine = animationEngine + this.scene = scene; } @@ -98,27 +111,6 @@ export class TimeLine { async play(): Promise { console.log(`Starting Timeline`) this.isPlaying = true - // const updateInterval = 100 - // const timerID = setInterval(async () => { - // console.log(`Current Time:${start}`) - - // // get last updated time when stopped - // for (const element of this.timelineItems) { - // if (element.start_offset >= start) { - // // run async - // // element.play() - // // remove the element - // this.timelineItems = this.timelineItems.filter(item => item !== element) - // } - // } - // if (start == this.timeLineLimit || this.timelineItems.length == 0) { - // this.stop() - // } - - // start += updateInterval - // }, updateInterval) - - // this.timerID = timerID } async reset_scene() { @@ -128,10 +120,11 @@ export class TimeLine { if(object) { this.transformEngine.clips[element.object_uuid].reset(object); } } else if (element.type == "audio") { - + } else if (element.type == "animation") { - } else { + } + else { this.stop() throw "Error New Type of element in the timeline" } @@ -156,20 +149,27 @@ export class TimeLine { // run async // element.play() // remove the element from the list + let object = this.scene.get_object_by_uuid(element.object_uuid) if (element.type == "transform") { - let object = this.scene.get_object_by_uuid(element.object_uuid); if(object) { - this.transformEngine.clips[element.object_uuid].length = (element.ending_offset-element.start_offset); - this.transformEngine.clips[element.object_uuid].step(object); + this.transformEngine.clips[element.object_uuid].length = (element.ending_offset-element.start_offset) + this.transformEngine.clips[element.object_uuid].step(object) } } - else if (element.type == "audio") { - + else if (element.type == "audio") { + // global audio track + this.audioEngine.playClip(element.media_id) + } + else if (element.type == "lipsync") { + // I think you just get the object verify it is a character ? then play the clip, but ... it needs + // need character face + this.lipSyncEngine.playClip(object,element.media_id) } else if (element.type == "animation") { - - } else { + // use the media id to figure out which animation clip belongs to who + } + else { this.stop() throw "Error New Type of element in the timeline" }