From 20ba33c36c4ebc65eecc335d1b1416378fbff6cc Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Sun, 24 Mar 2024 16:33:47 -0400 Subject: [PATCH] Clip Offset added lipsync type. added the lipsync clip type. animation engine mock. audio engine added to the timeline lipsync engine added and finished needs testing. timeline modified to address issues. --- .../datastructures/clips/clip_offset.ts | 20 +++--- .../clips/lipsync_track_clip.ts | 72 +++++++++++++++++++ app/pages/PageEnigma/js/animation_engine.ts | 7 ++ app/pages/PageEnigma/js/audio_engine.ts | 4 +- app/pages/PageEnigma/js/lip_sync_engine.ts | 50 +++++++++++++ app/pages/PageEnigma/js/lipsync.js | 67 +++++++++-------- app/pages/PageEnigma/js/timeline.ts | 68 +++++++++--------- 7 files changed, 215 insertions(+), 73 deletions(-) create mode 100644 app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts create mode 100644 app/pages/PageEnigma/js/animation_engine.ts create mode 100644 app/pages/PageEnigma/js/lip_sync_engine.ts diff --git a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts index b03e0e7d8f..9225276b74 100644 --- a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts +++ b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts @@ -1,8 +1,8 @@ export interface ClipUI { version: number - type: "transform" | "audio" | "animation" - media_id: number - object_uuid: number + type: "transform" | "audio" | "animation" | "lipsync" + media_id: string + object_uuid: string start_offset: number // in frames so 60 frames per second on the timeline. ending_offset: number // ending offset } @@ -11,17 +11,17 @@ export interface ClipUI { // it is created from a media id. export class ClipUI implements ClipUI { version: number - type: "transform" | "audio" | "animation" - media_id: number - object_uuid: number + type: "transform" | "audio" | "animation" | "lipsync" + media_id: string + object_uuid: string start_offset: number // in frames ending_offset: number // in frames constructor( version: number, - type: "transform" | "audio" | "animation", - media_id: number, - object_uuid: number, + type: "transform" | "audio" | "animation" | "lipsync", + media_id: string, + object_uuid: string, start_offset: number, ending_offset: number, ) { @@ -36,8 +36,8 @@ export class ClipUI implements ClipUI { toJSON(): string { return JSON.stringify({ version: this.version, - object_uuid: this.object_uuid, type: this.type, + object_uuid: this.object_uuid, media_id: this.media_id, start_offset: this.start_offset, ending_offset: this.ending_offset diff --git a/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts new file mode 100644 index 0000000000..94d8a49b86 --- /dev/null +++ b/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts @@ -0,0 +1,72 @@ +interface AudioData { + audioContext: AudioContext; + audioBuffer: AudioBuffer; + } + + class AudioData implements AudioData{ + audioContext: AudioContext; + audioBuffer: AudioBuffer; + source: AudioBufferSourceNode | undefined + + constructor (audioContext: AudioContext, audioBuffer: AudioBuffer) { + this.audioContext = audioContext; + this.audioBuffer = audioBuffer; + } + } + + export interface LipSyncTrackClip { + version: number; + media_id: string; + type: "lipsync"; + volume: number; + } + + export class LipSyncTrackClip implements LipSyncTrackClip { + version: number; + media_id: string; + type: "lipsync" = "lipsync"; + volume: number; + audio_data: AudioData | undefined; + + constructor(version: number, media_id: string, volume: number) { + this.version = version; + this.media_id = media_id; + this.type = "lipsync"; + this.volume = volume; + this.download_audio().then(data => { + this.audio_data = data; + }); + } + + // lip sync will be generated through TTS + async get_media_url() { + //This is for prod when we have the proper info on the url. + let api_base_url = "https://api.fakeyou.com"; + let url = `${api_base_url}/v1/media_files/file/${this.media_id}` + let responce = await fetch(url); + let json = await JSON.parse(await responce.text()); + let bucketPath = json["media_file"]["public_bucket_path"]; + let media_base_url = "https://storage.googleapis.com/vocodes-public" + let media_url = `${media_base_url}${bucketPath}` + return media_url; + } + + async download_audio() { + let url = await this.get_media_url(); + const audioContext = new AudioContext(); + const response = await fetch(url); + const arrayBuffer = await response.arrayBuffer(); + const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); + return new AudioData(audioContext, audioBuffer); + } + + toJSON(): string { + return JSON.stringify({ + version: this.version, + media_id: this.media_id, + type: this.type, + volume: this.volume, + }); + } + } + \ No newline at end of file diff --git a/app/pages/PageEnigma/js/animation_engine.ts b/app/pages/PageEnigma/js/animation_engine.ts new file mode 100644 index 0000000000..c8eba78ba9 --- /dev/null +++ b/app/pages/PageEnigma/js/animation_engine.ts @@ -0,0 +1,7 @@ + + +export class AnimationEngine { + + + +} \ No newline at end of file diff --git a/app/pages/PageEnigma/js/audio_engine.ts b/app/pages/PageEnigma/js/audio_engine.ts index 1dff3ee3c6..b7c3b5ee66 100644 --- a/app/pages/PageEnigma/js/audio_engine.ts +++ b/app/pages/PageEnigma/js/audio_engine.ts @@ -1,6 +1,5 @@ import { AudioTrackClip } from "../datastructures/clips/audio_track_clip"; - class AudioEngine { clips: { [key: string]: AudioTrackClip } = {}; audio_sources: { [key: string]: AudioBufferSourceNode } = {}; @@ -12,10 +11,12 @@ class AudioEngine { this.version = 1.0; } + // loads clips into the engine to cache loadClip(audio_media_id: string) { this.clips[audio_media_id] = new AudioTrackClip(this.version, audio_media_id, 1.0); } + // plays from the timeline. playClip(audio_media_id: string) { const clip = this.clips[audio_media_id]; if (clip.audio_data?.audioContext) { @@ -28,6 +29,7 @@ class AudioEngine { } } + // stops the clips stopClip(audio_media_id: string) { const clip = this.clips[audio_media_id]; if (clip.audio_data?.source) { diff --git a/app/pages/PageEnigma/js/lip_sync_engine.ts b/app/pages/PageEnigma/js/lip_sync_engine.ts new file mode 100644 index 0000000000..d6e32c4795 --- /dev/null +++ b/app/pages/PageEnigma/js/lip_sync_engine.ts @@ -0,0 +1,50 @@ + +import { LipSyncTrackClip } from "../datastructures/clips/lipsync_track_clip"; +import { LipSync } from "./lipsync"; +import * as THREE from 'three'; + +// needs to be apart of the editor window. +export class LipSyncEngine { + + clips: { [key: string]: LipSyncTrackClip } = {} + audio_sources: { [key: string]: AudioBufferSourceNode } = {} + version: number + lipsync: LipSync + + constructor() { + + this.clips = {} + this.audio_sources = {} + this.version = 1.0 + // we might need 3 of these one for each character ... + this.lipsync = new LipSync() + } + + loadClip(audio_media_id: string) { + this.clips[audio_media_id] = new LipSyncTrackClip(this.version, audio_media_id, 1.0); + } + + // needs to be called by the engine. + update(delta_time: number) { + this.lipsync.update(delta_time) + } + + playClip(face: THREE.Object3D,audio_media_id: string) { + // face movement here + this.lipsync.face = face + const clip = this.clips[audio_media_id]; + // this is a buffer right? + this.lipsync.startLipSyncFromAudioBuffer(clip) + } + + stopClip(audio_media_id: string) { + const clip = this.clips[audio_media_id]; + if (clip.audio_data?.source) { + clip.audio_data.source.stop(); + } else { + console.warn(`AudioManager: AudioClip with id "${audio_media_id}" not found.`); + } + } +} + +export default LipSyncEngine; diff --git a/app/pages/PageEnigma/js/lipsync.js b/app/pages/PageEnigma/js/lipsync.js index 2233bba786..c72356fd9a 100644 --- a/app/pages/PageEnigma/js/lipsync.js +++ b/app/pages/PageEnigma/js/lipsync.js @@ -7,32 +7,26 @@ const samplingFrequency = 44100; for (let m = 0; m < BoundingFrequencyMasc.length; m++) { IndicesFrequencyMale[m] = Math.round( - ((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyMasc[m] + ((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyMasc[m], ); } for (let m = 0; m < BoundingFrequencyFem.length; m++) { IndicesFrequencyFemale[m] = Math.round( - ((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyFem[m] + ((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyFem[m], ); } export class LipSync { - constructor(vrm) { - this.vrm = vrm; - - this.ee = this.vrm.morphTargetDictionary["E"]; - this.oh = this.vrm.morphTargetDictionary["O"]; - this.ah = this.vrm.morphTargetDictionary["aa"]; - - const update = (deltaTime, elapsedTime) => { - requestAnimationFrame(update); - this.update(deltaTime, elapsedTime); - }; - - update(); + constructor(face) { + this.face = face; + // we have to modify this for our + this.ee = this.face.morphTargetDictionary["E"]; + this.oh = this.face.morphTargetDictionary["O"]; + this.ah = this.face.morphTargetDictionary["aa"]; } + // for streaming sound? start(stream) { this.audioContext = new AudioContext(); this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream); @@ -40,6 +34,7 @@ export class LipSync { this.mediaStreamSource.connect(this.meter); } + // testing only startFromAudioFile(file) { if (!this.audioContext) this.audioContext = new AudioContext(); @@ -63,6 +58,25 @@ export class LipSync { }); } + startLipSyncFromAudioBuffer(buffer) { + if (!this.audioContext) this.audioContext = new AudioContext(); + if (!this.userSpeechAnalyzer) + this.userSpeechAnalyzer = this.audioContext.createAnalyser(); + this.userSpeechAnalyzer.smoothingTimeConstant = 0.5; + this.userSpeechAnalyzer.fftSize = FFT_SIZE; + + if (this.mediaStreamSource) this.mediaStreamSource.stop(); + + this.mediaStreamSource = this.audioContext.createBufferSource(); + this.mediaStreamSource.buffer = buffer; + this.meter = LipSync.createAudioMeter(this.audioContext); + this.mediaStreamSource.connect(this.meter); + this.mediaStreamSource.connect(this.audioContext.destination); + this.mediaStreamSource.start(); + // connect the output of mediaStreamSource to the input of userSpeechAnalyzer + this.mediaStreamSource.connect(this.userSpeechAnalyzer); + } + destroy() { this.meter?.shutdown(); this.meter = null; @@ -70,23 +84,20 @@ export class LipSync { return this.audioContext?.close().catch(() => {}) || Promise.resolve(); } - update(deltaTime) { + update() { if (this.meter) { const { volume } = this.meter; if (volume < 0.01) { - // get the shape keys from the VRM avatar + // get the shape keys from the face avatar - this.vrm.morphTargetInfluences[this.ee] = 0; - this.vrm.morphTargetInfluences[this.ah] = 0; - this.vrm.morphTargetInfluences[this.oh] = 0; + this.face.morphTargetInfluences[this.ee] = 0; + this.face.morphTargetInfluences[this.ah] = 0; + this.face.morphTargetInfluences[this.oh] = 0; } else { const { ah, oh, ee } = this.update2(); - this.vrm.morphTargetInfluences[this.ee] = ee; - this.vrm.morphTargetInfluences[this.ah] = ah; - this.vrm.morphTargetInfluences[this.oh] = oh; - - // update the shape keys - //this.vrm.expressionManager.update(deltaTime); + this.face.morphTargetInfluences[this.ee] = ee; + this.face.morphTargetInfluences[this.ah] = ah; + this.face.morphTargetInfluences[this.oh] = oh; } } } @@ -102,7 +113,7 @@ export class LipSync { } const spectrum = new Float32Array( - this.userSpeechAnalyzer.frequencyBinCount + this.userSpeechAnalyzer.frequencyBinCount, ); // Populate frequency data for computing frequency intensities this.userSpeechAnalyzer.getFloatFrequencyData(spectrum); // getByteTimeDomainData gets volumes over the sample time @@ -203,4 +214,4 @@ export class LipSync { return processor; } -} \ No newline at end of file +} diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 7e2acce6d0..1833bd9f86 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -1,9 +1,11 @@ import { AnyJson } from "three/examples/jsm/nodes/core/constants.js"; import { ClipUI } from "../datastructures/clips/clip_offset"; + +import Scene from "./scene.js"; import AudioEngine from "./audio_engine"; import TransformEngine from "./transform_engine"; -import Scene from "./scene.js"; -import * as THREE from 'three'; +import { LipSyncEngine } from "./lip_sync_engine"; +import { AnimationEngine } from "./animation_engine"; // Every object uuid / entity has a track. export class TimelineCurrentReactState { @@ -27,14 +29,22 @@ export class TimeLine { timelineState: TimelineCurrentReactState + // plays audio audioEngine: AudioEngine + // key framing transformEngine: TransformEngine // animation engine + animationEngine: AnimationEngine // lip sync engine + lipSyncEngine: LipSyncEngine scene: Scene - - constructor(audioEngine:AudioEngine, transformEngine:TransformEngine, scene:Scene) { + // ensure that the elements are loaded first. + constructor(audioEngine:AudioEngine, + transformEngine:TransformEngine, + lipsyncEngine:LipSyncEngine, + animationEngine:AnimationEngine, + scene:Scene) { this.timelineItems = [] this.timeLineLimit = 60 * 10 // 10 seconds this.runningClips = [] @@ -45,6 +55,9 @@ export class TimeLine { // this will be used to play the audio clips this.audioEngine = audioEngine this.transformEngine = transformEngine + this.lipSyncEngine = lipsyncEngine + this.animationEngine = animationEngine + this.scene = scene; } @@ -98,27 +111,6 @@ export class TimeLine { async play(): Promise { console.log(`Starting Timeline`) this.isPlaying = true - // const updateInterval = 100 - // const timerID = setInterval(async () => { - // console.log(`Current Time:${start}`) - - // // get last updated time when stopped - // for (const element of this.timelineItems) { - // if (element.start_offset >= start) { - // // run async - // // element.play() - // // remove the element - // this.timelineItems = this.timelineItems.filter(item => item !== element) - // } - // } - // if (start == this.timeLineLimit || this.timelineItems.length == 0) { - // this.stop() - // } - - // start += updateInterval - // }, updateInterval) - - // this.timerID = timerID } async reset_scene() { @@ -128,10 +120,11 @@ export class TimeLine { if(object) { this.transformEngine.clips[element.object_uuid].reset(object); } } else if (element.type == "audio") { - + } else if (element.type == "animation") { - } else { + } + else { this.stop() throw "Error New Type of element in the timeline" } @@ -156,20 +149,27 @@ export class TimeLine { // run async // element.play() // remove the element from the list + let object = this.scene.get_object_by_uuid(element.object_uuid) if (element.type == "transform") { - let object = this.scene.get_object_by_uuid(element.object_uuid); if(object) { - this.transformEngine.clips[element.object_uuid].length = (element.ending_offset-element.start_offset); - this.transformEngine.clips[element.object_uuid].step(object); + this.transformEngine.clips[element.object_uuid].length = (element.ending_offset-element.start_offset) + this.transformEngine.clips[element.object_uuid].step(object) } } - else if (element.type == "audio") { - + else if (element.type == "audio") { + // global audio track + this.audioEngine.playClip(element.media_id) + } + else if (element.type == "lipsync") { + // I think you just get the object verify it is a character ? then play the clip, but ... it needs + // need character face + this.lipSyncEngine.playClip(object,element.media_id) } else if (element.type == "animation") { - - } else { + // use the media id to figure out which animation clip belongs to who + } + else { this.stop() throw "Error New Type of element in the timeline" }