daniel543x/artcraft
1
0 Fork 0
mirror of https://github.com/storytold/artcraft.git synced 2026-11-05 09:48:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Clip Offset added lipsync type.

Browse Source 
added the lipsync clip type.
animation engine mock.
audio engine added to the timeline
lipsync engine added and finished needs testing.
timeline modified to address issues.
This commit is contained in:
Michael Chung committed 2024-03-24 16:33:47 -04:00
1 parent 1ef97d5668
commit 20ba33c36c
7 files changed
+215 -73
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/pages/PageEnigma/datastructures/clips/clip_offset.ts
+10 -10
View File
@@ -1,8 +1,8 @@
export interface ClipUI {
version: number
type: "transform" | "audio" | "animation"
media_id: number
object_uuid: number
type: "transform" | "audio" | "animation" | "lipsync"
media_id: string
object_uuid: string
start_offset: number // in frames so 60 frames per second on the timeline.
ending_offset: number // ending offset
}
@@ -11,17 +11,17 @@ export interface ClipUI {
// it is created from a media id.
export class ClipUI implements ClipUI {
version: number
type: "transform" | "audio" | "animation"
media_id: number
object_uuid: number
type: "transform" | "audio" | "animation" | "lipsync"
media_id: string
object_uuid: string
start_offset: number // in frames
ending_offset: number // in frames
constructor(
version: number,
type: "transform" | "audio" | "animation",
media_id: number,
object_uuid: number,
type: "transform" | "audio" | "animation" | "lipsync",
media_id: string,
object_uuid: string,
start_offset: number,
ending_offset: number,
) {
@@ -36,8 +36,8 @@ export class ClipUI implements ClipUI {
toJSON(): string {
return JSON.stringify({
version: this.version,
object_uuid: this.object_uuid,
type: this.type,
object_uuid: this.object_uuid,
media_id: this.media_id,
start_offset: this.start_offset,
ending_offset: this.ending_offset
app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts
+72
View File
@@ -0,0 +1,72 @@
interface AudioData {
audioContext: AudioContext;
audioBuffer: AudioBuffer;
}
class AudioData implements AudioData{
audioContext: AudioContext;
audioBuffer: AudioBuffer;
source: AudioBufferSourceNode | undefined
constructor (audioContext: AudioContext, audioBuffer: AudioBuffer) {
this.audioContext = audioContext;
this.audioBuffer = audioBuffer;
}
}
export interface LipSyncTrackClip {
version: number;
media_id: string;
type: "lipsync";
volume: number;
}
export class LipSyncTrackClip implements LipSyncTrackClip {
version: number;
media_id: string;
type: "lipsync" = "lipsync";
volume: number;
audio_data: AudioData | undefined;
constructor(version: number, media_id: string, volume: number) {
this.version = version;
this.media_id = media_id;
this.type = "lipsync";
this.volume = volume;
this.download_audio().then(data => {
this.audio_data = data;
});
}
// lip sync will be generated through TTS
async get_media_url() {
//This is for prod when we have the proper info on the url.
let api_base_url = "https://api.fakeyou.com";
let url = `${api_base_url}/v1/media_files/file/${this.media_id}`
let responce = await fetch(url);
let json = await JSON.parse(await responce.text());
let bucketPath = json["media_file"]["public_bucket_path"];
let media_base_url = "https://storage.googleapis.com/vocodes-public"
let media_url = `${media_base_url}${bucketPath}`
return media_url;
}
async download_audio() {
let url = await this.get_media_url();
const audioContext = new AudioContext();
const response = await fetch(url);
const arrayBuffer = await response.arrayBuffer();
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
return new AudioData(audioContext, audioBuffer);
}
toJSON(): string {
return JSON.stringify({
version: this.version,
media_id: this.media_id,
type: this.type,
volume: this.volume,
});
}
}
app/pages/PageEnigma/js/animation_engine.ts
+7
View File
@@ -0,0 +1,7 @@
export class AnimationEngine {
}
app/pages/PageEnigma/js/audio_engine.ts
+3 -1
View File
@@ -1,6 +1,5 @@
import { AudioTrackClip } from "../datastructures/clips/audio_track_clip";
class AudioEngine {
clips: { [key: string]: AudioTrackClip } = {};
audio_sources: { [key: string]: AudioBufferSourceNode } = {};
@@ -12,10 +11,12 @@ class AudioEngine {
this.version = 1.0;
}
// loads clips into the engine to cache
loadClip(audio_media_id: string) {
this.clips[audio_media_id] = new AudioTrackClip(this.version, audio_media_id, 1.0);
}
// plays from the timeline.
playClip(audio_media_id: string) {
const clip = this.clips[audio_media_id];
if (clip.audio_data?.audioContext) {
@@ -28,6 +29,7 @@ class AudioEngine {
}
}
// stops the clips
stopClip(audio_media_id: string) {
const clip = this.clips[audio_media_id];
if (clip.audio_data?.source) {
app/pages/PageEnigma/js/lip_sync_engine.ts
+50
View File
@@ -0,0 +1,50 @@
import { LipSyncTrackClip } from "../datastructures/clips/lipsync_track_clip";
import { LipSync } from "./lipsync";
import * as THREE from 'three';
// needs to be apart of the editor window.
export class LipSyncEngine {
clips: { [key: string]: LipSyncTrackClip } = {}
audio_sources: { [key: string]: AudioBufferSourceNode } = {}
version: number
lipsync: LipSync
constructor() {
this.clips = {}
this.audio_sources = {}
this.version = 1.0
// we might need 3 of these one for each character ...
this.lipsync = new LipSync()
}
loadClip(audio_media_id: string) {
this.clips[audio_media_id] = new LipSyncTrackClip(this.version, audio_media_id, 1.0);
}
// needs to be called by the engine.
update(delta_time: number) {
this.lipsync.update(delta_time)
}
playClip(face: THREE.Object3D,audio_media_id: string) {
// face movement here
this.lipsync.face = face
const clip = this.clips[audio_media_id];
// this is a buffer right?
this.lipsync.startLipSyncFromAudioBuffer(clip)
}
stopClip(audio_media_id: string) {
const clip = this.clips[audio_media_id];
if (clip.audio_data?.source) {
clip.audio_data.source.stop();
} else {
console.warn(`AudioManager: AudioClip with id "${audio_media_id}" not found.`);
}
}
}
export default LipSyncEngine;
app/pages/PageEnigma/js/lipsync.js
+39 -28
View File
@@ -7,32 +7,26 @@ const samplingFrequency = 44100;
for (let m = 0; m < BoundingFrequencyMasc.length; m++) {
IndicesFrequencyMale[m] = Math.round(
((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyMasc[m]
((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyMasc[m],
);
}
for (let m = 0; m < BoundingFrequencyFem.length; m++) {
IndicesFrequencyFemale[m] = Math.round(
((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyFem[m]
((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyFem[m],
);
}
export class LipSync {
constructor(vrm) {
this.vrm = vrm;
this.ee = this.vrm.morphTargetDictionary["E"];
this.oh = this.vrm.morphTargetDictionary["O"];
this.ah = this.vrm.morphTargetDictionary["aa"];
const update = (deltaTime, elapsedTime) => {
requestAnimationFrame(update);
this.update(deltaTime, elapsedTime);
};
update();
constructor(face) {
this.face = face;
// we have to modify this for our
this.ee = this.face.morphTargetDictionary["E"];
this.oh = this.face.morphTargetDictionary["O"];
this.ah = this.face.morphTargetDictionary["aa"];
}
// for streaming sound?
start(stream) {
this.audioContext = new AudioContext();
this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
@@ -40,6 +34,7 @@ export class LipSync {
this.mediaStreamSource.connect(this.meter);
}
// testing only
startFromAudioFile(file) {
if (!this.audioContext) this.audioContext = new AudioContext();
@@ -63,6 +58,25 @@ export class LipSync {
});
}
startLipSyncFromAudioBuffer(buffer) {
if (!this.audioContext) this.audioContext = new AudioContext();
if (!this.userSpeechAnalyzer)
this.userSpeechAnalyzer = this.audioContext.createAnalyser();
this.userSpeechAnalyzer.smoothingTimeConstant = 0.5;
this.userSpeechAnalyzer.fftSize = FFT_SIZE;
if (this.mediaStreamSource) this.mediaStreamSource.stop();
this.mediaStreamSource = this.audioContext.createBufferSource();
this.mediaStreamSource.buffer = buffer;
this.meter = LipSync.createAudioMeter(this.audioContext);
this.mediaStreamSource.connect(this.meter);
this.mediaStreamSource.connect(this.audioContext.destination);
this.mediaStreamSource.start();
// connect the output of mediaStreamSource to the input of userSpeechAnalyzer
this.mediaStreamSource.connect(this.userSpeechAnalyzer);
}
destroy() {
this.meter?.shutdown();
this.meter = null;
@@ -70,23 +84,20 @@ export class LipSync {
return this.audioContext?.close().catch(() => {}) || Promise.resolve();
}
update(deltaTime) {
update() {
if (this.meter) {
const { volume } = this.meter;
if (volume < 0.01) {
// get the shape keys from the VRM avatar
// get the shape keys from the face avatar
this.vrm.morphTargetInfluences[this.ee] = 0;
this.vrm.morphTargetInfluences[this.ah] = 0;
this.vrm.morphTargetInfluences[this.oh] = 0;
this.face.morphTargetInfluences[this.ee] = 0;
this.face.morphTargetInfluences[this.ah] = 0;
this.face.morphTargetInfluences[this.oh] = 0;
} else {
const { ah, oh, ee } = this.update2();
this.vrm.morphTargetInfluences[this.ee] = ee;
this.vrm.morphTargetInfluences[this.ah] = ah;
this.vrm.morphTargetInfluences[this.oh] = oh;
// update the shape keys
//this.vrm.expressionManager.update(deltaTime);
this.face.morphTargetInfluences[this.ee] = ee;
this.face.morphTargetInfluences[this.ah] = ah;
this.face.morphTargetInfluences[this.oh] = oh;
}
}
}
@@ -102,7 +113,7 @@ export class LipSync {
}
const spectrum = new Float32Array(
this.userSpeechAnalyzer.frequencyBinCount
this.userSpeechAnalyzer.frequencyBinCount,
);
// Populate frequency data for computing frequency intensities
this.userSpeechAnalyzer.getFloatFrequencyData(spectrum); // getByteTimeDomainData gets volumes over the sample time
@@ -203,4 +214,4 @@ export class LipSync {
return processor;
}
}
}
app/pages/PageEnigma/js/timeline.ts
+34 -34
View File
@@ -1,9 +1,11 @@
import { AnyJson } from "three/examples/jsm/nodes/core/constants.js";
import { ClipUI } from "../datastructures/clips/clip_offset";
import Scene from "./scene.js";
import AudioEngine from "./audio_engine";
import TransformEngine from "./transform_engine";
import Scene from "./scene.js";
import * as THREE from 'three';
import { LipSyncEngine } from "./lip_sync_engine";
import { AnimationEngine } from "./animation_engine";
// Every object uuid / entity has a track.
export class TimelineCurrentReactState {
@@ -27,14 +29,22 @@ export class TimeLine {
timelineState: TimelineCurrentReactState
// plays audio
audioEngine: AudioEngine
// key framing
transformEngine: TransformEngine
// animation engine
animationEngine: AnimationEngine
// lip sync engine
lipSyncEngine: LipSyncEngine
scene: Scene
constructor(audioEngine:AudioEngine, transformEngine:TransformEngine, scene:Scene) {
// ensure that the elements are loaded first.
constructor(audioEngine:AudioEngine,
transformEngine:TransformEngine,
lipsyncEngine:LipSyncEngine,
animationEngine:AnimationEngine,
scene:Scene) {
this.timelineItems = []
this.timeLineLimit = 60 * 10 // 10 seconds
this.runningClips = []
@@ -45,6 +55,9 @@ export class TimeLine {
// this will be used to play the audio clips
this.audioEngine = audioEngine
this.transformEngine = transformEngine
this.lipSyncEngine = lipsyncEngine
this.animationEngine = animationEngine
this.scene = scene;
}
@@ -98,27 +111,6 @@ export class TimeLine {
async play(): Promise<void> {
console.log(`Starting Timeline`)
this.isPlaying = true
// const updateInterval = 100
// const timerID = setInterval(async () => {
// console.log(`Current Time:${start}`)
// // get last updated time when stopped
// for (const element of this.timelineItems) {
// if (element.start_offset >= start) {
// // run async
// // element.play()
// // remove the element
// this.timelineItems = this.timelineItems.filter(item => item !== element)
// }
// }
// if (start == this.timeLineLimit || this.timelineItems.length == 0) {
// this.stop()
// }
// start += updateInterval
// }, updateInterval)
// this.timerID = timerID
}
async reset_scene() {
@@ -128,10 +120,11 @@ export class TimeLine {
if(object) { this.transformEngine.clips[element.object_uuid].reset(object); }
}
else if (element.type == "audio") {
}
else if (element.type == "animation") {
} else {
}
else {
this.stop()
throw "Error New Type of element in the timeline"
}
@@ -156,20 +149,27 @@ export class TimeLine {
// run async
// element.play()
// remove the element from the list
let object = this.scene.get_object_by_uuid(element.object_uuid)
if (element.type == "transform") {
let object = this.scene.get_object_by_uuid(element.object_uuid);
if(object)
{
this.transformEngine.clips[element.object_uuid].length = (element.ending_offset-element.start_offset);
this.transformEngine.clips[element.object_uuid].step(object);
this.transformEngine.clips[element.object_uuid].length = (element.ending_offset-element.start_offset)
this.transformEngine.clips[element.object_uuid].step(object)
}
}
else if (element.type == "audio") {
else if (element.type == "audio") {
// global audio track
this.audioEngine.playClip(element.media_id)
}
else if (element.type == "lipsync") {
// I think you just get the object verify it is a character ? then play the clip, but ... it needs
// need character face
this.lipSyncEngine.playClip(object,element.media_id)
}
else if (element.type == "animation") {
} else {
// use the media id to figure out which animation clip belongs to who
}
else {
this.stop()
throw "Error New Type of element in the timeline"
}