daniel543x/artcraft
1
0 Fork 0
mirror of https://github.com/storytold/artcraft.git synced 2026-11-05 09:48:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5 from storytold/styling/tabs-css

Browse Source 
Clip Offset added lipsync type.
This commit is contained in:
ArEnSc authored and GitHub committed 2024-03-24 17:42:34 -04:00
2 parents 22ec405e29 c54e5574bd
commit 74f08228fa
6 files changed
+202 -66
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/pages/PageEnigma/datastructures/clips/clip_offset.ts
+4 -4
View File
@@ -1,6 +1,6 @@
export interface ClipUI {
version: number
type: "transform" | "audio" | "animation"
type: "transform" | "audio" | "animation" | "lipsync"
media_id: string
object_uuid: string
start_offset: number // in frames so 60 frames per second on the timeline.
@@ -11,7 +11,7 @@ export interface ClipUI {
// it is created from a media id.
export class ClipUI implements ClipUI {
version: number
type: "transform" | "audio" | "animation"
type: "transform" | "audio" | "animation" | "lipsync"
media_id: string
object_uuid: string
start_offset: number // in frames
@@ -19,7 +19,7 @@ export class ClipUI implements ClipUI {
constructor(
version: number,
type: "transform" | "audio" | "animation",
type: "transform" | "audio" | "animation" | "lipsync",
media_id: string,
object_uuid: string,
start_offset: number,
@@ -36,8 +36,8 @@ export class ClipUI implements ClipUI {
toJSON(): string {
return JSON.stringify({
version: this.version,
object_uuid: this.object_uuid,
type: this.type,
object_uuid: this.object_uuid,
media_id: this.media_id,
start_offset: this.start_offset,
ending_offset: this.ending_offset
app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts
+72
View File
@@ -0,0 +1,72 @@
interface AudioData {
audioContext: AudioContext;
audioBuffer: AudioBuffer;
}
class AudioData implements AudioData{
audioContext: AudioContext;
audioBuffer: AudioBuffer;
source: AudioBufferSourceNode | undefined
constructor (audioContext: AudioContext, audioBuffer: AudioBuffer) {
this.audioContext = audioContext;
this.audioBuffer = audioBuffer;
}
}
export interface LipSyncTrackClip {
version: number;
media_id: string;
type: "lipsync";
volume: number;
}
export class LipSyncTrackClip implements LipSyncTrackClip {
version: number;
media_id: string;
type: "lipsync" = "lipsync";
volume: number;
audio_data: AudioData | undefined;
constructor(version: number, media_id: string, volume: number) {
this.version = version;
this.media_id = media_id;
this.type = "lipsync";
this.volume = volume;
this.download_audio().then(data => {
this.audio_data = data;
});
}
// lip sync will be generated through TTS
async get_media_url() {
//This is for prod when we have the proper info on the url.
let api_base_url = "https://api.fakeyou.com";
let url = `${api_base_url}/v1/media_files/file/${this.media_id}`
let responce = await fetch(url);
let json = await JSON.parse(await responce.text());
let bucketPath = json["media_file"]["public_bucket_path"];
let media_base_url = "https://storage.googleapis.com/vocodes-public"
let media_url = `${media_base_url}${bucketPath}`
return media_url;
}
async download_audio() {
let url = await this.get_media_url();
const audioContext = new AudioContext();
const response = await fetch(url);
const arrayBuffer = await response.arrayBuffer();
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
return new AudioData(audioContext, audioBuffer);
}
toJSON(): string {
return JSON.stringify({
version: this.version,
media_id: this.media_id,
type: this.type,
volume: this.volume,
});
}
}
app/pages/PageEnigma/js/audio_engine.ts
+3 -1
View File
@@ -1,7 +1,6 @@
import { faL } from "@fortawesome/pro-solid-svg-icons";
import { AudioTrackClip } from "../datastructures/clips/audio_track_clip";
class AudioEngine {
clips: { [key: string]: AudioTrackClip } = {};
audio_sources: { [key: string]: AudioBufferSourceNode } = {};
@@ -15,11 +14,13 @@ class AudioEngine {
this.version = 1.0;
}
// loads clips into the engine to cache
loadClip(audio_media_id: string) {
if(this.clips[audio_media_id] != null) { return; }
this.clips[audio_media_id] = new AudioTrackClip(this.version, audio_media_id, 1.0);
}
// plays from the timeline.
playClip(audio_media_id: string) {
if (this.clips[audio_media_id] == null) { this.loadClip(audio_media_id); }
let clip = this.clips[audio_media_id];
@@ -39,6 +40,7 @@ class AudioEngine {
}
}
// stops the clips
stopClip(audio_media_id: string) {
const clip = this.clips[audio_media_id];
if (clip.audio_data?.source) {
app/pages/PageEnigma/js/lip_sync_engine.ts
+50
View File
@@ -0,0 +1,50 @@
import { LipSyncTrackClip } from "../datastructures/clips/lipsync_track_clip";
import { LipSync } from "./lipsync";
import * as THREE from 'three';
// needs to be apart of the editor window.
export class LipSyncEngine {
clips: { [key: string]: LipSyncTrackClip } = {}
audio_sources: { [key: string]: AudioBufferSourceNode } = {}
version: number
lipsync: LipSync
constructor() {
this.clips = {}
this.audio_sources = {}
this.version = 1.0
// we might need 3 of these one for each character ...
this.lipsync = new LipSync()
}
loadClip(audio_media_id: string) {
this.clips[audio_media_id] = new LipSyncTrackClip(this.version, audio_media_id, 1.0);
}
// needs to be called by the engine.
update(delta_time: number) {
this.lipsync.update(delta_time)
}
playClip(face: THREE.Object3D,audio_media_id: string) {
// face movement here
this.lipsync.face = face
const clip = this.clips[audio_media_id];
// this is a buffer right?
this.lipsync.startLipSyncFromAudioBuffer(clip)
}
stopClip(audio_media_id: string) {
const clip = this.clips[audio_media_id];
if (clip.audio_data?.source) {
clip.audio_data.source.stop();
} else {
console.warn(`AudioManager: AudioClip with id "${audio_media_id}" not found.`);
}
}
}
export default LipSyncEngine;
app/pages/PageEnigma/js/lipsync.js
+39 -28
View File
@@ -7,32 +7,26 @@ const samplingFrequency = 44100;
for (let m = 0; m < BoundingFrequencyMasc.length; m++) {
IndicesFrequencyMale[m] = Math.round(
((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyMasc[m]
((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyMasc[m],
);
}
for (let m = 0; m < BoundingFrequencyFem.length; m++) {
IndicesFrequencyFemale[m] = Math.round(
((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyFem[m]
((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyFem[m],
);
}
export class LipSync {
constructor(vrm) {
this.vrm = vrm;
this.ee = this.vrm.morphTargetDictionary["E"];
this.oh = this.vrm.morphTargetDictionary["O"];
this.ah = this.vrm.morphTargetDictionary["aa"];
const update = (deltaTime, elapsedTime) => {
requestAnimationFrame(update);
this.update(deltaTime, elapsedTime);
};
update();
constructor(face) {
this.face = face;
// we have to modify this for our
this.ee = this.face.morphTargetDictionary["E"];
this.oh = this.face.morphTargetDictionary["O"];
this.ah = this.face.morphTargetDictionary["aa"];
}
// for streaming sound?
start(stream) {
this.audioContext = new AudioContext();
this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
@@ -40,6 +34,7 @@ export class LipSync {
this.mediaStreamSource.connect(this.meter);
}
// testing only
startFromAudioFile(file) {
if (!this.audioContext) this.audioContext = new AudioContext();
@@ -63,6 +58,25 @@ export class LipSync {
});
}
startLipSyncFromAudioBuffer(buffer) {
if (!this.audioContext) this.audioContext = new AudioContext();
if (!this.userSpeechAnalyzer)
this.userSpeechAnalyzer = this.audioContext.createAnalyser();
this.userSpeechAnalyzer.smoothingTimeConstant = 0.5;
this.userSpeechAnalyzer.fftSize = FFT_SIZE;
if (this.mediaStreamSource) this.mediaStreamSource.stop();
this.mediaStreamSource = this.audioContext.createBufferSource();
this.mediaStreamSource.buffer = buffer;
this.meter = LipSync.createAudioMeter(this.audioContext);
this.mediaStreamSource.connect(this.meter);
this.mediaStreamSource.connect(this.audioContext.destination);
this.mediaStreamSource.start();
// connect the output of mediaStreamSource to the input of userSpeechAnalyzer
this.mediaStreamSource.connect(this.userSpeechAnalyzer);
}
destroy() {
this.meter?.shutdown();
this.meter = null;
@@ -70,23 +84,20 @@ export class LipSync {
return this.audioContext?.close().catch(() => {}) || Promise.resolve();
}
update(deltaTime) {
update() {
if (this.meter) {
const { volume } = this.meter;
if (volume < 0.01) {
// get the shape keys from the VRM avatar
// get the shape keys from the face avatar
this.vrm.morphTargetInfluences[this.ee] = 0;
this.vrm.morphTargetInfluences[this.ah] = 0;
this.vrm.morphTargetInfluences[this.oh] = 0;
this.face.morphTargetInfluences[this.ee] = 0;
this.face.morphTargetInfluences[this.ah] = 0;
this.face.morphTargetInfluences[this.oh] = 0;
} else {
const { ah, oh, ee } = this.update2();
this.vrm.morphTargetInfluences[this.ee] = ee;
this.vrm.morphTargetInfluences[this.ah] = ah;
this.vrm.morphTargetInfluences[this.oh] = oh;
// update the shape keys
//this.vrm.expressionManager.update(deltaTime);
this.face.morphTargetInfluences[this.ee] = ee;
this.face.morphTargetInfluences[this.ah] = ah;
this.face.morphTargetInfluences[this.oh] = oh;
}
}
}
@@ -102,7 +113,7 @@ export class LipSync {
}
const spectrum = new Float32Array(
this.userSpeechAnalyzer.frequencyBinCount
this.userSpeechAnalyzer.frequencyBinCount,
);
// Populate frequency data for computing frequency intensities
this.userSpeechAnalyzer.getFloatFrequencyData(spectrum); // getByteTimeDomainData gets volumes over the sample time
@@ -203,4 +214,4 @@ export class LipSync {
return processor;
}
}
}
app/pages/PageEnigma/js/timeline.ts
+34 -33
View File
@@ -1,9 +1,11 @@
import { AnyJson } from "three/examples/jsm/nodes/core/constants.js";
import { ClipUI } from "../datastructures/clips/clip_offset";
import Scene from "./scene.js";
import AudioEngine from "./audio_engine";
import TransformEngine from "./transform_engine";
import Scene from "./scene.js";
import * as THREE from 'three';
import { LipSyncEngine } from "./lip_sync_engine";
import { AnimationEngine } from "./animation_engine";
// Every object uuid / entity has a track.
export class TimelineCurrentReactState {
@@ -27,14 +29,22 @@ export class TimeLine {
timelineState: TimelineCurrentReactState
// plays audio
audioEngine: AudioEngine
// key framing
transformEngine: TransformEngine
// animation engine
animationEngine: AnimationEngine
// lip sync engine
lipSyncEngine: LipSyncEngine
scene: Scene
constructor(audioEngine:AudioEngine, transformEngine:TransformEngine, scene:Scene) {
// ensure that the elements are loaded first.
constructor(audioEngine:AudioEngine,
transformEngine:TransformEngine,
lipsyncEngine:LipSyncEngine,
animationEngine:AnimationEngine,
scene:Scene) {
this.timelineItems = []
this.timeLineLimit = 60 * 10 // 10 seconds
this.runningClips = []
@@ -45,6 +55,9 @@ export class TimeLine {
// this will be used to play the audio clips
this.audioEngine = audioEngine
this.transformEngine = transformEngine
this.lipSyncEngine = lipsyncEngine
this.animationEngine = animationEngine
this.scene = scene;
}
@@ -98,27 +111,6 @@ export class TimeLine {
async play(): Promise<void> {
console.log(`Starting Timeline`)
this.isPlaying = true
// const updateInterval = 100
// const timerID = setInterval(async () => {
// console.log(`Current Time:${start}`)
// // get last updated time when stopped
// for (const element of this.timelineItems) {
// if (element.start_offset >= start) {
// // run async
// // element.play()
// // remove the element
// this.timelineItems = this.timelineItems.filter(item => item !== element)
// }
// }
// if (start == this.timeLineLimit || this.timelineItems.length == 0) {
// this.stop()
// }
// start += updateInterval
// }, updateInterval)
// this.timerID = timerID
}
async reset_scene() {
@@ -131,7 +123,8 @@ export class TimeLine {
this.audioEngine.loadClip(element.media_id);
}
else if (element.type == "animation") {
} else {
}
else {
this.stop()
throw "Error New Type of element in the timeline"
}
@@ -155,20 +148,28 @@ export class TimeLine {
// run async
// element.play()
// remove the element from the list
let object = this.scene.get_object_by_uuid(element.object_uuid)
if (element.type == "transform") {
let object = this.scene.get_object_by_uuid(element.object_uuid);
if(object)
{
this.transformEngine.clips[element.object_uuid].length = (element.ending_offset-element.start_offset);
this.transformEngine.clips[element.object_uuid].step(object);
this.transformEngine.clips[element.object_uuid].length = (element.ending_offset-element.start_offset)
this.transformEngine.clips[element.object_uuid].step(object)
}
}
else if (element.type == "audio") {
this.audioEngine.playClip(element.media_id);
else if (element.type == "audio") {
// global audio track
this.audioEngine.playClip(element.media_id)
}
else if (element.type == "lipsync") {
// I think you just get the object verify it is a character ? then play the clip, but ... it needs
// need character face
this.lipSyncEngine.playClip(object,element.media_id)
}
else if (element.type == "animation") {
} else {
// use the media id to figure out which animation clip belongs to who
}
else {
this.stop()
throw "Error New Type of element in the timeline"
}