From 54db4323615b147dc5a96ce4fcb3df0bd7f0b9e3 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Sat, 20 Apr 2024 20:38:05 -0400 Subject: [PATCH] ARKit engine. --- .../datastructures/clips/emotion_clip.ts | 30 ++++++++++++------- app/pages/PageEnigma/js/emotion_engine.ts | 7 ++--- app/pages/PageEnigma/js/timeline.ts | 25 +++++++++++++++- app/pages/PageEnigma/store/media.ts | 2 +- 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/app/pages/PageEnigma/datastructures/clips/emotion_clip.ts b/app/pages/PageEnigma/datastructures/clips/emotion_clip.ts index a5e5b324a8..b08d484d3c 100644 --- a/app/pages/PageEnigma/datastructures/clips/emotion_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/emotion_clip.ts @@ -52,14 +52,14 @@ export class EmotionClip { header.forEach(key => jsonData[key] = []); for (let i = 1; i < rows.length; i++) { - const values = rows[i].split(','); - values.forEach((value, index) => { - jsonData[header[index]].push(value); - }); + const values = rows[i].split(','); + values.forEach((value, index) => { + jsonData[header[index]].push(value); + }); } return jsonData; -} + } async _detect_face(object: THREE.Object3D): Promise { @@ -83,7 +83,8 @@ export class EmotionClip { this.faces.forEach((element: THREE.Mesh) => { if (element.morphTargetInfluences && shapes) { Object.keys(shapes).forEach(key => { - const index = element.morphTargetDictionary?.[key]; + let index = element.morphTargetDictionary?.[key]; + if (index === undefined) { index = element.morphTargetDictionary?.[key.charAt(0).toLowerCase() + key.slice(1)]; } if (typeof index === 'number' && element.morphTargetInfluences !== undefined) { element.morphTargetInfluences[index] = shapes[key]; } @@ -94,14 +95,21 @@ export class EmotionClip { async reset(object: THREE.Object3D) { await this._detect_face(object); - // setBlends() - console.log(this.emotion_json); + let keys: { [key: string]: number } = {}; + Object.keys(this.emotion_json).forEach(key => { + keys[key] = 0; + }); + this.setBlends(keys); } async step(frame: number, object: THREE.Object3D) { - await this._detect_face(object); - // setBlends() - console.log(this.emotion_json); + if(this.faces.length <= 0) { await this._detect_face(object); } + let keys: { [key: string]: number } = {}; + Object.keys(this.emotion_json).forEach(key => { + if (frame > this.emotion_json[key].length) { return; } + keys[key] = this.emotion_json[key][frame]; + }); + this.setBlends(keys); } toJSON(): any { diff --git a/app/pages/PageEnigma/js/emotion_engine.ts b/app/pages/PageEnigma/js/emotion_engine.ts index bd64fbb222..583383de0c 100644 --- a/app/pages/PageEnigma/js/emotion_engine.ts +++ b/app/pages/PageEnigma/js/emotion_engine.ts @@ -2,7 +2,6 @@ import { EmotionClip } from "../datastructures/clips/emotion_clip"; class EmotionEngine { clips: { [key: string]: EmotionClip } = {}; - audio_sources: { [key: string]: AudioBufferSourceNode } = {}; version: number; constructor(version: number) { @@ -11,9 +10,9 @@ class EmotionEngine { } // loads clips into the engine to cache - loadClip(object_uuid: string, audio_media_id: string) { - if(this.clips[audio_media_id+object_uuid] != null) { return; } - this.clips[audio_media_id+object_uuid] = new EmotionClip(this.version, audio_media_id); + loadClip(object_uuid: string, csv_media_id: string) { + if(this.clips[object_uuid+csv_media_id] != null) { return; } + this.clips[object_uuid+csv_media_id] = new EmotionClip(this.version, csv_media_id); } } diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 52981c81cc..fb4ffe1e44 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -197,6 +197,23 @@ export class TimeLine { 0, ), ); + + this.addPlayableClip( + new ClipUI( + data.data["version"], + ClipType.EMOTION, + ClipGroup.CHARACTER, + "Test", + "m_c0g50khzpg99rq8chjn8zgvxcwebc7", + obj.uuid, + obj.uuid, + name, + 0, + 200, + 0 + ) + ) + this.emotion_engine.loadClip(obj.uuid, "m_c0g50khzpg99rq8chjn8zgvxcwebc7") } public getPos() { @@ -504,7 +521,13 @@ export class TimeLine { this.lipSync_engine.clips[ element.object_uuid + element.media_id ].reset(); - } + } else if (element.type === ClipType.EMOTION) { + const object = this.scene.get_object_by_uuid(element.object_uuid); + if(object) + this.emotion_engine.clips[ + element.object_uuid + element.media_id + ].reset(object); + } } } diff --git a/app/pages/PageEnigma/store/media.ts b/app/pages/PageEnigma/store/media.ts index 0ad2f77013..7c04020b3f 100644 --- a/app/pages/PageEnigma/store/media.ts +++ b/app/pages/PageEnigma/store/media.ts @@ -539,7 +539,7 @@ export const characterItems = signal([ type: AssetType.CHARACTER, name: "Pop", thumbnail: "/resources/placeholders/placeholder.png", - }, + } ]); export const objectItems = signal([