ARKit engine.

This commit is contained in:
Quicksticks-oss committed 2024-04-20 20:38:05 -04:00
1 parent 4a45329fb8
commit 54db432361
4 files changed
+47 -17

No files matched your search

@@ -52,14 +52,14 @@ export class EmotionClip {
header.forEach(key => jsonData[key] = []);
for (let i = 1; i < rows.length; i++) {
const values = rows[i].split(',');
values.forEach((value, index) => {
jsonData[header[index]].push(value);
});
const values = rows[i].split(',');
values.forEach((value, index) => {
jsonData[header[index]].push(value);
});
}
return jsonData;
}
}
async _detect_face(object: THREE.Object3D): Promise<THREE.Mesh> {
@@ -83,7 +83,8 @@ export class EmotionClip {
this.faces.forEach((element: THREE.Mesh) => {
if (element.morphTargetInfluences && shapes) {
Object.keys(shapes).forEach(key => {
const index = element.morphTargetDictionary?.[key];
let index = element.morphTargetDictionary?.[key];
if (index === undefined) { index = element.morphTargetDictionary?.[key.charAt(0).toLowerCase() + key.slice(1)]; }
if (typeof index === 'number' && element.morphTargetInfluences !== undefined) {
element.morphTargetInfluences[index] = shapes[key];
}
@@ -94,14 +95,21 @@ export class EmotionClip {
async reset(object: THREE.Object3D) {
await this._detect_face(object);
// setBlends()
console.log(this.emotion_json);
let keys: { [key: string]: number } = {};
Object.keys(this.emotion_json).forEach(key => {
keys[key] = 0;
});
this.setBlends(keys);
}
async step(frame: number, object: THREE.Object3D) {
await this._detect_face(object);
// setBlends()
console.log(this.emotion_json);
if(this.faces.length <= 0) { await this._detect_face(object); }
let keys: { [key: string]: number } = {};
Object.keys(this.emotion_json).forEach(key => {
if (frame > this.emotion_json[key].length) { return; }
keys[key] = this.emotion_json[key][frame];
});
this.setBlends(keys);
}
toJSON(): any {
+3 -4
View File
@@ -2,7 +2,6 @@ import { EmotionClip } from "../datastructures/clips/emotion_clip";
class EmotionEngine {
clips: { [key: string]: EmotionClip } = {};
audio_sources: { [key: string]: AudioBufferSourceNode } = {};
version: number;
constructor(version: number) {
@@ -11,9 +10,9 @@ class EmotionEngine {
}
// loads clips into the engine to cache
loadClip(object_uuid: string, audio_media_id: string) {
if(this.clips[audio_media_id+object_uuid] != null) { return; }
this.clips[audio_media_id+object_uuid] = new EmotionClip(this.version, audio_media_id);
loadClip(object_uuid: string, csv_media_id: string) {
if(this.clips[object_uuid+csv_media_id] != null) { return; }
this.clips[object_uuid+csv_media_id] = new EmotionClip(this.version, csv_media_id);
}
}
+24 -1
View File
@@ -197,6 +197,23 @@ export class TimeLine {
0,
),
);
this.addPlayableClip(
new ClipUI(
data.data["version"],
ClipType.EMOTION,
ClipGroup.CHARACTER,
"Test",
"m_c0g50khzpg99rq8chjn8zgvxcwebc7",
obj.uuid,
obj.uuid,
name,
0,
200,
0
)
)
this.emotion_engine.loadClip(obj.uuid, "m_c0g50khzpg99rq8chjn8zgvxcwebc7")
}
public getPos() {
@@ -504,7 +521,13 @@ export class TimeLine {
this.lipSync_engine.clips[
element.object_uuid + element.media_id
].reset();
}
} else if (element.type === ClipType.EMOTION) {
const object = this.scene.get_object_by_uuid(element.object_uuid);
if(object)
this.emotion_engine.clips[
element.object_uuid + element.media_id
].reset(object);
}
}
}
+1 -1
View File
@@ -539,7 +539,7 @@ export const characterItems = signal<MediaItem[]>([
type: AssetType.CHARACTER,
name: "Pop",
thumbnail: "/resources/placeholders/placeholder.png",
},
}
]);
export const objectItems = signal<MediaItem[]>([