From 2cc29c6e25abcfab5c5a877d396649072dd7dafd Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Sat, 20 Apr 2024 20:04:22 -0400 Subject: [PATCH] Added emotion engine needs the step functing working but cant work until i get json. --- .../datastructures/clips/emotion_clip.ts | 81 +++++++++++++++++++ app/pages/PageEnigma/js/audio_engine.ts | 1 - app/pages/PageEnigma/js/editor.ts | 18 ++--- app/pages/PageEnigma/js/emotion_engine.ts | 20 +++++ app/pages/PageEnigma/js/timeline.ts | 19 ++++- app/pages/PageEnigma/models/track.ts | 1 + 6 files changed, 127 insertions(+), 13 deletions(-) create mode 100644 app/pages/PageEnigma/datastructures/clips/emotion_clip.ts create mode 100644 app/pages/PageEnigma/js/emotion_engine.ts diff --git a/app/pages/PageEnigma/datastructures/clips/emotion_clip.ts b/app/pages/PageEnigma/datastructures/clips/emotion_clip.ts new file mode 100644 index 0000000000..569f8a3af8 --- /dev/null +++ b/app/pages/PageEnigma/datastructures/clips/emotion_clip.ts @@ -0,0 +1,81 @@ +import * as THREE from "three"; + +export class EmotionClip { + version: number; + media_id: string; + type: "emotion" = "emotion"; + emotion_json: any; + faces: THREE.Mesh[]; + + constructor(version: number, media_id: string) { + this.version = version; + this.media_id = media_id; + this.type = "emotion"; + this.faces = []; + this.download_json().then(data => { + this.emotion_json = data; + }); + } + + async get_media_url() { + //This is for prod when we have the proper info on the url. + let api_base_url = "https://api.fakeyou.com"; + let url = `${api_base_url}/v1/media_files/file/${this.media_id}` + let responce = await fetch(url); + let json = await JSON.parse(await responce.text()); + let bucketPath = json["media_file"]["public_bucket_path"]; + let media_base_url = "https://storage.googleapis.com/vocodes-public" + let media_url = `${media_base_url}${bucketPath}` + return media_url; + } + + async download_json() { + let url = await this.get_media_url(); + const response = await fetch(url); + return await response.json(); + } + + async _detect_face(object: THREE.Object3D): Promise { + this.faces = []; + return new Promise((resolve) => { + object.traverse((c: THREE.Object3D) => { + if (c instanceof THREE.Mesh) { + if (c.morphTargetInfluences && c.morphTargetDictionary) { + const blendShapeIndexE = c.morphTargetDictionary["E"]; + if (blendShapeIndexE != null) { + this.faces.push(c); + resolve(c); + } + } + } + }); + }); + } + + setBlends(shapes: { [key: string]: number }) { + this.faces.forEach((element: THREE.Mesh) => { + if (element.morphTargetInfluences && shapes) { + Object.keys(shapes).forEach(key => { + const index = element.morphTargetDictionary?.[key]; + if (typeof index === 'number' && element.morphTargetInfluences !== undefined) { + element.morphTargetInfluences[index] = shapes[key]; + } + }); + } + }); + } + + + async step(frame: number, object: THREE.Object3D) { + await this._detect_face(object); + // setBlends() + } + + toJSON(): any { + return { + version: this.version, + media_id: this.media_id, + type: this.type, + }; + } +} diff --git a/app/pages/PageEnigma/js/audio_engine.ts b/app/pages/PageEnigma/js/audio_engine.ts index ffa789f783..4c1ac60262 100644 --- a/app/pages/PageEnigma/js/audio_engine.ts +++ b/app/pages/PageEnigma/js/audio_engine.ts @@ -1,4 +1,3 @@ -import { faL } from "@fortawesome/pro-solid-svg-icons"; import { AudioClip } from "../datastructures/clips/audio_clip"; class AudioEngine { diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index e0a4aa2da2..cdd0e25829 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -13,8 +13,11 @@ import { SAOPass } from "three/addons/postprocessing/SAOPass.js"; import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js"; import { BokehPass } from "three/addons/postprocessing/BokehPass.js"; import { createFFmpeg, fetchFile, FFmpeg } from "@ffmpeg/ffmpeg"; + import AudioEngine from "./audio_engine.js"; import TransformEngine from "./transform_engine.js"; +import EmotionEngine from "./emotion_engine"; + import { TimeLine } from "./timeline.js"; import { ClipUI } from "../datastructures/clips/clip_ui.js"; import { LipSyncEngine } from "./lip_sync_engine.js"; @@ -77,6 +80,7 @@ class Editor { max_length: number; audio_engine: AudioEngine; transform_engine: TransformEngine; + emotion_engine: EmotionEngine; lipsync_engine: LipSyncEngine; animation_engine: AnimationEngine; timeline: TimeLine; @@ -108,9 +112,6 @@ class Editor { recorder: MediaRecorder | undefined; // Default params. - // scene proxy for serialization - storyteller_proxy_scene: StoryTellerProxyScene; - constructor() { console.log( "If you see this message twice! then it rendered twice, if you see it once it's all good.", @@ -188,6 +189,7 @@ class Editor { this.canvasRenderCamReference; this.audio_engine = new AudioEngine(); + this.emotion_engine = new EmotionEngine(this.version); this.transform_engine = new TransformEngine(this.version); this.lipsync_engine = new LipSyncEngine(); this.animation_engine = new AnimationEngine(this.version); @@ -198,6 +200,7 @@ class Editor { this.transform_engine, this.lipsync_engine, this.animation_engine, + this.emotion_engine, this.activeScene, this.camera, this.mouse, @@ -217,11 +220,6 @@ class Editor { "((masterpiece, best quality, 8K, detailed)), colorful, epic, fantasy, (fox, red fox:1.2), no humans, 1other, ((koi pond)), outdoors, pond, rocks, stones, koi fish, ((watercolor))), lilypad, fish swimming around."; this.negative_prompt = ""; this.art_style = ArtStyle.Anime2DFlat; - - this.storyteller_proxy_scene = new StoryTellerProxyScene( - this.version, - this.activeScene.scene, - ); } isEmpty(value: string) { @@ -230,7 +228,7 @@ class Editor { ); } - initialize(config: any, sceneToken) { + initialize(config: any, sceneToken: any) { //setup reactland Callbacks this.dispatchAppUiState = config.dispatchAppUiState; @@ -252,6 +250,8 @@ class Editor { // Find the container element const container = document.getElementById("video-scene-container"); + if(container == null) { return; } + // Use the container's dimensions const width = container.offsetWidth; const height = container.offsetHeight; diff --git a/app/pages/PageEnigma/js/emotion_engine.ts b/app/pages/PageEnigma/js/emotion_engine.ts new file mode 100644 index 0000000000..bd64fbb222 --- /dev/null +++ b/app/pages/PageEnigma/js/emotion_engine.ts @@ -0,0 +1,20 @@ +import { EmotionClip } from "../datastructures/clips/emotion_clip"; + +class EmotionEngine { + clips: { [key: string]: EmotionClip } = {}; + audio_sources: { [key: string]: AudioBufferSourceNode } = {}; + version: number; + + constructor(version: number) { + this.clips = {}; + this.version = version; + } + + // loads clips into the engine to cache + loadClip(object_uuid: string, audio_media_id: string) { + if(this.clips[audio_media_id+object_uuid] != null) { return; } + this.clips[audio_media_id+object_uuid] = new EmotionClip(this.version, audio_media_id); + } +} + +export default EmotionEngine; diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 87bdcda920..52981c81cc 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -16,6 +16,7 @@ import { fromEngineActions } from "../Queue/fromEngineActions"; import { ClipGroup, ClipType } from "~/pages/PageEnigma/models/track"; import { MediaItem } from "~/pages/PageEnigma/models"; import Editor from "~/pages/PageEnigma/js/editor"; +import EmotionEngine from "./emotion_engine"; // Every object uuid / entity has a track. export class TimelineDataState { @@ -47,6 +48,10 @@ export class TimeLine { animation_engine: AnimationEngine; // lip sync engine lipSync_engine: LipSyncEngine; + // emotion engine + emotion_engine: EmotionEngine; + + // characters characters: { [key: string]: ClipGroup }; scene: Scene; @@ -62,6 +67,7 @@ export class TimeLine { transform_engine: TransformEngine, lipsync_engine: LipSyncEngine, animation_engine: AnimationEngine, + emotion_engine: EmotionEngine, scene: Scene, camera: THREE.Camera, mouse: THREE.Vector2 | undefined @@ -82,6 +88,7 @@ export class TimeLine { this.transform_engine = transform_engine; this.lipSync_engine = lipsync_engine; this.animation_engine = animation_engine; + this.emotion_engine = emotion_engine; this.scene = scene; @@ -364,9 +371,9 @@ export class TimeLine { keyframe_offset, ); this.transform_engine.clips[object_uuid].setTransform( - keyframe_uuid, - keyframe_pos, - keyframe_rot, + keyframe_uuid, + keyframe_pos, + keyframe_rot, keyframe_scl, ); } @@ -588,6 +595,12 @@ export class TimeLine { this.scrubber_frame_position / fps, // Double FPS for best result. ); } + } else if (element.type === ClipType.EMOTION) { + if (object) { + await this.emotion_engine.clips[ + object.uuid + element.media_id + ].step(this.scrubber_frame_position-element.offset, object); + } } //this.timelineItems = this.timelineItems.filter(item => item !== element) } diff --git a/app/pages/PageEnigma/models/track.ts b/app/pages/PageEnigma/models/track.ts index aede824c56..e74842fd02 100644 --- a/app/pages/PageEnigma/models/track.ts +++ b/app/pages/PageEnigma/models/track.ts @@ -17,6 +17,7 @@ export enum ClipType { TRANSFORM = "transform", AUDIO = "audio", ANIMATION = "animation", + EMOTION = "emotion", FAKE = "fake", }