daniel543x/artcraft
1
0 Fork 0
mirror of https://github.com/storytold/artcraft.git synced 2026-11-10 12:01:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'test' into feature/empty-track-text

Browse Source
This commit is contained in:
Michael Chung committed 2024-04-22 10:01:17 -04:00
2 parents 2866bd935d c1b5f052cc
commit 349ea9b0cb
27 files changed
+2116 -97
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/modules/TopBar/TopBar.tsx
+5 -1
View File
@@ -43,7 +43,11 @@ export const TopBar = ({ pageName }: Props) => {
<div className="flex items-center justify-center font-medium">
<span className="opacity-60">{pageName}&nbsp;&nbsp;/</span>
&nbsp;&nbsp;
{sceneTitle.value}
{sceneTitle.value}
&nbsp;&nbsp;
<span className="opacity-60">
@ CURRENT_STORYTELLER_GIT_VERSION
</span>
</div>
<div className="flex justify-end gap-2">
app/pages/PageEnigma/PageEditor.tsx
+36 -1
View File
@@ -18,6 +18,8 @@ import {
sidePanelVisible,
dndSidePanelWidth,
dndTimelineHeight,
dragItem,
canDrop,
} from "~/pages/PageEnigma/store";
import { useSignals } from "@preact/signals-react/runtime";
import { AppUiContext } from "~/contexts/AppUiContext";
@@ -25,6 +27,7 @@ import { pageHeight, pageWidth } from "~/store";
import { TopBar } from "~/modules/TopBar";
import { DialogueTTS } from "./comps/DialogueTTS/DialogueTTS";
import { Pages } from "~/pages/PageEnigma/constants/page";
import { AssetType } from "./models/assets";
interface Props {
setPage: (page: Pages) => void;
@@ -73,7 +76,39 @@ export const PageEditor = ({ setPage }: Props) => {
? "invisible"
: ""
}>
<canvas id="video-scene" width="1280px" height="720px" />
<div
id="video-scene-container"
style={{
width:
pageWidth.value -
(sidePanelVisible.value ? sidePanelWidth.value : 0) -
84,
height: pageHeight.value - timelineHeight.value - 68,
}}>
<canvas
id="video-scene"
width="1280px"
height="720px"
onPointerOver={() => {
if (
dragItem.value?.type === AssetType.OBJECT ||
dragItem.value?.type === AssetType.SHAPE ||
dragItem.value?.type === AssetType.CHARACTER
) {
canDrop.value = true;
}
}}
onPointerLeave={() => {
if (
dragItem.value?.type === AssetType.OBJECT ||
dragItem.value?.type === AssetType.SHAPE ||
dragItem.value?.type === AssetType.CHARACTER
) {
canDrop.value = false;
}
}}
/>
</div>
{/* Top controls */}
<div
app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/ObjectsTab.tsx
+2 -1
View File
@@ -10,6 +10,7 @@ import { ItemElements } from "~/pages/PageEnigma/comps/SidePanelTabs/itemTabs/It
import { Button } from "~/components";
import { faCirclePlus } from "@fortawesome/pro-solid-svg-icons";
import { twMerge } from "tailwind-merge";
import { shapeItems } from "~/pages/PageEnigma/store";
interface Props {
type: AssetType;
@@ -72,7 +73,7 @@ export const ObjectsTab = ({ type }: Props) => {
</Button>
</div>
<div className="h-full w-full overflow-y-auto px-4">
<ItemElements items={items.value} assetFilter={assetFilter.value} />
<ItemElements items={[ ...type !== AssetType.CHARACTER ? shapeItems.value : [], ...items.value ]} assetFilter={assetFilter.value} />
</div>
</>
);
app/pages/PageEnigma/comps/SidePanelTabs/tabList.tsx
-6
View File
@@ -24,12 +24,6 @@ export const tabList = [
value: AssetType.OBJECT,
component: <ObjectsTab type={AssetType.OBJECT} />,
},
{
icon: "/resources/icons/shapes.png",
title: "Shapes",
value: AssetType.SHAPE,
component: <ShapeTab />,
},
{
icon: "/resources/icons/camera.png",
title: "Camera",
app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsScrubber.ts
+21 -12
View File
@@ -1,4 +1,4 @@
import React, { useCallback, useEffect, useState } from "react";
import React, { useCallback, useEffect, useRef, useState } from "react";
import { currentTime, filmLength, scale } from "~/pages/PageEnigma/store";
import Queue from "~/pages/PageEnigma/Queue/Queue";
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
@@ -6,13 +6,27 @@ import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
export const useMouseEventsScrubber = () => {
const [isActive, setIsActive] = useState(false);
const [clientX, setClientX] = useState(0);
const clientX = useRef(0);
const [time, setTime] = useState(-1);
useEffect(() => {
const getDelta = useCallback((event: MouseEvent) => {
const max = filmLength.value * 60;
const delta = Math.round(
(event.clientX - clientX.current) / 4 / scale.value + currentTime.value,
);
if (delta < 0) {
return 0;
}
if (delta > max) {
return max;
}
return delta;
}, []);
useEffect(() => {
const onPointerUp = () => {
if (isActive) {
currentTime.value = Math.round(time);
@@ -27,16 +41,11 @@ export const useMouseEventsScrubber = () => {
};
const onMouseMove = (event: MouseEvent) => {
const delta = Math.round(
(event.clientX - clientX) / 4 / scale.value + currentTime.value,
);
if (isActive) {
event.stopPropagation();
event.preventDefault();
console.log(delta, max);
if (delta < 0 || delta > max) {
return;
}
const delta = getDelta(event);
setTime((oldTime) => {
if (oldTime !== delta) {
Queue.publish({
@@ -59,12 +68,12 @@ export const useMouseEventsScrubber = () => {
window.removeEventListener("pointerup", onPointerUp);
window.removeEventListener("pointermove", onMouseMove);
};
}, [clientX, isActive, time]);
}, [clientX, isActive, time, getDelta]);
return {
onPointerDown: useCallback((event: React.PointerEvent<HTMLDivElement>) => {
if (event.button === 0) {
setClientX(event.clientX);
clientX.current = event.clientX;
setIsActive(true);
setTime(currentTime.value);
}
app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx
+5
View File
@@ -33,6 +33,11 @@ export const TrackProvider = ({ children }: Props) => {
// cross group functions
const dropClip = useCallback(() => {
if (!canDrop.value) {
endDrag();
return;
}
if (dragItem.value) {
const mediaItem = dragItem.value;
if (mediaItem.type === AssetType.CHARACTER) {
app/pages/PageEnigma/datastructures/clips/emotion_clip.ts
+122
View File
@@ -0,0 +1,122 @@
import * as THREE from "three";
interface CsvJson {
[key: string]: string[];
}
export class EmotionClip {
version: number;
media_id: string;
type: "emotion" = "emotion";
emotion_json: any;
faces: THREE.Mesh[];
constructor(version: number, media_id: string) {
this.version = version;
this.media_id = media_id;
this.type = "emotion";
this.faces = [];
this.download_csv().then(data => {
this.emotion_json = data;
});
}
async get_media_url() {
//This is for prod when we have the proper info on the url.
let api_base_url = "https://api.fakeyou.com";
let url = `${api_base_url}/v1/media_files/file/${this.media_id}`
let responce = await fetch(url);
let json = await JSON.parse(await responce.text());
let bucketPath = json["media_file"]["public_bucket_path"];
let media_base_url = "https://storage.googleapis.com/vocodes-public"
let media_url = `${media_base_url}${bucketPath}`
return media_url;
}
async download_json() {
let url = await this.get_media_url();
const response = await fetch(url);
return await response.json();
}
async download_csv(): Promise<CsvJson> {
let url = await this.get_media_url();
const response = await fetch(url);
const csvText = await response.text();
// Parsing the CSV text
const rows = csvText.trim().split('\n');
const header = rows[0].split(',');
const jsonData: CsvJson = {};
header.forEach(key => jsonData[key] = []);
for (let i = 1; i < rows.length; i++) {
const values = rows[i].split(',');
values.forEach((value, index) => {
jsonData[header[index]].push(value);
});
}
return jsonData;
}
async _detect_face(object: THREE.Object3D): Promise<THREE.Mesh> {
this.faces = [];
return new Promise((resolve) => {
object.traverse((c: THREE.Object3D) => {
if (c instanceof THREE.Mesh) {
if (c.morphTargetInfluences && c.morphTargetDictionary) {
const blendShapeIndexE = c.morphTargetDictionary["E"];
if (blendShapeIndexE != null) {
this.faces.push(c);
resolve(c);
}
}
}
});
});
}
setBlends(shapes: { [key: string]: number }) {
this.faces.forEach((element: THREE.Mesh) => {
if (element.morphTargetInfluences && shapes) {
Object.keys(shapes).forEach(key => {
let index = element.morphTargetDictionary?.[key];
if (index === undefined) { index = element.morphTargetDictionary?.[key.charAt(0).toLowerCase() + key.slice(1)]; }
if (typeof index === 'number' && element.morphTargetInfluences !== undefined) {
element.morphTargetInfluences[index] = shapes[key];
}
});
}
});
}
async reset(object: THREE.Object3D) {
await this._detect_face(object);
let keys: { [key: string]: number } = {};
Object.keys(this.emotion_json).forEach(key => {
keys[key] = 0;
});
this.setBlends(keys);
}
async step(frame: number, object: THREE.Object3D) {
if(this.faces.length <= 0) { await this._detect_face(object); }
let keys: { [key: string]: number } = {};
Object.keys(this.emotion_json).forEach(key => {
if (frame > this.emotion_json[key].length) { return; }
keys[key] = this.emotion_json[key][frame];
});
this.setBlends(keys);
}
toJSON(): any {
return {
version: this.version,
media_id: this.media_id,
type: this.type,
};
}
}
app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts
+3 -2
View File
@@ -116,9 +116,10 @@ export class LipSyncClip {
});
}
step() {
step(frame: number, offset: number, rendering: boolean) {
if (this.lipsync == null) { return; }
const positions = this.lipsync.update();
const positions = this.lipsync.update(frame, offset, rendering);
if(positions)
this.setBlends(positions["ee"], positions["ah"], positions["oh"]);
}
app/pages/PageEnigma/js/TransformControls.js
+1574
View File
File diff suppressed because it is too large. Load diff
app/pages/PageEnigma/js/api_manager.ts
+2 -1
View File
@@ -354,7 +354,8 @@ export class APIManager {
positive_prompt: string,
negative_prompt: string,
): Promise<string> {
const url = 'http://213.173.110.135:15750/preview';
//const url = 'http://213.173.110.135:15750/preview';
const url = 'https://funnel.tailce84f.ts.net/preview/';
const payload = {
style: style,
app/pages/PageEnigma/js/audio_engine.ts
+2 -1
View File
@@ -1,4 +1,3 @@
import { faL } from "@fortawesome/pro-solid-svg-icons";
import { AudioClip } from "../datastructures/clips/audio_clip";
class AudioEngine {
@@ -6,12 +5,14 @@ class AudioEngine {
audio_sources: { [key: string]: AudioBufferSourceNode } = {};
version: number;
playing: string[];
last_frame: number;
constructor() {
this.clips = {};
this.playing = [];
this.audio_sources = {};
this.version = 1.0;
this.last_frame = 0;
}
// loads clips into the engine to cache
app/pages/PageEnigma/js/editor.ts
+112 -45
View File
@@ -1,7 +1,7 @@
import * as THREE from "three";
import { OrbitControls } from "three/addons/controls/OrbitControls.js";
import { FreeCam } from "./free_cam";
import { TransformControls } from "three/addons/controls/TransformControls.js";
import { TransformControls } from "./TransformControls.js";
import Scene from "./scene.js";
import { APIManager, ArtStyle, Visibility } from "./api_manager.js";
import { EffectComposer } from "three/addons/postprocessing/EffectComposer.js";
@@ -13,8 +13,11 @@ import { SAOPass } from "three/addons/postprocessing/SAOPass.js";
import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js";
import { BokehPass } from "three/addons/postprocessing/BokehPass.js";
import { createFFmpeg, fetchFile, FFmpeg } from "@ffmpeg/ffmpeg";
import AudioEngine from "./audio_engine.js";
import TransformEngine from "./transform_engine.js";
import EmotionEngine from "./emotion_engine";
import { TimeLine } from "./timeline.js";
import { ClipUI } from "../datastructures/clips/clip_ui.js";
import { LipSyncEngine } from "./lip_sync_engine.js";
@@ -77,12 +80,14 @@ class Editor {
max_length: number;
audio_engine: AudioEngine;
transform_engine: TransformEngine;
emotion_engine: EmotionEngine;
lipsync_engine: LipSyncEngine;
animation_engine: AnimationEngine;
timeline: TimeLine;
current_frame: number;
lockControls: PointerLockControls | undefined;
cam_obj: THREE.Object3D | undefined;
camera_last_pos: THREE.Vector3;
renderPass: RenderPass | undefined;
generating_preview: boolean;
frames: number;
@@ -107,9 +112,6 @@ class Editor {
recorder: MediaRecorder | undefined;
// Default params.
// scene proxy for serialization
storyteller_proxy_scene: StoryTellerProxyScene;
constructor() {
console.log(
"If you see this message twice! then it rendered twice, if you see it once it's all good.",
@@ -142,6 +144,7 @@ class Editor {
this.outlinePass;
this.last_cam_pos = new THREE.Vector3(0, 0, 0);
this.last_cam_rot = new THREE.Euler(0, 0, 0);
this.camera_last_pos = new THREE.Vector3(0, 0, 0);
this.lockControls;
this.saoPass;
this.outputPass;
@@ -186,6 +189,7 @@ class Editor {
this.canvasRenderCamReference;
this.audio_engine = new AudioEngine();
this.emotion_engine = new EmotionEngine(this.version);
this.transform_engine = new TransformEngine(this.version);
this.lipsync_engine = new LipSyncEngine();
this.animation_engine = new AnimationEngine(this.version);
@@ -196,9 +200,10 @@ class Editor {
this.transform_engine,
this.lipsync_engine,
this.animation_engine,
this.emotion_engine,
this.activeScene,
this.camera,
this.mouse
this.mouse,
);
this.current_frame = 0;
@@ -215,11 +220,6 @@ class Editor {
"((masterpiece, best quality, 8K, detailed)), colorful, epic, fantasy, (fox, red fox:1.2), no humans, 1other, ((koi pond)), outdoors, pond, rocks, stones, koi fish, ((watercolor))), lilypad, fish swimming around.";
this.negative_prompt = "";
this.art_style = ArtStyle.Anime2DFlat;
this.storyteller_proxy_scene = new StoryTellerProxyScene(
this.version,
this.activeScene.scene,
);
}
isEmpty(value: string) {
@@ -228,7 +228,7 @@ class Editor {
);
}
initialize(config: any, sceneToken) {
initialize(config: any, sceneToken: any) {
//setup reactland Callbacks
this.dispatchAppUiState = config.dispatchAppUiState;
@@ -247,9 +247,15 @@ class Editor {
this.canvReference = document.getElementById("video-scene");
this.canvasRenderCamReference = document.getElementById("camera-view");
// Base width and height.
const width = this.canvReference.width;
const height = this.canvReference.height;
// Find the container element
const container = document.getElementById("video-scene-container");
if(container == null) { return; }
// Use the container's dimensions
const width = container.offsetWidth;
const height = container.offsetHeight;
// Sets up camera and base position.
this.camera = new THREE.PerspectiveCamera(70, width / height, 0.15, 30);
this.camera.position.z = 3;
@@ -282,6 +288,7 @@ class Editor {
this.clock = new THREE.Clock();
// Resizes the renderer.
this.renderer.setSize(width, height);
this.renderer.setPixelRatio(window.devicePixelRatio);
//document.body.appendChild(this.renderer.domElement)
window.addEventListener("resize", this.onWindowResize.bind(this));
this._configurePostProcessing();
@@ -307,13 +314,20 @@ class Editor {
this.renderer.domElement,
);
this.orbitControls.mouseButtons = {
MIDDLE: THREE.MOUSE.ROTATE,
RIGHT: THREE.MOUSE.PAN,
}; // Blender Style
// this.orbitControls.mouseButtons = { LEFT: MOUSE.ROTATE, MIDDLE: MOUSE.DOLLY, RIGHT: MOUSE.PAN }; // Standard
this.control = new TransformControls(this.camera, this.renderer.domElement);
this.control.space = "local"; // Local transformation mode
// .space = 'world'; // Global mode
this.control.setScaleSnap(0.1);
this.control.setTranslationSnap(0.1);
this.control.setRotationSnap(0.1);
this.control.setScaleSnap(0.05);
this.control.setTranslationSnap(0.05);
this.control.setRotationSnap(0.05);
console.log("Control Sensitivity:", this.control.sensitivity);
// OnClick and MouseMove events.
window.addEventListener("mousemove", this.onMouseMove.bind(this), false);
@@ -329,6 +343,7 @@ class Editor {
}
this.orbitControls.enabled = !event.value;
this.updateSelectedUI();
this.camera_last_pos.copy(new THREE.Vector3(-99999, -99999, -99999));
// this.update_properties()
});
this.control.setSize(0.5); // Good default value for visuals.
@@ -338,6 +353,8 @@ class Editor {
// Resets canvas size.
this.onWindowResize();
this.setupResizeObserver();
this.timeline.scene = this.activeScene;
//this._test_demo();
@@ -368,19 +385,17 @@ class Editor {
this.loadScene(sceneToken);
}
document.addEventListener('mouseover', (event) => {
document.addEventListener("mouseover", (event) => {
if (this.orbitControls && this.cameraViewControls) {
if (event.target instanceof HTMLCanvasElement) {
if (this.camera_person_mode) {
this.orbitControls.enabled = false;
this.cameraViewControls.enabled = true;
}
else {
} else {
this.orbitControls.enabled = true;
this.cameraViewControls.enabled = false;
}
}
else {
} else {
this.orbitControls.enabled = false;
this.cameraViewControls.enabled = false;
}
@@ -404,7 +419,7 @@ class Editor {
console.log(result);
}
public async testTestTimelineEvents() { }
public async testTestTimelineEvents() {}
public async loadScene(scene_media_token: string) {
this.dispatchAppUiState({
@@ -436,6 +451,7 @@ class Editor {
this.animation_engine,
this.audio_engine,
this.lipsync_engine,
this.emotion_engine,
);
await proxyTimeline.loadFromJson(scene_json["timeline"]);
@@ -473,6 +489,7 @@ class Editor {
this.activeScene,
);
const scene_json = await proxyScene.saveToScene();
console.log("scene_json", scene_json);
const proxyTimeline = new StoryTellerProxyTimeline(
this.version,
@@ -481,6 +498,7 @@ class Editor {
this.animation_engine,
this.audio_engine,
this.lipsync_engine,
this.emotion_engine,
);
const timeline_json = await proxyTimeline.saveToJson();
@@ -678,7 +696,9 @@ class Editor {
deleteObject(uuid: string) {
const obj = this.activeScene.get_object_by_uuid(uuid);
if (obj?.name === "::CAM::") { return; }
if (obj?.name === "::CAM::") {
return;
}
if (obj) {
this.activeScene.scene.remove(obj);
}
@@ -753,7 +773,7 @@ class Editor {
this.render_timer += this.clock.getDelta();
this.frames += 1;
this.playback_location++;
const imgData = this.rawRenderer.domElement.toDataURL('image/png', 1.0); // Medium quality png for speed & size.
const imgData = this.rawRenderer.domElement.toDataURL("image/png", 1.0); // Medium quality png for speed & size.
this.frame_buffer.push(imgData);
this.render_timer += this.clock.getDelta();
if (this.timeline.is_playing == false) {
@@ -765,10 +785,19 @@ class Editor {
}
getselectedSum() {
if (this.selected === undefined) { return 0; }
let posCombo = this.selected.position.x + this.selected.position.y + this.selected.position.z;
let rotCombo = this.selected.rotation.x + this.selected.rotation.y + this.selected.rotation.z;
let sclCombo = this.selected.scale.x + this.selected.scale.y + this.selected.scale.z;
if (this.selected === undefined) {
return 0;
}
let posCombo =
this.selected.position.x +
this.selected.position.y +
this.selected.position.z;
let rotCombo =
this.selected.rotation.x +
this.selected.rotation.y +
this.selected.rotation.z;
let sclCombo =
this.selected.scale.x + this.selected.scale.y + this.selected.scale.z;
return posCombo + rotCombo + sclCombo;
}
@@ -828,8 +857,10 @@ class Editor {
if (changeView) {
this.switchCameraView();
}
}
else if (this.last_scrub === this.timeline.scrubber_frame_position && this.getselectedSum() !== this.last_selected_sum) {
} else if (
this.last_scrub === this.timeline.scrubber_frame_position &&
this.getselectedSum() !== this.last_selected_sum
) {
this.updateSelectedUI();
}
this.last_selected_sum = this.getselectedSum();
@@ -897,10 +928,10 @@ class Editor {
audioSegment,
"-filter_complex",
"[1:a]adelay=" +
startTime * 1000 +
"|" +
startTime * 1000 +
"[a1];[0:a][a1]amix=inputs=2[a]",
startTime * 1000 +
"|" +
startTime * 1000 +
"[a1];[0:a][a1]amix=inputs=2[a]",
"-map",
"[a]",
`${itteration}final_tmp.wav`,
@@ -1172,9 +1203,7 @@ class Editor {
element.visible = true;
});
}
}
else {
} else {
this.timeline.is_playing = true;
this.timeline.scrubber_frame_position = 0;
if (!this.camera_person_mode) {
@@ -1230,9 +1259,11 @@ class Editor {
// Automaticly resize scene.
onWindowResize() {
// Calculate the maximum possible dimensions while maintaining the aspect ratio
const width = window.innerWidth; // / aspect_adjust
const height = window.innerHeight; // / aspectRatio
const container = document.getElementById("video-scene-container");
if (!container) return;
const width = container.clientWidth;
const height = container.clientHeight;
if (this.camera == undefined || this.renderer == undefined) {
return;
@@ -1256,6 +1287,23 @@ class Editor {
this.render_camera.updateProjectionMatrix();
}
setupResizeObserver() {
const container = document.getElementById("video-scene-container");
if (!container) return;
const resizeObserver = new ResizeObserver((entries) => {
for (let entry of entries) {
const { width, height } = entry.contentRect;
this.camera.aspect = width / height;
this.camera.updateProjectionMatrix();
this.renderer?.setSize(width, height);
this.renderer?.setPixelRatio(window.devicePixelRatio);
}
});
resizeObserver.observe(container);
}
onMouseDown(event: any) {
if (event.button === 1 && this.camera_person_mode) {
this.lockControls?.lock();
@@ -1266,6 +1314,15 @@ class Editor {
if (event.button === 1) {
this.lockControls?.unlock();
}
if (event.button !== 0) {
let camera_pos = new THREE.Vector3(
parseFloat(this.camera.position.x.toFixed(2)),
parseFloat(this.camera.position.y.toFixed(2)),
parseFloat(this.camera.position.z.toFixed(2)),
);
this.camera_last_pos.copy(camera_pos);
}
}
// Sets new mouse location usually used in raycasts.
@@ -1281,15 +1338,27 @@ class Editor {
// When the mouse clicks the screen.
onMouseClick() {
let camera_pos = new THREE.Vector3(
parseFloat(this.camera.position.x.toFixed(2)),
parseFloat(this.camera.position.y.toFixed(2)),
parseFloat(this.camera.position.z.toFixed(2)),
);
if (this.camera_last_pos.equals(new THREE.Vector3(0, 0, 0))) {
this.camera_last_pos.copy(camera_pos);
}
if (
this.raycaster == undefined ||
this.mouse == undefined ||
this.control == undefined ||
this.outlinePass == undefined ||
this.camera_person_mode
this.camera_person_mode ||
!this.camera_last_pos.equals(camera_pos)
) {
this.camera_last_pos.copy(camera_pos);
return;
}
this.camera_last_pos.copy(camera_pos);
this.raycaster.setFromCamera(this.mouse, this.camera);
const interactable: any[] = [];
@@ -1333,13 +1402,11 @@ class Editor {
});
this.updateSelectedUI();
}
} else if (this.transform_interaction == false) {
} else {
this.removeTransformControls();
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.HIDE_CONTROLPANELS_SCENEOBJECT,
});
} else {
this.transform_interaction = false;
}
}
}
app/pages/PageEnigma/js/emotion_engine.ts
+19
View File
@@ -0,0 +1,19 @@
import { EmotionClip } from "../datastructures/clips/emotion_clip";
class EmotionEngine {
clips: { [key: string]: EmotionClip } = {};
version: number;
constructor(version: number) {
this.clips = {};
this.version = version;
}
// loads clips into the engine to cache
loadClip(object_uuid: string, csv_media_id: string) {
if(this.clips[object_uuid+csv_media_id] != null) { return; }
this.clips[object_uuid+csv_media_id] = new EmotionClip(this.version, csv_media_id);
}
}
export default EmotionEngine;
app/pages/PageEnigma/js/lipsync.js
+49 -12
View File
@@ -21,6 +21,8 @@ export class LipSync {
constructor(face) {
this.face = face;
this.buffer = null;
this.last_frame = 0;
// WE NEED THIS TO be run at RUNTIME
// we have to modify this for our
@@ -70,24 +72,59 @@ export class LipSync {
if (this.mediaStreamSource) this.mediaStreamSource.stop();
this.mediaStreamSource = this.audioContext.createBufferSource();
this.mediaStreamSource.buffer = buffer;
this.meter = LipSync.createAudioMeter(this.audioContext);
this.mediaStreamSource.connect(this.meter);
this.mediaStreamSource.connect(this.audioContext.destination);
this.mediaStreamSource.start();
// connect the output of mediaStreamSource to the input of userSpeechAnalyzer
this.mediaStreamSource.connect(this.userSpeechAnalyzer);
this.buffer = buffer;
//this.mediaStreamSource = this.audioContext.createBufferSource();
//this.mediaStreamSource.buffer = buffer;
//this.meter = LipSync.createAudioMeter(this.audioContext);
//this.mediaStreamSource.connect(this.meter);
//this.mediaStreamSource.connect(this.audioContext.destination);
////this.mediaStreamSource.start();
//// connect the output of mediaStreamSource to the input of userSpeechAnalyzer
//this.mediaStreamSource.connect(this.userSpeechAnalyzer);
}
destroy() {
this.meter?.shutdown();
this.meter = null;
this.mediaStreamSource?.disconnect();
return this.audioContext?.close().catch(() => {}) || Promise.resolve();
return this.audioContext?.close().catch(() => { }) || Promise.resolve();
}
update() {
update(frame, offset, rendering) {
const frameBuffer = 12;
let pos = frame - offset;
let doPlay = true;
if (Math.abs(this.last_frame - frame) < frameBuffer - 1 && this.last_frame !== 0) { doPlay = false }
if (pos <= 1 && !rendering) {
doPlay = true;
}
else if(!rendering){
doPlay = false;
}
if (doPlay) {
const startTime = pos / 60.0;
const endTime = frameBuffer / 60.0;
this.mediaStreamSource = this.audioContext.createBufferSource();
this.mediaStreamSource.buffer = this.buffer;
this.meter = LipSync.createAudioMeter(this.audioContext);
this.mediaStreamSource.connect(this.meter);
this.mediaStreamSource.connect(this.audioContext.destination);
if (rendering) {
this.mediaStreamSource.start(0, startTime, endTime);
}
else {
this.mediaStreamSource.start();
}
this.mediaStreamSource.connect(this.userSpeechAnalyzer);
this.last_frame = frame;
}
if (this.meter) {
const { volume } = this.meter;
let { ah, oh, ee } = this.update2();
@@ -156,8 +193,8 @@ export class LipSync {
Math.max(EnergyBinFem[1], EnergyBinMasc[1]) > 0.2
? 1 - 2 * Math.max(EnergyBinMasc[2], EnergyBinFem[2])
: (1 - 2 * Math.max(EnergyBinMasc[2], EnergyBinFem[2])) *
5 *
Math.max(EnergyBinMasc[1], EnergyBinFem[1]);
5 *
Math.max(EnergyBinMasc[1], EnergyBinFem[1]);
const ah = 3 * Math.max(EnergyBinMasc[3], EnergyBinFem[3]);
const ee =
app/pages/PageEnigma/js/scene.ts
+10 -7
View File
@@ -51,6 +51,10 @@ class Scene {
//obj.type = "Object3D";
obj.name = name;
obj.position.copy(pos);
obj.userData["color"] = "#FFFFFF";
obj.userData["metalness"] = 0.0;
obj.userData["shininess"] = 0.5;
obj.userData["specular"] = 0.0;
this.scene.add(obj);
return obj.uuid;
}
@@ -138,21 +142,19 @@ class Scene {
if (c instanceof THREE.Mesh) {
c.material.metalness = 0.0;
c.material.specular = 0.5;
c.material.shininess = 0.0;
c.castShadow = true;
c.receiveShadow = true;
c.frustumCulled = false;
c.material.transparent = false;
}
});
//console.log(child);
//if (child.type == "Group") {
// if (auto_add) { child.children.forEach(element => {
// this.scene.add(element);
// }); }
// resolve(child);
//}
child.frustumCulled = false;
child.userData["media_id"] = media_id;
child.userData["color"] = "#FFFFFF";
child.userData["metalness"] = 0.0;
child.userData["shininess"] = 0.5;
child.userData["specular"] = 0.5;
if (auto_add) { this.scene.add(child); }
resolve(child);
});
@@ -179,6 +181,7 @@ class Scene {
resolve(child.children[0]);
return;
}
child.userData["color"] = "#FFFFFF";
if (auto_add) { this.scene.add(child); }
resolve(child);
});
app/pages/PageEnigma/js/timeline.ts
+45 -6
View File
@@ -16,6 +16,7 @@ import { fromEngineActions } from "../Queue/fromEngineActions";
import { ClipGroup, ClipType } from "~/pages/PageEnigma/models/track";
import { MediaItem } from "~/pages/PageEnigma/models";
import Editor from "~/pages/PageEnigma/js/editor";
import EmotionEngine from "./emotion_engine";
// Every object uuid / entity has a track.
export class TimelineDataState {
@@ -47,6 +48,10 @@ export class TimeLine {
animation_engine: AnimationEngine;
// lip sync engine
lipSync_engine: LipSyncEngine;
// emotion engine
emotion_engine: EmotionEngine;
// characters
characters: { [key: string]: ClipGroup };
scene: Scene;
@@ -62,6 +67,7 @@ export class TimeLine {
transform_engine: TransformEngine,
lipsync_engine: LipSyncEngine,
animation_engine: AnimationEngine,
emotion_engine: EmotionEngine,
scene: Scene,
camera: THREE.Camera,
mouse: THREE.Vector2 | undefined
@@ -82,6 +88,7 @@ export class TimeLine {
this.transform_engine = transform_engine;
this.lipSync_engine = lipsync_engine;
this.animation_engine = animation_engine;
this.emotion_engine = emotion_engine;
this.scene = scene;
@@ -190,6 +197,23 @@ export class TimeLine {
0,
),
);
//this.addPlayableClip(
// new ClipUI(
// data.data["version"],
// ClipType.EMOTION,
// ClipGroup.CHARACTER,
// "Test",
// "m_c0g50khzpg99rq8chjn8zgvxcwebc7",
// obj.uuid,
// obj.uuid,
// name,
// 0,
// 200,
// 0
// )
//)
//this.emotion_engine.loadClip(obj.uuid, "m_c0g50khzpg99rq8chjn8zgvxcwebc7")
}
public getPos() {
@@ -301,6 +325,9 @@ export class TimeLine {
case "transform":
this.transform_engine.loadObject(object_uuid, data["data"]["length"]);
break;
case "emotion":
this.emotion_engine.loadClip(object_uuid, media_id);
break;
case "audio":
if (group === "character") {
this.lipSync_engine.load_object(object_uuid, media_id);
@@ -364,9 +391,9 @@ export class TimeLine {
keyframe_offset,
);
this.transform_engine.clips[object_uuid].setTransform(
keyframe_uuid,
keyframe_pos,
keyframe_rot,
keyframe_uuid,
keyframe_pos,
keyframe_rot,
keyframe_scl,
);
}
@@ -497,7 +524,13 @@ export class TimeLine {
this.lipSync_engine.clips[
element.object_uuid + element.media_id
].reset();
}
} else if (element.type === ClipType.EMOTION) {
const object = this.scene.get_object_by_uuid(element.object_uuid);
if(object)
this.emotion_engine.clips[
element.object_uuid + element.media_id
].reset(object);
}
}
}
@@ -553,7 +586,7 @@ export class TimeLine {
} else if (
element.type === ClipType.AUDIO &&
element.group !== ClipGroup.CHARACTER &&
this.is_playing
this.is_playing && !isRendering
) {
if (this.scrubber_frame_position + 1 >= element.length) {
this.audio_engine.stopClip(element.media_id);
@@ -576,7 +609,7 @@ export class TimeLine {
].play(object);
this.lipSync_engine.clips[
element.object_uuid + element.media_id
].step();
].step(this.scrubber_frame_position, element.offset, isRendering);
}
} else if (element.type === ClipType.ANIMATION) {
if (object) {
@@ -588,6 +621,12 @@ export class TimeLine {
this.scrubber_frame_position / fps, // Double FPS for best result.
);
}
} else if (element.type === ClipType.EMOTION) {
if (object) {
await this.emotion_engine.clips[
object.uuid + element.media_id
].step(this.scrubber_frame_position-element.offset, object);
}
}
//this.timelineItems = this.timelineItems.filter(item => item !== element)
}
app/pages/PageEnigma/models/track.ts
+1
View File
@@ -17,6 +17,7 @@ export enum ClipType {
TRANSFORM = "transform",
AUDIO = "audio",
ANIMATION = "animation",
EMOTION = "emotion",
FAKE = "fake",
}
app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts
+29
View File
@@ -21,6 +21,10 @@ export interface ObjectJSON {
object_uuid: string;
object_user_data_name: string;
media_file_token: string;
color: string;
metalness: number,
shininess: number,
specular: number,
}
export class StoryTellerProxy3DObject {
@@ -34,6 +38,10 @@ export class StoryTellerProxy3DObject {
object_name: string;
object_user_data_name: string; // changable name
media_file_token: string;
color: string;
metalness: number;
shininess: number;
specular: number;
constructor(
version: number,
@@ -49,6 +57,19 @@ export class StoryTellerProxy3DObject {
this.object_name = "";
this.object_user_data_name = "";
this.object_uuid = "";
this.color = "";
this.metalness = 0.0;
this.shininess = 0.0;
this.specular = 0.5;
}
getColorAsHexString(object: THREE.Object3D): string {
if (object instanceof THREE.Mesh && object.material instanceof THREE.MeshBasicMaterial) {
return '#' + object.material.color.getHexString();
}
return "#FFFFFF"
}
public async initialize(object: THREE.Object3D) {
@@ -59,6 +80,10 @@ export class StoryTellerProxy3DObject {
this.object_name = object.name;
this.object_user_data_name = object.userData.name;
this.object_uuid = object.uuid;
this.color = object.userData["color"];
this.metalness = object.userData["metalness"];
this.shininess = object.userData["shininess"];
this.specular = object.userData["specular"];
}
public async toJSON(): Promise<ObjectJSON> {
@@ -83,6 +108,10 @@ export class StoryTellerProxy3DObject {
object_uuid: this.object_uuid,
object_user_data_name: this.object_user_data_name,
media_file_token: this.media_file_token,
color: this.color,
metalness: this.metalness,
shininess: this.shininess,
specular: this.specular,
};
return json;
}
app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts
+4
View File
@@ -38,6 +38,10 @@ export class StoryTellerProxyScene {
proxyObject3D.object_user_data_name = child.userData.name;
proxyObject3D.object_name = child.name;
proxyObject3D.object_uuid = child.uuid;
proxyObject3D.color = child.userData["color"];
proxyObject3D.metalness = child.userData["metalness"];
proxyObject3D.shininess = child.userData["shininess"];
proxyObject3D.specular = child.userData["specular"];
const json_data = await proxyObject3D.toJSON();
results.push(json_data);
}
app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts
+18 -1
View File
@@ -1,10 +1,12 @@
import { AnimationClip } from "../datastructures/clips/animation_clip";
import { AudioClip } from "../datastructures/clips/audio_clip";
import { ClipUI } from "../datastructures/clips/clip_ui";
import { EmotionClip } from "../datastructures/clips/emotion_clip";
import { LipSyncClip } from "../datastructures/clips/lipsync_clip";
import { TransformClip } from "../datastructures/clips/transform_clip";
import AnimationEngine from "../js/animation_engine";
import AudioEngine from "../js/audio_engine";
import EmotionEngine from "../js/emotion_engine";
import LipSyncEngine from "../js/lip_sync_engine";
import { TimeLine } from "../js/timeline";
import TransformEngine from "../js/transform_engine";
@@ -16,13 +18,15 @@ export class StoryTellerProxyTimeline {
animation_engine: AnimationEngine
audio_engine: AudioEngine
lipsync_engine: LipSyncEngine
emotion_engine: EmotionEngine
constructor(version: number, timeline: TimeLine, transform_engine: TransformEngine, animation_engine: AnimationEngine, audio_engine: AudioEngine, lipsync_engine: LipSyncEngine) {
constructor(version: number, timeline: TimeLine, transform_engine: TransformEngine, animation_engine: AnimationEngine, audio_engine: AudioEngine, lipsync_engine: LipSyncEngine, emotion_engine: EmotionEngine) {
this.timeline = timeline;
this.transform_engine = transform_engine;
this.animation_engine = animation_engine;
this.audio_engine = audio_engine;
this.lipsync_engine = lipsync_engine;
this.emotion_engine = emotion_engine;
}
private async getItems(items: any[]): Promise<any[]> {
@@ -52,6 +56,7 @@ export class StoryTellerProxyTimeline {
animation: await this.getItemsDict(this.animation_engine.clips),
audio: await this.getItemsDict(this.audio_engine.clips),
lipsync: await this.getItemsDict(this.lipsync_engine.clips),
emotion: await this.getItemsDict(this.emotion_engine.clips),
}
console.log(timeline_json)
@@ -122,12 +127,24 @@ export class StoryTellerProxyTimeline {
return timeline_items_data;
}
private async loadEmotionClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> {
let timeline_items_data: { [key: string]: any } = {};
for (const key in items) {
if (items.hasOwnProperty(key)) {
const element = items[key];
timeline_items_data[key] = new EmotionClip(element.version, element.media_id);
}
}
return timeline_items_data;
}
public async loadFromJson(timeline: any) {
this.timeline.timeline_items = await this.loadTimelineClips(timeline['timeline']);
this.transform_engine.clips = await this.loadTransformClips(timeline['transform']);
this.audio_engine.clips = await this.loadAudioClips(timeline['audio']);
this.lipsync_engine.clips = await this.loadLipsyncClips(timeline['lipsync']);
this.animation_engine.clips = await this.loadAnimationClips(timeline['animation']);
this.emotion_engine.clips = await this.loadEmotionClips(timeline['emotion']);
console.log(this.timeline.timeline_items)
app/pages/PageEnigma/store/media.ts
+35
View File
@@ -525,6 +525,20 @@ export const characterItems = signal<MediaItem[]>([
type: AssetType.CHARACTER,
name: "Waifu H",
thumbnail: "/resources/characters/img14.png",
},
{
version: 1,
media_id: "m_qfqw9e5kz0b7kspbwcc3c9y7pym97q",
type: AssetType.CHARACTER,
name: "Skibidi Toilet",
thumbnail: "/resources/characters/img15.png",
},
{
version: 1,
media_id: "m_ecfxa94v4ftvh58hv7se3qqy8a385n",
type: AssetType.CHARACTER,
name: "Pop",
thumbnail: "/resources/placeholders/placeholder.png",
}
]);
@@ -536,6 +550,27 @@ export const objectItems = signal<MediaItem[]>([
name: "Pikachu Statue",
thumbnail: "/resources/placeholders/placeholder.png",
},
{
version: 1,
media_id: "m_4d2s2q1xy9m1xncppr3gf9ne6bm2m0",
type: AssetType.OBJECT,
name: "Moon",
thumbnail: "/resources/objects/moon.png",
},
{
version: 1,
media_id: "m_t0s6gbvfp78rvc0sd6129za41crr9d",
type: AssetType.OBJECT,
name: "Crown",
thumbnail: "/resources/objects/crown.png",
},
{
version: 1,
media_id: "m_zp0zykww0k7ka2zagg3q3pb9t123k4",
type: AssetType.OBJECT,
name: "Sakura Tree",
thumbnail: "/resources/objects/sakura.png",
},
]);
// In the future these will have shape id's
build-ci.sh
Executable
+21
View File
@@ -0,0 +1,21 @@
#!/bin/bash
# NB: This file is executed by Netlify to build the website
# Verbose printing, exit on failure.
set -euxo pipefail
# Add the GIT SHA to the build
# This must be done before everything else, or it will get cached with the build.
# (This might be making the builds less performant?)
# COMMIT_REF is defined by Netlify to be the commit SHA
# We want a short 8 character reference.
echo "Labelling build with short SHA..."
SHORT_SHA=$(echo "${COMMIT_REF}" | cut -c1-8)
find . -type f -exec sed -i "s/CURRENT_STORYTELLER_GIT_VERSION/${SHORT_SHA}/g" {} +
# TODO: Run tests when we add them.
# Run build.
npm run build
netlify.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[build]
command = "npm run build"
command = "./build-ci.sh"
publish = "build/client"
[dev]
public/resources/characters/img15.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 418 KiB

public/resources/objects/crown.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 245 KiB

public/resources/objects/moon.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 190 KiB

public/resources/objects/sakura.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 263 KiB