);
};
diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/ClipElements.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/ClipElements.tsx
index defcb65ac3..1c935c9fc2 100644
--- a/app/pages/PageEnigma/comps/SidePanelTabs/ClipElements.tsx
+++ b/app/pages/PageEnigma/comps/SidePanelTabs/ClipElements.tsx
@@ -1,14 +1,24 @@
-import { useContext } from "react";
+import { useContext, useEffect, useState } from "react";
import { ClipElement } from "./ClipElement";
import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext";
+import { useSignals } from "@preact/signals-react/runtime";
+import { timelineHeight } from "~/pages/PageEnigma/store";
export const ClipElements = () => {
const { animationClips } = useContext(TrackContext);
+ useSignals();
+ const [height, setHeight] = useState(0);
+
+ useEffect(() => {
+ setHeight(window.outerHeight - timelineHeight.value);
+ }, []);
return (
-
+
{animationClips.map((clip) => {
- return ;
+ return (
+
+ );
})}
);
diff --git a/app/pages/PageEnigma/comps/Timeline/Timeline.tsx b/app/pages/PageEnigma/comps/Timeline/Timeline.tsx
index 4c61defb6c..3fc61d7cad 100644
--- a/app/pages/PageEnigma/comps/Timeline/Timeline.tsx
+++ b/app/pages/PageEnigma/comps/Timeline/Timeline.tsx
@@ -21,7 +21,7 @@ import { EngineContext } from "../../contexts/EngineContext";
export const Timeline = () => {
const editorEngine = useContext(EngineContext);
-
+
const {
characters,
objects,
@@ -148,7 +148,7 @@ export const Timeline = () => {
style={{
left: 0,
top: 8,
- width: 2,
+ width: 4,
height: fullHeight,
}}
/>
diff --git a/app/pages/PageEnigma/comps/Timeline/TrackClips.tsx b/app/pages/PageEnigma/comps/Timeline/TrackClips.tsx
index fc37f21df9..f07ee1f593 100644
--- a/app/pages/PageEnigma/comps/Timeline/TrackClips.tsx
+++ b/app/pages/PageEnigma/comps/Timeline/TrackClips.tsx
@@ -1,7 +1,7 @@
import { TrackClip } from "~/pages/PageEnigma/comps/Timeline/TrackClip";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faVolume, faVolumeSlash } from "@fortawesome/pro-solid-svg-icons";
-import { BaseClip } from "~/pages/PageEnigma/models/track";
+import { Clip } from "~/pages/PageEnigma/models/track";
import { PointerEvent } from "react";
import {
canDrop,
@@ -15,7 +15,7 @@ import {
interface Props {
id: string;
- clips: BaseClip[];
+ clips: Clip[];
title: string;
style: "character" | "audio" | "camera" | "objects";
type?: "animations" | "positions" | "lipSync";
@@ -94,7 +94,7 @@ export const TrackClips = ({
>
{clips.map((clip, index) => (
0 ? clips[index - 1].offset + clips[index - 1].length : 0
}
diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsAnimation.ts b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsAnimation.ts
index e145579bb9..25de508f55 100644
--- a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsAnimation.ts
+++ b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsAnimation.ts
@@ -20,7 +20,7 @@ export const useMouseEventsAnimation = () => {
setTime(-1);
Queue.publish({
queueName: QueueNames.TO_ENGINE,
- action: toEngineActions.SCRUBBER_TIME,
+ action: toEngineActions.UPDATE_TIME,
data: { currentTime: Math.round(time) },
});
}
diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts
index 8d666599a2..1dbe7ed124 100644
--- a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts
+++ b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts
@@ -1,10 +1,10 @@
import React, { Dispatch, useCallback, useEffect, useRef } from "react";
-import { BaseClip } from "~/pages/PageEnigma/models/track";
+import { Clip } from "~/pages/PageEnigma/models/track";
import { canDrop, scale } from "~/pages/PageEnigma/store";
import { useSignals } from "@preact/signals-react/runtime";
export const useMouseEventsClip = (
- clip: BaseClip,
+ clip: Clip,
max: number,
min: number,
updateClip: (args: { id: string; offset: number; length: number }) => void,
@@ -21,21 +21,26 @@ export const useMouseEventsClip = (
const onPointerUp = useCallback(() => {
if (isActive.current) {
updateClip({
- id: clip.id,
+ id: clip.clip_uuid,
offset: Math.round(currOffset.current),
length: Math.round(currLength.current),
});
isActive.current = "";
canDrop.value = false;
}
- }, [updateClip, clip.id]);
+ }, [updateClip, clip.clip_uuid]);
const onMouseMove = useCallback(
(event: MouseEvent) => {
const delta = (event.clientX - clientX.current) / 4 / scale.value;
const deltaOffset = delta + initOffset.current;
if (isActive.current === "drag") {
- if (deltaOffset < min || deltaOffset + currLength.current > max) {
+ if (deltaOffset < min) {
+ currOffset.current = min;
+ return;
+ }
+ if (deltaOffset + currLength.current > max) {
+ currOffset.current = max - currLength.current;
return;
}
currOffset.current = deltaOffset;
diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts b/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts
index bc9b9176d1..3359ce8931 100644
--- a/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts
+++ b/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts
@@ -1,28 +1,27 @@
import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions";
import { useSignals } from "@preact/signals-react/runtime";
-import { useCallback, useContext, useEffect } from "react";
+import { useCallback, useEffect } from "react";
import { currentTime } from "~/pages/PageEnigma/store";
-import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext";
import Queue from "~/pages/PageEnigma/Queue/Queue";
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
+import { QueueClip } from "~/pages/PageEnigma/models/track";
+
+interface UpdateTime {
+ currentTime: number;
+}
interface Arguments {
action: fromEngineActions | toEngineActions;
- data: any;
+ data: QueueClip | UpdateTime;
}
export function useQueueHandler() {
useSignals();
- const { addObject } = useContext(TrackContext);
-
const handleActions = useCallback(({ action, data }: Arguments) => {
switch (action) {
- case fromEngineActions.FRAME_TICK:
- currentTime.value += 1;
- break;
- case fromEngineActions.OBJECT_ADDED:
- addObject(data);
+ case fromEngineActions.UPDATE_TIME:
+ currentTime.value = (data as UpdateTime).currentTime;
break;
default:
throw new Error(`Unknown action ${action}`);
diff --git a/app/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx b/app/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx
index bd523c4041..f761df637e 100644
--- a/app/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx
+++ b/app/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx
@@ -1,13 +1,11 @@
import { createContext } from "react";
import {
- AnimationClip,
- AudioClip,
AudioGroup,
- BaseClip,
CameraGroup,
CharacterGroup,
ObjectGroup,
ObjectTrack,
+ MediaClip,
} from "~/pages/PageEnigma/models/track";
export const TrackContext = createContext<{
@@ -23,13 +21,13 @@ export const TrackContext = createContext<{
addCharacterAnimation: (options: {
clipId: string;
characterId: string;
- animationClips: BaseClip[];
+ animationClips: MediaClip[];
offset: number;
}) => void;
addCharacterAudio: (options: {
clipId: string;
characterId: string;
- audioClips: BaseClip[];
+ audioClips: MediaClip[];
offset: number;
}) => void;
selectCharacterClip: (clipId: string) => void;
@@ -55,7 +53,7 @@ export const TrackContext = createContext<{
toggleAudioMute: () => void;
addGlobalAudio: (
clipId: string,
- animationClips: BaseClip[],
+ animationClips: MediaClip[],
offset: number,
) => void;
selectAudioClip: (clipId: string) => void;
@@ -71,8 +69,8 @@ export const TrackContext = createContext<{
selectClip: (clipId: string) => void;
// sidebar clips
- animationClips: AnimationClip[];
- audioClips: AudioClip[];
+ animationClips: MediaClip[];
+ audioClips: MediaClip[];
// drag and drop
startDrag: (
diff --git a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx
index 413b947e99..bc528311bd 100644
--- a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx
+++ b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx
@@ -4,7 +4,7 @@ import useUpdateCharacters from "~/pages/PageEnigma/contexts/TrackContext/utils/
import useUpdateCamera from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCamera";
import useUpdateAudio from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateAudio";
import useUpdateObject from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateObject";
-import { AnimationClip, AudioClip } from "~/pages/PageEnigma/models/track";
+import { ClipType, MediaClip } from "~/pages/PageEnigma/models/track";
import useUpdateDragDrop from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateDragDrop";
import {
canDrop,
@@ -13,6 +13,7 @@ import {
dropId,
dropOffset,
} from "~/pages/PageEnigma/store";
+import * as uuid from "uuid";
interface Props {
children: ReactNode;
@@ -31,8 +32,8 @@ export const TrackProvider = ({ children }: Props) => {
setSelectedClip(clipId);
}, []);
- const [animationClips, setAnimationClips] = useState([]);
- const [audioClips, setAudioClips] = useState([]);
+ const [animationClips, setAnimationClips] = useState([]);
+ const [audioClips, setAudioClips] = useState([]);
const [scale, setScale] = useState(1);
const [length, setLength] = useState(1);
@@ -74,26 +75,30 @@ export const TrackProvider = ({ children }: Props) => {
useEffect(() => {
setAnimationClips([
{
- id: "ani-id1",
- offset: 0,
+ version: 1,
+ media_id: uuid.v4(),
+ type: ClipType.ANIMATION,
length: 25,
name: "Sit",
},
{
- id: "ani-id2",
- offset: 0,
+ version: 1,
+ media_id: uuid.v4(),
+ type: ClipType.ANIMATION,
length: 25,
name: "Idle",
},
{
- id: "ani-id3",
- offset: 0,
+ version: 1,
+ media_id: uuid.v4(),
+ type: ClipType.ANIMATION,
length: 25,
name: "Stand",
},
{
- id: "ani-id4",
- offset: 0,
+ version: 1,
+ media_id: uuid.v4(),
+ type: ClipType.ANIMATION,
length: 25,
name: "Walk",
},
diff --git a/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateAudio.ts b/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateAudio.ts
index 2052926c16..5804527373 100644
--- a/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateAudio.ts
+++ b/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateAudio.ts
@@ -1,24 +1,21 @@
import { useCallback, useState } from "react";
-import { AudioGroup, BaseClip } from "~/pages/PageEnigma/models/track";
+import {
+ AudioGroup,
+ Clip,
+ ClipGroup,
+ ClipType,
+ MediaClip,
+} from "~/pages/PageEnigma/models/track";
+import Queue from "~/pages/PageEnigma/Queue/Queue";
+import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
+import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
+import * as uuid from "uuid";
export default function useUpdateAudio() {
const [audio, setAudio] = useState({
id: "AU1",
muted: false,
- clips: [
- {
- id: "AU1-1",
- length: 200,
- offset: 0,
- name: "audio 1",
- },
- {
- id: "AU1-2",
- length: 180,
- offset: 300,
- name: "audio 2",
- },
- ],
+ clips: [],
});
const updateAudio = useCallback(
@@ -33,44 +30,56 @@ export default function useUpdateAudio() {
}) => {
setAudio((oldAudio) => {
const newClips = [...oldAudio.clips];
- const clipIndex = newClips.findIndex((row) => row.id === id);
+ const clipIndex = newClips.findIndex((row) => row.clip_uuid === id);
if (clipIndex === -1) {
return { ...oldAudio };
}
const clip = newClips[clipIndex];
clip.offset = offset;
clip.length = length;
+
+ Queue.publish({
+ queueName: QueueNames.TO_ENGINE,
+ action: toEngineActions.UPDATE_CLIP,
+ data: clip,
+ });
+
return {
...oldAudio,
clips: newClips,
};
});
- console.log("message", {
- action: "UpdateGlobalAudio",
- id,
- data: { offset, length },
- });
},
[],
);
const addGlobalAudio = useCallback(
- (dragId: string, audioClips: BaseClip[], offset: number) => {
- const clip = audioClips.find((row) => row.id === dragId);
+ (dragId: string, audioClips: MediaClip[], offset: number) => {
+ const clip = audioClips.find((row) => row.media_id === dragId);
if (!clip) {
return;
}
+ const clip_uuid = uuid.v4();
+ const newClip = {
+ ...clip,
+ type: ClipType.AUDIO,
+ group: ClipGroup.GLOBAL_AUDIO,
+ offset,
+ clip_uuid,
+ } as Clip;
+
setAudio((oldAudio) => {
return {
...oldAudio,
- clips: [...oldAudio.clips, { ...clip, offset }],
+ clips: [...oldAudio.clips, newClip],
};
});
- console.log("message", {
- action: "UpdateGlobalAudio",
- id: dragId,
- data: { offset },
+
+ Queue.publish({
+ queueName: QueueNames.TO_ENGINE,
+ action: toEngineActions.ADD_CLIP,
+ data: newClip,
});
},
[],
@@ -78,15 +87,21 @@ export default function useUpdateAudio() {
const toggleAudioMute = useCallback(() => {
setAudio((oldAudio) => {
+ Queue.publish({
+ queueName: QueueNames.TO_ENGINE,
+ action: oldAudio?.muted ? toEngineActions.UNMUTE : toEngineActions.MUTE,
+ data: {
+ version: 1,
+ type: ClipType.AUDIO,
+ group: ClipGroup.GLOBAL_AUDIO,
+ },
+ });
+
return {
...oldAudio,
muted: !oldAudio.muted,
};
});
- console.log("message", {
- action: "ToggleGlobalAudioMute",
- id: "",
- });
}, []);
const selectAudioClip = useCallback((clipId: string) => {
@@ -97,7 +112,8 @@ export default function useUpdateAudio() {
...oldAudio.clips.map((clip) => {
return {
...clip,
- selected: clip.id === clipId ? !clip.selected : clip.selected,
+ selected:
+ clip.clip_uuid === clipId ? !clip.selected : clip.selected,
};
}),
],
@@ -109,13 +125,21 @@ export default function useUpdateAudio() {
setAudio((oldAudio) => {
return {
...oldAudio,
- clips: [...oldAudio.clips.filter((clip) => clip.id !== clipId)],
+ clips: [
+ ...oldAudio.clips.filter((clip) => {
+ if (clip.clip_uuid === clipId) {
+ Queue.publish({
+ queueName: QueueNames.TO_ENGINE,
+ action: toEngineActions.DELETE_CLIP,
+ data: clip!,
+ });
+ return false;
+ }
+ return true;
+ }),
+ ],
};
});
- console.log("message", {
- action: "UpdateGlobalAudio",
- id: clipId,
- });
}, []);
return {
diff --git a/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCamera.ts b/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCamera.ts
index b5b7d21da6..d7bc1395b8 100644
--- a/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCamera.ts
+++ b/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCamera.ts
@@ -1,23 +1,13 @@
import { useCallback, useState } from "react";
import { CameraGroup } from "~/pages/PageEnigma/models/track";
+import Queue from "~/pages/PageEnigma/Queue/Queue";
+import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
+import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
export default function useUpdateCamera() {
const [camera, setCamera] = useState({
id: "CA1",
- clips: [
- {
- id: "CA1-1",
- length: 200,
- offset: 0,
- name: "cam 1",
- },
- {
- id: "CA1-2",
- length: 180,
- offset: 300,
- name: "cam 2",
- },
- ],
+ clips: [],
});
const updateCamera = useCallback(
@@ -32,13 +22,20 @@ export default function useUpdateCamera() {
}) => {
setCamera((oldCamera) => {
const newClips = [...oldCamera.clips];
- const clipIndex = newClips.findIndex((row) => row.id === id);
+ const clipIndex = newClips.findIndex((row) => row.clip_uuid === id);
if (clipIndex === -1) {
return { ...oldCamera };
}
const clip = newClips[clipIndex];
clip.offset = offset;
clip.length = length;
+
+ Queue.publish({
+ queueName: QueueNames.TO_ENGINE,
+ action: toEngineActions.UPDATE_CLIP,
+ data: clip,
+ });
+
return {
...oldCamera,
clips: newClips,
@@ -56,7 +53,8 @@ export default function useUpdateCamera() {
...oldCamera.clips.map((clip) => {
return {
...clip,
- selected: clip.id === clipId ? !clip.selected : clip.selected,
+ selected:
+ clip.clip_uuid === clipId ? !clip.selected : clip.selected,
};
}),
],
@@ -68,7 +66,19 @@ export default function useUpdateCamera() {
setCamera((oldCamera) => {
return {
...oldCamera,
- clips: [...oldCamera.clips.filter((clip) => clip.id !== clipId)],
+ clips: [
+ ...oldCamera.clips.filter((clip) => {
+ if (clip.clip_uuid === clipId) {
+ Queue.publish({
+ queueName: QueueNames.TO_ENGINE,
+ action: toEngineActions.DELETE_CLIP,
+ data: clip!,
+ });
+ return false;
+ }
+ return true;
+ }),
+ ],
};
});
}, []);
diff --git a/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCharacters.ts b/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCharacters.ts
index 4f58f67f08..cd429c0024 100644
--- a/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCharacters.ts
+++ b/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCharacters.ts
@@ -1,5 +1,11 @@
import { useCallback, useState } from "react";
-import { BaseClip, CharacterGroup } from "~/pages/PageEnigma/models/track";
+import {
+ Clip,
+ CharacterGroup,
+ ClipGroup,
+ ClipType,
+ MediaClip,
+} from "~/pages/PageEnigma/models/track";
import * as uuid from "uuid";
import Queue from "~/pages/PageEnigma/Queue/Queue";
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
@@ -12,16 +18,10 @@ export default function useUpdateCharacters() {
muted: false,
animationClips: [],
positionClips: [],
- lipSyncClips: [
- {
- id: "CH1-L1",
- length: 400,
- offset: 150,
- name: "sync 1",
- },
- ],
+ lipSyncClips: [],
},
]);
+
const updateCharacters = useCallback(
({
type,
@@ -39,7 +39,7 @@ export default function useUpdateCharacters() {
return oldCharacters.map((character) => {
const newAnimationClips = [...character.animationClips];
const clipIndex = newAnimationClips.findIndex(
- (row) => row.id === id,
+ (row) => row.clip_uuid === id,
);
if (clipIndex === -1) {
return { ...character };
@@ -47,25 +47,28 @@ export default function useUpdateCharacters() {
const clip = newAnimationClips[clipIndex];
clip.offset = offset;
clip.length = length;
+
+ Queue.publish({
+ queueName: QueueNames.TO_ENGINE,
+ action: toEngineActions.UPDATE_CLIP,
+ data: clip,
+ });
+
return {
...character,
animationClips: newAnimationClips,
};
});
});
- console.log("message", {
- action: "UpdateAnimation",
- id,
- data: { offset, length },
- });
return;
}
+
if (type === "positions") {
setCharacters((oldCharacters) => {
return oldCharacters.map((character) => {
const newPositionClips = [...character.positionClips];
const clipIndex = newPositionClips.findIndex(
- (row) => row.id === id,
+ (row) => row.clip_uuid === id,
);
if (clipIndex === -1) {
return { ...character };
@@ -73,6 +76,13 @@ export default function useUpdateCharacters() {
const clip = newPositionClips[clipIndex];
clip.offset = offset;
clip.length = length;
+
+ Queue.publish({
+ queueName: QueueNames.TO_ENGINE,
+ action: toEngineActions.UPDATE_CLIP,
+ data: clip,
+ });
+
return {
...character,
positionClips: newPositionClips,
@@ -85,24 +95,28 @@ export default function useUpdateCharacters() {
setCharacters((oldCharacters) => {
return oldCharacters.map((character) => {
const newLipSyncClips = [...character.lipSyncClips];
- const clipIndex = newLipSyncClips.findIndex((row) => row.id === id);
+ const clipIndex = newLipSyncClips.findIndex(
+ (row) => row.clip_uuid === id,
+ );
if (clipIndex === -1) {
return { ...character };
}
const clip = newLipSyncClips[clipIndex];
clip.offset = offset;
clip.length = length;
+
+ Queue.publish({
+ queueName: QueueNames.TO_ENGINE,
+ action: toEngineActions.UPDATE_CLIP,
+ data: clip,
+ });
+
return {
...character,
lipSyncClips: newLipSyncClips,
};
});
});
- console.log("message", {
- action: "UpdateLipSync",
- id,
- data: { offset, length },
- });
}
},
[],
@@ -117,14 +131,23 @@ export default function useUpdateCharacters() {
}: {
clipId: string;
characterId: string;
- animationClips: BaseClip[];
+ animationClips: MediaClip[];
offset: number;
}) => {
- const clip = animationClips.find((row) => row.id === clipId);
+ const clip = animationClips.find((row) => row.media_id === clipId);
if (!clip) {
return;
}
+ const clip_uuid = uuid.v4();
+ const newClip = {
+ ...clip,
+ group: ClipGroup.CHARACTER,
+ offset,
+ clip_uuid,
+ object_uuid: characterId,
+ } as Clip;
+
setCharacters((oldCharacters) => {
return oldCharacters.map((character) => {
if (character.id !== characterId) {
@@ -132,20 +155,17 @@ export default function useUpdateCharacters() {
}
return {
...character,
- animationClips: [
- ...character.animationClips,
- { ...clip, id: uuid.v4(), offset },
- ],
+ animationClips: [...character.animationClips, newClip].sort(
+ (clipA, clipB) => clipA.offset - clipB.offset,
+ ),
};
});
});
+
Queue.publish({
queueName: QueueNames.TO_ENGINE,
- action: toEngineActions.ADD_ANIMATION,
- data: {
- clipId,
- offset,
- },
+ action: toEngineActions.ADD_CLIP,
+ data: newClip,
});
},
[],
@@ -160,14 +180,24 @@ export default function useUpdateCharacters() {
}: {
clipId: string;
characterId: string;
- audioClips: BaseClip[];
+ audioClips: MediaClip[];
offset: number;
}) => {
- const clip = audioClips.find((row) => row.id === clipId);
+ const clip = audioClips.find((row) => row.media_id === clipId);
if (!clip) {
return;
}
+ const clip_uuid = uuid.v4();
+ const newClip = {
+ ...clip,
+ type: ClipType.AUDIO,
+ group: ClipGroup.CHARACTER,
+ offset,
+ clip_uuid,
+ object_uuid: characterId,
+ } as Clip;
+
setCharacters((oldCharacters) => {
return oldCharacters.map((character) => {
if (character.id !== characterId) {
@@ -175,14 +205,17 @@ export default function useUpdateCharacters() {
}
return {
...character,
- lipSyncClips: [...character.lipSyncClips, { ...clip, offset }],
+ lipSyncClips: [...character.lipSyncClips, newClip].sort(
+ (clipA, clipB) => clipA.offset - clipB.offset,
+ ),
};
});
});
- console.log("message", {
- action: "AddLipSync",
- id: clipId,
- data: { offset },
+
+ Queue.publish({
+ queueName: QueueNames.TO_ENGINE,
+ action: toEngineActions.ADD_CLIP,
+ data: newClip,
});
},
[],
@@ -190,15 +223,28 @@ export default function useUpdateCharacters() {
const toggleLipSyncMute = useCallback((characterId: string) => {
setCharacters((oldCharacters) => {
- return oldCharacters.map((character) => ({
- ...character,
- muted:
- character.id === characterId ? !character.muted : character.muted,
- }));
- });
- console.log("message", {
- action: "ToggleLipSync",
- id: characterId,
+ return oldCharacters.map((character) => {
+ if (character.id === characterId) {
+ Queue.publish({
+ queueName: QueueNames.TO_ENGINE,
+ action: character?.muted
+ ? toEngineActions.UNMUTE
+ : toEngineActions.MUTE,
+ data: {
+ version: 1,
+ type: ClipType.AUDIO,
+ group: ClipGroup.CHARACTER,
+ object_uuid: characterId,
+ },
+ });
+ }
+
+ return {
+ ...character,
+ muted:
+ character.id === characterId ? !character.muted : character.muted,
+ };
+ });
});
}, []);
@@ -209,15 +255,18 @@ export default function useUpdateCharacters() {
...character,
animationClips: character.animationClips.map((clip) => ({
...clip,
- selected: clip.id === clipId ? !clip.selected : clip.selected,
+ selected:
+ clip.clip_uuid === clipId ? !clip.selected : clip.selected,
})),
positionClips: character.positionClips.map((clip) => ({
...clip,
- selected: clip.id === clipId ? !clip.selected : clip.selected,
+ selected:
+ clip.clip_uuid === clipId ? !clip.selected : clip.selected,
})),
lipSyncClips: character.lipSyncClips.map((clip) => ({
...clip,
- selected: clip.id === clipId ? !clip.selected : clip.selected,
+ selected:
+ clip.clip_uuid === clipId ? !clip.selected : clip.selected,
})),
})),
];
@@ -229,22 +278,42 @@ export default function useUpdateCharacters() {
return [
...oldCharacters.map((character) => ({
...character,
- animationClips: character.animationClips.filter(
- (clip) => clip.id !== clipId,
- ),
- positionClips: character.positionClips.filter(
- (clip) => clip.id !== clipId,
- ),
- lipSyncClips: character.lipSyncClips.filter(
- (clip) => clip.id !== clipId,
- ),
+ animationClips: character.animationClips.filter((clip) => {
+ if (clip.clip_uuid === clipId) {
+ Queue.publish({
+ queueName: QueueNames.TO_ENGINE,
+ action: toEngineActions.DELETE_CLIP,
+ data: clip!,
+ });
+ return false;
+ }
+ return true;
+ }),
+ positionClips: character.positionClips.filter((clip) => {
+ if (clip.clip_uuid === clipId) {
+ Queue.publish({
+ queueName: QueueNames.TO_ENGINE,
+ action: toEngineActions.DELETE_CLIP,
+ data: clip!,
+ });
+ return false;
+ }
+ return true;
+ }),
+ lipSyncClips: character.lipSyncClips.filter((clip) => {
+ if (clip.clip_uuid === clipId) {
+ Queue.publish({
+ queueName: QueueNames.TO_ENGINE,
+ action: toEngineActions.DELETE_CLIP,
+ data: clip!,
+ });
+ return false;
+ }
+ return true;
+ }),
})),
];
});
- console.log("message", {
- action: "DeleteCharacterClip",
- id: clipId,
- });
}, []);
return {
diff --git a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts
deleted file mode 100644
index 154c630561..0000000000
--- a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts
+++ /dev/null
@@ -1,41 +0,0 @@
-// Clip offsets represent the state of the clip on the timeline as well as what type of clip it is.
-// it is created from a media id.
-export class ClipUI {
- version: number
- type: "transform" | "audio" | "animation" | "lipsync"
- name: string
- media_id: string
- object_uuid: string
- start_offset: number // in frames
- ending_offset: number // in frames
-
- constructor(
- version: number,
- type: "transform" | "audio" | "animation" | "lipsync",
- name: string,
- media_id: string,
- object_uuid: string,
- start_offset: number,
- ending_offset: number,
- ) {
- this.version = version
- this.name = name
- this.type = type
- this.object_uuid = object_uuid
- this.media_id = media_id
- this.start_offset = start_offset
- this.ending_offset = ending_offset
- }
-
- toJSON(): string {
- return JSON.stringify({
- version: this.version,
- name: this.name,
- type: this.type,
- object_uuid: this.object_uuid,
- media_id: this.media_id,
- start_offset: this.start_offset,
- ending_offset: this.ending_offset
- })
- }
-}
diff --git a/app/pages/PageEnigma/datastructures/common.ts b/app/pages/PageEnigma/datastructures/common.ts
index da612df4aa..5d6c4d1889 100644
--- a/app/pages/PageEnigma/datastructures/common.ts
+++ b/app/pages/PageEnigma/datastructures/common.ts
@@ -2,4 +2,10 @@ export type XYZ = {
x :number;
y: number;
z: number;
+}
+
+export type Simple3DVector = {
+ position: XYZ;
+ rotation: XYZ;
+ scalar: XYZ;
}
\ No newline at end of file
diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts
index 0a3c768ac8..b3d4fb592f 100644
--- a/app/pages/PageEnigma/js/editor.ts
+++ b/app/pages/PageEnigma/js/editor.ts
@@ -16,7 +16,7 @@ import { FFmpeg, createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg";
import AudioEngine from "./audio_engine.js";
import TransformEngine from "./transform_engine.js";
import { TimeLine, TimelineDataState } from "./timeline.js";
-import { ClipUI } from "../datastructures/clips/clip_offset.js";
+// import { ClipUI } from "../datastructures/clips/clip_ui.js";
import { LipSyncEngine } from "./lip_sync_engine.js";
import { AnimationEngine } from "./animation_engine.js";
@@ -267,7 +267,6 @@ class Editor {
this._test_demo();
-
this.renderer.domElement.addEventListener("mousedown", this.onMouseDown.bind(this), false);
this.renderer.domElement.addEventListener("mouseup", this.onMouseUp.bind(this), false);
this.renderer.domElement.addEventListener("onContextMenu", this.onContextMenu.bind(this), false);
@@ -321,6 +320,7 @@ class Editor {
const result = await this.api_manager.getMediaBatch(["m_8fmp9hrvsqcryzka1fra597kg42s50","m_z4jzbst3xfh64h0qn4bqh4afenfps9"]);
console.log(result);
}
+
public async loadScene(scene_media_token: string) {
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER,
@@ -383,6 +383,7 @@ class Editor {
`saveScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`,
);
+ // TODO turn scene information into and object ...
const result = await this.api_manager.saveSceneState(
this.activeScene.scene,
name,
@@ -390,7 +391,7 @@ class Editor {
this.current_scene_media_token,
new TimelineDataState(),
);
-
+
if (result.data == null) {
return;
}
@@ -755,7 +756,7 @@ class Editor {
if (this.cameraViewControls && this.camera_person_mode) {
this.cameraViewControls.update(5 * delta_time);
if (this.cam_obj) {
- if (this.timeline.isPlaying == false) {
+ if (this.timeline.is_playing == false) {
this.cam_obj.position.copy(this.camera.position);
this.cam_obj.rotation.copy(this.camera.rotation);
} else {
@@ -810,8 +811,8 @@ class Editor {
let video_og = itteration + 'tmp.mp4';
let wav_name = itteration + 'tmp.wav';
let new_video = (itteration + 1) + 'tmp.mp4';
- let startFrame = clip.start_offset;
- let endFrame = clip.ending_offset;
+ let startFrame = clip.offset;
+ let endFrame = clip.length;
if (endFrame > this.timeline.timeLineLimit) {
@@ -890,7 +891,7 @@ class Editor {
let itteration = 0;
- for (const clip of this.timeline.timelineItems) {
+ for (const clip of this.timeline.timeline_items) {
if (clip.type == "lipsync" || clip.type == "audio") {
await this.convertAudioClip(itteration, ffmpeg, clip);
itteration += 1;
@@ -932,7 +933,7 @@ class Editor {
}
startPlayback() {
- this.timeline.isPlaying = true;
+ this.timeline.is_playing = true;
this.timeline.scrubberPosition = 0;
if (!this.camera_person_mode) {
this.switchCameraView();
@@ -979,7 +980,7 @@ class Editor {
let imgData = this.renderer.domElement.toDataURL();
this.frame_buffer.push(imgData);
this.render_timer += this.clock.getDelta();
- if (this.timeline.isPlaying == false) {
+ if (this.timeline.is_playing == false) {
this.stopPlayback();
this.playback_location = 0;
this.rendering = false;
diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts
index 267d23ba63..6873675c84 100644
--- a/app/pages/PageEnigma/js/timeline.ts
+++ b/app/pages/PageEnigma/js/timeline.ts
@@ -1,93 +1,74 @@
import { AnyJson } from "three/examples/jsm/nodes/core/constants.js";
-import { ClipUI } from "../datastructures/clips/clip_offset";
+import { ClipUI } from "../datastructures/clips/clip_ui";
import Scene from "./scene.js";
import AudioEngine from "./audio_engine";
import TransformEngine from "./transform_engine";
import LipSyncEngine from "./lip_sync_engine";
import AnimationEngine from "./animation_engine";
-import { AudioClip } from "../models/track";
-import { AnimationClip } from "three";
-import { LipSyncClip } from "../datastructures/clips/lipsync_clip";
-import { TransformClip } from "../datastructures/clips/transform_clip";
// Every object uuid / entity has a track.
export class TimelineCurrentState {
- isEditable: boolean
- selectedObjectID: number
+ is_editable: boolean
+ selected_object_ID: number
constructor() {
- this.isEditable = true // can add clips to it
- this.selectedObjectID = 0
+ this.is_editable = true // can add clips to it
+ this.selected_object_ID = 0
}
}
export class TimelineDataState {
- timelineItems: ClipUI[]
- scrubberPosition: number
-
- // Data to serialize with the scene, used to load into each engine.
- audioClips:AudioClip[]
- animationClips:AnimationClip[]
- lipSyncClips:LipSyncClip[]
- transformClips:TransformClip[]
-
- constructor(timelineItems: ClipUI[] = [],
- audioClips: AudioClip[] = [],
- animationClips: AnimationClip[] = [],
- transformClips: TransformClip[] = [],
- lipSyncClips: LipSyncClip[] = [],
- scrubberPosition:number = 0) {
- this.timelineItems = timelineItems
- this.scrubberPosition = scrubberPosition
- this.audioClips = audioClips
- this.animationClips = animationClips
- this.lipSyncClips = lipSyncClips
- this.transformClips = transformClips
+ timeline_items: ClipUI[]
+ scrubber_frame_position: number
+ constructor(timeline_items: ClipUI[] = [],
+ scrubber_frame_position: number = 0) {
+ this.timeline_items = timeline_items
+ this.scrubber_frame_position = scrubber_frame_position
}
}
export class TimeLine {
- timelineItems: ClipUI[]
+ timeline_items: ClipUI[]
- timeLineLimit: number
- scrubberPosition: number
- isPlaying: boolean
+ timeline_limit: number
+ scrubber_frame_position: number
+ is_playing: boolean
// plays audio
- audioEngine: AudioEngine
+ audio_engine: AudioEngine
// key framing
- transformEngine: TransformEngine
+ transform_engine: TransformEngine
// animation engine
- animationEngine: AnimationEngine
+ animation_engine: AnimationEngine
// lip sync engine
- lipSyncEngine: LipSyncEngine
+ lipSync_engine: LipSyncEngine
scene: Scene
// ensure that the elements are loaded first.
- constructor(audioEngine: AudioEngine,
- transformEngine: TransformEngine,
- lipsyncEngine: LipSyncEngine,
- animationEngine: AnimationEngine,
+ constructor(audio_engine: AudioEngine,
+ transform_engine: TransformEngine,
+ lipsync_engine: LipSyncEngine,
+ animation_engine: AnimationEngine,
scene: Scene) {
- this.timelineItems = []
- this.timeLineLimit = 60 * 5 // 5 seconds
+ this.timeline_items = []
+ this.timeline_limit = 60 * 5 // 5 seconds
- this.isPlaying = false
- this.scrubberPosition = 0 // in frames into the tl
+ this.is_playing = false
+ this.scrubber_frame_position = 0 // in frames into the tl
// this will be used to play the audio clips
- this.audioEngine = audioEngine
- this.transformEngine = transformEngine
- this.lipSyncEngine = lipsyncEngine
- this.animationEngine = animationEngine
+ this.audio_engine = audio_engine
+ this.transform_engine = transform_engine
+ this.lipSync_engine = lipsync_engine
+ this.animation_engine = animation_engine
this.scene = scene;
}
public async addPlayableClip(clip: ClipUI): Promise {
- this.timelineItems.push(clip)
+ this.timeline_items.push(clip)
}
// when given a media id item it will create the clip.
@@ -130,28 +111,28 @@ export class TimeLine {
}
// public streaming events into the timeline from
public async setScrubberPosition(offset: number) {
- this.scrubberPosition = offset // in ms
+ this.scrubber_frame_position = offset // in ms
}
// should play from the clip that is closest to the to scrubber
public async play(): Promise {
console.log(`Starting Timeline`)
- this.isPlaying = true
+ this.is_playing = true
}
- private async reset_scene() {
- for (const element of this.timelineItems) {
+ private async resetScene() {
+ for (const element of this.timeline_items) {
if (element.type == "transform") {
let object = this.scene.get_object_by_uuid(element.object_uuid);
- if (object && this.transformEngine.clips[element.object_uuid]) { this.transformEngine.clips[element.object_uuid].reset(object); }
+ if (object && this.transform_engine.clips[element.object_uuid]) { this.transform_engine.clips[element.object_uuid].reset(object); }
}
else if (element.type == "audio") {
- this.audioEngine.loadClip(element.media_id);
+ this.audio_engine.loadClip(element.media_id);
}
else if (element.type == "animation") {
}
else if (element.type == "lipsync") {
- this.lipSyncEngine.clips[element.object_uuid].reset();
+ this.lipSync_engine.clips[element.object_uuid].reset();
}
else {
this.stop()
@@ -162,48 +143,48 @@ export class TimeLine {
// called by the editor update loop on each frame
public async update(deltatime:number) {
- if (this.isPlaying == false) return; // start and stop
+ if (this.is_playing == false) return; // start and stop
- if (this.scrubberPosition <= 0) {
- await this.reset_scene();
+ if (this.scrubber_frame_position <= 0) {
+ await this.resetScene();
}
- this.scrubberPosition += 1;
+ this.scrubber_frame_position += 1;
//2. allow stopping.
//3. smallest unit is a frame and it is set by the scene and is in fps, our videos will be 60fps but we can reprocess them using the pipeline.
- for (const element of this.timelineItems) {
- if (element.start_offset <= this.scrubberPosition && this.scrubberPosition <= element.ending_offset) {
+ for (const element of this.timeline_items) {
+ if (element.offset <= this.scrubber_frame_position && this.scrubber_frame_position <= element.length) {
// run async
// element.play()
// remove the element from the list
let object = this.scene.get_object_by_uuid(element.object_uuid)
if (element.type == "transform") {
- if (object && this.transformEngine.clips[element.object_uuid]) {
- this.transformEngine.clips[element.object_uuid].length = (element.ending_offset - element.start_offset);
- this.transformEngine.clips[element.object_uuid].step(object, element.start_offset, this.scrubberPosition);
+ if (object && this.transform_engine.clips[element.object_uuid]) {
+ this.transform_engine.clips[element.object_uuid].length = (element.length - element.offset);
+ this.transform_engine.clips[element.object_uuid].step(object, element.offset, this.scrubber_frame_position);
}
}
else if (element.type == "audio") {
- if(this.scrubberPosition+1 >= element.ending_offset){
- this.audioEngine.stopClip(element.media_id);
+ if(this.scrubber_frame_position+1 >= element.length){
+ this.audio_engine.stopClip(element.media_id);
}
else {
- this.audioEngine.playClip(element.media_id);
+ this.audio_engine.playClip(element.media_id);
}
}
else if (element.type == "lipsync") {
- if(this.scrubberPosition+1 >= element.ending_offset){
- this.lipSyncEngine.clips[element.object_uuid].stop();
+ if(this.scrubber_frame_position+1 >= element.length){
+ this.lipSync_engine.clips[element.object_uuid].stop();
}
else if (object) {
- await this.lipSyncEngine.clips[element.object_uuid].play(object);
- this.lipSyncEngine.clips[element.object_uuid].step();
+ await this.lipSync_engine.clips[element.object_uuid].play(object);
+ this.lipSync_engine.clips[element.object_uuid].step();
}
}
else if (element.type == "animation") {
if (object) {
- await this.animationEngine.clips[object.uuid].play(object);
- this.animationEngine.clips[object.uuid].step(deltatime);
+ await this.animation_engine.clips[object.uuid].play(object);
+ this.animation_engine.clips[object.uuid].step(deltatime);
}
}
else {
@@ -214,14 +195,14 @@ export class TimeLine {
}
// find the offset of the longest clip and play until that clip is done
- if (this.scrubberPosition >= this.timeLineLimit) { // stops at where clips should // cannot throw clip
+ if (this.scrubber_frame_position >= this.timeline_limit) { // stops at where clips should // cannot throw clip
this.stop()
}
}
}
private async stop(): Promise {
- this.isPlaying = false
+ this.is_playing = false
console.log(`Stopping Timeline`)
}
}
diff --git a/app/pages/PageEnigma/models/track.ts b/app/pages/PageEnigma/models/track.ts
index 32299f513e..dd290db8c6 100644
--- a/app/pages/PageEnigma/models/track.ts
+++ b/app/pages/PageEnigma/models/track.ts
@@ -1,11 +1,50 @@
-export interface BaseClip {
- id: string;
+export interface QueueClip {
+ version: number;
+ type: ClipType;
+ group: ClipGroup;
+ object_uuid?: string;
+ clip_uuid?: string;
+ media_id?: string;
+ name?: string;
+ offset?: number;
+ length?: number;
+ selected?: boolean;
+}
+
+export interface MediaClip {
+ version: number;
+ type: ClipType;
+ media_id: string;
+ name: string;
+ length: number;
+}
+
+export interface Clip {
+ version: number;
+ clip_uuid: string;
+ type: ClipType;
+ group: ClipGroup;
+ media_id: string;
+ object_uuid?: string;
name: string;
offset: number;
length: number;
selected?: boolean;
}
+export enum ClipType {
+ TRANSFORM = "transform",
+ AUDIO = "audio",
+ ANIMATION = "animation",
+}
+
+export enum ClipGroup {
+ CHARACTER = "character",
+ CAMERA = "camera",
+ GLOBAL_AUDIO = "global_audio",
+ OBJECT = "object",
+}
+
export interface BaseKeyFrame {
id: string;
name: string;
@@ -13,35 +52,25 @@ export interface BaseKeyFrame {
selected?: boolean;
}
-export interface AnimationClip extends BaseClip {}
-
-export interface PositionClip extends BaseClip {}
-
-export interface LipSyncClip extends BaseClip {}
-
export interface CharacterGroup {
id: string;
muted: boolean;
- animationClips: AnimationClip[];
- positionClips: PositionClip[];
- lipSyncClips: LipSyncClip[];
+ animationClips: Clip[];
+ positionClips: Clip[];
+ lipSyncClips: Clip[];
}
export interface CameraGroup {
id: string;
- clips: CameraClip[];
+ clips: Clip[];
}
-export interface CameraClip extends BaseClip {}
-
export interface AudioGroup {
id: string;
muted: boolean;
- clips: AudioClip[];
+ clips: Clip[];
}
-export interface AudioClip extends BaseClip {}
-
export interface ObjectGroup {
id: string;
objects: ObjectTrack[];
diff --git a/app/pages/PageEnigma/reducers/appUiReducer/types.tsx b/app/pages/PageEnigma/reducers/appUiReducer/types.tsx
index aaec218cd0..75f85e627d 100644
--- a/app/pages/PageEnigma/reducers/appUiReducer/types.tsx
+++ b/app/pages/PageEnigma/reducers/appUiReducer/types.tsx
@@ -1,4 +1,4 @@
-import { XYZ } from "../../datastructures/common";
+import { Simple3DVector } from "../../datastructures/common";
export enum VIEW_MODES {
EDITOR = 'editor',
SIDE_BY_SIDE = 'side-by-side'
@@ -22,11 +22,7 @@ export type State = {
}
currentSceneObject:{
isShowing: boolean;
- objectVectors: {
- position:XYZ;
- rotation:XYZ;
- scalar: XYZ;
- };
+ objectVectors: Simple3DVector;
}
};
@@ -49,11 +45,7 @@ export type Action =
type: ACTION_TYPES.SHOW_CONTROLPANELS_SCENEOBJECT,
payload:{
currentSceneObject: {
- objectVectors: {
- position:XYZ;
- rotation:XYZ;
- scalar: XYZ;
- };
+ objectVectors: Simple3DVector;
};
}
}
diff --git a/app/pages/PageEnigma/store/dragAndDrop.ts b/app/pages/PageEnigma/store/dragAndDrop.ts
index e38765ce73..7d0c98a940 100644
--- a/app/pages/PageEnigma/store/dragAndDrop.ts
+++ b/app/pages/PageEnigma/store/dragAndDrop.ts
@@ -7,3 +7,11 @@ export const overTimeline = signal(false);
export const dropId = signal("");
export const dropOffset = signal(0);
export const clipLength = signal(0);
+export const initPosition = signal<{ initX: number; initY: number }>({
+ initX: 0,
+ initY: 0,
+});
+export const currPosition = signal<{ currX: number; currY: number }>({
+ currX: 0,
+ currY: 0,
+});
diff --git a/app/root.tsx b/app/root.tsx
index f22de825a3..855484c581 100644
--- a/app/root.tsx
+++ b/app/root.tsx
@@ -107,4 +107,4 @@ function CompleteTakeoverLoadingScreen({isShowing}:{isShowing:boolean}){
);
-};
\ No newline at end of file
+};