Merge branch 'test' into feature/asset-drawer

This commit is contained in:
FromTheLOLAshes committed 2024-04-06 18:38:48 -04:00
commit 9d814aaed6
33 files changed
+856 -391

No files matched your search

+10 -3
View File
@@ -2,17 +2,23 @@ import { twMerge } from "tailwind-merge";
import { IconDefinition } from "@fortawesome/pro-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { Label } from "./Typography";
import { kebabCase } from "~/utilities";
interface InputProps extends React.InputHTMLAttributes<HTMLInputElement> {
label?: string;
className?: string;
icon?: IconDefinition
}
export function Input({label, icon, className, ...rest }: InputProps) {
export function Input({
label, icon, className, id, ...rest
}: InputProps) {
return (
<div className={twMerge("flex flex-col gap-2", className)}>
{label && <Label>{label}</Label>}
{label &&
<Label htmlFor={id ? id : kebabCase(label)}>
{label}
</Label>
}
<div className="relative w-full">
{icon && (
@@ -22,6 +28,7 @@ export function Input({label, icon, className, ...rest }: InputProps) {
/>
)}
<input
id={id ? id : (label ? kebabCase(label) : undefined)}
className={twMerge(
"w-full h-10 rounded-md bg-brand-secondary p-3 text-white transition-all duration-150 ease-in-out outline-none outline-offset-0 focus:outline-brand-primary",
icon? "pl-12" : "",
+9 -1
View File
@@ -1,6 +1,7 @@
import { TextareaHTMLAttributes } from "react";
import { twMerge } from "tailwind-merge";
import { Label } from "./Typography";
import { kebabCase } from "~/utilities";
type ResizeType =
| "none"
@@ -15,6 +16,7 @@ export const Textarea = ({
className,
label,
resize = "vertical",
id,
...rest
}: TextareaHTMLAttributes<HTMLTextAreaElement> & {
label?: string;
@@ -22,8 +24,14 @@ export const Textarea = ({
}) => {
return (
<div className="flex flex-col">
{label && <Label>{label}</Label>}
{label &&
<Label htmlFor={id ? id : kebabCase(label)}>
{label}
</Label>
}
<textarea
id={id ? id : (label ? kebabCase(label) : undefined)}
className={twMerge(
"rounded-lg border border-ui-panel-border bg-ui-controls p-3",
className,
+10 -2
View File
@@ -1,4 +1,5 @@
import { NavLink, NavLinkProps } from "@remix-run/react";
import { LabelHTMLAttributes } from "react";
import { twMerge } from "tailwind-merge";
interface TypoProps {
@@ -23,8 +24,15 @@ export const H4 = ({ className, children }: TypoProps) => (
<h4 className={twMerge(baseTypo, "text-base font-medium", className)}>{children}</h4>
);
export const Label = ({ className, children }: TypoProps) => (
<label className={twMerge(baseTypo, "mb-2 text-base font-medium", className)}>{children}</label>
export const Label = ({
className, children, ...rest
}: LabelHTMLAttributes<HTMLLabelElement>) => (
<label
className={twMerge(baseTypo, "mb-2 text-base font-medium", className)}
{...rest}
>
{children}
</label>
);
export const H5 = ({ className, children }: TypoProps) => (
+73 -42
View File
@@ -1,7 +1,7 @@
import {
createContext,
useEffect,
useCallback,
useEffect,
useState,
ReactNode,
Dispatch,
@@ -12,52 +12,43 @@ import { useCookies } from 'react-cookie';
import {
SessionResponse,
AuthState,
STORAGE_KEYS
} from "./types";
import { CreateSession, DestroySession, GetSession } from "./utilities";
export const AuthenticationContext = createContext<{
authState?: AuthState,
setAuthState?: Dispatch<SetStateAction<AuthState | undefined>>,
persistLoginOrOut?: ()=>void,
authState: AuthState,
setAuthState?: Dispatch<SetStateAction<AuthState>>,
loginAndGetUserInfo?: (
usernameOrEmail: string,
password:string,
failureCallback?:()=>void
)=>void;
logout?: ()=>void;
}>({});
}>({
authState:{}
});
export const AuthenticationProvider = ({children}:{children:ReactNode})=>{
const [authCookies, setAuthCookie, removeAuthCookie] = useCookies(['userInfo']);
const [authState, setAuthState] = useState<AuthState | undefined>(undefined);
// console.log('auth provider rerender');
const [authCookies, setAuthCookie, removeAuthCookie] = useCookies([STORAGE_KEYS.USER_INFO]);
const [authState, setAuthState] = useState<AuthState>({});
const persistLoginOrOut = useCallback(()=>{
GetSession().then((
res: SessionResponse
)=>{
if(res.success && res.user){
setAuthCookie('userInfo', res.user);
setAuthState({
isLoggedIn: true,
userInfo: res.user
});
}else{
removeAuthCookie('userInfo');
}
});
}, []);
const loginAndGetUserInfo = (usernameOrEmail: string, password:string, failureCallback?:()=>void)=>{
const loginAndGetUserInfo = useCallback((
usernameOrEmail: string,
password:string,
failureCallback?:()=>void
)=>{
CreateSession({usernameOrEmail, password})
.then((respond)=>{
GetSession().then((
res: SessionResponse
)=>{
if(res.success && res.user && setAuthState){
setAuthCookie('userInfo', res.user);
setAuthState({
isLoggedIn: true,
sessionData: JSON.stringify(res.user),
userInfo: res.user
});
}else{
@@ -65,29 +56,69 @@ export const AuthenticationProvider = ({children}:{children:ReactNode})=>{
}
});
});
}
const logout = ()=>{
DestroySession().then((res)=>{
if (res && setAuthState) {
setAuthState({
isLoggedIn: false,
userInfo: undefined
});
}
});
}
useEffect(()=>{
if (authCookies !== undefined && authState===undefined){
persistLoginOrOut();
}
},[]);
const setLogoutState = useCallback(()=>{
setAuthState({
isLoggedIn: false,
sessionData: null,
userInfo: undefined
});
},[]);
const logout = useCallback(()=>{
DestroySession().then((res)=>{
if (res) setLogoutState();
});
},[]);
//Set Session upon Auth State Changes
useEffect(()=>{
//On First Load
if(authState.isLoggedIn === undefined){
//Persist Login if Session Data exist
const sessionData = localStorage.getItem(STORAGE_KEYS.USER_INFO);
// console.log(`Session Data: ${sessionData?.substring(0, 30)}${sessionData ? '...':''}`);
if (sessionData !== null){
// console.log('Getting session to persist login');
GetSession().then((
res: SessionResponse
)=>{
// console.log('Session Returned');
// console.log(res);
if(res.success && res.user){
// console.log('Setting Auth State');
setAuthState({
isLoggedIn: true,
sessionData: JSON.stringify(res.user),
userInfo: res.user
});
}else{
// console.log('Session expired, setting State');
setLogoutState();
}
});
}else{
setLogoutState();
}
}
// Propagate session data to localstorage
if(authState.sessionData !== null && authState.sessionData !== undefined){
localStorage.setItem(STORAGE_KEYS.USER_INFO, authState.sessionData);
setAuthCookie(STORAGE_KEYS.USER_INFO, authState.userInfo);
}
// Propagate Logout
else if (authState.sessionData === null){
localStorage.removeItem(STORAGE_KEYS.USER_INFO);
removeAuthCookie(STORAGE_KEYS.USER_INFO);
}
},[authState]);
return(
<AuthenticationContext.Provider value={{
authState,
setAuthState,
persistLoginOrOut,
loginAndGetUserInfo,
logout,
}}>
+5 -1
View File
@@ -1,3 +1,6 @@
export enum STORAGE_KEYS{
USER_INFO = "user_info",
}
export enum AUTH_STATUS {
LOGGED_IN = "logged_in",
LOGGING = "logging",
@@ -5,7 +8,8 @@ export enum AUTH_STATUS {
}
export type AuthState = {
isLoggedIn: boolean;
isLoggedIn?: boolean;
sessionData?: string | null;
userInfo?: UserInfo;
}
export interface UserInfo {
+11 -2
View File
@@ -1,6 +1,7 @@
import React, { useCallback } from "react";
import React, { useCallback, UIEvent } from "react";
import { useMouseEventsTimeline } from "~/pages/PageEnigma/comps/Timeline/utils/useMouseEventsTimeline";
import {
currentScroll,
currentTime,
overTimeline,
scale,
@@ -22,7 +23,10 @@ export const LowerPanel = ({ children }: LowerPanelPropsI) => {
const onTimelineClick = useCallback(
(event: React.PointerEvent<HTMLDivElement>) => {
if (event.button === 0) {
const newTime = Math.round((event.clientX - 92) / 4 / scale.value);
console.log("??", event.clientX, event.pageX, currentScroll.value);
const newTime = Math.round(
(event.clientX + currentScroll.value - 92) / 4 / scale.value,
);
if (newTime < 0) {
return;
}
@@ -37,6 +41,10 @@ export const LowerPanel = ({ children }: LowerPanelPropsI) => {
[],
);
const onScroll = useCallback((event: UIEvent<HTMLDivElement>) => {
currentScroll.value = event.currentTarget.scrollLeft;
}, []);
return (
<>
<div
@@ -56,6 +64,7 @@ export const LowerPanel = ({ children }: LowerPanelPropsI) => {
}}
onPointerLeave={() => (overTimeline.value = false)}
onPointerDown={onTimelineClick}
onScroll={onScroll}
>
{children}
</div>
+26
View File
@@ -0,0 +1,26 @@
import { useContext, useEffect, ComponentType } from "react";
import { useNavigate, useLocation } from "@remix-run/react";
import { AuthenticationContext } from "~/contexts/Authentication";
export const withProtectionRoute = <P extends {}>(Component: ComponentType<P>) => (rest:P) => {
const { authState } = useContext(AuthenticationContext);
// console.log(authState);
if (authState.isLoggedIn === undefined ) {
return <>waiting</>;
}
if (authState.isLoggedIn === false) {
return <RedirectToLogin />
}
return (
<Component {...rest}/>
);
}
const RedirectToLogin = ()=>{
const navigate = useNavigate();
const {pathname} = useLocation();
useEffect(()=>{
navigate(`/login?redirect=${pathname}`);
},[]);
return <>redirecting</>
}
+12 -3
View File
@@ -1,5 +1,4 @@
import { useContext } from "react";
import { faSparkles } from "@fortawesome/pro-solid-svg-icons";
import { Button, LoadingBar, LoadingDots } from "~/components";
@@ -18,9 +17,12 @@ import { APPUI_VIEW_MODES } from "./reducers";
import { timelineHeight } from "~/pages/PageEnigma/store";
import { useSignals } from "@preact/signals-react/runtime";
import { AppUiContext } from "~/pages/PageEnigma/contexts/AppUiContext";
import { EngineContext } from "./contexts/EngineContext";
export const PageEnigmaComponent = () => {
useSignals();
const editorEngine = useContext(EngineContext);
const [appUiState] = useContext(AppUiContext);
//To prevent the click event from propagating to the canvas: TODO: HANDLE THIS BETTER?
@@ -28,13 +30,20 @@ export const PageEnigmaComponent = () => {
event.stopPropagation();
};
const handleGenerateMovieClick=()=> {
if ( editorEngine != null) {
editorEngine?.generateVideo()
} else {
console.log("Tried to generate movie but editor was null")
}
};
const lowerHeight = timelineHeight.value;
return (
<div>
<TopBarHelmet>
<Button icon={faSparkles}>Generate Movie</Button>
<Button icon={faSparkles} onClick={handleGenerateMovieClick}>Generate Movie</Button>
</TopBarHelmet>
<div style={{ height: "calc(100vh - 68px)" }}>
{/* Engine section/side panel */}
<div
@@ -23,18 +23,18 @@ export const ControlPanelSceneObject = () => {
const editorEngine = useContext(EngineContext);
const [appUiState, dispatchAppUiState] = useContext(AppUiContext);
if (!appUiState.controlPanel.currentSceneObject) {
return null;
}
const position =
appUiState.controlPanel.currentSceneObject.objectVectors.position;
appUiState.controlPanel.currentSceneObject?.objectVectors?.position;
const rotation =
appUiState.controlPanel.currentSceneObject.objectVectors.rotation;
const scale = appUiState.controlPanel.currentSceneObject.objectVectors.scale;
appUiState.controlPanel.currentSceneObject?.objectVectors?.rotation;
const scale =
appUiState.controlPanel.currentSceneObject?.objectVectors?.scale;
const currentSceneObject = appUiState.controlPanel.currentSceneObject;
useEffect(() => {
// TODO this causes a subtle bug because it renders way too many times.
const vectors = appUiState.controlPanel.currentSceneObject.objectVectors;
editorEngine?.setSelectedObject(
vectors.position,
vectors.rotation,
@@ -42,6 +42,10 @@ export const ControlPanelSceneObject = () => {
);
}, [appUiState.controlPanel.currentSceneObject]);
if (!appUiState.controlPanel.currentSceneObject) {
return null;
}
const handlePositionChange = (xyz: XYZ) => {
if (!currentSceneObject) {
console.log("Missing Scene Object Position");
@@ -1,111 +1,47 @@
import { useContext, useState } from "react";
import { Button, Input } from "~/components";
import { ButtonDialogue } from "~/modules/ButtonDialogue";
import { AppUiContext } from "../../contexts/AppUiContext";
import { EngineContext } from "../../contexts/EngineContext";
import { APPUI_ACTION_TYPES } from "../../reducers";
import { ClipGroup } from "~/pages/PageEnigma/models";
export const ControlsTopButtons = () => {
const editorEngine = useContext(EngineContext);
const [appUiState, dispatchAppUiState] = useContext(AppUiContext);
const [sceneName, setSceneName] = useState<string>("");
const [sceneToken, setSceneToken] = useState<string>("");
// for testing
const [mediaToken, setMediaToken] = useState<string>("");
const handleButtonSave = () => {
console.log(`SceneName is ${sceneName}`);
editorEngine?.saveScene(sceneName);
};
const handleMediaToken = async () => {
// await editorEngine?.test_loadMediaToken(mediaToken)
};
const handleButtonLoadScene = () => {
console.log(`Scene Token is ${sceneToken}`);
editorEngine?.loadScene(sceneToken);
};
const handleButtonTest = () => {
editorEngine?.testStylizeRequest();
// if(appUiState?.showEditorLoadingBar.isShowing){
//TO GO TO A HUNDRED AND DISAPPER
// dispatchAppUiState({
// type: APPUI_ACTION_TYPES.UPDATE_EDITOR_LOADINGBAR,
// payload:{
// showEditorLoadingBar: {
// progress: 100,
// }
// }
// });
// setTimeout(() => {
// dispatchAppUiState({
// type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADINGBAR,
// });
// }, 1000);
//END :TO GO TO A HUNDRED AND DISAPPER
//EAMPLE OF CHANING LOADING MESSAGE
// dispatchAppUiState({
// type: APPUI_ACTION_TYPES.UPDATE_EDITOR_LOADINGBAR,
// payload:{
// showEditorLoadingBar: {
// message: "new message",
// }
// }
// });
// }else{
// dispatchAppUiState({
// type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADINGBAR,
// payload:{
// showEditorLoadingBar: {
// message: 'display of LoadingBar triggered by Test Button',
// progress: 10,
// }
// }
// });
//}
//editorEngine?.testStylizeRequest();
};
const handleObjectPanelTest = () => {
if (appUiState?.controlPanel.isShowing) {
dispatchAppUiState({
type: APPUI_ACTION_TYPES.HIDE_CONTROLPANELS_SCENEOBJECT,
});
} else {
dispatchAppUiState({
type: APPUI_ACTION_TYPES.SHOW_CONTROLPANELS_SCENEOBJECT,
payload: {
group: ClipGroup.OBJECT,
object_name: "TEST BUTTON",
object_uuid: "",
version: "1",
objectVectors: {
position: { x: 1, y: 2, z: 3 },
rotation: { x: 4, y: 5, z: 6 },
scale: { x: 7, y: 8, z: 9 },
},
},
});
}
const handleTestButton2 = () => {
console.log("Test Button 2");
};
const handleButtonCameraView = () => {
editorEngine?.switchCameraView();
};
const handleButtonPlayBack = () => {
editorEngine?.startPlayback();
};
const handleButtonRender = () => {
editorEngine?.generateVideo();
};
// const handleButtonPlayBack = () => {
// editorEngine?.startPlayback();
// };
const handleButtonTakeFrame = () => {
editorEngine?.take_timeline_cam_clip();
};
const handleButtonLoad = () => {
console.log("LOADING");
};
const handleButtonSingleFrame = () => {
editorEngine?.generateFrame();
};
@@ -153,6 +89,26 @@ export const ControlsTopButtons = () => {
/>
</ButtonDialogue>
<ButtonDialogue
buttonProps={{
variant: "secondary",
label: "Add Scene Object (Test)",
}}
confirmButtonProps={{
label: "Save",
disabled: mediaToken === "",
onClick: handleMediaToken,
}}
title="Add Scene Object via Media Token"
>
<Input
label="Please Enter a Media Token"
onChange={(e) => {
setMediaToken(e.target.value);
}}
/>
</ButtonDialogue>
<ButtonDialogue
buttonProps={{
variant: "secondary",
@@ -172,7 +128,7 @@ export const ControlsTopButtons = () => {
Toggle Camera View
</Button>
<Button
onClick={handleObjectPanelTest}
onClick={handleTestButton2}
className="hover:bg-brand-teriary-400 bg-brand-tertiary focus-visible:outline-brand-tertiary"
>
Test Button 2
@@ -181,9 +137,6 @@ export const ControlsTopButtons = () => {
<div className="flex gap-2">
<Button onClick={handleButtonSingleFrame}>Render Single Frame</Button>
<Button onClick={handleButtonTakeFrame}>Take Frame</Button>
<Button onClick={handleButtonRender}>Render</Button>
{/* <Button onClick={handleButtonLoad}>Load</Button>
<Button onClick={handleButtonRender}>Render</Button> */}
<Button
onClick={handleButtonTest}
className="hover:bg-brand-teriary-400 bg-brand-tertiary focus-visible:outline-brand-tertiary"
@@ -1,6 +1,6 @@
import { useContext, useState } from "react";
import { Button, ItemPicker, Label, Textarea } from "~/components";
import { Button, ItemPicker, Label, H4, Textarea } from "~/components";
import { AppUiContext } from "../../contexts/AppUiContext";
import { APPUI_ACTION_TYPES, APPUI_VIEW_MODES } from "../../reducers";
@@ -62,7 +62,7 @@ export const TabStylization = () => {
return (
<div className="flex h-full w-full flex-col gap-3">
<div className="flex flex-col">
<Label>Select Base Style</Label>
<H4>Select Base Style</H4>
<div className="grid grid-cols-4 gap-2">
<ItemPicker
label="Anime"
@@ -98,6 +98,7 @@ export const TabStylization = () => {
<Textarea
label="Positive Prompt"
className="h-32 w-full"
name="positive-prompt"
placeholder="Type here to describe your scene"
onChange={onChangeHandlerPositive}
resize="none"
@@ -105,6 +106,7 @@ export const TabStylization = () => {
<Textarea
label="Negative Prompt"
className="h-32 w-full"
name="negative-prompt"
placeholder="Type here to filter out the things you don't want in the scene"
onChange={onChangeHandlerNegative}
resize="none"
@@ -38,7 +38,7 @@ export const useMouseEventsClip = (
if (deltaOffset < min) {
currOffset.current = min;
} else if (deltaOffset + currLength.current > max) {
currOffset.current = max;
currOffset.current = max - currLength.current;
} else {
currOffset.current = deltaOffset;
}
@@ -1,5 +1,10 @@
import React, { useCallback, useEffect, useState } from "react";
import { currentTime, filmLength, scale } from "~/pages/PageEnigma/store";
import {
currentScroll,
currentTime,
filmLength,
scale,
} from "~/pages/PageEnigma/store";
import Queue from "~/pages/PageEnigma/Queue/Queue";
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
@@ -21,11 +26,13 @@ export const useMouseEventsScrubber = () => {
Queue.publish({
queueName: QueueNames.TO_ENGINE,
action: toEngineActions.UPDATE_TIME,
data: { currentTime: Math.round(time) },
data: { currentTime: Math.round(time) + currentScroll.value },
});
}
};
console.log("scroll", currentScroll.value);
const onMouseMove = (event: MouseEvent) => {
const delta = Math.round(
(event.clientX - clientX) / 4 / scale.value + currentTime.value,
@@ -41,7 +48,7 @@ export const useMouseEventsScrubber = () => {
Queue.publish({
queueName: QueueNames.TO_ENGINE,
action: toEngineActions.UPDATE_TIME,
data: { currentTime: delta },
data: { currentTime: delta + currentScroll.value },
});
}
return delta;
@@ -53,7 +53,6 @@ export const TrackProvider = ({ children }: Props) => {
addCharacterAnimation({
dragItem: dragItem.value!,
characterId: dropId.value,
animationClips,
offset: dropOffset.value,
});
}
@@ -61,13 +60,11 @@ export const TrackProvider = ({ children }: Props) => {
addCharacterAudio({
dragItem: dragItem.value!,
characterId: dropId.value,
audioClips,
offset: dropOffset.value,
});
addGlobalAudio({
dragItem: dragItem.value!,
audioId: dropId.value,
audioClips,
offset: dropOffset.value,
});
}
@@ -237,7 +234,7 @@ export const TrackProvider = ({ children }: Props) => {
setObjectItems([
{
version: 1,
media_id: uuid.v4(),
media_id: "m_w5nn3kjh1fbkmjrdac5b2qaba0pmyt",
type: AssetType.OBJECT,
name: "Block Stance",
thumbnail: "resources/characters/img01.png",
@@ -101,8 +101,8 @@ export class AnimationClip {
this.mixer?.setTime(0);
}
toJSON(): string {
return JSON.stringify({
toJSON(): any {
return {
version: this.version,
media_id: this.media_id,
object_uuid: this.object_uuid,
@@ -110,6 +110,6 @@ export class AnimationClip {
speed: this.speed,
length: this.length,
clip_name: this.clip_name,
})
};
}
}
@@ -52,12 +52,12 @@ export class AudioClip {
return new AudioData(audioContext, audioBuffer);
}
toJSON(): string {
return JSON.stringify({
toJSON(): any {
return {
version: this.version,
media_id: this.media_id,
type: this.type,
volume: this.volume,
});
};
}
}
@@ -34,8 +34,8 @@ export class ClipUI {
this.length = length;
}
toJSON(): string {
return JSON.stringify({
toJSON(): any {
return {
version: this.version,
group: this.group,
name: this.name,
@@ -44,6 +44,6 @@ export class ClipUI {
media_id: this.media_id,
start_offset: this.offset,
ending_offset: this.length,
});
};
}
}
@@ -129,12 +129,12 @@ export class LipSyncClip {
this.lipsync = new LipSync();
}
toJSON(): string {
return JSON.stringify({
toJSON(): any {
return {
version: this.version,
media_id: this.media_id,
type: this.type,
volume: this.volume,
});
};
}
}
@@ -140,14 +140,14 @@ export class TransformClip {
keyframe_uuid));
}
toJSON(): string {
return JSON.stringify({
toJSON(): any {
return {
version: this.version,
media_id: this.media_id,
object_uuid: this.object_uuid,
length: this.length,
type: this.type,
keyframes: this.keyframes,
});
};
}
}
+2 -2
View File
@@ -11,14 +11,14 @@ export class AnimationEngine {
}
load_object(object_uuid: string, media_id:string, clip_name: string) {
this.clips[object_uuid] = new AnimationClip(
this.clips[object_uuid+media_id] = new AnimationClip(
this.version,
media_id,
"glb",
object_uuid,
1.0, 1.0,
clip_name);
this.clips[object_uuid]._load_animation();
this.clips[object_uuid+media_id]._load_animation();
}
}
+59 -60
View File
@@ -88,43 +88,43 @@ export class APIManager {
* @returns APIManagerResponseMessage
*/
public async saveSceneState(
scene: THREE.Scene,
save_json: string,
scene_name: string,
scene_glb_media_file_token: string | null = null,
scene_media_file_token: string | null = null,
timeline_state: TimelineDataState | null = null,
): Promise<APIManagerResponseSuccess> {
const file = await this.gltfExport(scene);
): Promise<string> {
const file = new File([save_json], `${scene_name}.glb`, {
type: "application/json",
});
// will overwrite the scene on db if token exists
const upload_glb_response = await this.uploadGLB(
const upload_glb_response = await this.uploadEngineAsset(
file,
scene_glb_media_file_token,
);
const result_scene_glb_media_file_token =
upload_glb_response["media_file_token"];
// now write the scene
const save_scene_timeline_response = await this.saveSceneAndTimeLineState(
result_scene_glb_media_file_token,
scene_media_file_token,
scene_name,
timeline_state,
);
//const result_scene_glb_media_file_token =
// upload_glb_response["media_file_token"];
//// now write the scene
//const save_scene_timeline_response = await this.saveSceneAndTimeLineState(
// result_scene_glb_media_file_token,
// scene_media_file_token,
// scene_name,
// timeline_state,
//);
//const result_scene_media_file_token =
// save_scene_timeline_response["media_file_token"];
//const data = {
// scene_glb_media_file_token: result_scene_glb_media_file_token,
// scene_media_file_token: result_scene_media_file_token,
//};
const result_scene_media_file_token =
save_scene_timeline_response["media_file_token"];
const data = {
scene_glb_media_file_token: result_scene_glb_media_file_token,
scene_media_file_token: result_scene_media_file_token,
};
return new APIManagerResponseSuccess("Scene Saved", data);
return upload_glb_response["media_file_token"];
}
public async loadSceneState(
scene_media_file_token: string | null,
): Promise<APIManagerResponseSuccess> {
): Promise<any> {
const api_base_url = "https://api.fakeyou.com";
const url = `${api_base_url}/v1/media_files/file/${scene_media_file_token}`;
const response = await fetch(url);
@@ -148,38 +148,38 @@ export class APIManager {
});
console.log(`loadSceneState: ${JSON.stringify(json_result)}`);
return json_result;
const scene_glb_media_file_token: string =
json_result["scene_glb_media_file_token"];
// const scene_glb_media_file_token: string =
// json_result["scene_glb_media_file_token"];
// const media_bucket_path = await this.getMediaFile(
// scene_glb_media_file_token,
// );
// console.log(`GLB ${media_bucket_path}`);
// const glbLoader = new GLTFLoader();
// // promisify this
// const loadGlb = (
// bucket_path: string,
// ): Promise<APIManagerResponseSuccess> => {
// return new Promise((resolve, reject) => {
// glbLoader.load(bucket_path, (glb) => {
// if (glb) {
// const scene: THREE.Scene = glb.scene;
// const data = {
// scene_glb_media_file_token: scene_glb_media_file_token,
// scene_media_file_token: scene_media_file_token,
// scene: scene,
// };
// console.log(`Data: ${data}`);
// resolve(new APIManagerResponseSuccess("Success Loaded", data));
// } else {
// throw new APIManagerResponseError("Failed to Load GLB Scene");
// }
// });
// });
// };
const media_bucket_path = await this.getMediaFile(
scene_glb_media_file_token,
);
console.log(`GLB ${media_bucket_path}`);
const glbLoader = new GLTFLoader();
// promisify this
const loadGlb = (
bucket_path: string,
): Promise<APIManagerResponseSuccess> => {
return new Promise((resolve) => {
glbLoader.load(bucket_path, (glb) => {
if (glb) {
const scene: THREE.Scene = glb.scene;
const data = {
scene_glb_media_file_token: scene_glb_media_file_token,
scene_media_file_token: scene_media_file_token,
scene: scene,
};
console.log(`Data: ${data}`);
resolve(new APIManagerResponseSuccess("Success Loaded", data));
} else {
throw new APIManagerResponseError("Failed to Load GLB Scene");
}
});
});
};
return await loadGlb(media_bucket_path);
//return await loadGlb(media_bucket_path);
}
/**
@@ -213,18 +213,18 @@ export class APIManager {
return file;
}
private async uploadGLB(
private async uploadEngineAsset(
file: File,
scene_glb_media_file_token: string | null,
): Promise<string> {
media_file_token: string | null,
): Promise<any> {
const url = `${this.baseUrl}/v1/media_files/write/engine_asset`;
const uuid = uuidv4();
const form_data = new FormData();
form_data.append("uuid_idempotency_token", uuid);
// update existing scene otherwise create new glb scene and use it's media_file_id
if (scene_glb_media_file_token != null) {
form_data.append("media_file_token", scene_glb_media_file_token);
if (media_file_token != null) {
form_data.append("media_file_token", media_file_token);
}
form_data.append("file", file);
@@ -245,7 +245,6 @@ export class APIManager {
throw new Error("Failed to Send Data");
} else {
const json_data = await response.json();
console.log(`uploadGLB: ${JSON.stringify(json_data)}`);
return json_data; // or handle the response as appropriate
}
}
@@ -310,7 +309,7 @@ export class APIManager {
blob: any,
fileName: string,
): Promise<APIManagerResponseSuccess> {
const url = `${this.baseUrl}/v1/media_uploads/upload`;
const url = `${this.baseUrl}/v1/media_files/upload`;
const uuid = uuidv4();
const formData = new FormData();
+172 -113
View File
@@ -23,7 +23,10 @@ import { AnimationEngine } from "./animation_engine.js";
import { PointerLockControls } from "three/addons/controls/PointerLockControls.js";
import { APPUI_ACTION_TYPES } from "../reducers";
import { ClipGroup, ClipType } from "~/pages/PageEnigma/models";
import { XYZ } from "~/pages/PageEnigma/datastructures/common";
import { XYZ } from "../datastructures/common";
import { StoryTellerProxyScene } from "../proxy/storyteller_proxy_scene";
import { StoryTellerProxyTimeline } from "../proxy/storyteller_proxy_timeline";
class EditorState {
// {
@@ -91,6 +94,7 @@ class Editor {
cam_obj: THREE.Object3D | undefined;
renderPass: RenderPass | undefined;
generating_preview: boolean;
frames: number;
camera_person_mode: boolean;
current_scene_media_token: string | null;
@@ -107,6 +111,10 @@ class Editor {
last_scrub: number;
// Default params.
// scene proxy for serialization
storyteller_proxy_scene: StoryTellerProxyScene;
constructor() {
console.log(
"If you see this message twice! then it rendered twice, if you see it once it's all good.",
@@ -172,6 +180,7 @@ class Editor {
this.playback_location = 0;
this.max_length = 10;
this.last_scrub = 0;
this.frames = 0;
// Audio Engine Test.
this.render_width = 1280;
@@ -206,6 +215,11 @@ class Editor {
"((masterpiece, best quality, 8K, detailed)), colorful, epic, fantasy, (fox, red fox:1.2), no humans, 1other, ((koi pond)), outdoors, pond, rocks, stones, koi fish, ((watercolor))), lilypad, fish swimming around.";
this.negative_prompt = "";
this.art_style = ArtStyle.Anime2DFlat;
this.storyteller_proxy_scene = new StoryTellerProxyScene(
this.version,
this.activeScene.scene,
);
}
initialize(config: any) {
@@ -337,15 +351,6 @@ class Editor {
);
this.cam_obj = this.activeScene.get_object_by_name("::CAM::");
if (this.cam_obj) {
this.addTransformClipBase(
"Camera Object",
ClipGroup.CAMERA,
this.cam_obj,
0,
150,
);
}
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.UPDATE_EDITOR_LOADINGBAR,
@@ -372,19 +377,10 @@ class Editor {
public async testTestTimelineEvents() {}
public async testStylizeRequest() {
const result = await this.api_manager
.stylizeVideo(
"mu_6wy1570a0c3c0tpkkncf4tsvb5234",
this.art_style,
this.positive_prompt,
this.negative_prompt,
Visibility.Public,
)
.catch((error) => {
console.log(error);
});
console.log(result);
public async testLoadTokenAssets() {
await this.storyteller_proxy_scene.loadFromMediaFileToken(
"m_z4jzbst3xfh64h0qn4bqh4afenfps9",
);
}
public async loadScene(scene_media_token: string) {
@@ -392,49 +388,37 @@ class Editor {
type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER,
});
if (scene_media_token != null) {
this.current_scene_media_token = scene_media_token;
}
this.current_scene_media_token = scene_media_token;
const load_scene_state_response = await this.api_manager.loadSceneState(
const scene_json = await this.api_manager.loadSceneState(
this.current_scene_media_token,
);
console.log(load_scene_state_response);
if (load_scene_state_response.data == null) {
console.log("load_scene_state_response Missing Data");
return;
}
let proxyScene = new StoryTellerProxyScene(this.version, this.activeScene);
await proxyScene.loadFromSceneJson(scene_json["scene"]);
this.cam_obj = this.activeScene.get_object_by_name("::CAM::");
const loaded_scene = load_scene_state_response.data["scene"];
if (load_scene_state_response.data == null) {
return;
}
// Load these so you can rewrite the scene glb using it's token.
this.current_scene_media_token =
load_scene_state_response.data["scene_media_file_token"];
this.current_scene_glb_media_token =
load_scene_state_response.data["scene_glb_media_file_token"];
console.log(
`loadScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`,
let proxyTimeline = new StoryTellerProxyTimeline(
this.version,
this.timeline,
this.transform_engine,
this.animation_engine,
this.audio_engine,
this.lipsync_engine,
);
await proxyTimeline.loadFromJson(scene_json["timeline"]);
this.activeScene.scene.children = loaded_scene.children;
this.activeScene.scene.children.forEach((child: THREE.Object3D) => {
child.parent = this.activeScene.scene;
if (child.type == "DirectionalLight") {
const pos = child.position;
const rot = child.rotation;
const light = this.activeScene._create_base_lighting();
light.position.set(pos.x, pos.y, pos.z);
light.rotation.set(rot.x, rot.y, rot.z);
this.activeScene.scene.remove(child);
}
});
// this.activeScene.scene.children.forEach((child: THREE.Object3D) => {
// child.parent = this.activeScene.scene;
// if (child.type == "DirectionalLight") {
// const pos = child.position;
// const rot = child.rotation;
// const light = this.activeScene._create_base_lighting();
// light.position.set(pos.x, pos.y, pos.z);
// light.rotation.set(rot.x, rot.y, rot.z);
// this.activeScene.scene.remove(child);
// }
// });
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER,
@@ -443,7 +427,8 @@ class Editor {
// TO UPDATE selected objects in the scene might want to add to the scene ...
async setSelectedObject(position: XYZ, rotation: XYZ, scale: XYZ) {
if (this.selected != null) {
if (this.selected != undefined || this.selected != null) {
//console.log(`triggering setSelectedObject`)
this.selected.position.x = position.x;
this.selected.position.y = position.y;
this.selected.position.z = position.z;
@@ -455,41 +440,47 @@ class Editor {
this.selected.scale.x = scale.x;
this.selected.scale.y = scale.y;
this.selected.scale.z = scale.z;
this.selected.scale.x = scale.x;
this.selected.scale.y = scale.y;
this.selected.scale.z = scale.z;
}
}
public async test_loadMediaToken(media_file_token: string) {
this.activeScene.load_glb(media_file_token);
}
public async saveScene(name: string) {
// remove controls when saving scene.
this.removeTransformControls();
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER,
});
console.log(
`saveScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`,
let proxyScene = new StoryTellerProxyScene(this.version, this.activeScene);
let scene_json = await proxyScene.saveToScene();
let proxyTimeline = new StoryTellerProxyTimeline(
this.version,
this.timeline,
this.transform_engine,
this.animation_engine,
this.audio_engine,
this.lipsync_engine,
);
let timeline_json = await proxyTimeline.saveToJson();
let save_data = { scene: scene_json, timeline: timeline_json };
// TODO turn scene information into and object ...
const result = await this.api_manager.saveSceneState(
this.activeScene.scene,
JSON.stringify(save_data),
name,
this.current_scene_glb_media_token,
this.current_scene_media_token,
new TimelineDataState(),
);
if (result.data == null) {
return;
}
const scene_media_token = result.data["scene_media_file_token"];
if (scene_media_token != null) {
this.current_scene_media_token = scene_media_token;
}
const scene_glb_media_token = result.data["scene_glb_media_file_token"];
if (scene_glb_media_token != null) {
this.current_scene_glb_media_token = scene_glb_media_token;
}
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER,
});
@@ -649,10 +640,15 @@ class Editor {
// note the database from the server is the source of truth for all the data.
// Test code here
const object: THREE.Object3D = await this.activeScene.load_glb(
"m_9f3d3z94kk6m25zywyz6an3p43fjtw",
"m_fmxy8wjnep1hdaz7qdg4n7y15d2bsp",
);
object.uuid = "CH1";
// Stick Open Pose Man: m_9f3d3z94kk6m25zywyz6an3p43fjtw
// XBot: m_r7w1tmkx2jg8nznr3hyzj4k6zhfh7d
// YBot: m_9sqg0evpr23587jnr8z3zsvav1x077
// Load timeline creates the the clips from the datastructure and loads them in here.
// load object into the engine for lip syncing
//this.lipsync_engine.load_object(
@@ -814,18 +810,9 @@ class Editor {
}
// Basicly Unity 3D's update loop.
updateLoop(time: number) {
async updateLoop(time: number) {
if (this.cam_obj == undefined) {
this.cam_obj = this.activeScene.get_object_by_name("::CAM::");
if (this.cam_obj) {
this.addTransformClipBase(
"Camera Object",
ClipGroup.CAMERA,
this.cam_obj,
0,
150,
);
}
}
// Updates debug stats.
@@ -852,22 +839,26 @@ class Editor {
this.camera.position.copy(this.cam_obj.position);
this.camera.rotation.copy(this.cam_obj.rotation);
}
this.cam_obj.visible = false;
}
} else if (this.cam_obj) {
this.cam_obj.visible = true;
}
if (this.render_camera && this.cam_obj) {
this.render_camera.position.copy(this.cam_obj.position);
this.render_camera.rotation.copy(this.cam_obj.rotation);
this.cam_obj.scale.copy(new THREE.Vector3(1, 1, 1));
}
this.updateSelectedUI();
if (this.timeline.is_playing) {
this.timeline.update();
await this.timeline.update(this.rendering);
} else if (this.last_scrub == this.timeline.scrubber_frame_position) {
this.updateSelectedUI();
}
this.last_scrub = this.timeline.scrubber_frame_position;
this.renderScene();
}
@@ -965,7 +956,14 @@ class Editor {
}
async stopPlayback(compile_audio: boolean = true) {
console.log(this.frames, this.frame_buffer.length);
this.renderMode();
if (this.generating_preview) {
return;
}
this.generating_preview = true;
const ffmpeg = createFFmpeg({ log: true });
await ffmpeg.load();
for (let index = 0; index < this.frame_buffer.length; index++) {
@@ -978,7 +976,7 @@ class Editor {
}
await ffmpeg.run(
"-framerate",
"" + this.cap_fps,
"" + this.cap_fps / 2,
"-i",
"image%d.png",
"-f",
@@ -992,6 +990,10 @@ class Editor {
"-c:a",
"aac", // Specify audio codec (optional, but recommended for MP4)
"-shortest", // Ensure output duration matches the shortest stream (video or audio)
"-pix_fmt",
"yuv420p",
"-f",
"mp4",
"0tmp.mp4",
);
@@ -1006,21 +1008,40 @@ class Editor {
}
}
const output = await ffmpeg.FS("readFile", itteration + "tmp.mp4");
const output = ffmpeg.FS("readFile", itteration + "tmp.mp4");
ffmpeg.exit();
this.generating_preview = false;
// Create a Blob from the output file for downloading
const blob = new Blob([output.buffer], { type: "video/mp4" });
const url = URL.createObjectURL(blob);
const downloadLink = document.createElement("a");
downloadLink.href = url;
downloadLink.download = "render.mp4";
document.body.appendChild(downloadLink);
downloadLink.click();
// Clean up
URL.revokeObjectURL(url);
document.body.removeChild(downloadLink);
const data = await this.api_manager.uploadMedia(blob, "render.mp4");
// Create a link to download the file
const data: any = await this.api_manager.uploadMedia(blob, "render.mp4");
console.log("data", data);
if (data == null) {
return;
}
let upload_token = data["media_file_token"];
console.log(upload_token);
// Create a link to download the file stylize video using api ..
//{"success":true,"upload_token":"mu_x9kr5cfafn512pjbygdszvbdpktrr"} payload
const result = await this.api_manager
.stylizeVideo(
upload_token,
this.art_style,
this.positive_prompt,
this.negative_prompt,
Visibility.Public,
)
.catch((error) => {
console.log(error);
});
// {"success":true,"inference_job_token":"jinf_j3nbqbd15wqxb0xcks13qh3f3bz"}
console.log(result);
}
async generateFrame() {
@@ -1061,9 +1082,9 @@ class Editor {
const url = await this.api_manager.uploadMediaFrameGeneration(
blob,
"render.mp4",
"anime_ghibli",
"((masterpiece, best quality, 8K, detailed)), colorful, epic, fantasy, (fox, red fox:1.2), no humans, 1other, ((koi pond)), outdoors, pond, rocks, stones, koi fish, ((watercolor))), lilypad, fish swimming around.",
"",
this.art_style,
this.positive_prompt,
this.negative_prompt,
);
console.log(url);
@@ -1090,15 +1111,16 @@ class Editor {
}
}
// This initializes the generation of a video render scene is where the core work happens
generateVideo() {
console.log("Generating video...");
if (this.rendering) {
if (this.rendering || this.generating_preview) {
return;
}
this.rendering = true; // has to go first to debounce
this.startPlayback();
this.frame_buffer = [];
this.render_timer = 0;
this.rendering = true;
this.activeScene.renderMode(this.rendering);
if (this.activeScene.hot_items) {
this.activeScene.hot_items.forEach((element) => {
@@ -1135,7 +1157,7 @@ class Editor {
this.activeScene.scene.add(this.control);
}
// Render the scene to the camera.
// Render the scene to the camera, this is called in the update.
renderScene() {
if (this.composer != null && !this.rendering && this.rawRenderer) {
this.composer.render();
@@ -1148,6 +1170,7 @@ class Editor {
}
if (this.rendering && this.renderer && this.clock) {
this.frames += 1;
this.playback_location++;
const imgData = this.renderer.domElement.toDataURL();
this.frame_buffer.push(imgData);
@@ -1166,13 +1189,13 @@ class Editor {
if (this.selected == undefined) {
return;
}
const pos = this.selected.position;
const rot = this.selected.rotation;
const scale = this.selected.scale;
// TODO this is a bug we need to only show when clicked on and use UPDATE when updating.
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.SHOW_CONTROLPANELS_SCENEOBJECT,
type: APPUI_ACTION_TYPES.UPDATE_CONTROLPANELS_SCENEOBJECT,
payload: {
group:
this.selected.name === "::CAM::"
@@ -1285,6 +1308,8 @@ class Editor {
currentObject = currentObject.parent;
}
this.selected = currentObject;
// Show panel here
if (this.selected.type == "Scene") {
this.selected = intersects[0].object;
}
@@ -1295,6 +1320,40 @@ class Editor {
this.control.attach(this.selected);
this.outlinePass.selectedObjects = [this.selected];
this.transform_interaction = true;
// Contact react land
const pos = this.selected.position;
const rot = this.selected.rotation;
const scale = this.selected.scale;
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.SHOW_CONTROLPANELS_SCENEOBJECT,
payload: {
group:
this.selected.name === "::CAM::"
? ClipGroup.CAMERA
: ClipGroup.OBJECT, // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips
object_uuid: this.selected.uuid,
object_name: this.selected.name,
version: this.version,
objectVectors: {
position: {
x: parseFloat(pos.x.toFixed(2)),
y: parseFloat(pos.y.toFixed(2)),
z: parseFloat(pos.z.toFixed(2)),
},
rotation: {
x: parseFloat(rot.x.toFixed(2)),
y: parseFloat(rot.y.toFixed(2)),
z: parseFloat(rot.z.toFixed(2)),
},
scale: {
x: parseFloat(scale.x.toFixed(2)),
y: parseFloat(scale.y.toFixed(2)),
z: parseFloat(scale.z.toFixed(2)),
},
},
},
});
}
} else if (this.transform_interaction == false) {
this.removeTransformControls();
+3
View File
@@ -46,6 +46,7 @@ class Scene {
let obj = new THREE.Mesh(geometry, material);
obj.receiveShadow = true;
obj.castShadow = true;
obj.userData["media_id"] = "Parim";
//obj.type = "Object3D";
obj.name = name;
this.scene.add(obj);
@@ -128,6 +129,7 @@ class Scene {
resolve(child.children[0]);
return;
}
child.userData["media_id"] = media_id;
if (auto_add) { this.scene.add(child); }
resolve(child);
});
@@ -201,6 +203,7 @@ class Scene {
directional_light.shadow.map = null;
directional_light.castShadow = true;
directional_light.shadow.bias = 0.00004;
directional_light.userData["media_id"] = "DirectionalLight";
this.scene.add(directional_light);
this.scene.add(directional_light.target);
+42 -32
View File
@@ -11,8 +11,7 @@ import Queue from "~/pages/PageEnigma/Queue/Queue";
import { QueueNames } from "../Queue/QueueNames";
import { toEngineActions } from "../Queue/toEngineActions";
import { fromEngineActions } from "../Queue/fromEngineActions";
import { ClipGroup, ClipType } from "~/pages/PageEnigma/models";
import { ClipGroup, ClipType } from "~/pages/PageEnigma/models/track";
// Every object uuid / entity has a track.
export class TimelineCurrentState {
is_editable: boolean;
@@ -65,7 +64,7 @@ export class TimeLine {
) {
this.timeline_items = [];
this.absolute_end = 60 * 12;
this.timeline_limit = this.absolute_end; // 5 seconds
this.timeline_limit = 0; // 5 seconds
this.is_playing = false;
this.scrubber_frame_position = 0; // in frames into the tl
@@ -143,15 +142,15 @@ export class TimeLine {
// rotation: XYZ;
// scale: XYZ;
// selected?: boolean;
const data_json = data["data"];
const uuid = data_json["object_uuid"];
let data_json = data["data"];
let uuid = data_json["object_uuid"];
let object_name = this.scene.get_object_by_uuid(uuid)?.name;
if (object_name == undefined) {
object_name = "undefined";
}
const new_item = this.transform_engine.addFrame(
let new_item = this.transform_engine.addFrame(
uuid,
this.absolute_end,
data_json["position"],
@@ -200,7 +199,7 @@ export class TimeLine {
this.addPlayableClip(
new ClipUI(
version,
ClipType.AUDIO,
"lipsync", // TODO potiential bug ..
group,
name,
media_id,
@@ -211,7 +210,6 @@ export class TimeLine {
);
return;
} else {
console.log("Audio!");
this.audio_engine.loadClip(media_id);
}
break;
@@ -233,15 +231,15 @@ export class TimeLine {
}
public async deleteKeyFrame(data: any) {
const keyframe_uuid = data["data"]["keyframe_uuid"];
const object_uuid = data["data"]["object_uuid"];
let keyframe_uuid = data["data"]["keyframe_uuid"];
let object_uuid = data["data"]["object_uuid"];
this.transform_engine.clips[object_uuid].removeKeyframe(keyframe_uuid);
}
public async updateKeyFrame(data: any) {
const keyframe_uuid = data["data"]["keyframe_uuid"];
const keyframe_offset = data["data"]["offset"];
const object_uuid = data["data"]["object_uuid"];
let keyframe_uuid = data["data"]["keyframe_uuid"];
let keyframe_offset = data["data"]["offset"];
let object_uuid = data["data"]["object_uuid"];
this.transform_engine.clips[object_uuid].setOffset(
keyframe_uuid,
keyframe_offset,
@@ -250,9 +248,7 @@ export class TimeLine {
public async updateClip(data: any) {
// only length and offset changes here.
console.log(data);
const object_uuid = data["data"]["object_uuid"];
let object_uuid = data["data"]["object_uuid"];
const media_id = data["data"]["media_id"];
const offset = data["data"]["offset"];
const length = data["data"]["length"] + offset;
@@ -264,8 +260,20 @@ export class TimeLine {
}
}
}
public async deleteClip(data: any) {
console.log(data);
let json_data = data["data"];
let object_uuid = data["data"]["object_uuid"];
let media_id = data["data"]["media_id"];
let type = data["type"];
for (let i = 0; i < this.timeline_items.length; i++) {
const element = this.timeline_items[i];
if (element.media_id == media_id && element.object_uuid == object_uuid) {
this.timeline_items.splice(i, 1);
break;
}
}
}
public async scrubberUpdate(data: any) {
@@ -319,7 +327,6 @@ export class TimeLine {
}
private async resetScene() {
this.timeline_limit = this.getEndPoint();
for (const element of this.timeline_items) {
if (element.type == "transform") {
const object = this.scene.get_object_by_uuid(element.object_uuid);
@@ -329,7 +336,9 @@ export class TimeLine {
} else if (element.type == "audio") {
this.audio_engine.loadClip(element.media_id);
} else if (element.type == "animation") {
this.animation_engine.clips[element.object_uuid].stop();
this.animation_engine.clips[
element.object_uuid + element.media_id
].stop();
} else if (element.type == "lipsync") {
this.lipSync_engine.clips[element.object_uuid].reset();
} else {
@@ -350,9 +359,9 @@ export class TimeLine {
}
// called by the editor update loop on each frame
public async update() {
public async update(isRendering = false) {
//if (this.is_playing == false) return; // start and stop
this.timeline_limit = this.getEndPoint();
if (this.is_playing) {
this.current_time += 1;
this.pushEvent(fromEngineActions.UPDATE_TIME, {
@@ -387,19 +396,14 @@ export class TimeLine {
element.length =
this.transform_engine.clips[element.object_uuid].length;
}
} else if (
element.group !== ClipGroup.CHARACTER &&
element.type === ClipType.AUDIO
) {
} else if (element.type == "audio") {
if (this.scrubber_frame_position + 1 >= element.length) {
this.audio_engine.stopClip(element.media_id);
} else {
this.audio_engine.playClip(element.media_id);
}
} else if (
element.group === ClipGroup.CHARACTER &&
element.type === ClipType.AUDIO
) {
} else if (element.type == "lipsync") {
// we will remove this when we know which group it will come from character + audio == lip sync audio.
if (this.scrubber_frame_position + 1 >= element.length) {
this.lipSync_engine.clips[element.object_uuid].stop();
} else if (object) {
@@ -408,9 +412,15 @@ export class TimeLine {
}
} else if (element.type === ClipType.ANIMATION) {
if (object) {
await this.animation_engine.clips[object.uuid].play(object);
this.animation_engine.clips[object.uuid].step(
this.scrubber_frame_position / 120, // Double FPS for best result.
await this.animation_engine.clips[
object.uuid + element.media_id
].play(object);
let fps = 120;
if (isRendering) {
fps = 60;
}
this.animation_engine.clips[object.uuid + element.media_id].step(
this.scrubber_frame_position / fps, // Double FPS for best result.
);
}
} else {
+1
View File
@@ -17,6 +17,7 @@ export enum ClipType {
TRANSFORM = "transform",
AUDIO = "audio",
ANIMATION = "animation",
LIPSYNC = "lipsync"
}
export enum ClipGroup {
@@ -0,0 +1,89 @@
import * as THREE from "three";
export interface ObjectJSON {
version: number;
position: {
x: number;
y: number;
z: number;
};
rotation: {
x: number;
y: number;
z: number;
};
scale: {
x: number;
y: number;
z: number;
};
object_name: string;
object_uuid: string;
object_user_data_name: string;
media_file_token: string;
}
export class StoryTellerProxy3DObject {
version: number;
position: THREE.Vector3;
rotation: THREE.Euler;
scale: THREE.Vector3;
object_uuid: string;
object_name: string;
object_user_data_name: string; // changable name
media_file_token: string;
constructor(
version: number,
media_file_token: string,
) {
this.version = version;
this.media_file_token = media_file_token;
this.position = new THREE.Vector3(0.0, 0.0, 0.0);
this.rotation = new THREE.Euler(0.0, 0.0, 0.0);
this.scale = new THREE.Vector3(1.0, 1.0, 1.0);
this.object_name = "";
this.object_user_data_name = "";
this.object_uuid = "";
}
public async initialize(object: THREE.Object3D) {
this.position = object.position;
this.rotation = object.rotation;
this.scale = object.scale;
this.object_name = object.name;
this.object_user_data_name = object.userData.name;
this.object_uuid = object.uuid;
}
public async toJSON(): Promise<ObjectJSON> {
const json:ObjectJSON = {
version: this.version,
position: {
x: this.position.x,
y: this.position.y,
z: this.position.z,
},
rotation: {
x: this.rotation.x,
y: this.rotation.y,
z: this.rotation.z,
},
scale: {
x: this.scale.x,
y: this.scale.y,
z: this.scale.z,
},
object_name: this.object_name,
object_uuid: this.object_uuid,
object_user_data_name: this.object_user_data_name,
media_file_token: this.media_file_token,
};
return json;
}
}
@@ -0,0 +1,87 @@
import * as THREE from "three";
import { GLTFLoader, GLTF } from "three/addons/loaders/GLTFLoader.js";
import { StoryTellerProxy3DObject, ObjectJSON } from "./storyteller_proxy_3d_object"
import Scene from "../js/scene";
interface LookUpDictionary {
[key: string]: StoryTellerProxy3DObject;
}
export class StoryTellerProxyScene {
sceneItemProxy: StoryTellerProxy3DObject[];
scene: Scene;
glbLoader: GLTFLoader;
lookUpDictionary: LookUpDictionary;
version: number;
constructor(version: number, scene: Scene) {
this.version = version
this.scene = scene;
this.glbLoader = new GLTFLoader();
this.lookUpDictionary = {}
this.sceneItemProxy = []
}
public async saveToScene(): Promise<any> {
let results: ObjectJSON[] = [];
if (this.scene.scene != null) {
for (const child of this.scene.scene.children) {
if (child.userData['media_id'] != undefined) {
if (this.lookUpDictionary[child.uuid] == null) {
this.lookUpDictionary[child.uuid] = new StoryTellerProxy3DObject(this.version, child.userData['media_id']);
}
let proxyObject3D: StoryTellerProxy3DObject = this.lookUpDictionary[child.uuid];
proxyObject3D.position.copy(child.position);
proxyObject3D.rotation.copy(child.rotation);
proxyObject3D.scale.copy(child.scale);
proxyObject3D.object_user_data_name = child.userData.name;
proxyObject3D.object_name = child.name;
proxyObject3D.object_uuid = child.uuid;
const json_data = await proxyObject3D.toJSON();
results.push(json_data);
}
}
} else {
console.log("Scene doesn't exist needs to be assigned");
}
return results;
}
public async loadFromSceneJson(scene_json: ObjectJSON[]) {
if (scene_json != null && this.scene != null) {
while (this.scene.scene.children.length > 0) {
this.scene.scene.remove(this.scene.scene.children[0]);
}
for (const json_object of scene_json) {
let token: string = json_object.media_file_token;
let obj;
switch (token) {
case 'Parim':
let prim_uuid = this.scene.instantiate(json_object.object_name);
obj = this.scene.get_object_by_uuid(prim_uuid);
break;
case 'DirectionalLight':
obj = this.scene._create_base_lighting();
break;
default:
if(token.includes('m_')){
obj = await this.scene.load_glb(token);
}
break;
}
if (obj) {
obj.position.copy(json_object.position);
obj.rotation.copy(new THREE.Euler(json_object.rotation.x, json_object.rotation.y, json_object.rotation.z));
obj.scale.copy(json_object.scale);
obj.name = json_object.object_name;
obj.userData.name = json_object.object_user_data_name;
obj.uuid = json_object.object_uuid;
obj.userData['media_id'] = json_object.media_file_token;
}
}
this.scene._createGrid();
}
}
};
@@ -0,0 +1,127 @@
import { AnimationClip } from "../datastructures/clips/animation_clip";
import { AudioClip } from "../datastructures/clips/audio_clip";
import { ClipUI } from "../datastructures/clips/clip_ui";
import { LipSyncClip } from "../datastructures/clips/lipsync_clip";
import { TransformClip } from "../datastructures/clips/transform_clip";
import AnimationEngine from "../js/animation_engine";
import AudioEngine from "../js/audio_engine";
import LipSyncEngine from "../js/lip_sync_engine";
import { TimeLine } from "../js/timeline";
import TransformEngine from "../js/transform_engine";
export class StoryTellerProxyTimeline {
timeline: TimeLine
transform_engine: TransformEngine
animation_engine: AnimationEngine
audio_engine: AudioEngine
lipsync_engine: LipSyncEngine
constructor(version: number, timeline: TimeLine, transform_engine: TransformEngine, animation_engine: AnimationEngine, audio_engine: AudioEngine, lipsync_engine: LipSyncEngine) {
this.timeline = timeline;
this.transform_engine = transform_engine;
this.animation_engine = animation_engine;
this.audio_engine = audio_engine;
this.lipsync_engine = lipsync_engine;
}
private async getItems(items: any[]): Promise<any[]> {
let timeline_items_data = [];
for (let index = 0; index < items.length; index++) {
const element = items[index];
timeline_items_data.push(element.toJSON());
};
return timeline_items_data;
}
private async getItemsDict(items: { [key: string]: any }): Promise<{ [key: string]: any }> {
let timeline_items_data: { [key: string]: any } = {};
for (const key in items) {
if (items.hasOwnProperty(key)) {
const element = items[key];
timeline_items_data[key] = element.toJSON();
}
}
return timeline_items_data;
}
public async saveToJson(): Promise<any> {
let timeline_json = {
timeline: await this.getItems(this.timeline.timeline_items),
transform: await this.getItemsDict(this.transform_engine.clips),
animation: await this.getItemsDict(this.animation_engine.clips),
audio: await this.getItemsDict(this.audio_engine.clips),
lipsync: await this.getItemsDict(this.lipsync_engine.clips),
}
return timeline_json;
}
private async loadTimelineClips(timeline_clips: any[]): Promise<any[]> {
let new_clips = [];
if (timeline_clips) {
for (let index = 0; index < timeline_clips.length; index++) {
const element = timeline_clips[index];
let clip = new ClipUI(
element.version, element.type,
element.group, element.name,
element.media_id, element.object_uuid,
element.start_offset, element.ending_offset);
new_clips.push(clip);
}
}
return new_clips;
}
private async loadTransformClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> {
let timeline_items_data: { [key: string]: any } = {};
for (const key in items) {
if (items.hasOwnProperty(key)) {
const element = items[key];
timeline_items_data[key] = new TransformClip(element.version, element.object_uuid, element.length, element.media_id);
timeline_items_data[key].keyframes = element.keyframes;
}
}
return timeline_items_data;
}
private async loadAudioClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> {
let timeline_items_data: { [key: string]: any } = {};
for (const key in items) {
if (items.hasOwnProperty(key)) {
const element = items[key];
timeline_items_data[key] = new AudioClip(element.version, element.media_id, element.volume);
}
}
return timeline_items_data;
}
private async loadAnimationClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> {
let timeline_items_data: { [key: string]: any } = {};
for (const key in items) {
if (items.hasOwnProperty(key)) {
const element = items[key];
timeline_items_data[key] = new AnimationClip(element.version, element.media_id, "remote", element.object_uuid, element.speed, element.length, element.clip_name);
}
}
return timeline_items_data;
}
private async loadLipsyncClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> {
let timeline_items_data: { [key: string]: any } = {};
for (const key in items) {
if (items.hasOwnProperty(key)) {
const element = items[key];
timeline_items_data[key] = new LipSyncClip(element.version, element.media_id, element.volume);
}
}
return timeline_items_data;
}
public async loadFromJson(timeline: any) {
this.timeline.timeline_items = await this.loadTimelineClips(timeline['timeline']);
this.transform_engine.clips = await this.loadTransformClips(timeline['transform']);
this.audio_engine.clips = await this.loadAudioClips(timeline['audio']);
this.lipsync_engine.clips = await this.loadLipsyncClips(timeline['lipsync']);
this.animation_engine.clips = await this.loadAnimationClips(timeline['animation']);
}
}
+2
View File
@@ -1,3 +1,5 @@
import { signal } from "@preact/signals-core";
export const currentTime = signal(0);
export const currentScroll = signal(0);
+27
View File
@@ -0,0 +1,27 @@
import { json } from "@remix-run/router";
import { withProtectionRoute } from "~/modules/withProtectedRoute";
import { faChevronLeft } from "@fortawesome/pro-solid-svg-icons";
import { P, H1, ButtonLink } from "~/components";
export const loader = () => {
return json(null, { status: 404 });
};
const Page404 = withProtectionRoute(()=>{
return(
<div
className="fixed w-full overflow-scroll"
style={{height: "calc(100% - 72px)"}}
>
<div
className='bg-ui-panel w-10/12 max-w-7xl h-1/2 mx-auto my-24 rounded-lg p-6 flex flex-col items-center justify-center gap-8 text-center'
>
<P>Oops, seems like this is not a page that exists!</P>
<H1>Server Error: 404</H1>
<ButtonLink to="/" icon={faChevronLeft}>Back to Dashboard</ButtonLink>
</div>
</div>
)
});
export default Page404;
+4 -4
View File
@@ -1,5 +1,5 @@
import { PageEnigma } from "../pages/PageEnigma";
import { withProtectionRoute } from "~/modules/withProtectedRoute";
import { PageEnigma } from "~/pages/PageEnigma";
export default function IdealEnigma() {
return <PageEnigma />;
}
const IdealEnigma = withProtectionRoute(()=><PageEnigma />);
export default IdealEnigma;
+6 -14
View File
@@ -1,29 +1,19 @@
import { useContext, useEffect, useState, useRef } from "react";
import { useNavigate } from "@remix-run/react";
import { useCookies } from 'react-cookie';
import { useNavigate, useSearchParams } from "@remix-run/react";
import { faKey, faUser } from "@fortawesome/pro-solid-svg-icons";
import { AuthenticationContext } from "~/contexts/Authentication";
import {
Button,
H1,
Input,
Link,
P,
} from '~/components';
import { Button,H1,Input,Link,P } from '~/components';
import { LoadingDots } from "~/components";
export default function LoginScreen() {
const navigate = useNavigate();
const [searchParams] = useSearchParams();
const [showLoader, setShowLoader] = useState<string|undefined>(undefined);
const formRef = useRef<HTMLFormElement | null>(null);
const [authCookies, setAuthCookie, removeAuthCookie] = useCookies(['userInfo']);
const {authState, loginAndGetUserInfo} = useContext(AuthenticationContext);
const handleOnSumbit = (ev: React.FormEvent<HTMLFormElement>)=>{
ev.preventDefault();
if(formRef.current){
@@ -43,8 +33,9 @@ export default function LoginScreen() {
}// end handleOnSubmit
useEffect(()=>{
const redirectPath = searchParams.get('redirect');
if(authState && authState.isLoggedIn)
navigate("/");
navigate(redirectPath ? redirectPath:'/');
},[authState]);
return (
@@ -64,6 +55,7 @@ export default function LoginScreen() {
icon={faUser}
name="usernameOrEmail"
placeholder="Username or Email"
autoComplete="username"
required
/>
<br />
+4
View File
@@ -0,0 +1,4 @@
export const kebabCase = (str:string) => str
.replace(/([a-z])([A-Z])/g, "$1-$2")
.replace(/[\s_]+/g, '-')
.toLowerCase();