diff --git a/app/components/H4.tsx b/app/components/H4.tsx deleted file mode 100644 index bfbf79d630..0000000000 --- a/app/components/H4.tsx +++ /dev/null @@ -1,19 +0,0 @@ -import { twMerge } from "tailwind-merge" - -export const H4 = ({ - className, - children, -}:{ - className?: string, - children: React.ReactNode, -})=>{ - return( -

- {children} -

- ) -} \ No newline at end of file diff --git a/app/components/Typography.tsx b/app/components/Typography.tsx new file mode 100644 index 0000000000..f2c0bda536 --- /dev/null +++ b/app/components/Typography.tsx @@ -0,0 +1,31 @@ +import { twMerge } from "tailwind-merge"; + +interface TypogProps { + className?: string; + children: React.ReactNode; +} +const baseTypo = "text-white"; + +export const H1 = ({className,children,}:TypogProps)=> +

{children}

; + +export const H2 = ({className,children,}:TypogProps)=> +

{children}

; + +export const H3 = ({className,children,}:TypogProps)=> +

{children}

; + +export const H4 = ({className,children,}:TypogProps)=> +

{children}

; + + export const Label = ({className,children,}:TypogProps)=> + ; + +export const H5 = ({className,children,}:TypogProps)=> +
{children}
; + +export const H6 = ({className,children,}:TypogProps)=> +
{children}
; + +export const P = ({className,children,}:TypogProps)=> +

{children}

; \ No newline at end of file diff --git a/app/components/index.tsx b/app/components/index.tsx index 28e6e90ced..d3c627d58c 100644 --- a/app/components/index.tsx +++ b/app/components/index.tsx @@ -4,7 +4,7 @@ import { ButtonLink } from "./ButtonLink"; import { ButtonIconSelect } from "./ButtonIconSelect"; import { Input } from "./Input"; import { InputVector } from "./InputVector"; -import { H4 } from "./H4"; +import { H1, H2, H3, H4, H5, H6, P, Label} from "./Typography"; import { ItemPicker } from "./ItemPicker"; import { Textarea } from "./Textarea"; import { LoadingBar } from "./LoadingBar"; @@ -15,7 +15,7 @@ export { ButtonIcon, ButtonLink, ButtonIconSelect, - H4, + H1,H2,H3,H4,H5,H6,P,Label, ItemPicker, Textarea, Input, diff --git a/app/modules/SidePanel/SidePanel.tsx b/app/modules/SidePanel/SidePanel.tsx index a806569d8b..1181302cb7 100644 --- a/app/modules/SidePanel/SidePanel.tsx +++ b/app/modules/SidePanel/SidePanel.tsx @@ -27,7 +27,6 @@ export const SidePanel = ({ children }: Props) => {
{children}
- ); }; diff --git a/app/pages/PageEnigma/PageEnigmaComponent.tsx b/app/pages/PageEnigma/PageEnigmaComponent.tsx index e0cb3a0462..22ab49389a 100644 --- a/app/pages/PageEnigma/PageEnigmaComponent.tsx +++ b/app/pages/PageEnigma/PageEnigmaComponent.tsx @@ -104,4 +104,4 @@ export const PageEnigmaComponent = () => { ); -}; +}; \ No newline at end of file diff --git a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx index 8432174b6c..47cd64e8d8 100644 --- a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx +++ b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx @@ -70,15 +70,15 @@ export const ControlsTopButtons = () => { editorEngine?.switchCameraView(); }; const handleButtonPlayBack = () => { - editorEngine?.start_playback(); + editorEngine?.startPlayback(); }; const handleButtonRender = () => { - editorEngine?.take_timeline_cam_clip(); - }; - const handleButtonPlay = () => { - editorEngine?.start_playback(); + editorEngine?.generateVideo(); }; + const handleButtonTakeFrame = () => { + editorEngine?.take_timeline_cam_clip(); + }; const handleButtonLoad = () => {}; @@ -148,9 +148,11 @@ export const ControlsTopButtons = () => {
- - - + + + + {/* + */} + + ); +} \ No newline at end of file diff --git a/app/pages/PageEnigma/comps/PreviewEngineCamera/index.tsx b/app/pages/PageEnigma/comps/PreviewEngineCamera/index.tsx index 8630ed50d9..fb8e695aee 100644 --- a/app/pages/PageEnigma/comps/PreviewEngineCamera/index.tsx +++ b/app/pages/PageEnigma/comps/PreviewEngineCamera/index.tsx @@ -8,7 +8,7 @@ export const PreviewEngineCamera = ()=>{ const [showLoader, setShowLoader] = useState(true); useEffect(()=>{ - setTimeout(()=>setShowLoader(false), 5000); + setTimeout(()=>setShowLoader(false), 500); },[]); return (
{ + const editorEngine = useContext(EngineContext); + const { characters, objects, diff --git a/app/pages/PageEnigma/contexts/EngineProvider.tsx b/app/pages/PageEnigma/contexts/EngineProvider.tsx index 899de9ff72..fdaa9982ef 100644 --- a/app/pages/PageEnigma/contexts/EngineProvider.tsx +++ b/app/pages/PageEnigma/contexts/EngineProvider.tsx @@ -4,7 +4,7 @@ import Editor from "~/pages/PageEnigma/js/editor"; import { EngineContext } from "./EngineContext"; import { AppUiContext } from "./AppUiContext"; - +import { TrackContext } from "./TrackContext/TrackContext"; interface Props { children: ReactNode; @@ -13,6 +13,8 @@ interface Props { export const EngineProvider = ({ children }: Props) => { const [editor, setEditor] = useState(null); const [appUiState, dispatchAppUiState] = useContext(AppUiContext); + // To talk to react land. + const trackContext = useContext(TrackContext) useEffect(() => { //componentDidMount diff --git a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx index 2559041981..413b947e99 100644 --- a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx +++ b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx @@ -76,8 +76,26 @@ export const TrackProvider = ({ children }: Props) => { { id: "ani-id1", offset: 0, - length: 124, - name: "ani 11", + length: 25, + name: "Sit", + }, + { + id: "ani-id2", + offset: 0, + length: 25, + name: "Idle", + }, + { + id: "ani-id3", + offset: 0, + length: 25, + name: "Stand", + }, + { + id: "ani-id4", + offset: 0, + length: 25, + name: "Walk", }, ]); setAudioClips([]); diff --git a/app/pages/PageEnigma/datastructures/clips/animation_clip.ts b/app/pages/PageEnigma/datastructures/clips/animation_clip.ts index 50110653d4..30d9bf91e1 100644 --- a/app/pages/PageEnigma/datastructures/clips/animation_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/animation_clip.ts @@ -74,19 +74,16 @@ export class AnimationClip { this.clip_action.play(); } } - console.log("Play") } } step(deltatime: number) { if (this.mixer == null) { return; } this.mixer?.update(deltatime); - console.log("Update") } stop() { this.mixer?.stopAllAction(); - console.log("stop.") } toJSON(): string { diff --git a/app/pages/PageEnigma/datastructures/clips/transform_clip.ts b/app/pages/PageEnigma/datastructures/clips/transform_clip.ts index f167843319..66659207d7 100644 --- a/app/pages/PageEnigma/datastructures/clips/transform_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/transform_clip.ts @@ -29,10 +29,11 @@ export class TransformClip { this.looping = false; } - step(object: THREE.Object3D) { - //if (this.step_frame >= 60/this.length && this.looping == false) { return; } // Reached max frames. + step(object: THREE.Object3D, offset: number, frame: number) { + this.step_frame = frame-offset; + if (this.step_frame < 0) { this.step_frame = 0; } + if (this.step_frame >= this.length) { return; } // Reached max frames. if(this.positions.length < 2) { return; } // If there are enough points in the scene. - this.step_frame += 1; let time_frame = (this.step_frame/this.length); diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index ab3fa0a2ff..98f83e62ea 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -8,6 +8,21 @@ import { TimelineDataState } from "./timeline"; * Storyteller Studio API Manager * The source of truth of all these media items is the database in the cloud */ +class MediaFile { + public_bucket_path: string; + media_type: string; + media_token: string; + + constructor( + public_bucket_path: string, + media_type: string, + media_token: string, + ) { + this.media_token = media_token; + this.public_bucket_path = public_bucket_path; + this.media_type = media_type; + } +} /** * This is designed to surface user customer facing messages as errors. @@ -61,7 +76,8 @@ class APIManager { file, scene_glb_media_file_token, ); - const result_scene_glb_media_file_token = upload_glb_response["media_file_token"]; + const result_scene_glb_media_file_token = + upload_glb_response["media_file_token"]; // now write the scene const save_scene_timeline_response = await this.saveSceneAndTimeLineState( @@ -74,12 +90,16 @@ class APIManager { const result_scene_media_file_token = save_scene_timeline_response["media_file_token"]; - const data = { "scene_glb_media_file_token":result_scene_glb_media_file_token, - "scene_media_file_token": result_scene_media_file_token }; + const data = { + scene_glb_media_file_token: result_scene_glb_media_file_token, + scene_media_file_token: result_scene_media_file_token, + }; return new APIManagerResponseSuccess("Scene Saved", data); } - public async loadSceneState(scene_media_file_token: string | null): Promise { + public async loadSceneState( + scene_media_file_token: string | null, + ): Promise { const api_base_url = "https://api.fakeyou.com"; const url = `${api_base_url}/v1/media_files/file/${scene_media_file_token}`; const response = await fetch(url); @@ -104,19 +124,28 @@ class APIManager { console.log(`loadSceneState: ${JSON.stringify(json_result)}`); - const scene_glb_media_file_token: string = json_result["scene_glb_media_file_token"]; - - const media_bucket_path = await this.getMediaFile(scene_glb_media_file_token); - console.log(`GLB ${media_bucket_path}`) + const scene_glb_media_file_token: string = + json_result["scene_glb_media_file_token"]; + + const media_bucket_path = await this.getMediaFile( + scene_glb_media_file_token, + ); + console.log(`GLB ${media_bucket_path}`); let glbLoader = new GLTFLoader(); // promisify this - const loadGlb = (bucket_path: string): Promise => { + const loadGlb = ( + bucket_path: string, + ): Promise => { return new Promise((resolve, reject) => { glbLoader.load(bucket_path, (glb) => { if (glb) { const scene: THREE.Scene = glb.scene; - const data = { "scene_glb_media_file_token": scene_glb_media_file_token , "scene_media_file_token":scene_media_file_token, "scene": scene }; - console.log(`Data: ${data}`) + const data = { + scene_glb_media_file_token: scene_glb_media_file_token, + scene_media_file_token: scene_media_file_token, + scene: scene, + }; + console.log(`Data: ${data}`); resolve(new APIManagerResponseSuccess("Success Loaded", data)); } else { throw new APIManagerResponseError("Failed to Load GLB Scene"); @@ -133,7 +162,7 @@ class APIManager { * @param media_file_token * @returns */ - private async getMediaFile(media_file_token: string): Promise { + public async getMediaFile(media_file_token: string): Promise { let api_base_url = "https://api.fakeyou.com"; let url = `${api_base_url}/v1/media_files/file/${media_file_token}`; let response = await fetch(url); @@ -205,12 +234,14 @@ class APIManager { const url = `${this.baseUrl}/v1/media_files/write/scene_file`; let uuid = uuidv4(); - console.log(`Saving Scene scene_media_file_token:${scene_media_file_token} | scene_glb_media_file_token:${scene_glb_media_file_token}`) + console.log( + `Saving Scene scene_media_file_token:${scene_media_file_token} | scene_glb_media_file_token:${scene_glb_media_file_token}`, + ); // turn json into a blob const scene_schema = { - 'scene_glb_media_file_token': scene_glb_media_file_token, - 'scene_name': scene_file_name, - 'timeline': { 'objects': [] }, + scene_glb_media_file_token: scene_glb_media_file_token, + scene_name: scene_file_name, + timeline: { objects: [] }, }; const json = JSON.stringify(scene_schema); const blob = new Blob([json], { type: "application/json" }); @@ -282,10 +313,25 @@ class APIManager { } } - public async batchGetMedia(media_tokens: []): Promise { - return ""; + public async getMediaBatch(media_tokens: string[]): Promise { + const tokens = media_tokens; + const url = new URL(`${this.baseUrl}/v1/media_files/batch`); + tokens.forEach((token) => url.searchParams.append("tokens", token)); + const result = await fetch(url) + .then((response) => response.json()) + .then((data) => { + const result = data["media_files"].map((element) => { + return new MediaFile( + element["public_bucket_path"], + element["media_type"], + element["token"], + ); + }); + return result; + }) + .catch((error) => console.error("Error:", error)); + return result; } - } export default APIManager; diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 0fbdbb1072..0a3c768ac8 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -1,6 +1,6 @@ import * as THREE from "three"; import { OrbitControls } from "three/addons/controls/OrbitControls.js"; -import { FreeCam } from "./free_cam" +import { FreeCam } from "./free_cam"; import { TransformControls } from "three/addons/controls/TransformControls.js"; import Scene from "./scene.js"; import APIManager from "./api_manager.js"; @@ -12,7 +12,7 @@ import { SMAAPass } from "three/addons/postprocessing/SMAAPass.js"; import { SAOPass } from "three/addons/postprocessing/SAOPass.js"; import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js"; import { BokehPass } from "three/addons/postprocessing/BokehPass.js"; -import { createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg"; +import { FFmpeg, createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg"; import AudioEngine from "./audio_engine.js"; import TransformEngine from "./transform_engine.js"; import { TimeLine, TimelineDataState } from "./timeline.js"; @@ -20,10 +20,9 @@ import { ClipUI } from "../datastructures/clips/clip_offset.js"; import { LipSyncEngine } from "./lip_sync_engine.js"; import { AnimationEngine } from "./animation_engine.js"; -import { PointerLockControls } from 'three/addons/controls/PointerLockControls.js'; +import { PointerLockControls } from "three/addons/controls/PointerLockControls.js"; import { APPUI_ACTION_TYPES } from "../reducers"; - class EditorState { // { // action: "ShowLoadingIndicator" @@ -44,6 +43,7 @@ class Editor { version: number; activeScene: Scene; camera: any; + render_camera: any; renderer: THREE.WebGLRenderer | undefined; clock: THREE.Clock | undefined; canvReference: any; @@ -87,12 +87,14 @@ class Editor { cam_obj: THREE.Object3D | undefined; renderPass: RenderPass | undefined; + camera_person_mode: boolean; current_scene_media_token: string | null; current_scene_glb_media_token: string | null; can_initialize: boolean; dispatchAppUiState: any; // todo figure out the type - camera_person_mode: boolean; + render_width: number; + render_height: number; // Default params. constructor() { @@ -116,6 +118,7 @@ class Editor { this.activeScene = new Scene("" + this.version); this.activeScene.initialize(); this.camera; + this.render_camera; this.renderer; this.clock; this.canvReference = null; @@ -159,6 +162,9 @@ class Editor { this.max_length = 10; // Audio Engine Test. + this.render_width = 1280; + this.render_height = 720; + this.audio_engine = new AudioEngine(); this.transform_engine = new TransformEngine(this.version); this.lipsync_engine = new LipSyncEngine(); @@ -174,7 +180,7 @@ class Editor { this.current_frame = 0; - // Dispatcher + // Dispatcher this.dispatchAppUiState = null; // Scene State @@ -182,19 +188,17 @@ class Editor { this.current_scene_glb_media_token = null; } - initialize(config:any) { + initialize(config: any) { //setup reactland Callbacks - this.dispatchAppUiState = config.dispatchAppUiState + this.dispatchAppUiState = config.dispatchAppUiState; - // Show Loader - this.dispatchAppUiState({ - type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER - }); + // this is called by the parent for some reason + // this.dispatchAppUiState({ + // type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER + // }); if (this.can_initialize == false) { - console.log( - "Editor Already Initialized", - ); + console.log("Editor Already Initialized"); return; } this.can_initialize = false; @@ -209,6 +213,9 @@ class Editor { this.camera.position.z = 3; this.camera.position.y = 3; this.camera.position.x = -3; + + this.render_camera = new THREE.PerspectiveCamera(70, width / height, 0.1, 50); + // Base WebGL render and clock for delta time. this.renderer = new THREE.WebGLRenderer({ antialias: true, @@ -225,8 +232,14 @@ class Editor { this._configure_post_pro(); // Controls and movement. - this.lockControls = new PointerLockControls(this.camera, this.renderer.domElement); - this.cameraViewControls = new FreeCam(this.camera, this.renderer.domElement); + this.lockControls = new PointerLockControls( + this.camera, + this.renderer.domElement, + ); + this.cameraViewControls = new FreeCam( + this.camera, + this.renderer.domElement, + ); this.cameraViewControls.movementSpeed = 1; this.cameraViewControls.domElement = this.renderer.domElement; this.cameraViewControls.rollSpeed = Math.PI / 24; @@ -234,14 +247,17 @@ class Editor { this.cameraViewControls.dragToLook = true; this.cameraViewControls.enabled = false; - this.orbitControls = new OrbitControls(this.camera, this.renderer.domElement); + this.orbitControls = new OrbitControls( + this.camera, + this.renderer.domElement, + ); this.control = new TransformControls(this.camera, this.renderer.domElement); // OnClick and MouseMove events. window.addEventListener("mousemove", this.onMouseMove.bind(this), false); window.addEventListener("click", this.onMouseClick.bind(this), false); // Base control and debug stuff remove debug in prod. - this._initialize_control(); + this._initializeControl(); // Resets canvas size. this.onWindowResize(); // Creates the main update loop. @@ -251,32 +267,63 @@ class Editor { this._test_demo(); - // saving state of the scene - this.current_scene_media_token = null; - this.current_scene_glb_media_token = null; - this.renderer.domElement.addEventListener("mousedown", this.onMouseDown.bind(this), false); this.renderer.domElement.addEventListener("mouseup", this.onMouseUp.bind(this), false); this.renderer.domElement.addEventListener("onContextMenu", this.onContextMenu.bind(this), false); - + this.cam_obj = this.activeScene.get_object_by_name("::CAM::"); - if(this.cam_obj) - { + if (this.cam_obj) { this.add_transform_clip_base("Camera Object", this.cam_obj, 0, 150) } - // hide loader + // saving state of the scene + this.current_scene_media_token = null; + this.current_scene_glb_media_token = null; + + this.renderer.domElement.addEventListener( + "mousedown", + this.onMouseDown.bind(this), + false, + ); + this.renderer.domElement.addEventListener( + "mouseup", + this.onMouseUp.bind(this), + false, + ); + this.renderer.domElement.addEventListener( + "onContextMenu", + this.onContextMenu.bind(this), + false, + ); + + this.cam_obj = this.activeScene.get_object_by_name("::CAM::"); + if (this.cam_obj) { + this.add_transform_clip_base("Camera Object", this.cam_obj, 0, 150); + } + this.dispatchAppUiState({ - type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER + type: APPUI_ACTION_TYPES.UPDATE_EDITOR_LOADINGBAR, + payload: { + showEditorLoadingBar: { + progress: 100, + }, + }, }); + this.dispatchAppUiState({ + type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADINGBAR, + }); } // Token comes in from the front end to load the scene from the site. + public async testBatchRequest() { + const result = await this.api_manager.getMediaBatch(["m_8fmp9hrvsqcryzka1fra597kg42s50","m_z4jzbst3xfh64h0qn4bqh4afenfps9"]); + console.log(result); + } public async loadScene(scene_media_token: string) { this.dispatchAppUiState({ - type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER + type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER, }); if (scene_media_token != null) { @@ -287,19 +334,25 @@ class Editor { this.current_scene_media_token, ); - console.log(load_scene_state_response) - if(load_scene_state_response.data == null) { - console.log("load_scene_state_response Missing Data") - return; } + console.log(load_scene_state_response); + if (load_scene_state_response.data == null) { + console.log("load_scene_state_response Missing Data"); + return; + } - const loaded_scene = load_scene_state_response.data["scene"] + const loaded_scene = load_scene_state_response.data["scene"]; + if (load_scene_state_response.data == null) { return; } // Load these so you can rewrite the scene glb using it's token. - this.current_scene_media_token = load_scene_state_response.data["scene_media_file_token"] - this.current_scene_glb_media_token = load_scene_state_response.data["scene_glb_media_file_token"] + this.current_scene_media_token = + load_scene_state_response.data["scene_media_file_token"]; + this.current_scene_glb_media_token = + load_scene_state_response.data["scene_glb_media_file_token"]; + + console.log( + `loadScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`, + ); - console.log(`loadScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`); - this.activeScene.scene.children = loaded_scene.children; this.activeScene.scene.children.forEach((child: THREE.Object3D) => { @@ -316,53 +369,59 @@ class Editor { }); this.dispatchAppUiState({ - type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER + type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER, }); } - - public async saveScene(name: string) { // remove controls when saving scene. this.removeTransformControls(); this.dispatchAppUiState({ - type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER + type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER, }); - console.log(`saveScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`); - + console.log( + `saveScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`, + ); + const result = await this.api_manager.saveSceneState( this.activeScene.scene, name, this.current_scene_glb_media_token, this.current_scene_media_token, - new TimelineDataState() - ) + new TimelineDataState(), + ); - if(result.data == null) { return; } - - const scene_media_token = result.data["scene_media_file_token"] - if (scene_media_token != null) { - this.current_scene_media_token = scene_media_token + if (result.data == null) { + return; } - const scene_glb_media_token = result.data["scene_glb_media_file_token"] + + const scene_media_token = result.data["scene_media_file_token"]; + if (scene_media_token != null) { + this.current_scene_media_token = scene_media_token; + } + const scene_glb_media_token = result.data["scene_glb_media_file_token"]; if (scene_glb_media_token != null) { - this.current_scene_glb_media_token = scene_glb_media_token - } + this.current_scene_glb_media_token = scene_glb_media_token; + } this.dispatchAppUiState({ - type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER + type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER, }); } /** - * This cleans up the transform controls - * During saving it + * This cleans up the transform controls + * During saving it * Doesn't retain those controls. - * @returns + * @returns */ private removeTransformControls() { - if (this.control == undefined) { return }; - if (this.outlinePass == undefined) { return }; + if (this.control == undefined) { + return; + } + if (this.outlinePass == undefined) { + return; + } this.last_selected = this.selected; this.control.detach(); this.activeScene.scene.remove(this.control); @@ -422,29 +481,41 @@ class Editor { switchCameraView() { this.camera_person_mode = !this.camera_person_mode; if (this.cam_obj) { - if(this.camera_person_mode) { + if (this.camera_person_mode) { this.last_cam_pos.copy(this.camera.position); this.last_cam_rot.copy(this.camera.rotation); this.camera.position.copy(this.cam_obj.position); this.camera.rotation.copy(this.cam_obj.rotation); - if(this.orbitControls) { this.orbitControls.enabled = false; } - if(this.lockControls) { this.activeScene.scene.add(this.lockControls.getObject()); } - if(this.cameraViewControls) { this.cameraViewControls.enabled = true; } + if (this.orbitControls) { + this.orbitControls.enabled = false; + } + if (this.lockControls) { + this.activeScene.scene.add(this.lockControls.getObject()); + } + if (this.cameraViewControls) { + this.cameraViewControls.enabled = true; + } - if(this.activeScene.hot_items){ - this.activeScene.hot_items.forEach(element => { + if (this.activeScene.hot_items) { + this.activeScene.hot_items.forEach((element) => { element.visible = false; }); } } else { this.camera.position.copy(this.last_cam_pos); this.camera.rotation.copy(this.last_cam_rot); - if(this.orbitControls) { this.orbitControls.enabled = true; } - if(this.lockControls) { this.activeScene.scene.remove(this.lockControls.getObject()); } - if(this.cameraViewControls) { this.cameraViewControls.enabled = false; } - if(this.activeScene.hot_items){ - this.activeScene.hot_items.forEach(element => { + if (this.orbitControls) { + this.orbitControls.enabled = true; + } + if (this.lockControls) { + this.activeScene.scene.remove(this.lockControls.getObject()); + } + if (this.cameraViewControls) { + this.cameraViewControls.enabled = false; + } + if (this.activeScene.hot_items) { + this.activeScene.hot_items.forEach((element) => { element.visible = true; }); } @@ -452,9 +523,22 @@ class Editor { } } - async add_transform_clip_base(name: string = "New Clip", object: THREE.Object3D, start_offset:number, end_offset:number) { + async add_transform_clip_base( + name: string = "New Clip", + object: THREE.Object3D, + start_offset: number, + end_offset: number, + ) { this.timeline.addPlayableClip( - new ClipUI(1.0, "transform", "clip2", object.uuid, object.uuid, start_offset, end_offset), + new ClipUI( + 1.0, + "transform", + "clip2", + object.uuid, + object.uuid, + start_offset, + end_offset, + ), ); this.transform_engine.loadObject(object.uuid, end_offset); } @@ -510,6 +594,18 @@ class Editor { 400, ), ); + + + this.audio_engine.loadClip("m_h33vytxs5eqqqf07nsy14qzrf9ww4v"); + + await this.timeline.addPlayableClip(new ClipUI( + 1.0, + "audio", + "AudioClip", + "m_h33vytxs5eqqqf07nsy14qzrf9ww4v", + "", + 0, + 50)); } // Configure post processing. @@ -572,8 +668,8 @@ class Editor { this.composer.addPass(this.smaaPass); this.composer.addPass(this.bokehPass); - this.outputPass = new OutputPass() - this.composer.addPass(this.outputPass) + this.outputPass = new OutputPass(); + this.composer.addPass(this.outputPass); } create_parim(name: string) { @@ -583,6 +679,7 @@ class Editor { render_mode() { this.rendering = !this.rendering; this.activeScene.render_mode(this.rendering); + console.log("works.") //if (this.rendering) { // this._remove_post_processing() @@ -628,9 +725,16 @@ class Editor { } take_timeline_cam_clip() { - if(this.cam_obj == null) { return; } - if(!this.camera_person_mode) { return; } - this.transform_engine.addFrame(this.cam_obj, this.transform_engine.clips[this.cam_obj.uuid].length); + if (this.cam_obj == null) { + return; + } + if (!this.camera_person_mode) { + return; + } + this.transform_engine.addFrame( + this.cam_obj, + this.transform_engine.clips[this.cam_obj.uuid].length, + ); console.log("Camera frame taken."); this.activeScene.createPoint(this.cam_obj.position, false); } @@ -649,7 +753,7 @@ class Editor { let delta_time = this.clock.getDelta(); if (this.cameraViewControls && this.camera_person_mode) { - this.cameraViewControls.update(5 * delta_time) + this.cameraViewControls.update(5 * delta_time); if (this.cam_obj) { if (this.timeline.isPlaying == false) { this.cam_obj.position.copy(this.camera.position); @@ -661,19 +765,14 @@ class Editor { } } + if (this.render_camera && this.cam_obj) { + this.render_camera.position.copy(this.cam_obj.position); + this.render_camera.rotation.copy(this.cam_obj.rotation); + } + this.timeline.update(delta_time); - this.render_scene(); - if (this.capturer != null) { - this.capturer.capture(this.renderer.domElement); - } // Record scene. - } - - start_playback() { - this.timeline.isPlaying = true; - if(!this.camera_person_mode) { - this.switchCameraView(); - } + this.renderScene(); } change_mode(type: any) { @@ -707,6 +806,56 @@ class Editor { return blob; } + async convertAudioClip(itteration: number, ffmpeg: FFmpeg, clip: ClipUI) { + let video_og = itteration + 'tmp.mp4'; + let wav_name = itteration + 'tmp.wav'; + let new_video = (itteration + 1) + 'tmp.mp4'; + let startFrame = clip.start_offset; + let endFrame = clip.ending_offset; + + + if (endFrame > this.timeline.timeLineLimit) { + endFrame = this.timeline.timeLineLimit; + } + if (startFrame > this.timeline.timeLineLimit) { + startFrame = this.timeline.timeLineLimit-1; + } + + const startTime = startFrame / this.cap_fps; + const endTime = endFrame / this.cap_fps; + let end = endTime - startTime; + + let duration = this.timeline.timeLineLimit / this.cap_fps; + + let audioSegment = "as_" + wav_name; + await ffmpeg.FS('writeFile', wav_name, await fetchFile(await this.api_manager.getMediaFile(clip.media_id))); + await ffmpeg.run('-i', + wav_name, + '-ss', '0', + '-to', '' + end, + "-max_muxing_queue_size", "999999", + audioSegment); + + await ffmpeg.run('-i', video_og,"-max_muxing_queue_size", "999999", `${itteration}empty_tmp.wav`); + + await ffmpeg.run( + '-i', `${itteration}empty_tmp.wav`, + '-i', audioSegment, + '-filter_complex', "[1:a]adelay="+startTime*1000+"|"+startTime*1000+"[a1];[0:a][a1]amix=inputs=2[a]", + "-map", "[a]", + `${itteration}final_tmp.wav`) + + await ffmpeg.run( + '-i', video_og, + '-i', `${itteration}final_tmp.wav`, + '-c:v', 'copy', + '-c:a', 'aac', + '-map', '0:v:0', + '-map', '1:a:0', + '-strict', 'experimental', + new_video) + } + async stopPlayback() { this.render_mode(); let ffmpeg = createFFmpeg({ log: true }); @@ -724,35 +873,78 @@ class Editor { "" + this.cap_fps, "-i", "image%d.png", - "output.mp4", + "-f", + "lavfi", + "-i", + "anullsrc", // This adds a silent audio track + "-max_muxing_queue_size", + "999999", + "-c:v", + "libx264", // Specify video codec (optional, but recommended for MP4) + "-c:a", + "aac", // Specify audio codec (optional, but recommended for MP4) + "-shortest", // Ensure output duration matches the shortest stream (video or audio) + "0tmp.mp4" ); - let output = await ffmpeg.FS("readFile", "output.mp4"); + + + let itteration = 0; + + for (const clip of this.timeline.timelineItems) { + if (clip.type == "lipsync" || clip.type == "audio") { + await this.convertAudioClip(itteration, ffmpeg, clip); + itteration += 1; + } + }; + + let output = await ffmpeg.FS("readFile", itteration + "tmp.mp4"); // Create a Blob from the output file for downloading const blob = new Blob([output.buffer], { type: "video/mp4" }); const url = URL.createObjectURL(blob); - await this.api_manager.uploadMedia(blob, "output.mp4"); - // Create a link to download the file const downloadLink = document.createElement("a"); downloadLink.href = url; - downloadLink.download = "output.mp4"; + downloadLink.download = "render.mp4"; document.body.appendChild(downloadLink); downloadLink.click(); // Clean up URL.revokeObjectURL(url); document.body.removeChild(downloadLink); + + + let data = await this.api_manager.uploadMedia(blob, "tmp.wav"); + console.log(data); + // Create a link to download the file + + } + + generateVideo() { + console.log("Generating video..."); + if (this.rendering) { return; } + this.startPlayback(); + this._initializeRecording(); + this.rendering = true; + this.activeScene.render_mode(this.rendering); + if (this.activeScene.hot_items) { + this.activeScene.hot_items.forEach(element => { + element.visible = false; + }); + } } startPlayback() { - this.playback_location = 0; - this._initialize_recording(); + this.timeline.isPlaying = true; + this.timeline.scrubberPosition = 0; + if (!this.camera_person_mode) { + this.switchCameraView(); + } } // Initializes transform x y z changes. - _initialize_control() { + _initializeControl() { if (this.control == undefined) { return; } - this.control.addEventListener("change", this.render_scene.bind(this)); + this.control.addEventListener("change", this.renderScene.bind(this)); this.control.addEventListener("dragging-changed", (event: any) => { if (this.orbitControls == undefined) { return; @@ -766,18 +958,18 @@ class Editor { this.activeScene.scene.add(this.control); } - // Initializes CCapture for capturing the scene to send over to backend. - _initialize_recording() { + _initializeRecording() { this.frame_buffer = []; this.render_timer = 0; } // Render the scene to the camera. - render_scene() { - if (this.composer != null) { + renderScene() { + if (this.composer != null && !this.rendering) { this.composer.render(); - } else if (this.renderer && this.camera) { - this.renderer.render(this.activeScene.scene, this.camera); + } else if (this.renderer && this.render_camera) { + this.renderer.setSize(this.render_width, this.render_height); + this.renderer.render(this.activeScene.scene, this.render_camera); } else { console.error("Could not render to canvas no render or composer!"); } @@ -787,10 +979,11 @@ class Editor { let imgData = this.renderer.domElement.toDataURL(); this.frame_buffer.push(imgData); this.render_timer += this.clock.getDelta(); - if (this.playback_location >= this.fps_number * 3) { + if (this.timeline.isPlaying == false) { this.stopPlayback(); this.playback_location = 0; this.rendering = false; + this.onWindowResize(); } } } @@ -813,6 +1006,12 @@ class Editor { if (this.composer != null) { this.composer.setSize(width, height); } + + if (this.render_camera == undefined) { return; } + + //this.renderer.setSize(this.render_width, this.render_height); + this.render_camera.aspect = this.render_width / this.render_height; + this.render_camera.updateProjectionMatrix(); } onContextMenu(event: any) { diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 208a7e20e9..267d23ba63 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -72,7 +72,7 @@ export class TimeLine { scene: Scene) { this.timelineItems = [] - this.timeLineLimit = 60 * 10 // 10 seconds + this.timeLineLimit = 60 * 5 // 5 seconds this.isPlaying = false this.scrubberPosition = 0 // in frames into the tl @@ -151,7 +151,7 @@ export class TimeLine { else if (element.type == "animation") { } else if (element.type == "lipsync") { - + this.lipSyncEngine.clips[element.object_uuid].reset(); } else { this.stop() @@ -179,13 +179,17 @@ export class TimeLine { let object = this.scene.get_object_by_uuid(element.object_uuid) if (element.type == "transform") { if (object && this.transformEngine.clips[element.object_uuid]) { - this.transformEngine.clips[element.object_uuid].length = (element.ending_offset - element.start_offset) - this.transformEngine.clips[element.object_uuid].step(object) + this.transformEngine.clips[element.object_uuid].length = (element.ending_offset - element.start_offset); + this.transformEngine.clips[element.object_uuid].step(object, element.start_offset, this.scrubberPosition); } } else if (element.type == "audio") { - // global audio track - this.audioEngine.playClip(element.media_id) + if(this.scrubberPosition+1 >= element.ending_offset){ + this.audioEngine.stopClip(element.media_id); + } + else { + this.audioEngine.playClip(element.media_id); + } } else if (element.type == "lipsync") { if(this.scrubberPosition+1 >= element.ending_offset){ diff --git a/app/pages/PageEnigma/reducers/appUiReducer/values.tsx b/app/pages/PageEnigma/reducers/appUiReducer/values.tsx index 25aa12acab..b62b819a6f 100644 --- a/app/pages/PageEnigma/reducers/appUiReducer/values.tsx +++ b/app/pages/PageEnigma/reducers/appUiReducer/values.tsx @@ -4,11 +4,12 @@ export const initialStateValues : State = { timelineHeight: 260, viewMode: VIEW_MODES.EDITOR, showEditorLoader: { - isShowing:true, - message: 'loading Editor Engine 🦊' + isShowing:false, + message: 'Loading Editor Engine 🦊' }, showEditorLoadingBar: { - isShowing:false, - progress:0, + isShowing:true, + progress:5, + message: 'Loading Editor Engine 🦊' } } \ No newline at end of file diff --git a/app/routes/ComponentLibrary.tsx b/app/routes/ComponentLibrary.tsx index fd340f82a6..c0e6719021 100644 --- a/app/routes/ComponentLibrary.tsx +++ b/app/routes/ComponentLibrary.tsx @@ -2,7 +2,7 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { Button, ButtonLink, - H4, + H1,H2,H3,H4,H5,H6,P,Label, LoadingBar, LoadingDotsTyping, LoadingDotsBricks, @@ -35,65 +35,84 @@ export default function ComponentLibrary () { },[progress]); return( -
+
+
-
-

Buttons

-
- - - - -
-
- -
-

ButtonLink

- Back to / -
- -
-

Loading Dots

-
-
- -
-
- +
+

Typography

+

H1. This is an H1 Heading

+

H2. This is an H2 Heading

+

H3. This is an H3 Heading

+

H4. This is an H4 Heading

+ +

P. Is an normal Paragraph

+
H5. This is an H5 Heading
+
H6. This is an H6 Heading
+ +
+

Components

+

Buttons

+
+ + + +
-
-
-

Loading Bar

-
+
+

ButtonLink

+ Back to / +
+ +
+

Loading Dots

+
+
+ +
+
+ +
+
+
+ +
+

Loading Bar

+
+ + +
+ + -
- - -
);