diff --git a/app/pages/PageEnigma/PageEnigmaComponent.tsx b/app/pages/PageEnigma/PageEnigmaComponent.tsx index 0f8db07a61..df6bb91257 100644 --- a/app/pages/PageEnigma/PageEnigmaComponent.tsx +++ b/app/pages/PageEnigma/PageEnigmaComponent.tsx @@ -25,7 +25,7 @@ export const PageEnigmaComponent = () => { useEffect(()=>{ setTimeout(()=>dispatchAppUiState({ type: ACTION_TYPES.HIDE_EDITOR_LOADER - }), 3000); + }), 500); },[]); return (
diff --git a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx index bbf4dcd81e..1c5e7fe47b 100644 --- a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx +++ b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx @@ -9,8 +9,7 @@ export const ControlsTopButtons = () => { const handleButtonSave = () => { console.log(`SceneName is ${sceneName}`); - // editorEngine?.save(); - + editorEngine?.saveScene(); }; const handleButtonCameraView = () => { @@ -23,11 +22,11 @@ export const ControlsTopButtons = () => { const handleButtonLoad = () => { //document.getElementById("load-upload")?.click(); if(editorEngine == null) {return;} - editorEngine._load_for_testing(); + editorEngine.loadScene(); }; const handleButtonRender = () => { console.log("Saving GLB to server for reference"); - editorEngine?._upload_for_testing(); + editorEngine?.saveScene(); //editorEngine?.togglePlayback(); }; const handleButtonPlay = () => {}; diff --git a/app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/animation_clip.ts similarity index 88% rename from app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts rename to app/pages/PageEnigma/datastructures/clips/animation_clip.ts index ea38c59885..50110653d4 100644 --- a/app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/animation_clip.ts @@ -1,19 +1,7 @@ -import { faL } from '@fortawesome/pro-solid-svg-icons'; import * as THREE from 'three'; import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'; -export interface AnimationTrackClip { - version: number - media_id: string - object_uuid: string - type: "animation" - location: "glb" | "remote" - speed: number - length: number - clip_name: string -} - -export class AnimationTrackClip implements AnimationTrackClip { +export class AnimationClip { version: number; media_id: string; // comes from the server object_uuid: string; diff --git a/app/pages/PageEnigma/datastructures/clips/audio_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/audio_clip.ts similarity index 91% rename from app/pages/PageEnigma/datastructures/clips/audio_track_clip.ts rename to app/pages/PageEnigma/datastructures/clips/audio_clip.ts index 805741b550..cc5c19079c 100644 --- a/app/pages/PageEnigma/datastructures/clips/audio_track_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/audio_clip.ts @@ -14,14 +14,7 @@ class AudioData implements AudioData{ } } -export interface AudioTrackClip { - version: number; - media_id: string; - type: "audio"; - volume: number; -} - -export class AudioTrackClip implements AudioTrackClip { +export class AudioClip { version: number; media_id: string; type: "audio" = "audio"; diff --git a/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts similarity index 94% rename from app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts rename to app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts index 412b8b0bc3..a06808e72c 100644 --- a/app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts @@ -16,13 +16,6 @@ class AudioData implements AudioData { } } -export interface LipSyncTrackClip { - version: number; - media_id: string; - type: "lipsync"; - volume: number; -} - class BlendShapeHelper { ah: number; ee: number; @@ -34,7 +27,7 @@ class BlendShapeHelper { } } -export class LipSyncTrackClip implements LipSyncTrackClip { +export class LipSyncClip { version: number; media_id: string; type: "lipsync" = "lipsync"; @@ -98,7 +91,7 @@ export class LipSyncTrackClip implements LipSyncTrackClip { } - async play(object: THREE.Object3D) { +async play(object: THREE.Object3D) { if (this.audio_data?.audioBuffer == null) { await this.download_audio(); } if (this.lipsync.face == null) { this.lipsync = new LipSync(await this._detect_face(object)); diff --git a/app/pages/PageEnigma/datastructures/clips/transform_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/transform_clip.ts similarity index 89% rename from app/pages/PageEnigma/datastructures/clips/transform_track_clip.ts rename to app/pages/PageEnigma/datastructures/clips/transform_clip.ts index 729f36621f..f167843319 100644 --- a/app/pages/PageEnigma/datastructures/clips/transform_track_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/transform_clip.ts @@ -1,17 +1,6 @@ -import { faL } from '@fortawesome/pro-solid-svg-icons'; import * as THREE from 'three'; -export interface TransformTrackClip { - version: number; - media_id: string; - type: "transform"; - positions: THREE.Vector3[]; - rotations: THREE.Vector3[]; - scales: THREE.Vector3[]; - length: number; -} - -export class TransformTrackClip implements TransformTrackClip { +export class TransformClip { version: number; media_id: string; object_uuid: string; diff --git a/app/pages/PageEnigma/datastructures/scene/scene_object.ts b/app/pages/PageEnigma/datastructures/scene/scene_object.ts deleted file mode 100644 index 1242295f67..0000000000 --- a/app/pages/PageEnigma/datastructures/scene/scene_object.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { AudioTrackClip } from "../clips/audio_track_clip"; -import { AnimationTrackClip } from "../clips/animation_track_clip"; -import { TransformTrackClip } from "../clips/transform_track_clip"; -import { ClipOffset } from "../clips/clip_offset"; - -export interface Scene { - version: number; - glb_media_id: number; // Assuming a numerical ID, adjust as needed - skybox_type: "cube" | "six_sided_cube" | "sphere"; - skybox_media_ids: { [key: string]: number }; - timeline: Timeline; -} - -export interface Timeline { - version: number; - last_scrubber_position: number; - audio_track_clips: AudioTrackClip[]; // Assuming defined elsewhere - transform_track_clips: TransformTrackClip[]; // Placeholder, define as needed - animation_track_clips: AnimationTrackClip[]; // Placeholder, define as needed - entities: Entity[]; - globalAudio: GlobalAudio; - camera: Camera; -} - -export interface Entity { - object_uuid: number; - clip_offsets: ClipOffset[]; -} - -export interface GlobalAudio { - clip_offsets: ClipOffset[]; -} - -export interface Camera { - clip_offsets: ClipOffset[]; -} - -export class Scene implements Scene { - version: number; - glb_media_id: number; - skybox_type: "cube" | "six_sided_cube" | "sphere"; - skybox_media_ids: { [key: string]: number }; - timeline: Timeline; - - constructor( - version: number, - glb_media_id: number, - skybox_type: "cube" | "six_sided_cube" | "sphere", - skybox_image_ids: { [key: string]: number }, - timeline: Timeline, - ) - { - this.version = version; - this.glb_media_id = glb_media_id; - this.skybox_type = skybox_type; - this.skybox_media_ids = skybox_image_ids; - this.timeline = timeline; - } - - toJSON(): string { - return JSON.stringify(this); - } -} - -export class Timeline implements Timeline { - version: number; - last_scrubber_position: number; - audio_track_clips: AudioTrackClip[]; - transform_track_clips: TransformTrackClip[]; - animation_track_clips: AnimationTrackClip[]; - entities: Entity[]; - globalAudio: GlobalAudio; - camera: Camera; - - constructor( - version: number, - last_scrubber_position: number, - audio_track_clips: AudioTrackClip[], - transform_track_clips: TransformTrackClip[], - animation_track_clips: AnimationTrackClip[], - entities: Entity[], - globalAudio: GlobalAudio, - camera: Camera, - ) - { - this.version = version; - this.last_scrubber_position = last_scrubber_position; - this.audio_track_clips = audio_track_clips; - this.transform_track_clips = transform_track_clips; - this.animation_track_clips = animation_track_clips; - this.entities = entities; - this.globalAudio = globalAudio; - this.camera = camera; - } - - toJSON(): string { - return JSON.stringify(this); - } - // For testing purposes - static generateMockJson(): string { - const mock = { - version: 1, - last_scrubber_position: 5000, - audio_track_clips: [ - { version: 1, uuid: 0, media_id: 1, type: "audio", volume: 100 } - ], - transform_track_clips: [ - { version: 1, uuid: 1, media_id: 2, type: "transform", position: [{ x: 0, y: 0, z: 0 }], rotation: [{ x: 0, y: 0, z: 0 }], scale: [{ x: 1, y: 1, z: 1 }] } - ], - animation_track_clips: [ - { version: 1, uuid: 2, media_id: 3, type: "animation", speed: 1.0, length: 100, starting_offset: 0, clip_name: "some clip name" } - ], - entities: [ - { object_uuid: 0, clip_offsets: [] } // Assuming more details would be filled as necessary - ], - globalAudio: { clip_offsets: [] }, // Assuming more details would be filled as necessary - camera: { clip_offsets: [] } // Assuming more details would be filled as necessary - }; - return JSON.stringify(mock); - } -} diff --git a/app/pages/PageEnigma/js/animation_engine.ts b/app/pages/PageEnigma/js/animation_engine.ts index fb155ea7d2..a2fde07189 100644 --- a/app/pages/PageEnigma/js/animation_engine.ts +++ b/app/pages/PageEnigma/js/animation_engine.ts @@ -1,8 +1,8 @@ -import { AnimationTrackClip } from "../datastructures/clips/animation_track_clip"; +import { AnimationClip } from "../datastructures/clips/animation_clip"; import * as THREE from 'three'; export class AnimationEngine { - clips: { [key: string]: AnimationTrackClip } = {}; + clips: { [key: string]: AnimationClip } = {}; version: number; length: number; @@ -13,7 +13,7 @@ export class AnimationEngine { } load_object(object_uuid: string, media_id:string, clip_name: string) { - this.clips[object_uuid] = new AnimationTrackClip( + this.clips[object_uuid] = new AnimationClip( this.version, media_id, "glb", diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index 8167bcd0e4..a081738d41 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -3,9 +3,37 @@ import { Scene } from "../datastructures/scene/scene_object"; import * as THREE from "three"; import { GLTFExporter } from "three/addons/exporters/GLTFExporter.js"; import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; -import { TimelineCurrentReactState } from "./timeline"; +import { TimelineDataState } from "./timeline"; + +/** + * Storyteller Studio API Manager + * The source of truth of all these media items is the database in the cloud + */ + +/** + * This is designed to surface user customer facing messages as errors. + */ +type Tokens = { [key: string]: string }; +class APIManagerResponseSuccess { + public user_message:String + public data:Tokens | null + constructor(user_message:String = "", data:Tokens | null = null) { + this.data = data + this.user_message = user_message + } +} + +/** + * This is designed to surface user customer facing messages as errors. + * Errors shouldn't be 404 or something confusing should be + */ +class APIManagerResponseError extends Error { + constructor(message?: string) { + super(message); + this.name = "APIManagerResponseError"; + } +} -// Basically the data will exist on the database class APIManager { baseUrl: String; @@ -14,60 +42,116 @@ class APIManager { //this.baseUrl = "http://localhost:12345" } - // Function to save or update the scene - // it will always return a boolean success and the media_file_token - // they take an optional media_file_token as a parameter for the file to overwrite, which must belong to the user. - //If it's an anonymous user, we check the anonymous visitor token/cookie + /** + * @param scene The 3JS Scene we want to save. + * @param scene_name The Scene name we want to display + * @param scene_media_file_token If null we will we will create a new save or copy the scene, if provided we will overwrite the scene. + * @returns APIManagerResponseMessage + */ + public async saveSceneState(scene: THREE.Scene, + scene_name:string, + scene_media_file_token:string | null = null, + timeline_state:TimelineDataState):Promise { + + const file = await this.gltfExport(scene) + console.log(`FILE: ${file}`) + // will overwrite the scene on db if token exists + const upload_response = await this.uploadGLB(file,scene_media_file_token) + await this.saveSceneAndTimeLineState(scene_media_file_token,scene_name,timeline_state) + return new APIManagerResponseSuccess("Scene Saved") + } + + public async loadSceneState(scene_media_file_token: string): Promise { + const api_base_url = "https://api.fakeyou.com" + const url = `${api_base_url}/v1/media_files/file/${scene_media_file_token}` + const response = await fetch(url) + const json = await JSON.parse(await response.text()) + const bucket_path = json["media_file"]["public_bucket_path"] + const media_base_url = "https://storage.googleapis.com/vocodes-public" + const media_url = `${media_base_url}${bucket_path}` // gets you a bucket path - async saveSceneState( - scene: THREE.Scene, - timeline: TimelineCurrentReactState, - scene_name: string, - scene_media_file_token: string | null = null, - ): Promise { - // create new scene - - let file = await this.gltfExport(scene); - let response = await this.uploadGLB(file); - let success = response["success"]; - if (success) { - let media_file_token = response["media_file_token"]; - console.log(`media file token: ${media_file_token}`) - const result = await this._saveSceneAndTimelineToJSONSpec(media_file_token,scene_name, timeline); - return result - } - - - throw Error("Saving the Scene Resulted in an Error Success False"); + const file_response = await fetch(media_url) + + if (!file_response.ok) { + throw new APIManagerResponseError("Failed to download file") + } + // Convert the response to a Blob + let blob = await file_response.blob() + const json_result:string = await new Promise((resolve, reject) => { + let reader = new FileReader() + reader.onloadend = () => resolve(JSON.parse(reader.result as string)) + reader.onerror = reject + reader.readAsText(blob) + }) + + console.log(`JSON Result: ${json_result}`) + + const scene_glb_media_file_id:string = json_result["glb_media_file_id"] + const media_bucket_path = await this.getMediaFile(scene_glb_media_file_id) + let glbLoader = new GLTFLoader() + + // promisify this + const loadGlb = (bucket_path: string): Promise => { + return new Promise((resolve, reject) => { + glbLoader.load(bucket_path, (glb) => { + if (glb) { + const scene:THREE.Scene = glb.scene + resolve(scene) + } else { + reject(new APIManagerResponseError('Failed to Load GLB Scene')) + } + }) + }) + } + loadGlb(media_bucket_path) + } + + /** + * This recieves the bucket path of a media file + * @param media_file_token + * @returns + */ + private async getMediaFile(media_file_token: string): Promise { + let api_base_url = "https://api.fakeyou.com"; + let url = `${api_base_url}/v1/media_files/file/${media_file_token}` + let response = await fetch(url); + let json = await JSON.parse(await response.text()); + let bucketPath = json["media_file"]["public_bucket_path"]; + let media_base_url = "https://storage.googleapis.com/vocodes-public" + let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path + return media_url + } + + /** + This will save the scene to keep ids positions. + It will also give the file a name which will be a uuidv4() + @param scene The 3JS Scene we want to make a file to be uploaded as multipart form. + */ + private async gltfExport(scene: THREE.Scene):Promise { + const gltfExporter = new GLTFExporter(); + const uuid = uuidv4(); + const result = await gltfExporter.parseAsync(scene); + const file = new File([JSON.stringify(result)], `${uuid}.glb`, { + type: "application/json", + }); + return file; } - // For JSON files. - // Form-multipart POST with parameters: - // uuid_idempotency_token - // file - // media_file_token (optional; the file to replace if present) - async _saveSceneAndTimelineToJSONSpec( - media_file_token: string, - scene_file_name: string, - timeline: TimelineCurrentReactState, - ): Promise { - const url = `${this.baseUrl}/v1/media_files/write/scene_file`; + private async uploadGLB(file: File, scene_media_file_token:string | null): Promise { + const url = `${this.baseUrl}/v1/media_files/write/engine_asset`; let uuid = uuidv4(); + const form_data = new FormData(); + form_data.append("uuid_idempotency_token", uuid); - // turn json into a blob - const file_test = {"glb_media_file_id":media_file_token,"name":scene_file_name,"entities":[], "timeline":""} - - const json = JSON.stringify(file_test); - const blob = new Blob([json], {type: 'application/json'}); - const fileName = `${uuid}-${scene_file_name}.json`; - - const formData = new FormData(); - - formData.append("uuid_idempotency_token", uuid); - formData.append("file", blob, fileName); - formData.append("source", "file"); - formData.append("type", "scene_json"); - formData.append("source", "file"); + // update existing scene otherwise create new scene and use it's media_file_id + if (scene_media_file_token != null) { + form_data.append("media_file_token", scene_media_file_token); + } + + form_data.append("file", file); + form_data.append("source", "file"); + form_data.append("media_file_subtype", "scene_import"); + form_data.append("media_file_class", "scene"); const response = await fetch(url, { method: "POST", @@ -75,82 +159,71 @@ class APIManager { headers: { Accept: "application/json", }, - body: formData, + body: form_data, }); if (!response.ok) { throw new Error("Failed to Send Data"); } - - console.log(`saveSceneAndTimelineToJSONSpecResponse: ${response}`) - - return response.json(); // or handle the response as appropriate + const json_data = await response.json() + // const json_string = JSON.stringify(json_data) + // console.log(`uploadGLB: ${json_string}`) + return json_data; // or handle the response as appropriate } - // media_file_token that describes the scene - async loadScene(media_file_token: string): Promise { - let api_base_url = "https://api.fakeyou.com"; - let url = `${api_base_url}/v1/media_files/file/${media_file_token}` - let response = await fetch(url); - let json = await JSON.parse(await response.text()); - let bucketPath = json["media_file"]["public_bucket_path"]; - let media_base_url = "https://storage.googleapis.com/vocodes-public" - let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path - - let fileResponse = await fetch(media_url); + private async saveSceneAndTimeLineState( + scene_media_file_token: string | null, + scene_file_name: string, + timeline_state: TimelineDataState | null, // only for now. + ): Promise { + const url = `${this.baseUrl}/v1/media_files/write/scene_file` + let uuid = uuidv4() - if (!fileResponse.ok) { - throw new Error("Failed to download file"); + // turn json into a blob + const scene_schema = {"glb_media_file_id":scene_media_file_token,"scene_name":scene_file_name,"timeline": { "objects":[] } } + const json = JSON.stringify(scene_schema) + const blob = new Blob([json], {type: 'application/json'}) + const file_name = `${uuid}.json` + + const form_data = new FormData() + + form_data.append("uuid_idempotency_token", uuid) + form_data.append("file", blob, file_name) + form_data.append("source", "file") + form_data.append("type", "scene_json") + form_data.append("source", "file") + + const response = await fetch(url, { + method: "POST", + credentials: "include", + headers: { + Accept: "application/json", + }, + body: form_data, + }) + + if (!response.ok) { + throw new Error("Failed to Send Data") } - // Convert the response to a Blob - let blob = await fileResponse.blob(); - const json_value:string = await new Promise((resolve, reject) => { - let reader = new FileReader(); - reader.onloadend = () => resolve(JSON.parse(reader.result as string)); - reader.onerror = reject; - reader.readAsText(blob); - }); - console.log(json_value) - return json_value - } + const json_data = await response.json() - // Provides bucket path to the actual file to download from the media file - async getMediaFile(media_file_token: string): Promise { - let api_base_url = "https://api.fakeyou.com"; - let url = `${api_base_url}/v1/media_files/file/${media_file_token}` - let response = await fetch(url); - let json = await JSON.parse(await response.text()); - let bucketPath = json["media_file"]["public_bucket_path"]; - let media_base_url = "https://storage.googleapis.com/vocodes-public" - let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path - // TODO get it as binary. - return media_url - } - - async batchGetMedia(media_tokens: []): Promise { - return ""; + // example response {"success":true,"media_file_token":"m_r1ztnn501g2rn0vv2np08nd6zy2fvt"} + console.log(`saveSceneAndTimeLineState: ${JSON.stringify(json_data)}`) + // should return the same token if it is same file else new token. + return json_data // or handle the response as appropriate } - async gltfExport(scene: THREE.Scene) { - let gltfExporter = new GLTFExporter(); - let uuid = uuidv4(); - // save the scene to keep ids positions etc - let result = await gltfExporter.parseAsync(scene); - const file = new File([JSON.stringify(result)], `${uuid}.glb`, { - type: "application/json", - }); - return file; - } - // two core functions to upload things to the server and get id's - async uploadMedia(blob: any, fileName: string): Promise { + private async uploadMedia(blob:any,fileName:string):Promise { const url = `${this.baseUrl}/v1/media_uploads/upload`; let uuid = uuidv4(); + const formData = new FormData(); formData.append("uuid_idempotency_token", uuid); formData.append("file", blob, fileName); formData.append("source", "file"); formData.append("type", "video"); formData.append("source", "file"); + const response = await fetch(url, { method: "POST", credentials: "include", @@ -161,11 +234,167 @@ class APIManager { }); if (!response.ok) { - throw new Error("Failed to Send Data"); + throw new APIManagerResponseError("Upload Media Failed to send file") + } + else { + const json_data = await response.json() + return new APIManagerResponseSuccess("Uploaded Media Success") } - return response.json(); } + public async batchGetMedia(media_tokens: []): Promise { + return ""; + } + + // Function to save or update the scene + // it will always return a boolean success and the media_file_token + // they take an optional media_file_token as a parameter for the file to overwrite, which must belong to the user. + // If it's an anonymous user, we check the anonymous visitor token/cookie + + // async saveSceneState2( + // scene: THREE.Scene, + // timeline: TimelineDataState, + // scene_name: string, + // scene_media_file_token: string | null = null, + // ): Promise { + // // create new scene + + // let file = await this.gltfExport(scene); + // let response = await this.uploadGLB(file); + // let success = response["success"]; + // if (success) { + // let media_file_token = response["media_file_token"]; + // console.log(`media file token: ${media_file_token}`) + // const result = await this._saveSceneAndTimelineToJSONSpec(media_file_token,scene_name, timeline); + // return result + // } + + + // throw Error("Saving the Scene Resulted in an Error Success False"); + // } + + // // For JSON files. + // // Form-multipart POST with parameters: + // // uuid_idempotency_token + // // file + // // media_file_token (optional; the file to replace if present) + // async _saveSceneAndTimelineToJSONSpec( + // media_file_token: string, + // scene_file_name: string, + // timeline: TimelineDataState, + // ): Promise { + // const url = `${this.baseUrl}/v1/media_files/write/scene_file`; + // let uuid = uuidv4(); + + // // turn json into a blob + // const file_test = {"glb_media_file_id":media_file_token,"name":scene_file_name,"entities":[], "timeline":""} + + // const json = JSON.stringify(file_test); + // const blob = new Blob([json], {type: 'application/json'}); + // const fileName = `${uuid}-${scene_file_name}.json`; + + // const formData = new FormData(); + + // formData.append("uuid_idempotency_token", uuid); + // formData.append("file", blob, fileName); + // formData.append("source", "file"); + // formData.append("type", "scene_json"); + // formData.append("source", "file"); + + // const response = await fetch(url, { + // method: "POST", + // credentials: "include", + // headers: { + // Accept: "application/json", + // }, + // body: formData, + // }); + + // if (!response.ok) { + // throw new Error("Failed to Send Data"); + // } + + // console.log(`saveSceneAndTimelineToJSONSpecResponse: ${response}`) + + // return response.json(); // or handle the response as appropriate + // } + + // // media_file_token that describes the scene + // async loadScene(media_file_token: string): Promise { + // let api_base_url = "https://api.fakeyou.com"; + // let url = `${api_base_url}/v1/media_files/file/${media_file_token}` + // let response = await fetch(url); + // let json = await JSON.parse(await response.text()); + // let bucketPath = json["media_file"]["public_bucket_path"]; + // let media_base_url = "https://storage.googleapis.com/vocodes-public" + // let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path + + // let fileResponse = await fetch(media_url); + + // if (!fileResponse.ok) { + // throw new Error("Failed to download file"); + // } + // // Convert the response to a Blob + // let blob = await fileResponse.blob(); + // const json_value:string = await new Promise((resolve, reject) => { + // let reader = new FileReader(); + // reader.onloadend = () => resolve(JSON.parse(reader.result as string)); + // reader.onerror = reject; + // reader.readAsText(blob); + // }); + // console.log(json_value) + // return json_value + // } + + // // Provides bucket path to the actual file to download from the media file + // async getMediaFile(media_file_token: string): Promise { + // let api_base_url = "https://api.fakeyou.com"; + // let url = `${api_base_url}/v1/media_files/file/${media_file_token}` + // let response = await fetch(url); + // let json = await JSON.parse(await response.text()); + // let bucketPath = json["media_file"]["public_bucket_path"]; + // let media_base_url = "https://storage.googleapis.com/vocodes-public" + // let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path + // // TODO get it as binary. + // return media_url + // } + + + // async gltfExport2(scene: THREE.Scene) { + // let gltfExporter = new GLTFExporter(); + // let uuid = uuidv4(); + // // save the scene to keep ids positions etc + // let result = await gltfExporter.parseAsync(scene); + // const file = new File([JSON.stringify(result)], `${uuid}.glb`, { + // type: "application/json", + // }); + // return file; + // } + // // two core functions to upload things to the server and get id's + // async uploadMedia2(blob: any, fileName: string): Promise { + // const url = `${this.baseUrl}/v1/media_uploads/upload`; + // let uuid = uuidv4(); + // const formData = new FormData(); + // formData.append("uuid_idempotency_token", uuid); + // formData.append("file", blob, fileName); + // formData.append("source", "file"); + // formData.append("type", "video"); + // formData.append("source", "file"); + // const response = await fetch(url, { + // method: "POST", + // credentials: "include", + // headers: { + // Accept: "application/json", + // }, + // body: formData, + // }); + + // if (!response.ok) { + // throw new Error("Failed to Send Data"); + // } + // return response.json(); + // } + /** To save the scene as a file get the id back to serialize /v1/media_files/write/engine_asset @@ -179,35 +408,35 @@ class APIManager { media_file_subtype (optional; mixamo, mocap_net, scene_import, animation_only) media_file_class (audio, image, video, animation, character, prop, scene, unknown) **/ - async uploadGLB(file: File, scene_media_file_token:string | null): Promise { - const url = `${this.baseUrl}/v1/media_files/write/engine_asset`; - let uuid = uuidv4(); - const formData = new FormData(); - formData.append("uuid_idempotency_token", uuid); + // async uploadGLB2(file: File, scene_media_file_token:string | null): Promise { + // const url = `${this.baseUrl}/v1/media_files/write/engine_asset`; + // let uuid = uuidv4(); + // const formData = new FormData(); + // formData.append("uuid_idempotency_token", uuid); - // update existing scene otherwise create new scene and use it's media_file_id - if (scene_media_file_token != null) { - formData.append("media_file_token", scene_media_file_token); - } + // // update existing scene otherwise create new scene and use it's media_file_id + // if (scene_media_file_token != null) { + // formData.append("media_file_token", scene_media_file_token); + // } - formData.append("file", file); - formData.append("source", "file"); - formData.append("media_file_subtype", "scene_import"); - formData.append("media_file_class", "scene"); - const response = await fetch(url, { - method: "POST", - credentials: "include", - headers: { - Accept: "application/json", - }, - body: formData, - }); + // formData.append("file", file); + // formData.append("source", "file"); + // formData.append("media_file_subtype", "scene_import"); + // formData.append("media_file_class", "scene"); + // const response = await fetch(url, { + // method: "POST", + // credentials: "include", + // headers: { + // Accept: "application/json", + // }, + // body: formData, + // }); - if (!response.ok) { - throw new Error("Failed to Send Data"); - } - return response.json(); // or handle the response as appropriate - } + // if (!response.ok) { + // throw new Error("Failed to Send Data"); + // } + // return response.json(); // or handle the response as appropriate + // } } export default APIManager; diff --git a/app/pages/PageEnigma/js/audio_engine.ts b/app/pages/PageEnigma/js/audio_engine.ts index 17e36ac368..e6576636da 100644 --- a/app/pages/PageEnigma/js/audio_engine.ts +++ b/app/pages/PageEnigma/js/audio_engine.ts @@ -1,8 +1,8 @@ import { faL } from "@fortawesome/pro-solid-svg-icons"; -import { AudioTrackClip } from "../datastructures/clips/audio_track_clip"; +import { AudioClip } from "../datastructures/clips/audio_clip"; class AudioEngine { - clips: { [key: string]: AudioTrackClip } = {}; + clips: { [key: string]: AudioClip } = {}; audio_sources: { [key: string]: AudioBufferSourceNode } = {}; version: number; playing: string[]; @@ -17,7 +17,7 @@ class AudioEngine { // loads clips into the engine to cache loadClip(audio_media_id: string) { if(this.clips[audio_media_id] != null) { return; } - this.clips[audio_media_id] = new AudioTrackClip(this.version, audio_media_id, 1.0); + this.clips[audio_media_id] = new AudioClip(this.version, audio_media_id, 1.0); } // plays from the timeline. diff --git a/app/pages/PageEnigma/js/editor.js b/app/pages/PageEnigma/js/editor.js index 8cc07f9c87..565f38678f 100644 --- a/app/pages/PageEnigma/js/editor.js +++ b/app/pages/PageEnigma/js/editor.js @@ -16,7 +16,7 @@ import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; import AudioEngine from "./audio_engine.ts"; import TransformEngine from "./transform_engine.ts"; -import { TimeLine } from "./timeline.ts"; +import { TimeLine, TimelineDataState } from "./timeline.ts"; import { ClipUI } from "../datastructures/clips/clip_offset.ts"; import { LipSync } from "./lipsync.js"; @@ -36,6 +36,13 @@ if (typeof window !== "undefined") { } class EditorState { + + // { + // action: "ShowLoadingIndicator" + // source: "Editor" + // data: { "message" : "saving scene" } + // } + constructor() { this.selected_object = null; this.is_loading = false; @@ -104,6 +111,7 @@ class Editor { this.transform_engine = new TransformEngine(); this.lipsync_engine = new LipSyncEngine(); this.animation_engine = new AnimationEngine(); + this.timeline = new TimeLine( this.audio_engine, this.transform_engine, @@ -168,19 +176,39 @@ class Editor { this.timeline.scene = this.activeScene; - this._test_demo() - + //this._test_demo() // saving state of the scene this.scene_file_token = null } + // Token comes in from the front end to load the scene from the site. + async loadScene() { + + const scene = await this.api_manager.loadSceneState(this.test_scene_load_media_id); + this.activeScene.scene.children = scene.children; + + this.activeScene.scene.children.forEach(child => { + + child.parent = this.activeScene.scene; + + if(child.type == "DirectionalLight") { + let pos = child.position; + let rot = child.rotation; + let light = this.activeScene._create_base_lighting(); + light.position.set(pos.x, pos.y, pos.z); + light.rotation.set(rot.x, rot.y, rot.z); + this.activeScene.scene.remove(child); + } + }) + + } async saveScene(name) { - // check if I already have the scene token. - console.log("Saving..."); - const result = await this.api_manager.saveSceneState(this.activeScene.scene,this.scene_file_token); - console.log("Saved!"); + const result = await this.api_manager.saveSceneState(this.activeScene.scene,name, + this.scene_file_token, + new TimelineDataState()); + // dispatch call wil's engine. } // uploading some objects for testing to get their media ids from my account. diff --git a/app/pages/PageEnigma/js/lip_sync_engine.ts b/app/pages/PageEnigma/js/lip_sync_engine.ts index 15d808e1e7..1bfb590da2 100644 --- a/app/pages/PageEnigma/js/lip_sync_engine.ts +++ b/app/pages/PageEnigma/js/lip_sync_engine.ts @@ -1,24 +1,22 @@ -import { LipSyncTrackClip } from "../datastructures/clips/lipsync_track_clip"; -import { LipSync } from "./lipsync"; -import * as THREE from 'three'; +import { LipSyncClip } from "../datastructures/clips/lipsync_clip"; + // needs to be apart of the editor window. export class LipSyncEngine { - clips: { [key: string]: LipSyncTrackClip } = {} + clips: { [key: string]: LipSyncClip } = {} audio_sources: { [key: string]: AudioBufferSourceNode } = {} version: number constructor() { - this.clips = {} this.audio_sources = {} this.version = 1.0 } load_object(object_uuid: string, audio_media_id: string) { - this.clips[object_uuid] = new LipSyncTrackClip(this.version, audio_media_id, 1.0); + this.clips[object_uuid] = new LipSyncClip(this.version, audio_media_id, 1.0); } } diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 84fa27d49c..208a7e20e9 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -6,18 +6,46 @@ import AudioEngine from "./audio_engine"; import TransformEngine from "./transform_engine"; import LipSyncEngine from "./lip_sync_engine"; import AnimationEngine from "./animation_engine"; +import { AudioClip } from "../models/track"; +import { AnimationClip } from "three"; +import { LipSyncClip } from "../datastructures/clips/lipsync_clip"; +import { TransformClip } from "../datastructures/clips/transform_clip"; // Every object uuid / entity has a track. -export class TimelineCurrentReactState { -isEditable: boolean +export class TimelineCurrentState { + isEditable: boolean selectedObjectID: number - constructor() { this.isEditable = true // can add clips to it this.selectedObjectID = 0 } } +export class TimelineDataState { + timelineItems: ClipUI[] + scrubberPosition: number + + // Data to serialize with the scene, used to load into each engine. + audioClips:AudioClip[] + animationClips:AnimationClip[] + lipSyncClips:LipSyncClip[] + transformClips:TransformClip[] + + constructor(timelineItems: ClipUI[] = [], + audioClips: AudioClip[] = [], + animationClips: AnimationClip[] = [], + transformClips: TransformClip[] = [], + lipSyncClips: LipSyncClip[] = [], + scrubberPosition:number = 0) { + this.timelineItems = timelineItems + this.scrubberPosition = scrubberPosition + this.audioClips = audioClips + this.animationClips = animationClips + this.lipSyncClips = lipSyncClips + this.transformClips = transformClips + } +} + export class TimeLine { timelineItems: ClipUI[] @@ -26,8 +54,6 @@ export class TimeLine { scrubberPosition: number isPlaying: boolean - timelineState: TimelineCurrentReactState - // plays audio audioEngine: AudioEngine // key framing @@ -50,7 +76,6 @@ export class TimeLine { this.isPlaying = false this.scrubberPosition = 0 // in frames into the tl - this.timelineState = new TimelineCurrentReactState() // this will be used to play the audio clips this.audioEngine = audioEngine @@ -61,60 +86,60 @@ export class TimeLine { this.scene = scene; } - async addPlayableClip(clip: ClipUI): Promise { + public async addPlayableClip(clip: ClipUI): Promise { this.timelineItems.push(clip) } // when given a media id item it will create the clip. // Then the clip will be loaded by the engines, if they come from outside of the loaded scene. - async createClipOffset(media_id: string, object_uuid:string, type: string): Promise { + public async createClipOffset(media_id: string, object_uuid:string, type: string): Promise { // use engine to load based off media id and type animation | transform | } // this will update the state of the clips based off uuid easing? - async updateClip(clip_uuid: string, updates: AnyJson): Promise { + public async updateClip(clip_uuid: string, updates: AnyJson): Promise { } - async deleteClip(clip_uuid: string): Promise { + public async deleteClip(clip_uuid: string): Promise { } // Events that will trigger from react - async clipDidEnterDropZone() { + public async clipDidEnterDropZone() { } - async clipDidExitDropZone() { + public async clipDidExitDropZone() { } // timeline controls this. - async scrubberDidStart(offset_frame: number) { + public async scrubberDidStart(offset_frame: number) { } - async scrub(offset_frame: number): Promise { + public async scrub(offset_frame: number): Promise { // only stream through to the position and rotation keyframes // debounce not really } - async scrubberDidStop(offset_frame: number) { + public async scrubberDidStop(offset_frame: number) { } // public streaming events into the timeline from - async setScrubberPosition(offset: number) { + public async setScrubberPosition(offset: number) { this.scrubberPosition = offset // in ms } // should play from the clip that is closest to the to scrubber - async play(): Promise { + public async play(): Promise { console.log(`Starting Timeline`) this.isPlaying = true } - async reset_scene() { + private async reset_scene() { for (const element of this.timelineItems) { if (element.type == "transform") { let object = this.scene.get_object_by_uuid(element.object_uuid); @@ -136,7 +161,7 @@ export class TimeLine { } // called by the editor update loop on each frame - async update(deltatime:number) { + public async update(deltatime:number) { if (this.isPlaying == false) return; // start and stop if (this.scrubberPosition <= 0) { @@ -191,7 +216,7 @@ export class TimeLine { } } - async stop(): Promise { + private async stop(): Promise { this.isPlaying = false console.log(`Stopping Timeline`) } diff --git a/app/pages/PageEnigma/js/transform_engine.ts b/app/pages/PageEnigma/js/transform_engine.ts index c5eeb170b0..12c089f913 100644 --- a/app/pages/PageEnigma/js/transform_engine.ts +++ b/app/pages/PageEnigma/js/transform_engine.ts @@ -1,8 +1,8 @@ -import { TransformTrackClip } from "../datastructures/clips/transform_track_clip"; +import { TransformClip } from "../datastructures/clips/transform_clip"; import * as THREE from 'three'; class TransformEngine { - clips: { [key: string]: TransformTrackClip } = {}; + clips: { [key: string]: TransformClip } = {}; version: number; length: number; @@ -13,13 +13,13 @@ class TransformEngine { } loadObject(object_uuid: string, clip_length:number = 2) { - this.clips[object_uuid] = new TransformTrackClip(this.version, object_uuid, clip_length, object_uuid); // replace the last object_uuid with the media ID when its ready. + this.clips[object_uuid] = new TransformClip(this.version, object_uuid, clip_length, object_uuid); // replace the last object_uuid with the media ID when its ready. return this.clips[object_uuid] } addFrame(object: THREE.Object3D, clip_length:number = 2) { if(this.clips[object.uuid] == null) { - this.clips[object.uuid] = new TransformTrackClip(this.version, object.uuid, clip_length); + this.clips[object.uuid] = new TransformClip(this.version, object.uuid, clip_length); } this.clips[object.uuid].add_position(object.position); this.clips[object.uuid].add_rotation(new THREE.Vector3(object.rotation.x, object.rotation.y, object.rotation.z));