diff --git a/app/pages/PageEnigma/composable_engines/audio_engine.tsx b/app/pages/PageEnigma/composable_engines/audio_engine.tsx new file mode 100644 index 0000000000..e69de29bb2 diff --git a/app/pages/PageEnigma/composable_engines/lipsync_engine.tsx b/app/pages/PageEnigma/composable_engines/lipsync_engine.tsx new file mode 100644 index 0000000000..e69de29bb2 diff --git a/app/pages/PageEnigma/composable_engines/scene_format.tsx b/app/pages/PageEnigma/composable_engines/scene_format.tsx new file mode 100644 index 0000000000..e69de29bb2 diff --git a/app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts new file mode 100644 index 0000000000..4c4b41d2b5 --- /dev/null +++ b/app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts @@ -0,0 +1,41 @@ +export interface AnimationTrackClip { + version: number; + uuid: number; + media_id: number; // comes from the server + type: "animation"; + speed: number; + length: number; + clip_name: string; + } + +export class AnimationTrackClip implements AnimationTrackClip { + version: number; + uuid: number; + media_id: number; // comes from the server + type: "animation"; + speed: number; + length: number; + clip_name: string; + + constructor(version: number, uuid: number, media_id: number, speed: number, length: number, clip_name: string) { + this.version = version; + this.uuid = uuid; + this.media_id = media_id; + this.type = "animation"; + this.speed = speed; + this.length = length; + this.clip_name = clip_name; + } + + toJSON(): string { + return JSON.stringify({ + version: this.version, + uuid: this.uuid, + media_id: this.media_id, + type: this.type, + speed: this.speed, + length: this.length, + clip_name: this.clip_name, + }); + } + } \ No newline at end of file diff --git a/app/pages/PageEnigma/datastructures/clips/audio_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/audio_track_clip.ts new file mode 100644 index 0000000000..7a3fd172c0 --- /dev/null +++ b/app/pages/PageEnigma/datastructures/clips/audio_track_clip.ts @@ -0,0 +1,33 @@ +export interface AudioTrackClip { + version: number; + uuid: number; + media_id: number; + type: "audio"; + volume: number; +} + +export class AudioTrackClipClass implements AudioTrackClip { + version: number; + uuid: number; + media_id: number; + type: "audio"; + volume: number; + + constructor(version: number, uuid: number, media_id: number, volume: number) { + this.version = version; + this.uuid = uuid; + this.media_id = media_id; + this.type = "audio"; + this.volume = volume; + } + + toJSON(): string { + return JSON.stringify({ + version: this.version, + uuid: this.uuid, + media_id: this.media_id, + type: this.type, + volume: this.volume, + }); + } +} diff --git a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts new file mode 100644 index 0000000000..11b07b62d9 --- /dev/null +++ b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts @@ -0,0 +1,34 @@ +export interface ClipOffset { + version: number; + type: "transform" | "audio" | "animation"; + clip_uuid: number; + start_offset: number; // in milliseconds (ms) +} + +export class ClipOffset implements ClipOffset { + version: number; + type: "transform" | "audio" | "animation"; + clip_uuid: number; + start_offset: number; // in milliseconds (ms) + + constructor( + version: number, + type: "transform" | "audio" | "animation", + clip_uuid: number, + start_offset: number, + ) { + this.version = version; + this.type = type; + this.clip_uuid = clip_uuid; + this.start_offset = start_offset; + } + + toJSON(): string { + return JSON.stringify({ + version: this.version, + type: this.type, + clip_uuid: this.clip_uuid, + start_offset: this.start_offset, + }); + } +} diff --git a/app/pages/PageEnigma/datastructures/clips/transform_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/transform_track_clip.ts new file mode 100644 index 0000000000..f7e5960db5 --- /dev/null +++ b/app/pages/PageEnigma/datastructures/clips/transform_track_clip.ts @@ -0,0 +1,47 @@ +export interface Vector3 { + x: number; + y: number; + z: number; +} + +export interface TransformTrackClip { + version: number; + uuid: number; + media_id: number; + type: "transform"; + position: Vector3[]; + rotation: Vector3[]; + scale: Vector3[]; +} + +export class TransformTrackClipClass implements TransformTrackClip { + version: number; + uuid: number; + media_id: number; + type: "transform"; + position: Vector3[]; + rotation: Vector3[]; + scale: Vector3[]; + + constructor(version: number, uuid: number, media_id: number, position: Vector3[], rotation: Vector3[], scale: Vector3[]) { + this.version = version; + this.uuid = uuid; + this.media_id = media_id; + this.type = "transform"; + this.position = position; + this.rotation = rotation; + this.scale = scale; + } + + toJSON(): string { + return JSON.stringify({ + version: this.version, + uuid: this.uuid, + media_id: this.media_id, + type: this.type, + position: this.position, + rotation: this.rotation, + scale: this.scale, + }); + } + } diff --git a/app/pages/PageEnigma/datastructures/scene/scene_object.ts b/app/pages/PageEnigma/datastructures/scene/scene_object.ts new file mode 100644 index 0000000000..9f6cbffe9d --- /dev/null +++ b/app/pages/PageEnigma/datastructures/scene/scene_object.ts @@ -0,0 +1,99 @@ +import { AudioTrackClip } from "../clips/audio_track_clip"; +import { AnimationTrackClip } from "../clips/animation_track_clip"; +import { TransformTrackClip } from "../clips/transform_track_clip"; +import { ClipOffset } from "../clips/clip_offset"; + +export interface Scene { + version: number; + glb_media_id: number; // Assuming a numerical ID, adjust as needed + skybox_type: "cube" | "six_sided_cube" | "sphere"; + skybox_media_ids: { [key: string]: number }; + timeline: Timeline; +} + +export interface Timeline { + version: number; + last_scrubber_position: number; + audio_track_clips: AudioTrackClip[]; // Assuming defined elsewhere + transform_track_clips: TransformTrackClip[]; // Placeholder, define as needed + animation_track_clips: AnimationTrackClip[]; // Placeholder, define as needed + entities: Entity[]; + globalAudio: GlobalAudio; + camera: Camera; +} + +export interface Entity { + object_uuid: number; + clip_offsets: ClipOffset[]; +} + +export interface GlobalAudio { + clip_offsets: ClipOffset[]; +} + +export interface Camera { + clip_offsets: ClipOffset[]; +} + +export class Scene implements Scene { + version: number; + glb_media_id: number; + skybox_type: "cube" | "six_sided_cube" | "sphere"; + skybox_media_ids: { [key: string]: number }; + timeline: Timeline; + + constructor( + version: number, + glb_media_id: number, + skybox_type: "cube" | "six_sided_cube" | "sphere", + skybox_image_ids: { [key: string]: number }, + timeline: Timeline, + ) + { + this.version = version; + this.glb_media_id = glb_media_id; + this.skybox_type = skybox_type; + this.skybox_media_ids = skybox_image_ids; + this.timeline = timeline; + } + + toJSON(): string { + return JSON.stringify(this); + } +} + +export class Timeline implements Timeline { + version: number; + last_scrubber_position: number; + audio_track_clips: AudioTrackClip[]; + transform_track_clips: TransformTrackClip[]; + animation_track_clips: AnimationTrackClip[]; + entities: Entity[]; + globalAudio: GlobalAudio; + camera: Camera; + + constructor( + version: number, + last_scrubber_position: number, + audio_track_clips: AudioTrackClip[], + transform_track_clips: TransformTrackClip[], + animation_track_clips: AnimationTrackClip[], + entities: Entity[], + globalAudio: GlobalAudio, + camera: Camera, + ) + { + this.version = version; + this.last_scrubber_position = last_scrubber_position; + this.audio_track_clips = audio_track_clips; + this.transform_track_clips = transform_track_clips; + this.animation_track_clips = animation_track_clips; + this.entities = entities; + this.globalAudio = globalAudio; + this.camera = camera; + } + + toJSON(): string { + return JSON.stringify(this); + } +} diff --git a/app/pages/PageEnigma/js/animated_item.js b/app/pages/PageEnigma/js/animated_item.js index dafed19bc9..387bdc713e 100644 --- a/app/pages/PageEnigma/js/animated_item.js +++ b/app/pages/PageEnigma/js/animated_item.js @@ -1,6 +1,5 @@ import * as THREE from 'three'; import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'; -import { FBXLoader } from 'three/addons/loaders/FBXLoader'; import { LipSync } from './lipsync.js'; class AnimatedItem { @@ -67,16 +66,19 @@ class AnimatedItem { if (this.face != null && this.auto_blink) { this.blink(delta); if (this.mixer != null) { - this.mixer.update(delta); + this.mixer.update(delta); // add speed here later. } } } + // 3js clip animate(clip) { this.currentAction = this.mixer.clipAction(clip); this.currentAction.play(); } + // talking and plays the audio + // takes audio clip from folder on the server .wav sync_lips(filepath) { if(this.lipsync_comp != null){ return fetch(filepath) @@ -91,6 +93,7 @@ class AnimatedItem { } } + // Takes a glb animation loads from the server load_animation(filepath, callback = null) { let glbLoader = new GLTFLoader(); glbLoader.load(filepath, @@ -108,6 +111,7 @@ class AnimatedItem { ); } + // loads the mesh of the character load_glb(filepath, callback = null) { let glbLoader = new GLTFLoader(); glbLoader.load(filepath, (glb) => { diff --git a/app/pages/PageEnigma/js/api_manager.js b/app/pages/PageEnigma/js/api_manager.ts similarity index 81% rename from app/pages/PageEnigma/js/api_manager.js rename to app/pages/PageEnigma/js/api_manager.ts index 16497c260e..0bfc90e8ce 100644 --- a/app/pages/PageEnigma/js/api_manager.js +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -1,16 +1,23 @@ import { v4 as uuidv4 } from 'uuid'; +import { Scene } from '../datastructures/scene/scene_object'; + class MediaUploadManager { + baseUrl:String + constructor() { - //this.baseUrl = "https://api.fakeyou.com"; - this.baseUrl = "http://localhost:12345" + this.baseUrl = "https://api.fakeyou.com"; + //this.baseUrl = "http://localhost:12345" + } + + + async saveScene(scene:Scene) { + } - async uploadMedia(blob, fileName) { + async uploadMedia(blob:any, fileName:string) { const url = `${this.baseUrl}/v1/media_uploads/upload`; - let uuid = uuidv4(); - const formData = new FormData(); formData.append('uuid_idempotency_token', uuid); formData.append('file', blob, fileName); @@ -33,14 +40,12 @@ class MediaUploadManager { return response.json(); // or handle the response as appropriate } - async uploadGLB(blob) { + async uploadGLB(file:File) { const url = `${this.baseUrl}/v1/media_files/upload/engine_asset`; - let uuid = uuidv4(); - const formData = new FormData(); formData.append('uuid_idempotency_token', uuid); - formData.append('file', blob); + formData.append('file', file); formData.append('source','file'); formData.append('media_file_subtype', 'scene_import'); formData.append('media_file_class', 'scene'); @@ -56,7 +61,6 @@ class MediaUploadManager { if (!response.ok) { throw new Error('Failed to Send Data'); } - return response.json(); // or handle the response as appropriate } } diff --git a/app/pages/PageEnigma/js/audio_manager.js b/app/pages/PageEnigma/js/audio_engine.ts similarity index 97% rename from app/pages/PageEnigma/js/audio_manager.js rename to app/pages/PageEnigma/js/audio_engine.ts index c516504437..71eb36e906 100644 --- a/app/pages/PageEnigma/js/audio_manager.js +++ b/app/pages/PageEnigma/js/audio_engine.ts @@ -1,6 +1,6 @@ import * as THREE from "three"; -class AudioManager { +class AudioEngine { constructor() { this.listener = new THREE.AudioListener(); this.clips = {}; diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts new file mode 100644 index 0000000000..c4ced3da0e --- /dev/null +++ b/app/pages/PageEnigma/js/timeline.ts @@ -0,0 +1,106 @@ +// Write unsubcribe and run the remainder of the clip tech. + +// export class Clip { +// version: number; +// clip_id: number; + +// media_token_id: number; +// type: 'animation' | 'audio' | 'position_rotation'; +// time_start_offset: number; +// length_override: number | null; + +// constructor( +// version: number, +// clip_id: number, +// media_token_id: number, +// type: 'animation' | 'audio' | 'position_rotation', +// time_start_offset: number, +// length_override: number | null +// ) { +// this.version = version; +// this.clip_id = clip_id; +// this.media_token_id = media_token_id; +// this.type = type; +// this.time_start_offset = time_start_offset; +// this.length_override = length_override; +// } +// } + + +// export class Clip2 { +// name: string +// startOffset: number +// timerID: NodeJS.Timeout | null +// lengthOfClip: number +// scrubberEndingPosition: number + +// constructor(name: string, startOffset: number, lengthOfClip: number) { +// this.startOffset = startOffset +// this.lengthOfClip = lengthOfClip +// this.name = name +// this.timerID = null + +// this.scrubberEndingPosition = 100 +// } + +// async play(): Promise { +// console.log(`Playing ${this.name}`) +// this.timerID = setInterval(async () => { +// console.log(`Running async ${this.name}`) +// this.stop() +// }, this.lengthOfClip) +// } + +// async stop(): Promise { +// clearInterval(this.timerID!) +// console.log(`Stopping ${this.name}`) +// } +// } + +// export class API2 { +// timelineItems: Clip[] +// timerID: NodeJS.Timeout | null +// timeLineLimit: number + +// runningClips: Clip[] + +// constructor() { +// this.timelineItems = [] +// this.timerID = null +// this.timeLineLimit = 1000 * 10 +// this.runningClips = [] +// } + +// async addPlayableClip(clip: Clip): Promise { +// this.timelineItems.push(clip) +// } + +// async start(): Promise { +// let start = 0 +// const updateInterval = 100 +// const timerID = setInterval(async () => { +// console.log(`Current Time:${start}`) + +// // get last updated time when stopped +// for (const element of this.timelineItems) { +// if (element.startOffset >= start) { +// element.play() +// // remove the element +// this.timelineItems = this.timelineItems.filter(item => item !== element) +// } +// } +// if (start == this.timeLineLimit || this.timelineItems.length == 0) { +// this.stop() +// } + +// start += updateInterval +// }, updateInterval) + +// this.timerID = timerID +// } + +// async stop(): Promise { +// console.log(`Stopping Timeline`) +// clearInterval(this.timerID!) +// } +// } diff --git a/app/pages/PageEnigma/js/timeline.tsx b/app/pages/PageEnigma/js/timeline.tsx deleted file mode 100644 index 0d42d84a96..0000000000 --- a/app/pages/PageEnigma/js/timeline.tsx +++ /dev/null @@ -1,80 +0,0 @@ -// Write unsubcribe and run the remainder of the clip tech. - - -export class Clip2 { - name: string - startOffset: number - timerID: NodeJS.Timeout | null - lengthOfClip: number - scrubberEndingPosition: number - - constructor(name: string, startOffset: number, lengthOfClip: number) { - this.startOffset = startOffset - this.lengthOfClip = lengthOfClip - this.name = name - this.timerID = null - - this.scrubberEndingPosition = 100 - } - - async play(): Promise { - console.log(`Playing ${this.name}`) - this.timerID = setInterval(async () => { - console.log(`Running async ${this.name}`) - this.stop() - }, this.lengthOfClip) - } - - async stop(): Promise { - clearInterval(this.timerID!) - console.log(`Stopping ${this.name}`) - } -} - -export class API2 { - timelineItems: Clip[] - timerID: NodeJS.Timeout | null - timeLineLimit: number - - runningClips: Clip[] - - constructor() { - this.timelineItems = [] - this.timerID = null - this.timeLineLimit = 1000 * 10 - this.runningClips = [] - } - - async addPlayableClip(clip: Clip): Promise { - this.timelineItems.push(clip) - } - - async start(): Promise { - let start = 0 - const updateInterval = 100 - const timerID = setInterval(async () => { - console.log(`Current Time:${start}`) - - // get last updated time when stopped - for (const element of this.timelineItems) { - if (element.startOffset >= start) { - element.play() - // remove the element - this.timelineItems = this.timelineItems.filter(item => item !== element) - } - } - if (start == this.timeLineLimit || this.timelineItems.length == 0) { - this.stop() - } - - start += updateInterval - }, updateInterval) - - this.timerID = timerID - } - - async stop(): Promise { - console.log(`Stopping Timeline`) - clearInterval(this.timerID!) - } -} diff --git a/test/test_bench.tsx b/test/test_bench.tsx index 1714624864..158ad05afc 100644 --- a/test/test_bench.tsx +++ b/test/test_bench.tsx @@ -1,18 +1,15 @@ // Write unsubcribe and run the remainder of the clip tech. -class Clip { +class TrackClip { name: string - startOffset: number + startOffset: number // ms in the 0 100 | 10 second clip timerID: NodeJS.Timeout | null lengthOfClip: number - scrubberEndingPosition: number constructor(name: string, startOffset: number, lengthOfClip: number) { this.startOffset = startOffset this.lengthOfClip = lengthOfClip this.name = name this.timerID = null - - this.scrubberEndingPosition = 100 } async play(): Promise { @@ -29,12 +26,12 @@ class Clip { } } -class API { - timelineItems: Clip[] +class TimeLine { + timelineItems: TrackClip[] timerID: NodeJS.Timeout | null timeLineLimit: number - runningClips: Clip[] + runningClips: TrackClip[] constructor() { this.timelineItems = [] @@ -43,11 +40,11 @@ class API { this.runningClips = [] } - async addPlayableClip(clip: Clip): Promise { + async addPlayableClip(clip: TrackClip): Promise { this.timelineItems.push(clip) } - async start(): Promise { + async play(): Promise { let start = 0 const updateInterval = 100 const timerID = setInterval(async () => { @@ -82,34 +79,34 @@ const percision = 100 // Visual verification tests. function CheckIfBasicClipWorks() { - const api = new API() - api.addPlayableClip(new Clip("clip1", 0, 1000)) - api.start() + const api = new TimeLine() + api.addPlayableClip(new TrackClip("clip1", 0, 1000)) + api.play() } function CheckIfBasicClipWorks3SecondsAfterTimelineStops() { - const api = new API() - api.addPlayableClip(new Clip("clip1", 0, 10000)) - api.start() + const api = new TimeLine() + api.addPlayableClip(new TrackClip("clip1", 0, 10000)) + api.play() } function CheckIfTwoClipsAtTheSameTimeWorks() { - const api = new API() - api.addPlayableClip(new Clip("clip1", 0, 1000)) - api.start() + const api = new TimeLine() + api.addPlayableClip(new TrackClip("clip1", 0, 1000)) + api.play() } function CheckIfTwoClipsOneAfterAnotherWorks() { - const api = new API() - api.addPlayableClip(new Clip("clip1", 0, 1000)) - api.addPlayableClip(new Clip("clip2", 0, 2000)) - api.start() + const api = new TimeLine() + api.addPlayableClip(new TrackClip("clip1", 0, 1000)) + api.addPlayableClip(new TrackClip("clip2", 0, 2000)) + api.play() } function CheckIfTimeLineStopBeforeClipPlays() { - const api = new API() - api.addPlayableClip(new Clip("clip3",0,1000)) - api.start() + const api = new TimeLine() + api.addPlayableClip(new TrackClip("clip3",0,1000)) + api.play() setInterval(async ()=> { api.stop() },1100) @@ -117,9 +114,9 @@ function CheckIfTimeLineStopBeforeClipPlays() { } function CheckIfTimeLineStartAfterClipPlays() { - const api = new API() - api.addPlayableClip(new Clip("clip3",0,1000)) - api.start() + const api = new TimeLine() + api.addPlayableClip(new TrackClip("clip3",0,1000)) + api.play() setInterval(async ()=> { api.stop() },500) @@ -127,25 +124,25 @@ function CheckIfTimeLineStartAfterClipPlays() { } function CheckIfClipsPlayAllTogetherConcurrently() { - const api = new API() - api.addPlayableClip(new Clip("clip1",0,1000)) - api.addPlayableClip(new Clip("clip2",0,1000)) - api.addPlayableClip(new Clip("clip3",0,1000)) - api.addPlayableClip(new Clip("clip4",0,1000)) - api.start() + const api = new TimeLine() + api.addPlayableClip(new TrackClip("clip1",0,1000)) + api.addPlayableClip(new TrackClip("clip2",0,1000)) + api.addPlayableClip(new TrackClip("clip3",0,1000)) + api.addPlayableClip(new TrackClip("clip4",0,1000)) + api.play() } function CheckIfClipsPlayAllTogether() { - const api = new API() - api.addPlayableClip(new Clip("clip1",0,1000)) - api.addPlayableClip(new Clip("clip2",0,1000)) - api.addPlayableClip(new Clip("clip3",0,1000)) - api.addPlayableClip(new Clip("clip4",0,1000)) - api.start() + const api = new TimeLine() + api.addPlayableClip(new TrackClip("clip1",0,1000)) + api.addPlayableClip(new TrackClip("clip2",0,1000)) + api.addPlayableClip(new TrackClip("clip3",0,1000)) + api.addPlayableClip(new TrackClip("clip4",0,1000)) + api.play() } CheckIfBasicClipWorks() -CheckIfTwoClipsAtTheSameTimeWorks() +//CheckIfTwoClipsAtTheSameTimeWorks() // CheckIfTwoClipsOneAfterAnotherWorks() // CheckIfTimeLineStopBeforeClipPlays() // CheckIfTimeLineStartAfterClipPlays()