From 0067acea6e7706304d2621bddef0461b2b0f0ca8 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Sat, 23 Mar 2024 22:33:25 -0400 Subject: [PATCH] Ok clips updated to include objects Tracks --- .../clips/animation_track_clip.ts | 5 + .../datastructures/clips/clip_offset.ts | 38 +++-- app/pages/PageEnigma/js/timeline.ts | 131 ++++++++++-------- 3 files changed, 107 insertions(+), 67 deletions(-) diff --git a/app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts b/app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts index 7894d33f6c..d556db53d0 100644 --- a/app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts @@ -1,6 +1,7 @@ export interface AnimationTrackClip { version: number media_id: number + object_uuid: number type: "animation" location: "glb" | "remote" speed: number @@ -11,6 +12,7 @@ export interface AnimationTrackClip { export class AnimationTrackClip implements AnimationTrackClip { version: number media_id: number // comes from the server + object_uuid: number type: "animation" location: "glb" | "remote" speed: number @@ -21,6 +23,7 @@ export class AnimationTrackClip implements AnimationTrackClip { version: number, media_id: number, location: "glb" | "remote", + object_uuid: number, speed: number, length: number, clip_name: string, @@ -28,6 +31,7 @@ export class AnimationTrackClip implements AnimationTrackClip { this.version = version this.media_id = media_id this.type = "animation" + this.object_uuid = object_uuid this.location = location this.speed = speed this.length = length @@ -38,6 +42,7 @@ export class AnimationTrackClip implements AnimationTrackClip { return JSON.stringify({ version: this.version, media_id: this.media_id, + object_uuid: this.object_uuid, type: this.type, speed: this.speed, length: this.length, diff --git a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts index a6e2453c32..4b14bbe1d8 100644 --- a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts +++ b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts @@ -1,34 +1,46 @@ export interface ClipOffset { - version: number; - type: "transform" | "audio" | "animation"; - media_id: number; - start_offset: number; // in milliseconds (ms) + version: number + type: "transform" | "audio" | "animation" + media_id: number + object_uuid: number + start_offset: number // in frames so 60 frames per second on the timeline. + ending_offset: number // ending offset } +// Clip offsets represent the state of the clip on the timeline as well as what type of clip it is. +// it is created from a media id. export class ClipOffset implements ClipOffset { - version: number; - type: "transform" | "audio" | "animation"; - media_id: number; - start_offset: number; // in milliseconds (ms) + version: number + type: "transform" | "audio" | "animation" + media_id: number + object_uuid: number + start_offset: number // in frames + ending_offset: number // in frames constructor( version: number, type: "transform" | "audio" | "animation", media_id: number, + object_uuid: number, start_offset: number, + ending_offset: number, ) { - this.version = version; - this.type = type; - this.media_id = media_id; - this.start_offset = start_offset; + this.version = version + this.type = type + this.object_uuid = object_uuid + this.media_id = media_id + this.start_offset = start_offset + this.ending_offset = ending_offset } toJSON(): string { return JSON.stringify({ version: this.version, + object_uuid: this.object_uuid, type: this.type, media_id: this.media_id, start_offset: this.start_offset, - }); + ending_offset: this.ending_offset + }) } } diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 036d705ee8..fa11d29d7e 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -1,5 +1,6 @@ import { AnyJson } from "three/examples/jsm/nodes/core/constants.js" import { ClipOffset } from "../datastructures/clips/clip_offset" +import AudioEngine from "./audio_engine" // Every object uuid / entity has a track. export class TimelineCurrentReactState { @@ -22,34 +23,44 @@ export class TimeLine { isPlaying: boolean timelineState: TimelineCurrentReactState + + audioEngine: AudioEngine + // animation engine + // lip sync engine + // transform engine - constructor() { + constructor(audioEngine:AudioEngine) { this.timelineItems = [] - // this.timerID = null this.timeLineLimit = 1000 * 10 // 10 seconds this.runningClips = [] this.isPlaying = false this.scrubberPosition = 0 // in ms into the tl this.timelineState = new TimelineCurrentReactState() - } - - async createClipOffset(clip: ClipOffset): Promise { + // this will be used to play the audio clips + this.audioEngine = audioEngine } async addPlayableClip(clip: ClipOffset): Promise { this.timelineItems.push(clip) } + // when given a media id item it will create the clip. + // Then the clip will be loaded by the engines, if they come from outside of the loaded scene. + async createClipOffset(media_id: string,type: string): Promise { + // use engine to load based off media id and type animation | transform | + } + + // this will update the state of the clips based off uuid easing? + async updateClip(clip_uuid: string, updates: AnyJson): Promise { + + } + async deleteClip(clip_uuid: string): Promise { } - // this will update the state of the clips based off uuid easing? - async modifyClip(clip_uuid: string, updates: AnyJson): Promise { - - } - + // Events that will trigger from react async clipDidEnterDropZone() { } @@ -59,15 +70,16 @@ export class TimeLine { } // timeline controls this. - async scrubberDidStart(offset_in_ms:number) { + async scrubberDidStart(offset_frame:number) { } - async scrub(offset_in_ms:number): Promise { + async scrub(offset_frame:number): Promise { // only stream through to the position and rotation keyframes // debounce not really } - async scrubberDidStop(offset_in_ms:number) { + + async scrubberDidStop(offset_frame:number) { } // public streaming events into the timeline from @@ -106,7 +118,9 @@ export class TimeLine { // called by the editor update loop on each frame async update() { if (this.isPlaying == false) return; // start and stop - + + //2. allow stopping. + //3. smallest unit is a frame and it is set by the scene and is in fps, our videos will be 60fps but we can reprocess them using the pipeline. for (const element of this.timelineItems) { if (element.start_offset >= this.scrubberPosition) { // run async @@ -121,10 +135,13 @@ export class TimeLine { else if (element.type == "animation") { } else { - throw "Error New Type" + this.stop() + throw "Error New Type of element in the timeline" } this.timelineItems = this.timelineItems.filter(item => item !== element) } + + // find the offset of the longest clip and play until that clip is done if (this.scrubberPosition == this.timeLineLimit) { // stops at where clips should // cannot throw clip this.stop() } @@ -140,80 +157,86 @@ export class TimeLine { // How much timeline precision we have const percision = 100 // using a fake update loop mock +class AudioEngineMock { + async play(media_id:string) { + console.log("Audio Playing {media_id}") + } +} + // Visual verification tests. function CheckIfBasicAudioClipWorks() { - const api = new TimeLine() - api.addPlayableClip(new ClipOffset(1.0,'audio',1,0)) - api.play() + const timeline = new TimeLine() + timeline.addPlayableClip(new ClipOffset(1.0,'audio',1,0)) + timeline.play() } function CheckIfBasicTransformClipWorks() { - const api = new TimeLine() - api.addPlayableClip(new ClipOffset(1.0,'transform',2,0)) - api.play() + const timeline = new TimeLine() + timeline.addPlayableClip(new ClipOffset(1.0,'transform',2,0)) + timeline.play() } function CheckIfBasicClipWorks() { - const api = new TimeLine() - api.addPlayableClip(new ClipOffset(1.0,'animation',3,0)) - api.play() + const timeline = new TimeLine() + timeline.addPlayableClip(new ClipOffset(1.0,'animation',3,0)) + timeline.play() } // function CheckIfBasicClipWorks3SecondsAfterTimelineStops() { -// const api = new TimeLine() -// api.addPlayableClip(new ClipOffset("clip1", 0, 10000)) -// api.play() +// const timeline = new TimeLine() +// timeline.addPlayableClip(new ClipOffset("clip1", 0, 10000)) +// timeline.play() // } // function CheckIfTwoClipsAtTheSameTimeWorks() { -// const api = new TimeLine() -// api.addPlayableClip(new ClipOffset("clip1", 0, 1000)) -// api.play() +// const timeline = new TimeLine() +// timeline.addPlayableClip(new ClipOffset("clip1", 0, 1000)) +// timeline.play() // } // function CheckIfTwoClipsOneAfterAnotherWorks() { -// const api = new TimeLine() -// api.addPlayableClip(new ClipOffset("clip1", 0, 1000)) -// api.addPlayableClip(new ClipOffset("clip2", 0, 2000)) -// api.play() +// const timeline = new TimeLine() +// timeline.addPlayableClip(new ClipOffset("clip1", 0, 1000)) +// timeline.addPlayableClip(new ClipOffset("clip2", 0, 2000)) +// timeline.play() // } // function CheckIfTimeLineStopBeforeClipPlays() { -// const api = new TimeLine() -// api.addPlayableClip(new ClipOffset("clip3",0,1000)) -// api.play() +// const timeline = new TimeLine() +// timeline.addPlayableClip(new ClipOffset("clip3",0,1000)) +// timeline.play() // setInterval(async ()=> { -// api.stop() +// timeline.stop() // },1100) // console.log("Stopped") // } // function CheckIfTimeLineStartAfterClipPlays() { -// const api = new TimeLine() -// api.addPlayableClip(new ClipOffset("clip3",0,1000)) -// api.play() +// const timeline = new TimeLine() +// timeline.addPlayableClip(new ClipOffset("clip3",0,1000)) +// timeline.play() // setInterval(async ()=> { -// api.stop() +// timeline.stop() // },500) // console.log("Stopped") // } // function CheckIfClipsPlayAllTogetherConcurrently() { -// const api = new TimeLine() -// api.addPlayableClip(new ClipOffset("clip1",0,1000)) -// api.addPlayableClip(new ClipOffset("clip2",0,1000)) -// api.addPlayableClip(new ClipOffset("clip3",0,1000)) -// api.addPlayableClip(new ClipOffset("clip4",0,1000)) -// api.play() +// const timeline = new TimeLine() +// timeline.addPlayableClip(new ClipOffset("clip1",0,1000)) +// timeline.addPlayableClip(new ClipOffset("clip2",0,1000)) +// timeline.addPlayableClip(new ClipOffset("clip3",0,1000)) +// timeline.addPlayableClip(new ClipOffset("clip4",0,1000)) +// timeline.play() // } // function CheckIfClipsPlayAllTogether() { -// const api = new TimeLine() -// api.addPlayableClip(new TrackClip("clip1",0,1000)) -// api.addPlayableClip(new TrackClip("clip2",0,1000)) -// api.addPlayableClip(new TrackClip("clip3",0,1000)) -// api.addPlayableClip(new TrackClip("clip4",0,1000)) -// api.play() +// const timeline = new TimeLine() +// timeline.addPlayableClip(new TrackClip("clip1",0,1000)) +// timeline.addPlayableClip(new TrackClip("clip2",0,1000)) +// timeline.addPlayableClip(new TrackClip("clip3",0,1000)) +// timeline.addPlayableClip(new TrackClip("clip4",0,1000)) +// timeline.play() // } CheckIfBasicClipWorks()