diff --git a/app/pages/PageEnigma/Editor/timeline.ts b/app/pages/PageEnigma/Editor/timeline.ts index 3f6287085..9d88ed9b5 100644 --- a/app/pages/PageEnigma/Editor/timeline.ts +++ b/app/pages/PageEnigma/Editor/timeline.ts @@ -283,20 +283,20 @@ export class TimeLine { }); this.addPlayableClip( - new ClipUI({ - version: data.version, - type: ClipType.FAKE, - group: ClipGroup.CHARACTER, - name: "Default", - media_id: media_id, - object_uuid: obj.uuid, - clip_uuid: obj.uuid, - object_name: name, - start_offset: 0, - ending_offset: 0, - keyframe_offset: 0, - media_file_type: obj.userData["media_file_type"], - }), + new ClipUI( + data.version, + ClipType.FAKE, + ClipGroup.CHARACTER, + "Default", + media_id, + obj.uuid, + obj.uuid, + name, + 0, + 0, + 0, + obj.userData["media_file_type"], + ), ); await this.editorEngine.sceneManager?.add_creation_undostack(obj); @@ -424,20 +424,20 @@ export class TimeLine { ); await this.addPlayableClip( - new ClipUI({ - version: data_json["version"], - type: ClipType.TRANSFORM, - group: data_json["group"], - name: object_name, - media_id: "", - clip_uuid: keyframe_uuid, - object_uuid: uuid, - object_name: object_name, - start_offset: 0, - ending_offset: data_json["offset"], - keyframe_offset: data_json["offset"], - media_file_type: MediaFileType.None, - }), + new ClipUI( + data_json["version"], + ClipType.TRANSFORM, + data_json["group"], + object_name, + "", + keyframe_uuid, + uuid, + object_name, + 0, + data_json["offset"], + data_json["offset"], + MediaFileType.None, + ), ); const point = this.scene.createPoint( @@ -484,8 +484,8 @@ export class TimeLine { const group = data.group; const version = 1; const type = data.type; - const offset = data.start_offset; - const end_offset = data.ending_offset + offset; + const offset = data.offset; + const end_offset = data.length + offset; const object_name = this.scene.get_object_by_uuid(object_uuid)?.name ?? "undefined"; const clip_uuid = data.clip_uuid; @@ -495,7 +495,7 @@ export class TimeLine { this.animation_engine.load_object(object_uuid, media_id, name); break; case "transform": - this.transform_engine.loadObject(object_uuid, data.ending_offset); + this.transform_engine.loadObject(object_uuid, data.length); break; case "expression": this.emotion_engine.loadClip(object_uuid, media_id); @@ -505,23 +505,22 @@ export class TimeLine { this.lipSync_engine.load_object(object_uuid, media_id); // media id for this as well it can be downloaded this.addPlayableClip( - new ClipUI({ - version: version, - type: ClipType.AUDIO, - group: ClipGroup.CHARACTER, - name: name, - media_id: media_id, - clip_uuid: clip_uuid, - object_uuid: object_uuid, - object_name: object_name, - start_offset: offset, - ending_offset: end_offset, - length: 0, // length - media_file_type: - this.scene.get_object_by_uuid(object_uuid)?.userData[ - "media_file_type" - ], - }), + new ClipUI( + version, + ClipType.AUDIO, + ClipGroup.CHARACTER, + name, + media_id, + clip_uuid, + object_uuid, + object_name, + offset, + end_offset, + 0, // length + this.scene.get_object_by_uuid(object_uuid)?.userData[ + "media_file_type" + ], + ), ); return; } else { @@ -532,23 +531,20 @@ export class TimeLine { // media id for this as well it can be downloaded this.addPlayableClip( - new ClipUI({ - version: version, - type: type, - group: group, - name: name, - media_id: media_id, - clip_uuid: clip_uuid, - object_uuid: object_uuid, - object_name: object_name, - start_offset: offset, - ending_offset: end_offset, // length - keyframe_offset: 0, - media_file_type: - this.scene.get_object_by_uuid(object_uuid)?.userData[ - "media_file_type" - ], - }), + new ClipUI( + version, + type, + group, + name, + media_id, + clip_uuid, + object_uuid, + object_name, + offset, + end_offset, // length + 0, + this.scene.get_object_by_uuid(object_uuid)?.userData["media_file_type"], + ), ); this.checkEditorCanPlay(); @@ -608,8 +604,8 @@ export class TimeLine { // only length and offset changes here. const object_uuid = data.object_uuid; const media_id = data.media_id; - const offset = data.start_offset; - const length = data.ending_offset + offset; + const offset = data.offset; + const length = data.length + offset; const clip_uuid = data.clip_uuid; for (const element of this.timeline_items) { @@ -618,8 +614,8 @@ export class TimeLine { element.object_uuid === object_uuid && element.clip_uuid == clip_uuid ) { - element.ending_offset = length; - element.start_offset = offset; + element.length = length; + element.offset = offset; } } this.checkEditorCanPlay(); @@ -722,8 +718,8 @@ export class TimeLine { public getEndPoint(): number { let longest = 0; for (const element of this.timeline_items) { - if (longest < element.ending_offset) { - longest = element.ending_offset; + if (longest < element.length) { + longest = element.length; } } return longest; @@ -776,8 +772,8 @@ export class TimeLine { //3. smallest unit is a frame and it is set by the scene and is in fps, our videos will be 60fps but we can reprocess them using the pipeline. for (const element of this.timeline_items) { if ( - element.start_offset <= this.scrubber_frame_position && - this.scrubber_frame_position <= element.ending_offset && + element.offset <= this.scrubber_frame_position && + this.scrubber_frame_position <= element.length && element.should_play ) { // run async @@ -788,11 +784,11 @@ export class TimeLine { if (object && this.transform_engine.clips[element.object_uuid]) { this.transform_engine.clips[element.object_uuid].step( object, - element.start_offset, + element.offset, this.scrubber_frame_position, this.scene, ); - element.ending_offset = + element.length = this.transform_engine.clips[element.object_uuid].length; } } else if ( @@ -812,7 +808,7 @@ export class TimeLine { await this.audio_engine.step( element.media_id, this.scrubber_frame_position, - element.start_offset, + element.offset, ); } else if ( element.type === ClipType.AUDIO && @@ -825,7 +821,7 @@ export class TimeLine { ].play(object); this.lipSync_engine.clips[ element.object_uuid + element.media_id - ].step(this.scrubber_frame_position, element.start_offset); + ].step(this.scrubber_frame_position, element.offset); } } else if (element.type === ClipType.ANIMATION) { if (object) { @@ -836,7 +832,7 @@ export class TimeLine { await this.animation_engine.clips[ object.uuid + element.media_id ].step( - (this.scrubber_frame_position - element.start_offset) / fps, + (this.scrubber_frame_position - element.offset) / fps, this.is_playing, this.scrubber_frame_position, // Double FPS for best result. ); @@ -846,7 +842,7 @@ export class TimeLine { if (object) { await this.emotion_engine.clips[ object.uuid + element.media_id - ].step(this.scrubber_frame_position - element.start_offset, object); + ].step(this.scrubber_frame_position - element.offset, object); } } //this.timelineItems = this.timelineItems.filter(item => item !== element) diff --git a/app/pages/PageEnigma/Editor/video_generation.ts b/app/pages/PageEnigma/Editor/video_generation.ts index 73c36ffe1..272953faf 100644 --- a/app/pages/PageEnigma/Editor/video_generation.ts +++ b/app/pages/PageEnigma/Editor/video_generation.ts @@ -128,8 +128,8 @@ export class VideoGeneration { const video_og: string = itteration + "tmp.mp4"; const wav_name: string = itteration + "tmp.wav"; const new_video: string = itteration + 1 + "tmp.mp4"; - let startFrame: number = clip.start_offset; - let endFrame: number = clip.ending_offset; + let startFrame: number = clip.offset; + let endFrame: number = clip.length; if (endFrame > this.editor.timeline.timeline_limit) { endFrame = this.editor.timeline.timeline_limit; diff --git a/app/pages/PageEnigma/datastructures/clips/clip_ui.ts b/app/pages/PageEnigma/datastructures/clips/clip_ui.ts index 924e4719b..883a8491f 100644 --- a/app/pages/PageEnigma/datastructures/clips/clip_ui.ts +++ b/app/pages/PageEnigma/datastructures/clips/clip_ui.ts @@ -2,7 +2,7 @@ // it is created from a media id. import { ClipGroup, ClipType, MediaFileType } from "~/pages/PageEnigma/enums"; -export interface ClipUiItem { +export class ClipUI { version: number; type: ClipType; group: ClipGroup; @@ -10,63 +10,27 @@ export interface ClipUiItem { media_id: string; object_uuid: string; object_name: string; - start_offset: number; // in frames - ending_offset: number; // in frames - should_play?: boolean; - clip_uuid: string; - keyframe_offset: number; - media_file_type: MediaFileType; -} -export interface ClipUiClassInterface extends ClipUiItem { - toJSON: () => ClipUiItem; -} - -export class ClipUI implements ClipUiClassInterface { - version: number; - type: ClipType; - group: ClipGroup; - name: string; - media_id: string; - object_uuid: string; - object_name: string; - start_offset: number; // in frames - ending_offset: number; // in frames + offset: number; // in frames + length: number; // in frames should_play: boolean; clip_uuid: string; keyframe_offset: number; media_file_type: MediaFileType; - constructor({ - version, - type, - group, - name, - media_id, - clip_uuid, - object_uuid, - object_name, - start_offset, - offset, - ending_offset, - length, - keyframe_offset = 0, - media_file_type, - }: { - version: number; - type: ClipType; - group: ClipGroup; - name: string; - media_id: string; - clip_uuid: string; - object_uuid: string; - object_name: string; - start_offset?: number; - offset?: number; // =start_offset, kept for now just for backwards compatibility - ending_offset?: number; - length?: number; // =ending_offset, kept for now just for backwards compatibility - keyframe_offset?: number; - media_file_type: MediaFileType; - }) { + constructor( + version: number, + type: ClipType, + group: ClipGroup, + name: string, + media_id: string, + clip_uuid: string, + object_uuid: string, + object_name: string, + offset: number, + length: number, + keyframe_offset: number = 0, + media_file_type: MediaFileType + ) { this.version = version; this.group = group; // Only needed for UI @@ -78,16 +42,14 @@ export class ClipUI implements ClipUiClassInterface { this.object_uuid = object_uuid; // Animation / Audio / Lipsync / : Engine this.object_name = object_name; this.media_id = media_id; // Animation / Audio / Lipsync / : Engine - this.start_offset = start_offset ? start_offset : offset ? offset : 0; - this.ending_offset = ending_offset ? ending_offset : length ? length : 0; - + this.offset = offset; + this.length = length; this.should_play = true; - this.keyframe_offset = keyframe_offset ?? 0; - + this.keyframe_offset = keyframe_offset; this.media_file_type = media_file_type; } - toJSON(): ClipUiItem { + toJSON(): any { return { version: this.version, group: this.group, @@ -97,10 +59,10 @@ export class ClipUI implements ClipUiClassInterface { object_uuid: this.object_uuid, object_name: this.object_name, media_id: this.media_id, - start_offset: this.start_offset, - ending_offset: this.ending_offset, + start_offset: this.offset, + ending_offset: this.length, keyframe_offset: this.keyframe_offset, - media_file_type: this.media_file_type, + media_file_type: this.media_file_type }; } } diff --git a/app/pages/PageEnigma/models/track.ts b/app/pages/PageEnigma/models/track.ts index 6e4d244cf..ee957de24 100644 --- a/app/pages/PageEnigma/models/track.ts +++ b/app/pages/PageEnigma/models/track.ts @@ -1,4 +1,4 @@ -// import { MediaFileAnimationType } from "~/enums"; +import { MediaFileAnimationType } from "~/enums"; import { XYZ } from "../datastructures/common"; import { ClipGroup, ClipType, MediaFileType } from "~/pages/PageEnigma/enums"; @@ -36,7 +36,7 @@ export interface CharacterTrack { object_uuid: string; media_id: string; mediaType: MediaFileType; - // animationType?: MediaFileAnimationType; + animationType: MediaFileAnimationType; name: string; muted: boolean; minimized: boolean; diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts index 97c048426..5991804c4 100644 --- a/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts @@ -4,10 +4,10 @@ import { ClipUI } from "../datastructures/clips/clip_ui"; import { EmotionClip } from "../datastructures/clips/emotion_clip"; import { LipSyncClip } from "../datastructures/clips/lipsync_clip"; import { TransformClip } from "../datastructures/clips/transform_clip"; -import { AnimationEngine } from "../Editor/animation_engine"; +import AnimationEngine from "../Editor/animation_engine"; import AudioEngine from "../Editor/audio_engine"; import EmotionEngine from "../Editor/emotion_engine"; -import { LipSyncEngine } from "../Editor/lip_sync_engine"; +import LipSyncEngine from "../Editor/lip_sync_engine"; import { TimeLine } from "../Editor/timeline"; import TransformEngine from "../Editor/transform_engine"; @@ -77,20 +77,20 @@ export class StoryTellerProxyTimeline { if (timeline_clips) { for (let index = 0; index < timeline_clips.length; index++) { const element = timeline_clips[index]; - let clip = new ClipUI({ - version: element.version, - type: element.type, - group: element.group, - name: element.name, - media_id: element.media_id, - clip_uuid: element.clip_uuid, - object_uuid: element.object_uuid, - object_name: element.object_name, - start_offset: element.start_offset, - ending_offset: element.ending_offset, - keyframe_offset: element.keyframe_offset, - media_file_type: element.media_file_type, - }); + let clip = new ClipUI( + element.version, + element.type, + element.group, + element.name, + element.media_id, + element.clip_uuid, + element.object_uuid, + element.object_name, + element.start_offset, + element.ending_offset, + element.keyframe_offset, + element.media_file_type + ); new_clips.push(clip); } } diff --git a/app/pages/PageEnigma/signals/audioGroup.ts b/app/pages/PageEnigma/signals/audioGroup.ts index 3a65fd93a..44489fa37 100644 --- a/app/pages/PageEnigma/signals/audioGroup.ts +++ b/app/pages/PageEnigma/signals/audioGroup.ts @@ -144,8 +144,8 @@ export function loadAudioData(item: ClipUI) { object_uuid: item.object_uuid, media_id: item.media_id, name: item.name, - offset: item.start_offset, - length: item.ending_offset, + offset: item.offset, + length: item.length, } as Clip; existingAudio.clips.push(newItem); existingAudio.clips.sort((clipA, clipB) => clipA.offset - clipB.offset); diff --git a/app/pages/PageEnigma/signals/characterGroups/loadCharacterData.ts b/app/pages/PageEnigma/signals/characterGroups/loadCharacterData.ts index ad40bce8a..ec9e51014 100644 --- a/app/pages/PageEnigma/signals/characterGroups/loadCharacterData.ts +++ b/app/pages/PageEnigma/signals/characterGroups/loadCharacterData.ts @@ -23,7 +23,7 @@ function getAddCharacter(item: ClipUI) { positionKeyframes: [], lipSyncClips: [], expressionClips: [], - mediaType: item.media_file_type, + mediaType: item.media_file_type } as CharacterTrack; characterGroup.value = { @@ -53,9 +53,9 @@ export function loadCharacterData(item: ClipUI) { object_uuid: item.object_uuid, media_id: item.media_id, name: item.name, - offset: item.start_offset, - length: item.ending_offset, - mediaType: item.media_file_type, + offset: item.offset, + length: item.length, + mediaType: item.media_file_type } as Clip; existingCharacter.animationClips.push(newItem); existingCharacter.animationClips.sort( @@ -71,8 +71,8 @@ export function loadCharacterData(item: ClipUI) { object_uuid: item.object_uuid, media_id: item.media_id, name: item.name, - offset: item.start_offset, - length: item.ending_offset, + offset: item.offset, + length: item.length, } as Clip; existingCharacter.expressionClips.push(newItem); existingCharacter.expressionClips.sort( @@ -101,8 +101,8 @@ export function loadCharacterData(item: ClipUI) { object_uuid: item.object_uuid, media_id: item.media_id, name: item.name, - offset: item.start_offset, - length: item.ending_offset, + offset: item.offset, + length: item.length, } as Clip; existingCharacter.lipSyncClips.push(newItem); existingCharacter.lipSyncClips.sort(