revert clipui maintenance (#652)

Co-authored-by: wil <wil@storyteller.ai>

app/pages/PageEnigma/Editor/timeline.ts

@@ -283,20 +283,20 @@ export class TimeLine {
       });
 
       this.addPlayableClip(
-        new ClipUI({
-          version: data.version,
-          type: ClipType.FAKE,
-          group: ClipGroup.CHARACTER,
-          name: "Default",
-          media_id: media_id,
-          object_uuid: obj.uuid,
-          clip_uuid: obj.uuid,
-          object_name: name,
-          start_offset: 0,
-          ending_offset: 0,
-          keyframe_offset: 0,
-          media_file_type: obj.userData["media_file_type"],
-        }),
+        new ClipUI(
+          data.version,
+          ClipType.FAKE,
+          ClipGroup.CHARACTER,
+          "Default",
+          media_id,
+          obj.uuid,
+          obj.uuid,
+          name,
+          0,
+          0,
+          0,
+          obj.userData["media_file_type"],
+        ),
       );
 
       await this.editorEngine.sceneManager?.add_creation_undostack(obj);
@@ -424,20 +424,20 @@ export class TimeLine {
     );
 
     await this.addPlayableClip(
-      new ClipUI({
-        version: data_json["version"],
-        type: ClipType.TRANSFORM,
-        group: data_json["group"],
-        name: object_name,
-        media_id: "",
-        clip_uuid: keyframe_uuid,
-        object_uuid: uuid,
-        object_name: object_name,
-        start_offset: 0,
-        ending_offset: data_json["offset"],
-        keyframe_offset: data_json["offset"],
-        media_file_type: MediaFileType.None,
-      }),
+      new ClipUI(
+        data_json["version"],
+        ClipType.TRANSFORM,
+        data_json["group"],
+        object_name,
+        "",
+        keyframe_uuid,
+        uuid,
+        object_name,
+        0,
+        data_json["offset"],
+        data_json["offset"],
+        MediaFileType.None,
+      ),
     );
 
     const point = this.scene.createPoint(
@@ -484,8 +484,8 @@ export class TimeLine {
     const group = data.group;
     const version = 1;
     const type = data.type;
-    const offset = data.start_offset;
-    const end_offset = data.ending_offset + offset;
+    const offset = data.offset;
+    const end_offset = data.length + offset;
     const object_name =
       this.scene.get_object_by_uuid(object_uuid)?.name ?? "undefined";
     const clip_uuid = data.clip_uuid;
@@ -495,7 +495,7 @@ export class TimeLine {
         this.animation_engine.load_object(object_uuid, media_id, name);
         break;
       case "transform":
-        this.transform_engine.loadObject(object_uuid, data.ending_offset);
+        this.transform_engine.loadObject(object_uuid, data.length);
         break;
       case "expression":
         this.emotion_engine.loadClip(object_uuid, media_id);
@@ -505,23 +505,22 @@ export class TimeLine {
           this.lipSync_engine.load_object(object_uuid, media_id);
           // media id for this as well it can be downloaded
           this.addPlayableClip(
-            new ClipUI({
-              version: version,
-              type: ClipType.AUDIO,
-              group: ClipGroup.CHARACTER,
-              name: name,
-              media_id: media_id,
-              clip_uuid: clip_uuid,
-              object_uuid: object_uuid,
-              object_name: object_name,
-              start_offset: offset,
-              ending_offset: end_offset,
-              length: 0, // length
-              media_file_type:
-                this.scene.get_object_by_uuid(object_uuid)?.userData[
-                  "media_file_type"
-                ],
-            }),
+            new ClipUI(
+              version,
+              ClipType.AUDIO,
+              ClipGroup.CHARACTER,
+              name,
+              media_id,
+              clip_uuid,
+              object_uuid,
+              object_name,
+              offset,
+              end_offset,
+              0, // length
+              this.scene.get_object_by_uuid(object_uuid)?.userData[
+                "media_file_type"
+              ],
+            ),
           );
           return;
         } else {
@@ -532,23 +531,20 @@ export class TimeLine {
 
     // media id for this as well it can be downloaded
     this.addPlayableClip(
-      new ClipUI({
-        version: version,
-        type: type,
-        group: group,
-        name: name,
-        media_id: media_id,
-        clip_uuid: clip_uuid,
-        object_uuid: object_uuid,
-        object_name: object_name,
-        start_offset: offset,
-        ending_offset: end_offset, // length
-        keyframe_offset: 0,
-        media_file_type:
-          this.scene.get_object_by_uuid(object_uuid)?.userData[
-            "media_file_type"
-          ],
-      }),
+      new ClipUI(
+        version,
+        type,
+        group,
+        name,
+        media_id,
+        clip_uuid,
+        object_uuid,
+        object_name,
+        offset,
+        end_offset, // length
+        0,
+        this.scene.get_object_by_uuid(object_uuid)?.userData["media_file_type"],
+      ),
     );
 
     this.checkEditorCanPlay();
@@ -608,8 +604,8 @@ export class TimeLine {
     // only length and offset changes here.
     const object_uuid = data.object_uuid;
     const media_id = data.media_id;
-    const offset = data.start_offset;
-    const length = data.ending_offset + offset;
+    const offset = data.offset;
+    const length = data.length + offset;
     const clip_uuid = data.clip_uuid;
 
     for (const element of this.timeline_items) {
@@ -618,8 +614,8 @@ export class TimeLine {
         element.object_uuid === object_uuid &&
         element.clip_uuid == clip_uuid
       ) {
-        element.ending_offset = length;
-        element.start_offset = offset;
+        element.length = length;
+        element.offset = offset;
       }
     }
     this.checkEditorCanPlay();
@@ -722,8 +718,8 @@ export class TimeLine {
   public getEndPoint(): number {
     let longest = 0;
     for (const element of this.timeline_items) {
-      if (longest < element.ending_offset) {
-        longest = element.ending_offset;
+      if (longest < element.length) {
+        longest = element.length;
       }
     }
     return longest;
@@ -776,8 +772,8 @@ export class TimeLine {
     //3. smallest unit is a frame and it is set by the scene and is in fps, our videos will be 60fps but we can reprocess them using the pipeline.
     for (const element of this.timeline_items) {
       if (
-        element.start_offset <= this.scrubber_frame_position &&
-        this.scrubber_frame_position <= element.ending_offset &&
+        element.offset <= this.scrubber_frame_position &&
+        this.scrubber_frame_position <= element.length &&
         element.should_play
       ) {
         // run async
@@ -788,11 +784,11 @@ export class TimeLine {
           if (object && this.transform_engine.clips[element.object_uuid]) {
             this.transform_engine.clips[element.object_uuid].step(
               object,
-              element.start_offset,
+              element.offset,
               this.scrubber_frame_position,
               this.scene,
             );
-            element.ending_offset =
+            element.length =
               this.transform_engine.clips[element.object_uuid].length;
           }
         } else if (
@@ -812,7 +808,7 @@ export class TimeLine {
           await this.audio_engine.step(
             element.media_id,
             this.scrubber_frame_position,
-            element.start_offset,
+            element.offset,
           );
         } else if (
           element.type === ClipType.AUDIO &&
@@ -825,7 +821,7 @@ export class TimeLine {
             ].play(object);
             this.lipSync_engine.clips[
               element.object_uuid + element.media_id
-            ].step(this.scrubber_frame_position, element.start_offset);
+            ].step(this.scrubber_frame_position, element.offset);
           }
         } else if (element.type === ClipType.ANIMATION) {
           if (object) {
@@ -836,7 +832,7 @@ export class TimeLine {
             await this.animation_engine.clips[
               object.uuid + element.media_id
             ].step(
-              (this.scrubber_frame_position - element.start_offset) / fps,
+              (this.scrubber_frame_position - element.offset) / fps,
               this.is_playing,
               this.scrubber_frame_position, // Double FPS for best result.
             );
@@ -846,7 +842,7 @@ export class TimeLine {
           if (object) {
             await this.emotion_engine.clips[
               object.uuid + element.media_id
-            ].step(this.scrubber_frame_position - element.start_offset, object);
+            ].step(this.scrubber_frame_position - element.offset, object);
           }
         }
         //this.timelineItems = this.timelineItems.filter(item => item !== element)

app/pages/PageEnigma/Editor/video_generation.ts

@@ -128,8 +128,8 @@ export class VideoGeneration {
     const video_og: string = itteration + "tmp.mp4";
     const wav_name: string = itteration + "tmp.wav";
     const new_video: string = itteration + 1 + "tmp.mp4";
-    let startFrame: number = clip.start_offset;
-    let endFrame: number = clip.ending_offset;
+    let startFrame: number = clip.offset;
+    let endFrame: number = clip.length;
 
     if (endFrame > this.editor.timeline.timeline_limit) {
       endFrame = this.editor.timeline.timeline_limit;

app/pages/PageEnigma/datastructures/clips/clip_ui.ts

@@ -2,7 +2,7 @@
 // it is created from a media id.
 import { ClipGroup, ClipType, MediaFileType } from "~/pages/PageEnigma/enums";
 
-export interface ClipUiItem {
+export class ClipUI {
   version: number;
   type: ClipType;
   group: ClipGroup;
@@ -10,63 +10,27 @@ export interface ClipUiItem {
   media_id: string;
   object_uuid: string;
   object_name: string;
-  start_offset: number; // in frames
-  ending_offset: number; // in frames
-  should_play?: boolean;
-  clip_uuid: string;
-  keyframe_offset: number;
-  media_file_type: MediaFileType;
-}
-export interface ClipUiClassInterface extends ClipUiItem {
-  toJSON: () => ClipUiItem;
-}
-
-export class ClipUI implements ClipUiClassInterface {
-  version: number;
-  type: ClipType;
-  group: ClipGroup;
-  name: string;
-  media_id: string;
-  object_uuid: string;
-  object_name: string;
-  start_offset: number; // in frames
-  ending_offset: number; // in frames
+  offset: number; // in frames
+  length: number; // in frames
   should_play: boolean;
   clip_uuid: string;
   keyframe_offset: number;
   media_file_type: MediaFileType;
 
-  constructor({
-    version,
-    type,
-    group,
-    name,
-    media_id,
-    clip_uuid,
-    object_uuid,
-    object_name,
-    start_offset,
-    offset,
-    ending_offset,
-    length,
-    keyframe_offset = 0,
-    media_file_type,
-  }: {
-    version: number;
-    type: ClipType;
-    group: ClipGroup;
-    name: string;
-    media_id: string;
-    clip_uuid: string;
-    object_uuid: string;
-    object_name: string;
-    start_offset?: number;
-    offset?: number; // =start_offset, kept for now just for backwards compatibility
-    ending_offset?: number;
-    length?: number; // =ending_offset, kept for now just for backwards compatibility
-    keyframe_offset?: number;
-    media_file_type: MediaFileType;
-  }) {
+  constructor(
+    version: number,
+    type: ClipType,
+    group: ClipGroup,
+    name: string,
+    media_id: string,
+    clip_uuid: string,
+    object_uuid: string,
+    object_name: string,
+    offset: number,
+    length: number,
+    keyframe_offset: number = 0,
+    media_file_type: MediaFileType
+  ) {
     this.version = version;
     this.group = group; // Only needed for UI
 
@@ -78,16 +42,14 @@ export class ClipUI implements ClipUiClassInterface {
     this.object_uuid = object_uuid; // Animation / Audio / Lipsync /  : Engine
     this.object_name = object_name;
     this.media_id = media_id; //  Animation / Audio / Lipsync /  : Engine
-    this.start_offset = start_offset ? start_offset : offset ? offset : 0;
-    this.ending_offset = ending_offset ? ending_offset : length ? length : 0;
-
+    this.offset = offset;
+    this.length = length;
     this.should_play = true;
-    this.keyframe_offset = keyframe_offset ?? 0;
-
+    this.keyframe_offset = keyframe_offset;
     this.media_file_type = media_file_type;
   }
 
-  toJSON(): ClipUiItem {
+  toJSON(): any {
     return {
       version: this.version,
       group: this.group,
@@ -97,10 +59,10 @@ export class ClipUI implements ClipUiClassInterface {
       object_uuid: this.object_uuid,
       object_name: this.object_name,
       media_id: this.media_id,
-      start_offset: this.start_offset,
-      ending_offset: this.ending_offset,
+      start_offset: this.offset,
+      ending_offset: this.length,
       keyframe_offset: this.keyframe_offset,
-      media_file_type: this.media_file_type,
+      media_file_type: this.media_file_type
     };
   }
 }

app/pages/PageEnigma/models/track.ts

@@ -1,4 +1,4 @@
-// import { MediaFileAnimationType } from "~/enums";
+import { MediaFileAnimationType } from "~/enums";
 import { XYZ } from "../datastructures/common";
 import { ClipGroup, ClipType, MediaFileType } from "~/pages/PageEnigma/enums";
 
@@ -36,7 +36,7 @@ export interface CharacterTrack {
   object_uuid: string;
   media_id: string;
   mediaType: MediaFileType;
-  // animationType?: MediaFileAnimationType;
+  animationType: MediaFileAnimationType;
   name: string;
   muted: boolean;
   minimized: boolean;

app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts

@@ -4,10 +4,10 @@ import { ClipUI } from "../datastructures/clips/clip_ui";
 import { EmotionClip } from "../datastructures/clips/emotion_clip";
 import { LipSyncClip } from "../datastructures/clips/lipsync_clip";
 import { TransformClip } from "../datastructures/clips/transform_clip";
-import { AnimationEngine } from "../Editor/animation_engine";
+import AnimationEngine from "../Editor/animation_engine";
 import AudioEngine from "../Editor/audio_engine";
 import EmotionEngine from "../Editor/emotion_engine";
-import { LipSyncEngine } from "../Editor/lip_sync_engine";
+import LipSyncEngine from "../Editor/lip_sync_engine";
 import { TimeLine } from "../Editor/timeline";
 import TransformEngine from "../Editor/transform_engine";
 
@@ -77,20 +77,20 @@ export class StoryTellerProxyTimeline {
     if (timeline_clips) {
       for (let index = 0; index < timeline_clips.length; index++) {
         const element = timeline_clips[index];
-        let clip = new ClipUI({
-          version: element.version,
-          type: element.type,
-          group: element.group,
-          name: element.name,
-          media_id: element.media_id,
-          clip_uuid: element.clip_uuid,
-          object_uuid: element.object_uuid,
-          object_name: element.object_name,
-          start_offset: element.start_offset,
-          ending_offset: element.ending_offset,
-          keyframe_offset: element.keyframe_offset,
-          media_file_type: element.media_file_type,
-        });
+        let clip = new ClipUI(
+          element.version,
+          element.type,
+          element.group,
+          element.name,
+          element.media_id,
+          element.clip_uuid,
+          element.object_uuid,
+          element.object_name,
+          element.start_offset,
+          element.ending_offset,
+          element.keyframe_offset,
+          element.media_file_type
+        );
         new_clips.push(clip);
       }
     }

app/pages/PageEnigma/signals/audioGroup.ts

@@ -144,8 +144,8 @@ export function loadAudioData(item: ClipUI) {
     object_uuid: item.object_uuid,
     media_id: item.media_id,
     name: item.name,
-    offset: item.start_offset,
-    length: item.ending_offset,
+    offset: item.offset,
+    length: item.length,
   } as Clip;
   existingAudio.clips.push(newItem);
   existingAudio.clips.sort((clipA, clipB) => clipA.offset - clipB.offset);

app/pages/PageEnigma/signals/characterGroups/loadCharacterData.ts

@@ -23,7 +23,7 @@ function getAddCharacter(item: ClipUI) {
     positionKeyframes: [],
     lipSyncClips: [],
     expressionClips: [],
-    mediaType: item.media_file_type,
+    mediaType: item.media_file_type
   } as CharacterTrack;
 
   characterGroup.value = {
@@ -53,9 +53,9 @@ export function loadCharacterData(item: ClipUI) {
       object_uuid: item.object_uuid,
       media_id: item.media_id,
       name: item.name,
-      offset: item.start_offset,
-      length: item.ending_offset,
-      mediaType: item.media_file_type,
+      offset: item.offset,
+      length: item.length,
+      mediaType: item.media_file_type
     } as Clip;
     existingCharacter.animationClips.push(newItem);
     existingCharacter.animationClips.sort(
@@ -71,8 +71,8 @@ export function loadCharacterData(item: ClipUI) {
       object_uuid: item.object_uuid,
       media_id: item.media_id,
       name: item.name,
-      offset: item.start_offset,
-      length: item.ending_offset,
+      offset: item.offset,
+      length: item.length,
     } as Clip;
     existingCharacter.expressionClips.push(newItem);
     existingCharacter.expressionClips.sort(
@@ -101,8 +101,8 @@ export function loadCharacterData(item: ClipUI) {
       object_uuid: item.object_uuid,
       media_id: item.media_id,
       name: item.name,
-      offset: item.start_offset,
-      length: item.ending_offset,
+      offset: item.offset,
+      length: item.length,
     } as Clip;
     existingCharacter.lipSyncClips.push(newItem);
     existingCharacter.lipSyncClips.sort(