Play Button Icon and Functions (#283)

Play Button Icon and Functions
- Play Button changes icon depending on playback state
- Play Button can toggle between playing and pausing playback
- When Paused, current playtime wont be changed
- When Resumed, it resumes playing from where the play cursor is

Co-authored-by: wil <wil@storyteller.ai>

app/components/ButtonIcon.tsx

@@ -27,6 +27,7 @@ export const ButtonIcon = ({
     hoverFill
       ? "bg-ui-controls-button hover:bg-ui-controls-button/[0.75]"
       : "bg-transparent hover:bg-ui-panel/[0.4]",
+    disabled && "opacity-50 hover:bg-transparent",
     propsClassName,
   );
 

app/contexts/Authentication/AuthenticationProvider.tsx

@@ -4,7 +4,7 @@ import { CreateSession, DestroySession, GetSession } from "./utilities";
 import { AuthenticationContext } from "./AuthenticationContext";
 
 export const AuthenticationProvider = ({children}:{children:ReactNode})=>{
-  console.log('auth provider rerender');
+  // console.log('auth provider rerender');
 
   const [authState, setAuthState] = useState<AuthState>({
     authStatus: AUTH_STATUS.INIT
@@ -47,7 +47,7 @@ export const AuthenticationProvider = ({children}:{children:ReactNode})=>{
     password:string,
     failureCallback?:()=>void
   )=>{
-    console.log('CreateSession');
+    // console.log('CreateSession');
     startLoggingState();
     CreateSession({usernameOrEmail, password})
       .then((loginResponse)=>{
@@ -68,10 +68,10 @@ export const AuthenticationProvider = ({children}:{children:ReactNode})=>{
   const logout = useCallback(()=>{
     const sessionToken = localStorage.getItem(STORAGE_KEYS.SESSION_TOKEN);
     if (sessionToken !== null){
-      console.log('destroying the session');
+      // console.log('destroying the session');
       DestroySession(sessionToken).then((res)=>{
-        if (res) console.log('Logout Success'); // do nothing
-        else console.log(res); //TODO: error handling maybe necessary
+        // if (res) console.log('Logout Success'); // do nothing
+        // else console.log(res); //TODO: error handling maybe necessary
       })
     ;}
     // regarldess, clear the state and useeffect will rid of the localstorage
@@ -84,20 +84,20 @@ export const AuthenticationProvider = ({children}:{children:ReactNode})=>{
     if(authState.authStatus === AUTH_STATUS.INIT){
       //Persist Login if Session Data exist
       const sessionToken = localStorage.getItem(STORAGE_KEYS.SESSION_TOKEN);
-      console.log(`Session Data: ${sessionToken?.substring(0, 30)}${sessionToken ? '...':''}`);
+      // console.log(`Session Data: ${sessionToken?.substring(0, 30)}${sessionToken ? '...':''}`);
       if (sessionToken !== null){
-        console.log('Getting session to persist login');
+        // console.log('Getting session to persist login');
         updateLoggingState(sessionToken);
         GetSession(sessionToken).then((
           sessionResponse: SessionResponse
         )=>{
-          console.log('Session Returned');
-          console.log(sessionResponse);
+          // console.log('Session Returned');
+          // console.log(sessionResponse);
           if(sessionResponse.success && sessionResponse.user && sessionResponse.user !== null){
-            console.log('Setting Auth State');
+            // console.log('Setting Auth State');
             setLoginState(sessionResponse.user)
           }else{
-            console.log('Session expired, setting State');
+            // console.log('Session expired, setting State');
             setLogoutState(); 
           }
         });

app/pages/PageEnigma/comps/ControlsTopButtons/TestFeaturesButtons.tsx

@@ -60,7 +60,7 @@ export const TestFeaturesButtons = ({ debug }: { debug: boolean }) => {
     editorEngine?.generateFrame();
   };
   const handleButtonPlayBack = () => {
-    editorEngine?.startPlayback();
+    editorEngine?.togglePlayback();
   };
 
   const smallButtons = "text-xs p-2 h-6 ";

app/pages/PageEnigma/comps/ControlsVideo/ControlsVideo.tsx

@@ -1,8 +1,12 @@
-import { useContext, useRef } from "react";
+import { 
+  useContext,
+  // useRef
+} from "react";
 import {
   faBackwardStep,
   faForwardStep,
   faPlay,
+  faPause,
 } from "@fortawesome/pro-solid-svg-icons";
 import { ButtonIcon } from "~/components";
 import { EngineContext } from "~/contexts/EngineContext";
@@ -13,16 +17,19 @@ import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
 import { useSignals } from "@preact/signals-react/runtime";
 import { editorState, EditorStates } from "~/pages/PageEnigma/store/engine";
 
-const SCALE_VALUES = [0.5, 1, 1.25, 1.5, 1.75, 2];
-enum ScaleAdjustment {
-  UP,
-  DOWN,
-}
+// const SCALE_VALUES = [0.5, 1, 1.25, 1.5, 1.75, 2];
+// enum ScaleAdjustment {
+//   UP,
+//   DOWN,
+// }
 
 export const ControlsVideo = () => {
   useSignals();
   const editorEngine = useContext(EngineContext);
-  const scaleIndex = useRef(1);
+  // const scaleIndex = useRef(1);
+  // const canPlayback = editorEngine !== null && editorEngine.can_playback;
+  // console.log(`canplayback is ${canPlayback}`);
+  const isPlaying = editorEngine!==null ? editorEngine.timeline.is_playing : false;
 
   const handleBackwardStep = () => {
     currentTime.value = Math.max(currentTime.value - 1, 0);
@@ -32,8 +39,8 @@ export const ControlsVideo = () => {
       data: { currentTime: currentTime.value },
     });
   };
-  const handlePlay = () => {
-    editorEngine?.startPlayback();
+  const handlePlayback = () => {
+    editorEngine?.togglePlayback();
   };
   const handleForwardStep = () => {
     currentTime.value = Math.min(currentTime.value + 1, filmLength.value * 60);
@@ -70,9 +77,21 @@ export const ControlsVideo = () => {
     <div className="flex justify-center">
       <div className="rounded-t-lg border-x border-t border-ui-panel-border bg-ui-controls p-2 text-white">
         <div className="flex content-center gap-2">
-          <ButtonIcon icon={faBackwardStep} onClick={handleBackwardStep} />
-          <ButtonIcon icon={faPlay} onClick={handlePlay} />
-          <ButtonIcon icon={faForwardStep} onClick={handleForwardStep} />
+          <ButtonIcon
+            // disabled={!canPlayback}
+            icon={faBackwardStep}
+            onClick={handleBackwardStep}
+          />
+          <ButtonIcon
+            // disabled={!canPlayback}
+            icon={isPlaying ? faPause : faPlay}
+            onClick={handlePlayback}
+          />
+          <ButtonIcon
+            // disabled={!canPlayback}
+            icon={faForwardStep}
+            onClick={handleForwardStep}
+          />
           {/*<ButtonIcon*/}
           {/*  onClick={() => adjustScale(ScaleAdjustment.DOWN)}*/}
           {/*  icon={faMagnifyingGlassMinus}*/}

app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/AudioTab.tsx

@@ -50,7 +50,7 @@ export const AudioTab = () => {
 
   const handleListAudioByUser = useCallback((username:string, sessionToken:string)=>{
     function getTitle (item:any){
-      console.log(item);
+      // console.log(item);
       if (item.maybe_title) return item.maybe_title;
       if (item.origin && item.origin.maybe_model && item.origin.maybe_model.title) return item.origin.maybe_model.title;
       return "Media Audio";

app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts

@@ -132,7 +132,7 @@ export class LipSyncClip {
     if (this.lipsync == null) {
       return;
     }
-    const positions = this.lipsync.update(frame, offset, rendering);
+    const positions = this.lipsync.update(frame, offset, true);
     if (positions)
       this.setBlends(positions["ee"], positions["ah"], positions["oh"]);
   }

app/pages/PageEnigma/js/audio_engine.ts

@@ -32,24 +32,34 @@ class AudioEngine {
     if (this.clips[audio_media_id] == null) {
       this.loadClip(audio_media_id);
     }
+  }
+
+  async step(audio_media_id: string, frame: number, offset: number) {
+    const frameBuffer = 12;
+
+    let pos = frame - offset;
+
+    let doPlay = true;
+    if (
+      Math.abs(this.last_frame - frame) < frameBuffer - 1 &&
+      this.last_frame !== 0
+    ) {
+      doPlay = false;
+    }
+    if (pos <= 1) {
+      doPlay = true;
+    }
+
     let clip = this.clips[audio_media_id];
-    if (clip.audio_data?.audioContext) {
-      if (this.playing.includes(audio_media_id)) {
-        return;
-      }
-      this.playing.push(audio_media_id);
-      clip.audio_data.source =
-        clip.audio_data.audioContext.createBufferSource();
+
+    if (doPlay && clip.audio_data?.audioContext) {
+      const startTime = pos / 60.0;
+      const endTime = frameBuffer / 60.0;
+      clip.audio_data.source = clip.audio_data.audioContext.createBufferSource();
       clip.audio_data.source.buffer = clip.audio_data.audioBuffer;
       clip.audio_data.source.connect(clip.audio_data.audioContext.destination);
-      clip.audio_data.source.start();
-      clip.audio_data.source.onended = () => {
-        this.playing = this.playing.filter((id) => id !== audio_media_id);
-      };
-    } else {
-      console.warn(
-        `AudioManager: AudioClip buffer with id "${audio_media_id}" not found.`,
-      );
+      clip.audio_data.source.start(0, startTime, endTime);
+      this.last_frame = frame;
     }
   }
 

app/pages/PageEnigma/js/editor.ts

@@ -76,6 +76,7 @@ class Editor {
   render_timer: number;
   fps_number: number;
   cap_fps: number;
+  can_playback: boolean;
   playback: boolean;
   playback_location: number;
   max_length: number;
@@ -190,6 +191,7 @@ class Editor {
     this.fps_number = 60;
     this.cap_fps = 60;
     // Timeline settings.
+    this.can_playback = false;
     this.playback = false;
     this.playback_location = 0;
     this.max_length = 10;
@@ -446,7 +448,7 @@ class Editor {
     loadingBarIsShowing.value = false;
   }
 
-  public async newScene(){
+  public async newScene() {
     this.activeScene.clear();
     this.audio_engine = new AudioEngine();
     this.emotion_engine = new EmotionEngine(this.version);
@@ -515,6 +517,8 @@ class Editor {
     );
     await proxyTimeline.loadFromJson(scene_json["timeline"]);
 
+    this.timeline.checkEditorCanPlay();
+
     this.dispatchAppUiState({
       type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER,
     });
@@ -808,7 +812,7 @@ class Editor {
       if (this.timeline.is_playing == false) {
         //this.recorder.stop();
         this.playback_location = 0;
-        this.stopPlayback();
+        this.stopPlaybackAndUploadVideo();
       }
     }
   }
@@ -834,15 +838,15 @@ class Editor {
   async updateLoop() {
     setTimeout(() => {
       requestAnimationFrame(this.updateLoop.bind(this));
-    }, 1000 / (this.cap_fps*2)); // Get the most FPS we can out of the renderer.
+    }, 1000 / (this.cap_fps * 2)); // Get the most FPS we can out of the renderer.
 
-    if(this.container === undefined){
+    if (this.container === undefined) {
       this.container = document.getElementById("video-scene-container");
     }
-    if(!this.rendering && this.container !== undefined){
-      if (this.container.clientWidth+this.container.clientHeight !== this.lastCanvasSize) {
+    if (!this.rendering && this.container !== undefined) {
+      if (this.container.clientWidth + this.container.clientHeight !== this.lastCanvasSize) {
         this.onWindowResize();
-        this.lastCanvasSize = this.container.clientWidth+this.container.clientHeight;
+        this.lastCanvasSize = this.container.clientWidth + this.container.clientHeight;
       }
     }
 
@@ -850,7 +854,7 @@ class Editor {
       this.cam_obj = this.activeScene.get_object_by_name(this.camera_name);
 
     }
-    
+
     // Updates debug stats.
     if (this.stats != null) {
       this.stats.update();
@@ -1006,7 +1010,8 @@ class Editor {
     a.click(); // Trigger the download
   }
 
-  async stopPlayback(compile_audio: boolean = true) {
+  async stopPlaybackAndUploadVideo(compile_audio: boolean = true) {
+    this.playback = false;
     this.rendering = false;
 
     //const videoBlob = new Blob(this.frame_buffer, { type: "video/webm" });
@@ -1200,13 +1205,13 @@ class Editor {
         previewSrc.value = url;
 
         // const stylePreview: HTMLImageElement | null = document.getElementById(
-          // "video-scene",
+        // "video-scene",
         // ) as HTMLImageElement;
         // if (stylePreview) {
-          // stylePreview.src = url;
-          // console.log("Set preview source.");
+        // stylePreview.src = url;
+        // console.log("Set preview source.");
         // } else {
-          // console.log("No style preview window.");
+        // console.log("No style preview window.");
         // }
 
         return Promise.resolve(url);
@@ -1227,7 +1232,7 @@ class Editor {
     this.showLoading();
 
     this.rendering = true; // has to go first to debounce
-    this.startPlayback();
+    this.togglePlayback();
     this.frame_buffer = [];
     this.render_timer = 0;
     this.activeScene.renderMode(this.rendering);
@@ -1239,7 +1244,7 @@ class Editor {
     }
   }
 
-  startPlayback() {
+  togglePlayback() {
     this.updateLoad(25, "Starting Processing");
     if (this.rawRenderer) {
       this.startRenderWidth = this.rawRenderer.domElement.width;
@@ -1247,10 +1252,10 @@ class Editor {
     }
     if (!this.rendering && this.timeline.is_playing) {
       this.timeline.is_playing = false;
-      this.timeline.scrubber_frame_position = 0;
-      this.timeline.current_time = 0;
-      this.timeline.stepFrame(0);
-      this.timeline.resetScene();
+      // this.timeline.scrubber_frame_position = 0;
+      // this.timeline.current_time = 0;
+      // this.timeline.stepFrame(0);
+      // this.timeline.resetScene();
       this.switchCameraView();
       if (this.activeScene.hot_items) {
         this.activeScene.hot_items.forEach((element) => {
@@ -1393,7 +1398,7 @@ class Editor {
         this.switchPreviewToggle == false &&
         this.selectedCanvas
       ) {
-        this.startPlayback();
+        this.togglePlayback();
       }
     }
   }

app/pages/PageEnigma/js/timeline.ts

@@ -320,7 +320,7 @@ export class TimeLine {
 
     this.transform_engine.addFrame(
       uuid,
-      this.absolute_end,
+      data_json["offset"],
       data_json["position"],
       data_json["rotation"],
       data_json["scale"],
@@ -339,7 +339,7 @@ export class TimeLine {
         uuid,
         object_name,
         0,
-        this.absolute_end,
+        data_json["offset"],
         data_json["offset"],
       ),
     );
@@ -353,6 +353,16 @@ export class TimeLine {
     if (this.editorEngine.camera_person_mode) {
       point.visible = false;
     }
+    this.checkEditorCanPlay();
+  }
+
+  public checkEditorCanPlay(){
+    if(this.getEndPoint() <= 1){
+      this.editorEngine.can_playback = false;
+    } else {
+      this.editorEngine.can_playback = true;
+    }
+    this.editorEngine.updateSelectedUI();
   }
 
   public deleteObject(object_uuid: string) {
@@ -432,6 +442,8 @@ export class TimeLine {
         end_offset,
       ),
     );
+
+    this.checkEditorCanPlay();
   }
 
   public async deleteKeyFrame(data: any) {
@@ -439,6 +451,16 @@ export class TimeLine {
     const object_uuid = data["data"]["object_uuid"];
     this.transform_engine.clips[object_uuid].removeKeyframe(keyframe_uuid);
     this.scene.deletePoint(keyframe_uuid);
+    for (const element of this.timeline_items) {
+      if (
+        element.clip_uuid === keyframe_uuid &&
+        element.object_uuid === object_uuid
+      ) {
+         this.timeline_items = this.timeline_items.filter(element => !(element.clip_uuid === keyframe_uuid && element.object_uuid === object_uuid));
+         break;
+      }
+   }
+    this.checkEditorCanPlay();
   }
 
   public async updateKeyFrame(data: any) {
@@ -461,6 +483,7 @@ export class TimeLine {
       keyframe_scl,
     );
     this.scene.updatePoint(keyframe_uuid, keyframe_pos, keyframe_rot, keyframe_scl);
+    this.checkEditorCanPlay();
   }
 
   public async updateClip(data: any) {
@@ -481,6 +504,7 @@ export class TimeLine {
         element.offset = offset;
       }
     }
+    this.checkEditorCanPlay();
   }
 
   public async deleteClip(data: any) {
@@ -501,6 +525,8 @@ export class TimeLine {
         break;
       }
     }
+
+    this.checkEditorCanPlay();
   }
 
   public async scrubberUpdate(data: any) {
@@ -525,6 +551,7 @@ export class TimeLine {
     }
     const value = Math.floor(data["data"]["currentTime"]);
     this.setScrubberPosition(value);
+    this.current_time = value;
     this.update();
 
     if (this.editorEngine.switchPreviewToggle) {
@@ -588,9 +615,10 @@ export class TimeLine {
     }
   }
 
-  private getEndPoint(): number {
+  public getEndPoint(): number {
     let longest = 0;
     for (const element of this.timeline_items) {
+      console.log(element)
       if (longest < element.length) {
         longest = element.length;
       }
@@ -650,11 +678,15 @@ export class TimeLine {
           this.is_playing &&
           !isRendering
         ) {
-          if (this.scrubber_frame_position + 1 >= element.length) {
-            this.audio_engine.stopClip(element.media_id);
-          } else {
-            this.audio_engine.playClip(element.media_id);
-          }
+          // if (this.scrubber_frame_position + 1 >= element.length) {
+          //   this.audio_engine.playClip(element.media_id);
+          // } else {
+          //   await this.audio_engine.step(
+          //     element.object_uuid + element.media_id,
+          //   this.scrubber_frame_position, element.offset);
+          // }
+          await this.audio_engine.playClip(element.media_id);
+          await this.audio_engine.step(element.media_id, this.scrubber_frame_position, element.offset);
         } else if (
           element.type === ClipType.AUDIO &&
           element.group === ClipGroup.CHARACTER &&