diff --git a/app/components/ButtonIcon.tsx b/app/components/ButtonIcon.tsx index abedc9c683..77c010a9a6 100644 --- a/app/components/ButtonIcon.tsx +++ b/app/components/ButtonIcon.tsx @@ -27,6 +27,7 @@ export const ButtonIcon = ({ hoverFill ? "bg-ui-controls-button hover:bg-ui-controls-button/[0.75]" : "bg-transparent hover:bg-ui-panel/[0.4]", + disabled && "opacity-50 hover:bg-transparent", propsClassName, ); diff --git a/app/contexts/Authentication/AuthenticationProvider.tsx b/app/contexts/Authentication/AuthenticationProvider.tsx index f94a317936..6b4f80cd65 100644 --- a/app/contexts/Authentication/AuthenticationProvider.tsx +++ b/app/contexts/Authentication/AuthenticationProvider.tsx @@ -4,7 +4,7 @@ import { CreateSession, DestroySession, GetSession } from "./utilities"; import { AuthenticationContext } from "./AuthenticationContext"; export const AuthenticationProvider = ({children}:{children:ReactNode})=>{ - console.log('auth provider rerender'); + // console.log('auth provider rerender'); const [authState, setAuthState] = useState({ authStatus: AUTH_STATUS.INIT @@ -47,7 +47,7 @@ export const AuthenticationProvider = ({children}:{children:ReactNode})=>{ password:string, failureCallback?:()=>void )=>{ - console.log('CreateSession'); + // console.log('CreateSession'); startLoggingState(); CreateSession({usernameOrEmail, password}) .then((loginResponse)=>{ @@ -68,10 +68,10 @@ export const AuthenticationProvider = ({children}:{children:ReactNode})=>{ const logout = useCallback(()=>{ const sessionToken = localStorage.getItem(STORAGE_KEYS.SESSION_TOKEN); if (sessionToken !== null){ - console.log('destroying the session'); + // console.log('destroying the session'); DestroySession(sessionToken).then((res)=>{ - if (res) console.log('Logout Success'); // do nothing - else console.log(res); //TODO: error handling maybe necessary + // if (res) console.log('Logout Success'); // do nothing + // else console.log(res); //TODO: error handling maybe necessary }) ;} // regarldess, clear the state and useeffect will rid of the localstorage @@ -84,20 +84,20 @@ export const AuthenticationProvider = ({children}:{children:ReactNode})=>{ if(authState.authStatus === AUTH_STATUS.INIT){ //Persist Login if Session Data exist const sessionToken = localStorage.getItem(STORAGE_KEYS.SESSION_TOKEN); - console.log(`Session Data: ${sessionToken?.substring(0, 30)}${sessionToken ? '...':''}`); + // console.log(`Session Data: ${sessionToken?.substring(0, 30)}${sessionToken ? '...':''}`); if (sessionToken !== null){ - console.log('Getting session to persist login'); + // console.log('Getting session to persist login'); updateLoggingState(sessionToken); GetSession(sessionToken).then(( sessionResponse: SessionResponse )=>{ - console.log('Session Returned'); - console.log(sessionResponse); + // console.log('Session Returned'); + // console.log(sessionResponse); if(sessionResponse.success && sessionResponse.user && sessionResponse.user !== null){ - console.log('Setting Auth State'); + // console.log('Setting Auth State'); setLoginState(sessionResponse.user) }else{ - console.log('Session expired, setting State'); + // console.log('Session expired, setting State'); setLogoutState(); } }); diff --git a/app/pages/PageEnigma/comps/ControlsTopButtons/TestFeaturesButtons.tsx b/app/pages/PageEnigma/comps/ControlsTopButtons/TestFeaturesButtons.tsx index 7362b8d872..6809df312c 100644 --- a/app/pages/PageEnigma/comps/ControlsTopButtons/TestFeaturesButtons.tsx +++ b/app/pages/PageEnigma/comps/ControlsTopButtons/TestFeaturesButtons.tsx @@ -60,7 +60,7 @@ export const TestFeaturesButtons = ({ debug }: { debug: boolean }) => { editorEngine?.generateFrame(); }; const handleButtonPlayBack = () => { - editorEngine?.startPlayback(); + editorEngine?.togglePlayback(); }; const smallButtons = "text-xs p-2 h-6 "; diff --git a/app/pages/PageEnigma/comps/ControlsVideo/ControlsVideo.tsx b/app/pages/PageEnigma/comps/ControlsVideo/ControlsVideo.tsx index 0e6e7258a0..3c1b330fe6 100644 --- a/app/pages/PageEnigma/comps/ControlsVideo/ControlsVideo.tsx +++ b/app/pages/PageEnigma/comps/ControlsVideo/ControlsVideo.tsx @@ -1,8 +1,12 @@ -import { useContext, useRef } from "react"; +import { + useContext, + // useRef +} from "react"; import { faBackwardStep, faForwardStep, faPlay, + faPause, } from "@fortawesome/pro-solid-svg-icons"; import { ButtonIcon } from "~/components"; import { EngineContext } from "~/contexts/EngineContext"; @@ -13,16 +17,19 @@ import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; import { useSignals } from "@preact/signals-react/runtime"; import { editorState, EditorStates } from "~/pages/PageEnigma/store/engine"; -const SCALE_VALUES = [0.5, 1, 1.25, 1.5, 1.75, 2]; -enum ScaleAdjustment { - UP, - DOWN, -} +// const SCALE_VALUES = [0.5, 1, 1.25, 1.5, 1.75, 2]; +// enum ScaleAdjustment { +// UP, +// DOWN, +// } export const ControlsVideo = () => { useSignals(); const editorEngine = useContext(EngineContext); - const scaleIndex = useRef(1); + // const scaleIndex = useRef(1); + // const canPlayback = editorEngine !== null && editorEngine.can_playback; + // console.log(`canplayback is ${canPlayback}`); + const isPlaying = editorEngine!==null ? editorEngine.timeline.is_playing : false; const handleBackwardStep = () => { currentTime.value = Math.max(currentTime.value - 1, 0); @@ -32,8 +39,8 @@ export const ControlsVideo = () => { data: { currentTime: currentTime.value }, }); }; - const handlePlay = () => { - editorEngine?.startPlayback(); + const handlePlayback = () => { + editorEngine?.togglePlayback(); }; const handleForwardStep = () => { currentTime.value = Math.min(currentTime.value + 1, filmLength.value * 60); @@ -70,9 +77,21 @@ export const ControlsVideo = () => {
- - - + + + {/* adjustScale(ScaleAdjustment.DOWN)}*/} {/* icon={faMagnifyingGlassMinus}*/} diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/AudioTab.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/AudioTab.tsx index 4a425a4d44..fd6277d0ca 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/AudioTab.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/AudioTab.tsx @@ -50,7 +50,7 @@ export const AudioTab = () => { const handleListAudioByUser = useCallback((username:string, sessionToken:string)=>{ function getTitle (item:any){ - console.log(item); + // console.log(item); if (item.maybe_title) return item.maybe_title; if (item.origin && item.origin.maybe_model && item.origin.maybe_model.title) return item.origin.maybe_model.title; return "Media Audio"; diff --git a/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts index 0e43e5aabf..367b3b21ed 100644 --- a/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts @@ -132,7 +132,7 @@ export class LipSyncClip { if (this.lipsync == null) { return; } - const positions = this.lipsync.update(frame, offset, rendering); + const positions = this.lipsync.update(frame, offset, true); if (positions) this.setBlends(positions["ee"], positions["ah"], positions["oh"]); } diff --git a/app/pages/PageEnigma/js/audio_engine.ts b/app/pages/PageEnigma/js/audio_engine.ts index f05338d874..e6a7eafb79 100644 --- a/app/pages/PageEnigma/js/audio_engine.ts +++ b/app/pages/PageEnigma/js/audio_engine.ts @@ -32,24 +32,34 @@ class AudioEngine { if (this.clips[audio_media_id] == null) { this.loadClip(audio_media_id); } + } + + async step(audio_media_id: string, frame: number, offset: number) { + const frameBuffer = 12; + + let pos = frame - offset; + + let doPlay = true; + if ( + Math.abs(this.last_frame - frame) < frameBuffer - 1 && + this.last_frame !== 0 + ) { + doPlay = false; + } + if (pos <= 1) { + doPlay = true; + } + let clip = this.clips[audio_media_id]; - if (clip.audio_data?.audioContext) { - if (this.playing.includes(audio_media_id)) { - return; - } - this.playing.push(audio_media_id); - clip.audio_data.source = - clip.audio_data.audioContext.createBufferSource(); + + if (doPlay && clip.audio_data?.audioContext) { + const startTime = pos / 60.0; + const endTime = frameBuffer / 60.0; + clip.audio_data.source = clip.audio_data.audioContext.createBufferSource(); clip.audio_data.source.buffer = clip.audio_data.audioBuffer; clip.audio_data.source.connect(clip.audio_data.audioContext.destination); - clip.audio_data.source.start(); - clip.audio_data.source.onended = () => { - this.playing = this.playing.filter((id) => id !== audio_media_id); - }; - } else { - console.warn( - `AudioManager: AudioClip buffer with id "${audio_media_id}" not found.`, - ); + clip.audio_data.source.start(0, startTime, endTime); + this.last_frame = frame; } } diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 7e5c5bf288..055388ba33 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -76,6 +76,7 @@ class Editor { render_timer: number; fps_number: number; cap_fps: number; + can_playback: boolean; playback: boolean; playback_location: number; max_length: number; @@ -190,6 +191,7 @@ class Editor { this.fps_number = 60; this.cap_fps = 60; // Timeline settings. + this.can_playback = false; this.playback = false; this.playback_location = 0; this.max_length = 10; @@ -446,7 +448,7 @@ class Editor { loadingBarIsShowing.value = false; } - public async newScene(){ + public async newScene() { this.activeScene.clear(); this.audio_engine = new AudioEngine(); this.emotion_engine = new EmotionEngine(this.version); @@ -515,6 +517,8 @@ class Editor { ); await proxyTimeline.loadFromJson(scene_json["timeline"]); + this.timeline.checkEditorCanPlay(); + this.dispatchAppUiState({ type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER, }); @@ -808,7 +812,7 @@ class Editor { if (this.timeline.is_playing == false) { //this.recorder.stop(); this.playback_location = 0; - this.stopPlayback(); + this.stopPlaybackAndUploadVideo(); } } } @@ -834,15 +838,15 @@ class Editor { async updateLoop() { setTimeout(() => { requestAnimationFrame(this.updateLoop.bind(this)); - }, 1000 / (this.cap_fps*2)); // Get the most FPS we can out of the renderer. + }, 1000 / (this.cap_fps * 2)); // Get the most FPS we can out of the renderer. - if(this.container === undefined){ + if (this.container === undefined) { this.container = document.getElementById("video-scene-container"); } - if(!this.rendering && this.container !== undefined){ - if (this.container.clientWidth+this.container.clientHeight !== this.lastCanvasSize) { + if (!this.rendering && this.container !== undefined) { + if (this.container.clientWidth + this.container.clientHeight !== this.lastCanvasSize) { this.onWindowResize(); - this.lastCanvasSize = this.container.clientWidth+this.container.clientHeight; + this.lastCanvasSize = this.container.clientWidth + this.container.clientHeight; } } @@ -850,7 +854,7 @@ class Editor { this.cam_obj = this.activeScene.get_object_by_name(this.camera_name); } - + // Updates debug stats. if (this.stats != null) { this.stats.update(); @@ -1006,7 +1010,8 @@ class Editor { a.click(); // Trigger the download } - async stopPlayback(compile_audio: boolean = true) { + async stopPlaybackAndUploadVideo(compile_audio: boolean = true) { + this.playback = false; this.rendering = false; //const videoBlob = new Blob(this.frame_buffer, { type: "video/webm" }); @@ -1200,13 +1205,13 @@ class Editor { previewSrc.value = url; // const stylePreview: HTMLImageElement | null = document.getElementById( - // "video-scene", + // "video-scene", // ) as HTMLImageElement; // if (stylePreview) { - // stylePreview.src = url; - // console.log("Set preview source."); + // stylePreview.src = url; + // console.log("Set preview source."); // } else { - // console.log("No style preview window."); + // console.log("No style preview window."); // } return Promise.resolve(url); @@ -1227,7 +1232,7 @@ class Editor { this.showLoading(); this.rendering = true; // has to go first to debounce - this.startPlayback(); + this.togglePlayback(); this.frame_buffer = []; this.render_timer = 0; this.activeScene.renderMode(this.rendering); @@ -1239,7 +1244,7 @@ class Editor { } } - startPlayback() { + togglePlayback() { this.updateLoad(25, "Starting Processing"); if (this.rawRenderer) { this.startRenderWidth = this.rawRenderer.domElement.width; @@ -1247,10 +1252,10 @@ class Editor { } if (!this.rendering && this.timeline.is_playing) { this.timeline.is_playing = false; - this.timeline.scrubber_frame_position = 0; - this.timeline.current_time = 0; - this.timeline.stepFrame(0); - this.timeline.resetScene(); + // this.timeline.scrubber_frame_position = 0; + // this.timeline.current_time = 0; + // this.timeline.stepFrame(0); + // this.timeline.resetScene(); this.switchCameraView(); if (this.activeScene.hot_items) { this.activeScene.hot_items.forEach((element) => { @@ -1393,7 +1398,7 @@ class Editor { this.switchPreviewToggle == false && this.selectedCanvas ) { - this.startPlayback(); + this.togglePlayback(); } } } diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 341c5fd96b..2d43ef01c8 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -320,7 +320,7 @@ export class TimeLine { this.transform_engine.addFrame( uuid, - this.absolute_end, + data_json["offset"], data_json["position"], data_json["rotation"], data_json["scale"], @@ -339,7 +339,7 @@ export class TimeLine { uuid, object_name, 0, - this.absolute_end, + data_json["offset"], data_json["offset"], ), ); @@ -353,6 +353,16 @@ export class TimeLine { if (this.editorEngine.camera_person_mode) { point.visible = false; } + this.checkEditorCanPlay(); + } + + public checkEditorCanPlay(){ + if(this.getEndPoint() <= 1){ + this.editorEngine.can_playback = false; + } else { + this.editorEngine.can_playback = true; + } + this.editorEngine.updateSelectedUI(); } public deleteObject(object_uuid: string) { @@ -432,6 +442,8 @@ export class TimeLine { end_offset, ), ); + + this.checkEditorCanPlay(); } public async deleteKeyFrame(data: any) { @@ -439,6 +451,16 @@ export class TimeLine { const object_uuid = data["data"]["object_uuid"]; this.transform_engine.clips[object_uuid].removeKeyframe(keyframe_uuid); this.scene.deletePoint(keyframe_uuid); + for (const element of this.timeline_items) { + if ( + element.clip_uuid === keyframe_uuid && + element.object_uuid === object_uuid + ) { + this.timeline_items = this.timeline_items.filter(element => !(element.clip_uuid === keyframe_uuid && element.object_uuid === object_uuid)); + break; + } + } + this.checkEditorCanPlay(); } public async updateKeyFrame(data: any) { @@ -461,6 +483,7 @@ export class TimeLine { keyframe_scl, ); this.scene.updatePoint(keyframe_uuid, keyframe_pos, keyframe_rot, keyframe_scl); + this.checkEditorCanPlay(); } public async updateClip(data: any) { @@ -481,6 +504,7 @@ export class TimeLine { element.offset = offset; } } + this.checkEditorCanPlay(); } public async deleteClip(data: any) { @@ -501,6 +525,8 @@ export class TimeLine { break; } } + + this.checkEditorCanPlay(); } public async scrubberUpdate(data: any) { @@ -525,6 +551,7 @@ export class TimeLine { } const value = Math.floor(data["data"]["currentTime"]); this.setScrubberPosition(value); + this.current_time = value; this.update(); if (this.editorEngine.switchPreviewToggle) { @@ -588,9 +615,10 @@ export class TimeLine { } } - private getEndPoint(): number { + public getEndPoint(): number { let longest = 0; for (const element of this.timeline_items) { + console.log(element) if (longest < element.length) { longest = element.length; } @@ -650,11 +678,15 @@ export class TimeLine { this.is_playing && !isRendering ) { - if (this.scrubber_frame_position + 1 >= element.length) { - this.audio_engine.stopClip(element.media_id); - } else { - this.audio_engine.playClip(element.media_id); - } + // if (this.scrubber_frame_position + 1 >= element.length) { + // this.audio_engine.playClip(element.media_id); + // } else { + // await this.audio_engine.step( + // element.object_uuid + element.media_id, + // this.scrubber_frame_position, element.offset); + // } + await this.audio_engine.playClip(element.media_id); + await this.audio_engine.step(element.media_id, this.scrubber_frame_position, element.offset); } else if ( element.type === ClipType.AUDIO && element.group === ClipGroup.CHARACTER &&