From 97c12823c95e16fa00a422f6bb3028e31418b81b Mon Sep 17 00:00:00 2001 From: Miles Ryan <75916236+Quicksticks-oss@users.noreply.github.com> Date: Wed, 1 May 2024 17:18:22 -0400 Subject: [PATCH] Fixed lipsync for same clip. (#292) Cleaned up some console logs and fixed lip sync on same track with multiple of the same clip. --- app/pages/PageEnigma/js/editor.ts | 9 --------- app/pages/PageEnigma/js/lipsync.js | 16 +++++----------- app/pages/PageEnigma/js/timeline.ts | 28 ++++++++++------------------ 3 files changed, 15 insertions(+), 38 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index bc9d5c2982..bf724f1bbe 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -477,7 +477,6 @@ class Editor { "m_8fmp9hrvsqcryzka1fra597kg42s50", "m_z4jzbst3xfh64h0qn4bqh4afenfps9", ]); - console.log(result); } public async testTestTimelineEvents() { } @@ -553,7 +552,6 @@ class Editor { this.activeScene, ); const scene_json = await proxyScene.saveToScene(); - console.log("scene_json", scene_json); const proxyTimeline = new StoryTellerProxyTimeline( this.version, @@ -575,8 +573,6 @@ class Editor { this.current_scene_glb_media_token, ); - console.log(result); - this.dispatchAppUiState({ type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER, }); @@ -1091,7 +1087,6 @@ class Editor { return; } const upload_token = data["media_file_token"]; - console.log(upload_token); const result = await this.api_manager .stylizeVideo( @@ -1109,7 +1104,6 @@ class Editor { this.updateLoad(100, "Done Check Your Media Tab On Profile."); this.endLoading(); - console.log(result); this.recorder = undefined; if (this.rawRenderer) { this.rawRenderer.setSize(this.startRenderWidth, this.startRenderHeight); @@ -1188,7 +1182,6 @@ class Editor { this.positive_prompt, this.negative_prompt, ); - console.log(url); previewSrc.value = url; @@ -1302,8 +1295,6 @@ class Editor { const width = this.container.clientWidth; const height = this.container.clientHeight; - console.log(width, height) - if (this.camera == undefined || this.renderer == undefined) { return; } diff --git a/app/pages/PageEnigma/js/lipsync.js b/app/pages/PageEnigma/js/lipsync.js index a743f0267e..a0b59ea522 100644 --- a/app/pages/PageEnigma/js/lipsync.js +++ b/app/pages/PageEnigma/js/lipsync.js @@ -86,6 +86,10 @@ export class LipSync { let pos = frame - offset; + if (Math.floor(pos) <= 1){ + this.last_frame = 0; + } + let doPlay = true; if ( Math.abs(this.last_frame - frame) < frameBuffer - 1 && @@ -94,12 +98,6 @@ export class LipSync { doPlay = false; } - if (pos <= 1 && !rendering) { - doPlay = true; - } else if (!rendering) { - doPlay = false; - } - if (doPlay) { const startTime = pos / 60.0; const endTime = frameBuffer / 60.0; @@ -108,11 +106,7 @@ export class LipSync { this.meter = LipSync.createAudioMeter(this.audioContext); this.mediaStreamSource.connect(this.meter); this.mediaStreamSource.connect(this.audioContext.destination); - if (rendering) { - this.mediaStreamSource.start(0, startTime, endTime); - } else { - this.mediaStreamSource.start(); - } + this.mediaStreamSource.start(0, startTime, endTime); this.mediaStreamSource.connect(this.userSpeechAnalyzer); this.last_frame = frame; } diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 2d43ef01c8..c3f7779a17 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -356,8 +356,8 @@ export class TimeLine { this.checkEditorCanPlay(); } - public checkEditorCanPlay(){ - if(this.getEndPoint() <= 1){ + public checkEditorCanPlay() { + if (this.getEndPoint() <= 1) { this.editorEngine.can_playback = false; } else { this.editorEngine.can_playback = true; @@ -456,10 +456,10 @@ export class TimeLine { element.clip_uuid === keyframe_uuid && element.object_uuid === object_uuid ) { - this.timeline_items = this.timeline_items.filter(element => !(element.clip_uuid === keyframe_uuid && element.object_uuid === object_uuid)); - break; + this.timeline_items = this.timeline_items.filter(element => !(element.clip_uuid === keyframe_uuid && element.object_uuid === object_uuid)); + break; } - } + } this.checkEditorCanPlay(); } @@ -618,7 +618,6 @@ export class TimeLine { public getEndPoint(): number { let longest = 0; for (const element of this.timeline_items) { - console.log(element) if (longest < element.length) { longest = element.length; } @@ -627,12 +626,12 @@ export class TimeLine { } // called by the editor update loop on each frame - public async update(isRendering = false, delta_time:number=0): Promise { + public async update(isRendering = false, delta_time: number = 0): Promise { //if (this.is_playing === false) return; // start and stop this.timeline_limit = this.getEndPoint(); if (this.is_playing) { // When rendering we want to increase it by 1 but when in playback we want it dynamic based on deltatime. - if(isRendering){ + if (isRendering) { this.current_time += 1; } else { @@ -643,7 +642,7 @@ export class TimeLine { }); this.scrubber_frame_position = this.current_time; } - + if (this.scrubber_frame_position <= 0) { await this.resetScene(); @@ -692,18 +691,11 @@ export class TimeLine { element.group === ClipGroup.CHARACTER && this.is_playing ) { - // we will remove this when we know which group it will come from character + audio === lip sync audio. - if (this.scrubber_frame_position + 1 >= element.length) { - this.lipSync_engine.clips[ - element.object_uuid + element.media_id - ].stop(); - } else if (object) { + if (object) { await this.lipSync_engine.clips[ element.object_uuid + element.media_id ].play(object); - this.lipSync_engine.clips[ - element.object_uuid + element.media_id - ].step(this.scrubber_frame_position, element.offset, isRendering); + this.lipSync_engine.clips[element.object_uuid + element.media_id].step(this.scrubber_frame_position, element.offset, isRendering); } } else if (element.type === ClipType.ANIMATION) { if (object) {