From 6fef9167ce43ed4d02cc992b8007b6fa006d6a4d Mon Sep 17 00:00:00 2001 From: BFlat Date: Wed, 11 Sep 2024 18:22:50 +0700 Subject: [PATCH] live portrait cta to lipsync as source media (#699) * live portrait cta to lipsync as source media * lip icon --- .../SessionTtsInferenceResultsList.tsx | 1 - .../src/v2/view/pages/lipsync/GenerateTts.tsx | 31 +++- .../src/v2/view/pages/lipsync/Lipsync.tsx | 50 +++++- .../SessionLpInferenceResultsList.tsx | 150 +++++++++++------- .../live_portrait/ThumbnailMediaPicker.tsx | 20 +++ 5 files changed, 183 insertions(+), 69 deletions(-) diff --git a/src/website/packages/fakeyou.com/src/v2/view/_common/SessionTtsInferenceResultsList.tsx b/src/website/packages/fakeyou.com/src/v2/view/_common/SessionTtsInferenceResultsList.tsx index 8633a7e5b6..5f951f3ff8 100644 --- a/src/website/packages/fakeyou.com/src/v2/view/_common/SessionTtsInferenceResultsList.tsx +++ b/src/website/packages/fakeyou.com/src/v2/view/_common/SessionTtsInferenceResultsList.tsx @@ -64,7 +64,6 @@ function SessionTtsInferenceResultList(props: Props) { const fetchMedia = async (token: string) => { try { const response = await GetWeight(token, {}); - console.log("GetWeight response:", response); const publicBucketPath = response.cover_image.maybe_cover_image_public_bucket_path || ""; setMediaSrc(prev => ({ ...prev, [token]: publicBucketPath })); diff --git a/src/website/packages/fakeyou.com/src/v2/view/pages/lipsync/GenerateTts.tsx b/src/website/packages/fakeyou.com/src/v2/view/pages/lipsync/GenerateTts.tsx index 50f9cd55d6..2102606b1f 100644 --- a/src/website/packages/fakeyou.com/src/v2/view/pages/lipsync/GenerateTts.tsx +++ b/src/website/packages/fakeyou.com/src/v2/view/pages/lipsync/GenerateTts.tsx @@ -301,10 +301,21 @@ export const GenerateTts = ({ modelToken: string | undefined, resultToken: string | undefined | null ) => { - if (modelToken && resultToken && isAudioLoading === false) { + if (modelToken && resultToken && !isAudioLoading) { handleClearAudio(); setLoadingSelectedAudioResult(true); - const newUrl = `/ai-lip-sync?voice=${modelToken}&audio=${resultToken}`; + const url = new URL(window.location.href); + const searchParams = new URLSearchParams(url.search); + const sourceToken = searchParams.get("source"); + searchParams.set("voice", modelToken); + searchParams.set("audio", resultToken || ""); + + if (sourceToken) { + searchParams.set("source", sourceToken); + } + + const newUrl = `${url.pathname}?${searchParams.toString()}`; + history.push(newUrl); try { @@ -347,8 +358,18 @@ export const GenerateTts = ({ const handleVoiceSelect = async (data: any) => { if (data.weight_token) { - const newUrl = `/ai-lip-sync?voice=${data.weight_token}`; + const url = new URL(window.location.href); + const searchParams = new URLSearchParams(url.search); + const sourceToken = searchParams.get("source"); + searchParams.set("voice", data.weight_token); + + if (sourceToken) { + searchParams.set("source", sourceToken); + } + + const newUrl = `${url.pathname}?${searchParams.toString()}`; history.push(newUrl); + close(); } }; @@ -411,6 +432,7 @@ export const GenerateTts = ({
@@ -423,7 +445,6 @@ export const GenerateTts = ({ />
- Audio Transcript + {t("label.audioTranscript")}

{transcript}

diff --git a/src/website/packages/fakeyou.com/src/v2/view/pages/lipsync/Lipsync.tsx b/src/website/packages/fakeyou.com/src/v2/view/pages/lipsync/Lipsync.tsx index b461e7e192..f03e74d6c6 100644 --- a/src/website/packages/fakeyou.com/src/v2/view/pages/lipsync/Lipsync.tsx +++ b/src/website/packages/fakeyou.com/src/v2/view/pages/lipsync/Lipsync.tsx @@ -80,8 +80,11 @@ export default function Lipsync({ sessionSubscriptionsWrapper }: LipsyncProps) { const [sourceTokens, setSourceTokens] = useState([ ...PRECOMPUTED_SOURCE_TOKENS, ]); - const numberOfInitialSourceTokensRef = useRef(sourceTokens.length); - const numberOfInitialSourceTokens = numberOfInitialSourceTokensRef.current; + const [sourceTokenFromQuery, setSourceTokenFromQuery] = useState< + string | null + >(null); + // const numberOfInitialSourceTokensRef = useRef(sourceTokens.length); + // const numberOfInitialSourceTokens = numberOfInitialSourceTokensRef.current; const [isGenerating, setIsGenerating] = useState(false); const [jobProcessedTokens, setJobProcessedTokens] = useState([]); const [voiceToken, setVoiceToken] = useState(null); @@ -319,13 +322,26 @@ export default function Lipsync({ sessionSubscriptionsWrapper }: LipsyncProps) { useEffect(() => { const queryParams = new URLSearchParams(location.search); - setVoiceToken(queryParams.get("voice")); - }, [location.search]); + const voiceTokenFromQuery = queryParams.get("voice"); + const sourceTokenFromQuery = queryParams.get("source"); - useEffect(() => { - if (voiceToken) { - const precomputedSourceToken = LipsyncTokenMap[voiceToken]; + setVoiceToken(voiceTokenFromQuery); + setSourceTokenFromQuery(sourceTokenFromQuery); + if (sourceTokenFromQuery) { + setSourceTokens(prevTokens => { + const tokenIndex = prevTokens.indexOf(sourceTokenFromQuery); + if (tokenIndex !== -1) { + setSelectedSourceIndex(tokenIndex); + return prevTokens; + } else { + const updatedTokens = [...prevTokens, sourceTokenFromQuery]; + setSelectedSourceIndex(updatedTokens.length - 1); + return updatedTokens; + } + }); + } else if (voiceTokenFromQuery) { + const precomputedSourceToken = LipsyncTokenMap[voiceTokenFromQuery]; if (precomputedSourceToken) { setSourceTokens(prevTokens => { const tokenIndex = prevTokens.indexOf(precomputedSourceToken); @@ -340,7 +356,7 @@ export default function Lipsync({ sessionSubscriptionsWrapper }: LipsyncProps) { }); } } - }, [voiceToken, numberOfInitialSourceTokens]); + }, [location.search]); const signupCTA = ( <> @@ -373,6 +389,22 @@ export default function Lipsync({ sessionSubscriptionsWrapper }: LipsyncProps) { ); + const handleRemoveSourceMedia = () => { + if (sourceTokenFromQuery) { + setSourceTokenFromQuery(null); + setGeneratedVideoSrc(""); + setJobProcessedTokens([]); + setJobPercentage(null); + setLastEnqueuedJobToken(null); + setIsGenerating(false); + + const queryParams = new URLSearchParams(location.search); + queryParams.delete("source"); + history.replace({ search: queryParams.toString() }); + setSelectedSourceIndex(0); + } + }; + return ( <> @@ -422,6 +454,8 @@ export default function Lipsync({ sessionSubscriptionsWrapper }: LipsyncProps) { showUploadButton={false} showThumbnails={false} stepAlwaysOnTop={true} + showRemoveButton={!!sourceTokenFromQuery} + onRemoveMedia={handleRemoveSourceMedia} />
- ))} diff --git a/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/ThumbnailMediaPicker.tsx b/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/ThumbnailMediaPicker.tsx index 74a008501d..5f69903826 100644 --- a/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/ThumbnailMediaPicker.tsx +++ b/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/ThumbnailMediaPicker.tsx @@ -12,6 +12,7 @@ import { faUpload, faChevronLeft, faChevronRight, + faXmark, } from "@fortawesome/pro-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { isMobile } from "react-device-detect"; @@ -44,6 +45,8 @@ interface ThumbnailMediaPickerProps { showStep?: boolean; stepAlwaysOnTop?: boolean; videoRef?: React.RefObject; + showRemoveButton?: boolean; + onRemoveMedia?: () => void; } interface MediaData { @@ -72,6 +75,8 @@ const ThumbnailMediaPicker: React.FC = React.memo( showStep = true, stepAlwaysOnTop = false, videoRef, + showRemoveButton = false, + onRemoveMedia, }) => { const [isCropping, setIsCropping] = useState(false); const [mediaData, setMediaData] = useState<{ [key: string]: any }>({}); @@ -159,6 +164,21 @@ const ThumbnailMediaPicker: React.FC = React.memo(
+ {showRemoveButton && ( +
+ +
+ )} +
{!isCropping ? (