diff --git a/src/website/packages/components/src/api/model_inference/GetModelInferenceJobStatus.ts b/src/website/packages/components/src/api/model_inference/GetModelInferenceJobStatus.ts index a0655d319d..bc4ed283c3 100644 --- a/src/website/packages/components/src/api/model_inference/GetModelInferenceJobStatus.ts +++ b/src/website/packages/components/src/api/model_inference/GetModelInferenceJobStatus.ts @@ -25,6 +25,8 @@ export interface RequestDetails { maybe_model_title?: string; maybe_raw_inference_text?: string; + + maybe_live_portrait_details?: LivePortraitDetails; } export interface StatusDetails { @@ -42,6 +44,11 @@ export interface ResultDetails { maybe_public_bucket_media_path?: string; } +export type LivePortraitDetails = { + source_media_file_token: string; + face_driver_media_file_token: string; +}; + export interface GetModelInferenceJobStatusErrorResponse { success: boolean; } diff --git a/src/website/packages/components/src/jobs/InferenceJob.ts b/src/website/packages/components/src/jobs/InferenceJob.ts index 41e07893bc..50b949d0ca 100644 --- a/src/website/packages/components/src/jobs/InferenceJob.ts +++ b/src/website/packages/components/src/jobs/InferenceJob.ts @@ -1,4 +1,7 @@ -import { ModelInferenceJobStatus } from "../api/model_inference/GetModelInferenceJobStatus"; +import { + LivePortraitDetails, + ModelInferenceJobStatus, +} from "../api/model_inference/GetModelInferenceJobStatus"; import { JobState, jobStateFromString } from "./JobStates"; // Type of inference job (specified by the frontend, not backend) @@ -29,6 +32,7 @@ export type JobListOptions = FrontendInferenceJobType | AllInferenceJobs; export class InferenceJob { // PK jobToken: string; + createdAt: Date; frontendJobType: FrontendInferenceJobType; @@ -43,6 +47,7 @@ export class InferenceJob { maybeModelToken?: string; maybeModelTitle?: string; maybeRawInferenceText?: string; + maybeLivePortraitDetails?: LivePortraitDetails; // Result maybeResultType: string | undefined | null; @@ -53,6 +58,7 @@ export class InferenceJob { constructor( // PK jobToken: string, + createdAt: Date, // Frontend state frontendJobType: FrontendInferenceJobType, // Status @@ -65,6 +71,7 @@ export class InferenceJob { maybeModelToken: string | undefined = undefined, maybeModelTitle: string | undefined = undefined, maybeRawInferenceText: string | undefined = undefined, + maybeLivePortraitDetails: LivePortraitDetails | undefined = undefined, // Result maybeResultEntityType: string | undefined | null = null, maybeResultEntityToken: string | undefined | null = null, @@ -72,6 +79,7 @@ export class InferenceJob { maybeFailureCategory: string | undefined | null = null ) { this.jobToken = jobToken; + this.createdAt = createdAt; this.frontendJobType = frontendJobType; this.jobState = jobStateFromString(status); this.maybeExtraStatusDescription = maybeExtraStatusDescription; @@ -90,6 +98,8 @@ export class InferenceJob { this.maybeRawInferenceText = maybeRawInferenceText; + this.maybeLivePortraitDetails = maybeLivePortraitDetails; + if (!!maybeResultEntityType) { this.maybeResultType = maybeResultEntityType; } @@ -110,6 +120,7 @@ export class InferenceJob { ): InferenceJob { return new InferenceJob( response.job_token, + response.created_at, frontendJobType, response.status.status, response.status.maybe_extra_status_description || null, @@ -119,6 +130,7 @@ export class InferenceJob { response.request.maybe_model_token, response.request.maybe_model_title, response.request.maybe_raw_inference_text, + response.request.maybe_live_portrait_details, response.maybe_result?.entity_type, response.maybe_result?.entity_token, response.maybe_result?.maybe_public_bucket_media_path, diff --git a/src/website/packages/fakeyou.com/src/components/common/Card/VideoCard/VideoCard.tsx b/src/website/packages/fakeyou.com/src/components/common/Card/VideoCard/VideoCard.tsx index 1abd4a06b6..7396d065eb 100644 --- a/src/website/packages/fakeyou.com/src/components/common/Card/VideoCard/VideoCard.tsx +++ b/src/website/packages/fakeyou.com/src/components/common/Card/VideoCard/VideoCard.tsx @@ -145,6 +145,8 @@ export default function VideoCard({ const getLabel = (data: any) => { if (data.origin_product_category === "face_animator") { return "Lipsync"; + } else if (data.origin_product_category === "face_mirror") { + return "Live Portrait"; } else if (data.origin_product_category === "workflow") { return "Workflow"; } else if (data.origin_product_category === "unknown") { diff --git a/src/website/packages/fakeyou.com/src/hooks/useInferenceJobsPolling.tsx b/src/website/packages/fakeyou.com/src/hooks/useInferenceJobsPolling.tsx index 8eb9d51192..12a5ba85f0 100644 --- a/src/website/packages/fakeyou.com/src/hooks/useInferenceJobsPolling.tsx +++ b/src/website/packages/fakeyou.com/src/hooks/useInferenceJobsPolling.tsx @@ -40,6 +40,8 @@ const JobCategoryToType = (jobCategory: string) => { return FrontendInferenceJobType.ConvertFbxtoGltf; case "convert_bvh_to_workflow": return FrontendInferenceJobType.EngineComposition; + case "live_portrait": + return FrontendInferenceJobType.LivePortrait; default: return FrontendInferenceJobType.Unknown; } @@ -95,23 +97,23 @@ export default function useInferenceJobsPolling({ frontendJobType: FrontendInferenceJobType ) => { const categoryArray = categoryMap.get(frontendJobType) || []; - - if (user) { - categoryMap.set(frontendJobType, [ - ...categoryArray, - updatedJob, - ]); - } else { - const inArray = categoryArray.find((job,i) => job.jobToken === updatedJob.jobToken ); + if (user) { + categoryMap.set(frontendJobType, [...categoryArray, updatedJob]); + } else { + const inArray = categoryArray.find( + (job, i) => job.jobToken === updatedJob.jobToken + ); if (!inArray) { - categoryMap.set(frontendJobType, [ - updatedJob, - ...categoryArray, - ]); + categoryMap.set(frontendJobType, [updatedJob, ...categoryArray]); } else { - categoryMap.set(frontendJobType,categoryArray.map((job,i) => job.jobToken === updatedJob.jobToken ? updatedJob : job) ) + categoryMap.set( + frontendJobType, + categoryArray.map((job, i) => + job.jobToken === updatedJob.jobToken ? updatedJob : job + ) + ); } } }; @@ -150,7 +152,10 @@ export default function useInferenceJobsPolling({ } }); - const noSessionJobs = async (currentQueue: InferenceJob[], currentCategoryMap: CategoryMap) => { + const noSessionJobs = async ( + currentQueue: InferenceJob[], + currentCategoryMap: CategoryMap + ) => { let categoryMap = new Map(currentCategoryMap); Promise.all( currentQueue.map(async (job: InferenceJob) => { @@ -165,18 +170,17 @@ export default function useInferenceJobsPolling({ } ); }) - ).then((updatedJobs) => { + ).then(updatedJobs => { if (updatedJobs.length) { - updateState(updatedJobs, categoryMap) + updateState(updatedJobs, categoryMap); } }); - }; const onTick = async ({ eventProps: { inferenceJobs: currentQueue, byCategory: currentCategoryMap }, }: { - eventProps: { inferenceJobs: InferenceJob[], byCategory: CategoryMap }; + eventProps: { inferenceJobs: InferenceJob[]; byCategory: CategoryMap }; }) => { if (user) { sessionJobs(); @@ -187,14 +191,15 @@ export default function useInferenceJobsPolling({ const enqueueInferenceJob = ( jobToken: string, - frontendJobType: FrontendInferenceJobType + frontendJobType: FrontendInferenceJobType, + createdAt: Date ) => { onTick({ eventProps: { byCategory, inferenceJobs: [] } }); if (user) { // reserving this space for later uses } else { keepAliveSet(false); - const newJob = new InferenceJob(jobToken, frontendJobType); + const newJob = new InferenceJob(jobToken, createdAt, frontendJobType); inferenceJobsSet([newJob, ...(inferenceJobs || [])]); } @@ -207,7 +212,9 @@ export default function useInferenceJobsPolling({ clearJobsStatusSet(FetchStatus.in_progress); DismissFinishedJobs("", {}).then((res: DismissFinishedJobsResponse) => { if (res.success) { - onTick({ eventProps: { byCategory: newJobCategoryMap(), inferenceJobs: [] } }); + onTick({ + eventProps: { byCategory: newJobCategoryMap(), inferenceJobs: [] }, + }); keepAliveSet(true); clearJobsStatusSet(FetchStatus.ready); } diff --git a/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/LivePortrait.scss b/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/LivePortrait.scss index 2b09f5ac02..0fc92527bf 100644 --- a/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/LivePortrait.scss +++ b/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/LivePortrait.scss @@ -138,3 +138,35 @@ } } } + +.lp-jobs-list { + background-color: lighten($panel, 5%); + border-radius: 0.5rem; + transition: all 0.2s; + padding: 10px; + cursor: pointer; + align-items: center; + display: flex; + gap: 12px; + border: 2px solid transparent; + + &:hover { + background-color: lighten($panel, 10%); + border: 2px solid lighten($panel, 15%); + } + + img { + border: lighten($panel, 15%) solid 2px; + background-color: rgba($white, 0.06); + } + + &.no-hover { + background-color: lighten($panel, 3%) !important; + + &:hover { + cursor: default; + background-color: lighten($panel, 3%) !important; + border: 2px solid lighten($panel, 3%) !important; + } + } +} diff --git a/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/LivePortrait.tsx b/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/LivePortrait.tsx index 49572e88c4..9da9409308 100644 --- a/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/LivePortrait.tsx +++ b/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/LivePortrait.tsx @@ -33,7 +33,10 @@ import Tippy from "@tippyjs/react"; import "tippy.js/dist/tippy.css"; import { v4 as uuidv4 } from "uuid"; import { useInferenceJobs, useModal } from "hooks"; -import { FrontendInferenceJobType } from "@storyteller/components/src/jobs/InferenceJob"; +import { + FrontendInferenceJobType, + InferenceJob, +} from "@storyteller/components/src/jobs/InferenceJob"; import { SessionSubscriptionsWrapper } from "@storyteller/components/src/session/SessionSubscriptionsWrapper"; import { AITools } from "components/marketing"; import { EntityInput } from "components/entities"; @@ -42,6 +45,7 @@ import LoadingSpinner from "components/common/LoadingSpinner"; import SessionLpInferenceResultsList from "./SessionLpInferenceResultsList"; import { GetMedia } from "@storyteller/components/src/api/media_files/GetMedia"; import { useLocation } from "react-router-dom"; +import { LivePortraitDetails } from "@storyteller/components/src/api/model_inference/GetModelInferenceJobStatus"; interface LivePortraitProps { sessionSubscriptionsWrapper: SessionSubscriptionsWrapper; @@ -50,7 +54,11 @@ interface LivePortraitProps { interface GeneratedVideo { sourceIndex: number; motionIndex: number; + sourceToken: string; + motionToken: string; videoSrc: string; + jobToken: string; + createdAt: Date; } export default function LivePortrait({ @@ -70,7 +78,6 @@ export default function LivePortrait({ y: 0, }); const hasPremium = sessionSubscriptionsWrapper.hasPaidFeatures(); - const [videoSrc, setVideoSrc] = useState(""); const [generatedVideoSrc, setGeneratedVideoSrc] = useState(""); const [sourceTokens, setSourceTokens] = useState([ "m_2xrse9799wvy8hkv8tbxqxct8089t7", @@ -88,6 +95,10 @@ export default function LivePortrait({ const [isUserUploaded, setIsUserUploaded] = useState(false); const [isGenerating, setIsGenerating] = useState(false); const [generatedVideos, setGeneratedVideos] = useState([]); + const [jobProcessedTokens, setJobProcessedTokens] = useState([]); + const [currentlyGeneratingList, setCurrentlyGeneratingList] = useState< + { sourceIndex: number; motionIndex: number }[] + >([]); const location = useLocation(); @@ -103,20 +114,61 @@ export default function LivePortrait({ [] ); + const handleJobClick = (job: InferenceJob) => { + const livePortraitDetails = job.maybeLivePortraitDetails; + if (livePortraitDetails) { + const sourceIndex = sourceTokens.indexOf( + livePortraitDetails.source_media_file_token + ); + const motionIndex = motionTokens.indexOf( + livePortraitDetails.face_driver_media_file_token + ); + + if (sourceIndex !== -1) { + setSelectedSourceIndex(sourceIndex); + } else { + setSourceTokens(prevTokens => [ + ...prevTokens, + livePortraitDetails.source_media_file_token, + ]); + setSelectedSourceIndex(sourceTokens.length); + } + + if (motionIndex !== -1) { + setSelectedMotionIndex(motionIndex); + } else { + setMotionTokens(prevTokens => [ + ...prevTokens, + livePortraitDetails.face_driver_media_file_token, + ]); + setSelectedMotionIndex(motionTokens.length); + } + } + }; + const handleSourceSelect = (index: number) => { - console.log("Thumbnail clicked:", index); setIsUserUploaded(index >= numberOfInitialSourceTokens); setSelectedSourceIndex(index); }; const handleMotionSelect = (index: number) => { - console.log("Thumbnail clicked:", index); + setIsUserUploaded(index >= numberOfInitialMotionTokens); setSelectedMotionIndex(index); }; const enqueueClick = () => { - setIsEnqueuing(true); + // Clear the generated video when reanimating + setGeneratedVideoSrc(""); setIsGenerating(true); + + // Add the current source and motion combination to the generating list + setCurrentlyGeneratingList(prevList => [ + ...prevList, + { sourceIndex: selectedSourceIndex, motionIndex: selectedMotionIndex }, + ]); + + setIsEnqueuing(true); + EnqueueFaceMirror("", { creator_set_visibility: visibility, face_driver_media_file_token: motionTokens[selectedMotionIndex], @@ -128,7 +180,7 @@ export default function LivePortrait({ if (res.success && res.inference_job_token) { enqueueInferenceJob( res.inference_job_token, - FrontendInferenceJobType.TextToSpeech, + FrontendInferenceJobType.LivePortrait, false ); @@ -136,43 +188,29 @@ export default function LivePortrait({ } else { console.error("Failed to enqueue job", res); setIsGenerating(false); + // Remove the combination from currentlyGeneratingList if fail + setCurrentlyGeneratingList(prevList => + prevList.filter( + gen => + gen.sourceIndex !== selectedSourceIndex || + gen.motionIndex !== selectedMotionIndex + ) + ); } setIsEnqueuing(false); }); }; const renderVideoOrPlaceholder = () => { - if (generatedVideoSrc) { - return ( - - ); - } else if (videoSrc) { - return ( - - ); - } else if (isGenerating) { + // Check if the current combination is being generated + const isCurrentlyGenerating = currentlyGeneratingList.some( + gen => + gen.sourceIndex === selectedSourceIndex && + gen.motionIndex === selectedMotionIndex + ); + + if (isCurrentlyGenerating) { + // Show "Generating Video..." if the combination is currently generating return (
); + } else if (generatedVideoSrc) { + // Show dynamically generated video if available + return ( + + ); + } else if (getPrecomputedVideoSrc()) { + // Show precomputed video if it exists + return ( + + ); } else { + // Show "Click to Animate" if nothing is generating or precomputed return (
{ - if (isUserUploaded) { - setVideoSrc(""); - } else { - const newSrc = getPrecomputedVideoSrc(); - setVideoSrc(newSrc); - } - }, [ - selectedSourceIndex, - selectedMotionIndex, - isUserUploaded, - getPrecomputedVideoSrc, - ]); - useEffect(() => { if (userSourceToken) { setSourceTokens(prevTokens => { @@ -307,9 +364,9 @@ export default function LivePortrait({ }; const motionInputProps = { - accept: ["image", "video"], + accept: ["video"], className: "w-100", - label: "Upload Source Media (Image or Video)", + label: "Upload Motion Reference Video", onChange: ({ target }: { target: any }) => { setUserMotionToken(target.value); close(); @@ -347,10 +404,17 @@ export default function LivePortrait({ const handleJobTokens = async ( maybeResultToken: string, - jobToken: string + jobToken: string, + createdAt: Date, + livePortraitDetails?: LivePortraitDetails ) => { - console.log("Maybe Result Token:", maybeResultToken); - console.log("Job Token:", jobToken); + if (jobProcessedTokens.includes(jobToken)) { + return; + } + + if (!livePortraitDetails) { + return; + } const response = await GetMedia(maybeResultToken, {}); @@ -363,31 +427,69 @@ export default function LivePortrait({ response.media_file.public_bucket_path ); - const isSourceValid = selectedSourceIndex < sourceTokens.length; - const isMotionValid = selectedMotionIndex < motionTokens.length; + const sourceIndex = sourceTokens.indexOf( + livePortraitDetails.source_media_file_token + ); + const motionIndex = motionTokens.indexOf( + livePortraitDetails.face_driver_media_file_token + ); - if (isSourceValid && isMotionValid) { - const generatedVideo = { - sourceIndex: selectedSourceIndex, - motionIndex: selectedMotionIndex, - videoSrc: mediaLink, - }; + const newGeneratedVideo = { + sourceIndex, + motionIndex, + sourceToken: livePortraitDetails.source_media_file_token, + motionToken: livePortraitDetails.face_driver_media_file_token, + videoSrc: mediaLink, + jobToken, // Store the job token with the video + createdAt, + }; - // Update the state with the new combination - setGeneratedVideos(prevGeneratedVideos => [ - ...prevGeneratedVideos, - generatedVideo, - ]); + setGeneratedVideos(prevGeneratedVideos => { + const existingVideoIndex = prevGeneratedVideos.findIndex( + video => + video.sourceIndex === newGeneratedVideo.sourceIndex && + video.motionIndex === newGeneratedVideo.motionIndex + ); - // Only set generatedVideoSrc if it matches the current selected indices - if ( - selectedSourceIndex === generatedVideo.sourceIndex && - selectedMotionIndex === generatedVideo.motionIndex - ) { - setGeneratedVideoSrc(mediaLink); - setIsGenerating(false); + if (existingVideoIndex !== -1) { + const existingVideo = prevGeneratedVideos[existingVideoIndex]; + + // Replace the existing video only if the new jobToken is more recent + if ( + new Date(existingVideo.createdAt).getTime() < + new Date(newGeneratedVideo.createdAt).getTime() + ) { + const updatedVideos = [...prevGeneratedVideos]; + updatedVideos[existingVideoIndex] = newGeneratedVideo; + return updatedVideos; + } + return prevGeneratedVideos; + } else { + // Add the new video if it doesn't already exist + return [...prevGeneratedVideos, newGeneratedVideo]; } + }); + + // Set the new video for display if it's the latest + if ( + selectedSourceIndex === newGeneratedVideo.sourceIndex && + selectedMotionIndex === newGeneratedVideo.motionIndex + ) { + setGeneratedVideoSrc(mediaLink); + setIsGenerating(false); + + // Remove from generating list as job is complete + setCurrentlyGeneratingList(prevList => + prevList.filter( + gen => + gen.sourceIndex !== newGeneratedVideo.sourceIndex || + gen.motionIndex !== newGeneratedVideo.motionIndex + ) + ); } + + // Mark the job as processed + setJobProcessedTokens(prevTokens => [...prevTokens, jobToken]); } else { console.error( "Failed to retrieve media or media has no public bucket path", @@ -398,43 +500,45 @@ export default function LivePortrait({ }; useEffect(() => { - const getVideoForCombination = ( + const getLatestVideoForCombination = ( sourceIndex: number, motionIndex: number ) => { - const video = generatedVideos.find( + // Filter generated videos by sourceIndex and motionIndex + const matchingVideos = generatedVideos.filter( video => video.sourceIndex === sourceIndex && video.motionIndex === motionIndex ); - return video ? video.videoSrc : null; + + // Sort the videos by createdAt to get the latest one + const sortedVideos = matchingVideos.sort( + (a, b) => + new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime() + ); + + // Return the latest video source if it exists + return sortedVideos.length > 0 ? sortedVideos[0].videoSrc : null; }; - const existingVideoSrc = getVideoForCombination( + const latestVideoSrc = getLatestVideoForCombination( selectedSourceIndex, selectedMotionIndex ); - if (existingVideoSrc) { - // Show the generated video that matches the current selection - setGeneratedVideoSrc(existingVideoSrc); + if (latestVideoSrc) { + setGeneratedVideoSrc(latestVideoSrc); setIsGenerating(false); } else if (!isUserUploaded) { - // Show the precomputed video if no user-uploaded content is present - const newSrc = getPrecomputedVideoSrc(); - setVideoSrc(newSrc); setGeneratedVideoSrc(""); setIsGenerating(false); } else { - // Clear the videoSrc and generatedVideoSrc if no match is found - setVideoSrc(""); setGeneratedVideoSrc(""); } }, [ selectedSourceIndex, selectedMotionIndex, - isUserUploaded, generatedVideos, - getPrecomputedVideoSrc, + isUserUploaded, ]); useEffect(() => { @@ -507,15 +611,53 @@ export default function LivePortrait({ } }, [location.search]); + useEffect(() => { + // When switching source or motion indexes, check if the current combination is generating + const isCurrentlyGenerating = currentlyGeneratingList.some( + gen => + gen.sourceIndex === selectedSourceIndex && + gen.motionIndex === selectedMotionIndex + ); + + if (isCurrentlyGenerating) { + setIsGenerating(true); + } else if (generatedVideoSrc) { + setIsGenerating(false); + } else if (!isUserUploaded) { + setIsGenerating(false); + } else { + setIsGenerating(false); + } + }, [ + selectedSourceIndex, + selectedMotionIndex, + currentlyGeneratingList, + generatedVideoSrc, + isUserUploaded, + ]); + + const handleDownloadClick = () => { + if (generatedVideoSrc) { + const link = document.createElement("a"); + link.href = generatedVideoSrc; + link.download = "output_video.mp4"; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + } else { + console.error("No video source available for download"); + } + }; + return ( <> - +

Live Portrait

-

+

Use AI to transfer facial expressions, audio, and vocals from one face video to an image or video.

@@ -523,7 +665,7 @@ export default function LivePortrait({
@@ -656,12 +801,27 @@ export default function LivePortrait({
-
+
diff --git a/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/SessionLpInferenceResultsList.tsx b/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/SessionLpInferenceResultsList.tsx index fa5471239d..08602b61fa 100644 --- a/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/SessionLpInferenceResultsList.tsx +++ b/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/SessionLpInferenceResultsList.tsx @@ -1,24 +1,46 @@ -import React, { useEffect, useMemo, useRef } from "react"; -import { InferenceJob } from "@storyteller/components/src/jobs/InferenceJob"; +import React, { useEffect, useMemo, useRef, useState } from "react"; +import { + FrontendInferenceJobType, + InferenceJob, +} from "@storyteller/components/src/jobs/InferenceJob"; import { useInferenceJobs } from "hooks"; -import { Panel } from "components/common"; +import { LivePortraitDetails } from "@storyteller/components/src/api/model_inference/GetModelInferenceJobStatus"; +import { GetMedia } from "@storyteller/components/src/api/media_files/GetMedia"; +import { BucketConfig } from "@storyteller/components/src/api/BucketConfig"; +import { JobState } from "@storyteller/components/src/jobs/JobStates"; +import moment from "moment"; +import { Link } from "react-router-dom"; +import LoadingSpinner from "components/common/LoadingSpinner"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { faClipboardList } from "@fortawesome/pro-solid-svg-icons"; +import { faArrowTurnUp } from "@fortawesome/pro-solid-svg-icons"; interface SessionLpInferenceResultsListProps { sessionSubscriptionsWrapper: any; - onJobTokens: (maybeResultToken: string, jobToken: string) => void; + onJobTokens: ( + maybeResultToken: string, + jobToken: string, + createdAt: Date, + maybeLivePortraitDetails?: LivePortraitDetails + ) => void; + addSourceToken: (token: string) => void; + addMotionToken: (token: string) => void; + onJobClick: (job: InferenceJob) => void; } export default function SessionLpInferenceResultsList({ sessionSubscriptionsWrapper, onJobTokens, + addMotionToken, + addSourceToken, + onJobClick, }: SessionLpInferenceResultsListProps) { const { inferenceJobsByCategory } = useInferenceJobs(); const hasInitialized = useRef(false); const livePortraitJobs = useMemo(() => { - return inferenceJobsByCategory.get(0) || []; + return ( + inferenceJobsByCategory.get(FrontendInferenceJobType.LivePortrait) || [] + ); }, [inferenceJobsByCategory]); const lastProcessedJobToken = useRef(null); @@ -32,46 +54,142 @@ export default function SessionLpInferenceResultsList({ job.maybeResultToken && job.jobToken !== lastProcessedJobToken.current ) { - lastProcessedJobToken.current = job.jobToken; - onJobTokens(job.maybeResultToken, job.jobToken); + const livePortraitDetails = job.maybeLivePortraitDetails; + + if (livePortraitDetails) { + const { source_media_file_token, face_driver_media_file_token } = + livePortraitDetails; + + addSourceToken(source_media_file_token); + + addMotionToken(face_driver_media_file_token); + + lastProcessedJobToken.current = job.jobToken; + onJobTokens( + job.maybeResultToken, + job.jobToken, + job.createdAt, + livePortraitDetails + ); + } } }); } - }, [livePortraitJobs, onJobTokens]); + }, [livePortraitJobs, onJobTokens, addSourceToken, addMotionToken]); + + const [mediaSrc, setMediaSrc] = useState<{ [key: string]: string }>({}); + + useEffect(() => { + const fetchMedia = async (token: string) => { + try { + const response = await GetMedia(token, {}); + const publicBucketPath = response.media_file?.public_bucket_path || ""; + console.log(`Fetched media for token ${token}: ${publicBucketPath}`); + setMediaSrc(prev => ({ ...prev, [token]: publicBucketPath })); + } catch (error) { + console.error("Error fetching media:", error); + } + }; + + livePortraitJobs.forEach((job: InferenceJob) => { + const token = job.maybeLivePortraitDetails?.source_media_file_token; + if (token && !mediaSrc[token]) { + fetchMedia(token); + } + }); + }, [livePortraitJobs, mediaSrc]); + + const jobStateTextMap: { [key in JobState]: string } = { + [JobState.UNKNOWN]: "Unknown", + [JobState.PENDING]: "Pending", + [JobState.STARTED]: "Generating", + [JobState.COMPLETE_SUCCESS]: "Completed", + [JobState.COMPLETE_FAILURE]: "Completed (Failure)", + [JobState.ATTEMPT_FAILED]: "Attempt Failed", + [JobState.DEAD]: "Dead", + [JobState.CANCELED_BY_USER]: "Canceled by User", + }; const jobContent = (
{livePortraitJobs.length > 0 ? ( -
- {livePortraitJobs.map((job: InferenceJob, key: number) => ( -
{ - if (job.maybeResultToken) { - onJobTokens(job.maybeResultToken, job.jobToken); - } - }} - > - {job.maybeResultToken} - {job.jobToken} -
- ))} +
+ {livePortraitJobs + .slice(0, 4) + .map((job: InferenceJob, key: number) => ( +
{ + onJobClick(job); + }} + className="col-12 col-lg-3" + > +
+
+ +
+
+
+ {(job.jobState === JobState.PENDING || + job.jobState === JobState.STARTED) && ( + + )} + + {jobStateTextMap[job.jobState as JobState]} + +
+ + {moment(job.createdAt).fromNow()} + +
+ {job.maybeResultToken ? ( + + More Details + + ) : ( +
Waiting...
+ )} +
+
+ +
+
+ ))}
) : ( -
- -
-

title

-

subtitle

-
+
+ + Your latest live portrait generations will appear here. +
)}
); - return ( - - {jobContent} - - ); + return <>{jobContent}; } diff --git a/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/ThumbnailItem.tsx b/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/ThumbnailItem.tsx index 04abda69b2..e7e8f14bf6 100644 --- a/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/ThumbnailItem.tsx +++ b/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/ThumbnailItem.tsx @@ -19,15 +19,30 @@ const ThumbnailItem: React.FC = ({ const [isThumbReady, setIsThumbReady] = useState(false); useEffect(() => { + let isMounted = true; const checkImage = () => { const img = new Image(); img.src = poster + "-thumb.jpg"; - img.onload = () => setIsThumbReady(true); - img.onerror = () => setTimeout(checkImage, 1000); // Retry after 1 second if the image is not available + img.onload = () => { + if (isMounted) { + setIsThumbReady(true); + } + }; + img.onerror = () => { + if (isMounted && !isThumbReady) { + setTimeout(checkImage, 1000); // Retry after 1 second if the image is not available + } + }; }; - checkImage(); - }, [poster]); + if (poster && mediaType === "video") { + checkImage(); + } + + return () => { + isMounted = false; + }; + }, [poster, mediaType, isThumbReady]); return (
diff --git a/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/ThumbnailMediaPicker.tsx b/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/ThumbnailMediaPicker.tsx index ee7eaf3361..47e4fb59d4 100644 --- a/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/ThumbnailMediaPicker.tsx +++ b/src/website/packages/fakeyou.com/src/v2/view/pages/live_portrait/ThumbnailMediaPicker.tsx @@ -133,9 +133,10 @@ const ThumbnailMediaPicker: React.FC = React.memo( [isCropping] ); - const onCropComplete = useCallback((croppedArea, croppedAreaPixels) => { - console.log(croppedArea, croppedAreaPixels); - }, []); + const onCropComplete = useCallback( + (_croppedArea, _croppedAreaPixels) => {}, + [] + ); const handleNextPage = () => { setCurrentPage(prevPage => @@ -161,7 +162,9 @@ const ThumbnailMediaPicker: React.FC = React.memo( ) : ( diff --git a/src/website/packages/fakeyou.com/src/v2/view/pages/media/MediaPageSwitch.tsx b/src/website/packages/fakeyou.com/src/v2/view/pages/media/MediaPageSwitch.tsx index 028b0b06cc..b9a642cc09 100644 --- a/src/website/packages/fakeyou.com/src/v2/view/pages/media/MediaPageSwitch.tsx +++ b/src/website/packages/fakeyou.com/src/v2/view/pages/media/MediaPageSwitch.tsx @@ -95,6 +95,7 @@ export default function MediaPageSwitch() { pageTitle = mediaFile?.maybe_title || "Audio File"; break; case MediaFileType.Video: + case MediaFileType.Mp4: pageTitle = mediaFile?.maybe_title || "Video File"; break; case MediaFileType.Image: @@ -201,6 +202,7 @@ export default function MediaPageSwitch() { const MediaSwitch = () => { switch (mediaFile?.media_type) { case MediaFileType.Video: + case MediaFileType.Mp4: return ; default: return ; diff --git a/src/website/packages/fakeyou.com/src/v2/view/pages/profile/profile_view/tabs/VideosTab.tsx b/src/website/packages/fakeyou.com/src/v2/view/pages/profile/profile_view/tabs/VideosTab.tsx index 199306e1e4..9f3912c39d 100644 --- a/src/website/packages/fakeyou.com/src/v2/view/pages/profile/profile_view/tabs/VideosTab.tsx +++ b/src/website/packages/fakeyou.com/src/v2/view/pages/profile/profile_view/tabs/VideosTab.tsx @@ -19,7 +19,7 @@ export default function VideosTab({ username }: { username: string }) { const gridContainerRef = useRef(null); const [showMasonryGrid, setShowMasonryGrid] = useState(true); const [mediaType, mediaTypeSet] = useState( - urlQueries.get("filter_media_type") || "video" + urlQueries.get("filter_media_type") || "video,mp4" ); const [list, listSet] = useState([]); const media = useListContent({