cameraStartedSet(true),
- ref: webcamRef,
+ onUserMedia: () => camera.startedSet(true),
+ ref: camera.ref,
videoConstraints: {
width: 512,
height: 512,
@@ -196,40 +119,16 @@ export default function CameraCapture({
},
}}
/>
- {cameraStarted ? (
+ {camera.started ? (
-
+ />
{isMobile && (
;
+ reset: () => void;
+ started: boolean;
+ startedSet: (startedState: boolean) => void;
+ toggle: (e: RecordToggleEvent) => void;
+}
+
+export default function useCameraState(autoCapture = false): CameraState {
+ const ref = useRef(null);
+ const mediaRecorderRef = useRef(null);
+
+ const [blob, blobSet] = useState(null);
+ const [started, startedSet] = useState(false);
+ const [counter, counterSet] = useState(0);
+ const [capturing, capturingSet] = useState(false);
+ const [recordedChunks, recordedChunksSet] = useState([]);
+ const [autoTriggered, autoTriggeredSet] = useState(false);
+
+ // useEffect will only update when watching the stream value directly
+
+ const stream = ref?.current?.stream;
+
+ const handleDataAvailable = useCallback(
+ ({ data }) => {
+ if (data.size > 0) {
+ recordedChunksSet(prev => prev.concat(data));
+ }
+ },
+ [recordedChunksSet]
+ );
+
+ const startCapture = useCallback(
+ current => {
+ capturingSet(true);
+
+ if (current && current.stream) {
+ mediaRecorderRef.current = new MediaRecorder(current.stream, {
+ mimeType: "video/mp4",
+ });
+
+ mediaRecorderRef.current.addEventListener(
+ "dataavailable",
+ handleDataAvailable
+ );
+ mediaRecorderRef.current.start();
+ }
+ },
+ [handleDataAvailable]
+ );
+
+ const toggle = ({ target }: RecordToggleEvent) => {
+ if (target.value && ref.current) {
+ startCapture(ref.current);
+ } else {
+ capturingSet(false);
+
+ if (mediaRecorderRef.current !== null) {
+ mediaRecorderRef.current.stop();
+ }
+ }
+ };
+
+ const reset = () => {
+ startedSet(false);
+ blobSet(null);
+ recordedChunksSet([]);
+ counterSet(0);
+ };
+
+ useInterval({
+ eventProps: { capturing, counter },
+ interval: 1000,
+ locked: !capturing,
+ onTick: () => {
+ if (capturing) {
+ counterSet(currentCounter => currentCounter + 1);
+ }
+ },
+ });
+
+ useEffect(() => {
+ if (!blob && !capturing && recordedChunks.length) {
+ blobSet(
+ new Blob(recordedChunks, {
+ type: "video/mp4",
+ })
+ );
+ }
+
+ if (autoCapture && !capturing && !autoTriggered && ref.current && stream) {
+ autoTriggeredSet(true);
+ startCapture(ref.current);
+ }
+ }, [
+ autoCapture,
+ autoTriggered,
+ blob,
+ capturing,
+ recordedChunks,
+ startCapture,
+ stream,
+ ]);
+
+ return { blob, capturing, counter, ref, reset, started, startedSet, toggle };
+}
diff --git a/src/website/packages/fakeyou.com/src/hooks/useMedia/useMedia.tsx b/src/website/packages/fakeyou.com/src/hooks/useMedia/useMedia.tsx
index d31bba251d..e327601dd8 100644
--- a/src/website/packages/fakeyou.com/src/hooks/useMedia/useMedia.tsx
+++ b/src/website/packages/fakeyou.com/src/hooks/useMedia/useMedia.tsx
@@ -9,6 +9,7 @@ import {
} from "@storyteller/components/src/api/prompts/GetPrompts";
import { DeleteMedia } from "@storyteller/components/src/api/media_files/DeleteMedia";
import { FetchStatus } from "@storyteller/components/src/api/_common/SharedFetchTypes";
+import { BucketConfig } from "@storyteller/components/src/api/BucketConfig";
export default function useMedia({
debug = "",
@@ -41,6 +42,12 @@ export default function useMedia({
status === FetchStatus.in_progress ||
writeStatus === FetchStatus.in_progress;
+ // const bucketConfig = new BucketConfig();
+
+ const bucketUrl = media?.public_bucket_path
+ ? new BucketConfig().getGcsUrl(media.public_bucket_path)
+ : "";
+
useEffect(() => {
// this condidition handles all media file fetches
// it is triggered when status is ready and there is a media token but no media
@@ -80,6 +87,7 @@ export default function useMedia({
return {
busy,
+ bucketUrl,
media,
mediaFile: media,
mediaSet,
diff --git a/src/website/packages/fakeyou.com/src/scss/site-refresh.scss b/src/website/packages/fakeyou.com/src/scss/site-refresh.scss
index 255000f354..26e9091db7 100644
--- a/src/website/packages/fakeyou.com/src/scss/site-refresh.scss
+++ b/src/website/packages/fakeyou.com/src/scss/site-refresh.scss
@@ -63,6 +63,22 @@ styles with important tags:
backdrop-filter: blur(5px);
}
+@mixin video-controls {
+ @include video-tint;
+ position: absolute;
+ bottom: 1rem;
+ border-radius: 0.5rem;
+ padding: 0.5rem 0.5rem 0.25rem;
+ left: 0;
+ right: 0;
+ margin: auto;
+ width: 50%;
+ display: flex;
+ justify-content: center;
+ gap: 1rem;
+ align-items: center;
+}
+
.fakeyou-refresh {
//- BS5 classnames -//
.object-fit-cover {
diff --git a/src/website/packages/fakeyou.com/src/v2/view/PageContainer.tsx b/src/website/packages/fakeyou.com/src/v2/view/PageContainer.tsx
index 125c1234f5..83a9913daa 100644
--- a/src/website/packages/fakeyou.com/src/v2/view/PageContainer.tsx
+++ b/src/website/packages/fakeyou.com/src/v2/view/PageContainer.tsx
@@ -132,6 +132,7 @@ import SignUpSuccessPage from "./pages/beta_key/SignUpSuccessPage";
import { StudioMobileCheckPage } from "./pages/landing/storyteller/PostlaunchLanding/StudioMobileCheckPage";
import { UploadNewTtsModelPage } from "./pages/upload/UploadNewTtsModelPage";
import LivePortrait from "./pages/live_portrait/LivePortrait";
+import CameraLivePortrait from "./pages/live_portrait/CameraLivePortrait";
import { BetaLipSyncForm } from "./pages/beta_products/BetaLipSyncForm";
import { BetaLipSyncPage } from "./pages/beta_products/BetaLipSyncPage";
import { ToolsTestPage } from "./pages/tools_test/ToolsTestPage";
@@ -762,6 +763,10 @@ class PageContainer extends React.Component<
/>
+
+
+
+
{/* Route for initial voice creation */}
("public");
+ const hasPremium = sessionSubscriptions?.hasPaidFeatures();
+ const [generatedVideoSrc, setGeneratedVideoSrc] = useState("");
+ const [sourceTokens, setSourceTokens] = useState([
+ ...PRECOMPUTED_SOURCE_TOKENS,
+ ]);
+ const [motionTokens, setMotionTokens] = useState([
+ ...PRECOMPUTED_DRIVER_TOKENS,
+ ]);
+ const numberOfInitialSourceTokensRef = useRef(sourceTokens.length);
+ const numberOfInitialSourceTokens = numberOfInitialSourceTokensRef.current;
+ const numberOfInitialMotionTokensRef = useRef(motionTokens.length);
+ const numberOfInitialMotionTokens = numberOfInitialMotionTokensRef.current;
+ const [userSourceToken, setUserSourceToken] = useState("");
+ const [userMotionToken, setUserMotionToken] = useState("");
+ const [isUserUploaded, setIsUserUploaded] = useState(false);
+ const [isGenerating, setIsGenerating] = useState(false);
+ const [generatedVideos, setGeneratedVideos] = useState([]);
+ const [jobProcessedTokens, setJobProcessedTokens] = useState([]);
+ const [currentlyGeneratingList, setCurrentlyGeneratingList] = useState<
+ CurrentlyGenerating[]
+ >([]);
+ const [jobProgress, setJobProgress] = useState({});
+ const [currentCombinationKey, setCurrentCombinationKey] = useState("");
+ const getCombinationKey = (sourceIndex: number, motionIndex: number) =>
+ `s${sourceIndex}_m${motionIndex}`;
+ const [cameraInputToken, cameraInputTokenSet] = useState("");
+
+ const location = useLocation();
+ const history = useHistory();
+
+ const handleJobClick = (job: InferenceJob) => {
+ const livePortraitDetails = job.maybeLivePortraitDetails;
+ if (livePortraitDetails) {
+ const { source_media_file_token, face_driver_media_file_token } =
+ livePortraitDetails;
+
+ const sourceIndex = sourceTokens.indexOf(source_media_file_token);
+ const motionIndex = motionTokens.indexOf(face_driver_media_file_token);
+ setSelectedSourceIndex(sourceIndex);
+ setSelectedMotionIndex(motionIndex);
+ setCurrentCombinationKey(getCombinationKey(sourceIndex, motionIndex));
+
+ // Find the video for the clicked job
+ const videoFromJob = generatedVideos.find(
+ video => video.jobToken === job.jobToken
+ );
+ if (videoFromJob) {
+ setGeneratedVideoSrc(videoFromJob.videoSrc);
+ setIsGenerating(false);
+ } else {
+ // If there's no video and the job is still in the generating list, show progress
+ const isGenerating = currentlyGeneratingList.some(
+ gen =>
+ gen.sourceIndex === sourceIndex && gen.motionIndex === motionIndex
+ );
+ if (isGenerating) {
+ const progress =
+ jobProgress[getCombinationKey(sourceIndex, motionIndex)] || 0;
+ setJobProgress({
+ ...jobProgress,
+ [getCombinationKey(sourceIndex, motionIndex)]: progress,
+ });
+ setIsGenerating(true);
+ } else {
+ setIsGenerating(false);
+ setGeneratedVideoSrc("");
+ }
+ }
+ }
+ };
+
+ const handleJobStateChange = useCallback(
+ (jobToken: string, jobState: JobState) => {
+ const currentCombinationKey = getCombinationKey(
+ selectedSourceIndex,
+ selectedMotionIndex
+ );
+
+ if (jobState === JobState.COMPLETE_FAILURE) {
+ setCurrentlyGeneratingList(prevList =>
+ prevList.filter(
+ gen =>
+ !(
+ gen.sourceIndex === selectedSourceIndex &&
+ gen.motionIndex === selectedMotionIndex
+ )
+ )
+ );
+
+ setJobProgress(prevProgress => {
+ const updatedProgress = { ...prevProgress };
+ delete updatedProgress[currentCombinationKey];
+ return updatedProgress;
+ });
+
+ setIsGenerating(false);
+ }
+ },
+ [selectedSourceIndex, selectedMotionIndex]
+ );
+
+ const handleJobProgress = (progress: number | null) => {
+ setJobProgress(prevProgress => ({
+ ...prevProgress,
+ [currentCombinationKey]: progress,
+ }));
+ };
+
+ const handleSourceSelect = (index: number) => {
+ setIsUserUploaded(index >= numberOfInitialSourceTokens);
+ setSelectedSourceIndex(index);
+ setCurrentCombinationKey(getCombinationKey(index, selectedMotionIndex));
+ };
+
+ const enqueueClick = () => {
+ if (
+ !loggedInOrModal({
+ loginMessage: "Login to finish animating your portrait",
+ signupMessage: "Sign up to finish animating your portrait",
+ })
+ ) {
+ return;
+ }
+ // Clear the generated video when reanimating
+ setGeneratedVideoSrc("");
+ setIsGenerating(true);
+
+ // Add the current source and motion combination to the generating list
+ setCurrentlyGeneratingList(prevList => [
+ ...prevList,
+ { sourceIndex: selectedSourceIndex, motionIndex: selectedMotionIndex },
+ ]);
+
+ setIsEnqueuing(true);
+
+ const combinationKey = getCombinationKey(
+ selectedSourceIndex,
+ selectedMotionIndex
+ );
+ setCurrentCombinationKey(combinationKey);
+
+ EnqueueFaceMirror("", {
+ creator_set_visibility: visibility,
+ face_driver_media_file_token: cameraInputToken,
+ maybe_crop: {
+ height: 0,
+ width: 0,
+ x: 0,
+ y: 0,
+ },
+ remove_watermark: removeWatermark,
+ source_media_file_token: sourceTokens[selectedSourceIndex],
+ uuid_idempotency_token: uuidv4(),
+ }).then((res: EnqueueFaceMirrorResponse) => {
+ if (res.success && res.inference_job_token) {
+ enqueueInferenceJob(
+ res.inference_job_token,
+ FrontendInferenceJobType.LivePortrait,
+ false
+ );
+ } else {
+ // @ts-ignore
+ window.dataLayer.push({
+ event: "enqueue_failure",
+ page: "/webcam-acting",
+ user_id: "$user_id",
+ });
+ console.error("Failed to enqueue job", res);
+ setIsGenerating(false);
+ // Remove the combination from currentlyGeneratingList if fail
+ setCurrentlyGeneratingList(prevList =>
+ prevList.filter(
+ gen =>
+ gen.sourceIndex !== selectedSourceIndex ||
+ gen.motionIndex !== selectedMotionIndex
+ )
+ );
+ }
+ setIsEnqueuing(false);
+ });
+ };
+
+ useEffect(() => {
+ if (userSourceToken) {
+ setSourceTokens(prevTokens => {
+ const tokenIndex = prevTokens.indexOf(userSourceToken);
+ if (tokenIndex !== -1) {
+ setSelectedSourceIndex(tokenIndex);
+ setIsUserUploaded(tokenIndex >= numberOfInitialSourceTokens);
+ return prevTokens;
+ } else {
+ const updatedTokens = [...prevTokens, userSourceToken];
+ setSelectedSourceIndex(updatedTokens.length - 1);
+ setIsUserUploaded(true);
+ return updatedTokens;
+ }
+ });
+ setUserSourceToken("");
+ }
+ }, [userSourceToken, numberOfInitialSourceTokens]);
+
+ useEffect(() => {
+ if (userMotionToken) {
+ setMotionTokens(prevTokens => {
+ const tokenIndex = prevTokens.indexOf(userMotionToken);
+ if (tokenIndex !== -1) {
+ setSelectedMotionIndex(tokenIndex);
+ setIsUserUploaded(tokenIndex >= numberOfInitialMotionTokens);
+ return prevTokens;
+ } else {
+ const updatedTokens = [...prevTokens, userMotionToken];
+ setSelectedMotionIndex(updatedTokens.length - 1);
+ setIsUserUploaded(true);
+ return updatedTokens;
+ }
+ });
+ setUserMotionToken("");
+ }
+ }, [userMotionToken, numberOfInitialMotionTokens]);
+
+ const handleOpenUploadSourceModal = () => {
+ open({
+ component: SourceEntityInput,
+ props: {
+ onChange: ({ target }: { target: any }) => {
+ setUserSourceToken(target.value);
+ close();
+ },
+ },
+ width: "small",
+ });
+ };
+
+ const [, setSelectedSourceMedia] = useState<{
+ [key: string]: any;
+ }>({});
+
+ const handleSelectedMediaChange = (media: any) => {
+ setSelectedSourceMedia(media);
+ };
+
+ const handleJobTokens = async (
+ maybeResultToken: string,
+ jobToken: string,
+ createdAt: Date,
+ livePortraitDetails?: LivePortraitDetails
+ ) => {
+ if (!livePortraitDetails) {
+ return;
+ }
+
+ // makes sure that it only processes each job once and exactly when needed
+ if (jobProcessedTokens.includes(jobToken)) {
+ return;
+ }
+
+ const response = await GetMedia(maybeResultToken, {});
+
+ if (
+ response &&
+ response.media_file &&
+ response.media_file.public_bucket_path
+ ) {
+ const mediaLink = new BucketConfig().getGcsUrl(
+ response.media_file.public_bucket_path
+ );
+
+ const sourceIndex = sourceTokens.indexOf(
+ livePortraitDetails.source_media_file_token
+ );
+ const motionIndex = motionTokens.indexOf(
+ livePortraitDetails.face_driver_media_file_token
+ );
+
+ const newGeneratedVideo = {
+ sourceIndex,
+ motionIndex,
+ sourceToken: livePortraitDetails.source_media_file_token,
+ motionToken: livePortraitDetails.face_driver_media_file_token,
+ videoSrc: mediaLink,
+ jobToken,
+ createdAt,
+ };
+
+ setGeneratedVideos(prevGeneratedVideos => {
+ return [
+ ...prevGeneratedVideos.filter(v => v.jobToken !== jobToken),
+ newGeneratedVideo,
+ ];
+ });
+
+ // Set the video source from the clicked job, regardless of its timestamp relative to others
+ if (
+ selectedSourceIndex === newGeneratedVideo.sourceIndex &&
+ selectedMotionIndex === newGeneratedVideo.motionIndex
+ ) {
+ setGeneratedVideoSrc(mediaLink);
+ setIsGenerating(false);
+ }
+
+ setJobProcessedTokens(prevTokens => [...prevTokens, jobToken]);
+ } else {
+ console.error(
+ "Failed to retrieve media or media has no public bucket path",
+ response
+ );
+ setIsGenerating(false);
+ setGeneratedVideoSrc("");
+ setIsEnqueuing(false);
+ }
+ };
+
+ const getLatestVideoForCombination = useCallback(
+ (sourceIndex: number, motionIndex: number) => {
+ const matchingVideos = generatedVideos.filter(
+ video =>
+ video.sourceIndex === sourceIndex && video.motionIndex === motionIndex
+ );
+ const sortedVideos = matchingVideos.sort(
+ (a, b) =>
+ new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()
+ );
+ const latestVideo =
+ sortedVideos.length > 0 ? sortedVideos[0].videoSrc : null;
+
+ return latestVideo;
+ },
+ [generatedVideos]
+ );
+
+ useEffect(() => {
+ const latestVideoSrc = getLatestVideoForCombination(
+ selectedSourceIndex,
+ selectedMotionIndex
+ );
+
+ if (latestVideoSrc) {
+ setGeneratedVideoSrc(latestVideoSrc);
+ setIsGenerating(false);
+ } else if (!isUserUploaded) {
+ setGeneratedVideoSrc("");
+ setIsGenerating(false);
+ } else {
+ setGeneratedVideoSrc("");
+ }
+ }, [
+ selectedSourceIndex,
+ selectedMotionIndex,
+ generatedVideos,
+ isUserUploaded,
+ getLatestVideoForCombination,
+ ]);
+
+ useEffect(() => {
+ const queryParams = new URLSearchParams(location.search);
+ const sourceToken = queryParams.get("source");
+ const motionToken = queryParams.get("motion");
+
+ if (sourceToken) {
+ setSourceTokens(prevTokens => {
+ const tokenIndex = prevTokens.indexOf(sourceToken);
+ if (tokenIndex !== -1) {
+ setSelectedSourceIndex(tokenIndex);
+ setIsUserUploaded(tokenIndex >= numberOfInitialSourceTokens);
+ return prevTokens;
+ } else {
+ const updatedTokens = [...prevTokens, sourceToken];
+ setSelectedSourceIndex(updatedTokens.length - 1);
+ setIsUserUploaded(true);
+ return updatedTokens;
+ }
+ });
+ }
+
+ if (motionToken) {
+ setMotionTokens(prevTokens => {
+ const tokenIndex = prevTokens.indexOf(motionToken);
+ if (tokenIndex !== -1) {
+ setSelectedMotionIndex(tokenIndex);
+ setIsUserUploaded(tokenIndex >= numberOfInitialMotionTokens);
+ return prevTokens;
+ } else {
+ const updatedTokens = [...prevTokens, motionToken];
+ setSelectedMotionIndex(updatedTokens.length - 1);
+ setIsUserUploaded(true);
+ return updatedTokens;
+ }
+ });
+ }
+ }, [
+ location.search,
+ numberOfInitialSourceTokens,
+ numberOfInitialMotionTokens,
+ ]);
+
+ const uploadFocusPointSource = useMemo(() => {
+ const queryParams = new URLSearchParams(location.search);
+ const sourceToken = queryParams.get("source");
+ const motionToken = queryParams.get("motion");
+
+ if (motionToken && !sourceToken) {
+ return true;
+ } else if (!motionToken && sourceToken) {
+ return false;
+ } else {
+ return false;
+ }
+ }, [location.search]);
+
+ useEffect(() => {
+ // When switching source or motion indexes, check if the current combination is generating
+ const isCurrentlyGenerating = currentlyGeneratingList.some(
+ gen =>
+ gen.sourceIndex === selectedSourceIndex &&
+ gen.motionIndex === selectedMotionIndex
+ );
+
+ if (isCurrentlyGenerating) {
+ setIsGenerating(true);
+ } else if (generatedVideoSrc) {
+ setIsGenerating(false);
+ } else if (!isUserUploaded) {
+ setIsGenerating(false);
+ } else {
+ setIsGenerating(false);
+ }
+ }, [
+ selectedSourceIndex,
+ selectedMotionIndex,
+ currentlyGeneratingList,
+ generatedVideoSrc,
+ isUserUploaded,
+ ]);
+
+ useEffect(() => {
+ if (generatedVideos.length > 0) {
+ const relevantVideos = generatedVideos.filter(video =>
+ currentlyGeneratingList.some(
+ gen =>
+ gen.sourceIndex === video.sourceIndex &&
+ gen.motionIndex === video.motionIndex
+ )
+ );
+
+ relevantVideos.forEach(video => {
+ if (video.jobToken) {
+ const jobIndex = currentlyGeneratingList.findIndex(
+ gen =>
+ gen.sourceIndex === video.sourceIndex &&
+ gen.motionIndex === video.motionIndex
+ );
+ if (jobIndex !== -1) {
+ const updatedGeneratingList = [...currentlyGeneratingList];
+ updatedGeneratingList.splice(jobIndex, 1);
+ setCurrentlyGeneratingList(updatedGeneratingList);
+
+ const isCurrentJobGenerating = currentlyGeneratingList.some(
+ gen =>
+ gen.sourceIndex === selectedSourceIndex &&
+ gen.motionIndex === selectedMotionIndex
+ );
+
+ if (!isCurrentJobGenerating) {
+ setGeneratedVideoSrc(video.videoSrc);
+ setIsGenerating(false);
+ }
+ }
+ }
+ });
+ }
+ }, [
+ generatedVideos,
+ currentlyGeneratingList,
+ selectedSourceIndex,
+ selectedMotionIndex,
+ ]);
+
+ const handleDownloadClick = () => {
+ if (generatedVideoSrc) {
+ const link = document.createElement("a");
+ link.href = generatedVideoSrc;
+ link.download = "output_video.mp4";
+ document.body.appendChild(link);
+ link.click();
+ document.body.removeChild(link);
+ } else {
+ console.error("No video source available for download");
+ }
+ };
+
+ const signupCTA = (
+ <>
+ {!sessionFetched ? null : (
+
+
+
+ You need to be logged in to use Live Portrait
+
+
+ Please login or sign up to continue.
+
+
+
+
+
+ Webcam Acting
+
+
+ Use AI to transfer facial expressions, audio, and vocals from one
+ face video to an image or video.
+
+
+ {!loggedIn && (
+ {signupCTA}
+ )}
+
+
+
+
+
+
+
+
+
+ cameraInputTokenSet(target.value),
+ }}
+ />
+
+
+
+
+
+
+
+
+ {
+ setVisibility(prevVisibility =>
+ prevVisibility === "private" ? "public" : "private"
+ );
+ }}
+ checked={visibility === "private"}
+ />
+
+ {
+ setRemoveWatermark(
+ prevRemoveWatermark => !prevRemoveWatermark
+ );
+ }}
+ checked={removeWatermark}
+ />
+
+
+ {!hasPremium && (
+
+
+
+ )}
+
+
+
+
+
+ {loggedIn && (
+
+
+
+
+ setSourceTokens(prevTokens =>
+ prevTokens.includes(newToken)
+ ? prevTokens
+ : [...prevTokens, newToken]
+ )
+ }
+ addMotionToken={(newToken: string) =>
+ setMotionTokens(prevTokens =>
+ prevTokens.includes(newToken)
+ ? prevTokens
+ : [...prevTokens, newToken]
+ )
+ }
+ onJobClick={handleJobClick}
+ onJobProgress={handleJobProgress}
+ onJobStateChange={handleJobStateChange}
+ />
+
+
+ )}
+
Try other AI video tools
+
+
+
+
+ );
+}