new live portrait dev first push (#664)

* init

* updates

* load media from token

* thumbnail

* zoom speed change

* update

* updates

* updates

* update

* second video change

* updates
This commit is contained in:
BFlat authored and GitHub committed 2024-08-13 11:27:44 +07:00
1 parent 65dfeff889
commit 2c666ccf7e
12 files changed
+713 -2

No files matched your search

@@ -97,6 +97,7 @@
# /video* /website/index.html 200
# Test paths
/dev-lp* /website/index.html 200
/dev-tts* /website/index.html 200
/dev-vc* /website/index.html 200
/dev-upload* /website/index.html 200
@@ -1,6 +1,8 @@
@import "scss/colours.scss";
.fy-checkbox-frame {
display: flex;
align-items: center;
margin-bottom: 0.5rem;
white-space: nowrap;
cursor: pointer;
@@ -28,8 +28,13 @@
max-width: 50vw;
}
.fy-modal-body-small {
max-width: 600px;
}
.fy-modal-body-wide,
.fy-modal-body-narrow {
.fy-modal-body-narrow,
.fy-modal-body-small {
width: 100%;
height: fit-content;
overflow: hidden;
@@ -1,7 +1,7 @@
import { useEffect, useState } from "react";
import { isMobile } from "react-device-detect";
export type ModalWidth = "wide" | "narrow";
export type ModalWidth = "wide" | "narrow" | "small";
export interface ModalConfig {
component: React.ElementType;
@@ -132,6 +132,7 @@ import { CreatorTypeformPage } from "./pages/landing/storyteller/PostlaunchLandi
import SignUpSuccessPage from "./pages/beta_key/SignUpSuccessPage";
import { StudioMobileCheckPage } from "./pages/landing/storyteller/PostlaunchLanding/StudioMobileCheckPage";
import { UploadNewTtsModelPage } from "./pages/upload/UploadNewTtsModelPage";
import LivePortrait from "./pages/live_portrait/LivePortrait";
interface Props {
sessionWrapper: SessionWrapper;
@@ -753,6 +754,15 @@ class PageContainer extends React.Component<
/>
</Route>
{/* NEW LIVE PORTRAIT PAGE */}
<Route exact path="/dev-lp">
<LivePortrait
sessionSubscriptionsWrapper={
this.props.sessionSubscriptionsWrapper
}
/>
</Route>
{/* Route for initial voice creation */}
<Route exact path="/voice-designer/create">
<VoiceDesignerFormPage
@@ -0,0 +1,52 @@
import React from "react";
import Cropper from "react-easy-crop";
interface FaceCropperProps {
videoSrc: string;
crop: { x: number; y: number };
zoom: number;
onCropChange: (crop: { x: number; y: number }) => void;
onZoomChange: (zoom: number) => void;
zoomWithScroll?: boolean;
showGrid?: boolean;
isCropping?: boolean;
mediaProps?: any;
onCropComplete?: (croppedArea: any, croppedAreaPixels: any) => void;
}
const FaceCropper: React.FC<FaceCropperProps> = ({
videoSrc,
crop,
zoom,
onCropChange,
onZoomChange,
zoomWithScroll = false,
showGrid = false,
isCropping,
onCropComplete,
...props
}) => {
return (
<Cropper
video={videoSrc}
crop={crop}
zoom={zoom}
zoomWithScroll={zoomWithScroll}
aspect={1}
onCropChange={onCropChange}
onZoomChange={onZoomChange}
showGrid={showGrid}
objectFit="cover"
classes={{
cropAreaClassName: `border-0 ${
isCropping ? "" : "cursor-default"
}`.trim(),
}}
zoomSpeed={0.25}
onCropComplete={onCropComplete}
{...props}
/>
);
};
export default FaceCropper;
@@ -0,0 +1,68 @@
@import "scss/colours.scss";
.lp-media {
position: relative;
overflow: hidden;
border-radius: 0.5rem;
border: lighten($panel, 15%) solid 2px;
background-color: darken($panel, 4%);
aspect-ratio: 1 / 1;
display: flex;
align-items: center;
justify-content: center;
img,
video {
width: 100%;
height: 100%;
object-fit: contain;
}
}
.lp-section-between {
padding-top: 15%;
@media (max-width: 992px) {
padding-top: 0;
}
}
.lp-thumbnail {
cursor: pointer;
opacity: 0.6;
transition: all 0.2s;
border: lighten($panel, 15%) solid 2px;
border-radius: 0.5rem;
overflow: hidden;
&:hover {
border-color: lighten($panel, 30%);
opacity: 0.75;
}
&.active {
opacity: 1;
border: 2px solid $primary;
}
&.lp-add-media {
background-color: lighten($panel, 6%);
}
}
.lp-tag {
display: flex;
width: 100%;
justify-content: space-between;
position: absolute;
top: 0;
left: 0;
padding-top: 10px;
padding-left: 8px;
padding-right: 8px;
z-index: 2;
}
.cursor-default {
cursor: default;
}
@@ -0,0 +1,273 @@
import React, { useEffect, useState } from "react";
import {
Badge,
Button,
Checkbox,
Container,
Label,
Panel,
} from "components/common";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
faArrowDown,
faArrowDownToLine,
faEquals,
faImageUser,
faPlus,
faSparkles,
} from "@fortawesome/pro-solid-svg-icons";
import "./LivePortrait.scss";
import ThumbnailMediaPicker from "./ThumbnailMediaPicker";
import {
EnqueueFaceMirror,
EnqueueFaceMirrorResponse,
MediaFileCropArea,
} from "@storyteller/components/src/api/workflows/EnqueueFaceMirror";
import Tippy from "@tippyjs/react";
import "tippy.js/dist/tippy.css";
import { v4 as uuidv4 } from "uuid";
import { useInferenceJobs } from "hooks";
import { FrontendInferenceJobType } from "@storyteller/components/src/jobs/InferenceJob";
import { SessionSubscriptionsWrapper } from "@storyteller/components/src/session/SessionSubscriptionsWrapper";
interface LivePortraitProps {
sessionSubscriptionsWrapper: SessionSubscriptionsWrapper;
}
export default function LivePortrait({
sessionSubscriptionsWrapper,
}: LivePortraitProps) {
const { enqueue } = useInferenceJobs();
const [isEnqueuing, setIsEnqueuing] = useState(false);
const [selectedSourceIndex, setSelectedSourceIndex] = useState(0);
const [selectedMotionIndex, setSelectedMotionIndex] = useState(0);
const [removeWatermark, setRemoveWatermark] = useState(false);
const [visibility, setVisibility] = useState<"private" | "public">("public");
const [cropArea, setCropArea] = useState<MediaFileCropArea>({
height: 0,
width: 0,
x: 0,
y: 0,
});
const hasPremium = sessionSubscriptionsWrapper.hasPaidFeatures();
const [videoSrc, setVideoSrc] = useState("");
const sourceTokens = ["m_f68s71xwm1r8y2rv4q0xb4dmc4nx9f"];
const motionTokens = [
"m_x2t1kvyzw677b6kr5me2g4vcra4308",
"m_hv1n7nqbq5dmfmzcn1eyy1qw4mskvs",
];
const precomputedVideos = [
{
src: "/videos/live-portrait/1_1.mp4",
},
{
src: "/videos/live-portrait/1_2.mp4",
},
];
const handleSourceSelect = (index: number) => {
console.log("Thumbnail clicked:", index);
setSelectedSourceIndex(index);
};
const handleMotionSelect = (index: number) => {
console.log("Thumbnail clicked:", index);
setSelectedMotionIndex(index);
};
const enqueueClick = () => {
setIsEnqueuing(true);
EnqueueFaceMirror("", {
creator_set_visibility: visibility,
face_driver_media_file_token: motionTokens[selectedMotionIndex],
maybe_crop: cropArea,
remove_watermark: removeWatermark,
source_media_file_token: sourceTokens[selectedSourceIndex],
uuid_idempotency_token: uuidv4(),
}).then((res: EnqueueFaceMirrorResponse) => {
if (res.success && res.inference_job_token) {
enqueue(
res.inference_job_token,
FrontendInferenceJobType.VideoStyleTransfer,
false
);
} else {
console.error("Failed to enqueue job", res);
}
setIsEnqueuing(false);
});
};
const getPrecomputedVideoSrc = () => {
const index = selectedSourceIndex * 4 + selectedMotionIndex;
return precomputedVideos[index].src;
};
useEffect(() => {
setVideoSrc(getPrecomputedVideoSrc());
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [selectedSourceIndex, selectedMotionIndex]);
const isUserContent =
selectedSourceIndex >= sourceTokens.length ||
selectedMotionIndex >= motionTokens.length;
return (
<Container type="panel" className="mt-3 mt-lg-5">
<Panel padding={true}>
<h1 className="fw-bold fs-1 mb-0">
<FontAwesomeIcon icon={faImageUser} className="me-3 fs-2" />
Live Portrait
</h1>
<p className="opacity-75 fw-medium mb-3 mb-lg-0">
Use AI to transfer facial expressions, audio, and vocals from one face
video to an image or video.
</p>
<div className="row gx-0 gy-4 ">
<div
className="col-12 col-lg-3 d-flex gap-3 flex-column"
style={{ paddingTop: "4.2%" }}
>
<ThumbnailMediaPicker
mediaTokens={sourceTokens}
selectedIndex={selectedSourceIndex}
handleThumbnailClick={handleSourceSelect}
title="Select Source"
description="This image or video is what the final video will look like."
badgeLabel="Source Media"
/>
</div>
<div className="col-12 col-lg-1 d-flex justify-content-center lp-section-between">
<FontAwesomeIcon icon={faPlus} className="display-4 opacity-75" />
</div>
<div
className="col-12 col-lg-3 d-flex gap-3 flex-column"
style={{ paddingTop: "4.2%" }}
>
<ThumbnailMediaPicker
mediaTokens={motionTokens}
selectedIndex={selectedMotionIndex}
handleThumbnailClick={handleMotionSelect}
title="Select Motion Reference"
description="This is what the face video will move like (contains audio)."
badgeLabel="Motion Reference"
cropper={true}
cropArea={cropArea}
setCropArea={setCropArea}
/>
</div>
<div className="d-none col-12 col-lg-1 d-lg-flex justify-content-center lp-section-between">
<FontAwesomeIcon
icon={faEquals}
className="display-4 opacity-75 d-none d-lg-block"
/>
</div>
<div className="col-12 col-lg-4 d-flex gap-3 justify-content-center flex-column">
<div className="lp-media order-2 order-lg-1">
<video
loop
autoPlay
muted
playsInline
controls={true}
preload="auto"
key={videoSrc}
>
<source src={getPrecomputedVideoSrc()} type="video/mp4" />
Your browser does not support the video tag.
</video>
<div className="lp-tag">
<div className="d-flex gap-2 w-100">
<Badge
label="Output Video"
color="ultramarine"
overlay={true}
/>
{!isUserContent && (
<Badge
label="Pre-generated Example"
color="gray"
overlay={true}
/>
)}
</div>
</div>
</div>
<div className="d-flex flex-column gap-2 order-1 order-lg-2">
<div className="d-flex gap-3">
<Checkbox
label={"Make Public"}
onChange={() => {
setVisibility(prevVisibility =>
prevVisibility === "public" ? "private" : "public"
);
}}
checked={visibility === "public"}
/>
<div className="d-flex">
<Checkbox
disabled={!hasPremium}
label={"Remove Watermark"}
onChange={() => {
setRemoveWatermark(
prevRemoveWatermark => !prevRemoveWatermark
);
}}
checked={removeWatermark}
/>
{/* <div {...{ className: "fy-ai-face-mirror-premium-label" }}>
Watermark
{!hasPremium ? (
<Link {...{ to: "pricing" }}> subscribe to remove</Link>
) : null}
</div> */}
</div>
</div>
<div className="d-flex gap-2">
<Button
icon={faSparkles}
label="Generate Live Portrait"
onClick={enqueueClick}
className="flex-grow-1"
// disabled={!isUserContent}
isLoading={isEnqueuing}
/>
<Tippy theme="fakeyou" content="Download video">
<div>
<Button
square={true}
icon={faArrowDownToLine}
variant="action"
/>
</div>
</Tippy>
</div>
<div className="col-12 col-lg-1 d-flex d-lg-none justify-content-center lp-section-between my-3">
<FontAwesomeIcon
icon={faArrowDown}
className="display-4 opacity-75"
/>
</div>
</div>
<div className="mt-2 order-3">
<Label label="Output" />
<div className="panel panel-inner rounded p-3">Jobs</div>
<div className="panel panel-inner rounded p-3">Jobs</div>
<div className="panel panel-inner rounded p-3">Jobs</div>
</div>
</div>
</div>
</Panel>
</Container>
);
}
@@ -0,0 +1,62 @@
import LoadingSpinner from "components/common/LoadingSpinner";
import React from "react";
interface ThumbnailItemProps {
index: number;
selectedIndex: number;
handleThumbnailClick: (index: number) => void;
poster?: string;
mediaType?: string;
}
const ThumbnailItem: React.FC<ThumbnailItemProps> = ({
index,
selectedIndex,
handleThumbnailClick,
poster,
mediaType,
}) => {
return (
<div className="col-3" key={index}>
<div
className={`lp-thumbnail ratio ratio-1x1 ${
index === selectedIndex ? "active" : ""
}`}
onClick={() => handleThumbnailClick(index)}
>
{/* {thumbnail ? (
<img
src={thumbnail}
alt="Video Thumbnail"
className="w-100 h-100 object-fit-cover"
draggable="false"
/>
) : (
<LoadingSpinner padding={false} />
)} */}
{poster ? (
mediaType === "image" ? (
<img
src={poster}
alt="Media Thumbnail"
className="w-100 h-100 object-fit-cover"
draggable="false"
/>
) : (
<video
src={poster}
muted
playsInline
className="w-100 h-100 object-fit-cover"
draggable="false"
/>
)
) : (
<LoadingSpinner padding={false} />
)}
</div>
</div>
);
};
export default ThumbnailItem;
@@ -0,0 +1,238 @@
import React, { useCallback, useEffect, useState } from "react";
import { Badge, Button } from "components/common";
import FaceCropper from "./FaceCropper";
import ThumbnailItem from "./ThumbnailItem";
import { BucketConfig } from "@storyteller/components/src/api/BucketConfig";
import LoadingSpinner from "components/common/LoadingSpinner";
import {
GetMedia,
MediaFile,
} from "@storyteller/components/src/api/media_files/GetMedia";
import { faUpload } from "@fortawesome/pro-solid-svg-icons";
import { isMobile } from "react-device-detect";
interface ThumbnailMediaPickerProps {
mediaTokens: string[];
selectedIndex: number;
handleThumbnailClick: (index: number) => void;
title?: string;
description?: string;
badgeLabel?: string;
cropper?: true;
showCropButton?: boolean;
cropArea?: { x: number; y: number; height: number; width: number };
setCropArea?: (cropArea: {
x: number;
y: number;
height: number;
width: number;
}) => void;
}
interface MediaData {
[key: string]: MediaFile | undefined;
}
const ThumbnailMediaPicker: React.FC<ThumbnailMediaPickerProps> = React.memo(
({
selectedIndex,
handleThumbnailClick,
badgeLabel = "Media",
title = "Select Source",
description = "This image or video is what the final video will look like.",
cropper = false,
cropArea,
setCropArea,
mediaTokens,
showCropButton = true,
}) => {
const [zoom, setZoom] = useState(1);
const [isCropping, setIsCropping] = useState(false);
const [mediaData, setMediaData] = useState<{ [key: string]: any }>({});
useEffect(() => {
const fetchMediaData = async () => {
const mediaDataPromises = mediaTokens.map(async token => {
const response = await GetMedia(token, {});
return { token, media: response.media_file };
});
const mediaDataArray = await Promise.all(mediaDataPromises);
const mediaDataObject = mediaDataArray.reduce(
(acc, { token, media }) => {
acc[token] = media;
return acc;
},
{} as MediaData
);
setMediaData(mediaDataObject);
};
fetchMediaData();
}, [mediaTokens]);
const selectedMedia = mediaData[mediaTokens[selectedIndex]];
const mediaLink = selectedMedia?.public_bucket_path
? new BucketConfig().getGcsUrl(selectedMedia.public_bucket_path)
: null;
useEffect(() => {
if (setCropArea) {
setCropArea({ x: 0, y: 0, height: 0, width: 0 });
setZoom(1);
}
}, [selectedIndex, setCropArea]);
const onCropChange = useCallback(
crop => {
if (setCropArea) {
if (isCropping) {
setCropArea(crop);
}
}
},
[isCropping, setCropArea]
);
const onZoomChange = useCallback(
zoom => {
if (isCropping) {
setZoom(zoom);
}
},
[isCropping]
);
const onCropComplete = useCallback((croppedArea, croppedAreaPixels) => {
console.log(croppedArea, croppedAreaPixels, "HELLO");
}, []);
return (
<div className="d-flex gap-3 flex-column">
<div className="lp-media order-4 order-lg-1">
<div className="lp-tag">
<div>
{!isCropping ? (
<Badge label={badgeLabel} color="ultramarine" overlay={true} />
) : (
<Badge
label={`${isMobile ? "Pinch" : "Scroll"} and drag to crop`}
color="gray"
overlay={true}
/>
)}
</div>
{cropper && showCropButton && (
<div>
<Button
label={isCropping ? "Done" : "Crop Face"}
className="py-1 px-2 fs-7"
variant={isCropping ? "primary" : "action"}
onClick={() => setIsCropping(prev => !prev)}
/>
</div>
)}
</div>
{cropper && cropArea ? (
<>
{mediaLink ? (
<FaceCropper
videoSrc={mediaLink}
crop={cropArea}
zoom={zoom}
onCropChange={onCropChange}
onZoomChange={onZoomChange}
onCropComplete={onCropComplete}
showGrid={isCropping ? true : false}
zoomWithScroll={isCropping ? true : false}
isCropping={isCropping}
mediaProps={{
autoPlay: true,
loop: true,
}}
/>
) : (
<LoadingSpinner padding={false} />
)}
</>
) : (
<div className="w-100 h-100 object-fit-contain d-flex align-items-center justify-content-center">
{mediaLink ? (
selectedMedia?.media_type === "image" ? (
<img
key={selectedIndex}
src={mediaLink}
alt="Selected media"
/>
) : (
<video
key={selectedIndex}
autoPlay={true}
muted
loop={true}
playsInline
controls={false}
preload="auto"
draggable="false"
>
<source src={mediaLink} type="video/mp4" />
Your browser does not support the video tag.
</video>
)
) : (
<LoadingSpinner padding={false} />
)}
</div>
)}
</div>
<div className="order-1 order-lg-2">
<h2 className="fs-5 mb-1 fw-semibold">{title}</h2>
<p className="fw-medium fs-7 opacity-75">{description}</p>
</div>
<div className="row g-2 order-2 order-lg-3">
{mediaTokens.map((token, index) => {
const media = mediaData[token];
const mediaLink = media?.public_bucket_path
? new BucketConfig().getGcsUrl(media.public_bucket_path)
: null;
return (
<ThumbnailItem
key={index}
index={index}
selectedIndex={selectedIndex}
handleThumbnailClick={handleThumbnailClick}
poster={mediaLink || ""}
mediaType={media?.media_type}
/>
);
})}
{/* <div className="col-3">
<div className="lp-thumbnail lp-add-media ratio ratio-1x1">
<div className="d-flex gap-1 flex-column align-items-center justify-content-center">
<FontAwesomeIcon icon={faCirclePlus} className="fs-4 mt-1" />
<span className="fw-medium mb-0 user-select-none fs-7">
Upload
</span>
</div>
</div>
</div> */}
</div>
<Button
icon={faUpload}
label="Upload your own"
variant="secondary"
className="order-3 order-lg-4"
/>
</div>
);
}
);
export default ThumbnailMediaPicker;