From 05fc58da6cae859ad33bcbc4051844342c6454cc Mon Sep 17 00:00:00 2001 From: olivicmic Date: Fri, 8 Sep 2023 11:56:19 -0700 Subject: [PATCH] face animation editor with audio > image > animation > to job workflow --- packages/components/src/api/ApiConfig.ts | 8 ++ packages/components/src/api/upload/upload.ts | 54 ++++++++ packages/components/src/jobs/InferenceJob.ts | 1 + .../common/AudioInput/AudioInput.tsx | 2 +- .../fakeyou.com/src/v2/view/PageContainer.tsx | 7 +- .../lipsync/LipsyncEditor/LipsyncEditor.scss | 11 +- .../lipsync/LipsyncEditor/LipsyncEditor.tsx | 129 ++++++++++++------ .../src/v2/view/pages/media/index.tsx | 3 + .../src/v2/view/pages/media/media.tsx | 11 ++ 9 files changed, 184 insertions(+), 42 deletions(-) create mode 100644 packages/components/src/api/upload/upload.ts create mode 100644 packages/fakeyou.com/src/v2/view/pages/media/index.tsx create mode 100644 packages/fakeyou.com/src/v2/view/pages/media/media.tsx diff --git a/packages/components/src/api/ApiConfig.ts b/packages/components/src/api/ApiConfig.ts index 4a670ad06b..b98dcd4d0d 100644 --- a/packages/components/src/api/ApiConfig.ts +++ b/packages/components/src/api/ApiConfig.ts @@ -402,6 +402,14 @@ class ApiConfig { return `${this.getApiOrigin()}/v1/media_uploads/upload_audio`; } + uploadImage(): string { + return `${this.getApiOrigin()}/v1/media_uploads/upload_image`; + } + + uploadVideo(): string { + return `${this.getApiOrigin()}/v1/media_uploads/upload_video`; + } + uploadMedia(): string { return `${this.getApiOrigin()}/v1/media_uploads/upload`; } diff --git a/packages/components/src/api/upload/upload.ts b/packages/components/src/api/upload/upload.ts new file mode 100644 index 0000000000..fb0b23ee60 --- /dev/null +++ b/packages/components/src/api/upload/upload.ts @@ -0,0 +1,54 @@ +import { ApiConfig } from "../ApiConfig"; + +export interface UploadRequest { + uuid_idempotency_token: string, + source?: string, // eg. "device", "file" + type: string, // "audio", "image", "video" + file: any, +}; + +export interface UploadResponse { + success: boolean, + upload_token?: string, +} + +const { uploadAudio, uploadImage, uploadVideo } = new ApiConfig(); + +const audio = new ApiConfig().uploadAudio(); +const image = new ApiConfig().uploadImage(); +const video = new ApiConfig().uploadVideo(); + +const endpoints = { audio, image, video }; + +export async function Upload(request: UploadRequest) : Promise +{ + + const formData = new FormData(); + + formData.append('uuid_idempotency_token', request.uuid_idempotency_token); + formData.append('file', request.file); + + if (request.source !== undefined) { + formData.append('source', request.source); + } + + return fetch(endpoints[request.type as keyof typeof endpoints], { + method: 'POST', + credentials: 'include', + headers: { + 'Accept': 'application/json', + }, + body: formData, + }) + .then(res => res.json()) + .then(res => { + if (res && 'success' in res) { + return res; + } else { + return { success : false }; + } + }) + .catch(e => { + return { success : false }; + }); +} \ No newline at end of file diff --git a/packages/components/src/jobs/InferenceJob.ts b/packages/components/src/jobs/InferenceJob.ts index 4700206e3c..586126aa83 100644 --- a/packages/components/src/jobs/InferenceJob.ts +++ b/packages/components/src/jobs/InferenceJob.ts @@ -3,6 +3,7 @@ import { JobState, jobStateFromString } from "./JobStates"; // Type of inference job (specified by the frontend, not backend) export enum FrontendInferenceJobType { + FaceAnimation, TextToSpeech, VoiceConversion, } diff --git a/packages/fakeyou.com/src/components/common/AudioInput/AudioInput.tsx b/packages/fakeyou.com/src/components/common/AudioInput/AudioInput.tsx index 5531fbe7af..9466897ab9 100644 --- a/packages/fakeyou.com/src/components/common/AudioInput/AudioInput.tsx +++ b/packages/fakeyou.com/src/components/common/AudioInput/AudioInput.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { a, useSpring, useTransition } from '@react-spring/web'; +import { a, useTransition } from '@react-spring/web'; import { InputVcAudioPlayer } from "v2/view/_common/InputVcAudioPlayer"; import { FileActions, FileDetails, FileLabel, FileWrapper } from 'components/common'; import { faFileAudio } from "@fortawesome/pro-solid-svg-icons"; diff --git a/packages/fakeyou.com/src/v2/view/PageContainer.tsx b/packages/fakeyou.com/src/v2/view/PageContainer.tsx index 10dc7ec360..800c452614 100644 --- a/packages/fakeyou.com/src/v2/view/PageContainer.tsx +++ b/packages/fakeyou.com/src/v2/view/PageContainer.tsx @@ -6,6 +6,7 @@ import { LoginPage } from "./pages/login/LoginPage"; import { ModerationFc } from "./pages/moderation/moderation_main/ModerationFc"; import { ModerationIpBanListFc } from "./pages/moderation/moderation_ip_ban_list/ModerationIpBanListFc"; import { ModerationViewIpBanFc } from "./pages/moderation/moderation_view_ip_ban/ModerationViewIpBanFc"; +import { Media } from "./pages/media"; import { LipsyncEditor } from "./pages/lipsync"; import { FooterNav } from "./nav/FooterNav"; import { TopNav } from "./nav/TopNav"; @@ -431,7 +432,11 @@ class PageContainer extends React.Component { - + + + + + diff --git a/packages/fakeyou.com/src/v2/view/pages/lipsync/LipsyncEditor/LipsyncEditor.scss b/packages/fakeyou.com/src/v2/view/pages/lipsync/LipsyncEditor/LipsyncEditor.scss index ee3a9dfe82..3bfe8558ae 100644 --- a/packages/fakeyou.com/src/v2/view/pages/lipsync/LipsyncEditor/LipsyncEditor.scss +++ b/packages/fakeyou.com/src/v2/view/pages/lipsync/LipsyncEditor/LipsyncEditor.scss @@ -87,7 +87,7 @@ } } -.lipsync-panel { position: relative; height: fit-content; } +.lipsync-panel { position: relative; height: fit-content; overflow: hidden; } // .lipsync-editor, .lipsync-working, .lipsync-success { // position: absolute; @@ -97,8 +97,17 @@ display: flex; align-items: center; justify-content: center; + width: 100%; } +// .lipsync-working-notice { +// background-color: red; +// } + +// .lipsync-working { +// background-color: green; +// } + .lipsync-working-notice { flex-direction: column; margin: 5rem; diff --git a/packages/fakeyou.com/src/v2/view/pages/lipsync/LipsyncEditor/LipsyncEditor.tsx b/packages/fakeyou.com/src/v2/view/pages/lipsync/LipsyncEditor/LipsyncEditor.tsx index e0874cc5ff..1afbbc1555 100644 --- a/packages/fakeyou.com/src/v2/view/pages/lipsync/LipsyncEditor/LipsyncEditor.tsx +++ b/packages/fakeyou.com/src/v2/view/pages/lipsync/LipsyncEditor/LipsyncEditor.tsx @@ -1,11 +1,17 @@ import React, { useState } from "react"; import { animated, useSpring, useTransition } from '@react-spring/web'; -import { useFile } from "hooks"; +import { useFile, useIdempotency } from "hooks"; import { AudioInput, ImageInput, Spinner } from "components/common"; import DynamicButton from './DynamicButton'; +import { Upload } from "@storyteller/components/src/api/upload/upload"; +import { FrontendInferenceJobType } from "@storyteller/components/src/jobs/InferenceJob"; import './LipsyncEditor.scss'; -interface LipSyncProps { audioProps: any, imageProps: any, toggle: any, style: any }; +interface LipSyncProps { audioProps: any, children?: any, imageProps: any, toggle: any, style: any }; +interface EditorProps { enqueueInferenceJob: ( + jobToken: string, + frontendInferenceJobType: FrontendInferenceJobType + ) => void }; const softSpring = { config: { mass: 1, tension: 80, friction: 10 } } @@ -13,10 +19,7 @@ const SuccessPage = ({ audioProps, imageProps, style }: LipSyncProps ) => Result - + Your animation is in progress. ; const InputPage = ({ audioProps, imageProps, toggle, style }: LipSyncProps ) => { @@ -38,13 +41,17 @@ const InputPage = ({ audioProps, imageProps, toggle, style }: LipSyncProps ) => }; -const WorkingPage = ({ audioProps, imageProps, style }: LipSyncProps ) => +const Working = ({ audioProps, children = "", imageProps, style }: LipSyncProps ) =>
-

Working ...

+

{ children }

; +const AudioWorking = (props: any) => Uploading Audio ...; +const ImageWorking = (props: any) => Uploading Image ...; +const AniWorking = (props: any) => Starting animation ...; + const ProgressLi = ({ children, disabled = false }: { children?: any, disabled?: boolean }) => { const style = useSpring({ ...softSpring, @@ -54,11 +61,11 @@ const ProgressLi = ({ children, disabled = false }: { children?: any, disabled?: { } { children } @@ -72,7 +79,7 @@ const Title = ({ ...rest }) => { const incomplete = noAudio || noImg; const working = imageProps.working && audioProps.working; - const slides = ["Generate",,"Make another"]; + const slides = ["Generate",,,,"Make another"]; const onClick = () => { if (imageProps.success && audioProps.success) { @@ -80,7 +87,7 @@ const Title = ({ ...rest }) => { } else if (!incomplete && !working) submit(); }; - return
+ return

Lip Service

@@ -93,38 +100,82 @@ const Title = ({ ...rest }) => { -

+

Select and image with a clear face, and an audio sample, and generate a lipsynced video.

}; -export default function LipsyncEditor({ ...rest }) { +const fakeAPI = async (anything: any) => { // this just simulates the unifinished apis + await new Promise(resolve => { + setTimeout(() => resolve({ success: true }), 3000); + }); + return { success: true, inference_job_token: "fake", upload_token: "also fake", }; +}; + +export default function LipsyncEditor({ enqueueInferenceJob, ...rest }: EditorProps) { + + // the ready states are set by functions which run after the upload input animation is completed, which then illuminates the respective checkmark in a staggered way to draw attention to the workflow, and reduces concurrent animations + const [imageReady,imageReadySet] = useState(false); const [audioReady,audioReadySet] = useState(false); const readyMedia = (m:number) => (t:boolean) => [imageReadySet,audioReadySet][m](t); - const onSubmit = async () => { - await new Promise(resolve => { - setTimeout(function () { - resolve(); - }, 6000); + + const idempotency = useIdempotency(); + const audioProps = useFile({}); // contains upload inout state and controls, see docs + const imageProps = useFile({}); // contains upload inout state and controls, see docs + const [index,indexSet] = useState(0); // index = slideshow slide position + const [audioUploadToken,audioUploadTokenSet] = useState(); + const [imageUploadToken,imageUploadTokenSet] = useState(); + + const makeRequest = (mode: number) => ({ + uuid_idempotency_token: idempotency.token, + file: mode ? imageProps.file : audioProps.file, + source: "file", + type: mode ? "image" : "audio" + }); + + const submit = async () => { + if (!audioProps.file) return false; + + indexSet(1); // set audio working page + + Upload(makeRequest(0)) // start audio (0) upload + .then(res => { + if ("success" in res) { + audioUploadTokenSet(res.upload_token); // audio token stored + idempotency.reset(); // reset token for new request, I don't know if this needs to happen + indexSet(2); // set image working page + } + return fakeAPI(makeRequest(1)); // start image (1) upload, replace with Upload(imageRequest) + }) + .then(res => { + if ("success" in res) { + imageUploadTokenSet(res.upload_token); // image token stored + idempotency.reset(); + indexSet(3); // set face animator API working page + } + return fakeAPI({ + source_audio_media_upload_token: audioUploadToken, + source_image_media_upload_token: imageUploadToken, + uuid_idempotency_token: idempotency.token + }); + }) + .then(res => { + if ("success" in res) { + // this breaks because I don't get a real token + // enqueueInferenceJob(res.inference_job_token,FrontendInferenceJobType.FaceAnimation); + idempotency.reset(); + indexSet(4); // set face animator API working page + } + }) + .catch(e => { + return { success : false }; }); - return true; - }; - const audioProps = useFile({ debug: 'audio useFile', onSubmit }); - const imageProps = useFile({ onSubmit }); - const index = audioProps.status === imageProps.status ? audioProps.status : -1; - const headerProps = { - audioProps, - audioReady, - imageProps, - imageReady, - index, - submit: () => { - audioProps.submit(); imageProps.submit(); - } }; + const headerProps = { audioProps, audioReady, imageProps, imageReady, index, submit }; + const transitions = useTransition(index, { ...softSpring, from: { opacity: 0, position: "absolute" }, @@ -137,13 +188,13 @@ export default function LipsyncEditor({ ...rest }) {
{ transitions((style, i) => { - const Page = [InputPage,WorkingPage,SuccessPage][i]; - return i > -1 ? <Page {...{ + const Page = [InputPage,AudioWorking,ImageWorking,AniWorking,SuccessPage][i]; + return Page ? <Page {...{ audioProps, imageProps, toggle: { audio: readyMedia(1), image: readyMedia(0) }, style - }}/> : null + }}/> : <></> }) } </div> </div> diff --git a/packages/fakeyou.com/src/v2/view/pages/media/index.tsx b/packages/fakeyou.com/src/v2/view/pages/media/index.tsx new file mode 100644 index 0000000000..f7c67981af --- /dev/null +++ b/packages/fakeyou.com/src/v2/view/pages/media/index.tsx @@ -0,0 +1,3 @@ +import media from './media'; + +export { media as Media }; diff --git a/packages/fakeyou.com/src/v2/view/pages/media/media.tsx b/packages/fakeyou.com/src/v2/view/pages/media/media.tsx new file mode 100644 index 0000000000..ddc4db1725 --- /dev/null +++ b/packages/fakeyou.com/src/v2/view/pages/media/media.tsx @@ -0,0 +1,11 @@ +import React from 'react'; + +interface Props { + value?: any; +} + +export default function media({ value }: Props) { + return <div> + <h1>Media</h1> + </div>; +}; \ No newline at end of file