face animation editor with audio > image > animation > to job workflow

packages/components/src/api/ApiConfig.ts

@@ -402,6 +402,14 @@ class ApiConfig {
     return `${this.getApiOrigin()}/v1/media_uploads/upload_audio`;
   }
 
+  uploadImage(): string {
+    return `${this.getApiOrigin()}/v1/media_uploads/upload_image`;
+  }
+
+  uploadVideo(): string {
+    return `${this.getApiOrigin()}/v1/media_uploads/upload_video`;
+  }
+
   uploadMedia(): string {
     return `${this.getApiOrigin()}/v1/media_uploads/upload`;
   }

packages/components/src/api/upload/upload.ts

@@ -0,0 +1,54 @@
+import { ApiConfig } from "../ApiConfig";
+
+export interface UploadRequest {
+  uuid_idempotency_token: string,
+  source?: string, // eg. "device", "file"
+  type: string, // "audio", "image", "video"
+  file: any,
+};
+
+export interface UploadResponse {
+  success: boolean,
+  upload_token?: string,
+}
+
+const { uploadAudio, uploadImage, uploadVideo } = new ApiConfig();
+
+const audio =  new ApiConfig().uploadAudio();
+const image =  new ApiConfig().uploadImage();
+const video =  new ApiConfig().uploadVideo();
+
+const endpoints = { audio, image, video };
+
+export async function Upload(request: UploadRequest) : Promise<UploadResponse> 
+{
+  
+  const formData = new FormData();
+
+  formData.append('uuid_idempotency_token', request.uuid_idempotency_token);
+  formData.append('file', request.file);
+
+  if (request.source !== undefined) {
+    formData.append('source', request.source);
+  }
+
+  return fetch(endpoints[request.type as keyof typeof endpoints], {
+    method: 'POST',
+    credentials: 'include',
+    headers: {
+      'Accept': 'application/json',
+    },
+    body: formData,
+  })
+  .then(res => res.json())
+  .then(res => {
+    if (res && 'success' in res) {
+      return res;
+    } else {
+      return { success : false };
+    }
+  })
+  .catch(e => {
+    return { success : false };
+  });
+}

packages/components/src/jobs/InferenceJob.ts

@@ -3,6 +3,7 @@ import { JobState, jobStateFromString } from "./JobStates";
 
 // Type of inference job (specified by the frontend, not backend)
 export enum FrontendInferenceJobType {
+  FaceAnimation,
   TextToSpeech,
   VoiceConversion,
 }

packages/fakeyou.com/src/components/common/AudioInput/AudioInput.tsx

@@ -1,5 +1,5 @@
 import React from "react";
-import { a, useSpring, useTransition } from '@react-spring/web';
+import { a, useTransition } from '@react-spring/web';
 import { InputVcAudioPlayer } from "v2/view/_common/InputVcAudioPlayer";
 import { FileActions, FileDetails, FileLabel, FileWrapper } from 'components/common';
 import { faFileAudio } from "@fortawesome/pro-solid-svg-icons";

packages/fakeyou.com/src/v2/view/PageContainer.tsx

@@ -6,6 +6,7 @@ import { LoginPage } from "./pages/login/LoginPage";
 import { ModerationFc } from "./pages/moderation/moderation_main/ModerationFc";
 import { ModerationIpBanListFc } from "./pages/moderation/moderation_ip_ban_list/ModerationIpBanListFc";
 import { ModerationViewIpBanFc } from "./pages/moderation/moderation_view_ip_ban/ModerationViewIpBanFc";
+import { Media } from "./pages/media";
 import { LipsyncEditor } from "./pages/lipsync";
 import { FooterNav } from "./nav/FooterNav";
 import { TopNav } from "./nav/TopNav";
@@ -431,7 +432,11 @@ class PageContainer extends React.Component<Props, State> {
             </Route>
 
             <Route path="/dev-lipsync">
-              <LipsyncEditor />
+              <LipsyncEditor {...{ enqueueInferenceJob: this.props.enqueueInferenceJob }} />
+            </Route>
+
+            <Route path="/media/:token">
+              <Media/>
             </Route>
 
             <Route path="/moderation/user/list">

packages/fakeyou.com/src/v2/view/pages/lipsync/LipsyncEditor/LipsyncEditor.scss

@@ -87,7 +87,7 @@
 	}
 }
 
-.lipsync-panel { position: relative; height: fit-content; }
+.lipsync-panel { position: relative; height: fit-content; overflow: hidden; }
 
 // .lipsync-editor, .lipsync-working, .lipsync-success {
 // 	position: absolute;
@@ -97,8 +97,17 @@
 	display: flex;
 	align-items: center;
 	justify-content: center;
+	width: 100%;
 }
 
+// .lipsync-working-notice {
+// 	background-color: red;
+// }
+
+// .lipsync-working {
+// 	background-color: green;
+// }
+
 .lipsync-working-notice {
 	flex-direction: column;
 	margin: 5rem;

packages/fakeyou.com/src/v2/view/pages/lipsync/LipsyncEditor/LipsyncEditor.tsx

@@ -1,11 +1,17 @@
 import React, { useState } from "react";
 import { animated, useSpring, useTransition } from '@react-spring/web';
-import { useFile } from "hooks";
+import { useFile, useIdempotency } from "hooks";
 import { AudioInput, ImageInput, Spinner } from "components/common";
 import DynamicButton from './DynamicButton';
+import { Upload } from "@storyteller/components/src/api/upload/upload";
+import { FrontendInferenceJobType } from "@storyteller/components/src/jobs/InferenceJob";
 import './LipsyncEditor.scss';
 
-interface LipSyncProps { audioProps: any, imageProps: any, toggle: any, style: any };
+interface LipSyncProps { audioProps: any, children?: any, imageProps: any, toggle: any, style: any };
+interface EditorProps { enqueueInferenceJob: (
+    jobToken: string,
+    frontendInferenceJobType: FrontendInferenceJobType
+  ) => void };
 
 const softSpring = { config: { mass: 1, tension: 80, friction: 10 } }
 
@@ -13,10 +19,7 @@ const SuccessPage = ({ audioProps, imageProps, style }: LipSyncProps )  => <anim
   <h3 className=" fw-bold text-center text-lg-start">
     Result
   </h3>
-  <video width="100%" height="auto" controls={true} className="rounded">
-    <source src="https://pics.vics.pics/3027969248.mp4" />
-    Your device doesn't support video.
-  </video>
+  Your animation is in progress.
 </animated.div>;
 
 const InputPage = ({ audioProps, imageProps, toggle, style }: LipSyncProps )  => {
@@ -38,13 +41,17 @@ const InputPage = ({ audioProps, imageProps, toggle, style }: LipSyncProps )  =>
   </div>
 </animated.div>};
 
-const WorkingPage = ({ audioProps, imageProps, style }: LipSyncProps ) => <animated.div {...{ className: "lipsync-working", style }}>
+const Working = ({ audioProps, children = "", imageProps, style }: LipSyncProps ) => <animated.div {...{ className: "lipsync-working", style }}>
   <div {...{ className: "lipsync-working-notice" }}>
-    <h2>Working ...</h2>
+    <h2>{ children }</h2>
     <Spinner />
   </div>
 </animated.div>;
 
+const AudioWorking = (props: any) => <Working { ...props }>Uploading Audio ...</Working>;
+const ImageWorking = (props: any) => <Working { ...props }>Uploading Image ...</Working>;
+const AniWorking = (props: any) => <Working { ...props }>Starting animation ...</Working>;
+
 const ProgressLi = ({ children, disabled = false }: { children?: any, disabled?: boolean }) => {
   const style = useSpring({
     ...softSpring,
@@ -54,11 +61,11 @@ const ProgressLi = ({ children, disabled = false }: { children?: any, disabled?:
     <svg>
       <circle {...{ cx: 16, cy: 16, r: 15, strokeWidth: "2", }}/>
       { <polyline {...{
-        fill: 'none',
-        points: '9.5 18 14.5 22 22.5 12',
-        strokeLinecap: 'round',
-        strokeLinejoin: 'round',
-        strokeWidth: '4',
+        fill: "none",
+        points: "9.5 18 14.5 22 22.5 12",
+        strokeLinecap: "round",
+        strokeLinejoin: "round",
+        strokeWidth: "4",
       }}/> }
     </svg>
     { children }
@@ -72,7 +79,7 @@ const Title = ({ ...rest }) => {
   const incomplete = noAudio || noImg;
   const working = imageProps.working && audioProps.working;
 
-  const slides = ["Generate",<Spinner />,"Make another"];
+  const slides = ["Generate",<Spinner />,<Spinner />,<Spinner />,"Make another"];
 
   const onClick = () => {
     if (imageProps.success && audioProps.success) {
@@ -80,7 +87,7 @@ const Title = ({ ...rest }) => {
     } else if (!incomplete && !working) submit();
   };
 
-  return <div {...{ className: 'progress-header' }}>
+  return <div {...{ className: "progress-header" }}>
     <h1 {...{ className: "fw-bold text-center text-md-start progress-heading" }}>
       Lip Service
     </h1>
@@ -93,38 +100,82 @@ const Title = ({ ...rest }) => {
       </ProgressLi>
     </ul>
     <DynamicButton {...{ disabled: incomplete || working, onClick, slides, index }}/>
-    <p {...{ className: 'progress-description' }}> 
+    <p {...{ className: "progress-description" }}> 
       Select and image with a clear face, and an audio sample, and generate a lipsynced video.
     </p>
   </div>
 };
 
-export default function LipsyncEditor({ ...rest }) {
+const fakeAPI = async (anything: any) => { // this just simulates the unifinished apis
+  await new Promise(resolve => {
+      setTimeout(() => resolve({ success: true }), 3000);
+  });
+  return { success: true, inference_job_token: "fake", upload_token: "also fake", };
+};
+
+export default function LipsyncEditor({ enqueueInferenceJob, ...rest }: EditorProps) {
+
+  // the ready states are set by functions which run after the upload input animation is completed, which then illuminates the respective checkmark in a staggered way to draw attention to the workflow, and reduces concurrent animations
+
   const [imageReady,imageReadySet] = useState<boolean>(false);
   const [audioReady,audioReadySet] = useState<boolean>(false);
   const readyMedia = (m:number) => (t:boolean) => [imageReadySet,audioReadySet][m](t);
-  const onSubmit = async () => {
-    await new Promise(resolve => {
-        setTimeout(function () {
-            resolve();
-        }, 6000);
+
+  const idempotency = useIdempotency();
+  const audioProps = useFile({}); // contains upload inout state and controls, see docs
+  const imageProps = useFile({}); // contains upload inout state and controls, see docs
+  const [index,indexSet] = useState<number>(0); // index  = slideshow slide position
+  const [audioUploadToken,audioUploadTokenSet] = useState<string>();
+  const [imageUploadToken,imageUploadTokenSet] = useState<string>();
+
+  const makeRequest = (mode: number) => ({
+      uuid_idempotency_token: idempotency.token,
+      file: mode ? imageProps.file : audioProps.file,
+      source: "file",
+      type: mode ? "image" : "audio" 
+  });
+
+  const submit = async () => {
+    if (!audioProps.file) return false;
+
+    indexSet(1); // set audio working page
+
+    Upload(makeRequest(0)) // start audio (0) upload
+    .then(res => {
+      if ("success" in res) {
+        audioUploadTokenSet(res.upload_token); // audio token stored
+        idempotency.reset(); // reset token for new request, I don't know if this needs to happen
+        indexSet(2); // set image working page
+      }
+      return fakeAPI(makeRequest(1)); // start image (1) upload, replace with Upload(imageRequest)
+    })
+    .then(res => {
+      if ("success" in res) {
+        imageUploadTokenSet(res.upload_token); // image token stored
+        idempotency.reset(); 
+        indexSet(3); // set face animator API working page
+      }
+      return fakeAPI({ 
+        source_audio_media_upload_token: audioUploadToken, 
+        source_image_media_upload_token: imageUploadToken, 
+        uuid_idempotency_token: idempotency.token
+      });
+    })
+    .then(res => {
+      if ("success" in res) {
+      // this breaks because I don't get a real token
+      // enqueueInferenceJob(res.inference_job_token,FrontendInferenceJobType.FaceAnimation);
+        idempotency.reset();
+        indexSet(4); // set face animator API working page
+      }
+    })
+    .catch(e => {
+      return { success : false };
     });
-    return true;
-  };
-  const audioProps = useFile({ debug: 'audio useFile', onSubmit });
-  const imageProps = useFile({ onSubmit });
-  const index = audioProps.status === imageProps.status ? audioProps.status : -1;
-  const headerProps = {
-    audioProps,
-    audioReady,
-    imageProps,
-    imageReady,
-    index,
-    submit: () => {
-      audioProps.submit(); imageProps.submit();
-    }
   };
 
+  const headerProps = { audioProps, audioReady, imageProps, imageReady, index, submit };
+
   const transitions = useTransition(index, {
     ...softSpring,
     from: { opacity: 0, position: "absolute" },
@@ -137,13 +188,13 @@ export default function LipsyncEditor({ ...rest }) {
         <div {...{ className: "panel lipsync-panel" }}>
           <Title { ...headerProps }/>
           { transitions((style, i) => {
-            const Page = [InputPage,WorkingPage,SuccessPage][i];
-            return i > -1 ? <Page {...{ 
+            const Page = [InputPage,AudioWorking,ImageWorking,AniWorking,SuccessPage][i];
+            return Page ? <Page {...{ 
               audioProps,
               imageProps,
               toggle: { audio: readyMedia(1), image: readyMedia(0) },
               style 
-            }}/> : null
+            }}/> : <></>
           }) }
         </div>
       </div>

packages/fakeyou.com/src/v2/view/pages/media/index.tsx

@@ -0,0 +1,3 @@
+import media from './media';
+
+export { media as Media };

packages/fakeyou.com/src/v2/view/pages/media/media.tsx

@@ -0,0 +1,11 @@
+import React from 'react';
+
+interface Props {
+  value?: any;
+}
+
+export default function media({ value }: Props) {
+  return <div>
+  	<h1>Media</h1>
+  </div>;
+};