completed UI flow

packages/components/src/api/ApiConfig.ts

@@ -496,6 +496,11 @@ class ApiConfig {
     return `${this.getApiOrigin()}/v1/mocap/mocapnet/create`;
   }
 
+  // =============== Storyteller Filter ===============
+  enqueueStorytellerFilter(): string {
+    return `${this.getApiOrigin()}/v1/workflow/comfy/create`;
+  }
+
   // =============== Convert FBX to glTF ===============
   enqueueFbxToGltf(): string {
     return `${this.getApiOrigin()}/v1/conversion/enqueue_fbx_to_gltf`;

packages/components/src/api/storyteller_filter/EnqueueStorytellerFilter.tsx

@@ -0,0 +1,68 @@
+import { ApiConfig } from "@storyteller/components";
+
+export interface EnqueueStorytellerFilterRequest {
+  "uuid_idempotency_token": string,
+  
+  "maybe_sd_model": string,
+  "maybe_workflow_config": string,
+  "maybe_input_file": string,
+  "maybe_output_path": string,
+  "maybe_json_modifications": {
+    "$.510.inputs.Text": string,
+    "$.8.inputs.text": string,
+    "$.173.inputs.seed": string,
+    "$.401.inputs.Value": number,
+    "$.918.inputs.Value": number,
+    "$.137.inputs.Value": number,
+    "$.186.inputs.Value": number,
+    "$.140.inputs.Value":number,
+    "$.154.inputs.Value": number,
+    "$.445.inputs.number": number,
+    "$.947.inputs.Value": number,
+    "$.800.inputs.Value": number,
+    "$.797.inputs.Value": number,
+    "$.796.inputs.Value": number,
+    "$.772.inputs.Value": number,
+    "$.771.inputs.Value": number,
+    "$.527.inputs.Value": number,
+    "$.403.inputs.Value": number,
+  }
+}
+
+export interface EnqueueStorytellerFilterResponse {
+  success: boolean,
+  inference_job_token?: string,
+}
+
+export async function EnqueueStorytellerFilter(request: EnqueueStorytellerFilterRequest) : Promise<EnqueueStorytellerFilterResponse> 
+{
+  // const endpoint = "/v1/workflow/comfy/create";
+
+  const endpoint = new ApiConfig().enqueueStorytellerFilter();
+  
+  return await fetch(endpoint, {
+    method: 'POST',
+    headers: {
+      'Accept': 'application/json',
+      'Content-Type': 'application/json',
+    },
+    credentials: 'include',
+    body: JSON.stringify(request),
+  })
+  .then(res => res.json())
+  .then(res => {
+    if (!res) {
+      return { success : false };
+    }
+
+    if (res && 'success' in res) {
+      return res;
+    } else {
+      return { success : false };
+    }
+  })
+  .catch(e => {
+    return { success : false };
+  });
+}
+

packages/components/src/api/storyteller_filter/index.ts

@@ -0,0 +1,7 @@
+export type {
+  EnqueueStorytellerFilterRequest, EnqueueStorytellerFilterResponse,
+} from './EnqueueStorytellerFilter';
+
+export {
+  EnqueueStorytellerFilter
+} from './EnqueueStorytellerFilter';

packages/components/src/jobs/InferenceJob.ts

@@ -11,7 +11,8 @@ export enum FrontendInferenceJobType {
   ImageGeneration,
   VideoMotionCapture,
   ConvertFbxtoGltf,
-  EngineComposition
+  EngineComposition,
+  StorytellerFilter,
 }
 
 // NB: Many of these fields are optional despite the response payload containing them

packages/fakeyou.com/src/App.tsx

@@ -210,6 +210,7 @@ function initInferenceJobsByCategoryMap(): Map<
   inferenceJobsByCategory.set(FrontendInferenceJobType.VideoMotionCapture, []);
   inferenceJobsByCategory.set(FrontendInferenceJobType.ImageGeneration, []);
   inferenceJobsByCategory.set(FrontendInferenceJobType.ConvertFbxtoGltf, []);
+  inferenceJobsByCategory.set(FrontendInferenceJobType.StorytellerFilter, [])
   return inferenceJobsByCategory;
 }
 

packages/fakeyou.com/src/components/common/Input/Input.tsx

@@ -5,14 +5,14 @@ import { IconDefinition } from "@fortawesome/fontawesome-svg-core";
 interface InputProps extends React.InputHTMLAttributes<HTMLInputElement> {
   icon?: IconDefinition;
   label?: string;
-  textArea?: boolean;
+  // textArea?: boolean;
   className?: string;
 }
 
 export default function Input({
   label,
   icon,
-  textArea,
+  // textArea,
   className,
   ...rest
 }: InputProps) {

packages/fakeyou.com/src/v2/view/pages/storyteller_filter/StorytellerFilter.tsx

@@ -1,6 +1,6 @@
-import React, {useReducer } from 'react';
+import React, {useReducer, useEffect } from 'react';
 
-import { useLocalize } from "hooks";
+import { useInferenceJobs, useLocalize } from "hooks";
 import { FrontendInferenceJobType, InferenceJob } from '@storyteller/components/src/jobs/InferenceJob';
 
 import { Container } from "components/common";
@@ -24,6 +24,26 @@ export default function StorytellerFilter(props:{
     status: NO_FILE,
   });
 
+  const { enqueueInferenceJob } = props;
+  useInferenceJobs(
+    FrontendInferenceJobType.StorytellerFilter
+  );
+  useEffect(() => {
+    if (
+      pageState.status === states.FILTER_ENQUEUED &&
+      pageState.inferenceJobToken
+    ) {
+      enqueueInferenceJob(
+        pageState.inferenceJobToken,
+        FrontendInferenceJobType.StorytellerFilter
+      );
+      dispatchPageState({
+        type: "enqueueFilterSuccess",
+        payload: { inferenceJobToken: undefined },
+      });
+    }
+  }, [pageState, enqueueInferenceJob]);
+
   return(
     <Container type="panel" className="mb-5">
       <PageHeader

packages/fakeyou.com/src/v2/view/pages/storyteller_filter/components/pageFilterControls.tsx

@@ -1,11 +1,16 @@
-import React, {useState, useEffect} from "react";
-import { useParams } from "react-router-dom";
-
+import React, {useState} from "react";
+import { useParams, useHistory } from "react-router-dom";
+import { v4 as uuidv4 } from "uuid";
+import { faChevronRight } from "@fortawesome/pro-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { useMedia } from "hooks";
+
 import { BucketConfig } from "@storyteller/components/src/api/BucketConfig";
+import { EnqueueStorytellerFilter } from "@storyteller/components/src/api/storyteller_filter";
 import { states, Action, State } from "../storytellerFilterReducer";
 import {
   Accordion,
+  Button,
   BasicVideo,
   Checkbox,
   ErrorMessage,
@@ -15,13 +20,12 @@ import {
   Panel,
   SelectModal,
   Spinner,
-  TextAreaV2
+  TextArea
 } from "components/common";
-import { faChevronRight } from "@fortawesome/pro-solid-svg-icons";
-import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+
 
 export default function PageFilterControls({
-  debug=false, t, pageState, dispatchPageState
+  t, pageState, dispatchPageState
 }: {
   debug?: boolean;
   t: Function;
@@ -32,7 +36,6 @@ export default function PageFilterControls({
   useMedia({
     mediaToken: pageState.mediaFileToken || mediaToken,
     onSuccess: (res: any) => {
-      // ratings.gather({ res, key: "token" });
       dispatchPageState({
         type: 'loadFileSuccess',
         payload: {
@@ -44,6 +47,8 @@ export default function PageFilterControls({
   });
 
   const [filterState, setFilterState] = useState({
+    outputPath: "vid2vid/SparseUpscaleInterp_00001.mp4",
+    workflowConfig: "weight_q8sz47gmfw2zx02snrbz88ns9",
     seed: "",
     sdModelToken: "",
     loraModelToken: "",
@@ -58,16 +63,74 @@ export default function PageFilterControls({
     denoiseFaceDetailer: 0.45,
     useLCM: false,
     lcmCFG: 2,
-    lcmSteps: 8
+    lcmSteps: 8,
+    framesCap:16,
+    skipFrames:0,
+    nthFrames:2,
+    inputFps: 24,
+    interpolationMutiplier: 2,
+    cnTile: 0,
+    cnCanny: 0,
+    cnLinearAnime: 0,
+    cnLinearRealistic: 0,
+    cnDepth: 0,
+    cnOpenPose: 0,
+    cnPipeFace: 0,
+    cnSparse: 0.7,
   });
-  useEffect(()=>{
-    if(debug) console.log(filterState)
-  },[filterState, debug])
 
   const handleOnChange = (key: string, newValue:any,) => {
     setFilterState((curr)=>({...curr, [key]: newValue}));
   }
 
+  const history = useHistory();
+  const handleGenerate = ()=>{
+    console.log("filterState:")
+    console.log(filterState)
+
+    const request = {
+      "uuid_idempotency_token": uuidv4(),
+      "maybe_sd_model": filterState.sdModelToken,
+      "maybe_workflow_config": filterState.workflowConfig,
+      "maybe_input_file": pageState.mediaFileToken || mediaToken,
+      "maybe_output_path": filterState.outputPath,
+      "maybe_json_modifications": {
+        "$.510.inputs.Text": filterState.posPrompt,
+        "$.8.inputs.text": filterState.negPrompt,
+        "$.173.inputs.seed": filterState.seed,
+        "$.401.inputs.Value": filterState.firstPass,
+        "$.918.inputs.Value": filterState.motionScale,
+        "$.137.inputs.Value": filterState.framesCap,
+        "$.186.inputs.Value": filterState.skipFrames,
+        "$.140.inputs.Value": filterState.nthFrames,
+        "$.154.inputs.Value": filterState.inputFps,
+        "$.445.inputs.number": filterState.interpolationMutiplier,
+        "$.947.inputs.Value": filterState.cnTile,
+        "$.800.inputs.Value": filterState.cnCanny,
+        "$.797.inputs.Value": filterState.cnLinearAnime,
+        "$.796.inputs.Value": filterState.cnLinearRealistic,
+        "$.772.inputs.Value": filterState.cnDepth,
+        "$.771.inputs.Value": filterState.cnOpenPose,
+        "$.527.inputs.Value": filterState.cnPipeFace,
+        "$.403.inputs.Value": filterState.cnSparse
+      },
+    }
+    EnqueueStorytellerFilter(request).then(res => {
+      if (res.success && res.inference_job_token) {
+        dispatchPageState({
+          type: 'enqueueFilterSuccess',
+          payload: {
+            inferenceJobToken: res.inference_job_token
+          }
+        })
+      }else{
+        console.log(res);
+      }
+    })
+    dispatchPageState({type: 'enqueueFilter'})
+    history.push("/storyteller-filter/jobs");
+  }
+
   if (pageState.mediaFile){
     const mediaLink = new BucketConfig().getGcsUrl(pageState.mediaFile.public_bucket_path);
 
@@ -76,7 +139,6 @@ export default function PageFilterControls({
         <>
           <Panel>
             <div className="row g-3 p-3">
-              {debug && <p>{`File Token: ${pageState.mediaFileToken}`}</p> }
               <div className="col-5">
                 <BasicVideo src={mediaLink} />
               </div>
@@ -143,22 +205,22 @@ export default function PageFilterControls({
               </div>
               <div className="row g-3 p-3">
                 <div className="col-md-6">
-                  <TextAreaV2
+                  <TextArea
                     {...{
                       label: "Prompt",
                       placeholder: "Enter a prompt",
-                      onChange: (val:string)=>handleOnChange("posPrompt", val),
+                      onChange: (e:React.ChangeEvent<HTMLTextAreaElement>)=>handleOnChange("posPrompt", e.target.value),
                       value: filterState.posPrompt,
                       required: false,
                     }}
                   />
                 </div>
                 <div className="col-md-6">
-                  <TextAreaV2
+                  <TextArea
                     {...{
                       label: "Negative Prompt",
                       placeholder: "Enter Negative Prompt",
-                      onChange: (val:string)=>handleOnChange("negPrompt", val),
+                      onChange: (e:React.ChangeEvent<HTMLTextAreaElement>)=>handleOnChange("negPrompt", e.target.value),
                       value: filterState.negPrompt,
                       required: false,
                     }}
@@ -176,9 +238,7 @@ export default function PageFilterControls({
               <div className="row g-3 p-3">
                 <div className="col-md-6">
                   <NumberSliderV2 {...{
-                      min: 0.1,
-                      max: 1,
-                      step: 0.01,
+                      min: 0.1, max: 1, step: 0.01,
                       initialValue: filterState.firstPass,
                       label: " Denoise First Pass",
                       thumbTip: "Denoise First Pass",
@@ -187,9 +247,7 @@ export default function PageFilterControls({
                 </div>
                 <div className="col-md-6">
                   <NumberSliderV2 {...{
-                    min: 0.1,
-                    max: 1,
-                    step: 0.01,
+                    min: 0.1, max: 1, step: 0.01,
                     initialValue: filterState.upscalePass,
                     label: "Denoise Upscale Pass",
                     thumbTip: "Denoise Upscale Pass",
@@ -200,9 +258,7 @@ export default function PageFilterControls({
               <div className="row g-3 p-3">
                 <div className="col-md-6">
                   <NumberSliderV2 {...{
-                    min: 0.25,
-                    max: 2,
-                    step: 0.05,
+                    min: 0.25, max: 2, step: 0.05,
                     initialValue: filterState.motionScale,
                     label: "Motion Scale",
                     thumbTip: "Motion Scale",
@@ -211,9 +267,7 @@ export default function PageFilterControls({
                 </div>
                 <div className="col-md-6">
                   <NumberSliderV2 {...{
-                    min: 1.5,
-                    max: 2,
-                    step: 0.5,
+                    min: 1.5, max: 2, step: 0.5,
                     initialValue: filterState.upscaleMultiplier,
                     label: "Upscale Multiplier",
                     thumbTip: "Upscale Multiplier",
@@ -243,9 +297,7 @@ export default function PageFilterControls({
                 </div>
                 <div className="col-md-6">
                   <NumberSliderV2 {...{
-                    min: 0.1,
-                    max: 0.7,
-                    step: 0.01,
+                    min: 0.1, max: 0.7, step: 0.01,
                     initialValue: filterState.denoiseFaceDetailer,
                     label: "Denoise Face Detailer",
                     thumbTip: "Denoise Face Detailer",
@@ -263,9 +315,7 @@ export default function PageFilterControls({
               <div className="row g-3 p-3">
                 <div className="col-md-6">
                   <NumberSliderV2 {...{
-                    min: 1,
-                    max: 10,
-                    step: 1,
+                    min: 1, max: 10, step: 1,
                     initialValue: filterState.lcmCFG,
                     label: "LCM CFG",
                     thumbTip: "LCM CFG",
@@ -274,9 +324,7 @@ export default function PageFilterControls({
                 </div>
                 <div className="col-md-6">
                   <NumberSliderV2 {...{
-                    min: 5,
-                    max: 15,
-                    step: 1,
+                    min: 5, max: 15, step: 1,
                     initialValue: filterState.lcmSteps,
                     label: "LCM Steps",
                     thumbTip: "LCM Steps",
@@ -286,108 +334,83 @@ export default function PageFilterControls({
               </div>
             </Accordion.Item>
             <Accordion.Item title="Control Nets" defaultOpen>
-              <div className="row g-3 p-3">
-                <Checkbox 
-                  label="Use Control Nets" 
-                  checked={false}
-                  onChange={(e:{target:{ checked: boolean, name:string, type: string }})=>{
-                    console.log(`Use Control Nets: ${e.target.checked}`)
-                  }}
-                />
-              </div>
               <div className="row g-3 p-3">
                 <div className="col-md-6">
                   <NumberSliderV2 {...{
-                    min: 0,
-                    max: 1,
-                    step: 0.1,
-                    initialValue: 0,
+                    min: 0, max: 1, step: 0.1,
+                    initialValue: filterState.cnCanny,
                     label: "Canny",
                     thumbTip: "Canny",
-                    onChange: (val)=>{console.log(`Canny: ${val}`)}
+                    onChange: (val)=>{handleOnChange("cnCanny", val)}
                     }}/>
                 </div>
                 <div className="col-md-6">
                   <NumberSliderV2 {...{
-                    min: 0,
-                    max: 1,
-                    step: 0.1,
-                    initialValue: 0,
+                    min: 0, max: 1, step: 0.1,
+                    initialValue: filterState.cnLinearAnime,
                     label: "Line Art Anime",
                     thumbTip: "Line Art Anime",
-                    onChange: (val)=>{console.log(`Line Art Anime: ${val}`)}
+                    onChange: (val)=>{handleOnChange("cnLinearAnime", val)}
                     }}/>
                 </div>
               </div>
               <div className="row g-3 p-3">
                 <div className="col-md-6">
                   <NumberSliderV2 {...{
-                    min: 0,
-                    max: 1,
-                    step: 0.1,
-                    initialValue: 0,
+                    min: 0, max: 1, step: 0.1,
+                    initialValue: filterState.cnDepth,
                     label: "Depth",
                     thumbTip: "Depth",
-                    onChange: (val)=>{console.log(`Depth: ${val}`)}
+                    onChange: (val)=>{handleOnChange("cnDepth", val)}
                     }}/>
                 </div>
                 <div className="col-md-6">
                   <NumberSliderV2 {...{
-                    min: 0,
-                    max: 1,
-                    step: 0.1,
-                    initialValue: 0,
+                    min: 0, max: 1, step: 0.1,
+                    initialValue: filterState.cnOpenPose,
                     label: "OpenPose",
                     thumbTip: "OpenPose",
-                    onChange: (val)=>{console.log(`OpenPose: ${val}`)}
+                    onChange: (val)=>{handleOnChange("cnOpenPose", val)}
                     }}/>
                 </div>
               </div>
               <div className="row g-3 p-3">
                 <div className="col-md-6">
                   <NumberSliderV2 {...{
-                    min: 0,
-                    max: 1,
-                    step: 0.1,
-                    initialValue: 0,
+                    min: 0, max: 1, step: 0.1,
+                    initialValue: filterState.cnPipeFace,
                     label: "Media Pipe Face",
                     thumbTip: "Media Pipe Face",
-                    onChange: (val)=>{console.log(`Media Pipe Face: ${val}`)}
+                    onChange: (val)=>{handleOnChange("cnPipeFace", val)}
                     }}/>
                 </div>
                 <div className="col-md-6">
                   <NumberSliderV2 {...{
-                    min: 0,
-                    max: 1,
-                    step: 0.1,
-                    initialValue: 0.7,
+                    min: 0, max: 1, step: 0.1,
+                    initialValue: filterState.cnSparse,
                     label: "Sparse Scribble",
                     thumbTip: "Sparse Scribble",
-                    onChange: (val)=>{console.log(`Sparse Scribble: ${val}`)}
+                    onChange: (val)=>{handleOnChange("cnSparse", val)}
                     }}/>
                 </div>
               </div>
               <div className="row g-3 p-3">
                 <div className="col-md-6">
                   <NumberSliderV2 {...{
-                    min: 0,
-                    max: 1,
-                    step: 0.1,
-                    initialValue: 0.5,
-                    label: "Video CN",
+                    min: 0, max: 1, step: 0.1,
+                    initialValue: filterState.cnLinearRealistic,
+                    label: "Video CN (Linear Realistic)",
                     thumbTip: "Video CN",
-                    onChange: (val)=>{console.log(`Video CN: ${val}`)}
+                    onChange: (val)=>{handleOnChange("cnLinearRealistic", val)}
                     }}/>
                 </div>
                 <div className="col-md-6">
                   <NumberSliderV2 {...{
-                    min: 0,
-                    max: 1,
-                    step: 0.1,
-                    initialValue: 0,
+                    min: 0, max: 1, step: 0.1,
+                    initialValue: filterState.cnTile,
                     label: "Tile CN",
                     thumbTip: "Tile CN",
-                    onChange: (val)=>{console.log(`Tile CN: ${val}`)}
+                    onChange: (val)=>{handleOnChange("cnTile", val)}
                     }}/>
                 </div>
               </div>
@@ -414,7 +437,7 @@ export default function PageFilterControls({
                   <Input label="Every n-th Frame" />
                 </div>
                 <div className="col-md-6">
-                  <Input label="Input FPS" />
+                  <Input label="Input FPS" readOnly/>
                 </div>
               </div>
               <div className="row g-3 p-3">
@@ -426,12 +449,20 @@ export default function PageFilterControls({
               </div>
             </Accordion.Item>
           </Accordion>
+          <div className="row g-3 py-3">
+            <div className="col-12 d-flex justify-content-end">
+              <Button
+                label={t("Process Filter")}
+                onClick={handleGenerate}
+                variant="primary"
+              />
+            </div>
+          </div>
         </>
       );
   }else if (pageState.status <= states.FILE_LOADING){
     return (
       <Panel>
-        {debug && <p>{`File Token: ${pageState.mediaFileToken}`}</p> }
         <p>Loading Files</p>
         <Spinner />
       </Panel>

packages/fakeyou.com/src/v2/view/pages/storyteller_filter/components/pageJobList.tsx

@@ -0,0 +1,49 @@
+import React from 'react'
+
+import { Analytics } from "common/Analytics";
+import InferenceJobsList from "components/layout/InferenceJobsList";
+import { FrontendInferenceJobType } from "@storyteller/components/src/jobs/InferenceJob";
+import { states, Action, State } from "../storytellerFilterReducer";
+import { Spinner } from 'components/common';
+
+function StorytellerJobList(){
+  const failures = (fail = "") => {
+    switch (fail) {
+      case "sample case": 
+        return "Sample Case, this should not have been shown";
+      default:
+        return "Unknown failure";
+    }
+  };
+
+  return (
+    <InferenceJobsList {...{
+      failures,
+      onSelect: () => Analytics.voiceConversionClickDownload(),
+      jobType: FrontendInferenceJobType.StorytellerFilter,
+    }}/>
+  );
+}
+
+export default function PageJobList({
+  t, pageState, dispatchPageState
+}: {
+  debug?: boolean;
+  t: Function;
+  pageState: State;
+  dispatchPageState: (action: Action) => void;
+}) {
+  console.log(pageState);
+  return(
+    <>
+      <h1>Jobs</h1>
+      {pageState.status === states.FILTER_ENQUEUEING &&
+        <div>
+          <h2> Requesting Filter Job</h2>
+          <Spinner />
+        </div>
+      }
+      <StorytellerJobList />
+    </>
+  );
+}

packages/fakeyou.com/src/v2/view/pages/storyteller_filter/storytellerFilterReducer.tsx

@@ -8,6 +8,8 @@ export enum states{
   FILE_UPLOADED,
   FILE_LOADING,
   FILE_LOADED,
+  FILTER_ENQUEUEING,
+  FILTER_ENQUEUED
 }
 
 export type State = {
@@ -27,6 +29,8 @@ export type Action =
   | {type: 'loadFileSuccess', payload:{
       mediaFileToken: string, mediaFile: MediaFile
     }}
+  | {type: 'enqueueFilter'}
+  | {type: 'enqueueFilterSuccess', payload: {inferenceJobToken: string|undefined}}
 
 
 export function reducer (state: State, action: Action): State {
@@ -60,6 +64,17 @@ export function reducer (state: State, action: Action): State {
         mediaFile: action.payload.mediaFile,
         mediaFileToken : action.payload.mediaFileToken,
       }
+    case 'enqueueFilter':
+      return {
+        ...state,
+        status: states.FILTER_ENQUEUEING
+      }
+    case 'enqueueFilterSuccess':
+      return {
+        ...state,
+        status: states.FILTER_ENQUEUED,
+        inferenceJobToken: action.payload.inferenceJobToken
+      }
     default:
       return {status: states.NO_FILE};
       

packages/fakeyou.com/src/v2/view/pages/storyteller_filter/storytellerFilterRoutes.tsx

@@ -2,6 +2,7 @@ import React from "react";
 import {Switch, Route, useRouteMatch, Redirect} from "react-router-dom";
 import PageFilterControls from "./components/pageFilterControls";
 import PageVideoProvision from "./components/pageVideoProvision";
+import PageJobList from "./components/pageJobList";
 import {State,Action} from "./storytellerFilterReducer";
 
 export default function SubRoutes ({
@@ -25,6 +26,9 @@ export default function SubRoutes ({
       <Route exact path={`${path}/select-media`}>
         <PageVideoProvision {...{debug, t, pageState, dispatchPageState}}/>
       </Route>
+      <Route exact path={`${path}/jobs`}>
+        <PageJobList {...{debug, t, pageState, dispatchPageState}}/>
+      </Route>
       <Route path={`${path}`}>
         <Redirect to={`${path}/upload`} />
       </Route>