From 0dd87a9a6fcbe7149d2dd9f3a360edc3bbb2d34d Mon Sep 17 00:00:00 2001 From: wil Date: Mon, 5 Feb 2024 17:59:38 -0500 Subject: [PATCH] completed UI flow --- packages/components/src/api/ApiConfig.ts | 5 + .../EnqueueStorytellerFilter.tsx | 68 ++++++ .../src/api/storyteller_filter/index.ts | 7 + packages/components/src/jobs/InferenceJob.ts | 3 +- packages/fakeyou.com/src/App.tsx | 1 + .../src/components/common/Input/Input.tsx | 4 +- .../storyteller_filter/StorytellerFilter.tsx | 24 +- .../components/pageFilterControls.tsx | 211 ++++++++++-------- .../components/pageJobList.tsx | 49 ++++ .../storytellerFilterReducer.tsx | 15 ++ .../storytellerFilterRoutes.tsx | 4 + 11 files changed, 296 insertions(+), 95 deletions(-) create mode 100644 packages/components/src/api/storyteller_filter/EnqueueStorytellerFilter.tsx create mode 100644 packages/components/src/api/storyteller_filter/index.ts create mode 100644 packages/fakeyou.com/src/v2/view/pages/storyteller_filter/components/pageJobList.tsx diff --git a/packages/components/src/api/ApiConfig.ts b/packages/components/src/api/ApiConfig.ts index 103a13eb4e..59df11f2ba 100644 --- a/packages/components/src/api/ApiConfig.ts +++ b/packages/components/src/api/ApiConfig.ts @@ -496,6 +496,11 @@ class ApiConfig { return `${this.getApiOrigin()}/v1/mocap/mocapnet/create`; } + // =============== Storyteller Filter =============== + enqueueStorytellerFilter(): string { + return `${this.getApiOrigin()}/v1/workflow/comfy/create`; + } + // =============== Convert FBX to glTF =============== enqueueFbxToGltf(): string { return `${this.getApiOrigin()}/v1/conversion/enqueue_fbx_to_gltf`; diff --git a/packages/components/src/api/storyteller_filter/EnqueueStorytellerFilter.tsx b/packages/components/src/api/storyteller_filter/EnqueueStorytellerFilter.tsx new file mode 100644 index 0000000000..1da9000fc3 --- /dev/null +++ b/packages/components/src/api/storyteller_filter/EnqueueStorytellerFilter.tsx @@ -0,0 +1,68 @@ +import { ApiConfig } from "@storyteller/components"; + +export interface EnqueueStorytellerFilterRequest { + "uuid_idempotency_token": string, + + "maybe_sd_model": string, + "maybe_workflow_config": string, + "maybe_input_file": string, + "maybe_output_path": string, + "maybe_json_modifications": { + "$.510.inputs.Text": string, + "$.8.inputs.text": string, + "$.173.inputs.seed": string, + "$.401.inputs.Value": number, + "$.918.inputs.Value": number, + "$.137.inputs.Value": number, + "$.186.inputs.Value": number, + "$.140.inputs.Value":number, + "$.154.inputs.Value": number, + "$.445.inputs.number": number, + "$.947.inputs.Value": number, + "$.800.inputs.Value": number, + "$.797.inputs.Value": number, + "$.796.inputs.Value": number, + "$.772.inputs.Value": number, + "$.771.inputs.Value": number, + "$.527.inputs.Value": number, + "$.403.inputs.Value": number, + } +} + +export interface EnqueueStorytellerFilterResponse { + success: boolean, + inference_job_token?: string, +} + +export async function EnqueueStorytellerFilter(request: EnqueueStorytellerFilterRequest) : Promise +{ + // const endpoint = "/v1/workflow/comfy/create"; + + const endpoint = new ApiConfig().enqueueStorytellerFilter(); + + return await fetch(endpoint, { + method: 'POST', + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json', + }, + credentials: 'include', + body: JSON.stringify(request), + }) + .then(res => res.json()) + .then(res => { + if (!res) { + return { success : false }; + } + + if (res && 'success' in res) { + return res; + } else { + return { success : false }; + } + }) + .catch(e => { + return { success : false }; + }); +} + diff --git a/packages/components/src/api/storyteller_filter/index.ts b/packages/components/src/api/storyteller_filter/index.ts new file mode 100644 index 0000000000..aebe56e156 --- /dev/null +++ b/packages/components/src/api/storyteller_filter/index.ts @@ -0,0 +1,7 @@ +export type { + EnqueueStorytellerFilterRequest, EnqueueStorytellerFilterResponse, +} from './EnqueueStorytellerFilter'; + +export { + EnqueueStorytellerFilter +} from './EnqueueStorytellerFilter'; \ No newline at end of file diff --git a/packages/components/src/jobs/InferenceJob.ts b/packages/components/src/jobs/InferenceJob.ts index 53aa9ae0f7..88388e07b2 100644 --- a/packages/components/src/jobs/InferenceJob.ts +++ b/packages/components/src/jobs/InferenceJob.ts @@ -11,7 +11,8 @@ export enum FrontendInferenceJobType { ImageGeneration, VideoMotionCapture, ConvertFbxtoGltf, - EngineComposition + EngineComposition, + StorytellerFilter, } // NB: Many of these fields are optional despite the response payload containing them diff --git a/packages/fakeyou.com/src/App.tsx b/packages/fakeyou.com/src/App.tsx index 12e574f6ff..e6b7de0e21 100644 --- a/packages/fakeyou.com/src/App.tsx +++ b/packages/fakeyou.com/src/App.tsx @@ -210,6 +210,7 @@ function initInferenceJobsByCategoryMap(): Map< inferenceJobsByCategory.set(FrontendInferenceJobType.VideoMotionCapture, []); inferenceJobsByCategory.set(FrontendInferenceJobType.ImageGeneration, []); inferenceJobsByCategory.set(FrontendInferenceJobType.ConvertFbxtoGltf, []); + inferenceJobsByCategory.set(FrontendInferenceJobType.StorytellerFilter, []) return inferenceJobsByCategory; } diff --git a/packages/fakeyou.com/src/components/common/Input/Input.tsx b/packages/fakeyou.com/src/components/common/Input/Input.tsx index edfaef74c0..b9fe5ca2fb 100644 --- a/packages/fakeyou.com/src/components/common/Input/Input.tsx +++ b/packages/fakeyou.com/src/components/common/Input/Input.tsx @@ -5,14 +5,14 @@ import { IconDefinition } from "@fortawesome/fontawesome-svg-core"; interface InputProps extends React.InputHTMLAttributes { icon?: IconDefinition; label?: string; - textArea?: boolean; + // textArea?: boolean; className?: string; } export default function Input({ label, icon, - textArea, + // textArea, className, ...rest }: InputProps) { diff --git a/packages/fakeyou.com/src/v2/view/pages/storyteller_filter/StorytellerFilter.tsx b/packages/fakeyou.com/src/v2/view/pages/storyteller_filter/StorytellerFilter.tsx index 5ee9447359..624df75df1 100644 --- a/packages/fakeyou.com/src/v2/view/pages/storyteller_filter/StorytellerFilter.tsx +++ b/packages/fakeyou.com/src/v2/view/pages/storyteller_filter/StorytellerFilter.tsx @@ -1,6 +1,6 @@ -import React, {useReducer } from 'react'; +import React, {useReducer, useEffect } from 'react'; -import { useLocalize } from "hooks"; +import { useInferenceJobs, useLocalize } from "hooks"; import { FrontendInferenceJobType, InferenceJob } from '@storyteller/components/src/jobs/InferenceJob'; import { Container } from "components/common"; @@ -24,6 +24,26 @@ export default function StorytellerFilter(props:{ status: NO_FILE, }); + const { enqueueInferenceJob } = props; + useInferenceJobs( + FrontendInferenceJobType.StorytellerFilter + ); + useEffect(() => { + if ( + pageState.status === states.FILTER_ENQUEUED && + pageState.inferenceJobToken + ) { + enqueueInferenceJob( + pageState.inferenceJobToken, + FrontendInferenceJobType.StorytellerFilter + ); + dispatchPageState({ + type: "enqueueFilterSuccess", + payload: { inferenceJobToken: undefined }, + }); + } + }, [pageState, enqueueInferenceJob]); + return( { - // ratings.gather({ res, key: "token" }); dispatchPageState({ type: 'loadFileSuccess', payload: { @@ -44,6 +47,8 @@ export default function PageFilterControls({ }); const [filterState, setFilterState] = useState({ + outputPath: "vid2vid/SparseUpscaleInterp_00001.mp4", + workflowConfig: "weight_q8sz47gmfw2zx02snrbz88ns9", seed: "", sdModelToken: "", loraModelToken: "", @@ -58,16 +63,74 @@ export default function PageFilterControls({ denoiseFaceDetailer: 0.45, useLCM: false, lcmCFG: 2, - lcmSteps: 8 + lcmSteps: 8, + framesCap:16, + skipFrames:0, + nthFrames:2, + inputFps: 24, + interpolationMutiplier: 2, + cnTile: 0, + cnCanny: 0, + cnLinearAnime: 0, + cnLinearRealistic: 0, + cnDepth: 0, + cnOpenPose: 0, + cnPipeFace: 0, + cnSparse: 0.7, }); - useEffect(()=>{ - if(debug) console.log(filterState) - },[filterState, debug]) const handleOnChange = (key: string, newValue:any,) => { setFilterState((curr)=>({...curr, [key]: newValue})); } + const history = useHistory(); + const handleGenerate = ()=>{ + console.log("filterState:") + console.log(filterState) + + const request = { + "uuid_idempotency_token": uuidv4(), + "maybe_sd_model": filterState.sdModelToken, + "maybe_workflow_config": filterState.workflowConfig, + "maybe_input_file": pageState.mediaFileToken || mediaToken, + "maybe_output_path": filterState.outputPath, + "maybe_json_modifications": { + "$.510.inputs.Text": filterState.posPrompt, + "$.8.inputs.text": filterState.negPrompt, + "$.173.inputs.seed": filterState.seed, + "$.401.inputs.Value": filterState.firstPass, + "$.918.inputs.Value": filterState.motionScale, + "$.137.inputs.Value": filterState.framesCap, + "$.186.inputs.Value": filterState.skipFrames, + "$.140.inputs.Value": filterState.nthFrames, + "$.154.inputs.Value": filterState.inputFps, + "$.445.inputs.number": filterState.interpolationMutiplier, + "$.947.inputs.Value": filterState.cnTile, + "$.800.inputs.Value": filterState.cnCanny, + "$.797.inputs.Value": filterState.cnLinearAnime, + "$.796.inputs.Value": filterState.cnLinearRealistic, + "$.772.inputs.Value": filterState.cnDepth, + "$.771.inputs.Value": filterState.cnOpenPose, + "$.527.inputs.Value": filterState.cnPipeFace, + "$.403.inputs.Value": filterState.cnSparse + }, + } + EnqueueStorytellerFilter(request).then(res => { + if (res.success && res.inference_job_token) { + dispatchPageState({ + type: 'enqueueFilterSuccess', + payload: { + inferenceJobToken: res.inference_job_token + } + }) + }else{ + console.log(res); + } + }) + dispatchPageState({type: 'enqueueFilter'}) + history.push("/storyteller-filter/jobs"); + } + if (pageState.mediaFile){ const mediaLink = new BucketConfig().getGcsUrl(pageState.mediaFile.public_bucket_path); @@ -76,7 +139,6 @@ export default function PageFilterControls({ <>
- {debug &&

{`File Token: ${pageState.mediaFileToken}`}

}
@@ -143,22 +205,22 @@ export default function PageFilterControls({
- handleOnChange("posPrompt", val), + onChange: (e:React.ChangeEvent)=>handleOnChange("posPrompt", e.target.value), value: filterState.posPrompt, required: false, }} />
- handleOnChange("negPrompt", val), + onChange: (e:React.ChangeEvent)=>handleOnChange("negPrompt", e.target.value), value: filterState.negPrompt, required: false, }} @@ -176,9 +238,7 @@ export default function PageFilterControls({
-
- { - console.log(`Use Control Nets: ${e.target.checked}`) - }} - /> -
{console.log(`Canny: ${val}`)} + onChange: (val)=>{handleOnChange("cnCanny", val)} }}/>
{console.log(`Line Art Anime: ${val}`)} + onChange: (val)=>{handleOnChange("cnLinearAnime", val)} }}/>
{console.log(`Depth: ${val}`)} + onChange: (val)=>{handleOnChange("cnDepth", val)} }}/>
{console.log(`OpenPose: ${val}`)} + onChange: (val)=>{handleOnChange("cnOpenPose", val)} }}/>
{console.log(`Media Pipe Face: ${val}`)} + onChange: (val)=>{handleOnChange("cnPipeFace", val)} }}/>
{console.log(`Sparse Scribble: ${val}`)} + onChange: (val)=>{handleOnChange("cnSparse", val)} }}/>
{console.log(`Video CN: ${val}`)} + onChange: (val)=>{handleOnChange("cnLinearRealistic", val)} }}/>
{console.log(`Tile CN: ${val}`)} + onChange: (val)=>{handleOnChange("cnTile", val)} }}/>
@@ -414,7 +437,7 @@ export default function PageFilterControls({
- +
@@ -426,12 +449,20 @@ export default function PageFilterControls({
+
+
+
+
); }else if (pageState.status <= states.FILE_LOADING){ return ( - {debug &&

{`File Token: ${pageState.mediaFileToken}`}

}

Loading Files

 diff --git a/packages/fakeyou.com/src/v2/view/pages/storyteller_filter/components/pageJobList.tsx b/packages/fakeyou.com/src/v2/view/pages/storyteller_filter/components/pageJobList.tsx new file mode 100644 index 0000000000..b907495487 --- /dev/null +++ b/packages/fakeyou.com/src/v2/view/pages/storyteller_filter/components/pageJobList.tsx @@ -0,0 +1,49 @@ +import React from 'react' + +import { Analytics } from "common/Analytics"; +import InferenceJobsList from "components/layout/InferenceJobsList"; +import { FrontendInferenceJobType } from "@storyteller/components/src/jobs/InferenceJob"; +import { states, Action, State } from "../storytellerFilterReducer"; +import { Spinner } from 'components/common'; + +function StorytellerJobList(){ + const failures = (fail = "") => { + switch (fail) { + case "sample case": + return "Sample Case, this should not have been shown"; + default: + return "Unknown failure"; + } + }; + + return ( + Analytics.voiceConversionClickDownload(), + jobType: FrontendInferenceJobType.StorytellerFilter, + }}/> + ); +} + +export default function PageJobList({ + t, pageState, dispatchPageState +}: { + debug?: boolean; + t: Function; + pageState: State; + dispatchPageState: (action: Action) => void; +}) { + console.log(pageState); + return( + <> +

Jobs

+ {pageState.status === states.FILTER_ENQUEUEING && +
+

Requesting Filter Job

+ +
+ } + + + ); +} \ No newline at end of file diff --git a/packages/fakeyou.com/src/v2/view/pages/storyteller_filter/storytellerFilterReducer.tsx b/packages/fakeyou.com/src/v2/view/pages/storyteller_filter/storytellerFilterReducer.tsx index 87f44546a9..229eace4b2 100644 --- a/packages/fakeyou.com/src/v2/view/pages/storyteller_filter/storytellerFilterReducer.tsx +++ b/packages/fakeyou.com/src/v2/view/pages/storyteller_filter/storytellerFilterReducer.tsx @@ -8,6 +8,8 @@ export enum states{ FILE_UPLOADED, FILE_LOADING, FILE_LOADED, + FILTER_ENQUEUEING, + FILTER_ENQUEUED } export type State = { @@ -27,6 +29,8 @@ export type Action = | {type: 'loadFileSuccess', payload:{ mediaFileToken: string, mediaFile: MediaFile }} + | {type: 'enqueueFilter'} + | {type: 'enqueueFilterSuccess', payload: {inferenceJobToken: string|undefined}} export function reducer (state: State, action: Action): State { @@ -60,6 +64,17 @@ export function reducer (state: State, action: Action): State { mediaFile: action.payload.mediaFile, mediaFileToken : action.payload.mediaFileToken, } + case 'enqueueFilter': + return { + ...state, + status: states.FILTER_ENQUEUEING + } + case 'enqueueFilterSuccess': + return { + ...state, + status: states.FILTER_ENQUEUED, + inferenceJobToken: action.payload.inferenceJobToken + } default: return {status: states.NO_FILE}; diff --git a/packages/fakeyou.com/src/v2/view/pages/storyteller_filter/storytellerFilterRoutes.tsx b/packages/fakeyou.com/src/v2/view/pages/storyteller_filter/storytellerFilterRoutes.tsx index ce67ca1a19..cc4f7c516b 100644 --- a/packages/fakeyou.com/src/v2/view/pages/storyteller_filter/storytellerFilterRoutes.tsx +++ b/packages/fakeyou.com/src/v2/view/pages/storyteller_filter/storytellerFilterRoutes.tsx @@ -2,6 +2,7 @@ import React from "react"; import {Switch, Route, useRouteMatch, Redirect} from "react-router-dom"; import PageFilterControls from "./components/pageFilterControls"; import PageVideoProvision from "./components/pageVideoProvision"; +import PageJobList from "./components/pageJobList"; import {State,Action} from "./storytellerFilterReducer"; export default function SubRoutes ({ @@ -25,6 +26,9 @@ export default function SubRoutes ({ + + +