From 080d26b5029ebbbbe90c472123556ab36fe04fb1 Mon Sep 17 00:00:00 2001 From: ArEnSc <6252325+ArEnSc@users.noreply.github.com> Date: Sun, 23 Jun 2024 12:12:59 -0400 Subject: [PATCH] Feature/ip adapter (#522) --- app/Classes/ApiManager/ApiManager.ts | 2 - app/pages/PageEnigma/Editor/api_manager.ts | 2 + app/pages/PageEnigma/Editor/editor.ts | 99 +++++++++++++------ app/pages/PageEnigma/Editor/save_manager.ts | 15 +-- app/pages/PageEnigma/Editor/timeline.ts | 2 +- .../PageEnigma/Editor/video_generation.ts | 33 ++++++- .../ControlsTopButtons/ControlsTopButtons.tsx | 31 +++--- .../itemTabs/StylizeTab/IPAdapter.tsx | 21 +++- .../PageEnigma/models/generationOptions.ts | 5 +- .../models/sceneGenerationMetadata.ts | 2 +- 10 files changed, 156 insertions(+), 56 deletions(-) diff --git a/app/Classes/ApiManager/ApiManager.ts b/app/Classes/ApiManager/ApiManager.ts index c0ae70e33a..489378fe14 100644 --- a/app/Classes/ApiManager/ApiManager.ts +++ b/app/Classes/ApiManager/ApiManager.ts @@ -160,8 +160,6 @@ export class ApiManager { credentials: "include", body: formData, }); - - return response.json(); } protected camelToSnakeCase(str: string) { diff --git a/app/pages/PageEnigma/Editor/api_manager.ts b/app/pages/PageEnigma/Editor/api_manager.ts index 59bed8eb39..e46fa398e8 100644 --- a/app/pages/PageEnigma/Editor/api_manager.ts +++ b/app/pages/PageEnigma/Editor/api_manager.ts @@ -373,6 +373,7 @@ export class APIManager { use_strength: number = 1.0, use_lipsync: boolean = false, use_cinematic: boolean = false, + use_global_ipa_media_token: string | null = null, ) { const uuid = uuidv4(); @@ -390,6 +391,7 @@ export class APIManager { use_upscaler: use_upscaler, use_strength: use_strength, use_lipsync: use_lipsync, + global_ipa_media_token: use_global_ipa_media_token, use_cinematic: use_cinematic, }; diff --git a/app/pages/PageEnigma/Editor/editor.ts b/app/pages/PageEnigma/Editor/editor.ts index b2797097b1..f5ba059012 100644 --- a/app/pages/PageEnigma/Editor/editor.ts +++ b/app/pages/PageEnigma/Editor/editor.ts @@ -145,6 +145,9 @@ class Editor { generation_options: GenerationOptions; + // just a passive the image to be uploaded. we store the token and use that in snapshots. + globalIpAdapterImage: File | undefined; + media_upload: MediaUploadApi; sceneManager: SceneManager | undefined; @@ -244,9 +247,12 @@ class Editor { styleStrength: 1.0, lipSync: false, cinematic: false, + globalIpAdapterImageMediaToken: "", }; this.media_upload = new MediaUploadApi(); + this.globalIpAdapterImage = undefined; // used to display when loading in the app. and to serialize to an image token + // TODO REMOVE this.outliner_feature_flag = true; } getRenderDimensions() { @@ -487,20 +493,45 @@ class Editor { this.cam_obj = this.activeScene.get_object_by_name(this.camera_name); if (this.outliner_feature_flag) { - let result = this.sceneManager?.render_outliner(this.timeline.characters); + const result = this.sceneManager?.render_outliner( + this.timeline.characters, + ); if (result) outlinerState.items.value = result.items; } - this.mouse_controls = new MouseControls(this.camera, this.camera_person_mode, this.lockControls, - this.camera_last_pos, this.orbitControls, - this.selectedCanvas, this.switchPreviewToggle, this.rendering, this.togglePlayback.bind(this), - this.deleteObject.bind(this), this.canvReference, this.mouse, this.timeline.mouse, - this.raycaster, this.control, this.outlinePass, this.activeScene.scene, - this.publishSelect.bind(this), this.updateSelectedUI.bind(this), - this.transform_interaction, this.last_selected, this.getAssetType.bind(this), this.setSelected.bind(this)); + this.mouse_controls = new MouseControls( + this.camera, + this.camera_person_mode, + this.lockControls, + this.camera_last_pos, + this.orbitControls, + this.selectedCanvas, + this.switchPreviewToggle, + this.rendering, + this.togglePlayback.bind(this), + this.deleteObject.bind(this), + this.canvReference, + this.mouse, + this.timeline.mouse, + this.raycaster, + this.control, + this.outlinePass, + this.activeScene.scene, + this.publishSelect.bind(this), + this.updateSelectedUI.bind(this), + this.transform_interaction, + this.last_selected, + this.getAssetType.bind(this), + this.setSelected.bind(this), + ); if (this.outliner_feature_flag) { - this.sceneManager = new SceneManager(this.version, this.mouse_controls, this.activeScene, true); // Enabled dev mode. + this.sceneManager = new SceneManager( + this.version, + this.mouse_controls, + this.activeScene, + true, + ); // Enabled dev mode. this.mouse_controls.sceneManager = this.sceneManager; } @@ -588,15 +619,20 @@ class Editor { }); if (this.outliner_feature_flag) { - let result = this.sceneManager?.render_outliner(this.timeline.characters); + const result = this.sceneManager?.render_outliner( + this.timeline.characters, + ); if (result) outlinerState.items.value = result.items; } } public async loadScene(scene_media_token: string) { await this.save_manager.loadScene(scene_media_token); + if (this.outliner_feature_flag) { - let result = this.sceneManager?.render_outliner(this.timeline.characters); + const result = this.sceneManager?.render_outliner( + this.timeline.characters, + ); if (result) outlinerState.items.value = result.items; } // publish to the UI the values for the prompts and artistic style and settings? @@ -622,8 +658,8 @@ class Editor { } lockUnlockObject(object_uuid: string): boolean { - let res = this.utils.lockUnlockObject(object_uuid); - if(this.outliner_feature_flag) { + const res = this.utils.lockUnlockObject(object_uuid); + if (this.outliner_feature_flag) { this.updateSelectedUI(); } return res; @@ -734,7 +770,9 @@ class Editor { deleteObject(uuid: string) { this.utils.deleteObject(uuid); if (this.outliner_feature_flag) { - let result = this.sceneManager?.render_outliner(this.timeline.characters); + const result = this.sceneManager?.render_outliner( + this.timeline.characters, + ); if (result) outlinerState.items.value = result.items; } } @@ -763,14 +801,14 @@ class Editor { this.render_camera_aspect_ratio === CameraAspectRatio.HORIZONTAL_16_9 ? 1024 : this.render_camera_aspect_ratio === - CameraAspectRatio.VERTICAL_9_16 + CameraAspectRatio.VERTICAL_9_16 ? 576 : 1000; const height = this.render_camera_aspect_ratio === CameraAspectRatio.HORIZONTAL_16_9 ? 576 : this.render_camera_aspect_ratio === - CameraAspectRatio.VERTICAL_9_16 + CameraAspectRatio.VERTICAL_9_16 ? 1024 : 1000; @@ -818,7 +856,9 @@ class Editor { this.cam_obj = this.activeScene.get_object_by_name(this.camera_name); if (this.outliner_feature_flag) { - let result = this.sceneManager?.render_outliner(this.timeline.characters); + const result = this.sceneManager?.render_outliner( + this.timeline.characters, + ); if (result) outlinerState.items.value = result.items; } } @@ -929,6 +969,7 @@ class Editor { // This initializes the generation of a video render scene is where the core work happens generateVideo() { console.log("Generating video...", this.frame_buffer); + this.timeline.is_playing = false; this.timeline.scrubber_frame_position = 0; this.timeline.current_time = 0; @@ -987,7 +1028,10 @@ class Editor { updateSelectedUI() { let mainSelected; if (this.outliner_feature_flag) { - if (this.sceneManager?.selected_objects === undefined || this.timeline.is_playing) { + if ( + this.sceneManager?.selected_objects === undefined || + this.timeline.is_playing + ) { return; } if (this.sceneManager?.selected_objects.length <= 0) { @@ -995,8 +1039,7 @@ class Editor { } mainSelected = this.sceneManager?.selected_objects[0]; - } - else { + } else { if (this.timeline.is_playing) { return; } @@ -1099,8 +1142,11 @@ class Editor { } publishSelect() { - if (this, this.outliner_feature_flag) { - if (this.sceneManager?.selected_objects && this.sceneManager?.selected_objects?.length > 0) { + if ((this, this.outliner_feature_flag)) { + if ( + this.sceneManager?.selected_objects && + this.sceneManager?.selected_objects?.length > 0 + ) { Queue.publish({ queueName: QueueNames.FROM_ENGINE, action: fromEngineActions.SELECT_OBJECT, @@ -1113,16 +1159,14 @@ class Editor { } as MediaItem, }); return; - } - else { + } else { Queue.publish({ queueName: QueueNames.FROM_ENGINE, action: fromEngineActions.DESELECT_OBJECT, data: null, }); } - } - else { + } else { if (this.selected) { Queue.publish({ queueName: QueueNames.FROM_ENGINE, @@ -1136,8 +1180,7 @@ class Editor { } as MediaItem, }); return; - } - else { + } else { Queue.publish({ queueName: QueueNames.FROM_ENGINE, action: fromEngineActions.DESELECT_OBJECT, diff --git a/app/pages/PageEnigma/Editor/save_manager.ts b/app/pages/PageEnigma/Editor/save_manager.ts index 8770647698..592f6fd6a8 100644 --- a/app/pages/PageEnigma/Editor/save_manager.ts +++ b/app/pages/PageEnigma/Editor/save_manager.ts @@ -1,3 +1,4 @@ +import { scene } from "~/signals"; import { SceneGenereationMetaData } from "../models/sceneGenerationMetadata"; import { StoryTellerProxyScene } from "../proxy/storyteller_proxy_scene"; import { StoryTellerProxyTimeline } from "../proxy/storyteller_proxy_timeline"; @@ -52,13 +53,6 @@ export class SaveManager { timeline: timeline_json, }; - // const result = await this.editor.api_manager.saveSceneState({ - // saveJson: JSON.stringify(save_data), - // sceneTitle, - // sceneToken, - // sceneThumbnail, - // }); - return JSON.stringify(save_data); } @@ -124,6 +118,7 @@ export class SaveManager { return result; } + // TODO Refactor remove editor. public async loadScene(scene_media_token: string) { showEditorLoader(); @@ -150,6 +145,10 @@ export class SaveManager { // this calls the signal function to propagate the data to the UI restoreSceneGenerationMetadata(scene_json); // these propogate the values into the editor + if (scene_json.globalIpAdapterImageMediaToken) { + // this should be populated right after + this.editor.generation_options.globalIpAdapterImageMediaToken; + } if (scene_json.positivePrompt) { this.editor.positive_prompt = scene_json.positivePrompt; } @@ -170,6 +169,7 @@ export class SaveManager { this.editor.cam_obj = this.editor.activeScene.get_object_by_name( this.editor.camera_name, ); + this.editor.cam_obj?.layers.set(1); this.editor.cam_obj?.children.forEach((child) => { child.layers.set(1); @@ -189,6 +189,7 @@ export class SaveManager { this.editor.timeline.checkEditorCanPlay(); hideEditorLoader(); + // TODO figure out if this is a bug. this.editor.timeline.scrub({ data: { currentTime: 0 } }); } } diff --git a/app/pages/PageEnigma/Editor/timeline.ts b/app/pages/PageEnigma/Editor/timeline.ts index 2e9a09a603..9ed4f587e8 100644 --- a/app/pages/PageEnigma/Editor/timeline.ts +++ b/app/pages/PageEnigma/Editor/timeline.ts @@ -99,7 +99,6 @@ export class TimeLine { ); this.current_time = 0; - this.camera_name = camera_name; } @@ -198,6 +197,7 @@ export class TimeLine { break; case toEngineActions.GENERATE_VIDEO: { const options = data.data; // super overloaded talk to the devs about this. TODO... refactor + // pass this in ... rather than doing it implicitly ... this.editorEngine.generation_options = options as GenerationOptions; this.editorEngine.generateVideo(); break; diff --git a/app/pages/PageEnigma/Editor/video_generation.ts b/app/pages/PageEnigma/Editor/video_generation.ts index ce9d0f4541..73c112a0c3 100644 --- a/app/pages/PageEnigma/Editor/video_generation.ts +++ b/app/pages/PageEnigma/Editor/video_generation.ts @@ -12,14 +12,18 @@ import * as THREE from "three"; import { getSceneSignals } from "~/signals"; import { v4 as uuidv4 } from "uuid"; import { SceneGenereationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata"; +import { MediaUploadApi } from "~/Classes/ApiManager"; +import { adapterImage } from "../signals"; // TODO THIS CLASS MAKES NO SENSE Refactor so we generate all the frames first. then pass it through this pipeline as a data structure process it. through this class. export class VideoGeneration { editor: Editor; + mediaUploadAPI: MediaUploadApi; constructor(editor: Editor) { this.editor = editor; + this.mediaUploadAPI = new MediaUploadApi(); } async generateFrame() { @@ -258,17 +262,40 @@ export class VideoGeneration { const style_name = this.editor.art_style.toString(); const media_token = this.editor.current_scene_media_token || undefined; + // convert the ip adapter image and upload as a media token + const image_uuid = uuidv4(); + let ipa_image_token = undefined; + if (this.editor.globalIpAdapterImage != undefined) { + const response = await this.mediaUploadAPI.UploadImage({ + fileName: `${image_uuid}.ipa`, + blob: this.editor.globalIpAdapterImage, + uuid: image_uuid, + }); + if (response.success) { + if (response.data) { + ipa_image_token = response.data; + } + } + } + + let globalIPAMediaToken = ""; + + if (ipa_image_token) { + globalIPAMediaToken = ipa_image_token; + } + + // TODO Remove so many of these around wtf. SceneGenereationMetaData should only be one place const metaData: SceneGenereationMetaData = { artisticStyle: this.editor.art_style, positivePrompt: this.editor.positive_prompt, negativePrompt: this.editor.negative_prompt, cameraAspectRatio: this.editor.render_camera_aspect_ratio, - adapterImageToken: "", upscale: this.editor.generation_options.upscale, faceDetail: this.editor.generation_options.faceDetail, styleStrength: this.editor.generation_options.styleStrength, lipSync: this.editor.generation_options.lipSync, cinematic: this.editor.generation_options.cinematic, + globalIPAMediaToken: globalIPAMediaToken, }; // This is to save the snapshot of the scene for remixing... @@ -335,10 +362,13 @@ export class VideoGeneration { this.editor.generation_options.styleStrength, this.editor.generation_options.lipSync, this.editor.generation_options.cinematic, + globalIPAMediaToken, ) .catch((error) => { // TODO handle stylize error. console.log(error); + this.editor.updateLoad(100, "Failed To Render"); + this.editor.endLoading(); }); // Not sure if this is needed will double check TODO: MC //this.editor.generating_preview = false; @@ -362,6 +392,7 @@ export class VideoGeneration { canvas: this.editor.canvasRenderCamReference || undefined, preserveDrawingBuffer: true, }); + this.editor.activeScene.renderMode(false); this.editor.switchEdit(); diff --git a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx index 971d3a840e..4ff2ea3c70 100644 --- a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx +++ b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx @@ -26,7 +26,7 @@ import { LoadUserScenes } from "./LoadUserScenes"; import { NewSceneFromTemplate } from "./NewSceneFromTemplate"; import { getCurrentLocationWithoutParams } from "~/utilities"; -import { SceneGenereationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata"; +import { SceneGenereationMetaData as SceneGenerationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata"; import { cameraAspectRatio, cinematic, @@ -70,22 +70,26 @@ export const ControlsTopButtons = () => { editorEngine?.newScene(sceneTitleInput); }; - const getSceneGenereationMetaData = - useCallback((): SceneGenereationMetaData => { - // when this is called, editor engine is guarunteed by it's caller + const getSceneGenereationMetaData = useCallback((): + | SceneGenerationMetaData + | undefined => { + // when this is called, editor engine is guarunteed by it's caller + if (editorEngine) { return { - positivePrompt: editorEngine!.positive_prompt, - negativePrompt: editorEngine!.negative_prompt, - artisticStyle: getArtStyle(editorEngine!.art_style.toString()), + positivePrompt: editorEngine.positive_prompt, + negativePrompt: editorEngine.negative_prompt, + artisticStyle: getArtStyle(editorEngine.art_style.toString()), cameraAspectRatio: cameraAspectRatio.value, - adapterImageToken: "", + globalIPAMediaToken: + editorEngine.generation_options.globalIpAdapterImageMediaToken, upscale: upscale.value, faceDetail: faceDetail.value, styleStrength: styleStrength.value, lipSync: lipSync.value, cinematic: cinematic.value, }; - }, [editorEngine]); + } + }, [editorEngine]); const handleButtonSave = async () => { if (!editorEngine) { @@ -173,9 +177,9 @@ export const ControlsTopButtons = () => { } }); - const handleShowOutliner = () => { - outlinerIsShowing.value = !outlinerIsShowing.value; - }; + const handleShowOutliner = () => { + outlinerIsShowing.value = !outlinerIsShowing.value; + }; return (
@@ -309,7 +313,6 @@ export const ControlsTopButtons = () => { ]} /> - + { useSignals(); + const editorEngine = useContext(EngineContext); const [imageSrc, setImageSrc] = useState(null); const [crop, setCrop] = useState(); const [completedCrop, setCompletedCrop] = useState(null); @@ -30,10 +38,21 @@ export const IPAdapter: React.FC = () => { const imgRef = useRef(null); const FILE_TYPES = ["JPG", "PNG", "JPEG"]; + useEffect(() => { + if (editorEngine) { + // load the image token if availible + // Bombay load the image from the image token here. + editorEngine.generation_options.globalIpAdapterImageMediaToken; + } + }, []); + const onFileChange = (file: File) => { const reader = new FileReader(); reader.onloadend = () => { setImageSrc(reader.result as string); + if (editorEngine) { + editorEngine.globalIpAdapterImage = file; + } setIsDialogOpen(true); }; reader.readAsDataURL(file); diff --git a/app/pages/PageEnigma/models/generationOptions.ts b/app/pages/PageEnigma/models/generationOptions.ts index 1314497ae0..c77c734029 100644 --- a/app/pages/PageEnigma/models/generationOptions.ts +++ b/app/pages/PageEnigma/models/generationOptions.ts @@ -4,6 +4,7 @@ export interface GenerationOptions { styleStrength: number; lipSync: boolean; cinematic: boolean; + globalIpAdapterImageMediaToken: string; // base 64 image } export class GenerationOptions { @@ -12,18 +13,20 @@ export class GenerationOptions { public styleStrength: number; public lipSync: boolean; public cinematic: boolean; - + public globalIpAdapterImageMediaToken: string; constructor( upscale: boolean, faceDetail: boolean, styleStrength: number, lipSync: boolean, cinematic: boolean, + globalIpAdapterImageMediaToken: string, ) { this.upscale = upscale; this.faceDetail = faceDetail; this.styleStrength = styleStrength; this.lipSync = lipSync; this.cinematic = cinematic; + this.globalIpAdapterImageMediaToken = globalIpAdapterImageMediaToken; } } diff --git a/app/pages/PageEnigma/models/sceneGenerationMetadata.ts b/app/pages/PageEnigma/models/sceneGenerationMetadata.ts index ee533a406b..d394cae23a 100644 --- a/app/pages/PageEnigma/models/sceneGenerationMetadata.ts +++ b/app/pages/PageEnigma/models/sceneGenerationMetadata.ts @@ -7,7 +7,7 @@ export interface SceneGenereationMetaData { negativePrompt: string; cameraAspectRatio: CameraAspectRatio; - adapterImageToken: string; + globalIPAMediaToken: string; upscale: boolean; faceDetail: boolean;