From 8d4edcfafd670d88a3b497ab5dcc307c5abe6af2 Mon Sep 17 00:00:00 2001 From: BFlat Date: Mon, 28 Jul 2025 13:28:17 +0700 Subject: [PATCH] lightbox context images, generation details, canvas snapshot bug fix (#746) - canvas snapshot aspect ratio is fixed (it was snapshotting only as squared images) - added reference/context images in the lightbox under prompt, hovering shows a larger preview - added generation details: model used with logo and provider --- .../resources/images/services/recraft.svg | 1 + .../resources/images/services/tencent.svg | 1 + .../app/src/pages/PageDraw/PageDraw.tsx | 9 +- .../pages/PageDraw/hooks/useUpdateSnapshot.ts | 211 ++++++++-------- frontend/libs/api/src/lib/models/Prompts.ts | 18 ++ .../lightbox-modal/src/lib/lightbox-modal.tsx | 227 ++++++++++++++---- frontend/libs/model-list/src/index.ts | 1 + .../libs/model-list/src/lib/ModelCreator.ts | 3 +- .../model-list/src/lib/ModelCreatorIcons.ts | 2 + .../model-list/src/lib/ModelTypeMapping.ts | 94 ++++++++ 10 files changed, 426 insertions(+), 141 deletions(-) create mode 100644 frontend/apps/editor3d/app/public/resources/images/services/recraft.svg create mode 100644 frontend/apps/editor3d/app/public/resources/images/services/tencent.svg create mode 100644 frontend/libs/model-list/src/lib/ModelTypeMapping.ts diff --git a/frontend/apps/editor3d/app/public/resources/images/services/recraft.svg b/frontend/apps/editor3d/app/public/resources/images/services/recraft.svg new file mode 100644 index 0000000000..b4ea390a72 --- /dev/null +++ b/frontend/apps/editor3d/app/public/resources/images/services/recraft.svg @@ -0,0 +1 @@ +Recraft \ No newline at end of file diff --git a/frontend/apps/editor3d/app/public/resources/images/services/tencent.svg b/frontend/apps/editor3d/app/public/resources/images/services/tencent.svg new file mode 100644 index 0000000000..135a0c0f9d --- /dev/null +++ b/frontend/apps/editor3d/app/public/resources/images/services/tencent.svg @@ -0,0 +1 @@ +Tencent \ No newline at end of file diff --git a/frontend/apps/editor3d/app/src/pages/PageDraw/PageDraw.tsx b/frontend/apps/editor3d/app/src/pages/PageDraw/PageDraw.tsx index 40414350a9..79c6371d5b 100644 --- a/frontend/apps/editor3d/app/src/pages/PageDraw/PageDraw.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageDraw/PageDraw.tsx @@ -170,8 +170,15 @@ const PageDraw = () => { }; const onEnqueuedPressed = async () => { + const { width, height } = store.getAspectRatioDimensions(); + // takes snap shot and then a global variable in the engine will invoke the inference. - const image = await captureStageImageBitmap(stageRef, transformerRefs); + const image = await captureStageImageBitmap( + stageRef, + transformerRefs, + width, + height, + ); if (!image) { console.error("Failed to capture stage image"); return; diff --git a/frontend/apps/editor3d/app/src/pages/PageDraw/hooks/useUpdateSnapshot.ts b/frontend/apps/editor3d/app/src/pages/PageDraw/hooks/useUpdateSnapshot.ts index 8aa7f86d18..4bf2a868e8 100644 --- a/frontend/apps/editor3d/app/src/pages/PageDraw/hooks/useUpdateSnapshot.ts +++ b/frontend/apps/editor3d/app/src/pages/PageDraw/hooks/useUpdateSnapshot.ts @@ -1,114 +1,129 @@ -import { useState, useEffect } from 'react'; +import { useState, useEffect } from "react"; +import Konva from "konva"; -const debugOutput = (uri:string, name:string)=> { - var link = document.createElement('a'); - link.download = name; - link.href = uri; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); +export const captureStageImageBitmap = async ( + stageRef: React.RefObject, + transformerRefs: React.RefObject<{ [key: string]: Konva.Transformer }>, + width: number = 1024, + height: number = 1024, +): Promise => { + if (!stageRef.current) { + return undefined; } + const oldPos = { ...stageRef.current.position() }; + const oldScale = { + x: stageRef.current.scaleX(), + y: stageRef.current.scaleY(), + }; - export const captureStageImageBitmap = async (stageRef: any,transformerRefs: any): Promise => { - if (!stageRef.current) { - return undefined; - } - - const oldPos = { ...stageRef.current.position() }; - const oldScale = { x: stageRef.current.scaleX(), y: stageRef.current.scaleY() }; - - // Hide all transformers - Object.values(transformerRefs.current).forEach(transformer => { + // Hide all transformers + Object.values(transformerRefs.current || {}).forEach( + (transformer: Konva.Transformer) => { transformer.visible(false); + }, + ); + + stageRef.current.position({ x: 0, y: 0 }); + stageRef.current.scale({ x: 1, y: 1 }); + stageRef.current.draw(); + + try { + const canvas = stageRef.current.toCanvas({ + width, + height, + pixelRatio: 1, }); - + + return await createImageBitmap(canvas); + } catch (error) { + console.error("Failed to create ImageBitmap:", error); + return undefined; + } finally { + // Restore transformers + Object.values(transformerRefs.current || {}).forEach( + (transformer: Konva.Transformer) => { + transformer.visible(true); + }, + ); + + stageRef.current.position(oldPos); + stageRef.current.scale(oldScale); + stageRef.current.draw(); + } +}; + +export const useStageSnapshot = ( + stageRef: React.RefObject, + imageRef: React.RefObject, + isSelectingRef: React.RefObject, + transformerRefs: React.RefObject<{ [key: string]: Konva.Transformer }>, +) => { + const [animationFrameId, setAnimationFrameId] = useState(null); + + const captureStageImage = ( + stageRef: React.RefObject, + width: number = 1024, + height: number = 1024, + ): string => { + if (!stageRef.current) return ""; + + const oldPos = { ...stageRef.current.position() }; + const oldScale = { + x: stageRef.current.scaleX(), + y: stageRef.current.scaleY(), + }; + + // Hide all transformers + Object.values(transformerRefs.current || {}).forEach( + (transformer: Konva.Transformer) => { + transformer.visible(false); + }, + ); + stageRef.current.position({ x: 0, y: 0 }); stageRef.current.scale({ x: 1, y: 1 }); stageRef.current.draw(); - - try { - const canvas = stageRef.current.toCanvas({ - width: 1024, - height: 1024, - pixelRatio: 1 - }); - - return await createImageBitmap(canvas); - } catch (error) { - console.error('Failed to create ImageBitmap:', error); - return undefined; - } finally { - // Restore transformers - Object.values(transformerRefs.current).forEach(transformer => { + const uri = stageRef.current.toDataURL({ width, height, pixelRatio: 1 }); + + // Restore transformers + Object.values(transformerRefs.current || {}).forEach( + (transformer: Konva.Transformer) => { transformer.visible(true); - }); - - stageRef.current.position(oldPos); - stageRef.current.scale(oldScale); - stageRef.current.draw(); - } + }, + ); + + stageRef.current.position(oldPos); + stageRef.current.scale(oldScale); + stageRef.current.draw(); + return uri; }; - export const useStageSnapshot = ( - stageRef: React.RefObject, - imageRef: React.RefObject, - isSelectingRef: React.RefObject, - transformerRefs: React.RefObject<{ [key: string]: Konva.Transformer }> - ) => { - const [animationFrameId, setAnimationFrameId] = useState(null); - - const captureStageImage = (stageRef: any): string => { - const oldPos = { ...stageRef.current.position() }; - const oldScale = { x: stageRef.current.scaleX(), y: stageRef.current.scaleY() }; - - // Hide all transformers - Object.values(transformerRefs.current).forEach(transformer => { - transformer.visible(false); - }); - - stageRef.current.position({ x: 0, y: 0 }); - stageRef.current.scale({ x: 1, y: 1 }); - stageRef.current.draw(); - const uri = stageRef.current.toDataURL({ width: 1024, height: 1024, pixelRatio: 1 }); - - // Restore transformers - Object.values(transformerRefs.current).forEach(transformer => { - transformer.visible(true); - }); - - stageRef.current.position(oldPos); - stageRef.current.scale(oldScale); - stageRef.current.draw(); - return uri; - }; - - useEffect(() => { - const updateSnapshot = () => { - if (!isSelectingRef.current && stageRef.current && imageRef.current) { - const uri = captureStageImage(stageRef); - if (uri) { - const img = new window.Image(); - img.onload = () => { - imageRef.current?.setAttrs({ - image: img - }); - }; - img.src = uri; - } + useEffect(() => { + const updateSnapshot = () => { + if (!isSelectingRef.current && stageRef.current && imageRef.current) { + const uri = captureStageImage(stageRef); + if (uri) { + const img = new window.Image(); + img.onload = () => { + imageRef.current?.setAttrs({ + image: img, + }); + }; + img.src = uri; } - const frameId = requestAnimationFrame(updateSnapshot); - setAnimationFrameId(frameId); - }; - + } const frameId = requestAnimationFrame(updateSnapshot); setAnimationFrameId(frameId); - - return () => { - if (animationFrameId) { - cancelAnimationFrame(animationFrameId); - } + }; - }; - }, []); - }; \ No newline at end of file + const frameId = requestAnimationFrame(updateSnapshot); + setAnimationFrameId(frameId); + + return () => { + if (animationFrameId) { + cancelAnimationFrame(animationFrameId); + } + }; + }, []); +}; diff --git a/frontend/libs/api/src/lib/models/Prompts.ts b/frontend/libs/api/src/lib/models/Prompts.ts index fafe2fd394..bbf22905c1 100644 --- a/frontend/libs/api/src/lib/models/Prompts.ts +++ b/frontend/libs/api/src/lib/models/Prompts.ts @@ -13,4 +13,22 @@ export interface Prompts { use_cinematic: boolean; used_face_detailer: boolean; used_upscaler: boolean; + maybe_model_type: string; + maybe_generation_provider: string; + maybe_context_images: [ + { + media_links: { + cdn_url: string; + maybe_thumbnail_template: string; + maybe_video_previews: { + animated: string; + animated_thumbnail_template: string; + still: string; + still_thumbnail_template: string; + }; + }; + media_token: string; + semantic: string; + } + ]; } diff --git a/frontend/libs/components/lightbox-modal/src/lib/lightbox-modal.tsx b/frontend/libs/components/lightbox-modal/src/lib/lightbox-modal.tsx index 7067efa90e..074c080970 100644 --- a/frontend/libs/components/lightbox-modal/src/lib/lightbox-modal.tsx +++ b/frontend/libs/components/lightbox-modal/src/lib/lightbox-modal.tsx @@ -14,6 +14,12 @@ import { toast } from "@storyteller/ui-toaster"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faCopy } from "@fortawesome/pro-solid-svg-icons"; import { twMerge } from "tailwind-merge"; +import { + getModelCreatorIcon, + getModelDisplayName, + getProviderDisplayName, +} from "@storyteller/model-list"; +import { Tooltip } from "@storyteller/ui-tooltip"; interface LightboxModalProps { isOpen: boolean; @@ -63,6 +69,18 @@ export function LightboxModal({ const [promptLoading, setPromptLoading] = React.useState(false); const [hasPromptToken, setHasPromptToken] = React.useState(false); const [isPromptHovered, setIsPromptHovered] = React.useState(false); + const [generationProvider, setGenerationProvider] = React.useState< + string | null + >(null); + const [modelType, setModelType] = React.useState(null); + const [contextImages, setContextImages] = React.useState | null>(null); // Reset when imageUrl changes React.useEffect(() => { @@ -75,6 +93,9 @@ export function LightboxModal({ if (!mediaId) { setPrompt(null); setHasPromptToken(false); + setGenerationProvider(null); + setModelType(null); + setContextImages(null); return; } @@ -92,22 +113,32 @@ export function LightboxModal({ token: mediaResponse.data.maybe_prompt_token, }); - if ( - promptResponse.success && - promptResponse.data?.maybe_positive_prompt - ) { - setPrompt(promptResponse.data.maybe_positive_prompt); + if (promptResponse.success && promptResponse.data) { + const promptData = promptResponse.data; + setPrompt(promptData.maybe_positive_prompt || null); + setGenerationProvider(promptData.maybe_generation_provider || null); + setModelType(promptData.maybe_model_type || null); + setContextImages(promptData.maybe_context_images || null); } else { setPrompt(null); + setGenerationProvider(null); + setModelType(null); + setContextImages(null); } } else { setHasPromptToken(false); setPrompt(null); + setGenerationProvider(null); + setModelType(null); + setContextImages(null); } } catch (error) { console.error("Error fetching prompt:", error); setHasPromptToken(false); setPrompt(null); + setGenerationProvider(null); + setModelType(null); + setContextImages(null); } finally { setPromptLoading(false); } @@ -185,51 +216,165 @@ export function LightboxModal({ )} + {hasPromptToken && ( -
-
Prompt
-
setIsPromptHovered(true)} - onMouseLeave={() => setIsPromptHovered(false)} - onClick={() => { - if (!prompt) return; - navigator.clipboard.writeText(prompt).catch(() => {}); - toast.success("Prompt copied"); - }} - > - {promptLoading ? ( -
- - - Loading prompt... - + <> + {/* Prompt */} +
+
+ Prompt +
+
setIsPromptHovered(true)} + onMouseLeave={() => setIsPromptHovered(false)} + onClick={() => { + if (!prompt) return; + navigator.clipboard.writeText(prompt).catch(() => {}); + toast.success("Prompt copied"); + }} + > + {promptLoading ? ( +
+ + + Loading prompt... + +
+ ) : ( + prompt || ( + No prompt + ) + )} +
+ + {!promptLoading && ( +
+
+ + Copy prompt +
- ) : ( - prompt || ( - No prompt - ) )}
- {!promptLoading && ( -
-
- - Copy prompt + {/* Context Images */} + {contextImages && contextImages.length > 0 && ( +
+
+ Reference Images +
+
+ {contextImages.map((contextImage, index) => { + const thumbnailUrl = contextImage.media_links + .maybe_thumbnail_template + ? contextImage.media_links.maybe_thumbnail_template.replace( + "{WIDTH}", + "128" + ) + : contextImage.media_links.cdn_url; + + const fullSizeUrl = contextImage.media_links + .maybe_thumbnail_template + ? contextImage.media_links.maybe_thumbnail_template.replace( + "{WIDTH}", + "512" + ) + : contextImage.media_links.cdn_url; + + return ( + // Hover preview + +
+ {`Reference +
+ {contextImage.semantic && ( +
+ {contextImage.semantic} +
+ )} +
+ } + position="top" + delay={300} + closeOnClick={true} + > +
{ + window.open( + contextImage.media_links.cdn_url, + "_blank" + ); + }} + > +
+ {`Reference +
+
+ + ); + })}
)} -
+ + {/* Generation Details */} + {(generationProvider || modelType) && ( +
+
+ Generation Details +
+
+ {modelType && ( +
+ + Model + +
+ {getModelCreatorIcon(modelType)} + + {getModelDisplayName(modelType)} + +
+
+ )} + {generationProvider && ( +
+ + Provider + + + {getProviderDisplayName(generationProvider)} + +
+ )} +
+
+ )} + )} + {additionalInfo}
diff --git a/frontend/libs/model-list/src/index.ts b/frontend/libs/model-list/src/index.ts index 0214fc4184..94eea6b676 100644 --- a/frontend/libs/model-list/src/index.ts +++ b/frontend/libs/model-list/src/index.ts @@ -4,3 +4,4 @@ export * from "./lib/ModelCreator.js"; export * from "./lib/ImageModels.js"; export * from "./lib/VideoModels.js"; export * from "./lib/ModelCreatorIcons.js"; +export * from "./lib/ModelTypeMapping.js"; diff --git a/frontend/libs/model-list/src/lib/ModelCreator.ts b/frontend/libs/model-list/src/lib/ModelCreator.ts index 4e4bfd4f22..d7139e9fd9 100644 --- a/frontend/libs/model-list/src/lib/ModelCreator.ts +++ b/frontend/libs/model-list/src/lib/ModelCreator.ts @@ -1,4 +1,3 @@ - export enum ModelCreator { BlackForestLabs = "BlackForestLabs", Bytedance = "Bytedance", @@ -9,4 +8,6 @@ export enum ModelCreator { OpenAi = "OpenAi", Runway = "Runway", Stability = "Stability", + Tencent = "Tencent", + Recraft = "Recraft", } diff --git a/frontend/libs/model-list/src/lib/ModelCreatorIcons.ts b/frontend/libs/model-list/src/lib/ModelCreatorIcons.ts index 7982a43817..53738b40c2 100644 --- a/frontend/libs/model-list/src/lib/ModelCreatorIcons.ts +++ b/frontend/libs/model-list/src/lib/ModelCreatorIcons.ts @@ -9,6 +9,8 @@ export const CREATOR_ICON_PATHS: Partial> = { [ModelCreator.OpenAi]: "/resources/images/services/openai.svg", [ModelCreator.Bytedance]: "/resources/images/services/bytedance.svg", [ModelCreator.Google]: "/resources/images/services/google.svg", + [ModelCreator.Recraft]: "/resources/images/services/recraft.svg", + [ModelCreator.Tencent]: "/resources/images/services/tencent.svg", }; export const getCreatorIconPath = (creator: ModelCreator): string | undefined => diff --git a/frontend/libs/model-list/src/lib/ModelTypeMapping.ts b/frontend/libs/model-list/src/lib/ModelTypeMapping.ts new file mode 100644 index 0000000000..a100bfcedc --- /dev/null +++ b/frontend/libs/model-list/src/lib/ModelTypeMapping.ts @@ -0,0 +1,94 @@ +import React from "react"; +import { ModelCreator } from "./ModelCreator.js"; +import { getCreatorIcon } from "./ModelCreatorIcons.js"; + +// Map from model type strings to ModelCreator enum values +export const MODEL_TYPE_TO_CREATOR: Record = { + flux_1_dev: ModelCreator.BlackForestLabs, + flux_1_schnell: ModelCreator.BlackForestLabs, + flux_pro_1p1: ModelCreator.BlackForestLabs, + flux_pro_1p1_ultra: ModelCreator.BlackForestLabs, + gpt_image_1: ModelCreator.OpenAi, + kling_1p6_pro: ModelCreator.Kling, + kling_2p1_pro: ModelCreator.Kling, + kling_2p1_master: ModelCreator.Kling, + seedance_1p0_lite: ModelCreator.Bytedance, + veo_2: ModelCreator.Google, + recraft_3: ModelCreator.Recraft, + hunyuan_3d: ModelCreator.Tencent, +}; + +// Get creator icon for a model type +export const getModelCreatorIcon = ( + modelType: string +): React.ReactNode | null => { + const creator = MODEL_TYPE_TO_CREATOR[modelType]; + if (!creator) return null; + return getCreatorIcon(creator, "h-4 w-4 invert"); +}; + +// Get creator name for display +export const getModelCreatorName = (modelType: string): string | null => { + const creator = MODEL_TYPE_TO_CREATOR[modelType]; + + // Convert enum value to display name + switch (creator) { + case ModelCreator.BlackForestLabs: + return "Black Forest Labs"; + case ModelCreator.OpenAi: + return "OpenAI"; + case ModelCreator.Kling: + return "Kling AI"; + case ModelCreator.Bytedance: + return "ByteDance"; + case ModelCreator.Google: + return "Google"; + case ModelCreator.Midjourney: + return "Midjourney"; + case ModelCreator.Stability: + return "Stability AI"; + case ModelCreator.Runway: + return "Runway"; + case ModelCreator.Hailuo: + return "Hailuo"; + case ModelCreator.Recraft: + return "Recraft"; + case ModelCreator.Tencent: + return "Tencent"; + default: + return creator; + } +}; + +// Convert model type string to human-readable display name +export const getModelDisplayName = (modelType: string): string => { + const displayNames: Record = { + flux_1_dev: "Flux 1 Dev", + flux_1_schnell: "Flux 1 Schnell", + flux_pro_1p1: "Flux Pro 1.1", + flux_pro_1p1_ultra: "Flux Pro 1.1 Ultra", + gpt_image_1: "GPT Image 1", + kling_1p6_pro: "Kling 1.6 Pro", + kling_2p1_pro: "Kling 2.1 Pro", + kling_2p1_master: "Kling 2.1 Master", + seedance_1p0_lite: "Seedance 1.0 Lite", + veo_2: "Veo 2", + recraft_3: "Recraft 3", + hunyuan_3d_2p0: "Hunyuan 3D 2.0", + hunyuan_3d_2p1: "Hunyuan 3D 2.1", + hunyuan_3d: "Hunyuan 3D", + }; + + return displayNames[modelType] || modelType; +}; + +// Convert provider string to human-readable display name +export const getProviderDisplayName = (provider: string): string => { + const displayNames: Record = { + artcraft: "ArtCraft", + fal: "FAL", + sora: "Sora", + }; + + return displayNames[provider] || provider; +};