mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-10 12:01:36 +00:00
398 lines
13 KiB
TypeScript
398 lines
13 KiB
TypeScript
import { useState, useRef, useEffect } from "react";
|
|
import { useSignals } from "@preact/signals-react/runtime";
|
|
import { JobContextType } from "@storyteller/common";
|
|
import { downloadFileFromUrl } from "@storyteller/api";
|
|
import { PopoverMenu, PopoverItem } from "@storyteller/ui-popover";
|
|
import { Tooltip } from "@storyteller/ui-tooltip";
|
|
import { Button, ToggleButton } from "@storyteller/ui-button";
|
|
import { Modal } from "@storyteller/ui-modal";
|
|
import { EnqueueImageToVideo } from "@storyteller/tauri-api";
|
|
import {
|
|
faMessageXmark,
|
|
faMessageCheck,
|
|
faSparkles,
|
|
faSpinnerThird,
|
|
faTimes,
|
|
faPlus,
|
|
faImages,
|
|
} from "@fortawesome/pro-solid-svg-icons";
|
|
import { faRectangle } from "@fortawesome/pro-regular-svg-icons";
|
|
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
|
import { IconProp } from "@fortawesome/fontawesome-svg-core";
|
|
import { IsDesktopApp } from "@storyteller/tauri-utils";
|
|
import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal";
|
|
import { Model } from "@storyteller/model-list";
|
|
import { usePromptVideoStore, RefImage } from "./promptStore";
|
|
import { gtagEvent } from "@storyteller/google-analytics";
|
|
|
|
interface PromptBoxVideoProps {
|
|
useJobContext: () => JobContextType;
|
|
onEnqueuePressed?: () => void | Promise<void>;
|
|
selectedModel?: Model;
|
|
imageMediaId?: string;
|
|
url?: string;
|
|
}
|
|
|
|
export const PromptBoxVideo = ({
|
|
useJobContext,
|
|
onEnqueuePressed,
|
|
selectedModel,
|
|
imageMediaId,
|
|
url,
|
|
}: PromptBoxVideoProps) => {
|
|
useSignals();
|
|
|
|
// for the image media id and url, we need to set the reference image gallery panel.
|
|
useEffect(() => {
|
|
if (imageMediaId && url) {
|
|
const referenceImage: RefImage = {
|
|
id: Math.random().toString(36).substring(7),
|
|
url: url,
|
|
file: new File([], "library-image"),
|
|
mediaToken: imageMediaId,
|
|
};
|
|
setReferenceImages([referenceImage]);
|
|
}
|
|
}, [imageMediaId, url]);
|
|
|
|
console.log("Is this a desktop app?", IsDesktopApp());
|
|
const [isModalOpen, setIsModalOpen] = useState(false);
|
|
const [content, setContent] = useState("");
|
|
const prompt = usePromptVideoStore((s) => s.prompt);
|
|
const setPrompt = usePromptVideoStore((s) => s.setPrompt);
|
|
const useSystemPrompt = usePromptVideoStore((s) => s.useSystemPrompt);
|
|
const setUseSystemPrompt = usePromptVideoStore((s) => s.setUseSystemPrompt);
|
|
const resolution = usePromptVideoStore((s) => s.resolution);
|
|
const setResolution = usePromptVideoStore((s) => s.setResolution);
|
|
const [isEnqueueing, setIsEnqueueing] = useState(false);
|
|
const [selectedGalleryImages, setSelectedGalleryImages] = useState<string[]>(
|
|
[]
|
|
);
|
|
const referenceImages = usePromptVideoStore((s) => s.referenceImages);
|
|
const setReferenceImages = usePromptVideoStore((s) => s.setReferenceImages);
|
|
const [uploadingImages, _setUploadingImages] = useState<
|
|
{ id: string; file: File }[]
|
|
>([]);
|
|
const [resolutionList, setResolutionList] = useState<PopoverItem[]>([
|
|
{
|
|
label: "720p",
|
|
selected: resolution === "720p",
|
|
icon: <FontAwesomeIcon icon={faRectangle} className="h-4 w-4" />,
|
|
},
|
|
{
|
|
label: "480p",
|
|
selected: resolution === "480p",
|
|
icon: <FontAwesomeIcon icon={faRectangle} className="h-4 w-4" />,
|
|
},
|
|
]);
|
|
|
|
const fileInputRef = useRef<HTMLInputElement>(null);
|
|
const textareaRef = useRef<HTMLTextAreaElement>(null);
|
|
|
|
useEffect(() => {
|
|
if (textareaRef.current) {
|
|
textareaRef.current.style.height = "auto";
|
|
textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`;
|
|
}
|
|
});
|
|
|
|
useEffect(() => {
|
|
if (imageMediaId && url) {
|
|
const referenceImage: RefImage = {
|
|
id: Math.random().toString(36).substring(7),
|
|
url: url,
|
|
file: new File([], "library-image"),
|
|
mediaToken: imageMediaId,
|
|
};
|
|
setReferenceImages([referenceImage]);
|
|
}
|
|
}, [imageMediaId, url]);
|
|
|
|
const handleResolutionSelect = (selectedItem: PopoverItem) => {
|
|
setResolution(selectedItem.label as any);
|
|
setResolutionList((prev) =>
|
|
prev.map((item) => ({
|
|
...item,
|
|
selected: item.label === selectedItem.label,
|
|
}))
|
|
);
|
|
};
|
|
|
|
const handleRemoveReference = (id: string) => {
|
|
setReferenceImages(referenceImages.filter((img) => img.id !== id));
|
|
if (fileInputRef.current) {
|
|
fileInputRef.current.value = "";
|
|
}
|
|
};
|
|
|
|
const [isGalleryModalOpen, setIsGalleryModalOpen] = useState(false);
|
|
|
|
const handleGallerySelect = () => setIsGalleryModalOpen(true);
|
|
const handleGalleryClose = () => {
|
|
setIsGalleryModalOpen(false);
|
|
setSelectedGalleryImages([]);
|
|
};
|
|
|
|
const handleImageSelect = (id: string) => {
|
|
setSelectedGalleryImages((prev) => {
|
|
if (prev.includes(id)) {
|
|
return [];
|
|
}
|
|
return [id];
|
|
});
|
|
};
|
|
|
|
const handleGalleryImages = (selectedItems: GalleryItem[]) => {
|
|
// Clear existing reference images first
|
|
setReferenceImages([]);
|
|
|
|
// Only take the first selected item
|
|
const item = selectedItems[0];
|
|
if (!item || !item.fullImage) return;
|
|
|
|
const referenceImage: RefImage = {
|
|
id: Math.random().toString(36).substring(7),
|
|
url: item.fullImage,
|
|
file: new File([], "library-image"),
|
|
mediaToken: item.id,
|
|
};
|
|
setReferenceImages([referenceImage]);
|
|
setIsGalleryModalOpen(false);
|
|
setSelectedGalleryImages([]);
|
|
};
|
|
|
|
const handleAction = (action: string) => {
|
|
switch (action) {
|
|
case "gallery":
|
|
handleGallerySelect();
|
|
break;
|
|
default:
|
|
console.log("Unknown action:", action);
|
|
}
|
|
};
|
|
|
|
const handlePaste = (e: React.ClipboardEvent<HTMLTextAreaElement>) => {
|
|
e.preventDefault();
|
|
const pastedText = e.clipboardData.getData("text");
|
|
const target = e.currentTarget;
|
|
const { selectionStart, selectionEnd, value } = target;
|
|
const next =
|
|
value.slice(0, selectionStart) + pastedText + value.slice(selectionEnd);
|
|
setPrompt(next);
|
|
requestAnimationFrame(() => {
|
|
const pos = selectionStart + pastedText.length;
|
|
textareaRef.current?.setSelectionRange(pos, pos);
|
|
});
|
|
};
|
|
|
|
const handleChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
|
|
setPrompt(e.target.value);
|
|
};
|
|
|
|
const handleEnqueue = async () => {
|
|
setIsEnqueueing(true);
|
|
|
|
console.log("PromptBoxVideo - Prompting with model", selectedModel);
|
|
|
|
gtagEvent("enqueue_video");
|
|
|
|
setTimeout(() => {
|
|
// TODO(bt,2025-05-08): This is a hack so we don't accidentally wind up with a permanently disabled prompt box if
|
|
// the backend hangs on a given request.
|
|
console.debug("Turn off blocking of prompt box...");
|
|
setIsEnqueueing(false);
|
|
}, 10000);
|
|
|
|
const generateResponse = await EnqueueImageToVideo({
|
|
model: selectedModel,
|
|
image_media_token: referenceImages[0].mediaToken,
|
|
prompt: prompt,
|
|
});
|
|
|
|
console.log("generateResponse", generateResponse);
|
|
|
|
onEnqueuePressed?.();
|
|
|
|
setIsEnqueueing(false);
|
|
};
|
|
|
|
const getCurrentResolutionIcon = () => {
|
|
const selected = resolutionList.find((item) => item.selected);
|
|
if (!selected || !selected.icon) return faRectangle;
|
|
const iconElement = selected.icon as React.ReactElement<{ icon: IconProp }>;
|
|
return iconElement.props.icon;
|
|
};
|
|
|
|
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
|
|
if (e.key === "Enter" && !e.shiftKey) {
|
|
e.preventDefault();
|
|
handleEnqueue();
|
|
}
|
|
};
|
|
|
|
return (
|
|
<>
|
|
<Modal
|
|
isOpen={isModalOpen}
|
|
onClose={() => {
|
|
setIsModalOpen(false);
|
|
setContent("");
|
|
}}
|
|
>
|
|
{content}
|
|
</Modal>
|
|
<div className="relative z-20 flex flex-col gap-3">
|
|
{(referenceImages.length > 0 || uploadingImages.length > 0) && (
|
|
<div className="flex w-full gap-2">
|
|
{referenceImages.map((image) => (
|
|
<div
|
|
key={image.id}
|
|
className="glass relative aspect-square w-20 rounded-lg"
|
|
>
|
|
<img
|
|
src={image.url}
|
|
alt="Reference"
|
|
className="h-full w-full rounded-lg object-cover"
|
|
/>
|
|
<button
|
|
onClick={() => handleRemoveReference(image.id)}
|
|
className="absolute right-[2px] top-[2px] flex h-5 w-5 items-center justify-center rounded-full bg-black/40 text-white backdrop-blur-md transition-colors hover:bg-black"
|
|
>
|
|
<FontAwesomeIcon icon={faTimes} className="h-2.5 w-2.5" />
|
|
</button>
|
|
</div>
|
|
))}
|
|
{uploadingImages.map(({ id, file }) => {
|
|
const previewUrl = URL.createObjectURL(file);
|
|
return (
|
|
<div
|
|
key={id}
|
|
className="glass relative aspect-square w-20 overflow-hidden rounded-lg"
|
|
>
|
|
<div className="absolute inset-0">
|
|
<img
|
|
src={previewUrl}
|
|
alt="Uploading preview"
|
|
className="h-full w-full object-cover blur-sm"
|
|
/>
|
|
</div>
|
|
<div className="absolute inset-0 flex items-center justify-center bg-black/20">
|
|
<FontAwesomeIcon
|
|
icon={faSpinnerThird}
|
|
className="h-6 w-6 animate-spin text-white"
|
|
/>
|
|
</div>
|
|
</div>
|
|
);
|
|
})}
|
|
</div>
|
|
)}
|
|
<div className="glass w-[730px] rounded-xl p-4">
|
|
<div className="flex justify-center gap-2">
|
|
<PopoverMenu
|
|
mode="button"
|
|
panelTitle="Add Image"
|
|
items={[
|
|
{
|
|
label: "Choose from library",
|
|
selected: false,
|
|
icon: <FontAwesomeIcon icon={faImages} className="h-4 w-4" />,
|
|
action: "gallery",
|
|
},
|
|
]}
|
|
onPanelAction={handleAction}
|
|
showIconsInList
|
|
buttonClassName="bg-transparent hover:bg-transparent py-1.5 px-0 pr-1 m-0 hover:opacity-50 transition-opacity duration-100 ring-0 border-none focus:ring-0 outline-none"
|
|
triggerIcon={
|
|
<FontAwesomeIcon icon={faPlus} className="text-xl" />
|
|
}
|
|
/>
|
|
|
|
<textarea
|
|
ref={textareaRef}
|
|
rows={1}
|
|
placeholder="Choose the image from library (+) and describe what you want to happen in the video..."
|
|
className="text-md mb-2 max-h-[5.5em] flex-1 resize-none overflow-y-auto rounded bg-transparent pb-2 pr-2 pt-1 text-white placeholder-white placeholder:text-white/60 focus:outline-none"
|
|
value={prompt}
|
|
onChange={handleChange}
|
|
onPaste={handlePaste}
|
|
onKeyDown={handleKeyDown}
|
|
onFocus={() => {}}
|
|
onBlur={() => {}}
|
|
/>
|
|
</div>
|
|
<div className="mt-2 flex items-center justify-between gap-2">
|
|
<div className="flex items-center gap-2">
|
|
<Tooltip
|
|
content="Resolution"
|
|
position="top"
|
|
className="z-50"
|
|
closeOnClick={true}
|
|
>
|
|
<PopoverMenu
|
|
items={resolutionList}
|
|
onSelect={handleResolutionSelect}
|
|
mode="toggle"
|
|
panelTitle="Resolution"
|
|
showIconsInList
|
|
triggerIcon={
|
|
<FontAwesomeIcon
|
|
icon={getCurrentResolutionIcon()}
|
|
className="h-4 w-4"
|
|
/>
|
|
}
|
|
/>
|
|
</Tooltip>
|
|
<Tooltip
|
|
content={
|
|
useSystemPrompt
|
|
? "Use system prompt: ON"
|
|
: "Use system prompt: OFF"
|
|
}
|
|
position="top"
|
|
className="z-50"
|
|
delay={200}
|
|
>
|
|
<ToggleButton
|
|
isActive={useSystemPrompt}
|
|
icon={faMessageXmark}
|
|
activeIcon={faMessageCheck}
|
|
onClick={() => setUseSystemPrompt(!useSystemPrompt)}
|
|
/>
|
|
</Tooltip>
|
|
</div>
|
|
<div className="flex items-center gap-2">
|
|
<Button
|
|
className="flex items-center border-none bg-primary px-3 text-sm text-white disabled:cursor-not-allowed disabled:opacity-50"
|
|
icon={!isEnqueueing ? faSparkles : undefined}
|
|
onClick={handleEnqueue}
|
|
disabled={isEnqueueing || !prompt.trim()}
|
|
>
|
|
{isEnqueueing ? (
|
|
<FontAwesomeIcon
|
|
icon={faSpinnerThird}
|
|
className="animate-spin text-lg"
|
|
/>
|
|
) : (
|
|
"Generate"
|
|
)}
|
|
</Button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<GalleryModal
|
|
isOpen={!!isGalleryModalOpen}
|
|
onClose={handleGalleryClose}
|
|
mode="select"
|
|
selectedItemIds={selectedGalleryImages}
|
|
onSelectItem={handleImageSelect}
|
|
maxSelections={1}
|
|
onUseSelected={handleGalleryImages}
|
|
onDownloadClicked={downloadFileFromUrl}
|
|
forceFilter="image"
|
|
/>
|
|
</>
|
|
);
|
|
};
|