Webapp: drag & drop files onto the prompt box to add references (#1800)

- Drop images, videos, or audio anywhere on the prompt box - files are
sorted into the right reference type based on what the current model
supports
- Drag-over overlay shows which file kinds are accepted; turns red when
the hovered files aren't usable
- Video keyframe mode: dropped images fill the start frame, then the end
frame
- Unsupported kinds, full slots, and non-media files get a clear toast
instead of failing silently
- Reuses the existing `useDeckMedia` upload pipelines, so limits and
error handling match the "+" pickers

frontend/apps/artcraft-webapp/src/components/prompt-box/PromptBox.tsx

@@ -28,6 +28,12 @@ import {
 } from "@storyteller/ui-promptbox";
 import { arrayMove } from "@dnd-kit/sortable";
 import { MentionTextarea } from "./MentionTextarea";
+import {
+  PromptBoxDropOverlay,
+  usePromptBoxDrop,
+  type DroppedFiles,
+} from "./PromptBoxDropZone";
+import { toast } from "../toast/toast";
 import { getMentionColor, buildMentionColorMap } from "./mention-colors";
 import { uploadImage } from "./upload-image";
 import { uploadVideo, uploadAudio } from "./upload-media";
@@ -215,6 +221,57 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
       ownGalleryModal: false,
     });
 
+    // Drag & drop onto the box bounds: files route to the reference kind
+    // their MIME matches, gated on what the page's model supports. Keyframe
+    // mode renders no video/audio deck, so those kinds only land in
+    // reference mode where the user can see (and remove) them.
+    const isKeyframeMode = !!isVideo && !isReferenceMode;
+    const dropAcceptsImages =
+      !!supportsImagePrompts && maxImagePromptCount > 0;
+    const dropAcceptsVideos =
+      !isKeyframeMode && !!videoRefsSupported && !!onReferenceVideosChange;
+    const dropAcceptsAudio =
+      !isKeyframeMode && !!audioRefsSupported && !!onReferenceAudiosChange;
+
+    const handleDroppedFiles = ({ images, videos, audios }: DroppedFiles) => {
+      if (images.length > 0) {
+        if (isKeyframeMode) {
+          // Fill the empty keyframe slots in order: start frame, then end.
+          const queue = [...images];
+          const startOpen =
+            referenceImages.length === 0 && deck.uploadingImages.length === 0;
+          const endOpen =
+            !!showEndFrameSection && !endFrameImage && !deck.uploadingEnd;
+          if (startOpen) deck.processImageFiles([queue.shift()!], "start");
+          if (endOpen && queue.length > 0) {
+            deck.processImageFiles([queue.shift()!], "end");
+          }
+          if (!startOpen && !endOpen) {
+            toast.error(
+              showEndFrameSection
+                ? "Start and end frames are already set"
+                : "The start frame is already set",
+            );
+          }
+        } else if (deck.availableImageSlots <= 0) {
+          toast.error(
+            `Max ${maxImagePromptCount} image reference${maxImagePromptCount === 1 ? "" : "s"}`,
+          );
+        } else {
+          deck.processImageFiles(images, "start");
+        }
+      }
+      if (videos.length > 0) deck.processVideoFiles(videos);
+      if (audios.length > 0) deck.processAudioFiles(audios);
+    };
+
+    const drop = usePromptBoxDrop({
+      acceptsImages: dropAcceptsImages,
+      acceptsVideos: dropAcceptsVideos,
+      acceptsAudio: dropAcceptsAudio,
+      onDropFiles: handleDroppedFiles,
+    });
+
     // Mixed deck items ordered images → videos → audios so the page's
     // index-derived @ImageN/@VideoN/@AudioN mention labels stay aligned.
     const deckItems: DeckItem[] = useMemo(
@@ -424,7 +481,7 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
     // Left-of-textarea reference widget: image deck, keyframe cards, or the
     // mixed deck depending on page/mode.
     const renderReferenceWidget = (alwaysExpanded?: boolean) => {
-      if (isVideo && !isReferenceMode) {
+      if (isKeyframeMode) {
         if (!supportsImagePrompts) return null;
         return (
           <KeyframeCards
@@ -710,7 +767,15 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
               "glass rounded-2xl p-3 sm:p-4 !transition-all duration-200",
               isFocused && "ring-1 ring-primary",
             )}
+            {...drop.dropZoneProps}
           >
+            <PromptBoxDropOverlay
+              dragState={drop.dragState}
+              acceptsImages={dropAcceptsImages}
+              acceptsVideos={dropAcceptsVideos}
+              acceptsAudio={dropAcceptsAudio}
+              keyframeMode={isKeyframeMode}
+            />
             <div className="flex gap-3">
               {renderReferenceWidget()}
               {referenceSlots}

frontend/apps/artcraft-webapp/src/components/prompt-box/PromptBoxDropZone.tsx

@@ -0,0 +1,260 @@
+import { useCallback, useEffect, useState } from "react";
+import { twMerge } from "tailwind-merge";
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { faImages, faMusic, faVideo } from "@fortawesome/pro-solid-svg-icons";
+import { toast } from "../toast/toast";
+
+// ── Types ───────────────────────────────────────────────────────────────
+
+type DropKind = "image" | "video" | "audio";
+
+/** "accept" / "reject" reflect whether the hovered payload contains at
+ *  least one file the current model can take as a reference. */
+export type DropDragState = "idle" | "accept" | "reject";
+
+export interface DroppedFiles {
+  images: File[];
+  videos: File[];
+  audios: File[];
+}
+
+interface UsePromptBoxDropOptions {
+  acceptsImages: boolean;
+  acceptsVideos: boolean;
+  acceptsAudio: boolean;
+  onDropFiles: (files: DroppedFiles) => void;
+}
+
+interface PromptBoxDropOverlayProps {
+  dragState: DropDragState;
+  acceptsImages: boolean;
+  acceptsVideos: boolean;
+  acceptsAudio: boolean;
+  /** Video keyframe mode: images fill the start/end frame slots. */
+  keyframeMode?: boolean;
+}
+
+const kindOfMime = (mime: string): DropKind | null =>
+  mime.startsWith("image/")
+    ? "image"
+    : mime.startsWith("video/")
+      ? "video"
+      : mime.startsWith("audio/")
+        ? "audio"
+        : null;
+
+const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1);
+
+const listKinds = (labels: string[]) =>
+  labels.length <= 1
+    ? (labels[0] ?? "")
+    : `${labels.slice(0, -1).join(", ")} or ${labels[labels.length - 1]}`;
+
+/**
+ * Makes the prompt box a drop target for reference media. Files are routed
+ * by MIME type; kinds the current model doesn't take are rejected with a
+ * toast instead of silently vanishing.
+ */
+export function usePromptBoxDrop({
+  acceptsImages,
+  acceptsVideos,
+  acceptsAudio,
+  onDropFiles,
+}: UsePromptBoxDropOptions) {
+  const [dragState, setDragState] = useState<DropDragState>("idle");
+  const enabled = acceptsImages || acceptsVideos || acceptsAudio;
+
+  const acceptsKind = useCallback(
+    (kind: DropKind | null) =>
+      kind === "image"
+        ? acceptsImages
+        : kind === "video"
+          ? acceptsVideos
+          : kind === "audio"
+            ? acceptsAudio
+            : false,
+    [acceptsImages, acceptsVideos, acceptsAudio],
+  );
+
+  // While the box is on screen, a stray drop outside it must not navigate
+  // the browser to the file (which would blow away the user's session).
+  useEffect(() => {
+    if (!enabled) return;
+    const preventNavigation = (e: DragEvent) => {
+      if (e.dataTransfer?.types.includes("Files")) e.preventDefault();
+    };
+    // Drops that land outside the box still need the overlay dismissed.
+    const clearDragState = () => setDragState("idle");
+    window.addEventListener("dragover", preventNavigation);
+    window.addEventListener("drop", preventNavigation);
+    window.addEventListener("drop", clearDragState);
+    window.addEventListener("dragend", clearDragState);
+    return () => {
+      window.removeEventListener("dragover", preventNavigation);
+      window.removeEventListener("drop", preventNavigation);
+      window.removeEventListener("drop", clearDragState);
+      window.removeEventListener("dragend", clearDragState);
+    };
+  }, [enabled]);
+
+  // MIME types are unreliable mid-drag (browsers may report empty strings),
+  // so unknown types count as acceptable until the drop resolves them.
+  const inspectDrag = useCallback(
+    (dt: DataTransfer): DropDragState => {
+      const fileItems = Array.from(dt.items).filter((i) => i.kind === "file");
+      if (fileItems.length === 0) return "accept";
+      const ok = fileItems.some(
+        (i) => i.type === "" || acceptsKind(kindOfMime(i.type)),
+      );
+      return ok ? "accept" : "reject";
+    },
+    [acceptsKind],
+  );
+
+  const handleDragOver = (e: React.DragEvent<HTMLDivElement>) => {
+    if (!enabled || !e.dataTransfer.types.includes("Files")) return;
+    e.preventDefault();
+    e.stopPropagation();
+    e.dataTransfer.dropEffect = "copy";
+    setDragState(inspectDrag(e.dataTransfer));
+  };
+
+  const handleDragLeave = (e: React.DragEvent<HTMLDivElement>) => {
+    if (dragState === "idle") return;
+    // Child elements fire enter/leave pairs constantly; only reset when the
+    // pointer actually exits the drop target's bounds.
+    const rect = e.currentTarget.getBoundingClientRect();
+    if (
+      e.clientX < rect.left ||
+      e.clientX >= rect.right ||
+      e.clientY < rect.top ||
+      e.clientY >= rect.bottom
+    ) {
+      setDragState("idle");
+    }
+  };
+
+  const handleDrop = (e: React.DragEvent<HTMLDivElement>) => {
+    if (!enabled || !e.dataTransfer.types.includes("Files")) return;
+    e.preventDefault();
+    e.stopPropagation();
+    setDragState("idle");
+
+    const files = Array.from(e.dataTransfer.files);
+    if (files.length === 0) return;
+
+    const accepted: DroppedFiles = { images: [], videos: [], audios: [] };
+    const rejectedKinds = new Set<DropKind>();
+    let unknownCount = 0;
+    for (const file of files) {
+      const kind = kindOfMime(file.type);
+      if (kind === null) {
+        unknownCount++;
+      } else if (!acceptsKind(kind)) {
+        rejectedKinds.add(kind);
+      } else if (kind === "image") {
+        accepted.images.push(file);
+      } else if (kind === "video") {
+        accepted.videos.push(file);
+      } else {
+        accepted.audios.push(file);
+      }
+    }
+
+    const anyAccepted =
+      accepted.images.length > 0 ||
+      accepted.videos.length > 0 ||
+      accepted.audios.length > 0;
+    if (anyAccepted) onDropFiles(accepted);
+
+    // Mode-neutral wording: a kind can be off because the model lacks it or
+    // because the current input mode (e.g. keyframes) doesn't show it.
+    if (rejectedKinds.size > 0) {
+      toast.error(
+        `${capitalize(listKinds([...rejectedKinds]))} references aren't available here`,
+      );
+    } else if (unknownCount > 0 && !anyAccepted) {
+      toast.error(
+        `Only ${listKinds(acceptedKindLabels(acceptsImages, acceptsVideos, acceptsAudio))} files can be dropped here`,
+      );
+    }
+  };
+
+  return {
+    dragState,
+    dropZoneProps: {
+      onDragEnter: handleDragOver,
+      onDragOver: handleDragOver,
+      onDragLeave: handleDragLeave,
+      onDrop: handleDrop,
+    },
+  };
+}
+
+/**
+ * Full-bleed dashed overlay shown while files hover the prompt box, listing
+ * the reference kinds the current model takes (red when none match).
+ */
+export function PromptBoxDropOverlay({
+  dragState,
+  acceptsImages,
+  acceptsVideos,
+  acceptsAudio,
+  keyframeMode,
+}: PromptBoxDropOverlayProps) {
+  if (dragState === "idle") return null;
+
+  const kinds = [
+    ...(acceptsImages
+      ? [{ icon: faImages, label: keyframeMode ? "Frames" : "Images" }]
+      : []),
+    ...(acceptsVideos ? [{ icon: faVideo, label: "Video" }] : []),
+    ...(acceptsAudio ? [{ icon: faMusic, label: "Audio" }] : []),
+  ];
+  const rejected = dragState === "reject";
+
+  return (
+    <div
+      className={twMerge(
+        "promptbox-drop-overlay pointer-events-none absolute inset-0 z-40 flex flex-col items-center justify-center gap-2 rounded-2xl border-2 border-dashed bg-[#161618]/85 backdrop-blur-sm",
+        rejected ? "border-red-400/80" : "border-primary",
+      )}
+    >
+      <div className="flex items-center gap-2">
+        {kinds.map((kind, i) => (
+          <div
+            key={kind.label}
+            className="promptbox-drop-chip flex h-9 w-9 items-center justify-center rounded-lg bg-white/10 text-base-fg/90"
+            style={{ animationDelay: `${i * 50}ms` }}
+          >
+            <FontAwesomeIcon icon={kind.icon} className="text-sm" />
+          </div>
+        ))}
+      </div>
+      <div className="text-sm font-semibold text-base-fg">
+        {rejected
+          ? "That file type isn't supported"
+          : keyframeMode && !acceptsVideos && !acceptsAudio
+            ? "Drop to set your frames"
+            : "Drop to add references"}
+      </div>
+      <div className="-mt-1 text-xs text-base-fg/60">
+        {rejected
+          ? `Accepts ${listKinds(acceptedKindLabels(acceptsImages, acceptsVideos, acceptsAudio))} files`
+          : kinds.map((kind) => kind.label).join(" · ")}
+      </div>
+    </div>
+  );
+}
+
+function acceptedKindLabels(
+  acceptsImages: boolean,
+  acceptsVideos: boolean,
+  acceptsAudio: boolean,
+): string[] {
+  return [
+    ...(acceptsImages ? ["image"] : []),
+    ...(acceptsVideos ? ["video"] : []),
+    ...(acceptsAudio ? ["audio"] : []),
+  ];
+}

frontend/apps/artcraft-webapp/src/styles.css

@@ -341,6 +341,39 @@
   animation: toast-in 0.2s ease-out both;
 }
 
+/* Prompt box drag & drop overlay */
+@keyframes promptbox-drop-overlay-in {
+  from {
+    opacity: 0;
+  }
+
+  to {
+    opacity: 1;
+  }
+}
+
+.promptbox-drop-overlay {
+  animation: promptbox-drop-overlay-in 0.15s ease-out both;
+}
+
+@keyframes promptbox-drop-chip-in {
+  from {
+    opacity: 0;
+    translate: 0 6px;
+    scale: 0.9;
+  }
+
+  to {
+    opacity: 1;
+    translate: 0 0;
+    scale: 1;
+  }
+}
+
+.promptbox-drop-chip {
+  animation: promptbox-drop-chip-in 0.25s cubic-bezier(0.22, 1, 0.36, 1) both;
+}
+
 @keyframes marquee {
   0% {
     transform: translateX(0);

frontend/libs/components/promptbox/src/lib/deck/useDeckMedia.tsx

@@ -731,6 +731,9 @@ export function useDeckMedia<
     uploadingVideo,
     uploadingAudio,
     availableImageSlots,
+    processImageFiles,
+    processVideoFiles,
+    processAudioFiles,
     openImageUpload,
     openEndUpload,
     openVideoUpload,