diff --git a/frontend/apps/artcraft-webapp/src/app/app.tsx b/frontend/apps/artcraft-webapp/src/app/app.tsx index 6e7f705587..fb0420e68e 100644 --- a/frontend/apps/artcraft-webapp/src/app/app.tsx +++ b/frontend/apps/artcraft-webapp/src/app/app.tsx @@ -18,6 +18,7 @@ import CreateAudio from "../pages/create-audio"; import CreateObject from "../pages/create-object"; import CreateWorld from "../pages/create-world"; import CreateVFX from "../pages/create-vfx"; +import FrameExtractor from "../pages/frame-extractor"; import PageScene from "../pages/pagescene"; import PageDraw from "../pages/pagedraw"; import VideoEditorPage from "../pages/video-editor"; @@ -160,6 +161,7 @@ export function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/frontend/apps/artcraft-webapp/src/components/lightbox/LightboxDetails.tsx b/frontend/apps/artcraft-webapp/src/components/lightbox/LightboxDetails.tsx index 9fcf08b820..0081a5e347 100644 --- a/frontend/apps/artcraft-webapp/src/components/lightbox/LightboxDetails.tsx +++ b/frontend/apps/artcraft-webapp/src/components/lightbox/LightboxDetails.tsx @@ -15,12 +15,12 @@ import { faLink, faPause, faPencil, + faPhotoFilm, faPlay, faSpinnerThird, faTrashCan, faUser, faVideo, - faWaveformLines, faXmark, } from "@fortawesome/pro-solid-svg-icons"; import { @@ -77,6 +77,7 @@ export interface LightboxDetailsProps { onRecreate?: () => void; onMakeVideo?: () => void; onEditOnCanvas?: () => void; + onExtractFrames?: () => void; onDelete?: () => void; showDownloadAppCta?: boolean; } @@ -93,6 +94,7 @@ export function LightboxDetails({ onRecreate, onMakeVideo, onEditOnCanvas, + onExtractFrames, onDelete, showDownloadAppCta, }: LightboxDetailsProps) { @@ -495,6 +497,16 @@ export function LightboxDetails({ Edit on Canvas )} + {onExtractFrames && ( + + )} {onRecreate && promptData.hasToken && ( +
+ {disabled ? "Load a video to start" : "Press C or Enter to capture"} +
+ + +
+ + + {burstOpen && !disabled && burstAvailable && ( + +
+

+ Captures a sequence of frames from the playhead — useful for + finding the sharpest one. +

+
+ + +
+ {isExtracting && progress ? ( +
+
+
+
+ + {progress.done}/{progress.total} + + +
+ ) : ( + + )} +
+ + )} + +
+ +
+
+ Duration + + {!disabled && Number.isFinite(duration) ? formatTime(duration) : "—"} + +
+
+ Resolution + + {!disabled && resolution ? `${resolution.w} × ${resolution.h}` : "—"} + +
+
+
+ ); +}; diff --git a/frontend/apps/artcraft-webapp/src/pages/frame-extractor/frame-extractor.tsx b/frontend/apps/artcraft-webapp/src/pages/frame-extractor/frame-extractor.tsx new file mode 100644 index 0000000000..584f0761e0 --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/frame-extractor/frame-extractor.tsx @@ -0,0 +1,529 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { useNavigate, useSearchParams } from "react-router-dom"; +import { faArrowRotateRight } from "@fortawesome/pro-solid-svg-icons"; +import { Button } from "@storyteller/ui-button"; +import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal"; +import { MediaFilesApi } from "@storyteller/api"; +import { addCorsParam } from "@storyteller/common"; +import Seo from "../../components/seo"; +import { toast } from "../../components/toast/toast"; +import { useSignupCta } from "../../components/signup-cta-modal"; +import { isVideoUrl } from "../../components/lightbox/shared"; +import { VideoDropZone } from "./video-drop-zone"; +import { VideoScrubber } from "./video-scrubber"; +import { ExtractionPanel } from "./extraction-panel"; +import { FramesGrid, FrameActionState } from "./frames-grid"; +import { + captureFrameAt, + extractFrames, + FrameExtractionError, + type ExtractedFrame, +} from "./lib/extract-frames"; +import { + downloadFrame, + sendFrameToCreate, + uploadFrame, +} from "./lib/frame-actions"; + +type VideoSource = + | { kind: "local"; url: string } + | { kind: "library"; url: string; mediaToken: string }; + +// Tokens minted per frame: a visible library save and a hidden reference +// upload are different files server-side, but a library save also works as a +// reference, so `library` is preferred when both could apply. +interface FrameTokens { + library?: string; + intermediate?: string; +} + +export default function FrameExtractor() { + const navigate = useNavigate(); + const [searchParams, setSearchParams] = useSearchParams(); + const { loggedIn, openSignupCta } = useSignupCta(); + + const [source, setSource] = useState(null); + const [currentTime, setCurrentTime] = useState(0); + const [duration, setDuration] = useState(0); + const [resolution, setResolution] = useState<{ w: number; h: number } | null>( + null, + ); + const [numFrames, setNumFrames] = useState(10); + const [spacingMs, setSpacingMs] = useState(100); + const [frames, setFrames] = useState([]); + const [actionState, setActionState] = useState< + Record + >({}); + const [isExtracting, setIsExtracting] = useState(false); + const [isSavingAll, setIsSavingAll] = useState(false); + const [progress, setProgress] = useState<{ + done: number; + total: number; + } | null>(null); + const [isGalleryOpen, setIsGalleryOpen] = useState(false); + const [gallerySelection, setGallerySelection] = useState([]); + + const videoRef = useRef(null); + const abortRef = useRef(null); + const frameTokensRef = useRef>(new Map()); + // Frames handed off as prompt references keep their object URL alive — the + // reference deck on the create page renders it. + const handedOffUrlsRef = useRef>(new Set()); + + // ── Source loading ───────────────────────────────────────────────────────── + + const resetForNewSource = useCallback(() => { + abortRef.current?.abort(); + setFrames((prev) => { + prev.forEach((frame) => { + if (!handedOffUrlsRef.current.has(frame.objectUrl)) { + URL.revokeObjectURL(frame.objectUrl); + } + }); + return []; + }); + setActionState({}); + frameTokensRef.current.clear(); + setCurrentTime(0); + setDuration(0); + setResolution(null); + setProgress(null); + setIsExtracting(false); + }, []); + + const setLocalSource = useCallback( + (files: FileList) => { + const file = files[0]; + if (!file || !file.type.startsWith("video/")) { + toast.error("Please choose a video file"); + return; + } + resetForNewSource(); + setSource((prev) => { + if (prev?.kind === "local") URL.revokeObjectURL(prev.url); + return { kind: "local", url: URL.createObjectURL(file) }; + }); + }, + [resetForNewSource], + ); + + const setLibrarySource = useCallback( + (url: string, mediaToken: string) => { + resetForNewSource(); + setSource((prev) => { + if (prev?.kind === "local") URL.revokeObjectURL(prev.url); + return { kind: "library", url: addCorsParam(url) || url, mediaToken }; + }); + }, + [resetForNewSource], + ); + + const clearSource = useCallback(() => { + resetForNewSource(); + setSource((prev) => { + if (prev?.kind === "local") URL.revokeObjectURL(prev.url); + return null; + }); + }, [resetForNewSource]); + + // Deep link from the library: /frame-extractor?media=. Consumed once + // so switching videos later doesn't resurrect the param's source. + useEffect(() => { + const token = searchParams.get("media"); + if (!token) return; + setSearchParams({}, { replace: true }); + + (async () => { + try { + const response = await new MediaFilesApi().GetMediaFileByToken({ + mediaFileToken: token, + }); + const file = response.success ? response.data : null; + const url = file?.media_links?.cdn_url; + // MediaFileClass isn't exported from the @storyteller/api barrel; + // compare the string value (same approach as the video editor's + // media-source-adapter). isVideoUrl covers legacy "unknown" rows. + const isVideo = + (file?.media_class as string | null) === "video" || + (!!url && isVideoUrl(url)); + if (!url || !isVideo) { + toast.error("That media item isn't a video"); + return; + } + setLibrarySource(url, token); + } catch { + toast.error("Failed to load media"); + } + })(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + // Final cleanup: abort any in-flight burst and release blob URLs that no + // other page is displaying. + useEffect(() => { + return () => { + abortRef.current?.abort(); + setFrames((prev) => { + prev.forEach((frame) => { + if (!handedOffUrlsRef.current.has(frame.objectUrl)) { + URL.revokeObjectURL(frame.objectUrl); + } + }); + return prev; + }); + setSource((prev) => { + if (prev?.kind === "local") URL.revokeObjectURL(prev.url); + return prev; + }); + }; + }, []); + + // ── Extraction ───────────────────────────────────────────────────────────── + + const reportExtractionError = useCallback( + (error: unknown) => { + if (!(error instanceof FrameExtractionError)) { + toast.error("Failed to capture frame"); + return; + } + switch (error.kind) { + case "aborted": + break; + case "cors": + toast.error( + "This video can't be captured due to cross-origin protection. Download it and upload the file directly.", + ); + break; + case "timeout": + toast.error("The video didn't finish loading — try again."); + break; + default: + toast.error("Couldn't read this video. Try an MP4 (H.264) file."); + } + }, + [], + ); + + const handleCaptureCurrent = useCallback(async () => { + const video = videoRef.current; + if (!video || isExtracting) return; + setIsExtracting(true); + try { + const frame = await captureFrameAt(video, video.currentTime); + setFrames((prev) => [frame, ...prev]); + } catch (error) { + reportExtractionError(error); + } finally { + setIsExtracting(false); + } + }, [isExtracting, reportExtractionError]); + + const handleExtractBurst = useCallback(async () => { + const video = videoRef.current; + if (!video || isExtracting) return; + + const abort = new AbortController(); + abortRef.current = abort; + setIsExtracting(true); + setProgress({ done: 0, total: numFrames }); + + try { + const extracted = await extractFrames(video, { + startSec: video.currentTime, + count: numFrames, + spacingMs, + signal: abort.signal, + onProgress: (done, total) => setProgress({ done, total }), + }); + if (extracted.length === 0) { + toast.error("No frames could be captured before the video ended"); + } else { + setFrames((prev) => [...extracted.reverse(), ...prev]); + } + } catch (error) { + reportExtractionError(error); + } finally { + setIsExtracting(false); + setProgress(null); + abortRef.current = null; + } + }, [isExtracting, numFrames, spacingMs, reportExtractionError]); + + const handleCancelBurst = useCallback(() => { + abortRef.current?.abort(); + }, []); + + // ── Frame actions ────────────────────────────────────────────────────────── + + const patchFrameState = useCallback( + (frameId: string, patch: FrameActionState) => { + setActionState((prev) => ({ + ...prev, + [frameId]: { ...prev[frameId], ...patch }, + })); + }, + [], + ); + + // A frame saved to the library is also a valid reference; a hidden + // reference upload never doubles as a library save. + const tokenForReference = useCallback( + async (frame: ExtractedFrame): Promise => { + const cached = frameTokensRef.current.get(frame.id); + if (cached?.library || cached?.intermediate) { + return cached.library ?? cached.intermediate ?? null; + } + const result = await uploadFrame(frame, { intermediate: true }); + if (!result.success) { + toast.error(result.error); + return null; + } + frameTokensRef.current.set(frame.id, { + ...cached, + intermediate: result.mediaToken, + }); + return result.mediaToken; + }, + [], + ); + + const handleSendToCreate = useCallback( + async (frame: ExtractedFrame, destination: "image" | "video") => { + if (!loggedIn) { + openSignupCta(); + return; + } + patchFrameState(frame.id, { sending: true }); + const token = await tokenForReference(frame); + patchFrameState(frame.id, { sending: false }); + if (!token) return; + handedOffUrlsRef.current.add(frame.objectUrl); + sendFrameToCreate(frame, token, destination, navigate); + }, + [loggedIn, openSignupCta, patchFrameState, tokenForReference, navigate], + ); + + const saveFrameToLibrary = useCallback( + async (frame: ExtractedFrame): Promise => { + const cached = frameTokensRef.current.get(frame.id); + if (cached?.library) return true; + const result = await uploadFrame(frame, { intermediate: false }); + if (!result.success) return false; + frameTokensRef.current.set(frame.id, { + ...cached, + library: result.mediaToken, + }); + return true; + }, + [], + ); + + const handleSave = useCallback( + async (frame: ExtractedFrame) => { + if (!loggedIn) { + openSignupCta(); + return; + } + const state = actionState[frame.id]; + if (state?.saved || state?.saving) return; + patchFrameState(frame.id, { saving: true }); + const saved = await saveFrameToLibrary(frame); + patchFrameState(frame.id, { saving: false, saved }); + if (saved) { + toast.success("Frame saved to library"); + } else { + toast.error("Failed to save frame"); + } + }, + [loggedIn, openSignupCta, actionState, patchFrameState, saveFrameToLibrary], + ); + + const handleSaveAll = useCallback(async () => { + if (!loggedIn) { + openSignupCta(); + return; + } + if (isSavingAll) return; + setIsSavingAll(true); + let failures = 0; + // Sequential on purpose — parallel uploads of dozens of PNGs invite rate + // limiting, and per-frame progress reads better anyway. + for (const frame of frames) { + if (actionState[frame.id]?.saved) continue; + patchFrameState(frame.id, { saving: true }); + const saved = await saveFrameToLibrary(frame); + patchFrameState(frame.id, { saving: false, saved }); + if (!saved) failures++; + } + setIsSavingAll(false); + if (failures === 0) { + toast.success("All frames saved to library"); + } else { + toast.error(`Failed to save ${failures} ${failures === 1 ? "frame" : "frames"}`); + } + }, [ + loggedIn, + openSignupCta, + isSavingAll, + frames, + actionState, + patchFrameState, + saveFrameToLibrary, + ]); + + const handleRemove = useCallback((frame: ExtractedFrame) => { + setFrames((prev) => prev.filter((f) => f.id !== frame.id)); + if (!handedOffUrlsRef.current.has(frame.objectUrl)) { + URL.revokeObjectURL(frame.objectUrl); + } + frameTokensRef.current.delete(frame.id); + setActionState((prev) => { + const next = { ...prev }; + delete next[frame.id]; + return next; + }); + }, []); + + const handleClear = useCallback(() => { + setFrames((prev) => { + prev.forEach((frame) => { + if (!handedOffUrlsRef.current.has(frame.objectUrl)) { + URL.revokeObjectURL(frame.objectUrl); + } + }); + return []; + }); + frameTokensRef.current.clear(); + setActionState({}); + }, []); + + // ── Library picker ───────────────────────────────────────────────────────── + + const handleOpenGallery = useCallback(() => { + if (!loggedIn) { + openSignupCta(); + return; + } + setIsGalleryOpen(true); + }, [loggedIn, openSignupCta]); + + const handleGallerySelect = useCallback( + (id: string) => { + setGallerySelection((prev) => (prev.includes(id) ? [] : [id])); + }, + [], + ); + + const handleGalleryUse = useCallback( + (selectedItems: GalleryItem[]) => { + const item = selectedItems[0]; + if (!item?.fullImage) { + toast.error("No video selected"); + return; + } + setLibrarySource(item.fullImage, item.id); + setIsGalleryOpen(false); + setGallerySelection([]); + }, + [setLibrarySource], + ); + + const handleVideoError = useCallback(() => { + toast.error("Your browser can't play this video. Try an MP4 (H.264) file."); + clearSource(); + }, [clearSource]); + + // ── Render ───────────────────────────────────────────────────────────────── + + return ( +
+ +
+
+
+

+ Frame Extractor +

+ {source && ( + + )} +
+ +
+ {source ? ( + + ) : ( + + )} + +
+ + handleSendToCreate(frame, "image")} + onUseForVideo={(frame) => handleSendToCreate(frame, "video")} + onSave={handleSave} + onDownload={downloadFrame} + onRemove={handleRemove} + onSaveAll={handleSaveAll} + isSavingAll={isSavingAll} + onClear={handleClear} + /> +
+
+ + { + setIsGalleryOpen(false); + setGallerySelection([]); + }} + selectedItemIds={gallerySelection} + onSelectItem={handleGallerySelect} + maxSelections={1} + onUseSelected={handleGalleryUse} + forceFilter="video" + hideFilter + /> +
+ ); +} diff --git a/frontend/apps/artcraft-webapp/src/pages/frame-extractor/frames-grid.tsx b/frontend/apps/artcraft-webapp/src/pages/frame-extractor/frames-grid.tsx new file mode 100644 index 0000000000..fd4c19a6dc --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/frame-extractor/frames-grid.tsx @@ -0,0 +1,188 @@ +import { AnimatePresence, motion } from "framer-motion"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { + faArrowDownToLine, + faCheck, + faImage, + faSave, + faSpinnerThird, + faVideo, + faXmark, +} from "@fortawesome/pro-solid-svg-icons"; +import type { IconDefinition } from "@fortawesome/fontawesome-svg-core"; +import { Tooltip } from "@storyteller/ui-tooltip"; +import { EASE_EMPHASIS } from "../../lib/motion"; +import type { ExtractedFrame } from "./lib/extract-frames"; +import { formatTimePrecise } from "./lib/frame-actions"; + +export interface FrameActionState { + saving?: boolean; + saved?: boolean; + sending?: boolean; +} + +interface FramesGridProps { + frames: ExtractedFrame[]; + actionState: Record; + onUseAsImageRef: (frame: ExtractedFrame) => void; + onUseForVideo: (frame: ExtractedFrame) => void; + onSave: (frame: ExtractedFrame) => void; + onDownload: (frame: ExtractedFrame) => void; + onRemove: (frame: ExtractedFrame) => void; + onSaveAll: () => void; + isSavingAll: boolean; + onClear: () => void; +} + +export const FramesGrid = ({ + frames, + actionState, + onUseAsImageRef, + onUseForVideo, + onSave, + onDownload, + onRemove, + onSaveAll, + isSavingAll, + onClear, +}: FramesGridProps) => { + if (frames.length === 0) return null; + + const hasUnsaved = frames.some((frame) => !actionState[frame.id]?.saved); + + return ( +
+
+

+ Captured frames + + {frames.length} + +

+
+ {hasUnsaved && ( + + )} + +
+
+ +
+ + {frames.map((frame) => { + const state = actionState[frame.id] ?? {}; + return ( + +
+ {`Frame + + + {formatTimePrecise(frame.timestamp)} + +
+
+ onUseAsImageRef(frame)} + loading={state.sending} + /> + onUseForVideo(frame)} + loading={state.sending} + /> + onSave(frame)} + loading={state.saving} + disabled={state.saved} + success={state.saved} + /> + onDownload(frame)} + /> +
+ + ); + })} + +
+
+ ); +}; + +interface FrameActionProps { + icon: IconDefinition; + tooltip: string; + onClick: () => void; + loading?: boolean; + disabled?: boolean; + success?: boolean; +} + +const FrameAction = ({ + icon, + tooltip, + onClick, + loading, + disabled, + success, +}: FrameActionProps) => ( + // fixed strategy: the card is overflow-hidden (rounded corners), which + // would clip an absolutely-positioned tooltip. + + + +); diff --git a/frontend/apps/artcraft-webapp/src/pages/frame-extractor/index.ts b/frontend/apps/artcraft-webapp/src/pages/frame-extractor/index.ts new file mode 100644 index 0000000000..21a4fb2ea0 --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/frame-extractor/index.ts @@ -0,0 +1 @@ +export { default } from "./frame-extractor"; diff --git a/frontend/apps/artcraft-webapp/src/pages/frame-extractor/lib/extract-frames.ts b/frontend/apps/artcraft-webapp/src/pages/frame-extractor/lib/extract-frames.ts new file mode 100644 index 0000000000..b7f40f2644 --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/frame-extractor/lib/extract-frames.ts @@ -0,0 +1,239 @@ +// Client-side video frame extraction: seek the