From d27d5cec75b453ebcd9329ca0a0a20762ec6e4f3 Mon Sep 17 00:00:00 2001 From: Tanish Manku Date: Mon, 15 Jun 2026 17:53:15 -0700 Subject: [PATCH] PageDraw: Port and mount to web app (#1650) --- _docs/pagedraw_design_review.md | 203 +++++++++++++ frontend/apps/artcraft-webapp/src/app/app.tsx | 2 + .../components/lightbox/LightboxDetails.tsx | 12 + .../src/components/lightbox/lightbox.tsx | 13 + .../src/components/sidebar/app-sidebar.tsx | 2 + .../pages/pagedraw/base-image-selector.tsx | 223 ++++++++++++++ .../src/pages/pagedraw/edit-on-canvas.ts | 27 ++ .../src/pages/pagedraw/index.ts | 1 + .../src/pages/pagedraw/job-polling.ts | 141 +++++++++ .../src/pages/pagedraw/pagedraw.tsx | 25 ++ .../src/pages/pagedraw/web-adapter.tsx | 275 ++++++++++++++++++ 11 files changed, 924 insertions(+) create mode 100644 _docs/pagedraw_design_review.md create mode 100644 frontend/apps/artcraft-webapp/src/pages/pagedraw/base-image-selector.tsx create mode 100644 frontend/apps/artcraft-webapp/src/pages/pagedraw/edit-on-canvas.ts create mode 100644 frontend/apps/artcraft-webapp/src/pages/pagedraw/index.ts create mode 100644 frontend/apps/artcraft-webapp/src/pages/pagedraw/job-polling.ts create mode 100644 frontend/apps/artcraft-webapp/src/pages/pagedraw/pagedraw.tsx create mode 100644 frontend/apps/artcraft-webapp/src/pages/pagedraw/web-adapter.tsx diff --git a/_docs/pagedraw_design_review.md b/_docs/pagedraw_design_review.md new file mode 100644 index 0000000000..cdcd0cb93b --- /dev/null +++ b/_docs/pagedraw_design_review.md @@ -0,0 +1,203 @@ +# PageDraw — Visual Design & Architecture Review + +**Surface:** `@storyteller/ui-pagedraw` image editor + its `artcraft-webapp` integration (`/edit-image`). +**Lens:** Awwwards / Linear-tier "premium app chrome" rubric, adapted for an in-product creative tool. +**Date:** 2026-06-15 · **Branch:** `worktree-pagedraw-webapp` + +--- + +## 0. How to read this + +PageDraw is a **professional canvas tool**, not a marketing page. So the elite-design rubric is +applied in its *app-chrome* register — the north star is **Figma / Linear / Photoshop-web**, not an +Awwwards landing page. That means: + +- **Applies:** token discipline, concentric radii, hairline + inset-highlight surfaces ("double-bezel"), + island/pill controls, custom-eased micro-motion, z-index hygiene, premium type ramp. +- **Does _not_ apply:** `py-40` macro-whitespace, hero typography, editorial-split layouts, scroll-reveal + entrances. A tool surface is dense and persistent by design; forcing landing-page whitespace would hurt it. + +Citations are `file:line` against the current tree. + +--- + +## 1. Verdict + +| Axis | Score | One-liner | +|---|---|---| +| Color system | ⚠️ 5/10 | Good token foundation, but hardcoded greys (`#1b1b1b`, `#303030`, `#404040`) leak through a *themeable lib*. | +| Typography | ❌ 3/10 | No type ramp, no font-family ownership; rides whatever the host sets. Banned-by-rubric default stack risk. | +| Radii | ⚠️ 4/10 | Five radius scales in play (`sm`→`2xl`→`full`) with no concentric nesting. | +| Surfaces (border/shadow) | ⚠️ 5/10 | Flat `shadow-lg` everywhere; no inset highlights; no double-bezel. Reads "functional", not "machined". | +| Motion | ❌ 3/10 | Only `transition-colors`/`ease-in-out`, durations 75–200ms; **zero custom easing**, no spring, no entrance choreography. | +| Layout / z-index | ⚠️ 6/10 | Clean fixed-panel composition, but a `z-[9999999]` escape hatch and a 6-tier ad-hoc z-stack. | +| Icons | ✅ 7/10 | FontAwesome Pro (solid + regular) — acceptable, but solid weight is heavier than the rubric's "ultra-light line" ideal. | +| Architecture | ⚠️ 5/10 | Clean adapter boundary; but `PaintSurface.tsx` (2005 LOC), `SceneState.ts` (1539), `PageDraw.tsx` (1292) are oversized, and the lib hardcodes host-theme colors. | + +**Bottom line:** the editor is *structurally competent and functional* but visually reads as +"capable internal tool," not "$150k product surface." The single highest-leverage fix is a **shared +design-token layer** (color + radius + motion + type), because today the lib both (a) reaches for +host tokens (`bg-ui-panel`, `text-base-fg`) *and* (b) hardcodes its own greys — so it can never be +fully reskinned by the webapp vs. the desktop app. + +--- + +## 2. What's already good (keep) + +- **Token-aware base.** Most surfaces use semantic tokens (`bg-ui-panel`, `bg-ui-controls`, + `text-base-fg`, `border-ui-panel-border`) — the right instinct (SideToolbar, PromptEditor, HistoryStack). +- **Clean platform boundary.** The lib has zero REST/Tauri imports; everything I/O flows through + `PageDrawAdapter` (`adapter.ts`). The webapp adapter (`web-adapter.tsx`) and desktop adapter stay + swappable. This is genuinely good architecture and should be protected. +- **Performance-minded canvas.** Composite pre-baking on idle (`PageDraw.tsx` ~661–710) and worker + encoding (`generatePipeline.worker.ts`) keep the generate click off the main thread. Don't regress this. +- **CORS already handled** for browser canvas export (`crossOrigin="anonymous"` in `Node.ts:134`, + `SceneState.ts:1473`, `imageHelpers.ts:16`) — the web port works because of this. + +--- + +## 3. Findings & recommendations + +### 3.1 Color — stop hardcoding greys in a themeable lib ⚠️ P0 +**Current:** semantic tokens *and* raw hex coexist: +- `#1b1b1b` pegboard (`pagedraw.css:6`), `#1A1A1A`/`#333333` context menu (`ContextMenu.tsx:44`), + `#303030`/`#404040` dividers + popovers (`SideToolbar.tsx:381,550`), `bg-zinc-700/600` swatch + (`SideToolbar.tsx:173`), plus ad-hoc `bg-blue-600/500` accents (`PageDraw.tsx:157`, base selector). +- Accent color is inconsistent: the lib's primary action uses raw **blue-600**, while the webapp's + design system uses `--primary` / `brand-primary`. So the editor's blue doesn't match the product. + +**Why it fails the bar:** a premium component library exposes *one* surface ramp and *one* accent, both +driven by tokens, so it reskins per host. Hardcoded greys mean the desktop and web themes can drift, and +the raw `blue-600` clashes with `brand-primary` used elsewhere in the webapp. + +**Do:** +- Replace every raw grey with the existing ramp: `#1b1b1b`→`bg-ui-background`, `#1A1A1A`→`bg-ui-panel`, + `#303030`/`#404040`→`border-ui-panel-border` / `bg-ui-controls`. For the pegboard, add a token + `--st-canvas` so hosts can theme the canvas backdrop. +- Replace `blue-600/500` accents with `bg-primary` / `text-primary` (the token the rest of the webapp uses), + so the Generate/Edit-3D buttons match `brand-primary`. +- Move the react-colorful overrides off pure `rgb(255,255,255)` borders (`pagedraw.css:15,19`) onto + `white/15` hairlines for consistency with the rest of the chrome. + +### 3.2 Typography — own a type ramp ❌ P0 +**Current:** no `font-family` anywhere in the lib (grep: 0 hits); only `text-xs/sm/base/lg/xl` + +`font-medium/semibold/bold`. The editor inherits whatever the host's body font is. + +**Why it fails the bar:** the rubric explicitly bans the default web stack (Inter/Roboto/Arial/Helvetica). +Inheriting an unspecified stack means the tool can render in a banned font with no recourse, and there's +no deliberate hierarchy (numerics in sliders, labels, modal titles all share one undifferentiated ramp). + +**Do:** +- Adopt the product's display/grotesk token (e.g. a `font-sans` mapped to Geist / Plus Jakarta Sans at + the app level) and reference it in the lib's root container, not per-component. +- Define a small semantic ramp: `text-[11px] tracking-[0.16em] uppercase` for toolbar/section labels, + `text-sm` for controls, `text-base font-semibold` for modal titles. Apply `tabular-nums` to slider + read-outs (`SliderWithIndicator.tsx`) and dimension fields (`BlankCanvasModal`) so numbers don't jitter. + +### 3.3 Radii — collapse to a concentric scale ⚠️ P1 +**Current:** `rounded-sm / md / lg / xl / 2xl / full` all appear, with no parent→child relationship +(e.g. `rounded-2xl` card at `PageDraw.tsx:1123`, `rounded-xl` panels in SideToolbar, `rounded-lg` +buttons in InpaintToolBar). + +**Do:** pick a 3-step system — **`rounded-2xl` shells, `rounded-xl` inner cards, `rounded-lg` controls, +`rounded-full` pills** — and apply concentrically (the "double-bezel" curve math: inner radius = +outer − padding). Drop `rounded-sm`/`rounded-md` from the editor entirely. + +### 3.4 Surfaces — add the "machined" double-bezel ⚠️ P1 +**Current:** panels are a single `bg-ui-panel` + `border` + `shadow-lg`. No nested enclosure, no inset +top-highlight. Reads flat. + +**Do (for the four floating panels — SideToolbar, HistoryStack, PromptEditor, InpaintToolBar):** +- Outer shell: `bg-ui-panel/80 ring-1 ring-white/10 backdrop-blur-xl p-1.5 rounded-2xl`. +- Inner core: own bg + `shadow-[inset_0_1px_0_rgba(255,255,255,0.08)] rounded-xl`. +- This is the single change that most moves the surface from "tool" to "premium hardware." The panels are + already `fixed` (`PageDraw.tsx:1144,1175,1195`), so `backdrop-blur` here is GPU-safe per the perf rules. + +### 3.5 Motion — replace linear/ease with spring-eased micro-motion ❌ P0 (cheap, high impact) +**Current:** `transition-colors`/`transition-all` with `ease-in-out` and `duration-75/150/200` +(`SideToolbar.tsx:381,550`, `SliderWithIndicator.tsx:65`, `ContextMenu.tsx:53`). No custom easing, +no entrance/exit on popovers, color pickers, or the context menu (they pop instantly). + +**Why it fails the bar:** instant state changes and `ease-in-out` are explicitly banned. A pro tool still +deserves *interpolated* motion — it's what separates Figma from a jQuery plugin. + +**Do:** +- Standardize on one curve: `transition-[transform,opacity,background-color] duration-300 + ease-[cubic-bezier(0.32,0.72,0,1)]`. +- Tool buttons: `active:scale-[0.97]` press physics; selected tool gets a spring-eased indicator slide + rather than a hard background swap. +- Popovers / color picker / context menu: animate in with `opacity-0 translate-y-1 scale-[0.98]` → + resolved, ~180ms. Transform+opacity only (perf rule). +- History tiles: stagger their `opacity` reveal when a generation lands (they already use + `transition-opacity`, just add per-index delay). + +### 3.6 Layout & z-index — tame the stack ⚠️ P1 +**Current:** clean fixed-panel composition, but z-index is ad-hoc across 6 tiers and includes a +`z-[9999999]` on the context menu (`ContextMenu.tsx:44`) and a one-off `z-[60]` (`PageDraw.tsx:237`). + +**Do:** define a named scale in one place — `canvas:0, chrome:10, floating-controls:20, overlay-button:40, +menu:50` — and replace the magic numbers. The portal menu only needs to clear the chrome, not `9999999`. + +### 3.7 Responsive — declare intent ⚠️ P2 +**Current:** essentially desktop-only; fixed sidebars assume a wide viewport; only LightboxDetails has +`sm:` variants. The webapp wrapper scopes the lib correctly (`pagedraw.tsx` `translateZ(0)` containing +block) but there's no mobile story. + +**Do:** mirror the Edit-3D page's explicit **mobile gate** (`pagescene.tsx` `MobileGate`) on `/edit-image` +rather than letting fixed toolbars overlap on touch. A precision canvas tool is legitimately desktop-first; +just say so deliberately instead of degrading silently. + +--- + +## 4. Architecture notes + +- **Oversized modules.** `PaintSurface.tsx` (2005 LOC), `SceneState.ts` (1539), `PageDraw.tsx` (1292). + These are review/maintenance hotspots. Suggest extracting from `SceneState.ts`: history/undo slice, + base-image-load slice, and generation/pending slice into composable Zustand slices. `PageDraw.tsx`'s + generate-flow (cases 1–3, ~780–868) could move to a `useGenerate()` hook. +- **Theme ownership.** Per §3.1, the lib should *only* consume tokens. Today it mixes tokens + hex, which + defeats the adapter pattern's promise (one component, many themed hosts). +- **Adapter surface is healthy.** `PageDrawAdapter` is small and intentional; the web vs. Tauri split is + clean. The webapp's `enqueueInpaint` stub and 402→credits-modal handling are explicit and documented. +- **Icon weight.** FontAwesome **solid** is the heaviest option; if a line/light set is available + (Phosphor/Remix or FA light), the toolbar would read more refined. Low priority. + +--- + +## 5. Prioritized roadmap + +| P | Item | Effort | Payoff | +|---|---|---|---| +| **P0** | Token sweep: kill hardcoded greys; unify accent on `--primary` (§3.1) | M | Highest — enables true theming + product-consistent accent | +| **P0** | Own a font-family + semantic type ramp at the lib root (§3.2) | S | High — removes banned-font risk, adds hierarchy | +| **P0** | One custom easing curve + press/popover micro-motion (§3.5) | S | High — cheapest "feels expensive" win | +| **P1** | Double-bezel + inset highlights on the 4 floating panels (§3.4) | M | High — "machined" surface upgrade | +| **P1** | Concentric 3-step radius system (§3.3) | S | Medium — visual coherence | +| **P1** | Named z-index scale; remove `z-[9999999]` (§3.6) | S | Medium — maintainability | +| **P2** | Explicit mobile gate on `/edit-image` (§3.7) | S | Medium — correctness on touch | +| **P2** | Split `SceneState.ts` / `PaintSurface.tsx` into slices (§4) | L | Medium — maintainability | + +--- + +## 6. Appendix — proposed token additions + +```css +/* canvas + chrome surface ramp (lib should consume only these) */ +--st-canvas: ; /* replaces #1b1b1b pegboard */ +--st-accent: var(--primary); /* replaces blue-600/500 */ +--st-hairline: rgb(255 255 255 / 0.10); +--st-inset-hi: inset 0 1px 0 rgb(255 255 255 / 0.08); + +/* motion */ +--st-ease: cubic-bezier(0.32, 0.72, 0, 1); +--st-dur-fast: 180ms; +--st-dur: 300ms; + +/* radius scale (concentric) */ +--st-r-shell: 1.25rem; /* rounded-2xl */ +--st-r-card: 0.875rem; /* rounded-xl */ +--st-r-control: 0.5rem; /* rounded-lg */ +``` + +> Scope note: every recommendation here is **styling/architecture only** — none change the generate, +> polling, upload, bg-removal, or adapter contracts validated during the functional debug pass. diff --git a/frontend/apps/artcraft-webapp/src/app/app.tsx b/frontend/apps/artcraft-webapp/src/app/app.tsx index 2aae08da3e..4f0003483e 100644 --- a/frontend/apps/artcraft-webapp/src/app/app.tsx +++ b/frontend/apps/artcraft-webapp/src/app/app.tsx @@ -16,6 +16,7 @@ import CreateImage from "../pages/create-image"; import CreateVideo from "../pages/create-video"; import CreateVFX from "../pages/create-vfx"; import PageScene from "../pages/pagescene"; +import PageDraw from "../pages/pagedraw"; import VideoEditorPage from "../pages/video-editor"; import Pricing from "../pages/pricing"; import Support from "../pages/support/support"; @@ -149,6 +150,7 @@ export function App() { } /> } /> } /> + } /> } /> void; onRecreate?: () => void; onMakeVideo?: () => void; + onEditOnCanvas?: () => void; onDelete?: () => void; showDownloadAppCta?: boolean; } @@ -90,6 +91,7 @@ export function LightboxDetails({ onClose, onRecreate, onMakeVideo, + onEditOnCanvas, onDelete, showDownloadAppCta, }: LightboxDetailsProps) { @@ -474,6 +476,16 @@ export function LightboxDetails({ Make Video )} + {onEditOnCanvas && ( + + )} {onRecreate && promptData.hasToken && ( + + + + + { + setIsGalleryOpen(false); + setGallerySelectedIds([]); + }} + selectedItemIds={gallerySelectedIds} + onSelectItem={handleGallerySelect} + maxSelections={1} + onUseSelected={handleUseGalleryImages} + forceFilter="image" + /> + + setIsBlankCanvasOpen(false)} + onConfirm={handleBlankCanvasConfirm} + /> + + ); +} diff --git a/frontend/apps/artcraft-webapp/src/pages/pagedraw/edit-on-canvas.ts b/frontend/apps/artcraft-webapp/src/pages/pagedraw/edit-on-canvas.ts new file mode 100644 index 0000000000..efd68ed4dd --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/pagedraw/edit-on-canvas.ts @@ -0,0 +1,27 @@ +// "Edit on Canvas" handoff: open an existing image in the PageDraw editor. +// +// Mirrors the Tauri app's handleEditFromGallery — RESET the editor, seed the +// chosen image as the base (and into history), then route to /edit-image. +// Seeding the lib's store before navigating means the editor mounts straight +// into the image, same store-based handoff the create-image/create-video +// "Recreate" and "Make Video" buttons use (see lib/recreate.ts). + +import type { NavigateFunction } from "react-router-dom"; +import { useSceneStore } from "@storyteller/ui-pagedraw"; + +export function applyEditOnCanvasFromImage( + mediaToken: string, + mediaUrl: string, + navigate: NavigateFunction, +): void { + const store = useSceneStore.getState(); + const baseImage = { + url: mediaUrl, + mediaToken, + fullImageUrl: mediaUrl, + }; + store.RESET(); + store.addHistoryImageBundle({ images: [baseImage] }); + store.setBaseImageInfo(baseImage); + navigate("/edit-image"); +} diff --git a/frontend/apps/artcraft-webapp/src/pages/pagedraw/index.ts b/frontend/apps/artcraft-webapp/src/pages/pagedraw/index.ts new file mode 100644 index 0000000000..6de25f28d1 --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/pagedraw/index.ts @@ -0,0 +1 @@ +export { default } from "./pagedraw"; diff --git a/frontend/apps/artcraft-webapp/src/pages/pagedraw/job-polling.ts b/frontend/apps/artcraft-webapp/src/pages/pagedraw/job-polling.ts new file mode 100644 index 0000000000..7cb6567230 --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/pagedraw/job-polling.ts @@ -0,0 +1,141 @@ +// Job polling for the PageDraw web adapter. +// +// Mirrors the create-image / create-vfx job loops: enqueue returns an +// inference job token, then we poll `GET /v1/jobs/job/` until the +// job reaches a terminal state. On success we resolve the result media — +// preferring the batch (so multi-image generations all land) and falling +// back to the single media_links payload. +// +// Kept local to the pagedraw page (rather than imported from create-image) +// so the editor owns its own result shape and isn't coupled to that page's +// store types — same separation the artcraft-website pagedraw port used. + +import { JobsApi, MediaFilesApi } from "@storyteller/api"; + +export interface PolledImage { + media_token: string; + cdn_url: string; + maybe_thumbnail_template?: string; +} + +export type PollResult = + | { status: "pending"; images: PolledImage[] } + | { status: "complete"; images: PolledImage[] } + | { status: "failed"; images: PolledImage[]; error: string }; + +export async function pollJobOnce(jobToken: string): Promise { + const jobsApi = new JobsApi(); + const response = await jobsApi.GetJobByToken({ token: jobToken }); + + if (!response.success || !response.data) { + return { status: "pending", images: [] }; + } + + const state = response.data; + const statusStr = state.status?.status?.toLowerCase() ?? ""; + + if (statusStr === "complete_success" || statusStr === "complete") { + const result = state.maybe_result as Record | undefined; + const mediaLinks = (result as any)?.media_links; + const entityToken = (result as any)?.entity_token as string | undefined; + + if (entityToken) { + try { + const mediaApi = new MediaFilesApi(); + const mediaFile = await mediaApi.GetMediaFileByToken({ + mediaFileToken: entityToken, + }); + const batchToken = mediaFile.data?.maybe_batch_token; + if (batchToken) { + const batchResponse = await mediaApi.GetMediaFilesByBatchToken({ + batchToken, + }); + if (batchResponse.success && batchResponse.data?.length) { + const images: PolledImage[] = batchResponse.data + .map((file: any) => ({ + media_token: file.token, + cdn_url: file.media_links?.cdn_url, + maybe_thumbnail_template: + file.media_links?.maybe_thumbnail_template, + })) + .filter((img: PolledImage) => img.cdn_url); + if (images.length > 0) return { status: "complete", images }; + } + } + } catch { + // Fall through to single-image extraction. + } + } + + if (mediaLinks?.cdn_url) { + return { + status: "complete", + images: [ + { + media_token: entityToken ?? jobToken, + cdn_url: mediaLinks.cdn_url, + maybe_thumbnail_template: mediaLinks.maybe_thumbnail_template, + }, + ], + }; + } + + return { status: "complete", images: [] }; + } + + if ( + statusStr.includes("fail") || + statusStr.includes("error") || + statusStr === "dead" + ) { + return { + status: "failed", + images: [], + error: + state.status?.maybe_failure_message ?? + state.status?.maybe_extra_status_description ?? + "Generation failed", + }; + } + + return { status: "pending", images: [] }; +} + +export function startPolling( + jobToken: string, + onComplete: (images: PolledImage[]) => void, + onError: (reason: string) => void, + intervalMs = 3000, + maxAttempts = 120, +): () => void { + let attempts = 0; + let stopped = false; + + const poll = async () => { + if (stopped) return; + attempts++; + + try { + const result = await pollJobOnce(jobToken); + if (stopped) return; + + if (result.status === "complete") return onComplete(result.images); + if (result.status === "failed") return onError(result.error); + if (attempts >= maxAttempts) return onError("Generation timed out"); + + setTimeout(poll, intervalMs); + } catch { + if (!stopped && attempts < maxAttempts) { + setTimeout(poll, intervalMs * 2); + } else if (!stopped) { + onError("Network error during polling"); + } + } + }; + + setTimeout(poll, 2000); + + return () => { + stopped = true; + }; +} diff --git a/frontend/apps/artcraft-webapp/src/pages/pagedraw/pagedraw.tsx b/frontend/apps/artcraft-webapp/src/pages/pagedraw/pagedraw.tsx new file mode 100644 index 0000000000..eb9c17f92e --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/pagedraw/pagedraw.tsx @@ -0,0 +1,25 @@ +import { PageDraw as PageDrawLib } from "@storyteller/ui-pagedraw"; +import Seo from "../../components/seo"; +import { useWebPageDrawAdapter } from "./web-adapter"; + +export default function PageDraw() { + const adapter = useWebPageDrawAdapter(); + return ( + <> + + {/* PageDraw lays its canvas and toolbars out with position:fixed. + translateZ(0) makes this div their containing block so they scope + to the SidebarInset area instead of covering the whole window — + same technique the Edit 3D page uses for its lib overlays. */} +
+ +
+ + ); +} diff --git a/frontend/apps/artcraft-webapp/src/pages/pagedraw/web-adapter.tsx b/frontend/apps/artcraft-webapp/src/pages/pagedraw/web-adapter.tsx new file mode 100644 index 0000000000..ae316ce501 --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/pagedraw/web-adapter.tsx @@ -0,0 +1,275 @@ +// PageDrawAdapter implementation for the artcraft webapp. +// +// Plugs the shared @storyteller/ui-pagedraw lib into the webapp by speaking +// REST instead of Tauri commands: +// - enqueueEditImage → OmniGenApi.generateImage, then poll the job and feed +// results back into the lib's Zustand store (addHistoryImageBundle / +// resolvePendingGeneration), keyed on frontendSubscriberId. +// - enqueueBgRemoval → upload base64 → media token → POST +// /v1/generate/image/remove_background → poll → finishRemoveBackground. +// - uploadImage → UploadImageMedia (shared REST uploader). +// - renderBaseImageSelector → the webapp's own upload/library/blank picker. +// +// Inpaint is intentionally stubbed: the omni-gen REST endpoint doesn't yet +// accept a raw mask payload, so we surface a clear toast until that path +// exists. Mirrors the artcraft-website pagedraw port. + +import { useMemo } from "react"; +import { + type PageDrawAdapter, + type ImageBundle, + type BaseSelectorImage, + useSceneStore, +} from "@storyteller/ui-pagedraw"; +import { + OmniGenApi, + StorytellerApiHostStore, + UploadImageMedia, + type OmniGenImageRequest, +} from "@storyteller/api"; +import { UploaderStates } from "@storyteller/common"; +import { showToast } from "../../components/toast/toast"; +import { useInsufficientCredits } from "../../components/insufficient-credits-modal"; +import { BaseImageSelector } from "./base-image-selector"; +import { startPolling, type PolledImage } from "./job-polling"; + +// ─── Local bg-removal call ────────────────────────────────────────────────── +// Inlined (no dedicated method in @storyteller/api yet). Mirrors ApiManager's +// auth: cookies via credentials:"include", plus the signed-session header +// fallback for browsers that block third-party cookies. + +const SESSION_STORAGE_KEY = "artcraft_signed_session"; + +interface RemoveBgRequest { + uuid_idempotency_token: string; + media_file_token: string; +} + +interface RemoveBgResponse { + success: boolean; + inference_job_token: string; +} + +const sessionHeaders = (): Record => { + const headers: Record = { + Accept: "application/json", + "Content-Type": "application/json", + }; + try { + const session = localStorage.getItem(SESSION_STORAGE_KEY); + if (session) headers.session = session; + } catch { + // localStorage unavailable — rely on the cookie. + } + return headers; +}; + +const removeImageBackground = async ( + body: RemoveBgRequest, +): Promise => { + const host = StorytellerApiHostStore.getInstance().getApiSchemeAndHost(); + const response = await fetch(`${host}/v1/generate/image/remove_background`, { + method: "POST", + headers: sessionHeaders(), + credentials: "include", + body: JSON.stringify(body), + }); + if (!response.ok) { + throw new Error(`Background removal HTTP ${response.status}`); + } + return response.json(); +}; + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +const polledImagesToBundle = (images: PolledImage[]): ImageBundle => ({ + images: images.map( + (img) => + ({ + url: img.cdn_url, + mediaToken: img.media_token, + thumbnailUrlTemplate: img.maybe_thumbnail_template, + fullImageUrl: img.cdn_url, + }) as BaseSelectorImage, + ), +}); + +const startGenerationPoll = (jobToken: string, subscriberId: string) => { + startPolling( + jobToken, + (images) => { + const store = useSceneStore.getState(); + // The user may have dismissed the placeholder while the job ran. + if (!store.pendingGenerations.some((p) => p.id === subscriberId)) return; + store.addHistoryImageBundle(polledImagesToBundle(images)); + store.resolvePendingGeneration(subscriberId); + }, + (reason) => { + showToast("error", reason); + useSceneStore.getState().resolvePendingGeneration(subscriberId); + }, + ); +}; + +const base64ToBlob = (base64: string): Blob => { + const commaIdx = base64.indexOf(","); + const raw = commaIdx >= 0 ? base64.slice(commaIdx + 1) : base64; + const headerMatch = + commaIdx >= 0 ? base64.slice(0, commaIdx).match(/data:(.*?);base64/) : null; + const mime = headerMatch?.[1] ?? "image/png"; + const binary = atob(raw); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); + return new Blob([bytes], { type: mime }); +}; + +const uploadBlobAsImage = async ( + blob: Blob, + title: string, +): Promise => { + let mediaToken: string | undefined; + const file = new File([blob], `${title}.png`, { type: blob.type }); + await UploadImageMedia({ + title, + assetFile: file, + progressCallback: (state) => { + if (state.status === UploaderStates.success && state.data) { + mediaToken = state.data; + } + }, + }); + return mediaToken; +}; + +// PageDrawEditRequest.model is an ImageModel object (or possibly a raw id); +// the omni-gen REST body wants the string id. +const modelIdOf = (model: unknown): string => { + if (!model) return ""; + if (typeof model === "string") return model; + const m = model as { id?: string; tauriId?: string }; + return m.id ?? m.tauriId ?? String(model); +}; + +// ApiManager throws Error("HTTP error! status: 402") on a non-2xx response, and +// our inlined bg-removal fetch throws "Background removal HTTP 402"; the JSON +// body isn't surfaced either way. Pull the 3-digit status out of the message so +// callers can special-case 402 Payment Required. +const httpStatusFromError = (e: unknown): number | undefined => { + const message = e instanceof Error ? e.message : String(e); + const match = /(\d{3})/.exec(message); + return match ? Number(match[1]) : undefined; +}; + +// ─── Adapter ─────────────────────────────────────────────────────────────────── + +export const useWebPageDrawAdapter = (): PageDrawAdapter => { + const openInsufficientCredits = useInsufficientCredits(); + + return useMemo( + () => ({ + enqueueEditImage: async (req) => { + const refTokens = [ + ...(req.canvasImageMediaToken ? [req.canvasImageMediaToken] : []), + ...(req.imageMediaTokens ?? []), + ].filter((t): t is string => !!t); + + const body: OmniGenImageRequest = { + model: modelIdOf(req.model), + prompt: req.prompt ?? null, + idempotency_token: crypto.randomUUID(), + aspect_ratio: req.aspectRatio ?? null, + resolution: req.imageResolution ?? null, + image_batch_count: req.imageCount ?? 1, + image_media_tokens: refTokens.length ? refTokens : null, + }; + + try { + const response = await new OmniGenApi().generateImage(body); + if ( + response.success && + response.inference_job_token && + req.frontendSubscriberId + ) { + startGenerationPoll( + response.inference_job_token, + req.frontendSubscriberId, + ); + return { status: "success" }; + } + showToast("error", "Generation failed to enqueue"); + return { status: "fail" }; + } catch (e: any) { + // 402 Payment Required → the user is out of credits. Open the + // upgrade/credits modal instead of a raw error toast, matching the + // create-image / create-video generate handlers. + if (httpStatusFromError(e) === 402) { + openInsufficientCredits(); + return { status: "fail" }; + } + showToast("error", `Generation error: ${e?.message ?? e}`); + return { status: "fail" }; + } + }, + + enqueueInpaint: async (req) => { + // The omni-gen REST endpoint doesn't yet accept a raw mask payload; + // surface clearly until a multipart/mask-upload path exists. + if (!req.maskImageRawBytes) { + showToast("error", "Inpaint mask missing"); + return { status: "fail" }; + } + showToast("error", "Inpainting is not yet available on the web"); + return { status: "fail" }; + }, + + enqueueBgRemoval: async (base64Image, nodeId) => { + try { + const blob = base64ToBlob(base64Image); + const mediaToken = await uploadBlobAsImage( + blob, + `bg-removal-${nodeId}`, + ); + if (!mediaToken) { + showToast("error", "Background removal upload failed"); + return; + } + const response = await removeImageBackground({ + uuid_idempotency_token: crypto.randomUUID(), + media_file_token: mediaToken, + }); + if (!response.success || !response.inference_job_token) { + showToast("error", "Background removal failed to enqueue"); + return; + } + startPolling( + response.inference_job_token, + async (images) => { + const out = images[0]; + if (!out) return; + await useSceneStore + .getState() + .finishRemoveBackground(nodeId, out.media_token, out.cdn_url); + }, + (reason) => showToast("error", reason), + ); + } catch (e: any) { + if (httpStatusFromError(e) === 402) { + openInsufficientCredits(); + return; + } + showToast("error", `Background removal error: ${e?.message ?? e}`); + } + }, + + uploadImage: UploadImageMedia, + + renderBaseImageSelector: ({ onImageSelect, showLoading }) => ( + + ), + }), + [openInsufficientCredits], + ); +};