Test a local splat demo

frontend/apps/editor3d/app/src/components/reusable/UploadModal3D/utilities/loadPreviewOnCanvas.ts

@@ -2,6 +2,8 @@ import * as THREE from "three";
 import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js";
 import { FontLoader } from "three/addons/loaders/FontLoader.js";
 import { TextGeometry } from "three/addons/geometries/TextGeometry.js";
+import { SplatFileType, SplatMesh } from "@sparkjsdev/spark";
+import { SPLAT_FILE_TYPE } from "~/enums";
 
 interface LoaderInterface {
   file: File;
@@ -32,10 +34,13 @@ export const loadPreviewOnCanvas = ({
   const camera = new THREE.PerspectiveCamera(35, aspectRatio, 0.1, 1000);
   camera.position.z = 2;
 
+  const gl2ctx = canvas.getContext("webgl2");
+
   const renderer = new THREE.WebGLRenderer({
     alpha: true,
     antialias: true,
     canvas: canvas,
+    context: gl2ctx!,
     preserveDrawingBuffer: true,
   });
 
@@ -52,18 +57,30 @@ export const loadPreviewOnCanvas = ({
   scene.add(light);
 
   // load the file into the preview mini-scene depending of the file's type
-  if (file.name.includes(".glb")) {
+  if (file.name.endsWith(".glb")) {
     glbLoader({ file, scene, camera, renderer, statusCallback });
-  } else if (file.name.includes(".pmd")) {
+  } else if (file.name.endsWith(".pmd")) {
     pmdLoader({ file, scene, camera, renderer, statusCallback });
   } else if (
-    file.name.includes(".png") ||
-    file.name.includes(".jpg") ||
-    file.name.includes(".jpeg") ||
-    file.name.includes(".gif")
+    file.name.endsWith(".png") ||
+    file.name.endsWith(".jpg") ||
+    file.name.endsWith(".jpeg") ||
+    file.name.endsWith(".gif")
   ) {
     imagePlaneLoader({ file, scene, camera, renderer, statusCallback });
-  } else if (file.name.includes(".vmd")) {
+  } else if (file.name.endsWith(".spz")) {
+    file.arrayBuffer().then((arrayBuffer) => {
+      const splatMesh = new SplatMesh({
+        fileBytes: arrayBuffer, fileType: SplatFileType.SPZ,
+        onLoad: () => { scene.add(splatMesh) }
+      });
+    }).catch((loaderError) => {
+      statusCallback({
+        type: "SPLAT Loader Error",
+        message: String(loaderError),
+      });
+    });
+  } else if (file.name.endsWith(".vmd")) {
     statusCallback({
       type: "Preview Error",
       message: "Sorry, Preview is not available to VMD files yet",

frontend/apps/editor3d/app/src/components/reusable/UploadModalSplat/UploadFilesSplat.tsx

@@ -0,0 +1,188 @@
+import { useCallback, useEffect, useState, useRef } from "react";
+import { ListDropdown } from "@storyteller/ui-list-dropdown";
+import { Button } from "@storyteller/ui-button";
+import { FileUploader } from "@storyteller/ui-file-uploader";
+import { UploaderState } from "../../../models";
+import { FilterEngineCategories, MediaFileAnimationType, UploaderStates } from "../../../enums";
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { faCube } from "@fortawesome/pro-solid-svg-icons";
+import { loadPreviewOnCanvas, snapshotCanvasAsThumbnail } from "../UploadModal3D/utilities";
+
+interface Props {
+  title: string;
+  fileTypes: string[];
+  engineCategory: FilterEngineCategories;
+  options?: {
+    fileSubtypes?: { [key: string]: string }[];
+    hasLength?: boolean;
+    hasThumbnailUpload?: boolean;
+  };
+  onClose: () => void;
+  onUploadProgress: (newState: UploaderState) => void;
+  onLocalBytes: (bytes: ArrayBuffer) => void;
+}
+
+export const UploadFilesSplat = ({
+  fileTypes,
+  engineCategory,
+  options,
+  onClose,
+  onUploadProgress,
+  onLocalBytes,
+}: Props) => {
+  const canvasRef = useRef<HTMLCanvasElement | undefined>(undefined);
+  const canvasCallbackRef = useCallback((node: HTMLCanvasElement) => {
+    if (node !== null) {
+      canvasRef.current = node;
+    }
+  }, []);
+
+  const fileSubtypes = options?.fileSubtypes;
+  // const hasLength = options?.hasLength;
+  // const hasThumbnailUpload = options?.hasThumbnailUpload;
+
+  const [subtype, setSubtype] = useState<MediaFileAnimationType | undefined>(
+    fileSubtypes
+      ? (Object.values(fileSubtypes[0])[0] as MediaFileAnimationType)
+      : undefined,
+  );
+
+  const [assetFile, setAssetFile] = useState<{
+    value: File | null;
+    error?: string;
+  }>({ value: null });
+
+  const [previewStatus, setPreviewStatus] = useState<{
+    type: string;
+    message?: string;
+  }>({ type: "init" });
+
+  const [thumbnailFile, setThumbnailFile] = useState<Blob | undefined>(
+    undefined,
+  );
+
+  const handleSubmit = () => {
+    if (!assetFile.value) {
+      setAssetFile((curr) => ({
+        ...curr,
+        error: "Please select a file to upload.",
+      }));
+      return;
+    }
+
+    // Use the filename as the title
+    const title = assetFile.value.name.split(".")[0];
+
+    assetFile.value.arrayBuffer().then((arrayBuffer) => {
+      onLocalBytes(arrayBuffer);
+    }).catch((error) => {
+      console.error("Error reading file as ArrayBuffer:", error);
+    });
+
+    // upload3DObjects({
+    //   title: title,
+    //   assetFile: assetFile.value,
+    //   thumbnailSnapshot: thumbnailFile,
+    //   engineCategory: engineCategory,
+    //   animationType: subtype,
+    //   progressCallback: onUploadProgress,
+    // });
+  };
+
+  useEffect(() => {
+    // Reset subtype when fileSubtypes set changes (e.g., toggling character)
+    if (!fileSubtypes || fileSubtypes.length === 0) {
+      setSubtype(undefined);
+      return;
+    }
+    const nextDefault = Object.values(fileSubtypes[0])[0] as
+      | MediaFileAnimationType
+      | undefined;
+    setSubtype(nextDefault);
+  }, [fileSubtypes]);
+
+  useEffect(() => {
+    if (canvasRef.current && assetFile.value) {
+      loadPreviewOnCanvas({
+        file: assetFile.value,
+        canvas: canvasRef.current,
+        statusCallback: (statusObject: { type: string; message?: string }) => {
+          setPreviewStatus(statusObject);
+        },
+      });
+    }
+  }, [assetFile.value]);
+
+  useEffect(() => {
+    if (previewStatus.type === "OK" && canvasRef.current) {
+      snapshotCanvasAsThumbnail({
+        targetNode: canvasRef.current!,
+        resultCallback: (snapshotBlob) => {
+          if (snapshotBlob) {
+            setThumbnailFile(snapshotBlob);
+          }
+        },
+      });
+    }
+  }, [previewStatus]);
+
+  return (
+    <>
+      <div className="flex flex-col gap-3">
+        <FileUploader
+          fileTypes={fileTypes}
+          file={assetFile.value ?? undefined}
+          handleChange={(file: File) => {
+            setAssetFile({
+              value: file,
+            });
+          }}
+        />
+        {fileSubtypes && fileSubtypes.length > 1 && (
+          <ListDropdown
+            list={fileSubtypes}
+            onSelect={(value) => setSubtype(value as MediaFileAnimationType)}
+          />
+        )}
+        {assetFile.error && (
+          <h6 className="z-10 text-red">{assetFile.error}</h6>
+        )}
+
+        <div className="relative m-auto w-full overflow-hidden rounded-lg bg-brand-secondary">
+          <canvas
+            className="pointer-events-none h-full w-full"
+            ref={canvasCallbackRef}
+          />
+          {!assetFile.value && (
+            <h6 className="pointer-events-auto absolute left-0 top-1/2 -mt-5 flex w-full items-center justify-center gap-2.5 text-center opacity-50">
+              <FontAwesomeIcon icon={faCube} />
+              Your model preview will appear here
+            </h6>
+          )}
+          {previewStatus.type.includes("Error") && (
+            <>
+              <h6 className="pointer-events-auto absolute left-0 top-1/2 -mt-5 w-full text-center">
+                {previewStatus.type}
+                {previewStatus.message && <br />}
+                {previewStatus.message}
+              </h6>
+            </>
+          )}
+        </div>
+
+        <div className="flex justify-end gap-2">
+          <Button variant="secondary" onClick={onClose}>
+            Cancel
+          </Button>
+          <Button
+            variant="primary"
+            onClick={handleSubmit}
+            disabled={!assetFile.value}
+          >
+            Upload
+          </Button>
+        </div>
+      </div>
+    </>
+  );
+};

frontend/apps/editor3d/app/src/components/reusable/UploadModalSplat/UploadModalSplat.tsx

@@ -0,0 +1,143 @@
+import { useEffect, useMemo, useState } from "react";
+import { LoadingDots } from "@storyteller/ui-loading";
+import { Modal } from "@storyteller/ui-modal";
+import { UploadAssetError, UploadSuccess } from "@storyteller/ui-upload-modal";
+import { initialUploaderState, UploaderState } from "../../../models";
+import {
+  FilterEngineCategories,
+  UploaderStates,
+  MediaFileAnimationType,
+  SPLAT_FILE_TYPE,
+} from "../../../enums";
+import { IconDefinition } from "@fortawesome/fontawesome-svg-core";
+import {
+  galleryModalVisibleViewMode,
+  galleryModalVisibleDuringDrag,
+} from "@storyteller/ui-gallery-modal";
+import { UploadFilesSplat } from "./UploadFilesSplat";
+
+interface Props {
+  onClose: () => void;
+  onSuccess: (splatArrayBuffer: ArrayBuffer) => void;
+  isOpen: boolean;
+  title: string;
+  titleIcon: IconDefinition;
+  options?: {
+    fileSubtypes?: { [key: string]: string }[];
+    hasLength?: boolean;
+    hasThumbnailUpload?: boolean;
+  };
+}
+
+const splatFileTypes = Object.values(SPLAT_FILE_TYPE);
+
+export function UploadModalSplat(props: Props) {
+  const selectedCategory = FilterEngineCategories.SPLAT;
+  const { isOpen, onClose, onSuccess, title, titleIcon, options } = props;
+  const [uploaderState, setUploaderState] =
+    useState<UploaderState>(initialUploaderState);
+
+  const updateUploaderState = (newState: UploaderState) => {
+    setUploaderState(newState);
+  };
+
+  const resetModalState = () => {
+    setUploaderState(initialUploaderState);
+  };
+
+  useEffect(() => {
+    if (isOpen) {
+      resetModalState();
+    }
+  }, [isOpen]);
+
+  useEffect(() => {
+    if (uploaderState.status === UploaderStates.success) {
+      // Automatically open the global Gallery modal after a successful upload
+      // galleryModalVisibleViewMode.value = true;
+      // galleryModalVisibleDuringDrag.value = true;
+
+      // onSuccess(selectedCategory);
+      onClose();
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [uploaderState.status]);
+
+  const UploaderModalContent = () => {
+    switch (uploaderState.status) {
+      case UploaderStates.ready:
+        return (
+          <div className="space-y-4">
+            <UploadFilesSplat
+              title={title}
+              engineCategory={selectedCategory}
+              fileTypes={splatFileTypes}
+              options={{
+                ...(options ?? {}),
+              }}
+              onClose={onClose}
+              onUploadProgress={updateUploaderState}
+              onLocalBytes={(buffer) => { onSuccess(buffer); }}
+            />
+          </div>
+        );
+      case UploaderStates.uploadingAsset:
+      case UploaderStates.uploadingCover:
+      case UploaderStates.settingCover:
+        return (
+          <>
+            <LoadingDots className="mb-1 bg-transparent" />
+            <div className="w-100 text-center opacity-50">Uploading...</div>
+          </>
+        );
+      case UploaderStates.success: {
+        return (
+          <UploadSuccess
+            title="Splat"
+            onOk={() => {
+              onClose();
+              // onSuccess(selectedCategory);
+            }}
+          />
+        );
+      }
+      case UploaderStates.assetError:
+        return (
+          <UploadAssetError
+            onCancel={onClose}
+            onRetry={() => {
+              resetModalState();
+            }}
+            type={selectedCategory}
+            errorMessage={uploaderState.errorMessage}
+          />
+        );
+      case UploaderStates.coverCreateError:
+      case UploaderStates.coverSetError:
+        return (
+          <UploadAssetError
+            onCancel={onClose}
+            onRetry={() => {
+              resetModalState();
+            }}
+            type={"Thumbnail"}
+            errorMessage={uploaderState.errorMessage}
+          />
+        );
+    }
+    return undefined;
+  };
+
+  return (
+    <Modal
+      isOpen={isOpen}
+      onClose={onClose}
+      titleIcon={titleIcon}
+      title={title}
+      className="max-w-xl"
+      showClose={true}
+    >
+      <UploaderModalContent />
+    </Modal>
+  );
+}

frontend/apps/editor3d/app/src/components/reusable/UploadModalSplat/index.ts

@@ -0,0 +1 @@
+export * from "./UploadModalSplat";

frontend/apps/editor3d/app/src/components/reusable/index.ts

@@ -1,2 +1,3 @@
 export * from "./UploadModal3D";
 export * from "./UploadModalMedia";
+export * from "./UploadModalSplat";

frontend/apps/editor3d/app/src/enums/FileTypes.ts

@@ -42,6 +42,10 @@ export enum OBJECT_FILE_TYPE {
   GLB = "glb",
 }
 
+export enum SPLAT_FILE_TYPE {
+  SPZ = "spz",
+}
+
 export enum CHARACTER_MMD_FILE_TYPE {
   PMD = "pmd",
   PMX = "pmx",

frontend/apps/editor3d/app/src/enums/QueryFilters.ts

@@ -23,6 +23,7 @@ export enum FilterEngineCategories {
   IMAGE_PLANE = "image_plane",
   LOCATION = "location",
   OBJECT = "object",
+  SPLAT = "splat",
   SCENE = "scene",
   SET_DRESSING = "set_dressing",
   SKYBOX = "skybox",

frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/timeline.ts

@@ -24,6 +24,7 @@ import { Vector3 } from "three";
 
 import { filmLength, outlinerState } from "../signals";
 import { CharacterAnimationEngine } from "./Engines/CharacterAnimationEngine";
+import { SplatMesh } from "@sparkjsdev/spark";
 
 export class TimeLine {
   editorEngine: Editor;
@@ -399,6 +400,13 @@ export class TimeLine {
     return obj;
   }
 
+  // This method is for local dev testing of objects
+  // and bypasses the queue system, that's kinda redundant anyway
+  public addLocalSplat(fileBytes: ArrayBuffer | Uint8Array) {
+    const splatMesh = new SplatMesh({ fileBytes: fileBytes });
+    this.editorEngine.sceneManager?.scene.scene.add(splatMesh);
+  }
+
   public async addShape(data: MediaItem) {
     const pos = this.getPos();
     const parim = await this.editorEngine.create_parim(data.media_id, pos);

frontend/apps/editor3d/app/src/pages/PageEnigma/comps/Controls3D/Controls3D.tsx

@@ -45,6 +45,7 @@ import {
 } from "@storyteller/ui-gallery-modal";
 import { twMerge } from "tailwind-merge";
 import { UploadModalImage } from "../../../../components/reusable/UploadModalImage";
+import { UploadModalSplat } from "~/components/reusable/UploadModalSplat";
 
 export const Controls3D = () => {
   useSignals();
@@ -55,6 +56,7 @@ export const Controls3D = () => {
   const [upload3DIsShowing, setUpload3DIsShowing] = useState(false);
   const [isAddAssetPopoverOpen, setIsAddAssetPopoverOpen] = useState(false);
   const [uploadImageIsShowing, setUploadImageIsShowing] = useState(false);
+  const [uploadSplatIsShowing, setUploadSplatIsShowing] = useState(false);
 
   // Track processed 3D models by their media token to prevent duplicates
   const processedModelsRef = useRef<Record<string, boolean>>({});
@@ -68,7 +70,8 @@ export const Controls3D = () => {
         !galleryModalVisibleViewMode.value &&
         !isAddAssetPopoverOpen &&
         !upload3DIsShowing &&
-        !uploadImageIsShowing;
+        !uploadImageIsShowing &&
+        !uploadSplatIsShowing;
 
       setShowEmptySceneTooltip(isSceneEmpty);
     };
@@ -89,6 +92,7 @@ export const Controls3D = () => {
     isAddAssetPopoverOpen,
     upload3DIsShowing,
     uploadImageIsShowing,
+    uploadSplatIsShowing,
   ]);
 
   const handleModeChange = (value: string) => {
@@ -210,6 +214,9 @@ export const Controls3D = () => {
       case "upload-image":
         setUploadImageIsShowing(true);
         break;
+      case "upload-splat":
+        setUploadSplatIsShowing(true);
+        break;
       default:
         break;
     }
@@ -324,14 +331,24 @@ export const Controls3D = () => {
                         ),
                         action: "upload-image",
                       },
+                      {
+                        label: "Upload Splat",
+                        selected: false,
+                        icon: (
+                          <FontAwesomeIcon
+                            icon={faArrowUpFromBracket}
+                            className="h-4 w-4"
+                          />
+                        ),
+                        action: "upload-splat",
+                      },
                     ]}
                     onPanelAction={handleAddAssetAction}
                     showIconsInList
-                    buttonClassName={`h-9 w-9 rounded-[10px] text-lg ${
-                      showEmptySceneTooltip
-                        ? "bg-primary/90 hover:bg-primary/70"
-                        : "border-transparent bg-primary/90 hover:bg-primary/70"
-                    }`}
+                    buttonClassName={`h-9 w-9 rounded-[10px] text-lg ${showEmptySceneTooltip
+                      ? "bg-primary/90 hover:bg-primary/70"
+                      : "border-transparent bg-primary/90 hover:bg-primary/70"
+                      }`}
                     triggerIcon={
                       <FontAwesomeIcon icon={faPlus} className="text-xl" />
                     }
@@ -389,6 +406,17 @@ export const Controls3D = () => {
         title="Upload an Image"
         titleIcon={faImages}
       />
+
+      <UploadModalSplat
+        isOpen={uploadSplatIsShowing}
+        onClose={() => setUploadSplatIsShowing(false)}
+        onSuccess={(buffer) => {
+          setUploadSplatIsShowing(false)
+          editorEngine?.timeline.addLocalSplat(buffer);
+        }}
+        title="Upload an spz file"
+        titleIcon={faCube}
+      />
     </>
   );
 };

frontend/apps/editor3d/package.json

@@ -41,6 +41,7 @@
     "@remix-run/react": "^2.8.1",
     "@remix-run/router": "^1.15.3",
     "@remix-run/serve": "^2.8.1",
+    "@sparkjsdev/spark": "^0.1.10",
     "@tauri-apps/plugin-http": "^2.4.3",
     "@tweenjs/tween.js": "^18.6.4",
     "@types/lodash": "^4.17.6",

frontend/package.json

@@ -17,6 +17,7 @@
     "@preact/signals-react": "^3.0.1",
     "@radix-ui/react-dialog": "^1.1.14",
     "@remix-run/serve": "^2.15.0",
+    "@sparkjsdev/spark": "^0.1.10",
     "dayjs": "^1.11.10",
     "detect-gpu": "^5.0.38",
     "embla-carousel-react": "^8.6.0",