support arbitrary imported scenes (GLB, etc.)

packages/components/src/api/media_files/GetMedia.ts

@@ -3,10 +3,12 @@ import { UserDetailsLight } from "../_common/UserDetailsLight";
 import { MediaFileType } from "../_common/enums/MediaFileType";
 import { WeightCategory } from "../_common/enums/WeightCategory";
 import { WeightType } from "../_common/enums/WeightType";
+import { MediaFileSubtype } from "../enums/MediaFileSubtype";
 
 export interface MediaFile {
   token: string;
   media_type: MediaFileType;
+  maybe_media_subtype: MediaFileSubtype | null;
   public_bucket_path: string;
   maybe_batch_token: string;
   maybe_creator_user: UserDetailsLight | null;

packages/components/src/api/media_files/GetMediaFile.ts

@@ -3,11 +3,13 @@ import { UserDetailsLight } from "../_common/UserDetailsLight";
 import { MediaFileType } from "../_common/enums/MediaFileType";
 import { WeightCategory } from "../_common/enums/WeightCategory";
 import { WeightType } from "../_common/enums/WeightType";
+import { MediaFileSubtype } from "../enums/MediaFileSubtype";
 // import { Weight } from "../weights/GetWeight";
 
 export interface MediaFile {
   token: string;
   media_type: MediaFileType;
+  maybe_media_subtype: MediaFileSubtype | null;
   public_bucket_path: string;
   maybe_creator_user: UserDetailsLight | null;
   creator_set_visibility: string;

packages/fakeyou.com/src/components/common/Scene3D/Scene3D.tsx

@@ -8,27 +8,42 @@ import "./Scene3D.scss";
 // You can supply either an `objectId` or a `sceneMediaFileToken` when loading in studio mode.
 interface Scene3DProps {
   mode: string;
+  skybox?: string;
   fullScreen?: boolean;
   urlParams?: string;
-  objectId?: string;
-  sceneMediaFileToken?: string;
   className?: string;
   onSceneSavedCallback?: (mediaToken: string) => void;
+
+  // NB: The following parameters are mutally exclusive.
+  // We should use Typescript to enforce this.
+  objectId?: string; // built-in objects
+  sceneMediaFileToken?: string; // scn.ron native storyteller engine scenes
+  mixamoUrl?: string; // a mixamo animation
+  bvhUrl?: string; // a MocapNet animation in BVH format
+  sceneImportUrl?: string; // a generic (non-storyteller) scene: GLB, GLTF, etc.
 }
 
 export default function Scene3D({
   mode,
+  skybox,
   fullScreen = false,
-  objectId,
-  sceneMediaFileToken,
-  urlParams,
   className,
   onSceneSavedCallback,
+
+  // NB: The following parameters are mutally exclusive.
+  // We should use Typescript to enforce this.
+  objectId,
+  sceneMediaFileToken,
+  mixamoUrl,
+  bvhUrl,
+  sceneImportUrl,
+
 }: Scene3DProps) {
   const engineBaseUrl = "https://engine.fakeyou.com";
 
   const iframeRef = useRef<HTMLIFrameElement>(null);
   
+
   const onMessage = useCallback((event: MessageEvent) => {
     console.log("engine message received", event.data, event);
 
@@ -43,7 +58,8 @@ export default function Scene3D({
 
       console.log("studio-ready message (2)");
 
-      studio.postMessage("save-scene", engineBaseUrl);
+      // NB: Example of how to call the API in the other direction:
+      //  studio.postMessage("save-scene", engineBaseUrl);
     } else if (
       typeof event.data === "string"
       && event.data.startsWith("scene-saved:")
@@ -58,57 +74,50 @@ export default function Scene3D({
     } else if (event.data === "scene-save-failed") {
       console.error("Failed to save the scene!");
     }
-  }, []);
+  }, [onSceneSavedCallback]);
 
 
   useEffect(() => {
-    /*
-    if (iframeRef && iframeRef.current) {
-      console.log("installing event listener for messages");
-      
-      // NB: Compiler complains about ref going out of scope before destructor is called.
-      let ref = iframeRef.current;
-
-      // TODO(bt,2024-03-08): Fix typescript 'any' hack
-      (ref as any).addEventListener("message", onMessage);
-
-      return () => {
-        console.log("uninstalling event listener for messages");
-
-        // TODO(bt,2024-03-08): Fix typescript 'any' hack
-        (ref as any).removeEventListener("message", onMessage);
-      }
-    }
-    */
-
-    console.log("installing event listener for messages");
     window.addEventListener("message", onMessage, false);
-
     return () => {
-      console.log("uninstalling event listener for messages");
       window.removeEventListener("message", onMessage, false);
     }
   }, [onMessage]);
-  //}, [onMessage, iframeRef]);
 
 
   let engineUrl = `${engineBaseUrl}/?mode=${mode}`;
 
+  if (skybox) {
+    engineUrl += `&skybox=${skybox}`;
+  }
+
   if (sceneMediaFileToken) {
+    // NB: Storyteller Engine makes the API call to load the scene.
+    // We don't need to pass the bucket path.
+    // The engine, does, however, need a `.scn.ron` file extension.
     const sceneUrlRef = `remote://${sceneMediaFileToken}.scn.ron`;
     engineUrl += `&scene=${sceneUrlRef}`;
   } else if (objectId) {
+    // NB: This is an engine built-in, eg. `couch.gltf` or `sample-room.gltf`.
     engineUrl += `&objectId=${objectId}`;
+  } else if (mixamoUrl) {
+    // NB: This should be a full bucket path to an asset.
+    engineUrl += `&mixamo=${mixamoUrl}`;
+  } else if (bvhUrl) {
+    // NB: This should be a full bucket path to an asset.
+    engineUrl += `&bvh=${bvhUrl}`;
+  } else if (sceneImportUrl) {
+    // TODO: Not sure what the format of this should be.
+    engineUrl += `&sceneImport=${sceneImportUrl}`;
   }
 
-  console.log('installing iframe engine and callbacks');
-
   return (
     <div
       className={`${
         fullScreen ? "fy-scene-3d-fullscreen" : "fy-scene-3d-default"
       } ${className ? className : ""}`.trim()}
     >
+      {/* IframeResizer was causing some glitches passing messages. Need to test more. */}
       {/*<IframeResizer src={engineUrl} width="100%" height="100%" id="" />*/}
       <iframe
         title="Storyteller Engine"

packages/fakeyou.com/src/v2/view/pages/media/MediaPage.tsx

@@ -34,18 +34,11 @@ import BookmarkButton from "components/common/BookmarkButton";
 import LikeButton from "components/common/LikeButton";
 import { useBookmarks, useMedia, useRatings, useSession } from "hooks";
 import { WeightCategory } from "@storyteller/components/src/api/_common/enums/WeightCategory";
-import Iframe from "react-iframe";
+//import Iframe from "react-iframe";
 import SdBatchMediaPanel from "./components/SdBatchMediaPanel/SdBatchMediaPanel";
 import { GetMediaBatchImages } from "@storyteller/components/src/api/media_files/GetMediaBatchImages";
 import { mediaTypeLabels } from "utils/mediaTypeLabels";
-
-// Storyteller Engine parameters
-// These are documented here:
-// https://www.notion.so/storytellerai/Studio-Iframe-Query-Params-a748a9929ec3404780c3884e7fb89bdb
-const SKYBOX = "333348"; // Looks good (lighter)
-//const SKYBOX = "242433"; // Looks good
-//const SKYBOX = "1a1a27"; // Too dark
-//const SKYBOX = "3f3f55"; // too light
+import { EngineMediaPanel } from "./components/EngineMediaPanel/EngineMediaPanel";
 
 export default function MediaPage() {
   const { canEditTtsModel, user } = useSession();
@@ -230,6 +223,16 @@ export default function MediaPage() {
             {promptSection}
           </>
         );
+      case MediaFileType.BVH:
+      case MediaFileType.GLB:
+      case MediaFileType.GLTF:
+      case MediaFileType.SceneRon:
+        return (
+          <>
+            <EngineMediaPanel mediaFile={mediaFile} />
+          </>
+        )
+      /*
       case MediaFileType.BVH:
         const bvhUrl = bucketConfig.getGcsUrl(mediaFile.public_bucket_path);
         return (
@@ -274,6 +277,7 @@ export default function MediaPage() {
             }}
           />
         );
+      */
       case MediaFileType.FBX:
         return (
           <Panel padding={true}>

packages/fakeyou.com/src/v2/view/pages/media/components/EngineMediaPanel/EngineMediaPanel.tsx

@@ -0,0 +1,94 @@
+import React from "react";
+import { MediaFile } from "@storyteller/components/src/api/media_files/GetMediaFile";
+import { BucketConfig } from "@storyteller/components/src/api/BucketConfig";
+//import Iframe from "react-iframe";
+import { MediaFileType } from "@storyteller/components/src/api/_common/enums/MediaFileType";
+import { Scene3D } from "components/common";
+import { MediaFileSubtype } from "@storyteller/components/src/api/enums/MediaFileSubtype";
+
+// Storyteller Engine parameters
+// These are documented here:
+// https://www.notion.so/storytellerai/Studio-Iframe-Query-Params-a748a9929ec3404780c3884e7fb89bdb
+const SKYBOX = "333348"; // Looks good (lighter)
+//const SKYBOX = "242433"; // Looks good
+//const SKYBOX = "1a1a27"; // Too dark
+//const SKYBOX = "3f3f55"; // too light
+
+export interface EngineMediaPanelArgs {
+  mediaFile: MediaFile,
+}
+
+export function EngineMediaPanel({ mediaFile } : EngineMediaPanelArgs) {
+  const assetUrl = new BucketConfig().getGcsUrl(mediaFile.public_bucket_path);
+
+  let engineParams = {};
+
+  switch (mediaFile.maybe_media_subtype) {
+    case MediaFileSubtype.StorytellerScene:
+      engineParams = {
+        sceneMediaFileToken: mediaFile.token,
+      };
+      break;
+    case MediaFileSubtype.Mixamo:
+      engineParams = {
+        mixamoUrl: assetUrl,
+      };
+      break;
+    case MediaFileSubtype.MocapNet:
+      engineParams = {
+        bvhUrl: assetUrl,
+      };
+      break;
+    case MediaFileSubtype.AnimationOnly:
+    case MediaFileSubtype.Scene:
+    case MediaFileSubtype.SceneImport:
+    default:
+      // This should hypothetically be any type of generic scene asset
+      engineParams = {
+        sceneImportUrl: assetUrl,
+      };
+      break;
+  }
+
+  if (Object.keys(engineParams).length === 0) {
+    // Support for files without a media subtype
+    if (assetUrl.endsWith("bvh")) {
+      // Assume MocapNet. 
+      engineParams = {
+        bvhUrl: assetUrl,
+      };
+    }
+    // TODO(bt,2024-03-11): Figure this out
+    //else if (assetUrl.endsWith("glb")) {
+    //  // Assume MocapNet. 
+    //  engineParams = {
+    //    bvhUrl: assetUrl,
+    //  };
+    //}
+  }
+
+  if (mediaFile.media_type === MediaFileType.SceneRon) {
+    // This will always be a storyteller scene
+    engineParams = {
+      sceneMediaFileToken: mediaFile.token,
+    };
+  } 
+
+  return (
+    /*<Iframe
+      {...{
+        url: `https://engine.fakeyou.com?mode=viewer&bvh=${assetUrl}&skybox=${SKYBOX}`,
+        className: "fy-studio-frame",
+      }}
+    />*/
+    <Scene3D
+      mode="viewer"
+      skybox={SKYBOX}
+      fullScreen={false}
+      //className="flex-grow-1"
+      className="fy-studio-frame"
+      //onSceneSavedCallback={onSaveCallback}
+      {...engineParams}
+      />
+  );
+}