From a803bfb79b390eddc5c344396bca399cea9ecf20 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 11 Mar 2024 21:02:16 -0400 Subject: [PATCH 1/4] add unit-tested way to construct Engine URLs --- .../components/common/Scene3D/EngineMode.ts | 5 + .../common/Scene3D/GetEngineUrl.test.ts | 119 ++++++++++++++++++ .../components/common/Scene3D/GetEngineUrl.ts | 87 +++++++++++++ 3 files changed, 211 insertions(+) create mode 100644 packages/fakeyou.com/src/components/common/Scene3D/EngineMode.ts create mode 100644 packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.test.ts create mode 100644 packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.ts diff --git a/packages/fakeyou.com/src/components/common/Scene3D/EngineMode.ts b/packages/fakeyou.com/src/components/common/Scene3D/EngineMode.ts new file mode 100644 index 0000000000..d24186fb7e --- /dev/null +++ b/packages/fakeyou.com/src/components/common/Scene3D/EngineMode.ts @@ -0,0 +1,5 @@ + +export enum EngineMode { + Viewer = "viewer", + Studio = "studio", +} diff --git a/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.test.ts b/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.test.ts new file mode 100644 index 0000000000..050a12bae0 --- /dev/null +++ b/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.test.ts @@ -0,0 +1,119 @@ +import { MediaFileType } from "@storyteller/components/src/api/_common/enums/MediaFileType"; +import { EngineMode } from "./EngineMode"; +import { GetEngineUrl } from "./GetEngineUrl"; +import { WeightCategory } from "@storyteller/components/src/api/_common/enums/WeightCategory"; +import { WeightType } from "@storyteller/components/src/api/_common/enums/WeightType"; +import { MediaFileSubtype } from "@storyteller/components/src/api/enums/MediaFileSubtype"; + +describe('mode', () => { + test('studio', () => { + const url = GetEngineUrl({mode: EngineMode.Studio, load: { objectId: "foo" } }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&objectId=foo"); + }); + + test('viewer', () => { + const url = GetEngineUrl({mode: EngineMode.Viewer, load: { objectId: "foo" } }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=viewer&objectId=foo"); + }); +}); + +describe('skybox', () => { + test('from named skybox', () => { + const url = GetEngineUrl({mode: EngineMode.Studio, load: { objectId: "foo" }, skybox: "gum_trees_4k" }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&skybox=gum_trees_4k&objectId=foo"); + }); + + test('from hex color', () => { + const url = GetEngineUrl({mode: EngineMode.Studio, load: { objectId: "foo" }, skybox: "ff0000" }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&skybox=ff0000&objectId=foo"); + }); +}); + +describe('object ids', () => { + test('object id urls should work', () => { + const url = GetEngineUrl({mode: EngineMode.Studio, load: { objectId: "foo" } }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&objectId=foo"); + }); +}); + +// TODO(bt,2024-03-11): We're only testing one type of media file +describe('media files', () => { + let mediaFile = { + token: "MEDIA_FILE_TOKEN", + media_type: MediaFileType.GLB, + maybe_media_subtype: null, + public_bucket_path: "path/to/file", + maybe_batch_token: "BATCH_TOKEN", + created_at: new Date(), + updated_at: new Date(), + maybe_creator_user: null, + creator_set_visibility: "public", + // TODO(bt,2024-03-11): Make optional + maybe_model_weight_info: { + title: "title", + weight_token: "WEIGHT_TOKEN", + weight_category: WeightCategory.SD, + weight_type: WeightType.HIFIGAN_TT2, + maybe_cover_image_public_bucket_path: "image", + maybe_weight_creator: { + user_token: "USER_TOKEN", + username: "username", + display_name: "display_name", + gravatar_hash: "foo", + default_avatar: { + image_index: 1, + color_index: 2, + } + }, + } + }; + + describe('scene files', () => { + test('from media type', () => { + mediaFile.media_type = MediaFileType.SceneRon; + + // NB: Not the real subtype; forcing test to act on type. + mediaFile.maybe_media_subtype = null; + + const url = GetEngineUrl({mode: EngineMode.Studio, load: mediaFile }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&scene=remote://MEDIA_FILE_TOKEN.scn.ron"); + }); + + test('from media subtype', () => { + // NB: Not the real time; forcing test to act on subtype. + mediaFile.media_type = MediaFileType.Audio; + + // TODO(bt,2024-03-11): Why does the IDE complain here? + mediaFile.maybe_media_subtype = MediaFileSubtype.StorytellerScene as any; + + const url = GetEngineUrl({mode: EngineMode.Studio, load: mediaFile }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&scene=remote://MEDIA_FILE_TOKEN.scn.ron"); + }); + }); + + describe('mixamo animations', () => { + test('from media subtype', () => { + // NB: Not the real time; forcing test to act on subtype. + mediaFile.media_type = MediaFileType.Audio; + + // TODO(bt,2024-03-11): Why does the IDE complain here? + mediaFile.maybe_media_subtype = MediaFileSubtype.Mixamo as any; + + const url = GetEngineUrl({mode: EngineMode.Studio, load: mediaFile }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&mixamo=https://storage.googleapis.com/dev-vocodes-public/path/to/file"); + }); + }); + + describe('bvh animations', () => { + test('from media subtype', () => { + // NB: Not the real time; forcing test to act on subtype. + mediaFile.media_type = MediaFileType.Audio; + + // TODO(bt,2024-03-11): Why does the IDE complain here? + mediaFile.maybe_media_subtype = MediaFileSubtype.MocapNet as any; + + const url = GetEngineUrl({mode: EngineMode.Studio, load: mediaFile }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&bvh=https://storage.googleapis.com/dev-vocodes-public/path/to/file"); + }); + }); +}); diff --git a/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.ts b/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.ts new file mode 100644 index 0000000000..eb0d167663 --- /dev/null +++ b/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.ts @@ -0,0 +1,87 @@ +import { MediaFile as MediaFileOne } from "@storyteller/components/src/api/media_files/GetMedia"; +import { MediaFile as MediaFileTwo } from "@storyteller/components/src/api/media_files/GetMediaFile"; +import { EngineMode } from "./EngineMode"; +import { MediaFileSubtype } from "@storyteller/components/src/api/enums/MediaFileSubtype"; +import { MediaFileType } from "@storyteller/components/src/api/_common/enums/MediaFileType"; +import { BucketConfig } from "@storyteller/components/src/api/BucketConfig"; + +// Storyteller Engine parameters +// These are documented here: +// https://www.notion.so/storytellerai/Studio-Iframe-Query-Params-a748a9929ec3404780c3884e7fb89bdb + +interface GetEngineUrlArgs { + mode: EngineMode, + + // Polymorphic type of asset we're loading into the engine. + load: LoadableAsset + + // Optional skybox color (hex) or name + // - Predefined skyboxes, eg. `gum_trees_4k`, `kloofendal_28d_misty_4k`, `meadow_4k`, `promenade_de_vidy_4k`, `scythian_tombs_4k` + // - Hex colors, eg. `ff0000` or `1a1a27`. + skybox?: string, +} + +// Support multiple "Media File" API response payloads. +type MediaFile = MediaFileOne | MediaFileTwo ; + +// Used to load raw built-in objects into the engine. +interface EngineObject { + objectId: string; +} + +type LoadableAsset = MediaFile | EngineObject; + +const ENGINE_BASE_URL = "https://engine.fakeyou.com"; + +export function GetEngineUrl(args: GetEngineUrlArgs) : string { + let engineUrl = `${ENGINE_BASE_URL}/?mode=${args.mode}`; + + + if (args.skybox) { + engineUrl += `&skybox=${args.skybox}`; + } + + if (isMediaFile(args.load)) { + + const bucketAssetUrl = new BucketConfig().getGcsUrl(args.load.public_bucket_path); + + if ( + args.load.maybe_media_subtype === MediaFileSubtype.StorytellerScene || + args.load.media_type === MediaFileType.SceneRon + ) { + // Storyteller Engine Scenes + // NB: Storyteller Engine makes the API call to load the scene. + // We don't need to pass the bucket path. + // The engine, does, however, need a `.scn.ron` file extension. + const sceneUrlRef = `remote://${args.load.token}.scn.ron`; + engineUrl += `&scene=${sceneUrlRef}`; + } else if ( + args.load.maybe_media_subtype === MediaFileSubtype.Mixamo + ) { + engineUrl += `&mixamo=${bucketAssetUrl}`; + } else if ( + args.load.maybe_media_subtype === MediaFileSubtype.MocapNet + ) { + engineUrl += `&bvh=${bucketAssetUrl}`; + } else { + engineUrl += `&sceneImport=${bucketAssetUrl}`; + } + + } else if (isEngineObject(args.load)) { + // NB: This is an engine built-in, eg. `couch.gltf` or `sample-room.gltf`. + engineUrl += `&objectId=${args.load.objectId}`; + } + + return engineUrl; +} + +// Type guard for media files +const isMediaFile = (loadable: LoadableAsset) : loadable is MediaFile => { + return "token" in loadable; +} + +// Type guard for engine objects +const isEngineObject = (loadable: LoadableAsset) : loadable is EngineObject => { + return "objectId" in loadable; +} + From 8b96b0bd879a3c0359bdcc76654e0ac4a348a71f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 11 Mar 2024 21:25:49 -0400 Subject: [PATCH 2/4] use tested engine url construction for multiple uses of the engine --- .../common/Scene3D/GetEngineUrl.test.ts | 33 +++--- .../components/common/Scene3D/GetEngineUrl.ts | 41 +++++-- .../common/Scene3D/Scene3D.test.tsx | 6 - .../components/common/Scene3D/Scene3dNext.tsx | 103 ++++++++++++++++++ .../EngineMediaPanel/EngineMediaPanel.tsx | 76 +------------ .../StudioIntroPage.tsx | 18 +-- 6 files changed, 168 insertions(+), 109 deletions(-) delete mode 100644 packages/fakeyou.com/src/components/common/Scene3D/Scene3D.test.tsx create mode 100644 packages/fakeyou.com/src/components/common/Scene3D/Scene3dNext.tsx diff --git a/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.test.ts b/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.test.ts index 050a12bae0..e131101521 100644 --- a/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.test.ts +++ b/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.test.ts @@ -7,31 +7,38 @@ import { MediaFileSubtype } from "@storyteller/components/src/api/enums/MediaFil describe('mode', () => { test('studio', () => { - const url = GetEngineUrl({mode: EngineMode.Studio, load: { objectId: "foo" } }); + const url = GetEngineUrl({mode: EngineMode.Studio, asset: { objectId: "foo" } }); expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&objectId=foo"); }); test('viewer', () => { - const url = GetEngineUrl({mode: EngineMode.Viewer, load: { objectId: "foo" } }); + const url = GetEngineUrl({mode: EngineMode.Viewer, asset: { objectId: "foo" } }); expect(url).toEqual("https://engine.fakeyou.com/?mode=viewer&objectId=foo"); }); }); describe('skybox', () => { test('from named skybox', () => { - const url = GetEngineUrl({mode: EngineMode.Studio, load: { objectId: "foo" }, skybox: "gum_trees_4k" }); + const url = GetEngineUrl({mode: EngineMode.Studio, asset: { objectId: "foo" }, skybox: "gum_trees_4k" }); expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&skybox=gum_trees_4k&objectId=foo"); }); test('from hex color', () => { - const url = GetEngineUrl({mode: EngineMode.Studio, load: { objectId: "foo" }, skybox: "ff0000" }); + const url = GetEngineUrl({mode: EngineMode.Studio, asset: { objectId: "foo" }, skybox: "ff0000" }); expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&skybox=ff0000&objectId=foo"); }); }); +describe('storyteller scene media file tokens', () => { + test('scene token urls should work', () => { + const url = GetEngineUrl({mode: EngineMode.Studio, asset: { storytellerSceneMediaFileToken: "TOKEN" } }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&scene=remote://TOKEN.scn.ron"); + }); +}); + describe('object ids', () => { test('object id urls should work', () => { - const url = GetEngineUrl({mode: EngineMode.Studio, load: { objectId: "foo" } }); + const url = GetEngineUrl({mode: EngineMode.Studio, asset: { objectId: "foo" } }); expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&objectId=foo"); }); }); @@ -48,7 +55,7 @@ describe('media files', () => { updated_at: new Date(), maybe_creator_user: null, creator_set_visibility: "public", - // TODO(bt,2024-03-11): Make optional + // TODO(bt,2024-03-11): Make these fields optional maybe_model_weight_info: { title: "title", weight_token: "WEIGHT_TOKEN", @@ -75,7 +82,7 @@ describe('media files', () => { // NB: Not the real subtype; forcing test to act on type. mediaFile.maybe_media_subtype = null; - const url = GetEngineUrl({mode: EngineMode.Studio, load: mediaFile }); + const url = GetEngineUrl({mode: EngineMode.Studio, asset: mediaFile }); expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&scene=remote://MEDIA_FILE_TOKEN.scn.ron"); }); @@ -83,10 +90,10 @@ describe('media files', () => { // NB: Not the real time; forcing test to act on subtype. mediaFile.media_type = MediaFileType.Audio; - // TODO(bt,2024-03-11): Why does the IDE complain here? + // TODO(bt,2024-03-11): Why does the IDE complain about types here? mediaFile.maybe_media_subtype = MediaFileSubtype.StorytellerScene as any; - const url = GetEngineUrl({mode: EngineMode.Studio, load: mediaFile }); + const url = GetEngineUrl({mode: EngineMode.Studio, asset: mediaFile }); expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&scene=remote://MEDIA_FILE_TOKEN.scn.ron"); }); }); @@ -96,10 +103,10 @@ describe('media files', () => { // NB: Not the real time; forcing test to act on subtype. mediaFile.media_type = MediaFileType.Audio; - // TODO(bt,2024-03-11): Why does the IDE complain here? + // TODO(bt,2024-03-11): Why does the IDE complain about types here? mediaFile.maybe_media_subtype = MediaFileSubtype.Mixamo as any; - const url = GetEngineUrl({mode: EngineMode.Studio, load: mediaFile }); + const url = GetEngineUrl({mode: EngineMode.Studio, asset: mediaFile }); expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&mixamo=https://storage.googleapis.com/dev-vocodes-public/path/to/file"); }); }); @@ -109,10 +116,10 @@ describe('media files', () => { // NB: Not the real time; forcing test to act on subtype. mediaFile.media_type = MediaFileType.Audio; - // TODO(bt,2024-03-11): Why does the IDE complain here? + // TODO(bt,2024-03-11): Why does the IDE complain about types here? mediaFile.maybe_media_subtype = MediaFileSubtype.MocapNet as any; - const url = GetEngineUrl({mode: EngineMode.Studio, load: mediaFile }); + const url = GetEngineUrl({mode: EngineMode.Studio, asset: mediaFile }); expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&bvh=https://storage.googleapis.com/dev-vocodes-public/path/to/file"); }); }); diff --git a/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.ts b/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.ts index eb0d167663..7a9eb4530e 100644 --- a/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.ts +++ b/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.ts @@ -13,7 +13,7 @@ interface GetEngineUrlArgs { mode: EngineMode, // Polymorphic type of asset we're loading into the engine. - load: LoadableAsset + asset: LoadableAsset // Optional skybox color (hex) or name // - Predefined skyboxes, eg. `gum_trees_4k`, `kloofendal_28d_misty_4k`, `meadow_4k`, `promenade_de_vidy_4k`, `scythian_tombs_4k` @@ -29,7 +29,13 @@ interface EngineObject { objectId: string; } -type LoadableAsset = MediaFile | EngineObject; +// Used to load media files that are known a priori to be +// Storyteller Engine scenes (scene.ron files). +interface StorytellerSceneMediaFileToken { + storytellerSceneMediaFileToken: string; +} + +export type LoadableAsset = MediaFile | EngineObject | StorytellerSceneMediaFileToken; const ENGINE_BASE_URL = "https://engine.fakeyou.com"; @@ -41,35 +47,43 @@ export function GetEngineUrl(args: GetEngineUrlArgs) : string { engineUrl += `&skybox=${args.skybox}`; } - if (isMediaFile(args.load)) { + if (isMediaFile(args.asset)) { - const bucketAssetUrl = new BucketConfig().getGcsUrl(args.load.public_bucket_path); + const bucketAssetUrl = new BucketConfig().getGcsUrl(args.asset.public_bucket_path); if ( - args.load.maybe_media_subtype === MediaFileSubtype.StorytellerScene || - args.load.media_type === MediaFileType.SceneRon + args.asset.maybe_media_subtype === MediaFileSubtype.StorytellerScene || + args.asset.media_type === MediaFileType.SceneRon ) { // Storyteller Engine Scenes // NB: Storyteller Engine makes the API call to load the scene. // We don't need to pass the bucket path. // The engine, does, however, need a `.scn.ron` file extension. - const sceneUrlRef = `remote://${args.load.token}.scn.ron`; + const sceneUrlRef = `remote://${args.asset.token}.scn.ron`; engineUrl += `&scene=${sceneUrlRef}`; } else if ( - args.load.maybe_media_subtype === MediaFileSubtype.Mixamo + args.asset.maybe_media_subtype === MediaFileSubtype.Mixamo ) { engineUrl += `&mixamo=${bucketAssetUrl}`; } else if ( - args.load.maybe_media_subtype === MediaFileSubtype.MocapNet + args.asset.maybe_media_subtype === MediaFileSubtype.MocapNet ) { engineUrl += `&bvh=${bucketAssetUrl}`; } else { engineUrl += `&sceneImport=${bucketAssetUrl}`; } - } else if (isEngineObject(args.load)) { + } else if (isStorytellerSceneMediaFileToken(args.asset)) { + // Storyteller Engine Scenes + // NB: Storyteller Engine makes the API call to load the scene. + // We don't need to pass the bucket path. + // The engine, does, however, need a `.scn.ron` file extension. + const sceneUrlRef = `remote://${args.asset.storytellerSceneMediaFileToken}.scn.ron`; + engineUrl += `&scene=${sceneUrlRef}`; + + } else if (isEngineObject(args.asset)) { // NB: This is an engine built-in, eg. `couch.gltf` or `sample-room.gltf`. - engineUrl += `&objectId=${args.load.objectId}`; + engineUrl += `&objectId=${args.asset.objectId}`; } return engineUrl; @@ -85,3 +99,8 @@ const isEngineObject = (loadable: LoadableAsset) : loadable is EngineObject => { return "objectId" in loadable; } +// Type guard for engine objects +const isStorytellerSceneMediaFileToken = (loadable: LoadableAsset) : loadable is StorytellerSceneMediaFileToken => { + return "storytellerSceneMediaFileToken" in loadable; +} + diff --git a/packages/fakeyou.com/src/components/common/Scene3D/Scene3D.test.tsx b/packages/fakeyou.com/src/components/common/Scene3D/Scene3D.test.tsx deleted file mode 100644 index dad4cecb37..0000000000 --- a/packages/fakeyou.com/src/components/common/Scene3D/Scene3D.test.tsx +++ /dev/null @@ -1,6 +0,0 @@ -import React from 'react'; -import { render } from '@testing-library/react'; - -test('fake test', () => { - // No-op; just wiring up tests -}); diff --git a/packages/fakeyou.com/src/components/common/Scene3D/Scene3dNext.tsx b/packages/fakeyou.com/src/components/common/Scene3D/Scene3dNext.tsx new file mode 100644 index 0000000000..2d819fe109 --- /dev/null +++ b/packages/fakeyou.com/src/components/common/Scene3D/Scene3dNext.tsx @@ -0,0 +1,103 @@ +import React, { useCallback, useEffect, useRef } from "react"; +//import IframeResizer from "iframe-resizer-react"; +import "./Scene3D.scss"; +import { EngineMode } from "./EngineMode"; +import { GetEngineUrl, LoadableAsset } from "./GetEngineUrl"; + +const CHECK_FRAME_URL = "https://engine.fakeyou.com"; + +// See the following page for Storyteller Engine's query param documentation: +// https://www.notion.so/storytellerai/Studio-Iframe-Query-Params-a748a9929ec3404780c3884e7fb89bdb + +interface Scene3dNextProps { + // The mode to open the engine in. + mode: EngineMode; + + // A polymorphic asset type. + // See `GetEngineUrl()` for usage. + asset: LoadableAsset, + + // A skybox to use. + skybox?: string; + + // Whether the engine should be full screen + fullScreen?: boolean; + + // CSS classes to apply to the engine + className?: string; + + onSceneSavedCallback?: (mediaToken: string) => void; +} + +export default function Scene3dNext({ + mode, + asset, + skybox, + fullScreen = false, + className, + onSceneSavedCallback, +}: Scene3dNextProps) { + + const iframeRef = useRef(null); + + const onMessage = useCallback((event: MessageEvent) => { + console.log("engine message received", event.data, event); + + if (event.origin !== CHECK_FRAME_URL) + return; + + if (event.data === "studio-ready") { + console.log("studio-ready message (1)"); + + const studio = iframeRef.current?.contentWindow; + if (!studio) return; + + console.log("studio-ready message (2)"); + + // NB: Example of how to call the API in the other direction: + // studio.postMessage("save-scene", engineBaseUrl); + } else if ( + typeof event.data === "string" + && event.data.startsWith("scene-saved:") + ) { + const mediaToken = event.data.match(/scene-saved:(.+)/)?.[1]; + console.log("saved scene media token:", mediaToken); + + if (onSceneSavedCallback !== undefined && mediaToken !== undefined) { + onSceneSavedCallback(mediaToken); + } + + } else if (event.data === "scene-save-failed") { + console.error("Failed to save the scene!"); + } + }, [onSceneSavedCallback]); + + + useEffect(() => { + window.addEventListener("message", onMessage, false); + return () => { + window.removeEventListener("message", onMessage, false); + } + }, [onMessage]); + + + let engineUrl = GetEngineUrl({mode: mode, asset: asset, skybox: skybox}) + + return ( +
+ {/* IframeResizer was causing some glitches passing messages. Need to test more. */} + {/**/} +