diff --git a/packages/fakeyou.com/src/components/common/Scene3D/EngineMode.ts b/packages/fakeyou.com/src/components/common/Scene3D/EngineMode.ts new file mode 100644 index 0000000000..d24186fb7e --- /dev/null +++ b/packages/fakeyou.com/src/components/common/Scene3D/EngineMode.ts @@ -0,0 +1,5 @@ + +export enum EngineMode { + Viewer = "viewer", + Studio = "studio", +} diff --git a/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.test.ts b/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.test.ts new file mode 100644 index 0000000000..07d5881de2 --- /dev/null +++ b/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.test.ts @@ -0,0 +1,159 @@ +import { MediaFileType } from "@storyteller/components/src/api/_common/enums/MediaFileType"; +import { EngineMode } from "./EngineMode"; +import { GetEngineUrl } from "./GetEngineUrl"; +import { WeightCategory } from "@storyteller/components/src/api/_common/enums/WeightCategory"; +import { WeightType } from "@storyteller/components/src/api/_common/enums/WeightType"; +import { MediaFileSubtype } from "@storyteller/components/src/api/enums/MediaFileSubtype"; +import { MediaFile } from "@storyteller/components/src/api/media_files/GetMedia"; + +// These tests are important - constructing Storyteller Engine URLs is complicated, +// and we want to make sure we test the various cases that occur in production. + +describe('mode', () => { + test('studio', () => { + const url = GetEngineUrl({mode: EngineMode.Studio, asset: { objectId: "foo" } }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&objectId=foo"); + }); + + test('viewer', () => { + const url = GetEngineUrl({mode: EngineMode.Viewer, asset: { objectId: "foo" } }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=viewer&objectId=foo"); + }); +}); + +describe('skybox', () => { + test('from named skybox', () => { + const url = GetEngineUrl({mode: EngineMode.Studio, asset: { objectId: "foo" }, skybox: "gum_trees_4k" }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&skybox=gum_trees_4k&objectId=foo"); + }); + + test('from hex color', () => { + const url = GetEngineUrl({mode: EngineMode.Studio, asset: { objectId: "foo" }, skybox: "ff0000" }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&skybox=ff0000&objectId=foo"); + }); +}); + +describe('storyteller scene media file tokens', () => { + test('scene token urls should work', () => { + const url = GetEngineUrl({mode: EngineMode.Studio, asset: { storytellerSceneMediaFileToken: "TOKEN" } }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&scene=remote://TOKEN.scn.ron"); + }); +}); + +describe('object ids', () => { + test('object id urls should work', () => { + const url = GetEngineUrl({mode: EngineMode.Studio, asset: { objectId: "foo" } }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&objectId=foo"); + }); +}); + +// TODO(bt,2024-03-11): We're only testing one type of media file +describe('media files', () => { + let mediaFile : MediaFile; + + beforeEach(() => { + mediaFile = { + token: "MEDIA_FILE_TOKEN", + media_type: MediaFileType.GLB, + maybe_media_subtype: null, + maybe_prompt_token: null, + public_bucket_path: "path/to/file", + maybe_batch_token: "BATCH_TOKEN", + created_at: new Date(), + updated_at: new Date(), + maybe_creator_user: null, + creator_set_visibility: "public", + // TODO(bt,2024-03-11): Make these fields optional + maybe_model_weight_info: { + title: "title", + weight_token: "WEIGHT_TOKEN", + weight_category: WeightCategory.SD, + weight_type: WeightType.HIFIGAN_TT2, + maybe_cover_image_public_bucket_path: "image", + maybe_weight_creator: { + user_token: "USER_TOKEN", + username: "username", + display_name: "display_name", + gravatar_hash: "foo", + default_avatar: { + image_index: 1, + color_index: 2, + } + }, + } + }; + }); + + // Production example: + // [from media type] https://feature-mvp--fakeyou.netlify.app/studio-intro/m_ejhs95fc5aybp36h4a79k7523ds6an (scene file) + describe('storyteller scene files', () => { + test('from media type', () => { + mediaFile.media_type = MediaFileType.SceneRon; + mediaFile.maybe_media_subtype = null; // NB: Not the real subtype; forcing test to act on type. + + const url = GetEngineUrl({mode: EngineMode.Studio, asset: mediaFile }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&scene=remote://MEDIA_FILE_TOKEN.scn.ron"); + }); + + test('from media subtype', () => { + mediaFile.media_type = MediaFileType.Audio; // NB: Not the real time; forcing test to act on subtype. + mediaFile.maybe_media_subtype = MediaFileSubtype.StorytellerScene; + + const url = GetEngineUrl({mode: EngineMode.Studio, asset: mediaFile }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&scene=remote://MEDIA_FILE_TOKEN.scn.ron"); + }); + }); + + // Production examples: + // [without subtype] https://feature-mvp--fakeyou.netlify.app/media/m_tcj2zzncmvmn32f0yavmys5jdrc8cc (Majora's Mask GLB) + // [scene import subtype] https://feature-mvp--fakeyou.netlify.app/media/m_a504ma0n7vv3y80bw7bvgx7q2cecmb (Goron) + describe('generic scene file (not storyteller studio scene)', () => { + test('glb without subtype', () => { + mediaFile.media_type = MediaFileType.GLB; + mediaFile.maybe_media_subtype = null; // NB: Null in production (unless we backfill it) + mediaFile.public_bucket_path = "path/to/file.gltf"; // NB: We still write extension ".gltf" for glb. + + const url = GetEngineUrl({mode: EngineMode.Viewer, asset: mediaFile }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=viewer&sceneImport=https://storage.googleapis.com/dev-vocodes-public/path/to/file.gltf"); + }); + + test('glb with scene_import subtype', () => { + mediaFile.media_type = MediaFileType.GLB; + mediaFile.maybe_media_subtype = MediaFileSubtype.SceneImport; + mediaFile.public_bucket_path = "path/to/file.gltf"; // NB: We still write extension ".gltf" for glb. + + const url = GetEngineUrl({mode: EngineMode.Viewer, asset: mediaFile }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=viewer&sceneImport=https://storage.googleapis.com/dev-vocodes-public/path/to/file.gltf"); + }); + }); + + describe('mixamo animations', () => { + test('from media subtype', () => { + mediaFile.media_type = MediaFileType.Audio; // NB: Not the real time; forcing test to act on subtype. + mediaFile.maybe_media_subtype = MediaFileSubtype.Mixamo; + + const url = GetEngineUrl({mode: EngineMode.Studio, asset: mediaFile }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&mixamo=https://storage.googleapis.com/dev-vocodes-public/path/to/file"); + }); + }); + + // Production examples: + // [without subtype] https://feature-mvp--fakeyou.netlify.app/media/m_djbs4gjsjym41vn65py6ydzvy4ns3b (broken MocapNet animation) + describe('bvh animations', () => { + test('from media subtype', () => { + mediaFile.media_type = MediaFileType.Audio; // NB: Not the real time; forcing test to act on subtype. + mediaFile.maybe_media_subtype = MediaFileSubtype.MocapNet; + + const url = GetEngineUrl({mode: EngineMode.Studio, asset: mediaFile }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&bvh=https://storage.googleapis.com/dev-vocodes-public/path/to/file"); + }); + + test('bvh with empty subtype', () => { + mediaFile.media_type = MediaFileType.BVH; + mediaFile.maybe_media_subtype = null; // NB: Older BVH files do not specify the subtype + + const url = GetEngineUrl({mode: EngineMode.Studio, asset: mediaFile }); + expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&bvh=https://storage.googleapis.com/dev-vocodes-public/path/to/file"); + }); + }); +}); diff --git a/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.ts b/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.ts new file mode 100644 index 0000000000..b3f3937599 --- /dev/null +++ b/packages/fakeyou.com/src/components/common/Scene3D/GetEngineUrl.ts @@ -0,0 +1,115 @@ +import { MediaFile as MediaFileOne } from "@storyteller/components/src/api/media_files/GetMedia"; +import { MediaFile as MediaFileTwo } from "@storyteller/components/src/api/media_files/GetMediaFile"; +import { EngineMode } from "./EngineMode"; +import { MediaFileSubtype } from "@storyteller/components/src/api/enums/MediaFileSubtype"; +import { MediaFileType } from "@storyteller/components/src/api/_common/enums/MediaFileType"; +import { BucketConfig } from "@storyteller/components/src/api/BucketConfig"; + +// Storyteller Engine parameters +// These are documented here: +// https://www.notion.so/storytellerai/Studio-Iframe-Query-Params-a748a9929ec3404780c3884e7fb89bdb + +interface GetEngineUrlArgs { + mode: EngineMode, + + // Polymorphic type of asset we're loading into the engine. + asset: LoadableAsset + + // Optional skybox color (hex) or name + // - Predefined skyboxes, eg. `gum_trees_4k`, `kloofendal_28d_misty_4k`, `meadow_4k`, `promenade_de_vidy_4k`, `scythian_tombs_4k` + // - Hex colors, eg. `ff0000` or `1a1a27`. + skybox?: string, +} + +// Support multiple "Media File" API response payloads. +type MediaFile = MediaFileOne | MediaFileTwo ; + +// Used to load raw built-in objects into the engine. +interface EngineObject { + objectId: string; +} + +// Used to load media files that are known a priori to be +// Storyteller Engine scenes (scene.ron files). +interface StorytellerSceneMediaFileToken { + storytellerSceneMediaFileToken: string; +} + +export type LoadableAsset = MediaFile | EngineObject | StorytellerSceneMediaFileToken; + +const ENGINE_BASE_URL = "https://engine.fakeyou.com"; + +export function GetEngineUrl(args: GetEngineUrlArgs) : string { + let engineUrl = `${ENGINE_BASE_URL}/?mode=${args.mode}`; + + + if (args.skybox) { + engineUrl += `&skybox=${args.skybox}`; + } + + if (isMediaFile(args.asset)) { + + const bucketAssetUrl = new BucketConfig().getGcsUrl(args.asset.public_bucket_path); + + if ( + args.asset.maybe_media_subtype === MediaFileSubtype.StorytellerScene || + args.asset.media_type === MediaFileType.SceneRon + ) { + // Storyteller Engine Scenes + // NB: Storyteller Engine makes the API call to load the scene. + // We don't need to pass the bucket path. + // The engine, does, however, need a `.scn.ron` file extension. + const sceneUrlRef = `remote://${args.asset.token}.scn.ron`; + engineUrl += `&scene=${sceneUrlRef}`; + } else if ( + args.asset.maybe_media_subtype === MediaFileSubtype.Mixamo + ) { + engineUrl += `&mixamo=${bucketAssetUrl}`; + } else if ( + args.asset.maybe_media_subtype === MediaFileSubtype.MocapNet + ) { + engineUrl += `&bvh=${bucketAssetUrl}`; + } else if ( + args.asset.media_type === MediaFileType.BVH && ( + args.asset.maybe_media_subtype === null || + args.asset.maybe_media_subtype === undefined + ) + ){ + // NB: Older BVH files that do not specify a subtype are MocapNet, + // which will take a bare "bvh" argument. + engineUrl += `&bvh=${bucketAssetUrl}`; + } else { + engineUrl += `&sceneImport=${bucketAssetUrl}`; + } + + } else if (isStorytellerSceneMediaFileToken(args.asset)) { + // Storyteller Engine Scenes + // NB: Storyteller Engine makes the API call to load the scene. + // We don't need to pass the bucket path. + // The engine, does, however, need a `.scn.ron` file extension. + const sceneUrlRef = `remote://${args.asset.storytellerSceneMediaFileToken}.scn.ron`; + engineUrl += `&scene=${sceneUrlRef}`; + + } else if (isEngineObject(args.asset)) { + // NB: This is an engine built-in, eg. `couch.gltf` or `sample-room.gltf`. + engineUrl += `&objectId=${args.asset.objectId}`; + } + + return engineUrl; +} + +// Type guard for media files +const isMediaFile = (loadable: LoadableAsset) : loadable is MediaFile => { + return "token" in loadable; +} + +// Type guard for engine objects +const isEngineObject = (loadable: LoadableAsset) : loadable is EngineObject => { + return "objectId" in loadable; +} + +// Type guard for engine objects +const isStorytellerSceneMediaFileToken = (loadable: LoadableAsset) : loadable is StorytellerSceneMediaFileToken => { + return "storytellerSceneMediaFileToken" in loadable; +} + diff --git a/packages/fakeyou.com/src/components/common/Scene3D/Scene3D.test.tsx b/packages/fakeyou.com/src/components/common/Scene3D/Scene3D.test.tsx deleted file mode 100644 index dad4cecb37..0000000000 --- a/packages/fakeyou.com/src/components/common/Scene3D/Scene3D.test.tsx +++ /dev/null @@ -1,6 +0,0 @@ -import React from 'react'; -import { render } from '@testing-library/react'; - -test('fake test', () => { - // No-op; just wiring up tests -}); diff --git a/packages/fakeyou.com/src/components/common/Scene3D/Scene3D.tsx b/packages/fakeyou.com/src/components/common/Scene3D/Scene3D.tsx index d30377ee2e..1cc71914df 100644 --- a/packages/fakeyou.com/src/components/common/Scene3D/Scene3D.tsx +++ b/packages/fakeyou.com/src/components/common/Scene3D/Scene3D.tsx @@ -1,53 +1,49 @@ import React, { useCallback, useEffect, useRef } from "react"; //import IframeResizer from "iframe-resizer-react"; import "./Scene3D.scss"; +import { EngineMode } from "./EngineMode"; +import { GetEngineUrl, LoadableAsset } from "./GetEngineUrl"; + +const CHECK_FRAME_URL = "https://engine.fakeyou.com"; // See the following page for Storyteller Engine's query param documentation: // https://www.notion.so/storytellerai/Studio-Iframe-Query-Params-a748a9929ec3404780c3884e7fb89bdb -// -// You can supply either an `objectId` or a `sceneMediaFileToken` when loading in studio mode. -interface Scene3DProps { - mode: string; - skybox?: string; - fullScreen?: boolean; - urlParams?: string; - className?: string; - onSceneSavedCallback?: (mediaToken: string) => void; - // NB: The following parameters are mutally exclusive. - // We should use Typescript to enforce this. - objectId?: string; // built-in objects - sceneMediaFileToken?: string; // scn.ron native storyteller engine scenes - mixamoUrl?: string; // a mixamo animation - bvhUrl?: string; // a MocapNet animation in BVH format - sceneImportUrl?: string; // a generic (non-storyteller) scene: GLB, GLTF, etc. +interface Scene3DProps { + // The mode to open the engine in. + mode: EngineMode; + + // A polymorphic asset type. + // See `GetEngineUrl()` for usage. + asset: LoadableAsset, + + // A skybox to use. + skybox?: string; + + // Whether the engine should be full screen + fullScreen?: boolean; + + // CSS classes to apply to the engine + className?: string; + + onSceneSavedCallback?: (mediaToken: string) => void; } export default function Scene3D({ mode, + asset, skybox, fullScreen = false, className, onSceneSavedCallback, - - // NB: The following parameters are mutally exclusive. - // We should use Typescript to enforce this. - objectId, - sceneMediaFileToken, - mixamoUrl, - bvhUrl, - sceneImportUrl, - }: Scene3DProps) { - const engineBaseUrl = "https://engine.fakeyou.com"; const iframeRef = useRef(null); - const onMessage = useCallback((event: MessageEvent) => { console.log("engine message received", event.data, event); - if (event.origin !== engineBaseUrl) + if (event.origin !== CHECK_FRAME_URL) return; if (event.data === "studio-ready") { @@ -85,31 +81,7 @@ export default function Scene3D({ }, [onMessage]); - let engineUrl = `${engineBaseUrl}/?mode=${mode}`; - - if (skybox) { - engineUrl += `&skybox=${skybox}`; - } - - if (sceneMediaFileToken) { - // NB: Storyteller Engine makes the API call to load the scene. - // We don't need to pass the bucket path. - // The engine, does, however, need a `.scn.ron` file extension. - const sceneUrlRef = `remote://${sceneMediaFileToken}.scn.ron`; - engineUrl += `&scene=${sceneUrlRef}`; - } else if (objectId) { - // NB: This is an engine built-in, eg. `couch.gltf` or `sample-room.gltf`. - engineUrl += `&objectId=${objectId}`; - } else if (mixamoUrl) { - // NB: This should be a full bucket path to an asset. - engineUrl += `&mixamo=${mixamoUrl}`; - } else if (bvhUrl) { - // NB: This should be a full bucket path to an asset. - engineUrl += `&bvh=${bvhUrl}`; - } else if (sceneImportUrl) { - // TODO: Not sure what the format of this should be. - engineUrl += `&sceneImport=${sceneImportUrl}`; - } + let engineUrl = GetEngineUrl({mode: mode, asset: asset, skybox: skybox}) return (
*/ ); } diff --git a/packages/fakeyou.com/src/v2/view/pages/storyteller_studio/StorytellerStudioPage.tsx b/packages/fakeyou.com/src/v2/view/pages/storyteller_studio/StorytellerStudioPage.tsx index d79080a892..944b3893cb 100644 --- a/packages/fakeyou.com/src/v2/view/pages/storyteller_studio/StorytellerStudioPage.tsx +++ b/packages/fakeyou.com/src/v2/view/pages/storyteller_studio/StorytellerStudioPage.tsx @@ -2,9 +2,10 @@ import React from "react"; import { SessionWrapper } from "@storyteller/components/src/session/SessionWrapper"; import { SessionSubscriptionsWrapper } from "@storyteller/components/src/session/SessionSubscriptionsWrapper"; import { StudioNotAvailable } from "v2/view/_common/StudioNotAvailable"; -import { Scene3D } from "components/common"; import { usePrefixedDocumentTitle } from "common/UsePrefixedDocumentTitle"; import { useParams } from "react-router-dom"; +import Scene3D from "components/common/Scene3D/Scene3D"; +import { EngineMode } from "components/common/Scene3D/EngineMode"; interface Props { sessionWrapper: SessionWrapper; @@ -20,24 +21,25 @@ function StorytellerStudioListPage(props: Props) { return ; } - let engineParams = {}; + let assetDescriptor; if (mediaToken) { - engineParams = { - sceneMediaFileToken: mediaToken + assetDescriptor = { + storytellerSceneMediaFileToken: mediaToken }; } else { - engineParams = { + assetDescriptor = { objectId: "sample-room.gltf" }; } return ( <> - + mode={EngineMode.Studio} + asset={assetDescriptor} + /> ); } diff --git a/packages/fakeyou.com/src/v2/view/pages/storyteller_studio_intro/StudioIntroPage.tsx b/packages/fakeyou.com/src/v2/view/pages/storyteller_studio_intro/StudioIntroPage.tsx index 6046e2cd8e..de4a957a80 100644 --- a/packages/fakeyou.com/src/v2/view/pages/storyteller_studio_intro/StudioIntroPage.tsx +++ b/packages/fakeyou.com/src/v2/view/pages/storyteller_studio_intro/StudioIntroPage.tsx @@ -2,7 +2,7 @@ import React, { useCallback, useState } from "react"; import { SessionWrapper } from "@storyteller/components/src/session/SessionWrapper"; import { SessionSubscriptionsWrapper } from "@storyteller/components/src/session/SessionSubscriptionsWrapper"; import { StudioNotAvailable } from "v2/view/_common/StudioNotAvailable"; -import { Button, Scene3D } from "components/common"; +import { Button } from "components/common"; import { usePrefixedDocumentTitle } from "common/UsePrefixedDocumentTitle"; import { useParams, useHistory } from "react-router-dom"; import { useInferenceJobs } from "hooks"; @@ -10,6 +10,8 @@ import { EnqueueEngineCompositing } from "@storyteller/components/src/api/engine import { FrontendInferenceJobType } from "@storyteller/components/src/jobs/InferenceJob"; import { v4 as uuidv4 } from "uuid"; import "./StudioIntro.scss"; +import Scene3D from "components/common/Scene3D/Scene3D"; +import { EngineMode } from "components/common/Scene3D/EngineMode"; interface Props { sessionWrapper: SessionWrapper; @@ -39,16 +41,16 @@ function StudioIntroPage(props: Props) { return ; } - let engineParams = {}; + let assetDescriptor; // We should prefer to start the onboarding flow with an existing scene, but if // one is unavailable, we should show the sample room. if (mediaToken) { - engineParams = { - sceneMediaFileToken: mediaToken, + assetDescriptor = { + storytellerSceneMediaFileToken: mediaToken, }; } else { - engineParams = { + assetDescriptor = { objectId: "sample-room.gltf", }; } @@ -70,10 +72,10 @@ function StudioIntroPage(props: Props) {