Merge branch 'feature/mvp' of https://github.com/storytold/storyteller-frontend into feature/mvp

This commit is contained in:
Victor committed 2024-03-11 19:59:11 -07:00
commit 44b9c032df
10 files changed
+328 -146

No files matched your search

@@ -0,0 +1,5 @@
export enum EngineMode {
Viewer = "viewer",
Studio = "studio",
}
@@ -0,0 +1,159 @@
import { MediaFileType } from "@storyteller/components/src/api/_common/enums/MediaFileType";
import { EngineMode } from "./EngineMode";
import { GetEngineUrl } from "./GetEngineUrl";
import { WeightCategory } from "@storyteller/components/src/api/_common/enums/WeightCategory";
import { WeightType } from "@storyteller/components/src/api/_common/enums/WeightType";
import { MediaFileSubtype } from "@storyteller/components/src/api/enums/MediaFileSubtype";
import { MediaFile } from "@storyteller/components/src/api/media_files/GetMedia";
// These tests are important - constructing Storyteller Engine URLs is complicated,
// and we want to make sure we test the various cases that occur in production.
describe('mode', () => {
test('studio', () => {
const url = GetEngineUrl({mode: EngineMode.Studio, asset: { objectId: "foo" } });
expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&objectId=foo");
});
test('viewer', () => {
const url = GetEngineUrl({mode: EngineMode.Viewer, asset: { objectId: "foo" } });
expect(url).toEqual("https://engine.fakeyou.com/?mode=viewer&objectId=foo");
});
});
describe('skybox', () => {
test('from named skybox', () => {
const url = GetEngineUrl({mode: EngineMode.Studio, asset: { objectId: "foo" }, skybox: "gum_trees_4k" });
expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&skybox=gum_trees_4k&objectId=foo");
});
test('from hex color', () => {
const url = GetEngineUrl({mode: EngineMode.Studio, asset: { objectId: "foo" }, skybox: "ff0000" });
expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&skybox=ff0000&objectId=foo");
});
});
describe('storyteller scene media file tokens', () => {
test('scene token urls should work', () => {
const url = GetEngineUrl({mode: EngineMode.Studio, asset: { storytellerSceneMediaFileToken: "TOKEN" } });
expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&scene=remote://TOKEN.scn.ron");
});
});
describe('object ids', () => {
test('object id urls should work', () => {
const url = GetEngineUrl({mode: EngineMode.Studio, asset: { objectId: "foo" } });
expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&objectId=foo");
});
});
// TODO(bt,2024-03-11): We're only testing one type of media file
describe('media files', () => {
let mediaFile : MediaFile;
beforeEach(() => {
mediaFile = {
token: "MEDIA_FILE_TOKEN",
media_type: MediaFileType.GLB,
maybe_media_subtype: null,
maybe_prompt_token: null,
public_bucket_path: "path/to/file",
maybe_batch_token: "BATCH_TOKEN",
created_at: new Date(),
updated_at: new Date(),
maybe_creator_user: null,
creator_set_visibility: "public",
// TODO(bt,2024-03-11): Make these fields optional
maybe_model_weight_info: {
title: "title",
weight_token: "WEIGHT_TOKEN",
weight_category: WeightCategory.SD,
weight_type: WeightType.HIFIGAN_TT2,
maybe_cover_image_public_bucket_path: "image",
maybe_weight_creator: {
user_token: "USER_TOKEN",
username: "username",
display_name: "display_name",
gravatar_hash: "foo",
default_avatar: {
image_index: 1,
color_index: 2,
}
},
}
};
});
// Production example:
// [from media type] https://feature-mvp--fakeyou.netlify.app/studio-intro/m_ejhs95fc5aybp36h4a79k7523ds6an (scene file)
describe('storyteller scene files', () => {
test('from media type', () => {
mediaFile.media_type = MediaFileType.SceneRon;
mediaFile.maybe_media_subtype = null; // NB: Not the real subtype; forcing test to act on type.
const url = GetEngineUrl({mode: EngineMode.Studio, asset: mediaFile });
expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&scene=remote://MEDIA_FILE_TOKEN.scn.ron");
});
test('from media subtype', () => {
mediaFile.media_type = MediaFileType.Audio; // NB: Not the real time; forcing test to act on subtype.
mediaFile.maybe_media_subtype = MediaFileSubtype.StorytellerScene;
const url = GetEngineUrl({mode: EngineMode.Studio, asset: mediaFile });
expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&scene=remote://MEDIA_FILE_TOKEN.scn.ron");
});
});
// Production examples:
// [without subtype] https://feature-mvp--fakeyou.netlify.app/media/m_tcj2zzncmvmn32f0yavmys5jdrc8cc (Majora's Mask GLB)
// [scene import subtype] https://feature-mvp--fakeyou.netlify.app/media/m_a504ma0n7vv3y80bw7bvgx7q2cecmb (Goron)
describe('generic scene file (not storyteller studio scene)', () => {
test('glb without subtype', () => {
mediaFile.media_type = MediaFileType.GLB;
mediaFile.maybe_media_subtype = null; // NB: Null in production (unless we backfill it)
mediaFile.public_bucket_path = "path/to/file.gltf"; // NB: We still write extension ".gltf" for glb.
const url = GetEngineUrl({mode: EngineMode.Viewer, asset: mediaFile });
expect(url).toEqual("https://engine.fakeyou.com/?mode=viewer&sceneImport=https://storage.googleapis.com/dev-vocodes-public/path/to/file.gltf");
});
test('glb with scene_import subtype', () => {
mediaFile.media_type = MediaFileType.GLB;
mediaFile.maybe_media_subtype = MediaFileSubtype.SceneImport;
mediaFile.public_bucket_path = "path/to/file.gltf"; // NB: We still write extension ".gltf" for glb.
const url = GetEngineUrl({mode: EngineMode.Viewer, asset: mediaFile });
expect(url).toEqual("https://engine.fakeyou.com/?mode=viewer&sceneImport=https://storage.googleapis.com/dev-vocodes-public/path/to/file.gltf");
});
});
describe('mixamo animations', () => {
test('from media subtype', () => {
mediaFile.media_type = MediaFileType.Audio; // NB: Not the real time; forcing test to act on subtype.
mediaFile.maybe_media_subtype = MediaFileSubtype.Mixamo;
const url = GetEngineUrl({mode: EngineMode.Studio, asset: mediaFile });
expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&mixamo=https://storage.googleapis.com/dev-vocodes-public/path/to/file");
});
});
// Production examples:
// [without subtype] https://feature-mvp--fakeyou.netlify.app/media/m_djbs4gjsjym41vn65py6ydzvy4ns3b (broken MocapNet animation)
describe('bvh animations', () => {
test('from media subtype', () => {
mediaFile.media_type = MediaFileType.Audio; // NB: Not the real time; forcing test to act on subtype.
mediaFile.maybe_media_subtype = MediaFileSubtype.MocapNet;
const url = GetEngineUrl({mode: EngineMode.Studio, asset: mediaFile });
expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&bvh=https://storage.googleapis.com/dev-vocodes-public/path/to/file");
});
test('bvh with empty subtype', () => {
mediaFile.media_type = MediaFileType.BVH;
mediaFile.maybe_media_subtype = null; // NB: Older BVH files do not specify the subtype
const url = GetEngineUrl({mode: EngineMode.Studio, asset: mediaFile });
expect(url).toEqual("https://engine.fakeyou.com/?mode=studio&bvh=https://storage.googleapis.com/dev-vocodes-public/path/to/file");
});
});
});
@@ -0,0 +1,115 @@
import { MediaFile as MediaFileOne } from "@storyteller/components/src/api/media_files/GetMedia";
import { MediaFile as MediaFileTwo } from "@storyteller/components/src/api/media_files/GetMediaFile";
import { EngineMode } from "./EngineMode";
import { MediaFileSubtype } from "@storyteller/components/src/api/enums/MediaFileSubtype";
import { MediaFileType } from "@storyteller/components/src/api/_common/enums/MediaFileType";
import { BucketConfig } from "@storyteller/components/src/api/BucketConfig";
// Storyteller Engine parameters
// These are documented here:
// https://www.notion.so/storytellerai/Studio-Iframe-Query-Params-a748a9929ec3404780c3884e7fb89bdb
interface GetEngineUrlArgs {
mode: EngineMode,
// Polymorphic type of asset we're loading into the engine.
asset: LoadableAsset
// Optional skybox color (hex) or name
// - Predefined skyboxes, eg. `gum_trees_4k`, `kloofendal_28d_misty_4k`, `meadow_4k`, `promenade_de_vidy_4k`, `scythian_tombs_4k`
// - Hex colors, eg. `ff0000` or `1a1a27`.
skybox?: string,
}
// Support multiple "Media File" API response payloads.
type MediaFile = MediaFileOne | MediaFileTwo ;
// Used to load raw built-in objects into the engine.
interface EngineObject {
objectId: string;
}
// Used to load media files that are known a priori to be
// Storyteller Engine scenes (scene.ron files).
interface StorytellerSceneMediaFileToken {
storytellerSceneMediaFileToken: string;
}
export type LoadableAsset = MediaFile | EngineObject | StorytellerSceneMediaFileToken;
const ENGINE_BASE_URL = "https://engine.fakeyou.com";
export function GetEngineUrl(args: GetEngineUrlArgs) : string {
let engineUrl = `${ENGINE_BASE_URL}/?mode=${args.mode}`;
if (args.skybox) {
engineUrl += `&skybox=${args.skybox}`;
}
if (isMediaFile(args.asset)) {
const bucketAssetUrl = new BucketConfig().getGcsUrl(args.asset.public_bucket_path);
if (
args.asset.maybe_media_subtype === MediaFileSubtype.StorytellerScene ||
args.asset.media_type === MediaFileType.SceneRon
) {
// Storyteller Engine Scenes
// NB: Storyteller Engine makes the API call to load the scene.
// We don't need to pass the bucket path.
// The engine, does, however, need a `.scn.ron` file extension.
const sceneUrlRef = `remote://${args.asset.token}.scn.ron`;
engineUrl += `&scene=${sceneUrlRef}`;
} else if (
args.asset.maybe_media_subtype === MediaFileSubtype.Mixamo
) {
engineUrl += `&mixamo=${bucketAssetUrl}`;
} else if (
args.asset.maybe_media_subtype === MediaFileSubtype.MocapNet
) {
engineUrl += `&bvh=${bucketAssetUrl}`;
} else if (
args.asset.media_type === MediaFileType.BVH && (
args.asset.maybe_media_subtype === null ||
args.asset.maybe_media_subtype === undefined
)
){
// NB: Older BVH files that do not specify a subtype are MocapNet,
// which will take a bare "bvh" argument.
engineUrl += `&bvh=${bucketAssetUrl}`;
} else {
engineUrl += `&sceneImport=${bucketAssetUrl}`;
}
} else if (isStorytellerSceneMediaFileToken(args.asset)) {
// Storyteller Engine Scenes
// NB: Storyteller Engine makes the API call to load the scene.
// We don't need to pass the bucket path.
// The engine, does, however, need a `.scn.ron` file extension.
const sceneUrlRef = `remote://${args.asset.storytellerSceneMediaFileToken}.scn.ron`;
engineUrl += `&scene=${sceneUrlRef}`;
} else if (isEngineObject(args.asset)) {
// NB: This is an engine built-in, eg. `couch.gltf` or `sample-room.gltf`.
engineUrl += `&objectId=${args.asset.objectId}`;
}
return engineUrl;
}
// Type guard for media files
const isMediaFile = (loadable: LoadableAsset) : loadable is MediaFile => {
return "token" in loadable;
}
// Type guard for engine objects
const isEngineObject = (loadable: LoadableAsset) : loadable is EngineObject => {
return "objectId" in loadable;
}
// Type guard for engine objects
const isStorytellerSceneMediaFileToken = (loadable: LoadableAsset) : loadable is StorytellerSceneMediaFileToken => {
return "storytellerSceneMediaFileToken" in loadable;
}
@@ -1,6 +0,0 @@
import React from 'react';
import { render } from '@testing-library/react';
test('fake test', () => {
// No-op; just wiring up tests
});
@@ -1,53 +1,49 @@
import React, { useCallback, useEffect, useRef } from "react";
//import IframeResizer from "iframe-resizer-react";
import "./Scene3D.scss";
import { EngineMode } from "./EngineMode";
import { GetEngineUrl, LoadableAsset } from "./GetEngineUrl";
const CHECK_FRAME_URL = "https://engine.fakeyou.com";
// See the following page for Storyteller Engine's query param documentation:
// https://www.notion.so/storytellerai/Studio-Iframe-Query-Params-a748a9929ec3404780c3884e7fb89bdb
//
// You can supply either an `objectId` or a `sceneMediaFileToken` when loading in studio mode.
interface Scene3DProps {
mode: string;
skybox?: string;
fullScreen?: boolean;
urlParams?: string;
className?: string;
onSceneSavedCallback?: (mediaToken: string) => void;
// NB: The following parameters are mutally exclusive.
// We should use Typescript to enforce this.
objectId?: string; // built-in objects
sceneMediaFileToken?: string; // scn.ron native storyteller engine scenes
mixamoUrl?: string; // a mixamo animation
bvhUrl?: string; // a MocapNet animation in BVH format
sceneImportUrl?: string; // a generic (non-storyteller) scene: GLB, GLTF, etc.
interface Scene3DProps {
// The mode to open the engine in.
mode: EngineMode;
// A polymorphic asset type.
// See `GetEngineUrl()` for usage.
asset: LoadableAsset,
// A skybox to use.
skybox?: string;
// Whether the engine should be full screen
fullScreen?: boolean;
// CSS classes to apply to the engine
className?: string;
onSceneSavedCallback?: (mediaToken: string) => void;
}
export default function Scene3D({
mode,
asset,
skybox,
fullScreen = false,
className,
onSceneSavedCallback,
// NB: The following parameters are mutally exclusive.
// We should use Typescript to enforce this.
objectId,
sceneMediaFileToken,
mixamoUrl,
bvhUrl,
sceneImportUrl,
}: Scene3DProps) {
const engineBaseUrl = "https://engine.fakeyou.com";
const iframeRef = useRef<HTMLIFrameElement>(null);
const onMessage = useCallback((event: MessageEvent) => {
console.log("engine message received", event.data, event);
if (event.origin !== engineBaseUrl)
if (event.origin !== CHECK_FRAME_URL)
return;
if (event.data === "studio-ready") {
@@ -85,31 +81,7 @@ export default function Scene3D({
}, [onMessage]);
let engineUrl = `${engineBaseUrl}/?mode=${mode}`;
if (skybox) {
engineUrl += `&skybox=${skybox}`;
}
if (sceneMediaFileToken) {
// NB: Storyteller Engine makes the API call to load the scene.
// We don't need to pass the bucket path.
// The engine, does, however, need a `.scn.ron` file extension.
const sceneUrlRef = `remote://${sceneMediaFileToken}.scn.ron`;
engineUrl += `&scene=${sceneUrlRef}`;
} else if (objectId) {
// NB: This is an engine built-in, eg. `couch.gltf` or `sample-room.gltf`.
engineUrl += `&objectId=${objectId}`;
} else if (mixamoUrl) {
// NB: This should be a full bucket path to an asset.
engineUrl += `&mixamo=${mixamoUrl}`;
} else if (bvhUrl) {
// NB: This should be a full bucket path to an asset.
engineUrl += `&bvh=${bvhUrl}`;
} else if (sceneImportUrl) {
// TODO: Not sure what the format of this should be.
engineUrl += `&sceneImport=${sceneImportUrl}`;
}
let engineUrl = GetEngineUrl({mode: mode, asset: asset, skybox: skybox})
return (
<div
@@ -1 +0,0 @@
export { default } from "./Scene3D";
@@ -28,7 +28,7 @@ import NumberSliderV2 from "./NumberSliderV2";
import Pagination from "./Pagination";
import Panel from "./Panel";
import Radio from "./Radio";
import Scene3D from "./Scene3D";
import Scene3D from "./Scene3D/Scene3D";
import Searcher from "./Searcher";
import SegmentButtons from "./SegmentButtons";
import Select from "./Select";
@@ -1,10 +1,7 @@
import React from "react";
import { MediaFile } from "@storyteller/components/src/api/media_files/GetMediaFile";
import { BucketConfig } from "@storyteller/components/src/api/BucketConfig";
//import Iframe from "react-iframe";
import { MediaFileType } from "@storyteller/components/src/api/_common/enums/MediaFileType";
import { Scene3D } from "components/common";
import { MediaFileSubtype } from "@storyteller/components/src/api/enums/MediaFileSubtype";
import Scene3D from "components/common/Scene3D/Scene3D";
import { EngineMode } from "components/common/Scene3D/EngineMode";
// Storyteller Engine parameters
// These are documented here:
@@ -19,76 +16,13 @@ export interface EngineMediaPanelArgs {
}
export function EngineMediaPanel({ mediaFile } : EngineMediaPanelArgs) {
const assetUrl = new BucketConfig().getGcsUrl(mediaFile.public_bucket_path);
let engineParams = {};
switch (mediaFile.maybe_media_subtype) {
case MediaFileSubtype.StorytellerScene:
engineParams = {
sceneMediaFileToken: mediaFile.token,
};
break;
case MediaFileSubtype.Mixamo:
engineParams = {
mixamoUrl: assetUrl,
};
break;
case MediaFileSubtype.MocapNet:
engineParams = {
bvhUrl: assetUrl,
};
break;
case MediaFileSubtype.AnimationOnly:
case MediaFileSubtype.Scene:
case MediaFileSubtype.SceneImport:
default:
// This should hypothetically be any type of generic scene asset
engineParams = {
sceneImportUrl: assetUrl,
};
break;
}
if (Object.keys(engineParams).length === 0) {
// Support for files without a media subtype
if (assetUrl.endsWith("bvh")) {
// Assume MocapNet.
engineParams = {
bvhUrl: assetUrl,
};
}
// TODO(bt,2024-03-11): Figure this out
//else if (assetUrl.endsWith("glb")) {
// // Assume MocapNet.
// engineParams = {
// bvhUrl: assetUrl,
// };
//}
}
if (mediaFile.media_type === MediaFileType.SceneRon) {
// This will always be a storyteller scene
engineParams = {
sceneMediaFileToken: mediaFile.token,
};
}
return (
/*<Iframe
{...{
url: `https://engine.fakeyou.com?mode=viewer&bvh=${assetUrl}&skybox=${SKYBOX}`,
className: "fy-studio-frame",
}}
/>*/
<Scene3D
mode="viewer"
mode={EngineMode.Viewer}
skybox={SKYBOX}
fullScreen={false}
//className="flex-grow-1"
className="fy-studio-frame"
//onSceneSavedCallback={onSaveCallback}
{...engineParams}
asset={mediaFile}
/>
);
}
@@ -2,9 +2,10 @@ import React from "react";
import { SessionWrapper } from "@storyteller/components/src/session/SessionWrapper";
import { SessionSubscriptionsWrapper } from "@storyteller/components/src/session/SessionSubscriptionsWrapper";
import { StudioNotAvailable } from "v2/view/_common/StudioNotAvailable";
import { Scene3D } from "components/common";
import { usePrefixedDocumentTitle } from "common/UsePrefixedDocumentTitle";
import { useParams } from "react-router-dom";
import Scene3D from "components/common/Scene3D/Scene3D";
import { EngineMode } from "components/common/Scene3D/EngineMode";
interface Props {
sessionWrapper: SessionWrapper;
@@ -20,24 +21,25 @@ function StorytellerStudioListPage(props: Props) {
return <StudioNotAvailable />;
}
let engineParams = {};
let assetDescriptor;
if (mediaToken) {
engineParams = {
sceneMediaFileToken: mediaToken
assetDescriptor = {
storytellerSceneMediaFileToken: mediaToken
};
} else {
engineParams = {
assetDescriptor = {
objectId: "sample-room.gltf"
};
}
return (
<>
<Scene3D
<Scene3D
fullScreen={true}
mode="studio"
{...engineParams} />
mode={EngineMode.Studio}
asset={assetDescriptor}
/>
</>
);
}
@@ -2,7 +2,7 @@ import React, { useCallback, useState } from "react";
import { SessionWrapper } from "@storyteller/components/src/session/SessionWrapper";
import { SessionSubscriptionsWrapper } from "@storyteller/components/src/session/SessionSubscriptionsWrapper";
import { StudioNotAvailable } from "v2/view/_common/StudioNotAvailable";
import { Button, Scene3D } from "components/common";
import { Button } from "components/common";
import { usePrefixedDocumentTitle } from "common/UsePrefixedDocumentTitle";
import { useParams, useHistory } from "react-router-dom";
import { useInferenceJobs } from "hooks";
@@ -10,6 +10,8 @@ import { EnqueueEngineCompositing } from "@storyteller/components/src/api/engine
import { FrontendInferenceJobType } from "@storyteller/components/src/jobs/InferenceJob";
import { v4 as uuidv4 } from "uuid";
import "./StudioIntro.scss";
import Scene3D from "components/common/Scene3D/Scene3D";
import { EngineMode } from "components/common/Scene3D/EngineMode";
interface Props {
sessionWrapper: SessionWrapper;
@@ -39,16 +41,16 @@ function StudioIntroPage(props: Props) {
return <StudioNotAvailable />;
}
let engineParams = {};
let assetDescriptor;
// We should prefer to start the onboarding flow with an existing scene, but if
// one is unavailable, we should show the sample room.
if (mediaToken) {
engineParams = {
sceneMediaFileToken: mediaToken,
assetDescriptor = {
storytellerSceneMediaFileToken: mediaToken,
};
} else {
engineParams = {
assetDescriptor = {
objectId: "sample-room.gltf",
};
}
@@ -70,10 +72,10 @@ function StudioIntroPage(props: Props) {
<div className="studio-intro-page">
<Scene3D
fullScreen={true}
mode="studio"
mode={EngineMode.Studio}
asset={assetDescriptor}
className="flex-grow-1"
onSceneSavedCallback={onSaveCallback}
{...engineParams}
/>
<div className="d-flex justify-content-center p-3">
<Button label="Create Movie from 3D Scene" onClick={onClick} />