daniel543x/artcraft
1
0 Fork 0
mirror of https://github.com/storytold/artcraft.git synced 2026-11-04 17:38:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

1.Made changes to OOP for the timeline to be explicit to what the functions are intended for.

Browse Source 
2.Internal state vs external events added public and private modifiers.
3.Renamed the clips to from track clip to clip to reduce the confusion and solidify the communication.
4.Modified Scene Saving and Loading and refactored to make it more consistent and understandable.
This commit is contained in:
Michael Chung committed 2024-03-27 16:55:46 -04:00
1 parent 580bc86d7e
commit f3ca24183f
14 files changed
+467 -346
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/pages/PageEnigma/PageEnigmaComponent.tsx
+1 -1
View File
@@ -25,7 +25,7 @@ export const PageEnigmaComponent = () => {
useEffect(()=>{
setTimeout(()=>dispatchAppUiState({
type: ACTION_TYPES.HIDE_EDITOR_LOADER
}), 3000);
}), 500);
},[]);
return (
<div>
app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx
+3 -4
View File
@@ -9,8 +9,7 @@ export const ControlsTopButtons = () => {
const handleButtonSave = () => {
console.log(`SceneName is ${sceneName}`);
// editorEngine?.save();
editorEngine?.saveScene();
};
const handleButtonCameraView = () => {
@@ -23,11 +22,11 @@ export const ControlsTopButtons = () => {
const handleButtonLoad = () => {
//document.getElementById("load-upload")?.click();
if(editorEngine == null) {return;}
editorEngine._load_for_testing();
editorEngine.loadScene();
};
const handleButtonRender = () => {
console.log("Saving GLB to server for reference");
editorEngine?._upload_for_testing();
editorEngine?.saveScene();
//editorEngine?.togglePlayback();
};
const handleButtonPlay = () => {};
app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts → app/pages/PageEnigma/datastructures/clips/animation_clip.ts
+1 -13
View File
@@ -1,19 +1,7 @@
import { faL } from '@fortawesome/pro-solid-svg-icons';
import * as THREE from 'three';
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
export interface AnimationTrackClip {
version: number
media_id: string
object_uuid: string
type: "animation"
location: "glb" | "remote"
speed: number
length: number
clip_name: string
}
export class AnimationTrackClip implements AnimationTrackClip {
export class AnimationClip {
version: number;
media_id: string; // comes from the server
object_uuid: string;
app/pages/PageEnigma/datastructures/clips/audio_track_clip.ts → app/pages/PageEnigma/datastructures/clips/audio_clip.ts
+1 -8
View File
@@ -14,14 +14,7 @@ class AudioData implements AudioData{
}
}
export interface AudioTrackClip {
version: number;
media_id: string;
type: "audio";
volume: number;
}
export class AudioTrackClip implements AudioTrackClip {
export class AudioClip {
version: number;
media_id: string;
type: "audio" = "audio";
app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts → app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts
+2 -9
View File
@@ -16,13 +16,6 @@ class AudioData implements AudioData {
}
}
export interface LipSyncTrackClip {
version: number;
media_id: string;
type: "lipsync";
volume: number;
}
class BlendShapeHelper {
ah: number;
ee: number;
@@ -34,7 +27,7 @@ class BlendShapeHelper {
}
}
export class LipSyncTrackClip implements LipSyncTrackClip {
export class LipSyncClip {
version: number;
media_id: string;
type: "lipsync" = "lipsync";
@@ -98,7 +91,7 @@ export class LipSyncTrackClip implements LipSyncTrackClip {
}
async play(object: THREE.Object3D) {
async play(object: THREE.Object3D) {
if (this.audio_data?.audioBuffer == null) { await this.download_audio(); }
if (this.lipsync.face == null) {
this.lipsync = new LipSync(await this._detect_face(object));
app/pages/PageEnigma/datastructures/clips/transform_track_clip.ts → app/pages/PageEnigma/datastructures/clips/transform_clip.ts
+1 -12
View File
@@ -1,17 +1,6 @@
import { faL } from '@fortawesome/pro-solid-svg-icons';
import * as THREE from 'three';
export interface TransformTrackClip {
version: number;
media_id: string;
type: "transform";
positions: THREE.Vector3[];
rotations: THREE.Vector3[];
scales: THREE.Vector3[];
length: number;
}
export class TransformTrackClip implements TransformTrackClip {
export class TransformClip {
version: number;
media_id: string;
object_uuid: string;
app/pages/PageEnigma/datastructures/scene/scene_object.ts
-121
View File
@@ -1,121 +0,0 @@
import { AudioTrackClip } from "../clips/audio_track_clip";
import { AnimationTrackClip } from "../clips/animation_track_clip";
import { TransformTrackClip } from "../clips/transform_track_clip";
import { ClipOffset } from "../clips/clip_offset";
export interface Scene {
version: number;
glb_media_id: number; // Assuming a numerical ID, adjust as needed
skybox_type: "cube" | "six_sided_cube" | "sphere";
skybox_media_ids: { [key: string]: number };
timeline: Timeline;
}
export interface Timeline {
version: number;
last_scrubber_position: number;
audio_track_clips: AudioTrackClip[]; // Assuming defined elsewhere
transform_track_clips: TransformTrackClip[]; // Placeholder, define as needed
animation_track_clips: AnimationTrackClip[]; // Placeholder, define as needed
entities: Entity[];
globalAudio: GlobalAudio;
camera: Camera;
}
export interface Entity {
object_uuid: number;
clip_offsets: ClipOffset[];
}
export interface GlobalAudio {
clip_offsets: ClipOffset[];
}
export interface Camera {
clip_offsets: ClipOffset[];
}
export class Scene implements Scene {
version: number;
glb_media_id: number;
skybox_type: "cube" | "six_sided_cube" | "sphere";
skybox_media_ids: { [key: string]: number };
timeline: Timeline;
constructor(
version: number,
glb_media_id: number,
skybox_type: "cube" | "six_sided_cube" | "sphere",
skybox_image_ids: { [key: string]: number },
timeline: Timeline,
)
{
this.version = version;
this.glb_media_id = glb_media_id;
this.skybox_type = skybox_type;
this.skybox_media_ids = skybox_image_ids;
this.timeline = timeline;
}
toJSON(): string {
return JSON.stringify(this);
}
}
export class Timeline implements Timeline {
version: number;
last_scrubber_position: number;
audio_track_clips: AudioTrackClip[];
transform_track_clips: TransformTrackClip[];
animation_track_clips: AnimationTrackClip[];
entities: Entity[];
globalAudio: GlobalAudio;
camera: Camera;
constructor(
version: number,
last_scrubber_position: number,
audio_track_clips: AudioTrackClip[],
transform_track_clips: TransformTrackClip[],
animation_track_clips: AnimationTrackClip[],
entities: Entity[],
globalAudio: GlobalAudio,
camera: Camera,
)
{
this.version = version;
this.last_scrubber_position = last_scrubber_position;
this.audio_track_clips = audio_track_clips;
this.transform_track_clips = transform_track_clips;
this.animation_track_clips = animation_track_clips;
this.entities = entities;
this.globalAudio = globalAudio;
this.camera = camera;
}
toJSON(): string {
return JSON.stringify(this);
}
// For testing purposes
static generateMockJson(): string {
const mock = {
version: 1,
last_scrubber_position: 5000,
audio_track_clips: [
{ version: 1, uuid: 0, media_id: 1, type: "audio", volume: 100 }
],
transform_track_clips: [
{ version: 1, uuid: 1, media_id: 2, type: "transform", position: [{ x: 0, y: 0, z: 0 }], rotation: [{ x: 0, y: 0, z: 0 }], scale: [{ x: 1, y: 1, z: 1 }] }
],
animation_track_clips: [
{ version: 1, uuid: 2, media_id: 3, type: "animation", speed: 1.0, length: 100, starting_offset: 0, clip_name: "some clip name" }
],
entities: [
{ object_uuid: 0, clip_offsets: [] } // Assuming more details would be filled as necessary
],
globalAudio: { clip_offsets: [] }, // Assuming more details would be filled as necessary
camera: { clip_offsets: [] } // Assuming more details would be filled as necessary
};
return JSON.stringify(mock);
}
}
app/pages/PageEnigma/js/animation_engine.ts
+3 -3
View File
@@ -1,8 +1,8 @@
import { AnimationTrackClip } from "../datastructures/clips/animation_track_clip";
import { AnimationClip } from "../datastructures/clips/animation_clip";
import * as THREE from 'three';
export class AnimationEngine {
clips: { [key: string]: AnimationTrackClip } = {};
clips: { [key: string]: AnimationClip } = {};
version: number;
length: number;
@@ -13,7 +13,7 @@ export class AnimationEngine {
}
load_object(object_uuid: string, media_id:string, clip_name: string) {
this.clips[object_uuid] = new AnimationTrackClip(
this.clips[object_uuid] = new AnimationClip(
this.version,
media_id,
"glb",
app/pages/PageEnigma/js/api_manager.ts
+364 -135
View File
@@ -3,9 +3,37 @@ import { Scene } from "../datastructures/scene/scene_object";
import * as THREE from "three";
import { GLTFExporter } from "three/addons/exporters/GLTFExporter.js";
import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js";
import { TimelineCurrentReactState } from "./timeline";
import { TimelineDataState } from "./timeline";
/**
* Storyteller Studio API Manager
* The source of truth of all these media items is the database in the cloud
*/
/**
* This is designed to surface user customer facing messages as errors.
*/
type Tokens = { [key: string]: string };
class APIManagerResponseSuccess {
public user_message:String
public data:Tokens | null
constructor(user_message:String = "", data:Tokens | null = null) {
this.data = data
this.user_message = user_message
}
}
/**
* This is designed to surface user customer facing messages as errors.
* Errors shouldn't be 404 or something confusing should be
*/
class APIManagerResponseError extends Error {
constructor(message?: string) {
super(message);
this.name = "APIManagerResponseError";
}
}
// Basically the data will exist on the database
class APIManager {
baseUrl: String;
@@ -14,60 +42,116 @@ class APIManager {
//this.baseUrl = "http://localhost:12345"
}
// Function to save or update the scene
// it will always return a boolean success and the media_file_token
// they take an optional media_file_token as a parameter for the file to overwrite, which must belong to the user.
//If it's an anonymous user, we check the anonymous visitor token/cookie
/**
* @param scene The 3JS Scene we want to save.
* @param scene_name The Scene name we want to display
* @param scene_media_file_token If null we will we will create a new save or copy the scene, if provided we will overwrite the scene.
* @returns APIManagerResponseMessage
*/
public async saveSceneState(scene: THREE.Scene,
scene_name:string,
scene_media_file_token:string | null = null,
timeline_state:TimelineDataState):Promise<APIManagerResponseSuccess> {
const file = await this.gltfExport(scene)
console.log(`FILE: ${file}`)
// will overwrite the scene on db if token exists
const upload_response = await this.uploadGLB(file,scene_media_file_token)
await this.saveSceneAndTimeLineState(scene_media_file_token,scene_name,timeline_state)
return new APIManagerResponseSuccess("Scene Saved")
}
public async loadSceneState(scene_media_file_token: string): Promise<THREE.Scene> {
const api_base_url = "https://api.fakeyou.com"
const url = `${api_base_url}/v1/media_files/file/${scene_media_file_token}`
const response = await fetch(url)
const json = await JSON.parse(await response.text())
const bucket_path = json["media_file"]["public_bucket_path"]
const media_base_url = "https://storage.googleapis.com/vocodes-public"
const media_url = `${media_base_url}${bucket_path}` // gets you a bucket path
async saveSceneState(
scene: THREE.Scene,
timeline: TimelineCurrentReactState,
scene_name: string,
scene_media_file_token: string | null = null,
): Promise<string> {
// create new scene
let file = await this.gltfExport(scene);
let response = await this.uploadGLB(file);
let success = response["success"];
if (success) {
let media_file_token = response["media_file_token"];
console.log(`media file token: ${media_file_token}`)
const result = await this._saveSceneAndTimelineToJSONSpec(media_file_token,scene_name, timeline);
return result
}
throw Error("Saving the Scene Resulted in an Error Success False");
const file_response = await fetch(media_url)
if (!file_response.ok) {
throw new APIManagerResponseError("Failed to download file")
}
// Convert the response to a Blob
let blob = await file_response.blob()
const json_result:string = await new Promise((resolve, reject) => {
let reader = new FileReader()
reader.onloadend = () => resolve(JSON.parse(reader.result as string))
reader.onerror = reject
reader.readAsText(blob)
})
console.log(`JSON Result: ${json_result}`)
const scene_glb_media_file_id:string = json_result["glb_media_file_id"]
const media_bucket_path = await this.getMediaFile(scene_glb_media_file_id)
let glbLoader = new GLTFLoader()
// promisify this
const loadGlb = (bucket_path: string): Promise<THREE.Scene> => {
return new Promise((resolve, reject) => {
glbLoader.load(bucket_path, (glb) => {
if (glb) {
const scene:THREE.Scene = glb.scene
resolve(scene)
} else {
reject(new APIManagerResponseError('Failed to Load GLB Scene'))
}
})
})
}
loadGlb(media_bucket_path)
}
/**
* This recieves the bucket path of a media file
* @param media_file_token
* @returns
*/
private async getMediaFile(media_file_token: string): Promise<string> {
let api_base_url = "https://api.fakeyou.com";
let url = `${api_base_url}/v1/media_files/file/${media_file_token}`
let response = await fetch(url);
let json = await JSON.parse(await response.text());
let bucketPath = json["media_file"]["public_bucket_path"];
let media_base_url = "https://storage.googleapis.com/vocodes-public"
let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path
return media_url
}
/**
This will save the scene to keep ids positions.
It will also give the file a name which will be a uuidv4()
@param scene The 3JS Scene we want to make a file to be uploaded as multipart form.
*/
private async gltfExport(scene: THREE.Scene):Promise<File> {
const gltfExporter = new GLTFExporter();
const uuid = uuidv4();
const result = await gltfExporter.parseAsync(scene);
const file = new File([JSON.stringify(result)], `${uuid}.glb`, {
type: "application/json",
});
return file;
}
// For JSON files.
// Form-multipart POST with parameters:
// uuid_idempotency_token
// file
// media_file_token (optional; the file to replace if present)
async _saveSceneAndTimelineToJSONSpec(
media_file_token: string,
scene_file_name: string,
timeline: TimelineCurrentReactState,
): Promise<string> {
const url = `${this.baseUrl}/v1/media_files/write/scene_file`;
private async uploadGLB(file: File, scene_media_file_token:string | null): Promise<string> {
const url = `${this.baseUrl}/v1/media_files/write/engine_asset`;
let uuid = uuidv4();
const form_data = new FormData();
form_data.append("uuid_idempotency_token", uuid);
// turn json into a blob
const file_test = {"glb_media_file_id":media_file_token,"name":scene_file_name,"entities":[], "timeline":""}
const json = JSON.stringify(file_test);
const blob = new Blob([json], {type: 'application/json'});
const fileName = `${uuid}-${scene_file_name}.json`;
const formData = new FormData();
formData.append("uuid_idempotency_token", uuid);
formData.append("file", blob, fileName);
formData.append("source", "file");
formData.append("type", "scene_json");
formData.append("source", "file");
// update existing scene otherwise create new scene and use it's media_file_id
if (scene_media_file_token != null) {
form_data.append("media_file_token", scene_media_file_token);
}
form_data.append("file", file);
form_data.append("source", "file");
form_data.append("media_file_subtype", "scene_import");
form_data.append("media_file_class", "scene");
const response = await fetch(url, {
method: "POST",
@@ -75,82 +159,71 @@ class APIManager {
headers: {
Accept: "application/json",
},
body: formData,
body: form_data,
});
if (!response.ok) {
throw new Error("Failed to Send Data");
}
console.log(`saveSceneAndTimelineToJSONSpecResponse: ${response}`)
return response.json(); // or handle the response as appropriate
const json_data = await response.json()
// const json_string = JSON.stringify(json_data)
// console.log(`uploadGLB: ${json_string}`)
return json_data; // or handle the response as appropriate
}
// media_file_token that describes the scene
async loadScene(media_file_token: string): Promise<string> {
let api_base_url = "https://api.fakeyou.com";
let url = `${api_base_url}/v1/media_files/file/${media_file_token}`
let response = await fetch(url);
let json = await JSON.parse(await response.text());
let bucketPath = json["media_file"]["public_bucket_path"];
let media_base_url = "https://storage.googleapis.com/vocodes-public"
let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path
let fileResponse = await fetch(media_url);
private async saveSceneAndTimeLineState(
scene_media_file_token: string | null,
scene_file_name: string,
timeline_state: TimelineDataState | null, // only for now.
): Promise<string> {
const url = `${this.baseUrl}/v1/media_files/write/scene_file`
let uuid = uuidv4()
if (!fileResponse.ok) {
throw new Error("Failed to download file");
// turn json into a blob
const scene_schema = {"glb_media_file_id":scene_media_file_token,"scene_name":scene_file_name,"timeline": { "objects":[] } }
const json = JSON.stringify(scene_schema)
const blob = new Blob([json], {type: 'application/json'})
const file_name = `${uuid}.json`
const form_data = new FormData()
form_data.append("uuid_idempotency_token", uuid)
form_data.append("file", blob, file_name)
form_data.append("source", "file")
form_data.append("type", "scene_json")
form_data.append("source", "file")
const response = await fetch(url, {
method: "POST",
credentials: "include",
headers: {
Accept: "application/json",
},
body: form_data,
})
if (!response.ok) {
throw new Error("Failed to Send Data")
}
// Convert the response to a Blob
let blob = await fileResponse.blob();
const json_value:string = await new Promise((resolve, reject) => {
let reader = new FileReader();
reader.onloadend = () => resolve(JSON.parse(reader.result as string));
reader.onerror = reject;
reader.readAsText(blob);
});
console.log(json_value)
return json_value
}
const json_data = await response.json()
// Provides bucket path to the actual file to download from the media file
async getMediaFile(media_file_token: string): Promise<string> {
let api_base_url = "https://api.fakeyou.com";
let url = `${api_base_url}/v1/media_files/file/${media_file_token}`
let response = await fetch(url);
let json = await JSON.parse(await response.text());
let bucketPath = json["media_file"]["public_bucket_path"];
let media_base_url = "https://storage.googleapis.com/vocodes-public"
let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path
// TODO get it as binary.
return media_url
}
async batchGetMedia(media_tokens: []): Promise<string> {
return "";
// example response {"success":true,"media_file_token":"m_r1ztnn501g2rn0vv2np08nd6zy2fvt"}
console.log(`saveSceneAndTimeLineState: ${JSON.stringify(json_data)}`)
// should return the same token if it is same file else new token.
return json_data // or handle the response as appropriate
}
async gltfExport(scene: THREE.Scene) {
let gltfExporter = new GLTFExporter();
let uuid = uuidv4();
// save the scene to keep ids positions etc
let result = await gltfExporter.parseAsync(scene);
const file = new File([JSON.stringify(result)], `${uuid}.glb`, {
type: "application/json",
});
return file;
}
// two core functions to upload things to the server and get id's
async uploadMedia(blob: any, fileName: string): Promise<string> {
private async uploadMedia(blob:any,fileName:string):Promise<APIManagerResponseSuccess> {
const url = `${this.baseUrl}/v1/media_uploads/upload`;
let uuid = uuidv4();
const formData = new FormData();
formData.append("uuid_idempotency_token", uuid);
formData.append("file", blob, fileName);
formData.append("source", "file");
formData.append("type", "video");
formData.append("source", "file");
const response = await fetch(url, {
method: "POST",
credentials: "include",
@@ -161,11 +234,167 @@ class APIManager {
});
if (!response.ok) {
throw new Error("Failed to Send Data");
throw new APIManagerResponseError("Upload Media Failed to send file")
}
else {
const json_data = await response.json()
return new APIManagerResponseSuccess("Uploaded Media Success")
}
return response.json();
}
public async batchGetMedia(media_tokens: []): Promise<string> {
return "";
}
// Function to save or update the scene
// it will always return a boolean success and the media_file_token
// they take an optional media_file_token as a parameter for the file to overwrite, which must belong to the user.
// If it's an anonymous user, we check the anonymous visitor token/cookie
// async saveSceneState2(
// scene: THREE.Scene,
// timeline: TimelineDataState,
// scene_name: string,
// scene_media_file_token: string | null = null,
// ): Promise<string> {
// // create new scene
// let file = await this.gltfExport(scene);
// let response = await this.uploadGLB(file);
// let success = response["success"];
// if (success) {
// let media_file_token = response["media_file_token"];
// console.log(`media file token: ${media_file_token}`)
// const result = await this._saveSceneAndTimelineToJSONSpec(media_file_token,scene_name, timeline);
// return result
// }
// throw Error("Saving the Scene Resulted in an Error Success False");
// }
// // For JSON files.
// // Form-multipart POST with parameters:
// // uuid_idempotency_token
// // file
// // media_file_token (optional; the file to replace if present)
// async _saveSceneAndTimelineToJSONSpec(
// media_file_token: string,
// scene_file_name: string,
// timeline: TimelineDataState,
// ): Promise<string> {
// const url = `${this.baseUrl}/v1/media_files/write/scene_file`;
// let uuid = uuidv4();
// // turn json into a blob
// const file_test = {"glb_media_file_id":media_file_token,"name":scene_file_name,"entities":[], "timeline":""}
// const json = JSON.stringify(file_test);
// const blob = new Blob([json], {type: 'application/json'});
// const fileName = `${uuid}-${scene_file_name}.json`;
// const formData = new FormData();
// formData.append("uuid_idempotency_token", uuid);
// formData.append("file", blob, fileName);
// formData.append("source", "file");
// formData.append("type", "scene_json");
// formData.append("source", "file");
// const response = await fetch(url, {
// method: "POST",
// credentials: "include",
// headers: {
// Accept: "application/json",
// },
// body: formData,
// });
// if (!response.ok) {
// throw new Error("Failed to Send Data");
// }
// console.log(`saveSceneAndTimelineToJSONSpecResponse: ${response}`)
// return response.json(); // or handle the response as appropriate
// }
// // media_file_token that describes the scene
// async loadScene(media_file_token: string): Promise<string> {
// let api_base_url = "https://api.fakeyou.com";
// let url = `${api_base_url}/v1/media_files/file/${media_file_token}`
// let response = await fetch(url);
// let json = await JSON.parse(await response.text());
// let bucketPath = json["media_file"]["public_bucket_path"];
// let media_base_url = "https://storage.googleapis.com/vocodes-public"
// let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path
// let fileResponse = await fetch(media_url);
// if (!fileResponse.ok) {
// throw new Error("Failed to download file");
// }
// // Convert the response to a Blob
// let blob = await fileResponse.blob();
// const json_value:string = await new Promise((resolve, reject) => {
// let reader = new FileReader();
// reader.onloadend = () => resolve(JSON.parse(reader.result as string));
// reader.onerror = reject;
// reader.readAsText(blob);
// });
// console.log(json_value)
// return json_value
// }
// // Provides bucket path to the actual file to download from the media file
// async getMediaFile(media_file_token: string): Promise<string> {
// let api_base_url = "https://api.fakeyou.com";
// let url = `${api_base_url}/v1/media_files/file/${media_file_token}`
// let response = await fetch(url);
// let json = await JSON.parse(await response.text());
// let bucketPath = json["media_file"]["public_bucket_path"];
// let media_base_url = "https://storage.googleapis.com/vocodes-public"
// let media_url = `${media_base_url}${bucketPath}` // gets you a bucket path
// // TODO get it as binary.
// return media_url
// }
// async gltfExport2(scene: THREE.Scene) {
// let gltfExporter = new GLTFExporter();
// let uuid = uuidv4();
// // save the scene to keep ids positions etc
// let result = await gltfExporter.parseAsync(scene);
// const file = new File([JSON.stringify(result)], `${uuid}.glb`, {
// type: "application/json",
// });
// return file;
// }
// // two core functions to upload things to the server and get id's
// async uploadMedia2(blob: any, fileName: string): Promise<string> {
// const url = `${this.baseUrl}/v1/media_uploads/upload`;
// let uuid = uuidv4();
// const formData = new FormData();
// formData.append("uuid_idempotency_token", uuid);
// formData.append("file", blob, fileName);
// formData.append("source", "file");
// formData.append("type", "video");
// formData.append("source", "file");
// const response = await fetch(url, {
// method: "POST",
// credentials: "include",
// headers: {
// Accept: "application/json",
// },
// body: formData,
// });
// if (!response.ok) {
// throw new Error("Failed to Send Data");
// }
// return response.json();
// }
/**
To save the scene as a file get the id back to serialize
/v1/media_files/write/engine_asset
@@ -179,35 +408,35 @@ class APIManager {
media_file_subtype (optional; mixamo, mocap_net, scene_import, animation_only)
media_file_class (audio, image, video, animation, character, prop, scene, unknown)
**/
async uploadGLB(file: File, scene_media_file_token:string | null): Promise<string> {
const url = `${this.baseUrl}/v1/media_files/write/engine_asset`;
let uuid = uuidv4();
const formData = new FormData();
formData.append("uuid_idempotency_token", uuid);
// async uploadGLB2(file: File, scene_media_file_token:string | null): Promise<string> {
// const url = `${this.baseUrl}/v1/media_files/write/engine_asset`;
// let uuid = uuidv4();
// const formData = new FormData();
// formData.append("uuid_idempotency_token", uuid);
// update existing scene otherwise create new scene and use it's media_file_id
if (scene_media_file_token != null) {
formData.append("media_file_token", scene_media_file_token);
}
// // update existing scene otherwise create new scene and use it's media_file_id
// if (scene_media_file_token != null) {
// formData.append("media_file_token", scene_media_file_token);
// }
formData.append("file", file);
formData.append("source", "file");
formData.append("media_file_subtype", "scene_import");
formData.append("media_file_class", "scene");
const response = await fetch(url, {
method: "POST",
credentials: "include",
headers: {
Accept: "application/json",
},
body: formData,
});
// formData.append("file", file);
// formData.append("source", "file");
// formData.append("media_file_subtype", "scene_import");
// formData.append("media_file_class", "scene");
// const response = await fetch(url, {
// method: "POST",
// credentials: "include",
// headers: {
// Accept: "application/json",
// },
// body: formData,
// });
if (!response.ok) {
throw new Error("Failed to Send Data");
}
return response.json(); // or handle the response as appropriate
}
// if (!response.ok) {
// throw new Error("Failed to Send Data");
// }
// return response.json(); // or handle the response as appropriate
// }
}
export default APIManager;
app/pages/PageEnigma/js/audio_engine.ts
+3 -3
View File
@@ -1,8 +1,8 @@
import { faL } from "@fortawesome/pro-solid-svg-icons";
import { AudioTrackClip } from "../datastructures/clips/audio_track_clip";
import { AudioClip } from "../datastructures/clips/audio_clip";
class AudioEngine {
clips: { [key: string]: AudioTrackClip } = {};
clips: { [key: string]: AudioClip } = {};
audio_sources: { [key: string]: AudioBufferSourceNode } = {};
version: number;
playing: string[];
@@ -17,7 +17,7 @@ class AudioEngine {
// loads clips into the engine to cache
loadClip(audio_media_id: string) {
if(this.clips[audio_media_id] != null) { return; }
this.clips[audio_media_id] = new AudioTrackClip(this.version, audio_media_id, 1.0);
this.clips[audio_media_id] = new AudioClip(this.version, audio_media_id, 1.0);
}
// plays from the timeline.
app/pages/PageEnigma/js/editor.js
+35 -7
View File
@@ -16,7 +16,7 @@ import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js";
import AudioEngine from "./audio_engine.ts";
import TransformEngine from "./transform_engine.ts";
import { TimeLine } from "./timeline.ts";
import { TimeLine, TimelineDataState } from "./timeline.ts";
import { ClipUI } from "../datastructures/clips/clip_offset.ts";
import { LipSync } from "./lipsync.js";
@@ -36,6 +36,13 @@ if (typeof window !== "undefined") {
}
class EditorState {
// {
// action: "ShowLoadingIndicator"
// source: "Editor"
// data: { "message" : "saving scene" }
// }
constructor() {
this.selected_object = null;
this.is_loading = false;
@@ -104,6 +111,7 @@ class Editor {
this.transform_engine = new TransformEngine();
this.lipsync_engine = new LipSyncEngine();
this.animation_engine = new AnimationEngine();
this.timeline = new TimeLine(
this.audio_engine,
this.transform_engine,
@@ -168,19 +176,39 @@ class Editor {
this.timeline.scene = this.activeScene;
this._test_demo()
//this._test_demo()
// saving state of the scene
this.scene_file_token = null
}
// Token comes in from the front end to load the scene from the site.
async loadScene() {
const scene = await this.api_manager.loadSceneState(this.test_scene_load_media_id);
this.activeScene.scene.children = scene.children;
this.activeScene.scene.children.forEach(child => {
child.parent = this.activeScene.scene;
if(child.type == "DirectionalLight") {
let pos = child.position;
let rot = child.rotation;
let light = this.activeScene._create_base_lighting();
light.position.set(pos.x, pos.y, pos.z);
light.rotation.set(rot.x, rot.y, rot.z);
this.activeScene.scene.remove(child);
}
})
}
async saveScene(name) {
// check if I already have the scene token.
console.log("Saving...");
const result = await this.api_manager.saveSceneState(this.activeScene.scene,this.scene_file_token);
console.log("Saved!");
const result = await this.api_manager.saveSceneState(this.activeScene.scene,name,
this.scene_file_token,
new TimelineDataState());
// dispatch call wil's engine.
}
// uploading some objects for testing to get their media ids from my account.
app/pages/PageEnigma/js/lip_sync_engine.ts
+4 -6
View File
@@ -1,24 +1,22 @@
import { LipSyncTrackClip } from "../datastructures/clips/lipsync_track_clip";
import { LipSync } from "./lipsync";
import * as THREE from 'three';
import { LipSyncClip } from "../datastructures/clips/lipsync_clip";
// needs to be apart of the editor window.
export class LipSyncEngine {
clips: { [key: string]: LipSyncTrackClip } = {}
clips: { [key: string]: LipSyncClip } = {}
audio_sources: { [key: string]: AudioBufferSourceNode } = {}
version: number
constructor() {
this.clips = {}
this.audio_sources = {}
this.version = 1.0
}
load_object(object_uuid: string, audio_media_id: string) {
this.clips[object_uuid] = new LipSyncTrackClip(this.version, audio_media_id, 1.0);
this.clips[object_uuid] = new LipSyncClip(this.version, audio_media_id, 1.0);
}
}
app/pages/PageEnigma/js/timeline.ts
+45 -20
View File
@@ -6,18 +6,46 @@ import AudioEngine from "./audio_engine";
import TransformEngine from "./transform_engine";
import LipSyncEngine from "./lip_sync_engine";
import AnimationEngine from "./animation_engine";
import { AudioClip } from "../models/track";
import { AnimationClip } from "three";
import { LipSyncClip } from "../datastructures/clips/lipsync_clip";
import { TransformClip } from "../datastructures/clips/transform_clip";
// Every object uuid / entity has a track.
export class TimelineCurrentReactState {
isEditable: boolean
export class TimelineCurrentState {
isEditable: boolean
selectedObjectID: number
constructor() {
this.isEditable = true // can add clips to it
this.selectedObjectID = 0
}
}
export class TimelineDataState {
timelineItems: ClipUI[]
scrubberPosition: number
// Data to serialize with the scene, used to load into each engine.
audioClips:AudioClip[]
animationClips:AnimationClip[]
lipSyncClips:LipSyncClip[]
transformClips:TransformClip[]
constructor(timelineItems: ClipUI[] = [],
audioClips: AudioClip[] = [],
animationClips: AnimationClip[] = [],
transformClips: TransformClip[] = [],
lipSyncClips: LipSyncClip[] = [],
scrubberPosition:number = 0) {
this.timelineItems = timelineItems
this.scrubberPosition = scrubberPosition
this.audioClips = audioClips
this.animationClips = animationClips
this.lipSyncClips = lipSyncClips
this.transformClips = transformClips
}
}
export class TimeLine {
timelineItems: ClipUI[]
@@ -26,8 +54,6 @@ export class TimeLine {
scrubberPosition: number
isPlaying: boolean
timelineState: TimelineCurrentReactState
// plays audio
audioEngine: AudioEngine
// key framing
@@ -50,7 +76,6 @@ export class TimeLine {
this.isPlaying = false
this.scrubberPosition = 0 // in frames into the tl
this.timelineState = new TimelineCurrentReactState()
// this will be used to play the audio clips
this.audioEngine = audioEngine
@@ -61,60 +86,60 @@ export class TimeLine {
this.scene = scene;
}
async addPlayableClip(clip: ClipUI): Promise<void> {
public async addPlayableClip(clip: ClipUI): Promise<void> {
this.timelineItems.push(clip)
}
// when given a media id item it will create the clip.
// Then the clip will be loaded by the engines, if they come from outside of the loaded scene.
async createClipOffset(media_id: string, object_uuid:string, type: string): Promise<void> {
public async createClipOffset(media_id: string, object_uuid:string, type: string): Promise<void> {
// use engine to load based off media id and type animation | transform |
}
// this will update the state of the clips based off uuid easing?
async updateClip(clip_uuid: string, updates: AnyJson): Promise<void> {
public async updateClip(clip_uuid: string, updates: AnyJson): Promise<void> {
}
async deleteClip(clip_uuid: string): Promise<void> {
public async deleteClip(clip_uuid: string): Promise<void> {
}
// Events that will trigger from react
async clipDidEnterDropZone() {
public async clipDidEnterDropZone() {
}
async clipDidExitDropZone() {
public async clipDidExitDropZone() {
}
// timeline controls this.
async scrubberDidStart(offset_frame: number) {
public async scrubberDidStart(offset_frame: number) {
}
async scrub(offset_frame: number): Promise<void> {
public async scrub(offset_frame: number): Promise<void> {
// only stream through to the position and rotation keyframes
// debounce not really
}
async scrubberDidStop(offset_frame: number) {
public async scrubberDidStop(offset_frame: number) {
}
// public streaming events into the timeline from
async setScrubberPosition(offset: number) {
public async setScrubberPosition(offset: number) {
this.scrubberPosition = offset // in ms
}
// should play from the clip that is closest to the to scrubber
async play(): Promise<void> {
public async play(): Promise<void> {
console.log(`Starting Timeline`)
this.isPlaying = true
}
async reset_scene() {
private async reset_scene() {
for (const element of this.timelineItems) {
if (element.type == "transform") {
let object = this.scene.get_object_by_uuid(element.object_uuid);
@@ -136,7 +161,7 @@ export class TimeLine {
}
// called by the editor update loop on each frame
async update(deltatime:number) {
public async update(deltatime:number) {
if (this.isPlaying == false) return; // start and stop
if (this.scrubberPosition <= 0) {
@@ -191,7 +216,7 @@ export class TimeLine {
}
}
async stop(): Promise<void> {
private async stop(): Promise<void> {
this.isPlaying = false
console.log(`Stopping Timeline`)
}
app/pages/PageEnigma/js/transform_engine.ts
+4 -4
View File
@@ -1,8 +1,8 @@
import { TransformTrackClip } from "../datastructures/clips/transform_track_clip";
import { TransformClip } from "../datastructures/clips/transform_clip";
import * as THREE from 'three';
class TransformEngine {
clips: { [key: string]: TransformTrackClip } = {};
clips: { [key: string]: TransformClip } = {};
version: number;
length: number;
@@ -13,13 +13,13 @@ class TransformEngine {
}
loadObject(object_uuid: string, clip_length:number = 2) {
this.clips[object_uuid] = new TransformTrackClip(this.version, object_uuid, clip_length, object_uuid); // replace the last object_uuid with the media ID when its ready.
this.clips[object_uuid] = new TransformClip(this.version, object_uuid, clip_length, object_uuid); // replace the last object_uuid with the media ID when its ready.
return this.clips[object_uuid]
}
addFrame(object: THREE.Object3D, clip_length:number = 2) {
if(this.clips[object.uuid] == null) {
this.clips[object.uuid] = new TransformTrackClip(this.version, object.uuid, clip_length);
this.clips[object.uuid] = new TransformClip(this.version, object.uuid, clip_length);
}
this.clips[object.uuid].add_position(object.position);
this.clips[object.uuid].add_rotation(new THREE.Vector3(object.rotation.x, object.rotation.y, object.rotation.z));