daniel543x/artcraft
1
0 Fork 0
mirror of https://github.com/storytold/artcraft.git synced 2026-11-04 01:18:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

back merge

Browse Source
This commit is contained in:
wil committed 2024-04-06 21:15:44 -04:00
2 parents 59a308b0a4 9b80285c9f
commit 1fb20e2e54
18 files changed
+568 -227
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/modules/LowerPanel/LowerPanel.tsx
+11 -2
View File
@@ -1,6 +1,7 @@
import React, { useCallback } from "react";
import React, { UIEvent, UIEventHandler, useCallback } from "react";
import { useMouseEventsHeight } from "~/pages/PageEnigma/comps/Timeline/utils/useMouseEventsHeight";
import {
currentScroll,
currentTime,
overTimeline,
scale,
@@ -22,7 +23,10 @@ export const LowerPanel = ({ children }: LowerPanelPropsI) => {
const onTimelineClick = useCallback(
(event: React.PointerEvent<HTMLDivElement>) => {
if (event.button === 0) {
const newTime = Math.round((event.clientX - 92) / 4 / scale.value);
console.log("??", event.clientX, event.pageX, currentScroll.value);
const newTime = Math.round(
(event.clientX + currentScroll.value - 92) / 4 / scale.value,
);
if (newTime < 0) {
return;
}
@@ -37,6 +41,10 @@ export const LowerPanel = ({ children }: LowerPanelPropsI) => {
[],
);
const onScroll = useCallback((event: UIEvent<HTMLDivElement>) => {
currentScroll.value = event.currentTarget.scrollLeft;
}, []);
return (
<>
<div
@@ -56,6 +64,7 @@ export const LowerPanel = ({ children }: LowerPanelPropsI) => {
}}
onPointerLeave={() => (overTimeline.value = false)}
onPointerDown={onTimelineClick}
onScroll={onScroll}
>
{children}
</div>
app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx
+27
View File
@@ -11,10 +11,17 @@ export const ControlsTopButtons = () => {
const [sceneName, setSceneName] = useState<string>("");
const [sceneToken, setSceneToken] = useState<string>("");
// for testing
const [mediaToken, setMediaToken] = useState<string>("");
const handleButtonSave = () => {
editorEngine?.saveScene(sceneName);
};
const handleMediaToken = async () => {
await editorEngine?.test_loadMediaToken(mediaToken)
};
const handleButtonLoadScene = () => {
editorEngine?.loadScene(sceneToken);
};
@@ -62,6 +69,26 @@ export const ControlsTopButtons = () => {
/>
</ButtonDialogue>
<ButtonDialogue
buttonProps={{
variant: "secondary",
label: "Add Scene Object (Test)",
}}
confirmButtonProps={{
label: "Save",
disabled: mediaToken === "",
onClick: handleMediaToken,
}}
title="Add Scene Object via Media Token"
>
<Input
label="Please Enter a Media Token"
onChange={(e) => {
setMediaToken(e.target.value);
}}
/>
</ButtonDialogue>
<ButtonDialogue
buttonProps={{
variant: "secondary",
app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsAnimation.ts
+10 -3
View File
@@ -1,5 +1,10 @@
import React, { useCallback, useEffect, useState } from "react";
import { currentTime, filmLength, scale } from "~/pages/PageEnigma/store";
import {
currentScroll,
currentTime,
filmLength,
scale,
} from "~/pages/PageEnigma/store";
import Queue from "~/pages/PageEnigma/Queue/Queue";
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
@@ -21,11 +26,13 @@ export const useMouseEventsAnimation = () => {
Queue.publish({
queueName: QueueNames.TO_ENGINE,
action: toEngineActions.UPDATE_TIME,
data: { currentTime: Math.round(time) },
data: { currentTime: Math.round(time) + currentScroll.value },
});
}
};
console.log("scroll", currentScroll.value);
const onMouseMove = (event: MouseEvent) => {
const delta = Math.round(
(event.clientX - clientX) / 4 / scale.value + currentTime.value,
@@ -41,7 +48,7 @@ export const useMouseEventsAnimation = () => {
Queue.publish({
queueName: QueueNames.TO_ENGINE,
action: toEngineActions.UPDATE_TIME,
data: { currentTime: delta },
data: { currentTime: delta + currentScroll.value },
});
}
return delta;
app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts
+1 -1
View File
@@ -38,7 +38,7 @@ export const useMouseEventsClip = (
if (deltaOffset < min) {
currOffset.current = min;
} else if (deltaOffset + currLength.current > max) {
currOffset.current = max;
currOffset.current = max - currLength.current;
} else {
currOffset.current = deltaOffset;
}
app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx
+7
View File
@@ -129,6 +129,13 @@ export const TrackProvider = ({ children }: Props) => {
type: ClipType.AUDIO,
length: 25,
name: "Talk",
},
{
version: 1,
media_id: "m_w5nn3kjh1fbkmjrdac5b2qaba0pmyt",
type: ClipType.AUDIO,
length: 25,
name: "NCS Song",
}
]);
setScale(1);
app/pages/PageEnigma/datastructures/clips/animation_clip.ts
+3 -3
View File
@@ -101,8 +101,8 @@ export class AnimationClip {
this.mixer?.setTime(0);
}
toJSON(): string {
return JSON.stringify({
toJSON(): any {
return {
version: this.version,
media_id: this.media_id,
object_uuid: this.object_uuid,
@@ -110,6 +110,6 @@ export class AnimationClip {
speed: this.speed,
length: this.length,
clip_name: this.clip_name,
})
};
}
}
app/pages/PageEnigma/datastructures/clips/audio_clip.ts
+3 -3
View File
@@ -52,12 +52,12 @@ export class AudioClip {
return new AudioData(audioContext, audioBuffer);
}
toJSON(): string {
return JSON.stringify({
toJSON(): any {
return {
version: this.version,
media_id: this.media_id,
type: this.type,
volume: this.volume,
});
};
}
}
app/pages/PageEnigma/datastructures/clips/clip_ui.ts
+3 -3
View File
@@ -34,8 +34,8 @@ export class ClipUI {
this.length = length;
}
toJSON(): string {
return JSON.stringify({
toJSON(): any {
return {
version: this.version,
group: this.group,
name: this.name,
@@ -44,6 +44,6 @@ export class ClipUI {
media_id: this.media_id,
start_offset: this.offset,
ending_offset: this.length,
});
};
}
}
app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts
+3 -3
View File
@@ -129,12 +129,12 @@ export class LipSyncClip {
this.lipsync = new LipSync();
}
toJSON(): string {
return JSON.stringify({
toJSON(): any {
return {
version: this.version,
media_id: this.media_id,
type: this.type,
volume: this.volume,
});
};
}
}
app/pages/PageEnigma/datastructures/clips/transform_clip.ts
+3 -3
View File
@@ -140,14 +140,14 @@ export class TransformClip {
keyframe_uuid));
}
toJSON(): string {
return JSON.stringify({
toJSON(): any {
return {
version: this.version,
media_id: this.media_id,
object_uuid: this.object_uuid,
length: this.length,
type: this.type,
keyframes: this.keyframes,
});
};
}
}
app/pages/PageEnigma/js/api_manager.ts
+59 -60
View File
@@ -88,43 +88,43 @@ export class APIManager {
* @returns APIManagerResponseMessage
*/
public async saveSceneState(
scene: THREE.Scene,
save_json: string,
scene_name: string,
scene_glb_media_file_token: string | null = null,
scene_media_file_token: string | null = null,
timeline_state: TimelineDataState | null = null,
): Promise<APIManagerResponseSuccess> {
const file = await this.gltfExport(scene);
): Promise<string> {
const file = new File([save_json], `${scene_name}.glb`, {
type: "application/json",
});
// will overwrite the scene on db if token exists
const upload_glb_response = await this.uploadGLB(
const upload_glb_response = await this.uploadEngineAsset(
file,
scene_glb_media_file_token,
);
const result_scene_glb_media_file_token =
upload_glb_response["media_file_token"];
// now write the scene
const save_scene_timeline_response = await this.saveSceneAndTimeLineState(
result_scene_glb_media_file_token,
scene_media_file_token,
scene_name,
timeline_state,
);
//const result_scene_glb_media_file_token =
// upload_glb_response["media_file_token"];
//// now write the scene
//const save_scene_timeline_response = await this.saveSceneAndTimeLineState(
// result_scene_glb_media_file_token,
// scene_media_file_token,
// scene_name,
// timeline_state,
//);
//const result_scene_media_file_token =
// save_scene_timeline_response["media_file_token"];
//const data = {
// scene_glb_media_file_token: result_scene_glb_media_file_token,
// scene_media_file_token: result_scene_media_file_token,
//};
const result_scene_media_file_token =
save_scene_timeline_response["media_file_token"];
const data = {
scene_glb_media_file_token: result_scene_glb_media_file_token,
scene_media_file_token: result_scene_media_file_token,
};
return new APIManagerResponseSuccess("Scene Saved", data);
return upload_glb_response['media_file_token'];
}
public async loadSceneState(
scene_media_file_token: string | null,
): Promise<APIManagerResponseSuccess> {
): Promise<any> {
const api_base_url = "https://api.fakeyou.com";
const url = `${api_base_url}/v1/media_files/file/${scene_media_file_token}`;
const response = await fetch(url);
@@ -148,38 +148,38 @@ export class APIManager {
});
console.log(`loadSceneState: ${JSON.stringify(json_result)}`);
return json_result;
const scene_glb_media_file_token: string =
json_result["scene_glb_media_file_token"];
// const scene_glb_media_file_token: string =
// json_result["scene_glb_media_file_token"];
// const media_bucket_path = await this.getMediaFile(
// scene_glb_media_file_token,
// );
// console.log(`GLB ${media_bucket_path}`);
// const glbLoader = new GLTFLoader();
// // promisify this
// const loadGlb = (
// bucket_path: string,
// ): Promise<APIManagerResponseSuccess> => {
// return new Promise((resolve, reject) => {
// glbLoader.load(bucket_path, (glb) => {
// if (glb) {
// const scene: THREE.Scene = glb.scene;
// const data = {
// scene_glb_media_file_token: scene_glb_media_file_token,
// scene_media_file_token: scene_media_file_token,
// scene: scene,
// };
// console.log(`Data: ${data}`);
// resolve(new APIManagerResponseSuccess("Success Loaded", data));
// } else {
// throw new APIManagerResponseError("Failed to Load GLB Scene");
// }
// });
// });
// };
const media_bucket_path = await this.getMediaFile(
scene_glb_media_file_token,
);
console.log(`GLB ${media_bucket_path}`);
const glbLoader = new GLTFLoader();
// promisify this
const loadGlb = (
bucket_path: string,
): Promise<APIManagerResponseSuccess> => {
return new Promise((resolve, reject) => {
glbLoader.load(bucket_path, (glb) => {
if (glb) {
const scene: THREE.Scene = glb.scene;
const data = {
scene_glb_media_file_token: scene_glb_media_file_token,
scene_media_file_token: scene_media_file_token,
scene: scene,
};
console.log(`Data: ${data}`);
resolve(new APIManagerResponseSuccess("Success Loaded", data));
} else {
throw new APIManagerResponseError("Failed to Load GLB Scene");
}
});
});
};
return await loadGlb(media_bucket_path);
//return await loadGlb(media_bucket_path);
}
/**
@@ -213,18 +213,18 @@ export class APIManager {
return file;
}
private async uploadGLB(
private async uploadEngineAsset(
file: File,
scene_glb_media_file_token: string | null,
): Promise<string> {
media_file_token: string | null,
): Promise<any> {
const url = `${this.baseUrl}/v1/media_files/write/engine_asset`;
const uuid = uuidv4();
const form_data = new FormData();
form_data.append("uuid_idempotency_token", uuid);
// update existing scene otherwise create new glb scene and use it's media_file_id
if (scene_glb_media_file_token != null) {
form_data.append("media_file_token", scene_glb_media_file_token);
if (media_file_token != null) {
form_data.append("media_file_token", media_file_token);
}
form_data.append("file", file);
@@ -245,7 +245,6 @@ export class APIManager {
throw new Error("Failed to Send Data");
} else {
const json_data = await response.json();
console.log(`uploadGLB: ${JSON.stringify(json_data)}`);
return json_data; // or handle the response as appropriate
}
}
@@ -310,7 +309,7 @@ export class APIManager {
blob: any,
fileName: string,
): Promise<APIManagerResponseSuccess> {
const url = `${this.baseUrl}/v1/media_uploads/upload`;
const url = `${this.baseUrl}/v1/media_files/upload`;
const uuid = uuidv4();
const formData = new FormData();
app/pages/PageEnigma/js/editor.ts
+126 -144
View File
@@ -25,6 +25,9 @@ import { APPUI_ACTION_TYPES } from "../reducers";
import { ClipGroup } from "~/pages/PageEnigma/models/track";
import { XYZ } from "../datastructures/common";
import { StoryTellerProxyScene } from "../proxy/storyteller_proxy_scene";
import { StoryTellerProxyTimeline } from "../proxy/storyteller_proxy_timeline";
class EditorState {
// {
// action: "ShowLoadingIndicator"
@@ -91,6 +94,7 @@ class Editor {
cam_obj: THREE.Object3D | undefined;
renderPass: RenderPass | undefined;
generating_preview: boolean;
frames: number;
camera_person_mode: boolean;
current_scene_media_token: string | null;
@@ -107,6 +111,10 @@ class Editor {
last_scrub: number;
// Default params.
// scene proxy for serialization
storyteller_proxy_scene:StoryTellerProxyScene;
constructor() {
console.log(
"If you see this message twice! then it rendered twice, if you see it once it's all good.",
@@ -172,6 +180,7 @@ class Editor {
this.playback_location = 0;
this.max_length = 10;
this.last_scrub = 0;
this.frames = 0;
// Audio Engine Test.
this.render_width = 1280;
@@ -206,6 +215,9 @@ class Editor {
"((masterpiece, best quality, 8K, detailed)), colorful, epic, fantasy, (fox, red fox:1.2), no humans, 1other, ((koi pond)), outdoors, pond, rocks, stones, koi fish, ((watercolor))), lilypad, fish swimming around.";
this.negative_prompt = "";
this.art_style = ArtStyle.Anime2DFlat;
this.storyteller_proxy_scene = new StoryTellerProxyScene(this.version,this.activeScene.scene)
}
initialize(config: any) {
@@ -361,21 +373,10 @@ class Editor {
console.log(result);
}
public async testTestTimelineEvents() {}
public async testTestTimelineEvents() { }
public async testStylizeRequest() {
const result = await this.api_manager
.stylizeVideo(
"mu_6wy1570a0c3c0tpkkncf4tsvb5234",
this.art_style,
this.positive_prompt,
this.negative_prompt,
Visibility.Public,
)
.catch((error) => {
console.log(error);
});
console.log(result);
public async testLoadTokenAssets() {
await this.storyteller_proxy_scene.loadFromMediaFileToken("m_z4jzbst3xfh64h0qn4bqh4afenfps9")
}
public async loadScene(scene_media_token: string) {
@@ -383,49 +384,30 @@ class Editor {
type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER,
});
if (scene_media_token != null) {
this.current_scene_media_token = scene_media_token;
}
this.current_scene_media_token = scene_media_token;
const load_scene_state_response = await this.api_manager.loadSceneState(
const scene_json = await this.api_manager.loadSceneState(
this.current_scene_media_token,
);
console.log(load_scene_state_response);
if (load_scene_state_response.data == null) {
console.log("load_scene_state_response Missing Data");
return;
}
let proxyScene = new StoryTellerProxyScene(this.version, this.activeScene);
await proxyScene.loadFromSceneJson(scene_json['scene']);
this.cam_obj = this.activeScene.get_object_by_name("::CAM::");
const loaded_scene = load_scene_state_response.data["scene"];
let proxyTimeline = new StoryTellerProxyTimeline(this.version, this.timeline, this.transform_engine, this.animation_engine, this.audio_engine, this.lipsync_engine);
await proxyTimeline.loadFromJson(scene_json['timeline']);
if (load_scene_state_response.data == null) {
return;
}
// Load these so you can rewrite the scene glb using it's token.
this.current_scene_media_token =
load_scene_state_response.data["scene_media_file_token"];
this.current_scene_glb_media_token =
load_scene_state_response.data["scene_glb_media_file_token"];
console.log(
`loadScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`,
);
this.activeScene.scene.children = loaded_scene.children;
this.activeScene.scene.children.forEach((child: THREE.Object3D) => {
child.parent = this.activeScene.scene;
if (child.type == "DirectionalLight") {
const pos = child.position;
const rot = child.rotation;
const light = this.activeScene._create_base_lighting();
light.position.set(pos.x, pos.y, pos.z);
light.rotation.set(rot.x, rot.y, rot.z);
this.activeScene.scene.remove(child);
}
});
// this.activeScene.scene.children.forEach((child: THREE.Object3D) => {
// child.parent = this.activeScene.scene;
// if (child.type == "DirectionalLight") {
// const pos = child.position;
// const rot = child.rotation;
// const light = this.activeScene._create_base_lighting();
// light.position.set(pos.x, pos.y, pos.z);
// light.rotation.set(rot.x, rot.y, rot.z);
// this.activeScene.scene.remove(child);
// }
// });
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER,
@@ -433,8 +415,8 @@ class Editor {
}
// TO UPDATE selected objects in the scene might want to add to the scene ...
async setSelectedObject(position:XYZ,rotation:XYZ,scale:XYZ) {
if (this.selected !=undefined || this.selected !=null) {
async setSelectedObject(position: XYZ, rotation: XYZ, scale: XYZ) {
if (this.selected != undefined || this.selected != null) {
//console.log(`triggering setSelectedObject`)
this.selected.position.x = position.x
this.selected.position.y = position.y
@@ -449,8 +431,11 @@ class Editor {
this.selected.scale.z = scale.z
}
}
}
public async test_loadMediaToken(media_file_token: string) {
this.activeScene.load_glb(media_file_token);
}
public async saveScene(name: string) {
// remove controls when saving scene.
@@ -458,32 +443,23 @@ class Editor {
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER,
});
console.log(
`saveScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`,
);
let proxyScene = new StoryTellerProxyScene(this.version, this.activeScene);
let scene_json = await proxyScene.saveToScene();
let proxyTimeline = new StoryTellerProxyTimeline(this.version, this.timeline, this.transform_engine, this.animation_engine, this.audio_engine, this.lipsync_engine);
let timeline_json = await proxyTimeline.saveToJson();
let save_data = {scene: scene_json, timeline: timeline_json};
// TODO turn scene information into and object ...
const result = await this.api_manager.saveSceneState(
this.activeScene.scene,
JSON.stringify(save_data),
name,
this.current_scene_glb_media_token,
this.current_scene_media_token,
new TimelineDataState(),
);
if (result.data == null) {
return;
}
const scene_media_token = result.data["scene_media_file_token"];
if (scene_media_token != null) {
this.current_scene_media_token = scene_media_token;
}
const scene_glb_media_token = result.data["scene_glb_media_file_token"];
if (scene_glb_media_token != null) {
this.current_scene_glb_media_token = scene_glb_media_token;
}
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER,
});
@@ -643,7 +619,7 @@ class Editor {
// note the database from the server is the source of truth for all the data.
// Test code here
const object: THREE.Object3D = await this.activeScene.load_glb(
"m_77z28zfaxc3sdtt5cc68vpz2n40qed",
"m_fmxy8wjnep1hdaz7qdg4n7y15d2bsp",
);
object.uuid = "CH1";
@@ -813,7 +789,7 @@ class Editor {
}
// Basicly Unity 3D's update loop.
updateLoop(time: number) {
async updateLoop(time: number) {
if (this.cam_obj == undefined) {
this.cam_obj = this.activeScene.get_object_by_name("::CAM::");
}
@@ -831,11 +807,11 @@ class Editor {
if (this.cameraViewControls && this.camera_person_mode) {
this.cameraViewControls.update(5 * delta_time);
if(this.cam_obj){
if (this.cam_obj) {
if (this.last_scrub != this.timeline.scrubber_frame_position) {
this.camera.position.copy(this.cam_obj.position);
this.camera.rotation.copy(this.cam_obj.rotation);
} else if(this.timeline.is_playing == false) {
} else if (this.timeline.is_playing == false) {
this.cam_obj.position.copy(this.camera.position);
this.cam_obj.rotation.copy(this.camera.rotation);
} else {
@@ -846,25 +822,24 @@ class Editor {
this.cam_obj.visible = false;
}
}
else if (this.cam_obj){
else if (this.cam_obj) {
this.cam_obj.visible = true;
}
if (this.render_camera && this.cam_obj) {
this.render_camera.position.copy(this.cam_obj.position);
this.render_camera.rotation.copy(this.cam_obj.rotation);
this.cam_obj.scale.copy(new THREE.Vector3(1,1,1));
this.cam_obj.scale.copy(new THREE.Vector3(1, 1, 1));
}
if(this.timeline.is_playing){
this.timeline.update();
}
else if (this.last_scrub == this.timeline.scrubber_frame_position){
if (this.timeline.is_playing) {
await this.timeline.update(this.rendering);
}
else if (this.last_scrub == this.timeline.scrubber_frame_position) {
this.updateSelectedUI();
}
this.last_scrub = this.timeline.scrubber_frame_position;
this.renderScene();
}
@@ -933,10 +908,10 @@ class Editor {
audioSegment,
"-filter_complex",
"[1:a]adelay=" +
startTime * 1000 +
"|" +
startTime * 1000 +
"[a1];[0:a][a1]amix=inputs=2[a]",
startTime * 1000 +
"|" +
startTime * 1000 +
"[a1];[0:a][a1]amix=inputs=2[a]",
"-map",
"[a]",
`${itteration}final_tmp.wav`,
@@ -962,7 +937,12 @@ class Editor {
}
async stopPlayback(compile_audio: boolean = true) {
console.log(this.frames, this.frame_buffer.length);
this.renderMode();
if(this.generating_preview){return;}
this.generating_preview = true;
const ffmpeg = createFFmpeg({ log: true });
await ffmpeg.load();
for (let index = 0; index < this.frame_buffer.length; index++) {
@@ -975,7 +955,7 @@ class Editor {
}
await ffmpeg.run(
"-framerate",
"" + this.cap_fps,
"" + this.cap_fps/2,
"-i",
"image%d.png",
"-f",
@@ -989,6 +969,10 @@ class Editor {
"-c:a",
"aac", // Specify audio codec (optional, but recommended for MP4)
"-shortest", // Ensure output duration matches the shortest stream (video or audio)
"-pix_fmt" ,
"yuv420p",
"-f",
"mp4",
"0tmp.mp4",
);
@@ -1003,39 +987,36 @@ class Editor {
}
}
const output = await ffmpeg.FS("readFile", itteration + "tmp.mp4");
const output = ffmpeg.FS("readFile", itteration + "tmp.mp4");
ffmpeg.exit();
this.generating_preview = false;
// Create a Blob from the output file for downloading
const blob = new Blob([output.buffer], { type: "video/mp4" });
const url = URL.createObjectURL(blob);
const downloadLink = document.createElement("a");
downloadLink.href = url;
downloadLink.download = "render.mp4";
document.body.appendChild(downloadLink);
downloadLink.click();
// Clean up
URL.revokeObjectURL(url);
document.body.removeChild(downloadLink);
const data: any = await this.api_manager.uploadMedia(blob, "render.mp4");
console.log("data", data)
if (data == null) { return; }
let upload_token = data['upload_token'];
let upload_token = data['media_file_token'];
console.log(upload_token);
// Create a link to download the file stylize video using api ..
//{"success":true,"upload_token":"mu_x9kr5cfafn512pjbygdszvbdpktrr"} payload
const result = await this.api_manager
.stylizeVideo(
upload_token,
this.art_style,
this.positive_prompt,
this.negative_prompt,
Visibility.Public,
)
.catch((error) => {
console.log(error);
});
.stylizeVideo(
upload_token,
this.art_style,
this.positive_prompt,
this.negative_prompt,
Visibility.Public,
)
.catch((error) => {
console.log(error);
});
// {"success":true,"inference_job_token":"jinf_j3nbqbd15wqxb0xcks13qh3f3bz"}
console.log(result);
@@ -1111,7 +1092,7 @@ class Editor {
// This initializes the generation of a video render scene is where the core work happens
generateVideo() {
console.log("Generating video...");
if (this.rendering) {
if (this.rendering || this.generating_preview) {
return;
}
this.rendering = true; // has to go first to debounce
@@ -1167,6 +1148,7 @@ class Editor {
}
if (this.rendering && this.renderer && this.clock) {
this.frames += 1;
this.playback_location++;
const imgData = this.renderer.domElement.toDataURL();
this.frame_buffer.push(imgData);
@@ -1186,41 +1168,41 @@ class Editor {
if (this.selected == undefined) {
return;
}
const pos = this.selected.position;
const rot = this.selected.rotation;
const scale = this.selected.scale;
// TODO this is a bug we need to only show when clicked on and use UPDATE when updating.
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.UPDATE_CONTROLPANELS_SCENEOBJECT,
payload: {
group:
this.selected.name === "::CAM::"
? ClipGroup.CAMERA
: ClipGroup.OBJECT, // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips
object_uuid: this.selected.uuid,
object_name: this.selected.name,
version: this.version,
objectVectors: {
position: {
x: parseFloat(pos.x.toFixed(2)),
y: parseFloat(pos.y.toFixed(2)),
z: parseFloat(pos.z.toFixed(2)),
},
rotation: {
x: parseFloat(rot.x.toFixed(2)),
y: parseFloat(rot.y.toFixed(2)),
z: parseFloat(rot.z.toFixed(2)),
},
scale: {
x: parseFloat(scale.x.toFixed(2)),
y: parseFloat(scale.y.toFixed(2)),
z: parseFloat(scale.z.toFixed(2)),
},
const pos = this.selected.position;
const rot = this.selected.rotation;
const scale = this.selected.scale;
// TODO this is a bug we need to only show when clicked on and use UPDATE when updating.
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.UPDATE_CONTROLPANELS_SCENEOBJECT,
payload: {
group:
this.selected.name === "::CAM::"
? ClipGroup.CAMERA
: ClipGroup.OBJECT, // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips
object_uuid: this.selected.uuid,
object_name: this.selected.name,
version: this.version,
objectVectors: {
position: {
x: parseFloat(pos.x.toFixed(2)),
y: parseFloat(pos.y.toFixed(2)),
z: parseFloat(pos.z.toFixed(2)),
},
rotation: {
x: parseFloat(rot.x.toFixed(2)),
y: parseFloat(rot.y.toFixed(2)),
z: parseFloat(rot.z.toFixed(2)),
},
scale: {
x: parseFloat(scale.x.toFixed(2)),
y: parseFloat(scale.y.toFixed(2)),
z: parseFloat(scale.z.toFixed(2)),
},
},
});
},
});
}
// Automaticly resize scene.
app/pages/PageEnigma/js/scene.ts
+3
View File
@@ -46,6 +46,7 @@ class Scene {
let obj = new THREE.Mesh(geometry, material);
obj.receiveShadow = true;
obj.castShadow = true;
obj.userData["media_id"] = "Parim";
//obj.type = "Object3D";
obj.name = name;
this.scene.add(obj);
@@ -128,6 +129,7 @@ class Scene {
resolve(child.children[0]);
return;
}
child.userData["media_id"] = media_id;
if (auto_add) { this.scene.add(child); }
resolve(child);
});
@@ -201,6 +203,7 @@ class Scene {
directional_light.shadow.map = null;
directional_light.castShadow = true;
directional_light.shadow.bias = 0.00004;
directional_light.userData["media_id"] = "DirectionalLight";
this.scene.add(directional_light);
this.scene.add(directional_light.target);
app/pages/PageEnigma/js/timeline.ts
+4 -2
View File
@@ -347,7 +347,7 @@ export class TimeLine {
}
// called by the editor update loop on each frame
public async update() {
public async update(isRendering=false) {
//if (this.is_playing == false) return; // start and stop
this.timeline_limit = this.getEndPoint();
if (this.is_playing) {
@@ -399,8 +399,10 @@ export class TimeLine {
} else if (element.type === ClipType.ANIMATION) {
if (object) {
await this.animation_engine.clips[object.uuid+element.media_id].play(object);
let fps = 120;
if(isRendering) { fps = 60; }
this.animation_engine.clips[object.uuid+element.media_id].step(
this.scrubber_frame_position / 120, // Double FPS for best result.
this.scrubber_frame_position / fps, // Double FPS for best result.
);
}
} else {
app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts
+89
View File
@@ -0,0 +1,89 @@
import * as THREE from "three";
export interface ObjectJSON {
version: number;
position: {
x: number;
y: number;
z: number;
};
rotation: {
x: number;
y: number;
z: number;
};
scale: {
x: number;
y: number;
z: number;
};
object_name: string;
object_uuid: string;
object_user_data_name: string;
media_file_token: string;
}
export class StoryTellerProxy3DObject {
version: number;
position: THREE.Vector3;
rotation: THREE.Euler;
scale: THREE.Vector3;
object_uuid: string;
object_name: string;
object_user_data_name: string; // changable name
media_file_token: string;
constructor(
version: number,
media_file_token: string,
) {
this.version = version;
this.media_file_token = media_file_token;
this.position = new THREE.Vector3(0.0, 0.0, 0.0);
this.rotation = new THREE.Euler(0.0, 0.0, 0.0);
this.scale = new THREE.Vector3(1.0, 1.0, 1.0);
this.object_name = "";
this.object_user_data_name = "";
this.object_uuid = "";
}
public async initialize(object: THREE.Object3D) {
this.position = object.position;
this.rotation = object.rotation;
this.scale = object.scale;
this.object_name = object.name;
this.object_user_data_name = object.userData.name;
this.object_uuid = object.uuid;
}
public async toJSON(): Promise<ObjectJSON> {
const json:ObjectJSON = {
version: this.version,
position: {
x: this.position.x,
y: this.position.y,
z: this.position.z,
},
rotation: {
x: this.rotation.x,
y: this.rotation.y,
z: this.rotation.z,
},
scale: {
x: this.scale.x,
y: this.scale.y,
z: this.scale.z,
},
object_name: this.object_name,
object_uuid: this.object_uuid,
object_user_data_name: this.object_user_data_name,
media_file_token: this.media_file_token,
};
return json;
}
}
app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts
+87
View File
@@ -0,0 +1,87 @@
import * as THREE from "three";
import { GLTFLoader, GLTF } from "three/addons/loaders/GLTFLoader.js";
import { StoryTellerProxy3DObject, ObjectJSON } from "./storyteller_proxy_3d_object"
import Scene from "../js/scene";
interface LookUpDictionary {
[key: string]: StoryTellerProxy3DObject;
}
export class StoryTellerProxyScene {
sceneItemProxy: StoryTellerProxy3DObject[];
scene: Scene;
glbLoader: GLTFLoader;
lookUpDictionary: LookUpDictionary;
version: number;
constructor(version: number, scene: Scene) {
this.version = version
this.scene = scene;
this.glbLoader = new GLTFLoader();
this.lookUpDictionary = {}
this.sceneItemProxy = []
}
public async saveToScene(): Promise<any> {
let results: ObjectJSON[] = [];
if (this.scene.scene != null) {
for (const child of this.scene.scene.children) {
if (child.userData['media_id'] != undefined) {
if (this.lookUpDictionary[child.uuid] == null) {
this.lookUpDictionary[child.uuid] = new StoryTellerProxy3DObject(this.version, child.userData['media_id']);
}
let proxyObject3D: StoryTellerProxy3DObject = this.lookUpDictionary[child.uuid];
proxyObject3D.position.copy(child.position);
proxyObject3D.rotation.copy(child.rotation);
proxyObject3D.scale.copy(child.scale);
proxyObject3D.object_user_data_name = child.userData.name;
proxyObject3D.object_name = child.name;
proxyObject3D.object_uuid = child.uuid;
const json_data = await proxyObject3D.toJSON();
results.push(json_data);
}
}
} else {
console.log("Scene doesn't exist needs to be assigned");
}
return results;
}
public async loadFromSceneJson(scene_json: ObjectJSON[]) {
if (scene_json != null && this.scene != null) {
while (this.scene.scene.children.length > 0) {
this.scene.scene.remove(this.scene.scene.children[0]);
}
for (const json_object of scene_json) {
let token: string = json_object.media_file_token;
let obj;
switch (token) {
case 'Parim':
let prim_uuid = this.scene.instantiate(json_object.object_name);
obj = this.scene.get_object_by_uuid(prim_uuid);
break;
case 'DirectionalLight':
obj = this.scene._create_base_lighting();
break;
default:
if(token.includes('m_')){
obj = await this.scene.load_glb(token);
}
break;
}
if (obj) {
obj.position.copy(json_object.position);
obj.rotation.copy(new THREE.Euler(json_object.rotation.x, json_object.rotation.y, json_object.rotation.z));
obj.scale.copy(json_object.scale);
obj.name = json_object.object_name;
obj.userData.name = json_object.object_user_data_name;
obj.uuid = json_object.object_uuid;
obj.userData['media_id'] = json_object.media_file_token;
}
}
this.scene._createGrid();
}
}
};
app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts
+127
View File
@@ -0,0 +1,127 @@
import { AnimationClip } from "../datastructures/clips/animation_clip";
import { AudioClip } from "../datastructures/clips/audio_clip";
import { ClipUI } from "../datastructures/clips/clip_ui";
import { LipSyncClip } from "../datastructures/clips/lipsync_clip";
import { TransformClip } from "../datastructures/clips/transform_clip";
import AnimationEngine from "../js/animation_engine";
import AudioEngine from "../js/audio_engine";
import LipSyncEngine from "../js/lip_sync_engine";
import { TimeLine } from "../js/timeline";
import TransformEngine from "../js/transform_engine";
export class StoryTellerProxyTimeline {
timeline: TimeLine
transform_engine: TransformEngine
animation_engine: AnimationEngine
audio_engine: AudioEngine
lipsync_engine: LipSyncEngine
constructor(version: number, timeline: TimeLine, transform_engine: TransformEngine, animation_engine: AnimationEngine, audio_engine: AudioEngine, lipsync_engine: LipSyncEngine) {
this.timeline = timeline;
this.transform_engine = transform_engine;
this.animation_engine = animation_engine;
this.audio_engine = audio_engine;
this.lipsync_engine = lipsync_engine;
}
private async getItems(items: any[]): Promise<any[]> {
let timeline_items_data = [];
for (let index = 0; index < items.length; index++) {
const element = items[index];
timeline_items_data.push(element.toJSON());
};
return timeline_items_data;
}
private async getItemsDict(items: { [key: string]: any }): Promise<{ [key: string]: any }> {
let timeline_items_data: { [key: string]: any } = {};
for (const key in items) {
if (items.hasOwnProperty(key)) {
const element = items[key];
timeline_items_data[key] = element.toJSON();
}
}
return timeline_items_data;
}
public async saveToJson(): Promise<any> {
let timeline_json = {
timeline: await this.getItems(this.timeline.timeline_items),
transform: await this.getItemsDict(this.transform_engine.clips),
animation: await this.getItemsDict(this.animation_engine.clips),
audio: await this.getItemsDict(this.audio_engine.clips),
lipsync: await this.getItemsDict(this.lipsync_engine.clips),
}
return timeline_json;
}
private async loadTimelineClips(timeline_clips: any[]): Promise<any[]> {
let new_clips = [];
if (timeline_clips) {
for (let index = 0; index < timeline_clips.length; index++) {
const element = timeline_clips[index];
let clip = new ClipUI(
element.version, element.type,
element.group, element.name,
element.media_id, element.object_uuid,
element.start_offset, element.ending_offset);
new_clips.push(clip);
}
}
return new_clips;
}
private async loadTransformClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> {
let timeline_items_data: { [key: string]: any } = {};
for (const key in items) {
if (items.hasOwnProperty(key)) {
const element = items[key];
timeline_items_data[key] = new TransformClip(element.version, element.object_uuid, element.length, element.media_id);
timeline_items_data[key].keyframes = element.keyframes;
}
}
return timeline_items_data;
}
private async loadAudioClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> {
let timeline_items_data: { [key: string]: any } = {};
for (const key in items) {
if (items.hasOwnProperty(key)) {
const element = items[key];
timeline_items_data[key] = new AudioClip(element.version, element.media_id, element.volume);
}
}
return timeline_items_data;
}
private async loadAnimationClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> {
let timeline_items_data: { [key: string]: any } = {};
for (const key in items) {
if (items.hasOwnProperty(key)) {
const element = items[key];
timeline_items_data[key] = new AnimationClip(element.version, element.media_id, "remote", element.object_uuid, element.speed, element.length, element.clip_name);
}
}
return timeline_items_data;
}
private async loadLipsyncClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> {
let timeline_items_data: { [key: string]: any } = {};
for (const key in items) {
if (items.hasOwnProperty(key)) {
const element = items[key];
timeline_items_data[key] = new LipSyncClip(element.version, element.media_id, element.volume);
}
}
return timeline_items_data;
}
public async loadFromJson(timeline: any) {
this.timeline.timeline_items = await this.loadTimelineClips(timeline['timeline']);
this.transform_engine.clips = await this.loadTransformClips(timeline['transform']);
this.audio_engine.clips = await this.loadAudioClips(timeline['audio']);
this.lipsync_engine.clips = await this.loadLipsyncClips(timeline['lipsync']);
this.animation_engine.clips = await this.loadAnimationClips(timeline['animation']);
}
}
app/pages/PageEnigma/store/timing.ts
+2
View File
@@ -1,3 +1,5 @@
import { signal } from "@preact/signals-core";
export const currentTime = signal(0);
export const currentScroll = signal(0);