daniel543x/artcraft
1
0 Fork 0
mirror of https://github.com/storytold/artcraft.git synced 2026-11-04 17:38:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #99 from storytold/feature/quality-of-life-1

Browse Source 
Feature/quality of life 1
This commit is contained in:
ArEnSc authored and GitHub committed 2024-04-06 20:08:45 -04:00
2 parents 9b80285c9f dd1b13a01b
commit 45f4dd798f
5 files changed
+104 -251
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx
+4 -4
View File
@@ -30,7 +30,7 @@ export const ControlsTopButtons = () => {
};
const handleButtonTest = () => {
//editorEngine?.testStylizeRequest();
};
const handleTestButton2 = () => {
@@ -44,7 +44,7 @@ export const ControlsTopButtons = () => {
editorEngine?.startPlayback();
};
const handleButtonTakeFrame = () => {
editorEngine?.take_timeline_cam_clip();
// editorEngine?.take_timeline_cam_clip();
};
const handleButtonSingleFrame = () => {
editorEngine?.generateFrame();
@@ -139,8 +139,8 @@ export const ControlsTopButtons = () => {
</Button>
</div>
<div className="flex gap-2">
<Button onClick={handleButtonSingleFrame}>Render Single Frame</Button>
<Button onClick={handleButtonTakeFrame}>Take Frame</Button>
{/* <Button onClick={handleButtonSingleFrame}>Render Single Frame</Button>
<Button onClick={handleButtonTakeFrame}>Take Frame</Button> */}
<Button
onClick={handleButtonTest}
className="bg-brand-tertiary hover:bg-brand-teriary-400 focus-visible:outline-brand-tertiary"
app/pages/PageEnigma/comps/ControlsVideo/ControlsVideo.tsx
+2 -2
View File
@@ -16,13 +16,13 @@ export const ControlsVideo = () => {
console.log("Controls Video: Backward-Fast clicked");
};
const handleBackwardStep = () => {
console.log("Controls Video: Backward-Step clicked");
editorEngine?.stepFrame(-1);
};
const handlePlay = () => {
editorEngine?.startPlayback();
};
const handleForwardStep = () => {
console.log("Controls Video: Forward-Step clicked");
editorEngine?.stepFrame(1);
};
const handleForwardFast = () => {
console.log("Controls Video: Forward-Fast clicked");
app/pages/PageEnigma/datastructures/clips/clip_ui.ts
+2
View File
@@ -11,6 +11,7 @@ export class ClipUI {
object_uuid: string;
offset: number; // in frames
length: number; // in frames
should_play: boolean;
constructor(
version: number,
@@ -32,6 +33,7 @@ export class ClipUI {
this.media_id = media_id; // Animation / Audio / Lipsync / : Engine
this.offset = offset;
this.length = length;
this.should_play = true;
}
toJSON(): any {
app/pages/PageEnigma/js/editor.ts
+67 -233
View File
@@ -302,7 +302,22 @@ class Editor {
window.addEventListener("mousemove", this.onMouseMove.bind(this), false);
window.addEventListener("click", this.onMouseClick.bind(this), false);
// Base control and debug stuff remove debug in prod.
this._initializeControl();
if (this.control == undefined) {
return;
}
this.control.addEventListener("change", this.renderScene.bind(this));
this.control.addEventListener("dragging-changed", (event: any) => {
if (this.orbitControls == undefined) {
return;
}
this.orbitControls.enabled = !event.value;
this.updateSelectedUI();
// this.update_properties()
});
this.control.setSize(0.5); // Good default value for visuals.
this.raycaster = new THREE.Raycaster();
this.mouse = new THREE.Vector2();
this.activeScene.scene.add(this.control);
// Resets canvas size.
this.onWindowResize();
// Creates the main update loop.
@@ -322,32 +337,11 @@ class Editor {
this.onMouseUp.bind(this),
false,
);
this.renderer.domElement.addEventListener(
"onContextMenu",
this.onContextMenu.bind(this),
false,
);
// saving state of the scene
this.current_scene_media_token = null;
this.current_scene_glb_media_token = null;
this.renderer.domElement.addEventListener(
"mousedown",
this.onMouseDown.bind(this),
false,
);
this.renderer.domElement.addEventListener(
"mouseup",
this.onMouseUp.bind(this),
false,
);
this.renderer.domElement.addEventListener(
"onContextMenu",
this.onContextMenu.bind(this),
false,
);
this.cam_obj = this.activeScene.get_object_by_name("::CAM::");
this.dispatchAppUiState({
@@ -375,10 +369,6 @@ class Editor {
public async testTestTimelineEvents() { }
public async testLoadTokenAssets() {
await this.storyteller_proxy_scene.loadFromMediaFileToken("m_z4jzbst3xfh64h0qn4bqh4afenfps9")
}
public async loadScene(scene_media_token: string) {
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER,
@@ -433,9 +423,6 @@ class Editor {
}
}
public async test_loadMediaToken(media_file_token: string) {
this.activeScene.load_glb(media_file_token);
}
public async saveScene(name: string) {
// remove controls when saving scene.
@@ -548,9 +535,10 @@ class Editor {
switchCameraView() {
this.camera_person_mode = !this.camera_person_mode;
console.log("camera");
console.log(this.camera_person_mode);
if (this.cam_obj) {
if (this.camera_person_mode) {
this.last_cam_pos.copy(this.camera.position);
this.last_cam_rot.copy(this.camera.rotation);
@@ -571,6 +559,11 @@ class Editor {
element.visible = false;
});
}
this.removeTransformControls();
this.selected = this.cam_obj;
this.dispatchAppUiState({type: APPUI_ACTION_TYPES.SHOW_CONTROLPANELS_SCENEOBJECT});
this.updateSelectedUI();
} else {
this.camera.position.copy(this.last_cam_pos);
this.camera.rotation.copy(this.last_cam_rot);
@@ -589,32 +582,14 @@ class Editor {
element.visible = true;
});
}
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.HIDE_CONTROLPANELS_SCENEOBJECT,
});
}
}
}
async addTransformClipBase(
name: string = "New Clip",
group: "object" | "character" | "camera" | "global_audio",
object: THREE.Object3D,
offset: number,
length: number,
) {
this.timeline.addPlayableClip(
new ClipUI(
1.0,
"transform",
group,
name,
object.uuid, // object id here ...
object.uuid, // Need to change to media id ....
offset,
length,
),
);
this.transform_engine.loadObject(object.uuid, length);
}
async _test_demo() {
// note the database from the server is the source of truth for all the data.
// Test code here
@@ -627,60 +602,6 @@ class Editor {
// Stick Open Pose Man: m_9f3d3z94kk6m25zywyz6an3p43fjtw
// XBot: m_r7w1tmkx2jg8nznr3hyzj4k6zhfh7d
// YBot: m_9sqg0evpr23587jnr8z3zsvav1x077
// Load timeline creates the the clips from the datastructure and loads them in here.
// load object into the engine for lip syncing
//this.lipsync_engine.load_object(
// object.uuid,
// "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72",
//);
//this.animation_engine.load_object(
// object.uuid,
// "/resources/models/fox/fox_idle.glb",
// "clip3",
//);
// then it creates clip ui to load the playable clips
// then refreshes the timeline.
// create the clip with the same id for a reference to the media
//this.timeline.addPlayableClip(
// new ClipUI(
// 1.0,
// "lipsync",
// "character",
// "clip1",
// "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72",
// object.uuid,
// 150,
// 400,
// ),
//);
//// media id for this is up in the air but when a path is created you should be able to store and delete it
//this.timeline.addPlayableClip(
// new ClipUI(1.0, "transform", "character", "clip2", object.uuid, object.uuid, 0, 150),
//);
// media id for this as well it can be downloaded
//this.timeline.addPlayableClip(
// new ClipUI(
// 1.0,
// "animation",
// "character",
// "clip3",
// "/resources/models/fox/fox_idle.glb",
// object.uuid,
// 0,
// 400,
// ),
//);
//this.audio_engine.loadClip("m_h33vytxs5eqqqf07nsy14qzrf9ww4v");
//await this.timeline.addPlayableClip(new ClipUI(
// 1.0,
// "audio",
// "global_audio",
// "AudioClip",
// "m_h33vytxs5eqqqf07nsy14qzrf9ww4v",
// "",
// 0,
// 50));
}
// Configure post processing.
@@ -756,36 +677,36 @@ class Editor {
this.activeScene.renderMode(this.rendering);
}
save() {
//console.log(this.control)
//if(this.selected != null){ this.control.detach(this.selected) }
//this.activeScene.scene.remove(this.control)
//this.activeScene.scene.remove(this.activeScene.gridHelper)
//this.save_manager.save(this.activeScene.scene, this._save_to_cloud.bind(this), this.audio_manager, this.timeline, this.activeScene.animations)
//this.activeScene._createGrid()
//this.audio_engine.playClip("m_f7jnwt3d1ddchatdk5vaqt0n4mb1hg")
//console.log(this.selected)
if (this.selected == null) {
return;
}
this.transform_engine.addFrame(this.selected);
console.log("Frame taken.");
stepFrame(frames:number) {
this.timeline.stepFrame(frames);
}
take_timeline_cam_clip() {
if (this.cam_obj == null) {
return;
// Render the scene to the camera, this is called in the update.
renderScene() {
if (this.composer != null && !this.rendering && this.rawRenderer) {
this.composer.render();
this.rawRenderer.render(this.activeScene.scene, this.render_camera);
} else if (this.renderer && this.render_camera) {
this.renderer.setSize(this.render_width, this.render_height);
this.renderer.render(this.activeScene.scene, this.render_camera);
} else {
console.error("Could not render to canvas no render or composer!");
}
if (!this.camera_person_mode) {
return;
if (this.rendering && this.renderer && this.clock) {
this.frames += 1;
this.playback_location++;
const imgData = this.renderer.domElement.toDataURL();
this.frame_buffer.push(imgData);
this.render_timer += this.clock.getDelta();
if (this.timeline.is_playing == false) {
this.stopPlayback();
this.playback_location = 0;
this.rendering = false;
this.switchCameraView();
this.onWindowResize();
}
}
this.transform_engine.addFrame(
this.cam_obj,
this.transform_engine.clips[this.cam_obj.uuid].length,
);
console.log("Camera frame taken.");
this.activeScene.createPoint(this.cam_obj.position, false);
}
// Basicly Unity 3D's update loop.
@@ -851,12 +772,6 @@ class Editor {
this.transform_interaction = true;
}
async loadWavAsBlob(url: string) {
const response = await fetch(url);
const blob = await response.blob();
return blob;
}
async convertAudioClip(itteration: number, ffmpeg: FFmpeg, clip: ClipUI) {
const video_og = itteration + "tmp.mp4";
const wav_name = itteration + "tmp.wav";
@@ -1115,54 +1030,6 @@ class Editor {
}
}
// Initializes transform x y z changes.
_initializeControl() {
if (this.control == undefined) {
return;
}
this.control.addEventListener("change", this.renderScene.bind(this));
this.control.addEventListener("dragging-changed", (event: any) => {
if (this.orbitControls == undefined) {
return;
}
this.orbitControls.enabled = !event.value;
this.updateSelectedUI();
// this.update_properties()
});
this.control.setSize(0.5); // Good default value for visuals.
this.raycaster = new THREE.Raycaster();
this.mouse = new THREE.Vector2();
this.activeScene.scene.add(this.control);
}
// Render the scene to the camera, this is called in the update.
renderScene() {
if (this.composer != null && !this.rendering && this.rawRenderer) {
this.composer.render();
this.rawRenderer.render(this.activeScene.scene, this.render_camera);
} else if (this.renderer && this.render_camera) {
this.renderer.setSize(this.render_width, this.render_height);
this.renderer.render(this.activeScene.scene, this.render_camera);
} else {
console.error("Could not render to canvas no render or composer!");
}
if (this.rendering && this.renderer && this.clock) {
this.frames += 1;
this.playback_location++;
const imgData = this.renderer.domElement.toDataURL();
this.frame_buffer.push(imgData);
this.render_timer += this.clock.getDelta();
if (this.timeline.is_playing == false) {
this.stopPlayback();
this.playback_location = 0;
this.rendering = false;
this.switchCameraView();
this.onWindowResize();
}
}
}
updateSelectedUI() {
if (this.selected == undefined) {
@@ -1185,19 +1052,19 @@ class Editor {
version: this.version,
objectVectors: {
position: {
x: parseFloat(pos.x.toFixed(2)),
y: parseFloat(pos.y.toFixed(2)),
z: parseFloat(pos.z.toFixed(2)),
x: parseFloat(pos.x.toFixed(6)),
y: parseFloat(pos.y.toFixed(6)),
z: parseFloat(pos.z.toFixed(6)),
},
rotation: {
x: parseFloat(rot.x.toFixed(2)),
y: parseFloat(rot.y.toFixed(2)),
z: parseFloat(rot.z.toFixed(2)),
x: parseFloat(rot.x.toFixed(6)),
y: parseFloat(rot.y.toFixed(6)),
z: parseFloat(rot.z.toFixed(6)),
},
scale: {
x: parseFloat(scale.x.toFixed(2)),
y: parseFloat(scale.y.toFixed(2)),
z: parseFloat(scale.z.toFixed(2)),
x: parseFloat(scale.x.toFixed(6)),
y: parseFloat(scale.y.toFixed(6)),
z: parseFloat(scale.z.toFixed(6)),
},
},
},
@@ -1233,10 +1100,6 @@ class Editor {
this.render_camera.updateProjectionMatrix();
}
onContextMenu(event: any) {
return false;
}
onMouseDown(event: any) {
if (event.button === 1 && this.camera_person_mode) {
this.lockControls?.lock();
@@ -1265,7 +1128,8 @@ class Editor {
this.raycaster == undefined ||
this.mouse == undefined ||
this.control == undefined ||
this.outlinePass == undefined
this.outlinePass == undefined ||
this.camera_person_mode
) {
return;
}
@@ -1301,39 +1165,9 @@ class Editor {
this.outlinePass.selectedObjects = [this.selected];
this.transform_interaction = true;
// Contact react land
const pos = this.selected.position;
const rot = this.selected.rotation;
const scale = this.selected.scale;
this.dispatchAppUiState({
type: APPUI_ACTION_TYPES.SHOW_CONTROLPANELS_SCENEOBJECT,
payload: {
group:
this.selected.name === "::CAM::"
? ClipGroup.CAMERA
: ClipGroup.OBJECT, // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips
object_uuid: this.selected.uuid,
object_name: this.selected.name,
version: this.version,
objectVectors: {
position: {
x: parseFloat(pos.x.toFixed(2)),
y: parseFloat(pos.y.toFixed(2)),
z: parseFloat(pos.z.toFixed(2)),
},
rotation: {
x: parseFloat(rot.x.toFixed(2)),
y: parseFloat(rot.y.toFixed(2)),
z: parseFloat(rot.z.toFixed(2)),
},
scale: {
x: parseFloat(scale.x.toFixed(2)),
y: parseFloat(scale.y.toFixed(2)),
z: parseFloat(scale.z.toFixed(2)),
},
},
},
});
// Contact react land
this.dispatchAppUiState({type: APPUI_ACTION_TYPES.SHOW_CONTROLPANELS_SCENEOBJECT});
this.updateSelectedUI();
}
} else if (this.transform_interaction == false) {
this.removeTransformControls();
app/pages/PageEnigma/js/timeline.ts
+29 -12
View File
@@ -124,10 +124,13 @@ export class TimeLine {
await this.scrub(data);
break;
case toEngineActions.MUTE:
await this.mute(data);
await this.mute(data, false);
break;
case toEngineActions.UNMUTE:
await this.mute(data, true);
break;
default:
console.log("Action Not Wired");
console.log("Action Not Wired", action);
}
}
@@ -265,13 +268,18 @@ export class TimeLine {
}
}
}
public async scrubberUpdate(data: any) {
console.log(data);
}
public async mute(data: any) {
console.log(data);
public async mute(data: any, isMute: boolean) {
this.timeline_items.forEach(element => {
if (element.group == data.data['group']) {
element.should_play = isMute;
console.log(element);
}
});
}
public async addPlayableClip(clip: ClipUI): Promise<void> {
@@ -304,6 +312,14 @@ export class TimeLine {
this.setScrubberPosition(data["data"]["currentTime"]);
}
public async stepFrame(frames: number) {
this.update();
this.scrubber_frame_position += frames;
this.pushEvent(fromEngineActions.UPDATE_TIME, {
currentTime: this.scrubber_frame_position,
});
}
public async scrubberDidStop(offset_frame: number) { }
// public streaming events into the timeline from
public async setScrubberPosition(offset: number) {
@@ -326,7 +342,7 @@ export class TimeLine {
} else if (element.type == "audio") {
this.audio_engine.loadClip(element.media_id);
} else if (element.type == "animation") {
this.animation_engine.clips[element.object_uuid+element.media_id].stop();
this.animation_engine.clips[element.object_uuid + element.media_id].stop();
} else if (element.type == "lipsync") {
this.lipSync_engine.clips[element.object_uuid].reset();
} else {
@@ -347,7 +363,7 @@ export class TimeLine {
}
// called by the editor update loop on each frame
public async update(isRendering=false) {
public async update(isRendering = false) {
//if (this.is_playing == false) return; // start and stop
this.timeline_limit = this.getEndPoint();
if (this.is_playing) {
@@ -368,7 +384,8 @@ export class TimeLine {
for (const element of this.timeline_items) {
if (
element.offset <= this.scrubber_frame_position &&
this.scrubber_frame_position <= element.length
this.scrubber_frame_position <= element.length &&
element.should_play == true
) {
// run async
// element.play()
@@ -396,12 +413,12 @@ export class TimeLine {
await this.lipSync_engine.clips[element.object_uuid].play(object);
this.lipSync_engine.clips[element.object_uuid].step();
}
} else if (element.type === ClipType.ANIMATION) {
} else if (element.type === ClipType.ANIMATION) {
if (object) {
await this.animation_engine.clips[object.uuid+element.media_id].play(object);
await this.animation_engine.clips[object.uuid + element.media_id].play(object);
let fps = 120;
if(isRendering) { fps = 60; }
this.animation_engine.clips[object.uuid+element.media_id].step(
if (isRendering) { fps = 60; }
this.animation_engine.clips[object.uuid + element.media_id].step(
this.scrubber_frame_position / fps, // Double FPS for best result.
);
}