diff --git a/app/modules/TopBar/TopBar.tsx b/app/modules/TopBar/TopBar.tsx index decec636aa..2e4a2fad56 100644 --- a/app/modules/TopBar/TopBar.tsx +++ b/app/modules/TopBar/TopBar.tsx @@ -43,7 +43,11 @@ export const TopBar = ({ pageName }: Props) => {
{pageName}  /    - {sceneTitle.value} + {sceneTitle.value} +    + + @ CURRENT_STORYTELLER_GIT_VERSION +
diff --git a/app/pages/PageEnigma/PageEditor.tsx b/app/pages/PageEnigma/PageEditor.tsx index de45827031..d944354378 100644 --- a/app/pages/PageEnigma/PageEditor.tsx +++ b/app/pages/PageEnigma/PageEditor.tsx @@ -18,6 +18,8 @@ import { sidePanelVisible, dndSidePanelWidth, dndTimelineHeight, + dragItem, + canDrop, } from "~/pages/PageEnigma/store"; import { useSignals } from "@preact/signals-react/runtime"; import { AppUiContext } from "~/contexts/AppUiContext"; @@ -25,6 +27,7 @@ import { pageHeight, pageWidth } from "~/store"; import { TopBar } from "~/modules/TopBar"; import { DialogueTTS } from "./comps/DialogueTTS/DialogueTTS"; import { Pages } from "~/pages/PageEnigma/constants/page"; +import { AssetType } from "./models/assets"; interface Props { setPage: (page: Pages) => void; @@ -73,7 +76,39 @@ export const PageEditor = ({ setPage }: Props) => { ? "invisible" : "" }> - +
+ { + if ( + dragItem.value?.type === AssetType.OBJECT || + dragItem.value?.type === AssetType.SHAPE || + dragItem.value?.type === AssetType.CHARACTER + ) { + canDrop.value = true; + } + }} + onPointerLeave={() => { + if ( + dragItem.value?.type === AssetType.OBJECT || + dragItem.value?.type === AssetType.SHAPE || + dragItem.value?.type === AssetType.CHARACTER + ) { + canDrop.value = false; + } + }} + /> +
{/* Top controls */}
{
- +
); diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/tabList.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/tabList.tsx index eb1b393b40..9fda05468a 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/tabList.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/tabList.tsx @@ -24,12 +24,6 @@ export const tabList = [ value: AssetType.OBJECT, component: , }, - { - icon: "/resources/icons/shapes.png", - title: "Shapes", - value: AssetType.SHAPE, - component: , - }, { icon: "/resources/icons/camera.png", title: "Camera", diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsScrubber.ts b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsScrubber.ts index d75f09f967..7965490f2a 100644 --- a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsScrubber.ts +++ b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsScrubber.ts @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useState } from "react"; +import React, { useCallback, useEffect, useRef, useState } from "react"; import { currentTime, filmLength, scale } from "~/pages/PageEnigma/store"; import Queue from "~/pages/PageEnigma/Queue/Queue"; import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; @@ -6,13 +6,27 @@ import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; export const useMouseEventsScrubber = () => { const [isActive, setIsActive] = useState(false); - const [clientX, setClientX] = useState(0); + const clientX = useRef(0); const [time, setTime] = useState(-1); - useEffect(() => { + const getDelta = useCallback((event: MouseEvent) => { const max = filmLength.value * 60; + const delta = Math.round( + (event.clientX - clientX.current) / 4 / scale.value + currentTime.value, + ); + if (delta < 0) { + return 0; + } + if (delta > max) { + return max; + } + + return delta; + }, []); + + useEffect(() => { const onPointerUp = () => { if (isActive) { currentTime.value = Math.round(time); @@ -27,16 +41,11 @@ export const useMouseEventsScrubber = () => { }; const onMouseMove = (event: MouseEvent) => { - const delta = Math.round( - (event.clientX - clientX) / 4 / scale.value + currentTime.value, - ); if (isActive) { event.stopPropagation(); event.preventDefault(); - console.log(delta, max); - if (delta < 0 || delta > max) { - return; - } + + const delta = getDelta(event); setTime((oldTime) => { if (oldTime !== delta) { Queue.publish({ @@ -59,12 +68,12 @@ export const useMouseEventsScrubber = () => { window.removeEventListener("pointerup", onPointerUp); window.removeEventListener("pointermove", onMouseMove); }; - }, [clientX, isActive, time]); + }, [clientX, isActive, time, getDelta]); return { onPointerDown: useCallback((event: React.PointerEvent) => { if (event.button === 0) { - setClientX(event.clientX); + clientX.current = event.clientX; setIsActive(true); setTime(currentTime.value); } diff --git a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx index 471724640d..28daf77446 100644 --- a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx +++ b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx @@ -33,6 +33,11 @@ export const TrackProvider = ({ children }: Props) => { // cross group functions const dropClip = useCallback(() => { + if (!canDrop.value) { + endDrag(); + return; + } + if (dragItem.value) { const mediaItem = dragItem.value; if (mediaItem.type === AssetType.CHARACTER) { diff --git a/app/pages/PageEnigma/datastructures/clips/emotion_clip.ts b/app/pages/PageEnigma/datastructures/clips/emotion_clip.ts new file mode 100644 index 0000000000..b08d484d3c --- /dev/null +++ b/app/pages/PageEnigma/datastructures/clips/emotion_clip.ts @@ -0,0 +1,122 @@ +import * as THREE from "three"; + +interface CsvJson { + [key: string]: string[]; +} + +export class EmotionClip { + version: number; + media_id: string; + type: "emotion" = "emotion"; + emotion_json: any; + faces: THREE.Mesh[]; + + constructor(version: number, media_id: string) { + this.version = version; + this.media_id = media_id; + this.type = "emotion"; + this.faces = []; + this.download_csv().then(data => { + this.emotion_json = data; + }); + } + + async get_media_url() { + //This is for prod when we have the proper info on the url. + let api_base_url = "https://api.fakeyou.com"; + let url = `${api_base_url}/v1/media_files/file/${this.media_id}` + let responce = await fetch(url); + let json = await JSON.parse(await responce.text()); + let bucketPath = json["media_file"]["public_bucket_path"]; + let media_base_url = "https://storage.googleapis.com/vocodes-public" + let media_url = `${media_base_url}${bucketPath}` + return media_url; + } + + async download_json() { + let url = await this.get_media_url(); + const response = await fetch(url); + return await response.json(); + } + + async download_csv(): Promise { + let url = await this.get_media_url(); + const response = await fetch(url); + const csvText = await response.text(); + + // Parsing the CSV text + const rows = csvText.trim().split('\n'); + const header = rows[0].split(','); + const jsonData: CsvJson = {}; + + header.forEach(key => jsonData[key] = []); + + for (let i = 1; i < rows.length; i++) { + const values = rows[i].split(','); + values.forEach((value, index) => { + jsonData[header[index]].push(value); + }); + } + + return jsonData; + } + + + async _detect_face(object: THREE.Object3D): Promise { + this.faces = []; + return new Promise((resolve) => { + object.traverse((c: THREE.Object3D) => { + if (c instanceof THREE.Mesh) { + if (c.morphTargetInfluences && c.morphTargetDictionary) { + const blendShapeIndexE = c.morphTargetDictionary["E"]; + if (blendShapeIndexE != null) { + this.faces.push(c); + resolve(c); + } + } + } + }); + }); + } + + setBlends(shapes: { [key: string]: number }) { + this.faces.forEach((element: THREE.Mesh) => { + if (element.morphTargetInfluences && shapes) { + Object.keys(shapes).forEach(key => { + let index = element.morphTargetDictionary?.[key]; + if (index === undefined) { index = element.morphTargetDictionary?.[key.charAt(0).toLowerCase() + key.slice(1)]; } + if (typeof index === 'number' && element.morphTargetInfluences !== undefined) { + element.morphTargetInfluences[index] = shapes[key]; + } + }); + } + }); + } + + async reset(object: THREE.Object3D) { + await this._detect_face(object); + let keys: { [key: string]: number } = {}; + Object.keys(this.emotion_json).forEach(key => { + keys[key] = 0; + }); + this.setBlends(keys); + } + + async step(frame: number, object: THREE.Object3D) { + if(this.faces.length <= 0) { await this._detect_face(object); } + let keys: { [key: string]: number } = {}; + Object.keys(this.emotion_json).forEach(key => { + if (frame > this.emotion_json[key].length) { return; } + keys[key] = this.emotion_json[key][frame]; + }); + this.setBlends(keys); + } + + toJSON(): any { + return { + version: this.version, + media_id: this.media_id, + type: this.type, + }; + } +} diff --git a/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts index d484fda885..48b6c195e8 100644 --- a/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts @@ -116,9 +116,10 @@ export class LipSyncClip { }); } - step() { + step(frame: number, offset: number, rendering: boolean) { if (this.lipsync == null) { return; } - const positions = this.lipsync.update(); + const positions = this.lipsync.update(frame, offset, rendering); + if(positions) this.setBlends(positions["ee"], positions["ah"], positions["oh"]); } diff --git a/app/pages/PageEnigma/js/TransformControls.js b/app/pages/PageEnigma/js/TransformControls.js new file mode 100644 index 0000000000..faf753c25c --- /dev/null +++ b/app/pages/PageEnigma/js/TransformControls.js @@ -0,0 +1,1574 @@ +import { + BoxGeometry, + BufferGeometry, + CylinderGeometry, + DoubleSide, + Euler, + Float32BufferAttribute, + Line, + LineBasicMaterial, + Matrix4, + Mesh, + MeshBasicMaterial, + Object3D, + OctahedronGeometry, + PlaneGeometry, + Quaternion, + Raycaster, + SphereGeometry, + TorusGeometry, + Vector3 +} from 'three'; + +const _raycaster = new Raycaster(); + +const _tempVector = new Vector3(); +const _tempVector2 = new Vector3(); +const _tempQuaternion = new Quaternion(); +const _unit = { + X: new Vector3(1, 0, 0), + Y: new Vector3(0, 1, 0), + Z: new Vector3(0, 0, 1) +}; + +const _changeEvent = { type: 'change' }; +const _mouseDownEvent = { type: 'mouseDown' }; +const _mouseUpEvent = { type: 'mouseUp', mode: null }; +const _objectChangeEvent = { type: 'objectChange' }; + +class TransformControls extends Object3D { + + constructor(camera, domElement) { + + super(); + + if (domElement === undefined) { + + console.warn('THREE.TransformControls: The second parameter "domElement" is now mandatory.'); + domElement = document; + + } + + this.isTransformControls = true; + + this.sensitivity = 5; + + this.visible = false; + this.domElement = domElement; + this.domElement.style.touchAction = 'none'; // disable touch scroll + + const _gizmo = new TransformControlsGizmo(); + this._gizmo = _gizmo; + this.add(_gizmo); + + const _plane = new TransformControlsPlane(); + this._plane = _plane; + this.add(_plane); + + const scope = this; + + // Defined getter, setter and store for a property + function defineProperty(propName, defaultValue) { + + let propValue = defaultValue; + + Object.defineProperty(scope, propName, { + + get: function () { + + return propValue !== undefined ? propValue : defaultValue; + + }, + + set: function (value) { + + if (propValue !== value) { + + propValue = value; + _plane[propName] = value; + _gizmo[propName] = value; + + scope.dispatchEvent({ type: propName + '-changed', value: value }); + scope.dispatchEvent(_changeEvent); + + } + + } + + }); + + scope[propName] = defaultValue; + _plane[propName] = defaultValue; + _gizmo[propName] = defaultValue; + + } + + // Define properties with getters/setter + // Setting the defined property will automatically trigger change event + // Defined properties are passed down to gizmo and plane + + defineProperty('camera', camera); + defineProperty('object', undefined); + defineProperty('enabled', true); + defineProperty('axis', null); + defineProperty('mode', 'translate'); + defineProperty('translationSnap', null); + defineProperty('rotationSnap', null); + defineProperty('scaleSnap', null); + defineProperty('space', 'world'); + defineProperty('size', 1); + defineProperty('dragging', false); + defineProperty('showX', true); + defineProperty('showY', true); + defineProperty('showZ', true); + + // Reusable utility variables + + const worldPosition = new Vector3(); + const worldPositionStart = new Vector3(); + const worldQuaternion = new Quaternion(); + const worldQuaternionStart = new Quaternion(); + const cameraPosition = new Vector3(); + const cameraQuaternion = new Quaternion(); + const pointStart = new Vector3(); + const pointEnd = new Vector3(); + const rotationAxis = new Vector3(); + const rotationAngle = 0; + const eye = new Vector3(); + + // TODO: remove properties unused in plane and gizmo + + defineProperty('worldPosition', worldPosition); + defineProperty('worldPositionStart', worldPositionStart); + defineProperty('worldQuaternion', worldQuaternion); + defineProperty('worldQuaternionStart', worldQuaternionStart); + defineProperty('cameraPosition', cameraPosition); + defineProperty('cameraQuaternion', cameraQuaternion); + defineProperty('pointStart', pointStart); + defineProperty('pointEnd', pointEnd); + defineProperty('rotationAxis', rotationAxis); + defineProperty('rotationAngle', rotationAngle); + defineProperty('eye', eye); + + this._offset = new Vector3(); + this._startNorm = new Vector3(); + this._endNorm = new Vector3(); + this._cameraScale = new Vector3(); + + this._parentPosition = new Vector3(); + this._parentQuaternion = new Quaternion(); + this._parentQuaternionInv = new Quaternion(); + this._parentScale = new Vector3(); + + this._worldScaleStart = new Vector3(); + this._worldQuaternionInv = new Quaternion(); + this._worldScale = new Vector3(); + + this._positionStart = new Vector3(); + this._quaternionStart = new Quaternion(); + this._scaleStart = new Vector3(); + + this._getPointer = getPointer.bind(this); + this._onPointerDown = onPointerDown.bind(this); + this._onPointerHover = onPointerHover.bind(this); + this._onPointerMove = onPointerMove.bind(this); + this._onPointerUp = onPointerUp.bind(this); + + this.domElement.addEventListener('pointerdown', this._onPointerDown); + this.domElement.addEventListener('pointermove', this._onPointerHover); + this.domElement.addEventListener('pointerup', this._onPointerUp); + + } + + // updateMatrixWorld updates key transformation variables + updateMatrixWorld(force) { + + if (this.object !== undefined) { + + this.object.updateMatrixWorld(); + + if (this.object.parent === null) { + + console.error('TransformControls: The attached 3D object must be a part of the scene graph.'); + + } else { + + this.object.parent.matrixWorld.decompose(this._parentPosition, this._parentQuaternion, this._parentScale); + + } + + this.object.matrixWorld.decompose(this.worldPosition, this.worldQuaternion, this._worldScale); + + this._parentQuaternionInv.copy(this._parentQuaternion).invert(); + this._worldQuaternionInv.copy(this.worldQuaternion).invert(); + + } + + this.camera.updateMatrixWorld(); + this.camera.matrixWorld.decompose(this.cameraPosition, this.cameraQuaternion, this._cameraScale); + + if (this.camera.isOrthographicCamera) { + + this.camera.getWorldDirection(this.eye).negate(); + + } else { + + this.eye.copy(this.cameraPosition).sub(this.worldPosition).normalize(); + + } + + super.updateMatrixWorld(force); + + } + + pointerHover(pointer) { + + if (this.object === undefined || this.dragging === true) return; + + if (pointer !== null) _raycaster.setFromCamera(pointer, this.camera); + + const intersect = intersectObjectWithRay(this._gizmo.picker[this.mode], _raycaster); + + if (intersect) { + + this.axis = intersect.object.name; + + } else { + + this.axis = null; + + } + + } + + pointerDown(pointer) { + + if (this.object === undefined || this.dragging === true || (pointer != null && pointer.button !== 0)) return; + + if (this.axis !== null) { + + if (pointer !== null) _raycaster.setFromCamera(pointer, this.camera); + + const planeIntersect = intersectObjectWithRay(this._plane, _raycaster, true); + + if (planeIntersect) { + + this.object.updateMatrixWorld(); + this.object.parent.updateMatrixWorld(); + + this._positionStart.copy(this.object.position); + this._quaternionStart.copy(this.object.quaternion); + this._scaleStart.copy(this.object.scale); + + this.object.matrixWorld.decompose(this.worldPositionStart, this.worldQuaternionStart, this._worldScaleStart); + + this.pointStart.copy(planeIntersect.point).sub(this.worldPositionStart); + + } + + this.dragging = true; + _mouseDownEvent.mode = this.mode; + this.dispatchEvent(_mouseDownEvent); + + } + + } + + pointerMove(pointer) { + + const axis = this.axis; + const mode = this.mode; + const object = this.object; + let space = this.space; + + if (mode === 'scale') { + + space = 'local'; + + } else if (axis === 'E' || axis === 'XYZE' || axis === 'XYZ') { + + space = 'world'; + + } + + if (object === undefined || axis === null || this.dragging === false || (pointer !== null && pointer.button !== - 1)) return; + + if (pointer !== null) _raycaster.setFromCamera(pointer, this.camera); + + const planeIntersect = intersectObjectWithRay(this._plane, _raycaster, true); + + if (!planeIntersect) return; + + this.pointEnd.copy(planeIntersect.point).sub(this.worldPositionStart); + + if (mode === 'translate') { + + // Apply translate + + this._offset.copy(this.pointEnd).sub(this.pointStart); + + if (space === 'local' && axis !== 'XYZ') { + + this._offset.applyQuaternion(this._worldQuaternionInv); + + } + + if (axis.indexOf('X') === - 1) this._offset.x = 0; + if (axis.indexOf('Y') === - 1) this._offset.y = 0; + if (axis.indexOf('Z') === - 1) this._offset.z = 0; + + if (space === 'local' && axis !== 'XYZ') { + + this._offset.applyQuaternion(this._quaternionStart).divide(this._parentScale); + + } else { + + this._offset.applyQuaternion(this._parentQuaternionInv).divide(this._parentScale); + + } + + object.position.copy(this._offset).add(this._positionStart); + + // Apply translation snap + + if (this.translationSnap) { + + if (space === 'local') { + + object.position.applyQuaternion(_tempQuaternion.copy(this._quaternionStart).invert()); + + if (axis.search('X') !== - 1) { + + object.position.x = Math.round(object.position.x / this.translationSnap) * this.translationSnap; + + } + + if (axis.search('Y') !== - 1) { + + object.position.y = Math.round(object.position.y / this.translationSnap) * this.translationSnap; + + } + + if (axis.search('Z') !== - 1) { + + object.position.z = Math.round(object.position.z / this.translationSnap) * this.translationSnap; + + } + + object.position.applyQuaternion(this._quaternionStart); + + } + + if (space === 'world') { + + if (object.parent) { + + object.position.add(_tempVector.setFromMatrixPosition(object.parent.matrixWorld)); + + } + + if (axis.search('X') !== - 1) { + + object.position.x = Math.round(object.position.x / this.translationSnap) * this.translationSnap; + + } + + if (axis.search('Y') !== - 1) { + + object.position.y = Math.round(object.position.y / this.translationSnap) * this.translationSnap; + + } + + if (axis.search('Z') !== - 1) { + + object.position.z = Math.round(object.position.z / this.translationSnap) * this.translationSnap; + + } + + if (object.parent) { + + object.position.sub(_tempVector.setFromMatrixPosition(object.parent.matrixWorld)); + + } + + } + + } + + } else if (mode === 'scale') { + + if (axis.search('XYZ') !== - 1) { + + let d = (this.pointEnd.length() / this.pointStart.length())/this.sensitivity; + + if (this.pointEnd.dot(this.pointStart) < 0) d *= - 1; + + _tempVector2.set(d, d, d); + + } else { + + _tempVector.copy(this.pointStart); + _tempVector2.copy(this.pointEnd); + + _tempVector.applyQuaternion(this._worldQuaternionInv); + _tempVector2.applyQuaternion(this._worldQuaternionInv); + + _tempVector2.divide(_tempVector); + + if (axis.search('X') === - 1) { + + _tempVector2.x = 1; + + } + + if (axis.search('Y') === - 1) { + + _tempVector2.y = 1; + + } + + if (axis.search('Z') === - 1) { + + _tempVector2.z = 1; + + } + + } + + // Apply scale + object.scale.copy(this._scaleStart).multiply(_tempVector2); + + if (this.scaleSnap) { + + if (axis.search('X') !== - 1) { + + object.scale.x = Math.round(object.scale.x / this.scaleSnap) * this.scaleSnap || this.scaleSnap; + + } + + if (axis.search('Y') !== - 1) { + + object.scale.y = Math.round(object.scale.y / this.scaleSnap) * this.scaleSnap || this.scaleSnap; + + } + + if (axis.search('Z') !== - 1) { + + object.scale.z = Math.round(object.scale.z / this.scaleSnap) * this.scaleSnap || this.scaleSnap; + + } + + } + + } else if (mode === 'rotate') { + + this._offset.copy(this.pointEnd).sub(this.pointStart); + + const ROTATION_SPEED = 20 / this.worldPosition.distanceTo(_tempVector.setFromMatrixPosition(this.camera.matrixWorld)); + + let _inPlaneRotation = false; + + if (axis === 'XYZE') { + + this.rotationAxis.copy(this._offset).cross(this.eye).normalize(); + this.rotationAngle = this._offset.dot(_tempVector.copy(this.rotationAxis).cross(this.eye)) * ROTATION_SPEED; + + } else if (axis === 'X' || axis === 'Y' || axis === 'Z') { + + this.rotationAxis.copy(_unit[axis]); + + _tempVector.copy(_unit[axis]); + + if (space === 'local') { + + _tempVector.applyQuaternion(this.worldQuaternion); + + } + + _tempVector.cross(this.eye); + + // When _tempVector is 0 after cross with this.eye the vectors are parallel and should use in-plane rotation logic. + if (_tempVector.length() === 0) { + + _inPlaneRotation = true; + + } else { + + this.rotationAngle = this._offset.dot(_tempVector.normalize()) * ROTATION_SPEED; + + } + + + } + + if (axis === 'E' || _inPlaneRotation) { + + this.rotationAxis.copy(this.eye); + this.rotationAngle = this.pointEnd.angleTo(this.pointStart); + + this._startNorm.copy(this.pointStart).normalize(); + this._endNorm.copy(this.pointEnd).normalize(); + + this.rotationAngle *= (this._endNorm.cross(this._startNorm).dot(this.eye) < 0 ? 1 : - 1); + + } + + // Apply rotation snap + + if (this.rotationSnap) this.rotationAngle = Math.round(this.rotationAngle / this.rotationSnap) * this.rotationSnap; + + // Apply rotate + if (space === 'local' && axis !== 'E' && axis !== 'XYZE') { + + object.quaternion.copy(this._quaternionStart); + object.quaternion.multiply(_tempQuaternion.setFromAxisAngle(this.rotationAxis, this.rotationAngle)).normalize(); + + } else { + + this.rotationAxis.applyQuaternion(this._parentQuaternionInv); + object.quaternion.copy(_tempQuaternion.setFromAxisAngle(this.rotationAxis, this.rotationAngle)); + object.quaternion.multiply(this._quaternionStart).normalize(); + + } + + } + + this.dispatchEvent(_changeEvent); + this.dispatchEvent(_objectChangeEvent); + + } + + pointerUp(pointer) { + + if (pointer !== null && pointer.button !== 0) return; + + if (this.dragging && (this.axis !== null)) { + + _mouseUpEvent.mode = this.mode; + this.dispatchEvent(_mouseUpEvent); + + } + + this.dragging = false; + this.axis = null; + + } + + dispose() { + + this.domElement.removeEventListener('pointerdown', this._onPointerDown); + this.domElement.removeEventListener('pointermove', this._onPointerHover); + this.domElement.removeEventListener('pointermove', this._onPointerMove); + this.domElement.removeEventListener('pointerup', this._onPointerUp); + + this.traverse(function (child) { + + if (child.geometry) child.geometry.dispose(); + if (child.material) child.material.dispose(); + + }); + + } + + // Set current object + attach(object) { + + this.object = object; + this.visible = true; + + return this; + + } + + // Detach from object + detach() { + + this.object = undefined; + this.visible = false; + this.axis = null; + + return this; + + } + + reset() { + + if (!this.enabled) return; + + if (this.dragging) { + + this.object.position.copy(this._positionStart); + this.object.quaternion.copy(this._quaternionStart); + this.object.scale.copy(this._scaleStart); + + this.dispatchEvent(_changeEvent); + this.dispatchEvent(_objectChangeEvent); + + this.pointStart.copy(this.pointEnd); + + } + + } + + getRaycaster() { + + return _raycaster; + + } + + // TODO: deprecate + + getMode() { + + return this.mode; + + } + + setMode(mode) { + + this.mode = mode; + + } + + setTranslationSnap(translationSnap) { + + this.translationSnap = translationSnap; + + } + + setRotationSnap(rotationSnap) { + + this.rotationSnap = rotationSnap; + + } + + setScaleSnap(scaleSnap) { + + this.scaleSnap = scaleSnap; + + } + + setSize(size) { + + this.size = size; + + } + + setSpace(space) { + + this.space = space; + + } + +} + +// mouse / touch event handlers + +function getPointer(event) { + + if (this.domElement.ownerDocument.pointerLockElement) { + + return { + x: 0, + y: 0, + button: event.button + }; + + } else { + + const rect = this.domElement.getBoundingClientRect(); + + return { + x: (event.clientX - rect.left) / rect.width * 2 - 1, + y: - (event.clientY - rect.top) / rect.height * 2 + 1, + button: event.button + }; + + } + +} + +function onPointerHover(event) { + + if (!this.enabled) return; + + switch (event.pointerType) { + + case 'mouse': + case 'pen': + this.pointerHover(this._getPointer(event)); + break; + + } + +} + +function onPointerDown(event) { + + if (!this.enabled) return; + + if (!document.pointerLockElement) { + + this.domElement.setPointerCapture(event.pointerId); + + } + + this.domElement.addEventListener('pointermove', this._onPointerMove); + + this.pointerHover(this._getPointer(event)); + this.pointerDown(this._getPointer(event)); + +} + +function onPointerMove(event) { + + if (!this.enabled) return; + + this.pointerMove(this._getPointer(event)); + +} + +function onPointerUp(event) { + + if (!this.enabled) return; + + this.domElement.releasePointerCapture(event.pointerId); + + this.domElement.removeEventListener('pointermove', this._onPointerMove); + + this.pointerUp(this._getPointer(event)); + +} + +function intersectObjectWithRay(object, raycaster, includeInvisible) { + + const allIntersections = raycaster.intersectObject(object, true); + + for (let i = 0; i < allIntersections.length; i++) { + + if (allIntersections[i].object.visible || includeInvisible) { + + return allIntersections[i]; + + } + + } + + return false; + +} + +// + +// Reusable utility variables + +const _tempEuler = new Euler(); +const _alignVector = new Vector3(0, 1, 0); +const _zeroVector = new Vector3(0, 0, 0); +const _lookAtMatrix = new Matrix4(); +const _tempQuaternion2 = new Quaternion(); +const _identityQuaternion = new Quaternion(); +const _dirVector = new Vector3(); +const _tempMatrix = new Matrix4(); + +const _unitX = new Vector3(1, 0, 0); +const _unitY = new Vector3(0, 1, 0); +const _unitZ = new Vector3(0, 0, 1); + +const _v1 = new Vector3(); +const _v2 = new Vector3(); +const _v3 = new Vector3(); + +class TransformControlsGizmo extends Object3D { + + constructor() { + + super(); + + this.isTransformControlsGizmo = true; + + this.type = 'TransformControlsGizmo'; + + // shared materials + + const gizmoMaterial = new MeshBasicMaterial({ + depthTest: false, + depthWrite: false, + fog: false, + toneMapped: false, + transparent: true + }); + + const gizmoLineMaterial = new LineBasicMaterial({ + depthTest: false, + depthWrite: false, + fog: false, + toneMapped: false, + transparent: true + }); + + // Make unique material for each axis/color + + const matInvisible = gizmoMaterial.clone(); + matInvisible.opacity = 0.15; + + const matHelper = gizmoLineMaterial.clone(); + matHelper.opacity = 0.5; + + const matRed = gizmoMaterial.clone(); + matRed.color.setHex(0xff0000); + + const matGreen = gizmoMaterial.clone(); + matGreen.color.setHex(0x00ff00); + + const matBlue = gizmoMaterial.clone(); + matBlue.color.setHex(0x0000ff); + + const matRedTransparent = gizmoMaterial.clone(); + matRedTransparent.color.setHex(0xff0000); + matRedTransparent.opacity = 0.5; + + const matGreenTransparent = gizmoMaterial.clone(); + matGreenTransparent.color.setHex(0x00ff00); + matGreenTransparent.opacity = 0.5; + + const matBlueTransparent = gizmoMaterial.clone(); + matBlueTransparent.color.setHex(0x0000ff); + matBlueTransparent.opacity = 0.5; + + const matWhiteTransparent = gizmoMaterial.clone(); + matWhiteTransparent.opacity = 0.25; + + const matYellowTransparent = gizmoMaterial.clone(); + matYellowTransparent.color.setHex(0xffff00); + matYellowTransparent.opacity = 0.25; + + const matYellow = gizmoMaterial.clone(); + matYellow.color.setHex(0xffff00); + + const matGray = gizmoMaterial.clone(); + matGray.color.setHex(0x787878); + + // reusable geometry + + const arrowGeometry = new CylinderGeometry(0, 0.04, 0.1, 12); + arrowGeometry.translate(0, 0.05, 0); + + const scaleHandleGeometry = new BoxGeometry(0.08, 0.08, 0.08); + scaleHandleGeometry.translate(0, 0.04, 0); + + const lineGeometry = new BufferGeometry(); + lineGeometry.setAttribute('position', new Float32BufferAttribute([0, 0, 0, 1, 0, 0], 3)); + + const lineGeometry2 = new CylinderGeometry(0.0075, 0.0075, 0.5, 3); + lineGeometry2.translate(0, 0.25, 0); + + function CircleGeometry(radius, arc) { + + const geometry = new TorusGeometry(radius, 0.0075, 3, 64, arc * Math.PI * 2); + geometry.rotateY(Math.PI / 2); + geometry.rotateX(Math.PI / 2); + return geometry; + + } + + // Special geometry for transform helper. If scaled with position vector it spans from [0,0,0] to position + + function TranslateHelperGeometry() { + + const geometry = new BufferGeometry(); + + geometry.setAttribute('position', new Float32BufferAttribute([0, 0, 0, 1, 1, 1], 3)); + + return geometry; + + } + + // Gizmo definitions - custom hierarchy definitions for setupGizmo() function + + const gizmoTranslate = { + X: [ + [new Mesh(arrowGeometry, matRed), [0.5, 0, 0], [0, 0, - Math.PI / 2]], + [new Mesh(arrowGeometry, matRed), [- 0.5, 0, 0], [0, 0, Math.PI / 2]], + [new Mesh(lineGeometry2, matRed), [0, 0, 0], [0, 0, - Math.PI / 2]] + ], + Y: [ + [new Mesh(arrowGeometry, matGreen), [0, 0.5, 0]], + [new Mesh(arrowGeometry, matGreen), [0, - 0.5, 0], [Math.PI, 0, 0]], + [new Mesh(lineGeometry2, matGreen)] + ], + Z: [ + [new Mesh(arrowGeometry, matBlue), [0, 0, 0.5], [Math.PI / 2, 0, 0]], + [new Mesh(arrowGeometry, matBlue), [0, 0, - 0.5], [- Math.PI / 2, 0, 0]], + [new Mesh(lineGeometry2, matBlue), null, [Math.PI / 2, 0, 0]] + ], + XYZ: [ + [new Mesh(new OctahedronGeometry(0.1, 0), matWhiteTransparent.clone()), [0, 0, 0]] + ], + XY: [ + [new Mesh(new BoxGeometry(0.15, 0.15, 0.01), matBlueTransparent.clone()), [0.15, 0.15, 0]] + ], + YZ: [ + [new Mesh(new BoxGeometry(0.15, 0.15, 0.01), matRedTransparent.clone()), [0, 0.15, 0.15], [0, Math.PI / 2, 0]] + ], + XZ: [ + [new Mesh(new BoxGeometry(0.15, 0.15, 0.01), matGreenTransparent.clone()), [0.15, 0, 0.15], [- Math.PI / 2, 0, 0]] + ] + }; + + const pickerTranslate = { + X: [ + [new Mesh(new CylinderGeometry(0.2, 0, 0.6, 4), matInvisible), [0.3, 0, 0], [0, 0, - Math.PI / 2]], + [new Mesh(new CylinderGeometry(0.2, 0, 0.6, 4), matInvisible), [- 0.3, 0, 0], [0, 0, Math.PI / 2]] + ], + Y: [ + [new Mesh(new CylinderGeometry(0.2, 0, 0.6, 4), matInvisible), [0, 0.3, 0]], + [new Mesh(new CylinderGeometry(0.2, 0, 0.6, 4), matInvisible), [0, - 0.3, 0], [0, 0, Math.PI]] + ], + Z: [ + [new Mesh(new CylinderGeometry(0.2, 0, 0.6, 4), matInvisible), [0, 0, 0.3], [Math.PI / 2, 0, 0]], + [new Mesh(new CylinderGeometry(0.2, 0, 0.6, 4), matInvisible), [0, 0, - 0.3], [- Math.PI / 2, 0, 0]] + ], + XYZ: [ + [new Mesh(new OctahedronGeometry(0.2, 0), matInvisible)] + ], + XY: [ + [new Mesh(new BoxGeometry(0.2, 0.2, 0.01), matInvisible), [0.15, 0.15, 0]] + ], + YZ: [ + [new Mesh(new BoxGeometry(0.2, 0.2, 0.01), matInvisible), [0, 0.15, 0.15], [0, Math.PI / 2, 0]] + ], + XZ: [ + [new Mesh(new BoxGeometry(0.2, 0.2, 0.01), matInvisible), [0.15, 0, 0.15], [- Math.PI / 2, 0, 0]] + ] + }; + + const helperTranslate = { + START: [ + [new Mesh(new OctahedronGeometry(0.01, 2), matHelper), null, null, null, 'helper'] + ], + END: [ + [new Mesh(new OctahedronGeometry(0.01, 2), matHelper), null, null, null, 'helper'] + ], + DELTA: [ + [new Line(TranslateHelperGeometry(), matHelper), null, null, null, 'helper'] + ], + X: [ + [new Line(lineGeometry, matHelper.clone()), [- 1e3, 0, 0], null, [1e6, 1, 1], 'helper'] + ], + Y: [ + [new Line(lineGeometry, matHelper.clone()), [0, - 1e3, 0], [0, 0, Math.PI / 2], [1e6, 1, 1], 'helper'] + ], + Z: [ + [new Line(lineGeometry, matHelper.clone()), [0, 0, - 1e3], [0, - Math.PI / 2, 0], [1e6, 1, 1], 'helper'] + ] + }; + + const gizmoRotate = { + XYZE: [ + [new Mesh(CircleGeometry(0.5, 1), matGray), null, [0, Math.PI / 2, 0]] + ], + X: [ + [new Mesh(CircleGeometry(0.5, 0.5), matRed)] + ], + Y: [ + [new Mesh(CircleGeometry(0.5, 0.5), matGreen), null, [0, 0, - Math.PI / 2]] + ], + Z: [ + [new Mesh(CircleGeometry(0.5, 0.5), matBlue), null, [0, Math.PI / 2, 0]] + ], + E: [ + [new Mesh(CircleGeometry(0.75, 1), matYellowTransparent), null, [0, Math.PI / 2, 0]] + ] + }; + + const helperRotate = { + AXIS: [ + [new Line(lineGeometry, matHelper.clone()), [- 1e3, 0, 0], null, [1e6, 1, 1], 'helper'] + ] + }; + + const pickerRotate = { + XYZE: [ + [new Mesh(new SphereGeometry(0.25, 10, 8), matInvisible)] + ], + X: [ + [new Mesh(new TorusGeometry(0.5, 0.1, 4, 24), matInvisible), [0, 0, 0], [0, - Math.PI / 2, - Math.PI / 2]], + ], + Y: [ + [new Mesh(new TorusGeometry(0.5, 0.1, 4, 24), matInvisible), [0, 0, 0], [Math.PI / 2, 0, 0]], + ], + Z: [ + [new Mesh(new TorusGeometry(0.5, 0.1, 4, 24), matInvisible), [0, 0, 0], [0, 0, - Math.PI / 2]], + ], + E: [ + [new Mesh(new TorusGeometry(0.75, 0.1, 2, 24), matInvisible)] + ] + }; + + const gizmoScale = { + X: [ + [new Mesh(scaleHandleGeometry, matRed), [0.5, 0, 0], [0, 0, - Math.PI / 2]], + [new Mesh(lineGeometry2, matRed), [0, 0, 0], [0, 0, - Math.PI / 2]], + [new Mesh(scaleHandleGeometry, matRed), [- 0.5, 0, 0], [0, 0, Math.PI / 2]], + ], + Y: [ + [new Mesh(scaleHandleGeometry, matGreen), [0, 0.5, 0]], + [new Mesh(lineGeometry2, matGreen)], + [new Mesh(scaleHandleGeometry, matGreen), [0, - 0.5, 0], [0, 0, Math.PI]], + ], + Z: [ + [new Mesh(scaleHandleGeometry, matBlue), [0, 0, 0.5], [Math.PI / 2, 0, 0]], + [new Mesh(lineGeometry2, matBlue), [0, 0, 0], [Math.PI / 2, 0, 0]], + [new Mesh(scaleHandleGeometry, matBlue), [0, 0, - 0.5], [- Math.PI / 2, 0, 0]] + ], + XY: [ + [new Mesh(new BoxGeometry(0.15, 0.15, 0.01), matBlueTransparent), [0.15, 0.15, 0]] + ], + YZ: [ + [new Mesh(new BoxGeometry(0.15, 0.15, 0.01), matRedTransparent), [0, 0.15, 0.15], [0, Math.PI / 2, 0]] + ], + XZ: [ + [new Mesh(new BoxGeometry(0.15, 0.15, 0.01), matGreenTransparent), [0.15, 0, 0.15], [- Math.PI / 2, 0, 0]] + ], + XYZ: [ + [new Mesh(new BoxGeometry(0.1, 0.1, 0.1), matWhiteTransparent.clone())], + ] + }; + + const pickerScale = { + X: [ + [new Mesh(new CylinderGeometry(0.2, 0, 0.6, 4), matInvisible), [0.3, 0, 0], [0, 0, - Math.PI / 2]], + [new Mesh(new CylinderGeometry(0.2, 0, 0.6, 4), matInvisible), [- 0.3, 0, 0], [0, 0, Math.PI / 2]] + ], + Y: [ + [new Mesh(new CylinderGeometry(0.2, 0, 0.6, 4), matInvisible), [0, 0.3, 0]], + [new Mesh(new CylinderGeometry(0.2, 0, 0.6, 4), matInvisible), [0, - 0.3, 0], [0, 0, Math.PI]] + ], + Z: [ + [new Mesh(new CylinderGeometry(0.2, 0, 0.6, 4), matInvisible), [0, 0, 0.3], [Math.PI / 2, 0, 0]], + [new Mesh(new CylinderGeometry(0.2, 0, 0.6, 4), matInvisible), [0, 0, - 0.3], [- Math.PI / 2, 0, 0]] + ], + XY: [ + [new Mesh(new BoxGeometry(0.2, 0.2, 0.01), matInvisible), [0.15, 0.15, 0]], + ], + YZ: [ + [new Mesh(new BoxGeometry(0.2, 0.2, 0.01), matInvisible), [0, 0.15, 0.15], [0, Math.PI / 2, 0]], + ], + XZ: [ + [new Mesh(new BoxGeometry(0.2, 0.2, 0.01), matInvisible), [0.15, 0, 0.15], [- Math.PI / 2, 0, 0]], + ], + XYZ: [ + [new Mesh(new BoxGeometry(0.2, 0.2, 0.2), matInvisible), [0, 0, 0]], + ] + }; + + const helperScale = { + X: [ + [new Line(lineGeometry, matHelper.clone()), [- 1e3, 0, 0], null, [1e6, 1, 1], 'helper'] + ], + Y: [ + [new Line(lineGeometry, matHelper.clone()), [0, - 1e3, 0], [0, 0, Math.PI / 2], [1e6, 1, 1], 'helper'] + ], + Z: [ + [new Line(lineGeometry, matHelper.clone()), [0, 0, - 1e3], [0, - Math.PI / 2, 0], [1e6, 1, 1], 'helper'] + ] + }; + + // Creates an Object3D with gizmos described in custom hierarchy definition. + + function setupGizmo(gizmoMap) { + + const gizmo = new Object3D(); + + for (const name in gizmoMap) { + + for (let i = gizmoMap[name].length; i--;) { + + const object = gizmoMap[name][i][0].clone(); + const position = gizmoMap[name][i][1]; + const rotation = gizmoMap[name][i][2]; + const scale = gizmoMap[name][i][3]; + const tag = gizmoMap[name][i][4]; + + // name and tag properties are essential for picking and updating logic. + object.name = name; + object.tag = tag; + + if (position) { + + object.position.set(position[0], position[1], position[2]); + + } + + if (rotation) { + + object.rotation.set(rotation[0], rotation[1], rotation[2]); + + } + + if (scale) { + + object.scale.set(scale[0], scale[1], scale[2]); + + } + + object.updateMatrix(); + + const tempGeometry = object.geometry.clone(); + tempGeometry.applyMatrix4(object.matrix); + object.geometry = tempGeometry; + object.renderOrder = Infinity; + + object.position.set(0, 0, 0); + object.rotation.set(0, 0, 0); + object.scale.set(1, 1, 1); + + gizmo.add(object); + + } + + } + + return gizmo; + + } + + // Gizmo creation + + this.gizmo = {}; + this.picker = {}; + this.helper = {}; + + this.add(this.gizmo['translate'] = setupGizmo(gizmoTranslate)); + this.add(this.gizmo['rotate'] = setupGizmo(gizmoRotate)); + this.add(this.gizmo['scale'] = setupGizmo(gizmoScale)); + this.add(this.picker['translate'] = setupGizmo(pickerTranslate)); + this.add(this.picker['rotate'] = setupGizmo(pickerRotate)); + this.add(this.picker['scale'] = setupGizmo(pickerScale)); + this.add(this.helper['translate'] = setupGizmo(helperTranslate)); + this.add(this.helper['rotate'] = setupGizmo(helperRotate)); + this.add(this.helper['scale'] = setupGizmo(helperScale)); + + // Pickers should be hidden always + + this.picker['translate'].visible = false; + this.picker['rotate'].visible = false; + this.picker['scale'].visible = false; + + } + + // updateMatrixWorld will update transformations and appearance of individual handles + + updateMatrixWorld(force) { + + const space = (this.mode === 'scale') ? 'local' : this.space; // scale always oriented to local rotation + + const quaternion = (space === 'local') ? this.worldQuaternion : _identityQuaternion; + + // Show only gizmos for current transform mode + + this.gizmo['translate'].visible = this.mode === 'translate'; + this.gizmo['rotate'].visible = this.mode === 'rotate'; + this.gizmo['scale'].visible = this.mode === 'scale'; + + this.helper['translate'].visible = this.mode === 'translate'; + this.helper['rotate'].visible = this.mode === 'rotate'; + this.helper['scale'].visible = this.mode === 'scale'; + + + let handles = []; + handles = handles.concat(this.picker[this.mode].children); + handles = handles.concat(this.gizmo[this.mode].children); + handles = handles.concat(this.helper[this.mode].children); + + for (let i = 0; i < handles.length; i++) { + + const handle = handles[i]; + + // hide aligned to camera + + handle.visible = true; + handle.rotation.set(0, 0, 0); + handle.position.copy(this.worldPosition); + + let factor; + + if (this.camera.isOrthographicCamera) { + + factor = (this.camera.top - this.camera.bottom) / this.camera.zoom; + + } else { + + factor = this.worldPosition.distanceTo(this.cameraPosition) * Math.min(1.9 * Math.tan(Math.PI * this.camera.fov / 360) / this.camera.zoom, 7); + + } + + handle.scale.set(1, 1, 1).multiplyScalar(factor * this.size / 4); + + // TODO: simplify helpers and consider decoupling from gizmo + + if (handle.tag === 'helper') { + + handle.visible = false; + + if (handle.name === 'AXIS') { + + handle.visible = !!this.axis; + + if (this.axis === 'X') { + + _tempQuaternion.setFromEuler(_tempEuler.set(0, 0, 0)); + handle.quaternion.copy(quaternion).multiply(_tempQuaternion); + + if (Math.abs(_alignVector.copy(_unitX).applyQuaternion(quaternion).dot(this.eye)) > 0.9) { + + handle.visible = false; + + } + + } + + if (this.axis === 'Y') { + + _tempQuaternion.setFromEuler(_tempEuler.set(0, 0, Math.PI / 2)); + handle.quaternion.copy(quaternion).multiply(_tempQuaternion); + + if (Math.abs(_alignVector.copy(_unitY).applyQuaternion(quaternion).dot(this.eye)) > 0.9) { + + handle.visible = false; + + } + + } + + if (this.axis === 'Z') { + + _tempQuaternion.setFromEuler(_tempEuler.set(0, Math.PI / 2, 0)); + handle.quaternion.copy(quaternion).multiply(_tempQuaternion); + + if (Math.abs(_alignVector.copy(_unitZ).applyQuaternion(quaternion).dot(this.eye)) > 0.9) { + + handle.visible = false; + + } + + } + + if (this.axis === 'XYZE') { + + _tempQuaternion.setFromEuler(_tempEuler.set(0, Math.PI / 2, 0)); + _alignVector.copy(this.rotationAxis); + handle.quaternion.setFromRotationMatrix(_lookAtMatrix.lookAt(_zeroVector, _alignVector, _unitY)); + handle.quaternion.multiply(_tempQuaternion); + handle.visible = this.dragging; + + } + + if (this.axis === 'E') { + + handle.visible = false; + + } + + + } else if (handle.name === 'START') { + + handle.position.copy(this.worldPositionStart); + handle.visible = this.dragging; + + } else if (handle.name === 'END') { + + handle.position.copy(this.worldPosition); + handle.visible = this.dragging; + + } else if (handle.name === 'DELTA') { + + handle.position.copy(this.worldPositionStart); + handle.quaternion.copy(this.worldQuaternionStart); + _tempVector.set(1e-10, 1e-10, 1e-10).add(this.worldPositionStart).sub(this.worldPosition).multiplyScalar(- 1); + _tempVector.applyQuaternion(this.worldQuaternionStart.clone().invert()); + handle.scale.copy(_tempVector); + handle.visible = this.dragging; + + } else { + + handle.quaternion.copy(quaternion); + + if (this.dragging) { + + handle.position.copy(this.worldPositionStart); + + } else { + + handle.position.copy(this.worldPosition); + + } + + if (this.axis) { + + handle.visible = this.axis.search(handle.name) !== - 1; + + } + + } + + // If updating helper, skip rest of the loop + continue; + + } + + // Align handles to current local or world rotation + + handle.quaternion.copy(quaternion); + + if (this.mode === 'translate' || this.mode === 'scale') { + + // Hide translate and scale axis facing the camera + + const AXIS_HIDE_THRESHOLD = 0.99; + const PLANE_HIDE_THRESHOLD = 0.2; + + if (handle.name === 'X') { + + if (Math.abs(_alignVector.copy(_unitX).applyQuaternion(quaternion).dot(this.eye)) > AXIS_HIDE_THRESHOLD) { + + handle.scale.set(1e-10, 1e-10, 1e-10); + handle.visible = false; + + } + + } + + if (handle.name === 'Y') { + + if (Math.abs(_alignVector.copy(_unitY).applyQuaternion(quaternion).dot(this.eye)) > AXIS_HIDE_THRESHOLD) { + + handle.scale.set(1e-10, 1e-10, 1e-10); + handle.visible = false; + + } + + } + + if (handle.name === 'Z') { + + if (Math.abs(_alignVector.copy(_unitZ).applyQuaternion(quaternion).dot(this.eye)) > AXIS_HIDE_THRESHOLD) { + + handle.scale.set(1e-10, 1e-10, 1e-10); + handle.visible = false; + + } + + } + + if (handle.name === 'XY') { + + if (Math.abs(_alignVector.copy(_unitZ).applyQuaternion(quaternion).dot(this.eye)) < PLANE_HIDE_THRESHOLD) { + + handle.scale.set(1e-10, 1e-10, 1e-10); + handle.visible = false; + + } + + } + + if (handle.name === 'YZ') { + + if (Math.abs(_alignVector.copy(_unitX).applyQuaternion(quaternion).dot(this.eye)) < PLANE_HIDE_THRESHOLD) { + + handle.scale.set(1e-10, 1e-10, 1e-10); + handle.visible = false; + + } + + } + + if (handle.name === 'XZ') { + + if (Math.abs(_alignVector.copy(_unitY).applyQuaternion(quaternion).dot(this.eye)) < PLANE_HIDE_THRESHOLD) { + + handle.scale.set(1e-10, 1e-10, 1e-10); + handle.visible = false; + + } + + } + + } else if (this.mode === 'rotate') { + + // Align handles to current local or world rotation + + _tempQuaternion2.copy(quaternion); + _alignVector.copy(this.eye).applyQuaternion(_tempQuaternion.copy(quaternion).invert()); + + if (handle.name.search('E') !== - 1) { + + handle.quaternion.setFromRotationMatrix(_lookAtMatrix.lookAt(this.eye, _zeroVector, _unitY)); + + } + + if (handle.name === 'X') { + + _tempQuaternion.setFromAxisAngle(_unitX, Math.atan2(- _alignVector.y, _alignVector.z)); + _tempQuaternion.multiplyQuaternions(_tempQuaternion2, _tempQuaternion); + handle.quaternion.copy(_tempQuaternion); + + } + + if (handle.name === 'Y') { + + _tempQuaternion.setFromAxisAngle(_unitY, Math.atan2(_alignVector.x, _alignVector.z)); + _tempQuaternion.multiplyQuaternions(_tempQuaternion2, _tempQuaternion); + handle.quaternion.copy(_tempQuaternion); + + } + + if (handle.name === 'Z') { + + _tempQuaternion.setFromAxisAngle(_unitZ, Math.atan2(_alignVector.y, _alignVector.x)); + _tempQuaternion.multiplyQuaternions(_tempQuaternion2, _tempQuaternion); + handle.quaternion.copy(_tempQuaternion); + + } + + } + + // Hide disabled axes + handle.visible = handle.visible && (handle.name.indexOf('X') === - 1 || this.showX); + handle.visible = handle.visible && (handle.name.indexOf('Y') === - 1 || this.showY); + handle.visible = handle.visible && (handle.name.indexOf('Z') === - 1 || this.showZ); + handle.visible = handle.visible && (handle.name.indexOf('E') === - 1 || (this.showX && this.showY && this.showZ)); + + // highlight selected axis + + handle.material._color = handle.material._color || handle.material.color.clone(); + handle.material._opacity = handle.material._opacity || handle.material.opacity; + + handle.material.color.copy(handle.material._color); + handle.material.opacity = handle.material._opacity; + + if (this.enabled && this.axis) { + + if (handle.name === this.axis) { + + handle.material.color.setHex(0xffff00); + handle.material.opacity = 1.0; + + } else if (this.axis.split('').some(function (a) { + + return handle.name === a; + + })) { + + handle.material.color.setHex(0xffff00); + handle.material.opacity = 1.0; + + } + + } + + } + + super.updateMatrixWorld(force); + + } + +} + +// + +class TransformControlsPlane extends Mesh { + + constructor() { + + super( + new PlaneGeometry(100000, 100000, 2, 2), + new MeshBasicMaterial({ visible: false, wireframe: true, side: DoubleSide, transparent: true, opacity: 0.1, toneMapped: false }) + ); + + this.isTransformControlsPlane = true; + + this.type = 'TransformControlsPlane'; + + } + + updateMatrixWorld(force) { + + let space = this.space; + + this.position.copy(this.worldPosition); + + if (this.mode === 'scale') space = 'local'; // scale always oriented to local rotation + + _v1.copy(_unitX).applyQuaternion(space === 'local' ? this.worldQuaternion : _identityQuaternion); + _v2.copy(_unitY).applyQuaternion(space === 'local' ? this.worldQuaternion : _identityQuaternion); + _v3.copy(_unitZ).applyQuaternion(space === 'local' ? this.worldQuaternion : _identityQuaternion); + + // Align the plane for current transform mode, axis and space. + + _alignVector.copy(_v2); + + switch (this.mode) { + + case 'translate': + case 'scale': + switch (this.axis) { + + case 'X': + _alignVector.copy(this.eye).cross(_v1); + _dirVector.copy(_v1).cross(_alignVector); + break; + case 'Y': + _alignVector.copy(this.eye).cross(_v2); + _dirVector.copy(_v2).cross(_alignVector); + break; + case 'Z': + _alignVector.copy(this.eye).cross(_v3); + _dirVector.copy(_v3).cross(_alignVector); + break; + case 'XY': + _dirVector.copy(_v3); + break; + case 'YZ': + _dirVector.copy(_v1); + break; + case 'XZ': + _alignVector.copy(_v3); + _dirVector.copy(_v2); + break; + case 'XYZ': + case 'E': + _dirVector.set(0, 0, 0); + break; + + } + + break; + case 'rotate': + default: + // special case for rotate + _dirVector.set(0, 0, 0); + + } + + if (_dirVector.length() === 0) { + + // If in rotate mode, make the plane parallel to camera + this.quaternion.copy(this.cameraQuaternion); + + } else { + + _tempMatrix.lookAt(_tempVector.set(0, 0, 0), _dirVector, _alignVector); + + this.quaternion.setFromRotationMatrix(_tempMatrix); + + } + + super.updateMatrixWorld(force); + + } + +} + +export { TransformControls, TransformControlsGizmo, TransformControlsPlane }; diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index 466c900045..82de37ce35 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -354,7 +354,8 @@ export class APIManager { positive_prompt: string, negative_prompt: string, ): Promise { - const url = 'http://213.173.110.135:15750/preview'; + //const url = 'http://213.173.110.135:15750/preview'; + const url = 'https://funnel.tailce84f.ts.net/preview/'; const payload = { style: style, diff --git a/app/pages/PageEnigma/js/audio_engine.ts b/app/pages/PageEnigma/js/audio_engine.ts index e840cb9f06..4c1ac60262 100644 --- a/app/pages/PageEnigma/js/audio_engine.ts +++ b/app/pages/PageEnigma/js/audio_engine.ts @@ -1,4 +1,3 @@ -import { faL } from "@fortawesome/pro-solid-svg-icons"; import { AudioClip } from "../datastructures/clips/audio_clip"; class AudioEngine { @@ -6,12 +5,14 @@ class AudioEngine { audio_sources: { [key: string]: AudioBufferSourceNode } = {}; version: number; playing: string[]; + last_frame: number; constructor() { this.clips = {}; this.playing = []; this.audio_sources = {}; this.version = 1.0; + this.last_frame = 0; } // loads clips into the engine to cache diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 998eba1879..728b46a170 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -1,7 +1,7 @@ import * as THREE from "three"; import { OrbitControls } from "three/addons/controls/OrbitControls.js"; import { FreeCam } from "./free_cam"; -import { TransformControls } from "three/addons/controls/TransformControls.js"; +import { TransformControls } from "./TransformControls.js"; import Scene from "./scene.js"; import { APIManager, ArtStyle, Visibility } from "./api_manager.js"; import { EffectComposer } from "three/addons/postprocessing/EffectComposer.js"; @@ -13,8 +13,11 @@ import { SAOPass } from "three/addons/postprocessing/SAOPass.js"; import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js"; import { BokehPass } from "three/addons/postprocessing/BokehPass.js"; import { createFFmpeg, fetchFile, FFmpeg } from "@ffmpeg/ffmpeg"; + import AudioEngine from "./audio_engine.js"; import TransformEngine from "./transform_engine.js"; +import EmotionEngine from "./emotion_engine"; + import { TimeLine } from "./timeline.js"; import { ClipUI } from "../datastructures/clips/clip_ui.js"; import { LipSyncEngine } from "./lip_sync_engine.js"; @@ -77,12 +80,14 @@ class Editor { max_length: number; audio_engine: AudioEngine; transform_engine: TransformEngine; + emotion_engine: EmotionEngine; lipsync_engine: LipSyncEngine; animation_engine: AnimationEngine; timeline: TimeLine; current_frame: number; lockControls: PointerLockControls | undefined; cam_obj: THREE.Object3D | undefined; + camera_last_pos: THREE.Vector3; renderPass: RenderPass | undefined; generating_preview: boolean; frames: number; @@ -107,9 +112,6 @@ class Editor { recorder: MediaRecorder | undefined; // Default params. - // scene proxy for serialization - storyteller_proxy_scene: StoryTellerProxyScene; - constructor() { console.log( "If you see this message twice! then it rendered twice, if you see it once it's all good.", @@ -142,6 +144,7 @@ class Editor { this.outlinePass; this.last_cam_pos = new THREE.Vector3(0, 0, 0); this.last_cam_rot = new THREE.Euler(0, 0, 0); + this.camera_last_pos = new THREE.Vector3(0, 0, 0); this.lockControls; this.saoPass; this.outputPass; @@ -186,6 +189,7 @@ class Editor { this.canvasRenderCamReference; this.audio_engine = new AudioEngine(); + this.emotion_engine = new EmotionEngine(this.version); this.transform_engine = new TransformEngine(this.version); this.lipsync_engine = new LipSyncEngine(); this.animation_engine = new AnimationEngine(this.version); @@ -196,9 +200,10 @@ class Editor { this.transform_engine, this.lipsync_engine, this.animation_engine, + this.emotion_engine, this.activeScene, this.camera, - this.mouse + this.mouse, ); this.current_frame = 0; @@ -215,11 +220,6 @@ class Editor { "((masterpiece, best quality, 8K, detailed)), colorful, epic, fantasy, (fox, red fox:1.2), no humans, 1other, ((koi pond)), outdoors, pond, rocks, stones, koi fish, ((watercolor))), lilypad, fish swimming around."; this.negative_prompt = ""; this.art_style = ArtStyle.Anime2DFlat; - - this.storyteller_proxy_scene = new StoryTellerProxyScene( - this.version, - this.activeScene.scene, - ); } isEmpty(value: string) { @@ -228,7 +228,7 @@ class Editor { ); } - initialize(config: any, sceneToken) { + initialize(config: any, sceneToken: any) { //setup reactland Callbacks this.dispatchAppUiState = config.dispatchAppUiState; @@ -247,9 +247,15 @@ class Editor { this.canvReference = document.getElementById("video-scene"); this.canvasRenderCamReference = document.getElementById("camera-view"); - // Base width and height. - const width = this.canvReference.width; - const height = this.canvReference.height; + // Find the container element + const container = document.getElementById("video-scene-container"); + + if(container == null) { return; } + + // Use the container's dimensions + const width = container.offsetWidth; + const height = container.offsetHeight; + // Sets up camera and base position. this.camera = new THREE.PerspectiveCamera(70, width / height, 0.15, 30); this.camera.position.z = 3; @@ -282,6 +288,7 @@ class Editor { this.clock = new THREE.Clock(); // Resizes the renderer. this.renderer.setSize(width, height); + this.renderer.setPixelRatio(window.devicePixelRatio); //document.body.appendChild(this.renderer.domElement) window.addEventListener("resize", this.onWindowResize.bind(this)); this._configurePostProcessing(); @@ -307,13 +314,20 @@ class Editor { this.renderer.domElement, ); + this.orbitControls.mouseButtons = { + MIDDLE: THREE.MOUSE.ROTATE, + RIGHT: THREE.MOUSE.PAN, + }; // Blender Style + // this.orbitControls.mouseButtons = { LEFT: MOUSE.ROTATE, MIDDLE: MOUSE.DOLLY, RIGHT: MOUSE.PAN }; // Standard + this.control = new TransformControls(this.camera, this.renderer.domElement); this.control.space = "local"; // Local transformation mode // .space = 'world'; // Global mode - this.control.setScaleSnap(0.1); - this.control.setTranslationSnap(0.1); - this.control.setRotationSnap(0.1); + this.control.setScaleSnap(0.05); + this.control.setTranslationSnap(0.05); + this.control.setRotationSnap(0.05); + console.log("Control Sensitivity:", this.control.sensitivity); // OnClick and MouseMove events. window.addEventListener("mousemove", this.onMouseMove.bind(this), false); @@ -329,6 +343,7 @@ class Editor { } this.orbitControls.enabled = !event.value; this.updateSelectedUI(); + this.camera_last_pos.copy(new THREE.Vector3(-99999, -99999, -99999)); // this.update_properties() }); this.control.setSize(0.5); // Good default value for visuals. @@ -338,6 +353,8 @@ class Editor { // Resets canvas size. this.onWindowResize(); + this.setupResizeObserver(); + this.timeline.scene = this.activeScene; //this._test_demo(); @@ -368,19 +385,17 @@ class Editor { this.loadScene(sceneToken); } - document.addEventListener('mouseover', (event) => { + document.addEventListener("mouseover", (event) => { if (this.orbitControls && this.cameraViewControls) { if (event.target instanceof HTMLCanvasElement) { if (this.camera_person_mode) { this.orbitControls.enabled = false; this.cameraViewControls.enabled = true; - } - else { + } else { this.orbitControls.enabled = true; this.cameraViewControls.enabled = false; } - } - else { + } else { this.orbitControls.enabled = false; this.cameraViewControls.enabled = false; } @@ -404,7 +419,7 @@ class Editor { console.log(result); } - public async testTestTimelineEvents() { } + public async testTestTimelineEvents() {} public async loadScene(scene_media_token: string) { this.dispatchAppUiState({ @@ -436,6 +451,7 @@ class Editor { this.animation_engine, this.audio_engine, this.lipsync_engine, + this.emotion_engine, ); await proxyTimeline.loadFromJson(scene_json["timeline"]); @@ -473,6 +489,7 @@ class Editor { this.activeScene, ); const scene_json = await proxyScene.saveToScene(); + console.log("scene_json", scene_json); const proxyTimeline = new StoryTellerProxyTimeline( this.version, @@ -481,6 +498,7 @@ class Editor { this.animation_engine, this.audio_engine, this.lipsync_engine, + this.emotion_engine, ); const timeline_json = await proxyTimeline.saveToJson(); @@ -678,7 +696,9 @@ class Editor { deleteObject(uuid: string) { const obj = this.activeScene.get_object_by_uuid(uuid); - if (obj?.name === "::CAM::") { return; } + if (obj?.name === "::CAM::") { + return; + } if (obj) { this.activeScene.scene.remove(obj); } @@ -753,7 +773,7 @@ class Editor { this.render_timer += this.clock.getDelta(); this.frames += 1; this.playback_location++; - const imgData = this.rawRenderer.domElement.toDataURL('image/png', 1.0); // Medium quality png for speed & size. + const imgData = this.rawRenderer.domElement.toDataURL("image/png", 1.0); // Medium quality png for speed & size. this.frame_buffer.push(imgData); this.render_timer += this.clock.getDelta(); if (this.timeline.is_playing == false) { @@ -765,10 +785,19 @@ class Editor { } getselectedSum() { - if (this.selected === undefined) { return 0; } - let posCombo = this.selected.position.x + this.selected.position.y + this.selected.position.z; - let rotCombo = this.selected.rotation.x + this.selected.rotation.y + this.selected.rotation.z; - let sclCombo = this.selected.scale.x + this.selected.scale.y + this.selected.scale.z; + if (this.selected === undefined) { + return 0; + } + let posCombo = + this.selected.position.x + + this.selected.position.y + + this.selected.position.z; + let rotCombo = + this.selected.rotation.x + + this.selected.rotation.y + + this.selected.rotation.z; + let sclCombo = + this.selected.scale.x + this.selected.scale.y + this.selected.scale.z; return posCombo + rotCombo + sclCombo; } @@ -828,8 +857,10 @@ class Editor { if (changeView) { this.switchCameraView(); } - } - else if (this.last_scrub === this.timeline.scrubber_frame_position && this.getselectedSum() !== this.last_selected_sum) { + } else if ( + this.last_scrub === this.timeline.scrubber_frame_position && + this.getselectedSum() !== this.last_selected_sum + ) { this.updateSelectedUI(); } this.last_selected_sum = this.getselectedSum(); @@ -897,10 +928,10 @@ class Editor { audioSegment, "-filter_complex", "[1:a]adelay=" + - startTime * 1000 + - "|" + - startTime * 1000 + - "[a1];[0:a][a1]amix=inputs=2[a]", + startTime * 1000 + + "|" + + startTime * 1000 + + "[a1];[0:a][a1]amix=inputs=2[a]", "-map", "[a]", `${itteration}final_tmp.wav`, @@ -1172,9 +1203,7 @@ class Editor { element.visible = true; }); } - } - else { - + } else { this.timeline.is_playing = true; this.timeline.scrubber_frame_position = 0; if (!this.camera_person_mode) { @@ -1230,9 +1259,11 @@ class Editor { // Automaticly resize scene. onWindowResize() { - // Calculate the maximum possible dimensions while maintaining the aspect ratio - const width = window.innerWidth; // / aspect_adjust - const height = window.innerHeight; // / aspectRatio + const container = document.getElementById("video-scene-container"); + if (!container) return; + + const width = container.clientWidth; + const height = container.clientHeight; if (this.camera == undefined || this.renderer == undefined) { return; @@ -1256,6 +1287,23 @@ class Editor { this.render_camera.updateProjectionMatrix(); } + setupResizeObserver() { + const container = document.getElementById("video-scene-container"); + if (!container) return; + + const resizeObserver = new ResizeObserver((entries) => { + for (let entry of entries) { + const { width, height } = entry.contentRect; + this.camera.aspect = width / height; + this.camera.updateProjectionMatrix(); + this.renderer?.setSize(width, height); + this.renderer?.setPixelRatio(window.devicePixelRatio); + } + }); + + resizeObserver.observe(container); + } + onMouseDown(event: any) { if (event.button === 1 && this.camera_person_mode) { this.lockControls?.lock(); @@ -1266,6 +1314,15 @@ class Editor { if (event.button === 1) { this.lockControls?.unlock(); } + + if (event.button !== 0) { + let camera_pos = new THREE.Vector3( + parseFloat(this.camera.position.x.toFixed(2)), + parseFloat(this.camera.position.y.toFixed(2)), + parseFloat(this.camera.position.z.toFixed(2)), + ); + this.camera_last_pos.copy(camera_pos); + } } // Sets new mouse location usually used in raycasts. @@ -1281,15 +1338,27 @@ class Editor { // When the mouse clicks the screen. onMouseClick() { + let camera_pos = new THREE.Vector3( + parseFloat(this.camera.position.x.toFixed(2)), + parseFloat(this.camera.position.y.toFixed(2)), + parseFloat(this.camera.position.z.toFixed(2)), + ); + if (this.camera_last_pos.equals(new THREE.Vector3(0, 0, 0))) { + this.camera_last_pos.copy(camera_pos); + } + if ( this.raycaster == undefined || this.mouse == undefined || this.control == undefined || this.outlinePass == undefined || - this.camera_person_mode + this.camera_person_mode || + !this.camera_last_pos.equals(camera_pos) ) { + this.camera_last_pos.copy(camera_pos); return; } + this.camera_last_pos.copy(camera_pos); this.raycaster.setFromCamera(this.mouse, this.camera); const interactable: any[] = []; @@ -1333,13 +1402,11 @@ class Editor { }); this.updateSelectedUI(); } - } else if (this.transform_interaction == false) { + } else { this.removeTransformControls(); this.dispatchAppUiState({ type: APPUI_ACTION_TYPES.HIDE_CONTROLPANELS_SCENEOBJECT, }); - } else { - this.transform_interaction = false; } } } diff --git a/app/pages/PageEnigma/js/emotion_engine.ts b/app/pages/PageEnigma/js/emotion_engine.ts new file mode 100644 index 0000000000..583383de0c --- /dev/null +++ b/app/pages/PageEnigma/js/emotion_engine.ts @@ -0,0 +1,19 @@ +import { EmotionClip } from "../datastructures/clips/emotion_clip"; + +class EmotionEngine { + clips: { [key: string]: EmotionClip } = {}; + version: number; + + constructor(version: number) { + this.clips = {}; + this.version = version; + } + + // loads clips into the engine to cache + loadClip(object_uuid: string, csv_media_id: string) { + if(this.clips[object_uuid+csv_media_id] != null) { return; } + this.clips[object_uuid+csv_media_id] = new EmotionClip(this.version, csv_media_id); + } +} + +export default EmotionEngine; diff --git a/app/pages/PageEnigma/js/lipsync.js b/app/pages/PageEnigma/js/lipsync.js index a70ab40cf5..7a39ba0e50 100644 --- a/app/pages/PageEnigma/js/lipsync.js +++ b/app/pages/PageEnigma/js/lipsync.js @@ -21,6 +21,8 @@ export class LipSync { constructor(face) { this.face = face; + this.buffer = null; + this.last_frame = 0; // WE NEED THIS TO be run at RUNTIME // we have to modify this for our @@ -70,24 +72,59 @@ export class LipSync { if (this.mediaStreamSource) this.mediaStreamSource.stop(); - this.mediaStreamSource = this.audioContext.createBufferSource(); - this.mediaStreamSource.buffer = buffer; - this.meter = LipSync.createAudioMeter(this.audioContext); - this.mediaStreamSource.connect(this.meter); - this.mediaStreamSource.connect(this.audioContext.destination); - this.mediaStreamSource.start(); - // connect the output of mediaStreamSource to the input of userSpeechAnalyzer - this.mediaStreamSource.connect(this.userSpeechAnalyzer); + this.buffer = buffer; + + //this.mediaStreamSource = this.audioContext.createBufferSource(); + //this.mediaStreamSource.buffer = buffer; + //this.meter = LipSync.createAudioMeter(this.audioContext); + //this.mediaStreamSource.connect(this.meter); + //this.mediaStreamSource.connect(this.audioContext.destination); + ////this.mediaStreamSource.start(); + //// connect the output of mediaStreamSource to the input of userSpeechAnalyzer + //this.mediaStreamSource.connect(this.userSpeechAnalyzer); } destroy() { this.meter?.shutdown(); this.meter = null; this.mediaStreamSource?.disconnect(); - return this.audioContext?.close().catch(() => {}) || Promise.resolve(); + return this.audioContext?.close().catch(() => { }) || Promise.resolve(); } - update() { + update(frame, offset, rendering) { + + const frameBuffer = 12; + + let pos = frame - offset; + + let doPlay = true; + if (Math.abs(this.last_frame - frame) < frameBuffer - 1 && this.last_frame !== 0) { doPlay = false } + + if (pos <= 1 && !rendering) { + doPlay = true; + } + else if(!rendering){ + doPlay = false; + } + + if (doPlay) { + const startTime = pos / 60.0; + const endTime = frameBuffer / 60.0; + this.mediaStreamSource = this.audioContext.createBufferSource(); + this.mediaStreamSource.buffer = this.buffer; + this.meter = LipSync.createAudioMeter(this.audioContext); + this.mediaStreamSource.connect(this.meter); + this.mediaStreamSource.connect(this.audioContext.destination); + if (rendering) { + this.mediaStreamSource.start(0, startTime, endTime); + } + else { + this.mediaStreamSource.start(); + } + this.mediaStreamSource.connect(this.userSpeechAnalyzer); + this.last_frame = frame; + } + if (this.meter) { const { volume } = this.meter; let { ah, oh, ee } = this.update2(); @@ -156,8 +193,8 @@ export class LipSync { Math.max(EnergyBinFem[1], EnergyBinMasc[1]) > 0.2 ? 1 - 2 * Math.max(EnergyBinMasc[2], EnergyBinFem[2]) : (1 - 2 * Math.max(EnergyBinMasc[2], EnergyBinFem[2])) * - 5 * - Math.max(EnergyBinMasc[1], EnergyBinFem[1]); + 5 * + Math.max(EnergyBinMasc[1], EnergyBinFem[1]); const ah = 3 * Math.max(EnergyBinMasc[3], EnergyBinFem[3]); const ee = diff --git a/app/pages/PageEnigma/js/scene.ts b/app/pages/PageEnigma/js/scene.ts index bfc95e198c..043daf5e8c 100644 --- a/app/pages/PageEnigma/js/scene.ts +++ b/app/pages/PageEnigma/js/scene.ts @@ -51,6 +51,10 @@ class Scene { //obj.type = "Object3D"; obj.name = name; obj.position.copy(pos); + obj.userData["color"] = "#FFFFFF"; + obj.userData["metalness"] = 0.0; + obj.userData["shininess"] = 0.5; + obj.userData["specular"] = 0.0; this.scene.add(obj); return obj.uuid; } @@ -138,21 +142,19 @@ class Scene { if (c instanceof THREE.Mesh) { c.material.metalness = 0.0; c.material.specular = 0.5; + c.material.shininess = 0.0; c.castShadow = true; c.receiveShadow = true; c.frustumCulled = false; c.material.transparent = false; } }); - //console.log(child); - //if (child.type == "Group") { - // if (auto_add) { child.children.forEach(element => { - // this.scene.add(element); - // }); } - // resolve(child); - //} child.frustumCulled = false; child.userData["media_id"] = media_id; + child.userData["color"] = "#FFFFFF"; + child.userData["metalness"] = 0.0; + child.userData["shininess"] = 0.5; + child.userData["specular"] = 0.5; if (auto_add) { this.scene.add(child); } resolve(child); }); @@ -179,6 +181,7 @@ class Scene { resolve(child.children[0]); return; } + child.userData["color"] = "#FFFFFF"; if (auto_add) { this.scene.add(child); } resolve(child); }); diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index a27ef252d3..28ce3ae736 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -16,6 +16,7 @@ import { fromEngineActions } from "../Queue/fromEngineActions"; import { ClipGroup, ClipType } from "~/pages/PageEnigma/models/track"; import { MediaItem } from "~/pages/PageEnigma/models"; import Editor from "~/pages/PageEnigma/js/editor"; +import EmotionEngine from "./emotion_engine"; // Every object uuid / entity has a track. export class TimelineDataState { @@ -47,6 +48,10 @@ export class TimeLine { animation_engine: AnimationEngine; // lip sync engine lipSync_engine: LipSyncEngine; + // emotion engine + emotion_engine: EmotionEngine; + + // characters characters: { [key: string]: ClipGroup }; scene: Scene; @@ -62,6 +67,7 @@ export class TimeLine { transform_engine: TransformEngine, lipsync_engine: LipSyncEngine, animation_engine: AnimationEngine, + emotion_engine: EmotionEngine, scene: Scene, camera: THREE.Camera, mouse: THREE.Vector2 | undefined @@ -82,6 +88,7 @@ export class TimeLine { this.transform_engine = transform_engine; this.lipSync_engine = lipsync_engine; this.animation_engine = animation_engine; + this.emotion_engine = emotion_engine; this.scene = scene; @@ -190,6 +197,23 @@ export class TimeLine { 0, ), ); + + //this.addPlayableClip( + // new ClipUI( + // data.data["version"], + // ClipType.EMOTION, + // ClipGroup.CHARACTER, + // "Test", + // "m_c0g50khzpg99rq8chjn8zgvxcwebc7", + // obj.uuid, + // obj.uuid, + // name, + // 0, + // 200, + // 0 + // ) + //) + //this.emotion_engine.loadClip(obj.uuid, "m_c0g50khzpg99rq8chjn8zgvxcwebc7") } public getPos() { @@ -301,6 +325,9 @@ export class TimeLine { case "transform": this.transform_engine.loadObject(object_uuid, data["data"]["length"]); break; + case "emotion": + this.emotion_engine.loadClip(object_uuid, media_id); + break; case "audio": if (group === "character") { this.lipSync_engine.load_object(object_uuid, media_id); @@ -364,9 +391,9 @@ export class TimeLine { keyframe_offset, ); this.transform_engine.clips[object_uuid].setTransform( - keyframe_uuid, - keyframe_pos, - keyframe_rot, + keyframe_uuid, + keyframe_pos, + keyframe_rot, keyframe_scl, ); } @@ -497,7 +524,13 @@ export class TimeLine { this.lipSync_engine.clips[ element.object_uuid + element.media_id ].reset(); - } + } else if (element.type === ClipType.EMOTION) { + const object = this.scene.get_object_by_uuid(element.object_uuid); + if(object) + this.emotion_engine.clips[ + element.object_uuid + element.media_id + ].reset(object); + } } } @@ -553,7 +586,7 @@ export class TimeLine { } else if ( element.type === ClipType.AUDIO && element.group !== ClipGroup.CHARACTER && - this.is_playing + this.is_playing && !isRendering ) { if (this.scrubber_frame_position + 1 >= element.length) { this.audio_engine.stopClip(element.media_id); @@ -576,7 +609,7 @@ export class TimeLine { ].play(object); this.lipSync_engine.clips[ element.object_uuid + element.media_id - ].step(); + ].step(this.scrubber_frame_position, element.offset, isRendering); } } else if (element.type === ClipType.ANIMATION) { if (object) { @@ -588,6 +621,12 @@ export class TimeLine { this.scrubber_frame_position / fps, // Double FPS for best result. ); } + } else if (element.type === ClipType.EMOTION) { + if (object) { + await this.emotion_engine.clips[ + object.uuid + element.media_id + ].step(this.scrubber_frame_position-element.offset, object); + } } //this.timelineItems = this.timelineItems.filter(item => item !== element) } diff --git a/app/pages/PageEnigma/models/track.ts b/app/pages/PageEnigma/models/track.ts index aede824c56..e74842fd02 100644 --- a/app/pages/PageEnigma/models/track.ts +++ b/app/pages/PageEnigma/models/track.ts @@ -17,6 +17,7 @@ export enum ClipType { TRANSFORM = "transform", AUDIO = "audio", ANIMATION = "animation", + EMOTION = "emotion", FAKE = "fake", } diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts index d61734c79a..4f9f73ec6d 100644 --- a/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts @@ -21,6 +21,10 @@ export interface ObjectJSON { object_uuid: string; object_user_data_name: string; media_file_token: string; + color: string; + metalness: number, + shininess: number, + specular: number, } export class StoryTellerProxy3DObject { @@ -34,6 +38,10 @@ export class StoryTellerProxy3DObject { object_name: string; object_user_data_name: string; // changable name media_file_token: string; + color: string; + metalness: number; + shininess: number; + specular: number; constructor( version: number, @@ -49,6 +57,19 @@ export class StoryTellerProxy3DObject { this.object_name = ""; this.object_user_data_name = ""; this.object_uuid = ""; + + this.color = ""; + + this.metalness = 0.0; + this.shininess = 0.0; + this.specular = 0.5; + } + + getColorAsHexString(object: THREE.Object3D): string { + if (object instanceof THREE.Mesh && object.material instanceof THREE.MeshBasicMaterial) { + return '#' + object.material.color.getHexString(); + } + return "#FFFFFF" } public async initialize(object: THREE.Object3D) { @@ -59,6 +80,10 @@ export class StoryTellerProxy3DObject { this.object_name = object.name; this.object_user_data_name = object.userData.name; this.object_uuid = object.uuid; + this.color = object.userData["color"]; + this.metalness = object.userData["metalness"]; + this.shininess = object.userData["shininess"]; + this.specular = object.userData["specular"]; } public async toJSON(): Promise { @@ -83,6 +108,10 @@ export class StoryTellerProxy3DObject { object_uuid: this.object_uuid, object_user_data_name: this.object_user_data_name, media_file_token: this.media_file_token, + color: this.color, + metalness: this.metalness, + shininess: this.shininess, + specular: this.specular, }; return json; } diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts index c5fdddc1d9..7383c70d8a 100644 --- a/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts @@ -38,6 +38,10 @@ export class StoryTellerProxyScene { proxyObject3D.object_user_data_name = child.userData.name; proxyObject3D.object_name = child.name; proxyObject3D.object_uuid = child.uuid; + proxyObject3D.color = child.userData["color"]; + proxyObject3D.metalness = child.userData["metalness"]; + proxyObject3D.shininess = child.userData["shininess"]; + proxyObject3D.specular = child.userData["specular"]; const json_data = await proxyObject3D.toJSON(); results.push(json_data); } diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts index 8c0c08051f..a3e69a6259 100644 --- a/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts @@ -1,10 +1,12 @@ import { AnimationClip } from "../datastructures/clips/animation_clip"; import { AudioClip } from "../datastructures/clips/audio_clip"; import { ClipUI } from "../datastructures/clips/clip_ui"; +import { EmotionClip } from "../datastructures/clips/emotion_clip"; import { LipSyncClip } from "../datastructures/clips/lipsync_clip"; import { TransformClip } from "../datastructures/clips/transform_clip"; import AnimationEngine from "../js/animation_engine"; import AudioEngine from "../js/audio_engine"; +import EmotionEngine from "../js/emotion_engine"; import LipSyncEngine from "../js/lip_sync_engine"; import { TimeLine } from "../js/timeline"; import TransformEngine from "../js/transform_engine"; @@ -16,13 +18,15 @@ export class StoryTellerProxyTimeline { animation_engine: AnimationEngine audio_engine: AudioEngine lipsync_engine: LipSyncEngine + emotion_engine: EmotionEngine - constructor(version: number, timeline: TimeLine, transform_engine: TransformEngine, animation_engine: AnimationEngine, audio_engine: AudioEngine, lipsync_engine: LipSyncEngine) { + constructor(version: number, timeline: TimeLine, transform_engine: TransformEngine, animation_engine: AnimationEngine, audio_engine: AudioEngine, lipsync_engine: LipSyncEngine, emotion_engine: EmotionEngine) { this.timeline = timeline; this.transform_engine = transform_engine; this.animation_engine = animation_engine; this.audio_engine = audio_engine; this.lipsync_engine = lipsync_engine; + this.emotion_engine = emotion_engine; } private async getItems(items: any[]): Promise { @@ -52,6 +56,7 @@ export class StoryTellerProxyTimeline { animation: await this.getItemsDict(this.animation_engine.clips), audio: await this.getItemsDict(this.audio_engine.clips), lipsync: await this.getItemsDict(this.lipsync_engine.clips), + emotion: await this.getItemsDict(this.emotion_engine.clips), } console.log(timeline_json) @@ -122,12 +127,24 @@ export class StoryTellerProxyTimeline { return timeline_items_data; } + private async loadEmotionClips(items: { [key: string]: any }): Promise<{ [key: string]: any }> { + let timeline_items_data: { [key: string]: any } = {}; + for (const key in items) { + if (items.hasOwnProperty(key)) { + const element = items[key]; + timeline_items_data[key] = new EmotionClip(element.version, element.media_id); + } + } + return timeline_items_data; + } + public async loadFromJson(timeline: any) { this.timeline.timeline_items = await this.loadTimelineClips(timeline['timeline']); this.transform_engine.clips = await this.loadTransformClips(timeline['transform']); this.audio_engine.clips = await this.loadAudioClips(timeline['audio']); this.lipsync_engine.clips = await this.loadLipsyncClips(timeline['lipsync']); this.animation_engine.clips = await this.loadAnimationClips(timeline['animation']); + this.emotion_engine.clips = await this.loadEmotionClips(timeline['emotion']); console.log(this.timeline.timeline_items) diff --git a/app/pages/PageEnigma/store/media.ts b/app/pages/PageEnigma/store/media.ts index 2e6ee3ad5e..7c04020b3f 100644 --- a/app/pages/PageEnigma/store/media.ts +++ b/app/pages/PageEnigma/store/media.ts @@ -525,6 +525,20 @@ export const characterItems = signal([ type: AssetType.CHARACTER, name: "Waifu H", thumbnail: "/resources/characters/img14.png", + }, + { + version: 1, + media_id: "m_qfqw9e5kz0b7kspbwcc3c9y7pym97q", + type: AssetType.CHARACTER, + name: "Skibidi Toilet", + thumbnail: "/resources/characters/img15.png", + }, + { + version: 1, + media_id: "m_ecfxa94v4ftvh58hv7se3qqy8a385n", + type: AssetType.CHARACTER, + name: "Pop", + thumbnail: "/resources/placeholders/placeholder.png", } ]); @@ -536,6 +550,27 @@ export const objectItems = signal([ name: "Pikachu Statue", thumbnail: "/resources/placeholders/placeholder.png", }, + { + version: 1, + media_id: "m_4d2s2q1xy9m1xncppr3gf9ne6bm2m0", + type: AssetType.OBJECT, + name: "Moon", + thumbnail: "/resources/objects/moon.png", + }, + { + version: 1, + media_id: "m_t0s6gbvfp78rvc0sd6129za41crr9d", + type: AssetType.OBJECT, + name: "Crown", + thumbnail: "/resources/objects/crown.png", + }, + { + version: 1, + media_id: "m_zp0zykww0k7ka2zagg3q3pb9t123k4", + type: AssetType.OBJECT, + name: "Sakura Tree", + thumbnail: "/resources/objects/sakura.png", + }, ]); // In the future these will have shape id's diff --git a/build-ci.sh b/build-ci.sh new file mode 100755 index 0000000000..fd39fb8cc1 --- /dev/null +++ b/build-ci.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# NB: This file is executed by Netlify to build the website + +# Verbose printing, exit on failure. +set -euxo pipefail + +# Add the GIT SHA to the build +# This must be done before everything else, or it will get cached with the build. +# (This might be making the builds less performant?) +# COMMIT_REF is defined by Netlify to be the commit SHA +# We want a short 8 character reference. +echo "Labelling build with short SHA..." +SHORT_SHA=$(echo "${COMMIT_REF}" | cut -c1-8) + +find . -type f -exec sed -i "s/CURRENT_STORYTELLER_GIT_VERSION/${SHORT_SHA}/g" {} + + +# TODO: Run tests when we add them. + +# Run build. +npm run build + diff --git a/netlify.toml b/netlify.toml index 4f3f001c6d..52703a0246 100644 --- a/netlify.toml +++ b/netlify.toml @@ -1,5 +1,5 @@ [build] -command = "npm run build" +command = "./build-ci.sh" publish = "build/client" [dev] diff --git a/public/resources/characters/img15.png b/public/resources/characters/img15.png new file mode 100644 index 0000000000..039b9666c9 Binary files /dev/null and b/public/resources/characters/img15.png differ diff --git a/public/resources/objects/crown.png b/public/resources/objects/crown.png new file mode 100644 index 0000000000..1af3d75465 Binary files /dev/null and b/public/resources/objects/crown.png differ diff --git a/public/resources/objects/moon.png b/public/resources/objects/moon.png new file mode 100644 index 0000000000..2a6f3607f9 Binary files /dev/null and b/public/resources/objects/moon.png differ diff --git a/public/resources/objects/sakura.png b/public/resources/objects/sakura.png new file mode 100644 index 0000000000..f7ac5fe8da Binary files /dev/null and b/public/resources/objects/sakura.png differ