Merge branch 'test' of github.com:storytold/GottaGoFast into test

This commit is contained in:
wil committed 2024-03-23 16:00:20 -04:00
commit ea91431025
5 files changed
+301 -183

No files matched your search

@@ -1,21 +1,62 @@
interface AudioData {
audioContext: AudioContext;
audioBuffer: AudioBuffer;
}
class AudioData implements AudioData{
audioContext: AudioContext;
audioBuffer: AudioBuffer;
source: AudioBufferSourceNode | undefined
constructor (audioContext: AudioContext, audioBuffer: AudioBuffer) {
this.audioContext = audioContext;
this.audioBuffer = audioBuffer;
}
}
export interface AudioTrackClip {
version: number;
media_id: number;
media_id: string;
type: "audio";
volume: number;
}
export class AudioTrackClipClass implements AudioTrackClip {
export class AudioTrackClip implements AudioTrackClip {
version: number;
media_id: number;
type: "audio";
media_id: string;
type: "audio" = "audio";
volume: number;
audio_data: AudioData | undefined;
constructor(version: number, media_id: number, volume: number) {
constructor(version: number, media_id: string, volume: number) {
this.version = version;
this.media_id = media_id;
this.type = "audio";
this.volume = volume;
this.download_audio().then(data => {
this.audio_data = data;
});
}
async get_media_url() {
//This is for prod when we have the proper info on the url.
let api_base_url = "https://api.fakeyou.com";
let url = `${api_base_url}/v1/media_files/file/${this.media_id}`
let responce = await fetch(url);
let json = await JSON.parse(await responce.text());
let bucketPath = json["media_file"]["public_bucket_path"];
let media_base_url = "https://storage.googleapis.com/vocodes-public"
let media_url = `${media_base_url}${bucketPath}`
return media_url;
}
async download_audio() {
let url = await this.get_media_url();
const audioContext = new AudioContext();
const response = await fetch(url);
const arrayBuffer = await response.arrayBuffer();
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
return new AudioData(audioContext, audioBuffer);
}
toJSON(): string {
+33 -35
View File
@@ -1,43 +1,41 @@
import * as THREE from "three";
import { AudioTrackClip } from "../datastructures/clips/audio_track_clip";
class AudioEngine {
constructor() {
this.listener = new THREE.AudioListener();
this.clips = {};
}
clips: { [key: string]: AudioTrackClip } = {};
audio_sources: { [key: string]: AudioBufferSourceNode } = {};
version: number;
addCamera(camera) {
// Assuming you have a camera and you want to add the listener to it
camera.add(this.listener);
}
constructor() {
this.clips = {};
this.audio_sources = {};
this.version = 1.0;
}
loadClip(id, audioURL) {
if (this.clips[id]) {
console.warn(`AudioManager: AudioClip already exists with id "${id}".`);
return;
}
loadClip(audio_media_id: string) {
this.clips[audio_media_id] = new AudioTrackClip(this.version, audio_media_id, 1.0);
}
const clip = new AudioClip(this.listener, audioURL);
this.clips[id] = clip;
}
playClip(audio_media_id: string) {
const clip = this.clips[audio_media_id];
if (clip.audio_data?.audioContext) {
clip.audio_data.source = clip.audio_data.audioContext.createBufferSource();
clip.audio_data.source.buffer = clip.audio_data.audioBuffer;
clip.audio_data.source.connect(clip.audio_data.audioContext.destination);
clip.audio_data.source.start();
} else {
console.warn(`AudioManager: AudioClip buffer with id "${audio_media_id}" not found.`);
}
}
playClip(id) {
const clip = this.clips[id];
if (clip) {
clip.play();
} else {
console.warn(`AudioManager: AudioClip with id "${id}" not found.`);
}
}
stopClip(id) {
const clip = this.clips[id];
if (clip) {
clip.stop();
} else {
console.warn(`AudioManager: AudioClip with id "${id}" not found.`);
}
}
stopClip(audio_media_id: string) {
const clip = this.clips[audio_media_id];
if (clip.audio_data?.source) {
clip.audio_data.source.stop();
} else {
console.warn(`AudioManager: AudioClip with id "${audio_media_id}" not found.`);
}
}
}
export default AudioManager;
export default AudioEngine;
+12 -20
View File
@@ -14,6 +14,8 @@ import { UnrealBloomPass } from 'three/addons/postprocessing/UnrealBloomPass.js'
import { BokehPass } from 'three/addons/postprocessing/BokehPass.js';
import { createFFmpeg, fetchFile } from '@ffmpeg/ffmpeg';
import AudioEngine from './audio_engine.ts';
if (typeof window !== 'undefined') {
import('ccapture.js').then(module => {
const CCapture = module.CCapture;
@@ -28,7 +30,6 @@ class Editor {
// Default params.
constructor() {
// For making sure the editor only gets created onece.
this.can_initailize = false;
let one_element = document.getElementById("created-one-element");
@@ -36,10 +37,8 @@ class Editor {
let newElement = document.createElement("div");
newElement.id = "created-one-element";
document.body.appendChild(newElement);
// Version and name.
this.version = "v0.1";
// Clock, scene and camera essentials.
this.activeScene = null;
this.camera = null;
@@ -55,7 +54,6 @@ class Editor {
this.bloomPass;
this.smaaPass;
this.bokehPass;
// Transform control and selection.
this.control = null;
this.raycaster = null;
@@ -64,41 +62,35 @@ class Editor {
this.last_selected = null;
this.transform_interaction;
this.rendering = false;
// API.
this.api_manager = new MediaUploadManager();
// Debug & Movement.
this.stats = null;
this.orbit = null;
this.locked = false;
// Recording params.
this.capturer = null;
this.frame_buffer = [];
this.render_timer = 0;
this.fps_number = 60;
this.cap_fps = 60;
// Timeline settings.
this.playback = false;
this.playback_location = 0;
this.max_length = 10;
this.timeline = null;
// Save & Load.
this.save_manager = new SaveManager(this.version);
// Audio Engine Test.
console.log("Created Editor.");
this.audio_engine = new AudioEngine();
}
// Initializes the main scene and ThreeJS essentials.
initialize() {
//if (this.can_initailize == false) { return; }
// Gets the canvas.
this.canvReference = document.getElementById("video-scene");
// Base width and height.
let width = this.canvReference.width;
let height = this.canvReference.height;
@@ -126,14 +118,13 @@ class Editor {
// OnClick and MouseMove events.
window.addEventListener('mousemove', this.onMouseMove.bind(this), false);
window.addEventListener('click', this.onMouseClick.bind(this), false);
// Base control and debug stuff remove debug in prod.
this._initialize_control();
// Resets canvas size.
this.onWindowResize();
// Creates the main update loop.
this.renderer.setAnimationLoop(this.update_loop.bind(this));
this.audio_engine.loadClip("m_f7jnwt3d1ddchatdk5vaqt0n4mb1hg");
}
// Configure post processing.
@@ -209,12 +200,13 @@ class Editor {
}
save() {
console.log(this.control);
if(this.selected != null){ this.control.detach(this.selected); }
this.activeScene.scene.remove(this.control);
this.activeScene.scene.remove(this.activeScene.gridHelper);
this.save_manager.save(this.activeScene.scene, this._save_to_cloud.bind(this), this.audio_manager, this.timeline, this.activeScene.animations);
this.activeScene._createGrid();
//console.log(this.control);
//if(this.selected != null){ this.control.detach(this.selected); }
//this.activeScene.scene.remove(this.control);
//this.activeScene.scene.remove(this.activeScene.gridHelper);
//this.save_manager.save(this.activeScene.scene, this._save_to_cloud.bind(this), this.audio_manager, this.timeline, this.activeScene.animations);
//this.activeScene._createGrid();
this.audio_engine.playClip("m_f7jnwt3d1ddchatdk5vaqt0n4mb1hg");
}
_save_to_cloud(blob) {
+187 -100
View File
@@ -1,106 +1,193 @@
// Write unsubcribe and run the remainder of the clip tech.
import { AnyJson } from "three/examples/jsm/nodes/core/constants.js"
import { ClipOffset } from "../datastructures/clips/clip_offset"
// export class Clip {
// version: number;
// clip_id: number;
export class TimelineCurrentReactState {
isEditable:boolean
constructor() {
this.isEditable = true // can add clips to it
}
}
export class TimeLine {
timelineItems: ClipOffset[]
runningClips: ClipOffset[]
//timerID: NodeJS.Timeout | null
timeLineLimit: number
scrubberPosition: number
isPlaying: boolean
timelineState: TimelineCurrentReactState
// media_token_id: number;
// type: 'animation' | 'audio' | 'position_rotation';
// time_start_offset: number;
// length_override: number | null;
constructor() {
this.timelineItems = []
// this.timerID = null
this.timeLineLimit = 1000 * 10 // 10 seconds
this.runningClips = []
this.isPlaying = false
this.scrubberPosition = 0 // in ms into the tl
this.timelineState = new TimelineCurrentReactState()
}
// constructor(
// version: number,
// clip_id: number,
// media_token_id: number,
// type: 'animation' | 'audio' | 'position_rotation',
// time_start_offset: number,
// length_override: number | null
// ) {
// this.version = version;
// this.clip_id = clip_id;
// this.media_token_id = media_token_id;
// this.type = type;
// this.time_start_offset = time_start_offset;
// this.length_override = length_override;
// }
// }
// export class Clip2 {
// name: string
// startOffset: number
// timerID: NodeJS.Timeout | null
// lengthOfClip: number
// scrubberEndingPosition: number
// constructor(name: string, startOffset: number, lengthOfClip: number) {
// this.startOffset = startOffset
// this.lengthOfClip = lengthOfClip
// this.name = name
// this.timerID = null
// this.scrubberEndingPosition = 100
// }
// async play(): Promise<void> {
// console.log(`Playing ${this.name}`)
// this.timerID = setInterval(async () => {
// console.log(`Running async ${this.name}`)
// this.stop()
// }, this.lengthOfClip)
// }
// async stop(): Promise<void> {
// clearInterval(this.timerID!)
// console.log(`Stopping ${this.name}`)
// }
// }
// export class API2 {
// timelineItems: Clip[]
// timerID: NodeJS.Timeout | null
// timeLineLimit: number
// runningClips: Clip[]
// constructor() {
// this.timelineItems = []
// this.timerID = null
// this.timeLineLimit = 1000 * 10
// this.runningClips = []
// }
// async addPlayableClip(clip: Clip): Promise<void> {
// this.timelineItems.push(clip)
// }
// async start(): Promise<void> {
// let start = 0
// const updateInterval = 100
// const timerID = setInterval(async () => {
// console.log(`Current Time:${start}`)
// // get last updated time when stopped
// for (const element of this.timelineItems) {
// if (element.startOffset >= start) {
// element.play()
// // remove the element
// this.timelineItems = this.timelineItems.filter(item => item !== element)
// }
// }
// if (start == this.timeLineLimit || this.timelineItems.length == 0) {
// this.stop()
// }
// start += updateInterval
// }, updateInterval)
// public streaming events into the timeline from
async setScrubberPosition(offset:number) {
this.scrubberPosition = offset // in ms
}
// this.timerID = timerID
// }
async addPlayableClip(clip: ClipOffset): Promise<void> {
this.timelineItems.push(clip)
}
// async stop(): Promise<void> {
// console.log(`Stopping Timeline`)
// clearInterval(this.timerID!)
// }
async deleteClip(clip_uuid: string): Promise<void> {
}
// this will update the state of the clips based off uuid
async modifyClip(clip_uuid: string, updates: AnyJson): Promise<void> {
}
// timeline controls this.
async scrub(offset_in_ms:number): Promise<void> {
// only stream through to the position and rotation keyframes
}
// should play from the clip that is closest to the to scrubber
async play(): Promise<void> {
console.log(`Starting Timeline`)
this.isPlaying = true
// const updateInterval = 100
// const timerID = setInterval(async () => {
// console.log(`Current Time:${start}`)
// // get last updated time when stopped
// for (const element of this.timelineItems) {
// if (element.start_offset >= start) {
// // run async
// // element.play()
// // remove the element
// this.timelineItems = this.timelineItems.filter(item => item !== element)
// }
// }
// if (start == this.timeLineLimit || this.timelineItems.length == 0) {
// this.stop()
// }
// start += updateInterval
// }, updateInterval)
// this.timerID = timerID
}
// called by the editor update loop on each frame
async update() {
if (this.isPlaying == false) return; // start and stop
for (const element of this.timelineItems) {
if (element.start_offset >= this.scrubberPosition) {
// run async
// element.play()
// remove the element from the list
this.timelineItems = this.timelineItems.filter(item => item !== element)
}
if (this.scrubberPosition == this.timeLineLimit) { // stops at where clips should // cannot throw clip
this.stop()
}
}
}
async stop(): Promise<void> {
this.isPlaying = false
console.log(`Stopping Timeline`)
}
}
// How much timeline precision we have
const percision = 100 // using a fake update loop mock
// Visual verification tests.
function CheckIfBasicAudioClipWorks() {
const api = new TimeLine()
api.addPlayableClip(new ClipOffset(1.0,'audio',1,0))
api.play()
}
function CheckIfBasicTransformClipWorks() {
const api = new TimeLine()
api.addPlayableClip(new ClipOffset(1.0,'transform',2,0))
api.play()
}
function CheckIfBasicClipWorks() {
const api = new TimeLine()
api.addPlayableClip(new ClipOffset(1.0,'animation',3,0))
api.play()
}
// function CheckIfBasicClipWorks3SecondsAfterTimelineStops() {
// const api = new TimeLine()
// api.addPlayableClip(new ClipOffset("clip1", 0, 10000))
// api.play()
// }
// function CheckIfTwoClipsAtTheSameTimeWorks() {
// const api = new TimeLine()
// api.addPlayableClip(new ClipOffset("clip1", 0, 1000))
// api.play()
// }
// function CheckIfTwoClipsOneAfterAnotherWorks() {
// const api = new TimeLine()
// api.addPlayableClip(new ClipOffset("clip1", 0, 1000))
// api.addPlayableClip(new ClipOffset("clip2", 0, 2000))
// api.play()
// }
// function CheckIfTimeLineStopBeforeClipPlays() {
// const api = new TimeLine()
// api.addPlayableClip(new ClipOffset("clip3",0,1000))
// api.play()
// setInterval(async ()=> {
// api.stop()
// },1100)
// console.log("Stopped")
// }
// function CheckIfTimeLineStartAfterClipPlays() {
// const api = new TimeLine()
// api.addPlayableClip(new ClipOffset("clip3",0,1000))
// api.play()
// setInterval(async ()=> {
// api.stop()
// },500)
// console.log("Stopped")
// }
// function CheckIfClipsPlayAllTogetherConcurrently() {
// const api = new TimeLine()
// api.addPlayableClip(new ClipOffset("clip1",0,1000))
// api.addPlayableClip(new ClipOffset("clip2",0,1000))
// api.addPlayableClip(new ClipOffset("clip3",0,1000))
// api.addPlayableClip(new ClipOffset("clip4",0,1000))
// api.play()
// }
// function CheckIfClipsPlayAllTogether() {
// const api = new TimeLine()
// api.addPlayableClip(new TrackClip("clip1",0,1000))
// api.addPlayableClip(new TrackClip("clip2",0,1000))
// api.addPlayableClip(new TrackClip("clip3",0,1000))
// api.addPlayableClip(new TrackClip("clip4",0,1000))
// api.play()
// }
CheckIfBasicClipWorks()
//CheckIfTwoClipsAtTheSameTimeWorks()
// CheckIfTwoClipsOneAfterAnotherWorks()
// CheckIfTimeLineStopBeforeClipPlays()
// CheckIfTimeLineStartAfterClipPlays()
// CheckIfClipsPlayAllTogetherConcurrently()
// CheckIfClipsPlayAllTogether()
+23 -23
View File
@@ -1,30 +1,30 @@
// Write unsubcribe and run the remainder of the clip tech.
class TrackClip {
name: string
startOffset: number // ms in the 0 100 | 10 second clip
timerID: NodeJS.Timeout | null
lengthOfClip: number
// class TrackClip {
// name: string
// startOffset: number // ms in the 0 100 | 10 second clip
// timerID: NodeJS.Timeout | null
// lengthOfClip: number
constructor(name: string, startOffset: number, lengthOfClip: number) {
this.startOffset = startOffset
this.lengthOfClip = lengthOfClip
this.name = name
this.timerID = null
}
// constructor(name: string, startOffset: number, lengthOfClip: number) {
// this.startOffset = startOffset
// this.lengthOfClip = lengthOfClip
// this.name = name
// this.timerID = null
// }
async play(): Promise<void> {
console.log(`Playing ${this.name}`)
this.timerID = setInterval(async () => {
console.log(`Running async ${this.name}`)
this.stop()
}, this.lengthOfClip)
}
// async play(): Promise<void> {
// console.log(`Playing ${this.name}`)
// this.timerID = setInterval(async () => {
// console.log(`Running async ${this.name}`)
// this.stop()
// }, this.lengthOfClip)
// }
async stop(): Promise<void> {
clearInterval(this.timerID!)
console.log(`Stopping ${this.name}`)
}
}
// async stop(): Promise<void> {
// clearInterval(this.timerID!)
// console.log(`Stopping ${this.name}`)
// }
// }
class TimeLine {
timelineItems: TrackClip[]