mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-17 07:04:36 +00:00
Merge pull request #94 from storytold/feature/render-audio-buf-fix
Feature/render audio buf fix
This commit is contained in:
4 files changed
+82
-67
No files matched your search
@@ -129,6 +129,13 @@ export const TrackProvider = ({ children }: Props) => {
|
||||
type: ClipType.AUDIO,
|
||||
length: 25,
|
||||
name: "Talk",
|
||||
},
|
||||
{
|
||||
version: 1,
|
||||
media_id: "m_w5nn3kjh1fbkmjrdac5b2qaba0pmyt",
|
||||
type: ClipType.AUDIO,
|
||||
length: 25,
|
||||
name: "NCS Song",
|
||||
}
|
||||
]);
|
||||
setScale(1);
|
||||
|
||||
@@ -310,7 +310,7 @@ export class APIManager {
|
||||
blob: any,
|
||||
fileName: string,
|
||||
): Promise<APIManagerResponseSuccess> {
|
||||
const url = `${this.baseUrl}/v1/media_uploads/upload`;
|
||||
const url = `${this.baseUrl}/v1/media_files/upload`;
|
||||
const uuid = uuidv4();
|
||||
|
||||
const formData = new FormData();
|
||||
|
||||
@@ -91,6 +91,7 @@ class Editor {
|
||||
cam_obj: THREE.Object3D | undefined;
|
||||
renderPass: RenderPass | undefined;
|
||||
generating_preview: boolean;
|
||||
frames: number;
|
||||
|
||||
camera_person_mode: boolean;
|
||||
current_scene_media_token: string | null;
|
||||
@@ -172,6 +173,7 @@ class Editor {
|
||||
this.playback_location = 0;
|
||||
this.max_length = 10;
|
||||
this.last_scrub = 0;
|
||||
this.frames = 0;
|
||||
// Audio Engine Test.
|
||||
|
||||
this.render_width = 1280;
|
||||
@@ -361,7 +363,7 @@ class Editor {
|
||||
console.log(result);
|
||||
}
|
||||
|
||||
public async testTestTimelineEvents() {}
|
||||
public async testTestTimelineEvents() { }
|
||||
|
||||
public async testStylizeRequest() {
|
||||
const result = await this.api_manager
|
||||
@@ -433,8 +435,8 @@ class Editor {
|
||||
}
|
||||
|
||||
// TO UPDATE selected objects in the scene might want to add to the scene ...
|
||||
async setSelectedObject(position:XYZ,rotation:XYZ,scale:XYZ) {
|
||||
if (this.selected !=undefined || this.selected !=null) {
|
||||
async setSelectedObject(position: XYZ, rotation: XYZ, scale: XYZ) {
|
||||
if (this.selected != undefined || this.selected != null) {
|
||||
//console.log(`triggering setSelectedObject`)
|
||||
this.selected.position.x = position.x
|
||||
this.selected.position.y = position.y
|
||||
@@ -449,7 +451,7 @@ class Editor {
|
||||
this.selected.scale.z = scale.z
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public async saveScene(name: string) {
|
||||
@@ -813,7 +815,7 @@ class Editor {
|
||||
}
|
||||
|
||||
// Basicly Unity 3D's update loop.
|
||||
updateLoop(time: number) {
|
||||
async updateLoop(time: number) {
|
||||
if (this.cam_obj == undefined) {
|
||||
this.cam_obj = this.activeScene.get_object_by_name("::CAM::");
|
||||
}
|
||||
@@ -831,11 +833,11 @@ class Editor {
|
||||
|
||||
if (this.cameraViewControls && this.camera_person_mode) {
|
||||
this.cameraViewControls.update(5 * delta_time);
|
||||
if(this.cam_obj){
|
||||
if (this.cam_obj) {
|
||||
if (this.last_scrub != this.timeline.scrubber_frame_position) {
|
||||
this.camera.position.copy(this.cam_obj.position);
|
||||
this.camera.rotation.copy(this.cam_obj.rotation);
|
||||
} else if(this.timeline.is_playing == false) {
|
||||
} else if (this.timeline.is_playing == false) {
|
||||
this.cam_obj.position.copy(this.camera.position);
|
||||
this.cam_obj.rotation.copy(this.camera.rotation);
|
||||
} else {
|
||||
@@ -846,25 +848,24 @@ class Editor {
|
||||
this.cam_obj.visible = false;
|
||||
}
|
||||
}
|
||||
else if (this.cam_obj){
|
||||
else if (this.cam_obj) {
|
||||
this.cam_obj.visible = true;
|
||||
}
|
||||
|
||||
if (this.render_camera && this.cam_obj) {
|
||||
this.render_camera.position.copy(this.cam_obj.position);
|
||||
this.render_camera.rotation.copy(this.cam_obj.rotation);
|
||||
this.cam_obj.scale.copy(new THREE.Vector3(1,1,1));
|
||||
this.cam_obj.scale.copy(new THREE.Vector3(1, 1, 1));
|
||||
}
|
||||
|
||||
if(this.timeline.is_playing){
|
||||
this.timeline.update();
|
||||
}
|
||||
else if (this.last_scrub == this.timeline.scrubber_frame_position){
|
||||
|
||||
if (this.timeline.is_playing) {
|
||||
await this.timeline.update(this.rendering);
|
||||
}
|
||||
else if (this.last_scrub == this.timeline.scrubber_frame_position) {
|
||||
this.updateSelectedUI();
|
||||
}
|
||||
|
||||
this.last_scrub = this.timeline.scrubber_frame_position;
|
||||
|
||||
this.renderScene();
|
||||
}
|
||||
|
||||
@@ -933,10 +934,10 @@ class Editor {
|
||||
audioSegment,
|
||||
"-filter_complex",
|
||||
"[1:a]adelay=" +
|
||||
startTime * 1000 +
|
||||
"|" +
|
||||
startTime * 1000 +
|
||||
"[a1];[0:a][a1]amix=inputs=2[a]",
|
||||
startTime * 1000 +
|
||||
"|" +
|
||||
startTime * 1000 +
|
||||
"[a1];[0:a][a1]amix=inputs=2[a]",
|
||||
"-map",
|
||||
"[a]",
|
||||
`${itteration}final_tmp.wav`,
|
||||
@@ -962,6 +963,8 @@ class Editor {
|
||||
}
|
||||
|
||||
async stopPlayback(compile_audio: boolean = true) {
|
||||
console.log(this.frames, this.frame_buffer.length);
|
||||
|
||||
this.renderMode();
|
||||
const ffmpeg = createFFmpeg({ log: true });
|
||||
await ffmpeg.load();
|
||||
@@ -975,7 +978,7 @@ class Editor {
|
||||
}
|
||||
await ffmpeg.run(
|
||||
"-framerate",
|
||||
"" + this.cap_fps,
|
||||
"" + this.cap_fps/2,
|
||||
"-i",
|
||||
"image%d.png",
|
||||
"-f",
|
||||
@@ -1020,22 +1023,24 @@ class Editor {
|
||||
console.log("data", data)
|
||||
|
||||
if (data == null) { return; }
|
||||
let upload_token = data['upload_token'];
|
||||
let upload_token = data['media_file_token'];
|
||||
console.log(upload_token);
|
||||
// Create a link to download the file stylize video using api ..
|
||||
//{"success":true,"upload_token":"mu_x9kr5cfafn512pjbygdszvbdpktrr"} payload
|
||||
|
||||
const result = await this.api_manager
|
||||
.stylizeVideo(
|
||||
upload_token,
|
||||
this.art_style,
|
||||
this.positive_prompt,
|
||||
this.negative_prompt,
|
||||
Visibility.Public,
|
||||
)
|
||||
.catch((error) => {
|
||||
console.log(error);
|
||||
});
|
||||
.stylizeVideo(
|
||||
upload_token,
|
||||
this.art_style,
|
||||
this.positive_prompt,
|
||||
this.negative_prompt,
|
||||
Visibility.Public,
|
||||
)
|
||||
.catch((error) => {
|
||||
console.log(error);
|
||||
});
|
||||
|
||||
// {"success":true,"inference_job_token":"jinf_j3nbqbd15wqxb0xcks13qh3f3bz"}
|
||||
|
||||
console.log(result);
|
||||
|
||||
@@ -1167,6 +1172,7 @@ class Editor {
|
||||
}
|
||||
|
||||
if (this.rendering && this.renderer && this.clock) {
|
||||
this.frames += 1;
|
||||
this.playback_location++;
|
||||
const imgData = this.renderer.domElement.toDataURL();
|
||||
this.frame_buffer.push(imgData);
|
||||
@@ -1186,41 +1192,41 @@ class Editor {
|
||||
if (this.selected == undefined) {
|
||||
return;
|
||||
}
|
||||
const pos = this.selected.position;
|
||||
const rot = this.selected.rotation;
|
||||
const scale = this.selected.scale;
|
||||
|
||||
// TODO this is a bug we need to only show when clicked on and use UPDATE when updating.
|
||||
this.dispatchAppUiState({
|
||||
type: APPUI_ACTION_TYPES.UPDATE_CONTROLPANELS_SCENEOBJECT,
|
||||
payload: {
|
||||
group:
|
||||
this.selected.name === "::CAM::"
|
||||
? ClipGroup.CAMERA
|
||||
: ClipGroup.OBJECT, // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips
|
||||
object_uuid: this.selected.uuid,
|
||||
object_name: this.selected.name,
|
||||
version: this.version,
|
||||
objectVectors: {
|
||||
position: {
|
||||
x: parseFloat(pos.x.toFixed(2)),
|
||||
y: parseFloat(pos.y.toFixed(2)),
|
||||
z: parseFloat(pos.z.toFixed(2)),
|
||||
},
|
||||
rotation: {
|
||||
x: parseFloat(rot.x.toFixed(2)),
|
||||
y: parseFloat(rot.y.toFixed(2)),
|
||||
z: parseFloat(rot.z.toFixed(2)),
|
||||
},
|
||||
scale: {
|
||||
x: parseFloat(scale.x.toFixed(2)),
|
||||
y: parseFloat(scale.y.toFixed(2)),
|
||||
z: parseFloat(scale.z.toFixed(2)),
|
||||
},
|
||||
const pos = this.selected.position;
|
||||
const rot = this.selected.rotation;
|
||||
const scale = this.selected.scale;
|
||||
|
||||
// TODO this is a bug we need to only show when clicked on and use UPDATE when updating.
|
||||
this.dispatchAppUiState({
|
||||
type: APPUI_ACTION_TYPES.UPDATE_CONTROLPANELS_SCENEOBJECT,
|
||||
payload: {
|
||||
group:
|
||||
this.selected.name === "::CAM::"
|
||||
? ClipGroup.CAMERA
|
||||
: ClipGroup.OBJECT, // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips
|
||||
object_uuid: this.selected.uuid,
|
||||
object_name: this.selected.name,
|
||||
version: this.version,
|
||||
objectVectors: {
|
||||
position: {
|
||||
x: parseFloat(pos.x.toFixed(2)),
|
||||
y: parseFloat(pos.y.toFixed(2)),
|
||||
z: parseFloat(pos.z.toFixed(2)),
|
||||
},
|
||||
rotation: {
|
||||
x: parseFloat(rot.x.toFixed(2)),
|
||||
y: parseFloat(rot.y.toFixed(2)),
|
||||
z: parseFloat(rot.z.toFixed(2)),
|
||||
},
|
||||
scale: {
|
||||
x: parseFloat(scale.x.toFixed(2)),
|
||||
y: parseFloat(scale.y.toFixed(2)),
|
||||
z: parseFloat(scale.z.toFixed(2)),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
},
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
// Automaticly resize scene.
|
||||
|
||||
@@ -347,7 +347,7 @@ export class TimeLine {
|
||||
}
|
||||
|
||||
// called by the editor update loop on each frame
|
||||
public async update() {
|
||||
public async update(isRendering=false) {
|
||||
//if (this.is_playing == false) return; // start and stop
|
||||
this.timeline_limit = this.getEndPoint();
|
||||
if (this.is_playing) {
|
||||
@@ -399,8 +399,10 @@ export class TimeLine {
|
||||
} else if (element.type === ClipType.ANIMATION) {
|
||||
if (object) {
|
||||
await this.animation_engine.clips[object.uuid+element.media_id].play(object);
|
||||
let fps = 120;
|
||||
if(isRendering) { fps = 60; }
|
||||
this.animation_engine.clips[object.uuid+element.media_id].step(
|
||||
this.scrubber_frame_position / 120, // Double FPS for best result.
|
||||
this.scrubber_frame_position / fps, // Double FPS for best result.
|
||||
);
|
||||
}
|
||||
} else {
|
||||
|
||||
Reference in new issue
Block a user